Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e16efc86ad | |||
| 010e253d66 | |||
| 245f258f20 | |||
| c62d57d5ac | |||
| 34bb7263fa |
@@ -20,9 +20,3 @@ omit =
|
||||
bot_bottle/cli/tui.py
|
||||
bot_bottle/cli/init.py
|
||||
tests/*
|
||||
# Build-time only: setuptools invokes it out-of-process to build the
|
||||
# wheel/sdist (it's never imported by the running app), so in-process
|
||||
# coverage can't reach it. Its one job — bundling the root resources into
|
||||
# bot_bottle/_resources/ — is exercised end-to-end by test_wheel_install,
|
||||
# which builds and installs a real wheel and checks the result.
|
||||
setup.py
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
name: prd-number-check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, reopened, synchronize]
|
||||
|
||||
jobs:
|
||||
require-numbered-prds:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Reject unnumbered PRDs
|
||||
run: |
|
||||
unnumbered=$(find docs/prds -maxdepth 1 -type f \
|
||||
-name 'prd-new-*.md' -print | sort)
|
||||
|
||||
if [ -n "$unnumbered" ]; then
|
||||
echo "::error::Assign every new PRD its final sequential number before merge."
|
||||
echo "Unnumbered PRDs:"
|
||||
echo "$unnumbered"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "All PRDs have final numbers."
|
||||
@@ -0,0 +1,122 @@
|
||||
# Assign sequential numbers to prd-new-*.md files on merge to main.
|
||||
#
|
||||
# When a PR merges to main and includes prd-new-*.md files this workflow:
|
||||
# 1. Finds the next available NNNN number by scanning existing PRDs.
|
||||
# 2. Renames each prd-new-*.md to NNNN-<slug>.md.
|
||||
# 3. Updates the title header (# PRD prd-new: → # PRD NNNN:).
|
||||
# 4. Flips Status: Draft → Active when the push touched files outside
|
||||
# docs/prds/ anywhere in its commit range (i.e. the implementation
|
||||
# shipped together with the PRD).
|
||||
# 5. Commits the renaming back to main.
|
||||
#
|
||||
# No-op if the working tree contains no prd-new-*.md files.
|
||||
#
|
||||
# NOTE: The workflow scans the working tree (not just HEAD~1..HEAD) because
|
||||
# PRs land as multi-commit pushes and the prd-new file is often added in an
|
||||
# earlier commit on the branch, not in the final squash/merge commit.
|
||||
|
||||
name: prd-number
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'docs/prds/prd-new-*.md'
|
||||
|
||||
jobs:
|
||||
assign-numbers:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# No actions/setup-python: the inline script is stdlib-only on the
|
||||
# image's system Python 3.12 (older act_runner mishandles its PATH).
|
||||
- name: Configure git
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Assign PRD numbers
|
||||
run: |
|
||||
python3 - <<'EOF'
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
prds_dir = Path("docs/prds")
|
||||
|
||||
# Scan the working tree — prd-new files may have landed in any
|
||||
# commit of a multi-commit push, not just HEAD.
|
||||
new_prds = sorted(prds_dir.glob("prd-new-*.md"))
|
||||
|
||||
if not new_prds:
|
||||
print("No prd-new-*.md files found — nothing to do.")
|
||||
sys.exit(0)
|
||||
|
||||
# Determine whether non-PRD files were also changed anywhere in
|
||||
# the push range (BEFORE_SHA → HEAD). Falls back to HEAD~1 when
|
||||
# the env var isn't set (e.g. local act runs).
|
||||
before_sha = os.environ.get("GITHUB_EVENT_BEFORE", "HEAD~1")
|
||||
all_changed = subprocess.run(
|
||||
["git", "diff", "--name-only", before_sha, "HEAD"],
|
||||
capture_output=True, text=True, check=True,
|
||||
).stdout.splitlines()
|
||||
non_prd_changed = any(
|
||||
not f.startswith("docs/prds/") for f in all_changed
|
||||
)
|
||||
|
||||
# Find next available number.
|
||||
existing = sorted(
|
||||
int(m.group(1))
|
||||
for p in prds_dir.glob("*.md")
|
||||
if (m := re.match(r"^(\d{4})-", p.name))
|
||||
)
|
||||
next_num = (max(existing) + 1) if existing else 1
|
||||
|
||||
for prd_path in sorted(new_prds):
|
||||
slug = re.sub(r"^prd-new-", "", prd_path.stem)
|
||||
new_name = f"{next_num:04d}-{slug}.md"
|
||||
new_path = prds_dir / new_name
|
||||
print(f" {prd_path.name} → {new_name}")
|
||||
|
||||
content = prd_path.read_text()
|
||||
|
||||
# Update title header.
|
||||
content = re.sub(
|
||||
r"^(#\s+PRD\s+)prd-new(:)",
|
||||
rf"\g<1>{next_num:04d}\2",
|
||||
content,
|
||||
count=1,
|
||||
flags=re.MULTILINE,
|
||||
)
|
||||
|
||||
# Conditionally flip Status.
|
||||
if non_prd_changed:
|
||||
content = re.sub(
|
||||
r"(\*\*Status:\*\*\s*)Draft",
|
||||
r"\g<1>Active",
|
||||
content,
|
||||
count=1,
|
||||
)
|
||||
|
||||
new_path.write_text(content)
|
||||
subprocess.run(["git", "rm", str(prd_path)], check=True)
|
||||
subprocess.run(["git", "add", str(new_path)], check=True)
|
||||
next_num += 1
|
||||
|
||||
subprocess.run(
|
||||
["git", "commit", "-m", "ci(prd): assign sequential numbers to new PRDs"],
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(["git", "push"], check=True)
|
||||
EOF
|
||||
@@ -1,337 +0,0 @@
|
||||
# Run the complete backend test suite before a release. This workflow is
|
||||
# intentionally manual because Firecracker and macOS use privileged,
|
||||
# self-hosted runners.
|
||||
#
|
||||
# The suite uses stdlib `unittest` discovery — no external Python
|
||||
# dependencies are required to execute it. Tests are split by directory:
|
||||
#
|
||||
# tests/unit/ — pure unit tests; always run
|
||||
# tests/integration/ — need a reachable backend; skip cleanly when
|
||||
# the backend isn't available on the runner
|
||||
# tests/canaries/ — upstream regression canaries; run on a separate
|
||||
# schedule (see canaries.yml), not here
|
||||
#
|
||||
# Unit, Docker, and Firecracker run once under coverage and upload a small
|
||||
# .coverage.* artifact for the combined coverage job. macOS reports coverage
|
||||
# in place because it is an advisory host-mode runner.
|
||||
|
||||
name: pre-release-test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
unit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# No actions/setup-python: the runner image already ships Python 3.12,
|
||||
# and older act_runner engines mishandle setup-python's PATH (coverage
|
||||
# lands in one interpreter, `python3` resolves to another). Install
|
||||
# straight into the ephemeral job container's system Python —
|
||||
# --break-system-packages is safe because the container is disposable.
|
||||
- name: Install dev requirements
|
||||
run: python3 -m pip install --break-system-packages -r requirements-dev.txt
|
||||
|
||||
- name: Run unit tests with coverage
|
||||
env:
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.unit
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/unit -v
|
||||
|
||||
- name: Report unit coverage
|
||||
env:
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.unit
|
||||
run: python3 -m coverage report -m
|
||||
|
||||
# upload-artifact@v3's glob skips dotfiles, so a bare `.coverage.unit`
|
||||
# silently uploads nothing ("No files were found"). Stage it under a
|
||||
# non-dot name; the coverage job renames it back before `coverage
|
||||
# combine`. `cp` also fails loudly if coverage never wrote the file.
|
||||
- name: Stage unit coverage for upload
|
||||
run: cp .coverage.unit coverage-unit.dat
|
||||
|
||||
- name: Upload unit coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-unit
|
||||
path: coverage-unit.dat
|
||||
|
||||
integration-docker:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# No actions/setup-python (see the note in the `unit` job); the
|
||||
# container's system Python 3.12 runs the stdlib test suite directly.
|
||||
- name: Install coverage
|
||||
run: python3 -m pip install --break-system-packages coverage
|
||||
|
||||
# Fail loudly if the backend this job promises isn't actually usable,
|
||||
# rather than letting every test silently `unittest.skip` and the job
|
||||
# go green on zero coverage. `backend status` prints a clear per-check
|
||||
# summary (docker on PATH, daemon reachable) and exits non-zero when a
|
||||
# prerequisite is missing — the same readiness check the skip guards
|
||||
# gate on via `has_backend`.
|
||||
- name: Preflight — Docker backend is ready
|
||||
run: |
|
||||
python3 --version
|
||||
python3 cli.py backend status --backend=docker
|
||||
|
||||
- name: Run integration tests (docker) with coverage
|
||||
env:
|
||||
BOT_BOTTLE_BACKEND: docker
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.docker
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
|
||||
- name: Stage docker coverage for upload
|
||||
run: cp .coverage.docker coverage-docker.dat
|
||||
|
||||
- name: Upload docker coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-docker
|
||||
path: coverage-docker.dat
|
||||
|
||||
# Integration tests against the Firecracker backend. Runs on a self-hosted
|
||||
# KVM runner (label `kvm`) where /dev/kvm and the TAP/nft pool are available.
|
||||
#
|
||||
# Manual only: the privileged KVM runner does not execute proposed changes
|
||||
# unattended.
|
||||
#
|
||||
# Runner prerequisites (provision once; see README "Firecracker on Linux"):
|
||||
# `firecracker` on PATH, `/dev/kvm` accessible, cached kernel +
|
||||
# static dropbear at /var/cache/bot-bottle-fc/dropbear, and the pool as a
|
||||
# persistent systemd unit.
|
||||
#
|
||||
# The infra candidate is built here directly (no artifact download) to
|
||||
# eliminate the ~70 s ubuntu-latest upload + ~83 s combined download that
|
||||
# the old build-infra → integration-firecracker + coverage chain incurred.
|
||||
integration-firecracker:
|
||||
runs-on: [self-hosted, kvm]
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Preflight — Firecracker host is ready
|
||||
run: |
|
||||
command -v firecracker >/dev/null || {
|
||||
echo "firecracker not on PATH — provision the runner (README: Firecracker on Linux)"; exit 1; }
|
||||
test -e /dev/kvm || { echo "/dev/kvm missing — KVM not available on this runner"; exit 1; }
|
||||
# `backend status` exits non-zero unless the TAP pool is up + no
|
||||
# range overlap; it prints the exact `backend setup` fix.
|
||||
python3 cli.py backend status --backend=firecracker
|
||||
|
||||
- name: Build infra candidate from this checkout
|
||||
env:
|
||||
BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear
|
||||
run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate --reuse-published
|
||||
|
||||
- name: Replace the persistent infra VM with the candidate
|
||||
run: python3 -c 'from bot_bottle.backend.firecracker import infra_vm; infra_vm.stop()'
|
||||
|
||||
# No dev-requirements install: `coverage` is already provided by the
|
||||
# self-hosted runner's Nix python env, and that env has no `pip`
|
||||
# module to install into anyway.
|
||||
- name: Run integration tests (firecracker) with coverage
|
||||
env:
|
||||
BOT_BOTTLE_BACKEND: firecracker
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.firecracker
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
- name: Stage firecracker coverage for upload
|
||||
run: cp .coverage.firecracker coverage-firecracker.dat
|
||||
|
||||
- name: Upload firecracker coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-firecracker
|
||||
path: coverage-firecracker.dat
|
||||
|
||||
- name: Upload tested rootfs
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate/
|
||||
|
||||
- name: Upload dropbear for publish verification
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: /var/cache/bot-bottle-fc/dropbear
|
||||
|
||||
# Integration tests against the macOS Apple Container backend. Runs on a
|
||||
# self-hosted macOS runner (label `macos`) registered in HOST mode — Apple
|
||||
# Container needs the host `container` CLI + virtualization framework and
|
||||
# cannot run inside a Linux container, so this cannot reuse the KVM runner.
|
||||
#
|
||||
# Advisory only: workflow_dispatch (manual) exclusively — never push or
|
||||
# pull_request. A single non-redundant laptop that sleeps/roams must not run
|
||||
# unattended on every push to main, let alone block a PR merge, so this job is
|
||||
# deliberately NOT in the `coverage` job's `needs` and its coverage never
|
||||
# feeds the diff-coverage gate. Dispatch-only also means no fork PR (or any
|
||||
# push) ever executes on the host-mode runner.
|
||||
#
|
||||
# The infra container is a singleton (`bot-bottle-mac-infra`); the
|
||||
# `concurrency` group serializes runs so two never collide on it (#425), and
|
||||
# the always-run teardown removes it so a crashed run can't wedge the next.
|
||||
#
|
||||
# Runner prerequisites (provision once; see README "macOS Apple Container"):
|
||||
# the `container` CLI on PATH with `container system status` running, and a
|
||||
# Python >=3.11 with `coverage` importable on the launchd service PATH.
|
||||
integration-macos:
|
||||
runs-on: [self-hosted, macos]
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
concurrency:
|
||||
group: integration-macos-infra
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Fail loudly if the backend this job promises isn't actually usable,
|
||||
# rather than letting every test silently `unittest.skip` and the job go
|
||||
# green on zero coverage. `backend status` exits non-zero (and prints the
|
||||
# per-check summary) when the `container` CLI or its system service is
|
||||
# missing — the same readiness check the skip guards gate on.
|
||||
- name: Preflight — Apple Container backend is ready
|
||||
run: |
|
||||
command -v container >/dev/null || {
|
||||
echo "container CLI not on PATH — provision the runner (README: macOS Apple Container)"; exit 1; }
|
||||
container system status || {
|
||||
echo "container system service not running — run 'container system start'"; exit 1; }
|
||||
python3 cli.py backend status --backend=macos-container
|
||||
|
||||
# `coverage` comes from the runner's provisioned Python (no pip install
|
||||
# into the host interpreter). Advisory job: report coverage in-line for
|
||||
# visibility but don't upload — it never feeds the combined gate.
|
||||
- name: Run integration tests (macos-container) with coverage
|
||||
env:
|
||||
BOT_BOTTLE_BACKEND: macos-container
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.macos
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
- name: Report macos coverage
|
||||
env:
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.macos
|
||||
run: python3 -m coverage report -m
|
||||
|
||||
# On failure, capture the infra containers' state and logs BEFORE the
|
||||
# teardown below removes them — otherwise a control-plane crash is
|
||||
# undiagnosable from CI, since `stop()` deletes the orchestrator (and its
|
||||
# logs) on every run. Best-effort: never let the diagnostics themselves
|
||||
# fail the job, and keep going if a container is already gone.
|
||||
- name: Dump infra diagnostics (on failure)
|
||||
if: failure()
|
||||
run: |
|
||||
set +e
|
||||
echo "=== containers ==="
|
||||
container ls -a | grep bot-bottle-mac || echo "(no bot-bottle-mac containers)"
|
||||
echo "=== networks ==="
|
||||
container network ls | grep bot-bottle-mac || echo "(no bot-bottle-mac networks)"
|
||||
for c in bot-bottle-mac-orchestrator bot-bottle-mac-infra; do
|
||||
echo "=== inspect $c ==="
|
||||
container inspect "$c" || echo "($c not found)"
|
||||
echo "=== logs $c ==="
|
||||
container logs "$c" || echo "($c logs unavailable)"
|
||||
done
|
||||
exit 0
|
||||
|
||||
# Remove the singleton infra container so a crashed or cancelled run
|
||||
# cannot leave `bot-bottle-mac-infra` wedged for the next job.
|
||||
- name: Teardown infra singleton
|
||||
if: always()
|
||||
run: python3 -c 'from bot_bottle.backend.macos_container.infra import MacosInfraService; MacosInfraService().stop()'
|
||||
|
||||
# Combined coverage gate: aggregates .coverage.* artifacts uploaded by each
|
||||
# test job, then runs the diff-coverage gate (new/changed lines >= 90%).
|
||||
#
|
||||
# Runs on ubuntu-latest — no KVM needed, no test reruns. Coverage files use
|
||||
# relative_files = True (.coveragerc) so they combine cleanly across runners.
|
||||
# Each test job sets COVERAGE_FILE to an absolute path so coverage.py writes
|
||||
# to a known location that upload-artifact can find regardless of runner env.
|
||||
#
|
||||
coverage:
|
||||
needs: [unit, integration-docker, integration-firecracker]
|
||||
timeout-minutes: 15
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install coverage
|
||||
run: python3 -m pip install --break-system-packages coverage
|
||||
|
||||
- name: Download unit coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: coverage-unit
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
- name: Download docker coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: coverage-docker
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
- name: Download firecracker coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: coverage-firecracker
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
# Rename the non-dot upload names back to the .coverage.* files that
|
||||
# `coverage combine` discovers (see the staging steps in each test job).
|
||||
- name: Reassemble coverage data files
|
||||
run: |
|
||||
mv coverage-unit.dat .coverage.unit
|
||||
mv coverage-docker.dat .coverage.docker
|
||||
mv coverage-firecracker.dat .coverage.firecracker
|
||||
|
||||
- name: Combined coverage (unit + integration, incl. firecracker)
|
||||
run: PYTHON=python3 bash scripts/coverage.sh aggregate critical
|
||||
|
||||
- name: Diff-coverage gate (changed lines >= 90%)
|
||||
run: |
|
||||
git fetch --no-tags origin main:refs/remotes/origin/main
|
||||
python3 scripts/diff_coverage.py --base origin/main --min 90
|
||||
|
||||
publish-infra:
|
||||
needs:
|
||||
- unit
|
||||
- integration-docker
|
||||
- integration-firecracker
|
||||
- integration-macos
|
||||
- coverage
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout the tested revision
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download the tested rootfs
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate
|
||||
|
||||
# publish_infra re-derives the version from the checkout to confirm the
|
||||
# bundle matches before uploading, and the version hashes the dropbear
|
||||
# bytes. Download the same dropbear integration-firecracker used.
|
||||
- name: Download the staged dropbear
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: firecracker-inputs
|
||||
|
||||
- name: Publish the tested candidate
|
||||
env:
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
|
||||
BOT_BOTTLE_FC_DROPBEAR: ${{ github.workspace }}/firecracker-inputs/dropbear
|
||||
run: python3 -m bot_bottle.backend.firecracker.publish_infra --publish-dir infra-candidate
|
||||
+176
-7
@@ -1,6 +1,21 @@
|
||||
# Run the automated test gate when package or runtime inputs change on a PR
|
||||
# or on push to main. Privileged self-hosted backends live in the manually
|
||||
# dispatched pre-release-test workflow.
|
||||
# Run the project's test suite when package or runtime inputs change on a PR
|
||||
# or on push to main.
|
||||
#
|
||||
# The suite uses stdlib `unittest` discovery — no external Python
|
||||
# dependencies are required to execute it. Tests are split by directory:
|
||||
#
|
||||
# tests/unit/ — pure unit tests; always run
|
||||
# tests/integration/ — need a reachable backend; skip cleanly when
|
||||
# the backend isn't available on the runner
|
||||
# tests/canaries/ — upstream regression canaries; run on a separate
|
||||
# schedule (see canaries.yml), not here
|
||||
#
|
||||
# Each test job runs once under coverage and uploads a small .coverage.*
|
||||
# artifact. The `coverage` job combines them — no test reruns, no KVM
|
||||
# dependency on that job. For main-branch pushes only, the tested rootfs
|
||||
# and matching dropbear are uploaded so `publish-infra` can publish the
|
||||
# byte-identical artifact that was tested. PRs avoid the ~194 MB rootfs
|
||||
# transfer entirely.
|
||||
|
||||
name: test
|
||||
|
||||
@@ -22,7 +37,6 @@ on:
|
||||
- 'requirements-dev.txt'
|
||||
- '.coveragerc'
|
||||
- '.dockerignore'
|
||||
- '.gitea/workflows/test.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'bot_bottle/**'
|
||||
@@ -38,7 +52,7 @@ on:
|
||||
- 'requirements-dev.txt'
|
||||
- '.coveragerc'
|
||||
- '.dockerignore'
|
||||
- '.gitea/workflows/test.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
unit:
|
||||
@@ -47,6 +61,11 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# No actions/setup-python: the runner image already ships Python 3.12,
|
||||
# and older act_runner engines mishandle setup-python's PATH (coverage
|
||||
# lands in one interpreter, `python3` resolves to another). Install
|
||||
# straight into the ephemeral job container's system Python —
|
||||
# --break-system-packages is safe because the container is disposable.
|
||||
- name: Install dev requirements
|
||||
run: python3 -m pip install --break-system-packages -r requirements-dev.txt
|
||||
|
||||
@@ -60,6 +79,10 @@ jobs:
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.unit
|
||||
run: python3 -m coverage report -m
|
||||
|
||||
# upload-artifact@v3's glob skips dotfiles, so a bare `.coverage.unit`
|
||||
# silently uploads nothing ("No files were found"). Stage it under a
|
||||
# non-dot name; the coverage job renames it back before `coverage
|
||||
# combine`. `cp` also fails loudly if coverage never wrote the file.
|
||||
- name: Stage unit coverage for upload
|
||||
run: cp .coverage.unit coverage-unit.dat
|
||||
|
||||
@@ -75,9 +98,17 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# No actions/setup-python (see the note in the `unit` job); the
|
||||
# container's system Python 3.12 runs the stdlib test suite directly.
|
||||
- name: Install coverage
|
||||
run: python3 -m pip install --break-system-packages coverage
|
||||
|
||||
# Fail loudly if the backend this job promises isn't actually usable,
|
||||
# rather than letting every test silently `unittest.skip` and the job
|
||||
# go green on zero coverage. `backend status` prints a clear per-check
|
||||
# summary (docker on PATH, daemon reachable) and exits non-zero when a
|
||||
# prerequisite is missing — the same readiness check the skip guards
|
||||
# gate on via `has_backend`.
|
||||
- name: Preflight — Docker backend is ready
|
||||
run: |
|
||||
python3 --version
|
||||
@@ -89,6 +120,7 @@ jobs:
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.docker
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
|
||||
- name: Stage docker coverage for upload
|
||||
run: cp .coverage.docker coverage-docker.dat
|
||||
|
||||
@@ -98,10 +130,107 @@ jobs:
|
||||
name: coverage-docker
|
||||
path: coverage-docker.dat
|
||||
|
||||
# Integration tests against the Firecracker backend. Runs on a self-hosted
|
||||
# KVM runner (label `kvm`) where /dev/kvm and the TAP/nft pool are available.
|
||||
#
|
||||
# Restricted to same-repo PRs, push to main, and workflow_dispatch — fork
|
||||
# PRs don't execute untrusted code on the privileged runner.
|
||||
#
|
||||
# Runner prerequisites (provision once; see README "Firecracker on Linux"):
|
||||
# `firecracker` on PATH, `/dev/kvm` accessible, cached kernel +
|
||||
# static dropbear at /var/cache/bot-bottle-fc/dropbear, and the pool as a
|
||||
# persistent systemd unit.
|
||||
#
|
||||
# The infra candidate is built here directly (no artifact download) to
|
||||
# eliminate the ~70 s ubuntu-latest upload + ~83 s combined download that
|
||||
# the old build-infra → integration-firecracker + coverage chain incurred.
|
||||
# For main-branch pushes the tested rootfs and matching dropbear are
|
||||
# uploaded so publish-infra can publish the byte-identical artifact; PRs
|
||||
# skip those uploads entirely.
|
||||
integration-firecracker:
|
||||
runs-on: [self-hosted, kvm]
|
||||
if: >-
|
||||
github.event_name == 'push' ||
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'pull_request' &&
|
||||
github.event.pull_request.head.repo.full_name == github.repository)
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Preflight — Firecracker host is ready
|
||||
run: |
|
||||
command -v firecracker >/dev/null || {
|
||||
echo "firecracker not on PATH — provision the runner (README: Firecracker on Linux)"; exit 1; }
|
||||
test -e /dev/kvm || { echo "/dev/kvm missing — KVM not available on this runner"; exit 1; }
|
||||
# `backend status` exits non-zero unless the TAP pool is up + no
|
||||
# range overlap; it prints the exact `backend setup` fix.
|
||||
python3 cli.py backend status --backend=firecracker
|
||||
|
||||
- name: Build infra candidate from this checkout
|
||||
env:
|
||||
BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear
|
||||
run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate --reuse-published
|
||||
|
||||
- name: Replace the persistent infra VM with the candidate
|
||||
run: python3 -c 'from bot_bottle.backend.firecracker import infra_vm; infra_vm.stop()'
|
||||
|
||||
# No dev-requirements install: `coverage` is already provided by the
|
||||
# self-hosted runner's Nix python env, and that env has no `pip`
|
||||
# module to install into anyway.
|
||||
- name: Run integration tests (firecracker) with coverage
|
||||
env:
|
||||
BOT_BOTTLE_BACKEND: firecracker
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.firecracker
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
|
||||
- name: Stage firecracker coverage for upload
|
||||
run: cp .coverage.firecracker coverage-firecracker.dat
|
||||
|
||||
- name: Upload firecracker coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-firecracker
|
||||
path: coverage-firecracker.dat
|
||||
|
||||
# Only upload the large rootfs artifact on main-branch pushes;
|
||||
# PRs avoid the ~194 MB transfer. publish-infra only runs on main
|
||||
# and downloads these to publish the byte-identical tested rootfs.
|
||||
- name: Upload tested rootfs (main branch only)
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate/
|
||||
|
||||
- name: Upload dropbear for publish verification (main branch only)
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: /var/cache/bot-bottle-fc/dropbear
|
||||
|
||||
# Combined coverage gate: aggregates .coverage.* artifacts uploaded by each
|
||||
# test job, then runs the diff-coverage gate (new/changed lines >= 90%).
|
||||
#
|
||||
# Runs on ubuntu-latest — no KVM needed, no test reruns. Coverage files use
|
||||
# relative_files = True (.coveragerc) so they combine cleanly across runners.
|
||||
# Each test job sets COVERAGE_FILE to an absolute path so coverage.py writes
|
||||
# to a known location that upload-artifact can find regardless of runner env.
|
||||
#
|
||||
# Restricted to the same events as integration-firecracker: it depends on
|
||||
# that job's coverage artifact and skips for fork PRs alongside it.
|
||||
coverage:
|
||||
needs: [unit, integration-docker]
|
||||
needs: [unit, integration-docker, integration-firecracker]
|
||||
timeout-minutes: 15
|
||||
runs-on: ubuntu-latest
|
||||
if: >-
|
||||
github.event_name == 'push' ||
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'pull_request' &&
|
||||
github.event.pull_request.head.repo.full_name == github.repository)
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
@@ -123,15 +252,55 @@ jobs:
|
||||
name: coverage-docker
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
- name: Download firecracker coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: coverage-firecracker
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
# Rename the non-dot upload names back to the .coverage.* files that
|
||||
# `coverage combine` discovers (see the staging steps in each test job).
|
||||
- name: Reassemble coverage data files
|
||||
run: |
|
||||
mv coverage-unit.dat .coverage.unit
|
||||
mv coverage-docker.dat .coverage.docker
|
||||
mv coverage-firecracker.dat .coverage.firecracker
|
||||
|
||||
- name: Combined coverage (unit + docker integration)
|
||||
- name: Combined coverage (unit + integration, incl. firecracker)
|
||||
run: PYTHON=python3 bash scripts/coverage.sh aggregate critical
|
||||
|
||||
- name: Diff-coverage gate (changed lines >= 90%)
|
||||
run: |
|
||||
git fetch --no-tags origin main:refs/remotes/origin/main
|
||||
python3 scripts/diff_coverage.py --base origin/main --min 90
|
||||
|
||||
publish-infra:
|
||||
needs: [unit, integration-docker, integration-firecracker, coverage]
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Checkout the tested revision
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download the tested rootfs
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate
|
||||
|
||||
# publish_infra re-derives the version from the checkout to confirm the
|
||||
# bundle matches before uploading, and the version hashes the dropbear
|
||||
# bytes. Download the SAME dropbear integration-firecracker used, or
|
||||
# the recheck computes a "<missing>"-dropbear version and rejects the
|
||||
# candidate.
|
||||
- name: Download the staged dropbear (matches build's version)
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: firecracker-inputs
|
||||
|
||||
- name: Publish the tested candidate
|
||||
env:
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
|
||||
BOT_BOTTLE_FC_DROPBEAR: ${{ github.workspace }}/firecracker-inputs/dropbear
|
||||
run: python3 -m bot_bottle.backend.firecracker.publish_infra --publish-dir infra-candidate
|
||||
|
||||
@@ -17,9 +17,6 @@ __pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.egg-info/
|
||||
# setuptools/build_meta output (wheels, sdists, build tree)
|
||||
/build/
|
||||
/dist/
|
||||
.venv/
|
||||
venv/
|
||||
.pytest_cache/
|
||||
|
||||
@@ -44,10 +44,10 @@ backend remains available with `BOT_BOTTLE_BACKEND=docker` or
|
||||
|
||||
- Three kinds of doc, each with its own conventions in-folder; see
|
||||
`docs/README.md` for when to write which:
|
||||
- **PRDs** (`docs/prds/`) — one feature per file. A draft may initially
|
||||
use `prd-new-<kebab>.md`, but its author must assign the next
|
||||
sequential number before merge; CI rejects unnumbered PRDs. A
|
||||
`Status:` line tracks lifecycle: Draft → Active (shipped to `main`) →
|
||||
- **PRDs** (`docs/prds/`) — one feature per file. While a PR is open
|
||||
the file is named `prd-new-<kebab>.md`; CI assigns a sequential
|
||||
number on merge to `main` and renames it. A `Status:` line tracks
|
||||
lifecycle: Draft → Active (shipped to `main`) →
|
||||
Superseded/Retargeted. Format in `docs/prds/README.md`.
|
||||
- **Research notes** (`docs/research/`) — opinionated investigations;
|
||||
unnumbered kebab-case, freeform and verdict-first. See
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
# Root-level build resources copied into bot_bottle/_resources/ at build time
|
||||
# (see setup.py). Included in the sdist so `pip install` from an sdist can
|
||||
# still bundle them into the wheel.
|
||||
include Dockerfile.gateway
|
||||
include Dockerfile.orchestrator
|
||||
include Dockerfile.orchestrator.fc
|
||||
include nix/firecracker-netpool.nix
|
||||
include scripts/firecracker-netpool.sh
|
||||
@@ -5,7 +5,7 @@
|
||||
# bot-bottle
|
||||
|
||||
[](https://gitea.dideric.is/didericis/bot-bottle/actions?workflow=test.yml)
|
||||
[](https://coverage.readthedocs.io/)
|
||||
[](https://coverage.readthedocs.io/)
|
||||
[](https://gitea.dideric.is/didericis/bot-bottle/src/branch/main/docs/decisions/0004-coverage-policy.md)
|
||||
|
||||
**Problem:** Developer wants to run a coding agent without supervision, but they don't want a prompt injected or misbehaving agent wrecking their environment or exfiltrating sensitive data.
|
||||
@@ -30,7 +30,7 @@
|
||||
|
||||
## Architecture
|
||||
|
||||
On the default macOS Apple Container backend, a bottle is an agent container on a host-only internal network plus a gateway attached to both that internal network and a NAT egress network. The agent gets HTTP(S)_PROXY and CA bundle env vars pointing at the gateway's internal-network IP, so HTTP/HTTPS traffic flows through the gateway instead of direct egress. git-gate runs over the gateway's consolidated `git-http` daemon (the legacy per-bottle `git://` daemon is not used on this backend); keys are provisioned dynamically at launch and revoked on teardown.
|
||||
On the default macOS Apple Container backend, a bottle is an agent container on a host-only internal network plus a gateway attached to both that internal network and a NAT egress network. The agent gets HTTP(S)_PROXY and CA bundle env vars pointing at the gateway's internal-network IP, so HTTP/HTTPS traffic flows through the gateway instead of direct egress. `bottle.git` / git-gate is intentionally deferred on this backend until a safe Apple Container key-delivery path exists.
|
||||
|
||||
On the Firecracker backend, a bottle is an agent microVM plus a Docker gateway for egress, git-gate, and supervise. The VM reaches the gateway over a per-bottle point-to-point TAP link; a dedicated fail-closed `nftables` table (`inet bot_bottle_fc`) confines the guest to that link, so nothing leaves the box except through the gateway. The TAP pool and nft table are provisioned once (root); per-launch needs no privilege.
|
||||
|
||||
@@ -75,8 +75,6 @@ On compatible macOS hosts, the default backend requires Apple's `container` CLI
|
||||
|
||||
Use `BOT_BOTTLE_BACKEND=docker ./cli.py start <agent>` on hosts where neither Apple Container nor KVM is available and Docker is the desired backend.
|
||||
|
||||
> **CI (macOS Apple Container):** the `integration-macos` job (`.gitea/workflows/test.yml`) runs the integration suite against `BOT_BOTTLE_BACKEND=macos-container` on a self-hosted macOS runner labelled `macos`, because Apple Container needs the host virtualization framework and cannot run in a Linux container (so it can't reuse the `kvm` runner). Provision an Apple Silicon host with the `container` CLI on `PATH` and `container system status` running, then register the runner in **host mode** (not docker mode) with the `macos` label — `brew install gitea-runner` (the `act_runner` rename). Give it a Python ≥ 3.11 with `coverage` importable on the launchd service's `PATH` (a launchd service doesn't inherit your shell profile, so pin `node` and the Python env explicitly). The job is **advisory** — `workflow_dispatch` (manual) only, never triggered by push or PR — since a single laptop that sleeps/roams must not block merges or churn on every push to main; its coverage doesn't feed the gate. The infra container is a singleton (`bot-bottle-mac-infra`), so keep runner concurrency at 1.
|
||||
|
||||
### Containers inside a bottle
|
||||
|
||||
A bottle may set `nested_containers: true`. On the macOS backend this starts a
|
||||
|
||||
@@ -10,10 +10,9 @@ from ...paths import (
|
||||
ORCHESTRATOR_AUTH_JWT_ENV,
|
||||
host_gateway_ca_dir,
|
||||
)
|
||||
from ... import resources
|
||||
from ...gateway import (
|
||||
Gateway, GatewayTransport, GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK,
|
||||
GATEWAY_DOCKERFILE, GATEWAY_LABEL, MITMPROXY_HOME,
|
||||
GATEWAY_DOCKERFILE, REPO_ROOT, GATEWAY_LABEL, MITMPROXY_HOME,
|
||||
DEFAULT_CA_TIMEOUT_SECONDS, CA_POLL_SECONDS, GATEWAY_CA_CERT, GatewayError
|
||||
)
|
||||
|
||||
@@ -51,9 +50,7 @@ class DockerGateway(Gateway):
|
||||
# `address` / `stop` work on an already-running gateway without it.
|
||||
self._orchestrator_url = ""
|
||||
self._gateway_token = ""
|
||||
# Resolved lazily in ensure_built() so merely constructing a gateway to
|
||||
# read its CA never stages a build root from an installed wheel.
|
||||
self._build_context = build_context
|
||||
self._build_context = build_context or REPO_ROOT
|
||||
self._dockerfile = dockerfile
|
||||
# Ports published on the host (0.0.0.0). Used by the Firecracker
|
||||
# backend's dev-harness gateway so VMs can reach it via their TAP link;
|
||||
@@ -75,10 +72,9 @@ class DockerGateway(Gateway):
|
||||
forces a full rebuild (parity with `start --no-cache`)."""
|
||||
if self._dockerfile is None:
|
||||
return
|
||||
context = self._build_context or resources.build_root()
|
||||
argv = ["docker", "build", "-t", self.image_ref,
|
||||
"-f", str(context / self._dockerfile),
|
||||
str(context)]
|
||||
"-f", str(self._build_context / self._dockerfile),
|
||||
str(self._build_context)]
|
||||
if os.environ.get("BOT_BOTTLE_NO_CACHE"):
|
||||
argv.insert(2, "--no-cache")
|
||||
proc = run_docker(argv)
|
||||
|
||||
@@ -34,7 +34,6 @@ from .orchestrator import (
|
||||
ORCHESTRATOR_NETWORK,
|
||||
)
|
||||
from ...paths import bot_bottle_root
|
||||
from ... import resources
|
||||
from ...gateway import (
|
||||
GATEWAY_IMAGE,
|
||||
GATEWAY_NAME,
|
||||
@@ -51,6 +50,8 @@ from ...orchestrator.lifecycle import (
|
||||
# the pair's public identity.
|
||||
INFRA_NAME = GATEWAY_NAME # the container agents attribute against is the gateway
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
class DockerInfraService(InfraService):
|
||||
"""Composes the per-host control plane + gateway as two containers.
|
||||
@@ -67,7 +68,7 @@ class DockerInfraService(InfraService):
|
||||
control_network: str = ORCHESTRATOR_NETWORK,
|
||||
orchestrator_image: str = ORCHESTRATOR_IMAGE,
|
||||
gateway_image: str = GATEWAY_IMAGE,
|
||||
repo_root: Path | None = None,
|
||||
repo_root: Path = _REPO_ROOT,
|
||||
host_root: Path | None = None,
|
||||
orchestrator_name: str = ORCHESTRATOR_NAME,
|
||||
orchestrator_label: str = ORCHESTRATOR_LABEL,
|
||||
@@ -78,9 +79,7 @@ class DockerInfraService(InfraService):
|
||||
self.control_network = control_network
|
||||
self.orchestrator_image = orchestrator_image
|
||||
self.gateway_image = gateway_image
|
||||
# Build context / bind-mount source: the repo root in a checkout, a
|
||||
# staged copy from the installed wheel otherwise (bot_bottle.resources).
|
||||
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||
self._repo_root = repo_root
|
||||
self._host_root = host_root or bot_bottle_root()
|
||||
self._orchestrator_name = orchestrator_name
|
||||
self._orchestrator_label = orchestrator_label
|
||||
|
||||
@@ -33,6 +33,7 @@ from __future__ import annotations
|
||||
import dataclasses
|
||||
import os
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Callable, Generator
|
||||
|
||||
from ...agent_provider import runtime_for
|
||||
@@ -64,7 +65,10 @@ from ...orchestrator.store.config_store import resolve_teardown_timeout
|
||||
from .consolidated_launch import launch_consolidated, deprovision_consolidated
|
||||
from .infra import INFRA_NAME
|
||||
from .gateway import DockerGateway
|
||||
from ... import resources
|
||||
|
||||
|
||||
# Where the repo root lives, for `docker build` context. Computed once.
|
||||
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
|
||||
|
||||
|
||||
def build_or_load_images(plan: DockerBottlePlan) -> BottleImages:
|
||||
@@ -84,7 +88,7 @@ def build_or_load_images(plan: DockerBottlePlan) -> BottleImages:
|
||||
)
|
||||
info(f"using cached agent image {plan.image!r}")
|
||||
return BottleImages(agent=plan.image)
|
||||
docker_mod.build_image(plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path)
|
||||
docker_mod.build_image(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path)
|
||||
docker_mod.verify_agent_image(
|
||||
plan.image, runtime_for(plan.agent_provider_template).smoke_test,
|
||||
)
|
||||
|
||||
@@ -15,7 +15,6 @@ import time
|
||||
from pathlib import Path
|
||||
|
||||
from ... import log
|
||||
from ... import resources
|
||||
from .util import run_docker
|
||||
from ...paths import (
|
||||
ORCHESTRATOR_TOKEN_ENV,
|
||||
@@ -56,6 +55,8 @@ _ROOT_IN_CONTAINER = "/bot-bottle-root"
|
||||
|
||||
_HEALTH_POLL_SECONDS = 0.25
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
class DockerOrchestrator(Orchestrator):
|
||||
"""The control plane as a single fixed-name container. `ensure_built` builds
|
||||
@@ -70,7 +71,7 @@ class DockerOrchestrator(Orchestrator):
|
||||
label: str = ORCHESTRATOR_LABEL,
|
||||
port: int = DEFAULT_PORT,
|
||||
control_network: str = ORCHESTRATOR_NETWORK,
|
||||
repo_root: Path | None = None,
|
||||
repo_root: Path = _REPO_ROOT,
|
||||
host_root: Path | None = None,
|
||||
dockerfile: str | None = ORCHESTRATOR_DOCKERFILE,
|
||||
) -> None:
|
||||
@@ -79,9 +80,7 @@ class DockerOrchestrator(Orchestrator):
|
||||
self.label = label
|
||||
self.port = port
|
||||
self.control_network = control_network
|
||||
# Build context / bind-mount source: the repo root in a checkout, a
|
||||
# staged copy from the installed wheel otherwise (bot_bottle.resources).
|
||||
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||
self._repo_root = repo_root
|
||||
self._host_root = host_root or bot_bottle_root()
|
||||
self._dockerfile = dockerfile
|
||||
|
||||
|
||||
@@ -37,7 +37,6 @@ import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from ... import resources
|
||||
from ...log import die, info
|
||||
from . import util
|
||||
|
||||
@@ -45,6 +44,8 @@ from . import util
|
||||
# scheme can't collide with a cached/published artifact of the old one.
|
||||
_ARTIFACT_FORMAT = "1"
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
# The two per-plane infra VM roles. Each publishes/pulls its own rootfs artifact
|
||||
# from its own generic package; the Dockerfiles baked into each differ (only the
|
||||
# orchestrator rootfs carries buildah), so the versions are hashed separately.
|
||||
@@ -73,7 +74,7 @@ def local_build_requested() -> bool:
|
||||
|
||||
|
||||
def infra_artifact_version(
|
||||
init_script: str, role: str, *, repo_root: Path | None = None,
|
||||
init_script: str, role: str, *, repo_root: Path = _REPO_ROOT,
|
||||
) -> str:
|
||||
"""Content hash (16 hex) of everything baked into `role`'s infra rootfs: the
|
||||
whole shipped `bot_bottle` package, that role's Dockerfiles, and its guest
|
||||
@@ -88,8 +89,6 @@ def infra_artifact_version(
|
||||
version or a launch host could boot a stale rootfs whose code differs from
|
||||
its checkout. `__pycache__`/`.pyc` are the only exclusions — build artifacts,
|
||||
never copied."""
|
||||
if repo_root is None:
|
||||
repo_root = resources.build_root()
|
||||
h = hashlib.sha256()
|
||||
h.update(f"format={_ARTIFACT_FORMAT}\nrole={role}\n".encode())
|
||||
pkg = repo_root / "bot_bottle"
|
||||
|
||||
@@ -42,7 +42,6 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Generator
|
||||
|
||||
from ... import resources
|
||||
from ...log import die, info
|
||||
from ..docker import util as docker_mod
|
||||
from . import firecracker_vm, infra_artifact, netpool, util
|
||||
@@ -66,6 +65,7 @@ _GUEST_GATEWAY_JWT_PATH = "/var/lib/bot-bottle/gateway-jwt"
|
||||
_GATEWAY_IMAGE = "bot-bottle-gateway:latest"
|
||||
_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest"
|
||||
_ORCHESTRATOR_FC_IMAGE = "bot-bottle-orchestrator-fc:latest"
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
# Per-role rootfs source image + the extra free space `mke2fs` leaves for the
|
||||
# guest to grow into. The orchestrator keeps buildah's large build slack; the
|
||||
@@ -130,13 +130,12 @@ def build_infra_images_with_docker() -> None:
|
||||
orchestrator + buildah). The gateway VM boots the gateway image directly.
|
||||
The launch host uses this only in `BOT_BOTTLE_INFRA_BUILD=local` mode;
|
||||
`publish_infra` uses it off-host to produce the published artifacts."""
|
||||
root = str(resources.build_root())
|
||||
docker_mod.build_image(
|
||||
_ORCHESTRATOR_IMAGE, root, dockerfile="Dockerfile.orchestrator")
|
||||
_ORCHESTRATOR_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.orchestrator")
|
||||
docker_mod.build_image(
|
||||
_GATEWAY_IMAGE, root, dockerfile="Dockerfile.gateway")
|
||||
_GATEWAY_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.gateway")
|
||||
docker_mod.build_image(
|
||||
_ORCHESTRATOR_FC_IMAGE, root, dockerfile="Dockerfile.orchestrator.fc")
|
||||
_ORCHESTRATOR_FC_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.orchestrator.fc")
|
||||
|
||||
|
||||
def build_rootfs_dir(role: str) -> Path:
|
||||
|
||||
@@ -20,7 +20,6 @@ import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from ... import resources
|
||||
from . import netpool
|
||||
from . import util
|
||||
|
||||
@@ -43,13 +42,13 @@ def _has_systemd() -> bool:
|
||||
|
||||
|
||||
def _module_path() -> str:
|
||||
"""Absolute path to the importable NixOS module (checkout or wheel)."""
|
||||
return str(resources.nix_netpool_module())
|
||||
"""Absolute path to the importable NixOS module in this checkout."""
|
||||
return str(Path(__file__).resolve().parents[3] / "nix" / "firecracker-netpool.nix")
|
||||
|
||||
|
||||
def _script_path() -> str:
|
||||
"""Absolute path to the bundled bring-up script (checkout or wheel)."""
|
||||
return str(resources.netpool_script())
|
||||
"""Absolute path to the bundled bring-up script in this checkout."""
|
||||
return str(Path(__file__).resolve().parents[3] / "scripts" / "firecracker-netpool.sh")
|
||||
|
||||
|
||||
def _print_prereqs() -> None:
|
||||
|
||||
@@ -28,7 +28,6 @@ from ...paths import (
|
||||
ORCHESTRATOR_AUTH_JWT_ENV,
|
||||
host_gateway_ca_dir,
|
||||
)
|
||||
from ... import resources
|
||||
from .. import util as backend_util
|
||||
from . import util as container_mod
|
||||
|
||||
@@ -53,6 +52,8 @@ GATEWAY_DAEMONS = "egress,git-http,supervise"
|
||||
|
||||
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def ensure_networks(
|
||||
network: str = GATEWAY_NETWORK,
|
||||
@@ -83,16 +84,14 @@ class MacosGateway(Gateway):
|
||||
network: str = GATEWAY_NETWORK,
|
||||
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
||||
control_network: str = CONTROL_NETWORK,
|
||||
repo_root: Path | None = None,
|
||||
repo_root: Path = _REPO_ROOT,
|
||||
) -> None:
|
||||
self.image_ref = image_ref
|
||||
self.name = name
|
||||
self.network = network
|
||||
self.egress_network = egress_network
|
||||
self.control_network = control_network
|
||||
# Build context: the repo root in a checkout, a staged copy from the
|
||||
# installed wheel otherwise (bot_bottle.resources).
|
||||
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||
self._repo_root = repo_root
|
||||
# Set by `connect_to_orchestrator`: the URL the daemons resolve policy
|
||||
# against + the pre-minted `gateway` token they present. The gateway
|
||||
# never mints, so it never holds the signing key (#469).
|
||||
|
||||
@@ -24,7 +24,6 @@ from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from ... import resources
|
||||
from ...orchestrator.lifecycle import (
|
||||
DEFAULT_PORT,
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
@@ -54,6 +53,8 @@ from .orchestrator import (
|
||||
# still import it (probe / reprovision attribute against the gateway).
|
||||
INFRA_NAME = GATEWAY_NAME
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
class MacosInfraService(InfraService):
|
||||
"""Composes the per-host orchestrator + gateway containers. Callers use
|
||||
@@ -69,7 +70,7 @@ class MacosInfraService(InfraService):
|
||||
control_network: str = CONTROL_NETWORK,
|
||||
gateway_image: str = GATEWAY_IMAGE,
|
||||
orchestrator_image: str = ORCHESTRATOR_IMAGE,
|
||||
repo_root: Path | None = None,
|
||||
repo_root: Path = _REPO_ROOT,
|
||||
orchestrator_name: str = ORCHESTRATOR_NAME,
|
||||
gateway_name: str = INFRA_NAME,
|
||||
db_volume: str = ORCHESTRATOR_DB_VOLUME,
|
||||
@@ -80,9 +81,7 @@ class MacosInfraService(InfraService):
|
||||
self.control_network = control_network
|
||||
self.gateway_image = gateway_image
|
||||
self.orchestrator_image = orchestrator_image
|
||||
# Build context / bind-mount source: the repo root in a checkout, a
|
||||
# staged copy from the installed wheel otherwise (bot_bottle.resources).
|
||||
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||
self._repo_root = repo_root
|
||||
self._orchestrator_name = orchestrator_name
|
||||
self._gateway_name = gateway_name
|
||||
self._db_volume = db_volume
|
||||
|
||||
@@ -36,6 +36,7 @@ import dataclasses
|
||||
import os
|
||||
import subprocess
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Callable, Generator
|
||||
|
||||
from ...bottle_state import (
|
||||
@@ -48,7 +49,6 @@ from ...git_gate import GitGate
|
||||
from ...gateway.git_gate.http_backend import DEFAULT_PORT as _GIT_HTTP_PORT
|
||||
from ...image_cache import check_stale
|
||||
from ...log import die, info, warn
|
||||
from ... import resources
|
||||
from .. import BottleImages
|
||||
from ...supervisor.types import SUPERVISE_PORT
|
||||
from ..docker.egress import EGRESS_PORT
|
||||
@@ -71,6 +71,7 @@ from .consolidated_launch import (
|
||||
deprovision_consolidated,
|
||||
)
|
||||
|
||||
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
|
||||
_AGENT_SLEEP_SECONDS = "2147483647"
|
||||
|
||||
|
||||
@@ -93,7 +94,7 @@ def _agent_image(plan: MacosContainerBottlePlan) -> str:
|
||||
)
|
||||
info(f"using cached agent image {plan.image!r}")
|
||||
return plan.image
|
||||
container_mod.build_image(plan.image, str(resources.build_root()), dockerfile=plan.dockerfile_path)
|
||||
container_mod.build_image(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path)
|
||||
return plan.image
|
||||
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from ... import log
|
||||
from ... import resources
|
||||
from ...paths import ORCHESTRATOR_TOKEN_ENV
|
||||
from ...orchestrator.lifecycle import (
|
||||
DEFAULT_HEALTH_TIMEOUT_SECONDS,
|
||||
@@ -46,6 +45,7 @@ _DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle"
|
||||
_SRC_IN_CONTAINER = "/bot-bottle-src"
|
||||
|
||||
_HEALTH_POLL_SECONDS = 0.25
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
class MacosOrchestrator(Orchestrator):
|
||||
@@ -61,7 +61,7 @@ class MacosOrchestrator(Orchestrator):
|
||||
label: str = ORCHESTRATOR_LABEL,
|
||||
port: int = DEFAULT_PORT,
|
||||
control_network: str = CONTROL_NETWORK,
|
||||
repo_root: Path | None = None,
|
||||
repo_root: Path = _REPO_ROOT,
|
||||
db_volume: str = ORCHESTRATOR_DB_VOLUME,
|
||||
) -> None:
|
||||
self.image_ref = image_ref
|
||||
@@ -69,9 +69,7 @@ class MacosOrchestrator(Orchestrator):
|
||||
self.label = label
|
||||
self.port = port
|
||||
self.control_network = control_network
|
||||
# Build context / bind-mount source: the repo root in a checkout, a
|
||||
# staged copy from the installed wheel otherwise (bot_bottle.resources).
|
||||
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||
self._repo_root = repo_root
|
||||
self._db_volume = db_volume
|
||||
|
||||
def url(self) -> str:
|
||||
|
||||
@@ -21,7 +21,6 @@ _HANDLERS: dict[str, str] = {
|
||||
"backend": "backend:cmd_backend",
|
||||
"cleanup": "cleanup:cmd_cleanup",
|
||||
"commit": "commit:cmd_commit",
|
||||
"doctor": "doctor:cmd_doctor",
|
||||
"edit": "edit:cmd_edit",
|
||||
"help": "help:cmd_help",
|
||||
"init": "init:cmd_init",
|
||||
@@ -54,6 +53,6 @@ COMMANDS = {name: _lazy(spec) for name, spec in _HANDLERS.items()}
|
||||
# gating it on the schema breaks preflight on a fresh CI runner where stdin
|
||||
# isn't a TTY and the migration prompt can't be answered. `help` and `login`
|
||||
# likewise never touch the store.
|
||||
NO_MIGRATION_COMMANDS = frozenset({"backend", "doctor", "help", "login"})
|
||||
NO_MIGRATION_COMMANDS = frozenset({"backend", "help", "login"})
|
||||
|
||||
__all__ = ["COMMANDS", "NO_MIGRATION_COMMANDS"]
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
"""`doctor` CLI command — validate host prerequisites for running
|
||||
bot-bottle and report what's ready.
|
||||
|
||||
Fails (non-zero exit) only on the two hard requirements: a new-enough
|
||||
Python and at least one backend that is *ready* (passes its full status
|
||||
checks, so `start` can actually work). The config directory is a soft
|
||||
check — `install.sh` creates it, but a missing one only warrants a note,
|
||||
not a failure, since `start` provisions what it needs on first run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from ...backend import is_backend_ready, known_backend_names
|
||||
from ..constants import PROG
|
||||
|
||||
MIN_PYTHON = (3, 11)
|
||||
CONFIG_DIR = ".bot-bottle"
|
||||
|
||||
|
||||
def _ok(label: str, detail: str) -> None:
|
||||
print(f"ok: {label}: {detail}")
|
||||
|
||||
|
||||
def _warn(label: str, detail: str) -> None:
|
||||
print(f"warn: {label}: {detail}")
|
||||
|
||||
|
||||
def _fail(label: str, detail: str) -> None:
|
||||
print(f"fail: {label}: {detail}")
|
||||
|
||||
|
||||
def _check_python() -> bool:
|
||||
v = sys.version_info
|
||||
detail = f"{v.major}.{v.minor}.{v.micro}"
|
||||
if (v.major, v.minor) >= MIN_PYTHON:
|
||||
_ok("python", detail)
|
||||
return True
|
||||
_fail("python", f"{detail}; need {MIN_PYTHON[0]}.{MIN_PYTHON[1]} or newer")
|
||||
return False
|
||||
|
||||
|
||||
def _check_backends() -> bool:
|
||||
"""At least one backend must be *ready* to run a bottle — i.e. pass its
|
||||
full status() checks (daemon reachable, network pool present, KVM usable),
|
||||
not merely have a binary on PATH. A binary-only check would report `ok`
|
||||
on a host with a stopped Docker daemon or a half-configured Firecracker,
|
||||
where `start` still can't work. Each not-ready backend prints its own
|
||||
diagnostics (quiet=False) so the operator sees exactly what's missing."""
|
||||
ready = []
|
||||
for name in known_backend_names():
|
||||
if is_backend_ready(name, quiet=False):
|
||||
_ok("backend", f"{name}: ready")
|
||||
ready.append(name)
|
||||
else:
|
||||
_warn("backend", f"{name}: not ready (see diagnostics above)")
|
||||
if ready:
|
||||
return True
|
||||
_fail(
|
||||
"backend",
|
||||
"no backend is ready to run a bottle; start Docker, or finish "
|
||||
"Apple Container (macOS) / Firecracker (Linux) setup",
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _check_config_dir() -> None:
|
||||
config = Path.home() / CONFIG_DIR
|
||||
if config.is_dir():
|
||||
_ok("config", str(config))
|
||||
else:
|
||||
_warn("config", f"{config} does not exist yet (created on first use)")
|
||||
|
||||
|
||||
def cmd_doctor(argv: list[str]) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog=f"{PROG} doctor",
|
||||
description="Check host prerequisites for running bot-bottle.",
|
||||
)
|
||||
parser.parse_args(argv)
|
||||
|
||||
# Hard requirements gate the exit code; the config note is advisory.
|
||||
required = [_check_python(), _check_backends()]
|
||||
_check_config_dir()
|
||||
return 0 if all(required) else 1
|
||||
@@ -25,7 +25,6 @@ def cmd_help(argv: list[str] | None = None) -> int:
|
||||
w(" backend set up / check / undo a backend's host prerequisites (setup|status|teardown)\n")
|
||||
w(" cleanup stop and remove all active bot-bottle containers\n")
|
||||
w(" commit snapshot a running bottle's container state to a Docker image\n")
|
||||
w(" doctor check host prerequisites (Python, backend, config dir)\n")
|
||||
w(" edit open an agent in vim for editing\n")
|
||||
w(" help show this command list\n")
|
||||
w(" init interactively create a new agent and add it to bot-bottle.json\n")
|
||||
|
||||
@@ -62,6 +62,7 @@ GATEWAY_CA_GLOB = "mitmproxy-ca*"
|
||||
# that lands. Env override matches the backend's BOT_BOTTLE_GATEWAY_IMAGE.
|
||||
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
||||
GATEWAY_DOCKERFILE = "Dockerfile.gateway"
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]:
|
||||
|
||||
@@ -17,7 +17,7 @@ Each queued proposal tool call:
|
||||
4. On a decision within the window, returns the operator's
|
||||
`{status, notes}`. On timeout, returns `status: pending` **with the
|
||||
proposal id** and leaves the proposal queued — the flow is
|
||||
non-blocking past the grace window (PRD 0072 / issue #412).
|
||||
non-blocking past the grace window (PRD prd-new / issue #412).
|
||||
|
||||
`check-proposal` is the non-blocking companion: given a `proposal_id`
|
||||
returned by a `pending` response, it reports the current decision
|
||||
|
||||
@@ -66,7 +66,12 @@ if TYPE_CHECKING:
|
||||
from .agent import ManifestAgent, ManifestAgentProvider
|
||||
from .bottle import ManifestBottle
|
||||
from .egress import EGRESS_AUTH_SCHEMES, ManifestEgressConfig, ManifestEgressRoute
|
||||
from .git import ManifestGitEntry, ManifestGitUser, ManifestKeyConfig
|
||||
from .git import (
|
||||
ManifestGitEntry,
|
||||
ManifestGitSigning,
|
||||
ManifestGitUser,
|
||||
ManifestKeyConfig,
|
||||
)
|
||||
|
||||
|
||||
# Facade name -> submodule that defines it. The aggregate model (`Manifest`,
|
||||
@@ -82,6 +87,7 @@ _LAZY_MODULES: dict[str, str] = {
|
||||
"ManifestEgressRoute": "egress",
|
||||
"ManifestEgressConfig": "egress",
|
||||
"ManifestGitEntry": "git",
|
||||
"ManifestGitSigning": "git",
|
||||
"ManifestGitUser": "git",
|
||||
"ManifestKeyConfig": "git",
|
||||
}
|
||||
@@ -107,6 +113,7 @@ __all__ = [
|
||||
"ManifestIndex",
|
||||
"ManifestError",
|
||||
"ManifestGitEntry",
|
||||
"ManifestGitSigning",
|
||||
"ManifestGitUser",
|
||||
"ManifestKeyConfig",
|
||||
"ManifestAgentProvider",
|
||||
|
||||
@@ -18,7 +18,12 @@ from typing import Mapping
|
||||
from .util import ManifestError, as_json_object
|
||||
from .agent import ManifestAgentProvider
|
||||
from .egress import ManifestEgressConfig
|
||||
from .git import ManifestGitEntry, ManifestGitUser, parse_git_gate_config
|
||||
from .git import (
|
||||
ManifestGitEntry,
|
||||
ManifestGitSigning,
|
||||
ManifestGitUser,
|
||||
parse_git_gate_config,
|
||||
)
|
||||
from .schema import BOTTLE_KEYS
|
||||
|
||||
__all__ = ["ManifestBottle"]
|
||||
@@ -38,6 +43,10 @@ class ManifestBottle:
|
||||
# `git config --global` step entirely. A bottle can declare a user
|
||||
# identity without any git-gate.repos upstreams, and vice versa.
|
||||
git_user: ManifestGitUser = field(default_factory=ManifestGitUser)
|
||||
# Per-bottle commit signing (PRD prd-new: signed commits & audit
|
||||
# attribution). Off by default; `git-gate.signing.enabled: true`
|
||||
# opts a bottle into per-activation signing + audit. Bottle-only.
|
||||
git_signing: ManifestGitSigning = field(default_factory=ManifestGitSigning)
|
||||
egress: ManifestEgressConfig = field(default_factory=ManifestEgressConfig)
|
||||
# Per-bottle stuck-recovery daemon (PRD 0013). When true (the
|
||||
# default, issue #249), the launch step brings up a supervise
|
||||
@@ -109,9 +118,10 @@ class ManifestBottle:
|
||||
|
||||
git: tuple[ManifestGitEntry, ...] = ()
|
||||
git_user = ManifestGitUser()
|
||||
git_signing = ManifestGitSigning()
|
||||
git_raw = d.get("git-gate")
|
||||
if git_raw is not None:
|
||||
git, git_user = parse_git_gate_config(name, git_raw)
|
||||
git, git_user, git_signing = parse_git_gate_config(name, git_raw)
|
||||
|
||||
agent_provider = (
|
||||
ManifestAgentProvider.from_dict(name, d["agent_provider"])
|
||||
@@ -141,7 +151,8 @@ class ManifestBottle:
|
||||
|
||||
return cls(
|
||||
env=env, agent_provider=agent_provider, git=git,
|
||||
git_user=git_user, egress=egress, supervise=supervise_raw,
|
||||
git_user=git_user, git_signing=git_signing, egress=egress,
|
||||
supervise=supervise_raw,
|
||||
nested_containers=nested_raw,
|
||||
declared_fields=frozenset(d),
|
||||
)
|
||||
|
||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
||||
|
||||
from .bottle import ManifestBottle
|
||||
from .egress import ManifestEgressConfig, validate_egress_routes
|
||||
from .git import ManifestGitUser, parse_git_gate_config
|
||||
from .git import ManifestGitSigning, ManifestGitUser, parse_git_gate_config
|
||||
from .util import ManifestError, as_json_object
|
||||
|
||||
|
||||
@@ -19,6 +19,17 @@ def _overlay_declared_bool(
|
||||
return value
|
||||
|
||||
|
||||
def _overlay_signing(
|
||||
base: ManifestBottle, override: ManifestBottle
|
||||
) -> ManifestGitSigning:
|
||||
"""Overlay `git-gate.signing`: an override that enables signing wins;
|
||||
otherwise the base's value is inherited. Mirrors the non-empty-wins
|
||||
overlay used for `git_user` — a child cannot un-set a parent's signing
|
||||
by declaring `enabled: false` (that reads as the default), the same
|
||||
way a child cannot blank a parent's git-gate.user field."""
|
||||
return override.git_signing if override.git_signing.enabled else base.git_signing
|
||||
|
||||
|
||||
def merge_bottles_runtime(bottles: "list[ManifestBottle]") -> "ManifestBottle":
|
||||
"""Merge an ordered list of pre-resolved ManifestBottle objects.
|
||||
|
||||
@@ -69,6 +80,7 @@ def _merge_two_bottles_runtime(base: "ManifestBottle", override: "ManifestBottle
|
||||
agent_provider=override.agent_provider,
|
||||
git=merged_git,
|
||||
git_user=merged_git_user,
|
||||
git_signing=_overlay_signing(base, override),
|
||||
egress=merged_egress,
|
||||
supervise=_overlay_declared_bool(base, override, "supervise"),
|
||||
nested_containers=_overlay_declared_bool(
|
||||
@@ -210,7 +222,7 @@ def _fold_two_bottles(
|
||||
for n in names
|
||||
}
|
||||
if merged_repos_raw:
|
||||
merged_git, _ = parse_git_gate_config("_fold", {"repos": merged_repos_raw})
|
||||
merged_git, _, _ = parse_git_gate_config("_fold", {"repos": merged_repos_raw})
|
||||
else:
|
||||
merged_git = ()
|
||||
|
||||
@@ -225,6 +237,7 @@ def _fold_two_bottles(
|
||||
agent_provider=later.agent_provider,
|
||||
git=merged_git,
|
||||
git_user=merged_git_user,
|
||||
git_signing=_overlay_signing(earlier, later),
|
||||
egress=merged_egress,
|
||||
supervise=_overlay_declared_bool(earlier, later, "supervise"),
|
||||
nested_containers=_overlay_declared_bool(
|
||||
@@ -299,6 +312,7 @@ def _merge_bottles(
|
||||
agent_provider=merged_agent_provider,
|
||||
git=merged_git,
|
||||
git_user=merged_git_user,
|
||||
git_signing=_overlay_signing(parent, child),
|
||||
egress=merged_egress,
|
||||
supervise=merged_supervise,
|
||||
nested_containers=merged_nested_containers,
|
||||
|
||||
@@ -283,16 +283,52 @@ class ManifestGitUser:
|
||||
return not self.name and not self.email
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ManifestGitSigning:
|
||||
"""Per-bottle commit-signing switch (PRD prd-new: signed commits &
|
||||
audit attribution).
|
||||
|
||||
When `enabled`, the launcher mints a per-activation Ed25519 signing
|
||||
key host-side, holds the private half in the sidecar ssh-agent, and
|
||||
configures the bottle to sign every commit at commit time; the gate
|
||||
rejects any commit it forwards that is not signed by the activation
|
||||
key, and the control plane records an audit binding.
|
||||
|
||||
Bottle-only: it carries data-plane / audit policy, not an
|
||||
agent-overlayable identity, so — like `git-gate.repos` — it is
|
||||
rejected at the agent level. Defaults to off: bottles that omit
|
||||
`git-gate.signing` behave exactly as before."""
|
||||
|
||||
enabled: bool = False
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, bottle_name: str, raw: object) -> "ManifestGitSigning":
|
||||
d = as_json_object(raw, f"bottle '{bottle_name}' git-gate.signing")
|
||||
for k in d:
|
||||
if k != "enabled":
|
||||
raise ManifestError(
|
||||
f"bottle '{bottle_name}' git-gate.signing has unknown key "
|
||||
f"{k!r}; allowed: enabled"
|
||||
)
|
||||
enabled = d.get("enabled", False)
|
||||
if not isinstance(enabled, bool):
|
||||
raise ManifestError(
|
||||
f"bottle '{bottle_name}' git-gate.signing.enabled must be a "
|
||||
f"boolean (was {type(enabled).__name__})"
|
||||
)
|
||||
return cls(enabled=enabled)
|
||||
|
||||
|
||||
def parse_git_gate_config(
|
||||
bottle_name: str,
|
||||
raw: object,
|
||||
) -> tuple[tuple[ManifestGitEntry, ...], ManifestGitUser]:
|
||||
) -> tuple[tuple[ManifestGitEntry, ...], ManifestGitUser, ManifestGitSigning]:
|
||||
d = as_json_object(raw, f"bottle '{bottle_name}' git-gate")
|
||||
for k in d:
|
||||
if k not in {"user", "repos"}:
|
||||
if k not in {"user", "repos", "signing"}:
|
||||
raise ManifestError(
|
||||
f"bottle '{bottle_name}' git-gate has unknown key {k!r}; "
|
||||
f"allowed: user, repos"
|
||||
f"allowed: user, repos, signing"
|
||||
)
|
||||
|
||||
git_user = (
|
||||
@@ -301,6 +337,12 @@ def parse_git_gate_config(
|
||||
else ManifestGitUser()
|
||||
)
|
||||
|
||||
git_signing = (
|
||||
ManifestGitSigning.from_dict(bottle_name, d["signing"])
|
||||
if "signing" in d
|
||||
else ManifestGitSigning()
|
||||
)
|
||||
|
||||
git: tuple[ManifestGitEntry, ...] = ()
|
||||
repos_raw = d.get("repos")
|
||||
if repos_raw is not None:
|
||||
@@ -311,4 +353,4 @@ def parse_git_gate_config(
|
||||
)
|
||||
validate_unique_git_names(bottle_name, git)
|
||||
|
||||
return git, git_user
|
||||
return git, git_user, git_signing
|
||||
|
||||
@@ -17,10 +17,7 @@ from pathlib import Path
|
||||
|
||||
from .. import log
|
||||
from .store.store_manager import StoreManager
|
||||
from ..paths import LAUNCH_BROKER_KEY_ENV
|
||||
from .broker import StubBroker, SubmitBroker
|
||||
from .broker_client import BrokerClient
|
||||
from .host_server import DEFAULT_PORT, broker_secret
|
||||
from .broker import LaunchBroker, StubBroker
|
||||
from .server import make_server
|
||||
from .docker_broker import DockerBroker
|
||||
from .store.registry_store import RegistryStore, default_db_path
|
||||
@@ -37,13 +34,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
help=f"registry DB path (default: {default_db_path()})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--broker", choices=("stub", "docker", "http"), default="stub",
|
||||
help="launch broker: 'stub' records requests; 'docker' runs containers "
|
||||
"in-process; 'http' relays signed requests to a host control server",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host-controller-url", default=f"http://127.0.0.1:{DEFAULT_PORT}",
|
||||
help="host control server URL (used only with --broker http)",
|
||||
"--broker", choices=("stub", "docker"), default="stub",
|
||||
help="launch broker: 'stub' records requests; 'docker' runs containers",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
@@ -55,27 +47,11 @@ def main(argv: list[str] | None = None) -> int:
|
||||
# operator reaches it over HTTP (never a second, disconnected DB).
|
||||
StoreManager(registry.db_path).migrate()
|
||||
|
||||
# A signing secret ties the orchestrator (signer) to its broker (verifier).
|
||||
# 'stub' records launches instead of starting anything; 'docker' runs real
|
||||
# containers in-process; 'http' relays signed requests to a separate host
|
||||
# control server, which verifies and launches. For 'stub'/'docker' the secret
|
||||
# is ephemeral (signer and verifier share this process). For 'http' it must be
|
||||
# the SAME key the host controller holds — and this process is the *guest*
|
||||
# (signer), so it must be given that key by injection, NOT mint its own
|
||||
# process-local one (which would diverge from the host's and 401 every launch).
|
||||
broker: SubmitBroker
|
||||
if args.broker == "http":
|
||||
secret = broker_secret() # env-injected only; no host-file fallback here
|
||||
if secret is None:
|
||||
parser.error(
|
||||
f"--broker http requires the launch-broker key injected as "
|
||||
f"${LAUNCH_BROKER_KEY_ENV} (the host controller owns/mints it); the "
|
||||
"orchestrator must not mint its own or it would diverge from the host's"
|
||||
)
|
||||
broker = BrokerClient(args.host_controller_url)
|
||||
else:
|
||||
secret = secrets.token_bytes(32)
|
||||
broker = DockerBroker(secret) if args.broker == "docker" else StubBroker(secret)
|
||||
# An ephemeral signing secret ties the orchestrator (signer) to its
|
||||
# broker (verifier). 'stub' records launches instead of starting
|
||||
# anything; 'docker' runs real containers (firecracker drops in later).
|
||||
secret = secrets.token_bytes(32)
|
||||
broker: LaunchBroker = DockerBroker(secret) if args.broker == "docker" else StubBroker(secret)
|
||||
orchestrator = OrchestratorCore(registry, broker, secret)
|
||||
|
||||
server = make_server(orchestrator, host=args.host, port=args.port)
|
||||
|
||||
@@ -29,7 +29,6 @@ import json
|
||||
import secrets
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
|
||||
_JWT_HEADER = {"alg": "HS256", "typ": "JWT"}
|
||||
_ALLOWED_OPS = ("launch", "teardown")
|
||||
@@ -38,21 +37,7 @@ _ALLOWED_OPS = ("launch", "teardown")
|
||||
class BrokerAuthError(Exception):
|
||||
"""A broker request failed provenance or schema verification —
|
||||
bad/absent signature, malformed token, or a payload that doesn't match
|
||||
the fixed launch-request shape. Fail-closed: the broker must not act.
|
||||
|
||||
A **definite** negative: nothing was launched, so a caller may safely roll
|
||||
back as if the op never happened."""
|
||||
|
||||
|
||||
class BrokerUnavailableError(Exception):
|
||||
"""A brokered request could not be carried to a verdict: the broker (or the
|
||||
wire to it) was unreachable, timed out, or dropped the response.
|
||||
|
||||
Crucially **ambiguous** — unlike `BrokerAuthError`, the op MAY already have
|
||||
taken effect on the backend before the response was lost, so a caller must
|
||||
NOT assume it did nothing (e.g. must not roll a registry row back as if no
|
||||
launch happened, which would orphan a running container). Only the in-process
|
||||
brokers never raise this; the out-of-process `BrokerClient` does."""
|
||||
the fixed launch-request shape. Fail-closed: the broker must not act."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -138,16 +123,6 @@ def verify_request(token: str, secret: bytes) -> LaunchRequest:
|
||||
|
||||
# --- the broker itself ------------------------------------------------------
|
||||
|
||||
class SubmitBroker(Protocol):
|
||||
"""The single method `OrchestratorCore` depends on: verify a signed token and
|
||||
perform its op, returning the verified request. Both the in-process
|
||||
`LaunchBroker` and the out-of-process `BrokerClient` (which relays the token
|
||||
to the host control server) satisfy it structurally, so the core is unchanged
|
||||
whether the backend is local or a real host service."""
|
||||
|
||||
def submit(self, token: str) -> LaunchRequest: ...
|
||||
|
||||
|
||||
class LaunchBroker(abc.ABC):
|
||||
"""Verifies a signed request came from the orchestrator, then performs
|
||||
the backend-native launch/teardown. Subclasses implement `_launch` /
|
||||
@@ -193,9 +168,7 @@ class StubBroker(LaunchBroker):
|
||||
|
||||
__all__ = [
|
||||
"BrokerAuthError",
|
||||
"BrokerUnavailableError",
|
||||
"LaunchRequest",
|
||||
"SubmitBroker",
|
||||
"LaunchBroker",
|
||||
"StubBroker",
|
||||
"sign_request",
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
"""Orchestrator-side broker transport (issue #468, chunk 1).
|
||||
|
||||
The signer's half of the launch-broker transport gap. `BrokerClient` satisfies
|
||||
the exact `submit(token)` contract `OrchestratorCore` already depends on (see
|
||||
`broker.SubmitBroker`), but instead of verifying and launching in-process it POSTs
|
||||
the signed token to the host control server over HTTP (stdlib `urllib`, like
|
||||
`orchestrator/client.py`). Because it is drop-in for that interface, wiring a real
|
||||
out-of-process backend does not change the core: it still signs a request and
|
||||
calls `submit()`; only the wire is new.
|
||||
|
||||
A provenance/schema rejection from the host controller (HTTP 401) is re-raised as
|
||||
the same `BrokerAuthError` the in-process broker raises, so the launch path's
|
||||
rollback-on-failure (`OrchestratorCore.launch_bottle`) behaves identically whether
|
||||
the broker is local or remote.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from .broker import BrokerAuthError, BrokerUnavailableError, LaunchRequest
|
||||
|
||||
DEFAULT_TIMEOUT_SECONDS = 5.0
|
||||
|
||||
|
||||
class BrokerClientError(RuntimeError):
|
||||
"""The host control server *responded*, but with an unexpected status other
|
||||
than the fail-closed 401 (which surfaces as `BrokerAuthError`) — e.g. a 502
|
||||
backend failure or a malformed body. A definite negative: the host processed
|
||||
the request and it did not launch. (A *no-response* failure — unreachable /
|
||||
timeout / dropped — is the ambiguous `BrokerUnavailableError` instead.)"""
|
||||
|
||||
|
||||
class BrokerClient:
|
||||
"""Drop-in `submit(token)` that relays a signed request to the host control
|
||||
server. Holds no secret — provenance rides entirely in the signed token, so a
|
||||
caller that can reach this client still cannot forge a launch."""
|
||||
|
||||
def __init__(self, base_url: str, *, timeout: float = DEFAULT_TIMEOUT_SECONDS) -> None:
|
||||
self._base = base_url.rstrip("/")
|
||||
self._timeout = timeout
|
||||
|
||||
def submit(self, token: str) -> LaunchRequest:
|
||||
"""POST the signed token to the host controller and return the request it
|
||||
verified and acted on.
|
||||
|
||||
Raises `BrokerAuthError` on a fail-closed 401 (bad provenance/schema —
|
||||
the same exception the in-process broker raises); `BrokerClientError` if
|
||||
the host *responds* with any other non-success status or a malformed
|
||||
body (a definite negative); or `BrokerUnavailableError` if no response is
|
||||
obtained (unreachable / timeout / dropped) — the **ambiguous** case, where
|
||||
the host may already have acted, so the caller must not roll back."""
|
||||
data = json.dumps({"token": token}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{self._base}/broker", data=data, method="POST",
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=self._timeout) as resp:
|
||||
return _request_from(_json_object(resp.read()))
|
||||
except urllib.error.HTTPError as e:
|
||||
detail = _error_detail(e)
|
||||
if e.code == 401:
|
||||
raise BrokerAuthError(
|
||||
detail or "host controller rejected the request"
|
||||
) from e
|
||||
raise BrokerClientError(
|
||||
f"POST /broker: HTTP {e.code} {detail}".rstrip()
|
||||
) from e
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
||||
# No usable response — unreachable, timed out, or the connection
|
||||
# dropped mid-exchange. Ambiguous: the request may already have
|
||||
# launched the bottle, so this is NOT a definite failure.
|
||||
raise BrokerUnavailableError(f"POST /broker: {e}") from e
|
||||
|
||||
|
||||
def _json_object(raw: bytes) -> dict[str, object]:
|
||||
"""Parse a JSON object, tolerating an empty or malformed body (→ {}), like
|
||||
the orchestrator client — a bad body becomes a clean 'missing field' error
|
||||
downstream rather than an opaque JSON crash."""
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
obj = json.loads(raw)
|
||||
except ValueError:
|
||||
return {}
|
||||
return obj if isinstance(obj, dict) else {}
|
||||
|
||||
|
||||
def _error_detail(e: urllib.error.HTTPError) -> str:
|
||||
"""The `error` string from a structured error response, best-effort — an
|
||||
error body may be absent or unreadable, in which case there is no detail."""
|
||||
try:
|
||||
detail = _json_object(e.read()).get("error", "")
|
||||
except Exception: # noqa: BLE001 — the error body is advisory only
|
||||
return ""
|
||||
return detail if isinstance(detail, str) else ""
|
||||
|
||||
|
||||
def _request_from(payload: dict[str, object]) -> LaunchRequest:
|
||||
"""Reconstruct the verified `LaunchRequest` the controller echoed, so the
|
||||
returned value matches the in-process broker's (which returns the request it
|
||||
acted on). A missing op/bottle_id means a malformed response."""
|
||||
op = payload.get("op")
|
||||
bottle_id = payload.get("bottle_id")
|
||||
if not isinstance(op, str) or not isinstance(bottle_id, str) or not bottle_id:
|
||||
raise BrokerClientError("host controller response missing op/bottle_id")
|
||||
source_ip = payload.get("source_ip")
|
||||
image_ref = payload.get("image_ref")
|
||||
slot = payload.get("slot")
|
||||
return LaunchRequest(
|
||||
op=op,
|
||||
bottle_id=bottle_id,
|
||||
source_ip=source_ip if isinstance(source_ip, str) else "",
|
||||
image_ref=image_ref if isinstance(image_ref, str) else "",
|
||||
slot=slot if isinstance(slot, int) and not isinstance(slot, bool) else None,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BrokerClient",
|
||||
"BrokerClientError",
|
||||
"DEFAULT_TIMEOUT_SECONDS",
|
||||
]
|
||||
@@ -1,287 +0,0 @@
|
||||
"""Host control server (issue #468) — the launch broker as a real host service.
|
||||
|
||||
Chunk 1 of the host-control-server stack closes the **transport** gap the PRD
|
||||
opens with: today `LaunchBroker.submit(token)` is an in-process method call from
|
||||
`OrchestratorCore`, and a real host service needs it reachable over the wire.
|
||||
This module is that service — the single privileged host component — reached over
|
||||
**HTTP** (the universal transport 0070 chose), mirroring the orchestrator control
|
||||
plane's shape (`orchestrator/server.py`): a pure `dispatch()` for socket-free
|
||||
testing, wrapped by a thin stdlib `http.server` adapter.
|
||||
|
||||
GET /health -> 200 {"status": "ok"}
|
||||
POST /broker -> 200 {"op", "bottle_id", "source_ip", "image_ref", "slot"}
|
||||
400 (bad body) | 401 (bad provenance/schema) | 502 (backend)
|
||||
body: {"token": "<signed launch/teardown JWT>"}
|
||||
|
||||
Only the **signed token** crosses the wire; the server holds the shared HS256
|
||||
secret and a real `LaunchBroker` (e.g. `DockerBroker`) and runs the existing
|
||||
`verify_request` + `_launch`/`_teardown` path behind the endpoint, so nothing
|
||||
free-form ever reaches it. Provenance/schema failures are fail-closed 401s that
|
||||
never touch the backend (`LaunchBroker.submit` verifies before acting), and a
|
||||
backend launch failure is a 502 the caller must surface — neither takes the
|
||||
controller down.
|
||||
|
||||
The signed launch token *is* the endpoint's authentication (its provenance is the
|
||||
whole point of the JWS), so `/broker` needs no separate caller credential; the
|
||||
host controller's own lifecycle endpoints, which do, arrive with the `host`-role
|
||||
tokens of the separate `HOST_CONTROLLER` trust domain in a later chunk.
|
||||
|
||||
The shared signing secret is the durable **launch-broker `TrustDomain` key**
|
||||
(#468/#476): a host-canonical key file minted 0600 on first use, provisioned to
|
||||
the orchestrator (signer) and this server (verifier). A backend launcher injects
|
||||
it via `$BOT_BOTTLE_LAUNCH_BROKER_KEY`; a host-side dev-harness process reads the
|
||||
key file directly. Durability is the point — a restarted orchestrator re-verifies
|
||||
against the same key, so re-adoption works.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import socketserver
|
||||
import sys
|
||||
import typing
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from .. import log
|
||||
from ..paths import LAUNCH_BROKER_KEY_ENV
|
||||
from ..trust_domain import LAUNCH_BROKER
|
||||
from .broker import BrokerAuthError, LaunchBroker
|
||||
from .docker_broker import DockerBroker
|
||||
|
||||
# JSON body payload type (parsed request / rendered response).
|
||||
Json = dict[str, object]
|
||||
|
||||
# Default host-controller port. Distinct from the orchestrator control plane
|
||||
# (8099) — a separate privileged component listening on its own socket.
|
||||
DEFAULT_PORT = 8091
|
||||
|
||||
# Cap on the request body. A signed broker request is tiny, so rejecting anything
|
||||
# larger *before reading it* keeps a caller that can merely reach the socket (no
|
||||
# signed token needed) from exhausting memory or a handler thread with a huge
|
||||
# Content-Length — the signed token, not mere reachability, is the authority.
|
||||
MAX_BODY_BYTES = 64 * 1024
|
||||
|
||||
# Per-request socket timeout, bounding how long a stalled / slow-loris caller can
|
||||
# hold a handler thread on this privileged listener.
|
||||
REQUEST_TIMEOUT_SECONDS = 15
|
||||
|
||||
|
||||
def _parse_json_object(body: bytes) -> Json:
|
||||
"""Parse a JSON object body. Raises ValueError for non-objects / bad JSON."""
|
||||
if not body:
|
||||
return {}
|
||||
obj = json.loads(body) # raises json.JSONDecodeError (a ValueError)
|
||||
if not isinstance(obj, dict):
|
||||
raise ValueError("request body must be a JSON object")
|
||||
return obj
|
||||
|
||||
|
||||
def broker_secret(
|
||||
environ: typing.Mapping[str, str] | None = None, *, allow_host_file: bool = False,
|
||||
) -> bytes | None:
|
||||
"""The shared launch-broker HS256 secret, as this process should use it.
|
||||
|
||||
Always prefers the key injected into this process's env
|
||||
(`$BOT_BOTTLE_LAUNCH_BROKER_KEY`). `allow_host_file` decides the fallback when
|
||||
it is absent, and the distinction is a security boundary:
|
||||
|
||||
- **Host-side** processes — the host controller and the host dev-harness — pass
|
||||
``allow_host_file=True`` to read (minting on first use) the durable host key
|
||||
file (``bot_bottle_root()/launch-broker-key``) they legitimately own.
|
||||
- The **guest orchestrator** (``--broker http``) keeps the default ``False``.
|
||||
It runs inside a container/VM whose ``bot_bottle_root()`` is process-local,
|
||||
so minting a file there would silently create a key UNRELATED to the host
|
||||
controller's — startup would succeed but every launch would be rejected 401.
|
||||
It must instead be *given* the key by its launcher, and fail closed (None)
|
||||
if it wasn't, rather than diverge.
|
||||
|
||||
None when no key is available (a guest with no injection, or an unwritable
|
||||
host root)."""
|
||||
key = LAUNCH_BROKER.key_from_env(environ)
|
||||
if not key and allow_host_file:
|
||||
try:
|
||||
key = LAUNCH_BROKER.signing_key() # host-canonical, minted on first use
|
||||
except OSError:
|
||||
return None
|
||||
return key.encode("utf-8") if key else None
|
||||
|
||||
|
||||
def dispatch( # pylint: disable=too-many-return-statements
|
||||
broker: LaunchBroker, method: str, path: str, body: bytes,
|
||||
) -> tuple[int, Json]:
|
||||
"""Route one host-control request to a (status, payload) pair. Pure — the
|
||||
only side effect is the broker's own backend launch — so routing is testable
|
||||
without a socket.
|
||||
|
||||
Total by design: a provenance/schema failure becomes 401 and a backend launch
|
||||
failure becomes 502 rather than raising, so one bad request can neither act
|
||||
on the backend nor take the controller down for the next caller."""
|
||||
route = urlsplit(path).path.rstrip("/") or "/"
|
||||
|
||||
if method == "GET" and route == "/health":
|
||||
return 200, {"status": "ok"}
|
||||
|
||||
if method == "POST" and route == "/broker":
|
||||
try:
|
||||
data = _parse_json_object(body)
|
||||
except ValueError as e:
|
||||
return 400, {"error": f"invalid JSON: {e}"}
|
||||
token = data.get("token")
|
||||
if not isinstance(token, str) or not token:
|
||||
return 400, {"error": "token (string) is required"}
|
||||
try:
|
||||
req = broker.submit(token)
|
||||
except BrokerAuthError as e:
|
||||
# Fail-closed: bad signature, malformed token, or off-schema payload.
|
||||
# `submit` verifies before acting, so nothing was launched.
|
||||
return 401, {"error": f"broker auth failed: {e}"}
|
||||
except Exception as e: # noqa: BLE001 — a backend launch failure (docker
|
||||
# down, image gone) is operational, not a control-plane bug; the
|
||||
# caller must see it as a distinct 502, and the server must stay up.
|
||||
return 502, {"error": f"backend launch failed: {e}"}
|
||||
return 200, {
|
||||
"op": req.op,
|
||||
"bottle_id": req.bottle_id,
|
||||
"source_ip": req.source_ip,
|
||||
"image_ref": req.image_ref,
|
||||
"slot": req.slot,
|
||||
}
|
||||
|
||||
return 404, {"error": "not found"}
|
||||
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
"""Thin stdlib adapter: read the body, call `dispatch`, write JSON."""
|
||||
|
||||
# Socket timeout per request (applied by StreamRequestHandler.setup) so a
|
||||
# stalled caller can't pin a handler thread on this privileged listener.
|
||||
timeout = REQUEST_TIMEOUT_SECONDS
|
||||
|
||||
# Quiet by default; opt back into stdlib access logging with
|
||||
# BOT_BOTTLE_HOST_CONTROLLER_DEBUG (the controller has its own logging).
|
||||
def log_message(self, format: str, *args: typing.Any) -> None: # noqa: A002
|
||||
if os.environ.get("BOT_BOTTLE_HOST_CONTROLLER_DEBUG"):
|
||||
super().log_message(format, *args)
|
||||
|
||||
def _serve(self, method: str) -> None:
|
||||
"""Read the request body (bounded), dispatch it, and write the JSON
|
||||
reply. A dispatch that raises (it shouldn't — dispatch is total) still
|
||||
returns a 500 rather than dropping the connection."""
|
||||
server = self.server
|
||||
assert isinstance(server, HostControlServer)
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
except ValueError:
|
||||
self._reply(400, {"error": "invalid Content-Length"})
|
||||
return
|
||||
if length < 0 or length > MAX_BODY_BYTES:
|
||||
# Reject before reading: nothing legitimate is this big, so an
|
||||
# oversized declared length is a bug or a resource-exhaustion attempt.
|
||||
self._reply(413, {"error": "request body too large"})
|
||||
return
|
||||
body = self.rfile.read(length) if length > 0 else b""
|
||||
try:
|
||||
status, payload = dispatch(server.broker, method, self.path, body)
|
||||
except Exception as e: # noqa: BLE001 — the controller must stay up
|
||||
sys.stderr.write(f"host controller: {method} {self.path} failed: {e!r}\n")
|
||||
sys.stderr.flush()
|
||||
status, payload = 500, {"error": f"internal error: {e}"}
|
||||
self._reply(status, payload)
|
||||
|
||||
def _reply(self, status: int, payload: typing.Mapping[str, object]) -> None:
|
||||
"""Write one JSON response with an explicit Content-Length."""
|
||||
data = json.dumps(payload).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def do_GET(self) -> None:
|
||||
self._serve("GET")
|
||||
|
||||
def do_POST(self) -> None:
|
||||
self._serve("POST")
|
||||
|
||||
|
||||
class HostControlServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||||
"""Threading HTTP server that carries the launch broker for its handlers.
|
||||
|
||||
The broker holds the shared signing secret and performs the backend-native
|
||||
launch/teardown; the server itself keeps no secret of its own — provenance
|
||||
rides entirely in each request's signed token."""
|
||||
|
||||
daemon_threads = True
|
||||
allow_reuse_address = True
|
||||
|
||||
def __init__(self, address: tuple[str, int], broker: LaunchBroker) -> None:
|
||||
self.broker = broker
|
||||
super().__init__(address, Handler)
|
||||
|
||||
|
||||
def make_host_server(
|
||||
broker: LaunchBroker, host: str = "127.0.0.1", port: int = DEFAULT_PORT
|
||||
) -> HostControlServer:
|
||||
"""Build (but do not start) a host control server. `port=0` binds an
|
||||
ephemeral port — read `server.server_address` for the actual one."""
|
||||
return HostControlServer((host, port), broker)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
"""Run the host control server as a plain process (dev-harness).
|
||||
|
||||
python -m bot_bottle.orchestrator.host_server [--host H] [--port P]
|
||||
|
||||
Fail-closed: without the launch-broker key the server can verify no request's
|
||||
provenance, so it refuses to start rather than run a launcher that accepts
|
||||
unsigned input. As the host-side owner of the key, it may mint/read the host
|
||||
key file (`allow_host_file=True`)."""
|
||||
parser = argparse.ArgumentParser(prog="bot_bottle.orchestrator.host_server")
|
||||
parser.add_argument("--host", default="127.0.0.1", help="bind address")
|
||||
parser.add_argument("--port", type=int, default=DEFAULT_PORT, help="bind port (0 = ephemeral)")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
secret = broker_secret(allow_host_file=True)
|
||||
if secret is None:
|
||||
sys.stderr.write(
|
||||
f"host controller: refusing to start without the launch-broker key "
|
||||
f"(${LAUNCH_BROKER_KEY_ENV}, or a writable host root to mint it) — it "
|
||||
"could verify no request's provenance and would relay unsigned "
|
||||
"launches to the backend\n"
|
||||
)
|
||||
sys.stderr.flush()
|
||||
return 2
|
||||
|
||||
broker = DockerBroker(secret)
|
||||
server = make_host_server(broker, host=args.host, port=args.port)
|
||||
bound_host, bound_port = server.server_address[0], server.server_address[1]
|
||||
log.info(
|
||||
"host control server listening",
|
||||
context={"host": bound_host, "port": bound_port},
|
||||
)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
log.info("host controller shutting down")
|
||||
finally:
|
||||
server.server_close()
|
||||
return 0
|
||||
|
||||
|
||||
__all__ = [
|
||||
"dispatch",
|
||||
"Handler",
|
||||
"HostControlServer",
|
||||
"make_host_server",
|
||||
"broker_secret",
|
||||
"main",
|
||||
"Json",
|
||||
"DEFAULT_PORT",
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -25,7 +25,7 @@ import json
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .broker import BrokerUnavailableError, LaunchRequest, SubmitBroker, sign_request
|
||||
from .broker import LaunchBroker, LaunchRequest, sign_request
|
||||
from .store.registry_store import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore
|
||||
from .supervisor import (
|
||||
AuditEntry,
|
||||
@@ -62,7 +62,7 @@ class OrchestratorCore:
|
||||
def __init__(
|
||||
self,
|
||||
registry: RegistryStore,
|
||||
broker: SubmitBroker,
|
||||
broker: LaunchBroker,
|
||||
sign_secret: bytes,
|
||||
supervisor: Supervisor | None = None,
|
||||
) -> None:
|
||||
@@ -111,23 +111,14 @@ class OrchestratorCore:
|
||||
image_ref=image_ref,
|
||||
slot=slot,
|
||||
)
|
||||
launched = False
|
||||
try:
|
||||
self._broker.submit(sign_request(req, self._secret))
|
||||
except BrokerUnavailableError:
|
||||
# Ambiguous delivery failure (timeout / dropped response): the broker
|
||||
# may already have launched the bottle before the response was lost.
|
||||
# Do NOT deregister — that would orphan a running container with no
|
||||
# registry row (reconcile reaps rows, never containers). Keep the row
|
||||
# so reconcile reaps it iff the bottle is not actually live; surface
|
||||
# the error so the caller knows the launch is unconfirmed.
|
||||
raise
|
||||
except Exception:
|
||||
# A definite failure — a fail-closed rejection, a backend launch
|
||||
# error, or the host reporting it did not launch: nothing is running,
|
||||
# so roll the registry entry back to leave no orphan.
|
||||
self.registry.deregister(rec.bottle_id)
|
||||
self._tokens.pop(rec.bottle_id, None)
|
||||
raise
|
||||
launched = True
|
||||
finally:
|
||||
if not launched:
|
||||
self.registry.deregister(rec.bottle_id)
|
||||
self._tokens.pop(rec.bottle_id, None)
|
||||
return rec
|
||||
|
||||
def teardown_bottle(self, bottle_id: str) -> bool:
|
||||
|
||||
@@ -113,7 +113,7 @@ _MIGRATIONS = TableMigrations(
|
||||
# egress allowlist / routes / git config selected by source IP. The
|
||||
# multi-tenant gateway resolves it per request via `attribute`.
|
||||
"ALTER TABLE orchestrator_bottles ADD COLUMN policy TEXT NOT NULL DEFAULT ''",
|
||||
# v4 — per-bottle encrypted egress secrets (PRD 0080).
|
||||
# v4 — per-bottle encrypted egress secrets (PRD prd-new-secret-provider).
|
||||
# One row per env-var: key (env-var name) is plaintext for auditing;
|
||||
# value is the encrypted token string. The encryption key (ENV_VAR_SECRET)
|
||||
# lives only in the agent's environment — a row alone cannot recover the
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Symmetric encryption for per-bottle egress secrets (PRD 0080).
|
||||
"""Symmetric encryption for per-bottle egress secrets (PRD prd-new-secret-provider).
|
||||
|
||||
Each agent receives a random ENV_VAR_SECRET at startup — passed as an env var,
|
||||
never logged or persisted. The host uses this key to encrypt each egress auth
|
||||
@@ -12,22 +12,12 @@ reattachment path reads ENV_VAR_SECRET from the running agent container via
|
||||
``POST /bottles/<id>/reprovision_gateway``; the orchestrator decrypts the
|
||||
stored rows and re-populates ``_tokens``.
|
||||
|
||||
Encryption scheme: HMAC-SHA256 used as a PRF in CTR mode, **authenticated**
|
||||
encrypt-then-MAC (stdlib-only, no external deps). Each value is encrypted
|
||||
independently. The output blob is ``nonce (16 bytes) || ciphertext || tag
|
||||
(32 bytes)`` encoded as URL-safe base64 (no padding).
|
||||
Encryption scheme: HMAC-SHA256 used as a PRF in CTR mode (stdlib-only,
|
||||
no external deps). Each value is encrypted independently. The output blob is
|
||||
``nonce (16 bytes) || ciphertext`` encoded as URL-safe base64 (no padding).
|
||||
|
||||
keystream_block_i = HMAC-SHA256(key, nonce || i.to_bytes(4, "big"))
|
||||
ciphertext_i = plaintext_i XOR keystream_block_i[:len(plaintext_i)]
|
||||
mac_key = HMAC-SHA256(key, "bottled-secret-mac-v1")
|
||||
tag = HMAC-SHA256(mac_key, nonce || ciphertext)
|
||||
|
||||
The tag is what makes a **wrong key deterministically detectable**: without it,
|
||||
CTR decryption with the wrong key yields garbage that only fails when it isn't
|
||||
valid UTF-8 (so ``reprovision`` would sometimes "succeed" with a wrong
|
||||
ENV_VAR_SECRET and inject garbage egress tokens). The MAC key is derived from
|
||||
the ENV_VAR_SECRET by a domain-separated HMAC so the same key never both
|
||||
generates the keystream and signs the tag with the same message shape.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -39,7 +29,6 @@ import secrets
|
||||
|
||||
_KEY_BYTES = 32 # 256-bit key from ENV_VAR_SECRET
|
||||
_NONCE_BYTES = 16 # 128-bit random nonce per encrypt call
|
||||
_TAG_BYTES = 32 # HMAC-SHA256 authentication tag
|
||||
_BLOCK = 32 # HMAC-SHA256 output width == one keystream block
|
||||
|
||||
# Env-var name the agent container receives at startup.
|
||||
@@ -61,58 +50,45 @@ def _keystream(key: bytes, nonce: bytes, block_index: int) -> bytes:
|
||||
).digest()
|
||||
|
||||
|
||||
def _tag(key: bytes, nonce: bytes, ciphertext: bytes) -> bytes:
|
||||
"""The authentication tag over ``nonce || ciphertext``, keyed by a MAC
|
||||
subkey domain-separated from the keystream key."""
|
||||
mac_key = hmac.new(key, b"bottled-secret-mac-v1", hashlib.sha256).digest()
|
||||
return hmac.new(mac_key, nonce + ciphertext, hashlib.sha256).digest()
|
||||
|
||||
|
||||
def _ctr(key: bytes, nonce: bytes, data: bytes) -> bytes:
|
||||
"""CTR keystream XOR — its own inverse, so it both encrypts and decrypts."""
|
||||
out = bytearray()
|
||||
for i in range(0, len(data), _BLOCK):
|
||||
chunk = data[i : i + _BLOCK]
|
||||
ks = _keystream(key, nonce, i)[: len(chunk)]
|
||||
out.extend(b ^ k for b, k in zip(chunk, ks))
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def encrypt_value(secret_b64: str, plaintext: str) -> str:
|
||||
"""Encrypt a single string value with *secret_b64* (the ENV_VAR_SECRET).
|
||||
|
||||
Returns a URL-safe base64 blob ``nonce || ciphertext || tag`` suitable for
|
||||
Returns a URL-safe base64 blob ``nonce || ciphertext`` suitable for
|
||||
the ``bottled_agent_secrets.value`` column."""
|
||||
key = _b64dec(secret_b64)
|
||||
pt = plaintext.encode()
|
||||
nonce = secrets.token_bytes(_NONCE_BYTES)
|
||||
ct = _ctr(key, nonce, plaintext.encode())
|
||||
tag = _tag(key, nonce, ct)
|
||||
return base64.urlsafe_b64encode(nonce + ct + tag).rstrip(b"=").decode()
|
||||
ct = bytearray()
|
||||
for i in range(0, len(pt), _BLOCK):
|
||||
chunk = pt[i : i + _BLOCK]
|
||||
ks = _keystream(key, nonce, i)[: len(chunk)]
|
||||
ct.extend(p ^ k for p, k in zip(chunk, ks))
|
||||
return base64.urlsafe_b64encode(nonce + bytes(ct)).rstrip(b"=").decode()
|
||||
|
||||
|
||||
def decrypt_value(secret_b64: str, blob_b64: str) -> str:
|
||||
"""Decrypt a blob produced by :func:`encrypt_value`.
|
||||
|
||||
Returns the original plaintext string. Raises ``ValueError`` for malformed
|
||||
input, a **wrong key**, or a tampered ciphertext — all caught by the
|
||||
authentication tag before any plaintext is returned, so a wrong
|
||||
ENV_VAR_SECRET is rejected deterministically (never a garbage token)."""
|
||||
input or a key mismatch (wrong key produces garbage, not an error, unless
|
||||
the plaintext is non-UTF-8 — treat all such failures as wrong key)."""
|
||||
key = _b64dec(secret_b64)
|
||||
try:
|
||||
blob = _b64dec(blob_b64)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"invalid ciphertext blob: {exc}") from exc
|
||||
if len(blob) < _NONCE_BYTES + _TAG_BYTES:
|
||||
if len(blob) < _NONCE_BYTES:
|
||||
raise ValueError("ciphertext blob too short")
|
||||
nonce = blob[:_NONCE_BYTES]
|
||||
tag = blob[-_TAG_BYTES:]
|
||||
ciphertext = blob[_NONCE_BYTES:-_TAG_BYTES]
|
||||
if not hmac.compare_digest(tag, _tag(key, nonce, ciphertext)):
|
||||
raise ValueError("ciphertext failed authentication (wrong key or tampered)")
|
||||
nonce, ciphertext = blob[:_NONCE_BYTES], blob[_NONCE_BYTES:]
|
||||
pt = bytearray()
|
||||
for i in range(0, len(ciphertext), _BLOCK):
|
||||
chunk = ciphertext[i : i + _BLOCK]
|
||||
ks = _keystream(key, nonce, i)[: len(chunk)]
|
||||
pt.extend(c ^ k for c, k in zip(chunk, ks))
|
||||
try:
|
||||
return _ctr(key, nonce, ciphertext).decode()
|
||||
except UnicodeDecodeError as exc: # pragma: no cover - authenticated, so unreachable
|
||||
raise ValueError(f"decryption produced non-UTF-8 output: {exc}") from exc
|
||||
return bytes(pt).decode()
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValueError(f"decryption produced non-UTF-8 output (wrong key?): {exc}") from exc
|
||||
|
||||
|
||||
__all__ = ["ENV_VAR_SECRET_NAME", "new_env_var_secret", "encrypt_value", "decrypt_value"]
|
||||
|
||||
@@ -36,13 +36,6 @@ ROLE_GATEWAY = "gateway"
|
||||
ROLE_CLI = "cli"
|
||||
ROLES: frozenset[str] = frozenset({ROLE_GATEWAY, ROLE_CLI})
|
||||
|
||||
# The host controller's own lifecycle role (#468). Deliberately OUTSIDE `ROLES`:
|
||||
# it belongs to a separate trust domain (`HOST_CONTROLLER`) signed by a key the
|
||||
# orchestrator never holds, so the orchestrator's control-plane key can neither
|
||||
# mint nor accept it — the orchestrator must not be able to forge the credential
|
||||
# used to start and stop it.
|
||||
ROLE_HOST = "host"
|
||||
|
||||
_ALG = "HS256"
|
||||
|
||||
|
||||
@@ -110,4 +103,4 @@ def verify(token: str, secret: str, *, roles: frozenset[str] = ROLES) -> str | N
|
||||
return role if isinstance(role, str) and role in roles else None
|
||||
|
||||
|
||||
__all__ = ["ROLE_GATEWAY", "ROLE_CLI", "ROLE_HOST", "ROLES", "mint", "verify"]
|
||||
__all__ = ["ROLE_GATEWAY", "ROLE_CLI", "ROLES", "mint", "verify"]
|
||||
|
||||
@@ -47,22 +47,6 @@ ORCHESTRATOR_TOKEN_ENV = "BOT_BOTTLE_ORCHESTRATOR_TOKEN"
|
||||
# cannot forge a higher-privilege `cli` token (issue #469 review).
|
||||
ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT"
|
||||
|
||||
# The durable launch-broker signing key: the HS256 secret the orchestrator
|
||||
# (signer) and the host control server (verifier) share to sign/verify launch
|
||||
# requests (#468). A host-canonical key file (minted 0600 on first use) so it
|
||||
# survives orchestrator restarts — re-adoption re-verifies against the same key —
|
||||
# instead of the ephemeral per-process secret of the in-process broker.
|
||||
LAUNCH_BROKER_KEY_FILENAME = "launch-broker-key"
|
||||
LAUNCH_BROKER_KEY_ENV = "BOT_BOTTLE_LAUNCH_BROKER_KEY"
|
||||
# The host controller's OWN key, for its lifecycle endpoints (the direct
|
||||
# cli -> host controller path that starts/stops the orchestrator). Separate from
|
||||
# the launch-broker key and never held by the orchestrator: the controller starts
|
||||
# and stops the orchestrator, so the orchestrator must not be able to mint the
|
||||
# credentials used to drive it (#468/#476).
|
||||
HOST_CONTROLLER_KEY_FILENAME = "host-controller-key"
|
||||
HOST_CONTROLLER_KEY_ENV = "BOT_BOTTLE_HOST_CONTROLLER_KEY"
|
||||
HOST_CONTROLLER_AUTH_JWT_ENV = "BOT_BOTTLE_HOST_CONTROLLER_AUTH_JWT"
|
||||
|
||||
# The host directory holding the gateway's persistent mitmproxy CA. Bind-mounted
|
||||
# into the infra/gateway container at mitmproxy's confdir so the self-generated
|
||||
# CA survives container recreation — every agent installs this one CA to trust
|
||||
@@ -158,11 +142,6 @@ __all__ = [
|
||||
"ORCHESTRATOR_TOKEN_FILENAME",
|
||||
"ORCHESTRATOR_TOKEN_ENV",
|
||||
"ORCHESTRATOR_AUTH_JWT_ENV",
|
||||
"LAUNCH_BROKER_KEY_FILENAME",
|
||||
"LAUNCH_BROKER_KEY_ENV",
|
||||
"HOST_CONTROLLER_KEY_FILENAME",
|
||||
"HOST_CONTROLLER_KEY_ENV",
|
||||
"HOST_CONTROLLER_AUTH_JWT_ENV",
|
||||
"GATEWAY_CA_DIRNAME",
|
||||
"bot_bottle_root",
|
||||
"host_db_path",
|
||||
|
||||
@@ -1,159 +0,0 @@
|
||||
"""Locate build-time resources whether bot-bottle runs from a source
|
||||
checkout or an installed wheel.
|
||||
|
||||
The gateway / infra / orchestrator images are built from a Docker (or Apple
|
||||
`container`) build context that must contain the `bot_bottle` package,
|
||||
`pyproject.toml`, and the root-level Dockerfiles as siblings. In a source
|
||||
checkout that context is simply the repo root, one level above the package.
|
||||
An installed wheel has no repo root: the same root-level files are shipped
|
||||
inside the package under ``bot_bottle/_resources/`` (see ``setup.py``), and a
|
||||
repo-root-shaped build context is staged on demand into the app-data dir.
|
||||
|
||||
``build_root()`` is the single source of truth — it returns a directory laid
|
||||
out like a repo root (has ``bot_bottle/``, ``pyproject.toml``, the
|
||||
Dockerfiles, ``nix/``, ``scripts/``). Every caller that needs a build
|
||||
context, a Dockerfile path, the nix netpool module, or the netpool script
|
||||
derives from it, so checkout and wheel installs share one downstream path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from .paths import bot_bottle_root
|
||||
|
||||
_PKG = Path(__file__).resolve().parent # …/bot_bottle
|
||||
_CHECKOUT_ROOT = _PKG.parent # repo root in a checkout
|
||||
_BUNDLED = _PKG / "_resources" # wheel-shipped copies
|
||||
|
||||
# Root-level files bundled into the wheel under ``_resources/`` (paths are
|
||||
# relative to the checkout root, and preserved verbatim under ``_resources/``
|
||||
# and in the staged build root). ``setup.py`` copies exactly this set; keep
|
||||
# the two lists in sync (``test_resources`` guards that every entry exists).
|
||||
BUNDLED_RESOURCES: tuple[str, ...] = (
|
||||
"pyproject.toml",
|
||||
"Dockerfile.gateway",
|
||||
"Dockerfile.orchestrator",
|
||||
"Dockerfile.orchestrator.fc",
|
||||
"nix/firecracker-netpool.nix",
|
||||
"scripts/firecracker-netpool.sh",
|
||||
)
|
||||
|
||||
# Present at a checkout root, never in a bare installed package — the cheap
|
||||
# tell for which layout we're in.
|
||||
_CHECKOUT_MARKER = "Dockerfile.gateway"
|
||||
|
||||
|
||||
class ResourceError(RuntimeError):
|
||||
"""Build resources are missing from the install (corrupt/partial wheel)."""
|
||||
|
||||
|
||||
def is_source_checkout() -> bool:
|
||||
"""True when running from a source tree (the root Dockerfiles sit beside
|
||||
the package); False from an installed wheel."""
|
||||
return (_CHECKOUT_ROOT / _CHECKOUT_MARKER).is_file()
|
||||
|
||||
|
||||
def build_root() -> Path:
|
||||
"""A directory shaped like a repo root: ``bot_bottle/``, ``pyproject.toml``,
|
||||
the root Dockerfiles, ``nix/``, and ``scripts/``.
|
||||
|
||||
A checkout returns the repo root itself (no copying). An installed wheel
|
||||
returns a staged copy under the app-data dir, materialized once and reused.
|
||||
The stage is keyed by a digest of the installed package + bundled resources
|
||||
(not the distribution version), so a force-reinstall of a newer commit that
|
||||
keeps ``version = 0.1.0`` still rebuilds instead of reusing a stale tree."""
|
||||
if is_source_checkout():
|
||||
return _CHECKOUT_ROOT
|
||||
return _stage_build_root()
|
||||
|
||||
|
||||
def dockerfile(name: str) -> Path:
|
||||
"""Absolute path to a root-level Dockerfile, e.g. ``Dockerfile.gateway``."""
|
||||
return build_root() / name
|
||||
|
||||
|
||||
def nix_netpool_module() -> Path:
|
||||
"""Absolute path to the firecracker netpool NixOS module."""
|
||||
return build_root() / "nix" / "firecracker-netpool.nix"
|
||||
|
||||
|
||||
def netpool_script() -> Path:
|
||||
"""Absolute path to the firecracker netpool bring-up script."""
|
||||
return build_root() / "scripts" / "firecracker-netpool.sh"
|
||||
|
||||
|
||||
def _content_digest() -> str:
|
||||
"""A 16-hex digest of the installed package + bundled resources.
|
||||
|
||||
Keys the staged build root by *content*, so a force-reinstall over the same
|
||||
version string (the installer defaults to a git branch + ``pipx install
|
||||
--force``, and ``version`` stays ``0.1.0``) yields a different key and
|
||||
re-stages, rather than reusing an old commit's tree. ``_PKG`` already
|
||||
contains ``_resources``, so walking it covers both."""
|
||||
h = hashlib.sha256()
|
||||
for path in sorted(_PKG.rglob("*")):
|
||||
if not path.is_file() or "__pycache__" in path.parts or path.suffix == ".pyc":
|
||||
continue
|
||||
h.update(str(path.relative_to(_PKG)).encode())
|
||||
h.update(b"\0")
|
||||
h.update(path.read_bytes())
|
||||
return h.hexdigest()[:16]
|
||||
|
||||
|
||||
def _stage_build_root() -> Path:
|
||||
"""Materialize a repo-root-shaped build context from the installed wheel's
|
||||
bundled resources, keyed by content digest. Idempotent and concurrency-safe:
|
||||
a file lock serializes staging, a partial/stale tree is replaced, and the
|
||||
finished tree is published with an atomic rename."""
|
||||
if not _BUNDLED.is_dir():
|
||||
raise ResourceError(
|
||||
"bot-bottle build resources are missing from this install "
|
||||
f"(expected {_BUNDLED}). Reinstall the package."
|
||||
)
|
||||
base = bot_bottle_root() / "build-root"
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
dest = base / _content_digest()
|
||||
if (dest / ".complete").is_file():
|
||||
return dest
|
||||
|
||||
# Serialize staging across processes: a concurrent `start` after an install
|
||||
# must not race on the shared tree. The lock is held only around stage +
|
||||
# atomic publish; the fast path above never blocks.
|
||||
with open(base / ".stage.lock", "w", encoding="utf-8") as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||||
if (dest / ".complete").is_file(): # another process staged while we waited
|
||||
return dest
|
||||
# Stage into a private temp dir on the same filesystem, then publish by
|
||||
# rename — never populate a shared path other processes might read.
|
||||
staging = Path(tempfile.mkdtemp(prefix=".staging-", dir=base))
|
||||
try:
|
||||
# The package itself, minus caches and the bundled-resource copies,
|
||||
# so the staged ``bot_bottle/`` matches a checkout's (keeps the
|
||||
# firecracker infra-artifact hash stable across checkout and wheel).
|
||||
shutil.copytree(
|
||||
_PKG,
|
||||
staging / "bot_bottle",
|
||||
ignore=shutil.ignore_patterns("__pycache__", "*.pyc", "_resources"),
|
||||
)
|
||||
# The bundled root files, restored to their checkout-relative layout.
|
||||
for rel in BUNDLED_RESOURCES:
|
||||
dst = staging / rel
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(_BUNDLED / rel, dst)
|
||||
(staging / ".complete").write_text("")
|
||||
# Replace any partial leftover for this digest (safe: we hold the
|
||||
# lock), then publish atomically.
|
||||
if dest.exists():
|
||||
shutil.rmtree(dest)
|
||||
os.replace(staging, dest)
|
||||
staging = None # published; nothing to clean up
|
||||
finally:
|
||||
if staging is not None:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
return dest
|
||||
@@ -17,7 +17,7 @@ instead would defeat that — the orchestrator holds that key, so it could forge
|
||||
`ControlPlaneProvisioning` is the one seam every backend launcher uses to get the
|
||||
orchestrator its key and the gateway its token, instead of re-deriving that
|
||||
wiring per backend (the bug class behind PR #471 — see
|
||||
`docs/prds/0079-control-plane-auth-provisioning.md`).
|
||||
`docs/prds/prd-new-control-plane-auth-provisioning.md`).
|
||||
|
||||
Stdlib-only: the HMAC lives in `orchestrator_auth`, the key file in `paths`.
|
||||
"""
|
||||
@@ -29,13 +29,8 @@ from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
|
||||
from . import orchestrator_auth
|
||||
from .orchestrator_auth import ROLE_GATEWAY, ROLE_HOST
|
||||
from .orchestrator_auth import ROLE_GATEWAY
|
||||
from .paths import (
|
||||
HOST_CONTROLLER_AUTH_JWT_ENV,
|
||||
HOST_CONTROLLER_KEY_ENV,
|
||||
HOST_CONTROLLER_KEY_FILENAME,
|
||||
LAUNCH_BROKER_KEY_ENV,
|
||||
LAUNCH_BROKER_KEY_FILENAME,
|
||||
ORCHESTRATOR_AUTH_JWT_ENV,
|
||||
ORCHESTRATOR_TOKEN_ENV,
|
||||
ORCHESTRATOR_TOKEN_FILENAME,
|
||||
@@ -104,40 +99,6 @@ CONTROL_PLANE = TrustDomain(
|
||||
)
|
||||
|
||||
|
||||
# The launch-broker domain (#468): durable key material for the broker's own
|
||||
# signed launch requests (`broker.py`'s HS256 launch JWT), shared by the
|
||||
# orchestrator (signer) and the host control server (verifier). Unlike
|
||||
# `CONTROL_PLANE` it mints no role tokens — the broker's provenance is the launch
|
||||
# JWT, not a role token — so its `roles` set is empty and it is used only as a
|
||||
# provider of durable, host-canonical key material (`signing_key` / `key_from_env`).
|
||||
# The durability is the point: the key survives orchestrator restarts, so a
|
||||
# restarted orchestrator re-verifies against the same key instead of the
|
||||
# ephemeral per-process secret the in-process broker used.
|
||||
LAUNCH_BROKER = TrustDomain(
|
||||
name="launch-broker",
|
||||
key_filename=LAUNCH_BROKER_KEY_FILENAME,
|
||||
roles=frozenset(),
|
||||
key_env=LAUNCH_BROKER_KEY_ENV,
|
||||
token_env="",
|
||||
)
|
||||
|
||||
# The host controller's own domain (#468) — the SECOND domain #476 reserves. Its
|
||||
# key, which the orchestrator never holds, signs the `host`-role tokens the CLI
|
||||
# presents on the host controller's lifecycle endpoints (start / restart / status
|
||||
# of the orchestrator itself). Keeping it separate from `CONTROL_PLANE` is the
|
||||
# whole point: the host controller starts and stops the orchestrator, so the
|
||||
# orchestrator must not be able to mint the credentials used to drive it. (The
|
||||
# lifecycle endpoints themselves arrive in a later chunk; the domain is
|
||||
# established here alongside the durable launch-broker key.)
|
||||
HOST_CONTROLLER = TrustDomain(
|
||||
name="host-controller",
|
||||
key_filename=HOST_CONTROLLER_KEY_FILENAME,
|
||||
roles=frozenset({ROLE_HOST}),
|
||||
key_env=HOST_CONTROLLER_KEY_ENV,
|
||||
token_env=HOST_CONTROLLER_AUTH_JWT_ENV,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ControlPlaneProvisioning:
|
||||
"""The one seam every backend launcher uses to provision control-plane auth,
|
||||
@@ -171,56 +132,9 @@ class ControlPlaneProvisioning:
|
||||
return self.domain.mint(ROLE_GATEWAY)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LaunchBrokerProvisioning:
|
||||
"""The seam that provisions the host-side launch broker's durable keys (#468),
|
||||
the counterpart to `ControlPlaneProvisioning`. Both the orchestrator (signer)
|
||||
and the host control server (verifier) receive the SAME launch-broker key
|
||||
(carry it in `broker_domain.key_env`); the host controller ALSO receives its
|
||||
own lifecycle key (`controller_domain.key_env`) the orchestrator never holds.
|
||||
|
||||
Fail-closed like the control-plane seam: minting returns "" only if the host
|
||||
root is unwritable, and an empty launch-broker key would leave the verifier
|
||||
unable to authenticate any launch — so we raise rather than hand back a key
|
||||
that would make the host controller reject (or, if a caller defaulted it,
|
||||
accept) unsigned input."""
|
||||
|
||||
broker_domain: TrustDomain = LAUNCH_BROKER
|
||||
controller_domain: TrustDomain = HOST_CONTROLLER
|
||||
|
||||
def broker_key(self) -> str:
|
||||
"""The durable launch-broker key both the orchestrator and the host
|
||||
control server must receive (in `broker_domain.key_env`). Raises rather
|
||||
than return ""."""
|
||||
key = self.broker_domain.signing_key()
|
||||
if not key:
|
||||
raise ProvisioningError(
|
||||
f"refusing to provision the {self.broker_domain.name} broker "
|
||||
"without a signing key: the host controller could then verify no "
|
||||
"launch request's provenance"
|
||||
)
|
||||
return key
|
||||
|
||||
def controller_key(self) -> str:
|
||||
"""The host controller's own lifecycle key — provisioned ONLY to the host
|
||||
controller (in `controller_domain.key_env`), never to the orchestrator, so
|
||||
the orchestrator cannot mint the `host`-role tokens that start and stop
|
||||
it. Raises rather than return ""."""
|
||||
key = self.controller_domain.signing_key()
|
||||
if not key:
|
||||
raise ProvisioningError(
|
||||
f"refusing to provision the {self.controller_domain.name} without "
|
||||
"a signing key: its lifecycle endpoints would authenticate no one"
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ProvisioningError",
|
||||
"TrustDomain",
|
||||
"CONTROL_PLANE",
|
||||
"LAUNCH_BROKER",
|
||||
"HOST_CONTROLLER",
|
||||
"ControlPlaneProvisioning",
|
||||
"LaunchBrokerProvisioning",
|
||||
]
|
||||
|
||||
@@ -7,7 +7,6 @@ picking the right document for what you're capturing.
|
||||
|
||||
| Artifact | For |
|
||||
|---|---|
|
||||
| **Design workflow** (`docs/design-workflow.md`) | How discussion becomes canonical design, how dependencies are recorded, and when implementation may begin. |
|
||||
| **Glossary** (`docs/glossary.md`) | Canonical term definitions — what words mean in this project. |
|
||||
| **PRD** (`docs/prds/`) | A feature: what to build, scope, success criteria. |
|
||||
| **Research note** (`docs/research/`) | A landscape/tradeoff investigation. |
|
||||
|
||||
+1
-12
@@ -2,22 +2,11 @@
|
||||
|
||||
The test workflow lives at [`.gitea/workflows/test.yml`](../.gitea/workflows/test.yml).
|
||||
It runs the unit suite plus one integration job per backend
|
||||
(`integration-docker`, `integration-firecracker`, `integration-macos`) on:
|
||||
(`integration-docker`, `integration-firecracker`) on:
|
||||
|
||||
- every push to a branch with an open pull request, and
|
||||
- every push to `main`.
|
||||
|
||||
`integration-macos` is the exception: it is **advisory**, running only on
|
||||
`workflow_dispatch` (manual dispatch), never on push or pull requests. It targets the
|
||||
Apple Container backend on a self-hosted macOS runner (label `macos`,
|
||||
registered in host mode — Apple Container can't run in a Linux container, so it
|
||||
can't reuse the `kvm` runner). A single non-redundant laptop must not be able
|
||||
to block a PR merge, so the job stays out of the `coverage` job's `needs` and
|
||||
its coverage never feeds the diff-coverage gate. Because the infra container is
|
||||
a singleton (`bot-bottle-mac-infra`), the job declares a `concurrency` group
|
||||
and tears the container down on exit; keep runner concurrency at 1. See the
|
||||
README "macOS Apple Container" CI note for runner provisioning.
|
||||
|
||||
Each integration job selects its backend via `BOT_BOTTLE_BACKEND` and
|
||||
runs a **preflight** (`./cli.py backend status --backend=<name>`) that
|
||||
prints a clear per-check readiness summary and fails the job when the
|
||||
|
||||
@@ -1,13 +1,9 @@
|
||||
# ADR 0005: Keep tracker metadata on one tracker object
|
||||
# ADR 0005: Keep tracker metadata on issues
|
||||
|
||||
- **Status:** Accepted
|
||||
- **Date:** 2026-07-18
|
||||
- **Deciders:** didericis
|
||||
|
||||
> **Amended 2026-07-26.** A pull request may carry labels directly instead of
|
||||
> linking a tracking issue. When a PR does link an issue, the issue remains the
|
||||
> canonical owner of planning metadata and the reference is validated.
|
||||
|
||||
## Context
|
||||
|
||||
Gitea exposes labels on both issues and pull requests. Applying the same labels
|
||||
@@ -24,29 +20,19 @@ would make the issue history less truthful.
|
||||
|
||||
## Decision
|
||||
|
||||
Issues are the canonical tracker records and own labels when a separate work
|
||||
item exists. Every issue has at least one label. An issue opened or left
|
||||
without labels receives `Status/Needs Triage` automatically until it is
|
||||
classified.
|
||||
Issues are the canonical tracker records and own labels. Every issue has at
|
||||
least one label. An issue opened or left without labels receives
|
||||
`Status/Needs Triage` automatically until it is classified.
|
||||
|
||||
Every new pull request is tracked in exactly one of two mutually exclusive
|
||||
ways:
|
||||
|
||||
1. It deliberately references at least one existing issue in its title or
|
||||
description. Tracker metadata stays on that issue and the PR remains
|
||||
unlabelled.
|
||||
2. It carries at least one label directly when a separate issue would add no
|
||||
useful planning context.
|
||||
|
||||
Issue references use one of these forms:
|
||||
Pull requests carry no labels. Every new PR deliberately references at least
|
||||
one existing issue in its title or description with one of these forms:
|
||||
|
||||
- `Closes #123`, `Fixes #123`, or `Resolves #123` when merging completes it.
|
||||
- `Part of #123`, `Related to #123`, `Refs #123`, or `References #123` when it
|
||||
contributes without completing it.
|
||||
|
||||
Gitea Actions enforces the exclusive either/or PR rule, validates any issue
|
||||
references, and repairs the empty issue-label state. Branch protection makes
|
||||
the PR policy check required.
|
||||
Gitea Actions enforces both PR rules as a status check and repairs the empty
|
||||
issue-label state. Branch protection makes the PR policy check required.
|
||||
|
||||
The policy applies from 2026-07-18 onward. Existing issues may be labelled as
|
||||
they are encountered, but closed PRs are grandfathered: no retrospective
|
||||
@@ -54,13 +40,9 @@ issues or PR labels are created solely to make history conform.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Classification, priority, and workflow metadata have one source of truth for
|
||||
each change: the linked issue when one exists, otherwise the PR.
|
||||
- For issue-backed changes, the PR's issue link is the navigation path to its
|
||||
planning metadata.
|
||||
- Classification, priority, and workflow metadata have one source of truth.
|
||||
- A PR's issue link is the navigation path to its planning metadata.
|
||||
- Multi-PR issues do not require copied or synchronized labels.
|
||||
- Small standalone changes do not require a tracking issue created solely to
|
||||
satisfy automation.
|
||||
- `Status/Needs Triage` is an intentional fallback, not a final
|
||||
classification.
|
||||
- Direct issue creation remains convenient; automation repairs a missing label
|
||||
@@ -71,6 +53,5 @@ issues or PR labels are created solely to make history conform.
|
||||
## Links
|
||||
|
||||
- Issue #405.
|
||||
- `.gitea/workflows/tracker-policy-pr.yml`.
|
||||
- `.gitea/workflows/tracker-policy-issues.yml`.
|
||||
- `.gitea/workflows/tracker-policy.yml`.
|
||||
- `scripts/tracker_policy.py`.
|
||||
|
||||
@@ -1,218 +0,0 @@
|
||||
# Design workflow
|
||||
|
||||
How bot-bottle turns discussion into canonical design and then into
|
||||
implementation without leaving the repository's architecture scattered across
|
||||
issue and review threads.
|
||||
|
||||
The goal is not more documentation. The goal is one discoverable current answer
|
||||
for every load-bearing design question.
|
||||
|
||||
## Sources of truth
|
||||
|
||||
Design artifacts have different jobs:
|
||||
|
||||
| Artifact | Authority |
|
||||
|---|---|
|
||||
| Decision records | Stable system-wide boundaries, policies, and invariants |
|
||||
| PRDs | The current design for a feature |
|
||||
| Research notes | Evidence and tradeoff analysis; informative, not normative |
|
||||
| Issues | Work tracking, open questions, and discussion |
|
||||
| Pull-request comments | Review history; never the final home of a design decision |
|
||||
|
||||
When a discussion changes the design, update the relevant PRD or decision
|
||||
record before treating the discussion as resolved. A comment may explain why a
|
||||
decision changed, but future implementers must not need to reconstruct the
|
||||
decision from a thread.
|
||||
|
||||
Avoid duplicating the same rule in several canonical documents. Prefer one
|
||||
canonical statement and links from dependent documents.
|
||||
|
||||
## Choosing the canonical artifact
|
||||
|
||||
Use a PRD when the decision describes a feature: its behavior, scope, success
|
||||
criteria, trust model, implementation slices, and tests.
|
||||
|
||||
Use a decision record when the choice is broader than one feature or will
|
||||
constrain several future features. Examples include state ownership, credential
|
||||
boundaries, compatibility policy, and what the project does or does not claim
|
||||
as a security guarantee.
|
||||
|
||||
Use a research note when the conclusion depends on comparing external systems,
|
||||
protocols, or approaches. Promote any resulting project decision into a PRD or
|
||||
decision record.
|
||||
|
||||
## From discussion to implementation
|
||||
|
||||
### 1. Open the design discussion
|
||||
|
||||
An issue may start with incomplete requirements. Record:
|
||||
|
||||
- the problem and desired outcome;
|
||||
- known security or compatibility constraints;
|
||||
- the current owner of affected state and credentials;
|
||||
- related PRDs, decisions, issues, and pull requests;
|
||||
- open questions that would materially change the implementation.
|
||||
|
||||
Do not disguise an unresolved trust-boundary or state-ownership decision as an
|
||||
implementation detail.
|
||||
|
||||
### 2. Draft or update the canonical design
|
||||
|
||||
Before substantial implementation, write the feature PRD and update any
|
||||
system-wide decision it changes.
|
||||
|
||||
An active design should make these relationships visible near its top:
|
||||
|
||||
```markdown
|
||||
Status: Draft | Active | Superseded | Retargeted
|
||||
Depends on: #...
|
||||
Supersedes: ...
|
||||
```
|
||||
|
||||
Record dependencies only on the dependent document. Do not maintain reverse
|
||||
`Blocks` lists that can drift as dependent work changes.
|
||||
|
||||
For security-sensitive work, state:
|
||||
|
||||
- the exact guarantee and explicit non-guarantees;
|
||||
- trusted and untrusted components;
|
||||
- who creates each identity or attribution field;
|
||||
- who owns durable state;
|
||||
- failure and recovery behavior;
|
||||
- how the design is tested at its boundaries.
|
||||
|
||||
### 3. Resolve review into the repository
|
||||
|
||||
When review settles a design-changing question:
|
||||
|
||||
1. Update the canonical document in the same pull request.
|
||||
2. Mark conflicting documents Superseded or Retargeted, or update them.
|
||||
3. Add or adjust dependency links.
|
||||
4. Leave a concise resolution comment linking to the canonical change.
|
||||
|
||||
A useful resolution comment is:
|
||||
|
||||
```text
|
||||
Resolution: <what was decided>
|
||||
Canonicalized in: <document/section/commit>
|
||||
Supersedes: <older statement, if any>
|
||||
Follow-up: <remaining implementation or question>
|
||||
```
|
||||
|
||||
The resolution is incomplete until the repository reflects it.
|
||||
|
||||
### 4. Check design readiness
|
||||
|
||||
Implementation may begin when:
|
||||
|
||||
- the PRD's material trust, ownership, and compatibility questions are settled;
|
||||
- dependencies and blockers are explicit;
|
||||
- the design agrees with current architecture and decision records;
|
||||
- superseded documents are marked or updated;
|
||||
- success criteria and boundary tests are concrete;
|
||||
- remaining open questions can be answered during implementation without
|
||||
changing the feature's guarantee or component ownership.
|
||||
|
||||
Small exploratory spikes may happen earlier. A spike proves feasibility; it does
|
||||
not establish a production contract or silently settle the design.
|
||||
|
||||
### 5. Implement in ordered slices
|
||||
|
||||
Prefer small, independently reviewable slices after the parent design is
|
||||
accepted. Record the dependency chain explicitly.
|
||||
|
||||
Parallel work is safe when slices do not compete for the same unsettled
|
||||
interface or ownership boundary. If a foundational change will alter the
|
||||
transport, schema, state owner, or trust domain used by another slice, land the
|
||||
foundation first.
|
||||
|
||||
An implementation pull request should identify:
|
||||
|
||||
- the PRD or decision it implements;
|
||||
- the implementation chunk;
|
||||
- its base and blockers;
|
||||
- any design deviation discovered during implementation.
|
||||
|
||||
If implementation reveals a load-bearing design change, pause that slice and
|
||||
update the canonical design. Do not let the code and review thread become an
|
||||
undocumented replacement for the PRD.
|
||||
|
||||
## Dependency and staleness management
|
||||
|
||||
### Dependency direction
|
||||
|
||||
Write dependencies in terms of contracts, not chronology:
|
||||
|
||||
```text
|
||||
credential provisioning contract
|
||||
-> host-controller authentication
|
||||
-> privileged host operations
|
||||
```
|
||||
|
||||
If only part of a feature is blocked, say so. For example, a manifest parser may
|
||||
proceed while that feature's durable audit-storage chunk waits for the canonical
|
||||
audit schema.
|
||||
|
||||
### Superseding documents
|
||||
|
||||
Do not silently edit history to make an old design appear to have always said
|
||||
the new thing. Preserve the rationale, but make current status unmistakable:
|
||||
|
||||
```markdown
|
||||
Status: Superseded
|
||||
Superseded by: <document>
|
||||
Reason: <one paragraph>
|
||||
```
|
||||
|
||||
If part of a PRD remains valid, mark it Retargeted and identify which scope moved
|
||||
elsewhere.
|
||||
|
||||
Add a short supersession note near the top explaining what changed, why the old
|
||||
design is no longer current, and where the current design lives. For a research
|
||||
note whose original analysis remains useful, preserve that analysis and append
|
||||
a dated addendum with the newer finding instead of rewriting the note as though
|
||||
it had always reached the new conclusion.
|
||||
|
||||
### Architecture sweeps
|
||||
|
||||
After a foundational change, do a targeted architecture sweep before building
|
||||
more features on it:
|
||||
|
||||
1. Identify the concepts the change affects, such as `bot-bottle.db`, host
|
||||
controller, orchestrator, audit ownership, or signing key.
|
||||
2. Search active PRDs, decisions, and open issues for those concepts.
|
||||
3. Update or supersede contradictory statements.
|
||||
4. Refresh dependency links and the current architecture summary.
|
||||
5. Confirm stacked implementation branches still have the correct base.
|
||||
|
||||
This is a milestone activity, not a recurring documentation ceremony.
|
||||
|
||||
## Pull-request checklist
|
||||
|
||||
Use the relevant items in design and implementation pull requests:
|
||||
|
||||
- [ ] The canonical PRD or decision is linked.
|
||||
- [ ] Design-changing review decisions are reflected in-repo.
|
||||
- [ ] Dependencies and blockers are explicit.
|
||||
- [ ] State, credential, and trust ownership agree with current architecture.
|
||||
- [ ] Superseded or retargeted documents are marked.
|
||||
- [ ] Security guarantees and non-guarantees are precise.
|
||||
- [ ] Open questions do not change the promised guarantee or ownership model.
|
||||
- [ ] Implementation deviations updated the canonical design.
|
||||
|
||||
## Lightweight maintenance
|
||||
|
||||
Automation should enforce document shape, not pretend to understand
|
||||
architecture. Useful checks include:
|
||||
|
||||
- active PRDs contain status and dependency metadata;
|
||||
- superseded PRDs link to their replacement;
|
||||
- referenced documents and issues exist;
|
||||
- implementation pull requests identify their PRD and chunk;
|
||||
- document filenames and lifecycle states follow repository conventions.
|
||||
|
||||
Human review remains responsible for detecting conflicting guarantees or
|
||||
ownership claims.
|
||||
|
||||
The durable rule is simple: **discussion discovers the decision; the repository
|
||||
records it; implementation follows it.**
|
||||
@@ -1,14 +1,9 @@
|
||||
# PRD 0001: Per-agent egress proxy via pipelock
|
||||
|
||||
- **Status:** Superseded by [PRD 0017](0017-egress-proxy-via-mitmproxy.md)
|
||||
and [PRD 0052](0052-egress-dlp-addon.md)
|
||||
- **Status:** Active
|
||||
- **Author:** didericis
|
||||
- **Created:** 2026-05-08
|
||||
|
||||
> **Superseded.** Pipelock was removed in issue #193. PRD 0017 moved
|
||||
> egress enforcement and credential injection to mitmproxy; PRD 0052 moved DLP
|
||||
> enforcement into the egress addon. The design below is retained as history.
|
||||
|
||||
## Summary
|
||||
|
||||
Run pipelock as a sidecar container on each bot-bottle agent's only
|
||||
|
||||
@@ -1,14 +1,9 @@
|
||||
# PRD 0006: pipelock native TLS interception
|
||||
|
||||
- **Status:** Superseded by [PRD 0017](0017-egress-proxy-via-mitmproxy.md)
|
||||
and [PRD 0052](0052-egress-dlp-addon.md)
|
||||
- **Status:** Active
|
||||
- **Author:** didericis
|
||||
- **Created:** 2026-05-12
|
||||
|
||||
> **Superseded.** Pipelock was removed in issue #193. TLS interception now
|
||||
> belongs to the mitmproxy egress design in PRD 0017, with DLP implemented by
|
||||
> the egress addon in PRD 0052. The design below is retained as history.
|
||||
|
||||
## Summary
|
||||
|
||||
Turn on pipelock's built-in `tls_interception` so its DLP / URL /
|
||||
|
||||
@@ -1,17 +1,11 @@
|
||||
# PRD 0015: pipelock block remediation
|
||||
|
||||
- **Status:** Superseded by [PRD 0017](0017-egress-proxy-via-mitmproxy.md)
|
||||
and [PRD 0052](0052-egress-dlp-addon.md)
|
||||
- **Status:** Active
|
||||
- **Author:** didericis
|
||||
- **Created:** 2026-05-25
|
||||
- **Parent:** PRD 0012
|
||||
- **Depends on:** PRD 0013
|
||||
|
||||
> **Superseded.** Pipelock and its restart-based allowlist remediation path
|
||||
> were removed in issue #193. Current egress enforcement is the mitmproxy
|
||||
> design from PRD 0017 with DLP in PRD 0052. The design below is retained as
|
||||
> history.
|
||||
|
||||
## Summary
|
||||
|
||||
Wires the **pipelock block** path (PRD 0012 *Stuck categories*) end-to-end. The supervisor, on approval of a `pipelock-block` proposal, writes the new pipelock allowlist to the host and restarts pipelock; the agent's in-flight outbound calls may drop and rely on retry. The TUI gains a proactive `pipelock edit <bottle>` verb for operator-initiated edits unrelated to a tool call. The pipelock audit log (format defined in PRD 0013) is filled in with real entries on every edit.
|
||||
|
||||
@@ -4,11 +4,6 @@
|
||||
- **Author:** didericis
|
||||
- **Created:** 2026-05-26
|
||||
|
||||
> **Superseded.** PRD 0049 removed the active-agents pane and agent-scoped
|
||||
> operator edit verbs when the dashboard was narrowed back to a proposal-only
|
||||
> supervise TUI. A future agent-management surface was deferred rather than
|
||||
> carried forward from this design. The design below is retained as history.
|
||||
|
||||
## Summary
|
||||
|
||||
The dashboard today is proposal-centric: it lists every pending
|
||||
|
||||
@@ -4,11 +4,6 @@
|
||||
- **Author:** didericis
|
||||
- **Created:** 2026-05-26
|
||||
|
||||
> **Superseded.** PRD 0049 removed start, re-attach, and stop actions from the
|
||||
> dashboard when it became the proposal-only supervise TUI. Bottle lifecycle
|
||||
> remains in the dedicated CLI commands; no dashboard replacement from this
|
||||
> design remains active. The design below is retained as history.
|
||||
|
||||
## Summary
|
||||
|
||||
Today the dashboard is read-only: it surfaces pending proposals
|
||||
|
||||
@@ -4,11 +4,6 @@
|
||||
- **Author:** didericis
|
||||
- **Created:** 2026-05-26
|
||||
|
||||
> **Superseded.** PRD 0049 removed agent handoff and tmux pane management when
|
||||
> the dashboard was reduced to the proposal-only supervise TUI. The split-pane
|
||||
> interaction described below has no active replacement and is retained only
|
||||
> as design history.
|
||||
|
||||
## Summary
|
||||
|
||||
When the dashboard runs inside tmux, lay it out as the **left
|
||||
|
||||
@@ -1,13 +1,9 @@
|
||||
# PRD 0024: Consolidate per-bottle sidecars into a single bundle
|
||||
|
||||
- **Status:** Superseded by [PRD 0070](0070-per-host-orchestrator.md)
|
||||
- **Status:** Active
|
||||
- **Author:** didericis
|
||||
- **Created:** 2026-05-26
|
||||
|
||||
> **Superseded.** PRD 0070 replaced the per-bottle sidecar bundle with a
|
||||
> persistent per-host gateway and separate orchestrator control plane. The
|
||||
> design below is retained as history.
|
||||
|
||||
## Summary
|
||||
|
||||
Replace the four per-bottle sidecar containers in the Docker
|
||||
|
||||
@@ -1,16 +1,10 @@
|
||||
# PRD 0037: Pipelock YAML Render Contract
|
||||
|
||||
- **Status:** Superseded by [PRD 0017](0017-egress-proxy-via-mitmproxy.md)
|
||||
and [PRD 0052](0052-egress-dlp-addon.md)
|
||||
- **Status:** Active
|
||||
- **Author:** didericis-codex
|
||||
- **Created:** 2026-06-02
|
||||
- **Issue:** #130
|
||||
|
||||
> **Superseded.** Pipelock and its YAML renderer were removed in issue #193.
|
||||
> Current egress configuration is consumed by the mitmproxy design from PRD
|
||||
> 0017 and its DLP addon from PRD 0052. The contract below is retained as
|
||||
> history.
|
||||
|
||||
## Summary
|
||||
|
||||
Lock down the contract between `pipelock_build_config` and
|
||||
|
||||
@@ -1,17 +1,10 @@
|
||||
# PRD 0067: SQLite local storage
|
||||
|
||||
- **Status:** Retargeted by [PRD 0070](0070-per-host-orchestrator.md) and
|
||||
issues #469/#471
|
||||
- **Status:** Active
|
||||
- **Author:** codex
|
||||
- **Created:** 2026-07-01
|
||||
- **Issue:** #319
|
||||
|
||||
> **Retargeted.** The SQLite storage and migration foundation remains in use,
|
||||
> but the writable data-plane database mount described below is no longer the
|
||||
> active ownership model. Issues #469/#471 removed `bot-bottle.db` from the
|
||||
> data plane; under PRD 0070 only the orchestrator control plane opens the
|
||||
> operational database, and gateway components reach state through RPC.
|
||||
|
||||
## Summary
|
||||
|
||||
Add a small stdlib SQLite storage layer for bot-bottle host runtime state,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# PRD 0069: Firecracker-native, Docker-free backend
|
||||
|
||||
- **Status:** Retargeted by [PRD 0070](0070-per-host-orchestrator.md)
|
||||
- **Status:** Draft (partially superseded)
|
||||
- **Author:** Claude
|
||||
- **Created:** 2026-07-12
|
||||
- **Issue:** #348
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# PRD 0070: Per-host orchestrator service
|
||||
|
||||
- **Status:** Active
|
||||
- **Status:** Draft
|
||||
- **Author:** Claude
|
||||
- **Created:** 2026-07-12
|
||||
- **Issue:** #351
|
||||
|
||||
@@ -1,142 +0,0 @@
|
||||
# PRD 0077: macOS (Apple Container) CI runner
|
||||
|
||||
- **Status:** Active
|
||||
- **Author:** Claude
|
||||
- **Created:** 2026-07-25
|
||||
- **Issue:** #426
|
||||
|
||||
## Summary
|
||||
|
||||
CI has no runner for the `macos-container` (Apple Container) backend.
|
||||
`.gitea/workflows/test.yml` exercises Docker (`ubuntu-latest`) and
|
||||
Firecracker (self-hosted `kvm`) but never the macOS backend. This PRD adds a
|
||||
self-hosted macOS runner (label `macos`) and an advisory `integration-macos`
|
||||
job that runs the integration suite against `BOT_BOTTLE_BACKEND=macos-container`,
|
||||
so the backend that is the default on macOS stops shipping unexercised.
|
||||
|
||||
## Problem
|
||||
|
||||
The gap is not theoretical. `5ad3449` moved `bot_bottle` from flat files under
|
||||
`/app` into a pip-installed package but left init scripts spawning the
|
||||
supervisor as `python3 /app/gateway_init.py`, which no longer exists. Both the
|
||||
Firecracker and macOS backends carried the identical bug:
|
||||
|
||||
- **firecracker** — caught and fixed in `127ba49` because the KVM runner
|
||||
(added in `c193b04`, PR #349) runs that backend's integration suite.
|
||||
- **macos-container** — survived on `main` and only surfaced when a human ran
|
||||
`bot-bottle start` by hand.
|
||||
|
||||
The failure mode is expensive to debug: the supervisor never starts, so
|
||||
mitmdump never generates its CA, and launch dies downstream with
|
||||
`GatewayError: gateway CA not available`, which points at TLS rather than at
|
||||
the supervisor. Unit tests did not help — `test_macos_infra` asserted the
|
||||
substring `"gateway_init.py"`, which the *broken* path satisfies. (That
|
||||
specific assertion has since been tightened to the module form
|
||||
`bot_bottle.gateway_init`, matching its Firecracker twin, so the exact
|
||||
regression is now covered on `ubuntu-latest`. What remains missing is the
|
||||
end-to-end runner that would catch the *next* macOS-only launch regression.)
|
||||
|
||||
PR #470 (#414) already made the integration suite backend-agnostic:
|
||||
`skip_unless_selected_backend_available()` gates on the *selected* backend's
|
||||
own `is_backend_ready()` rather than `docker_available()`, and each
|
||||
integration job runs `./cli.py backend status --backend=<name>` as a preflight
|
||||
that fails loudly when the backend is missing. That is the machinery this job
|
||||
plugs into; this PRD supplies the runner and the job.
|
||||
|
||||
## Goals / Success criteria
|
||||
|
||||
- A macOS runner is registered and picks up jobs by the `macos` label.
|
||||
- An `integration-macos` job runs the integration suite against
|
||||
`BOT_BOTTLE_BACKEND=macos-container`.
|
||||
- The job **fails, not skips**, when the backend is unavailable on the runner
|
||||
(via the `backend status` preflight).
|
||||
- Reverting the `macos_container/infra.py` supervisor fix makes the job fail:
|
||||
the broken supervisor path throws `GatewayError` at bottle launch, which is
|
||||
`TestSandboxEscape.setUpClass`, failing the whole class before any individual
|
||||
attack runs.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Making `integration-macos` a **required** PR check. It runs on
|
||||
`workflow_dispatch` (manual dispatch) only — never on push or PRs. A single
|
||||
non-redundant laptop that sleeps and roams must never be able to block a PR
|
||||
merge or churn unattended on every push to main, and it is deliberately kept
|
||||
out of the `coverage` job's `needs` so the diff-coverage gate never depends on
|
||||
it.
|
||||
- Multi-machine or hosted macOS runners. Apple Container needs the host
|
||||
virtualization framework, so the runner must be a physical/VM macOS host on
|
||||
Apple Silicon — it cannot reuse the KVM runner or run in a Linux container.
|
||||
- Coverage aggregation from the macOS job into the combined gate (would couple
|
||||
the gate to the laptop).
|
||||
|
||||
## Design
|
||||
|
||||
### Runner (operational, provisioned once)
|
||||
|
||||
- Apple Silicon macOS host with Apple's `container` CLI installed and
|
||||
`container system status` reporting `running`.
|
||||
- Install the runner: `brew install gitea-runner` (the `act_runner` rename),
|
||||
registered in **host mode** with label `macos` — not docker mode, because
|
||||
Apple Container needs the host `container` CLI and virtualization framework,
|
||||
not a nested container.
|
||||
- A Python ≥ 3.11 with `coverage` importable on the runner's `PATH`. Because a
|
||||
launchd service does not inherit an interactive shell's `PATH`, pin `node`
|
||||
(for the JS `actions/*`) and the Python env explicitly in the service
|
||||
environment rather than relying on `nvm`/shell profile.
|
||||
- Concurrency 1. The infra container is a singleton (`bot-bottle-mac-infra`),
|
||||
so two simultaneous runs on one host collide (#425). The job also declares a
|
||||
`concurrency` group as belt-and-suspenders and tears the singleton down after
|
||||
each run.
|
||||
|
||||
### `integration-macos` job
|
||||
|
||||
Modeled on `integration-firecracker`:
|
||||
|
||||
- `runs-on: [self-hosted, macos]`.
|
||||
- `if:` `workflow_dispatch` only (advisory, manual dispatch; never push or PRs,
|
||||
so no fork-PR exposure, no merge-blocking, and no unattended runs on push).
|
||||
- `concurrency: { group: integration-macos-infra, cancel-in-progress: false }`
|
||||
to serialize runs against the singleton.
|
||||
- **Preflight** — `command -v container`, `container system status`, then
|
||||
`./cli.py backend status --backend=macos-container`; any failure exits
|
||||
non-zero so a misprovisioned runner fails loudly instead of silently
|
||||
skipping.
|
||||
- Run the integration suite under coverage with
|
||||
`BOT_BOTTLE_BACKEND=macos-container` and print a `coverage report -m` for
|
||||
visibility (no upload, not in the gate).
|
||||
- **Teardown** (`if: always()`) — `MacosInfraService().stop()` removes the
|
||||
singleton so a crashed run cannot wedge the next one.
|
||||
|
||||
### The `test_sandbox_escape` CI guard (the trap #470 left)
|
||||
|
||||
`TestSandboxEscape` is the only backend-agnostic integration test that boots a
|
||||
real bottle, so it is the one that would catch a macOS launch regression. It
|
||||
still carries a second guard that skips under `GITEA_ACTIONS` for every backend
|
||||
except `firecracker`:
|
||||
|
||||
```python
|
||||
@unittest.skipIf(
|
||||
os.environ.get("GITEA_ACTIONS") == "true"
|
||||
and os.environ.get("BOT_BOTTLE_BACKEND") != "firecracker",
|
||||
...,
|
||||
)
|
||||
```
|
||||
|
||||
The skip exists because the *containerized* `act_runner` (docker on
|
||||
`ubuntu-latest`) can't see a host bind mount and hides sibling-gateway network
|
||||
topology. Those constraints do not apply to a **host-mode** runner — neither
|
||||
the KVM host runner nor a macOS host runner is containerized. This PRD relaxes
|
||||
the guard to allow both host-mode backends (`firecracker`, `macos-container`)
|
||||
through while still skipping on the containerized Docker job. Without this
|
||||
change the macOS job would run green while skipping the exact test that proves
|
||||
the backend launches — the very false-green this issue is about.
|
||||
|
||||
## Open questions
|
||||
|
||||
- None known that block the job. git-gate is fully implemented on the macOS
|
||||
backend (the gateway's consolidated `git-http` daemon plus dynamic key
|
||||
provisioning/revocation), so `TestSandboxEscape` attack 5 — secret exfil
|
||||
pushed through git-gate, rejected by the gitleaks hook before the upstream
|
||||
push — runs the same as on the other backends. Any genuinely
|
||||
macOS-specific test adjustment would surface at first runner bring-up, but
|
||||
none is anticipated from the current backend implementation.
|
||||
@@ -1,172 +0,0 @@
|
||||
# PRD 0078: Quick install script
|
||||
|
||||
- **Status:** Active
|
||||
- **Author:** claude
|
||||
- **Created:** 2026-07-25
|
||||
- **Issue:** #197
|
||||
|
||||
## Summary
|
||||
|
||||
Add a proper Python package distribution (`pyproject.toml` with a
|
||||
`bot-bottle` entry point) plus a thin `install.sh` bootstrapper, so users
|
||||
can install bot-bottle with a single command instead of cloning the repo
|
||||
and invoking `cli.py` directly. A new `bot-bottle doctor` subcommand
|
||||
verifies host prerequisites after install.
|
||||
|
||||
## Problem
|
||||
|
||||
There is currently no install path for new users. The only way to run
|
||||
bot-bottle is to clone the repo and invoke `./cli.py`. This blocks any
|
||||
public demo: readers want `curl | sh` or `pipx install`, not a manual
|
||||
clone-and-configure flow. There is also no single command that tells a
|
||||
user whether their host is actually ready to run a bottle.
|
||||
|
||||
## Goals / Success Criteria
|
||||
|
||||
- `curl -fsSL <raw-url>/install.sh | sh` leaves a working `bot-bottle`
|
||||
command on PATH.
|
||||
- Python-native users can install with `pipx install bot-bottle` or
|
||||
`uv tool install bot-bottle` (once published) — or from a local
|
||||
checkout today.
|
||||
- `install.sh` validates prerequisites (Python ≥ 3.11), creates the
|
||||
`~/.bot-bottle/` config tree, installs the package, and runs
|
||||
`bot-bottle doctor`. It never installs Docker or a VM backend silently
|
||||
and never uses `sudo`.
|
||||
- `install.sh` is idempotent — safe to re-run.
|
||||
- `bot-bottle doctor` reports Python version, backend *readiness*, and
|
||||
config-dir presence, exiting non-zero when a hard prerequisite is unmet.
|
||||
- The package keeps **zero runtime pip dependencies** (stdlib-only,
|
||||
matching the existing constraint in `AGENTS.md`).
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Bundling a Python runtime or producing a standalone binary.
|
||||
- Automatic Docker / VM-backend installation.
|
||||
- Plugin-architecture changes (issue #197 floats a containerized-plugin
|
||||
direction; that's a separate feature).
|
||||
- Publishing to a package index in this PR — the package *structure* is
|
||||
the deliverable; publishing is a follow-up step.
|
||||
|
||||
## Design
|
||||
|
||||
### Package structure (`pyproject.toml`)
|
||||
|
||||
Fill out the previously-stub `pyproject.toml` with project metadata, a
|
||||
console-script entry point, and package-data for the non-Python assets the
|
||||
runtime reads from inside the package:
|
||||
|
||||
```toml
|
||||
[project]
|
||||
name = "bot-bottle"
|
||||
version = "0.1.0"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = []
|
||||
|
||||
[project.scripts]
|
||||
bot-bottle = "bot_bottle.cli:main"
|
||||
```
|
||||
|
||||
`bot_bottle.cli:main` already exists (the `cli.py` shim calls it), so no
|
||||
refactor of the entry point is needed. `package-data` ships the non-Python
|
||||
assets that live *inside* the package (`egress_entrypoint.sh`, the contrib
|
||||
Dockerfiles, the firecracker netpool defaults, the macos-container init
|
||||
script).
|
||||
|
||||
### Self-contained wheel (build resources)
|
||||
|
||||
The gateway / infra / orchestrator images are built from a Docker (or Apple
|
||||
`container`) build context that must contain the `bot_bottle` package,
|
||||
`pyproject.toml`, and the **root-level** Dockerfiles as siblings. Several
|
||||
modules used to locate that context by walking `__file__`'s parents to the
|
||||
repo root (`_REPO_ROOT = Path(__file__)…parents[N]`) and reading
|
||||
`Dockerfile.gateway`, `nix/firecracker-netpool.nix`, and
|
||||
`scripts/firecracker-netpool.sh` from it. In an installed wheel the package
|
||||
lives in `site-packages` with no repo root above it, so those reads fail —
|
||||
`doctor` passes but `start` / backend setup breaks.
|
||||
|
||||
Fix: a single resolver, `bot_bottle/resources.py`.
|
||||
|
||||
- `build_root()` returns a directory shaped like a repo root (has
|
||||
`bot_bottle/`, `pyproject.toml`, the Dockerfiles, `nix/`, `scripts/`).
|
||||
In a **checkout** it's the repo root itself — unchanged behavior. From an
|
||||
**installed wheel** it stages a copy under the app-data dir, keyed by a
|
||||
**content digest** of the installed package + bundled resources (not the
|
||||
distribution version): the installer defaults to a git branch and
|
||||
`pipx install --force` while `version` stays `0.1.0`, so a version key
|
||||
would reuse a previous commit's tree — the digest key re-stages instead.
|
||||
Staging is concurrency-safe: a file lock serializes it, each writer builds
|
||||
into a private temp dir, and the finished tree is published with an atomic
|
||||
rename (never populating a shared path another process might read).
|
||||
- The root-level resources are shipped inside the wheel under
|
||||
`bot_bottle/_resources/` by a `setup.py` `build_py` step (kept in sync
|
||||
with `resources.BUNDLED_RESOURCES`); `MANIFEST.in` includes them in the
|
||||
sdist.
|
||||
- Every former `_REPO_ROOT` / `_REPO_DIR` call site now derives from
|
||||
`resources`: the docker/macos agent-image launch, each backend's
|
||||
`orchestrator` / `gateway` / `infra` service, firecracker `infra_vm` /
|
||||
`infra_artifact` / `setup`, and the shared `gateway` build context. So
|
||||
checkout and wheel installs share one downstream path.
|
||||
|
||||
Verification: `test_resources` exercises both layouts — including the staged
|
||||
wheel context, a re-stage when package content changes at the same version,
|
||||
and a rebuild of a partial (crashed) stage. `test_wheel_install` builds the
|
||||
wheel, installs it into an isolated venv, and asserts `bot-bottle doctor`
|
||||
runs and `build_root()` produces a valid context; `build` is in
|
||||
`requirements-dev.txt` so it runs in CI, and a build/install failure fails
|
||||
the test (it does not skip). Running `start` end-to-end still needs a
|
||||
Docker/KVM host (CI), not a source checkout.
|
||||
|
||||
### `install.sh`
|
||||
|
||||
A POSIX `sh` bootstrapper that:
|
||||
|
||||
1. Checks `python3` is present and ≥ 3.11; exits with a clear message
|
||||
otherwise.
|
||||
2. Checks `git` when installing a `git+` spec, and — when falling back to
|
||||
pip — that pip is usable and the interpreter isn't externally managed
|
||||
(PEP 668), pointing at pipx otherwise.
|
||||
3. Creates `~/.bot-bottle/{agents,bottles,contrib}`.
|
||||
4. Installs via `pipx` if available, else `python3 -m pip install --user`.
|
||||
The spec defaults to the git URL and is overridable via
|
||||
`BOT_BOTTLE_INSTALL_SPEC` (used by tests / local installs).
|
||||
5. Locates the `bot-bottle` entry point: PATH first, else the
|
||||
interpreter's own user-scheme scripts dir resolved via `sysconfig`
|
||||
(`~/.local/bin` on Linux, `~/Library/Python/<X.Y>/bin` on a python.org
|
||||
macOS interpreter — not hardcoded).
|
||||
6. Runs `bot-bottle doctor` and reports the result.
|
||||
|
||||
It is idempotent and never calls `sudo`.
|
||||
|
||||
### `bot-bottle doctor`
|
||||
|
||||
A new store-free subcommand (no DB migration required) that checks and
|
||||
reports:
|
||||
|
||||
- **python** — interpreter version (hard requirement: ≥ 3.11).
|
||||
- **backend** — at least one backend *ready* on this host
|
||||
(macos-container / firecracker / docker), via `is_backend_ready()` — a
|
||||
full backend `status()` probe (daemon reachable, network pool present,
|
||||
KVM usable), not a PATH-only check: a stopped daemon or half-configured
|
||||
backend must not report `ok` when `start` can't work. Each not-ready
|
||||
backend prints its own diagnostics. Hard requirement.
|
||||
- **config** — whether `~/.bot-bottle/` exists (advisory only; `start`
|
||||
provisions on first run).
|
||||
|
||||
Exits 0 when both hard requirements pass, non-zero otherwise.
|
||||
|
||||
## Testing strategy
|
||||
|
||||
- Unit test `bot-bottle doctor` success/failure paths with backend
|
||||
readiness (`is_backend_ready`) and Python version mocked, including the
|
||||
available-but-not-ready → fail case.
|
||||
- Unit test that `pyproject.toml` parses, declares the entry point and an
|
||||
empty `dependencies` list, and that every `package-data` glob resolves
|
||||
to a file that exists on disk (guards against drift).
|
||||
- Unit test that `install.sh` is executable, POSIX-ish (`set -eu`), never
|
||||
calls `sudo`, and runs `doctor` after install.
|
||||
|
||||
## Open questions
|
||||
|
||||
- Should `version` be derived from a git tag at build time (e.g.
|
||||
`hatch-vcs`) or kept static? Static (`0.1.0`) is simpler for now.
|
||||
- Publishing target (PyPI vs. a self-hosted index) is deferred.
|
||||
+5
-8
@@ -7,13 +7,10 @@ document vs. a research note or a decision record).
|
||||
|
||||
## Naming and numbering
|
||||
|
||||
New PRDs may use a `prd-new-<kebab-title>.md` placeholder name while the
|
||||
design is being drafted. Before merge, assign the next sequential number
|
||||
after the highest-numbered PRD on `main`, rename the file to
|
||||
`NNNN-<kebab-title>.md`, and update the title header. CI blocks merging
|
||||
while any `prd-new-*.md` placeholder remains. If concurrent PRs select the
|
||||
same number, the later PR must take the next available number before it
|
||||
merges. Numbers are never reused; gaps are fine.
|
||||
New PRDs use a `prd-new-<kebab-title>.md` placeholder name while the PR
|
||||
is open. On merge to `main` a CI workflow assigns the next sequential
|
||||
number (`0024-…`, `0025-…`), renames the file, and updates the title
|
||||
header. Numbers are never reused; gaps are fine.
|
||||
|
||||
Once numbered, the filename stays fixed for the life of the doc.
|
||||
|
||||
@@ -29,7 +26,7 @@ The `Status:` line near the top tracks the PRD's lifecycle:
|
||||
## Format
|
||||
|
||||
```markdown
|
||||
# PRD prd-new: <short title> ← replace with the final number before merge
|
||||
# PRD prd-new: <short title> ← placeholder; CI fills in the number on merge
|
||||
|
||||
- **Status:** Draft
|
||||
- **Author:** <who>
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# PRD 0074: CI artifact-based coverage and local Firecracker candidate flow
|
||||
# PRD prd-new: CI artifact-based coverage and local Firecracker candidate flow
|
||||
|
||||
- **Status:** Active
|
||||
- **Author:** Claude
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# PRD 0071: Claude forward_host_credentials
|
||||
# PRD prd-new: Claude forward_host_credentials
|
||||
|
||||
- **Status:** Active
|
||||
- **Status:** Draft
|
||||
- **Author:** claude
|
||||
- **Created:** 2026-07-01
|
||||
- **Issue:** #325
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# PRD 0073: Consolidate infra backend for Docker
|
||||
# PRD prd-new: Consolidate infra backend for Docker
|
||||
|
||||
- **Status:** Active
|
||||
- **Author:** Claude
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# PRD 0079: Per-service signing keys for control-plane auth
|
||||
# PRD prd-new: Per-service signing keys for control-plane auth
|
||||
|
||||
- **Status:** Active
|
||||
- **Status:** Draft
|
||||
- **Author:** claude
|
||||
- **Created:** 2026-07-26
|
||||
- **Issue:** #476
|
||||
@@ -1,273 +0,0 @@
|
||||
# PRD prd-new: Host control server
|
||||
|
||||
- **Status:** Draft
|
||||
- **Author:** Claude
|
||||
- **Created:** 2026-07-26
|
||||
- **Issue:** #468
|
||||
|
||||
## Summary
|
||||
|
||||
Promote the in-process launch broker into a standalone **host control
|
||||
server**: the single privileged component on the host. Both the CLI and the
|
||||
orchestrator drive it over HTTP; it brokers agent launches, owns the
|
||||
orchestrator's own lifecycle, and is the sole writer of host-durable state (the
|
||||
tamper-evident audit record). This closes the three gaps between today's
|
||||
well-formed broker *contract* ([`orchestrator/broker.py`](../../bot_bottle/orchestrator/broker.py))
|
||||
and a real out-of-process service — transport, durable provisioned secret,
|
||||
and a disciplined op vocabulary — and splits host state by
|
||||
owner and lifetime. The prize: **the CLI no longer needs the Docker socket**,
|
||||
which is what finally lets a dedicated Gitea runner user drop the
|
||||
root-equivalent `docker` group (PRD 0070, "Relationship to other work").
|
||||
|
||||
## Problem
|
||||
|
||||
Container launches run directly from a short-lived CLI process against the
|
||||
Docker socket. That socket is root-equivalent, so every host that launches
|
||||
bottles hands root to whoever invokes the CLI — including a CI runner user we
|
||||
want to keep unprivileged. PRD 0070 already argues for replacing the fat socket
|
||||
with a **thin, structured, auditable** launch broker, and the contract for that
|
||||
broker exists and is tested in-process. But it is *only* in-process:
|
||||
`LaunchBroker.submit(token)` is a method call from
|
||||
`OrchestratorCore.launch_bottle` ([`service.py:116`](../../bot_bottle/orchestrator/service.py)),
|
||||
and `DockerBroker` is on no production path — every backend starts the
|
||||
orchestrator with `--broker stub` ([`__main__.py:54`](../../bot_bottle/orchestrator/__main__.py)).
|
||||
|
||||
Three gaps stand between that scaffold and a host service:
|
||||
|
||||
1. **No transport.** `submit` is an in-process call. A real service needs a
|
||||
`BrokerClient` that POSTs the signed token and a host-side HTTP server that
|
||||
verifies and acts.
|
||||
2. **The signing secret is ephemeral and self-generated.**
|
||||
[`__main__.py:53`](../../bot_bottle/orchestrator/__main__.py) does
|
||||
`secrets.token_bytes(32)` and hands the *same value* to signer and verifier —
|
||||
viable only because they share a process. A separate daemon needs the secret
|
||||
provisioned out of band and durable across orchestrator restarts.
|
||||
3. **The op vocabulary is `launch` / `teardown` only.** Everything else
|
||||
host-privileged still lives in the CLI, so the schema has to grow — carefully,
|
||||
since PRD 0070's security argument rests on "structured requests only, static
|
||||
flags + ids."
|
||||
|
||||
Separately, host state has no clear owner. `OrchestratorCore.reconcile` takes
|
||||
`live_source_ips` as a parameter *only because the orchestrator cannot see the
|
||||
backend* ([`service.py:137`](../../bot_bottle/orchestrator/service.py)); the
|
||||
egress traffic log is written to the container's stderr; and there is no durable,
|
||||
tamper-evident home for the audit record that survives orchestrator destruction.
|
||||
|
||||
## Goals / Success Criteria
|
||||
|
||||
- A standalone host control server that the CLI and orchestrator reach over
|
||||
**HTTP**, with three entry paths working end to end:
|
||||
- `web console -(iroh)-> orchestrator -(http)-> host controller -> launch`
|
||||
- `cli -(http)-> orchestrator -(http)-> host controller -> launch`
|
||||
- `cli -(http)-> host controller` — start / restart / status of the
|
||||
orchestrator **itself** (the bootstrap/recovery path #391 targets).
|
||||
- The launch op is expressed as a **signed JWT of static flags + ids only**,
|
||||
verified against a closed schema.
|
||||
- The signing secret is **provisioned out of band and durable** across
|
||||
orchestrator restarts (a `TrustDomain` per #476, with a key the orchestrator
|
||||
never holds for the host controller's *own* endpoints).
|
||||
- Host-privileged operations move off the CLI to the control server; **the CLI
|
||||
no longer opens the Docker socket** for bottle operations.
|
||||
- `Orchestrator.reconcile` no longer takes `live_source_ips` — live-bottle
|
||||
enumeration becomes an internal control-server call.
|
||||
- Host-durable state lands as an **append-only, hash-chained JSONL** audit log
|
||||
owned solely by the host controller; operational state stays SQLite owned
|
||||
solely by the orchestrator.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- **Removing standing privilege.** This converts on-demand privilege (a CLI the
|
||||
user invokes) into standing privilege (a daemon under launchd/systemd). The
|
||||
win is that the privilege is *narrower* (structured requests vs. a raw socket),
|
||||
not that it disappears. "Always running" is an accepted new property.
|
||||
- **Asymmetric signing.** We stay HS256 — see Design / "Signing stays
|
||||
symmetric."
|
||||
- **Integrity against a live compromised orchestrator.** Host-location of the
|
||||
audit log does not buy this: the orchestrator makes the decisions being audited
|
||||
and can forge or omit entries wherever the file lives. An off-box copy is the
|
||||
answer, tracked separately.
|
||||
- **A single unified DB for all state.** Impossible over a guest-kernel share
|
||||
(SQLite locking is not coherent); state is split by owner and lifetime instead.
|
||||
- **The generic `SecretProvider` (#355)** and **remote terminal design (#478)** —
|
||||
both ride the same door but are their own work.
|
||||
|
||||
## Design
|
||||
|
||||
### Topology
|
||||
|
||||
The host controller is the sole privileged component. The orchestrator becomes a
|
||||
client of it for launches, and the CLI becomes a client of it for *both* bottle
|
||||
operations (indirectly, through the orchestrator) and orchestrator lifecycle
|
||||
(directly, for bootstrap/recovery — startup can't route through the thing being
|
||||
started).
|
||||
|
||||
```
|
||||
web console ─(iroh)─▶ orchestrator ─┐
|
||||
├─(http, signed JWT)─▶ host controller ─▶ launch
|
||||
cli ────────(http)──▶ orchestrator ─┘
|
||||
cli ────────(http, bearer)──────────────────────────────▶ host controller (orchestrator lifecycle)
|
||||
```
|
||||
|
||||
### Transport: `BrokerClient` + host server
|
||||
|
||||
`LaunchBroker.submit(token)` keeps its exact signature and semantics; only the
|
||||
*wire* changes. A new `BrokerClient` implements the same submit contract by
|
||||
POSTing the signed token to the host controller (stdlib `urllib`, like the
|
||||
existing [`orchestrator/client.py`](../../bot_bottle/orchestrator/client.py)),
|
||||
and the host controller's launch handler is the existing `verify_request` +
|
||||
`_launch`/`_teardown` path, now reached over HTTP instead of a method call. The
|
||||
in-process `StubBroker` stays for the dev-harness and tests; `DockerBroker`'s
|
||||
`_launch`/`_teardown` bodies move behind the server unchanged. Because the client
|
||||
satisfies the same interface `OrchestratorCore` already depends on, the core does
|
||||
not change to gain a real backend.
|
||||
|
||||
### Signing stays symmetric (HS256)
|
||||
|
||||
PRD 0070 nominally specifies asymmetric; the code is HS256 and we keep it.
|
||||
Asymmetric matters when the verifier is *less* privileged than the signer — here
|
||||
it is the reverse: the host controller (verifier) is strictly more privileged
|
||||
than the orchestrator (signer), and a controller that could forge orchestrator
|
||||
requests gains nothing, since it is already the component that launches. Staying
|
||||
symmetric also honors the no-runtime-deps policy (stdlib has no Ed25519). This
|
||||
matches the reasoning already inlined in `broker.py`'s module docstring.
|
||||
|
||||
### Replay protection is out of scope (tracked in #494)
|
||||
|
||||
Once the launch token travels over a wire, a captured token could be replayed —
|
||||
`sign_request` already emits `jti`/`iat` but `verify_request` reads neither, so
|
||||
there is no expiry window or `jti` cache today. Enforcing that (an `iat` window +
|
||||
a self-trimming `jti` cache) is a pure in-process change that lands independently
|
||||
of this work, and it is deferred to **#494** rather than gating the MVP of the
|
||||
host control server. Nothing here depends on it; it can merge before or after.
|
||||
|
||||
### Op vocabulary and the "ids + static flags" rule (gap 3)
|
||||
|
||||
Each op moved off the CLI widens the privileged surface, so growth is governed by
|
||||
one explicit rule, enforced in `verify_request`'s schema check:
|
||||
|
||||
> A broker op carries **only ids and enumerated static flags** — a bottle id, a
|
||||
> pool slot, a **content-addressed** image ref chosen from a fixed set, an op
|
||||
> name from a closed vocabulary. Never a free-form path, argv, command, or
|
||||
> caller-supplied filesystem location. If an operation cannot be expressed that
|
||||
> way, it does not become a broker op.
|
||||
|
||||
Operations that fit and move off the CLI (all today in
|
||||
`backend/*/consolidated_launch.py`, driven by a short-lived CLI process):
|
||||
|
||||
| Op | What it does | Fits the rule because |
|
||||
|---|---|---|
|
||||
| `launch` / `teardown` | existing | ids + slot + image ref |
|
||||
| `orchestrator.ensure_running` | start the infra container | no arguments |
|
||||
| `orchestrator.{start,restart,status}` | lifecycle (the #391 path) | no arguments |
|
||||
| `list_live` | enumerate running bottles for reconcile | no arguments; returns ids/IPs |
|
||||
| `allocate_ip` | `next_free_ip` over `_network_container_ips` | no arguments; returns an IP |
|
||||
| `provision_git_gate` | `cp`/`exec` a per-bottle deploy key into the gateway | bottle id + key handle, no path |
|
||||
| `reprovision` | `docker exec printenv <ENV_VAR_SECRET>` on a live agent | bottle id + secret *name* |
|
||||
|
||||
Image **builds** stay with the orchestrator for v1 (PRD 0070 §Memory: builds run
|
||||
control-plane-side; a dedicated slim build unit is later, #468-adjacent), so no
|
||||
`build` broker op is added here.
|
||||
|
||||
With `list_live` as an internal control-server call, `Orchestrator.reconcile`'s
|
||||
`live_source_ips` parameter goes away — the tell PRD 0070 called out that the
|
||||
orchestrator couldn't see the backend disappears with it.
|
||||
|
||||
### Secret provisioning (gap 2)
|
||||
|
||||
The shared HS256 secret becomes a durable, out-of-band artifact via the
|
||||
**`TrustDomain`** seam (#476,
|
||||
[`trust_domain.py`](../../bot_bottle/trust_domain.py)):
|
||||
|
||||
- The **launch-broker secret** is a `TrustDomain` whose key
|
||||
(`host_signing_key(<file>)`, minted 0600 on first use, durable under
|
||||
`bot_bottle_root()`) is provisioned to the orchestrator (signer) and the host
|
||||
controller (verifier). Durability across orchestrator restarts is what makes
|
||||
re-adoption work — a restart re-verifies against the same key.
|
||||
- The **host controller's own lifecycle endpoints** (the direct `cli -> host
|
||||
controller` path) get a **separate** `TrustDomain` key the orchestrator never
|
||||
holds — exactly the second domain #476's PRD reserves. The orchestrator must
|
||||
not be able to mint the credentials used to start and stop it.
|
||||
|
||||
This reuses the seam #476 landed rather than re-deriving provisioning per
|
||||
backend (the PR #471 bug class).
|
||||
|
||||
### One daemon, structurally separate handlers (open decision 1)
|
||||
|
||||
The audit writer and the broker live in **one daemon** for install simplicity,
|
||||
but with **no shared parsing** and **different credentials per handler**:
|
||||
|
||||
- the **launch** handler requires the signed launch **JWT** (provenance +
|
||||
un-coercible schema);
|
||||
- the **audit-append** handler takes a plain **bearer token** and writes to the
|
||||
JSONL log.
|
||||
|
||||
This does not defend against orchestrator compromise (it holds both creds) — it
|
||||
stops a bug in the boring audit path from reaching the privileged launch path.
|
||||
The launcher stays small enough to audit line-by-line, per PRD 0070.
|
||||
|
||||
### State ownership: split by owner and lifetime
|
||||
|
||||
A single mounted DB is impossible — SQLite locking is not coherent across guest
|
||||
kernels over a share, which is why the macOS backend already uses a container-only
|
||||
volume (`INFRA_DB_VOLUME`). So state splits three ways (depends on #469, which
|
||||
gets `bot-bottle.db` off the data plane first):
|
||||
|
||||
| Owner | State | Home | Shape |
|
||||
|---|---|---|---|
|
||||
| **Orchestrator** | `orchestrator_bottles` registry; `bottled_agent_secrets` (encrypted egress tokens); `supervise_proposals` / `supervise_responses` | volume nothing else mounts (generalizing the macOS design) | **SQLite** — mutable, transactional, queried |
|
||||
| **Host controller** | supervise audit entries; egress traffic log (today → container stderr); host-side config | host filesystem, survives orchestrator/volume destruction | **JSONL** — append-only |
|
||||
| **Gateway** | none | — | after #469 the data plane holds no DB state |
|
||||
|
||||
The historical record is **JSONL, not SQLite**, because it is append-only, never
|
||||
updated, never transactionally queried: `O_APPEND` writes are atomic, there is no
|
||||
locking protocol to get wrong, hash-chaining for tamper-evidence is cheap, and it
|
||||
survives container-runtime volume pruning (the #450 lesson) and stays readable
|
||||
without the orchestrator running. Both halves of "the audit record" — supervise
|
||||
decisions and the egress traffic log — land in the one place.
|
||||
|
||||
The orchestrator is **sole mounter and sole writer** of its SQLite volume; the
|
||||
host controller is **sole writer** of the JSONL log, over the authenticated
|
||||
audit-append channel.
|
||||
|
||||
## Implementation chunks
|
||||
|
||||
Ordered, each independently mergeable:
|
||||
|
||||
1. **`BrokerClient` + host launch server** over HTTP, reusing `verify_request`
|
||||
and the existing `DockerBroker` bodies. Wire `OrchestratorCore` to a
|
||||
`BrokerClient` behind a flag; keep `StubBroker` for the dev-harness. Closes
|
||||
gap 1.
|
||||
2. **Durable secret via `TrustDomain`** — provision the launch-broker key to
|
||||
signer + verifier; add the host controller's own lifecycle `TrustDomain`.
|
||||
Closes gap 2.
|
||||
3. **Grow the op vocabulary** one op at a time (`list_live` first — it also
|
||||
removes `reconcile`'s `live_source_ips`), each behind the ids + static-flags
|
||||
rule. Closes gap 3.
|
||||
4. **JSONL audit log** — the host-controller-owned, hash-chained historical
|
||||
record with the plain-bearer audit-append handler; redirect the egress traffic
|
||||
log into it.
|
||||
5. **Drop the Docker socket from the CLI** once every host-privileged op it used
|
||||
is a broker op — the payoff that unblocks the unprivileged Gitea runner user.
|
||||
|
||||
## Open questions
|
||||
|
||||
1. **Schema-width rule enforcement.** The "ids + static flags" rule is stated;
|
||||
should `verify_request` reject unknown claim keys outright (strict schema) to
|
||||
keep the surface from drifting? Leaning yes.
|
||||
2. **Audit-append back-pressure.** What the audit handler does if the JSONL sink
|
||||
is unavailable (fail-closed vs. buffer) — resolve before shipping chunk 5.
|
||||
|
||||
## References
|
||||
|
||||
- **PRD 0070** — the contract, the launch broker, and the state tiers this
|
||||
implements.
|
||||
- **#469** — get `bot-bottle.db` off the data plane (lands underneath this).
|
||||
- **#476** ([`prd-new-control-plane-auth-provisioning`](prd-new-control-plane-auth-provisioning.md))
|
||||
— the `TrustDomain` seam this plugs the host controller's key into.
|
||||
- **#391** — backend-agnostic orchestrator restart (the bootstrap path).
|
||||
- **#494** — enforce broker replay protection (`iat` window + `jti` cache); split
|
||||
out of this PRD as an independent in-process change.
|
||||
- **#386** — prebuilt images from the Gitea OCI registry (the fixed image set the
|
||||
broker validates against).
|
||||
- **#355** — generic `SecretProvider`.
|
||||
- **#478** — remote terminal design.
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# PRD 0076: Modernize built-in agent images
|
||||
# PRD prd-new: Modernize built-in agent images
|
||||
|
||||
- **Status:** Active
|
||||
- **Status:** Draft
|
||||
- **Author:** Codex
|
||||
- **Created:** 2026-07-21
|
||||
- **Issue:** #451
|
||||
@@ -1,6 +1,6 @@
|
||||
# PRD 0075: Containers inside a bottle
|
||||
# PRD prd-new: Containers inside a bottle
|
||||
|
||||
- **Status:** Active
|
||||
- **Status:** Draft
|
||||
- **Author:** Claude
|
||||
- **Created:** 2026-07-21
|
||||
- **Issue:** #392
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# PRD 0072: Non-blocking supervise (async approval + proposal polling)
|
||||
# PRD prd-new: Non-blocking supervise (async approval + proposal polling)
|
||||
|
||||
- **Status:** Active
|
||||
- **Status:** Draft
|
||||
- **Author:** didericis
|
||||
- **Created:** 2026-07-18
|
||||
- **Issue:** #412
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# PRD 0080: Encrypted at-rest egress secrets (SecretProvider, interim slice)
|
||||
# PRD prd-new: Encrypted at-rest egress secrets (SecretProvider, interim slice)
|
||||
|
||||
- **Status:** Draft
|
||||
- **Author:** didericis
|
||||
@@ -0,0 +1,463 @@
|
||||
# PRD prd-new: Per-bottle signed commits & audit attribution
|
||||
|
||||
- **Status:** Draft
|
||||
- **Author:** didericis-claude
|
||||
- **Created:** 2026-07-25
|
||||
- **Issue:** #423
|
||||
|
||||
## Summary
|
||||
|
||||
Give each bottled agent a **per-activation signing key** so that every commit it
|
||||
produces is signed in the git-gate trust boundary (outside the bottle) and
|
||||
recorded in bot-bottle's own **host-owned audit store**, which is the portable
|
||||
source of truth. Each row cryptographically binds a commit's bytes (and their
|
||||
control-plane-recomputed SHA) to **access to that activation's signing key**, and
|
||||
binds the key to control-plane-owned activation metadata — bottle, host,
|
||||
manifest, agent, activation interval, retained public key — plus the commit's
|
||||
*claimed* author. The gate mints a short-lived Ed25519 key at spin-up, holds the
|
||||
private half in the sidecar `ssh-agent`, and forwards only `SSH_AUTH_SOCK` into
|
||||
the bottle. The gate rejects any commit it forwards that is not signed by the
|
||||
activation key; separately, the **control plane** independently recomputes each
|
||||
commit's object ID and verifies its signature before recording attribution — it
|
||||
never trusts a SHA, key, or verdict asserted by the gate.
|
||||
|
||||
This PRD deliberately does **not** enforce or vouch author/committer identity.
|
||||
Author/committer name/email are recorded as claims carried inside the signed
|
||||
object; making the gate reject a mismatching author/committer is a possible
|
||||
future add (see **Non-goals** and **Deferred: identity enforcement**). Push
|
||||
capability stays exactly as PRD 0048 deploy keys; forge subuser accounts,
|
||||
provisioned API tokens, and forge-side status/"Verified" badges remain out of
|
||||
scope (a future "forge actors" PRD).
|
||||
|
||||
Successor to:
|
||||
|
||||
- **PRD 0027 (agent git identity, #94)** / **ADR 0002** — established that
|
||||
`git-gate.user` name/email is *claimed, not vouched*. This PRD keeps that
|
||||
posture: it adds signed **provenance** and a durable host record, not identity
|
||||
enforcement.
|
||||
- **PRD 0048 (deploy-key provisioning, #169)** — the host-side mint-at-spin-up /
|
||||
revoke-at-teardown lifecycle the signing key follows. Deploy keys are
|
||||
unchanged.
|
||||
- **PRD 0070 (per-host orchestrator, #351)** — the orchestrator/control plane is
|
||||
the sole owner of `bot-bottle.db`; audit verification and recording live
|
||||
there, not in the data-plane gate (see **Trust boundary**).
|
||||
|
||||
## The guarantee
|
||||
|
||||
The crisp property this feature provides:
|
||||
|
||||
> The **host-owned audit store** binds a set of commit bytes — whose Git object
|
||||
> ID the control plane **recomputes** itself — to **access to this activation's
|
||||
> signing key**, and binds that key to **control-plane-owned activation
|
||||
> metadata**: bottle, host, manifest, agent, activation interval, retained public
|
||||
> key. An agent may author and sign arbitrary commit contents, but it cannot make
|
||||
> that signature verify as a *different* activation, and it cannot choose the
|
||||
> activation metadata the control plane records. The commit's author/committer
|
||||
> identity is **recorded as a claim**, not enforced or vouched. The forge remains
|
||||
> only the repository transport/capability layer.
|
||||
|
||||
What this does and does not prove (issue #423, comments #5554 / #5607 / #5608):
|
||||
|
||||
- It proves **access to activation *Y*'s signing key**: whoever assembled these
|
||||
commit bytes could sign with that key. Recomputing the object ID and verifying
|
||||
the embedded signature binds the SHA to activation *Y*, and the control plane's
|
||||
own records bind *Y*'s key to *Y*'s metadata.
|
||||
- It does **not** prove the commit was ever pushed, observed upstream, kept
|
||||
(vs. later reverted or dropped), or produced by the *agent* rather than by any
|
||||
other holder of the activation signing capability (the sidecar itself). The
|
||||
store deliberately makes no claim about publication or sole-agent authorship
|
||||
— the owner's requirement is attribution of *what manifest/agent/etc. was in
|
||||
use when a commit was signed*, not proof of where the commit went (#5607).
|
||||
- It does **not** make author/committer identity cryptographically vouched. The
|
||||
bottle chooses every byte sent through the forwarded agent, so a signature over
|
||||
`author Mallory <mallory@example>` is just as valid. Those fields are a claim
|
||||
carried inside the signed object and recorded as-is.
|
||||
- The binding is trustworthy because the **control plane** supplies the SHA (it
|
||||
recomputes it), the public key, and the activation metadata from its own state
|
||||
— never from a value the gateway asserts (see **Trust boundary**). The
|
||||
residual, by design: anything that holds the activation signing capability can
|
||||
produce commits that attribute to that activation. That is inherent to a
|
||||
binding on *activation-key access*, not a defect.
|
||||
|
||||
## Problem
|
||||
|
||||
An agent runs on the developer's machine as a *subrole*, scoped down per role.
|
||||
Locally that is fine because the machine is single-tenant. The git history an
|
||||
agent produces, however, is a durable artifact that outlives the session and
|
||||
can be pushed to shared repositories, and today bot-bottle offers no
|
||||
tamper-evidence over it:
|
||||
|
||||
- **No provenance.** Nothing ties a pushed commit to the bottle/activation that
|
||||
actually produced it. `git-gate.user` name/email is forgeable and cosmetic
|
||||
(ADR 0002); a commit could be produced anywhere.
|
||||
- **No durable, portable record.** There is no host-side ledger that says "SHA
|
||||
*X* was produced by agent *A* in bottle *B* on host *H* during interval
|
||||
*[t0,t1]*, signed by key *K*," independent of any forge and surviving key
|
||||
rotation.
|
||||
|
||||
## Goals / Success Criteria
|
||||
|
||||
- **Per-activation signing key.** A fresh Ed25519 keypair is minted host-side at
|
||||
each activation; the private half lives only in the sidecar `ssh-agent`, never
|
||||
in the bottle. Only `SSH_AUTH_SOCK` crosses the boundary.
|
||||
- **Signed commits with no SHA divergence.** Commits produced in the bottle are
|
||||
signed at commit time; the SHA the agent observes is the SHA that reaches the
|
||||
upstream through the gate.
|
||||
- **Gate rejects unsigned commits.** Before the gate forwards a push, every
|
||||
newly-introduced commit (those not already reachable from the advertised
|
||||
upstream refs) must verify against the activation public key; a push with any
|
||||
unsigned or wrong-key new commit is rejected, loudly, with the offending SHA.
|
||||
This is a **signature** check only — no author/committer matching.
|
||||
- **Control-plane-owned attribution.** The orchestrator/control plane (sole
|
||||
owner of `bot-bottle.db`, PRD 0070) recomputes each commit's object ID from the
|
||||
bytes, verifies the embedded signature against the activation public key it
|
||||
holds, and attaches activation metadata from its own state — accepting no SHA,
|
||||
key, verdict, or metadata asserted by the gateway. No upstream fetch is
|
||||
required.
|
||||
- **Host is the source of truth.** The audit record binds each recomputed SHA to
|
||||
the bottle, host, manifest, agent, activation interval, and retained public
|
||||
key, and records the commit's claimed author/committer.
|
||||
- **Verifiable after teardown.** The audit record retains the **full public
|
||||
key, fingerprint, principal, and validity interval** — enough to regenerate an
|
||||
allowed-signers file and run `git verify-commit` long after the activation
|
||||
ends and the key is gone.
|
||||
- **Reprovision-per-activation, fail-loud teardown.** The signing key is minted
|
||||
once per activation (persists across restarts within that activation) and
|
||||
discarded at teardown; deploy-key revocation continues to follow PRD 0048's
|
||||
fail-loud discipline.
|
||||
- **Push capability unchanged.** Forge access remains PRD 0048 deploy keys; no
|
||||
new forge API dependency beyond 0048's existing deploy-key registration.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- **Author/committer enforcement.** Explicitly out of scope for this PRD (issue
|
||||
#423, comment #5590). The gate does not reject a commit for carrying a foreign
|
||||
author or committer; those fields are recorded as claims. We rely on the
|
||||
cross-forge audit store of signed commits and the authors recorded there. See
|
||||
**Deferred: identity enforcement** for what a future add would look like.
|
||||
- **Cryptographically-vouched author identity.** Not claimed — see **The
|
||||
guarantee**.
|
||||
- **Forge subuser accounts / provisioned API tokens / PAT minting.** Dropped.
|
||||
Gitea's `POST /users/:name/tokens` requires Basic Auth *as the target user*
|
||||
(an admin PAT cannot mint one for another user; the only server-side path is
|
||||
the `gitea admin user generate-access-token` CLI), so a token-minting
|
||||
bootstrap is a design in its own right (issue #423, comments #5518 / #5554).
|
||||
This PRD needs no subrole API token, so that bootstrap problem does not arise.
|
||||
- **Forge-side attribution surfaces.** No commit-status badges, no forge
|
||||
"Verified" badge. The latter is doubly unsuitable: it renders dynamically
|
||||
against a *currently registered* key (so it would lie the moment a
|
||||
reprovisioned key is revoked), and on Gitea registering a signing key also
|
||||
grants push. Attribution lives in the host record and local `git
|
||||
verify-commit`, not the forge.
|
||||
- **Non-Gitea forges, dashboard UI for orphan cleanup, mid-session rotation,
|
||||
dirty-teardown reconciliation.** As before; a separate cleanup/sync pass
|
||||
handles orphans left by a crash or discarded snapshot.
|
||||
|
||||
## Scope narrowing
|
||||
|
||||
This PRD started as "forge subroles" (forge subuser accounts + provisioned API
|
||||
tokens + optional forge status posting + signing). Review (issue #423, comments
|
||||
#5518 → #5590) narrowed it in two steps:
|
||||
|
||||
1. **Dropped the forge-account and API-token machinery** (#5518 → #5556):
|
||||
the PAT bootstrap is not implementable as sketched (Basic-Auth-as-target-user
|
||||
constraint); the signature never vouched the author anyway; and making the
|
||||
host audit store the portable source of truth is a cleaner boundary that
|
||||
removes the forge-specific token lifecycle and commit-status dependence.
|
||||
2. **Dropped author/committer enforcement** (#5590): rely on the audit store of
|
||||
signed commits and the authors recorded there; gate enforcement of the
|
||||
identity fields is a possible future add, not part of this slice.
|
||||
|
||||
What remains is the core that stands on its own: **signed commits + a
|
||||
host-owned, independently-verified audit record.** Forge *actors* (a per-bottle
|
||||
account that comments/opens PRs) and *identity enforcement* (the gate rejecting a
|
||||
foreign author/committer) are each candidate future PRDs.
|
||||
|
||||
## Design
|
||||
|
||||
### Trust boundary (control plane vs data plane)
|
||||
|
||||
git-gate is the **data plane**: it parses hostile bytes from inside the bottle
|
||||
and forwards pushes. The orchestrator is the **control plane** and, per PRD
|
||||
0070, is the sole owner of `bot-bottle.db`. These are different trust boundaries,
|
||||
and the audit record must be anchored in the control plane:
|
||||
|
||||
- The gate performs a **synchronous pre-forward signature check** (below) and
|
||||
can reject a push before it reaches the upstream. This is a data-plane gate on
|
||||
what leaves the bottle, not the audit binding.
|
||||
- The **control plane** takes the commit bytes to attribute (gateway-delivered
|
||||
opaque bytes are fine), **recomputes the Git object ID**, **verifies the
|
||||
embedded signature** against the activation public key it minted and holds, and
|
||||
writes `attributed_commit` attaching metadata from its own state. It accepts
|
||||
**no** gateway-supplied `verified` flag, claimed SHA, public key, or activation
|
||||
identity.
|
||||
|
||||
The precise trust statement (issue #423, review by didericis-codex on d8362ec,
|
||||
resolved in #5608): the row binds *these commit bytes / this recomputed SHA* to
|
||||
*access to this activation's signing key*, and the control plane binds that key
|
||||
to the recorded activation metadata. It does **not** assert forge observation or
|
||||
that only the agent (not the signing sidecar) authored the commit — so this PRD
|
||||
does **not** claim a compromised gateway cannot obtain an attribution row.
|
||||
Because the sidecar holds the activation signing capability, a compromised
|
||||
gateway *can* assemble and sign a commit and have it attributed to that
|
||||
activation; what it cannot do is make the signature verify as a *different*
|
||||
activation or choose the metadata the control plane records. That residual is
|
||||
acceptable under the intended guarantee (#5607) and is why the guarantee is
|
||||
worded as activation-key access, not agent-only authorship or upstream
|
||||
publication. The gate therefore cannot stand in for host-side verification: the
|
||||
control plane recomputes the object ID and verifies the signature itself rather
|
||||
than trusting the gate's word.
|
||||
|
||||
### Identity model
|
||||
|
||||
Per **bottled agent** (agent definition ∘ sealed bottle), realized per
|
||||
activation:
|
||||
|
||||
| Part | Value | Source | Role |
|
||||
|------|-------|--------|------|
|
||||
| Signing key | one Ed25519 keypair | minted host-side per activation | signs every commit; private half sidecar-only; the anchor of provenance |
|
||||
| Author/committer | name + email | `git-gate.user` (PRD 0027 overlay) | written into commits and **recorded** as a claim; **not** enforced |
|
||||
|
||||
### Manifest surface
|
||||
|
||||
No new top-level keys and no `git-forge`/`forge-accounts` blocks. A single
|
||||
opt-in flag under the existing `git-gate` key turns on per-activation signing;
|
||||
`git-gate.user` (PRD 0027) supplies the author string as today.
|
||||
|
||||
```yaml
|
||||
git-gate:
|
||||
user: # PRD 0027 — author string; recorded, not enforced
|
||||
name: didericis-claude
|
||||
email: eric+claude@dideric.is
|
||||
signing:
|
||||
enabled: true # NEW — opt-in per-activation signing + audit
|
||||
repos:
|
||||
bot-bottle:
|
||||
url: ssh://git@100.78.141.42:30009/didericis/bot-bottle.git
|
||||
provisioned_key: # PRD 0048 — push capability, UNCHANGED
|
||||
provider: gitea
|
||||
token_env: GITEA_DEPLOY_TOKEN
|
||||
host_key: "ssh-ed25519 AAAA..."
|
||||
```
|
||||
|
||||
- `git-gate.signing.enabled: true` opts a bottle in. Without it, behavior is
|
||||
exactly as today. There is **no `enforce` sub-key** — this PRD does not enforce
|
||||
identity fields, so no knob is needed (and a knob that weakened a guarantee
|
||||
was flagged as a contradiction in review).
|
||||
- `git-gate.signing` is **bottle-only** (home-only policy), rejected at the
|
||||
agent level with a clear pointer. `git-gate.user` keeps its PRD 0027
|
||||
agent-overlay semantics.
|
||||
|
||||
### Signing: sign at commit time via a forwarded ssh-agent
|
||||
|
||||
The reason SHAs never diverge:
|
||||
|
||||
- The **sidecar** (the git-gate trust boundary) runs an `ssh-agent` holding the
|
||||
short-lived signing private key.
|
||||
- **Only `SSH_AUTH_SOCK`** is forwarded into the bottle — a bounded signing
|
||||
capability, not the key.
|
||||
- The provisioner writes the bottle `.gitconfig`:
|
||||
|
||||
```ini
|
||||
[commit]
|
||||
gpgsign = true
|
||||
[gpg]
|
||||
format = ssh
|
||||
[user]
|
||||
name = didericis-claude
|
||||
email = eric+claude@dideric.is
|
||||
signingkey = ssh-ed25519 AAAA... # activation signing PUBLIC key
|
||||
```
|
||||
|
||||
- `git commit` asks the forwarded agent to sign; the signature is embedded at
|
||||
object creation, so the agent-space SHA equals the pushed SHA. No transcoder,
|
||||
no SHA translation table.
|
||||
|
||||
### Gate pre-forward signature check (data plane)
|
||||
|
||||
The gate already fetches from upstream before every `upload-pack` and mirrors
|
||||
bidirectionally (PRD 0008). When `git-gate.signing.enabled` is set, after
|
||||
gitleaks and before forwarding a push upstream:
|
||||
|
||||
1. **Compute the newly-introduced set.** Commits reachable from the pushed ref
|
||||
tips but **not** reachable from any ref already advertised by the upstream
|
||||
(which the gate knows because it fetches upstream first) — equivalent to
|
||||
`git rev-list <new-tips> --not <all-known-upstream-refs>`. This excludes
|
||||
pulled/merged existing history; a merge commit the bottle creates is itself
|
||||
new and is checked, its already-upstream ancestors are not.
|
||||
2. **Verify each new commit's signature** against the activation public key. A
|
||||
commit that is unsigned or signed by any other key causes the push to be
|
||||
**rejected** with the offending SHA.
|
||||
3. No author/committer matching is performed.
|
||||
|
||||
This is a synchronous safety gate on what leaves the bottle; it is not the audit
|
||||
record.
|
||||
|
||||
### Control-plane verification & recording
|
||||
|
||||
For each commit to attribute (the gate hands the control plane the commit bytes;
|
||||
opaque gateway-delivered bytes are acceptable because nothing the gateway *says*
|
||||
about them is trusted), the orchestrator/control plane:
|
||||
|
||||
1. **Recomputes the Git object ID** from the bytes itself. The stored `sha` is
|
||||
this recomputed value, never a SHA the gateway claims.
|
||||
2. **Verifies the embedded signature** against the activation public key it
|
||||
minted and holds for that activation (via a generated allowed-signers file) —
|
||||
ignoring any `verified` flag, key, or activation identity supplied by the
|
||||
gateway.
|
||||
3. Writes `attributed_commit` only for bytes that pass, stamping the activation
|
||||
metadata (bottle/manifest/agent/host/interval) from its **own** state — not
|
||||
from anything the gateway provides — and recording the commit's claimed
|
||||
author/committer.
|
||||
|
||||
No upstream fetch is required: the guarantee is a byte↔activation-key binding, so
|
||||
the object does not need to come from the forge (issue #423, #5608). Bytes that
|
||||
do not verify against the activation key are **not** recorded as attributed (they
|
||||
may be logged as an anomaly instead).
|
||||
|
||||
### Audit trail
|
||||
|
||||
The host SQLite store (PRD 0067, `~/.bot-bottle/bot-bottle.db`, owned by the
|
||||
control plane per PRD 0070) records the signing-key lifecycle and per-commit
|
||||
attribution. Retention is the **full public key, fingerprint, principal, and
|
||||
validity interval** — enough to regenerate an allowed-signers file and verify
|
||||
commits after teardown (issue #423, comment #5554, resolution 3). Never any
|
||||
private key material.
|
||||
|
||||
```sql
|
||||
CREATE TABLE bottled_agent_activation (
|
||||
bottled_agent_slug TEXT NOT NULL,
|
||||
activation_id TEXT NOT NULL, -- one per activation cycle
|
||||
host TEXT NOT NULL,
|
||||
manifest_digest TEXT NOT NULL, -- ties the record to the sealed manifest
|
||||
agent TEXT NOT NULL,
|
||||
signing_pubkey TEXT NOT NULL, -- full ssh-ed25519 public key (for verify-commit)
|
||||
signing_fpr TEXT NOT NULL, -- SHA256:... fingerprint (stable handle)
|
||||
principal TEXT NOT NULL, -- allowed-signers principal, e.g. the author email
|
||||
valid_from TEXT NOT NULL,
|
||||
valid_until TEXT, -- NULL while active; set at teardown
|
||||
status TEXT NOT NULL, -- active | retired
|
||||
PRIMARY KEY (bottled_agent_slug, activation_id)
|
||||
);
|
||||
|
||||
CREATE TABLE attributed_commit (
|
||||
sha TEXT NOT NULL, -- control-plane-RECOMPUTED object ID, not gateway-claimed
|
||||
bottled_agent_slug TEXT NOT NULL,
|
||||
activation_id TEXT NOT NULL,
|
||||
repo TEXT NOT NULL,
|
||||
author_name TEXT NOT NULL, -- CLAIMED, recorded as-is (not enforced)
|
||||
author_email TEXT NOT NULL, -- CLAIMED
|
||||
committer_name TEXT NOT NULL, -- CLAIMED
|
||||
committer_email TEXT NOT NULL, -- CLAIMED
|
||||
observed_at TEXT NOT NULL,
|
||||
PRIMARY KEY (sha, repo)
|
||||
);
|
||||
```
|
||||
|
||||
Verification/allowed-signers generation is a stated part of the design: for a
|
||||
given SHA, join `attributed_commit → bottled_agent_activation`, emit
|
||||
`<principal> <signing_pubkey>` to a temporary allowed-signers file, and
|
||||
`git verify-commit` (or `ssh-keygen -Y verify`) against it. The
|
||||
`(pubkey, principal, valid_from/until)` tuple is exactly what that requires. The
|
||||
recorded author/committer columns are the *claim*; a consumer that wants to know
|
||||
"who says they wrote this" reads them, understanding they are unenforced.
|
||||
|
||||
### Credential lifecycle
|
||||
|
||||
Follows PRD 0048, minus the API-token kind (dropped):
|
||||
|
||||
- **Activation:** mint a fresh Ed25519 signing keypair; load the private half
|
||||
into the sidecar `ssh-agent`; write the public half into `.gitconfig` and the
|
||||
`bottled_agent_activation` row (`active`, `valid_from` set). Deploy keys are
|
||||
provisioned exactly as PRD 0048. Minting is **per activation** (a restart
|
||||
re-attaches the same key; a new activation mints a new key and retires the old
|
||||
row), so frozen snapshots don't accumulate live keys.
|
||||
- **Teardown (fail-loud):** revoke provisioned deploy keys via the forge API
|
||||
(0048); discard the signing key from the sidecar agent and set the activation
|
||||
row to `retired` with `valid_until`. The signing key was never on the forge,
|
||||
so there is nothing to revoke there — only the local retire. Deploy-key
|
||||
revocation failure halts teardown (0048); 404 = already-gone = success.
|
||||
- **Dirty teardown** is assumed handled; a separate cleanup/sync pass reconciles
|
||||
orphaned deploy keys.
|
||||
|
||||
## Deferred: identity enforcement
|
||||
|
||||
If a future PRD wants the gate to *enforce* that new commits carry the manifest
|
||||
identity, the natural shape is: extend the gate pre-forward check to also require
|
||||
each new commit's author **and** committer name/email to equal `git-gate.user`,
|
||||
rejecting mismatches — with the same control-plane re-verification before
|
||||
recording. This is deliberately left out now (issue #423, comment #5590); it is
|
||||
noted so the door stays open and the current schema (which records the claimed
|
||||
author/committer) already carries what such a check would compare against. Note
|
||||
that even then the property would be gate-*enforced*, not signature-*vouched*; a
|
||||
validating signing broker in front of the key would be required for the latter.
|
||||
|
||||
## Implementation chunks
|
||||
|
||||
1. **This PRD.** Sets the (narrowed) design.
|
||||
2. **Manifest surface.** Add `git-gate.signing` (bottle-only; `enabled` only);
|
||||
reject it at the agent level. Unit tests for parse/validation and the
|
||||
agent-level rejection.
|
||||
3. **Signing pipeline.** Sidecar `ssh-agent` provisioning; forward
|
||||
`SSH_AUTH_SOCK` into the bottle across docker, smolmachines, macOS-container,
|
||||
and firecracker backends; emit the `commit.gpgsign` / `gpg.format=ssh` /
|
||||
`user.signingkey` gitconfig. Integration test: a bottle commit is
|
||||
`verify-commit`-valid and its SHA is unchanged through the gate; the private
|
||||
key is absent from the bottle.
|
||||
4. **Gate pre-forward signature check.** Compute the newly-introduced set
|
||||
(excluding upstream-reachable commits), verify each against the activation
|
||||
key, reject unsigned/wrong-key with the offending SHA. Tests: unsigned
|
||||
rejected; wrong-key rejected; pulled/merged upstream history passes; an
|
||||
all-signed push succeeds. A foreign-author commit that is correctly signed
|
||||
**passes the gate** (identity is not enforced here).
|
||||
5. **Control-plane verification + audit.** `bottled_agent_activation` /
|
||||
`attributed_commit` tables (PRD 0067 store, control-plane-owned per PRD 0070);
|
||||
the control plane recomputes each commit's object ID and verifies the
|
||||
signature before writing a row; retain full pubkey + fingerprint + principal +
|
||||
validity interval; record claimed author/committer; allowed-signers generation
|
||||
+ a post-teardown `verify-commit` helper. Tests: a gateway-claimed SHA/key/
|
||||
verdict is ignored — the row's `sha` is the recomputed ID and bytes not signed
|
||||
by the activation key produce **no** row.
|
||||
6. **Docs.** Glossary entry ("per-bottle signed commits"); README manifest
|
||||
section; ADR note that signing-enabled bottles gain signed *provenance* and a
|
||||
host-owned audit record while authorship stays *claimed* (ADR 0002 unchanged).
|
||||
|
||||
## Testing strategy
|
||||
|
||||
- **Unit (must):** `git-gate.signing` parse/validation; agent-level `signing`
|
||||
rejection.
|
||||
- **Integration — signing (must):** end-to-end signed commit verifies with
|
||||
`git verify-commit`; SHA observed in the bottle equals the SHA upstream; the
|
||||
private key is absent from the bottle.
|
||||
- **Integration — gate check (must):** unsigned rejected; wrong-key rejected;
|
||||
the upstream-reachable exclusion (pull + merge human history and push a signed
|
||||
merge); a correctly-signed foreign-author commit **passes** (no identity
|
||||
enforcement); a clean all-signed push succeeds.
|
||||
- **Control plane (must):** the control plane recomputes the object ID and
|
||||
records a row for bytes genuinely signed by the activation key; a gateway-
|
||||
supplied SHA/key/verdict is ignored (the stored `sha` is the recomputed value);
|
||||
bytes signed by a foreign/invalid key produce **no** row.
|
||||
- **Lifecycle:** activation mints the key and writes an `active` row; teardown
|
||||
retires it (`valid_until`) and revokes deploy keys fail-loud; a restart
|
||||
re-attaches the same key (no new row); a fresh activation mints a new key and
|
||||
retires the old.
|
||||
- **Post-teardown verification:** regenerate the allowed-signers file from a
|
||||
`retired` row and confirm `verify-commit` still succeeds for an attributed SHA.
|
||||
|
||||
## Resolved: control-plane transport
|
||||
|
||||
Raised in review and **resolved** (issue #423, #5608): the commit object does not
|
||||
need to come from the forge, and reading the gateway-owned mirror is no stronger
|
||||
than accepting gateway-delivered bytes — both are fabricatable, and neither
|
||||
matters because the control plane trusts nothing the gateway *asserts*. The
|
||||
transport is therefore: the gate hands the control plane the raw commit bytes,
|
||||
the control plane **recomputes the object ID** and **verifies the signature**
|
||||
against the activation key, and stamps its own activation metadata. No upstream
|
||||
fetch. This is exactly what makes the byte↔activation-key binding sound
|
||||
regardless of transport.
|
||||
|
||||
## Open questions
|
||||
|
||||
- **Where the gate check slots into PRD 0008 ordering.** Modeled as a
|
||||
pre-forward step after gitleaks; confirm it composes with the existing
|
||||
access-hook / mirror ordering rather than needing a separate hook.
|
||||
@@ -4,7 +4,7 @@ Spike branch: `spike/rootless-docker-macos` (`a4d8461`)
|
||||
|
||||
**Outcome:** the podman recommendation below shipped as the
|
||||
`nested_containers` bottle flag — see
|
||||
[`docs/prds/0075-nested-containers.md`](../prds/0075-nested-containers.md).
|
||||
[`docs/prds/prd-new-nested-containers.md`](../prds/prd-new-nested-containers.md).
|
||||
The `docker_access` name used throughout the spike text was renamed on the
|
||||
way in; it granted no access to anything on the host.
|
||||
|
||||
|
||||
-123
@@ -1,123 +0,0 @@
|
||||
#!/bin/sh
|
||||
# bot-bottle quick installer.
|
||||
#
|
||||
# Usage:
|
||||
# curl -fsSL https://gitea.dideric.is/didericis/bot-bottle/raw/branch/main/install.sh | sh
|
||||
#
|
||||
# Python-native users can skip this entirely:
|
||||
# pipx install bot-bottle # from a checkout or a published index
|
||||
# uv tool install bot-bottle
|
||||
#
|
||||
# This script is a thin bootstrapper: it checks prerequisites, installs the
|
||||
# package with pipx (falling back to pip --user), creates the config dir, and
|
||||
# runs `bot-bottle doctor`. It is idempotent (safe to re-run) and never uses
|
||||
# sudo. It does NOT install Docker or a VM backend for you — `doctor` reports
|
||||
# what's missing after install.
|
||||
set -eu
|
||||
|
||||
PACKAGE_SPEC="${BOT_BOTTLE_INSTALL_SPEC:-git+https://gitea.dideric.is/didericis/bot-bottle.git}"
|
||||
MIN_PYTHON_MAJOR=3
|
||||
MIN_PYTHON_MINOR=11
|
||||
|
||||
say() {
|
||||
printf 'bot-bottle install: %s\n' "$*" >&2
|
||||
}
|
||||
|
||||
die() {
|
||||
say "error: $*"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# --- prerequisites -----------------------------------------------------------
|
||||
|
||||
command -v python3 >/dev/null 2>&1 \
|
||||
|| die "python3 ${MIN_PYTHON_MAJOR}.${MIN_PYTHON_MINOR}+ is required but was not found"
|
||||
|
||||
python3 - "$MIN_PYTHON_MAJOR" "$MIN_PYTHON_MINOR" <<'PY' || die "python3 ${MIN_PYTHON_MAJOR}.${MIN_PYTHON_MINOR} or newer is required"
|
||||
import sys
|
||||
|
||||
want = (int(sys.argv[1]), int(sys.argv[2]))
|
||||
raise SystemExit(0 if sys.version_info[:2] >= want else 1)
|
||||
PY
|
||||
|
||||
# Installing a `git+` spec (the default) shells out to git under the hood,
|
||||
# whether via pipx or pip. Fail early with a clear message rather than deep
|
||||
# inside the installer's output.
|
||||
case "${PACKAGE_SPEC}" in
|
||||
git+*|*.git)
|
||||
command -v git >/dev/null 2>&1 || die \
|
||||
"git is required to install from '${PACKAGE_SPEC}'. Install git, or set "\
|
||||
"BOT_BOTTLE_INSTALL_SPEC to a non-git spec (e.g. a wheel path or a package index name)."
|
||||
;;
|
||||
esac
|
||||
|
||||
# The pip fallback needs a usable pip. Externally-managed interpreters
|
||||
# (PEP 668, common on Debian/Ubuntu/Homebrew) reject `pip install --user`;
|
||||
# pipx sidesteps that, so recommend it when pip can't be used.
|
||||
if ! command -v pipx >/dev/null 2>&1; then
|
||||
python3 -m pip --version >/dev/null 2>&1 || die \
|
||||
"neither pipx nor a usable 'python3 -m pip' was found. Install pipx "\
|
||||
"(recommended): 'python3 -m pip install --user pipx' or your OS package manager."
|
||||
if python3 - <<'PY'
|
||||
import os
|
||||
import sys
|
||||
import sysconfig
|
||||
|
||||
# PEP 668: an EXTERNALLY-MANAGED marker in the stdlib dir means pip refuses
|
||||
# to install into this interpreter without --break-system-packages.
|
||||
marker = os.path.join(sysconfig.get_path("stdlib"), "EXTERNALLY-MANAGED")
|
||||
raise SystemExit(0 if os.path.exists(marker) else 1)
|
||||
PY
|
||||
then
|
||||
die "this Python is externally managed (PEP 668), so 'pip install --user' is "\
|
||||
"blocked. Install pipx and re-run: 'python3 -m pip install --user --break-system-packages pipx', "\
|
||||
"then 'pipx ensurepath'."
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- config directories ------------------------------------------------------
|
||||
|
||||
mkdir -p \
|
||||
"${HOME}/.bot-bottle/agents" \
|
||||
"${HOME}/.bot-bottle/bottles" \
|
||||
"${HOME}/.bot-bottle/contrib"
|
||||
|
||||
# --- install -----------------------------------------------------------------
|
||||
|
||||
if command -v pipx >/dev/null 2>&1; then
|
||||
say "installing with pipx"
|
||||
pipx install --force "${PACKAGE_SPEC}"
|
||||
else
|
||||
say "pipx not found; installing with 'python3 -m pip install --user'"
|
||||
python3 -m pip install --user --upgrade "${PACKAGE_SPEC}"
|
||||
fi
|
||||
|
||||
# --- locate the entry point --------------------------------------------------
|
||||
|
||||
# The pip --user scripts directory is platform-specific: ~/.local/bin on Linux,
|
||||
# but ~/Library/Python/<X.Y>/bin on a python.org macOS interpreter. Ask the
|
||||
# interpreter for its own user-scheme scripts dir instead of hardcoding.
|
||||
USER_SCRIPTS="$(python3 - <<'PY'
|
||||
import sysconfig
|
||||
print(sysconfig.get_path("scripts", sysconfig.get_preferred_scheme("user")))
|
||||
PY
|
||||
)"
|
||||
|
||||
if command -v bot-bottle >/dev/null 2>&1; then
|
||||
BOT_BOTTLE_BIN="bot-bottle"
|
||||
elif [ -n "${USER_SCRIPTS}" ] && [ -x "${USER_SCRIPTS}/bot-bottle" ]; then
|
||||
BOT_BOTTLE_BIN="${USER_SCRIPTS}/bot-bottle"
|
||||
say "note: add ${USER_SCRIPTS} to your PATH to run 'bot-bottle' directly"
|
||||
else
|
||||
die "bot-bottle was installed but is not on PATH; add ${USER_SCRIPTS:-your user scripts dir} to PATH and re-run"
|
||||
fi
|
||||
|
||||
# --- verify ------------------------------------------------------------------
|
||||
|
||||
say "running '${BOT_BOTTLE_BIN} doctor'"
|
||||
if "${BOT_BOTTLE_BIN}" doctor; then
|
||||
say "done. Run '${BOT_BOTTLE_BIN} --help' to get started."
|
||||
else
|
||||
say "install completed, but 'doctor' reported unmet prerequisites (see above)."
|
||||
say "resolve them, then re-run '${BOT_BOTTLE_BIN} doctor'."
|
||||
fi
|
||||
+1
-38
@@ -4,42 +4,5 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "bot-bottle"
|
||||
version = "0.1.0"
|
||||
description = "Self-hosted sandbox for running AI coding agents with egress controls"
|
||||
readme = "README.md"
|
||||
version = "0.0.0"
|
||||
requires-python = ">=3.11"
|
||||
license = { text = "Apache-2.0" }
|
||||
authors = [{ name = "didericis" }]
|
||||
keywords = ["ai", "agents", "sandbox", "security", "egress"]
|
||||
classifiers = [
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3 :: Only",
|
||||
"Operating System :: POSIX :: Linux",
|
||||
"Operating System :: MacOS",
|
||||
]
|
||||
# The package itself has no runtime pip dependencies (stdlib-only); the
|
||||
# only language runtime is the Python interpreter. Keep this empty.
|
||||
dependencies = []
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://gitea.dideric.is/didericis/bot-bottle"
|
||||
Source = "https://gitea.dideric.is/didericis/bot-bottle"
|
||||
|
||||
[project.scripts]
|
||||
bot-bottle = "bot_bottle.cli:main"
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
include = ["bot_bottle*"]
|
||||
|
||||
# Non-Python assets the runtime reads from inside the package (container
|
||||
# build contexts, entrypoints, netpool defaults). Keep in sync with the
|
||||
# files shipped under bot_bottle/; test_pyproject.py asserts they exist.
|
||||
[tool.setuptools.package-data]
|
||||
bot_bottle = [
|
||||
"gateway/egress/entrypoint.sh",
|
||||
"contrib/claude/Dockerfile",
|
||||
"contrib/codex/Dockerfile",
|
||||
"contrib/pi/Dockerfile",
|
||||
"backend/firecracker/netpool.defaults.env",
|
||||
"backend/macos_container/nested-containers-init.sh",
|
||||
]
|
||||
|
||||
@@ -5,6 +5,3 @@
|
||||
pylint>=3.0.0
|
||||
pyright>=1.1.411
|
||||
coverage>=7.0.0
|
||||
# PEP 517 build front-end used by tests/unit/test_wheel_install.py to build and
|
||||
# install a real wheel (proves the installed distribution is self-contained).
|
||||
build>=1.0.0
|
||||
|
||||
@@ -54,20 +54,18 @@ def check_pull_request(event: dict[str, Any], api: GiteaApi) -> list[str]:
|
||||
pull = event["pull_request"]
|
||||
errors: list[str] = []
|
||||
labels = pull.get("labels") or []
|
||||
numbers = deliberate_issue_numbers(pull.get("title", ""), pull.get("body", ""))
|
||||
if labels and numbers:
|
||||
if labels:
|
||||
errors.append(
|
||||
"PR must use exactly one tracking mode: remove PR labels when "
|
||||
"linking an issue, or remove the issue reference when labels "
|
||||
"belong on the PR."
|
||||
"PRs must be unlabeled; put tracker metadata on the linked issue "
|
||||
f"(found: {', '.join(label['name'] for label in labels)})."
|
||||
)
|
||||
|
||||
numbers = deliberate_issue_numbers(pull.get("title", ""), pull.get("body", ""))
|
||||
if not numbers:
|
||||
if not labels:
|
||||
errors.append(
|
||||
"PR must either have a label or reference an issue with "
|
||||
"Closes/Fixes/Resolves #N, Part of #N, Related to #N, "
|
||||
"Refs #N, or References #N."
|
||||
)
|
||||
errors.append(
|
||||
"PR must reference an issue with Closes/Fixes/Resolves #N, "
|
||||
"Part of #N, Related to #N, Refs #N, or References #N."
|
||||
)
|
||||
return errors
|
||||
|
||||
real_issues = 0
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
"""Build shim. Project metadata lives in ``pyproject.toml``; this only adds a
|
||||
build step that copies the root-level build resources (the Dockerfiles, the
|
||||
nix netpool module, the netpool script, and ``pyproject.toml``) into
|
||||
``bot_bottle/_resources/`` so an installed wheel is self-contained and can
|
||||
build its gateway/infra/orchestrator images without a source checkout.
|
||||
|
||||
Kept in sync with ``bot_bottle.resources.BUNDLED_RESOURCES`` — the
|
||||
``test_resources`` suite guards against drift between the two lists.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from setuptools import setup
|
||||
from setuptools.command.build_py import build_py
|
||||
|
||||
_ROOT = Path(__file__).resolve().parent
|
||||
|
||||
# Must match bot_bottle.resources.BUNDLED_RESOURCES (paths relative to root).
|
||||
_BUNDLED_RESOURCES = (
|
||||
"pyproject.toml",
|
||||
"Dockerfile.gateway",
|
||||
"Dockerfile.orchestrator",
|
||||
"Dockerfile.orchestrator.fc",
|
||||
"nix/firecracker-netpool.nix",
|
||||
"scripts/firecracker-netpool.sh",
|
||||
)
|
||||
|
||||
|
||||
class _BundleResources(build_py):
|
||||
"""Copy the root-level build resources into the built package tree so they
|
||||
ship inside the wheel under ``bot_bottle/_resources/``."""
|
||||
|
||||
def run(self) -> None:
|
||||
super().run()
|
||||
pkg_resources = Path(self.build_lib) / "bot_bottle" / "_resources"
|
||||
for rel in _BUNDLED_RESOURCES:
|
||||
src = _ROOT / rel
|
||||
dst = pkg_resources / rel
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dst)
|
||||
|
||||
|
||||
setup(cmdclass={"build_py": _BundleResources})
|
||||
@@ -67,25 +67,14 @@ _DUMMY_HOST_KEY = (
|
||||
)
|
||||
|
||||
|
||||
# Backends whose CI runner is HOST-mode (self-hosted), so the test process
|
||||
# and the backend share a host. The containerized act_runner (docker on
|
||||
# ubuntu-latest) is the one that can't see the host bind mount egress_tls_init
|
||||
# uses and hides sibling-gateway network topology; host-mode runners
|
||||
# (firecracker/KVM, macos-container) don't have those constraints, so the test
|
||||
# runs there. Keep this in sync with the `runs-on` labels in
|
||||
# .gitea/workflows/test.yml.
|
||||
_HOST_MODE_CI_BACKENDS = frozenset({"firecracker", "macos-container"})
|
||||
|
||||
|
||||
@skip_unless_selected_backend_available()
|
||||
@unittest.skipIf(
|
||||
os.environ.get("GITEA_ACTIONS") == "true"
|
||||
and os.environ.get("BOT_BOTTLE_BACKEND") not in _HOST_MODE_CI_BACKENDS,
|
||||
"skipped under the containerized act_runner (docker on ubuntu-latest): "
|
||||
and os.environ.get("BOT_BOTTLE_BACKEND") != "firecracker",
|
||||
"skipped under act_runner unless BOT_BOTTLE_BACKEND=firecracker: "
|
||||
"egress_tls_init uses a host bind mount the runner container can't "
|
||||
"see, and the network topology hides sibling-gateway visibility — "
|
||||
"these constraints don't apply on the self-hosted host-mode runners "
|
||||
"(firecracker/KVM, macos-container)",
|
||||
"these constraints don't apply on the self-hosted KVM runner",
|
||||
)
|
||||
class TestSandboxEscape(unittest.TestCase):
|
||||
"""End-to-end attacks against a real bottle. The bottle stays
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
"""Unit: `bot-bottle doctor` host prerequisite checks (ADR 0004).
|
||||
|
||||
`doctor` is a store-free diagnostic — it must run on a fresh install
|
||||
before any DB migration, and its exit code gates only the two hard
|
||||
prerequisites (Python and at least one *ready* backend). The config-dir
|
||||
check is advisory and never affects the exit code.
|
||||
|
||||
Backend readiness is probed with `is_backend_ready()` (a full status()
|
||||
check), not the cheap PATH-only `is_backend_available()` — a host with a
|
||||
stopped daemon or half-configured backend must not report `ok`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import redirect_stdout
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.cli.commands import doctor
|
||||
|
||||
|
||||
def _run(argv: list[str] | None = None) -> tuple[int, str]:
|
||||
buf = io.StringIO()
|
||||
with redirect_stdout(buf):
|
||||
code = doctor.cmd_doctor(argv or [])
|
||||
return code, buf.getvalue()
|
||||
|
||||
|
||||
class TestDoctor(unittest.TestCase):
|
||||
def test_passes_when_python_and_backend_ready(self):
|
||||
with patch.object(doctor, "known_backend_names", return_value=("docker",)), \
|
||||
patch.object(doctor, "is_backend_ready", return_value=True):
|
||||
code, out = _run()
|
||||
self.assertEqual(0, code)
|
||||
self.assertIn("ok: python", out)
|
||||
self.assertIn("ok: backend: docker: ready", out)
|
||||
|
||||
def test_fails_when_no_backend_ready(self):
|
||||
# The regression the reviewer flagged: a backend whose binary is on PATH
|
||||
# but whose daemon/pool isn't ready must NOT pass. is_backend_ready is
|
||||
# the full status() check, so returning False here means "not ready".
|
||||
with patch.object(doctor, "known_backend_names", return_value=("docker", "firecracker")), \
|
||||
patch.object(doctor, "is_backend_ready", return_value=False):
|
||||
code, out = _run()
|
||||
self.assertEqual(1, code)
|
||||
self.assertIn("fail: backend", out)
|
||||
self.assertIn("warn: backend: docker: not ready", out)
|
||||
|
||||
def test_passes_when_at_least_one_backend_ready(self):
|
||||
# docker not ready, firecracker ready → overall pass, mixed report.
|
||||
def ready(name: str, *, quiet: bool = False) -> bool:
|
||||
del quiet
|
||||
return name == "firecracker"
|
||||
|
||||
with patch.object(doctor, "known_backend_names", return_value=("docker", "firecracker")), \
|
||||
patch.object(doctor, "is_backend_ready", side_effect=ready):
|
||||
code, out = _run()
|
||||
self.assertEqual(0, code)
|
||||
self.assertIn("warn: backend: docker: not ready", out)
|
||||
self.assertIn("ok: backend: firecracker: ready", out)
|
||||
|
||||
def test_fails_when_python_too_old(self):
|
||||
# Force the version gate to fail without touching the interpreter.
|
||||
with patch.object(doctor, "MIN_PYTHON", (99, 0)), \
|
||||
patch.object(doctor, "known_backend_names", return_value=("docker",)), \
|
||||
patch.object(doctor, "is_backend_ready", return_value=True):
|
||||
code, out = _run()
|
||||
self.assertEqual(1, code)
|
||||
self.assertIn("fail: python", out)
|
||||
|
||||
def test_missing_config_dir_is_advisory_not_fatal(self):
|
||||
# A missing ~/.bot-bottle warns but must not fail. Point home at a
|
||||
# fresh empty dir so the shared suite HOME (which other tests may
|
||||
# populate) can't turn this into an "ok: config".
|
||||
with tempfile.TemporaryDirectory() as tmp, \
|
||||
patch.object(doctor.Path, "home", return_value=Path(tmp)), \
|
||||
patch.object(doctor, "known_backend_names", return_value=("docker",)), \
|
||||
patch.object(doctor, "is_backend_ready", return_value=True):
|
||||
code, out = _run()
|
||||
self.assertEqual(0, code)
|
||||
self.assertIn("warn: config", out)
|
||||
|
||||
def test_present_config_dir_reports_ok(self):
|
||||
with tempfile.TemporaryDirectory() as tmp, \
|
||||
patch.object(doctor.Path, "home", return_value=Path(tmp)), \
|
||||
patch.object(doctor, "known_backend_names", return_value=("docker",)), \
|
||||
patch.object(doctor, "is_backend_ready", return_value=True):
|
||||
(Path(tmp) / ".bot-bottle").mkdir()
|
||||
code, out = _run()
|
||||
self.assertEqual(0, code)
|
||||
self.assertIn("ok: config", out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,101 +0,0 @@
|
||||
"""Unit: install.sh bootstrapper contract.
|
||||
|
||||
The installer is a thin, sudo-free, idempotent bootstrapper. These are
|
||||
static checks on the script text (no network / no real install) so CI can
|
||||
run them anywhere: it must be executable, fail-fast, never call sudo,
|
||||
create the config tree, install the package, and verify with `doctor`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sysconfig
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
INSTALL_SH = REPO_ROOT / "install.sh"
|
||||
|
||||
|
||||
class TestInstallScript(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.text = INSTALL_SH.read_text()
|
||||
|
||||
def test_exists_and_executable(self):
|
||||
self.assertTrue(INSTALL_SH.is_file())
|
||||
self.assertTrue(os.access(INSTALL_SH, os.X_OK), "install.sh must be executable")
|
||||
|
||||
def test_posix_shebang_and_failfast(self):
|
||||
first = self.text.splitlines()[0]
|
||||
self.assertEqual("#!/bin/sh", first)
|
||||
self.assertIn("set -eu", self.text)
|
||||
|
||||
def test_never_uses_sudo(self):
|
||||
# Only executable lines matter; the header comment may mention sudo.
|
||||
code = [
|
||||
ln for ln in self.text.splitlines()
|
||||
if ln.strip() and not ln.lstrip().startswith("#")
|
||||
]
|
||||
self.assertNotIn("sudo", "\n".join(code))
|
||||
|
||||
def test_creates_config_tree(self):
|
||||
self.assertIn(".bot-bottle/agents", self.text)
|
||||
self.assertIn(".bot-bottle/bottles", self.text)
|
||||
|
||||
def test_installs_via_pipx_with_pip_fallback(self):
|
||||
self.assertIn("pipx install", self.text)
|
||||
self.assertIn("pip install --user", self.text)
|
||||
|
||||
def test_runs_doctor_after_install(self):
|
||||
self.assertIn("doctor", self.text)
|
||||
|
||||
def test_install_spec_is_overridable(self):
|
||||
# Tests / local installs point BOT_BOTTLE_INSTALL_SPEC at a checkout.
|
||||
self.assertIn("BOT_BOTTLE_INSTALL_SPEC", self.text)
|
||||
|
||||
def test_requires_git_for_git_specs(self):
|
||||
# A git+ / .git spec (the default) shells out to git; the script must
|
||||
# gate on it rather than failing opaquely inside pipx/pip.
|
||||
self.assertIn("command -v git", self.text)
|
||||
self.assertIn("git+*|*.git", self.text)
|
||||
|
||||
def test_checks_pip_usable_before_fallback(self):
|
||||
self.assertIn("python3 -m pip --version", self.text)
|
||||
|
||||
def test_detects_externally_managed_python(self):
|
||||
# PEP 668: 'pip install --user' is blocked on externally-managed
|
||||
# interpreters; the script must detect this and point at pipx.
|
||||
self.assertIn("EXTERNALLY-MANAGED", self.text)
|
||||
self.assertIn("pipx", self.text)
|
||||
|
||||
def test_resolves_user_scripts_dir_not_hardcoded(self):
|
||||
# The pip --user scripts dir differs by platform; the script must ask
|
||||
# the interpreter (sysconfig + the preferred *user* scheme) rather than
|
||||
# hardcoding Linux's ~/.local/bin (which is wrong on macOS python.org).
|
||||
self.assertIn("get_preferred_scheme", self.text)
|
||||
self.assertIn("sysconfig", self.text)
|
||||
# No hardcoded Linux path in executable lines (a comment may mention it).
|
||||
code = "\n".join(
|
||||
ln for ln in self.text.splitlines()
|
||||
if ln.strip() and not ln.lstrip().startswith("#")
|
||||
)
|
||||
self.assertNotIn(".local/bin", code)
|
||||
|
||||
def test_macos_user_scheme_is_not_dot_local_bin(self):
|
||||
# The case the fix exists for: a python.org macOS interpreter uses the
|
||||
# osx_framework_user scheme, whose scripts land under
|
||||
# ~/Library/Python/<X.Y>/bin — NOT ~/.local/bin. Drive the same
|
||||
# sysconfig lookup install.sh uses, with a mac-like userbase, to prove
|
||||
# it resolves a non-~/.local/bin directory.
|
||||
self.assertIn("osx_framework_user", sysconfig.get_scheme_names())
|
||||
scripts = sysconfig.get_path(
|
||||
"scripts", "osx_framework_user",
|
||||
vars={"userbase": "/Users/dev/Library/Python/3.11"},
|
||||
)
|
||||
self.assertEqual("/Users/dev/Library/Python/3.11/bin", scripts)
|
||||
self.assertNotIn("/.local/bin", scripts)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -283,7 +283,7 @@ class TestBuildOrLoadImages(unittest.TestCase):
|
||||
images = launch_mod.build_or_load_images(plan)
|
||||
|
||||
build.assert_called_once_with(
|
||||
"agent:base", str(launch_mod.resources.build_root()),
|
||||
"agent:base", launch_mod._REPO_DIR, # pylint: disable=protected-access
|
||||
dockerfile="/repo/Dockerfile",
|
||||
)
|
||||
derived.assert_called_once_with("agent:base", build)
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Unit: git-gate.signing manifest parsing + validation.
|
||||
|
||||
PRD prd-new (signed commits & audit attribution): a bottle opts into
|
||||
per-activation commit signing with `git-gate.signing.enabled: true`.
|
||||
The block is bottle-only (rejected at the agent level, like
|
||||
git-gate.repos) and defaults to off.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
from bot_bottle.manifest import ManifestError, ManifestGitSigning, ManifestIndex
|
||||
|
||||
|
||||
def _bottle(git_gate: dict) -> dict: # type: ignore
|
||||
return {
|
||||
"bottles": {"dev": {"git-gate": git_gate}},
|
||||
"agents": {"demo": {"skills": [], "prompt": "", "bottle": "dev"}},
|
||||
}
|
||||
|
||||
|
||||
class TestSigningParsing(unittest.TestCase):
|
||||
def test_default_is_disabled(self):
|
||||
"""A bottle with no git-gate block signs nothing."""
|
||||
m = ManifestIndex.from_json_obj({
|
||||
"bottles": {"dev": {}},
|
||||
"agents": {"demo": {"skills": [], "prompt": "", "bottle": "dev"}},
|
||||
})
|
||||
self.assertEqual(ManifestGitSigning(), m.bottles["dev"].git_signing)
|
||||
self.assertFalse(m.bottles["dev"].git_signing.enabled)
|
||||
|
||||
def test_git_gate_without_signing_is_disabled(self):
|
||||
"""git-gate present (e.g. user only) but no signing → off."""
|
||||
m = ManifestIndex.from_json_obj(_bottle({
|
||||
"user": {"name": "claude", "email": "eric+claude@dideric.is"},
|
||||
}))
|
||||
self.assertFalse(m.bottles["dev"].git_signing.enabled)
|
||||
|
||||
def test_enabled_true(self):
|
||||
m = ManifestIndex.from_json_obj(_bottle({"signing": {"enabled": True}}))
|
||||
self.assertTrue(m.bottles["dev"].git_signing.enabled)
|
||||
|
||||
def test_enabled_false(self):
|
||||
m = ManifestIndex.from_json_obj(_bottle({"signing": {"enabled": False}}))
|
||||
self.assertFalse(m.bottles["dev"].git_signing.enabled)
|
||||
|
||||
def test_signing_coexists_with_user_and_repos(self):
|
||||
m = ManifestIndex.from_json_obj(_bottle({
|
||||
"user": {"name": "claude", "email": "eric+claude@dideric.is"},
|
||||
"signing": {"enabled": True},
|
||||
"repos": {
|
||||
"bot-bottle": {
|
||||
"url": "ssh://git@gitea.dideric.is:30009/didericis/bot-bottle.git",
|
||||
"key": {"provider": "static", "path": "/dev/null"},
|
||||
},
|
||||
},
|
||||
}))
|
||||
b = m.bottles["dev"]
|
||||
self.assertTrue(b.git_signing.enabled)
|
||||
self.assertEqual("claude", b.git_user.name)
|
||||
self.assertEqual(1, len(b.git))
|
||||
|
||||
|
||||
class TestSigningValidation(unittest.TestCase):
|
||||
def test_unknown_key_under_signing_dies(self):
|
||||
with self.assertRaises(ManifestError) as cm:
|
||||
ManifestIndex.from_json_obj(_bottle({
|
||||
"signing": {"enabled": True, "enforce": ["author"]},
|
||||
}))
|
||||
msg = str(cm.exception)
|
||||
self.assertIn("git-gate.signing", msg)
|
||||
self.assertIn("enforce", msg)
|
||||
|
||||
def test_non_bool_enabled_dies(self):
|
||||
with self.assertRaises(ManifestError) as cm:
|
||||
ManifestIndex.from_json_obj(_bottle({"signing": {"enabled": "yes"}}))
|
||||
self.assertIn("git-gate.signing.enabled must be a boolean", str(cm.exception))
|
||||
|
||||
def test_signing_not_a_mapping_dies(self):
|
||||
with self.assertRaises(ManifestError):
|
||||
ManifestIndex.from_json_obj(_bottle({"signing": ["enabled"]}))
|
||||
|
||||
def test_unknown_git_gate_key_lists_signing(self):
|
||||
"""The git-gate allowed-key error names signing as valid."""
|
||||
with self.assertRaises(ManifestError) as cm:
|
||||
ManifestIndex.from_json_obj(_bottle({"bogus": {}}))
|
||||
self.assertIn("allowed: user, repos, signing", str(cm.exception))
|
||||
|
||||
|
||||
class TestSigningIsBottleOnly(unittest.TestCase):
|
||||
def test_agent_level_signing_rejected(self):
|
||||
"""git-gate.signing on an agent dies — it is bottle-only."""
|
||||
with self.assertRaises(ManifestError) as cm:
|
||||
ManifestIndex.from_json_obj({
|
||||
"bottles": {"dev": {}},
|
||||
"agents": {
|
||||
"demo": {
|
||||
"skills": [],
|
||||
"prompt": "",
|
||||
"bottle": "dev",
|
||||
"git-gate": {"signing": {"enabled": True}},
|
||||
},
|
||||
},
|
||||
})
|
||||
msg = str(cm.exception)
|
||||
self.assertIn("git-gate.signing", msg)
|
||||
self.assertIn("not allowed at the agent level", msg)
|
||||
|
||||
|
||||
class TestSigningExtendsOverlay(unittest.TestCase):
|
||||
def test_child_inherits_parent_signing(self):
|
||||
"""A child that omits signing inherits the parent's enabled flag."""
|
||||
m = ManifestIndex.from_json_obj({
|
||||
"bottles": {
|
||||
"base": {"git-gate": {"signing": {"enabled": True}}},
|
||||
"dev": {"extends": "base"},
|
||||
},
|
||||
"agents": {"demo": {"skills": [], "prompt": "", "bottle": "dev"}},
|
||||
})
|
||||
self.assertTrue(m.bottles["dev"].git_signing.enabled)
|
||||
|
||||
def test_child_enables_over_disabled_parent(self):
|
||||
m = ManifestIndex.from_json_obj({
|
||||
"bottles": {
|
||||
"base": {},
|
||||
"dev": {"extends": "base", "git-gate": {"signing": {"enabled": True}}},
|
||||
},
|
||||
"agents": {"demo": {"skills": [], "prompt": "", "bottle": "dev"}},
|
||||
})
|
||||
self.assertTrue(m.bottles["dev"].git_signing.enabled)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,117 +0,0 @@
|
||||
"""Unit: orchestrator-side broker client (issue #468, chunk 1). HTTP mocked."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import unittest
|
||||
import urllib.error
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bot_bottle.orchestrator.broker import (
|
||||
BrokerAuthError,
|
||||
BrokerUnavailableError,
|
||||
LaunchRequest,
|
||||
)
|
||||
from bot_bottle.orchestrator.broker_client import BrokerClient, BrokerClientError
|
||||
|
||||
_URLOPEN = "bot_bottle.orchestrator.broker_client.urllib.request.urlopen"
|
||||
|
||||
|
||||
def _resp(payload: object) -> MagicMock:
|
||||
m = MagicMock()
|
||||
m.__enter__.return_value.read.return_value = json.dumps(payload).encode()
|
||||
return m
|
||||
|
||||
|
||||
def _http_error(code: int, payload: object = None) -> urllib.error.HTTPError:
|
||||
body = json.dumps(payload).encode() if payload is not None else b""
|
||||
return urllib.error.HTTPError(
|
||||
"http://host/broker", code, "err", {}, io.BytesIO(body)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
class TestSubmit(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.c = BrokerClient("http://host:8091")
|
||||
|
||||
def test_returns_the_verified_request(self) -> None:
|
||||
echo = {
|
||||
"op": "launch", "bottle_id": "b1", "source_ip": "10.0.0.1",
|
||||
"image_ref": "img", "slot": 3,
|
||||
}
|
||||
with patch(_URLOPEN, return_value=_resp(echo)):
|
||||
got = self.c.submit("tok")
|
||||
self.assertEqual(
|
||||
LaunchRequest(op="launch", bottle_id="b1", source_ip="10.0.0.1",
|
||||
image_ref="img", slot=3),
|
||||
got,
|
||||
)
|
||||
|
||||
def test_posts_token_to_broker_endpoint(self) -> None:
|
||||
with patch(_URLOPEN, return_value=_resp({"op": "teardown", "bottle_id": "b1"})) as m:
|
||||
self.c.submit("signed-token")
|
||||
request = m.call_args.args[0]
|
||||
self.assertEqual("POST", request.get_method())
|
||||
self.assertTrue(request.full_url.endswith("/broker"))
|
||||
self.assertEqual({"token": "signed-token"}, json.loads(request.data))
|
||||
|
||||
def test_401_raises_broker_auth_error(self) -> None:
|
||||
# A fail-closed provenance/schema rejection surfaces as the SAME exception
|
||||
# the in-process broker raises, so the launch path's rollback is identical.
|
||||
with patch(_URLOPEN, side_effect=_http_error(401, {"error": "bad signature"})):
|
||||
with self.assertRaises(BrokerAuthError):
|
||||
self.c.submit("forged")
|
||||
|
||||
def test_502_is_a_definite_client_error(self) -> None:
|
||||
# The host responded — it processed the request and did not launch, so a
|
||||
# definite BrokerClientError (the caller may safely roll back).
|
||||
with patch(_URLOPEN, side_effect=_http_error(502, {"error": "docker down"})):
|
||||
with self.assertRaises(BrokerClientError):
|
||||
self.c.submit("tok")
|
||||
|
||||
def test_unreachable_is_ambiguous_unavailable(self) -> None:
|
||||
# No response at all — the request may already have launched, so the
|
||||
# AMBIGUOUS BrokerUnavailableError (the caller must NOT roll back).
|
||||
with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
|
||||
with self.assertRaises(BrokerUnavailableError):
|
||||
self.c.submit("tok")
|
||||
|
||||
def test_timeout_is_ambiguous_unavailable(self) -> None:
|
||||
# A dropped/late response after the request was sent is the exact orphan
|
||||
# risk: the host may have launched. Must be ambiguous, not a definite fail.
|
||||
with patch(_URLOPEN, side_effect=TimeoutError("read timed out")):
|
||||
with self.assertRaises(BrokerUnavailableError):
|
||||
self.c.submit("tok")
|
||||
|
||||
def test_malformed_success_body_raises(self) -> None:
|
||||
with patch(_URLOPEN, return_value=_resp({"op": "launch"})): # missing bottle_id
|
||||
with self.assertRaises(BrokerClientError):
|
||||
self.c.submit("tok")
|
||||
|
||||
def test_empty_error_body_is_tolerated(self) -> None:
|
||||
# An error with no readable JSON body still classifies by status code.
|
||||
with patch(_URLOPEN, side_effect=_http_error(401)):
|
||||
with self.assertRaises(BrokerAuthError):
|
||||
self.c.submit("forged")
|
||||
|
||||
def test_non_json_success_body_raises(self) -> None:
|
||||
# A 200 whose body isn't JSON is tolerated into {} then fails the
|
||||
# missing-field check — a definite client error, not a crash.
|
||||
m = MagicMock()
|
||||
m.__enter__.return_value.read.return_value = b"not json at all"
|
||||
with patch(_URLOPEN, return_value=m):
|
||||
with self.assertRaises(BrokerClientError):
|
||||
self.c.submit("tok")
|
||||
|
||||
def test_unreadable_error_body_is_tolerated(self) -> None:
|
||||
# An HTTPError whose body can't be read (fp=None) still classifies by
|
||||
# status — the error detail is best-effort.
|
||||
err = urllib.error.HTTPError(
|
||||
"http://host/broker", 502, "err", {}, None) # type: ignore[arg-type]
|
||||
with patch(_URLOPEN, side_effect=err):
|
||||
with self.assertRaises(BrokerClientError):
|
||||
self.c.submit("tok")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,306 +0,0 @@
|
||||
"""Unit tests for the host control server (issue #468, chunk 1).
|
||||
|
||||
Mostly exercises the pure `dispatch()` (socket-free, like the orchestrator
|
||||
server tests), plus a real-socket round-trip through `BrokerClient` that proves
|
||||
the full sign -> POST -> verify -> act seam over HTTP.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import tempfile
|
||||
import threading
|
||||
import typing
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bot_bottle.orchestrator.broker import (
|
||||
BrokerAuthError,
|
||||
LaunchBroker,
|
||||
LaunchRequest,
|
||||
StubBroker,
|
||||
sign_request,
|
||||
)
|
||||
from bot_bottle.orchestrator.broker_client import BrokerClient
|
||||
from bot_bottle.orchestrator.host_server import (
|
||||
MAX_BODY_BYTES,
|
||||
Handler,
|
||||
HostControlServer,
|
||||
broker_secret,
|
||||
dispatch,
|
||||
main,
|
||||
make_host_server,
|
||||
)
|
||||
from bot_bottle.paths import LAUNCH_BROKER_KEY_ENV
|
||||
|
||||
|
||||
def _body(obj: object) -> bytes:
|
||||
return json.dumps(obj).encode()
|
||||
|
||||
|
||||
class _RaisingBroker(LaunchBroker):
|
||||
"""A broker whose backend launch always fails — exercises the 502 path (an
|
||||
operational backend failure, distinct from a fail-closed provenance 401)."""
|
||||
|
||||
def _launch(self, req: LaunchRequest) -> None:
|
||||
raise RuntimeError("docker down")
|
||||
|
||||
def _teardown(self, req: LaunchRequest) -> None:
|
||||
raise RuntimeError("docker down")
|
||||
|
||||
|
||||
class TestDispatch(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.secret = secrets.token_bytes(16)
|
||||
self.broker = StubBroker(self.secret)
|
||||
|
||||
def _token(self, **kwargs: object) -> str:
|
||||
return sign_request(LaunchRequest(**kwargs), self.secret) # type: ignore[arg-type]
|
||||
|
||||
def test_health(self) -> None:
|
||||
status, payload = dispatch(self.broker, "GET", "/health", b"")
|
||||
self.assertEqual(200, status)
|
||||
self.assertEqual("ok", payload["status"])
|
||||
|
||||
def test_broker_launch_verifies_and_acts(self) -> None:
|
||||
token = self._token(
|
||||
op="launch", bottle_id="b1", source_ip="10.243.0.1",
|
||||
image_ref="img", slot=2,
|
||||
)
|
||||
status, payload = dispatch(self.broker, "POST", "/broker", _body({"token": token}))
|
||||
self.assertEqual(200, status)
|
||||
self.assertEqual("launch", payload["op"])
|
||||
self.assertEqual("b1", payload["bottle_id"])
|
||||
self.assertEqual("img", payload["image_ref"])
|
||||
self.assertEqual(2, payload["slot"])
|
||||
self.assertEqual(["b1"], [r.bottle_id for r in self.broker.launched])
|
||||
|
||||
def test_broker_teardown_acts(self) -> None:
|
||||
token = self._token(op="teardown", bottle_id="b1")
|
||||
status, _ = dispatch(self.broker, "POST", "/broker", _body({"token": token}))
|
||||
self.assertEqual(200, status)
|
||||
self.assertEqual(["b1"], [r.bottle_id for r in self.broker.torn_down])
|
||||
|
||||
def test_forged_token_is_401_and_nothing_acted(self) -> None:
|
||||
forged = sign_request(
|
||||
LaunchRequest(op="launch", bottle_id="b1"), secrets.token_bytes(16))
|
||||
status, payload = dispatch(self.broker, "POST", "/broker", _body({"token": forged}))
|
||||
self.assertEqual(401, status)
|
||||
self.assertIn("broker auth failed", str(payload["error"]))
|
||||
self.assertEqual([], self.broker.launched) # fail-closed: never launched
|
||||
|
||||
def test_backend_failure_is_502(self) -> None:
|
||||
broker = _RaisingBroker(self.secret)
|
||||
token = self._token(op="launch", bottle_id="b1", image_ref="img")
|
||||
status, payload = dispatch(broker, "POST", "/broker", _body({"token": token}))
|
||||
self.assertEqual(502, status)
|
||||
self.assertIn("backend launch failed", str(payload["error"]))
|
||||
|
||||
def test_missing_token_is_400(self) -> None:
|
||||
status, _ = dispatch(self.broker, "POST", "/broker", _body({}))
|
||||
self.assertEqual(400, status)
|
||||
|
||||
def test_bad_json_is_400(self) -> None:
|
||||
status, _ = dispatch(self.broker, "POST", "/broker", b"{not json")
|
||||
self.assertEqual(400, status)
|
||||
|
||||
def test_empty_body_is_missing_token_400(self) -> None:
|
||||
# Empty body parses to {} (no token) → 400, never reaching the broker.
|
||||
status, _ = dispatch(self.broker, "POST", "/broker", b"")
|
||||
self.assertEqual(400, status)
|
||||
self.assertEqual([], self.broker.launched)
|
||||
|
||||
def test_non_object_body_is_400(self) -> None:
|
||||
status, _ = dispatch(self.broker, "POST", "/broker", b"[1, 2]")
|
||||
self.assertEqual(400, status)
|
||||
|
||||
def test_unknown_route_404(self) -> None:
|
||||
status, _ = dispatch(self.broker, "GET", "/nope", b"")
|
||||
self.assertEqual(404, status)
|
||||
|
||||
def test_trailing_slash_normalized(self) -> None:
|
||||
status, _ = dispatch(self.broker, "GET", "/health/", b"")
|
||||
self.assertEqual(200, status)
|
||||
|
||||
|
||||
class TestBrokerSecret(unittest.TestCase):
|
||||
"""The durable launch-broker key (#468/#476): prefer the env-injected key,
|
||||
else the durable host key file, so signer and verifier resolve the same one."""
|
||||
|
||||
def test_reads_injected_key_from_env(self) -> None:
|
||||
# The injected key is honoured regardless of allow_host_file — both the
|
||||
# host controller and the guest orchestrator take an injected key.
|
||||
self.assertEqual(
|
||||
b"injected-key", broker_secret({LAUNCH_BROKER_KEY_ENV: "injected-key"}))
|
||||
self.assertEqual(
|
||||
b"injected-key",
|
||||
broker_secret({LAUNCH_BROKER_KEY_ENV: "injected-key"}, allow_host_file=True))
|
||||
|
||||
def test_guest_without_injection_fails_closed(self) -> None:
|
||||
# The default (guest orchestrator): no env key and NO host-file fallback,
|
||||
# so it returns None rather than mint a divergent process-local key.
|
||||
self.assertIsNone(broker_secret({}))
|
||||
|
||||
def test_host_side_falls_back_to_the_durable_key_file(self) -> None:
|
||||
# allow_host_file=True (host controller / dev-harness): mint/read the
|
||||
# durable host key file, the same key on every call (restart re-adoption).
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
with patch.dict("os.environ", {"BOT_BOTTLE_ROOT": root}, clear=False):
|
||||
os.environ.pop(LAUNCH_BROKER_KEY_ENV, None)
|
||||
first = broker_secret(allow_host_file=True)
|
||||
second = broker_secret(allow_host_file=True)
|
||||
self.assertTrue(first)
|
||||
self.assertEqual(first, second)
|
||||
|
||||
|
||||
class TestSeamRoundTrip(unittest.TestCase):
|
||||
"""The whole point of chunk 1: a request signed by the orchestrator side is
|
||||
POSTed to a real host control server, verified there, and acted on — over
|
||||
HTTP, not an in-process call."""
|
||||
|
||||
def _serve(self, broker: LaunchBroker) -> BrokerClient:
|
||||
server = make_host_server(broker, "127.0.0.1", 0)
|
||||
self.addCleanup(server.server_close)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
self.addCleanup(server.shutdown)
|
||||
host, port = server.server_address[0], server.server_address[1]
|
||||
return BrokerClient(f"http://{host}:{port}")
|
||||
|
||||
def test_sign_post_verify_act_over_http(self) -> None:
|
||||
secret = secrets.token_bytes(16)
|
||||
broker = StubBroker(secret)
|
||||
client = self._serve(broker)
|
||||
req = LaunchRequest(
|
||||
op="launch", bottle_id="b1", source_ip="10.0.0.1", image_ref="img", slot=1)
|
||||
got = client.submit(sign_request(req, secret))
|
||||
self.assertEqual(req, got) # the controller echoes the verified request
|
||||
self.assertEqual(["b1"], [r.bottle_id for r in broker.launched])
|
||||
|
||||
def test_forged_token_raises_broker_auth_error_over_http(self) -> None:
|
||||
secret = secrets.token_bytes(16)
|
||||
broker = StubBroker(secret)
|
||||
client = self._serve(broker)
|
||||
forged = sign_request(
|
||||
LaunchRequest(op="launch", bottle_id="b1"), secrets.token_bytes(16))
|
||||
with self.assertRaises(BrokerAuthError):
|
||||
client.submit(forged)
|
||||
self.assertEqual([], broker.launched) # fail-closed across the wire
|
||||
|
||||
|
||||
class TestRequestLimits(unittest.TestCase):
|
||||
"""The privileged listener must not let a caller that can merely reach the
|
||||
socket (no signed token) exhaust it via an oversized declared body — and it
|
||||
rejects on the Content-Length *header*, before reading the body."""
|
||||
|
||||
def _addr(self) -> tuple[str, int]:
|
||||
self.broker = StubBroker(secrets.token_bytes(16))
|
||||
server = make_host_server(self.broker, "127.0.0.1", 0)
|
||||
self.addCleanup(server.server_close)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
self.addCleanup(server.shutdown)
|
||||
host, port = server.server_address[:2]
|
||||
return typing.cast(str, host), port
|
||||
|
||||
def test_oversized_content_length_is_rejected_before_reading(self) -> None:
|
||||
host, port = self._addr()
|
||||
conn = http.client.HTTPConnection(host, port, timeout=5)
|
||||
self.addCleanup(conn.close)
|
||||
# Declare an oversized body but send only a sliver: the server must reject
|
||||
# on the header before reading, so the caller gets a clean, deterministic
|
||||
# 413 (no large unread body to race a connection reset).
|
||||
conn.putrequest("POST", "/broker", skip_accept_encoding=True)
|
||||
conn.putheader("Content-Type", "application/json")
|
||||
conn.putheader("Content-Length", str(MAX_BODY_BYTES + 1))
|
||||
conn.endheaders()
|
||||
conn.send(b"{}") # far short of the declared length; never read
|
||||
resp = conn.getresponse()
|
||||
self.assertEqual(413, resp.status)
|
||||
self.assertEqual([], self.broker.launched) # never reached the broker
|
||||
|
||||
|
||||
class TestServeUnit(unittest.TestCase):
|
||||
"""Drive `Handler._serve` directly (no socket). The real per-request handler
|
||||
runs in a daemon thread whose coverage/trace data is lost, so the
|
||||
bounded-body and error paths are exercised here in the main thread instead."""
|
||||
|
||||
def _handler(self, broker: LaunchBroker, headers: dict[str, str],
|
||||
body: bytes = b"") -> tuple[Handler, MagicMock]:
|
||||
server = HostControlServer.__new__(HostControlServer)
|
||||
server.broker = broker
|
||||
h = Handler.__new__(Handler)
|
||||
h.server = server
|
||||
h.headers = headers # type: ignore[assignment] — dict is a valid .get() stand-in
|
||||
h.path = "/broker"
|
||||
h.rfile = io.BytesIO(body)
|
||||
h.wfile = io.BytesIO()
|
||||
send_response = MagicMock()
|
||||
h.send_response = send_response # type: ignore[method-assign]
|
||||
h.send_header = MagicMock() # type: ignore[method-assign]
|
||||
h.end_headers = MagicMock() # type: ignore[method-assign]
|
||||
return h, send_response
|
||||
|
||||
def test_oversized_content_length_is_413(self) -> None:
|
||||
broker = StubBroker(secrets.token_bytes(16))
|
||||
h, send_response = self._handler(broker, {"Content-Length": str(MAX_BODY_BYTES + 1)})
|
||||
h.do_POST() # exercises do_POST -> _serve
|
||||
send_response.assert_called_once_with(413)
|
||||
self.assertEqual([], broker.launched) # rejected before the broker
|
||||
|
||||
def test_invalid_content_length_is_400(self) -> None:
|
||||
h, send_response = self._handler(StubBroker(secrets.token_bytes(16)),
|
||||
{"Content-Length": "not-a-number"})
|
||||
h._serve("POST")
|
||||
send_response.assert_called_once_with(400)
|
||||
|
||||
def test_valid_request_dispatches_200(self) -> None:
|
||||
secret = secrets.token_bytes(16)
|
||||
broker = StubBroker(secret)
|
||||
body = _body({"token": sign_request(
|
||||
LaunchRequest(op="teardown", bottle_id="b1"), secret)})
|
||||
h, send_response = self._handler(broker, {"Content-Length": str(len(body))}, body)
|
||||
h._serve("POST")
|
||||
send_response.assert_called_once_with(200)
|
||||
self.assertEqual(["b1"], [r.bottle_id for r in broker.torn_down])
|
||||
|
||||
def test_dispatch_exception_becomes_500(self) -> None:
|
||||
# dispatch is total, but the handler still guards it: a raised dispatch
|
||||
# returns 500 rather than dropping the connection.
|
||||
h, send_response = self._handler(
|
||||
StubBroker(secrets.token_bytes(16)), {"Content-Length": "0"})
|
||||
with patch("bot_bottle.orchestrator.host_server.dispatch",
|
||||
side_effect=RuntimeError("boom")):
|
||||
h._serve("POST")
|
||||
send_response.assert_called_once_with(500)
|
||||
|
||||
def test_health_over_do_get(self) -> None:
|
||||
h, send_response = self._handler(StubBroker(secrets.token_bytes(16)), {})
|
||||
h.path = "/health"
|
||||
h.do_GET()
|
||||
send_response.assert_called_once_with(200)
|
||||
|
||||
|
||||
class TestMain(unittest.TestCase):
|
||||
def test_fail_closed_without_secret(self) -> None:
|
||||
with patch("bot_bottle.orchestrator.host_server.broker_secret",
|
||||
return_value=None):
|
||||
self.assertEqual(2, main(["--port", "0"]))
|
||||
|
||||
def test_serves_then_shuts_down_cleanly(self) -> None:
|
||||
fake = MagicMock()
|
||||
fake.server_address = ("127.0.0.1", 0)
|
||||
fake.serve_forever.side_effect = KeyboardInterrupt
|
||||
with patch("bot_bottle.orchestrator.host_server.broker_secret",
|
||||
return_value=b"k"), \
|
||||
patch("bot_bottle.orchestrator.host_server.make_host_server",
|
||||
return_value=fake):
|
||||
self.assertEqual(0, main(["--port", "0"]))
|
||||
fake.serve_forever.assert_called_once()
|
||||
fake.server_close.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,67 +0,0 @@
|
||||
"""Unit: the orchestrator dev-harness entrypoint (`python -m bot_bottle.orchestrator`).
|
||||
|
||||
Exercises broker selection (stub / docker / http) and the fail-closed http path,
|
||||
patching `make_server` so the serve loop returns instead of blocking.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import secrets
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bot_bottle.orchestrator.__main__ import main
|
||||
from bot_bottle.paths import LAUNCH_BROKER_KEY_ENV
|
||||
|
||||
|
||||
def _fake_server() -> MagicMock:
|
||||
fake = MagicMock()
|
||||
fake.server_address = ("127.0.0.1", 0)
|
||||
# Break out of serve_forever immediately, exercising the try/finally.
|
||||
fake.serve_forever.side_effect = KeyboardInterrupt
|
||||
return fake
|
||||
|
||||
|
||||
class TestMain(unittest.TestCase):
|
||||
def _run(self, broker: str, env: dict[str, str] | None = None) -> tuple[int, MagicMock]:
|
||||
fake = _fake_server()
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
argv = ["--db", str(Path(d) / "r.db"), "--port", "0", "--broker", broker]
|
||||
with patch("bot_bottle.orchestrator.__main__.make_server", return_value=fake), \
|
||||
patch.dict("os.environ", env or {}, clear=False):
|
||||
if env is None:
|
||||
os.environ.pop(LAUNCH_BROKER_KEY_ENV, None)
|
||||
rc = main(argv)
|
||||
return rc, fake
|
||||
|
||||
def test_stub_broker_serves_and_closes(self) -> None:
|
||||
rc, fake = self._run("stub")
|
||||
self.assertEqual(0, rc)
|
||||
fake.serve_forever.assert_called_once()
|
||||
fake.server_close.assert_called_once()
|
||||
|
||||
def test_docker_broker_serves(self) -> None:
|
||||
rc, _ = self._run("docker")
|
||||
self.assertEqual(0, rc)
|
||||
|
||||
def test_http_broker_with_injected_key_serves(self) -> None:
|
||||
# The guest orchestrator takes the launch-broker key by injection.
|
||||
rc, _ = self._run(
|
||||
"http", env={LAUNCH_BROKER_KEY_ENV: secrets.token_urlsafe(16)})
|
||||
self.assertEqual(0, rc)
|
||||
|
||||
def test_http_broker_without_injected_key_exits(self) -> None:
|
||||
# Fail-closed: no host-file fallback for the guest, so a missing injected
|
||||
# key is a usage error rather than a silently-minted divergent key.
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
with patch.dict("os.environ", {}, clear=False):
|
||||
os.environ.pop(LAUNCH_BROKER_KEY_ENV, None)
|
||||
with self.assertRaises(SystemExit):
|
||||
main(["--db", str(Path(d) / "r.db"), "--broker", "http"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,13 +1,11 @@
|
||||
"""Unit tests for per-bottle egress secret encryption (PRD 0080)."""
|
||||
"""Unit tests for per-bottle egress secret encryption (PRD prd-new-secret-provider)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import unittest
|
||||
|
||||
from bot_bottle.orchestrator.store.secret_store import (
|
||||
ENV_VAR_SECRET_NAME,
|
||||
_NONCE_BYTES,
|
||||
decrypt_value,
|
||||
encrypt_value,
|
||||
new_env_var_secret,
|
||||
@@ -67,27 +65,22 @@ class TestDecryptErrors(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.secret = new_env_var_secret()
|
||||
|
||||
def test_wrong_key_always_raises_value_error(self) -> None:
|
||||
# Deterministic: the authentication tag rejects a wrong key every time,
|
||||
# so reprovision can never inject a garbage token. Repeat across many
|
||||
# random keys (the old unauthenticated scheme let ~5% through when the
|
||||
# garbage happened to decode as valid UTF-8).
|
||||
for _ in range(200):
|
||||
ct = encrypt_value(self.secret, "secret-token")
|
||||
with self.assertRaises(ValueError):
|
||||
decrypt_value(new_env_var_secret(), ct)
|
||||
|
||||
def test_tampered_ciphertext_raises_value_error(self) -> None:
|
||||
def test_wrong_key_raises_value_error(self) -> None:
|
||||
ct = encrypt_value(self.secret, "secret-token")
|
||||
raw = bytearray(base64.urlsafe_b64decode(ct + "=" * (-len(ct) % 4)))
|
||||
raw[_NONCE_BYTES] ^= 0x01 # flip a bit in the ciphertext body → tag mismatch
|
||||
tampered = base64.urlsafe_b64encode(bytes(raw)).rstrip(b"=").decode()
|
||||
with self.assertRaises(ValueError):
|
||||
decrypt_value(self.secret, tampered)
|
||||
other_key = new_env_var_secret()
|
||||
# Wrong key produces garbage bytes; decrypt_value raises ValueError
|
||||
# when the result is non-UTF-8 (which is very likely for 12-char data).
|
||||
# We allow it to succeed only if garbage happens to be valid UTF-8, but
|
||||
# the plaintext must not match.
|
||||
try:
|
||||
result = decrypt_value(other_key, ct)
|
||||
self.assertNotEqual("secret-token", result)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
def test_truncated_blob_raises_value_error(self) -> None:
|
||||
with self.assertRaises(ValueError):
|
||||
decrypt_value(self.secret, "dG9vc2hvcnQ") # "tooshort" — under nonce+tag
|
||||
decrypt_value(self.secret, "dG9vc2hvcnQ") # "tooshort" — under 16 nonce bytes
|
||||
|
||||
def test_invalid_base64_raises_value_error(self) -> None:
|
||||
with self.assertRaises(ValueError):
|
||||
|
||||
@@ -11,12 +11,7 @@ from contextlib import closing
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.orchestrator.broker import (
|
||||
BrokerUnavailableError,
|
||||
LaunchBroker,
|
||||
LaunchRequest,
|
||||
StubBroker,
|
||||
)
|
||||
from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBroker
|
||||
from bot_bottle.orchestrator.store.registry_store import RegistryStore
|
||||
from bot_bottle.orchestrator.service import OrchestratorCore
|
||||
from bot_bottle.orchestrator.store.secret_store import new_env_var_secret
|
||||
@@ -30,8 +25,8 @@ from bot_bottle.orchestrator.supervisor import (
|
||||
|
||||
|
||||
class _FailingBroker(LaunchBroker):
|
||||
"""Verifies the token like any broker, then fails the launch *definitely* —
|
||||
to exercise the orchestrator's registry rollback."""
|
||||
"""Verifies the token like any broker, then fails the launch — to
|
||||
exercise the orchestrator's registry rollback."""
|
||||
|
||||
def _launch(self, req: LaunchRequest) -> None:
|
||||
raise RuntimeError("launch failed")
|
||||
@@ -40,18 +35,6 @@ class _FailingBroker(LaunchBroker):
|
||||
pass
|
||||
|
||||
|
||||
class _UnavailableBroker(LaunchBroker):
|
||||
"""Verifies the token, then raises the *ambiguous* BrokerUnavailableError —
|
||||
the host may already have launched — so the orchestrator must KEEP the
|
||||
registry row rather than orphan a running container."""
|
||||
|
||||
def _launch(self, req: LaunchRequest) -> None:
|
||||
raise BrokerUnavailableError("delivery dropped after send")
|
||||
|
||||
def _teardown(self, req: LaunchRequest) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class TestOrchestrator(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
@@ -153,20 +136,11 @@ class TestOrchestrator(unittest.TestCase):
|
||||
self.assertIsNotNone(self.orch.resolve("10.243.0.3", rec.identity_token))
|
||||
self.assertIsNone(self.orch.resolve("10.243.0.3", "wrong-token"))
|
||||
|
||||
def test_launch_rolls_back_registry_on_definite_broker_failure(self) -> None:
|
||||
def test_launch_rolls_back_registry_on_broker_failure(self) -> None:
|
||||
orch = OrchestratorCore(self.store, _FailingBroker(self.secret), self.secret)
|
||||
with self.assertRaises(RuntimeError):
|
||||
orch.launch_bottle("10.243.0.9")
|
||||
self.assertEqual([], self.store.all()) # no orphan row
|
||||
|
||||
def test_launch_keeps_registry_on_ambiguous_broker_failure(self) -> None:
|
||||
# The host may already have launched the bottle before the response was
|
||||
# lost, so deregistering would orphan a running container with no row.
|
||||
# The row is kept for reconcile to reap iff the bottle is not live.
|
||||
orch = OrchestratorCore(self.store, _UnavailableBroker(self.secret), self.secret)
|
||||
with self.assertRaises(BrokerUnavailableError):
|
||||
orch.launch_bottle("10.243.0.9")
|
||||
self.assertEqual(1, len(self.store.all())) # row survives — no orphan container
|
||||
self.assertEqual([], self.store.all()) # no orphan
|
||||
|
||||
def test_gateway_status_reports_unconfigured(self) -> None:
|
||||
# The orchestrator no longer owns a standalone gateway lifecycle; the
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
"""Unit: pyproject.toml packaging contract.
|
||||
|
||||
Guards the install/distribution surface: the console-script entry point,
|
||||
the stdlib-only (empty) dependency list, and that every package-data glob
|
||||
still points at a file that exists (so an installed wheel isn't missing a
|
||||
Dockerfile or entrypoint the runtime reads).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tomllib
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
PYPROJECT = REPO_ROOT / "pyproject.toml"
|
||||
|
||||
|
||||
class TestPyproject(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
with PYPROJECT.open("rb") as fh:
|
||||
cls.data = tomllib.load(fh)
|
||||
|
||||
def test_entry_point_targets_cli_main(self):
|
||||
scripts = self.data["project"]["scripts"]
|
||||
self.assertEqual("bot_bottle.cli:main", scripts["bot-bottle"])
|
||||
|
||||
def test_no_runtime_dependencies(self):
|
||||
# AGENTS.md: the package has no runtime pip dependencies.
|
||||
self.assertEqual([], self.data["project"]["dependencies"])
|
||||
|
||||
def test_requires_python_311(self):
|
||||
self.assertEqual(">=3.11", self.data["project"]["requires-python"])
|
||||
|
||||
def test_package_data_files_exist(self):
|
||||
pkg_data = self.data["tool"]["setuptools"]["package-data"]["bot_bottle"]
|
||||
self.assertTrue(pkg_data, "expected package-data entries")
|
||||
for rel in pkg_data:
|
||||
path = REPO_ROOT / "bot_bottle" / rel
|
||||
self.assertTrue(path.is_file(), f"package-data missing: {path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,184 +0,0 @@
|
||||
"""Unit: bot_bottle.resources — build-resource resolution for both a source
|
||||
checkout and an installed wheel.
|
||||
|
||||
The checkout path is what the whole test suite already runs under; the wheel
|
||||
path is exercised here by faking an installed layout (a package dir with a
|
||||
bundled ``_resources/`` and no sibling Dockerfiles) and asserting that
|
||||
``build_root()`` stages a repo-root-shaped context. See
|
||||
``test_wheel_install.py`` for the end-to-end build+install check.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle import resources
|
||||
|
||||
from tests.unit import use_bottle_root
|
||||
|
||||
|
||||
class TestCheckoutMode(unittest.TestCase):
|
||||
"""The environment the suite runs in: a real source checkout."""
|
||||
|
||||
def test_is_source_checkout(self):
|
||||
self.assertTrue(resources.is_source_checkout())
|
||||
|
||||
def test_build_root_is_repo_root(self):
|
||||
root = resources.build_root()
|
||||
self.assertTrue((root / "bot_bottle").is_dir())
|
||||
self.assertTrue((root / "pyproject.toml").is_file())
|
||||
self.assertTrue((root / "Dockerfile.gateway").is_file())
|
||||
|
||||
def test_resource_helpers_resolve(self):
|
||||
self.assertTrue(resources.dockerfile("Dockerfile.gateway").is_file())
|
||||
self.assertTrue(resources.nix_netpool_module().is_file())
|
||||
self.assertTrue(resources.netpool_script().is_file())
|
||||
|
||||
def test_bundled_resources_all_exist_at_root(self):
|
||||
# Drift guard: every path setup.py bundles must exist in the checkout.
|
||||
root = resources.build_root()
|
||||
for rel in resources.BUNDLED_RESOURCES:
|
||||
self.assertTrue((root / rel).is_file(), f"missing bundled resource: {rel}")
|
||||
|
||||
|
||||
class TestWheelMode(unittest.TestCase):
|
||||
"""Fake an installed wheel: a package dir with _resources/ and no
|
||||
checkout Dockerfiles beside it."""
|
||||
|
||||
def _fake_install(self, tmp: Path) -> Path:
|
||||
pkg = tmp / "site-packages" / "bot_bottle"
|
||||
(pkg / "cli").mkdir(parents=True)
|
||||
(pkg / "__init__.py").write_text("")
|
||||
(pkg / "cli" / "__init__.py").write_text("# module\n")
|
||||
bundled = pkg / "_resources"
|
||||
for rel in resources.BUNDLED_RESOURCES:
|
||||
dst = bundled / rel
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
dst.write_text(f"# fake {rel}\n")
|
||||
return pkg
|
||||
|
||||
@contextmanager
|
||||
def _wheel(self):
|
||||
"""Point `resources` at a fake installed wheel with an isolated
|
||||
app-data dir; yields the package dir so a test can mutate it."""
|
||||
with tempfile.TemporaryDirectory() as tmpname:
|
||||
tmp = Path(tmpname)
|
||||
pkg = self._fake_install(tmp)
|
||||
self.addCleanup(use_bottle_root(tmp / "appdata"))
|
||||
with patch.object(resources, "_PKG", pkg), \
|
||||
patch.object(resources, "_BUNDLED", pkg / "_resources"), \
|
||||
patch.object(resources, "_CHECKOUT_ROOT", tmp / "no-checkout"):
|
||||
yield pkg
|
||||
|
||||
def test_stage_and_resolve(self):
|
||||
with self._wheel():
|
||||
self.assertFalse(resources.is_source_checkout())
|
||||
root = resources.build_root()
|
||||
# Staged context looks like a repo root.
|
||||
self.assertTrue((root / "bot_bottle" / "__init__.py").is_file())
|
||||
self.assertTrue((root / "bot_bottle" / "cli" / "__init__.py").is_file())
|
||||
self.assertTrue((root / "pyproject.toml").is_file())
|
||||
self.assertTrue((root / "Dockerfile.gateway").is_file())
|
||||
self.assertTrue((root / "nix" / "firecracker-netpool.nix").is_file())
|
||||
self.assertTrue((root / "scripts" / "firecracker-netpool.sh").is_file())
|
||||
# The bundled-resource copies are NOT re-nested under the staged
|
||||
# package (keeps it byte-identical to a checkout package).
|
||||
self.assertFalse((root / "bot_bottle" / "_resources").exists())
|
||||
# Helpers resolve off the staged root.
|
||||
self.assertEqual(root / "Dockerfile.gateway",
|
||||
resources.dockerfile("Dockerfile.gateway"))
|
||||
# Idempotent: second call returns the same completed dir.
|
||||
self.assertEqual(root, resources.build_root())
|
||||
|
||||
def test_refreshes_when_content_changes_at_same_version(self):
|
||||
# Regression for the stale-cache bug: `pipx install --force` of a newer
|
||||
# commit keeps version 0.1.0, so keying on version would reuse the old
|
||||
# tree. Keying on content must re-stage when a package file changes.
|
||||
with self._wheel() as pkg:
|
||||
root1 = resources.build_root()
|
||||
self.assertTrue((root1 / ".complete").is_file())
|
||||
(pkg / "cli" / "__init__.py").write_text("# new commit, same version\n")
|
||||
root2 = resources.build_root()
|
||||
self.assertNotEqual(root1, root2)
|
||||
self.assertEqual(
|
||||
"# new commit, same version\n",
|
||||
(root2 / "bot_bottle" / "cli" / "__init__.py").read_text(),
|
||||
)
|
||||
|
||||
def test_failed_stage_cleans_up_temp_dir(self):
|
||||
# A failure mid-stage must not leave a half-written temp dir behind.
|
||||
with self._wheel():
|
||||
base = resources.bot_bottle_root() / "build-root"
|
||||
with patch.object(resources.shutil, "copytree", side_effect=OSError("boom")):
|
||||
with self.assertRaises(OSError):
|
||||
resources.build_root()
|
||||
self.assertEqual([], list(base.glob(".staging-*")))
|
||||
|
||||
def test_rebuilds_when_stage_incomplete(self):
|
||||
# A crash mid-stage can leave a dir without its `.complete` marker; the
|
||||
# next call must rebuild it rather than trust the partial tree.
|
||||
with self._wheel():
|
||||
root = resources.build_root()
|
||||
(root / ".complete").unlink()
|
||||
(root / "sentinel").write_text("stale")
|
||||
again = resources.build_root()
|
||||
self.assertEqual(root, again) # same content digest → same dir
|
||||
self.assertTrue((again / ".complete").is_file())
|
||||
self.assertFalse((again / "sentinel").exists()) # rebuilt clean
|
||||
|
||||
def test_reuses_peer_stage_after_lock_wait(self):
|
||||
# Regression for the staging race: a caller that loses the lock must,
|
||||
# once it wins, see the peer's completed tree and reuse it — never
|
||||
# re-clobber a shared path. Drive it deterministically: hold the lock,
|
||||
# let a worker block after its fast-path miss, publish a complete tree
|
||||
# as the "peer", then release so the worker takes the reuse path.
|
||||
import fcntl
|
||||
import threading
|
||||
import time
|
||||
|
||||
with self._wheel():
|
||||
base = resources.bot_bottle_root() / "build-root"
|
||||
base.mkdir(parents=True, exist_ok=True)
|
||||
dest = base / resources._content_digest() # pylint: disable=protected-access
|
||||
|
||||
result: dict[str, Path] = {}
|
||||
with open(base / ".stage.lock", "w", encoding="utf-8") as held:
|
||||
fcntl.flock(held, fcntl.LOCK_EX)
|
||||
|
||||
def worker() -> None:
|
||||
result["root"] = resources.build_root()
|
||||
|
||||
t = threading.Thread(target=worker)
|
||||
t.start()
|
||||
# Let the worker miss the fast path (dest not yet complete) and
|
||||
# block on the held lock, then publish a complete tree as a peer
|
||||
# would have and release the lock.
|
||||
time.sleep(0.3)
|
||||
dest.mkdir(parents=True)
|
||||
(dest / ".complete").write_text("")
|
||||
fcntl.flock(held, fcntl.LOCK_UN)
|
||||
t.join(timeout=10)
|
||||
|
||||
self.assertEqual(dest, result["root"])
|
||||
self.assertFalse(t.is_alive())
|
||||
|
||||
def test_missing_bundle_raises(self):
|
||||
with tempfile.TemporaryDirectory() as tmpname:
|
||||
tmp = Path(tmpname)
|
||||
pkg = tmp / "bot_bottle"
|
||||
pkg.mkdir()
|
||||
restore = use_bottle_root(tmp / "appdata")
|
||||
self.addCleanup(restore)
|
||||
with patch.object(resources, "_PKG", pkg), \
|
||||
patch.object(resources, "_BUNDLED", pkg / "_resources"), \
|
||||
patch.object(resources, "_CHECKOUT_ROOT", tmp / "no-checkout"):
|
||||
with self.assertRaises(resources.ResourceError):
|
||||
resources.build_root()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -728,7 +728,7 @@ class TestResolvedRoutesPayload(unittest.TestCase):
|
||||
|
||||
|
||||
class TestNonBlockingSupervise(unittest.TestCase):
|
||||
"""PRD 0072 / issue #412: pending responses carry the proposal id, and
|
||||
"""PRD prd-new / issue #412: pending responses carry the proposal id, and
|
||||
`check-proposal` polls a queued proposal without blocking or re-proposing."""
|
||||
|
||||
_ROUTES = "routes:\n - host: example.com\n"
|
||||
|
||||
@@ -27,41 +27,7 @@ class TestCheckPullRequest(unittest.TestCase):
|
||||
event = {"pull_request": {"title": "Change", "body": "Part of #12", "labels": []}}
|
||||
self.assertEqual(check_pull_request(event, api), [])
|
||||
|
||||
def test_accepts_labelled_pr_without_issue(self):
|
||||
api = Mock()
|
||||
event = {
|
||||
"pull_request": {
|
||||
"title": "Change",
|
||||
"body": "",
|
||||
"labels": [{"name": "Kind/Documentation"}],
|
||||
}
|
||||
}
|
||||
self.assertEqual(check_pull_request(event, api), [])
|
||||
api.request.assert_not_called()
|
||||
|
||||
def test_rejects_unlabelled_pr_without_issue(self):
|
||||
api = Mock()
|
||||
event = {"pull_request": {"title": "Change", "body": "", "labels": []}}
|
||||
errors = check_pull_request(event, api)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("either have a label or reference an issue", errors[0])
|
||||
api.request.assert_not_called()
|
||||
|
||||
def test_rejects_labelled_pr_linked_to_real_issue(self):
|
||||
api = Mock()
|
||||
api.request.return_value = {"number": 12, "pull_request": None}
|
||||
event = {
|
||||
"pull_request": {
|
||||
"title": "Change",
|
||||
"body": "Closes #12",
|
||||
"labels": [{"name": "Kind/Documentation"}],
|
||||
}
|
||||
}
|
||||
errors = check_pull_request(event, api)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("exactly one tracking mode", errors[0])
|
||||
|
||||
def test_still_validates_issue_reference_when_both_modes_are_used(self):
|
||||
def test_rejects_labels_and_pr_reference(self):
|
||||
api = Mock()
|
||||
api.request.return_value = {"number": 12, "pull_request": {}}
|
||||
event = {
|
||||
@@ -73,7 +39,7 @@ class TestCheckPullRequest(unittest.TestCase):
|
||||
}
|
||||
errors = check_pull_request(event, api)
|
||||
self.assertEqual(len(errors), 2)
|
||||
self.assertIn("exactly one tracking mode", errors[0])
|
||||
self.assertIn("unlabeled", errors[0])
|
||||
self.assertIn("not an issue", errors[1])
|
||||
|
||||
|
||||
|
||||
@@ -6,13 +6,10 @@ import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle import orchestrator_auth
|
||||
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, ROLE_HOST
|
||||
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY
|
||||
from bot_bottle.trust_domain import (
|
||||
CONTROL_PLANE,
|
||||
HOST_CONTROLLER,
|
||||
LAUNCH_BROKER,
|
||||
ControlPlaneProvisioning,
|
||||
LaunchBrokerProvisioning,
|
||||
ProvisioningError,
|
||||
TrustDomain,
|
||||
)
|
||||
@@ -104,73 +101,5 @@ class TestControlPlaneProvisioning(unittest.TestCase):
|
||||
self.assertNotEqual(ROLE_CLI, CONTROL_PLANE.verify(tok, "k"))
|
||||
|
||||
|
||||
class TestLaunchBrokerAndHostControllerDomains(unittest.TestCase):
|
||||
"""The real #468 domains: the launch-broker key (shared by orchestrator +
|
||||
host controller) and the host controller's own lifecycle key."""
|
||||
|
||||
def test_launch_broker_mints_no_role_tokens(self) -> None:
|
||||
# Empty role set — it provides durable key material for the broker's own
|
||||
# launch JWT, not orchestrator_auth role tokens.
|
||||
self.assertEqual(frozenset(), LAUNCH_BROKER.roles)
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
with self.assertRaises(ValueError):
|
||||
LAUNCH_BROKER.mint(ROLE_CLI)
|
||||
|
||||
def test_host_controller_signs_host_role_only(self) -> None:
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
tok = HOST_CONTROLLER.mint(ROLE_HOST)
|
||||
self.assertEqual(ROLE_HOST, HOST_CONTROLLER.verify(tok, "k"))
|
||||
# A control-plane `cli` token (the orchestrator's key) never verifies as a
|
||||
# host-controller role — the orchestrator can't forge lifecycle creds.
|
||||
cli_tok = orchestrator_auth.mint(ROLE_CLI, "k")
|
||||
self.assertIsNone(HOST_CONTROLLER.verify(cli_tok, "k"))
|
||||
|
||||
def test_control_plane_cannot_mint_the_host_role(self) -> None:
|
||||
# `host` is outside the control-plane role set on purpose.
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||
with self.assertRaises(ValueError):
|
||||
CONTROL_PLANE.mint(ROLE_HOST)
|
||||
|
||||
def test_the_three_domains_use_distinct_keys_and_env_vars(self) -> None:
|
||||
self.assertEqual(3, len({
|
||||
CONTROL_PLANE.key_filename,
|
||||
LAUNCH_BROKER.key_filename,
|
||||
HOST_CONTROLLER.key_filename,
|
||||
}))
|
||||
self.assertEqual(3, len({
|
||||
CONTROL_PLANE.key_env, LAUNCH_BROKER.key_env, HOST_CONTROLLER.key_env,
|
||||
}))
|
||||
|
||||
|
||||
class TestLaunchBrokerProvisioning(unittest.TestCase):
|
||||
def test_broker_key_returns_the_durable_key(self) -> None:
|
||||
prov = LaunchBrokerProvisioning()
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value="bk"):
|
||||
self.assertEqual("bk", prov.broker_key())
|
||||
|
||||
def test_broker_key_fail_closes_when_empty(self) -> None:
|
||||
# An empty key would leave the host controller unable to verify any
|
||||
# launch — fail-closed rather than hand back a useless/dangerous key.
|
||||
prov = LaunchBrokerProvisioning()
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value=""):
|
||||
with self.assertRaises(ProvisioningError):
|
||||
prov.broker_key()
|
||||
|
||||
def test_controller_key_is_distinct_from_the_broker_key(self) -> None:
|
||||
# The orchestrator holds the broker key but NEVER the controller key.
|
||||
prov = LaunchBrokerProvisioning()
|
||||
keys = {"launch-broker-key": "bk", "host-controller-key": "ck"}
|
||||
with patch("bot_bottle.trust_domain.host_signing_key",
|
||||
side_effect=keys.__getitem__):
|
||||
self.assertEqual("bk", prov.broker_key())
|
||||
self.assertEqual("ck", prov.controller_key())
|
||||
|
||||
def test_controller_key_fail_closes_when_empty(self) -> None:
|
||||
prov = LaunchBrokerProvisioning()
|
||||
with patch("bot_bottle.trust_domain.host_signing_key", return_value=""):
|
||||
with self.assertRaises(ProvisioningError):
|
||||
prov.controller_key()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
"""Integration: build the wheel, install it into an isolated venv, and prove
|
||||
the installed distribution is self-contained.
|
||||
|
||||
This is the boundary a source-tree existence test can't reach (issue #197
|
||||
review): under an installed wheel the package lives in ``site-packages`` with
|
||||
no repo root above it, so anything resolving Dockerfiles / nix / scripts from
|
||||
``__file__``'s parents would break. Here we install for real and assert that
|
||||
``bot-bottle doctor`` runs from the console script and that
|
||||
``bot_bottle.resources`` stages a valid, repo-root-shaped build context.
|
||||
|
||||
It does NOT run `start` — building images needs a Docker/KVM host (CI). It
|
||||
skips cleanly when the build/venv toolchain isn't available.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class TestWheelInstall(unittest.TestCase):
|
||||
"""`build` is a declared dev dependency (requirements-dev.txt), so this runs
|
||||
in CI. A build/install failure is a real packaging regression and FAILS —
|
||||
only genuinely-unsupported infra (no `venv`/`ensurepip`) skips."""
|
||||
|
||||
_tmp: "tempfile.TemporaryDirectory[str]"
|
||||
venv_py: Path
|
||||
app_root: Path
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls._tmp = tempfile.TemporaryDirectory( # pylint: disable=consider-using-with
|
||||
prefix="bb-wheel-")
|
||||
tmp = Path(cls._tmp.name)
|
||||
dist = tmp / "dist"
|
||||
|
||||
# A failed wheel build is exactly the regression this test guards — fail,
|
||||
# don't skip. `build` is installed via requirements-dev.txt.
|
||||
built = subprocess.run(
|
||||
[sys.executable, "-m", "build", "--wheel", "--outdir", str(dist), str(REPO_ROOT)],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
if built.returncode != 0:
|
||||
raise AssertionError(f"wheel build failed:\n{built.stderr[-2000:]}")
|
||||
wheels = list(dist.glob("*.whl"))
|
||||
if not wheels:
|
||||
raise AssertionError(f"no wheel produced:\n{built.stdout[-2000:]}")
|
||||
|
||||
# A missing `venv`/`ensurepip` is unsupported optional infra, not a
|
||||
# packaging bug — skip only here.
|
||||
venv = tmp / "venv"
|
||||
made = subprocess.run([sys.executable, "-m", "venv", str(venv)],
|
||||
capture_output=True, text=True, check=False)
|
||||
if made.returncode != 0:
|
||||
raise unittest.SkipTest(f"venv/ensurepip unavailable:\n{made.stderr[-1500:]}")
|
||||
cls.venv_py = venv / "bin" / "python"
|
||||
|
||||
# Installing the freshly-built wheel must succeed — fail if it doesn't.
|
||||
install = subprocess.run(
|
||||
[str(cls.venv_py), "-m", "pip", "install", "--quiet", str(wheels[0])],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
if install.returncode != 0:
|
||||
raise AssertionError(f"pip install of the wheel failed:\n{install.stderr[-2000:]}")
|
||||
|
||||
# Isolate the staged build root the wheel writes under the app-data dir.
|
||||
cls.app_root = tmp / "appdata"
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls._tmp.cleanup()
|
||||
|
||||
def _run(self, tail: "list[str]") -> "subprocess.CompletedProcess[str]":
|
||||
"""Run the installed venv's python with `tail` appended, from a neutral
|
||||
cwd so the source checkout isn't on sys.path — we must import the
|
||||
*installed* package, not the repo we built from."""
|
||||
env = {"BOT_BOTTLE_ROOT": str(self.app_root), "PATH": "/usr/bin:/bin"}
|
||||
return subprocess.run(
|
||||
[str(self.venv_py), *tail],
|
||||
capture_output=True, text=True, env=env, cwd=self._tmp.name, check=False,
|
||||
)
|
||||
|
||||
def test_console_entry_point_installed(self):
|
||||
# The `bot-bottle` script the wheel declares must exist in the venv.
|
||||
script = self.venv_py.parent / "bot-bottle"
|
||||
self.assertTrue(script.is_file(), "bot-bottle console script not installed")
|
||||
|
||||
def test_doctor_runs_from_installed_package(self):
|
||||
proc = self._run(["-m", "bot_bottle.cli", "doctor"])
|
||||
# doctor exits non-zero here (no backend), but it must RUN and report.
|
||||
self.assertIn("python", proc.stdout)
|
||||
self.assertIn(proc.returncode, (0, 1))
|
||||
|
||||
def test_installed_wheel_is_self_contained(self):
|
||||
# From the installed layout (not a checkout), resources must resolve
|
||||
# Dockerfiles and stage a repo-root-shaped build context.
|
||||
script = (
|
||||
"import bot_bottle.resources as r\n"
|
||||
"assert not r.is_source_checkout(), 'should not look like a checkout'\n"
|
||||
"assert r.dockerfile('Dockerfile.gateway').is_file()\n"
|
||||
"assert r.nix_netpool_module().is_file()\n"
|
||||
"assert r.netpool_script().is_file()\n"
|
||||
"root = r.build_root()\n"
|
||||
"assert (root / 'bot_bottle' / '__init__.py').is_file(), 'no package in context'\n"
|
||||
"assert (root / 'pyproject.toml').is_file(), 'no pyproject in context'\n"
|
||||
"assert (root / 'Dockerfile.gateway').is_file(), 'no Dockerfile in context'\n"
|
||||
"print('SELF_CONTAINED_OK')\n"
|
||||
)
|
||||
proc = self._run(["-c", script])
|
||||
self.assertIn("SELF_CONTAINED_OK", proc.stdout, msg=proc.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user