Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 016e59e029 | |||
| f3664dea9f | |||
| 88b82a169e | |||
| 5a9428cc86 | |||
| 60039f2eb3 | |||
| bc42836327 | |||
| 0fc5457e41 | |||
| c0493f0b01 | |||
| 5828f5e900 | |||
| be025ff8fb | |||
| 9537c96586 | |||
| 6b43fe73c1 | |||
| d3370a88bb | |||
| 39167528db | |||
| b25ace4c00 |
@@ -0,0 +1,26 @@
|
||||
name: prd-number-check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, reopened, synchronize]
|
||||
|
||||
jobs:
|
||||
require-numbered-prds:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Reject unnumbered PRDs
|
||||
run: |
|
||||
unnumbered=$(find docs/prds -maxdepth 1 -type f \
|
||||
-name 'prd-new-*.md' -print | sort)
|
||||
|
||||
if [ -n "$unnumbered" ]; then
|
||||
echo "::error::Assign every new PRD its final sequential number before merge."
|
||||
echo "Unnumbered PRDs:"
|
||||
echo "$unnumbered"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "All PRDs have final numbers."
|
||||
@@ -1,122 +0,0 @@
|
||||
# Assign sequential numbers to prd-new-*.md files on merge to main.
|
||||
#
|
||||
# When a PR merges to main and includes prd-new-*.md files this workflow:
|
||||
# 1. Finds the next available NNNN number by scanning existing PRDs.
|
||||
# 2. Renames each prd-new-*.md to NNNN-<slug>.md.
|
||||
# 3. Updates the title header (# PRD prd-new: → # PRD NNNN:).
|
||||
# 4. Flips Status: Draft → Active when the push touched files outside
|
||||
# docs/prds/ anywhere in its commit range (i.e. the implementation
|
||||
# shipped together with the PRD).
|
||||
# 5. Commits the renaming back to main.
|
||||
#
|
||||
# No-op if the working tree contains no prd-new-*.md files.
|
||||
#
|
||||
# NOTE: The workflow scans the working tree (not just HEAD~1..HEAD) because
|
||||
# PRs land as multi-commit pushes and the prd-new file is often added in an
|
||||
# earlier commit on the branch, not in the final squash/merge commit.
|
||||
|
||||
name: prd-number
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'docs/prds/prd-new-*.md'
|
||||
|
||||
jobs:
|
||||
assign-numbers:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# No actions/setup-python: the inline script is stdlib-only on the
|
||||
# image's system Python 3.12 (older act_runner mishandles its PATH).
|
||||
- name: Configure git
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Assign PRD numbers
|
||||
run: |
|
||||
python3 - <<'EOF'
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
prds_dir = Path("docs/prds")
|
||||
|
||||
# Scan the working tree — prd-new files may have landed in any
|
||||
# commit of a multi-commit push, not just HEAD.
|
||||
new_prds = sorted(prds_dir.glob("prd-new-*.md"))
|
||||
|
||||
if not new_prds:
|
||||
print("No prd-new-*.md files found — nothing to do.")
|
||||
sys.exit(0)
|
||||
|
||||
# Determine whether non-PRD files were also changed anywhere in
|
||||
# the push range (BEFORE_SHA → HEAD). Falls back to HEAD~1 when
|
||||
# the env var isn't set (e.g. local act runs).
|
||||
before_sha = os.environ.get("GITHUB_EVENT_BEFORE", "HEAD~1")
|
||||
all_changed = subprocess.run(
|
||||
["git", "diff", "--name-only", before_sha, "HEAD"],
|
||||
capture_output=True, text=True, check=True,
|
||||
).stdout.splitlines()
|
||||
non_prd_changed = any(
|
||||
not f.startswith("docs/prds/") for f in all_changed
|
||||
)
|
||||
|
||||
# Find next available number.
|
||||
existing = sorted(
|
||||
int(m.group(1))
|
||||
for p in prds_dir.glob("*.md")
|
||||
if (m := re.match(r"^(\d{4})-", p.name))
|
||||
)
|
||||
next_num = (max(existing) + 1) if existing else 1
|
||||
|
||||
for prd_path in sorted(new_prds):
|
||||
slug = re.sub(r"^prd-new-", "", prd_path.stem)
|
||||
new_name = f"{next_num:04d}-{slug}.md"
|
||||
new_path = prds_dir / new_name
|
||||
print(f" {prd_path.name} → {new_name}")
|
||||
|
||||
content = prd_path.read_text()
|
||||
|
||||
# Update title header.
|
||||
content = re.sub(
|
||||
r"^(#\s+PRD\s+)prd-new(:)",
|
||||
rf"\g<1>{next_num:04d}\2",
|
||||
content,
|
||||
count=1,
|
||||
flags=re.MULTILINE,
|
||||
)
|
||||
|
||||
# Conditionally flip Status.
|
||||
if non_prd_changed:
|
||||
content = re.sub(
|
||||
r"(\*\*Status:\*\*\s*)Draft",
|
||||
r"\g<1>Active",
|
||||
content,
|
||||
count=1,
|
||||
)
|
||||
|
||||
new_path.write_text(content)
|
||||
subprocess.run(["git", "rm", str(prd_path)], check=True)
|
||||
subprocess.run(["git", "add", str(new_path)], check=True)
|
||||
next_num += 1
|
||||
|
||||
subprocess.run(
|
||||
["git", "commit", "-m", "ci(prd): assign sequential numbers to new PRDs"],
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(["git", "push"], check=True)
|
||||
EOF
|
||||
@@ -0,0 +1,337 @@
|
||||
# Run the complete backend test suite before a release. This workflow is
|
||||
# intentionally manual because Firecracker and macOS use privileged,
|
||||
# self-hosted runners.
|
||||
#
|
||||
# The suite uses stdlib `unittest` discovery — no external Python
|
||||
# dependencies are required to execute it. Tests are split by directory:
|
||||
#
|
||||
# tests/unit/ — pure unit tests; always run
|
||||
# tests/integration/ — need a reachable backend; skip cleanly when
|
||||
# the backend isn't available on the runner
|
||||
# tests/canaries/ — upstream regression canaries; run on a separate
|
||||
# schedule (see canaries.yml), not here
|
||||
#
|
||||
# Unit, Docker, and Firecracker run once under coverage and upload a small
|
||||
# .coverage.* artifact for the combined coverage job. macOS reports coverage
|
||||
# in place because it is an advisory host-mode runner.
|
||||
|
||||
name: pre-release-test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
unit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# No actions/setup-python: the runner image already ships Python 3.12,
|
||||
# and older act_runner engines mishandle setup-python's PATH (coverage
|
||||
# lands in one interpreter, `python3` resolves to another). Install
|
||||
# straight into the ephemeral job container's system Python —
|
||||
# --break-system-packages is safe because the container is disposable.
|
||||
- name: Install dev requirements
|
||||
run: python3 -m pip install --break-system-packages -r requirements-dev.txt
|
||||
|
||||
- name: Run unit tests with coverage
|
||||
env:
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.unit
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/unit -v
|
||||
|
||||
- name: Report unit coverage
|
||||
env:
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.unit
|
||||
run: python3 -m coverage report -m
|
||||
|
||||
# upload-artifact@v3's glob skips dotfiles, so a bare `.coverage.unit`
|
||||
# silently uploads nothing ("No files were found"). Stage it under a
|
||||
# non-dot name; the coverage job renames it back before `coverage
|
||||
# combine`. `cp` also fails loudly if coverage never wrote the file.
|
||||
- name: Stage unit coverage for upload
|
||||
run: cp .coverage.unit coverage-unit.dat
|
||||
|
||||
- name: Upload unit coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-unit
|
||||
path: coverage-unit.dat
|
||||
|
||||
integration-docker:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# No actions/setup-python (see the note in the `unit` job); the
|
||||
# container's system Python 3.12 runs the stdlib test suite directly.
|
||||
- name: Install coverage
|
||||
run: python3 -m pip install --break-system-packages coverage
|
||||
|
||||
# Fail loudly if the backend this job promises isn't actually usable,
|
||||
# rather than letting every test silently `unittest.skip` and the job
|
||||
# go green on zero coverage. `backend status` prints a clear per-check
|
||||
# summary (docker on PATH, daemon reachable) and exits non-zero when a
|
||||
# prerequisite is missing — the same readiness check the skip guards
|
||||
# gate on via `has_backend`.
|
||||
- name: Preflight — Docker backend is ready
|
||||
run: |
|
||||
python3 --version
|
||||
python3 cli.py backend status --backend=docker
|
||||
|
||||
- name: Run integration tests (docker) with coverage
|
||||
env:
|
||||
BOT_BOTTLE_BACKEND: docker
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.docker
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
|
||||
- name: Stage docker coverage for upload
|
||||
run: cp .coverage.docker coverage-docker.dat
|
||||
|
||||
- name: Upload docker coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-docker
|
||||
path: coverage-docker.dat
|
||||
|
||||
# Integration tests against the Firecracker backend. Runs on a self-hosted
|
||||
# KVM runner (label `kvm`) where /dev/kvm and the TAP/nft pool are available.
|
||||
#
|
||||
# Manual only: the privileged KVM runner does not execute proposed changes
|
||||
# unattended.
|
||||
#
|
||||
# Runner prerequisites (provision once; see README "Firecracker on Linux"):
|
||||
# `firecracker` on PATH, `/dev/kvm` accessible, cached kernel +
|
||||
# static dropbear at /var/cache/bot-bottle-fc/dropbear, and the pool as a
|
||||
# persistent systemd unit.
|
||||
#
|
||||
# The infra candidate is built here directly (no artifact download) to
|
||||
# eliminate the ~70 s ubuntu-latest upload + ~83 s combined download that
|
||||
# the old build-infra → integration-firecracker + coverage chain incurred.
|
||||
integration-firecracker:
|
||||
runs-on: [self-hosted, kvm]
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Preflight — Firecracker host is ready
|
||||
run: |
|
||||
command -v firecracker >/dev/null || {
|
||||
echo "firecracker not on PATH — provision the runner (README: Firecracker on Linux)"; exit 1; }
|
||||
test -e /dev/kvm || { echo "/dev/kvm missing — KVM not available on this runner"; exit 1; }
|
||||
# `backend status` exits non-zero unless the TAP pool is up + no
|
||||
# range overlap; it prints the exact `backend setup` fix.
|
||||
python3 cli.py backend status --backend=firecracker
|
||||
|
||||
- name: Build infra candidate from this checkout
|
||||
env:
|
||||
BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear
|
||||
run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate --reuse-published
|
||||
|
||||
- name: Replace the persistent infra VM with the candidate
|
||||
run: python3 -c 'from bot_bottle.backend.firecracker import infra_vm; infra_vm.stop()'
|
||||
|
||||
# No dev-requirements install: `coverage` is already provided by the
|
||||
# self-hosted runner's Nix python env, and that env has no `pip`
|
||||
# module to install into anyway.
|
||||
- name: Run integration tests (firecracker) with coverage
|
||||
env:
|
||||
BOT_BOTTLE_BACKEND: firecracker
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.firecracker
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
- name: Stage firecracker coverage for upload
|
||||
run: cp .coverage.firecracker coverage-firecracker.dat
|
||||
|
||||
- name: Upload firecracker coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-firecracker
|
||||
path: coverage-firecracker.dat
|
||||
|
||||
- name: Upload tested rootfs
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate/
|
||||
|
||||
- name: Upload dropbear for publish verification
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: /var/cache/bot-bottle-fc/dropbear
|
||||
|
||||
# Integration tests against the macOS Apple Container backend. Runs on a
|
||||
# self-hosted macOS runner (label `macos`) registered in HOST mode — Apple
|
||||
# Container needs the host `container` CLI + virtualization framework and
|
||||
# cannot run inside a Linux container, so this cannot reuse the KVM runner.
|
||||
#
|
||||
# Advisory only: workflow_dispatch (manual) exclusively — never push or
|
||||
# pull_request. A single non-redundant laptop that sleeps/roams must not run
|
||||
# unattended on every push to main, let alone block a PR merge, so this job is
|
||||
# deliberately NOT in the `coverage` job's `needs` and its coverage never
|
||||
# feeds the diff-coverage gate. Dispatch-only also means no fork PR (or any
|
||||
# push) ever executes on the host-mode runner.
|
||||
#
|
||||
# The infra container is a singleton (`bot-bottle-mac-infra`); the
|
||||
# `concurrency` group serializes runs so two never collide on it (#425), and
|
||||
# the always-run teardown removes it so a crashed run can't wedge the next.
|
||||
#
|
||||
# Runner prerequisites (provision once; see README "macOS Apple Container"):
|
||||
# the `container` CLI on PATH with `container system status` running, and a
|
||||
# Python >=3.11 with `coverage` importable on the launchd service PATH.
|
||||
integration-macos:
|
||||
runs-on: [self-hosted, macos]
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
concurrency:
|
||||
group: integration-macos-infra
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Fail loudly if the backend this job promises isn't actually usable,
|
||||
# rather than letting every test silently `unittest.skip` and the job go
|
||||
# green on zero coverage. `backend status` exits non-zero (and prints the
|
||||
# per-check summary) when the `container` CLI or its system service is
|
||||
# missing — the same readiness check the skip guards gate on.
|
||||
- name: Preflight — Apple Container backend is ready
|
||||
run: |
|
||||
command -v container >/dev/null || {
|
||||
echo "container CLI not on PATH — provision the runner (README: macOS Apple Container)"; exit 1; }
|
||||
container system status || {
|
||||
echo "container system service not running — run 'container system start'"; exit 1; }
|
||||
python3 cli.py backend status --backend=macos-container
|
||||
|
||||
# `coverage` comes from the runner's provisioned Python (no pip install
|
||||
# into the host interpreter). Advisory job: report coverage in-line for
|
||||
# visibility but don't upload — it never feeds the combined gate.
|
||||
- name: Run integration tests (macos-container) with coverage
|
||||
env:
|
||||
BOT_BOTTLE_BACKEND: macos-container
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.macos
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
- name: Report macos coverage
|
||||
env:
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.macos
|
||||
run: python3 -m coverage report -m
|
||||
|
||||
# On failure, capture the infra containers' state and logs BEFORE the
|
||||
# teardown below removes them — otherwise a control-plane crash is
|
||||
# undiagnosable from CI, since `stop()` deletes the orchestrator (and its
|
||||
# logs) on every run. Best-effort: never let the diagnostics themselves
|
||||
# fail the job, and keep going if a container is already gone.
|
||||
- name: Dump infra diagnostics (on failure)
|
||||
if: failure()
|
||||
run: |
|
||||
set +e
|
||||
echo "=== containers ==="
|
||||
container ls -a | grep bot-bottle-mac || echo "(no bot-bottle-mac containers)"
|
||||
echo "=== networks ==="
|
||||
container network ls | grep bot-bottle-mac || echo "(no bot-bottle-mac networks)"
|
||||
for c in bot-bottle-mac-orchestrator bot-bottle-mac-infra; do
|
||||
echo "=== inspect $c ==="
|
||||
container inspect "$c" || echo "($c not found)"
|
||||
echo "=== logs $c ==="
|
||||
container logs "$c" || echo "($c logs unavailable)"
|
||||
done
|
||||
exit 0
|
||||
|
||||
# Remove the singleton infra container so a crashed or cancelled run
|
||||
# cannot leave `bot-bottle-mac-infra` wedged for the next job.
|
||||
- name: Teardown infra singleton
|
||||
if: always()
|
||||
run: python3 -c 'from bot_bottle.backend.macos_container.infra import MacosInfraService; MacosInfraService().stop()'
|
||||
|
||||
# Combined coverage gate: aggregates .coverage.* artifacts uploaded by each
|
||||
# test job, then runs the diff-coverage gate (new/changed lines >= 90%).
|
||||
#
|
||||
# Runs on ubuntu-latest — no KVM needed, no test reruns. Coverage files use
|
||||
# relative_files = True (.coveragerc) so they combine cleanly across runners.
|
||||
# Each test job sets COVERAGE_FILE to an absolute path so coverage.py writes
|
||||
# to a known location that upload-artifact can find regardless of runner env.
|
||||
#
|
||||
coverage:
|
||||
needs: [unit, integration-docker, integration-firecracker]
|
||||
timeout-minutes: 15
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install coverage
|
||||
run: python3 -m pip install --break-system-packages coverage
|
||||
|
||||
- name: Download unit coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: coverage-unit
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
- name: Download docker coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: coverage-docker
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
- name: Download firecracker coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: coverage-firecracker
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
# Rename the non-dot upload names back to the .coverage.* files that
|
||||
# `coverage combine` discovers (see the staging steps in each test job).
|
||||
- name: Reassemble coverage data files
|
||||
run: |
|
||||
mv coverage-unit.dat .coverage.unit
|
||||
mv coverage-docker.dat .coverage.docker
|
||||
mv coverage-firecracker.dat .coverage.firecracker
|
||||
|
||||
- name: Combined coverage (unit + integration, incl. firecracker)
|
||||
run: PYTHON=python3 bash scripts/coverage.sh aggregate critical
|
||||
|
||||
- name: Diff-coverage gate (changed lines >= 90%)
|
||||
run: |
|
||||
git fetch --no-tags origin main:refs/remotes/origin/main
|
||||
python3 scripts/diff_coverage.py --base origin/main --min 90
|
||||
|
||||
publish-infra:
|
||||
needs:
|
||||
- unit
|
||||
- integration-docker
|
||||
- integration-firecracker
|
||||
- integration-macos
|
||||
- coverage
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout the tested revision
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download the tested rootfs
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate
|
||||
|
||||
# publish_infra re-derives the version from the checkout to confirm the
|
||||
# bundle matches before uploading, and the version hashes the dropbear
|
||||
# bytes. Download the same dropbear integration-firecracker used.
|
||||
- name: Download the staged dropbear
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: firecracker-inputs
|
||||
|
||||
- name: Publish the tested candidate
|
||||
env:
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
|
||||
BOT_BOTTLE_FC_DROPBEAR: ${{ github.workspace }}/firecracker-inputs/dropbear
|
||||
run: python3 -m bot_bottle.backend.firecracker.publish_infra --publish-dir infra-candidate
|
||||
+7
-259
@@ -1,21 +1,6 @@
|
||||
# Run the project's test suite when package or runtime inputs change on a PR
|
||||
# or on push to main.
|
||||
#
|
||||
# The suite uses stdlib `unittest` discovery — no external Python
|
||||
# dependencies are required to execute it. Tests are split by directory:
|
||||
#
|
||||
# tests/unit/ — pure unit tests; always run
|
||||
# tests/integration/ — need a reachable backend; skip cleanly when
|
||||
# the backend isn't available on the runner
|
||||
# tests/canaries/ — upstream regression canaries; run on a separate
|
||||
# schedule (see canaries.yml), not here
|
||||
#
|
||||
# Each test job runs once under coverage and uploads a small .coverage.*
|
||||
# artifact. The `coverage` job combines them — no test reruns, no KVM
|
||||
# dependency on that job. For main-branch pushes only, the tested rootfs
|
||||
# and matching dropbear are uploaded so `publish-infra` can publish the
|
||||
# byte-identical artifact that was tested. PRs avoid the ~194 MB rootfs
|
||||
# transfer entirely.
|
||||
# Run the automated test gate when package or runtime inputs change on a PR
|
||||
# or on push to main. Privileged self-hosted backends live in the manually
|
||||
# dispatched pre-release-test workflow.
|
||||
|
||||
name: test
|
||||
|
||||
@@ -37,6 +22,7 @@ on:
|
||||
- 'requirements-dev.txt'
|
||||
- '.coveragerc'
|
||||
- '.dockerignore'
|
||||
- '.gitea/workflows/test.yml'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'bot_bottle/**'
|
||||
@@ -52,7 +38,7 @@ on:
|
||||
- 'requirements-dev.txt'
|
||||
- '.coveragerc'
|
||||
- '.dockerignore'
|
||||
workflow_dispatch:
|
||||
- '.gitea/workflows/test.yml'
|
||||
|
||||
jobs:
|
||||
unit:
|
||||
@@ -61,11 +47,6 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# No actions/setup-python: the runner image already ships Python 3.12,
|
||||
# and older act_runner engines mishandle setup-python's PATH (coverage
|
||||
# lands in one interpreter, `python3` resolves to another). Install
|
||||
# straight into the ephemeral job container's system Python —
|
||||
# --break-system-packages is safe because the container is disposable.
|
||||
- name: Install dev requirements
|
||||
run: python3 -m pip install --break-system-packages -r requirements-dev.txt
|
||||
|
||||
@@ -79,10 +60,6 @@ jobs:
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.unit
|
||||
run: python3 -m coverage report -m
|
||||
|
||||
# upload-artifact@v3's glob skips dotfiles, so a bare `.coverage.unit`
|
||||
# silently uploads nothing ("No files were found"). Stage it under a
|
||||
# non-dot name; the coverage job renames it back before `coverage
|
||||
# combine`. `cp` also fails loudly if coverage never wrote the file.
|
||||
- name: Stage unit coverage for upload
|
||||
run: cp .coverage.unit coverage-unit.dat
|
||||
|
||||
@@ -98,17 +75,9 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# No actions/setup-python (see the note in the `unit` job); the
|
||||
# container's system Python 3.12 runs the stdlib test suite directly.
|
||||
- name: Install coverage
|
||||
run: python3 -m pip install --break-system-packages coverage
|
||||
|
||||
# Fail loudly if the backend this job promises isn't actually usable,
|
||||
# rather than letting every test silently `unittest.skip` and the job
|
||||
# go green on zero coverage. `backend status` prints a clear per-check
|
||||
# summary (docker on PATH, daemon reachable) and exits non-zero when a
|
||||
# prerequisite is missing — the same readiness check the skip guards
|
||||
# gate on via `has_backend`.
|
||||
- name: Preflight — Docker backend is ready
|
||||
run: |
|
||||
python3 --version
|
||||
@@ -120,7 +89,6 @@ jobs:
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.docker
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
|
||||
- name: Stage docker coverage for upload
|
||||
run: cp .coverage.docker coverage-docker.dat
|
||||
|
||||
@@ -130,190 +98,10 @@ jobs:
|
||||
name: coverage-docker
|
||||
path: coverage-docker.dat
|
||||
|
||||
# Integration tests against the Firecracker backend. Runs on a self-hosted
|
||||
# KVM runner (label `kvm`) where /dev/kvm and the TAP/nft pool are available.
|
||||
#
|
||||
# Restricted to same-repo PRs, push to main, and workflow_dispatch — fork
|
||||
# PRs don't execute untrusted code on the privileged runner.
|
||||
#
|
||||
# Runner prerequisites (provision once; see README "Firecracker on Linux"):
|
||||
# `firecracker` on PATH, `/dev/kvm` accessible, cached kernel +
|
||||
# static dropbear at /var/cache/bot-bottle-fc/dropbear, and the pool as a
|
||||
# persistent systemd unit.
|
||||
#
|
||||
# The infra candidate is built here directly (no artifact download) to
|
||||
# eliminate the ~70 s ubuntu-latest upload + ~83 s combined download that
|
||||
# the old build-infra → integration-firecracker + coverage chain incurred.
|
||||
# For main-branch pushes the tested rootfs and matching dropbear are
|
||||
# uploaded so publish-infra can publish the byte-identical artifact; PRs
|
||||
# skip those uploads entirely.
|
||||
integration-firecracker:
|
||||
runs-on: [self-hosted, kvm]
|
||||
if: >-
|
||||
github.event_name == 'push' ||
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'pull_request' &&
|
||||
github.event.pull_request.head.repo.full_name == github.repository)
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Preflight — Firecracker host is ready
|
||||
run: |
|
||||
command -v firecracker >/dev/null || {
|
||||
echo "firecracker not on PATH — provision the runner (README: Firecracker on Linux)"; exit 1; }
|
||||
test -e /dev/kvm || { echo "/dev/kvm missing — KVM not available on this runner"; exit 1; }
|
||||
# `backend status` exits non-zero unless the TAP pool is up + no
|
||||
# range overlap; it prints the exact `backend setup` fix.
|
||||
python3 cli.py backend status --backend=firecracker
|
||||
|
||||
- name: Build infra candidate from this checkout
|
||||
env:
|
||||
BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear
|
||||
run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate --reuse-published
|
||||
|
||||
- name: Replace the persistent infra VM with the candidate
|
||||
run: python3 -c 'from bot_bottle.backend.firecracker import infra_vm; infra_vm.stop()'
|
||||
|
||||
# No dev-requirements install: `coverage` is already provided by the
|
||||
# self-hosted runner's Nix python env, and that env has no `pip`
|
||||
# module to install into anyway.
|
||||
- name: Run integration tests (firecracker) with coverage
|
||||
env:
|
||||
BOT_BOTTLE_BACKEND: firecracker
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.firecracker
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
|
||||
- name: Stage firecracker coverage for upload
|
||||
run: cp .coverage.firecracker coverage-firecracker.dat
|
||||
|
||||
- name: Upload firecracker coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-firecracker
|
||||
path: coverage-firecracker.dat
|
||||
|
||||
# Only upload the large rootfs artifact on main-branch pushes;
|
||||
# PRs avoid the ~194 MB transfer. publish-infra only runs on main
|
||||
# and downloads these to publish the byte-identical tested rootfs.
|
||||
- name: Upload tested rootfs (main branch only)
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate/
|
||||
|
||||
- name: Upload dropbear for publish verification (main branch only)
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: /var/cache/bot-bottle-fc/dropbear
|
||||
|
||||
# Integration tests against the macOS Apple Container backend. Runs on a
|
||||
# self-hosted macOS runner (label `macos`) registered in HOST mode — Apple
|
||||
# Container needs the host `container` CLI + virtualization framework and
|
||||
# cannot run inside a Linux container, so this cannot reuse the KVM runner.
|
||||
#
|
||||
# Advisory only: workflow_dispatch (manual) exclusively — never push or
|
||||
# pull_request. A single non-redundant laptop that sleeps/roams must not run
|
||||
# unattended on every push to main, let alone block a PR merge, so this job is
|
||||
# deliberately NOT in the `coverage` job's `needs` and its coverage never
|
||||
# feeds the diff-coverage gate. Dispatch-only also means no fork PR (or any
|
||||
# push) ever executes on the host-mode runner.
|
||||
#
|
||||
# The infra container is a singleton (`bot-bottle-mac-infra`); the
|
||||
# `concurrency` group serializes runs so two never collide on it (#425), and
|
||||
# the always-run teardown removes it so a crashed run can't wedge the next.
|
||||
#
|
||||
# Runner prerequisites (provision once; see README "macOS Apple Container"):
|
||||
# the `container` CLI on PATH with `container system status` running, and a
|
||||
# Python >=3.11 with `coverage` importable on the launchd service PATH.
|
||||
integration-macos:
|
||||
runs-on: [self-hosted, macos]
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
concurrency:
|
||||
group: integration-macos-infra
|
||||
cancel-in-progress: false
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Fail loudly if the backend this job promises isn't actually usable,
|
||||
# rather than letting every test silently `unittest.skip` and the job go
|
||||
# green on zero coverage. `backend status` exits non-zero (and prints the
|
||||
# per-check summary) when the `container` CLI or its system service is
|
||||
# missing — the same readiness check the skip guards gate on.
|
||||
- name: Preflight — Apple Container backend is ready
|
||||
run: |
|
||||
command -v container >/dev/null || {
|
||||
echo "container CLI not on PATH — provision the runner (README: macOS Apple Container)"; exit 1; }
|
||||
container system status || {
|
||||
echo "container system service not running — run 'container system start'"; exit 1; }
|
||||
python3 cli.py backend status --backend=macos-container
|
||||
|
||||
# `coverage` comes from the runner's provisioned Python (no pip install
|
||||
# into the host interpreter). Advisory job: report coverage in-line for
|
||||
# visibility but don't upload — it never feeds the combined gate.
|
||||
- name: Run integration tests (macos-container) with coverage
|
||||
env:
|
||||
BOT_BOTTLE_BACKEND: macos-container
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.macos
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
- name: Report macos coverage
|
||||
env:
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.macos
|
||||
run: python3 -m coverage report -m
|
||||
|
||||
# On failure, capture the infra containers' state and logs BEFORE the
|
||||
# teardown below removes them — otherwise a control-plane crash is
|
||||
# undiagnosable from CI, since `stop()` deletes the orchestrator (and its
|
||||
# logs) on every run. Best-effort: never let the diagnostics themselves
|
||||
# fail the job, and keep going if a container is already gone.
|
||||
- name: Dump infra diagnostics (on failure)
|
||||
if: failure()
|
||||
run: |
|
||||
set +e
|
||||
echo "=== containers ==="
|
||||
container ls -a | grep bot-bottle-mac || echo "(no bot-bottle-mac containers)"
|
||||
echo "=== networks ==="
|
||||
container network ls | grep bot-bottle-mac || echo "(no bot-bottle-mac networks)"
|
||||
for c in bot-bottle-mac-orchestrator bot-bottle-mac-infra; do
|
||||
echo "=== inspect $c ==="
|
||||
container inspect "$c" || echo "($c not found)"
|
||||
echo "=== logs $c ==="
|
||||
container logs "$c" || echo "($c logs unavailable)"
|
||||
done
|
||||
exit 0
|
||||
|
||||
# Remove the singleton infra container so a crashed or cancelled run
|
||||
# cannot leave `bot-bottle-mac-infra` wedged for the next job.
|
||||
- name: Teardown infra singleton
|
||||
if: always()
|
||||
run: python3 -c 'from bot_bottle.backend.macos_container.infra import MacosInfraService; MacosInfraService().stop()'
|
||||
|
||||
# Combined coverage gate: aggregates .coverage.* artifacts uploaded by each
|
||||
# test job, then runs the diff-coverage gate (new/changed lines >= 90%).
|
||||
#
|
||||
# Runs on ubuntu-latest — no KVM needed, no test reruns. Coverage files use
|
||||
# relative_files = True (.coveragerc) so they combine cleanly across runners.
|
||||
# Each test job sets COVERAGE_FILE to an absolute path so coverage.py writes
|
||||
# to a known location that upload-artifact can find regardless of runner env.
|
||||
#
|
||||
# Restricted to the same events as integration-firecracker: it depends on
|
||||
# that job's coverage artifact and skips for fork PRs alongside it.
|
||||
coverage:
|
||||
needs: [unit, integration-docker, integration-firecracker]
|
||||
needs: [unit, integration-docker]
|
||||
timeout-minutes: 15
|
||||
runs-on: ubuntu-latest
|
||||
if: >-
|
||||
github.event_name == 'push' ||
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'pull_request' &&
|
||||
github.event.pull_request.head.repo.full_name == github.repository)
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
@@ -335,55 +123,15 @@ jobs:
|
||||
name: coverage-docker
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
- name: Download firecracker coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: coverage-firecracker
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
# Rename the non-dot upload names back to the .coverage.* files that
|
||||
# `coverage combine` discovers (see the staging steps in each test job).
|
||||
- name: Reassemble coverage data files
|
||||
run: |
|
||||
mv coverage-unit.dat .coverage.unit
|
||||
mv coverage-docker.dat .coverage.docker
|
||||
mv coverage-firecracker.dat .coverage.firecracker
|
||||
|
||||
- name: Combined coverage (unit + integration, incl. firecracker)
|
||||
- name: Combined coverage (unit + docker integration)
|
||||
run: PYTHON=python3 bash scripts/coverage.sh aggregate critical
|
||||
|
||||
- name: Diff-coverage gate (changed lines >= 90%)
|
||||
run: |
|
||||
git fetch --no-tags origin main:refs/remotes/origin/main
|
||||
python3 scripts/diff_coverage.py --base origin/main --min 90
|
||||
|
||||
publish-infra:
|
||||
needs: [unit, integration-docker, integration-firecracker, coverage]
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Checkout the tested revision
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download the tested rootfs
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate
|
||||
|
||||
# publish_infra re-derives the version from the checkout to confirm the
|
||||
# bundle matches before uploading, and the version hashes the dropbear
|
||||
# bytes. Download the SAME dropbear integration-firecracker used, or
|
||||
# the recheck computes a "<missing>"-dropbear version and rejects the
|
||||
# candidate.
|
||||
- name: Download the staged dropbear (matches build's version)
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: firecracker-inputs
|
||||
|
||||
- name: Publish the tested candidate
|
||||
env:
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_TOKEN: ${{ secrets.BOT_BOTTLE_INFRA_ARTIFACT_TOKEN }}
|
||||
BOT_BOTTLE_FC_DROPBEAR: ${{ github.workspace }}/firecracker-inputs/dropbear
|
||||
run: python3 -m bot_bottle.backend.firecracker.publish_infra --publish-dir infra-candidate
|
||||
|
||||
@@ -44,10 +44,10 @@ backend remains available with `BOT_BOTTLE_BACKEND=docker` or
|
||||
|
||||
- Three kinds of doc, each with its own conventions in-folder; see
|
||||
`docs/README.md` for when to write which:
|
||||
- **PRDs** (`docs/prds/`) — one feature per file. While a PR is open
|
||||
the file is named `prd-new-<kebab>.md`; CI assigns a sequential
|
||||
number on merge to `main` and renames it. A `Status:` line tracks
|
||||
lifecycle: Draft → Active (shipped to `main`) →
|
||||
- **PRDs** (`docs/prds/`) — one feature per file. A draft may initially
|
||||
use `prd-new-<kebab>.md`, but its author must assign the next
|
||||
sequential number before merge; CI rejects unnumbered PRDs. A
|
||||
`Status:` line tracks lifecycle: Draft → Active (shipped to `main`) →
|
||||
Superseded/Retargeted. Format in `docs/prds/README.md`.
|
||||
- **Research notes** (`docs/research/`) — opinionated investigations;
|
||||
unnumbered kebab-case, freeform and verdict-first. See
|
||||
|
||||
+75
-11
@@ -23,14 +23,14 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Generator, Generic, Sequence, TypeVar
|
||||
|
||||
from ..agent_provider import AgentProvisionPlan, get_provider
|
||||
from ..agent_provider import AgentProvisionPlan, get_provider, build_agent_provision_plan
|
||||
from ..egress import EgressPlan
|
||||
from ..git_gate import GitGatePlan
|
||||
from ..log import die, info
|
||||
from ..util import expand_tilde
|
||||
from ..manifest import Manifest, ManifestIndex
|
||||
from ..supervisor.plan import SupervisePlan
|
||||
from ..env import ResolvedEnv
|
||||
from ..env import resolve_env, ResolvedEnv
|
||||
from ..workspace import WorkspacePlan, workspace_plan
|
||||
from .print_util import print_multi, visible_agent_env_names
|
||||
from .util import host_skill_dir
|
||||
@@ -296,18 +296,82 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]):
|
||||
backend-specific resolution (names, scratch files, etc.). The
|
||||
validation step is enforced here so a future backend cannot
|
||||
accidentally skip it. No remote/runtime resources are created."""
|
||||
from .preparation import BottlePreparationPlanner
|
||||
prepared = BottlePreparationPlanner(self).prepare(spec)
|
||||
from .resolve_common import (
|
||||
merge_provision_env_vars,
|
||||
mint_slug,
|
||||
prepare_agent_state_dir,
|
||||
prepare_egress,
|
||||
prepare_git_gate,
|
||||
prepare_supervise,
|
||||
reject_nested_containers,
|
||||
resolve_manifest_dockerfile,
|
||||
write_launch_metadata,
|
||||
)
|
||||
|
||||
manifest = self._validate(spec)
|
||||
|
||||
if not self.supports_nested_containers:
|
||||
reject_nested_containers(self.name, manifest)
|
||||
|
||||
self._preflight()
|
||||
|
||||
from ..git_gate import GitGate
|
||||
manifest = GitGate().preflight_host_keys(
|
||||
manifest,
|
||||
headless=spec.headless,
|
||||
home_md=spec.manifest.home_md,
|
||||
)
|
||||
|
||||
manifest_bottle = manifest.bottle
|
||||
manifest_agent_provider = manifest_bottle.agent_provider
|
||||
agent_provider = get_provider(manifest_agent_provider.template)
|
||||
resolved_env = resolve_env(manifest)
|
||||
workspace = workspace_plan(spec, guest_home=agent_provider.guest_home)
|
||||
|
||||
slug = mint_slug(spec)
|
||||
write_launch_metadata(slug, spec, compose_project="", backend=self.name)
|
||||
|
||||
# Manifest may override the Dockerfile per-bottle; otherwise fall
|
||||
# back to the provider plugin's bundled Dockerfile (next to its
|
||||
# agent_provider.py module).
|
||||
if manifest_agent_provider.dockerfile:
|
||||
agent_dockerfile_path = resolve_manifest_dockerfile(
|
||||
manifest_agent_provider.dockerfile, spec,
|
||||
)
|
||||
else:
|
||||
agent_dockerfile_path = str(agent_provider.dockerfile)
|
||||
|
||||
agent_dir, prompt_file = prepare_agent_state_dir(slug, manifest)
|
||||
|
||||
agent_provision_plan = build_agent_provision_plan(
|
||||
template=manifest_agent_provider.template,
|
||||
dockerfile=agent_dockerfile_path,
|
||||
state_dir=agent_dir,
|
||||
instance_name=f"bot-bottle-{slug}",
|
||||
prompt_file=prompt_file,
|
||||
guest_env=self._build_guest_env(resolved_env),
|
||||
forward_host_credentials=manifest_agent_provider.forward_host_credentials,
|
||||
auth_token=manifest_agent_provider.auth_token,
|
||||
host_env=dict(os.environ),
|
||||
trusted_project_path=workspace.workdir,
|
||||
label=spec.label,
|
||||
color=spec.color,
|
||||
provider_settings=manifest_agent_provider.settings,
|
||||
)
|
||||
agent_provision_plan = merge_provision_env_vars(agent_provision_plan)
|
||||
egress_plan = prepare_egress(manifest_bottle, slug, agent_provision_plan)
|
||||
supervise_plan = prepare_supervise(manifest_bottle, slug)
|
||||
git_gate_plan = prepare_git_gate(manifest_bottle, slug)
|
||||
|
||||
return self._resolve_plan(
|
||||
spec,
|
||||
manifest=prepared.manifest,
|
||||
slug=prepared.slug,
|
||||
resolved_env=prepared.resolved_env,
|
||||
agent_provision_plan=prepared.agent_provision_plan,
|
||||
egress_plan=prepared.egress_plan,
|
||||
supervise_plan=prepared.supervise_plan,
|
||||
git_gate_plan=prepared.git_gate_plan,
|
||||
manifest=manifest,
|
||||
slug=slug,
|
||||
resolved_env=resolved_env,
|
||||
agent_provision_plan=agent_provision_plan,
|
||||
egress_plan=egress_plan,
|
||||
supervise_plan=supervise_plan,
|
||||
git_gate_plan=git_gate_plan,
|
||||
stage_dir=stage_dir,
|
||||
)
|
||||
|
||||
|
||||
@@ -6,17 +6,12 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
from typing import Iterator
|
||||
|
||||
from ...log import die, info
|
||||
from ...util import slugify as _slugify
|
||||
|
||||
|
||||
def slugify(name: str) -> str:
|
||||
"""Compatibility wrapper; new generic callers import ``bot_bottle.util``."""
|
||||
return _slugify(name)
|
||||
|
||||
|
||||
def run_docker(
|
||||
@@ -119,6 +114,19 @@ def docker_cp(src: str, dest: str) -> None:
|
||||
f"{(result.stderr or '').strip() or '<no stderr>'}")
|
||||
|
||||
|
||||
_SLUG_RE = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
|
||||
def slugify(name: str) -> str:
|
||||
"""Lowercase, non-alnum runs → '-', trimmed. Dies on empty result."""
|
||||
if not name:
|
||||
die("slugify: missing name")
|
||||
slug = _SLUG_RE.sub("-", name.lower()).strip("-")
|
||||
if not slug:
|
||||
die(f"name '{name}' produced an empty slug; use alphanumeric characters")
|
||||
return slug
|
||||
|
||||
|
||||
def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
|
||||
"""Invokes `docker build` every call. Layer cache makes no-change
|
||||
rebuilds cheap; running every time means Dockerfile edits land
|
||||
|
||||
@@ -11,8 +11,7 @@ from pathlib import Path
|
||||
|
||||
from ..bottle_state import egress_state_dir
|
||||
from ..egress import EGRESS_ROUTES_FILENAME
|
||||
from ..gateway.egress.schema import load_config
|
||||
from ..gateway.egress.types import LOG_OFF
|
||||
from ..gateway.egress.addon_core import LOG_OFF, load_config
|
||||
|
||||
|
||||
class EgressApplyError(RuntimeError):
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
"""Backend-neutral preparation planner.
|
||||
|
||||
This module owns the shared transformation from a CLI ``BottleSpec`` to the
|
||||
typed inputs consumed by a concrete backend's ``_resolve_plan``. Backend
|
||||
classes retain only their validation/preflight/env hooks and their
|
||||
backend-specific final resolution.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
|
||||
from ..agent_provider import AgentProvisionPlan, build_agent_provision_plan, get_provider
|
||||
from ..egress import EgressPlan
|
||||
from ..env import ResolvedEnv, resolve_env
|
||||
from ..git_gate import GitGate, GitGatePlan
|
||||
from ..manifest import Manifest
|
||||
from ..supervisor.plan import SupervisePlan
|
||||
from ..workspace import workspace_plan
|
||||
from .resolve_common import (
|
||||
merge_provision_env_vars,
|
||||
mint_slug,
|
||||
prepare_agent_state_dir,
|
||||
prepare_egress,
|
||||
prepare_git_gate,
|
||||
prepare_supervise,
|
||||
reject_nested_containers,
|
||||
resolve_manifest_dockerfile,
|
||||
write_launch_metadata,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .base import BottleSpec
|
||||
|
||||
|
||||
class PreparationBackend(Protocol):
|
||||
"""Backend hooks needed by the shared planner."""
|
||||
|
||||
name: str
|
||||
supports_nested_containers: bool
|
||||
|
||||
def _validate(self, spec: BottleSpec) -> Manifest: ...
|
||||
def _preflight(self) -> None: ...
|
||||
def _build_guest_env(self, resolved_env: ResolvedEnv) -> dict[str, str]: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PreparedBottle:
|
||||
"""Typed, backend-neutral result of shared launch preparation."""
|
||||
|
||||
manifest: Manifest
|
||||
slug: str
|
||||
resolved_env: ResolvedEnv
|
||||
agent_provision_plan: AgentProvisionPlan
|
||||
egress_plan: EgressPlan
|
||||
git_gate_plan: GitGatePlan
|
||||
supervise_plan: SupervisePlan | None
|
||||
|
||||
|
||||
class BottlePreparationPlanner:
|
||||
"""Run the common, side-effect-limited part of bottle preparation."""
|
||||
|
||||
def __init__(self, backend: PreparationBackend) -> None:
|
||||
self._backend = backend
|
||||
|
||||
def prepare(self, spec: BottleSpec) -> PreparedBottle:
|
||||
backend = self._backend
|
||||
# These are deliberately protected backend hooks: only this shared
|
||||
# planner orchestrates them, while concrete backends provide the
|
||||
# implementation.
|
||||
manifest = backend._validate(spec) # pylint: disable=protected-access
|
||||
if not backend.supports_nested_containers:
|
||||
reject_nested_containers(backend.name, manifest)
|
||||
|
||||
backend._preflight() # pylint: disable=protected-access
|
||||
manifest = GitGate().preflight_host_keys(
|
||||
manifest,
|
||||
headless=spec.headless,
|
||||
home_md=spec.manifest.home_md,
|
||||
)
|
||||
|
||||
bottle = manifest.bottle
|
||||
provider_config = bottle.agent_provider
|
||||
provider = get_provider(provider_config.template)
|
||||
resolved_env = resolve_env(manifest)
|
||||
workspace = workspace_plan(spec, guest_home=provider.guest_home)
|
||||
slug = mint_slug(spec)
|
||||
write_launch_metadata(slug, spec, compose_project="", backend=backend.name)
|
||||
|
||||
dockerfile = (
|
||||
resolve_manifest_dockerfile(provider_config.dockerfile, spec)
|
||||
if provider_config.dockerfile
|
||||
else str(provider.dockerfile)
|
||||
)
|
||||
agent_dir, prompt_file = prepare_agent_state_dir(slug, manifest)
|
||||
provision = build_agent_provision_plan(
|
||||
template=provider_config.template,
|
||||
dockerfile=dockerfile,
|
||||
state_dir=agent_dir,
|
||||
instance_name=f"bot-bottle-{slug}",
|
||||
prompt_file=prompt_file,
|
||||
guest_env=backend._build_guest_env( # pylint: disable=protected-access
|
||||
resolved_env
|
||||
),
|
||||
forward_host_credentials=provider_config.forward_host_credentials,
|
||||
auth_token=provider_config.auth_token,
|
||||
host_env=dict(os.environ),
|
||||
trusted_project_path=workspace.workdir,
|
||||
label=spec.label,
|
||||
color=spec.color,
|
||||
provider_settings=provider_config.settings,
|
||||
)
|
||||
provision = merge_provision_env_vars(provision)
|
||||
return PreparedBottle(
|
||||
manifest=manifest,
|
||||
slug=slug,
|
||||
resolved_env=resolved_env,
|
||||
agent_provision_plan=provision,
|
||||
egress_plan=prepare_egress(bottle, slug, provision),
|
||||
git_gate_plan=prepare_git_gate(bottle, slug),
|
||||
supervise_plan=prepare_supervise(bottle, slug),
|
||||
)
|
||||
@@ -30,7 +30,6 @@ from ..log import die
|
||||
from ..manifest import Manifest, ManifestBottle
|
||||
from ..supervisor.plan import SupervisePlan
|
||||
from ..orchestrator.supervisor import Supervisor
|
||||
from ..util import slugify
|
||||
from . import BottleSpec
|
||||
|
||||
|
||||
@@ -45,7 +44,8 @@ def mint_slug(spec: BottleSpec) -> str:
|
||||
if spec.identity:
|
||||
return spec.identity
|
||||
if spec.label:
|
||||
return slugify(spec.label)
|
||||
from .docker import util as docker_mod
|
||||
return docker_mod.slugify(spec.label)
|
||||
return bottle_identity(spec.agent_name)
|
||||
|
||||
|
||||
|
||||
@@ -25,11 +25,12 @@ from typing import Callable
|
||||
from ...agent_provider import get_provider, runtime_for
|
||||
from ...backend import (
|
||||
Bottle,
|
||||
BottlePlan,
|
||||
BottleSpec,
|
||||
enumerate_active_agents,
|
||||
get_bottle_backend,
|
||||
)
|
||||
from ...backend.docker import util as docker_mod
|
||||
from ...backend.docker.bottle_plan import DockerBottlePlan
|
||||
from ...bottle_state import (
|
||||
cleanup_state,
|
||||
is_preserved,
|
||||
@@ -39,7 +40,7 @@ from ...image_cache import StaleImageError
|
||||
from ...log import info, die
|
||||
from ...manifest import Manifest, ManifestIndex
|
||||
from ..constants import PROG
|
||||
from ...util import read_tty_line, slugify
|
||||
from ...util import read_tty_line
|
||||
from .. import tui
|
||||
|
||||
|
||||
@@ -256,10 +257,10 @@ def _uniquify_label_headless(label: str) -> str:
|
||||
logging the chosen label. Orchestrators fire-and-forget many bottles,
|
||||
so silently picking a free name beats erroring on every collision."""
|
||||
active_slugs = {a.slug for a in enumerate_active_agents()}
|
||||
if slugify(label) not in active_slugs:
|
||||
if docker_mod.slugify(label) not in active_slugs:
|
||||
return label
|
||||
n = 2
|
||||
while slugify(f"{label}-{n}") in active_slugs:
|
||||
while docker_mod.slugify(f"{label}-{n}") in active_slugs:
|
||||
n += 1
|
||||
chosen = f"{label}-{n}"
|
||||
info(f"label '{label}' already in use; using '{chosen}'")
|
||||
@@ -273,11 +274,11 @@ def prepare_with_preflight(
|
||||
spec: BottleSpec,
|
||||
*,
|
||||
stage_dir: Path,
|
||||
render_preflight: Callable[[BottlePlan, str], None],
|
||||
render_preflight: Callable[[DockerBottlePlan, str], None],
|
||||
prompt_yes: Callable[[], bool],
|
||||
dry_run: bool = False,
|
||||
backend_name: str | None = None,
|
||||
) -> tuple[BottlePlan | None, str]:
|
||||
) -> tuple[DockerBottlePlan | None, str]:
|
||||
"""Run `backend.prepare`, render the preflight summary via the
|
||||
injected callable, prompt y/N via the injected callable.
|
||||
|
||||
@@ -404,7 +405,7 @@ def _resolve_unique_label(label: str, color: str) -> tuple[str, str]:
|
||||
in use among running bottles. Passes through unchanged when no
|
||||
collision is found on the first check."""
|
||||
while True:
|
||||
slug_candidate = slugify(label)
|
||||
slug_candidate = docker_mod.slugify(label)
|
||||
active_slugs = {a.slug for a in enumerate_active_agents()}
|
||||
if slug_candidate not in active_slugs:
|
||||
return label, color
|
||||
@@ -431,7 +432,7 @@ def _select_image_policy() -> str | None:
|
||||
|
||||
|
||||
def _text_render_preflight():
|
||||
def _render(plan: BottlePlan, backend_name: str) -> None:
|
||||
def _render(plan: DockerBottlePlan, backend_name: str) -> None:
|
||||
print(file=sys.stderr)
|
||||
print(f"backend: {backend_name}", file=sys.stderr)
|
||||
print(_manifest_to_yaml(plan.manifest), file=sys.stderr)
|
||||
|
||||
@@ -368,7 +368,7 @@ def _main_loop(stdscr: "curses._CursesWindow") -> None: # type: ignore # pragm
|
||||
elif key in (curses.KEY_UP, ord("k")):
|
||||
selected = max(selected - 1, 0)
|
||||
elif key in (curses.KEY_ENTER, 10, 13):
|
||||
status_line = _detail_view(stdscr, qp, green_attr=green_attr)
|
||||
_detail_view(stdscr, qp, green_attr=green_attr)
|
||||
elif key == ord("a"):
|
||||
try:
|
||||
status_line = _approve_from_tui(stdscr, qp)
|
||||
@@ -456,7 +456,7 @@ def _detail_view(
|
||||
qp: QueuedProposal,
|
||||
*,
|
||||
green_attr: int = 0,
|
||||
) -> str: # pragma: no cover
|
||||
) -> None: # pragma: no cover
|
||||
"""Render the full proposal. Scrollable. Press q to return."""
|
||||
lines = _detail_lines(qp, green_attr=green_attr)
|
||||
offset = 0
|
||||
@@ -473,7 +473,7 @@ def _detail_view(
|
||||
stdscr.refresh()
|
||||
key = stdscr.getch()
|
||||
if key in (ord("q"), 27):
|
||||
return ""
|
||||
return
|
||||
if key in (curses.KEY_DOWN, ord("j")):
|
||||
offset = min(offset + 1, max(0, len(lines) - 1))
|
||||
elif key in (curses.KEY_UP, ord("k")):
|
||||
@@ -484,34 +484,31 @@ def _detail_view(
|
||||
offset = max(0, len(lines) - 1)
|
||||
elif key == ord("a"):
|
||||
try:
|
||||
return _approve_from_tui(stdscr, qp)
|
||||
except ApplyError as exc:
|
||||
return f"apply failed: {exc}"
|
||||
_approve_from_tui(stdscr, qp)
|
||||
except ApplyError:
|
||||
pass
|
||||
return
|
||||
elif key == ord("m"):
|
||||
if qp.proposal.tool in _REPORT_ONLY_TOOLS:
|
||||
return f"modify unavailable for {qp.proposal.tool}"
|
||||
return
|
||||
edited = _modify(stdscr, qp)
|
||||
if edited is None:
|
||||
return "modify aborted (no change)"
|
||||
try:
|
||||
return _approve_from_tui(
|
||||
stdscr, qp, final_file=edited,
|
||||
notes="operator modified before approving",
|
||||
)
|
||||
except ApplyError as exc:
|
||||
return f"apply failed: {exc}"
|
||||
if edited is not None:
|
||||
try:
|
||||
_approve_from_tui(
|
||||
stdscr, qp, final_file=edited,
|
||||
notes="operator modified before approving",
|
||||
)
|
||||
except ApplyError:
|
||||
pass
|
||||
return
|
||||
elif key == ord("r"):
|
||||
reason = _prompt(stdscr, "reject reason: ")
|
||||
if reason:
|
||||
reject(qp, reason=reason)
|
||||
return f"rejected {qp.proposal.tool} for [{qp.label}]"
|
||||
return "reject aborted (empty reason)"
|
||||
return
|
||||
|
||||
|
||||
def _modify(
|
||||
stdscr: "curses._CursesWindow", # type: ignore
|
||||
qp: QueuedProposal,
|
||||
) -> str | None: # pragma: no cover
|
||||
def _modify(stdscr: "curses._CursesWindow", qp: QueuedProposal) -> str | None: # type: ignore # pragma: no cover
|
||||
"""Suspend curses, open $EDITOR on the proposed file, return edited content."""
|
||||
suffix = _suffix_for_tool(qp.proposal.tool)
|
||||
curses.endwin()
|
||||
|
||||
+6
-32
@@ -16,8 +16,6 @@ import os
|
||||
import sys
|
||||
from typing import Any, Optional
|
||||
|
||||
from ..log import debug
|
||||
|
||||
|
||||
def filter_multiselect(
|
||||
items: list[str],
|
||||
@@ -44,11 +42,7 @@ def filter_multiselect(
|
||||
|
||||
try:
|
||||
tty_fd = open(tty_path, "r+b", buffering=0)
|
||||
except OSError as exc:
|
||||
debug(
|
||||
"multi-select unavailable; treating it as cancellation",
|
||||
context={"error_type": type(exc).__name__, "tty": tty_path},
|
||||
)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
try:
|
||||
@@ -79,11 +73,7 @@ def filter_select(
|
||||
|
||||
try:
|
||||
tty_fd = open(tty_path, "r+b", buffering=0)
|
||||
except OSError as exc:
|
||||
debug(
|
||||
"filter-select unavailable; treating it as cancellation",
|
||||
context={"error_type": type(exc).__name__, "tty": tty_path},
|
||||
)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
try:
|
||||
@@ -139,11 +129,7 @@ def _run_picker(items: list[str], *, title: str, tty_fd: int) -> Optional[str]:
|
||||
curses.nocbreak()
|
||||
curses.echo()
|
||||
curses.endwin()
|
||||
except Exception as exc: # noqa: W0718 — curses can raise many error types
|
||||
debug(
|
||||
"filter-select display failed; treating it as cancellation",
|
||||
context={"error_type": type(exc).__name__},
|
||||
)
|
||||
except Exception: # noqa: W0718 — curses can raise many error types
|
||||
return None
|
||||
finally:
|
||||
sys.__stdin__ = orig_stdin # type: ignore[assignment]
|
||||
@@ -306,11 +292,7 @@ def _run_multiselect(
|
||||
curses.nocbreak()
|
||||
curses.echo()
|
||||
curses.endwin()
|
||||
except Exception as exc: # noqa: W0718
|
||||
debug(
|
||||
"multi-select display failed; treating it as cancellation",
|
||||
context={"error_type": type(exc).__name__},
|
||||
)
|
||||
except Exception: # noqa: W0718
|
||||
return None
|
||||
finally:
|
||||
sys.__stdin__ = orig_stdin # type: ignore[assignment]
|
||||
@@ -576,21 +558,13 @@ def name_color_modal(
|
||||
"""
|
||||
try:
|
||||
tty_fd = open(tty_path, "r+b", buffering=0) # pylint: disable=consider-using-with
|
||||
except OSError as exc:
|
||||
debug(
|
||||
"name/color picker unavailable; using defaults",
|
||||
context={"error_type": type(exc).__name__, "tty": tty_path},
|
||||
)
|
||||
except OSError:
|
||||
return default_label, ""
|
||||
|
||||
try:
|
||||
fd_dup = os.dup(tty_fd.fileno())
|
||||
return _run_name_color(default_label, tty_fd=fd_dup, disclaimer=disclaimer)
|
||||
except Exception as exc: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
||||
debug(
|
||||
"name/color picker failed; using defaults",
|
||||
context={"error_type": type(exc).__name__},
|
||||
)
|
||||
except Exception: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
||||
return default_label, ""
|
||||
finally:
|
||||
tty_fd.close()
|
||||
|
||||
@@ -11,7 +11,7 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ..gateway.egress.types import Route
|
||||
from ..gateway.egress.addon_core import Route
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -19,7 +19,7 @@ class EgressRoute(Route):
|
||||
"""Host-side extension of the addon's `Route`.
|
||||
|
||||
Inherits `host`, `matches`, `auth_scheme`, and `token_env`
|
||||
from the gateway's wire `Route` — those are the fields that cross the
|
||||
from `egress_addon_core.Route` — those are the fields that cross the
|
||||
YAML wire into the gateway. The fields below are host-only and
|
||||
are never serialised to the addon.
|
||||
|
||||
|
||||
@@ -14,8 +14,8 @@ import secrets
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ..gateway.egress.dlp_config import ON_MATCH_REDACT
|
||||
from ..gateway.egress.types import (
|
||||
from ..gateway.egress.addon_core import (
|
||||
ON_MATCH_REDACT,
|
||||
HeaderMatch as CoreHeaderMatch,
|
||||
MatchEntry as CoreMatchEntry,
|
||||
PathMatch as CorePathMatch,
|
||||
|
||||
@@ -17,34 +17,28 @@ from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=
|
||||
|
||||
from bot_bottle.constants import IDENTITY_HEADER
|
||||
from bot_bottle.gateway.egress.dlp_detectors import redact_tokens, strip_crlf
|
||||
from bot_bottle.gateway.egress.dlp_config import (
|
||||
from bot_bottle.gateway.egress.addon_core import (
|
||||
LOG_BLOCKS,
|
||||
LOG_FULL,
|
||||
DEFAULT_OUTBOUND_ON_MATCH,
|
||||
ON_MATCH_BLOCK,
|
||||
ON_MATCH_REDACT,
|
||||
)
|
||||
from bot_bottle.gateway.egress.context import resolve_client_context
|
||||
from bot_bottle.gateway.egress.dlp import (
|
||||
Config,
|
||||
Route,
|
||||
ScanResult,
|
||||
build_inbound_scan_text,
|
||||
build_outbound_scan_text,
|
||||
build_token_allow_payload,
|
||||
outbound_scan_headers,
|
||||
scan_inbound,
|
||||
scan_outbound,
|
||||
)
|
||||
from bot_bottle.gateway.egress.matching import (
|
||||
decide,
|
||||
decide_git_fetch,
|
||||
is_git_fetch_request,
|
||||
is_git_push_request,
|
||||
match_route,
|
||||
)
|
||||
from bot_bottle.gateway.egress.schema import route_to_yaml_dict
|
||||
from bot_bottle.gateway.egress.types import (
|
||||
LOG_BLOCKS,
|
||||
LOG_FULL,
|
||||
Config,
|
||||
Route,
|
||||
ScanResult,
|
||||
resolve_client_context,
|
||||
outbound_scan_headers,
|
||||
route_to_yaml_dict,
|
||||
scan_inbound,
|
||||
scan_outbound,
|
||||
)
|
||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||
from bot_bottle.supervisor.types import (
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,77 +0,0 @@
|
||||
"""Fail-closed resolution of a client's policy and egress credentials."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
from ...log import debug
|
||||
from .types import Config
|
||||
|
||||
|
||||
DENY_UNATTRIBUTED = (
|
||||
"egress: this request was not attributed to any bottle, so no egress policy "
|
||||
"applies and every host is denied. Either the bottle's registry row is "
|
||||
"missing/ambiguous (torn down, or another bottle claimed its source IP), or "
|
||||
"the request carried no matching identity token — check that the caller's "
|
||||
"proxy URL includes it. This is not an allowlist problem."
|
||||
)
|
||||
DENY_UNPARSEABLE = (
|
||||
"egress: this bottle's egress policy could not be parsed, so it is being "
|
||||
"treated as deny-all. Fix the bottle's egress.routes; every host is denied "
|
||||
"until it loads."
|
||||
)
|
||||
DENY_RESOLVER_ERROR = (
|
||||
"egress: the orchestrator could not be reached to resolve this bottle's "
|
||||
"egress policy, so every host is denied (fail-closed). Check that the "
|
||||
"control plane is up; this is not an allowlist problem."
|
||||
)
|
||||
|
||||
|
||||
class PolicyResolverLike(typing.Protocol):
|
||||
def resolve(self, source_ip: str, identity_token: str = ...) -> str | None: ...
|
||||
|
||||
|
||||
class ContextResolverLike(typing.Protocol):
|
||||
def resolve_policy_and_bottle_id(
|
||||
self, source_ip: str, identity_token: str = ...,
|
||||
) -> tuple[str | None, str | None, dict[str, str]]: ...
|
||||
|
||||
|
||||
def _config_from_policy(policy: str | None) -> Config:
|
||||
# Local import keeps schema parsing independent of resolver protocols.
|
||||
from .schema import load_config
|
||||
if not policy:
|
||||
return Config(routes=(), deny_reason=DENY_UNATTRIBUTED)
|
||||
try:
|
||||
return load_config(policy)
|
||||
except ValueError:
|
||||
return Config(routes=(), deny_reason=DENY_UNPARSEABLE)
|
||||
|
||||
|
||||
def resolve_client_config(
|
||||
resolver: PolicyResolverLike, client_ip: str, identity_token: str = "",
|
||||
) -> Config:
|
||||
try:
|
||||
policy = resolver.resolve(client_ip, identity_token)
|
||||
except Exception as exc: # noqa: BLE001 - a policy lookup failure must deny
|
||||
debug(
|
||||
"egress policy resolution failed; applying deny-all",
|
||||
context={"error_type": type(exc).__name__},
|
||||
)
|
||||
return Config(routes=(), deny_reason=DENY_RESOLVER_ERROR)
|
||||
return _config_from_policy(policy)
|
||||
|
||||
|
||||
def resolve_client_context(
|
||||
resolver: ContextResolverLike, client_ip: str, identity_token: str = "",
|
||||
) -> tuple[Config, str, dict[str, str]]:
|
||||
try:
|
||||
policy, bottle_id, tokens = resolver.resolve_policy_and_bottle_id(
|
||||
client_ip, identity_token)
|
||||
except Exception as exc: # noqa: BLE001 - a policy lookup failure must deny
|
||||
debug(
|
||||
"egress context resolution failed; applying deny-all",
|
||||
context={"error_type": type(exc).__name__},
|
||||
)
|
||||
return Config(routes=(), deny_reason=DENY_RESOLVER_ERROR), "", {}
|
||||
return _config_from_policy(policy), (bottle_id or ""), tokens
|
||||
@@ -1,99 +0,0 @@
|
||||
"""DLP scan dispatch and safe proposal rendering for egress requests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
from .types import Route, ScanResult
|
||||
|
||||
|
||||
def build_outbound_scan_text(host: str, path: str, query: str,
|
||||
headers: typing.Mapping[str, str], body: str) -> str:
|
||||
parts = [host, path]
|
||||
if query:
|
||||
parts.append(query)
|
||||
parts.extend(f"{name}: {value}" for name, value in headers.items())
|
||||
if body:
|
||||
parts.append(body)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def outbound_scan_headers(route: Route, headers: typing.Mapping[str, str]) -> dict[str, str]:
|
||||
"""Drop agent Authorization when the route injects gateway-owned auth."""
|
||||
skip_auth = bool(route.auth_scheme and route.token_env)
|
||||
return {name: value for name, value in headers.items()
|
||||
if not (skip_auth and name.lower() == "authorization")}
|
||||
|
||||
|
||||
def build_inbound_scan_text(headers: typing.Mapping[str, str], body: str) -> str:
|
||||
parts = [f"{name}: {value}" for name, value in headers.items()]
|
||||
if body:
|
||||
parts.append(body)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _enabled(configured: tuple[str, ...] | None, name: str) -> bool:
|
||||
return configured is None or name in configured
|
||||
|
||||
|
||||
def scan_outbound(route: Route, body: str | bytes, environ: typing.Mapping[str, str], *,
|
||||
safe_tokens: typing.AbstractSet[str] | None = None,
|
||||
crlf_text: str | None = None) -> ScanResult | None:
|
||||
if not route.inspect:
|
||||
return None
|
||||
try:
|
||||
from dlp_detectors import ( # type: ignore[import-not-found]
|
||||
scan_crlf_injection, scan_entropy, scan_known_secrets, scan_token_patterns)
|
||||
except ImportError: # pragma: no cover - gateway's flat module path
|
||||
from .dlp_detectors import (
|
||||
scan_crlf_injection, scan_entropy, scan_known_secrets, scan_token_patterns)
|
||||
if isinstance(body, bytes):
|
||||
try:
|
||||
text = body.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
text = body.decode("latin-1")
|
||||
else:
|
||||
text = body
|
||||
result = scan_crlf_injection(text if crlf_text is None else crlf_text)
|
||||
if result is not None:
|
||||
return result
|
||||
if _enabled(route.outbound_detectors, "token_patterns"):
|
||||
result = scan_token_patterns(text, location="body", safe_tokens=safe_tokens)
|
||||
if result is not None:
|
||||
return result
|
||||
if _enabled(route.outbound_detectors, "known_secrets"):
|
||||
extra = tuple(prefix for prefix in environ.get(
|
||||
"BOT_BOTTLE_SENSITIVE_PREFIXES", "").split(",") if prefix)
|
||||
result = scan_known_secrets(text, location="body", env=environ,
|
||||
sensitive_prefixes=("EGRESS_TOKEN_",) + extra,
|
||||
safe_tokens=safe_tokens)
|
||||
if result is not None:
|
||||
return result
|
||||
if route.outbound_detectors is not None and "entropy" in route.outbound_detectors:
|
||||
return scan_entropy(text, location="body")
|
||||
return None
|
||||
|
||||
|
||||
def build_token_allow_payload(host: str, method: str, path: str, result: ScanResult) -> str:
|
||||
"""Render redacted operator context; the raw matched secret is excluded."""
|
||||
lines = [
|
||||
"egress blocked an outbound request carrying a detected token",
|
||||
f"host: {host}", f"method: {method}", f"path: {path}",
|
||||
f"detector: {result.reason}",
|
||||
]
|
||||
if result.context:
|
||||
lines.append(f"context: {result.context}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def scan_inbound(route: Route, body: str | bytes) -> ScanResult | None:
|
||||
if not route.inspect:
|
||||
return None
|
||||
try:
|
||||
from dlp_detectors import scan_naive_injection # type: ignore[import-not-found]
|
||||
except ImportError: # pragma: no cover - gateway's flat module path
|
||||
from .dlp_detectors import scan_naive_injection
|
||||
text = body if isinstance(body, str) else body.decode("utf-8", errors="replace")
|
||||
if _enabled(route.inbound_detectors, "naive_injection_detection"):
|
||||
return scan_naive_injection(text)
|
||||
return None
|
||||
@@ -19,7 +19,7 @@ from math import log2
|
||||
from collections import Counter
|
||||
from urllib.parse import quote as url_quote
|
||||
|
||||
from .types import ScanResult
|
||||
from .addon_core import ScanResult
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
"""Route matching and request-policy decisions for the egress gateway."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
from .types import Decision, MatchEntry, PathMatch, Route
|
||||
|
||||
|
||||
def _path_matches(pm: PathMatch, request_path: str) -> bool:
|
||||
if pm.type == "exact":
|
||||
return request_path == pm.value
|
||||
if pm.type == "prefix":
|
||||
if request_path == pm.value:
|
||||
return True
|
||||
if not pm.value.endswith("/"):
|
||||
return request_path.startswith(pm.value + "/")
|
||||
return request_path.startswith(pm.value)
|
||||
return (
|
||||
pm.type == "regex"
|
||||
and pm.compiled is not None
|
||||
and pm.compiled.search(request_path) is not None
|
||||
)
|
||||
|
||||
|
||||
def _entry_matches(
|
||||
entry: MatchEntry, request_path: str, request_method: str,
|
||||
request_headers: typing.Mapping[str, str],
|
||||
) -> bool:
|
||||
if entry.paths and not any(_path_matches(pm, request_path) for pm in entry.paths):
|
||||
return False
|
||||
if entry.methods and request_method.upper() not in entry.methods:
|
||||
return False
|
||||
for match in entry.headers:
|
||||
value = request_headers.get(match.name.lower())
|
||||
if value is None:
|
||||
return False
|
||||
if match.type == "exact" and value != match.value:
|
||||
return False
|
||||
if match.type == "regex" and (
|
||||
match.compiled is None or match.compiled.search(value) is None
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def evaluate_matches(
|
||||
route: Route, request_path: str, request_method: str = "GET",
|
||||
request_headers: typing.Mapping[str, str] | None = None,
|
||||
) -> bool:
|
||||
"""Return whether a request satisfies a route's optional match entries."""
|
||||
if not route.matches:
|
||||
return True
|
||||
return any(_entry_matches(entry, request_path, request_method, request_headers or {})
|
||||
for entry in route.matches)
|
||||
|
||||
|
||||
def is_git_push_request(path: str, query: str) -> bool:
|
||||
return path.endswith("/git-receive-pack") or (
|
||||
path.endswith("/info/refs") and any(
|
||||
pair.partition("=") == ("service", "=", "git-receive-pack")
|
||||
for pair in query.split("&")
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def is_git_fetch_request(path: str, query: str) -> bool:
|
||||
return path.endswith("/git-upload-pack") or (
|
||||
path.endswith("/info/refs") and any(
|
||||
pair.partition("=") == ("service", "=", "git-upload-pack")
|
||||
for pair in query.split("&")
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def match_route(routes: typing.Sequence[Route], request_host: str) -> Route | None:
|
||||
target = request_host.lower()
|
||||
return next((route for route in routes if route.host.lower() == target), None)
|
||||
|
||||
|
||||
def decide(
|
||||
routes: typing.Sequence[Route], request_host: str, request_path: str,
|
||||
environ: typing.Mapping[str, str], *, request_method: str = "GET",
|
||||
request_headers: typing.Mapping[str, str] | None = None, deny_reason: str = "",
|
||||
) -> Decision:
|
||||
route = match_route(routes, request_host)
|
||||
if route is None:
|
||||
return Decision("block", deny_reason or (
|
||||
f"egress: host {request_host!r} is not in the bottle's egress.routes "
|
||||
"allowlist. Declare a route for it or remove the request."))
|
||||
if not evaluate_matches(route, request_path, request_method, request_headers):
|
||||
return Decision("block", (
|
||||
f"egress: request {request_method} {request_path!r} does not match any "
|
||||
f"entry in matches for {route.host!r}"))
|
||||
if route.auth_scheme and route.token_env:
|
||||
token = environ.get(route.token_env, "")
|
||||
if not token:
|
||||
return Decision("block", (
|
||||
f"egress: route for {route.host!r} declared auth but env var "
|
||||
f"{route.token_env!r} is unset"))
|
||||
return Decision("forward", inject_authorization=f"{route.auth_scheme} {token}")
|
||||
return Decision("forward")
|
||||
|
||||
|
||||
def decide_git_fetch(routes: typing.Sequence[Route], request_host: str) -> Decision:
|
||||
route = match_route(routes, request_host)
|
||||
if route is not None and route.git_fetch:
|
||||
return Decision("forward")
|
||||
return Decision("block", (
|
||||
"egress: git fetch/clone over HTTPS is not allowed by default; use git-gate "
|
||||
"for declared repos or set egress.routes[].git.fetch=true for explicit "
|
||||
"read-only HTTPS Git access."))
|
||||
@@ -1,349 +0,0 @@
|
||||
"""Egress policy schema parsing and serialization (PRD 0017 / 0053)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import typing
|
||||
|
||||
from ...yaml_subset import YamlSubsetError, parse_yaml_subset
|
||||
from .dlp_config import parse_inspect_block
|
||||
from .types import (
|
||||
HEADER_MATCH_TYPES,
|
||||
LOG_BLOCKS,
|
||||
LOG_FULL,
|
||||
LOG_OFF,
|
||||
PATH_MATCH_TYPES,
|
||||
VALID_METHODS,
|
||||
Config,
|
||||
HeaderMatch,
|
||||
MatchEntry,
|
||||
PathMatch,
|
||||
Route,
|
||||
)
|
||||
|
||||
# Parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _parse_path_match(idx: int, j: int, raw: object) -> PathMatch:
|
||||
label = f"route[{idx}] matches paths[{j}]"
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"{label}: must be an object")
|
||||
raw_dict: dict[str, object] = typing.cast(dict[str, object], raw)
|
||||
ptype = raw_dict.get("type", "prefix")
|
||||
if not isinstance(ptype, str) or ptype not in PATH_MATCH_TYPES:
|
||||
raise ValueError(
|
||||
f"{label}: 'type' must be one of {', '.join(PATH_MATCH_TYPES)} "
|
||||
f"(got {ptype!r})"
|
||||
)
|
||||
value = raw_dict.get("value")
|
||||
if not isinstance(value, str) or not value:
|
||||
raise ValueError(f"{label}: 'value' must be a non-empty string")
|
||||
if ptype in ("exact", "prefix") and not value.startswith("/"):
|
||||
raise ValueError(
|
||||
f"{label}: value {value!r} must start with '/' for "
|
||||
f"type {ptype!r}"
|
||||
)
|
||||
compiled: re.Pattern[str] | None = None
|
||||
if ptype == "regex":
|
||||
try:
|
||||
compiled = re.compile(value)
|
||||
except re.error as e:
|
||||
raise ValueError(
|
||||
f"{label}: regex {value!r} failed to compile: {e}"
|
||||
) from e
|
||||
for k in raw_dict:
|
||||
if k not in ("type", "value"):
|
||||
raise ValueError(f"{label}: unknown key {k!r}")
|
||||
return PathMatch(type=ptype, value=value, compiled=compiled)
|
||||
|
||||
|
||||
def _parse_header_match(idx: int, j: int, raw: object) -> HeaderMatch:
|
||||
label = f"route[{idx}] matches headers[{j}]"
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"{label}: must be an object")
|
||||
raw_dict: dict[str, object] = typing.cast(dict[str, object], raw)
|
||||
name = raw_dict.get("name")
|
||||
if not isinstance(name, str) or not name:
|
||||
raise ValueError(f"{label}: 'name' must be a non-empty string")
|
||||
value = raw_dict.get("value")
|
||||
if not isinstance(value, str):
|
||||
raise ValueError(f"{label}: 'value' must be a string")
|
||||
htype = raw_dict.get("type", "exact")
|
||||
if not isinstance(htype, str) or htype not in HEADER_MATCH_TYPES:
|
||||
raise ValueError(
|
||||
f"{label}: 'type' must be one of {', '.join(HEADER_MATCH_TYPES)} "
|
||||
f"(got {htype!r})"
|
||||
)
|
||||
compiled: re.Pattern[str] | None = None
|
||||
if htype == "regex":
|
||||
try:
|
||||
compiled = re.compile(value)
|
||||
except re.error as e:
|
||||
raise ValueError(
|
||||
f"{label}: regex {value!r} failed to compile: {e}"
|
||||
) from e
|
||||
for k in raw_dict:
|
||||
if k not in ("name", "value", "type"):
|
||||
raise ValueError(f"{label}: unknown key {k!r}")
|
||||
return HeaderMatch(name=name, value=value, type=htype, compiled=compiled)
|
||||
|
||||
|
||||
def _parse_match_entry(idx: int, k: int, raw: object) -> MatchEntry:
|
||||
label = f"route[{idx}] matches[{k}]"
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"{label}: must be an object")
|
||||
raw_dict: dict[str, object] = typing.cast(dict[str, object], raw)
|
||||
|
||||
paths: tuple[PathMatch, ...] = ()
|
||||
paths_raw = raw_dict.get("paths")
|
||||
if paths_raw is not None:
|
||||
if not isinstance(paths_raw, list):
|
||||
raise ValueError(f"{label}: 'paths' must be a list")
|
||||
paths_list = typing.cast(list[object], paths_raw)
|
||||
paths = tuple(_parse_path_match(idx, j, p) for j, p in enumerate(paths_list))
|
||||
|
||||
methods: tuple[str, ...] = ()
|
||||
methods_raw = raw_dict.get("methods")
|
||||
if methods_raw is not None:
|
||||
if not isinstance(methods_raw, list):
|
||||
raise ValueError(f"{label}: 'methods' must be a list")
|
||||
methods_list = typing.cast(list[object], methods_raw)
|
||||
normalised: list[str] = []
|
||||
for j, m in enumerate(methods_list):
|
||||
if not isinstance(m, str):
|
||||
raise ValueError(f"{label}: methods[{j}] must be a string")
|
||||
upper = m.upper()
|
||||
if upper not in VALID_METHODS:
|
||||
raise ValueError(
|
||||
f"{label}: methods[{j}] {m!r} is not a valid HTTP method"
|
||||
)
|
||||
normalised.append(upper)
|
||||
methods = tuple(normalised)
|
||||
|
||||
headers: tuple[HeaderMatch, ...] = ()
|
||||
headers_raw = raw_dict.get("headers")
|
||||
if headers_raw is not None:
|
||||
if not isinstance(headers_raw, list):
|
||||
raise ValueError(f"{label}: 'headers' must be a list")
|
||||
headers_list = typing.cast(list[object], headers_raw)
|
||||
headers = tuple(
|
||||
_parse_header_match(idx, j, h) for j, h in enumerate(headers_list)
|
||||
)
|
||||
|
||||
for key in raw_dict:
|
||||
if key not in ("paths", "methods", "headers"):
|
||||
raise ValueError(f"{label}: unknown key {key!r}")
|
||||
|
||||
return MatchEntry(paths=paths, methods=methods, headers=headers)
|
||||
|
||||
|
||||
def parse_routes(payload: object) -> tuple[Route, ...]:
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("routes payload: top-level must be an object")
|
||||
payload_dict: dict[str, object] = typing.cast(dict[str, object], payload)
|
||||
raw: object = payload_dict.get("routes")
|
||||
if not isinstance(raw, list):
|
||||
raise ValueError("routes payload: 'routes' must be a list")
|
||||
raw_list: list[object] = typing.cast(list[object], raw)
|
||||
out: list[Route] = []
|
||||
for i, r in enumerate(raw_list):
|
||||
out.append(_parse_one(i, r))
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _parse_one(idx: int, raw: object) -> Route:
|
||||
label = f"route[{idx}]"
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"{label}: must be an object (got {type(raw).__name__})")
|
||||
raw_dict: dict[str, object] = typing.cast(dict[str, object], raw)
|
||||
host: object = raw_dict.get("host")
|
||||
if not isinstance(host, str) or not host:
|
||||
raise ValueError(f"{label}: 'host' must be a non-empty string")
|
||||
legacy_flat = "inspect" not in raw_dict
|
||||
inspect_raw = raw_dict.get("inspect", {})
|
||||
if inspect_raw is False:
|
||||
inspect = False
|
||||
settings: dict[str, object] = {}
|
||||
elif isinstance(inspect_raw, dict):
|
||||
inspect = True
|
||||
settings = (
|
||||
{k: v for k, v in raw_dict.items() if k != "host"}
|
||||
if legacy_flat
|
||||
else typing.cast(dict[str, object], inspect_raw)
|
||||
)
|
||||
legacy_dlp = settings.pop("dlp", None)
|
||||
if isinstance(legacy_dlp, dict):
|
||||
settings.update(typing.cast(dict[str, object], legacy_dlp))
|
||||
elif legacy_dlp is not None:
|
||||
raise ValueError(
|
||||
f"{label} ({host}): legacy 'dlp' must be an object"
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"{label} ({host}): 'inspect' must be false or an object")
|
||||
|
||||
# matches
|
||||
matches: tuple[MatchEntry, ...] = ()
|
||||
matches_raw = settings.get("matches")
|
||||
if matches_raw is not None:
|
||||
if not isinstance(matches_raw, list):
|
||||
raise ValueError(f"{label} ({host}): 'matches' must be a list")
|
||||
matches_list = typing.cast(list[object], matches_raw)
|
||||
matches = tuple(
|
||||
_parse_match_entry(idx, k, m) for k, m in enumerate(matches_list)
|
||||
)
|
||||
|
||||
# auth (unchanged wire format)
|
||||
auth_scheme: object = settings.get("auth_scheme", "")
|
||||
token_env: object = settings.get("token_env", "")
|
||||
if not isinstance(auth_scheme, str):
|
||||
raise ValueError(f"{label} ({host}): 'auth_scheme' must be a string")
|
||||
if not isinstance(token_env, str):
|
||||
raise ValueError(f"{label} ({host}): 'token_env' must be a string")
|
||||
if bool(auth_scheme) != bool(token_env):
|
||||
raise ValueError(
|
||||
f"{label} ({host}): 'auth_scheme' and 'token_env' must be both "
|
||||
f"set or both empty (got auth_scheme={auth_scheme!r}, "
|
||||
f"token_env={token_env!r})"
|
||||
)
|
||||
|
||||
# git-over-HTTPS policy
|
||||
git_fetch = False
|
||||
git_raw = settings.get("git")
|
||||
if git_raw is not None:
|
||||
if not isinstance(git_raw, dict):
|
||||
raise ValueError(f"{label} ({host}): 'git' must be an object")
|
||||
git_dict: dict[str, object] = typing.cast(dict[str, object], git_raw)
|
||||
fetch_raw = git_dict.get("fetch", False)
|
||||
if fetch_raw is True or fetch_raw is False:
|
||||
git_fetch = fetch_raw
|
||||
else:
|
||||
raise ValueError(f"{label} ({host}): 'git.fetch' must be a boolean")
|
||||
for k in git_dict:
|
||||
if k != "fetch":
|
||||
raise ValueError(
|
||||
f"{label} ({host}): git has unknown key {k!r}; "
|
||||
"accepted key is 'fetch'"
|
||||
)
|
||||
|
||||
# dlp detectors
|
||||
outbound_detectors, inbound_detectors, outbound_on_match = parse_inspect_block(
|
||||
idx, host, settings,
|
||||
)
|
||||
|
||||
preserve_auth_raw = settings.get("preserve_auth", False)
|
||||
if preserve_auth_raw is not True and preserve_auth_raw is not False:
|
||||
raise ValueError(
|
||||
f"{label} ({host}): 'preserve_auth' must be a boolean"
|
||||
)
|
||||
preserve_auth: bool = preserve_auth_raw
|
||||
|
||||
for k in settings:
|
||||
if k not in (
|
||||
"matches", "auth_scheme", "token_env", "git", "preserve_auth",
|
||||
"outbound_detectors", "inbound_detectors", "outbound_on_match",
|
||||
):
|
||||
raise ValueError(
|
||||
f"{label} ({host}): inspect has unknown key {k!r}"
|
||||
)
|
||||
for k in raw_dict:
|
||||
if not legacy_flat and k not in ("host", "inspect"):
|
||||
raise ValueError(
|
||||
f"{label} ({host}): unknown key {k!r}; accepted keys "
|
||||
f"are 'host' and 'inspect'"
|
||||
)
|
||||
|
||||
return Route(
|
||||
host=host,
|
||||
matches=matches,
|
||||
auth_scheme=auth_scheme,
|
||||
token_env=token_env,
|
||||
git_fetch=git_fetch,
|
||||
outbound_detectors=outbound_detectors,
|
||||
inbound_detectors=inbound_detectors,
|
||||
outbound_on_match=outbound_on_match,
|
||||
preserve_auth=preserve_auth,
|
||||
inspect=inspect,
|
||||
)
|
||||
|
||||
|
||||
def _path_match_to_dict(pm: PathMatch) -> dict[str, object]:
|
||||
d: dict[str, object] = {"value": pm.value}
|
||||
if pm.type != "prefix":
|
||||
d["type"] = pm.type
|
||||
return d
|
||||
|
||||
|
||||
def _header_match_to_dict(hm: HeaderMatch) -> dict[str, object]:
|
||||
d: dict[str, object] = {"name": hm.name, "value": hm.value}
|
||||
if hm.type != "exact":
|
||||
d["type"] = hm.type
|
||||
return d
|
||||
|
||||
|
||||
def _match_entry_to_dict(me: MatchEntry) -> dict[str, object]:
|
||||
d: dict[str, object] = {}
|
||||
if me.paths:
|
||||
d["paths"] = [_path_match_to_dict(p) for p in me.paths]
|
||||
if me.methods:
|
||||
d["methods"] = list(me.methods)
|
||||
if me.headers:
|
||||
d["headers"] = [_header_match_to_dict(h) for h in me.headers]
|
||||
return d
|
||||
|
||||
|
||||
def route_to_yaml_dict(r: Route) -> dict[str, object]:
|
||||
"""Serialize a Route to YAML-schema-compatible dict.
|
||||
|
||||
Uses the same field names the YAML parser accepts, so the output
|
||||
can be round-tripped directly into an `allow` or `egress-block`
|
||||
proposal without translation. Fields that are empty/default are
|
||||
omitted so the agent doesn't copy irrelevant keys."""
|
||||
d: dict[str, object] = {"host": r.host}
|
||||
if not r.inspect:
|
||||
d["inspect"] = False
|
||||
return d
|
||||
inspected: dict[str, object] = {}
|
||||
if r.auth_scheme:
|
||||
inspected["auth_scheme"] = r.auth_scheme
|
||||
inspected["token_env"] = r.token_env
|
||||
if r.matches:
|
||||
inspected["matches"] = [_match_entry_to_dict(m) for m in r.matches]
|
||||
if r.git_fetch:
|
||||
inspected["git"] = {"fetch": True}
|
||||
if r.outbound_detectors is not None:
|
||||
inspected["outbound_detectors"] = list(r.outbound_detectors)
|
||||
if r.inbound_detectors is not None:
|
||||
inspected["inbound_detectors"] = list(r.inbound_detectors)
|
||||
if r.outbound_on_match:
|
||||
inspected["outbound_on_match"] = r.outbound_on_match
|
||||
if r.preserve_auth:
|
||||
inspected["preserve_auth"] = True
|
||||
if inspected:
|
||||
d["inspect"] = inspected
|
||||
return d
|
||||
|
||||
|
||||
def parse_config(payload: object) -> "Config":
|
||||
"""Parse a full egress config payload (top-level log level + routes)."""
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("routes payload: top-level must be an object")
|
||||
payload_dict: dict[str, object] = typing.cast(dict[str, object], payload)
|
||||
|
||||
log_raw: object = payload_dict.get("log", LOG_OFF)
|
||||
if log_raw is True or log_raw is False or not isinstance(log_raw, int) \
|
||||
or log_raw not in (LOG_OFF, LOG_BLOCKS, LOG_FULL):
|
||||
raise ValueError(
|
||||
f"routes payload: 'log' must be {LOG_OFF}, {LOG_BLOCKS}, or {LOG_FULL}"
|
||||
)
|
||||
|
||||
routes = parse_routes(payload)
|
||||
return Config(routes=routes, log=log_raw)
|
||||
|
||||
|
||||
def load_config(text: str) -> "Config":
|
||||
"""Parse YAML text → Config (routes + log flag)."""
|
||||
try:
|
||||
payload = parse_yaml_subset(text)
|
||||
except YamlSubsetError as e:
|
||||
raise ValueError(f"routes payload: invalid YAML: {e}") from e
|
||||
return parse_config(payload)
|
||||
@@ -1,81 +0,0 @@
|
||||
"""Shared egress policy value objects.
|
||||
|
||||
Kept dependency-free so the schema parser, matcher, DLP scanner, and addon
|
||||
adapter can use the same immutable public shapes without importing each other.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
PATH_MATCH_TYPES = ("exact", "prefix", "regex")
|
||||
HEADER_MATCH_TYPES = ("exact", "regex")
|
||||
VALID_METHODS = frozenset({
|
||||
"GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "TRACE",
|
||||
"CONNECT",
|
||||
})
|
||||
|
||||
LOG_OFF = 0
|
||||
LOG_BLOCKS = 1
|
||||
LOG_FULL = 2
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PathMatch:
|
||||
type: str
|
||||
value: str
|
||||
compiled: re.Pattern[str] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HeaderMatch:
|
||||
name: str
|
||||
value: str
|
||||
type: str = "exact"
|
||||
compiled: re.Pattern[str] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MatchEntry:
|
||||
paths: tuple[PathMatch, ...] = ()
|
||||
methods: tuple[str, ...] = ()
|
||||
headers: tuple[HeaderMatch, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Route:
|
||||
host: str
|
||||
matches: tuple[MatchEntry, ...] = ()
|
||||
auth_scheme: str = ""
|
||||
token_env: str = ""
|
||||
git_fetch: bool = False
|
||||
outbound_detectors: tuple[str, ...] | None = None
|
||||
inbound_detectors: tuple[str, ...] | None = None
|
||||
outbound_on_match: str = ""
|
||||
preserve_auth: bool = False
|
||||
inspect: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
routes: tuple[Route, ...]
|
||||
log: int = LOG_OFF
|
||||
deny_reason: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Decision:
|
||||
action: str
|
||||
reason: str = ""
|
||||
inject_authorization: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScanResult:
|
||||
severity: str
|
||||
reason: str
|
||||
location: str = ""
|
||||
context: str = ""
|
||||
matched: str = ""
|
||||
@@ -58,9 +58,9 @@ import typing
|
||||
from dataclasses import dataclass
|
||||
|
||||
from bot_bottle.constants import IDENTITY_HEADER
|
||||
from bot_bottle.gateway.egress.context import resolve_client_context
|
||||
from bot_bottle.gateway.egress.schema import load_config, route_to_yaml_dict
|
||||
from bot_bottle.gateway.egress.types import LOG_OFF
|
||||
from bot_bottle.gateway.egress.addon_core import (
|
||||
LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict,
|
||||
)
|
||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||
from bot_bottle.supervisor import types as _sv
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ import urllib.request
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..log import debug
|
||||
from ..orchestrator_auth import ROLE_CLI
|
||||
from ..trust_domain import CONTROL_PLANE
|
||||
from .server import ORCHESTRATOR_AUTH_HEADER
|
||||
@@ -54,23 +53,6 @@ class RegisteredBottle:
|
||||
env_var_secret: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackendProbeFailure:
|
||||
"""Safe diagnostic for an optional backend discovery probe."""
|
||||
|
||||
backend: str
|
||||
error_type: str
|
||||
|
||||
|
||||
def _probe_failure(backend: str, exc: BaseException) -> BackendProbeFailure:
|
||||
failure = BackendProbeFailure(backend, type(exc).__name__)
|
||||
debug(
|
||||
"orchestrator discovery probe unavailable",
|
||||
context={"backend": failure.backend, "error_type": failure.error_type},
|
||||
)
|
||||
return failure
|
||||
|
||||
|
||||
class OrchestratorClient:
|
||||
"""Trusted host-side client for the orchestrator control plane.
|
||||
|
||||
@@ -263,41 +245,32 @@ def discover_orchestrator_url(*, timeout: float = 2.0) -> str:
|
||||
orchestrator TAP. Returns the first that answers `/health`; raises if none
|
||||
do (no orchestrator up — launch a bottle first)."""
|
||||
candidates: list[str] = []
|
||||
failures: list[BackendProbeFailure] = []
|
||||
try: # docker: loopback-published control plane
|
||||
from .lifecycle import DEFAULT_PORT as _DOCKER_PORT
|
||||
candidates.append(f"http://127.0.0.1:{_DOCKER_PORT}")
|
||||
except Exception as exc: # noqa: BLE001 — backend optional
|
||||
failures.append(_probe_failure("docker", exc))
|
||||
except Exception: # noqa: BLE001 — backend optional
|
||||
candidates.append("http://127.0.0.1:8099")
|
||||
try: # firecracker: infra VM control plane on the orchestrator TAP
|
||||
from ..backend.firecracker import netpool
|
||||
from ..backend.firecracker.infra_vm import ORCHESTRATOR_PORT
|
||||
candidates.append(
|
||||
f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}")
|
||||
except Exception as exc: # noqa: BLE001 — backend optional / not firecracker
|
||||
failures.append(_probe_failure("firecracker", exc))
|
||||
except Exception: # noqa: BLE001 — backend optional / not firecracker
|
||||
pass
|
||||
try: # macOS: orchestrator container on its host-only address
|
||||
from ..backend.macos_container.infra import probe_orchestrator_url
|
||||
url = probe_orchestrator_url()
|
||||
if url:
|
||||
candidates.append(url)
|
||||
except Exception as exc: # noqa: BLE001 — backend optional / not macOS
|
||||
failures.append(_probe_failure("macos-container", exc))
|
||||
except Exception: # noqa: BLE001 — backend optional / not macOS
|
||||
pass
|
||||
for url in candidates:
|
||||
if OrchestratorClient(url, timeout=timeout).health():
|
||||
return url
|
||||
detail = ""
|
||||
if failures:
|
||||
detail = "; optional probes unavailable: " + ", ".join(
|
||||
f"{failure.backend} ({failure.error_type})" for failure in failures
|
||||
)
|
||||
raise OrchestratorClientError(
|
||||
"no running orchestrator control plane found (tried "
|
||||
+ ", ".join(candidates)
|
||||
+ ")"
|
||||
+ detail
|
||||
+ "; launch a bottle first"
|
||||
+ "); launch a bottle first"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..log import debug
|
||||
from .client import OrchestratorClient, OrchestratorClientError
|
||||
|
||||
|
||||
@@ -28,14 +27,7 @@ def reprovision_bottles(
|
||||
try:
|
||||
if client.reprovision_gateway(bottle_id, secret):
|
||||
restored += 1
|
||||
except OrchestratorClientError as exc:
|
||||
debug(
|
||||
"gateway secret reprovision failed; continuing with other bottles",
|
||||
context={
|
||||
"bottle_id": bottle_id,
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
)
|
||||
except OrchestratorClientError:
|
||||
continue
|
||||
return restored
|
||||
|
||||
|
||||
@@ -57,7 +57,6 @@ from __future__ import annotations
|
||||
|
||||
import http.server
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import socketserver
|
||||
import sys
|
||||
@@ -218,18 +217,13 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches
|
||||
raw_ips = data.get("live_source_ips")
|
||||
if not isinstance(raw_ips, list):
|
||||
return 400, {"error": "live_source_ips (list of strings) is required"}
|
||||
if any(not isinstance(ip, str) or not ip for ip in raw_ips):
|
||||
return 400, {"error": "live_source_ips must contain non-empty strings"}
|
||||
live = raw_ips
|
||||
live = [ip for ip in raw_ips if isinstance(ip, str) and ip]
|
||||
grace = data.get("grace_seconds")
|
||||
kwargs: dict[str, float] = {}
|
||||
if grace is not None:
|
||||
if isinstance(grace, bool) or not isinstance(grace, (int, float)):
|
||||
return 400, {"error": "grace_seconds must be a non-negative finite number"}
|
||||
parsed_grace = float(grace)
|
||||
if not math.isfinite(parsed_grace) or parsed_grace < 0:
|
||||
return 400, {"error": "grace_seconds must be a non-negative finite number"}
|
||||
kwargs["grace_seconds"] = parsed_grace
|
||||
kwargs = (
|
||||
{"grace_seconds": float(grace)}
|
||||
if isinstance(grace, (int, float)) and not isinstance(grace, bool)
|
||||
else {}
|
||||
)
|
||||
return 200, {"reaped": orch.reconcile(live, **kwargs)}
|
||||
|
||||
if method == "POST" and route == "/attribute":
|
||||
@@ -379,15 +373,9 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
||||
status, payload = dispatch(
|
||||
server.orchestrator, method, self.path, body, role=role)
|
||||
except Exception as e: # noqa: BLE001 — the control plane must stay up
|
||||
# Do not echo exception messages to the caller or logs: broker and
|
||||
# persistence exceptions can contain request data. The operation,
|
||||
# route, and exception type are enough to correlate a traceback.
|
||||
sys.stderr.write(
|
||||
f"orchestrator: {method} {self.path} failed "
|
||||
f"[error_type={type(e).__name__}]\n"
|
||||
)
|
||||
sys.stderr.write(f"orchestrator: {method} {self.path} failed: {e!r}\n")
|
||||
sys.stderr.flush()
|
||||
status, payload = 500, {"error": "internal error"}
|
||||
status, payload = 500, {"error": f"internal error: {e}"}
|
||||
data = json.dumps(payload).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
|
||||
@@ -113,7 +113,7 @@ _MIGRATIONS = TableMigrations(
|
||||
# egress allowlist / routes / git config selected by source IP. The
|
||||
# multi-tenant gateway resolves it per request via `attribute`.
|
||||
"ALTER TABLE orchestrator_bottles ADD COLUMN policy TEXT NOT NULL DEFAULT ''",
|
||||
# v4 — per-bottle encrypted egress secrets (PRD prd-new-secret-provider).
|
||||
# v4 — per-bottle encrypted egress secrets (PRD 0080).
|
||||
# One row per env-var: key (env-var name) is plaintext for auditing;
|
||||
# value is the encrypted token string. The encryption key (ENV_VAR_SECRET)
|
||||
# lives only in the agent's environment — a row alone cannot recover the
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Symmetric encryption for per-bottle egress secrets (PRD prd-new-secret-provider).
|
||||
"""Symmetric encryption for per-bottle egress secrets (PRD 0080).
|
||||
|
||||
Each agent receives a random ENV_VAR_SECRET at startup — passed as an env var,
|
||||
never logged or persisted. The host uses this key to encrypt each egress auth
|
||||
|
||||
@@ -9,11 +9,8 @@ import difflib
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from .log import die
|
||||
|
||||
|
||||
def sha256_hex(content: str) -> str:
|
||||
"""Hex SHA-256 of a UTF-8 string."""
|
||||
@@ -70,20 +67,3 @@ def expand_tilde(path: str) -> str:
|
||||
home = os.environ.get("HOME", "")
|
||||
return home + path[1:]
|
||||
return path
|
||||
|
||||
|
||||
_SLUG_RE = re.compile(r"[^a-z0-9]+")
|
||||
|
||||
|
||||
def slugify(name: str) -> str:
|
||||
"""Return a portable bottle identifier from a human-readable name.
|
||||
|
||||
This is deliberately a root utility: names are part of the generic CLI
|
||||
and state model, not a Docker container concern.
|
||||
"""
|
||||
if not name:
|
||||
die("slugify: missing name")
|
||||
slug = _SLUG_RE.sub("-", name.lower()).strip("-")
|
||||
if not slug:
|
||||
die(f"name '{name}' produced an empty slug; use alphanumeric characters")
|
||||
return slug
|
||||
|
||||
@@ -7,6 +7,7 @@ picking the right document for what you're capturing.
|
||||
|
||||
| Artifact | For |
|
||||
|---|---|
|
||||
| **Design workflow** (`docs/design-workflow.md`) | How discussion becomes canonical design, how dependencies are recorded, and when implementation may begin. |
|
||||
| **Glossary** (`docs/glossary.md`) | Canonical term definitions — what words mean in this project. |
|
||||
| **PRD** (`docs/prds/`) | A feature: what to build, scope, success criteria. |
|
||||
| **Research note** (`docs/research/`) | A landscape/tradeoff investigation. |
|
||||
|
||||
@@ -1,9 +1,13 @@
|
||||
# ADR 0005: Keep tracker metadata on issues
|
||||
# ADR 0005: Keep tracker metadata on one tracker object
|
||||
|
||||
- **Status:** Accepted
|
||||
- **Date:** 2026-07-18
|
||||
- **Deciders:** didericis
|
||||
|
||||
> **Amended 2026-07-26.** A pull request may carry labels directly instead of
|
||||
> linking a tracking issue. When a PR does link an issue, the issue remains the
|
||||
> canonical owner of planning metadata and the reference is validated.
|
||||
|
||||
## Context
|
||||
|
||||
Gitea exposes labels on both issues and pull requests. Applying the same labels
|
||||
@@ -20,19 +24,29 @@ would make the issue history less truthful.
|
||||
|
||||
## Decision
|
||||
|
||||
Issues are the canonical tracker records and own labels. Every issue has at
|
||||
least one label. An issue opened or left without labels receives
|
||||
`Status/Needs Triage` automatically until it is classified.
|
||||
Issues are the canonical tracker records and own labels when a separate work
|
||||
item exists. Every issue has at least one label. An issue opened or left
|
||||
without labels receives `Status/Needs Triage` automatically until it is
|
||||
classified.
|
||||
|
||||
Pull requests carry no labels. Every new PR deliberately references at least
|
||||
one existing issue in its title or description with one of these forms:
|
||||
Every new pull request is tracked in exactly one of two mutually exclusive
|
||||
ways:
|
||||
|
||||
1. It deliberately references at least one existing issue in its title or
|
||||
description. Tracker metadata stays on that issue and the PR remains
|
||||
unlabelled.
|
||||
2. It carries at least one label directly when a separate issue would add no
|
||||
useful planning context.
|
||||
|
||||
Issue references use one of these forms:
|
||||
|
||||
- `Closes #123`, `Fixes #123`, or `Resolves #123` when merging completes it.
|
||||
- `Part of #123`, `Related to #123`, `Refs #123`, or `References #123` when it
|
||||
contributes without completing it.
|
||||
|
||||
Gitea Actions enforces both PR rules as a status check and repairs the empty
|
||||
issue-label state. Branch protection makes the PR policy check required.
|
||||
Gitea Actions enforces the exclusive either/or PR rule, validates any issue
|
||||
references, and repairs the empty issue-label state. Branch protection makes
|
||||
the PR policy check required.
|
||||
|
||||
The policy applies from 2026-07-18 onward. Existing issues may be labelled as
|
||||
they are encountered, but closed PRs are grandfathered: no retrospective
|
||||
@@ -40,9 +54,13 @@ issues or PR labels are created solely to make history conform.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Classification, priority, and workflow metadata have one source of truth.
|
||||
- A PR's issue link is the navigation path to its planning metadata.
|
||||
- Classification, priority, and workflow metadata have one source of truth for
|
||||
each change: the linked issue when one exists, otherwise the PR.
|
||||
- For issue-backed changes, the PR's issue link is the navigation path to its
|
||||
planning metadata.
|
||||
- Multi-PR issues do not require copied or synchronized labels.
|
||||
- Small standalone changes do not require a tracking issue created solely to
|
||||
satisfy automation.
|
||||
- `Status/Needs Triage` is an intentional fallback, not a final
|
||||
classification.
|
||||
- Direct issue creation remains convenient; automation repairs a missing label
|
||||
@@ -53,5 +71,6 @@ issues or PR labels are created solely to make history conform.
|
||||
## Links
|
||||
|
||||
- Issue #405.
|
||||
- `.gitea/workflows/tracker-policy.yml`.
|
||||
- `.gitea/workflows/tracker-policy-pr.yml`.
|
||||
- `.gitea/workflows/tracker-policy-issues.yml`.
|
||||
- `scripts/tracker_policy.py`.
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
# Design workflow
|
||||
|
||||
How bot-bottle turns discussion into canonical design and then into
|
||||
implementation without leaving the repository's architecture scattered across
|
||||
issue and review threads.
|
||||
|
||||
The goal is not more documentation. The goal is one discoverable current answer
|
||||
for every load-bearing design question.
|
||||
|
||||
## Sources of truth
|
||||
|
||||
Design artifacts have different jobs:
|
||||
|
||||
| Artifact | Authority |
|
||||
|---|---|
|
||||
| Decision records | Stable system-wide boundaries, policies, and invariants |
|
||||
| PRDs | The current design for a feature |
|
||||
| Research notes | Evidence and tradeoff analysis; informative, not normative |
|
||||
| Issues | Work tracking, open questions, and discussion |
|
||||
| Pull-request comments | Review history; never the final home of a design decision |
|
||||
|
||||
When a discussion changes the design, update the relevant PRD or decision
|
||||
record before treating the discussion as resolved. A comment may explain why a
|
||||
decision changed, but future implementers must not need to reconstruct the
|
||||
decision from a thread.
|
||||
|
||||
Avoid duplicating the same rule in several canonical documents. Prefer one
|
||||
canonical statement and links from dependent documents.
|
||||
|
||||
## Choosing the canonical artifact
|
||||
|
||||
Use a PRD when the decision describes a feature: its behavior, scope, success
|
||||
criteria, trust model, implementation slices, and tests.
|
||||
|
||||
Use a decision record when the choice is broader than one feature or will
|
||||
constrain several future features. Examples include state ownership, credential
|
||||
boundaries, compatibility policy, and what the project does or does not claim
|
||||
as a security guarantee.
|
||||
|
||||
Use a research note when the conclusion depends on comparing external systems,
|
||||
protocols, or approaches. Promote any resulting project decision into a PRD or
|
||||
decision record.
|
||||
|
||||
## From discussion to implementation
|
||||
|
||||
### 1. Open the design discussion
|
||||
|
||||
An issue may start with incomplete requirements. Record:
|
||||
|
||||
- the problem and desired outcome;
|
||||
- known security or compatibility constraints;
|
||||
- the current owner of affected state and credentials;
|
||||
- related PRDs, decisions, issues, and pull requests;
|
||||
- open questions that would materially change the implementation.
|
||||
|
||||
Do not disguise an unresolved trust-boundary or state-ownership decision as an
|
||||
implementation detail.
|
||||
|
||||
### 2. Draft or update the canonical design
|
||||
|
||||
Before substantial implementation, write the feature PRD and update any
|
||||
system-wide decision it changes.
|
||||
|
||||
An active design should make these relationships visible near its top:
|
||||
|
||||
```markdown
|
||||
Status: Draft | Active | Superseded | Retargeted
|
||||
Depends on: #...
|
||||
Supersedes: ...
|
||||
```
|
||||
|
||||
Record dependencies only on the dependent document. Do not maintain reverse
|
||||
`Blocks` lists that can drift as dependent work changes.
|
||||
|
||||
For security-sensitive work, state:
|
||||
|
||||
- the exact guarantee and explicit non-guarantees;
|
||||
- trusted and untrusted components;
|
||||
- who creates each identity or attribution field;
|
||||
- who owns durable state;
|
||||
- failure and recovery behavior;
|
||||
- how the design is tested at its boundaries.
|
||||
|
||||
### 3. Resolve review into the repository
|
||||
|
||||
When review settles a design-changing question:
|
||||
|
||||
1. Update the canonical document in the same pull request.
|
||||
2. Mark conflicting documents Superseded or Retargeted, or update them.
|
||||
3. Add or adjust dependency links.
|
||||
4. Leave a concise resolution comment linking to the canonical change.
|
||||
|
||||
A useful resolution comment is:
|
||||
|
||||
```text
|
||||
Resolution: <what was decided>
|
||||
Canonicalized in: <document/section/commit>
|
||||
Supersedes: <older statement, if any>
|
||||
Follow-up: <remaining implementation or question>
|
||||
```
|
||||
|
||||
The resolution is incomplete until the repository reflects it.
|
||||
|
||||
### 4. Check design readiness
|
||||
|
||||
Implementation may begin when:
|
||||
|
||||
- the PRD's material trust, ownership, and compatibility questions are settled;
|
||||
- dependencies and blockers are explicit;
|
||||
- the design agrees with current architecture and decision records;
|
||||
- superseded documents are marked or updated;
|
||||
- success criteria and boundary tests are concrete;
|
||||
- remaining open questions can be answered during implementation without
|
||||
changing the feature's guarantee or component ownership.
|
||||
|
||||
Small exploratory spikes may happen earlier. A spike proves feasibility; it does
|
||||
not establish a production contract or silently settle the design.
|
||||
|
||||
### 5. Implement in ordered slices
|
||||
|
||||
Prefer small, independently reviewable slices after the parent design is
|
||||
accepted. Record the dependency chain explicitly.
|
||||
|
||||
Parallel work is safe when slices do not compete for the same unsettled
|
||||
interface or ownership boundary. If a foundational change will alter the
|
||||
transport, schema, state owner, or trust domain used by another slice, land the
|
||||
foundation first.
|
||||
|
||||
An implementation pull request should identify:
|
||||
|
||||
- the PRD or decision it implements;
|
||||
- the implementation chunk;
|
||||
- its base and blockers;
|
||||
- any design deviation discovered during implementation.
|
||||
|
||||
If implementation reveals a load-bearing design change, pause that slice and
|
||||
update the canonical design. Do not let the code and review thread become an
|
||||
undocumented replacement for the PRD.
|
||||
|
||||
## Dependency and staleness management
|
||||
|
||||
### Dependency direction
|
||||
|
||||
Write dependencies in terms of contracts, not chronology:
|
||||
|
||||
```text
|
||||
credential provisioning contract
|
||||
-> host-controller authentication
|
||||
-> privileged host operations
|
||||
```
|
||||
|
||||
If only part of a feature is blocked, say so. For example, a manifest parser may
|
||||
proceed while that feature's durable audit-storage chunk waits for the canonical
|
||||
audit schema.
|
||||
|
||||
### Superseding documents
|
||||
|
||||
Do not silently edit history to make an old design appear to have always said
|
||||
the new thing. Preserve the rationale, but make current status unmistakable:
|
||||
|
||||
```markdown
|
||||
Status: Superseded
|
||||
Superseded by: <document>
|
||||
Reason: <one paragraph>
|
||||
```
|
||||
|
||||
If part of a PRD remains valid, mark it Retargeted and identify which scope moved
|
||||
elsewhere.
|
||||
|
||||
Add a short supersession note near the top explaining what changed, why the old
|
||||
design is no longer current, and where the current design lives. For a research
|
||||
note whose original analysis remains useful, preserve that analysis and append
|
||||
a dated addendum with the newer finding instead of rewriting the note as though
|
||||
it had always reached the new conclusion.
|
||||
|
||||
### Architecture sweeps
|
||||
|
||||
After a foundational change, do a targeted architecture sweep before building
|
||||
more features on it:
|
||||
|
||||
1. Identify the concepts the change affects, such as `bot-bottle.db`, host
|
||||
controller, orchestrator, audit ownership, or signing key.
|
||||
2. Search active PRDs, decisions, and open issues for those concepts.
|
||||
3. Update or supersede contradictory statements.
|
||||
4. Refresh dependency links and the current architecture summary.
|
||||
5. Confirm stacked implementation branches still have the correct base.
|
||||
|
||||
This is a milestone activity, not a recurring documentation ceremony.
|
||||
|
||||
## Pull-request checklist
|
||||
|
||||
Use the relevant items in design and implementation pull requests:
|
||||
|
||||
- [ ] The canonical PRD or decision is linked.
|
||||
- [ ] Design-changing review decisions are reflected in-repo.
|
||||
- [ ] Dependencies and blockers are explicit.
|
||||
- [ ] State, credential, and trust ownership agree with current architecture.
|
||||
- [ ] Superseded or retargeted documents are marked.
|
||||
- [ ] Security guarantees and non-guarantees are precise.
|
||||
- [ ] Open questions do not change the promised guarantee or ownership model.
|
||||
- [ ] Implementation deviations updated the canonical design.
|
||||
|
||||
## Lightweight maintenance
|
||||
|
||||
Automation should enforce document shape, not pretend to understand
|
||||
architecture. Useful checks include:
|
||||
|
||||
- active PRDs contain status and dependency metadata;
|
||||
- superseded PRDs link to their replacement;
|
||||
- referenced documents and issues exist;
|
||||
- implementation pull requests identify their PRD and chunk;
|
||||
- document filenames and lifecycle states follow repository conventions.
|
||||
|
||||
Human review remains responsible for detecting conflicting guarantees or
|
||||
ownership claims.
|
||||
|
||||
The durable rule is simple: **discussion discovers the decision; the repository
|
||||
records it; implementation follows it.**
|
||||
@@ -4,6 +4,11 @@
|
||||
- **Author:** didericis
|
||||
- **Created:** 2026-05-26
|
||||
|
||||
> **Superseded.** PRD 0049 removed the active-agents pane and agent-scoped
|
||||
> operator edit verbs when the dashboard was narrowed back to a proposal-only
|
||||
> supervise TUI. A future agent-management surface was deferred rather than
|
||||
> carried forward from this design. The design below is retained as history.
|
||||
|
||||
## Summary
|
||||
|
||||
The dashboard today is proposal-centric: it lists every pending
|
||||
|
||||
@@ -4,6 +4,11 @@
|
||||
- **Author:** didericis
|
||||
- **Created:** 2026-05-26
|
||||
|
||||
> **Superseded.** PRD 0049 removed start, re-attach, and stop actions from the
|
||||
> dashboard when it became the proposal-only supervise TUI. Bottle lifecycle
|
||||
> remains in the dedicated CLI commands; no dashboard replacement from this
|
||||
> design remains active. The design below is retained as history.
|
||||
|
||||
## Summary
|
||||
|
||||
Today the dashboard is read-only: it surfaces pending proposals
|
||||
|
||||
@@ -4,6 +4,11 @@
|
||||
- **Author:** didericis
|
||||
- **Created:** 2026-05-26
|
||||
|
||||
> **Superseded.** PRD 0049 removed agent handoff and tmux pane management when
|
||||
> the dashboard was reduced to the proposal-only supervise TUI. The split-pane
|
||||
> interaction described below has no active replacement and is retained only
|
||||
> as design history.
|
||||
|
||||
## Summary
|
||||
|
||||
When the dashboard runs inside tmux, lay it out as the **left
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# PRD prd-new: Encrypted at-rest egress secrets (SecretProvider, interim slice)
|
||||
# PRD 0080: Encrypted at-rest egress secrets (SecretProvider, interim slice)
|
||||
|
||||
- **Status:** Draft
|
||||
- **Author:** didericis
|
||||
+8
-5
@@ -7,10 +7,13 @@ document vs. a research note or a decision record).
|
||||
|
||||
## Naming and numbering
|
||||
|
||||
New PRDs use a `prd-new-<kebab-title>.md` placeholder name while the PR
|
||||
is open. On merge to `main` a CI workflow assigns the next sequential
|
||||
number (`0024-…`, `0025-…`), renames the file, and updates the title
|
||||
header. Numbers are never reused; gaps are fine.
|
||||
New PRDs may use a `prd-new-<kebab-title>.md` placeholder name while the
|
||||
design is being drafted. Before merge, assign the next sequential number
|
||||
after the highest-numbered PRD on `main`, rename the file to
|
||||
`NNNN-<kebab-title>.md`, and update the title header. CI blocks merging
|
||||
while any `prd-new-*.md` placeholder remains. If concurrent PRs select the
|
||||
same number, the later PR must take the next available number before it
|
||||
merges. Numbers are never reused; gaps are fine.
|
||||
|
||||
Once numbered, the filename stays fixed for the life of the doc.
|
||||
|
||||
@@ -26,7 +29,7 @@ The `Status:` line near the top tracks the PRD's lifecycle:
|
||||
## Format
|
||||
|
||||
```markdown
|
||||
# PRD prd-new: <short title> ← placeholder; CI fills in the number on merge
|
||||
# PRD prd-new: <short title> ← replace with the final number before merge
|
||||
|
||||
- **Status:** Draft
|
||||
- **Author:** <who>
|
||||
|
||||
@@ -0,0 +1,732 @@
|
||||
# PRD prd-new: Canonical tamper-evident audit-event schema and local query contract
|
||||
|
||||
- **Status:** Draft
|
||||
- **Author:** didericis-claude
|
||||
- **Created:** 2026-07-26
|
||||
- **Issue:** #487
|
||||
|
||||
## Summary
|
||||
|
||||
bot-bottle already emits security- and provenance-relevant events from
|
||||
several producers — supervise operator decisions (PRD 0013's
|
||||
`AuditStore`), egress allow/block enforcement, git-gate push decisions,
|
||||
control-plane token minting, and (next) host-controller lifecycle
|
||||
transitions — but each writes its own shape to its own sink. There is no
|
||||
shared envelope, no tamper-evidence, and no single place to search. Local
|
||||
incident reconstruction means grepping several stores that don't agree on
|
||||
field names, timestamps, or how a bottled agent is identified.
|
||||
|
||||
This PRD defines **one canonical audit-event contract** every producer
|
||||
emits into:
|
||||
|
||||
1. A **versioned envelope** — schema version, event id, event/observed
|
||||
timestamps, host-attributed identity (`bottle`/`bottled_agent`/
|
||||
`activation`), provenance (`manifest_digest` — the manifest is the
|
||||
policy — and `engine` = bot-bottle version/SHA),
|
||||
host-observed `actor`/`action`/`resource`/`outcome`, correlation/
|
||||
causation ids, a sensitivity class, a typed payload, and an explicit
|
||||
**trust boundary** between host-supplied and agent-claimed fields.
|
||||
2. **Canonical JSON serialization + a per-writer hash chain** (with
|
||||
normative test vectors), so any deletion, edit, or reorder of a past
|
||||
record breaks the chain and is detectable offline.
|
||||
3. An **append-only JSONL journal as the source of truth**, with a
|
||||
**rebuildable SQLite index** and a local `audit query`/`verify` surface
|
||||
— no paid platform, no network dependency.
|
||||
4. An **initial event registry** covering lifecycle, host-controller,
|
||||
supervise decision, egress (request/decision/cutoff/anomaly), git-gate
|
||||
and signed-commit (#480), auth/authz, and audit self-events — each with
|
||||
its trusted-vs-claimed fields and redaction rules.
|
||||
5. A **stable export projection** (CloudEvents / OpenTelemetry Logs) and
|
||||
the **#324 delivery contract** (payload, per-chain
|
||||
`(host, epoch, seq)` cursor, dedup,
|
||||
backpressure, retention ordering).
|
||||
|
||||
It is explicitly scheduled to land **immediately after the host
|
||||
controller (#468)** so the host controller's lifecycle transitions are the
|
||||
first producer wired onto the new contract (per the directive on #487).
|
||||
|
||||
## Problem
|
||||
|
||||
Audit infrastructure is fragmented across #468, #324, and #480 with no
|
||||
shared schema. Concretely:
|
||||
|
||||
- **No shared envelope.** `supervise_audit_entries` (PRD 0013) has
|
||||
`timestamp, bottle_slug, component, operator_action, ...`. The egress
|
||||
proxy and git-gate log their own ad-hoc lines. There is no common
|
||||
`event_id`, `event_type`, or version, so cross-producer correlation
|
||||
("what did bottled agent X do between its start and this rejected push?") is
|
||||
manual and lossy.
|
||||
- **No tamper-evidence.** The audit store is a plain SQLite table. Anyone
|
||||
who can write the DB can delete or rewrite a row and leave no trace.
|
||||
Audit that an attacker (or a buggy agent) can silently rewrite is not
|
||||
audit.
|
||||
- **Trusted and untrusted data are mixed.** A bottled agent is attributed by
|
||||
**source IP → slug** at the gateway (host-supplied, trustworthy). An
|
||||
agent can also *claim* things about itself in a tool call
|
||||
(agent-claimed, adversarial). Today nothing in the record marks which is
|
||||
which, so a reader can be misled by an agent-supplied field that looks
|
||||
authoritative.
|
||||
- **No local search.** Reconstructing an incident means reading multiple
|
||||
sinks with different schemas. There is no query contract and no promise
|
||||
that the index can be rebuilt from the journal if it drifts or is lost.
|
||||
- **No redaction rule.** Nothing prohibits a producer from writing a raw
|
||||
token or secret into an audit record, which would turn the audit log
|
||||
itself into a credential store.
|
||||
|
||||
## Goals / Success Criteria
|
||||
|
||||
- A single `AuditEvent` envelope type, versioned, that every producer
|
||||
emits. The trust boundary is **one `untrusted` region**: everything
|
||||
outside it is host-established and trusted (source-IP → bottled-agent
|
||||
attribution, host wall-clock, producer identity, chain metadata);
|
||||
`untrusted` is the sole place anything an agent or a remote claimed may
|
||||
go. The boundary is structural, not a convention.
|
||||
- **Canonical serialization** (`sort_keys`, `(",", ":")` separators,
|
||||
UTF-8, `ensure_ascii=False`) is defined once and reused, so the same
|
||||
logical event always hashes identically across producers and hosts.
|
||||
- Each writer maintains a **hash chain**: `hash = sha256(prev_hash ||
|
||||
canonical(event))`. Deleting or editing any past record breaks every
|
||||
subsequent link; a standalone verifier detects the break offline with no
|
||||
secret material.
|
||||
- The **JSONL journal is the source of truth**; the **SQLite index is
|
||||
fully rebuildable** from it (`audit rebuild` reconstructs the DB and
|
||||
re-verifies the chain).
|
||||
- **Local query** works with no paid platform and no egress: filter by
|
||||
bottled agent, event type, time range, and producer, and follow a bottled
|
||||
agent's events in order.
|
||||
- A **redaction rule** is enforced at the envelope boundary: known
|
||||
credential-shaped fields are rejected/redacted before a record is
|
||||
written; the writer refuses raw secrets rather than storing them.
|
||||
- The envelope **projects onto the OpenTelemetry Logs data model and a
|
||||
CloudEvents JSON envelope** by field re-mapping alone (no reformat),
|
||||
preserving the trust boundary and carrying the integrity fields — per
|
||||
#487's export/interop requirement. (The export adapters are follow-up;
|
||||
the *schema* must make them a re-map.)
|
||||
- The **host controller (#468)** emits `lifecycle.*` events through this
|
||||
contract as the first consumer; existing supervise/egress producers are
|
||||
migrated behind the same envelope without changing operator-facing
|
||||
behavior.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- **Cross-host aggregation / shipping.** This PRD makes each host's journal
|
||||
canonical and correlatable *by construction* (stable ids, hash chain),
|
||||
but the transport that merges multiple hosts into one timeline is a
|
||||
follow-up (#324). The schema is designed so that merge is a later append,
|
||||
not a reformat.
|
||||
- **Cryptographic signing / external anchoring.** Hash-chaining gives
|
||||
tamper-**evidence** (you can detect edits), not tamper-**resistance**
|
||||
against an attacker who can rewrite the whole chain. Per-writer signing
|
||||
keys and periodic external anchoring are a follow-up; the chain-head hash
|
||||
is the seam they attach to.
|
||||
- **Real-time alerting / SIEM rules.** Query is local and pull-based here.
|
||||
- **Retention / rotation policy.** Journal rotation and TTL are operator
|
||||
policy, tracked separately; the format must survive rotation (chain head
|
||||
carried across segments) but this PRD does not set the schedule.
|
||||
- **Replacing PRD 0013's operator queue.** The supervise proposal/response
|
||||
queue is unchanged; only its terminal *audit* record is re-emitted onto
|
||||
the new envelope.
|
||||
|
||||
## Design
|
||||
|
||||
### The envelope
|
||||
|
||||
One dataclass, `AuditEvent`, serialized to a JSON object with a small,
|
||||
stable top level:
|
||||
|
||||
```
|
||||
{
|
||||
// ---- schema + integrity (host-owned) ----
|
||||
"v": 1, // schema version — bumped only on a breaking change
|
||||
"id": "<uuid4>", // globally unique event id; stable across export/replay (dedup key)
|
||||
"type": "egress.decision", // dotted event type from the registry
|
||||
"epoch": 7, // writer-boot counter, bumped once per host-controller (writer) start
|
||||
"seq": 1287, // monotonic sequence within this epoch (gap-detectable)
|
||||
"segment": "20260726T000000Z", // journal segment id (rotation boundary); chain continues across segments
|
||||
"prev": "<hex>", // prior record hash ("" only for the first record of a new chain)
|
||||
"hash": "<hex>", // sha256(prev + canonical(this event with hash=""))
|
||||
|
||||
// ---- timestamps (host-owned) ----
|
||||
"ts_event": "2026-07-26T18:22:04.061Z", // when the underlying event occurred at the boundary
|
||||
"ts_recorded": "2026-07-26T18:22:04.113Z", // when the single writer appended it (authoritative)
|
||||
"ts_mono": 90142.55, // monotonic secs since this epoch's boot (intra-epoch ordering only)
|
||||
|
||||
// ---- attribution + provenance (host-established) ----
|
||||
"producer": "egress", // host component that emitted the event
|
||||
"host": "mac-studio-1",
|
||||
"engine": "bot-bottle/0.1.0+abc1234", // bot-bottle version + git SHA of the enforcing host code
|
||||
"bottle": "amber-fox", // bottle (container/VM) identity
|
||||
"bottled_agent": "amber-fox-12", // bottled-agent slug from source-IP attribution (null for host-level events)
|
||||
"activation": "01J8Z...", // activation id: one run/session of the bottled agent (null if n/a)
|
||||
"manifest_digest": "sha256:9f2…", // digest of the manifest — which IS the policy (egress routes etc.); null if n/a
|
||||
|
||||
// ---- semantics: host-observed facts of what happened ----
|
||||
"actor": "bottled-agent:amber-fox-12", // who acted, as a host-attributed identity
|
||||
"action": "egress.connect", // what was attempted / done
|
||||
"resource": "registry.npmjs.org:443", // what it acted on, as observed at the boundary
|
||||
"outcome": "blocked", // host-decided result: allowed|blocked|deferred|success|failure
|
||||
"sensitivity": "security", // classification: normal|security|restricted (drives redaction + export)
|
||||
|
||||
// ---- correlation (host-assigned) ----
|
||||
"correlation_id": "flow-9c2a…", // groups a related flow (request → decision → cutoff)
|
||||
"causation_id": "<event id>", // the event that directly caused this one ("" if root)
|
||||
|
||||
// ---- typed, trusted, event-specific payload (shape fixed per type in the registry) ----
|
||||
"payload": {
|
||||
"route_id": 4,
|
||||
"detector": "token_patterns"
|
||||
},
|
||||
|
||||
// ---- the ONLY untrusted region: agent- or remote-claimed data ----
|
||||
"untrusted": {
|
||||
"reason": "npm install needs registry.npmjs.org" // the agent's stated justification
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The **trust boundary is a single region, not a split.** Everything outside
|
||||
`untrusted` is trusted by construction — the host established it: schema and
|
||||
chain metadata, both timestamps, the attribution/provenance fields
|
||||
(source-IP → `bottled_agent`/`bottle`/`activation`, `manifest_digest`,
|
||||
`engine`), the host-observed semantics
|
||||
(`actor`/`action`/`resource`/`outcome`), the host-assigned correlation ids,
|
||||
and the typed `payload`. `untrusted` is the **one** place anything an agent
|
||||
or a remote claimed may go (e.g. the agent's free-text `reason`). A reader
|
||||
(or a future policy engine) trusts every field outside `untrusted` for
|
||||
attribution and treats `untrusted.*` — and only `untrusted.*` — as
|
||||
adversarial claims.
|
||||
|
||||
Framing it as "one untrusted region, everything else trusted" removes the
|
||||
mistake where a producer forgets to mark a claimed field: a field is
|
||||
trusted unless it is deliberately placed inside `untrusted`. The
|
||||
construction API enforces this — producers pass trusted fields explicitly
|
||||
and hand all agent/remote-claimed data as the single `untrusted` mapping,
|
||||
so there is no way to emit a top-level field that *looks* authoritative but
|
||||
isn't.
|
||||
|
||||
**Two timestamps** because they answer different questions and can diverge
|
||||
under backpressure: `ts_event` is when the thing happened at the boundary
|
||||
(the proxy saw the connect, the gate saw the push); `ts_recorded` is when
|
||||
the single writer durably appended it. Ordering and the chain use
|
||||
`(epoch, seq)`, never either wall clock. Both are host-set — a bottled
|
||||
agent never supplies a timestamp.
|
||||
|
||||
**The manifest *is* the policy.** bot-bottle has no separate policy
|
||||
artifact — a bottled agent's egress routes and other constraints are
|
||||
declared in its manifest (`bot_bottle/manifest/egress.py`), so
|
||||
`manifest_digest` already pins the ruleset in force; there is no distinct
|
||||
`policy_version`. Given a fixed manifest, the only other thing that can
|
||||
change a decision's outcome is the enforcing code — captured by `engine`
|
||||
(bot-bottle version + git SHA). So two `egress.decision` records with the
|
||||
same `resource` but different `outcome` are explained by exactly one of:
|
||||
different `manifest_digest` (the rules changed) or different `engine` (the
|
||||
enforcer changed). Runtime operator overrides (a supervise `egress-allow`)
|
||||
are themselves audit events, so the effective ruleset at any instant is
|
||||
`manifest_digest` plus the logged, approved deltas — reconstructable from
|
||||
the chain, not from a version stamp.
|
||||
|
||||
**Optionality.** `bottle`/`bottled_agent`/`activation`, `manifest_digest`,
|
||||
and `payload`/`untrusted` are `null`/absent for events that have no such
|
||||
subject (a host-level `hostctl.*` or `audit.*` event has no bottled agent).
|
||||
Absent ≠ empty: a reader distinguishes "no subject" from "unknown". `id`,
|
||||
`type`, the chain fields, both timestamps, `producer`, `host`, `engine`,
|
||||
`actor`, `action`, `outcome`, and `sensitivity` are always present.
|
||||
|
||||
#### Trust provenance of every common field
|
||||
|
||||
| Field | Trust | Set by |
|
||||
|---|---|---|
|
||||
| `v` `id` `type` `epoch` `seq` `segment` `prev` `hash` | trusted | the single writer |
|
||||
| `ts_event` | trusted | emitting host component (boundary) |
|
||||
| `ts_recorded` `ts_mono` | trusted | the single writer |
|
||||
| `producer` `host` `engine` | trusted | the single writer |
|
||||
| `bottle` `bottled_agent` `activation` | trusted | gateway source-IP → slug attribution |
|
||||
| `manifest_digest` | trusted | control plane (the manifest = the policy in force) |
|
||||
| `actor` `action` `resource` `outcome` | trusted | host component that observed/decided it |
|
||||
| `sensitivity` | trusted | registry default for `type`, overridable up (never down) by the producer |
|
||||
| `correlation_id` `causation_id` | trusted | the single writer (assigned as it threads the flow) |
|
||||
| `payload.*` | trusted | emitting host component (shape fixed per `type`) |
|
||||
| `untrusted.*` | **claimed** | copied verbatim from a bottle / gateway / forge / remote |
|
||||
|
||||
Every registry entry (below) restates, per event type, which `payload`
|
||||
keys are required and names any `untrusted` keys it carries — so "trusted
|
||||
vs claimed" is explicit for every event-specific attribute, not just the
|
||||
common ones.
|
||||
|
||||
### Canonical serialization + hash chain
|
||||
|
||||
Serialization is defined once (extends the existing `sha256_hex` /
|
||||
`util.py` helpers):
|
||||
|
||||
```
|
||||
def canonical(event: dict) -> str:
|
||||
return json.dumps(event, sort_keys=True, separators=(",", ":"),
|
||||
ensure_ascii=False)
|
||||
```
|
||||
|
||||
The `hash` field is computed over the canonical form of the event **with
|
||||
`hash` set to `""`**, prefixed by the previous record's hash:
|
||||
|
||||
```
|
||||
digest = sha256_hex(prev_hash + canonical({**event, "hash": ""}))
|
||||
```
|
||||
|
||||
`prev` is the prior record's `hash`; only the first record of a brand-new
|
||||
chain uses `prev = ""`. The first record of a rotated segment carries the
|
||||
preceding segment's head, as specified under *Rotation* below.
|
||||
Two exact rules pin the bytes so the chain is reproducible anywhere:
|
||||
|
||||
1. **Serialize the record with its own `hash` field set to `""`** (present,
|
||||
empty), never omitted — the key set is identical before and after
|
||||
hashing.
|
||||
2. **Digest = `sha256_hex(prev + canonical(record_with_empty_hash))`**,
|
||||
where `prev` is the previous record's `hash` string (`""` at genesis),
|
||||
`+` is string concatenation, and `canonical` is the function above.
|
||||
`ts_mono`, being a float, is serialized by Python's shortest-round-trip
|
||||
`repr` via `json.dumps`; producers therefore emit it as a JSON number
|
||||
they do not post-process. (All other fields are strings/ints/objects,
|
||||
which serialize unambiguously.)
|
||||
|
||||
Editing or deleting record *n* changes its hash, so record *n+1*'s `prev`
|
||||
no longer matches — the break is local and names the tampered record.
|
||||
Verification needs only the journal itself (no keys), so it runs offline
|
||||
and in CI.
|
||||
|
||||
#### Test vectors (normative)
|
||||
|
||||
Two records, reduced to the chain-relevant fields, demonstrate the exact
|
||||
serialization and linkage. An implementation is conformant iff it
|
||||
reproduces these bytes and hashes.
|
||||
|
||||
```
|
||||
# Record 0 — brand-new chain genesis (prev = "")
|
||||
canonical(record0, hash=""):
|
||||
{"hash":"","id":"11111111-1111-4111-8111-111111111111","prev":"","seq":0,"type":"audit.segment_open"}
|
||||
hash0 = sha256("" + canonical) =
|
||||
942ea5729bcac6efdbdea942396bfa574ab0d6ebf5615402595359422f2aeb83
|
||||
|
||||
# Record 1 — chains onto record 0 (prev = hash0)
|
||||
canonical(record1, hash=""):
|
||||
{"hash":"","id":"22222222-2222-4222-8222-222222222222","prev":"942ea5729bcac6efdbdea942396bfa574ab0d6ebf5615402595359422f2aeb83","seq":1,"type":"lifecycle.bottled_agent_start"}
|
||||
hash1 = sha256(hash0 + canonical) =
|
||||
bc082347680405fee50b60a9c304611aa026950b15d869b7e3ae56e1c451b856
|
||||
|
||||
# Tamper check: flip record0.type → recompute →
|
||||
# 4553eda647f33f0c608cfea44be28efbbaca45ed30b873fcbd4405fa5ce737ed
|
||||
# which no longer equals record1.prev (942ea5…) — the break is detected at record1.
|
||||
```
|
||||
|
||||
The implementation PR ships these plus full-envelope vectors (every field
|
||||
populated, and a redaction case) as committed fixtures, so a schema-version
|
||||
bump that changes the bytes fails a golden test loudly.
|
||||
|
||||
### Ordering, idempotency, and duplicate handling
|
||||
|
||||
- **Ordering.** `(epoch, seq)` is a strict total order within one host's
|
||||
native chain and, because
|
||||
the writer is single, a strict order per bottle/activation within that
|
||||
host — satisfying "at least strict causal order per activation/bottle".
|
||||
`causation_id` records the explicit cause edges (a DAG) on top of the
|
||||
total order, so a consumer can reconstruct request → decision → cutoff
|
||||
even if unrelated events interleave between them. There is deliberately
|
||||
no invented total order across imported host chains; a cross-host key is
|
||||
`(host, epoch, seq)`, and consumers use correlation/causation edges where
|
||||
causal ordering across hosts is known.
|
||||
- **Idempotency.** `id` is the idempotency key. A producer that retries an
|
||||
emit (e.g. after a writer restart mid-handoff) **reuses the same `id`**;
|
||||
before appending, the writer checks a durable, host-wide id ledger that
|
||||
spans every segment and survives restart/retention, and drops an id
|
||||
already present. The ledger is operational metadata, not an audit source
|
||||
of truth: after a crash it is reconciled from the journal before appends
|
||||
resume, and retention preserves id tombstones after journal segments are
|
||||
pruned. An in-memory set may cache the ledger but is never the authority.
|
||||
The index additionally has a unique key on `id`; its `UPSERT` is
|
||||
defensive and does not substitute for the pre-append check. Thus a
|
||||
duplicate never enters the source-of-truth journal, double-counts, or
|
||||
forks the chain.
|
||||
- **Deduplication downstream.** Because `id` is stable across export and
|
||||
replay, #324's cursor replay and any cross-host merge dedup on `id` — no
|
||||
consumer needs to invent a second identity.
|
||||
|
||||
### Behavior across rotation, restart, import, truncation
|
||||
|
||||
- **Rotation.** At a segment boundary the writer opens a new segment file,
|
||||
sets its `segment` id, and carries the rotated-out segment's head as the
|
||||
new segment's first `prev` — so the chain is continuous *across* segments
|
||||
(`prev = ""` is reserved for the first record of a brand-new chain) while
|
||||
each file stays independently openable. `verify` walks segments in order
|
||||
and checks the preceding-head-to-first-record link at each seam.
|
||||
- **Restart.** Covered above: read last line → adopt its `hash` as `prev`,
|
||||
bump `epoch`, reset `seq`. The chain never restarts even though the
|
||||
counters do.
|
||||
- **Import.** `audit import <segment>` appends an externally supplied
|
||||
segment (e.g. recovered from another host or a backup). Import verifies
|
||||
the incoming chain in isolation first. A continuation whose first `prev`
|
||||
matches a known head extends that chain. A foreign chain is registered as
|
||||
a separate immutable chain namespace rather than rewriting or grafting
|
||||
its records (which would invalidate their hashes). Imported records keep
|
||||
their original `host`, `id`, `epoch`, and `seq`; the index keys their
|
||||
native order by `(host, epoch, seq)` so attribution is not laundered and
|
||||
tuples from different hosts cannot collide.
|
||||
- **Truncation.** A crash can leave a partial final line; `verify` reports
|
||||
it as `truncated-tail` (recoverable — replay resumes from the last intact
|
||||
record). A chain that ends before a persisted head, or a missing interior
|
||||
`seq`, is reported as `gap`/`missing-suffix` (evidence of deletion, not a
|
||||
clean crash). The two are distinguished so an operator can tell "power
|
||||
loss" from "someone trimmed the log".
|
||||
|
||||
### Single writer; ordering across restarts
|
||||
|
||||
**Decided: one writer per host** (reviewed — the host controller owns it).
|
||||
Producers hand events to the host controller, which is the sole appender,
|
||||
so the chain has one well-defined total order and one `seq`/`epoch`
|
||||
counter. This ties audit availability to the host controller being up,
|
||||
which is acceptable because the host controller already gates every
|
||||
lifecycle transition; per-producer chains are noted only as a future
|
||||
scaling path, not built now.
|
||||
|
||||
**Restarts** are handled by the chain, not the clock. `ts_mono` resets to
|
||||
~0 on every writer start, so it orders events only *within* one boot. On
|
||||
start the writer:
|
||||
|
||||
1. reads the last line of the journal, adopts its `hash` as the next
|
||||
record's `prev` (the chain is continuous across the restart), and
|
||||
2. bumps `epoch` (persisted alongside the chain head) and resets `seq` to
|
||||
0 for the new boot.
|
||||
|
||||
Total order within this host's native chain is therefore `(epoch, seq)` —
|
||||
monotonic across restarts by construction — with
|
||||
`ts_event`/`ts_recorded` for human reading and `ts_mono` for sub-second
|
||||
ordering inside an epoch. Imported chains retain their own
|
||||
`(host, epoch, seq)` order and do not acquire a fictional order relative to
|
||||
the local chain. A crash mid-append truncates at most the last (partial)
|
||||
line; the verifier flags it and replay resumes from the last intact record.
|
||||
|
||||
### Journal (source of truth) + SQLite index (rebuildable)
|
||||
|
||||
- **Journal:** one append-only JSONL file per host (path from `paths.py`,
|
||||
alongside `host_db_path()`), one canonical event per line, opened
|
||||
`O_APPEND`. This is authoritative.
|
||||
- **Index:** a new `audit_events` table via the existing `DbStore` /
|
||||
`TableMigrations` machinery. It is a **derived cache, not a second source
|
||||
of truth**: `audit rebuild` truncates and replays the journal,
|
||||
re-verifying the chain as it goes, so a deleted or drifted DB is
|
||||
regenerated from the journal with no data loss. On a `verify` failure
|
||||
during rebuild it stops and reports rather than indexing past a break.
|
||||
(This supersedes the free-standing `supervise_audit_entries` table, which
|
||||
becomes a producer onto the new index.)
|
||||
|
||||
**Indexable fields** (columns + indices): `ts_event`, `ts_recorded`,
|
||||
`type`, `host`, `bottle`, `bottled_agent`, `activation`, `actor`,
|
||||
`outcome`, `sensitivity`, `correlation_id`, `causation_id`, plus two
|
||||
event-specific projections promoted out of `payload` for query —
|
||||
`repository` and `commit_sha` (populated for `forge.*`/`commit.*`, null
|
||||
otherwise). The unique event id and native-order index are respectively
|
||||
`id` and `(host, epoch, seq)`; a local-only `ingest_seq` provides stable
|
||||
display order when a query intentionally mixes chains without pretending
|
||||
that it is causal order. The full canonical record is stored verbatim in a
|
||||
`raw` column so the index never loses fidelity to the journal.
|
||||
|
||||
**Local query surface** — `audit query`, no egress, no paid platform:
|
||||
|
||||
```
|
||||
audit query \
|
||||
[--since T] [--until T] [--type egress.*] [--host H] [--bottle B] \
|
||||
[--activation A] [--agent SLUG] [--actor ID] [--outcome blocked] \
|
||||
[--repository R] [--correlation-id C] [--commit SHA] \
|
||||
[--follow BOTTLE] # one host chain's events in (epoch, seq) order
|
||||
[--json | --table]
|
||||
audit verify [--segment S] # offline chain check; exit non-zero on any break
|
||||
audit rebuild # drop + replay journal → index
|
||||
audit import <segment> # graft an external segment (see above)
|
||||
```
|
||||
|
||||
Type filters accept a `group.*` glob. A read-only local HTTP endpoint
|
||||
mirrors the same filters for the future review console; both are pure reads
|
||||
over the index and can never mutate the journal.
|
||||
|
||||
### Event registry (initial)
|
||||
|
||||
Dotted `type` names, grouped. The registry is a table mapping each type to
|
||||
its required `payload` keys, its `untrusted` keys (if any), a default
|
||||
`sensitivity`, and its correlation behavior — so producers and the verifier
|
||||
agree on shape and "trusted vs claimed" is pinned per type. Initial
|
||||
coverage (the issue's mandated set):
|
||||
|
||||
| Group / type | Producer | Required `payload` (trusted) | `untrusted` | Default sensitivity |
|
||||
|---|---|---|---|---|
|
||||
| **lifecycle.*** — `bottled_agent_start` / `_stop` / `_crash` | host-controller (#468) | `manifest_digest`, `exit` (for stop/crash) | — | normal |
|
||||
| **hostctl.*** — `broker_launch`, `broker_teardown`, `broker_reject` | host-controller (#468) | `op`, `request_digest` | — | security |
|
||||
| **decision.*** — `proposed`, `resolved` | supervise | `tool`, `operator_action`, `justification`, `diff_digest` | `agent_rationale` | security |
|
||||
| **egress.*** — `request`, `decision`, `cutoff`, `anomaly` | egress proxy | `route_id`, `detector` (on match), `bytes` (cutoff) | `reason`, `target_claimed` | security |
|
||||
| **forge.*** — `push_accepted`, `push_rejected`, `pr_opened` | git-gate | `repository`, `ref`, `gitleaks_result` | `title`, `description` | security |
|
||||
| **commit.signed** (#480) | git-gate | `repository`, `commit_sha`, `activation_key_id`, `signature_ref` | `commit_message` | security |
|
||||
| **auth.*** — `token_minted`, `token_rejected`, `authz_denied` | control plane | `role`, `token_id`, `reason_code` | — | security |
|
||||
| **audit.*** — `segment_open`, `verify_failed`, `truncation_detected`, `export_failed` | audit writer/verifier | `segment`, `detail` | — | security |
|
||||
|
||||
Notes:
|
||||
- **`egress.request` vs `egress.decision`** share a `correlation_id`; the
|
||||
`decision`'s `causation_id` points at the `request`, and a later `cutoff`
|
||||
chains onto the `decision` — so a flow is reconstructable.
|
||||
- **`audit.*` self-events** make the audit subsystem audit itself: a failed
|
||||
verification, a detected truncation, or a dropped export is itself a
|
||||
chained, tamper-evident record — you cannot silence the alarm without
|
||||
breaking the chain that carries it.
|
||||
- **Free-text and remote-echoed fields are always `untrusted`** (`reason`,
|
||||
`agent_rationale`, PR `title`/`description`, `target_claimed`), because
|
||||
they originate in the bottle or a remote response; the host-observed
|
||||
counterpart (`resource`, `outcome`, `gitleaks_result`) is the trusted
|
||||
fact.
|
||||
|
||||
**Sensitivity + redaction per type.** Every type's default `sensitivity`
|
||||
is listed above; a producer may raise it (never lower it). `restricted`
|
||||
events keep their `payload` in the journal but the export projection ships
|
||||
only the envelope + a payload digest unless the consumer is authorized —
|
||||
so a `security`/`restricted` record is still counted and correlated
|
||||
downstream without leaking its body. The credential-shape redaction rules
|
||||
(next) apply to **every** type regardless of sensitivity.
|
||||
|
||||
**Schema evolution & backward-compatible readers.** The registry is
|
||||
append-only: **adding** a type, an optional `payload` key, or an
|
||||
`untrusted` key does **not** bump `v`; readers ignore unknown fields
|
||||
(forward-compatible) and treat absent optional fields as `null`.
|
||||
**Removing** or **re-typing** a field, or making an optional field
|
||||
required, bumps `v`. A reader declares the max `v` it understands and
|
||||
refuses to *interpret* a higher-`v` record, but the **verifier is
|
||||
version-agnostic** — the hash covers whatever fields exist, so chain
|
||||
integrity is checkable across versions without understanding semantics.
|
||||
Every `v` bump ships a migration note and updated golden vectors.
|
||||
|
||||
### Redaction rule
|
||||
|
||||
Redaction runs at the envelope boundary, before a record is written, in two
|
||||
layers:
|
||||
|
||||
1. **Key deny-list (structural).** A field whose *key* matches a known
|
||||
credential shape (`token`, `secret`, `password`, `authorization`,
|
||||
`*_key`) is refused — the producer must pass a reference (a token *id*
|
||||
or `sha256` fingerprint), never the raw value. `auth.token_minted`
|
||||
therefore records the token id and role, not the JWT. This is the
|
||||
primary guard: it is cheap, deterministic, and catches the intended
|
||||
mistake (a producer stuffing a credential into a named field).
|
||||
|
||||
2. **Value scan — reuse the egress DLP detectors.** Per review, the value
|
||||
layer reuses the *same* deterministic credential-shape detectors the
|
||||
egress proxy already ships:
|
||||
`bot_bottle/gateway/egress/dlp_detectors.py` —
|
||||
`scan_token_patterns` / `redact_tokens` (and `scan_known_secrets` for
|
||||
host-known secret material). They are pure-Python, mitmproxy-free, and
|
||||
already the project's source of truth for "what a leaked credential
|
||||
looks like," so a single detector set governs both what may leave over
|
||||
the wire and what may land in the journal — they can't drift apart.
|
||||
|
||||
**Scoped deliberately:** only the pattern/known-secret detectors are
|
||||
reused, **not** `scan_entropy`. Entropy scoring is tuned for large
|
||||
streamed request bodies; on the short, high-entropy structured values an
|
||||
audit event legitimately carries (hashes, uuids, base64 ids) it would
|
||||
false-positive and start redacting the very fingerprints the log needs.
|
||||
So the shared layer is the deterministic detectors; entropy stays an
|
||||
egress-only concern. (This is the "evaluate how reasonable that is" from
|
||||
review: reuse the deterministic detectors — yes; share the entropy
|
||||
heuristic — no.)
|
||||
|
||||
On a value-layer match the default is **redact** (scrub to a placeholder
|
||||
and keep the event) rather than drop, so a producer bug can never make an
|
||||
audit event vanish; the key deny-list stays a hard refusal because a
|
||||
credential in a named field is always a producer bug worth surfacing.
|
||||
|
||||
**No raw-payload capture by default.** The envelope carries *decisions and
|
||||
metadata*, not traffic. Prompts, model responses, request/response bodies,
|
||||
and file contents are **not** recorded unless a producer opts a specific,
|
||||
reviewed field in — and such a field is `untrusted` and subject to both
|
||||
redaction layers. This keeps the audit log from becoming a covert copy of
|
||||
the very data the sandbox exists to contain (the issue's "unsafe payload
|
||||
capture" non-goal).
|
||||
|
||||
### Export / interoperability (CloudEvents, OpenTelemetry Logs)
|
||||
|
||||
#487 requires the envelope to map onto the **OpenTelemetry Logs data
|
||||
model** and/or a **CloudEvents JSON** envelope *without losing integrity or
|
||||
attribution semantics*. The flattened shape (one `untrusted` region,
|
||||
everything else trusted at top level) does **not** conflict with either — it
|
||||
maps *more* cleanly than a nested `trusted`/`untrusted` pair would, because
|
||||
both target models expect a flat set of top-level fields plus one payload
|
||||
subtree.
|
||||
|
||||
**CloudEvents.** Context attributes MUST be scalar simple types — a map
|
||||
cannot be a context attribute — so a nested `trusted` block would have had
|
||||
to be flattened for CloudEvents anyway. Our flat top level maps directly:
|
||||
`id`→`id`, `type`→`type`, `producer`+`host`→`source`,
|
||||
`bottled_agent`→`subject`, `ts_event`→`time` (`ts_recorded` as an extension); the integrity/chain fields
|
||||
(`epoch`, `seq`, `prev`, `hash`, `v`) ride as **extension attributes**
|
||||
(scalars — legal). The `untrusted` map goes in `data`. Only mechanical
|
||||
transform needed: extension attribute names must be lowercase-alphanumeric,
|
||||
so `bottled_agent`/`ts_mono`/etc. are renamed at export (e.g. a
|
||||
`botbottle`-prefixed form) — a naming rule, not a schema conflict.
|
||||
|
||||
**OpenTelemetry Logs.** `ts_event`→`Timestamp`, `ts_recorded`→`ObservedTimestamp`; `type`→the `event.name`
|
||||
attribute; the flat trusted fields → `Attributes` under a `botbottle.*`
|
||||
namespace (`botbottle.bottled_agent`, `botbottle.producer`,
|
||||
`botbottle.chain.hash`, …); `untrusted.*` → `Attributes` under
|
||||
`botbottle.untrusted.*` (or `Body`). OTel attributes are a dotted map that
|
||||
happily carries the nested subtree.
|
||||
|
||||
**Attribution is preserved** precisely because the boundary is now
|
||||
structural: on export, top-level fields become trusted context/attributes
|
||||
and the `untrusted` subtree stays a single, clearly-named region — so a
|
||||
downstream consumer still sees exactly which fields an agent claimed.
|
||||
Nothing agent-claimed is promoted to a trusted-looking position.
|
||||
|
||||
**Integrity has one deliberate caveat.** CloudEvents/OTel are
|
||||
representation envelopes with their own (or no) canonicalization; `hash`
|
||||
and `prev` are computed over **our** canonical JSON, not over the exported
|
||||
form. So the chain fields travel *as data* for reference, but
|
||||
tamper-evidence is always verified against the **native journal** (the
|
||||
source of truth) — never re-derived from an exported CloudEvents/OTel
|
||||
record, whose key ordering / number formatting the exporter may change.
|
||||
Export is thus a lossless-for-attribution **projection** that carries the
|
||||
integrity fields along; verification stays on the canonical journal. This
|
||||
satisfies "without losing integrity or attribution semantics": both are
|
||||
carried, neither is *relied upon* in the foreign format.
|
||||
|
||||
The export adapters themselves are follow-up implementation — this PRD
|
||||
fixes the *schema* so that projection is a field re-map, never a reformat.
|
||||
|
||||
#### The #324 delivery contract (payload, cursor, backpressure)
|
||||
|
||||
#324 transports events off-box; it must not invent a second envelope. This
|
||||
PRD fixes the contract it depends on:
|
||||
|
||||
- **Payload.** #324 ships the **native canonical record verbatim** (the
|
||||
exact bytes the hash covers), optionally wrapped in the CloudEvents
|
||||
projection whose `data` *is* that record. Either way the integrity fields
|
||||
travel intact and the receiver can verify against the same bytes.
|
||||
- **Cursor.** Export maintains one cursor per native host chain:
|
||||
`(host, epoch, seq)` (equivalently that chain's last exported `hash`).
|
||||
It advances **only on acknowledgement**, so delivery is at-least-once and
|
||||
gap-free; a crash re-sends from the last acked cursor. Imported foreign
|
||||
chains use independent cursors and never share a bare `(epoch, seq)`
|
||||
namespace with the local chain.
|
||||
- **Idempotency / replay.** Dedup is on `id` (stable across replay), so
|
||||
at-least-once delivery is safe — the receiver collapses re-sends.
|
||||
- **Backpressure.** The outbox is the journal itself plus a cursor; when
|
||||
the endpoint is slow the cursor simply lags — the writer never blocks on
|
||||
export, and audit never applies backpressure to the data plane it
|
||||
records.
|
||||
- **Retention interaction.** Retention/rotation **must not** prune a
|
||||
segment whose records are still behind the export cursor; the reaper
|
||||
honors `min(cursor)` across all configured consumers. (The schedule
|
||||
itself stays the retention follow-up; this is the *ordering* constraint
|
||||
that follow-up must respect.)
|
||||
|
||||
#### #480 signed-commit attribution maps in without weakening it
|
||||
|
||||
#480 binds a commit's bytes to a per-activation signing key. It maps to the
|
||||
`commit.signed` event: `payload` carries `repository`, `commit_sha`,
|
||||
`activation_key_id`, and a `signature_ref` (the detached-signature
|
||||
location or its digest) — **not** the private key and not a re-derived
|
||||
signature. The audit event therefore *references and timestamps* #480's
|
||||
existing byte-to-activation-key proof inside the tamper-evident chain; it
|
||||
does not re-implement or replace it, so #480's guarantee is unweakened —
|
||||
the signature still verifies against the commit bytes independently, and
|
||||
the audit record adds only "this binding was observed at this point in the
|
||||
chain". The trusted `actor`/`activation` fields and the `commit_sha`
|
||||
payload are host-observed at the gate, so attribution cannot be forged by
|
||||
the committing agent.
|
||||
|
||||
## Implementation chunks
|
||||
|
||||
1. **(this PR — PRD only.)** The contract above. No code; scheduled to land
|
||||
right after #468.
|
||||
2. **Envelope + canonical + chain core.** `AuditEvent` dataclass,
|
||||
`canonical()`, chain hashing, and the single-writer journal appender in
|
||||
`bot_bottle/store/` (reusing `sha256_hex`); redaction wired to the
|
||||
existing `gateway/egress/dlp_detectors` (`scan_token_patterns` /
|
||||
`redact_tokens`); embed the git SHA at build so `engine` is populated
|
||||
(only `version = "0.1.0"` exists in `pyproject.toml` today — the build
|
||||
must stamp the SHA); unit tests for determinism, chain-break detection,
|
||||
`epoch`/`seq` continuity across a simulated restart, and redaction of
|
||||
both a deny-listed key and a token-shaped value.
|
||||
3. **SQLite index + `audit` CLI.** New `audit_events` migration (indexable
|
||||
fields above); replay-from-journal; offline chain verifier
|
||||
(`truncated-tail` vs `gap`); `query` / `verify` / `rebuild` / `import`;
|
||||
idempotent `UPSERT` by `id`.
|
||||
4. **Host controller as first producer (#468).** Wire
|
||||
`lifecycle.bottled_agent_*` and `hostctl.*` emission into the host
|
||||
controller; establish the `epoch` bump + chain-head carry + segment
|
||||
rotation on writer restart here (it owns the single writer).
|
||||
5. **Migrate existing producers.** Re-emit supervise `decision.*` (retiring
|
||||
the standalone `supervise_audit_entries` shape behind the index), egress
|
||||
`egress.*`, git-gate `forge.*` + `commit.signed` (#480), control-plane
|
||||
`auth.*`; add the `audit.*` self-events (verify/truncation/export
|
||||
failure).
|
||||
6. **CloudEvents / OTel export adapters + #324 delivery.** Projection layer
|
||||
(field re-map per *Export / interoperability*) plus the outbox cursor,
|
||||
ack-driven advance, and retention-ordering guard the #324 contract
|
||||
specifies.
|
||||
7. **(follow-up.)** Cross-host merge transport; per-writer signing +
|
||||
external anchoring on the chain head; retention/rotation *schedule*.
|
||||
|
||||
## Acceptance-criteria coverage (#487)
|
||||
|
||||
The issue defines the contract; implementation is explicitly split into
|
||||
follow-up PRs. This PRD is the durable decision record; each acceptance box
|
||||
maps to a section:
|
||||
|
||||
| #487 acceptance criterion | Where |
|
||||
|---|---|
|
||||
| Durable PRD defines versioned envelope + initial registry | *The envelope*, *Event registry* |
|
||||
| Canonical JSON + hash-chain rules, unambiguous, with test vectors | *Canonical serialization + hash chain* → *Test vectors* |
|
||||
| Trust provenance explicit for every common + event-specific field | *Trust provenance of every common field*; per-type `untrusted` in *Event registry* |
|
||||
| Redaction prohibits credentials / raw secrets / unsafe capture by default | *Redaction rule*; `untrusted`-only claims; sensitivity classes |
|
||||
| JSONL journal canonical; SQLite index fully rebuildable | *Journal + SQLite index* (`audit rebuild`) |
|
||||
| Minimum local search/query contract | *Journal + SQLite index* → *Local query surface* |
|
||||
| #324 can transport/replay without a second envelope | *The #324 delivery contract* |
|
||||
| #480 maps in without weakening its byte-to-activation-key guarantee | *#480 signed-commit attribution maps in…* |
|
||||
| Schema evolution + backward-compatible readers | *Schema evolution & backward-compatible readers* |
|
||||
| Integrity detects modification / deletion / reorder / bad continuation | *Test vectors* (tamper), *Behavior across rotation…truncation*, `audit verify` |
|
||||
|
||||
Two acceptance items are **specified here, implemented later** by design
|
||||
(the issue permits this): the concrete test-vector *fixtures* and the
|
||||
`audit` CLI land in impl chunks 2–3; the #324 outbox lands in chunk 6.
|
||||
Nothing in the contract is left undefined — only its code is deferred.
|
||||
|
||||
## Resolved in review (#495)
|
||||
|
||||
- **Single writer per host — decided.** The host controller owns the sole
|
||||
appender; per-producer chains are a future scaling path only. (Design →
|
||||
*Single writer; ordering across restarts*.)
|
||||
- **Restarts — decided.** An `epoch` counter (bumped per writer boot) plus
|
||||
carrying the last chain head as the next `prev` gives a total order of
|
||||
`(epoch, seq)` that survives restarts; `ts_mono` orders only within an
|
||||
epoch. (Design → *ordering across restarts*.)
|
||||
- **Flatten to one `untrusted` region — decided.** Everything outside
|
||||
`untrusted` (chain metadata, `producer`/`host`, `bottled_agent`, `ts_*`)
|
||||
is trusted by construction, so the separate `trusted` sub-block is
|
||||
removed; a field is trusted unless deliberately placed under `untrusted`.
|
||||
(Design → *The envelope*.)
|
||||
- **Subject term is `bottled_agent` everywhere** — the top-level field and
|
||||
the `lifecycle.bottled_agent_*` leaf names. (Design → *The envelope* /
|
||||
*Event registry*.)
|
||||
- **Retention head-carry — yes.** When a journal segment is rotated out,
|
||||
the new segment's first record carries the rotated-out head as `prev`, so
|
||||
the verifier still trusts the current head across a rotation. (Folds into the
|
||||
retention follow-up.)
|
||||
- **Redaction reuses the egress detectors — yes, scoped.** Reuse the
|
||||
deterministic `dlp_detectors` (`scan_token_patterns` / `redact_tokens` /
|
||||
`scan_known_secrets`); exclude `scan_entropy` as brittle on the short,
|
||||
high-entropy structured values audit records carry. (Design → *Redaction
|
||||
rule*.)
|
||||
|
||||
## Open questions
|
||||
|
||||
- **Value-scan cost on the hot path.** The single writer runs the reused
|
||||
detectors on every event's `untrusted` block inline. Is that cheap enough
|
||||
at lifecycle-event volume, or should the value scan move to index-build
|
||||
time (journal stays raw, index stores the redacted view)? Leaning inline
|
||||
so the raw journal never contains a leaked value in the first place.
|
||||
- **`epoch` persistence location.** Store the per-writer `epoch` + chain
|
||||
head in the SQLite index (rebuildable, but then the writer needs the DB
|
||||
at boot) or in a tiny sidecar file next to the journal (independent of
|
||||
the index)? Leaning sidecar, so the writer can start and append without
|
||||
the index present.
|
||||
@@ -54,18 +54,20 @@ def check_pull_request(event: dict[str, Any], api: GiteaApi) -> list[str]:
|
||||
pull = event["pull_request"]
|
||||
errors: list[str] = []
|
||||
labels = pull.get("labels") or []
|
||||
if labels:
|
||||
errors.append(
|
||||
"PRs must be unlabeled; put tracker metadata on the linked issue "
|
||||
f"(found: {', '.join(label['name'] for label in labels)})."
|
||||
)
|
||||
|
||||
numbers = deliberate_issue_numbers(pull.get("title", ""), pull.get("body", ""))
|
||||
if not numbers:
|
||||
if labels and numbers:
|
||||
errors.append(
|
||||
"PR must reference an issue with Closes/Fixes/Resolves #N, "
|
||||
"Part of #N, Related to #N, Refs #N, or References #N."
|
||||
"PR must use exactly one tracking mode: remove PR labels when "
|
||||
"linking an issue, or remove the issue reference when labels "
|
||||
"belong on the PR."
|
||||
)
|
||||
if not numbers:
|
||||
if not labels:
|
||||
errors.append(
|
||||
"PR must either have a label or reference an issue with "
|
||||
"Closes/Fixes/Resolves #N, Part of #N, Related to #N, "
|
||||
"Refs #N, or References #N."
|
||||
)
|
||||
return errors
|
||||
|
||||
real_issues = 0
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
"""Architecture rules that should fail before coupling becomes entrenched."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class TestCliBackendBoundaries(unittest.TestCase):
|
||||
def test_cli_does_not_import_a_concrete_backend(self) -> None:
|
||||
forbidden = (
|
||||
"backend.docker", "backend.firecracker", "backend.macos_container",
|
||||
"bot_bottle.backend.docker", "bot_bottle.backend.firecracker",
|
||||
"bot_bottle.backend.macos_container",
|
||||
)
|
||||
violations: list[str] = []
|
||||
for path in (ROOT / "bot_bottle" / "cli").rglob("*.py"):
|
||||
tree = ast.parse(path.read_text(), filename=str(path))
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom):
|
||||
module = node.module
|
||||
if module and module.startswith(forbidden):
|
||||
violations.append(
|
||||
f"{path.relative_to(ROOT)}:{node.lineno}: {module}"
|
||||
)
|
||||
if isinstance(node, ast.Import):
|
||||
violations.extend(
|
||||
f"{path.relative_to(ROOT)}:{node.lineno}: {alias.name}"
|
||||
for alias in node.names if alias.name.startswith(forbidden)
|
||||
)
|
||||
self.assertEqual([], violations, "generic CLI imports concrete backend internals:\n" +
|
||||
"\n".join(violations))
|
||||
|
||||
|
||||
class TestRuntimeModuleSizes(unittest.TestCase):
|
||||
def test_no_runtime_module_grows_beyond_global_ceiling(self) -> None:
|
||||
"""A coarse ceiling catches new monoliths; focused caps stay tighter."""
|
||||
ceiling = 850
|
||||
oversized = [
|
||||
f"{path.relative_to(ROOT)} ({len(path.read_text().splitlines())})"
|
||||
for path in (ROOT / "bot_bottle").rglob("*.py")
|
||||
if len(path.read_text().splitlines()) > ceiling
|
||||
]
|
||||
self.assertEqual(
|
||||
[], oversized,
|
||||
f"runtime modules must stay at or below {ceiling} lines: "
|
||||
+ ", ".join(oversized),
|
||||
)
|
||||
|
||||
def test_egress_modules_stay_focused(self) -> None:
|
||||
caps = {
|
||||
"addon_core.py": 100,
|
||||
"schema.py": 400,
|
||||
"types.py": 180,
|
||||
"matching.py": 180,
|
||||
"dlp.py": 180,
|
||||
"context.py": 140,
|
||||
}
|
||||
directory = ROOT / "bot_bottle" / "gateway" / "egress"
|
||||
oversized = [f"{name} ({len((directory / name).read_text().splitlines())}>{cap})"
|
||||
for name, cap in caps.items()
|
||||
if len((directory / name).read_text().splitlines()) > cap]
|
||||
self.assertEqual([], oversized, "split a module rather than raising its cap: " +
|
||||
", ".join(oversized))
|
||||
|
||||
def test_runtime_code_uses_focused_egress_modules(self) -> None:
|
||||
"""addon_core is compatibility-only, never an internal dependency."""
|
||||
violations: list[str] = []
|
||||
package = ROOT / "bot_bottle"
|
||||
facade = package / "gateway" / "egress" / "addon_core.py"
|
||||
package_init = package / "gateway" / "egress" / "__init__.py"
|
||||
for path in package.rglob("*.py"):
|
||||
if path in (facade, package_init):
|
||||
continue
|
||||
text = path.read_text()
|
||||
if "gateway.egress.addon_core import" in text or \
|
||||
".addon_core import" in text:
|
||||
violations.append(str(path.relative_to(ROOT)))
|
||||
self.assertEqual([], violations)
|
||||
|
||||
def test_backend_contract_does_not_absorb_preparation_logic(self) -> None:
|
||||
caps = {
|
||||
ROOT / "bot_bottle" / "backend" / "base.py": 580,
|
||||
ROOT / "bot_bottle" / "backend" / "preparation.py": 160,
|
||||
}
|
||||
oversized = [
|
||||
f"{path.relative_to(ROOT)} "
|
||||
f"({len(path.read_text().splitlines())}>{cap})"
|
||||
for path, cap in caps.items()
|
||||
if len(path.read_text().splitlines()) > cap
|
||||
]
|
||||
self.assertEqual([], oversized)
|
||||
@@ -48,13 +48,12 @@ class TestSharedReprovision(unittest.TestCase):
|
||||
client.reprovision_gateway.side_effect = [
|
||||
OrchestratorClientError("bad key"), True,
|
||||
]
|
||||
with patch("bot_bottle.orchestrator.reprovision.debug") as debug:
|
||||
count = reprovision_bottles(
|
||||
self.assertEqual(
|
||||
1,
|
||||
reprovision_bottles(
|
||||
client, {"10.0.0.1": "key-1", "10.0.0.2": "key-2"},
|
||||
)
|
||||
self.assertEqual(1, count)
|
||||
self.assertEqual("b1", debug.call_args.kwargs["context"]["bottle_id"])
|
||||
self.assertNotIn("bad key", repr(debug.call_args))
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class TestMacosReprovision(unittest.TestCase):
|
||||
|
||||
@@ -9,7 +9,6 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from typing import Any, Optional
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.cli.tui import _filter_items, _multiselect_loop, filter_multiselect, filter_select
|
||||
|
||||
@@ -50,10 +49,8 @@ class TestFilterSelectEmptyItems(unittest.TestCase):
|
||||
|
||||
def test_returns_none_when_tty_unavailable(self):
|
||||
# /nonexistent is guaranteed to not open.
|
||||
with patch("bot_bottle.cli.tui.debug") as debug:
|
||||
result = filter_select(["a", "b"], tty_path="/nonexistent/tty")
|
||||
result = filter_select(["a", "b"], tty_path="/nonexistent/tty")
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual("FileNotFoundError", debug.call_args.kwargs["context"]["error_type"])
|
||||
|
||||
|
||||
class TestFilterMultiselectEmptyItems(unittest.TestCase):
|
||||
@@ -63,10 +60,8 @@ class TestFilterMultiselectEmptyItems(unittest.TestCase):
|
||||
self.assertEqual([], result)
|
||||
|
||||
def test_returns_none_when_tty_unavailable(self):
|
||||
with patch("bot_bottle.cli.tui.debug") as debug:
|
||||
result = filter_multiselect(["a", "b"], tty_path="/nonexistent/tty")
|
||||
result = filter_multiselect(["a", "b"], tty_path="/nonexistent/tty")
|
||||
self.assertIsNone(result)
|
||||
self.assertEqual("FileNotFoundError", debug.call_args.kwargs["context"]["error_type"])
|
||||
|
||||
|
||||
class TestMultiselectLoopReordering(unittest.TestCase):
|
||||
|
||||
@@ -8,18 +8,16 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from bot_bottle.gateway.egress.matching import evaluate_matches
|
||||
from bot_bottle.gateway.egress.schema import (
|
||||
load_config,
|
||||
parse_config,
|
||||
parse_routes,
|
||||
route_to_yaml_dict,
|
||||
)
|
||||
from bot_bottle.gateway.egress.types import (
|
||||
from bot_bottle.gateway.egress.addon_core import (
|
||||
HeaderMatch,
|
||||
MatchEntry,
|
||||
PathMatch,
|
||||
Route,
|
||||
evaluate_matches,
|
||||
load_config,
|
||||
parse_config,
|
||||
parse_routes,
|
||||
route_to_yaml_dict,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -3,16 +3,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.gateway.egress.context import (
|
||||
from bot_bottle.gateway.egress.addon_core import (
|
||||
DENY_RESOLVER_ERROR,
|
||||
DENY_UNATTRIBUTED,
|
||||
DENY_UNPARSEABLE,
|
||||
decide,
|
||||
resolve_client_config,
|
||||
resolve_client_context,
|
||||
)
|
||||
from bot_bottle.gateway.egress.matching import decide
|
||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError
|
||||
|
||||
|
||||
@@ -45,13 +44,7 @@ class TestResolveClientConfig(unittest.TestCase):
|
||||
|
||||
def test_resolver_error_denies_all(self) -> None:
|
||||
# Orchestrator unreachable/errored must never widen egress.
|
||||
with patch("bot_bottle.gateway.egress.context.debug") as debug:
|
||||
config = resolve_client_config(_FakeResolver(raises=True), "10.243.0.1")
|
||||
self.assertEqual((), config.routes)
|
||||
self.assertEqual(
|
||||
"PolicyResolveError", debug.call_args.kwargs["context"]["error_type"],
|
||||
)
|
||||
self.assertNotIn("orchestrator down", repr(debug.call_args))
|
||||
self.assertEqual((), resolve_client_config(_FakeResolver(raises=True), "10.243.0.1").routes)
|
||||
|
||||
def test_unparseable_policy_denies_all(self) -> None:
|
||||
cfg = resolve_client_config(_FakeResolver(result="routes: notalist\n"), "10.243.0.1")
|
||||
|
||||
@@ -12,9 +12,7 @@ from bot_bottle.orchestrator.client import (
|
||||
OrchestratorClient,
|
||||
OrchestratorClientError,
|
||||
RegisteredBottle,
|
||||
BackendProbeFailure,
|
||||
_host_auth_token,
|
||||
_probe_failure,
|
||||
)
|
||||
|
||||
_URLOPEN = "bot_bottle.orchestrator.client.urllib.request.urlopen"
|
||||
@@ -35,15 +33,6 @@ class TestHostAuthToken(unittest.TestCase):
|
||||
self.assertEqual("", _host_auth_token())
|
||||
|
||||
|
||||
class TestBackendProbeFailure(unittest.TestCase):
|
||||
def test_records_safe_typed_diagnostic(self) -> None:
|
||||
with patch("bot_bottle.orchestrator.client.debug") as debug:
|
||||
result = _probe_failure("firecracker", RuntimeError("secret detail"))
|
||||
self.assertEqual(BackendProbeFailure("firecracker", "RuntimeError"), result)
|
||||
rendered = repr(debug.call_args)
|
||||
self.assertNotIn("secret detail", rendered)
|
||||
|
||||
|
||||
def _resp(status: int, payload: object) -> MagicMock:
|
||||
m = MagicMock()
|
||||
inner = m.__enter__.return_value
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Unit tests for per-bottle egress secret encryption (PRD prd-new-secret-provider)."""
|
||||
"""Unit tests for per-bottle egress secret encryption (PRD 0080)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ server tests), plus one real-socket round-trip to prove the handler wiring.
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import secrets
|
||||
import sqlite3
|
||||
@@ -18,7 +17,7 @@ import urllib.error
|
||||
import urllib.request
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint
|
||||
from bot_bottle.orchestrator.broker import StubBroker
|
||||
@@ -284,25 +283,6 @@ class TestServerRoundTrip(unittest.TestCase):
|
||||
))
|
||||
self.assertEqual(reg["bottle_id"], attr["bottle_id"])
|
||||
|
||||
def test_internal_failure_is_contextual_but_redacted(self) -> None:
|
||||
orch = MagicMock()
|
||||
orch.registry.all.side_effect = RuntimeError("SENSITIVE request value")
|
||||
with patch("sys.stderr", io.StringIO()) as stderr:
|
||||
server = make_server(orch, "127.0.0.1", 0)
|
||||
self.addCleanup(server.server_close)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
self.addCleanup(server.shutdown)
|
||||
host, port = server.server_address[0], server.server_address[1]
|
||||
with self.assertRaises(urllib.error.HTTPError) as raised:
|
||||
urllib.request.urlopen(f"http://{host}:{port}/bottles", timeout=5)
|
||||
payload = json.loads(raised.exception.read())
|
||||
output = stderr.getvalue()
|
||||
self.assertEqual({"error": "internal error"}, payload)
|
||||
self.assertIn("GET /bottles", output)
|
||||
self.assertIn("RuntimeError", output)
|
||||
self.assertNotIn("SENSITIVE", output)
|
||||
|
||||
|
||||
class TestOrchestratorAuth(unittest.TestCase):
|
||||
"""Role-scoped control-plane tokens (issue #400 / #469 review): every route
|
||||
@@ -667,24 +647,10 @@ class TestReconcileRoute(unittest.TestCase):
|
||||
self.assertEqual(200, status)
|
||||
self.assertEqual([], payload["reaped"])
|
||||
|
||||
def test_non_string_entries_are_rejected(self) -> None:
|
||||
def test_non_string_entries_are_ignored(self) -> None:
|
||||
dead = self._old("10.0.0.4")
|
||||
status, payload = dispatch(
|
||||
self.orch, "POST", "/reconcile",
|
||||
_body({"live_source_ips": [None, 7, "10.0.0.9"]}))
|
||||
self.assertEqual(400, status)
|
||||
self.assertIn("live_source_ips", str(payload["error"]))
|
||||
|
||||
def test_empty_live_source_ip_is_rejected(self) -> None:
|
||||
status, payload = dispatch(
|
||||
self.orch, "POST", "/reconcile", _body({"live_source_ips": [""]}))
|
||||
self.assertEqual(400, status)
|
||||
self.assertIn("live_source_ips", str(payload["error"]))
|
||||
|
||||
def test_invalid_grace_seconds_is_rejected(self) -> None:
|
||||
for value in (True, "30", -1, float("inf"), float("nan")):
|
||||
with self.subTest(value=value):
|
||||
status, payload = dispatch(
|
||||
self.orch, "POST", "/reconcile",
|
||||
_body({"live_source_ips": [], "grace_seconds": value}))
|
||||
self.assertEqual(400, status)
|
||||
self.assertIn("grace_seconds", str(payload["error"]))
|
||||
self.assertEqual(200, status)
|
||||
self.assertEqual([dead], payload["reaped"])
|
||||
|
||||
@@ -27,7 +27,41 @@ class TestCheckPullRequest(unittest.TestCase):
|
||||
event = {"pull_request": {"title": "Change", "body": "Part of #12", "labels": []}}
|
||||
self.assertEqual(check_pull_request(event, api), [])
|
||||
|
||||
def test_rejects_labels_and_pr_reference(self):
|
||||
def test_accepts_labelled_pr_without_issue(self):
|
||||
api = Mock()
|
||||
event = {
|
||||
"pull_request": {
|
||||
"title": "Change",
|
||||
"body": "",
|
||||
"labels": [{"name": "Kind/Documentation"}],
|
||||
}
|
||||
}
|
||||
self.assertEqual(check_pull_request(event, api), [])
|
||||
api.request.assert_not_called()
|
||||
|
||||
def test_rejects_unlabelled_pr_without_issue(self):
|
||||
api = Mock()
|
||||
event = {"pull_request": {"title": "Change", "body": "", "labels": []}}
|
||||
errors = check_pull_request(event, api)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("either have a label or reference an issue", errors[0])
|
||||
api.request.assert_not_called()
|
||||
|
||||
def test_rejects_labelled_pr_linked_to_real_issue(self):
|
||||
api = Mock()
|
||||
api.request.return_value = {"number": 12, "pull_request": None}
|
||||
event = {
|
||||
"pull_request": {
|
||||
"title": "Change",
|
||||
"body": "Closes #12",
|
||||
"labels": [{"name": "Kind/Documentation"}],
|
||||
}
|
||||
}
|
||||
errors = check_pull_request(event, api)
|
||||
self.assertEqual(len(errors), 1)
|
||||
self.assertIn("exactly one tracking mode", errors[0])
|
||||
|
||||
def test_still_validates_issue_reference_when_both_modes_are_used(self):
|
||||
api = Mock()
|
||||
api.request.return_value = {"number": 12, "pull_request": {}}
|
||||
event = {
|
||||
@@ -39,7 +73,7 @@ class TestCheckPullRequest(unittest.TestCase):
|
||||
}
|
||||
errors = check_pull_request(event, api)
|
||||
self.assertEqual(len(errors), 2)
|
||||
self.assertIn("unlabeled", errors[0])
|
||||
self.assertIn("exactly one tracking mode", errors[0])
|
||||
self.assertIn("not an issue", errors[1])
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user