Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 60039f2eb3 | |||
| bc42836327 |
@@ -2,7 +2,7 @@
|
|||||||
# digest, etc.) without coupling every dev push to upstream registry
|
# digest, etc.) without coupling every dev push to upstream registry
|
||||||
# availability.
|
# availability.
|
||||||
#
|
#
|
||||||
# Opt-in via BOT_BOTTLE_RUN_CANARIES=1 so the same files can be run
|
# Opt-in via CLAUDE_BOTTLE_RUN_CANARIES=1 so the same files can be run
|
||||||
# locally with the same gating.
|
# locally with the same gating.
|
||||||
|
|
||||||
name: canaries
|
name: canaries
|
||||||
@@ -17,7 +17,7 @@ jobs:
|
|||||||
canaries:
|
canaries:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
env:
|
env:
|
||||||
BOT_BOTTLE_RUN_CANARIES: "1"
|
CLAUDE_BOTTLE_RUN_CANARIES: "1"
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
@@ -25,7 +25,4 @@ jobs:
|
|||||||
# No actions/setup-python: canaries are stdlib unittest on the image's
|
# No actions/setup-python: canaries are stdlib unittest on the image's
|
||||||
# system Python 3.12 (older act_runner mishandles setup-python's PATH).
|
# system Python 3.12 (older act_runner mishandles setup-python's PATH).
|
||||||
- name: Run canaries
|
- name: Run canaries
|
||||||
run: |
|
run: python3 -m unittest discover -t . -s tests/canaries -v
|
||||||
python3 -m scripts.unittest_gate \
|
|
||||||
-t . -s tests/canaries -v \
|
|
||||||
--minimum-executed 1 --fail-on-skip
|
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ name: prd-number-check
|
|||||||
on:
|
on:
|
||||||
pull_request:
|
pull_request:
|
||||||
types: [opened, reopened, synchronize]
|
types: [opened, reopened, synchronize]
|
||||||
branches: [main]
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
require-numbered-prds:
|
require-numbered-prds:
|
||||||
|
|||||||
@@ -60,9 +60,6 @@ jobs:
|
|||||||
|
|
||||||
integration-docker:
|
integration-docker:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
concurrency:
|
|
||||||
group: integration-docker-infra
|
|
||||||
cancel-in-progress: false
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
@@ -87,34 +84,7 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
BOT_BOTTLE_BACKEND: docker
|
BOT_BOTTLE_BACKEND: docker
|
||||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.docker
|
COVERAGE_FILE: ${{ github.workspace }}/.coverage.docker
|
||||||
run: |
|
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||||
set -euo pipefail
|
|
||||||
DOCKER_CLIENT_NETWORK=$(
|
|
||||||
docker inspect "$(hostname)" |
|
|
||||||
python3 -c 'import json,sys; n=json.load(sys.stdin)[0]["NetworkSettings"]["Networks"]; print(next(iter(n)))'
|
|
||||||
)
|
|
||||||
test -n "$DOCKER_CLIENT_NETWORK"
|
|
||||||
RUN_KEY="${GITHUB_RUN_ID:-${GITHUB_RUN_NUMBER:-0}}"
|
|
||||||
export NO_PROXY="*"
|
|
||||||
export no_proxy="*"
|
|
||||||
export BOT_BOTTLE_DOCKER_CLIENT_NETWORK="$DOCKER_CLIENT_NETWORK"
|
|
||||||
export BOT_BOTTLE_DOCKER_ROOT_MOUNT="bot-bottle-ci-root-$RUN_KEY"
|
|
||||||
export BOT_BOTTLE_DOCKER_CA_MOUNT="bot-bottle-ci-ca-$RUN_KEY"
|
|
||||||
export BOT_BOTTLE_DOCKER_GIT_MOUNT="bot-bottle-ci-git-$RUN_KEY"
|
|
||||||
export BOT_BOTTLE_DOCKER_CREDS_MOUNT="bot-bottle-ci-creds-$RUN_KEY"
|
|
||||||
python3 -m coverage run -m scripts.unittest_gate \
|
|
||||||
-t . -s tests/integration -v \
|
|
||||||
--minimum-executed 22 --fail-on-skip
|
|
||||||
|
|
||||||
- name: Clean Docker integration volumes
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
RUN_KEY="${GITHUB_RUN_ID:-${GITHUB_RUN_NUMBER:-0}}"
|
|
||||||
docker volume rm --force \
|
|
||||||
"bot-bottle-ci-root-$RUN_KEY" \
|
|
||||||
"bot-bottle-ci-ca-$RUN_KEY" \
|
|
||||||
"bot-bottle-ci-git-$RUN_KEY" \
|
|
||||||
"bot-bottle-ci-creds-$RUN_KEY" 2>/dev/null || true
|
|
||||||
|
|
||||||
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
|
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
|
||||||
- name: Stage docker coverage for upload
|
- name: Stage docker coverage for upload
|
||||||
|
|||||||
@@ -12,14 +12,10 @@ on:
|
|||||||
- 'bot_bottle/**'
|
- 'bot_bottle/**'
|
||||||
- 'tests/**/*.py'
|
- 'tests/**/*.py'
|
||||||
- 'cli.py'
|
- 'cli.py'
|
||||||
- 'install.sh'
|
|
||||||
- 'setup.py'
|
|
||||||
- 'MANIFEST.in'
|
|
||||||
- 'flake.nix'
|
|
||||||
- 'nix/firecracker-netpool.nix'
|
|
||||||
- 'scripts/coverage.sh'
|
- 'scripts/coverage.sh'
|
||||||
- 'scripts/critical-modules.txt'
|
- 'scripts/critical-modules.txt'
|
||||||
- 'scripts/**/*.py'
|
- 'scripts/diff_coverage.py'
|
||||||
|
- 'scripts/tracker_policy.py'
|
||||||
- 'scripts/firecracker-netpool.sh'
|
- 'scripts/firecracker-netpool.sh'
|
||||||
- 'Dockerfile*'
|
- 'Dockerfile*'
|
||||||
- 'pyproject.toml'
|
- 'pyproject.toml'
|
||||||
@@ -27,20 +23,15 @@ on:
|
|||||||
- '.coveragerc'
|
- '.coveragerc'
|
||||||
- '.dockerignore'
|
- '.dockerignore'
|
||||||
- '.gitea/workflows/test.yml'
|
- '.gitea/workflows/test.yml'
|
||||||
- '.gitea/workflows/pre-release-test.yml'
|
|
||||||
pull_request:
|
pull_request:
|
||||||
paths:
|
paths:
|
||||||
- 'bot_bottle/**'
|
- 'bot_bottle/**'
|
||||||
- 'tests/**/*.py'
|
- 'tests/**/*.py'
|
||||||
- 'cli.py'
|
- 'cli.py'
|
||||||
- 'install.sh'
|
|
||||||
- 'setup.py'
|
|
||||||
- 'MANIFEST.in'
|
|
||||||
- 'flake.nix'
|
|
||||||
- 'nix/firecracker-netpool.nix'
|
|
||||||
- 'scripts/coverage.sh'
|
- 'scripts/coverage.sh'
|
||||||
- 'scripts/critical-modules.txt'
|
- 'scripts/critical-modules.txt'
|
||||||
- 'scripts/**/*.py'
|
- 'scripts/diff_coverage.py'
|
||||||
|
- 'scripts/tracker_policy.py'
|
||||||
- 'scripts/firecracker-netpool.sh'
|
- 'scripts/firecracker-netpool.sh'
|
||||||
- 'Dockerfile*'
|
- 'Dockerfile*'
|
||||||
- 'pyproject.toml'
|
- 'pyproject.toml'
|
||||||
@@ -48,7 +39,6 @@ on:
|
|||||||
- '.coveragerc'
|
- '.coveragerc'
|
||||||
- '.dockerignore'
|
- '.dockerignore'
|
||||||
- '.gitea/workflows/test.yml'
|
- '.gitea/workflows/test.yml'
|
||||||
- '.gitea/workflows/pre-release-test.yml'
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
unit:
|
unit:
|
||||||
@@ -81,9 +71,6 @@ jobs:
|
|||||||
|
|
||||||
integration-docker:
|
integration-docker:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
concurrency:
|
|
||||||
group: integration-docker-infra
|
|
||||||
cancel-in-progress: false
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
@@ -100,37 +87,7 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
BOT_BOTTLE_BACKEND: docker
|
BOT_BOTTLE_BACKEND: docker
|
||||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.docker
|
COVERAGE_FILE: ${{ github.workspace }}/.coverage.docker
|
||||||
run: |
|
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||||
set -euo pipefail
|
|
||||||
# act_runner executes this job in a container while sharing the host
|
|
||||||
# Docker socket. Attach control-plane siblings to the job's network,
|
|
||||||
# and use named volumes for state the host daemon must mount.
|
|
||||||
DOCKER_CLIENT_NETWORK=$(
|
|
||||||
docker inspect "$(hostname)" |
|
|
||||||
python3 -c 'import json,sys; n=json.load(sys.stdin)[0]["NetworkSettings"]["Networks"]; print(next(iter(n)))'
|
|
||||||
)
|
|
||||||
test -n "$DOCKER_CLIENT_NETWORK"
|
|
||||||
RUN_KEY="${GITHUB_RUN_ID:-${GITHUB_RUN_NUMBER:-0}}"
|
|
||||||
export NO_PROXY="*"
|
|
||||||
export no_proxy="*"
|
|
||||||
export BOT_BOTTLE_DOCKER_CLIENT_NETWORK="$DOCKER_CLIENT_NETWORK"
|
|
||||||
export BOT_BOTTLE_DOCKER_ROOT_MOUNT="bot-bottle-ci-root-$RUN_KEY"
|
|
||||||
export BOT_BOTTLE_DOCKER_CA_MOUNT="bot-bottle-ci-ca-$RUN_KEY"
|
|
||||||
export BOT_BOTTLE_DOCKER_GIT_MOUNT="bot-bottle-ci-git-$RUN_KEY"
|
|
||||||
export BOT_BOTTLE_DOCKER_CREDS_MOUNT="bot-bottle-ci-creds-$RUN_KEY"
|
|
||||||
python3 -m coverage run -m scripts.unittest_gate \
|
|
||||||
-t . -s tests/integration -v \
|
|
||||||
--minimum-executed 22 --fail-on-skip
|
|
||||||
|
|
||||||
- name: Clean Docker integration volumes
|
|
||||||
if: always()
|
|
||||||
run: |
|
|
||||||
RUN_KEY="${GITHUB_RUN_ID:-${GITHUB_RUN_NUMBER:-0}}"
|
|
||||||
docker volume rm --force \
|
|
||||||
"bot-bottle-ci-root-$RUN_KEY" \
|
|
||||||
"bot-bottle-ci-ca-$RUN_KEY" \
|
|
||||||
"bot-bottle-ci-git-$RUN_KEY" \
|
|
||||||
"bot-bottle-ci-creds-$RUN_KEY" 2>/dev/null || true
|
|
||||||
|
|
||||||
- name: Stage docker coverage for upload
|
- name: Stage docker coverage for upload
|
||||||
run: cp .coverage.docker coverage-docker.dat
|
run: cp .coverage.docker coverage-docker.dat
|
||||||
|
|||||||
@@ -33,28 +33,19 @@ jobs:
|
|||||||
- name: Run coverage and extract percentage
|
- name: Run coverage and extract percentage
|
||||||
id: coverage
|
id: coverage
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
python3 -m coverage run -m unittest discover -t . -s tests/unit > /dev/null 2>&1 || true
|
||||||
# Never publish a badge from a failed or partial test run.
|
PERCENT=$(python3 -m coverage report 2>/dev/null | grep '^TOTAL' | grep -oP '\d+(?=%)' | tail -1)
|
||||||
python3 -m coverage run -m unittest discover -t . -s tests/unit
|
|
||||||
REPORT=$(python3 -m coverage report)
|
|
||||||
printf '%s\n' "$REPORT"
|
|
||||||
PERCENT=$(printf '%s\n' "$REPORT" | awk '$1 == "TOTAL" {gsub("%", "", $NF); print $NF}')
|
|
||||||
test -n "$PERCENT"
|
|
||||||
echo "percent=$PERCENT" >> $GITHUB_OUTPUT
|
echo "percent=$PERCENT" >> $GITHUB_OUTPUT
|
||||||
echo "Coverage: $PERCENT%"
|
echo "Coverage: $PERCENT%"
|
||||||
|
|
||||||
- name: Extract core (critical-module) coverage percentage
|
- name: Extract core (critical-module) coverage percentage
|
||||||
id: core_coverage
|
id: core_coverage
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
|
||||||
# Reuses the .coverage data from the previous step. The core list is
|
# Reuses the .coverage data from the previous step. The core list is
|
||||||
# validated single source of truth. Fail if a listed path disappeared
|
# the single source of truth in scripts/critical-modules.txt; every
|
||||||
# or if the measured core falls below ADR 0004's 90% minimum.
|
# core module is unit-tested, so the unit-only run is accurate for it.
|
||||||
INCLUDE=$(python3 scripts/critical_modules.py)
|
INCLUDE=$(grep -vE '^[[:space:]]*(#|$)' scripts/critical-modules.txt | paste -sd, -)
|
||||||
REPORT=$(python3 -m coverage report --include="$INCLUDE" --fail-under=90)
|
PERCENT=$(python3 -m coverage report --include="$INCLUDE" 2>/dev/null | grep '^TOTAL' | grep -oP '\d+(?=%)' | tail -1)
|
||||||
printf '%s\n' "$REPORT"
|
|
||||||
PERCENT=$(printf '%s\n' "$REPORT" | awk '$1 == "TOTAL" {gsub("%", "", $NF); print $NF}')
|
|
||||||
test -n "$PERCENT"
|
|
||||||
echo "percent=$PERCENT" >> $GITHUB_OUTPUT
|
echo "percent=$PERCENT" >> $GITHUB_OUTPUT
|
||||||
echo "Core coverage: $PERCENT%"
|
echo "Core coverage: $PERCENT%"
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
# bot-bottle
|
# bot-bottle
|
||||||
|
|
||||||
[](https://gitea.dideric.is/didericis/bot-bottle/actions?workflow=test.yml)
|
[](https://gitea.dideric.is/didericis/bot-bottle/actions?workflow=test.yml)
|
||||||
[](https://coverage.readthedocs.io/)
|
[](https://coverage.readthedocs.io/)
|
||||||
[](https://gitea.dideric.is/didericis/bot-bottle/src/branch/main/docs/decisions/0004-coverage-policy.md)
|
[](https://gitea.dideric.is/didericis/bot-bottle/src/branch/main/docs/decisions/0004-coverage-policy.md)
|
||||||
|
|
||||||
**Problem:** Developer wants to run a coding agent without supervision, but they don't want a prompt injected or misbehaving agent wrecking their environment or exfiltrating sensitive data.
|
**Problem:** Developer wants to run a coding agent without supervision, but they don't want a prompt injected or misbehaving agent wrecking their environment or exfiltrating sensitive data.
|
||||||
@@ -75,7 +75,7 @@ On compatible macOS hosts, the default backend requires Apple's `container` CLI
|
|||||||
|
|
||||||
Use `BOT_BOTTLE_BACKEND=docker ./cli.py start <agent>` on hosts where neither Apple Container nor KVM is available and Docker is the desired backend.
|
Use `BOT_BOTTLE_BACKEND=docker ./cli.py start <agent>` on hosts where neither Apple Container nor KVM is available and Docker is the desired backend.
|
||||||
|
|
||||||
> **CI (macOS Apple Container):** the advisory `integration-macos` job in `.gitea/workflows/pre-release-test.yml` runs only on manual dispatch. It targets a self-hosted host-mode runner labelled `macos`; Apple Container cannot run inside the Linux pull-request runner. Provision an Apple Silicon host with the `container` CLI running and Python ≥ 3.11 plus `coverage` on the launchd service's explicit `PATH`. The infra container is a singleton (`bot-bottle-mac-infra`), so keep runner concurrency at 1. Its coverage is reported separately and never feeds the required pull-request gate.
|
> **CI (macOS Apple Container):** the `integration-macos` job (`.gitea/workflows/test.yml`) runs the integration suite against `BOT_BOTTLE_BACKEND=macos-container` on a self-hosted macOS runner labelled `macos`, because Apple Container needs the host virtualization framework and cannot run in a Linux container (so it can't reuse the `kvm` runner). Provision an Apple Silicon host with the `container` CLI on `PATH` and `container system status` running, then register the runner in **host mode** (not docker mode) with the `macos` label — `brew install gitea-runner` (the `act_runner` rename). Give it a Python ≥ 3.11 with `coverage` importable on the launchd service's `PATH` (a launchd service doesn't inherit your shell profile, so pin `node` and the Python env explicitly). The job is **advisory** — `workflow_dispatch` (manual) only, never triggered by push or PR — since a single laptop that sleeps/roams must not block merges or churn on every push to main; its coverage doesn't feed the gate. The infra container is a singleton (`bot-bottle-mac-infra`), so keep runner concurrency at 1.
|
||||||
|
|
||||||
### Containers inside a bottle
|
### Containers inside a bottle
|
||||||
|
|
||||||
@@ -174,7 +174,7 @@ BOT_BOTTLE_BACKEND=firecracker ./cli.py start <agent>
|
|||||||
|
|
||||||
> **NixOS:** enable `virtualisation.docker`, ensure the KVM module is loaded (`boot.kernelModules = [ "kvm-intel" ];` or `kvm-amd`), and add your user to the `kvm` and `docker` groups. For the network pool, consume the flake module — `imports = [ inputs.bot-bottle.nixosModules.firecracker-netpool ]; services.bot-bottle-firecracker = { enable = true; owner = "you"; };` — then `nixos-rebuild switch` (imperative nft/TAP rules don't survive a rebuild; channel users can `imports = [ <bot-bottle>/nix/firecracker-netpool.nix ]`). `firecracker` isn't in nixpkgs by default as a user binary — install the release binary (pin the version) and put it on `PATH`.
|
> **NixOS:** enable `virtualisation.docker`, ensure the KVM module is loaded (`boot.kernelModules = [ "kvm-intel" ];` or `kvm-amd`), and add your user to the `kvm` and `docker` groups. For the network pool, consume the flake module — `imports = [ inputs.bot-bottle.nixosModules.firecracker-netpool ]; services.bot-bottle-firecracker = { enable = true; owner = "you"; };` — then `nixos-rebuild switch` (imperative nft/TAP rules don't survive a rebuild; channel users can `imports = [ <bot-bottle>/nix/firecracker-netpool.nix ]`). `firecracker` isn't in nixpkgs by default as a user binary — install the release binary (pin the version) and put it on `PATH`.
|
||||||
|
|
||||||
> **CI:** Firecracker integration runs in the manually dispatched `.gitea/workflows/pre-release-test.yml` on a self-hosted runner labelled `kvm`; privileged KVM hosts never execute unreviewed PR code automatically. Provision it like a normal Firecracker host: `firecracker` on `PATH`, `/dev/kvm`, the cached guest kernel and static dropbear, and the persistent TAP/nft pool. The required pull-request workflow runs unit plus the complete Docker integration suite on `ubuntu-latest`; see `docs/ci.md`.
|
> **CI:** the coverage gate (`.gitea/workflows/test.yml` → `coverage` job) runs on a self-hosted runner labelled `kvm`, because the Firecracker backend's VM/SSH orchestration is exercised only by the integration suite, which needs `/dev/kvm` + the provisioned pool (a container runner would skip it and read as uncovered). Provision that runner exactly like a normal Firecracker host — `firecracker` on `PATH`, `/dev/kvm`, the cached guest kernel + static dropbear, and the pool installed as the persistent systemd unit — then register it with the `kvm` label. A Docker-capable hosted job builds the candidate once; KVM tests boot those exact bytes, and a successful main run publishes them. The unit/lint jobs still run on `ubuntu-latest`.
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
./cli.py start <agent> # builds the image on first run, drops you into claude
|
./cli.py start <agent> # builds the image on first run, drops you into claude
|
||||||
|
|||||||
+75
-11
@@ -23,14 +23,14 @@ from dataclasses import dataclass
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Generator, Generic, Sequence, TypeVar
|
from typing import Generator, Generic, Sequence, TypeVar
|
||||||
|
|
||||||
from ..agent_provider import AgentProvisionPlan, get_provider
|
from ..agent_provider import AgentProvisionPlan, get_provider, build_agent_provision_plan
|
||||||
from ..egress import EgressPlan
|
from ..egress import EgressPlan
|
||||||
from ..git_gate import GitGatePlan
|
from ..git_gate import GitGatePlan
|
||||||
from ..log import die, info
|
from ..log import die, info
|
||||||
from ..util import expand_tilde
|
from ..util import expand_tilde
|
||||||
from ..manifest import Manifest, ManifestIndex
|
from ..manifest import Manifest, ManifestIndex
|
||||||
from ..supervisor.plan import SupervisePlan
|
from ..supervisor.plan import SupervisePlan
|
||||||
from ..env import ResolvedEnv
|
from ..env import resolve_env, ResolvedEnv
|
||||||
from ..workspace import WorkspacePlan, workspace_plan
|
from ..workspace import WorkspacePlan, workspace_plan
|
||||||
from .print_util import print_multi, visible_agent_env_names
|
from .print_util import print_multi, visible_agent_env_names
|
||||||
from .util import host_skill_dir
|
from .util import host_skill_dir
|
||||||
@@ -296,18 +296,82 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]):
|
|||||||
backend-specific resolution (names, scratch files, etc.). The
|
backend-specific resolution (names, scratch files, etc.). The
|
||||||
validation step is enforced here so a future backend cannot
|
validation step is enforced here so a future backend cannot
|
||||||
accidentally skip it. No remote/runtime resources are created."""
|
accidentally skip it. No remote/runtime resources are created."""
|
||||||
from .preparation import BottlePreparationPlanner
|
from .resolve_common import (
|
||||||
prepared = BottlePreparationPlanner(self).prepare(spec)
|
merge_provision_env_vars,
|
||||||
|
mint_slug,
|
||||||
|
prepare_agent_state_dir,
|
||||||
|
prepare_egress,
|
||||||
|
prepare_git_gate,
|
||||||
|
prepare_supervise,
|
||||||
|
reject_nested_containers,
|
||||||
|
resolve_manifest_dockerfile,
|
||||||
|
write_launch_metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
manifest = self._validate(spec)
|
||||||
|
|
||||||
|
if not self.supports_nested_containers:
|
||||||
|
reject_nested_containers(self.name, manifest)
|
||||||
|
|
||||||
|
self._preflight()
|
||||||
|
|
||||||
|
from ..git_gate import GitGate
|
||||||
|
manifest = GitGate().preflight_host_keys(
|
||||||
|
manifest,
|
||||||
|
headless=spec.headless,
|
||||||
|
home_md=spec.manifest.home_md,
|
||||||
|
)
|
||||||
|
|
||||||
|
manifest_bottle = manifest.bottle
|
||||||
|
manifest_agent_provider = manifest_bottle.agent_provider
|
||||||
|
agent_provider = get_provider(manifest_agent_provider.template)
|
||||||
|
resolved_env = resolve_env(manifest)
|
||||||
|
workspace = workspace_plan(spec, guest_home=agent_provider.guest_home)
|
||||||
|
|
||||||
|
slug = mint_slug(spec)
|
||||||
|
write_launch_metadata(slug, spec, compose_project="", backend=self.name)
|
||||||
|
|
||||||
|
# Manifest may override the Dockerfile per-bottle; otherwise fall
|
||||||
|
# back to the provider plugin's bundled Dockerfile (next to its
|
||||||
|
# agent_provider.py module).
|
||||||
|
if manifest_agent_provider.dockerfile:
|
||||||
|
agent_dockerfile_path = resolve_manifest_dockerfile(
|
||||||
|
manifest_agent_provider.dockerfile, spec,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
agent_dockerfile_path = str(agent_provider.dockerfile)
|
||||||
|
|
||||||
|
agent_dir, prompt_file = prepare_agent_state_dir(slug, manifest)
|
||||||
|
|
||||||
|
agent_provision_plan = build_agent_provision_plan(
|
||||||
|
template=manifest_agent_provider.template,
|
||||||
|
dockerfile=agent_dockerfile_path,
|
||||||
|
state_dir=agent_dir,
|
||||||
|
instance_name=f"bot-bottle-{slug}",
|
||||||
|
prompt_file=prompt_file,
|
||||||
|
guest_env=self._build_guest_env(resolved_env),
|
||||||
|
forward_host_credentials=manifest_agent_provider.forward_host_credentials,
|
||||||
|
auth_token=manifest_agent_provider.auth_token,
|
||||||
|
host_env=dict(os.environ),
|
||||||
|
trusted_project_path=workspace.workdir,
|
||||||
|
label=spec.label,
|
||||||
|
color=spec.color,
|
||||||
|
provider_settings=manifest_agent_provider.settings,
|
||||||
|
)
|
||||||
|
agent_provision_plan = merge_provision_env_vars(agent_provision_plan)
|
||||||
|
egress_plan = prepare_egress(manifest_bottle, slug, agent_provision_plan)
|
||||||
|
supervise_plan = prepare_supervise(manifest_bottle, slug)
|
||||||
|
git_gate_plan = prepare_git_gate(manifest_bottle, slug)
|
||||||
|
|
||||||
return self._resolve_plan(
|
return self._resolve_plan(
|
||||||
spec,
|
spec,
|
||||||
manifest=prepared.manifest,
|
manifest=manifest,
|
||||||
slug=prepared.slug,
|
slug=slug,
|
||||||
resolved_env=prepared.resolved_env,
|
resolved_env=resolved_env,
|
||||||
agent_provision_plan=prepared.agent_provision_plan,
|
agent_provision_plan=agent_provision_plan,
|
||||||
egress_plan=prepared.egress_plan,
|
egress_plan=egress_plan,
|
||||||
supervise_plan=prepared.supervise_plan,
|
supervise_plan=supervise_plan,
|
||||||
git_gate_plan=prepared.git_gate_plan,
|
git_gate_plan=git_gate_plan,
|
||||||
stage_dir=stage_dir,
|
stage_dir=stage_dir,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -9,8 +9,6 @@ from .gateway_transport import DockerGatewayTransport
|
|||||||
from ...paths import (
|
from ...paths import (
|
||||||
ORCHESTRATOR_AUTH_JWT_ENV,
|
ORCHESTRATOR_AUTH_JWT_ENV,
|
||||||
host_gateway_ca_dir,
|
host_gateway_ca_dir,
|
||||||
host_gateway_git_dir,
|
|
||||||
host_gateway_creds_dir,
|
|
||||||
)
|
)
|
||||||
from ... import resources
|
from ... import resources
|
||||||
from ...gateway import (
|
from ...gateway import (
|
||||||
@@ -19,10 +17,6 @@ from ...gateway import (
|
|||||||
DEFAULT_CA_TIMEOUT_SECONDS, CA_POLL_SECONDS, GATEWAY_CA_CERT, GatewayError
|
DEFAULT_CA_TIMEOUT_SECONDS, CA_POLL_SECONDS, GATEWAY_CA_CERT, GatewayError
|
||||||
)
|
)
|
||||||
|
|
||||||
DEFAULT_GATEWAY_SUBNET = "10.242.255.0/24"
|
|
||||||
_GATEWAY_SUBNET_LABEL = "bot-bottle.gateway-subnet"
|
|
||||||
|
|
||||||
|
|
||||||
class DockerGateway(Gateway):
|
class DockerGateway(Gateway):
|
||||||
"""The consolidated gateway as a single, fixed-name Docker container.
|
"""The consolidated gateway as a single, fixed-name Docker container.
|
||||||
|
|
||||||
@@ -41,10 +35,6 @@ class DockerGateway(Gateway):
|
|||||||
build_context: Path | None = None,
|
build_context: Path | None = None,
|
||||||
dockerfile: str | None = GATEWAY_DOCKERFILE,
|
dockerfile: str | None = GATEWAY_DOCKERFILE,
|
||||||
host_port_bindings: tuple[int, ...] = (),
|
host_port_bindings: tuple[int, ...] = (),
|
||||||
ca_mount_source: str | Path | None = None,
|
|
||||||
git_mount_source: str | Path | None = None,
|
|
||||||
creds_mount_source: str | Path | None = None,
|
|
||||||
subnet: str | None = None,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
self.image_ref = image_ref
|
self.image_ref = image_ref
|
||||||
self.name = name
|
self.name = name
|
||||||
@@ -69,26 +59,6 @@ class DockerGateway(Gateway):
|
|||||||
# backend's dev-harness gateway so VMs can reach it via their TAP link;
|
# backend's dev-harness gateway so VMs can reach it via their TAP link;
|
||||||
# Docker's DNAT + the nft `ct status dnat accept` rule handle the rest.
|
# Docker's DNAT + the nft `ct status dnat accept` rule handle the rest.
|
||||||
self._host_port_bindings = host_port_bindings
|
self._host_port_bindings = host_port_bindings
|
||||||
self._subnet = (
|
|
||||||
subnet
|
|
||||||
or os.environ.get("BOT_BOTTLE_DOCKER_GATEWAY_SUBNET", "").strip()
|
|
||||||
or DEFAULT_GATEWAY_SUBNET
|
|
||||||
)
|
|
||||||
configured_ca = os.environ.get("BOT_BOTTLE_DOCKER_CA_MOUNT", "").strip()
|
|
||||||
self._ca_mount_source = str(
|
|
||||||
ca_mount_source or configured_ca or host_gateway_ca_dir()
|
|
||||||
)
|
|
||||||
# The persistent git-gate mounts (/git bare repos, /git-gate/creds deploy
|
|
||||||
# creds) — same host-bind-mount rationale as the CA (issue #512). The env
|
|
||||||
# overrides let CI point them at per-run named volumes it cleans up.
|
|
||||||
configured_git = os.environ.get("BOT_BOTTLE_DOCKER_GIT_MOUNT", "").strip()
|
|
||||||
self._git_mount_source = str(
|
|
||||||
git_mount_source or configured_git or host_gateway_git_dir()
|
|
||||||
)
|
|
||||||
configured_creds = os.environ.get("BOT_BOTTLE_DOCKER_CREDS_MOUNT", "").strip()
|
|
||||||
self._creds_mount_source = str(
|
|
||||||
creds_mount_source or configured_creds or host_gateway_creds_dir()
|
|
||||||
)
|
|
||||||
|
|
||||||
def image_exists(self) -> bool:
|
def image_exists(self) -> bool:
|
||||||
return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0
|
return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0
|
||||||
@@ -139,34 +109,10 @@ class DockerGateway(Gateway):
|
|||||||
def _ensure_network(self) -> None:
|
def _ensure_network(self) -> None:
|
||||||
"""Create the shared gateway network if it doesn't exist. Idempotent —
|
"""Create the shared gateway network if it doesn't exist. Idempotent —
|
||||||
a concurrent create loses harmlessly (the loser sees 'already exists').
|
a concurrent create loses harmlessly (the loser sees 'already exists').
|
||||||
The explicit subnet is required because bottle attribution pins source
|
Docker picks the subnet; the launcher reads it back to allocate IPs."""
|
||||||
IPs; Docker rejects static endpoint addresses on an auto-IPAM network."""
|
if run_docker(["docker", "network", "inspect", self.network]).returncode == 0:
|
||||||
inspected = run_docker([
|
return
|
||||||
"docker", "network", "inspect",
|
proc = run_docker(["docker", "network", "create", self.network])
|
||||||
"--format", f'{{{{index .Labels "{_GATEWAY_SUBNET_LABEL}"}}}}',
|
|
||||||
self.network,
|
|
||||||
])
|
|
||||||
if inspected.returncode == 0:
|
|
||||||
marker = inspected.stdout.strip()
|
|
||||||
if marker in {"", self._subnet}:
|
|
||||||
return
|
|
||||||
if inspected.returncode == 0:
|
|
||||||
# Migrate the stale auto-IPAM network created by older releases.
|
|
||||||
# Removing the fixed gateway is safe here: this launch recreates it.
|
|
||||||
run_docker(["docker", "rm", "--force", self.name])
|
|
||||||
removed = run_docker(["docker", "network", "rm", self.network])
|
|
||||||
if removed.returncode != 0:
|
|
||||||
raise GatewayError(
|
|
||||||
f"gateway network {self.network} needs explicit subnet "
|
|
||||||
f"{self._subnet} but could not be replaced: "
|
|
||||||
f"{removed.stderr.strip()}"
|
|
||||||
)
|
|
||||||
proc = run_docker([
|
|
||||||
"docker", "network", "create",
|
|
||||||
"--subnet", self._subnet,
|
|
||||||
"--label", f"{_GATEWAY_SUBNET_LABEL}={self._subnet}",
|
|
||||||
self.network,
|
|
||||||
])
|
|
||||||
if proc.returncode != 0 and "already exists" not in proc.stderr:
|
if proc.returncode != 0 and "already exists" not in proc.stderr:
|
||||||
raise GatewayError(
|
raise GatewayError(
|
||||||
f"gateway network {self.network} failed to create: {proc.stderr.strip()}"
|
f"gateway network {self.network} failed to create: {proc.stderr.strip()}"
|
||||||
@@ -197,9 +143,9 @@ class DockerGateway(Gateway):
|
|||||||
# Recreate when the running container's image is stale (a rebuild),
|
# Recreate when the running container's image is stale (a rebuild),
|
||||||
# so source changes to the gateway's flat daemons take effect — not
|
# so source changes to the gateway's flat daemons take effect — not
|
||||||
# just when the container is absent.
|
# just when the container is absent.
|
||||||
self._ensure_network()
|
|
||||||
if self.is_running() and self._running_image_is_current():
|
if self.is_running() and self._running_image_is_current():
|
||||||
return
|
return
|
||||||
|
self._ensure_network()
|
||||||
# Clear any stale (stopped OR outdated-image) container holding the
|
# Clear any stale (stopped OR outdated-image) container holding the
|
||||||
# fixed name, then start fresh. `rm --force` on an absent name is a
|
# fixed name, then start fresh. `rm --force` on an absent name is a
|
||||||
# tolerated no-op.
|
# tolerated no-op.
|
||||||
@@ -212,13 +158,7 @@ class DockerGateway(Gateway):
|
|||||||
# Persist the self-generated CA on the host so it survives both
|
# Persist the self-generated CA on the host so it survives both
|
||||||
# container recreation AND docker volume pruning (agents trust it)
|
# container recreation AND docker volume pruning (agents trust it)
|
||||||
# — see host_gateway_ca_dir / issue #450.
|
# — see host_gateway_ca_dir / issue #450.
|
||||||
"--volume", f"{self._ca_mount_source}:{MITMPROXY_HOME}",
|
"--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}",
|
||||||
# Persist per-bottle git-gate state (bare repos + deploy creds) on
|
|
||||||
# the host so a gateway restart doesn't drop already-running bottles'
|
|
||||||
# repos — they would otherwise 404 on fetch/push (issue #512). Same
|
|
||||||
# host-bind-mount rationale as the CA.
|
|
||||||
"--volume", f"{self._git_mount_source}:/git",
|
|
||||||
"--volume", f"{self._creds_mount_source}:/git-gate/creds",
|
|
||||||
# No DB mount: the data plane (egress / supervise / git-gate) reaches
|
# No DB mount: the data plane (egress / supervise / git-gate) reaches
|
||||||
# the supervise queue over the control-plane RPC and never opens
|
# the supervise queue over the control-plane RPC and never opens
|
||||||
# bot-bottle.db, so the gateway container gets no file handle on it
|
# bot-bottle.db, so the gateway container gets no file handle on it
|
||||||
@@ -313,4 +253,4 @@ class DockerGateway(Gateway):
|
|||||||
def provisioning_transport(self) -> GatewayTransport:
|
def provisioning_transport(self) -> GatewayTransport:
|
||||||
"""The exec/cp transport git-gate provisioning stages per-bottle repos +
|
"""The exec/cp transport git-gate provisioning stages per-bottle repos +
|
||||||
deploy keys through (over the docker socket)."""
|
deploy keys through (over the docker socket)."""
|
||||||
return DockerGatewayTransport(self.name)
|
return DockerGatewayTransport(self.name)
|
||||||
@@ -33,6 +33,7 @@ from .orchestrator import (
|
|||||||
ORCHESTRATOR_NAME,
|
ORCHESTRATOR_NAME,
|
||||||
ORCHESTRATOR_NETWORK,
|
ORCHESTRATOR_NETWORK,
|
||||||
)
|
)
|
||||||
|
from ...paths import bot_bottle_root
|
||||||
from ... import resources
|
from ... import resources
|
||||||
from ...gateway import (
|
from ...gateway import (
|
||||||
GATEWAY_IMAGE,
|
GATEWAY_IMAGE,
|
||||||
@@ -68,8 +69,6 @@ class DockerInfraService(InfraService):
|
|||||||
gateway_image: str = GATEWAY_IMAGE,
|
gateway_image: str = GATEWAY_IMAGE,
|
||||||
repo_root: Path | None = None,
|
repo_root: Path | None = None,
|
||||||
host_root: Path | None = None,
|
host_root: Path | None = None,
|
||||||
root_mount_source: str | Path | None = None,
|
|
||||||
gateway_ca_mount_source: str | Path | None = None,
|
|
||||||
orchestrator_name: str = ORCHESTRATOR_NAME,
|
orchestrator_name: str = ORCHESTRATOR_NAME,
|
||||||
orchestrator_label: str = ORCHESTRATOR_LABEL,
|
orchestrator_label: str = ORCHESTRATOR_LABEL,
|
||||||
gateway_name: str = GATEWAY_NAME,
|
gateway_name: str = GATEWAY_NAME,
|
||||||
@@ -79,14 +78,10 @@ class DockerInfraService(InfraService):
|
|||||||
self.control_network = control_network
|
self.control_network = control_network
|
||||||
self.orchestrator_image = orchestrator_image
|
self.orchestrator_image = orchestrator_image
|
||||||
self.gateway_image = gateway_image
|
self.gateway_image = gateway_image
|
||||||
# Build context: the repo root in a checkout, a staged copy from the
|
# Build context / bind-mount source: the repo root in a checkout, a
|
||||||
# installed wheel otherwise (bot_bottle.resources).
|
# staged copy from the installed wheel otherwise (bot_bottle.resources).
|
||||||
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||||
if host_root is not None and root_mount_source is not None:
|
self._host_root = host_root or bot_bottle_root()
|
||||||
raise ValueError("pass host_root or root_mount_source, not both")
|
|
||||||
self._host_root = host_root
|
|
||||||
self._root_mount_source = root_mount_source
|
|
||||||
self._gateway_ca_mount_source = gateway_ca_mount_source
|
|
||||||
self._orchestrator_name = orchestrator_name
|
self._orchestrator_name = orchestrator_name
|
||||||
self._orchestrator_label = orchestrator_label
|
self._orchestrator_label = orchestrator_label
|
||||||
self._gateway_name = gateway_name
|
self._gateway_name = gateway_name
|
||||||
@@ -103,7 +98,6 @@ class DockerInfraService(InfraService):
|
|||||||
control_network=self.control_network,
|
control_network=self.control_network,
|
||||||
repo_root=self._repo_root,
|
repo_root=self._repo_root,
|
||||||
host_root=self._host_root,
|
host_root=self._host_root,
|
||||||
root_mount_source=self._root_mount_source,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def gateway(self) -> DockerGateway:
|
def gateway(self) -> DockerGateway:
|
||||||
@@ -118,7 +112,6 @@ class DockerInfraService(InfraService):
|
|||||||
network=self.network,
|
network=self.network,
|
||||||
control_network=self.control_network,
|
control_network=self.control_network,
|
||||||
build_context=self._repo_root,
|
build_context=self._repo_root,
|
||||||
ca_mount_source=self._gateway_ca_mount_source,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def ensure_running(
|
def ensure_running(
|
||||||
|
|||||||
@@ -42,9 +42,13 @@ ORCHESTRATOR_IMAGE = os.environ.get(
|
|||||||
)
|
)
|
||||||
ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator"
|
ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator"
|
||||||
# Baked as a container label so `ensure_running` can detect whether the running
|
# Baked as a container label so `ensure_running` can detect whether the running
|
||||||
# orchestrator image was built from the current source.
|
# orchestrator is executing the current bind-mounted source.
|
||||||
ORCHESTRATOR_SOURCE_HASH_LABEL = "bot-bottle-orchestrator-source-hash"
|
ORCHESTRATOR_SOURCE_HASH_LABEL = "bot-bottle-orchestrator-source-hash"
|
||||||
|
|
||||||
|
# The bind-mount path for the live control-plane source inside the container.
|
||||||
|
# PYTHONPATH points here so a code change takes effect on the next launch
|
||||||
|
# without an image rebuild.
|
||||||
|
_SRC_IN_CONTAINER = "/bot-bottle-src"
|
||||||
# Bot-bottle host-root bind-mount (DB + state) inside the orchestrator. The
|
# Bot-bottle host-root bind-mount (DB + state) inside the orchestrator. The
|
||||||
# control plane opens bot-bottle.db under here (via BOT_BOTTLE_ROOT ->
|
# control plane opens bot-bottle.db under here (via BOT_BOTTLE_ROOT ->
|
||||||
# host_db_path()); it is the ONLY container with a handle on it (issue #469).
|
# host_db_path()); it is the ONLY container with a handle on it (issue #469).
|
||||||
@@ -68,51 +72,23 @@ class DockerOrchestrator(Orchestrator):
|
|||||||
control_network: str = ORCHESTRATOR_NETWORK,
|
control_network: str = ORCHESTRATOR_NETWORK,
|
||||||
repo_root: Path | None = None,
|
repo_root: Path | None = None,
|
||||||
host_root: Path | None = None,
|
host_root: Path | None = None,
|
||||||
root_mount_source: str | Path | None = None,
|
|
||||||
client_host: str | None = None,
|
|
||||||
client_network: str | None = None,
|
|
||||||
bind_host: str | None = None,
|
|
||||||
dockerfile: str | None = ORCHESTRATOR_DOCKERFILE,
|
dockerfile: str | None = ORCHESTRATOR_DOCKERFILE,
|
||||||
) -> None:
|
) -> None:
|
||||||
if host_root is not None and root_mount_source is not None:
|
|
||||||
raise ValueError("pass host_root or root_mount_source, not both")
|
|
||||||
self.image_ref = image_ref
|
self.image_ref = image_ref
|
||||||
self.name = name
|
self.name = name
|
||||||
self.label = label
|
self.label = label
|
||||||
self.port = port
|
self.port = port
|
||||||
self.control_network = control_network
|
self.control_network = control_network
|
||||||
# Build context: the repo root in a checkout, a staged copy from the
|
# Build context / bind-mount source: the repo root in a checkout, a
|
||||||
# installed wheel otherwise (bot_bottle.resources).
|
# staged copy from the installed wheel otherwise (bot_bottle.resources).
|
||||||
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
self._repo_root = repo_root if repo_root is not None else resources.build_root()
|
||||||
configured_root = os.environ.get("BOT_BOTTLE_DOCKER_ROOT_MOUNT", "").strip()
|
self._host_root = host_root or bot_bottle_root()
|
||||||
self._root_mount_source = str(
|
|
||||||
root_mount_source or configured_root or host_root or bot_bottle_root()
|
|
||||||
)
|
|
||||||
configured_network = os.environ.get(
|
|
||||||
"BOT_BOTTLE_DOCKER_CLIENT_NETWORK", ""
|
|
||||||
).strip()
|
|
||||||
self._client_network = client_network or configured_network or None
|
|
||||||
configured_host = os.environ.get(
|
|
||||||
"BOT_BOTTLE_DOCKER_HOST_ADDRESS", ""
|
|
||||||
).strip()
|
|
||||||
self._client_host = (
|
|
||||||
client_host or configured_host
|
|
||||||
or (self.name if self._client_network else "127.0.0.1")
|
|
||||||
)
|
|
||||||
# A socket-shared CI runner reaches published ports through its Docker
|
|
||||||
# network rather than its own loopback. Production stays bound to host
|
|
||||||
# loopback unless a caller explicitly selects another client.
|
|
||||||
self._bind_host = bind_host or (
|
|
||||||
"0.0.0.0"
|
|
||||||
if not self._client_network and self._client_host != "127.0.0.1"
|
|
||||||
else "127.0.0.1"
|
|
||||||
)
|
|
||||||
self._dockerfile = dockerfile
|
self._dockerfile = dockerfile
|
||||||
|
|
||||||
def url(self) -> str:
|
def url(self) -> str:
|
||||||
"""Control-plane URL reachable by this Docker client."""
|
"""Host-side control-plane URL — the orchestrator's published loopback,
|
||||||
port = DEFAULT_PORT if self._client_network else self.port
|
which the CLI reaches."""
|
||||||
return f"http://{self._client_host}:{port}"
|
return f"http://127.0.0.1:{self.port}"
|
||||||
|
|
||||||
def gateway_url(self) -> str:
|
def gateway_url(self) -> str:
|
||||||
"""The URL the gateway's data plane resolves policy against — the
|
"""The URL the gateway's data plane resolves policy against — the
|
||||||
@@ -143,7 +119,8 @@ class DockerOrchestrator(Orchestrator):
|
|||||||
return self.name in proc.stdout.split()
|
return self.name in proc.stdout.split()
|
||||||
|
|
||||||
def _source_current(self, current_hash: str) -> bool:
|
def _source_current(self, current_hash: str) -> bool:
|
||||||
"""True iff the running orchestrator image matches current source."""
|
"""True iff the running orchestrator was started from the current
|
||||||
|
bind-mounted source."""
|
||||||
if not self.is_running():
|
if not self.is_running():
|
||||||
return False
|
return False
|
||||||
proc = run_docker([
|
proc = run_docker([
|
||||||
@@ -205,19 +182,15 @@ class DockerOrchestrator(Orchestrator):
|
|||||||
# Control network only — agents are never on it, so they have no
|
# Control network only — agents are never on it, so they have no
|
||||||
# route to the control plane (the L3 block, not just the JWT).
|
# route to the control plane (the L3 block, not just the JWT).
|
||||||
"--network", self.control_network,
|
"--network", self.control_network,
|
||||||
# Host CLI reaches the control plane here (loopback by default).
|
# Host CLI reaches the control plane here (loopback only). The
|
||||||
# Socket-shared CI joins the container directly to the job network;
|
|
||||||
# the host-side mapping remains loopback-only in that topology. The
|
|
||||||
# orchestrator listens on the fixed DEFAULT_PORT inside the
|
# orchestrator listens on the fixed DEFAULT_PORT inside the
|
||||||
# container; self.port is the host-side published port.
|
# container; self.port is the host-side published port.
|
||||||
"--publish", f"{self._bind_host}:{self.port}:{DEFAULT_PORT}",
|
"--publish", f"127.0.0.1:{self.port}:{DEFAULT_PORT}",
|
||||||
# The image was rebuilt from `_repo_root` immediately before this
|
# Live control-plane source (code changes without an image rebuild).
|
||||||
# launch. Running its baked package avoids a host-path bind mount,
|
"--volume", f"{self._repo_root}:{_SRC_IN_CONTAINER}:ro",
|
||||||
# which is both more production-like and works with socket-shared
|
"--env", f"PYTHONPATH={_SRC_IN_CONTAINER}",
|
||||||
# CI where the daemon cannot see the job container's workspace.
|
|
||||||
# Orchestrator registry DB on the host (sole writer: control plane).
|
# Orchestrator registry DB on the host (sole writer: control plane).
|
||||||
# `root_mount_source` may be a host path or a named Docker volume.
|
"--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}",
|
||||||
"--volume", f"{self._root_mount_source}:{_ROOT_IN_CONTAINER}",
|
|
||||||
"--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}",
|
"--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}",
|
||||||
# The signing key — held ONLY by the orchestrator (it verifies
|
# The signing key — held ONLY by the orchestrator (it verifies
|
||||||
# tokens); the gateway gets the pre-minted `gateway` JWT, never the
|
# tokens); the gateway gets the pre-minted `gateway` JWT, never the
|
||||||
@@ -232,15 +205,6 @@ class DockerOrchestrator(Orchestrator):
|
|||||||
raise OrchestratorStartError(
|
raise OrchestratorStartError(
|
||||||
f"orchestrator container failed to start: {proc.stderr.strip()}"
|
f"orchestrator container failed to start: {proc.stderr.strip()}"
|
||||||
)
|
)
|
||||||
if self._client_network:
|
|
||||||
proc = run_docker([
|
|
||||||
"docker", "network", "connect", self._client_network, self.name,
|
|
||||||
])
|
|
||||||
if proc.returncode != 0:
|
|
||||||
raise OrchestratorStartError(
|
|
||||||
f"orchestrator container failed to join client network "
|
|
||||||
f"{self._client_network}: {proc.stderr.strip()}"
|
|
||||||
)
|
|
||||||
|
|
||||||
def stop(self) -> None:
|
def stop(self) -> None:
|
||||||
"""Remove the control-plane container (idempotent)."""
|
"""Remove the control-plane container (idempotent)."""
|
||||||
|
|||||||
@@ -6,17 +6,12 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
from typing import Iterator
|
from typing import Iterator
|
||||||
|
|
||||||
from ...log import die, info
|
from ...log import die, info
|
||||||
from ...util import slugify as _slugify
|
|
||||||
|
|
||||||
|
|
||||||
def slugify(name: str) -> str:
|
|
||||||
"""Compatibility wrapper; new generic callers import ``bot_bottle.util``."""
|
|
||||||
return _slugify(name)
|
|
||||||
|
|
||||||
|
|
||||||
def run_docker(
|
def run_docker(
|
||||||
@@ -119,6 +114,19 @@ def docker_cp(src: str, dest: str) -> None:
|
|||||||
f"{(result.stderr or '').strip() or '<no stderr>'}")
|
f"{(result.stderr or '').strip() or '<no stderr>'}")
|
||||||
|
|
||||||
|
|
||||||
|
_SLUG_RE = re.compile(r"[^a-z0-9]+")
|
||||||
|
|
||||||
|
|
||||||
|
def slugify(name: str) -> str:
|
||||||
|
"""Lowercase, non-alnum runs → '-', trimmed. Dies on empty result."""
|
||||||
|
if not name:
|
||||||
|
die("slugify: missing name")
|
||||||
|
slug = _SLUG_RE.sub("-", name.lower()).strip("-")
|
||||||
|
if not slug:
|
||||||
|
die(f"name '{name}' produced an empty slug; use alphanumeric characters")
|
||||||
|
return slug
|
||||||
|
|
||||||
|
|
||||||
def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
|
def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
|
||||||
"""Invokes `docker build` every call. Layer cache makes no-change
|
"""Invokes `docker build` every call. Layer cache makes no-change
|
||||||
rebuilds cheap; running every time means Dockerfile edits land
|
rebuilds cheap; running every time means Dockerfile edits land
|
||||||
|
|||||||
@@ -11,8 +11,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
from ..bottle_state import egress_state_dir
|
from ..bottle_state import egress_state_dir
|
||||||
from ..egress import EGRESS_ROUTES_FILENAME
|
from ..egress import EGRESS_ROUTES_FILENAME
|
||||||
from ..gateway.egress.schema import load_config
|
from ..gateway.egress.addon_core import LOG_OFF, load_config
|
||||||
from ..gateway.egress.types import LOG_OFF
|
|
||||||
|
|
||||||
|
|
||||||
class EgressApplyError(RuntimeError):
|
class EgressApplyError(RuntimeError):
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ def _config(
|
|||||||
vcpus: int,
|
vcpus: int,
|
||||||
mem_mib: int,
|
mem_mib: int,
|
||||||
guest_mac: str,
|
guest_mac: str,
|
||||||
data_drives: tuple[Path, ...] = (),
|
data_drive: Path | None = None,
|
||||||
extra_boot_args: str = "",
|
extra_boot_args: str = "",
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
drives: list[dict[str, object]] = [
|
drives: list[dict[str, object]] = [
|
||||||
@@ -95,15 +95,12 @@ def _config(
|
|||||||
"is_read_only": False,
|
"is_read_only": False,
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
# Extra virtio-block devices — the infra VMs' persistent "volumes",
|
# A second virtio-block device (guest /dev/vdb) — the infra VM's
|
||||||
# host-side ext4 files that outlive the ephemeral rootfs across VM restarts.
|
# persistent registry "volume", a host-side ext4 file that outlives the
|
||||||
# They appear to the guest as /dev/vdb, /dev/vdc, ... in list order (after
|
# ephemeral rootfs across VM restarts.
|
||||||
# the root /dev/vda), so callers must keep the order stable: the orchestrator
|
if data_drive is not None:
|
||||||
# attaches its registry (vdb); the gateway attaches its CA (vdb) then its
|
|
||||||
# git-gate state (vdc).
|
|
||||||
for i, data_drive in enumerate(data_drives):
|
|
||||||
drives.append({
|
drives.append({
|
||||||
"drive_id": f"data{i}",
|
"drive_id": "data",
|
||||||
"path_on_host": str(data_drive),
|
"path_on_host": str(data_drive),
|
||||||
"is_root_device": False,
|
"is_root_device": False,
|
||||||
"is_read_only": False,
|
"is_read_only": False,
|
||||||
@@ -141,7 +138,7 @@ def boot(
|
|||||||
mem_mib: int = 2048,
|
mem_mib: int = 2048,
|
||||||
guest_mac: str = "06:00:AC:10:00:02",
|
guest_mac: str = "06:00:AC:10:00:02",
|
||||||
detached: bool = False,
|
detached: bool = False,
|
||||||
data_drives: tuple[Path, ...] = (),
|
data_drive: Path | None = None,
|
||||||
extra_boot_args: str = "",
|
extra_boot_args: str = "",
|
||||||
) -> VmHandle:
|
) -> VmHandle:
|
||||||
"""Write the config and launch the VMM. Returns once the process is
|
"""Write the config and launch the VMM. Returns once the process is
|
||||||
@@ -158,7 +155,7 @@ def boot(
|
|||||||
_config(
|
_config(
|
||||||
rootfs=rootfs, tap=tap, guest_ip=guest_ip, host_ip=host_ip,
|
rootfs=rootfs, tap=tap, guest_ip=guest_ip, host_ip=host_ip,
|
||||||
pubkey=pubkey, vcpus=vcpus, mem_mib=mem_mib, guest_mac=guest_mac,
|
pubkey=pubkey, vcpus=vcpus, mem_mib=mem_mib, guest_mac=guest_mac,
|
||||||
data_drives=data_drives, extra_boot_args=extra_boot_args,
|
data_drive=data_drive, extra_boot_args=extra_boot_args,
|
||||||
),
|
),
|
||||||
indent=2,
|
indent=2,
|
||||||
))
|
))
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ singleton flock) is `FirecrackerInfraService` (`infra.py`).
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import subprocess
|
import subprocess
|
||||||
from pathlib import Path
|
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from ...gateway import (
|
from ...gateway import (
|
||||||
@@ -28,7 +27,6 @@ from ...gateway import (
|
|||||||
GatewayError,
|
GatewayError,
|
||||||
GatewayTransport,
|
GatewayTransport,
|
||||||
)
|
)
|
||||||
from ...log import die, info
|
|
||||||
from .. import util as backend_util
|
from .. import util as backend_util
|
||||||
from . import infra_vm, netpool, util
|
from . import infra_vm, netpool, util
|
||||||
from .gateway_transport import FirecrackerGatewayTransport
|
from .gateway_transport import FirecrackerGatewayTransport
|
||||||
@@ -51,26 +49,6 @@ _GUEST_GATEWAY_JWT_PATH = infra_vm._GUEST_GATEWAY_JWT_PATH
|
|||||||
# the gateway's TLS interception. Host-side only (SSH cat), so it lives here.
|
# the gateway's TLS interception. Host-side only (SSH cat), so it lives here.
|
||||||
_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem"
|
_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem"
|
||||||
|
|
||||||
# The gateway VM's persistent CA volume — a small ext4 file the gateway init
|
|
||||||
# mounts at mitmproxy's confdir (see `infra_vm._GATEWAY_CA_MOUNT`) so the
|
|
||||||
# self-generated CA SURVIVES a gateway-VM rebuild/restart. Without it the CA
|
|
||||||
# lives only in the ephemeral per-boot rootfs, so every rebuild mints a fresh CA
|
|
||||||
# that every already-running bottle distrusts, failing the TLS handshake (the
|
|
||||||
# firecracker analogue of the docker fix's persistent CA bind-mount — issue
|
|
||||||
# #450). Co-located with the orchestrator's registry volume under the infra dir,
|
|
||||||
# which outlives the ephemeral rootfs. mitmproxy is tiny; 16M is ample.
|
|
||||||
_CA_VOLUME_SIZE = "16M"
|
|
||||||
|
|
||||||
# The gateway VM's persistent git-gate volume — a (sparse) ext4 file the gateway
|
|
||||||
# init mounts, then bind-mounts onto /git + /git-gate/creds (see
|
|
||||||
# `infra_vm._GATEWAY_GIT_MOUNT`), so per-bottle bare repos and deploy creds
|
|
||||||
# SURVIVE a gateway-VM rebuild. Without it a restart drops every already-running
|
|
||||||
# bottle's git-gate state and its agent 404s on fetch/push (same class as the CA
|
|
||||||
# — issue #512). Bare repos hold upstream history, so this is sized far larger
|
|
||||||
# than the CA volume; mke2fs leaves the file sparse, so the host only stores
|
|
||||||
# blocks actually used.
|
|
||||||
_GIT_VOLUME_SIZE = "8G"
|
|
||||||
|
|
||||||
_CA_FETCH_TIMEOUT_SECONDS = 15.0
|
_CA_FETCH_TIMEOUT_SECONDS = 15.0
|
||||||
|
|
||||||
|
|
||||||
@@ -116,18 +94,11 @@ class FirecrackerGateway(Gateway):
|
|||||||
f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}"
|
f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}"
|
||||||
)
|
)
|
||||||
# Boot on the gateway link from the gateway rootfs, then push the token
|
# Boot on the gateway link from the gateway rootfs, then push the token
|
||||||
# the init waits for before starting the data plane. Two persistent
|
# the init waits for before starting the data plane.
|
||||||
# volumes ride along, attached in a FIXED order the gateway init depends
|
|
||||||
# on: the CA volume as /dev/vdb (mounted at mitmproxy's confdir) and the
|
|
||||||
# git-gate volume as /dev/vdc (bind-mounted onto /git + /git-gate/creds).
|
|
||||||
# Both keep gateway-side state STABLE across rebuilds so already-running
|
|
||||||
# bottles keep working — TLS interception (issue #450) and git-gate fetch
|
|
||||||
# /push (issue #512) respectively.
|
|
||||||
vm = infra_vm.boot_vm(
|
vm = infra_vm.boot_vm(
|
||||||
name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(),
|
name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(),
|
||||||
role="gateway", mem_mib=_GW_MEM_MIB,
|
role="gateway", mem_mib=_GW_MEM_MIB,
|
||||||
extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
|
extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
|
||||||
data_drives=(self._ensure_ca_volume(), self._ensure_git_volume()),
|
|
||||||
)
|
)
|
||||||
infra_vm.push_secret(
|
infra_vm.push_secret(
|
||||||
vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH,
|
vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH,
|
||||||
@@ -145,46 +116,6 @@ class FirecrackerGateway(Gateway):
|
|||||||
infra_vm._kill_pidfile(infra_vm._gw_dir())
|
infra_vm._kill_pidfile(infra_vm._gw_dir())
|
||||||
infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True)
|
infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True)
|
||||||
|
|
||||||
def _ensure_ca_volume(self) -> Path:
|
|
||||||
"""Create the empty ext4 CA volume on first use; reuse it after.
|
|
||||||
|
|
||||||
A fresh (empty) volume makes mitmproxy generate a CA into it on first
|
|
||||||
boot; every later boot reuses the CA already on the volume — which is
|
|
||||||
what keeps the CA stable across gateway-VM rebuilds. The mirror of
|
|
||||||
`FirecrackerOrchestrator._ensure_registry_volume`."""
|
|
||||||
vol = infra_vm._gw_dir() / "gateway-ca.ext4"
|
|
||||||
if vol.exists():
|
|
||||||
return vol
|
|
||||||
info(f"creating gateway CA volume {vol} ({_CA_VOLUME_SIZE})")
|
|
||||||
proc = subprocess.run(
|
|
||||||
["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _CA_VOLUME_SIZE],
|
|
||||||
capture_output=True, text=True, check=False,
|
|
||||||
)
|
|
||||||
if proc.returncode != 0:
|
|
||||||
vol.unlink(missing_ok=True)
|
|
||||||
die(f"creating gateway CA volume failed: {proc.stderr.strip()}")
|
|
||||||
return vol
|
|
||||||
|
|
||||||
def _ensure_git_volume(self) -> Path:
|
|
||||||
"""Create the empty ext4 git-gate volume on first use; reuse it after.
|
|
||||||
|
|
||||||
Empty on first boot (the gateway init lays out `git/` + `creds/` subdirs
|
|
||||||
and bind-mounts them); every later boot reuses whatever repos + creds the
|
|
||||||
volume already holds — which is what keeps git-gate state stable across a
|
|
||||||
gateway-VM rebuild (issue #512). Sibling of `_ensure_ca_volume`."""
|
|
||||||
vol = infra_vm._gw_dir() / "gateway-git.ext4"
|
|
||||||
if vol.exists():
|
|
||||||
return vol
|
|
||||||
info(f"creating gateway git-gate volume {vol} ({_GIT_VOLUME_SIZE})")
|
|
||||||
proc = subprocess.run(
|
|
||||||
["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _GIT_VOLUME_SIZE],
|
|
||||||
capture_output=True, text=True, check=False,
|
|
||||||
)
|
|
||||||
if proc.returncode != 0:
|
|
||||||
vol.unlink(missing_ok=True)
|
|
||||||
die(f"creating gateway git-gate volume failed: {proc.stderr.strip()}")
|
|
||||||
return vol
|
|
||||||
|
|
||||||
def address(self) -> str:
|
def address(self) -> str:
|
||||||
"""The gateway VM's guest IP — the agent-facing target agent VMs'
|
"""The gateway VM's guest IP — the agent-facing target agent VMs'
|
||||||
gateway-port traffic is DNAT'd to."""
|
gateway-port traffic is DNAT'd to."""
|
||||||
|
|||||||
@@ -53,25 +53,10 @@ from . import firecracker_vm, infra_artifact, netpool, util
|
|||||||
# tokens with the same key the host CLI signs with — the host token file stays
|
# tokens with the same key the host CLI signs with — the host token file stays
|
||||||
# the single source of truth, never clobbered per-backend (issue #469 review).
|
# the single source of truth, never clobbered per-backend (issue #469 review).
|
||||||
_GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/orchestrator-token"
|
_GUEST_SIGNING_KEY_PATH = "/var/lib/bot-bottle/orchestrator-token"
|
||||||
# The gateway VM's pre-minted `gateway` JWT path (rootfs, not /dev/vdb — the JWT
|
# The gateway VM's pre-minted `gateway` JWT path (rootfs, not /dev/vdb — the
|
||||||
# is re-pushed every boot, so it needn't persist). Pushed post-boot; the gateway
|
# data plane has no registry volume and never opens the DB). Pushed post-boot;
|
||||||
# daemons present it to the orchestrator, and never see the key.
|
# the gateway daemons present it to the orchestrator, and never see the key.
|
||||||
_GUEST_GATEWAY_JWT_PATH = "/var/lib/bot-bottle/gateway-jwt"
|
_GUEST_GATEWAY_JWT_PATH = "/var/lib/bot-bottle/gateway-jwt"
|
||||||
# The gateway VM's persistent CA volume mount point — mitmproxy's confdir. The
|
|
||||||
# gateway boots with a persistent /dev/vdb CA volume (see
|
|
||||||
# `FirecrackerGateway._ensure_ca_volume`) mounted here so the self-generated CA
|
|
||||||
# survives a gateway-VM rebuild; without it every rebuild mints a fresh CA that
|
|
||||||
# already-running bottles distrust, breaking the TLS handshake (issue #450).
|
|
||||||
_GATEWAY_CA_MOUNT = "/home/mitmproxy/.mitmproxy"
|
|
||||||
# The gateway VM's persistent git-gate volume staging mount (/dev/vdc). The
|
|
||||||
# gateway boots with this volume (see `FirecrackerGateway._ensure_git_volume`)
|
|
||||||
# and the init bind-mounts its `git/` + `creds/` subdirs onto the load-bearing
|
|
||||||
# `/git` and `/git-gate/creds` paths, so per-bottle bare repos + deploy creds
|
|
||||||
# survive a gateway-VM rebuild; without it a restart drops every already-running
|
|
||||||
# bottle's git-gate state and its agent 404s on fetch/push (issue #512).
|
|
||||||
_GATEWAY_GIT_MOUNT = "/var/lib/bot-bottle-gitgate"
|
|
||||||
_GATEWAY_GIT_REPO_ROOT = "/git"
|
|
||||||
_GATEWAY_GIT_CREDS_DIR = "/git-gate/creds"
|
|
||||||
|
|
||||||
# The two per-plane rootfs source images. The orchestrator VM boots a control
|
# The two per-plane rootfs source images. The orchestrator VM boots a control
|
||||||
# plane + buildah rootfs (Dockerfile.orchestrator.fc, FROM orchestrator); the
|
# plane + buildah rootfs (Dockerfile.orchestrator.fc, FROM orchestrator); the
|
||||||
@@ -207,12 +192,11 @@ def boot_vm(
|
|||||||
run_dir: Path,
|
run_dir: Path,
|
||||||
role: str,
|
role: str,
|
||||||
mem_mib: int,
|
mem_mib: int,
|
||||||
data_drives: tuple[Path, ...] = (),
|
data_drive: Path | None = None,
|
||||||
extra_boot_args: str = "",
|
extra_boot_args: str = "",
|
||||||
) -> InfraVm:
|
) -> InfraVm:
|
||||||
"""Boot the `role` infra VM from its per-plane rootfs on `slot`'s link.
|
"""Boot the `role` infra VM from its per-plane rootfs on `slot`'s link.
|
||||||
Records the PID. `data_drives` are attached as /dev/vdb, /dev/vdc, ... in
|
Records the PID."""
|
||||||
order, so callers must keep the order stable (see `firecracker_vm._config`)."""
|
|
||||||
if not netpool.tap_present(slot.iface):
|
if not netpool.tap_present(slot.iface):
|
||||||
die(f"infra link {slot.iface} not present.\n"
|
die(f"infra link {slot.iface} not present.\n"
|
||||||
f" ./cli.py backend setup --backend=firecracker")
|
f" ./cli.py backend setup --backend=firecracker")
|
||||||
@@ -234,7 +218,7 @@ def boot_vm(
|
|||||||
name=name, rootfs=rootfs, tap=slot.iface,
|
name=name, rootfs=rootfs, tap=slot.iface,
|
||||||
guest_ip=slot.guest_ip, host_ip=slot.host_ip, pubkey=pubkey,
|
guest_ip=slot.guest_ip, host_ip=slot.host_ip, pubkey=pubkey,
|
||||||
run_dir=run_dir, mem_mib=mem_mib, detached=True,
|
run_dir=run_dir, mem_mib=mem_mib, detached=True,
|
||||||
data_drives=data_drives, extra_boot_args=boot_args,
|
data_drive=data_drive, extra_boot_args=boot_args,
|
||||||
)
|
)
|
||||||
_pid_file(run_dir).write_text(str(vm.process.pid))
|
_pid_file(run_dir).write_text(str(vm.process.pid))
|
||||||
return InfraVm(guest_ip=slot.guest_ip, private_key=private_key, vm=vm)
|
return InfraVm(guest_ip=slot.guest_ip, private_key=private_key, vm=vm)
|
||||||
@@ -464,24 +448,6 @@ def _gateway_init() -> str:
|
|||||||
bot-bottle.db (PRD 0070 / #469). If the JWT never arrives, REFUSE to start
|
bot-bottle.db (PRD 0070 / #469). If the JWT never arrives, REFUSE to start
|
||||||
rather than run without auth."""
|
rather than run without auth."""
|
||||||
return _init_head() + f"""
|
return _init_head() + f"""
|
||||||
# Persistent CA volume (second virtio-block device, /dev/vdb) mounted at
|
|
||||||
# mitmproxy's confdir, so the self-generated mitmproxy CA survives gateway-VM
|
|
||||||
# rebuilds (issue #450). On first boot the volume is empty and mitmproxy mints a
|
|
||||||
# CA into it; every later boot reuses it. Must mount BEFORE the data plane (hence
|
|
||||||
# mitmproxy) starts.
|
|
||||||
mkdir -p {_GATEWAY_CA_MOUNT}
|
|
||||||
mount -t ext4 /dev/vdb {_GATEWAY_CA_MOUNT} 2>/dev/null || true
|
|
||||||
# Persistent git-gate volume (/dev/vdc): its git/ + creds/ subdirs are
|
|
||||||
# bind-mounted onto the load-bearing /git and /git-gate/creds so per-bottle bare
|
|
||||||
# repos + deploy creds survive a gateway-VM rebuild (issue #512). On first boot
|
|
||||||
# the volume is empty; the subdirs are created here. Must mount BEFORE the data
|
|
||||||
# plane (hence git-http) starts, and before any per-bottle provisioning writes.
|
|
||||||
mkdir -p {_GATEWAY_GIT_MOUNT}
|
|
||||||
mount -t ext4 /dev/vdc {_GATEWAY_GIT_MOUNT} 2>/dev/null || true
|
|
||||||
mkdir -p {_GATEWAY_GIT_MOUNT}/git {_GATEWAY_GIT_MOUNT}/creds
|
|
||||||
mkdir -p {_GATEWAY_GIT_REPO_ROOT} {_GATEWAY_GIT_CREDS_DIR}
|
|
||||||
mount --bind {_GATEWAY_GIT_MOUNT}/git {_GATEWAY_GIT_REPO_ROOT} 2>/dev/null || true
|
|
||||||
mount --bind {_GATEWAY_GIT_MOUNT}/creds {_GATEWAY_GIT_CREDS_DIR} 2>/dev/null || true
|
|
||||||
ORCH=$(sed -n 's/.*bb_orch=\\([^ ]*\\).*/\\1/p' /proc/cmdline)
|
ORCH=$(sed -n 's/.*bb_orch=\\([^ ]*\\).*/\\1/p' /proc/cmdline)
|
||||||
GW_JWT=""
|
GW_JWT=""
|
||||||
i=0
|
i=0
|
||||||
|
|||||||
@@ -104,7 +104,7 @@ class FirecrackerOrchestrator(Orchestrator):
|
|||||||
vm = infra_vm.boot_vm(
|
vm = infra_vm.boot_vm(
|
||||||
name=ORCHESTRATOR_NAME, slot=netpool.orch_slot(),
|
name=ORCHESTRATOR_NAME, slot=netpool.orch_slot(),
|
||||||
run_dir=infra_vm._orch_dir(), role="orchestrator", mem_mib=_ORCH_MEM_MIB,
|
run_dir=infra_vm._orch_dir(), role="orchestrator", mem_mib=_ORCH_MEM_MIB,
|
||||||
data_drives=(self._ensure_registry_volume(),),
|
data_drive=self._ensure_registry_volume(),
|
||||||
)
|
)
|
||||||
# Push the host-canonical signing key (the init waits for it before
|
# Push the host-canonical signing key (the init waits for it before
|
||||||
# starting the control plane). It comes through the shared provisioning
|
# starting the control plane). It comes through the shared provisioning
|
||||||
|
|||||||
@@ -1,124 +0,0 @@
|
|||||||
"""Backend-neutral preparation planner.
|
|
||||||
|
|
||||||
This module owns the shared transformation from a CLI ``BottleSpec`` to the
|
|
||||||
typed inputs consumed by a concrete backend's ``_resolve_plan``. Backend
|
|
||||||
classes retain only their validation/preflight/env hooks and their
|
|
||||||
backend-specific final resolution.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from typing import TYPE_CHECKING, Protocol
|
|
||||||
|
|
||||||
from ..agent_provider import AgentProvisionPlan, build_agent_provision_plan, get_provider
|
|
||||||
from ..egress import EgressPlan
|
|
||||||
from ..env import ResolvedEnv, resolve_env
|
|
||||||
from ..git_gate import GitGate, GitGatePlan
|
|
||||||
from ..manifest import Manifest
|
|
||||||
from ..supervisor.plan import SupervisePlan
|
|
||||||
from ..workspace import workspace_plan
|
|
||||||
from .resolve_common import (
|
|
||||||
merge_provision_env_vars,
|
|
||||||
mint_slug,
|
|
||||||
prepare_agent_state_dir,
|
|
||||||
prepare_egress,
|
|
||||||
prepare_git_gate,
|
|
||||||
prepare_supervise,
|
|
||||||
reject_nested_containers,
|
|
||||||
resolve_manifest_dockerfile,
|
|
||||||
write_launch_metadata,
|
|
||||||
)
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from .base import BottleSpec
|
|
||||||
|
|
||||||
|
|
||||||
class PreparationBackend(Protocol):
|
|
||||||
"""Backend hooks needed by the shared planner."""
|
|
||||||
|
|
||||||
name: str
|
|
||||||
supports_nested_containers: bool
|
|
||||||
|
|
||||||
def _validate(self, spec: BottleSpec) -> Manifest: ...
|
|
||||||
def _preflight(self) -> None: ...
|
|
||||||
def _build_guest_env(self, resolved_env: ResolvedEnv) -> dict[str, str]: ...
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class PreparedBottle:
|
|
||||||
"""Typed, backend-neutral result of shared launch preparation."""
|
|
||||||
|
|
||||||
manifest: Manifest
|
|
||||||
slug: str
|
|
||||||
resolved_env: ResolvedEnv
|
|
||||||
agent_provision_plan: AgentProvisionPlan
|
|
||||||
egress_plan: EgressPlan
|
|
||||||
git_gate_plan: GitGatePlan
|
|
||||||
supervise_plan: SupervisePlan | None
|
|
||||||
|
|
||||||
|
|
||||||
class BottlePreparationPlanner:
|
|
||||||
"""Run the common, side-effect-limited part of bottle preparation."""
|
|
||||||
|
|
||||||
def __init__(self, backend: PreparationBackend) -> None:
|
|
||||||
self._backend = backend
|
|
||||||
|
|
||||||
def prepare(self, spec: BottleSpec) -> PreparedBottle:
|
|
||||||
backend = self._backend
|
|
||||||
# These are deliberately protected backend hooks: only this shared
|
|
||||||
# planner orchestrates them, while concrete backends provide the
|
|
||||||
# implementation.
|
|
||||||
manifest = backend._validate(spec) # pylint: disable=protected-access
|
|
||||||
if not backend.supports_nested_containers:
|
|
||||||
reject_nested_containers(backend.name, manifest)
|
|
||||||
|
|
||||||
backend._preflight() # pylint: disable=protected-access
|
|
||||||
manifest = GitGate().preflight_host_keys(
|
|
||||||
manifest,
|
|
||||||
headless=spec.headless,
|
|
||||||
home_md=spec.manifest.home_md,
|
|
||||||
)
|
|
||||||
|
|
||||||
bottle = manifest.bottle
|
|
||||||
provider_config = bottle.agent_provider
|
|
||||||
provider = get_provider(provider_config.template)
|
|
||||||
resolved_env = resolve_env(manifest)
|
|
||||||
workspace = workspace_plan(spec, guest_home=provider.guest_home)
|
|
||||||
slug = mint_slug(spec)
|
|
||||||
write_launch_metadata(slug, spec, compose_project="", backend=backend.name)
|
|
||||||
|
|
||||||
dockerfile = (
|
|
||||||
resolve_manifest_dockerfile(provider_config.dockerfile, spec)
|
|
||||||
if provider_config.dockerfile
|
|
||||||
else str(provider.dockerfile)
|
|
||||||
)
|
|
||||||
agent_dir, prompt_file = prepare_agent_state_dir(slug, manifest)
|
|
||||||
provision = build_agent_provision_plan(
|
|
||||||
template=provider_config.template,
|
|
||||||
dockerfile=dockerfile,
|
|
||||||
state_dir=agent_dir,
|
|
||||||
instance_name=f"bot-bottle-{slug}",
|
|
||||||
prompt_file=prompt_file,
|
|
||||||
guest_env=backend._build_guest_env( # pylint: disable=protected-access
|
|
||||||
resolved_env
|
|
||||||
),
|
|
||||||
forward_host_credentials=provider_config.forward_host_credentials,
|
|
||||||
auth_token=provider_config.auth_token,
|
|
||||||
host_env=dict(os.environ),
|
|
||||||
trusted_project_path=workspace.workdir,
|
|
||||||
label=spec.label,
|
|
||||||
color=spec.color,
|
|
||||||
provider_settings=provider_config.settings,
|
|
||||||
)
|
|
||||||
provision = merge_provision_env_vars(provision)
|
|
||||||
return PreparedBottle(
|
|
||||||
manifest=manifest,
|
|
||||||
slug=slug,
|
|
||||||
resolved_env=resolved_env,
|
|
||||||
agent_provision_plan=provision,
|
|
||||||
egress_plan=prepare_egress(bottle, slug, provision),
|
|
||||||
git_gate_plan=prepare_git_gate(bottle, slug),
|
|
||||||
supervise_plan=prepare_supervise(bottle, slug),
|
|
||||||
)
|
|
||||||
@@ -30,7 +30,6 @@ from ..log import die
|
|||||||
from ..manifest import Manifest, ManifestBottle
|
from ..manifest import Manifest, ManifestBottle
|
||||||
from ..supervisor.plan import SupervisePlan
|
from ..supervisor.plan import SupervisePlan
|
||||||
from ..orchestrator.supervisor import Supervisor
|
from ..orchestrator.supervisor import Supervisor
|
||||||
from ..util import slugify
|
|
||||||
from . import BottleSpec
|
from . import BottleSpec
|
||||||
|
|
||||||
|
|
||||||
@@ -45,7 +44,8 @@ def mint_slug(spec: BottleSpec) -> str:
|
|||||||
if spec.identity:
|
if spec.identity:
|
||||||
return spec.identity
|
return spec.identity
|
||||||
if spec.label:
|
if spec.label:
|
||||||
return slugify(spec.label)
|
from .docker import util as docker_mod
|
||||||
|
return docker_mod.slugify(spec.label)
|
||||||
return bottle_identity(spec.agent_name)
|
return bottle_identity(spec.agent_name)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -25,11 +25,12 @@ from typing import Callable
|
|||||||
from ...agent_provider import get_provider, runtime_for
|
from ...agent_provider import get_provider, runtime_for
|
||||||
from ...backend import (
|
from ...backend import (
|
||||||
Bottle,
|
Bottle,
|
||||||
BottlePlan,
|
|
||||||
BottleSpec,
|
BottleSpec,
|
||||||
enumerate_active_agents,
|
enumerate_active_agents,
|
||||||
get_bottle_backend,
|
get_bottle_backend,
|
||||||
)
|
)
|
||||||
|
from ...backend.docker import util as docker_mod
|
||||||
|
from ...backend.docker.bottle_plan import DockerBottlePlan
|
||||||
from ...bottle_state import (
|
from ...bottle_state import (
|
||||||
cleanup_state,
|
cleanup_state,
|
||||||
is_preserved,
|
is_preserved,
|
||||||
@@ -39,7 +40,7 @@ from ...image_cache import StaleImageError
|
|||||||
from ...log import info, die
|
from ...log import info, die
|
||||||
from ...manifest import Manifest, ManifestIndex
|
from ...manifest import Manifest, ManifestIndex
|
||||||
from ..constants import PROG
|
from ..constants import PROG
|
||||||
from ...util import read_tty_line, slugify
|
from ...util import read_tty_line
|
||||||
from .. import tui
|
from .. import tui
|
||||||
|
|
||||||
|
|
||||||
@@ -256,10 +257,10 @@ def _uniquify_label_headless(label: str) -> str:
|
|||||||
logging the chosen label. Orchestrators fire-and-forget many bottles,
|
logging the chosen label. Orchestrators fire-and-forget many bottles,
|
||||||
so silently picking a free name beats erroring on every collision."""
|
so silently picking a free name beats erroring on every collision."""
|
||||||
active_slugs = {a.slug for a in enumerate_active_agents()}
|
active_slugs = {a.slug for a in enumerate_active_agents()}
|
||||||
if slugify(label) not in active_slugs:
|
if docker_mod.slugify(label) not in active_slugs:
|
||||||
return label
|
return label
|
||||||
n = 2
|
n = 2
|
||||||
while slugify(f"{label}-{n}") in active_slugs:
|
while docker_mod.slugify(f"{label}-{n}") in active_slugs:
|
||||||
n += 1
|
n += 1
|
||||||
chosen = f"{label}-{n}"
|
chosen = f"{label}-{n}"
|
||||||
info(f"label '{label}' already in use; using '{chosen}'")
|
info(f"label '{label}' already in use; using '{chosen}'")
|
||||||
@@ -273,11 +274,11 @@ def prepare_with_preflight(
|
|||||||
spec: BottleSpec,
|
spec: BottleSpec,
|
||||||
*,
|
*,
|
||||||
stage_dir: Path,
|
stage_dir: Path,
|
||||||
render_preflight: Callable[[BottlePlan, str], None],
|
render_preflight: Callable[[DockerBottlePlan, str], None],
|
||||||
prompt_yes: Callable[[], bool],
|
prompt_yes: Callable[[], bool],
|
||||||
dry_run: bool = False,
|
dry_run: bool = False,
|
||||||
backend_name: str | None = None,
|
backend_name: str | None = None,
|
||||||
) -> tuple[BottlePlan | None, str]:
|
) -> tuple[DockerBottlePlan | None, str]:
|
||||||
"""Run `backend.prepare`, render the preflight summary via the
|
"""Run `backend.prepare`, render the preflight summary via the
|
||||||
injected callable, prompt y/N via the injected callable.
|
injected callable, prompt y/N via the injected callable.
|
||||||
|
|
||||||
@@ -404,7 +405,7 @@ def _resolve_unique_label(label: str, color: str) -> tuple[str, str]:
|
|||||||
in use among running bottles. Passes through unchanged when no
|
in use among running bottles. Passes through unchanged when no
|
||||||
collision is found on the first check."""
|
collision is found on the first check."""
|
||||||
while True:
|
while True:
|
||||||
slug_candidate = slugify(label)
|
slug_candidate = docker_mod.slugify(label)
|
||||||
active_slugs = {a.slug for a in enumerate_active_agents()}
|
active_slugs = {a.slug for a in enumerate_active_agents()}
|
||||||
if slug_candidate not in active_slugs:
|
if slug_candidate not in active_slugs:
|
||||||
return label, color
|
return label, color
|
||||||
@@ -431,7 +432,7 @@ def _select_image_policy() -> str | None:
|
|||||||
|
|
||||||
|
|
||||||
def _text_render_preflight():
|
def _text_render_preflight():
|
||||||
def _render(plan: BottlePlan, backend_name: str) -> None:
|
def _render(plan: DockerBottlePlan, backend_name: str) -> None:
|
||||||
print(file=sys.stderr)
|
print(file=sys.stderr)
|
||||||
print(f"backend: {backend_name}", file=sys.stderr)
|
print(f"backend: {backend_name}", file=sys.stderr)
|
||||||
print(_manifest_to_yaml(plan.manifest), file=sys.stderr)
|
print(_manifest_to_yaml(plan.manifest), file=sys.stderr)
|
||||||
|
|||||||
@@ -368,7 +368,7 @@ def _main_loop(stdscr: "curses._CursesWindow") -> None: # type: ignore # pragm
|
|||||||
elif key in (curses.KEY_UP, ord("k")):
|
elif key in (curses.KEY_UP, ord("k")):
|
||||||
selected = max(selected - 1, 0)
|
selected = max(selected - 1, 0)
|
||||||
elif key in (curses.KEY_ENTER, 10, 13):
|
elif key in (curses.KEY_ENTER, 10, 13):
|
||||||
status_line = _detail_view(stdscr, qp, green_attr=green_attr)
|
_detail_view(stdscr, qp, green_attr=green_attr)
|
||||||
elif key == ord("a"):
|
elif key == ord("a"):
|
||||||
try:
|
try:
|
||||||
status_line = _approve_from_tui(stdscr, qp)
|
status_line = _approve_from_tui(stdscr, qp)
|
||||||
@@ -456,7 +456,7 @@ def _detail_view(
|
|||||||
qp: QueuedProposal,
|
qp: QueuedProposal,
|
||||||
*,
|
*,
|
||||||
green_attr: int = 0,
|
green_attr: int = 0,
|
||||||
) -> str: # pragma: no cover
|
) -> None: # pragma: no cover
|
||||||
"""Render the full proposal. Scrollable. Press q to return."""
|
"""Render the full proposal. Scrollable. Press q to return."""
|
||||||
lines = _detail_lines(qp, green_attr=green_attr)
|
lines = _detail_lines(qp, green_attr=green_attr)
|
||||||
offset = 0
|
offset = 0
|
||||||
@@ -473,7 +473,7 @@ def _detail_view(
|
|||||||
stdscr.refresh()
|
stdscr.refresh()
|
||||||
key = stdscr.getch()
|
key = stdscr.getch()
|
||||||
if key in (ord("q"), 27):
|
if key in (ord("q"), 27):
|
||||||
return ""
|
return
|
||||||
if key in (curses.KEY_DOWN, ord("j")):
|
if key in (curses.KEY_DOWN, ord("j")):
|
||||||
offset = min(offset + 1, max(0, len(lines) - 1))
|
offset = min(offset + 1, max(0, len(lines) - 1))
|
||||||
elif key in (curses.KEY_UP, ord("k")):
|
elif key in (curses.KEY_UP, ord("k")):
|
||||||
@@ -484,34 +484,31 @@ def _detail_view(
|
|||||||
offset = max(0, len(lines) - 1)
|
offset = max(0, len(lines) - 1)
|
||||||
elif key == ord("a"):
|
elif key == ord("a"):
|
||||||
try:
|
try:
|
||||||
return _approve_from_tui(stdscr, qp)
|
_approve_from_tui(stdscr, qp)
|
||||||
except ApplyError as exc:
|
except ApplyError:
|
||||||
return f"apply failed: {exc}"
|
pass
|
||||||
|
return
|
||||||
elif key == ord("m"):
|
elif key == ord("m"):
|
||||||
if qp.proposal.tool in _REPORT_ONLY_TOOLS:
|
if qp.proposal.tool in _REPORT_ONLY_TOOLS:
|
||||||
return f"modify unavailable for {qp.proposal.tool}"
|
return
|
||||||
edited = _modify(stdscr, qp)
|
edited = _modify(stdscr, qp)
|
||||||
if edited is None:
|
if edited is not None:
|
||||||
return "modify aborted (no change)"
|
try:
|
||||||
try:
|
_approve_from_tui(
|
||||||
return _approve_from_tui(
|
stdscr, qp, final_file=edited,
|
||||||
stdscr, qp, final_file=edited,
|
notes="operator modified before approving",
|
||||||
notes="operator modified before approving",
|
)
|
||||||
)
|
except ApplyError:
|
||||||
except ApplyError as exc:
|
pass
|
||||||
return f"apply failed: {exc}"
|
return
|
||||||
elif key == ord("r"):
|
elif key == ord("r"):
|
||||||
reason = _prompt(stdscr, "reject reason: ")
|
reason = _prompt(stdscr, "reject reason: ")
|
||||||
if reason:
|
if reason:
|
||||||
reject(qp, reason=reason)
|
reject(qp, reason=reason)
|
||||||
return f"rejected {qp.proposal.tool} for [{qp.label}]"
|
return
|
||||||
return "reject aborted (empty reason)"
|
|
||||||
|
|
||||||
|
|
||||||
def _modify(
|
def _modify(stdscr: "curses._CursesWindow", qp: QueuedProposal) -> str | None: # type: ignore # pragma: no cover
|
||||||
stdscr: "curses._CursesWindow", # type: ignore
|
|
||||||
qp: QueuedProposal,
|
|
||||||
) -> str | None: # pragma: no cover
|
|
||||||
"""Suspend curses, open $EDITOR on the proposed file, return edited content."""
|
"""Suspend curses, open $EDITOR on the proposed file, return edited content."""
|
||||||
suffix = _suffix_for_tool(qp.proposal.tool)
|
suffix = _suffix_for_tool(qp.proposal.tool)
|
||||||
curses.endwin()
|
curses.endwin()
|
||||||
|
|||||||
+6
-32
@@ -16,8 +16,6 @@ import os
|
|||||||
import sys
|
import sys
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
from ..log import debug
|
|
||||||
|
|
||||||
|
|
||||||
def filter_multiselect(
|
def filter_multiselect(
|
||||||
items: list[str],
|
items: list[str],
|
||||||
@@ -44,11 +42,7 @@ def filter_multiselect(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
tty_fd = open(tty_path, "r+b", buffering=0)
|
tty_fd = open(tty_path, "r+b", buffering=0)
|
||||||
except OSError as exc:
|
except OSError:
|
||||||
debug(
|
|
||||||
"multi-select unavailable; treating it as cancellation",
|
|
||||||
context={"error_type": type(exc).__name__, "tty": tty_path},
|
|
||||||
)
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -79,11 +73,7 @@ def filter_select(
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
tty_fd = open(tty_path, "r+b", buffering=0)
|
tty_fd = open(tty_path, "r+b", buffering=0)
|
||||||
except OSError as exc:
|
except OSError:
|
||||||
debug(
|
|
||||||
"filter-select unavailable; treating it as cancellation",
|
|
||||||
context={"error_type": type(exc).__name__, "tty": tty_path},
|
|
||||||
)
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -139,11 +129,7 @@ def _run_picker(items: list[str], *, title: str, tty_fd: int) -> Optional[str]:
|
|||||||
curses.nocbreak()
|
curses.nocbreak()
|
||||||
curses.echo()
|
curses.echo()
|
||||||
curses.endwin()
|
curses.endwin()
|
||||||
except Exception as exc: # noqa: W0718 — curses can raise many error types
|
except Exception: # noqa: W0718 — curses can raise many error types
|
||||||
debug(
|
|
||||||
"filter-select display failed; treating it as cancellation",
|
|
||||||
context={"error_type": type(exc).__name__},
|
|
||||||
)
|
|
||||||
return None
|
return None
|
||||||
finally:
|
finally:
|
||||||
sys.__stdin__ = orig_stdin # type: ignore[assignment]
|
sys.__stdin__ = orig_stdin # type: ignore[assignment]
|
||||||
@@ -306,11 +292,7 @@ def _run_multiselect(
|
|||||||
curses.nocbreak()
|
curses.nocbreak()
|
||||||
curses.echo()
|
curses.echo()
|
||||||
curses.endwin()
|
curses.endwin()
|
||||||
except Exception as exc: # noqa: W0718
|
except Exception: # noqa: W0718
|
||||||
debug(
|
|
||||||
"multi-select display failed; treating it as cancellation",
|
|
||||||
context={"error_type": type(exc).__name__},
|
|
||||||
)
|
|
||||||
return None
|
return None
|
||||||
finally:
|
finally:
|
||||||
sys.__stdin__ = orig_stdin # type: ignore[assignment]
|
sys.__stdin__ = orig_stdin # type: ignore[assignment]
|
||||||
@@ -576,21 +558,13 @@ def name_color_modal(
|
|||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
tty_fd = open(tty_path, "r+b", buffering=0) # pylint: disable=consider-using-with
|
tty_fd = open(tty_path, "r+b", buffering=0) # pylint: disable=consider-using-with
|
||||||
except OSError as exc:
|
except OSError:
|
||||||
debug(
|
|
||||||
"name/color picker unavailable; using defaults",
|
|
||||||
context={"error_type": type(exc).__name__, "tty": tty_path},
|
|
||||||
)
|
|
||||||
return default_label, ""
|
return default_label, ""
|
||||||
|
|
||||||
try:
|
try:
|
||||||
fd_dup = os.dup(tty_fd.fileno())
|
fd_dup = os.dup(tty_fd.fileno())
|
||||||
return _run_name_color(default_label, tty_fd=fd_dup, disclaimer=disclaimer)
|
return _run_name_color(default_label, tty_fd=fd_dup, disclaimer=disclaimer)
|
||||||
except Exception as exc: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
except Exception: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
||||||
debug(
|
|
||||||
"name/color picker failed; using defaults",
|
|
||||||
context={"error_type": type(exc).__name__},
|
|
||||||
)
|
|
||||||
return default_label, ""
|
return default_label, ""
|
||||||
finally:
|
finally:
|
||||||
tty_fd.close()
|
tty_fd.close()
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from __future__ import annotations
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ..gateway.egress.types import Route
|
from ..gateway.egress.addon_core import Route
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -19,7 +19,7 @@ class EgressRoute(Route):
|
|||||||
"""Host-side extension of the addon's `Route`.
|
"""Host-side extension of the addon's `Route`.
|
||||||
|
|
||||||
Inherits `host`, `matches`, `auth_scheme`, and `token_env`
|
Inherits `host`, `matches`, `auth_scheme`, and `token_env`
|
||||||
from the gateway's wire `Route` — those are the fields that cross the
|
from `egress_addon_core.Route` — those are the fields that cross the
|
||||||
YAML wire into the gateway. The fields below are host-only and
|
YAML wire into the gateway. The fields below are host-only and
|
||||||
are never serialised to the addon.
|
are never serialised to the addon.
|
||||||
|
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ import secrets
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from ..gateway.egress.dlp_config import ON_MATCH_REDACT
|
from ..gateway.egress.addon_core import (
|
||||||
from ..gateway.egress.types import (
|
ON_MATCH_REDACT,
|
||||||
HeaderMatch as CoreHeaderMatch,
|
HeaderMatch as CoreHeaderMatch,
|
||||||
MatchEntry as CoreMatchEntry,
|
MatchEntry as CoreMatchEntry,
|
||||||
PathMatch as CorePathMatch,
|
PathMatch as CorePathMatch,
|
||||||
|
|||||||
@@ -17,34 +17,28 @@ from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=
|
|||||||
|
|
||||||
from bot_bottle.constants import IDENTITY_HEADER
|
from bot_bottle.constants import IDENTITY_HEADER
|
||||||
from bot_bottle.gateway.egress.dlp_detectors import redact_tokens, strip_crlf
|
from bot_bottle.gateway.egress.dlp_detectors import redact_tokens, strip_crlf
|
||||||
from bot_bottle.gateway.egress.dlp_config import (
|
from bot_bottle.gateway.egress.addon_core import (
|
||||||
|
LOG_BLOCKS,
|
||||||
|
LOG_FULL,
|
||||||
DEFAULT_OUTBOUND_ON_MATCH,
|
DEFAULT_OUTBOUND_ON_MATCH,
|
||||||
ON_MATCH_BLOCK,
|
ON_MATCH_BLOCK,
|
||||||
ON_MATCH_REDACT,
|
ON_MATCH_REDACT,
|
||||||
)
|
Config,
|
||||||
from bot_bottle.gateway.egress.context import resolve_client_context
|
Route,
|
||||||
from bot_bottle.gateway.egress.dlp import (
|
ScanResult,
|
||||||
build_inbound_scan_text,
|
build_inbound_scan_text,
|
||||||
build_outbound_scan_text,
|
build_outbound_scan_text,
|
||||||
build_token_allow_payload,
|
build_token_allow_payload,
|
||||||
outbound_scan_headers,
|
|
||||||
scan_inbound,
|
|
||||||
scan_outbound,
|
|
||||||
)
|
|
||||||
from bot_bottle.gateway.egress.matching import (
|
|
||||||
decide,
|
decide,
|
||||||
decide_git_fetch,
|
decide_git_fetch,
|
||||||
is_git_fetch_request,
|
is_git_fetch_request,
|
||||||
is_git_push_request,
|
is_git_push_request,
|
||||||
match_route,
|
match_route,
|
||||||
)
|
resolve_client_context,
|
||||||
from bot_bottle.gateway.egress.schema import route_to_yaml_dict
|
outbound_scan_headers,
|
||||||
from bot_bottle.gateway.egress.types import (
|
route_to_yaml_dict,
|
||||||
LOG_BLOCKS,
|
scan_inbound,
|
||||||
LOG_FULL,
|
scan_outbound,
|
||||||
Config,
|
|
||||||
Route,
|
|
||||||
ScanResult,
|
|
||||||
)
|
)
|
||||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||||
from bot_bottle.supervisor.types import (
|
from bot_bottle.supervisor.types import (
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,77 +0,0 @@
|
|||||||
"""Fail-closed resolution of a client's policy and egress credentials."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import typing
|
|
||||||
|
|
||||||
from ...log import debug
|
|
||||||
from .types import Config
|
|
||||||
|
|
||||||
|
|
||||||
DENY_UNATTRIBUTED = (
|
|
||||||
"egress: this request was not attributed to any bottle, so no egress policy "
|
|
||||||
"applies and every host is denied. Either the bottle's registry row is "
|
|
||||||
"missing/ambiguous (torn down, or another bottle claimed its source IP), or "
|
|
||||||
"the request carried no matching identity token — check that the caller's "
|
|
||||||
"proxy URL includes it. This is not an allowlist problem."
|
|
||||||
)
|
|
||||||
DENY_UNPARSEABLE = (
|
|
||||||
"egress: this bottle's egress policy could not be parsed, so it is being "
|
|
||||||
"treated as deny-all. Fix the bottle's egress.routes; every host is denied "
|
|
||||||
"until it loads."
|
|
||||||
)
|
|
||||||
DENY_RESOLVER_ERROR = (
|
|
||||||
"egress: the orchestrator could not be reached to resolve this bottle's "
|
|
||||||
"egress policy, so every host is denied (fail-closed). Check that the "
|
|
||||||
"control plane is up; this is not an allowlist problem."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class PolicyResolverLike(typing.Protocol):
|
|
||||||
def resolve(self, source_ip: str, identity_token: str = ...) -> str | None: ...
|
|
||||||
|
|
||||||
|
|
||||||
class ContextResolverLike(typing.Protocol):
|
|
||||||
def resolve_policy_and_bottle_id(
|
|
||||||
self, source_ip: str, identity_token: str = ...,
|
|
||||||
) -> tuple[str | None, str | None, dict[str, str]]: ...
|
|
||||||
|
|
||||||
|
|
||||||
def _config_from_policy(policy: str | None) -> Config:
|
|
||||||
# Local import keeps schema parsing independent of resolver protocols.
|
|
||||||
from .schema import load_config
|
|
||||||
if not policy:
|
|
||||||
return Config(routes=(), deny_reason=DENY_UNATTRIBUTED)
|
|
||||||
try:
|
|
||||||
return load_config(policy)
|
|
||||||
except ValueError:
|
|
||||||
return Config(routes=(), deny_reason=DENY_UNPARSEABLE)
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_client_config(
|
|
||||||
resolver: PolicyResolverLike, client_ip: str, identity_token: str = "",
|
|
||||||
) -> Config:
|
|
||||||
try:
|
|
||||||
policy = resolver.resolve(client_ip, identity_token)
|
|
||||||
except Exception as exc: # noqa: BLE001 - a policy lookup failure must deny
|
|
||||||
debug(
|
|
||||||
"egress policy resolution failed; applying deny-all",
|
|
||||||
context={"error_type": type(exc).__name__},
|
|
||||||
)
|
|
||||||
return Config(routes=(), deny_reason=DENY_RESOLVER_ERROR)
|
|
||||||
return _config_from_policy(policy)
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_client_context(
|
|
||||||
resolver: ContextResolverLike, client_ip: str, identity_token: str = "",
|
|
||||||
) -> tuple[Config, str, dict[str, str]]:
|
|
||||||
try:
|
|
||||||
policy, bottle_id, tokens = resolver.resolve_policy_and_bottle_id(
|
|
||||||
client_ip, identity_token)
|
|
||||||
except Exception as exc: # noqa: BLE001 - a policy lookup failure must deny
|
|
||||||
debug(
|
|
||||||
"egress context resolution failed; applying deny-all",
|
|
||||||
context={"error_type": type(exc).__name__},
|
|
||||||
)
|
|
||||||
return Config(routes=(), deny_reason=DENY_RESOLVER_ERROR), "", {}
|
|
||||||
return _config_from_policy(policy), (bottle_id or ""), tokens
|
|
||||||
@@ -1,99 +0,0 @@
|
|||||||
"""DLP scan dispatch and safe proposal rendering for egress requests."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import typing
|
|
||||||
|
|
||||||
from .types import Route, ScanResult
|
|
||||||
|
|
||||||
|
|
||||||
def build_outbound_scan_text(host: str, path: str, query: str,
|
|
||||||
headers: typing.Mapping[str, str], body: str) -> str:
|
|
||||||
parts = [host, path]
|
|
||||||
if query:
|
|
||||||
parts.append(query)
|
|
||||||
parts.extend(f"{name}: {value}" for name, value in headers.items())
|
|
||||||
if body:
|
|
||||||
parts.append(body)
|
|
||||||
return "\n".join(parts)
|
|
||||||
|
|
||||||
|
|
||||||
def outbound_scan_headers(route: Route, headers: typing.Mapping[str, str]) -> dict[str, str]:
|
|
||||||
"""Drop agent Authorization when the route injects gateway-owned auth."""
|
|
||||||
skip_auth = bool(route.auth_scheme and route.token_env)
|
|
||||||
return {name: value for name, value in headers.items()
|
|
||||||
if not (skip_auth and name.lower() == "authorization")}
|
|
||||||
|
|
||||||
|
|
||||||
def build_inbound_scan_text(headers: typing.Mapping[str, str], body: str) -> str:
|
|
||||||
parts = [f"{name}: {value}" for name, value in headers.items()]
|
|
||||||
if body:
|
|
||||||
parts.append(body)
|
|
||||||
return "\n".join(parts)
|
|
||||||
|
|
||||||
|
|
||||||
def _enabled(configured: tuple[str, ...] | None, name: str) -> bool:
|
|
||||||
return configured is None or name in configured
|
|
||||||
|
|
||||||
|
|
||||||
def scan_outbound(route: Route, body: str | bytes, environ: typing.Mapping[str, str], *,
|
|
||||||
safe_tokens: typing.AbstractSet[str] | None = None,
|
|
||||||
crlf_text: str | None = None) -> ScanResult | None:
|
|
||||||
if not route.inspect:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
from dlp_detectors import ( # type: ignore[import-not-found]
|
|
||||||
scan_crlf_injection, scan_entropy, scan_known_secrets, scan_token_patterns)
|
|
||||||
except ImportError: # pragma: no cover - gateway's flat module path
|
|
||||||
from .dlp_detectors import (
|
|
||||||
scan_crlf_injection, scan_entropy, scan_known_secrets, scan_token_patterns)
|
|
||||||
if isinstance(body, bytes):
|
|
||||||
try:
|
|
||||||
text = body.decode("utf-8")
|
|
||||||
except UnicodeDecodeError:
|
|
||||||
text = body.decode("latin-1")
|
|
||||||
else:
|
|
||||||
text = body
|
|
||||||
result = scan_crlf_injection(text if crlf_text is None else crlf_text)
|
|
||||||
if result is not None:
|
|
||||||
return result
|
|
||||||
if _enabled(route.outbound_detectors, "token_patterns"):
|
|
||||||
result = scan_token_patterns(text, location="body", safe_tokens=safe_tokens)
|
|
||||||
if result is not None:
|
|
||||||
return result
|
|
||||||
if _enabled(route.outbound_detectors, "known_secrets"):
|
|
||||||
extra = tuple(prefix for prefix in environ.get(
|
|
||||||
"BOT_BOTTLE_SENSITIVE_PREFIXES", "").split(",") if prefix)
|
|
||||||
result = scan_known_secrets(text, location="body", env=environ,
|
|
||||||
sensitive_prefixes=("EGRESS_TOKEN_",) + extra,
|
|
||||||
safe_tokens=safe_tokens)
|
|
||||||
if result is not None:
|
|
||||||
return result
|
|
||||||
if route.outbound_detectors is not None and "entropy" in route.outbound_detectors:
|
|
||||||
return scan_entropy(text, location="body")
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def build_token_allow_payload(host: str, method: str, path: str, result: ScanResult) -> str:
|
|
||||||
"""Render redacted operator context; the raw matched secret is excluded."""
|
|
||||||
lines = [
|
|
||||||
"egress blocked an outbound request carrying a detected token",
|
|
||||||
f"host: {host}", f"method: {method}", f"path: {path}",
|
|
||||||
f"detector: {result.reason}",
|
|
||||||
]
|
|
||||||
if result.context:
|
|
||||||
lines.append(f"context: {result.context}")
|
|
||||||
return "\n".join(lines) + "\n"
|
|
||||||
|
|
||||||
|
|
||||||
def scan_inbound(route: Route, body: str | bytes) -> ScanResult | None:
|
|
||||||
if not route.inspect:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
from dlp_detectors import scan_naive_injection # type: ignore[import-not-found]
|
|
||||||
except ImportError: # pragma: no cover - gateway's flat module path
|
|
||||||
from .dlp_detectors import scan_naive_injection
|
|
||||||
text = body if isinstance(body, str) else body.decode("utf-8", errors="replace")
|
|
||||||
if _enabled(route.inbound_detectors, "naive_injection_detection"):
|
|
||||||
return scan_naive_injection(text)
|
|
||||||
return None
|
|
||||||
@@ -19,7 +19,7 @@ from math import log2
|
|||||||
from collections import Counter
|
from collections import Counter
|
||||||
from urllib.parse import quote as url_quote
|
from urllib.parse import quote as url_quote
|
||||||
|
|
||||||
from .types import ScanResult
|
from .addon_core import ScanResult
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -1,112 +0,0 @@
|
|||||||
"""Route matching and request-policy decisions for the egress gateway."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import typing
|
|
||||||
|
|
||||||
from .types import Decision, MatchEntry, PathMatch, Route
|
|
||||||
|
|
||||||
|
|
||||||
def _path_matches(pm: PathMatch, request_path: str) -> bool:
|
|
||||||
if pm.type == "exact":
|
|
||||||
return request_path == pm.value
|
|
||||||
if pm.type == "prefix":
|
|
||||||
if request_path == pm.value:
|
|
||||||
return True
|
|
||||||
if not pm.value.endswith("/"):
|
|
||||||
return request_path.startswith(pm.value + "/")
|
|
||||||
return request_path.startswith(pm.value)
|
|
||||||
return (
|
|
||||||
pm.type == "regex"
|
|
||||||
and pm.compiled is not None
|
|
||||||
and pm.compiled.search(request_path) is not None
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _entry_matches(
|
|
||||||
entry: MatchEntry, request_path: str, request_method: str,
|
|
||||||
request_headers: typing.Mapping[str, str],
|
|
||||||
) -> bool:
|
|
||||||
if entry.paths and not any(_path_matches(pm, request_path) for pm in entry.paths):
|
|
||||||
return False
|
|
||||||
if entry.methods and request_method.upper() not in entry.methods:
|
|
||||||
return False
|
|
||||||
for match in entry.headers:
|
|
||||||
value = request_headers.get(match.name.lower())
|
|
||||||
if value is None:
|
|
||||||
return False
|
|
||||||
if match.type == "exact" and value != match.value:
|
|
||||||
return False
|
|
||||||
if match.type == "regex" and (
|
|
||||||
match.compiled is None or match.compiled.search(value) is None
|
|
||||||
):
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def evaluate_matches(
|
|
||||||
route: Route, request_path: str, request_method: str = "GET",
|
|
||||||
request_headers: typing.Mapping[str, str] | None = None,
|
|
||||||
) -> bool:
|
|
||||||
"""Return whether a request satisfies a route's optional match entries."""
|
|
||||||
if not route.matches:
|
|
||||||
return True
|
|
||||||
return any(_entry_matches(entry, request_path, request_method, request_headers or {})
|
|
||||||
for entry in route.matches)
|
|
||||||
|
|
||||||
|
|
||||||
def is_git_push_request(path: str, query: str) -> bool:
|
|
||||||
return path.endswith("/git-receive-pack") or (
|
|
||||||
path.endswith("/info/refs") and any(
|
|
||||||
pair.partition("=") == ("service", "=", "git-receive-pack")
|
|
||||||
for pair in query.split("&")
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def is_git_fetch_request(path: str, query: str) -> bool:
|
|
||||||
return path.endswith("/git-upload-pack") or (
|
|
||||||
path.endswith("/info/refs") and any(
|
|
||||||
pair.partition("=") == ("service", "=", "git-upload-pack")
|
|
||||||
for pair in query.split("&")
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def match_route(routes: typing.Sequence[Route], request_host: str) -> Route | None:
|
|
||||||
target = request_host.lower()
|
|
||||||
return next((route for route in routes if route.host.lower() == target), None)
|
|
||||||
|
|
||||||
|
|
||||||
def decide(
|
|
||||||
routes: typing.Sequence[Route], request_host: str, request_path: str,
|
|
||||||
environ: typing.Mapping[str, str], *, request_method: str = "GET",
|
|
||||||
request_headers: typing.Mapping[str, str] | None = None, deny_reason: str = "",
|
|
||||||
) -> Decision:
|
|
||||||
route = match_route(routes, request_host)
|
|
||||||
if route is None:
|
|
||||||
return Decision("block", deny_reason or (
|
|
||||||
f"egress: host {request_host!r} is not in the bottle's egress.routes "
|
|
||||||
"allowlist. Declare a route for it or remove the request."))
|
|
||||||
if not evaluate_matches(route, request_path, request_method, request_headers):
|
|
||||||
return Decision("block", (
|
|
||||||
f"egress: request {request_method} {request_path!r} does not match any "
|
|
||||||
f"entry in matches for {route.host!r}"))
|
|
||||||
if route.auth_scheme and route.token_env:
|
|
||||||
token = environ.get(route.token_env, "")
|
|
||||||
if not token:
|
|
||||||
return Decision("block", (
|
|
||||||
f"egress: route for {route.host!r} declared auth but env var "
|
|
||||||
f"{route.token_env!r} is unset"))
|
|
||||||
return Decision("forward", inject_authorization=f"{route.auth_scheme} {token}")
|
|
||||||
return Decision("forward")
|
|
||||||
|
|
||||||
|
|
||||||
def decide_git_fetch(routes: typing.Sequence[Route], request_host: str) -> Decision:
|
|
||||||
route = match_route(routes, request_host)
|
|
||||||
if route is not None and route.git_fetch:
|
|
||||||
return Decision("forward")
|
|
||||||
return Decision("block", (
|
|
||||||
"egress: git fetch/clone over HTTPS is not allowed by default; use git-gate "
|
|
||||||
"for declared repos or set egress.routes[].git.fetch=true for explicit "
|
|
||||||
"read-only HTTPS Git access."))
|
|
||||||
@@ -1,349 +0,0 @@
|
|||||||
"""Egress policy schema parsing and serialization (PRD 0017 / 0053)."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import re
|
|
||||||
import typing
|
|
||||||
|
|
||||||
from ...yaml_subset import YamlSubsetError, parse_yaml_subset
|
|
||||||
from .dlp_config import parse_inspect_block
|
|
||||||
from .types import (
|
|
||||||
HEADER_MATCH_TYPES,
|
|
||||||
LOG_BLOCKS,
|
|
||||||
LOG_FULL,
|
|
||||||
LOG_OFF,
|
|
||||||
PATH_MATCH_TYPES,
|
|
||||||
VALID_METHODS,
|
|
||||||
Config,
|
|
||||||
HeaderMatch,
|
|
||||||
MatchEntry,
|
|
||||||
PathMatch,
|
|
||||||
Route,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Parsing
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
def _parse_path_match(idx: int, j: int, raw: object) -> PathMatch:
|
|
||||||
label = f"route[{idx}] matches paths[{j}]"
|
|
||||||
if not isinstance(raw, dict):
|
|
||||||
raise ValueError(f"{label}: must be an object")
|
|
||||||
raw_dict: dict[str, object] = typing.cast(dict[str, object], raw)
|
|
||||||
ptype = raw_dict.get("type", "prefix")
|
|
||||||
if not isinstance(ptype, str) or ptype not in PATH_MATCH_TYPES:
|
|
||||||
raise ValueError(
|
|
||||||
f"{label}: 'type' must be one of {', '.join(PATH_MATCH_TYPES)} "
|
|
||||||
f"(got {ptype!r})"
|
|
||||||
)
|
|
||||||
value = raw_dict.get("value")
|
|
||||||
if not isinstance(value, str) or not value:
|
|
||||||
raise ValueError(f"{label}: 'value' must be a non-empty string")
|
|
||||||
if ptype in ("exact", "prefix") and not value.startswith("/"):
|
|
||||||
raise ValueError(
|
|
||||||
f"{label}: value {value!r} must start with '/' for "
|
|
||||||
f"type {ptype!r}"
|
|
||||||
)
|
|
||||||
compiled: re.Pattern[str] | None = None
|
|
||||||
if ptype == "regex":
|
|
||||||
try:
|
|
||||||
compiled = re.compile(value)
|
|
||||||
except re.error as e:
|
|
||||||
raise ValueError(
|
|
||||||
f"{label}: regex {value!r} failed to compile: {e}"
|
|
||||||
) from e
|
|
||||||
for k in raw_dict:
|
|
||||||
if k not in ("type", "value"):
|
|
||||||
raise ValueError(f"{label}: unknown key {k!r}")
|
|
||||||
return PathMatch(type=ptype, value=value, compiled=compiled)
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_header_match(idx: int, j: int, raw: object) -> HeaderMatch:
|
|
||||||
label = f"route[{idx}] matches headers[{j}]"
|
|
||||||
if not isinstance(raw, dict):
|
|
||||||
raise ValueError(f"{label}: must be an object")
|
|
||||||
raw_dict: dict[str, object] = typing.cast(dict[str, object], raw)
|
|
||||||
name = raw_dict.get("name")
|
|
||||||
if not isinstance(name, str) or not name:
|
|
||||||
raise ValueError(f"{label}: 'name' must be a non-empty string")
|
|
||||||
value = raw_dict.get("value")
|
|
||||||
if not isinstance(value, str):
|
|
||||||
raise ValueError(f"{label}: 'value' must be a string")
|
|
||||||
htype = raw_dict.get("type", "exact")
|
|
||||||
if not isinstance(htype, str) or htype not in HEADER_MATCH_TYPES:
|
|
||||||
raise ValueError(
|
|
||||||
f"{label}: 'type' must be one of {', '.join(HEADER_MATCH_TYPES)} "
|
|
||||||
f"(got {htype!r})"
|
|
||||||
)
|
|
||||||
compiled: re.Pattern[str] | None = None
|
|
||||||
if htype == "regex":
|
|
||||||
try:
|
|
||||||
compiled = re.compile(value)
|
|
||||||
except re.error as e:
|
|
||||||
raise ValueError(
|
|
||||||
f"{label}: regex {value!r} failed to compile: {e}"
|
|
||||||
) from e
|
|
||||||
for k in raw_dict:
|
|
||||||
if k not in ("name", "value", "type"):
|
|
||||||
raise ValueError(f"{label}: unknown key {k!r}")
|
|
||||||
return HeaderMatch(name=name, value=value, type=htype, compiled=compiled)
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_match_entry(idx: int, k: int, raw: object) -> MatchEntry:
|
|
||||||
label = f"route[{idx}] matches[{k}]"
|
|
||||||
if not isinstance(raw, dict):
|
|
||||||
raise ValueError(f"{label}: must be an object")
|
|
||||||
raw_dict: dict[str, object] = typing.cast(dict[str, object], raw)
|
|
||||||
|
|
||||||
paths: tuple[PathMatch, ...] = ()
|
|
||||||
paths_raw = raw_dict.get("paths")
|
|
||||||
if paths_raw is not None:
|
|
||||||
if not isinstance(paths_raw, list):
|
|
||||||
raise ValueError(f"{label}: 'paths' must be a list")
|
|
||||||
paths_list = typing.cast(list[object], paths_raw)
|
|
||||||
paths = tuple(_parse_path_match(idx, j, p) for j, p in enumerate(paths_list))
|
|
||||||
|
|
||||||
methods: tuple[str, ...] = ()
|
|
||||||
methods_raw = raw_dict.get("methods")
|
|
||||||
if methods_raw is not None:
|
|
||||||
if not isinstance(methods_raw, list):
|
|
||||||
raise ValueError(f"{label}: 'methods' must be a list")
|
|
||||||
methods_list = typing.cast(list[object], methods_raw)
|
|
||||||
normalised: list[str] = []
|
|
||||||
for j, m in enumerate(methods_list):
|
|
||||||
if not isinstance(m, str):
|
|
||||||
raise ValueError(f"{label}: methods[{j}] must be a string")
|
|
||||||
upper = m.upper()
|
|
||||||
if upper not in VALID_METHODS:
|
|
||||||
raise ValueError(
|
|
||||||
f"{label}: methods[{j}] {m!r} is not a valid HTTP method"
|
|
||||||
)
|
|
||||||
normalised.append(upper)
|
|
||||||
methods = tuple(normalised)
|
|
||||||
|
|
||||||
headers: tuple[HeaderMatch, ...] = ()
|
|
||||||
headers_raw = raw_dict.get("headers")
|
|
||||||
if headers_raw is not None:
|
|
||||||
if not isinstance(headers_raw, list):
|
|
||||||
raise ValueError(f"{label}: 'headers' must be a list")
|
|
||||||
headers_list = typing.cast(list[object], headers_raw)
|
|
||||||
headers = tuple(
|
|
||||||
_parse_header_match(idx, j, h) for j, h in enumerate(headers_list)
|
|
||||||
)
|
|
||||||
|
|
||||||
for key in raw_dict:
|
|
||||||
if key not in ("paths", "methods", "headers"):
|
|
||||||
raise ValueError(f"{label}: unknown key {key!r}")
|
|
||||||
|
|
||||||
return MatchEntry(paths=paths, methods=methods, headers=headers)
|
|
||||||
|
|
||||||
|
|
||||||
def parse_routes(payload: object) -> tuple[Route, ...]:
|
|
||||||
if not isinstance(payload, dict):
|
|
||||||
raise ValueError("routes payload: top-level must be an object")
|
|
||||||
payload_dict: dict[str, object] = typing.cast(dict[str, object], payload)
|
|
||||||
raw: object = payload_dict.get("routes")
|
|
||||||
if not isinstance(raw, list):
|
|
||||||
raise ValueError("routes payload: 'routes' must be a list")
|
|
||||||
raw_list: list[object] = typing.cast(list[object], raw)
|
|
||||||
out: list[Route] = []
|
|
||||||
for i, r in enumerate(raw_list):
|
|
||||||
out.append(_parse_one(i, r))
|
|
||||||
return tuple(out)
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_one(idx: int, raw: object) -> Route:
|
|
||||||
label = f"route[{idx}]"
|
|
||||||
if not isinstance(raw, dict):
|
|
||||||
raise ValueError(f"{label}: must be an object (got {type(raw).__name__})")
|
|
||||||
raw_dict: dict[str, object] = typing.cast(dict[str, object], raw)
|
|
||||||
host: object = raw_dict.get("host")
|
|
||||||
if not isinstance(host, str) or not host:
|
|
||||||
raise ValueError(f"{label}: 'host' must be a non-empty string")
|
|
||||||
legacy_flat = "inspect" not in raw_dict
|
|
||||||
inspect_raw = raw_dict.get("inspect", {})
|
|
||||||
if inspect_raw is False:
|
|
||||||
inspect = False
|
|
||||||
settings: dict[str, object] = {}
|
|
||||||
elif isinstance(inspect_raw, dict):
|
|
||||||
inspect = True
|
|
||||||
settings = (
|
|
||||||
{k: v for k, v in raw_dict.items() if k != "host"}
|
|
||||||
if legacy_flat
|
|
||||||
else typing.cast(dict[str, object], inspect_raw)
|
|
||||||
)
|
|
||||||
legacy_dlp = settings.pop("dlp", None)
|
|
||||||
if isinstance(legacy_dlp, dict):
|
|
||||||
settings.update(typing.cast(dict[str, object], legacy_dlp))
|
|
||||||
elif legacy_dlp is not None:
|
|
||||||
raise ValueError(
|
|
||||||
f"{label} ({host}): legacy 'dlp' must be an object"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
raise ValueError(f"{label} ({host}): 'inspect' must be false or an object")
|
|
||||||
|
|
||||||
# matches
|
|
||||||
matches: tuple[MatchEntry, ...] = ()
|
|
||||||
matches_raw = settings.get("matches")
|
|
||||||
if matches_raw is not None:
|
|
||||||
if not isinstance(matches_raw, list):
|
|
||||||
raise ValueError(f"{label} ({host}): 'matches' must be a list")
|
|
||||||
matches_list = typing.cast(list[object], matches_raw)
|
|
||||||
matches = tuple(
|
|
||||||
_parse_match_entry(idx, k, m) for k, m in enumerate(matches_list)
|
|
||||||
)
|
|
||||||
|
|
||||||
# auth (unchanged wire format)
|
|
||||||
auth_scheme: object = settings.get("auth_scheme", "")
|
|
||||||
token_env: object = settings.get("token_env", "")
|
|
||||||
if not isinstance(auth_scheme, str):
|
|
||||||
raise ValueError(f"{label} ({host}): 'auth_scheme' must be a string")
|
|
||||||
if not isinstance(token_env, str):
|
|
||||||
raise ValueError(f"{label} ({host}): 'token_env' must be a string")
|
|
||||||
if bool(auth_scheme) != bool(token_env):
|
|
||||||
raise ValueError(
|
|
||||||
f"{label} ({host}): 'auth_scheme' and 'token_env' must be both "
|
|
||||||
f"set or both empty (got auth_scheme={auth_scheme!r}, "
|
|
||||||
f"token_env={token_env!r})"
|
|
||||||
)
|
|
||||||
|
|
||||||
# git-over-HTTPS policy
|
|
||||||
git_fetch = False
|
|
||||||
git_raw = settings.get("git")
|
|
||||||
if git_raw is not None:
|
|
||||||
if not isinstance(git_raw, dict):
|
|
||||||
raise ValueError(f"{label} ({host}): 'git' must be an object")
|
|
||||||
git_dict: dict[str, object] = typing.cast(dict[str, object], git_raw)
|
|
||||||
fetch_raw = git_dict.get("fetch", False)
|
|
||||||
if fetch_raw is True or fetch_raw is False:
|
|
||||||
git_fetch = fetch_raw
|
|
||||||
else:
|
|
||||||
raise ValueError(f"{label} ({host}): 'git.fetch' must be a boolean")
|
|
||||||
for k in git_dict:
|
|
||||||
if k != "fetch":
|
|
||||||
raise ValueError(
|
|
||||||
f"{label} ({host}): git has unknown key {k!r}; "
|
|
||||||
"accepted key is 'fetch'"
|
|
||||||
)
|
|
||||||
|
|
||||||
# dlp detectors
|
|
||||||
outbound_detectors, inbound_detectors, outbound_on_match = parse_inspect_block(
|
|
||||||
idx, host, settings,
|
|
||||||
)
|
|
||||||
|
|
||||||
preserve_auth_raw = settings.get("preserve_auth", False)
|
|
||||||
if preserve_auth_raw is not True and preserve_auth_raw is not False:
|
|
||||||
raise ValueError(
|
|
||||||
f"{label} ({host}): 'preserve_auth' must be a boolean"
|
|
||||||
)
|
|
||||||
preserve_auth: bool = preserve_auth_raw
|
|
||||||
|
|
||||||
for k in settings:
|
|
||||||
if k not in (
|
|
||||||
"matches", "auth_scheme", "token_env", "git", "preserve_auth",
|
|
||||||
"outbound_detectors", "inbound_detectors", "outbound_on_match",
|
|
||||||
):
|
|
||||||
raise ValueError(
|
|
||||||
f"{label} ({host}): inspect has unknown key {k!r}"
|
|
||||||
)
|
|
||||||
for k in raw_dict:
|
|
||||||
if not legacy_flat and k not in ("host", "inspect"):
|
|
||||||
raise ValueError(
|
|
||||||
f"{label} ({host}): unknown key {k!r}; accepted keys "
|
|
||||||
f"are 'host' and 'inspect'"
|
|
||||||
)
|
|
||||||
|
|
||||||
return Route(
|
|
||||||
host=host,
|
|
||||||
matches=matches,
|
|
||||||
auth_scheme=auth_scheme,
|
|
||||||
token_env=token_env,
|
|
||||||
git_fetch=git_fetch,
|
|
||||||
outbound_detectors=outbound_detectors,
|
|
||||||
inbound_detectors=inbound_detectors,
|
|
||||||
outbound_on_match=outbound_on_match,
|
|
||||||
preserve_auth=preserve_auth,
|
|
||||||
inspect=inspect,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _path_match_to_dict(pm: PathMatch) -> dict[str, object]:
|
|
||||||
d: dict[str, object] = {"value": pm.value}
|
|
||||||
if pm.type != "prefix":
|
|
||||||
d["type"] = pm.type
|
|
||||||
return d
|
|
||||||
|
|
||||||
|
|
||||||
def _header_match_to_dict(hm: HeaderMatch) -> dict[str, object]:
|
|
||||||
d: dict[str, object] = {"name": hm.name, "value": hm.value}
|
|
||||||
if hm.type != "exact":
|
|
||||||
d["type"] = hm.type
|
|
||||||
return d
|
|
||||||
|
|
||||||
|
|
||||||
def _match_entry_to_dict(me: MatchEntry) -> dict[str, object]:
|
|
||||||
d: dict[str, object] = {}
|
|
||||||
if me.paths:
|
|
||||||
d["paths"] = [_path_match_to_dict(p) for p in me.paths]
|
|
||||||
if me.methods:
|
|
||||||
d["methods"] = list(me.methods)
|
|
||||||
if me.headers:
|
|
||||||
d["headers"] = [_header_match_to_dict(h) for h in me.headers]
|
|
||||||
return d
|
|
||||||
|
|
||||||
|
|
||||||
def route_to_yaml_dict(r: Route) -> dict[str, object]:
|
|
||||||
"""Serialize a Route to YAML-schema-compatible dict.
|
|
||||||
|
|
||||||
Uses the same field names the YAML parser accepts, so the output
|
|
||||||
can be round-tripped directly into an `allow` or `egress-block`
|
|
||||||
proposal without translation. Fields that are empty/default are
|
|
||||||
omitted so the agent doesn't copy irrelevant keys."""
|
|
||||||
d: dict[str, object] = {"host": r.host}
|
|
||||||
if not r.inspect:
|
|
||||||
d["inspect"] = False
|
|
||||||
return d
|
|
||||||
inspected: dict[str, object] = {}
|
|
||||||
if r.auth_scheme:
|
|
||||||
inspected["auth_scheme"] = r.auth_scheme
|
|
||||||
inspected["token_env"] = r.token_env
|
|
||||||
if r.matches:
|
|
||||||
inspected["matches"] = [_match_entry_to_dict(m) for m in r.matches]
|
|
||||||
if r.git_fetch:
|
|
||||||
inspected["git"] = {"fetch": True}
|
|
||||||
if r.outbound_detectors is not None:
|
|
||||||
inspected["outbound_detectors"] = list(r.outbound_detectors)
|
|
||||||
if r.inbound_detectors is not None:
|
|
||||||
inspected["inbound_detectors"] = list(r.inbound_detectors)
|
|
||||||
if r.outbound_on_match:
|
|
||||||
inspected["outbound_on_match"] = r.outbound_on_match
|
|
||||||
if r.preserve_auth:
|
|
||||||
inspected["preserve_auth"] = True
|
|
||||||
if inspected:
|
|
||||||
d["inspect"] = inspected
|
|
||||||
return d
|
|
||||||
|
|
||||||
|
|
||||||
def parse_config(payload: object) -> "Config":
|
|
||||||
"""Parse a full egress config payload (top-level log level + routes)."""
|
|
||||||
if not isinstance(payload, dict):
|
|
||||||
raise ValueError("routes payload: top-level must be an object")
|
|
||||||
payload_dict: dict[str, object] = typing.cast(dict[str, object], payload)
|
|
||||||
|
|
||||||
log_raw: object = payload_dict.get("log", LOG_OFF)
|
|
||||||
if log_raw is True or log_raw is False or not isinstance(log_raw, int) \
|
|
||||||
or log_raw not in (LOG_OFF, LOG_BLOCKS, LOG_FULL):
|
|
||||||
raise ValueError(
|
|
||||||
f"routes payload: 'log' must be {LOG_OFF}, {LOG_BLOCKS}, or {LOG_FULL}"
|
|
||||||
)
|
|
||||||
|
|
||||||
routes = parse_routes(payload)
|
|
||||||
return Config(routes=routes, log=log_raw)
|
|
||||||
|
|
||||||
|
|
||||||
def load_config(text: str) -> "Config":
|
|
||||||
"""Parse YAML text → Config (routes + log flag)."""
|
|
||||||
try:
|
|
||||||
payload = parse_yaml_subset(text)
|
|
||||||
except YamlSubsetError as e:
|
|
||||||
raise ValueError(f"routes payload: invalid YAML: {e}") from e
|
|
||||||
return parse_config(payload)
|
|
||||||
@@ -1,81 +0,0 @@
|
|||||||
"""Shared egress policy value objects.
|
|
||||||
|
|
||||||
Kept dependency-free so the schema parser, matcher, DLP scanner, and addon
|
|
||||||
adapter can use the same immutable public shapes without importing each other.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import re
|
|
||||||
from dataclasses import dataclass
|
|
||||||
|
|
||||||
|
|
||||||
PATH_MATCH_TYPES = ("exact", "prefix", "regex")
|
|
||||||
HEADER_MATCH_TYPES = ("exact", "regex")
|
|
||||||
VALID_METHODS = frozenset({
|
|
||||||
"GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "TRACE",
|
|
||||||
"CONNECT",
|
|
||||||
})
|
|
||||||
|
|
||||||
LOG_OFF = 0
|
|
||||||
LOG_BLOCKS = 1
|
|
||||||
LOG_FULL = 2
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class PathMatch:
|
|
||||||
type: str
|
|
||||||
value: str
|
|
||||||
compiled: re.Pattern[str] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class HeaderMatch:
|
|
||||||
name: str
|
|
||||||
value: str
|
|
||||||
type: str = "exact"
|
|
||||||
compiled: re.Pattern[str] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class MatchEntry:
|
|
||||||
paths: tuple[PathMatch, ...] = ()
|
|
||||||
methods: tuple[str, ...] = ()
|
|
||||||
headers: tuple[HeaderMatch, ...] = ()
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class Route:
|
|
||||||
host: str
|
|
||||||
matches: tuple[MatchEntry, ...] = ()
|
|
||||||
auth_scheme: str = ""
|
|
||||||
token_env: str = ""
|
|
||||||
git_fetch: bool = False
|
|
||||||
outbound_detectors: tuple[str, ...] | None = None
|
|
||||||
inbound_detectors: tuple[str, ...] | None = None
|
|
||||||
outbound_on_match: str = ""
|
|
||||||
preserve_auth: bool = False
|
|
||||||
inspect: bool = True
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class Config:
|
|
||||||
routes: tuple[Route, ...]
|
|
||||||
log: int = LOG_OFF
|
|
||||||
deny_reason: str = ""
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class Decision:
|
|
||||||
action: str
|
|
||||||
reason: str = ""
|
|
||||||
inject_authorization: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class ScanResult:
|
|
||||||
severity: str
|
|
||||||
reason: str
|
|
||||||
location: str = ""
|
|
||||||
context: str = ""
|
|
||||||
matched: str = ""
|
|
||||||
@@ -252,27 +252,6 @@ cat > "$refs_file"
|
|||||||
|
|
||||||
zero=0000000000000000000000000000000000000000
|
zero=0000000000000000000000000000000000000000
|
||||||
|
|
||||||
# Phase 0: reject Gitea AGit review refs before scanning or forwarding.
|
|
||||||
# A push to refs/for/*, refs/draft/*, or refs/for-review/* asks Gitea to
|
|
||||||
# open a pull request backed by a server-managed refs/pull/<n>/head rather
|
|
||||||
# than an ordinary refs/heads/* branch. That breaks the git-gate workflow:
|
|
||||||
# follow-up commits can't be pushed back through the branch, and Gitea
|
|
||||||
# rejects later direct updates to the generated review ref. Fail the whole
|
|
||||||
# push here (before any gitleaks scan or upstream forward) so the caller
|
|
||||||
# pushes a real branch and opens the PR against it instead. Deletions
|
|
||||||
# (new == zero) stay allowed so stale AGit refs can still be cleaned up.
|
|
||||||
while IFS=' ' read -r old new ref; do
|
|
||||||
[ -z "$ref" ] && continue
|
|
||||||
[ "$new" = "$zero" ] && continue
|
|
||||||
case "$ref" in
|
|
||||||
refs/for/*|refs/draft/*|refs/for-review/*)
|
|
||||||
echo "git-gate: refusing AGit review ref $ref" >&2
|
|
||||||
echo "git-gate: push to refs/heads/<branch> and open a branch-backed pull request instead" >&2
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done < "$refs_file"
|
|
||||||
|
|
||||||
supervise_gitleaks_allow() {
|
supervise_gitleaks_allow() {
|
||||||
log_opts=$1
|
log_opts=$1
|
||||||
ref=$2
|
ref=$2
|
||||||
|
|||||||
@@ -58,9 +58,9 @@ import typing
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from bot_bottle.constants import IDENTITY_HEADER
|
from bot_bottle.constants import IDENTITY_HEADER
|
||||||
from bot_bottle.gateway.egress.context import resolve_client_context
|
from bot_bottle.gateway.egress.addon_core import (
|
||||||
from bot_bottle.gateway.egress.schema import load_config, route_to_yaml_dict
|
LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict,
|
||||||
from bot_bottle.gateway.egress.types import LOG_OFF
|
)
|
||||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||||
from bot_bottle.supervisor import types as _sv
|
from bot_bottle.supervisor import types as _sv
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ import urllib.request
|
|||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from ..log import debug
|
|
||||||
from ..orchestrator_auth import ROLE_CLI
|
from ..orchestrator_auth import ROLE_CLI
|
||||||
from ..trust_domain import CONTROL_PLANE
|
from ..trust_domain import CONTROL_PLANE
|
||||||
from .server import ORCHESTRATOR_AUTH_HEADER
|
from .server import ORCHESTRATOR_AUTH_HEADER
|
||||||
@@ -54,23 +53,6 @@ class RegisteredBottle:
|
|||||||
env_var_secret: str = ""
|
env_var_secret: str = ""
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class BackendProbeFailure:
|
|
||||||
"""Safe diagnostic for an optional backend discovery probe."""
|
|
||||||
|
|
||||||
backend: str
|
|
||||||
error_type: str
|
|
||||||
|
|
||||||
|
|
||||||
def _probe_failure(backend: str, exc: BaseException) -> BackendProbeFailure:
|
|
||||||
failure = BackendProbeFailure(backend, type(exc).__name__)
|
|
||||||
debug(
|
|
||||||
"orchestrator discovery probe unavailable",
|
|
||||||
context={"backend": failure.backend, "error_type": failure.error_type},
|
|
||||||
)
|
|
||||||
return failure
|
|
||||||
|
|
||||||
|
|
||||||
class OrchestratorClient:
|
class OrchestratorClient:
|
||||||
"""Trusted host-side client for the orchestrator control plane.
|
"""Trusted host-side client for the orchestrator control plane.
|
||||||
|
|
||||||
@@ -263,41 +245,32 @@ def discover_orchestrator_url(*, timeout: float = 2.0) -> str:
|
|||||||
orchestrator TAP. Returns the first that answers `/health`; raises if none
|
orchestrator TAP. Returns the first that answers `/health`; raises if none
|
||||||
do (no orchestrator up — launch a bottle first)."""
|
do (no orchestrator up — launch a bottle first)."""
|
||||||
candidates: list[str] = []
|
candidates: list[str] = []
|
||||||
failures: list[BackendProbeFailure] = []
|
|
||||||
try: # docker: loopback-published control plane
|
try: # docker: loopback-published control plane
|
||||||
from .lifecycle import DEFAULT_PORT as _DOCKER_PORT
|
from .lifecycle import DEFAULT_PORT as _DOCKER_PORT
|
||||||
candidates.append(f"http://127.0.0.1:{_DOCKER_PORT}")
|
candidates.append(f"http://127.0.0.1:{_DOCKER_PORT}")
|
||||||
except Exception as exc: # noqa: BLE001 — backend optional
|
except Exception: # noqa: BLE001 — backend optional
|
||||||
failures.append(_probe_failure("docker", exc))
|
|
||||||
candidates.append("http://127.0.0.1:8099")
|
candidates.append("http://127.0.0.1:8099")
|
||||||
try: # firecracker: infra VM control plane on the orchestrator TAP
|
try: # firecracker: infra VM control plane on the orchestrator TAP
|
||||||
from ..backend.firecracker import netpool
|
from ..backend.firecracker import netpool
|
||||||
from ..backend.firecracker.infra_vm import ORCHESTRATOR_PORT
|
from ..backend.firecracker.infra_vm import ORCHESTRATOR_PORT
|
||||||
candidates.append(
|
candidates.append(
|
||||||
f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}")
|
f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}")
|
||||||
except Exception as exc: # noqa: BLE001 — backend optional / not firecracker
|
except Exception: # noqa: BLE001 — backend optional / not firecracker
|
||||||
failures.append(_probe_failure("firecracker", exc))
|
pass
|
||||||
try: # macOS: orchestrator container on its host-only address
|
try: # macOS: orchestrator container on its host-only address
|
||||||
from ..backend.macos_container.infra import probe_orchestrator_url
|
from ..backend.macos_container.infra import probe_orchestrator_url
|
||||||
url = probe_orchestrator_url()
|
url = probe_orchestrator_url()
|
||||||
if url:
|
if url:
|
||||||
candidates.append(url)
|
candidates.append(url)
|
||||||
except Exception as exc: # noqa: BLE001 — backend optional / not macOS
|
except Exception: # noqa: BLE001 — backend optional / not macOS
|
||||||
failures.append(_probe_failure("macos-container", exc))
|
pass
|
||||||
for url in candidates:
|
for url in candidates:
|
||||||
if OrchestratorClient(url, timeout=timeout).health():
|
if OrchestratorClient(url, timeout=timeout).health():
|
||||||
return url
|
return url
|
||||||
detail = ""
|
|
||||||
if failures:
|
|
||||||
detail = "; optional probes unavailable: " + ", ".join(
|
|
||||||
f"{failure.backend} ({failure.error_type})" for failure in failures
|
|
||||||
)
|
|
||||||
raise OrchestratorClientError(
|
raise OrchestratorClientError(
|
||||||
"no running orchestrator control plane found (tried "
|
"no running orchestrator control plane found (tried "
|
||||||
+ ", ".join(candidates)
|
+ ", ".join(candidates)
|
||||||
+ ")"
|
+ "); launch a bottle first"
|
||||||
+ detail
|
|
||||||
+ "; launch a bottle first"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from ..log import debug
|
|
||||||
from .client import OrchestratorClient, OrchestratorClientError
|
from .client import OrchestratorClient, OrchestratorClientError
|
||||||
|
|
||||||
|
|
||||||
@@ -28,14 +27,7 @@ def reprovision_bottles(
|
|||||||
try:
|
try:
|
||||||
if client.reprovision_gateway(bottle_id, secret):
|
if client.reprovision_gateway(bottle_id, secret):
|
||||||
restored += 1
|
restored += 1
|
||||||
except OrchestratorClientError as exc:
|
except OrchestratorClientError:
|
||||||
debug(
|
|
||||||
"gateway secret reprovision failed; continuing with other bottles",
|
|
||||||
context={
|
|
||||||
"bottle_id": bottle_id,
|
|
||||||
"error_type": type(exc).__name__,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
continue
|
continue
|
||||||
return restored
|
return restored
|
||||||
|
|
||||||
|
|||||||
@@ -57,7 +57,6 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import http.server
|
import http.server
|
||||||
import json
|
import json
|
||||||
import math
|
|
||||||
import os
|
import os
|
||||||
import socketserver
|
import socketserver
|
||||||
import sys
|
import sys
|
||||||
@@ -218,18 +217,13 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches
|
|||||||
raw_ips = data.get("live_source_ips")
|
raw_ips = data.get("live_source_ips")
|
||||||
if not isinstance(raw_ips, list):
|
if not isinstance(raw_ips, list):
|
||||||
return 400, {"error": "live_source_ips (list of strings) is required"}
|
return 400, {"error": "live_source_ips (list of strings) is required"}
|
||||||
if any(not isinstance(ip, str) or not ip for ip in raw_ips):
|
live = [ip for ip in raw_ips if isinstance(ip, str) and ip]
|
||||||
return 400, {"error": "live_source_ips must contain non-empty strings"}
|
|
||||||
live = raw_ips
|
|
||||||
grace = data.get("grace_seconds")
|
grace = data.get("grace_seconds")
|
||||||
kwargs: dict[str, float] = {}
|
kwargs = (
|
||||||
if grace is not None:
|
{"grace_seconds": float(grace)}
|
||||||
if isinstance(grace, bool) or not isinstance(grace, (int, float)):
|
if isinstance(grace, (int, float)) and not isinstance(grace, bool)
|
||||||
return 400, {"error": "grace_seconds must be a non-negative finite number"}
|
else {}
|
||||||
parsed_grace = float(grace)
|
)
|
||||||
if not math.isfinite(parsed_grace) or parsed_grace < 0:
|
|
||||||
return 400, {"error": "grace_seconds must be a non-negative finite number"}
|
|
||||||
kwargs["grace_seconds"] = parsed_grace
|
|
||||||
return 200, {"reaped": orch.reconcile(live, **kwargs)}
|
return 200, {"reaped": orch.reconcile(live, **kwargs)}
|
||||||
|
|
||||||
if method == "POST" and route == "/attribute":
|
if method == "POST" and route == "/attribute":
|
||||||
@@ -379,15 +373,9 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
|||||||
status, payload = dispatch(
|
status, payload = dispatch(
|
||||||
server.orchestrator, method, self.path, body, role=role)
|
server.orchestrator, method, self.path, body, role=role)
|
||||||
except Exception as e: # noqa: BLE001 — the control plane must stay up
|
except Exception as e: # noqa: BLE001 — the control plane must stay up
|
||||||
# Do not echo exception messages to the caller or logs: broker and
|
sys.stderr.write(f"orchestrator: {method} {self.path} failed: {e!r}\n")
|
||||||
# persistence exceptions can contain request data. The operation,
|
|
||||||
# route, and exception type are enough to correlate a traceback.
|
|
||||||
sys.stderr.write(
|
|
||||||
f"orchestrator: {method} {self.path} failed "
|
|
||||||
f"[error_type={type(e).__name__}]\n"
|
|
||||||
)
|
|
||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
status, payload = 500, {"error": "internal error"}
|
status, payload = 500, {"error": f"internal error: {e}"}
|
||||||
data = json.dumps(payload).encode()
|
data = json.dumps(payload).encode()
|
||||||
self.send_response(status)
|
self.send_response(status)
|
||||||
self.send_header("Content-Type", "application/json")
|
self.send_header("Content-Type", "application/json")
|
||||||
|
|||||||
@@ -53,15 +53,6 @@ ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT"
|
|||||||
# the shared gateway's TLS interception, so it must not rotate on restart. See
|
# the shared gateway's TLS interception, so it must not rotate on restart. See
|
||||||
# host_gateway_ca_dir() for why this is a host bind-mount, not a named volume.
|
# host_gateway_ca_dir() for why this is a host bind-mount, not a named volume.
|
||||||
GATEWAY_CA_DIRNAME = "gateway-ca"
|
GATEWAY_CA_DIRNAME = "gateway-ca"
|
||||||
# The host directories holding the gateway's persistent git-gate state — the
|
|
||||||
# per-bottle bare repos (`gateway-git`) and deploy creds (`gateway-creds`).
|
|
||||||
# Bind-mounted into the gateway container at /git and /git-gate/creds so they
|
|
||||||
# survive container recreation; without it a gateway restart drops every
|
|
||||||
# already-running bottle's git-gate state and its agent 404s on fetch/push
|
|
||||||
# (issue #512). Host bind-mounts (not named volumes) for the same reason as the
|
|
||||||
# CA dir — see host_gateway_ca_dir().
|
|
||||||
GATEWAY_GIT_DIRNAME = "gateway-git"
|
|
||||||
GATEWAY_CREDS_DIRNAME = "gateway-creds"
|
|
||||||
|
|
||||||
|
|
||||||
def bot_bottle_root() -> Path:
|
def bot_bottle_root() -> Path:
|
||||||
@@ -106,27 +97,6 @@ def host_gateway_ca_dir() -> Path:
|
|||||||
return ca_dir
|
return ca_dir
|
||||||
|
|
||||||
|
|
||||||
def host_gateway_git_dir() -> Path:
|
|
||||||
"""The directory holding the gateway's persistent per-bottle bare repos,
|
|
||||||
created if missing. Bind-mounted into the gateway container at /git so the
|
|
||||||
repos survive container recreation (issue #512). A host bind-mount under the
|
|
||||||
app-data root, never pruned — same rationale as host_gateway_ca_dir()."""
|
|
||||||
git_dir = bot_bottle_root() / GATEWAY_GIT_DIRNAME
|
|
||||||
git_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
return git_dir
|
|
||||||
|
|
||||||
|
|
||||||
def host_gateway_creds_dir() -> Path:
|
|
||||||
"""The directory holding the gateway's persistent per-bottle git-gate deploy
|
|
||||||
creds, created if missing. Bind-mounted into the gateway container at
|
|
||||||
/git-gate/creds so the creds survive container recreation (issue #512). A
|
|
||||||
host bind-mount under the app-data root — same rationale as
|
|
||||||
host_gateway_ca_dir()."""
|
|
||||||
creds_dir = bot_bottle_root() / GATEWAY_CREDS_DIRNAME
|
|
||||||
creds_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
return creds_dir
|
|
||||||
|
|
||||||
|
|
||||||
def host_signing_key(filename: str) -> str:
|
def host_signing_key(filename: str) -> str:
|
||||||
"""A per-host signing key at `<root>/<filename>`, minted (256-bit, url-safe)
|
"""A per-host signing key at `<root>/<filename>`, minted (256-bit, url-safe)
|
||||||
and persisted 0600 on first use, then reused.
|
and persisted 0600 on first use, then reused.
|
||||||
@@ -173,14 +143,10 @@ __all__ = [
|
|||||||
"ORCHESTRATOR_TOKEN_ENV",
|
"ORCHESTRATOR_TOKEN_ENV",
|
||||||
"ORCHESTRATOR_AUTH_JWT_ENV",
|
"ORCHESTRATOR_AUTH_JWT_ENV",
|
||||||
"GATEWAY_CA_DIRNAME",
|
"GATEWAY_CA_DIRNAME",
|
||||||
"GATEWAY_GIT_DIRNAME",
|
|
||||||
"GATEWAY_CREDS_DIRNAME",
|
|
||||||
"bot_bottle_root",
|
"bot_bottle_root",
|
||||||
"host_db_path",
|
"host_db_path",
|
||||||
"host_db_dir",
|
"host_db_dir",
|
||||||
"host_gateway_ca_dir",
|
"host_gateway_ca_dir",
|
||||||
"host_gateway_git_dir",
|
|
||||||
"host_gateway_creds_dir",
|
|
||||||
"host_signing_key",
|
"host_signing_key",
|
||||||
"host_orchestrator_token",
|
"host_orchestrator_token",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -9,11 +9,8 @@ import difflib
|
|||||||
import hashlib
|
import hashlib
|
||||||
import ipaddress
|
import ipaddress
|
||||||
import os
|
import os
|
||||||
import re
|
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from .log import die
|
|
||||||
|
|
||||||
|
|
||||||
def sha256_hex(content: str) -> str:
|
def sha256_hex(content: str) -> str:
|
||||||
"""Hex SHA-256 of a UTF-8 string."""
|
"""Hex SHA-256 of a UTF-8 string."""
|
||||||
@@ -70,20 +67,3 @@ def expand_tilde(path: str) -> str:
|
|||||||
home = os.environ.get("HOME", "")
|
home = os.environ.get("HOME", "")
|
||||||
return home + path[1:]
|
return home + path[1:]
|
||||||
return path
|
return path
|
||||||
|
|
||||||
|
|
||||||
_SLUG_RE = re.compile(r"[^a-z0-9]+")
|
|
||||||
|
|
||||||
|
|
||||||
def slugify(name: str) -> str:
|
|
||||||
"""Return a portable bottle identifier from a human-readable name.
|
|
||||||
|
|
||||||
This is deliberately a root utility: names are part of the generic CLI
|
|
||||||
and state model, not a Docker container concern.
|
|
||||||
"""
|
|
||||||
if not name:
|
|
||||||
die("slugify: missing name")
|
|
||||||
slug = _SLUG_RE.sub("-", name.lower()).strip("-")
|
|
||||||
if not slug:
|
|
||||||
die(f"name '{name}' produced an empty slug; use alphanumeric characters")
|
|
||||||
return slug
|
|
||||||
|
|||||||
+42
-45
@@ -1,53 +1,50 @@
|
|||||||
# CI
|
# CI
|
||||||
|
|
||||||
## Required pull-request gate
|
The test workflow lives at [`.gitea/workflows/test.yml`](../.gitea/workflows/test.yml).
|
||||||
|
It runs the unit suite plus one integration job per backend
|
||||||
|
(`integration-docker`, `integration-firecracker`, `integration-macos`) on:
|
||||||
|
|
||||||
[`.gitea/workflows/test.yml`](../.gitea/workflows/test.yml) runs the unit
|
- every push to a branch with an open pull request, and
|
||||||
suite, Docker integration suite, combined coverage report, and diff-coverage
|
- every push to `main`.
|
||||||
gate when tested package/build inputs change on a pull request or on `main`.
|
|
||||||
|
|
||||||
The Docker job preflights the backend before discovery. Gitea's `act_runner`
|
`integration-macos` is the exception: it is **advisory**, running only on
|
||||||
runs the job in a container with the host Docker socket, so the test process
|
`workflow_dispatch` (manual dispatch), never on push or pull requests. It targets the
|
||||||
reaches control-plane siblings through the job's Docker network and uses named
|
Apple Container backend on a self-hosted macOS runner (label `macos`,
|
||||||
Docker volumes for orchestrator/CA state the host daemon must mount. The
|
registered in host mode — Apple Container can't run in a Linux container, so it
|
||||||
orchestrator runs the package baked into the image built from the checkout; it
|
can't reuse the `kvm` runner). A single non-redundant laptop must not be able
|
||||||
does not bind the job container's invisible workspace into a sibling container.
|
to block a PR merge, so the job stays out of the `coverage` job's `needs` and
|
||||||
Docker integration jobs share fixed singleton names, so required and manual
|
its coverage never feeds the diff-coverage gate. Because the infra container is
|
||||||
runs use one non-cancelling concurrency group. The shared agent/gateway network
|
a singleton (`bot-bottle-mac-infra`), the job declares a `concurrency` group
|
||||||
has an explicit subnet, which Docker requires for the pinned source IPs used as
|
and tears the container down on exit; keep runner concurrency at 1. See the
|
||||||
the isolation/attribution key.
|
README "macOS Apple Container" CI note for runner provisioning.
|
||||||
|
|
||||||
`scripts.unittest_gate` enforces the Docker job's contract: all 22 integration
|
Each integration job selects its backend via `BOT_BOTTLE_BACKEND` and
|
||||||
tests must execute and none may skip. This includes the real gateway-image,
|
runs a **preflight** (`./cli.py backend status --backend=<name>`) that
|
||||||
control-plane authentication, multitenant policy/token isolation,
|
prints a clear per-check readiness summary and fails the job when the
|
||||||
sandbox-escape, and orphan-network tests. Backend skip decorators remain useful
|
backend is missing — so absent infrastructure is visible at the job level
|
||||||
for local runs, but the CI preflight plus execution-count gate prevents a
|
rather than hidden among per-test `unittest.skip` lines. The skip guards in
|
||||||
missing backend or runner-topology regression from becoming a green job.
|
[`tests/_backend.py`](../tests/_backend.py) gate on the same readiness
|
||||||
|
check (`bot_bottle.backend.has_backend`): backend-agnostic tests use
|
||||||
|
`skip_unless_selected_backend_available()` and run through whichever
|
||||||
|
backend is selected (checking, e.g., Linux + `/dev/kvm` for Firecracker
|
||||||
|
rather than unrelated Docker availability); Docker-implementation tests use
|
||||||
|
`skip_unless_backend("docker")` and no-op under a non-Docker run.
|
||||||
|
|
||||||
Combined unit + Docker coverage is informational globally. Two focused gates
|
A small subset of integration tests skip when running specifically
|
||||||
are enforced:
|
under Gitea Actions (`GITEA_ACTIONS=true`), because `act_runner` runs
|
||||||
|
the job inside a container with the host's `/var/run/docker.sock`
|
||||||
|
mounted in. That topology breaks two assumptions those tests make:
|
||||||
|
|
||||||
- changed executable Python lines must be at least 90% covered; and
|
- networks created via the host daemon aren't always visible to a
|
||||||
- the validated critical security/logic core must remain at least 90% covered.
|
same-process `docker network ls` call from inside the job container,
|
||||||
|
and
|
||||||
|
- ports published by sibling containers land on the host's loopback,
|
||||||
|
not on the job container's `127.0.0.1` — so HTTP probes against
|
||||||
|
`http://127.0.0.1:<host_port>` from inside the job time out.
|
||||||
|
|
||||||
## Privileged pre-release matrix
|
The affected tests (`test_orphan_cleanup.test_create_and_remove`,
|
||||||
|
`test_gateway_image.TestGatewayImage`) still run
|
||||||
[`.gitea/workflows/pre-release-test.yml`](../.gitea/workflows/pre-release-test.yml)
|
locally where the test process and Docker daemon share a host.
|
||||||
is manually dispatched before a release. It repeats unit and Docker integration
|
Making them work in CI is a follow-up: either re-write them to
|
||||||
coverage, then runs:
|
discover container IPs via `docker inspect`, or reconfigure the
|
||||||
|
runner with host networking.
|
||||||
- Firecracker integration on the self-hosted `kvm` runner; and
|
|
||||||
- advisory Apple Container integration on the self-hosted `macos` runner.
|
|
||||||
|
|
||||||
These privileged host-mode runners never execute unreviewed pull-request code
|
|
||||||
automatically. Firecracker coverage is combined in the manual pre-release
|
|
||||||
report; macOS reports advisory coverage in its own job. The macOS infra
|
|
||||||
container is a singleton, so its job uses a concurrency group and always tears
|
|
||||||
the service down.
|
|
||||||
|
|
||||||
## Scheduled canary
|
|
||||||
|
|
||||||
[`.gitea/workflows/canaries.yml`](../.gitea/workflows/canaries.yml) runs weekly
|
|
||||||
and on manual dispatch. It verifies the pinned gitleaks release URL, checksum,
|
|
||||||
archive shape, and executable. The same unittest execution gate requires at
|
|
||||||
least one executed canary and rejects skips.
|
|
||||||
|
|||||||
@@ -34,13 +34,12 @@ a regression (Goodhart's law).
|
|||||||
Coverage is **risk-weighted**, measured over the **combined unit +
|
Coverage is **risk-weighted**, measured over the **combined unit +
|
||||||
integration** suites, with three rules:
|
integration** suites, with three rules:
|
||||||
|
|
||||||
1. **Critical modules must remain ≥ 90%.** The curated security/logic core
|
1. **Critical modules target ≥ 90%.** The security/logic core —
|
||||||
covers the host and gateway egress policy, manifest trust boundary,
|
`egress_addon{,_core}.py`, `dlp_detectors.py`, `egress.py`,
|
||||||
git-gate enforcement, supervise protocol/server, YAML parser, and bottle
|
`manifest*.py`, `git_gate.py`, `git_http_backend.py`, `supervise.py`,
|
||||||
state. The concrete module list lives in `scripts/critical-modules.txt`;
|
`yaml_subset.py`, `bottle_state.py` — is Docker-independent and
|
||||||
`scripts/critical_modules.py` rejects stale or ambiguous entries before
|
unit-testable, so it carries the high bar. We ratchet toward 90% as
|
||||||
Coverage.py can silently ignore them. These modules are unit-testable, so
|
these modules are touched; new gaps in them are not acceptable.
|
||||||
CI enforces the aggregate minimum independently of diff coverage.
|
|
||||||
|
|
||||||
2. **Subprocess/backend orchestration is covered by the integration
|
2. **Subprocess/backend orchestration is covered by the integration
|
||||||
suite, not omitted.** `scripts/coverage.sh` runs unit + integration
|
suite, not omitted.** `scripts/coverage.sh` runs unit + integration
|
||||||
@@ -83,9 +82,6 @@ omit list.
|
|||||||
(critical-module standard + diff coverage) are Docker-independent.
|
(critical-module standard + diff coverage) are Docker-independent.
|
||||||
- "We're at N%" is now a curated figure; outsiders should read the
|
- "We're at N%" is now a curated figure; outsiders should read the
|
||||||
policy, not just the badge.
|
policy, not just the badge.
|
||||||
- A rename or removal in the curated list fails CI. Updating the list is an
|
|
||||||
explicit review of where the security-critical behavior moved, not a way to
|
|
||||||
improve the percentage by omission.
|
|
||||||
|
|
||||||
## Links
|
## Links
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,350 @@
|
|||||||
|
# PRD prd-new: Canonical tamper-evident audit-event schema and local query contract
|
||||||
|
|
||||||
|
- **Status:** Draft
|
||||||
|
- **Author:** didericis-claude
|
||||||
|
- **Created:** 2026-07-26
|
||||||
|
- **Issue:** #487
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
bot-bottle already emits security- and provenance-relevant events from
|
||||||
|
several producers — supervise operator decisions (PRD 0013's
|
||||||
|
`AuditStore`), egress allow/block enforcement, git-gate push decisions,
|
||||||
|
control-plane token minting, and (next) host-controller lifecycle
|
||||||
|
transitions — but each writes its own shape to its own sink. There is no
|
||||||
|
shared envelope, no tamper-evidence, and no single place to search. Local
|
||||||
|
incident reconstruction means grepping several stores that don't agree on
|
||||||
|
field names, timestamps, or how a bottled agent is identified.
|
||||||
|
|
||||||
|
This PRD defines **one canonical audit-event contract** every producer
|
||||||
|
emits into:
|
||||||
|
|
||||||
|
1. A **versioned envelope** — schema version, event id, event type,
|
||||||
|
monotonic + wall-clock timestamps, and an explicit **trust boundary**
|
||||||
|
between host-supplied and agent-claimed fields.
|
||||||
|
2. **Canonical JSON serialization + a per-writer hash chain**, so any
|
||||||
|
deletion or edit of a past record breaks the chain and is detectable
|
||||||
|
offline.
|
||||||
|
3. An **append-only JSONL journal as the source of truth**, with a
|
||||||
|
**rebuildable SQLite index** for local search — no paid platform, no
|
||||||
|
network dependency.
|
||||||
|
4. An **initial event registry** covering lifecycle, decision, egress,
|
||||||
|
auth, and forge events.
|
||||||
|
|
||||||
|
It is explicitly scheduled to land **immediately after the host
|
||||||
|
controller (#468)** so the host controller's lifecycle transitions are the
|
||||||
|
first producer wired onto the new contract (per the directive on #487).
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
Audit infrastructure is fragmented across #468, #324, and #480 with no
|
||||||
|
shared schema. Concretely:
|
||||||
|
|
||||||
|
- **No shared envelope.** `supervise_audit_entries` (PRD 0013) has
|
||||||
|
`timestamp, bottle_slug, component, operator_action, ...`. The egress
|
||||||
|
proxy and git-gate log their own ad-hoc lines. There is no common
|
||||||
|
`event_id`, `event_type`, or version, so cross-producer correlation
|
||||||
|
("what did bottled agent X do between its start and this rejected push?") is
|
||||||
|
manual and lossy.
|
||||||
|
- **No tamper-evidence.** The audit store is a plain SQLite table. Anyone
|
||||||
|
who can write the DB can delete or rewrite a row and leave no trace.
|
||||||
|
Audit that an attacker (or a buggy agent) can silently rewrite is not
|
||||||
|
audit.
|
||||||
|
- **Trusted and untrusted data are mixed.** A bottled agent is attributed by
|
||||||
|
**source IP → slug** at the gateway (host-supplied, trustworthy). An
|
||||||
|
agent can also *claim* things about itself in a tool call
|
||||||
|
(agent-claimed, adversarial). Today nothing in the record marks which is
|
||||||
|
which, so a reader can be misled by an agent-supplied field that looks
|
||||||
|
authoritative.
|
||||||
|
- **No local search.** Reconstructing an incident means reading multiple
|
||||||
|
sinks with different schemas. There is no query contract and no promise
|
||||||
|
that the index can be rebuilt from the journal if it drifts or is lost.
|
||||||
|
- **No redaction rule.** Nothing prohibits a producer from writing a raw
|
||||||
|
token or secret into an audit record, which would turn the audit log
|
||||||
|
itself into a credential store.
|
||||||
|
|
||||||
|
## Goals / Success Criteria
|
||||||
|
|
||||||
|
- A single `AuditEvent` envelope type, versioned, that every producer
|
||||||
|
emits. Fields are split into a **`trusted`** block (host-supplied:
|
||||||
|
bottled-agent slug from source-IP attribution, host wall-clock, producer
|
||||||
|
identity) and an **`untrusted`** block (anything the agent or a remote
|
||||||
|
claimed), and the split is structural, not a convention.
|
||||||
|
- **Canonical serialization** (`sort_keys`, `(",", ":")` separators,
|
||||||
|
UTF-8, `ensure_ascii=False`) is defined once and reused, so the same
|
||||||
|
logical event always hashes identically across producers and hosts.
|
||||||
|
- Each writer maintains a **hash chain**: `hash = sha256(prev_hash ||
|
||||||
|
canonical(event))`. Deleting or editing any past record breaks every
|
||||||
|
subsequent link; a standalone verifier detects the break offline with no
|
||||||
|
secret material.
|
||||||
|
- The **JSONL journal is the source of truth**; the **SQLite index is
|
||||||
|
fully rebuildable** from it (`audit rebuild` reconstructs the DB and
|
||||||
|
re-verifies the chain).
|
||||||
|
- **Local query** works with no paid platform and no egress: filter by
|
||||||
|
bottled agent, event type, time range, and producer, and follow a bottled
|
||||||
|
agent's events in order.
|
||||||
|
- A **redaction rule** is enforced at the envelope boundary: known
|
||||||
|
credential-shaped fields are rejected/redacted before a record is
|
||||||
|
written; the writer refuses raw secrets rather than storing them.
|
||||||
|
- The **host controller (#468)** emits `lifecycle.*` events through this
|
||||||
|
contract as the first consumer; existing supervise/egress producers are
|
||||||
|
migrated behind the same envelope without changing operator-facing
|
||||||
|
behavior.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- **Cross-host aggregation / shipping.** This PRD makes each host's journal
|
||||||
|
canonical and correlatable *by construction* (stable ids, hash chain),
|
||||||
|
but the transport that merges multiple hosts into one timeline is a
|
||||||
|
follow-up (#324). The schema is designed so that merge is a later append,
|
||||||
|
not a reformat.
|
||||||
|
- **Cryptographic signing / external anchoring.** Hash-chaining gives
|
||||||
|
tamper-**evidence** (you can detect edits), not tamper-**resistance**
|
||||||
|
against an attacker who can rewrite the whole chain. Per-writer signing
|
||||||
|
keys and periodic external anchoring are a follow-up; the chain-head hash
|
||||||
|
is the seam they attach to.
|
||||||
|
- **Real-time alerting / SIEM rules.** Query is local and pull-based here.
|
||||||
|
- **Retention / rotation policy.** Journal rotation and TTL are operator
|
||||||
|
policy, tracked separately; the format must survive rotation (chain head
|
||||||
|
carried across segments) but this PRD does not set the schedule.
|
||||||
|
- **Replacing PRD 0013's operator queue.** The supervise proposal/response
|
||||||
|
queue is unchanged; only its terminal *audit* record is re-emitted onto
|
||||||
|
the new envelope.
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
### The envelope
|
||||||
|
|
||||||
|
One dataclass, `AuditEvent`, serialized to a JSON object with a small,
|
||||||
|
stable top level:
|
||||||
|
|
||||||
|
```
|
||||||
|
{
|
||||||
|
"v": 1, // schema version — bumped only on a breaking change
|
||||||
|
"id": "<uuid4>", // globally unique event id
|
||||||
|
"type": "egress.decision", // dotted event type from the registry
|
||||||
|
|
||||||
|
// --- chain / ordering (structural, host-owned) ---
|
||||||
|
"epoch": 7, // writer-boot counter, bumped once per host-controller (writer) start
|
||||||
|
"seq": 1287, // monotonic sequence within this epoch (gap-detectable)
|
||||||
|
"prev": "<hex>", // hash of the previous record in the chain ("" for genesis)
|
||||||
|
"hash": "<hex>", // sha256(prev + canonical(this event with hash=""))
|
||||||
|
|
||||||
|
// --- trusted: everything the *host* established; authoritative ---
|
||||||
|
"trusted": {
|
||||||
|
"producer": "host-controller", // which host component wrote this
|
||||||
|
"host": "mac-studio-1",
|
||||||
|
"bottled_agent": "amber-fox-12", // slug from source-IP attribution (null for host-level events)
|
||||||
|
"ts_wall": "2026-07-26T18:22:04.113Z", // host wall-clock, RFC3339 UTC
|
||||||
|
"ts_mono": 90142.55 // host monotonic secs since this epoch's boot (intra-epoch ordering only)
|
||||||
|
},
|
||||||
|
|
||||||
|
// --- untrusted: anything the agent or a remote claimed; never authoritative ---
|
||||||
|
"untrusted": {
|
||||||
|
"reason": "npm install needs registry.npmjs.org",
|
||||||
|
"target": "registry.npmjs.org:443"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The **`trusted` / `untrusted` split is the core invariant.** A producer may
|
||||||
|
only place a field in `trusted` if the *host* established it: the
|
||||||
|
source-IP → `bottled_agent` slug attribution, the host's own clock
|
||||||
|
(`ts_wall`/`ts_mono`), and the producer's own identity. **`producer` and
|
||||||
|
`ts_*` live inside `trusted` on purpose** — they are host-supplied, so
|
||||||
|
grouping them there (rather than as loose top-level fields) keeps the
|
||||||
|
"authoritative ⇔ inside `trusted`" rule structural, with nothing
|
||||||
|
host-established leaking outside it. Everything an agent or a remote said
|
||||||
|
goes in `untrusted`. A reader (or a future policy engine) can therefore
|
||||||
|
trust `trusted.bottled_agent` for attribution and treat `untrusted.*` as
|
||||||
|
adversarial claims — the distinction the current stores lack.
|
||||||
|
|
||||||
|
Only the small structural set — `v`, `id`, `type`, `epoch`, `seq`, `prev`,
|
||||||
|
`hash` — sits at the top level; it is host-owned too, but it is chain
|
||||||
|
metadata rather than event data, so it stays out of the `trusted` body to
|
||||||
|
keep that body purely about *what happened*.
|
||||||
|
|
||||||
|
### Canonical serialization + hash chain
|
||||||
|
|
||||||
|
Serialization is defined once (extends the existing `sha256_hex` /
|
||||||
|
`util.py` helpers):
|
||||||
|
|
||||||
|
```
|
||||||
|
def canonical(event: dict) -> str:
|
||||||
|
return json.dumps(event, sort_keys=True, separators=(",", ":"),
|
||||||
|
ensure_ascii=False)
|
||||||
|
```
|
||||||
|
|
||||||
|
The `hash` field is computed over the canonical form of the event **with
|
||||||
|
`hash` set to `""`**, prefixed by the previous record's hash:
|
||||||
|
|
||||||
|
```
|
||||||
|
digest = sha256_hex(prev_hash + canonical({**event, "hash": ""}))
|
||||||
|
```
|
||||||
|
|
||||||
|
`prev` is the prior record's `hash`; genesis uses `prev = ""`. This makes
|
||||||
|
the journal an append-only Merkle-style chain: editing or deleting record
|
||||||
|
*n* changes its hash, so record *n+1*'s `prev` no longer matches — the
|
||||||
|
break is local and points at the tampered record. Verification needs only
|
||||||
|
the journal itself (no keys), so it runs offline and in CI.
|
||||||
|
|
||||||
|
### Single writer; ordering across restarts
|
||||||
|
|
||||||
|
**Decided: one writer per host** (reviewed — the host controller owns it).
|
||||||
|
Producers hand events to the host controller, which is the sole appender,
|
||||||
|
so the chain has one well-defined total order and one `seq`/`epoch`
|
||||||
|
counter. This ties audit availability to the host controller being up,
|
||||||
|
which is acceptable because the host controller already gates every
|
||||||
|
lifecycle transition; per-producer chains are noted only as a future
|
||||||
|
scaling path, not built now.
|
||||||
|
|
||||||
|
**Restarts** are handled by the chain, not the clock. `ts_mono` resets to
|
||||||
|
~0 on every writer start, so it orders events only *within* one boot. On
|
||||||
|
start the writer:
|
||||||
|
|
||||||
|
1. reads the last line of the journal, adopts its `hash` as the next
|
||||||
|
record's `prev` (the chain is continuous across the restart), and
|
||||||
|
2. bumps `epoch` (persisted alongside the chain head) and resets `seq` to
|
||||||
|
0 for the new boot.
|
||||||
|
|
||||||
|
Total order is therefore `(epoch, seq)` — monotonic across restarts by
|
||||||
|
construction — with `ts_wall` for human reading and `ts_mono` for
|
||||||
|
sub-second ordering inside an epoch. A crash mid-append truncates at most
|
||||||
|
the last (partial) line; the verifier flags it and replay resumes from the
|
||||||
|
last intact record.
|
||||||
|
|
||||||
|
### Journal (source of truth) + SQLite index (rebuildable)
|
||||||
|
|
||||||
|
- **Journal:** one append-only JSONL file per host (path from `paths.py`,
|
||||||
|
alongside `host_db_path()`), one canonical event per line, opened
|
||||||
|
`O_APPEND`. This is authoritative.
|
||||||
|
- **Index:** a new `audit_events` table via the existing `DbStore` /
|
||||||
|
`TableMigrations` machinery, holding the envelope columns plus JSON
|
||||||
|
blobs, indexed on `(bottled_agent, type, ts_wall)`. It is a **derived
|
||||||
|
cache**: `audit rebuild` truncates and replays the journal, re-verifying
|
||||||
|
the chain as it goes. If the DB is deleted or drifts, it is regenerated
|
||||||
|
from the journal with no data loss. (This supersedes the free-standing
|
||||||
|
`supervise_audit_entries` table, which becomes a view/producer onto the
|
||||||
|
new index.)
|
||||||
|
|
||||||
|
### Event registry (initial)
|
||||||
|
|
||||||
|
Dotted `type` names, grouped; the registry is a table mapping type →
|
||||||
|
required `untrusted` keys so producers and the verifier agree on shape:
|
||||||
|
|
||||||
|
- **lifecycle.*** — `lifecycle.bottled_agent_start`,
|
||||||
|
`lifecycle.bottled_agent_stop`, `lifecycle.bottled_agent_crash`
|
||||||
|
(producer: host-controller, #468). Leaf names use `bottled_agent` to match
|
||||||
|
the `trusted.bottled_agent` field — one term for the subject everywhere.
|
||||||
|
- **decision.*** — `decision.proposed`, `decision.resolved`
|
||||||
|
(producer: supervise; carries operator action + justification, replacing
|
||||||
|
PRD 0013's row shape).
|
||||||
|
- **egress.*** — `egress.decision` (allow/block at the proxy),
|
||||||
|
`egress.route_added`.
|
||||||
|
- **auth.*** — `auth.token_minted`, `auth.token_rejected` (control-plane;
|
||||||
|
**never** the token itself — see redaction).
|
||||||
|
- **forge.*** — `forge.push_accepted`, `forge.push_rejected` (git-gate),
|
||||||
|
`forge.pr_opened`.
|
||||||
|
|
||||||
|
New types are additive; adding one does not bump `v`. Removing or
|
||||||
|
re-typing a field bumps `v`.
|
||||||
|
|
||||||
|
### Redaction rule
|
||||||
|
|
||||||
|
Redaction runs at the envelope boundary, before a record is written, in two
|
||||||
|
layers:
|
||||||
|
|
||||||
|
1. **Key deny-list (structural).** A field whose *key* matches a known
|
||||||
|
credential shape (`token`, `secret`, `password`, `authorization`,
|
||||||
|
`*_key`) is refused — the producer must pass a reference (a token *id*
|
||||||
|
or `sha256` fingerprint), never the raw value. `auth.token_minted`
|
||||||
|
therefore records the token id and role, not the JWT. This is the
|
||||||
|
primary guard: it is cheap, deterministic, and catches the intended
|
||||||
|
mistake (a producer stuffing a credential into a named field).
|
||||||
|
|
||||||
|
2. **Value scan — reuse the egress DLP detectors.** Per review, the value
|
||||||
|
layer reuses the *same* deterministic credential-shape detectors the
|
||||||
|
egress proxy already ships:
|
||||||
|
`bot_bottle/gateway/egress/dlp_detectors.py` —
|
||||||
|
`scan_token_patterns` / `redact_tokens` (and `scan_known_secrets` for
|
||||||
|
host-known secret material). They are pure-Python, mitmproxy-free, and
|
||||||
|
already the project's source of truth for "what a leaked credential
|
||||||
|
looks like," so a single detector set governs both what may leave over
|
||||||
|
the wire and what may land in the journal — they can't drift apart.
|
||||||
|
|
||||||
|
**Scoped deliberately:** only the pattern/known-secret detectors are
|
||||||
|
reused, **not** `scan_entropy`. Entropy scoring is tuned for large
|
||||||
|
streamed request bodies; on the short, high-entropy structured values an
|
||||||
|
audit event legitimately carries (hashes, uuids, base64 ids) it would
|
||||||
|
false-positive and start redacting the very fingerprints the log needs.
|
||||||
|
So the shared layer is the deterministic detectors; entropy stays an
|
||||||
|
egress-only concern. (This is the "evaluate how reasonable that is" from
|
||||||
|
review: reuse the deterministic detectors — yes; share the entropy
|
||||||
|
heuristic — no.)
|
||||||
|
|
||||||
|
On a value-layer match the default is **redact** (scrub to a placeholder
|
||||||
|
and keep the event) rather than drop, so a producer bug can never make an
|
||||||
|
audit event vanish; the key deny-list stays a hard refusal because a
|
||||||
|
credential in a named field is always a producer bug worth surfacing.
|
||||||
|
|
||||||
|
## Implementation chunks
|
||||||
|
|
||||||
|
1. **(this PR — PRD only.)** The contract above. No code; scheduled to land
|
||||||
|
right after #468.
|
||||||
|
2. **Envelope + canonical + chain core.** `AuditEvent` dataclass,
|
||||||
|
`canonical()`, chain hashing, and the single-writer journal appender in
|
||||||
|
`bot_bottle/store/` (reusing `sha256_hex`); redaction wired to the
|
||||||
|
existing `gateway/egress/dlp_detectors` (`scan_token_patterns` /
|
||||||
|
`redact_tokens`); unit tests for determinism, chain-break detection,
|
||||||
|
`epoch`/`seq` continuity across a simulated restart, and redaction of
|
||||||
|
both a deny-listed key and a token-shaped value.
|
||||||
|
3. **SQLite index + `audit rebuild` / `audit verify` CLI.** New
|
||||||
|
`audit_events` migration; replay-from-journal; offline chain verifier;
|
||||||
|
local query commands (by bottled-agent / type / time / producer).
|
||||||
|
4. **Host controller as first producer (#468).** Wire
|
||||||
|
`lifecycle.bottled_agent_*` emission into the host controller's
|
||||||
|
start/stop/crash paths; establish the `epoch` bump + chain-head carry on
|
||||||
|
writer restart here (the host controller owns the single writer).
|
||||||
|
5. **Migrate existing producers.** Re-emit supervise `decision.*` (retiring
|
||||||
|
the standalone `supervise_audit_entries` shape behind the index), egress
|
||||||
|
`egress.*`, git-gate `forge.*`, control-plane `auth.*`.
|
||||||
|
6. **(follow-up.)** Cross-host merge transport (#324); per-writer signing +
|
||||||
|
external anchoring on the chain head; retention/rotation policy.
|
||||||
|
|
||||||
|
## Resolved in review (#495)
|
||||||
|
|
||||||
|
- **Single writer per host — decided.** The host controller owns the sole
|
||||||
|
appender; per-producer chains are a future scaling path only. (Design →
|
||||||
|
*Single writer; ordering across restarts*.)
|
||||||
|
- **Restarts — decided.** An `epoch` counter (bumped per writer boot) plus
|
||||||
|
carrying the last chain head as the next `prev` gives a total order of
|
||||||
|
`(epoch, seq)` that survives restarts; `ts_mono` orders only within an
|
||||||
|
epoch. (Design → *ordering across restarts*.)
|
||||||
|
- **`ts_*` and `producer` belong in `trusted`.** They are host-supplied, so
|
||||||
|
they now sit inside the `trusted` block; only chain metadata stays at the
|
||||||
|
top level. (Design → *The envelope*.)
|
||||||
|
- **Subject term is `bottled_agent` everywhere** — the `trusted` field and
|
||||||
|
the `lifecycle.bottled_agent_*` leaf names. (Design → *The envelope* /
|
||||||
|
*Event registry*.)
|
||||||
|
- **Retention head-carry — yes.** When a journal segment is rotated out,
|
||||||
|
the new segment's genesis `prev` is the rotated-out head, so the verifier
|
||||||
|
still trusts the current head across a rotation. (Folds into the
|
||||||
|
retention follow-up.)
|
||||||
|
- **Redaction reuses the egress detectors — yes, scoped.** Reuse the
|
||||||
|
deterministic `dlp_detectors` (`scan_token_patterns` / `redact_tokens` /
|
||||||
|
`scan_known_secrets`); exclude `scan_entropy` as brittle on the short,
|
||||||
|
high-entropy structured values audit records carry. (Design → *Redaction
|
||||||
|
rule*.)
|
||||||
|
|
||||||
|
## Open questions
|
||||||
|
|
||||||
|
- **Value-scan cost on the hot path.** The single writer runs the reused
|
||||||
|
detectors on every event's `untrusted` block inline. Is that cheap enough
|
||||||
|
at lifecycle-event volume, or should the value scan move to index-build
|
||||||
|
time (journal stays raw, index stores the redacted view)? Leaning inline
|
||||||
|
so the raw journal never contains a leaked value in the first place.
|
||||||
|
- **`epoch` persistence location.** Store the per-writer `epoch` + chain
|
||||||
|
head in the SQLite index (rebuildable, but then the writer needs the DB
|
||||||
|
at boot) or in a tiny sidecar file next to the journal (independent of
|
||||||
|
the index)? Leaning sidecar, so the writer can start and append without
|
||||||
|
the index present.
|
||||||
+8
-8
@@ -20,10 +20,10 @@ cd "$(dirname "$0")/.."
|
|||||||
|
|
||||||
PY="${PYTHON:-python3}"
|
PY="${PYTHON:-python3}"
|
||||||
|
|
||||||
# Critical security/logic core held to the high bar by ADR 0004. The helper
|
# Critical security/logic core held to the high bar by ADR 0004. The list
|
||||||
# fails before coverage when a curated path was renamed or removed; Coverage.py
|
# lives in one place (scripts/critical-modules.txt) so this report and the
|
||||||
# itself would silently ignore that stale include and inflate the score.
|
# README "core coverage" badge can't drift; comma-join it for --include.
|
||||||
CRITICAL=$("$PY" scripts/critical_modules.py)
|
CRITICAL=$(grep -vE '^[[:space:]]*(#|$)' scripts/critical-modules.txt | paste -sd, -)
|
||||||
|
|
||||||
if [ "${1:-}" = "aggregate" ]; then
|
if [ "${1:-}" = "aggregate" ]; then
|
||||||
# Aggregate mode: combine .coverage.* artifacts already in the workspace.
|
# Aggregate mode: combine .coverage.* artifacts already in the workspace.
|
||||||
@@ -34,8 +34,8 @@ if [ "${1:-}" = "aggregate" ]; then
|
|||||||
"$PY" -m coverage report -m
|
"$PY" -m coverage report -m
|
||||||
|
|
||||||
if [ "${2:-}" = "critical" ]; then
|
if [ "${2:-}" = "critical" ]; then
|
||||||
echo "== critical modules (ADR 0004 minimum: 90%) ==" >&2
|
echo "== critical modules (ADR 0004 target: 90%) ==" >&2
|
||||||
"$PY" -m coverage report --include="$CRITICAL" --fail-under=90
|
"$PY" -m coverage report --include="$CRITICAL"
|
||||||
fi
|
fi
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
@@ -55,6 +55,6 @@ echo "== combined report ==" >&2
|
|||||||
"$PY" -m coverage report -m
|
"$PY" -m coverage report -m
|
||||||
|
|
||||||
if [ "${1:-}" = "critical" ]; then
|
if [ "${1:-}" = "critical" ]; then
|
||||||
echo "== critical modules (ADR 0004 minimum: 90%) ==" >&2
|
echo "== critical modules (ADR 0004 target: 90%) ==" >&2
|
||||||
"$PY" -m coverage report --include="$CRITICAL" --fail-under=90
|
"$PY" -m coverage report --include="$CRITICAL"
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -7,48 +7,19 @@
|
|||||||
# number that silently stops measuring a module is worse than no badge.
|
# number that silently stops measuring a module is worse than no badge.
|
||||||
#
|
#
|
||||||
# One module path per line, relative to the repo root. Blank lines and
|
# One module path per line, relative to the repo root. Blank lines and
|
||||||
# `#` comments are ignored. scripts/critical_modules.py rejects missing,
|
# `#` comments are ignored.
|
||||||
# duplicate, non-Python, and out-of-repository entries before coverage runs.
|
|
||||||
|
|
||||||
# Host-side egress planning and secret preparation.
|
|
||||||
bot_bottle/egress/plan.py
|
|
||||||
bot_bottle/egress/service.py
|
|
||||||
|
|
||||||
# Gateway egress policy, matching, and DLP enforcement.
|
|
||||||
bot_bottle/gateway/egress/addon.py
|
bot_bottle/gateway/egress/addon.py
|
||||||
bot_bottle/gateway/egress/addon_core.py
|
bot_bottle/gateway/egress/addon_core.py
|
||||||
bot_bottle/gateway/egress/context.py
|
|
||||||
bot_bottle/gateway/egress/dlp.py
|
|
||||||
bot_bottle/gateway/egress/dlp_config.py
|
|
||||||
bot_bottle/gateway/egress/dlp_detectors.py
|
bot_bottle/gateway/egress/dlp_detectors.py
|
||||||
bot_bottle/gateway/egress/matching.py
|
bot_bottle/egress.py
|
||||||
bot_bottle/gateway/egress/schema.py
|
bot_bottle/manifest.py
|
||||||
bot_bottle/gateway/egress/types.py
|
bot_bottle/manifest_egress.py
|
||||||
|
bot_bottle/manifest_agent.py
|
||||||
# Manifest trust boundary and schema.
|
bot_bottle/manifest_schema.py
|
||||||
bot_bottle/manifest/agent.py
|
bot_bottle/git_gate.py
|
||||||
bot_bottle/manifest/bottle.py
|
|
||||||
bot_bottle/manifest/egress.py
|
|
||||||
bot_bottle/manifest/extends.py
|
|
||||||
bot_bottle/manifest/git.py
|
|
||||||
bot_bottle/manifest/index.py
|
|
||||||
bot_bottle/manifest/loader.py
|
|
||||||
bot_bottle/manifest/schema.py
|
|
||||||
bot_bottle/manifest/util.py
|
|
||||||
|
|
||||||
# Host-side and gateway-side git policy enforcement.
|
|
||||||
bot_bottle/git_gate/host_key.py
|
|
||||||
bot_bottle/git_gate/plan.py
|
|
||||||
bot_bottle/git_gate/provision.py
|
|
||||||
bot_bottle/git_gate/service.py
|
|
||||||
bot_bottle/gateway/git_gate/render.py
|
bot_bottle/gateway/git_gate/render.py
|
||||||
|
bot_bottle/git_gate_provision.py
|
||||||
bot_bottle/gateway/git_gate/http_backend.py
|
bot_bottle/gateway/git_gate/http_backend.py
|
||||||
|
bot_bottle/supervise.py
|
||||||
# Supervise proposal protocol and data plane.
|
|
||||||
bot_bottle/supervisor/plan.py
|
|
||||||
bot_bottle/supervisor/types.py
|
|
||||||
bot_bottle/gateway/supervisor/server.py
|
|
||||||
|
|
||||||
# Shared parsers and state validation.
|
|
||||||
bot_bottle/yaml_subset.py
|
bot_bottle/yaml_subset.py
|
||||||
bot_bottle/bottle_state.py
|
bot_bottle/bottle_state.py
|
||||||
|
|||||||
@@ -1,101 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Validate and render the critical-module coverage manifest.
|
|
||||||
|
|
||||||
Coverage.py silently ignores an ``--include`` path that does not exist. That
|
|
||||||
is useful for broad globs, but dangerous for bot-bottle's curated security
|
|
||||||
core: a rename could otherwise improve the reported percentage by removing a
|
|
||||||
module from the measurement. Keep the validation in one small stdlib helper
|
|
||||||
and make every coverage consumer call it.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
DEFAULT_MANIFEST = REPO_ROOT / "scripts" / "critical-modules.txt"
|
|
||||||
|
|
||||||
|
|
||||||
class CriticalModulesError(ValueError):
|
|
||||||
"""The critical-module manifest is empty, ambiguous, or stale."""
|
|
||||||
|
|
||||||
|
|
||||||
def load_critical_modules(manifest: Path, *, root: Path) -> list[str]:
|
|
||||||
"""Return validated module paths relative to *root*.
|
|
||||||
|
|
||||||
Entries must be unique, concrete Python files inside the repository.
|
|
||||||
Globs are deliberately rejected by the file check: each rename must update
|
|
||||||
this explicit security review surface.
|
|
||||||
"""
|
|
||||||
|
|
||||||
root = root.resolve()
|
|
||||||
try:
|
|
||||||
lines = manifest.read_text(encoding="utf-8").splitlines()
|
|
||||||
except OSError as exc:
|
|
||||||
raise CriticalModulesError(
|
|
||||||
f"cannot read critical-module manifest {manifest}: {exc}"
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
modules: list[str] = []
|
|
||||||
seen: set[str] = set()
|
|
||||||
errors: list[str] = []
|
|
||||||
for line_number, raw in enumerate(lines, start=1):
|
|
||||||
entry = raw.strip()
|
|
||||||
if not entry or entry.startswith("#"):
|
|
||||||
continue
|
|
||||||
path = Path(entry)
|
|
||||||
prefix = f"{manifest}:{line_number}: {entry!r}"
|
|
||||||
if path.is_absolute():
|
|
||||||
errors.append(f"{prefix} must be relative to the repository root")
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
resolved = (root / path).resolve()
|
|
||||||
resolved.relative_to(root)
|
|
||||||
except ValueError:
|
|
||||||
errors.append(f"{prefix} escapes the repository root")
|
|
||||||
continue
|
|
||||||
if entry in seen:
|
|
||||||
errors.append(f"{prefix} is duplicated")
|
|
||||||
continue
|
|
||||||
seen.add(entry)
|
|
||||||
if path.suffix != ".py":
|
|
||||||
errors.append(f"{prefix} is not a Python module")
|
|
||||||
continue
|
|
||||||
if not resolved.is_file():
|
|
||||||
errors.append(f"{prefix} does not exist")
|
|
||||||
continue
|
|
||||||
modules.append(path.as_posix())
|
|
||||||
|
|
||||||
if not modules and not errors:
|
|
||||||
errors.append(f"{manifest}: contains no critical modules")
|
|
||||||
if errors:
|
|
||||||
raise CriticalModulesError("\n".join(errors))
|
|
||||||
return modules
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="validate and print the critical coverage include list"
|
|
||||||
)
|
|
||||||
parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST)
|
|
||||||
parser.add_argument("--root", type=Path, default=REPO_ROOT)
|
|
||||||
parser.add_argument(
|
|
||||||
"--check", action="store_true",
|
|
||||||
help="validate only; do not print the comma-separated include list",
|
|
||||||
)
|
|
||||||
args = parser.parse_args(argv)
|
|
||||||
try:
|
|
||||||
modules = load_critical_modules(args.manifest, root=args.root)
|
|
||||||
except CriticalModulesError as exc:
|
|
||||||
print(f"critical-modules: {exc}", file=sys.stderr)
|
|
||||||
return 1
|
|
||||||
if not args.check:
|
|
||||||
print(",".join(modules))
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Run unittest discovery with explicit execution-count assurances.
|
|
||||||
|
|
||||||
The standard unittest CLI exits successfully when a suite contains skipped
|
|
||||||
tests. That is normally useful, but it let the Docker integration job stay
|
|
||||||
green while its security-boundary classes were all skipped under act_runner.
|
|
||||||
This wrapper keeps normal unittest output and adds opt-in minimum-executed and
|
|
||||||
no-skip gates for jobs that promise a concrete integration surface.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import sys
|
|
||||||
import unittest
|
|
||||||
|
|
||||||
|
|
||||||
def assurance_errors(
|
|
||||||
*, tests_run: int, skipped: int, minimum_executed: int, fail_on_skip: bool
|
|
||||||
) -> list[str]:
|
|
||||||
"""Return human-readable assurance failures for a completed suite."""
|
|
||||||
|
|
||||||
executed = tests_run - skipped
|
|
||||||
errors: list[str] = []
|
|
||||||
if executed < minimum_executed:
|
|
||||||
errors.append(
|
|
||||||
f"executed {executed} test(s), below required minimum "
|
|
||||||
f"{minimum_executed} (discovered {tests_run}, skipped {skipped})"
|
|
||||||
)
|
|
||||||
if fail_on_skip and skipped:
|
|
||||||
errors.append(f"{skipped} test(s) skipped in a no-skip suite")
|
|
||||||
return errors
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
|
||||||
parser = argparse.ArgumentParser(
|
|
||||||
description="unittest discovery with execution-count assurance"
|
|
||||||
)
|
|
||||||
parser.add_argument("-s", "--start-directory", default=".")
|
|
||||||
parser.add_argument("-t", "--top-level-directory", default=None)
|
|
||||||
parser.add_argument("-p", "--pattern", default="test*.py")
|
|
||||||
parser.add_argument("--minimum-executed", type=int, default=0)
|
|
||||||
parser.add_argument("--fail-on-skip", action="store_true")
|
|
||||||
parser.add_argument("-v", "--verbose", action="store_true")
|
|
||||||
args = parser.parse_args(argv)
|
|
||||||
|
|
||||||
suite = unittest.defaultTestLoader.discover(
|
|
||||||
args.start_directory,
|
|
||||||
pattern=args.pattern,
|
|
||||||
top_level_dir=args.top_level_directory,
|
|
||||||
)
|
|
||||||
result = unittest.TextTestRunner(
|
|
||||||
verbosity=2 if args.verbose else 1,
|
|
||||||
).run(suite)
|
|
||||||
failures = assurance_errors(
|
|
||||||
tests_run=result.testsRun,
|
|
||||||
skipped=len(result.skipped),
|
|
||||||
minimum_executed=args.minimum_executed,
|
|
||||||
fail_on_skip=args.fail_on_skip,
|
|
||||||
)
|
|
||||||
for failure in failures:
|
|
||||||
print(f"unittest-gate: {failure}", file=sys.stderr)
|
|
||||||
return 0 if result.wasSuccessful() and not failures else 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
raise SystemExit(main())
|
|
||||||
+8
-12
@@ -20,11 +20,10 @@ tests/
|
|||||||
... # many others; see unit/ directory
|
... # many others; see unit/ directory
|
||||||
integration/
|
integration/
|
||||||
test_gateway_image.py
|
test_gateway_image.py
|
||||||
test_sandbox_escape.py
|
test_dry_run_plan.py
|
||||||
test_orphan_cleanup.py
|
test_orphan_cleanup.py
|
||||||
...
|
...
|
||||||
canaries/
|
canaries/ # opt-in; see below (currently empty)
|
||||||
test_gitleaks_release.py # opt-in upstream artifact check
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Classification falls out of the directory — no hand-maintained list to
|
Classification falls out of the directory — no hand-maintained list to
|
||||||
@@ -44,27 +43,24 @@ Discovery is invoked with `-t .` (top-level dir = repo root) so the
|
|||||||
|
|
||||||
## What the integration tests cover
|
## What the integration tests cover
|
||||||
|
|
||||||
|
- `test_dry_run_plan.py` — `cli.py start --dry-run --format=json` emits
|
||||||
|
a structured plan that contains the resolved egress allowlist and
|
||||||
|
the bottle's runtime, and creates zero Docker resources.
|
||||||
- `test_orphan_cleanup.py` — `network_remove` is idempotent against
|
- `test_orphan_cleanup.py` — `network_remove` is idempotent against
|
||||||
missing resources, so the EXIT trap can call it unconditionally.
|
missing resources, so the EXIT trap can call it unconditionally.
|
||||||
- `test_gateway_image.py` — builds Dockerfile.gateway and
|
- `test_gateway_image.py` — builds Dockerfile.gateway and
|
||||||
probes that gitleaks / mitmdump / supervise are all reachable
|
probes that gitleaks / mitmdump / supervise are all reachable
|
||||||
inside the gateway image.
|
inside the gateway image.
|
||||||
- `test_orchestrator_docker_auth.py` — drives the real control-plane
|
|
||||||
container and verifies role-scoped authentication.
|
|
||||||
- `test_multitenant_isolation.py` and `test_sandbox_escape.py` — exercise
|
|
||||||
token/allowlist separation and end-to-end escape attempts.
|
|
||||||
|
|
||||||
## Canaries
|
## Canaries
|
||||||
|
|
||||||
`tests/canaries/` holds upstream-regression checks gated on
|
`tests/canaries/` holds upstream-regression checks gated on
|
||||||
`BOT_BOTTLE_RUN_CANARIES=1` and not part of the per-push suite.
|
`BOT_BOTTLE_RUN_CANARIES=1` and not part of the per-push suite.
|
||||||
They're invoked by the scheduled `canaries` workflow. The gitleaks canary
|
They're invoked by the scheduled `canaries` workflow. Currently
|
||||||
downloads the exact release archive pinned by `Dockerfile.gateway`, verifies
|
no canaries are defined.
|
||||||
its architecture-specific checksum, and executes the binary.
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
BOT_BOTTLE_RUN_CANARIES=1 python -m scripts.unittest_gate \
|
BOT_BOTTLE_RUN_CANARIES=1 python -m unittest discover -t . -s tests/canaries -v
|
||||||
-t . -s tests/canaries -v --minimum-executed 1 --fail-on-skip
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## What's NOT covered
|
## What's NOT covered
|
||||||
|
|||||||
@@ -1,85 +0,0 @@
|
|||||||
"""Canary: the pinned gitleaks release remains downloadable and executable.
|
|
||||||
|
|
||||||
The gateway Dockerfile verifies this archive during an image build. Repeating
|
|
||||||
the upstream check weekly keeps registry/release drift out of normal pull
|
|
||||||
requests while proving that the pinned URL, architecture checksum, archive
|
|
||||||
shape, and binary still agree.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import os
|
|
||||||
import platform
|
|
||||||
import re
|
|
||||||
import subprocess
|
|
||||||
import tarfile
|
|
||||||
import tempfile
|
|
||||||
import unittest
|
|
||||||
import urllib.request
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
|
||||||
DOCKERFILE = ROOT / "Dockerfile.gateway"
|
|
||||||
|
|
||||||
|
|
||||||
def _docker_arg(text: str, name: str) -> str:
|
|
||||||
match = re.search(rf"^ARG {re.escape(name)}=(\S+)$", text, re.MULTILINE)
|
|
||||||
if match is None:
|
|
||||||
raise AssertionError(f"Dockerfile.gateway has no concrete ARG {name}")
|
|
||||||
return match.group(1)
|
|
||||||
|
|
||||||
|
|
||||||
@unittest.skipUnless(
|
|
||||||
os.environ.get("BOT_BOTTLE_RUN_CANARIES") == "1",
|
|
||||||
"canary suite is opt-in; set BOT_BOTTLE_RUN_CANARIES=1 to run",
|
|
||||||
)
|
|
||||||
class TestGitleaksRelease(unittest.TestCase):
|
|
||||||
def test_pinned_archive_checksum_and_binary(self) -> None:
|
|
||||||
dockerfile = DOCKERFILE.read_text(encoding="utf-8")
|
|
||||||
version = _docker_arg(dockerfile, "GITLEAKS_VERSION")
|
|
||||||
machine = platform.machine().lower()
|
|
||||||
architectures = {
|
|
||||||
"x86_64": ("linux_x64", "GITLEAKS_SHA256_AMD64"),
|
|
||||||
"amd64": ("linux_x64", "GITLEAKS_SHA256_AMD64"),
|
|
||||||
"aarch64": ("linux_arm64", "GITLEAKS_SHA256_ARM64"),
|
|
||||||
"arm64": ("linux_arm64", "GITLEAKS_SHA256_ARM64"),
|
|
||||||
}
|
|
||||||
if machine not in architectures:
|
|
||||||
self.fail(f"unsupported canary runner architecture: {machine}")
|
|
||||||
asset, checksum_arg = architectures[machine]
|
|
||||||
expected_checksum = _docker_arg(dockerfile, checksum_arg)
|
|
||||||
url = (
|
|
||||||
"https://github.com/gitleaks/gitleaks/releases/download/"
|
|
||||||
f"v{version}/gitleaks_{version}_{asset}.tar.gz"
|
|
||||||
)
|
|
||||||
|
|
||||||
with tempfile.TemporaryDirectory(prefix="bot-bottle-gitleaks-canary.") as tmp:
|
|
||||||
archive = Path(tmp) / "gitleaks.tar.gz"
|
|
||||||
urllib.request.urlretrieve(url, archive)
|
|
||||||
self.assertEqual(
|
|
||||||
expected_checksum,
|
|
||||||
hashlib.sha256(archive.read_bytes()).hexdigest(),
|
|
||||||
"the pinned upstream archive no longer matches Dockerfile.gateway",
|
|
||||||
)
|
|
||||||
with tarfile.open(archive, "r:gz") as bundle:
|
|
||||||
member = bundle.getmember("gitleaks")
|
|
||||||
source = bundle.extractfile(member)
|
|
||||||
if source is None:
|
|
||||||
self.fail("gitleaks archive member is not a regular file")
|
|
||||||
binary = Path(tmp) / "gitleaks"
|
|
||||||
binary.write_bytes(source.read())
|
|
||||||
binary.chmod(0o755)
|
|
||||||
result = subprocess.run(
|
|
||||||
[str(binary), "version"],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
self.assertEqual(0, result.returncode, result.stderr)
|
|
||||||
self.assertIn(version, result.stdout + result.stderr)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -14,7 +14,9 @@ the chunk-1 contract:
|
|||||||
expected "no daemons selected" line when the supervisor is
|
expected "no daemons selected" line when the supervisor is
|
||||||
pointed at an empty daemon set.
|
pointed at an empty daemon set.
|
||||||
|
|
||||||
Skips cleanly only when the selected Docker backend is unavailable.
|
Skips cleanly when docker is unavailable, or under act_runner
|
||||||
|
where the host bind-mount topology breaks multi-stage builds
|
||||||
|
that pull large bases.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -31,6 +33,12 @@ _DOCKERFILE = "Dockerfile.gateway"
|
|||||||
|
|
||||||
|
|
||||||
@skip_unless_backend("docker")
|
@skip_unless_backend("docker")
|
||||||
|
@unittest.skipIf(
|
||||||
|
os.environ.get("GITEA_ACTIONS") == "true",
|
||||||
|
"skipped under act_runner: multi-stage build pulls a 200+MB "
|
||||||
|
"mitmproxy base + two upstream gateway images; runner storage "
|
||||||
|
"+ time budget make this an interactive-only test",
|
||||||
|
)
|
||||||
class TestGatewayImage(unittest.TestCase):
|
class TestGatewayImage(unittest.TestCase):
|
||||||
"""Builds the image once for the class, then runs a few
|
"""Builds the image once for the class, then runs a few
|
||||||
`docker run` probes against it."""
|
`docker run` probes against it."""
|
||||||
@@ -43,11 +51,10 @@ class TestGatewayImage(unittest.TestCase):
|
|||||||
"-f", _DOCKERFILE, "."],
|
"-f", _DOCKERFILE, "."],
|
||||||
cwd=repo_root,
|
cwd=repo_root,
|
||||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||||
check=False,
|
|
||||||
)
|
)
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise AssertionError(
|
raise unittest.SkipTest(
|
||||||
f"docker build failed; image probes cannot run.\n"
|
f"docker build failed; skipping image probes.\n"
|
||||||
f"{proc.stdout.decode('utf-8', errors='replace')[-2000:]}"
|
f"{proc.stdout.decode('utf-8', errors='replace')[-2000:]}"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -56,16 +63,14 @@ class TestGatewayImage(unittest.TestCase):
|
|||||||
subprocess.run(
|
subprocess.run(
|
||||||
["docker", "image", "rm", "-f", _IMAGE],
|
["docker", "image", "rm", "-f", _IMAGE],
|
||||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||||
check=False,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def _run_in_image(self, *cmd: str, timeout: float = 30.0) -> tuple[int, str]:
|
def _run_in_image(self, *cmd: str, timeout: float = 30.0) -> tuple[int, str]:
|
||||||
proc = subprocess.run(
|
proc = subprocess.run(
|
||||||
["docker", "run", "--rm", "--entrypoint", cmd[0], _IMAGE,
|
["docker", "run", "--rm", "--entrypoint", cmd[0], _IMAGE,
|
||||||
*cmd[1:]],
|
*cmd[1:]],
|
||||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
check=False,
|
|
||||||
)
|
)
|
||||||
return proc.returncode, proc.stdout.decode("utf-8", errors="replace")
|
return proc.returncode, proc.stdout.decode("utf-8", errors="replace")
|
||||||
|
|
||||||
@@ -86,9 +91,7 @@ class TestGatewayImage(unittest.TestCase):
|
|||||||
# Probe that the package imports resolve inside the image.
|
# Probe that the package imports resolve inside the image.
|
||||||
rc, out = self._run_in_image(
|
rc, out = self._run_in_image(
|
||||||
"python3", "-c",
|
"python3", "-c",
|
||||||
"from bot_bottle.supervisor import types; "
|
"from bot_bottle.supervisor import types; from bot_bottle.gateway.supervisor import server as supervise_server; print('ok')",
|
||||||
"from bot_bottle.gateway.supervisor import server as supervise_server; "
|
|
||||||
"print('ok')",
|
|
||||||
)
|
)
|
||||||
self.assertEqual(0, rc, msg=out)
|
self.assertEqual(0, rc, msg=out)
|
||||||
self.assertIn("ok", out)
|
self.assertIn("ok", out)
|
||||||
@@ -103,7 +106,6 @@ class TestGatewayImage(unittest.TestCase):
|
|||||||
_IMAGE],
|
_IMAGE],
|
||||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||||
timeout=10.0,
|
timeout=10.0,
|
||||||
check=False,
|
|
||||||
)
|
)
|
||||||
out = proc.stdout.decode("utf-8", errors="replace")
|
out = proc.stdout.decode("utf-8", errors="replace")
|
||||||
self.assertEqual(0, proc.returncode, msg=out)
|
self.assertEqual(0, proc.returncode, msg=out)
|
||||||
|
|||||||
@@ -16,10 +16,11 @@ throwaway BOT_BOTTLE_ROOT for a clean registry and tears everything down.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import secrets
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import time
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from bot_bottle.backend.docker.consolidated_launch import (
|
from bot_bottle.backend.docker.consolidated_launch import (
|
||||||
_network_cidr,
|
_network_cidr,
|
||||||
@@ -72,12 +73,19 @@ _PROBE_SRC = (
|
|||||||
|
|
||||||
|
|
||||||
@skip_unless_backend("docker")
|
@skip_unless_backend("docker")
|
||||||
|
@unittest.skipIf(
|
||||||
|
os.environ.get("GITEA_ACTIONS") == "true",
|
||||||
|
"skipped under act_runner: the orchestrator container bind-mounts the repo "
|
||||||
|
"path into a container on the socket-shared host daemon, which can't see the "
|
||||||
|
"runner's /workspace — same host-bind-mount constraint as the other "
|
||||||
|
"bottle-bringup integration tests",
|
||||||
|
)
|
||||||
class TestMultitenantIsolation(unittest.TestCase):
|
class TestMultitenantIsolation(unittest.TestCase):
|
||||||
def setUp(self) -> None:
|
def setUp(self) -> None:
|
||||||
# Named volume → a clean registry DB that is also visible to a
|
self._tmp = tempfile.TemporaryDirectory()
|
||||||
# socket-shared host daemon when the test process runs in act_runner.
|
self.addCleanup(self._tmp.cleanup)
|
||||||
self._root_volume = "bot-bottle-mtitest-root-" + secrets.token_hex(4)
|
# Throwaway root → a clean registry DB, independent of the host's.
|
||||||
self.svc = DockerInfraService(root_mount_source=self._root_volume)
|
self.svc = DockerInfraService(host_root=Path(self._tmp.name))
|
||||||
self.addCleanup(self._teardown_docker)
|
self.addCleanup(self._teardown_docker)
|
||||||
# ensure_running builds the bundle image (slow on a cold cache) and
|
# ensure_running builds the bundle image (slow on a cold cache) and
|
||||||
# brings up the shared network + gateway + orchestrator.
|
# brings up the shared network + gateway + orchestrator.
|
||||||
@@ -92,8 +100,13 @@ class TestMultitenantIsolation(unittest.TestCase):
|
|||||||
self.svc.stop()
|
self.svc.stop()
|
||||||
subprocess.run(["docker", "network", "rm", GATEWAY_NETWORK],
|
subprocess.run(["docker", "network", "rm", GATEWAY_NETWORK],
|
||||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
|
||||||
|
# The orchestrator container wrote the registry DB as root into the
|
||||||
|
# throwaway root; chown it back so the (non-root) tempdir cleanup can
|
||||||
|
# remove it.
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["docker", "volume", "rm", "--force", self._root_volume],
|
["docker", "run", "--rm", "-v", f"{self._tmp.name}:/r",
|
||||||
|
"--entrypoint", "chown", GATEWAY_IMAGE, "-R",
|
||||||
|
f"{os.getuid()}:{os.getgid()}", "/r"],
|
||||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -119,61 +132,29 @@ class TestMultitenantIsolation(unittest.TestCase):
|
|||||||
taken = _network_container_ips(GATEWAY_NETWORK) + extra_taken
|
taken = _network_container_ips(GATEWAY_NETWORK) + extra_taken
|
||||||
return next_free_ip(_network_cidr(GATEWAY_NETWORK), taken)
|
return next_free_ip(_network_cidr(GATEWAY_NETWORK), taken)
|
||||||
|
|
||||||
def _probe(self, source_ip: str, identity_token: str, host: str) -> str:
|
def _probe(self, source_ip: str, host: str) -> str:
|
||||||
deadline = time.monotonic() + 30
|
proc = subprocess.run(
|
||||||
last = subprocess.CompletedProcess([], 1, "", "probe not attempted")
|
["docker", "run", "--rm", "--network", GATEWAY_NETWORK, "--ip", source_ip,
|
||||||
while time.monotonic() < deadline:
|
"--entrypoint", "python3", GATEWAY_IMAGE, "-c", _PROBE_SRC,
|
||||||
last = subprocess.run(
|
f"http://{self.gw_ip}:{EGRESS_PORT}", host],
|
||||||
[
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False, timeout=90,
|
||||||
"docker", "run", "--rm",
|
|
||||||
"--network", GATEWAY_NETWORK, "--ip", source_ip,
|
|
||||||
"--entrypoint", "python3", GATEWAY_IMAGE, "-c", _PROBE_SRC,
|
|
||||||
f"http://bottle:{identity_token}@{self.gw_ip}:{EGRESS_PORT}",
|
|
||||||
host,
|
|
||||||
],
|
|
||||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
|
|
||||||
check=False, timeout=90,
|
|
||||||
)
|
|
||||||
output = last.stdout.strip()
|
|
||||||
if last.returncode == 0 and output:
|
|
||||||
return output
|
|
||||||
time.sleep(0.25)
|
|
||||||
self.fail(
|
|
||||||
f"gateway probe did not become ready: "
|
|
||||||
f"exit={last.returncode}, stderr={last.stderr.strip()!r}"
|
|
||||||
)
|
)
|
||||||
|
return proc.stdout.strip()
|
||||||
|
|
||||||
def test_two_bottles_share_gateway_with_isolated_tokens_and_allowlists(self) -> None:
|
def test_two_bottles_share_gateway_with_isolated_tokens_and_allowlists(self) -> None:
|
||||||
ip_a = self._free_ip([])
|
ip_a = self._free_ip([])
|
||||||
ip_b = self._free_ip([ip_a])
|
ip_b = self._free_ip([ip_a])
|
||||||
bottle_a = self.client.register_bottle(
|
self.client.register_bottle(ip_a, policy=_POLICY_A, tokens={"EGRESS_TOKEN_0": _TOKEN_A})
|
||||||
ip_a, policy=_POLICY_A, tokens={"EGRESS_TOKEN_0": _TOKEN_A}
|
self.client.register_bottle(ip_b, policy=_POLICY_B, tokens={"EGRESS_TOKEN_0": _TOKEN_B})
|
||||||
)
|
|
||||||
bottle_b = self.client.register_bottle(
|
|
||||||
ip_b, policy=_POLICY_B, tokens={"EGRESS_TOKEN_0": _TOKEN_B}
|
|
||||||
)
|
|
||||||
|
|
||||||
# Each bottle gets its OWN token injected on the shared route — no bleed.
|
# Each bottle gets its OWN token injected on the shared route — no bleed.
|
||||||
self.assertEqual(
|
self.assertEqual(f"200 AUTH=Bearer {_TOKEN_A}", self._probe(ip_a, "echo-shared"))
|
||||||
f"200 AUTH=Bearer {_TOKEN_A}",
|
self.assertEqual(f"200 AUTH=Bearer {_TOKEN_B}", self._probe(ip_b, "echo-shared"))
|
||||||
self._probe(ip_a, bottle_a.identity_token, "echo-shared"),
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
f"200 AUTH=Bearer {_TOKEN_B}",
|
|
||||||
self._probe(ip_b, bottle_b.identity_token, "echo-shared"),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Allowlist is per-bottle: echo-bonly is only in B's policy.
|
# Allowlist is per-bottle: echo-bonly is only in B's policy.
|
||||||
self.assertTrue(
|
self.assertTrue(self._probe(ip_a, "echo-bonly").startswith("403"), # fail-closed for A
|
||||||
self._probe(
|
"A reached a host outside its allowlist")
|
||||||
ip_a, bottle_a.identity_token, "echo-bonly"
|
self.assertEqual("200 AUTH=NONE", self._probe(ip_b, "echo-bonly")) # allowed, unauthed for B
|
||||||
).startswith("403"), # fail-closed for A
|
|
||||||
"A reached a host outside its allowlist",
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
"200 AUTH=NONE",
|
|
||||||
self._probe(ip_b, bottle_b.identity_token, "echo-bonly"),
|
|
||||||
) # allowed, unauthed for B
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import secrets
|
|||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint
|
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint
|
||||||
from bot_bottle.orchestrator.client import OrchestratorClient
|
from bot_bottle.orchestrator.client import OrchestratorClient
|
||||||
@@ -37,6 +38,13 @@ _TEST_GATEWAY_IMAGE = "bot-bottle-gateway:itest"
|
|||||||
|
|
||||||
|
|
||||||
@skip_unless_backend("docker")
|
@skip_unless_backend("docker")
|
||||||
|
@unittest.skipIf(
|
||||||
|
os.environ.get("GITEA_ACTIONS") == "true",
|
||||||
|
"skipped under act_runner: the orchestrator container bind-mounts the repo "
|
||||||
|
"path into a container on the socket-shared host daemon, which can't see the "
|
||||||
|
"runner's /workspace — same host-bind-mount constraint as the other "
|
||||||
|
"bottle-bringup integration tests",
|
||||||
|
)
|
||||||
class TestDockerOrchestratorAuthIntegration(unittest.TestCase):
|
class TestDockerOrchestratorAuthIntegration(unittest.TestCase):
|
||||||
@classmethod
|
@classmethod
|
||||||
def setUpClass(cls) -> None:
|
def setUpClass(cls) -> None:
|
||||||
@@ -66,10 +74,10 @@ class TestDockerOrchestratorAuthIntegration(unittest.TestCase):
|
|||||||
gateway_name = f"bot-bottle-gateway-itest-{suffix}"
|
gateway_name = f"bot-bottle-gateway-itest-{suffix}"
|
||||||
network = f"bot-bottle-net-itest-{suffix}"
|
network = f"bot-bottle-net-itest-{suffix}"
|
||||||
control_network = f"bot-bottle-ctrl-itest-{suffix}"
|
control_network = f"bot-bottle-ctrl-itest-{suffix}"
|
||||||
root_volume = f"bot-bottle-root-itest-{suffix}"
|
host_root = Path(cls._tmp.name)
|
||||||
cls.addClassCleanup(
|
cls.addClassCleanup(
|
||||||
cls._teardown_docker,
|
cls._teardown_docker,
|
||||||
orchestrator_name, gateway_name, network, control_network, root_volume,
|
orchestrator_name, gateway_name, network, control_network, host_root,
|
||||||
)
|
)
|
||||||
|
|
||||||
cls.svc = DockerInfraService(
|
cls.svc = DockerInfraService(
|
||||||
@@ -80,7 +88,7 @@ class TestDockerOrchestratorAuthIntegration(unittest.TestCase):
|
|||||||
orchestrator_image=_TEST_ORCHESTRATOR_IMAGE,
|
orchestrator_image=_TEST_ORCHESTRATOR_IMAGE,
|
||||||
gateway_image=_TEST_GATEWAY_IMAGE,
|
gateway_image=_TEST_GATEWAY_IMAGE,
|
||||||
port=20000 + secrets.randbelow(10000),
|
port=20000 + secrets.randbelow(10000),
|
||||||
root_mount_source=root_volume,
|
host_root=host_root,
|
||||||
)
|
)
|
||||||
cls.svc.ensure_running()
|
cls.svc.ensure_running()
|
||||||
# The control plane now verifies role-scoped signed tokens, not the raw
|
# The control plane now verifies role-scoped signed tokens, not the raw
|
||||||
@@ -92,7 +100,7 @@ class TestDockerOrchestratorAuthIntegration(unittest.TestCase):
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def _teardown_docker(
|
def _teardown_docker(
|
||||||
orchestrator_name: str, gateway_name: str,
|
orchestrator_name: str, gateway_name: str,
|
||||||
network: str, control_network: str, root_volume: str,
|
network: str, control_network: str, host_root: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
for name in (gateway_name, orchestrator_name):
|
for name in (gateway_name, orchestrator_name):
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
@@ -104,8 +112,14 @@ class TestDockerOrchestratorAuthIntegration(unittest.TestCase):
|
|||||||
["docker", "network", "rm", net],
|
["docker", "network", "rm", net],
|
||||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
||||||
)
|
)
|
||||||
|
# The orchestrator container (no USER directive) wrote the registry
|
||||||
|
# DB as root into the throwaway host_root; chown it back so the
|
||||||
|
# (non-root) tempdir cleanup can remove it. Same workaround
|
||||||
|
# test_multitenant_isolation.py uses for the identical bind mount.
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["docker", "volume", "rm", "--force", root_volume],
|
["docker", "run", "--rm", "-v", f"{host_root}:/r",
|
||||||
|
"--entrypoint", "chown", _TEST_ORCHESTRATOR_IMAGE, "-R",
|
||||||
|
f"{os.getuid()}:{os.getgid()}", "/r"],
|
||||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -42,6 +42,11 @@ class TestOrphanCleanup(unittest.TestCase):
|
|||||||
# Returning True == idempotent success.
|
# Returning True == idempotent success.
|
||||||
self.assertTrue(network_remove(f"bot-bottle-net-{self.slug}-does-not-exist"))
|
self.assertTrue(network_remove(f"bot-bottle-net-{self.slug}-does-not-exist"))
|
||||||
|
|
||||||
|
@unittest.skipIf(
|
||||||
|
os.environ.get("GITEA_ACTIONS") == "true",
|
||||||
|
"skipped under act_runner: docker socket mount topology breaks "
|
||||||
|
"in-process visibility of networks created on the host daemon",
|
||||||
|
)
|
||||||
def test_create_and_remove(self):
|
def test_create_and_remove(self):
|
||||||
self.internal_name = network_create_internal(self.slug)
|
self.internal_name = network_create_internal(self.slug)
|
||||||
self.egress_name = network_create_egress(self.slug)
|
self.egress_name = network_create_egress(self.slug)
|
||||||
|
|||||||
@@ -67,7 +67,26 @@ _DUMMY_HOST_KEY = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Backends whose CI runner is HOST-mode (self-hosted), so the test process
|
||||||
|
# and the backend share a host. The containerized act_runner (docker on
|
||||||
|
# ubuntu-latest) is the one that can't see the host bind mount egress_tls_init
|
||||||
|
# uses and hides sibling-gateway network topology; host-mode runners
|
||||||
|
# (firecracker/KVM, macos-container) don't have those constraints, so the test
|
||||||
|
# runs there. Keep this in sync with the `runs-on` labels in
|
||||||
|
# .gitea/workflows/test.yml.
|
||||||
|
_HOST_MODE_CI_BACKENDS = frozenset({"firecracker", "macos-container"})
|
||||||
|
|
||||||
|
|
||||||
@skip_unless_selected_backend_available()
|
@skip_unless_selected_backend_available()
|
||||||
|
@unittest.skipIf(
|
||||||
|
os.environ.get("GITEA_ACTIONS") == "true"
|
||||||
|
and os.environ.get("BOT_BOTTLE_BACKEND") not in _HOST_MODE_CI_BACKENDS,
|
||||||
|
"skipped under the containerized act_runner (docker on ubuntu-latest): "
|
||||||
|
"egress_tls_init uses a host bind mount the runner container can't "
|
||||||
|
"see, and the network topology hides sibling-gateway visibility — "
|
||||||
|
"these constraints don't apply on the self-hosted host-mode runners "
|
||||||
|
"(firecracker/KVM, macos-container)",
|
||||||
|
)
|
||||||
class TestSandboxEscape(unittest.TestCase):
|
class TestSandboxEscape(unittest.TestCase):
|
||||||
"""End-to-end attacks against a real bottle. The bottle stays
|
"""End-to-end attacks against a real bottle. The bottle stays
|
||||||
up for the whole class — bringup is ~10-30s, so per-test
|
up for the whole class — bringup is ~10-30s, so per-test
|
||||||
@@ -170,7 +189,7 @@ class TestSandboxEscape(unittest.TestCase):
|
|||||||
missing.append(tool)
|
missing.append(tool)
|
||||||
if missing:
|
if missing:
|
||||||
cls._teardown_resources()
|
cls._teardown_resources()
|
||||||
raise AssertionError(
|
raise unittest.SkipTest(
|
||||||
f"agent missing required tools: {', '.join(missing)} — "
|
f"agent missing required tools: {', '.join(missing)} — "
|
||||||
f"add them to the backend's base image"
|
f"add them to the backend's base image"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,96 +0,0 @@
|
|||||||
"""Architecture rules that should fail before coupling becomes entrenched."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import ast
|
|
||||||
import unittest
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
|
||||||
|
|
||||||
|
|
||||||
class TestCliBackendBoundaries(unittest.TestCase):
|
|
||||||
def test_cli_does_not_import_a_concrete_backend(self) -> None:
|
|
||||||
forbidden = (
|
|
||||||
"backend.docker", "backend.firecracker", "backend.macos_container",
|
|
||||||
"bot_bottle.backend.docker", "bot_bottle.backend.firecracker",
|
|
||||||
"bot_bottle.backend.macos_container",
|
|
||||||
)
|
|
||||||
violations: list[str] = []
|
|
||||||
for path in (ROOT / "bot_bottle" / "cli").rglob("*.py"):
|
|
||||||
tree = ast.parse(path.read_text(), filename=str(path))
|
|
||||||
for node in ast.walk(tree):
|
|
||||||
if isinstance(node, ast.ImportFrom):
|
|
||||||
module = node.module
|
|
||||||
if module and module.startswith(forbidden):
|
|
||||||
violations.append(
|
|
||||||
f"{path.relative_to(ROOT)}:{node.lineno}: {module}"
|
|
||||||
)
|
|
||||||
if isinstance(node, ast.Import):
|
|
||||||
violations.extend(
|
|
||||||
f"{path.relative_to(ROOT)}:{node.lineno}: {alias.name}"
|
|
||||||
for alias in node.names if alias.name.startswith(forbidden)
|
|
||||||
)
|
|
||||||
self.assertEqual([], violations, "generic CLI imports concrete backend internals:\n" +
|
|
||||||
"\n".join(violations))
|
|
||||||
|
|
||||||
|
|
||||||
class TestRuntimeModuleSizes(unittest.TestCase):
|
|
||||||
def test_no_runtime_module_grows_beyond_global_ceiling(self) -> None:
|
|
||||||
"""A coarse ceiling catches new monoliths; focused caps stay tighter."""
|
|
||||||
ceiling = 850
|
|
||||||
oversized = [
|
|
||||||
f"{path.relative_to(ROOT)} ({len(path.read_text().splitlines())})"
|
|
||||||
for path in (ROOT / "bot_bottle").rglob("*.py")
|
|
||||||
if len(path.read_text().splitlines()) > ceiling
|
|
||||||
]
|
|
||||||
self.assertEqual(
|
|
||||||
[], oversized,
|
|
||||||
f"runtime modules must stay at or below {ceiling} lines: "
|
|
||||||
+ ", ".join(oversized),
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_egress_modules_stay_focused(self) -> None:
|
|
||||||
caps = {
|
|
||||||
"addon_core.py": 100,
|
|
||||||
"schema.py": 400,
|
|
||||||
"types.py": 180,
|
|
||||||
"matching.py": 180,
|
|
||||||
"dlp.py": 180,
|
|
||||||
"context.py": 140,
|
|
||||||
}
|
|
||||||
directory = ROOT / "bot_bottle" / "gateway" / "egress"
|
|
||||||
oversized = [f"{name} ({len((directory / name).read_text().splitlines())}>{cap})"
|
|
||||||
for name, cap in caps.items()
|
|
||||||
if len((directory / name).read_text().splitlines()) > cap]
|
|
||||||
self.assertEqual([], oversized, "split a module rather than raising its cap: " +
|
|
||||||
", ".join(oversized))
|
|
||||||
|
|
||||||
def test_runtime_code_uses_focused_egress_modules(self) -> None:
|
|
||||||
"""addon_core is compatibility-only, never an internal dependency."""
|
|
||||||
violations: list[str] = []
|
|
||||||
package = ROOT / "bot_bottle"
|
|
||||||
facade = package / "gateway" / "egress" / "addon_core.py"
|
|
||||||
package_init = package / "gateway" / "egress" / "__init__.py"
|
|
||||||
for path in package.rglob("*.py"):
|
|
||||||
if path in (facade, package_init):
|
|
||||||
continue
|
|
||||||
text = path.read_text()
|
|
||||||
if "gateway.egress.addon_core import" in text or \
|
|
||||||
".addon_core import" in text:
|
|
||||||
violations.append(str(path.relative_to(ROOT)))
|
|
||||||
self.assertEqual([], violations)
|
|
||||||
|
|
||||||
def test_backend_contract_does_not_absorb_preparation_logic(self) -> None:
|
|
||||||
caps = {
|
|
||||||
ROOT / "bot_bottle" / "backend" / "base.py": 580,
|
|
||||||
ROOT / "bot_bottle" / "backend" / "preparation.py": 160,
|
|
||||||
}
|
|
||||||
oversized = [
|
|
||||||
f"{path.relative_to(ROOT)} "
|
|
||||||
f"({len(path.read_text().splitlines())}>{cap})"
|
|
||||||
for path, cap in caps.items()
|
|
||||||
if len(path.read_text().splitlines()) > cap
|
|
||||||
]
|
|
||||||
self.assertEqual([], oversized)
|
|
||||||
@@ -48,13 +48,12 @@ class TestSharedReprovision(unittest.TestCase):
|
|||||||
client.reprovision_gateway.side_effect = [
|
client.reprovision_gateway.side_effect = [
|
||||||
OrchestratorClientError("bad key"), True,
|
OrchestratorClientError("bad key"), True,
|
||||||
]
|
]
|
||||||
with patch("bot_bottle.orchestrator.reprovision.debug") as debug:
|
self.assertEqual(
|
||||||
count = reprovision_bottles(
|
1,
|
||||||
|
reprovision_bottles(
|
||||||
client, {"10.0.0.1": "key-1", "10.0.0.2": "key-2"},
|
client, {"10.0.0.1": "key-1", "10.0.0.2": "key-2"},
|
||||||
)
|
),
|
||||||
self.assertEqual(1, count)
|
)
|
||||||
self.assertEqual("b1", debug.call_args.kwargs["context"]["bottle_id"])
|
|
||||||
self.assertNotIn("bad key", repr(debug.call_args))
|
|
||||||
|
|
||||||
|
|
||||||
class TestMacosReprovision(unittest.TestCase):
|
class TestMacosReprovision(unittest.TestCase):
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
from bot_bottle.cli.tui import _filter_items, _multiselect_loop, filter_multiselect, filter_select
|
from bot_bottle.cli.tui import _filter_items, _multiselect_loop, filter_multiselect, filter_select
|
||||||
|
|
||||||
@@ -50,10 +49,8 @@ class TestFilterSelectEmptyItems(unittest.TestCase):
|
|||||||
|
|
||||||
def test_returns_none_when_tty_unavailable(self):
|
def test_returns_none_when_tty_unavailable(self):
|
||||||
# /nonexistent is guaranteed to not open.
|
# /nonexistent is guaranteed to not open.
|
||||||
with patch("bot_bottle.cli.tui.debug") as debug:
|
result = filter_select(["a", "b"], tty_path="/nonexistent/tty")
|
||||||
result = filter_select(["a", "b"], tty_path="/nonexistent/tty")
|
|
||||||
self.assertIsNone(result)
|
self.assertIsNone(result)
|
||||||
self.assertEqual("FileNotFoundError", debug.call_args.kwargs["context"]["error_type"])
|
|
||||||
|
|
||||||
|
|
||||||
class TestFilterMultiselectEmptyItems(unittest.TestCase):
|
class TestFilterMultiselectEmptyItems(unittest.TestCase):
|
||||||
@@ -63,10 +60,8 @@ class TestFilterMultiselectEmptyItems(unittest.TestCase):
|
|||||||
self.assertEqual([], result)
|
self.assertEqual([], result)
|
||||||
|
|
||||||
def test_returns_none_when_tty_unavailable(self):
|
def test_returns_none_when_tty_unavailable(self):
|
||||||
with patch("bot_bottle.cli.tui.debug") as debug:
|
result = filter_multiselect(["a", "b"], tty_path="/nonexistent/tty")
|
||||||
result = filter_multiselect(["a", "b"], tty_path="/nonexistent/tty")
|
|
||||||
self.assertIsNone(result)
|
self.assertIsNone(result)
|
||||||
self.assertEqual("FileNotFoundError", debug.call_args.kwargs["context"]["error_type"])
|
|
||||||
|
|
||||||
|
|
||||||
class TestMultiselectLoopReordering(unittest.TestCase):
|
class TestMultiselectLoopReordering(unittest.TestCase):
|
||||||
|
|||||||
@@ -1,101 +0,0 @@
|
|||||||
"""Tests for the fail-closed critical coverage manifest."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import tempfile
|
|
||||||
import unittest
|
|
||||||
from contextlib import redirect_stderr, redirect_stdout
|
|
||||||
from io import StringIO
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from scripts.critical_modules import (
|
|
||||||
DEFAULT_MANIFEST,
|
|
||||||
REPO_ROOT,
|
|
||||||
CriticalModulesError,
|
|
||||||
load_critical_modules,
|
|
||||||
main,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestCriticalModules(unittest.TestCase):
|
|
||||||
def test_repository_manifest_is_valid(self) -> None:
|
|
||||||
modules = load_critical_modules(DEFAULT_MANIFEST, root=REPO_ROOT)
|
|
||||||
self.assertGreater(len(modules), 20)
|
|
||||||
self.assertEqual(len(modules), len(set(modules)))
|
|
||||||
|
|
||||||
def test_missing_module_fails(self) -> None:
|
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
|
||||||
root = Path(tmp)
|
|
||||||
manifest = root / "critical-modules.txt"
|
|
||||||
manifest.write_text("bot_bottle/renamed.py\n", encoding="utf-8")
|
|
||||||
with self.assertRaisesRegex(CriticalModulesError, "does not exist"):
|
|
||||||
load_critical_modules(manifest, root=root)
|
|
||||||
|
|
||||||
def test_duplicate_module_fails(self) -> None:
|
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
|
||||||
root = Path(tmp)
|
|
||||||
module = root / "bot_bottle" / "core.py"
|
|
||||||
module.parent.mkdir()
|
|
||||||
module.write_text("", encoding="utf-8")
|
|
||||||
manifest = root / "critical-modules.txt"
|
|
||||||
manifest.write_text(
|
|
||||||
"bot_bottle/core.py\nbot_bottle/core.py\n", encoding="utf-8"
|
|
||||||
)
|
|
||||||
with self.assertRaisesRegex(CriticalModulesError, "duplicated"):
|
|
||||||
load_critical_modules(manifest, root=root)
|
|
||||||
|
|
||||||
def test_entry_cannot_escape_repository(self) -> None:
|
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
|
||||||
root = Path(tmp)
|
|
||||||
manifest = root / "critical-modules.txt"
|
|
||||||
manifest.write_text("../outside.py\n", encoding="utf-8")
|
|
||||||
with self.assertRaisesRegex(CriticalModulesError, "escapes"):
|
|
||||||
load_critical_modules(manifest, root=root)
|
|
||||||
|
|
||||||
def test_invalid_entry_forms_are_reported_together(self) -> None:
|
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
|
||||||
root = Path(tmp)
|
|
||||||
manifest = root / "critical-modules.txt"
|
|
||||||
manifest.write_text(
|
|
||||||
f"{root / 'absolute.py'}\nREADME.md\n",
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
with self.assertRaises(CriticalModulesError) as raised:
|
|
||||||
load_critical_modules(manifest, root=root)
|
|
||||||
self.assertIn("must be relative", str(raised.exception))
|
|
||||||
self.assertIn("is not a Python module", str(raised.exception))
|
|
||||||
|
|
||||||
def test_empty_manifest_fails(self) -> None:
|
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
|
||||||
root = Path(tmp)
|
|
||||||
manifest = root / "critical-modules.txt"
|
|
||||||
manifest.write_text("# comments do not define modules\n", encoding="utf-8")
|
|
||||||
with self.assertRaisesRegex(CriticalModulesError, "contains no"):
|
|
||||||
load_critical_modules(manifest, root=root)
|
|
||||||
|
|
||||||
def test_unreadable_manifest_fails(self) -> None:
|
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
|
||||||
missing = Path(tmp) / "missing.txt"
|
|
||||||
with self.assertRaisesRegex(CriticalModulesError, "cannot read"):
|
|
||||||
load_critical_modules(missing, root=Path(tmp))
|
|
||||||
|
|
||||||
def test_main_prints_include_list_or_checks_silently(self) -> None:
|
|
||||||
output = StringIO()
|
|
||||||
with redirect_stdout(output):
|
|
||||||
self.assertEqual(0, main([]))
|
|
||||||
self.assertIn("bot_bottle/manifest/egress.py", output.getvalue())
|
|
||||||
|
|
||||||
output = StringIO()
|
|
||||||
with redirect_stdout(output):
|
|
||||||
self.assertEqual(0, main(["--check"]))
|
|
||||||
self.assertEqual("", output.getvalue())
|
|
||||||
|
|
||||||
def test_main_reports_manifest_error(self) -> None:
|
|
||||||
error = StringIO()
|
|
||||||
with redirect_stderr(error):
|
|
||||||
self.assertEqual(1, main(["--manifest", "/definitely/missing"]))
|
|
||||||
self.assertIn("critical-modules:", error.getvalue())
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -9,7 +9,6 @@ a freshly minted token."""
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
from bot_bottle.backend.docker.infra import (
|
from bot_bottle.backend.docker.infra import (
|
||||||
@@ -68,21 +67,6 @@ class TestDockerInfraService(unittest.TestCase):
|
|||||||
self.assertTrue(any(GATEWAY_NAME in a for a in rms))
|
self.assertTrue(any(GATEWAY_NAME in a for a in rms))
|
||||||
self.assertTrue(any(ORCHESTRATOR_NAME in a for a in rms))
|
self.assertTrue(any(ORCHESTRATOR_NAME in a for a in rms))
|
||||||
|
|
||||||
def test_named_mounts_propagate_to_both_planes(self) -> None:
|
|
||||||
svc = DockerInfraService(
|
|
||||||
root_mount_source="registry-volume",
|
|
||||||
gateway_ca_mount_source="ca-volume",
|
|
||||||
)
|
|
||||||
self.assertEqual("registry-volume", svc.orchestrator()._root_mount_source)
|
|
||||||
self.assertEqual("ca-volume", svc.gateway()._ca_mount_source)
|
|
||||||
|
|
||||||
def test_host_path_and_named_root_mount_are_mutually_exclusive(self) -> None:
|
|
||||||
with self.assertRaisesRegex(ValueError, "host_root or root_mount_source"):
|
|
||||||
DockerInfraService(
|
|
||||||
host_root=Path("/host/path"),
|
|
||||||
root_mount_source="registry-volume",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
import urllib.error
|
import urllib.error
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import MagicMock, Mock, patch
|
from unittest.mock import MagicMock, Mock, patch
|
||||||
|
|
||||||
from bot_bottle.backend.docker.orchestrator import (
|
from bot_bottle.backend.docker.orchestrator import (
|
||||||
@@ -48,55 +47,6 @@ class TestDockerOrchestrator(unittest.TestCase):
|
|||||||
def test_url_is_host_loopback(self) -> None:
|
def test_url_is_host_loopback(self) -> None:
|
||||||
self.assertEqual("http://127.0.0.1:8099", self.orch.url())
|
self.assertEqual("http://127.0.0.1:8099", self.orch.url())
|
||||||
|
|
||||||
def test_socket_shared_client_uses_explicit_host_and_open_bind(self) -> None:
|
|
||||||
orch = DockerOrchestrator(
|
|
||||||
port=8099, client_host="172.17.0.1", root_mount_source="state-volume"
|
|
||||||
)
|
|
||||||
with patch(_TOKEN, return_value="k"), \
|
|
||||||
patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
|
||||||
patch(_RUN, return_value=_proc()) as run, patch(_SLEEP):
|
|
||||||
orch.ensure_running()
|
|
||||||
self.assertEqual("http://172.17.0.1:8099", orch.url())
|
|
||||||
argv = next(c.args[0] for c in run.call_args_list
|
|
||||||
if c.args[0][:2] == ["docker", "run"])
|
|
||||||
self.assertEqual("0.0.0.0:8099:8099", argv[argv.index("--publish") + 1])
|
|
||||||
self.assertIn("state-volume:/bot-bottle-root", argv)
|
|
||||||
|
|
||||||
def test_host_path_and_named_root_mount_are_mutually_exclusive(self) -> None:
|
|
||||||
with self.assertRaisesRegex(ValueError, "host_root or root_mount_source"):
|
|
||||||
DockerOrchestrator(
|
|
||||||
host_root=Path("/host/path"),
|
|
||||||
root_mount_source="state-volume",
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_socket_shared_job_network_uses_container_dns(self) -> None:
|
|
||||||
orch = DockerOrchestrator(
|
|
||||||
name="orchestrator-itest",
|
|
||||||
port=22001,
|
|
||||||
client_network="runner-job-network",
|
|
||||||
root_mount_source="state-volume",
|
|
||||||
)
|
|
||||||
with patch(_TOKEN, return_value="k"), \
|
|
||||||
patch(
|
|
||||||
_URLOPEN,
|
|
||||||
side_effect=[urllib.error.URLError("down"), _health(200)],
|
|
||||||
), patch(_RUN, return_value=_proc()) as run, patch(_SLEEP):
|
|
||||||
orch.ensure_running()
|
|
||||||
self.assertEqual("http://orchestrator-itest:8099", orch.url())
|
|
||||||
calls = [call.args[0] for call in run.call_args_list]
|
|
||||||
self.assertIn(
|
|
||||||
[
|
|
||||||
"docker", "network", "connect",
|
|
||||||
"runner-job-network", "orchestrator-itest",
|
|
||||||
],
|
|
||||||
calls,
|
|
||||||
)
|
|
||||||
argv = next(call for call in calls if call[:2] == ["docker", "run"])
|
|
||||||
self.assertEqual(
|
|
||||||
"127.0.0.1:22001:8099",
|
|
||||||
argv[argv.index("--publish") + 1],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_gateway_url_is_the_container_dns_name(self) -> None:
|
def test_gateway_url_is_the_container_dns_name(self) -> None:
|
||||||
# The gateway reaches the orchestrator by name on the control network.
|
# The gateway reaches the orchestrator by name on the control network.
|
||||||
self.assertEqual(f"http://{ORCHESTRATOR_NAME}:8099", self.orch.gateway_url())
|
self.assertEqual(f"http://{ORCHESTRATOR_NAME}:8099", self.orch.gateway_url())
|
||||||
@@ -160,8 +110,6 @@ class TestDockerOrchestrator(unittest.TestCase):
|
|||||||
# The lean control plane: no mitmproxy CA mount, no gateway daemons.
|
# The lean control plane: no mitmproxy CA mount, no gateway daemons.
|
||||||
self.assertFalse([a for a in argv if a.endswith(":/home/mitmproxy/.mitmproxy")])
|
self.assertFalse([a for a in argv if a.endswith(":/home/mitmproxy/.mitmproxy")])
|
||||||
self.assertNotIn("BOT_BOTTLE_GATEWAY_DAEMONS", " ".join(argv))
|
self.assertNotIn("BOT_BOTTLE_GATEWAY_DAEMONS", " ".join(argv))
|
||||||
self.assertNotIn("/bot-bottle-src", " ".join(argv))
|
|
||||||
self.assertNotIn("PYTHONPATH", " ".join(argv))
|
|
||||||
# Orchestrator entrypoint args (image ENTRYPOINT is `-m bot_bottle.orchestrator`).
|
# Orchestrator entrypoint args (image ENTRYPOINT is `-m bot_bottle.orchestrator`).
|
||||||
self.assertIn("--broker", argv)
|
self.assertIn("--broker", argv)
|
||||||
self.assertIn("stub", argv)
|
self.assertIn("stub", argv)
|
||||||
|
|||||||
@@ -8,18 +8,16 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from bot_bottle.gateway.egress.matching import evaluate_matches
|
from bot_bottle.gateway.egress.addon_core import (
|
||||||
from bot_bottle.gateway.egress.schema import (
|
|
||||||
load_config,
|
|
||||||
parse_config,
|
|
||||||
parse_routes,
|
|
||||||
route_to_yaml_dict,
|
|
||||||
)
|
|
||||||
from bot_bottle.gateway.egress.types import (
|
|
||||||
HeaderMatch,
|
HeaderMatch,
|
||||||
MatchEntry,
|
MatchEntry,
|
||||||
PathMatch,
|
PathMatch,
|
||||||
Route,
|
Route,
|
||||||
|
evaluate_matches,
|
||||||
|
load_config,
|
||||||
|
parse_config,
|
||||||
|
parse_routes,
|
||||||
|
route_to_yaml_dict,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -3,16 +3,15 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
from bot_bottle.gateway.egress.context import (
|
from bot_bottle.gateway.egress.addon_core import (
|
||||||
DENY_RESOLVER_ERROR,
|
DENY_RESOLVER_ERROR,
|
||||||
DENY_UNATTRIBUTED,
|
DENY_UNATTRIBUTED,
|
||||||
DENY_UNPARSEABLE,
|
DENY_UNPARSEABLE,
|
||||||
|
decide,
|
||||||
resolve_client_config,
|
resolve_client_config,
|
||||||
resolve_client_context,
|
resolve_client_context,
|
||||||
)
|
)
|
||||||
from bot_bottle.gateway.egress.matching import decide
|
|
||||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError
|
from bot_bottle.gateway.policy_resolver import PolicyResolveError
|
||||||
|
|
||||||
|
|
||||||
@@ -45,13 +44,7 @@ class TestResolveClientConfig(unittest.TestCase):
|
|||||||
|
|
||||||
def test_resolver_error_denies_all(self) -> None:
|
def test_resolver_error_denies_all(self) -> None:
|
||||||
# Orchestrator unreachable/errored must never widen egress.
|
# Orchestrator unreachable/errored must never widen egress.
|
||||||
with patch("bot_bottle.gateway.egress.context.debug") as debug:
|
self.assertEqual((), resolve_client_config(_FakeResolver(raises=True), "10.243.0.1").routes)
|
||||||
config = resolve_client_config(_FakeResolver(raises=True), "10.243.0.1")
|
|
||||||
self.assertEqual((), config.routes)
|
|
||||||
self.assertEqual(
|
|
||||||
"PolicyResolveError", debug.call_args.kwargs["context"]["error_type"],
|
|
||||||
)
|
|
||||||
self.assertNotIn("orchestrator down", repr(debug.call_args))
|
|
||||||
|
|
||||||
def test_unparseable_policy_denies_all(self) -> None:
|
def test_unparseable_policy_denies_all(self) -> None:
|
||||||
cfg = resolve_client_config(_FakeResolver(result="routes: notalist\n"), "10.243.0.1")
|
cfg = resolve_client_config(_FakeResolver(result="routes: notalist\n"), "10.243.0.1")
|
||||||
|
|||||||
@@ -404,19 +404,16 @@ class TestBootArgs(unittest.TestCase):
|
|||||||
self.assertEqual("bbfc0", cfg["network-interfaces"][0]["host_dev_name"])
|
self.assertEqual("bbfc0", cfg["network-interfaces"][0]["host_dev_name"])
|
||||||
self.assertEqual(1, len(cfg["drives"])) # no data drive by default
|
self.assertEqual(1, len(cfg["drives"])) # no data drive by default
|
||||||
|
|
||||||
def test_config_adds_data_drives_in_order(self):
|
def test_config_adds_data_drive(self):
|
||||||
cfg = cast(Any, firecracker_vm._config(
|
cfg = cast(Any, firecracker_vm._config(
|
||||||
rootfs=Path("/run/rootfs.ext4"), tap="bbfc0",
|
rootfs=Path("/run/rootfs.ext4"), tap="bbfc0",
|
||||||
guest_ip="100.64.0.1", host_ip="100.64.0.0", pubkey="k",
|
guest_ip="100.64.0.1", host_ip="100.64.0.0", pubkey="k",
|
||||||
vcpus=2, mem_mib=2048, guest_mac="06:00:AC:10:00:02",
|
vcpus=2, mem_mib=2048, guest_mac="06:00:AC:10:00:02",
|
||||||
data_drives=(Path("/run/ca.ext4"), Path("/run/git.ext4")),
|
data_drive=Path("/run/registry.ext4"),
|
||||||
))
|
))
|
||||||
# rootfs (vda) + two data drives, attached in list order so the guest
|
self.assertEqual(2, len(cfg["drives"]))
|
||||||
# sees them as /dev/vdb, /dev/vdc — an order callers depend on.
|
|
||||||
self.assertEqual(3, len(cfg["drives"]))
|
|
||||||
self.assertFalse(cfg["drives"][1]["is_root_device"])
|
self.assertFalse(cfg["drives"][1]["is_root_device"])
|
||||||
self.assertEqual("/run/ca.ext4", cfg["drives"][1]["path_on_host"])
|
self.assertEqual("/run/registry.ext4", cfg["drives"][1]["path_on_host"])
|
||||||
self.assertEqual("/run/git.ext4", cfg["drives"][2]["path_on_host"])
|
|
||||||
|
|
||||||
|
|
||||||
class TestBottleExecClose(unittest.TestCase):
|
class TestBottleExecClose(unittest.TestCase):
|
||||||
|
|||||||
@@ -59,13 +59,7 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
|
|||||||
def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None:
|
def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None:
|
||||||
gw = FirecrackerGateway()
|
gw = FirecrackerGateway()
|
||||||
booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))
|
booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))
|
||||||
ca_vol = Path("/gw/gateway-ca.ext4")
|
|
||||||
git_vol = Path("/gw/gateway-git.ext4")
|
|
||||||
with patch.object(infra_vm, "boot_vm", return_value=booted) as boot, \
|
with patch.object(infra_vm, "boot_vm", return_value=booted) as boot, \
|
||||||
patch.object(FirecrackerGateway, "_ensure_ca_volume",
|
|
||||||
return_value=ca_vol), \
|
|
||||||
patch.object(FirecrackerGateway, "_ensure_git_volume",
|
|
||||||
return_value=git_vol), \
|
|
||||||
patch.object(infra_vm, "push_secret") as push:
|
patch.object(infra_vm, "push_secret") as push:
|
||||||
gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
||||||
# Booted on the gateway link with the gateway role; the orchestrator's
|
# Booted on the gateway link with the gateway role; the orchestrator's
|
||||||
@@ -73,71 +67,12 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
|
|||||||
kw = boot.call_args.kwargs
|
kw = boot.call_args.kwargs
|
||||||
self.assertEqual("gateway", kw["role"])
|
self.assertEqual("gateway", kw["role"])
|
||||||
self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"])
|
self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"])
|
||||||
# The persistent volumes ride in a FIXED order — CA as /dev/vdb, git-gate
|
self.assertIsNone(kw.get("data_drive")) # data plane never opens the DB
|
||||||
# as /dev/vdc — so both survive a gateway-VM rebuild (issues #450, #512).
|
|
||||||
self.assertEqual((ca_vol, git_vol), kw.get("data_drives"))
|
|
||||||
# The host-minted token (never the key — #469) is pushed to the guest.
|
# The host-minted token (never the key — #469) is pushed to the guest.
|
||||||
push.assert_called_once()
|
push.assert_called_once()
|
||||||
self.assertEqual(_TOKEN, push.call_args.args[1])
|
self.assertEqual(_TOKEN, push.call_args.args[1])
|
||||||
|
|
||||||
|
|
||||||
class TestGatewayCaVolume(unittest.TestCase):
|
|
||||||
"""The persistent CA volume that keeps the mitmproxy CA stable across a
|
|
||||||
gateway-VM rebuild (issue #450) — the mirror of the orchestrator's registry
|
|
||||||
volume."""
|
|
||||||
|
|
||||||
def test_reuses_existing_volume(self) -> None:
|
|
||||||
gw = FirecrackerGateway()
|
|
||||||
with tempfile.TemporaryDirectory() as td:
|
|
||||||
vol = Path(td) / "gateway-ca.ext4"
|
|
||||||
vol.write_bytes(b"") # already present
|
|
||||||
with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
|
|
||||||
patch(f"{_GW}.subprocess.run") as run:
|
|
||||||
out = gw._ensure_ca_volume()
|
|
||||||
run.assert_not_called() # no mke2fs when it exists — the CA survives
|
|
||||||
self.assertEqual(vol, out)
|
|
||||||
|
|
||||||
def test_creates_volume_when_missing(self) -> None:
|
|
||||||
gw = FirecrackerGateway()
|
|
||||||
with tempfile.TemporaryDirectory() as td:
|
|
||||||
with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
|
|
||||||
patch(f"{_GW}.subprocess.run",
|
|
||||||
return_value=CompletedProcess([], 0)) as run:
|
|
||||||
out = gw._ensure_ca_volume()
|
|
||||||
argv = run.call_args.args[0]
|
|
||||||
self.assertIn("mke2fs", argv)
|
|
||||||
self.assertEqual(str(Path(td) / "gateway-ca.ext4"), out.__fspath__())
|
|
||||||
self.assertIn(str(out), argv)
|
|
||||||
|
|
||||||
|
|
||||||
class TestGatewayGitVolume(unittest.TestCase):
|
|
||||||
"""The persistent git-gate volume that keeps per-bottle bare repos + creds
|
|
||||||
stable across a gateway-VM rebuild (issue #512)."""
|
|
||||||
|
|
||||||
def test_reuses_existing_volume(self) -> None:
|
|
||||||
gw = FirecrackerGateway()
|
|
||||||
with tempfile.TemporaryDirectory() as td:
|
|
||||||
vol = Path(td) / "gateway-git.ext4"
|
|
||||||
vol.write_bytes(b"") # already present
|
|
||||||
with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
|
|
||||||
patch(f"{_GW}.subprocess.run") as run:
|
|
||||||
out = gw._ensure_git_volume()
|
|
||||||
run.assert_not_called() # no mke2fs when it exists — the repos survive
|
|
||||||
self.assertEqual(vol, out)
|
|
||||||
|
|
||||||
def test_creates_volume_when_missing(self) -> None:
|
|
||||||
gw = FirecrackerGateway()
|
|
||||||
with tempfile.TemporaryDirectory() as td:
|
|
||||||
with patch.object(infra_vm, "_gw_dir", return_value=Path(td)), \
|
|
||||||
patch(f"{_GW}.subprocess.run",
|
|
||||||
return_value=CompletedProcess([], 0)) as run:
|
|
||||||
out = gw._ensure_git_volume()
|
|
||||||
argv = run.call_args.args[0]
|
|
||||||
self.assertIn("mke2fs", argv)
|
|
||||||
self.assertEqual(str(Path(td) / "gateway-git.ext4"), out.__fspath__())
|
|
||||||
self.assertIn(str(out), argv)
|
|
||||||
|
|
||||||
|
|
||||||
class TestFirecrackerGatewaySurface(unittest.TestCase):
|
class TestFirecrackerGatewaySurface(unittest.TestCase):
|
||||||
def test_is_running_reads_the_gateway_pidfile(self) -> None:
|
def test_is_running_reads_the_gateway_pidfile(self) -> None:
|
||||||
gw = FirecrackerGateway()
|
gw = FirecrackerGateway()
|
||||||
|
|||||||
@@ -61,23 +61,7 @@ class TestRoleInits(unittest.TestCase):
|
|||||||
init = infra_vm.role_init("gateway")
|
init = infra_vm.role_init("gateway")
|
||||||
self.assertNotIn("bb_role=", init)
|
self.assertNotIn("bb_role=", init)
|
||||||
self.assertNotIn("bot_bottle.orchestrator", init)
|
self.assertNotIn("bot_bottle.orchestrator", init)
|
||||||
# Persistent CA volume mounted at mitmproxy's confdir so the CA survives
|
self.assertNotIn("/dev/vdb", init) # no registry volume on the data plane
|
||||||
# a gateway-VM rebuild (issue #450). Mounted BEFORE the data plane starts.
|
|
||||||
self.assertIn(f"mount -t ext4 /dev/vdb {infra_vm._GATEWAY_CA_MOUNT}", init)
|
|
||||||
self.assertLess(init.index("/dev/vdb"),
|
|
||||||
init.index("bot_bottle.gateway.bootstrap"))
|
|
||||||
# Persistent git-gate volume (/dev/vdc) bind-mounted onto /git and
|
|
||||||
# /git-gate/creds so per-bottle repos + creds survive a rebuild (#512),
|
|
||||||
# also before the data plane (and any provisioning writes).
|
|
||||||
self.assertIn(f"mount -t ext4 /dev/vdc {infra_vm._GATEWAY_GIT_MOUNT}", init)
|
|
||||||
self.assertIn(
|
|
||||||
f"mount --bind {infra_vm._GATEWAY_GIT_MOUNT}/git "
|
|
||||||
f"{infra_vm._GATEWAY_GIT_REPO_ROOT}", init)
|
|
||||||
self.assertIn(
|
|
||||||
f"mount --bind {infra_vm._GATEWAY_GIT_MOUNT}/creds "
|
|
||||||
f"{infra_vm._GATEWAY_GIT_CREDS_DIR}", init)
|
|
||||||
self.assertLess(init.index("/dev/vdc"),
|
|
||||||
init.index("bot_bottle.gateway.bootstrap"))
|
|
||||||
self.assertIn("export PATH=", init) # shared preamble
|
self.assertIn("export PATH=", init) # shared preamble
|
||||||
self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init)
|
self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", init)
|
||||||
self.assertIn(f"cat {infra_vm._GUEST_GATEWAY_JWT_PATH}", init) # host-minted JWT
|
self.assertIn(f"cat {infra_vm._GUEST_GATEWAY_JWT_PATH}", init) # host-minted JWT
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ class TestEnsureRunning(unittest.TestCase):
|
|||||||
kw = boot.call_args.kwargs
|
kw = boot.call_args.kwargs
|
||||||
self.assertEqual("orchestrator", kw["role"])
|
self.assertEqual("orchestrator", kw["role"])
|
||||||
self.assertEqual(4096, kw["mem_mib"]) # orchestrator keeps build headroom
|
self.assertEqual(4096, kw["mem_mib"]) # orchestrator keeps build headroom
|
||||||
self.assertEqual((Path("/reg"),), kw["data_drives"]) # DB volume on the CP
|
self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP
|
||||||
# The host-canonical signing key is pushed to the guest signing-key path.
|
# The host-canonical signing key is pushed to the guest signing-key path.
|
||||||
self.assertEqual("host-key", push.call_args.args[1])
|
self.assertEqual("host-key", push.call_args.args[1])
|
||||||
self.assertEqual(infra_vm._GUEST_SIGNING_KEY_PATH, push.call_args.args[2])
|
self.assertEqual(infra_vm._GUEST_SIGNING_KEY_PATH, push.call_args.args[2])
|
||||||
|
|||||||
@@ -206,75 +206,6 @@ class TestHookRender(unittest.TestCase):
|
|||||||
self.assertIn('set -- "$@" --push-option="$opt"', hook)
|
self.assertIn('set -- "$@" --push-option="$opt"', hook)
|
||||||
self.assertIn('git push "$@" origin "$refspec"', hook)
|
self.assertIn('git push "$@" origin "$refspec"', hook)
|
||||||
|
|
||||||
def test_agit_review_refs_rejected_before_scan(self):
|
|
||||||
# Creating/updating refs/for/*, refs/draft/*, or refs/for-review/*
|
|
||||||
# opens a Gitea AGit pull request backed by a server-managed review
|
|
||||||
# ref instead of a normal branch, which the git-gate branch workflow
|
|
||||||
# can't push follow-ups to. The guard rejects those refs, and it runs
|
|
||||||
# in Phase 0 — before the gitleaks scan and the upstream forward.
|
|
||||||
hook = git_gate_render_hook()
|
|
||||||
self.assertIn(
|
|
||||||
"refs/for/*|refs/draft/*|refs/for-review/*", hook,
|
|
||||||
)
|
|
||||||
self.assertIn("refusing AGit review ref", hook)
|
|
||||||
self.assertIn("branch-backed pull request", hook)
|
|
||||||
guard = hook.index("refusing AGit review ref")
|
|
||||||
self.assertLess(
|
|
||||||
guard, hook.index("gitleaks scanning"),
|
|
||||||
"AGit guard must run before the gitleaks scan",
|
|
||||||
)
|
|
||||||
self.assertLess(
|
|
||||||
guard, hook.index("forwarding $ref to origin"),
|
|
||||||
"AGit guard must run before the upstream forward",
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_agit_review_ref_deletion_still_allowed(self):
|
|
||||||
# Cleanup of a legacy AGit ref (new == zero) must not be blocked, so
|
|
||||||
# the reject is guarded by the same delete short-circuit the scan and
|
|
||||||
# forward phases use.
|
|
||||||
hook = git_gate_render_hook()
|
|
||||||
guard_block = hook[
|
|
||||||
hook.index("Phase 0"):hook.index("supervise_gitleaks_allow()")
|
|
||||||
]
|
|
||||||
self.assertIn('[ "$new" = "$zero" ] && continue', guard_block)
|
|
||||||
self.assertIn("refs/for/*", guard_block)
|
|
||||||
|
|
||||||
def _run_hook_stdin(self, stdin: str):
|
|
||||||
# Execute the rendered hook far enough to exercise Phase 0. The guard
|
|
||||||
# touches only mktemp + read, so it rejects (or falls through) without
|
|
||||||
# a bare repo, gitleaks, or ssh — anything past Phase 0 fails for
|
|
||||||
# unrelated reasons, which is fine for the reject cases asserted here.
|
|
||||||
import subprocess
|
|
||||||
fd, path = tempfile.mkstemp(suffix=".sh")
|
|
||||||
try:
|
|
||||||
with os.fdopen(fd, "w") as f:
|
|
||||||
f.write(git_gate_render_hook())
|
|
||||||
return subprocess.run(
|
|
||||||
["sh", path], input=stdin, capture_output=True, text=True,
|
|
||||||
)
|
|
||||||
finally:
|
|
||||||
os.unlink(path)
|
|
||||||
|
|
||||||
def test_agit_review_ref_create_is_rejected(self):
|
|
||||||
one = "1" * 40
|
|
||||||
for ref in ("refs/for/main", "refs/draft/main", "refs/for-review/main"):
|
|
||||||
with self.subTest(ref=ref):
|
|
||||||
result = self._run_hook_stdin(f"{'0' * 40} {one} {ref}\n")
|
|
||||||
self.assertEqual(1, result.returncode)
|
|
||||||
self.assertIn("refusing AGit review ref", result.stderr)
|
|
||||||
self.assertIn("branch-backed pull request", result.stderr)
|
|
||||||
# Rejected in Phase 0, before the gitleaks scan runs.
|
|
||||||
self.assertNotIn("gitleaks scanning", result.stderr)
|
|
||||||
|
|
||||||
def test_ordinary_branch_passes_agit_guard(self):
|
|
||||||
# A normal refs/heads push must fall through Phase 0 and reach the
|
|
||||||
# gitleaks scan (which then fails for lack of a real repo — proving
|
|
||||||
# only that the guard did not short-circuit it).
|
|
||||||
one = "1" * 40
|
|
||||||
result = self._run_hook_stdin(f"{'0' * 40} {one} refs/heads/main\n")
|
|
||||||
self.assertNotIn("refusing AGit review ref", result.stderr)
|
|
||||||
self.assertIn("gitleaks scanning", result.stderr)
|
|
||||||
|
|
||||||
def test_inline_gitleaks_allow_routes_to_supervisor(self):
|
def test_inline_gitleaks_allow_routes_to_supervisor(self):
|
||||||
hook = git_gate_render_hook()
|
hook = git_gate_render_hook()
|
||||||
# First gitleaks runs normally; only if that passes does the
|
# First gitleaks runs normally; only if that passes does the
|
||||||
|
|||||||
@@ -12,9 +12,7 @@ from bot_bottle.orchestrator.client import (
|
|||||||
OrchestratorClient,
|
OrchestratorClient,
|
||||||
OrchestratorClientError,
|
OrchestratorClientError,
|
||||||
RegisteredBottle,
|
RegisteredBottle,
|
||||||
BackendProbeFailure,
|
|
||||||
_host_auth_token,
|
_host_auth_token,
|
||||||
_probe_failure,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
_URLOPEN = "bot_bottle.orchestrator.client.urllib.request.urlopen"
|
_URLOPEN = "bot_bottle.orchestrator.client.urllib.request.urlopen"
|
||||||
@@ -35,15 +33,6 @@ class TestHostAuthToken(unittest.TestCase):
|
|||||||
self.assertEqual("", _host_auth_token())
|
self.assertEqual("", _host_auth_token())
|
||||||
|
|
||||||
|
|
||||||
class TestBackendProbeFailure(unittest.TestCase):
|
|
||||||
def test_records_safe_typed_diagnostic(self) -> None:
|
|
||||||
with patch("bot_bottle.orchestrator.client.debug") as debug:
|
|
||||||
result = _probe_failure("firecracker", RuntimeError("secret detail"))
|
|
||||||
self.assertEqual(BackendProbeFailure("firecracker", "RuntimeError"), result)
|
|
||||||
rendered = repr(debug.call_args)
|
|
||||||
self.assertNotIn("secret detail", rendered)
|
|
||||||
|
|
||||||
|
|
||||||
def _resp(status: int, payload: object) -> MagicMock:
|
def _resp(status: int, payload: object) -> MagicMock:
|
||||||
m = MagicMock()
|
m = MagicMock()
|
||||||
inner = m.__enter__.return_value
|
inner = m.__enter__.return_value
|
||||||
|
|||||||
@@ -7,10 +7,7 @@ import unittest
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
from bot_bottle.backend.docker.gateway import (
|
from bot_bottle.backend.docker.gateway import DockerGateway
|
||||||
DEFAULT_GATEWAY_SUBNET,
|
|
||||||
DockerGateway,
|
|
||||||
)
|
|
||||||
from bot_bottle.gateway import (
|
from bot_bottle.gateway import (
|
||||||
GATEWAY_CA_CERT,
|
GATEWAY_CA_CERT,
|
||||||
GATEWAY_NAME,
|
GATEWAY_NAME,
|
||||||
@@ -145,42 +142,6 @@ class TestDockerGateway(unittest.TestCase):
|
|||||||
# Data plane resolves policy against the orchestrator control plane.
|
# Data plane resolves policy against the orchestrator control plane.
|
||||||
self.assertIn(f"BOT_BOTTLE_ORCHESTRATOR_URL={_ORCH_URL}", runs[0])
|
self.assertIn(f"BOT_BOTTLE_ORCHESTRATOR_URL={_ORCH_URL}", runs[0])
|
||||||
|
|
||||||
def test_named_ca_volume_supports_socket_shared_runner(self) -> None:
|
|
||||||
sc = DockerGateway(
|
|
||||||
"bot-bottle-gateway:latest", ca_mount_source="ci-ca-volume"
|
|
||||||
)
|
|
||||||
|
|
||||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
|
||||||
return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc()
|
|
||||||
|
|
||||||
with patch(_RUN_DOCKER, side_effect=fake) as run:
|
|
||||||
sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
|
||||||
argv = next(c.args[0] for c in run.call_args_list
|
|
||||||
if c.args[0][:2] == ["docker", "run"])
|
|
||||||
self.assertIn(
|
|
||||||
"ci-ca-volume:/home/mitmproxy/.mitmproxy", argv
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_persists_git_gate_state_across_recreation(self) -> None:
|
|
||||||
# Per-bottle bare repos + deploy creds are bind-mounted from the host so
|
|
||||||
# a gateway restart doesn't drop already-running bottles' git-gate state
|
|
||||||
# (issue #512). Named sources here stand in for CI's per-run volumes.
|
|
||||||
sc = DockerGateway(
|
|
||||||
"bot-bottle-gateway:latest",
|
|
||||||
git_mount_source="ci-git-volume",
|
|
||||||
creds_mount_source="ci-creds-volume",
|
|
||||||
)
|
|
||||||
|
|
||||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
|
||||||
return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc()
|
|
||||||
|
|
||||||
with patch(_RUN_DOCKER, side_effect=fake) as run:
|
|
||||||
sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
|
||||||
argv = next(c.args[0] for c in run.call_args_list
|
|
||||||
if c.args[0][:2] == ["docker", "run"])
|
|
||||||
self.assertIn("ci-git-volume:/git", argv)
|
|
||||||
self.assertIn("ci-creds-volume:/git-gate/creds", argv)
|
|
||||||
|
|
||||||
def test_connect_injects_the_pre_minted_gateway_token(self) -> None:
|
def test_connect_injects_the_pre_minted_gateway_token(self) -> None:
|
||||||
# The gateway presents the token the orchestrator handed it — it never
|
# The gateway presents the token the orchestrator handed it — it never
|
||||||
# mints (holds no signing key). The value rides the env (bare `--env
|
# mints (holds no signing key). The value rides the env (bare `--env
|
||||||
@@ -232,38 +193,7 @@ class TestDockerGateway(unittest.TestCase):
|
|||||||
with patch(_RUN_DOCKER, side_effect=fake):
|
with patch(_RUN_DOCKER, side_effect=fake):
|
||||||
self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
||||||
creates = [c for c in calls if c[:3] == ["docker", "network", "create"]]
|
creates = [c for c in calls if c[:3] == ["docker", "network", "create"]]
|
||||||
self.assertEqual(
|
self.assertEqual([["docker", "network", "create", self.sc.network]], creates)
|
||||||
[[
|
|
||||||
"docker", "network", "create",
|
|
||||||
"--subnet", DEFAULT_GATEWAY_SUBNET,
|
|
||||||
"--label",
|
|
||||||
f"bot-bottle.gateway-subnet={DEFAULT_GATEWAY_SUBNET}",
|
|
||||||
self.sc.network,
|
|
||||||
]],
|
|
||||||
creates,
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_ensure_running_replaces_stale_auto_ipam_network(self) -> None:
|
|
||||||
calls: list[list[str]] = []
|
|
||||||
|
|
||||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
|
||||||
calls.append(argv)
|
|
||||||
if argv[:2] == ["docker", "ps"]:
|
|
||||||
return _proc(stdout="")
|
|
||||||
if argv[:3] == ["docker", "network", "inspect"]:
|
|
||||||
return _proc(stdout="<no value>\n")
|
|
||||||
return _proc()
|
|
||||||
|
|
||||||
with patch(_RUN_DOCKER, side_effect=fake):
|
|
||||||
self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
|
||||||
self.assertIn(
|
|
||||||
["docker", "rm", "--force", self.sc.name],
|
|
||||||
calls,
|
|
||||||
)
|
|
||||||
self.assertIn(
|
|
||||||
["docker", "network", "rm", self.sc.network],
|
|
||||||
calls,
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_ca_cert_pem_reads_from_container(self) -> None:
|
def test_ca_cert_pem_reads_from_container(self) -> None:
|
||||||
with patch(_RUN_DOCKER, return_value=_proc(stdout=_CA_PEM)) as m:
|
with patch(_RUN_DOCKER, return_value=_proc(stdout=_CA_PEM)) as m:
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ server tests), plus one real-socket round-trip to prove the handler wiring.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
import io
|
|
||||||
import json
|
import json
|
||||||
import secrets
|
import secrets
|
||||||
import sqlite3
|
import sqlite3
|
||||||
@@ -18,7 +17,7 @@ import urllib.error
|
|||||||
import urllib.request
|
import urllib.request
|
||||||
from contextlib import closing
|
from contextlib import closing
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint
|
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint
|
||||||
from bot_bottle.orchestrator.broker import StubBroker
|
from bot_bottle.orchestrator.broker import StubBroker
|
||||||
@@ -284,25 +283,6 @@ class TestServerRoundTrip(unittest.TestCase):
|
|||||||
))
|
))
|
||||||
self.assertEqual(reg["bottle_id"], attr["bottle_id"])
|
self.assertEqual(reg["bottle_id"], attr["bottle_id"])
|
||||||
|
|
||||||
def test_internal_failure_is_contextual_but_redacted(self) -> None:
|
|
||||||
orch = MagicMock()
|
|
||||||
orch.registry.all.side_effect = RuntimeError("SENSITIVE request value")
|
|
||||||
with patch("sys.stderr", io.StringIO()) as stderr:
|
|
||||||
server = make_server(orch, "127.0.0.1", 0)
|
|
||||||
self.addCleanup(server.server_close)
|
|
||||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
|
||||||
thread.start()
|
|
||||||
self.addCleanup(server.shutdown)
|
|
||||||
host, port = server.server_address[0], server.server_address[1]
|
|
||||||
with self.assertRaises(urllib.error.HTTPError) as raised:
|
|
||||||
urllib.request.urlopen(f"http://{host}:{port}/bottles", timeout=5)
|
|
||||||
payload = json.loads(raised.exception.read())
|
|
||||||
output = stderr.getvalue()
|
|
||||||
self.assertEqual({"error": "internal error"}, payload)
|
|
||||||
self.assertIn("GET /bottles", output)
|
|
||||||
self.assertIn("RuntimeError", output)
|
|
||||||
self.assertNotIn("SENSITIVE", output)
|
|
||||||
|
|
||||||
|
|
||||||
class TestOrchestratorAuth(unittest.TestCase):
|
class TestOrchestratorAuth(unittest.TestCase):
|
||||||
"""Role-scoped control-plane tokens (issue #400 / #469 review): every route
|
"""Role-scoped control-plane tokens (issue #400 / #469 review): every route
|
||||||
@@ -667,24 +647,10 @@ class TestReconcileRoute(unittest.TestCase):
|
|||||||
self.assertEqual(200, status)
|
self.assertEqual(200, status)
|
||||||
self.assertEqual([], payload["reaped"])
|
self.assertEqual([], payload["reaped"])
|
||||||
|
|
||||||
def test_non_string_entries_are_rejected(self) -> None:
|
def test_non_string_entries_are_ignored(self) -> None:
|
||||||
|
dead = self._old("10.0.0.4")
|
||||||
status, payload = dispatch(
|
status, payload = dispatch(
|
||||||
self.orch, "POST", "/reconcile",
|
self.orch, "POST", "/reconcile",
|
||||||
_body({"live_source_ips": [None, 7, "10.0.0.9"]}))
|
_body({"live_source_ips": [None, 7, "10.0.0.9"]}))
|
||||||
self.assertEqual(400, status)
|
self.assertEqual(200, status)
|
||||||
self.assertIn("live_source_ips", str(payload["error"]))
|
self.assertEqual([dead], payload["reaped"])
|
||||||
|
|
||||||
def test_empty_live_source_ip_is_rejected(self) -> None:
|
|
||||||
status, payload = dispatch(
|
|
||||||
self.orch, "POST", "/reconcile", _body({"live_source_ips": [""]}))
|
|
||||||
self.assertEqual(400, status)
|
|
||||||
self.assertIn("live_source_ips", str(payload["error"]))
|
|
||||||
|
|
||||||
def test_invalid_grace_seconds_is_rejected(self) -> None:
|
|
||||||
for value in (True, "30", -1, float("inf"), float("nan")):
|
|
||||||
with self.subTest(value=value):
|
|
||||||
status, payload = dispatch(
|
|
||||||
self.orch, "POST", "/reconcile",
|
|
||||||
_body({"live_source_ips": [], "grace_seconds": value}))
|
|
||||||
self.assertEqual(400, status)
|
|
||||||
self.assertIn("grace_seconds", str(payload["error"]))
|
|
||||||
|
|||||||
@@ -105,19 +105,11 @@ class TestOrchestrator(unittest.TestCase):
|
|||||||
|
|
||||||
def test_reprovision_rejects_missing_rows_and_wrong_key(self) -> None:
|
def test_reprovision_rejects_missing_rows_and_wrong_key(self) -> None:
|
||||||
self.assertFalse(self.orch.reprovision_from_secret("missing", new_env_var_secret()))
|
self.assertFalse(self.orch.reprovision_from_secret("missing", new_env_var_secret()))
|
||||||
key = "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE"
|
rec = self.orch.launch_bottle(
|
||||||
wrong_key = "FBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQ"
|
"10.243.0.13", tokens={"K": "value"},
|
||||||
# Pin the nonce so this is a deterministic wrong-key/decryption vector
|
env_var_secret=new_env_var_secret(),
|
||||||
# instead of a probabilistic assertion over random bytes.
|
)
|
||||||
with patch(
|
self.assertFalse(self.orch.reprovision_from_secret(rec.bottle_id, new_env_var_secret()))
|
||||||
"bot_bottle.orchestrator.store.secret_store.secrets.token_bytes",
|
|
||||||
return_value=b"\0" * 16,
|
|
||||||
):
|
|
||||||
rec = self.orch.launch_bottle(
|
|
||||||
"10.243.0.13", tokens={"K": "value"},
|
|
||||||
env_var_secret=key,
|
|
||||||
)
|
|
||||||
self.assertFalse(self.orch.reprovision_from_secret(rec.bottle_id, wrong_key))
|
|
||||||
|
|
||||||
def test_set_policy_live_reload(self) -> None:
|
def test_set_policy_live_reload(self) -> None:
|
||||||
rec = self.orch.launch_bottle("10.243.0.3")
|
rec = self.orch.launch_bottle("10.243.0.3")
|
||||||
|
|||||||
@@ -1,91 +0,0 @@
|
|||||||
"""Unit tests for CI's unittest execution-count gate."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import unittest
|
|
||||||
from contextlib import redirect_stderr
|
|
||||||
from io import StringIO
|
|
||||||
from unittest.mock import Mock, patch
|
|
||||||
|
|
||||||
from scripts.unittest_gate import assurance_errors, main
|
|
||||||
|
|
||||||
|
|
||||||
class TestAssuranceErrors(unittest.TestCase):
|
|
||||||
def test_accepts_suite_that_meets_minimum_without_skips(self) -> None:
|
|
||||||
self.assertEqual(
|
|
||||||
[],
|
|
||||||
assurance_errors(
|
|
||||||
tests_run=22, skipped=0, minimum_executed=22, fail_on_skip=True
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_rejects_green_suite_below_execution_minimum(self) -> None:
|
|
||||||
errors = assurance_errors(
|
|
||||||
tests_run=22, skipped=18, minimum_executed=22, fail_on_skip=False
|
|
||||||
)
|
|
||||||
self.assertEqual(1, len(errors))
|
|
||||||
self.assertIn("executed 4", errors[0])
|
|
||||||
|
|
||||||
def test_rejects_any_skip_when_required(self) -> None:
|
|
||||||
errors = assurance_errors(
|
|
||||||
tests_run=23, skipped=1, minimum_executed=22, fail_on_skip=True
|
|
||||||
)
|
|
||||||
self.assertEqual(["1 test(s) skipped in a no-skip suite"], errors)
|
|
||||||
|
|
||||||
def test_main_accepts_successful_assured_suite(self) -> None:
|
|
||||||
result = Mock(
|
|
||||||
testsRun=22,
|
|
||||||
skipped=[],
|
|
||||||
wasSuccessful=Mock(return_value=True),
|
|
||||||
)
|
|
||||||
runner = Mock()
|
|
||||||
runner.run.return_value = result
|
|
||||||
with patch(
|
|
||||||
"scripts.unittest_gate.unittest.defaultTestLoader.discover",
|
|
||||||
return_value=Mock(),
|
|
||||||
) as discover, patch(
|
|
||||||
"scripts.unittest_gate.unittest.TextTestRunner",
|
|
||||||
return_value=runner,
|
|
||||||
) as runner_type:
|
|
||||||
self.assertEqual(
|
|
||||||
0,
|
|
||||||
main([
|
|
||||||
"-s", "tests/integration",
|
|
||||||
"-t", ".",
|
|
||||||
"-p", "test_*.py",
|
|
||||||
"--minimum-executed", "22",
|
|
||||||
"--fail-on-skip",
|
|
||||||
"-v",
|
|
||||||
]),
|
|
||||||
)
|
|
||||||
discover.assert_called_once_with(
|
|
||||||
"tests/integration", pattern="test_*.py", top_level_dir="."
|
|
||||||
)
|
|
||||||
runner_type.assert_called_once_with(verbosity=2)
|
|
||||||
|
|
||||||
def test_main_rejects_unsuccessful_underfilled_suite(self) -> None:
|
|
||||||
result = Mock(
|
|
||||||
testsRun=1,
|
|
||||||
skipped=[(Mock(), "not available")],
|
|
||||||
wasSuccessful=Mock(return_value=False),
|
|
||||||
)
|
|
||||||
runner = Mock()
|
|
||||||
runner.run.return_value = result
|
|
||||||
error = StringIO()
|
|
||||||
with patch(
|
|
||||||
"scripts.unittest_gate.unittest.defaultTestLoader.discover",
|
|
||||||
return_value=Mock(),
|
|
||||||
), patch(
|
|
||||||
"scripts.unittest_gate.unittest.TextTestRunner",
|
|
||||||
return_value=runner,
|
|
||||||
), redirect_stderr(error):
|
|
||||||
self.assertEqual(
|
|
||||||
1,
|
|
||||||
main(["--minimum-executed", "2", "--fail-on-skip"]),
|
|
||||||
)
|
|
||||||
self.assertIn("below required minimum", error.getvalue())
|
|
||||||
self.assertIn("skipped in a no-skip suite", error.getvalue())
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
Reference in New Issue
Block a user