Compare commits
41 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 01cee056be | |||
| ef89ed084f | |||
| ccd987a501 | |||
| 2cd44cf79a | |||
| 8a1b833aaa | |||
| 3b5c55bc8e | |||
| 95220b4808 | |||
| b032562d74 | |||
| 1d925172ec | |||
| f6ae485b68 | |||
| e3258d0683 | |||
| 0ff11d8ed7 | |||
| a970f974a2 | |||
| c845d3fed4 | |||
| c6a9419b95 | |||
| 36fb019007 | |||
| adc033a902 | |||
| 21b253c7eb | |||
| d4e2bc5f93 | |||
| 4998a5ec6a | |||
| efd413c1ba | |||
| d3d468532f | |||
| ad2927b3b1 | |||
| 2582373490 | |||
| 17ac1be93b | |||
| c53254e9d5 | |||
| 58ecd8cb90 | |||
| 310b36196d | |||
| c473e5e5d8 | |||
| 137426d9ac | |||
| 2a3a7dfb5c | |||
| 16c12177d2 | |||
| 0f1734b823 | |||
| 2bf28e03f4 | |||
| d3428b8c14 | |||
| 4199de5e3e | |||
| 8348714e3e | |||
| 26002b75ca | |||
| 0c91c75a05 | |||
| 8ce8a8cc62 | |||
| 3fba385513 |
@@ -1,6 +1,10 @@
|
||||
[run]
|
||||
branch = True
|
||||
source = .
|
||||
# Store paths relative to the project root so .coverage.* files produced on
|
||||
# different runners (ubuntu-latest vs self-hosted KVM) can be combined by the
|
||||
# coverage job without a [paths] remapping section.
|
||||
relative_files = True
|
||||
|
||||
[report]
|
||||
# Coverage policy: see docs/decisions/0004-coverage-policy.md.
|
||||
|
||||
+128
-110
@@ -9,10 +9,12 @@
|
||||
# tests/canaries/ — upstream regression canaries; run on a separate
|
||||
# schedule (see canaries.yml), not here
|
||||
#
|
||||
# Integration tests run once per backend in separate jobs. Each job sets
|
||||
# BOT_BOTTLE_BACKEND explicitly so the test suite uses the right backend.
|
||||
# Backends that aren't available on the runner fail the preflight step
|
||||
# rather than silently skipping inside the test output.
|
||||
# Each test job runs once under coverage and uploads a small .coverage.*
|
||||
# artifact. The `coverage` job combines them — no test reruns, no KVM
|
||||
# dependency on that job. For main-branch pushes only, the tested rootfs
|
||||
# and matching dropbear are uploaded so `publish-infra` can publish the
|
||||
# byte-identical artifact that was tested. PRs avoid the ~194 MB rootfs
|
||||
# transfer entirely.
|
||||
|
||||
name: test
|
||||
|
||||
@@ -40,53 +42,6 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
stage-firecracker-inputs:
|
||||
runs-on: [self-hosted, kvm]
|
||||
# Same guard as the other KVM-runner jobs: don't spin the privileged
|
||||
# runner for fork PRs (this only copies a non-secret static binary, but
|
||||
# keep the posture consistent — build-infra/integration/coverage all
|
||||
# depend on it, so gating here gates the whole Firecracker chain).
|
||||
if: >-
|
||||
github.event_name == 'push' ||
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'pull_request' &&
|
||||
github.event.pull_request.head.repo.full_name == github.repository)
|
||||
steps:
|
||||
- name: Stage the provisioned static dropbear
|
||||
run: |
|
||||
mkdir -p firecracker-inputs
|
||||
cp /var/cache/bot-bottle-fc/dropbear firecracker-inputs/dropbear
|
||||
|
||||
- name: Upload Firecracker build inputs
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: firecracker-inputs/
|
||||
|
||||
build-infra:
|
||||
needs: stage-firecracker-inputs
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download Firecracker build inputs
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: firecracker-inputs
|
||||
|
||||
- name: Build infra candidate from this checkout
|
||||
env:
|
||||
BOT_BOTTLE_FC_DROPBEAR: ${{ github.workspace }}/firecracker-inputs/dropbear
|
||||
run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate
|
||||
|
||||
- name: Upload infra candidate
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate/
|
||||
|
||||
unit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -101,12 +56,29 @@ jobs:
|
||||
- name: Install dev requirements
|
||||
run: python3 -m pip install --break-system-packages -r requirements-dev.txt
|
||||
|
||||
- name: Run unit tests
|
||||
- name: Run unit tests with coverage
|
||||
env:
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.unit
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/unit -v
|
||||
|
||||
- name: Report unit coverage
|
||||
env:
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.unit
|
||||
run: python3 -m coverage report -m
|
||||
|
||||
# upload-artifact@v3's glob skips dotfiles, so a bare `.coverage.unit`
|
||||
# silently uploads nothing ("No files were found"). Stage it under a
|
||||
# non-dot name; the coverage job renames it back before `coverage
|
||||
# combine`. `cp` also fails loudly if coverage never wrote the file.
|
||||
- name: Stage unit coverage for upload
|
||||
run: cp .coverage.unit coverage-unit.dat
|
||||
|
||||
- name: Upload unit coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-unit
|
||||
path: coverage-unit.dat
|
||||
|
||||
integration-docker:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -115,6 +87,9 @@ jobs:
|
||||
|
||||
# No actions/setup-python (see the note in the `unit` job); the
|
||||
# container's system Python 3.12 runs the stdlib test suite directly.
|
||||
- name: Install coverage
|
||||
run: python3 -m pip install --break-system-packages coverage
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
python3 --version
|
||||
@@ -124,10 +99,21 @@ jobs:
|
||||
echo "docker not on PATH — integration tests will skip"
|
||||
fi
|
||||
|
||||
- name: Run integration tests (docker)
|
||||
- name: Run integration tests (docker) with coverage
|
||||
env:
|
||||
BOT_BOTTLE_BACKEND: docker
|
||||
run: python3 -m unittest discover -t . -s tests/integration -v
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.docker
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
|
||||
- name: Stage docker coverage for upload
|
||||
run: cp .coverage.docker coverage-docker.dat
|
||||
|
||||
- name: Upload docker coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-docker
|
||||
path: coverage-docker.dat
|
||||
|
||||
# Integration tests against the Firecracker backend. Runs on a self-hosted
|
||||
# KVM runner (label `kvm`) where /dev/kvm and the TAP/nft pool are available.
|
||||
@@ -137,9 +123,16 @@ jobs:
|
||||
#
|
||||
# Runner prerequisites (provision once; see README "Firecracker on Linux"):
|
||||
# `firecracker` on PATH, `/dev/kvm` accessible, cached kernel +
|
||||
# static dropbear, and the pool as a persistent systemd unit.
|
||||
# static dropbear at /var/cache/bot-bottle-fc/dropbear, and the pool as a
|
||||
# persistent systemd unit.
|
||||
#
|
||||
# The infra candidate is built here directly (no artifact download) to
|
||||
# eliminate the ~70 s ubuntu-latest upload + ~83 s combined download that
|
||||
# the old build-infra → integration-firecracker + coverage chain incurred.
|
||||
# For main-branch pushes the tested rootfs and matching dropbear are
|
||||
# uploaded so publish-infra can publish the byte-identical artifact; PRs
|
||||
# skip those uploads entirely.
|
||||
integration-firecracker:
|
||||
needs: build-infra
|
||||
runs-on: [self-hosted, kvm]
|
||||
if: >-
|
||||
github.event_name == 'push' ||
|
||||
@@ -159,49 +152,65 @@ jobs:
|
||||
# range overlap; it prints the exact `backend setup` fix.
|
||||
python3 cli.py backend status --backend=firecracker
|
||||
|
||||
- name: Download the candidate built from this checkout
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate
|
||||
- name: Build infra candidate from this checkout
|
||||
env:
|
||||
BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear
|
||||
run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate --reuse-published
|
||||
|
||||
- name: Replace the persistent infra VM with the candidate
|
||||
run: python3 -c 'from bot_bottle.backend.firecracker import infra_vm; infra_vm.stop()'
|
||||
|
||||
# No dev-requirements install: the integration suite runs on stdlib
|
||||
# `unittest` (pylint/pyright are lint.yml's concern, not this job's),
|
||||
# and the self-hosted runner's Nix python env has no `pip` module
|
||||
# (`python3 -m pip` → "No module named pip"). Nothing to install.
|
||||
- name: Run integration tests (firecracker)
|
||||
# No dev-requirements install: `coverage` is already provided by the
|
||||
# self-hosted runner's Nix python env, and that env has no `pip`
|
||||
# module to install into anyway.
|
||||
- name: Run integration tests (firecracker) with coverage
|
||||
env:
|
||||
BOT_BOTTLE_BACKEND: firecracker
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
|
||||
run: python3 -m unittest discover -t . -s tests/integration -v
|
||||
COVERAGE_FILE: ${{ github.workspace }}/.coverage.firecracker
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
# Combined unit+integration coverage + the diff-coverage gate (the hard
|
||||
# gate: new/changed lines >= 90%). See docs/decisions/0004-coverage-policy.md.
|
||||
# Non-dot name so upload-artifact's dotfile-skipping glob picks it up.
|
||||
- name: Stage firecracker coverage for upload
|
||||
run: cp .coverage.firecracker coverage-firecracker.dat
|
||||
|
||||
- name: Upload firecracker coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-firecracker
|
||||
path: coverage-firecracker.dat
|
||||
|
||||
# Only upload the large rootfs artifact on main-branch pushes;
|
||||
# PRs avoid the ~194 MB transfer. publish-infra only runs on main
|
||||
# and downloads these to publish the byte-identical tested rootfs.
|
||||
- name: Upload tested rootfs (main branch only)
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate/
|
||||
|
||||
- name: Upload dropbear for publish verification (main branch only)
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: /var/cache/bot-bottle-fc/dropbear
|
||||
|
||||
# Combined coverage gate: aggregates .coverage.* artifacts uploaded by each
|
||||
# test job, then runs the diff-coverage gate (new/changed lines >= 90%).
|
||||
#
|
||||
# This runs on a self-hosted KVM runner (label `kvm`), NOT ubuntu-latest,
|
||||
# because the Firecracker backend's subprocess/VM orchestration
|
||||
# (launch/boot/SSH/isolation-probe) is covered by the integration suite,
|
||||
# and that suite needs `/dev/kvm` + the provisioned TAP/nft pool — which a
|
||||
# container-based runner doesn't have. On such a runner the firecracker
|
||||
# integration test skips and its ~230 orchestration lines read as
|
||||
# uncovered, so the gate can't pass there.
|
||||
# Runs on ubuntu-latest — no KVM needed, no test reruns. Coverage files use
|
||||
# relative_files = True (.coveragerc) so they combine cleanly across runners.
|
||||
# Each test job sets COVERAGE_FILE to an absolute path so coverage.py writes
|
||||
# to a known location that upload-artifact can find regardless of runner env.
|
||||
#
|
||||
# Restricted to the same events as integration-firecracker (same-repo PRs,
|
||||
# push, workflow_dispatch) for the same security reason.
|
||||
#
|
||||
# See #414 for the planned follow-up: artifact-based coverage combination
|
||||
# (run tests once in their respective jobs, combine .coverage files here).
|
||||
#
|
||||
# build-infra creates one candidate from the checkout. This job boots that
|
||||
# same candidate after integration-firecracker has exercised it; the main
|
||||
# push path publishes the identical bytes only after every required job.
|
||||
# Restricted to the same events as integration-firecracker: it depends on
|
||||
# that job's coverage artifact and skips for fork PRs alongside it.
|
||||
coverage:
|
||||
needs: [build-infra, integration-firecracker]
|
||||
needs: [unit, integration-docker, integration-firecracker]
|
||||
timeout-minutes: 15
|
||||
runs-on: [self-hosted, kvm]
|
||||
runs-on: ubuntu-latest
|
||||
if: >-
|
||||
github.event_name == 'push' ||
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
@@ -213,29 +222,37 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Preflight — Firecracker host is ready
|
||||
run: |
|
||||
command -v firecracker >/dev/null || {
|
||||
echo "firecracker not on PATH — provision the runner (README: Firecracker on Linux)"; exit 1; }
|
||||
test -e /dev/kvm || { echo "/dev/kvm missing — KVM not available on this runner"; exit 1; }
|
||||
# `backend status` exits non-zero unless the TAP pool is up + no
|
||||
# range overlap; it prints the exact `backend setup` fix.
|
||||
python3 cli.py backend status --backend=firecracker
|
||||
- name: Install coverage
|
||||
run: python3 -m pip install --break-system-packages coverage
|
||||
|
||||
- name: Download the candidate already exercised by integration
|
||||
- name: Download unit coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate
|
||||
name: coverage-unit
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
- name: Download docker coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: coverage-docker
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
- name: Download firecracker coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: coverage-firecracker
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
# Rename the non-dot upload names back to the .coverage.* files that
|
||||
# `coverage combine` discovers (see the staging steps in each test job).
|
||||
- name: Reassemble coverage data files
|
||||
run: |
|
||||
mv coverage-unit.dat .coverage.unit
|
||||
mv coverage-docker.dat .coverage.docker
|
||||
mv coverage-firecracker.dat .coverage.firecracker
|
||||
|
||||
# No dev-requirements install: `coverage` is already provided by the
|
||||
# self-hosted runner's Nix python env, and that env has no `pip`
|
||||
# module to install into anyway. `scripts/coverage.sh` +
|
||||
# `diff_coverage.py` need only `coverage` (not pylint/pyright).
|
||||
- name: Combined coverage (unit + integration, incl. firecracker)
|
||||
env:
|
||||
BOT_BOTTLE_CI_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
|
||||
run: PYTHON=python3 bash scripts/coverage.sh critical
|
||||
run: PYTHON=python3 bash scripts/coverage.sh aggregate critical
|
||||
|
||||
- name: Diff-coverage gate (changed lines >= 90%)
|
||||
run: |
|
||||
@@ -243,14 +260,14 @@ jobs:
|
||||
python3 scripts/diff_coverage.py --base origin/main --min 90
|
||||
|
||||
publish-infra:
|
||||
needs: [stage-firecracker-inputs, build-infra, unit, integration-docker, integration-firecracker, coverage]
|
||||
needs: [unit, integration-docker, integration-firecracker, coverage]
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Checkout the tested revision
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download the tested candidate
|
||||
- name: Download the tested rootfs
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
@@ -258,9 +275,10 @@ jobs:
|
||||
|
||||
# publish_infra re-derives the version from the checkout to confirm the
|
||||
# bundle matches before uploading, and the version hashes the dropbear
|
||||
# bytes. Stage the SAME dropbear build-infra used, or the recheck
|
||||
# computes a "<missing>"-dropbear version and rejects the candidate.
|
||||
- name: Download the staged dropbear (matches build-infra's version)
|
||||
# bytes. Download the SAME dropbear integration-firecracker used, or
|
||||
# the recheck computes a "<missing>"-dropbear version and rejects the
|
||||
# candidate.
|
||||
- name: Download the staged dropbear (matches build's version)
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
|
||||
@@ -14,6 +14,9 @@ on:
|
||||
jobs:
|
||||
update-badges:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
|
||||
@@ -45,6 +45,10 @@ PROVIDER_TEMPLATES = frozenset({PROVIDER_CLAUDE, PROVIDER_CODEX, PROVIDER_PI})
|
||||
# forward_host_credentials is enabled. Pipelock must pass these through
|
||||
# (no TLS MITM) or its header DLP blocks the injected JWT.
|
||||
CODEX_HOST_CREDENTIAL_HOSTS = ("api.openai.com", "chatgpt.com")
|
||||
|
||||
# Host that egress injects the host Claude bearer on when Claude
|
||||
# forward_host_credentials is enabled.
|
||||
CLAUDE_HOST_CREDENTIAL_HOSTS = ("api.anthropic.com",)
|
||||
PromptMode = Literal[
|
||||
"append_file",
|
||||
"read_prompt_file",
|
||||
@@ -257,7 +261,28 @@ class AgentProvider(ABC):
|
||||
Default: Debian/node — writes the git-gate insteadOf gitconfig
|
||||
and sets user.name/email as node. Workspace copy runs through
|
||||
BottleBackend.provision_workspace against the running bottle."""
|
||||
from .log import info
|
||||
from .log import die, info
|
||||
|
||||
# Firecracker exports image rootfs files through an unprivileged host
|
||||
# tar extraction, so image-time ownership of XDG directories is not
|
||||
# preserved. Git consults ~/.config/git even when the actual config
|
||||
# is ~/.gitconfig; an unreadable directory there can prevent the
|
||||
# git-gate insteadOf rules below from taking effect. Repair this at
|
||||
# runtime, after every backend's copy/export path has completed.
|
||||
git_xdg_dir = f"{plan.guest_home}/.config/git"
|
||||
repair = bottle.exec(
|
||||
f"chown node:node {shlex.quote(plan.guest_home)} && "
|
||||
f"chmod 755 {shlex.quote(plan.guest_home)} && "
|
||||
f"mkdir -p {shlex.quote(git_xdg_dir)} && "
|
||||
f"chown -R node:node {shlex.quote(f'{plan.guest_home}/.config')} && "
|
||||
f"chmod -R u+rwX,go+rX {shlex.quote(f'{plan.guest_home}/.config')}",
|
||||
user="root",
|
||||
)
|
||||
if repair.returncode != 0:
|
||||
die(
|
||||
"git provisioning: could not make the runtime Git config "
|
||||
f"directory readable: {(repair.stderr or repair.stdout).strip()}"
|
||||
)
|
||||
|
||||
manifest_bottle = plan.manifest.bottle
|
||||
if manifest_bottle.git:
|
||||
@@ -280,11 +305,27 @@ class AgentProvider(ABC):
|
||||
f"{len(manifest_bottle.git)} insteadOf rule(s)"
|
||||
)
|
||||
bottle.cp_in(str(config_file), guest_gitconfig)
|
||||
bottle.exec(
|
||||
permissions = bottle.exec(
|
||||
f"chown node:node {shlex.quote(guest_gitconfig)} && "
|
||||
f"chmod 644 {shlex.quote(guest_gitconfig)}",
|
||||
user="root",
|
||||
)
|
||||
if permissions.returncode != 0:
|
||||
die(
|
||||
"git provisioning: could not set ownership on "
|
||||
f"{guest_gitconfig}: "
|
||||
f"{(permissions.stderr or permissions.stdout).strip()}"
|
||||
)
|
||||
configured = bottle.exec(
|
||||
"git config --global --get-regexp '^url\\..*\\.insteadof$'",
|
||||
user="node",
|
||||
)
|
||||
if configured.returncode != 0:
|
||||
die(
|
||||
"git provisioning: the runtime user cannot read the "
|
||||
f"git-gate insteadOf rules from {guest_gitconfig}: "
|
||||
f"{(configured.stderr or configured.stdout).strip()}"
|
||||
)
|
||||
|
||||
gu = manifest_bottle.git_user
|
||||
if not gu.is_empty():
|
||||
|
||||
@@ -37,10 +37,10 @@ import os
|
||||
import shlex
|
||||
import sys
|
||||
from abc import ABC, abstractmethod
|
||||
from contextlib import AbstractContextManager
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Generic, Sequence, TypeVar
|
||||
from typing import TYPE_CHECKING, Any, Generator, Generic, Sequence, TypeVar
|
||||
|
||||
from ..agent_provider import AgentProvisionPlan, get_provider, build_agent_provision_plan
|
||||
from ..egress import EgressPlan
|
||||
@@ -83,6 +83,9 @@ class BottleSpec:
|
||||
# True when launched via --headless (no TTY, no interactive prompts).
|
||||
# The git-gate host-key preflight uses this to error rather than prompt.
|
||||
headless: bool = False
|
||||
# Image startup policy. "fresh" preserves the normal build path;
|
||||
# "cached" reuses the current local image/artifact without rebuilding.
|
||||
image_policy: str = "fresh"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -278,6 +281,18 @@ PlanT = TypeVar("PlanT", bound=BottlePlan)
|
||||
CleanupT = TypeVar("CleanupT", bound=BottleCleanupPlan)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BottleImages:
|
||||
"""Resolved image references (or artifact paths) for a bottle launch.
|
||||
|
||||
For Docker/macOS-container backends, `agent` and `sidecar` are string
|
||||
image refs. For the smolmachines backend they are Path objects pointing
|
||||
to pre-built `.smolmachine` artifacts."""
|
||||
|
||||
agent: str | Path
|
||||
sidecar: str | Path = ""
|
||||
|
||||
|
||||
class BottleBackend(ABC, Generic[PlanT, CleanupT]):
|
||||
"""Abstract base for selectable bottle backends. Concrete subclasses
|
||||
(e.g. DockerBottleBackend) own their own prepare/launch impls.
|
||||
@@ -437,9 +452,27 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]):
|
||||
prompt file, Dockerfile path, and guest home all live on
|
||||
`agent_provision_plan` — the source of truth."""
|
||||
|
||||
def prelaunch_checks(self, plan: PlanT) -> None:
|
||||
"""Raise StaleImageError if any cached image used by this plan is stale.
|
||||
No-op default; backends override to call the shared check_stale*
|
||||
helpers on their image/artifact timestamps. Called by the CLI before
|
||||
launch so the operator can be prompted outside the launch context."""
|
||||
|
||||
@contextmanager
|
||||
def launch(self, plan: PlanT) -> Generator[Bottle, None, None]:
|
||||
"""Template: build or load images, then delegate to _launch_impl."""
|
||||
images = self._build_or_load_images(plan)
|
||||
with self._launch_impl(plan, images) as bottle:
|
||||
yield bottle
|
||||
|
||||
@abstractmethod
|
||||
def launch(self, plan: PlanT) -> AbstractContextManager[Bottle]:
|
||||
"""Build/run the bottle and yield a handle; tear down on exit."""
|
||||
def _build_or_load_images(self, plan: PlanT) -> BottleImages:
|
||||
"""Return the agent and sidecar image references (or artifact paths)
|
||||
for this plan, building fresh images when the policy requires it."""
|
||||
|
||||
@abstractmethod
|
||||
def _launch_impl(self, plan: PlanT, images: BottleImages) -> AbstractContextManager[Bottle]:
|
||||
"""Bring up the bottle using pre-resolved images; yield a handle; tear down on exit."""
|
||||
|
||||
def provision(self, plan: PlanT, bottle: "Bottle") -> str | None:
|
||||
"""Copy host-side files (CA cert, prompt, skills, .git) into
|
||||
|
||||
@@ -7,10 +7,13 @@ imports it rather than re-implementing it.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
|
||||
from ..egress import EgressPlan
|
||||
from ..git_gate import GitGatePlan
|
||||
from ..orchestrator.client import OrchestratorClient
|
||||
from ..orchestrator.client import OrchestratorClient, RegisteredBottle
|
||||
from ..orchestrator.registration import registration_inputs
|
||||
from ..orchestrator.secret_store import new_env_var_secret
|
||||
from .docker.gateway_provision import GatewayTransport, deprovision_git_gate, provision_git_gate
|
||||
|
||||
|
||||
@@ -23,21 +26,26 @@ def provision_bottle(
|
||||
*,
|
||||
image_ref: str = "",
|
||||
tokens: dict[str, str] | None = None,
|
||||
):
|
||||
) -> RegisteredBottle:
|
||||
"""Register the bottle and provision its git-gate state. Rolls back the
|
||||
registration if provisioning fails so no orphan is left. Returns the
|
||||
`RegisteredBottle` from the orchestrator."""
|
||||
registration if provisioning fails so no orphan is left.
|
||||
|
||||
Generates a fresh ENV_VAR_SECRET, passes it to the orchestrator so it can
|
||||
encrypt the token values at rest, and stamps the secret onto the returned
|
||||
``RegisteredBottle`` so callers can inject it into the agent container's
|
||||
environment."""
|
||||
inputs = registration_inputs(egress_plan)
|
||||
env_var_secret = new_env_var_secret()
|
||||
reg = client.register_bottle(
|
||||
source_ip, image_ref=image_ref, policy=inputs.policy,
|
||||
metadata=inputs.metadata, tokens=tokens,
|
||||
metadata=inputs.metadata, tokens=tokens, env_var_secret=env_var_secret,
|
||||
)
|
||||
try:
|
||||
provision_git_gate(transport, reg.bottle_id, git_gate_plan)
|
||||
except Exception:
|
||||
client.teardown_bottle(reg.bottle_id)
|
||||
raise
|
||||
return reg
|
||||
return dataclasses.replace(reg, env_var_secret=env_var_secret)
|
||||
|
||||
|
||||
def teardown_consolidated(
|
||||
|
||||
@@ -31,7 +31,7 @@ from ...env import ResolvedEnv
|
||||
from ...git_gate import GitGatePlan
|
||||
from ...supervise import SupervisePlan
|
||||
from ...manifest import Manifest
|
||||
from .. import ActiveAgent, BottleBackend, BottleSpec
|
||||
from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec
|
||||
from . import cleanup as _cleanup
|
||||
from . import enumerate as _enumerate
|
||||
from . import launch as _launch
|
||||
@@ -100,9 +100,15 @@ class DockerBottleBackend(BottleBackend["DockerBottlePlan", "DockerBottleCleanup
|
||||
stage_dir=stage_dir,
|
||||
)
|
||||
|
||||
def prelaunch_checks(self, plan: DockerBottlePlan) -> None:
|
||||
_launch.stale_checks(plan)
|
||||
|
||||
def _build_or_load_images(self, plan: DockerBottlePlan) -> BottleImages:
|
||||
return _launch.build_or_load_images(plan)
|
||||
|
||||
@contextmanager
|
||||
def launch(self, plan: DockerBottlePlan) -> Generator[DockerBottle, None, None]:
|
||||
with _launch.launch(plan, provision=self.provision) as bottle:
|
||||
def _launch_impl(self, plan: DockerBottlePlan, images: BottleImages) -> Generator[DockerBottle, None, None]:
|
||||
with _launch.launch(plan, images, provision=self.provision) as bottle:
|
||||
yield bottle
|
||||
|
||||
def ensure_orchestrator(self) -> str:
|
||||
|
||||
@@ -39,6 +39,10 @@ class DockerBottlePlan(BottlePlan):
|
||||
# (egress proxy credentials, git-gate/supervise headers); set by launch
|
||||
# from the orchestrator registration. Empty pre-registration.
|
||||
identity_token: str = ""
|
||||
# Encryption key for the agent's stored egress secrets; injected into the
|
||||
# agent container as ENV_VAR_SECRET via the compose subprocess env (bare
|
||||
# name — value never written to the compose file). Empty pre-registration.
|
||||
env_var_secret: str = ""
|
||||
|
||||
@property
|
||||
def container_name(self) -> str:
|
||||
|
||||
@@ -17,6 +17,7 @@ from __future__ import annotations
|
||||
from typing import Any
|
||||
|
||||
from ...egress import egress_agent_env_entries
|
||||
from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
|
||||
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||
from .bottle_plan import DockerBottlePlan
|
||||
from .egress import EGRESS_PORT
|
||||
@@ -58,6 +59,10 @@ def consolidated_agent_compose(
|
||||
# the secret value never lands on argv or in the compose file.
|
||||
for name in sorted(plan.forwarded_env.keys()):
|
||||
env.append(name)
|
||||
# ENV_VAR_SECRET: bare name so the value comes from the compose subprocess
|
||||
# env (set in launch.py) and is never written to the compose file on disk.
|
||||
if getattr(plan, "env_var_secret", ""):
|
||||
env.append(ENV_VAR_SECRET_NAME)
|
||||
env.extend(egress_agent_env_entries(plan.egress_plan))
|
||||
|
||||
service: dict[str, Any] = {
|
||||
|
||||
@@ -15,12 +15,14 @@ from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ... import log
|
||||
from ...docker_cmd import run_docker
|
||||
from ...egress import EgressPlan
|
||||
from ...git_gate import GitGatePlan
|
||||
from ...orchestrator.client import OrchestratorClient
|
||||
from ...orchestrator.gateway import GATEWAY_NETWORK
|
||||
from ...orchestrator.lifecycle import INFRA_NAME, OrchestratorService
|
||||
from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
|
||||
from ..consolidated_util import provision_bottle
|
||||
from ..consolidated_util import teardown_consolidated as _teardown_util
|
||||
from .gateway_provision import DockerGatewayTransport
|
||||
@@ -41,6 +43,7 @@ class LaunchContext:
|
||||
network: str # the shared gateway network to attach to
|
||||
gateway_ip: str # the gateway's address — the agent's proxy target
|
||||
orchestrator_url: str
|
||||
env_var_secret: str = "" # encryption key injected into the agent's env
|
||||
|
||||
|
||||
def _network_cidr(network: str) -> str:
|
||||
@@ -85,6 +88,66 @@ def _network_container_ips(network: str) -> list[str]:
|
||||
return ips
|
||||
|
||||
|
||||
def _reprovision_running_bottles(
|
||||
orchestrator_url: str,
|
||||
network: str = GATEWAY_NETWORK,
|
||||
infra_name: str = INFRA_NAME,
|
||||
) -> None:
|
||||
"""Re-inject egress tokens for any registered bottles that lost their
|
||||
in-memory tokens (e.g., after an infra container restart).
|
||||
|
||||
For each registered bottle whose source IP maps to a live container on the
|
||||
gateway network, reads ENV_VAR_SECRET via ``docker exec … printenv`` and
|
||||
calls ``POST /bottles/<id>/reprovision_gateway``. Idempotent — a no-op
|
||||
when the orchestrator already has all tokens loaded. Best-effort: a single
|
||||
container exec failure never blocks a new bottle launch."""
|
||||
client = OrchestratorClient(orchestrator_url)
|
||||
bottles = client.list_bottles()
|
||||
if not bottles:
|
||||
return
|
||||
|
||||
# Build {source_ip: container_name} from live containers on the gateway
|
||||
# network, excluding the infra container itself.
|
||||
proc = run_docker([
|
||||
"docker", "network", "inspect",
|
||||
"--format", "{{range .Containers}}{{.Name}} {{.IPv4Address}}\n{{end}}",
|
||||
network,
|
||||
])
|
||||
ip_to_container: dict[str, str] = {}
|
||||
for line in proc.stdout.splitlines():
|
||||
parts = line.strip().split()
|
||||
if len(parts) >= 2 and parts[0] != infra_name:
|
||||
ip = parts[1].split("/", 1)[0]
|
||||
if ip:
|
||||
ip_to_container[ip] = parts[0]
|
||||
|
||||
reprovisioned = 0
|
||||
for bottle in bottles:
|
||||
bottle_id = bottle.get("bottle_id")
|
||||
source_ip = bottle.get("source_ip")
|
||||
if not isinstance(bottle_id, str) or not isinstance(source_ip, str):
|
||||
continue
|
||||
container_name = ip_to_container.get(source_ip)
|
||||
if not container_name:
|
||||
continue
|
||||
proc = run_docker(
|
||||
["docker", "exec", container_name, "printenv", ENV_VAR_SECRET_NAME]
|
||||
)
|
||||
if proc.returncode != 0 or not proc.stdout.strip():
|
||||
continue
|
||||
try:
|
||||
if client.reprovision_gateway(bottle_id, proc.stdout.strip()):
|
||||
reprovisioned += 1
|
||||
except Exception: # noqa: BLE001 — best-effort, never block a launch
|
||||
pass
|
||||
|
||||
if reprovisioned:
|
||||
log.info(
|
||||
"reprovisioned egress tokens",
|
||||
context={"count": reprovisioned},
|
||||
)
|
||||
|
||||
|
||||
def launch_consolidated(
|
||||
egress_plan: EgressPlan,
|
||||
git_gate_plan: GitGatePlan,
|
||||
@@ -96,9 +159,14 @@ def launch_consolidated(
|
||||
network: str = GATEWAY_NETWORK,
|
||||
) -> LaunchContext:
|
||||
"""Ensure the infra container is up, allocate + register the bottle, and
|
||||
provision its git-gate state. Returns the agent's attach context."""
|
||||
provision its git-gate state. Returns the agent's attach context.
|
||||
|
||||
Also reprovisiones egress tokens for any already-running bottles that lost
|
||||
their in-memory credentials (e.g. after an infra container restart), so
|
||||
they regain egress access before the new bottle is registered."""
|
||||
service = service or OrchestratorService()
|
||||
url = service.ensure_running()
|
||||
_reprovision_running_bottles(url, network=network, infra_name=infra_name)
|
||||
client = OrchestratorClient(url)
|
||||
|
||||
cidr = _network_cidr(network)
|
||||
@@ -117,6 +185,7 @@ def launch_consolidated(
|
||||
network=network,
|
||||
gateway_ip=gateway_ip,
|
||||
orchestrator_url=url,
|
||||
env_var_secret=reg.env_var_secret,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -42,7 +42,9 @@ from ...git_gate import (
|
||||
provision_git_gate_dynamic_keys,
|
||||
revoke_git_gate_provisioned_keys,
|
||||
)
|
||||
from ...log import info, warn
|
||||
from ...image_cache import check_stale
|
||||
from ...log import die, info, warn
|
||||
from .. import BottleImages
|
||||
from . import util as docker_mod
|
||||
from .bottle import DockerBottle
|
||||
from .bottle_plan import DockerBottlePlan
|
||||
@@ -71,16 +73,47 @@ from ...orchestrator.gateway import DockerGateway
|
||||
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
|
||||
|
||||
|
||||
def build_or_load_images(plan: DockerBottlePlan) -> BottleImages:
|
||||
"""Resolve the agent image ref for this plan.
|
||||
|
||||
Returns the committed snapshot if one exists, the cached image when the
|
||||
policy is 'cached', or builds a fresh image and returns that."""
|
||||
committed = read_committed_image(plan.slug)
|
||||
if committed and docker_mod.image_exists(committed):
|
||||
info(f"using committed image {committed!r}")
|
||||
return BottleImages(agent=committed)
|
||||
if plan.spec.image_policy == "cached":
|
||||
if not docker_mod.image_exists(plan.image):
|
||||
die(
|
||||
f"cached agent image {plan.image!r} not found; "
|
||||
"run without --cached-images to build it"
|
||||
)
|
||||
info(f"using cached agent image {plan.image!r}")
|
||||
return BottleImages(agent=plan.image)
|
||||
docker_mod.build_image(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path)
|
||||
docker_mod.verify_agent_image(
|
||||
plan.image, runtime_for(plan.agent_provider_template).smoke_test,
|
||||
)
|
||||
return BottleImages(agent=plan.image)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def launch(
|
||||
plan: DockerBottlePlan,
|
||||
images: BottleImages,
|
||||
*,
|
||||
provision: Callable[[DockerBottlePlan, "DockerBottle"], str | None],
|
||||
) -> Generator[DockerBottle, None, None]:
|
||||
"""Build, launch, and provision a Docker bottle via compose.
|
||||
Teardown on exit."""
|
||||
"""Launch and provision a Docker bottle via compose. Teardown on exit."""
|
||||
stack = ExitStack()
|
||||
|
||||
# Stamp the resolved agent image ref into the plan so compose rendering
|
||||
# picks up the right image (may be a committed snapshot or cached ref).
|
||||
plan = dataclasses.replace(
|
||||
plan,
|
||||
agent_provision=dataclasses.replace(plan.agent_provision, image=str(images.agent)),
|
||||
)
|
||||
|
||||
_bottle_for_revoke = plan.manifest.bottle
|
||||
_git_gate_dir_for_revoke = git_gate_state_dir(plan.slug)
|
||||
|
||||
@@ -97,25 +130,6 @@ def launch(
|
||||
)
|
||||
|
||||
try:
|
||||
# Step 1: agent image. Use a committed snapshot when one exists
|
||||
# and is present in the local daemon; otherwise build from the
|
||||
# Dockerfile. (The gateway image is built by the orchestrator.)
|
||||
committed = read_committed_image(plan.slug)
|
||||
if committed and docker_mod.image_exists(committed):
|
||||
info(f"using committed image {committed!r}")
|
||||
plan = dataclasses.replace(
|
||||
plan,
|
||||
agent_provision=dataclasses.replace(plan.agent_provision, image=committed),
|
||||
)
|
||||
else:
|
||||
docker_mod.build_image(
|
||||
plan.image, _REPO_DIR,
|
||||
dockerfile=plan.dockerfile_path,
|
||||
)
|
||||
docker_mod.verify_agent_image(
|
||||
plan.image, runtime_for(plan.agent_provider_template).smoke_test,
|
||||
)
|
||||
|
||||
# Step 2: mint the git-gate dynamic (gitea) deploy keys, if any, before
|
||||
# provisioning the bottle's repos into the shared gateway.
|
||||
git_gate_plan = plan.git_gate_plan
|
||||
@@ -172,6 +186,7 @@ def launch(
|
||||
agent_git_gate_url=git_gate_url,
|
||||
agent_supervise_url=supervise_url,
|
||||
identity_token=ctx.identity_token,
|
||||
env_var_secret=ctx.env_var_secret,
|
||||
)
|
||||
|
||||
# Step 5: render + up the agent-only compose, pinned on the shared
|
||||
@@ -184,7 +199,12 @@ def launch(
|
||||
project = compose_project_name(plan.slug)
|
||||
# Forwarded vars (OAuth token, host interpolations) flow through the
|
||||
# subprocess env as bare names so values never land in the file.
|
||||
# ENV_VAR_SECRET follows the same pattern: bare name in the compose
|
||||
# spec, value only in the subprocess env so it is never written to disk.
|
||||
compose_env: dict[str, str] = {**os.environ, **plan.forwarded_env}
|
||||
if plan.env_var_secret:
|
||||
from ...orchestrator.secret_store import ENV_VAR_SECRET_NAME
|
||||
compose_env[ENV_VAR_SECRET_NAME] = plan.env_var_secret
|
||||
info(
|
||||
f"docker compose up -d (project {project}, agent on shared "
|
||||
f"gateway {ctx.gateway_ip}, ip {ctx.source_ip})"
|
||||
@@ -211,3 +231,21 @@ def launch(
|
||||
yield bottle
|
||||
finally:
|
||||
teardown()
|
||||
|
||||
|
||||
def stale_checks(plan: DockerBottlePlan) -> None:
|
||||
"""Raise StaleImageError if a cached image is older than the configured
|
||||
threshold. Only runs when image_policy is 'cached'. Called by the backend
|
||||
class's _image_stale_checks before _launch_impl starts any resources."""
|
||||
if plan.spec.image_policy != "cached":
|
||||
return
|
||||
committed = read_committed_image(plan.slug)
|
||||
if committed and docker_mod.image_exists(committed):
|
||||
ts = docker_mod.image_created_at(committed)
|
||||
if ts is not None:
|
||||
check_stale(f"agent image {committed!r}", ts)
|
||||
return
|
||||
if docker_mod.image_exists(plan.image):
|
||||
ts = docker_mod.image_created_at(plan.image)
|
||||
if ts is not None:
|
||||
check_stale(f"agent image {plan.image!r}", ts)
|
||||
|
||||
@@ -5,6 +5,7 @@ existence, and building images."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -197,3 +198,46 @@ def commit_container(container_name: str, image_tag: str) -> None:
|
||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||
)
|
||||
info(f"committed {container_name!r} → {image_tag!r}")
|
||||
|
||||
|
||||
def image_created_at(ref: str) -> datetime | None:
|
||||
"""Return Docker's image Created timestamp as an aware UTC datetime, or
|
||||
None when the field is absent or unparseable. Callers should skip the
|
||||
stale check when None is returned."""
|
||||
r = subprocess.run(
|
||||
["docker", "image", "inspect", "--format", "{{.Created}}", ref],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if r.returncode != 0:
|
||||
die(
|
||||
f"docker image inspect for {ref!r} failed: "
|
||||
f"{(r.stderr or '').strip() or '<no stderr>'}"
|
||||
)
|
||||
raw = r.stdout.strip()
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return _parse_docker_timestamp(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_docker_timestamp(raw: str) -> datetime:
|
||||
text = raw.strip()
|
||||
if text.endswith("Z"):
|
||||
text = text[:-1] + "+00:00"
|
||||
dot = text.find(".")
|
||||
if dot != -1:
|
||||
tz_plus = text.find("+", dot)
|
||||
tz_minus = text.find("-", dot)
|
||||
tz_candidates = [pos for pos in (tz_plus, tz_minus) if pos != -1]
|
||||
if tz_candidates:
|
||||
tz_pos = min(tz_candidates)
|
||||
frac = text[dot + 1:tz_pos]
|
||||
text = text[:dot + 1] + frac[:6].ljust(6, "0") + text[tz_pos:]
|
||||
dt = datetime.fromisoformat(text)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
|
||||
@@ -18,7 +18,7 @@ from ...env import ResolvedEnv
|
||||
from ...git_gate import GitGatePlan
|
||||
from ...manifest import Manifest
|
||||
from ...supervise import SupervisePlan
|
||||
from .. import ActiveAgent, BottleBackend, BottleSpec
|
||||
from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec
|
||||
from . import cleanup as _cleanup
|
||||
from . import enumerate as _enumerate
|
||||
from . import launch as _launch
|
||||
@@ -92,11 +92,18 @@ class FirecrackerBottleBackend(
|
||||
stage_dir=stage_dir,
|
||||
)
|
||||
|
||||
def _build_or_load_images(self, plan: FirecrackerBottlePlan) -> BottleImages:
|
||||
return BottleImages(agent=_launch.build_or_load_agent_base(plan))
|
||||
|
||||
def prelaunch_checks(self, plan: FirecrackerBottlePlan) -> None:
|
||||
_launch.stale_checks(plan)
|
||||
|
||||
@contextmanager
|
||||
def launch(
|
||||
self, plan: FirecrackerBottlePlan
|
||||
def _launch_impl(
|
||||
self, plan: FirecrackerBottlePlan, images: BottleImages,
|
||||
) -> Generator[FirecrackerBottle, None, None]:
|
||||
with _launch.launch(plan, provision=self.provision) as bottle:
|
||||
assert isinstance(images.agent, Path)
|
||||
with _launch.launch(plan, images.agent, provision=self.provision) as bottle:
|
||||
yield bottle
|
||||
|
||||
def prepare_cleanup(self) -> FirecrackerBottleCleanupPlan:
|
||||
|
||||
@@ -50,6 +50,7 @@ class LaunchContext:
|
||||
source_ip: str # the VM's guest IP — the attribution key
|
||||
gateway_ca_pem: str # the shared gateway CA the provisioner installs
|
||||
orchestrator_url: str
|
||||
env_var_secret: str = "" # encryption key injected into the agent's env
|
||||
|
||||
|
||||
def launch_consolidated(
|
||||
@@ -80,6 +81,7 @@ def launch_consolidated(
|
||||
source_ip=guest_ip,
|
||||
gateway_ca_pem=infra.gateway_ca_pem(),
|
||||
orchestrator_url=url,
|
||||
env_var_secret=reg.env_var_secret,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ backend — we stream the guest root filesystem out over the control
|
||||
channel (SSH here). Unlike the other backends this needs no Docker: the
|
||||
tar *is* the resumable artifact. `resume` extracts it and rebuilds a
|
||||
fresh per-bottle ext4 with `mke2fs -d` (see `util.build_committed_rootfs_dir`
|
||||
and `launch._build_agent_base`). The bottle keeps running after the
|
||||
and `launch.build_or_load_agent_base`). The bottle keeps running after the
|
||||
snapshot.
|
||||
"""
|
||||
|
||||
|
||||
@@ -58,6 +58,12 @@ def _rootfs_digest(dockerfile: Path) -> str:
|
||||
return h.hexdigest()[:16]
|
||||
|
||||
|
||||
def cached_agent_rootfs_dir(dockerfile: Path) -> Path | None:
|
||||
"""Return the ready cached rootfs for ``dockerfile``, if one exists."""
|
||||
base = util.cache_dir() / "rootfs" / f"agent-{_rootfs_digest(dockerfile)}"
|
||||
return base if (base / ".bb-ready").is_file() else None
|
||||
|
||||
|
||||
def build_agent_rootfs_dir(
|
||||
dockerfile: Path, *, image_tag: str, smoke_test: tuple[str, ...] = (),
|
||||
) -> Path:
|
||||
@@ -72,7 +78,7 @@ def build_agent_rootfs_dir(
|
||||
silent-failure image at build time rather than at first agent use."""
|
||||
digest = _rootfs_digest(dockerfile)
|
||||
base = util.cache_dir() / "rootfs" / f"agent-{digest}"
|
||||
if (base / ".bb-ready").is_file():
|
||||
if cached_agent_rootfs_dir(dockerfile) is not None:
|
||||
info(f"using cached agent rootfs {base.name}")
|
||||
return base
|
||||
|
||||
|
||||
@@ -45,7 +45,8 @@ from ...git_gate import (
|
||||
provision_git_gate_dynamic_keys,
|
||||
revoke_git_gate_provisioned_keys,
|
||||
)
|
||||
from ...log import info, warn
|
||||
from ...image_cache import check_stale_path
|
||||
from ...log import die, info, warn
|
||||
from ...supervise import SUPERVISE_PORT
|
||||
from ..docker.egress import EGRESS_PORT
|
||||
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||
@@ -65,6 +66,7 @@ _GIT_HTTP_PORT = 9420
|
||||
@contextmanager
|
||||
def launch(
|
||||
plan: FirecrackerBottlePlan,
|
||||
agent_base: Path,
|
||||
*,
|
||||
provision: Callable[[FirecrackerBottlePlan, "FirecrackerBottle"], str | None],
|
||||
) -> Generator[FirecrackerBottle, None, None]:
|
||||
@@ -86,11 +88,9 @@ def launch(
|
||||
raise teardown_exc
|
||||
|
||||
try:
|
||||
# Step 1: agent rootfs. Built from the Dockerfile inside a Firecracker
|
||||
# builder VM (buildah, no host docker); a committed snapshot is reused
|
||||
# when present. Returns the base dir the per-bottle ext4 is made from.
|
||||
plan, agent_base = _build_agent_base(plan)
|
||||
|
||||
# Step 1 (rootfs resolution/build) runs in BottleBackend.launch before
|
||||
# this context starts resources. ``agent_base`` is the selected cache,
|
||||
# fresh build, or committed snapshot.
|
||||
# Step 2: mint the git-gate dynamic (gitea) deploy keys, if any.
|
||||
git_gate_plan = plan.git_gate_plan
|
||||
if git_gate_plan.upstreams:
|
||||
@@ -209,9 +209,7 @@ def launch(
|
||||
teardown()
|
||||
|
||||
|
||||
def _build_agent_base(
|
||||
plan: FirecrackerBottlePlan,
|
||||
) -> tuple[FirecrackerBottlePlan, Path]:
|
||||
def build_or_load_agent_base(plan: FirecrackerBottlePlan) -> Path:
|
||||
"""Produce the agent's base rootfs dir. Primary path: build the Dockerfile
|
||||
inside a Firecracker builder VM (buildah, no host docker), smoke-testing
|
||||
the image before export. A committed snapshot (freeze/migrate) is resumed
|
||||
@@ -220,13 +218,36 @@ def _build_agent_base(
|
||||
committed_tar = committed_rootfs_path(plan.slug)
|
||||
if committed and committed_tar.is_file():
|
||||
info(f"resuming from committed rootfs {committed_tar}")
|
||||
return plan, util.build_committed_rootfs_dir(committed_tar)
|
||||
base = image_builder.build_agent_rootfs_dir(
|
||||
Path(plan.dockerfile_path),
|
||||
return util.build_committed_rootfs_dir(committed_tar)
|
||||
dockerfile = Path(plan.dockerfile_path)
|
||||
if plan.spec.image_policy == "cached":
|
||||
cached = image_builder.cached_agent_rootfs_dir(dockerfile)
|
||||
if cached is None:
|
||||
die(
|
||||
f"cached agent rootfs for {plan.image!r} not found; "
|
||||
"run without --cached-images to build it"
|
||||
)
|
||||
info(f"using cached agent rootfs {cached.name}")
|
||||
return cached
|
||||
return image_builder.build_agent_rootfs_dir(
|
||||
dockerfile,
|
||||
image_tag=plan.image,
|
||||
smoke_test=runtime_for(plan.agent_provider_template).smoke_test,
|
||||
)
|
||||
return plan, base
|
||||
|
||||
|
||||
def stale_checks(plan: FirecrackerBottlePlan) -> None:
|
||||
"""Raise when the cached rootfs selected by this plan is stale."""
|
||||
if plan.spec.image_policy != "cached":
|
||||
return
|
||||
committed = read_committed_image(plan.slug)
|
||||
committed_tar = committed_rootfs_path(plan.slug)
|
||||
if committed and committed_tar.is_file():
|
||||
check_stale_path(f"agent rootfs {committed_tar}", committed_tar)
|
||||
return
|
||||
cached = image_builder.cached_agent_rootfs_dir(Path(plan.dockerfile_path))
|
||||
if cached is not None:
|
||||
check_stale_path(f"agent rootfs {cached}", cached / ".bb-ready")
|
||||
|
||||
|
||||
# --- agent guest env -------------------------------------------------
|
||||
@@ -242,6 +263,11 @@ def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str
|
||||
"HTTPS_PROXY": proxy_url, "HTTP_PROXY": proxy_url,
|
||||
"https_proxy": proxy_url, "http_proxy": proxy_url,
|
||||
"NO_PROXY": no_proxy, "no_proxy": no_proxy,
|
||||
# Rootfs export can leave Git's implicit XDG paths unreadable even
|
||||
# after the runtime repair. Bypass that discovery and name the
|
||||
# provisioned global config explicitly so insteadOf can never fall
|
||||
# through to the credential-bearing upstream URL.
|
||||
"GIT_CONFIG_GLOBAL": f"{plan.guest_home}/.gitconfig",
|
||||
"NODE_EXTRA_CA_CERTS": AGENT_CA_PATH,
|
||||
"SSL_CERT_FILE": AGENT_CA_BUNDLE,
|
||||
"REQUESTS_CA_BUNDLE": AGENT_CA_BUNDLE,
|
||||
|
||||
@@ -131,6 +131,29 @@ def build_artifact(out_dir: Path) -> tuple[str, Path, Path]:
|
||||
return version, gz, sha
|
||||
|
||||
|
||||
def _try_download_published(out_dir: Path) -> tuple[str, Path, Path] | None:
|
||||
"""If this version's artifact is already in the registry, download the gz
|
||||
and sha to out_dir and return (version, gz_path, sha_path). Returns None
|
||||
when not yet published."""
|
||||
version = infra_artifact.infra_artifact_version(infra_vm._infra_init())
|
||||
sha_url = infra_artifact.artifact_url(version, "rootfs.ext4.gz.sha256")
|
||||
try:
|
||||
with urllib.request.urlopen(infra_artifact._open(sha_url)):
|
||||
pass
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
return None
|
||||
raise SystemExit(f"registry check failed (HTTP {e.code}): {sha_url}")
|
||||
except urllib.error.URLError as e:
|
||||
raise SystemExit(f"registry unreachable: {sha_url} ({e.reason})")
|
||||
print(f"infra rootfs {version} already published — downloading instead of building")
|
||||
gz = out_dir / "rootfs.ext4.gz"
|
||||
sha = out_dir / "rootfs.ext4.gz.sha256"
|
||||
infra_artifact._download(infra_artifact.artifact_url(version, "rootfs.ext4.gz"), gz)
|
||||
infra_artifact._download(infra_artifact.artifact_url(version, "rootfs.ext4.gz.sha256"), sha)
|
||||
return version, gz, sha
|
||||
|
||||
|
||||
def _publish_bundle(root: Path, token: str) -> str:
|
||||
version_file = root / "version.txt"
|
||||
# Guard the read so a missing version.txt is a clean error, not a raw
|
||||
@@ -187,6 +210,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
help="build a candidate bundle in DIR without publishing")
|
||||
mode.add_argument("--publish-dir", type=Path,
|
||||
help="publish an already-built and tested candidate bundle")
|
||||
parser.add_argument("--reuse-published", action="store_true",
|
||||
help="with --output: download from registry if already published instead of building")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
_, _, token = infra_artifact._config()
|
||||
@@ -197,6 +222,14 @@ def main(argv: list[str] | None = None) -> int:
|
||||
|
||||
if args.output is not None:
|
||||
args.output.mkdir(parents=True, exist_ok=True)
|
||||
reused = None
|
||||
if args.reuse_published:
|
||||
reused = _try_download_published(args.output)
|
||||
if reused is not None:
|
||||
version, _, _ = reused
|
||||
(args.output / "version.txt").write_text(version + "\n", encoding="utf-8")
|
||||
print(f"reused published infra rootfs candidate {version}")
|
||||
return 0
|
||||
version, _gz, _sha = build_artifact(args.output)
|
||||
(args.output / "version.txt").write_text(version + "\n", encoding="utf-8")
|
||||
print(f"built infra rootfs candidate {version}")
|
||||
|
||||
@@ -373,6 +373,12 @@ mount -o remount,rw / 2>/dev/null
|
||||
# scratch dirs there — git worktrees, build temp, `git init /tmp/...`, etc.
|
||||
mkdir -p /tmp && chmod 1777 /tmp
|
||||
|
||||
# Rootfs export also maps the image's original owners to the unprivileged
|
||||
# host build uid. That uid is not guaranteed to be node's uid in the guest;
|
||||
# restore the home-directory boundary before any SSH provisioning runs.
|
||||
chown node:node /home/node 2>/dev/null || true
|
||||
chmod 755 /home/node 2>/dev/null || true
|
||||
|
||||
# Install the per-bottle SSH pubkey from the kernel cmdline.
|
||||
KEY=$(sed -n 's/.*bb_pubkey=\([^ ]*\).*/\1/p' /proc/cmdline | base64 -d 2>/dev/null)
|
||||
if [ -n "$KEY" ]; then
|
||||
|
||||
@@ -12,7 +12,7 @@ from ...env import ResolvedEnv
|
||||
from ...git_gate import GitGatePlan
|
||||
from ...supervise import SupervisePlan
|
||||
from ...manifest import Manifest
|
||||
from .. import ActiveAgent, BottleBackend, BottleSpec
|
||||
from .. import ActiveAgent, BottleBackend, BottleImages, BottleSpec
|
||||
from . import cleanup as _cleanup
|
||||
from . import enumerate as _enumerate
|
||||
from . import launch as _launch
|
||||
@@ -82,11 +82,17 @@ class MacosContainerBottleBackend(
|
||||
stage_dir=stage_dir,
|
||||
)
|
||||
|
||||
def prelaunch_checks(self, plan: MacosContainerBottlePlan) -> None:
|
||||
_launch.stale_checks(plan)
|
||||
|
||||
def _build_or_load_images(self, plan: MacosContainerBottlePlan) -> BottleImages:
|
||||
return _launch.build_or_load_images(plan)
|
||||
|
||||
@contextmanager
|
||||
def launch(
|
||||
self, plan: MacosContainerBottlePlan
|
||||
def _launch_impl(
|
||||
self, plan: MacosContainerBottlePlan, images: BottleImages
|
||||
) -> Generator[MacosContainerBottle, None, None]:
|
||||
with _launch.launch(plan, provision=self.provision) as bottle:
|
||||
with _launch.launch(plan, images, provision=self.provision) as bottle:
|
||||
yield bottle
|
||||
|
||||
def ensure_orchestrator(self) -> str:
|
||||
|
||||
@@ -72,6 +72,7 @@ class LaunchContext:
|
||||
gateway_ip: str
|
||||
network: str
|
||||
orchestrator_url: str
|
||||
env_var_secret: str = "" # encryption key injected into the agent's env
|
||||
|
||||
|
||||
def ensure_gateway(
|
||||
@@ -152,6 +153,7 @@ def register_agent(
|
||||
gateway_ip=endpoint.gateway_ip,
|
||||
network=endpoint.network,
|
||||
orchestrator_url=endpoint.orchestrator_url,
|
||||
env_var_secret=reg.env_var_secret,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ... import log
|
||||
from ...orchestrator.gateway import GATEWAY_CA_CERT
|
||||
from ...orchestrator.gateway import GATEWAY_CA_CERT, MITMPROXY_HOME
|
||||
from ...orchestrator.lifecycle import (
|
||||
DEFAULT_PORT,
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
@@ -52,6 +52,7 @@ from ...paths import (
|
||||
CONTROL_PLANE_TOKEN_ENV,
|
||||
HOST_DB_FILENAME,
|
||||
host_control_plane_token,
|
||||
host_gateway_ca_dir,
|
||||
)
|
||||
from .. import util as backend_util
|
||||
from . import util as container_mod
|
||||
@@ -218,6 +219,14 @@ class MacosInfraService:
|
||||
# Container-only DB volume: one kernel writes bot-bottle.db, never
|
||||
# shared with the host or another guest.
|
||||
"--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}",
|
||||
# The DB needs a container-only ext4 volume for coherent SQLite
|
||||
# locking, but the CA has no such constraint. Keep it in the host
|
||||
# app-data root so infra-container recreation and Apple Container
|
||||
# volume pruning cannot silently rotate every bottle's trust
|
||||
# anchor (issue #450).
|
||||
"--mount",
|
||||
container_mod.bind_mount_spec(
|
||||
str(host_gateway_ca_dir()), MITMPROXY_HOME),
|
||||
# Bind-mount the control-plane source (read-only); a code change
|
||||
# takes effect on relaunch with no image rebuild.
|
||||
"--mount",
|
||||
@@ -261,9 +270,9 @@ class MacosInfraService:
|
||||
|
||||
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
||||
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
|
||||
interception. Read out of the container (the CA lives on a
|
||||
container-internal path, not a host mount); polls because mitmproxy
|
||||
writes it a beat after start."""
|
||||
interception. Read through the container path backed by the persistent
|
||||
host CA directory; polls because mitmproxy writes it a beat after
|
||||
start."""
|
||||
def _fetch() -> str | None:
|
||||
result = container_mod.run_container_argv(
|
||||
["container", "exec", self._name, "cat", GATEWAY_CA_CERT])
|
||||
|
||||
@@ -53,7 +53,9 @@ from ...git_gate import (
|
||||
revoke_git_gate_provisioned_keys,
|
||||
)
|
||||
from ...git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT
|
||||
from ...image_cache import check_stale
|
||||
from ...log import die, info, warn
|
||||
from .. import BottleImages
|
||||
from ...supervise import SUPERVISE_PORT
|
||||
from ..docker.egress import EGRESS_PORT
|
||||
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||
@@ -77,18 +79,43 @@ _REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
|
||||
_AGENT_SLEEP_SECONDS = "2147483647"
|
||||
|
||||
|
||||
def build_or_load_images(plan: MacosContainerBottlePlan) -> BottleImages:
|
||||
"""Resolve the agent image ref for this plan. The gateway's own image is
|
||||
built by `ensure_gateway` — it belongs to the shared singleton."""
|
||||
committed = read_committed_image(plan.slug)
|
||||
if committed and container_mod.image_exists(committed):
|
||||
info(f"using committed image {committed!r}")
|
||||
return BottleImages(agent=committed)
|
||||
if plan.spec.image_policy == "cached":
|
||||
if not container_mod.image_exists(plan.image):
|
||||
die(
|
||||
f"cached agent image {plan.image!r} not found; "
|
||||
"run without --cached-images to build it"
|
||||
)
|
||||
info(f"using cached agent image {plan.image!r}")
|
||||
return BottleImages(agent=plan.image)
|
||||
container_mod.build_image(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path)
|
||||
return BottleImages(agent=plan.image)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def launch(
|
||||
plan: MacosContainerBottlePlan,
|
||||
images: BottleImages,
|
||||
*,
|
||||
provision: Callable[[MacosContainerBottlePlan, "MacosContainerBottle"], str | None],
|
||||
) -> Generator[MacosContainerBottle, None, None]:
|
||||
"""Build, run, register, provision, and yield an Apple Container bottle on
|
||||
the shared per-host gateway."""
|
||||
"""Run, register, provision, and yield an Apple Container bottle on the
|
||||
shared per-host gateway."""
|
||||
stack = ExitStack()
|
||||
bottle_for_revoke = plan.manifest.bottle
|
||||
git_gate_dir_for_revoke = git_gate_state_dir(plan.slug)
|
||||
|
||||
plan = dataclasses.replace(
|
||||
plan,
|
||||
agent_provision=dataclasses.replace(plan.agent_provision, image=str(images.agent)),
|
||||
)
|
||||
|
||||
def teardown() -> None:
|
||||
teardown_exc: BaseException | None = None
|
||||
try:
|
||||
@@ -101,8 +128,6 @@ def launch(
|
||||
raise teardown_exc
|
||||
|
||||
try:
|
||||
plan = _build_images(plan)
|
||||
|
||||
# Step 1: the per-host singletons. Must precede the agent run — its
|
||||
# proxy env needs the gateway's address at `container run` time.
|
||||
endpoint = ensure_gateway()
|
||||
@@ -193,22 +218,23 @@ def launch(
|
||||
teardown()
|
||||
|
||||
|
||||
def _build_images(plan: MacosContainerBottlePlan) -> MacosContainerBottlePlan:
|
||||
"""Build the agent image. The gateway's own image is built by
|
||||
`ensure_gateway` — it belongs to the shared singleton, not to a bottle."""
|
||||
|
||||
def stale_checks(plan: MacosContainerBottlePlan) -> None:
|
||||
"""Raise StaleImageError if a cached image is older than the configured
|
||||
threshold. Only runs when image_policy is 'cached'. Called by the backend
|
||||
class's _image_stale_checks before _launch_impl starts any resources."""
|
||||
if plan.spec.image_policy != "cached":
|
||||
return
|
||||
committed = read_committed_image(plan.slug)
|
||||
if committed and container_mod.image_exists(committed):
|
||||
info(f"using committed image {committed!r}")
|
||||
return dataclasses.replace(
|
||||
plan,
|
||||
agent_provision=dataclasses.replace(
|
||||
plan.agent_provision, image=committed,
|
||||
),
|
||||
)
|
||||
container_mod.build_image(
|
||||
plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path,
|
||||
)
|
||||
return plan
|
||||
ts = container_mod.image_created_at(committed)
|
||||
if ts is not None:
|
||||
check_stale(f"agent image {committed!r}", ts)
|
||||
return
|
||||
if container_mod.image_exists(plan.image):
|
||||
ts = container_mod.image_created_at(plan.image)
|
||||
if ts is not None:
|
||||
check_stale(f"agent image {plan.image!r}", ts)
|
||||
|
||||
|
||||
def _provision_git_gate_keys(
|
||||
|
||||
@@ -10,6 +10,7 @@ import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Iterable
|
||||
|
||||
from ...log import die, info
|
||||
@@ -661,6 +662,39 @@ def image_id(ref: str) -> str:
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
def image_created_at(ref: str) -> datetime | None:
|
||||
"""Return the image creation timestamp as an aware UTC datetime, or None
|
||||
when the field is absent or unparseable (e.g. FROM-scratch images, images
|
||||
pulled from registries that omit the field). Callers should skip the stale
|
||||
check when None is returned rather than treating it as an error."""
|
||||
result = subprocess.run(
|
||||
[_CONTAINER, "image", "inspect", ref],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
die(
|
||||
f"container image inspect for {ref!r} failed: "
|
||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||
)
|
||||
try:
|
||||
data = json.loads(result.stdout or "{}")
|
||||
except json.JSONDecodeError as exc:
|
||||
die(f"container image inspect for {ref!r} returned malformed JSON: {exc}")
|
||||
if isinstance(data, list) and data:
|
||||
data = data[0]
|
||||
if isinstance(data, dict):
|
||||
value = data.get("created") or data.get("Created")
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
ts = value.rstrip("Z")
|
||||
return datetime.fromisoformat(ts).replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def save(ref: str, output: str) -> None:
|
||||
subprocess.run([_CONTAINER, "image", "save", ref, "-o", output], check=True)
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ from .commit import cmd_commit
|
||||
from .edit import cmd_edit
|
||||
from .info import cmd_info
|
||||
from .init import cmd_init
|
||||
from .login import cmd_login
|
||||
from .resume import cmd_resume
|
||||
from .start import cmd_start
|
||||
from .supervise import cmd_supervise
|
||||
@@ -33,6 +34,7 @@ COMMANDS = {
|
||||
"info": cmd_info,
|
||||
"init": cmd_init,
|
||||
"list": cmd_list,
|
||||
"login": cmd_login,
|
||||
"resume": cmd_resume,
|
||||
"start": cmd_start,
|
||||
"supervise": cmd_supervise,
|
||||
@@ -43,7 +45,7 @@ COMMANDS = {
|
||||
# the host (TAP pool, /dev/kvm, firecracker) and never opens the store, so
|
||||
# gating it on the schema breaks preflight on a fresh CI runner where stdin
|
||||
# isn't a TTY and the migration prompt can't be answered.
|
||||
NO_MIGRATION_COMMANDS = frozenset({"backend"})
|
||||
NO_MIGRATION_COMMANDS = frozenset({"backend", "login"})
|
||||
|
||||
|
||||
def usage() -> None:
|
||||
@@ -56,6 +58,7 @@ def usage() -> None:
|
||||
sys.stderr.write(" info print env, skills, and prompt details for a named agent\n")
|
||||
sys.stderr.write(" init interactively create a new agent and add it to bot-bottle.json\n")
|
||||
sys.stderr.write(" list list available agents or active containers\n")
|
||||
sys.stderr.write(" login register this host with a bot-bottle console\n")
|
||||
sys.stderr.write(
|
||||
" resume re-launch a bottle by its identity "
|
||||
"(continues state from PRD 0016)\n"
|
||||
@@ -111,7 +114,3 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return e.code if isinstance(e.code, int) else 1
|
||||
except KeyboardInterrupt:
|
||||
return 130
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
"""Entry point for `python -m bot_bottle.cli`.
|
||||
|
||||
`cli.py` at the repo root is the usual way in; this makes the package
|
||||
runnable too, so the CLI works from an installed copy where there is no
|
||||
`cli.py` on disk to point at.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from . import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,168 @@
|
||||
"""bb login — register this host with a bot-bottle console.
|
||||
|
||||
Opens a device-authorization flow against the target console, waits for the
|
||||
operator to approve, then writes access and refresh tokens to
|
||||
~/.bot-bottle/console.json (or $BOT_BOTTLE_ROOT/console.json).
|
||||
|
||||
Usage:
|
||||
bb login [--console-url URL] [--label LABEL]
|
||||
|
||||
Flags:
|
||||
--console-url URL Target console URL (overrides BB_CONSOLE_URL env var)
|
||||
--label LABEL Host label shown in the console (default: hostname)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from ..paths import bot_bottle_root
|
||||
|
||||
_CONSOLE_URL_ENV = "BB_CONSOLE_URL"
|
||||
_POLL_SLEEP = 2 # seconds between polls; matches console's poll_interval default
|
||||
|
||||
|
||||
def _usage() -> None:
|
||||
sys.stderr.write(
|
||||
"usage: bb login [--console-url URL] [--label LABEL]\n"
|
||||
"\n"
|
||||
"Options:\n"
|
||||
" --console-url URL Console base URL (or BB_CONSOLE_URL env var)\n"
|
||||
" --label LABEL Host label shown in the console (default: hostname)\n"
|
||||
)
|
||||
|
||||
|
||||
def _flag(argv: list[str], name: str) -> str | None:
|
||||
for i, arg in enumerate(argv):
|
||||
if arg == name and i + 1 < len(argv):
|
||||
return argv[i + 1]
|
||||
if arg.startswith(f"{name}="):
|
||||
return arg[len(name) + 1:]
|
||||
return None
|
||||
|
||||
|
||||
def _post(url: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
data = json.dumps(payload).encode()
|
||||
req = urllib.request.Request(
|
||||
url, data=data, headers={"Content-Type": "application/json"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
return json.loads(resp.read())
|
||||
|
||||
|
||||
def _get(url: str) -> tuple[int, dict[str, Any]]:
|
||||
req = urllib.request.Request(url)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
return resp.status, json.loads(resp.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, {}
|
||||
|
||||
|
||||
def _save_credentials(
|
||||
console_url: str, host_id: str, access_token: str, refresh_token: str
|
||||
) -> Path:
|
||||
path = bot_bottle_root() / "console.json"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
content = (
|
||||
json.dumps(
|
||||
{
|
||||
"url": console_url,
|
||||
"host_id": host_id,
|
||||
"access_token": access_token,
|
||||
"refresh_token": refresh_token,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
fd, tmp_path_str = tempfile.mkstemp(dir=path.parent, prefix=".console-")
|
||||
tmp = Path(tmp_path_str)
|
||||
try:
|
||||
tmp.chmod(0o600)
|
||||
with os.fdopen(fd, "w") as f:
|
||||
f.write(content)
|
||||
os.replace(tmp, path)
|
||||
except OSError:
|
||||
try:
|
||||
tmp.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
return path
|
||||
|
||||
|
||||
def cmd_login(argv: list[str]) -> int:
|
||||
if "--help" in argv or "-h" in argv:
|
||||
_usage()
|
||||
return 0
|
||||
|
||||
console_url = _flag(argv, "--console-url") or os.environ.get(_CONSOLE_URL_ENV)
|
||||
if not console_url:
|
||||
sys.stderr.write(
|
||||
"bb login: --console-url or BB_CONSOLE_URL is required\n"
|
||||
)
|
||||
return 1
|
||||
console_url = console_url.rstrip("/")
|
||||
|
||||
label = _flag(argv, "--label") or socket.gethostname()
|
||||
|
||||
try:
|
||||
resp = _post(f"{console_url}/api/v1/hosts/authorize", {"label": label})
|
||||
except (OSError, ValueError) as exc:
|
||||
sys.stderr.write(f"bb login: failed to start authorization: {exc}\n")
|
||||
return 1
|
||||
|
||||
device_code = resp["device_code"]
|
||||
user_code = resp["user_code"]
|
||||
expires_in = resp.get("expires_in", 300)
|
||||
poll_sleep = max(1, min(int(resp.get("poll_interval", _POLL_SLEEP)), 60))
|
||||
|
||||
sys.stderr.write(
|
||||
f"\nOpen this URL in your browser to authorize this host:\n\n"
|
||||
f" {console_url}/hosts/authorize?code={user_code}\n\n"
|
||||
f"Waiting for approval"
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + expires_in
|
||||
while time.monotonic() < deadline:
|
||||
sys.stderr.write(".")
|
||||
sys.stderr.flush()
|
||||
time.sleep(poll_sleep)
|
||||
|
||||
try:
|
||||
code, result = _get(
|
||||
f"{console_url}/api/v1/hosts/authorize/{device_code}"
|
||||
)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
|
||||
if code == 410:
|
||||
break
|
||||
|
||||
st = result.get("status")
|
||||
if st == "approved":
|
||||
sys.stderr.write("\n\nApproved.\n")
|
||||
path = _save_credentials(
|
||||
console_url,
|
||||
result["host_id"],
|
||||
result["access_token"],
|
||||
result["refresh_token"],
|
||||
)
|
||||
sys.stderr.write(f"Credentials saved to {path}\n")
|
||||
return 0
|
||||
if st == "denied":
|
||||
sys.stderr.write("\n\nDenied by operator.\n")
|
||||
return 1
|
||||
|
||||
sys.stderr.write("\n\nAuthorization timed out.\n")
|
||||
return 1
|
||||
+33
-4
@@ -35,6 +35,7 @@ from ..bottle_state import (
|
||||
is_preserved,
|
||||
mark_preserved,
|
||||
)
|
||||
from ..image_cache import StaleImageError
|
||||
from ..log import info, die
|
||||
from ..manifest import Manifest, ManifestIndex
|
||||
from ._common import PROG, USER_CWD, read_tty_line
|
||||
@@ -64,6 +65,14 @@ def cmd_start(argv: list[str]) -> int:
|
||||
"skip all prompts. For orchestrators, CI, and webhooks."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cached-images",
|
||||
action="store_true",
|
||||
help=(
|
||||
"quickstart with existing local agent and sidecar images; "
|
||||
"only valid with --headless"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bottle",
|
||||
action="append",
|
||||
@@ -96,6 +105,8 @@ def cmd_start(argv: list[str]) -> int:
|
||||
help="agent name defined in bot-bottle.json (omit to pick interactively)",
|
||||
)
|
||||
args = parser.parse_args(argv)
|
||||
if args.cached_images and not args.headless:
|
||||
die("--cached-images is only supported with --headless")
|
||||
|
||||
dry_run = args.dry_run or os.environ.get("BOT_BOTTLE_DRY_RUN") == "1"
|
||||
if args.no_cache or os.environ.get("BOT_BOTTLE_NO_CACHE") == "1":
|
||||
@@ -147,6 +158,10 @@ def cmd_start(argv: list[str]) -> int:
|
||||
label, color = tui.name_color_modal(default_label=agent_name)
|
||||
label, color = _resolve_unique_label(label, color)
|
||||
|
||||
image_policy = _select_image_policy()
|
||||
if image_policy is None:
|
||||
return 0
|
||||
|
||||
spec = BottleSpec(
|
||||
manifest=manifest,
|
||||
agent_name=agent_name,
|
||||
@@ -155,6 +170,7 @@ def cmd_start(argv: list[str]) -> int:
|
||||
label=label,
|
||||
color=color,
|
||||
bottle_names=bottle_names,
|
||||
image_policy=image_policy,
|
||||
)
|
||||
return _launch_bottle(
|
||||
spec,
|
||||
@@ -213,6 +229,7 @@ def _start_headless(
|
||||
color=args.color or "",
|
||||
bottle_names=bottle_names,
|
||||
headless=True,
|
||||
image_policy="cached" if args.cached_images else "fresh",
|
||||
)
|
||||
return _launch_bottle(
|
||||
spec,
|
||||
@@ -395,6 +412,13 @@ def _text_prompt_yes() -> bool:
|
||||
return reply in ("y", "Y", "yes", "YES")
|
||||
|
||||
|
||||
def _select_image_policy() -> str | None:
|
||||
return tui.filter_select(
|
||||
["fresh", "cached"],
|
||||
title="Select image startup mode",
|
||||
)
|
||||
|
||||
|
||||
def _text_render_preflight():
|
||||
def _render(plan: DockerBottlePlan, backend_name: str) -> None:
|
||||
print(file=sys.stderr)
|
||||
@@ -537,6 +561,15 @@ def _launch_bottle(
|
||||
return 0
|
||||
|
||||
backend = get_bottle_backend(backend_name)
|
||||
try:
|
||||
backend.prelaunch_checks(plan)
|
||||
except StaleImageError as exc:
|
||||
if assume_yes:
|
||||
die(str(exc))
|
||||
sys.stderr.write(f"bot-bottle: {exc}\nLaunch anyway? [y/N] ")
|
||||
sys.stderr.flush()
|
||||
if read_tty_line() not in ("y", "Y", "yes", "YES"):
|
||||
return 0
|
||||
with backend.launch(plan) as bottle:
|
||||
agent_provider_template = getattr(plan, "agent_provider_template", "claude")
|
||||
extra_args: tuple[str, ...] = ()
|
||||
@@ -555,10 +588,6 @@ def _launch_bottle(
|
||||
f"session ended (exit {exit_code}); "
|
||||
f"container {bottle.name} will be removed"
|
||||
)
|
||||
# While the container is still alive: always snapshot the
|
||||
# transcript and — if the agent exited non-zero — mark
|
||||
# the state for preservation. This picks up crashes /
|
||||
# Ctrl-Cs / OOM kills before cleanup removes the state dir.
|
||||
if agent_provider_template == "claude":
|
||||
capture_claude_session_state(identity, exit_code)
|
||||
return 0
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""SQLite-backed bot-bottle configuration store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from .db_store import DbStore
|
||||
from .migrations import TableMigrations
|
||||
from .paths import host_db_path
|
||||
except ImportError:
|
||||
from db_store import DbStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
|
||||
from migrations import TableMigrations # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
|
||||
from paths import host_db_path # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
|
||||
|
||||
|
||||
DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS = 1
|
||||
|
||||
|
||||
class ConfigStore(DbStore):
|
||||
"""SQLite configuration for host-side bot-bottle settings."""
|
||||
|
||||
def __init__(self, db_path: Path | None = None) -> None:
|
||||
migrations = TableMigrations("config_store", [
|
||||
# v1 — host-side bot-bottle settings
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS bot_bottle_config (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
cached_image_stale_warning_days INTEGER NOT NULL DEFAULT 1
|
||||
)
|
||||
""",
|
||||
])
|
||||
super().__init__(db_path or host_db_path(), migrations)
|
||||
|
||||
def cached_image_stale_warning_days(self) -> int:
|
||||
if not self.db_path.is_file():
|
||||
return DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT cached_image_stale_warning_days
|
||||
FROM bot_bottle_config
|
||||
WHERE id = 1
|
||||
""",
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS
|
||||
try:
|
||||
return int(row["cached_image_stale_warning_days"])
|
||||
except (TypeError, ValueError):
|
||||
return DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS
|
||||
|
||||
def set_cached_image_stale_warning_days(self, days: int) -> Path:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO bot_bottle_config (id, cached_image_stale_warning_days)
|
||||
VALUES (1, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
cached_image_stale_warning_days = excluded.cached_image_stale_warning_days
|
||||
""",
|
||||
(days,),
|
||||
)
|
||||
self._chmod()
|
||||
return self.db_path
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS",
|
||||
"ConfigStore",
|
||||
]
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
# Current Node LTS; slim variant keeps the image small while still
|
||||
# providing apt-get for any future additions.
|
||||
FROM node:22-slim
|
||||
FROM node:22-trixie-slim
|
||||
|
||||
# Install runtime system deps. claude-code shells out to git for several
|
||||
# features (status checks, commits, PR creation) — without git in the
|
||||
@@ -21,7 +21,15 @@ FROM node:22-slim
|
||||
# to it) works against egress's bumped TLS without the agent needing
|
||||
# local DNS.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git ca-certificates curl ripgrep iproute2 dnsutils \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
git \
|
||||
ca-certificates \
|
||||
curl \
|
||||
openssh-client \
|
||||
podman \
|
||||
ripgrep \
|
||||
iproute2 \
|
||||
dnsutils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# App-specific deps. Python isn't required by claude-code itself
|
||||
@@ -39,6 +47,11 @@ RUN apt-get update \
|
||||
RUN npm install -g --no-fund --no-audit @anthropic-ai/claude-code@2.1.172 \
|
||||
&& npm cache clean --force
|
||||
|
||||
# Git reads both ~/.gitconfig and ~/.config/git/config. Keep its XDG config
|
||||
# path traversable by the non-root runtime user so permission errors do not
|
||||
# suppress bot-bottle's git-gate insteadOf rules.
|
||||
RUN install -d -o node -g node -m 755 /home/node/.config /home/node/.config/git
|
||||
|
||||
# Run as a non-root user. The node image already provides a `node` user
|
||||
# (uid 1000) with a home directory, which is where claude-code will write
|
||||
# its session state.
|
||||
|
||||
@@ -23,8 +23,9 @@ from ...agent_provider import (
|
||||
provider_startup_args,
|
||||
)
|
||||
from ...backend.docker import util as docker_mod
|
||||
from ...egress import EgressRoute
|
||||
from ...egress import CLAUDE_HOST_CREDENTIAL_TOKEN_REF, EgressRoute
|
||||
from ...log import die, info, warn
|
||||
from .claude_auth import claude_host_access_token
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -118,7 +119,6 @@ class ClaudeAgentProvider(AgentProvider):
|
||||
color: str = "",
|
||||
provider_settings: dict[str, object] | None = None,
|
||||
) -> AgentProvisionPlan:
|
||||
del forward_host_credentials, host_env
|
||||
resolved_guest_env = dict(guest_env or {})
|
||||
startup_args = provider_startup_args(provider_settings)
|
||||
guest_home = self.guest_home
|
||||
@@ -180,13 +180,24 @@ class ClaudeAgentProvider(AgentProvider):
|
||||
claude_settings,
|
||||
f"{guest_home}/.claude/settings.json",
|
||||
))
|
||||
provisioned_env: dict[str, str] = {}
|
||||
if forward_host_credentials:
|
||||
_host_env = host_env or dict(os.environ)
|
||||
provisioned_env[CLAUDE_HOST_CREDENTIAL_TOKEN_REF] = (
|
||||
claude_host_access_token(_host_env)
|
||||
)
|
||||
|
||||
cred_token_ref = (
|
||||
CLAUDE_HOST_CREDENTIAL_TOKEN_REF if forward_host_credentials
|
||||
else auth_token
|
||||
)
|
||||
egress_routes = (EgressRoute(
|
||||
host="api.anthropic.com",
|
||||
auth_scheme="Bearer" if auth_token else "",
|
||||
token_ref=auth_token,
|
||||
auth_scheme="Bearer" if (auth_token or forward_host_credentials) else "",
|
||||
token_ref=cred_token_ref,
|
||||
),)
|
||||
hidden_env_names: frozenset[str] = frozenset()
|
||||
if auth_token:
|
||||
if auth_token or forward_host_credentials:
|
||||
env_vars["CLAUDE_CODE_OAUTH_TOKEN"] = "egress-placeholder"
|
||||
hidden_env_names = frozenset({"CLAUDE_CODE_OAUTH_TOKEN"})
|
||||
|
||||
@@ -208,6 +219,7 @@ class ClaudeAgentProvider(AgentProvider):
|
||||
files=tuple(files),
|
||||
egress_routes=egress_routes,
|
||||
hidden_env_names=hidden_env_names,
|
||||
provisioned_env=provisioned_env,
|
||||
)
|
||||
|
||||
def provision_skills(self, plan: "BottlePlan", bottle: "Bottle") -> None:
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Host Claude auth helpers.
|
||||
|
||||
Reads the host's Claude Code credentials and returns only the access
|
||||
token needed by egress. Does not expose refresh tokens or raw payloads.
|
||||
|
||||
Credential storage by platform:
|
||||
Linux — ~/.claude/.credentials.json
|
||||
macOS — macOS Keychain, service "Claude Code-credentials"
|
||||
(file path is tried first; Keychain is the fallback)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from ...log import die
|
||||
|
||||
|
||||
_KEYCHAIN_SERVICE = "Claude Code-credentials"
|
||||
|
||||
|
||||
def claude_auth_path(host_env: dict[str, str] | None = None) -> Path:
|
||||
env = os.environ if host_env is None else host_env
|
||||
home = env.get("HOME")
|
||||
if home:
|
||||
return Path(home) / ".claude" / ".credentials.json"
|
||||
return Path.home() / ".claude" / ".credentials.json"
|
||||
|
||||
|
||||
def _read_keychain() -> dict[str, object] | None:
|
||||
"""Try the macOS Keychain. Returns parsed JSON dict or None."""
|
||||
if sys.platform != "darwin":
|
||||
return None
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["security", "find-generic-password", "-s", _KEYCHAIN_SERVICE, "-w"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
if result.returncode != 0 or not result.stdout.strip():
|
||||
return None
|
||||
try:
|
||||
raw = json.loads(result.stdout.strip())
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
return raw if isinstance(raw, dict) else None
|
||||
|
||||
|
||||
def claude_host_access_token(
|
||||
host_env: dict[str, str] | None = None,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
) -> str:
|
||||
path = claude_auth_path(host_env)
|
||||
raw: dict[str, object] | None = None
|
||||
|
||||
if path.is_file():
|
||||
try:
|
||||
raw = json.loads(path.read_text())
|
||||
except (OSError, json.JSONDecodeError) as e:
|
||||
die(f"claude host credentials: could not read valid JSON at {path}: {e}")
|
||||
if not isinstance(raw, dict):
|
||||
die(f"claude host credentials: {path} must contain a JSON object")
|
||||
else:
|
||||
raw = _read_keychain()
|
||||
if raw is None:
|
||||
die(
|
||||
f"claude host credentials: auth file missing at {path} and "
|
||||
f"macOS Keychain lookup for '{_KEYCHAIN_SERVICE}' failed. "
|
||||
"Run `claude login` on the host or disable "
|
||||
"agent_provider.forward_host_credentials."
|
||||
)
|
||||
|
||||
oauth = raw.get("claudeAiOauth")
|
||||
if not isinstance(oauth, dict):
|
||||
die(
|
||||
"claude host credentials: claudeAiOauth is missing from credentials. "
|
||||
"Run `claude login` on the host or disable "
|
||||
"agent_provider.forward_host_credentials."
|
||||
)
|
||||
|
||||
access_token = oauth.get("accessToken")
|
||||
if not isinstance(access_token, str) or not access_token:
|
||||
die(
|
||||
"claude host credentials: claudeAiOauth.accessToken is missing or empty. "
|
||||
"Run `claude login` on the host and restart the bottle."
|
||||
)
|
||||
|
||||
# expiresAt is in milliseconds
|
||||
expires_at = oauth.get("expiresAt")
|
||||
if isinstance(expires_at, (int, float)):
|
||||
check_now = now or datetime.now(timezone.utc)
|
||||
exp_dt = datetime.fromtimestamp(float(expires_at) / 1000.0, timezone.utc)
|
||||
if exp_dt <= check_now:
|
||||
die(
|
||||
"claude host credentials: host Claude access token is expired. "
|
||||
"Run `claude login` on the host and restart the bottle."
|
||||
)
|
||||
|
||||
return access_token
|
||||
|
||||
|
||||
__all__ = [
|
||||
"claude_auth_path",
|
||||
"claude_host_access_token",
|
||||
]
|
||||
@@ -3,10 +3,17 @@
|
||||
# Mirrors the default Claude image shape: Node LTS, git/network tooling,
|
||||
# non-root node user, and the provider CLI installed for that user.
|
||||
|
||||
FROM node:22-slim
|
||||
FROM node:22-trixie-slim
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends git ca-certificates curl procps ripgrep \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
git \
|
||||
ca-certificates \
|
||||
curl \
|
||||
openssh-client \
|
||||
podman \
|
||||
procps \
|
||||
ripgrep \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# App-specific deps. Python isn't required by codex itself
|
||||
@@ -17,6 +24,8 @@ RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends python3 python3-pip python3-venv \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN install -d -o node -g node -m 755 /home/node/.config /home/node/.config/git
|
||||
|
||||
USER node
|
||||
WORKDIR /home/node
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# Node LTS, git/network tooling, and the Pi coding-agent CLI installed globally.
|
||||
|
||||
FROM node:22-slim
|
||||
FROM node:22-trixie-slim
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
@@ -10,6 +10,8 @@ RUN apt-get update \
|
||||
ca-certificates \
|
||||
curl \
|
||||
fd-find \
|
||||
openssh-client \
|
||||
podman \
|
||||
ripgrep \
|
||||
&& ln -s /usr/bin/fdfind /usr/local/bin/fd \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
@@ -21,7 +23,8 @@ RUN apt-get update \
|
||||
RUN npm install -g --ignore-scripts --no-fund --no-audit @earendil-works/pi-coding-agent \
|
||||
&& npm cache clean --force
|
||||
|
||||
RUN mkdir -p /home/node/.pi/agent \
|
||||
RUN install -d -o node -g node -m 755 /home/node/.config /home/node/.config/git \
|
||||
&& mkdir -p /home/node/.pi/agent \
|
||||
/home/node/.pi/context-mode/sessions \
|
||||
/tmp/pi-subagents-uid-1000 \
|
||||
&& chown -R node:node /home/node/.pi /tmp \
|
||||
|
||||
@@ -30,6 +30,7 @@ if TYPE_CHECKING:
|
||||
from .manifest import ManifestBottle
|
||||
|
||||
CODEX_HOST_CREDENTIAL_TOKEN_REF = "BOT_BOTTLE_CODEX_HOST_ACCESS_TOKEN"
|
||||
CLAUDE_HOST_CREDENTIAL_TOKEN_REF = "BOT_BOTTLE_CLAUDE_HOST_ACCESS_TOKEN"
|
||||
|
||||
EGRESS_HOSTNAME = "egress"
|
||||
|
||||
@@ -145,6 +146,7 @@ def egress_manifest_routes(
|
||||
outbound_detectors=r.OutboundDetectors,
|
||||
inbound_detectors=r.InboundDetectors,
|
||||
outbound_on_match=r.OutboundOnMatch,
|
||||
preserve_auth=r.PreserveAuth,
|
||||
))
|
||||
return tuple(out)
|
||||
|
||||
@@ -400,6 +402,7 @@ class Egress(ABC):
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CLAUDE_HOST_CREDENTIAL_TOKEN_REF",
|
||||
"CODEX_HOST_CREDENTIAL_TOKEN_REF",
|
||||
"EGRESS_HOSTNAME",
|
||||
"EGRESS_ROUTES_FILENAME",
|
||||
|
||||
@@ -367,7 +367,10 @@ class EgressAddon:
|
||||
|
||||
# Strip agent-set Authorization after DLP scan so smuggled tokens
|
||||
# are caught above; the route may inject gateway-owned auth below.
|
||||
flow.request.headers.pop("authorization", None)
|
||||
# Routes with preserve_auth=True pass the header through as-is so the
|
||||
# agent's own credentials (e.g. registry bearer tokens) reach the upstream.
|
||||
if route is None or not route.preserve_auth:
|
||||
flow.request.headers.pop("authorization", None)
|
||||
|
||||
# Build headers mapping for match evaluation
|
||||
req_headers = {k.lower(): v for k, v in flow.request.headers.items()}
|
||||
|
||||
@@ -78,6 +78,7 @@ class Route:
|
||||
inbound_detectors: tuple[str, ...] | None = None
|
||||
# "" means unset → DEFAULT_OUTBOUND_ON_MATCH. See OUTBOUND_ON_MATCH_VALUES.
|
||||
outbound_on_match: str = ""
|
||||
preserve_auth: bool = False
|
||||
|
||||
|
||||
LOG_OFF = 0 # no logging
|
||||
@@ -308,11 +309,18 @@ def _parse_one(idx: int, raw: object) -> Route:
|
||||
idx, host, raw_dict,
|
||||
)
|
||||
|
||||
preserve_auth_raw = raw_dict.get("preserve_auth", False)
|
||||
if preserve_auth_raw is not True and preserve_auth_raw is not False:
|
||||
raise ValueError(
|
||||
f"{label} ({host}): 'preserve_auth' must be a boolean"
|
||||
)
|
||||
preserve_auth: bool = preserve_auth_raw
|
||||
|
||||
for k in raw_dict:
|
||||
if k not in ("host", "matches", "auth_scheme", "token_env", "dlp", "git"):
|
||||
if k not in ("host", "matches", "auth_scheme", "token_env", "dlp", "git", "preserve_auth"):
|
||||
raise ValueError(
|
||||
f"{label} ({host}): unknown key {k!r}; accepted keys "
|
||||
f"are 'host', 'matches', 'auth_scheme', 'token_env', 'dlp', 'git'"
|
||||
f"are 'host', 'matches', 'auth_scheme', 'token_env', 'dlp', 'git', 'preserve_auth'"
|
||||
)
|
||||
|
||||
return Route(
|
||||
@@ -324,6 +332,7 @@ def _parse_one(idx: int, raw: object) -> Route:
|
||||
outbound_detectors=outbound_detectors,
|
||||
inbound_detectors=inbound_detectors,
|
||||
outbound_on_match=outbound_on_match,
|
||||
preserve_auth=preserve_auth,
|
||||
)
|
||||
|
||||
|
||||
@@ -376,6 +385,8 @@ def route_to_yaml_dict(r: Route) -> dict[str, object]:
|
||||
dlp["outbound_on_match"] = r.outbound_on_match
|
||||
if dlp:
|
||||
d["dlp"] = dlp
|
||||
if r.preserve_auth:
|
||||
d["preserve_auth"] = True
|
||||
return d
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Shared helpers for cached-image quickstart stale checks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from .config_store import ConfigStore
|
||||
except ImportError:
|
||||
from config_store import ConfigStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
|
||||
|
||||
|
||||
class StaleImageError(Exception):
|
||||
"""Raised when a cached image or artifact exceeds the configured staleness
|
||||
threshold. Callers can catch this to prompt interactively; headless paths
|
||||
let it propagate as a fatal error."""
|
||||
|
||||
|
||||
def check_stale(label: str, created_at: datetime) -> None:
|
||||
"""Raise StaleImageError if `created_at` is older than the configured
|
||||
stale-warning threshold. Negative threshold disables the check."""
|
||||
threshold_days = ConfigStore().cached_image_stale_warning_days()
|
||||
if threshold_days < 0:
|
||||
return
|
||||
now = datetime.now(timezone.utc)
|
||||
created = created_at.astimezone(timezone.utc)
|
||||
age = now - created
|
||||
if age.total_seconds() <= threshold_days * 86400:
|
||||
return
|
||||
raise StaleImageError(
|
||||
f"cached {label} is {age.days} day(s) old; "
|
||||
"quickstart does not verify it matches the current Dockerfile/context"
|
||||
)
|
||||
|
||||
|
||||
def check_stale_path(label: str, path: Path) -> None:
|
||||
"""Raise StaleImageError if `path`'s mtime exceeds the staleness threshold."""
|
||||
check_stale(label, datetime.fromtimestamp(path.stat().st_mtime, tz=timezone.utc))
|
||||
|
||||
|
||||
__all__ = ["StaleImageError", "check_stale", "check_stale_path"]
|
||||
@@ -25,8 +25,9 @@ class ManifestAgentProvider:
|
||||
header, and sets a placeholder CLAUDE_CODE_OAUTH_TOKEN in the agent
|
||||
so the Claude Code CLI starts.
|
||||
|
||||
`forward_host_credentials` forwards the host Codex auth token into
|
||||
the egress daemon (Codex only).
|
||||
`forward_host_credentials` forwards the host provider auth token into
|
||||
the egress sidecar (Codex and Claude). For Codex this reads
|
||||
`~/.codex/auth.json`; for Claude it reads `~/.claude/.credentials.json`.
|
||||
"""
|
||||
|
||||
template: str = "claude"
|
||||
@@ -92,10 +93,15 @@ class ManifestAgentProvider:
|
||||
f"is only supported for built-in templates "
|
||||
f"({', '.join(sorted(PROVIDER_TEMPLATES))})"
|
||||
)
|
||||
if forward_host_credentials and template != "codex":
|
||||
if forward_host_credentials and template not in {"codex", "claude"}:
|
||||
raise ManifestError(
|
||||
f"bottle '{bottle_name}' agent_provider.forward_host_credentials "
|
||||
"is currently only supported for template 'codex'"
|
||||
"is only supported for templates 'codex' and 'claude'"
|
||||
)
|
||||
if forward_host_credentials and auth_token:
|
||||
raise ManifestError(
|
||||
f"bottle '{bottle_name}' agent_provider.forward_host_credentials "
|
||||
"and auth_token both set; use one or the other"
|
||||
)
|
||||
settings = _parse_provider_settings(bottle_name, template, d.get("settings"))
|
||||
return cls(
|
||||
|
||||
@@ -71,6 +71,7 @@ class ManifestEgressRoute:
|
||||
OutboundDetectors: tuple[str, ...] | None = None
|
||||
InboundDetectors: tuple[str, ...] | None = None
|
||||
OutboundOnMatch: str = ""
|
||||
PreserveAuth: bool = False
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, bottle_name: str, idx: int, raw: object) -> "ManifestEgressRoute":
|
||||
@@ -190,11 +191,22 @@ class ManifestEgressRoute:
|
||||
f"only 'fetch' is accepted"
|
||||
)
|
||||
|
||||
# --- preserve_auth ---
|
||||
preserve_auth = False
|
||||
if "preserve_auth" in d:
|
||||
raw_preserve_auth = d.get("preserve_auth")
|
||||
if not isinstance(raw_preserve_auth, bool):
|
||||
raise ManifestError(
|
||||
f"{label} preserve_auth must be a boolean "
|
||||
f"(was {type(raw_preserve_auth).__name__})"
|
||||
)
|
||||
preserve_auth = raw_preserve_auth
|
||||
|
||||
for k in d:
|
||||
if k not in ("host", "matches", "auth", "role", "dlp", "git"):
|
||||
if k not in ("host", "matches", "auth", "role", "dlp", "git", "preserve_auth"):
|
||||
raise ManifestError(
|
||||
f"{label} has unknown key {k!r}; accepted keys are "
|
||||
f"'host', 'matches', 'auth', 'role', 'dlp', 'git'"
|
||||
f"'host', 'matches', 'auth', 'role', 'dlp', 'git', 'preserve_auth'"
|
||||
)
|
||||
|
||||
return cls(
|
||||
@@ -207,6 +219,7 @@ class ManifestEgressRoute:
|
||||
OutboundDetectors=outbound_detectors,
|
||||
InboundDetectors=inbound_detectors,
|
||||
OutboundOnMatch=outbound_on_match,
|
||||
PreserveAuth=preserve_auth,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -41,10 +41,13 @@ class OrchestratorClientError(RuntimeError):
|
||||
@dataclass(frozen=True)
|
||||
class RegisteredBottle:
|
||||
"""What `POST /bottles` returns: the minted bottle id and the per-bottle
|
||||
identity token the agent presents for app-layer attribution."""
|
||||
identity token the agent presents for app-layer attribution. `env_var_secret`
|
||||
is set by the caller (not from the server response) and carries the
|
||||
encryption key so it can be injected into the agent container's env."""
|
||||
|
||||
bottle_id: str
|
||||
identity_token: str
|
||||
env_var_secret: str = ""
|
||||
|
||||
|
||||
class OrchestratorClient:
|
||||
@@ -120,17 +123,21 @@ class OrchestratorClient:
|
||||
metadata: str = "",
|
||||
policy: str = "",
|
||||
tokens: dict[str, str] | None = None,
|
||||
env_var_secret: str = "",
|
||||
) -> RegisteredBottle:
|
||||
"""Register a bottle and broker its launch (`POST /bottles`). `tokens`
|
||||
are the per-bottle egress auth values (env_name -> value) the
|
||||
orchestrator holds in memory for the gateway to inject. Returns the
|
||||
minted id + identity token."""
|
||||
orchestrator holds in memory for the gateway to inject. When
|
||||
*env_var_secret* is provided, the orchestrator also encrypts the token
|
||||
values and stores them in ``bottled_agent_secrets`` for restart
|
||||
recovery. Returns the minted id + identity token."""
|
||||
payload = self._ok("POST", "/bottles", {
|
||||
"source_ip": source_ip,
|
||||
"image_ref": image_ref,
|
||||
"metadata": metadata,
|
||||
"policy": policy,
|
||||
"tokens": tokens or {},
|
||||
"env_var_secret": env_var_secret,
|
||||
})
|
||||
bottle_id = payload.get("bottle_id")
|
||||
token = payload.get("identity_token")
|
||||
@@ -138,6 +145,24 @@ class OrchestratorClient:
|
||||
raise OrchestratorClientError("register: response missing bottle_id/identity_token")
|
||||
return RegisteredBottle(bottle_id=bottle_id, identity_token=token)
|
||||
|
||||
def reprovision_gateway(self, bottle_id: str, env_var_secret: str) -> bool:
|
||||
"""Re-inject a bottle's egress tokens from its ENV_VAR_SECRET
|
||||
(`POST /bottles/<id>/reprovision_gateway`). Returns True when the
|
||||
orchestrator successfully decrypted and restored the tokens, False
|
||||
when it had no stored secrets for this bottle (404)."""
|
||||
status, _ = self._request(
|
||||
"POST",
|
||||
f"/bottles/{bottle_id}/reprovision_gateway",
|
||||
{"env_var_secret": env_var_secret},
|
||||
)
|
||||
if status == 404:
|
||||
return False
|
||||
if not 200 <= status < 300:
|
||||
raise OrchestratorClientError(
|
||||
f"reprovision_gateway {bottle_id}: HTTP {status}"
|
||||
)
|
||||
return True
|
||||
|
||||
def teardown_bottle(self, bottle_id: str) -> bool:
|
||||
"""Tear a bottle down (`DELETE /bottles/<id>`). False if the
|
||||
orchestrator didn't know it (404) — idempotent for cleanup paths."""
|
||||
|
||||
@@ -9,9 +9,13 @@ vsock / unix-socket portability caveats):
|
||||
GET /bottles -> 200 {"bottles": [ <redacted record>, ...]}
|
||||
POST /bottles -> 201 {"bottle_id","identity_token"} (launch)
|
||||
body: {"source_ip", ["image_ref"],
|
||||
["metadata"], ["policy"]}
|
||||
["metadata"], ["policy"],
|
||||
["tokens"], ["env_var_secret"]}
|
||||
PUT /bottles/<bottle_id>/policy -> 200 {"updated": true} | 404 (live reload)
|
||||
body: {"policy"}
|
||||
POST /bottles/<bottle_id>/reprovision_gateway
|
||||
-> 200 {"reprovisioned": true} | 404
|
||||
body: {"env_var_secret"}
|
||||
DELETE /bottles/<bottle_id> -> 200 {"torn_down": true} | 404 (teardown)
|
||||
POST /reconcile -> 200 {"reaped": [bottle_id, ...]}
|
||||
body: {"live_source_ips": [...],
|
||||
@@ -116,12 +120,14 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches
|
||||
tokens = {
|
||||
k: v for k, v in raw_tokens.items() if isinstance(k, str) and isinstance(v, str)
|
||||
} if isinstance(raw_tokens, dict) else {}
|
||||
env_var_secret = data.get("env_var_secret", "")
|
||||
rec = orch.launch_bottle(
|
||||
source_ip,
|
||||
image_ref=image_ref if isinstance(image_ref, str) else "",
|
||||
metadata=metadata if isinstance(metadata, str) else "",
|
||||
policy=policy if isinstance(policy, str) else "",
|
||||
tokens=tokens,
|
||||
env_var_secret=env_var_secret if isinstance(env_var_secret, str) else "",
|
||||
)
|
||||
return 201, {"bottle_id": rec.bottle_id, "identity_token": rec.identity_token}
|
||||
|
||||
@@ -138,6 +144,23 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches
|
||||
return 200, {"updated": True}
|
||||
return 404, {"error": "no such bottle"}
|
||||
|
||||
if (
|
||||
method == "POST"
|
||||
and route.startswith("/bottles/")
|
||||
and route.endswith("/reprovision_gateway")
|
||||
):
|
||||
bottle_id = route[len("/bottles/") : -len("/reprovision_gateway")]
|
||||
try:
|
||||
data = _parse_json_object(body)
|
||||
except ValueError as e:
|
||||
return 400, {"error": f"invalid JSON: {e}"}
|
||||
env_var_secret = data.get("env_var_secret")
|
||||
if not isinstance(env_var_secret, str) or not env_var_secret:
|
||||
return 400, {"error": "env_var_secret (string) is required"}
|
||||
if orch.reprovision_from_secret(bottle_id, env_var_secret):
|
||||
return 200, {"reprovisioned": True}
|
||||
return 404, {"error": "no stored secrets for this bottle"}
|
||||
|
||||
if method == "DELETE" and route.startswith("/bottles/"):
|
||||
bottle_id = route[len("/bottles/"):]
|
||||
if orch.teardown_bottle(bottle_id):
|
||||
|
||||
@@ -27,6 +27,7 @@ from ..paths import (
|
||||
CONTROL_PLANE_TOKEN_ENV,
|
||||
host_control_plane_token,
|
||||
host_db_path,
|
||||
host_gateway_ca_dir,
|
||||
)
|
||||
from ..supervise import DB_PATH_IN_CONTAINER
|
||||
|
||||
@@ -48,14 +49,23 @@ GATEWAY_LABEL = "bot-bottle-orch-gateway=1"
|
||||
# the source IP the gateway attributes by is the address on this network.
|
||||
GATEWAY_NETWORK = "bot-bottle-gateway"
|
||||
|
||||
# mitmproxy's CA dir in the bundle. A persistent named volume here keeps the
|
||||
# gateway's self-generated CA STABLE across container recreation — every agent
|
||||
# installs this one CA to trust the shared gateway's TLS interception, so it
|
||||
# must not rotate when the gateway restarts.
|
||||
# mitmproxy's CA dir in the bundle. The host's gateway-CA dir (see
|
||||
# `host_gateway_ca_dir`) is bind-mounted here so the gateway's self-generated
|
||||
# CA stays STABLE across container recreation — every agent installs this one
|
||||
# CA to trust the shared gateway's TLS interception, so it must not rotate when
|
||||
# the gateway restarts. A host bind-mount rather than a named volume: a named
|
||||
# volume is silently wiped by `docker volume prune`, minting a fresh CA that
|
||||
# breaks every running bottle (issue #450).
|
||||
MITMPROXY_HOME = "/home/mitmproxy/.mitmproxy"
|
||||
GATEWAY_CA_VOLUME = "bot-bottle-gateway-mitmproxy"
|
||||
GATEWAY_CA_CERT = f"{MITMPROXY_HOME}/mitmproxy-ca-cert.pem"
|
||||
|
||||
# The CA material mitmproxy writes into its confdir. mitmproxy reuses these on
|
||||
# startup when present and generates them only on first run, so persisting them
|
||||
# is what makes the CA stable; deleting them (see `rotate_gateway_ca`) forces a
|
||||
# fresh CA on the next start. `mitmproxy-ca.pem` (cert + private key) is the
|
||||
# signing identity; the rest are derived encodings agents/clients consume.
|
||||
GATEWAY_CA_GLOB = "mitmproxy-ca*"
|
||||
|
||||
# The gateway data-plane image + its Dockerfile. Kept as a local constant
|
||||
# rather than imported from the backend layer, which would drag
|
||||
# the whole backend layer into the lean orchestrator (see #359); unify when
|
||||
@@ -73,6 +83,26 @@ def _host_db_dir() -> str:
|
||||
return str(db_dir)
|
||||
|
||||
|
||||
def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]:
|
||||
"""Delete the persisted mitmproxy CA so the next gateway start mints a
|
||||
fresh one — the explicit, deliberate CA-rollover path (issue #450).
|
||||
|
||||
Persistence keeps the CA stable across restarts precisely because mitmproxy
|
||||
reuses the on-disk CA; rotation is therefore just removing that material.
|
||||
Returns the files removed (empty when there was no CA yet); idempotent.
|
||||
|
||||
This only clears the on-disk CA. It does NOT stop the running gateway (whose
|
||||
mitmproxy still holds the old CA in memory) or re-provision agents — the
|
||||
caller recreates the gateway to mint the new CA and re-attaches bottles.
|
||||
`rotate-ca` on the orchestrator CLI wires those steps together."""
|
||||
ca_dir = ca_dir if ca_dir is not None else host_gateway_ca_dir()
|
||||
removed: list[Path] = []
|
||||
for path in sorted(ca_dir.glob(GATEWAY_CA_GLOB)):
|
||||
path.unlink()
|
||||
removed.append(path)
|
||||
return removed
|
||||
|
||||
|
||||
class GatewayError(Exception):
|
||||
"""The shared gateway failed to build/start/stop (non-zero `docker` exit)."""
|
||||
|
||||
@@ -221,9 +251,10 @@ class DockerGateway(Gateway):
|
||||
"--name", self.name,
|
||||
"--label", GATEWAY_LABEL,
|
||||
"--network", self.network,
|
||||
# Persist the self-generated CA so it survives restarts (agents
|
||||
# trust it) — see GATEWAY_CA_VOLUME.
|
||||
"--volume", f"{GATEWAY_CA_VOLUME}:{MITMPROXY_HOME}",
|
||||
# Persist the self-generated CA on the host so it survives both
|
||||
# container recreation AND docker volume pruning (agents trust it)
|
||||
# — see host_gateway_ca_dir / issue #450.
|
||||
"--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}",
|
||||
# Share the one host DB: the supervise daemon queues proposals
|
||||
# into the same file the orchestrator (and the operator, over
|
||||
# HTTP) reads — no second, disconnected DB in the container.
|
||||
@@ -272,7 +303,7 @@ class DockerGateway(Gateway):
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Gateway", "DockerGateway", "GatewayError",
|
||||
"Gateway", "DockerGateway", "GatewayError", "rotate_gateway_ca",
|
||||
"GATEWAY_NAME", "GATEWAY_LABEL", "GATEWAY_IMAGE", "GATEWAY_NETWORK",
|
||||
"GATEWAY_CA_VOLUME", "GATEWAY_CA_CERT",
|
||||
"GATEWAY_CA_CERT", "GATEWAY_CA_GLOB",
|
||||
]
|
||||
|
||||
@@ -23,10 +23,14 @@ from pathlib import Path
|
||||
|
||||
from .. import log
|
||||
from ..docker_cmd import run_docker
|
||||
from ..paths import CONTROL_PLANE_TOKEN_ENV, bot_bottle_root, host_control_plane_token
|
||||
from ..paths import (
|
||||
CONTROL_PLANE_TOKEN_ENV,
|
||||
bot_bottle_root,
|
||||
host_control_plane_token,
|
||||
host_gateway_ca_dir,
|
||||
)
|
||||
from ..supervise import DB_PATH_IN_CONTAINER
|
||||
from .gateway import (
|
||||
GATEWAY_CA_VOLUME,
|
||||
GATEWAY_DOCKERFILE,
|
||||
GATEWAY_IMAGE,
|
||||
GATEWAY_NETWORK,
|
||||
@@ -194,8 +198,10 @@ class OrchestratorService:
|
||||
# gateway_init always starts the orchestrator on DEFAULT_PORT (8099)
|
||||
# inside the container; self.port is the host-side published port.
|
||||
"--publish", f"127.0.0.1:{self.port}:{DEFAULT_PORT}",
|
||||
# Persist the mitmproxy CA so it survives container recreation.
|
||||
"--volume", f"{GATEWAY_CA_VOLUME}:{MITMPROXY_HOME}",
|
||||
# Persist the mitmproxy CA on the host so it survives container
|
||||
# recreation AND docker volume pruning (issue #450): every agent
|
||||
# trusts this one CA, so a fresh one would break all running bottles.
|
||||
"--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}",
|
||||
# Shared supervise DB (same file the operator reads over HTTP).
|
||||
"--volume", f"{_host_db_dir()}:{_SUPERVISE_DB_DIR_IN_CONTAINER}",
|
||||
"--env", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}",
|
||||
|
||||
@@ -113,6 +113,22 @@ _MIGRATIONS = TableMigrations(
|
||||
# egress allowlist / routes / git config selected by source IP. The
|
||||
# multi-tenant gateway resolves it per request via `attribute`.
|
||||
"ALTER TABLE orchestrator_bottles ADD COLUMN policy TEXT NOT NULL DEFAULT ''",
|
||||
# v4 — per-bottle encrypted egress secrets (PRD prd-new-secret-provider).
|
||||
# One row per env-var: key (env-var name) is plaintext for auditing;
|
||||
# value is the encrypted token string. The encryption key (ENV_VAR_SECRET)
|
||||
# lives only in the agent's environment — a row alone cannot recover the
|
||||
# credential.
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS bottled_agent_secrets (
|
||||
bottled_agent_id TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
type TEXT NOT NULL DEFAULT 'injected_env_var'
|
||||
)
|
||||
""",
|
||||
# v5 — index for fast per-bottle lookups and bulk DELETE on teardown.
|
||||
"CREATE INDEX IF NOT EXISTS idx_bottled_agent_secrets_id "
|
||||
"ON bottled_agent_secrets (bottled_agent_id, type)",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -326,6 +342,57 @@ class RegistryStore(DbStore):
|
||||
return None
|
||||
return rec
|
||||
|
||||
# --- encrypted egress secret store ------------------------------------
|
||||
|
||||
def store_agent_secrets(
|
||||
self,
|
||||
bottle_id: str,
|
||||
encrypted_values: dict[str, str],
|
||||
secret_type: str = "injected_env_var",
|
||||
) -> None:
|
||||
"""Replace all stored secrets for *bottle_id* with *encrypted_values*
|
||||
(env-var name → encrypted ciphertext). Deletes then re-inserts so a
|
||||
re-registration is always consistent with the current token set."""
|
||||
with self._connection() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM bottled_agent_secrets "
|
||||
"WHERE bottled_agent_id = ? AND type = ?",
|
||||
(bottle_id, secret_type),
|
||||
)
|
||||
conn.executemany(
|
||||
"INSERT INTO bottled_agent_secrets "
|
||||
"(bottled_agent_id, key, value, type) VALUES (?, ?, ?, ?)",
|
||||
[(bottle_id, k, v, secret_type) for k, v in encrypted_values.items()],
|
||||
)
|
||||
self._chmod()
|
||||
|
||||
def get_agent_secrets(
|
||||
self,
|
||||
bottle_id: str,
|
||||
secret_type: str = "injected_env_var",
|
||||
) -> dict[str, str]:
|
||||
"""Return {env_var_name: encrypted_value} for *bottle_id*, or {} if none."""
|
||||
with self._connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT key, value FROM bottled_agent_secrets "
|
||||
"WHERE bottled_agent_id = ? AND type = ?",
|
||||
(bottle_id, secret_type),
|
||||
).fetchall()
|
||||
return {row[0]: row[1] for row in rows}
|
||||
|
||||
def delete_agent_secrets(
|
||||
self,
|
||||
bottle_id: str,
|
||||
secret_type: str = "injected_env_var",
|
||||
) -> None:
|
||||
"""Remove all stored secrets for *bottle_id* (e.g. on teardown)."""
|
||||
with self._connection() as conn:
|
||||
conn.execute(
|
||||
"DELETE FROM bottled_agent_secrets "
|
||||
"WHERE bottled_agent_id = ? AND type = ?",
|
||||
(bottle_id, secret_type),
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BottleRecord",
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Rotate the shared gateway's mitmproxy CA (issue #450).
|
||||
|
||||
python -m bot_bottle.orchestrator.rotate_ca
|
||||
|
||||
A deliberate CA rollover has two halves: drop the *persisted* CA so a fresh one
|
||||
is minted, and drop the *running* gateway so its mitmproxy (which holds the old
|
||||
CA in memory) is replaced. This one-shot command does both:
|
||||
|
||||
1. Delete the persisted CA under the host gateway-CA dir — the next gateway
|
||||
start generates a new one (mitmproxy reuses an existing CA, generates only
|
||||
when absent).
|
||||
2. Force-remove the infra / standalone-gateway containers so the stale
|
||||
in-memory CA is gone; the next bottle launch's idempotent `ensure_running`
|
||||
brings the gateway back up and mints the fresh CA.
|
||||
|
||||
It does NOT re-provision the new CA into already-running bottles — those must be
|
||||
re-attached so they install the new trust anchor. Rotation is thus an explicit,
|
||||
operator-driven action with a brief egress interruption, not an automatic one.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from ..docker_cmd import run_docker
|
||||
from ..paths import host_gateway_ca_dir
|
||||
from .gateway import GATEWAY_NAME, rotate_gateway_ca
|
||||
from .lifecycle import INFRA_NAME
|
||||
|
||||
# The containers whose mitmproxy would still be serving the old CA from memory:
|
||||
# the consolidated infra container and the standalone per-host gateway.
|
||||
_GATEWAY_CONTAINERS = (INFRA_NAME, GATEWAY_NAME)
|
||||
|
||||
|
||||
def _out(msg: str) -> None:
|
||||
sys.stdout.write(f"rotate-ca: {msg}\n")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
del argv # no flags — a single deliberate action
|
||||
ca_dir: Path = host_gateway_ca_dir()
|
||||
removed = rotate_gateway_ca(ca_dir)
|
||||
if removed:
|
||||
_out(f"removed {len(removed)} CA file(s) from {ca_dir}")
|
||||
else:
|
||||
_out(f"no persisted CA under {ca_dir}; a fresh one is minted on next start")
|
||||
|
||||
# Drop any running gateway so its in-memory (now-stale) CA is replaced on
|
||||
# the next launch. `rm --force` on an absent name is a tolerated no-op.
|
||||
for name in _GATEWAY_CONTAINERS:
|
||||
proc = run_docker(["docker", "rm", "--force", name])
|
||||
if proc.returncode == 0 and proc.stdout.strip():
|
||||
_out(f"removed running container {name}")
|
||||
|
||||
_out("done — the next bottle launch remints the CA; re-attach bottles to "
|
||||
"install the new trust anchor")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Symmetric encryption for per-bottle egress secrets (PRD prd-new-secret-provider).
|
||||
|
||||
Each agent receives a random ENV_VAR_SECRET at startup — passed as an env var,
|
||||
never logged or persisted. The host uses this key to encrypt each egress auth
|
||||
token value before writing it to the bottled_agent_secrets table; the DB rows
|
||||
(ciphertext, plaintext env-var name) without the key are insufficient to
|
||||
recover the credentials.
|
||||
|
||||
On orchestrator restart the in-memory token map is lost. The host-side
|
||||
reattachment path reads ENV_VAR_SECRET from the running agent container via
|
||||
``docker exec … printenv ENV_VAR_SECRET`` and posts it to
|
||||
``POST /bottles/<id>/reprovision_gateway``; the orchestrator decrypts the
|
||||
stored rows and re-populates ``_tokens``.
|
||||
|
||||
Encryption scheme: HMAC-SHA256 used as a PRF in CTR mode (stdlib-only,
|
||||
no external deps). Each value is encrypted independently. The output blob is
|
||||
``nonce (16 bytes) || ciphertext`` encoded as URL-safe base64 (no padding).
|
||||
|
||||
keystream_block_i = HMAC-SHA256(key, nonce || i.to_bytes(4, "big"))
|
||||
ciphertext_i = plaintext_i XOR keystream_block_i[:len(plaintext_i)]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
|
||||
_KEY_BYTES = 32 # 256-bit key from ENV_VAR_SECRET
|
||||
_NONCE_BYTES = 16 # 128-bit random nonce per encrypt call
|
||||
_BLOCK = 32 # HMAC-SHA256 output width == one keystream block
|
||||
|
||||
# Env-var name the agent container receives at startup.
|
||||
ENV_VAR_SECRET_NAME = "ENV_VAR_SECRET"
|
||||
|
||||
|
||||
def new_env_var_secret() -> str:
|
||||
"""Generate a fresh ENV_VAR_SECRET: 32 random bytes as URL-safe base64."""
|
||||
return base64.urlsafe_b64encode(secrets.token_bytes(_KEY_BYTES)).rstrip(b"=").decode()
|
||||
|
||||
|
||||
def _b64dec(s: str) -> bytes:
|
||||
return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4))
|
||||
|
||||
|
||||
def _keystream(key: bytes, nonce: bytes, block_index: int) -> bytes:
|
||||
return hmac.new(
|
||||
key, nonce + block_index.to_bytes(4, "big"), hashlib.sha256
|
||||
).digest()
|
||||
|
||||
|
||||
def encrypt_value(secret_b64: str, plaintext: str) -> str:
|
||||
"""Encrypt a single string value with *secret_b64* (the ENV_VAR_SECRET).
|
||||
|
||||
Returns a URL-safe base64 blob ``nonce || ciphertext`` suitable for
|
||||
the ``bottled_agent_secrets.value`` column."""
|
||||
key = _b64dec(secret_b64)
|
||||
pt = plaintext.encode()
|
||||
nonce = secrets.token_bytes(_NONCE_BYTES)
|
||||
ct = bytearray()
|
||||
for i in range(0, len(pt), _BLOCK):
|
||||
chunk = pt[i : i + _BLOCK]
|
||||
ks = _keystream(key, nonce, i)[: len(chunk)]
|
||||
ct.extend(p ^ k for p, k in zip(chunk, ks))
|
||||
return base64.urlsafe_b64encode(nonce + bytes(ct)).rstrip(b"=").decode()
|
||||
|
||||
|
||||
def decrypt_value(secret_b64: str, blob_b64: str) -> str:
|
||||
"""Decrypt a blob produced by :func:`encrypt_value`.
|
||||
|
||||
Returns the original plaintext string. Raises ``ValueError`` for malformed
|
||||
input or a key mismatch (wrong key produces garbage, not an error, unless
|
||||
the plaintext is non-UTF-8 — treat all such failures as wrong key)."""
|
||||
key = _b64dec(secret_b64)
|
||||
try:
|
||||
blob = _b64dec(blob_b64)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"invalid ciphertext blob: {exc}") from exc
|
||||
if len(blob) < _NONCE_BYTES:
|
||||
raise ValueError("ciphertext blob too short")
|
||||
nonce, ciphertext = blob[:_NONCE_BYTES], blob[_NONCE_BYTES:]
|
||||
pt = bytearray()
|
||||
for i in range(0, len(ciphertext), _BLOCK):
|
||||
chunk = ciphertext[i : i + _BLOCK]
|
||||
ks = _keystream(key, nonce, i)[: len(chunk)]
|
||||
pt.extend(c ^ k for c, k in zip(chunk, ks))
|
||||
try:
|
||||
return bytes(pt).decode()
|
||||
except UnicodeDecodeError as exc:
|
||||
raise ValueError(f"decryption produced non-UTF-8 output (wrong key?): {exc}") from exc
|
||||
|
||||
|
||||
__all__ = ["ENV_VAR_SECRET_NAME", "new_env_var_secret", "encrypt_value", "decrypt_value"]
|
||||
@@ -87,13 +87,22 @@ class Orchestrator:
|
||||
metadata: str = "",
|
||||
policy: str = "",
|
||||
tokens: dict[str, str] | None = None,
|
||||
env_var_secret: str = "",
|
||||
) -> BottleRecord:
|
||||
"""Register a bottle (with its gateway policy + in-memory egress auth
|
||||
tokens) and broker its launch. Rolls the registry entry back if the
|
||||
launch doesn't take, so a failure leaves no orphan."""
|
||||
launch doesn't take, so a failure leaves no orphan.
|
||||
|
||||
When *env_var_secret* is provided alongside *tokens*, the token values
|
||||
are also encrypted and written to ``bottled_agent_secrets`` so they can
|
||||
survive an orchestrator restart (see ``reprovision_from_secret``)."""
|
||||
rec = self.registry.register(source_ip, metadata=metadata, policy=policy)
|
||||
if tokens:
|
||||
self._tokens[rec.bottle_id] = dict(tokens)
|
||||
if env_var_secret:
|
||||
from .secret_store import encrypt_value
|
||||
encrypted = {k: encrypt_value(env_var_secret, v) for k, v in tokens.items()}
|
||||
self.registry.store_agent_secrets(rec.bottle_id, encrypted)
|
||||
req = LaunchRequest(
|
||||
op="launch",
|
||||
bottle_id=rec.bottle_id,
|
||||
@@ -284,6 +293,26 @@ class Orchestrator:
|
||||
))
|
||||
return True, ""
|
||||
|
||||
# --- secret reprovision -----------------------------------------------
|
||||
|
||||
def reprovision_from_secret(self, bottle_id: str, env_var_secret: str) -> bool:
|
||||
"""Re-inject a bottle's egress tokens from its ENV_VAR_SECRET.
|
||||
|
||||
Reads the encrypted rows from ``bottled_agent_secrets``, decrypts each
|
||||
value with *env_var_secret*, and restores ``_tokens[bottle_id]``.
|
||||
Returns True on success, False when no stored secrets exist for this
|
||||
bottle or decryption fails (wrong key / corrupt data)."""
|
||||
from .secret_store import decrypt_value
|
||||
encrypted = self.registry.get_agent_secrets(bottle_id)
|
||||
if not encrypted:
|
||||
return False
|
||||
try:
|
||||
self._tokens[bottle_id] = {k: decrypt_value(env_var_secret, v)
|
||||
for k, v in encrypted.items()}
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
# --- consolidated gateway ----------------------------------------------
|
||||
|
||||
def ensure_gateway(self) -> None:
|
||||
|
||||
@@ -33,6 +33,13 @@ HOST_DB_FILENAME = "bot-bottle.db"
|
||||
CONTROL_PLANE_TOKEN_FILENAME = "control-plane-token"
|
||||
CONTROL_PLANE_TOKEN_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN"
|
||||
|
||||
# The host directory holding the gateway's persistent mitmproxy CA. Bind-mounted
|
||||
# into the infra/gateway container at mitmproxy's confdir so the self-generated
|
||||
# CA survives container recreation — every agent installs this one CA to trust
|
||||
# the shared gateway's TLS interception, so it must not rotate on restart. See
|
||||
# host_gateway_ca_dir() for why this is a host bind-mount, not a named volume.
|
||||
GATEWAY_CA_DIRNAME = "gateway-ca"
|
||||
|
||||
|
||||
def bot_bottle_root() -> Path:
|
||||
"""The app data root — `$BOT_BOTTLE_ROOT` if set, else `~/.bot-bottle`."""
|
||||
@@ -59,6 +66,23 @@ def host_db_dir() -> Path:
|
||||
return db_dir
|
||||
|
||||
|
||||
def host_gateway_ca_dir() -> Path:
|
||||
"""The directory holding the gateway's persistent mitmproxy CA, created if
|
||||
missing. Backends bind-mount this into the infra/gateway container at
|
||||
mitmproxy's confdir so the CA persists across container recreation.
|
||||
|
||||
A host bind-mount under the app-data root — deliberately NOT a Docker
|
||||
named volume. A named volume survives `docker rm` but is silently wiped by
|
||||
`docker volume prune` / `docker system prune --volumes` during routine host
|
||||
maintenance; the gateway then mints a fresh CA that every already-running
|
||||
bottle distrusts, failing the TLS handshake even after it reconnects to the
|
||||
moved gateway (issue #450). A path under the root docker never prunes it,
|
||||
and it stays directly inspectable + rotatable from the host."""
|
||||
ca_dir = bot_bottle_root() / GATEWAY_CA_DIRNAME
|
||||
ca_dir.mkdir(parents=True, exist_ok=True)
|
||||
return ca_dir
|
||||
|
||||
|
||||
def host_control_plane_token() -> str:
|
||||
"""The per-host control-plane secret, minted (256-bit, url-safe) and
|
||||
persisted 0600 on first use, then reused.
|
||||
@@ -94,8 +118,10 @@ __all__ = [
|
||||
"HOST_DB_FILENAME",
|
||||
"CONTROL_PLANE_TOKEN_FILENAME",
|
||||
"CONTROL_PLANE_TOKEN_ENV",
|
||||
"GATEWAY_CA_DIRNAME",
|
||||
"bot_bottle_root",
|
||||
"host_db_path",
|
||||
"host_db_dir",
|
||||
"host_gateway_ca_dir",
|
||||
"host_control_plane_token",
|
||||
]
|
||||
|
||||
@@ -6,9 +6,11 @@ from pathlib import Path
|
||||
|
||||
try:
|
||||
from .audit_store import AuditStore
|
||||
from .config_store import ConfigStore
|
||||
from .queue_store import QueueStore
|
||||
except ImportError:
|
||||
from audit_store import AuditStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
|
||||
from config_store import ConfigStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
|
||||
from queue_store import QueueStore # type: ignore[import-not-found] # pylint: disable=import-error,no-name-in-module
|
||||
|
||||
_instance: StoreManager | None = None
|
||||
@@ -47,11 +49,13 @@ class StoreManager:
|
||||
return (
|
||||
QueueStore("", self.db_path).is_migrated()
|
||||
and AuditStore(self.db_path).is_migrated()
|
||||
and ConfigStore(self.db_path).is_migrated()
|
||||
)
|
||||
|
||||
def migrate(self) -> None:
|
||||
QueueStore("", self.db_path).migrate()
|
||||
AuditStore(self.db_path).migrate()
|
||||
ConfigStore(self.db_path).migrate()
|
||||
|
||||
|
||||
__all__ = ["StoreManager"]
|
||||
|
||||
@@ -312,6 +312,44 @@ reaches over the RPC rather than a shared mount into the VM. WAL on the
|
||||
shared DB is therefore a deliberate, tested future change — not enabled ad
|
||||
hoc. `sqlite3` itself is stdlib, so "the host needs SQLite" is a non-cost.
|
||||
|
||||
### Gateway CA: host-resident, like the DB
|
||||
|
||||
The shared gateway bumps TLS with a self-generated mitmproxy CA, and **every
|
||||
bottle installs that CA** into its trust store to accept the bumped leaves. So
|
||||
the CA is durable per-host state with the same rule as the DB: it must outlive
|
||||
any single gateway container, or a restart mints a fresh CA that every
|
||||
already-running bottle distrusts — the TLS handshake then fails even after the
|
||||
bottle re-resolves and reconnects to the moved gateway (issue #450, a
|
||||
re-attachment blocker distinct from #443/#445).
|
||||
|
||||
The CA lives on the **host filesystem** at `bot_bottle_root()/gateway-ca`
|
||||
(`host_gateway_ca_dir()`), bind-mounted into the container at mitmproxy's
|
||||
confdir. This is deliberately a host bind-mount, **not a container-runtime
|
||||
named volume**: a named volume survives ordinary container removal but can be
|
||||
silently wiped by Docker's or Apple Container's volume-prune commands during
|
||||
routine host maintenance, which is exactly how the ephemeral-CA symptom shows
|
||||
up in practice. A path under the app-data root is not managed or pruned by the
|
||||
container runtime, and stays directly inspectable and rotatable from the host.
|
||||
mitmproxy reuses an existing CA and generates one only on first run, so the
|
||||
bind-mount alone gives
|
||||
"adopt-existing, generate-on-first-run" for free.
|
||||
|
||||
The macOS backend uses the same host-resident CA directory and bind-mounts it
|
||||
into the consolidated Apple infra container. Its `bot-bottle-mac-db` named
|
||||
volume remains container-only because that prevents incoherent cross-kernel
|
||||
SQLite locking, but the CA is deliberately not stored there: Apple Container
|
||||
also has a `container volume prune` operation, and the named volume is
|
||||
temporarily unreferenced while the infra container is recreated. Keeping the
|
||||
CA on the host makes both ordinary recreation and volume pruning safe.
|
||||
|
||||
**Deliberate rollover** is the explicit inverse: `rotate_gateway_ca()` removes
|
||||
the persisted CA material so the next start remints it, and the
|
||||
`python -m bot_bottle.orchestrator.rotate_ca` one-shot wires that together with
|
||||
dropping the running gateway container (whose mitmproxy still holds the old CA
|
||||
in memory). Rotation does not auto-re-provision the new CA into running bottles
|
||||
— those re-attach to install the new anchor — so it is an operator action with
|
||||
a brief egress interruption, never an implicit one.
|
||||
|
||||
## Sequencing
|
||||
|
||||
Jump straight to the **virtualized** end state (not a host-daemon stepping
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
# PRD prd-new: CI artifact-based coverage and local Firecracker candidate flow
|
||||
|
||||
- **Status:** Active
|
||||
- **Author:** Claude
|
||||
- **Created:** 2026-07-21
|
||||
- **Issue:** #446
|
||||
|
||||
## Summary
|
||||
|
||||
Restructure the CI test pipeline to run each test suite exactly once, upload
|
||||
small `.coverage.*` artifacts, and combine them in a lightweight aggregation
|
||||
job. Move the infra build onto the KVM runner so the ~194 MB rootfs never
|
||||
crosses the network for PRs. On main-branch pushes, publish the byte-identical
|
||||
rootfs that was tested.
|
||||
|
||||
## Motivation
|
||||
|
||||
The prior pipeline had two redundant costs:
|
||||
|
||||
1. **Duplicate artifact transfers.** `build-infra` (ubuntu-latest) built and
|
||||
uploaded the ~194 MB rootfs; `integration-firecracker` downloaded it; the
|
||||
`coverage` job downloaded it a second time. Combined download overhead: ~83
|
||||
seconds per run, plus the ~70-second upload.
|
||||
|
||||
2. **Duplicate test execution.** `integration-firecracker` ran the Firecracker
|
||||
integration suite; `coverage` ran the entire unit + integration suite again
|
||||
on the same KVM runner to collect coverage data. Every line of Firecracker
|
||||
code was tested twice per CI run.
|
||||
|
||||
## Goals
|
||||
|
||||
- Each test suite (unit, integration-docker, integration-firecracker) executes
|
||||
exactly once per workflow run.
|
||||
- PRs incur no large artifact transfers — the rootfs stays on the KVM runner.
|
||||
- Main-branch pushes publish a byte-for-byte identical rootfs to the one that
|
||||
passed the integration tests.
|
||||
- Concurrent workflow runs cannot cross-publish candidates (naturally enforced
|
||||
by Gitea Actions' per-run artifact scoping).
|
||||
- Failed or cancelled runs block publication (enforced by the `needs:` chain on
|
||||
`publish-infra`).
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Changing test semantics or the coverage policy (ADR 0004).
|
||||
- Removing the KVM runner guard on `integration-firecracker` and `coverage`.
|
||||
- Changing how `publish_infra.py` builds or uploads the rootfs.
|
||||
|
||||
## Design
|
||||
|
||||
### Job graph
|
||||
|
||||
```
|
||||
unit ──────────────────────────────────┐
|
||||
integration-docker ────────────────────┤──► coverage ──► publish-infra (main only)
|
||||
integration-firecracker (KVM) ─────────┘
|
||||
```
|
||||
|
||||
### `unit`
|
||||
|
||||
Unchanged except: `coverage run` writes `--data-file=.coverage.unit`; the file
|
||||
is uploaded as the `coverage-unit` artifact.
|
||||
|
||||
### `integration-docker`
|
||||
|
||||
Adds a `coverage` install step. `coverage run` writes `--data-file=.coverage.docker`;
|
||||
the file is uploaded as `coverage-docker`.
|
||||
|
||||
### `integration-firecracker` (KVM runner)
|
||||
|
||||
Replaces the old `stage-firecracker-inputs` → `build-infra` → download chain:
|
||||
|
||||
1. Builds the infra candidate locally with
|
||||
`BOT_BOTTLE_FC_DROPBEAR=/var/cache/bot-bottle-fc/dropbear`.
|
||||
2. Boots the candidate and runs integration tests with coverage, writing
|
||||
`.coverage.firecracker`.
|
||||
3. Uploads the small `coverage-firecracker` artifact unconditionally.
|
||||
4. On main-branch pushes only, uploads the rootfs as `infra-candidate` and the
|
||||
dropbear as `firecracker-inputs` so `publish-infra` can verify and publish
|
||||
the byte-identical artifact.
|
||||
|
||||
### `coverage`
|
||||
|
||||
Moves from a KVM runner to `ubuntu-latest`. No tests are re-executed:
|
||||
|
||||
1. Downloads `coverage-unit`, `coverage-docker`, and `coverage-firecracker`.
|
||||
2. Runs `scripts/coverage.sh aggregate critical`, which calls
|
||||
`coverage combine` then `coverage report`.
|
||||
3. Runs the diff-coverage gate (`scripts/diff_coverage.py`).
|
||||
|
||||
Coverage files use `relative_files = True` (`.coveragerc`) so they combine
|
||||
cleanly across runners with different absolute workspace paths.
|
||||
|
||||
### `publish-infra`
|
||||
|
||||
Depends on all four predecessor jobs (unchanged gate). Downloads `infra-candidate`
|
||||
and `firecracker-inputs` that were uploaded by `integration-firecracker` on
|
||||
main — the same byte sequence that passed the integration tests.
|
||||
|
||||
### Eliminated jobs
|
||||
|
||||
- `stage-firecracker-inputs`: existed only to copy the dropbear to ubuntu-latest
|
||||
for `build-infra`. No longer needed.
|
||||
- `build-infra`: the infra candidate is now built on the KVM runner in
|
||||
`integration-firecracker`.
|
||||
|
||||
### Script changes
|
||||
|
||||
`scripts/coverage.sh` gains an `aggregate` mode (`coverage.sh aggregate [critical]`)
|
||||
that combines pre-existing `.coverage.*` files instead of re-running tests.
|
||||
The existing run mode (`coverage.sh [critical]`) is preserved for local dev.
|
||||
@@ -0,0 +1,146 @@
|
||||
# PRD prd-new: Claude forward_host_credentials
|
||||
|
||||
- **Status:** Draft
|
||||
- **Author:** claude
|
||||
- **Created:** 2026-07-01
|
||||
- **Issue:** #325
|
||||
|
||||
## Summary
|
||||
|
||||
Add `agent_provider.forward_host_credentials: true` support for the
|
||||
`claude` template, mirroring the existing Codex flow. When enabled,
|
||||
bot-bottle reads the host's Claude OAuth session key from
|
||||
`~/.claude/.credentials.json` at launch, forwards it only to the egress sidecar,
|
||||
and injects a placeholder `CLAUDE_CODE_OAUTH_TOKEN` into the agent so
|
||||
Claude Code starts without ever seeing the real credential.
|
||||
|
||||
## Problem
|
||||
|
||||
Running a Claude agent in a container today requires the operator to
|
||||
manually extract a long-lived OAuth token (`claude setup-token`), export
|
||||
it as `BOT_BOTTLE_CLAUDE_OAUTH_TOKEN`, and reference it explicitly in
|
||||
the manifest with `agent_provider.auth_token:
|
||||
"BOT_BOTTLE_CLAUDE_OAUTH_TOKEN"`. This is a two-step manual ceremony
|
||||
that is easy to skip or do incorrectly.
|
||||
|
||||
The host already stores a valid Claude session in `~/.claude/.credentials.json`
|
||||
after `claude login`. Codex already automates an
|
||||
equivalent extraction from `~/.codex/auth.json`. There is no reason
|
||||
Claude bottles cannot do the same.
|
||||
|
||||
## Goals / Success Criteria
|
||||
|
||||
- A Claude bottle with `forward_host_credentials: true` in the manifest
|
||||
uses the host's `~/.claude/.credentials.json` session key at launch with no
|
||||
additional operator steps.
|
||||
- The agent container receives only `CLAUDE_CODE_OAUTH_TOKEN=egress-placeholder`
|
||||
— never the real token.
|
||||
- The real session key lives only in the egress sidecar's environment.
|
||||
- Missing, malformed, or expired host Claude auth fails launch with a
|
||||
clear operator-facing message.
|
||||
- Existing `auth_token` behavior is unchanged.
|
||||
- `forward_host_credentials: true` is rejected in the manifest when both
|
||||
`auth_token` and `forward_host_credentials` are set, since they serve
|
||||
the same purpose.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Refreshing Claude OAuth tokens in the sidecar.
|
||||
- Writing a dummy `~/.claude.json` auth state to the agent (unlike the
|
||||
Codex flow, Claude Code reads its credential from `CLAUDE_CODE_OAUTH_TOKEN`
|
||||
in env, not from an auth file — no guest-side auth marker is needed).
|
||||
- Supporting `forward_host_credentials` for providers other than `codex`
|
||||
and `claude`.
|
||||
|
||||
## Design
|
||||
|
||||
### Manifest schema
|
||||
|
||||
```yaml
|
||||
agent_provider:
|
||||
template: claude
|
||||
forward_host_credentials: true
|
||||
```
|
||||
|
||||
Rejects in manifest validation when:
|
||||
- Template is not `codex` or `claude`.
|
||||
- Both `auth_token` and `forward_host_credentials` are set.
|
||||
|
||||
### Host auth extraction (`contrib/claude/claude_auth.py`)
|
||||
|
||||
Claude Code credential storage varies by platform:
|
||||
|
||||
- **Linux**: `~/.claude/.credentials.json`
|
||||
- **macOS**: macOS Keychain, service `"Claude Code-credentials"`
|
||||
(the file path is tried first; Keychain is the fallback when the file
|
||||
is absent)
|
||||
|
||||
`~/.claude.json` contains only UI state and profile metadata — no token.
|
||||
|
||||
The credentials JSON schema (same whether from file or Keychain):
|
||||
|
||||
```json
|
||||
{
|
||||
"claudeAiOauth": {
|
||||
"accessToken": "<access-token>",
|
||||
"refreshToken": "<refresh-token>",
|
||||
"expiresAt": 1748276587173,
|
||||
"scopes": ["user:inference", "user:profile"]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`expiresAt` is in **milliseconds** (not seconds).
|
||||
|
||||
At prepare/launch time, when `forward_host_credentials: true`:
|
||||
|
||||
1. Try `~/.claude/.credentials.json`; on macOS, if absent, run
|
||||
`security find-generic-password -s "Claude Code-credentials" -w`
|
||||
and parse its stdout as JSON.
|
||||
2. Require a `claudeAiOauth` dict.
|
||||
3. Require a non-empty `claudeAiOauth.accessToken` string.
|
||||
4. If `claudeAiOauth.expiresAt` is present, divide by 1000 and require
|
||||
the result to be in the future.
|
||||
5. Return only the access token to the launch path.
|
||||
|
||||
Errors name the missing or invalid condition and point the operator at
|
||||
`claude login`, without printing token values.
|
||||
|
||||
### Egress route
|
||||
|
||||
When `forward_host_credentials: true`:
|
||||
|
||||
- Provision the session key in `provisioned_env` under
|
||||
`BOT_BOTTLE_CLAUDE_HOST_ACCESS_TOKEN` (new constant in `egress.py`).
|
||||
- Set up the `api.anthropic.com` egress route with `auth_scheme: Bearer`
|
||||
and `token_ref: BOT_BOTTLE_CLAUDE_HOST_ACCESS_TOKEN`.
|
||||
- Set `CLAUDE_CODE_OAUTH_TOKEN=egress-placeholder` in the agent env and
|
||||
add it to `hidden_env_names`.
|
||||
|
||||
No dummy auth file and no `verify` step are needed — Claude Code reads
|
||||
the credential from the env var, not from a file.
|
||||
|
||||
### Constants
|
||||
|
||||
- `CLAUDE_HOST_CREDENTIAL_TOKEN_REF = "BOT_BOTTLE_CLAUDE_HOST_ACCESS_TOKEN"`
|
||||
in `egress.py` (alongside the existing `CODEX_HOST_CREDENTIAL_TOKEN_REF`).
|
||||
- `CLAUDE_HOST_CREDENTIAL_HOSTS = ("api.anthropic.com",)` in
|
||||
`agent_provider.py` (alongside the existing `CODEX_HOST_CREDENTIAL_HOSTS`).
|
||||
|
||||
### Data flow
|
||||
|
||||
```
|
||||
Host ~/.claude/.credentials.json → bot-bottle launch
|
||||
│
|
||||
├──► egress sidecar env (real token only)
|
||||
│
|
||||
└──► agent env: CLAUDE_CODE_OAUTH_TOKEN=egress-placeholder
|
||||
|
||||
Agent → HTTPS to api.anthropic.com (via egress)
|
||||
Egress → injects Authorization: Bearer <real token>
|
||||
Egress → forwards to api.anthropic.com
|
||||
```
|
||||
|
||||
## Open questions
|
||||
|
||||
None — the Codex precedent makes the design clear.
|
||||
@@ -0,0 +1,47 @@
|
||||
# PRD prd-new: Modernize built-in agent images
|
||||
|
||||
- **Status:** Draft
|
||||
- **Author:** Codex
|
||||
- **Created:** 2026-07-21
|
||||
- **Issue:** #451
|
||||
|
||||
## Summary
|
||||
|
||||
Keep every built-in agent provider on Debian's current stable release and make
|
||||
Podman available inside each image. This gives agents a consistent, modern
|
||||
userspace and an OCI container tool without requiring per-project setup.
|
||||
|
||||
## Problem
|
||||
|
||||
The Claude, Codex, and Pi images inherit the generic `node:22-slim` tag. That
|
||||
tag does not state which Debian release the project supports and currently
|
||||
leaves the images on the older Bookworm release. None of the built-in images
|
||||
installs Podman, so tasks that need to inspect or build OCI images must first
|
||||
modify the bottle or cannot run at all.
|
||||
|
||||
## Goals / success criteria
|
||||
|
||||
- Every Dockerfile under `bot_bottle/contrib/*/Dockerfile` explicitly inherits
|
||||
`node:22-trixie-slim`, based on Debian 13 (the current stable release).
|
||||
- Every built-in agent image installs Podman from Debian stable.
|
||||
- Every built-in agent image retains an SSH client for Git-over-SSH workflows.
|
||||
- The non-root agent user owns a traversable XDG Git configuration directory,
|
||||
so Git can load bot-bottle's global git-gate rewrites without permission
|
||||
errors.
|
||||
- A shared test enforces both requirements for current and future built-in
|
||||
providers.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Configuring privileged or nested-container execution for bottles.
|
||||
- Pinning Podman outside Debian's stable package repository.
|
||||
- Changing the Node.js or agent CLI release policy.
|
||||
|
||||
## Design
|
||||
|
||||
Use the explicit `node:22-trixie-slim` base rather than the floating `slim`
|
||||
variant. Install the `podman` package with each image's existing `apt-get`
|
||||
dependency layer, so package metadata and caches are still removed in the same
|
||||
layer. Treat Debian stable as the Podman stability and update channel; this
|
||||
keeps the images stdlib/distribution-first and avoids adding a third-party
|
||||
package repository.
|
||||
+28
-8
@@ -1,15 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# Combined unit + integration coverage (see docs/decisions/0004-coverage-policy.md).
|
||||
#
|
||||
# Runs the unit suite, then appends the integration suite (which skips
|
||||
# cleanly when Docker / the backend CLIs are unavailable), and prints one
|
||||
# combined report. The integration suite is what scores the subprocess /
|
||||
# backend orchestration modules, so the number here is the policy's
|
||||
# yardstick — not the unit-only badge.
|
||||
# Two modes:
|
||||
#
|
||||
# Usage:
|
||||
# scripts/coverage.sh # combined report
|
||||
# scripts/coverage.sh critical # also report just the critical modules
|
||||
# scripts/coverage.sh [critical]
|
||||
# Run mode (default, for local dev): executes the unit suite then the
|
||||
# integration suite under coverage and prints a combined report.
|
||||
#
|
||||
# scripts/coverage.sh aggregate [critical]
|
||||
# Aggregate mode (used by CI): combines pre-existing .coverage.* files
|
||||
# produced by individual test jobs and prints a combined report. No tests
|
||||
# are re-executed; no KVM or Docker dependency.
|
||||
#
|
||||
# Pass "critical" as the last argument in either mode to also report just the
|
||||
# critical modules (ADR 0004 target: 90%).
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
@@ -21,6 +25,22 @@ PY="${PYTHON:-python3}"
|
||||
# README "core coverage" badge can't drift; comma-join it for --include.
|
||||
CRITICAL=$(grep -vE '^[[:space:]]*(#|$)' scripts/critical-modules.txt | paste -sd, -)
|
||||
|
||||
if [ "${1:-}" = "aggregate" ]; then
|
||||
# Aggregate mode: combine .coverage.* artifacts already in the workspace.
|
||||
echo "== combining coverage artifacts ==" >&2
|
||||
"$PY" -m coverage combine
|
||||
|
||||
echo "== combined report ==" >&2
|
||||
"$PY" -m coverage report -m
|
||||
|
||||
if [ "${2:-}" = "critical" ]; then
|
||||
echo "== critical modules (ADR 0004 target: 90%) ==" >&2
|
||||
"$PY" -m coverage report --include="$CRITICAL"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Run mode (default): execute both suites under coverage in this process.
|
||||
rm -f .coverage
|
||||
|
||||
echo "== unit ==" >&2
|
||||
|
||||
@@ -172,7 +172,7 @@ class TestSandboxEscape(unittest.TestCase):
|
||||
# base image without producing five confusing
|
||||
# command-not-found failures down the suite.
|
||||
missing: list[str] = []
|
||||
for tool in ("curl", "git", "dig"):
|
||||
for tool in ("curl", "git", "dig", "ssh"):
|
||||
r = cls._bottle.exec(f"command -v {tool} >/dev/null 2>&1")
|
||||
if r.returncode != 0:
|
||||
missing.append(tool)
|
||||
|
||||
@@ -9,11 +9,15 @@ import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from bot_bottle.agent_provider import (
|
||||
CLAUDE_HOST_CREDENTIAL_HOSTS,
|
||||
CODEX_HOST_CREDENTIAL_HOSTS,
|
||||
build_agent_provision_plan,
|
||||
prompt_args,
|
||||
)
|
||||
from bot_bottle.egress import CODEX_HOST_CREDENTIAL_TOKEN_REF
|
||||
from bot_bottle.egress import (
|
||||
CLAUDE_HOST_CREDENTIAL_TOKEN_REF,
|
||||
CODEX_HOST_CREDENTIAL_TOKEN_REF,
|
||||
)
|
||||
|
||||
|
||||
def _jwt(exp: int) -> str:
|
||||
@@ -292,6 +296,67 @@ class TestAgentProviderRuntime(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual({}, plan.provisioned_env)
|
||||
|
||||
def test_claude_forward_host_credentials_populates_egress_route(self):
|
||||
access_token = "sk-ant-oat01-test-key" # gitleaks:allow
|
||||
with tempfile.TemporaryDirectory(prefix="bb-provider.") as tmp:
|
||||
home = Path(tmp) / "host-claude"
|
||||
cred_dir = home / ".claude"
|
||||
cred_dir.mkdir(parents=True)
|
||||
(cred_dir / ".credentials.json").write_text(json.dumps({
|
||||
"claudeAiOauth": {"accessToken": access_token},
|
||||
}))
|
||||
plan = build_agent_provision_plan(
|
||||
template="claude",
|
||||
dockerfile="",
|
||||
state_dir=Path(tmp),
|
||||
instance_name="bot-bottle-test",
|
||||
prompt_file=Path(tmp) / "prompt.txt",
|
||||
forward_host_credentials=True,
|
||||
host_env={"HOME": str(home)},
|
||||
)
|
||||
self.assertEqual(1, len(plan.egress_routes))
|
||||
route = plan.egress_routes[0]
|
||||
self.assertIn(route.host, CLAUDE_HOST_CREDENTIAL_HOSTS)
|
||||
self.assertEqual("Bearer", route.auth_scheme)
|
||||
self.assertEqual(CLAUDE_HOST_CREDENTIAL_TOKEN_REF, route.token_ref)
|
||||
self.assertEqual("egress-placeholder", plan.env_vars["CLAUDE_CODE_OAUTH_TOKEN"])
|
||||
self.assertEqual(frozenset({"CLAUDE_CODE_OAUTH_TOKEN"}), plan.hidden_env_names)
|
||||
|
||||
def test_claude_forward_host_credentials_populates_provisioned_env(self):
|
||||
access_token = "sk-ant-oat01-test-key" # gitleaks:allow
|
||||
with tempfile.TemporaryDirectory(prefix="bb-provider.") as tmp:
|
||||
home = Path(tmp) / "host-claude"
|
||||
cred_dir = home / ".claude"
|
||||
cred_dir.mkdir(parents=True)
|
||||
(cred_dir / ".credentials.json").write_text(json.dumps({
|
||||
"claudeAiOauth": {"accessToken": access_token},
|
||||
}))
|
||||
plan = build_agent_provision_plan(
|
||||
template="claude",
|
||||
dockerfile="",
|
||||
state_dir=Path(tmp),
|
||||
instance_name="bot-bottle-test",
|
||||
prompt_file=Path(tmp) / "prompt.txt",
|
||||
forward_host_credentials=True,
|
||||
host_env={"HOME": str(home)},
|
||||
)
|
||||
self.assertEqual(
|
||||
{CLAUDE_HOST_CREDENTIAL_TOKEN_REF: access_token},
|
||||
plan.provisioned_env,
|
||||
)
|
||||
|
||||
def test_claude_without_forward_host_credentials_has_empty_provisioned_env(self):
|
||||
with tempfile.TemporaryDirectory(prefix="bb-provider.") as tmp:
|
||||
plan = build_agent_provision_plan(
|
||||
template="claude",
|
||||
dockerfile="",
|
||||
state_dir=Path(tmp),
|
||||
instance_name="bot-bottle-test",
|
||||
prompt_file=Path(tmp) / "prompt.txt",
|
||||
forward_host_credentials=False,
|
||||
)
|
||||
self.assertEqual({}, plan.provisioned_env)
|
||||
|
||||
def test_pi_plan_writes_default_ollama_models(self):
|
||||
with tempfile.TemporaryDirectory(prefix="bb-provider.") as tmp:
|
||||
plan = build_agent_provision_plan(
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Unit contracts shared by all built-in agent images."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
_CONTRIB_DIR = Path(__file__).resolve().parents[2] / "bot_bottle/contrib"
|
||||
_AGENT_DOCKERFILES = tuple(sorted(_CONTRIB_DIR.glob("*/Dockerfile")))
|
||||
|
||||
|
||||
class TestBuiltinAgentImages(unittest.TestCase):
|
||||
def test_all_use_debian_trixie_stable(self):
|
||||
self.assertTrue(_AGENT_DOCKERFILES)
|
||||
for dockerfile in _AGENT_DOCKERFILES:
|
||||
with self.subTest(provider=dockerfile.parent.name):
|
||||
self.assertRegex(
|
||||
dockerfile.read_text(),
|
||||
r"(?m)^FROM node:22-trixie-slim\s*$",
|
||||
)
|
||||
|
||||
def test_all_install_podman(self):
|
||||
for dockerfile in _AGENT_DOCKERFILES:
|
||||
with self.subTest(provider=dockerfile.parent.name):
|
||||
self.assertRegex(
|
||||
dockerfile.read_text(),
|
||||
re.compile(r"(?m)^\s*podman(?:\s|\\|$)"),
|
||||
)
|
||||
|
||||
def test_all_install_ssh_client(self):
|
||||
for dockerfile in _AGENT_DOCKERFILES:
|
||||
with self.subTest(provider=dockerfile.parent.name):
|
||||
self.assertRegex(
|
||||
dockerfile.read_text(),
|
||||
re.compile(r"(?m)^\s*openssh-client(?:\s|\\|$)"),
|
||||
)
|
||||
|
||||
def test_all_prepare_node_git_config_directory(self):
|
||||
for dockerfile in _AGENT_DOCKERFILES:
|
||||
with self.subTest(provider=dockerfile.parent.name):
|
||||
dockerfile_text = dockerfile.read_text()
|
||||
self.assertIn("install -d -o node -g node", dockerfile_text)
|
||||
self.assertIn("/home/node/.config/git", dockerfile_text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,268 @@
|
||||
"""Unit tests for bb login command."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
import urllib.error
|
||||
from email.message import Message
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
class TestFlagParsing(unittest.TestCase):
|
||||
def test_console_url_flag(self) -> None:
|
||||
from bot_bottle.cli.login import _flag
|
||||
self.assertEqual(_flag(["--console-url", "http://x"], "--console-url"), "http://x")
|
||||
|
||||
def test_console_url_equals_form(self) -> None:
|
||||
from bot_bottle.cli.login import _flag
|
||||
self.assertEqual(
|
||||
_flag(["--console-url=http://x"], "--console-url"), "http://x"
|
||||
)
|
||||
|
||||
def test_label_flag(self) -> None:
|
||||
from bot_bottle.cli.login import _flag
|
||||
self.assertEqual(_flag(["--label", "my-mac"], "--label"), "my-mac")
|
||||
|
||||
def test_missing_flag_returns_none(self) -> None:
|
||||
from bot_bottle.cli.login import _flag
|
||||
self.assertIsNone(_flag([], "--console-url"))
|
||||
|
||||
|
||||
class TestHttpHelpers(unittest.TestCase):
|
||||
def test_post_sends_json_and_decodes_response(self) -> None:
|
||||
from bot_bottle.cli.login import _post
|
||||
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value.read.return_value = b'{"ok": true}'
|
||||
with patch("urllib.request.urlopen", return_value=response) as urlopen:
|
||||
self.assertEqual(
|
||||
_post("http://console/start", {"label": "host"}), {"ok": True}
|
||||
)
|
||||
|
||||
request = urlopen.call_args.args[0]
|
||||
self.assertEqual(request.data, b'{"label": "host"}')
|
||||
self.assertEqual(request.get_header("Content-type"), "application/json")
|
||||
|
||||
def test_get_decodes_success_response(self) -> None:
|
||||
from bot_bottle.cli.login import _get
|
||||
|
||||
response = MagicMock()
|
||||
response.__enter__.return_value.status = 200
|
||||
response.__enter__.return_value.read.return_value = b'{"status": "pending"}'
|
||||
with patch("urllib.request.urlopen", return_value=response):
|
||||
self.assertEqual(
|
||||
_get("http://console/status"), (200, {"status": "pending"})
|
||||
)
|
||||
|
||||
def test_get_returns_http_error_status(self) -> None:
|
||||
from bot_bottle.cli.login import _get
|
||||
|
||||
error = urllib.error.HTTPError(
|
||||
"http://console/status", 410, "gone", Message(), None
|
||||
)
|
||||
with patch("urllib.request.urlopen", side_effect=error):
|
||||
self.assertEqual(_get("http://console/status"), (410, {}))
|
||||
|
||||
|
||||
class TestSaveCredentials(unittest.TestCase):
|
||||
def test_writes_json_and_sets_perms(self) -> None:
|
||||
from bot_bottle.cli.login import _save_credentials
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}):
|
||||
path = _save_credentials("http://c", "hid", "at", "rt")
|
||||
self.assertTrue(path.exists())
|
||||
data = json.loads(path.read_text())
|
||||
self.assertEqual(data["url"], "http://c")
|
||||
self.assertEqual(data["host_id"], "hid")
|
||||
self.assertEqual(data["access_token"], "at")
|
||||
self.assertEqual(data["refresh_token"], "rt")
|
||||
self.assertEqual(oct(path.stat().st_mode & 0o777), oct(0o600))
|
||||
|
||||
def test_temp_file_is_private_before_replace(self) -> None:
|
||||
"""Temp file must be 0600 at the moment os.replace is called."""
|
||||
from bot_bottle.cli.login import _save_credentials
|
||||
from pathlib import Path as _Path
|
||||
|
||||
tmp_perms_at_replace: list[int] = []
|
||||
real_replace = os.replace
|
||||
|
||||
def _spy_replace(
|
||||
src: str | os.PathLike[str], dst: str | os.PathLike[str]
|
||||
) -> None:
|
||||
tmp_perms_at_replace.append(_Path(src).stat().st_mode & 0o777)
|
||||
real_replace(src, dst)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}):
|
||||
with patch("os.replace", side_effect=_spy_replace):
|
||||
path = _save_credentials("http://c", "hid", "at", "rt")
|
||||
self.assertEqual(len(tmp_perms_at_replace), 1)
|
||||
self.assertEqual(oct(tmp_perms_at_replace[0]), oct(0o600))
|
||||
self.assertEqual(oct(path.stat().st_mode & 0o777), oct(0o600))
|
||||
|
||||
def test_cleanup_on_write_failure(self) -> None:
|
||||
"""Temp file is removed and no credentials remain if replace fails."""
|
||||
from bot_bottle.cli.login import _save_credentials
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}):
|
||||
with patch("os.replace", side_effect=OSError("disk full")):
|
||||
with self.assertRaises(OSError):
|
||||
_save_credentials("http://c", "hid", "at", "rt")
|
||||
leftovers = [f for f in os.listdir(tmp) if f.startswith(".console-")]
|
||||
self.assertEqual(leftovers, [])
|
||||
|
||||
|
||||
class TestCmdLoginMissingUrl(unittest.TestCase):
|
||||
def test_help_returns_0(self) -> None:
|
||||
from bot_bottle.cli.login import cmd_login
|
||||
|
||||
self.assertEqual(cmd_login(["--help"]), 0)
|
||||
|
||||
def test_returns_1_without_url(self) -> None:
|
||||
from bot_bottle.cli.login import cmd_login
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
os.environ.pop("BB_CONSOLE_URL", None)
|
||||
result = cmd_login([])
|
||||
self.assertEqual(result, 1)
|
||||
|
||||
def test_reads_env_var(self) -> None:
|
||||
"""Exits 1 (network error) not because of missing URL when env var is set."""
|
||||
from bot_bottle.cli.login import cmd_login
|
||||
|
||||
def _fail_post(_url: str, _payload: dict[str, Any]) -> dict[str, Any]:
|
||||
raise OSError("connection refused")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"BB_CONSOLE_URL": "http://localhost:9999",
|
||||
"BOT_BOTTLE_ROOT": tmp,
|
||||
},
|
||||
):
|
||||
with patch("bot_bottle.cli.login._post", side_effect=_fail_post):
|
||||
result = cmd_login([])
|
||||
self.assertEqual(result, 1)
|
||||
|
||||
|
||||
class TestCmdLoginFlow(unittest.TestCase):
|
||||
def _run_with_mocks(
|
||||
self, poll_responses: list[dict[str, Any]], tmp: str
|
||||
) -> int:
|
||||
from bot_bottle.cli.login import cmd_login
|
||||
|
||||
start_resp = {
|
||||
"device_code": "dc123",
|
||||
"user_code": "ABC-DEF",
|
||||
"expires_in": 300,
|
||||
"poll_interval": 0,
|
||||
}
|
||||
|
||||
poll_iter = iter(poll_responses)
|
||||
|
||||
def _fake_post(
|
||||
_url: str, _payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
return start_resp
|
||||
|
||||
def _fake_get(_url: str) -> tuple[int, dict[str, Any]]:
|
||||
return 200, next(poll_iter, {"status": "pending"})
|
||||
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}):
|
||||
with patch("bot_bottle.cli.login._post", side_effect=_fake_post):
|
||||
with patch("bot_bottle.cli.login._get", side_effect=_fake_get):
|
||||
with patch("time.sleep"):
|
||||
return cmd_login(["--console-url", "http://console"])
|
||||
|
||||
def test_approved_flow_returns_0(self) -> None:
|
||||
approved = {
|
||||
"status": "approved",
|
||||
"host_id": "hid",
|
||||
"access_token": "at",
|
||||
"refresh_token": "rt",
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
result = self._run_with_mocks(
|
||||
[{"status": "pending"}, approved], tmp
|
||||
)
|
||||
self.assertEqual(result, 0)
|
||||
with open(os.path.join(tmp, "console.json"), encoding="utf-8") as f:
|
||||
creds = json.loads(f.read())
|
||||
self.assertEqual(creds["host_id"], "hid")
|
||||
|
||||
def test_denied_flow_returns_1(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
result = self._run_with_mocks([{"status": "denied"}], tmp)
|
||||
self.assertEqual(result, 1)
|
||||
|
||||
def test_timeout_returns_1(self) -> None:
|
||||
from bot_bottle.cli.login import cmd_login
|
||||
|
||||
start_resp = {
|
||||
"device_code": "dc",
|
||||
"user_code": "ZZZ-ZZZ",
|
||||
"expires_in": 0, # already expired; loop never runs
|
||||
"poll_interval": 2,
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}):
|
||||
with patch("bot_bottle.cli.login._post", return_value=start_resp):
|
||||
result = cmd_login(["--console-url", "http://console"])
|
||||
self.assertEqual(result, 1)
|
||||
|
||||
def test_poll_interval_from_server_is_used(self) -> None:
|
||||
"""time.sleep must be called with the server-provided poll_interval."""
|
||||
from bot_bottle.cli.login import cmd_login
|
||||
|
||||
server_interval = 7
|
||||
start_resp = {
|
||||
"device_code": "dc",
|
||||
"user_code": "ABC-DEF",
|
||||
"expires_in": 300,
|
||||
"poll_interval": server_interval,
|
||||
}
|
||||
approved = {
|
||||
"status": "approved",
|
||||
"host_id": "hid",
|
||||
"access_token": "at",
|
||||
"refresh_token": "rt",
|
||||
}
|
||||
poll_iter = iter([{"status": "pending"}, approved])
|
||||
|
||||
def _fake_get(_url: str) -> tuple[int, dict[str, str]]:
|
||||
return 200, next(poll_iter)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}):
|
||||
with patch("bot_bottle.cli.login._post", return_value=start_resp):
|
||||
with patch(
|
||||
"bot_bottle.cli.login._get", side_effect=_fake_get
|
||||
):
|
||||
with patch("time.sleep") as mock_sleep:
|
||||
result = cmd_login(["--console-url", "http://console"])
|
||||
|
||||
self.assertEqual(result, 0)
|
||||
self.assertTrue(mock_sleep.called)
|
||||
for call in mock_sleep.call_args_list:
|
||||
self.assertEqual(call.args[0], server_interval)
|
||||
|
||||
|
||||
class TestDispatcherRegistration(unittest.TestCase):
|
||||
def test_login_in_commands(self) -> None:
|
||||
from bot_bottle.cli import COMMANDS
|
||||
self.assertIn("login", COMMANDS)
|
||||
|
||||
def test_login_in_no_migration(self) -> None:
|
||||
from bot_bottle.cli import NO_MIGRATION_COMMANDS
|
||||
self.assertIn("login", NO_MIGRATION_COMMANDS)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""The CLI package is runnable as `python -m bot_bottle.cli`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def _run(*args: str) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[sys.executable, "-m", "bot_bottle.cli", *args],
|
||||
cwd=_REPO_ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
class TestModuleEntry(unittest.TestCase):
|
||||
def test_help_exits_zero(self) -> None:
|
||||
result = _run("--help")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("login", result.stderr)
|
||||
|
||||
def test_no_args_prints_usage(self) -> None:
|
||||
# main() returns 2 with no command, matching the cli.py entry point.
|
||||
self.assertEqual(_run().returncode, 2)
|
||||
|
||||
def test_subcommand_help_reaches_handler(self) -> None:
|
||||
result = _run("login", "--help")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("--console-url", result.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -176,6 +176,18 @@ class TestCmdStartHeadless(unittest.TestCase):
|
||||
self.assertEqual("researcher-2", self._spec().label)
|
||||
|
||||
|
||||
def test_cached_images_sets_cached_policy(self):
|
||||
start_mod.cmd_start(
|
||||
["--headless", "--cached-images", "researcher", "--bottle", "claude",
|
||||
"--prompt", "Do it"]
|
||||
)
|
||||
self.assertEqual("cached", self._spec().image_policy)
|
||||
|
||||
def test_cached_images_requires_headless(self):
|
||||
with self.assertRaises(Die):
|
||||
start_mod.cmd_start(["--cached-images", "researcher"])
|
||||
self._launch_mock.assert_not_called()
|
||||
|
||||
|
||||
class TestPrepareWithPreflight(unittest.TestCase):
|
||||
"""prepare_with_preflight calls render_preflight with the plan and backend name."""
|
||||
|
||||
@@ -65,6 +65,12 @@ class TestCmdStartSelector(unittest.TestCase):
|
||||
)
|
||||
self._modal_patch.start()
|
||||
|
||||
self._image_policy_patch = patch(
|
||||
"bot_bottle.cli.start._select_image_policy",
|
||||
return_value="fresh",
|
||||
)
|
||||
self._image_policy_patch.start()
|
||||
|
||||
self._env_patch = patch.dict(os.environ, {}, clear=False)
|
||||
self._env_patch.start()
|
||||
os.environ.pop("BOT_BOTTLE_BACKEND", None)
|
||||
@@ -75,6 +81,7 @@ class TestCmdStartSelector(unittest.TestCase):
|
||||
self._agent_picker_patch.stop()
|
||||
self._bottle_picker_patch.stop()
|
||||
self._modal_patch.stop()
|
||||
self._image_policy_patch.stop()
|
||||
self._env_patch.stop()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
@@ -133,6 +140,19 @@ class TestCmdStartSelector(unittest.TestCase):
|
||||
spec = self._launch_mock.call_args[0][0]
|
||||
self.assertEqual(("claude", "dev"), spec.bottle_names)
|
||||
|
||||
def test_image_policy_forwarded_to_spec(self):
|
||||
with patch("bot_bottle.cli.start._select_image_policy", return_value="cached"):
|
||||
start_mod.cmd_start(["researcher"])
|
||||
self._launch_mock.assert_called_once()
|
||||
spec = self._launch_mock.call_args[0][0]
|
||||
self.assertEqual("cached", spec.image_policy)
|
||||
|
||||
def test_image_policy_cancel_returns_0(self):
|
||||
with patch("bot_bottle.cli.start._select_image_policy", return_value=None):
|
||||
rc = start_mod.cmd_start(["researcher"])
|
||||
self.assertEqual(0, rc)
|
||||
self._launch_mock.assert_not_called()
|
||||
|
||||
def test_empty_bottle_selection_forwarded(self):
|
||||
self._bottle_picker_mock.return_value = []
|
||||
start_mod.cmd_start(["researcher"])
|
||||
@@ -214,6 +234,7 @@ class TestCmdStartLabelCollision(unittest.TestCase):
|
||||
).start()
|
||||
# Stub the bottle picker to always return a selection.
|
||||
patch.object(tui_mod, "filter_multiselect", return_value=["claude"]).start()
|
||||
patch("bot_bottle.cli.start._select_image_policy", return_value="fresh").start()
|
||||
self.addCleanup(patch.stopall)
|
||||
|
||||
def test_no_collision_proceeds_without_reprompt(self):
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Unit: _launch_bottle StaleImageError handling.
|
||||
|
||||
Exercises prelaunch_checks / backend.launch flow:
|
||||
- headless mode → die on stale
|
||||
- interactive mode, user declines → stop without launching
|
||||
- interactive mode, user confirms → skip stale check and launch once
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import tempfile
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bot_bottle.image_cache import StaleImageError
|
||||
from bot_bottle.log import Die
|
||||
|
||||
|
||||
def _fake_plan() -> Any:
|
||||
provision = SimpleNamespace(startup_args=())
|
||||
return cast(Any, SimpleNamespace(
|
||||
agent_provision=provision,
|
||||
agent_provider_template="claude",
|
||||
slug="dev-abc",
|
||||
))
|
||||
|
||||
|
||||
def _ok_cm(bottle: Any) -> MagicMock:
|
||||
"""Return a context-manager mock that yields `bottle`."""
|
||||
cm = MagicMock()
|
||||
cm.__enter__ = MagicMock(return_value=bottle)
|
||||
cm.__exit__ = MagicMock(return_value=False)
|
||||
return cm
|
||||
|
||||
|
||||
class TestLaunchBottleStaleHandling(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.mkdtemp(prefix="cli-stale-test.")
|
||||
|
||||
def _spec(self) -> Any:
|
||||
from bot_bottle.backend import BottleSpec
|
||||
from bot_bottle.manifest import ManifestIndex
|
||||
idx = ManifestIndex.from_json_obj({
|
||||
"bottles": {"dev": {}},
|
||||
"agents": {"demo": {"skills": [], "prompt": "", "bottle": "dev"}},
|
||||
})
|
||||
return BottleSpec(
|
||||
manifest=idx,
|
||||
agent_name="demo",
|
||||
copy_cwd=False,
|
||||
user_cwd=self._tmp,
|
||||
identity="dev-abc",
|
||||
)
|
||||
|
||||
def _run_launch(self, **patch_kwargs: Any) -> int:
|
||||
import bot_bottle.cli.start as start_mod
|
||||
spec = self._spec()
|
||||
with patch.object(start_mod, "prepare_with_preflight",
|
||||
return_value=(_fake_plan(), "dev-abc")), \
|
||||
patch.object(start_mod, "settle_state"), \
|
||||
patch.object(start_mod, "info"):
|
||||
return start_mod._launch_bottle(
|
||||
spec,
|
||||
dry_run=False,
|
||||
backend_name="docker",
|
||||
**patch_kwargs,
|
||||
)
|
||||
|
||||
def test_headless_stale_calls_die(self) -> None:
|
||||
"""In headless mode (assume_yes=True), a StaleImageError from prelaunch_checks must call die()."""
|
||||
import bot_bottle.cli.start as start_mod
|
||||
|
||||
backend_mock = MagicMock()
|
||||
backend_mock.prelaunch_checks.side_effect = StaleImageError("image is 5 day(s) old")
|
||||
|
||||
with patch.object(start_mod, "get_bottle_backend", return_value=backend_mock), \
|
||||
patch.object(start_mod, "die", side_effect=Die()):
|
||||
with self.assertRaises(Die):
|
||||
self._run_launch(assume_yes=True)
|
||||
|
||||
backend_mock.launch.assert_not_called()
|
||||
|
||||
def test_interactive_user_declines_stops_before_launch(self) -> None:
|
||||
"""Interactive user answering 'n' → launch is never called."""
|
||||
import bot_bottle.cli.start as start_mod
|
||||
|
||||
backend_mock = MagicMock()
|
||||
backend_mock.prelaunch_checks.side_effect = StaleImageError("image is 5 day(s) old")
|
||||
|
||||
with patch.object(start_mod, "get_bottle_backend", return_value=backend_mock), \
|
||||
patch.object(start_mod, "read_tty_line", return_value="n"), \
|
||||
patch("sys.stderr", new_callable=io.StringIO):
|
||||
rc = self._run_launch(assume_yes=False)
|
||||
|
||||
self.assertEqual(0, rc)
|
||||
backend_mock.launch.assert_not_called()
|
||||
|
||||
def test_interactive_user_confirms_launches_once(self) -> None:
|
||||
"""Interactive user answering 'y' → prelaunch stale error is bypassed; launch called once."""
|
||||
import bot_bottle.cli.start as start_mod
|
||||
|
||||
bottle_mock = MagicMock()
|
||||
bottle_mock.name = "dev-abc"
|
||||
|
||||
backend_mock = MagicMock()
|
||||
backend_mock.prelaunch_checks.side_effect = StaleImageError("image is 5 day(s) old")
|
||||
backend_mock.launch.return_value = _ok_cm(bottle_mock)
|
||||
|
||||
with patch.object(start_mod, "get_bottle_backend", return_value=backend_mock), \
|
||||
patch.object(start_mod, "read_tty_line", return_value="y"), \
|
||||
patch.object(start_mod, "attach_agent", return_value=0), \
|
||||
patch.object(start_mod, "capture_claude_session_state"), \
|
||||
patch("sys.stderr", new_callable=io.StringIO):
|
||||
rc = self._run_launch(assume_yes=False)
|
||||
|
||||
self.assertEqual(0, rc)
|
||||
backend_mock.prelaunch_checks.assert_called_once()
|
||||
backend_mock.launch.assert_called_once()
|
||||
|
||||
def test_interactive_yes_uppercase_also_accepted(self) -> None:
|
||||
"""'Y' or 'YES' should also be accepted as confirmation."""
|
||||
import bot_bottle.cli.start as start_mod
|
||||
|
||||
bottle_mock = MagicMock()
|
||||
bottle_mock.name = "dev-abc"
|
||||
|
||||
backend_mock = MagicMock()
|
||||
backend_mock.prelaunch_checks.side_effect = StaleImageError("image is 5 day(s) old")
|
||||
backend_mock.launch.return_value = _ok_cm(bottle_mock)
|
||||
|
||||
with patch.object(start_mod, "get_bottle_backend", return_value=backend_mock), \
|
||||
patch.object(start_mod, "read_tty_line", return_value="YES"), \
|
||||
patch.object(start_mod, "attach_agent", return_value=0), \
|
||||
patch.object(start_mod, "capture_claude_session_state"), \
|
||||
patch("sys.stderr", new_callable=io.StringIO):
|
||||
rc = self._run_launch(assume_yes=False)
|
||||
|
||||
self.assertEqual(0, rc)
|
||||
backend_mock.launch.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Unit tests for the host-side configuration store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from bot_bottle.config_store import (
|
||||
DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS,
|
||||
ConfigStore,
|
||||
)
|
||||
from bot_bottle.store_manager import StoreManager
|
||||
|
||||
|
||||
class TestConfigStore(unittest.TestCase):
|
||||
def test_cached_image_warning_days_defaults_to_one(self) -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="config-store.") as tmp:
|
||||
store = ConfigStore(Path(tmp) / "bot-bottle.db")
|
||||
store.migrate()
|
||||
self.assertEqual(
|
||||
DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS,
|
||||
store.cached_image_stale_warning_days(),
|
||||
)
|
||||
|
||||
def test_cached_image_warning_days_reads_value(self) -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="config-store.") as tmp:
|
||||
store = ConfigStore(Path(tmp) / "bot-bottle.db")
|
||||
store.migrate()
|
||||
store.set_cached_image_stale_warning_days(7)
|
||||
self.assertEqual(7, store.cached_image_stale_warning_days())
|
||||
|
||||
def test_config_schema_uses_explicit_settings_columns(self) -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="config-store.") as tmp:
|
||||
store = ConfigStore(Path(tmp) / "bot-bottle.db")
|
||||
store.migrate()
|
||||
with sqlite3.connect(store.db_path) as conn:
|
||||
conn.row_factory = sqlite3.Row
|
||||
columns = [
|
||||
row["name"]
|
||||
for row in conn.execute("PRAGMA table_info(bot_bottle_config)")
|
||||
]
|
||||
self.assertEqual([
|
||||
"id",
|
||||
"cached_image_stale_warning_days",
|
||||
], columns)
|
||||
|
||||
def test_store_manager_includes_config_store(self) -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="config-store.") as tmp:
|
||||
db = Path(tmp) / "bot-bottle.db"
|
||||
manager = StoreManager(db)
|
||||
self.assertFalse(manager.is_migrated())
|
||||
manager.migrate()
|
||||
self.assertTrue(manager.is_migrated())
|
||||
|
||||
def test_cached_image_warning_days_returns_default_when_db_missing(self) -> None:
|
||||
# When the db file doesn't exist yet (parent exists, file doesn't),
|
||||
# the store returns the default without touching the file.
|
||||
with tempfile.TemporaryDirectory(prefix="config-store.") as tmp:
|
||||
store = ConfigStore(Path(tmp) / "missing.db")
|
||||
self.assertEqual(
|
||||
DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS,
|
||||
store.cached_image_stale_warning_days(),
|
||||
)
|
||||
|
||||
def test_cached_image_warning_days_returns_default_on_null_value(self) -> None:
|
||||
# If the row exists but the value is NULL (or not castable to int),
|
||||
# the store falls back to the default.
|
||||
with tempfile.TemporaryDirectory(prefix="config-store.") as tmp:
|
||||
db_path = Path(tmp) / "bot-bottle.db"
|
||||
store = ConfigStore(db_path)
|
||||
store.migrate()
|
||||
# Write a NULL value directly.
|
||||
with sqlite3.connect(db_path) as conn:
|
||||
conn.execute(
|
||||
"UPDATE bot_bottle_config SET cached_image_stale_warning_days = NULL WHERE id = 1"
|
||||
)
|
||||
self.assertEqual(
|
||||
DEFAULT_CACHED_IMAGE_STALE_WARNING_DAYS,
|
||||
store.cached_image_stale_warning_days(),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Unit: host Claude auth extraction."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bot_bottle.contrib.claude.claude_auth import (
|
||||
claude_auth_path,
|
||||
claude_host_access_token,
|
||||
)
|
||||
from bot_bottle.log import Die
|
||||
|
||||
|
||||
def _cred_json(access_token: str, **extra: object) -> str:
|
||||
payload: dict[str, object] = {"claudeAiOauth": {"accessToken": access_token, **extra}}
|
||||
return json.dumps(payload)
|
||||
|
||||
|
||||
class TestClaudeHostAccessToken(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory(prefix="bb-claude-auth.")
|
||||
self.home = Path(self.tmp.name)
|
||||
self.cred_dir = self.home / ".claude"
|
||||
self.cred_dir.mkdir()
|
||||
self.auth_path = self.cred_dir / ".credentials.json"
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _write(self, payload: dict) -> None: # type: ignore[no-untyped-def]
|
||||
self.auth_path.write_text(json.dumps(payload))
|
||||
|
||||
def test_auth_path_uses_home_env(self):
|
||||
self.assertEqual(
|
||||
self.auth_path,
|
||||
claude_auth_path({"HOME": str(self.home)}),
|
||||
)
|
||||
|
||||
# --- file-based (Linux) ---
|
||||
|
||||
def test_file_returns_access_token(self):
|
||||
key = "sk-ant-oat01-real-key" # gitleaks:allow
|
||||
self._write({"claudeAiOauth": {"accessToken": key}})
|
||||
out = claude_host_access_token({"HOME": str(self.home)})
|
||||
self.assertEqual(key, out)
|
||||
|
||||
def test_file_missing_claude_ai_oauth_dies(self):
|
||||
self._write({"hasCompletedOnboarding": True})
|
||||
with self.assertRaises(Die):
|
||||
claude_host_access_token({"HOME": str(self.home)})
|
||||
|
||||
def test_file_missing_access_token_dies(self):
|
||||
self._write({"claudeAiOauth": {"expiresAt": 2000000000000}})
|
||||
with self.assertRaises(Die):
|
||||
claude_host_access_token({"HOME": str(self.home)})
|
||||
|
||||
def test_file_empty_access_token_dies(self):
|
||||
self._write({"claudeAiOauth": {"accessToken": ""}})
|
||||
with self.assertRaises(Die):
|
||||
claude_host_access_token({"HOME": str(self.home)})
|
||||
|
||||
def test_file_expired_token_dies(self):
|
||||
# expiresAt is milliseconds; 1_000_000 ms is year 1970
|
||||
self._write({
|
||||
"claudeAiOauth": {"accessToken": "sk-ant-oat01-x", "expiresAt": 1_000_000}, # gitleaks:allow
|
||||
})
|
||||
with self.assertRaises(Die):
|
||||
claude_host_access_token(
|
||||
{"HOME": str(self.home)},
|
||||
now=datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
)
|
||||
|
||||
def test_file_future_expiry_is_accepted(self):
|
||||
key = "sk-ant-oat01-y" # gitleaks:allow
|
||||
# 2_000_000_000_000 ms ≈ year 2033
|
||||
self._write({
|
||||
"claudeAiOauth": {"accessToken": key, "expiresAt": 2_000_000_000_000},
|
||||
})
|
||||
out = claude_host_access_token(
|
||||
{"HOME": str(self.home)},
|
||||
now=datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
)
|
||||
self.assertEqual(key, out)
|
||||
|
||||
def test_file_absent_expiry_is_accepted(self):
|
||||
key = "sk-ant-oat01-z" # gitleaks:allow
|
||||
self._write({"claudeAiOauth": {"accessToken": key}})
|
||||
out = claude_host_access_token({"HOME": str(self.home)})
|
||||
self.assertEqual(key, out)
|
||||
|
||||
def test_file_non_json_dies(self):
|
||||
self.auth_path.write_text("not json {{{")
|
||||
with self.assertRaises(Die):
|
||||
claude_host_access_token({"HOME": str(self.home)})
|
||||
|
||||
def test_file_json_array_root_dies(self):
|
||||
self.auth_path.write_text("[]")
|
||||
with self.assertRaises(Die):
|
||||
claude_host_access_token({"HOME": str(self.home)})
|
||||
|
||||
def test_file_extra_fields_are_ignored(self):
|
||||
key = "sk-ant-oat01-real" # gitleaks:allow
|
||||
self._write({
|
||||
"claudeAiOauth": {
|
||||
"accessToken": key,
|
||||
"refreshToken": "sk-ant-ort01-secret", # gitleaks:allow
|
||||
"scopes": ["user:inference"],
|
||||
"expiresAt": 2_000_000_000_000,
|
||||
},
|
||||
})
|
||||
out = claude_host_access_token({"HOME": str(self.home)})
|
||||
self.assertEqual(key, out)
|
||||
|
||||
# --- macOS Keychain fallback ---
|
||||
|
||||
def _home_without_creds(self) -> Path:
|
||||
"""A home dir that has .claude/ but no .credentials.json."""
|
||||
empty = self.home / "no-creds"
|
||||
(empty / ".claude").mkdir(parents=True)
|
||||
return empty
|
||||
|
||||
def _mock_keychain(self, stdout: str, returncode: int = 0) -> MagicMock:
|
||||
mock = MagicMock()
|
||||
mock.returncode = returncode
|
||||
mock.stdout = stdout
|
||||
return mock
|
||||
|
||||
def test_keychain_used_when_file_absent(self):
|
||||
key = "sk-ant-oat01-keychain" # gitleaks:allow
|
||||
home = self._home_without_creds()
|
||||
with patch(
|
||||
"bot_bottle.contrib.claude.claude_auth.subprocess.run",
|
||||
return_value=self._mock_keychain(_cred_json(key)),
|
||||
), patch(
|
||||
"bot_bottle.contrib.claude.claude_auth.sys.platform", "darwin",
|
||||
):
|
||||
out = claude_host_access_token({"HOME": str(home)})
|
||||
self.assertEqual(key, out)
|
||||
|
||||
def test_keychain_failure_when_file_absent_dies(self):
|
||||
home = self._home_without_creds()
|
||||
with patch(
|
||||
"bot_bottle.contrib.claude.claude_auth.subprocess.run",
|
||||
return_value=self._mock_keychain("", returncode=44),
|
||||
), patch(
|
||||
"bot_bottle.contrib.claude.claude_auth.sys.platform", "darwin",
|
||||
):
|
||||
with self.assertRaises(Die):
|
||||
claude_host_access_token({"HOME": str(home)})
|
||||
|
||||
def test_no_file_no_keychain_on_linux_dies(self):
|
||||
home = self._home_without_creds()
|
||||
with patch("bot_bottle.contrib.claude.claude_auth.sys.platform", "linux"):
|
||||
with self.assertRaises(Die):
|
||||
claude_host_access_token({"HOME": str(home)})
|
||||
|
||||
def test_keychain_non_json_dies(self):
|
||||
home = self._home_without_creds()
|
||||
with patch(
|
||||
"bot_bottle.contrib.claude.claude_auth.subprocess.run",
|
||||
return_value=self._mock_keychain("not-json"),
|
||||
), patch(
|
||||
"bot_bottle.contrib.claude.claude_auth.sys.platform", "darwin",
|
||||
):
|
||||
with self.assertRaises(Die):
|
||||
claude_host_access_token({"HOME": str(home)})
|
||||
|
||||
def test_keychain_security_not_found_dies(self):
|
||||
home = self._home_without_creds()
|
||||
with patch(
|
||||
"bot_bottle.contrib.claude.claude_auth.subprocess.run",
|
||||
side_effect=FileNotFoundError,
|
||||
), patch(
|
||||
"bot_bottle.contrib.claude.claude_auth.sys.platform", "darwin",
|
||||
):
|
||||
with self.assertRaises(Die):
|
||||
claude_host_access_token({"HOME": str(home)})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import dataclasses
|
||||
import io
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -18,6 +19,7 @@ from bot_bottle.backend.docker.bottle_plan import DockerBottlePlan
|
||||
from bot_bottle.backend.docker.consolidated_launch import LaunchContext
|
||||
from bot_bottle.egress import EgressPlan
|
||||
from bot_bottle.git_gate import GitGatePlan
|
||||
from bot_bottle.log import Die
|
||||
from bot_bottle.manifest import ManifestIndex
|
||||
from tests.unit import use_bottle_root
|
||||
|
||||
@@ -92,6 +94,7 @@ class TestLaunchCommittedImage(unittest.TestCase):
|
||||
mock.patch.object(launch_mod.docker_mod, "image_exists", return_value=image_present), \
|
||||
mock.patch.object(launch_mod.docker_mod, "build_image", side_effect=_build), \
|
||||
mock.patch.object(launch_mod.docker_mod, "verify_agent_image"), \
|
||||
mock.patch.object(launch_mod.docker_mod, "image_created_at"), \
|
||||
mock.patch.object(launch_mod, "launch_consolidated", return_value=_CTX), \
|
||||
mock.patch.object(launch_mod, "teardown_consolidated"), \
|
||||
mock.patch.object(launch_mod, "DockerGateway", return_value=gw), \
|
||||
@@ -112,7 +115,8 @@ class TestLaunchCommittedImage(unittest.TestCase):
|
||||
with self._patched(
|
||||
committed_tag=committed_tag, image_present=image_present, compose=compose,
|
||||
) as built:
|
||||
with launch_mod.launch(plan, provision=mock.Mock(return_value=None)):
|
||||
images = launch_mod.build_or_load_images(plan)
|
||||
with launch_mod.launch(plan, images, provision=mock.Mock(return_value=None)):
|
||||
pass
|
||||
return built
|
||||
|
||||
@@ -127,14 +131,34 @@ class TestLaunchCommittedImage(unittest.TestCase):
|
||||
captured.append(p)
|
||||
return {"services": {"agent": {}}}
|
||||
|
||||
with self._patched(committed_tag=_COMMITTED_TAG, image_present=True, compose=compose):
|
||||
with launch_mod.launch(_plan(self._tmp), provision=mock.Mock(return_value=None)):
|
||||
with self._patched(committed_tag=_COMMITTED_TAG, image_present=True, compose=compose) as _:
|
||||
plan = _plan(self._tmp)
|
||||
images = launch_mod.build_or_load_images(plan)
|
||||
with launch_mod.launch(plan, images, provision=mock.Mock(return_value=None)):
|
||||
pass
|
||||
self.assertEqual(_COMMITTED_TAG, captured[0].image)
|
||||
|
||||
def test_falls_back_to_build_when_no_committed_image(self) -> None:
|
||||
self.assertEqual([_DEFAULT_IMAGE], self._run_launch(_plan(self._tmp), committed_tag=None))
|
||||
|
||||
def test_cached_images_skip_build_when_present(self) -> None:
|
||||
base = _plan(self._tmp)
|
||||
plan = dataclasses.replace(
|
||||
base,
|
||||
spec=dataclasses.replace(base.spec, image_policy="cached"),
|
||||
)
|
||||
built = self._run_launch(plan, committed_tag=None, image_present=True)
|
||||
self.assertEqual([], built)
|
||||
|
||||
def test_cached_images_die_when_agent_missing(self) -> None:
|
||||
base = _plan(self._tmp)
|
||||
plan = dataclasses.replace(
|
||||
base,
|
||||
spec=dataclasses.replace(base.spec, image_policy="cached"),
|
||||
)
|
||||
with self.assertRaises(Die):
|
||||
self._run_launch(plan, committed_tag=None, image_present=False)
|
||||
|
||||
def test_falls_back_to_build_when_committed_image_missing_from_daemon(self) -> None:
|
||||
built = self._run_launch(_plan(self._tmp), committed_tag=_COMMITTED_TAG, image_present=False)
|
||||
self.assertEqual([_DEFAULT_IMAGE], built)
|
||||
|
||||
@@ -16,7 +16,7 @@ from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
from bot_bottle.agent_provider import AgentProvisionPlan
|
||||
from bot_bottle.backend import BottleSpec
|
||||
from bot_bottle.backend import BottleImages, BottleSpec
|
||||
from bot_bottle.backend.docker import launch as launch_mod
|
||||
from bot_bottle.backend.docker.bottle_plan import DockerBottlePlan
|
||||
from bot_bottle.backend.docker.consolidated_launch import LaunchContext
|
||||
@@ -93,6 +93,8 @@ class TestTeardownWarning(unittest.TestCase):
|
||||
orchestrator_url="http://orch:8099",
|
||||
)
|
||||
|
||||
images = BottleImages(agent="bot-bottle-claude:latest", sidecar="bot-bottle-sidecars:latest")
|
||||
|
||||
with mock.patch.object(launch_mod.docker_mod, "build_image"), \
|
||||
mock.patch.object(launch_mod.docker_mod, "verify_agent_image"), \
|
||||
mock.patch.object(launch_mod, "launch_consolidated", return_value=ctx), \
|
||||
@@ -113,7 +115,7 @@ class TestTeardownWarning(unittest.TestCase):
|
||||
), \
|
||||
contextlib.redirect_stderr(buf):
|
||||
provision = mock.Mock(return_value=None)
|
||||
with launch_mod.launch(plan, provision=provision):
|
||||
with launch_mod.launch(plan, images, provision=provision):
|
||||
pass
|
||||
|
||||
output = buf.getvalue()
|
||||
|
||||
@@ -45,12 +45,15 @@ _PROVIDER = _Provider()
|
||||
|
||||
|
||||
def _plan(*, git_user: dict | None = None, # type: ignore
|
||||
git_repos: dict | None = None, # type: ignore
|
||||
copy_cwd: bool = False,
|
||||
user_cwd: str = "/tmp/x",
|
||||
stage_dir: Path | None = None) -> DockerBottlePlan:
|
||||
bottle_json: dict = {} # type: ignore
|
||||
if git_user is not None:
|
||||
bottle_json["git-gate"] = {"user": git_user}
|
||||
if git_repos is not None:
|
||||
bottle_json.setdefault("git-gate", {})["repos"] = git_repos
|
||||
index = ManifestIndex.from_json_obj({
|
||||
"bottles": {"dev": bottle_json},
|
||||
"agents": {"demo": {"skills": [], "prompt": "", "bottle": "dev"}},
|
||||
@@ -125,6 +128,62 @@ class TestProvisionGitUser(unittest.TestCase):
|
||||
_PROVIDER.provision_git(bottle, _plan(stage_dir=self.stage))
|
||||
self.assertEqual([], _git_config_exec_calls(bottle))
|
||||
|
||||
def test_repairs_git_xdg_directory_for_runtime_user(self):
|
||||
bottle = _make_bottle()
|
||||
_PROVIDER.provision_git(bottle, _plan(stage_dir=self.stage))
|
||||
|
||||
script, user = next(
|
||||
(call.args[0], call.kwargs.get("user", "node"))
|
||||
for call in bottle.exec.call_args_list
|
||||
if "/home/node/.config/git" in call.args[0]
|
||||
)
|
||||
self.assertEqual("root", user)
|
||||
self.assertIn("chown node:node /home/node", script)
|
||||
self.assertIn("chmod 755 /home/node", script)
|
||||
self.assertIn("mkdir -p /home/node/.config/git", script)
|
||||
self.assertIn("chown -R node:node /home/node/.config", script)
|
||||
self.assertIn("chmod -R u+rwX,go+rX /home/node/.config", script)
|
||||
|
||||
def test_fails_closed_when_home_permissions_cannot_be_repaired(self):
|
||||
bottle = _make_bottle()
|
||||
bottle.exec.return_value = ExecResult(1, "", "read-only filesystem")
|
||||
|
||||
with self.assertRaises(SystemExit):
|
||||
_PROVIDER.provision_git(bottle, _plan(stage_dir=self.stage))
|
||||
|
||||
def _git_plan(self) -> DockerBottlePlan:
|
||||
return _plan(
|
||||
git_repos={
|
||||
"repo": {
|
||||
"url": "ssh://git@example.com/repo.git",
|
||||
"key": {"provider": "static", "path": "/dev/null"},
|
||||
"host_key": "ssh-ed25519 AAAA",
|
||||
},
|
||||
},
|
||||
stage_dir=self.stage,
|
||||
)
|
||||
|
||||
def test_fails_closed_when_gitconfig_permissions_cannot_be_set(self):
|
||||
bottle = _make_bottle()
|
||||
bottle.exec.side_effect = [
|
||||
ExecResult(0, "", ""),
|
||||
ExecResult(1, "", "chown failed"),
|
||||
]
|
||||
|
||||
with self.assertRaises(SystemExit):
|
||||
_PROVIDER.provision_git(bottle, self._git_plan())
|
||||
|
||||
def test_fails_closed_when_runtime_user_cannot_read_gitconfig(self):
|
||||
bottle = _make_bottle()
|
||||
bottle.exec.side_effect = [
|
||||
ExecResult(0, "", ""),
|
||||
ExecResult(0, "", ""),
|
||||
ExecResult(1, "", "permission denied"),
|
||||
]
|
||||
|
||||
with self.assertRaises(SystemExit):
|
||||
_PROVIDER.provision_git(bottle, self._git_plan())
|
||||
|
||||
def test_sets_name_and_email(self):
|
||||
plan = _plan(
|
||||
git_user={"name": "Eric Bauerfeld", "email": "eric@dideric.is"},
|
||||
|
||||
@@ -9,6 +9,7 @@ from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import unittest
|
||||
from datetime import timezone
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.backend.docker import util as docker_mod
|
||||
@@ -26,6 +27,59 @@ def _fail(stderr: str = "boom") -> subprocess.CompletedProcess: # type: ignore
|
||||
)
|
||||
|
||||
|
||||
|
||||
class TestImageCreatedAt(unittest.TestCase):
|
||||
def test_parses_docker_timestamp_with_nanoseconds(self):
|
||||
with patch.object(
|
||||
docker_mod.subprocess, "run",
|
||||
return_value=_ok(stdout="2026-07-06T15:33:47.123456789Z\n"),
|
||||
) as run:
|
||||
created = docker_mod.image_created_at("bot-bottle-claude:latest")
|
||||
self.assertIsNotNone(created)
|
||||
assert created is not None
|
||||
self.assertEqual(2026, created.year)
|
||||
self.assertEqual(123456, created.microsecond)
|
||||
self.assertEqual(timezone.utc, created.tzinfo)
|
||||
self.assertEqual(
|
||||
["docker", "image", "inspect", "--format", "{{.Created}}", "bot-bottle-claude:latest"],
|
||||
run.call_args.args[0],
|
||||
)
|
||||
|
||||
def test_dies_on_inspect_failure(self):
|
||||
with patch.object(
|
||||
docker_mod.subprocess, "run", return_value=_fail("No such image"),
|
||||
), patch.object(
|
||||
docker_mod, "die", side_effect=SystemExit("die"),
|
||||
) as die:
|
||||
with self.assertRaises(SystemExit):
|
||||
docker_mod.image_created_at("missing:tag")
|
||||
die.assert_called_once()
|
||||
self.assertIn("missing:tag", die.call_args.args[0])
|
||||
|
||||
def test_returns_none_on_invalid_timestamp(self):
|
||||
with patch.object(
|
||||
docker_mod.subprocess, "run",
|
||||
return_value=_ok(stdout="not-a-timestamp\n"),
|
||||
):
|
||||
result = docker_mod.image_created_at("some:tag")
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_returns_none_on_empty_stdout(self):
|
||||
with patch.object(
|
||||
docker_mod.subprocess, "run",
|
||||
return_value=_ok(stdout=""),
|
||||
):
|
||||
result = docker_mod.image_created_at("some:tag")
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_parse_docker_timestamp_no_tzinfo_defaults_to_utc(self):
|
||||
# A bare datetime with no tz offset should be treated as UTC.
|
||||
dt = docker_mod._parse_docker_timestamp("2024-05-01T10:00:00.000000")
|
||||
self.assertIsNotNone(dt.tzinfo)
|
||||
self.assertEqual(timezone.utc, dt.tzinfo)
|
||||
|
||||
|
||||
|
||||
class TestCommitContainer(unittest.TestCase):
|
||||
def test_runs_docker_commit(self):
|
||||
with patch.object(
|
||||
|
||||
@@ -413,6 +413,28 @@ class TestAuthInjection(unittest.TestCase):
|
||||
assert flow.response is not None
|
||||
self.assertEqual(403, flow.response.status_code)
|
||||
|
||||
def test_preserve_auth_passes_agent_token_through(self) -> None:
|
||||
route = Route(host="registry-1.docker.io", preserve_auth=True)
|
||||
addon = _addon(Config(routes=(route,)))
|
||||
flow = _Flow(_Request(
|
||||
host="registry-1.docker.io",
|
||||
headers={"authorization": "Bearer agent-registry-token"},
|
||||
))
|
||||
_run_request(addon, flow)
|
||||
self.assertEqual("Bearer agent-registry-token", flow.request.headers.get("authorization"))
|
||||
self.assertIsNone(flow.response)
|
||||
|
||||
def test_default_route_strips_agent_auth(self) -> None:
|
||||
route = Route(host="registry-1.docker.io")
|
||||
addon = _addon(Config(routes=(route,)))
|
||||
flow = _Flow(_Request(
|
||||
host="registry-1.docker.io",
|
||||
headers={"authorization": "Bearer agent-registry-token"},
|
||||
))
|
||||
_run_request(addon, flow)
|
||||
self.assertIsNone(flow.request.headers.get("authorization"))
|
||||
self.assertIsNone(flow.response)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# git push / fetch over HTTPS
|
||||
|
||||
@@ -61,6 +61,12 @@ class TestNetpoolSlots(unittest.TestCase):
|
||||
|
||||
|
||||
class TestNetpoolRenderers(unittest.TestCase):
|
||||
def test_guest_init_restores_node_home_boundary(self):
|
||||
from bot_bottle.backend.firecracker import util
|
||||
|
||||
self.assertIn("chown node:node /home/node", util._GUEST_INIT)
|
||||
self.assertIn("chmod 755 /home/node", util._GUEST_INIT)
|
||||
|
||||
def test_nixos_module_is_non_invasive(self):
|
||||
# The NixOS module must NOT flip the host firewall backend or
|
||||
# hand interfaces to systemd-networkd; it brings the pool up via
|
||||
@@ -422,10 +428,15 @@ class TestBottlePlanProperties(unittest.TestCase):
|
||||
ap.command = "claude"
|
||||
ap.prompt_mode = "append_file"
|
||||
ap.template = "claude"
|
||||
ap.guest_home = "/home/node"
|
||||
ap.guest_env = {}
|
||||
egress_plan = cast(Any, MagicMock())
|
||||
egress_plan.canary = ""
|
||||
egress_plan.canary_env = ""
|
||||
fields = dict(
|
||||
spec=cast(Any, MagicMock()), manifest=cast(Any, MagicMock()),
|
||||
stage_dir=Path("/stage"), git_gate_plan=cast(Any, MagicMock()),
|
||||
egress_plan=cast(Any, MagicMock()), supervise_plan=None,
|
||||
egress_plan=egress_plan, supervise_plan=None,
|
||||
agent_provision=ap, slug="demo-x", forwarded_env={},
|
||||
)
|
||||
fields.update(overrides)
|
||||
@@ -451,6 +462,12 @@ class TestBottlePlanProperties(unittest.TestCase):
|
||||
self.assertEqual("10.243.0.0:9420", p.git_gate_insteadof_host)
|
||||
self.assertEqual("http", p.git_gate_insteadof_scheme)
|
||||
|
||||
def test_guest_env_pins_global_git_config(self):
|
||||
from bot_bottle.backend.firecracker.launch import _agent_guest_env
|
||||
|
||||
env = _agent_guest_env(self._plan(), "10.243.0.0")
|
||||
self.assertEqual("/home/node/.gitconfig", env["GIT_CONFIG_GLOBAL"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -35,6 +35,15 @@ class TestBuildAgentRootfsDir(unittest.TestCase):
|
||||
build.assert_not_called()
|
||||
self.assertEqual(base, out)
|
||||
|
||||
def test_cached_lookup_requires_ready_marker(self):
|
||||
digest = image_builder._rootfs_digest(self.dockerfile)
|
||||
base = self.cache / "rootfs" / f"agent-{digest}"
|
||||
base.mkdir(parents=True)
|
||||
with patch.object(image_builder.util, "cache_dir", return_value=self.cache):
|
||||
self.assertIsNone(image_builder.cached_agent_rootfs_dir(self.dockerfile))
|
||||
(base / ".bb-ready").write_text("ok\n")
|
||||
self.assertEqual(base, image_builder.cached_agent_rootfs_dir(self.dockerfile))
|
||||
|
||||
def test_cache_miss_builds_injects_and_marks_ready(self):
|
||||
with patch.object(image_builder.util, "cache_dir", return_value=self.cache), \
|
||||
patch.object(image_builder, "_build_in_infra") as build, \
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Unit: image_cache.py — check_stale / check_stale_path."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.image_cache import StaleImageError, check_stale, check_stale_path
|
||||
|
||||
|
||||
class TestCheckStale(unittest.TestCase):
|
||||
def _run(self, threshold: int, age_days: float) -> None:
|
||||
created = datetime.now(tz=timezone.utc) - timedelta(days=age_days)
|
||||
with patch("bot_bottle.image_cache.ConfigStore") as cs:
|
||||
cs.return_value.cached_image_stale_warning_days.return_value = threshold
|
||||
check_stale("test image", created)
|
||||
|
||||
def test_negative_threshold_never_raises(self):
|
||||
# Threshold < 0 means the check is disabled — always passes.
|
||||
self._run(threshold=-1, age_days=9999)
|
||||
|
||||
def test_zero_threshold_raises_immediately(self):
|
||||
# threshold=0 means any image is stale the moment it exists.
|
||||
with self.assertRaises(StaleImageError):
|
||||
self._run(threshold=0, age_days=0.1)
|
||||
|
||||
def test_within_threshold_does_not_raise(self):
|
||||
# Age well under threshold — should pass silently.
|
||||
self._run(threshold=7, age_days=2)
|
||||
|
||||
def test_at_threshold_does_not_raise(self):
|
||||
# Exactly at the boundary is fine (<=, not <).
|
||||
self._run(threshold=1, age_days=0.9999)
|
||||
|
||||
def test_exceeds_threshold_raises(self):
|
||||
created = datetime.now(tz=timezone.utc) - timedelta(days=3)
|
||||
with patch("bot_bottle.image_cache.ConfigStore") as cs:
|
||||
cs.return_value.cached_image_stale_warning_days.return_value = 1
|
||||
with self.assertRaises(StaleImageError) as ctx:
|
||||
check_stale("agent image 'bot-bottle:latest'", created)
|
||||
self.assertIn("agent image", str(ctx.exception))
|
||||
self.assertIn("day(s) old", str(ctx.exception))
|
||||
|
||||
def test_naive_datetime_treated_as_utc(self):
|
||||
# check_stale calls .astimezone(utc) on the input; naive datetimes
|
||||
# that would be interpreted as local time should still work.
|
||||
# We can't control the local tz in a unit test, so just ensure
|
||||
# no exception is thrown for a very recent naive datetime.
|
||||
naive_now = datetime(2099, 1, 1) # far future, always "fresh"
|
||||
with patch("bot_bottle.image_cache.ConfigStore") as cs:
|
||||
cs.return_value.cached_image_stale_warning_days.return_value = 1
|
||||
# Should not raise — the image is brand new.
|
||||
check_stale("test image", naive_now)
|
||||
|
||||
|
||||
class TestCheckStalePath(unittest.TestCase):
|
||||
def test_delegates_to_check_stale_with_mtime(self):
|
||||
with tempfile.NamedTemporaryFile() as f:
|
||||
path = Path(f.name)
|
||||
with patch("bot_bottle.image_cache.check_stale") as mock_check:
|
||||
check_stale_path("some artifact", path)
|
||||
mock_check.assert_called_once()
|
||||
label, dt = mock_check.call_args.args
|
||||
self.assertEqual("some artifact", label)
|
||||
self.assertIsInstance(dt, datetime)
|
||||
self.assertIsNotNone(dt.tzinfo)
|
||||
|
||||
def test_raises_stale_for_old_file(self):
|
||||
with tempfile.NamedTemporaryFile() as f:
|
||||
path = Path(f.name)
|
||||
with patch("bot_bottle.image_cache.ConfigStore") as cs:
|
||||
cs.return_value.cached_image_stale_warning_days.return_value = 0
|
||||
with self.assertRaises(StaleImageError):
|
||||
check_stale_path("cached artifact", path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -12,7 +12,7 @@ import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import ANY, patch
|
||||
|
||||
from bot_bottle.backend.macos_container.bottle import MacosContainerBottle
|
||||
from bot_bottle.backend.macos_container.bottle_plan import MacosContainerBottlePlan
|
||||
@@ -21,7 +21,9 @@ from bot_bottle.backend.macos_container.gateway_hosts import GATEWAY_HOSTNAME
|
||||
from bot_bottle.backend.macos_container.launch import (
|
||||
_agent_run_argv,
|
||||
_identity_proxy_env,
|
||||
build_or_load_images,
|
||||
)
|
||||
from bot_bottle.log import Die
|
||||
from bot_bottle.manifest import ManifestIndex
|
||||
|
||||
_BOTTLE = "bot_bottle.backend.macos_container.bottle"
|
||||
@@ -46,6 +48,7 @@ def _plan(
|
||||
*,
|
||||
agent_git_gate_url: str = "",
|
||||
agent_supervise_url: str = "",
|
||||
image_policy: str = "fresh",
|
||||
) -> MacosContainerBottlePlan:
|
||||
routes_path = stage_dir / "routes.yaml"
|
||||
routes_path.write_text("routes: []\n", encoding="utf-8")
|
||||
@@ -60,12 +63,13 @@ def _plan(
|
||||
canary_env="",
|
||||
)
|
||||
return cast(MacosContainerBottlePlan, SimpleNamespace(
|
||||
spec=SimpleNamespace(),
|
||||
spec=SimpleNamespace(image_policy=image_policy),
|
||||
manifest=_MANIFEST,
|
||||
stage_dir=stage_dir,
|
||||
slug="dev-abc",
|
||||
container_name="bot-bottle-dev-abc",
|
||||
image="bot-bottle-agent:latest",
|
||||
dockerfile_path="/repo/Dockerfile",
|
||||
forwarded_env={"OAUTH_TOKEN": "host-value"},
|
||||
egress_plan=egress_plan,
|
||||
git_gate_plan=SimpleNamespace(upstreams=()),
|
||||
@@ -79,6 +83,88 @@ def _plan(
|
||||
))
|
||||
|
||||
|
||||
class TestBuildOrLoadImages(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.plan = _plan(Path(self._tmp.name))
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def test_reuses_present_committed_image(self) -> None:
|
||||
with (
|
||||
patch(
|
||||
"bot_bottle.backend.macos_container.launch.read_committed_image",
|
||||
return_value="committed:latest",
|
||||
),
|
||||
patch(
|
||||
"bot_bottle.backend.macos_container.launch.container_mod.image_exists",
|
||||
return_value=True,
|
||||
),
|
||||
patch(
|
||||
"bot_bottle.backend.macos_container.launch.container_mod.build_image"
|
||||
) as build,
|
||||
):
|
||||
images = build_or_load_images(self.plan)
|
||||
|
||||
self.assertEqual("committed:latest", images.agent)
|
||||
build.assert_not_called()
|
||||
|
||||
def test_reuses_present_cached_image(self) -> None:
|
||||
plan = _plan(Path(self._tmp.name), image_policy="cached")
|
||||
with (
|
||||
patch(
|
||||
"bot_bottle.backend.macos_container.launch.read_committed_image",
|
||||
return_value=None,
|
||||
),
|
||||
patch(
|
||||
"bot_bottle.backend.macos_container.launch.container_mod.image_exists",
|
||||
return_value=True,
|
||||
),
|
||||
patch(
|
||||
"bot_bottle.backend.macos_container.launch.container_mod.build_image"
|
||||
) as build,
|
||||
):
|
||||
images = build_or_load_images(plan)
|
||||
|
||||
self.assertEqual(plan.image, images.agent)
|
||||
build.assert_not_called()
|
||||
|
||||
def test_cached_policy_rejects_missing_image(self) -> None:
|
||||
plan = _plan(Path(self._tmp.name), image_policy="cached")
|
||||
with (
|
||||
patch(
|
||||
"bot_bottle.backend.macos_container.launch.read_committed_image",
|
||||
return_value=None,
|
||||
),
|
||||
patch(
|
||||
"bot_bottle.backend.macos_container.launch.container_mod.image_exists",
|
||||
return_value=False,
|
||||
),
|
||||
):
|
||||
with self.assertRaises(Die):
|
||||
build_or_load_images(plan)
|
||||
|
||||
def test_fresh_policy_builds_image(self) -> None:
|
||||
with (
|
||||
patch(
|
||||
"bot_bottle.backend.macos_container.launch.read_committed_image",
|
||||
return_value=None,
|
||||
),
|
||||
patch(
|
||||
"bot_bottle.backend.macos_container.launch.container_mod.build_image"
|
||||
) as build,
|
||||
):
|
||||
images = build_or_load_images(self.plan)
|
||||
|
||||
self.assertEqual(self.plan.image, images.agent)
|
||||
build.assert_called_once_with(
|
||||
self.plan.image,
|
||||
ANY,
|
||||
dockerfile=self.plan.dockerfile_path,
|
||||
)
|
||||
|
||||
|
||||
class TestAgentRunArgv(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
|
||||
@@ -391,5 +391,72 @@ class TestWaitContainerIpv4(unittest.TestCase):
|
||||
self.assertEqual("", util.wait_container_ipv4_on_network("c", "net", timeout=-1))
|
||||
|
||||
|
||||
class TestMacosContainerImageCreatedAt(unittest.TestCase):
|
||||
def _ok(self, stdout: str) -> "util.subprocess.CompletedProcess": # type: ignore
|
||||
return util.subprocess.CompletedProcess(
|
||||
args=[], returncode=0, stdout=stdout, stderr="",
|
||||
)
|
||||
|
||||
def _fail(self, stderr: str = "no such image") -> "util.subprocess.CompletedProcess": # type: ignore
|
||||
return util.subprocess.CompletedProcess(
|
||||
args=[], returncode=1, stdout="", stderr=stderr,
|
||||
)
|
||||
|
||||
def test_parses_iso_timestamp_from_dict(self):
|
||||
payload = '[{"created": "2025-06-01T12:00:00"}]'
|
||||
with patch.object(util.subprocess, "run", return_value=self._ok(payload)):
|
||||
dt = util.image_created_at("bot-bottle-agent:latest")
|
||||
self.assertIsNotNone(dt)
|
||||
assert dt is not None
|
||||
self.assertEqual(2025, dt.year)
|
||||
self.assertEqual(6, dt.month)
|
||||
self.assertEqual(1, dt.day)
|
||||
|
||||
def test_accepts_list_or_dict_input(self):
|
||||
# Container CLI may return a list; we take the first element.
|
||||
payload = '[{"created": "2024-01-15T08:30:00"}]'
|
||||
with patch.object(util.subprocess, "run", return_value=self._ok(payload)):
|
||||
dt = util.image_created_at("some-image:latest")
|
||||
self.assertIsNotNone(dt)
|
||||
assert dt is not None
|
||||
self.assertEqual(2024, dt.year)
|
||||
|
||||
def test_accepts_uppercase_Created_field(self):
|
||||
payload = '[{"Created": "2024-03-20T10:00:00"}]'
|
||||
with patch.object(util.subprocess, "run", return_value=self._ok(payload)):
|
||||
dt = util.image_created_at("some-image:latest")
|
||||
self.assertIsNotNone(dt)
|
||||
assert dt is not None
|
||||
self.assertEqual(2024, dt.year)
|
||||
self.assertEqual(3, dt.month)
|
||||
|
||||
def test_dies_on_nonzero_returncode(self):
|
||||
with patch.object(util.subprocess, "run", return_value=self._fail("not found")), \
|
||||
patch.object(util, "die", side_effect=SystemExit("die")) as die:
|
||||
with self.assertRaises(SystemExit):
|
||||
util.image_created_at("missing:tag")
|
||||
die.assert_called_once()
|
||||
self.assertIn("missing:tag", die.call_args.args[0])
|
||||
|
||||
def test_dies_on_malformed_json(self):
|
||||
with patch.object(util.subprocess, "run", return_value=self._ok("not-json {")), \
|
||||
patch.object(util, "die", side_effect=SystemExit("die")) as die:
|
||||
with self.assertRaises(SystemExit):
|
||||
util.image_created_at("some:tag")
|
||||
die.assert_called_once()
|
||||
|
||||
def test_returns_none_when_no_created_field(self):
|
||||
payload = '[{"id": "sha256:abc123"}]'
|
||||
with patch.object(util.subprocess, "run", return_value=self._ok(payload)):
|
||||
result = util.image_created_at("some:tag")
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_returns_none_on_invalid_timestamp_format(self):
|
||||
payload = '[{"created": "not-a-date"}]'
|
||||
with patch.object(util.subprocess, "run", return_value=self._ok(payload)):
|
||||
result = util.image_created_at("some:tag")
|
||||
self.assertIsNone(result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -61,6 +61,20 @@ class TestInfraRun(unittest.TestCase):
|
||||
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
|
||||
self.assertTrue(all("bot-bottle.db" not in m for m in mounts))
|
||||
|
||||
def test_ca_is_persisted_on_the_host_not_the_container_volume(self) -> None:
|
||||
"""The CA survives infra recreation and cannot be removed by Apple
|
||||
Container's volume-prune command."""
|
||||
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
||||
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
|
||||
ca_mounts = [
|
||||
m for m in mounts
|
||||
if "target=/home/mitmproxy/.mitmproxy" in m
|
||||
]
|
||||
self.assertEqual(1, len(ca_mounts))
|
||||
self.assertIn("source=", ca_mounts[0])
|
||||
self.assertIn("/gateway-ca", ca_mounts[0])
|
||||
self.assertNotIn(",readonly", ca_mounts[0])
|
||||
|
||||
def test_nat_network_precedes_the_host_only_network(self) -> None:
|
||||
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
||||
nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
||||
|
||||
@@ -80,11 +80,19 @@ class TestAgentProviderHostCredentials(unittest.TestCase):
|
||||
"forward_host_credentials": "yes",
|
||||
})
|
||||
|
||||
def test_forward_host_credentials_rejected_for_claude(self):
|
||||
def test_forward_host_credentials_allowed_for_claude(self):
|
||||
b = _provider_config_bottle({
|
||||
"template": "claude",
|
||||
"forward_host_credentials": True,
|
||||
})
|
||||
self.assertTrue(b.agent_provider.forward_host_credentials)
|
||||
|
||||
def test_forward_host_credentials_and_auth_token_rejected_together(self):
|
||||
with self.assertRaises(ManifestError):
|
||||
_provider_config_bottle({
|
||||
"template": "claude",
|
||||
"forward_host_credentials": True,
|
||||
"auth_token": "SOME_TOKEN",
|
||||
})
|
||||
|
||||
def test_auth_token_defaults_empty(self):
|
||||
@@ -450,6 +458,24 @@ class TestRole(unittest.TestCase):
|
||||
_bottle([{"host": "x.example", "role": ["x", 42]}])
|
||||
|
||||
|
||||
class TestPreserveAuth(unittest.TestCase):
|
||||
def test_omitted_defaults_false(self):
|
||||
b = _bottle([{"host": "registry-1.docker.io"}])
|
||||
self.assertFalse(b.egress.routes[0].PreserveAuth)
|
||||
|
||||
def test_true_accepted(self):
|
||||
b = _bottle([{"host": "registry-1.docker.io", "preserve_auth": True}])
|
||||
self.assertTrue(b.egress.routes[0].PreserveAuth)
|
||||
|
||||
def test_false_accepted(self):
|
||||
b = _bottle([{"host": "registry-1.docker.io", "preserve_auth": False}])
|
||||
self.assertFalse(b.egress.routes[0].PreserveAuth)
|
||||
|
||||
def test_non_bool_rejected(self):
|
||||
with self.assertRaises(ManifestError):
|
||||
_bottle([{"host": "registry-1.docker.io", "preserve_auth": "yes"}])
|
||||
|
||||
|
||||
class TestPipelockKeyRejected(unittest.TestCase):
|
||||
def test_pipelock_key_rejected_as_unknown(self):
|
||||
with self.assertRaises(ManifestError):
|
||||
|
||||
@@ -86,10 +86,22 @@ class TestAgentProviderValidation(unittest.TestCase):
|
||||
"b", {"forward_host_credentials": True, "template": "weird"}
|
||||
)
|
||||
|
||||
def test_forward_creds_non_codex_template(self) -> None:
|
||||
def test_forward_creds_pi_template_rejected(self) -> None:
|
||||
with self.assertRaises(ManifestError):
|
||||
ManifestAgentProvider.from_dict(
|
||||
"b", {"forward_host_credentials": True, "template": "claude"}
|
||||
"b", {"forward_host_credentials": True, "template": "pi"}
|
||||
)
|
||||
|
||||
def test_forward_creds_claude_allowed(self) -> None:
|
||||
p = ManifestAgentProvider.from_dict(
|
||||
"b", {"forward_host_credentials": True, "template": "claude"}
|
||||
)
|
||||
self.assertTrue(p.forward_host_credentials)
|
||||
|
||||
def test_forward_creds_and_auth_token_rejected(self) -> None:
|
||||
with self.assertRaises(ManifestError):
|
||||
ManifestAgentProvider.from_dict(
|
||||
"b", {"forward_host_credentials": True, "auth_token": "T", "template": "claude"}
|
||||
)
|
||||
|
||||
def test_valid_claude_auth_token(self) -> None:
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from bot_bottle.orchestrator.gateway import (
|
||||
@@ -10,7 +12,10 @@ from bot_bottle.orchestrator.gateway import (
|
||||
GATEWAY_NAME,
|
||||
DockerGateway,
|
||||
GatewayError,
|
||||
rotate_gateway_ca,
|
||||
)
|
||||
from bot_bottle.paths import GATEWAY_CA_DIRNAME, host_gateway_ca_dir
|
||||
from tests.unit import use_bottle_root
|
||||
|
||||
|
||||
_CA_PEM = "-----BEGIN CERTIFICATE-----\nMII...\n-----END CERTIFICATE-----\n"
|
||||
@@ -27,6 +32,11 @@ _ORCH_URL = "http://orchestrator:9000"
|
||||
|
||||
class TestDockerGateway(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
# Redirect the app-data root so ensure_running's host-dir mkdirs (CA +
|
||||
# DB) land in a throwaway dir, not the real ~/.bot-bottle.
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
self.addCleanup(use_bottle_root(Path(self._tmp.name)))
|
||||
# Resolver-only data plane (PRD 0070): running the gateway requires an
|
||||
# orchestrator URL, so the fixture supplies one.
|
||||
self.sc = DockerGateway("bot-bottle-gateway:latest", orchestrator_url=_ORCH_URL)
|
||||
@@ -103,8 +113,17 @@ class TestDockerGateway(unittest.TestCase):
|
||||
self.assertIn("bot-bottle-gateway:latest", runs[0])
|
||||
# Runs on the shared gateway network so agents can reach it by IP.
|
||||
self.assertEqual(self.sc.network, runs[0][runs[0].index("--network") + 1])
|
||||
# Persists its CA on a named volume so agents keep trusting it.
|
||||
self.assertTrue(any("mitmproxy" in a for a in runs[0]))
|
||||
# Persists its CA on a HOST bind-mount (not a docker named volume, which
|
||||
# `docker volume prune` would wipe — issue #450) so agents keep trusting
|
||||
# it across restarts. The mount source is the host gateway-CA dir.
|
||||
ca_mounts = [
|
||||
a for a in runs[0]
|
||||
if a.endswith(":/home/mitmproxy/.mitmproxy")
|
||||
]
|
||||
self.assertEqual(1, len(ca_mounts))
|
||||
src = ca_mounts[0].rsplit(":", 1)[0]
|
||||
self.assertTrue(src.endswith("/" + GATEWAY_CA_DIRNAME), src)
|
||||
self.assertTrue(Path(src).is_absolute(), src)
|
||||
# Shares the ONE host DB: the supervise daemon queues into the same
|
||||
# file the orchestrator + operator (over HTTP) use.
|
||||
self.assertTrue(any(
|
||||
@@ -234,5 +253,49 @@ class TestDockerGatewayBuild(unittest.TestCase):
|
||||
self.sc.ensure_built()
|
||||
|
||||
|
||||
class TestRotateGatewayCa(unittest.TestCase):
|
||||
"""rotate_gateway_ca clears the persisted CA so the next start remints it."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
self.addCleanup(use_bottle_root(Path(self._tmp.name)))
|
||||
|
||||
def _seed_ca(self) -> Path:
|
||||
ca_dir = host_gateway_ca_dir()
|
||||
# A representative mitmproxy confdir: the CA identity + derived encodings,
|
||||
# plus one non-CA file that rotation must leave untouched.
|
||||
for name in (
|
||||
"mitmproxy-ca.pem",
|
||||
"mitmproxy-ca-cert.pem",
|
||||
"mitmproxy-ca-cert.cer",
|
||||
"mitmproxy-ca-cert.p12",
|
||||
):
|
||||
(ca_dir / name).write_text("x")
|
||||
(ca_dir / "combined-trust.pem").write_text("keep")
|
||||
return ca_dir
|
||||
|
||||
def test_removes_ca_material_only(self) -> None:
|
||||
ca_dir = self._seed_ca()
|
||||
removed = rotate_gateway_ca(ca_dir)
|
||||
self.assertEqual(4, len(removed))
|
||||
self.assertTrue(all(p.name.startswith("mitmproxy-ca") for p in removed))
|
||||
# The CA files are gone; the non-CA trust bundle survives.
|
||||
self.assertEqual(
|
||||
{"combined-trust.pem"}, {p.name for p in ca_dir.iterdir()}
|
||||
)
|
||||
|
||||
def test_defaults_to_host_ca_dir(self) -> None:
|
||||
self._seed_ca()
|
||||
removed = rotate_gateway_ca() # no arg → host_gateway_ca_dir()
|
||||
self.assertTrue(removed)
|
||||
self.assertEqual(
|
||||
[], list(host_gateway_ca_dir().glob("mitmproxy-ca*"))
|
||||
)
|
||||
|
||||
def test_idempotent_when_no_ca(self) -> None:
|
||||
self.assertEqual([], rotate_gateway_ca(host_gateway_ca_dir()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -16,6 +16,7 @@ from bot_bottle.orchestrator.lifecycle import (
|
||||
OrchestratorStartError,
|
||||
source_hash,
|
||||
)
|
||||
from bot_bottle.paths import GATEWAY_CA_DIRNAME
|
||||
from tests.unit import use_bottle_root
|
||||
|
||||
_URLOPEN = "bot_bottle.orchestrator.lifecycle.urllib.request.urlopen"
|
||||
@@ -115,6 +116,14 @@ class TestOrchestratorService(unittest.TestCase):
|
||||
# Gateway daemons + orchestrator explicitly opted in.
|
||||
daemons_flag = "BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise,orchestrator"
|
||||
self.assertIn("orchestrator", argv[argv.index(daemons_flag)])
|
||||
# The mitmproxy CA persists on a HOST bind-mount under the app-data root
|
||||
# (not a docker named volume `docker volume prune` would wipe — #450), so
|
||||
# a restarted infra container keeps the CA every running bottle trusts.
|
||||
ca_mounts = [a for a in argv if a.endswith(":/home/mitmproxy/.mitmproxy")]
|
||||
self.assertEqual(1, len(ca_mounts))
|
||||
src = ca_mounts[0].rsplit(":", 1)[0]
|
||||
self.assertTrue(src.startswith(self._tmp.name), src)
|
||||
self.assertTrue(src.endswith("/" + GATEWAY_CA_DIRNAME), src)
|
||||
|
||||
def test_ensure_running_builds_all_images(self) -> None:
|
||||
calls: list[list[str]] = []
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Unit: the `rotate_ca` one-shot CLI (issue #450). Docker mocked."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from bot_bottle.orchestrator import rotate_ca
|
||||
from bot_bottle.orchestrator.gateway import GATEWAY_NAME
|
||||
from bot_bottle.orchestrator.lifecycle import INFRA_NAME
|
||||
from bot_bottle.paths import host_gateway_ca_dir
|
||||
from tests.unit import use_bottle_root
|
||||
|
||||
_RUN = "bot_bottle.orchestrator.rotate_ca.run_docker"
|
||||
|
||||
|
||||
def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock:
|
||||
return Mock(returncode=returncode, stdout=stdout, stderr=stderr)
|
||||
|
||||
|
||||
class TestRotateCaCli(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self._tmp.cleanup)
|
||||
self.addCleanup(use_bottle_root(Path(self._tmp.name)))
|
||||
|
||||
def test_clears_ca_and_drops_gateway_containers(self) -> None:
|
||||
ca_dir = host_gateway_ca_dir()
|
||||
(ca_dir / "mitmproxy-ca.pem").write_text("x")
|
||||
(ca_dir / "mitmproxy-ca-cert.pem").write_text("x")
|
||||
|
||||
calls: list[list[str]] = []
|
||||
|
||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||
calls.append(argv)
|
||||
# Report a removed container name so the CLI logs it.
|
||||
return _proc(stdout=argv[-1])
|
||||
|
||||
with patch(_RUN, side_effect=fake):
|
||||
self.assertEqual(0, rotate_ca.main([]))
|
||||
|
||||
# Persisted CA is gone → next start remints it.
|
||||
self.assertEqual([], list(ca_dir.glob("mitmproxy-ca*")))
|
||||
# Both the infra container and the standalone gateway are force-removed
|
||||
# so no mitmproxy keeps serving the old CA from memory.
|
||||
removed = {c[-1] for c in calls if c[:3] == ["docker", "rm", "--force"]}
|
||||
self.assertEqual({INFRA_NAME, GATEWAY_NAME}, removed)
|
||||
|
||||
def test_succeeds_with_no_persisted_ca(self) -> None:
|
||||
with patch(_RUN, return_value=_proc()) as m:
|
||||
self.assertEqual(0, rotate_ca.main([]))
|
||||
# Still tears down any running gateway even when there was no CA on disk.
|
||||
self.assertTrue(m.called)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -136,6 +136,49 @@ class TestPublishBundle(unittest.TestCase):
|
||||
self.assertIn("registry unreachable", str(ctx.exception))
|
||||
|
||||
|
||||
class TestTryDownloadPublished(unittest.TestCase):
|
||||
def test_downloads_existing_artifact(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as d, \
|
||||
mock.patch.object(pub.infra_vm, "_infra_init", return_value="init"), \
|
||||
mock.patch.object(
|
||||
pub.infra_artifact, "infra_artifact_version", return_value="v1"
|
||||
), mock.patch.object(
|
||||
pub.urllib.request, "urlopen", return_value=_Resp()
|
||||
), mock.patch.object(pub.infra_artifact, "_download") as download:
|
||||
root = Path(d)
|
||||
result = pub._try_download_published(root)
|
||||
|
||||
self.assertEqual(
|
||||
("v1", root / "rootfs.ext4.gz", root / "rootfs.ext4.gz.sha256"),
|
||||
result,
|
||||
)
|
||||
self.assertEqual(2, download.call_count)
|
||||
|
||||
def test_missing_artifact_returns_none(self) -> None:
|
||||
missing = urllib.error.HTTPError("u", 404, "missing", Message(), None)
|
||||
with tempfile.TemporaryDirectory() as d, mock.patch.object(
|
||||
pub.urllib.request, "urlopen", side_effect=missing
|
||||
):
|
||||
self.assertIsNone(pub._try_download_published(Path(d)))
|
||||
|
||||
def test_registry_http_failure_is_reported(self) -> None:
|
||||
failure = urllib.error.HTTPError("u", 500, "failed", Message(), None)
|
||||
with tempfile.TemporaryDirectory() as d, mock.patch.object(
|
||||
pub.urllib.request, "urlopen", side_effect=failure
|
||||
):
|
||||
with self.assertRaises(SystemExit) as ctx:
|
||||
pub._try_download_published(Path(d))
|
||||
self.assertIn("registry check failed (HTTP 500)", str(ctx.exception))
|
||||
|
||||
def test_registry_connection_failure_is_reported(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as d, mock.patch.object(
|
||||
pub.urllib.request, "urlopen", side_effect=urllib.error.URLError("offline")
|
||||
):
|
||||
with self.assertRaises(SystemExit) as ctx:
|
||||
pub._try_download_published(Path(d))
|
||||
self.assertIn("registry unreachable", str(ctx.exception))
|
||||
|
||||
|
||||
class TestMain(unittest.TestCase):
|
||||
def test_output_builds_candidate_and_records_version(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
@@ -147,6 +190,20 @@ class TestMain(unittest.TestCase):
|
||||
build.assert_called_once_with(root)
|
||||
self.assertEqual("v1\n", (root / "version.txt").read_text())
|
||||
|
||||
def test_output_reuses_published_candidate(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
root = Path(d) / "candidate"
|
||||
reused = ("v1", root / "rootfs.ext4.gz", root / "rootfs.ext4.gz.sha256")
|
||||
with mock.patch.object(
|
||||
pub, "_try_download_published", return_value=reused
|
||||
) as reuse, mock.patch.object(pub, "build_artifact") as build:
|
||||
self.assertEqual(
|
||||
0, pub.main(["--output", str(root), "--reuse-published"])
|
||||
)
|
||||
reuse.assert_called_once_with(root)
|
||||
build.assert_not_called()
|
||||
self.assertEqual("v1\n", (root / "version.txt").read_text())
|
||||
|
||||
def test_publish_dir_publishes_existing_candidate(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as d, \
|
||||
mock.patch.object(pub.infra_artifact, "_config", return_value=("", "", "t")), \
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Unit: stale-image check functions across backends, and the
|
||||
BottleBackend.launch template method (prelaunch_checks + build_or_load_images).
|
||||
|
||||
No real images or containers are used — all Docker/container/smolmachine
|
||||
calls are mocked at the module boundary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
def _bottle_cm(bottle: Any) -> MagicMock:
|
||||
"""Return a mock context manager that yields `bottle`."""
|
||||
cm = MagicMock()
|
||||
cm.__enter__ = MagicMock(return_value=bottle)
|
||||
cm.__exit__ = MagicMock(return_value=False)
|
||||
return cm
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# BottleBackend.launch template — prelaunch_checks + _build_or_load_images
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestBottleBackendLaunchTemplate(unittest.TestCase):
|
||||
"""Verify the concrete launch() method on BottleBackend calls
|
||||
_build_or_load_images and _launch_impl, and that prelaunch_checks is a no-op
|
||||
on the base class."""
|
||||
|
||||
def _make_backend(self) -> Any:
|
||||
from bot_bottle.backend.docker.backend import DockerBottleBackend
|
||||
return DockerBottleBackend()
|
||||
|
||||
def test_launch_delegates_to_build_or_load_and_launch_impl(self) -> None:
|
||||
from bot_bottle.backend import BottleImages
|
||||
backend = self._make_backend()
|
||||
plan = cast(Any, SimpleNamespace())
|
||||
bottle = MagicMock()
|
||||
images = BottleImages(agent="agent:latest", sidecar="sidecar:latest")
|
||||
with patch.object(
|
||||
backend, "_build_or_load_images", return_value=images,
|
||||
) as build_mock, patch.object(
|
||||
backend, "_launch_impl",
|
||||
return_value=_bottle_cm(bottle),
|
||||
) as impl_mock:
|
||||
with backend.launch(plan):
|
||||
pass
|
||||
build_mock.assert_called_once_with(plan)
|
||||
impl_mock.assert_called_once_with(plan, images)
|
||||
|
||||
def test_noop_default_prelaunch_checks(self) -> None:
|
||||
from bot_bottle.backend.docker.backend import DockerBottleBackend
|
||||
from bot_bottle.backend import BottleBackend
|
||||
backend = DockerBottleBackend()
|
||||
# Base-class prelaunch_checks is a no-op — must not raise.
|
||||
BottleBackend.prelaunch_checks(backend, cast(Any, SimpleNamespace())) # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Docker backend stale_checks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestDockerStaleChecks(unittest.TestCase):
|
||||
def _plan(self, policy: str = "cached", slug: str = "dev-abc") -> Any:
|
||||
spec = SimpleNamespace(image_policy=policy)
|
||||
provision = SimpleNamespace(image="bot-bottle-agent:latest")
|
||||
return cast(Any, SimpleNamespace(
|
||||
spec=spec,
|
||||
slug=slug,
|
||||
image="bot-bottle-agent:latest",
|
||||
agent_provision=provision,
|
||||
))
|
||||
|
||||
def test_fresh_policy_is_noop(self) -> None:
|
||||
from bot_bottle.backend.docker import launch as mod
|
||||
with patch.object(mod, "read_committed_image") as rci, \
|
||||
patch.object(mod, "check_stale") as cs:
|
||||
mod.stale_checks(self._plan("fresh"))
|
||||
rci.assert_not_called()
|
||||
cs.assert_not_called()
|
||||
|
||||
def test_committed_image_present_checks_only_committed(self) -> None:
|
||||
from bot_bottle.backend.docker import launch as mod
|
||||
from datetime import datetime, timezone
|
||||
ts = datetime(2025, 1, 1, tzinfo=timezone.utc)
|
||||
with patch.object(mod, "read_committed_image", return_value="committed:latest"), \
|
||||
patch.object(mod.docker_mod, "image_exists", return_value=True), \
|
||||
patch.object(mod.docker_mod, "image_created_at", return_value=ts), \
|
||||
patch.object(mod, "check_stale") as cs:
|
||||
mod.stale_checks(self._plan())
|
||||
cs.assert_called_once()
|
||||
self.assertIn("committed:latest", cs.call_args.args[0])
|
||||
|
||||
def test_no_committed_image_checks_agent(self) -> None:
|
||||
from bot_bottle.backend.docker import launch as mod
|
||||
from datetime import datetime, timezone
|
||||
ts = datetime(2025, 1, 1, tzinfo=timezone.utc)
|
||||
plan = self._plan()
|
||||
with patch.object(mod, "read_committed_image", return_value=""), \
|
||||
patch.object(mod.docker_mod, "image_exists", return_value=True), \
|
||||
patch.object(mod.docker_mod, "image_created_at", return_value=ts), \
|
||||
patch.object(mod, "check_stale") as cs:
|
||||
mod.stale_checks(plan)
|
||||
cs.assert_called_once()
|
||||
self.assertIn(plan.image, cs.call_args.args[0])
|
||||
|
||||
def test_image_not_present_skips_check(self) -> None:
|
||||
from bot_bottle.backend.docker import launch as mod
|
||||
with patch.object(mod, "read_committed_image", return_value=""), \
|
||||
patch.object(mod.docker_mod, "image_exists", return_value=False), \
|
||||
patch.object(mod, "check_stale") as cs:
|
||||
mod.stale_checks(self._plan())
|
||||
cs.assert_not_called()
|
||||
|
||||
def test_only_sidecar_missing_checks_only_agent(self) -> None:
|
||||
from bot_bottle.backend.docker import launch as mod
|
||||
from datetime import datetime, timezone
|
||||
ts = datetime(2025, 1, 1, tzinfo=timezone.utc)
|
||||
plan = self._plan()
|
||||
|
||||
def image_exists(ref: str) -> bool:
|
||||
return ref == plan.image # Only agent present; sidecar missing.
|
||||
|
||||
with patch.object(mod, "read_committed_image", return_value=""), \
|
||||
patch.object(mod.docker_mod, "image_exists", side_effect=image_exists), \
|
||||
patch.object(mod.docker_mod, "image_created_at", return_value=ts), \
|
||||
patch.object(mod, "check_stale") as cs:
|
||||
mod.stale_checks(plan)
|
||||
self.assertEqual(1, cs.call_count)
|
||||
self.assertIn(plan.image, cs.call_args.args[0])
|
||||
|
||||
def test_backend_prelaunch_checks_delegates(self) -> None:
|
||||
from bot_bottle.backend.docker.backend import DockerBottleBackend
|
||||
from bot_bottle.backend.docker import launch as mod
|
||||
backend = DockerBottleBackend()
|
||||
plan = self._plan()
|
||||
with patch.object(mod, "stale_checks") as sc:
|
||||
backend.prelaunch_checks(plan) # type: ignore[arg-type]
|
||||
sc.assert_called_once_with(plan)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# macOS container backend stale_checks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestMacosContainerStaleChecks(unittest.TestCase):
|
||||
def _plan(self, policy: str = "cached", slug: str = "dev-abc") -> Any:
|
||||
return cast(Any, SimpleNamespace(
|
||||
spec=SimpleNamespace(image_policy=policy),
|
||||
slug=slug,
|
||||
image="bot-bottle-agent:latest",
|
||||
))
|
||||
|
||||
def test_fresh_policy_is_noop(self) -> None:
|
||||
from bot_bottle.backend.macos_container import launch as mod
|
||||
with patch.object(mod, "read_committed_image") as rci, \
|
||||
patch.object(mod, "check_stale") as cs:
|
||||
mod.stale_checks(self._plan("fresh"))
|
||||
rci.assert_not_called()
|
||||
cs.assert_not_called()
|
||||
|
||||
def test_committed_image_present_checks_only_committed(self) -> None:
|
||||
from bot_bottle.backend.macos_container import launch as mod
|
||||
from datetime import datetime, timezone
|
||||
ts = datetime(2025, 1, 1, tzinfo=timezone.utc)
|
||||
with patch.object(mod, "read_committed_image", return_value="committed:latest"), \
|
||||
patch.object(mod.container_mod, "image_exists", return_value=True), \
|
||||
patch.object(mod.container_mod, "image_created_at", return_value=ts), \
|
||||
patch.object(mod, "check_stale") as cs:
|
||||
mod.stale_checks(self._plan())
|
||||
cs.assert_called_once()
|
||||
self.assertIn("committed:latest", cs.call_args.args[0])
|
||||
|
||||
def test_no_committed_image_checks_agent(self) -> None:
|
||||
from bot_bottle.backend.macos_container import launch as mod
|
||||
from datetime import datetime, timezone
|
||||
ts = datetime(2025, 1, 1, tzinfo=timezone.utc)
|
||||
plan = self._plan()
|
||||
with patch.object(mod, "read_committed_image", return_value=""), \
|
||||
patch.object(mod.container_mod, "image_exists", return_value=True), \
|
||||
patch.object(mod.container_mod, "image_created_at", return_value=ts), \
|
||||
patch.object(mod, "check_stale") as cs:
|
||||
mod.stale_checks(plan)
|
||||
cs.assert_called_once()
|
||||
self.assertIn(plan.image, cs.call_args.args[0])
|
||||
|
||||
def test_image_not_present_skips_check(self) -> None:
|
||||
from bot_bottle.backend.macos_container import launch as mod
|
||||
with patch.object(mod, "read_committed_image", return_value=""), \
|
||||
patch.object(mod.container_mod, "image_exists", return_value=False), \
|
||||
patch.object(mod, "check_stale") as cs:
|
||||
mod.stale_checks(self._plan())
|
||||
cs.assert_not_called()
|
||||
|
||||
def test_backend_prelaunch_checks_delegates(self) -> None:
|
||||
from bot_bottle.backend.macos_container.backend import MacosContainerBottleBackend
|
||||
from bot_bottle.backend.macos_container import launch as mod
|
||||
backend = MacosContainerBottleBackend()
|
||||
plan = self._plan()
|
||||
with patch.object(mod, "stale_checks") as sc:
|
||||
backend.prelaunch_checks(plan) # type: ignore[arg-type]
|
||||
sc.assert_called_once_with(plan)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Firecracker cached rootfs selection and stale checks
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestFirecrackerCachedRootfs(unittest.TestCase):
|
||||
def _plan(self, policy: str = "cached") -> Any:
|
||||
return cast(Any, SimpleNamespace(
|
||||
spec=SimpleNamespace(image_policy=policy),
|
||||
slug="dev-abc",
|
||||
image="bot-bottle-agent:latest",
|
||||
dockerfile_path="/repo/Dockerfile",
|
||||
agent_provider_template="claude",
|
||||
))
|
||||
|
||||
def test_cached_policy_reuses_ready_rootfs_without_building(self) -> None:
|
||||
from bot_bottle.backend.firecracker import launch as mod
|
||||
cached = Path("/cache/rootfs/agent-deadbeef")
|
||||
with patch.object(mod, "read_committed_image", return_value=""), \
|
||||
patch.object(
|
||||
mod.image_builder, "cached_agent_rootfs_dir", return_value=cached,
|
||||
), \
|
||||
patch.object(mod.image_builder, "build_agent_rootfs_dir") as build:
|
||||
self.assertEqual(cached, mod.build_or_load_agent_base(self._plan()))
|
||||
build.assert_not_called()
|
||||
|
||||
def test_cached_policy_fails_when_rootfs_is_missing(self) -> None:
|
||||
from bot_bottle.backend.firecracker import launch as mod
|
||||
with patch.object(mod, "read_committed_image", return_value=""), \
|
||||
patch.object(
|
||||
mod.image_builder, "cached_agent_rootfs_dir", return_value=None,
|
||||
), \
|
||||
patch.object(mod.image_builder, "build_agent_rootfs_dir") as build, \
|
||||
self.assertRaises(SystemExit):
|
||||
mod.build_or_load_agent_base(self._plan())
|
||||
build.assert_not_called()
|
||||
|
||||
def test_stale_checks_use_ready_marker_timestamp(self) -> None:
|
||||
from bot_bottle.backend.firecracker import launch as mod
|
||||
cached = Path("/cache/rootfs/agent-deadbeef")
|
||||
with patch.object(mod, "read_committed_image", return_value=""), \
|
||||
patch.object(
|
||||
mod.image_builder, "cached_agent_rootfs_dir", return_value=cached,
|
||||
), \
|
||||
patch.object(mod, "check_stale_path") as check:
|
||||
mod.stale_checks(self._plan())
|
||||
check.assert_called_once_with(f"agent rootfs {cached}", cached / ".bb-ready")
|
||||
|
||||
def test_backend_prelaunch_checks_delegates(self) -> None:
|
||||
from bot_bottle.backend.firecracker.backend import FirecrackerBottleBackend
|
||||
from bot_bottle.backend.firecracker import launch as mod
|
||||
plan = self._plan()
|
||||
with patch.object(mod, "stale_checks") as checks:
|
||||
FirecrackerBottleBackend().prelaunch_checks(plan)
|
||||
checks.assert_called_once_with(plan)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user