Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c527841d55 | |||
| 5940b75bb7 |
@@ -1,6 +1,10 @@
|
||||
[run]
|
||||
branch = True
|
||||
source = .
|
||||
# Store paths relative to the project root so .coverage.* files produced on
|
||||
# different runners (ubuntu-latest vs self-hosted KVM) can be combined by the
|
||||
# coverage job without a [paths] remapping section.
|
||||
relative_files = True
|
||||
|
||||
[report]
|
||||
# Coverage policy: see docs/decisions/0004-coverage-policy.md.
|
||||
|
||||
+99
-112
@@ -9,10 +9,12 @@
|
||||
# tests/canaries/ — upstream regression canaries; run on a separate
|
||||
# schedule (see canaries.yml), not here
|
||||
#
|
||||
# Integration tests run once per backend in separate jobs. Each job sets
|
||||
# BOT_BOTTLE_BACKEND explicitly so the test suite uses the right backend.
|
||||
# Backends that aren't available on the runner fail the preflight step
|
||||
# rather than silently skipping inside the test output.
|
||||
# Each test job runs once under coverage and uploads a small .coverage.*
|
||||
# artifact. The `coverage` job combines them — no test reruns, no KVM
|
||||
# dependency on that job. For main-branch pushes only, the tested rootfs
|
||||
# and matching dropbear are uploaded so `publish-infra` can publish the
|
||||
# byte-identical artifact that was tested. PRs avoid the ~194 MB rootfs
|
||||
# transfer entirely.
|
||||
|
||||
name: test
|
||||
|
||||
@@ -40,53 +42,6 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
stage-firecracker-inputs:
|
||||
runs-on: [self-hosted, kvm]
|
||||
# Same guard as the other KVM-runner jobs: don't spin the privileged
|
||||
# runner for fork PRs (this only copies a non-secret static binary, but
|
||||
# keep the posture consistent — build-infra/integration/coverage all
|
||||
# depend on it, so gating here gates the whole Firecracker chain).
|
||||
if: >-
|
||||
github.event_name == 'push' ||
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'pull_request' &&
|
||||
github.event.pull_request.head.repo.full_name == github.repository)
|
||||
steps:
|
||||
- name: Stage the provisioned static dropbear
|
||||
run: |
|
||||
mkdir -p firecracker-inputs
|
||||
cp /var/cache/bot-bottle-fc/dropbear firecracker-inputs/dropbear
|
||||
|
||||
- name: Upload Firecracker build inputs
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: firecracker-inputs/
|
||||
|
||||
build-infra:
|
||||
needs: stage-firecracker-inputs
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download Firecracker build inputs
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: firecracker-inputs
|
||||
|
||||
- name: Build infra candidate from this checkout
|
||||
env:
|
||||
BOT_BOTTLE_FC_DROPBEAR: ${{ github.workspace }}/firecracker-inputs/dropbear
|
||||
run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate
|
||||
|
||||
- name: Upload infra candidate
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate/
|
||||
|
||||
unit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -101,11 +56,17 @@ jobs:
|
||||
- name: Install dev requirements
|
||||
run: python3 -m pip install --break-system-packages -r requirements-dev.txt
|
||||
|
||||
- name: Run unit tests
|
||||
run: python3 -m coverage run -m unittest discover -t . -s tests/unit -v
|
||||
- name: Run unit tests with coverage
|
||||
run: python3 -m coverage run --data-file=.coverage.unit -m unittest discover -t . -s tests/unit -v
|
||||
|
||||
- name: Report unit coverage
|
||||
run: python3 -m coverage report -m
|
||||
run: python3 -m coverage report --data-file=.coverage.unit -m
|
||||
|
||||
- name: Upload unit coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-unit
|
||||
path: ${{ github.workspace }}/.coverage.unit
|
||||
|
||||
integration-docker:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -115,6 +76,9 @@ jobs:
|
||||
|
||||
# No actions/setup-python (see the note in the `unit` job); the
|
||||
# container's system Python 3.12 runs the stdlib test suite directly.
|
||||
- name: Install coverage
|
||||
run: python3 -m pip install --break-system-packages coverage
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
python3 --version
|
||||
@@ -124,10 +88,16 @@ jobs:
|
||||
echo "docker not on PATH — integration tests will skip"
|
||||
fi
|
||||
|
||||
- name: Run integration tests (docker)
|
||||
- name: Run integration tests (docker) with coverage
|
||||
env:
|
||||
BOT_BOTTLE_BACKEND: docker
|
||||
run: python3 -m unittest discover -t . -s tests/integration -v
|
||||
run: python3 -m coverage run --data-file=.coverage.docker -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
- name: Upload docker coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-docker
|
||||
path: ${{ github.workspace }}/.coverage.docker
|
||||
|
||||
# Integration tests against the Firecracker backend. Runs on a self-hosted
|
||||
# KVM runner (label `kvm`) where /dev/kvm and the TAP/nft pool are available.
|
||||
@@ -137,9 +107,16 @@ jobs:
|
||||
#
|
||||
# Runner prerequisites (provision once; see README "Firecracker on Linux"):
|
||||
# `firecracker` on PATH, `/dev/kvm` accessible, cached kernel +
|
||||
# static dropbear, and the pool as a persistent systemd unit.
|
||||
# static dropbear at /var/cache/bot-bottle-fc/dropbear, and the pool as a
|
||||
# persistent systemd unit.
|
||||
#
|
||||
# The infra candidate is built here directly (no artifact download) to
|
||||
# eliminate the ~70 s ubuntu-latest upload + ~83 s combined download that
|
||||
# the old build-infra → integration-firecracker + coverage chain incurred.
|
||||
# For main-branch pushes the tested rootfs and matching dropbear are
|
||||
# uploaded so publish-infra can publish the byte-identical artifact; PRs
|
||||
# skip those uploads entirely.
|
||||
integration-firecracker:
|
||||
needs: build-infra
|
||||
runs-on: [self-hosted, kvm]
|
||||
if: >-
|
||||
github.event_name == 'push' ||
|
||||
@@ -159,49 +136,58 @@ jobs:
|
||||
# range overlap; it prints the exact `backend setup` fix.
|
||||
python3 cli.py backend status --backend=firecracker
|
||||
|
||||
- name: Download the candidate built from this checkout
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate
|
||||
- name: Build infra candidate from this checkout
|
||||
env:
|
||||
BOT_BOTTLE_FC_DROPBEAR: /var/cache/bot-bottle-fc/dropbear
|
||||
run: python3 -m bot_bottle.backend.firecracker.publish_infra --output infra-candidate
|
||||
|
||||
- name: Replace the persistent infra VM with the candidate
|
||||
run: python3 -c 'from bot_bottle.backend.firecracker import infra_vm; infra_vm.stop()'
|
||||
|
||||
# No dev-requirements install: the integration suite runs on stdlib
|
||||
# `unittest` (pylint/pyright are lint.yml's concern, not this job's),
|
||||
# and the self-hosted runner's Nix python env has no `pip` module
|
||||
# (`python3 -m pip` → "No module named pip"). Nothing to install.
|
||||
- name: Run integration tests (firecracker)
|
||||
# No dev-requirements install: `coverage` is already provided by the
|
||||
# self-hosted runner's Nix python env, and that env has no `pip`
|
||||
# module to install into anyway.
|
||||
- name: Run integration tests (firecracker) with coverage
|
||||
env:
|
||||
BOT_BOTTLE_BACKEND: firecracker
|
||||
BOT_BOTTLE_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
|
||||
run: python3 -m unittest discover -t . -s tests/integration -v
|
||||
run: python3 -m coverage run --data-file=.coverage.firecracker -m unittest discover -t . -s tests/integration -v
|
||||
|
||||
# Combined unit+integration coverage + the diff-coverage gate (the hard
|
||||
# gate: new/changed lines >= 90%). See docs/decisions/0004-coverage-policy.md.
|
||||
- name: Upload firecracker coverage artifact
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: coverage-firecracker
|
||||
path: ${{ github.workspace }}/.coverage.firecracker
|
||||
|
||||
# Only upload the large rootfs artifact on main-branch pushes;
|
||||
# PRs avoid the ~194 MB transfer. publish-infra only runs on main
|
||||
# and downloads these to publish the byte-identical tested rootfs.
|
||||
- name: Upload tested rootfs (main branch only)
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate/
|
||||
|
||||
- name: Upload dropbear for publish verification (main branch only)
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
path: /var/cache/bot-bottle-fc/dropbear
|
||||
|
||||
# Combined coverage gate: aggregates .coverage.* artifacts uploaded by each
|
||||
# test job, then runs the diff-coverage gate (new/changed lines >= 90%).
|
||||
#
|
||||
# This runs on a self-hosted KVM runner (label `kvm`), NOT ubuntu-latest,
|
||||
# because the Firecracker backend's subprocess/VM orchestration
|
||||
# (launch/boot/SSH/isolation-probe) is covered by the integration suite,
|
||||
# and that suite needs `/dev/kvm` + the provisioned TAP/nft pool — which a
|
||||
# container-based runner doesn't have. On such a runner the firecracker
|
||||
# integration test skips and its ~230 orchestration lines read as
|
||||
# uncovered, so the gate can't pass there.
|
||||
# Runs on ubuntu-latest — no KVM needed, no test reruns. Coverage files use
|
||||
# relative_files = True (.coveragerc) so they combine cleanly across runners.
|
||||
#
|
||||
# Restricted to the same events as integration-firecracker (same-repo PRs,
|
||||
# push, workflow_dispatch) for the same security reason.
|
||||
#
|
||||
# See #414 for the planned follow-up: artifact-based coverage combination
|
||||
# (run tests once in their respective jobs, combine .coverage files here).
|
||||
#
|
||||
# build-infra creates one candidate from the checkout. This job boots that
|
||||
# same candidate after integration-firecracker has exercised it; the main
|
||||
# push path publishes the identical bytes only after every required job.
|
||||
# Restricted to the same events as integration-firecracker: it depends on
|
||||
# that job's coverage artifact and skips for fork PRs alongside it.
|
||||
coverage:
|
||||
needs: [build-infra, integration-firecracker]
|
||||
needs: [unit, integration-docker, integration-firecracker]
|
||||
timeout-minutes: 15
|
||||
runs-on: [self-hosted, kvm]
|
||||
runs-on: ubuntu-latest
|
||||
if: >-
|
||||
github.event_name == 'push' ||
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
@@ -213,29 +199,29 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Preflight — Firecracker host is ready
|
||||
run: |
|
||||
command -v firecracker >/dev/null || {
|
||||
echo "firecracker not on PATH — provision the runner (README: Firecracker on Linux)"; exit 1; }
|
||||
test -e /dev/kvm || { echo "/dev/kvm missing — KVM not available on this runner"; exit 1; }
|
||||
# `backend status` exits non-zero unless the TAP pool is up + no
|
||||
# range overlap; it prints the exact `backend setup` fix.
|
||||
python3 cli.py backend status --backend=firecracker
|
||||
- name: Install coverage
|
||||
run: python3 -m pip install --break-system-packages coverage
|
||||
|
||||
- name: Download the candidate already exercised by integration
|
||||
- name: Download unit coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
path: infra-candidate
|
||||
name: coverage-unit
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
- name: Download docker coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: coverage-docker
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
- name: Download firecracker coverage artifact
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: coverage-firecracker
|
||||
path: ${{ github.workspace }}
|
||||
|
||||
# No dev-requirements install: `coverage` is already provided by the
|
||||
# self-hosted runner's Nix python env, and that env has no `pip`
|
||||
# module to install into anyway. `scripts/coverage.sh` +
|
||||
# `diff_coverage.py` need only `coverage` (not pylint/pyright).
|
||||
- name: Combined coverage (unit + integration, incl. firecracker)
|
||||
env:
|
||||
BOT_BOTTLE_CI_INFRA_ARTIFACT_DIR: ${{ github.workspace }}/infra-candidate
|
||||
run: PYTHON=python3 bash scripts/coverage.sh critical
|
||||
run: PYTHON=python3 bash scripts/coverage.sh aggregate critical
|
||||
|
||||
- name: Diff-coverage gate (changed lines >= 90%)
|
||||
run: |
|
||||
@@ -243,14 +229,14 @@ jobs:
|
||||
python3 scripts/diff_coverage.py --base origin/main --min 90
|
||||
|
||||
publish-infra:
|
||||
needs: [stage-firecracker-inputs, build-infra, unit, integration-docker, integration-firecracker, coverage]
|
||||
needs: [unit, integration-docker, integration-firecracker, coverage]
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
steps:
|
||||
- name: Checkout the tested revision
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download the tested candidate
|
||||
- name: Download the tested rootfs
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: infra-candidate
|
||||
@@ -258,9 +244,10 @@ jobs:
|
||||
|
||||
# publish_infra re-derives the version from the checkout to confirm the
|
||||
# bundle matches before uploading, and the version hashes the dropbear
|
||||
# bytes. Stage the SAME dropbear build-infra used, or the recheck
|
||||
# computes a "<missing>"-dropbear version and rejects the candidate.
|
||||
- name: Download the staged dropbear (matches build-infra's version)
|
||||
# bytes. Download the SAME dropbear integration-firecracker used, or
|
||||
# the recheck computes a "<missing>"-dropbear version and rejects the
|
||||
# candidate.
|
||||
- name: Download the staged dropbear (matches build's version)
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
name: firecracker-inputs
|
||||
|
||||
@@ -19,7 +19,6 @@ from .commit import cmd_commit
|
||||
from .edit import cmd_edit
|
||||
from .info import cmd_info
|
||||
from .init import cmd_init
|
||||
from .login import cmd_login
|
||||
from .resume import cmd_resume
|
||||
from .start import cmd_start
|
||||
from .supervise import cmd_supervise
|
||||
@@ -34,7 +33,6 @@ COMMANDS = {
|
||||
"info": cmd_info,
|
||||
"init": cmd_init,
|
||||
"list": cmd_list,
|
||||
"login": cmd_login,
|
||||
"resume": cmd_resume,
|
||||
"start": cmd_start,
|
||||
"supervise": cmd_supervise,
|
||||
@@ -45,7 +43,7 @@ COMMANDS = {
|
||||
# the host (TAP pool, /dev/kvm, firecracker) and never opens the store, so
|
||||
# gating it on the schema breaks preflight on a fresh CI runner where stdin
|
||||
# isn't a TTY and the migration prompt can't be answered.
|
||||
NO_MIGRATION_COMMANDS = frozenset({"backend", "login"})
|
||||
NO_MIGRATION_COMMANDS = frozenset({"backend"})
|
||||
|
||||
|
||||
def usage() -> None:
|
||||
@@ -58,7 +56,6 @@ def usage() -> None:
|
||||
sys.stderr.write(" info print env, skills, and prompt details for a named agent\n")
|
||||
sys.stderr.write(" init interactively create a new agent and add it to bot-bottle.json\n")
|
||||
sys.stderr.write(" list list available agents or active containers\n")
|
||||
sys.stderr.write(" login register this host with a bot-bottle console\n")
|
||||
sys.stderr.write(
|
||||
" resume re-launch a bottle by its identity "
|
||||
"(continues state from PRD 0016)\n"
|
||||
|
||||
@@ -1,167 +0,0 @@
|
||||
"""bb login — register this host with a bot-bottle console.
|
||||
|
||||
Opens a device-authorization flow against the target console, waits for the
|
||||
operator to approve, then writes access and refresh tokens to
|
||||
~/.bot-bottle/console.json (or $BOT_BOTTLE_ROOT/console.json).
|
||||
|
||||
Usage:
|
||||
bb login [--console-url URL] [--label LABEL]
|
||||
|
||||
Flags:
|
||||
--console-url URL Target console URL (overrides BB_CONSOLE_URL env var)
|
||||
--label LABEL Host label shown in the console (default: hostname)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from ..paths import bot_bottle_root
|
||||
|
||||
_CONSOLE_URL_ENV = "BB_CONSOLE_URL"
|
||||
_POLL_SLEEP = 2 # seconds between polls; matches console's poll_interval default
|
||||
|
||||
|
||||
def _usage() -> None:
|
||||
sys.stderr.write(
|
||||
"usage: bb login [--console-url URL] [--label LABEL]\n"
|
||||
"\n"
|
||||
"Options:\n"
|
||||
" --console-url URL Console base URL (or BB_CONSOLE_URL env var)\n"
|
||||
" --label LABEL Host label shown in the console (default: hostname)\n"
|
||||
)
|
||||
|
||||
|
||||
def _flag(argv: list[str], name: str) -> str | None:
|
||||
for i, arg in enumerate(argv):
|
||||
if arg == name and i + 1 < len(argv):
|
||||
return argv[i + 1]
|
||||
if arg.startswith(f"{name}="):
|
||||
return arg[len(name) + 1:]
|
||||
return None
|
||||
|
||||
|
||||
def _post(url: str, payload: dict) -> dict:
|
||||
data = json.dumps(payload).encode()
|
||||
req = urllib.request.Request(
|
||||
url, data=data, headers={"Content-Type": "application/json"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
return json.loads(resp.read())
|
||||
|
||||
|
||||
def _get(url: str) -> tuple[int, dict]:
|
||||
req = urllib.request.Request(url)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
return resp.status, json.loads(resp.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, {}
|
||||
|
||||
|
||||
def _save_credentials(
|
||||
console_url: str, host_id: str, access_token: str, refresh_token: str
|
||||
) -> Path:
|
||||
path = bot_bottle_root() / "console.json"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
content = (
|
||||
json.dumps(
|
||||
{
|
||||
"url": console_url,
|
||||
"host_id": host_id,
|
||||
"access_token": access_token,
|
||||
"refresh_token": refresh_token,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
fd, tmp_path_str = tempfile.mkstemp(dir=path.parent, prefix=".console-")
|
||||
tmp = Path(tmp_path_str)
|
||||
try:
|
||||
tmp.chmod(0o600)
|
||||
with os.fdopen(fd, "w") as f:
|
||||
f.write(content)
|
||||
os.replace(tmp, path)
|
||||
except OSError:
|
||||
try:
|
||||
tmp.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
return path
|
||||
|
||||
|
||||
def cmd_login(argv: list[str]) -> int:
|
||||
if "--help" in argv or "-h" in argv:
|
||||
_usage()
|
||||
return 0
|
||||
|
||||
console_url = _flag(argv, "--console-url") or os.environ.get(_CONSOLE_URL_ENV)
|
||||
if not console_url:
|
||||
sys.stderr.write(
|
||||
"bb login: --console-url or BB_CONSOLE_URL is required\n"
|
||||
)
|
||||
return 1
|
||||
console_url = console_url.rstrip("/")
|
||||
|
||||
label = _flag(argv, "--label") or socket.gethostname()
|
||||
|
||||
try:
|
||||
resp = _post(f"{console_url}/api/v1/hosts/authorize", {"label": label})
|
||||
except (OSError, ValueError) as exc:
|
||||
sys.stderr.write(f"bb login: failed to start authorization: {exc}\n")
|
||||
return 1
|
||||
|
||||
device_code = resp["device_code"]
|
||||
user_code = resp["user_code"]
|
||||
expires_in = resp.get("expires_in", 300)
|
||||
poll_sleep = max(1, min(int(resp.get("poll_interval", _POLL_SLEEP)), 60))
|
||||
|
||||
sys.stderr.write(
|
||||
f"\nOpen this URL in your browser to authorize this host:\n\n"
|
||||
f" {console_url}/authorize?code={user_code}\n\n"
|
||||
f"Waiting for approval"
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + expires_in
|
||||
while time.monotonic() < deadline:
|
||||
sys.stderr.write(".")
|
||||
sys.stderr.flush()
|
||||
time.sleep(poll_sleep)
|
||||
|
||||
try:
|
||||
code, result = _get(
|
||||
f"{console_url}/api/v1/hosts/authorize/{device_code}"
|
||||
)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
|
||||
if code == 410:
|
||||
break
|
||||
|
||||
st = result.get("status")
|
||||
if st == "approved":
|
||||
sys.stderr.write("\n\nApproved.\n")
|
||||
path = _save_credentials(
|
||||
console_url,
|
||||
result["host_id"],
|
||||
result["access_token"],
|
||||
result["refresh_token"],
|
||||
)
|
||||
sys.stderr.write(f"Credentials saved to {path}\n")
|
||||
return 0
|
||||
if st == "denied":
|
||||
sys.stderr.write("\n\nDenied by operator.\n")
|
||||
return 1
|
||||
|
||||
sys.stderr.write("\n\nAuthorization timed out.\n")
|
||||
return 1
|
||||
@@ -0,0 +1,110 @@
|
||||
# PRD prd-new: CI artifact-based coverage and local Firecracker candidate flow
|
||||
|
||||
- **Status:** Active
|
||||
- **Author:** Claude
|
||||
- **Created:** 2026-07-21
|
||||
- **Issue:** #446
|
||||
|
||||
## Summary
|
||||
|
||||
Restructure the CI test pipeline to run each test suite exactly once, upload
|
||||
small `.coverage.*` artifacts, and combine them in a lightweight aggregation
|
||||
job. Move the infra build onto the KVM runner so the ~194 MB rootfs never
|
||||
crosses the network for PRs. On main-branch pushes, publish the byte-identical
|
||||
rootfs that was tested.
|
||||
|
||||
## Motivation
|
||||
|
||||
The prior pipeline had two redundant costs:
|
||||
|
||||
1. **Duplicate artifact transfers.** `build-infra` (ubuntu-latest) built and
|
||||
uploaded the ~194 MB rootfs; `integration-firecracker` downloaded it; the
|
||||
`coverage` job downloaded it a second time. Combined download overhead: ~83
|
||||
seconds per run, plus the ~70-second upload.
|
||||
|
||||
2. **Duplicate test execution.** `integration-firecracker` ran the Firecracker
|
||||
integration suite; `coverage` ran the entire unit + integration suite again
|
||||
on the same KVM runner to collect coverage data. Every line of Firecracker
|
||||
code was tested twice per CI run.
|
||||
|
||||
## Goals
|
||||
|
||||
- Each test suite (unit, integration-docker, integration-firecracker) executes
|
||||
exactly once per workflow run.
|
||||
- PRs incur no large artifact transfers — the rootfs stays on the KVM runner.
|
||||
- Main-branch pushes publish a byte-for-byte identical rootfs to the one that
|
||||
passed the integration tests.
|
||||
- Concurrent workflow runs cannot cross-publish candidates (naturally enforced
|
||||
by Gitea Actions' per-run artifact scoping).
|
||||
- Failed or cancelled runs block publication (enforced by the `needs:` chain on
|
||||
`publish-infra`).
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Changing test semantics or the coverage policy (ADR 0004).
|
||||
- Removing the KVM runner guard on `integration-firecracker` and `coverage`.
|
||||
- Changing how `publish_infra.py` builds or uploads the rootfs.
|
||||
|
||||
## Design
|
||||
|
||||
### Job graph
|
||||
|
||||
```
|
||||
unit ──────────────────────────────────┐
|
||||
integration-docker ────────────────────┤──► coverage ──► publish-infra (main only)
|
||||
integration-firecracker (KVM) ─────────┘
|
||||
```
|
||||
|
||||
### `unit`
|
||||
|
||||
Unchanged except: `coverage run` writes `--data-file=.coverage.unit`; the file
|
||||
is uploaded as the `coverage-unit` artifact.
|
||||
|
||||
### `integration-docker`
|
||||
|
||||
Adds a `coverage` install step. `coverage run` writes `--data-file=.coverage.docker`;
|
||||
the file is uploaded as `coverage-docker`.
|
||||
|
||||
### `integration-firecracker` (KVM runner)
|
||||
|
||||
Replaces the old `stage-firecracker-inputs` → `build-infra` → download chain:
|
||||
|
||||
1. Builds the infra candidate locally with
|
||||
`BOT_BOTTLE_FC_DROPBEAR=/var/cache/bot-bottle-fc/dropbear`.
|
||||
2. Boots the candidate and runs integration tests with coverage, writing
|
||||
`.coverage.firecracker`.
|
||||
3. Uploads the small `coverage-firecracker` artifact unconditionally.
|
||||
4. On main-branch pushes only, uploads the rootfs as `infra-candidate` and the
|
||||
dropbear as `firecracker-inputs` so `publish-infra` can verify and publish
|
||||
the byte-identical artifact.
|
||||
|
||||
### `coverage`
|
||||
|
||||
Moves from a KVM runner to `ubuntu-latest`. No tests are re-executed:
|
||||
|
||||
1. Downloads `coverage-unit`, `coverage-docker`, and `coverage-firecracker`.
|
||||
2. Runs `scripts/coverage.sh aggregate critical`, which calls
|
||||
`coverage combine` then `coverage report`.
|
||||
3. Runs the diff-coverage gate (`scripts/diff_coverage.py`).
|
||||
|
||||
Coverage files use `relative_files = True` (`.coveragerc`) so they combine
|
||||
cleanly across runners with different absolute workspace paths.
|
||||
|
||||
### `publish-infra`
|
||||
|
||||
Depends on all four predecessor jobs (unchanged gate). Downloads `infra-candidate`
|
||||
and `firecracker-inputs` that were uploaded by `integration-firecracker` on
|
||||
main — the same byte sequence that passed the integration tests.
|
||||
|
||||
### Eliminated jobs
|
||||
|
||||
- `stage-firecracker-inputs`: existed only to copy the dropbear to ubuntu-latest
|
||||
for `build-infra`. No longer needed.
|
||||
- `build-infra`: the infra candidate is now built on the KVM runner in
|
||||
`integration-firecracker`.
|
||||
|
||||
### Script changes
|
||||
|
||||
`scripts/coverage.sh` gains an `aggregate` mode (`coverage.sh aggregate [critical]`)
|
||||
that combines pre-existing `.coverage.*` files instead of re-running tests.
|
||||
The existing run mode (`coverage.sh [critical]`) is preserved for local dev.
|
||||
+28
-8
@@ -1,15 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# Combined unit + integration coverage (see docs/decisions/0004-coverage-policy.md).
|
||||
#
|
||||
# Runs the unit suite, then appends the integration suite (which skips
|
||||
# cleanly when Docker / the backend CLIs are unavailable), and prints one
|
||||
# combined report. The integration suite is what scores the subprocess /
|
||||
# backend orchestration modules, so the number here is the policy's
|
||||
# yardstick — not the unit-only badge.
|
||||
# Two modes:
|
||||
#
|
||||
# Usage:
|
||||
# scripts/coverage.sh # combined report
|
||||
# scripts/coverage.sh critical # also report just the critical modules
|
||||
# scripts/coverage.sh [critical]
|
||||
# Run mode (default, for local dev): executes the unit suite then the
|
||||
# integration suite under coverage and prints a combined report.
|
||||
#
|
||||
# scripts/coverage.sh aggregate [critical]
|
||||
# Aggregate mode (used by CI): combines pre-existing .coverage.* files
|
||||
# produced by individual test jobs and prints a combined report. No tests
|
||||
# are re-executed; no KVM or Docker dependency.
|
||||
#
|
||||
# Pass "critical" as the last argument in either mode to also report just the
|
||||
# critical modules (ADR 0004 target: 90%).
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
@@ -21,6 +25,22 @@ PY="${PYTHON:-python3}"
|
||||
# README "core coverage" badge can't drift; comma-join it for --include.
|
||||
CRITICAL=$(grep -vE '^[[:space:]]*(#|$)' scripts/critical-modules.txt | paste -sd, -)
|
||||
|
||||
if [ "${1:-}" = "aggregate" ]; then
|
||||
# Aggregate mode: combine .coverage.* artifacts already in the workspace.
|
||||
echo "== combining coverage artifacts ==" >&2
|
||||
"$PY" -m coverage combine
|
||||
|
||||
echo "== combined report ==" >&2
|
||||
"$PY" -m coverage report -m
|
||||
|
||||
if [ "${2:-}" = "critical" ]; then
|
||||
echo "== critical modules (ADR 0004 target: 90%) ==" >&2
|
||||
"$PY" -m coverage report --include="$CRITICAL"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Run mode (default): execute both suites under coverage in this process.
|
||||
rm -f .coverage
|
||||
|
||||
echo "== unit ==" >&2
|
||||
|
||||
@@ -1,205 +0,0 @@
|
||||
"""Unit tests for bb login command."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
class TestFlagParsing(unittest.TestCase):
|
||||
def test_console_url_flag(self) -> None:
|
||||
from bot_bottle.cli.login import _flag
|
||||
self.assertEqual(_flag(["--console-url", "http://x"], "--console-url"), "http://x")
|
||||
|
||||
def test_console_url_equals_form(self) -> None:
|
||||
from bot_bottle.cli.login import _flag
|
||||
self.assertEqual(_flag(["--console-url=http://x"], "--console-url"), "http://x")
|
||||
|
||||
def test_label_flag(self) -> None:
|
||||
from bot_bottle.cli.login import _flag
|
||||
self.assertEqual(_flag(["--label", "my-mac"], "--label"), "my-mac")
|
||||
|
||||
def test_missing_flag_returns_none(self) -> None:
|
||||
from bot_bottle.cli.login import _flag
|
||||
self.assertIsNone(_flag([], "--console-url"))
|
||||
|
||||
|
||||
class TestSaveCredentials(unittest.TestCase):
|
||||
def test_writes_json_and_sets_perms(self) -> None:
|
||||
from bot_bottle.cli.login import _save_credentials
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}):
|
||||
path = _save_credentials("http://c", "hid", "at", "rt")
|
||||
self.assertTrue(path.exists())
|
||||
data = json.loads(path.read_text())
|
||||
self.assertEqual(data["url"], "http://c")
|
||||
self.assertEqual(data["host_id"], "hid")
|
||||
self.assertEqual(data["access_token"], "at")
|
||||
self.assertEqual(data["refresh_token"], "rt")
|
||||
self.assertEqual(oct(path.stat().st_mode & 0o777), oct(0o600))
|
||||
|
||||
def test_cleanup_on_write_failure(self) -> None:
|
||||
"""Temp file is removed and no credentials remain if replace fails."""
|
||||
from bot_bottle.cli.login import _save_credentials
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}):
|
||||
with patch("os.replace", side_effect=OSError("disk full")):
|
||||
with self.assertRaises(OSError):
|
||||
_save_credentials("http://c", "hid", "at", "rt")
|
||||
leftovers = [f for f in os.listdir(tmp) if f.startswith(".console-")]
|
||||
self.assertEqual(leftovers, [])
|
||||
|
||||
def test_temp_file_is_private_before_replace(self) -> None:
|
||||
"""Temp file must be 0600 at the moment os.replace is called."""
|
||||
from bot_bottle.cli.login import _save_credentials
|
||||
from pathlib import Path as _Path
|
||||
|
||||
tmp_perms_at_replace: list[int] = []
|
||||
real_replace = os.replace
|
||||
|
||||
def _spy_replace(src: "str | os.PathLike", dst: "str | os.PathLike") -> None:
|
||||
tmp_perms_at_replace.append(_Path(src).stat().st_mode & 0o777)
|
||||
real_replace(src, dst)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}):
|
||||
with patch("os.replace", side_effect=_spy_replace):
|
||||
path = _save_credentials("http://c", "hid", "at", "rt")
|
||||
self.assertEqual(len(tmp_perms_at_replace), 1)
|
||||
self.assertEqual(oct(tmp_perms_at_replace[0]), oct(0o600))
|
||||
self.assertEqual(oct(path.stat().st_mode & 0o777), oct(0o600))
|
||||
|
||||
|
||||
class TestCmdLoginMissingUrl(unittest.TestCase):
|
||||
def test_returns_1_without_url(self) -> None:
|
||||
from bot_bottle.cli.login import cmd_login
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
os.environ.pop("BB_CONSOLE_URL", None)
|
||||
result = cmd_login([])
|
||||
self.assertEqual(result, 1)
|
||||
|
||||
def test_reads_env_var(self) -> None:
|
||||
"""Exits 1 (network error) not because of missing URL when env var is set."""
|
||||
from bot_bottle.cli.login import cmd_login
|
||||
|
||||
def _fail_post(url, payload):
|
||||
raise OSError("connection refused")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.dict(os.environ, {"BB_CONSOLE_URL": "http://localhost:9999", "BOT_BOTTLE_ROOT": tmp}):
|
||||
with patch("bot_bottle.cli.login._post", side_effect=_fail_post):
|
||||
result = cmd_login([])
|
||||
self.assertEqual(result, 1)
|
||||
|
||||
|
||||
class TestCmdLoginFlow(unittest.TestCase):
|
||||
def _run_with_mocks(self, poll_responses: list[dict], tmp: str) -> int:
|
||||
from bot_bottle.cli.login import cmd_login
|
||||
|
||||
start_resp = {
|
||||
"device_code": "dc123",
|
||||
"user_code": "ABC-DEF",
|
||||
"expires_in": 300,
|
||||
"poll_interval": 0,
|
||||
}
|
||||
|
||||
poll_iter = iter(poll_responses)
|
||||
|
||||
def _fake_post(url, payload):
|
||||
return start_resp
|
||||
|
||||
def _fake_get(url):
|
||||
return 200, next(poll_iter, {"status": "pending"})
|
||||
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}):
|
||||
with patch("bot_bottle.cli.login._post", side_effect=_fake_post):
|
||||
with patch("bot_bottle.cli.login._get", side_effect=_fake_get):
|
||||
with patch("time.sleep"):
|
||||
return cmd_login(["--console-url", "http://console"])
|
||||
|
||||
def test_approved_flow_returns_0(self) -> None:
|
||||
approved = {
|
||||
"status": "approved",
|
||||
"host_id": "hid",
|
||||
"access_token": "at",
|
||||
"refresh_token": "rt",
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
result = self._run_with_mocks(
|
||||
[{"status": "pending"}, approved], tmp
|
||||
)
|
||||
self.assertEqual(result, 0)
|
||||
with open(os.path.join(tmp, "console.json")) as f:
|
||||
creds = json.loads(f.read())
|
||||
self.assertEqual(creds["host_id"], "hid")
|
||||
|
||||
def test_denied_flow_returns_1(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
result = self._run_with_mocks([{"status": "denied"}], tmp)
|
||||
self.assertEqual(result, 1)
|
||||
|
||||
def test_timeout_returns_1(self) -> None:
|
||||
from bot_bottle.cli.login import cmd_login
|
||||
|
||||
start_resp = {
|
||||
"device_code": "dc",
|
||||
"user_code": "ZZZ-ZZZ",
|
||||
"expires_in": 0, # already expired; loop never runs
|
||||
"poll_interval": 2,
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}):
|
||||
with patch("bot_bottle.cli.login._post", return_value=start_resp):
|
||||
result = cmd_login(["--console-url", "http://console"])
|
||||
self.assertEqual(result, 1)
|
||||
|
||||
def test_poll_interval_from_server_is_used(self) -> None:
|
||||
"""time.sleep must be called with the server-provided poll_interval."""
|
||||
from bot_bottle.cli.login import cmd_login
|
||||
|
||||
server_interval = 7
|
||||
start_resp = {
|
||||
"device_code": "dc",
|
||||
"user_code": "ABC-DEF",
|
||||
"expires_in": 300,
|
||||
"poll_interval": server_interval,
|
||||
}
|
||||
approved = {
|
||||
"status": "approved",
|
||||
"host_id": "hid",
|
||||
"access_token": "at",
|
||||
"refresh_token": "rt",
|
||||
}
|
||||
poll_iter = iter([{"status": "pending"}, approved])
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_ROOT": tmp}):
|
||||
with patch("bot_bottle.cli.login._post", return_value=start_resp):
|
||||
with patch("bot_bottle.cli.login._get", side_effect=lambda u: (200, next(poll_iter))):
|
||||
with patch("time.sleep") as mock_sleep:
|
||||
result = cmd_login(["--console-url", "http://console"])
|
||||
|
||||
self.assertEqual(result, 0)
|
||||
self.assertTrue(mock_sleep.called)
|
||||
for call in mock_sleep.call_args_list:
|
||||
self.assertEqual(call.args[0], server_interval)
|
||||
|
||||
|
||||
class TestDispatcherRegistration(unittest.TestCase):
|
||||
def test_login_in_commands(self) -> None:
|
||||
from bot_bottle.cli import COMMANDS
|
||||
self.assertIn("login", COMMANDS)
|
||||
|
||||
def test_login_in_no_migration(self) -> None:
|
||||
from bot_bottle.cli import NO_MIGRATION_COMMANDS
|
||||
self.assertIn("login", NO_MIGRATION_COMMANDS)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user