Compare commits

..

2 Commits

Author SHA1 Message Date
didericis d89d389bef ci(coverage): add auto-updated "core coverage" badge
Surface the metric ADR 0004 says matters — the critical security/logic
core, currently 95% — as a README badge, distinct from the
informational global `coverage` badge.

- scripts/critical-modules.txt: single source of truth for the core
  module list. scripts/coverage.sh now reads it (instead of a hardcoded
  string) and update-badges.yml reads the same file, so the badge and
  the `critical` report cannot drift.
- update-badges.yml: a `core coverage` step reuses the unit-coverage
  data (every core module is unit-tested, so unit-only is accurate for
  it) and sed-updates the new badge, like the existing ones.
- README: `core coverage 95%` badge linking to ADR 0004 so a reader can
  find out what "core" means.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NkwFXLFff9PYPy4wgVBJp9
2026-06-26 02:19:35 -04:00
didericis 8caa79ee76 test(supervise): ratchet supervise coverage to >=90%
test / unit (pull_request) Successful in 46s
test / integration (pull_request) Successful in 17s
test / coverage (pull_request) Successful in 58s
lint / lint (push) Successful in 1m54s
Sixth per-module ratchet under ADR 0004. Cover the queue/audit
malformed-input and fallback branches:

- path helpers (bot_bottle_root, queue_dir_for_slug,
  _id_from_proposal_filename non-match)
- read_proposal / read_response reject non-object JSON
- list_pending_proposals skips unreadable/non-dict/incomplete
  proposals and ones with a response already present
- wait_for_response tolerates a malformed or incomplete response file
  and then times out at the deadline
- read_audit_entries returns [] for a missing log and skips blank /
  non-JSON / non-dict / missing-field lines
- the fcntl flock helpers swallow OSError on a bad fd

supervise.py: 89% -> 99%. The one remaining line is an unreachable
`continue` (glob already guarantees the .proposal.json suffix).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01NkwFXLFff9PYPy4wgVBJp9
2026-06-25 22:19:37 -04:00
6 changed files with 183 additions and 9 deletions
+17 -2
View File
@@ -54,11 +54,23 @@ jobs:
echo "percent=$PERCENT" >> $GITHUB_OUTPUT
echo "Coverage: $PERCENT%"
- name: Extract core (critical-module) coverage percentage
id: core_coverage
run: |
# Reuses the .coverage data from the previous step. The core list is
# the single source of truth in scripts/critical-modules.txt; every
# core module is unit-tested, so the unit-only run is accurate for it.
INCLUDE=$(grep -vE '^[[:space:]]*(#|$)' scripts/critical-modules.txt | paste -sd, -)
PERCENT=$(python -m coverage report --include="$INCLUDE" 2>/dev/null | grep '^TOTAL' | grep -oP '\d+(?=%)' | tail -1)
echo "percent=$PERCENT" >> $GITHUB_OUTPUT
echo "Core coverage: $PERCENT%"
- name: Update badges in README
run: |
PYLINT_SCORE="${{ steps.pylint.outputs.score }}"
PYRIGHT_ERRORS="${{ steps.pyright.outputs.errors }}"
COVERAGE_PERCENT="${{ steps.coverage.outputs.percent }}"
CORE_COVERAGE_PERCENT="${{ steps.core_coverage.outputs.percent }}"
PYLINT_SCORE_ENCODED=$(echo "$PYLINT_SCORE" | sed 's|/|%2F|g')
@@ -71,9 +83,12 @@ jobs:
if [ -n "$COVERAGE_PERCENT" ]; then
sed -i "s|/badge/coverage-[^)]*|/badge/coverage-${COVERAGE_PERCENT}%25-brightgreen|" README.md
fi
if [ -n "$CORE_COVERAGE_PERCENT" ]; then
sed -i "s|/badge/core%20coverage-[^)]*|/badge/core%20coverage-${CORE_COVERAGE_PERCENT}%25-brightgreen|" README.md
fi
echo "Updated badges:"
grep -E "pylint|pyright|coverage" README.md | head -3
grep -E "pylint|pyright|coverage" README.md | head -4
- name: Commit and push badge updates
run: |
@@ -86,7 +101,7 @@ jobs:
else
echo "Badge changes detected, committing..."
git add README.md
MSG="chore: update quality badges"$'\n\n'"- Pylint: ${{ steps.pylint.outputs.score }}"$'\n'"- Pyright: ${{ steps.pyright.outputs.errors }} errors"$'\n'"- Coverage: ${{ steps.coverage.outputs.percent }}%"$'\n\n'"[skip ci]"
MSG="chore: update quality badges"$'\n\n'"- Pylint: ${{ steps.pylint.outputs.score }}"$'\n'"- Pyright: ${{ steps.pyright.outputs.errors }} errors"$'\n'"- Coverage: ${{ steps.coverage.outputs.percent }}%"$'\n'"- Core coverage: ${{ steps.core_coverage.outputs.percent }}%"$'\n\n'"[skip ci]"
git commit -m "$MSG"
git push
fi
+1
View File
@@ -8,6 +8,7 @@
[![pylint](https://img.shields.io/badge/pylint-9.93%2F10-brightgreen)](https://github.com/PyCQA/pylint)
[![pyright](https://img.shields.io/badge/pyright-0%20errors-brightgreen)](https://github.com/microsoft/pyright)
[![coverage](https://img.shields.io/badge/coverage-79%25-brightgreen)](https://coverage.readthedocs.io/)
[![core coverage](https://img.shields.io/badge/core%20coverage-95%25-brightgreen)](https://gitea.dideric.is/didericis/bot-bottle/src/branch/main/docs/decisions/0004-coverage-policy.md)
**Problem:** Developer wants to run a coding agent without supervision, but they don't want a prompt injected or misbehaving agent wrecking their environment or exfiltrating sensitive data.
+6
View File
@@ -88,3 +88,9 @@ omit list.
- PRs #290 (cover the egress adapter), and the coverage-policy PR that
introduces this record.
- `.coveragerc`, `scripts/coverage.sh`, `scripts/diff_coverage.py`.
- `scripts/critical-modules.txt` — the single source of truth for the
core-module list; read by both `scripts/coverage.sh` and the
`update-badges.yml` "core coverage" badge so they cannot drift.
- The README carries a `core coverage` badge (auto-updated from that
list) — the headline number, distinct from the informational global
`coverage` badge.
+4 -7
View File
@@ -16,13 +16,10 @@ cd "$(dirname "$0")/.."
PY="${PYTHON:-python3}"
# Critical security/logic core held to the high bar by ADR 0004.
CRITICAL="bot_bottle/egress_addon.py,bot_bottle/egress_addon_core.py,\
bot_bottle/dlp_detectors.py,bot_bottle/egress.py,bot_bottle/manifest.py,\
bot_bottle/manifest_egress.py,bot_bottle/manifest_agent.py,\
bot_bottle/manifest_schema.py,bot_bottle/git_gate.py,\
bot_bottle/git_http_backend.py,bot_bottle/supervise.py,\
bot_bottle/yaml_subset.py,bot_bottle/bottle_state.py"
# Critical security/logic core held to the high bar by ADR 0004. The list
# lives in one place (scripts/critical-modules.txt) so this report and the
# README "core coverage" badge can't drift; comma-join it for --include.
CRITICAL=$(grep -vE '^[[:space:]]*(#|$)' scripts/critical-modules.txt | paste -sd, -)
rm -f .coverage
+23
View File
@@ -0,0 +1,23 @@
# Critical security/logic core held to the >=90% coverage bar by
# docs/decisions/0004-coverage-policy.md.
#
# SINGLE SOURCE OF TRUTH: scripts/coverage.sh (the `critical` report) and
# .gitea/workflows/update-badges.yml (the "core coverage" badge) both read
# this file. Add a module here when it becomes part of the core; a coverage
# number that silently stops measuring a module is worse than no badge.
#
# One module path per line, relative to the repo root. Blank lines and
# `#` comments are ignored.
bot_bottle/egress_addon.py
bot_bottle/egress_addon_core.py
bot_bottle/dlp_detectors.py
bot_bottle/egress.py
bot_bottle/manifest.py
bot_bottle/manifest_egress.py
bot_bottle/manifest_agent.py
bot_bottle/manifest_schema.py
bot_bottle/git_gate.py
bot_bottle/git_http_backend.py
bot_bottle/supervise.py
bot_bottle/yaml_subset.py
bot_bottle/bottle_state.py
+132
View File
@@ -0,0 +1,132 @@
"""Unit: supervise queue/audit error + edge branches (coverage ratchet,
ADR 0004). Complements test_supervise.py with the malformed-input and
fallback paths."""
from __future__ import annotations
import os
import tempfile
import time
import unittest
from pathlib import Path
from unittest.mock import patch
from bot_bottle import supervise
from bot_bottle.supervise import (
Proposal,
TOOL_EGRESS_ALLOW,
list_pending_proposals,
read_audit_entries,
read_proposal,
read_response,
wait_for_response,
)
def _proposal() -> Proposal:
return Proposal.new(
bottle_slug="slug",
tool=TOOL_EGRESS_ALLOW,
proposed_file="x",
justification="j",
current_file_hash="h",
)
class TestPathHelpers(unittest.TestCase):
def test_bot_bottle_root(self) -> None:
self.assertTrue(str(supervise.bot_bottle_root()).endswith(".bot-bottle"))
def test_queue_dir_for_slug(self) -> None:
self.assertIn("slug", str(supervise.queue_dir_for_slug("slug")))
def test_id_from_non_proposal_filename(self) -> None:
self.assertIsNone(supervise._id_from_proposal_filename(Path("x.response.json")))
class TestReadMalformed(unittest.TestCase):
def test_read_proposal_non_dict(self) -> None:
with tempfile.TemporaryDirectory() as d:
(Path(d) / "p.proposal.json").write_text("[]")
with self.assertRaises(ValueError):
read_proposal(Path(d), "p")
def test_read_response_non_dict(self) -> None:
with tempfile.TemporaryDirectory() as d:
(Path(d) / "p.response.json").write_text("[]")
with self.assertRaises(ValueError):
read_response(Path(d), "p")
def test_list_pending_skips_malformed(self) -> None:
with tempfile.TemporaryDirectory() as d:
qd = Path(d)
(qd / "bad.proposal.json").write_text("{ not json")
(qd / "arr.proposal.json").write_text("[]")
(qd / "incomplete.proposal.json").write_text("{}") # from_dict raises
supervise.write_proposal(qd, _proposal()) # one valid
pending = list_pending_proposals(qd)
self.assertEqual(1, len(pending))
self.assertEqual("slug", pending[0].bottle_slug)
def test_list_pending_skips_when_response_present(self) -> None:
with tempfile.TemporaryDirectory() as d:
qd = Path(d)
p = _proposal()
supervise.write_proposal(qd, p)
(qd / f"{p.id}.response.json").write_text("{}") # response exists -> skipped
self.assertEqual([], list_pending_proposals(qd))
class TestWaitForResponse(unittest.TestCase):
def test_malformed_response_then_timeout(self) -> None:
with tempfile.TemporaryDirectory() as d:
(Path(d) / "p.response.json").write_text("{ not json")
with self.assertRaises(TimeoutError):
wait_for_response(Path(d), "p", deadline=time.monotonic())
def test_incomplete_response_then_timeout(self) -> None:
with tempfile.TemporaryDirectory() as d:
(Path(d) / "p.response.json").write_text("{}") # dict but from_dict raises
with self.assertRaises(TimeoutError):
wait_for_response(Path(d), "p", deadline=time.monotonic())
class TestReadAuditEntries(unittest.TestCase):
def test_missing_log_returns_empty(self) -> None:
with tempfile.TemporaryDirectory() as home, \
patch.dict("os.environ", {"HOME": home}):
self.assertEqual([], read_audit_entries("egress", "nope"))
def test_skips_malformed_lines(self) -> None:
with tempfile.TemporaryDirectory() as home, \
patch.dict("os.environ", {"HOME": home}):
path = supervise.audit_log_path("egress", "slug")
path.parent.mkdir(parents=True, exist_ok=True)
valid = (
'{"timestamp": "t", "bottle_slug": "slug", "component": "egress",'
' "operator_action": "approve", "operator_notes": "",'
' "justification": "", "diff": ""}'
)
path.write_text(
"\n" # blank line skipped
"{ not json\n" # JSONDecodeError skipped
"[]\n" # not a dict skipped
"{}\n" # missing fields -> ValueError skipped
+ valid + "\n"
)
entries = read_audit_entries("egress", "slug")
self.assertEqual(1, len(entries))
self.assertEqual("approve", entries[0].operator_action)
class TestFlockFallback(unittest.TestCase):
def test_flock_on_closed_fd_is_swallowed(self) -> None:
# flock on a closed fd raises OSError(EBADF), which the helpers swallow.
fd = os.open(os.devnull, os.O_RDONLY)
os.close(fd)
supervise._try_flock(fd)
supervise._try_funlock(fd)
if __name__ == "__main__":
unittest.main()