diff --git a/.gitea/workflows/lint.yml b/.gitea/workflows/lint.yml index b562619b..c60accc8 100644 --- a/.gitea/workflows/lint.yml +++ b/.gitea/workflows/lint.yml @@ -4,6 +4,12 @@ on: push: paths: - "**.py" + - "Dockerfile*" + - "bot_bottle/contrib/*/Dockerfile" + - "bot_bottle/contrib/*/package.json" + - "bot_bottle/contrib/*/package-lock.json" + - "bot_bottle/contrib/codex/install.sh.sha256" + - "requirements.gateway.*" - ".pylintrc" - ".gitea/workflows/lint.yml" @@ -13,6 +19,9 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Enforce immutable image inputs + run: python3 scripts/check_image_inputs.py + # No actions/setup-python: the runner image already ships Python 3.12, # and older act_runner engines mishandle setup-python's PATH. Install # into the ephemeral job container's system Python — the pylint/pyright diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 963e7d0f..2cbbde6f 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -24,9 +24,11 @@ on: - 'Dockerfile*' - 'pyproject.toml' - 'requirements-dev.txt' + - 'requirements.gateway.*' - '.coveragerc' - '.dockerignore' - '.gitea/workflows/test.yml' + - '.gitea/workflows/refresh-image-locks.yml' - '.gitea/workflows/pre-release-test.yml' pull_request: paths: @@ -45,9 +47,11 @@ on: - 'Dockerfile*' - 'pyproject.toml' - 'requirements-dev.txt' + - 'requirements.gateway.*' - '.coveragerc' - '.dockerignore' - '.gitea/workflows/test.yml' + - '.gitea/workflows/refresh-image-locks.yml' - '.gitea/workflows/pre-release-test.yml' jobs: @@ -137,6 +141,63 @@ jobs: name: coverage-docker path: coverage-docker.dat + image-input-builds: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Verify shared bases cover supported architectures + run: | + python_ref='python:3.12.13-slim-trixie@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de' + node_ref='node:22.23.1-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba' + for ref in "$python_ref" "$node_ref"; do + docker buildx imagetools inspect --raw "$ref" | + python3 -c ' + import json + import sys + manifest = json.load(sys.stdin) + platforms = { + (item["platform"]["os"], item["platform"]["architecture"]) + for item in manifest["manifests"] + if item.get("platform", {}).get("os") != "unknown" + } + required = {("linux", "amd64"), ("linux", "arm64")} + missing = required - platforms + if missing: + raise SystemExit(f"base manifest lacks supported platforms: {missing}") + ' + done + + - name: Build and smoke-test all supported images + run: | + set -euo pipefail + suffix="${GITHUB_RUN_ID:-${GITHUB_RUN_NUMBER:-image-inputs}}" + orchestrator="bot-bottle-orchestrator-inputs:${suffix}" + gateway="bot-bottle-gateway-inputs:${suffix}" + orchestrator_fc="bot-bottle-orchestrator-fc-inputs:${suffix}" + claude="bot-bottle-claude-inputs:${suffix}" + codex="bot-bottle-codex-inputs:${suffix}" + pi="bot-bottle-pi-inputs:${suffix}" + + docker build -t "$orchestrator" -f Dockerfile.orchestrator . + orchestrator_id=$(docker image inspect --format '{{.Id}}' "$orchestrator") + case "$orchestrator_id" in sha256:*) ;; *) exit 1 ;; esac + docker build -t "$gateway" -f Dockerfile.gateway . + docker build \ + --build-arg "ORCHESTRATOR_BASE_IMAGE=$orchestrator_id" \ + -t "$orchestrator_fc" -f Dockerfile.orchestrator.fc . + docker build -t "$claude" -f bot_bottle/contrib/claude/Dockerfile . + docker build -t "$codex" -f bot_bottle/contrib/codex/Dockerfile . + docker build -t "$pi" -f bot_bottle/contrib/pi/Dockerfile . + + docker run --rm --entrypoint python3 "$orchestrator" -c \ + 'import bot_bottle.orchestrator' + docker run --rm --entrypoint mitmdump "$gateway" --version + docker run --rm "$claude" claude --version + docker run --rm "$codex" codex --version + docker run --rm "$pi" pi --version + coverage: needs: [unit, integration-docker] timeout-minutes: 15 diff --git a/scripts/check_image_inputs.py b/scripts/check_image_inputs.py new file mode 100644 index 00000000..ab7b1e55 --- /dev/null +++ b/scripts/check_image_inputs.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Fail when a supported image reintroduces mutable build inputs.""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +DOCKERFILES = ( + Path("Dockerfile.gateway"), + Path("Dockerfile.orchestrator"), + Path("Dockerfile.orchestrator.fc"), + Path("bot_bottle/contrib/claude/Dockerfile"), + Path("bot_bottle/contrib/codex/Dockerfile"), + Path("bot_bottle/contrib/pi/Dockerfile"), +) +NPM_MANIFESTS = ( + Path("bot_bottle/contrib/claude/package.json"), + Path("bot_bottle/contrib/pi/package.json"), +) + +_DIGEST = re.compile(r"^[0-9a-f]{64}$") +_EXACT_NPM_VERSION = re.compile( + r"^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z][0-9A-Za-z.-]*)?$", +) +_SNAPSHOT = re.compile(r"ARG\s+DEBIAN_SNAPSHOT=\d{8}T\d{6}Z") +_NETWORK_TO_SHELL = re.compile( + r"(?:curl|wget)\b[^|\n]*\|\s*(?:ba)?sh\b", + re.IGNORECASE, +) + + +def _logical_lines(text: str) -> str: + """Join Dockerfile continuations so policy regexes see one instruction.""" + return re.sub(r"\\\r?\n", " ", text) + + +def check_dockerfile(path: Path, text: str) -> list[str]: + """Return policy violations for one Dockerfile.""" + problems: list[str] = [] + logical = _logical_lines(text) + instructions = "\n".join( + line for line in logical.splitlines() + if not line.lstrip().startswith("#") + ) + from_values = re.findall(r"(?im)^\s*FROM\s+(\S+)", logical) + if not from_values: + problems.append(f"{path}: missing FROM") + for value in from_values: + if value == "${ORCHESTRATOR_BASE_IMAGE}": + if not re.search( + r"(?m)^\s*ARG\s+ORCHESTRATOR_BASE_IMAGE\s*$", + text, + ): + problems.append( + f"{path}: local base argument must have no mutable default", + ) + continue + if "$" in value: + problems.append(f"{path}: dynamic base image is not content-pinned: {value}") + continue + try: + image, digest = value.rsplit("@sha256:", 1) + except ValueError: + problems.append(f"{path}: base image lacks a sha256 digest: {value}") + continue + leaf = image.rsplit("/", 1)[-1] + if ":" not in leaf: + problems.append(f"{path}: base image lacks a version-qualified tag: {value}") + elif leaf.rsplit(":", 1)[1] == "latest": + problems.append(f"{path}: base image uses latest: {value}") + if not _DIGEST.fullmatch(digest): + problems.append(f"{path}: base image digest is not 64 lowercase hex: {value}") + + if _NETWORK_TO_SHELL.search(instructions): + problems.append(f"{path}: downloads network content directly into a shell") + if re.search(r"\b(?:npm\s+(?:i|install)|pi\s+install)\b", instructions): + problems.append(f"{path}: use npm ci with a committed integrity lock") + if "apt-get install" in instructions: + if not _SNAPSHOT.search(text): + problems.append(f"{path}: apt install lacks a fixed Debian snapshot") + if "snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT}" not in text: + problems.append(f"{path}: apt sources are not routed to the snapshot") + if "Acquire::Check-Valid-Until=false" not in text: + problems.append(f"{path}: snapshot apt update lacks validity override") + if path == Path("Dockerfile.gateway"): + if "--require-hashes" not in instructions: + problems.append(f"{path}: gateway Python install does not require hashes") + if "requirements.gateway.lock" not in text: + problems.append(f"{path}: gateway Python lock is not consumed") + if path == Path("bot_bottle/contrib/codex/Dockerfile"): + required = ( + "install.sh.sha256", + "sha256sum -c", + '--release "${CODEX_VERSION}"', + ) + for marker in required: + if marker not in instructions: + problems.append(f"{path}: verified pinned Codex install lacks {marker!r}") + return problems + + +def check_npm_manifest(root: Path, manifest_path: Path) -> list[str]: + """Validate exact direct versions and integrity-complete npm locks.""" + problems: list[str] = [] + manifest_file = root / manifest_path + lock_file = manifest_file.with_name("package-lock.json") + try: + manifest = json.loads(manifest_file.read_text(encoding="utf-8")) + lock = json.loads(lock_file.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return [f"{manifest_path}: cannot read manifest and lock: {exc}"] + + direct = manifest.get("dependencies", {}) + for package, version in direct.items(): + if not isinstance(version, str) or not _EXACT_NPM_VERSION.fullmatch(version): + problems.append( + f"{manifest_path}: {package} is not an exact version: {version!r}", + ) + root_lock = lock.get("packages", {}).get("", {}).get("dependencies", {}) + if root_lock != direct: + problems.append(f"{lock_file.relative_to(root)}: root dependencies are stale") + if lock.get("lockfileVersion") != 3: + problems.append(f"{lock_file.relative_to(root)}: require lockfileVersion 3") + for package_path, package in lock.get("packages", {}).items(): + if not package_path or package.get("link"): + continue + resolved = package.get("resolved", "") + if isinstance(resolved, str) and resolved.startswith(("http://", "https://")): + integrity = package.get("integrity", "") + if not isinstance(integrity, str) or not integrity.startswith("sha512-"): + problems.append( + f"{lock_file.relative_to(root)}: " + f"{package_path} lacks sha512 integrity", + ) + return problems + + +def check_python_lock(root: Path) -> list[str]: + """Validate that every gateway Python requirement is exact and hashed.""" + path = root / "requirements.gateway.lock" + try: + text = path.read_text(encoding="utf-8") + except OSError as exc: + return [f"requirements.gateway.lock: cannot read lock: {exc}"] + problems: list[str] = [] + blocks = re.split(r"(?m)(?=^[A-Za-z0-9_.-]+==)", text) + requirements = [block for block in blocks if re.match(r"^[A-Za-z0-9_.-]+==", block)] + if not requirements: + problems.append("requirements.gateway.lock: contains no exact requirements") + for block in requirements: + name = block.split("==", 1)[0] + first_line = block.splitlines()[0] + if not re.match(r"^[A-Za-z0-9_.-]+==[^\s\\]+", first_line): + problems.append(f"requirements.gateway.lock: {name} is not exact") + if not re.search(r"--hash=sha256:[0-9a-f]{64}", block): + problems.append(f"requirements.gateway.lock: {name} lacks a sha256 hash") + try: + direct = [ + line.strip() + for line in (root / "requirements.gateway.in").read_text().splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + except OSError as exc: + problems.append(f"requirements.gateway.in: cannot read input: {exc}") + direct = [] + for requirement in direct: + if not re.fullmatch(r"[A-Za-z0-9_.-]+==[^\s]+", requirement): + problems.append( + f"requirements.gateway.in: direct dependency is not exact: {requirement}", + ) + if not re.search(rf"(?m)^{re.escape(requirement)}(?:\s|\\)", text): + problems.append( + f"requirements.gateway.lock: missing direct dependency {requirement}", + ) + return problems + + +def check_repo(root: Path = REPO_ROOT) -> list[str]: + """Return every repository image-input policy violation.""" + problems: list[str] = [] + for path in DOCKERFILES: + try: + text = (root / path).read_text(encoding="utf-8") + except OSError as exc: + problems.append(f"{path}: cannot read Dockerfile: {exc}") + continue + problems.extend(check_dockerfile(path, text)) + for manifest in NPM_MANIFESTS: + problems.extend(check_npm_manifest(root, manifest)) + problems.extend(check_python_lock(root)) + return problems + + +def main() -> int: + problems = check_repo() + if problems: + for problem in problems: + print(f"image-input policy: {problem}", file=sys.stderr) + return 1 + print("image-input policy: all supported image inputs are pinned and verified") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/test_image_input_policy.py b/tests/unit/test_image_input_policy.py new file mode 100644 index 00000000..feab748e --- /dev/null +++ b/tests/unit/test_image_input_policy.py @@ -0,0 +1,107 @@ +"""Unit tests for the immutable container-build input policy.""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from scripts import check_image_inputs as policy + + +class TestDockerfilePolicy(unittest.TestCase): + def test_accepts_versioned_digest_base(self): + text = "FROM node:22.23.1-trixie-slim@sha256:" + "a" * 64 + "\n" + self.assertEqual([], policy.check_dockerfile(Path("Dockerfile"), text)) + + def test_rejects_mutable_and_latest_bases(self): + for value in ( + "node:22-trixie-slim", + "node:latest@sha256:" + "a" * 64, + "node@sha256:" + "a" * 64, + ): + with self.subTest(value=value): + problems = policy.check_dockerfile( + Path("Dockerfile"), f"FROM {value}\n", + ) + self.assertTrue(problems) + + def test_rejects_network_pipe_to_shell(self): + text = ( + "FROM node:22.23.1@sha256:" + "a" * 64 + + "\nRUN curl -fsSL https://example.invalid/install.sh | sh\n" + ) + problems = policy.check_dockerfile(Path("Dockerfile"), text) + self.assertTrue(any("directly into a shell" in item for item in problems)) + + def test_rejects_unlocked_npm_and_pi_installs(self): + base = "FROM node:22.23.1@sha256:" + "a" * 64 + "\nRUN " + for install in ( + "npm install -g package@1.2.3", + "pi install npm:extension@1.2.3", + ): + with self.subTest(install=install): + problems = policy.check_dockerfile( + Path("Dockerfile"), base + install + "\n", + ) + self.assertTrue(any("npm ci" in item for item in problems)) + + def test_requires_snapshot_for_apt(self): + text = ( + "FROM node:22.23.1@sha256:" + "a" * 64 + + "\nRUN apt-get update && apt-get install git\n" + ) + problems = policy.check_dockerfile(Path("Dockerfile"), text) + self.assertTrue(any("fixed Debian snapshot" in item for item in problems)) + + +class TestNpmLockPolicy(unittest.TestCase): + def _write(self, root: Path, *, version: str, integrity: str = "") -> Path: + package_dir = root / "provider" + package_dir.mkdir() + manifest = { + "name": "test", + "private": True, + "dependencies": {"example": version}, + } + package = { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/example/-/example-1.2.3.tgz", + } + if integrity: + package["integrity"] = integrity + lock = { + "name": "test", + "lockfileVersion": 3, + "packages": { + "": {"dependencies": {"example": version}}, + "node_modules/example": package, + }, + } + (package_dir / "package.json").write_text(json.dumps(manifest)) + (package_dir / "package-lock.json").write_text(json.dumps(lock)) + return Path("provider/package.json") + + def test_accepts_exact_integrity_locked_package(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = self._write(root, version="1.2.3", integrity="sha512-abc") + self.assertEqual([], policy.check_npm_manifest(root, path)) + + def test_rejects_range_and_missing_integrity(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = self._write(root, version="^1.2.3") + problems = policy.check_npm_manifest(root, path) + self.assertTrue(any("not an exact version" in item for item in problems)) + self.assertTrue(any("lacks sha512 integrity" in item for item in problems)) + + +class TestRepositoryPolicy(unittest.TestCase): + def test_repository_image_inputs_pass(self): + self.assertEqual([], policy.check_repo()) + + +if __name__ == "__main__": + unittest.main()