diff --git a/.gitea/workflows/lint.yml b/.gitea/workflows/lint.yml index c60accc8..db663ac6 100644 --- a/.gitea/workflows/lint.yml +++ b/.gitea/workflows/lint.yml @@ -10,6 +10,7 @@ on: - "bot_bottle/contrib/*/package-lock.json" - "bot_bottle/contrib/codex/install.sh.sha256" - "requirements.gateway.*" + - "image-build-args.json" - ".pylintrc" - ".gitea/workflows/lint.yml" diff --git a/.gitea/workflows/refresh-image-locks.yml b/.gitea/workflows/refresh-image-locks.yml index 16d0ce0d..4b650892 100644 --- a/.gitea/workflows/refresh-image-locks.yml +++ b/.gitea/workflows/refresh-image-locks.yml @@ -13,6 +13,7 @@ on: - 'bot_bottle/contrib/claude/package.json' - 'bot_bottle/contrib/pi/package.json' - 'bot_bottle/contrib/codex/Dockerfile' + - 'image-build-args.json' - '.gitea/workflows/refresh-image-locks.yml' permissions: @@ -25,15 +26,32 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Resolve image runtime versions + id: runtimes + run: | + python3 - <<'PY' >> "$GITHUB_OUTPUT" + import json + import re + inputs = json.load(open("image-build-args.json")) + for output, name in ( + ("python-version", "PYTHON_BASE_IMAGE"), + ("node-version", "NODE_BASE_IMAGE"), + ): + match = re.search(r":(\d+\.\d+\.\d+)-", inputs[name]) + if match is None: + raise SystemExit(f"cannot resolve runtime version from {name}") + print(f"{output}={match.group(1)}") + PY + - name: Use the image Python version uses: actions/setup-python@v5 with: - python-version: '3.12.13' + python-version: '${{ steps.runtimes.outputs.python-version }}' - name: Use the image Node version uses: actions/setup-node@v4 with: - node-version: '22.23.1' + node-version: '${{ steps.runtimes.outputs.node-version }}' - name: Compile gateway Python lock run: | diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index c1676161..de2ff31c 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -22,6 +22,7 @@ on: - 'scripts/**/*.py' - 'scripts/firecracker-netpool.sh' - 'Dockerfile*' + - 'image-build-args.json' - 'pyproject.toml' - 'requirements-dev.txt' - 'requirements.gateway.*' @@ -45,6 +46,7 @@ on: - 'scripts/**/*.py' - 'scripts/firecracker-netpool.sh' - 'Dockerfile*' + - 'image-build-args.json' - 'pyproject.toml' - 'requirements-dev.txt' - 'requirements.gateway.*' @@ -149,8 +151,13 @@ jobs: - name: Verify shared bases cover supported architectures run: | - python_ref='python:3.12.13-slim-trixie@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de' - node_ref='node:22.23.1-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba' + set -euo pipefail + python_ref=$(python3 -c \ + 'import json; print(json.load(open("image-build-args.json"))["PYTHON_BASE_IMAGE"])') + node_ref=$(python3 -c \ + 'import json; print(json.load(open("image-build-args.json"))["NODE_BASE_IMAGE"])') + test -n "$python_ref" + test -n "$node_ref" for ref in "$python_ref" "$node_ref"; do docker buildx imagetools inspect --raw "$ref" | python3 -c ' @@ -179,8 +186,13 @@ jobs: claude="bot-bottle-claude-inputs:${suffix}" codex="bot-bottle-codex-inputs:${suffix}" pi="bot-bottle-pi-inputs:${suffix}" + python_base=$(python3 -c \ + 'import json; print(json.load(open("image-build-args.json"))["PYTHON_BASE_IMAGE"])') + node_base=$(python3 -c \ + 'import json; print(json.load(open("image-build-args.json"))["NODE_BASE_IMAGE"])') - docker build -t "$orchestrator" -f Dockerfile.orchestrator . + docker build --build-arg "PYTHON_BASE_IMAGE=$python_base" \ + -t "$orchestrator" -f Dockerfile.orchestrator . orchestrator_id=$(docker image inspect --format '{{.Id}}' "$orchestrator") case "$orchestrator_id" in sha256:*) ;; *) exit 1 ;; esac orchestrator_base="bot-bottle-orchestrator-inputs:sha256-${orchestrator_id#sha256:}" @@ -188,13 +200,17 @@ jobs: test "$( docker image inspect --format '{{.Id}}' "$orchestrator_base" )" = "$orchestrator_id" - docker build -t "$gateway" -f Dockerfile.gateway . + docker build --build-arg "PYTHON_BASE_IMAGE=$python_base" \ + -t "$gateway" -f Dockerfile.gateway . docker build \ --build-arg "ORCHESTRATOR_BASE_IMAGE=$orchestrator_base" \ -t "$orchestrator_fc" -f Dockerfile.orchestrator.fc . - docker build -t "$claude" -f bot_bottle/contrib/claude/Dockerfile . - docker build -t "$codex" -f bot_bottle/contrib/codex/Dockerfile . - docker build -t "$pi" -f bot_bottle/contrib/pi/Dockerfile . + docker build --build-arg "NODE_BASE_IMAGE=$node_base" \ + -t "$claude" -f bot_bottle/contrib/claude/Dockerfile . + docker build --build-arg "NODE_BASE_IMAGE=$node_base" \ + -t "$codex" -f bot_bottle/contrib/codex/Dockerfile . + docker build --build-arg "NODE_BASE_IMAGE=$node_base" \ + -t "$pi" -f bot_bottle/contrib/pi/Dockerfile . docker run --rm --entrypoint python3 "$orchestrator" -c \ 'import bot_bottle.orchestrator' diff --git a/Dockerfile.gateway b/Dockerfile.gateway index 1c7a2765..1fe7454f 100644 --- a/Dockerfile.gateway +++ b/Dockerfile.gateway @@ -41,7 +41,8 @@ # `mitmproxy/mitmproxy` image (Debian bookworm), matching the trixie base the # orchestrator image needs for buildah (Dockerfile.orchestrator.fc). mitmproxy # is pip-installed to the same effect as the upstream image. -FROM python:3.12.13-slim-trixie@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de +ARG PYTHON_BASE_IMAGE +FROM ${PYTHON_BASE_IMAGE} # Freeze apt's package universe as well as the base filesystem. Without a # snapshot, the same Dockerfile resolves different package versions over time. diff --git a/Dockerfile.orchestrator b/Dockerfile.orchestrator index d01ee9bf..44a246d4 100644 --- a/Dockerfile.orchestrator +++ b/Dockerfile.orchestrator @@ -20,7 +20,8 @@ # image. The version-qualified tag keeps the human-readable upstream version; # the digest makes the bytes immutable. -FROM python:3.12.13-slim-trixie@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de +ARG PYTHON_BASE_IMAGE +FROM ${PYTHON_BASE_IMAGE} WORKDIR /app diff --git a/MANIFEST.in b/MANIFEST.in index 92dafc83..7bdc3047 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,7 @@ include Dockerfile.gateway include Dockerfile.orchestrator include Dockerfile.orchestrator.fc +include image-build-args.json include requirements.gateway.in include requirements.gateway.lock include nix/firecracker-netpool.nix diff --git a/bot_bottle/backend/docker/gateway.py b/bot_bottle/backend/docker/gateway.py index 689ba5f0..7ec74fcf 100644 --- a/bot_bottle/backend/docker/gateway.py +++ b/bot_bottle/backend/docker/gateway.py @@ -96,6 +96,11 @@ class DockerGateway(Gateway): str(context)] if os.environ.get("BOT_BOTTLE_NO_CACHE"): argv.insert(2, "--no-cache") + for name, value in resources.image_build_args( + self._dockerfile, + context=context, + ).items(): + argv[-1:-1] = ["--build-arg", f"{name}={value}"] proc = run_docker(argv) if proc.returncode != 0: raise GatewayError(f"gateway image build failed: {proc.stderr.strip()}") diff --git a/bot_bottle/backend/docker/orchestrator.py b/bot_bottle/backend/docker/orchestrator.py index b293c9c1..a4e890da 100644 --- a/bot_bottle/backend/docker/orchestrator.py +++ b/bot_bottle/backend/docker/orchestrator.py @@ -131,6 +131,11 @@ class DockerOrchestrator(Orchestrator): str(self._repo_root)] if os.environ.get("BOT_BOTTLE_NO_CACHE"): argv.insert(2, "--no-cache") + for name, value in resources.image_build_args( + self._dockerfile, + context=self._repo_root, + ).items(): + argv[-1:-1] = ["--build-arg", f"{name}={value}"] proc = run_docker(argv) if proc.returncode != 0: raise GatewayError( diff --git a/bot_bottle/backend/docker/util.py b/bot_bottle/backend/docker/util.py index 05c13d28..d43103bb 100644 --- a/bot_bottle/backend/docker/util.py +++ b/bot_bottle/backend/docker/util.py @@ -10,6 +10,7 @@ import shutil import subprocess from typing import Iterator +from ... import resources from ...log import die, info from ...util import slugify as _slugify @@ -145,7 +146,12 @@ def build_image( args.append("--no-cache") if dockerfile: args.extend(["-f", dockerfile]) - for name, value in (build_args or {}).items(): + effective_build_args = resources.image_build_args( + dockerfile, + context=context, + ) if dockerfile else {} + effective_build_args.update(build_args or {}) + for name, value in effective_build_args.items(): args.extend(["--build-arg", f"{name}={value}"]) args.append(context) subprocess.run(args, check=True) diff --git a/bot_bottle/backend/firecracker/image_builder.py b/bot_bottle/backend/firecracker/image_builder.py index 77da0ace..9930415d 100644 --- a/bot_bottle/backend/firecracker/image_builder.py +++ b/bot_bottle/backend/firecracker/image_builder.py @@ -19,12 +19,14 @@ from __future__ import annotations import fcntl import hashlib import os +import shlex import shutil import subprocess from contextlib import contextmanager from pathlib import Path from typing import Generator +from ... import resources from ...log import die, info from . import util from .infra import FirecrackerInfraService @@ -55,6 +57,11 @@ def _rootfs_digest(dockerfile: Path) -> str: h = hashlib.sha256() h.update(_dockerfile_hash(dockerfile).encode()) h.update(b"\0") + for name, value in resources.image_build_args(dockerfile).items(): + h.update(name.encode()) + h.update(b"=") + h.update(value.encode()) + h.update(b"\0") h.update(util._GUEST_INIT.encode()) return h.hexdigest()[:16] @@ -147,7 +154,13 @@ def _build_in_infra( if prep.returncode != 0: die(f"preparing build dir in the infra VM failed: {prep.stderr.strip()}") _send_dockerfile(key, ip, dockerfile, ctx) - _buildah_build(key, ip, ctx, tag) + _buildah_build( + key, + ip, + ctx, + tag, + resources.image_build_args(dockerfile), + ) _smoke_test(key, ip, tag, smoke_ctr, smoke_test) _stream_rootfs(key, ip, tag, export_ctr, base) finally: @@ -184,15 +197,26 @@ def _send_dockerfile(private_key: Path, guest_ip: str, dockerfile: Path, ctx: st f"{proc.stderr.decode(errors='replace').strip()}") -def _buildah_build(private_key: Path, guest_ip: str, ctx: str, tag: str) -> None: +def _buildah_build( + private_key: Path, + guest_ip: str, + ctx: str, + tag: str, + build_args: dict[str, str], +) -> None: # Stream buildah's step-by-step output straight to our stderr (like the # docker backend's `docker build`), so a long first build (base pull + # apt/npm installs) shows live progress instead of a silent wait. The # remote stderr is where buildah writes its `STEP i/n` lines. info(f"buildah build {tag} in the infra VM (streaming output)") + arg_flags = " ".join( + f"--build-arg {shlex.quote(f'{name}={value}')}" + for name, value in build_args.items() + ) rc = _ssh_streamed( private_key, guest_ip, - f"buildah build {_BUILD_FLAGS} -t {tag} -f {ctx}/Dockerfile {ctx}/ctx", + f"buildah build {_BUILD_FLAGS} {arg_flags} " + f"-t {tag} -f {ctx}/Dockerfile {ctx}/ctx", timeout=_BUILD_TIMEOUT_SECONDS, ) if rc != 0: diff --git a/bot_bottle/backend/firecracker/infra_artifact.py b/bot_bottle/backend/firecracker/infra_artifact.py index 3ef23d1d..f2e5c1a2 100644 --- a/bot_bottle/backend/firecracker/infra_artifact.py +++ b/bot_bottle/backend/firecracker/infra_artifact.py @@ -50,8 +50,16 @@ _ARTIFACT_FORMAT = "1" # orchestrator rootfs carries buildah), so the versions are hashed separately. ROLES = ("orchestrator", "gateway") _BUILD_INPUTS = { - "orchestrator": ("Dockerfile.orchestrator", "Dockerfile.orchestrator.fc"), - "gateway": ("Dockerfile.gateway", "requirements.gateway.lock"), + "orchestrator": ( + "image-build-args.json", + "Dockerfile.orchestrator", + "Dockerfile.orchestrator.fc", + ), + "gateway": ( + "image-build-args.json", + "Dockerfile.gateway", + "requirements.gateway.lock", + ), } _DEFAULT_BASE = "https://gitea.dideric.is" diff --git a/bot_bottle/backend/macos_container/util.py b/bot_bottle/backend/macos_container/util.py index 23a6403e..34794017 100644 --- a/bot_bottle/backend/macos_container/util.py +++ b/bot_bottle/backend/macos_container/util.py @@ -13,6 +13,7 @@ import time from datetime import datetime, timezone from typing import Iterable +from ... import resources from ...log import die, info @@ -60,7 +61,13 @@ def dns_server() -> str: return _host_ipv4_dns() or _DEFAULT_DNS -def build_image(ref: str, context: str, *, dockerfile: str = "") -> None: +def build_image( + ref: str, + context: str, + *, + dockerfile: str = "", + build_args: dict[str, str] | None = None, +) -> None: """Build an OCI image with Apple's BuildKit-backed `container build`. Set `BOT_BOTTLE_NO_CACHE=1` (the `start --no-cache` flag) to force @@ -83,6 +90,13 @@ def build_image(ref: str, context: str, *, dockerfile: str = "") -> None: if not os.path.isabs(dockerfile): dockerfile = os.path.join(context, dockerfile) args.extend(["-f", dockerfile]) + effective_build_args = resources.image_build_args( + dockerfile, + context=context, + ) if dockerfile else {} + effective_build_args.update(build_args or {}) + for name, value in effective_build_args.items(): + args.extend(["--build-arg", f"{name}={value}"]) args.append(context) subprocess.run(args, check=True) diff --git a/bot_bottle/contrib/claude/Dockerfile b/bot_bottle/contrib/claude/Dockerfile index f76a8e8a..5d6bc791 100644 --- a/bot_bottle/contrib/claude/Dockerfile +++ b/bot_bottle/contrib/claude/Dockerfile @@ -9,7 +9,8 @@ # changes to the rest of the repo (or to the CMD) don't bust it. # Version-qualified Node LTS, pinned to its multi-architecture manifest. -FROM node:22.23.1-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba +ARG NODE_BASE_IMAGE +FROM ${NODE_BASE_IMAGE} ARG DEBIAN_SNAPSHOT=20260724T000000Z RUN sed -i \ diff --git a/bot_bottle/contrib/codex/Dockerfile b/bot_bottle/contrib/codex/Dockerfile index add00e16..9adb2b38 100644 --- a/bot_bottle/contrib/codex/Dockerfile +++ b/bot_bottle/contrib/codex/Dockerfile @@ -3,7 +3,8 @@ # Mirrors the default Claude image shape: Node LTS, git/network tooling, # non-root node user, and the provider CLI installed for that user. -FROM node:22.23.1-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba +ARG NODE_BASE_IMAGE +FROM ${NODE_BASE_IMAGE} # The standalone installer is used below because remote-control requires its # managed package layout. Keep this exact release in sync with the verified diff --git a/bot_bottle/contrib/pi/Dockerfile b/bot_bottle/contrib/pi/Dockerfile index 54c70031..60ad915d 100644 --- a/bot_bottle/contrib/pi/Dockerfile +++ b/bot_bottle/contrib/pi/Dockerfile @@ -2,7 +2,8 @@ # # Node LTS, git/network tooling, and the Pi coding-agent CLI installed globally. -FROM node:22.23.1-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba +ARG NODE_BASE_IMAGE +FROM ${NODE_BASE_IMAGE} ARG DEBIAN_SNAPSHOT=20260724T000000Z RUN sed -i \ diff --git a/bot_bottle/resources.py b/bot_bottle/resources.py index 7d88ee3a..0d8e6581 100644 --- a/bot_bottle/resources.py +++ b/bot_bottle/resources.py @@ -20,7 +20,9 @@ from __future__ import annotations import fcntl import hashlib +import json import os +import re import shutil import tempfile from pathlib import Path @@ -37,6 +39,7 @@ _BUNDLED = _PKG / "_resources" # wheel-shipped copies # the two lists in sync (``test_resources`` guards that every entry exists). BUNDLED_RESOURCES: tuple[str, ...] = ( "pyproject.toml", + "image-build-args.json", "Dockerfile.gateway", "Dockerfile.orchestrator", "Dockerfile.orchestrator.fc", @@ -48,6 +51,8 @@ BUNDLED_RESOURCES: tuple[str, ...] = ( # Present at a checkout root, never in a bare installed package — the cheap # tell for which layout we're in. _CHECKOUT_MARKER = "Dockerfile.gateway" +_IMAGE_BUILD_ARGS_FILE = "image-build-args.json" +_CENTRAL_BUILD_ARG_NAMES = frozenset({"NODE_BASE_IMAGE", "PYTHON_BASE_IMAGE"}) class ResourceError(RuntimeError): @@ -79,6 +84,59 @@ def dockerfile(name: str) -> Path: return build_root() / name +def image_build_args( + dockerfile_path: str | Path, + *, + context: str | Path | None = None, +) -> dict[str, str]: + """Return centralized arguments declared by ``dockerfile_path``. + + Base-image arguments deliberately have no Dockerfile defaults. Their + digest-pinned values live in one repository input file and every supported + build path calls this helper before invoking its OCI builder. Explicit + caller-supplied arguments may still override this returned mapping. + """ + path = Path(dockerfile_path) + if not path.is_absolute() and context is not None: + path = Path(context) / path + try: + text = path.read_text(encoding="utf-8") + except OSError: + # Generic callers and tests may build an ephemeral Dockerfile outside + # bot-bottle. The builder will report a genuinely missing file. + return {} + declared = set(re.findall( + r"(?m)^\s*ARG\s+([A-Za-z_][A-Za-z0-9_]*)\s*$", + text, + )) + wanted = declared & _CENTRAL_BUILD_ARG_NAMES + if not wanted: + return {} + root = Path(context) if context is not None else build_root() + inputs_path = root / _IMAGE_BUILD_ARGS_FILE + try: + inputs = json.loads(inputs_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ResourceError( + f"cannot read centralized image build arguments from {inputs_path}: {exc}" + ) from exc + if not isinstance(inputs, dict): + raise ResourceError(f"{inputs_path} must contain a JSON object") + missing = wanted - inputs.keys() + if missing: + raise ResourceError( + f"{inputs_path} lacks required image build arguments: " + f"{', '.join(sorted(missing))}" + ) + invalid = [name for name in wanted if not isinstance(inputs[name], str)] + if invalid: + raise ResourceError( + f"{inputs_path} has non-string image build arguments: " + f"{', '.join(sorted(invalid))}" + ) + return {name: inputs[name] for name in sorted(wanted)} + + def nix_netpool_module() -> Path: """Absolute path to the firecracker netpool NixOS module.""" return build_root() / "nix" / "firecracker-netpool.nix" diff --git a/docs/image-build-inputs.md b/docs/image-build-inputs.md index 61e8ed88..e49b3fa8 100644 --- a/docs/image-build-inputs.md +++ b/docs/image-build-inputs.md @@ -4,8 +4,10 @@ Bot-bottle's supported images are intended to rebuild from the same declared inputs on Linux amd64 and arm64. The repository enforces four layers of immutability: -- Python and Node base images use version-qualified tags plus multi-platform - OCI index digests. +- Python and Node base images are required, defaultless Docker build arguments. + Their version-qualified tags and multi-platform OCI index digests live + together in `image-build-args.json`; every supported builder reads that + file and passes only the arguments its Dockerfile declares. - Debian packages resolve from the same dated `snapshot.debian.org` archive in every image that runs `apt-get install`. The snapshot endpoint uses HTTP so a fresh image does not need a host-specific TLS interception CA; APT still @@ -31,9 +33,9 @@ BuildKit, which treats a bare image ID in `FROM` as a registry repository. Make refreshes on a feature branch and review them like an application dependency update: -1. For a Python or Node base update, select a version-qualified tag and record - its multi-platform index digest in every Dockerfile that shares the base. - Confirm the index still contains `linux/amd64` and `linux/arm64`. +1. For a Python or Node base update, select a version-qualified tag and update + its one digest-pinned value in `image-build-args.json`. Confirm the index + still contains `linux/amd64` and `linux/arm64`. 2. For Debian packages, advance `DEBIAN_SNAPSHOT` to one fixed UTC timestamp in every Dockerfile that uses apt. Do not use the moving Debian mirrors. 3. Change direct Python versions in `requirements.gateway.in`, direct npm diff --git a/image-build-args.json b/image-build-args.json new file mode 100644 index 00000000..2247da49 --- /dev/null +++ b/image-build-args.json @@ -0,0 +1,4 @@ +{ + "NODE_BASE_IMAGE": "node:22.23.1-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba", + "PYTHON_BASE_IMAGE": "python:3.12.13-slim-trixie@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de" +} diff --git a/scripts/check_image_inputs.py b/scripts/check_image_inputs.py index ab7b1e55..dac6e011 100644 --- a/scripts/check_image_inputs.py +++ b/scripts/check_image_inputs.py @@ -21,6 +21,8 @@ NPM_MANIFESTS = ( Path("bot_bottle/contrib/claude/package.json"), Path("bot_bottle/contrib/pi/package.json"), ) +IMAGE_BUILD_ARGS = Path("image-build-args.json") +REQUIRED_BASE_ARGS = frozenset({"NODE_BASE_IMAGE", "PYTHON_BASE_IMAGE"}) _DIGEST = re.compile(r"^[0-9a-f]{64}$") _EXACT_NPM_VERSION = re.compile( @@ -38,7 +40,11 @@ def _logical_lines(text: str) -> str: return re.sub(r"\\\r?\n", " ", text) -def check_dockerfile(path: Path, text: str) -> list[str]: +def check_dockerfile( + path: Path, + text: str, + image_build_args: dict[str, str] | None = None, +) -> list[str]: """Return policy violations for one Dockerfile.""" problems: list[str] = [] logical = _logical_lines(text) @@ -46,19 +52,47 @@ def check_dockerfile(path: Path, text: str) -> list[str]: line for line in logical.splitlines() if not line.lstrip().startswith("#") ) - from_values = re.findall(r"(?im)^\s*FROM\s+(\S+)", logical) + from_matches = list(re.finditer(r"(?im)^\s*FROM\s+(\S+)", logical)) + from_values = [match.group(1) for match in from_matches] + preamble = logical[:from_matches[0].start()] if from_matches else logical + base_args = { + match.group(1): match.group(2) + for match in re.finditer( + r"(?im)^\s*ARG\s+([A-Za-z_][A-Za-z0-9_]*)(?:=(\S+))?\s*$", + preamble, + ) + } if not from_values: problems.append(f"{path}: missing FROM") for value in from_values: - if value == "${ORCHESTRATOR_BASE_IMAGE}": - if not re.search( - r"(?m)^\s*ARG\s+ORCHESTRATOR_BASE_IMAGE\s*$", - text, + variable = re.fullmatch(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", value) + if variable and variable.group(1) == "ORCHESTRATOR_BASE_IMAGE": + if ( + "ORCHESTRATOR_BASE_IMAGE" not in base_args + or base_args["ORCHESTRATOR_BASE_IMAGE"] is not None ): problems.append( f"{path}: local base argument must have no mutable default", ) continue + if variable: + name = variable.group(1) + if name not in base_args: + problems.append( + f"{path}: dynamic base argument {name} is not declared before FROM", + ) + continue + if base_args[name] is not None: + problems.append( + f"{path}: base argument {name} must not define a default", + ) + continue + if image_build_args is None or name not in image_build_args: + problems.append( + f"{path}: base argument {name} lacks a centralized value", + ) + continue + value = image_build_args[name] if "$" in value: problems.append(f"{path}: dynamic base image is not content-pinned: {value}") continue @@ -103,6 +137,37 @@ def check_dockerfile(path: Path, text: str) -> list[str]: return problems +def load_image_build_args(root: Path) -> tuple[dict[str, str], list[str]]: + """Load and validate the one repository location for base-image args.""" + path = root / IMAGE_BUILD_ARGS + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return {}, [f"{IMAGE_BUILD_ARGS}: cannot read image build arguments: {exc}"] + if not isinstance(data, dict): + return {}, [f"{IMAGE_BUILD_ARGS}: must contain a JSON object"] + problems: list[str] = [] + names = set(data) + missing = REQUIRED_BASE_ARGS - names + unexpected = names - REQUIRED_BASE_ARGS + if missing: + problems.append( + f"{IMAGE_BUILD_ARGS}: missing arguments: {', '.join(sorted(missing))}", + ) + if unexpected: + problems.append( + f"{IMAGE_BUILD_ARGS}: unexpected arguments: " + f"{', '.join(sorted(unexpected))}", + ) + for name, value in data.items(): + if not isinstance(value, str): + problems.append(f"{IMAGE_BUILD_ARGS}: {name} must be a string") + return { + name: value for name, value in data.items() + if isinstance(name, str) and isinstance(value, str) + }, problems + + def check_npm_manifest(root: Path, manifest_path: Path) -> list[str]: """Validate exact direct versions and integrity-complete npm locks.""" problems: list[str] = [] @@ -181,14 +246,14 @@ def check_python_lock(root: Path) -> list[str]: def check_repo(root: Path = REPO_ROOT) -> list[str]: """Return every repository image-input policy violation.""" - problems: list[str] = [] + image_build_args, problems = load_image_build_args(root) for path in DOCKERFILES: try: text = (root / path).read_text(encoding="utf-8") except OSError as exc: problems.append(f"{path}: cannot read Dockerfile: {exc}") continue - problems.extend(check_dockerfile(path, text)) + problems.extend(check_dockerfile(path, text, image_build_args)) for manifest in NPM_MANIFESTS: problems.extend(check_npm_manifest(root, manifest)) problems.extend(check_python_lock(root)) diff --git a/scripts/demo-setup.sh b/scripts/demo-setup.sh index 840e699c..e19c06d6 100755 --- a/scripts/demo-setup.sh +++ b/scripts/demo-setup.sh @@ -35,5 +35,12 @@ chmod 600 "$fake_key_dir/fake-key" # Build the image graph quietly so the recorded run shows only the # bottle launch and the four `!` probes, not BuildKit progress. -docker build -q -f bot_bottle/contrib/claude/Dockerfile -t bot-bottle-claude:latest . >/dev/null 2>&1 || true +node_base_image=$( + python3 -c \ + 'import json; print(json.load(open("image-build-args.json"))["NODE_BASE_IMAGE"])' +) +docker build -q \ + --build-arg "NODE_BASE_IMAGE=$node_base_image" \ + -f bot_bottle/contrib/claude/Dockerfile \ + -t bot-bottle-claude:latest . >/dev/null 2>&1 || true docker build -q -f Dockerfile.git-gate -t bot-bottle-git-gate:latest . >/dev/null 2>&1 || true diff --git a/setup.py b/setup.py index f61268ef..a53c3923 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,7 @@ _ROOT = Path(__file__).resolve().parent # Must match bot_bottle.resources.BUNDLED_RESOURCES (paths relative to root). _BUNDLED_RESOURCES = ( "pyproject.toml", + "image-build-args.json", "Dockerfile.gateway", "Dockerfile.orchestrator", "Dockerfile.orchestrator.fc", diff --git a/tests/integration/test_gateway_image.py b/tests/integration/test_gateway_image.py index a0e7b878..3a8c4f30 100644 --- a/tests/integration/test_gateway_image.py +++ b/tests/integration/test_gateway_image.py @@ -23,6 +23,7 @@ import os import subprocess import unittest +from bot_bottle import resources from tests._backend import skip_unless_backend @@ -38,9 +39,18 @@ class TestGatewayImage(unittest.TestCase): @classmethod def setUpClass(cls) -> None: repo_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + build_args = resources.image_build_args( + _DOCKERFILE, + context=repo_root, + ) + arg_flags = [ + item + for name, value in build_args.items() + for item in ("--build-arg", f"{name}={value}") + ] proc = subprocess.run( ["docker", "build", "-t", _IMAGE, - "-f", _DOCKERFILE, "."], + "-f", _DOCKERFILE, *arg_flags, "."], cwd=repo_root, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False, diff --git a/tests/unit/test_builtin_agent_images.py b/tests/unit/test_builtin_agent_images.py index 860bf851..4799f07f 100644 --- a/tests/unit/test_builtin_agent_images.py +++ b/tests/unit/test_builtin_agent_images.py @@ -2,30 +2,43 @@ from __future__ import annotations +import json import re import unittest from pathlib import Path -_CONTRIB_DIR = Path(__file__).resolve().parents[2] / "bot_bottle/contrib" +_REPO_ROOT = Path(__file__).resolve().parents[2] +_CONTRIB_DIR = _REPO_ROOT / "bot_bottle/contrib" _AGENT_DOCKERFILES = tuple(sorted(_CONTRIB_DIR.glob("*/Dockerfile"))) class TestBuiltinAgentImages(unittest.TestCase): def test_all_share_one_digest_pinned_node_trixie_base(self): self.assertTrue(_AGENT_DOCKERFILES) - bases = [] + inputs = json.loads((_REPO_ROOT / "image-build-args.json").read_text()) + self.assertRegex( + inputs["NODE_BASE_IMAGE"], + r"^node:22\.\d+\.\d+-trixie-slim@sha256:[0-9a-f]{64}$", + ) for dockerfile in _AGENT_DOCKERFILES: with self.subTest(provider=dockerfile.parent.name): - match = re.search( - r"(?m)^FROM " - r"(node:22\.\d+\.\d+-trixie-slim@sha256:[0-9a-f]{64})\s*$", - dockerfile.read_text(), - ) - if match is None: - self.fail(f"{dockerfile} does not use a digest-pinned Node base") - bases.append(match.group(1)) - self.assertEqual(1, len(set(bases))) + text = dockerfile.read_text() + self.assertRegex(text, r"(?m)^ARG NODE_BASE_IMAGE$") + self.assertIn("FROM ${NODE_BASE_IMAGE}", text) + + def test_orchestrator_and_gateway_share_configurable_python_base(self): + inputs = json.loads((_REPO_ROOT / "image-build-args.json").read_text()) + self.assertRegex( + inputs["PYTHON_BASE_IMAGE"], + r"^python:3\.12\.\d+-slim-trixie@sha256:[0-9a-f]{64}$", + ) + for name in ("Dockerfile.orchestrator", "Dockerfile.gateway"): + dockerfile = _REPO_ROOT / name + text = dockerfile.read_text() + with self.subTest(dockerfile=name): + self.assertRegex(text, r"(?m)^ARG PYTHON_BASE_IMAGE$") + self.assertIn("FROM ${PYTHON_BASE_IMAGE}", text) def test_none_install_podman(self): # podman lives in the nested-containers derived layer (nested_containers.py), diff --git a/tests/unit/test_docker_orchestrator.py b/tests/unit/test_docker_orchestrator.py index 82fafed2..604b417b 100644 --- a/tests/unit/test_docker_orchestrator.py +++ b/tests/unit/test_docker_orchestrator.py @@ -221,6 +221,10 @@ class TestDockerOrchestrator(unittest.TestCase): builds = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "build"]] self.assertEqual(1, len(builds)) self.assertTrue(any(a.endswith("Dockerfile.orchestrator") for a in builds[0])) + arg_index = builds[0].index("--build-arg") + self.assertTrue( + builds[0][arg_index + 1].startswith("PYTHON_BASE_IMAGE=python:"), + ) def test_ensure_built_raises_on_build_failure(self) -> None: with patch(_RUN, return_value=_proc(returncode=1, stderr="no space left")): diff --git a/tests/unit/test_docker_util_image.py b/tests/unit/test_docker_util_image.py index bfbdaaed..3b66a74b 100644 --- a/tests/unit/test_docker_util_image.py +++ b/tests/unit/test_docker_util_image.py @@ -122,6 +122,27 @@ class TestCommitContainer(unittest.TestCase): class TestBuildImage(unittest.TestCase): + def test_passes_centralized_image_build_args(self): + with patch.object( + docker_mod.resources, + "image_build_args", + return_value={"NODE_BASE_IMAGE": "node:pinned"}, + ), patch.object( + docker_mod.subprocess, "run", return_value=_ok(), + ) as run, patch.object(docker_mod, "info"): + docker_mod.build_image( + "agent:test", + "/context", + dockerfile="Dockerfile.agent", + ) + self.assertIn( + ["--build-arg", "NODE_BASE_IMAGE=node:pinned"], + [ + run.call_args.args[0][index:index + 2] + for index in range(len(run.call_args.args[0]) - 1) + ], + ) + def test_passes_build_args_without_mutating_the_value(self): with patch.object( docker_mod.subprocess, "run", return_value=_ok(), diff --git a/tests/unit/test_firecracker_image_builder.py b/tests/unit/test_firecracker_image_builder.py index 7cc324db..d8c1c1d1 100644 --- a/tests/unit/test_firecracker_image_builder.py +++ b/tests/unit/test_firecracker_image_builder.py @@ -75,8 +75,39 @@ class TestBuildAgentRootfsDir(unittest.TestCase): other.write_text("FROM python:3.12-slim\n") self.assertNotEqual(base, image_builder._rootfs_digest(other)) + def test_rootfs_digest_tracks_centralized_build_args(self): + with patch.object( + image_builder.resources, + "image_build_args", + return_value={"NODE_BASE_IMAGE": "node:first"}, + ): + first = image_builder._rootfs_digest(self.dockerfile) + with patch.object( + image_builder.resources, + "image_build_args", + return_value={"NODE_BASE_IMAGE": "node:second"}, + ): + second = image_builder._rootfs_digest(self.dockerfile) + self.assertNotEqual(first, second) + class TestSmokeTest(unittest.TestCase): + def test_buildah_receives_centralized_image_build_args(self): + with patch.object( + image_builder, + "_ssh_streamed", + return_value=0, + ) as ssh, patch.object(image_builder, "info"): + image_builder._buildah_build( + Path("/k"), + "10.0.0.1", + "/tmp/context", + "agent:test", + {"NODE_BASE_IMAGE": "node:pinned"}, + ) + script = ssh.call_args.args[2] + self.assertIn("--build-arg NODE_BASE_IMAGE=node:pinned", script) + def test_empty_argv_is_noop(self): with patch.object(image_builder, "_ssh") as ssh: image_builder._smoke_test(Path("/k"), "10.0.0.1", "tag", "ctr", ()) diff --git a/tests/unit/test_image_input_policy.py b/tests/unit/test_image_input_policy.py index 2402cd98..d058b1e1 100644 --- a/tests/unit/test_image_input_policy.py +++ b/tests/unit/test_image_input_policy.py @@ -14,10 +14,26 @@ from scripts import check_image_inputs as policy class TestDockerfilePolicy(unittest.TestCase): + _NODE_BASE = { + "NODE_BASE_IMAGE": + "node:22.23.1-trixie-slim@sha256:" + "a" * 64, + } + def test_accepts_versioned_digest_base(self): text = "FROM node:22.23.1-trixie-slim@sha256:" + "a" * 64 + "\n" self.assertEqual([], policy.check_dockerfile(Path("Dockerfile"), text)) + def test_accepts_defaultless_base_argument_with_centralized_value(self): + text = "ARG NODE_BASE_IMAGE\nFROM ${NODE_BASE_IMAGE}\n" + self.assertEqual( + [], + policy.check_dockerfile( + Path("Dockerfile"), + text, + self._NODE_BASE, + ), + ) + def test_rejects_mutable_and_latest_bases(self): for value in ( "node:22-trixie-slim", @@ -61,7 +77,15 @@ class TestDockerfilePolicy(unittest.TestCase): def test_rejects_missing_dynamic_and_malformed_bases(self): cases = ( ("RUN true\n", "missing FROM"), - ("FROM ${BASE_IMAGE}\n", "dynamic base"), + ("FROM ${BASE_IMAGE}\n", "not declared before FROM"), + ( + "ARG BASE_IMAGE=node:latest\nFROM ${BASE_IMAGE}\n", + "must not define a default", + ), + ( + "ARG BASE_IMAGE\nFROM ${BASE_IMAGE}\n", + "lacks a centralized value", + ), ( "ARG ORCHESTRATOR_BASE_IMAGE=base:latest\n" "FROM ${ORCHESTRATOR_BASE_IMAGE}\n", @@ -74,6 +98,15 @@ class TestDockerfilePolicy(unittest.TestCase): problems = policy.check_dockerfile(Path("Dockerfile"), text) self.assertTrue(any(expected in item for item in problems)) + def test_rejects_mutable_centralized_base_value(self): + text = "ARG NODE_BASE_IMAGE\nFROM ${NODE_BASE_IMAGE}\n" + problems = policy.check_dockerfile( + Path("Dockerfile"), + text, + {"NODE_BASE_IMAGE": "node:latest"}, + ) + self.assertTrue(any("lacks a sha256 digest" in item for item in problems)) + def test_requires_gateway_lock_and_verified_codex_markers(self): base = "FROM node:22.23.1@sha256:" + "a" * 64 + "\n" gateway = policy.check_dockerfile(Path("Dockerfile.gateway"), base) diff --git a/tests/unit/test_infra_artifact.py b/tests/unit/test_infra_artifact.py index 3affe72b..8ada9bd4 100644 --- a/tests/unit/test_infra_artifact.py +++ b/tests/unit/test_infra_artifact.py @@ -110,6 +110,9 @@ class TestVersionInputs(unittest.TestCase): for name in ("Dockerfile.orchestrator", "Dockerfile.orchestrator.fc", "Dockerfile.gateway"): (root / name).write_text(f"FROM scratch # {name}\n") + (root / "image-build-args.json").write_text( + '{"PYTHON_BASE_IMAGE": "python:pinned"}\n', + ) (root / "requirements.gateway.lock").write_text("mitmproxy==11.1.3\n") (root / "pyproject.toml").write_text("[project]\nname = 'bot-bottle'\n") @@ -162,6 +165,23 @@ class TestVersionInputs(unittest.TestCase): ) self.assertNotEqual(before, after) + def test_base_image_argument_change_bumps_both_role_versions(self) -> None: + with tempfile.TemporaryDirectory() as d: + root = Path(d) + self._fake_repo(root) + before = { + role: ia.infra_artifact_version("init", role, repo_root=root) + for role in ia.ROLES + } + (root / "image-build-args.json").write_text( + '{"PYTHON_BASE_IMAGE": "python:different"}\n', + ) + after = { + role: ia.infra_artifact_version("init", role, repo_root=root) + for role in ia.ROLES + } + self.assertTrue(all(before[role] != after[role] for role in ia.ROLES)) + def test_pyc_and_pycache_ignored(self) -> None: with tempfile.TemporaryDirectory() as d: root = Path(d) diff --git a/tests/unit/test_macos_container_util.py b/tests/unit/test_macos_container_util.py index 3a419c87..f3a20708 100644 --- a/tests/unit/test_macos_container_util.py +++ b/tests/unit/test_macos_container_util.py @@ -115,6 +115,37 @@ resolver #2 run.call_args_list[-1].args[0], ) + def test_build_image_passes_centralized_image_build_args(self): + status = util.subprocess.CompletedProcess( + args=[], + returncode=0, + stdout=( + '[{"status":{"state":"running"},' + '"configuration":{"dns":{"nameservers":["9.9.9.9"]}}}]' + ), + stderr="", + ) + with patch.object(util.subprocess, "run", return_value=status) as run, \ + patch.object( + util.resources, + "image_build_args", + return_value={"NODE_BASE_IMAGE": "node:pinned"}, + ), patch.object(util.os, "environ", { + "BOT_BOTTLE_MACOS_CONTAINER_DNS": "9.9.9.9", + }): + util.build_image( + "bot-bottle-agent:latest", + "/repo", + dockerfile="Dockerfile.agent", + ) + self.assertIn( + ["--build-arg", "NODE_BASE_IMAGE=node:pinned"], + [ + run.call_args_list[-1].args[0][index:index + 2] + for index in range(len(run.call_args_list[-1].args[0]) - 1) + ], + ) + def test_commit_container_execs_tar_and_builds_image(self): # stderr is bytes because subprocess.run uses stderr=PIPE without text=True completed = util.subprocess.CompletedProcess( diff --git a/tests/unit/test_orchestrator_gateway.py b/tests/unit/test_orchestrator_gateway.py index 7c8cbd2e..0ad49c7e 100644 --- a/tests/unit/test_orchestrator_gateway.py +++ b/tests/unit/test_orchestrator_gateway.py @@ -323,6 +323,10 @@ class TestDockerGatewayBuild(unittest.TestCase): self.assertEqual(1, len(builds)) self.assertIn(self.sc.image_ref, builds[0]) self.assertTrue(any(a.endswith("Dockerfile.gateway") for a in builds[0])) + arg_index = builds[0].index("--build-arg") + self.assertTrue( + builds[0][arg_index + 1].startswith("PYTHON_BASE_IMAGE=python:"), + ) self.assertNotIn("--no-cache", builds[0]) def test_ensure_built_no_cache_env_forces_full_rebuild(self) -> None: diff --git a/tests/unit/test_resources.py b/tests/unit/test_resources.py index 110353aa..ef1e8f82 100644 --- a/tests/unit/test_resources.py +++ b/tests/unit/test_resources.py @@ -38,6 +38,18 @@ class TestCheckoutMode(unittest.TestCase): self.assertTrue(resources.nix_netpool_module().is_file()) self.assertTrue(resources.netpool_script().is_file()) + def test_image_build_args_come_from_the_central_input_file(self): + root = resources.build_root() + args = resources.image_build_args( + "Dockerfile.gateway", + context=root, + ) + self.assertEqual({"PYTHON_BASE_IMAGE"}, set(args)) + self.assertRegex( + args["PYTHON_BASE_IMAGE"], + r"^python:\d+\.\d+\.\d+-.+@sha256:[0-9a-f]{64}$", + ) + def test_bundled_resources_all_exist_at_root(self): # Drift guard: every path setup.py bundles must exist in the checkout. root = resources.build_root()