From 73c566f3ff44c86ac2fef92b1e54a8e5b7755d70 Mon Sep 17 00:00:00 2001 From: codex Date: Sun, 26 Jul 2026 20:41:29 +0000 Subject: [PATCH] fix(build): close remaining mutable image inputs --- .gitea/workflows/lint.yml | 2 +- .gitea/workflows/refresh-image-locks.yml | 12 ++---- bot_bottle/backend/macos_container/launch.py | 6 ++- .../macos_container/nested_containers.py | 7 +++- bot_bottle/backend/macos_container/util.py | 34 ++++++++++++++++ bot_bottle/contrib/codex/Dockerfile | 40 ++++++++++++------- .../contrib/codex/codex-package_SHA256SUMS | 2 + bot_bottle/contrib/codex/install.sh.sha256 | 1 - bot_bottle/resources.py | 18 ++++++++- docs/image-build-inputs.md | 9 +++-- image-build-args.json | 1 + pyproject.toml | 2 +- scripts/check_image_inputs.py | 30 ++++++++++++-- tests/unit/test_macos_container_util.py | 23 +++++++++++ tests/unit/test_macos_nested_containers.py | 26 +++++++++--- tests/unit/test_resources.py | 15 +++++++ 16 files changed, 184 insertions(+), 44 deletions(-) create mode 100644 bot_bottle/contrib/codex/codex-package_SHA256SUMS delete mode 100644 bot_bottle/contrib/codex/install.sh.sha256 diff --git a/.gitea/workflows/lint.yml b/.gitea/workflows/lint.yml index db663ac6..614728dc 100644 --- a/.gitea/workflows/lint.yml +++ b/.gitea/workflows/lint.yml @@ -8,7 +8,7 @@ on: - "bot_bottle/contrib/*/Dockerfile" - "bot_bottle/contrib/*/package.json" - "bot_bottle/contrib/*/package-lock.json" - - "bot_bottle/contrib/codex/install.sh.sha256" + - "bot_bottle/contrib/codex/codex-package_SHA256SUMS" - "requirements.gateway.*" - "image-build-args.json" - ".pylintrc" diff --git a/.gitea/workflows/refresh-image-locks.yml b/.gitea/workflows/refresh-image-locks.yml index 4b650892..05d26bc7 100644 --- a/.gitea/workflows/refresh-image-locks.yml +++ b/.gitea/workflows/refresh-image-locks.yml @@ -76,19 +76,15 @@ jobs: bot_bottle/contrib/claude/package-lock.json \ bot_bottle/contrib/pi/package-lock.json - - name: Refresh pinned Codex installer checksum + - name: Refresh pinned Codex archive checksums run: | CODEX_VERSION=$( sed -n 's/^ARG CODEX_VERSION=//p' bot_bottle/contrib/codex/Dockerfile ) test -n "$CODEX_VERSION" curl -fsSL \ - "https://raw.githubusercontent.com/openai/codex/rust-v${CODEX_VERSION}/scripts/install/install.sh" \ - -o bot_bottle/contrib/codex/install.sh - ( - cd bot_bottle/contrib/codex - sha256sum install.sh > install.sh.sha256 - ) + "https://github.com/openai/codex/releases/download/rust-v${CODEX_VERSION}/codex-package_SHA256SUMS" \ + -o bot_bottle/contrib/codex/codex-package_SHA256SUMS - name: Upload refreshed inputs uses: actions/upload-artifact@v3 @@ -98,4 +94,4 @@ jobs: requirements.gateway.lock bot_bottle/contrib/claude/package-lock.json bot_bottle/contrib/pi/package-lock.json - bot_bottle/contrib/codex/install.sh.sha256 + bot_bottle/contrib/codex/codex-package_SHA256SUMS diff --git a/bot_bottle/backend/macos_container/launch.py b/bot_bottle/backend/macos_container/launch.py index e3b87fa4..ae9faf85 100644 --- a/bot_bottle/backend/macos_container/launch.py +++ b/bot_bottle/backend/macos_container/launch.py @@ -116,7 +116,11 @@ def _layer_nested_containers( ) info(f"using cached nested-container image {derived!r}") return derived - return nested_containers_mod.build_image(agent_image, container_mod.build_image) + return nested_containers_mod.build_image( + agent_image, + container_mod.build_image, + container_mod.pinned_local_image_ref, + ) @contextmanager diff --git a/bot_bottle/backend/macos_container/nested_containers.py b/bot_bottle/backend/macos_container/nested_containers.py index b8d4292f..7b17c59b 100644 --- a/bot_bottle/backend/macos_container/nested_containers.py +++ b/bot_bottle/backend/macos_container/nested_containers.py @@ -57,6 +57,7 @@ _GUEST_DEVICES = ("/dev/fuse", "/dev/net/tun") def build_image( base_image: str, build: Callable[..., None], + pin_local_base: Callable[[str], str], ) -> str: """Layer the nested-container tooling onto an already-built agent image. @@ -67,13 +68,15 @@ def build_image( # abstraction once that infrastructure exists. """ image = f"{base_image}{IMAGE_SUFFIX}" + pinned_base = pin_local_base(base_image) init_script = Path(__file__).with_name("nested-containers-init.sh") with tempfile.TemporaryDirectory(prefix="bot-bottle-nested-containers.") as tmp: context = Path(tmp) shutil.copy2(init_script, context / "nested-containers-init.sh") (context / "Dockerfile").write_text( - "FROM docker:28-cli AS docker_cli\n" - f"FROM {base_image}\n" + "ARG DOCKER_CLI_BASE_IMAGE\n" + "FROM ${DOCKER_CLI_BASE_IMAGE} AS docker_cli\n" + f"FROM {pinned_base}\n" "USER root\n" "COPY --from=docker_cli /usr/local/bin/docker /usr/local/bin/docker\n" "COPY --from=docker_cli /usr/local/libexec/docker/cli-plugins/" diff --git a/bot_bottle/backend/macos_container/util.py b/bot_bottle/backend/macos_container/util.py index 34794017..0794d01e 100644 --- a/bot_bottle/backend/macos_container/util.py +++ b/bot_bottle/backend/macos_container/util.py @@ -682,6 +682,40 @@ def image_id(ref: str) -> str: raise AssertionError("unreachable") +def pinned_local_image_ref(ref: str) -> str: + """Tag a local image with its complete content ID for a stable ``FROM``. + + Agent images are immediately used as bases for the optional + nested-containers layer. A content-derived tag prevents another concurrent + build from moving the provider's ordinary ``:latest`` tag between those + two builds. + """ + image = image_id(ref) + digest = image.removeprefix("sha256:") + if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest): + die(f"could not derive a local base tag from invalid image ID {image!r}") + repository = ref.split("@", 1)[0] + last_slash = repository.rfind("/") + last_colon = repository.rfind(":") + if last_colon > last_slash: + repository = repository[:last_colon] + pinned_ref = f"{repository}:sha256-{digest}" + result = subprocess.run( + [_CONTAINER, "image", "tag", image, pinned_ref], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + die( + f"could not tag exact local image {image}: " + f"{(result.stderr or result.stdout or '').strip() or ''}" + ) + if image_id(pinned_ref) != image: + die(f"content-derived local tag {pinned_ref!r} did not resolve to {image}") + return pinned_ref + + def image_created_at(ref: str) -> datetime | None: """Return the image creation timestamp as an aware UTC datetime, or None when the field is absent or unparseable (e.g. FROM-scratch images, images diff --git a/bot_bottle/contrib/codex/Dockerfile b/bot_bottle/contrib/codex/Dockerfile index 9adb2b38..c80111f3 100644 --- a/bot_bottle/contrib/codex/Dockerfile +++ b/bot_bottle/contrib/codex/Dockerfile @@ -6,9 +6,8 @@ ARG NODE_BASE_IMAGE FROM ${NODE_BASE_IMAGE} -# The standalone installer is used below because remote-control requires its -# managed package layout. Keep this exact release in sync with the verified -# installer source and checksum. +# Remote-control requires the standalone package layout. Keep this exact release +# in sync with the committed upstream archive checksums. ARG CODEX_VERSION=0.145.0 ARG DEBIAN_SNAPSHOT=20260724T000000Z @@ -44,18 +43,29 @@ WORKDIR /home/node ENV PATH="/home/node/.local/bin:${PATH}" -# Remote-control support requires the standalone Codex install layout under -# ~/.codex/packages/standalone/current. Fetch the installer from the same -# immutable release tag as the requested CLI, verify the committed checksum -# before executing it, and let the official installer verify the selected -# release archive against upstream release metadata. -COPY --chown=node:node bot_bottle/contrib/codex/install.sh.sha256 /tmp/install.sh.sha256 -RUN curl -fsSL \ - "https://raw.githubusercontent.com/openai/codex/rust-v${CODEX_VERSION}/scripts/install/install.sh" \ - -o /tmp/install.sh \ - && cd /tmp \ - && sha256sum -c install.sh.sha256 \ - && CODEX_NON_INTERACTIVE=1 sh /tmp/install.sh --release "${CODEX_VERSION}" \ +# Install the exact standalone release archive selected by the target +# architecture. The checksum list is copied from the immutable upstream release +# and committed so a rebuild cannot silently accept changed release bytes. +COPY --chown=node:node bot_bottle/contrib/codex/codex-package_SHA256SUMS /tmp/codex-package_SHA256SUMS +RUN case "$(dpkg --print-architecture)" in \ + amd64) codex_target=x86_64-unknown-linux-musl ;; \ + arm64) codex_target=aarch64-unknown-linux-musl ;; \ + *) echo "unsupported Codex architecture: $(dpkg --print-architecture)" >&2; exit 1 ;; \ + esac \ + && codex_asset="codex-package-${codex_target}.tar.gz" \ + && codex_sha256="$(awk -v asset="${codex_asset}" '$2 == asset { print $1 }' /tmp/codex-package_SHA256SUMS)" \ + && test -n "${codex_sha256}" \ + && curl -fsSL \ + "https://github.com/openai/codex/releases/download/rust-v${CODEX_VERSION}/${codex_asset}" \ + -o "/tmp/${codex_asset}" \ + && echo "${codex_sha256} /tmp/${codex_asset}" | sha256sum -c - \ + && codex_release="/home/node/.codex/packages/standalone/releases/${CODEX_VERSION}-${codex_target}" \ + && mkdir -p "${codex_release}" /home/node/.local/bin \ + && tar -xzf "/tmp/${codex_asset}" -C "${codex_release}" \ + && ln -s bin/codex "${codex_release}/codex" \ + && ln -s "${codex_release}" /home/node/.codex/packages/standalone/current \ + && ln -s /home/node/.codex/packages/standalone/current/bin/codex /home/node/.local/bin/codex \ + && rm "/tmp/${codex_asset}" \ && test "$(codex --version)" = "codex-cli ${CODEX_VERSION}" CMD ["codex"] diff --git a/bot_bottle/contrib/codex/codex-package_SHA256SUMS b/bot_bottle/contrib/codex/codex-package_SHA256SUMS new file mode 100644 index 00000000..7c2121fa --- /dev/null +++ b/bot_bottle/contrib/codex/codex-package_SHA256SUMS @@ -0,0 +1,2 @@ +54f79a05aba6f9abf8ef988abcae8bf2fcefba20beb549b4ff2b3acdb2cb6f54 codex-package-aarch64-unknown-linux-musl.tar.gz +71a28d362c96ac9829bf8203a2c71be451aeb726adb843167fdaf0eae8fe7dd9 codex-package-x86_64-unknown-linux-musl.tar.gz diff --git a/bot_bottle/contrib/codex/install.sh.sha256 b/bot_bottle/contrib/codex/install.sh.sha256 deleted file mode 100644 index 0eb7b946..00000000 --- a/bot_bottle/contrib/codex/install.sh.sha256 +++ /dev/null @@ -1 +0,0 @@ -1154e9daf713aacd1534efca8042bfd6665ad24bc1d1dfd86b8f439fe60a7a5d install.sh diff --git a/bot_bottle/resources.py b/bot_bottle/resources.py index 0d8e6581..001fda7b 100644 --- a/bot_bottle/resources.py +++ b/bot_bottle/resources.py @@ -52,7 +52,11 @@ BUNDLED_RESOURCES: tuple[str, ...] = ( # tell for which layout we're in. _CHECKOUT_MARKER = "Dockerfile.gateway" _IMAGE_BUILD_ARGS_FILE = "image-build-args.json" -_CENTRAL_BUILD_ARG_NAMES = frozenset({"NODE_BASE_IMAGE", "PYTHON_BASE_IMAGE"}) +_CENTRAL_BUILD_ARG_NAMES = frozenset({ + "DOCKER_CLI_BASE_IMAGE", + "NODE_BASE_IMAGE", + "PYTHON_BASE_IMAGE", +}) class ResourceError(RuntimeError): @@ -112,7 +116,17 @@ def image_build_args( wanted = declared & _CENTRAL_BUILD_ARG_NAMES if not wanted: return {} - root = Path(context) if context is not None else build_root() + # Generated Dockerfiles (notably the macOS nested-container layer) live in + # temporary build contexts. Resolve their declaration there, but take the + # centralized values from that context only when it carries its own input + # file; otherwise use bot-bottle's staged build root. + context_root = Path(context) if context is not None else None + root = ( + context_root + if context_root is not None + and (context_root / _IMAGE_BUILD_ARGS_FILE).is_file() + else build_root() + ) inputs_path = root / _IMAGE_BUILD_ARGS_FILE try: inputs = json.loads(inputs_path.read_text(encoding="utf-8")) diff --git a/docs/image-build-inputs.md b/docs/image-build-inputs.md index e49b3fa8..3c1987e2 100644 --- a/docs/image-build-inputs.md +++ b/docs/image-build-inputs.md @@ -4,7 +4,8 @@ Bot-bottle's supported images are intended to rebuild from the same declared inputs on Linux amd64 and arm64. The repository enforces four layers of immutability: -- Python and Node base images are required, defaultless Docker build arguments. +- Python, Node, and nested-container Docker CLI base images are required, + defaultless Docker build arguments. Their version-qualified tags and multi-platform OCI index digests live together in `image-build-args.json`; every supported builder reads that file and passes only the arguments its Dockerfile declares. @@ -16,9 +17,9 @@ immutability: - Gateway Python dependencies install from `requirements.gateway.lock` with `pip --require-hashes`; provider npm packages install with `npm ci` from committed lockfiles. -- The Codex standalone installer is fetched from the selected Codex release - tag, checked against `install.sh.sha256`, invoked with an exact `--release`, - and then verifies the selected release archive against upstream metadata. +- The Codex standalone archive is fetched from the exact `CODEX_VERSION` + release and checked against the committed upstream + `codex-package_SHA256SUMS` before extraction. The standalone layout is retained because Codex remote control requires it. `Dockerfile.orchestrator.fc` is the one intentionally dynamic `FROM`. It has no diff --git a/image-build-args.json b/image-build-args.json index 2247da49..17471436 100644 --- a/image-build-args.json +++ b/image-build-args.json @@ -1,4 +1,5 @@ { + "DOCKER_CLI_BASE_IMAGE": "docker:28-cli@sha256:625d9431a9f54c5a2bc90f24f0e1c3d55b1349fd857dd85035f98c2c9acbdd4d", "NODE_BASE_IMAGE": "node:22.23.1-trixie-slim@sha256:e6d9a389d34ff9678438af985c9913fbd1eb6ed36e80fea56644f4b4f6dd70ba", "PYTHON_BASE_IMAGE": "python:3.12.13-slim-trixie@sha256:57cd7c3a7a273101a6485ba99423ee568157882804b1124b4dd04266317710de" } diff --git a/pyproject.toml b/pyproject.toml index d7cb0e63..9a2a1308 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ bot_bottle = [ "contrib/claude/package.json", "contrib/claude/package-lock.json", "contrib/codex/Dockerfile", - "contrib/codex/install.sh.sha256", + "contrib/codex/codex-package_SHA256SUMS", "contrib/pi/Dockerfile", "contrib/pi/package.json", "contrib/pi/package-lock.json", diff --git a/scripts/check_image_inputs.py b/scripts/check_image_inputs.py index dac6e011..464bc4a4 100644 --- a/scripts/check_image_inputs.py +++ b/scripts/check_image_inputs.py @@ -22,7 +22,11 @@ NPM_MANIFESTS = ( Path("bot_bottle/contrib/pi/package.json"), ) IMAGE_BUILD_ARGS = Path("image-build-args.json") -REQUIRED_BASE_ARGS = frozenset({"NODE_BASE_IMAGE", "PYTHON_BASE_IMAGE"}) +REQUIRED_BASE_ARGS = frozenset({ + "DOCKER_CLI_BASE_IMAGE", + "NODE_BASE_IMAGE", + "PYTHON_BASE_IMAGE", +}) _DIGEST = re.compile(r"^[0-9a-f]{64}$") _EXACT_NPM_VERSION = re.compile( @@ -127,9 +131,9 @@ def check_dockerfile( problems.append(f"{path}: gateway Python lock is not consumed") if path == Path("bot_bottle/contrib/codex/Dockerfile"): required = ( - "install.sh.sha256", + "codex-package_SHA256SUMS", "sha256sum -c", - '--release "${CODEX_VERSION}"', + "codex-package-${codex_target}.tar.gz", ) for marker in required: if marker not in instructions: @@ -256,6 +260,26 @@ def check_repo(root: Path = REPO_ROOT) -> list[str]: problems.extend(check_dockerfile(path, text, image_build_args)) for manifest in NPM_MANIFESTS: problems.extend(check_npm_manifest(root, manifest)) + nested_path = Path( + "bot_bottle/backend/macos_container/nested_containers.py", + ) + try: + nested_source = (root / nested_path).read_text(encoding="utf-8") + except OSError as exc: + problems.append(f"{nested_path}: cannot read generated image source: {exc}") + else: + if "FROM docker:" in nested_source: + problems.append( + "nested-containers image uses a mutable Docker CLI base", + ) + if '"ARG DOCKER_CLI_BASE_IMAGE\\n"' not in nested_source: + problems.append( + "nested-containers image does not consume DOCKER_CLI_BASE_IMAGE", + ) + if "pin_local_base(base_image)" not in nested_source: + problems.append( + "nested-containers image does not pin its local agent base", + ) problems.extend(check_python_lock(root)) return problems diff --git a/tests/unit/test_macos_container_util.py b/tests/unit/test_macos_container_util.py index f3a20708..e1ee1004 100644 --- a/tests/unit/test_macos_container_util.py +++ b/tests/unit/test_macos_container_util.py @@ -146,6 +146,29 @@ resolver #2 ], ) + def test_pinned_local_image_ref_tags_and_verifies_exact_id(self): + image = "sha256:" + "a" * 64 + completed = util.subprocess.CompletedProcess( + args=[], returncode=0, stdout="", stderr="", + ) + with patch.object(util, "image_id", return_value=image) as inspect, \ + patch.object(util.subprocess, "run", return_value=completed) as run: + pinned = util.pinned_local_image_ref("registry:5000/agent:latest") + self.assertEqual( + f"registry:5000/agent:sha256-{'a' * 64}", + pinned, + ) + run.assert_called_once_with( + ["container", "image", "tag", image, pinned], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual( + [("registry:5000/agent:latest",), (pinned,)], + [call.args for call in inspect.call_args_list], + ) + def test_commit_container_execs_tar_and_builds_image(self): # stderr is bytes because subprocess.run uses stderr=PIPE without text=True completed = util.subprocess.CompletedProcess( diff --git a/tests/unit/test_macos_nested_containers.py b/tests/unit/test_macos_nested_containers.py index 731dc320..6cc5bff1 100644 --- a/tests/unit/test_macos_nested_containers.py +++ b/tests/unit/test_macos_nested_containers.py @@ -107,12 +107,16 @@ class TestNestedContainersImage(unittest.TestCase): def build(image: str, context: str, *, dockerfile: str) -> None: calls.append((image, context, dockerfile)) text = Path(dockerfile).read_text(encoding="utf-8") - self.assertIn("FROM agent:base", text) + self.assertIn("ARG DOCKER_CLI_BASE_IMAGE", text) + self.assertIn("FROM ${DOCKER_CLI_BASE_IMAGE} AS docker_cli", text) + self.assertIn("FROM agent:sha256-deadbeef", text) self.assertIn("aardvark-dns fuse-overlayfs netavark nftables passt podman", text) self.assertIn("USER node", text) self.assertTrue((Path(context) / "nested-containers-init.sh").is_file()) - image = nested_containers.build_image("agent:base", build) + image = nested_containers.build_image( + "agent:base", build, lambda _ref: "agent:sha256-deadbeef", + ) self.assertEqual("agent:base-nested-containers", image) self.assertEqual("agent:base-nested-containers", calls[0][0]) @@ -127,7 +131,9 @@ class TestNestedContainersImage(unittest.TestCase): def build(_image: str, _context: str, *, dockerfile: str) -> None: seen.append(Path(dockerfile).read_text(encoding="utf-8")) - nested_containers.build_image("agent:base", build) + nested_containers.build_image( + "agent:base", build, lambda _ref: "agent:sha256-deadbeef", + ) for package in ("podman", "passt", "nftables", "aardvark-dns"): self.assertIn(package, seen[0]) @@ -143,7 +149,9 @@ class TestNestedContainersImage(unittest.TestCase): def build(_image: str, _context: str, *, dockerfile: str) -> None: seen.append(Path(dockerfile).read_text(encoding="utf-8")) - nested_containers.build_image("agent:base", build) + nested_containers.build_image( + "agent:base", build, lambda _ref: "agent:sha256-deadbeef", + ) text = seen[0] self.assertIn("sed -i '/^node:/d' /etc/subuid /etc/subgid", text) self.assertNotIn("subuid", text.replace( @@ -286,7 +294,9 @@ class TestBuildOrLoadImages(unittest.TestCase): "agent:base", str(launch_mod.resources.build_root()), dockerfile="/repo/Dockerfile", ) - derived.assert_called_once_with("agent:base", build) + derived.assert_called_once_with( + "agent:base", build, launch_mod.container_mod.pinned_local_image_ref, + ) self.assertEqual("agent:base-nested-containers", images.agent) def test_derived_image_layers_onto_a_committed_image(self) -> None: @@ -307,7 +317,11 @@ class TestBuildOrLoadImages(unittest.TestCase): images = launch_mod.build_or_load_images(plan) build.assert_not_called() - derived.assert_called_once_with("agent:committed", build) + derived.assert_called_once_with( + "agent:committed", + build, + launch_mod.container_mod.pinned_local_image_ref, + ) self.assertEqual("agent:committed-nested-containers", images.agent) def test_cached_policy_refuses_to_build_the_derived_image(self) -> None: diff --git a/tests/unit/test_resources.py b/tests/unit/test_resources.py index ef1e8f82..4815b79c 100644 --- a/tests/unit/test_resources.py +++ b/tests/unit/test_resources.py @@ -50,6 +50,21 @@ class TestCheckoutMode(unittest.TestCase): r"^python:\d+\.\d+\.\d+-.+@sha256:[0-9a-f]{64}$", ) + def test_generated_dockerfile_uses_central_input_from_build_root(self): + with tempfile.TemporaryDirectory() as tmp: + dockerfile = Path(tmp) / "Dockerfile" + dockerfile.write_text( + "ARG DOCKER_CLI_BASE_IMAGE\n" + "FROM ${DOCKER_CLI_BASE_IMAGE}\n", + encoding="utf-8", + ) + args = resources.image_build_args(dockerfile, context=tmp) + self.assertEqual({"DOCKER_CLI_BASE_IMAGE"}, set(args)) + self.assertRegex( + args["DOCKER_CLI_BASE_IMAGE"], + r"^docker:\d+-cli@sha256:[0-9a-f]{64}$", + ) + def test_bundled_resources_all_exist_at_root(self): # Drift guard: every path setup.py bundles must exist in the checkout. root = resources.build_root()