feat(firecracker): run the post-build agent-image smoke test too

Closes the gap left in #354: docker and macos-container already
smoke-test a freshly built agent image before launch; firecracker
built the image via its own docker_mod-based path but never ran the
check. Same one-liner as the other two backends now that _build_agent_image
uses docker_mod.build_image.

bot_bottle/agent_provider.py

@@ -61,6 +61,13 @@ class AgentProviderRuntime:
     prompt_mode: PromptMode
     bypass_args: tuple[str, ...]
     resume_args: tuple[str, ...]
+    # argv run inside a throwaway container of a freshly built agent
+    # image, right after `build_image()`, to catch a build that
+    # exited 0 but produced a broken CLI (e.g. an npm
+    # optionalDependencies fetch for a platform-native binary that
+    # silently no-ops on a transient failure). Empty tuple skips the
+    # check — not every provider has opted in yet.
+    smoke_test: tuple[str, ...] = ()
 
 
 @dataclass(frozen=True)

bot_bottle/backend/docker/launch.py

@@ -36,6 +36,7 @@ from contextlib import ExitStack, contextmanager
 from pathlib import Path
 from typing import Callable, Generator
 
+from ...agent_provider import runtime_for
 from ...egress import egress_resolve_token_values
 from ...git_gate import (
     provision_git_gate_dynamic_keys,
@@ -111,6 +112,9 @@ def launch(
                 plan.image, _REPO_DIR,
                 dockerfile=plan.dockerfile_path,
             )
+            docker_mod.verify_agent_image(
+                plan.image, runtime_for(plan.agent_provider_template).smoke_test,
+            )
 
         internal_network = network_mod.network_name_for_slug(plan.slug)
         egress_network = network_mod.network_egress_name_for_slug(plan.slug)

bot_bottle/backend/docker/util.py

@@ -4,6 +4,7 @@ existence, and building images."""
 
 from __future__ import annotations
 
+import os
 import re
 import shutil
 import subprocess
@@ -88,6 +89,31 @@ def docker_exec_root(container: str, argv: list[str]) -> None:
     )
 
 
+def docker_exec(container: str, argv: list[str], *, user: str = "") -> None:
+    """Run `docker exec` in the named container, dying with the
+    command's own stderr on failure. Pass `user=\"0\"` to run as root."""
+    cmd = ["docker", "exec"]
+    if user:
+        cmd += ["-u", user]
+    cmd += [container, *argv]
+    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        die(
+            f"docker exec in {container} failed: "
+            f"{(result.stderr or '').strip() or '<no stderr>'}"
+        )
+
+
+def docker_cp(src: str, dest: str) -> None:
+    """Run `docker cp`, dying with the command's own stderr on failure."""
+    result = subprocess.run(
+        ["docker", "cp", src, dest], capture_output=True, text=True, check=False,
+    )
+    if result.returncode != 0:
+        die(f"docker cp {src} -> {dest} failed: "
+            f"{(result.stderr or '').strip() or '<no stderr>'}")
+
+
 _SLUG_RE = re.compile(r"[^a-z0-9]+")
 
 
@@ -108,15 +134,45 @@ def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
 
     `dockerfile` is an optional path (relative to `context`, or
     absolute) for callers that need to build from a non-default
-    Dockerfile in the same context — e.g. `Dockerfile.git-gate`."""
+    Dockerfile in the same context — e.g. `Dockerfile.git-gate`.
+
+    Set `BOT_BOTTLE_NO_CACHE=1` (the `start --no-cache` flag) to force
+    `--no-cache`. The npm/curl installers some provider Dockerfiles
+    shell out to can silently no-op on a transient network failure —
+    e.g. an `optionalDependencies` fetch for a platform-native binary —
+    and Docker will then cache that broken layer indefinitely."""
     info(f"building image {ref} from {context} (layer cache keeps repeat builds fast)")
     args = ["docker", "build", "-t", ref]
+    if os.environ.get("BOT_BOTTLE_NO_CACHE") == "1":
+        args.append("--no-cache")
     if dockerfile:
         args.extend(["-f", dockerfile])
     args.append(context)
     subprocess.run(args, check=True)
 
 
+def verify_agent_image(image: str, argv: tuple[str, ...]) -> None:
+    """Run `argv` inside a throwaway container of a freshly built agent
+    image and die loudly if it fails, instead of shipping an image
+    whose CLI only breaks at first real use. No-op when the provider
+    hasn't declared a smoke test (`AgentProviderRuntime.smoke_test`)."""
+    if not argv:
+        return
+    result = subprocess.run(
+        ["docker", "run", "--rm", "--entrypoint", argv[0], image, *argv[1:]],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if result.returncode != 0:
+        detail = (result.stderr or result.stdout or "").strip()
+        die(
+            f"agent image {image!r} failed its post-build smoke test "
+            f"({' '.join(argv)}): {detail}\n"
+            f"Try rebuilding from scratch: bot-bottle start --no-cache"
+        )
+
+
 # def build_image_with_cwd(
 #     derived: str,
 #     base: str,

bot_bottle/backend/firecracker/launch.py

@@ -24,6 +24,7 @@ from contextlib import ExitStack, contextmanager
 from pathlib import Path
 from typing import Callable, Generator
 
+from ...agent_provider import runtime_for
 from ...bottle_state import (
     egress_state_dir,
     git_gate_state_dir,
@@ -42,6 +43,7 @@ from ...git_gate import (
 from ...log import die, info, warn
 from ...supervise import DB_PATH_IN_CONTAINER, SUPERVISE_PORT
 from ...util import expand_tilde
+from ..docker import util as docker_mod
 from ..docker.egress import (
     EGRESS_CA_IN_CONTAINER,
     EGRESS_PORT,
@@ -106,9 +108,9 @@ def launch(
         plan = _provision_git_gate_keys(plan)
 
         sidecar_name = sidecar_container_name(plan.slug)
-        _force_remove_container(sidecar_name)
+        docker_mod.force_remove_container(sidecar_name)
         _start_sidecar_bundle(plan, sidecar_name, slot.host_ip)
-        stack.callback(_force_remove_container, sidecar_name)
+        stack.callback(docker_mod.force_remove_container, sidecar_name)
         _stage_git_gate(plan, sidecar_name)
 
         plan = _stamp_agent_urls(plan, slot.host_ip)
@@ -170,15 +172,18 @@ def _mint_certs(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
 
 
 def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
-    _docker_build(SIDECAR_BUNDLE_IMAGE, _REPO_DIR, dockerfile=SIDECAR_BUNDLE_DOCKERFILE)
+    docker_mod.build_image(SIDECAR_BUNDLE_IMAGE, _REPO_DIR, dockerfile=SIDECAR_BUNDLE_DOCKERFILE)
     committed = read_committed_image(plan.slug)
-    if committed and _image_exists(committed):
+    if committed and docker_mod.image_exists(committed):
         info(f"using committed image {committed!r}")
         return dataclasses.replace(
             plan,
             agent_provision=dataclasses.replace(plan.agent_provision, image=committed),
         )
-    _docker_build(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path)
+    docker_mod.build_image(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path)
+    docker_mod.verify_agent_image(
+        plan.image, runtime_for(plan.agent_provider_template).smoke_test,
+    )
     return plan
 
 
@@ -291,15 +296,15 @@ def _stage_git_gate(plan: FirecrackerBottlePlan, sidecar_name: str) -> None:
     gp = plan.git_gate_plan
     if not gp.upstreams:
         return
-    _docker_exec(sidecar_name, [
+    docker_mod.docker_exec(sidecar_name, [
         "mkdir", "-p",
         str(Path(GIT_GATE_HOOK_IN_CONTAINER).parent),
         GIT_GATE_CREDS_DIR_IN_CONTAINER, "/git",
         str(Path(_GIT_GATE_READY_FILE).parent),
     ])
     for host_path, container_path in _git_gate_files(plan):
-        _docker_cp(host_path, f"{sidecar_name}:{container_path}")
-    _docker_exec(sidecar_name, [
+        docker_mod.docker_cp(host_path, f"{sidecar_name}:{container_path}")
+    docker_mod.docker_exec(sidecar_name, [
         "sh", "-c",
         f"chmod 755 {GIT_GATE_ENTRYPOINT_IN_CONTAINER} "
         f"{GIT_GATE_HOOK_IN_CONTAINER} {GIT_GATE_ACCESS_HOOK_IN_CONTAINER} && "
@@ -355,50 +360,3 @@ def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str
         if value is not None:
             env[name] = value
     return env
-
-
-# --- docker helpers --------------------------------------------------
-
-def _docker_build(ref: str, context: str, *, dockerfile: str = "") -> None:
-    info(f"docker build {ref}")
-    args = ["docker", "build", "-t", ref]
-    if dockerfile:
-        if not os.path.isabs(dockerfile):
-            dockerfile = os.path.join(context, dockerfile)
-        args += ["-f", dockerfile]
-    args.append(context)
-    result = subprocess.run(args, check=False)
-    if result.returncode != 0:
-        die(f"docker build for {ref!r} failed")
-
-
-def _image_exists(ref: str) -> bool:
-    return subprocess.run(
-        ["docker", "image", "inspect", ref],
-        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
-    ).returncode == 0
-
-
-def _force_remove_container(name: str) -> None:
-    subprocess.run(
-        ["docker", "rm", "-f", name],
-        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
-    )
-
-
-def _docker_exec(name: str, argv: list[str]) -> None:
-    result = subprocess.run(
-        ["docker", "exec", name, *argv], capture_output=True, text=True, check=False,
-    )
-    if result.returncode != 0:
-        die(f"docker exec in {name} failed: "
-            f"{(result.stderr or '').strip() or '<no stderr>'}")
-
-
-def _docker_cp(host_path: str, dest: str) -> None:
-    result = subprocess.run(
-        ["docker", "cp", host_path, dest], capture_output=True, text=True, check=False,
-    )
-    if result.returncode != 0:
-        die(f"docker cp {host_path} -> {dest} failed: "
-            f"{(result.stderr or '').strip() or '<no stderr>'}")

bot_bottle/backend/macos_container/launch.py

@@ -17,6 +17,7 @@ from contextlib import ExitStack, contextmanager
 from pathlib import Path
 from typing import Callable, Generator
 
+from ...agent_provider import runtime_for
 from ...bottle_state import (
     egress_state_dir,
     git_gate_state_dir,
@@ -169,6 +170,9 @@ def _build_images(plan: MacosContainerBottlePlan) -> MacosContainerBottlePlan:
         _REPO_DIR,
         dockerfile=plan.dockerfile_path,
     )
+    container_mod.verify_agent_image(
+        plan.image, runtime_for(plan.agent_provider_template).smoke_test,
+    )
     return plan
 
 

bot_bottle/backend/macos_container/util.py

@@ -60,13 +60,21 @@ def dns_server() -> str:
 
 
 def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
-    """Build an OCI image with Apple's BuildKit-backed `container build`."""
+    """Build an OCI image with Apple's BuildKit-backed `container build`.
+
+    Set `BOT_BOTTLE_NO_CACHE=1` (the `start --no-cache` flag) to force
+    `--no-cache`. The npm/curl installers some provider Dockerfiles
+    shell out to can silently no-op on a transient network failure —
+    e.g. an `optionalDependencies` fetch for a platform-native binary —
+    and the builder will then cache that broken layer indefinitely."""
     info(
         f"building image {ref} from {context} with Apple Container "
         "(layer cache keeps repeat builds fast)"
     )
     _ensure_builder_dns()
     args = [_CONTAINER, "build", "-t", ref, "--dns", dns_server()]
+    if os.environ.get("BOT_BOTTLE_NO_CACHE") == "1":
+        args.append("--no-cache")
     if dockerfile:
         # `container build` resolves -f relative to the current working
         # directory, not the build context. Anchor a relative Dockerfile to
@@ -78,6 +86,28 @@ def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
     subprocess.run(args, check=True)
 
 
+def verify_agent_image(image: str, argv: tuple[str, ...]) -> None:
+    """Run `argv` inside a throwaway container of a freshly built agent
+    image and die loudly if it fails, instead of shipping an image
+    whose CLI only breaks at first real use. No-op when the provider
+    hasn't declared a smoke test (`AgentProviderRuntime.smoke_test`)."""
+    if not argv:
+        return
+    result = subprocess.run(
+        [_CONTAINER, "run", "--rm", "--entrypoint", argv[0], image, *argv[1:]],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if result.returncode != 0:
+        detail = (result.stderr or result.stdout or "").strip()
+        die(
+            f"agent image {image!r} failed its post-build smoke test "
+            f"({' '.join(argv)}): {detail}\n"
+            f"Try rebuilding from scratch: bot-bottle start --no-cache"
+        )
+
+
 def commit_container(container_name: str, image_tag: str) -> None:
     """Snapshot a running Apple Container as a local image.
 

bot_bottle/cli/start.py

@@ -46,6 +46,17 @@ def cmd_start(argv: list[str]) -> int:
     parser = argparse.ArgumentParser(prog=f"{PROG} start", add_help=True)
     parser.add_argument("--dry-run", action="store_true")
     parser.add_argument("--cwd", action="store_true", help="copy host cwd into the running bottle")
+    parser.add_argument(
+        "--no-cache",
+        action="store_true",
+        help=(
+            "rebuild agent/sidecar images from scratch, bypassing the "
+            "build layer cache. Use when an image looks broken after a "
+            "dependency bump — e.g. an installer's optionalDependencies "
+            "fetch silently no-op'd on a transient failure and got baked "
+            "into a cached layer."
+        ),
+    )
     parser.add_argument(
         "--backend",
         choices=known_backend_names(),
@@ -97,6 +108,11 @@ def cmd_start(argv: list[str]) -> int:
     args = parser.parse_args(argv)
 
     dry_run = args.dry_run or os.environ.get("BOT_BOTTLE_DRY_RUN") == "1"
+    if args.no_cache or os.environ.get("BOT_BOTTLE_NO_CACHE") == "1":
+        # Read by build_image() in each backend's util module — set here
+        # so both the interactive and --headless paths pick it up without
+        # threading a no_cache field through every backend's plan dataclass.
+        os.environ["BOT_BOTTLE_NO_CACHE"] = "1"
 
     manifest = ManifestIndex.resolve(USER_CWD)
     backend_name: str | None = args.backend

bot_bottle/contrib/claude/agent_provider.py

@@ -91,6 +91,7 @@ _RUNTIME = AgentProviderRuntime(
     prompt_mode="append_file",
     bypass_args=("--dangerously-skip-permissions",),
     resume_args=("--continue",),
+    smoke_test=("claude", "--version"),
 )
 
 

bot_bottle/contrib/codex/agent_provider.py

@@ -61,6 +61,7 @@ _RUNTIME = AgentProviderRuntime(
     prompt_mode="read_prompt_file",
     bypass_args=("--dangerously-bypass-approvals-and-sandbox",),
     resume_args=("resume", "--last"),
+    smoke_test=(_CODEX_CLI, "--version"),
 )
 
 

tests/unit/test_docker_launch_committed_image.py

@@ -103,6 +103,8 @@ class TestLaunchCommittedImage(unittest.TestCase):
             launch_mod.docker_mod, "image_exists", return_value=image_present,
         ), mock.patch.object(
             launch_mod.docker_mod, "build_image", side_effect=fake_build,
+        ), mock.patch.object(
+            launch_mod.docker_mod, "verify_agent_image",
         ), mock.patch.object(
             launch_mod, "egress_tls_init",
             return_value=(Path("/egress_ca"), Path("/egress_cert")),

tests/unit/test_docker_launch_teardown.py

@@ -87,6 +87,7 @@ class TestTeardownWarning(unittest.TestCase):
         buf = io.StringIO()
 
         with mock.patch.object(launch_mod.docker_mod, "build_image"), \
+             mock.patch.object(launch_mod.docker_mod, "verify_agent_image"), \
              mock.patch.object(
                  launch_mod, "egress_tls_init",
                  return_value=(Path("/egress_ca"), Path("/egress_cert")),

tests/unit/test_macos_container_launch.py

@@ -357,12 +357,17 @@ class TestMacosContainerLaunchCommittedImage(unittest.TestCase):
             launch.container_mod, "image_exists", return_value=False,
         ), patch.object(
             launch.container_mod, "build_image", side_effect=fake_build,
-        ):
+        ), patch.object(
+            launch.container_mod, "verify_agent_image",
+        ) as verify:
             updated = launch._build_images(plan)
 
         self.assertEqual("bot-bottle-agent:latest", updated.image)
         self.assertEqual(2, len(calls))
         self.assertEqual("bot-bottle-agent:latest", calls[1][0])
+        verify.assert_called_once_with(
+            "bot-bottle-agent:latest", ("claude", "--version"),
+        )
 
 
 if __name__ == "__main__":