From 1d3e9e859cfe2d0f5ac49dea02b3e709b621d2bc Mon Sep 17 00:00:00 2001
From: didericis <eric@dideric.is>
Date: Mon, 13 Jul 2026 04:04:08 -0400
Subject: [PATCH 1/6] fix: smoke-test agent images after build, add start
 --no-cache
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

npm treats optionalDependencies failures as non-fatal, so a transient
network blip fetching claude-code's platform-native binary during
`npm install -g` left a stub CLI in an image that still "built"
successfully — then got baked into the Docker/Container layer cache
until forced to rebuild. Post-build smoke test (provider-declared
argv, run in a throwaway container of the freshly built image) fails
the launch loudly instead of shipping a broken image; --no-cache
gives an escape hatch to force a from-scratch rebuild.

Closes #353.
---
 bot_bottle/agent_provider.py                  |  7 ++++
 bot_bottle/backend/docker/launch.py           |  4 +++
 bot_bottle/backend/docker/util.py             | 33 ++++++++++++++++++-
 bot_bottle/backend/macos_container/util.py    | 32 +++++++++++++++++-
 bot_bottle/cli/start.py                       | 16 +++++++++
 bot_bottle/contrib/claude/agent_provider.py   |  1 +
 bot_bottle/contrib/codex/agent_provider.py    |  1 +
 .../test_docker_launch_committed_image.py     |  2 ++
 tests/unit/test_docker_launch_teardown.py     |  1 +
 9 files changed, 95 insertions(+), 2 deletions(-)

diff --git a/bot_bottle/agent_provider.py b/bot_bottle/agent_provider.py
index 4f26726..f6b0425 100644
--- a/bot_bottle/agent_provider.py
+++ b/bot_bottle/agent_provider.py
@@ -61,6 +61,13 @@ class AgentProviderRuntime:
     prompt_mode: PromptMode
     bypass_args: tuple[str, ...]
     resume_args: tuple[str, ...]
+    # argv run inside a throwaway container of a freshly built agent
+    # image, right after `build_image()`, to catch a build that
+    # exited 0 but produced a broken CLI (e.g. an npm
+    # optionalDependencies fetch for a platform-native binary that
+    # silently no-ops on a transient failure). Empty tuple skips the
+    # check — not every provider has opted in yet.
+    smoke_test: tuple[str, ...] = ()
 
 
 @dataclass(frozen=True)
diff --git a/bot_bottle/backend/docker/launch.py b/bot_bottle/backend/docker/launch.py
index def5036..e99d87c 100644
--- a/bot_bottle/backend/docker/launch.py
+++ b/bot_bottle/backend/docker/launch.py
@@ -36,6 +36,7 @@ from contextlib import ExitStack, contextmanager
 from pathlib import Path
 from typing import Callable, Generator
 
+from ...agent_provider import runtime_for
 from ...egress import egress_resolve_token_values
 from ...git_gate import (
     provision_git_gate_dynamic_keys,
@@ -110,6 +111,9 @@ def launch(
                 plan.image, _REPO_DIR,
                 dockerfile=plan.dockerfile_path,
             )
+            docker_mod.verify_agent_image(
+                plan.image, runtime_for(plan.agent_provider_template).smoke_test,
+            )
 
         # Step 2: mint the git-gate dynamic (gitea) deploy keys, if any, before
         # provisioning the bottle's repos into the shared gateway.
diff --git a/bot_bottle/backend/docker/util.py b/bot_bottle/backend/docker/util.py
index 6a4abdd..e827c24 100644
--- a/bot_bottle/backend/docker/util.py
+++ b/bot_bottle/backend/docker/util.py
@@ -4,6 +4,7 @@ existence, and building images."""
 
 from __future__ import annotations
 
+import os
 import re
 import shutil
 import subprocess
@@ -108,15 +109,45 @@ def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
 
     `dockerfile` is an optional path (relative to `context`, or
     absolute) for callers that need to build from a non-default
-    Dockerfile in the same context — e.g. `Dockerfile.git-gate`."""
+    Dockerfile in the same context — e.g. `Dockerfile.git-gate`.
+
+    Set `BOT_BOTTLE_NO_CACHE=1` (the `start --no-cache` flag) to force
+    `--no-cache`. The npm/curl installers some provider Dockerfiles
+    shell out to can silently no-op on a transient network failure —
+    e.g. an `optionalDependencies` fetch for a platform-native binary —
+    and Docker will then cache that broken layer indefinitely."""
     info(f"building image {ref} from {context} (layer cache keeps repeat builds fast)")
     args = ["docker", "build", "-t", ref]
+    if os.environ.get("BOT_BOTTLE_NO_CACHE") == "1":
+        args.append("--no-cache")
     if dockerfile:
         args.extend(["-f", dockerfile])
     args.append(context)
     subprocess.run(args, check=True)
 
 
+def verify_agent_image(image: str, argv: tuple[str, ...]) -> None:
+    """Run `argv` inside a throwaway container of a freshly built agent
+    image and die loudly if it fails, instead of shipping an image
+    whose CLI only breaks at first real use. No-op when the provider
+    hasn't declared a smoke test (`AgentProviderRuntime.smoke_test`)."""
+    if not argv:
+        return
+    result = subprocess.run(
+        ["docker", "run", "--rm", "--entrypoint", argv[0], image, *argv[1:]],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if result.returncode != 0:
+        detail = (result.stderr or result.stdout or "").strip()
+        die(
+            f"agent image {image!r} failed its post-build smoke test "
+            f"({' '.join(argv)}): {detail}\n"
+            f"Try rebuilding from scratch: bot-bottle start --no-cache"
+        )
+
+
 # def build_image_with_cwd(
 #     derived: str,
 #     base: str,
diff --git a/bot_bottle/backend/macos_container/util.py b/bot_bottle/backend/macos_container/util.py
index ccb8f06..5853948 100644
--- a/bot_bottle/backend/macos_container/util.py
+++ b/bot_bottle/backend/macos_container/util.py
@@ -60,13 +60,21 @@ def dns_server() -> str:
 
 
 def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
-    """Build an OCI image with Apple's BuildKit-backed `container build`."""
+    """Build an OCI image with Apple's BuildKit-backed `container build`.
+
+    Set `BOT_BOTTLE_NO_CACHE=1` (the `start --no-cache` flag) to force
+    `--no-cache`. The npm/curl installers some provider Dockerfiles
+    shell out to can silently no-op on a transient network failure —
+    e.g. an `optionalDependencies` fetch for a platform-native binary —
+    and the builder will then cache that broken layer indefinitely."""
     info(
         f"building image {ref} from {context} with Apple Container "
         "(layer cache keeps repeat builds fast)"
     )
     _ensure_builder_dns()
     args = [_CONTAINER, "build", "-t", ref, "--dns", dns_server()]
+    if os.environ.get("BOT_BOTTLE_NO_CACHE") == "1":
+        args.append("--no-cache")
     if dockerfile:
         # `container build` resolves -f relative to the current working
         # directory, not the build context. Anchor a relative Dockerfile to
@@ -78,6 +86,28 @@ def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
     subprocess.run(args, check=True)
 
 
+def verify_agent_image(image: str, argv: tuple[str, ...]) -> None:
+    """Run `argv` inside a throwaway container of a freshly built agent
+    image and die loudly if it fails, instead of shipping an image
+    whose CLI only breaks at first real use. No-op when the provider
+    hasn't declared a smoke test (`AgentProviderRuntime.smoke_test`)."""
+    if not argv:
+        return
+    result = subprocess.run(
+        [_CONTAINER, "run", "--rm", "--entrypoint", argv[0], image, *argv[1:]],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if result.returncode != 0:
+        detail = (result.stderr or result.stdout or "").strip()
+        die(
+            f"agent image {image!r} failed its post-build smoke test "
+            f"({' '.join(argv)}): {detail}\n"
+            f"Try rebuilding from scratch: bot-bottle start --no-cache"
+        )
+
+
 def commit_container(container_name: str, image_tag: str) -> None:
     """Snapshot a running Apple Container as a local image.
 
diff --git a/bot_bottle/cli/start.py b/bot_bottle/cli/start.py
index 5a22ce0..4101804 100644
--- a/bot_bottle/cli/start.py
+++ b/bot_bottle/cli/start.py
@@ -46,6 +46,17 @@ def cmd_start(argv: list[str]) -> int:
     parser = argparse.ArgumentParser(prog=f"{PROG} start", add_help=True)
     parser.add_argument("--dry-run", action="store_true")
     parser.add_argument("--cwd", action="store_true", help="copy host cwd into the running bottle")
+    parser.add_argument(
+        "--no-cache",
+        action="store_true",
+        help=(
+            "rebuild agent/sidecar images from scratch, bypassing the "
+            "build layer cache. Use when an image looks broken after a "
+            "dependency bump — e.g. an installer's optionalDependencies "
+            "fetch silently no-op'd on a transient failure and got baked "
+            "into a cached layer."
+        ),
+    )
     parser.add_argument(
         "--backend",
         choices=known_backend_names(),
@@ -97,6 +108,11 @@ def cmd_start(argv: list[str]) -> int:
     args = parser.parse_args(argv)
 
     dry_run = args.dry_run or os.environ.get("BOT_BOTTLE_DRY_RUN") == "1"
+    if args.no_cache or os.environ.get("BOT_BOTTLE_NO_CACHE") == "1":
+        # Read by build_image() in each backend's util module — set here
+        # so both the interactive and --headless paths pick it up without
+        # threading a no_cache field through every backend's plan dataclass.
+        os.environ["BOT_BOTTLE_NO_CACHE"] = "1"
 
     manifest = ManifestIndex.resolve(USER_CWD)
     backend_name: str | None = args.backend
diff --git a/bot_bottle/contrib/claude/agent_provider.py b/bot_bottle/contrib/claude/agent_provider.py
index c6c6943..8d6dd47 100644
--- a/bot_bottle/contrib/claude/agent_provider.py
+++ b/bot_bottle/contrib/claude/agent_provider.py
@@ -91,6 +91,7 @@ _RUNTIME = AgentProviderRuntime(
     prompt_mode="append_file",
     bypass_args=("--dangerously-skip-permissions",),
     resume_args=("--continue",),
+    smoke_test=("claude", "--version"),
 )
 
 
diff --git a/bot_bottle/contrib/codex/agent_provider.py b/bot_bottle/contrib/codex/agent_provider.py
index eb09977..6a148f2 100644
--- a/bot_bottle/contrib/codex/agent_provider.py
+++ b/bot_bottle/contrib/codex/agent_provider.py
@@ -61,6 +61,7 @@ _RUNTIME = AgentProviderRuntime(
     prompt_mode="read_prompt_file",
     bypass_args=("--dangerously-bypass-approvals-and-sandbox",),
     resume_args=("resume", "--last"),
+    smoke_test=(_CODEX_CLI, "--version"),
 )
 
 
diff --git a/tests/unit/test_docker_launch_committed_image.py b/tests/unit/test_docker_launch_committed_image.py
index d0919b6..47a3a52 100644
--- a/tests/unit/test_docker_launch_committed_image.py
+++ b/tests/unit/test_docker_launch_committed_image.py
@@ -88,9 +88,11 @@ class TestLaunchCommittedImage(unittest.TestCase):
             del ctx, dockerfile
             built.append(image)
 
+<<<<<<< HEAD
         with mock.patch.object(launch_mod, "read_committed_image", return_value=committed_tag), \
                 mock.patch.object(launch_mod.docker_mod, "image_exists", return_value=image_present), \
                 mock.patch.object(launch_mod.docker_mod, "build_image", side_effect=_build), \
+                mock.patch.object(launch_mod.docker_mod, "verify_agent_image"), \
                 mock.patch.object(launch_mod, "launch_consolidated", return_value=_CTX), \
                 mock.patch.object(launch_mod, "teardown_consolidated"), \
                 mock.patch.object(launch_mod, "DockerGateway", return_value=gw), \
diff --git a/tests/unit/test_docker_launch_teardown.py b/tests/unit/test_docker_launch_teardown.py
index c942f73..06c24ac 100644
--- a/tests/unit/test_docker_launch_teardown.py
+++ b/tests/unit/test_docker_launch_teardown.py
@@ -94,6 +94,7 @@ class TestTeardownWarning(unittest.TestCase):
         )
 
         with mock.patch.object(launch_mod.docker_mod, "build_image"), \
+             mock.patch.object(launch_mod.docker_mod, "verify_agent_image"), \
              mock.patch.object(launch_mod, "launch_consolidated", return_value=ctx), \
              mock.patch.object(launch_mod, "teardown_consolidated"), \
              mock.patch.object(launch_mod, "DockerGateway", return_value=gw), \
-- 
2.52.0


From 3c426f45f5120ed8514c9a0378b974248cfa6453 Mon Sep 17 00:00:00 2001
From: didericis <eric@dideric.is>
Date: Mon, 13 Jul 2026 04:21:57 -0400
Subject: [PATCH 2/6] refactor(firecracker): use docker_mod instead of
 hand-rolled docker helpers

firecracker/launch.py reimplemented docker build/image-exists/rm/exec/cp
as private functions instead of the shared docker_mod used by the
docker and macos-container backends. Switching to docker_mod dedupes
the logic and gets --no-cache support for free (docker_mod.build_image
already reads BOT_BOTTLE_NO_CACHE); docker_mod gains docker_exec/
docker_cp general-purpose helpers to cover what the private versions did.
---
 bot_bottle/backend/docker/util.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/bot_bottle/backend/docker/util.py b/bot_bottle/backend/docker/util.py
index e827c24..43c5c54 100644
--- a/bot_bottle/backend/docker/util.py
+++ b/bot_bottle/backend/docker/util.py
@@ -89,6 +89,31 @@ def docker_exec_root(container: str, argv: list[str]) -> None:
     )
 
 
+def docker_exec(container: str, argv: list[str], *, user: str = "") -> None:
+    """Run `docker exec` in the named container, dying with the
+    command's own stderr on failure. Pass `user=\"0\"` to run as root."""
+    cmd = ["docker", "exec"]
+    if user:
+        cmd += ["-u", user]
+    cmd += [container, *argv]
+    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    if result.returncode != 0:
+        die(
+            f"docker exec in {container} failed: "
+            f"{(result.stderr or '').strip() or '<no stderr>'}"
+        )
+
+
+def docker_cp(src: str, dest: str) -> None:
+    """Run `docker cp`, dying with the command's own stderr on failure."""
+    result = subprocess.run(
+        ["docker", "cp", src, dest], capture_output=True, text=True, check=False,
+    )
+    if result.returncode != 0:
+        die(f"docker cp {src} -> {dest} failed: "
+            f"{(result.stderr or '').strip() or '<no stderr>'}")
+
+
 _SLUG_RE = re.compile(r"[^a-z0-9]+")
 
 
-- 
2.52.0


From 64ca39ddd408c2f591fb2025eea0f39657801ac9 Mon Sep 17 00:00:00 2001
From: claude <eric+claude@dideric.is>
Date: Tue, 14 Jul 2026 07:57:01 +0000
Subject: [PATCH 3/6] feat(firecracker): implement consolidated orchestrator
 launch (PRD 0070)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Replace the per-bottle Docker sidecar bundle with the shared per-host
orchestrator + gateway, mirroring what the Docker backend already has.

- Add `bot_bottle/backend/firecracker/consolidated_launch.py`:
  `_FirecrackerOrchestratorService` (subclasses `OrchestratorService`,
  overrides `_gateway()` to return a `DockerGateway` with host port
  bindings so Firecracker VMs can reach it via their TAP link);
  `launch_consolidated()` registers the bottle by guest IP (attribution
  key), provisions git-gate into the shared gateway, and returns the
  shared CA + orchestrator URL for teardown; `teardown_consolidated()`
  deregisters and cleans up.

- Rewrite `bot_bottle/backend/firecracker/launch.py`: removes the
  per-bottle sidecar bundle (`_start_sidecar_bundle`, `_stage_git_gate`,
  etc.) and `_mint_certs`; wires `launch_consolidated()` instead. The VM
  still sends to `host_tap_ip:PORT` — Docker's PREROUTING DNAT + the nft
  `ct status dnat accept` rule in the forward chain route the traffic to
  the shared gateway container.

- Extend `DockerGateway` with `host_port_bindings` so the Firecracker
  gateway publishes its ports on the host (`0.0.0.0:PORT`).

- Parameterise `OrchestratorService` with `orchestrator_name` /
  `orchestrator_label` so Docker and Firecracker orchestrators can
  coexist on the same host (`bot-bottle-orchestrator` vs
  `bot-bottle-fc-orchestrator`).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .../firecracker/consolidated_launch.py        | 164 ++++++++++++
 bot_bottle/backend/firecracker/launch.py      | 248 ++++++++++++++++--
 bot_bottle/orchestrator/gateway.py            |   7 +
 bot_bottle/orchestrator/lifecycle.py          |  31 ++-
 .../test_docker_launch_committed_image.py     |   1 -
 5 files changed, 421 insertions(+), 30 deletions(-)
 create mode 100644 bot_bottle/backend/firecracker/consolidated_launch.py

diff --git a/bot_bottle/backend/firecracker/consolidated_launch.py b/bot_bottle/backend/firecracker/consolidated_launch.py
new file mode 100644
index 0000000..6ac26b3
--- /dev/null
+++ b/bot_bottle/backend/firecracker/consolidated_launch.py
@@ -0,0 +1,164 @@
+"""Consolidated bottle launch sequence for the Firecracker backend (PRD 0070).
+
+Mirrors bot_bottle.backend.docker.consolidated_launch but wired for
+Firecracker's TAP-based network topology instead of a shared Docker bridge.
+
+The per-bottle sidecar bundle (one `docker run` per bottle, published on the
+slot's host-side TAP IP) is replaced by a single persistent gateway that
+every Firecracker VM shares. The gateway runs as a Docker container in the
+dev-harness (a Firecracker VM is stage B per PRD 0070), with its ports
+published on the host (`0.0.0.0:PORT`). VMs reach it at their slot's
+host-side TAP IP because Docker's iptables PREROUTING DNAT redirects
+port 9099/9100/9420 traffic to the gateway container — a path the nft
+isolation table already allows via `ct status dnat accept` in the forward
+chain.
+
+Attribution is by the VM's guest IP, which is unspoofable by construction:
+the /31 point-to-point TAP topology + the `bot_bottle_fc` nft table ensure
+that only the expected VM can source-IP that address.
+
+Sequence:
+  1. ensure the Firecracker-flavoured orchestrator + gateway are up;
+  2. register the bottle by its guest IP (attribution key) → bottle id +
+     identity token;
+  3. provision its git-gate repos/creds into the running gateway;
+  4. fetch the shared gateway CA for the provisioner to install in the rootfs.
+
+The TAP slot allocation, rootfs build, and VM boot are the caller's job.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from ...egress import EgressPlan
+from ...git_gate import GitGatePlan
+from ...orchestrator.client import OrchestratorClient
+from ...orchestrator.gateway import (
+    GATEWAY_NETWORK,
+    DockerGateway,
+)
+from ...orchestrator.lifecycle import (
+    OrchestratorService,
+    OrchestratorStartError,  # re-exported so callers can catch it
+)
+from ...orchestrator.registration import registration_inputs
+from ..docker.egress import EGRESS_PORT
+from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate
+from ...supervise import SUPERVISE_PORT
+
+_GIT_HTTP_PORT = 9420
+
+# Separate names from the Docker gateway so both backends can coexist on one
+# host (and for clarity in `docker ps` output).
+_FC_GATEWAY_NAME = "bot-bottle-fc-gateway"
+_FC_ORCHESTRATOR_NAME = "bot-bottle-fc-orchestrator"
+_FC_ORCHESTRATOR_LABEL = "bot-bottle-fc-orchestrator=1"
+# Ports the gateway publishes on the host so Firecracker VMs can reach it
+# via their TAP link. Docker's PREROUTING DNAT + nft's `ct status dnat
+# accept` in the forward chain route the traffic.
+_FC_GATEWAY_HOST_PORTS = (EGRESS_PORT, SUPERVISE_PORT, _GIT_HTTP_PORT)
+
+
+class ConsolidatedLaunchError(RuntimeError):
+    """The consolidated register/provision sequence could not complete."""
+
+
+@dataclass(frozen=True)
+class LaunchContext:
+    """What the Firecracker launch needs from the consolidated sequence."""
+
+    bottle_id: str
+    identity_token: str
+    source_ip: str          # the VM's guest IP — the attribution key
+    gateway_ca_pem: str     # the shared gateway CA the provisioner installs
+    orchestrator_url: str
+
+
+class _FirecrackerOrchestratorService(OrchestratorService):
+    """Dev-harness orchestrator for the Firecracker backend.
+
+    Uses a gateway that publishes its ports on the host so Firecracker VMs can
+    reach it via their TAP link. The gateway and orchestrator containers use
+    `*-fc-*` names so both backends can run independently on the same host.
+    """
+
+    def __init__(self, **kwargs: object) -> None:
+        super().__init__(
+            orchestrator_name=_FC_ORCHESTRATOR_NAME,
+            orchestrator_label=_FC_ORCHESTRATOR_LABEL,
+            **kwargs,  # type: ignore[arg-type]
+        )
+
+    def _gateway(self) -> DockerGateway:
+        # The heavy data-plane image (#384 split it from the lean control-plane
+        # `image` this service's orchestrator container runs).
+        return DockerGateway(
+            self._gateway_image,
+            name=_FC_GATEWAY_NAME,
+            network=self.network,
+            orchestrator_url=self.internal_url,
+            host_port_bindings=_FC_GATEWAY_HOST_PORTS,
+        )
+
+
+def launch_consolidated(
+    egress_plan: EgressPlan,
+    git_gate_plan: GitGatePlan,
+    *,
+    guest_ip: str,
+    image_ref: str = "",
+    tokens: dict[str, str] | None = None,
+    service: OrchestratorService | None = None,
+    gateway_name: str = _FC_GATEWAY_NAME,
+) -> LaunchContext:
+    """Ensure the orchestrator + Firecracker gateway are up, register the
+    bottle by its guest IP, and provision its git-gate state. Returns the
+    context the VM launch needs. Raises `ConsolidatedLaunchError` (or the
+    primitives' own errors) on failure — the caller tears down on failure."""
+    service = service or _FirecrackerOrchestratorService()
+    url = service.ensure_running()
+    client = OrchestratorClient(url)
+
+    inputs = registration_inputs(egress_plan)
+    reg = client.register_bottle(
+        guest_ip, image_ref=image_ref, policy=inputs.policy,
+        metadata=inputs.metadata, tokens=tokens,
+    )
+    try:
+        provision_git_gate(gateway_name, reg.bottle_id, git_gate_plan)
+    except Exception:
+        client.teardown_bottle(reg.bottle_id)
+        raise
+
+    # Fetch the shared gateway CA here so the caller can install it in the
+    # rootfs (the same CA every agent on this host trusts for TLS interception).
+    gateway_ca_pem = DockerGateway(
+        name=gateway_name, network=GATEWAY_NETWORK,
+    ).ca_cert_pem()
+
+    return LaunchContext(
+        bottle_id=reg.bottle_id,
+        identity_token=reg.identity_token,
+        source_ip=guest_ip,
+        gateway_ca_pem=gateway_ca_pem,
+        orchestrator_url=url,
+    )
+
+
+def teardown_consolidated(
+    bottle_id: str, *, orchestrator_url: str, gateway_name: str = _FC_GATEWAY_NAME,
+) -> None:
+    """Deregister the bottle and remove its git-gate state from the gateway.
+    Both steps are idempotent so this is safe from a cleanup trap."""
+    OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
+    deprovision_git_gate(gateway_name, bottle_id)
+
+
+__all__ = [
+    "LaunchContext",
+    "launch_consolidated",
+    "teardown_consolidated",
+    "ConsolidatedLaunchError",
+    "OrchestratorStartError",
+]
diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py
index 438a5eb..246441e 100644
--- a/bot_bottle/backend/firecracker/launch.py
+++ b/bot_bottle/backend/firecracker/launch.py
@@ -1,25 +1,64 @@
-"""Launch flow for the Firecracker backend — temporarily disabled (#385).
+"""Launch flow for the Firecracker backend (PRD 0070, consolidated).
 
-The firecracker backend launched a per-bottle companion container (the
-egress / git-gate / supervise data plane) alongside each microVM. That
-per-bottle-companion architecture was removed in the companion-container removal;
-firecracker's replacement — the consolidated per-host gateway — lands in
-its own cutover (#354).
+Per bottle:
+  1. build the agent image (docker), export it to a cached ext4 rootfs;
+  2. ensure the per-host orchestrator + shared gateway are up;
+  3. claim a free TAP pool slot (rootless flock);
+  4. register the bottle on the orchestrator by the VM's guest IP (the
+     attribution key) and provision its git-gate state into the gateway;
+  5. boot the microVM on that TAP; wait for SSH;
+  6. provision (shared gateway CA, prompt, skills, workspace, git, supervise)
+     over SSH.
 
-Until that lands, launching a firecracker bottle fails closed rather than
-silently running the removed path. `prepare` / `status` / cleanup still
-work, so `backend status --backend=firecracker` and orphan cleanup are
-unaffected.
+The per-bottle Docker sidecar bundle is gone. The shared gateway handles
+egress / git-gate / supervise for every VM; Docker's PREROUTING DNAT routes
+the VMs' traffic to it, and the nft table's `ct status dnat accept` rule
+in the forward chain lets it pass. The VM still sends to `host_tap_ip:PORT`
+— the address its world is, by nft design, limited to.
+
+Isolation is enforced by the operator-provisioned nft table (checked
+fail-closed in preflight): a VM reaches only the sidecar (DNAT'd from
+the host TAP IP) and nothing else.
 """
 
 from __future__ import annotations
 
-from contextlib import contextmanager
+import dataclasses
+import os
+from contextlib import ExitStack, contextmanager
+from pathlib import Path
 from typing import Callable, Generator
 
-from ...log import die
+from ...agent_provider import runtime_for
+from ...bottle_state import (
+    egress_state_dir,
+    git_gate_state_dir,
+    read_committed_image,
+)
+from ...egress import (
+    egress_agent_env_entries,
+    egress_resolve_token_values,
+)
+from ...git_gate import (
+    provision_git_gate_dynamic_keys,
+    revoke_git_gate_provisioned_keys,
+)
+from ...log import info, warn
+from ...supervise import SUPERVISE_PORT
+from ..docker import util as docker_mod
+from ..docker.egress import EGRESS_PORT
+from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
+from . import firecracker_vm, isolation_probe, netpool, util
 from .bottle import FirecrackerBottle
 from .bottle_plan import FirecrackerBottlePlan
+from .consolidated_launch import (
+    launch_consolidated,
+    teardown_consolidated,
+)
+
+
+_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
+_GIT_HTTP_PORT = 9420
 
 
 @contextmanager
@@ -28,12 +67,181 @@ def launch(
     *,
     provision: Callable[[FirecrackerBottlePlan, "FirecrackerBottle"], str | None],
 ) -> Generator[FirecrackerBottle, None, None]:
-    """Fail closed: the firecracker backend is disabled while its
-    consolidated (gateway-backed) launch is built in #354."""
-    del plan, provision
-    die(
-        "the firecracker backend is temporarily disabled during the "
-        "companion-container removal (#385); its consolidated relaunch "
-        "lands in #354. Use --backend=docker for now."
+    """Build, launch, and provision a Firecracker bottle via the consolidated
+    orchestrator. Teardown on exit."""
+    stack = ExitStack()
+    bottle_for_revoke = plan.manifest.bottle
+    git_gate_dir_for_revoke = git_gate_state_dir(plan.slug)
+
+    def teardown() -> None:
+        teardown_exc: BaseException | None = None
+        try:
+            stack.close()
+        except BaseException as exc:  # noqa: W0718 - teardown must continue
+            teardown_exc = exc
+            warn(f"firecracker teardown failed: {exc!r}")
+        revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke)
+        if teardown_exc is not None:
+            raise teardown_exc
+
+    try:
+        # Step 1: agent image. The sidecar bundle image is built by the
+        # orchestrator service (ensure_running → ensure_built); we only
+        # build the agent image here. Use a committed snapshot when available.
+        plan = _build_agent_image(plan)
+
+        # Step 2: mint the git-gate dynamic (gitea) deploy keys, if any.
+        git_gate_plan = plan.git_gate_plan
+        if git_gate_plan.upstreams:
+            git_gate_plan = provision_git_gate_dynamic_keys(
+                plan.manifest.bottle, git_gate_plan, git_gate_state_dir(plan.slug),
+            )
+
+        # Step 3: claim a TAP slot; the flock is held until teardown.
+        slot, lock = netpool.allocate(plan.slug)
+        stack.callback(lock.close)
+        info(f"firecracker slot {slot.iface}: host={slot.host_ip} "
+             f"guest={slot.guest_ip}")
+
+        # Step 4: register on the orchestrator + provision this bottle's
+        # git-gate state into the shared gateway. The per-bottle egress tokens
+        # are resolved from the host env now and handed to the orchestrator
+        # (in memory) for the gateway to inject — the agent never sees them.
+        # Attribution is by the VM's guest IP (unspoofable via /31 TAP + nft).
+        effective_env = {**os.environ, **plan.agent_provision.provisioned_env}
+        token_values = egress_resolve_token_values(
+            plan.egress_plan.token_env_map, effective_env,
+        )
+        ctx = launch_consolidated(
+            plan.egress_plan, git_gate_plan,
+            guest_ip=slot.guest_ip,
+            image_ref=plan.image,
+            tokens=token_values,
+        )
+        stack.callback(
+            teardown_consolidated, ctx.bottle_id,
+            orchestrator_url=ctx.orchestrator_url,
+        )
+
+        # Step 5: install the SHARED gateway CA (replaces the per-bottle CA).
+        # Write it to a stable host path so the provisioner can copy it over SSH.
+        ca_dir = egress_state_dir(plan.slug) / "gateway-ca"
+        ca_dir.mkdir(parents=True, exist_ok=True)
+        ca_file = ca_dir / "gateway-ca.pem"
+        ca_file.write_text(ctx.gateway_ca_pem)
+        egress_plan = dataclasses.replace(
+            plan.egress_plan,
+            mitmproxy_ca_host_path=ca_file,
+            mitmproxy_ca_cert_only_host_path=ca_file,
+        )
+        # Point the agent's git-gate insteadOf rewrites and supervise MCP URL
+        # at the shared gateway (reached at the slot's host TAP IP — the VM
+        # sends there and Docker DNAT routes to the gateway container).
+        git_gate_url = (
+            f"http://{slot.host_ip}:{_GIT_HTTP_PORT}" if git_gate_plan.upstreams else ""
+        )
+        supervise_url = (
+            f"http://{slot.host_ip}:{SUPERVISE_PORT}/"
+            if plan.supervise_plan is not None else ""
+        )
+        plan = dataclasses.replace(
+            plan,
+            git_gate_plan=git_gate_plan,
+            egress_plan=egress_plan,
+            agent_proxy_url=f"http://{slot.host_ip}:{EGRESS_PORT}",
+            agent_git_gate_url=git_gate_url,
+            agent_supervise_url=supervise_url,
+        )
+
+        # Step 6: build the per-bottle rootfs + SSH key, then boot.
+        base_dir = util.build_base_rootfs_dir(plan.image)
+        run_dir = util.cache_dir() / "run" / plan.slug
+        run_dir.mkdir(parents=True, exist_ok=True)
+        rootfs = run_dir / "rootfs.ext4"
+        util.build_rootfs_ext4(base_dir, rootfs)
+        private_key, pubkey = util.generate_keypair(run_dir)
+
+        vm = firecracker_vm.boot(
+            name=plan.container_name,
+            rootfs=rootfs,
+            tap=slot.iface,
+            guest_ip=slot.guest_ip,
+            host_ip=slot.host_ip,
+            pubkey=pubkey,
+            run_dir=run_dir,
+        )
+        stack.callback(vm.terminate)
+        firecracker_vm.wait_for_ssh(vm, private_key)
+
+        # Authoritative fail-closed egress-boundary check, before the agent
+        # runs: prove the VM cannot reach the host directly.
+        isolation_probe.verify_isolation(private_key, slot.guest_ip)
+
+        bottle = FirecrackerBottle(
+            plan.container_name,
+            private_key=private_key,
+            guest_ip=slot.guest_ip,
+            guest_env=_agent_guest_env(plan, slot.host_ip),
+            agent_command=plan.agent_command,
+            agent_prompt_mode=plan.agent_prompt_mode,
+            agent_provider_template=plan.agent_provider_template,
+            terminal_title=(
+                f"{plan.spec.label} ({plan.spec.agent_name})"
+                if plan.spec.label else plan.spec.agent_name
+            ),
+            terminal_color=plan.spec.color,
+            agent_workdir=plan.workspace_plan.workdir,
+        )
+        bottle.prompt_path = provision(plan, bottle)
+
+        yield bottle
+    finally:
+        teardown()
+
+
+def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
+    committed = read_committed_image(plan.slug)
+    if committed and docker_mod.image_exists(committed):
+        info(f"using committed image {committed!r}")
+        return dataclasses.replace(
+            plan,
+            agent_provision=dataclasses.replace(plan.agent_provision, image=committed),
+        )
+    docker_mod.build_image(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path)
+    docker_mod.verify_agent_image(
+        plan.image, runtime_for(plan.agent_provider_template).smoke_test,
     )
-    yield  # unreachable — `die` raises; keeps this a generator/contextmanager
+    return plan
+
+
+# --- agent guest env -------------------------------------------------
+
+def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str]:
+    """Env injected into every agent/exec call over SSH. The VM has no
+    baked process env (it just runs init), so the proxy/CA/git/supervise
+    wiring is applied per-invocation."""
+    proxy_url = f"http://{host_ip}:{EGRESS_PORT}"
+    no_proxy = f"localhost,127.0.0.1,{host_ip}"
+    env: dict[str, str] = {
+        "HTTPS_PROXY": proxy_url, "HTTP_PROXY": proxy_url,
+        "https_proxy": proxy_url, "http_proxy": proxy_url,
+        "NO_PROXY": no_proxy, "no_proxy": no_proxy,
+        "NODE_EXTRA_CA_CERTS": AGENT_CA_PATH,
+        "SSL_CERT_FILE": AGENT_CA_BUNDLE,
+        "REQUESTS_CA_BUNDLE": AGENT_CA_BUNDLE,
+    }
+    if plan.agent_git_gate_url:
+        env["GIT_GATE_URL"] = plan.agent_git_gate_url
+    if plan.agent_supervise_url:
+        env["MCP_SUPERVISE_URL"] = plan.agent_supervise_url
+    for entry in egress_agent_env_entries(plan.egress_plan):
+        key, _, value = entry.partition("=")
+        env[key] = value
+    env.update(plan.agent_provision.guest_env)
+    # Forwarded (bare-name) env: resolve host values now, since the VM
+    # can't inherit them from a `docker run --env NAME`.
+    for name in plan.forwarded_env:
+        value = os.environ.get(name)
+        if value is not None:
+            env[name] = value
+    return env
diff --git a/bot_bottle/orchestrator/gateway.py b/bot_bottle/orchestrator/gateway.py
index f474cd9..f501f39 100644
--- a/bot_bottle/orchestrator/gateway.py
+++ b/bot_bottle/orchestrator/gateway.py
@@ -100,6 +100,7 @@ class DockerGateway(Gateway):
         orchestrator_url: str = "",
         build_context: Path | None = None,
         dockerfile: str | None = GATEWAY_DOCKERFILE,
+        host_port_bindings: tuple[int, ...] = (),
     ) -> None:
         self.image_ref = image_ref
         self.name = name
@@ -110,6 +111,10 @@ class DockerGateway(Gateway):
         self._orchestrator_url = orchestrator_url
         self._build_context = build_context or _REPO_ROOT
         self._dockerfile = dockerfile
+        # Ports published on the host (0.0.0.0). Used by the Firecracker
+        # backend's dev-harness gateway so VMs can reach it via their TAP link;
+        # Docker's DNAT + the nft `ct status dnat accept` rule handle the rest.
+        self._host_port_bindings = host_port_bindings
 
     def image_exists(self) -> bool:
         return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0
@@ -188,6 +193,8 @@ class DockerGateway(Gateway):
             # trust it) — see GATEWAY_CA_VOLUME.
             "--volume", f"{GATEWAY_CA_VOLUME}:{MITMPROXY_HOME}",
         ]
+        for port in self._host_port_bindings:
+            argv += ["--publish", f"0.0.0.0:{port}:{port}"]
         if self._orchestrator_url:
             # Makes the gateway's egress / git / supervise daemons multi-tenant:
             # each request resolves source-IP -> policy against the control plane.
diff --git a/bot_bottle/orchestrator/lifecycle.py b/bot_bottle/orchestrator/lifecycle.py
index ad80bd3..d6d94b2 100644
--- a/bot_bottle/orchestrator/lifecycle.py
+++ b/bot_bottle/orchestrator/lifecycle.py
@@ -78,7 +78,13 @@ def _source_hash(repo_root: Path) -> str:
 
 class OrchestratorService:
     """Manages the orchestrator control-plane container + the shared gateway.
-    Callers only need `ensure_running()` + `url`."""
+    Callers only need `ensure_running()` + `url`.
+
+    `orchestrator_name` / `orchestrator_label` let backends run independent
+    orchestrators on the same host without name collisions (e.g. the
+    Firecracker backend uses `bot-bottle-fc-orchestrator` alongside the Docker
+    backend's `bot-bottle-orchestrator`). Subclass and override `_gateway()`
+    to supply a backend-specific gateway variant."""
 
     def __init__(
         self,
@@ -89,6 +95,8 @@ class OrchestratorService:
         gateway_image: str = GATEWAY_IMAGE,
         repo_root: Path = _REPO_ROOT,
         host_root: Path | None = None,
+        orchestrator_name: str = ORCHESTRATOR_NAME,
+        orchestrator_label: str = ORCHESTRATOR_LABEL,
     ) -> None:
         self.port = port
         self.network = network
@@ -100,6 +108,8 @@ class OrchestratorService:
         self._gateway_image = gateway_image
         self._repo_root = repo_root
         self._host_root = host_root or bot_bottle_root()
+        self._orchestrator_name = orchestrator_name
+        self._orchestrator_label = orchestrator_label
 
     @property
     def url(self) -> str:
@@ -111,7 +121,7 @@ class OrchestratorService:
         """Control-plane URL as the gateway container reaches it — by name over
         docker DNS on the shared network. This is the gateway's
         BOT_BOTTLE_ORCHESTRATOR_URL."""
-        return f"http://{ORCHESTRATOR_NAME}:{self.port}"
+        return f"http://{self._orchestrator_name}:{self.port}"
 
     def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool:
         try:
@@ -129,11 +139,11 @@ class OrchestratorService:
         fixed-name container first). Register-only broker → no docker socket.
         Labels the container with `source_hash` so a later `ensure_running`
         can detect a real code change (see `_source_hash`)."""
-        run_docker(["docker", "rm", "--force", ORCHESTRATOR_NAME])
+        run_docker(["docker", "rm", "--force", self._orchestrator_name])
         proc = run_docker([
             "docker", "run", "--detach",
-            "--name", ORCHESTRATOR_NAME,
-            "--label", ORCHESTRATOR_LABEL,
+            "--name", self._orchestrator_name,
+            "--label", self._orchestrator_label,
             "--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={source_hash}",
             "--network", self.network,
             # Host CLI reaches the control plane here; bound to loopback so it
@@ -185,12 +195,12 @@ class OrchestratorService:
         *current* bind-mounted source. Mirrors `DockerGateway`'s
         image-staleness check, but by content hash rather than image id since
         the orchestrator runs bind-mounted source, not a built image."""
-        if not self._container_running(ORCHESTRATOR_NAME):
+        if not self._container_running(self._orchestrator_name):
             return False
         proc = run_docker([
             "docker", "inspect", "--format",
             "{{ index .Config.Labels \"" + ORCHESTRATOR_SOURCE_HASH_LABEL + "\" }}",
-            ORCHESTRATOR_NAME,
+            self._orchestrator_name,
         ])
         if proc.returncode != 0:
             return True  # can't compare -> don't churn a working container
@@ -219,7 +229,10 @@ class OrchestratorService:
             return self.url
 
         self._ensure_orchestrator_image()
-        log.info("starting orchestrator container", context={"name": ORCHESTRATOR_NAME})
+        log.info(
+            "starting orchestrator container",
+            context={"name": self._orchestrator_name},
+        )
         self._run_orchestrator_container(current_hash)
 
         deadline = time.monotonic() + startup_timeout
@@ -234,7 +247,7 @@ class OrchestratorService:
 
     def stop(self) -> None:
         """Remove the orchestrator + gateway containers (idempotent)."""
-        run_docker(["docker", "rm", "--force", ORCHESTRATOR_NAME])
+        run_docker(["docker", "rm", "--force", self._orchestrator_name])
         self._gateway().stop()
 
 
diff --git a/tests/unit/test_docker_launch_committed_image.py b/tests/unit/test_docker_launch_committed_image.py
index 47a3a52..7e9c997 100644
--- a/tests/unit/test_docker_launch_committed_image.py
+++ b/tests/unit/test_docker_launch_committed_image.py
@@ -88,7 +88,6 @@ class TestLaunchCommittedImage(unittest.TestCase):
             del ctx, dockerfile
             built.append(image)
 
-<<<<<<< HEAD
         with mock.patch.object(launch_mod, "read_committed_image", return_value=committed_tag), \
                 mock.patch.object(launch_mod.docker_mod, "image_exists", return_value=image_present), \
                 mock.patch.object(launch_mod.docker_mod, "build_image", side_effect=_build), \
-- 
2.52.0


From 0a1f9499378acdcff1ec879e35459456fe5ccd14 Mon Sep 17 00:00:00 2001
From: claude <eric+claude@dideric.is>
Date: Tue, 14 Jul 2026 08:38:37 +0000
Subject: [PATCH 4/6] fix(docker): use run_docker in docker_exec, docker_cp,
 verify_agent_image

The rebase onto lazy-backend-imports converts existing helpers (image_exists,
container_exists, etc.) to run_docker; the three new functions added in this
branch still called subprocess.run directly. Switch them over for consistency.
---
 bot_bottle/backend/docker/util.py | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/bot_bottle/backend/docker/util.py b/bot_bottle/backend/docker/util.py
index 43c5c54..a7887d9 100644
--- a/bot_bottle/backend/docker/util.py
+++ b/bot_bottle/backend/docker/util.py
@@ -10,6 +10,7 @@ import shutil
 import subprocess
 from typing import Iterable, Iterator
 
+from ...docker_cmd import run_docker
 from ...log import die, info
 # from ...workspace import WorkspacePlan
 
@@ -96,7 +97,7 @@ def docker_exec(container: str, argv: list[str], *, user: str = "") -> None:
     if user:
         cmd += ["-u", user]
     cmd += [container, *argv]
-    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
+    result = run_docker(cmd)
     if result.returncode != 0:
         die(
             f"docker exec in {container} failed: "
@@ -106,9 +107,7 @@ def docker_exec(container: str, argv: list[str], *, user: str = "") -> None:
 
 def docker_cp(src: str, dest: str) -> None:
     """Run `docker cp`, dying with the command's own stderr on failure."""
-    result = subprocess.run(
-        ["docker", "cp", src, dest], capture_output=True, text=True, check=False,
-    )
+    result = run_docker(["docker", "cp", src, dest])
     if result.returncode != 0:
         die(f"docker cp {src} -> {dest} failed: "
             f"{(result.stderr or '').strip() or '<no stderr>'}")
@@ -158,12 +157,7 @@ def verify_agent_image(image: str, argv: tuple[str, ...]) -> None:
     hasn't declared a smoke test (`AgentProviderRuntime.smoke_test`)."""
     if not argv:
         return
-    result = subprocess.run(
-        ["docker", "run", "--rm", "--entrypoint", argv[0], image, *argv[1:]],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
+    result = run_docker(["docker", "run", "--rm", "--entrypoint", argv[0], image, *argv[1:]])
     if result.returncode != 0:
         detail = (result.stderr or result.stdout or "").strip()
         die(
-- 
2.52.0


From bab44dbe97769367428e9b8b99fb04c60bbd63f4 Mon Sep 17 00:00:00 2001
From: didericis <eric@dideric.is>
Date: Tue, 14 Jul 2026 17:48:17 -0400
Subject: [PATCH 5/6] feat(orchestrator): add buildah for in-VM agent-image
 builds (PRD 0069 Stage 3)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The Firecracker backend's remaining host-Docker dependency is building
users' agent Dockerfiles. Move that build *into the orchestrator VM*:
give the orchestrator image buildah (rootless, daemonless) so it builds
agent images itself, and the host needs no Docker daemon and no
root-equivalent `docker` group. An untrusted Dockerfile then builds
inside the confined orchestrator VM rather than on the host — strictly
more isolated than host `docker build`.

- buildah + crun (OCI runtime) + netavark/aardvark-dns (network backend
  for FROM pulls and RUN egress), installed explicitly since
  --no-install-recommends strips buildah's helper deps.
- vfs storage + chroot isolation (STORAGE_DRIVER/BUILDAH_ISOLATION) so
  buildah needs neither fuse-overlayfs / an overlay kernel module nor
  configured subuid maps — it works unconditionally as root in a minimal
  microVM rootfs. (Slower than overlay; a build-cache pass is deferred.)

Bootstrap: the orchestrator rootfs is still produced from this image via
host docker export; a later step pulls a pre-built image instead.

Verified: the orchestrator image boots as a Firecracker VM and buildah
runs inside it as real VM root (reaches the base-image pull; build-time
egress wiring is the next step).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck
---
 Dockerfile.orchestrator | 39 +++++++++++++++++++++++++++++++++------
 1 file changed, 33 insertions(+), 6 deletions(-)

diff --git a/Dockerfile.orchestrator b/Dockerfile.orchestrator
index daa8129..e81ae3b 100644
--- a/Dockerfile.orchestrator
+++ b/Dockerfile.orchestrator
@@ -1,9 +1,10 @@
-# Orchestrator control-plane image (PRD 0070, #384).
+# Orchestrator control-plane image (PRD 0070, #384) + in-VM agent-image
+# builder (PRD 0069 Stage 3).
 #
 # The per-host orchestrator runs `python3 -m bot_bottle.orchestrator`.
 # The `bot_bottle` package is **stdlib-only** by design, so the control
-# plane needs nothing but a Python runtime — none of the gateway's
-# mitmproxy / git / gitleaks payload (that is the separate
+# plane itself needs nothing but a Python runtime — none of the
+# gateway's mitmproxy / git / gitleaks payload (that is the separate
 # `bot-bottle-gateway` image, Dockerfile.gateway). Splitting them keeps
 # the secret-dense control plane (it concentrates every bottle's egress
 # tokens — see PRD 0070's "secret concentration") on a minimal
@@ -17,9 +18,35 @@
 
 FROM python:3.12-slim
 
-# No third-party deps to install — stdlib only. Kept as an explicit,
-# self-documenting stage so a future confinement step (baking the
-# package, dropping the bind mount) has an obvious home.
+# --- in-VM agent-image builder (PRD 0069 Stage 3) -------------------
+# The Firecracker backend removes the host Docker daemon by building
+# users' agent Dockerfiles *inside the orchestrator VM* with buildah
+# (rootless, daemonless) instead of on the host. The host then needs no
+# Docker daemon and no root-equivalent `docker` group; an untrusted
+# Dockerfile builds inside the confined orchestrator VM, never on the
+# host — strictly more isolated than host `docker build`.
+#
+# `git` lets buildah resolve git-context builds and lets the backend
+# clone/pull; `ca-certificates` is needed for `FROM` pulls over TLS.
+# buildah's runtime helpers (stripped by --no-install-recommends, so
+# listed explicitly): `crun` is the OCI runtime that executes build
+# steps; `netavark` + `aardvark-dns` are the network backend buildah
+# uses to give `FROM` pulls and `RUN` steps network access.
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+      buildah crun netavark aardvark-dns git ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+# vfs storage + chroot isolation: buildah then needs neither
+# fuse-overlayfs / an overlay kernel module nor configured subuid maps
+# (newuidmap/newgidmap), so it works unconditionally as root in a
+# minimal microVM rootfs. vfs copies layers rather than stacking them —
+# slower and heavier than overlay; a build-cache optimization (overlay
+# where available, a persistent cache disk) is tracked for later.
+ENV STORAGE_DRIVER=vfs \
+    BUILDAH_ISOLATION=chroot
+
+# No third-party *Python* deps — the control plane stays stdlib only.
 WORKDIR /app
 
 # Documentation only; lifecycle.py overrides the entrypoint to
-- 
2.52.0


From 95981ea9d3c0c93ac875b528192ac9833f447b67 Mon Sep 17 00:00:00 2001
From: didericis <eric@dideric.is>
Date: Tue, 14 Jul 2026 18:13:14 -0400
Subject: [PATCH 6/6] feat(firecracker): NAT'd egress link for the
 orchestrator/builder VM
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The orchestrator/gateway VM is trusted infra, not an isolated agent: it
builds agent images in-VM (buildah must FROM-pull + apt/npm) and, in the
Stage B cutover, forwards agent egress upstream. Give it a dedicated TAP
(`bborch0`) on a /31 at the top of the IP_BASE /16 (clear of the bbfc*
agent pool at the bottom), NAT'd out the host uplink — while agent VMs
keep their fail-closed, gateway-only isolation table.

- netpool.defaults.env / netpool.py: new BOT_BOTTLE_FC_ORCH_IFACE +
  `orch_slot()` (index -1 sentinel; host x.y.255.0 / guest x.y.255.1).
- scripts/firecracker-netpool.sh: create + address the orchestrator TAP;
  `bot_bottle_fc_nat` table masquerades its /31 out the uplink and
  accepts its forward path. Because bootstrap still runs Docker (whose
  FORWARD policy is DROP), a best-effort, guarded, idempotent DOCKER-USER
  ACCEPT is added too (skipped once Docker is gone). down/status updated.
- nix/firecracker-netpool.nix: mirror the option, pass it via the unit
  Environment= (the store-copied script can't read the defaults file),
  and add iptables to the unit path for the DOCKER-USER step.

Agent isolation is unchanged: the new rules only ever accept/masquerade
the orchestrator link and never drop, so they can't weaken the bbfc*
drops. Applied by re-running `sudo ./scripts/firecracker-netpool.sh up`.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck
---
 .../backend/firecracker/netpool.defaults.env  |  8 ++
 bot_bottle/backend/firecracker/netpool.py     | 25 ++++++
 nix/firecracker-netpool.nix                   | 25 +++++-
 scripts/firecracker-netpool.sh                | 81 ++++++++++++++++++-
 tests/unit/test_firecracker_helpers.py        | 15 ++++
 5 files changed, 149 insertions(+), 5 deletions(-)

diff --git a/bot_bottle/backend/firecracker/netpool.defaults.env b/bot_bottle/backend/firecracker/netpool.defaults.env
index 7532b3d..2277f1f 100644
--- a/bot_bottle/backend/firecracker/netpool.defaults.env
+++ b/bot_bottle/backend/firecracker/netpool.defaults.env
@@ -15,3 +15,11 @@ BOT_BOTTLE_FC_POOL_SIZE=8
 BOT_BOTTLE_FC_IP_BASE=10.243.0.0
 BOT_BOTTLE_FC_IFACE_PREFIX=bbfc
 BOT_BOTTLE_FC_NFT_TABLE=bot_bottle_fc
+# The orchestrator/gateway VM's own TAP — a dedicated link OUTSIDE the
+# bbfc* agent pool. Unlike agent VMs (which reach only their gateway),
+# the orchestrator is trusted infra that needs real NAT'd internet
+# egress: to FROM-pull + apt/npm during in-VM agent-image builds
+# (buildah) and to forward agent egress upstream (Stage B gateway). Its
+# /31 is the top of the IP_BASE /16 (host x.y.255.0, guest x.y.255.1),
+# clear of the pool near the bottom of the block.
+BOT_BOTTLE_FC_ORCH_IFACE=bborch0
diff --git a/bot_bottle/backend/firecracker/netpool.py b/bot_bottle/backend/firecracker/netpool.py
index faad11f..3ee6653 100644
--- a/bot_bottle/backend/firecracker/netpool.py
+++ b/bot_bottle/backend/firecracker/netpool.py
@@ -79,6 +79,12 @@ def _cfg(key: str) -> str:
 IFACE_PREFIX = _cfg("BOT_BOTTLE_FC_IFACE_PREFIX")
 NFT_TABLE = _cfg("BOT_BOTTLE_FC_NFT_TABLE")
 
+# The orchestrator/gateway VM's dedicated TAP — outside the bbfc* agent
+# pool and, unlike it, NAT'd to the internet (see `orch_slot`). The
+# orchestrator is trusted infra: it builds agent images in-VM (buildah
+# needs to FROM-pull + apt/npm) and forwards agent egress upstream.
+ORCH_IFACE = _cfg("BOT_BOTTLE_FC_ORCH_IFACE")
+
 
 def pool_size() -> int:
     return int(_cfg("BOT_BOTTLE_FC_POOL_SIZE"))
@@ -123,6 +129,25 @@ def all_slots() -> list[Slot]:
     return [slot(i) for i in range(pool_size())]
 
 
+def orch_slot() -> Slot:
+    """The orchestrator/gateway VM's dedicated link — its own TAP
+    (`ORCH_IFACE`) on a /31 at the TOP of the IP_BASE /16 (host
+    x.y.255.0, guest x.y.255.1), well clear of the agent pool near the
+    bottom of the block. Unlike a pool `Slot`, this link is NAT'd out to
+    the internet by the setup (the orchestrator is trusted infra), so it
+    is deliberately *not* one of the isolated `bbfc*` slots.
+
+    `index` is -1 (sentinel: not a pool index)."""
+    base16 = int(ipaddress.IPv4Address(ip_base())) & 0xFFFF0000
+    host = base16 + 0xFF00
+    return Slot(
+        index=-1,
+        iface=ORCH_IFACE,
+        host_ip=str(ipaddress.IPv4Address(host)),
+        guest_ip=str(ipaddress.IPv4Address(host + 1)),
+    )
+
+
 # --- fail-closed verification ---------------------------------------
 
 def _run_ok(argv: list[str]) -> bool:
diff --git a/nix/firecracker-netpool.nix b/nix/firecracker-netpool.nix
index 7def4fa..8e63cc1 100644
--- a/nix/firecracker-netpool.nix
+++ b/nix/firecracker-netpool.nix
@@ -49,10 +49,12 @@ let
   # /31 alignment == an even final octet (only bit 0 matters for base+2i).
   lastOctet = lib.toInt (lib.last (lib.splitString "." cfg.ipBase));
 
-  # The script needs ip/nft/sysctl + the usual coreutils. It gets every
-  # pool value via the unit's Environment=, so it never reads the shared
-  # defaults file (which isn't beside it once copied to the store).
-  runtimePath = with pkgs; [ iproute2 nftables procps coreutils gnused ];
+  # The script needs ip/nft/sysctl + the usual coreutils, plus iptables
+  # for the orchestrator link's DOCKER-USER accept (best-effort; skipped
+  # when Docker is absent). It gets every pool value via the unit's
+  # Environment=, so it never reads the shared defaults file (which isn't
+  # beside it once copied to the store).
+  runtimePath = with pkgs; [ iproute2 nftables iptables procps coreutils gnused ];
 
   ownEnv =
     if cfg.group != null
@@ -64,6 +66,7 @@ let
     BOT_BOTTLE_FC_IP_BASE = cfg.ipBase;
     BOT_BOTTLE_FC_IFACE_PREFIX = cfg.ifacePrefix;
     BOT_BOTTLE_FC_NFT_TABLE = cfg.tableName;
+    BOT_BOTTLE_FC_ORCH_IFACE = cfg.orchIface;
   } // ownEnv;
 in
 {
@@ -127,6 +130,19 @@ in
       description = "nftables table name for the isolation boundary. Must match netpool.NFT_TABLE.";
     };
 
+    orchIface = lib.mkOption {
+      type = lib.types.str;
+      default = defaults.BOT_BOTTLE_FC_ORCH_IFACE;
+      defaultText = lib.literalMD "the shared `netpool.defaults.env` value";
+      description = ''
+        TAP name for the orchestrator/gateway VM's dedicated link. Unlike
+        the isolated bbfc* agent pool, this link is NAT'd to the internet
+        (the orchestrator is trusted infra that builds agent images in-VM
+        and forwards agent egress upstream). Must match
+        BOT_BOTTLE_FC_ORCH_IFACE / netpool.ORCH_IFACE.
+      '';
+    };
+
     writeEnvFile = lib.mkOption {
       type = lib.types.bool;
       default = false;
@@ -176,6 +192,7 @@ in
         BOT_BOTTLE_FC_IP_BASE=${cfg.ipBase}
         BOT_BOTTLE_FC_IFACE_PREFIX=${cfg.ifacePrefix}
         BOT_BOTTLE_FC_NFT_TABLE=${cfg.tableName}
+        BOT_BOTTLE_FC_ORCH_IFACE=${cfg.orchIface}
       '';
     };
   };
diff --git a/scripts/firecracker-netpool.sh b/scripts/firecracker-netpool.sh
index 9f9d444..43161e1 100755
--- a/scripts/firecracker-netpool.sh
+++ b/scripts/firecracker-netpool.sh
@@ -63,12 +63,13 @@ POOL_SIZE="${BOT_BOTTLE_FC_POOL_SIZE:-$(_default BOT_BOTTLE_FC_POOL_SIZE)}"
 IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-$(_default BOT_BOTTLE_FC_IP_BASE)}"
 PREFIX="${BOT_BOTTLE_FC_IFACE_PREFIX:-$(_default BOT_BOTTLE_FC_IFACE_PREFIX)}"
 TABLE="${BOT_BOTTLE_FC_NFT_TABLE:-$(_default BOT_BOTTLE_FC_NFT_TABLE)}"
+ORCH_IFACE="${BOT_BOTTLE_FC_ORCH_IFACE:-$(_default BOT_BOTTLE_FC_ORCH_IFACE)}"
 OWNER="${BOT_BOTTLE_FC_OWNER:-${SUDO_USER:-$USER}}"
 GROUP="${BOT_BOTTLE_FC_GROUP:-}"
 
 # Fail loudly rather than provisioning a half/empty range if a value
 # resolved to nothing (env unset AND the shared file unreadable).
-for _v in POOL_SIZE IP_BASE PREFIX TABLE; do
+for _v in POOL_SIZE IP_BASE PREFIX TABLE ORCH_IFACE; do
     [ -n "${!_v}" ] || { echo "error: $_v unresolved (set BOT_BOTTLE_FC_* or fix $_DEFAULTS)" >&2; exit 1; }
 done
 
@@ -89,6 +90,13 @@ host_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 )); }
 guest_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 + 1 )); }
 iface()   { echo "${PREFIX}$1"; }
 
+# Orchestrator/gateway VM link: a /31 at the TOP of the IP_BASE /16
+# (host x.y.255.0, guest x.y.255.1), well clear of the agent pool near
+# the bottom of the block. Must match netpool.py:orch_slot().
+_orch_base() { echo $(( ($(_ip_to_int "$IP_BASE") & 0xFFFF0000) + 0xFF00 )); }
+orch_host()  { _int_to_ip "$(_orch_base)"; }
+orch_guest() { _int_to_ip $(( $(_orch_base) + 1 )); }
+
 require_root() {
     if [ "$(id -u)" -ne 0 ]; then
         echo "error: '$1' needs root (run under sudo)" >&2
@@ -125,8 +133,19 @@ cmd_up() {
         echo "  $dev  host=$host  guest=$(guest_ip "$i")  $own_desc"
     done
 
+    # The orchestrator/gateway VM's dedicated link. Same rootless-open
+    # ownership as the pool, but NAT'd to the internet (below) — it is
+    # trusted infra, not an isolated agent slot.
+    ip link show "$ORCH_IFACE" >/dev/null 2>&1 \
+        || ip tuntap add dev "$ORCH_IFACE" mode tap "${own_args[@]}"
+    ip addr replace "$(orch_host)/31" dev "$ORCH_IFACE"
+    ip link set "$ORCH_IFACE" up
+    echo "  $ORCH_IFACE  host=$(orch_host)  guest=$(orch_guest)  (NAT'd egress)  $own_desc"
+
     _install_nft
     echo "nftables table inet $TABLE installed (fail-closed boundary)"
+    _install_orch_egress
+    echo "orchestrator egress installed ($ORCH_IFACE -> NAT out)"
     echo "done."
 }
 
@@ -161,8 +180,63 @@ table inet $TABLE {
 EOF
 }
 
+# Give the orchestrator/gateway VM real internet egress (agent VMs get
+# none — that's the isolation table above). Three parts, because the
+# path must work both during bootstrap (Docker still present) and after
+# Docker is removed:
+#   * masquerade   — SNAT the orch guest /31 out the host uplink so its
+#                    RFC-1918 address can reach the internet.
+#   * nft forward  — accept the orch link's forward path (load-bearing
+#                    on a pure-nft host whose FORWARD policy drops; a
+#                    harmless no-op where forwarding is already open).
+#                    It never drops, so it can't weaken the isolation
+#                    table's agent drops.
+#   * DOCKER-USER  — during bootstrap Docker's FORWARD chain policy is
+#                    DROP; its sanctioned DOCKER-USER hook is the only
+#                    place a user ACCEPT survives. Best-effort + guarded
+#                    (skipped once Docker is gone).
+_install_orch_egress() {
+    nft -f - <<EOF
+table inet ${TABLE}_nat {
+    chain forward {
+        type filter hook forward priority -10; policy accept;
+        iifname "$ORCH_IFACE" accept
+        oifname "$ORCH_IFACE" ct state established,related accept
+    }
+    chain postrouting {
+        type nat hook postrouting priority 100; policy accept;
+        ip saddr $(orch_guest) oifname != "$ORCH_IFACE" masquerade
+    }
+}
+EOF
+    _docker_user_orch add
+}
+
+# Insert (add) or delete (del) the DOCKER-USER ACCEPT rules for the
+# orchestrator link, idempotently, only when the chain exists.
+_docker_user_orch() {
+    local op="$1" flag
+    command -v iptables >/dev/null 2>&1 || return 0
+    iptables -t filter -L DOCKER-USER >/dev/null 2>&1 || return 0
+    for flag in "-i" "-o"; do
+        if [ "$op" = add ]; then
+            iptables -C DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT 2>/dev/null \
+                || iptables -I DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT
+        else
+            iptables -D DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT 2>/dev/null || true
+        fi
+    done
+}
+
 cmd_down() {
     require_root down
+    _docker_user_orch del
+    nft delete table inet "${TABLE}_nat" 2>/dev/null || true
+    if ip link show "$ORCH_IFACE" >/dev/null 2>&1; then
+        ip link set "$ORCH_IFACE" down 2>/dev/null || true
+        ip tuntap del dev "$ORCH_IFACE" mode tap 2>/dev/null || true
+        echo "  removed $ORCH_IFACE"
+    fi
     nft delete table inet "$TABLE" 2>/dev/null || true
     for i in $(seq 0 $((POOL_SIZE-1))); do
         local dev ; dev="$(iface "$i")"
@@ -178,6 +252,8 @@ cmd_down() {
 cmd_status() {
     echo "table inet $TABLE:"
     nft list table inet "$TABLE" 2>/dev/null || echo "  (absent)"
+    echo "table inet ${TABLE}_nat (orchestrator egress):"
+    nft list table inet "${TABLE}_nat" 2>/dev/null || echo "  (absent)"
     echo "taps:"
     for i in $(seq 0 $((POOL_SIZE-1))); do
         local dev ; dev="$(iface "$i")"
@@ -185,6 +261,9 @@ cmd_status() {
             ip -brief addr show "$dev" | sed 's/^/  /'
         fi
     done
+    if ip -brief addr show "$ORCH_IFACE" >/dev/null 2>&1; then
+        ip -brief addr show "$ORCH_IFACE" | sed 's/^/  /'
+    fi
 }
 
 case "${1:-}" in
diff --git a/tests/unit/test_firecracker_helpers.py b/tests/unit/test_firecracker_helpers.py
index a0c7ada..d48c7a2 100644
--- a/tests/unit/test_firecracker_helpers.py
+++ b/tests/unit/test_firecracker_helpers.py
@@ -64,6 +64,21 @@ class TestNetpoolProbes(unittest.TestCase):
              patch.object(netpool, "tap_present", side_effect=[True, False]):
             self.assertEqual(["bbfc1"], netpool.missing_taps())
 
+    def test_orch_slot_is_top_of_ip_base_16(self):
+        # Dedicated orchestrator link: /31 at the top of the IP_BASE /16,
+        # clear of the pool (bottom of the block). Must match the shell
+        # script's orch_host()/orch_guest() and be a non-pool index.
+        with patch.dict("os.environ", {"BOT_BOTTLE_FC_IP_BASE": "10.243.0.0"}):
+            s = netpool.orch_slot()
+        self.assertEqual(netpool.ORCH_IFACE, s.iface)
+        self.assertEqual("10.243.255.0", s.host_ip)
+        self.assertEqual("10.243.255.1", s.guest_ip)
+        self.assertEqual(-1, s.index)
+        # Never collides with a pool slot's guest address.
+        with patch.dict("os.environ", {"BOT_BOTTLE_FC_IP_BASE": "10.243.0.0"}):
+            pool_guests = {sl.guest_ip for sl in netpool.all_slots()}
+        self.assertNotIn(s.guest_ip, pool_guests)
+
 
 class TestConsoleTail(unittest.TestCase):
     def test_reads_tail(self):
-- 
2.52.0