From 8d7e5956e14f1a2d24724539685ed1fd86dc8b2a Mon Sep 17 00:00:00 2001 From: didericis Date: Thu, 16 Jul 2026 12:47:13 -0400 Subject: [PATCH] feat(firecracker): run the gateway data plane in the infra VM too (Stage B, 2/n) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single infra VM now runs BOTH the orchestrator control plane and the gateway data plane (egress / supervise / git-http), multi-tenant against the local control plane — the single-VM shape from the Stage B design. - Dockerfile.infra: the firecracker infra image = the gateway image + the baked control-plane source (FROM bot-bottle-gateway, COPY bot_bottle). Reuses the gateway payload rather than copying mitmproxy/ gitleaks into a third image; the docker backend keeps its two separate images. - infra_vm: build from source (gateway then infra image), and the PID-1 init now also launches `gateway_init` with BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:8099. Adds `gateway_ca_pem` (fetch the mitmproxy CA over SSH) and the agent-facing port constants. - init exports PATH — a bare-init shell resolves its own execs via a built-in default path, but that isn't in the environment, so gateway_init's `python3 ...` daemons would otherwise fail to spawn. Verified on a KVM host: infra VM boots, control plane /health -> 200, and egress:9099 / supervise:9100 / git-http:9420 all listen and are reachable from the host over the TAP link; the gateway CA is retrievable. (git-gate stays down until a bottle provisions its per-bottle entrypoint/creds, same as a fresh docker gateway — non-fatal, the supervisor keeps the rest up.) Next: agent->gateway VM-to-VM routing so bbfc* VMs reach these ports at the infra VM and nowhere else. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck --- Dockerfile.infra | 21 +++++++ bot_bottle/backend/firecracker/infra_vm.py | 69 ++++++++++++++++++++-- tests/unit/test_firecracker_infra_vm.py | 18 +++++- 3 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 Dockerfile.infra diff --git a/Dockerfile.infra b/Dockerfile.infra new file mode 100644 index 0000000..8a9e446 --- /dev/null +++ b/Dockerfile.infra @@ -0,0 +1,21 @@ +# Firecracker single infra-VM image (PRD 0070 Stage B). +# +# The per-host infra VM runs BOTH the orchestrator control plane and the +# gateway data plane in one microVM (see backend/firecracker/infra_vm.py). +# It layers the stdlib-only control-plane source onto the gateway +# data-plane image, so the single VM has mitmproxy / git / gitleaks / +# supervise (gateway) AND `bot_bottle.orchestrator` (control plane) — +# reusing the gateway payload rather than copying it into a third image. +# +# The docker backend keeps the two images separate (a lean orchestrator +# container + a gateway container); this combined image exists only for +# the Firecracker single-VM cut. If the egress blast radius ever warrants +# it, splitting egress back into its own VM is a routing change, not a +# repackaging (see PRD 0070's "secret concentration"). +FROM bot-bottle-gateway:latest + +# The gateway image copies only its daemon modules flat under /app; the +# control plane needs the whole (stdlib-only) package. Both coexist: +# `/app/gateway_init.py` keeps its flat-sibling imports, and +# `python3 -m bot_bottle.orchestrator` resolves /app/bot_bottle. +COPY bot_bottle /app/bot_bottle diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index 1568a1a..838bfe4 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -17,6 +17,7 @@ surface. from __future__ import annotations +import subprocess import time import urllib.error import urllib.request @@ -24,10 +25,25 @@ from dataclasses import dataclass from pathlib import Path from ...log import die, info +from ..docker import util as docker_mod from . import firecracker_vm, netpool, util -_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest" +# The single infra-VM image: gateway data plane + baked control-plane source +# (Dockerfile.infra FROM the gateway image). Built from source by default; +# a pull-from-registry mode lands later. +_INFRA_IMAGE = "bot-bottle-infra:latest" +_GATEWAY_IMAGE = "bot-bottle-gateway:latest" +_REPO_ROOT = Path(__file__).resolve().parents[3] + CONTROL_PLANE_PORT = 8099 +# Gateway data-plane ports (agent-facing): egress proxy, supervise MCP, +# git-http. Reached by agent VMs over VM-to-VM routing (added next). +EGRESS_PORT = 9099 +SUPERVISE_PORT = 9100 +GIT_HTTP_PORT = 9420 +# mitmproxy writes its CA here a beat after start; agents install it to trust +# the gateway's TLS interception. +_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem" # The infra VM makes direct upstream connections (gateway egress, and buildah # during builds), and the kernel `ip=` cmdline sets no resolver. Public for @@ -36,6 +52,7 @@ _INFRA_RESOLVER = "1.1.1.1" _HEALTH_TIMEOUT_SECONDS = 45.0 _HEALTH_POLL_SECONDS = 0.5 +_CA_TIMEOUT_SECONDS = 30.0 @dataclass @@ -53,12 +70,41 @@ class InfraVm: def terminate(self) -> None: self.vm.terminate() + def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str: + """The gateway's mitmproxy CA (PEM) that agents install to trust its + TLS interception. Generated a moment after boot, so this polls over + SSH until it appears (mirrors DockerGateway.ca_cert_pem).""" + deadline = time.monotonic() + timeout + while True: + proc = subprocess.run( + util.ssh_base_argv(self.private_key, self.guest_ip) + + [f"cat {_GATEWAY_CA_PATH}"], + capture_output=True, text=True, timeout=15, check=False, + ) + if proc.returncode == 0 and "BEGIN CERTIFICATE" in proc.stdout: + return proc.stdout + if time.monotonic() >= deadline: + die(f"gateway CA not available after {timeout:g}s: " + f"{proc.stderr.strip() or 'empty'}") + time.sleep(_HEALTH_POLL_SECONDS) + + +def ensure_built() -> None: + """Build the infra image from source (bootstrap via host docker): the + gateway data-plane image, then the infra image that layers the + control-plane source onto it. A pull-from-registry mode replaces this + later.""" + docker_mod.build_image( + _GATEWAY_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.gateway") + docker_mod.build_image( + _INFRA_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.infra") + def build_infra_rootfs_dir() -> Path: - """The infra VM's base rootfs: the orchestrator image (with its baked, - stdlib-only source) prepared with the control-plane init as PID 1.""" + """The infra VM's base rootfs: the infra image prepared with the + control-plane + gateway init as PID 1.""" return util.build_base_rootfs_dir( - _ORCHESTRATOR_IMAGE, variant="-infra", init_script=_infra_init(), + _INFRA_IMAGE, variant="-infra", init_script=_infra_init(), ) @@ -111,8 +157,8 @@ def wait_for_health( def _infra_init() -> str: """PID-1 init for the infra VM: mount the pseudo-filesystems, wire a - resolver, start dropbear (debug SSH), then launch the control plane. The - gateway daemons are added here in the next step.""" + resolver, start dropbear (debug SSH), then launch the control plane and + the gateway data plane (multi-tenant against the local control plane).""" return f"""#!/bin/sh # bot-bottle Firecracker infra VM init (PID 1). mount -t proc proc /proc 2>/dev/null @@ -121,6 +167,12 @@ mount -t devtmpfs dev /dev 2>/dev/null mkdir -p /dev/pts && mount -t devpts devpts /dev/pts 2>/dev/null mount -o remount,rw / 2>/dev/null +# Export a real PATH: a bare-init shell resolves its own execs via a +# built-in default path, but that isn't in the *environment*, so +# gateway_init's subprocess daemons (spawned as `python3 ...`) would +# inherit no PATH and fail to find python3. Export it for all children. +export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + # Direct upstream resolver (control-plane / gateway egress + buildah). printf 'nameserver {_INFRA_RESOLVER}\\n' > /etc/resolv.conf 2>/dev/null @@ -140,6 +192,11 @@ cd /app BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -m bot_bottle.orchestrator \\ --host 0.0.0.0 --port {CONTROL_PLANE_PORT} --broker stub & +# Gateway data plane (egress / git-gate / supervise), multi-tenant: each +# request resolves source-IP -> policy against the local control plane. +BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{CONTROL_PLANE_PORT} \\ + python3 /app/gateway_init.py & + # Reap as PID 1; children are backgrounded, so `wait` blocks. while : ; do wait ; done """ diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index ab21758..3e571a9 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -30,10 +30,22 @@ class TestBuildInfraRootfs(unittest.TestCase): build.return_value = Path("/cache/rootfs/x-infra") infra_vm.build_infra_rootfs_dir() build.assert_called_once() - self.assertEqual(infra_vm._ORCHESTRATOR_IMAGE, build.call_args.args[0]) + self.assertEqual(infra_vm._INFRA_IMAGE, build.call_args.args[0]) self.assertEqual("-infra", build.call_args.kwargs["variant"]) - # The init runs the control plane, not the SSH-only agent init. - self.assertIn("bot_bottle.orchestrator", build.call_args.kwargs["init_script"]) + # The init runs BOTH the control plane and the gateway data plane, + # and exports PATH so gateway_init's subprocess daemons find python3. + init = build.call_args.kwargs["init_script"] + self.assertIn("bot_bottle.orchestrator", init) + self.assertIn("gateway_init.py", init) + self.assertIn("export PATH=", init) + + +class TestEnsureBuilt(unittest.TestCase): + def test_builds_gateway_then_infra(self): + with patch.object(infra_vm.docker_mod, "build_image") as build: + infra_vm.ensure_built() + tags = [c.args[0] for c in build.call_args_list] + self.assertEqual([infra_vm._GATEWAY_IMAGE, infra_vm._INFRA_IMAGE], tags) class TestWaitForHealth(unittest.TestCase):