feat(firecracker): run the gateway data plane in the infra VM too (Stage B, 2/n)
The single infra VM now runs BOTH the orchestrator control plane and the gateway data plane (egress / supervise / git-http), multi-tenant against the local control plane — the single-VM shape from the Stage B design. - Dockerfile.infra: the firecracker infra image = the gateway image + the baked control-plane source (FROM bot-bottle-gateway, COPY bot_bottle). Reuses the gateway payload rather than copying mitmproxy/ gitleaks into a third image; the docker backend keeps its two separate images. - infra_vm: build from source (gateway then infra image), and the PID-1 init now also launches `gateway_init` with BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:8099. Adds `gateway_ca_pem` (fetch the mitmproxy CA over SSH) and the agent-facing port constants. - init exports PATH — a bare-init shell resolves its own execs via a built-in default path, but that isn't in the environment, so gateway_init's `python3 ...` daemons would otherwise fail to spawn. Verified on a KVM host: infra VM boots, control plane /health -> 200, and egress:9099 / supervise:9100 / git-http:9420 all listen and are reachable from the host over the TAP link; the gateway CA is retrievable. (git-gate stays down until a bottle provisions its per-bottle entrypoint/creds, same as a fresh docker gateway — non-fatal, the supervisor keeps the rest up.) Next: agent->gateway VM-to-VM routing so bbfc* VMs reach these ports at the infra VM and nowhere else. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
# Firecracker single infra-VM image (PRD 0070 Stage B).
|
||||
#
|
||||
# The per-host infra VM runs BOTH the orchestrator control plane and the
|
||||
# gateway data plane in one microVM (see backend/firecracker/infra_vm.py).
|
||||
# It layers the stdlib-only control-plane source onto the gateway
|
||||
# data-plane image, so the single VM has mitmproxy / git / gitleaks /
|
||||
# supervise (gateway) AND `bot_bottle.orchestrator` (control plane) —
|
||||
# reusing the gateway payload rather than copying it into a third image.
|
||||
#
|
||||
# The docker backend keeps the two images separate (a lean orchestrator
|
||||
# container + a gateway container); this combined image exists only for
|
||||
# the Firecracker single-VM cut. If the egress blast radius ever warrants
|
||||
# it, splitting egress back into its own VM is a routing change, not a
|
||||
# repackaging (see PRD 0070's "secret concentration").
|
||||
FROM bot-bottle-gateway:latest
|
||||
|
||||
# The gateway image copies only its daemon modules flat under /app; the
|
||||
# control plane needs the whole (stdlib-only) package. Both coexist:
|
||||
# `/app/gateway_init.py` keeps its flat-sibling imports, and
|
||||
# `python3 -m bot_bottle.orchestrator` resolves /app/bot_bottle.
|
||||
COPY bot_bottle /app/bot_bottle
|
||||
@@ -17,6 +17,7 @@ surface.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
@@ -24,10 +25,25 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ...log import die, info
|
||||
from ..docker import util as docker_mod
|
||||
from . import firecracker_vm, netpool, util
|
||||
|
||||
_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:latest"
|
||||
# The single infra-VM image: gateway data plane + baked control-plane source
|
||||
# (Dockerfile.infra FROM the gateway image). Built from source by default;
|
||||
# a pull-from-registry mode lands later.
|
||||
_INFRA_IMAGE = "bot-bottle-infra:latest"
|
||||
_GATEWAY_IMAGE = "bot-bottle-gateway:latest"
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
CONTROL_PLANE_PORT = 8099
|
||||
# Gateway data-plane ports (agent-facing): egress proxy, supervise MCP,
|
||||
# git-http. Reached by agent VMs over VM-to-VM routing (added next).
|
||||
EGRESS_PORT = 9099
|
||||
SUPERVISE_PORT = 9100
|
||||
GIT_HTTP_PORT = 9420
|
||||
# mitmproxy writes its CA here a beat after start; agents install it to trust
|
||||
# the gateway's TLS interception.
|
||||
_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem"
|
||||
|
||||
# The infra VM makes direct upstream connections (gateway egress, and buildah
|
||||
# during builds), and the kernel `ip=` cmdline sets no resolver. Public for
|
||||
@@ -36,6 +52,7 @@ _INFRA_RESOLVER = "1.1.1.1"
|
||||
|
||||
_HEALTH_TIMEOUT_SECONDS = 45.0
|
||||
_HEALTH_POLL_SECONDS = 0.5
|
||||
_CA_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -53,12 +70,41 @@ class InfraVm:
|
||||
def terminate(self) -> None:
|
||||
self.vm.terminate()
|
||||
|
||||
def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str:
|
||||
"""The gateway's mitmproxy CA (PEM) that agents install to trust its
|
||||
TLS interception. Generated a moment after boot, so this polls over
|
||||
SSH until it appears (mirrors DockerGateway.ca_cert_pem)."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
proc = subprocess.run(
|
||||
util.ssh_base_argv(self.private_key, self.guest_ip)
|
||||
+ [f"cat {_GATEWAY_CA_PATH}"],
|
||||
capture_output=True, text=True, timeout=15, check=False,
|
||||
)
|
||||
if proc.returncode == 0 and "BEGIN CERTIFICATE" in proc.stdout:
|
||||
return proc.stdout
|
||||
if time.monotonic() >= deadline:
|
||||
die(f"gateway CA not available after {timeout:g}s: "
|
||||
f"{proc.stderr.strip() or 'empty'}")
|
||||
time.sleep(_HEALTH_POLL_SECONDS)
|
||||
|
||||
|
||||
def ensure_built() -> None:
|
||||
"""Build the infra image from source (bootstrap via host docker): the
|
||||
gateway data-plane image, then the infra image that layers the
|
||||
control-plane source onto it. A pull-from-registry mode replaces this
|
||||
later."""
|
||||
docker_mod.build_image(
|
||||
_GATEWAY_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.gateway")
|
||||
docker_mod.build_image(
|
||||
_INFRA_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.infra")
|
||||
|
||||
|
||||
def build_infra_rootfs_dir() -> Path:
|
||||
"""The infra VM's base rootfs: the orchestrator image (with its baked,
|
||||
stdlib-only source) prepared with the control-plane init as PID 1."""
|
||||
"""The infra VM's base rootfs: the infra image prepared with the
|
||||
control-plane + gateway init as PID 1."""
|
||||
return util.build_base_rootfs_dir(
|
||||
_ORCHESTRATOR_IMAGE, variant="-infra", init_script=_infra_init(),
|
||||
_INFRA_IMAGE, variant="-infra", init_script=_infra_init(),
|
||||
)
|
||||
|
||||
|
||||
@@ -111,8 +157,8 @@ def wait_for_health(
|
||||
|
||||
def _infra_init() -> str:
|
||||
"""PID-1 init for the infra VM: mount the pseudo-filesystems, wire a
|
||||
resolver, start dropbear (debug SSH), then launch the control plane. The
|
||||
gateway daemons are added here in the next step."""
|
||||
resolver, start dropbear (debug SSH), then launch the control plane and
|
||||
the gateway data plane (multi-tenant against the local control plane)."""
|
||||
return f"""#!/bin/sh
|
||||
# bot-bottle Firecracker infra VM init (PID 1).
|
||||
mount -t proc proc /proc 2>/dev/null
|
||||
@@ -121,6 +167,12 @@ mount -t devtmpfs dev /dev 2>/dev/null
|
||||
mkdir -p /dev/pts && mount -t devpts devpts /dev/pts 2>/dev/null
|
||||
mount -o remount,rw / 2>/dev/null
|
||||
|
||||
# Export a real PATH: a bare-init shell resolves its own execs via a
|
||||
# built-in default path, but that isn't in the *environment*, so
|
||||
# gateway_init's subprocess daemons (spawned as `python3 ...`) would
|
||||
# inherit no PATH and fail to find python3. Export it for all children.
|
||||
export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
|
||||
# Direct upstream resolver (control-plane / gateway egress + buildah).
|
||||
printf 'nameserver {_INFRA_RESOLVER}\\n' > /etc/resolv.conf 2>/dev/null
|
||||
|
||||
@@ -140,6 +192,11 @@ cd /app
|
||||
BOT_BOTTLE_ROOT=/var/lib/bot-bottle python3 -m bot_bottle.orchestrator \\
|
||||
--host 0.0.0.0 --port {CONTROL_PLANE_PORT} --broker stub &
|
||||
|
||||
# Gateway data plane (egress / git-gate / supervise), multi-tenant: each
|
||||
# request resolves source-IP -> policy against the local control plane.
|
||||
BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{CONTROL_PLANE_PORT} \\
|
||||
python3 /app/gateway_init.py &
|
||||
|
||||
# Reap as PID 1; children are backgrounded, so `wait` blocks.
|
||||
while : ; do wait ; done
|
||||
"""
|
||||
|
||||
@@ -30,10 +30,22 @@ class TestBuildInfraRootfs(unittest.TestCase):
|
||||
build.return_value = Path("/cache/rootfs/x-infra")
|
||||
infra_vm.build_infra_rootfs_dir()
|
||||
build.assert_called_once()
|
||||
self.assertEqual(infra_vm._ORCHESTRATOR_IMAGE, build.call_args.args[0])
|
||||
self.assertEqual(infra_vm._INFRA_IMAGE, build.call_args.args[0])
|
||||
self.assertEqual("-infra", build.call_args.kwargs["variant"])
|
||||
# The init runs the control plane, not the SSH-only agent init.
|
||||
self.assertIn("bot_bottle.orchestrator", build.call_args.kwargs["init_script"])
|
||||
# The init runs BOTH the control plane and the gateway data plane,
|
||||
# and exports PATH so gateway_init's subprocess daemons find python3.
|
||||
init = build.call_args.kwargs["init_script"]
|
||||
self.assertIn("bot_bottle.orchestrator", init)
|
||||
self.assertIn("gateway_init.py", init)
|
||||
self.assertIn("export PATH=", init)
|
||||
|
||||
|
||||
class TestEnsureBuilt(unittest.TestCase):
|
||||
def test_builds_gateway_then_infra(self):
|
||||
with patch.object(infra_vm.docker_mod, "build_image") as build:
|
||||
infra_vm.ensure_built()
|
||||
tags = [c.args[0] for c in build.call_args_list]
|
||||
self.assertEqual([infra_vm._GATEWAY_IMAGE, infra_vm._INFRA_IMAGE], tags)
|
||||
|
||||
|
||||
class TestWaitForHealth(unittest.TestCase):
|
||||
|
||||
Reference in New Issue
Block a user