From 95981ea9d3c0c93ac875b528192ac9833f447b67 Mon Sep 17 00:00:00 2001 From: didericis Date: Tue, 14 Jul 2026 18:13:14 -0400 Subject: [PATCH] feat(firecracker): NAT'd egress link for the orchestrator/builder VM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The orchestrator/gateway VM is trusted infra, not an isolated agent: it builds agent images in-VM (buildah must FROM-pull + apt/npm) and, in the Stage B cutover, forwards agent egress upstream. Give it a dedicated TAP (`bborch0`) on a /31 at the top of the IP_BASE /16 (clear of the bbfc* agent pool at the bottom), NAT'd out the host uplink — while agent VMs keep their fail-closed, gateway-only isolation table. - netpool.defaults.env / netpool.py: new BOT_BOTTLE_FC_ORCH_IFACE + `orch_slot()` (index -1 sentinel; host x.y.255.0 / guest x.y.255.1). - scripts/firecracker-netpool.sh: create + address the orchestrator TAP; `bot_bottle_fc_nat` table masquerades its /31 out the uplink and accepts its forward path. Because bootstrap still runs Docker (whose FORWARD policy is DROP), a best-effort, guarded, idempotent DOCKER-USER ACCEPT is added too (skipped once Docker is gone). down/status updated. - nix/firecracker-netpool.nix: mirror the option, pass it via the unit Environment= (the store-copied script can't read the defaults file), and add iptables to the unit path for the DOCKER-USER step. Agent isolation is unchanged: the new rules only ever accept/masquerade the orchestrator link and never drop, so they can't weaken the bbfc* drops. Applied by re-running `sudo ./scripts/firecracker-netpool.sh up`. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck --- .../backend/firecracker/netpool.defaults.env | 8 ++ bot_bottle/backend/firecracker/netpool.py | 25 ++++++ nix/firecracker-netpool.nix | 25 +++++- scripts/firecracker-netpool.sh | 81 ++++++++++++++++++- tests/unit/test_firecracker_helpers.py | 15 ++++ 5 files changed, 149 insertions(+), 5 deletions(-) diff --git a/bot_bottle/backend/firecracker/netpool.defaults.env b/bot_bottle/backend/firecracker/netpool.defaults.env index 7532b3d..2277f1f 100644 --- a/bot_bottle/backend/firecracker/netpool.defaults.env +++ b/bot_bottle/backend/firecracker/netpool.defaults.env @@ -15,3 +15,11 @@ BOT_BOTTLE_FC_POOL_SIZE=8 BOT_BOTTLE_FC_IP_BASE=10.243.0.0 BOT_BOTTLE_FC_IFACE_PREFIX=bbfc BOT_BOTTLE_FC_NFT_TABLE=bot_bottle_fc +# The orchestrator/gateway VM's own TAP — a dedicated link OUTSIDE the +# bbfc* agent pool. Unlike agent VMs (which reach only their gateway), +# the orchestrator is trusted infra that needs real NAT'd internet +# egress: to FROM-pull + apt/npm during in-VM agent-image builds +# (buildah) and to forward agent egress upstream (Stage B gateway). Its +# /31 is the top of the IP_BASE /16 (host x.y.255.0, guest x.y.255.1), +# clear of the pool near the bottom of the block. +BOT_BOTTLE_FC_ORCH_IFACE=bborch0 diff --git a/bot_bottle/backend/firecracker/netpool.py b/bot_bottle/backend/firecracker/netpool.py index faad11f..3ee6653 100644 --- a/bot_bottle/backend/firecracker/netpool.py +++ b/bot_bottle/backend/firecracker/netpool.py @@ -79,6 +79,12 @@ def _cfg(key: str) -> str: IFACE_PREFIX = _cfg("BOT_BOTTLE_FC_IFACE_PREFIX") NFT_TABLE = _cfg("BOT_BOTTLE_FC_NFT_TABLE") +# The orchestrator/gateway VM's dedicated TAP — outside the bbfc* agent +# pool and, unlike it, NAT'd to the internet (see `orch_slot`). The +# orchestrator is trusted infra: it builds agent images in-VM (buildah +# needs to FROM-pull + apt/npm) and forwards agent egress upstream. +ORCH_IFACE = _cfg("BOT_BOTTLE_FC_ORCH_IFACE") + def pool_size() -> int: return int(_cfg("BOT_BOTTLE_FC_POOL_SIZE")) @@ -123,6 +129,25 @@ def all_slots() -> list[Slot]: return [slot(i) for i in range(pool_size())] +def orch_slot() -> Slot: + """The orchestrator/gateway VM's dedicated link — its own TAP + (`ORCH_IFACE`) on a /31 at the TOP of the IP_BASE /16 (host + x.y.255.0, guest x.y.255.1), well clear of the agent pool near the + bottom of the block. Unlike a pool `Slot`, this link is NAT'd out to + the internet by the setup (the orchestrator is trusted infra), so it + is deliberately *not* one of the isolated `bbfc*` slots. + + `index` is -1 (sentinel: not a pool index).""" + base16 = int(ipaddress.IPv4Address(ip_base())) & 0xFFFF0000 + host = base16 + 0xFF00 + return Slot( + index=-1, + iface=ORCH_IFACE, + host_ip=str(ipaddress.IPv4Address(host)), + guest_ip=str(ipaddress.IPv4Address(host + 1)), + ) + + # --- fail-closed verification --------------------------------------- def _run_ok(argv: list[str]) -> bool: diff --git a/nix/firecracker-netpool.nix b/nix/firecracker-netpool.nix index 7def4fa..8e63cc1 100644 --- a/nix/firecracker-netpool.nix +++ b/nix/firecracker-netpool.nix @@ -49,10 +49,12 @@ let # /31 alignment == an even final octet (only bit 0 matters for base+2i). lastOctet = lib.toInt (lib.last (lib.splitString "." cfg.ipBase)); - # The script needs ip/nft/sysctl + the usual coreutils. It gets every - # pool value via the unit's Environment=, so it never reads the shared - # defaults file (which isn't beside it once copied to the store). - runtimePath = with pkgs; [ iproute2 nftables procps coreutils gnused ]; + # The script needs ip/nft/sysctl + the usual coreutils, plus iptables + # for the orchestrator link's DOCKER-USER accept (best-effort; skipped + # when Docker is absent). It gets every pool value via the unit's + # Environment=, so it never reads the shared defaults file (which isn't + # beside it once copied to the store). + runtimePath = with pkgs; [ iproute2 nftables iptables procps coreutils gnused ]; ownEnv = if cfg.group != null @@ -64,6 +66,7 @@ let BOT_BOTTLE_FC_IP_BASE = cfg.ipBase; BOT_BOTTLE_FC_IFACE_PREFIX = cfg.ifacePrefix; BOT_BOTTLE_FC_NFT_TABLE = cfg.tableName; + BOT_BOTTLE_FC_ORCH_IFACE = cfg.orchIface; } // ownEnv; in { @@ -127,6 +130,19 @@ in description = "nftables table name for the isolation boundary. Must match netpool.NFT_TABLE."; }; + orchIface = lib.mkOption { + type = lib.types.str; + default = defaults.BOT_BOTTLE_FC_ORCH_IFACE; + defaultText = lib.literalMD "the shared `netpool.defaults.env` value"; + description = '' + TAP name for the orchestrator/gateway VM's dedicated link. Unlike + the isolated bbfc* agent pool, this link is NAT'd to the internet + (the orchestrator is trusted infra that builds agent images in-VM + and forwards agent egress upstream). Must match + BOT_BOTTLE_FC_ORCH_IFACE / netpool.ORCH_IFACE. + ''; + }; + writeEnvFile = lib.mkOption { type = lib.types.bool; default = false; @@ -176,6 +192,7 @@ in BOT_BOTTLE_FC_IP_BASE=${cfg.ipBase} BOT_BOTTLE_FC_IFACE_PREFIX=${cfg.ifacePrefix} BOT_BOTTLE_FC_NFT_TABLE=${cfg.tableName} + BOT_BOTTLE_FC_ORCH_IFACE=${cfg.orchIface} ''; }; }; diff --git a/scripts/firecracker-netpool.sh b/scripts/firecracker-netpool.sh index 9f9d444..43161e1 100755 --- a/scripts/firecracker-netpool.sh +++ b/scripts/firecracker-netpool.sh @@ -63,12 +63,13 @@ POOL_SIZE="${BOT_BOTTLE_FC_POOL_SIZE:-$(_default BOT_BOTTLE_FC_POOL_SIZE)}" IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-$(_default BOT_BOTTLE_FC_IP_BASE)}" PREFIX="${BOT_BOTTLE_FC_IFACE_PREFIX:-$(_default BOT_BOTTLE_FC_IFACE_PREFIX)}" TABLE="${BOT_BOTTLE_FC_NFT_TABLE:-$(_default BOT_BOTTLE_FC_NFT_TABLE)}" +ORCH_IFACE="${BOT_BOTTLE_FC_ORCH_IFACE:-$(_default BOT_BOTTLE_FC_ORCH_IFACE)}" OWNER="${BOT_BOTTLE_FC_OWNER:-${SUDO_USER:-$USER}}" GROUP="${BOT_BOTTLE_FC_GROUP:-}" # Fail loudly rather than provisioning a half/empty range if a value # resolved to nothing (env unset AND the shared file unreadable). -for _v in POOL_SIZE IP_BASE PREFIX TABLE; do +for _v in POOL_SIZE IP_BASE PREFIX TABLE ORCH_IFACE; do [ -n "${!_v}" ] || { echo "error: $_v unresolved (set BOT_BOTTLE_FC_* or fix $_DEFAULTS)" >&2; exit 1; } done @@ -89,6 +90,13 @@ host_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 )); } guest_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 + 1 )); } iface() { echo "${PREFIX}$1"; } +# Orchestrator/gateway VM link: a /31 at the TOP of the IP_BASE /16 +# (host x.y.255.0, guest x.y.255.1), well clear of the agent pool near +# the bottom of the block. Must match netpool.py:orch_slot(). +_orch_base() { echo $(( ($(_ip_to_int "$IP_BASE") & 0xFFFF0000) + 0xFF00 )); } +orch_host() { _int_to_ip "$(_orch_base)"; } +orch_guest() { _int_to_ip $(( $(_orch_base) + 1 )); } + require_root() { if [ "$(id -u)" -ne 0 ]; then echo "error: '$1' needs root (run under sudo)" >&2 @@ -125,8 +133,19 @@ cmd_up() { echo " $dev host=$host guest=$(guest_ip "$i") $own_desc" done + # The orchestrator/gateway VM's dedicated link. Same rootless-open + # ownership as the pool, but NAT'd to the internet (below) — it is + # trusted infra, not an isolated agent slot. + ip link show "$ORCH_IFACE" >/dev/null 2>&1 \ + || ip tuntap add dev "$ORCH_IFACE" mode tap "${own_args[@]}" + ip addr replace "$(orch_host)/31" dev "$ORCH_IFACE" + ip link set "$ORCH_IFACE" up + echo " $ORCH_IFACE host=$(orch_host) guest=$(orch_guest) (NAT'd egress) $own_desc" + _install_nft echo "nftables table inet $TABLE installed (fail-closed boundary)" + _install_orch_egress + echo "orchestrator egress installed ($ORCH_IFACE -> NAT out)" echo "done." } @@ -161,8 +180,63 @@ table inet $TABLE { EOF } +# Give the orchestrator/gateway VM real internet egress (agent VMs get +# none — that's the isolation table above). Three parts, because the +# path must work both during bootstrap (Docker still present) and after +# Docker is removed: +# * masquerade — SNAT the orch guest /31 out the host uplink so its +# RFC-1918 address can reach the internet. +# * nft forward — accept the orch link's forward path (load-bearing +# on a pure-nft host whose FORWARD policy drops; a +# harmless no-op where forwarding is already open). +# It never drops, so it can't weaken the isolation +# table's agent drops. +# * DOCKER-USER — during bootstrap Docker's FORWARD chain policy is +# DROP; its sanctioned DOCKER-USER hook is the only +# place a user ACCEPT survives. Best-effort + guarded +# (skipped once Docker is gone). +_install_orch_egress() { + nft -f - </dev/null 2>&1 || return 0 + iptables -t filter -L DOCKER-USER >/dev/null 2>&1 || return 0 + for flag in "-i" "-o"; do + if [ "$op" = add ]; then + iptables -C DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT 2>/dev/null \ + || iptables -I DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT + else + iptables -D DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT 2>/dev/null || true + fi + done +} + cmd_down() { require_root down + _docker_user_orch del + nft delete table inet "${TABLE}_nat" 2>/dev/null || true + if ip link show "$ORCH_IFACE" >/dev/null 2>&1; then + ip link set "$ORCH_IFACE" down 2>/dev/null || true + ip tuntap del dev "$ORCH_IFACE" mode tap 2>/dev/null || true + echo " removed $ORCH_IFACE" + fi nft delete table inet "$TABLE" 2>/dev/null || true for i in $(seq 0 $((POOL_SIZE-1))); do local dev ; dev="$(iface "$i")" @@ -178,6 +252,8 @@ cmd_down() { cmd_status() { echo "table inet $TABLE:" nft list table inet "$TABLE" 2>/dev/null || echo " (absent)" + echo "table inet ${TABLE}_nat (orchestrator egress):" + nft list table inet "${TABLE}_nat" 2>/dev/null || echo " (absent)" echo "taps:" for i in $(seq 0 $((POOL_SIZE-1))); do local dev ; dev="$(iface "$i")" @@ -185,6 +261,9 @@ cmd_status() { ip -brief addr show "$dev" | sed 's/^/ /' fi done + if ip -brief addr show "$ORCH_IFACE" >/dev/null 2>&1; then + ip -brief addr show "$ORCH_IFACE" | sed 's/^/ /' + fi } case "${1:-}" in diff --git a/tests/unit/test_firecracker_helpers.py b/tests/unit/test_firecracker_helpers.py index a0c7ada..d48c7a2 100644 --- a/tests/unit/test_firecracker_helpers.py +++ b/tests/unit/test_firecracker_helpers.py @@ -64,6 +64,21 @@ class TestNetpoolProbes(unittest.TestCase): patch.object(netpool, "tap_present", side_effect=[True, False]): self.assertEqual(["bbfc1"], netpool.missing_taps()) + def test_orch_slot_is_top_of_ip_base_16(self): + # Dedicated orchestrator link: /31 at the top of the IP_BASE /16, + # clear of the pool (bottom of the block). Must match the shell + # script's orch_host()/orch_guest() and be a non-pool index. + with patch.dict("os.environ", {"BOT_BOTTLE_FC_IP_BASE": "10.243.0.0"}): + s = netpool.orch_slot() + self.assertEqual(netpool.ORCH_IFACE, s.iface) + self.assertEqual("10.243.255.0", s.host_ip) + self.assertEqual("10.243.255.1", s.guest_ip) + self.assertEqual(-1, s.index) + # Never collides with a pool slot's guest address. + with patch.dict("os.environ", {"BOT_BOTTLE_FC_IP_BASE": "10.243.0.0"}): + pool_guests = {sl.guest_ip for sl in netpool.all_slots()} + self.assertNotIn(s.guest_ip, pool_guests) + class TestConsoleTail(unittest.TestCase): def test_reads_tail(self):