feat(firecracker): NAT'd egress link for the orchestrator/builder VM

The orchestrator/gateway VM is trusted infra, not an isolated agent: it
builds agent images in-VM (buildah must FROM-pull + apt/npm) and, in the
Stage B cutover, forwards agent egress upstream. Give it a dedicated TAP
(`bborch0`) on a /31 at the top of the IP_BASE /16 (clear of the bbfc*
agent pool at the bottom), NAT'd out the host uplink — while agent VMs
keep their fail-closed, gateway-only isolation table.

- netpool.defaults.env / netpool.py: new BOT_BOTTLE_FC_ORCH_IFACE +
  `orch_slot()` (index -1 sentinel; host x.y.255.0 / guest x.y.255.1).
- scripts/firecracker-netpool.sh: create + address the orchestrator TAP;
  `bot_bottle_fc_nat` table masquerades its /31 out the uplink and
  accepts its forward path. Because bootstrap still runs Docker (whose
  FORWARD policy is DROP), a best-effort, guarded, idempotent DOCKER-USER
  ACCEPT is added too (skipped once Docker is gone). down/status updated.
- nix/firecracker-netpool.nix: mirror the option, pass it via the unit
  Environment= (the store-copied script can't read the defaults file),
  and add iptables to the unit path for the DOCKER-USER step.

Agent isolation is unchanged: the new rules only ever accept/masquerade
the orchestrator link and never drop, so they can't weaken the bbfc*
drops. Applied by re-running `sudo ./scripts/firecracker-netpool.sh up`.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck

bot_bottle/backend/firecracker/netpool.defaults.env

@@ -15,3 +15,11 @@ BOT_BOTTLE_FC_POOL_SIZE=8
 BOT_BOTTLE_FC_IP_BASE=10.243.0.0
 BOT_BOTTLE_FC_IFACE_PREFIX=bbfc
 BOT_BOTTLE_FC_NFT_TABLE=bot_bottle_fc
+# The orchestrator/gateway VM's own TAP — a dedicated link OUTSIDE the
+# bbfc* agent pool. Unlike agent VMs (which reach only their gateway),
+# the orchestrator is trusted infra that needs real NAT'd internet
+# egress: to FROM-pull + apt/npm during in-VM agent-image builds
+# (buildah) and to forward agent egress upstream (Stage B gateway). Its
+# /31 is the top of the IP_BASE /16 (host x.y.255.0, guest x.y.255.1),
+# clear of the pool near the bottom of the block.
+BOT_BOTTLE_FC_ORCH_IFACE=bborch0

bot_bottle/backend/firecracker/netpool.py

@@ -79,6 +79,12 @@ def _cfg(key: str) -> str:
 IFACE_PREFIX = _cfg("BOT_BOTTLE_FC_IFACE_PREFIX")
 NFT_TABLE = _cfg("BOT_BOTTLE_FC_NFT_TABLE")
 
+# The orchestrator/gateway VM's dedicated TAP — outside the bbfc* agent
+# pool and, unlike it, NAT'd to the internet (see `orch_slot`). The
+# orchestrator is trusted infra: it builds agent images in-VM (buildah
+# needs to FROM-pull + apt/npm) and forwards agent egress upstream.
+ORCH_IFACE = _cfg("BOT_BOTTLE_FC_ORCH_IFACE")
+
 
 def pool_size() -> int:
     return int(_cfg("BOT_BOTTLE_FC_POOL_SIZE"))
@@ -123,6 +129,25 @@ def all_slots() -> list[Slot]:
     return [slot(i) for i in range(pool_size())]
 
 
+def orch_slot() -> Slot:
+    """The orchestrator/gateway VM's dedicated link — its own TAP
+    (`ORCH_IFACE`) on a /31 at the TOP of the IP_BASE /16 (host
+    x.y.255.0, guest x.y.255.1), well clear of the agent pool near the
+    bottom of the block. Unlike a pool `Slot`, this link is NAT'd out to
+    the internet by the setup (the orchestrator is trusted infra), so it
+    is deliberately *not* one of the isolated `bbfc*` slots.
+
+    `index` is -1 (sentinel: not a pool index)."""
+    base16 = int(ipaddress.IPv4Address(ip_base())) & 0xFFFF0000
+    host = base16 + 0xFF00
+    return Slot(
+        index=-1,
+        iface=ORCH_IFACE,
+        host_ip=str(ipaddress.IPv4Address(host)),
+        guest_ip=str(ipaddress.IPv4Address(host + 1)),
+    )
+
+
 # --- fail-closed verification ---------------------------------------
 
 def _run_ok(argv: list[str]) -> bool:

nix/firecracker-netpool.nix

@@ -49,10 +49,12 @@ let
   # /31 alignment == an even final octet (only bit 0 matters for base+2i).
   lastOctet = lib.toInt (lib.last (lib.splitString "." cfg.ipBase));
 
-  # The script needs ip/nft/sysctl + the usual coreutils. It gets every
-  # pool value via the unit's Environment=, so it never reads the shared
-  # defaults file (which isn't beside it once copied to the store).
-  runtimePath = with pkgs; [ iproute2 nftables procps coreutils gnused ];
+  # The script needs ip/nft/sysctl + the usual coreutils, plus iptables
+  # for the orchestrator link's DOCKER-USER accept (best-effort; skipped
+  # when Docker is absent). It gets every pool value via the unit's
+  # Environment=, so it never reads the shared defaults file (which isn't
+  # beside it once copied to the store).
+  runtimePath = with pkgs; [ iproute2 nftables iptables procps coreutils gnused ];
 
   ownEnv =
     if cfg.group != null
@@ -64,6 +66,7 @@ let
     BOT_BOTTLE_FC_IP_BASE = cfg.ipBase;
     BOT_BOTTLE_FC_IFACE_PREFIX = cfg.ifacePrefix;
     BOT_BOTTLE_FC_NFT_TABLE = cfg.tableName;
+    BOT_BOTTLE_FC_ORCH_IFACE = cfg.orchIface;
   } // ownEnv;
 in
 {
@@ -127,6 +130,19 @@ in
       description = "nftables table name for the isolation boundary. Must match netpool.NFT_TABLE.";
     };
 
+    orchIface = lib.mkOption {
+      type = lib.types.str;
+      default = defaults.BOT_BOTTLE_FC_ORCH_IFACE;
+      defaultText = lib.literalMD "the shared `netpool.defaults.env` value";
+      description = ''
+        TAP name for the orchestrator/gateway VM's dedicated link. Unlike
+        the isolated bbfc* agent pool, this link is NAT'd to the internet
+        (the orchestrator is trusted infra that builds agent images in-VM
+        and forwards agent egress upstream). Must match
+        BOT_BOTTLE_FC_ORCH_IFACE / netpool.ORCH_IFACE.
+      '';
+    };
+
     writeEnvFile = lib.mkOption {
       type = lib.types.bool;
       default = false;
@@ -176,6 +192,7 @@ in
         BOT_BOTTLE_FC_IP_BASE=${cfg.ipBase}
         BOT_BOTTLE_FC_IFACE_PREFIX=${cfg.ifacePrefix}
         BOT_BOTTLE_FC_NFT_TABLE=${cfg.tableName}
+        BOT_BOTTLE_FC_ORCH_IFACE=${cfg.orchIface}
       '';
     };
   };

scripts/firecracker-netpool.sh

@@ -63,12 +63,13 @@ POOL_SIZE="${BOT_BOTTLE_FC_POOL_SIZE:-$(_default BOT_BOTTLE_FC_POOL_SIZE)}"
 IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-$(_default BOT_BOTTLE_FC_IP_BASE)}"
 PREFIX="${BOT_BOTTLE_FC_IFACE_PREFIX:-$(_default BOT_BOTTLE_FC_IFACE_PREFIX)}"
 TABLE="${BOT_BOTTLE_FC_NFT_TABLE:-$(_default BOT_BOTTLE_FC_NFT_TABLE)}"
+ORCH_IFACE="${BOT_BOTTLE_FC_ORCH_IFACE:-$(_default BOT_BOTTLE_FC_ORCH_IFACE)}"
 OWNER="${BOT_BOTTLE_FC_OWNER:-${SUDO_USER:-$USER}}"
 GROUP="${BOT_BOTTLE_FC_GROUP:-}"
 
 # Fail loudly rather than provisioning a half/empty range if a value
 # resolved to nothing (env unset AND the shared file unreadable).
-for _v in POOL_SIZE IP_BASE PREFIX TABLE; do
+for _v in POOL_SIZE IP_BASE PREFIX TABLE ORCH_IFACE; do
     [ -n "${!_v}" ] || { echo "error: $_v unresolved (set BOT_BOTTLE_FC_* or fix $_DEFAULTS)" >&2; exit 1; }
 done
 
@@ -89,6 +90,13 @@ host_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 )); }
 guest_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 + 1 )); }
 iface()   { echo "${PREFIX}$1"; }
 
+# Orchestrator/gateway VM link: a /31 at the TOP of the IP_BASE /16
+# (host x.y.255.0, guest x.y.255.1), well clear of the agent pool near
+# the bottom of the block. Must match netpool.py:orch_slot().
+_orch_base() { echo $(( ($(_ip_to_int "$IP_BASE") & 0xFFFF0000) + 0xFF00 )); }
+orch_host()  { _int_to_ip "$(_orch_base)"; }
+orch_guest() { _int_to_ip $(( $(_orch_base) + 1 )); }
+
 require_root() {
     if [ "$(id -u)" -ne 0 ]; then
         echo "error: '$1' needs root (run under sudo)" >&2
@@ -125,8 +133,19 @@ cmd_up() {
         echo "  $dev  host=$host  guest=$(guest_ip "$i")  $own_desc"
     done
 
+    # The orchestrator/gateway VM's dedicated link. Same rootless-open
+    # ownership as the pool, but NAT'd to the internet (below) — it is
+    # trusted infra, not an isolated agent slot.
+    ip link show "$ORCH_IFACE" >/dev/null 2>&1 \
+        || ip tuntap add dev "$ORCH_IFACE" mode tap "${own_args[@]}"
+    ip addr replace "$(orch_host)/31" dev "$ORCH_IFACE"
+    ip link set "$ORCH_IFACE" up
+    echo "  $ORCH_IFACE  host=$(orch_host)  guest=$(orch_guest)  (NAT'd egress)  $own_desc"
+
     _install_nft
     echo "nftables table inet $TABLE installed (fail-closed boundary)"
+    _install_orch_egress
+    echo "orchestrator egress installed ($ORCH_IFACE -> NAT out)"
     echo "done."
 }
 
@@ -161,8 +180,63 @@ table inet $TABLE {
 EOF
 }
 
+# Give the orchestrator/gateway VM real internet egress (agent VMs get
+# none — that's the isolation table above). Three parts, because the
+# path must work both during bootstrap (Docker still present) and after
+# Docker is removed:
+#   * masquerade   — SNAT the orch guest /31 out the host uplink so its
+#                    RFC-1918 address can reach the internet.
+#   * nft forward  — accept the orch link's forward path (load-bearing
+#                    on a pure-nft host whose FORWARD policy drops; a
+#                    harmless no-op where forwarding is already open).
+#                    It never drops, so it can't weaken the isolation
+#                    table's agent drops.
+#   * DOCKER-USER  — during bootstrap Docker's FORWARD chain policy is
+#                    DROP; its sanctioned DOCKER-USER hook is the only
+#                    place a user ACCEPT survives. Best-effort + guarded
+#                    (skipped once Docker is gone).
+_install_orch_egress() {
+    nft -f - <<EOF
+table inet ${TABLE}_nat {
+    chain forward {
+        type filter hook forward priority -10; policy accept;
+        iifname "$ORCH_IFACE" accept
+        oifname "$ORCH_IFACE" ct state established,related accept
+    }
+    chain postrouting {
+        type nat hook postrouting priority 100; policy accept;
+        ip saddr $(orch_guest) oifname != "$ORCH_IFACE" masquerade
+    }
+}
+EOF
+    _docker_user_orch add
+}
+
+# Insert (add) or delete (del) the DOCKER-USER ACCEPT rules for the
+# orchestrator link, idempotently, only when the chain exists.
+_docker_user_orch() {
+    local op="$1" flag
+    command -v iptables >/dev/null 2>&1 || return 0
+    iptables -t filter -L DOCKER-USER >/dev/null 2>&1 || return 0
+    for flag in "-i" "-o"; do
+        if [ "$op" = add ]; then
+            iptables -C DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT 2>/dev/null \
+                || iptables -I DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT
+        else
+            iptables -D DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT 2>/dev/null || true
+        fi
+    done
+}
+
 cmd_down() {
     require_root down
+    _docker_user_orch del
+    nft delete table inet "${TABLE}_nat" 2>/dev/null || true
+    if ip link show "$ORCH_IFACE" >/dev/null 2>&1; then
+        ip link set "$ORCH_IFACE" down 2>/dev/null || true
+        ip tuntap del dev "$ORCH_IFACE" mode tap 2>/dev/null || true
+        echo "  removed $ORCH_IFACE"
+    fi
     nft delete table inet "$TABLE" 2>/dev/null || true
     for i in $(seq 0 $((POOL_SIZE-1))); do
         local dev ; dev="$(iface "$i")"
@@ -178,6 +252,8 @@ cmd_down() {
 cmd_status() {
     echo "table inet $TABLE:"
     nft list table inet "$TABLE" 2>/dev/null || echo "  (absent)"
+    echo "table inet ${TABLE}_nat (orchestrator egress):"
+    nft list table inet "${TABLE}_nat" 2>/dev/null || echo "  (absent)"
     echo "taps:"
     for i in $(seq 0 $((POOL_SIZE-1))); do
         local dev ; dev="$(iface "$i")"
@@ -185,6 +261,9 @@ cmd_status() {
             ip -brief addr show "$dev" | sed 's/^/  /'
         fi
     done
+    if ip -brief addr show "$ORCH_IFACE" >/dev/null 2>&1; then
+        ip -brief addr show "$ORCH_IFACE" | sed 's/^/  /'
+    fi
 }
 
 case "${1:-}" in

tests/unit/test_firecracker_helpers.py

@@ -64,6 +64,21 @@ class TestNetpoolProbes(unittest.TestCase):
              patch.object(netpool, "tap_present", side_effect=[True, False]):
             self.assertEqual(["bbfc1"], netpool.missing_taps())
 
+    def test_orch_slot_is_top_of_ip_base_16(self):
+        # Dedicated orchestrator link: /31 at the top of the IP_BASE /16,
+        # clear of the pool (bottom of the block). Must match the shell
+        # script's orch_host()/orch_guest() and be a non-pool index.
+        with patch.dict("os.environ", {"BOT_BOTTLE_FC_IP_BASE": "10.243.0.0"}):
+            s = netpool.orch_slot()
+        self.assertEqual(netpool.ORCH_IFACE, s.iface)
+        self.assertEqual("10.243.255.0", s.host_ip)
+        self.assertEqual("10.243.255.1", s.guest_ip)
+        self.assertEqual(-1, s.index)
+        # Never collides with a pool slot's guest address.
+        with patch.dict("os.environ", {"BOT_BOTTLE_FC_IP_BASE": "10.243.0.0"}):
+            pool_guests = {sl.guest_ip for sl in netpool.all_slots()}
+        self.assertNotIn(s.guest_ip, pool_guests)
+
 
 class TestConsoleTail(unittest.TestCase):
     def test_reads_tail(self):