feat(firecracker): NAT'd egress link for the orchestrator/builder VM
The orchestrator/gateway VM is trusted infra, not an isolated agent: it builds agent images in-VM (buildah must FROM-pull + apt/npm) and, in the Stage B cutover, forwards agent egress upstream. Give it a dedicated TAP (`bborch0`) on a /31 at the top of the IP_BASE /16 (clear of the bbfc* agent pool at the bottom), NAT'd out the host uplink — while agent VMs keep their fail-closed, gateway-only isolation table. - netpool.defaults.env / netpool.py: new BOT_BOTTLE_FC_ORCH_IFACE + `orch_slot()` (index -1 sentinel; host x.y.255.0 / guest x.y.255.1). - scripts/firecracker-netpool.sh: create + address the orchestrator TAP; `bot_bottle_fc_nat` table masquerades its /31 out the uplink and accepts its forward path. Because bootstrap still runs Docker (whose FORWARD policy is DROP), a best-effort, guarded, idempotent DOCKER-USER ACCEPT is added too (skipped once Docker is gone). down/status updated. - nix/firecracker-netpool.nix: mirror the option, pass it via the unit Environment= (the store-copied script can't read the defaults file), and add iptables to the unit path for the DOCKER-USER step. Agent isolation is unchanged: the new rules only ever accept/masquerade the orchestrator link and never drop, so they can't weaken the bbfc* drops. Applied by re-running `sudo ./scripts/firecracker-netpool.sh up`. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck
This commit is contained in:
@@ -63,12 +63,13 @@ POOL_SIZE="${BOT_BOTTLE_FC_POOL_SIZE:-$(_default BOT_BOTTLE_FC_POOL_SIZE)}"
|
||||
IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-$(_default BOT_BOTTLE_FC_IP_BASE)}"
|
||||
PREFIX="${BOT_BOTTLE_FC_IFACE_PREFIX:-$(_default BOT_BOTTLE_FC_IFACE_PREFIX)}"
|
||||
TABLE="${BOT_BOTTLE_FC_NFT_TABLE:-$(_default BOT_BOTTLE_FC_NFT_TABLE)}"
|
||||
ORCH_IFACE="${BOT_BOTTLE_FC_ORCH_IFACE:-$(_default BOT_BOTTLE_FC_ORCH_IFACE)}"
|
||||
OWNER="${BOT_BOTTLE_FC_OWNER:-${SUDO_USER:-$USER}}"
|
||||
GROUP="${BOT_BOTTLE_FC_GROUP:-}"
|
||||
|
||||
# Fail loudly rather than provisioning a half/empty range if a value
|
||||
# resolved to nothing (env unset AND the shared file unreadable).
|
||||
for _v in POOL_SIZE IP_BASE PREFIX TABLE; do
|
||||
for _v in POOL_SIZE IP_BASE PREFIX TABLE ORCH_IFACE; do
|
||||
[ -n "${!_v}" ] || { echo "error: $_v unresolved (set BOT_BOTTLE_FC_* or fix $_DEFAULTS)" >&2; exit 1; }
|
||||
done
|
||||
|
||||
@@ -89,6 +90,13 @@ host_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 )); }
|
||||
guest_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 + 1 )); }
|
||||
iface() { echo "${PREFIX}$1"; }
|
||||
|
||||
# Orchestrator/gateway VM link: a /31 at the TOP of the IP_BASE /16
|
||||
# (host x.y.255.0, guest x.y.255.1), well clear of the agent pool near
|
||||
# the bottom of the block. Must match netpool.py:orch_slot().
|
||||
_orch_base() { echo $(( ($(_ip_to_int "$IP_BASE") & 0xFFFF0000) + 0xFF00 )); }
|
||||
orch_host() { _int_to_ip "$(_orch_base)"; }
|
||||
orch_guest() { _int_to_ip $(( $(_orch_base) + 1 )); }
|
||||
|
||||
require_root() {
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "error: '$1' needs root (run under sudo)" >&2
|
||||
@@ -125,8 +133,19 @@ cmd_up() {
|
||||
echo " $dev host=$host guest=$(guest_ip "$i") $own_desc"
|
||||
done
|
||||
|
||||
# The orchestrator/gateway VM's dedicated link. Same rootless-open
|
||||
# ownership as the pool, but NAT'd to the internet (below) — it is
|
||||
# trusted infra, not an isolated agent slot.
|
||||
ip link show "$ORCH_IFACE" >/dev/null 2>&1 \
|
||||
|| ip tuntap add dev "$ORCH_IFACE" mode tap "${own_args[@]}"
|
||||
ip addr replace "$(orch_host)/31" dev "$ORCH_IFACE"
|
||||
ip link set "$ORCH_IFACE" up
|
||||
echo " $ORCH_IFACE host=$(orch_host) guest=$(orch_guest) (NAT'd egress) $own_desc"
|
||||
|
||||
_install_nft
|
||||
echo "nftables table inet $TABLE installed (fail-closed boundary)"
|
||||
_install_orch_egress
|
||||
echo "orchestrator egress installed ($ORCH_IFACE -> NAT out)"
|
||||
echo "done."
|
||||
}
|
||||
|
||||
@@ -161,8 +180,63 @@ table inet $TABLE {
|
||||
EOF
|
||||
}
|
||||
|
||||
# Give the orchestrator/gateway VM real internet egress (agent VMs get
|
||||
# none — that's the isolation table above). Three parts, because the
|
||||
# path must work both during bootstrap (Docker still present) and after
|
||||
# Docker is removed:
|
||||
# * masquerade — SNAT the orch guest /31 out the host uplink so its
|
||||
# RFC-1918 address can reach the internet.
|
||||
# * nft forward — accept the orch link's forward path (load-bearing
|
||||
# on a pure-nft host whose FORWARD policy drops; a
|
||||
# harmless no-op where forwarding is already open).
|
||||
# It never drops, so it can't weaken the isolation
|
||||
# table's agent drops.
|
||||
# * DOCKER-USER — during bootstrap Docker's FORWARD chain policy is
|
||||
# DROP; its sanctioned DOCKER-USER hook is the only
|
||||
# place a user ACCEPT survives. Best-effort + guarded
|
||||
# (skipped once Docker is gone).
|
||||
_install_orch_egress() {
|
||||
nft -f - <<EOF
|
||||
table inet ${TABLE}_nat {
|
||||
chain forward {
|
||||
type filter hook forward priority -10; policy accept;
|
||||
iifname "$ORCH_IFACE" accept
|
||||
oifname "$ORCH_IFACE" ct state established,related accept
|
||||
}
|
||||
chain postrouting {
|
||||
type nat hook postrouting priority 100; policy accept;
|
||||
ip saddr $(orch_guest) oifname != "$ORCH_IFACE" masquerade
|
||||
}
|
||||
}
|
||||
EOF
|
||||
_docker_user_orch add
|
||||
}
|
||||
|
||||
# Insert (add) or delete (del) the DOCKER-USER ACCEPT rules for the
|
||||
# orchestrator link, idempotently, only when the chain exists.
|
||||
_docker_user_orch() {
|
||||
local op="$1" flag
|
||||
command -v iptables >/dev/null 2>&1 || return 0
|
||||
iptables -t filter -L DOCKER-USER >/dev/null 2>&1 || return 0
|
||||
for flag in "-i" "-o"; do
|
||||
if [ "$op" = add ]; then
|
||||
iptables -C DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT 2>/dev/null \
|
||||
|| iptables -I DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT
|
||||
else
|
||||
iptables -D DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT 2>/dev/null || true
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
cmd_down() {
|
||||
require_root down
|
||||
_docker_user_orch del
|
||||
nft delete table inet "${TABLE}_nat" 2>/dev/null || true
|
||||
if ip link show "$ORCH_IFACE" >/dev/null 2>&1; then
|
||||
ip link set "$ORCH_IFACE" down 2>/dev/null || true
|
||||
ip tuntap del dev "$ORCH_IFACE" mode tap 2>/dev/null || true
|
||||
echo " removed $ORCH_IFACE"
|
||||
fi
|
||||
nft delete table inet "$TABLE" 2>/dev/null || true
|
||||
for i in $(seq 0 $((POOL_SIZE-1))); do
|
||||
local dev ; dev="$(iface "$i")"
|
||||
@@ -178,6 +252,8 @@ cmd_down() {
|
||||
cmd_status() {
|
||||
echo "table inet $TABLE:"
|
||||
nft list table inet "$TABLE" 2>/dev/null || echo " (absent)"
|
||||
echo "table inet ${TABLE}_nat (orchestrator egress):"
|
||||
nft list table inet "${TABLE}_nat" 2>/dev/null || echo " (absent)"
|
||||
echo "taps:"
|
||||
for i in $(seq 0 $((POOL_SIZE-1))); do
|
||||
local dev ; dev="$(iface "$i")"
|
||||
@@ -185,6 +261,9 @@ cmd_status() {
|
||||
ip -brief addr show "$dev" | sed 's/^/ /'
|
||||
fi
|
||||
done
|
||||
if ip -brief addr show "$ORCH_IFACE" >/dev/null 2>&1; then
|
||||
ip -brief addr show "$ORCH_IFACE" | sed 's/^/ /'
|
||||
fi
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
|
||||
Reference in New Issue
Block a user