feat(firecracker): move pool off CGNAT, add overlap guard + flake module
The default TAP-pool base was 100.64.0.0/10 (RFC-6598 CGNAT) — chosen to dodge RFC-1918, but that's exactly the range Tailscale assigns node addresses from, so on a Tailscale host it's the worst pick. Move the default to 10.243.0.0/16, an obscure RFC-1918 block that steers clear of docker/libvirt/k8s/LAN and Tailscale. No default is collision-proof, so add netpool.overlapping_routes(): it parses `ip -json route show table all` and flags any route intersecting the pool range (excluding our own bbfc* TAPs and the default route). The launch preflight warns on overlap; `backend status` reports it. Distribute the NixOS host setup as a flake module instead of a copy-pasted blob: nix/firecracker-netpool.nix computes the taps / nft table from typed options (poolSize, ipBase, ifacePrefix, owner) with a /31-alignment assertion, and flake.nix exposes it as nixosModules.firecracker-netpool. Defaults mirror the backend constants; writeEnvFile emits the matching BOT_BOTTLE_FC_* so the host pool and the launcher can't drift. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck
This commit is contained in:
@@ -17,8 +17,16 @@ Topology (per slot i):
|
||||
* isolation via ``table inet bot_bottle_fc``: a VM reaches only its
|
||||
own sidecar (DNAT'd from the host TAP IP) and nothing else.
|
||||
|
||||
The default IP block is RFC-6598 shared address space (100.64.0.0/10),
|
||||
which is purpose-built to avoid colliding with LAN/VPN private ranges.
|
||||
The default IP block is ``10.243.0.0/16`` — an intentionally obscure
|
||||
corner of RFC-1918 private space. RFC-1918 is the range *designated*
|
||||
for private links; we pick a high, unusual /16 to steer clear of the
|
||||
common occupants (Docker's 172.17-31, libvirt's 192.168.122, k8s
|
||||
10.42/10.244, and typical home LANs on 192.168.0/1.x or 10.0.0.x).
|
||||
We deliberately avoid ``100.64.0.0/10`` (RFC-6598 CGNAT) because
|
||||
Tailscale hands out node addresses from exactly that range. No default
|
||||
is collision-proof, so `overlapping_routes()` is the real guard —
|
||||
`backend setup`/`status` and the launch preflight call it to catch
|
||||
a base that clashes with something already on the host.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -45,7 +53,7 @@ def pool_size() -> int:
|
||||
|
||||
|
||||
def ip_base() -> str:
|
||||
return os.environ.get("BOT_BOTTLE_FC_IP_BASE", "100.64.0.0")
|
||||
return os.environ.get("BOT_BOTTLE_FC_IP_BASE", "10.243.0.0")
|
||||
|
||||
|
||||
# Sidecar ports the VM reaches at its host-side TAP IP. Kept in sync
|
||||
@@ -119,6 +127,73 @@ def missing_taps() -> list[str]:
|
||||
return [s.iface for s in all_slots() if not tap_present(s.iface)]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RouteConflict:
|
||||
"""An existing host route whose destination overlaps the pool range
|
||||
but isn't one of our own bbfc TAPs — i.e. the chosen IP base clashes
|
||||
with something already on the host (a Tailscale CGNAT peer, a
|
||||
docker/libvirt bridge, the LAN…)."""
|
||||
|
||||
dst: str
|
||||
dev: str
|
||||
|
||||
|
||||
def _pool_span() -> tuple[int, int]:
|
||||
"""Inclusive [first, last] integer bounds of every address the pool
|
||||
occupies: base .. base + 2*pool_size - 1."""
|
||||
base = int(ipaddress.IPv4Address(ip_base()))
|
||||
return base, base + 2 * pool_size() - 1
|
||||
|
||||
|
||||
def overlapping_routes() -> list[RouteConflict]:
|
||||
"""Existing routes whose destination overlaps the pool's address
|
||||
range, excluding our own `bbfc*` TAP routes and the default route.
|
||||
|
||||
A non-empty result means `BOT_BOTTLE_FC_IP_BASE` collides with
|
||||
something already configured on this host, so the pool would shadow
|
||||
(or be shadowed by) it. Empty on success — or when `ip` is missing
|
||||
or unparseable, since this is an advisory guard, not the fail-closed
|
||||
isolation check (that's the nft table + post-boot probe)."""
|
||||
import json
|
||||
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["ip", "-json", "route", "show", "table", "all"],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
if proc.returncode != 0 or not proc.stdout.strip():
|
||||
return []
|
||||
try:
|
||||
routes = json.loads(proc.stdout)
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
|
||||
lo, hi = _pool_span()
|
||||
conflicts: list[RouteConflict] = []
|
||||
for r in routes:
|
||||
if not isinstance(r, dict):
|
||||
continue
|
||||
dst = r.get("dst")
|
||||
dev = r.get("dev", "")
|
||||
if not isinstance(dst, str) or dst in ("", "default"):
|
||||
continue
|
||||
if isinstance(dev, str) and dev.startswith(IFACE_PREFIX):
|
||||
continue # our own pool link
|
||||
try:
|
||||
net = ipaddress.ip_network(dst, strict=False)
|
||||
except ValueError:
|
||||
continue
|
||||
if net.version != 4:
|
||||
continue
|
||||
r_lo = int(net.network_address)
|
||||
r_hi = int(net.broadcast_address)
|
||||
if r_lo <= hi and lo <= r_hi: # ranges intersect
|
||||
conflicts.append(RouteConflict(dst=dst, dev=str(dev)))
|
||||
return conflicts
|
||||
|
||||
|
||||
# --- allocation ------------------------------------------------------
|
||||
|
||||
def _lock_dir() -> Path:
|
||||
@@ -149,11 +224,11 @@ def allocate(slug: str) -> tuple[Slot, IO[str]]:
|
||||
return s, handle
|
||||
die(f"Firecracker TAP pool exhausted ({pool_size()} slots, all in "
|
||||
f"use). Stop a running bottle or raise BOT_BOTTLE_FC_POOL_SIZE "
|
||||
f"and re-run `./cli.py firecracker setup`.")
|
||||
f"and re-run `./cli.py backend setup --backend=firecracker`.")
|
||||
raise AssertionError("unreachable")
|
||||
|
||||
|
||||
# --- config renderers (shown by `./cli.py firecracker setup`) --------
|
||||
# --- config renderers (shown by `./cli.py backend setup`) -----------
|
||||
|
||||
def render_shell_setup() -> str:
|
||||
"""The imperative command for non-NixOS hosts."""
|
||||
@@ -190,7 +265,7 @@ def render_nixos_module() -> str:
|
||||
)
|
||||
|
||||
return f"""# bot-bottle Firecracker network pool ({len(slots)} slots, base {ip_base()}).
|
||||
# Generated by `./cli.py firecracker setup`; owner user = {owner!r}.
|
||||
# Generated by `./cli.py backend setup --backend=firecracker`; owner user = {owner!r}.
|
||||
{{ ... }}:
|
||||
{{
|
||||
boot.kernel.sysctl."net.ipv4.ip_forward" = 1;
|
||||
|
||||
Reference in New Issue
Block a user