From ce3fad9320896ed38d2f5615ab7f8eb7c80963a0 Mon Sep 17 00:00:00 2001 From: didericis Date: Sat, 11 Jul 2026 15:18:21 -0400 Subject: [PATCH] feat(firecracker): move pool off CGNAT, add overlap guard + flake module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default TAP-pool base was 100.64.0.0/10 (RFC-6598 CGNAT) — chosen to dodge RFC-1918, but that's exactly the range Tailscale assigns node addresses from, so on a Tailscale host it's the worst pick. Move the default to 10.243.0.0/16, an obscure RFC-1918 block that steers clear of docker/libvirt/k8s/LAN and Tailscale. No default is collision-proof, so add netpool.overlapping_routes(): it parses `ip -json route show table all` and flags any route intersecting the pool range (excluding our own bbfc* TAPs and the default route). The launch preflight warns on overlap; `backend status` reports it. Distribute the NixOS host setup as a flake module instead of a copy-pasted blob: nix/firecracker-netpool.nix computes the taps / nft table from typed options (poolSize, ipBase, ifacePrefix, owner) with a /31-alignment assertion, and flake.nix exposes it as nixosModules.firecracker-netpool. Defaults mirror the backend constants; writeEnvFile emits the matching BOT_BOTTLE_FC_* so the host pool and the launcher can't drift. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck --- README.md | 4 +- bot_bottle/backend/firecracker/netpool.py | 87 ++++++++++++- bot_bottle/backend/firecracker/util.py | 14 +- flake.nix | 18 +++ nix/firecracker-netpool.nix | 152 ++++++++++++++++++++++ scripts/firecracker-netpool.sh | 13 +- tests/unit/test_firecracker_backend.py | 53 +++++++- 7 files changed, 324 insertions(+), 17 deletions(-) create mode 100644 flake.nix create mode 100644 nix/firecracker-netpool.nix diff --git a/README.md b/README.md index 7cad616..c117e1a 100644 --- a/README.md +++ b/README.md @@ -82,13 +82,13 @@ On Linux, a KVM-capable host defaults to the Firecracker backend. It needs: - **`/dev/kvm`** present and accessible. Load `kvm-intel` or `kvm-amd` (and enable virtualization in BIOS/firmware). The invoking user must be in the `kvm` group: `sudo usermod -aG kvm "$USER"` then re-login. bot-bottle preflights this and reports exactly what's missing. - **`firecracker`** on `PATH`: grab a release from . Start flows print this pointer when the binary is missing. - **Docker** for the sidecar bundle and image build. -- **A one-time privileged network setup** — the per-bottle TAP pool plus the fail-closed `nftables` isolation table. Run `./cli.py firecracker setup` for the host-appropriate command (a declarative module on NixOS, a `sudo` script elsewhere); `./cli.py firecracker status` reports what's present. +- **A one-time privileged network setup** — the per-bottle TAP pool plus the fail-closed `nftables` isolation table. Run `./cli.py backend setup --backend=firecracker` for the host-appropriate config (a NixOS module, a `sudo` script elsewhere); `./cli.py backend status --backend=firecracker` reports what's present, including whether the pool range collides with an existing route. The pool defaults to `10.243.0.0/16` (an obscure RFC-1918 block that dodges docker/libvirt/LAN and, deliberately, Tailscale's `100.64.0.0/10` CGNAT range); override with `BOT_BOTTLE_FC_IP_BASE` if it clashes on your host. ```sh BOT_BOTTLE_BACKEND=firecracker ./cli.py start ``` -> **NixOS:** enable `virtualisation.docker`, ensure the KVM module is loaded (`boot.kernelModules = [ "kvm-intel" ];` or `kvm-amd`), and add your user to the `kvm` and `docker` groups. Apply the network-pool module from `./cli.py firecracker setup` and `nixos-rebuild switch` (imperative nft/TAP rules don't survive a rebuild). `firecracker` isn't in nixpkgs by default as a user binary — install the release binary (pin the version) and put it on `PATH`. +> **NixOS:** enable `virtualisation.docker`, ensure the KVM module is loaded (`boot.kernelModules = [ "kvm-intel" ];` or `kvm-amd`), and add your user to the `kvm` and `docker` groups. For the network pool, consume the flake module — `imports = [ inputs.bot-bottle.nixosModules.firecracker-netpool ]; services.bot-bottle-firecracker = { enable = true; owner = "you"; };` — then `nixos-rebuild switch` (imperative nft/TAP rules don't survive a rebuild; channel users can `imports = [ /nix/firecracker-netpool.nix ]`). `firecracker` isn't in nixpkgs by default as a user binary — install the release binary (pin the version) and put it on `PATH`. ```sh ./cli.py start # builds the image on first run, drops you into claude diff --git a/bot_bottle/backend/firecracker/netpool.py b/bot_bottle/backend/firecracker/netpool.py index 2d02417..5c9078d 100644 --- a/bot_bottle/backend/firecracker/netpool.py +++ b/bot_bottle/backend/firecracker/netpool.py @@ -17,8 +17,16 @@ Topology (per slot i): * isolation via ``table inet bot_bottle_fc``: a VM reaches only its own sidecar (DNAT'd from the host TAP IP) and nothing else. -The default IP block is RFC-6598 shared address space (100.64.0.0/10), -which is purpose-built to avoid colliding with LAN/VPN private ranges. +The default IP block is ``10.243.0.0/16`` — an intentionally obscure +corner of RFC-1918 private space. RFC-1918 is the range *designated* +for private links; we pick a high, unusual /16 to steer clear of the +common occupants (Docker's 172.17-31, libvirt's 192.168.122, k8s +10.42/10.244, and typical home LANs on 192.168.0/1.x or 10.0.0.x). +We deliberately avoid ``100.64.0.0/10`` (RFC-6598 CGNAT) because +Tailscale hands out node addresses from exactly that range. No default +is collision-proof, so `overlapping_routes()` is the real guard — +`backend setup`/`status` and the launch preflight call it to catch +a base that clashes with something already on the host. """ from __future__ import annotations @@ -45,7 +53,7 @@ def pool_size() -> int: def ip_base() -> str: - return os.environ.get("BOT_BOTTLE_FC_IP_BASE", "100.64.0.0") + return os.environ.get("BOT_BOTTLE_FC_IP_BASE", "10.243.0.0") # Sidecar ports the VM reaches at its host-side TAP IP. Kept in sync @@ -119,6 +127,73 @@ def missing_taps() -> list[str]: return [s.iface for s in all_slots() if not tap_present(s.iface)] +@dataclass(frozen=True) +class RouteConflict: + """An existing host route whose destination overlaps the pool range + but isn't one of our own bbfc TAPs — i.e. the chosen IP base clashes + with something already on the host (a Tailscale CGNAT peer, a + docker/libvirt bridge, the LAN…).""" + + dst: str + dev: str + + +def _pool_span() -> tuple[int, int]: + """Inclusive [first, last] integer bounds of every address the pool + occupies: base .. base + 2*pool_size - 1.""" + base = int(ipaddress.IPv4Address(ip_base())) + return base, base + 2 * pool_size() - 1 + + +def overlapping_routes() -> list[RouteConflict]: + """Existing routes whose destination overlaps the pool's address + range, excluding our own `bbfc*` TAP routes and the default route. + + A non-empty result means `BOT_BOTTLE_FC_IP_BASE` collides with + something already configured on this host, so the pool would shadow + (or be shadowed by) it. Empty on success — or when `ip` is missing + or unparseable, since this is an advisory guard, not the fail-closed + isolation check (that's the nft table + post-boot probe).""" + import json + + try: + proc = subprocess.run( + ["ip", "-json", "route", "show", "table", "all"], + capture_output=True, text=True, check=False, + ) + except FileNotFoundError: + return [] + if proc.returncode != 0 or not proc.stdout.strip(): + return [] + try: + routes = json.loads(proc.stdout) + except json.JSONDecodeError: + return [] + + lo, hi = _pool_span() + conflicts: list[RouteConflict] = [] + for r in routes: + if not isinstance(r, dict): + continue + dst = r.get("dst") + dev = r.get("dev", "") + if not isinstance(dst, str) or dst in ("", "default"): + continue + if isinstance(dev, str) and dev.startswith(IFACE_PREFIX): + continue # our own pool link + try: + net = ipaddress.ip_network(dst, strict=False) + except ValueError: + continue + if net.version != 4: + continue + r_lo = int(net.network_address) + r_hi = int(net.broadcast_address) + if r_lo <= hi and lo <= r_hi: # ranges intersect + conflicts.append(RouteConflict(dst=dst, dev=str(dev))) + return conflicts + + # --- allocation ------------------------------------------------------ def _lock_dir() -> Path: @@ -149,11 +224,11 @@ def allocate(slug: str) -> tuple[Slot, IO[str]]: return s, handle die(f"Firecracker TAP pool exhausted ({pool_size()} slots, all in " f"use). Stop a running bottle or raise BOT_BOTTLE_FC_POOL_SIZE " - f"and re-run `./cli.py firecracker setup`.") + f"and re-run `./cli.py backend setup --backend=firecracker`.") raise AssertionError("unreachable") -# --- config renderers (shown by `./cli.py firecracker setup`) -------- +# --- config renderers (shown by `./cli.py backend setup`) ----------- def render_shell_setup() -> str: """The imperative command for non-NixOS hosts.""" @@ -190,7 +265,7 @@ def render_nixos_module() -> str: ) return f"""# bot-bottle Firecracker network pool ({len(slots)} slots, base {ip_base()}). -# Generated by `./cli.py firecracker setup`; owner user = {owner!r}. +# Generated by `./cli.py backend setup --backend=firecracker`; owner user = {owner!r}. {{ ... }}: {{ boot.kernel.sysctl."net.ipv4.ip_forward" = 1; diff --git a/bot_bottle/backend/firecracker/util.py b/bot_bottle/backend/firecracker/util.py index 4e251fa..15b0bb5 100644 --- a/bot_bottle/backend/firecracker/util.py +++ b/bot_bottle/backend/firecracker/util.py @@ -9,7 +9,7 @@ generation. The privileged network setup (TAP pool + nft table) is a one-time operator step — see `netpool.py`, `scripts/firecracker-netpool.sh`, -and `./cli.py firecracker setup`. +and `./cli.py backend setup --backend=firecracker`. """ from __future__ import annotations @@ -124,17 +124,25 @@ def _require_network_pool() -> None: empirical isolation probe run after boot, before the agent starts (see isolation_probe.verify_isolation). What we must never do is boot without the TAP pool.""" + conflicts = netpool.overlapping_routes() + if conflicts: + detail = "; ".join(f"{c.dst} dev {c.dev}" for c in conflicts) + warn(f"Firecracker pool range ({netpool.ip_base()}, " + f"{netpool.pool_size()} slots) overlaps existing routes: " + f"{detail}. This can shadow or be shadowed by that route; " + f"set BOT_BOTTLE_FC_IP_BASE to a free range and re-run " + f"./cli.py backend setup --backend=firecracker.") missing = netpool.missing_taps() if missing: die(f"network pool incomplete — missing TAP devices: " - f"{', '.join(missing)}.\n ./cli.py firecracker setup") + f"{', '.join(missing)}.\n ./cli.py backend setup --backend=firecracker") if shutil.which("nft") is not None and not netpool.nft_table_present(): # nft is queryable and says the table is absent — that's a # definite, catchable misconfiguration; fail early. warn(f"isolation table `inet {netpool.NFT_TABLE}` not found via nft. " "If this is a permissions issue it will be re-checked " "empirically after boot; otherwise run: " - "./cli.py firecracker setup") + "./cli.py backend setup --backend=firecracker") # --- rootfs pipeline (rootless) ------------------------------------- diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..8637d7b --- /dev/null +++ b/flake.nix @@ -0,0 +1,18 @@ +{ + description = "bot-bottle — sandboxed runtime for AI coding agents"; + + outputs = { self, ... }: { + # Declarative host setup for the Firecracker backend's network pool. + # Consume from a flake-based NixOS config: + # + # inputs.bot-bottle.url = "git+ssh://"; # or path:/… + # # then, in your host module: + # imports = [ inputs.bot-bottle.nixosModules.firecracker-netpool ]; + # services.bot-bottle-firecracker = { enable = true; owner = "you"; }; + # + # The module is plain (no nixpkgs pin), so channel users can import + # ./nix/firecracker-netpool.nix directly without the flake. + nixosModules.firecracker-netpool = import ./nix/firecracker-netpool.nix; + nixosModules.default = self.nixosModules.firecracker-netpool; + }; +} diff --git a/nix/firecracker-netpool.nix b/nix/firecracker-netpool.nix new file mode 100644 index 0000000..8d5f6b3 --- /dev/null +++ b/nix/firecracker-netpool.nix @@ -0,0 +1,152 @@ +# bot-bottle Firecracker network pool — declarative NixOS module. +# +# The one-time privileged setup the Firecracker backend needs: a pool of +# user-owned point-to-point TAP devices plus a fail-closed nftables table +# that confines every microVM to its own sidecar. Enabling this module is +# the NixOS equivalent of `sudo ./scripts/firecracker-netpool.sh up` — but +# declarative, so the taps/table survive `nixos-rebuild` and are only +# reconfigured when this config changes (running VMs aren't dropped). +# +# The option defaults MUST match the backend constants in +# bot_bottle/backend/firecracker/netpool.py (pool size, IP base, iface +# prefix, table name). The backend reads the same values at launch from +# the BOT_BOTTLE_FC_* env vars; if you override an option here, override +# the matching env var for the CLI too (or set writeEnvFile = true below +# to have this module emit them). Keep the two sides in lockstep. +{ config, lib, ... }: + +let + cfg = config.services.bot-bottle-firecracker; + + # --- IPv4 <-> int (full 32-bit carry math) ------------------------- + toOctets = s: map lib.toInt (lib.splitString "." s); + ipToInt = s: + let o = toOctets s; in + (lib.elemAt o 0) * 16777216 + + (lib.elemAt o 1) * 65536 + + (lib.elemAt o 2) * 256 + + (lib.elemAt o 3); + intToIp = n: + let + b0 = n / 16777216; r0 = n - b0 * 16777216; + b1 = r0 / 65536; r1 = r0 - b1 * 65536; + b2 = r1 / 256; + b3 = r1 - b2 * 256; + in "${toString b0}.${toString b1}.${toString b2}.${toString b3}"; + + baseInt = ipToInt cfg.ipBase; + + # Slot i: host = base + 2i (the VM's gateway), on its own /31. + slots = lib.genList (i: { + iface = "${cfg.ifacePrefix}${toString i}"; + hostIp = intToIp (baseInt + 2 * i); + }) cfg.poolSize; + + mkAttr = f: lib.listToAttrs (map (s: lib.nameValuePair "10-${s.iface}" (f s)) slots); +in +{ + options.services.bot-bottle-firecracker = { + enable = lib.mkEnableOption "the bot-bottle Firecracker network pool (TAP devices + isolation nftables table)"; + + poolSize = lib.mkOption { + type = lib.types.ints.positive; + default = 8; + description = "Number of pool slots (concurrent bottles). Must match BOT_BOTTLE_FC_POOL_SIZE."; + }; + + ipBase = lib.mkOption { + type = lib.types.str; + default = "10.243.0.0"; + description = '' + Base IPv4 of the /31 pool; slot i uses host = base + 2i. Must + be /31-aligned (even final address) and must match + BOT_BOTTLE_FC_IP_BASE. Default is an obscure RFC-1918 /16 that + dodges docker/libvirt/k8s/LAN and, deliberately, Tailscale's + 100.64.0.0/10 CGNAT range. + ''; + }; + + ifacePrefix = lib.mkOption { + type = lib.types.str; + default = "bbfc"; + description = "TAP interface name prefix. Must match BOT_BOTTLE_FC_IFACE_PREFIX."; + }; + + owner = lib.mkOption { + type = lib.types.str; + example = "alice"; + description = "User that owns the TAP devices, so `./cli.py start` opens them without root."; + }; + + tableName = lib.mkOption { + type = lib.types.str; + default = "bot_bottle_fc"; + description = "nftables table name for the isolation boundary. Must match netpool.NFT_TABLE."; + }; + + writeEnvFile = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + When true, write /etc/bot-bottle/firecracker.env with the + matching BOT_BOTTLE_FC_* values, so the host pool and the CLI + launcher can't drift. Source it before running `./cli.py`. + ''; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = lib.mod baseInt 2 == 0; + message = "services.bot-bottle-firecracker.ipBase must be /31-aligned (even final octet); got ${cfg.ipBase}."; + } + ]; + + # VM->sidecar traffic is DNAT'd and forwarded, so forwarding must be on. + boot.kernel.sysctl."net.ipv4.ip_forward" = 1; + + systemd.network.enable = true; + systemd.network.netdevs = mkAttr (s: { + netdevConfig = { Name = s.iface; Kind = "tap"; }; + tapConfig = { User = cfg.owner; }; + }); + systemd.network.networks = mkAttr (s: { + matchConfig.Name = s.iface; + address = [ "${s.hostIp}/31" ]; + networkConfig.ConfigureWithoutCarrier = true; + }); + + # Independent, fail-closed table — coexists with Docker/ufw/firewalld + # rather than replacing the ruleset. A VM (traffic from a + # ${cfg.ifacePrefix}* TAP) may reach only its own sidecar (DNAT'd from + # the host TAP IP); everything else is dropped. + networking.nftables.enable = true; + networking.nftables.tables.${cfg.tableName} = { + family = "inet"; + content = '' + chain forward { + type filter hook forward priority -10; policy accept; + iifname != "${cfg.ifacePrefix}*" return + ct state established,related accept + ct status dnat accept + drop + } + chain input { + type filter hook input priority -10; policy accept; + iifname != "${cfg.ifacePrefix}*" return + ct state established,related accept + drop + } + ''; + }; + + environment.etc."bot-bottle/firecracker.env" = lib.mkIf cfg.writeEnvFile { + text = '' + BOT_BOTTLE_FC_POOL_SIZE=${toString cfg.poolSize} + BOT_BOTTLE_FC_IP_BASE=${cfg.ipBase} + BOT_BOTTLE_FC_IFACE_PREFIX=${cfg.ifacePrefix} + ''; + }; + }; +} diff --git a/scripts/firecracker-netpool.sh b/scripts/firecracker-netpool.sh index 5bd394e..14681c9 100755 --- a/scripts/firecracker-netpool.sh +++ b/scripts/firecracker-netpool.sh @@ -19,11 +19,14 @@ # virbr0 (libvirt), cni0 (k8s) or br-* (docker networks). # * Own nftables table `bot_bottle_fc` — independent of the iptables # filter/nat tables Docker/ufw/firewalld use, so nothing is stomped. -# * Default IP block is RFC-6598 shared address space (100.64.0.0/10), -# purpose-built to not collide with LAN/VPN private ranges. +# * Default IP block is an obscure RFC-1918 /16 (10.243.0.0/16), +# chosen to dodge the usual occupants (docker 172.17-31, libvirt +# 192.168.122, k8s 10.42/10.244, home LANs). NOT 100.64.0.0/10 — +# that's RFC-6598 CGNAT, which Tailscale hands node addresses from. # # NixOS: imperative rules here do NOT survive nixos-rebuild. Use the -# declarative module in docs/firecracker/nixos-netpool.nix instead. +# declarative module in nix/firecracker-netpool.nix instead (exposed as +# the flake output nixosModules.firecracker-netpool). # # Usage: # sudo ./scripts/firecracker-netpool.sh up @@ -32,14 +35,14 @@ # # Env overrides (must match the backend's util.py constants): # BOT_BOTTLE_FC_POOL_SIZE number of slots (default 8) -# BOT_BOTTLE_FC_IP_BASE base IPv4 of the /31 pool (default 100.64.0.0) +# BOT_BOTTLE_FC_IP_BASE base IPv4 of the /31 pool (default 10.243.0.0) # BOT_BOTTLE_FC_IFACE_PREFIX TAP name prefix (default bbfc) # BOT_BOTTLE_FC_OWNER owning user (default $SUDO_USER or $USER) set -euo pipefail POOL_SIZE="${BOT_BOTTLE_FC_POOL_SIZE:-8}" -IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-100.64.0.0}" +IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-10.243.0.0}" PREFIX="${BOT_BOTTLE_FC_IFACE_PREFIX:-bbfc}" OWNER="${BOT_BOTTLE_FC_OWNER:-${SUDO_USER:-$USER}}" TABLE="bot_bottle_fc" diff --git a/tests/unit/test_firecracker_backend.py b/tests/unit/test_firecracker_backend.py index 00321dc..79e6954 100644 --- a/tests/unit/test_firecracker_backend.py +++ b/tests/unit/test_firecracker_backend.py @@ -38,7 +38,15 @@ class TestNetpoolSlots(unittest.TestCase): (s1.iface, s1.host_ip, s1.guest_ip)) def test_guest_cidr_is_31(self): - self.assertEqual("100.64.0.1/31", netpool.slot(0).guest_cidr) + with patch.dict(os.environ, {"BOT_BOTTLE_FC_IP_BASE": "100.64.0.0"}): + self.assertEqual("100.64.0.1/31", netpool.slot(0).guest_cidr) + + def test_default_base_avoids_cgnat_and_common_private_ranges(self): + # The default base must NOT sit in 100.64.0.0/10 (RFC-6598 CGNAT, + # which Tailscale hands node addresses from); it's an obscure + # RFC-1918 block instead. + with patch.dict(os.environ, {}, clear=True): + self.assertEqual("10.243.0.0", netpool.ip_base()) def test_pool_size_env_override(self): with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "4"}): @@ -65,6 +73,49 @@ class TestNetpoolRenderers(unittest.TestCase): self.assertIn("firecracker-netpool.sh up", out) +class TestNetpoolOverlap(unittest.TestCase): + """`overlapping_routes()` flags a pool base that collides with an + existing host route (Tailscale CGNAT peer, docker/libvirt bridge, + LAN) and ignores our own bbfc TAPs + the default route.""" + + def _routes(self, entries: list[dict]) -> object: + import json + import subprocess + + def fake_run(argv, **kwargs): + return subprocess.CompletedProcess( + argv, 0, stdout=json.dumps(entries), stderr="", + ) + return patch.object(netpool.subprocess, "run", side_effect=fake_run) + + def test_flags_route_overlapping_pool(self): + # base 100.64.0.0 + a Tailscale peer /32 inside the pool window. + with patch.dict(os.environ, {"BOT_BOTTLE_FC_IP_BASE": "100.64.0.0", + "BOT_BOTTLE_FC_POOL_SIZE": "8"}), \ + self._routes([ + {"dst": "default", "dev": "enp4s0"}, + {"dst": "100.64.0.5", "dev": "tailscale0"}, + ]): + conflicts = netpool.overlapping_routes() + self.assertEqual(1, len(conflicts)) + self.assertEqual("tailscale0", conflicts[0].dev) + + def test_ignores_own_taps_and_default(self): + with patch.dict(os.environ, {"BOT_BOTTLE_FC_IP_BASE": "10.243.0.0", + "BOT_BOTTLE_FC_POOL_SIZE": "8"}), \ + self._routes([ + {"dst": "default", "dev": "enp4s0"}, + {"dst": "10.243.0.0/31", "dev": "bbfc0"}, + {"dst": "192.168.1.0/24", "dev": "enp4s0"}, + ]): + self.assertEqual([], netpool.overlapping_routes()) + + def test_empty_when_ip_missing(self): + with self._routes([]) as run: + run.side_effect = FileNotFoundError() + self.assertEqual([], netpool.overlapping_routes()) + + class TestNetpoolAllocation(unittest.TestCase): def test_allocate_is_mutually_exclusive(self): with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "1"}):