diff --git a/bot_bottle/backend/firecracker/netpool.py b/bot_bottle/backend/firecracker/netpool.py index 5c9078d..ddbb968 100644 --- a/bot_bottle/backend/firecracker/netpool.py +++ b/bot_bottle/backend/firecracker/netpool.py @@ -6,8 +6,9 @@ a pool of point-to-point TAP devices (owned by the invoking user, so `./cli.py start` never needs root) and a dedicated nftables table that isolates every VM. This module is the single source of truth for the pool parameters — the shell script (`scripts/firecracker-netpool.sh`), -the printed NixOS snippet, and the backend's fail-closed preflight all -derive from the constants here so they can't drift. +the NixOS module (`nix/firecracker-netpool.nix`), and the backend's +fail-closed preflight all derive from these constants so they can't +drift. Topology (per slot i): * TAP ``bbfc{i}`` — no shared bridge, so no docker0 / virbr0 / cni0 @@ -237,72 +238,9 @@ def render_shell_setup() -> str: return f"sudo {prefix}./scripts/firecracker-netpool.sh up" -def render_nixos_module() -> str: - """A paste-ready NixOS snippet, derived from the live constants. - - Uses systemd-networkd tap netdevs (so TAPs are only reconfigured - when this config changes, not torn down on every rebuild — which - would drop running VMs) and `networking.nftables.tables.` - (an independent table that coexists with Docker's iptables rules - rather than replacing the whole ruleset).""" - slots = all_slots() - owner = os.environ.get("USER", "youruser") - - netdevs = "\n".join( - f''' "10-{s.iface}" = {{ - netdevConfig = {{ Name = "{s.iface}"; Kind = "tap"; }}; - tapConfig = {{ User = "{owner}"; }}; - }};''' - for s in slots - ) - networks = "\n".join( - f''' "10-{s.iface}" = {{ - matchConfig.Name = "{s.iface}"; - address = [ "{s.host_ip}/31" ]; - networkConfig.ConfigureWithoutCarrier = true; - }};''' - for s in slots - ) - - return f"""# bot-bottle Firecracker network pool ({len(slots)} slots, base {ip_base()}). -# Generated by `./cli.py backend setup --backend=firecracker`; owner user = {owner!r}. -{{ ... }}: -{{ - boot.kernel.sysctl."net.ipv4.ip_forward" = 1; - - systemd.network.enable = true; - systemd.network.netdevs = {{ -{netdevs} - }}; - systemd.network.networks = {{ -{networks} - }}; - - # Independent table — does not touch Docker/ufw/firewalld rules. - # A bottle VM (traffic from a bbfc* TAP) may reach only its own - # sidecar (DNAT'd from the host TAP IP); everything else is dropped, - # so there is no egress except through the sidecar proxy. - networking.nftables.enable = true; - networking.nftables.tables."{NFT_TABLE}" = {{ - family = "inet"; - content = '' - chain forward {{ - type filter hook forward priority -10; policy accept; - iifname != "{IFACE_PREFIX}*" return - ct state established,related accept - ct status dnat accept - drop - }} - chain input {{ - type filter hook input priority -10; policy accept; - iifname != "{IFACE_PREFIX}*" return - ct state established,related accept - drop - }} - ''; - }}; -}} -""" +# The NixOS setup is a real, importable module (nix/firecracker-netpool.nix, +# exposed as the flake output nixosModules.firecracker-netpool) rather than a +# generated paste — see `backend setup` output. def _nondefault_env() -> str: diff --git a/bot_bottle/backend/firecracker/setup.py b/bot_bottle/backend/firecracker/setup.py index 95a1e41..931055a 100644 --- a/bot_bottle/backend/firecracker/setup.py +++ b/bot_bottle/backend/firecracker/setup.py @@ -29,6 +29,11 @@ def _owner() -> str: return os.environ.get("USER", "youruser") +def _module_path() -> str: + """Absolute path to the importable NixOS module in this checkout.""" + return str(Path(__file__).resolve().parents[3] / "nix" / "firecracker-netpool.nix") + + def _print_prereqs() -> None: """The firecracker binary + KVM + guest artifacts, shown before the privileged network-pool step so operators see the full picture.""" @@ -96,18 +101,18 @@ def setup() -> int: _warn_overlaps() if _is_nixos(): sys.stderr.write( - "Detected NixOS. Preferred: consume the flake module (versioned, " - "no copy-paste drift):\n\n" - " # flake inputs (point at wherever you host bot-bottle):\n" + "Detected NixOS. Import the module — it is NON-INVASIVE: it does " + "not flip networking.nftables.enable or systemd.network.enable, so " + "your existing (iptables) firewall and Docker are untouched. A " + "systemd oneshot brings the pool up alongside them.\n\n" + " # flake users:\n" " inputs.bot-bottle.url = \"git+ssh://\";\n" - " # host module:\n" " imports = [ inputs.bot-bottle.nixosModules.firecracker-netpool ];\n" + " # channel (non-flake) users — import the file directly:\n" + f" imports = [ {_module_path()} ];\n\n" f" services.bot-bottle-firecracker = {{ enable = true; owner = \"{_owner()}\"; }};\n\n" - "Then `nixos-rebuild switch`. Channel (non-flake) users can " - "`imports = [ /nix/firecracker-netpool.nix ];` instead.\n\n" - "Fallback — paste this generated module directly:\n\n" + "Then `nixos-rebuild switch`.\n" ) - sys.stdout.write(netpool.render_nixos_module()) else: sys.stderr.write("Run the one-time setup as root:\n\n") sys.stdout.write(netpool.render_shell_setup() + "\n") diff --git a/nix/firecracker-netpool.nix b/nix/firecracker-netpool.nix index 8d5f6b3..0357bda 100644 --- a/nix/firecracker-netpool.nix +++ b/nix/firecracker-netpool.nix @@ -2,10 +2,20 @@ # # The one-time privileged setup the Firecracker backend needs: a pool of # user-owned point-to-point TAP devices plus a fail-closed nftables table -# that confines every microVM to its own sidecar. Enabling this module is -# the NixOS equivalent of `sudo ./scripts/firecracker-netpool.sh up` — but -# declarative, so the taps/table survive `nixos-rebuild` and are only -# reconfigured when this config changes (running VMs aren't dropped). +# that confines every microVM to its own sidecar. +# +# NON-INVASIVE BY DESIGN. It does NOT flip `networking.nftables.enable` +# (which would switch your whole host firewall backend) or +# `systemd.network.enable` (which would hand your interfaces to +# systemd-networkd). Instead a single systemd oneshot service brings the +# pool up on boot — the declarative equivalent of +# `sudo ./scripts/firecracker-netpool.sh up`. The `inet ` table +# is independent (its own hooks at priority -10), so it coexists with an +# iptables `networking.firewall`, Docker, ufw, firewalld, etc. +# +# The oneshot is only restarted when this config changes, so a +# `nixos-rebuild switch` doesn't tear down TAPs out from under running +# VMs unless you actually changed the pool. # # The option defaults MUST match the backend constants in # bot_bottle/backend/firecracker/netpool.py (pool size, IP base, iface @@ -13,7 +23,7 @@ # the BOT_BOTTLE_FC_* env vars; if you override an option here, override # the matching env var for the CLI too (or set writeEnvFile = true below # to have this module emit them). Keep the two sides in lockstep. -{ config, lib, ... }: +{ config, lib, pkgs, ... }: let cfg = config.services.bot-bottle-firecracker; @@ -42,7 +52,47 @@ let hostIp = intToIp (baseInt + 2 * i); }) cfg.poolSize; - mkAttr = f: lib.listToAttrs (map (s: lib.nameValuePair "10-${s.iface}" (f s)) slots); + ip = "${pkgs.iproute2}/bin/ip"; + nft = "${pkgs.nftables}/bin/nft"; + + # Idempotent ruleset: (create-if-absent → delete → recreate) is atomic + # within one `nft -f`, so re-running `up` always lands a clean table. + nftFile = pkgs.writeText "bot-bottle-fc.nft" '' + table inet ${cfg.tableName} + delete table inet ${cfg.tableName} + table inet ${cfg.tableName} { + chain forward { + type filter hook forward priority -10; policy accept; + iifname != "${cfg.ifacePrefix}*" return + ct state established,related accept + ct status dnat accept + drop + } + chain input { + type filter hook input priority -10; policy accept; + iifname != "${cfg.ifacePrefix}*" return + ct state established,related accept + drop + } + } + ''; + + upScript = pkgs.writeShellScript "bot-bottle-fc-netpool-up" '' + set -eu + ${lib.concatMapStringsSep "\n" (s: '' + ${ip} link show ${s.iface} >/dev/null 2>&1 || ${ip} tuntap add dev ${s.iface} mode tap user ${cfg.owner} + ${ip} addr replace ${s.hostIp}/31 dev ${s.iface} + ${ip} link set ${s.iface} up + '') slots} + ${nft} -f ${nftFile} + ''; + + downScript = pkgs.writeShellScript "bot-bottle-fc-netpool-down" '' + ${nft} delete table inet ${cfg.tableName} 2>/dev/null || true + ${lib.concatMapStringsSep "\n" (s: '' + ${ip} link show ${s.iface} >/dev/null 2>&1 && ${ip} tuntap del dev ${s.iface} mode tap || true + '') slots} + ''; in { options.services.bot-bottle-firecracker = { @@ -106,39 +156,18 @@ in # VM->sidecar traffic is DNAT'd and forwarded, so forwarding must be on. boot.kernel.sysctl."net.ipv4.ip_forward" = 1; - systemd.network.enable = true; - systemd.network.netdevs = mkAttr (s: { - netdevConfig = { Name = s.iface; Kind = "tap"; }; - tapConfig = { User = cfg.owner; }; - }); - systemd.network.networks = mkAttr (s: { - matchConfig.Name = s.iface; - address = [ "${s.hostIp}/31" ]; - networkConfig.ConfigureWithoutCarrier = true; - }); - - # Independent, fail-closed table — coexists with Docker/ufw/firewalld - # rather than replacing the ruleset. A VM (traffic from a - # ${cfg.ifacePrefix}* TAP) may reach only its own sidecar (DNAT'd from - # the host TAP IP); everything else is dropped. - networking.nftables.enable = true; - networking.nftables.tables.${cfg.tableName} = { - family = "inet"; - content = '' - chain forward { - type filter hook forward priority -10; policy accept; - iifname != "${cfg.ifacePrefix}*" return - ct state established,related accept - ct status dnat accept - drop - } - chain input { - type filter hook input priority -10; policy accept; - iifname != "${cfg.ifacePrefix}*" return - ct state established,related accept - drop - } - ''; + # One oneshot brings up the whole pool (TAPs + independent nft table). + # No networking.nftables.enable / systemd.network.enable — see header. + systemd.services."bot-bottle-firecracker-netpool" = { + description = "bot-bottle Firecracker TAP pool + nft isolation table"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-pre.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = upScript; + ExecStop = downScript; + }; }; environment.etc."bot-bottle/firecracker.env" = lib.mkIf cfg.writeEnvFile { diff --git a/tests/unit/test_firecracker_backend.py b/tests/unit/test_firecracker_backend.py index 79e6954..28f92c7 100644 --- a/tests/unit/test_firecracker_backend.py +++ b/tests/unit/test_firecracker_backend.py @@ -55,16 +55,17 @@ class TestNetpoolSlots(unittest.TestCase): class TestNetpoolRenderers(unittest.TestCase): - def test_nixos_module_has_table_and_taps(self): - with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "2"}): - out = netpool.render_nixos_module() - self.assertIn('networking.nftables.tables."bot_bottle_fc"', out) - self.assertIn('Name = "bbfc0"', out) - self.assertIn('Name = "bbfc1"', out) - self.assertIn("net.ipv4.ip_forward", out) - # fail-closed isolation rules present - self.assertIn('iifname != "bbfc*" return', out) - self.assertIn("ct status dnat accept", out) + def test_nixos_module_is_non_invasive(self): + # The NixOS module must NOT flip the host firewall backend or + # hand interfaces to systemd-networkd; it brings the pool up via + # a systemd oneshot that coexists with an iptables firewall. + root = Path(__file__).resolve().parents[2] + mod = (root / "nix" / "firecracker-netpool.nix").read_text() + self.assertNotIn("networking.nftables.enable = true", mod) + self.assertNotIn("systemd.network.enable = true", mod) + self.assertIn("bot-bottle-firecracker-netpool", mod) # the oneshot + self.assertIn(netpool.NFT_TABLE, mod) + self.assertIn('iifname != "${cfg.ifacePrefix}*" return', mod) def test_shell_setup_reflects_overrides(self): with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "3"}):