refactor(firecracker): non-invasive NixOS module (no firewall switch)
The module used `networking.nftables.tables.*` (which forces `networking.nftables.enable = true`, flipping the host firewall backend) and `systemd.network.enable` (handing interfaces to systemd-networkd) — disruptive on an iptables + Docker daily driver. Rewrite it to a single systemd oneshot that brings the pool up: idempotent `ip tuntap` for the TAPs + `nft -f` for the independent `inet bot_bottle_fc` table (its own hooks at priority -10), with ExecStop teardown. Same as the imperative script, but declarative. It touches neither the firewall backend nor networkd, so it coexists with iptables/Docker/ufw/firewalld. Verified through the NixOS module system (service present, firewall untouched). Drop the now-redundant `render_nixos_module()` paste generator (the module is a real importable file) and point `backend setup` at importing it (flake output or the file path), noting it's non-invasive. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck
This commit is contained in:
@@ -6,8 +6,9 @@ a pool of point-to-point TAP devices (owned by the invoking user, so
|
||||
`./cli.py start` never needs root) and a dedicated nftables table that
|
||||
isolates every VM. This module is the single source of truth for the
|
||||
pool parameters — the shell script (`scripts/firecracker-netpool.sh`),
|
||||
the printed NixOS snippet, and the backend's fail-closed preflight all
|
||||
derive from the constants here so they can't drift.
|
||||
the NixOS module (`nix/firecracker-netpool.nix`), and the backend's
|
||||
fail-closed preflight all derive from these constants so they can't
|
||||
drift.
|
||||
|
||||
Topology (per slot i):
|
||||
* TAP ``bbfc{i}`` — no shared bridge, so no docker0 / virbr0 / cni0
|
||||
@@ -237,72 +238,9 @@ def render_shell_setup() -> str:
|
||||
return f"sudo {prefix}./scripts/firecracker-netpool.sh up"
|
||||
|
||||
|
||||
def render_nixos_module() -> str:
|
||||
"""A paste-ready NixOS snippet, derived from the live constants.
|
||||
|
||||
Uses systemd-networkd tap netdevs (so TAPs are only reconfigured
|
||||
when this config changes, not torn down on every rebuild — which
|
||||
would drop running VMs) and `networking.nftables.tables.<name>`
|
||||
(an independent table that coexists with Docker's iptables rules
|
||||
rather than replacing the whole ruleset)."""
|
||||
slots = all_slots()
|
||||
owner = os.environ.get("USER", "youruser")
|
||||
|
||||
netdevs = "\n".join(
|
||||
f''' "10-{s.iface}" = {{
|
||||
netdevConfig = {{ Name = "{s.iface}"; Kind = "tap"; }};
|
||||
tapConfig = {{ User = "{owner}"; }};
|
||||
}};'''
|
||||
for s in slots
|
||||
)
|
||||
networks = "\n".join(
|
||||
f''' "10-{s.iface}" = {{
|
||||
matchConfig.Name = "{s.iface}";
|
||||
address = [ "{s.host_ip}/31" ];
|
||||
networkConfig.ConfigureWithoutCarrier = true;
|
||||
}};'''
|
||||
for s in slots
|
||||
)
|
||||
|
||||
return f"""# bot-bottle Firecracker network pool ({len(slots)} slots, base {ip_base()}).
|
||||
# Generated by `./cli.py backend setup --backend=firecracker`; owner user = {owner!r}.
|
||||
{{ ... }}:
|
||||
{{
|
||||
boot.kernel.sysctl."net.ipv4.ip_forward" = 1;
|
||||
|
||||
systemd.network.enable = true;
|
||||
systemd.network.netdevs = {{
|
||||
{netdevs}
|
||||
}};
|
||||
systemd.network.networks = {{
|
||||
{networks}
|
||||
}};
|
||||
|
||||
# Independent table — does not touch Docker/ufw/firewalld rules.
|
||||
# A bottle VM (traffic from a bbfc* TAP) may reach only its own
|
||||
# sidecar (DNAT'd from the host TAP IP); everything else is dropped,
|
||||
# so there is no egress except through the sidecar proxy.
|
||||
networking.nftables.enable = true;
|
||||
networking.nftables.tables."{NFT_TABLE}" = {{
|
||||
family = "inet";
|
||||
content = ''
|
||||
chain forward {{
|
||||
type filter hook forward priority -10; policy accept;
|
||||
iifname != "{IFACE_PREFIX}*" return
|
||||
ct state established,related accept
|
||||
ct status dnat accept
|
||||
drop
|
||||
}}
|
||||
chain input {{
|
||||
type filter hook input priority -10; policy accept;
|
||||
iifname != "{IFACE_PREFIX}*" return
|
||||
ct state established,related accept
|
||||
drop
|
||||
}}
|
||||
'';
|
||||
}};
|
||||
}}
|
||||
"""
|
||||
# The NixOS setup is a real, importable module (nix/firecracker-netpool.nix,
|
||||
# exposed as the flake output nixosModules.firecracker-netpool) rather than a
|
||||
# generated paste — see `backend setup` output.
|
||||
|
||||
|
||||
def _nondefault_env() -> str:
|
||||
|
||||
@@ -29,6 +29,11 @@ def _owner() -> str:
|
||||
return os.environ.get("USER", "youruser")
|
||||
|
||||
|
||||
def _module_path() -> str:
|
||||
"""Absolute path to the importable NixOS module in this checkout."""
|
||||
return str(Path(__file__).resolve().parents[3] / "nix" / "firecracker-netpool.nix")
|
||||
|
||||
|
||||
def _print_prereqs() -> None:
|
||||
"""The firecracker binary + KVM + guest artifacts, shown before the
|
||||
privileged network-pool step so operators see the full picture."""
|
||||
@@ -96,18 +101,18 @@ def setup() -> int:
|
||||
_warn_overlaps()
|
||||
if _is_nixos():
|
||||
sys.stderr.write(
|
||||
"Detected NixOS. Preferred: consume the flake module (versioned, "
|
||||
"no copy-paste drift):\n\n"
|
||||
" # flake inputs (point at wherever you host bot-bottle):\n"
|
||||
"Detected NixOS. Import the module — it is NON-INVASIVE: it does "
|
||||
"not flip networking.nftables.enable or systemd.network.enable, so "
|
||||
"your existing (iptables) firewall and Docker are untouched. A "
|
||||
"systemd oneshot brings the pool up alongside them.\n\n"
|
||||
" # flake users:\n"
|
||||
" inputs.bot-bottle.url = \"git+ssh://<your-bot-bottle-remote>\";\n"
|
||||
" # host module:\n"
|
||||
" imports = [ inputs.bot-bottle.nixosModules.firecracker-netpool ];\n"
|
||||
" # channel (non-flake) users — import the file directly:\n"
|
||||
f" imports = [ {_module_path()} ];\n\n"
|
||||
f" services.bot-bottle-firecracker = {{ enable = true; owner = \"{_owner()}\"; }};\n\n"
|
||||
"Then `nixos-rebuild switch`. Channel (non-flake) users can "
|
||||
"`imports = [ <bot-bottle>/nix/firecracker-netpool.nix ];` instead.\n\n"
|
||||
"Fallback — paste this generated module directly:\n\n"
|
||||
"Then `nixos-rebuild switch`.\n"
|
||||
)
|
||||
sys.stdout.write(netpool.render_nixos_module())
|
||||
else:
|
||||
sys.stderr.write("Run the one-time setup as root:\n\n")
|
||||
sys.stdout.write(netpool.render_shell_setup() + "\n")
|
||||
|
||||
+68
-39
@@ -2,10 +2,20 @@
|
||||
#
|
||||
# The one-time privileged setup the Firecracker backend needs: a pool of
|
||||
# user-owned point-to-point TAP devices plus a fail-closed nftables table
|
||||
# that confines every microVM to its own sidecar. Enabling this module is
|
||||
# the NixOS equivalent of `sudo ./scripts/firecracker-netpool.sh up` — but
|
||||
# declarative, so the taps/table survive `nixos-rebuild` and are only
|
||||
# reconfigured when this config changes (running VMs aren't dropped).
|
||||
# that confines every microVM to its own sidecar.
|
||||
#
|
||||
# NON-INVASIVE BY DESIGN. It does NOT flip `networking.nftables.enable`
|
||||
# (which would switch your whole host firewall backend) or
|
||||
# `systemd.network.enable` (which would hand your interfaces to
|
||||
# systemd-networkd). Instead a single systemd oneshot service brings the
|
||||
# pool up on boot — the declarative equivalent of
|
||||
# `sudo ./scripts/firecracker-netpool.sh up`. The `inet <tableName>` table
|
||||
# is independent (its own hooks at priority -10), so it coexists with an
|
||||
# iptables `networking.firewall`, Docker, ufw, firewalld, etc.
|
||||
#
|
||||
# The oneshot is only restarted when this config changes, so a
|
||||
# `nixos-rebuild switch` doesn't tear down TAPs out from under running
|
||||
# VMs unless you actually changed the pool.
|
||||
#
|
||||
# The option defaults MUST match the backend constants in
|
||||
# bot_bottle/backend/firecracker/netpool.py (pool size, IP base, iface
|
||||
@@ -13,7 +23,7 @@
|
||||
# the BOT_BOTTLE_FC_* env vars; if you override an option here, override
|
||||
# the matching env var for the CLI too (or set writeEnvFile = true below
|
||||
# to have this module emit them). Keep the two sides in lockstep.
|
||||
{ config, lib, ... }:
|
||||
{ config, lib, pkgs, ... }:
|
||||
|
||||
let
|
||||
cfg = config.services.bot-bottle-firecracker;
|
||||
@@ -42,7 +52,47 @@ let
|
||||
hostIp = intToIp (baseInt + 2 * i);
|
||||
}) cfg.poolSize;
|
||||
|
||||
mkAttr = f: lib.listToAttrs (map (s: lib.nameValuePair "10-${s.iface}" (f s)) slots);
|
||||
ip = "${pkgs.iproute2}/bin/ip";
|
||||
nft = "${pkgs.nftables}/bin/nft";
|
||||
|
||||
# Idempotent ruleset: (create-if-absent → delete → recreate) is atomic
|
||||
# within one `nft -f`, so re-running `up` always lands a clean table.
|
||||
nftFile = pkgs.writeText "bot-bottle-fc.nft" ''
|
||||
table inet ${cfg.tableName}
|
||||
delete table inet ${cfg.tableName}
|
||||
table inet ${cfg.tableName} {
|
||||
chain forward {
|
||||
type filter hook forward priority -10; policy accept;
|
||||
iifname != "${cfg.ifacePrefix}*" return
|
||||
ct state established,related accept
|
||||
ct status dnat accept
|
||||
drop
|
||||
}
|
||||
chain input {
|
||||
type filter hook input priority -10; policy accept;
|
||||
iifname != "${cfg.ifacePrefix}*" return
|
||||
ct state established,related accept
|
||||
drop
|
||||
}
|
||||
}
|
||||
'';
|
||||
|
||||
upScript = pkgs.writeShellScript "bot-bottle-fc-netpool-up" ''
|
||||
set -eu
|
||||
${lib.concatMapStringsSep "\n" (s: ''
|
||||
${ip} link show ${s.iface} >/dev/null 2>&1 || ${ip} tuntap add dev ${s.iface} mode tap user ${cfg.owner}
|
||||
${ip} addr replace ${s.hostIp}/31 dev ${s.iface}
|
||||
${ip} link set ${s.iface} up
|
||||
'') slots}
|
||||
${nft} -f ${nftFile}
|
||||
'';
|
||||
|
||||
downScript = pkgs.writeShellScript "bot-bottle-fc-netpool-down" ''
|
||||
${nft} delete table inet ${cfg.tableName} 2>/dev/null || true
|
||||
${lib.concatMapStringsSep "\n" (s: ''
|
||||
${ip} link show ${s.iface} >/dev/null 2>&1 && ${ip} tuntap del dev ${s.iface} mode tap || true
|
||||
'') slots}
|
||||
'';
|
||||
in
|
||||
{
|
||||
options.services.bot-bottle-firecracker = {
|
||||
@@ -106,39 +156,18 @@ in
|
||||
# VM->sidecar traffic is DNAT'd and forwarded, so forwarding must be on.
|
||||
boot.kernel.sysctl."net.ipv4.ip_forward" = 1;
|
||||
|
||||
systemd.network.enable = true;
|
||||
systemd.network.netdevs = mkAttr (s: {
|
||||
netdevConfig = { Name = s.iface; Kind = "tap"; };
|
||||
tapConfig = { User = cfg.owner; };
|
||||
});
|
||||
systemd.network.networks = mkAttr (s: {
|
||||
matchConfig.Name = s.iface;
|
||||
address = [ "${s.hostIp}/31" ];
|
||||
networkConfig.ConfigureWithoutCarrier = true;
|
||||
});
|
||||
|
||||
# Independent, fail-closed table — coexists with Docker/ufw/firewalld
|
||||
# rather than replacing the ruleset. A VM (traffic from a
|
||||
# ${cfg.ifacePrefix}* TAP) may reach only its own sidecar (DNAT'd from
|
||||
# the host TAP IP); everything else is dropped.
|
||||
networking.nftables.enable = true;
|
||||
networking.nftables.tables.${cfg.tableName} = {
|
||||
family = "inet";
|
||||
content = ''
|
||||
chain forward {
|
||||
type filter hook forward priority -10; policy accept;
|
||||
iifname != "${cfg.ifacePrefix}*" return
|
||||
ct state established,related accept
|
||||
ct status dnat accept
|
||||
drop
|
||||
}
|
||||
chain input {
|
||||
type filter hook input priority -10; policy accept;
|
||||
iifname != "${cfg.ifacePrefix}*" return
|
||||
ct state established,related accept
|
||||
drop
|
||||
}
|
||||
'';
|
||||
# One oneshot brings up the whole pool (TAPs + independent nft table).
|
||||
# No networking.nftables.enable / systemd.network.enable — see header.
|
||||
systemd.services."bot-bottle-firecracker-netpool" = {
|
||||
description = "bot-bottle Firecracker TAP pool + nft isolation table";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "network-pre.target" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
ExecStart = upScript;
|
||||
ExecStop = downScript;
|
||||
};
|
||||
};
|
||||
|
||||
environment.etc."bot-bottle/firecracker.env" = lib.mkIf cfg.writeEnvFile {
|
||||
|
||||
@@ -55,16 +55,17 @@ class TestNetpoolSlots(unittest.TestCase):
|
||||
|
||||
|
||||
class TestNetpoolRenderers(unittest.TestCase):
|
||||
def test_nixos_module_has_table_and_taps(self):
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "2"}):
|
||||
out = netpool.render_nixos_module()
|
||||
self.assertIn('networking.nftables.tables."bot_bottle_fc"', out)
|
||||
self.assertIn('Name = "bbfc0"', out)
|
||||
self.assertIn('Name = "bbfc1"', out)
|
||||
self.assertIn("net.ipv4.ip_forward", out)
|
||||
# fail-closed isolation rules present
|
||||
self.assertIn('iifname != "bbfc*" return', out)
|
||||
self.assertIn("ct status dnat accept", out)
|
||||
def test_nixos_module_is_non_invasive(self):
|
||||
# The NixOS module must NOT flip the host firewall backend or
|
||||
# hand interfaces to systemd-networkd; it brings the pool up via
|
||||
# a systemd oneshot that coexists with an iptables firewall.
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
mod = (root / "nix" / "firecracker-netpool.nix").read_text()
|
||||
self.assertNotIn("networking.nftables.enable = true", mod)
|
||||
self.assertNotIn("systemd.network.enable = true", mod)
|
||||
self.assertIn("bot-bottle-firecracker-netpool", mod) # the oneshot
|
||||
self.assertIn(netpool.NFT_TABLE, mod)
|
||||
self.assertIn('iifname != "${cfg.ifacePrefix}*" return', mod)
|
||||
|
||||
def test_shell_setup_reflects_overrides(self):
|
||||
with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "3"}):
|
||||
|
||||
Reference in New Issue
Block a user