feat(firecracker): add Linux microVM backend to replace smolmachines

Adds a Firecracker-based backend for Linux, providing mature KVM-based microVM isolation to replace smolmachines/libkrun (issue #342, closes the dead-end tracked in #332). Architecture: - Guest control over SSH (dropbear injected into the rootfs) on a point-to-point TAP link. `ssh -t` forwards SIGWINCH natively, so no resize bridge is needed. - Networking: a one-time, root-provisioned pool of user-owned TAP devices (no shared bridge → no docker0/virbr0/cni0 collisions) plus a dedicated `table inet bot_bottle_fc` nftables table (independent of Docker/ufw/firewalld rules). `./cli.py firecracker setup` prints the host-appropriate config (NixOS module or sudo script). - Rootfs: `docker export` → ext4 via `mke2fs -d` (rootless, no mount), cached by image digest; per-bottle SSH pubkey + IP passed via the kernel cmdline. - Sidecar: reuses the Docker bundle, published on the slot's host TAP IP. - Fail-closed isolation: TAP pool verified at preflight; the egress boundary is proven empirically post-boot (before the agent runs) by a canary probe — the VM must fail to reach the host directly, or launch is refused. Linux hosts with Firecracker + KVM now default to this backend; macOS stays on macos-container. Not yet validated end-to-end on live hardware (requires the one-time network pool). Unit tests + pyright pass. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01G8p32HJgPoS1hLPWubbftM
2026-07-11 10:32:55 -04:00
parent 814c7338a1
commit c276f7b0b1
21 changed files with 2309 additions and 5 deletions
@@ -0,0 +1,243 @@
+"""Firecracker network pool: constants, IP math, allocation, and the
+config renderers (shell command + NixOS module) shown to operators.
+
+The Firecracker backend needs a privileged one-time network setup:
+a pool of point-to-point TAP devices (owned by the invoking user, so
+`./cli.py start` never needs root) and a dedicated nftables table that
+isolates every VM. This module is the single source of truth for the
+pool parameters — the shell script (`scripts/firecracker-netpool.sh`),
+the printed NixOS snippet, and the backend's fail-closed preflight all
+derive from the constants here so they can't drift.
+
+Topology (per slot i):
+  * TAP  ``bbfc{i}``  — no shared bridge, so no docker0 / virbr0 / cni0
+    / br-* collisions.
+  * a /31 host<->guest link: host = base + 2i (the gateway the VM
+    routes through), guest = base + 2i + 1 (the VM's address).
+  * isolation via ``table inet bot_bottle_fc``: a VM reaches only its
+    own sidecar (DNAT'd from the host TAP IP) and nothing else.
+
+The default IP block is RFC-6598 shared address space (100.64.0.0/10),
+which is purpose-built to avoid colliding with LAN/VPN private ranges.
+"""
+
+from __future__ import annotations
+
+import fcntl
+import ipaddress
+import os
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import IO
+
+from ...log import die
+
+
+# Interface names are capped at 15 chars (IFNAMSIZ-1); "bbfc" + a small
+# index stays well under that and is distinctive enough to grep for.
+IFACE_PREFIX = os.environ.get("BOT_BOTTLE_FC_IFACE_PREFIX", "bbfc")
+NFT_TABLE = "bot_bottle_fc"
+
+
+def pool_size() -> int:
+    return int(os.environ.get("BOT_BOTTLE_FC_POOL_SIZE", "8"))
+
+
+def ip_base() -> str:
+    return os.environ.get("BOT_BOTTLE_FC_IP_BASE", "100.64.0.0")
+
+
+# Sidecar ports the VM reaches at its host-side TAP IP. Kept in sync
+# with the backend constants (egress 9099, supervise 9100, git-http
+# 9420); rendered into the setup output for operator visibility.
+SIDECAR_PORTS = (9099, 9100, 9420)
+
+
+@dataclass(frozen=True)
+class Slot:
+    """One pool slot: a TAP device and its host/guest /31 addresses."""
+
+    index: int
+    iface: str
+    host_ip: str
+    guest_ip: str
+
+    @property
+    def guest_cidr(self) -> str:
+        """Guest address with the /31 prefix, for the kernel `ip=` arg."""
+        return f"{self.guest_ip}/31"
+
+
+def slot(index: int) -> Slot:
+    base = int(ipaddress.IPv4Address(ip_base()))
+    return Slot(
+        index=index,
+        iface=f"{IFACE_PREFIX}{index}",
+        host_ip=str(ipaddress.IPv4Address(base + 2 * index)),
+        guest_ip=str(ipaddress.IPv4Address(base + 2 * index + 1)),
+    )
+
+
+def all_slots() -> list[Slot]:
+    return [slot(i) for i in range(pool_size())]
+
+
+# --- fail-closed verification ---------------------------------------
+
+def _run_ok(argv: list[str]) -> bool:
+    """Run a probe command, treating a missing binary as failure
+    (rather than crashing) so callers can stay fail-closed."""
+    try:
+        return subprocess.run(
+            argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+            check=False,
+        ).returncode == 0
+    except FileNotFoundError:
+        return False
+
+
+def nft_table_present() -> bool:
+    """True iff the isolation table exists in the active nftables
+    backend. The backend's preflight treats absence as fatal — the VM
+    must not boot without its egress boundary in place.
+
+    Listing may require root; when it can't be confirmed here the
+    launch path falls back to an empirical post-boot isolation probe.
+    Returns False (fail-closed) if `nft` is unavailable."""
+    return _run_ok(["nft", "list", "table", "inet", NFT_TABLE])
+
+
+def tap_present(iface: str) -> bool:
+    # `ip link show` is unprivileged, so the TAP-pool check is reliable
+    # for the non-root launcher.
+    return _run_ok(["ip", "link", "show", iface])
+
+
+def missing_taps() -> list[str]:
+    """Pool TAPs that the one-time setup has not created yet."""
+    return [s.iface for s in all_slots() if not tap_present(s.iface)]
+
+
+# --- allocation ------------------------------------------------------
+
+def _lock_dir() -> Path:
+    d = Path.home() / ".cache" / "bot-bottle" / "firecracker" / "pool"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def allocate(slug: str) -> tuple[Slot, IO[str]]:
+    """Claim a free pool slot for one bottle. Returns the slot and the
+    held lock file — the caller keeps it open for the VM's lifetime and
+    closes it on teardown; the flock auto-releases if the launcher
+    crashes, so a slot is never leaked. Dies when the pool is
+    exhausted.
+
+    A per-slot `flock` (rather than inspecting running processes) makes
+    allocation race-free across concurrent launches without a central
+    registry: the first launcher to grab the lock owns the slot."""
+    del slug  # logged by the caller; allocation is purely lock-driven
+    for s in all_slots():
+        lock_path = _lock_dir() / f"{s.iface}.lock"
+        handle = open(lock_path, "w", encoding="utf-8")
+        try:
+            fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError:
+            handle.close()
+            continue
+        return s, handle
+    die(f"Firecracker TAP pool exhausted ({pool_size()} slots, all in "
+        f"use). Stop a running bottle or raise BOT_BOTTLE_FC_POOL_SIZE "
+        f"and re-run `./cli.py firecracker setup`.")
+    raise AssertionError("unreachable")
+
+
+# --- config renderers (shown by `./cli.py firecracker setup`) --------
+
+def render_shell_setup() -> str:
+    """The imperative command for non-NixOS hosts."""
+    env = _nondefault_env()
+    prefix = f"{env} " if env else ""
+    return f"sudo {prefix}./scripts/firecracker-netpool.sh up"
+
+
+def render_nixos_module() -> str:
+    """A paste-ready NixOS snippet, derived from the live constants.
+
+    Uses systemd-networkd tap netdevs (so TAPs are only reconfigured
+    when this config changes, not torn down on every rebuild — which
+    would drop running VMs) and `networking.nftables.tables.<name>`
+    (an independent table that coexists with Docker's iptables rules
+    rather than replacing the whole ruleset)."""
+    slots = all_slots()
+    owner = os.environ.get("USER", "youruser")
+
+    netdevs = "\n".join(
+        f'''        "10-{s.iface}" = {{
+          netdevConfig = {{ Name = "{s.iface}"; Kind = "tap"; }};
+          tapConfig = {{ User = "{owner}"; }};
+        }};'''
+        for s in slots
+    )
+    networks = "\n".join(
+        f'''        "10-{s.iface}" = {{
+          matchConfig.Name = "{s.iface}";
+          address = [ "{s.host_ip}/31" ];
+          networkConfig.ConfigureWithoutCarrier = true;
+        }};'''
+        for s in slots
+    )
+
+    return f"""# bot-bottle Firecracker network pool ({len(slots)} slots, base {ip_base()}).
+# Generated by `./cli.py firecracker setup`; owner user = {owner!r}.
+{{ ... }}:
+{{
+  boot.kernel.sysctl."net.ipv4.ip_forward" = 1;
+
+  systemd.network.enable = true;
+  systemd.network.netdevs = {{
+{netdevs}
+  }};
+  systemd.network.networks = {{
+{networks}
+  }};
+
+  # Independent table — does not touch Docker/ufw/firewalld rules.
+  # A bottle VM (traffic from a bbfc* TAP) may reach only its own
+  # sidecar (DNAT'd from the host TAP IP); everything else is dropped,
+  # so there is no egress except through the sidecar proxy.
+  networking.nftables.enable = true;
+  networking.nftables.tables."{NFT_TABLE}" = {{
+    family = "inet";
+    content = ''
+      chain forward {{
+        type filter hook forward priority -10; policy accept;
+        iifname != "{IFACE_PREFIX}*" return
+        ct state established,related accept
+        ct status dnat accept
+        drop
+      }}
+      chain input {{
+        type filter hook input priority -10; policy accept;
+        iifname != "{IFACE_PREFIX}*" return
+        ct state established,related accept
+        drop
+      }}
+    '';
+  }};
+}}
+"""
+
+
+def _nondefault_env() -> str:
+    """Render any non-default pool env overrides so the printed shell
+    command reproduces the operator's current settings."""
+    pairs = []
+    if os.environ.get("BOT_BOTTLE_FC_POOL_SIZE"):
+        pairs.append(f"BOT_BOTTLE_FC_POOL_SIZE={pool_size()}")
+    if os.environ.get("BOT_BOTTLE_FC_IP_BASE"):
+        pairs.append(f"BOT_BOTTLE_FC_IP_BASE={ip_base()}")
+    if os.environ.get("BOT_BOTTLE_FC_IFACE_PREFIX"):
+        pairs.append(f"BOT_BOTTLE_FC_IFACE_PREFIX={IFACE_PREFIX}")
+    return " ".join(pairs)