diff --git a/bot_bottle/backend/__init__.py b/bot_bottle/backend/__init__.py index ed8b894..41559c5 100644 --- a/bot_bottle/backend/__init__.py +++ b/bot_bottle/backend/__init__.py @@ -26,8 +26,9 @@ backend exposes five methods: Selection is driven by `--backend` on `start` or BOT_BOTTLE_BACKEND (env var). When neither is set, compatible macOS hosts default to -`macos-container`; other hosts default to `smolmachines`. Per PRD 0003 -the manifest does not carry a backend field; the host picks. +`macos-container`; Linux hosts with Firecracker + KVM default to +`firecracker`; otherwise `smolmachines`. Per PRD 0003 the manifest +does not carry a backend field; the host picks. """ from __future__ import annotations @@ -542,6 +543,7 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]): # each backend module can pull BottleSpec / BottlePlan / BottleBackend # via `from . import ...` without hitting a partially-initialized module. from .docker import DockerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position +from .firecracker import FirecrackerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position from .macos_container import MacosContainerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position from .smolmachines import SmolmachinesBottleBackend # noqa: E402 # pylint: disable=wrong-import-position @@ -557,6 +559,7 @@ from .freeze import CommitCancelled, Freezer, get_freezer # noqa: E402 # pylin # unparameterized methods (prepare → plan → launch(plan), cleanup, etc.). _BACKENDS: dict[str, BottleBackend[Any, Any]] = { "docker": DockerBottleBackend(), + "firecracker": FirecrackerBottleBackend(), "macos-container": MacosContainerBottleBackend(), "smolmachines": SmolmachinesBottleBackend(), } @@ -585,6 +588,8 @@ def get_bottle_backend( def _default_backend_name() -> str: if has_backend("macos-container"): return "macos-container" + if has_backend("firecracker"): + return "firecracker" return "smolmachines" diff --git a/bot_bottle/backend/firecracker/__init__.py b/bot_bottle/backend/firecracker/__init__.py new file mode 100644 index 0000000..57b5840 --- /dev/null +++ b/bot_bottle/backend/firecracker/__init__.py @@ -0,0 +1,7 @@ +"""Firecracker backend: Linux KVM microVM isolation (issue #342).""" + +from __future__ import annotations + +from .backend import FirecrackerBottleBackend + +__all__ = ["FirecrackerBottleBackend"] diff --git a/bot_bottle/backend/firecracker/backend.py b/bot_bottle/backend/firecracker/backend.py new file mode 100644 index 0000000..8431dca --- /dev/null +++ b/bot_bottle/backend/firecracker/backend.py @@ -0,0 +1,89 @@ +"""FirecrackerBottleBackend — Linux microVM implementation. + +Replaces smolmachines on Linux (issue #342): mature KVM-based +isolation via Firecracker, SSH control over a point-to-point TAP, and a +fail-closed nftables egress boundary. Selected by +`BOT_BOTTLE_BACKEND=firecracker` or `--backend=firecracker`. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from pathlib import Path +from typing import Generator, Sequence + +from ...agent_provider import AgentProvisionPlan +from ...egress import EgressPlan +from ...env import ResolvedEnv +from ...git_gate import GitGatePlan +from ...manifest import Manifest +from ...supervise import SupervisePlan +from .. import ActiveAgent, BottleBackend, BottleSpec +from . import cleanup as _cleanup +from . import enumerate as _enumerate +from . import launch as _launch +from . import resolve_plan as _resolve_plan +from . import util as _util +from .bottle import FirecrackerBottle +from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan +from .bottle_plan import FirecrackerBottlePlan + + +class FirecrackerBottleBackend( + BottleBackend["FirecrackerBottlePlan", "FirecrackerBottleCleanupPlan"] +): + name = "firecracker" + + @classmethod + def is_available(cls) -> bool: + return _util.is_available() + + def _preflight(self) -> None: + _resolve_plan.preflight() + + def _build_guest_env(self, resolved_env: ResolvedEnv) -> dict[str, str]: + return _resolve_plan.build_guest_env(resolved_env) + + def _resolve_plan( + self, + spec: BottleSpec, + *, + manifest: Manifest, + slug: str, + resolved_env: ResolvedEnv, + agent_provision_plan: AgentProvisionPlan, + egress_plan: EgressPlan, + git_gate_plan: GitGatePlan, + supervise_plan: SupervisePlan | None, + stage_dir: Path, + ) -> FirecrackerBottlePlan: + return _resolve_plan.resolve_plan( + spec, + manifest=manifest, + slug=slug, + resolved_env=resolved_env, + agent_provision_plan=agent_provision_plan, + egress_plan=egress_plan, + supervise_plan=supervise_plan, + git_gate_plan=git_gate_plan, + stage_dir=stage_dir, + ) + + @contextmanager + def launch( + self, plan: FirecrackerBottlePlan + ) -> Generator[FirecrackerBottle, None, None]: + with _launch.launch(plan, provision=self.provision) as bottle: + yield bottle + + def prepare_cleanup(self) -> FirecrackerBottleCleanupPlan: + return _cleanup.prepare_cleanup() + + def cleanup(self, plan: FirecrackerBottleCleanupPlan) -> None: + _cleanup.cleanup(plan) + + def enumerate_active(self) -> Sequence[ActiveAgent]: + return _enumerate.enumerate_active() + + def supervise_mcp_url(self, plan: FirecrackerBottlePlan) -> str: + return plan.agent_supervise_url diff --git a/bot_bottle/backend/firecracker/bottle.py b/bot_bottle/backend/firecracker/bottle.py new file mode 100644 index 0000000..3805dc9 --- /dev/null +++ b/bot_bottle/backend/firecracker/bottle.py @@ -0,0 +1,170 @@ +"""Bottle handle for a Firecracker microVM, over SSH. + +Every operation routes through SSH to the guest (dropbear, listening on +the TAP link). `exec` pipes a script over stdin and captures output; +`cp_in` uses scp; `exec_agent` uses `ssh -t` for an interactive PTY +session. `ssh -t` forwards the host terminal's SIGWINCH to the remote +PTY natively, so — unlike the smolmachines backend — no resize bridge +is needed. + +Commands run as the image's `node` user via `runuser`, with HOME/USER/ +PATH and the bottle env (HTTPS_PROXY at the sidecar, CA paths, …) set +per-invocation through `env` (the VM itself just runs the init; it has +no baked-in process env like a `docker run` container would). +""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import sys +from pathlib import Path +from typing import Mapping, cast + +from ...agent_provider import PromptMode, prompt_args +from .. import Bottle, ExecResult +from ..terminal import exec_shell_script +from . import util + + +_HOME_FOR = {"node": "/home/node", "root": "/root"} +_DEFAULT_PATH_FOR = { + "node": ( + "/home/node/.local/bin:" + "/home/node/.codex/packages/standalone/current/bin:" + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ), + "root": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +} + + +def _env_assignments_for(user: str, env: Mapping[str, str]) -> list[str]: + home = _HOME_FOR.get(user, f"/home/{user}") + out = [f"HOME={home}", f"USER={user}"] + if "PATH" not in env: + out.append(f"PATH={_DEFAULT_PATH_FOR.get(user, _DEFAULT_PATH_FOR['root'])}") + for k, v in env.items(): + out.append(f"{k}={v}") + return out + + +class FirecrackerBottle(Bottle): + def __init__( + self, + name: str, + *, + private_key: Path, + guest_ip: str, + guest_env: Mapping[str, str] | None = None, + prompt_path_in_guest: str | None = None, + agent_command: str = "claude", + agent_prompt_mode: PromptMode = "append_file", + agent_provider_template: str = "claude", + terminal_title: str = "", + terminal_color: str = "", + agent_workdir: str = "/home/node", + ) -> None: + self.name = name + self._private_key = private_key + self._guest_ip = guest_ip + self._guest_env = dict(guest_env or {}) + self.prompt_path = prompt_path_in_guest + self._agent_prompt_mode = agent_prompt_mode + self.agent_command = agent_command + self.terminal_title = terminal_title + self.terminal_color = terminal_color + self.agent_provider_template = agent_provider_template + self.agent_workdir = agent_workdir + + def _ssh(self, *, tty: bool) -> list[str]: + argv = util.ssh_base_argv(self._private_key, self._guest_ip) + if tty: + # -t: allocate a remote PTY and forward window-size changes. + argv.insert(1, "-t") + return argv + + def _agent_remote_argv(self, argv: list[str]) -> list[str]: + """The `runuser … …` command run inside the guest.""" + full_argv = list(argv) + full_argv.extend( + prompt_args( + cast(PromptMode, self._agent_prompt_mode), + self.prompt_path, + argv=full_argv, + ) + ) + remote = ["runuser", "-u", "node", "--", + "env", *_env_assignments_for("node", self._guest_env)] + if self.agent_workdir and self.agent_workdir != _HOME_FOR["node"]: + remote += ["sh", "-lc", f"cd {self.agent_workdir} && exec \"$@\"", + "bot-bottle-agent"] + remote += [self.agent_command, *full_argv] + return remote + + def agent_argv(self, argv: list[str], *, tty: bool = True) -> list[str]: + return [*self._ssh(tty=tty), "--", *self._agent_remote_argv(argv)] + + def exec_agent(self, argv: list[str], *, tty: bool = True) -> int: + agent_argv = self.agent_argv(argv, tty=tty) + script = ( + exec_shell_script(agent_argv, self.terminal_title, self.terminal_color) + if tty else None + ) + if script is None: + return subprocess.run(agent_argv, check=False).returncode + return subprocess.run(["sh", "-lc", script], check=False).returncode + + def exec(self, script: str, *, user: str = "node") -> ExecResult: + # Pipe the script over stdin (`sh -s`) so nothing needs shell + # quoting through the SSH command line. + remote = ["runuser", "-u", user, "--", + "env", *_env_assignments_for(user, self._guest_env), + "/bin/sh", "-s"] + result = subprocess.run( + [*self._ssh(tty=False), "--", *remote], + input=script, capture_output=True, text=True, check=False, + ) + return ExecResult( + returncode=result.returncode, + stdout=result.stdout, + stderr=result.stderr, + ) + + def cp_in(self, host_path: str, container_path: str) -> None: + # Stream a tar over the SSH exec channel rather than scp: the + # guest runs dropbear, which ships no sftp-server/scp, but every + # agent image has `tar`. Copy so `container_path` becomes a copy + # of `host_path` (docker cp semantics — callers rm -rf the + # destination first). + host_parent = os.path.dirname(host_path.rstrip("/")) or "/" + host_base = os.path.basename(host_path.rstrip("/")) + guest_parent = os.path.dirname(container_path.rstrip("/")) or "/" + guest_base = os.path.basename(container_path.rstrip("/")) + remote = ( + f"mkdir -p {shlex.quote(guest_parent)} && " + f"cd {shlex.quote(guest_parent)} && " + f"rm -rf {shlex.quote(guest_base)} && tar -xf - && " + f"{{ [ {shlex.quote(host_base)} = {shlex.quote(guest_base)} ] || " + f"mv {shlex.quote(host_base)} {shlex.quote(guest_base)}; }}" + ) + tar = subprocess.Popen( + ["tar", "-C", host_parent, "-cf", "-", host_base], + stdout=subprocess.PIPE, + ) + ssh = subprocess.run( + [*self._ssh(tty=False), "--", "sh", "-c", remote], + stdin=tar.stdout, capture_output=True, text=True, check=False, + ) + tar.wait() + if tar.returncode != 0 or ssh.returncode != 0: + sys.stderr.write(ssh.stderr) + raise subprocess.CalledProcessError( + ssh.returncode or tar.returncode, + ["cp_in", host_path, container_path], + ) + + def close(self) -> None: + # Real teardown (VM terminate, TAP release) lives on the launch + # ExitStack; this is the idempotent alias the ABC expects. + pass diff --git a/bot_bottle/backend/firecracker/bottle_cleanup_plan.py b/bot_bottle/backend/firecracker/bottle_cleanup_plan.py new file mode 100644 index 0000000..3fd8eec --- /dev/null +++ b/bot_bottle/backend/firecracker/bottle_cleanup_plan.py @@ -0,0 +1,32 @@ +"""Cleanup plan for the Firecracker backend.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from ...log import info +from .. import BottleCleanupPlan + + +@dataclass(frozen=True) +class FirecrackerBottleCleanupPlan(BottleCleanupPlan): + # PIDs of orphaned firecracker VMM processes and the sidecar + # containers left behind by previous bottles. + vm_pids: tuple[int, ...] = () + containers: tuple[str, ...] = () + run_dirs: tuple[str, ...] = () + + def print(self) -> None: + if self.empty: + info("firecracker cleanup: nothing to remove") + return + for pid in self.vm_pids: + info(f"firecracker VM process: pid {pid}") + for name in self.containers: + info(f"firecracker sidecar container: {name}") + for path in self.run_dirs: + info(f"firecracker run dir: {path}") + + @property + def empty(self) -> bool: + return not (self.vm_pids or self.containers or self.run_dirs) diff --git a/bot_bottle/backend/firecracker/bottle_plan.py b/bot_bottle/backend/firecracker/bottle_plan.py new file mode 100644 index 0000000..91decae --- /dev/null +++ b/bot_bottle/backend/firecracker/bottle_plan.py @@ -0,0 +1,63 @@ +"""Plan type for the Firecracker backend.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from ...agent_provider import PromptMode +from .. import BottlePlan + + +@dataclass(frozen=True) +class FirecrackerBottlePlan(BottlePlan): + slug: str + forwarded_env: dict[str, str] = field(repr=False) + # Stamped by launch once the sidecar is up and its ports are + # published on the host-side TAP IP (empty at prepare time). + agent_proxy_url: str = "" + agent_git_gate_url: str = "" + agent_supervise_url: str = "" + + @property + def container_name(self) -> str: + """Instance name, reused for the sidecar container + VM run dir. + Matches the `bot-bottle-` convention the other backends + use so cleanup/enumerate discovery-by-prefix keeps working.""" + return self.agent_provision.instance_name + + @property + def image(self) -> str: + return self.agent_provision.image + + @property + def dockerfile_path(self) -> str: + return self.agent_provision.dockerfile + + @property + def prompt_file(self) -> Path: + return self.agent_provision.prompt_file + + @property + def agent_command(self) -> str: + return self.agent_provision.command + + @property + def agent_prompt_mode(self) -> PromptMode: + return self.agent_provision.prompt_mode + + @property + def agent_provider_template(self) -> str: + return self.agent_provision.template + + @property + def git_gate_insteadof_host(self) -> str: + if self.agent_git_gate_url.startswith("http://"): + return self.agent_git_gate_url.removeprefix("http://").rstrip("/") + return super().git_gate_insteadof_host + + @property + def git_gate_insteadof_scheme(self) -> str: + if self.agent_git_gate_url.startswith("http://"): + return "http" + return super().git_gate_insteadof_scheme diff --git a/bot_bottle/backend/firecracker/cleanup.py b/bot_bottle/backend/firecracker/cleanup.py new file mode 100644 index 0000000..938bca9 --- /dev/null +++ b/bot_bottle/backend/firecracker/cleanup.py @@ -0,0 +1,90 @@ +"""Cleanup for the Firecracker backend. + +Orphans are: firecracker VMM processes whose config lives under our run +dir, the `bot-bottle-sidecars-*` containers, and the per-bottle run +dirs. TAP slots free themselves (the flock drops when the launcher +exits), so there is nothing to reclaim there. +""" + +from __future__ import annotations + +import os +import shutil +import signal +import subprocess +from pathlib import Path + +from ...log import info +from . import util +from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan + +_SIDECAR_PREFIX = "bot-bottle-sidecars-" + + +def _run_root() -> Path: + return util.cache_dir() / "run" + + +def _orphan_vm_pids() -> list[int]: + """firecracker processes whose --config-file is under our run dir.""" + run_root = str(_run_root()) + result = subprocess.run( + ["pgrep", "-a", "firecracker"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + return [] + pids: list[int] = [] + for line in result.stdout.splitlines(): + parts = line.split(None, 1) + if len(parts) != 2 or run_root not in parts[1]: + continue + try: + pids.append(int(parts[0])) + except ValueError: + continue + return pids + + +def _sidecar_containers() -> list[str]: + result = subprocess.run( + ["docker", "ps", "-a", "--format", "{{.Names}}", + "--filter", f"name={_SIDECAR_PREFIX}"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + return [] + return sorted(n.strip() for n in result.stdout.splitlines() if n.strip()) + + +def _run_dirs() -> list[str]: + run_root = _run_root() + if not run_root.is_dir(): + return [] + return sorted(str(p) for p in run_root.iterdir() if p.is_dir()) + + +def prepare_cleanup() -> FirecrackerBottleCleanupPlan: + return FirecrackerBottleCleanupPlan( + vm_pids=tuple(_orphan_vm_pids()), + containers=tuple(_sidecar_containers()), + run_dirs=tuple(_run_dirs()), + ) + + +def cleanup(plan: FirecrackerBottleCleanupPlan) -> None: + for pid in plan.vm_pids: + info(f"kill firecracker VM pid {pid}") + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + for name in plan.containers: + info(f"docker rm -f {name}") + subprocess.run( + ["docker", "rm", "-f", name], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, + ) + for path in plan.run_dirs: + info(f"rm -rf {path}") + shutil.rmtree(path, ignore_errors=True) diff --git a/bot_bottle/backend/firecracker/enumerate.py b/bot_bottle/backend/firecracker/enumerate.py new file mode 100644 index 0000000..b4a7f42 --- /dev/null +++ b/bot_bottle/backend/firecracker/enumerate.py @@ -0,0 +1,43 @@ +"""Active-agent enumeration for the Firecracker backend. + +The agent runs in a VM (no container to list), so a live bottle is +identified by its running sidecar container `bot-bottle-sidecars-` +— the same discovery-by-prefix the other backends use. +""" + +from __future__ import annotations + +import subprocess + +from ...bottle_state import read_metadata +from .. import ActiveAgent + +_SIDECAR_PREFIX = "bot-bottle-sidecars-" + + +def enumerate_active() -> list[ActiveAgent]: + result = subprocess.run( + ["docker", "ps", "--format", "{{.Names}}", + "--filter", f"name={_SIDECAR_PREFIX}"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + return [] + out: list[ActiveAgent] = [] + for name in sorted(n.strip() for n in result.stdout.splitlines() if n.strip()): + slug = name[len(_SIDECAR_PREFIX):] + metadata = read_metadata(slug) + if metadata is None or metadata.backend != "firecracker": + # Skip sidecars owned by another backend (docker/smolmachines + # share the container-name prefix). + continue + out.append(ActiveAgent( + backend_name="firecracker", + slug=slug, + agent_name=metadata.agent_name, + started_at=metadata.started_at, + services=(), + label=metadata.label, + color=metadata.color, + )) + return out diff --git a/bot_bottle/backend/firecracker/firecracker_vm.py b/bot_bottle/backend/firecracker/firecracker_vm.py new file mode 100644 index 0000000..113065d --- /dev/null +++ b/bot_bottle/backend/firecracker/firecracker_vm.py @@ -0,0 +1,173 @@ +"""Firecracker microVM process lifecycle. + +A bottle VM is one `firecracker` process booted from a JSON config +(kernel + writable rootfs ext4 + one TAP network interface). We run it +`--no-api`: all configuration is in the config file, and teardown is a +process signal — the microVM dies with its VMM, which is exactly the +ephemeral-bottle semantics we want. No API socket, no runtime +reconfiguration. + +The guest gets its IP from the kernel `ip=` cmdline (kernel-level +autoconfig, no in-guest iproute2) and its SSH pubkey from a `bb_pubkey=` +cmdline arg the init decodes. +""" + +from __future__ import annotations + +import base64 +import json +import signal +import subprocess +import time +from dataclasses import dataclass +from pathlib import Path + +from ...log import die, info +from . import util + + +# Serial console off the critical path but captured to a log for +# debugging boot failures. `reboot=k panic=1 pci=off` are the standard +# Firecracker guest args; `i8042.*` skip the (absent) PS/2 probe to +# shave boot time. +_BASE_BOOT_ARGS = ( + "console=ttyS0 reboot=k panic=1 pci=off " + "i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd " + "root=/dev/vda rw init=/bb-init" +) + + +@dataclass +class VmHandle: + """A running microVM: its VMM process, guest IP, and console log.""" + + process: subprocess.Popen[bytes] + guest_ip: str + console_log: Path + + def is_alive(self) -> bool: + return self.process.poll() is None + + def terminate(self) -> None: + """Stop the VMM (and thus the guest). SIGTERM, then SIGKILL.""" + if self.process.poll() is not None: + return + self.process.send_signal(signal.SIGTERM) + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + + +def _boot_args(guest_ip: str, host_ip: str, pubkey: str) -> str: + # ip=::::::off — /31 point-to-point link, + # so netmask is 255.255.255.254 and the gateway is the host TAP IP. + ip_arg = f"ip={guest_ip}::{host_ip}:255.255.255.254::eth0:off" + pub_b64 = base64.b64encode(pubkey.encode()).decode() + return f"{_BASE_BOOT_ARGS} {ip_arg} bb_pubkey={pub_b64}" + + +def _config( + *, + rootfs: Path, + tap: str, + guest_ip: str, + host_ip: str, + pubkey: str, + vcpus: int, + mem_mib: int, + guest_mac: str, +) -> dict[str, object]: + return { + "boot-source": { + "kernel_image_path": str(util.kernel_path()), + "boot_args": _boot_args(guest_ip, host_ip, pubkey), + }, + "drives": [ + { + "drive_id": "rootfs", + "path_on_host": str(rootfs), + "is_root_device": True, + "is_read_only": False, + } + ], + "network-interfaces": [ + { + "iface_id": "eth0", + "host_dev_name": tap, + "guest_mac": guest_mac, + } + ], + "machine-config": { + "vcpu_count": vcpus, + "mem_size_mib": mem_mib, + }, + } + + +def boot( + *, + name: str, + rootfs: Path, + tap: str, + guest_ip: str, + host_ip: str, + pubkey: str, + run_dir: Path, + vcpus: int = 2, + mem_mib: int = 2048, + guest_mac: str = "06:00:AC:10:00:02", +) -> VmHandle: + """Write the config and launch the VMM. Returns once the process is + spawned; callers wait for SSH readiness separately.""" + run_dir.mkdir(parents=True, exist_ok=True) + config_path = run_dir / "config.json" + console_log = run_dir / "console.log" + config_path.write_text(json.dumps( + _config( + rootfs=rootfs, tap=tap, guest_ip=guest_ip, host_ip=host_ip, + pubkey=pubkey, vcpus=vcpus, mem_mib=mem_mib, guest_mac=guest_mac, + ), + indent=2, + )) + + info(f"booting microVM {name} on {tap} (guest {guest_ip})") + log_fh = console_log.open("wb") + process = subprocess.Popen( + ["firecracker", "--no-api", "--config-file", str(config_path)], + stdout=log_fh, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + ) + return VmHandle(process=process, guest_ip=guest_ip, console_log=console_log) + + +def wait_for_ssh( + vm: VmHandle, private_key: Path, *, timeout: float = 30.0, +) -> None: + """Poll SSH until the guest accepts a command or the deadline + passes. Dies (with the console tail) if the VMM exits early or the + guest never comes up — a boot failure must be loud, not a hang.""" + deadline = time.monotonic() + timeout + probe = util.ssh_base_argv(private_key, vm.guest_ip) + ["true"] + while time.monotonic() < deadline: + if not vm.is_alive(): + die(f"microVM exited during boot (rc={vm.process.returncode}).\n" + f"{_console_tail(vm.console_log)}") + result = subprocess.run( + probe, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + check=False, + ) + if result.returncode == 0: + return + time.sleep(0.5) + die(f"microVM {vm.guest_ip} did not accept SSH within {timeout:.0f}s.\n" + f"{_console_tail(vm.console_log)}") + + +def _console_tail(console_log: Path, lines: int = 25) -> str: + try: + text = console_log.read_text(errors="replace").splitlines() + except OSError: + return "(no console log)" + tail = "\n".join(text[-lines:]) + return f"--- guest console (last {lines} lines) ---\n{tail}" diff --git a/bot_bottle/backend/firecracker/freezer.py b/bot_bottle/backend/firecracker/freezer.py new file mode 100644 index 0000000..16c6370 --- /dev/null +++ b/bot_bottle/backend/firecracker/freezer.py @@ -0,0 +1,76 @@ +"""FirecrackerFreezer — snapshot a running microVM to a Docker image. + +The VM is live and can't be block-copied safely, so — like the macOS +backend — we stream the guest root filesystem out over the control +channel (SSH here) and rebuild an image from it. The bottle keeps +running after the snapshot. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +from pathlib import Path + +from ...log import die, info +from .. import ActiveAgent +from ..freeze import Freezer +from . import util + + +class FirecrackerFreezer(Freezer): + backend_name = "firecracker" + + def _freeze(self, agent: ActiveAgent) -> str: + run_dir = util.cache_dir() / "run" / agent.slug + private_key = run_dir / "bottle_id_ed25519" + guest_ip = _guest_ip_from_config(run_dir / "config.json") + if not private_key.is_file() or not guest_ip: + die(f"cannot freeze {agent.slug}: run dir {run_dir} is missing the " + f"SSH key or VM config (is the bottle still running?)") + image_tag = f"bot-bottle-committed-{agent.slug}:latest" + _commit_via_ssh(private_key, guest_ip, image_tag) + info(f"committed {agent.slug} -> {image_tag!r}") + return image_tag + + def _export_hint(self, slug: str, image_ref: str) -> None: + info(f"to export for migration: docker image save {image_ref} " + f"-o {slug}.tar") + + +def _guest_ip_from_config(config_path: Path) -> str: + try: + cfg = json.loads(config_path.read_text()) + except (OSError, json.JSONDecodeError): + return "" + boot_args = cfg.get("boot-source", {}).get("boot_args", "") + for token in boot_args.split(): + if token.startswith("ip="): + # ip=::::::off + return token[len("ip="):].split(":", 1)[0] + return "" + + +def _commit_via_ssh(private_key: Path, guest_ip: str, image_tag: str) -> None: + with tempfile.TemporaryDirectory(prefix="bot-bottle-fc-commit.") as tmp: + rootfs_tar = os.path.join(tmp, "rootfs.tar") + ssh = util.ssh_base_argv(private_key, guest_ip) + with open(rootfs_tar, "wb") as tar_out: + result = subprocess.run( + [*ssh, "--", "tar", "--create", "--one-file-system", + "--exclude=./proc", "--exclude=./sys", "--exclude=./dev", + "--exclude=./run", "--file=-", "--directory=/", "."], + stdout=tar_out, stderr=subprocess.PIPE, check=False, + ) + if result.returncode != 0: + die(f"ssh tar for {guest_ip} failed: " + f"{(result.stderr or b'').decode().strip() or ''}") + with open(os.path.join(tmp, "Dockerfile"), "w", encoding="utf-8") as f: + f.write("FROM scratch\nADD rootfs.tar /\nUSER node\nWORKDIR /home/node\n") + build = subprocess.run( + ["docker", "build", "-t", image_tag, tmp], check=False, + ) + if build.returncode != 0: + die(f"docker build for {image_tag!r} failed") diff --git a/bot_bottle/backend/firecracker/isolation_probe.py b/bot_bottle/backend/firecracker/isolation_probe.py new file mode 100644 index 0000000..233665d --- /dev/null +++ b/bot_bottle/backend/firecracker/isolation_probe.py @@ -0,0 +1,116 @@ +"""Empirical, post-boot isolation probe — the authoritative fail-closed +egress-boundary check. + +The pre-boot nft check can't be trusted on every host (listing an +nftables table typically needs root; the launcher runs unprivileged). +So before the agent ever runs, we prove the boundary directly: open a +canary TCP listener on a host IP the VM must NOT be able to reach (the +host's primary non-TAP address), then have the guest try to connect. If +the isolation rules are in force the connection is dropped and times +out; if it succeeds, the VM can reach host services it shouldn't — a +sandbox escape — and we refuse to continue. + +The listener is load-bearing: without it a "blocked" result could just +be connection-refused. With it, a reachable canary means a real leak +and a timeout means a real drop. +""" + +from __future__ import annotations + +import socket +import subprocess +import threading +from pathlib import Path + +from ...log import die, info, warn +from . import util + + +def _host_primary_ip() -> str: + """The source IP the host uses for off-box traffic (its LAN + address) — a canary the VM must not reach. Empty if the host has no + such route (isolated box).""" + result = subprocess.run( + ["ip", "-o", "route", "get", "1.1.1.1"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + return "" + tokens = result.stdout.split() + # `... src ...` — the address the host would use as source. + if "src" in tokens: + idx = tokens.index("src") + if idx + 1 < len(tokens): + return tokens[idx + 1] + return "" + + +# Guest-side connect test: exit 0 = reached the canary (LEAK), 1 = +# blocked (good), 2 = no usable tool (inconclusive -> fail-closed). +_GUEST_PROBE = r""" +h="$1"; p="$2" +if command -v python3 >/dev/null 2>&1; then + python3 - "$h" "$p" <<'PY' +import socket, sys +s = socket.socket(); s.settimeout(3) +sys.exit(0 if s.connect_ex((sys.argv[1], int(sys.argv[2]))) == 0 else 1) +PY +elif command -v bash >/dev/null 2>&1; then + timeout 3 bash -c "exec 3<>/dev/tcp/$h/$p" >/dev/null 2>&1 +elif command -v nc >/dev/null 2>&1; then + nc -w3 -z "$h" "$p" >/dev/null 2>&1 +else + exit 2 +fi +""" + + +def verify_isolation(private_key: Path, guest_ip: str) -> None: + """Prove the VM cannot reach the host's primary IP. Dies + (fail-closed) on a confirmed leak or if the guest has no tool to + run the test. Warns (does not fail) when the host has no canary + address to test against.""" + canary_ip = _host_primary_ip() + if not canary_ip: + warn("isolation probe skipped: host has no non-TAP route to test " + "against (isolated box).") + return + + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind((canary_ip, 0)) + listener.listen(1) + listener.settimeout(6) + canary_port = listener.getsockname()[1] + + accepted: list[bool] = [] + + def _accept() -> None: + try: + conn, _ = listener.accept() + accepted.append(True) + conn.close() + except OSError: + pass + + thread = threading.Thread(target=_accept, daemon=True) + thread.start() + + ssh = util.ssh_base_argv(private_key, guest_ip) + result = subprocess.run( + [*ssh, "--", "sh", "-s", "--", canary_ip, str(canary_port)], + input=_GUEST_PROBE, capture_output=True, text=True, check=False, + ) + thread.join(timeout=7) + listener.close() + + if result.returncode == 0 or accepted: + die(f"ISOLATION FAILURE: the VM reached the host canary " + f"{canary_ip}:{canary_port}. The egress boundary is not in " + f"force — refusing to run the agent (fail-closed). Verify the " + f"nft table with: ./cli.py firecracker setup") + if result.returncode == 2: + die("isolation probe inconclusive: the guest has no python3/bash/nc " + "to run the connectivity test. Refusing to continue " + "(fail-closed).") + info(f"isolation verified: VM cannot reach host {canary_ip}") diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py new file mode 100644 index 0000000..a0b0486 --- /dev/null +++ b/bot_bottle/backend/firecracker/launch.py @@ -0,0 +1,404 @@ +"""Launch flow for the Firecracker backend. + +Per bottle: + 1. mint the egress CA, build the agent image (docker), export it to a + cached ext4 rootfs; + 2. claim a free TAP pool slot (rootless flock); + 3. bring up the Docker sidecar bundle, publishing egress / git-gate / + supervise on the slot's host-side TAP IP at fixed ports; + 4. boot the microVM on that TAP; wait for SSH; + 5. provision (CA, prompt, skills, workspace, git, supervise) over SSH. + +Isolation is enforced by the operator-provisioned nft table (checked +fail-closed in preflight): a VM reaches only its sidecar (DNAT'd from +the host TAP IP) and nothing else. The agent's HTTPS_PROXY therefore +points at `http://:9099`, its only route to the world. +""" + +from __future__ import annotations + +import dataclasses +import os +import subprocess +from contextlib import ExitStack, contextmanager +from pathlib import Path +from typing import Callable, Generator + +from ...bottle_state import ( + egress_state_dir, + git_gate_state_dir, + read_committed_image, +) +from ...egress import ( + EGRESS_ROUTES_IN_CONTAINER, + egress_agent_env_entries, + egress_resolve_token_values, + egress_sidecar_env_entries, +) +from ...git_gate import ( + provision_git_gate_dynamic_keys, + revoke_git_gate_provisioned_keys, +) +from ...log import die, info, warn +from ...supervise import DB_PATH_IN_CONTAINER, SUPERVISE_PORT +from ...util import expand_tilde +from ..docker.egress import ( + EGRESS_CA_IN_CONTAINER, + EGRESS_PORT, + egress_tls_init, +) +from ..docker.git_gate import ( + GIT_GATE_ACCESS_HOOK_IN_CONTAINER, + GIT_GATE_CREDS_DIR_IN_CONTAINER, + GIT_GATE_ENTRYPOINT_IN_CONTAINER, + GIT_GATE_HOOK_IN_CONTAINER, +) +from ..docker.sidecar_bundle import ( + SIDECAR_BUNDLE_DOCKERFILE, + SIDECAR_BUNDLE_IMAGE, +) +from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH +from . import firecracker_vm, isolation_probe, netpool, util +from .bottle import FirecrackerBottle +from .bottle_plan import FirecrackerBottlePlan + + +_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent) +_GIT_HTTP_PORT = 9420 +_GIT_GATE_READY_FILE = "/run/git-gate/ready" + + +def sidecar_container_name(slug: str) -> str: + return f"bot-bottle-sidecars-{slug}" + + +@contextmanager +def launch( + plan: FirecrackerBottlePlan, + *, + provision: Callable[[FirecrackerBottlePlan, "FirecrackerBottle"], str | None], +) -> Generator[FirecrackerBottle, None, None]: + stack = ExitStack() + bottle_for_revoke = plan.manifest.bottle + git_gate_dir_for_revoke = git_gate_state_dir(plan.slug) + + def teardown() -> None: + teardown_exc: BaseException | None = None + try: + stack.close() + except BaseException as exc: # noqa: W0718 - teardown must continue + teardown_exc = exc + warn(f"firecracker teardown failed: {exc!r}") + revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) + if teardown_exc is not None: + raise teardown_exc + + try: + plan = _mint_certs(plan) + plan = _build_agent_image(plan) + + # Claim a TAP slot; the flock is held until teardown closes it. + slot, lock = netpool.allocate(plan.slug) + stack.callback(lock.close) + info(f"firecracker slot {slot.iface}: host={slot.host_ip} " + f"guest={slot.guest_ip}") + + plan = _provision_git_gate_keys(plan) + + sidecar_name = sidecar_container_name(plan.slug) + _force_remove_container(sidecar_name) + _start_sidecar_bundle(plan, sidecar_name, slot.host_ip) + stack.callback(_force_remove_container, sidecar_name) + _stage_git_gate(plan, sidecar_name) + + plan = _stamp_agent_urls(plan, slot.host_ip) + + # Build the per-bottle rootfs + SSH key, then boot. + base_dir = util.build_base_rootfs_dir(plan.image) + run_dir = util.cache_dir() / "run" / plan.slug + run_dir.mkdir(parents=True, exist_ok=True) + rootfs = run_dir / "rootfs.ext4" + util.build_rootfs_ext4(base_dir, rootfs) + private_key, pubkey = util.generate_keypair(run_dir) + + vm = firecracker_vm.boot( + name=plan.container_name, + rootfs=rootfs, + tap=slot.iface, + guest_ip=slot.guest_ip, + host_ip=slot.host_ip, + pubkey=pubkey, + run_dir=run_dir, + ) + stack.callback(vm.terminate) + firecracker_vm.wait_for_ssh(vm, private_key) + + # Authoritative fail-closed egress-boundary check, before the + # agent runs: prove the VM cannot reach the host directly. + isolation_probe.verify_isolation(private_key, slot.guest_ip) + + bottle = FirecrackerBottle( + plan.container_name, + private_key=private_key, + guest_ip=slot.guest_ip, + guest_env=_agent_guest_env(plan, slot.host_ip), + agent_command=plan.agent_command, + agent_prompt_mode=plan.agent_prompt_mode, + agent_provider_template=plan.agent_provider_template, + terminal_title=( + f"{plan.spec.label} ({plan.spec.agent_name})" + if plan.spec.label else plan.spec.agent_name + ), + terminal_color=plan.spec.color, + agent_workdir=plan.workspace_plan.workdir, + ) + bottle.prompt_path = provision(plan, bottle) + + yield bottle + finally: + teardown() + + +def _mint_certs(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan: + egress_ca_host, egress_ca_cert_only = egress_tls_init(egress_state_dir(plan.slug)) + egress_plan = dataclasses.replace( + plan.egress_plan, + mitmproxy_ca_host_path=egress_ca_host, + mitmproxy_ca_cert_only_host_path=egress_ca_cert_only, + ) + return dataclasses.replace(plan, egress_plan=egress_plan) + + +def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan: + _docker_build(SIDECAR_BUNDLE_IMAGE, _REPO_DIR, dockerfile=SIDECAR_BUNDLE_DOCKERFILE) + committed = read_committed_image(plan.slug) + if committed and _image_exists(committed): + info(f"using committed image {committed!r}") + return dataclasses.replace( + plan, + agent_provision=dataclasses.replace(plan.agent_provision, image=committed), + ) + _docker_build(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path) + return plan + + +def _provision_git_gate_keys(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan: + if not plan.git_gate_plan.upstreams: + return plan + git_gate_plan = provision_git_gate_dynamic_keys( + plan.manifest.bottle, plan.git_gate_plan, git_gate_state_dir(plan.slug), + ) + return dataclasses.replace(plan, git_gate_plan=git_gate_plan) + + +def _stamp_agent_urls( + plan: FirecrackerBottlePlan, host_ip: str, +) -> FirecrackerBottlePlan: + proxy_url = f"http://{host_ip}:{EGRESS_PORT}" + supervise_url = ( + f"http://{host_ip}:{SUPERVISE_PORT}/" if plan.supervise_plan is not None else "" + ) + git_gate_url = ( + f"http://{host_ip}:{_GIT_HTTP_PORT}" if plan.git_gate_plan.upstreams else "" + ) + return dataclasses.replace( + plan, + agent_proxy_url=proxy_url, + agent_git_gate_url=git_gate_url, + agent_supervise_url=supervise_url, + ) + + +# --- sidecar bundle (Docker) ---------------------------------------- + +def _start_sidecar_bundle( + plan: FirecrackerBottlePlan, sidecar_name: str, host_ip: str, +) -> None: + argv = ["docker", "run", "--name", sidecar_name, "--detach", "--rm", + "-e", f"BOT_BOTTLE_SIDECAR_DAEMONS={','.join(_sidecar_daemons(plan))}"] + for entry in _sidecar_env_entries(plan): + argv += ["-e", entry] + for host_path, container_path, read_only in _sidecar_mounts(plan): + argv += ["-v", f"{host_path}:{container_path}{':ro' if read_only else ''}"] + # Publish on the slot's host TAP IP at fixed ports — each bottle + # has a distinct host_ip, so fixed ports never collide, and the VM + # reaches them at a stable, well-known address (its only route out). + for port in _sidecar_ports(plan): + argv += ["-p", f"{host_ip}:{port}:{port}"] + argv.append(SIDECAR_BUNDLE_IMAGE) + + effective_env = {**dict(os.environ), **plan.agent_provision.provisioned_env} + token_values = egress_resolve_token_values( + plan.egress_plan.token_env_map, effective_env, + ) + env = {**os.environ, **token_values} + info(f"docker run sidecar bundle {sidecar_name} (published on {host_ip})") + result = subprocess.run(argv, capture_output=True, text=True, env=env, check=False) + if result.returncode != 0: + die(f"docker run for sidecar bundle {sidecar_name} failed: " + f"{(result.stderr or '').strip() or ''}") + + +def _sidecar_daemons(plan: FirecrackerBottlePlan) -> tuple[str, ...]: + daemons = ["egress"] + if plan.git_gate_plan.upstreams: + daemons += ["git-gate", "git-http"] + if plan.supervise_plan is not None: + daemons.append("supervise") + return tuple(daemons) + + +def _sidecar_ports(plan: FirecrackerBottlePlan) -> tuple[int, ...]: + ports = [EGRESS_PORT] + if plan.git_gate_plan.upstreams: + ports.append(_GIT_HTTP_PORT) + if plan.supervise_plan is not None: + ports.append(SUPERVISE_PORT) + return tuple(ports) + + +def _sidecar_env_entries(plan: FirecrackerBottlePlan) -> tuple[str, ...]: + env: list[str] = list(egress_sidecar_env_entries(plan.egress_plan)) + if plan.git_gate_plan.upstreams: + env.append(f"BOT_BOTTLE_GIT_GATE_READY_FILE={_GIT_GATE_READY_FILE}") + if plan.supervise_plan is not None: + env += [ + f"SUPERVISE_BOTTLE_SLUG={plan.slug}", + f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", + f"SUPERVISE_PORT={SUPERVISE_PORT}", + ] + return tuple(env) + + +def _sidecar_mounts( + plan: FirecrackerBottlePlan, +) -> tuple[tuple[str, str, bool], ...]: + mounts: list[tuple[str, str, bool]] = [] + ep = plan.egress_plan + mounts.append((str(ep.mitmproxy_ca_host_path.parent), + str(Path(EGRESS_CA_IN_CONTAINER).parent), False)) + if ep.routes: + mounts.append((str(ep.routes_path.parent), + str(Path(EGRESS_ROUTES_IN_CONTAINER).parent), True)) + sp = plan.supervise_plan + if sp is not None: + mounts.append((str(sp.db_path.parent), + str(Path(DB_PATH_IN_CONTAINER).parent), False)) + return tuple(mounts) + + +def _stage_git_gate(plan: FirecrackerBottlePlan, sidecar_name: str) -> None: + gp = plan.git_gate_plan + if not gp.upstreams: + return + _docker_exec(sidecar_name, [ + "mkdir", "-p", + str(Path(GIT_GATE_HOOK_IN_CONTAINER).parent), + GIT_GATE_CREDS_DIR_IN_CONTAINER, "/git", + str(Path(_GIT_GATE_READY_FILE).parent), + ]) + for host_path, container_path in _git_gate_files(plan): + _docker_cp(host_path, f"{sidecar_name}:{container_path}") + _docker_exec(sidecar_name, [ + "sh", "-c", + f"chmod 755 {GIT_GATE_ENTRYPOINT_IN_CONTAINER} " + f"{GIT_GATE_HOOK_IN_CONTAINER} {GIT_GATE_ACCESS_HOOK_IN_CONTAINER} && " + f"chmod 600 {GIT_GATE_CREDS_DIR_IN_CONTAINER}/* && " + f"touch {_GIT_GATE_READY_FILE}", + ]) + + +def _git_gate_files(plan: FirecrackerBottlePlan) -> tuple[tuple[str, str], ...]: + gp = plan.git_gate_plan + files: list[tuple[str, str]] = [ + (str(gp.entrypoint_script), GIT_GATE_ENTRYPOINT_IN_CONTAINER), + (str(gp.hook_script), GIT_GATE_HOOK_IN_CONTAINER), + (str(gp.access_hook_script), GIT_GATE_ACCESS_HOOK_IN_CONTAINER), + ] + for upstream in gp.upstreams: + files.append((expand_tilde(upstream.identity_file), + f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{upstream.name}-key")) + if upstream.known_hosts_file: + files.append((str(upstream.known_hosts_file), + f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{upstream.name}-known_hosts")) + return tuple(files) + + +# --- agent guest env ------------------------------------------------- + +def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str]: + """Env injected into every agent/exec call over SSH. The VM has no + baked process env (it just runs init), so the proxy/CA/git/supervise + wiring is applied per-invocation.""" + proxy_url = f"http://{host_ip}:{EGRESS_PORT}" + no_proxy = f"localhost,127.0.0.1,{host_ip}" + env: dict[str, str] = { + "HTTPS_PROXY": proxy_url, "HTTP_PROXY": proxy_url, + "https_proxy": proxy_url, "http_proxy": proxy_url, + "NO_PROXY": no_proxy, "no_proxy": no_proxy, + "NODE_EXTRA_CA_CERTS": AGENT_CA_PATH, + "SSL_CERT_FILE": AGENT_CA_BUNDLE, + "REQUESTS_CA_BUNDLE": AGENT_CA_BUNDLE, + } + if plan.agent_git_gate_url: + env["GIT_GATE_URL"] = plan.agent_git_gate_url + if plan.agent_supervise_url: + env["MCP_SUPERVISE_URL"] = plan.agent_supervise_url + for entry in egress_agent_env_entries(plan.egress_plan): + key, _, value = entry.partition("=") + env[key] = value + env.update(plan.agent_provision.guest_env) + # Forwarded (bare-name) env: resolve host values now, since the VM + # can't inherit them from a `docker run --env NAME`. + for name in plan.forwarded_env: + value = os.environ.get(name) + if value is not None: + env[name] = value + return env + + +# --- docker helpers -------------------------------------------------- + +def _docker_build(ref: str, context: str, *, dockerfile: str = "") -> None: + info(f"docker build {ref}") + args = ["docker", "build", "-t", ref] + if dockerfile: + if not os.path.isabs(dockerfile): + dockerfile = os.path.join(context, dockerfile) + args += ["-f", dockerfile] + args.append(context) + result = subprocess.run(args, check=False) + if result.returncode != 0: + die(f"docker build for {ref!r} failed") + + +def _image_exists(ref: str) -> bool: + return subprocess.run( + ["docker", "image", "inspect", ref], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, + ).returncode == 0 + + +def _force_remove_container(name: str) -> None: + subprocess.run( + ["docker", "rm", "-f", name], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, + ) + + +def _docker_exec(name: str, argv: list[str]) -> None: + result = subprocess.run( + ["docker", "exec", name, *argv], capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + die(f"docker exec in {name} failed: " + f"{(result.stderr or '').strip() or ''}") + + +def _docker_cp(host_path: str, dest: str) -> None: + result = subprocess.run( + ["docker", "cp", host_path, dest], capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + die(f"docker cp {host_path} -> {dest} failed: " + f"{(result.stderr or '').strip() or ''}") diff --git a/bot_bottle/backend/firecracker/netpool.py b/bot_bottle/backend/firecracker/netpool.py new file mode 100644 index 0000000..2d02417 --- /dev/null +++ b/bot_bottle/backend/firecracker/netpool.py @@ -0,0 +1,243 @@ +"""Firecracker network pool: constants, IP math, allocation, and the +config renderers (shell command + NixOS module) shown to operators. + +The Firecracker backend needs a privileged one-time network setup: +a pool of point-to-point TAP devices (owned by the invoking user, so +`./cli.py start` never needs root) and a dedicated nftables table that +isolates every VM. This module is the single source of truth for the +pool parameters — the shell script (`scripts/firecracker-netpool.sh`), +the printed NixOS snippet, and the backend's fail-closed preflight all +derive from the constants here so they can't drift. + +Topology (per slot i): + * TAP ``bbfc{i}`` — no shared bridge, so no docker0 / virbr0 / cni0 + / br-* collisions. + * a /31 host<->guest link: host = base + 2i (the gateway the VM + routes through), guest = base + 2i + 1 (the VM's address). + * isolation via ``table inet bot_bottle_fc``: a VM reaches only its + own sidecar (DNAT'd from the host TAP IP) and nothing else. + +The default IP block is RFC-6598 shared address space (100.64.0.0/10), +which is purpose-built to avoid colliding with LAN/VPN private ranges. +""" + +from __future__ import annotations + +import fcntl +import ipaddress +import os +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import IO + +from ...log import die + + +# Interface names are capped at 15 chars (IFNAMSIZ-1); "bbfc" + a small +# index stays well under that and is distinctive enough to grep for. +IFACE_PREFIX = os.environ.get("BOT_BOTTLE_FC_IFACE_PREFIX", "bbfc") +NFT_TABLE = "bot_bottle_fc" + + +def pool_size() -> int: + return int(os.environ.get("BOT_BOTTLE_FC_POOL_SIZE", "8")) + + +def ip_base() -> str: + return os.environ.get("BOT_BOTTLE_FC_IP_BASE", "100.64.0.0") + + +# Sidecar ports the VM reaches at its host-side TAP IP. Kept in sync +# with the backend constants (egress 9099, supervise 9100, git-http +# 9420); rendered into the setup output for operator visibility. +SIDECAR_PORTS = (9099, 9100, 9420) + + +@dataclass(frozen=True) +class Slot: + """One pool slot: a TAP device and its host/guest /31 addresses.""" + + index: int + iface: str + host_ip: str + guest_ip: str + + @property + def guest_cidr(self) -> str: + """Guest address with the /31 prefix, for the kernel `ip=` arg.""" + return f"{self.guest_ip}/31" + + +def slot(index: int) -> Slot: + base = int(ipaddress.IPv4Address(ip_base())) + return Slot( + index=index, + iface=f"{IFACE_PREFIX}{index}", + host_ip=str(ipaddress.IPv4Address(base + 2 * index)), + guest_ip=str(ipaddress.IPv4Address(base + 2 * index + 1)), + ) + + +def all_slots() -> list[Slot]: + return [slot(i) for i in range(pool_size())] + + +# --- fail-closed verification --------------------------------------- + +def _run_ok(argv: list[str]) -> bool: + """Run a probe command, treating a missing binary as failure + (rather than crashing) so callers can stay fail-closed.""" + try: + return subprocess.run( + argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + except FileNotFoundError: + return False + + +def nft_table_present() -> bool: + """True iff the isolation table exists in the active nftables + backend. The backend's preflight treats absence as fatal — the VM + must not boot without its egress boundary in place. + + Listing may require root; when it can't be confirmed here the + launch path falls back to an empirical post-boot isolation probe. + Returns False (fail-closed) if `nft` is unavailable.""" + return _run_ok(["nft", "list", "table", "inet", NFT_TABLE]) + + +def tap_present(iface: str) -> bool: + # `ip link show` is unprivileged, so the TAP-pool check is reliable + # for the non-root launcher. + return _run_ok(["ip", "link", "show", iface]) + + +def missing_taps() -> list[str]: + """Pool TAPs that the one-time setup has not created yet.""" + return [s.iface for s in all_slots() if not tap_present(s.iface)] + + +# --- allocation ------------------------------------------------------ + +def _lock_dir() -> Path: + d = Path.home() / ".cache" / "bot-bottle" / "firecracker" / "pool" + d.mkdir(parents=True, exist_ok=True) + return d + + +def allocate(slug: str) -> tuple[Slot, IO[str]]: + """Claim a free pool slot for one bottle. Returns the slot and the + held lock file — the caller keeps it open for the VM's lifetime and + closes it on teardown; the flock auto-releases if the launcher + crashes, so a slot is never leaked. Dies when the pool is + exhausted. + + A per-slot `flock` (rather than inspecting running processes) makes + allocation race-free across concurrent launches without a central + registry: the first launcher to grab the lock owns the slot.""" + del slug # logged by the caller; allocation is purely lock-driven + for s in all_slots(): + lock_path = _lock_dir() / f"{s.iface}.lock" + handle = open(lock_path, "w", encoding="utf-8") + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + handle.close() + continue + return s, handle + die(f"Firecracker TAP pool exhausted ({pool_size()} slots, all in " + f"use). Stop a running bottle or raise BOT_BOTTLE_FC_POOL_SIZE " + f"and re-run `./cli.py firecracker setup`.") + raise AssertionError("unreachable") + + +# --- config renderers (shown by `./cli.py firecracker setup`) -------- + +def render_shell_setup() -> str: + """The imperative command for non-NixOS hosts.""" + env = _nondefault_env() + prefix = f"{env} " if env else "" + return f"sudo {prefix}./scripts/firecracker-netpool.sh up" + + +def render_nixos_module() -> str: + """A paste-ready NixOS snippet, derived from the live constants. + + Uses systemd-networkd tap netdevs (so TAPs are only reconfigured + when this config changes, not torn down on every rebuild — which + would drop running VMs) and `networking.nftables.tables.` + (an independent table that coexists with Docker's iptables rules + rather than replacing the whole ruleset).""" + slots = all_slots() + owner = os.environ.get("USER", "youruser") + + netdevs = "\n".join( + f''' "10-{s.iface}" = {{ + netdevConfig = {{ Name = "{s.iface}"; Kind = "tap"; }}; + tapConfig = {{ User = "{owner}"; }}; + }};''' + for s in slots + ) + networks = "\n".join( + f''' "10-{s.iface}" = {{ + matchConfig.Name = "{s.iface}"; + address = [ "{s.host_ip}/31" ]; + networkConfig.ConfigureWithoutCarrier = true; + }};''' + for s in slots + ) + + return f"""# bot-bottle Firecracker network pool ({len(slots)} slots, base {ip_base()}). +# Generated by `./cli.py firecracker setup`; owner user = {owner!r}. +{{ ... }}: +{{ + boot.kernel.sysctl."net.ipv4.ip_forward" = 1; + + systemd.network.enable = true; + systemd.network.netdevs = {{ +{netdevs} + }}; + systemd.network.networks = {{ +{networks} + }}; + + # Independent table — does not touch Docker/ufw/firewalld rules. + # A bottle VM (traffic from a bbfc* TAP) may reach only its own + # sidecar (DNAT'd from the host TAP IP); everything else is dropped, + # so there is no egress except through the sidecar proxy. + networking.nftables.enable = true; + networking.nftables.tables."{NFT_TABLE}" = {{ + family = "inet"; + content = '' + chain forward {{ + type filter hook forward priority -10; policy accept; + iifname != "{IFACE_PREFIX}*" return + ct state established,related accept + ct status dnat accept + drop + }} + chain input {{ + type filter hook input priority -10; policy accept; + iifname != "{IFACE_PREFIX}*" return + ct state established,related accept + drop + }} + ''; + }}; +}} +""" + + +def _nondefault_env() -> str: + """Render any non-default pool env overrides so the printed shell + command reproduces the operator's current settings.""" + pairs = [] + if os.environ.get("BOT_BOTTLE_FC_POOL_SIZE"): + pairs.append(f"BOT_BOTTLE_FC_POOL_SIZE={pool_size()}") + if os.environ.get("BOT_BOTTLE_FC_IP_BASE"): + pairs.append(f"BOT_BOTTLE_FC_IP_BASE={ip_base()}") + if os.environ.get("BOT_BOTTLE_FC_IFACE_PREFIX"): + pairs.append(f"BOT_BOTTLE_FC_IFACE_PREFIX={IFACE_PREFIX}") + return " ".join(pairs) diff --git a/bot_bottle/backend/firecracker/resolve_plan.py b/bot_bottle/backend/firecracker/resolve_plan.py new file mode 100644 index 0000000..6292175 --- /dev/null +++ b/bot_bottle/backend/firecracker/resolve_plan.py @@ -0,0 +1,47 @@ +"""Prepare step for the Firecracker backend.""" + +from __future__ import annotations + +from pathlib import Path + +from ...agent_provider import AgentProvisionPlan +from ...egress import EgressPlan +from ...env import ResolvedEnv +from ...git_gate import GitGatePlan +from ...manifest import Manifest +from ...supervise import SupervisePlan +from .. import BottleSpec +from . import util +from .bottle_plan import FirecrackerBottlePlan + + +def preflight() -> None: + util.require_firecracker() + + +def build_guest_env(resolved_env: ResolvedEnv) -> dict[str, str]: + return dict(resolved_env.literals) + + +def resolve_plan( + spec: BottleSpec, + manifest: Manifest, + slug: str, + resolved_env: ResolvedEnv, + agent_provision_plan: AgentProvisionPlan, + egress_plan: EgressPlan, + supervise_plan: SupervisePlan | None, + git_gate_plan: GitGatePlan, + stage_dir: Path, +) -> FirecrackerBottlePlan: + return FirecrackerBottlePlan( + spec=spec, + manifest=manifest, + stage_dir=stage_dir, + slug=slug, + forwarded_env=dict(resolved_env.forwarded), + git_gate_plan=git_gate_plan, + egress_plan=egress_plan, + supervise_plan=supervise_plan, + agent_provision=agent_provision_plan, + ) diff --git a/bot_bottle/backend/firecracker/util.py b/bot_bottle/backend/firecracker/util.py new file mode 100644 index 0000000..1b72dd6 --- /dev/null +++ b/bot_bottle/backend/firecracker/util.py @@ -0,0 +1,296 @@ +"""Host-side primitives for the Firecracker backend. + +Covers the pieces that don't need root at launch time: locating the +firecracker binary / guest kernel / injected dropbear, the fail-closed +preflight (KVM + kernel + isolation table + TAP pool must all be +present before a VM boots), the rootless rootfs pipeline +(`docker export` -> `mke2fs -d`, no mount), and per-bottle SSH key +generation. + +The privileged network setup (TAP pool + nft table) is a one-time +operator step — see `netpool.py`, `scripts/firecracker-netpool.sh`, +and `./cli.py firecracker setup`. +""" + +from __future__ import annotations + +import os +import platform +import shutil +import subprocess +from pathlib import Path + +from ...log import die, info, warn +from . import netpool + + +# Guest agent images are Debian-family with USER node; the VM is +# reached over SSH as root (init drops the pubkey into both root's and +# node's authorized_keys) and commands `runuser` down to node. +GUEST_SSH_USER = "root" + +# `/dev/kvm` must exist and be openable by the invoking user. +_KVM_DEVICE = "/dev/kvm" + + +def cache_dir() -> Path: + d = Path( + os.environ.get( + "BOT_BOTTLE_FC_CACHE", + str(Path.home() / ".cache" / "bot-bottle" / "firecracker"), + ) + ) + d.mkdir(parents=True, exist_ok=True) + return d + + +def kernel_path() -> Path: + return Path(os.environ.get("BOT_BOTTLE_FC_KERNEL", str(cache_dir() / "vmlinux"))) + + +def dropbear_path() -> Path: + """The static dropbear injected into every guest rootfs as its SSH + server. Must be statically linked — the guest has none of the + host's shared libraries.""" + return Path( + os.environ.get("BOT_BOTTLE_FC_DROPBEAR", str(cache_dir() / "dropbear")) + ) + + +# --- availability + preflight --------------------------------------- + +def is_linux() -> bool: + return platform.system() == "Linux" + + +def is_available() -> bool: + """Cheap capability probe used by cross-backend enumeration — + firecracker on PATH, on Linux, with KVM. Does not check the + (operator-provisioned) kernel / pool, so an available-but-unset + host still shows up and gets an actionable error at launch.""" + return ( + is_linux() + and shutil.which("firecracker") is not None + and os.path.exists(_KVM_DEVICE) + ) + + +def require_firecracker() -> None: + """Fail-closed preflight. Every check that gates the security + boundary (the isolation table, the TAP pool) errors rather than + booting a VM without it.""" + if not is_linux(): + die("firecracker backend is only supported on Linux (KVM). " + "On macOS use --backend=macos-container.") + if shutil.which("firecracker") is None: + info("Install: https://github.com/firecracker-microvm/firecracker/releases") + die("firecracker not found on PATH") + _require_kvm() + if not kernel_path().is_file(): + die(f"guest kernel not found at {kernel_path()}. Set " + "BOT_BOTTLE_FC_KERNEL or place a vmlinux there.") + if not dropbear_path().is_file(): + die(f"static dropbear not found at {dropbear_path()}. Set " + "BOT_BOTTLE_FC_DROPBEAR or cache one there.") + if shutil.which("mke2fs") is None: + die("mke2fs (e2fsprogs) not found — required to build guest rootfs") + _require_network_pool() + + +def _require_kvm() -> None: + if not os.path.exists(_KVM_DEVICE): + die(f"{_KVM_DEVICE} is missing. Enable KVM (load kvm-intel/kvm-amd; " + "confirm virtualization is on in firmware).") + if not os.access(_KVM_DEVICE, os.R_OK | os.W_OK): + die(f"{_KVM_DEVICE} exists but is not accessible. Add your user to " + "the `kvm` group and re-login.") + + +def _require_network_pool() -> None: + """Fail-closed on the parts we can check unprivileged; defer the + rest to the post-boot isolation probe. + + The TAP pool is verified here (`ip link show` is unprivileged). The + nft table can only be *confirmed present* here when `nft` is + queryable — which usually needs root — so a missing/absent nft is + not treated as fatal at this stage: the authoritative check is the + empirical isolation probe run after boot, before the agent starts + (see isolation_probe.verify_isolation). What we must never do is + boot without the TAP pool.""" + missing = netpool.missing_taps() + if missing: + die(f"network pool incomplete — missing TAP devices: " + f"{', '.join(missing)}.\n ./cli.py firecracker setup") + if shutil.which("nft") is not None and not netpool.nft_table_present(): + # nft is queryable and says the table is absent — that's a + # definite, catchable misconfiguration; fail early. + warn(f"isolation table `inet {netpool.NFT_TABLE}` not found via nft. " + "If this is a permissions issue it will be re-checked " + "empirically after boot; otherwise run: " + "./cli.py firecracker setup") + + +# --- rootfs pipeline (rootless) ------------------------------------- + +def docker_image_id(ref: str) -> str: + """The image's content digest, used as the rootfs cache key.""" + result = subprocess.run( + ["docker", "image", "inspect", "--format", "{{.Id}}", ref], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0 or not result.stdout.strip(): + die(f"docker image inspect for {ref!r} failed: " + f"{(result.stderr or '').strip() or ''}") + return result.stdout.strip().replace("sha256:", "")[:16] + + +def build_base_rootfs_dir(image_ref: str) -> Path: + """Export the agent image's filesystem and inject the guest init + + static dropbear. Cached by image digest — the per-bottle bits + (authorized_keys, IP) are passed at boot via the kernel cmdline, so + this tree carries nothing bottle-specific and is safely shared. + + Returns the prepared directory (read as the `mke2fs -d` source).""" + digest = docker_image_id(image_ref) + base = cache_dir() / "rootfs" / digest + ready = base / ".bb-ready" + if ready.is_file(): + return base + + if base.exists(): + shutil.rmtree(base, ignore_errors=True) + base.mkdir(parents=True) + + info(f"exporting {image_ref} rootfs -> {base}") + cid = subprocess.run( + ["docker", "create", image_ref, "sleep", "infinity"], + capture_output=True, text=True, check=False, + ) + if cid.returncode != 0 or not cid.stdout.strip(): + die(f"docker create {image_ref!r} failed: {cid.stderr.strip()}") + container = cid.stdout.strip() + try: + export = subprocess.Popen( + ["docker", "export", container], stdout=subprocess.PIPE, + ) + untar = subprocess.run( + ["tar", "-x", "-C", str(base)], stdin=export.stdout, check=False, + ) + export.wait() + if export.returncode != 0 or untar.returncode != 0: + die(f"exporting rootfs for {image_ref!r} failed") + finally: + subprocess.run( + ["docker", "rm", "-f", container], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, + ) + + _inject_guest_boot(base) + ready.write_text("ok\n") + return base + + +def _inject_guest_boot(rootfs: Path) -> None: + """Drop the static dropbear and the PID-1 init into the rootfs.""" + shutil.copy2(dropbear_path(), rootfs / "bb-dropbear") + os.chmod(rootfs / "bb-dropbear", 0o755) + init = rootfs / "bb-init" + init.write_text(_GUEST_INIT) + os.chmod(init, 0o755) + + +def build_rootfs_ext4(base_dir: Path, out_path: Path, *, slack_mib: int = 1024) -> None: + """Build a fresh, writable ext4 for one bottle from the cached base + dir. Rootless: `mke2fs -d` populates the image from a directory + without mounting. Each call produces an independent disk, so the + shared base dir stays untouched and concurrent bottles don't race.""" + used_mib = _dir_size_mib(base_dir) + size_mib = used_mib + slack_mib + out_path.parent.mkdir(parents=True, exist_ok=True) + result = subprocess.run( + ["mke2fs", "-q", "-t", "ext4", "-d", str(base_dir), "-F", + str(out_path), f"{size_mib}M"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + die(f"mke2fs for {out_path} failed: {result.stderr.strip()}") + + +def _dir_size_mib(path: Path) -> int: + result = subprocess.run( + ["du", "-sm", str(path)], capture_output=True, text=True, check=False, + ) + try: + return int(result.stdout.split()[0]) + except (ValueError, IndexError): + return 2048 + + +# --- per-bottle SSH keys -------------------------------------------- + +def generate_keypair(dest_dir: Path) -> tuple[Path, str]: + """Mint a per-bottle ed25519 keypair. Returns (private_key_path, + public_key_line). The public line is injected into the guest via + the boot cmdline; the private key authenticates host->guest SSH.""" + dest_dir.mkdir(parents=True, exist_ok=True) + key = dest_dir / "bottle_id_ed25519" + if key.exists(): + key.unlink() + (dest_dir / "bottle_id_ed25519.pub").unlink(missing_ok=True) + subprocess.run( + ["ssh-keygen", "-t", "ed25519", "-N", "", "-q", "-f", str(key), + "-C", "bot-bottle-firecracker"], + check=True, + ) + pub = (dest_dir / "bottle_id_ed25519.pub").read_text().strip() + return key, pub + + +def ssh_base_argv(private_key: Path, guest_ip: str) -> list[str]: + """Common SSH options for host->guest control. The VM is ephemeral + and per-bottle, so host-key TOFU is meaningless — pin no known_hosts + and skip the check rather than accumulate churn.""" + return [ + "ssh", + "-i", str(private_key), + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=5", + f"{GUEST_SSH_USER}@{guest_ip}", + ] + + +# PID-1 init injected into every guest. Kept dependency-light: relies +# only on coreutils + a POSIX shell (present in the Debian-family agent +# images). The kernel `ip=` cmdline configures eth0 before init runs, +# so no iproute2 is needed. The per-bottle SSH pubkey arrives base64 on +# the cmdline; dropbear generates ephemeral host keys with -R. +_GUEST_INIT = r"""#!/bin/sh +# bot-bottle Firecracker guest init (PID 1). +mount -t proc proc /proc 2>/dev/null +mount -t sysfs sys /sys 2>/dev/null +mount -t devtmpfs dev /dev 2>/dev/null +mkdir -p /dev/pts && mount -t devpts devpts /dev/pts 2>/dev/null +mount -o remount,rw / 2>/dev/null + +# Install the per-bottle SSH pubkey from the kernel cmdline. +KEY=$(sed -n 's/.*bb_pubkey=\([^ ]*\).*/\1/p' /proc/cmdline | base64 -d 2>/dev/null) +if [ -n "$KEY" ]; then + for home in /root /home/node; do + mkdir -p "$home/.ssh" + printf '%s\n' "$KEY" > "$home/.ssh/authorized_keys" + chmod 700 "$home/.ssh" + chmod 600 "$home/.ssh/authorized_keys" + done + chown -R node:node /home/node/.ssh 2>/dev/null || true +fi + +mkdir -p /etc/dropbear /run +# -R: generate host keys on demand. -E: log to stderr (guest console). +/bb-dropbear -R -E -p 22 2>/dev/null & + +# Reap zombies as PID 1. dropbear is always a child, so `wait` blocks +# rather than busy-looping. +while : ; do wait ; done +""" diff --git a/bot_bottle/backend/freeze.py b/bot_bottle/backend/freeze.py index d9a8013..90b394c 100644 --- a/bot_bottle/backend/freeze.py +++ b/bot_bottle/backend/freeze.py @@ -90,11 +90,15 @@ def get_freezer(backend_name: str) -> Freezer: if resolved == "macos-container": from .macos_container.freezer import MacosContainerFreezer return MacosContainerFreezer() + if resolved == "firecracker": + from .firecracker.freezer import FirecrackerFreezer + return FirecrackerFreezer() if resolved == "smolmachines": from .smolmachines.freezer import SmolmachinesFreezer return SmolmachinesFreezer() die( - f"commit is only supported for docker, macos-container, and " - f"smolmachines; backend {backend_name!r} has no freezer" + f"commit is only supported for docker, macos-container, " + f"firecracker, and smolmachines; backend {backend_name!r} has " + f"no freezer" ) raise AssertionError("unreachable") diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index dfc9cb0..6c6db94 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -16,6 +16,7 @@ from . import list as _list_mod from .cleanup import cmd_cleanup from .commit import cmd_commit from .edit import cmd_edit +from .firecracker import cmd_firecracker from .info import cmd_info from .init import cmd_init from .resume import cmd_resume @@ -28,6 +29,7 @@ COMMANDS = { "cleanup": cmd_cleanup, "commit": cmd_commit, "edit": cmd_edit, + "firecracker": cmd_firecracker, "info": cmd_info, "init": cmd_init, "list": cmd_list, @@ -43,6 +45,7 @@ def usage() -> None: sys.stderr.write(" cleanup stop and remove all active bot-bottle containers\n") sys.stderr.write(" commit snapshot a running bottle's container state to a Docker image\n") sys.stderr.write(" edit open an agent in vim for editing\n") + sys.stderr.write(" firecracker one-time network setup for the Firecracker backend\n") sys.stderr.write(" info print env, skills, and prompt details for a named agent\n") sys.stderr.write(" init interactively create a new agent and add it to bot-bottle.json\n") sys.stderr.write(" list list available agents or active containers\n") diff --git a/bot_bottle/cli/firecracker.py b/bot_bottle/cli/firecracker.py new file mode 100644 index 0000000..3d37736 --- /dev/null +++ b/bot_bottle/cli/firecracker.py @@ -0,0 +1,79 @@ +"""`firecracker` CLI command — one-time network setup helper. + +`./cli.py firecracker setup` prints the host-appropriate config for the +privileged, one-time network pool (TAP devices + isolation nftables +table) the Firecracker backend needs. On NixOS it prints a declarative +module to paste into your config (imperative rules don't survive +nixos-rebuild); elsewhere it prints the sudo command for the bundled +setup script. `./cli.py firecracker status` reports what's present. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from ..backend.firecracker import netpool + + +def _is_nixos() -> bool: + if Path("/etc/NIXOS").exists(): + return True + try: + return "ID=nixos" in Path("/etc/os-release").read_text() + except OSError: + return False + + +def _cmd_setup() -> int: + slots = netpool.all_slots() + sys.stderr.write( + f"Firecracker network pool: {len(slots)} slots " + f"({slots[0].iface}..{slots[-1].iface}), base {netpool.ip_base()}.\n" + f"This is a one-time privileged setup (needs root once).\n\n" + ) + if _is_nixos(): + sys.stderr.write( + "Detected NixOS. Paste this module into your configuration and " + "`nixos-rebuild switch` (imperative rules would not survive a " + "rebuild):\n\n" + ) + sys.stdout.write(netpool.render_nixos_module()) + else: + sys.stderr.write("Run the one-time setup as root:\n\n") + sys.stdout.write(netpool.render_shell_setup() + "\n") + sys.stderr.write( + "\n(On NixOS, use the declarative module instead — this host " + "was not detected as NixOS.)\n" + ) + return 0 + + +def _cmd_status() -> int: + ok = True + if netpool.nft_table_present(): + sys.stderr.write(f"nft table inet {netpool.NFT_TABLE}: present\n") + else: + sys.stderr.write(f"nft table inet {netpool.NFT_TABLE}: MISSING\n") + ok = False + missing = netpool.missing_taps() + total = netpool.pool_size() + if missing: + sys.stderr.write(f"TAP pool: {total - len(missing)}/{total} present " + f"(missing: {', '.join(missing)})\n") + ok = False + else: + sys.stderr.write(f"TAP pool: {total}/{total} present\n") + if not ok: + sys.stderr.write("\nRun: ./cli.py firecracker setup\n") + return 0 if ok else 1 + + +def cmd_firecracker(args: list[str]) -> int: + sub = args[0] if args else "" + if sub == "setup": + return _cmd_setup() + if sub == "status": + return _cmd_status() + sys.stderr.write("usage: firecracker {setup|status}\n") + return 2 diff --git a/scripts/firecracker-netpool.sh b/scripts/firecracker-netpool.sh new file mode 100755 index 0000000..5bd394e --- /dev/null +++ b/scripts/firecracker-netpool.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# One-time privileged network setup for the Firecracker backend. +# +# Creates a pool of point-to-point TAP devices (owned by the invoking +# user so the backend can open them without root at launch) and a +# dedicated nftables table that isolates every VM: a bottle VM can +# reach only its own sidecar (published on the host-side TAP IP) and +# nothing else on the host or network. +# +# Why a pool + one-time setup: creating a TAP and assigning it an IP +# needs CAP_NET_ADMIN. Pre-creating user-owned, pre-addressed TAPs +# means `./cli.py start` never needs root. The nft table is static +# (keyed on the `bbfc*` interface wildcard), so it covers every slot +# without per-launch changes. +# +# Design notes: +# * No shared bridge — each slot is an isolated /31 host<->guest link, +# so there are no bridge name/subnet collisions with docker0, +# virbr0 (libvirt), cni0 (k8s) or br-* (docker networks). +# * Own nftables table `bot_bottle_fc` — independent of the iptables +# filter/nat tables Docker/ufw/firewalld use, so nothing is stomped. +# * Default IP block is RFC-6598 shared address space (100.64.0.0/10), +# purpose-built to not collide with LAN/VPN private ranges. +# +# NixOS: imperative rules here do NOT survive nixos-rebuild. Use the +# declarative module in docs/firecracker/nixos-netpool.nix instead. +# +# Usage: +# sudo ./scripts/firecracker-netpool.sh up +# sudo ./scripts/firecracker-netpool.sh down +# ./scripts/firecracker-netpool.sh status +# +# Env overrides (must match the backend's util.py constants): +# BOT_BOTTLE_FC_POOL_SIZE number of slots (default 8) +# BOT_BOTTLE_FC_IP_BASE base IPv4 of the /31 pool (default 100.64.0.0) +# BOT_BOTTLE_FC_IFACE_PREFIX TAP name prefix (default bbfc) +# BOT_BOTTLE_FC_OWNER owning user (default $SUDO_USER or $USER) + +set -euo pipefail + +POOL_SIZE="${BOT_BOTTLE_FC_POOL_SIZE:-8}" +IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-100.64.0.0}" +PREFIX="${BOT_BOTTLE_FC_IFACE_PREFIX:-bbfc}" +OWNER="${BOT_BOTTLE_FC_OWNER:-${SUDO_USER:-$USER}}" +TABLE="bot_bottle_fc" + +# Sidecar ports (must match the backend). egress=9099, supervise=9100, +# git-http=9420. Reached by the VM at its host-side TAP IP. +SIDECAR_PORTS="9099,9100,9420" + +# --- IP math --------------------------------------------------------- +# Slot i occupies the /31 {base+2i, base+2i+1}: host = base+2i (the +# gateway the VM routes through), guest = base+2i+1 (the VM's address). +_ip_to_int() { + local IFS=. ; read -r a b c d <<<"$1" ; echo $(( (a<<24) + (b<<16) + (c<<8) + d )) +} +_int_to_ip() { + local n=$1 ; echo "$(( (n>>24)&255 )).$(( (n>>16)&255 )).$(( (n>>8)&255 )).$(( n&255 ))" +} +host_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 )); } +guest_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 + 1 )); } +iface() { echo "${PREFIX}$1"; } + +require_root() { + if [ "$(id -u)" -ne 0 ]; then + echo "error: '$1' needs root (run under sudo)" >&2 + exit 1 + fi +} + +cmd_up() { + require_root up + echo "firecracker net pool: $POOL_SIZE slots, base $IP_BASE, owner $OWNER" + + # VM->sidecar traffic is DNAT'd to the sidecar container and + # forwarded, so forwarding must be enabled (Docker also sets this). + sysctl -qw net.ipv4.ip_forward=1 + + for i in $(seq 0 $((POOL_SIZE-1))); do + local dev host + dev="$(iface "$i")" ; host="$(host_ip "$i")" + if ip link show "$dev" >/dev/null 2>&1; then + ip link set "$dev" down 2>/dev/null || true + ip tuntap del dev "$dev" mode tap 2>/dev/null || true + fi + ip tuntap add dev "$dev" mode tap user "$OWNER" + ip addr add "$host/31" dev "$dev" + ip link set "$dev" up + echo " $dev host=$host guest=$(guest_ip "$i") owner=$OWNER" + done + + _install_nft + echo "nftables table inet $TABLE installed (fail-closed boundary)" + echo "done." +} + +_install_nft() { + # Own table: dropping only matches our bbfc* interfaces, so no other + # tool's traffic is affected. Priority -10 runs before Docker's + # filter hooks (priority 0); a drop here is terminal for the packet. + # + # forward: VM egress is DNAT'd to the sidecar (established via + # `ct status dnat`); return traffic via `ct state established`. + # Anything else from a VM is dropped -> no route to the internet + # or the rest of the host except through the sidecar proxy. + # input: a VM never needs host-local delivery (its sidecar is + # reached via DNAT->forward), so drop all direct input from VMs + # -> host services bound on 0.0.0.0 are unreachable from the VM. + nft -f - </dev/null || true + for i in $(seq 0 $((POOL_SIZE-1))); do + local dev ; dev="$(iface "$i")" + if ip link show "$dev" >/dev/null 2>&1; then + ip link set "$dev" down 2>/dev/null || true + ip tuntap del dev "$dev" mode tap 2>/dev/null || true + echo " removed $dev" + fi + done + echo "done." +} + +cmd_status() { + echo "table inet $TABLE:" + nft list table inet "$TABLE" 2>/dev/null || echo " (absent)" + echo "taps:" + for i in $(seq 0 $((POOL_SIZE-1))); do + local dev ; dev="$(iface "$i")" + if ip -brief addr show "$dev" >/dev/null 2>&1; then + ip -brief addr show "$dev" | sed 's/^/ /' + fi + done +} + +case "${1:-}" in + up) cmd_up ;; + down) cmd_down ;; + status) cmd_status ;; + *) echo "usage: $0 {up|down|status}" >&2 ; exit 2 ;; +esac diff --git a/tests/unit/test_backend_selection.py b/tests/unit/test_backend_selection.py index 4db3932..3511d49 100644 --- a/tests/unit/test_backend_selection.py +++ b/tests/unit/test_backend_selection.py @@ -64,6 +64,24 @@ class TestGetBottleBackend(unittest.TestCase): b = get_bottle_backend() self.assertEqual("smolmachines", b.name) + def test_default_firecracker_when_macos_unavailable_but_fc_available(self): + class _FakeBackend: + def __init__(self, name: str, available: bool) -> None: + self.name = name + self._available = available + + def is_available(self) -> bool: + return self._available + + with patch.dict(os.environ, {}, clear=True), \ + patch.object(backend_mod, "_BACKENDS", { + "macos-container": _FakeBackend("macos-container", False), + "firecracker": _FakeBackend("firecracker", True), + "smolmachines": _FakeBackend("smolmachines", True), + }): + b = get_bottle_backend() + self.assertEqual("firecracker", b.name) + def test_unknown_dies(self): with patch.object(backend_mod, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): @@ -73,7 +91,7 @@ class TestGetBottleBackend(unittest.TestCase): class TestKnownBackendNames(unittest.TestCase): def test_returns_backends_sorted(self): self.assertEqual( - ("docker", "macos-container", "smolmachines"), + ("docker", "firecracker", "macos-container", "smolmachines"), known_backend_names(), ) diff --git a/tests/unit/test_firecracker_backend.py b/tests/unit/test_firecracker_backend.py new file mode 100644 index 0000000..00321dc --- /dev/null +++ b/tests/unit/test_firecracker_backend.py @@ -0,0 +1,187 @@ +"""Unit: Firecracker backend helpers. + +Covers the pieces that run without KVM/root: pool IP math + config +renderers + allocation, the SSH-backed bottle's argv construction, and +the VM boot-arg assembly. +""" + +from __future__ import annotations + +import base64 +import os +import unittest +from pathlib import Path +from typing import Any, cast +from unittest.mock import patch + +from bot_bottle.backend.firecracker import firecracker_vm, netpool +from bot_bottle.backend.firecracker.bottle import FirecrackerBottle + + +def _bottle(**kw: Any) -> FirecrackerBottle: + defaults = dict( + name="bot-bottle-dev-abc", + private_key=Path("/tmp/key"), + guest_ip="100.64.0.1", + ) + defaults.update(kw) + return FirecrackerBottle(**defaults) # type: ignore[arg-type] + + +class TestNetpoolSlots(unittest.TestCase): + def test_slot_ip_math_31_pairs(self): + with patch.dict(os.environ, {"BOT_BOTTLE_FC_IP_BASE": "100.64.0.0"}): + s0, s1 = netpool.slot(0), netpool.slot(1) + self.assertEqual(("bbfc0", "100.64.0.0", "100.64.0.1"), + (s0.iface, s0.host_ip, s0.guest_ip)) + self.assertEqual(("bbfc1", "100.64.0.2", "100.64.0.3"), + (s1.iface, s1.host_ip, s1.guest_ip)) + + def test_guest_cidr_is_31(self): + self.assertEqual("100.64.0.1/31", netpool.slot(0).guest_cidr) + + def test_pool_size_env_override(self): + with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "4"}): + self.assertEqual(4, netpool.pool_size()) + self.assertEqual(4, len(netpool.all_slots())) + + +class TestNetpoolRenderers(unittest.TestCase): + def test_nixos_module_has_table_and_taps(self): + with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "2"}): + out = netpool.render_nixos_module() + self.assertIn('networking.nftables.tables."bot_bottle_fc"', out) + self.assertIn('Name = "bbfc0"', out) + self.assertIn('Name = "bbfc1"', out) + self.assertIn("net.ipv4.ip_forward", out) + # fail-closed isolation rules present + self.assertIn('iifname != "bbfc*" return', out) + self.assertIn("ct status dnat accept", out) + + def test_shell_setup_reflects_overrides(self): + with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "3"}): + out = netpool.render_shell_setup() + self.assertIn("BOT_BOTTLE_FC_POOL_SIZE=3", out) + self.assertIn("firecracker-netpool.sh up", out) + + +class TestNetpoolAllocation(unittest.TestCase): + def test_allocate_is_mutually_exclusive(self): + with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "1"}): + # First allocation grabs the only slot; the lock is held + # until the handle is closed, so a second call must fail + # over (and here, exhaust the pool). + slot, lock = netpool.allocate("first") + self.addCleanup(lock.close) + self.assertEqual("bbfc0", slot.iface) + with patch.object(netpool, "die", + side_effect=SystemExit("exhausted")): + with self.assertRaises(SystemExit): + netpool.allocate("second") + + def test_allocate_releases_on_close(self): + with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "1"}): + slot, lock = netpool.allocate("first") + lock.close() + # Slot is free again after close. + slot2, lock2 = netpool.allocate("second") + self.addCleanup(lock2.close) + self.assertEqual(slot.iface, slot2.iface) + + +class TestBottleAgentArgv(unittest.TestCase): + def test_interactive_uses_ssh_tty_and_runuser(self): + argv = _bottle().agent_argv([], tty=True) + self.assertEqual("ssh", argv[0]) + self.assertIn("-t", argv) + self.assertIn("100.64.0.1", " ".join(argv)) + idx = argv.index("--") + self.assertEqual(["runuser", "-u", "node", "--"], argv[idx + 1:idx + 5]) + self.assertIn("claude", argv) + + def test_non_interactive_has_no_tty(self): + argv = _bottle().agent_argv([], tty=False) + self.assertEqual("ssh", argv[0]) + self.assertNotIn("-t", argv) + + def test_appends_extra_args_after_command(self): + argv = _bottle().agent_argv( + ["--dangerously-skip-permissions", "--continue"], tty=False, + ) + idx = argv.index("claude") + self.assertEqual( + ["claude", "--dangerously-skip-permissions", "--continue"], + argv[idx:], + ) + + def test_prompt_file_flag_injected(self): + argv = _bottle( + prompt_path_in_guest="/home/node/.bot-bottle-prompt.txt", + ).agent_argv(["--continue"], tty=False) + idx = argv.index("claude") + self.assertEqual( + ["claude", "--continue", + "--append-system-prompt-file", "/home/node/.bot-bottle-prompt.txt"], + argv[idx:], + ) + + def test_workdir_wraps_command(self): + argv = _bottle(agent_workdir="/home/node/workspace").agent_argv([], tty=False) + self.assertIn("sh", argv) + joined = " ".join(argv) + self.assertIn("cd /home/node/workspace", joined) + + def test_guest_env_injected(self): + argv = _bottle(guest_env={"HTTPS_PROXY": "http://100.64.0.0:9099"}).agent_argv( + [], tty=False, + ) + self.assertIn("HTTPS_PROXY=http://100.64.0.0:9099", argv) + self.assertIn("HOME=/home/node", argv) + self.assertIn("USER=node", argv) + + +class TestSetupScriptConsistency(unittest.TestCase): + """The shell setup script duplicates the pool defaults; keep them in + lockstep with the Python constants so the two setup paths agree.""" + + def _script(self) -> str: + root = Path(__file__).resolve().parent.parent.parent + return (root / "scripts" / "firecracker-netpool.sh").read_text() + + def test_defaults_match_python(self): + script = self._script() + with patch.dict(os.environ, {}, clear=True): + self.assertIn(f'POOL_SIZE:-{netpool.pool_size()}', script) + self.assertIn(f'BOT_BOTTLE_FC_IP_BASE:-{netpool.ip_base()}', script) + self.assertIn(f'IFACE_PREFIX:-{netpool.IFACE_PREFIX}', script) + + def test_table_name_and_ports_match(self): + script = self._script() + self.assertIn(f'TABLE="{netpool.NFT_TABLE}"', script) + for port in netpool.SIDECAR_PORTS: + self.assertIn(str(port), script) + + +class TestBootArgs(unittest.TestCase): + def test_boot_args_carry_ip_and_pubkey(self): + args = firecracker_vm._boot_args( + guest_ip="100.64.0.1", host_ip="100.64.0.0", pubkey="ssh-ed25519 AAAA x", + ) + self.assertIn("ip=100.64.0.1::100.64.0.0:255.255.255.254::eth0:off", args) + self.assertIn("init=/bb-init", args) + b64 = base64.b64encode(b"ssh-ed25519 AAAA x").decode() + self.assertIn(f"bb_pubkey={b64}", args) + + def test_config_has_rootfs_and_tap(self): + cfg = cast(Any, firecracker_vm._config( + rootfs=Path("/run/rootfs.ext4"), tap="bbfc0", + guest_ip="100.64.0.1", host_ip="100.64.0.0", pubkey="k", + vcpus=2, mem_mib=2048, guest_mac="06:00:AC:10:00:02", + )) + self.assertEqual("/run/rootfs.ext4", cfg["drives"][0]["path_on_host"]) + self.assertFalse(cfg["drives"][0]["is_read_only"]) + self.assertEqual("bbfc0", cfg["network-interfaces"][0]["host_dev_name"]) + + +if __name__ == "__main__": + unittest.main()