From c276f7b0b18c3490e84841270a38eb0275081b3b Mon Sep 17 00:00:00 2001 From: didericis Date: Sat, 11 Jul 2026 10:32:55 -0400 Subject: [PATCH 01/13] feat(firecracker): add Linux microVM backend to replace smolmachines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a Firecracker-based backend for Linux, providing mature KVM-based microVM isolation to replace smolmachines/libkrun (issue #342, closes the dead-end tracked in #332). Architecture: - Guest control over SSH (dropbear injected into the rootfs) on a point-to-point TAP link. `ssh -t` forwards SIGWINCH natively, so no resize bridge is needed. - Networking: a one-time, root-provisioned pool of user-owned TAP devices (no shared bridge → no docker0/virbr0/cni0 collisions) plus a dedicated `table inet bot_bottle_fc` nftables table (independent of Docker/ufw/firewalld rules). `./cli.py firecracker setup` prints the host-appropriate config (NixOS module or sudo script). - Rootfs: `docker export` → ext4 via `mke2fs -d` (rootless, no mount), cached by image digest; per-bottle SSH pubkey + IP passed via the kernel cmdline. - Sidecar: reuses the Docker bundle, published on the slot's host TAP IP. - Fail-closed isolation: TAP pool verified at preflight; the egress boundary is proven empirically post-boot (before the agent runs) by a canary probe — the VM must fail to reach the host directly, or launch is refused. Linux hosts with Firecracker + KVM now default to this backend; macOS stays on macos-container. Not yet validated end-to-end on live hardware (requires the one-time network pool). Unit tests + pyright pass. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01G8p32HJgPoS1hLPWubbftM --- bot_bottle/backend/__init__.py | 9 +- bot_bottle/backend/firecracker/__init__.py | 7 + bot_bottle/backend/firecracker/backend.py | 89 ++++ bot_bottle/backend/firecracker/bottle.py | 170 ++++++++ .../firecracker/bottle_cleanup_plan.py | 32 ++ bot_bottle/backend/firecracker/bottle_plan.py | 63 +++ bot_bottle/backend/firecracker/cleanup.py | 90 ++++ bot_bottle/backend/firecracker/enumerate.py | 43 ++ .../backend/firecracker/firecracker_vm.py | 173 ++++++++ bot_bottle/backend/firecracker/freezer.py | 76 ++++ .../backend/firecracker/isolation_probe.py | 116 +++++ bot_bottle/backend/firecracker/launch.py | 404 ++++++++++++++++++ bot_bottle/backend/firecracker/netpool.py | 243 +++++++++++ .../backend/firecracker/resolve_plan.py | 47 ++ bot_bottle/backend/firecracker/util.py | 296 +++++++++++++ bot_bottle/backend/freeze.py | 8 +- bot_bottle/cli/__init__.py | 3 + bot_bottle/cli/firecracker.py | 79 ++++ scripts/firecracker-netpool.sh | 159 +++++++ tests/unit/test_backend_selection.py | 20 +- tests/unit/test_firecracker_backend.py | 187 ++++++++ 21 files changed, 2309 insertions(+), 5 deletions(-) create mode 100644 bot_bottle/backend/firecracker/__init__.py create mode 100644 bot_bottle/backend/firecracker/backend.py create mode 100644 bot_bottle/backend/firecracker/bottle.py create mode 100644 bot_bottle/backend/firecracker/bottle_cleanup_plan.py create mode 100644 bot_bottle/backend/firecracker/bottle_plan.py create mode 100644 bot_bottle/backend/firecracker/cleanup.py create mode 100644 bot_bottle/backend/firecracker/enumerate.py create mode 100644 bot_bottle/backend/firecracker/firecracker_vm.py create mode 100644 bot_bottle/backend/firecracker/freezer.py create mode 100644 bot_bottle/backend/firecracker/isolation_probe.py create mode 100644 bot_bottle/backend/firecracker/launch.py create mode 100644 bot_bottle/backend/firecracker/netpool.py create mode 100644 bot_bottle/backend/firecracker/resolve_plan.py create mode 100644 bot_bottle/backend/firecracker/util.py create mode 100644 bot_bottle/cli/firecracker.py create mode 100755 scripts/firecracker-netpool.sh create mode 100644 tests/unit/test_firecracker_backend.py diff --git a/bot_bottle/backend/__init__.py b/bot_bottle/backend/__init__.py index ed8b894..41559c5 100644 --- a/bot_bottle/backend/__init__.py +++ b/bot_bottle/backend/__init__.py @@ -26,8 +26,9 @@ backend exposes five methods: Selection is driven by `--backend` on `start` or BOT_BOTTLE_BACKEND (env var). When neither is set, compatible macOS hosts default to -`macos-container`; other hosts default to `smolmachines`. Per PRD 0003 -the manifest does not carry a backend field; the host picks. +`macos-container`; Linux hosts with Firecracker + KVM default to +`firecracker`; otherwise `smolmachines`. Per PRD 0003 the manifest +does not carry a backend field; the host picks. """ from __future__ import annotations @@ -542,6 +543,7 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]): # each backend module can pull BottleSpec / BottlePlan / BottleBackend # via `from . import ...` without hitting a partially-initialized module. from .docker import DockerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position +from .firecracker import FirecrackerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position from .macos_container import MacosContainerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position from .smolmachines import SmolmachinesBottleBackend # noqa: E402 # pylint: disable=wrong-import-position @@ -557,6 +559,7 @@ from .freeze import CommitCancelled, Freezer, get_freezer # noqa: E402 # pylin # unparameterized methods (prepare → plan → launch(plan), cleanup, etc.). _BACKENDS: dict[str, BottleBackend[Any, Any]] = { "docker": DockerBottleBackend(), + "firecracker": FirecrackerBottleBackend(), "macos-container": MacosContainerBottleBackend(), "smolmachines": SmolmachinesBottleBackend(), } @@ -585,6 +588,8 @@ def get_bottle_backend( def _default_backend_name() -> str: if has_backend("macos-container"): return "macos-container" + if has_backend("firecracker"): + return "firecracker" return "smolmachines" diff --git a/bot_bottle/backend/firecracker/__init__.py b/bot_bottle/backend/firecracker/__init__.py new file mode 100644 index 0000000..57b5840 --- /dev/null +++ b/bot_bottle/backend/firecracker/__init__.py @@ -0,0 +1,7 @@ +"""Firecracker backend: Linux KVM microVM isolation (issue #342).""" + +from __future__ import annotations + +from .backend import FirecrackerBottleBackend + +__all__ = ["FirecrackerBottleBackend"] diff --git a/bot_bottle/backend/firecracker/backend.py b/bot_bottle/backend/firecracker/backend.py new file mode 100644 index 0000000..8431dca --- /dev/null +++ b/bot_bottle/backend/firecracker/backend.py @@ -0,0 +1,89 @@ +"""FirecrackerBottleBackend — Linux microVM implementation. + +Replaces smolmachines on Linux (issue #342): mature KVM-based +isolation via Firecracker, SSH control over a point-to-point TAP, and a +fail-closed nftables egress boundary. Selected by +`BOT_BOTTLE_BACKEND=firecracker` or `--backend=firecracker`. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from pathlib import Path +from typing import Generator, Sequence + +from ...agent_provider import AgentProvisionPlan +from ...egress import EgressPlan +from ...env import ResolvedEnv +from ...git_gate import GitGatePlan +from ...manifest import Manifest +from ...supervise import SupervisePlan +from .. import ActiveAgent, BottleBackend, BottleSpec +from . import cleanup as _cleanup +from . import enumerate as _enumerate +from . import launch as _launch +from . import resolve_plan as _resolve_plan +from . import util as _util +from .bottle import FirecrackerBottle +from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan +from .bottle_plan import FirecrackerBottlePlan + + +class FirecrackerBottleBackend( + BottleBackend["FirecrackerBottlePlan", "FirecrackerBottleCleanupPlan"] +): + name = "firecracker" + + @classmethod + def is_available(cls) -> bool: + return _util.is_available() + + def _preflight(self) -> None: + _resolve_plan.preflight() + + def _build_guest_env(self, resolved_env: ResolvedEnv) -> dict[str, str]: + return _resolve_plan.build_guest_env(resolved_env) + + def _resolve_plan( + self, + spec: BottleSpec, + *, + manifest: Manifest, + slug: str, + resolved_env: ResolvedEnv, + agent_provision_plan: AgentProvisionPlan, + egress_plan: EgressPlan, + git_gate_plan: GitGatePlan, + supervise_plan: SupervisePlan | None, + stage_dir: Path, + ) -> FirecrackerBottlePlan: + return _resolve_plan.resolve_plan( + spec, + manifest=manifest, + slug=slug, + resolved_env=resolved_env, + agent_provision_plan=agent_provision_plan, + egress_plan=egress_plan, + supervise_plan=supervise_plan, + git_gate_plan=git_gate_plan, + stage_dir=stage_dir, + ) + + @contextmanager + def launch( + self, plan: FirecrackerBottlePlan + ) -> Generator[FirecrackerBottle, None, None]: + with _launch.launch(plan, provision=self.provision) as bottle: + yield bottle + + def prepare_cleanup(self) -> FirecrackerBottleCleanupPlan: + return _cleanup.prepare_cleanup() + + def cleanup(self, plan: FirecrackerBottleCleanupPlan) -> None: + _cleanup.cleanup(plan) + + def enumerate_active(self) -> Sequence[ActiveAgent]: + return _enumerate.enumerate_active() + + def supervise_mcp_url(self, plan: FirecrackerBottlePlan) -> str: + return plan.agent_supervise_url diff --git a/bot_bottle/backend/firecracker/bottle.py b/bot_bottle/backend/firecracker/bottle.py new file mode 100644 index 0000000..3805dc9 --- /dev/null +++ b/bot_bottle/backend/firecracker/bottle.py @@ -0,0 +1,170 @@ +"""Bottle handle for a Firecracker microVM, over SSH. + +Every operation routes through SSH to the guest (dropbear, listening on +the TAP link). `exec` pipes a script over stdin and captures output; +`cp_in` uses scp; `exec_agent` uses `ssh -t` for an interactive PTY +session. `ssh -t` forwards the host terminal's SIGWINCH to the remote +PTY natively, so — unlike the smolmachines backend — no resize bridge +is needed. + +Commands run as the image's `node` user via `runuser`, with HOME/USER/ +PATH and the bottle env (HTTPS_PROXY at the sidecar, CA paths, …) set +per-invocation through `env` (the VM itself just runs the init; it has +no baked-in process env like a `docker run` container would). +""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import sys +from pathlib import Path +from typing import Mapping, cast + +from ...agent_provider import PromptMode, prompt_args +from .. import Bottle, ExecResult +from ..terminal import exec_shell_script +from . import util + + +_HOME_FOR = {"node": "/home/node", "root": "/root"} +_DEFAULT_PATH_FOR = { + "node": ( + "/home/node/.local/bin:" + "/home/node/.codex/packages/standalone/current/bin:" + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + ), + "root": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +} + + +def _env_assignments_for(user: str, env: Mapping[str, str]) -> list[str]: + home = _HOME_FOR.get(user, f"/home/{user}") + out = [f"HOME={home}", f"USER={user}"] + if "PATH" not in env: + out.append(f"PATH={_DEFAULT_PATH_FOR.get(user, _DEFAULT_PATH_FOR['root'])}") + for k, v in env.items(): + out.append(f"{k}={v}") + return out + + +class FirecrackerBottle(Bottle): + def __init__( + self, + name: str, + *, + private_key: Path, + guest_ip: str, + guest_env: Mapping[str, str] | None = None, + prompt_path_in_guest: str | None = None, + agent_command: str = "claude", + agent_prompt_mode: PromptMode = "append_file", + agent_provider_template: str = "claude", + terminal_title: str = "", + terminal_color: str = "", + agent_workdir: str = "/home/node", + ) -> None: + self.name = name + self._private_key = private_key + self._guest_ip = guest_ip + self._guest_env = dict(guest_env or {}) + self.prompt_path = prompt_path_in_guest + self._agent_prompt_mode = agent_prompt_mode + self.agent_command = agent_command + self.terminal_title = terminal_title + self.terminal_color = terminal_color + self.agent_provider_template = agent_provider_template + self.agent_workdir = agent_workdir + + def _ssh(self, *, tty: bool) -> list[str]: + argv = util.ssh_base_argv(self._private_key, self._guest_ip) + if tty: + # -t: allocate a remote PTY and forward window-size changes. + argv.insert(1, "-t") + return argv + + def _agent_remote_argv(self, argv: list[str]) -> list[str]: + """The `runuser … …` command run inside the guest.""" + full_argv = list(argv) + full_argv.extend( + prompt_args( + cast(PromptMode, self._agent_prompt_mode), + self.prompt_path, + argv=full_argv, + ) + ) + remote = ["runuser", "-u", "node", "--", + "env", *_env_assignments_for("node", self._guest_env)] + if self.agent_workdir and self.agent_workdir != _HOME_FOR["node"]: + remote += ["sh", "-lc", f"cd {self.agent_workdir} && exec \"$@\"", + "bot-bottle-agent"] + remote += [self.agent_command, *full_argv] + return remote + + def agent_argv(self, argv: list[str], *, tty: bool = True) -> list[str]: + return [*self._ssh(tty=tty), "--", *self._agent_remote_argv(argv)] + + def exec_agent(self, argv: list[str], *, tty: bool = True) -> int: + agent_argv = self.agent_argv(argv, tty=tty) + script = ( + exec_shell_script(agent_argv, self.terminal_title, self.terminal_color) + if tty else None + ) + if script is None: + return subprocess.run(agent_argv, check=False).returncode + return subprocess.run(["sh", "-lc", script], check=False).returncode + + def exec(self, script: str, *, user: str = "node") -> ExecResult: + # Pipe the script over stdin (`sh -s`) so nothing needs shell + # quoting through the SSH command line. + remote = ["runuser", "-u", user, "--", + "env", *_env_assignments_for(user, self._guest_env), + "/bin/sh", "-s"] + result = subprocess.run( + [*self._ssh(tty=False), "--", *remote], + input=script, capture_output=True, text=True, check=False, + ) + return ExecResult( + returncode=result.returncode, + stdout=result.stdout, + stderr=result.stderr, + ) + + def cp_in(self, host_path: str, container_path: str) -> None: + # Stream a tar over the SSH exec channel rather than scp: the + # guest runs dropbear, which ships no sftp-server/scp, but every + # agent image has `tar`. Copy so `container_path` becomes a copy + # of `host_path` (docker cp semantics — callers rm -rf the + # destination first). + host_parent = os.path.dirname(host_path.rstrip("/")) or "/" + host_base = os.path.basename(host_path.rstrip("/")) + guest_parent = os.path.dirname(container_path.rstrip("/")) or "/" + guest_base = os.path.basename(container_path.rstrip("/")) + remote = ( + f"mkdir -p {shlex.quote(guest_parent)} && " + f"cd {shlex.quote(guest_parent)} && " + f"rm -rf {shlex.quote(guest_base)} && tar -xf - && " + f"{{ [ {shlex.quote(host_base)} = {shlex.quote(guest_base)} ] || " + f"mv {shlex.quote(host_base)} {shlex.quote(guest_base)}; }}" + ) + tar = subprocess.Popen( + ["tar", "-C", host_parent, "-cf", "-", host_base], + stdout=subprocess.PIPE, + ) + ssh = subprocess.run( + [*self._ssh(tty=False), "--", "sh", "-c", remote], + stdin=tar.stdout, capture_output=True, text=True, check=False, + ) + tar.wait() + if tar.returncode != 0 or ssh.returncode != 0: + sys.stderr.write(ssh.stderr) + raise subprocess.CalledProcessError( + ssh.returncode or tar.returncode, + ["cp_in", host_path, container_path], + ) + + def close(self) -> None: + # Real teardown (VM terminate, TAP release) lives on the launch + # ExitStack; this is the idempotent alias the ABC expects. + pass diff --git a/bot_bottle/backend/firecracker/bottle_cleanup_plan.py b/bot_bottle/backend/firecracker/bottle_cleanup_plan.py new file mode 100644 index 0000000..3fd8eec --- /dev/null +++ b/bot_bottle/backend/firecracker/bottle_cleanup_plan.py @@ -0,0 +1,32 @@ +"""Cleanup plan for the Firecracker backend.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from ...log import info +from .. import BottleCleanupPlan + + +@dataclass(frozen=True) +class FirecrackerBottleCleanupPlan(BottleCleanupPlan): + # PIDs of orphaned firecracker VMM processes and the sidecar + # containers left behind by previous bottles. + vm_pids: tuple[int, ...] = () + containers: tuple[str, ...] = () + run_dirs: tuple[str, ...] = () + + def print(self) -> None: + if self.empty: + info("firecracker cleanup: nothing to remove") + return + for pid in self.vm_pids: + info(f"firecracker VM process: pid {pid}") + for name in self.containers: + info(f"firecracker sidecar container: {name}") + for path in self.run_dirs: + info(f"firecracker run dir: {path}") + + @property + def empty(self) -> bool: + return not (self.vm_pids or self.containers or self.run_dirs) diff --git a/bot_bottle/backend/firecracker/bottle_plan.py b/bot_bottle/backend/firecracker/bottle_plan.py new file mode 100644 index 0000000..91decae --- /dev/null +++ b/bot_bottle/backend/firecracker/bottle_plan.py @@ -0,0 +1,63 @@ +"""Plan type for the Firecracker backend.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from ...agent_provider import PromptMode +from .. import BottlePlan + + +@dataclass(frozen=True) +class FirecrackerBottlePlan(BottlePlan): + slug: str + forwarded_env: dict[str, str] = field(repr=False) + # Stamped by launch once the sidecar is up and its ports are + # published on the host-side TAP IP (empty at prepare time). + agent_proxy_url: str = "" + agent_git_gate_url: str = "" + agent_supervise_url: str = "" + + @property + def container_name(self) -> str: + """Instance name, reused for the sidecar container + VM run dir. + Matches the `bot-bottle-` convention the other backends + use so cleanup/enumerate discovery-by-prefix keeps working.""" + return self.agent_provision.instance_name + + @property + def image(self) -> str: + return self.agent_provision.image + + @property + def dockerfile_path(self) -> str: + return self.agent_provision.dockerfile + + @property + def prompt_file(self) -> Path: + return self.agent_provision.prompt_file + + @property + def agent_command(self) -> str: + return self.agent_provision.command + + @property + def agent_prompt_mode(self) -> PromptMode: + return self.agent_provision.prompt_mode + + @property + def agent_provider_template(self) -> str: + return self.agent_provision.template + + @property + def git_gate_insteadof_host(self) -> str: + if self.agent_git_gate_url.startswith("http://"): + return self.agent_git_gate_url.removeprefix("http://").rstrip("/") + return super().git_gate_insteadof_host + + @property + def git_gate_insteadof_scheme(self) -> str: + if self.agent_git_gate_url.startswith("http://"): + return "http" + return super().git_gate_insteadof_scheme diff --git a/bot_bottle/backend/firecracker/cleanup.py b/bot_bottle/backend/firecracker/cleanup.py new file mode 100644 index 0000000..938bca9 --- /dev/null +++ b/bot_bottle/backend/firecracker/cleanup.py @@ -0,0 +1,90 @@ +"""Cleanup for the Firecracker backend. + +Orphans are: firecracker VMM processes whose config lives under our run +dir, the `bot-bottle-sidecars-*` containers, and the per-bottle run +dirs. TAP slots free themselves (the flock drops when the launcher +exits), so there is nothing to reclaim there. +""" + +from __future__ import annotations + +import os +import shutil +import signal +import subprocess +from pathlib import Path + +from ...log import info +from . import util +from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan + +_SIDECAR_PREFIX = "bot-bottle-sidecars-" + + +def _run_root() -> Path: + return util.cache_dir() / "run" + + +def _orphan_vm_pids() -> list[int]: + """firecracker processes whose --config-file is under our run dir.""" + run_root = str(_run_root()) + result = subprocess.run( + ["pgrep", "-a", "firecracker"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + return [] + pids: list[int] = [] + for line in result.stdout.splitlines(): + parts = line.split(None, 1) + if len(parts) != 2 or run_root not in parts[1]: + continue + try: + pids.append(int(parts[0])) + except ValueError: + continue + return pids + + +def _sidecar_containers() -> list[str]: + result = subprocess.run( + ["docker", "ps", "-a", "--format", "{{.Names}}", + "--filter", f"name={_SIDECAR_PREFIX}"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + return [] + return sorted(n.strip() for n in result.stdout.splitlines() if n.strip()) + + +def _run_dirs() -> list[str]: + run_root = _run_root() + if not run_root.is_dir(): + return [] + return sorted(str(p) for p in run_root.iterdir() if p.is_dir()) + + +def prepare_cleanup() -> FirecrackerBottleCleanupPlan: + return FirecrackerBottleCleanupPlan( + vm_pids=tuple(_orphan_vm_pids()), + containers=tuple(_sidecar_containers()), + run_dirs=tuple(_run_dirs()), + ) + + +def cleanup(plan: FirecrackerBottleCleanupPlan) -> None: + for pid in plan.vm_pids: + info(f"kill firecracker VM pid {pid}") + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + for name in plan.containers: + info(f"docker rm -f {name}") + subprocess.run( + ["docker", "rm", "-f", name], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, + ) + for path in plan.run_dirs: + info(f"rm -rf {path}") + shutil.rmtree(path, ignore_errors=True) diff --git a/bot_bottle/backend/firecracker/enumerate.py b/bot_bottle/backend/firecracker/enumerate.py new file mode 100644 index 0000000..b4a7f42 --- /dev/null +++ b/bot_bottle/backend/firecracker/enumerate.py @@ -0,0 +1,43 @@ +"""Active-agent enumeration for the Firecracker backend. + +The agent runs in a VM (no container to list), so a live bottle is +identified by its running sidecar container `bot-bottle-sidecars-` +— the same discovery-by-prefix the other backends use. +""" + +from __future__ import annotations + +import subprocess + +from ...bottle_state import read_metadata +from .. import ActiveAgent + +_SIDECAR_PREFIX = "bot-bottle-sidecars-" + + +def enumerate_active() -> list[ActiveAgent]: + result = subprocess.run( + ["docker", "ps", "--format", "{{.Names}}", + "--filter", f"name={_SIDECAR_PREFIX}"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + return [] + out: list[ActiveAgent] = [] + for name in sorted(n.strip() for n in result.stdout.splitlines() if n.strip()): + slug = name[len(_SIDECAR_PREFIX):] + metadata = read_metadata(slug) + if metadata is None or metadata.backend != "firecracker": + # Skip sidecars owned by another backend (docker/smolmachines + # share the container-name prefix). + continue + out.append(ActiveAgent( + backend_name="firecracker", + slug=slug, + agent_name=metadata.agent_name, + started_at=metadata.started_at, + services=(), + label=metadata.label, + color=metadata.color, + )) + return out diff --git a/bot_bottle/backend/firecracker/firecracker_vm.py b/bot_bottle/backend/firecracker/firecracker_vm.py new file mode 100644 index 0000000..113065d --- /dev/null +++ b/bot_bottle/backend/firecracker/firecracker_vm.py @@ -0,0 +1,173 @@ +"""Firecracker microVM process lifecycle. + +A bottle VM is one `firecracker` process booted from a JSON config +(kernel + writable rootfs ext4 + one TAP network interface). We run it +`--no-api`: all configuration is in the config file, and teardown is a +process signal — the microVM dies with its VMM, which is exactly the +ephemeral-bottle semantics we want. No API socket, no runtime +reconfiguration. + +The guest gets its IP from the kernel `ip=` cmdline (kernel-level +autoconfig, no in-guest iproute2) and its SSH pubkey from a `bb_pubkey=` +cmdline arg the init decodes. +""" + +from __future__ import annotations + +import base64 +import json +import signal +import subprocess +import time +from dataclasses import dataclass +from pathlib import Path + +from ...log import die, info +from . import util + + +# Serial console off the critical path but captured to a log for +# debugging boot failures. `reboot=k panic=1 pci=off` are the standard +# Firecracker guest args; `i8042.*` skip the (absent) PS/2 probe to +# shave boot time. +_BASE_BOOT_ARGS = ( + "console=ttyS0 reboot=k panic=1 pci=off " + "i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd " + "root=/dev/vda rw init=/bb-init" +) + + +@dataclass +class VmHandle: + """A running microVM: its VMM process, guest IP, and console log.""" + + process: subprocess.Popen[bytes] + guest_ip: str + console_log: Path + + def is_alive(self) -> bool: + return self.process.poll() is None + + def terminate(self) -> None: + """Stop the VMM (and thus the guest). SIGTERM, then SIGKILL.""" + if self.process.poll() is not None: + return + self.process.send_signal(signal.SIGTERM) + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + + +def _boot_args(guest_ip: str, host_ip: str, pubkey: str) -> str: + # ip=::::::off — /31 point-to-point link, + # so netmask is 255.255.255.254 and the gateway is the host TAP IP. + ip_arg = f"ip={guest_ip}::{host_ip}:255.255.255.254::eth0:off" + pub_b64 = base64.b64encode(pubkey.encode()).decode() + return f"{_BASE_BOOT_ARGS} {ip_arg} bb_pubkey={pub_b64}" + + +def _config( + *, + rootfs: Path, + tap: str, + guest_ip: str, + host_ip: str, + pubkey: str, + vcpus: int, + mem_mib: int, + guest_mac: str, +) -> dict[str, object]: + return { + "boot-source": { + "kernel_image_path": str(util.kernel_path()), + "boot_args": _boot_args(guest_ip, host_ip, pubkey), + }, + "drives": [ + { + "drive_id": "rootfs", + "path_on_host": str(rootfs), + "is_root_device": True, + "is_read_only": False, + } + ], + "network-interfaces": [ + { + "iface_id": "eth0", + "host_dev_name": tap, + "guest_mac": guest_mac, + } + ], + "machine-config": { + "vcpu_count": vcpus, + "mem_size_mib": mem_mib, + }, + } + + +def boot( + *, + name: str, + rootfs: Path, + tap: str, + guest_ip: str, + host_ip: str, + pubkey: str, + run_dir: Path, + vcpus: int = 2, + mem_mib: int = 2048, + guest_mac: str = "06:00:AC:10:00:02", +) -> VmHandle: + """Write the config and launch the VMM. Returns once the process is + spawned; callers wait for SSH readiness separately.""" + run_dir.mkdir(parents=True, exist_ok=True) + config_path = run_dir / "config.json" + console_log = run_dir / "console.log" + config_path.write_text(json.dumps( + _config( + rootfs=rootfs, tap=tap, guest_ip=guest_ip, host_ip=host_ip, + pubkey=pubkey, vcpus=vcpus, mem_mib=mem_mib, guest_mac=guest_mac, + ), + indent=2, + )) + + info(f"booting microVM {name} on {tap} (guest {guest_ip})") + log_fh = console_log.open("wb") + process = subprocess.Popen( + ["firecracker", "--no-api", "--config-file", str(config_path)], + stdout=log_fh, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + ) + return VmHandle(process=process, guest_ip=guest_ip, console_log=console_log) + + +def wait_for_ssh( + vm: VmHandle, private_key: Path, *, timeout: float = 30.0, +) -> None: + """Poll SSH until the guest accepts a command or the deadline + passes. Dies (with the console tail) if the VMM exits early or the + guest never comes up — a boot failure must be loud, not a hang.""" + deadline = time.monotonic() + timeout + probe = util.ssh_base_argv(private_key, vm.guest_ip) + ["true"] + while time.monotonic() < deadline: + if not vm.is_alive(): + die(f"microVM exited during boot (rc={vm.process.returncode}).\n" + f"{_console_tail(vm.console_log)}") + result = subprocess.run( + probe, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + check=False, + ) + if result.returncode == 0: + return + time.sleep(0.5) + die(f"microVM {vm.guest_ip} did not accept SSH within {timeout:.0f}s.\n" + f"{_console_tail(vm.console_log)}") + + +def _console_tail(console_log: Path, lines: int = 25) -> str: + try: + text = console_log.read_text(errors="replace").splitlines() + except OSError: + return "(no console log)" + tail = "\n".join(text[-lines:]) + return f"--- guest console (last {lines} lines) ---\n{tail}" diff --git a/bot_bottle/backend/firecracker/freezer.py b/bot_bottle/backend/firecracker/freezer.py new file mode 100644 index 0000000..16c6370 --- /dev/null +++ b/bot_bottle/backend/firecracker/freezer.py @@ -0,0 +1,76 @@ +"""FirecrackerFreezer — snapshot a running microVM to a Docker image. + +The VM is live and can't be block-copied safely, so — like the macOS +backend — we stream the guest root filesystem out over the control +channel (SSH here) and rebuild an image from it. The bottle keeps +running after the snapshot. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +from pathlib import Path + +from ...log import die, info +from .. import ActiveAgent +from ..freeze import Freezer +from . import util + + +class FirecrackerFreezer(Freezer): + backend_name = "firecracker" + + def _freeze(self, agent: ActiveAgent) -> str: + run_dir = util.cache_dir() / "run" / agent.slug + private_key = run_dir / "bottle_id_ed25519" + guest_ip = _guest_ip_from_config(run_dir / "config.json") + if not private_key.is_file() or not guest_ip: + die(f"cannot freeze {agent.slug}: run dir {run_dir} is missing the " + f"SSH key or VM config (is the bottle still running?)") + image_tag = f"bot-bottle-committed-{agent.slug}:latest" + _commit_via_ssh(private_key, guest_ip, image_tag) + info(f"committed {agent.slug} -> {image_tag!r}") + return image_tag + + def _export_hint(self, slug: str, image_ref: str) -> None: + info(f"to export for migration: docker image save {image_ref} " + f"-o {slug}.tar") + + +def _guest_ip_from_config(config_path: Path) -> str: + try: + cfg = json.loads(config_path.read_text()) + except (OSError, json.JSONDecodeError): + return "" + boot_args = cfg.get("boot-source", {}).get("boot_args", "") + for token in boot_args.split(): + if token.startswith("ip="): + # ip=::::::off + return token[len("ip="):].split(":", 1)[0] + return "" + + +def _commit_via_ssh(private_key: Path, guest_ip: str, image_tag: str) -> None: + with tempfile.TemporaryDirectory(prefix="bot-bottle-fc-commit.") as tmp: + rootfs_tar = os.path.join(tmp, "rootfs.tar") + ssh = util.ssh_base_argv(private_key, guest_ip) + with open(rootfs_tar, "wb") as tar_out: + result = subprocess.run( + [*ssh, "--", "tar", "--create", "--one-file-system", + "--exclude=./proc", "--exclude=./sys", "--exclude=./dev", + "--exclude=./run", "--file=-", "--directory=/", "."], + stdout=tar_out, stderr=subprocess.PIPE, check=False, + ) + if result.returncode != 0: + die(f"ssh tar for {guest_ip} failed: " + f"{(result.stderr or b'').decode().strip() or ''}") + with open(os.path.join(tmp, "Dockerfile"), "w", encoding="utf-8") as f: + f.write("FROM scratch\nADD rootfs.tar /\nUSER node\nWORKDIR /home/node\n") + build = subprocess.run( + ["docker", "build", "-t", image_tag, tmp], check=False, + ) + if build.returncode != 0: + die(f"docker build for {image_tag!r} failed") diff --git a/bot_bottle/backend/firecracker/isolation_probe.py b/bot_bottle/backend/firecracker/isolation_probe.py new file mode 100644 index 0000000..233665d --- /dev/null +++ b/bot_bottle/backend/firecracker/isolation_probe.py @@ -0,0 +1,116 @@ +"""Empirical, post-boot isolation probe — the authoritative fail-closed +egress-boundary check. + +The pre-boot nft check can't be trusted on every host (listing an +nftables table typically needs root; the launcher runs unprivileged). +So before the agent ever runs, we prove the boundary directly: open a +canary TCP listener on a host IP the VM must NOT be able to reach (the +host's primary non-TAP address), then have the guest try to connect. If +the isolation rules are in force the connection is dropped and times +out; if it succeeds, the VM can reach host services it shouldn't — a +sandbox escape — and we refuse to continue. + +The listener is load-bearing: without it a "blocked" result could just +be connection-refused. With it, a reachable canary means a real leak +and a timeout means a real drop. +""" + +from __future__ import annotations + +import socket +import subprocess +import threading +from pathlib import Path + +from ...log import die, info, warn +from . import util + + +def _host_primary_ip() -> str: + """The source IP the host uses for off-box traffic (its LAN + address) — a canary the VM must not reach. Empty if the host has no + such route (isolated box).""" + result = subprocess.run( + ["ip", "-o", "route", "get", "1.1.1.1"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + return "" + tokens = result.stdout.split() + # `... src ...` — the address the host would use as source. + if "src" in tokens: + idx = tokens.index("src") + if idx + 1 < len(tokens): + return tokens[idx + 1] + return "" + + +# Guest-side connect test: exit 0 = reached the canary (LEAK), 1 = +# blocked (good), 2 = no usable tool (inconclusive -> fail-closed). +_GUEST_PROBE = r""" +h="$1"; p="$2" +if command -v python3 >/dev/null 2>&1; then + python3 - "$h" "$p" <<'PY' +import socket, sys +s = socket.socket(); s.settimeout(3) +sys.exit(0 if s.connect_ex((sys.argv[1], int(sys.argv[2]))) == 0 else 1) +PY +elif command -v bash >/dev/null 2>&1; then + timeout 3 bash -c "exec 3<>/dev/tcp/$h/$p" >/dev/null 2>&1 +elif command -v nc >/dev/null 2>&1; then + nc -w3 -z "$h" "$p" >/dev/null 2>&1 +else + exit 2 +fi +""" + + +def verify_isolation(private_key: Path, guest_ip: str) -> None: + """Prove the VM cannot reach the host's primary IP. Dies + (fail-closed) on a confirmed leak or if the guest has no tool to + run the test. Warns (does not fail) when the host has no canary + address to test against.""" + canary_ip = _host_primary_ip() + if not canary_ip: + warn("isolation probe skipped: host has no non-TAP route to test " + "against (isolated box).") + return + + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind((canary_ip, 0)) + listener.listen(1) + listener.settimeout(6) + canary_port = listener.getsockname()[1] + + accepted: list[bool] = [] + + def _accept() -> None: + try: + conn, _ = listener.accept() + accepted.append(True) + conn.close() + except OSError: + pass + + thread = threading.Thread(target=_accept, daemon=True) + thread.start() + + ssh = util.ssh_base_argv(private_key, guest_ip) + result = subprocess.run( + [*ssh, "--", "sh", "-s", "--", canary_ip, str(canary_port)], + input=_GUEST_PROBE, capture_output=True, text=True, check=False, + ) + thread.join(timeout=7) + listener.close() + + if result.returncode == 0 or accepted: + die(f"ISOLATION FAILURE: the VM reached the host canary " + f"{canary_ip}:{canary_port}. The egress boundary is not in " + f"force — refusing to run the agent (fail-closed). Verify the " + f"nft table with: ./cli.py firecracker setup") + if result.returncode == 2: + die("isolation probe inconclusive: the guest has no python3/bash/nc " + "to run the connectivity test. Refusing to continue " + "(fail-closed).") + info(f"isolation verified: VM cannot reach host {canary_ip}") diff --git a/bot_bottle/backend/firecracker/launch.py b/bot_bottle/backend/firecracker/launch.py new file mode 100644 index 0000000..a0b0486 --- /dev/null +++ b/bot_bottle/backend/firecracker/launch.py @@ -0,0 +1,404 @@ +"""Launch flow for the Firecracker backend. + +Per bottle: + 1. mint the egress CA, build the agent image (docker), export it to a + cached ext4 rootfs; + 2. claim a free TAP pool slot (rootless flock); + 3. bring up the Docker sidecar bundle, publishing egress / git-gate / + supervise on the slot's host-side TAP IP at fixed ports; + 4. boot the microVM on that TAP; wait for SSH; + 5. provision (CA, prompt, skills, workspace, git, supervise) over SSH. + +Isolation is enforced by the operator-provisioned nft table (checked +fail-closed in preflight): a VM reaches only its sidecar (DNAT'd from +the host TAP IP) and nothing else. The agent's HTTPS_PROXY therefore +points at `http://:9099`, its only route to the world. +""" + +from __future__ import annotations + +import dataclasses +import os +import subprocess +from contextlib import ExitStack, contextmanager +from pathlib import Path +from typing import Callable, Generator + +from ...bottle_state import ( + egress_state_dir, + git_gate_state_dir, + read_committed_image, +) +from ...egress import ( + EGRESS_ROUTES_IN_CONTAINER, + egress_agent_env_entries, + egress_resolve_token_values, + egress_sidecar_env_entries, +) +from ...git_gate import ( + provision_git_gate_dynamic_keys, + revoke_git_gate_provisioned_keys, +) +from ...log import die, info, warn +from ...supervise import DB_PATH_IN_CONTAINER, SUPERVISE_PORT +from ...util import expand_tilde +from ..docker.egress import ( + EGRESS_CA_IN_CONTAINER, + EGRESS_PORT, + egress_tls_init, +) +from ..docker.git_gate import ( + GIT_GATE_ACCESS_HOOK_IN_CONTAINER, + GIT_GATE_CREDS_DIR_IN_CONTAINER, + GIT_GATE_ENTRYPOINT_IN_CONTAINER, + GIT_GATE_HOOK_IN_CONTAINER, +) +from ..docker.sidecar_bundle import ( + SIDECAR_BUNDLE_DOCKERFILE, + SIDECAR_BUNDLE_IMAGE, +) +from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH +from . import firecracker_vm, isolation_probe, netpool, util +from .bottle import FirecrackerBottle +from .bottle_plan import FirecrackerBottlePlan + + +_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent) +_GIT_HTTP_PORT = 9420 +_GIT_GATE_READY_FILE = "/run/git-gate/ready" + + +def sidecar_container_name(slug: str) -> str: + return f"bot-bottle-sidecars-{slug}" + + +@contextmanager +def launch( + plan: FirecrackerBottlePlan, + *, + provision: Callable[[FirecrackerBottlePlan, "FirecrackerBottle"], str | None], +) -> Generator[FirecrackerBottle, None, None]: + stack = ExitStack() + bottle_for_revoke = plan.manifest.bottle + git_gate_dir_for_revoke = git_gate_state_dir(plan.slug) + + def teardown() -> None: + teardown_exc: BaseException | None = None + try: + stack.close() + except BaseException as exc: # noqa: W0718 - teardown must continue + teardown_exc = exc + warn(f"firecracker teardown failed: {exc!r}") + revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke) + if teardown_exc is not None: + raise teardown_exc + + try: + plan = _mint_certs(plan) + plan = _build_agent_image(plan) + + # Claim a TAP slot; the flock is held until teardown closes it. + slot, lock = netpool.allocate(plan.slug) + stack.callback(lock.close) + info(f"firecracker slot {slot.iface}: host={slot.host_ip} " + f"guest={slot.guest_ip}") + + plan = _provision_git_gate_keys(plan) + + sidecar_name = sidecar_container_name(plan.slug) + _force_remove_container(sidecar_name) + _start_sidecar_bundle(plan, sidecar_name, slot.host_ip) + stack.callback(_force_remove_container, sidecar_name) + _stage_git_gate(plan, sidecar_name) + + plan = _stamp_agent_urls(plan, slot.host_ip) + + # Build the per-bottle rootfs + SSH key, then boot. + base_dir = util.build_base_rootfs_dir(plan.image) + run_dir = util.cache_dir() / "run" / plan.slug + run_dir.mkdir(parents=True, exist_ok=True) + rootfs = run_dir / "rootfs.ext4" + util.build_rootfs_ext4(base_dir, rootfs) + private_key, pubkey = util.generate_keypair(run_dir) + + vm = firecracker_vm.boot( + name=plan.container_name, + rootfs=rootfs, + tap=slot.iface, + guest_ip=slot.guest_ip, + host_ip=slot.host_ip, + pubkey=pubkey, + run_dir=run_dir, + ) + stack.callback(vm.terminate) + firecracker_vm.wait_for_ssh(vm, private_key) + + # Authoritative fail-closed egress-boundary check, before the + # agent runs: prove the VM cannot reach the host directly. + isolation_probe.verify_isolation(private_key, slot.guest_ip) + + bottle = FirecrackerBottle( + plan.container_name, + private_key=private_key, + guest_ip=slot.guest_ip, + guest_env=_agent_guest_env(plan, slot.host_ip), + agent_command=plan.agent_command, + agent_prompt_mode=plan.agent_prompt_mode, + agent_provider_template=plan.agent_provider_template, + terminal_title=( + f"{plan.spec.label} ({plan.spec.agent_name})" + if plan.spec.label else plan.spec.agent_name + ), + terminal_color=plan.spec.color, + agent_workdir=plan.workspace_plan.workdir, + ) + bottle.prompt_path = provision(plan, bottle) + + yield bottle + finally: + teardown() + + +def _mint_certs(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan: + egress_ca_host, egress_ca_cert_only = egress_tls_init(egress_state_dir(plan.slug)) + egress_plan = dataclasses.replace( + plan.egress_plan, + mitmproxy_ca_host_path=egress_ca_host, + mitmproxy_ca_cert_only_host_path=egress_ca_cert_only, + ) + return dataclasses.replace(plan, egress_plan=egress_plan) + + +def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan: + _docker_build(SIDECAR_BUNDLE_IMAGE, _REPO_DIR, dockerfile=SIDECAR_BUNDLE_DOCKERFILE) + committed = read_committed_image(plan.slug) + if committed and _image_exists(committed): + info(f"using committed image {committed!r}") + return dataclasses.replace( + plan, + agent_provision=dataclasses.replace(plan.agent_provision, image=committed), + ) + _docker_build(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path) + return plan + + +def _provision_git_gate_keys(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan: + if not plan.git_gate_plan.upstreams: + return plan + git_gate_plan = provision_git_gate_dynamic_keys( + plan.manifest.bottle, plan.git_gate_plan, git_gate_state_dir(plan.slug), + ) + return dataclasses.replace(plan, git_gate_plan=git_gate_plan) + + +def _stamp_agent_urls( + plan: FirecrackerBottlePlan, host_ip: str, +) -> FirecrackerBottlePlan: + proxy_url = f"http://{host_ip}:{EGRESS_PORT}" + supervise_url = ( + f"http://{host_ip}:{SUPERVISE_PORT}/" if plan.supervise_plan is not None else "" + ) + git_gate_url = ( + f"http://{host_ip}:{_GIT_HTTP_PORT}" if plan.git_gate_plan.upstreams else "" + ) + return dataclasses.replace( + plan, + agent_proxy_url=proxy_url, + agent_git_gate_url=git_gate_url, + agent_supervise_url=supervise_url, + ) + + +# --- sidecar bundle (Docker) ---------------------------------------- + +def _start_sidecar_bundle( + plan: FirecrackerBottlePlan, sidecar_name: str, host_ip: str, +) -> None: + argv = ["docker", "run", "--name", sidecar_name, "--detach", "--rm", + "-e", f"BOT_BOTTLE_SIDECAR_DAEMONS={','.join(_sidecar_daemons(plan))}"] + for entry in _sidecar_env_entries(plan): + argv += ["-e", entry] + for host_path, container_path, read_only in _sidecar_mounts(plan): + argv += ["-v", f"{host_path}:{container_path}{':ro' if read_only else ''}"] + # Publish on the slot's host TAP IP at fixed ports — each bottle + # has a distinct host_ip, so fixed ports never collide, and the VM + # reaches them at a stable, well-known address (its only route out). + for port in _sidecar_ports(plan): + argv += ["-p", f"{host_ip}:{port}:{port}"] + argv.append(SIDECAR_BUNDLE_IMAGE) + + effective_env = {**dict(os.environ), **plan.agent_provision.provisioned_env} + token_values = egress_resolve_token_values( + plan.egress_plan.token_env_map, effective_env, + ) + env = {**os.environ, **token_values} + info(f"docker run sidecar bundle {sidecar_name} (published on {host_ip})") + result = subprocess.run(argv, capture_output=True, text=True, env=env, check=False) + if result.returncode != 0: + die(f"docker run for sidecar bundle {sidecar_name} failed: " + f"{(result.stderr or '').strip() or ''}") + + +def _sidecar_daemons(plan: FirecrackerBottlePlan) -> tuple[str, ...]: + daemons = ["egress"] + if plan.git_gate_plan.upstreams: + daemons += ["git-gate", "git-http"] + if plan.supervise_plan is not None: + daemons.append("supervise") + return tuple(daemons) + + +def _sidecar_ports(plan: FirecrackerBottlePlan) -> tuple[int, ...]: + ports = [EGRESS_PORT] + if plan.git_gate_plan.upstreams: + ports.append(_GIT_HTTP_PORT) + if plan.supervise_plan is not None: + ports.append(SUPERVISE_PORT) + return tuple(ports) + + +def _sidecar_env_entries(plan: FirecrackerBottlePlan) -> tuple[str, ...]: + env: list[str] = list(egress_sidecar_env_entries(plan.egress_plan)) + if plan.git_gate_plan.upstreams: + env.append(f"BOT_BOTTLE_GIT_GATE_READY_FILE={_GIT_GATE_READY_FILE}") + if plan.supervise_plan is not None: + env += [ + f"SUPERVISE_BOTTLE_SLUG={plan.slug}", + f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", + f"SUPERVISE_PORT={SUPERVISE_PORT}", + ] + return tuple(env) + + +def _sidecar_mounts( + plan: FirecrackerBottlePlan, +) -> tuple[tuple[str, str, bool], ...]: + mounts: list[tuple[str, str, bool]] = [] + ep = plan.egress_plan + mounts.append((str(ep.mitmproxy_ca_host_path.parent), + str(Path(EGRESS_CA_IN_CONTAINER).parent), False)) + if ep.routes: + mounts.append((str(ep.routes_path.parent), + str(Path(EGRESS_ROUTES_IN_CONTAINER).parent), True)) + sp = plan.supervise_plan + if sp is not None: + mounts.append((str(sp.db_path.parent), + str(Path(DB_PATH_IN_CONTAINER).parent), False)) + return tuple(mounts) + + +def _stage_git_gate(plan: FirecrackerBottlePlan, sidecar_name: str) -> None: + gp = plan.git_gate_plan + if not gp.upstreams: + return + _docker_exec(sidecar_name, [ + "mkdir", "-p", + str(Path(GIT_GATE_HOOK_IN_CONTAINER).parent), + GIT_GATE_CREDS_DIR_IN_CONTAINER, "/git", + str(Path(_GIT_GATE_READY_FILE).parent), + ]) + for host_path, container_path in _git_gate_files(plan): + _docker_cp(host_path, f"{sidecar_name}:{container_path}") + _docker_exec(sidecar_name, [ + "sh", "-c", + f"chmod 755 {GIT_GATE_ENTRYPOINT_IN_CONTAINER} " + f"{GIT_GATE_HOOK_IN_CONTAINER} {GIT_GATE_ACCESS_HOOK_IN_CONTAINER} && " + f"chmod 600 {GIT_GATE_CREDS_DIR_IN_CONTAINER}/* && " + f"touch {_GIT_GATE_READY_FILE}", + ]) + + +def _git_gate_files(plan: FirecrackerBottlePlan) -> tuple[tuple[str, str], ...]: + gp = plan.git_gate_plan + files: list[tuple[str, str]] = [ + (str(gp.entrypoint_script), GIT_GATE_ENTRYPOINT_IN_CONTAINER), + (str(gp.hook_script), GIT_GATE_HOOK_IN_CONTAINER), + (str(gp.access_hook_script), GIT_GATE_ACCESS_HOOK_IN_CONTAINER), + ] + for upstream in gp.upstreams: + files.append((expand_tilde(upstream.identity_file), + f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{upstream.name}-key")) + if upstream.known_hosts_file: + files.append((str(upstream.known_hosts_file), + f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{upstream.name}-known_hosts")) + return tuple(files) + + +# --- agent guest env ------------------------------------------------- + +def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str]: + """Env injected into every agent/exec call over SSH. The VM has no + baked process env (it just runs init), so the proxy/CA/git/supervise + wiring is applied per-invocation.""" + proxy_url = f"http://{host_ip}:{EGRESS_PORT}" + no_proxy = f"localhost,127.0.0.1,{host_ip}" + env: dict[str, str] = { + "HTTPS_PROXY": proxy_url, "HTTP_PROXY": proxy_url, + "https_proxy": proxy_url, "http_proxy": proxy_url, + "NO_PROXY": no_proxy, "no_proxy": no_proxy, + "NODE_EXTRA_CA_CERTS": AGENT_CA_PATH, + "SSL_CERT_FILE": AGENT_CA_BUNDLE, + "REQUESTS_CA_BUNDLE": AGENT_CA_BUNDLE, + } + if plan.agent_git_gate_url: + env["GIT_GATE_URL"] = plan.agent_git_gate_url + if plan.agent_supervise_url: + env["MCP_SUPERVISE_URL"] = plan.agent_supervise_url + for entry in egress_agent_env_entries(plan.egress_plan): + key, _, value = entry.partition("=") + env[key] = value + env.update(plan.agent_provision.guest_env) + # Forwarded (bare-name) env: resolve host values now, since the VM + # can't inherit them from a `docker run --env NAME`. + for name in plan.forwarded_env: + value = os.environ.get(name) + if value is not None: + env[name] = value + return env + + +# --- docker helpers -------------------------------------------------- + +def _docker_build(ref: str, context: str, *, dockerfile: str = "") -> None: + info(f"docker build {ref}") + args = ["docker", "build", "-t", ref] + if dockerfile: + if not os.path.isabs(dockerfile): + dockerfile = os.path.join(context, dockerfile) + args += ["-f", dockerfile] + args.append(context) + result = subprocess.run(args, check=False) + if result.returncode != 0: + die(f"docker build for {ref!r} failed") + + +def _image_exists(ref: str) -> bool: + return subprocess.run( + ["docker", "image", "inspect", ref], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, + ).returncode == 0 + + +def _force_remove_container(name: str) -> None: + subprocess.run( + ["docker", "rm", "-f", name], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, + ) + + +def _docker_exec(name: str, argv: list[str]) -> None: + result = subprocess.run( + ["docker", "exec", name, *argv], capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + die(f"docker exec in {name} failed: " + f"{(result.stderr or '').strip() or ''}") + + +def _docker_cp(host_path: str, dest: str) -> None: + result = subprocess.run( + ["docker", "cp", host_path, dest], capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + die(f"docker cp {host_path} -> {dest} failed: " + f"{(result.stderr or '').strip() or ''}") diff --git a/bot_bottle/backend/firecracker/netpool.py b/bot_bottle/backend/firecracker/netpool.py new file mode 100644 index 0000000..2d02417 --- /dev/null +++ b/bot_bottle/backend/firecracker/netpool.py @@ -0,0 +1,243 @@ +"""Firecracker network pool: constants, IP math, allocation, and the +config renderers (shell command + NixOS module) shown to operators. + +The Firecracker backend needs a privileged one-time network setup: +a pool of point-to-point TAP devices (owned by the invoking user, so +`./cli.py start` never needs root) and a dedicated nftables table that +isolates every VM. This module is the single source of truth for the +pool parameters — the shell script (`scripts/firecracker-netpool.sh`), +the printed NixOS snippet, and the backend's fail-closed preflight all +derive from the constants here so they can't drift. + +Topology (per slot i): + * TAP ``bbfc{i}`` — no shared bridge, so no docker0 / virbr0 / cni0 + / br-* collisions. + * a /31 host<->guest link: host = base + 2i (the gateway the VM + routes through), guest = base + 2i + 1 (the VM's address). + * isolation via ``table inet bot_bottle_fc``: a VM reaches only its + own sidecar (DNAT'd from the host TAP IP) and nothing else. + +The default IP block is RFC-6598 shared address space (100.64.0.0/10), +which is purpose-built to avoid colliding with LAN/VPN private ranges. +""" + +from __future__ import annotations + +import fcntl +import ipaddress +import os +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import IO + +from ...log import die + + +# Interface names are capped at 15 chars (IFNAMSIZ-1); "bbfc" + a small +# index stays well under that and is distinctive enough to grep for. +IFACE_PREFIX = os.environ.get("BOT_BOTTLE_FC_IFACE_PREFIX", "bbfc") +NFT_TABLE = "bot_bottle_fc" + + +def pool_size() -> int: + return int(os.environ.get("BOT_BOTTLE_FC_POOL_SIZE", "8")) + + +def ip_base() -> str: + return os.environ.get("BOT_BOTTLE_FC_IP_BASE", "100.64.0.0") + + +# Sidecar ports the VM reaches at its host-side TAP IP. Kept in sync +# with the backend constants (egress 9099, supervise 9100, git-http +# 9420); rendered into the setup output for operator visibility. +SIDECAR_PORTS = (9099, 9100, 9420) + + +@dataclass(frozen=True) +class Slot: + """One pool slot: a TAP device and its host/guest /31 addresses.""" + + index: int + iface: str + host_ip: str + guest_ip: str + + @property + def guest_cidr(self) -> str: + """Guest address with the /31 prefix, for the kernel `ip=` arg.""" + return f"{self.guest_ip}/31" + + +def slot(index: int) -> Slot: + base = int(ipaddress.IPv4Address(ip_base())) + return Slot( + index=index, + iface=f"{IFACE_PREFIX}{index}", + host_ip=str(ipaddress.IPv4Address(base + 2 * index)), + guest_ip=str(ipaddress.IPv4Address(base + 2 * index + 1)), + ) + + +def all_slots() -> list[Slot]: + return [slot(i) for i in range(pool_size())] + + +# --- fail-closed verification --------------------------------------- + +def _run_ok(argv: list[str]) -> bool: + """Run a probe command, treating a missing binary as failure + (rather than crashing) so callers can stay fail-closed.""" + try: + return subprocess.run( + argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + except FileNotFoundError: + return False + + +def nft_table_present() -> bool: + """True iff the isolation table exists in the active nftables + backend. The backend's preflight treats absence as fatal — the VM + must not boot without its egress boundary in place. + + Listing may require root; when it can't be confirmed here the + launch path falls back to an empirical post-boot isolation probe. + Returns False (fail-closed) if `nft` is unavailable.""" + return _run_ok(["nft", "list", "table", "inet", NFT_TABLE]) + + +def tap_present(iface: str) -> bool: + # `ip link show` is unprivileged, so the TAP-pool check is reliable + # for the non-root launcher. + return _run_ok(["ip", "link", "show", iface]) + + +def missing_taps() -> list[str]: + """Pool TAPs that the one-time setup has not created yet.""" + return [s.iface for s in all_slots() if not tap_present(s.iface)] + + +# --- allocation ------------------------------------------------------ + +def _lock_dir() -> Path: + d = Path.home() / ".cache" / "bot-bottle" / "firecracker" / "pool" + d.mkdir(parents=True, exist_ok=True) + return d + + +def allocate(slug: str) -> tuple[Slot, IO[str]]: + """Claim a free pool slot for one bottle. Returns the slot and the + held lock file — the caller keeps it open for the VM's lifetime and + closes it on teardown; the flock auto-releases if the launcher + crashes, so a slot is never leaked. Dies when the pool is + exhausted. + + A per-slot `flock` (rather than inspecting running processes) makes + allocation race-free across concurrent launches without a central + registry: the first launcher to grab the lock owns the slot.""" + del slug # logged by the caller; allocation is purely lock-driven + for s in all_slots(): + lock_path = _lock_dir() / f"{s.iface}.lock" + handle = open(lock_path, "w", encoding="utf-8") + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + handle.close() + continue + return s, handle + die(f"Firecracker TAP pool exhausted ({pool_size()} slots, all in " + f"use). Stop a running bottle or raise BOT_BOTTLE_FC_POOL_SIZE " + f"and re-run `./cli.py firecracker setup`.") + raise AssertionError("unreachable") + + +# --- config renderers (shown by `./cli.py firecracker setup`) -------- + +def render_shell_setup() -> str: + """The imperative command for non-NixOS hosts.""" + env = _nondefault_env() + prefix = f"{env} " if env else "" + return f"sudo {prefix}./scripts/firecracker-netpool.sh up" + + +def render_nixos_module() -> str: + """A paste-ready NixOS snippet, derived from the live constants. + + Uses systemd-networkd tap netdevs (so TAPs are only reconfigured + when this config changes, not torn down on every rebuild — which + would drop running VMs) and `networking.nftables.tables.` + (an independent table that coexists with Docker's iptables rules + rather than replacing the whole ruleset).""" + slots = all_slots() + owner = os.environ.get("USER", "youruser") + + netdevs = "\n".join( + f''' "10-{s.iface}" = {{ + netdevConfig = {{ Name = "{s.iface}"; Kind = "tap"; }}; + tapConfig = {{ User = "{owner}"; }}; + }};''' + for s in slots + ) + networks = "\n".join( + f''' "10-{s.iface}" = {{ + matchConfig.Name = "{s.iface}"; + address = [ "{s.host_ip}/31" ]; + networkConfig.ConfigureWithoutCarrier = true; + }};''' + for s in slots + ) + + return f"""# bot-bottle Firecracker network pool ({len(slots)} slots, base {ip_base()}). +# Generated by `./cli.py firecracker setup`; owner user = {owner!r}. +{{ ... }}: +{{ + boot.kernel.sysctl."net.ipv4.ip_forward" = 1; + + systemd.network.enable = true; + systemd.network.netdevs = {{ +{netdevs} + }}; + systemd.network.networks = {{ +{networks} + }}; + + # Independent table — does not touch Docker/ufw/firewalld rules. + # A bottle VM (traffic from a bbfc* TAP) may reach only its own + # sidecar (DNAT'd from the host TAP IP); everything else is dropped, + # so there is no egress except through the sidecar proxy. + networking.nftables.enable = true; + networking.nftables.tables."{NFT_TABLE}" = {{ + family = "inet"; + content = '' + chain forward {{ + type filter hook forward priority -10; policy accept; + iifname != "{IFACE_PREFIX}*" return + ct state established,related accept + ct status dnat accept + drop + }} + chain input {{ + type filter hook input priority -10; policy accept; + iifname != "{IFACE_PREFIX}*" return + ct state established,related accept + drop + }} + ''; + }}; +}} +""" + + +def _nondefault_env() -> str: + """Render any non-default pool env overrides so the printed shell + command reproduces the operator's current settings.""" + pairs = [] + if os.environ.get("BOT_BOTTLE_FC_POOL_SIZE"): + pairs.append(f"BOT_BOTTLE_FC_POOL_SIZE={pool_size()}") + if os.environ.get("BOT_BOTTLE_FC_IP_BASE"): + pairs.append(f"BOT_BOTTLE_FC_IP_BASE={ip_base()}") + if os.environ.get("BOT_BOTTLE_FC_IFACE_PREFIX"): + pairs.append(f"BOT_BOTTLE_FC_IFACE_PREFIX={IFACE_PREFIX}") + return " ".join(pairs) diff --git a/bot_bottle/backend/firecracker/resolve_plan.py b/bot_bottle/backend/firecracker/resolve_plan.py new file mode 100644 index 0000000..6292175 --- /dev/null +++ b/bot_bottle/backend/firecracker/resolve_plan.py @@ -0,0 +1,47 @@ +"""Prepare step for the Firecracker backend.""" + +from __future__ import annotations + +from pathlib import Path + +from ...agent_provider import AgentProvisionPlan +from ...egress import EgressPlan +from ...env import ResolvedEnv +from ...git_gate import GitGatePlan +from ...manifest import Manifest +from ...supervise import SupervisePlan +from .. import BottleSpec +from . import util +from .bottle_plan import FirecrackerBottlePlan + + +def preflight() -> None: + util.require_firecracker() + + +def build_guest_env(resolved_env: ResolvedEnv) -> dict[str, str]: + return dict(resolved_env.literals) + + +def resolve_plan( + spec: BottleSpec, + manifest: Manifest, + slug: str, + resolved_env: ResolvedEnv, + agent_provision_plan: AgentProvisionPlan, + egress_plan: EgressPlan, + supervise_plan: SupervisePlan | None, + git_gate_plan: GitGatePlan, + stage_dir: Path, +) -> FirecrackerBottlePlan: + return FirecrackerBottlePlan( + spec=spec, + manifest=manifest, + stage_dir=stage_dir, + slug=slug, + forwarded_env=dict(resolved_env.forwarded), + git_gate_plan=git_gate_plan, + egress_plan=egress_plan, + supervise_plan=supervise_plan, + agent_provision=agent_provision_plan, + ) diff --git a/bot_bottle/backend/firecracker/util.py b/bot_bottle/backend/firecracker/util.py new file mode 100644 index 0000000..1b72dd6 --- /dev/null +++ b/bot_bottle/backend/firecracker/util.py @@ -0,0 +1,296 @@ +"""Host-side primitives for the Firecracker backend. + +Covers the pieces that don't need root at launch time: locating the +firecracker binary / guest kernel / injected dropbear, the fail-closed +preflight (KVM + kernel + isolation table + TAP pool must all be +present before a VM boots), the rootless rootfs pipeline +(`docker export` -> `mke2fs -d`, no mount), and per-bottle SSH key +generation. + +The privileged network setup (TAP pool + nft table) is a one-time +operator step — see `netpool.py`, `scripts/firecracker-netpool.sh`, +and `./cli.py firecracker setup`. +""" + +from __future__ import annotations + +import os +import platform +import shutil +import subprocess +from pathlib import Path + +from ...log import die, info, warn +from . import netpool + + +# Guest agent images are Debian-family with USER node; the VM is +# reached over SSH as root (init drops the pubkey into both root's and +# node's authorized_keys) and commands `runuser` down to node. +GUEST_SSH_USER = "root" + +# `/dev/kvm` must exist and be openable by the invoking user. +_KVM_DEVICE = "/dev/kvm" + + +def cache_dir() -> Path: + d = Path( + os.environ.get( + "BOT_BOTTLE_FC_CACHE", + str(Path.home() / ".cache" / "bot-bottle" / "firecracker"), + ) + ) + d.mkdir(parents=True, exist_ok=True) + return d + + +def kernel_path() -> Path: + return Path(os.environ.get("BOT_BOTTLE_FC_KERNEL", str(cache_dir() / "vmlinux"))) + + +def dropbear_path() -> Path: + """The static dropbear injected into every guest rootfs as its SSH + server. Must be statically linked — the guest has none of the + host's shared libraries.""" + return Path( + os.environ.get("BOT_BOTTLE_FC_DROPBEAR", str(cache_dir() / "dropbear")) + ) + + +# --- availability + preflight --------------------------------------- + +def is_linux() -> bool: + return platform.system() == "Linux" + + +def is_available() -> bool: + """Cheap capability probe used by cross-backend enumeration — + firecracker on PATH, on Linux, with KVM. Does not check the + (operator-provisioned) kernel / pool, so an available-but-unset + host still shows up and gets an actionable error at launch.""" + return ( + is_linux() + and shutil.which("firecracker") is not None + and os.path.exists(_KVM_DEVICE) + ) + + +def require_firecracker() -> None: + """Fail-closed preflight. Every check that gates the security + boundary (the isolation table, the TAP pool) errors rather than + booting a VM without it.""" + if not is_linux(): + die("firecracker backend is only supported on Linux (KVM). " + "On macOS use --backend=macos-container.") + if shutil.which("firecracker") is None: + info("Install: https://github.com/firecracker-microvm/firecracker/releases") + die("firecracker not found on PATH") + _require_kvm() + if not kernel_path().is_file(): + die(f"guest kernel not found at {kernel_path()}. Set " + "BOT_BOTTLE_FC_KERNEL or place a vmlinux there.") + if not dropbear_path().is_file(): + die(f"static dropbear not found at {dropbear_path()}. Set " + "BOT_BOTTLE_FC_DROPBEAR or cache one there.") + if shutil.which("mke2fs") is None: + die("mke2fs (e2fsprogs) not found — required to build guest rootfs") + _require_network_pool() + + +def _require_kvm() -> None: + if not os.path.exists(_KVM_DEVICE): + die(f"{_KVM_DEVICE} is missing. Enable KVM (load kvm-intel/kvm-amd; " + "confirm virtualization is on in firmware).") + if not os.access(_KVM_DEVICE, os.R_OK | os.W_OK): + die(f"{_KVM_DEVICE} exists but is not accessible. Add your user to " + "the `kvm` group and re-login.") + + +def _require_network_pool() -> None: + """Fail-closed on the parts we can check unprivileged; defer the + rest to the post-boot isolation probe. + + The TAP pool is verified here (`ip link show` is unprivileged). The + nft table can only be *confirmed present* here when `nft` is + queryable — which usually needs root — so a missing/absent nft is + not treated as fatal at this stage: the authoritative check is the + empirical isolation probe run after boot, before the agent starts + (see isolation_probe.verify_isolation). What we must never do is + boot without the TAP pool.""" + missing = netpool.missing_taps() + if missing: + die(f"network pool incomplete — missing TAP devices: " + f"{', '.join(missing)}.\n ./cli.py firecracker setup") + if shutil.which("nft") is not None and not netpool.nft_table_present(): + # nft is queryable and says the table is absent — that's a + # definite, catchable misconfiguration; fail early. + warn(f"isolation table `inet {netpool.NFT_TABLE}` not found via nft. " + "If this is a permissions issue it will be re-checked " + "empirically after boot; otherwise run: " + "./cli.py firecracker setup") + + +# --- rootfs pipeline (rootless) ------------------------------------- + +def docker_image_id(ref: str) -> str: + """The image's content digest, used as the rootfs cache key.""" + result = subprocess.run( + ["docker", "image", "inspect", "--format", "{{.Id}}", ref], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0 or not result.stdout.strip(): + die(f"docker image inspect for {ref!r} failed: " + f"{(result.stderr or '').strip() or ''}") + return result.stdout.strip().replace("sha256:", "")[:16] + + +def build_base_rootfs_dir(image_ref: str) -> Path: + """Export the agent image's filesystem and inject the guest init + + static dropbear. Cached by image digest — the per-bottle bits + (authorized_keys, IP) are passed at boot via the kernel cmdline, so + this tree carries nothing bottle-specific and is safely shared. + + Returns the prepared directory (read as the `mke2fs -d` source).""" + digest = docker_image_id(image_ref) + base = cache_dir() / "rootfs" / digest + ready = base / ".bb-ready" + if ready.is_file(): + return base + + if base.exists(): + shutil.rmtree(base, ignore_errors=True) + base.mkdir(parents=True) + + info(f"exporting {image_ref} rootfs -> {base}") + cid = subprocess.run( + ["docker", "create", image_ref, "sleep", "infinity"], + capture_output=True, text=True, check=False, + ) + if cid.returncode != 0 or not cid.stdout.strip(): + die(f"docker create {image_ref!r} failed: {cid.stderr.strip()}") + container = cid.stdout.strip() + try: + export = subprocess.Popen( + ["docker", "export", container], stdout=subprocess.PIPE, + ) + untar = subprocess.run( + ["tar", "-x", "-C", str(base)], stdin=export.stdout, check=False, + ) + export.wait() + if export.returncode != 0 or untar.returncode != 0: + die(f"exporting rootfs for {image_ref!r} failed") + finally: + subprocess.run( + ["docker", "rm", "-f", container], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, + ) + + _inject_guest_boot(base) + ready.write_text("ok\n") + return base + + +def _inject_guest_boot(rootfs: Path) -> None: + """Drop the static dropbear and the PID-1 init into the rootfs.""" + shutil.copy2(dropbear_path(), rootfs / "bb-dropbear") + os.chmod(rootfs / "bb-dropbear", 0o755) + init = rootfs / "bb-init" + init.write_text(_GUEST_INIT) + os.chmod(init, 0o755) + + +def build_rootfs_ext4(base_dir: Path, out_path: Path, *, slack_mib: int = 1024) -> None: + """Build a fresh, writable ext4 for one bottle from the cached base + dir. Rootless: `mke2fs -d` populates the image from a directory + without mounting. Each call produces an independent disk, so the + shared base dir stays untouched and concurrent bottles don't race.""" + used_mib = _dir_size_mib(base_dir) + size_mib = used_mib + slack_mib + out_path.parent.mkdir(parents=True, exist_ok=True) + result = subprocess.run( + ["mke2fs", "-q", "-t", "ext4", "-d", str(base_dir), "-F", + str(out_path), f"{size_mib}M"], + capture_output=True, text=True, check=False, + ) + if result.returncode != 0: + die(f"mke2fs for {out_path} failed: {result.stderr.strip()}") + + +def _dir_size_mib(path: Path) -> int: + result = subprocess.run( + ["du", "-sm", str(path)], capture_output=True, text=True, check=False, + ) + try: + return int(result.stdout.split()[0]) + except (ValueError, IndexError): + return 2048 + + +# --- per-bottle SSH keys -------------------------------------------- + +def generate_keypair(dest_dir: Path) -> tuple[Path, str]: + """Mint a per-bottle ed25519 keypair. Returns (private_key_path, + public_key_line). The public line is injected into the guest via + the boot cmdline; the private key authenticates host->guest SSH.""" + dest_dir.mkdir(parents=True, exist_ok=True) + key = dest_dir / "bottle_id_ed25519" + if key.exists(): + key.unlink() + (dest_dir / "bottle_id_ed25519.pub").unlink(missing_ok=True) + subprocess.run( + ["ssh-keygen", "-t", "ed25519", "-N", "", "-q", "-f", str(key), + "-C", "bot-bottle-firecracker"], + check=True, + ) + pub = (dest_dir / "bottle_id_ed25519.pub").read_text().strip() + return key, pub + + +def ssh_base_argv(private_key: Path, guest_ip: str) -> list[str]: + """Common SSH options for host->guest control. The VM is ephemeral + and per-bottle, so host-key TOFU is meaningless — pin no known_hosts + and skip the check rather than accumulate churn.""" + return [ + "ssh", + "-i", str(private_key), + "-o", "StrictHostKeyChecking=no", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "LogLevel=ERROR", + "-o", "ConnectTimeout=5", + f"{GUEST_SSH_USER}@{guest_ip}", + ] + + +# PID-1 init injected into every guest. Kept dependency-light: relies +# only on coreutils + a POSIX shell (present in the Debian-family agent +# images). The kernel `ip=` cmdline configures eth0 before init runs, +# so no iproute2 is needed. The per-bottle SSH pubkey arrives base64 on +# the cmdline; dropbear generates ephemeral host keys with -R. +_GUEST_INIT = r"""#!/bin/sh +# bot-bottle Firecracker guest init (PID 1). +mount -t proc proc /proc 2>/dev/null +mount -t sysfs sys /sys 2>/dev/null +mount -t devtmpfs dev /dev 2>/dev/null +mkdir -p /dev/pts && mount -t devpts devpts /dev/pts 2>/dev/null +mount -o remount,rw / 2>/dev/null + +# Install the per-bottle SSH pubkey from the kernel cmdline. +KEY=$(sed -n 's/.*bb_pubkey=\([^ ]*\).*/\1/p' /proc/cmdline | base64 -d 2>/dev/null) +if [ -n "$KEY" ]; then + for home in /root /home/node; do + mkdir -p "$home/.ssh" + printf '%s\n' "$KEY" > "$home/.ssh/authorized_keys" + chmod 700 "$home/.ssh" + chmod 600 "$home/.ssh/authorized_keys" + done + chown -R node:node /home/node/.ssh 2>/dev/null || true +fi + +mkdir -p /etc/dropbear /run +# -R: generate host keys on demand. -E: log to stderr (guest console). +/bb-dropbear -R -E -p 22 2>/dev/null & + +# Reap zombies as PID 1. dropbear is always a child, so `wait` blocks +# rather than busy-looping. +while : ; do wait ; done +""" diff --git a/bot_bottle/backend/freeze.py b/bot_bottle/backend/freeze.py index d9a8013..90b394c 100644 --- a/bot_bottle/backend/freeze.py +++ b/bot_bottle/backend/freeze.py @@ -90,11 +90,15 @@ def get_freezer(backend_name: str) -> Freezer: if resolved == "macos-container": from .macos_container.freezer import MacosContainerFreezer return MacosContainerFreezer() + if resolved == "firecracker": + from .firecracker.freezer import FirecrackerFreezer + return FirecrackerFreezer() if resolved == "smolmachines": from .smolmachines.freezer import SmolmachinesFreezer return SmolmachinesFreezer() die( - f"commit is only supported for docker, macos-container, and " - f"smolmachines; backend {backend_name!r} has no freezer" + f"commit is only supported for docker, macos-container, " + f"firecracker, and smolmachines; backend {backend_name!r} has " + f"no freezer" ) raise AssertionError("unreachable") diff --git a/bot_bottle/cli/__init__.py b/bot_bottle/cli/__init__.py index dfc9cb0..6c6db94 100644 --- a/bot_bottle/cli/__init__.py +++ b/bot_bottle/cli/__init__.py @@ -16,6 +16,7 @@ from . import list as _list_mod from .cleanup import cmd_cleanup from .commit import cmd_commit from .edit import cmd_edit +from .firecracker import cmd_firecracker from .info import cmd_info from .init import cmd_init from .resume import cmd_resume @@ -28,6 +29,7 @@ COMMANDS = { "cleanup": cmd_cleanup, "commit": cmd_commit, "edit": cmd_edit, + "firecracker": cmd_firecracker, "info": cmd_info, "init": cmd_init, "list": cmd_list, @@ -43,6 +45,7 @@ def usage() -> None: sys.stderr.write(" cleanup stop and remove all active bot-bottle containers\n") sys.stderr.write(" commit snapshot a running bottle's container state to a Docker image\n") sys.stderr.write(" edit open an agent in vim for editing\n") + sys.stderr.write(" firecracker one-time network setup for the Firecracker backend\n") sys.stderr.write(" info print env, skills, and prompt details for a named agent\n") sys.stderr.write(" init interactively create a new agent and add it to bot-bottle.json\n") sys.stderr.write(" list list available agents or active containers\n") diff --git a/bot_bottle/cli/firecracker.py b/bot_bottle/cli/firecracker.py new file mode 100644 index 0000000..3d37736 --- /dev/null +++ b/bot_bottle/cli/firecracker.py @@ -0,0 +1,79 @@ +"""`firecracker` CLI command — one-time network setup helper. + +`./cli.py firecracker setup` prints the host-appropriate config for the +privileged, one-time network pool (TAP devices + isolation nftables +table) the Firecracker backend needs. On NixOS it prints a declarative +module to paste into your config (imperative rules don't survive +nixos-rebuild); elsewhere it prints the sudo command for the bundled +setup script. `./cli.py firecracker status` reports what's present. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from ..backend.firecracker import netpool + + +def _is_nixos() -> bool: + if Path("/etc/NIXOS").exists(): + return True + try: + return "ID=nixos" in Path("/etc/os-release").read_text() + except OSError: + return False + + +def _cmd_setup() -> int: + slots = netpool.all_slots() + sys.stderr.write( + f"Firecracker network pool: {len(slots)} slots " + f"({slots[0].iface}..{slots[-1].iface}), base {netpool.ip_base()}.\n" + f"This is a one-time privileged setup (needs root once).\n\n" + ) + if _is_nixos(): + sys.stderr.write( + "Detected NixOS. Paste this module into your configuration and " + "`nixos-rebuild switch` (imperative rules would not survive a " + "rebuild):\n\n" + ) + sys.stdout.write(netpool.render_nixos_module()) + else: + sys.stderr.write("Run the one-time setup as root:\n\n") + sys.stdout.write(netpool.render_shell_setup() + "\n") + sys.stderr.write( + "\n(On NixOS, use the declarative module instead — this host " + "was not detected as NixOS.)\n" + ) + return 0 + + +def _cmd_status() -> int: + ok = True + if netpool.nft_table_present(): + sys.stderr.write(f"nft table inet {netpool.NFT_TABLE}: present\n") + else: + sys.stderr.write(f"nft table inet {netpool.NFT_TABLE}: MISSING\n") + ok = False + missing = netpool.missing_taps() + total = netpool.pool_size() + if missing: + sys.stderr.write(f"TAP pool: {total - len(missing)}/{total} present " + f"(missing: {', '.join(missing)})\n") + ok = False + else: + sys.stderr.write(f"TAP pool: {total}/{total} present\n") + if not ok: + sys.stderr.write("\nRun: ./cli.py firecracker setup\n") + return 0 if ok else 1 + + +def cmd_firecracker(args: list[str]) -> int: + sub = args[0] if args else "" + if sub == "setup": + return _cmd_setup() + if sub == "status": + return _cmd_status() + sys.stderr.write("usage: firecracker {setup|status}\n") + return 2 diff --git a/scripts/firecracker-netpool.sh b/scripts/firecracker-netpool.sh new file mode 100755 index 0000000..5bd394e --- /dev/null +++ b/scripts/firecracker-netpool.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# One-time privileged network setup for the Firecracker backend. +# +# Creates a pool of point-to-point TAP devices (owned by the invoking +# user so the backend can open them without root at launch) and a +# dedicated nftables table that isolates every VM: a bottle VM can +# reach only its own sidecar (published on the host-side TAP IP) and +# nothing else on the host or network. +# +# Why a pool + one-time setup: creating a TAP and assigning it an IP +# needs CAP_NET_ADMIN. Pre-creating user-owned, pre-addressed TAPs +# means `./cli.py start` never needs root. The nft table is static +# (keyed on the `bbfc*` interface wildcard), so it covers every slot +# without per-launch changes. +# +# Design notes: +# * No shared bridge — each slot is an isolated /31 host<->guest link, +# so there are no bridge name/subnet collisions with docker0, +# virbr0 (libvirt), cni0 (k8s) or br-* (docker networks). +# * Own nftables table `bot_bottle_fc` — independent of the iptables +# filter/nat tables Docker/ufw/firewalld use, so nothing is stomped. +# * Default IP block is RFC-6598 shared address space (100.64.0.0/10), +# purpose-built to not collide with LAN/VPN private ranges. +# +# NixOS: imperative rules here do NOT survive nixos-rebuild. Use the +# declarative module in docs/firecracker/nixos-netpool.nix instead. +# +# Usage: +# sudo ./scripts/firecracker-netpool.sh up +# sudo ./scripts/firecracker-netpool.sh down +# ./scripts/firecracker-netpool.sh status +# +# Env overrides (must match the backend's util.py constants): +# BOT_BOTTLE_FC_POOL_SIZE number of slots (default 8) +# BOT_BOTTLE_FC_IP_BASE base IPv4 of the /31 pool (default 100.64.0.0) +# BOT_BOTTLE_FC_IFACE_PREFIX TAP name prefix (default bbfc) +# BOT_BOTTLE_FC_OWNER owning user (default $SUDO_USER or $USER) + +set -euo pipefail + +POOL_SIZE="${BOT_BOTTLE_FC_POOL_SIZE:-8}" +IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-100.64.0.0}" +PREFIX="${BOT_BOTTLE_FC_IFACE_PREFIX:-bbfc}" +OWNER="${BOT_BOTTLE_FC_OWNER:-${SUDO_USER:-$USER}}" +TABLE="bot_bottle_fc" + +# Sidecar ports (must match the backend). egress=9099, supervise=9100, +# git-http=9420. Reached by the VM at its host-side TAP IP. +SIDECAR_PORTS="9099,9100,9420" + +# --- IP math --------------------------------------------------------- +# Slot i occupies the /31 {base+2i, base+2i+1}: host = base+2i (the +# gateway the VM routes through), guest = base+2i+1 (the VM's address). +_ip_to_int() { + local IFS=. ; read -r a b c d <<<"$1" ; echo $(( (a<<24) + (b<<16) + (c<<8) + d )) +} +_int_to_ip() { + local n=$1 ; echo "$(( (n>>24)&255 )).$(( (n>>16)&255 )).$(( (n>>8)&255 )).$(( n&255 ))" +} +host_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 )); } +guest_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 + 1 )); } +iface() { echo "${PREFIX}$1"; } + +require_root() { + if [ "$(id -u)" -ne 0 ]; then + echo "error: '$1' needs root (run under sudo)" >&2 + exit 1 + fi +} + +cmd_up() { + require_root up + echo "firecracker net pool: $POOL_SIZE slots, base $IP_BASE, owner $OWNER" + + # VM->sidecar traffic is DNAT'd to the sidecar container and + # forwarded, so forwarding must be enabled (Docker also sets this). + sysctl -qw net.ipv4.ip_forward=1 + + for i in $(seq 0 $((POOL_SIZE-1))); do + local dev host + dev="$(iface "$i")" ; host="$(host_ip "$i")" + if ip link show "$dev" >/dev/null 2>&1; then + ip link set "$dev" down 2>/dev/null || true + ip tuntap del dev "$dev" mode tap 2>/dev/null || true + fi + ip tuntap add dev "$dev" mode tap user "$OWNER" + ip addr add "$host/31" dev "$dev" + ip link set "$dev" up + echo " $dev host=$host guest=$(guest_ip "$i") owner=$OWNER" + done + + _install_nft + echo "nftables table inet $TABLE installed (fail-closed boundary)" + echo "done." +} + +_install_nft() { + # Own table: dropping only matches our bbfc* interfaces, so no other + # tool's traffic is affected. Priority -10 runs before Docker's + # filter hooks (priority 0); a drop here is terminal for the packet. + # + # forward: VM egress is DNAT'd to the sidecar (established via + # `ct status dnat`); return traffic via `ct state established`. + # Anything else from a VM is dropped -> no route to the internet + # or the rest of the host except through the sidecar proxy. + # input: a VM never needs host-local delivery (its sidecar is + # reached via DNAT->forward), so drop all direct input from VMs + # -> host services bound on 0.0.0.0 are unreachable from the VM. + nft -f - </dev/null || true + for i in $(seq 0 $((POOL_SIZE-1))); do + local dev ; dev="$(iface "$i")" + if ip link show "$dev" >/dev/null 2>&1; then + ip link set "$dev" down 2>/dev/null || true + ip tuntap del dev "$dev" mode tap 2>/dev/null || true + echo " removed $dev" + fi + done + echo "done." +} + +cmd_status() { + echo "table inet $TABLE:" + nft list table inet "$TABLE" 2>/dev/null || echo " (absent)" + echo "taps:" + for i in $(seq 0 $((POOL_SIZE-1))); do + local dev ; dev="$(iface "$i")" + if ip -brief addr show "$dev" >/dev/null 2>&1; then + ip -brief addr show "$dev" | sed 's/^/ /' + fi + done +} + +case "${1:-}" in + up) cmd_up ;; + down) cmd_down ;; + status) cmd_status ;; + *) echo "usage: $0 {up|down|status}" >&2 ; exit 2 ;; +esac diff --git a/tests/unit/test_backend_selection.py b/tests/unit/test_backend_selection.py index 4db3932..3511d49 100644 --- a/tests/unit/test_backend_selection.py +++ b/tests/unit/test_backend_selection.py @@ -64,6 +64,24 @@ class TestGetBottleBackend(unittest.TestCase): b = get_bottle_backend() self.assertEqual("smolmachines", b.name) + def test_default_firecracker_when_macos_unavailable_but_fc_available(self): + class _FakeBackend: + def __init__(self, name: str, available: bool) -> None: + self.name = name + self._available = available + + def is_available(self) -> bool: + return self._available + + with patch.dict(os.environ, {}, clear=True), \ + patch.object(backend_mod, "_BACKENDS", { + "macos-container": _FakeBackend("macos-container", False), + "firecracker": _FakeBackend("firecracker", True), + "smolmachines": _FakeBackend("smolmachines", True), + }): + b = get_bottle_backend() + self.assertEqual("firecracker", b.name) + def test_unknown_dies(self): with patch.object(backend_mod, "die", side_effect=SystemExit("die")): with self.assertRaises(SystemExit): @@ -73,7 +91,7 @@ class TestGetBottleBackend(unittest.TestCase): class TestKnownBackendNames(unittest.TestCase): def test_returns_backends_sorted(self): self.assertEqual( - ("docker", "macos-container", "smolmachines"), + ("docker", "firecracker", "macos-container", "smolmachines"), known_backend_names(), ) diff --git a/tests/unit/test_firecracker_backend.py b/tests/unit/test_firecracker_backend.py new file mode 100644 index 0000000..00321dc --- /dev/null +++ b/tests/unit/test_firecracker_backend.py @@ -0,0 +1,187 @@ +"""Unit: Firecracker backend helpers. + +Covers the pieces that run without KVM/root: pool IP math + config +renderers + allocation, the SSH-backed bottle's argv construction, and +the VM boot-arg assembly. +""" + +from __future__ import annotations + +import base64 +import os +import unittest +from pathlib import Path +from typing import Any, cast +from unittest.mock import patch + +from bot_bottle.backend.firecracker import firecracker_vm, netpool +from bot_bottle.backend.firecracker.bottle import FirecrackerBottle + + +def _bottle(**kw: Any) -> FirecrackerBottle: + defaults = dict( + name="bot-bottle-dev-abc", + private_key=Path("/tmp/key"), + guest_ip="100.64.0.1", + ) + defaults.update(kw) + return FirecrackerBottle(**defaults) # type: ignore[arg-type] + + +class TestNetpoolSlots(unittest.TestCase): + def test_slot_ip_math_31_pairs(self): + with patch.dict(os.environ, {"BOT_BOTTLE_FC_IP_BASE": "100.64.0.0"}): + s0, s1 = netpool.slot(0), netpool.slot(1) + self.assertEqual(("bbfc0", "100.64.0.0", "100.64.0.1"), + (s0.iface, s0.host_ip, s0.guest_ip)) + self.assertEqual(("bbfc1", "100.64.0.2", "100.64.0.3"), + (s1.iface, s1.host_ip, s1.guest_ip)) + + def test_guest_cidr_is_31(self): + self.assertEqual("100.64.0.1/31", netpool.slot(0).guest_cidr) + + def test_pool_size_env_override(self): + with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "4"}): + self.assertEqual(4, netpool.pool_size()) + self.assertEqual(4, len(netpool.all_slots())) + + +class TestNetpoolRenderers(unittest.TestCase): + def test_nixos_module_has_table_and_taps(self): + with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "2"}): + out = netpool.render_nixos_module() + self.assertIn('networking.nftables.tables."bot_bottle_fc"', out) + self.assertIn('Name = "bbfc0"', out) + self.assertIn('Name = "bbfc1"', out) + self.assertIn("net.ipv4.ip_forward", out) + # fail-closed isolation rules present + self.assertIn('iifname != "bbfc*" return', out) + self.assertIn("ct status dnat accept", out) + + def test_shell_setup_reflects_overrides(self): + with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "3"}): + out = netpool.render_shell_setup() + self.assertIn("BOT_BOTTLE_FC_POOL_SIZE=3", out) + self.assertIn("firecracker-netpool.sh up", out) + + +class TestNetpoolAllocation(unittest.TestCase): + def test_allocate_is_mutually_exclusive(self): + with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "1"}): + # First allocation grabs the only slot; the lock is held + # until the handle is closed, so a second call must fail + # over (and here, exhaust the pool). + slot, lock = netpool.allocate("first") + self.addCleanup(lock.close) + self.assertEqual("bbfc0", slot.iface) + with patch.object(netpool, "die", + side_effect=SystemExit("exhausted")): + with self.assertRaises(SystemExit): + netpool.allocate("second") + + def test_allocate_releases_on_close(self): + with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "1"}): + slot, lock = netpool.allocate("first") + lock.close() + # Slot is free again after close. + slot2, lock2 = netpool.allocate("second") + self.addCleanup(lock2.close) + self.assertEqual(slot.iface, slot2.iface) + + +class TestBottleAgentArgv(unittest.TestCase): + def test_interactive_uses_ssh_tty_and_runuser(self): + argv = _bottle().agent_argv([], tty=True) + self.assertEqual("ssh", argv[0]) + self.assertIn("-t", argv) + self.assertIn("100.64.0.1", " ".join(argv)) + idx = argv.index("--") + self.assertEqual(["runuser", "-u", "node", "--"], argv[idx + 1:idx + 5]) + self.assertIn("claude", argv) + + def test_non_interactive_has_no_tty(self): + argv = _bottle().agent_argv([], tty=False) + self.assertEqual("ssh", argv[0]) + self.assertNotIn("-t", argv) + + def test_appends_extra_args_after_command(self): + argv = _bottle().agent_argv( + ["--dangerously-skip-permissions", "--continue"], tty=False, + ) + idx = argv.index("claude") + self.assertEqual( + ["claude", "--dangerously-skip-permissions", "--continue"], + argv[idx:], + ) + + def test_prompt_file_flag_injected(self): + argv = _bottle( + prompt_path_in_guest="/home/node/.bot-bottle-prompt.txt", + ).agent_argv(["--continue"], tty=False) + idx = argv.index("claude") + self.assertEqual( + ["claude", "--continue", + "--append-system-prompt-file", "/home/node/.bot-bottle-prompt.txt"], + argv[idx:], + ) + + def test_workdir_wraps_command(self): + argv = _bottle(agent_workdir="/home/node/workspace").agent_argv([], tty=False) + self.assertIn("sh", argv) + joined = " ".join(argv) + self.assertIn("cd /home/node/workspace", joined) + + def test_guest_env_injected(self): + argv = _bottle(guest_env={"HTTPS_PROXY": "http://100.64.0.0:9099"}).agent_argv( + [], tty=False, + ) + self.assertIn("HTTPS_PROXY=http://100.64.0.0:9099", argv) + self.assertIn("HOME=/home/node", argv) + self.assertIn("USER=node", argv) + + +class TestSetupScriptConsistency(unittest.TestCase): + """The shell setup script duplicates the pool defaults; keep them in + lockstep with the Python constants so the two setup paths agree.""" + + def _script(self) -> str: + root = Path(__file__).resolve().parent.parent.parent + return (root / "scripts" / "firecracker-netpool.sh").read_text() + + def test_defaults_match_python(self): + script = self._script() + with patch.dict(os.environ, {}, clear=True): + self.assertIn(f'POOL_SIZE:-{netpool.pool_size()}', script) + self.assertIn(f'BOT_BOTTLE_FC_IP_BASE:-{netpool.ip_base()}', script) + self.assertIn(f'IFACE_PREFIX:-{netpool.IFACE_PREFIX}', script) + + def test_table_name_and_ports_match(self): + script = self._script() + self.assertIn(f'TABLE="{netpool.NFT_TABLE}"', script) + for port in netpool.SIDECAR_PORTS: + self.assertIn(str(port), script) + + +class TestBootArgs(unittest.TestCase): + def test_boot_args_carry_ip_and_pubkey(self): + args = firecracker_vm._boot_args( + guest_ip="100.64.0.1", host_ip="100.64.0.0", pubkey="ssh-ed25519 AAAA x", + ) + self.assertIn("ip=100.64.0.1::100.64.0.0:255.255.255.254::eth0:off", args) + self.assertIn("init=/bb-init", args) + b64 = base64.b64encode(b"ssh-ed25519 AAAA x").decode() + self.assertIn(f"bb_pubkey={b64}", args) + + def test_config_has_rootfs_and_tap(self): + cfg = cast(Any, firecracker_vm._config( + rootfs=Path("/run/rootfs.ext4"), tap="bbfc0", + guest_ip="100.64.0.1", host_ip="100.64.0.0", pubkey="k", + vcpus=2, mem_mib=2048, guest_mac="06:00:AC:10:00:02", + )) + self.assertEqual("/run/rootfs.ext4", cfg["drives"][0]["path_on_host"]) + self.assertFalse(cfg["drives"][0]["is_read_only"]) + self.assertEqual("bbfc0", cfg["network-interfaces"][0]["host_dev_name"]) + + +if __name__ == "__main__": + unittest.main() -- 2.52.0 From c07ebca867f884b4be7934015265ba90b2008045 Mon Sep 17 00:00:00 2001 From: didericis Date: Sat, 11 Jul 2026 13:24:47 -0400 Subject: [PATCH 02/13] feat(backend): remove smolmachines; firecracker is the Linux default Delete the smolmachines backend (the whole bot_bottle/backend/smolmachines package and its tests). It had fatal Linux issues (TSI networking under sustained use, exec-channel contention, no SIGWINCH) and is superseded by the Firecracker backend (issue #342). Backend selection now: - default is macos-container on macOS, firecracker on KVM-capable Linux hosts, and docker as the last resort (was smolmachines). - firecracker is selected on a KVM host even when the `firecracker` binary isn't installed, so start routes through its preflight and prints an install pointer (same UX as require_container), instead of silently falling back. Split is_host_capable() (Linux + KVM) out of is_available() (adds the binary check) to drive this. Retarget the cross-backend tests (parity, print-parity, prepare, workspace, freezer, selection) from smolmachines to firecracker rather than dropping the coverage. Remove docker.util.image_id/save, which only smolmachines used. Update README/AGENTS/example bottles and stale comments; historical docs/prds are left as a point-in-time record. BREAKING: BOT_BOTTLE_BACKEND=smolmachines now errors as unknown. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck --- AGENTS.md | 10 +- Dockerfile.sidecars | 2 +- README.md | 25 +- bot_bottle/agent_provider.py | 6 +- bot_bottle/backend/__init__.py | 45 +- bot_bottle/backend/docker/cleanup.py | 9 +- bot_bottle/backend/docker/enumerate.py | 3 +- bot_bottle/backend/docker/util.py | 30 - bot_bottle/backend/firecracker/backend.py | 8 + bot_bottle/backend/firecracker/bottle.py | 3 +- bot_bottle/backend/firecracker/enumerate.py | 4 +- bot_bottle/backend/firecracker/util.py | 17 +- bot_bottle/backend/freeze.py | 8 +- bot_bottle/backend/print_util.py | 7 +- bot_bottle/backend/resolve_common.py | 4 +- bot_bottle/backend/smolmachines/__init__.py | 15 - bot_bottle/backend/smolmachines/backend.py | 101 ---- bot_bottle/backend/smolmachines/bottle.py | 217 ------- .../smolmachines/bottle_cleanup_plan.py | 55 -- .../backend/smolmachines/bottle_plan.py | 109 ---- bot_bottle/backend/smolmachines/cleanup.py | 159 ----- .../backend/smolmachines/egress_apply.py | 21 - bot_bottle/backend/smolmachines/enumerate.py | 123 ---- bot_bottle/backend/smolmachines/freezer.py | 145 ----- bot_bottle/backend/smolmachines/launch.py | 521 ---------------- .../backend/smolmachines/local_registry.py | 236 -------- .../backend/smolmachines/loopback_alias.py | 314 ---------- .../smolmachines/provision/__init__.py | 12 - bot_bottle/backend/smolmachines/pty_resize.py | 151 ----- .../backend/smolmachines/resolve_plan.py | 83 --- .../backend/smolmachines/sidecar_bundle.py | 242 -------- bot_bottle/backend/smolmachines/smolvm.py | 268 --------- bot_bottle/backend/smolmachines/util.py | 82 --- bot_bottle/bottle_state.py | 6 +- bot_bottle/cli/cleanup.py | 10 +- bot_bottle/cli/commit.py | 8 +- bot_bottle/cli/list.py | 5 +- bot_bottle/cli/supervise.py | 5 - bot_bottle/egress_entrypoint.sh | 4 +- bot_bottle/git_gate_render.py | 6 +- bot_bottle/git_http_backend.py | 7 +- bot_bottle/supervise.py | 2 +- examples/bottles/claude.md | 6 +- examples/bottles/dev.md | 3 +- tests/integration/test_sandbox_escape.py | 32 +- .../test_smolmachines_bundle_bringup.py | 111 ---- tests/integration/test_smolmachines_launch.py | 223 ------- .../test_smolmachines_smolvm_smoke.py | 64 -- tests/unit/test_backend_freezer.py | 56 +- tests/unit/test_backend_parity.py | 83 ++- tests/unit/test_backend_prepare.py | 20 +- tests/unit/test_backend_selection.py | 59 +- tests/unit/test_backend_terminal.py | 4 +- tests/unit/test_backend_workspace.py | 14 +- tests/unit/test_bottle_state.py | 6 +- tests/unit/test_cli_cleanup_cross_backend.py | 36 +- tests/unit/test_cli_commit.py | 6 +- tests/unit/test_cli_start_backend_flag.py | 6 +- tests/unit/test_docker_cleanup.py | 14 +- tests/unit/test_docker_util_image.py | 43 +- tests/unit/test_egress_entrypoint.py | 6 +- tests/unit/test_plan_print_parity.py | 31 +- tests/unit/test_provision_git.py | 7 +- tests/unit/test_smolmachines_bottle.py | 195 ------ tests/unit/test_smolmachines_cleanup.py | 150 ----- tests/unit/test_smolmachines_launch_image.py | 192 ------ .../unit/test_smolmachines_local_registry.py | 250 -------- .../unit/test_smolmachines_loopback_alias.py | 430 -------------- tests/unit/test_smolmachines_provision.py | 559 ------------------ tests/unit/test_smolmachines_pty_resize.py | 164 ----- .../unit/test_smolmachines_sidecar_bundle.py | 226 ------- tests/unit/test_smolmachines_smolvm.py | 284 --------- tests/unit/test_smolmachines_util.py | 155 ----- 73 files changed, 305 insertions(+), 6218 deletions(-) delete mode 100644 bot_bottle/backend/smolmachines/__init__.py delete mode 100644 bot_bottle/backend/smolmachines/backend.py delete mode 100644 bot_bottle/backend/smolmachines/bottle.py delete mode 100644 bot_bottle/backend/smolmachines/bottle_cleanup_plan.py delete mode 100644 bot_bottle/backend/smolmachines/bottle_plan.py delete mode 100644 bot_bottle/backend/smolmachines/cleanup.py delete mode 100644 bot_bottle/backend/smolmachines/egress_apply.py delete mode 100644 bot_bottle/backend/smolmachines/enumerate.py delete mode 100644 bot_bottle/backend/smolmachines/freezer.py delete mode 100644 bot_bottle/backend/smolmachines/launch.py delete mode 100644 bot_bottle/backend/smolmachines/local_registry.py delete mode 100644 bot_bottle/backend/smolmachines/loopback_alias.py delete mode 100644 bot_bottle/backend/smolmachines/provision/__init__.py delete mode 100644 bot_bottle/backend/smolmachines/pty_resize.py delete mode 100644 bot_bottle/backend/smolmachines/resolve_plan.py delete mode 100644 bot_bottle/backend/smolmachines/sidecar_bundle.py delete mode 100644 bot_bottle/backend/smolmachines/smolvm.py delete mode 100644 bot_bottle/backend/smolmachines/util.py delete mode 100644 tests/integration/test_smolmachines_bundle_bringup.py delete mode 100644 tests/integration/test_smolmachines_launch.py delete mode 100644 tests/integration/test_smolmachines_smolvm_smoke.py delete mode 100644 tests/unit/test_smolmachines_bottle.py delete mode 100644 tests/unit/test_smolmachines_cleanup.py delete mode 100644 tests/unit/test_smolmachines_launch_image.py delete mode 100644 tests/unit/test_smolmachines_local_registry.py delete mode 100644 tests/unit/test_smolmachines_loopback_alias.py delete mode 100644 tests/unit/test_smolmachines_provision.py delete mode 100644 tests/unit/test_smolmachines_pty_resize.py delete mode 100644 tests/unit/test_smolmachines_sidecar_bundle.py delete mode 100644 tests/unit/test_smolmachines_smolvm.py delete mode 100644 tests/unit/test_smolmachines_util.py diff --git a/AGENTS.md b/AGENTS.md index ff7ca05..6920191 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,11 +9,11 @@ host. A Python CLI (entry point `cli.py`, package `bot_bottle/`) orchestrates the runtime lifecycle and the copying of skills and env vars into it. The default backend on compatible macOS hosts is macos-container: agents and sidecar bundles run through Apple's `container` CLI without -requiring Docker. The smolmachines backend remains available with -`BOT_BOTTLE_BACKEND=smolmachines` or `--backend=smolmachines`; agents -run in a libkrun micro-VM, while the sidecar bundle still uses Docker. -The legacy Docker backend remains available with `BOT_BOTTLE_BACKEND=docker` -or `--backend=docker`. +requiring Docker. On KVM-capable Linux hosts the default is firecracker: +agents run in a Firecracker microVM reached over SSH on a point-to-point +TAP, while the sidecar bundle still uses Docker. The legacy Docker +backend remains available with `BOT_BOTTLE_BACKEND=docker` or +`--backend=docker`. ## Goals diff --git a/Dockerfile.sidecars b/Dockerfile.sidecars index 89b33b3..e581856 100644 --- a/Dockerfile.sidecars +++ b/Dockerfile.sidecars @@ -24,7 +24,7 @@ # Exposed ports inside the container: # 9099 egress (mitmproxy, agent-facing HTTPS proxy) # 9418 git-gate (git-daemon) -# 9420 git-gate smart HTTP (smolmachines agent-facing transport) +# 9420 git-gate smart HTTP (VM-backend agent-facing transport) # 9100 supervise (MCP HTTP) # Stage 1: gitleaks binary. The upstream gitleaks image is alpine diff --git a/README.md b/README.md index 6f24346..7cad616 100644 --- a/README.md +++ b/README.md @@ -25,14 +25,14 @@ - **Provider templates (Claude, Codex)** — `Dockerfile.claude` / `Dockerfile.codex`, or a bottle-supplied Dockerfile. Claude auth via long-lived OAuth token; Codex via opt-in host device-auth forwarding. - **gVisor auto-detect** — on Linux hosts where `runsc` is registered with Docker, every bottle launches under it for a userspace syscall barrier; no manifest config required. - **Apple Container backend (macOS default when available)** — runs the agent and sidecar bundle with Apple's `container` CLI, using a host-only agent network plus a separate sidecar egress network. -- **Smolmachines backend** — runs the agent in a libkrun micro-VM while the sidecar bundle stays in Docker. TSI and smolmachines DNS filtering close the raw DNS exfiltration gap that exists in the legacy Docker backend. Runs on macOS (Hypervisor.framework) and Linux (KVM, `/dev/kvm`). -- **Legacy Docker backend** — still available for examples, CI, and hosts without Apple Container via `BOT_BOTTLE_BACKEND=docker` or `--backend=docker`. +- **Firecracker backend (Linux default when available)** — runs the agent in a KVM Firecracker microVM reached over SSH on a point-to-point TAP, with the sidecar bundle in Docker. A dedicated, fail-closed `nftables` table isolates the guest, closing the raw DNS/IP exfiltration gap that exists in the legacy Docker backend. Requires KVM (`/dev/kvm`) and a one-time privileged network-pool setup. +- **Legacy Docker backend** — still available for examples, CI, and hosts without Apple Container or KVM via `BOT_BOTTLE_BACKEND=docker` or `--backend=docker`. ## Architecture On the default macOS Apple Container backend, a bottle is an agent container on a host-only internal network plus a sidecar bundle attached to both that internal network and a NAT egress network. The agent gets HTTP(S)_PROXY and CA bundle env vars pointing at the sidecar's internal-network IP, so HTTP/HTTPS traffic flows through the sidecar instead of direct egress. `bottle.git` / git-gate is intentionally deferred on this backend until a safe Apple Container key-delivery path exists. -On the smolmachines backend, a bottle is an agent micro-VM plus a Docker sidecar bundle for egress, git-gate, and supervise. The VM reaches the sidecars through a per-bottle loopback alias allowed by TSI; smolmachines handles DNS filtering below the guest OS. +On the Firecracker backend, a bottle is an agent microVM plus a Docker sidecar bundle for egress, git-gate, and supervise. The VM reaches the sidecars over a per-bottle point-to-point TAP link; a dedicated fail-closed `nftables` table (`inet bot_bottle_fc`) confines the guest to that link, so nothing leaves the box except through the sidecars. The TAP pool and nft table are provisioned once (root); per-launch needs no privilege. On the legacy Docker backend, the same logical bottle is two containers per agent: an `agent` container and a `sidecars` container. They share a per-agent Docker `--internal` network; the agent has no default route off-box. @@ -71,25 +71,24 @@ When the agent exits, `cli.py` tears down every sidecar and both networks; nothi ## Quickstart -On compatible macOS hosts, the default backend requires Apple's `container` CLI and does not require Docker. The smolmachines backend requires Docker on the host for the sidecar bundle plus `smolvm` (macOS or Linux). The legacy Docker backend requires Docker. Claude bottles also need a long-lived Claude Code OAuth token (`claude setup-token`) exported as `BOT_BOTTLE_CLAUDE_OAUTH_TOKEN`. +On compatible macOS hosts, the default backend requires Apple's `container` CLI and does not require Docker. The Firecracker backend (Linux) requires Docker on the host for the sidecar bundle plus the `firecracker` binary and KVM. The legacy Docker backend requires Docker. Claude bottles also need a long-lived Claude Code OAuth token (`claude setup-token`) exported as `BOT_BOTTLE_CLAUDE_OAUTH_TOKEN`. -Use `BOT_BOTTLE_BACKEND=docker ./cli.py start ` on hosts where Apple Container is not installed and Docker is the desired backend. +Use `BOT_BOTTLE_BACKEND=docker ./cli.py start ` on hosts where neither Apple Container nor KVM is available and Docker is the desired backend. -### smolmachines on Linux +### Firecracker on Linux -The smolmachines backend runs on Linux as well as macOS. On Linux, `smolvm`/libkrun use KVM, so the host needs: +On Linux, a KVM-capable host defaults to the Firecracker backend. It needs: - **`/dev/kvm`** present and accessible. Load `kvm-intel` or `kvm-amd` (and enable virtualization in BIOS/firmware). The invoking user must be in the `kvm` group: `sudo usermod -aG kvm "$USER"` then re-login. bot-bottle preflights this and reports exactly what's missing. -- **`smolvm`** on `PATH`: `curl -sSL https://smolmachines.com/install.sh | sh`. -- **Docker** for the sidecar bundle and image build, same as macOS. - -Per-bottle isolation works the same as macOS without any `ifconfig`/sudo step — all of `127.0.0.0/8` is already loopback on Linux, so each bottle's sidecar bundle is published on its own `127.0.0.` and TSI's allowlist is scoped to that `/32`. +- **`firecracker`** on `PATH`: grab a release from . Start flows print this pointer when the binary is missing. +- **Docker** for the sidecar bundle and image build. +- **A one-time privileged network setup** — the per-bottle TAP pool plus the fail-closed `nftables` isolation table. Run `./cli.py firecracker setup` for the host-appropriate command (a declarative module on NixOS, a `sudo` script elsewhere); `./cli.py firecracker status` reports what's present. ```sh -BOT_BOTTLE_BACKEND=smolmachines ./cli.py start +BOT_BOTTLE_BACKEND=firecracker ./cli.py start ``` -> **NixOS:** enable `virtualisation.docker`, ensure the KVM module is loaded (`boot.kernelModules = [ "kvm-intel" ];` or `kvm-amd`), and add your user to the `kvm` and `docker` groups. If you run bottles from a Gitea Actions runner, use a `host`-label runner so Docker, `smolvm`, and `/dev/kvm` are all reachable from the job. `smolvm` isn't in nixpkgs — install the release binary (pin the version) and put it on the runner's `PATH`. +> **NixOS:** enable `virtualisation.docker`, ensure the KVM module is loaded (`boot.kernelModules = [ "kvm-intel" ];` or `kvm-amd`), and add your user to the `kvm` and `docker` groups. Apply the network-pool module from `./cli.py firecracker setup` and `nixos-rebuild switch` (imperative nft/TAP rules don't survive a rebuild). `firecracker` isn't in nixpkgs by default as a user binary — install the release binary (pin the version) and put it on `PATH`. ```sh ./cli.py start # builds the image on first run, drops you into claude diff --git a/bot_bottle/agent_provider.py b/bot_bottle/agent_provider.py index feb8a50..398f239 100644 --- a/bot_bottle/agent_provider.py +++ b/bot_bottle/agent_provider.py @@ -227,9 +227,9 @@ class AgentProvider(ABC): from .backend.util import AGENT_CA_PATH, log_ca_fingerprint, select_ca_cert from .log import die cert_host_path, label = select_ca_cert(plan.egress_plan) - # Ensure the target directory exists. smolvm's pack step may not - # preserve the empty /usr/local/share/ca-certificates/ directory - # on Linux; mkdir -p is idempotent and safe for all backends. + # Ensure the target directory exists. A backend's rootfs build + # may not preserve the empty /usr/local/share/ca-certificates/ + # directory; mkdir -p is idempotent and safe for all backends. bottle.exec("mkdir -p /usr/local/share/ca-certificates", user="root") bottle.cp_in(str(cert_host_path), AGENT_CA_PATH) r = bottle.exec( diff --git a/bot_bottle/backend/__init__.py b/bot_bottle/backend/__init__.py index 41559c5..9080681 100644 --- a/bot_bottle/backend/__init__.py +++ b/bot_bottle/backend/__init__.py @@ -26,9 +26,9 @@ backend exposes five methods: Selection is driven by `--backend` on `start` or BOT_BOTTLE_BACKEND (env var). When neither is set, compatible macOS hosts default to -`macos-container`; Linux hosts with Firecracker + KVM default to -`firecracker`; otherwise `smolmachines`. Per PRD 0003 the manifest -does not carry a backend field; the host picks. +`macos-container`; Linux hosts with KVM default to `firecracker`; +otherwise `docker`. Per PRD 0003 the manifest does not carry a +backend field; the host picks. """ from __future__ import annotations @@ -98,14 +98,14 @@ class BottlePlan(ABC): @property def git_gate_insteadof_host(self) -> str: """Host (and optional port) used in git-gate insteadOf URLs. - Docker uses the compose-network DNS alias; smolmachines - overrides with a loopback IP:port since TSI has no DNS.""" + Docker uses the compose-network DNS alias; VM backends may + override with an IP:port when the guest has no DNS.""" return "git-gate" @property def git_gate_insteadof_scheme(self) -> str: """URL scheme for git-gate insteadOf rewrites. 'git' for - Docker (git daemon); 'http' for smolmachines (HTTP proxy + Docker (git daemon); VM backends may override (e.g. 'http' over a published host port).""" return "git" egress_plan: EgressPlan @@ -183,8 +183,8 @@ class BottleCleanupPlan(ABC): class ExecResult: """Captured result of `Bottle.exec`. Backend-neutral: the Docker impl populates it from a `subprocess.CompletedProcess`, but a - future fly/smolmachines backend could populate it from any source - that produces a returncode + captured streams.""" + VM backend could populate it from any source that produces a + returncode + captured streams.""" returncode: int stdout: str @@ -202,7 +202,7 @@ class ActiveAgent: of sidecar daemons currently up for this bottle (`egress`, `git-gate`, `supervise`); the dashboard uses it to gate edit verbs. `backend_name` is the matching key in - `_BACKENDS` (`docker` / `smolmachines` / `macos-container`) — used by the active- + `_BACKENDS` (`docker` / `firecracker` / `macos-container`) — used by the active- list rendering to disambiguate and by the dashboard's re-attach path.""" @@ -507,7 +507,7 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]): MCP entry inside the guest. Default returns "" so backends without supervise support - don't have to implement it. Docker and smolmachines override.""" + don't have to implement it. Docker and firecracker override.""" del plan return "" @@ -524,19 +524,19 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]): def enumerate_active(self) -> Sequence[ActiveAgent]: """Return every currently-running agent on this backend. Empty when none. Backend-specific: docker queries `docker - compose ls`; smolmachines queries `smolvm machine ls --json` - + cross-references its bundle container.""" + compose ls`; firecracker cross-references its running sidecar + containers against per-bottle metadata.""" @classmethod @abstractmethod def is_available(cls) -> bool: """Whether this backend's runtime prerequisites are satisfied - on the current host. Docker → `docker` on PATH; smolmachines - → `smolvm` on PATH. Used by the cross-backend + on the current host. Docker → `docker` on PATH; firecracker → + Linux + KVM. Used by the cross-backend `enumerate_active_agents` / `cmd_cleanup` to skip backends the operator hasn't installed, so a docker-only host doesn't fail when `cli.py list active` walks past - smolmachines.""" + firecracker.""" # Import concrete backend classes AFTER the base types are defined, so @@ -545,7 +545,6 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]): from .docker import DockerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position from .firecracker import FirecrackerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position from .macos_container import MacosContainerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position -from .smolmachines import SmolmachinesBottleBackend # noqa: E402 # pylint: disable=wrong-import-position # Freezer is imported after the backend classes for the same reason: # Freezer.commit_slug constructs ActiveAgent, which must be fully @@ -561,7 +560,6 @@ _BACKENDS: dict[str, BottleBackend[Any, Any]] = { "docker": DockerBottleBackend(), "firecracker": FirecrackerBottleBackend(), "macos-container": MacosContainerBottleBackend(), - "smolmachines": SmolmachinesBottleBackend(), } @@ -574,7 +572,8 @@ def get_bottle_backend( 1. explicit arg (CLI `--backend=` passes through here) 2. BOT_BOTTLE_BACKEND env var 3. `macos-container` on compatible macOS hosts - 4. default `smolmachines` + 4. `firecracker` on KVM-capable Linux hosts + 5. default `docker` Dies with a pointer at the known backends if the chosen name isn't implemented.""" @@ -588,9 +587,13 @@ def get_bottle_backend( def _default_backend_name() -> str: if has_backend("macos-container"): return "macos-container" - if has_backend("firecracker"): + # A KVM-capable Linux host defaults to firecracker even when the + # `firecracker` binary isn't installed yet: selecting it here routes + # start through firecracker's preflight, which prints an install + # pointer, instead of silently falling back to docker. + if FirecrackerBottleBackend.is_host_capable(): return "firecracker" - return "smolmachines" + return "docker" def known_backend_names() -> tuple[str, ...]: @@ -604,7 +607,7 @@ def has_backend(name: str) -> bool: """Whether the named backend's runtime prerequisites are available on the current host. Cross-backend callers (list, cleanup) skip unavailable backends so a docker-only host - doesn't fail when the smolmachines backend isn't installed, + doesn't fail when the firecracker backend isn't usable, and vice versa. Returns False for unknown names so callers can pass diff --git a/bot_bottle/backend/docker/cleanup.py b/bot_bottle/backend/docker/cleanup.py index 079de35..254c63f 100644 --- a/bot_bottle/backend/docker/cleanup.py +++ b/bot_bottle/backend/docker/cleanup.py @@ -18,8 +18,7 @@ scan, just as a fallback bucket alongside the project list. `cleanup` removes everything in the plan. -Active-agent enumeration lives in `backend/docker/enumerate.py` -(mirror of `backend/smolmachines/enumerate.py`). +Active-agent enumeration lives in `backend/docker/enumerate.py`. """ from __future__ import annotations @@ -93,8 +92,8 @@ def _list_orphan_state_dirs( `protected_identities` is the set of slugs that are live in ANY backend — used so this docker-side check doesn't reap a - running smolmachines bottle's state dir (the layout is shared - across both backends).""" + running non-docker bottle's state dir (the layout is shared + across backends).""" state_root = _supervise.bot_bottle_root() / "state" if not state_root.is_dir(): return [] @@ -119,7 +118,7 @@ def prepare_cleanup() -> DockerBottleCleanupPlan: Pulls the union of live identities across backends via `enumerate_active_agents()` so the orphan-state-dir bucket - doesn't include slugs whose smolmachines VM is still up.""" + doesn't include slugs whose non-docker bottle is still up.""" docker_mod.require_docker() projects = list_compose_projects() project_set = set(projects) diff --git a/bot_bottle/backend/docker/enumerate.py b/bot_bottle/backend/docker/enumerate.py index af12af3..0337d5a 100644 --- a/bot_bottle/backend/docker/enumerate.py +++ b/bot_bottle/backend/docker/enumerate.py @@ -1,7 +1,6 @@ """Active-agent enumeration for the docker backend. -Mirrors `backend/smolmachines/enumerate.py`: returns -`ActiveAgent` records the CLI `list active` command and the +Returns `ActiveAgent` records the CLI `list active` command and the dashboard agents pane consume. Empty when docker isn't reachable — gated by `has_backend('docker')` at the cross-backend caller so this module trusts that docker is available when called. diff --git a/bot_bottle/backend/docker/util.py b/bot_bottle/backend/docker/util.py index 8ac0339..6a4abdd 100644 --- a/bot_bottle/backend/docker/util.py +++ b/bot_bottle/backend/docker/util.py @@ -167,36 +167,6 @@ def commit_container(container_name: str, image_tag: str) -> None: info(f"committed {container_name!r} → {image_tag!r}") -def image_id(ref: str) -> str: - """Return the content-addressed image ID (e.g. - `sha256:abcd...`) for `ref`. The smolmachines backend keys its - `.smolmachine` artifact cache on this, so a Dockerfile change - that produces a new image automatically invalidates the cache.""" - r = subprocess.run( - ["docker", "image", "inspect", "--format", "{{.Id}}", ref], - capture_output=True, - text=True, - check=False, - ) - if r.returncode != 0: - die( - f"docker image inspect for {ref!r} failed: " - f"{(r.stderr or '').strip() or ''}" - ) - return r.stdout.strip() - - -def save(ref: str, output: str) -> None: - """`docker save REF -o OUTPUT`. Writes a tarball of the image - layers + manifest to the host path. Used by smolmachines - prepare to hand the agent image to a containerized crane that - pushes it to the ephemeral registry — bypassing the docker - daemon's `docker push` (which on Docker Desktop can't reach a - host-loopback registry and refuses plain-HTTP pushes to - non-loopback hosts).""" - subprocess.run(["docker", "save", ref, "-o", output], check=True) - - def _silent_run(cmd: Iterable[str]) -> int: return subprocess.run( list(cmd), diff --git a/bot_bottle/backend/firecracker/backend.py b/bot_bottle/backend/firecracker/backend.py index 8431dca..58d49f9 100644 --- a/bot_bottle/backend/firecracker/backend.py +++ b/bot_bottle/backend/firecracker/backend.py @@ -38,6 +38,14 @@ class FirecrackerBottleBackend( def is_available(cls) -> bool: return _util.is_available() + @classmethod + def is_host_capable(cls) -> bool: + """Linux + KVM, regardless of whether the `firecracker` binary + is installed. Drives default-backend selection so a capable host + without the binary still lands on firecracker and gets an + install pointer at launch.""" + return _util.is_host_capable() + def _preflight(self) -> None: _resolve_plan.preflight() diff --git a/bot_bottle/backend/firecracker/bottle.py b/bot_bottle/backend/firecracker/bottle.py index 3805dc9..aff4fcf 100644 --- a/bot_bottle/backend/firecracker/bottle.py +++ b/bot_bottle/backend/firecracker/bottle.py @@ -4,8 +4,7 @@ Every operation routes through SSH to the guest (dropbear, listening on the TAP link). `exec` pipes a script over stdin and captures output; `cp_in` uses scp; `exec_agent` uses `ssh -t` for an interactive PTY session. `ssh -t` forwards the host terminal's SIGWINCH to the remote -PTY natively, so — unlike the smolmachines backend — no resize bridge -is needed. +PTY natively, so no separate resize bridge is needed. Commands run as the image's `node` user via `runuser`, with HOME/USER/ PATH and the bottle env (HTTPS_PROXY at the sidecar, CA paths, …) set diff --git a/bot_bottle/backend/firecracker/enumerate.py b/bot_bottle/backend/firecracker/enumerate.py index b4a7f42..4c9e90d 100644 --- a/bot_bottle/backend/firecracker/enumerate.py +++ b/bot_bottle/backend/firecracker/enumerate.py @@ -28,8 +28,8 @@ def enumerate_active() -> list[ActiveAgent]: slug = name[len(_SIDECAR_PREFIX):] metadata = read_metadata(slug) if metadata is None or metadata.backend != "firecracker": - # Skip sidecars owned by another backend (docker/smolmachines - # share the container-name prefix). + # Skip sidecars owned by another backend (docker shares the + # container-name prefix). continue out.append(ActiveAgent( backend_name="firecracker", diff --git a/bot_bottle/backend/firecracker/util.py b/bot_bottle/backend/firecracker/util.py index 1b72dd6..4e251fa 100644 --- a/bot_bottle/backend/firecracker/util.py +++ b/bot_bottle/backend/firecracker/util.py @@ -63,16 +63,22 @@ def is_linux() -> bool: return platform.system() == "Linux" +def is_host_capable() -> bool: + """Whether this host *could* run firecracker — Linux with KVM — + regardless of whether the `firecracker` binary is installed. Used + for default-backend selection so a KVM Linux host that hasn't + installed firecracker yet still selects it and gets an install + pointer at launch (see `require_firecracker`), rather than silently + falling back to docker.""" + return is_linux() and os.path.exists(_KVM_DEVICE) + + def is_available() -> bool: """Cheap capability probe used by cross-backend enumeration — firecracker on PATH, on Linux, with KVM. Does not check the (operator-provisioned) kernel / pool, so an available-but-unset host still shows up and gets an actionable error at launch.""" - return ( - is_linux() - and shutil.which("firecracker") is not None - and os.path.exists(_KVM_DEVICE) - ) + return is_host_capable() and shutil.which("firecracker") is not None def require_firecracker() -> None: @@ -83,6 +89,7 @@ def require_firecracker() -> None: die("firecracker backend is only supported on Linux (KVM). " "On macOS use --backend=macos-container.") if shutil.which("firecracker") is None: + info("Firecracker is required but was not found on PATH.") info("Install: https://github.com/firecracker-microvm/firecracker/releases") die("firecracker not found on PATH") _require_kvm() diff --git a/bot_bottle/backend/freeze.py b/bot_bottle/backend/freeze.py index 90b394c..ec4f424 100644 --- a/bot_bottle/backend/freeze.py +++ b/bot_bottle/backend/freeze.py @@ -93,12 +93,8 @@ def get_freezer(backend_name: str) -> Freezer: if resolved == "firecracker": from .firecracker.freezer import FirecrackerFreezer return FirecrackerFreezer() - if resolved == "smolmachines": - from .smolmachines.freezer import SmolmachinesFreezer - return SmolmachinesFreezer() die( - f"commit is only supported for docker, macos-container, " - f"firecracker, and smolmachines; backend {backend_name!r} has " - f"no freezer" + f"commit is only supported for docker, macos-container, and " + f"firecracker; backend {backend_name!r} has no freezer" ) raise AssertionError("unreachable") diff --git a/bot_bottle/backend/print_util.py b/bot_bottle/backend/print_util.py index 4b4ec3d..5f18cdb 100644 --- a/bot_bottle/backend/print_util.py +++ b/bot_bottle/backend/print_util.py @@ -1,9 +1,8 @@ """Shared print helpers for BottlePlan.print implementations. -Lifts the multi-value label printer out of DockerBottlePlan so the -smolmachines backend (and any future backend) renders the same -two-column scannable preflight without duplicating the indent -math.""" +Lifts the multi-value label printer out of DockerBottlePlan so every +backend (and any future backend) renders the same two-column +scannable preflight without duplicating the indent math.""" from __future__ import annotations diff --git a/bot_bottle/backend/resolve_common.py b/bot_bottle/backend/resolve_common.py index d1ad0dc..7270d37 100644 --- a/bot_bottle/backend/resolve_common.py +++ b/bot_bottle/backend/resolve_common.py @@ -1,7 +1,7 @@ -"""Shared helpers used by both backends' resolve_plan steps. +"""Shared helpers used across backends' resolve_plan steps. Each helper owns one well-defined step of the per-bottle plan -resolution so docker and smolmachines don't repeat the same logic. +resolution so the backends don't repeat the same logic. Backend-specific steps (container names, env-file, per-bottle Dockerfile overrides, subnet allocation) stay in the backend's own resolve_plan.py. diff --git a/bot_bottle/backend/smolmachines/__init__.py b/bot_bottle/backend/smolmachines/__init__.py deleted file mode 100644 index 6b65870..0000000 --- a/bot_bottle/backend/smolmachines/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -"""smolmachines bottle backend (PRD 0023). - -Selectable via `BOT_BOTTLE_BACKEND=smolmachines`. Runs each -bottle inside a per-agent microVM (libkrun / Hypervisor.framework -on macOS) with a userspace gvproxy gateway as the egress -primitive. The sidecar bundle (PRD 0024) runs as a host-side -docker container reached only through gvproxy's port-forward list. - -Chunk 1 (this commit) ships the backend skeleton + Smolfile + -gvproxy renderers + preflight check. VM lifecycle, sidecar -bringup, and provisioning land in later chunks.""" - -from .backend import SmolmachinesBottleBackend # noqa: F401 - -__all__ = ["SmolmachinesBottleBackend"] diff --git a/bot_bottle/backend/smolmachines/backend.py b/bot_bottle/backend/smolmachines/backend.py deleted file mode 100644 index f24be61..0000000 --- a/bot_bottle/backend/smolmachines/backend.py +++ /dev/null @@ -1,101 +0,0 @@ -"""SmolmachinesBottleBackend — the smolmachines implementation of -BottleBackend (PRD 0023). - -Per PRD 0050 the per-provider provisioning steps (prompt, skills, -the declarative provision-plan apply, supervise MCP registration) -live on the `AgentProvider` plugin under `bot_bottle/contrib/`. The -smolmachines backend only owns the steps that are about backend -infrastructure: CA install (no-op for now), workspace, git copy-in.""" - -from __future__ import annotations - -from contextlib import contextmanager -from pathlib import Path -from typing import Generator, Sequence - -from ...agent_provider import AgentProvisionPlan -from ...egress import EgressPlan -from ...env import ResolvedEnv -from ...git_gate import GitGatePlan -from ...supervise import SupervisePlan -from ...manifest import Manifest -from .. import ActiveAgent, BottleBackend, BottleSpec -from . import cleanup as _cleanup -from . import enumerate as _enumerate -from . import launch as _launch -from . import resolve_plan as _resolve_plan -from . import smolvm as _smolvm -from .bottle import SmolmachinesBottle -from .bottle_cleanup_plan import SmolmachinesBottleCleanupPlan -from .bottle_plan import SmolmachinesBottlePlan - - -class SmolmachinesBottleBackend( - BottleBackend["SmolmachinesBottlePlan", "SmolmachinesBottleCleanupPlan"] -): - """smolmachines backend. Selected by - `BOT_BOTTLE_BACKEND=smolmachines`.""" - - name = "smolmachines" - - @classmethod - def is_available(cls) -> bool: - """`smolvm` on PATH. The backend additionally needs macOS - for libkrun + TSI, but `enumerate_active` / `cleanup` are - host-shell ops that gracefully no-op on Linux too — the - runtime check happens at `prepare`.""" - return _smolvm.is_available() - - def _preflight(self) -> None: - _resolve_plan.preflight() - - def _build_guest_env(self, resolved_env: ResolvedEnv) -> dict[str, str]: - return _resolve_plan.build_guest_env(resolved_env) - - def _resolve_plan( - self, - spec: BottleSpec, - *, - manifest: Manifest, - slug: str, - resolved_env: ResolvedEnv, - agent_provision_plan: AgentProvisionPlan, - egress_plan: EgressPlan, - git_gate_plan: GitGatePlan, - supervise_plan: SupervisePlan | None, - stage_dir: Path, - ) -> SmolmachinesBottlePlan: - return _resolve_plan.resolve_plan( - spec, - manifest=manifest, - slug=slug, - resolved_env=resolved_env, - agent_provision_plan=agent_provision_plan, - egress_plan=egress_plan, - supervise_plan=supervise_plan, - git_gate_plan=git_gate_plan, - stage_dir=stage_dir, - ) - - @contextmanager - def launch( - self, plan: SmolmachinesBottlePlan - ) -> Generator[SmolmachinesBottle, None, None]: - with _launch.launch(plan, provision=self.provision) as bottle: - yield bottle - - def supervise_mcp_url(self, plan: SmolmachinesBottlePlan) -> str: - """The smolmachines guest reaches the supervise sidecar via a - host-published random port the launch step pinned earlier - (`http://:/`). `agent_supervise_url` - on the plan is "" when the bottle has no sidecar.""" - return plan.agent_supervise_url - - def prepare_cleanup(self) -> SmolmachinesBottleCleanupPlan: - return _cleanup.prepare_cleanup() - - def cleanup(self, plan: SmolmachinesBottleCleanupPlan) -> None: - _cleanup.cleanup(plan) - - def enumerate_active(self) -> Sequence[ActiveAgent]: - return _enumerate.enumerate_active() diff --git a/bot_bottle/backend/smolmachines/bottle.py b/bot_bottle/backend/smolmachines/bottle.py deleted file mode 100644 index c496919..0000000 --- a/bot_bottle/backend/smolmachines/bottle.py +++ /dev/null @@ -1,217 +0,0 @@ -"""SmolmachinesBottle — running-instance handle (PRD 0023 chunk 2d). - -Routes `exec_agent` / `exec` / `cp_in` through `smolvm machine -exec` / `smolvm machine cp`. The handle is yielded by `launch` -and torn down via the surrounding ExitStack on context exit; -`close` is a no-op idempotent alias so the BottleBackend ABC's -context-manager contract is satisfied. - -User context: `smolvm machine exec` runs commands as root in the -VM, but the agent image's USER is `node` and agent CLIs may refuse -to run as root in bypass modes. Both -`exec_agent` and `exec` switch to the requested user (default -`node`) via `runuser -u --` and set `HOME` / `USER` -through `smolvm -e` — avoiding `runuser -l`'s login-shell wiring -(PAM session setup, /etc/profile sourcing) which can hang on a -minimal Debian VM with no PAM session config.""" - -from __future__ import annotations - -import subprocess -import sys -import time -import shlex -from typing import Mapping, cast - -from ...agent_provider import PromptMode, prompt_args -from .. import Bottle, ExecResult -from ..terminal import exec_shell_script -from . import pty_resize as _pty_resize -from . import smolvm as _smolvm - - -# Absolute path to the pty_resize wrapper. Invoke as -# `python ` rather than `python -m ` so the -# wrapper runs regardless of cwd / sys.path — it has no -# bot_bottle.* imports, so it's self-contained. -_PTY_RESIZE_SCRIPT = _pty_resize.__file__ - - -# Per-user env the agent image's USER (node) expects. Some providers -# write session state under the user's home directory; -# bare `runuser -u` inherits root's HOME=/root, which claude -# can't write to. Set HOME / USER explicitly through smolvm -e -# so the child process sees them. -_HOME_FOR = { - "node": "/home/node", - "root": "/root", -} - -_DEFAULT_PATH_FOR = { - # Committed smolmachine snapshots are rebuilt from a rootfs tarball and - # lose Docker image ENV metadata. Restore the provider CLI path here so - # resumed Codex bottles can still find the per-user install. - "node": ( - "/home/node/.local/bin:" - "/home/node/.codex/packages/standalone/current/bin:" - "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - ), - "root": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", -} - - -def _env_assignments_for(user: str, env: Mapping[str, str]) -> list[str]: - home = _HOME_FOR.get(user, f"/home/{user}") - out = [f"HOME={home}", f"USER={user}"] - if "PATH" not in env: - out.append(f"PATH={_DEFAULT_PATH_FOR.get(user, _DEFAULT_PATH_FOR['root'])}") - for k, v in env.items(): - out.append(f"{k}={v}") - return out - - -class SmolmachinesBottle(Bottle): - """Handle returned by `SmolmachinesBottleBackend.launch`. The - underlying VM lifecycle (create / start / stop / delete) lives - on the launch ExitStack — this class only routes runtime - operations to the right `smolvm machine ...` subcommand.""" - - def __init__( - self, - machine_name: str, - *, - prompt_path: str | None = None, - guest_env: Mapping[str, str] | None = None, - agent_command: str = "claude", - agent_prompt_mode: PromptMode = "append_file", - agent_provider_template: str = "claude", - terminal_title: str = "", - terminal_color: str = "", - agent_workdir: str = "/home/node", - ) -> None: - self.name = machine_name - # In-VM path to the agent's prompt file. None when the - # agent declared no prompt (file still exists; we just - # don't pass --append-system-prompt-file). - self.prompt_path = prompt_path - # Env vars the agent process needs (HTTPS_PROXY, - # CLAUDE_CODE_OAUTH_TOKEN, manifest-declared bottle env, …). - # Forwarded on every `smolvm machine exec` via `-e K=V` - # because exec doesn't inherit from machine_create's env. - self._guest_env = dict(guest_env or {}) - self._agent_prompt_mode = agent_prompt_mode - self.agent_command = agent_command - self.terminal_title = terminal_title - self.terminal_color = terminal_color - self.agent_provider_template = agent_provider_template - self.agent_workdir = agent_workdir - - def agent_argv( - self, argv: list[str], *, tty: bool = True, - ) -> list[str]: - flags = ["smolvm", "machine", "exec", "--name", self.name] - if tty: - flags += ["-i", "-t"] - agent_tail = ["env", *_env_assignments_for("node", self._guest_env)] - if self.agent_workdir and self.agent_workdir != _HOME_FOR["node"]: - agent_tail += [ - "sh", "-lc", - f"cd {shlex.quote(self.agent_workdir)} && exec \"$@\"", - "bot-bottle-agent", - ] - agent_tail.append(self.agent_command) - provider_prompt_args = prompt_args( - cast(PromptMode, self._agent_prompt_mode), self.prompt_path, argv=argv, - ) - if cast(PromptMode, self._agent_prompt_mode) == "read_prompt_file": - agent_tail += argv - agent_tail += provider_prompt_args - else: - agent_tail += provider_prompt_args - agent_tail += argv - flags += ["--", "runuser", "-u", "node", "--", *agent_tail] - if not tty: - # No PTY allocated — no SIGWINCH to forward, no resize - # bridge needed. Skip the wrapper so non-interactive - # exec paths (e.g., provisioning shell-outs that - # happen to go through this method) stay light. - return flags - return [ - sys.executable, _PTY_RESIZE_SCRIPT, - self.name, "--", *flags, - ] - - def exec_agent(self, argv: list[str], *, tty: bool = True) -> int: - """Run the selected agent interactively inside the VM as the `node` - user. Inherits the operator's terminal (stdin / stdout / - stderr) so the session feels native. Blocks until the agent - exits; returns the in-VM exit code. - - We bypass the captured-output `machine_exec` helper here - because that one wraps stdout/stderr in pipes — fine for - scripted exec, wrong for an interactive shell. Drop down - to `subprocess.run` with the TTY inherited. - - UID switches via `runuser -u node --` (not `-l`) so we - avoid login-shell wiring. HOME / USER come from `smolvm - -e` instead, which sets them on the process env.""" - agent_argv = self.agent_argv(argv, tty=tty) - script = exec_shell_script(agent_argv, self.terminal_title, self.terminal_color) if tty else None - if script is None: - return subprocess.run(agent_argv, check=False).returncode - # Use sh -c (not -lc) so the script inherits PATH from the calling - # process. sh -l sources login-shell init files (e.g. /etc/profile) - # which may NOT include smolvm's location when it was installed via - # homebrew. The calling process (./cli.py) already has smolvm on PATH - # (provision steps succeed), so -c is sufficient. - return subprocess.run(["sh", "-c", script], check=False).returncode - - # smolvm/libkrun can SIGKILL an otherwise-normal exec during - # early-VM provisioning. Retry once after a short settle so - # callers (provision_ca, etc.) don't have to handle it themselves. - _SIGKILL_EXIT = 128 + 9 - - def exec(self, script: str, *, user: str = "node") -> ExecResult: - """Run a POSIX shell script as `user` (default `node`) and - capture the result. Matches the docker backend's `exec`, - which defaults to the image's USER (also node) — so test - helpers / provision shell-outs run with the same identity - on both backends. Pass `user="root"` for tests that need - root. - - `runuser -u -- env ... /bin/sh -c