feat(firecracker): add Linux microVM backend to replace smolmachines
Adds a Firecracker-based backend for Linux, providing mature KVM-based microVM isolation to replace smolmachines/libkrun (issue #342, closes the dead-end tracked in #332). Architecture: - Guest control over SSH (dropbear injected into the rootfs) on a point-to-point TAP link. `ssh -t` forwards SIGWINCH natively, so no resize bridge is needed. - Networking: a one-time, root-provisioned pool of user-owned TAP devices (no shared bridge → no docker0/virbr0/cni0 collisions) plus a dedicated `table inet bot_bottle_fc` nftables table (independent of Docker/ufw/firewalld rules). `./cli.py firecracker setup` prints the host-appropriate config (NixOS module or sudo script). - Rootfs: `docker export` → ext4 via `mke2fs -d` (rootless, no mount), cached by image digest; per-bottle SSH pubkey + IP passed via the kernel cmdline. - Sidecar: reuses the Docker bundle, published on the slot's host TAP IP. - Fail-closed isolation: TAP pool verified at preflight; the egress boundary is proven empirically post-boot (before the agent runs) by a canary probe — the VM must fail to reach the host directly, or launch is refused. Linux hosts with Firecracker + KVM now default to this backend; macOS stays on macos-container. Not yet validated end-to-end on live hardware (requires the one-time network pool). Unit tests + pyright pass. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01G8p32HJgPoS1hLPWubbftM
This commit is contained in:
@@ -26,8 +26,9 @@ backend exposes five methods:
|
|||||||
|
|
||||||
Selection is driven by `--backend` on `start` or BOT_BOTTLE_BACKEND
|
Selection is driven by `--backend` on `start` or BOT_BOTTLE_BACKEND
|
||||||
(env var). When neither is set, compatible macOS hosts default to
|
(env var). When neither is set, compatible macOS hosts default to
|
||||||
`macos-container`; other hosts default to `smolmachines`. Per PRD 0003
|
`macos-container`; Linux hosts with Firecracker + KVM default to
|
||||||
the manifest does not carry a backend field; the host picks.
|
`firecracker`; otherwise `smolmachines`. Per PRD 0003 the manifest
|
||||||
|
does not carry a backend field; the host picks.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -542,6 +543,7 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]):
|
|||||||
# each backend module can pull BottleSpec / BottlePlan / BottleBackend
|
# each backend module can pull BottleSpec / BottlePlan / BottleBackend
|
||||||
# via `from . import ...` without hitting a partially-initialized module.
|
# via `from . import ...` without hitting a partially-initialized module.
|
||||||
from .docker import DockerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position
|
from .docker import DockerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position
|
||||||
|
from .firecracker import FirecrackerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position
|
||||||
from .macos_container import MacosContainerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position
|
from .macos_container import MacosContainerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position
|
||||||
from .smolmachines import SmolmachinesBottleBackend # noqa: E402 # pylint: disable=wrong-import-position
|
from .smolmachines import SmolmachinesBottleBackend # noqa: E402 # pylint: disable=wrong-import-position
|
||||||
|
|
||||||
@@ -557,6 +559,7 @@ from .freeze import CommitCancelled, Freezer, get_freezer # noqa: E402 # pylin
|
|||||||
# unparameterized methods (prepare → plan → launch(plan), cleanup, etc.).
|
# unparameterized methods (prepare → plan → launch(plan), cleanup, etc.).
|
||||||
_BACKENDS: dict[str, BottleBackend[Any, Any]] = {
|
_BACKENDS: dict[str, BottleBackend[Any, Any]] = {
|
||||||
"docker": DockerBottleBackend(),
|
"docker": DockerBottleBackend(),
|
||||||
|
"firecracker": FirecrackerBottleBackend(),
|
||||||
"macos-container": MacosContainerBottleBackend(),
|
"macos-container": MacosContainerBottleBackend(),
|
||||||
"smolmachines": SmolmachinesBottleBackend(),
|
"smolmachines": SmolmachinesBottleBackend(),
|
||||||
}
|
}
|
||||||
@@ -585,6 +588,8 @@ def get_bottle_backend(
|
|||||||
def _default_backend_name() -> str:
|
def _default_backend_name() -> str:
|
||||||
if has_backend("macos-container"):
|
if has_backend("macos-container"):
|
||||||
return "macos-container"
|
return "macos-container"
|
||||||
|
if has_backend("firecracker"):
|
||||||
|
return "firecracker"
|
||||||
return "smolmachines"
|
return "smolmachines"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,7 @@
|
|||||||
|
"""Firecracker backend: Linux KVM microVM isolation (issue #342)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from .backend import FirecrackerBottleBackend
|
||||||
|
|
||||||
|
__all__ = ["FirecrackerBottleBackend"]
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
"""FirecrackerBottleBackend — Linux microVM implementation.
|
||||||
|
|
||||||
|
Replaces smolmachines on Linux (issue #342): mature KVM-based
|
||||||
|
isolation via Firecracker, SSH control over a point-to-point TAP, and a
|
||||||
|
fail-closed nftables egress boundary. Selected by
|
||||||
|
`BOT_BOTTLE_BACKEND=firecracker` or `--backend=firecracker`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Generator, Sequence
|
||||||
|
|
||||||
|
from ...agent_provider import AgentProvisionPlan
|
||||||
|
from ...egress import EgressPlan
|
||||||
|
from ...env import ResolvedEnv
|
||||||
|
from ...git_gate import GitGatePlan
|
||||||
|
from ...manifest import Manifest
|
||||||
|
from ...supervise import SupervisePlan
|
||||||
|
from .. import ActiveAgent, BottleBackend, BottleSpec
|
||||||
|
from . import cleanup as _cleanup
|
||||||
|
from . import enumerate as _enumerate
|
||||||
|
from . import launch as _launch
|
||||||
|
from . import resolve_plan as _resolve_plan
|
||||||
|
from . import util as _util
|
||||||
|
from .bottle import FirecrackerBottle
|
||||||
|
from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan
|
||||||
|
from .bottle_plan import FirecrackerBottlePlan
|
||||||
|
|
||||||
|
|
||||||
|
class FirecrackerBottleBackend(
|
||||||
|
BottleBackend["FirecrackerBottlePlan", "FirecrackerBottleCleanupPlan"]
|
||||||
|
):
|
||||||
|
name = "firecracker"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def is_available(cls) -> bool:
|
||||||
|
return _util.is_available()
|
||||||
|
|
||||||
|
def _preflight(self) -> None:
|
||||||
|
_resolve_plan.preflight()
|
||||||
|
|
||||||
|
def _build_guest_env(self, resolved_env: ResolvedEnv) -> dict[str, str]:
|
||||||
|
return _resolve_plan.build_guest_env(resolved_env)
|
||||||
|
|
||||||
|
def _resolve_plan(
|
||||||
|
self,
|
||||||
|
spec: BottleSpec,
|
||||||
|
*,
|
||||||
|
manifest: Manifest,
|
||||||
|
slug: str,
|
||||||
|
resolved_env: ResolvedEnv,
|
||||||
|
agent_provision_plan: AgentProvisionPlan,
|
||||||
|
egress_plan: EgressPlan,
|
||||||
|
git_gate_plan: GitGatePlan,
|
||||||
|
supervise_plan: SupervisePlan | None,
|
||||||
|
stage_dir: Path,
|
||||||
|
) -> FirecrackerBottlePlan:
|
||||||
|
return _resolve_plan.resolve_plan(
|
||||||
|
spec,
|
||||||
|
manifest=manifest,
|
||||||
|
slug=slug,
|
||||||
|
resolved_env=resolved_env,
|
||||||
|
agent_provision_plan=agent_provision_plan,
|
||||||
|
egress_plan=egress_plan,
|
||||||
|
supervise_plan=supervise_plan,
|
||||||
|
git_gate_plan=git_gate_plan,
|
||||||
|
stage_dir=stage_dir,
|
||||||
|
)
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def launch(
|
||||||
|
self, plan: FirecrackerBottlePlan
|
||||||
|
) -> Generator[FirecrackerBottle, None, None]:
|
||||||
|
with _launch.launch(plan, provision=self.provision) as bottle:
|
||||||
|
yield bottle
|
||||||
|
|
||||||
|
def prepare_cleanup(self) -> FirecrackerBottleCleanupPlan:
|
||||||
|
return _cleanup.prepare_cleanup()
|
||||||
|
|
||||||
|
def cleanup(self, plan: FirecrackerBottleCleanupPlan) -> None:
|
||||||
|
_cleanup.cleanup(plan)
|
||||||
|
|
||||||
|
def enumerate_active(self) -> Sequence[ActiveAgent]:
|
||||||
|
return _enumerate.enumerate_active()
|
||||||
|
|
||||||
|
def supervise_mcp_url(self, plan: FirecrackerBottlePlan) -> str:
|
||||||
|
return plan.agent_supervise_url
|
||||||
@@ -0,0 +1,170 @@
|
|||||||
|
"""Bottle handle for a Firecracker microVM, over SSH.
|
||||||
|
|
||||||
|
Every operation routes through SSH to the guest (dropbear, listening on
|
||||||
|
the TAP link). `exec` pipes a script over stdin and captures output;
|
||||||
|
`cp_in` uses scp; `exec_agent` uses `ssh -t` for an interactive PTY
|
||||||
|
session. `ssh -t` forwards the host terminal's SIGWINCH to the remote
|
||||||
|
PTY natively, so — unlike the smolmachines backend — no resize bridge
|
||||||
|
is needed.
|
||||||
|
|
||||||
|
Commands run as the image's `node` user via `runuser`, with HOME/USER/
|
||||||
|
PATH and the bottle env (HTTPS_PROXY at the sidecar, CA paths, …) set
|
||||||
|
per-invocation through `env` (the VM itself just runs the init; it has
|
||||||
|
no baked-in process env like a `docker run` container would).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shlex
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Mapping, cast
|
||||||
|
|
||||||
|
from ...agent_provider import PromptMode, prompt_args
|
||||||
|
from .. import Bottle, ExecResult
|
||||||
|
from ..terminal import exec_shell_script
|
||||||
|
from . import util
|
||||||
|
|
||||||
|
|
||||||
|
_HOME_FOR = {"node": "/home/node", "root": "/root"}
|
||||||
|
_DEFAULT_PATH_FOR = {
|
||||||
|
"node": (
|
||||||
|
"/home/node/.local/bin:"
|
||||||
|
"/home/node/.codex/packages/standalone/current/bin:"
|
||||||
|
"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||||
|
),
|
||||||
|
"root": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _env_assignments_for(user: str, env: Mapping[str, str]) -> list[str]:
|
||||||
|
home = _HOME_FOR.get(user, f"/home/{user}")
|
||||||
|
out = [f"HOME={home}", f"USER={user}"]
|
||||||
|
if "PATH" not in env:
|
||||||
|
out.append(f"PATH={_DEFAULT_PATH_FOR.get(user, _DEFAULT_PATH_FOR['root'])}")
|
||||||
|
for k, v in env.items():
|
||||||
|
out.append(f"{k}={v}")
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
class FirecrackerBottle(Bottle):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
name: str,
|
||||||
|
*,
|
||||||
|
private_key: Path,
|
||||||
|
guest_ip: str,
|
||||||
|
guest_env: Mapping[str, str] | None = None,
|
||||||
|
prompt_path_in_guest: str | None = None,
|
||||||
|
agent_command: str = "claude",
|
||||||
|
agent_prompt_mode: PromptMode = "append_file",
|
||||||
|
agent_provider_template: str = "claude",
|
||||||
|
terminal_title: str = "",
|
||||||
|
terminal_color: str = "",
|
||||||
|
agent_workdir: str = "/home/node",
|
||||||
|
) -> None:
|
||||||
|
self.name = name
|
||||||
|
self._private_key = private_key
|
||||||
|
self._guest_ip = guest_ip
|
||||||
|
self._guest_env = dict(guest_env or {})
|
||||||
|
self.prompt_path = prompt_path_in_guest
|
||||||
|
self._agent_prompt_mode = agent_prompt_mode
|
||||||
|
self.agent_command = agent_command
|
||||||
|
self.terminal_title = terminal_title
|
||||||
|
self.terminal_color = terminal_color
|
||||||
|
self.agent_provider_template = agent_provider_template
|
||||||
|
self.agent_workdir = agent_workdir
|
||||||
|
|
||||||
|
def _ssh(self, *, tty: bool) -> list[str]:
|
||||||
|
argv = util.ssh_base_argv(self._private_key, self._guest_ip)
|
||||||
|
if tty:
|
||||||
|
# -t: allocate a remote PTY and forward window-size changes.
|
||||||
|
argv.insert(1, "-t")
|
||||||
|
return argv
|
||||||
|
|
||||||
|
def _agent_remote_argv(self, argv: list[str]) -> list[str]:
|
||||||
|
"""The `runuser … <agent> …` command run inside the guest."""
|
||||||
|
full_argv = list(argv)
|
||||||
|
full_argv.extend(
|
||||||
|
prompt_args(
|
||||||
|
cast(PromptMode, self._agent_prompt_mode),
|
||||||
|
self.prompt_path,
|
||||||
|
argv=full_argv,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
remote = ["runuser", "-u", "node", "--",
|
||||||
|
"env", *_env_assignments_for("node", self._guest_env)]
|
||||||
|
if self.agent_workdir and self.agent_workdir != _HOME_FOR["node"]:
|
||||||
|
remote += ["sh", "-lc", f"cd {self.agent_workdir} && exec \"$@\"",
|
||||||
|
"bot-bottle-agent"]
|
||||||
|
remote += [self.agent_command, *full_argv]
|
||||||
|
return remote
|
||||||
|
|
||||||
|
def agent_argv(self, argv: list[str], *, tty: bool = True) -> list[str]:
|
||||||
|
return [*self._ssh(tty=tty), "--", *self._agent_remote_argv(argv)]
|
||||||
|
|
||||||
|
def exec_agent(self, argv: list[str], *, tty: bool = True) -> int:
|
||||||
|
agent_argv = self.agent_argv(argv, tty=tty)
|
||||||
|
script = (
|
||||||
|
exec_shell_script(agent_argv, self.terminal_title, self.terminal_color)
|
||||||
|
if tty else None
|
||||||
|
)
|
||||||
|
if script is None:
|
||||||
|
return subprocess.run(agent_argv, check=False).returncode
|
||||||
|
return subprocess.run(["sh", "-lc", script], check=False).returncode
|
||||||
|
|
||||||
|
def exec(self, script: str, *, user: str = "node") -> ExecResult:
|
||||||
|
# Pipe the script over stdin (`sh -s`) so nothing needs shell
|
||||||
|
# quoting through the SSH command line.
|
||||||
|
remote = ["runuser", "-u", user, "--",
|
||||||
|
"env", *_env_assignments_for(user, self._guest_env),
|
||||||
|
"/bin/sh", "-s"]
|
||||||
|
result = subprocess.run(
|
||||||
|
[*self._ssh(tty=False), "--", *remote],
|
||||||
|
input=script, capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
return ExecResult(
|
||||||
|
returncode=result.returncode,
|
||||||
|
stdout=result.stdout,
|
||||||
|
stderr=result.stderr,
|
||||||
|
)
|
||||||
|
|
||||||
|
def cp_in(self, host_path: str, container_path: str) -> None:
|
||||||
|
# Stream a tar over the SSH exec channel rather than scp: the
|
||||||
|
# guest runs dropbear, which ships no sftp-server/scp, but every
|
||||||
|
# agent image has `tar`. Copy so `container_path` becomes a copy
|
||||||
|
# of `host_path` (docker cp semantics — callers rm -rf the
|
||||||
|
# destination first).
|
||||||
|
host_parent = os.path.dirname(host_path.rstrip("/")) or "/"
|
||||||
|
host_base = os.path.basename(host_path.rstrip("/"))
|
||||||
|
guest_parent = os.path.dirname(container_path.rstrip("/")) or "/"
|
||||||
|
guest_base = os.path.basename(container_path.rstrip("/"))
|
||||||
|
remote = (
|
||||||
|
f"mkdir -p {shlex.quote(guest_parent)} && "
|
||||||
|
f"cd {shlex.quote(guest_parent)} && "
|
||||||
|
f"rm -rf {shlex.quote(guest_base)} && tar -xf - && "
|
||||||
|
f"{{ [ {shlex.quote(host_base)} = {shlex.quote(guest_base)} ] || "
|
||||||
|
f"mv {shlex.quote(host_base)} {shlex.quote(guest_base)}; }}"
|
||||||
|
)
|
||||||
|
tar = subprocess.Popen(
|
||||||
|
["tar", "-C", host_parent, "-cf", "-", host_base],
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
)
|
||||||
|
ssh = subprocess.run(
|
||||||
|
[*self._ssh(tty=False), "--", "sh", "-c", remote],
|
||||||
|
stdin=tar.stdout, capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
tar.wait()
|
||||||
|
if tar.returncode != 0 or ssh.returncode != 0:
|
||||||
|
sys.stderr.write(ssh.stderr)
|
||||||
|
raise subprocess.CalledProcessError(
|
||||||
|
ssh.returncode or tar.returncode,
|
||||||
|
["cp_in", host_path, container_path],
|
||||||
|
)
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
# Real teardown (VM terminate, TAP release) lives on the launch
|
||||||
|
# ExitStack; this is the idempotent alias the ABC expects.
|
||||||
|
pass
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
"""Cleanup plan for the Firecracker backend."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from ...log import info
|
||||||
|
from .. import BottleCleanupPlan
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FirecrackerBottleCleanupPlan(BottleCleanupPlan):
|
||||||
|
# PIDs of orphaned firecracker VMM processes and the sidecar
|
||||||
|
# containers left behind by previous bottles.
|
||||||
|
vm_pids: tuple[int, ...] = ()
|
||||||
|
containers: tuple[str, ...] = ()
|
||||||
|
run_dirs: tuple[str, ...] = ()
|
||||||
|
|
||||||
|
def print(self) -> None:
|
||||||
|
if self.empty:
|
||||||
|
info("firecracker cleanup: nothing to remove")
|
||||||
|
return
|
||||||
|
for pid in self.vm_pids:
|
||||||
|
info(f"firecracker VM process: pid {pid}")
|
||||||
|
for name in self.containers:
|
||||||
|
info(f"firecracker sidecar container: {name}")
|
||||||
|
for path in self.run_dirs:
|
||||||
|
info(f"firecracker run dir: {path}")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def empty(self) -> bool:
|
||||||
|
return not (self.vm_pids or self.containers or self.run_dirs)
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
"""Plan type for the Firecracker backend."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ...agent_provider import PromptMode
|
||||||
|
from .. import BottlePlan
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FirecrackerBottlePlan(BottlePlan):
|
||||||
|
slug: str
|
||||||
|
forwarded_env: dict[str, str] = field(repr=False)
|
||||||
|
# Stamped by launch once the sidecar is up and its ports are
|
||||||
|
# published on the host-side TAP IP (empty at prepare time).
|
||||||
|
agent_proxy_url: str = ""
|
||||||
|
agent_git_gate_url: str = ""
|
||||||
|
agent_supervise_url: str = ""
|
||||||
|
|
||||||
|
@property
|
||||||
|
def container_name(self) -> str:
|
||||||
|
"""Instance name, reused for the sidecar container + VM run dir.
|
||||||
|
Matches the `bot-bottle-<slug>` convention the other backends
|
||||||
|
use so cleanup/enumerate discovery-by-prefix keeps working."""
|
||||||
|
return self.agent_provision.instance_name
|
||||||
|
|
||||||
|
@property
|
||||||
|
def image(self) -> str:
|
||||||
|
return self.agent_provision.image
|
||||||
|
|
||||||
|
@property
|
||||||
|
def dockerfile_path(self) -> str:
|
||||||
|
return self.agent_provision.dockerfile
|
||||||
|
|
||||||
|
@property
|
||||||
|
def prompt_file(self) -> Path:
|
||||||
|
return self.agent_provision.prompt_file
|
||||||
|
|
||||||
|
@property
|
||||||
|
def agent_command(self) -> str:
|
||||||
|
return self.agent_provision.command
|
||||||
|
|
||||||
|
@property
|
||||||
|
def agent_prompt_mode(self) -> PromptMode:
|
||||||
|
return self.agent_provision.prompt_mode
|
||||||
|
|
||||||
|
@property
|
||||||
|
def agent_provider_template(self) -> str:
|
||||||
|
return self.agent_provision.template
|
||||||
|
|
||||||
|
@property
|
||||||
|
def git_gate_insteadof_host(self) -> str:
|
||||||
|
if self.agent_git_gate_url.startswith("http://"):
|
||||||
|
return self.agent_git_gate_url.removeprefix("http://").rstrip("/")
|
||||||
|
return super().git_gate_insteadof_host
|
||||||
|
|
||||||
|
@property
|
||||||
|
def git_gate_insteadof_scheme(self) -> str:
|
||||||
|
if self.agent_git_gate_url.startswith("http://"):
|
||||||
|
return "http"
|
||||||
|
return super().git_gate_insteadof_scheme
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
"""Cleanup for the Firecracker backend.
|
||||||
|
|
||||||
|
Orphans are: firecracker VMM processes whose config lives under our run
|
||||||
|
dir, the `bot-bottle-sidecars-*` containers, and the per-bottle run
|
||||||
|
dirs. TAP slots free themselves (the flock drops when the launcher
|
||||||
|
exits), so there is nothing to reclaim there.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ...log import info
|
||||||
|
from . import util
|
||||||
|
from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan
|
||||||
|
|
||||||
|
_SIDECAR_PREFIX = "bot-bottle-sidecars-"
|
||||||
|
|
||||||
|
|
||||||
|
def _run_root() -> Path:
|
||||||
|
return util.cache_dir() / "run"
|
||||||
|
|
||||||
|
|
||||||
|
def _orphan_vm_pids() -> list[int]:
|
||||||
|
"""firecracker processes whose --config-file is under our run dir."""
|
||||||
|
run_root = str(_run_root())
|
||||||
|
result = subprocess.run(
|
||||||
|
["pgrep", "-a", "firecracker"],
|
||||||
|
capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
return []
|
||||||
|
pids: list[int] = []
|
||||||
|
for line in result.stdout.splitlines():
|
||||||
|
parts = line.split(None, 1)
|
||||||
|
if len(parts) != 2 or run_root not in parts[1]:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
pids.append(int(parts[0]))
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
return pids
|
||||||
|
|
||||||
|
|
||||||
|
def _sidecar_containers() -> list[str]:
|
||||||
|
result = subprocess.run(
|
||||||
|
["docker", "ps", "-a", "--format", "{{.Names}}",
|
||||||
|
"--filter", f"name={_SIDECAR_PREFIX}"],
|
||||||
|
capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
return []
|
||||||
|
return sorted(n.strip() for n in result.stdout.splitlines() if n.strip())
|
||||||
|
|
||||||
|
|
||||||
|
def _run_dirs() -> list[str]:
|
||||||
|
run_root = _run_root()
|
||||||
|
if not run_root.is_dir():
|
||||||
|
return []
|
||||||
|
return sorted(str(p) for p in run_root.iterdir() if p.is_dir())
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_cleanup() -> FirecrackerBottleCleanupPlan:
|
||||||
|
return FirecrackerBottleCleanupPlan(
|
||||||
|
vm_pids=tuple(_orphan_vm_pids()),
|
||||||
|
containers=tuple(_sidecar_containers()),
|
||||||
|
run_dirs=tuple(_run_dirs()),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def cleanup(plan: FirecrackerBottleCleanupPlan) -> None:
|
||||||
|
for pid in plan.vm_pids:
|
||||||
|
info(f"kill firecracker VM pid {pid}")
|
||||||
|
try:
|
||||||
|
os.kill(pid, signal.SIGTERM)
|
||||||
|
except ProcessLookupError:
|
||||||
|
pass
|
||||||
|
for name in plan.containers:
|
||||||
|
info(f"docker rm -f {name}")
|
||||||
|
subprocess.run(
|
||||||
|
["docker", "rm", "-f", name],
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
||||||
|
)
|
||||||
|
for path in plan.run_dirs:
|
||||||
|
info(f"rm -rf {path}")
|
||||||
|
shutil.rmtree(path, ignore_errors=True)
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
"""Active-agent enumeration for the Firecracker backend.
|
||||||
|
|
||||||
|
The agent runs in a VM (no container to list), so a live bottle is
|
||||||
|
identified by its running sidecar container `bot-bottle-sidecars-<slug>`
|
||||||
|
— the same discovery-by-prefix the other backends use.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
from ...bottle_state import read_metadata
|
||||||
|
from .. import ActiveAgent
|
||||||
|
|
||||||
|
_SIDECAR_PREFIX = "bot-bottle-sidecars-"
|
||||||
|
|
||||||
|
|
||||||
|
def enumerate_active() -> list[ActiveAgent]:
|
||||||
|
result = subprocess.run(
|
||||||
|
["docker", "ps", "--format", "{{.Names}}",
|
||||||
|
"--filter", f"name={_SIDECAR_PREFIX}"],
|
||||||
|
capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
return []
|
||||||
|
out: list[ActiveAgent] = []
|
||||||
|
for name in sorted(n.strip() for n in result.stdout.splitlines() if n.strip()):
|
||||||
|
slug = name[len(_SIDECAR_PREFIX):]
|
||||||
|
metadata = read_metadata(slug)
|
||||||
|
if metadata is None or metadata.backend != "firecracker":
|
||||||
|
# Skip sidecars owned by another backend (docker/smolmachines
|
||||||
|
# share the container-name prefix).
|
||||||
|
continue
|
||||||
|
out.append(ActiveAgent(
|
||||||
|
backend_name="firecracker",
|
||||||
|
slug=slug,
|
||||||
|
agent_name=metadata.agent_name,
|
||||||
|
started_at=metadata.started_at,
|
||||||
|
services=(),
|
||||||
|
label=metadata.label,
|
||||||
|
color=metadata.color,
|
||||||
|
))
|
||||||
|
return out
|
||||||
@@ -0,0 +1,173 @@
|
|||||||
|
"""Firecracker microVM process lifecycle.
|
||||||
|
|
||||||
|
A bottle VM is one `firecracker` process booted from a JSON config
|
||||||
|
(kernel + writable rootfs ext4 + one TAP network interface). We run it
|
||||||
|
`--no-api`: all configuration is in the config file, and teardown is a
|
||||||
|
process signal — the microVM dies with its VMM, which is exactly the
|
||||||
|
ephemeral-bottle semantics we want. No API socket, no runtime
|
||||||
|
reconfiguration.
|
||||||
|
|
||||||
|
The guest gets its IP from the kernel `ip=` cmdline (kernel-level
|
||||||
|
autoconfig, no in-guest iproute2) and its SSH pubkey from a `bb_pubkey=`
|
||||||
|
cmdline arg the init decodes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ...log import die, info
|
||||||
|
from . import util
|
||||||
|
|
||||||
|
|
||||||
|
# Serial console off the critical path but captured to a log for
|
||||||
|
# debugging boot failures. `reboot=k panic=1 pci=off` are the standard
|
||||||
|
# Firecracker guest args; `i8042.*` skip the (absent) PS/2 probe to
|
||||||
|
# shave boot time.
|
||||||
|
_BASE_BOOT_ARGS = (
|
||||||
|
"console=ttyS0 reboot=k panic=1 pci=off "
|
||||||
|
"i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd "
|
||||||
|
"root=/dev/vda rw init=/bb-init"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class VmHandle:
|
||||||
|
"""A running microVM: its VMM process, guest IP, and console log."""
|
||||||
|
|
||||||
|
process: subprocess.Popen[bytes]
|
||||||
|
guest_ip: str
|
||||||
|
console_log: Path
|
||||||
|
|
||||||
|
def is_alive(self) -> bool:
|
||||||
|
return self.process.poll() is None
|
||||||
|
|
||||||
|
def terminate(self) -> None:
|
||||||
|
"""Stop the VMM (and thus the guest). SIGTERM, then SIGKILL."""
|
||||||
|
if self.process.poll() is not None:
|
||||||
|
return
|
||||||
|
self.process.send_signal(signal.SIGTERM)
|
||||||
|
try:
|
||||||
|
self.process.wait(timeout=5)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
self.process.kill()
|
||||||
|
self.process.wait(timeout=5)
|
||||||
|
|
||||||
|
|
||||||
|
def _boot_args(guest_ip: str, host_ip: str, pubkey: str) -> str:
|
||||||
|
# ip=<client>::<gw>:<netmask>::<dev>:off — /31 point-to-point link,
|
||||||
|
# so netmask is 255.255.255.254 and the gateway is the host TAP IP.
|
||||||
|
ip_arg = f"ip={guest_ip}::{host_ip}:255.255.255.254::eth0:off"
|
||||||
|
pub_b64 = base64.b64encode(pubkey.encode()).decode()
|
||||||
|
return f"{_BASE_BOOT_ARGS} {ip_arg} bb_pubkey={pub_b64}"
|
||||||
|
|
||||||
|
|
||||||
|
def _config(
|
||||||
|
*,
|
||||||
|
rootfs: Path,
|
||||||
|
tap: str,
|
||||||
|
guest_ip: str,
|
||||||
|
host_ip: str,
|
||||||
|
pubkey: str,
|
||||||
|
vcpus: int,
|
||||||
|
mem_mib: int,
|
||||||
|
guest_mac: str,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"boot-source": {
|
||||||
|
"kernel_image_path": str(util.kernel_path()),
|
||||||
|
"boot_args": _boot_args(guest_ip, host_ip, pubkey),
|
||||||
|
},
|
||||||
|
"drives": [
|
||||||
|
{
|
||||||
|
"drive_id": "rootfs",
|
||||||
|
"path_on_host": str(rootfs),
|
||||||
|
"is_root_device": True,
|
||||||
|
"is_read_only": False,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"network-interfaces": [
|
||||||
|
{
|
||||||
|
"iface_id": "eth0",
|
||||||
|
"host_dev_name": tap,
|
||||||
|
"guest_mac": guest_mac,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"machine-config": {
|
||||||
|
"vcpu_count": vcpus,
|
||||||
|
"mem_size_mib": mem_mib,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def boot(
|
||||||
|
*,
|
||||||
|
name: str,
|
||||||
|
rootfs: Path,
|
||||||
|
tap: str,
|
||||||
|
guest_ip: str,
|
||||||
|
host_ip: str,
|
||||||
|
pubkey: str,
|
||||||
|
run_dir: Path,
|
||||||
|
vcpus: int = 2,
|
||||||
|
mem_mib: int = 2048,
|
||||||
|
guest_mac: str = "06:00:AC:10:00:02",
|
||||||
|
) -> VmHandle:
|
||||||
|
"""Write the config and launch the VMM. Returns once the process is
|
||||||
|
spawned; callers wait for SSH readiness separately."""
|
||||||
|
run_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
config_path = run_dir / "config.json"
|
||||||
|
console_log = run_dir / "console.log"
|
||||||
|
config_path.write_text(json.dumps(
|
||||||
|
_config(
|
||||||
|
rootfs=rootfs, tap=tap, guest_ip=guest_ip, host_ip=host_ip,
|
||||||
|
pubkey=pubkey, vcpus=vcpus, mem_mib=mem_mib, guest_mac=guest_mac,
|
||||||
|
),
|
||||||
|
indent=2,
|
||||||
|
))
|
||||||
|
|
||||||
|
info(f"booting microVM {name} on {tap} (guest {guest_ip})")
|
||||||
|
log_fh = console_log.open("wb")
|
||||||
|
process = subprocess.Popen(
|
||||||
|
["firecracker", "--no-api", "--config-file", str(config_path)],
|
||||||
|
stdout=log_fh, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL,
|
||||||
|
)
|
||||||
|
return VmHandle(process=process, guest_ip=guest_ip, console_log=console_log)
|
||||||
|
|
||||||
|
|
||||||
|
def wait_for_ssh(
|
||||||
|
vm: VmHandle, private_key: Path, *, timeout: float = 30.0,
|
||||||
|
) -> None:
|
||||||
|
"""Poll SSH until the guest accepts a command or the deadline
|
||||||
|
passes. Dies (with the console tail) if the VMM exits early or the
|
||||||
|
guest never comes up — a boot failure must be loud, not a hang."""
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
probe = util.ssh_base_argv(private_key, vm.guest_ip) + ["true"]
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
if not vm.is_alive():
|
||||||
|
die(f"microVM exited during boot (rc={vm.process.returncode}).\n"
|
||||||
|
f"{_console_tail(vm.console_log)}")
|
||||||
|
result = subprocess.run(
|
||||||
|
probe, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if result.returncode == 0:
|
||||||
|
return
|
||||||
|
time.sleep(0.5)
|
||||||
|
die(f"microVM {vm.guest_ip} did not accept SSH within {timeout:.0f}s.\n"
|
||||||
|
f"{_console_tail(vm.console_log)}")
|
||||||
|
|
||||||
|
|
||||||
|
def _console_tail(console_log: Path, lines: int = 25) -> str:
|
||||||
|
try:
|
||||||
|
text = console_log.read_text(errors="replace").splitlines()
|
||||||
|
except OSError:
|
||||||
|
return "(no console log)"
|
||||||
|
tail = "\n".join(text[-lines:])
|
||||||
|
return f"--- guest console (last {lines} lines) ---\n{tail}"
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
"""FirecrackerFreezer — snapshot a running microVM to a Docker image.
|
||||||
|
|
||||||
|
The VM is live and can't be block-copied safely, so — like the macOS
|
||||||
|
backend — we stream the guest root filesystem out over the control
|
||||||
|
channel (SSH here) and rebuild an image from it. The bottle keeps
|
||||||
|
running after the snapshot.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ...log import die, info
|
||||||
|
from .. import ActiveAgent
|
||||||
|
from ..freeze import Freezer
|
||||||
|
from . import util
|
||||||
|
|
||||||
|
|
||||||
|
class FirecrackerFreezer(Freezer):
|
||||||
|
backend_name = "firecracker"
|
||||||
|
|
||||||
|
def _freeze(self, agent: ActiveAgent) -> str:
|
||||||
|
run_dir = util.cache_dir() / "run" / agent.slug
|
||||||
|
private_key = run_dir / "bottle_id_ed25519"
|
||||||
|
guest_ip = _guest_ip_from_config(run_dir / "config.json")
|
||||||
|
if not private_key.is_file() or not guest_ip:
|
||||||
|
die(f"cannot freeze {agent.slug}: run dir {run_dir} is missing the "
|
||||||
|
f"SSH key or VM config (is the bottle still running?)")
|
||||||
|
image_tag = f"bot-bottle-committed-{agent.slug}:latest"
|
||||||
|
_commit_via_ssh(private_key, guest_ip, image_tag)
|
||||||
|
info(f"committed {agent.slug} -> {image_tag!r}")
|
||||||
|
return image_tag
|
||||||
|
|
||||||
|
def _export_hint(self, slug: str, image_ref: str) -> None:
|
||||||
|
info(f"to export for migration: docker image save {image_ref} "
|
||||||
|
f"-o {slug}.tar")
|
||||||
|
|
||||||
|
|
||||||
|
def _guest_ip_from_config(config_path: Path) -> str:
|
||||||
|
try:
|
||||||
|
cfg = json.loads(config_path.read_text())
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return ""
|
||||||
|
boot_args = cfg.get("boot-source", {}).get("boot_args", "")
|
||||||
|
for token in boot_args.split():
|
||||||
|
if token.startswith("ip="):
|
||||||
|
# ip=<client>::<gw>:<mask>::<dev>:off
|
||||||
|
return token[len("ip="):].split(":", 1)[0]
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def _commit_via_ssh(private_key: Path, guest_ip: str, image_tag: str) -> None:
|
||||||
|
with tempfile.TemporaryDirectory(prefix="bot-bottle-fc-commit.") as tmp:
|
||||||
|
rootfs_tar = os.path.join(tmp, "rootfs.tar")
|
||||||
|
ssh = util.ssh_base_argv(private_key, guest_ip)
|
||||||
|
with open(rootfs_tar, "wb") as tar_out:
|
||||||
|
result = subprocess.run(
|
||||||
|
[*ssh, "--", "tar", "--create", "--one-file-system",
|
||||||
|
"--exclude=./proc", "--exclude=./sys", "--exclude=./dev",
|
||||||
|
"--exclude=./run", "--file=-", "--directory=/", "."],
|
||||||
|
stdout=tar_out, stderr=subprocess.PIPE, check=False,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
die(f"ssh tar for {guest_ip} failed: "
|
||||||
|
f"{(result.stderr or b'').decode().strip() or '<no stderr>'}")
|
||||||
|
with open(os.path.join(tmp, "Dockerfile"), "w", encoding="utf-8") as f:
|
||||||
|
f.write("FROM scratch\nADD rootfs.tar /\nUSER node\nWORKDIR /home/node\n")
|
||||||
|
build = subprocess.run(
|
||||||
|
["docker", "build", "-t", image_tag, tmp], check=False,
|
||||||
|
)
|
||||||
|
if build.returncode != 0:
|
||||||
|
die(f"docker build for {image_tag!r} failed")
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
"""Empirical, post-boot isolation probe — the authoritative fail-closed
|
||||||
|
egress-boundary check.
|
||||||
|
|
||||||
|
The pre-boot nft check can't be trusted on every host (listing an
|
||||||
|
nftables table typically needs root; the launcher runs unprivileged).
|
||||||
|
So before the agent ever runs, we prove the boundary directly: open a
|
||||||
|
canary TCP listener on a host IP the VM must NOT be able to reach (the
|
||||||
|
host's primary non-TAP address), then have the guest try to connect. If
|
||||||
|
the isolation rules are in force the connection is dropped and times
|
||||||
|
out; if it succeeds, the VM can reach host services it shouldn't — a
|
||||||
|
sandbox escape — and we refuse to continue.
|
||||||
|
|
||||||
|
The listener is load-bearing: without it a "blocked" result could just
|
||||||
|
be connection-refused. With it, a reachable canary means a real leak
|
||||||
|
and a timeout means a real drop.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ...log import die, info, warn
|
||||||
|
from . import util
|
||||||
|
|
||||||
|
|
||||||
|
def _host_primary_ip() -> str:
|
||||||
|
"""The source IP the host uses for off-box traffic (its LAN
|
||||||
|
address) — a canary the VM must not reach. Empty if the host has no
|
||||||
|
such route (isolated box)."""
|
||||||
|
result = subprocess.run(
|
||||||
|
["ip", "-o", "route", "get", "1.1.1.1"],
|
||||||
|
capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
return ""
|
||||||
|
tokens = result.stdout.split()
|
||||||
|
# `... src <addr> ...` — the address the host would use as source.
|
||||||
|
if "src" in tokens:
|
||||||
|
idx = tokens.index("src")
|
||||||
|
if idx + 1 < len(tokens):
|
||||||
|
return tokens[idx + 1]
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
# Guest-side connect test: exit 0 = reached the canary (LEAK), 1 =
|
||||||
|
# blocked (good), 2 = no usable tool (inconclusive -> fail-closed).
|
||||||
|
_GUEST_PROBE = r"""
|
||||||
|
h="$1"; p="$2"
|
||||||
|
if command -v python3 >/dev/null 2>&1; then
|
||||||
|
python3 - "$h" "$p" <<'PY'
|
||||||
|
import socket, sys
|
||||||
|
s = socket.socket(); s.settimeout(3)
|
||||||
|
sys.exit(0 if s.connect_ex((sys.argv[1], int(sys.argv[2]))) == 0 else 1)
|
||||||
|
PY
|
||||||
|
elif command -v bash >/dev/null 2>&1; then
|
||||||
|
timeout 3 bash -c "exec 3<>/dev/tcp/$h/$p" >/dev/null 2>&1
|
||||||
|
elif command -v nc >/dev/null 2>&1; then
|
||||||
|
nc -w3 -z "$h" "$p" >/dev/null 2>&1
|
||||||
|
else
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def verify_isolation(private_key: Path, guest_ip: str) -> None:
|
||||||
|
"""Prove the VM cannot reach the host's primary IP. Dies
|
||||||
|
(fail-closed) on a confirmed leak or if the guest has no tool to
|
||||||
|
run the test. Warns (does not fail) when the host has no canary
|
||||||
|
address to test against."""
|
||||||
|
canary_ip = _host_primary_ip()
|
||||||
|
if not canary_ip:
|
||||||
|
warn("isolation probe skipped: host has no non-TAP route to test "
|
||||||
|
"against (isolated box).")
|
||||||
|
return
|
||||||
|
|
||||||
|
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||||
|
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||||
|
listener.bind((canary_ip, 0))
|
||||||
|
listener.listen(1)
|
||||||
|
listener.settimeout(6)
|
||||||
|
canary_port = listener.getsockname()[1]
|
||||||
|
|
||||||
|
accepted: list[bool] = []
|
||||||
|
|
||||||
|
def _accept() -> None:
|
||||||
|
try:
|
||||||
|
conn, _ = listener.accept()
|
||||||
|
accepted.append(True)
|
||||||
|
conn.close()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
thread = threading.Thread(target=_accept, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
|
||||||
|
ssh = util.ssh_base_argv(private_key, guest_ip)
|
||||||
|
result = subprocess.run(
|
||||||
|
[*ssh, "--", "sh", "-s", "--", canary_ip, str(canary_port)],
|
||||||
|
input=_GUEST_PROBE, capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
thread.join(timeout=7)
|
||||||
|
listener.close()
|
||||||
|
|
||||||
|
if result.returncode == 0 or accepted:
|
||||||
|
die(f"ISOLATION FAILURE: the VM reached the host canary "
|
||||||
|
f"{canary_ip}:{canary_port}. The egress boundary is not in "
|
||||||
|
f"force — refusing to run the agent (fail-closed). Verify the "
|
||||||
|
f"nft table with: ./cli.py firecracker setup")
|
||||||
|
if result.returncode == 2:
|
||||||
|
die("isolation probe inconclusive: the guest has no python3/bash/nc "
|
||||||
|
"to run the connectivity test. Refusing to continue "
|
||||||
|
"(fail-closed).")
|
||||||
|
info(f"isolation verified: VM cannot reach host {canary_ip}")
|
||||||
@@ -0,0 +1,404 @@
|
|||||||
|
"""Launch flow for the Firecracker backend.
|
||||||
|
|
||||||
|
Per bottle:
|
||||||
|
1. mint the egress CA, build the agent image (docker), export it to a
|
||||||
|
cached ext4 rootfs;
|
||||||
|
2. claim a free TAP pool slot (rootless flock);
|
||||||
|
3. bring up the Docker sidecar bundle, publishing egress / git-gate /
|
||||||
|
supervise on the slot's host-side TAP IP at fixed ports;
|
||||||
|
4. boot the microVM on that TAP; wait for SSH;
|
||||||
|
5. provision (CA, prompt, skills, workspace, git, supervise) over SSH.
|
||||||
|
|
||||||
|
Isolation is enforced by the operator-provisioned nft table (checked
|
||||||
|
fail-closed in preflight): a VM reaches only its sidecar (DNAT'd from
|
||||||
|
the host TAP IP) and nothing else. The agent's HTTPS_PROXY therefore
|
||||||
|
points at `http://<host_tap_ip>:9099`, its only route to the world.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import dataclasses
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
from contextlib import ExitStack, contextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Callable, Generator
|
||||||
|
|
||||||
|
from ...bottle_state import (
|
||||||
|
egress_state_dir,
|
||||||
|
git_gate_state_dir,
|
||||||
|
read_committed_image,
|
||||||
|
)
|
||||||
|
from ...egress import (
|
||||||
|
EGRESS_ROUTES_IN_CONTAINER,
|
||||||
|
egress_agent_env_entries,
|
||||||
|
egress_resolve_token_values,
|
||||||
|
egress_sidecar_env_entries,
|
||||||
|
)
|
||||||
|
from ...git_gate import (
|
||||||
|
provision_git_gate_dynamic_keys,
|
||||||
|
revoke_git_gate_provisioned_keys,
|
||||||
|
)
|
||||||
|
from ...log import die, info, warn
|
||||||
|
from ...supervise import DB_PATH_IN_CONTAINER, SUPERVISE_PORT
|
||||||
|
from ...util import expand_tilde
|
||||||
|
from ..docker.egress import (
|
||||||
|
EGRESS_CA_IN_CONTAINER,
|
||||||
|
EGRESS_PORT,
|
||||||
|
egress_tls_init,
|
||||||
|
)
|
||||||
|
from ..docker.git_gate import (
|
||||||
|
GIT_GATE_ACCESS_HOOK_IN_CONTAINER,
|
||||||
|
GIT_GATE_CREDS_DIR_IN_CONTAINER,
|
||||||
|
GIT_GATE_ENTRYPOINT_IN_CONTAINER,
|
||||||
|
GIT_GATE_HOOK_IN_CONTAINER,
|
||||||
|
)
|
||||||
|
from ..docker.sidecar_bundle import (
|
||||||
|
SIDECAR_BUNDLE_DOCKERFILE,
|
||||||
|
SIDECAR_BUNDLE_IMAGE,
|
||||||
|
)
|
||||||
|
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||||
|
from . import firecracker_vm, isolation_probe, netpool, util
|
||||||
|
from .bottle import FirecrackerBottle
|
||||||
|
from .bottle_plan import FirecrackerBottlePlan
|
||||||
|
|
||||||
|
|
||||||
|
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
|
||||||
|
_GIT_HTTP_PORT = 9420
|
||||||
|
_GIT_GATE_READY_FILE = "/run/git-gate/ready"
|
||||||
|
|
||||||
|
|
||||||
|
def sidecar_container_name(slug: str) -> str:
|
||||||
|
return f"bot-bottle-sidecars-{slug}"
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def launch(
|
||||||
|
plan: FirecrackerBottlePlan,
|
||||||
|
*,
|
||||||
|
provision: Callable[[FirecrackerBottlePlan, "FirecrackerBottle"], str | None],
|
||||||
|
) -> Generator[FirecrackerBottle, None, None]:
|
||||||
|
stack = ExitStack()
|
||||||
|
bottle_for_revoke = plan.manifest.bottle
|
||||||
|
git_gate_dir_for_revoke = git_gate_state_dir(plan.slug)
|
||||||
|
|
||||||
|
def teardown() -> None:
|
||||||
|
teardown_exc: BaseException | None = None
|
||||||
|
try:
|
||||||
|
stack.close()
|
||||||
|
except BaseException as exc: # noqa: W0718 - teardown must continue
|
||||||
|
teardown_exc = exc
|
||||||
|
warn(f"firecracker teardown failed: {exc!r}")
|
||||||
|
revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke)
|
||||||
|
if teardown_exc is not None:
|
||||||
|
raise teardown_exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
plan = _mint_certs(plan)
|
||||||
|
plan = _build_agent_image(plan)
|
||||||
|
|
||||||
|
# Claim a TAP slot; the flock is held until teardown closes it.
|
||||||
|
slot, lock = netpool.allocate(plan.slug)
|
||||||
|
stack.callback(lock.close)
|
||||||
|
info(f"firecracker slot {slot.iface}: host={slot.host_ip} "
|
||||||
|
f"guest={slot.guest_ip}")
|
||||||
|
|
||||||
|
plan = _provision_git_gate_keys(plan)
|
||||||
|
|
||||||
|
sidecar_name = sidecar_container_name(plan.slug)
|
||||||
|
_force_remove_container(sidecar_name)
|
||||||
|
_start_sidecar_bundle(plan, sidecar_name, slot.host_ip)
|
||||||
|
stack.callback(_force_remove_container, sidecar_name)
|
||||||
|
_stage_git_gate(plan, sidecar_name)
|
||||||
|
|
||||||
|
plan = _stamp_agent_urls(plan, slot.host_ip)
|
||||||
|
|
||||||
|
# Build the per-bottle rootfs + SSH key, then boot.
|
||||||
|
base_dir = util.build_base_rootfs_dir(plan.image)
|
||||||
|
run_dir = util.cache_dir() / "run" / plan.slug
|
||||||
|
run_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
rootfs = run_dir / "rootfs.ext4"
|
||||||
|
util.build_rootfs_ext4(base_dir, rootfs)
|
||||||
|
private_key, pubkey = util.generate_keypair(run_dir)
|
||||||
|
|
||||||
|
vm = firecracker_vm.boot(
|
||||||
|
name=plan.container_name,
|
||||||
|
rootfs=rootfs,
|
||||||
|
tap=slot.iface,
|
||||||
|
guest_ip=slot.guest_ip,
|
||||||
|
host_ip=slot.host_ip,
|
||||||
|
pubkey=pubkey,
|
||||||
|
run_dir=run_dir,
|
||||||
|
)
|
||||||
|
stack.callback(vm.terminate)
|
||||||
|
firecracker_vm.wait_for_ssh(vm, private_key)
|
||||||
|
|
||||||
|
# Authoritative fail-closed egress-boundary check, before the
|
||||||
|
# agent runs: prove the VM cannot reach the host directly.
|
||||||
|
isolation_probe.verify_isolation(private_key, slot.guest_ip)
|
||||||
|
|
||||||
|
bottle = FirecrackerBottle(
|
||||||
|
plan.container_name,
|
||||||
|
private_key=private_key,
|
||||||
|
guest_ip=slot.guest_ip,
|
||||||
|
guest_env=_agent_guest_env(plan, slot.host_ip),
|
||||||
|
agent_command=plan.agent_command,
|
||||||
|
agent_prompt_mode=plan.agent_prompt_mode,
|
||||||
|
agent_provider_template=plan.agent_provider_template,
|
||||||
|
terminal_title=(
|
||||||
|
f"{plan.spec.label} ({plan.spec.agent_name})"
|
||||||
|
if plan.spec.label else plan.spec.agent_name
|
||||||
|
),
|
||||||
|
terminal_color=plan.spec.color,
|
||||||
|
agent_workdir=plan.workspace_plan.workdir,
|
||||||
|
)
|
||||||
|
bottle.prompt_path = provision(plan, bottle)
|
||||||
|
|
||||||
|
yield bottle
|
||||||
|
finally:
|
||||||
|
teardown()
|
||||||
|
|
||||||
|
|
||||||
|
def _mint_certs(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
|
||||||
|
egress_ca_host, egress_ca_cert_only = egress_tls_init(egress_state_dir(plan.slug))
|
||||||
|
egress_plan = dataclasses.replace(
|
||||||
|
plan.egress_plan,
|
||||||
|
mitmproxy_ca_host_path=egress_ca_host,
|
||||||
|
mitmproxy_ca_cert_only_host_path=egress_ca_cert_only,
|
||||||
|
)
|
||||||
|
return dataclasses.replace(plan, egress_plan=egress_plan)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
|
||||||
|
_docker_build(SIDECAR_BUNDLE_IMAGE, _REPO_DIR, dockerfile=SIDECAR_BUNDLE_DOCKERFILE)
|
||||||
|
committed = read_committed_image(plan.slug)
|
||||||
|
if committed and _image_exists(committed):
|
||||||
|
info(f"using committed image {committed!r}")
|
||||||
|
return dataclasses.replace(
|
||||||
|
plan,
|
||||||
|
agent_provision=dataclasses.replace(plan.agent_provision, image=committed),
|
||||||
|
)
|
||||||
|
_docker_build(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path)
|
||||||
|
return plan
|
||||||
|
|
||||||
|
|
||||||
|
def _provision_git_gate_keys(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
|
||||||
|
if not plan.git_gate_plan.upstreams:
|
||||||
|
return plan
|
||||||
|
git_gate_plan = provision_git_gate_dynamic_keys(
|
||||||
|
plan.manifest.bottle, plan.git_gate_plan, git_gate_state_dir(plan.slug),
|
||||||
|
)
|
||||||
|
return dataclasses.replace(plan, git_gate_plan=git_gate_plan)
|
||||||
|
|
||||||
|
|
||||||
|
def _stamp_agent_urls(
|
||||||
|
plan: FirecrackerBottlePlan, host_ip: str,
|
||||||
|
) -> FirecrackerBottlePlan:
|
||||||
|
proxy_url = f"http://{host_ip}:{EGRESS_PORT}"
|
||||||
|
supervise_url = (
|
||||||
|
f"http://{host_ip}:{SUPERVISE_PORT}/" if plan.supervise_plan is not None else ""
|
||||||
|
)
|
||||||
|
git_gate_url = (
|
||||||
|
f"http://{host_ip}:{_GIT_HTTP_PORT}" if plan.git_gate_plan.upstreams else ""
|
||||||
|
)
|
||||||
|
return dataclasses.replace(
|
||||||
|
plan,
|
||||||
|
agent_proxy_url=proxy_url,
|
||||||
|
agent_git_gate_url=git_gate_url,
|
||||||
|
agent_supervise_url=supervise_url,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --- sidecar bundle (Docker) ----------------------------------------
|
||||||
|
|
||||||
|
def _start_sidecar_bundle(
|
||||||
|
plan: FirecrackerBottlePlan, sidecar_name: str, host_ip: str,
|
||||||
|
) -> None:
|
||||||
|
argv = ["docker", "run", "--name", sidecar_name, "--detach", "--rm",
|
||||||
|
"-e", f"BOT_BOTTLE_SIDECAR_DAEMONS={','.join(_sidecar_daemons(plan))}"]
|
||||||
|
for entry in _sidecar_env_entries(plan):
|
||||||
|
argv += ["-e", entry]
|
||||||
|
for host_path, container_path, read_only in _sidecar_mounts(plan):
|
||||||
|
argv += ["-v", f"{host_path}:{container_path}{':ro' if read_only else ''}"]
|
||||||
|
# Publish on the slot's host TAP IP at fixed ports — each bottle
|
||||||
|
# has a distinct host_ip, so fixed ports never collide, and the VM
|
||||||
|
# reaches them at a stable, well-known address (its only route out).
|
||||||
|
for port in _sidecar_ports(plan):
|
||||||
|
argv += ["-p", f"{host_ip}:{port}:{port}"]
|
||||||
|
argv.append(SIDECAR_BUNDLE_IMAGE)
|
||||||
|
|
||||||
|
effective_env = {**dict(os.environ), **plan.agent_provision.provisioned_env}
|
||||||
|
token_values = egress_resolve_token_values(
|
||||||
|
plan.egress_plan.token_env_map, effective_env,
|
||||||
|
)
|
||||||
|
env = {**os.environ, **token_values}
|
||||||
|
info(f"docker run sidecar bundle {sidecar_name} (published on {host_ip})")
|
||||||
|
result = subprocess.run(argv, capture_output=True, text=True, env=env, check=False)
|
||||||
|
if result.returncode != 0:
|
||||||
|
die(f"docker run for sidecar bundle {sidecar_name} failed: "
|
||||||
|
f"{(result.stderr or '').strip() or '<no stderr>'}")
|
||||||
|
|
||||||
|
|
||||||
|
def _sidecar_daemons(plan: FirecrackerBottlePlan) -> tuple[str, ...]:
|
||||||
|
daemons = ["egress"]
|
||||||
|
if plan.git_gate_plan.upstreams:
|
||||||
|
daemons += ["git-gate", "git-http"]
|
||||||
|
if plan.supervise_plan is not None:
|
||||||
|
daemons.append("supervise")
|
||||||
|
return tuple(daemons)
|
||||||
|
|
||||||
|
|
||||||
|
def _sidecar_ports(plan: FirecrackerBottlePlan) -> tuple[int, ...]:
|
||||||
|
ports = [EGRESS_PORT]
|
||||||
|
if plan.git_gate_plan.upstreams:
|
||||||
|
ports.append(_GIT_HTTP_PORT)
|
||||||
|
if plan.supervise_plan is not None:
|
||||||
|
ports.append(SUPERVISE_PORT)
|
||||||
|
return tuple(ports)
|
||||||
|
|
||||||
|
|
||||||
|
def _sidecar_env_entries(plan: FirecrackerBottlePlan) -> tuple[str, ...]:
|
||||||
|
env: list[str] = list(egress_sidecar_env_entries(plan.egress_plan))
|
||||||
|
if plan.git_gate_plan.upstreams:
|
||||||
|
env.append(f"BOT_BOTTLE_GIT_GATE_READY_FILE={_GIT_GATE_READY_FILE}")
|
||||||
|
if plan.supervise_plan is not None:
|
||||||
|
env += [
|
||||||
|
f"SUPERVISE_BOTTLE_SLUG={plan.slug}",
|
||||||
|
f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}",
|
||||||
|
f"SUPERVISE_PORT={SUPERVISE_PORT}",
|
||||||
|
]
|
||||||
|
return tuple(env)
|
||||||
|
|
||||||
|
|
||||||
|
def _sidecar_mounts(
|
||||||
|
plan: FirecrackerBottlePlan,
|
||||||
|
) -> tuple[tuple[str, str, bool], ...]:
|
||||||
|
mounts: list[tuple[str, str, bool]] = []
|
||||||
|
ep = plan.egress_plan
|
||||||
|
mounts.append((str(ep.mitmproxy_ca_host_path.parent),
|
||||||
|
str(Path(EGRESS_CA_IN_CONTAINER).parent), False))
|
||||||
|
if ep.routes:
|
||||||
|
mounts.append((str(ep.routes_path.parent),
|
||||||
|
str(Path(EGRESS_ROUTES_IN_CONTAINER).parent), True))
|
||||||
|
sp = plan.supervise_plan
|
||||||
|
if sp is not None:
|
||||||
|
mounts.append((str(sp.db_path.parent),
|
||||||
|
str(Path(DB_PATH_IN_CONTAINER).parent), False))
|
||||||
|
return tuple(mounts)
|
||||||
|
|
||||||
|
|
||||||
|
def _stage_git_gate(plan: FirecrackerBottlePlan, sidecar_name: str) -> None:
|
||||||
|
gp = plan.git_gate_plan
|
||||||
|
if not gp.upstreams:
|
||||||
|
return
|
||||||
|
_docker_exec(sidecar_name, [
|
||||||
|
"mkdir", "-p",
|
||||||
|
str(Path(GIT_GATE_HOOK_IN_CONTAINER).parent),
|
||||||
|
GIT_GATE_CREDS_DIR_IN_CONTAINER, "/git",
|
||||||
|
str(Path(_GIT_GATE_READY_FILE).parent),
|
||||||
|
])
|
||||||
|
for host_path, container_path in _git_gate_files(plan):
|
||||||
|
_docker_cp(host_path, f"{sidecar_name}:{container_path}")
|
||||||
|
_docker_exec(sidecar_name, [
|
||||||
|
"sh", "-c",
|
||||||
|
f"chmod 755 {GIT_GATE_ENTRYPOINT_IN_CONTAINER} "
|
||||||
|
f"{GIT_GATE_HOOK_IN_CONTAINER} {GIT_GATE_ACCESS_HOOK_IN_CONTAINER} && "
|
||||||
|
f"chmod 600 {GIT_GATE_CREDS_DIR_IN_CONTAINER}/* && "
|
||||||
|
f"touch {_GIT_GATE_READY_FILE}",
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def _git_gate_files(plan: FirecrackerBottlePlan) -> tuple[tuple[str, str], ...]:
|
||||||
|
gp = plan.git_gate_plan
|
||||||
|
files: list[tuple[str, str]] = [
|
||||||
|
(str(gp.entrypoint_script), GIT_GATE_ENTRYPOINT_IN_CONTAINER),
|
||||||
|
(str(gp.hook_script), GIT_GATE_HOOK_IN_CONTAINER),
|
||||||
|
(str(gp.access_hook_script), GIT_GATE_ACCESS_HOOK_IN_CONTAINER),
|
||||||
|
]
|
||||||
|
for upstream in gp.upstreams:
|
||||||
|
files.append((expand_tilde(upstream.identity_file),
|
||||||
|
f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{upstream.name}-key"))
|
||||||
|
if upstream.known_hosts_file:
|
||||||
|
files.append((str(upstream.known_hosts_file),
|
||||||
|
f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{upstream.name}-known_hosts"))
|
||||||
|
return tuple(files)
|
||||||
|
|
||||||
|
|
||||||
|
# --- agent guest env -------------------------------------------------
|
||||||
|
|
||||||
|
def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str]:
|
||||||
|
"""Env injected into every agent/exec call over SSH. The VM has no
|
||||||
|
baked process env (it just runs init), so the proxy/CA/git/supervise
|
||||||
|
wiring is applied per-invocation."""
|
||||||
|
proxy_url = f"http://{host_ip}:{EGRESS_PORT}"
|
||||||
|
no_proxy = f"localhost,127.0.0.1,{host_ip}"
|
||||||
|
env: dict[str, str] = {
|
||||||
|
"HTTPS_PROXY": proxy_url, "HTTP_PROXY": proxy_url,
|
||||||
|
"https_proxy": proxy_url, "http_proxy": proxy_url,
|
||||||
|
"NO_PROXY": no_proxy, "no_proxy": no_proxy,
|
||||||
|
"NODE_EXTRA_CA_CERTS": AGENT_CA_PATH,
|
||||||
|
"SSL_CERT_FILE": AGENT_CA_BUNDLE,
|
||||||
|
"REQUESTS_CA_BUNDLE": AGENT_CA_BUNDLE,
|
||||||
|
}
|
||||||
|
if plan.agent_git_gate_url:
|
||||||
|
env["GIT_GATE_URL"] = plan.agent_git_gate_url
|
||||||
|
if plan.agent_supervise_url:
|
||||||
|
env["MCP_SUPERVISE_URL"] = plan.agent_supervise_url
|
||||||
|
for entry in egress_agent_env_entries(plan.egress_plan):
|
||||||
|
key, _, value = entry.partition("=")
|
||||||
|
env[key] = value
|
||||||
|
env.update(plan.agent_provision.guest_env)
|
||||||
|
# Forwarded (bare-name) env: resolve host values now, since the VM
|
||||||
|
# can't inherit them from a `docker run --env NAME`.
|
||||||
|
for name in plan.forwarded_env:
|
||||||
|
value = os.environ.get(name)
|
||||||
|
if value is not None:
|
||||||
|
env[name] = value
|
||||||
|
return env
|
||||||
|
|
||||||
|
|
||||||
|
# --- docker helpers --------------------------------------------------
|
||||||
|
|
||||||
|
def _docker_build(ref: str, context: str, *, dockerfile: str = "") -> None:
|
||||||
|
info(f"docker build {ref}")
|
||||||
|
args = ["docker", "build", "-t", ref]
|
||||||
|
if dockerfile:
|
||||||
|
if not os.path.isabs(dockerfile):
|
||||||
|
dockerfile = os.path.join(context, dockerfile)
|
||||||
|
args += ["-f", dockerfile]
|
||||||
|
args.append(context)
|
||||||
|
result = subprocess.run(args, check=False)
|
||||||
|
if result.returncode != 0:
|
||||||
|
die(f"docker build for {ref!r} failed")
|
||||||
|
|
||||||
|
|
||||||
|
def _image_exists(ref: str) -> bool:
|
||||||
|
return subprocess.run(
|
||||||
|
["docker", "image", "inspect", ref],
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
||||||
|
).returncode == 0
|
||||||
|
|
||||||
|
|
||||||
|
def _force_remove_container(name: str) -> None:
|
||||||
|
subprocess.run(
|
||||||
|
["docker", "rm", "-f", name],
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _docker_exec(name: str, argv: list[str]) -> None:
|
||||||
|
result = subprocess.run(
|
||||||
|
["docker", "exec", name, *argv], capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
die(f"docker exec in {name} failed: "
|
||||||
|
f"{(result.stderr or '').strip() or '<no stderr>'}")
|
||||||
|
|
||||||
|
|
||||||
|
def _docker_cp(host_path: str, dest: str) -> None:
|
||||||
|
result = subprocess.run(
|
||||||
|
["docker", "cp", host_path, dest], capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
die(f"docker cp {host_path} -> {dest} failed: "
|
||||||
|
f"{(result.stderr or '').strip() or '<no stderr>'}")
|
||||||
@@ -0,0 +1,243 @@
|
|||||||
|
"""Firecracker network pool: constants, IP math, allocation, and the
|
||||||
|
config renderers (shell command + NixOS module) shown to operators.
|
||||||
|
|
||||||
|
The Firecracker backend needs a privileged one-time network setup:
|
||||||
|
a pool of point-to-point TAP devices (owned by the invoking user, so
|
||||||
|
`./cli.py start` never needs root) and a dedicated nftables table that
|
||||||
|
isolates every VM. This module is the single source of truth for the
|
||||||
|
pool parameters — the shell script (`scripts/firecracker-netpool.sh`),
|
||||||
|
the printed NixOS snippet, and the backend's fail-closed preflight all
|
||||||
|
derive from the constants here so they can't drift.
|
||||||
|
|
||||||
|
Topology (per slot i):
|
||||||
|
* TAP ``bbfc{i}`` — no shared bridge, so no docker0 / virbr0 / cni0
|
||||||
|
/ br-* collisions.
|
||||||
|
* a /31 host<->guest link: host = base + 2i (the gateway the VM
|
||||||
|
routes through), guest = base + 2i + 1 (the VM's address).
|
||||||
|
* isolation via ``table inet bot_bottle_fc``: a VM reaches only its
|
||||||
|
own sidecar (DNAT'd from the host TAP IP) and nothing else.
|
||||||
|
|
||||||
|
The default IP block is RFC-6598 shared address space (100.64.0.0/10),
|
||||||
|
which is purpose-built to avoid colliding with LAN/VPN private ranges.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import fcntl
|
||||||
|
import ipaddress
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import IO
|
||||||
|
|
||||||
|
from ...log import die
|
||||||
|
|
||||||
|
|
||||||
|
# Interface names are capped at 15 chars (IFNAMSIZ-1); "bbfc" + a small
|
||||||
|
# index stays well under that and is distinctive enough to grep for.
|
||||||
|
IFACE_PREFIX = os.environ.get("BOT_BOTTLE_FC_IFACE_PREFIX", "bbfc")
|
||||||
|
NFT_TABLE = "bot_bottle_fc"
|
||||||
|
|
||||||
|
|
||||||
|
def pool_size() -> int:
|
||||||
|
return int(os.environ.get("BOT_BOTTLE_FC_POOL_SIZE", "8"))
|
||||||
|
|
||||||
|
|
||||||
|
def ip_base() -> str:
|
||||||
|
return os.environ.get("BOT_BOTTLE_FC_IP_BASE", "100.64.0.0")
|
||||||
|
|
||||||
|
|
||||||
|
# Sidecar ports the VM reaches at its host-side TAP IP. Kept in sync
|
||||||
|
# with the backend constants (egress 9099, supervise 9100, git-http
|
||||||
|
# 9420); rendered into the setup output for operator visibility.
|
||||||
|
SIDECAR_PORTS = (9099, 9100, 9420)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Slot:
|
||||||
|
"""One pool slot: a TAP device and its host/guest /31 addresses."""
|
||||||
|
|
||||||
|
index: int
|
||||||
|
iface: str
|
||||||
|
host_ip: str
|
||||||
|
guest_ip: str
|
||||||
|
|
||||||
|
@property
|
||||||
|
def guest_cidr(self) -> str:
|
||||||
|
"""Guest address with the /31 prefix, for the kernel `ip=` arg."""
|
||||||
|
return f"{self.guest_ip}/31"
|
||||||
|
|
||||||
|
|
||||||
|
def slot(index: int) -> Slot:
|
||||||
|
base = int(ipaddress.IPv4Address(ip_base()))
|
||||||
|
return Slot(
|
||||||
|
index=index,
|
||||||
|
iface=f"{IFACE_PREFIX}{index}",
|
||||||
|
host_ip=str(ipaddress.IPv4Address(base + 2 * index)),
|
||||||
|
guest_ip=str(ipaddress.IPv4Address(base + 2 * index + 1)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def all_slots() -> list[Slot]:
|
||||||
|
return [slot(i) for i in range(pool_size())]
|
||||||
|
|
||||||
|
|
||||||
|
# --- fail-closed verification ---------------------------------------
|
||||||
|
|
||||||
|
def _run_ok(argv: list[str]) -> bool:
|
||||||
|
"""Run a probe command, treating a missing binary as failure
|
||||||
|
(rather than crashing) so callers can stay fail-closed."""
|
||||||
|
try:
|
||||||
|
return subprocess.run(
|
||||||
|
argv, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||||
|
check=False,
|
||||||
|
).returncode == 0
|
||||||
|
except FileNotFoundError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def nft_table_present() -> bool:
|
||||||
|
"""True iff the isolation table exists in the active nftables
|
||||||
|
backend. The backend's preflight treats absence as fatal — the VM
|
||||||
|
must not boot without its egress boundary in place.
|
||||||
|
|
||||||
|
Listing may require root; when it can't be confirmed here the
|
||||||
|
launch path falls back to an empirical post-boot isolation probe.
|
||||||
|
Returns False (fail-closed) if `nft` is unavailable."""
|
||||||
|
return _run_ok(["nft", "list", "table", "inet", NFT_TABLE])
|
||||||
|
|
||||||
|
|
||||||
|
def tap_present(iface: str) -> bool:
|
||||||
|
# `ip link show` is unprivileged, so the TAP-pool check is reliable
|
||||||
|
# for the non-root launcher.
|
||||||
|
return _run_ok(["ip", "link", "show", iface])
|
||||||
|
|
||||||
|
|
||||||
|
def missing_taps() -> list[str]:
|
||||||
|
"""Pool TAPs that the one-time setup has not created yet."""
|
||||||
|
return [s.iface for s in all_slots() if not tap_present(s.iface)]
|
||||||
|
|
||||||
|
|
||||||
|
# --- allocation ------------------------------------------------------
|
||||||
|
|
||||||
|
def _lock_dir() -> Path:
|
||||||
|
d = Path.home() / ".cache" / "bot-bottle" / "firecracker" / "pool"
|
||||||
|
d.mkdir(parents=True, exist_ok=True)
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
def allocate(slug: str) -> tuple[Slot, IO[str]]:
|
||||||
|
"""Claim a free pool slot for one bottle. Returns the slot and the
|
||||||
|
held lock file — the caller keeps it open for the VM's lifetime and
|
||||||
|
closes it on teardown; the flock auto-releases if the launcher
|
||||||
|
crashes, so a slot is never leaked. Dies when the pool is
|
||||||
|
exhausted.
|
||||||
|
|
||||||
|
A per-slot `flock` (rather than inspecting running processes) makes
|
||||||
|
allocation race-free across concurrent launches without a central
|
||||||
|
registry: the first launcher to grab the lock owns the slot."""
|
||||||
|
del slug # logged by the caller; allocation is purely lock-driven
|
||||||
|
for s in all_slots():
|
||||||
|
lock_path = _lock_dir() / f"{s.iface}.lock"
|
||||||
|
handle = open(lock_path, "w", encoding="utf-8")
|
||||||
|
try:
|
||||||
|
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
except OSError:
|
||||||
|
handle.close()
|
||||||
|
continue
|
||||||
|
return s, handle
|
||||||
|
die(f"Firecracker TAP pool exhausted ({pool_size()} slots, all in "
|
||||||
|
f"use). Stop a running bottle or raise BOT_BOTTLE_FC_POOL_SIZE "
|
||||||
|
f"and re-run `./cli.py firecracker setup`.")
|
||||||
|
raise AssertionError("unreachable")
|
||||||
|
|
||||||
|
|
||||||
|
# --- config renderers (shown by `./cli.py firecracker setup`) --------
|
||||||
|
|
||||||
|
def render_shell_setup() -> str:
|
||||||
|
"""The imperative command for non-NixOS hosts."""
|
||||||
|
env = _nondefault_env()
|
||||||
|
prefix = f"{env} " if env else ""
|
||||||
|
return f"sudo {prefix}./scripts/firecracker-netpool.sh up"
|
||||||
|
|
||||||
|
|
||||||
|
def render_nixos_module() -> str:
|
||||||
|
"""A paste-ready NixOS snippet, derived from the live constants.
|
||||||
|
|
||||||
|
Uses systemd-networkd tap netdevs (so TAPs are only reconfigured
|
||||||
|
when this config changes, not torn down on every rebuild — which
|
||||||
|
would drop running VMs) and `networking.nftables.tables.<name>`
|
||||||
|
(an independent table that coexists with Docker's iptables rules
|
||||||
|
rather than replacing the whole ruleset)."""
|
||||||
|
slots = all_slots()
|
||||||
|
owner = os.environ.get("USER", "youruser")
|
||||||
|
|
||||||
|
netdevs = "\n".join(
|
||||||
|
f''' "10-{s.iface}" = {{
|
||||||
|
netdevConfig = {{ Name = "{s.iface}"; Kind = "tap"; }};
|
||||||
|
tapConfig = {{ User = "{owner}"; }};
|
||||||
|
}};'''
|
||||||
|
for s in slots
|
||||||
|
)
|
||||||
|
networks = "\n".join(
|
||||||
|
f''' "10-{s.iface}" = {{
|
||||||
|
matchConfig.Name = "{s.iface}";
|
||||||
|
address = [ "{s.host_ip}/31" ];
|
||||||
|
networkConfig.ConfigureWithoutCarrier = true;
|
||||||
|
}};'''
|
||||||
|
for s in slots
|
||||||
|
)
|
||||||
|
|
||||||
|
return f"""# bot-bottle Firecracker network pool ({len(slots)} slots, base {ip_base()}).
|
||||||
|
# Generated by `./cli.py firecracker setup`; owner user = {owner!r}.
|
||||||
|
{{ ... }}:
|
||||||
|
{{
|
||||||
|
boot.kernel.sysctl."net.ipv4.ip_forward" = 1;
|
||||||
|
|
||||||
|
systemd.network.enable = true;
|
||||||
|
systemd.network.netdevs = {{
|
||||||
|
{netdevs}
|
||||||
|
}};
|
||||||
|
systemd.network.networks = {{
|
||||||
|
{networks}
|
||||||
|
}};
|
||||||
|
|
||||||
|
# Independent table — does not touch Docker/ufw/firewalld rules.
|
||||||
|
# A bottle VM (traffic from a bbfc* TAP) may reach only its own
|
||||||
|
# sidecar (DNAT'd from the host TAP IP); everything else is dropped,
|
||||||
|
# so there is no egress except through the sidecar proxy.
|
||||||
|
networking.nftables.enable = true;
|
||||||
|
networking.nftables.tables."{NFT_TABLE}" = {{
|
||||||
|
family = "inet";
|
||||||
|
content = ''
|
||||||
|
chain forward {{
|
||||||
|
type filter hook forward priority -10; policy accept;
|
||||||
|
iifname != "{IFACE_PREFIX}*" return
|
||||||
|
ct state established,related accept
|
||||||
|
ct status dnat accept
|
||||||
|
drop
|
||||||
|
}}
|
||||||
|
chain input {{
|
||||||
|
type filter hook input priority -10; policy accept;
|
||||||
|
iifname != "{IFACE_PREFIX}*" return
|
||||||
|
ct state established,related accept
|
||||||
|
drop
|
||||||
|
}}
|
||||||
|
'';
|
||||||
|
}};
|
||||||
|
}}
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _nondefault_env() -> str:
|
||||||
|
"""Render any non-default pool env overrides so the printed shell
|
||||||
|
command reproduces the operator's current settings."""
|
||||||
|
pairs = []
|
||||||
|
if os.environ.get("BOT_BOTTLE_FC_POOL_SIZE"):
|
||||||
|
pairs.append(f"BOT_BOTTLE_FC_POOL_SIZE={pool_size()}")
|
||||||
|
if os.environ.get("BOT_BOTTLE_FC_IP_BASE"):
|
||||||
|
pairs.append(f"BOT_BOTTLE_FC_IP_BASE={ip_base()}")
|
||||||
|
if os.environ.get("BOT_BOTTLE_FC_IFACE_PREFIX"):
|
||||||
|
pairs.append(f"BOT_BOTTLE_FC_IFACE_PREFIX={IFACE_PREFIX}")
|
||||||
|
return " ".join(pairs)
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
"""Prepare step for the Firecracker backend."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ...agent_provider import AgentProvisionPlan
|
||||||
|
from ...egress import EgressPlan
|
||||||
|
from ...env import ResolvedEnv
|
||||||
|
from ...git_gate import GitGatePlan
|
||||||
|
from ...manifest import Manifest
|
||||||
|
from ...supervise import SupervisePlan
|
||||||
|
from .. import BottleSpec
|
||||||
|
from . import util
|
||||||
|
from .bottle_plan import FirecrackerBottlePlan
|
||||||
|
|
||||||
|
|
||||||
|
def preflight() -> None:
|
||||||
|
util.require_firecracker()
|
||||||
|
|
||||||
|
|
||||||
|
def build_guest_env(resolved_env: ResolvedEnv) -> dict[str, str]:
|
||||||
|
return dict(resolved_env.literals)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_plan(
|
||||||
|
spec: BottleSpec,
|
||||||
|
manifest: Manifest,
|
||||||
|
slug: str,
|
||||||
|
resolved_env: ResolvedEnv,
|
||||||
|
agent_provision_plan: AgentProvisionPlan,
|
||||||
|
egress_plan: EgressPlan,
|
||||||
|
supervise_plan: SupervisePlan | None,
|
||||||
|
git_gate_plan: GitGatePlan,
|
||||||
|
stage_dir: Path,
|
||||||
|
) -> FirecrackerBottlePlan:
|
||||||
|
return FirecrackerBottlePlan(
|
||||||
|
spec=spec,
|
||||||
|
manifest=manifest,
|
||||||
|
stage_dir=stage_dir,
|
||||||
|
slug=slug,
|
||||||
|
forwarded_env=dict(resolved_env.forwarded),
|
||||||
|
git_gate_plan=git_gate_plan,
|
||||||
|
egress_plan=egress_plan,
|
||||||
|
supervise_plan=supervise_plan,
|
||||||
|
agent_provision=agent_provision_plan,
|
||||||
|
)
|
||||||
@@ -0,0 +1,296 @@
|
|||||||
|
"""Host-side primitives for the Firecracker backend.
|
||||||
|
|
||||||
|
Covers the pieces that don't need root at launch time: locating the
|
||||||
|
firecracker binary / guest kernel / injected dropbear, the fail-closed
|
||||||
|
preflight (KVM + kernel + isolation table + TAP pool must all be
|
||||||
|
present before a VM boots), the rootless rootfs pipeline
|
||||||
|
(`docker export` -> `mke2fs -d`, no mount), and per-bottle SSH key
|
||||||
|
generation.
|
||||||
|
|
||||||
|
The privileged network setup (TAP pool + nft table) is a one-time
|
||||||
|
operator step — see `netpool.py`, `scripts/firecracker-netpool.sh`,
|
||||||
|
and `./cli.py firecracker setup`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import platform
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ...log import die, info, warn
|
||||||
|
from . import netpool
|
||||||
|
|
||||||
|
|
||||||
|
# Guest agent images are Debian-family with USER node; the VM is
|
||||||
|
# reached over SSH as root (init drops the pubkey into both root's and
|
||||||
|
# node's authorized_keys) and commands `runuser` down to node.
|
||||||
|
GUEST_SSH_USER = "root"
|
||||||
|
|
||||||
|
# `/dev/kvm` must exist and be openable by the invoking user.
|
||||||
|
_KVM_DEVICE = "/dev/kvm"
|
||||||
|
|
||||||
|
|
||||||
|
def cache_dir() -> Path:
|
||||||
|
d = Path(
|
||||||
|
os.environ.get(
|
||||||
|
"BOT_BOTTLE_FC_CACHE",
|
||||||
|
str(Path.home() / ".cache" / "bot-bottle" / "firecracker"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
d.mkdir(parents=True, exist_ok=True)
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
def kernel_path() -> Path:
|
||||||
|
return Path(os.environ.get("BOT_BOTTLE_FC_KERNEL", str(cache_dir() / "vmlinux")))
|
||||||
|
|
||||||
|
|
||||||
|
def dropbear_path() -> Path:
|
||||||
|
"""The static dropbear injected into every guest rootfs as its SSH
|
||||||
|
server. Must be statically linked — the guest has none of the
|
||||||
|
host's shared libraries."""
|
||||||
|
return Path(
|
||||||
|
os.environ.get("BOT_BOTTLE_FC_DROPBEAR", str(cache_dir() / "dropbear"))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --- availability + preflight ---------------------------------------
|
||||||
|
|
||||||
|
def is_linux() -> bool:
|
||||||
|
return platform.system() == "Linux"
|
||||||
|
|
||||||
|
|
||||||
|
def is_available() -> bool:
|
||||||
|
"""Cheap capability probe used by cross-backend enumeration —
|
||||||
|
firecracker on PATH, on Linux, with KVM. Does not check the
|
||||||
|
(operator-provisioned) kernel / pool, so an available-but-unset
|
||||||
|
host still shows up and gets an actionable error at launch."""
|
||||||
|
return (
|
||||||
|
is_linux()
|
||||||
|
and shutil.which("firecracker") is not None
|
||||||
|
and os.path.exists(_KVM_DEVICE)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def require_firecracker() -> None:
|
||||||
|
"""Fail-closed preflight. Every check that gates the security
|
||||||
|
boundary (the isolation table, the TAP pool) errors rather than
|
||||||
|
booting a VM without it."""
|
||||||
|
if not is_linux():
|
||||||
|
die("firecracker backend is only supported on Linux (KVM). "
|
||||||
|
"On macOS use --backend=macos-container.")
|
||||||
|
if shutil.which("firecracker") is None:
|
||||||
|
info("Install: https://github.com/firecracker-microvm/firecracker/releases")
|
||||||
|
die("firecracker not found on PATH")
|
||||||
|
_require_kvm()
|
||||||
|
if not kernel_path().is_file():
|
||||||
|
die(f"guest kernel not found at {kernel_path()}. Set "
|
||||||
|
"BOT_BOTTLE_FC_KERNEL or place a vmlinux there.")
|
||||||
|
if not dropbear_path().is_file():
|
||||||
|
die(f"static dropbear not found at {dropbear_path()}. Set "
|
||||||
|
"BOT_BOTTLE_FC_DROPBEAR or cache one there.")
|
||||||
|
if shutil.which("mke2fs") is None:
|
||||||
|
die("mke2fs (e2fsprogs) not found — required to build guest rootfs")
|
||||||
|
_require_network_pool()
|
||||||
|
|
||||||
|
|
||||||
|
def _require_kvm() -> None:
|
||||||
|
if not os.path.exists(_KVM_DEVICE):
|
||||||
|
die(f"{_KVM_DEVICE} is missing. Enable KVM (load kvm-intel/kvm-amd; "
|
||||||
|
"confirm virtualization is on in firmware).")
|
||||||
|
if not os.access(_KVM_DEVICE, os.R_OK | os.W_OK):
|
||||||
|
die(f"{_KVM_DEVICE} exists but is not accessible. Add your user to "
|
||||||
|
"the `kvm` group and re-login.")
|
||||||
|
|
||||||
|
|
||||||
|
def _require_network_pool() -> None:
|
||||||
|
"""Fail-closed on the parts we can check unprivileged; defer the
|
||||||
|
rest to the post-boot isolation probe.
|
||||||
|
|
||||||
|
The TAP pool is verified here (`ip link show` is unprivileged). The
|
||||||
|
nft table can only be *confirmed present* here when `nft` is
|
||||||
|
queryable — which usually needs root — so a missing/absent nft is
|
||||||
|
not treated as fatal at this stage: the authoritative check is the
|
||||||
|
empirical isolation probe run after boot, before the agent starts
|
||||||
|
(see isolation_probe.verify_isolation). What we must never do is
|
||||||
|
boot without the TAP pool."""
|
||||||
|
missing = netpool.missing_taps()
|
||||||
|
if missing:
|
||||||
|
die(f"network pool incomplete — missing TAP devices: "
|
||||||
|
f"{', '.join(missing)}.\n ./cli.py firecracker setup")
|
||||||
|
if shutil.which("nft") is not None and not netpool.nft_table_present():
|
||||||
|
# nft is queryable and says the table is absent — that's a
|
||||||
|
# definite, catchable misconfiguration; fail early.
|
||||||
|
warn(f"isolation table `inet {netpool.NFT_TABLE}` not found via nft. "
|
||||||
|
"If this is a permissions issue it will be re-checked "
|
||||||
|
"empirically after boot; otherwise run: "
|
||||||
|
"./cli.py firecracker setup")
|
||||||
|
|
||||||
|
|
||||||
|
# --- rootfs pipeline (rootless) -------------------------------------
|
||||||
|
|
||||||
|
def docker_image_id(ref: str) -> str:
|
||||||
|
"""The image's content digest, used as the rootfs cache key."""
|
||||||
|
result = subprocess.run(
|
||||||
|
["docker", "image", "inspect", "--format", "{{.Id}}", ref],
|
||||||
|
capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
if result.returncode != 0 or not result.stdout.strip():
|
||||||
|
die(f"docker image inspect for {ref!r} failed: "
|
||||||
|
f"{(result.stderr or '').strip() or '<no output>'}")
|
||||||
|
return result.stdout.strip().replace("sha256:", "")[:16]
|
||||||
|
|
||||||
|
|
||||||
|
def build_base_rootfs_dir(image_ref: str) -> Path:
|
||||||
|
"""Export the agent image's filesystem and inject the guest init +
|
||||||
|
static dropbear. Cached by image digest — the per-bottle bits
|
||||||
|
(authorized_keys, IP) are passed at boot via the kernel cmdline, so
|
||||||
|
this tree carries nothing bottle-specific and is safely shared.
|
||||||
|
|
||||||
|
Returns the prepared directory (read as the `mke2fs -d` source)."""
|
||||||
|
digest = docker_image_id(image_ref)
|
||||||
|
base = cache_dir() / "rootfs" / digest
|
||||||
|
ready = base / ".bb-ready"
|
||||||
|
if ready.is_file():
|
||||||
|
return base
|
||||||
|
|
||||||
|
if base.exists():
|
||||||
|
shutil.rmtree(base, ignore_errors=True)
|
||||||
|
base.mkdir(parents=True)
|
||||||
|
|
||||||
|
info(f"exporting {image_ref} rootfs -> {base}")
|
||||||
|
cid = subprocess.run(
|
||||||
|
["docker", "create", image_ref, "sleep", "infinity"],
|
||||||
|
capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
if cid.returncode != 0 or not cid.stdout.strip():
|
||||||
|
die(f"docker create {image_ref!r} failed: {cid.stderr.strip()}")
|
||||||
|
container = cid.stdout.strip()
|
||||||
|
try:
|
||||||
|
export = subprocess.Popen(
|
||||||
|
["docker", "export", container], stdout=subprocess.PIPE,
|
||||||
|
)
|
||||||
|
untar = subprocess.run(
|
||||||
|
["tar", "-x", "-C", str(base)], stdin=export.stdout, check=False,
|
||||||
|
)
|
||||||
|
export.wait()
|
||||||
|
if export.returncode != 0 or untar.returncode != 0:
|
||||||
|
die(f"exporting rootfs for {image_ref!r} failed")
|
||||||
|
finally:
|
||||||
|
subprocess.run(
|
||||||
|
["docker", "rm", "-f", container],
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
_inject_guest_boot(base)
|
||||||
|
ready.write_text("ok\n")
|
||||||
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
def _inject_guest_boot(rootfs: Path) -> None:
|
||||||
|
"""Drop the static dropbear and the PID-1 init into the rootfs."""
|
||||||
|
shutil.copy2(dropbear_path(), rootfs / "bb-dropbear")
|
||||||
|
os.chmod(rootfs / "bb-dropbear", 0o755)
|
||||||
|
init = rootfs / "bb-init"
|
||||||
|
init.write_text(_GUEST_INIT)
|
||||||
|
os.chmod(init, 0o755)
|
||||||
|
|
||||||
|
|
||||||
|
def build_rootfs_ext4(base_dir: Path, out_path: Path, *, slack_mib: int = 1024) -> None:
|
||||||
|
"""Build a fresh, writable ext4 for one bottle from the cached base
|
||||||
|
dir. Rootless: `mke2fs -d` populates the image from a directory
|
||||||
|
without mounting. Each call produces an independent disk, so the
|
||||||
|
shared base dir stays untouched and concurrent bottles don't race."""
|
||||||
|
used_mib = _dir_size_mib(base_dir)
|
||||||
|
size_mib = used_mib + slack_mib
|
||||||
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
result = subprocess.run(
|
||||||
|
["mke2fs", "-q", "-t", "ext4", "-d", str(base_dir), "-F",
|
||||||
|
str(out_path), f"{size_mib}M"],
|
||||||
|
capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
die(f"mke2fs for {out_path} failed: {result.stderr.strip()}")
|
||||||
|
|
||||||
|
|
||||||
|
def _dir_size_mib(path: Path) -> int:
|
||||||
|
result = subprocess.run(
|
||||||
|
["du", "-sm", str(path)], capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
return int(result.stdout.split()[0])
|
||||||
|
except (ValueError, IndexError):
|
||||||
|
return 2048
|
||||||
|
|
||||||
|
|
||||||
|
# --- per-bottle SSH keys --------------------------------------------
|
||||||
|
|
||||||
|
def generate_keypair(dest_dir: Path) -> tuple[Path, str]:
|
||||||
|
"""Mint a per-bottle ed25519 keypair. Returns (private_key_path,
|
||||||
|
public_key_line). The public line is injected into the guest via
|
||||||
|
the boot cmdline; the private key authenticates host->guest SSH."""
|
||||||
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
key = dest_dir / "bottle_id_ed25519"
|
||||||
|
if key.exists():
|
||||||
|
key.unlink()
|
||||||
|
(dest_dir / "bottle_id_ed25519.pub").unlink(missing_ok=True)
|
||||||
|
subprocess.run(
|
||||||
|
["ssh-keygen", "-t", "ed25519", "-N", "", "-q", "-f", str(key),
|
||||||
|
"-C", "bot-bottle-firecracker"],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
pub = (dest_dir / "bottle_id_ed25519.pub").read_text().strip()
|
||||||
|
return key, pub
|
||||||
|
|
||||||
|
|
||||||
|
def ssh_base_argv(private_key: Path, guest_ip: str) -> list[str]:
|
||||||
|
"""Common SSH options for host->guest control. The VM is ephemeral
|
||||||
|
and per-bottle, so host-key TOFU is meaningless — pin no known_hosts
|
||||||
|
and skip the check rather than accumulate churn."""
|
||||||
|
return [
|
||||||
|
"ssh",
|
||||||
|
"-i", str(private_key),
|
||||||
|
"-o", "StrictHostKeyChecking=no",
|
||||||
|
"-o", "UserKnownHostsFile=/dev/null",
|
||||||
|
"-o", "LogLevel=ERROR",
|
||||||
|
"-o", "ConnectTimeout=5",
|
||||||
|
f"{GUEST_SSH_USER}@{guest_ip}",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# PID-1 init injected into every guest. Kept dependency-light: relies
|
||||||
|
# only on coreutils + a POSIX shell (present in the Debian-family agent
|
||||||
|
# images). The kernel `ip=` cmdline configures eth0 before init runs,
|
||||||
|
# so no iproute2 is needed. The per-bottle SSH pubkey arrives base64 on
|
||||||
|
# the cmdline; dropbear generates ephemeral host keys with -R.
|
||||||
|
_GUEST_INIT = r"""#!/bin/sh
|
||||||
|
# bot-bottle Firecracker guest init (PID 1).
|
||||||
|
mount -t proc proc /proc 2>/dev/null
|
||||||
|
mount -t sysfs sys /sys 2>/dev/null
|
||||||
|
mount -t devtmpfs dev /dev 2>/dev/null
|
||||||
|
mkdir -p /dev/pts && mount -t devpts devpts /dev/pts 2>/dev/null
|
||||||
|
mount -o remount,rw / 2>/dev/null
|
||||||
|
|
||||||
|
# Install the per-bottle SSH pubkey from the kernel cmdline.
|
||||||
|
KEY=$(sed -n 's/.*bb_pubkey=\([^ ]*\).*/\1/p' /proc/cmdline | base64 -d 2>/dev/null)
|
||||||
|
if [ -n "$KEY" ]; then
|
||||||
|
for home in /root /home/node; do
|
||||||
|
mkdir -p "$home/.ssh"
|
||||||
|
printf '%s\n' "$KEY" > "$home/.ssh/authorized_keys"
|
||||||
|
chmod 700 "$home/.ssh"
|
||||||
|
chmod 600 "$home/.ssh/authorized_keys"
|
||||||
|
done
|
||||||
|
chown -R node:node /home/node/.ssh 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p /etc/dropbear /run
|
||||||
|
# -R: generate host keys on demand. -E: log to stderr (guest console).
|
||||||
|
/bb-dropbear -R -E -p 22 2>/dev/null &
|
||||||
|
|
||||||
|
# Reap zombies as PID 1. dropbear is always a child, so `wait` blocks
|
||||||
|
# rather than busy-looping.
|
||||||
|
while : ; do wait ; done
|
||||||
|
"""
|
||||||
@@ -90,11 +90,15 @@ def get_freezer(backend_name: str) -> Freezer:
|
|||||||
if resolved == "macos-container":
|
if resolved == "macos-container":
|
||||||
from .macos_container.freezer import MacosContainerFreezer
|
from .macos_container.freezer import MacosContainerFreezer
|
||||||
return MacosContainerFreezer()
|
return MacosContainerFreezer()
|
||||||
|
if resolved == "firecracker":
|
||||||
|
from .firecracker.freezer import FirecrackerFreezer
|
||||||
|
return FirecrackerFreezer()
|
||||||
if resolved == "smolmachines":
|
if resolved == "smolmachines":
|
||||||
from .smolmachines.freezer import SmolmachinesFreezer
|
from .smolmachines.freezer import SmolmachinesFreezer
|
||||||
return SmolmachinesFreezer()
|
return SmolmachinesFreezer()
|
||||||
die(
|
die(
|
||||||
f"commit is only supported for docker, macos-container, and "
|
f"commit is only supported for docker, macos-container, "
|
||||||
f"smolmachines; backend {backend_name!r} has no freezer"
|
f"firecracker, and smolmachines; backend {backend_name!r} has "
|
||||||
|
f"no freezer"
|
||||||
)
|
)
|
||||||
raise AssertionError("unreachable")
|
raise AssertionError("unreachable")
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from . import list as _list_mod
|
|||||||
from .cleanup import cmd_cleanup
|
from .cleanup import cmd_cleanup
|
||||||
from .commit import cmd_commit
|
from .commit import cmd_commit
|
||||||
from .edit import cmd_edit
|
from .edit import cmd_edit
|
||||||
|
from .firecracker import cmd_firecracker
|
||||||
from .info import cmd_info
|
from .info import cmd_info
|
||||||
from .init import cmd_init
|
from .init import cmd_init
|
||||||
from .resume import cmd_resume
|
from .resume import cmd_resume
|
||||||
@@ -28,6 +29,7 @@ COMMANDS = {
|
|||||||
"cleanup": cmd_cleanup,
|
"cleanup": cmd_cleanup,
|
||||||
"commit": cmd_commit,
|
"commit": cmd_commit,
|
||||||
"edit": cmd_edit,
|
"edit": cmd_edit,
|
||||||
|
"firecracker": cmd_firecracker,
|
||||||
"info": cmd_info,
|
"info": cmd_info,
|
||||||
"init": cmd_init,
|
"init": cmd_init,
|
||||||
"list": cmd_list,
|
"list": cmd_list,
|
||||||
@@ -43,6 +45,7 @@ def usage() -> None:
|
|||||||
sys.stderr.write(" cleanup stop and remove all active bot-bottle containers\n")
|
sys.stderr.write(" cleanup stop and remove all active bot-bottle containers\n")
|
||||||
sys.stderr.write(" commit snapshot a running bottle's container state to a Docker image\n")
|
sys.stderr.write(" commit snapshot a running bottle's container state to a Docker image\n")
|
||||||
sys.stderr.write(" edit open an agent in vim for editing\n")
|
sys.stderr.write(" edit open an agent in vim for editing\n")
|
||||||
|
sys.stderr.write(" firecracker one-time network setup for the Firecracker backend\n")
|
||||||
sys.stderr.write(" info print env, skills, and prompt details for a named agent\n")
|
sys.stderr.write(" info print env, skills, and prompt details for a named agent\n")
|
||||||
sys.stderr.write(" init interactively create a new agent and add it to bot-bottle.json\n")
|
sys.stderr.write(" init interactively create a new agent and add it to bot-bottle.json\n")
|
||||||
sys.stderr.write(" list list available agents or active containers\n")
|
sys.stderr.write(" list list available agents or active containers\n")
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
"""`firecracker` CLI command — one-time network setup helper.
|
||||||
|
|
||||||
|
`./cli.py firecracker setup` prints the host-appropriate config for the
|
||||||
|
privileged, one-time network pool (TAP devices + isolation nftables
|
||||||
|
table) the Firecracker backend needs. On NixOS it prints a declarative
|
||||||
|
module to paste into your config (imperative rules don't survive
|
||||||
|
nixos-rebuild); elsewhere it prints the sudo command for the bundled
|
||||||
|
setup script. `./cli.py firecracker status` reports what's present.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ..backend.firecracker import netpool
|
||||||
|
|
||||||
|
|
||||||
|
def _is_nixos() -> bool:
|
||||||
|
if Path("/etc/NIXOS").exists():
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
return "ID=nixos" in Path("/etc/os-release").read_text()
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _cmd_setup() -> int:
|
||||||
|
slots = netpool.all_slots()
|
||||||
|
sys.stderr.write(
|
||||||
|
f"Firecracker network pool: {len(slots)} slots "
|
||||||
|
f"({slots[0].iface}..{slots[-1].iface}), base {netpool.ip_base()}.\n"
|
||||||
|
f"This is a one-time privileged setup (needs root once).\n\n"
|
||||||
|
)
|
||||||
|
if _is_nixos():
|
||||||
|
sys.stderr.write(
|
||||||
|
"Detected NixOS. Paste this module into your configuration and "
|
||||||
|
"`nixos-rebuild switch` (imperative rules would not survive a "
|
||||||
|
"rebuild):\n\n"
|
||||||
|
)
|
||||||
|
sys.stdout.write(netpool.render_nixos_module())
|
||||||
|
else:
|
||||||
|
sys.stderr.write("Run the one-time setup as root:\n\n")
|
||||||
|
sys.stdout.write(netpool.render_shell_setup() + "\n")
|
||||||
|
sys.stderr.write(
|
||||||
|
"\n(On NixOS, use the declarative module instead — this host "
|
||||||
|
"was not detected as NixOS.)\n"
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _cmd_status() -> int:
|
||||||
|
ok = True
|
||||||
|
if netpool.nft_table_present():
|
||||||
|
sys.stderr.write(f"nft table inet {netpool.NFT_TABLE}: present\n")
|
||||||
|
else:
|
||||||
|
sys.stderr.write(f"nft table inet {netpool.NFT_TABLE}: MISSING\n")
|
||||||
|
ok = False
|
||||||
|
missing = netpool.missing_taps()
|
||||||
|
total = netpool.pool_size()
|
||||||
|
if missing:
|
||||||
|
sys.stderr.write(f"TAP pool: {total - len(missing)}/{total} present "
|
||||||
|
f"(missing: {', '.join(missing)})\n")
|
||||||
|
ok = False
|
||||||
|
else:
|
||||||
|
sys.stderr.write(f"TAP pool: {total}/{total} present\n")
|
||||||
|
if not ok:
|
||||||
|
sys.stderr.write("\nRun: ./cli.py firecracker setup\n")
|
||||||
|
return 0 if ok else 1
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_firecracker(args: list[str]) -> int:
|
||||||
|
sub = args[0] if args else ""
|
||||||
|
if sub == "setup":
|
||||||
|
return _cmd_setup()
|
||||||
|
if sub == "status":
|
||||||
|
return _cmd_status()
|
||||||
|
sys.stderr.write("usage: firecracker {setup|status}\n")
|
||||||
|
return 2
|
||||||
Executable
+159
@@ -0,0 +1,159 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# One-time privileged network setup for the Firecracker backend.
|
||||||
|
#
|
||||||
|
# Creates a pool of point-to-point TAP devices (owned by the invoking
|
||||||
|
# user so the backend can open them without root at launch) and a
|
||||||
|
# dedicated nftables table that isolates every VM: a bottle VM can
|
||||||
|
# reach only its own sidecar (published on the host-side TAP IP) and
|
||||||
|
# nothing else on the host or network.
|
||||||
|
#
|
||||||
|
# Why a pool + one-time setup: creating a TAP and assigning it an IP
|
||||||
|
# needs CAP_NET_ADMIN. Pre-creating user-owned, pre-addressed TAPs
|
||||||
|
# means `./cli.py start` never needs root. The nft table is static
|
||||||
|
# (keyed on the `bbfc*` interface wildcard), so it covers every slot
|
||||||
|
# without per-launch changes.
|
||||||
|
#
|
||||||
|
# Design notes:
|
||||||
|
# * No shared bridge — each slot is an isolated /31 host<->guest link,
|
||||||
|
# so there are no bridge name/subnet collisions with docker0,
|
||||||
|
# virbr0 (libvirt), cni0 (k8s) or br-* (docker networks).
|
||||||
|
# * Own nftables table `bot_bottle_fc` — independent of the iptables
|
||||||
|
# filter/nat tables Docker/ufw/firewalld use, so nothing is stomped.
|
||||||
|
# * Default IP block is RFC-6598 shared address space (100.64.0.0/10),
|
||||||
|
# purpose-built to not collide with LAN/VPN private ranges.
|
||||||
|
#
|
||||||
|
# NixOS: imperative rules here do NOT survive nixos-rebuild. Use the
|
||||||
|
# declarative module in docs/firecracker/nixos-netpool.nix instead.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# sudo ./scripts/firecracker-netpool.sh up
|
||||||
|
# sudo ./scripts/firecracker-netpool.sh down
|
||||||
|
# ./scripts/firecracker-netpool.sh status
|
||||||
|
#
|
||||||
|
# Env overrides (must match the backend's util.py constants):
|
||||||
|
# BOT_BOTTLE_FC_POOL_SIZE number of slots (default 8)
|
||||||
|
# BOT_BOTTLE_FC_IP_BASE base IPv4 of the /31 pool (default 100.64.0.0)
|
||||||
|
# BOT_BOTTLE_FC_IFACE_PREFIX TAP name prefix (default bbfc)
|
||||||
|
# BOT_BOTTLE_FC_OWNER owning user (default $SUDO_USER or $USER)
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
POOL_SIZE="${BOT_BOTTLE_FC_POOL_SIZE:-8}"
|
||||||
|
IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-100.64.0.0}"
|
||||||
|
PREFIX="${BOT_BOTTLE_FC_IFACE_PREFIX:-bbfc}"
|
||||||
|
OWNER="${BOT_BOTTLE_FC_OWNER:-${SUDO_USER:-$USER}}"
|
||||||
|
TABLE="bot_bottle_fc"
|
||||||
|
|
||||||
|
# Sidecar ports (must match the backend). egress=9099, supervise=9100,
|
||||||
|
# git-http=9420. Reached by the VM at its host-side TAP IP.
|
||||||
|
SIDECAR_PORTS="9099,9100,9420"
|
||||||
|
|
||||||
|
# --- IP math ---------------------------------------------------------
|
||||||
|
# Slot i occupies the /31 {base+2i, base+2i+1}: host = base+2i (the
|
||||||
|
# gateway the VM routes through), guest = base+2i+1 (the VM's address).
|
||||||
|
_ip_to_int() {
|
||||||
|
local IFS=. ; read -r a b c d <<<"$1" ; echo $(( (a<<24) + (b<<16) + (c<<8) + d ))
|
||||||
|
}
|
||||||
|
_int_to_ip() {
|
||||||
|
local n=$1 ; echo "$(( (n>>24)&255 )).$(( (n>>16)&255 )).$(( (n>>8)&255 )).$(( n&255 ))"
|
||||||
|
}
|
||||||
|
host_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 )); }
|
||||||
|
guest_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 + 1 )); }
|
||||||
|
iface() { echo "${PREFIX}$1"; }
|
||||||
|
|
||||||
|
require_root() {
|
||||||
|
if [ "$(id -u)" -ne 0 ]; then
|
||||||
|
echo "error: '$1' needs root (run under sudo)" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_up() {
|
||||||
|
require_root up
|
||||||
|
echo "firecracker net pool: $POOL_SIZE slots, base $IP_BASE, owner $OWNER"
|
||||||
|
|
||||||
|
# VM->sidecar traffic is DNAT'd to the sidecar container and
|
||||||
|
# forwarded, so forwarding must be enabled (Docker also sets this).
|
||||||
|
sysctl -qw net.ipv4.ip_forward=1
|
||||||
|
|
||||||
|
for i in $(seq 0 $((POOL_SIZE-1))); do
|
||||||
|
local dev host
|
||||||
|
dev="$(iface "$i")" ; host="$(host_ip "$i")"
|
||||||
|
if ip link show "$dev" >/dev/null 2>&1; then
|
||||||
|
ip link set "$dev" down 2>/dev/null || true
|
||||||
|
ip tuntap del dev "$dev" mode tap 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
ip tuntap add dev "$dev" mode tap user "$OWNER"
|
||||||
|
ip addr add "$host/31" dev "$dev"
|
||||||
|
ip link set "$dev" up
|
||||||
|
echo " $dev host=$host guest=$(guest_ip "$i") owner=$OWNER"
|
||||||
|
done
|
||||||
|
|
||||||
|
_install_nft
|
||||||
|
echo "nftables table inet $TABLE installed (fail-closed boundary)"
|
||||||
|
echo "done."
|
||||||
|
}
|
||||||
|
|
||||||
|
_install_nft() {
|
||||||
|
# Own table: dropping only matches our bbfc* interfaces, so no other
|
||||||
|
# tool's traffic is affected. Priority -10 runs before Docker's
|
||||||
|
# filter hooks (priority 0); a drop here is terminal for the packet.
|
||||||
|
#
|
||||||
|
# forward: VM egress is DNAT'd to the sidecar (established via
|
||||||
|
# `ct status dnat`); return traffic via `ct state established`.
|
||||||
|
# Anything else from a VM is dropped -> no route to the internet
|
||||||
|
# or the rest of the host except through the sidecar proxy.
|
||||||
|
# input: a VM never needs host-local delivery (its sidecar is
|
||||||
|
# reached via DNAT->forward), so drop all direct input from VMs
|
||||||
|
# -> host services bound on 0.0.0.0 are unreachable from the VM.
|
||||||
|
nft -f - <<EOF
|
||||||
|
table inet $TABLE {
|
||||||
|
chain forward {
|
||||||
|
type filter hook forward priority -10; policy accept;
|
||||||
|
iifname != "${PREFIX}*" return
|
||||||
|
ct state established,related accept
|
||||||
|
ct status dnat accept
|
||||||
|
drop
|
||||||
|
}
|
||||||
|
chain input {
|
||||||
|
type filter hook input priority -10; policy accept;
|
||||||
|
iifname != "${PREFIX}*" return
|
||||||
|
ct state established,related accept
|
||||||
|
drop
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_down() {
|
||||||
|
require_root down
|
||||||
|
nft delete table inet "$TABLE" 2>/dev/null || true
|
||||||
|
for i in $(seq 0 $((POOL_SIZE-1))); do
|
||||||
|
local dev ; dev="$(iface "$i")"
|
||||||
|
if ip link show "$dev" >/dev/null 2>&1; then
|
||||||
|
ip link set "$dev" down 2>/dev/null || true
|
||||||
|
ip tuntap del dev "$dev" mode tap 2>/dev/null || true
|
||||||
|
echo " removed $dev"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo "done."
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd_status() {
|
||||||
|
echo "table inet $TABLE:"
|
||||||
|
nft list table inet "$TABLE" 2>/dev/null || echo " (absent)"
|
||||||
|
echo "taps:"
|
||||||
|
for i in $(seq 0 $((POOL_SIZE-1))); do
|
||||||
|
local dev ; dev="$(iface "$i")"
|
||||||
|
if ip -brief addr show "$dev" >/dev/null 2>&1; then
|
||||||
|
ip -brief addr show "$dev" | sed 's/^/ /'
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
case "${1:-}" in
|
||||||
|
up) cmd_up ;;
|
||||||
|
down) cmd_down ;;
|
||||||
|
status) cmd_status ;;
|
||||||
|
*) echo "usage: $0 {up|down|status}" >&2 ; exit 2 ;;
|
||||||
|
esac
|
||||||
@@ -64,6 +64,24 @@ class TestGetBottleBackend(unittest.TestCase):
|
|||||||
b = get_bottle_backend()
|
b = get_bottle_backend()
|
||||||
self.assertEqual("smolmachines", b.name)
|
self.assertEqual("smolmachines", b.name)
|
||||||
|
|
||||||
|
def test_default_firecracker_when_macos_unavailable_but_fc_available(self):
|
||||||
|
class _FakeBackend:
|
||||||
|
def __init__(self, name: str, available: bool) -> None:
|
||||||
|
self.name = name
|
||||||
|
self._available = available
|
||||||
|
|
||||||
|
def is_available(self) -> bool:
|
||||||
|
return self._available
|
||||||
|
|
||||||
|
with patch.dict(os.environ, {}, clear=True), \
|
||||||
|
patch.object(backend_mod, "_BACKENDS", {
|
||||||
|
"macos-container": _FakeBackend("macos-container", False),
|
||||||
|
"firecracker": _FakeBackend("firecracker", True),
|
||||||
|
"smolmachines": _FakeBackend("smolmachines", True),
|
||||||
|
}):
|
||||||
|
b = get_bottle_backend()
|
||||||
|
self.assertEqual("firecracker", b.name)
|
||||||
|
|
||||||
def test_unknown_dies(self):
|
def test_unknown_dies(self):
|
||||||
with patch.object(backend_mod, "die", side_effect=SystemExit("die")):
|
with patch.object(backend_mod, "die", side_effect=SystemExit("die")):
|
||||||
with self.assertRaises(SystemExit):
|
with self.assertRaises(SystemExit):
|
||||||
@@ -73,7 +91,7 @@ class TestGetBottleBackend(unittest.TestCase):
|
|||||||
class TestKnownBackendNames(unittest.TestCase):
|
class TestKnownBackendNames(unittest.TestCase):
|
||||||
def test_returns_backends_sorted(self):
|
def test_returns_backends_sorted(self):
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
("docker", "macos-container", "smolmachines"),
|
("docker", "firecracker", "macos-container", "smolmachines"),
|
||||||
known_backend_names(),
|
known_backend_names(),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,187 @@
|
|||||||
|
"""Unit: Firecracker backend helpers.
|
||||||
|
|
||||||
|
Covers the pieces that run without KVM/root: pool IP math + config
|
||||||
|
renderers + allocation, the SSH-backed bottle's argv construction, and
|
||||||
|
the VM boot-arg assembly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import os
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, cast
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from bot_bottle.backend.firecracker import firecracker_vm, netpool
|
||||||
|
from bot_bottle.backend.firecracker.bottle import FirecrackerBottle
|
||||||
|
|
||||||
|
|
||||||
|
def _bottle(**kw: Any) -> FirecrackerBottle:
|
||||||
|
defaults = dict(
|
||||||
|
name="bot-bottle-dev-abc",
|
||||||
|
private_key=Path("/tmp/key"),
|
||||||
|
guest_ip="100.64.0.1",
|
||||||
|
)
|
||||||
|
defaults.update(kw)
|
||||||
|
return FirecrackerBottle(**defaults) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
class TestNetpoolSlots(unittest.TestCase):
|
||||||
|
def test_slot_ip_math_31_pairs(self):
|
||||||
|
with patch.dict(os.environ, {"BOT_BOTTLE_FC_IP_BASE": "100.64.0.0"}):
|
||||||
|
s0, s1 = netpool.slot(0), netpool.slot(1)
|
||||||
|
self.assertEqual(("bbfc0", "100.64.0.0", "100.64.0.1"),
|
||||||
|
(s0.iface, s0.host_ip, s0.guest_ip))
|
||||||
|
self.assertEqual(("bbfc1", "100.64.0.2", "100.64.0.3"),
|
||||||
|
(s1.iface, s1.host_ip, s1.guest_ip))
|
||||||
|
|
||||||
|
def test_guest_cidr_is_31(self):
|
||||||
|
self.assertEqual("100.64.0.1/31", netpool.slot(0).guest_cidr)
|
||||||
|
|
||||||
|
def test_pool_size_env_override(self):
|
||||||
|
with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "4"}):
|
||||||
|
self.assertEqual(4, netpool.pool_size())
|
||||||
|
self.assertEqual(4, len(netpool.all_slots()))
|
||||||
|
|
||||||
|
|
||||||
|
class TestNetpoolRenderers(unittest.TestCase):
|
||||||
|
def test_nixos_module_has_table_and_taps(self):
|
||||||
|
with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "2"}):
|
||||||
|
out = netpool.render_nixos_module()
|
||||||
|
self.assertIn('networking.nftables.tables."bot_bottle_fc"', out)
|
||||||
|
self.assertIn('Name = "bbfc0"', out)
|
||||||
|
self.assertIn('Name = "bbfc1"', out)
|
||||||
|
self.assertIn("net.ipv4.ip_forward", out)
|
||||||
|
# fail-closed isolation rules present
|
||||||
|
self.assertIn('iifname != "bbfc*" return', out)
|
||||||
|
self.assertIn("ct status dnat accept", out)
|
||||||
|
|
||||||
|
def test_shell_setup_reflects_overrides(self):
|
||||||
|
with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "3"}):
|
||||||
|
out = netpool.render_shell_setup()
|
||||||
|
self.assertIn("BOT_BOTTLE_FC_POOL_SIZE=3", out)
|
||||||
|
self.assertIn("firecracker-netpool.sh up", out)
|
||||||
|
|
||||||
|
|
||||||
|
class TestNetpoolAllocation(unittest.TestCase):
|
||||||
|
def test_allocate_is_mutually_exclusive(self):
|
||||||
|
with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "1"}):
|
||||||
|
# First allocation grabs the only slot; the lock is held
|
||||||
|
# until the handle is closed, so a second call must fail
|
||||||
|
# over (and here, exhaust the pool).
|
||||||
|
slot, lock = netpool.allocate("first")
|
||||||
|
self.addCleanup(lock.close)
|
||||||
|
self.assertEqual("bbfc0", slot.iface)
|
||||||
|
with patch.object(netpool, "die",
|
||||||
|
side_effect=SystemExit("exhausted")):
|
||||||
|
with self.assertRaises(SystemExit):
|
||||||
|
netpool.allocate("second")
|
||||||
|
|
||||||
|
def test_allocate_releases_on_close(self):
|
||||||
|
with patch.dict(os.environ, {"BOT_BOTTLE_FC_POOL_SIZE": "1"}):
|
||||||
|
slot, lock = netpool.allocate("first")
|
||||||
|
lock.close()
|
||||||
|
# Slot is free again after close.
|
||||||
|
slot2, lock2 = netpool.allocate("second")
|
||||||
|
self.addCleanup(lock2.close)
|
||||||
|
self.assertEqual(slot.iface, slot2.iface)
|
||||||
|
|
||||||
|
|
||||||
|
class TestBottleAgentArgv(unittest.TestCase):
|
||||||
|
def test_interactive_uses_ssh_tty_and_runuser(self):
|
||||||
|
argv = _bottle().agent_argv([], tty=True)
|
||||||
|
self.assertEqual("ssh", argv[0])
|
||||||
|
self.assertIn("-t", argv)
|
||||||
|
self.assertIn("100.64.0.1", " ".join(argv))
|
||||||
|
idx = argv.index("--")
|
||||||
|
self.assertEqual(["runuser", "-u", "node", "--"], argv[idx + 1:idx + 5])
|
||||||
|
self.assertIn("claude", argv)
|
||||||
|
|
||||||
|
def test_non_interactive_has_no_tty(self):
|
||||||
|
argv = _bottle().agent_argv([], tty=False)
|
||||||
|
self.assertEqual("ssh", argv[0])
|
||||||
|
self.assertNotIn("-t", argv)
|
||||||
|
|
||||||
|
def test_appends_extra_args_after_command(self):
|
||||||
|
argv = _bottle().agent_argv(
|
||||||
|
["--dangerously-skip-permissions", "--continue"], tty=False,
|
||||||
|
)
|
||||||
|
idx = argv.index("claude")
|
||||||
|
self.assertEqual(
|
||||||
|
["claude", "--dangerously-skip-permissions", "--continue"],
|
||||||
|
argv[idx:],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_prompt_file_flag_injected(self):
|
||||||
|
argv = _bottle(
|
||||||
|
prompt_path_in_guest="/home/node/.bot-bottle-prompt.txt",
|
||||||
|
).agent_argv(["--continue"], tty=False)
|
||||||
|
idx = argv.index("claude")
|
||||||
|
self.assertEqual(
|
||||||
|
["claude", "--continue",
|
||||||
|
"--append-system-prompt-file", "/home/node/.bot-bottle-prompt.txt"],
|
||||||
|
argv[idx:],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_workdir_wraps_command(self):
|
||||||
|
argv = _bottle(agent_workdir="/home/node/workspace").agent_argv([], tty=False)
|
||||||
|
self.assertIn("sh", argv)
|
||||||
|
joined = " ".join(argv)
|
||||||
|
self.assertIn("cd /home/node/workspace", joined)
|
||||||
|
|
||||||
|
def test_guest_env_injected(self):
|
||||||
|
argv = _bottle(guest_env={"HTTPS_PROXY": "http://100.64.0.0:9099"}).agent_argv(
|
||||||
|
[], tty=False,
|
||||||
|
)
|
||||||
|
self.assertIn("HTTPS_PROXY=http://100.64.0.0:9099", argv)
|
||||||
|
self.assertIn("HOME=/home/node", argv)
|
||||||
|
self.assertIn("USER=node", argv)
|
||||||
|
|
||||||
|
|
||||||
|
class TestSetupScriptConsistency(unittest.TestCase):
|
||||||
|
"""The shell setup script duplicates the pool defaults; keep them in
|
||||||
|
lockstep with the Python constants so the two setup paths agree."""
|
||||||
|
|
||||||
|
def _script(self) -> str:
|
||||||
|
root = Path(__file__).resolve().parent.parent.parent
|
||||||
|
return (root / "scripts" / "firecracker-netpool.sh").read_text()
|
||||||
|
|
||||||
|
def test_defaults_match_python(self):
|
||||||
|
script = self._script()
|
||||||
|
with patch.dict(os.environ, {}, clear=True):
|
||||||
|
self.assertIn(f'POOL_SIZE:-{netpool.pool_size()}', script)
|
||||||
|
self.assertIn(f'BOT_BOTTLE_FC_IP_BASE:-{netpool.ip_base()}', script)
|
||||||
|
self.assertIn(f'IFACE_PREFIX:-{netpool.IFACE_PREFIX}', script)
|
||||||
|
|
||||||
|
def test_table_name_and_ports_match(self):
|
||||||
|
script = self._script()
|
||||||
|
self.assertIn(f'TABLE="{netpool.NFT_TABLE}"', script)
|
||||||
|
for port in netpool.SIDECAR_PORTS:
|
||||||
|
self.assertIn(str(port), script)
|
||||||
|
|
||||||
|
|
||||||
|
class TestBootArgs(unittest.TestCase):
|
||||||
|
def test_boot_args_carry_ip_and_pubkey(self):
|
||||||
|
args = firecracker_vm._boot_args(
|
||||||
|
guest_ip="100.64.0.1", host_ip="100.64.0.0", pubkey="ssh-ed25519 AAAA x",
|
||||||
|
)
|
||||||
|
self.assertIn("ip=100.64.0.1::100.64.0.0:255.255.255.254::eth0:off", args)
|
||||||
|
self.assertIn("init=/bb-init", args)
|
||||||
|
b64 = base64.b64encode(b"ssh-ed25519 AAAA x").decode()
|
||||||
|
self.assertIn(f"bb_pubkey={b64}", args)
|
||||||
|
|
||||||
|
def test_config_has_rootfs_and_tap(self):
|
||||||
|
cfg = cast(Any, firecracker_vm._config(
|
||||||
|
rootfs=Path("/run/rootfs.ext4"), tap="bbfc0",
|
||||||
|
guest_ip="100.64.0.1", host_ip="100.64.0.0", pubkey="k",
|
||||||
|
vcpus=2, mem_mib=2048, guest_mac="06:00:AC:10:00:02",
|
||||||
|
))
|
||||||
|
self.assertEqual("/run/rootfs.ext4", cfg["drives"][0]["path_on_host"])
|
||||||
|
self.assertFalse(cfg["drives"][0]["is_read_only"])
|
||||||
|
self.assertEqual("bbfc0", cfg["network-interfaces"][0]["host_dev_name"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user