bot_bottle/backend/firecracker/freezer.py

@@ -1,9 +1,12 @@
-"""FirecrackerFreezer — snapshot a running microVM to a Docker image.
+"""FirecrackerFreezer — snapshot a running microVM to a rootfs tar.
 
 The VM is live and can't be block-copied safely, so — like the macOS
 backend — we stream the guest root filesystem out over the control
-channel (SSH here) and rebuild an image from it. The bottle keeps
+channel (SSH here). Unlike the other backends this needs no Docker: the
-running after the snapshot.
+tar *is* the resumable artifact. `resume` extracts it and rebuilds a
+fresh per-bottle ext4 with `mke2fs -d` (see `util.build_committed_rootfs_dir`
+and `launch._build_agent_base`). The bottle keeps running after the
+snapshot.
 """
 
 from __future__ import annotations
@@ -11,9 +14,9 @@ from __future__ import annotations
 import json
 import os
 import subprocess
-import tempfile
 from pathlib import Path
 
+from ...bottle_state import committed_rootfs_path
 from ...log import die, info
 from .. import ActiveAgent
 from ..freeze import Freezer
@@ -30,14 +33,13 @@ class FirecrackerFreezer(Freezer):
         if not private_key.is_file() or not guest_ip:
             die(f"cannot freeze {agent.slug}: run dir {run_dir} is missing the "
                 f"SSH key or VM config (is the bottle still running?)")
-        image_tag = f"bot-bottle-committed-{agent.slug}:latest"
+        tar_path = committed_rootfs_path(agent.slug)
-        _commit_via_ssh(private_key, guest_ip, image_tag)
+        _commit_rootfs_via_ssh(private_key, guest_ip, tar_path)
-        info(f"committed {agent.slug} -> {image_tag!r}")
+        info(f"committed {agent.slug} -> {tar_path}")
-        return image_tag
+        return str(tar_path)
 
     def _export_hint(self, slug: str, image_ref: str) -> None:
-        info(f"to export for migration:      docker image save {image_ref} "
+        info(f"to export for migration:      cp {image_ref} {slug}.tar")
-             f"-o {slug}.tar")
 
 
 def _guest_ip_from_config(config_path: Path) -> str:
@@ -53,24 +55,36 @@ def _guest_ip_from_config(config_path: Path) -> str:
     return ""
 
 
-def _commit_via_ssh(private_key: Path, guest_ip: str, image_tag: str) -> None:
+def _commit_rootfs_via_ssh(private_key: Path, guest_ip: str, tar_path: Path) -> None:
-    with tempfile.TemporaryDirectory(prefix="bot-bottle-fc-commit.") as tmp:
+    """Stream the guest rootfs out over SSH into `tar_path`. Excludes the
-        rootfs_tar = os.path.join(tmp, "rootfs.tar")
+    virtual/live mounts (proc/sys/dev/run) — resume recreates those empty
-        ssh = util.ssh_base_argv(private_key, guest_ip)
+    mount points. Written to a `.partial` sibling and renamed on success so
-        with open(rootfs_tar, "wb") as tar_out:
+    a failed freeze never leaves a truncated artifact in its place."""
-            result = subprocess.run(
+    tar_path.parent.mkdir(parents=True, exist_ok=True)
-                [*ssh, "--", "tar", "--create", "--one-file-system",
+    partial = tar_path.with_name(tar_path.name + ".partial")
-                 "--exclude=./proc", "--exclude=./sys", "--exclude=./dev",
+    ssh = util.ssh_base_argv(private_key, guest_ip)
-                 "--exclude=./run", "--file=-", "--directory=/", "."],
+    # The snapshot can contain the bottle's private workspace, so keep it
-                stdout=tar_out, stderr=subprocess.PIPE, check=False,
+    # owner-only (0600) for the whole stream. The `os.open` mode only applies
-            )
+    # on *creation*, so unlink any leftover partial (a prior interrupted run
-        if result.returncode != 0:
+    # could have left it world-readable, or something could swap in a symlink
-            die(f"ssh tar for {guest_ip} failed: "
+    # at this predictable name) and exclusively recreate it — O_EXCL|O_NOFOLLOW
-                f"{(result.stderr or b'').decode().strip() or '<no stderr>'}")
+    # — then fchmod immediately so umask can't loosen it. Re-assert after the
-        with open(os.path.join(tmp, "Dockerfile"), "w", encoding="utf-8") as f:
+    # rename too (os.replace carries the source mode, but be explicit).
-            f.write("FROM scratch\nADD rootfs.tar /\nUSER node\nWORKDIR /home/node\n")
+    partial.unlink(missing_ok=True)
-        build = subprocess.run(
+    fd = os.open(
-            ["docker", "build", "-t", image_tag, tmp], check=False,
+        partial, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600
+    )
+    os.fchmod(fd, 0o600)
+    with os.fdopen(fd, "wb") as tar_out:
+        result = subprocess.run(
+            [*ssh, "--", "tar", "--create", "--one-file-system",
+             "--exclude=./proc", "--exclude=./sys", "--exclude=./dev",
+             "--exclude=./run", "--file=-", "--directory=/", "."],
+            stdout=tar_out, stderr=subprocess.PIPE, check=False,
         )
-        if build.returncode != 0:
+    if result.returncode != 0:
-            die(f"docker build for {image_tag!r} failed")
+        partial.unlink(missing_ok=True)
+        die(f"ssh tar for {guest_ip} failed: "
+            f"{(result.stderr or b'').decode().strip() or '<no stderr>'}")
+    os.replace(partial, tar_path)
+    os.chmod(tar_path, 0o600)

bot_bottle/backend/firecracker/launch.py

@@ -1,7 +1,8 @@
 """Launch flow for the Firecracker backend (PRD 0070, consolidated).
 
 Per bottle:
-  1. build the agent image (docker), export it to a cached ext4 rootfs;
+  1. build the agent rootfs in a builder VM (buildah, no host docker), or
+     resume a frozen bottle from its committed rootfs tar; cache the ext4;
   2. ensure the per-host orchestrator + shared gateway are up;
   3. claim a free TAP pool slot (rootless flock);
   4. register the bottle on the orchestrator by the VM's guest IP (the
@@ -31,6 +32,7 @@ from typing import Callable, Generator
 
 from ...agent_provider import runtime_for
 from ...bottle_state import (
+    committed_rootfs_path,
     egress_state_dir,
     git_gate_state_dir,
     read_committed_image,
@@ -45,7 +47,6 @@ from ...git_gate import (
 )
 from ...log import info, warn
 from ...supervise import SUPERVISE_PORT
-from ..docker import util as docker_mod
 from ..docker.egress import EGRESS_PORT
 from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
 from . import firecracker_vm, image_builder, isolation_probe, netpool, util
@@ -210,16 +211,13 @@ def _build_agent_base(
 ) -> tuple[FirecrackerBottlePlan, Path]:
     """Produce the agent's base rootfs dir. Primary path: build the Dockerfile
     inside a Firecracker builder VM (buildah, no host docker), smoke-testing
-    the image before export. A committed snapshot (freeze/migrate) is still
+    the image before export. A committed snapshot (freeze/migrate) is resumed
-    exported via the host docker path until that is ported too."""
+    directly from the rootfs tar the freezer wrote — no host docker either."""
     committed = read_committed_image(plan.slug)
-    if committed and docker_mod.image_exists(committed):
+    committed_tar = committed_rootfs_path(plan.slug)
-        info(f"using committed image {committed!r}")
+    if committed and committed_tar.is_file():
-        plan = dataclasses.replace(
+        info(f"resuming from committed rootfs {committed_tar}")
-            plan,
+        return plan, util.build_committed_rootfs_dir(committed_tar)
-            agent_provision=dataclasses.replace(plan.agent_provision, image=committed),
-        )
-        return plan, util.build_base_rootfs_dir(committed)
     base = image_builder.build_agent_rootfs_dir(
         Path(plan.dockerfile_path),
         image_tag=plan.image,

bot_bottle/backend/firecracker/util.py

@@ -14,6 +14,7 @@ and `./cli.py backend setup --backend=firecracker`.
 
 from __future__ import annotations
 
+import hashlib
 import os
 import platform
 import shutil
@@ -212,15 +213,80 @@ def build_base_rootfs_dir(
     return base
 
 
+def build_committed_rootfs_dir(tar_path: Path) -> Path:
+    """Prepare a base rootfs dir from a frozen-bottle snapshot tar (the
+    freeze/resume path — no Docker). Extracts the snapshot, recreates the
+    virtual mount points the freezer excluded, and injects the guest init +
+    static dropbear, mirroring `build_base_rootfs_dir` but sourced from a tar
+    we control rather than a Docker image.
+
+    Cached under the rootfs cache, keyed by the tar's size+mtime so a
+    re-freeze re-extracts but repeated resumes of the same snapshot don't.
+    Returns the prepared directory (read as the `mke2fs -d` source)."""
+    st = tar_path.stat()
+    fingerprint = hashlib.sha256(
+        f"{tar_path}:{st.st_size}:{st.st_mtime_ns}".encode()
+    ).hexdigest()[:16]
+    base = cache_dir() / "rootfs" / f"committed-{fingerprint}"
+    ready = base / ".bb-ready"
+    if ready.is_file():
+        return base
+
+    if base.exists():
+        shutil.rmtree(base, ignore_errors=True)
+    base.mkdir(parents=True)
+
+    info(f"extracting committed rootfs {tar_path} -> {base}")
+    result = subprocess.run(
+        ["tar", "-x", "-f", str(tar_path), "-C", str(base)],
+        capture_output=True, text=True, check=False,
+    )
+    if result.returncode != 0:
+        die(f"extracting committed rootfs {tar_path} failed: "
+            f"{result.stderr.strip() or '<no stderr>'}")
+
+    # The freezer excludes the live/virtual filesystems from the snapshot;
+    # recreate them as empty mount points so the guest init can mount
+    # proc/sys/dev and dropbear has a writable /run.
+    for mount_point in ("proc", "sys", "dev", "run"):
+        (base / mount_point).mkdir(mode=0o755, exist_ok=True)
+
+    inject_guest_boot(base)
+    ready.write_text("ok\n")
+    return base
+
+
 def inject_guest_boot(rootfs: Path, init_script: str | None = None) -> None:
     """Drop the static dropbear and the PID-1 init into the rootfs.
     `init_script` defaults to the SSH-only agent init; the infra VM
-    passes its own (control plane + gateway) init."""
+    passes its own (control plane + gateway) init.
-    shutil.copy2(dropbear_path(), rootfs / "bb-dropbear")
+
-    os.chmod(rootfs / "bb-dropbear", 0o755)
+    A committed snapshot is guest-controlled, so `bb-dropbear`/`bb-init`
-    init = rootfs / "bb-init"
+    may already exist as symlinks aimed at a host file (e.g. bb-init ->
-    init.write_text(init_script or _GUEST_INIT)
+    ~/.bashrc). Replace whatever is there and create the files with
-    os.chmod(init, 0o755)
+    O_EXCL|O_NOFOLLOW so the write always lands a fresh regular file in
+    the staging tree and never follows a planted symlink out of it."""
+    _write_staged_file(rootfs / "bb-dropbear", dropbear_path().read_bytes())
+    _write_staged_file(rootfs / "bb-init", (init_script or _GUEST_INIT).encode())
+
+
+def _write_staged_file(path: Path, data: bytes) -> None:
+    """Write `data` to `path` (mode 0755) as a fresh regular file inside a
+    staging rootfs, replacing any pre-existing entry without following a
+    symlink at `path`. Fails closed on anything unexpected there."""
+    if path.is_symlink() or path.exists():
+        if path.is_dir() and not path.is_symlink():
+            shutil.rmtree(path)
+        else:
+            path.unlink()
+    fd = os.open(
+        path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o755
+    )
+    try:
+        os.write(fd, data)
+    finally:
+        os.close(fd)
+    os.chmod(path, 0o755)
 
 
 def build_rootfs_ext4(base_dir: Path, out_path: Path, *, slack_mib: int = 1024) -> None:

bot_bottle/bottle_state.py

@@ -44,6 +44,7 @@ from .paths import bot_bottle_root
 _STATE_SUBDIR = "state"
 _PER_BOTTLE_DOCKERFILE_NAME = "Dockerfile"
 _COMMITTED_IMAGE_NAME = "committed-image"
+_COMMITTED_ROOTFS_NAME = "committed-rootfs.tar"
 _TRANSCRIPT_SUBDIR = "transcript"
 # Per-daemon scratch subdirs. PRD 0018 chunk 2: bind-mount sources
 # live here so chunk 3's `docker compose up` can find them at stable
@@ -200,6 +201,15 @@ def committed_image_path(identity: str) -> Path:
     return bottle_state_dir(identity) / _COMMITTED_IMAGE_NAME
 
 
+def committed_rootfs_path(identity: str) -> Path:
+    """Where the Firecracker freezer stores a snapshot of the bottle's
+    guest rootfs (a plain tar). This is the resumable/migratable artifact
+    the Firecracker backend boots from — no Docker image involved. The
+    matching `committed-image` state file records that a snapshot exists
+    (and its path); `resume` boots from this tar when both are present."""
+    return bottle_state_dir(identity) / _COMMITTED_ROOTFS_NAME
+
+
 def write_committed_image(identity: str, image_tag: str) -> Path:
     """Persist the committed image tag for `identity`. The next
     `cli.py resume <identity>` will boot from this image instead of
@@ -354,6 +364,7 @@ __all__ = [
     "cleanup_state",
     "clear_preserve_marker",
     "committed_image_path",
+    "committed_rootfs_path",
     "egress_state_dir",
     "git_gate_state_dir",
     "is_preserved",

tests/unit/test_backend_freezer.py

@@ -205,25 +205,29 @@ class TestFirecrackerFreezer(_FakeHomeMixin, unittest.TestCase):
         )
 
     def test_snapshots_running_vm_without_stopping(self):
-        """Commit should tar the running guest rootfs over SSH, not stop it."""
+        """Commit should tar the running guest rootfs over SSH into the
+        committed-rootfs artifact (no Docker), not stop the VM."""
         slug = "dev-abc12"
         self._write_meta(slug)
         self._stage_run_dir(slug)
         freezer = FirecrackerFreezer()
         agent = _make_agent(slug, "firecracker")
 
-        with patch("bot_bottle.backend.firecracker.freezer._commit_via_ssh") as mock_commit, \
+        commit_fn = "bot_bottle.backend.firecracker.freezer._commit_rootfs_via_ssh"
+        with patch(commit_fn) as mock_commit, \
              patch("bot_bottle.backend.freeze.info"), \
              patch("bot_bottle.backend.firecracker.freezer.info"):
             freezer.commit(agent)
 
-        image_tag = f"bot-bottle-committed-{slug}:latest"
+        tar_path = bottle_state.committed_rootfs_path(slug)
         self.assertEqual(1, mock_commit.call_count)
-        # (private_key, guest_ip, image_tag) — guest_ip parsed from config.
+        # (private_key, guest_ip, tar_path) — guest_ip parsed from config.
         args = mock_commit.call_args.args
         self.assertEqual("100.64.0.1", args[1])
-        self.assertEqual(image_tag, args[2])
+        self.assertEqual(tar_path, args[2])
-        self.assertEqual(image_tag, bottle_state.read_committed_image(slug))
+        # The committed-image state records the artifact path; resume boots
+        # from the tar rather than a Docker image.
+        self.assertEqual(str(tar_path), bottle_state.read_committed_image(slug))
         self.assertTrue(bottle_state.is_preserved(slug))
 
 

tests/unit/test_firecracker_helpers.py

@@ -6,10 +6,13 @@ branches. Mock subprocess/os so nothing needs KVM or a live VM.
 from __future__ import annotations
 
 import json
+import os
+import stat
 import subprocess
 import tempfile
 import unittest
 from pathlib import Path
+from typing import Any
 from unittest.mock import patch
 
 from bot_bottle.backend.firecracker import firecracker_vm, freezer, netpool, util
@@ -128,5 +131,172 @@ class TestRequireFirecracker(unittest.TestCase):
                 util.require_firecracker()
 
 
+class TestBuildCommittedRootfsDir(unittest.TestCase):
+    """Resume prepares the base rootfs dir from the freezer's snapshot tar
+    with no Docker: extract, recreate the excluded mount points, inject the
+    guest boot bits."""
+
+    def _make_tar(self, tmp: Path) -> Path:
+        import tarfile
+
+        src = tmp / "src"
+        (src / "home" / "node").mkdir(parents=True)
+        (src / "home" / "node" / "hello").write_text("hi")
+        tar_path = tmp / "rootfs.tar"
+        with tarfile.open(tar_path, "w") as tar:
+            tar.add(src, arcname=".")
+        return tar_path
+
+    def test_extracts_recreates_mountpoints_and_injects_boot(self):
+        with tempfile.TemporaryDirectory(prefix="fc-committed.") as d:
+            tmp = Path(d)
+            tar_path = self._make_tar(tmp)
+            # Stand in for the static dropbear that inject_guest_boot copies.
+            dropbear = tmp / "dropbear"
+            dropbear.write_text("#!/bin/true\n")
+            cache = tmp / "cache"
+            cache.mkdir()
+
+            with patch.object(util, "cache_dir", return_value=cache), \
+                 patch.object(util, "dropbear_path", return_value=dropbear), \
+                 patch.object(util, "info"):
+                base = util.build_committed_rootfs_dir(tar_path)
+
+            self.assertEqual("hi", (base / "home" / "node" / "hello").read_text())
+            for mount_point in ("proc", "sys", "dev", "run"):
+                self.assertTrue((base / mount_point).is_dir(),
+                                f"missing recreated mount point /{mount_point}")
+            self.assertTrue((base / "bb-dropbear").is_file())
+            self.assertTrue((base / "bb-init").is_file())
+            self.assertTrue((base / ".bb-ready").is_file())
+
+    def test_caches_on_repeat_and_reextracts_after_refreeze(self):
+        with tempfile.TemporaryDirectory(prefix="fc-committed.") as d:
+            tmp = Path(d)
+            tar_path = self._make_tar(tmp)
+            dropbear = tmp / "dropbear"
+            dropbear.write_text("#!/bin/true\n")
+            cache = tmp / "cache"
+            cache.mkdir()
+
+            ctx = [
+                patch.object(util, "cache_dir", return_value=cache),
+                patch.object(util, "dropbear_path", return_value=dropbear),
+                patch.object(util, "info"),
+            ]
+            for c in ctx:
+                c.start()
+            self.addCleanup(lambda: [c.stop() for c in ctx])
+
+            real_run = subprocess.run
+            calls = {"n": 0}
+
+            def counting_run(argv: list[str], *a: Any, **k: Any) -> Any:
+                if argv and argv[0] == "tar":
+                    calls["n"] += 1
+                return real_run(argv, *a, **k)
+
+            with patch.object(util.subprocess, "run", side_effect=counting_run):
+                first = util.build_committed_rootfs_dir(tar_path)
+                second = util.build_committed_rootfs_dir(tar_path)
+                self.assertEqual(first, second)
+                self.assertEqual(1, calls["n"])  # cached — no re-extract
+
+                # A re-freeze rewrites the tar; a new size/mtime -> new cache
+                # key -> re-extract. Force a distinct mtime so the test isn't
+                # at the mercy of filesystem timestamp granularity.
+                import tarfile
+                extra = tmp / "extra"
+                extra.mkdir()
+                (extra / "note").write_text("v2")
+                with tarfile.open(tar_path, "w") as tar:
+                    tar.add(extra, arcname=".")
+                st = tar_path.stat()
+                os.utime(tar_path, ns=(st.st_atime_ns, st.st_mtime_ns + 1_000_000_000))
+
+                third = util.build_committed_rootfs_dir(tar_path)
+                self.assertNotEqual(first, third)
+                self.assertEqual(2, calls["n"])
+
+
+class TestInjectGuestBootSymlinkSafe(unittest.TestCase):
+    """A committed snapshot is guest-controlled: inject_guest_boot must not
+    follow a planted symlink and overwrite a host file during resume."""
+
+    def test_planted_symlink_does_not_escape_staging_tree(self):
+        with tempfile.TemporaryDirectory(prefix="fc-inject.") as d:
+            tmp = Path(d)
+            dropbear = tmp / "dropbear"
+            dropbear.write_bytes(b"DROPBEAR")
+            # A host file the malicious snapshot tries to clobber.
+            victim = tmp / "victim"
+            victim.write_text("original")
+
+            rootfs = tmp / "rootfs"
+            rootfs.mkdir()
+            # The snapshot planted bb-init/bb-dropbear as symlinks to it.
+            (rootfs / "bb-init").symlink_to(victim)
+            (rootfs / "bb-dropbear").symlink_to(victim)
+
+            with patch.object(util, "dropbear_path", return_value=dropbear):
+                util.inject_guest_boot(rootfs, init_script="#!/bin/sh\nreal\n")
+
+            # Host file untouched; the staged paths are fresh regular files.
+            self.assertEqual("original", victim.read_text())
+            self.assertFalse((rootfs / "bb-init").is_symlink())
+            self.assertFalse((rootfs / "bb-dropbear").is_symlink())
+            self.assertEqual("#!/bin/sh\nreal\n", (rootfs / "bb-init").read_text())
+            self.assertEqual(b"DROPBEAR", (rootfs / "bb-dropbear").read_bytes())
+
+
+class TestCommitRootfsPermissions(unittest.TestCase):
+    """The snapshot tar can hold the bottle's private workspace, so the
+    freezer must write it owner-only (0600)."""
+
+    def _commit(self, tar_path: Path) -> int:
+        """Run _commit_rootfs_via_ssh with a stubbed ssh|tar pipe; return the
+        mode of the open partial observed mid-stream (from subprocess.run)."""
+        key = tar_path.parent.parent / "key"
+        key.write_text("K")
+        seen: dict[str, int] = {}
+
+        def fake_run(argv: list[str], *a: Any, **k: Any) -> Any:
+            out = k["stdout"]
+            seen["mode"] = stat.S_IMODE(os.fstat(out.fileno()).st_mode)
+            out.write(b"TARDATA")
+            return subprocess.CompletedProcess(argv, 0, b"", b"")
+
+        with patch.object(freezer.util, "ssh_base_argv", return_value=["ssh"]), \
+             patch.object(freezer.subprocess, "run", side_effect=fake_run):
+            freezer._commit_rootfs_via_ssh(key, "10.0.0.1", tar_path)
+        return seen["mode"]
+
+    def test_snapshot_created_owner_only(self):
+        with tempfile.TemporaryDirectory(prefix="fc-freeze.") as d:
+            tar_path = Path(d) / "state" / "committed-rootfs.tar"
+            tar_path.parent.mkdir()
+            stream_mode = self._commit(tar_path)
+
+            self.assertEqual(0o600, stream_mode)  # private during the stream
+            self.assertEqual(b"TARDATA", tar_path.read_bytes())
+            self.assertEqual(0o600, stat.S_IMODE(tar_path.stat().st_mode))
+
+    def test_leftover_world_readable_partial_is_recreated_private(self):
+        """A partial left 0644 by an interrupted prior run must not keep the
+        new snapshot world-readable while it streams."""
+        with tempfile.TemporaryDirectory(prefix="fc-freeze.") as d:
+            tar_path = Path(d) / "state" / "committed-rootfs.tar"
+            tar_path.parent.mkdir()
+            partial = tar_path.with_name(tar_path.name + ".partial")
+            partial.write_bytes(b"stale")
+            os.chmod(partial, 0o644)
+
+            stream_mode = self._commit(tar_path)
+
+            self.assertEqual(0o600, stream_mode)
+            self.assertEqual(b"TARDATA", tar_path.read_bytes())
+            self.assertEqual(0o600, stat.S_IMODE(tar_path.stat().st_mode))
+
+
 if __name__ == "__main__":
     unittest.main()