fix(firecracker): make cleanup reap orphans only, never live VMs

`cleanup()` enumerated *every* firecracker process under the run root and
*every* run dir, so `./cli.py cleanup` would kill running bottles and
delete their rootfs — despite being described as orphan cleanup. The
backend's `enumerate_active` registry is still a stub (#354), so there
was no live/dead signal.

Use the running firecracker processes themselves as that signal: a run
dir with a live VM is protected (never killed, never removed); only run
dirs with no live process (leaked by a hard-killed launch) are reaped,
plus firecracker pids whose run dir is already gone (lingering VMMs).

This makes cleanup safe to run alongside live bottles and lets it back a
periodic GC for the run-dir leak the launch.py teardown fix closes on the
clean-exit path.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

bot_bottle/backend/firecracker/cleanup.py

@@ -1,8 +1,23 @@
 """Cleanup for the Firecracker backend.
 
-Orphans are: firecracker VMM processes whose config lives under our run
-dir, and the per-bottle run dirs. TAP slots free themselves (the flock
-drops when the launcher exits), so there is nothing to reclaim there.
+Reaps *orphans* only — resources with no live VM behind them:
+
+  * orphan run dirs: a per-bottle run dir (holding the ~1G rootfs.ext4)
+    whose firecracker process has exited. These leak when a launch is
+    hard-killed before its teardown runs (host OOM/crash, a cancelled CI
+    job, `kill -9`); the clean-exit path already removes its own dir in
+    launch.py.
+  * orphan VM pids: a firecracker process whose run dir is already gone
+    — a VMM left lingering after its dir was removed.
+
+A run dir with a *live* firecracker process is a running bottle and is
+left strictly alone: it is neither killed nor removed. (The backend's
+`enumerate_active` registry is still a stub — #354 — so a live process
+is the only reliable "this bottle is in use" signal we have. Once the
+registry lands, registry-orphaned-but-running VMs can be reaped too.)
+
+TAP slots free themselves (the flock drops when the launcher exits), so
+there is nothing to reclaim there.
 """
 
 from __future__ import annotations
@@ -22,38 +37,73 @@ def _run_root() -> Path:
     return util.cache_dir() / "run"
 
 
-def _orphan_vm_pids() -> list[int]:
-    """firecracker processes whose --config-file is under our run dir."""
-    run_root = str(_run_root())
+def _run_dir_of(cmd: str, run_root: Path) -> Path | None:
+    """The bottle run dir a firecracker cmdline belongs to, or None.
+
+    A bottle VM is launched with `--config-file <run_root>/<slug>/config.json`,
+    so the run dir is the config file's parent when it sits directly under
+    the run root. Anything else (a builder VM, the infra VM elsewhere) is
+    not ours to reap here.
+    """
+    toks = cmd.split()
+    for i, tok in enumerate(toks):
+        if tok == "--config-file" and i + 1 < len(toks):
+            parent = Path(toks[i + 1]).parent
+            if parent.parent == run_root:
+                return parent
+    return None
+
+
+def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
+    """Inspect running firecracker VMs under ``run_root``.
+
+    Returns ``(live_run_dirs, orphan_pids)``:
+      * ``live_run_dirs`` — run dirs backed by a running VM (never reaped);
+      * ``orphan_pids`` — firecracker pids whose run dir no longer exists
+        (a lingering VMM to kill).
+    """
     result = subprocess.run(
         ["pgrep", "-a", "firecracker"],
         capture_output=True, text=True, check=False,
     )
     if result.returncode != 0:
-        return []
-    pids: list[int] = []
+        return set(), []
+    live: set[str] = set()
+    orphan_pids: list[int] = []
     for line in result.stdout.splitlines():
         parts = line.split(None, 1)
-        if len(parts) != 2 or run_root not in parts[1]:
+        if len(parts) != 2:
             continue
         try:
-            pids.append(int(parts[0]))
+            pid = int(parts[0])
         except ValueError:
             continue
-    return pids
+        run_dir = _run_dir_of(parts[1], run_root)
+        if run_dir is None:
+            continue
+        if run_dir.is_dir():
+            live.add(str(run_dir))
+        else:
+            orphan_pids.append(pid)
+    return live, orphan_pids
 
 
-def _run_dirs() -> list[str]:
-    run_root = _run_root()
+def _orphan_run_dirs(run_root: Path, live: set[str]) -> list[str]:
+    """Run dirs with no live VM behind them — the leaked ones to remove."""
     if not run_root.is_dir():
         return []
-    return sorted(str(p) for p in run_root.iterdir() if p.is_dir())
+    return sorted(
+        str(p) for p in run_root.iterdir()
+        if p.is_dir() and str(p) not in live
+    )
 
 
 def prepare_cleanup() -> FirecrackerBottleCleanupPlan:
+    run_root = _run_root()
+    live, orphan_pids = _scan_processes(run_root)
     return FirecrackerBottleCleanupPlan(
-        vm_pids=tuple(_orphan_vm_pids()),
-        run_dirs=tuple(_run_dirs()),
+        vm_pids=tuple(orphan_pids),
+        run_dirs=tuple(_orphan_run_dirs(run_root, live)),
     )
 
 

tests/unit/test_firecracker_cleanup.py

@@ -10,7 +10,9 @@ classmethods forward to their module.
 from __future__ import annotations
 
 import subprocess
+import tempfile
 import unittest
+from pathlib import Path
 from unittest.mock import patch
 
 from bot_bottle.backend.firecracker import cleanup as fc_cleanup
@@ -23,32 +25,69 @@ def _proc(stdout: str = "", returncode: int = 0) -> "subprocess.CompletedProcess
     return subprocess.CompletedProcess([], returncode, stdout=stdout, stderr="")
 
 
-class TestOrphanEnumeration(unittest.TestCase):
-    def test_orphan_vm_pids_filters_by_run_dir(self):
-        run_root = str(fc_cleanup._run_root())
-        out = (
-            f"111 firecracker --config-file {run_root}/dev-a/config.json\n"
-            "222 firecracker --config-file /somewhere/else/config.json\n"
-            "notanint firecracker --config-file " + run_root + "/x\n"
+class TestProcessScan(unittest.TestCase):
+    def test_run_dir_of_matches_only_direct_children(self):
+        run_root = Path("/cache/run")
+        self.assertEqual(
+            Path("/cache/run/dev-a"),
+            fc_cleanup._run_dir_of(
+                f"firecracker --config-file {run_root}/dev-a/config.json", run_root
+            ),
+        )
+        # infra/builder VMs elsewhere, or nested paths, are not ours.
+        self.assertIsNone(
+            fc_cleanup._run_dir_of("firecracker --config-file /elsewhere/config.json", run_root)
+        )
+        self.assertIsNone(
+            fc_cleanup._run_dir_of("firecracker --no-config", run_root)
         )
-        with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
-            self.assertEqual([111], fc_cleanup._orphan_vm_pids())
 
-    def test_orphan_vm_pids_empty_when_pgrep_fails(self):
+    def test_scan_splits_live_dirs_from_orphan_pids(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            run_root = Path(tmp)
+            (run_root / "live-a").mkdir()  # dir present -> live VM, protected
+            # "gone-b" dir intentionally absent -> lingering VMM, orphan pid
+            out = (
+                f"111 firecracker --config-file {run_root}/live-a/config.json\n"
+                f"222 firecracker --config-file {run_root}/gone-b/config.json\n"
+                "333 firecracker --config-file /elsewhere/config.json\n"
+                "notanint firecracker --config-file x\n"
+            )
+            with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
+                live, orphan_pids = fc_cleanup._scan_processes(run_root)
+        self.assertEqual({str(run_root / "live-a")}, live)
+        self.assertEqual([222], orphan_pids)
+
+    def test_scan_empty_when_pgrep_fails(self):
         with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(returncode=1)):
-            self.assertEqual([], fc_cleanup._orphan_vm_pids())
+            self.assertEqual((set(), []), fc_cleanup._scan_processes(Path("/x")))
 
-    def test_run_dirs_empty_when_absent(self):
-        with patch.object(fc_cleanup.util, "cache_dir") as cache:
-            cache.return_value.__truediv__.return_value.is_dir.return_value = False
-            self.assertEqual([], fc_cleanup._run_dirs())
+    def test_orphan_run_dirs_excludes_live_and_missing_root(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            run_root = Path(tmp)
+            (run_root / "live-a").mkdir()
+            (run_root / "dead-b").mkdir()
+            live = {str(run_root / "live-a")}
+            self.assertEqual(
+                [str(run_root / "dead-b")],
+                fc_cleanup._orphan_run_dirs(run_root, live),
+            )
+        # absent run root -> nothing to reap
+        self.assertEqual([], fc_cleanup._orphan_run_dirs(Path("/nope/run"), set()))
 
-    def test_prepare_cleanup_assembles_plan(self):
-        with patch.object(fc_cleanup, "_orphan_vm_pids", return_value=[7]), \
-             patch.object(fc_cleanup, "_run_dirs", return_value=["/run/x"]):
-            plan = fc_cleanup.prepare_cleanup()
-        self.assertEqual((7,), plan.vm_pids)
-        self.assertEqual(("/run/x",), plan.run_dirs)
+    def test_prepare_cleanup_reaps_orphans_only(self):
+        """The live VM's dir is never in the plan; the dead one is."""
+        with tempfile.TemporaryDirectory() as tmp:
+            run_root = Path(tmp)
+            (run_root / "live-a").mkdir()
+            (run_root / "dead-b").mkdir()
+            out = f"111 firecracker --config-file {run_root}/live-a/config.json\n"
+            with patch.object(fc_cleanup, "_run_root", return_value=run_root), \
+                 patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
+                plan = fc_cleanup.prepare_cleanup()
+        self.assertEqual((), plan.vm_pids)
+        self.assertEqual((str(run_root / "dead-b"),), plan.run_dirs)
+        self.assertNotIn(str(run_root / "live-a"), plan.run_dirs)
 
 
 class TestCleanupRemoval(unittest.TestCase):