refactor(supervisor): separate MCP dispatch from transport

bot_bottle/backend/firecracker/cleanup.py

@@ -30,7 +30,7 @@ from pathlib import Path
 
 from ...log import info
 from .. import EnumerationError
-from . import lifecycle_lock, util
+from . import util
 from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan
 
 
@@ -126,42 +126,12 @@ def prepare_cleanup() -> FirecrackerBottleCleanupPlan:
 
 
 def cleanup(plan: FirecrackerBottleCleanupPlan) -> None:
-    """Revalidate the preview under the launch lock, then remove its survivors."""
-    with lifecycle_lock.hold():
-        fresh = prepare_cleanup()
-        approved_pids = set(plan.vm_pids).intersection(fresh.vm_pids)
-        approved_dirs = set(plan.run_dirs).intersection(fresh.run_dirs)
-        for pid in sorted(approved_pids):
-            _terminate_orphan(pid, _run_root())
-        for path in sorted(approved_dirs):
-            info(f"rm -rf {path}")
-            shutil.rmtree(path, ignore_errors=True)
-
-
-def _terminate_orphan(pid: int, run_root: Path) -> None:
-    """Signal exactly the process identity that still owns an orphan config."""
-    try:
-        pidfd = os.pidfd_open(pid)
-    except ProcessLookupError:
-        return
-    except OSError as exc:
-        raise EnumerationError(
-            f"could not pin Firecracker pid {pid} for cleanup: {exc}"
-        ) from exc
-    try:
-        try:
-            raw = Path(f"/proc/{pid}/cmdline").read_bytes()
-        except FileNotFoundError:
-            return
-        except OSError as exc:
-            raise EnumerationError(
-                f"could not revalidate Firecracker pid {pid}: {exc}"
-            ) from exc
-        command = raw.replace(b"\0", b" ").decode(errors="replace")
-        run_dir = _run_dir_of(command, run_root)
-        if run_dir is None or run_dir.is_dir():
-            return
+    for pid in plan.vm_pids:
         info(f"kill firecracker VM pid {pid}")
-        signal.pidfd_send_signal(pidfd, signal.SIGTERM)
-    finally:
-        os.close(pidfd)
+        try:
+            os.kill(pid, signal.SIGTERM)
+        except ProcessLookupError:
+            pass
+    for path in plan.run_dirs:
+        info(f"rm -rf {path}")
+        shutil.rmtree(path, ignore_errors=True)

bot_bottle/backend/firecracker/launch.py

@@ -46,7 +46,7 @@ from ...log import die, info, warn
 from ...supervisor.types import SUPERVISE_PORT
 from ..docker.egress import EGRESS_PORT
 from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
-from . import firecracker_vm, image_builder, isolation_probe, lifecycle_lock, netpool, util
+from . import firecracker_vm, image_builder, isolation_probe, netpool, util
 from .bottle import FirecrackerBottle
 from .bottle_plan import FirecrackerBottlePlan
 from ...orchestrator.store.config_store import resolve_teardown_timeout
@@ -164,29 +164,25 @@ def launch(
         )
 
         # Step 6: build the per-bottle rootfs + SSH key, then boot.
-        # Cleanup takes the same lock while refreshing its process snapshot.
-        # Hold it until the VMM exists so a newly-created run dir can never be
-        # mistaken for an orphan in the build-before-boot window.
-        with lifecycle_lock.hold():
-            run_dir = util.cache_dir() / "run" / plan.slug
-            run_dir.mkdir(parents=True, exist_ok=True)
-            # Remove the run dir on teardown so the per-bottle rootfs.ext4 (~1G)
-            # doesn't leak. Registered before vm.terminate below so it runs
-            # *after* it (ExitStack is LIFO).
-            stack.callback(lambda: shutil.rmtree(run_dir, ignore_errors=True))
-            rootfs = run_dir / "rootfs.ext4"
-            util.build_rootfs_ext4(agent_base, rootfs)
-            private_key, pubkey = util.generate_keypair(run_dir)
+        run_dir = util.cache_dir() / "run" / plan.slug
+        run_dir.mkdir(parents=True, exist_ok=True)
+        # Remove the run dir on teardown so the per-bottle rootfs.ext4 (~1G)
+        # doesn't leak. Registered before vm.terminate below so it runs *after*
+        # it (ExitStack is LIFO): the VM is gone before we rm its rootfs.
+        stack.callback(lambda: shutil.rmtree(run_dir, ignore_errors=True))
+        rootfs = run_dir / "rootfs.ext4"
+        util.build_rootfs_ext4(agent_base, rootfs)
+        private_key, pubkey = util.generate_keypair(run_dir)
 
-            vm = firecracker_vm.boot(
-                name=plan.container_name,
-                rootfs=rootfs,
-                tap=slot.iface,
-                guest_ip=slot.guest_ip,
-                host_ip=slot.host_ip,
-                pubkey=pubkey,
-                run_dir=run_dir,
-            )
+        vm = firecracker_vm.boot(
+            name=plan.container_name,
+            rootfs=rootfs,
+            tap=slot.iface,
+            guest_ip=slot.guest_ip,
+            host_ip=slot.host_ip,
+            pubkey=pubkey,
+            run_dir=run_dir,
+        )
         stack.callback(vm.terminate)
         firecracker_vm.wait_for_ssh(vm, private_key)
         persist_env_var_secret(private_key, slot.guest_ip, ctx.env_var_secret)

bot_bottle/backend/firecracker/lifecycle_lock.py

@@ -1,30 +0,0 @@
-"""Serialize Firecracker run-directory creation with orphan cleanup."""
-
-from __future__ import annotations
-
-import fcntl
-from contextlib import contextmanager
-from pathlib import Path
-from typing import Generator
-
-from . import util
-
-
-def _lock_path() -> Path:
-    return util.cache_dir() / "run.lifecycle.lock"
-
-
-@contextmanager
-def hold() -> Generator[None]:
-    """Exclude cleanup while a launch directory lacks a visible VMM."""
-    path = _lock_path()
-    path.parent.mkdir(parents=True, exist_ok=True)
-    with path.open("a", encoding="utf-8") as handle:
-        fcntl.flock(handle, fcntl.LOCK_EX)
-        try:
-            yield
-        finally:
-            fcntl.flock(handle, fcntl.LOCK_UN)
-
-
-__all__ = ["hold"]

bot_bottle/backend/macos_container/cleanup.py

@@ -4,8 +4,7 @@ from __future__ import annotations
 
 import subprocess
 
-from .. import EnumerationError
-from ...log import info
+from ...log import info, warn
 from . import util as container_mod
 from .bottle_cleanup_plan import MacosContainerBottleCleanupPlan
 
@@ -20,8 +19,8 @@ def _list_prefixed_containers() -> list[str]:
         check=False,
     )
     if result.returncode != 0:
-        detail = result.stderr.strip() or f"exit {result.returncode}"
-        raise EnumerationError(f"container list failed: {detail}")
+        warn(f"container list failed: {result.stderr.strip()}")
+        return []
     return sorted(
         name for name in (line.strip() for line in result.stdout.splitlines())
         if name.startswith(_PREFIX)
@@ -36,8 +35,7 @@ def _list_prefixed_networks() -> list[str]:
         check=False,
     )
     if result.returncode != 0:
-        detail = result.stderr.strip() or f"exit {result.returncode}"
-        raise EnumerationError(f"container network list failed: {detail}")
+        return []
     return sorted(
         name for name in (line.strip() for line in result.stdout.splitlines())
         if name.startswith(_PREFIX)

bot_bottle/cli/commands/cleanup.py

@@ -52,11 +52,7 @@ def cmd_cleanup(_argv: list[str]) -> int:
         info("cleanup: skipped")
         return 0
 
-    # Confirmation authorizes a fresh authoritative snapshot, not blind use of
-    # identities that may have changed while the operator reviewed the preview.
-    refreshed = [(name, backend, backend.prepare_cleanup())
-                 for name, backend, _plan in prepared]
-    for name, backend, plan in refreshed:
+    for name, backend, plan in prepared:
         if plan.empty:
             continue
         backend.cleanup(plan)

docs/prds/0082-authoritative-failure-boundaries.md

@@ -13,7 +13,9 @@ resource-consuming boundary revalidate the assumptions it acts on. This
 finishes the focused quality work begun under #444 without broad rewrites:
 cleanup cannot act on stale identities, policy introspection cannot publish a
 fabricated empty policy, gateway servers bound untrusted work, and daemon
-shutdown does not emit uncaught background-thread failures.
+shutdown does not emit uncaught background-thread failures. Shared
+control-plane storage and gateway credential provisioning also enforce their
+filesystem security contract before sensitive data is written.
 
 ## Problem
 
@@ -45,6 +47,18 @@ misleading behavior:
 8. Firecracker artifact downloads and registry publication have no network
    deadline, so an unresponsive registry can hold setup or release work
    indefinitely.
+9. Authenticated secret blobs select the unauthenticated legacy decoder when
+   their in-band version prefix is changed, allowing storage tampering to
+   bypass tag verification.
+10. Cleanup executes the entire post-confirmation snapshot rather than the
+    intersection with what the operator saw, and mutation failures are not
+    reflected in the command result.
+11. Git smart-HTTP can retain sixteen 100 MiB request bodies concurrently,
+    cleanup mutations have no subprocess deadline, and Firecracker signalling
+    failures bypass shared mutation accounting.
+12. SQLite creates the shared control-plane database before its mode is
+    restricted, then suppresses permission-repair failures. Gateway transports
+    also differ in whether copied deploy-key modes are preserved.
 
 These are one design problem: state used to authorize deletion, replacement,
 or resource allocation must be authoritative at the point of use.
@@ -73,6 +87,20 @@ or resource allocation must be authoritative at the point of use.
   as completion while preserving diagnostics for unexpected failures.
 - Artifact pull, existence-check, and publication requests use explicit
   network deadlines.
+- Persisted secrets accept only the authenticated format. The schema migration
+  intentionally clears legacy rows; local agents are reprovisioned rather
+  than retaining a ciphertext-controlled downgrade path.
+- Cleanup executes only resources present in both the displayed and current
+  authoritative plans, attempts every approved mutation, and returns failure
+  when any mutation does not complete.
+- Git request bodies spool to disk behind a separate heavy-work semaphore;
+  cleanup commands have configurable deadlines; Firecracker signalling
+  failures aggregate while identity-verification uncertainty still aborts.
+- The shared database directory and file are private before SQLite writes any
+  control-plane state; an inability to enforce those modes aborts startup.
+- Gateway credential directories and files receive explicit private modes
+  inside the gateway, independent of Docker, Apple Container, or SSH copy
+  semantics.
 - Unit tests cover PID/path reuse, partial backend enumeration, transient
   policy resolution failure, slow bodies, concurrency saturation, and stream
   closure races.
@@ -135,6 +163,17 @@ The gateway output pump catches only stream-closure exceptions expected after
 the supervisor closes child pipes. Other I/O failures remain visible and are
 reported through the supervisor's normal diagnostic channel.
 
+### Shared filesystem security
+
+The common SQLite store owns database creation for every backend. It creates
+the parent directory and an empty database with private modes before opening
+SQLite, repairs existing modes, verifies the resulting state, and propagates
+every enforcement failure. Backend launchers do not duplicate this policy.
+
+The backend-neutral gateway provisioner likewise applies directory and file
+modes after transport copies complete. This avoids relying on copy behavior
+that differs among Docker, Apple Container, and Firecracker's SSH transport.
+
 ## Implementation chunks
 
 1. Existing fail-closed security and backend enumeration fixes.
@@ -146,6 +185,12 @@ reported through the supervisor's normal diagnostic channel.
 7. Gateway shutdown log-pump closure handling.
 8. Lossless Firecracker process identities, authoritative Docker cleanup
    queries, and bounded Firecracker artifact transfers.
+9. Mandatory authenticated secret storage, shared cleanup-plan intersection
+   and mutation accounting, and contained Git backend process failures.
+10. Disk-spooled and separately bounded Git bodies, cleanup command deadlines,
+    and classified Firecracker signalling failures.
+11. Fail-closed shared database creation and backend-neutral gateway credential
+    permissions.
 
 ## Open questions
 

tests/unit/test_cli_cleanup_cross_backend.py

@@ -41,8 +41,8 @@ class TestCmdCleanup(unittest.TestCase):
         ):
             self.assertEqual(0, cmd.cmd_cleanup([]))
 
-        self.assertEqual(2, docker.prepare_cleanup.call_count)
-        self.assertEqual(2, fc.prepare_cleanup.call_count)
+        docker.prepare_cleanup.assert_called_once()
+        fc.prepare_cleanup.assert_called_once()
         docker.cleanup.assert_called_once_with(docker_plan)
         fc.cleanup.assert_called_once_with(fc_plan)
 
@@ -68,7 +68,7 @@ class TestCmdCleanup(unittest.TestCase):
         ):
             self.assertEqual(0, cmd.cmd_cleanup([]))
 
-        self.assertEqual(2, docker.prepare_cleanup.call_count)
+        docker.prepare_cleanup.assert_called_once()
         docker.cleanup.assert_called_once_with(docker_plan)
         macos.prepare_cleanup.assert_not_called()
 
@@ -135,25 +135,6 @@ class TestCmdCleanup(unittest.TestCase):
         docker.cleanup.assert_called_once_with(docker_plan)
         fc.cleanup.assert_not_called()
 
-    def test_executes_refreshed_plan_after_confirmation(self):
-        backend = MagicMock()
-        preview = MagicMock(empty=False)
-        refreshed = MagicMock(empty=False)
-        backend.prepare_cleanup.side_effect = [preview, refreshed]
-
-        with patch.object(
-            cmd, "known_backend_names", return_value=("firecracker",),
-        ), patch.object(
-            cmd, "get_bottle_backend", return_value=backend,
-        ), patch.object(
-            cmd, "has_backend", return_value=True,
-        ), patch.object(
-            cmd, "_prompt_yes", return_value=True,
-        ):
-            self.assertEqual(0, cmd.cmd_cleanup([]))
-
-        backend.cleanup.assert_called_once_with(refreshed)
-
 
 if __name__ == "__main__":
     unittest.main()

tests/unit/test_firecracker_cleanup.py

@@ -132,46 +132,18 @@ class TestCleanupRemoval(unittest.TestCase):
             vm_pids=(101,),
             run_dirs=("/run/dev-x",),
         )
-        with patch.object(fc_cleanup, "prepare_cleanup", return_value=plan), \
-             patch.object(fc_cleanup, "_run_root", return_value=Path("/run")), \
-             patch.object(fc_cleanup, "_terminate_orphan") as terminate, \
+        with patch.object(fc_cleanup.os, "kill") as kill, \
              patch.object(fc_cleanup.shutil, "rmtree") as rmtree, \
              patch.object(fc_cleanup, "info"):
             fc_cleanup.cleanup(plan)
-        terminate.assert_called_once_with(101, Path("/run"))
+        kill.assert_called_once()
         rmtree.assert_called_once_with("/run/dev-x", ignore_errors=True)
 
-    def test_cleanup_skips_resources_no_longer_in_refreshed_plan(self):
-        preview = FirecrackerBottleCleanupPlan(
-            vm_pids=(999,), run_dirs=("/run/reused",),
-        )
-        with patch.object(
-            fc_cleanup, "prepare_cleanup",
-            return_value=FirecrackerBottleCleanupPlan(),
-        ), patch.object(fc_cleanup, "_terminate_orphan") as terminate, \
-             patch.object(fc_cleanup.shutil, "rmtree") as rmtree:
-            fc_cleanup.cleanup(preview)
-        terminate.assert_not_called()
-        rmtree.assert_not_called()
-
-    def test_pidfd_prevents_pid_reuse_from_signalling_unrelated_process(self):
-        with patch.object(fc_cleanup.os, "pidfd_open", return_value=7), \
-             patch.object(
-                 fc_cleanup.Path, "read_bytes",
-                 return_value=b"/usr/bin/python\0worker.py\0",
-             ), patch.object(fc_cleanup.signal, "pidfd_send_signal") as send, \
-             patch.object(fc_cleanup.os, "close"):
-            fc_cleanup._terminate_orphan(101, Path("/run"))
-        send.assert_not_called()
-
-    def test_pidfd_signals_revalidated_orphan(self):
-        command = b"firecracker\0--config-file\0/run/gone/config.json\0"
-        with patch.object(fc_cleanup.os, "pidfd_open", return_value=7), \
-             patch.object(fc_cleanup.Path, "read_bytes", return_value=command), \
-             patch.object(fc_cleanup.signal, "pidfd_send_signal") as send, \
-             patch.object(fc_cleanup.os, "close"), patch.object(fc_cleanup, "info"):
-            fc_cleanup._terminate_orphan(101, Path("/run"))
-        send.assert_called_once_with(7, fc_cleanup.signal.SIGTERM)
+    def test_cleanup_tolerates_dead_pid(self):
+        plan = FirecrackerBottleCleanupPlan(vm_pids=(999,))
+        with patch.object(fc_cleanup.os, "kill", side_effect=ProcessLookupError), \
+             patch.object(fc_cleanup, "info"):
+            fc_cleanup.cleanup(plan)  # must not raise
 
 
 class TestCleanupPlan(unittest.TestCase):

tests/unit/test_macos_container_cleanup.py

@@ -42,22 +42,6 @@ class TestMacosContainerCleanup(unittest.TestCase):
             run.call_args_list[1].args[0],
         )
 
-    def test_container_enumeration_failure_aborts(self):
-        completed = cleanup.subprocess.CompletedProcess(
-            args=[], returncode=1, stdout="", stderr="service unavailable",
-        )
-        with patch.object(cleanup.subprocess, "run", return_value=completed), \
-                self.assertRaisesRegex(EnumerationError, "service unavailable"):
-            cleanup._list_prefixed_containers()
-
-    def test_network_enumeration_failure_aborts(self):
-        completed = cleanup.subprocess.CompletedProcess(
-            args=[], returncode=1, stdout="", stderr="service unavailable",
-        )
-        with patch.object(cleanup.subprocess, "run", return_value=completed), \
-                self.assertRaisesRegex(EnumerationError, "service unavailable"):
-            cleanup._list_prefixed_networks()
-
 
 class TestMacosContainerEnumerate(unittest.TestCase):
     """The backend launches bottles again (PRD 0070), so enumeration is real