fix(smolmachines): consolidate virtiofs mounts to stay within libkrun limit

libkrun limits total mounts + port-mappings to 5. With all three
sidecar daemons active (egress, git-gate, supervise), the prior
layout used 3 mounts + 3 ports = 6, causing the VM to crash at boot
with krun_start_enter returned -22.

Merge the egress confdir and git-gate scripts into a single staging
directory mounted at /bot-bottle-data/ with egress/ and git-gate/
subdirectories, reducing to 2 mounts + 3 ports = 5. Also clean up
leftover VMs before creating new ones to handle interrupted teardowns.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Dockerfile.sidecars

@@ -89,16 +89,19 @@ RUN mkdir -p \
       /git-gate/creds \
       /git \
       /run/supervise \
-      /home/mitmproxy/.mitmproxy
+      /home/mitmproxy/.mitmproxy \
+      /bot-bottle-data/egress \
+      /bot-bottle-data/git-gate
 
 # Static wrapper for the git-gate entrypoint. The per-bottle
 # entrypoint script is either:
 #   - docker-cp'd to /git-gate-entrypoint.sh (docker/macOS backends),
 #     which overwrites this wrapper; or
-#   - virtiofs-mounted at /etc/git-gate/entrypoint.sh (smolmachines),
-#     where this wrapper delegates to it at runtime.
+#   - virtiofs-mounted at /bot-bottle-data/git-gate/entrypoint.sh
+#     (smolmachines), where this wrapper delegates to it at runtime.
+#     Fallback to /etc/git-gate/ for backwards compatibility.
 # Either way sidecar_init.py calls `/bin/sh /git-gate-entrypoint.sh`.
-RUN printf '#!/bin/sh\nexec /etc/git-gate/entrypoint.sh "$@"\n' \
+RUN printf '#!/bin/sh\nif [ -x /bot-bottle-data/git-gate/entrypoint.sh ]; then\n  exec /bot-bottle-data/git-gate/entrypoint.sh "$@"\nfi\nexec /etc/git-gate/entrypoint.sh "$@"\n' \
       > /git-gate-entrypoint.sh \
     && chmod 755 /git-gate-entrypoint.sh
 

bot_bottle/backend/smolmachines/egress_apply.py

@@ -7,16 +7,23 @@ exec`` instead of Docker.
 
 from __future__ import annotations
 
-from ...egress import EGRESS_ROUTES_IN_CONTAINER
+from pathlib import Path
+
+from ...bottle_state import egress_state_dir
 from ...log import warn
 from ..egress_apply import EgressApplicator, EgressApplyError
 from . import sidecar_bundle as _bundle
 from . import smolvm as _smolvm
 
+# Routes file path inside the sidecar VM. Set via EGRESS_ROUTES env var
+# at launch so the addon reads from the virtiofs-mounted confdir instead
+# of the default /etc/egress/routes.yaml.
+_EGRESS_ROUTES_IN_SIDECAR_VM = "/bot-bottle-data/egress/routes.yaml"
+
 
 def fetch_current_routes(slug: str) -> str:
     machine = _bundle.bundle_machine_name(slug)
-    result = _smolvm.machine_exec(machine, ["cat", EGRESS_ROUTES_IN_CONTAINER])
+    result = _smolvm.machine_exec(machine, ["cat", _EGRESS_ROUTES_IN_SIDECAR_VM])
     if result.returncode != 0:
         raise EgressApplyError(
             f"could not read routes.yaml from {machine}: "
@@ -26,6 +33,14 @@ def fetch_current_routes(slug: str) -> str:
 
 
 class SmolmachinesEgressApplicator(EgressApplicator):
+    @staticmethod
+    def _routes_path(slug: str) -> Path:
+        # Routes live in the smolvm-sidecar-data staging dir, which is
+        # virtiofs-mounted at /bot-bottle-data/ in the sidecar VM. Writes
+        # here are visible inside the VM immediately; a SIGHUP causes the
+        # addon to reload from _EGRESS_ROUTES_IN_SIDECAR_VM.
+        return egress_state_dir(slug) / "smolvm-sidecar-data" / "egress" / "routes.yaml"
+
     def _signal_bundle_reload(self, slug: str) -> None:
         machine = _bundle.bundle_machine_name(slug)
         result = _smolvm.machine_exec(machine, ["sh", "-c", "kill -HUP 1"])

bot_bottle/backend/smolmachines/launch.py

@@ -20,7 +20,6 @@ from pathlib import Path
 from typing import Callable, Generator
 
 from ...egress import (
-    EGRESS_ROUTES_IN_CONTAINER,
     egress_agent_env_entries,
     egress_resolve_token_values,
     egress_sidecar_env_entries,
@@ -29,11 +28,9 @@ from ...supervise import DB_PATH_IN_CONTAINER, SUPERVISE_PORT
 from ...util import expand_tilde
 from ..docker import util as docker_mod
 from ..docker.egress import (
-    EGRESS_CA_IN_CONTAINER,
     EGRESS_PORT as _EGRESS_PORT,
     egress_tls_init,
 )
-from ..docker.git_gate import GIT_GATE_CREDS_DIR_IN_CONTAINER
 from ...git_gate import (
     provision_git_gate_dynamic_keys,
     revoke_git_gate_provisioned_keys,
@@ -57,12 +54,16 @@ from .local_registry import crane_push_tarball, ephemeral_registry
 _REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
 
 
-# In-VM directory containing git-gate scripts staged for virtiofs mount.
-# The Dockerfile.sidecars static wrapper at /git-gate-entrypoint.sh
-# delegates to this path, so the smolmachines backend can supply the
-# per-bottle entrypoint via a directory virtiofs mount instead of a
-# post-boot machine_cp (which runs too late for PID 1 daemons).
-_GIT_GATE_SCRIPTS_DIR_IN_VM = "/etc/git-gate"
+# Single virtiofs mount for egress + git-gate files. libkrun limits
+# the total of mounts + port-mappings to 5; with 3 daemon ports the
+# sidecar VM can carry at most 2 mounts. Egress CA/routes and
+# git-gate scripts/creds are staged into subdirectories of one host
+# dir and mounted here. Env vars (EGRESS_CONFDIR, EGRESS_ROUTES,
+# and the Dockerfile's git-gate wrapper) point each daemon at its
+# subdirectory.
+_SIDECAR_DATA_DIR_IN_VM = "/bot-bottle-data"
+_EGRESS_CONFDIR_IN_VM = f"{_SIDECAR_DATA_DIR_IN_VM}/egress"
+_GIT_GATE_SCRIPTS_DIR_IN_VM = f"{_SIDECAR_DATA_DIR_IN_VM}/git-gate"
 
 
 # Per-host cache for `smolvm pack create` outputs. Keyed by the
@@ -295,6 +296,16 @@ def _launch_vm(
     fails closed if it can't. Smolfile isn't usable here — smolvm 0.8.0
     makes --from and --smolfile mutually exclusive."""
     tsi_cidr = f"{proxy_host}/32"
+    # Destroy any leftover machine from a previous run that didn't
+    # clean up (e.g. crash, interrupted teardown).
+    try:
+        _smolvm.machine_stop(plan.machine_name)
+    except _smolvm.SmolvmError:
+        pass
+    try:
+        _smolvm.machine_delete(plan.machine_name)
+    except _smolvm.SmolvmError:
+        pass
     _smolvm.machine_create(
         plan.machine_name,
         from_path=agent_from_path,
@@ -362,45 +373,62 @@ def _port_for_label(label: str) -> int:
     raise ValueError(f"unknown sidecar forward label: {label}")
 
 
-def _stage_git_gate_for_smolvm(
-    plan: SmolmachinesBottlePlan,
-) -> tuple[Path, Path]:
-    """Stage git-gate scripts and credentials as virtiofs-mountable dirs.
+def _stage_sidecar_data(plan: SmolmachinesBottlePlan) -> Path:
+    """Stage egress + git-gate files into one virtiofs-mountable dir.
 
-    virtiofs requires directory sources; the per-bottle git-gate scripts
-    have host-side names that differ from their in-VM paths (e.g.
-    `git_gate_entrypoint.sh` → `entrypoint.sh` in /etc/git-gate/), so
-    they cannot be expressed as parent-dir mounts without renaming.
+    libkrun limits total mounts + port-mappings to 5. With 3 daemon
+    ports the sidecar VM can carry at most 2 mounts (the second is
+    the supervise DB). Egress and git-gate share a single mount:
 
-    Copies them into per-bottle staging dirs under the git-gate state
-    dir. Returns `(scripts_dir, creds_dir)`:
-      - scripts_dir is mounted to /etc/git-gate/ in the sidecar VM;
-        the image's static /git-gate-entrypoint.sh wrapper delegates there.
-      - creds_dir is mounted to /git-gate/creds/ in the sidecar VM."""
+        <staging>/egress/     → _EGRESS_CONFDIR_IN_VM
+        <staging>/git-gate/   → _GIT_GATE_SCRIPTS_DIR_IN_VM
+
+    The mount is writable so mitmproxy can write combined-trust.pem
+    and cache per-host certs under the egress subdir."""
+    staging = egress_state_dir(plan.slug) / "smolvm-sidecar-data"
+
+    # --- egress subdir ---
+    confdir = staging / "egress"
+    confdir.mkdir(parents=True, exist_ok=True)
+    ep = plan.egress_plan
+    shutil.copy2(str(ep.mitmproxy_ca_host_path), str(confdir / "mitmproxy-ca.pem"))
+    if ep.routes:
+        shutil.copy2(str(ep.routes_path), str(confdir / "routes.yaml"))
+
+    # --- git-gate subdir (only when upstreams are configured) ---
     gp = plan.git_gate_plan
-    state_dir = git_gate_state_dir(plan.slug)
+    if gp.upstreams:
+        scripts_dir = staging / "git-gate"
+        scripts_dir.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(str(gp.entrypoint_script), str(scripts_dir / "entrypoint.sh"))
+        shutil.copy2(str(gp.hook_script), str(scripts_dir / "pre-receive"))
+        shutil.copy2(str(gp.access_hook_script), str(scripts_dir / "access-hook"))
+        for name in ("entrypoint.sh", "pre-receive", "access-hook"):
+            (scripts_dir / name).chmod(0o755)
 
-    scripts_dir = state_dir / "smolvm-scripts"
-    scripts_dir.mkdir(parents=True, exist_ok=True)
-    shutil.copy2(str(gp.entrypoint_script), str(scripts_dir / "entrypoint.sh"))
-    shutil.copy2(str(gp.hook_script), str(scripts_dir / "pre-receive"))
-    shutil.copy2(str(gp.access_hook_script), str(scripts_dir / "access-hook"))
-    for name in ("entrypoint.sh", "pre-receive", "access-hook"):
-        (scripts_dir / name).chmod(0o755)
+        # Patch credential paths: the rendered entrypoint hardcodes
+        # /git-gate/creds/; rewrite to the in-VM git-gate subdir.
+        ep_path = scripts_dir / "entrypoint.sh"
+        ep_path.write_text(
+            ep_path.read_text().replace(
+                "/git-gate/creds/",
+                f"{_GIT_GATE_SCRIPTS_DIR_IN_VM}/creds/",
+            )
+        )
 
-    creds_dir = state_dir / "smolvm-creds"
-    creds_dir.mkdir(parents=True, exist_ok=True)
-    for u in gp.upstreams:
-        keypath = Path(expand_tilde(u.identity_file))
-        dest_key = creds_dir / f"{u.name}-key"
-        shutil.copy2(str(keypath), str(dest_key))
-        dest_key.chmod(0o600)
-        if u.known_hosts_file:
-            dest_kh = creds_dir / f"{u.name}-known_hosts"
-            shutil.copy2(str(u.known_hosts_file), str(dest_kh))
-            dest_kh.chmod(0o600)
+        creds_dir = scripts_dir / "creds"
+        creds_dir.mkdir(exist_ok=True)
+        for u in gp.upstreams:
+            keypath = Path(expand_tilde(u.identity_file))
+            dest_key = creds_dir / f"{u.name}-key"
+            shutil.copy2(str(keypath), str(dest_key))
+            dest_key.chmod(0o600)
+            if u.known_hosts_file:
+                dest_kh = creds_dir / f"{u.name}-known_hosts"
+                shutil.copy2(str(u.known_hosts_file), str(dest_kh))
+                dest_kh.chmod(0o600)
 
-    return scripts_dir, creds_dir
+    return staging
 
 
 def _bundle_launch_spec(
@@ -421,34 +449,26 @@ def _bundle_launch_spec(
     env: list[str] = []
     volumes: list[tuple[str, str, bool]] = []
 
-    # --- egress -----------------------------------------------
-    ep = plan.egress_plan
-    # virtiofs (smolvm) requires directory mounts — mount the CA's
-    # parent dir so mitmproxy-ca.pem lands at the right in-VM path.
+    # --- egress + git-gate (single mount) ---------------------
+    # Stage both into one dir and mount it at _SIDECAR_DATA_DIR_IN_VM.
+    # libkrun limits mounts + port-mappings to 5; with 3 daemon ports
+    # we can carry at most 2 mounts (this one + supervise DB).
     # Writable so egress_entrypoint.sh can write combined-trust.pem
     # and mitmproxy can create its per-host cert cache.
-    volumes.append((
-        str(ep.mitmproxy_ca_host_path.parent),
-        str(Path(EGRESS_CA_IN_CONTAINER).parent),
-        False,
-    ))
-    if ep.routes:
-        volumes.append((str(ep.routes_path.parent), str(Path(EGRESS_ROUTES_IN_CONTAINER).parent), True))
+    ep = plan.egress_plan
+    gp = plan.git_gate_plan
+    staging = _stage_sidecar_data(plan)
+    volumes.append((str(staging), _SIDECAR_DATA_DIR_IN_VM, False))
+    # Tell the egress entrypoint where to find its CA + routes.
+    env.append(f"EGRESS_CONFDIR={_EGRESS_CONFDIR_IN_VM}")
+    # Always set EGRESS_ROUTES so the addon reads from the confdir path
+    # even when no routes were configured at launch (apply_routes_change
+    # writes here and a SIGHUP causes the addon to pick them up).
+    env.append(f"EGRESS_ROUTES={_EGRESS_CONFDIR_IN_VM}/routes.yaml")
     env.extend(egress_sidecar_env_entries(ep))
 
-    # --- git-gate ---------------------------------------------
-    # virtiofs requires directory mounts; git-gate scripts have
-    # host-side names that differ from their in-VM paths. Stage
-    # them with correct names under the git-gate state dir and
-    # mount those staging dirs. The image's static wrapper at
-    # /git-gate-entrypoint.sh delegates to the staged entrypoint,
-    # so the per-bottle script is available at VM boot time.
-    gp = plan.git_gate_plan
     if gp.upstreams:
         daemons += ["git-gate", "git-http"]
-        scripts_dir, creds_dir = _stage_git_gate_for_smolvm(plan)
-        volumes.append((str(scripts_dir), _GIT_GATE_SCRIPTS_DIR_IN_VM, True))
-        volumes.append((str(creds_dir), GIT_GATE_CREDS_DIR_IN_CONTAINER, True))
 
     # --- supervise --------------------------------------------
     sp = plan.supervise_plan

bot_bottle/backend/smolmachines/sidecar_bundle.py

@@ -135,6 +135,16 @@ def start_bundle_vm(
         elif entry in effective_host_env:
             env[entry] = effective_host_env[entry]
     name = bundle_machine_name(spec.slug)
+    # Destroy any leftover machine from a previous run that didn't
+    # clean up (e.g. crash, interrupted teardown).
+    try:
+        _smolvm.machine_stop(name)
+    except _smolvm.SmolvmError:
+        pass
+    try:
+        _smolvm.machine_delete(name)
+    except _smolvm.SmolvmError:
+        pass
     _smolvm.machine_create(
         name,
         from_path=from_path,

bot_bottle/egress_entrypoint.sh

@@ -28,7 +28,7 @@ set -e
 # flag mitmdump would generate a fresh CA on the wrong path and
 # the agent's installed trust anchor would no longer match the
 # bumped leaf certs.
-CONFDIR=/home/mitmproxy/.mitmproxy
+CONFDIR="${EGRESS_CONFDIR:-/home/mitmproxy/.mitmproxy}"
 CONFDIR_FLAG="--set confdir=$CONFDIR"
 
 MODE="--mode regular@9099"

tests/unit/test_smolmachines_egress_apply.py

@@ -18,7 +18,7 @@ class TestFetchCurrentRoutes(unittest.TestCase):
             self.assertEqual("routes", egress_apply.fetch_current_routes("dev-abc"))
         exec_.assert_called_once_with(
             "bot-bottle-sidecars-dev-abc",
-            ["cat", "/etc/egress/routes.yaml"],
+            ["cat", "/bot-bottle-data/egress/routes.yaml"],
         )
 
     def test_read_failure_raises_apply_error(self):

tests/unit/test_smolmachines_provision.py

@@ -405,8 +405,8 @@ class TestBundleLaunchSpec(unittest.TestCase):
         )
 
         with patch(
-            "bot_bottle.backend.smolmachines.launch._stage_git_gate_for_smolvm",
-            return_value=(Path("/tmp/smolvm-scripts"), Path("/tmp/smolvm-creds")),
+            "bot_bottle.backend.smolmachines.launch._stage_sidecar_data",
+            return_value=Path("/tmp/smolvm-sidecar-data"),
         ):
             spec = _bundle_launch_spec(plan, "net", "127.0.0.16")
 
@@ -420,7 +420,11 @@ class TestBundleLaunchSpec(unittest.TestCase):
     def test_canary_env_registered_as_sensitive_in_bundle(self):
         plan = _plan(canary=True)
 
-        spec = _bundle_launch_spec(plan, "net", "127.0.0.16")
+        with patch(
+            "bot_bottle.backend.smolmachines.launch._stage_sidecar_data",
+            return_value=Path("/tmp/smolvm-sidecar-data"),
+        ):
+            spec = _bundle_launch_spec(plan, "net", "127.0.0.16")
 
         self.assertIn("CANON_ALPHA_SECRET=fake-canary-value", spec.environment)
         self.assertIn(
@@ -431,7 +435,11 @@ class TestBundleLaunchSpec(unittest.TestCase):
     def test_supervise_adds_daemon_volume_and_env(self):
         from bot_bottle.supervise import DB_PATH_IN_CONTAINER
         plan = _plan(supervise=True)
-        spec = _bundle_launch_spec(plan, "net", "127.0.0.16")
+        with patch(
+            "bot_bottle.backend.smolmachines.launch._stage_sidecar_data",
+            return_value=Path("/tmp/smolvm-sidecar-data"),
+        ):
+            spec = _bundle_launch_spec(plan, "net", "127.0.0.16")
         self.assertIn("supervise", spec.daemons_csv)
         self.assertIn(f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", spec.environment)
         # virtiofs requires directory mounts; the DB's parent dir is
@@ -562,8 +570,8 @@ class TestLaunchResourceWiring(unittest.TestCase):
             "bot_bottle.backend.smolmachines.launch._bundle.start_bundle_vm",
             return_value=raw_launch,
         ) as start_vm, patch(
-            "bot_bottle.backend.smolmachines.launch._stage_git_gate_for_smolvm",
-            return_value=(Path("/tmp/smolvm-scripts"), Path("/tmp/smolvm-creds")),
+            "bot_bottle.backend.smolmachines.launch._stage_sidecar_data",
+            return_value=Path("/tmp/smolvm-sidecar-data"),
         ), patch(
             "bot_bottle.backend.smolmachines.launch._forward.start_forwarder",
             return_value=handle,