diff --git a/Dockerfile.sidecars b/Dockerfile.sidecars index c287391..a7f309b 100644 --- a/Dockerfile.sidecars +++ b/Dockerfile.sidecars @@ -89,16 +89,19 @@ RUN mkdir -p \ /git-gate/creds \ /git \ /run/supervise \ - /home/mitmproxy/.mitmproxy + /home/mitmproxy/.mitmproxy \ + /bot-bottle-data/egress \ + /bot-bottle-data/git-gate # Static wrapper for the git-gate entrypoint. The per-bottle # entrypoint script is either: # - docker-cp'd to /git-gate-entrypoint.sh (docker/macOS backends), # which overwrites this wrapper; or -# - virtiofs-mounted at /etc/git-gate/entrypoint.sh (smolmachines), -# where this wrapper delegates to it at runtime. +# - virtiofs-mounted at /bot-bottle-data/git-gate/entrypoint.sh +# (smolmachines), where this wrapper delegates to it at runtime. +# Fallback to /etc/git-gate/ for backwards compatibility. # Either way sidecar_init.py calls `/bin/sh /git-gate-entrypoint.sh`. -RUN printf '#!/bin/sh\nexec /etc/git-gate/entrypoint.sh "$@"\n' \ +RUN printf '#!/bin/sh\nif [ -x /bot-bottle-data/git-gate/entrypoint.sh ]; then\n exec /bot-bottle-data/git-gate/entrypoint.sh "$@"\nfi\nexec /etc/git-gate/entrypoint.sh "$@"\n' \ > /git-gate-entrypoint.sh \ && chmod 755 /git-gate-entrypoint.sh diff --git a/bot_bottle/backend/smolmachines/egress_apply.py b/bot_bottle/backend/smolmachines/egress_apply.py index 661a13f..63d28db 100644 --- a/bot_bottle/backend/smolmachines/egress_apply.py +++ b/bot_bottle/backend/smolmachines/egress_apply.py @@ -7,16 +7,23 @@ exec`` instead of Docker. from __future__ import annotations -from ...egress import EGRESS_ROUTES_IN_CONTAINER +from pathlib import Path + +from ...bottle_state import egress_state_dir from ...log import warn from ..egress_apply import EgressApplicator, EgressApplyError from . import sidecar_bundle as _bundle from . import smolvm as _smolvm +# Routes file path inside the sidecar VM. Set via EGRESS_ROUTES env var +# at launch so the addon reads from the virtiofs-mounted confdir instead +# of the default /etc/egress/routes.yaml. +_EGRESS_ROUTES_IN_SIDECAR_VM = "/bot-bottle-data/egress/routes.yaml" + def fetch_current_routes(slug: str) -> str: machine = _bundle.bundle_machine_name(slug) - result = _smolvm.machine_exec(machine, ["cat", EGRESS_ROUTES_IN_CONTAINER]) + result = _smolvm.machine_exec(machine, ["cat", _EGRESS_ROUTES_IN_SIDECAR_VM]) if result.returncode != 0: raise EgressApplyError( f"could not read routes.yaml from {machine}: " @@ -26,6 +33,14 @@ def fetch_current_routes(slug: str) -> str: class SmolmachinesEgressApplicator(EgressApplicator): + @staticmethod + def _routes_path(slug: str) -> Path: + # Routes live in the smolvm-sidecar-data staging dir, which is + # virtiofs-mounted at /bot-bottle-data/ in the sidecar VM. Writes + # here are visible inside the VM immediately; a SIGHUP causes the + # addon to reload from _EGRESS_ROUTES_IN_SIDECAR_VM. + return egress_state_dir(slug) / "smolvm-sidecar-data" / "egress" / "routes.yaml" + def _signal_bundle_reload(self, slug: str) -> None: machine = _bundle.bundle_machine_name(slug) result = _smolvm.machine_exec(machine, ["sh", "-c", "kill -HUP 1"]) diff --git a/bot_bottle/backend/smolmachines/launch.py b/bot_bottle/backend/smolmachines/launch.py index c40d685..7227d23 100644 --- a/bot_bottle/backend/smolmachines/launch.py +++ b/bot_bottle/backend/smolmachines/launch.py @@ -20,7 +20,6 @@ from pathlib import Path from typing import Callable, Generator from ...egress import ( - EGRESS_ROUTES_IN_CONTAINER, egress_agent_env_entries, egress_resolve_token_values, egress_sidecar_env_entries, @@ -29,11 +28,9 @@ from ...supervise import DB_PATH_IN_CONTAINER, SUPERVISE_PORT from ...util import expand_tilde from ..docker import util as docker_mod from ..docker.egress import ( - EGRESS_CA_IN_CONTAINER, EGRESS_PORT as _EGRESS_PORT, egress_tls_init, ) -from ..docker.git_gate import GIT_GATE_CREDS_DIR_IN_CONTAINER from ...git_gate import ( provision_git_gate_dynamic_keys, revoke_git_gate_provisioned_keys, @@ -57,12 +54,16 @@ from .local_registry import crane_push_tarball, ephemeral_registry _REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent) -# In-VM directory containing git-gate scripts staged for virtiofs mount. -# The Dockerfile.sidecars static wrapper at /git-gate-entrypoint.sh -# delegates to this path, so the smolmachines backend can supply the -# per-bottle entrypoint via a directory virtiofs mount instead of a -# post-boot machine_cp (which runs too late for PID 1 daemons). -_GIT_GATE_SCRIPTS_DIR_IN_VM = "/etc/git-gate" +# Single virtiofs mount for egress + git-gate files. libkrun limits +# the total of mounts + port-mappings to 5; with 3 daemon ports the +# sidecar VM can carry at most 2 mounts. Egress CA/routes and +# git-gate scripts/creds are staged into subdirectories of one host +# dir and mounted here. Env vars (EGRESS_CONFDIR, EGRESS_ROUTES, +# and the Dockerfile's git-gate wrapper) point each daemon at its +# subdirectory. +_SIDECAR_DATA_DIR_IN_VM = "/bot-bottle-data" +_EGRESS_CONFDIR_IN_VM = f"{_SIDECAR_DATA_DIR_IN_VM}/egress" +_GIT_GATE_SCRIPTS_DIR_IN_VM = f"{_SIDECAR_DATA_DIR_IN_VM}/git-gate" # Per-host cache for `smolvm pack create` outputs. Keyed by the @@ -295,6 +296,16 @@ def _launch_vm( fails closed if it can't. Smolfile isn't usable here — smolvm 0.8.0 makes --from and --smolfile mutually exclusive.""" tsi_cidr = f"{proxy_host}/32" + # Destroy any leftover machine from a previous run that didn't + # clean up (e.g. crash, interrupted teardown). + try: + _smolvm.machine_stop(plan.machine_name) + except _smolvm.SmolvmError: + pass + try: + _smolvm.machine_delete(plan.machine_name) + except _smolvm.SmolvmError: + pass _smolvm.machine_create( plan.machine_name, from_path=agent_from_path, @@ -362,45 +373,62 @@ def _port_for_label(label: str) -> int: raise ValueError(f"unknown sidecar forward label: {label}") -def _stage_git_gate_for_smolvm( - plan: SmolmachinesBottlePlan, -) -> tuple[Path, Path]: - """Stage git-gate scripts and credentials as virtiofs-mountable dirs. +def _stage_sidecar_data(plan: SmolmachinesBottlePlan) -> Path: + """Stage egress + git-gate files into one virtiofs-mountable dir. - virtiofs requires directory sources; the per-bottle git-gate scripts - have host-side names that differ from their in-VM paths (e.g. - `git_gate_entrypoint.sh` → `entrypoint.sh` in /etc/git-gate/), so - they cannot be expressed as parent-dir mounts without renaming. + libkrun limits total mounts + port-mappings to 5. With 3 daemon + ports the sidecar VM can carry at most 2 mounts (the second is + the supervise DB). Egress and git-gate share a single mount: - Copies them into per-bottle staging dirs under the git-gate state - dir. Returns `(scripts_dir, creds_dir)`: - - scripts_dir is mounted to /etc/git-gate/ in the sidecar VM; - the image's static /git-gate-entrypoint.sh wrapper delegates there. - - creds_dir is mounted to /git-gate/creds/ in the sidecar VM.""" + /egress/ → _EGRESS_CONFDIR_IN_VM + /git-gate/ → _GIT_GATE_SCRIPTS_DIR_IN_VM + + The mount is writable so mitmproxy can write combined-trust.pem + and cache per-host certs under the egress subdir.""" + staging = egress_state_dir(plan.slug) / "smolvm-sidecar-data" + + # --- egress subdir --- + confdir = staging / "egress" + confdir.mkdir(parents=True, exist_ok=True) + ep = plan.egress_plan + shutil.copy2(str(ep.mitmproxy_ca_host_path), str(confdir / "mitmproxy-ca.pem")) + if ep.routes: + shutil.copy2(str(ep.routes_path), str(confdir / "routes.yaml")) + + # --- git-gate subdir (only when upstreams are configured) --- gp = plan.git_gate_plan - state_dir = git_gate_state_dir(plan.slug) + if gp.upstreams: + scripts_dir = staging / "git-gate" + scripts_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(str(gp.entrypoint_script), str(scripts_dir / "entrypoint.sh")) + shutil.copy2(str(gp.hook_script), str(scripts_dir / "pre-receive")) + shutil.copy2(str(gp.access_hook_script), str(scripts_dir / "access-hook")) + for name in ("entrypoint.sh", "pre-receive", "access-hook"): + (scripts_dir / name).chmod(0o755) - scripts_dir = state_dir / "smolvm-scripts" - scripts_dir.mkdir(parents=True, exist_ok=True) - shutil.copy2(str(gp.entrypoint_script), str(scripts_dir / "entrypoint.sh")) - shutil.copy2(str(gp.hook_script), str(scripts_dir / "pre-receive")) - shutil.copy2(str(gp.access_hook_script), str(scripts_dir / "access-hook")) - for name in ("entrypoint.sh", "pre-receive", "access-hook"): - (scripts_dir / name).chmod(0o755) + # Patch credential paths: the rendered entrypoint hardcodes + # /git-gate/creds/; rewrite to the in-VM git-gate subdir. + ep_path = scripts_dir / "entrypoint.sh" + ep_path.write_text( + ep_path.read_text().replace( + "/git-gate/creds/", + f"{_GIT_GATE_SCRIPTS_DIR_IN_VM}/creds/", + ) + ) - creds_dir = state_dir / "smolvm-creds" - creds_dir.mkdir(parents=True, exist_ok=True) - for u in gp.upstreams: - keypath = Path(expand_tilde(u.identity_file)) - dest_key = creds_dir / f"{u.name}-key" - shutil.copy2(str(keypath), str(dest_key)) - dest_key.chmod(0o600) - if u.known_hosts_file: - dest_kh = creds_dir / f"{u.name}-known_hosts" - shutil.copy2(str(u.known_hosts_file), str(dest_kh)) - dest_kh.chmod(0o600) + creds_dir = scripts_dir / "creds" + creds_dir.mkdir(exist_ok=True) + for u in gp.upstreams: + keypath = Path(expand_tilde(u.identity_file)) + dest_key = creds_dir / f"{u.name}-key" + shutil.copy2(str(keypath), str(dest_key)) + dest_key.chmod(0o600) + if u.known_hosts_file: + dest_kh = creds_dir / f"{u.name}-known_hosts" + shutil.copy2(str(u.known_hosts_file), str(dest_kh)) + dest_kh.chmod(0o600) - return scripts_dir, creds_dir + return staging def _bundle_launch_spec( @@ -421,34 +449,26 @@ def _bundle_launch_spec( env: list[str] = [] volumes: list[tuple[str, str, bool]] = [] - # --- egress ----------------------------------------------- - ep = plan.egress_plan - # virtiofs (smolvm) requires directory mounts — mount the CA's - # parent dir so mitmproxy-ca.pem lands at the right in-VM path. + # --- egress + git-gate (single mount) --------------------- + # Stage both into one dir and mount it at _SIDECAR_DATA_DIR_IN_VM. + # libkrun limits mounts + port-mappings to 5; with 3 daemon ports + # we can carry at most 2 mounts (this one + supervise DB). # Writable so egress_entrypoint.sh can write combined-trust.pem # and mitmproxy can create its per-host cert cache. - volumes.append(( - str(ep.mitmproxy_ca_host_path.parent), - str(Path(EGRESS_CA_IN_CONTAINER).parent), - False, - )) - if ep.routes: - volumes.append((str(ep.routes_path.parent), str(Path(EGRESS_ROUTES_IN_CONTAINER).parent), True)) + ep = plan.egress_plan + gp = plan.git_gate_plan + staging = _stage_sidecar_data(plan) + volumes.append((str(staging), _SIDECAR_DATA_DIR_IN_VM, False)) + # Tell the egress entrypoint where to find its CA + routes. + env.append(f"EGRESS_CONFDIR={_EGRESS_CONFDIR_IN_VM}") + # Always set EGRESS_ROUTES so the addon reads from the confdir path + # even when no routes were configured at launch (apply_routes_change + # writes here and a SIGHUP causes the addon to pick them up). + env.append(f"EGRESS_ROUTES={_EGRESS_CONFDIR_IN_VM}/routes.yaml") env.extend(egress_sidecar_env_entries(ep)) - # --- git-gate --------------------------------------------- - # virtiofs requires directory mounts; git-gate scripts have - # host-side names that differ from their in-VM paths. Stage - # them with correct names under the git-gate state dir and - # mount those staging dirs. The image's static wrapper at - # /git-gate-entrypoint.sh delegates to the staged entrypoint, - # so the per-bottle script is available at VM boot time. - gp = plan.git_gate_plan if gp.upstreams: daemons += ["git-gate", "git-http"] - scripts_dir, creds_dir = _stage_git_gate_for_smolvm(plan) - volumes.append((str(scripts_dir), _GIT_GATE_SCRIPTS_DIR_IN_VM, True)) - volumes.append((str(creds_dir), GIT_GATE_CREDS_DIR_IN_CONTAINER, True)) # --- supervise -------------------------------------------- sp = plan.supervise_plan diff --git a/bot_bottle/backend/smolmachines/sidecar_bundle.py b/bot_bottle/backend/smolmachines/sidecar_bundle.py index 6118014..8bb9a66 100644 --- a/bot_bottle/backend/smolmachines/sidecar_bundle.py +++ b/bot_bottle/backend/smolmachines/sidecar_bundle.py @@ -135,6 +135,16 @@ def start_bundle_vm( elif entry in effective_host_env: env[entry] = effective_host_env[entry] name = bundle_machine_name(spec.slug) + # Destroy any leftover machine from a previous run that didn't + # clean up (e.g. crash, interrupted teardown). + try: + _smolvm.machine_stop(name) + except _smolvm.SmolvmError: + pass + try: + _smolvm.machine_delete(name) + except _smolvm.SmolvmError: + pass _smolvm.machine_create( name, from_path=from_path, diff --git a/bot_bottle/egress_entrypoint.sh b/bot_bottle/egress_entrypoint.sh index fb64ff4..f2189c1 100644 --- a/bot_bottle/egress_entrypoint.sh +++ b/bot_bottle/egress_entrypoint.sh @@ -28,7 +28,7 @@ set -e # flag mitmdump would generate a fresh CA on the wrong path and # the agent's installed trust anchor would no longer match the # bumped leaf certs. -CONFDIR=/home/mitmproxy/.mitmproxy +CONFDIR="${EGRESS_CONFDIR:-/home/mitmproxy/.mitmproxy}" CONFDIR_FLAG="--set confdir=$CONFDIR" MODE="--mode regular@9099" diff --git a/tests/unit/test_smolmachines_egress_apply.py b/tests/unit/test_smolmachines_egress_apply.py index 3bb2d25..309babb 100644 --- a/tests/unit/test_smolmachines_egress_apply.py +++ b/tests/unit/test_smolmachines_egress_apply.py @@ -18,7 +18,7 @@ class TestFetchCurrentRoutes(unittest.TestCase): self.assertEqual("routes", egress_apply.fetch_current_routes("dev-abc")) exec_.assert_called_once_with( "bot-bottle-sidecars-dev-abc", - ["cat", "/etc/egress/routes.yaml"], + ["cat", "/bot-bottle-data/egress/routes.yaml"], ) def test_read_failure_raises_apply_error(self): diff --git a/tests/unit/test_smolmachines_provision.py b/tests/unit/test_smolmachines_provision.py index 959fc87..f8a2c15 100644 --- a/tests/unit/test_smolmachines_provision.py +++ b/tests/unit/test_smolmachines_provision.py @@ -405,8 +405,8 @@ class TestBundleLaunchSpec(unittest.TestCase): ) with patch( - "bot_bottle.backend.smolmachines.launch._stage_git_gate_for_smolvm", - return_value=(Path("/tmp/smolvm-scripts"), Path("/tmp/smolvm-creds")), + "bot_bottle.backend.smolmachines.launch._stage_sidecar_data", + return_value=Path("/tmp/smolvm-sidecar-data"), ): spec = _bundle_launch_spec(plan, "net", "127.0.0.16") @@ -420,7 +420,11 @@ class TestBundleLaunchSpec(unittest.TestCase): def test_canary_env_registered_as_sensitive_in_bundle(self): plan = _plan(canary=True) - spec = _bundle_launch_spec(plan, "net", "127.0.0.16") + with patch( + "bot_bottle.backend.smolmachines.launch._stage_sidecar_data", + return_value=Path("/tmp/smolvm-sidecar-data"), + ): + spec = _bundle_launch_spec(plan, "net", "127.0.0.16") self.assertIn("CANON_ALPHA_SECRET=fake-canary-value", spec.environment) self.assertIn( @@ -431,7 +435,11 @@ class TestBundleLaunchSpec(unittest.TestCase): def test_supervise_adds_daemon_volume_and_env(self): from bot_bottle.supervise import DB_PATH_IN_CONTAINER plan = _plan(supervise=True) - spec = _bundle_launch_spec(plan, "net", "127.0.0.16") + with patch( + "bot_bottle.backend.smolmachines.launch._stage_sidecar_data", + return_value=Path("/tmp/smolvm-sidecar-data"), + ): + spec = _bundle_launch_spec(plan, "net", "127.0.0.16") self.assertIn("supervise", spec.daemons_csv) self.assertIn(f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", spec.environment) # virtiofs requires directory mounts; the DB's parent dir is @@ -562,8 +570,8 @@ class TestLaunchResourceWiring(unittest.TestCase): "bot_bottle.backend.smolmachines.launch._bundle.start_bundle_vm", return_value=raw_launch, ) as start_vm, patch( - "bot_bottle.backend.smolmachines.launch._stage_git_gate_for_smolvm", - return_value=(Path("/tmp/smolvm-scripts"), Path("/tmp/smolvm-creds")), + "bot_bottle.backend.smolmachines.launch._stage_sidecar_data", + return_value=Path("/tmp/smolvm-sidecar-data"), ), patch( "bot_bottle.backend.smolmachines.launch._forward.start_forwarder", return_value=handle,