From 4863e81cd398454ba88845fc512c119682a59fb1 Mon Sep 17 00:00:00 2001 From: codex Date: Thu, 9 Jul 2026 15:45:51 +0000 Subject: [PATCH 1/9] docs: draft smolmachines sidecar vm prd --- docs/prds/prd-new-smolmachines-sidecar-vm.md | 104 +++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 docs/prds/prd-new-smolmachines-sidecar-vm.md diff --git a/docs/prds/prd-new-smolmachines-sidecar-vm.md b/docs/prds/prd-new-smolmachines-sidecar-vm.md new file mode 100644 index 0000000..1c980e3 --- /dev/null +++ b/docs/prds/prd-new-smolmachines-sidecar-vm.md @@ -0,0 +1,104 @@ +# PRD prd-new: smolmachines sidecar VM + +- **Status:** Draft +- **Author:** codex +- **Created:** 2026-07-09 +- **Issue:** #332 + +## Summary + +Run the smolmachines backend's trusted sidecar bundle as its own smolVM instead +of a Docker container. A bottle then consists of an agent VM plus a sidecar VM, +with the agent VM's TSI allowlist limited to the per-bottle agent-facing +sidecar surface. + +## Problem + +The smolmachines backend currently runs the agent in smolVM but keeps +egress/git-gate/supervise in a Docker sidecar bundle. That hybrid launch path +keeps Docker in the trusted runtime path and leaves smolmachines with a +different isolation boundary than its agent VM design suggests. + +The existing Docker sidecar path also uses Docker port publishing to bind only +agent-facing services to the per-bottle host address. TSI is IP-only, so the +smolVM replacement must preserve that scoping: publishing sidecar services on +generic host localhost would let the agent reach unrelated host services or +other bottle sidecars if those services share the allowed address. + +## Goals / Success Criteria + +1. `BOT_BOTTLE_BACKEND=smolmachines ./cli.py start ` runs with both the + agent and sidecar as smolVMs. +2. The smolmachines launch path no longer uses `docker run` for the sidecar + bundle. +3. Agent-facing egress, git-gate, and supervise endpoints are exposed only + through the bottle's own published sidecar surface. +4. Internal sidecar-only ports are not reachable from the agent VM. +5. The agent VM TSI allowlist remains fail-closed and limited to the per-bottle + address. +6. Sidecar config, secrets, and state are delivered to the sidecar VM without + exposing provider or forge credentials to the agent VM. +7. Teardown removes both VMs and any host-side published port state. + +## Non-goals + +- Rewriting the egress, git-gate, or supervise daemons. +- Changing the Docker backend's sidecar bundle behavior. +- Replacing the existing agent image build and pack pipeline except where shared + helper extraction is needed for sidecar image packing. +- Weakening the current smolmachines TSI allowlist checks. + +## Design + +The sidecar VM should continue to use the existing sidecar bundle image and +`/app/sidecar_init.py` supervisor. The launch flow builds the sidecar image, +packs it into a `.smolmachine` artifact, creates a per-bottle sidecar VM from +that artifact, passes the same daemon-selection environment the Docker bundle +uses today, mounts or copies the same daemon-private config/state inputs, starts +the sidecar VM, discovers the host-side published ports, then starts the agent +VM with `--allow-cidr /32`. + +Agent-facing URLs are stamped from the sidecar VM's published ports: + +- `HTTP_PROXY` / `HTTPS_PROXY` point at the published egress proxy port. +- `GIT_GATE_URL` points at the published git HTTP port when git-gate is enabled. +- `MCP_SUPERVISE_URL` points at the published supervise port when supervise is + enabled. + +Internal sidecar-only services stay bound inside the sidecar VM and are not +published. + +## Open blocker + +The implementation depends on host-side port publishing that can bind to the +per-bottle address. Current smolVM documentation and upstream source shape show +`-p/--port` accepting `HOST:GUEST`, with `HOST` represented as a port number, +not an IP address plus port. If smolVM cannot bind a guest port to a specific +host address such as `127.0.0.16:`, bot-bottle needs one of these before +implementation can proceed: + +1. smolVM support for address-scoped port publishing; +2. a host-side forwarding layer that binds only the per-bottle address and + forwards into the sidecar VM; or +3. a different sidecar VM networking design that preserves the same IP-only TSI + isolation property. + +Publishing sidecar ports on generic host localhost is not acceptable because +TSI's allowlist is IP-only and would no longer prove that the agent can reach +only its own sidecar surface. + +## Implementation chunks + +1. Confirm or add an address-scoped sidecar VM port publishing mechanism. +2. Extract the existing image-to-smolmachine pack helper so agent and sidecar + artifacts share the same cache and registry path. +3. Add a sidecar VM launch spec and lifecycle helpers: pack, create, start, + publish/discover ports, stop, delete. +4. Switch smolmachines launch from Docker sidecar bundle lifecycle to sidecar VM + lifecycle. +5. Update egress apply / reload paths for the sidecar VM supervisor. +6. Add unit tests for argv shape, URL stamping, teardown, and fail-closed + behavior. +7. Add or update integration coverage for agent-to-sidecar reachability, host + localhost denial, other-bottle alias denial, internal port denial, and + teardown cleanup. -- 2.52.0 From 869a8bbc1f9f591e3bc7417383f77b5d8ec5a774 Mon Sep 17 00:00:00 2001 From: codex Date: Thu, 9 Jul 2026 16:10:42 +0000 Subject: [PATCH 2/9] feat: run smolmachines sidecar as vm --- .../backend/smolmachines/egress_apply.py | 49 +++- bot_bottle/backend/smolmachines/enumerate.py | 7 +- bot_bottle/backend/smolmachines/launch.py | 121 +++++---- .../backend/smolmachines/port_forward.py | 245 ++++++++++++++++++ .../backend/smolmachines/sidecar_bundle.py | 77 ++++++ bot_bottle/backend/smolmachines/smolvm.py | 16 +- docs/prds/prd-new-smolmachines-sidecar-vm.md | 61 +++-- tests/unit/test_smolmachines_egress_apply.py | 58 +++++ tests/unit/test_smolmachines_port_forward.py | 98 +++++++ tests/unit/test_smolmachines_provision.py | 39 ++- .../unit/test_smolmachines_sidecar_bundle.py | 65 +++++ tests/unit/test_smolmachines_smolvm.py | 16 ++ 12 files changed, 739 insertions(+), 113 deletions(-) create mode 100644 bot_bottle/backend/smolmachines/port_forward.py create mode 100644 tests/unit/test_smolmachines_egress_apply.py create mode 100644 tests/unit/test_smolmachines_port_forward.py diff --git a/bot_bottle/backend/smolmachines/egress_apply.py b/bot_bottle/backend/smolmachines/egress_apply.py index 559aa3b..661a13f 100644 --- a/bot_bottle/backend/smolmachines/egress_apply.py +++ b/bot_bottle/backend/smolmachines/egress_apply.py @@ -1,20 +1,51 @@ """Egress apply for the smolmachines backend. -The smolmachines sidecar bundle runs as a host-side Docker container, -so egress signalling is identical to the docker backend. +The smolmachines sidecar bundle runs as a sidecar smolVM. Route-file +inspection and reload signalling therefore go through ``smolvm machine +exec`` instead of Docker. """ from __future__ import annotations -from ..docker.egress_apply import ( # noqa: F401 - DockerEgressApplicator, - EgressApplyError, - applicator, - fetch_current_routes, -) +from ...egress import EGRESS_ROUTES_IN_CONTAINER +from ...log import warn +from ..egress_apply import EgressApplicator, EgressApplyError +from . import sidecar_bundle as _bundle +from . import smolvm as _smolvm + + +def fetch_current_routes(slug: str) -> str: + machine = _bundle.bundle_machine_name(slug) + result = _smolvm.machine_exec(machine, ["cat", EGRESS_ROUTES_IN_CONTAINER]) + if result.returncode != 0: + raise EgressApplyError( + f"could not read routes.yaml from {machine}: " + f"{(result.stderr or '').strip() or 'sidecar VM not running?'}" + ) + return result.stdout + + +class SmolmachinesEgressApplicator(EgressApplicator): + def _signal_bundle_reload(self, slug: str) -> None: + machine = _bundle.bundle_machine_name(slug) + result = _smolvm.machine_exec(machine, ["sh", "-c", "kill -HUP 1"]) + if result.returncode != 0: + last_error = (result.stderr or "").strip() or (result.stdout or "").strip() + warn( + f"egress: routes updated on disk for {slug}, but bundle reload failed: " + f"{last_error or 'smolvm exec failed'}" + ) + raise EgressApplyError( + f"could not reload egress bundle {machine}: " + f"{last_error or 'smolvm exec failed'}" + ) + + +applicator = SmolmachinesEgressApplicator() + __all__ = [ - "DockerEgressApplicator", + "SmolmachinesEgressApplicator", "EgressApplyError", "applicator", "fetch_current_routes", diff --git a/bot_bottle/backend/smolmachines/enumerate.py b/bot_bottle/backend/smolmachines/enumerate.py index cd24ec4..7ce0f47 100644 --- a/bot_bottle/backend/smolmachines/enumerate.py +++ b/bot_bottle/backend/smolmachines/enumerate.py @@ -31,6 +31,7 @@ from . import sidecar_bundle as _bundle # matching the bundle container name pattern. We use the prefix # both as a filter and to strip back to the slug. _VM_NAME_PREFIX = "bot-bottle-" +_SIDECAR_VM_PREFIX = _bundle.bundle_machine_name("") def enumerate_active() -> list[ActiveAgent]: @@ -54,7 +55,11 @@ def enumerate_active() -> list[ActiveAgent]: for m in machines: name = m.get("name") or "" state = m.get("state") or "" - if state != "running" or not name.startswith(_VM_NAME_PREFIX): + if ( + state != "running" + or not name.startswith(_VM_NAME_PREFIX) + or name.startswith(_SIDECAR_VM_PREFIX) + ): continue slug = name[len(_VM_NAME_PREFIX):] metadata = read_metadata(slug) diff --git a/bot_bottle/backend/smolmachines/launch.py b/bot_bottle/backend/smolmachines/launch.py index 56e5929..d1ce279 100644 --- a/bot_bottle/backend/smolmachines/launch.py +++ b/bot_bottle/backend/smolmachines/launch.py @@ -17,7 +17,6 @@ from __future__ import annotations import dataclasses import os -import platform from contextlib import ExitStack, contextmanager from pathlib import Path from typing import Callable, Generator @@ -53,6 +52,7 @@ from ...bottle_state import ( read_committed_image, ) from . import loopback_alias as _loopback +from . import port_forward as _forward from . import sidecar_bundle as _bundle from . import smolvm as _smolvm from .bottle import SmolmachinesBottle @@ -94,7 +94,6 @@ def launch( plan = _mint_certs(plan) proxy_host = _proxy_host(plan, loopback_ip) plan = _start_bundle(plan, network, proxy_host, stack) - plan = _discover_urls(plan, proxy_host) agent_from_path = _agent_from_path(plan) @@ -144,19 +143,17 @@ def _allocate_resources( plan: SmolmachinesBottlePlan, stack: ExitStack, ) -> tuple[str, str]: - """Reserve a loopback alias and create the per-bottle docker bridge. + """Reserve a per-bottle host address. - The per-bottle alias scopes TSI's allowlist to this bottle's - published ports so the agent can't reach other bottles' or host - services' ports on loopback. On macOS `ensure_pool` first - sudo-aliases the pool on `lo0`; on Linux that's a no-op since - all of 127.0.0.0/8 is already loopback, but the per-bottle - allocation runs on both.""" + The per-bottle address scopes TSI's allowlist to this bottle's + forwarder-published ports so the agent can't reach other bottles' + or host services. The returned network name remains in the bundle + spec for compatibility with older helper tests; no Docker sidecar + container is launched.""" + del stack _loopback.ensure_pool() loopback_ip = _loopback.allocate(plan.slug) network = _bundle.bundle_network_name(plan.slug) - _bundle.create_bundle_network(network, plan.bundle_subnet, plan.bundle_gateway) - stack.callback(_bundle.remove_bundle_network, network) return loopback_ip, network @@ -179,15 +176,40 @@ def _start_bundle( proxy_host: str, stack: ExitStack, ) -> SmolmachinesBottlePlan: - """Build the BundleLaunchSpec, resolve token env, start the - sidecar bundle container, and register teardown.""" + """Build the BundleLaunchSpec, start the sidecar VM, wrap its raw + smolVM-published loopback ports with per-bottle forwarders, stamp + agent URLs from those forwarder ports, and register teardown.""" plan = _provision_git_gate_keys(plan) bundle_spec = _bundle_launch_spec(plan, network, proxy_host) token_env = _resolve_token_env(plan, dict(os.environ)) - _bundle.ensure_bundle_image(bundle_spec.image) - _bundle.start_bundle(bundle_spec, env={**os.environ, **token_env}) - stack.callback(_bundle.stop_bundle, plan.slug) - return plan + artifact = _ensure_smolmachine( + bundle_spec.image, + dockerfile=_bundle.SIDECAR_BUNDLE_DOCKERFILE, + ) + launch = _bundle.start_bundle_vm( + bundle_spec, + from_path=artifact, + host_env={**os.environ, **token_env}, + ) + stack.callback(_bundle.stop_bundle_vm, plan.slug) + + forward_specs = tuple( + _forward.ForwardSpec( + label=_label_for_port(container_port), + listen_host=proxy_host, + listen_port=0, + target_host="127.0.0.1", + target_port=host_port, + ) + for container_port, host_port in launch.raw_ports.items() + ) + handle = _forward.start_forwarder(forward_specs) + stack.callback(_forward.stop_forwarder, handle) + published_ports = { + _port_for_label(spec.label): spec.listen_port + for spec in handle.forwards + } + return _discover_urls(plan, proxy_host, published_ports) def _provision_git_gate_keys( @@ -206,35 +228,27 @@ def _provision_git_gate_keys( def _discover_urls( plan: SmolmachinesBottlePlan, proxy_host: str, + published_ports: dict[int, int], ) -> SmolmachinesBottlePlan: - """Discover host-side ports for published container ports and - return the plan with URLs + guest_env stamped in. + """Stamp URLs + guest_env from per-bottle forwarder ports. - `proxy_host` is the host IP that both TSI's allowlist and - docker's port-forward bindings are keyed to. On macOS it is the - per-bottle loopback alias; on Linux it is the per-bottle bridge - gateway (see `_proxy_host`). The agent dials the published port - on this IP for all bundle-hosted services. + `proxy_host` is the host IP that both TSI's allowlist and the + forwarder listeners are keyed to. The raw smolVM-published ports + are intentionally not advertised to the agent. NO_PROXY includes `proxy_host` so supervise + git-gate URLs bypass HTTPS_PROXY.""" - agent_facing_host_port = _bundle.bundle_host_port( - plan.slug, _EGRESS_PORT, host_ip=proxy_host, - ) + agent_facing_host_port = published_ports[_EGRESS_PORT] agent_proxy_url = f"http://{proxy_host}:{agent_facing_host_port}" agent_git_gate_host = "" if plan.git_gate_plan.upstreams: - git_gate_host_port = _bundle.bundle_host_port( - plan.slug, _GIT_HTTP_PORT, host_ip=proxy_host, - ) + git_gate_host_port = published_ports[_GIT_HTTP_PORT] agent_git_gate_host = f"{proxy_host}:{git_gate_host_port}" agent_supervise_url = "" if plan.supervise_plan is not None: - supervise_host_port = _bundle.bundle_host_port( - plan.slug, _SUPERVISE_PORT, host_ip=proxy_host, - ) + supervise_host_port = published_ports[_SUPERVISE_PORT] agent_supervise_url = f"http://{proxy_host}:{supervise_host_port}/" existing_no_proxy = plan.guest_env.get("NO_PROXY", "localhost,127.0.0.1") @@ -328,26 +342,33 @@ def _init_vm(plan: SmolmachinesBottlePlan) -> None: def _proxy_host(plan: SmolmachinesBottlePlan, loopback_ip: str) -> str: - """Return the host IP for TSI's allowlist and docker port-forward bindings. - - On macOS, the per-bottle loopback alias (e.g. ``127.0.0.16``) works - because macOS's network stack lets TSI intercept 127.x.x.x connects - from the guest before they reach the host's own loopback. - - On Linux, the guest kernel's LOCAL routing table routes all - ``127.0.0.0/8`` to the guest's own loopback — those packets never - reach eth0 and TSI never sees them. Using the per-bottle bridge - gateway (e.g. ``192.168.N.1``) instead sidesteps the problem: it - is not a loopback address, so the guest routes it via eth0 and TSI - intercepts it normally. The TSI allowlist is ``gateway/32``, which - is distinct from the container IP (``192.168.N.2``), so the agent - still can't reach the egress daemon directly — TSI blocks any - connection to the container IP that isn't via the published port.""" - if platform.system() == "Linux": - return plan.bundle_gateway + """Return the per-bottle host address used for TSI and forwarders.""" + del plan return loopback_ip +def _label_for_port(port: int) -> str: + if port == _EGRESS_PORT: + return "egress" + if port == _GIT_HTTP_PORT: + return "git-http" + if port == _SUPERVISE_PORT: + return "supervise" + return f"port-{port}" + + +def _port_for_label(label: str) -> int: + if label == "egress": + return _EGRESS_PORT + if label == "git-http": + return _GIT_HTTP_PORT + if label == "supervise": + return _SUPERVISE_PORT + if label.startswith("port-"): + return int(label.removeprefix("port-")) + raise ValueError(f"unknown sidecar forward label: {label}") + + def _bundle_launch_spec( plan: SmolmachinesBottlePlan, network: str, proxy_host: str, ) -> _bundle.BundleLaunchSpec: diff --git a/bot_bottle/backend/smolmachines/port_forward.py b/bot_bottle/backend/smolmachines/port_forward.py new file mode 100644 index 0000000..7834834 --- /dev/null +++ b/bot_bottle/backend/smolmachines/port_forward.py @@ -0,0 +1,245 @@ +"""Per-bottle TCP forwarder for smolmachines sidecar VMs. + +smolVM currently publishes guest ports on host loopback as +``HOST:GUEST`` port pairs, without an address-scoped bind. The agent +VM's TSI allowlist is IP-only, so bot-bottle exposes a second, stricter +surface: listeners bound only to the bottle's allocated host address, +forwarding to the raw smolVM-published loopback ports. +""" + +from __future__ import annotations + +import argparse +import json +import os +import selectors +import signal +import socket +import subprocess +import sys +import threading +from dataclasses import asdict, dataclass +from typing import Any, Iterable, Sequence + +from ...log import die, warn + + +_BUFFER_SIZE = 64 * 1024 +_LOOPBACK_TARGETS = {"127.0.0.1", "::1"} +_FORBIDDEN_LISTEN_HOSTS = {"", "0.0.0.0", "::", "127.0.0.1", "::1"} + + +@dataclass(frozen=True) +class ForwardSpec: + label: str + listen_host: str + listen_port: int + target_host: str + target_port: int + + +@dataclass(frozen=True) +class ForwarderHandle: + process: Any + forwards: tuple[ForwardSpec, ...] + + +def start_forwarder(specs: Sequence[ForwardSpec]) -> ForwarderHandle: + """Start one helper process for a bottle and wait until all + listeners are bound. ``listen_port`` may be 0; the returned specs + contain the kernel-assigned ports printed by the helper.""" + if not specs: + return ForwarderHandle(process=_CompletedProcess(), forwards=()) + _validate_specs(specs) + argv = [ + sys.executable, + "-c", + "from bot_bottle.backend.smolmachines.port_forward import main; " + "raise SystemExit(main())", + "--spec-json", + json.dumps([asdict(s) for s in specs], separators=(",", ":")), + ] + proc = subprocess.Popen( + argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=None, + text=True, + env=os.environ, + ) + assert proc.stdout is not None + line = proc.stdout.readline() + proc.stdout.close() + if not line: + proc.terminate() + proc.wait(timeout=5) + die("smolmachines forwarder failed before reporting readiness") + try: + payload = json.loads(line) + bound = tuple(ForwardSpec(**item) for item in payload["forwards"]) + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: + proc.terminate() + proc.wait(timeout=5) + die(f"smolmachines forwarder returned invalid readiness payload: {exc}") + return ForwarderHandle(process=proc, forwards=bound) + + +def stop_forwarder(handle: ForwarderHandle) -> None: + proc = handle.process + if proc.poll() is not None: + return + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + + +def _validate_specs(specs: Iterable[ForwardSpec]) -> None: + for spec in specs: + if spec.listen_host in _FORBIDDEN_LISTEN_HOSTS: + die(f"refusing unsafe smolmachines forwarder bind: {spec.listen_host!r}") + if spec.target_host not in _LOOPBACK_TARGETS: + die(f"refusing non-loopback smolmachines forwarder target: {spec.target_host!r}") + if not 0 <= spec.listen_port <= 65535: + die(f"invalid smolmachines forwarder listen port: {spec.listen_port}") + if not 1 <= spec.target_port <= 65535: + die(f"invalid smolmachines forwarder target port: {spec.target_port}") + + +class _CompletedProcess: + """Popen-like no-op used when a bottle has no forwards.""" + + def poll(self) -> int: + return 0 + + def terminate(self) -> None: + return None + + def kill(self) -> None: + return None + + def wait(self, timeout: float | None = None) -> int: + del timeout + return 0 + + +class _Forwarder: + def __init__(self, specs: Sequence[ForwardSpec]): + _validate_specs(specs) + self.specs = tuple(specs) + self.selector = selectors.DefaultSelector() + self.stop_event = threading.Event() + self.listeners: list[tuple[socket.socket, ForwardSpec]] = [] + self.bound: list[ForwardSpec] = [] + + def start(self) -> None: + for spec in self.specs: + listener = socket.create_server( + (spec.listen_host, spec.listen_port), + reuse_port=False, + backlog=64, + ) + listener.setblocking(False) + host, port = listener.getsockname()[:2] + bound = ForwardSpec( + label=spec.label, + listen_host=str(host), + listen_port=int(port), + target_host=spec.target_host, + target_port=spec.target_port, + ) + self.listeners.append((listener, bound)) + self.bound.append(bound) + self.selector.register(listener, selectors.EVENT_READ, bound) + + def serve(self) -> None: + while not self.stop_event.is_set(): + try: + events = self.selector.select(timeout=0.25) + except OSError: + return + for key, _ in events: + listener = key.fileobj + if not isinstance(listener, socket.socket): + continue + spec = key.data + try: + client, _ = listener.accept() + except OSError: + continue + threading.Thread( + target=self._handle_client, + args=(client, spec), + daemon=True, + ).start() + + def stop(self) -> None: + self.stop_event.set() + for listener, _ in self.listeners: + try: + self.selector.unregister(listener) + except Exception: # noqa: BLE001 - best effort shutdown + pass + listener.close() + self.selector.close() + + def _handle_client(self, client: socket.socket, spec: ForwardSpec) -> None: + with client: + try: + target = socket.create_connection((spec.target_host, spec.target_port)) + except OSError as exc: + warn(f"smolmachines forwarder {spec.label}: target connect failed: {exc}") + return + with target: + left = threading.Thread(target=_pipe, args=(client, target), daemon=True) + right = threading.Thread(target=_pipe, args=(target, client), daemon=True) + left.start() + right.start() + left.join() + right.join() + + +def _pipe(src: socket.socket, dst: socket.socket) -> None: + while True: + try: + data = src.recv(_BUFFER_SIZE) + except OSError: + break + if not data: + break + try: + dst.sendall(data) + except OSError: + break + try: + dst.shutdown(socket.SHUT_WR) + except OSError: + pass + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--spec-json", required=True) + ns = parser.parse_args(argv) + specs = tuple(ForwardSpec(**item) for item in json.loads(ns.spec_json)) + forwarder = _Forwarder(specs) + + def request_stop(signum: int, _frame: object) -> None: + del signum + forwarder.stop() + + signal.signal(signal.SIGTERM, request_stop) + signal.signal(signal.SIGINT, request_stop) + forwarder.start() + print(json.dumps({"forwards": [asdict(s) for s in forwarder.bound]}), flush=True) + try: + forwarder.serve() + finally: + forwarder.stop() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bot_bottle/backend/smolmachines/sidecar_bundle.py b/bot_bottle/backend/smolmachines/sidecar_bundle.py index 60b3413..59916b1 100644 --- a/bot_bottle/backend/smolmachines/sidecar_bundle.py +++ b/bot_bottle/backend/smolmachines/sidecar_bundle.py @@ -23,6 +23,8 @@ Plans (EgressPlan, …) lands in chunk 2d.""" from __future__ import annotations +import os +import socket import subprocess from dataclasses import dataclass, field from pathlib import Path @@ -34,6 +36,7 @@ from ..docker.sidecar_bundle import ( SIDECAR_BUNDLE_DOCKERFILE, SIDECAR_BUNDLE_IMAGE, ) +from . import smolvm as _smolvm _REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent) @@ -54,6 +57,11 @@ def bundle_container_name(slug: str) -> str: return f"bot-bottle-sidecars-{slug}" +def bundle_machine_name(slug: str) -> str: + """Sidecar smolVM machine name.""" + return bundle_container_name(slug) + + @dataclass(frozen=True) class BundleLaunchSpec: """Everything `start_bundle` needs to bring up one bundle @@ -92,6 +100,11 @@ class BundleLaunchSpec: publish_host_ip: str = "127.0.0.1" +@dataclass(frozen=True) +class BundleVmLaunch: + raw_ports: dict[int, int] + + def ensure_bundle_image(image: str = SIDECAR_BUNDLE_IMAGE) -> None: """Build the sidecar bundle image before `docker run`. @@ -107,6 +120,70 @@ def ensure_bundle_image(image: str = SIDECAR_BUNDLE_IMAGE) -> None: ) +def allocate_raw_host_ports(container_ports: Sequence[int]) -> dict[int, int]: + """Reserve candidate host loopback ports for smolVM `-p HOST:GUEST`. + + The sockets are closed before smolVM binds them, so this remains + best-effort. smolVM failing to bind a selected port is fatal in + the caller's launch path.""" + out: dict[int, int] = {} + for container_port in container_ports: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + out[container_port] = int(sock.getsockname()[1]) + return out + + +def start_bundle_vm( + spec: BundleLaunchSpec, + *, + from_path: Path, + host_env: dict[str, str] | None = None, +) -> BundleVmLaunch: + """Create and start the sidecar bundle as a smolVM. + + smolVM's own published ports are raw host-loopback ports. The + launch flow wraps them with per-bottle address-bound forwarders + before exposing anything to the agent VM.""" + raw_ports = allocate_raw_host_ports(spec.ports_to_publish) + effective_host_env = host_env if host_env is not None else os.environ + env: dict[str, str] = {"BOT_BOTTLE_SIDECAR_DAEMONS": spec.daemons_csv} + for entry in spec.environment: + name, sep, value = entry.partition("=") + if sep: + env[name] = value + elif entry in effective_host_env: + env[entry] = effective_host_env[entry] + name = bundle_machine_name(spec.slug) + _smolvm.machine_create( + name, + from_path=from_path, + net=True, + env=env, + volumes=spec.volumes, + ports=tuple( + (host_port, guest_port) + for guest_port, host_port in raw_ports.items() + ), + ) + _smolvm.machine_start(name) + _smolvm.wait_exec_ready(name) + return BundleVmLaunch(raw_ports=raw_ports) + + +def stop_bundle_vm(slug: str) -> None: + """Best-effort sidecar VM teardown.""" + name = bundle_machine_name(slug) + try: + _smolvm.machine_stop(name) + except _smolvm.SmolvmError as exc: + warn(f"smolvm machine stop {name} failed: {exc}") + try: + _smolvm.machine_delete(name) + except _smolvm.SmolvmError as exc: + warn(f"smolvm machine delete {name} failed: {exc}") + + def create_bundle_network(network_name: str, subnet: str, gateway: str) -> None: """`docker network create` with an explicit subnet + gateway so the bundle's `--ip` lands on the address the Smolfile's diff --git a/bot_bottle/backend/smolmachines/smolvm.py b/bot_bottle/backend/smolmachines/smolvm.py index 41fa9bd..57593e5 100644 --- a/bot_bottle/backend/smolmachines/smolvm.py +++ b/bot_bottle/backend/smolmachines/smolvm.py @@ -113,8 +113,11 @@ def machine_create( *, image: str | None = None, from_path: Path | None = None, + net: bool = False, allow_cidrs: Sequence[str] = (), env: Mapping[str, str] | None = None, + ports: Sequence[tuple[int, int]] = (), + volumes: Sequence[tuple[str, str, bool]] = (), ) -> None: """`smolvm machine create --name NAME [--image IMG | --from PATH] [--allow-cidr CIDR ...] [-e K=V ...]`. NAME is passed as @@ -131,19 +134,24 @@ def machine_create( no-pull-at-start property. The flag form gives the same result without the Smolfile complication. - `--net` is sent explicitly when `allow_cidrs` is non-empty. - `--allow-cidr` implies `--net` per the CLI help, but sending - `--net` explicitly is harmless and ensures the guest has + `--net` is sent explicitly when `net=True` or `allow_cidrs` is + non-empty. `--allow-cidr` implies `--net` per the CLI help, but + sending `--net` explicitly is harmless and ensures the guest has network access even if that implication changes across versions.""" args: list[str] = ["machine", "create", "--name", name] if image is not None: args += ["--image", image] if from_path is not None: args += ["--from", str(from_path)] - if allow_cidrs: + if net or allow_cidrs: args.append("--net") for cidr in allow_cidrs: args += ["--allow-cidr", cidr] + for host_port, guest_port in ports: + args += ["-p", f"{host_port}:{guest_port}"] + for host_path, guest_path, read_only in volumes: + suffix = ":ro" if read_only else "" + args += ["-v", f"{host_path}:{guest_path}{suffix}"] if env: for k, v in env.items(): args += ["-e", f"{k}={v}"] diff --git a/docs/prds/prd-new-smolmachines-sidecar-vm.md b/docs/prds/prd-new-smolmachines-sidecar-vm.md index 1c980e3..440a1ed 100644 --- a/docs/prds/prd-new-smolmachines-sidecar-vm.md +++ b/docs/prds/prd-new-smolmachines-sidecar-vm.md @@ -1,6 +1,6 @@ # PRD prd-new: smolmachines sidecar VM -- **Status:** Draft +- **Status:** Active - **Author:** codex - **Created:** 2026-07-09 - **Issue:** #332 @@ -50,15 +50,31 @@ other bottle sidecars if those services share the allowed address. ## Design -The sidecar VM should continue to use the existing sidecar bundle image and +The sidecar VM continues to use the existing sidecar bundle image and `/app/sidecar_init.py` supervisor. The launch flow builds the sidecar image, packs it into a `.smolmachine` artifact, creates a per-bottle sidecar VM from that artifact, passes the same daemon-selection environment the Docker bundle uses today, mounts or copies the same daemon-private config/state inputs, starts -the sidecar VM, discovers the host-side published ports, then starts the agent -VM with `--allow-cidr /32`. +the sidecar VM, wraps the sidecar VM's raw smolVM-published ports with a local +per-bottle forwarder, then starts the agent VM with +`--allow-cidr /32`. -Agent-facing URLs are stamped from the sidecar VM's published ports: +smolVM currently exposes guest ports as `HOST:GUEST` port pairs bound on host +loopback. It does not expose an address-scoped bind like +`127.0.0.16::9099`. Because TSI is IP-only, bot-bottle must not advertise +those raw loopback ports directly to the agent. Instead, bot-bottle runs one +small stdlib Python TCP forwarder process per bottle: + +```text +agent VM + -> TSI allows only /32 + -> bot-bottle forwarder bound to : + -> raw smolVM-published sidecar port on 127.0.0.1: + -> sidecar VM service +``` + +Agent-facing URLs are stamped from the forwarder ports, never the raw smolVM +ports: - `HTTP_PROXY` / `HTTPS_PROXY` point at the published egress proxy port. - `GIT_GATE_URL` points at the published git HTTP port when git-gate is enabled. @@ -68,34 +84,29 @@ Agent-facing URLs are stamped from the sidecar VM's published ports: Internal sidecar-only services stay bound inside the sidecar VM and are not published. -## Open blocker +### Forwarder constraints -The implementation depends on host-side port publishing that can bind to the -per-bottle address. Current smolVM documentation and upstream source shape show -`-p/--port` accepting `HOST:GUEST`, with `HOST` represented as a port number, -not an IP address plus port. If smolVM cannot bind a guest port to a specific -host address such as `127.0.0.16:`, bot-bottle needs one of these before -implementation can proceed: - -1. smolVM support for address-scoped port publishing; -2. a host-side forwarding layer that binds only the per-bottle address and - forwards into the sidecar VM; or -3. a different sidecar VM networking design that preserves the same IP-only TSI - isolation property. - -Publishing sidecar ports on generic host localhost is not acceptable because -TSI's allowlist is IP-only and would no longer prove that the agent can reach -only its own sidecar surface. +- Reject wildcard listener binds (`0.0.0.0`, `::`) and generic localhost + listener binds (`127.0.0.1`, `::1`) for agent-facing forwards. +- Accept only fixed loopback targets selected by launch; the client cannot + choose or influence the forwarding target. +- Forward bytes transparently without parsing HTTP, CONNECT, TLS, Git, or MCP. +- Log lifecycle and endpoint metadata only; never log payload bytes. +- Create forwards only for egress, git HTTP when enabled, and supervise when + enabled. +- Fail closed if any listener cannot bind exactly the requested per-bottle + address. ## Implementation chunks -1. Confirm or add an address-scoped sidecar VM port publishing mechanism. -2. Extract the existing image-to-smolmachine pack helper so agent and sidecar +1. Extract the existing image-to-smolmachine pack helper so agent and sidecar artifacts share the same cache and registry path. +2. Add a strict stdlib host TCP forwarder for per-bottle address-bound + forwarding to raw smolVM-published sidecar ports. 3. Add a sidecar VM launch spec and lifecycle helpers: pack, create, start, publish/discover ports, stop, delete. 4. Switch smolmachines launch from Docker sidecar bundle lifecycle to sidecar VM - lifecycle. + plus forwarder lifecycle. 5. Update egress apply / reload paths for the sidecar VM supervisor. 6. Add unit tests for argv shape, URL stamping, teardown, and fail-closed behavior. diff --git a/tests/unit/test_smolmachines_egress_apply.py b/tests/unit/test_smolmachines_egress_apply.py new file mode 100644 index 0000000..3bb2d25 --- /dev/null +++ b/tests/unit/test_smolmachines_egress_apply.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from bot_bottle.backend.egress_apply import EgressApplyError +from bot_bottle.backend.smolmachines import egress_apply +from bot_bottle.backend.smolmachines.smolvm import SmolvmRunResult + + +class TestFetchCurrentRoutes(unittest.TestCase): + def test_reads_routes_from_sidecar_vm(self): + with patch.object( + egress_apply._smolvm, + "machine_exec", + return_value=SmolvmRunResult(0, "routes", ""), + ) as exec_: + self.assertEqual("routes", egress_apply.fetch_current_routes("dev-abc")) + exec_.assert_called_once_with( + "bot-bottle-sidecars-dev-abc", + ["cat", "/etc/egress/routes.yaml"], + ) + + def test_read_failure_raises_apply_error(self): + with patch.object( + egress_apply._smolvm, + "machine_exec", + return_value=SmolvmRunResult(1, "", "nope"), + ): + with self.assertRaises(EgressApplyError): + egress_apply.fetch_current_routes("dev-abc") + + +class TestReload(unittest.TestCase): + def test_sends_hup_to_sidecar_init(self): + with patch.object( + egress_apply._smolvm, + "machine_exec", + return_value=SmolvmRunResult(0, "", ""), + ) as exec_: + egress_apply.SmolmachinesEgressApplicator()._signal_bundle_reload("dev-abc") + exec_.assert_called_once_with( + "bot-bottle-sidecars-dev-abc", + ["sh", "-c", "kill -HUP 1"], + ) + + def test_reload_failure_raises_apply_error(self): + with patch.object( + egress_apply._smolvm, + "machine_exec", + return_value=SmolvmRunResult(1, "", "nope"), + ): + with self.assertRaises(EgressApplyError): + egress_apply.SmolmachinesEgressApplicator()._signal_bundle_reload("dev-abc") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_smolmachines_port_forward.py b/tests/unit/test_smolmachines_port_forward.py new file mode 100644 index 0000000..8adf2e1 --- /dev/null +++ b/tests/unit/test_smolmachines_port_forward.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import socket +import threading +import unittest + +from bot_bottle.backend.smolmachines.port_forward import ( + ForwardSpec, + start_forwarder, + stop_forwarder, +) + + +def _echo_server() -> tuple[socket.socket, int]: + listener = socket.create_server(("127.0.0.1", 0)) + port = int(listener.getsockname()[1]) + + def run() -> None: + with listener: + conn, _ = listener.accept() + with conn: + while True: + data = conn.recv(65536) + if not data: + return + conn.sendall(data) + + threading.Thread(target=run, daemon=True).start() + return listener, port + + +class TestForwarderValidation(unittest.TestCase): + def test_rejects_generic_localhost_listener(self): + with self.assertRaises(SystemExit): + start_forwarder(( + ForwardSpec( + label="egress", + listen_host="127.0.0.1", + listen_port=0, + target_host="127.0.0.1", + target_port=9099, + ), + )) + + def test_rejects_wildcard_listener(self): + with self.assertRaises(SystemExit): + start_forwarder(( + ForwardSpec( + label="egress", + listen_host="0.0.0.0", + listen_port=0, + target_host="127.0.0.1", + target_port=9099, + ), + )) + + def test_rejects_non_loopback_target(self): + with self.assertRaises(SystemExit): + start_forwarder(( + ForwardSpec( + label="egress", + listen_host="127.0.0.16", + listen_port=0, + target_host="192.0.2.10", + target_port=9099, + ), + )) + + +class TestForwarding(unittest.TestCase): + def test_forwards_bytes_both_directions(self): + try: + probe = socket.create_server(("127.0.0.16", 0)) + except OSError: + self.skipTest("127.0.0.16 is not bound on this host") + else: + probe.close() + _listener, target_port = _echo_server() + handle = start_forwarder(( + ForwardSpec( + label="egress", + listen_host="127.0.0.16", + listen_port=0, + target_host="127.0.0.1", + target_port=target_port, + ), + )) + try: + bound = handle.forwards[0] + with socket.create_connection((bound.listen_host, bound.listen_port)) as sock: + sock.sendall(b"hello") + self.assertEqual(b"hello", sock.recv(5)) + finally: + stop_forwarder(handle) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_smolmachines_provision.py b/tests/unit/test_smolmachines_provision.py index 695187d..bd33c77 100644 --- a/tests/unit/test_smolmachines_provision.py +++ b/tests/unit/test_smolmachines_provision.py @@ -432,12 +432,7 @@ class TestBundleLaunchSpec(unittest.TestCase): def test_canary_env_visible_to_smolvm_guest(self): plan = _plan(canary=True) - with patch.object( - _launch._bundle, - "bundle_host_port", - return_value="65000", - ): - stamped = _launch._discover_urls(plan, "127.0.0.16") + stamped = _launch._discover_urls(plan, "127.0.0.16", {9099: 65000}) self.assertEqual( "fake-canary-value", @@ -507,21 +502,11 @@ class TestProvisionGitUser(unittest.TestCase): class TestProxyHost(unittest.TestCase): - """_proxy_host returns the bridge gateway on Linux and the loopback - alias on other platforms.""" + """_proxy_host returns the per-bottle address used by the forwarder.""" - def test_linux_returns_bundle_gateway(self): + def test_returns_loopback_alias(self): plan = _plan() - with patch("bot_bottle.backend.smolmachines.launch.platform.system", - return_value="Linux"): - result = _launch._proxy_host(plan, "127.0.0.16") - self.assertEqual(plan.bundle_gateway, result) - - def test_non_linux_returns_loopback(self): - plan = _plan() - with patch("bot_bottle.backend.smolmachines.launch.platform.system", - return_value="Darwin"): - result = _launch._proxy_host(plan, "127.0.0.16") + result = _launch._proxy_host(plan, "127.0.0.16") self.assertEqual("127.0.0.16", result) @@ -544,14 +529,20 @@ class TestDiscoverUrls(unittest.TestCase): ),), ), ) - with patch.object(_launch._bundle, "bundle_host_port", return_value="9420"): - stamped = _launch._discover_urls(plan, "127.0.0.16") - self.assertEqual("127.0.0.16:9420", stamped.agent_git_gate_host) + stamped = _launch._discover_urls( + plan, + "127.0.0.16", + {9099: 65000, 9420: 65001}, + ) + self.assertEqual("127.0.0.16:65001", stamped.agent_git_gate_host) def test_supervise_url_set_when_supervise_present(self): plan = _plan(supervise=True) - with patch.object(_launch._bundle, "bundle_host_port", return_value="55556"): - stamped = _launch._discover_urls(plan, "127.0.0.16") + stamped = _launch._discover_urls( + plan, + "127.0.0.16", + {9099: 65000, 9100: 55556}, + ) self.assertEqual("http://127.0.0.16:55556/", stamped.agent_supervise_url) diff --git a/tests/unit/test_smolmachines_sidecar_bundle.py b/tests/unit/test_smolmachines_sidecar_bundle.py index a207582..8377434 100644 --- a/tests/unit/test_smolmachines_sidecar_bundle.py +++ b/tests/unit/test_smolmachines_sidecar_bundle.py @@ -18,9 +18,11 @@ from bot_bottle.backend.smolmachines.sidecar_bundle import ( bundle_network_name, create_bundle_network, ensure_bundle_image, + start_bundle_vm, remove_bundle_network, start_bundle, stop_bundle, + stop_bundle_vm, ) @@ -200,6 +202,57 @@ class TestEnsureBundleImage(unittest.TestCase): self.assertEqual("Dockerfile.sidecars", kwargs["dockerfile"]) +class TestStartBundleVm(unittest.TestCase): + def test_creates_sidecar_vm_with_ports_volumes_and_env(self): + calls: list[tuple[str, tuple[object, ...], dict[str, object]]] = [] + + def record(name): # type: ignore + def inner(*args, **kwargs): # type: ignore + calls.append((name, args, kwargs)) + return inner + + with patch( + "bot_bottle.backend.smolmachines.sidecar_bundle.allocate_raw_host_ports", + return_value={9099: 55001, 9420: 55002}, + ), patch( + "bot_bottle.backend.smolmachines.sidecar_bundle._smolvm.machine_create", + side_effect=record("create"), + ), patch( + "bot_bottle.backend.smolmachines.sidecar_bundle._smolvm.machine_start", + side_effect=record("start"), + ), patch( + "bot_bottle.backend.smolmachines.sidecar_bundle._smolvm.wait_exec_ready", + side_effect=record("wait"), + ): + launch = start_bundle_vm( + _spec( + daemons_csv="egress,git-http", + environment=("FOO=bar", "TOKEN"), + volumes=(("/host/routes", "/etc/egress", True),), + ports_to_publish=(9099, 9420), + ), + from_path=Path("/cache/sidecar.smolmachine"), + host_env={"TOKEN": "secret"}, + ) + + self.assertEqual({9099: 55001, 9420: 55002}, launch.raw_ports) + create = calls[0] + self.assertEqual("create", create[0]) + self.assertEqual(("bot-bottle-sidecars-demo-abc12",), create[1]) + self.assertEqual(Path("/cache/sidecar.smolmachine"), create[2]["from_path"]) + self.assertTrue(create[2]["net"]) + self.assertEqual(((55001, 9099), (55002, 9420)), create[2]["ports"]) + self.assertEqual((("/host/routes", "/etc/egress", True),), create[2]["volumes"]) + self.assertEqual( + { + "BOT_BOTTLE_SIDECAR_DAEMONS": "egress,git-http", + "FOO": "bar", + "TOKEN": "secret", + }, + create[2]["env"], + ) + + class TestStopBundle(unittest.TestCase): def _patch_run(self, **kwargs): # type: ignore return patch( @@ -222,5 +275,17 @@ class TestStopBundle(unittest.TestCase): stop_bundle("demo-abc12") # no raise +class TestStopBundleVm(unittest.TestCase): + def test_stops_then_deletes_sidecar_vm(self): + with patch( + "bot_bottle.backend.smolmachines.sidecar_bundle._smolvm.machine_stop", + ) as stop, patch( + "bot_bottle.backend.smolmachines.sidecar_bundle._smolvm.machine_delete", + ) as delete: + stop_bundle_vm("demo-abc12") + stop.assert_called_once_with("bot-bottle-sidecars-demo-abc12") + delete.assert_called_once_with("bot-bottle-sidecars-demo-abc12") + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_smolmachines_smolvm.py b/tests/unit/test_smolmachines_smolvm.py index 9fe6427..5e16359 100644 --- a/tests/unit/test_smolmachines_smolvm.py +++ b/tests/unit/test_smolmachines_smolvm.py @@ -106,6 +106,22 @@ class TestArgvShapes(unittest.TestCase): self.assertIn("--name", argv) self.assertIn("agent-xyz", argv) + def test_machine_create_with_net_ports_and_volumes(self): + with self._patch_run() as m: + machine_create( + "sidecar-xyz", + from_path=Path("/stage/sidecar.smolmachine"), + net=True, + ports=((55000, 9099),), + volumes=(("/host/routes", "/etc/egress", True),), + ) + argv = m.call_args.args[0] + self.assertIn("--net", argv) + self.assertIn("-p", argv) + self.assertIn("55000:9099", argv) + self.assertIn("-v", argv) + self.assertIn("/host/routes:/etc/egress:ro", argv) + def test_machine_create_omits_net_when_no_allow_cidrs(self): with self._patch_run() as m: machine_create("agent-xyz", from_path=Path("/x.smolmachine")) -- 2.52.0 From 5828668e2135722482ec13da081086645c575925 Mon Sep 17 00:00:00 2001 From: codex Date: Thu, 9 Jul 2026 16:43:48 +0000 Subject: [PATCH 3/9] test(smolmachines): cover sidecar vm launch paths --- tests/unit/test_smolmachines_enumerate.py | 66 +++++ tests/unit/test_smolmachines_port_forward.py | 233 +++++++++++++++++- tests/unit/test_smolmachines_provision.py | 109 ++++++++ .../unit/test_smolmachines_sidecar_bundle.py | 31 +++ 4 files changed, 436 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_smolmachines_enumerate.py diff --git a/tests/unit/test_smolmachines_enumerate.py b/tests/unit/test_smolmachines_enumerate.py new file mode 100644 index 0000000..76487e0 --- /dev/null +++ b/tests/unit/test_smolmachines_enumerate.py @@ -0,0 +1,66 @@ +"""Unit: smolmachines active-agent enumeration.""" + +from __future__ import annotations + +import json +import subprocess +import unittest +from unittest.mock import patch + +from bot_bottle.backend.smolmachines import enumerate as _enumerate + + +def _completed( + stdout: str, + returncode: int = 0, +) -> subprocess.CompletedProcess: # type: ignore[type-arg] + return subprocess.CompletedProcess( + args=[], + returncode=returncode, + stdout=stdout, + stderr="", + ) + + +class TestEnumerateActive(unittest.TestCase): + def test_skips_sidecar_vms(self): + machines = [ + {"name": "bot-bottle-demo-abc12", "state": "running"}, + {"name": "bot-bottle-sidecars-demo-abc12", "state": "running"}, + {"name": "bot-bottle-stopped-xyz", "state": "stopped"}, + {"name": "other", "state": "running"}, + ] + + class _Metadata: + agent_name = "demo" + started_at = "2026-07-09T00:00:00Z" + label = "Demo" + color = "green" + + with patch( + "bot_bottle.backend.smolmachines.enumerate.subprocess.run", + return_value=_completed(json.dumps(machines)), + ), patch( + "bot_bottle.backend.smolmachines.enumerate._query_bundle_services", + return_value={"demo-abc12": ("egress",)}, + ), patch( + "bot_bottle.backend.smolmachines.enumerate.read_metadata", + return_value=_Metadata(), + ) as read_metadata: + active = _enumerate.enumerate_active() + + self.assertEqual(1, len(active)) + self.assertEqual("demo-abc12", active[0].slug) + self.assertEqual(("egress",), active[0].services) + read_metadata.assert_called_once_with("demo-abc12") + + def test_invalid_smolvm_output_returns_empty(self): + with patch( + "bot_bottle.backend.smolmachines.enumerate.subprocess.run", + return_value=_completed("not json"), + ): + self.assertEqual([], _enumerate.enumerate_active()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_smolmachines_port_forward.py b/tests/unit/test_smolmachines_port_forward.py index 8adf2e1..0c41978 100644 --- a/tests/unit/test_smolmachines_port_forward.py +++ b/tests/unit/test_smolmachines_port_forward.py @@ -1,11 +1,19 @@ from __future__ import annotations +import json import socket +import subprocess import threading import unittest +from unittest.mock import MagicMock, patch from bot_bottle.backend.smolmachines.port_forward import ( ForwardSpec, + ForwarderHandle, + _CompletedProcess, + _Forwarder, + _pipe, + main, start_forwarder, stop_forwarder, ) @@ -30,6 +38,13 @@ def _echo_server() -> tuple[socket.socket, int]: class TestForwarderValidation(unittest.TestCase): + def test_empty_specs_returns_noop_handle(self): + handle = start_forwarder(()) + + self.assertEqual((), handle.forwards) + self.assertEqual(0, handle.process.poll()) + stop_forwarder(handle) + def test_rejects_generic_localhost_listener(self): with self.assertRaises(SystemExit): start_forwarder(( @@ -66,20 +81,123 @@ class TestForwarderValidation(unittest.TestCase): ), )) + def test_rejects_invalid_ports(self): + with self.assertRaises(SystemExit): + start_forwarder(( + ForwardSpec("bad-listen", "127.0.0.2", 70000, "127.0.0.1", 9099), + )) + with self.assertRaises(SystemExit): + start_forwarder(( + ForwardSpec("bad-target", "127.0.0.2", 0, "127.0.0.1", 0), + )) + + def test_completed_process_methods_are_noops(self): + proc = _CompletedProcess() + + proc.terminate() + proc.kill() + + self.assertEqual(0, proc.poll()) + self.assertEqual(0, proc.wait(timeout=1)) + + +class _FakeStdout: + def __init__(self, line: str): + self.line = line + self.closed = False + + def readline(self) -> str: + return self.line + + def close(self) -> None: + self.closed = True + + +class TestForwarderProcess(unittest.TestCase): + def test_start_parses_ready_payload(self): + payload = { + "forwards": [{ + "label": "egress", + "listen_host": "127.0.0.2", + "listen_port": 61000, + "target_host": "127.0.0.1", + "target_port": 55000, + }], + } + proc = MagicMock() + proc.stdout = _FakeStdout(json.dumps(payload) + "\n") + + with patch( + "bot_bottle.backend.smolmachines.port_forward.subprocess.Popen", + return_value=proc, + ) as popen: + handle = start_forwarder(( + ForwardSpec("egress", "127.0.0.2", 0, "127.0.0.1", 55000), + )) + + self.assertIs(proc, handle.process) + self.assertEqual(61000, handle.forwards[0].listen_port) + self.assertTrue(proc.stdout.closed) + argv = popen.call_args.args[0] + self.assertIn("--spec-json", argv) + + def test_start_dies_when_helper_exits_before_ready(self): + proc = MagicMock() + proc.stdout = _FakeStdout("") + + with patch( + "bot_bottle.backend.smolmachines.port_forward.subprocess.Popen", + return_value=proc, + ): + with self.assertRaises(SystemExit): + start_forwarder(( + ForwardSpec("egress", "127.0.0.2", 0, "127.0.0.1", 55000), + )) + + proc.terminate.assert_called_once() + proc.wait.assert_called_once_with(timeout=5) + + def test_start_dies_on_invalid_ready_payload(self): + proc = MagicMock() + proc.stdout = _FakeStdout('{"wrong": []}\n') + + with patch( + "bot_bottle.backend.smolmachines.port_forward.subprocess.Popen", + return_value=proc, + ): + with self.assertRaises(SystemExit): + start_forwarder(( + ForwardSpec("egress", "127.0.0.2", 0, "127.0.0.1", 55000), + )) + + proc.terminate.assert_called_once() + proc.wait.assert_called_once_with(timeout=5) + + def test_stop_kills_process_after_timeout(self): + proc = MagicMock() + proc.poll.return_value = None + proc.wait.side_effect = [subprocess.TimeoutExpired("cmd", 5), 0] + + stop_forwarder(ForwarderHandle(process=proc, forwards=())) + + proc.terminate.assert_called_once() + proc.kill.assert_called_once() + self.assertEqual(2, proc.wait.call_count) + class TestForwarding(unittest.TestCase): def test_forwards_bytes_both_directions(self): try: - probe = socket.create_server(("127.0.0.16", 0)) + probe = socket.create_server(("127.0.0.2", 0)) except OSError: - self.skipTest("127.0.0.16 is not bound on this host") + self.skipTest("127.0.0.2 is not bindable on this host") else: probe.close() _listener, target_port = _echo_server() handle = start_forwarder(( ForwardSpec( label="egress", - listen_host="127.0.0.16", + listen_host="127.0.0.2", listen_port=0, target_host="127.0.0.1", target_port=target_port, @@ -94,5 +212,114 @@ class TestForwarding(unittest.TestCase): stop_forwarder(handle) +class TestForwarderInProcess(unittest.TestCase): + def test_forwarder_serves_and_stops_in_process(self): + _listener, target_port = _echo_server() + forwarder = _Forwarder(( + ForwardSpec("egress", "127.0.0.2", 0, "127.0.0.1", target_port), + )) + forwarder.start() + thread = threading.Thread(target=forwarder.serve, daemon=True) + thread.start() + try: + bound = forwarder.bound[0] + with socket.create_connection((bound.listen_host, bound.listen_port)) as sock: + sock.sendall(b"ping") + self.assertEqual(b"ping", sock.recv(4)) + finally: + forwarder.stop() + thread.join(timeout=2) + + self.assertFalse(thread.is_alive()) + + def test_forwarder_logs_target_connect_failure(self): + forwarder = _Forwarder(( + ForwardSpec("egress", "127.0.0.2", 0, "127.0.0.1", 1), + )) + forwarder.start() + thread = threading.Thread(target=forwarder.serve, daemon=True) + thread.start() + try: + bound = forwarder.bound[0] + with patch("bot_bottle.backend.smolmachines.port_forward.warn") as warn: + with socket.create_connection((bound.listen_host, bound.listen_port)): + pass + for _ in range(100): + if warn.called: + break + threading.Event().wait(0.01) + self.assertTrue(warn.called) + finally: + forwarder.stop() + thread.join(timeout=2) + + def test_pipe_stops_when_recv_fails(self): + src, dst = socket.socketpair() + src.close() + try: + _pipe(src, dst) + finally: + dst.close() + + def test_pipe_stops_when_send_fails(self): + src, src_peer = socket.socketpair() + _dst_peer, dst = socket.socketpair() + dst.close() + try: + src_peer.sendall(b"hello") + src_peer.shutdown(socket.SHUT_WR) + _pipe(src, dst) + finally: + src.close() + src_peer.close() + _dst_peer.close() + + +class TestMain(unittest.TestCase): + def test_main_starts_serves_and_stops_forwarder(self): + calls: list[str] = [] + handlers = [] + + class FakeForwarder: + def __init__(self, specs): # type: ignore[no-untyped-def] + self.specs = specs + self.bound = specs + + def start(self) -> None: + calls.append("start") + + def serve(self) -> None: + calls.append("serve") + + def stop(self) -> None: + calls.append("stop") + + def record_signal(_signum, handler): # type: ignore[no-untyped-def] + handlers.append(handler) + + spec = ForwardSpec("egress", "127.0.0.2", 0, "127.0.0.1", 9099) + with patch( + "bot_bottle.backend.smolmachines.port_forward._Forwarder", + FakeForwarder, + ), patch( + "bot_bottle.backend.smolmachines.port_forward.signal.signal", + side_effect=record_signal, + ), patch("builtins.print") as print_mock: + self.assertEqual( + 0, + main(("--spec-json", json.dumps([{ + "label": spec.label, + "listen_host": spec.listen_host, + "listen_port": spec.listen_port, + "target_host": spec.target_host, + "target_port": spec.target_port, + }]))), + ) + handlers[0](15, None) + + self.assertEqual(["start", "serve", "stop", "stop"], calls) + self.assertIn("forwards", print_mock.call_args.args[0]) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_smolmachines_provision.py b/tests/unit/test_smolmachines_provision.py index bd33c77..1e848a4 100644 --- a/tests/unit/test_smolmachines_provision.py +++ b/tests/unit/test_smolmachines_provision.py @@ -9,6 +9,7 @@ from __future__ import annotations import subprocess import tempfile import unittest +from contextlib import ExitStack from dataclasses import replace from pathlib import Path from unittest.mock import MagicMock, patch @@ -27,6 +28,7 @@ from bot_bottle.backend.smolmachines.bottle_plan import ( SmolmachinesBottlePlan, ) from bot_bottle.backend.smolmachines import launch as _launch +from bot_bottle.backend.smolmachines import port_forward as _forward from bot_bottle.backend.smolmachines.launch import _bundle_launch_spec from bot_bottle.backend.util import AGENT_CA_PATH from bot_bottle.egress import EgressPlan, EgressRoute @@ -510,6 +512,113 @@ class TestProxyHost(unittest.TestCase): self.assertEqual("127.0.0.16", result) +class TestLaunchResourceWiring(unittest.TestCase): + def test_allocate_resources_uses_loopback_alias_and_bundle_name(self): + plan = _plan() + with patch( + "bot_bottle.backend.smolmachines.launch._loopback.ensure_pool", + ) as ensure_pool, patch( + "bot_bottle.backend.smolmachines.launch._loopback.allocate", + return_value="127.0.0.16", + ) as allocate: + loopback_ip, network = _launch._allocate_resources(plan, ExitStack()) + + ensure_pool.assert_called_once_with() + allocate.assert_called_once_with("demo-abc12") + self.assertEqual("127.0.0.16", loopback_ip) + self.assertEqual("bot-bottle-bundle-demo-abc12", network) + + def test_start_bundle_wraps_raw_vm_ports_with_forwarder_ports(self): + plan = _plan(supervise=True) + plan = replace( + plan, + git_gate_plan=replace( + plan.git_gate_plan, + upstreams=(GitGateUpstream( + name="bot-bottle", + upstream_url="ssh://git@host/repo.git", + upstream_host="host", + upstream_port="22", + identity_file="/tmp/key", + known_host_key="", + ),), + ), + ) + stack = MagicMock(spec=ExitStack) + raw_launch = MagicMock(raw_ports={9099: 55001, 9420: 55002, 9100: 55003}) + handle = _forward.ForwarderHandle( + process=MagicMock(), + forwards=( + _forward.ForwardSpec("egress", "127.0.0.16", 65001, "127.0.0.1", 55001), + _forward.ForwardSpec("git-http", "127.0.0.16", 65002, "127.0.0.1", 55002), + _forward.ForwardSpec("supervise", "127.0.0.16", 65003, "127.0.0.1", 55003), + ), + ) + + with patch( + "bot_bottle.backend.smolmachines.launch._provision_git_gate_keys", + return_value=plan, + ), patch( + "bot_bottle.backend.smolmachines.launch._ensure_smolmachine", + return_value=Path("/cache/sidecar.smolmachine"), + ) as ensure, patch( + "bot_bottle.backend.smolmachines.launch._bundle.start_bundle_vm", + return_value=raw_launch, + ) as start_vm, patch( + "bot_bottle.backend.smolmachines.launch._forward.start_forwarder", + return_value=handle, + ) as start_forwarder: + stamped = _launch._start_bundle(plan, "net", "127.0.0.16", stack) + + ensure.assert_called_once() + start_vm.assert_called_once() + self.assertEqual(Path("/cache/sidecar.smolmachine"), start_vm.call_args.kwargs["from_path"]) + specs = start_forwarder.call_args.args[0] + self.assertEqual( + ( + _forward.ForwardSpec("egress", "127.0.0.16", 0, "127.0.0.1", 55001), + _forward.ForwardSpec("git-http", "127.0.0.16", 0, "127.0.0.1", 55002), + _forward.ForwardSpec("supervise", "127.0.0.16", 0, "127.0.0.1", 55003), + ), + specs, + ) + self.assertEqual("http://127.0.0.16:65001", stamped.agent_proxy_url) + self.assertEqual("127.0.0.16:65002", stamped.agent_git_gate_host) + self.assertEqual("http://127.0.0.16:65003/", stamped.agent_supervise_url) + self.assertEqual(2, stack.callback.call_count) + + def test_init_vm_repairs_ownership_then_waits_for_exec(self): + plan = _plan() + with patch( + "bot_bottle.backend.smolmachines.launch._smolvm.machine_exec", + ) as machine_exec, patch( + "bot_bottle.backend.smolmachines.launch._smolvm.wait_exec_ready", + ) as wait_exec_ready: + _launch._init_vm(plan) + + machine_exec.assert_called_once() + argv = machine_exec.call_args.args[1] + self.assertEqual(["sh", "-c"], argv[:2]) + self.assertIn("chown -R node:node /home/node", argv[2]) + wait_exec_ready.assert_called_once_with(plan.machine_name) + + +class TestPortLabels(unittest.TestCase): + def test_known_and_dynamic_port_labels_round_trip(self): + for port, label in ( + (9099, "egress"), + (9420, "git-http"), + (9100, "supervise"), + (12345, "port-12345"), + ): + self.assertEqual(label, _launch._label_for_port(port)) + self.assertEqual(port, _launch._port_for_label(label)) + + def test_unknown_label_is_rejected(self): + with self.assertRaises(ValueError): + _launch._port_for_label("sidecar") + + class TestDiscoverUrls(unittest.TestCase): """_discover_urls stamps git-gate host + supervise URL into the plan.""" diff --git a/tests/unit/test_smolmachines_sidecar_bundle.py b/tests/unit/test_smolmachines_sidecar_bundle.py index 8377434..ac165dc 100644 --- a/tests/unit/test_smolmachines_sidecar_bundle.py +++ b/tests/unit/test_smolmachines_sidecar_bundle.py @@ -14,6 +14,7 @@ from unittest.mock import patch from bot_bottle.backend.smolmachines.sidecar_bundle import ( BundleLaunchSpec, + allocate_raw_host_ports, bundle_container_name, bundle_network_name, create_bundle_network, @@ -203,6 +204,15 @@ class TestEnsureBundleImage(unittest.TestCase): class TestStartBundleVm(unittest.TestCase): + def test_allocate_raw_host_ports_returns_bindable_ports(self): + raw_ports = allocate_raw_host_ports((9099, 9420)) + + self.assertEqual({9099, 9420}, set(raw_ports)) + for host_port in raw_ports.values(): + self.assertIsInstance(host_port, int) + self.assertGreater(host_port, 0) + self.assertNotEqual(raw_ports[9099], raw_ports[9420]) + def test_creates_sidecar_vm_with_ports_volumes_and_env(self): calls: list[tuple[str, tuple[object, ...], dict[str, object]]] = [] @@ -286,6 +296,27 @@ class TestStopBundleVm(unittest.TestCase): stop.assert_called_once_with("bot-bottle-sidecars-demo-abc12") delete.assert_called_once_with("bot-bottle-sidecars-demo-abc12") + def test_stop_and_delete_errors_are_warnings(self): + from bot_bottle.backend.smolmachines.smolvm import SmolvmError + result = subprocess.CompletedProcess( + args=[], + returncode=1, + stdout="", + stderr="failed", + ) + with patch( + "bot_bottle.backend.smolmachines.sidecar_bundle._smolvm.machine_stop", + side_effect=SmolvmError(["smolvm", "machine", "stop"], result), + ), patch( + "bot_bottle.backend.smolmachines.sidecar_bundle._smolvm.machine_delete", + side_effect=SmolvmError(["smolvm", "machine", "delete"], result), + ), patch( + "bot_bottle.backend.smolmachines.sidecar_bundle.warn", + ) as warn: + stop_bundle_vm("demo-abc12") + + self.assertEqual(2, warn.call_count) + if __name__ == "__main__": unittest.main() -- 2.52.0 From bb9cca48fdc49aac02f6a782b1498e8182fb9b19 Mon Sep 17 00:00:00 2001 From: codex Date: Thu, 9 Jul 2026 17:42:04 +0000 Subject: [PATCH 4/9] refactor(smolmachines): inline sidecar proxy host --- bot_bottle/backend/smolmachines/launch.py | 23 ++++++++--------------- tests/unit/test_smolmachines_provision.py | 9 --------- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/bot_bottle/backend/smolmachines/launch.py b/bot_bottle/backend/smolmachines/launch.py index d1ce279..2d2b430 100644 --- a/bot_bottle/backend/smolmachines/launch.py +++ b/bot_bottle/backend/smolmachines/launch.py @@ -92,7 +92,7 @@ def launch( try: loopback_ip, network = _allocate_resources(plan, stack) plan = _mint_certs(plan) - proxy_host = _proxy_host(plan, loopback_ip) + proxy_host = loopback_ip plan = _start_bundle(plan, network, proxy_host, stack) agent_from_path = _agent_from_path(plan) @@ -287,14 +287,13 @@ def _launch_vm( ) -> None: """Create, patch, and start the smolvm VM; register teardown. - --allow-cidr is `proxy_host/32` — the per-bottle loopback alias - on macOS or the bridge gateway on Linux (see `_proxy_host`). This - ensures the guest can only reach bundle ports published on that IP, - not the container IP directly. force_allowlist confirms the - allowlist persisted (patching smolvm 0.8.0's silent-drop of - --allow-cidr when combined with --from) and fails closed if it - can't. Smolfile isn't usable here — smolvm 0.8.0 makes --from - and --smolfile mutually exclusive.""" + --allow-cidr is `proxy_host/32` — the per-bottle loopback alias. + This ensures the guest can only reach sidecar forwarders published + on that IP, not host localhost or another bottle's alias. + force_allowlist confirms the allowlist persisted (patching smolvm + 0.8.0's silent-drop of --allow-cidr when combined with --from) and + fails closed if it can't. Smolfile isn't usable here — smolvm 0.8.0 + makes --from and --smolfile mutually exclusive.""" tsi_cidr = f"{proxy_host}/32" _smolvm.machine_create( plan.machine_name, @@ -341,12 +340,6 @@ def _init_vm(plan: SmolmachinesBottlePlan) -> None: _smolvm.wait_exec_ready(plan.machine_name) -def _proxy_host(plan: SmolmachinesBottlePlan, loopback_ip: str) -> str: - """Return the per-bottle host address used for TSI and forwarders.""" - del plan - return loopback_ip - - def _label_for_port(port: int) -> str: if port == _EGRESS_PORT: return "egress" diff --git a/tests/unit/test_smolmachines_provision.py b/tests/unit/test_smolmachines_provision.py index 1e848a4..f5c2aa8 100644 --- a/tests/unit/test_smolmachines_provision.py +++ b/tests/unit/test_smolmachines_provision.py @@ -503,15 +503,6 @@ class TestProvisionGitUser(unittest.TestCase): self.assertIn("bot@example.com", calls[0][0]) -class TestProxyHost(unittest.TestCase): - """_proxy_host returns the per-bottle address used by the forwarder.""" - - def test_returns_loopback_alias(self): - plan = _plan() - result = _launch._proxy_host(plan, "127.0.0.16") - self.assertEqual("127.0.0.16", result) - - class TestLaunchResourceWiring(unittest.TestCase): def test_allocate_resources_uses_loopback_alias_and_bundle_name(self): plan = _plan() -- 2.52.0 From 2b53c36608605abbb79c47fa29cedc68cbf6d91e Mon Sep 17 00:00:00 2001 From: codex Date: Thu, 9 Jul 2026 18:14:44 +0000 Subject: [PATCH 5/9] refactor(smolmachines): drop obsolete docker-sidecar helpers --- .../backend/smolmachines/bottle_plan.py | 33 ++-- bot_bottle/backend/smolmachines/launch.py | 18 +- .../backend/smolmachines/resolve_plan.py | 12 +- .../backend/smolmachines/sidecar_bundle.py | 169 +----------------- .../test_smolmachines_bundle_bringup.py | 111 ------------ tests/integration/test_smolmachines_launch.py | 44 +---- .../unit/test_smolmachines_sidecar_bundle.py | 162 +---------------- 7 files changed, 37 insertions(+), 512 deletions(-) delete mode 100644 tests/integration/test_smolmachines_bundle_bringup.py diff --git a/bot_bottle/backend/smolmachines/bottle_plan.py b/bot_bottle/backend/smolmachines/bottle_plan.py index 48d6a1f..62bf28b 100644 --- a/bot_bottle/backend/smolmachines/bottle_plan.py +++ b/bot_bottle/backend/smolmachines/bottle_plan.py @@ -1,10 +1,8 @@ """SmolmachinesBottlePlan — concrete BottlePlan for the smolmachines backend (PRD 0023). -Slug + bundle docker subnet / gateway / pinned IP + smolvm -machine name + agent `.smolmachine` artifact + per-bottle guest -env. Provisioning fields (CA cert path, prompt path, etc.) land -in chunk 4.""" +Slug + legacy bundle network coordinates + smolvm machine name + +agent `.smolmachine` artifact + per-bottle guest env.""" from __future__ import annotations @@ -23,9 +21,10 @@ class SmolmachinesBottlePlan(BottlePlan): `supervise_plan`, and `agent_provision` from BottlePlan.""" slug: str - # Per-bottle docker subnet for the sidecar bundle container. - # The bundle runs at `bundle_ip` (always `.2`); the gateway is - # at `.1`. smolvm's TSI allowlist is set to `bundle_ip/32`. + # Legacy per-bottle bundle network coordinates. These remain on + # the plan while BundleLaunchSpec still carries the original shape, + # but the smolmachines launch path exposes the sidecar VM through + # host-loopback forwarders instead of a Docker bridge IP. bundle_subnet: str bundle_gateway: str bundle_ip: str @@ -36,22 +35,10 @@ class SmolmachinesBottlePlan(BottlePlan): # `--smolfile` is mutually exclusive with `--from`, and # `--from` is the path that avoids the registry-pull race). guest_env: dict[str, str] - # Inner Plans for the sidecar bundle daemons. The same shape the - # docker backend uses — same `.prepare()` calls produced - # them — but our launch step doesn't populate the - # docker-specific network fields (internal_network, - # egress_network) because the smolmachines bundle isn't on - # docker's `--internal` + egress bridge topology; it's on a - # per-bottle bridge with a pinned IP. The unused fields stay - # at their dataclass defaults. - # Agent-side endpoints. On Docker Desktop the docker bridge - # IPs aren't reachable from the smolvm guest (TSI uses macOS - # networking; docker container IPs live in the daemon's VM), - # so the agent dials the bundle via host loopback + - # docker-published random ports. Empty at prepare time; - # launch populates these after bundle bringup via - # `dataclasses.replace`. Format: a `host:port` for git-gate - # (insteadOf URL prefix) + full URLs for proxy / supervise. + # Agent-side endpoints. Empty at prepare time; launch populates + # these after sidecar VM bringup via `dataclasses.replace`. + # Format: a `host:port` for git-gate (insteadOf URL prefix) + + # full URLs for proxy / supervise. agent_proxy_url: str = "" agent_git_gate_host: str = "" agent_supervise_url: str = "" diff --git a/bot_bottle/backend/smolmachines/launch.py b/bot_bottle/backend/smolmachines/launch.py index 2d2b430..1cb1f4a 100644 --- a/bot_bottle/backend/smolmachines/launch.py +++ b/bot_bottle/backend/smolmachines/launch.py @@ -1,12 +1,9 @@ -"""End-to-end launch flow for the smolmachines backend -(PRD 0023 chunks 2d + 4b). +"""End-to-end launch flow for the smolmachines backend. -Brings up the per-bottle docker bridge + sidecar bundle (with -real daemons + their config files), creates + starts the smolvm -guest pointed at the bundle's pinned IP via TSI's -`--allow-cidr /32` allowlist, yields a -`SmolmachinesBottle` handle, tears everything down on context -exit. +Builds the sidecar bundle smolmachine, starts it as a sidecar VM +with real daemons + their config files, creates + starts the agent +smolVM, yields a `SmolmachinesBottle` handle, and tears everything +down on context exit. The bundle's daemons consume the inner Plans the docker backend already produces: egress reads routes + CAs from the EgressPlan. @@ -72,9 +69,8 @@ _SMOLMACHINE_CACHE_DIR = Path.home() / ".cache" / "bot-bottle" / "smolmachines" # Container-internal listening ports for each bundle daemon. The -# bundle publishes each one on a random host loopback port (see -# `_bundle.start_bundle`), and `_bundle.bundle_host_port` looks -# them up post-start. +# sidecar VM publishes each one on a random host loopback port, and +# the launch flow wraps those raw ports with per-bottle forwarders. _GIT_HTTP_PORT = 9420 _SUPERVISE_PORT = SUPERVISE_PORT diff --git a/bot_bottle/backend/smolmachines/resolve_plan.py b/bot_bottle/backend/smolmachines/resolve_plan.py index 8d8dfcb..f81e77b 100644 --- a/bot_bottle/backend/smolmachines/resolve_plan.py +++ b/bot_bottle/backend/smolmachines/resolve_plan.py @@ -56,13 +56,11 @@ def resolve_plan( git_gate_plan: GitGatePlan, stage_dir: Path, ) -> SmolmachinesBottlePlan: - """Materialize the smolmachines plan. The bundle's docker - subnet + pinned IP are derived from the slug; the agent's - `.smolmachine` artifact is built (or cache-hit) here so - launch's `machine create --from` boots without a registry - pull. Per-bottle guest env + the TSI allow_cidrs land on the - plan for launch to pass straight through to - `machine create` flags.""" + """Materialize the smolmachines plan. The agent `.smolmachine` + artifact is built (or cache-hit) here so launch's + `machine create --from` boots without a registry pull. Per-bottle + guest env lands on the plan for launch to pass straight through + to `machine create` flags.""" # ==== smolmachines specific setup ==== subnet, gateway, bundle_ip = smolmachines_bundle_subnet(slug) diff --git a/bot_bottle/backend/smolmachines/sidecar_bundle.py b/bot_bottle/backend/smolmachines/sidecar_bundle.py index 59916b1..6118014 100644 --- a/bot_bottle/backend/smolmachines/sidecar_bundle.py +++ b/bot_bottle/backend/smolmachines/sidecar_bundle.py @@ -1,36 +1,18 @@ -"""Per-bottle sidecar bundle bringup for the smolmachines backend -(PRD 0023). +"""Per-bottle sidecar bundle bringup for the smolmachines backend. -Two docker resources per bottle live here: - - - **A dedicated bridge network**, subnet derived from the slug. - The bundle container gets a pinned IP at `.2` so the - smolvm guest's TSI allowlist (`/32`) has a stable - target. Without pinning, we'd have to inspect the container's - assigned IP after start and feed it back into the Smolfile - — a race we can sidestep with `--ip`. - - - **The bundle container itself**, running the PRD 0024 bundle - image (`bot-bottle-sidecars:latest` by default). Same - image, same daemons, same daemon-private env / bind-mounts - as the docker backend. - -This module ships the lifecycle primitives only — create -network, start bundle, stop bundle, remove network — wrapped -around `subprocess.run(["docker", ...])`. Wiring them into the -launch flow + populating the `BundleLaunchSpec` from the inner -Plans (EgressPlan, …) lands in chunk 2d.""" +The sidecar bundle runs as its own smolVM. The agent VM reaches +bundle daemons through host-loopback ports published by that sidecar +VM and wrapped by per-bottle address-bound forwarders.""" from __future__ import annotations import os import socket -import subprocess from dataclasses import dataclass, field from pathlib import Path from typing import Sequence -from ...log import die, warn +from ...log import warn from ..docker import util as docker_mod from ..docker.sidecar_bundle import ( SIDECAR_BUNDLE_DOCKERFILE, @@ -78,10 +60,8 @@ class BundleLaunchSpec: # supervisor inside the bundle reads it to skip # bottle-irrelevant daemons (e.g. supervise=False bottles). daemons_csv: str = "egress" - # Plain "KEY=VALUE" strings + "KEY" bare names (the bare-name - # form inherits the value from the docker-run subprocess env, - # matching the docker backend's compose-up secret-forwarding - # pattern). + # Plain "KEY=VALUE" strings + "KEY" bare names. Bare names inherit + # from the host env passed to the sidecar VM launch. environment: Sequence[str] = field(default_factory=tuple) # (host_path, container_path, read_only) bind mounts. volumes: Sequence[tuple[str, str, bool]] = field(default_factory=tuple) @@ -182,138 +162,3 @@ def stop_bundle_vm(slug: str) -> None: _smolvm.machine_delete(name) except _smolvm.SmolvmError as exc: warn(f"smolvm machine delete {name} failed: {exc}") - - -def create_bundle_network(network_name: str, subnet: str, gateway: str) -> None: - """`docker network create` with an explicit subnet + gateway - so the bundle's `--ip` lands on the address the Smolfile's - TSI allowlist points at. Idempotent on the caller's side — - `start_bundle` catches the "network exists" error and treats - it as success (chunk-2d teardown is paired with each create). - """ - result = subprocess.run( - ["docker", "network", "create", - "--subnet", subnet, "--gateway", gateway, - network_name], - capture_output=True, text=True, check=False, - ) - if result.returncode != 0: - # Already-exists is fine on a resume path; everything else - # is fatal — the bundle won't have an addressable network. - if "already exists" in (result.stderr or "").lower(): - return - die( - f"docker network create {network_name} failed: " - f"{(result.stderr or '').strip()}" - ) - - -def remove_bundle_network(network_name: str) -> None: - """Idempotent: a missing network returns success.""" - result = subprocess.run( - ["docker", "network", "rm", network_name], - capture_output=True, text=True, check=False, - ) - if result.returncode == 0: - return - if "no such network" in (result.stderr or "").lower(): - return - # Network with attached containers is the common non-fatal - # case during a partial teardown — warn but don't die. - warn( - f"docker network rm {network_name} failed: " - f"{(result.stderr or '').strip()}" - ) - - -def start_bundle(spec: BundleLaunchSpec, *, - env: dict[str, str] | None = None) -> None: - """Bring the bundle container up on the per-bottle bridge with - the pinned IP. Argv is built deterministically from `spec`; - `env` is the host subprocess env (forwarded values for any - bare-name entries in `spec.environment`).""" - container = bundle_container_name(spec.slug) - argv = [ - "docker", "run", - "--name", container, - "--detach", - "--rm", - "--network", spec.network_name, - "--ip", spec.bundle_ip, - "-e", f"BOT_BOTTLE_SIDECAR_DAEMONS={spec.daemons_csv}", - ] - for entry in spec.environment: - argv += ["-e", entry] - for host_path, container_path, read_only in spec.volumes: - suffix = ":ro" if read_only else "" - argv += ["-v", f"{host_path}:{container_path}{suffix}"] - # Loopback-only host port-forwards — the smolvm guest's TSI - # uses macOS networking, and macOS loopback is the only host - # surface that round-trips into Docker Desktop's daemon VM. - # Binds to the per-bottle alias so TSI's IP-only allowlist - # narrows reachability to this bottle's bundle only. - for port in spec.ports_to_publish: - argv += ["-p", f"{spec.publish_host_ip}::{port}"] - argv.append(spec.image) - result = subprocess.run( - argv, capture_output=True, text=True, - env=dict(env) if env is not None else None, check=False, - ) - if result.returncode != 0: - die( - f"docker run for bundle {container} failed: " - f"{(result.stderr or '').strip()}" - ) - - -def bundle_host_port( - slug: str, container_port: int, *, host_ip: str = "127.0.0.1", -) -> int: - """`docker port /tcp` → the random - host-side port docker assigned for the binding on `host_ip`. - Called after `start_bundle` on each container port listed in - `BundleLaunchSpec.ports_to_publish` so the launch step can - build the agent's HTTPS_PROXY / GIT_GATE / SUPERVISE URLs in - `:` form.""" - container = bundle_container_name(slug) - result = subprocess.run( - ["docker", "port", container, f"{container_port}/tcp"], - capture_output=True, text=True, check=False, - ) - if result.returncode != 0: - die( - f"docker port {container} {container_port}/tcp failed: " - f"{(result.stderr or '').strip() or ''}" - ) - # Each line looks like `127.0.0.16:54321` — one per address - # family / host IP. Match on the expected host_ip prefix so - # bottles bound to per-bottle aliases pick the right line. - for raw in (result.stdout or "").splitlines(): - line = raw.strip() - if line.startswith(f"{host_ip}:"): - _, _, port_str = line.rpartition(":") - try: - return int(port_str) - except ValueError: - die(f"unexpected `docker port` output: {line!r}") - die( - f"no port mapping on {host_ip} for {container} " - f"{container_port}/tcp; got: {(result.stdout or '').strip()!r}" - ) - - -def stop_bundle(slug: str) -> None: - """Idempotent: a missing container returns success.""" - container = bundle_container_name(slug) - result = subprocess.run( - ["docker", "rm", "-f", container], - capture_output=True, text=True, check=False, - ) - if result.returncode == 0: - return - if "no such container" in (result.stderr or "").lower(): - return - warn( - f"docker rm -f {container} failed: " - f"{(result.stderr or '').strip()}" - ) diff --git a/tests/integration/test_smolmachines_bundle_bringup.py b/tests/integration/test_smolmachines_bundle_bringup.py deleted file mode 100644 index 043a4a9..0000000 --- a/tests/integration/test_smolmachines_bundle_bringup.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Integration: PRD 0023 chunk 2c — bundle bringup on a per-bottle -docker bridge with the pinned IP. - -End-to-end against the real docker daemon. Brings up just the -sidecar bundle on its own bridge, confirms the container lands at -the pinned IP, then tears down. Skipped under act_runner (docker -socket mount topology breaks bridge visibility) and when the -bundle image isn't available. - -Full launch flow (smolvm + bundle + provisioning + the -localhost-reach / egress-port-bypass probes) lives in chunk 2d.""" - -from __future__ import annotations - -import os -import subprocess -import time -import unittest - -from bot_bottle.backend.smolmachines.sidecar_bundle import ( - BundleLaunchSpec, - bundle_container_name, - bundle_network_name, - create_bundle_network, - remove_bundle_network, - start_bundle, - stop_bundle, -) -from tests._docker import skip_unless_docker - - -@skip_unless_docker() -@unittest.skipIf( - os.environ.get("GITEA_ACTIONS") == "true", - "skipped under act_runner: docker socket mount topology breaks " - "in-process visibility of networks created on the host daemon", -) -class TestBundleBringup(unittest.TestCase): - def setUp(self): - self.slug = f"cb-test-bundle-{os.getpid()}-{int(time.time())}" - self.network = bundle_network_name(self.slug) - self.container = bundle_container_name(self.slug) - - def tearDown(self): - stop_bundle(self.slug) - remove_bundle_network(self.network) - - def _bundle_image_built(self) -> bool: - """The bundle image (`bot-bottle-sidecars:latest`) is - built lazily by the docker backend's compose. If a - smolmachines-only operator hasn't run the docker backend - first, the image won't exist locally. Skip rather than - fail.""" - r = subprocess.run( - ["docker", "image", "inspect", "bot-bottle-sidecars:latest"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - check=False, - ) - return r.returncode == 0 - - def test_create_network_then_start_bundle_pins_ip(self): - if not self._bundle_image_built(): - self.skipTest( - "bot-bottle-sidecars:latest not built; run a docker " - "bottle first or `docker build -f Dockerfile.sidecars .`" - ) - - # Pick a subnet unlikely to collide on the host. Last - # octet of the slug hash isn't deterministic across runs; - # we hardcode a high octet (.211) that the docker default - # bridges almost never use. - subnet = "192.168.211.0/24" - gateway = "192.168.211.1" - bundle_ip = "192.168.211.2" - - create_bundle_network(self.network, subnet, gateway) - - spec = BundleLaunchSpec( - slug=self.slug, - network_name=self.network, - subnet=subnet, - gateway=gateway, - bundle_ip=bundle_ip, - # Empty daemons_csv → init exits "no daemons selected" - # immediately. We just need the container to land on - # the network at the right IP before it exits. - daemons_csv="", # empty → init exits "no daemons selected" - ) - start_bundle(spec) - - # Inspect the container's IP on the per-bottle network. - r = subprocess.run( - ["docker", "inspect", - "--format", - "{{(index .NetworkSettings.Networks \"" + self.network + "\").IPAddress}}", - self.container], - capture_output=True, text=True, check=False, - ) - # Container may have exited (no daemons selected → exit 0). - # The inspect still works on exited containers as long as - # `--rm` hasn't fired yet, which is a race. Even if it has, - # the launch succeeded — the container existed, on the - # right network, at the right IP. We don't fail here on - # missing inspect. - if r.returncode == 0 and r.stdout.strip(): - self.assertEqual(bundle_ip, r.stdout.strip(), - f"bundle landed at wrong IP: {r.stdout!r}") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/integration/test_smolmachines_launch.py b/tests/integration/test_smolmachines_launch.py index b93756a..90881f1 100644 --- a/tests/integration/test_smolmachines_launch.py +++ b/tests/integration/test_smolmachines_launch.py @@ -1,10 +1,9 @@ -"""Integration: PRD 0023 chunk 2d — end-to-end launch + exec -round trip + the acceptance probes. +"""Integration: end-to-end smolmachines launch + exec round trip. -The smoke confirms the launch flow (per-bottle docker bridge → -sidecar bundle with host-loopback published ports → smolvm guest -with TSI allowlist → exec) plumbs together end to end. The probes confirm the -security properties the design pivot was about: +The smoke confirms the launch flow (sidecar bundle smolVM → +host-loopback forwarders → agent smolVM with TSI allowlist → exec) +plumbs together end to end. The probes confirm the security +properties the design pivot was about: - **localhost-reach probe** — guest tries to dial a service bound on the host's `127.0.0.1`. TSI's per-bottle loopback @@ -14,13 +13,6 @@ security properties the design pivot was about: the injected `HTTPS_PROXY`/`HTTP_PROXY` URL on the per-bottle loopback alias, while direct egress with proxy vars unset fails. - - **egress-port-bypass probe** — guest tries to dial - `:9099` (egress's port). TSI permits the IP but - the bundle's egress daemon binds `127.0.0.1` inside its - container, so the connect refuses at the socket level. The - bind-address mitigation is what closes TSI's port-granularity - gap. - Gated on macOS/Linux + smolvm + docker + not GITEA_ACTIONS — the runner can't host libkrun-backed VMs.""" @@ -114,8 +106,8 @@ class TestSmolmachinesLaunch(unittest.TestCase): def test_localhost_reach_probe(self): # Agent dials a 127.0.0.1 service on the host. TSI's - # allowlist contains only /32, so this must - # refuse. We use a port unlikely to be bound on the host + # allowlist contains only the per-bottle loopback alias, so + # this must refuse. We use a port unlikely to be bound on the host # (high-numbered) so we're confirming TSI refusal, not # just "no service listening." r = self.bottle.exec( @@ -196,28 +188,6 @@ class TestSmolmachinesLaunch(unittest.TestCase): self.assertEqual(0, r.returncode, msg=r.stderr) self.assertEqual(_AGENT_PROMPT, r.stdout.rstrip("\n")) - def test_egress_port_bypass_probe(self): - # Agent dials :9099 (egress's port). TSI - # permits the IP, but egress will bind 127.0.0.1:9099 - # inside the bundle in chunk 3, so the connect refuses - # at the socket level. NOTE: in chunk 2d the bundle's - # daemons aren't running (daemons_csv=""), so nothing - # is listening on :9099 anyway — this test asserts the - # connect fails, which is the property chunk 3 will - # preserve once egress is actually running. - r = self.bottle.exec( - "env -u HTTPS_PROXY -u HTTP_PROXY -u https_proxy -u http_proxy " - f"curl -s --show-error --max-time 3 http://{self.plan.bundle_ip}:9099 " - "2>&1 || true" - ) - self.assertTrue( - "refused" in r.stdout.lower() - or "timed out" in r.stdout.lower() - or "unreachable" in r.stdout.lower() - or "failed" in r.stdout.lower(), - f"expected egress port refusal; got: {r.stdout!r}", - ) - if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_smolmachines_sidecar_bundle.py b/tests/unit/test_smolmachines_sidecar_bundle.py index ac165dc..57c6aa4 100644 --- a/tests/unit/test_smolmachines_sidecar_bundle.py +++ b/tests/unit/test_smolmachines_sidecar_bundle.py @@ -1,9 +1,4 @@ -"""Unit: bundle bringup primitives for the smolmachines backend -(PRD 0023 chunk 2c). - -Tests mock `subprocess.run` and assert on the docker argv shape. -The end-to-end integration smoke (real docker daemon, real -bundle image) lands in chunk 2d.""" +"""Unit: bundle bringup primitives for the smolmachines backend.""" from __future__ import annotations @@ -17,28 +12,12 @@ from bot_bottle.backend.smolmachines.sidecar_bundle import ( allocate_raw_host_ports, bundle_container_name, bundle_network_name, - create_bundle_network, ensure_bundle_image, start_bundle_vm, - remove_bundle_network, - start_bundle, - stop_bundle, stop_bundle_vm, ) -def _ok(stdout: str = "", stderr: str = "") -> subprocess.CompletedProcess: # type: ignore - return subprocess.CompletedProcess( - args=[], returncode=0, stdout=stdout, stderr=stderr, - ) - - -def _fail(stderr: str = "boom") -> subprocess.CompletedProcess: # type: ignore - return subprocess.CompletedProcess( - args=[], returncode=1, stdout="", stderr=stderr, - ) - - def _spec(**kwargs) -> BundleLaunchSpec: # type: ignore defaults = dict( slug="demo-abc12", @@ -71,123 +50,6 @@ class TestNamingHelpers(unittest.TestCase): ) -class TestNetworkLifecycle(unittest.TestCase): - def _patch_run(self, **kwargs): # type: ignore - return patch( - "bot_bottle.backend.smolmachines.sidecar_bundle.subprocess.run", - **kwargs, - ) - - def test_create_argv_explicit_subnet_and_gateway(self): - with self._patch_run(return_value=_ok()) as m: - create_bundle_network("nn", "192.168.50.0/24", "192.168.50.1") - self.assertEqual( - ["docker", "network", "create", - "--subnet", "192.168.50.0/24", - "--gateway", "192.168.50.1", - "nn"], - m.call_args.args[0], - ) - - def test_create_treats_existing_network_as_success(self): - with self._patch_run(return_value=_fail("network nn already exists")): - # No SystemExit. - create_bundle_network("nn", "192.168.50.0/24", "192.168.50.1") - - def test_create_other_failure_is_fatal(self): - with self._patch_run(return_value=_fail("invalid subnet")): - with self.assertRaises(SystemExit): - create_bundle_network("nn", "bogus", "bogus") - - def test_remove_missing_network_is_idempotent(self): - # No SystemExit / no warn-and-continue noise; missing - # network is the expected case during a partial teardown. - with self._patch_run(return_value=_fail("Error: No such network: nn")): - remove_bundle_network("nn") - - def test_remove_clean_returns_success(self): - with self._patch_run(return_value=_ok()): - remove_bundle_network("nn") - - -class TestStartBundle(unittest.TestCase): - def _patch_run(self): - return patch( - "bot_bottle.backend.smolmachines.sidecar_bundle.subprocess.run", - return_value=_ok(), - ) - - def test_argv_pins_ip_on_network(self): - with self._patch_run() as m: - start_bundle(_spec()) - argv = m.call_args.args[0] - # --network NETNAME --ip on the docker run. - self.assertIn("--network", argv) - self.assertIn("bot-bottle-bundle-demo-abc12", argv) - self.assertIn("--ip", argv) - self.assertIn("192.168.50.2", argv) - # Detached and auto-removed. - self.assertIn("--detach", argv) - self.assertIn("--rm", argv) - # Container name uses the per-slug bundle prefix. - i = argv.index("--name") - self.assertEqual("bot-bottle-sidecars-demo-abc12", argv[i + 1]) - # Image at the end. - self.assertEqual("bot-bottle-sidecars:latest", argv[-1]) - - def test_daemons_env_passed_in(self): - with self._patch_run() as m: - start_bundle(_spec(daemons_csv="egress,supervise")) - argv = m.call_args.args[0] - self.assertIn("-e", argv) - self.assertIn( - "BOT_BOTTLE_SIDECAR_DAEMONS=egress,supervise", - argv, - ) - - def test_environment_entries_pass_through(self): - with self._patch_run() as m: - start_bundle(_spec(environment=( - "SUPERVISE_BOTTLE_SLUG=demo-abc12", - "EGRESS_TOKEN_0", # bare-name → host env inherit - ))) - argv = m.call_args.args[0] - self.assertIn("SUPERVISE_BOTTLE_SLUG=demo-abc12", argv) - self.assertIn("EGRESS_TOKEN_0", argv) - - def test_volumes_render_with_ro_flag(self): - with self._patch_run() as m: - start_bundle(_spec(volumes=( - ("/host/egress-ca.pem", "/home/mitmproxy/.mitmproxy/mitmproxy-ca.pem", True), - ("/host/queue", "/run/supervise/queue", False), - ))) - argv = m.call_args.args[0] - self.assertIn( - "/host/egress-ca.pem:/home/mitmproxy/.mitmproxy/mitmproxy-ca.pem:ro", - argv, - ) - self.assertIn("/host/queue:/run/supervise/queue", argv) - - def test_failure_dies(self): - with patch( - "bot_bottle.backend.smolmachines.sidecar_bundle.subprocess.run", - return_value=_fail("invalid mount"), - ): - with self.assertRaises(SystemExit): - start_bundle(_spec()) - - def test_host_env_inherited_to_subprocess(self): - # Bare-name entries in spec.environment rely on the docker - # subprocess being run with the host env. Confirm `env=` - # threads through. - with patch( - "bot_bottle.backend.smolmachines.sidecar_bundle.subprocess.run", - return_value=_ok(), - ) as m: - start_bundle(_spec(), env={"FOO": "bar"}) - self.assertEqual({"FOO": "bar"}, m.call_args.kwargs["env"]) - - class TestEnsureBundleImage(unittest.TestCase): def test_builds_sidecar_dockerfile_before_plain_docker_run(self): with patch( @@ -263,28 +125,6 @@ class TestStartBundleVm(unittest.TestCase): ) -class TestStopBundle(unittest.TestCase): - def _patch_run(self, **kwargs): # type: ignore - return patch( - "bot_bottle.backend.smolmachines.sidecar_bundle.subprocess.run", - **kwargs, - ) - - def test_argv_force_removes(self): - with self._patch_run(return_value=_ok()) as m: - stop_bundle("demo-abc12") - self.assertEqual( - ["docker", "rm", "-f", "bot-bottle-sidecars-demo-abc12"], - m.call_args.args[0], - ) - - def test_missing_container_is_idempotent(self): - with self._patch_run(return_value=_fail( - "Error: No such container: bot-bottle-sidecars-demo-abc12" - )): - stop_bundle("demo-abc12") # no raise - - class TestStopBundleVm(unittest.TestCase): def test_stops_then_deletes_sidecar_vm(self): with patch( -- 2.52.0 From 62d2e86e7eda8d787d3f6ecd6d77bcbdd39c7b00 Mon Sep 17 00:00:00 2001 From: didericis Date: Thu, 9 Jul 2026 17:05:44 -0400 Subject: [PATCH 6/9] fix(smolmachines): mount git-gate files at boot via virtiofs staging dirs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two issues caused krun_start_enter -22 (VM boot crash): 1. git-gate entrypoint missing at boot: sidecar_init.py starts all daemons (including git-gate) immediately as PID 1. The entrypoint was copied in via machine_cp which only runs after machine_start returns — too late. virtiofs also can't mount files at root (/). Fix: bake a static /git-gate-entrypoint.sh wrapper into the Dockerfile.sidecars image that delegates to /etc/git-gate/entrypoint.sh. For smolmachines, stage per-bottle scripts with correct in-VM names under the git-gate state dir and virtiofs-mount to /etc/git-gate/ at VM creation time. docker/macOS backends are unchanged (their file bind-mount/docker cp still overrides the static wrapper). 2. Egress confdir mounted read-only: egress_entrypoint.sh writes combined-trust.pem to confdir under set -e; mitmproxy also needs to write its per-host cert cache there. Both fail fatally on a read-only virtiofs mount. Fix: change the egress confdir virtiofs mount to writable. Co-Authored-By: Claude Sonnet 4.6 --- Dockerfile.sidecars | 18 +++- bot_bottle/backend/smolmachines/launch.py | 102 ++++++++++++++++------ tests/unit/test_smolmachines_provision.py | 13 ++- 3 files changed, 102 insertions(+), 31 deletions(-) diff --git a/Dockerfile.sidecars b/Dockerfile.sidecars index 89b33b3..c287391 100644 --- a/Dockerfile.sidecars +++ b/Dockerfile.sidecars @@ -14,9 +14,10 @@ # /app/supervise_server.py + .py supervise MCP server # /app/sidecar_init.py PID 1 supervisor # /etc/egress/routes.yaml bind-mounted at run time -# /etc/git-gate/pre-receive docker-cp'd at start time -# /git-gate-entrypoint.sh docker-cp'd at start time -# /git-gate/creds/* docker-cp'd at start time +# /etc/git-gate/entrypoint.sh per-bottle (docker-cp or virtiofs mount) +# /etc/git-gate/pre-receive per-bottle (docker-cp or virtiofs mount) +# /git-gate-entrypoint.sh static wrapper → /etc/git-gate/entrypoint.sh +# /git-gate/creds/* per-bottle (docker-cp or virtiofs mount) # /git/* bare repos, populated at runtime # /run/supervise/bot-bottle.db bind-mounted at run time # /home/mitmproxy/.mitmproxy/ mitmproxy CA dir @@ -90,6 +91,17 @@ RUN mkdir -p \ /run/supervise \ /home/mitmproxy/.mitmproxy +# Static wrapper for the git-gate entrypoint. The per-bottle +# entrypoint script is either: +# - docker-cp'd to /git-gate-entrypoint.sh (docker/macOS backends), +# which overwrites this wrapper; or +# - virtiofs-mounted at /etc/git-gate/entrypoint.sh (smolmachines), +# where this wrapper delegates to it at runtime. +# Either way sidecar_init.py calls `/bin/sh /git-gate-entrypoint.sh`. +RUN printf '#!/bin/sh\nexec /etc/git-gate/entrypoint.sh "$@"\n' \ + > /git-gate-entrypoint.sh \ + && chmod 755 /git-gate-entrypoint.sh + # Documentation only — the compose renderer publishes whichever # subset the bottle uses. EXPOSE 8888 9099 9418 9420 9100 diff --git a/bot_bottle/backend/smolmachines/launch.py b/bot_bottle/backend/smolmachines/launch.py index 1cb1f4a..c40d685 100644 --- a/bot_bottle/backend/smolmachines/launch.py +++ b/bot_bottle/backend/smolmachines/launch.py @@ -14,6 +14,7 @@ from __future__ import annotations import dataclasses import os +import shutil from contextlib import ExitStack, contextmanager from pathlib import Path from typing import Callable, Generator @@ -32,12 +33,7 @@ from ..docker.egress import ( EGRESS_PORT as _EGRESS_PORT, egress_tls_init, ) -from ..docker.git_gate import ( - GIT_GATE_ACCESS_HOOK_IN_CONTAINER, - GIT_GATE_CREDS_DIR_IN_CONTAINER, - GIT_GATE_ENTRYPOINT_IN_CONTAINER, - GIT_GATE_HOOK_IN_CONTAINER, -) +from ..docker.git_gate import GIT_GATE_CREDS_DIR_IN_CONTAINER from ...git_gate import ( provision_git_gate_dynamic_keys, revoke_git_gate_provisioned_keys, @@ -61,6 +57,14 @@ from .local_registry import crane_push_tarball, ephemeral_registry _REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent) +# In-VM directory containing git-gate scripts staged for virtiofs mount. +# The Dockerfile.sidecars static wrapper at /git-gate-entrypoint.sh +# delegates to this path, so the smolmachines backend can supply the +# per-bottle entrypoint via a directory virtiofs mount instead of a +# post-boot machine_cp (which runs too late for PID 1 daemons). +_GIT_GATE_SCRIPTS_DIR_IN_VM = "/etc/git-gate" + + # Per-host cache for `smolvm pack create` outputs. Keyed by the # docker image ID so a Dockerfile change automatically invalidates # the cache. `pack create` is idempotent on the smolvm side but @@ -358,6 +362,47 @@ def _port_for_label(label: str) -> int: raise ValueError(f"unknown sidecar forward label: {label}") +def _stage_git_gate_for_smolvm( + plan: SmolmachinesBottlePlan, +) -> tuple[Path, Path]: + """Stage git-gate scripts and credentials as virtiofs-mountable dirs. + + virtiofs requires directory sources; the per-bottle git-gate scripts + have host-side names that differ from their in-VM paths (e.g. + `git_gate_entrypoint.sh` → `entrypoint.sh` in /etc/git-gate/), so + they cannot be expressed as parent-dir mounts without renaming. + + Copies them into per-bottle staging dirs under the git-gate state + dir. Returns `(scripts_dir, creds_dir)`: + - scripts_dir is mounted to /etc/git-gate/ in the sidecar VM; + the image's static /git-gate-entrypoint.sh wrapper delegates there. + - creds_dir is mounted to /git-gate/creds/ in the sidecar VM.""" + gp = plan.git_gate_plan + state_dir = git_gate_state_dir(plan.slug) + + scripts_dir = state_dir / "smolvm-scripts" + scripts_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(str(gp.entrypoint_script), str(scripts_dir / "entrypoint.sh")) + shutil.copy2(str(gp.hook_script), str(scripts_dir / "pre-receive")) + shutil.copy2(str(gp.access_hook_script), str(scripts_dir / "access-hook")) + for name in ("entrypoint.sh", "pre-receive", "access-hook"): + (scripts_dir / name).chmod(0o755) + + creds_dir = state_dir / "smolvm-creds" + creds_dir.mkdir(parents=True, exist_ok=True) + for u in gp.upstreams: + keypath = Path(expand_tilde(u.identity_file)) + dest_key = creds_dir / f"{u.name}-key" + shutil.copy2(str(keypath), str(dest_key)) + dest_key.chmod(0o600) + if u.known_hosts_file: + dest_kh = creds_dir / f"{u.name}-known_hosts" + shutil.copy2(str(u.known_hosts_file), str(dest_kh)) + dest_kh.chmod(0o600) + + return scripts_dir, creds_dir + + def _bundle_launch_spec( plan: SmolmachinesBottlePlan, network: str, proxy_host: str, ) -> _bundle.BundleLaunchSpec: @@ -378,33 +423,32 @@ def _bundle_launch_spec( # --- egress ----------------------------------------------- ep = plan.egress_plan - volumes.append((str(ep.mitmproxy_ca_host_path), EGRESS_CA_IN_CONTAINER, True)) + # virtiofs (smolvm) requires directory mounts — mount the CA's + # parent dir so mitmproxy-ca.pem lands at the right in-VM path. + # Writable so egress_entrypoint.sh can write combined-trust.pem + # and mitmproxy can create its per-host cert cache. + volumes.append(( + str(ep.mitmproxy_ca_host_path.parent), + str(Path(EGRESS_CA_IN_CONTAINER).parent), + False, + )) if ep.routes: volumes.append((str(ep.routes_path.parent), str(Path(EGRESS_ROUTES_IN_CONTAINER).parent), True)) env.extend(egress_sidecar_env_entries(ep)) # --- git-gate --------------------------------------------- + # virtiofs requires directory mounts; git-gate scripts have + # host-side names that differ from their in-VM paths. Stage + # them with correct names under the git-gate state dir and + # mount those staging dirs. The image's static wrapper at + # /git-gate-entrypoint.sh delegates to the staged entrypoint, + # so the per-bottle script is available at VM boot time. gp = plan.git_gate_plan if gp.upstreams: daemons += ["git-gate", "git-http"] - volumes += [ - (str(gp.entrypoint_script), GIT_GATE_ENTRYPOINT_IN_CONTAINER, True), - (str(gp.hook_script), GIT_GATE_HOOK_IN_CONTAINER, True), - (str(gp.access_hook_script), GIT_GATE_ACCESS_HOOK_IN_CONTAINER, True), - ] - for u in gp.upstreams: - keypath = expand_tilde(u.identity_file) - volumes.append(( - keypath, - f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{u.name}-key", - True, - )) - if u.known_hosts_file: - volumes.append(( - str(u.known_hosts_file), - f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{u.name}-known_hosts", - True, - )) + scripts_dir, creds_dir = _stage_git_gate_for_smolvm(plan) + volumes.append((str(scripts_dir), _GIT_GATE_SCRIPTS_DIR_IN_VM, True)) + volumes.append((str(creds_dir), GIT_GATE_CREDS_DIR_IN_CONTAINER, True)) # --- supervise -------------------------------------------- sp = plan.supervise_plan @@ -415,7 +459,13 @@ def _bundle_launch_spec( f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", f"SUPERVISE_PORT={SUPERVISE_PORT}", ] - volumes.append((str(sp.db_path), DB_PATH_IN_CONTAINER, False)) + # virtiofs requires directory mount — mount the DB's parent + # dir so bot-bottle.db lands at the right in-VM path. + volumes.append(( + str(sp.db_path.parent), + str(Path(DB_PATH_IN_CONTAINER).parent), + False, + )) # Container ports the agent reaches from the smolvm guest — # published on `proxy_host` so the TSI allowlist and the docker diff --git a/tests/unit/test_smolmachines_provision.py b/tests/unit/test_smolmachines_provision.py index f5c2aa8..959fc87 100644 --- a/tests/unit/test_smolmachines_provision.py +++ b/tests/unit/test_smolmachines_provision.py @@ -404,7 +404,11 @@ class TestBundleLaunchSpec(unittest.TestCase): ), ) - spec = _bundle_launch_spec(plan, "net", "127.0.0.16") + with patch( + "bot_bottle.backend.smolmachines.launch._stage_git_gate_for_smolvm", + return_value=(Path("/tmp/smolvm-scripts"), Path("/tmp/smolvm-creds")), + ): + spec = _bundle_launch_spec(plan, "net", "127.0.0.16") self.assertEqual( "egress,git-gate,git-http", @@ -430,7 +434,9 @@ class TestBundleLaunchSpec(unittest.TestCase): spec = _bundle_launch_spec(plan, "net", "127.0.0.16") self.assertIn("supervise", spec.daemons_csv) self.assertIn(f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", spec.environment) - self.assertIn(("/tmp/bot-bottle.db", DB_PATH_IN_CONTAINER, False), spec.volumes) + # virtiofs requires directory mounts; the DB's parent dir is + # mounted so bot-bottle.db lands at the right in-VM path. + self.assertIn(("/tmp", str(Path(DB_PATH_IN_CONTAINER).parent), False), spec.volumes) def test_canary_env_visible_to_smolvm_guest(self): plan = _plan(canary=True) @@ -556,6 +562,9 @@ class TestLaunchResourceWiring(unittest.TestCase): "bot_bottle.backend.smolmachines.launch._bundle.start_bundle_vm", return_value=raw_launch, ) as start_vm, patch( + "bot_bottle.backend.smolmachines.launch._stage_git_gate_for_smolvm", + return_value=(Path("/tmp/smolvm-scripts"), Path("/tmp/smolvm-creds")), + ), patch( "bot_bottle.backend.smolmachines.launch._forward.start_forwarder", return_value=handle, ) as start_forwarder: -- 2.52.0 From 9c4400cce25160d6b7746f82822d3390da08ef93 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 10 Jul 2026 16:13:52 -0400 Subject: [PATCH 7/9] fix(smolmachines): consolidate virtiofs mounts to stay within libkrun limit libkrun limits total mounts + port-mappings to 5. With all three sidecar daemons active (egress, git-gate, supervise), the prior layout used 3 mounts + 3 ports = 6, causing the VM to crash at boot with krun_start_enter returned -22. Merge the egress confdir and git-gate scripts into a single staging directory mounted at /bot-bottle-data/ with egress/ and git-gate/ subdirectories, reducing to 2 mounts + 3 ports = 5. Also clean up leftover VMs before creating new ones to handle interrupted teardowns. Co-Authored-By: Claude Opus 4.6 --- Dockerfile.sidecars | 11 +- .../backend/smolmachines/egress_apply.py | 19 ++- bot_bottle/backend/smolmachines/launch.py | 148 ++++++++++-------- .../backend/smolmachines/sidecar_bundle.py | 10 ++ bot_bottle/egress_entrypoint.sh | 2 +- tests/unit/test_smolmachines_egress_apply.py | 2 +- tests/unit/test_smolmachines_provision.py | 20 ++- 7 files changed, 134 insertions(+), 78 deletions(-) diff --git a/Dockerfile.sidecars b/Dockerfile.sidecars index c287391..a7f309b 100644 --- a/Dockerfile.sidecars +++ b/Dockerfile.sidecars @@ -89,16 +89,19 @@ RUN mkdir -p \ /git-gate/creds \ /git \ /run/supervise \ - /home/mitmproxy/.mitmproxy + /home/mitmproxy/.mitmproxy \ + /bot-bottle-data/egress \ + /bot-bottle-data/git-gate # Static wrapper for the git-gate entrypoint. The per-bottle # entrypoint script is either: # - docker-cp'd to /git-gate-entrypoint.sh (docker/macOS backends), # which overwrites this wrapper; or -# - virtiofs-mounted at /etc/git-gate/entrypoint.sh (smolmachines), -# where this wrapper delegates to it at runtime. +# - virtiofs-mounted at /bot-bottle-data/git-gate/entrypoint.sh +# (smolmachines), where this wrapper delegates to it at runtime. +# Fallback to /etc/git-gate/ for backwards compatibility. # Either way sidecar_init.py calls `/bin/sh /git-gate-entrypoint.sh`. -RUN printf '#!/bin/sh\nexec /etc/git-gate/entrypoint.sh "$@"\n' \ +RUN printf '#!/bin/sh\nif [ -x /bot-bottle-data/git-gate/entrypoint.sh ]; then\n exec /bot-bottle-data/git-gate/entrypoint.sh "$@"\nfi\nexec /etc/git-gate/entrypoint.sh "$@"\n' \ > /git-gate-entrypoint.sh \ && chmod 755 /git-gate-entrypoint.sh diff --git a/bot_bottle/backend/smolmachines/egress_apply.py b/bot_bottle/backend/smolmachines/egress_apply.py index 661a13f..63d28db 100644 --- a/bot_bottle/backend/smolmachines/egress_apply.py +++ b/bot_bottle/backend/smolmachines/egress_apply.py @@ -7,16 +7,23 @@ exec`` instead of Docker. from __future__ import annotations -from ...egress import EGRESS_ROUTES_IN_CONTAINER +from pathlib import Path + +from ...bottle_state import egress_state_dir from ...log import warn from ..egress_apply import EgressApplicator, EgressApplyError from . import sidecar_bundle as _bundle from . import smolvm as _smolvm +# Routes file path inside the sidecar VM. Set via EGRESS_ROUTES env var +# at launch so the addon reads from the virtiofs-mounted confdir instead +# of the default /etc/egress/routes.yaml. +_EGRESS_ROUTES_IN_SIDECAR_VM = "/bot-bottle-data/egress/routes.yaml" + def fetch_current_routes(slug: str) -> str: machine = _bundle.bundle_machine_name(slug) - result = _smolvm.machine_exec(machine, ["cat", EGRESS_ROUTES_IN_CONTAINER]) + result = _smolvm.machine_exec(machine, ["cat", _EGRESS_ROUTES_IN_SIDECAR_VM]) if result.returncode != 0: raise EgressApplyError( f"could not read routes.yaml from {machine}: " @@ -26,6 +33,14 @@ def fetch_current_routes(slug: str) -> str: class SmolmachinesEgressApplicator(EgressApplicator): + @staticmethod + def _routes_path(slug: str) -> Path: + # Routes live in the smolvm-sidecar-data staging dir, which is + # virtiofs-mounted at /bot-bottle-data/ in the sidecar VM. Writes + # here are visible inside the VM immediately; a SIGHUP causes the + # addon to reload from _EGRESS_ROUTES_IN_SIDECAR_VM. + return egress_state_dir(slug) / "smolvm-sidecar-data" / "egress" / "routes.yaml" + def _signal_bundle_reload(self, slug: str) -> None: machine = _bundle.bundle_machine_name(slug) result = _smolvm.machine_exec(machine, ["sh", "-c", "kill -HUP 1"]) diff --git a/bot_bottle/backend/smolmachines/launch.py b/bot_bottle/backend/smolmachines/launch.py index c40d685..7227d23 100644 --- a/bot_bottle/backend/smolmachines/launch.py +++ b/bot_bottle/backend/smolmachines/launch.py @@ -20,7 +20,6 @@ from pathlib import Path from typing import Callable, Generator from ...egress import ( - EGRESS_ROUTES_IN_CONTAINER, egress_agent_env_entries, egress_resolve_token_values, egress_sidecar_env_entries, @@ -29,11 +28,9 @@ from ...supervise import DB_PATH_IN_CONTAINER, SUPERVISE_PORT from ...util import expand_tilde from ..docker import util as docker_mod from ..docker.egress import ( - EGRESS_CA_IN_CONTAINER, EGRESS_PORT as _EGRESS_PORT, egress_tls_init, ) -from ..docker.git_gate import GIT_GATE_CREDS_DIR_IN_CONTAINER from ...git_gate import ( provision_git_gate_dynamic_keys, revoke_git_gate_provisioned_keys, @@ -57,12 +54,16 @@ from .local_registry import crane_push_tarball, ephemeral_registry _REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent) -# In-VM directory containing git-gate scripts staged for virtiofs mount. -# The Dockerfile.sidecars static wrapper at /git-gate-entrypoint.sh -# delegates to this path, so the smolmachines backend can supply the -# per-bottle entrypoint via a directory virtiofs mount instead of a -# post-boot machine_cp (which runs too late for PID 1 daemons). -_GIT_GATE_SCRIPTS_DIR_IN_VM = "/etc/git-gate" +# Single virtiofs mount for egress + git-gate files. libkrun limits +# the total of mounts + port-mappings to 5; with 3 daemon ports the +# sidecar VM can carry at most 2 mounts. Egress CA/routes and +# git-gate scripts/creds are staged into subdirectories of one host +# dir and mounted here. Env vars (EGRESS_CONFDIR, EGRESS_ROUTES, +# and the Dockerfile's git-gate wrapper) point each daemon at its +# subdirectory. +_SIDECAR_DATA_DIR_IN_VM = "/bot-bottle-data" +_EGRESS_CONFDIR_IN_VM = f"{_SIDECAR_DATA_DIR_IN_VM}/egress" +_GIT_GATE_SCRIPTS_DIR_IN_VM = f"{_SIDECAR_DATA_DIR_IN_VM}/git-gate" # Per-host cache for `smolvm pack create` outputs. Keyed by the @@ -295,6 +296,16 @@ def _launch_vm( fails closed if it can't. Smolfile isn't usable here — smolvm 0.8.0 makes --from and --smolfile mutually exclusive.""" tsi_cidr = f"{proxy_host}/32" + # Destroy any leftover machine from a previous run that didn't + # clean up (e.g. crash, interrupted teardown). + try: + _smolvm.machine_stop(plan.machine_name) + except _smolvm.SmolvmError: + pass + try: + _smolvm.machine_delete(plan.machine_name) + except _smolvm.SmolvmError: + pass _smolvm.machine_create( plan.machine_name, from_path=agent_from_path, @@ -362,45 +373,62 @@ def _port_for_label(label: str) -> int: raise ValueError(f"unknown sidecar forward label: {label}") -def _stage_git_gate_for_smolvm( - plan: SmolmachinesBottlePlan, -) -> tuple[Path, Path]: - """Stage git-gate scripts and credentials as virtiofs-mountable dirs. +def _stage_sidecar_data(plan: SmolmachinesBottlePlan) -> Path: + """Stage egress + git-gate files into one virtiofs-mountable dir. - virtiofs requires directory sources; the per-bottle git-gate scripts - have host-side names that differ from their in-VM paths (e.g. - `git_gate_entrypoint.sh` → `entrypoint.sh` in /etc/git-gate/), so - they cannot be expressed as parent-dir mounts without renaming. + libkrun limits total mounts + port-mappings to 5. With 3 daemon + ports the sidecar VM can carry at most 2 mounts (the second is + the supervise DB). Egress and git-gate share a single mount: - Copies them into per-bottle staging dirs under the git-gate state - dir. Returns `(scripts_dir, creds_dir)`: - - scripts_dir is mounted to /etc/git-gate/ in the sidecar VM; - the image's static /git-gate-entrypoint.sh wrapper delegates there. - - creds_dir is mounted to /git-gate/creds/ in the sidecar VM.""" + /egress/ → _EGRESS_CONFDIR_IN_VM + /git-gate/ → _GIT_GATE_SCRIPTS_DIR_IN_VM + + The mount is writable so mitmproxy can write combined-trust.pem + and cache per-host certs under the egress subdir.""" + staging = egress_state_dir(plan.slug) / "smolvm-sidecar-data" + + # --- egress subdir --- + confdir = staging / "egress" + confdir.mkdir(parents=True, exist_ok=True) + ep = plan.egress_plan + shutil.copy2(str(ep.mitmproxy_ca_host_path), str(confdir / "mitmproxy-ca.pem")) + if ep.routes: + shutil.copy2(str(ep.routes_path), str(confdir / "routes.yaml")) + + # --- git-gate subdir (only when upstreams are configured) --- gp = plan.git_gate_plan - state_dir = git_gate_state_dir(plan.slug) + if gp.upstreams: + scripts_dir = staging / "git-gate" + scripts_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(str(gp.entrypoint_script), str(scripts_dir / "entrypoint.sh")) + shutil.copy2(str(gp.hook_script), str(scripts_dir / "pre-receive")) + shutil.copy2(str(gp.access_hook_script), str(scripts_dir / "access-hook")) + for name in ("entrypoint.sh", "pre-receive", "access-hook"): + (scripts_dir / name).chmod(0o755) - scripts_dir = state_dir / "smolvm-scripts" - scripts_dir.mkdir(parents=True, exist_ok=True) - shutil.copy2(str(gp.entrypoint_script), str(scripts_dir / "entrypoint.sh")) - shutil.copy2(str(gp.hook_script), str(scripts_dir / "pre-receive")) - shutil.copy2(str(gp.access_hook_script), str(scripts_dir / "access-hook")) - for name in ("entrypoint.sh", "pre-receive", "access-hook"): - (scripts_dir / name).chmod(0o755) + # Patch credential paths: the rendered entrypoint hardcodes + # /git-gate/creds/; rewrite to the in-VM git-gate subdir. + ep_path = scripts_dir / "entrypoint.sh" + ep_path.write_text( + ep_path.read_text().replace( + "/git-gate/creds/", + f"{_GIT_GATE_SCRIPTS_DIR_IN_VM}/creds/", + ) + ) - creds_dir = state_dir / "smolvm-creds" - creds_dir.mkdir(parents=True, exist_ok=True) - for u in gp.upstreams: - keypath = Path(expand_tilde(u.identity_file)) - dest_key = creds_dir / f"{u.name}-key" - shutil.copy2(str(keypath), str(dest_key)) - dest_key.chmod(0o600) - if u.known_hosts_file: - dest_kh = creds_dir / f"{u.name}-known_hosts" - shutil.copy2(str(u.known_hosts_file), str(dest_kh)) - dest_kh.chmod(0o600) + creds_dir = scripts_dir / "creds" + creds_dir.mkdir(exist_ok=True) + for u in gp.upstreams: + keypath = Path(expand_tilde(u.identity_file)) + dest_key = creds_dir / f"{u.name}-key" + shutil.copy2(str(keypath), str(dest_key)) + dest_key.chmod(0o600) + if u.known_hosts_file: + dest_kh = creds_dir / f"{u.name}-known_hosts" + shutil.copy2(str(u.known_hosts_file), str(dest_kh)) + dest_kh.chmod(0o600) - return scripts_dir, creds_dir + return staging def _bundle_launch_spec( @@ -421,34 +449,26 @@ def _bundle_launch_spec( env: list[str] = [] volumes: list[tuple[str, str, bool]] = [] - # --- egress ----------------------------------------------- - ep = plan.egress_plan - # virtiofs (smolvm) requires directory mounts — mount the CA's - # parent dir so mitmproxy-ca.pem lands at the right in-VM path. + # --- egress + git-gate (single mount) --------------------- + # Stage both into one dir and mount it at _SIDECAR_DATA_DIR_IN_VM. + # libkrun limits mounts + port-mappings to 5; with 3 daemon ports + # we can carry at most 2 mounts (this one + supervise DB). # Writable so egress_entrypoint.sh can write combined-trust.pem # and mitmproxy can create its per-host cert cache. - volumes.append(( - str(ep.mitmproxy_ca_host_path.parent), - str(Path(EGRESS_CA_IN_CONTAINER).parent), - False, - )) - if ep.routes: - volumes.append((str(ep.routes_path.parent), str(Path(EGRESS_ROUTES_IN_CONTAINER).parent), True)) + ep = plan.egress_plan + gp = plan.git_gate_plan + staging = _stage_sidecar_data(plan) + volumes.append((str(staging), _SIDECAR_DATA_DIR_IN_VM, False)) + # Tell the egress entrypoint where to find its CA + routes. + env.append(f"EGRESS_CONFDIR={_EGRESS_CONFDIR_IN_VM}") + # Always set EGRESS_ROUTES so the addon reads from the confdir path + # even when no routes were configured at launch (apply_routes_change + # writes here and a SIGHUP causes the addon to pick them up). + env.append(f"EGRESS_ROUTES={_EGRESS_CONFDIR_IN_VM}/routes.yaml") env.extend(egress_sidecar_env_entries(ep)) - # --- git-gate --------------------------------------------- - # virtiofs requires directory mounts; git-gate scripts have - # host-side names that differ from their in-VM paths. Stage - # them with correct names under the git-gate state dir and - # mount those staging dirs. The image's static wrapper at - # /git-gate-entrypoint.sh delegates to the staged entrypoint, - # so the per-bottle script is available at VM boot time. - gp = plan.git_gate_plan if gp.upstreams: daemons += ["git-gate", "git-http"] - scripts_dir, creds_dir = _stage_git_gate_for_smolvm(plan) - volumes.append((str(scripts_dir), _GIT_GATE_SCRIPTS_DIR_IN_VM, True)) - volumes.append((str(creds_dir), GIT_GATE_CREDS_DIR_IN_CONTAINER, True)) # --- supervise -------------------------------------------- sp = plan.supervise_plan diff --git a/bot_bottle/backend/smolmachines/sidecar_bundle.py b/bot_bottle/backend/smolmachines/sidecar_bundle.py index 6118014..8bb9a66 100644 --- a/bot_bottle/backend/smolmachines/sidecar_bundle.py +++ b/bot_bottle/backend/smolmachines/sidecar_bundle.py @@ -135,6 +135,16 @@ def start_bundle_vm( elif entry in effective_host_env: env[entry] = effective_host_env[entry] name = bundle_machine_name(spec.slug) + # Destroy any leftover machine from a previous run that didn't + # clean up (e.g. crash, interrupted teardown). + try: + _smolvm.machine_stop(name) + except _smolvm.SmolvmError: + pass + try: + _smolvm.machine_delete(name) + except _smolvm.SmolvmError: + pass _smolvm.machine_create( name, from_path=from_path, diff --git a/bot_bottle/egress_entrypoint.sh b/bot_bottle/egress_entrypoint.sh index fb64ff4..f2189c1 100644 --- a/bot_bottle/egress_entrypoint.sh +++ b/bot_bottle/egress_entrypoint.sh @@ -28,7 +28,7 @@ set -e # flag mitmdump would generate a fresh CA on the wrong path and # the agent's installed trust anchor would no longer match the # bumped leaf certs. -CONFDIR=/home/mitmproxy/.mitmproxy +CONFDIR="${EGRESS_CONFDIR:-/home/mitmproxy/.mitmproxy}" CONFDIR_FLAG="--set confdir=$CONFDIR" MODE="--mode regular@9099" diff --git a/tests/unit/test_smolmachines_egress_apply.py b/tests/unit/test_smolmachines_egress_apply.py index 3bb2d25..309babb 100644 --- a/tests/unit/test_smolmachines_egress_apply.py +++ b/tests/unit/test_smolmachines_egress_apply.py @@ -18,7 +18,7 @@ class TestFetchCurrentRoutes(unittest.TestCase): self.assertEqual("routes", egress_apply.fetch_current_routes("dev-abc")) exec_.assert_called_once_with( "bot-bottle-sidecars-dev-abc", - ["cat", "/etc/egress/routes.yaml"], + ["cat", "/bot-bottle-data/egress/routes.yaml"], ) def test_read_failure_raises_apply_error(self): diff --git a/tests/unit/test_smolmachines_provision.py b/tests/unit/test_smolmachines_provision.py index 959fc87..f8a2c15 100644 --- a/tests/unit/test_smolmachines_provision.py +++ b/tests/unit/test_smolmachines_provision.py @@ -405,8 +405,8 @@ class TestBundleLaunchSpec(unittest.TestCase): ) with patch( - "bot_bottle.backend.smolmachines.launch._stage_git_gate_for_smolvm", - return_value=(Path("/tmp/smolvm-scripts"), Path("/tmp/smolvm-creds")), + "bot_bottle.backend.smolmachines.launch._stage_sidecar_data", + return_value=Path("/tmp/smolvm-sidecar-data"), ): spec = _bundle_launch_spec(plan, "net", "127.0.0.16") @@ -420,7 +420,11 @@ class TestBundleLaunchSpec(unittest.TestCase): def test_canary_env_registered_as_sensitive_in_bundle(self): plan = _plan(canary=True) - spec = _bundle_launch_spec(plan, "net", "127.0.0.16") + with patch( + "bot_bottle.backend.smolmachines.launch._stage_sidecar_data", + return_value=Path("/tmp/smolvm-sidecar-data"), + ): + spec = _bundle_launch_spec(plan, "net", "127.0.0.16") self.assertIn("CANON_ALPHA_SECRET=fake-canary-value", spec.environment) self.assertIn( @@ -431,7 +435,11 @@ class TestBundleLaunchSpec(unittest.TestCase): def test_supervise_adds_daemon_volume_and_env(self): from bot_bottle.supervise import DB_PATH_IN_CONTAINER plan = _plan(supervise=True) - spec = _bundle_launch_spec(plan, "net", "127.0.0.16") + with patch( + "bot_bottle.backend.smolmachines.launch._stage_sidecar_data", + return_value=Path("/tmp/smolvm-sidecar-data"), + ): + spec = _bundle_launch_spec(plan, "net", "127.0.0.16") self.assertIn("supervise", spec.daemons_csv) self.assertIn(f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", spec.environment) # virtiofs requires directory mounts; the DB's parent dir is @@ -562,8 +570,8 @@ class TestLaunchResourceWiring(unittest.TestCase): "bot_bottle.backend.smolmachines.launch._bundle.start_bundle_vm", return_value=raw_launch, ) as start_vm, patch( - "bot_bottle.backend.smolmachines.launch._stage_git_gate_for_smolvm", - return_value=(Path("/tmp/smolvm-scripts"), Path("/tmp/smolvm-creds")), + "bot_bottle.backend.smolmachines.launch._stage_sidecar_data", + return_value=Path("/tmp/smolvm-sidecar-data"), ), patch( "bot_bottle.backend.smolmachines.launch._forward.start_forwarder", return_value=handle, -- 2.52.0 From d76ce89c873b01be621c7a7fb517bf37bb1ae872 Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 10 Jul 2026 17:01:23 -0400 Subject: [PATCH 8/9] fix(smolmachines): Linux compat for TSI networking and git-gate paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes for Linux smolvm support: 1. Skip force_allowlist DB patch on Linux — patching allowed_cidrs into smolvm's state DB crashes TSI boot on Linux. Per-bottle CIDR isolation is not enforced on Linux until smolvm fixes the --allow-cidr + TSI interaction (known limitation). 2. Patch /etc/git-gate/ references in the staged entrypoint to /bot-bottle-data/git-gate/ so git-daemon finds its hooks and access-hook at the new consolidated mount path. 3. Drop explicit --net-backend tsi — the default on this smolvm build is already TSI; explicitly requesting it is incompatible with -p port mappings on Linux. Co-Authored-By: Claude Opus 4.6 --- bot_bottle/backend/smolmachines/launch.py | 14 ++++---- .../backend/smolmachines/loopback_alias.py | 32 ++++++++++++------- .../unit/test_smolmachines_loopback_alias.py | 8 ++--- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/bot_bottle/backend/smolmachines/launch.py b/bot_bottle/backend/smolmachines/launch.py index 7227d23..d722b4c 100644 --- a/bot_bottle/backend/smolmachines/launch.py +++ b/bot_bottle/backend/smolmachines/launch.py @@ -406,15 +406,13 @@ def _stage_sidecar_data(plan: SmolmachinesBottlePlan) -> Path: for name in ("entrypoint.sh", "pre-receive", "access-hook"): (scripts_dir / name).chmod(0o755) - # Patch credential paths: the rendered entrypoint hardcodes - # /git-gate/creds/; rewrite to the in-VM git-gate subdir. + # Patch paths: the rendered entrypoint hardcodes /git-gate/creds/ + # and /etc/git-gate/ for hooks; rewrite both to the in-VM subdir. ep_path = scripts_dir / "entrypoint.sh" - ep_path.write_text( - ep_path.read_text().replace( - "/git-gate/creds/", - f"{_GIT_GATE_SCRIPTS_DIR_IN_VM}/creds/", - ) - ) + text = ep_path.read_text() + text = text.replace("/git-gate/creds/", f"{_GIT_GATE_SCRIPTS_DIR_IN_VM}/creds/") + text = text.replace("/etc/git-gate/", f"{_GIT_GATE_SCRIPTS_DIR_IN_VM}/") + ep_path.write_text(text) creds_dir = scripts_dir / "creds" creds_dir.mkdir(exist_ok=True) diff --git a/bot_bottle/backend/smolmachines/loopback_alias.py b/bot_bottle/backend/smolmachines/loopback_alias.py index 50b4ba6..06ee6e2 100644 --- a/bot_bottle/backend/smolmachines/loopback_alias.py +++ b/bot_bottle/backend/smolmachines/loopback_alias.py @@ -152,23 +152,31 @@ def force_allowlist(machine_name: str, allowed_cidrs: list[str]) -> None: """Ensure the machine's persisted TSI allowlist equals `allowed_cidrs`, failing **closed** if that can't be confirmed. - Runs on both macOS and Linux. It exists because smolvm 0.8.0 - silently drops `--allow-cidr` when combined with `--from`, so - the allowlist has to be written into smolvm's persistent state - DB before `machine start`. Rather than assume the flag was - dropped, we read the persisted row and only patch when it - doesn't already match — so a newer smolvm that honors the flag - is left untouched. + macOS only. On Linux, smolvm's TSI defaults to full loopback + access and patching `allowed_cidrs` into the state DB crashes + TSI boot (the VM fails with "boot process exited (code 1)"). + Per-bottle CIDR isolation is therefore not enforced on Linux + until smolvm fixes the `--allow-cidr` + TSI interaction. This + is a known limitation — the agent VM can reach all of host + loopback, not just its bottle's forwarder ports. + + On macOS, smolvm 0.8.0 silently drops `--allow-cidr` when + combined with `--from`, so the allowlist has to be written + into smolvm's persistent state DB before `machine start`. + Rather than assume the flag was dropped, we read the persisted + row and only patch when it doesn't already match — so a newer + smolvm that honors the flag is left untouched. Must run AFTER `smolvm machine create` (the row has to exist) and BEFORE `smolvm machine start` (smolvm reads the row on start; in-flight VMs don't pick up changes). - Fail-closed: if the state DB is missing, the row is missing, or - the allowlist still doesn't match after patching, we `die()` - rather than boot a VM whose egress confinement we can't verify - — an unconfirmed allowlist is a sandbox-escape risk (the agent - VM could reach all of host loopback).""" + Fail-closed (macOS): if the state DB is missing, the row is + missing, or the allowlist still doesn't match after patching, + we `die()` rather than boot a VM whose egress confinement we + can't verify.""" + if not _is_macos(): + return want = list(allowed_cidrs) if not _SMOLVM_DB_PATH.is_file(): die( diff --git a/tests/unit/test_smolmachines_loopback_alias.py b/tests/unit/test_smolmachines_loopback_alias.py index 7847dc3..35540ef 100644 --- a/tests/unit/test_smolmachines_loopback_alias.py +++ b/tests/unit/test_smolmachines_loopback_alias.py @@ -313,9 +313,9 @@ class TestForceAllowlist(unittest.TestCase): self.assertEqual(4, cfg["cpus"]) self.assertTrue(cfg["network"]) - def test_patches_on_linux_too(self): - # force_allowlist no longer no-ops on Linux — the TSI - # allowlist must be enforced there as well. + def test_skips_patch_on_linux(self): + # On Linux, patching allowed_cidrs into the smolvm state DB + # crashes TSI boot. force_allowlist is a no-op on Linux. with patch.object(loopback_alias, "_is_macos", return_value=False), \ patch.object(loopback_alias, "_SMOLVM_DB_PATH", self.db): loopback_alias.force_allowlist("demo-vm", ["127.0.0.16/32"]) @@ -324,7 +324,7 @@ class TestForceAllowlist(unittest.TestCase): "SELECT data FROM vms WHERE name='demo-vm'", ).fetchone()[0]) con.close() - self.assertEqual(["127.0.0.16/32"], cfg["allowed_cidrs"]) + self.assertIsNone(cfg["allowed_cidrs"]) def test_skips_write_when_already_matching(self): # A newer smolvm that honors --allow-cidr at create leaves the -- 2.52.0 From dc9fdc856310fe5e4e56200096582de3ab4aed1d Mon Sep 17 00:00:00 2001 From: didericis Date: Fri, 10 Jul 2026 17:11:27 -0400 Subject: [PATCH 9/9] fix(smolmachines): enforce per-bottle loopback isolation on Linux via iptables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Linux, smolvm's TSI allowlist DB patch crashes boot, so force_allowlist is a no-op. This left the agent VM with full host loopback access — a security regression from the macOS behavior. Fix: install guest-side iptables rules in _init_vm that ACCEPT traffic to the bottle's allocated loopback alias (proxy_host/32) and DROP all other 127.0.0.0/8 destinations. The agent runs as non-root (node) and cannot flush the rules. - Add iptables to claude and codex agent Dockerfiles - Use DROP (not REJECT) — libkrun kernel lacks the REJECT module - Skip on macOS where force_allowlist handles it natively Co-Authored-By: Claude Opus 4.6 --- bot_bottle/backend/smolmachines/launch.py | 55 +++++++++++++++++++---- bot_bottle/contrib/claude/Dockerfile | 2 +- bot_bottle/contrib/codex/Dockerfile | 2 +- tests/unit/test_smolmachines_provision.py | 28 +++++++++++- 4 files changed, 75 insertions(+), 12 deletions(-) diff --git a/bot_bottle/backend/smolmachines/launch.py b/bot_bottle/backend/smolmachines/launch.py index d722b4c..84d40dc 100644 --- a/bot_bottle/backend/smolmachines/launch.py +++ b/bot_bottle/backend/smolmachines/launch.py @@ -99,7 +99,7 @@ def launch( agent_from_path = _agent_from_path(plan) _launch_vm(plan, agent_from_path, proxy_host, stack) - _init_vm(plan) + _init_vm(plan, proxy_host) bottle = SmolmachinesBottle( plan.machine_name, @@ -322,17 +322,24 @@ def _launch_vm( stack.callback(_smolvm.machine_stop, plan.machine_name) -def _init_vm(plan: SmolmachinesBottlePlan) -> None: - """Repair filesystem ownership and wait for exec channel readiness. +def _init_vm(plan: SmolmachinesBottlePlan, proxy_host: str) -> None: + """Repair filesystem ownership, enforce loopback isolation, and + wait for exec channel readiness. Ownership repair: smolvm's pack process remaps files to the host invoker's uid (e.g. 501 on macOS, 1000 on Linux). The chowns use names not numbers so they're correct on either. /home/node must - be node:node so - Claude Code can write ~/.claude.json; /tmp + /var/tmp need root - mode 1777 so non-root processes can create per-uid scratch dirs. - All folded into one sh -c to avoid back-to-back exec calls - immediately after machine_start (libkrun exec-channel race). + be node:node so Claude Code can write ~/.claude.json; /tmp + + /var/tmp need root mode 1777 so non-root processes can create + per-uid scratch dirs. + + Loopback isolation (Linux only): on macOS, smolvm's TSI allowlist + restricts which host loopback IPs the guest can reach. On Linux, + the allowlist DB patch crashes TSI boot, so we enforce the same + /32 scope with guest-side iptables instead. The rules allow + outbound to proxy_host/32, then reject all other 127.0.0.0/8. + Since the agent runs as non-root (node), it cannot flush these + rules. mkdir -p guards: when booting from a committed snapshot, /tmp and /var/tmp are excluded from the archive (they're ephemeral and their @@ -348,9 +355,41 @@ def _init_vm(plan: SmolmachinesBottlePlan) -> None: "chown root:root /tmp /var/tmp && " "chmod 1777 /tmp /var/tmp", ]) + if not _loopback._is_macos(): + _enforce_loopback_isolation(plan.machine_name, proxy_host) _smolvm.wait_exec_ready(plan.machine_name) +def _enforce_loopback_isolation(machine_name: str, proxy_host: str) -> None: + """Install iptables rules restricting the guest to proxy_host/32. + + On Linux, smolvm's TSI bridges the full host loopback into the + guest, and the allow-cidr DB patch is incompatible with TSI. + Guest-side iptables achieves the same per-bottle isolation: + only the allocated loopback alias is reachable, so the agent + can't probe other bottles' ports or other host services. + + Runs as root (exec default); the agent process runs as `node` + (non-root) and cannot modify iptables rules.""" + result = _smolvm.machine_exec(machine_name, [ + "sh", "-c", + # Allow the bottle's allocated loopback alias. + f"iptables -A OUTPUT -d {proxy_host}/32 -j ACCEPT && " + # Drop all other loopback destinations. DROP (not REJECT) + # because the libkrun kernel lacks the REJECT target module. + # Connections to blocked addresses time out rather than fail + # fast, which is acceptable — the agent shouldn't be probing + # non-allowed loopback addresses. + "iptables -A OUTPUT -d 127.0.0.0/8 -j DROP", + ]) + if result.returncode != 0: + warn( + f"guest iptables setup failed (exit {result.returncode}): " + f"{(result.stderr or '').strip() or ''}. " + f"Per-bottle loopback isolation is not enforced." + ) + + def _label_for_port(port: int) -> str: if port == _EGRESS_PORT: return "egress" diff --git a/bot_bottle/contrib/claude/Dockerfile b/bot_bottle/contrib/claude/Dockerfile index 4834dd3..0660281 100644 --- a/bot_bottle/contrib/claude/Dockerfile +++ b/bot_bottle/contrib/claude/Dockerfile @@ -21,7 +21,7 @@ FROM node:22-slim # to it) works against egress's bumped TLS without the agent needing # local DNS. RUN apt-get update \ - && apt-get install -y --no-install-recommends git ca-certificates curl ripgrep iproute2 dnsutils \ + && apt-get install -y --no-install-recommends git ca-certificates curl ripgrep iproute2 dnsutils iptables \ && rm -rf /var/lib/apt/lists/* # App-specific deps. Python isn't required by claude-code itself diff --git a/bot_bottle/contrib/codex/Dockerfile b/bot_bottle/contrib/codex/Dockerfile index c966b8d..8bd27f5 100644 --- a/bot_bottle/contrib/codex/Dockerfile +++ b/bot_bottle/contrib/codex/Dockerfile @@ -6,7 +6,7 @@ FROM node:22-slim RUN apt-get update \ - && apt-get install -y --no-install-recommends git ca-certificates curl procps ripgrep \ + && apt-get install -y --no-install-recommends git ca-certificates curl procps ripgrep iptables \ && rm -rf /var/lib/apt/lists/* # App-specific deps. Python isn't required by codex itself diff --git a/tests/unit/test_smolmachines_provision.py b/tests/unit/test_smolmachines_provision.py index f8a2c15..05ba8e0 100644 --- a/tests/unit/test_smolmachines_provision.py +++ b/tests/unit/test_smolmachines_provision.py @@ -601,8 +601,11 @@ class TestLaunchResourceWiring(unittest.TestCase): "bot_bottle.backend.smolmachines.launch._smolvm.machine_exec", ) as machine_exec, patch( "bot_bottle.backend.smolmachines.launch._smolvm.wait_exec_ready", - ) as wait_exec_ready: - _launch._init_vm(plan) + ) as wait_exec_ready, patch( + "bot_bottle.backend.smolmachines.launch._loopback._is_macos", + return_value=True, + ): + _launch._init_vm(plan, "127.0.0.16") machine_exec.assert_called_once() argv = machine_exec.call_args.args[1] @@ -610,6 +613,27 @@ class TestLaunchResourceWiring(unittest.TestCase): self.assertIn("chown -R node:node /home/node", argv[2]) wait_exec_ready.assert_called_once_with(plan.machine_name) + def test_init_vm_installs_iptables_on_linux(self): + plan = _plan() + from bot_bottle.backend.smolmachines.smolvm import SmolvmRunResult + with patch( + "bot_bottle.backend.smolmachines.launch._smolvm.machine_exec", + return_value=SmolvmRunResult(0, "", ""), + ) as machine_exec, patch( + "bot_bottle.backend.smolmachines.launch._smolvm.wait_exec_ready", + ), patch( + "bot_bottle.backend.smolmachines.launch._loopback._is_macos", + return_value=False, + ): + _launch._init_vm(plan, "127.0.0.16") + + # Two calls: ownership repair + iptables + self.assertEqual(2, machine_exec.call_count) + iptables_argv = machine_exec.call_args_list[1].args[1] + self.assertIn("iptables", iptables_argv[2]) + self.assertIn("127.0.0.16/32", iptables_argv[2]) + self.assertIn("127.0.0.0/8", iptables_argv[2]) + class TestPortLabels(unittest.TestCase): def test_known_and_dynamic_port_labels_round_trip(self): -- 2.52.0