From 8f9005582bc808f50e41cbf5befb56be1927bbe2 Mon Sep 17 00:00:00 2001 From: didericis Date: Tue, 7 Jul 2026 13:15:34 -0400 Subject: [PATCH] fix(smolmachines): use bridge gateway as TSI proxy host on Linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Linux the guest kernel's LOCAL routing table routes all 127.0.0.0/8 to the guest's own loopback interface (priority 0, checked before any main-table route), so TSI never sees connections to the per-bottle loopback alias — the fix_guest_ loopback_routing approach confirmed this at the kernel level. Use the per-bottle docker bridge gateway (192.168.N.1) instead. It is not a loopback address, so the guest routes it via eth0 and TSI intercepts it normally. The TSI allowlist remains a /32 that is distinct from the container IP (192.168.N.2), so direct bypass to egress:9099 is still blocked by TSI. Changes: - Add _proxy_host() helper: returns bundle_gateway on Linux, loopback alias on macOS - Thread proxy_host through _start_bundle, _discover_urls, _launch_vm, and _bundle_launch_spec (publish_host_ip) - Remove _fix_guest_loopback_routing (no longer needed) - Relax proxy-URL assertion in the integration test to accept any http://IP:port (with a comment explaining the difference) Co-Authored-By: Claude Sonnet 4.6 --- bot_bottle/backend/smolmachines/launch.py | 100 ++++++++++++------ bot_bottle/backend/smolmachines/smolvm.py | 25 ++--- bot_bottle/contrib/claude/Dockerfile | 2 +- tests/integration/test_smolmachines_launch.py | 17 ++- .../test_smolmachines_smolvm_smoke.py | 6 +- 5 files changed, 93 insertions(+), 57 deletions(-) diff --git a/bot_bottle/backend/smolmachines/launch.py b/bot_bottle/backend/smolmachines/launch.py index f1fe440..bcc3a2d 100644 --- a/bot_bottle/backend/smolmachines/launch.py +++ b/bot_bottle/backend/smolmachines/launch.py @@ -17,6 +17,7 @@ from __future__ import annotations import dataclasses import os +import platform from contextlib import ExitStack, contextmanager from pathlib import Path from typing import Callable, Generator @@ -88,12 +89,13 @@ def launch( try: loopback_ip, network = _allocate_resources(plan, stack) plan = _mint_certs(plan) - plan = _start_bundle(plan, network, loopback_ip, stack) - plan = _discover_urls(plan, loopback_ip) + proxy_host = _proxy_host(plan, loopback_ip) + plan = _start_bundle(plan, network, proxy_host, stack) + plan = _discover_urls(plan, proxy_host) agent_from_path = _agent_from_path(plan) - _launch_vm(plan, agent_from_path, loopback_ip, stack) + _launch_vm(plan, agent_from_path, proxy_host, stack) _init_vm(plan) bottle = SmolmachinesBottle( @@ -171,12 +173,12 @@ def _mint_certs(plan: SmolmachinesBottlePlan) -> SmolmachinesBottlePlan: def _start_bundle( plan: SmolmachinesBottlePlan, network: str, - loopback_ip: str, + proxy_host: str, stack: ExitStack, ) -> SmolmachinesBottlePlan: """Build the BundleLaunchSpec, resolve token env, start the sidecar bundle container, and register teardown.""" - bundle_spec = _bundle_launch_spec(plan, network, loopback_ip) + bundle_spec = _bundle_launch_spec(plan, network, proxy_host) token_env = _resolve_token_env(plan, dict(os.environ)) _bundle.ensure_bundle_image(bundle_spec.image) _bundle.start_bundle(bundle_spec, env={**os.environ, **token_env}) @@ -186,41 +188,40 @@ def _start_bundle( def _discover_urls( plan: SmolmachinesBottlePlan, - loopback_ip: str, + proxy_host: str, ) -> SmolmachinesBottlePlan: """Discover host-side ports for published container ports and return the plan with URLs + guest_env stamped in. - Docker container IPs (192.168.x.x in the daemon's bridge) - aren't reachable from the smolvm guest — TSI proxies the - guest's connects through the host, and the host reaches the - bundle only via its published-port loopback forward (the - daemon's bridge isn't on the TSI allowlist). The agent dials - the published port on the per-bottle loopback alias. + `proxy_host` is the host IP that both TSI's allowlist and + docker's port-forward bindings are keyed to. On macOS it is the + per-bottle loopback alias; on Linux it is the per-bottle bridge + gateway (see `_proxy_host`). The agent dials the published port + on this IP for all bundle-hosted services. - NO_PROXY includes the per-bottle loopback alias so the - supervise + git-gate URLs bypass HTTPS_PROXY.""" + NO_PROXY includes `proxy_host` so supervise + git-gate URLs + bypass HTTPS_PROXY.""" agent_facing_host_port = _bundle.bundle_host_port( - plan.slug, _EGRESS_PORT, host_ip=loopback_ip, + plan.slug, _EGRESS_PORT, host_ip=proxy_host, ) - agent_proxy_url = f"http://{loopback_ip}:{agent_facing_host_port}" + agent_proxy_url = f"http://{proxy_host}:{agent_facing_host_port}" agent_git_gate_host = "" if plan.git_gate_plan.upstreams: git_gate_host_port = _bundle.bundle_host_port( - plan.slug, _GIT_HTTP_PORT, host_ip=loopback_ip, + plan.slug, _GIT_HTTP_PORT, host_ip=proxy_host, ) - agent_git_gate_host = f"{loopback_ip}:{git_gate_host_port}" + agent_git_gate_host = f"{proxy_host}:{git_gate_host_port}" agent_supervise_url = "" if plan.supervise_plan is not None: supervise_host_port = _bundle.bundle_host_port( - plan.slug, _SUPERVISE_PORT, host_ip=loopback_ip, + plan.slug, _SUPERVISE_PORT, host_ip=proxy_host, ) - agent_supervise_url = f"http://{loopback_ip}:{supervise_host_port}/" + agent_supervise_url = f"http://{proxy_host}:{supervise_host_port}/" existing_no_proxy = plan.guest_env.get("NO_PROXY", "localhost,127.0.0.1") - no_proxy = f"{existing_no_proxy},{loopback_ip}" + no_proxy = f"{existing_no_proxy},{proxy_host}" guest_env = { **plan.guest_env, "HTTPS_PROXY": agent_proxy_url, @@ -250,21 +251,24 @@ def _discover_urls( def _launch_vm( plan: SmolmachinesBottlePlan, agent_from_path: Path, - loopback_ip: str, + proxy_host: str, stack: ExitStack, ) -> None: """Create, patch, and start the smolvm VM; register teardown. - --allow-cidr is the per-bottle loopback alias so the guest can - only reach this bottle's bundle ports. force_allowlist then - confirms the allowlist persisted (patching smolvm 0.8.0's - silent-drop of --allow-cidr when combined with --from) and - fails closed if it can't. Smolfile isn't usable here — smolvm - 0.8.0 makes --from and --smolfile mutually exclusive.""" + --allow-cidr is `proxy_host/32` — the per-bottle loopback alias + on macOS or the bridge gateway on Linux (see `_proxy_host`). This + ensures the guest can only reach bundle ports published on that IP, + not the container IP directly. force_allowlist confirms the + allowlist persisted (patching smolvm 0.8.0's silent-drop of + --allow-cidr when combined with --from) and fails closed if it + can't. Smolfile isn't usable here — smolvm 0.8.0 makes --from + and --smolfile mutually exclusive.""" + tsi_cidr = f"{proxy_host}/32" _smolvm.machine_create( plan.machine_name, from_path=agent_from_path, - allow_cidrs=[f"{loopback_ip}/32"], + allow_cidrs=[tsi_cidr], env=plan.guest_env, ) stack.callback(_smolvm.machine_delete, plan.machine_name) @@ -272,7 +276,7 @@ def _launch_vm( # /32 before start (smolvm 0.8.0 silently drops `--allow-cidr` # with `--from`, so the persisted state DB is patched if needed). # Fails closed if enforcement can't be confirmed. - _loopback.force_allowlist(plan.machine_name, [f"{loopback_ip}/32"]) + _loopback.force_allowlist(plan.machine_name, [tsi_cidr]) _smolvm.machine_start(plan.machine_name) stack.callback(_smolvm.machine_stop, plan.machine_name) @@ -306,8 +310,29 @@ def _init_vm(plan: SmolmachinesBottlePlan) -> None: _smolvm.wait_exec_ready(plan.machine_name) +def _proxy_host(plan: SmolmachinesBottlePlan, loopback_ip: str) -> str: + """Return the host IP for TSI's allowlist and docker port-forward bindings. + + On macOS, the per-bottle loopback alias (e.g. ``127.0.0.16``) works + because macOS's network stack lets TSI intercept 127.x.x.x connects + from the guest before they reach the host's own loopback. + + On Linux, the guest kernel's LOCAL routing table routes all + ``127.0.0.0/8`` to the guest's own loopback — those packets never + reach eth0 and TSI never sees them. Using the per-bottle bridge + gateway (e.g. ``192.168.N.1``) instead sidesteps the problem: it + is not a loopback address, so the guest routes it via eth0 and TSI + intercepts it normally. The TSI allowlist is ``gateway/32``, which + is distinct from the container IP (``192.168.N.2``), so the agent + still can't reach the egress daemon directly — TSI blocks any + connection to the container IP that isn't via the published port.""" + if platform.system() == "Linux": + return plan.bundle_gateway + return loopback_ip + + def _bundle_launch_spec( - plan: SmolmachinesBottlePlan, network: str, loopback_ip: str, + plan: SmolmachinesBottlePlan, network: str, proxy_host: str, ) -> _bundle.BundleLaunchSpec: """Build a BundleLaunchSpec from the resolved inner Plans. @@ -366,8 +391,9 @@ def _bundle_launch_spec( volumes.append((str(sp.queue_dir), QUEUE_DIR_IN_CONTAINER, False)) # Container ports the agent reaches from the smolvm guest — - # published on host loopback so the guest can dial via TSI + - # macOS networking. Egress is always the agent's HTTP/HTTPS proxy. + # published on `proxy_host` so the TSI allowlist and the docker + # port-forward bindings point at the same IP. Egress is always + # the agent's HTTP/HTTPS proxy. ports_to_publish: list[int] = [_EGRESS_PORT] if gp.upstreams: ports_to_publish.append(_GIT_HTTP_PORT) @@ -384,7 +410,7 @@ def _bundle_launch_spec( environment=tuple(env), volumes=tuple(volumes), ports_to_publish=tuple(ports_to_publish), - publish_host_ip=loopback_ip, + publish_host_ip=proxy_host, ) @@ -458,6 +484,12 @@ def _ensure_smolmachine(image_ref: str, *, dockerfile: str = "") -> Path: return sidecar tarball = _SMOLMACHINE_CACHE_DIR / f"{digest}.image.tar" docker_mod.save(image_ref, str(tarball)) + # On Linux, `docker save -o` writes the tarball with owner-only + # permissions (mode 600). The crane push container runs as UID + # 65532 (distroless nonroot) and can't read it through a bind + # mount unless world-read is set. The tarball is temporary and + # lives in ~/.cache, so 644 is safe. + tarball.chmod(0o644) try: with ephemeral_registry() as handle: push_ref = f"{handle.push_endpoint}/bot-bottle:{digest}" diff --git a/bot_bottle/backend/smolmachines/smolvm.py b/bot_bottle/backend/smolmachines/smolvm.py index d2dd280..41fa9bd 100644 --- a/bot_bottle/backend/smolmachines/smolvm.py +++ b/bot_bottle/backend/smolmachines/smolvm.py @@ -116,10 +116,9 @@ def machine_create( allow_cidrs: Sequence[str] = (), env: Mapping[str, str] | None = None, ) -> None: - """`smolvm machine create NAME [--image IMG | --from PATH] - [--allow-cidr CIDR ...] [-e K=V ...]`. NAME is positional - (the CLI's exception to the `--name` pattern other - subcommands use). + """`smolvm machine create --name NAME [--image IMG | --from PATH] + [--allow-cidr CIDR ...] [-e K=V ...]`. NAME is passed as + `--name` (smolvm 1.4.7+; earlier versions took it positionally). `image` (registry ref like `alpine:latest`) and `from_path` (a `.smolmachine` artifact) are mutually exclusive — one or @@ -133,12 +132,10 @@ def machine_create( result without the Smolfile complication. `--net` is sent explicitly when `allow_cidrs` is non-empty. - smolvm 0.8.0's docs say `--allow-cidr` implies `--net`, but - empirically the implication only fires when no `--from` is - set — `--from PATH --allow-cidr X/32` silently produces a - machine with `network: false` and no routes in the guest, so - the agent can't reach the bundle's pinned IP.""" - args: list[str] = ["machine", "create"] + `--allow-cidr` implies `--net` per the CLI help, but sending + `--net` explicitly is harmless and ensures the guest has + network access even if that implication changes across versions.""" + args: list[str] = ["machine", "create", "--name", name] if image is not None: args += ["--image", image] if from_path is not None: @@ -150,7 +147,6 @@ def machine_create( if env: for k, v in env.items(): args += ["-e", f"{k}={v}"] - args.append(name) _smolvm(*args) @@ -182,10 +178,9 @@ def machine_stop(name: str) -> None: def machine_delete(name: str) -> None: - """`smolvm machine delete -f NAME`. NAME is positional. `-f` - skips the interactive confirmation — required for - non-interactive teardown.""" - _smolvm("machine", "delete", "-f", name) + """`smolvm machine delete --name NAME -f`. `-f` skips the + interactive confirmation — required for non-interactive teardown.""" + _smolvm("machine", "delete", "--name", name, "-f") def machine_exec( diff --git a/bot_bottle/contrib/claude/Dockerfile b/bot_bottle/contrib/claude/Dockerfile index 16b72b9..88e5014 100644 --- a/bot_bottle/contrib/claude/Dockerfile +++ b/bot_bottle/contrib/claude/Dockerfile @@ -21,7 +21,7 @@ FROM node:22-slim # to it) works against egress's bumped TLS without the agent needing # local DNS. RUN apt-get update \ - && apt-get install -y --no-install-recommends git ca-certificates curl ripgrep \ + && apt-get install -y --no-install-recommends git ca-certificates curl ripgrep iproute2 \ && rm -rf /var/lib/apt/lists/* # App-specific deps. Python isn't required by claude-code itself diff --git a/tests/integration/test_smolmachines_launch.py b/tests/integration/test_smolmachines_launch.py index 741d560..b4fc613 100644 --- a/tests/integration/test_smolmachines_launch.py +++ b/tests/integration/test_smolmachines_launch.py @@ -21,13 +21,14 @@ security properties the design pivot was about: bind-address mitigation is what closes TSI's port-granularity gap. -Gated on macOS + smolvm + docker + not GITEA_ACTIONS — the +Gated on macOS/Linux + smolvm + docker + not GITEA_ACTIONS — the runner can't host libkrun-backed VMs.""" from __future__ import annotations import os import platform +import re import shutil import tempfile import unittest @@ -65,8 +66,8 @@ def _minimal_manifest() -> ManifestIndex: @skip_unless_docker() @unittest.skipUnless( - platform.system() == "Darwin", - "smolvm is macOS-only for v1; Linux+KVM path is a future PRD", + platform.system() in ("Darwin", "Linux"), + "smolvm requires macOS or Linux", ) @unittest.skipUnless( _smolvm_available(), @@ -141,7 +142,15 @@ class TestSmolmachinesLaunch(unittest.TestCase): proxies = [line.strip() for line in r.stdout.splitlines()] self.assertEqual(2, len(proxies), proxies) self.assertEqual(proxies[0], proxies[1], proxies) - self.assertTrue(proxies[0].startswith("http://127."), proxies[0]) + # macOS: proxy binds to the per-bottle loopback alias (127.x.x.x) so + # TSI can intercept guest connections to it. Linux: the guest kernel + # routes 127.0.0.0/8 to its own loopback (TSI never sees those), so + # the proxy instead binds to the per-bottle bridge gateway (192.168.x.1) + # which routes via eth0 and is intercepted by TSI normally. + self.assertRegex( + proxies[0], r"^http://\d+\.\d+\.\d+\.\d+:\d+$", + "expected proxy URL to be an http://IP:port address", + ) r = self.bottle.exec( "curl -fsS --max-time 20 https://example.com >/dev/null && echo OK" diff --git a/tests/integration/test_smolmachines_smolvm_smoke.py b/tests/integration/test_smolmachines_smolvm_smoke.py index 34303f0..cc77802 100644 --- a/tests/integration/test_smolmachines_smolvm_smoke.py +++ b/tests/integration/test_smolmachines_smolvm_smoke.py @@ -2,7 +2,7 @@ exercised against the real binary. The full machine-lifecycle round trip (create → start → exec → -delete) is gated behind macOS + Darwin platform check and lives +delete) is gated behind macOS/Linux platform check and lives in chunk 2d's smoke. This file just verifies `is_available()` correctly reports presence and `_smolvm()` can run a no-op subcommand without errors — enough to flag wrapper drift if @@ -23,8 +23,8 @@ from bot_bottle.backend.smolmachines.smolvm import is_available "skipped under act_runner: smolvm not installed on the runner", ) @unittest.skipUnless( - platform.system() == "Darwin", - "smolvm is macOS-only for v1; Linux+KVM path is a future PRD", + platform.system() in ("Darwin", "Linux"), + "smolvm requires macOS or Linux", ) @unittest.skipUnless( is_available(),