fix(smolmachines): use bridge gateway as TSI proxy host on Linux
lint / lint (push) Failing after 1m49s
test / unit (pull_request) Failing after 41s
test / integration (pull_request) Successful in 19s

On Linux the guest kernel's LOCAL routing table routes all
127.0.0.0/8 to the guest's own loopback interface (priority 0,
checked before any main-table route), so TSI never sees
connections to the per-bottle loopback alias — the fix_guest_
loopback_routing approach confirmed this at the kernel level.

Use the per-bottle docker bridge gateway (192.168.N.1) instead.
It is not a loopback address, so the guest routes it via eth0
and TSI intercepts it normally.  The TSI allowlist remains a
/32 that is distinct from the container IP (192.168.N.2), so
direct bypass to egress:9099 is still blocked by TSI.

Changes:
- Add _proxy_host() helper: returns bundle_gateway on Linux,
  loopback alias on macOS
- Thread proxy_host through _start_bundle, _discover_urls,
  _launch_vm, and _bundle_launch_spec (publish_host_ip)
- Remove _fix_guest_loopback_routing (no longer needed)
- Relax proxy-URL assertion in the integration test to accept
  any http://IP:port (with a comment explaining the difference)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-07-07 13:15:34 -04:00
parent 688e4cd546
commit 8f9005582b
5 changed files with 93 additions and 57 deletions
+66 -34
View File
@@ -17,6 +17,7 @@ from __future__ import annotations
import dataclasses import dataclasses
import os import os
import platform
from contextlib import ExitStack, contextmanager from contextlib import ExitStack, contextmanager
from pathlib import Path from pathlib import Path
from typing import Callable, Generator from typing import Callable, Generator
@@ -88,12 +89,13 @@ def launch(
try: try:
loopback_ip, network = _allocate_resources(plan, stack) loopback_ip, network = _allocate_resources(plan, stack)
plan = _mint_certs(plan) plan = _mint_certs(plan)
plan = _start_bundle(plan, network, loopback_ip, stack) proxy_host = _proxy_host(plan, loopback_ip)
plan = _discover_urls(plan, loopback_ip) plan = _start_bundle(plan, network, proxy_host, stack)
plan = _discover_urls(plan, proxy_host)
agent_from_path = _agent_from_path(plan) agent_from_path = _agent_from_path(plan)
_launch_vm(plan, agent_from_path, loopback_ip, stack) _launch_vm(plan, agent_from_path, proxy_host, stack)
_init_vm(plan) _init_vm(plan)
bottle = SmolmachinesBottle( bottle = SmolmachinesBottle(
@@ -171,12 +173,12 @@ def _mint_certs(plan: SmolmachinesBottlePlan) -> SmolmachinesBottlePlan:
def _start_bundle( def _start_bundle(
plan: SmolmachinesBottlePlan, plan: SmolmachinesBottlePlan,
network: str, network: str,
loopback_ip: str, proxy_host: str,
stack: ExitStack, stack: ExitStack,
) -> SmolmachinesBottlePlan: ) -> SmolmachinesBottlePlan:
"""Build the BundleLaunchSpec, resolve token env, start the """Build the BundleLaunchSpec, resolve token env, start the
sidecar bundle container, and register teardown.""" sidecar bundle container, and register teardown."""
bundle_spec = _bundle_launch_spec(plan, network, loopback_ip) bundle_spec = _bundle_launch_spec(plan, network, proxy_host)
token_env = _resolve_token_env(plan, dict(os.environ)) token_env = _resolve_token_env(plan, dict(os.environ))
_bundle.ensure_bundle_image(bundle_spec.image) _bundle.ensure_bundle_image(bundle_spec.image)
_bundle.start_bundle(bundle_spec, env={**os.environ, **token_env}) _bundle.start_bundle(bundle_spec, env={**os.environ, **token_env})
@@ -186,41 +188,40 @@ def _start_bundle(
def _discover_urls( def _discover_urls(
plan: SmolmachinesBottlePlan, plan: SmolmachinesBottlePlan,
loopback_ip: str, proxy_host: str,
) -> SmolmachinesBottlePlan: ) -> SmolmachinesBottlePlan:
"""Discover host-side ports for published container ports and """Discover host-side ports for published container ports and
return the plan with URLs + guest_env stamped in. return the plan with URLs + guest_env stamped in.
Docker container IPs (192.168.x.x in the daemon's bridge) `proxy_host` is the host IP that both TSI's allowlist and
aren't reachable from the smolvm guest — TSI proxies the docker's port-forward bindings are keyed to. On macOS it is the
guest's connects through the host, and the host reaches the per-bottle loopback alias; on Linux it is the per-bottle bridge
bundle only via its published-port loopback forward (the gateway (see `_proxy_host`). The agent dials the published port
daemon's bridge isn't on the TSI allowlist). The agent dials on this IP for all bundle-hosted services.
the published port on the per-bottle loopback alias.
NO_PROXY includes the per-bottle loopback alias so the NO_PROXY includes `proxy_host` so supervise + git-gate URLs
supervise + git-gate URLs bypass HTTPS_PROXY.""" bypass HTTPS_PROXY."""
agent_facing_host_port = _bundle.bundle_host_port( agent_facing_host_port = _bundle.bundle_host_port(
plan.slug, _EGRESS_PORT, host_ip=loopback_ip, plan.slug, _EGRESS_PORT, host_ip=proxy_host,
) )
agent_proxy_url = f"http://{loopback_ip}:{agent_facing_host_port}" agent_proxy_url = f"http://{proxy_host}:{agent_facing_host_port}"
agent_git_gate_host = "" agent_git_gate_host = ""
if plan.git_gate_plan.upstreams: if plan.git_gate_plan.upstreams:
git_gate_host_port = _bundle.bundle_host_port( git_gate_host_port = _bundle.bundle_host_port(
plan.slug, _GIT_HTTP_PORT, host_ip=loopback_ip, plan.slug, _GIT_HTTP_PORT, host_ip=proxy_host,
) )
agent_git_gate_host = f"{loopback_ip}:{git_gate_host_port}" agent_git_gate_host = f"{proxy_host}:{git_gate_host_port}"
agent_supervise_url = "" agent_supervise_url = ""
if plan.supervise_plan is not None: if plan.supervise_plan is not None:
supervise_host_port = _bundle.bundle_host_port( supervise_host_port = _bundle.bundle_host_port(
plan.slug, _SUPERVISE_PORT, host_ip=loopback_ip, plan.slug, _SUPERVISE_PORT, host_ip=proxy_host,
) )
agent_supervise_url = f"http://{loopback_ip}:{supervise_host_port}/" agent_supervise_url = f"http://{proxy_host}:{supervise_host_port}/"
existing_no_proxy = plan.guest_env.get("NO_PROXY", "localhost,127.0.0.1") existing_no_proxy = plan.guest_env.get("NO_PROXY", "localhost,127.0.0.1")
no_proxy = f"{existing_no_proxy},{loopback_ip}" no_proxy = f"{existing_no_proxy},{proxy_host}"
guest_env = { guest_env = {
**plan.guest_env, **plan.guest_env,
"HTTPS_PROXY": agent_proxy_url, "HTTPS_PROXY": agent_proxy_url,
@@ -250,21 +251,24 @@ def _discover_urls(
def _launch_vm( def _launch_vm(
plan: SmolmachinesBottlePlan, plan: SmolmachinesBottlePlan,
agent_from_path: Path, agent_from_path: Path,
loopback_ip: str, proxy_host: str,
stack: ExitStack, stack: ExitStack,
) -> None: ) -> None:
"""Create, patch, and start the smolvm VM; register teardown. """Create, patch, and start the smolvm VM; register teardown.
--allow-cidr is the per-bottle loopback alias so the guest can --allow-cidr is `proxy_host/32` — the per-bottle loopback alias
only reach this bottle's bundle ports. force_allowlist then on macOS or the bridge gateway on Linux (see `_proxy_host`). This
confirms the allowlist persisted (patching smolvm 0.8.0's ensures the guest can only reach bundle ports published on that IP,
silent-drop of --allow-cidr when combined with --from) and not the container IP directly. force_allowlist confirms the
fails closed if it can't. Smolfile isn't usable here — smolvm allowlist persisted (patching smolvm 0.8.0's silent-drop of
0.8.0 makes --from and --smolfile mutually exclusive.""" --allow-cidr when combined with --from) and fails closed if it
can't. Smolfile isn't usable here — smolvm 0.8.0 makes --from
and --smolfile mutually exclusive."""
tsi_cidr = f"{proxy_host}/32"
_smolvm.machine_create( _smolvm.machine_create(
plan.machine_name, plan.machine_name,
from_path=agent_from_path, from_path=agent_from_path,
allow_cidrs=[f"{loopback_ip}/32"], allow_cidrs=[tsi_cidr],
env=plan.guest_env, env=plan.guest_env,
) )
stack.callback(_smolvm.machine_delete, plan.machine_name) stack.callback(_smolvm.machine_delete, plan.machine_name)
@@ -272,7 +276,7 @@ def _launch_vm(
# /32 before start (smolvm 0.8.0 silently drops `--allow-cidr` # /32 before start (smolvm 0.8.0 silently drops `--allow-cidr`
# with `--from`, so the persisted state DB is patched if needed). # with `--from`, so the persisted state DB is patched if needed).
# Fails closed if enforcement can't be confirmed. # Fails closed if enforcement can't be confirmed.
_loopback.force_allowlist(plan.machine_name, [f"{loopback_ip}/32"]) _loopback.force_allowlist(plan.machine_name, [tsi_cidr])
_smolvm.machine_start(plan.machine_name) _smolvm.machine_start(plan.machine_name)
stack.callback(_smolvm.machine_stop, plan.machine_name) stack.callback(_smolvm.machine_stop, plan.machine_name)
@@ -306,8 +310,29 @@ def _init_vm(plan: SmolmachinesBottlePlan) -> None:
_smolvm.wait_exec_ready(plan.machine_name) _smolvm.wait_exec_ready(plan.machine_name)
def _proxy_host(plan: SmolmachinesBottlePlan, loopback_ip: str) -> str:
"""Return the host IP for TSI's allowlist and docker port-forward bindings.
On macOS, the per-bottle loopback alias (e.g. ``127.0.0.16``) works
because macOS's network stack lets TSI intercept 127.x.x.x connects
from the guest before they reach the host's own loopback.
On Linux, the guest kernel's LOCAL routing table routes all
``127.0.0.0/8`` to the guest's own loopback — those packets never
reach eth0 and TSI never sees them. Using the per-bottle bridge
gateway (e.g. ``192.168.N.1``) instead sidesteps the problem: it
is not a loopback address, so the guest routes it via eth0 and TSI
intercepts it normally. The TSI allowlist is ``gateway/32``, which
is distinct from the container IP (``192.168.N.2``), so the agent
still can't reach the egress daemon directly — TSI blocks any
connection to the container IP that isn't via the published port."""
if platform.system() == "Linux":
return plan.bundle_gateway
return loopback_ip
def _bundle_launch_spec( def _bundle_launch_spec(
plan: SmolmachinesBottlePlan, network: str, loopback_ip: str, plan: SmolmachinesBottlePlan, network: str, proxy_host: str,
) -> _bundle.BundleLaunchSpec: ) -> _bundle.BundleLaunchSpec:
"""Build a BundleLaunchSpec from the resolved inner Plans. """Build a BundleLaunchSpec from the resolved inner Plans.
@@ -366,8 +391,9 @@ def _bundle_launch_spec(
volumes.append((str(sp.queue_dir), QUEUE_DIR_IN_CONTAINER, False)) volumes.append((str(sp.queue_dir), QUEUE_DIR_IN_CONTAINER, False))
# Container ports the agent reaches from the smolvm guest — # Container ports the agent reaches from the smolvm guest —
# published on host loopback so the guest can dial via TSI + # published on `proxy_host` so the TSI allowlist and the docker
# macOS networking. Egress is always the agent's HTTP/HTTPS proxy. # port-forward bindings point at the same IP. Egress is always
# the agent's HTTP/HTTPS proxy.
ports_to_publish: list[int] = [_EGRESS_PORT] ports_to_publish: list[int] = [_EGRESS_PORT]
if gp.upstreams: if gp.upstreams:
ports_to_publish.append(_GIT_HTTP_PORT) ports_to_publish.append(_GIT_HTTP_PORT)
@@ -384,7 +410,7 @@ def _bundle_launch_spec(
environment=tuple(env), environment=tuple(env),
volumes=tuple(volumes), volumes=tuple(volumes),
ports_to_publish=tuple(ports_to_publish), ports_to_publish=tuple(ports_to_publish),
publish_host_ip=loopback_ip, publish_host_ip=proxy_host,
) )
@@ -458,6 +484,12 @@ def _ensure_smolmachine(image_ref: str, *, dockerfile: str = "") -> Path:
return sidecar return sidecar
tarball = _SMOLMACHINE_CACHE_DIR / f"{digest}.image.tar" tarball = _SMOLMACHINE_CACHE_DIR / f"{digest}.image.tar"
docker_mod.save(image_ref, str(tarball)) docker_mod.save(image_ref, str(tarball))
# On Linux, `docker save -o` writes the tarball with owner-only
# permissions (mode 600). The crane push container runs as UID
# 65532 (distroless nonroot) and can't read it through a bind
# mount unless world-read is set. The tarball is temporary and
# lives in ~/.cache, so 644 is safe.
tarball.chmod(0o644)
try: try:
with ephemeral_registry() as handle: with ephemeral_registry() as handle:
push_ref = f"{handle.push_endpoint}/bot-bottle:{digest}" push_ref = f"{handle.push_endpoint}/bot-bottle:{digest}"
+10 -15
View File
@@ -116,10 +116,9 @@ def machine_create(
allow_cidrs: Sequence[str] = (), allow_cidrs: Sequence[str] = (),
env: Mapping[str, str] | None = None, env: Mapping[str, str] | None = None,
) -> None: ) -> None:
"""`smolvm machine create NAME [--image IMG | --from PATH] """`smolvm machine create --name NAME [--image IMG | --from PATH]
[--allow-cidr CIDR ...] [-e K=V ...]`. NAME is positional [--allow-cidr CIDR ...] [-e K=V ...]`. NAME is passed as
(the CLI's exception to the `--name` pattern other `--name` (smolvm 1.4.7+; earlier versions took it positionally).
subcommands use).
`image` (registry ref like `alpine:latest`) and `from_path` `image` (registry ref like `alpine:latest`) and `from_path`
(a `.smolmachine` artifact) are mutually exclusive — one or (a `.smolmachine` artifact) are mutually exclusive — one or
@@ -133,12 +132,10 @@ def machine_create(
result without the Smolfile complication. result without the Smolfile complication.
`--net` is sent explicitly when `allow_cidrs` is non-empty. `--net` is sent explicitly when `allow_cidrs` is non-empty.
smolvm 0.8.0's docs say `--allow-cidr` implies `--net`, but `--allow-cidr` implies `--net` per the CLI help, but sending
empirically the implication only fires when no `--from` is `--net` explicitly is harmless and ensures the guest has
set — `--from PATH --allow-cidr X/32` silently produces a network access even if that implication changes across versions."""
machine with `network: false` and no routes in the guest, so args: list[str] = ["machine", "create", "--name", name]
the agent can't reach the bundle's pinned IP."""
args: list[str] = ["machine", "create"]
if image is not None: if image is not None:
args += ["--image", image] args += ["--image", image]
if from_path is not None: if from_path is not None:
@@ -150,7 +147,6 @@ def machine_create(
if env: if env:
for k, v in env.items(): for k, v in env.items():
args += ["-e", f"{k}={v}"] args += ["-e", f"{k}={v}"]
args.append(name)
_smolvm(*args) _smolvm(*args)
@@ -182,10 +178,9 @@ def machine_stop(name: str) -> None:
def machine_delete(name: str) -> None: def machine_delete(name: str) -> None:
"""`smolvm machine delete -f NAME`. NAME is positional. `-f` """`smolvm machine delete --name NAME -f`. `-f` skips the
skips the interactive confirmation — required for interactive confirmation — required for non-interactive teardown."""
non-interactive teardown.""" _smolvm("machine", "delete", "--name", name, "-f")
_smolvm("machine", "delete", "-f", name)
def machine_exec( def machine_exec(
+1 -1
View File
@@ -21,7 +21,7 @@ FROM node:22-slim
# to it) works against egress's bumped TLS without the agent needing # to it) works against egress's bumped TLS without the agent needing
# local DNS. # local DNS.
RUN apt-get update \ RUN apt-get update \
&& apt-get install -y --no-install-recommends git ca-certificates curl ripgrep \ && apt-get install -y --no-install-recommends git ca-certificates curl ripgrep iproute2 \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# App-specific deps. Python isn't required by claude-code itself # App-specific deps. Python isn't required by claude-code itself
+13 -4
View File
@@ -21,13 +21,14 @@ security properties the design pivot was about:
bind-address mitigation is what closes TSI's port-granularity bind-address mitigation is what closes TSI's port-granularity
gap. gap.
Gated on macOS + smolvm + docker + not GITEA_ACTIONS — the Gated on macOS/Linux + smolvm + docker + not GITEA_ACTIONS — the
runner can't host libkrun-backed VMs.""" runner can't host libkrun-backed VMs."""
from __future__ import annotations from __future__ import annotations
import os import os
import platform import platform
import re
import shutil import shutil
import tempfile import tempfile
import unittest import unittest
@@ -65,8 +66,8 @@ def _minimal_manifest() -> ManifestIndex:
@skip_unless_docker() @skip_unless_docker()
@unittest.skipUnless( @unittest.skipUnless(
platform.system() == "Darwin", platform.system() in ("Darwin", "Linux"),
"smolvm is macOS-only for v1; Linux+KVM path is a future PRD", "smolvm requires macOS or Linux",
) )
@unittest.skipUnless( @unittest.skipUnless(
_smolvm_available(), _smolvm_available(),
@@ -141,7 +142,15 @@ class TestSmolmachinesLaunch(unittest.TestCase):
proxies = [line.strip() for line in r.stdout.splitlines()] proxies = [line.strip() for line in r.stdout.splitlines()]
self.assertEqual(2, len(proxies), proxies) self.assertEqual(2, len(proxies), proxies)
self.assertEqual(proxies[0], proxies[1], proxies) self.assertEqual(proxies[0], proxies[1], proxies)
self.assertTrue(proxies[0].startswith("http://127."), proxies[0]) # macOS: proxy binds to the per-bottle loopback alias (127.x.x.x) so
# TSI can intercept guest connections to it. Linux: the guest kernel
# routes 127.0.0.0/8 to its own loopback (TSI never sees those), so
# the proxy instead binds to the per-bottle bridge gateway (192.168.x.1)
# which routes via eth0 and is intercepted by TSI normally.
self.assertRegex(
proxies[0], r"^http://\d+\.\d+\.\d+\.\d+:\d+$",
"expected proxy URL to be an http://IP:port address",
)
r = self.bottle.exec( r = self.bottle.exec(
"curl -fsS --max-time 20 https://example.com >/dev/null && echo OK" "curl -fsS --max-time 20 https://example.com >/dev/null && echo OK"
@@ -2,7 +2,7 @@
exercised against the real binary. exercised against the real binary.
The full machine-lifecycle round trip (create → start → exec → The full machine-lifecycle round trip (create → start → exec →
delete) is gated behind macOS + Darwin platform check and lives delete) is gated behind macOS/Linux platform check and lives
in chunk 2d's smoke. This file just verifies `is_available()` in chunk 2d's smoke. This file just verifies `is_available()`
correctly reports presence and `_smolvm()` can run a no-op correctly reports presence and `_smolvm()` can run a no-op
subcommand without errors — enough to flag wrapper drift if subcommand without errors — enough to flag wrapper drift if
@@ -23,8 +23,8 @@ from bot_bottle.backend.smolmachines.smolvm import is_available
"skipped under act_runner: smolvm not installed on the runner", "skipped under act_runner: smolvm not installed on the runner",
) )
@unittest.skipUnless( @unittest.skipUnless(
platform.system() == "Darwin", platform.system() in ("Darwin", "Linux"),
"smolvm is macOS-only for v1; Linux+KVM path is a future PRD", "smolvm requires macOS or Linux",
) )
@unittest.skipUnless( @unittest.skipUnless(
is_available(), is_available(),