From 1dfc3591412e27e2f156bb6e2e9387a2ff4c9758 Mon Sep 17 00:00:00 2001 From: claude Date: Wed, 27 May 2026 05:29:02 -0400 Subject: [PATCH] feat(smolmachines): thread inner Plans + bundle daemons run (PRD 0023 chunk 4b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bundle daemons (pipelock, egress, optionally git-gate + supervise) now actually start with their config files bind-mounted from the inner Plans the docker backend already produces. Chunks 2d + 3 ran with daemons_csv="" so the bundle's init supervisor idled; chunk 4b wires up the real path: agent → pipelock → egress → internet (when routes declared) is now functional, modulo agent- image gaps (claude-code / TLS-trust-store / git in the guest) that chunk 4c addresses. bottle_plan.py — added the four inner Plan fields: proxy_plan: PipelockProxyPlan git_gate_plan: GitGatePlan egress_plan: EgressPlan supervise_plan: SupervisePlan | None Same shape the docker backend's plan uses. Docker-network-only fields (internal_network, egress_network) stay at dataclass defaults — the smolmachines bundle is on a per-bottle bridge with a pinned IP, not docker's --internal + egress topology. prepare.py — instantiates DockerPipelockProxy / DockerEgress / DockerGitGate / DockerSupervise and calls their .prepare() methods to write the per-bottle config files (pipelock.yaml, routes.yaml, git-gate entrypoint/hooks, supervise queue dir) under the per-bottle state dir. (The "Docker" prefix on the class names is a misnomer here — .prepare() is platform-neutral, inherited from each sidecar's ABC. A future cleanup could factor the prepare logic out of the docker subpackage.) launch.py — major rewrite: - pipelock_tls_init at launch (always); egress_tls_init only when the bottle declares routes (otherwise the CA files aren't bind-mounted and openssl runs would be wasted). - Inner Plans updated in place with launch-time CA paths + EGRESS_UPSTREAM_PROXY = http://127.0.0.1:8888 (egress's upstream is pipelock on the bundle's own loopback; same container's network namespace). - BundleLaunchSpec env + volumes built from the inner Plans: pipelock.yaml + CA + key (always); egress routes + CAs + upstream env + token-slot bare names (when routes); git-gate entrypoint + hooks + per-upstream identity files (when upstreams); supervise queue dir + env (when enabled). - daemons_csv = ["egress", "pipelock"] + ["git-gate"] (if upstreams) + ["supervise"] (if enabled). - Token env values resolved from host env via `egress_resolve_token_values` and threaded into the docker-run subprocess env (bare-name -e entries in spec inherit from there — values never land on argv). Tests: - 552 unit passing (no new unit cases; fixture updated to populate the new plan fields). - 5 integration cases passing locally (Darwin + smolvm + docker + not GITEA_ACTIONS): * test_smoke_exec_echo — still works. * test_localhost_reach_probe — host loopback still refused. * test_egress_port_bypass_probe — :9099 still refused, NOW WITH EGRESS ACTUALLY RUNNING (chunk 3's 127.0.0.1 bind-address is doing its job). * test_prompt_file_lands_in_guest — still works. * test_pipelock_answers_on_bundle_ip — NEW. From inside the guest, wget to :8888 gets an HTTP response (not "connection refused") — proves pipelock is actually listening and the bind-mount + CA generation path works. What's left in chunk 4: - 4c: agent-image-conversion (claude-code + git + curl + ca-certificates in the guest). Chunk 2d's alpine placeholder stays for now. - 4d: provision_ca + provision_git + provision_supervise once the agent image has the required tools. Co-Authored-By: Claude Opus 4.7 --- .../backend/smolmachines/bottle_plan.py | 18 ++ claude_bottle/backend/smolmachines/launch.py | 228 ++++++++++++++---- claude_bottle/backend/smolmachines/prepare.py | 38 +++ tests/integration/test_smolmachines_launch.py | 26 ++ tests/unit/test_smolmachines_provision.py | 21 ++ 5 files changed, 287 insertions(+), 44 deletions(-) diff --git a/claude_bottle/backend/smolmachines/bottle_plan.py b/claude_bottle/backend/smolmachines/bottle_plan.py index 1213553..3bffb6d 100644 --- a/claude_bottle/backend/smolmachines/bottle_plan.py +++ b/claude_bottle/backend/smolmachines/bottle_plan.py @@ -12,7 +12,11 @@ import sys from dataclasses import dataclass from pathlib import Path +from ...egress import EgressPlan +from ...git_gate import GitGatePlan from ...log import info +from ...pipelock import PipelockProxyPlan +from ...supervise import SupervisePlan from .. import BottlePlan from ..print_util import print_multi @@ -57,6 +61,20 @@ class SmolmachinesBottlePlan(BottlePlan): # empty when the agent has no prompt — claude-code reads it # via --append-system-prompt-file only when non-empty. prompt_file: Path + # Inner Plans for the four bundle daemons. The same shape the + # docker backend uses — same `.prepare()` calls produced + # them — but our launch step doesn't populate the + # docker-specific network fields (internal_network, + # egress_network) because the smolmachines bundle isn't on + # docker's `--internal` + egress bridge topology; it's on a + # per-bottle bridge with a pinned IP. The unused fields stay + # at their dataclass defaults. + proxy_plan: PipelockProxyPlan + git_gate_plan: GitGatePlan + egress_plan: EgressPlan + # None when bottle.supervise is False, matching the docker + # backend's convention. + supervise_plan: SupervisePlan | None def print(self, *, remote_control: bool) -> None: """Compact y/N preflight. Same shape as the Docker diff --git a/claude_bottle/backend/smolmachines/launch.py b/claude_bottle/backend/smolmachines/launch.py index c0aa550..1bfe514 100644 --- a/claude_bottle/backend/smolmachines/launch.py +++ b/claude_bottle/backend/smolmachines/launch.py @@ -1,28 +1,60 @@ """End-to-end launch flow for the smolmachines backend -(PRD 0023 chunk 2d). +(PRD 0023 chunks 2d + 4b). -Brings up the per-bottle docker bridge + sidecar bundle, creates -+ starts the smolvm guest pointed at the bundle's pinned IP via -the Smolfile's TSI allowlist, yields a `SmolmachinesBottle` -handle, tears everything down on context exit. +Brings up the per-bottle docker bridge + sidecar bundle (with +real daemons + their config files), creates + starts the smolvm +guest pointed at the bundle's pinned IP via TSI's +`--allow-cidr /32` allowlist, yields a +`SmolmachinesBottle` handle, tears everything down on context +exit. -Chunk-2d scope: smoke-test plumbing for the launch + exec round -trip. The bundle daemons aren't supplied with config files yet -(pipelock.yaml, routes.yaml, etc.); the bundle's init supervisor -exits cleanly when nothing is configured. Real provisioning + CA -install + the inner Plan plumbing land in chunk 4.""" +The bundle's daemons consume the inner Plans the docker backend +already produces: pipelock reads its yaml + CA from the +PipelockProxyPlan; egress reads routes + CAs from the EgressPlan ++ EGRESS_UPSTREAM_PROXY pointing at `127.0.0.1:8888` (bundle +local), since the agent dials pipelock first (not egress) on the +smolmachines path. Git-gate + supervise plumb through the same +plans the docker backend uses, minus the docker-network fields +that don't apply here.""" from __future__ import annotations +import dataclasses +import os from contextlib import ExitStack, contextmanager from typing import Callable, Generator -from . import smolvm as _smolvm +from ...egress import EGRESS_ROUTES_IN_CONTAINER, egress_resolve_token_values +from ...supervise import QUEUE_DIR_IN_CONTAINER, SUPERVISE_PORT +from ...util import expand_tilde +from ..docker.egress import ( + EGRESS_CA_IN_CONTAINER, + EGRESS_PIPELOCK_CA_IN_CONTAINER, + egress_tls_init, +) +from ..docker.git_gate import ( + GIT_GATE_ACCESS_HOOK_IN_CONTAINER, + GIT_GATE_CREDS_DIR_IN_CONTAINER, + GIT_GATE_ENTRYPOINT_IN_CONTAINER, + GIT_GATE_HOOK_IN_CONTAINER, +) +from ..docker.pipelock import ( + PIPELOCK_CA_CERT_IN_CONTAINER, + PIPELOCK_CA_KEY_IN_CONTAINER, + pipelock_tls_init, +) from . import sidecar_bundle as _bundle +from . import smolvm as _smolvm from .bottle import SmolmachinesBottle from .bottle_plan import SmolmachinesBottlePlan +# Pipelock's upstream when egress is in the bundle: localhost on +# the bundle's own loopback. No docker DNS aliases involved — +# pipelock + egress share the same container's network namespace. +_BUNDLE_LOCAL_PIPELOCK_URL = "http://127.0.0.1:8888" + + @contextmanager def launch( plan: SmolmachinesBottlePlan, @@ -34,40 +66,53 @@ def launch( via the ExitStack.""" stack = ExitStack() try: - # 1. Per-bottle docker bridge + bundle container. + # 1. Per-bottle docker bridge. network = _bundle.bundle_network_name(plan.slug) _bundle.create_bundle_network(network, plan.bundle_subnet, plan.bundle_gateway) stack.callback(_bundle.remove_bundle_network, network) - bundle_spec = _bundle.BundleLaunchSpec( - slug=plan.slug, - network_name=network, - subnet=plan.bundle_subnet, - gateway=plan.bundle_gateway, - bundle_ip=plan.bundle_ip, - # Chunk 2d: empty daemon set — the init supervisor - # logs "no daemons selected" and idles. Real daemon - # bringup with inner-Plan-driven env + volumes lands - # in chunk 4 alongside provisioning. - daemons_csv="", - # PRD 0023 chunk 3: pin egress to localhost INSIDE the - # bundle so the agent's TSI-permitted `:*` - # connect to :9099 refuses at the socket level. Always - # set in smolmachines mode — agent dials pipelock, not - # egress, so egress is bundle-internal regardless of - # whether routes are declared. The docker backend - # doesn't set this env (egress on 0.0.0.0 by default) - # since the docker agent goes via the egress alias. - environment=("EGRESS_LISTEN_HOST=127.0.0.1",), + # 2. Mint per-bottle CAs and update the inner Plans with + # their launch-time paths. pipelock always runs in the + # bundle; egress's CA is only minted when the bottle + # declares routes (otherwise egress runs idle without + # MITM and the CA files would be unused). + ca_cert_host, ca_key_host = pipelock_tls_init(plan.proxy_plan.yaml_path.parent) + proxy_plan = dataclasses.replace( + plan.proxy_plan, + ca_cert_host_path=ca_cert_host, + ca_key_host_path=ca_key_host, ) - _bundle.start_bundle(bundle_spec) + egress_plan = plan.egress_plan + if egress_plan.routes: + egress_ca_host, egress_ca_cert_only = egress_tls_init( + plan.egress_plan.routes_path.parent, + ) + egress_plan = dataclasses.replace( + egress_plan, + mitmproxy_ca_host_path=egress_ca_host, + mitmproxy_ca_cert_only_host_path=egress_ca_cert_only, + pipelock_ca_host_path=ca_cert_host, + # On smolmachines, egress's upstream is pipelock + # on the bundle's localhost — they're in the same + # container's network namespace. + pipelock_proxy_url=_BUNDLE_LOCAL_PIPELOCK_URL, + ) + plan = dataclasses.replace( + plan, proxy_plan=proxy_plan, egress_plan=egress_plan, + ) + + # 3. Build the BundleLaunchSpec from the (now-resolved) + # inner Plans: daemon subset, env, bind-mounts. + bundle_spec = _bundle_launch_spec(plan, network) + token_env = _resolve_token_env(plan, os.environ) + _bundle.start_bundle(bundle_spec, env={**os.environ, **token_env}) stack.callback(_bundle.stop_bundle, plan.slug) - # 2. smolvm VM. --from carries the pre-packed - # .smolmachine artifact (built by prepare); --allow-cidr - # + -e carry the per-bottle TSI allowlist + env. Smolfile - # isn't usable here — smolvm 0.8.0 makes `--from` and - # `--smolfile` mutually exclusive. + # 4. smolvm VM. --from carries the pre-packed .smolmachine + # artifact (built by prepare); --allow-cidr + -e carry the + # per-bottle TSI allowlist + env. Smolfile isn't usable + # here — smolvm 0.8.0 makes `--from` and `--smolfile` + # mutually exclusive. _smolvm.machine_create( plan.machine_name, from_path=plan.agent_from_path, @@ -78,14 +123,109 @@ def launch( _smolvm.machine_start(plan.machine_name) stack.callback(_smolvm.machine_stop, plan.machine_name) - # 3. Provision (CA / prompt / skills / git / supervise). - # The orchestrator runs each one in order; provision_* - # methods left as stubs (chunk 4 follow-ons) are no-ops. + # 5. Provision (CA / prompt / skills / git / supervise). prompt_path = provision(plan, plan.machine_name) - # 4. Yield the handle. The prompt_path drives whether - # exec_claude adds --append-system-prompt-file to claude's - # argv (None → no flag). yield SmolmachinesBottle(plan.machine_name, prompt_path=prompt_path) finally: stack.close() + + +def _bundle_launch_spec( + plan: SmolmachinesBottlePlan, network: str +) -> _bundle.BundleLaunchSpec: + """Build a BundleLaunchSpec from the resolved inner Plans. + + Daemons in the CSV: + - egress + pipelock are always present (pipelock is the + agent's first hop; egress is its upstream). + - git-gate is conditional on plan.git_gate_plan.upstreams. + - supervise is conditional on plan.supervise_plan. + + Env + volumes are the union of the four daemons' needs, with + daemon-private values only (HTTPS_PROXY is scoped to the + egress process by egress_entrypoint.sh — see PRD 0024's bundle + bind-address PR).""" + daemons: list[str] = ["egress", "pipelock"] + env: list[str] = [] + volumes: list[tuple[str, str, bool]] = [] + + # PRD 0023 chunk 3: egress binds 127.0.0.1 inside the bundle + # so TSI's IP-only allowlist can't bypass pipelock. + env.append("EGRESS_LISTEN_HOST=127.0.0.1") + + # --- pipelock --------------------------------------------- + pp = plan.proxy_plan + volumes += [ + (str(pp.yaml_path), "/etc/pipelock.yaml", True), + (str(pp.ca_cert_host_path), PIPELOCK_CA_CERT_IN_CONTAINER, True), + (str(pp.ca_key_host_path), PIPELOCK_CA_KEY_IN_CONTAINER, True), + ] + + # --- egress ----------------------------------------------- + ep = plan.egress_plan + if ep.routes: + env.append(f"EGRESS_UPSTREAM_PROXY={ep.pipelock_proxy_url}") + env.append(f"EGRESS_UPSTREAM_CA={EGRESS_PIPELOCK_CA_IN_CONTAINER}") + volumes += [ + (str(ep.routes_path), EGRESS_ROUTES_IN_CONTAINER, True), + (str(ep.mitmproxy_ca_host_path), EGRESS_CA_IN_CONTAINER, True), + (str(ep.pipelock_ca_host_path), EGRESS_PIPELOCK_CA_IN_CONTAINER, True), + ] + # Bare-name entries for upstream-token slots. Their values + # come from the docker-run subprocess env (inherited from + # the operator's shell), never landing on argv. + for token_env in sorted(ep.token_env_map.keys()): + env.append(token_env) + + # --- git-gate --------------------------------------------- + extra_hosts: list[str] = [] + gp = plan.git_gate_plan + if gp.upstreams: + daemons.append("git-gate") + volumes += [ + (str(gp.entrypoint_script), GIT_GATE_ENTRYPOINT_IN_CONTAINER, True), + (str(gp.hook_script), GIT_GATE_HOOK_IN_CONTAINER, True), + (str(gp.access_hook_script), GIT_GATE_ACCESS_HOOK_IN_CONTAINER, True), + ] + for u in gp.upstreams: + keypath = expand_tilde(u.identity_file) + volumes.append(( + keypath, + f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{u.name}-key", + True, + )) + + # --- supervise -------------------------------------------- + sp = plan.supervise_plan + if sp is not None: + daemons.append("supervise") + env += [ + f"SUPERVISE_BOTTLE_SLUG={plan.slug}", + f"SUPERVISE_QUEUE_DIR={QUEUE_DIR_IN_CONTAINER}", + f"SUPERVISE_PORT={SUPERVISE_PORT}", + ] + volumes.append((str(sp.queue_dir), QUEUE_DIR_IN_CONTAINER, False)) + + return _bundle.BundleLaunchSpec( + slug=plan.slug, + network_name=network, + subnet=plan.bundle_subnet, + gateway=plan.bundle_gateway, + bundle_ip=plan.bundle_ip, + daemons_csv=",".join(daemons), + environment=tuple(env), + volumes=tuple(volumes), + ) + + +def _resolve_token_env( + plan: SmolmachinesBottlePlan, host_env: object +) -> dict[str, str]: + """Resolve the egress token env-var values from the host's + environ so they reach the bundle's process env via docker's + `-e NAME` inheritance. Empty when no routes declare auth.""" + ep = plan.egress_plan + if not ep.routes: + return {} + return egress_resolve_token_values(ep.token_env_map, dict(host_env)) diff --git a/claude_bottle/backend/smolmachines/prepare.py b/claude_bottle/backend/smolmachines/prepare.py index 0aee9ba..b0cf4e3 100644 --- a/claude_bottle/backend/smolmachines/prepare.py +++ b/claude_bottle/backend/smolmachines/prepare.py @@ -15,8 +15,16 @@ from ...backend.docker.bottle_state import ( BottleMetadata, agent_state_dir, bottle_identity, + egress_state_dir, + git_gate_state_dir, + pipelock_state_dir, + supervise_state_dir, write_metadata, ) +from ...backend.docker.egress import DockerEgress +from ...backend.docker.git_gate import DockerGitGate +from ...backend.docker.pipelock import DockerPipelockProxy +from ...backend.docker.supervise import DockerSupervise from . import smolvm as _smolvm from .bottle_plan import SmolmachinesBottlePlan from .util import smolmachines_bundle_subnet, smolmachines_preflight @@ -86,6 +94,32 @@ def resolve_plan( f"http://{bundle_ip}:{_BUNDLE_SUPERVISE_PORT}" ) + # Inner Plans for the four bundle daemons. Use the docker + # backend's concrete subclasses — the `.prepare()` method + # they inherit is platform-neutral (writes config files + + # returns a Plan dataclass); the docker-specific subclasses + # exist only to satisfy ABC instantiation. Future: factor + # the prepare logic out of the docker subpackage so + # smolmachines doesn't have to reach across the backend + # boundary. + pipelock_dir = pipelock_state_dir(slug) + pipelock_dir.mkdir(parents=True, exist_ok=True) + proxy_plan = DockerPipelockProxy().prepare(bottle, slug, pipelock_dir) + + git_gate_dir = git_gate_state_dir(slug) + git_gate_dir.mkdir(parents=True, exist_ok=True) + git_gate_plan = DockerGitGate().prepare(bottle, slug, git_gate_dir) + + egress_dir = egress_state_dir(slug) + egress_dir.mkdir(parents=True, exist_ok=True) + egress_plan = DockerEgress().prepare(bottle, slug, egress_dir) + + supervise_plan = None + if bottle.supervise: + supervise_dir = supervise_state_dir(slug) + supervise_dir.mkdir(parents=True, exist_ok=True) + supervise_plan = DockerSupervise().prepare(slug, supervise_dir) + # Prompt file is always written (mode 0o600) so the in-VM # path always exists. Content is the agent's `prompt` # field (markdown body) — empty for agents with no prompt. @@ -118,6 +152,10 @@ def resolve_plan( agent_from_path=agent_from_path, guest_env=guest_env, prompt_file=prompt_file, + proxy_plan=proxy_plan, + git_gate_plan=git_gate_plan, + egress_plan=egress_plan, + supervise_plan=supervise_plan, ) diff --git a/tests/integration/test_smolmachines_launch.py b/tests/integration/test_smolmachines_launch.py index 7c9dd35..b8f907d 100644 --- a/tests/integration/test_smolmachines_launch.py +++ b/tests/integration/test_smolmachines_launch.py @@ -124,6 +124,32 @@ class TestSmolmachinesLaunch(unittest.TestCase): f"expected a connect-refusal message; got: {r.stdout!r}", ) + def test_pipelock_answers_on_bundle_ip(self): + # Chunk 4b: the bundle's pipelock daemon is now actually + # running (was daemons_csv="" in chunks 2d/3). From inside + # the guest, a TCP connect to :8888 must succeed + # — distinct from the egress-port-bypass probe below where + # the connect must FAIL. + # + # We don't try to speak proxy protocol here — pipelock will + # 4xx a bare GET — we just verify the socket answers. + r = self.bottle.exec( + f"wget -T 5 -t 1 -O - http://{self.plan.bundle_ip}:8888/ " + "2>&1 || true" + ) + # Any HTTP response (even a 4xx) proves pipelock is up. + # "connection refused" / "unable to connect" / "timed out" + # would mean it isn't. + msg = r.stdout.lower() + self.assertNotIn( + "connection refused", msg, + f"pipelock connect refused — daemon not listening? {r.stdout!r}", + ) + self.assertNotIn( + "timed out", msg, + f"pipelock connect timed out: {r.stdout!r}", + ) + def test_prompt_file_lands_in_guest(self): # provision_prompt copies the host-side prompt.txt into the # guest at /root/.claude-bottle-prompt.txt. The content diff --git a/tests/unit/test_smolmachines_provision.py b/tests/unit/test_smolmachines_provision.py index 49e02e2..d30ecb9 100644 --- a/tests/unit/test_smolmachines_provision.py +++ b/tests/unit/test_smolmachines_provision.py @@ -18,7 +18,10 @@ from claude_bottle.backend.smolmachines.provision import ( prompt as _prompt, skills as _skills, ) +from claude_bottle.egress import EgressPlan +from claude_bottle.git_gate import GitGatePlan from claude_bottle.manifest import Manifest +from claude_bottle.pipelock import PipelockProxyPlan def _plan( @@ -53,6 +56,24 @@ def _plan( agent_from_path=Path("/tmp/agent.smolmachine"), guest_env={}, prompt_file=Path("/tmp/state/demo-abc12/agent/prompt.txt"), + proxy_plan=PipelockProxyPlan( + yaml_path=Path("/tmp/pipelock.yaml"), + slug="demo-abc12", + ), + git_gate_plan=GitGatePlan( + slug="demo-abc12", + entrypoint_script=Path("/tmp/git-gate-entrypoint.sh"), + hook_script=Path("/tmp/git-gate-hook"), + access_hook_script=Path("/tmp/git-gate-access-hook"), + upstreams=(), + ), + egress_plan=EgressPlan( + slug="demo-abc12", + routes_path=Path("/tmp/routes.yaml"), + routes=(), + token_env_map={}, + ), + supervise_plan=None, )