PRD 0005: mitmproxy TLS interception for pipelock content scanning #8

Closed
didericis wants to merge 6 commits from mitmproxy-tls-interception into main
6 changed files with 128 additions and 14 deletions
Showing only changes of commit 21054212d4 - Show all commits
+22 -11
View File
@@ -204,24 +204,35 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]):
"""Build/run the bottle and yield a handle; tear down on exit.""" """Build/run the bottle and yield a handle; tear down on exit."""
def provision(self, plan: PlanT, target: str) -> str | None: def provision(self, plan: PlanT, target: str) -> str | None:
"""Copy host-side files (prompt, skills, SSH keys, .git) into """Copy host-side files (CA cert, prompt, skills, SSH keys,
the running bottle. Called from `launch` after the container/ .git) into the running bottle. Called from `launch` after the
machine is up. `target` identifies the running instance in container/machine is up. `target` identifies the running
backend-specific terms (Docker: resolved container name; fly: instance in backend-specific terms (Docker: resolved container
machine id). Returns the in-container prompt path if a prompt name; fly: machine id). Returns the in-container prompt path
was provisioned, else None — the Bottle handle uses it to if a prompt was provisioned, else None — the Bottle handle
decide whether to add --append-system-prompt-file to claude's uses it to decide whether to add --append-system-prompt-file
argv. to claude's argv.
Default orchestration: prompt → skills → ssh → git. Subclasses Default orchestration: ca → prompt → skills → ssh → git.
typically don't override this; they implement the four CA goes first because it changes how the agent process trusts
sub-methods below.""" the network; the rest don't depend on it but the order keeps
trust setup adjacent to the launch step. Subclasses typically
don't override this; they implement the sub-methods below."""
self.provision_ca(plan, target)
prompt_path = self.provision_prompt(plan, target) prompt_path = self.provision_prompt(plan, target)
self.provision_skills(plan, target) self.provision_skills(plan, target)
self.provision_ssh(plan, target) self.provision_ssh(plan, target)
self.provision_git(plan, target) self.provision_git(plan, target)
return prompt_path return prompt_path
def provision_ca(self, plan: PlanT, target: str) -> None:
"""Install the egress-proxy's CA into the running bottle's
trust store. Default impl is a no-op so backends that don't
yet support TLS interception (every backend except Docker
today) aren't forced to implement it. The Docker backend
overrides to extract mitmproxy's CA and run
`update-ca-certificates` inside the agent container."""
@abstractmethod @abstractmethod
def provision_prompt(self, plan: PlanT, target: str) -> str | None: def provision_prompt(self, plan: PlanT, target: str) -> str | None:
"""Copy the prompt file into the running bottle. Returns the """Copy the prompt file into the running bottle. Returns the
+12 -2
View File
@@ -23,7 +23,9 @@ from . import prepare as _prepare
from .bottle import DockerBottle from .bottle import DockerBottle
from .bottle_cleanup_plan import DockerBottleCleanupPlan from .bottle_cleanup_plan import DockerBottleCleanupPlan
from .bottle_plan import DockerBottlePlan from .bottle_plan import DockerBottlePlan
from .mitmproxy import DockerMitmproxyProxy
from .pipelock import DockerPipelockProxy from .pipelock import DockerPipelockProxy
from .provision import ca as _ca
from .provision import git as _git from .provision import git as _git
from .provision import prompt as _prompt from .provision import prompt as _prompt
from .provision import skills as _skills from .provision import skills as _skills
@@ -38,15 +40,23 @@ class DockerBottleBackend(BottleBackend["DockerBottlePlan", "DockerBottleCleanup
def __init__(self) -> None: def __init__(self) -> None:
self._proxy = DockerPipelockProxy() self._proxy = DockerPipelockProxy()
self._mitm = DockerMitmproxyProxy()
def _resolve_plan(self, spec: BottleSpec, *, stage_dir: Path) -> DockerBottlePlan: def _resolve_plan(self, spec: BottleSpec, *, stage_dir: Path) -> DockerBottlePlan:
return _prepare.resolve_plan(spec, stage_dir=stage_dir, proxy=self._proxy) return _prepare.resolve_plan(
spec, stage_dir=stage_dir, proxy=self._proxy, mitm=self._mitm,
)
@contextmanager @contextmanager
def launch(self, plan: DockerBottlePlan) -> Generator[DockerBottle, None, None]: def launch(self, plan: DockerBottlePlan) -> Generator[DockerBottle, None, None]:
with _launch.launch(plan, proxy=self._proxy, provision=self.provision) as bottle: with _launch.launch(
plan, proxy=self._proxy, mitm=self._mitm, provision=self.provision,
) as bottle:
yield bottle yield bottle
def provision_ca(self, plan: DockerBottlePlan, target: str) -> None:
_ca.provision_ca(plan, target)
def provision_prompt(self, plan: DockerBottlePlan, target: str) -> str | None: def provision_prompt(self, plan: DockerBottlePlan, target: str) -> str | None:
return _prompt.provision_prompt(plan, target) return _prompt.provision_prompt(plan, target)
@@ -13,6 +13,7 @@ from pathlib import Path
from ...log import info from ...log import info
from ...manifest import Agent, Bottle from ...manifest import Agent, Bottle
from ...mitmproxy import MitmproxyProxyPlan
from ...pipelock import PipelockProxyPlan, pipelock_effective_allowlist from ...pipelock import PipelockProxyPlan, pipelock_effective_allowlist
from .. import BottlePlan from .. import BottlePlan
@@ -49,6 +50,7 @@ class DockerBottlePlan(BottlePlan):
forwarded_env: dict[str, str] = field(repr=False) forwarded_env: dict[str, str] = field(repr=False)
prompt_file: Path prompt_file: Path
proxy_plan: PipelockProxyPlan proxy_plan: PipelockProxyPlan
mitmproxy_plan: MitmproxyProxyPlan
allowlist_summary: str allowlist_summary: str
use_runsc: bool use_runsc: bool
+33 -1
View File
@@ -22,8 +22,15 @@ from . import network as network_mod
from . import util as docker_mod from . import util as docker_mod
from .bottle import DockerBottle from .bottle import DockerBottle
from .bottle_plan import DockerBottlePlan from .bottle_plan import DockerBottlePlan
from .mitmproxy import DockerMitmproxyProxy, mitmproxy_proxy_url
from .pipelock import DockerPipelockProxy, pipelock_proxy_url from .pipelock import DockerPipelockProxy, pipelock_proxy_url
# Path inside the agent container where the mitmproxy CA cert lives
# after provision_ca runs. Exported as a module-level constant so
# both the agent's docker-run env trio and the provisioner agree.
AGENT_CA_PATH = "/usr/local/share/ca-certificates/claude-bottle-mitm.crt"
AGENT_CA_BUNDLE = "/etc/ssl/certs/ca-certificates.crt"
# Where the repo root lives, for `docker build` context. Computed once. # Where the repo root lives, for `docker build` context. Computed once.
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent) _REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
@@ -34,6 +41,7 @@ def launch(
plan: DockerBottlePlan, plan: DockerBottlePlan,
*, *,
proxy: DockerPipelockProxy, proxy: DockerPipelockProxy,
mitm: DockerMitmproxyProxy,
provision: Callable[[DockerBottlePlan, str], str | None], provision: Callable[[DockerBottlePlan, str], str | None],
) -> Generator[DockerBottle, None, None]: ) -> Generator[DockerBottle, None, None]:
"""Build, launch, and provision a Docker bottle. Teardown on exit. """Build, launch, and provision a Docker bottle. Teardown on exit.
@@ -71,6 +79,17 @@ def launch(
pipelock_name = proxy.start(proxy_plan) pipelock_name = proxy.start(proxy_plan)
stack.callback(proxy.stop, pipelock_name) stack.callback(proxy.stop, pipelock_name)
# mitmproxy sits in front of pipelock on the agent's egress
# path. mitmproxy's `addon.py` reaches pipelock via the
# service-name URL we hand it here.
mitm_plan = dataclasses.replace(
plan.mitmproxy_plan,
internal_network=internal_network,
egress_network=egress_network,
)
mitm_name = mitm.start(mitm_plan, pipelock_url=pipelock_proxy_url(plan.slug))
stack.callback(mitm.stop, mitm_name)
container = _run_agent_container(plan, internal_network) container = _run_agent_container(plan, internal_network)
stack.callback(docker_mod.force_remove_container, container) stack.callback(docker_mod.force_remove_container, container)
@@ -85,7 +104,10 @@ def _run_agent_container(plan: DockerBottlePlan, internal_network: str) -> str:
"""Build the `docker run` argv and execute it, handling name- """Build the `docker run` argv and execute it, handling name-
conflict races by incrementing the suffix (unless the name was conflict races by incrementing the suffix (unless the name was
user-pinned). Returns the resolved container name.""" user-pinned). Returns the resolved container name."""
proxy_url = pipelock_proxy_url(plan.slug) # Agent traffic routes through mitmproxy, not pipelock directly.
# mitmproxy decrypts and hands the plaintext to pipelock via its
# addon; pipelock is unchanged from PRD 0001.
proxy_url = mitmproxy_proxy_url(plan.slug)
docker_args: list[str] = [ docker_args: list[str] = [
"--rm", "-d", "--rm", "-d",
"--name", plan.container_name, "--name", plan.container_name,
@@ -93,6 +115,16 @@ def _run_agent_container(plan: DockerBottlePlan, internal_network: str) -> str:
"-e", f"HTTPS_PROXY={proxy_url}", "-e", f"HTTPS_PROXY={proxy_url}",
"-e", f"HTTP_PROXY={proxy_url}", "-e", f"HTTP_PROXY={proxy_url}",
"-e", "NO_PROXY=localhost,127.0.0.1", "-e", "NO_PROXY=localhost,127.0.0.1",
# CA trust trio for the agent process. Docker propagates
# run-time env into `docker exec`, so `claude` sees these
# without per-exec threading. NODE_EXTRA_CA_CERTS points at
# the cert file (Node appends it to its bundled roots);
# SSL_CERT_FILE / REQUESTS_CA_BUNDLE point at the system
# bundle that `update-ca-certificates` rebuilds in
# provision_ca.
"-e", f"NODE_EXTRA_CA_CERTS={AGENT_CA_PATH}",
"-e", f"SSL_CERT_FILE={AGENT_CA_BUNDLE}",
"-e", f"REQUESTS_CA_BUNDLE={AGENT_CA_BUNDLE}",
] ]
if plan.use_runsc: if plan.use_runsc:
docker_args.extend(["--runtime", "runsc"]) docker_args.extend(["--runtime", "runsc"])
+4
View File
@@ -19,6 +19,7 @@ from ...log import die
from .. import BottleSpec from .. import BottleSpec
from . import util as docker_mod from . import util as docker_mod
from .bottle_plan import DockerBottlePlan from .bottle_plan import DockerBottlePlan
from .mitmproxy import DockerMitmproxyProxy
from .pipelock import DockerPipelockProxy from .pipelock import DockerPipelockProxy
@@ -27,6 +28,7 @@ def resolve_plan(
*, *,
stage_dir: Path, stage_dir: Path,
proxy: DockerPipelockProxy, proxy: DockerPipelockProxy,
mitm: DockerMitmproxyProxy,
) -> DockerBottlePlan: ) -> DockerBottlePlan:
"""Resolve Docker-specific names and write scratch files. Trusts """Resolve Docker-specific names and write scratch files. Trusts
that the agent and its skills/SSH keys are present — validation that the agent and its skills/SSH keys are present — validation
@@ -78,6 +80,7 @@ def resolve_plan(
prompt_file.chmod(0o600) prompt_file.chmod(0o600)
proxy_plan = proxy.prepare(bottle, slug, stage_dir) proxy_plan = proxy.prepare(bottle, slug, stage_dir)
mitmproxy_plan = mitm.prepare(slug)
resolved = resolve_env(manifest, spec.agent_name) resolved = resolve_env(manifest, spec.agent_name)
# Everything that should reach the bottle by-name (so its value # Everything that should reach the bottle by-name (so its value
# never lands on argv or in env_file) goes into one dict. The # never lands on argv or in env_file) goes into one dict. The
@@ -105,6 +108,7 @@ def resolve_plan(
forwarded_env=forwarded_env, forwarded_env=forwarded_env,
prompt_file=prompt_file, prompt_file=prompt_file,
proxy_plan=proxy_plan, proxy_plan=proxy_plan,
mitmproxy_plan=mitmproxy_plan,
allowlist_summary=allowlist_summary, allowlist_summary=allowlist_summary,
use_runsc=use_runsc, use_runsc=use_runsc,
) )
@@ -0,0 +1,55 @@
"""Extract mitmproxy's CA cert and install it into the agent
container's trust store.
mitmproxy generates a fresh CA on first launch inside its sidecar.
This provisioner pulls the public cert through a host stage dir,
drops it into the agent at `/usr/local/share/ca-certificates/...`,
runs `update-ca-certificates` to rebuild the system bundle, and
emits a single stderr log line with the SHA-256 fingerprint."""
from __future__ import annotations
import hashlib
import ssl
import subprocess
from ....log import info
from ..bottle_plan import DockerBottlePlan
from ..launch import AGENT_CA_PATH
from ..mitmproxy import DockerMitmproxyProxy, mitmproxy_container_name
def provision_ca(plan: DockerBottlePlan, target: str) -> None:
"""Pull mitmproxy's CA cert, install in the agent, log fingerprint.
Called from BottleBackend.provision after the agent container is
up. The mitmproxy sidecar is already running (started during
`launch`)."""
sidecar = mitmproxy_container_name(plan.mitmproxy_plan.slug)
stage_cert = plan.stage_dir / "mitm-ca.crt"
DockerMitmproxyProxy().extract_ca_cert(sidecar, stage_cert)
container = target
subprocess.run(
["docker", "cp", str(stage_cert), f"{container}:{AGENT_CA_PATH}"],
stdout=subprocess.DEVNULL,
check=True,
)
subprocess.run(
["docker", "exec", "-u", "0", container, "chmod", "644", AGENT_CA_PATH],
stdout=subprocess.DEVNULL,
check=True,
)
subprocess.run(
["docker", "exec", "-u", "0", container, "update-ca-certificates"],
stdout=subprocess.DEVNULL,
check=True,
)
# SHA-256 of the cert's DER bytes — the standard fingerprint
# form. stdlib only; never the private key (which stays in the
# sidecar). Logged once at launch as an audit signal.
pem = stage_cert.read_text()
der = ssl.PEM_cert_to_DER_cert(pem)
fingerprint = hashlib.sha256(der).hexdigest()
info(f"mitm ca fingerprint: sha256:{fingerprint[:32]}...")