chore: strip pipelock from Docker backend
- Remove pipelock_state_dir, _PIPELOCK_SUBDIR from bottle_state.py - Remove proxy_plan: PipelockProxyPlan from DockerBottlePlan - Remove EGRESS_PIPELOCK_CA_IN_CONTAINER from docker/egress.py - Remove pipelock TLS init and proxy_plan population from launch.py - Remove PipelockProxy import and pipelock_dir setup from prepare.py - Remove pipelock volumes, daemon entry, and network alias from compose.py - Remove pipelock mirroring entirely from egress_apply.py - Agent HTTP_PROXY now always points at egress (no pipelock fallback)
This commit is contained in:
@@ -11,7 +11,6 @@ from dataclasses import dataclass, field
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ...agent_provider import PromptMode
|
from ...agent_provider import PromptMode
|
||||||
from ...pipelock import PipelockProxyPlan
|
|
||||||
from .. import BottlePlan
|
from .. import BottlePlan
|
||||||
|
|
||||||
|
|
||||||
@@ -40,7 +39,6 @@ class DockerBottlePlan(BottlePlan):
|
|||||||
# accidental log of the plan dataclass.
|
# accidental log of the plan dataclass.
|
||||||
forwarded_env: dict[str, str] = field(repr=False)
|
forwarded_env: dict[str, str] = field(repr=False)
|
||||||
prompt_file: Path
|
prompt_file: Path
|
||||||
proxy_plan: PipelockProxyPlan
|
|
||||||
use_runsc: bool
|
use_runsc: bool
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|||||||
@@ -49,7 +49,6 @@ _TRANSCRIPT_SUBDIR = "transcript"
|
|||||||
# live here so chunk 3's `docker compose up` can find them at stable
|
# live here so chunk 3's `docker compose up` can find them at stable
|
||||||
# paths. Each sidecar's `prepare()` writes config + CAs into its own
|
# paths. Each sidecar's `prepare()` writes config + CAs into its own
|
||||||
# subdir; the launch step is unchanged today (still `docker cp`).
|
# subdir; the launch step is unchanged today (still `docker cp`).
|
||||||
_PIPELOCK_SUBDIR = "pipelock"
|
|
||||||
_EGRESS_SUBDIR = "egress"
|
_EGRESS_SUBDIR = "egress"
|
||||||
_GIT_GATE_SUBDIR = "git-gate"
|
_GIT_GATE_SUBDIR = "git-gate"
|
||||||
_SUPERVISE_SUBDIR = "supervise"
|
_SUPERVISE_SUBDIR = "supervise"
|
||||||
@@ -234,12 +233,6 @@ def transcript_snapshot_dir(identity: str) -> Path:
|
|||||||
# nothing requested preservation.
|
# nothing requested preservation.
|
||||||
|
|
||||||
|
|
||||||
def pipelock_state_dir(identity: str) -> Path:
|
|
||||||
"""State subdir for the pipelock sidecar: pipelock.yaml + the
|
|
||||||
per-bottle CA cert/key. Bind-mount source from chunk 3 onward."""
|
|
||||||
return bottle_state_dir(identity) / _PIPELOCK_SUBDIR
|
|
||||||
|
|
||||||
|
|
||||||
def egress_state_dir(identity: str) -> Path:
|
def egress_state_dir(identity: str) -> Path:
|
||||||
"""State subdir for the egress sidecar: routes.yaml + the
|
"""State subdir for the egress sidecar: routes.yaml + the
|
||||||
per-bottle mitmproxy CA. Bind-mount source from chunk 3 onward."""
|
per-bottle mitmproxy CA. Bind-mount source from chunk 3 onward."""
|
||||||
@@ -325,7 +318,6 @@ __all__ = [
|
|||||||
"per_bottle_dockerfile",
|
"per_bottle_dockerfile",
|
||||||
"per_bottle_dockerfile_path",
|
"per_bottle_dockerfile_path",
|
||||||
"per_bottle_image_tag",
|
"per_bottle_image_tag",
|
||||||
"pipelock_state_dir",
|
|
||||||
"preserve_marker_path",
|
"preserve_marker_path",
|
||||||
"read_metadata",
|
"read_metadata",
|
||||||
"supervise_state_dir",
|
"supervise_state_dir",
|
||||||
|
|||||||
@@ -7,34 +7,14 @@ two networks, no named volumes.
|
|||||||
|
|
||||||
Pure function. No I/O, no subprocess. Expects every launch-time
|
Pure function. No I/O, no subprocess. Expects every launch-time
|
||||||
field (network names, CA host paths, etc.) on the plan's inner
|
field (network names, CA host paths, etc.) on the plan's inner
|
||||||
plans to be populated; chunks 2+3 own that ordering. Chunk 1 just
|
plans to be populated; chunks 2+3 own that ordering.
|
||||||
encodes the translation so it can be unit-tested in isolation.
|
|
||||||
|
|
||||||
Conditional services follow the plan content (matches the
|
Conditional services follow the plan content:
|
||||||
SDK-call branching in `launch.py` today):
|
|
||||||
|
|
||||||
- pipelock + agent: always.
|
- agent + sidecars bundle: always.
|
||||||
- git-gate: iff plan.git_gate_plan.upstreams.
|
- git-gate: iff plan.git_gate_plan.upstreams.
|
||||||
- egress: iff plan.egress_plan.routes.
|
- egress: iff plan.egress_plan.routes.
|
||||||
- supervise: iff plan.supervise_plan is not None.
|
- supervise: iff plan.supervise_plan is not None.
|
||||||
|
|
||||||
Naming:
|
|
||||||
|
|
||||||
- Compose project: `bot-bottle-<slug>`.
|
|
||||||
- Service names (inside the file): `agent`, `pipelock`,
|
|
||||||
`egress`, `git-gate`, `supervise`.
|
|
||||||
- `container_name:` matches today's pattern
|
|
||||||
(`bot-bottle-<service>-<slug>`) so dashboard/cleanup discovery
|
|
||||||
via the prefix scan keeps working through the transition.
|
|
||||||
- Network aliases preserve the current dial-by-shortname pattern
|
|
||||||
for `egress` / `supervise`, and add the long container-name as
|
|
||||||
an internal-network alias for `pipelock` / `git-gate` so any
|
|
||||||
caller still referencing the long name resolves.
|
|
||||||
|
|
||||||
Sidecars that are built (egress, git-gate, supervise) get a
|
|
||||||
compose `build:` block pointing at the repo Dockerfile; the
|
|
||||||
`image:` tag is set explicitly so cached images on the daemon
|
|
||||||
aren't rebuilt on every up.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -51,7 +31,6 @@ from ...egress import (
|
|||||||
)
|
)
|
||||||
from ...git_gate import GIT_GATE_HOSTNAME
|
from ...git_gate import GIT_GATE_HOSTNAME
|
||||||
from ...log import die, warn
|
from ...log import die, warn
|
||||||
from ...pipelock import PIPELOCK_HOSTNAME
|
|
||||||
from ...supervise import (
|
from ...supervise import (
|
||||||
CURRENT_CONFIG_DIR_IN_AGENT,
|
CURRENT_CONFIG_DIR_IN_AGENT,
|
||||||
QUEUE_DIR_IN_CONTAINER,
|
QUEUE_DIR_IN_CONTAINER,
|
||||||
@@ -63,7 +42,7 @@ from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
|||||||
from .bottle_plan import DockerBottlePlan
|
from .bottle_plan import DockerBottlePlan
|
||||||
from .egress import (
|
from .egress import (
|
||||||
EGRESS_CA_IN_CONTAINER,
|
EGRESS_CA_IN_CONTAINER,
|
||||||
EGRESS_PIPELOCK_CA_IN_CONTAINER,
|
EGRESS_PORT,
|
||||||
)
|
)
|
||||||
from .git_gate import (
|
from .git_gate import (
|
||||||
GIT_GATE_ACCESS_HOOK_IN_CONTAINER,
|
GIT_GATE_ACCESS_HOOK_IN_CONTAINER,
|
||||||
@@ -71,11 +50,6 @@ from .git_gate import (
|
|||||||
GIT_GATE_ENTRYPOINT_IN_CONTAINER,
|
GIT_GATE_ENTRYPOINT_IN_CONTAINER,
|
||||||
GIT_GATE_HOOK_IN_CONTAINER,
|
GIT_GATE_HOOK_IN_CONTAINER,
|
||||||
)
|
)
|
||||||
from ...pipelock import (
|
|
||||||
PIPELOCK_CA_CERT_IN_CONTAINER,
|
|
||||||
PIPELOCK_CA_KEY_IN_CONTAINER,
|
|
||||||
)
|
|
||||||
from .pipelock import PIPELOCK_PORT
|
|
||||||
from .sidecar_bundle import (
|
from .sidecar_bundle import (
|
||||||
SIDECAR_BUNDLE_DOCKERFILE,
|
SIDECAR_BUNDLE_DOCKERFILE,
|
||||||
SIDECAR_BUNDLE_IMAGE,
|
SIDECAR_BUNDLE_IMAGE,
|
||||||
@@ -91,12 +65,11 @@ def bottle_plan_to_compose(plan: DockerBottlePlan) -> dict[str, Any]:
|
|||||||
"""Render a Compose v2 spec dict from a fully-resolved
|
"""Render a Compose v2 spec dict from a fully-resolved
|
||||||
DockerBottlePlan.
|
DockerBottlePlan.
|
||||||
|
|
||||||
The plan must have its inner plans (`proxy_plan`,
|
The plan must have its inner plans (`git_gate_plan`,
|
||||||
`git_gate_plan`, `egress_plan`, `supervise_plan`) populated
|
`egress_plan`, `supervise_plan`) populated with launch-time
|
||||||
with launch-time fields — network names, CA host paths,
|
fields — network names, CA host paths. The renderer doesn't
|
||||||
pipelock_proxy_url. The renderer doesn't validate; callers
|
validate; callers feed it a fully-resolved plan or get an
|
||||||
feed it a fully-resolved plan or get an incomplete compose
|
incomplete compose spec back.
|
||||||
spec back.
|
|
||||||
"""
|
"""
|
||||||
project = f"bot-bottle-{plan.slug}"
|
project = f"bot-bottle-{plan.slug}"
|
||||||
services: dict[str, Any] = {
|
services: dict[str, Any] = {
|
||||||
@@ -142,29 +115,12 @@ def _bind(host: str | Path, target: str, *, read_only: bool = True) -> dict[str,
|
|||||||
|
|
||||||
def _sidecar_bundle_service(plan: DockerBottlePlan) -> dict[str, Any]:
|
def _sidecar_bundle_service(plan: DockerBottlePlan) -> dict[str, Any]:
|
||||||
"""The `sidecars` service: one container per bottle, bundle
|
"""The `sidecars` service: one container per bottle, bundle
|
||||||
image, all four daemons under a Python init supervisor.
|
image, all daemons under a Python init supervisor.
|
||||||
|
|
||||||
Mechanics:
|
Daemon subset narrows via `BOT_BOTTLE_SIDECAR_DAEMONS` env.
|
||||||
|
egress is always present; git-gate / supervise are conditional.
|
||||||
- Daemon subset narrows via `BOT_BOTTLE_SIDECAR_DAEMONS`
|
|
||||||
env. pipelock is always present; egress / git-gate /
|
|
||||||
supervise are conditional on the plan.
|
|
||||||
- Volumes are the union of the four daemons' bind-mounts,
|
|
||||||
preserving the same in-container paths so each daemon
|
|
||||||
finds its config / hooks / CA where it expects.
|
|
||||||
- Environment is the union of *daemon-private* env vars
|
|
||||||
(EGRESS_UPSTREAM_PROXY, SUPERVISE_BOTTLE_SLUG, etc).
|
|
||||||
HTTPS_PROXY is NOT propagated here — see the comment in
|
|
||||||
egress_entrypoint.sh; setting it at the container level
|
|
||||||
would route git-gate's git fetches through pipelock,
|
|
||||||
which is wrong.
|
|
||||||
- Network aliases register every legacy short/long
|
|
||||||
hostname (pipelock, egress, git-gate, supervise plus
|
|
||||||
their `bot-bottle-<service>-<slug>` long forms) so
|
|
||||||
the agent's HTTPS_PROXY URL and any other inter-service
|
|
||||||
reference resolves to the bundle.
|
|
||||||
"""
|
"""
|
||||||
daemons: list[str] = ["egress", "pipelock"]
|
daemons: list[str] = ["egress"]
|
||||||
if plan.git_gate_plan.upstreams:
|
if plan.git_gate_plan.upstreams:
|
||||||
daemons.append("git-gate")
|
daemons.append("git-gate")
|
||||||
if plan.supervise_plan is not None:
|
if plan.supervise_plan is not None:
|
||||||
@@ -173,31 +129,17 @@ def _sidecar_bundle_service(plan: DockerBottlePlan) -> dict[str, Any]:
|
|||||||
env: list[str] = [f"BOT_BOTTLE_SIDECAR_DAEMONS={','.join(daemons)}"]
|
env: list[str] = [f"BOT_BOTTLE_SIDECAR_DAEMONS={','.join(daemons)}"]
|
||||||
volumes: list[dict[str, Any]] = []
|
volumes: list[dict[str, Any]] = []
|
||||||
|
|
||||||
# --- pipelock ----------------------------------------------------
|
# --- egress -------------------------------------------------------
|
||||||
pp = plan.proxy_plan
|
|
||||||
volumes += [
|
|
||||||
_bind(pp.yaml_path, "/etc/pipelock.yaml"),
|
|
||||||
_bind(pp.ca_cert_host_path, PIPELOCK_CA_CERT_IN_CONTAINER),
|
|
||||||
_bind(pp.ca_key_host_path, PIPELOCK_CA_KEY_IN_CONTAINER),
|
|
||||||
]
|
|
||||||
|
|
||||||
# --- egress (always part of the bundle; the EGRESS_UPSTREAM_*
|
|
||||||
# env vars + ca bind-mounts are needed iff routes exist; when
|
|
||||||
# the bottle has no routes the egress daemon falls back to its
|
|
||||||
# `regular@9099` mode and is unused) -----------------------------
|
|
||||||
ep = plan.egress_plan
|
ep = plan.egress_plan
|
||||||
if ep.routes:
|
if ep.routes:
|
||||||
env.append(f"EGRESS_UPSTREAM_PROXY={ep.pipelock_proxy_url}")
|
|
||||||
env.append(f"EGRESS_UPSTREAM_CA={EGRESS_PIPELOCK_CA_IN_CONTAINER}")
|
|
||||||
volumes += [
|
volumes += [
|
||||||
_bind(ep.routes_path, EGRESS_ROUTES_IN_CONTAINER),
|
_bind(ep.routes_path, EGRESS_ROUTES_IN_CONTAINER),
|
||||||
_bind(ep.mitmproxy_ca_host_path, EGRESS_CA_IN_CONTAINER),
|
_bind(ep.mitmproxy_ca_host_path, EGRESS_CA_IN_CONTAINER),
|
||||||
_bind(ep.pipelock_ca_host_path, EGRESS_PIPELOCK_CA_IN_CONTAINER),
|
|
||||||
]
|
]
|
||||||
for token_env in sorted(ep.token_env_map.keys()):
|
for token_env in sorted(ep.token_env_map.keys()):
|
||||||
env.append(token_env)
|
env.append(token_env)
|
||||||
|
|
||||||
# --- git-gate ----------------------------------------------------
|
# --- git-gate -----------------------------------------------------
|
||||||
gp = plan.git_gate_plan
|
gp = plan.git_gate_plan
|
||||||
if gp.upstreams:
|
if gp.upstreams:
|
||||||
volumes += [
|
volumes += [
|
||||||
@@ -217,7 +159,7 @@ def _sidecar_bundle_service(plan: DockerBottlePlan) -> dict[str, Any]:
|
|||||||
f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{u.name}-known_hosts",
|
f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{u.name}-known_hosts",
|
||||||
))
|
))
|
||||||
|
|
||||||
# --- supervise ---------------------------------------------------
|
# --- supervise ----------------------------------------------------
|
||||||
sp = plan.supervise_plan
|
sp = plan.supervise_plan
|
||||||
if sp is not None:
|
if sp is not None:
|
||||||
env += [
|
env += [
|
||||||
@@ -232,13 +174,7 @@ def _sidecar_bundle_service(plan: DockerBottlePlan) -> dict[str, Any]:
|
|||||||
"read_only": False,
|
"read_only": False,
|
||||||
})
|
})
|
||||||
|
|
||||||
# Internal-network aliases: the agent reaches each daemon through
|
internal_aliases = [EGRESS_HOSTNAME]
|
||||||
# its short name (pipelock / egress / git-gate / supervise) which
|
|
||||||
# the bundle answers as if it were the daemon itself.
|
|
||||||
internal_aliases = [
|
|
||||||
PIPELOCK_HOSTNAME,
|
|
||||||
EGRESS_HOSTNAME,
|
|
||||||
]
|
|
||||||
if gp.upstreams:
|
if gp.upstreams:
|
||||||
internal_aliases.append(GIT_GATE_HOSTNAME)
|
internal_aliases.append(GIT_GATE_HOSTNAME)
|
||||||
if sp is not None:
|
if sp is not None:
|
||||||
@@ -263,11 +199,8 @@ def _sidecar_bundle_service(plan: DockerBottlePlan) -> dict[str, Any]:
|
|||||||
|
|
||||||
def _agent_service(plan: DockerBottlePlan) -> dict[str, Any]:
|
def _agent_service(plan: DockerBottlePlan) -> dict[str, Any]:
|
||||||
"""Agent container. Runs `sleep infinity`; claude is `docker
|
"""Agent container. Runs `sleep infinity`; claude is `docker
|
||||||
exec -it`'d into it later. No TTY at the container level —
|
exec -it`'d into it later. HTTP_PROXY/HTTPS_PROXY point at the
|
||||||
interactivity is per-exec. HTTP_PROXY/HTTPS_PROXY point at the
|
egress sidecar."""
|
||||||
egress short-alias when an egress is declared, otherwise
|
|
||||||
straight at pipelock's container name. CA trust trio matches
|
|
||||||
the existing launch.py wiring."""
|
|
||||||
proxy_url = _agent_proxy_url(plan)
|
proxy_url = _agent_proxy_url(plan)
|
||||||
no_proxy = _agent_no_proxy(plan)
|
no_proxy = _agent_no_proxy(plan)
|
||||||
env: list[str] = [
|
env: list[str] = [
|
||||||
@@ -319,21 +252,14 @@ def _agent_service(plan: DockerBottlePlan) -> dict[str, Any]:
|
|||||||
|
|
||||||
|
|
||||||
def _agent_proxy_url(plan: DockerBottlePlan) -> str:
|
def _agent_proxy_url(plan: DockerBottlePlan) -> str:
|
||||||
"""Pick the agent's HTTP_PROXY. With egress declared, the agent
|
"""Agent's HTTP_PROXY — always points at egress."""
|
||||||
goes through egress (which in turn HTTPS_PROXYs to pipelock on
|
return f"http://{EGRESS_HOSTNAME}:{EGRESS_PORT}"
|
||||||
its outbound leg). Without egress, the agent talks straight to
|
|
||||||
pipelock."""
|
|
||||||
if plan.egress_plan.routes:
|
|
||||||
from .egress import EGRESS_PORT
|
|
||||||
return f"http://{EGRESS_HOSTNAME}:{EGRESS_PORT}"
|
|
||||||
return f"http://{PIPELOCK_HOSTNAME}:{PIPELOCK_PORT}"
|
|
||||||
|
|
||||||
|
|
||||||
def _agent_no_proxy(plan: DockerBottlePlan) -> str:
|
def _agent_no_proxy(plan: DockerBottlePlan) -> str:
|
||||||
"""NO_PROXY for the agent. Matches the launch.py rules:
|
"""NO_PROXY for the agent: loopback always; supervise hostname
|
||||||
loopback always, supervise hostname when the supervise sidecar
|
when the supervise sidecar is up (MCP long-poll must bypass
|
||||||
is up (the MCP long-poll pattern needs to bypass pipelock's
|
the egress proxy)."""
|
||||||
idle timeout)."""
|
|
||||||
hosts = ["localhost", "127.0.0.1"]
|
hosts = ["localhost", "127.0.0.1"]
|
||||||
if plan.supervise_plan is not None:
|
if plan.supervise_plan is not None:
|
||||||
hosts.append(SUPERVISE_HOSTNAME)
|
hosts.append(SUPERVISE_HOSTNAME)
|
||||||
|
|||||||
@@ -22,14 +22,8 @@ from ...log import die
|
|||||||
EGRESS_PORT = int(os.environ.get("BOT_BOTTLE_EGRESS_PORT", "9099"))
|
EGRESS_PORT = int(os.environ.get("BOT_BOTTLE_EGRESS_PORT", "9099"))
|
||||||
|
|
||||||
# In-container path for mitmproxy's CA. The format is a single PEM
|
# In-container path for mitmproxy's CA. The format is a single PEM
|
||||||
# file holding BOTH the cert and the private key, concatenated. The
|
# file holding BOTH the cert and the private key, concatenated.
|
||||||
# upstream-trust CA (pipelock's, so egress trusts the upstream
|
|
||||||
# leg) is a separate file because pipelock keeps a different CA on
|
|
||||||
# its end.
|
|
||||||
EGRESS_CA_IN_CONTAINER = "/home/mitmproxy/.mitmproxy/mitmproxy-ca.pem"
|
EGRESS_CA_IN_CONTAINER = "/home/mitmproxy/.mitmproxy/mitmproxy-ca.pem"
|
||||||
EGRESS_PIPELOCK_CA_IN_CONTAINER = (
|
|
||||||
"/home/mitmproxy/.mitmproxy/pipelock-ca.pem"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def egress_tls_init(stage_dir: Path) -> tuple[Path, Path]:
|
def egress_tls_init(stage_dir: Path) -> tuple[Path, Path]:
|
||||||
@@ -42,16 +36,8 @@ def egress_tls_init(stage_dir: Path) -> tuple[Path, Path]:
|
|||||||
trust store by `provision_ca` so the agent trusts the bumped
|
trust store by `provision_ca` so the agent trusts the bumped
|
||||||
CONNECT cert egress presents.
|
CONNECT cert egress presents.
|
||||||
|
|
||||||
Why openssl req (not the pipelock binary's `tls init`):
|
openssl req's `subjectKeyIdentifier=hash` extension uses
|
||||||
pipelock's CA generator stamps a non-standard `Subject Key
|
SHA-1(pubkey), matching mitmproxy's AKI computation on leaves.
|
||||||
Identifier` on the CA (random rather than SHA-1 of the pubkey).
|
|
||||||
mitmproxy computes the `Authority Key Identifier` on each leaf
|
|
||||||
it mints as SHA-1(issuer's pubkey). openssl's chain validator
|
|
||||||
uses the leaf's AKI to find the issuer cert by SKI; pipelock's
|
|
||||||
SKI doesn't match → openssl reports "unable to get local issuer
|
|
||||||
certificate" even though the CA is right there in the trust
|
|
||||||
store. openssl req's `subjectKeyIdentifier=hash` extension uses
|
|
||||||
SHA-1(pubkey), matching mitmproxy's computation.
|
|
||||||
|
|
||||||
Both files live under `<stage_dir>/egress-ca/` (mode 644 —
|
Both files live under `<stage_dir>/egress-ca/` (mode 644 —
|
||||||
`docker cp` preserves the mode into the container, where the
|
`docker cp` preserves the mode into the container, where the
|
||||||
|
|||||||
@@ -8,13 +8,6 @@ egress-block proposal (or runs the operator-initiated
|
|||||||
sidecar via `docker cp`, then `docker kill --signal HUP` to make
|
sidecar via `docker cp`, then `docker kill --signal HUP` to make
|
||||||
the addon reload without dropping connections.
|
the addon reload without dropping connections.
|
||||||
|
|
||||||
Also mirrors the new route hosts into pipelock's hostname allowlist
|
|
||||||
so the downstream leg lets them through — egress enforces
|
|
||||||
the path-aware allowlist on the agent leg, pipelock enforces the
|
|
||||||
hostname allowlist + DLP body scan on the upstream leg, and a
|
|
||||||
host added to one must be in the other or the request 403s
|
|
||||||
somewhere along the chain.
|
|
||||||
|
|
||||||
Raises EgressApplyError on any failure — the dashboard
|
Raises EgressApplyError on any failure — the dashboard
|
||||||
surfaces the message and keeps the proposal pending so the
|
surfaces the message and keeps the proposal pending so the
|
||||||
operator can retry.
|
operator can retry.
|
||||||
@@ -23,7 +16,6 @@ operator can retry.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import re
|
|
||||||
import subprocess
|
import subprocess
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import cast
|
from typing import cast
|
||||||
@@ -33,13 +25,6 @@ from ...egress_addon_core import load_routes
|
|||||||
from ...yaml_subset import YamlSubsetError, parse_yaml_subset
|
from ...yaml_subset import YamlSubsetError, parse_yaml_subset
|
||||||
from .bottle_state import egress_state_dir
|
from .bottle_state import egress_state_dir
|
||||||
from .sidecar_bundle import sidecar_bundle_container_name
|
from .sidecar_bundle import sidecar_bundle_container_name
|
||||||
from .pipelock_apply import (
|
|
||||||
PipelockApplyError,
|
|
||||||
apply_allowlist_change,
|
|
||||||
fetch_current_allowlist,
|
|
||||||
parse_allowlist_content,
|
|
||||||
render_allowlist_content,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _render_routes_payload(routes_list: list[dict[str, object]]) -> str:
|
def _render_routes_payload(routes_list: list[dict[str, object]]) -> str:
|
||||||
@@ -108,82 +93,12 @@ def validate_routes_content(content: str) -> None:
|
|||||||
) from e
|
) from e
|
||||||
|
|
||||||
|
|
||||||
def _hosts_in_routes(content: str) -> list[str]:
|
|
||||||
"""Extract the host list from a routes.yaml content string.
|
|
||||||
Uses the addon's own parser so any host the addon will match on
|
|
||||||
also lands in pipelock's allowlist. Returns sorted+deduped."""
|
|
||||||
try:
|
|
||||||
routes = load_routes(content)
|
|
||||||
except ValueError as e:
|
|
||||||
raise EgressApplyError(
|
|
||||||
f"proposed routes.yaml is not valid: {e}"
|
|
||||||
) from e
|
|
||||||
return sorted({r.host for r in routes if r.host})
|
|
||||||
|
|
||||||
|
|
||||||
# Pipelock's allowlist parser accepts only literal hostnames:
|
|
||||||
# `[A-Za-z0-9_.-]+`. Anything else (wildcards, IPv6 literals,
|
|
||||||
# stray characters) is silently dropped from the mirror so the
|
|
||||||
# pipelock apply doesn't fail parse before the new yaml is even
|
|
||||||
# written. The dropped hosts stay on egress's route table —
|
|
||||||
# but the addon does exact-host match only, so they'll never
|
|
||||||
# match anything either. (Wildcard host matching was removed —
|
|
||||||
# see `match_route` in egress_addon_core for the rationale.)
|
|
||||||
_PIPELOCK_HOST_RE = re.compile(r"^[A-Za-z0-9_.-]+$")
|
|
||||||
|
|
||||||
|
|
||||||
def _pipelock_safe_hosts(hosts: list[str]) -> list[str]:
|
|
||||||
"""Drop any host pipelock's allowlist parser would reject.
|
|
||||||
Order preserved."""
|
|
||||||
return [h for h in hosts if _PIPELOCK_HOST_RE.match(h)]
|
|
||||||
|
|
||||||
|
|
||||||
def _mirror_hosts_to_pipelock(slug: str, hosts: list[str]) -> None:
|
|
||||||
"""Ensure every pipelock-compatible `hosts` entry is on
|
|
||||||
pipelock's allowlist. Fetches pipelock's current allowlist,
|
|
||||||
merges, re-applies. Hosts pipelock can't represent (wildcards,
|
|
||||||
etc.) are silently skipped — they stay live on egress
|
|
||||||
but aren't enforced at pipelock. No-op if every host is already
|
|
||||||
present (apply still restarts pipelock if any host is new).
|
|
||||||
Raises EgressApplyError on pipelock failures so the
|
|
||||||
caller's diff/audit reflects the half-state."""
|
|
||||||
safe_hosts = _pipelock_safe_hosts(hosts)
|
|
||||||
try:
|
|
||||||
current = fetch_current_allowlist(slug)
|
|
||||||
existing = parse_allowlist_content(current)
|
|
||||||
merged = sorted(set(existing) | set(safe_hosts))
|
|
||||||
if merged == sorted(existing):
|
|
||||||
return # nothing to add
|
|
||||||
apply_allowlist_change(slug, render_allowlist_content(merged))
|
|
||||||
except PipelockApplyError as e:
|
|
||||||
# Mirror runs BEFORE the egress write, so egress
|
|
||||||
# is unchanged on this failure path. Report it as a
|
|
||||||
# pipelock-side problem so the operator looks in the right
|
|
||||||
# place; their `pipelock edit` flow can repair manually.
|
|
||||||
raise EgressApplyError(
|
|
||||||
f"pipelock allowlist mirror failed (egress NOT "
|
|
||||||
f"updated): {e}. Fix pipelock's allowlist manually with "
|
|
||||||
f"`pipelock edit <bottle>` then retry the proposal."
|
|
||||||
) from e
|
|
||||||
|
|
||||||
|
|
||||||
def apply_routes_change(slug: str, new_content: str) -> tuple[str, str]:
|
def apply_routes_change(slug: str, new_content: str) -> tuple[str, str]:
|
||||||
"""Apply `new_content` to the egress sidecar for `slug`:
|
"""Apply `new_content` to the egress sidecar for `slug`:
|
||||||
1. Fetch current routes.yaml (for the before-diff).
|
1. Fetch current routes.yaml (for the before-diff).
|
||||||
2. Validate the new content via the addon's own parser.
|
2. Validate the new content via the addon's own parser.
|
||||||
3. Mirror the route hosts onto pipelock's allowlist (so the
|
3. Write to the bind-mount source path.
|
||||||
downstream hostname gate lets them through).
|
4. `docker kill --signal HUP` so the addon reloads.
|
||||||
4. Write to a temp file, `docker cp` into the egress
|
|
||||||
sidecar.
|
|
||||||
5. `docker kill --signal HUP` so the addon reloads.
|
|
||||||
|
|
||||||
Order matters: pipelock first, then egress. If the
|
|
||||||
pipelock step fails, egress hasn't been touched and the
|
|
||||||
old routes stay live. If the egress step fails after
|
|
||||||
pipelock succeeded, pipelock has the host in its allowlist but
|
|
||||||
egress doesn't enforce it yet — harmless extra-permissive
|
|
||||||
state at pipelock, and a re-approval will land the egress
|
|
||||||
side.
|
|
||||||
|
|
||||||
Returns (before, after) where `after` == `new_content`. Raises
|
Returns (before, after) where `after` == `new_content`. Raises
|
||||||
EgressApplyError on any step."""
|
EgressApplyError on any step."""
|
||||||
@@ -191,10 +106,6 @@ def apply_routes_change(slug: str, new_content: str) -> tuple[str, str]:
|
|||||||
before = fetch_current_routes(slug)
|
before = fetch_current_routes(slug)
|
||||||
validate_routes_content(new_content)
|
validate_routes_content(new_content)
|
||||||
|
|
||||||
# Pipelock mirror first — if it fails, egress stays intact
|
|
||||||
# and the operator gets a clear error about the half-state.
|
|
||||||
_mirror_hosts_to_pipelock(slug, _hosts_in_routes(new_content))
|
|
||||||
|
|
||||||
# routes.yaml is bind-mounted into the egress container as a
|
# routes.yaml is bind-mounted into the egress container as a
|
||||||
# SINGLE FILE. Docker single-file bind mounts pin the source
|
# SINGLE FILE. Docker single-file bind mounts pin the source
|
||||||
# inode at mount time; write-temp-then-rename swaps the inode
|
# inode at mount time; write-temp-then-rename swaps the inode
|
||||||
@@ -209,12 +120,6 @@ def apply_routes_change(slug: str, new_content: str) -> tuple[str, str]:
|
|||||||
target = _egress_routes_host_path(slug)
|
target = _egress_routes_host_path(slug)
|
||||||
target.parent.mkdir(parents=True, exist_ok=True)
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
target.write_text(new_content)
|
target.write_text(new_content)
|
||||||
# mitmproxy in the container reads through the bind mount as
|
|
||||||
# uid 1000; the host file has to be world-readable for that
|
|
||||||
# read to succeed (parent dir at 0o700 still restricts who
|
|
||||||
# can reach the file on the host). Routes content is not
|
|
||||||
# secret — tokens live in the container's environ — so 0o644
|
|
||||||
# is the right trade-off.
|
|
||||||
target.chmod(0o644)
|
target.chmod(0o644)
|
||||||
sig = subprocess.run(
|
sig = subprocess.run(
|
||||||
["docker", "kill", "--signal", "HUP", container],
|
["docker", "kill", "--signal", "HUP", container],
|
||||||
@@ -311,13 +216,6 @@ def _merge_single_route(
|
|||||||
next_idx = len(existing_slots)
|
next_idx = len(existing_slots)
|
||||||
entry_typed["auth_scheme"] = str(cast(object, auth_typed.get("scheme")))
|
entry_typed["auth_scheme"] = str(cast(object, auth_typed.get("scheme")))
|
||||||
entry_typed["token_env"] = f"EGRESS_TOKEN_{next_idx}"
|
entry_typed["token_env"] = f"EGRESS_TOKEN_{next_idx}"
|
||||||
# NOTE: the addon reads token VALUES from its container's
|
|
||||||
# environ keyed by token_env. A newly-added auth route at
|
|
||||||
# runtime points at a slot that has no env value → the
|
|
||||||
# addon will 403 with "token env unset" until the operator
|
|
||||||
# arranges for the value to land in the container's env.
|
|
||||||
# Recording this here so the operator-facing diff carries
|
|
||||||
# the slot name they'll need to provision.
|
|
||||||
routes_typed.append(entry_typed)
|
routes_typed.append(entry_typed)
|
||||||
|
|
||||||
return _render_routes_payload(cast(list[dict[str, object]], routes_typed))
|
return _render_routes_payload(cast(list[dict[str, object]], routes_typed))
|
||||||
|
|||||||
@@ -6,16 +6,10 @@ The flow is:
|
|||||||
|
|
||||||
1. Build the agent's base + derived image (compose builds the
|
1. Build the agent's base + derived image (compose builds the
|
||||||
sidecar images via the `build:` directive on first up).
|
sidecar images via the `build:` directive on first up).
|
||||||
2. Pre-create the per-bottle networks. We do this outside compose
|
2. Mint the per-bottle egress CA (chunk 2 writes it under
|
||||||
so we can inspect the assigned internal CIDR and embed it in
|
state/<slug>/egress/).
|
||||||
pipelock's yaml (compose's `external: true` lets the compose
|
3. Populate the inner plans with launch-time fields so the
|
||||||
file reference these pre-existing networks).
|
renderer can read network names, CA paths.
|
||||||
3. Mint the per-bottle CAs (chunk 2 writes them under
|
|
||||||
state/<slug>/{pipelock,egress}/).
|
|
||||||
4. Re-render pipelock yaml with the now-known internal CIDR so
|
|
||||||
the SSRF allowlist exempts the bottle's own subnet.
|
|
||||||
5. Populate the inner plans with launch-time fields so the
|
|
||||||
renderer can read network names, CA paths, pipelock URL.
|
|
||||||
6. Render the compose spec, write it to
|
6. Render the compose spec, write it to
|
||||||
state/<slug>/docker-compose.yml, write metadata.json.
|
state/<slug>/docker-compose.yml, write metadata.json.
|
||||||
7. `docker compose up -d` (token + OAuth values flow into the
|
7. `docker compose up -d` (token + OAuth values flow into the
|
||||||
@@ -53,7 +47,6 @@ from .bottle_state import (
|
|||||||
bottle_state_dir,
|
bottle_state_dir,
|
||||||
egress_state_dir,
|
egress_state_dir,
|
||||||
git_gate_state_dir,
|
git_gate_state_dir,
|
||||||
pipelock_state_dir,
|
|
||||||
)
|
)
|
||||||
from .compose import (
|
from .compose import (
|
||||||
bottle_plan_to_compose,
|
bottle_plan_to_compose,
|
||||||
@@ -66,10 +59,6 @@ from .compose import (
|
|||||||
write_compose_file,
|
write_compose_file,
|
||||||
)
|
)
|
||||||
from .egress import egress_tls_init
|
from .egress import egress_tls_init
|
||||||
from .pipelock import (
|
|
||||||
BUNDLE_LOCAL_PIPELOCK_URL,
|
|
||||||
pipelock_tls_init,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# Where the repo root lives, for `docker build` context. Computed once.
|
# Where the repo root lives, for `docker build` context. Computed once.
|
||||||
@@ -113,35 +102,13 @@ def launch(
|
|||||||
plan.derived_image, plan.image, plan.workspace_plan
|
plan.derived_image, plan.image, plan.workspace_plan
|
||||||
)
|
)
|
||||||
|
|
||||||
# Networks: compose-managed. The names are derived
|
|
||||||
# deterministically from the slug so the renderer can put
|
|
||||||
# them on the services and `compose up` creates them with
|
|
||||||
# those names. The empirical spike confirmed pipelock's
|
|
||||||
# SSRF guard only checks proxied-request destinations, not
|
|
||||||
# source IPs — so the bottle's own internal CIDR doesn't
|
|
||||||
# need to be in `ssrf.ip_allowlist`. Pre-create + CIDR
|
|
||||||
# introspection are gone; compose owns the network
|
|
||||||
# lifecycle.
|
|
||||||
internal_network = network_mod.network_name_for_slug(plan.slug)
|
internal_network = network_mod.network_name_for_slug(plan.slug)
|
||||||
egress_network = network_mod.network_egress_name_for_slug(plan.slug)
|
egress_network = network_mod.network_egress_name_for_slug(plan.slug)
|
||||||
|
|
||||||
# Mint per-bottle CAs into state/<slug>/{pipelock,egress}/.
|
|
||||||
ca_cert_host, ca_key_host = pipelock_tls_init(pipelock_state_dir(plan.slug))
|
|
||||||
egress_ca_host, egress_ca_cert_only = egress_tls_init(
|
egress_ca_host, egress_ca_cert_only = egress_tls_init(
|
||||||
egress_state_dir(plan.slug),
|
egress_state_dir(plan.slug),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Populate launch-time fields on every inner plan so the
|
|
||||||
# renderer reads concrete network names, CA paths, and
|
|
||||||
# pipelock URL.
|
|
||||||
proxy_plan = dataclasses.replace(
|
|
||||||
plan.proxy_plan,
|
|
||||||
internal_network=internal_network,
|
|
||||||
internal_network_cidr="",
|
|
||||||
egress_network=egress_network,
|
|
||||||
ca_cert_host_path=ca_cert_host,
|
|
||||||
ca_key_host_path=ca_key_host,
|
|
||||||
)
|
|
||||||
git_gate_plan = plan.git_gate_plan
|
git_gate_plan = plan.git_gate_plan
|
||||||
if git_gate_plan.upstreams:
|
if git_gate_plan.upstreams:
|
||||||
git_gate_plan = dataclasses.replace(
|
git_gate_plan = dataclasses.replace(
|
||||||
@@ -157,8 +124,6 @@ def launch(
|
|||||||
egress_network=egress_network,
|
egress_network=egress_network,
|
||||||
mitmproxy_ca_host_path=egress_ca_host,
|
mitmproxy_ca_host_path=egress_ca_host,
|
||||||
mitmproxy_ca_cert_only_host_path=egress_ca_cert_only,
|
mitmproxy_ca_cert_only_host_path=egress_ca_cert_only,
|
||||||
pipelock_ca_host_path=ca_cert_host,
|
|
||||||
pipelock_proxy_url=BUNDLE_LOCAL_PIPELOCK_URL,
|
|
||||||
)
|
)
|
||||||
supervise_plan = plan.supervise_plan
|
supervise_plan = plan.supervise_plan
|
||||||
if supervise_plan is not None:
|
if supervise_plan is not None:
|
||||||
@@ -168,7 +133,6 @@ def launch(
|
|||||||
)
|
)
|
||||||
plan = dataclasses.replace(
|
plan = dataclasses.replace(
|
||||||
plan,
|
plan,
|
||||||
proxy_plan=proxy_plan,
|
|
||||||
git_gate_plan=git_gate_plan,
|
git_gate_plan=git_gate_plan,
|
||||||
egress_plan=egress_plan,
|
egress_plan=egress_plan,
|
||||||
supervise_plan=supervise_plan,
|
supervise_plan=supervise_plan,
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ from ...egress import Egress
|
|||||||
from ...env import ResolvedEnv, resolve_env
|
from ...env import ResolvedEnv, resolve_env
|
||||||
from ...git_gate import GitGate
|
from ...git_gate import GitGate
|
||||||
from ...log import die
|
from ...log import die
|
||||||
from ...pipelock import PipelockProxy
|
|
||||||
from ...supervise import Supervise
|
from ...supervise import Supervise
|
||||||
from ...workspace import workspace_plan as resolve_workspace_plan
|
from ...workspace import workspace_plan as resolve_workspace_plan
|
||||||
from .. import BottleSpec
|
from .. import BottleSpec
|
||||||
@@ -36,7 +35,6 @@ from .bottle_state import (
|
|||||||
per_bottle_dockerfile,
|
per_bottle_dockerfile,
|
||||||
per_bottle_dockerfile_path,
|
per_bottle_dockerfile_path,
|
||||||
per_bottle_image_tag,
|
per_bottle_image_tag,
|
||||||
pipelock_state_dir,
|
|
||||||
supervise_state_dir,
|
supervise_state_dir,
|
||||||
write_metadata,
|
write_metadata,
|
||||||
)
|
)
|
||||||
@@ -53,7 +51,6 @@ def resolve_plan(
|
|||||||
validation already ran in the base class."""
|
validation already ran in the base class."""
|
||||||
docker_mod.require_docker()
|
docker_mod.require_docker()
|
||||||
|
|
||||||
proxy = PipelockProxy()
|
|
||||||
git_gate = GitGate()
|
git_gate = GitGate()
|
||||||
egress = Egress()
|
egress = Egress()
|
||||||
supervise = Supervise()
|
supervise = Supervise()
|
||||||
@@ -191,12 +188,6 @@ def resolve_plan(
|
|||||||
guest_env.setdefault(key, val)
|
guest_env.setdefault(key, val)
|
||||||
agent_provision = replace(agent_provision, guest_env=guest_env)
|
agent_provision = replace(agent_provision, guest_env=guest_env)
|
||||||
|
|
||||||
pipelock_dir = pipelock_state_dir(slug)
|
|
||||||
pipelock_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
proxy_plan = proxy.prepare(
|
|
||||||
bottle, slug, pipelock_dir, agent_provision.egress_routes,
|
|
||||||
)
|
|
||||||
|
|
||||||
egress_dir = egress_state_dir(slug)
|
egress_dir = egress_state_dir(slug)
|
||||||
egress_dir.mkdir(parents=True, exist_ok=True)
|
egress_dir.mkdir(parents=True, exist_ok=True)
|
||||||
egress_plan = egress.prepare(
|
egress_plan = egress.prepare(
|
||||||
@@ -244,7 +235,6 @@ def resolve_plan(
|
|||||||
env_file=env_file,
|
env_file=env_file,
|
||||||
forwarded_env=forwarded_env,
|
forwarded_env=forwarded_env,
|
||||||
prompt_file=prompt_file,
|
prompt_file=prompt_file,
|
||||||
proxy_plan=proxy_plan,
|
|
||||||
git_gate_plan=git_gate_plan,
|
git_gate_plan=git_gate_plan,
|
||||||
egress_plan=egress_plan,
|
egress_plan=egress_plan,
|
||||||
supervise_plan=supervise_plan,
|
supervise_plan=supervise_plan,
|
||||||
|
|||||||
Reference in New Issue
Block a user