refactor(gateway): move the data-plane daemons into a bot_bottle.gateway package
test / integration-docker (pull_request) Successful in 11s
test / unit (pull_request) Successful in 43s
lint / lint (push) Successful in 56s
test / integration-firecracker (pull_request) Successful in 3m19s
test / coverage (pull_request) Successful in 19s
test / publish-infra (pull_request) Has been skipped
tracker-policy-pr / check-pr (pull_request) Successful in 7s
test / integration-docker (pull_request) Successful in 11s
test / unit (pull_request) Successful in 43s
lint / lint (push) Successful in 56s
test / integration-firecracker (pull_request) Successful in 3m19s
test / coverage (pull_request) Successful in 19s
test / publish-infra (pull_request) Has been skipped
tracker-policy-pr / check-pr (pull_request) Successful in 7s
Separate the gateway (data plane) from the orchestrator (control plane) at the
module level. The gateway runtime files move out of the package root — and the
backend-neutral Gateway lifecycle ABC + GATEWAY_* constants move out of
orchestrator/ — into a new bot_bottle/gateway/ package:
gateway/__init__.py (was orchestrator/gateway.py: Gateway ABC + consts
+ rotate_gateway_ca)
gateway/gateway_init.py (the PID-1 daemon supervisor)
gateway/egress_addon.py, egress_addon_core.py, egress_dlp_config.py,
dlp_detectors.py (the egress mitmproxy daemon)
gateway/git_http_backend.py (the git-http daemon)
gateway/git_gate_render.py (the git-gate pre-receive rendering)
gateway/supervise_server.py (the supervise MCP daemon)
gateway/policy_resolver.py (the data-plane control-plane RPC client)
orchestrator/ now holds only control-plane files. The shared plan/types/auth
layer (egress.py=EgressPlan, git_gate.py=GitGatePlan, supervise.py,
supervise_types.py, control_auth.py) and the launch-time git-gate provisioning
helpers stay at root, so orchestrator/ and backend/ still own them.
Because these daemons are invoked as `python3 -m bot_bottle.<name>`, loaded flat
by mitmproxy, and referenced in Dockerfile.gateway, the move updates more than
Python imports: the `-m` invocations (firecracker/macOS infra scripts), the
Dockerfile.gateway addon shim + ENTRYPOINT, gateway_init's _DAEMONS module
paths, and the git-gate CGI heredocs all now point at bot_bottle.gateway.*.
No behavior change; full unit suite green (2251).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,119 @@
|
||||
"""The consolidated per-host gateway (PRD 0070).
|
||||
|
||||
The core consolidation win: **one** persistent gateway per host, shared by
|
||||
every bottle, instead of a gateway per bottle. It's safe to share
|
||||
because the attribution invariant (source IP + identity token, see
|
||||
`registry`) lets the gateway attribute each request to the right bottle —
|
||||
so per-bottle policy lives in one long-lived process keyed on who's calling.
|
||||
|
||||
`Gateway` is the backend-neutral lifecycle contract (mirrors `LaunchBroker`):
|
||||
ensure the single instance is up, report it, tear it down. The docker
|
||||
implementation (`DockerGateway`) lives in `backend/docker/gateway.py`; a
|
||||
firecracker gateway VM slots in later.
|
||||
|
||||
The defining behaviour is **idempotent singleton**: `ensure_running` starts
|
||||
the instance if absent and is a no-op if it's already up, so N bottle
|
||||
launches never spawn N gateways.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from ..paths import host_gateway_ca_dir
|
||||
|
||||
# The gateway's mitmproxy writes its CA a beat after the container starts, so
|
||||
# reads poll for it rather than assuming it's there on a fresh launch.
|
||||
CA_POLL_SECONDS = 0.5
|
||||
DEFAULT_CA_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
GATEWAY_NAME = "bot-bottle-orch-gateway"
|
||||
GATEWAY_LABEL = "bot-bottle-orch-gateway=1"
|
||||
# The single user-defined network the gateway and every agent bottle share.
|
||||
# Agents attach here with a pinned IP and reach the gateway's egress /
|
||||
# git-http / supervise ports by its address — no host port publishing, and
|
||||
# the source IP the gateway attributes by is the address on this network.
|
||||
GATEWAY_NETWORK = "bot-bottle-gateway"
|
||||
|
||||
# mitmproxy's CA dir in the bundle. The host's gateway-CA dir (see
|
||||
# `host_gateway_ca_dir`) is bind-mounted here so the gateway's self-generated
|
||||
# CA stays STABLE across container recreation — every agent installs this one
|
||||
# CA to trust the shared gateway's TLS interception, so it must not rotate when
|
||||
# the gateway restarts. A host bind-mount rather than a named volume: a named
|
||||
# volume is silently wiped by `docker volume prune`, minting a fresh CA that
|
||||
# breaks every running bottle (issue #450).
|
||||
MITMPROXY_HOME = "/home/mitmproxy/.mitmproxy"
|
||||
GATEWAY_CA_CERT = f"{MITMPROXY_HOME}/mitmproxy-ca-cert.pem"
|
||||
|
||||
# The CA material mitmproxy writes into its confdir. mitmproxy reuses these on
|
||||
# startup when present and generates them only on first run, so persisting them
|
||||
# is what makes the CA stable; deleting them (see `rotate_gateway_ca`) forces a
|
||||
# fresh CA on the next start. `mitmproxy-ca.pem` (cert + private key) is the
|
||||
# signing identity; the rest are derived encodings agents/clients consume.
|
||||
GATEWAY_CA_GLOB = "mitmproxy-ca*"
|
||||
|
||||
# The gateway data-plane image + its Dockerfile. Kept as a local constant
|
||||
# rather than imported from the backend layer, which would drag
|
||||
# the whole backend layer into the lean orchestrator (see #359); unify when
|
||||
# that lands. Env override matches the backend's BOT_BOTTLE_GATEWAY_IMAGE.
|
||||
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
||||
GATEWAY_DOCKERFILE = "Dockerfile.gateway"
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]:
|
||||
"""Delete the persisted mitmproxy CA so the next gateway start mints a
|
||||
fresh one — the explicit, deliberate CA-rollover path (issue #450).
|
||||
|
||||
Persistence keeps the CA stable across restarts precisely because mitmproxy
|
||||
reuses the on-disk CA; rotation is therefore just removing that material.
|
||||
Returns the files removed (empty when there was no CA yet); idempotent.
|
||||
|
||||
This only clears the on-disk CA. It does NOT stop the running gateway (whose
|
||||
mitmproxy still holds the old CA in memory) or re-provision agents — the
|
||||
caller recreates the gateway to mint the new CA and re-attaches bottles.
|
||||
`rotate-ca` on the orchestrator CLI wires those steps together."""
|
||||
ca_dir = ca_dir if ca_dir is not None else host_gateway_ca_dir()
|
||||
removed: list[Path] = []
|
||||
for path in sorted(ca_dir.glob(GATEWAY_CA_GLOB)):
|
||||
path.unlink()
|
||||
removed.append(path)
|
||||
return removed
|
||||
|
||||
|
||||
class GatewayError(Exception):
|
||||
"""The shared gateway failed to build/start/stop (non-zero `docker` exit)."""
|
||||
|
||||
|
||||
class Gateway(abc.ABC):
|
||||
"""Lifecycle of the single per-host gateway. Backend-neutral."""
|
||||
|
||||
name: str
|
||||
|
||||
def ensure_built(self) -> None:
|
||||
"""Ensure the gateway's image / rootfs exists, building it if needed.
|
||||
Default: nothing to build (e.g. a stub or a pre-pulled image)."""
|
||||
return
|
||||
|
||||
@abc.abstractmethod
|
||||
def ensure_running(self) -> None:
|
||||
"""Start the gateway if it isn't already up. Idempotent: a no-op
|
||||
when it's already running (that's the whole point — one per host).
|
||||
Assumes the image exists — call `ensure_built()` first."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def is_running(self) -> bool:
|
||||
"""True iff the gateway instance is currently up."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def stop(self) -> None:
|
||||
"""Remove the gateway. Idempotent — absent is success."""
|
||||
|
||||
|
||||
__all__ = [
|
||||
"Gateway", "GatewayError", "rotate_gateway_ca",
|
||||
"GATEWAY_NAME", "GATEWAY_LABEL", "GATEWAY_IMAGE", "GATEWAY_NETWORK",
|
||||
"GATEWAY_CA_CERT", "GATEWAY_CA_GLOB",
|
||||
]
|
||||
@@ -0,0 +1,531 @@
|
||||
"""DLP detectors for the egress proxy (PRD 0053).
|
||||
|
||||
Pure Python, no mitmproxy dependency. Each detector is a module-level
|
||||
function returning `ScanResult | None`.
|
||||
|
||||
Available in the gateway via the installed `bot_bottle` package
|
||||
(see `Dockerfile.gateway`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import functools
|
||||
import gzip
|
||||
import re
|
||||
import typing
|
||||
import unicodedata
|
||||
from math import log2
|
||||
from collections import Counter
|
||||
from urllib.parse import quote as url_quote
|
||||
|
||||
from .egress_addon_core import ScanResult
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Snippet helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SNIPPET_CONTEXT = 40 # chars of surrounding text to include on each side
|
||||
REDACT = "********" # fixed-width replacement for the matched sensitive value
|
||||
|
||||
|
||||
def _snippet(text: str, start: int, end: int) -> str:
|
||||
"""Return context around a match with the matched span replaced by REDACT."""
|
||||
before = text[max(0, start - SNIPPET_CONTEXT):start].replace("\n", " ").replace("\r", " ")
|
||||
after = text[end:end + SNIPPET_CONTEXT].replace("\n", " ").replace("\r", " ")
|
||||
return f"{before}{REDACT}{after}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unicode normalization (defeats confusable-char and combining-mark evasion)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _normalize_text(text: str) -> str:
|
||||
# NFKD separates base characters from combining marks and resolves
|
||||
# compatibility equivalents (fullwidth ASCII, ligatures, etc.)
|
||||
decomposed = unicodedata.normalize("NFKD", text)
|
||||
return "".join(
|
||||
ch for ch in decomposed
|
||||
# Strip combining marks inserted between chars to break patterns
|
||||
if unicodedata.category(ch) != "Mn"
|
||||
# Strip control chars; keep common whitespace (\n \r \t)
|
||||
and (unicodedata.category(ch) != "Cc" or ch in "\n\r\t")
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Token patterns detector
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TOKEN_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
|
||||
("AWS access key", re.compile(r"AKIA[0-9A-Z]{16}")),
|
||||
("GitHub token (classic)", re.compile(r"ghp_[A-Za-z0-9_]{36}")),
|
||||
("GitHub fine-grained token", re.compile(r"github_pat_[A-Za-z0-9_]{82}")),
|
||||
("Anthropic API key", re.compile(r"sk-ant-[A-Za-z0-9\-_]{93}")),
|
||||
("OpenAI API key", re.compile(r"sk-[A-Za-z0-9]{48}")),
|
||||
("OpenAI project API key", re.compile(r"sk-proj-[A-Za-z0-9_\-]{48,}")),
|
||||
("Stripe live key", re.compile(r"sk_live_[A-Za-z0-9]{24}")),
|
||||
("Generic Bearer JWT", re.compile(r"Bearer\s+[A-Za-z0-9._\-]{50,}")),
|
||||
("HuggingFace token", re.compile(r"hf_[A-Za-z0-9]{34,}")),
|
||||
("Databricks token", re.compile(r"dapi[A-Za-z0-9]{32}")),
|
||||
("Slack token", re.compile(r"xox[baprs]-[A-Za-z0-9]+-[A-Za-z0-9]+-[A-Za-z0-9]{24,}")),
|
||||
("npm token", re.compile(r"npm_[A-Za-z0-9]{36}")),
|
||||
("SendGrid API key", re.compile(r"SG\.[A-Za-z0-9_\-]{22}\.[A-Za-z0-9_\-]{43}")),
|
||||
("PyPI token", re.compile(r"pypi-[A-Za-z0-9_\-]{80,}")),
|
||||
("HashiCorp Vault token", re.compile(r"hvs\.[A-Za-z0-9_\-]{24,}")),
|
||||
)
|
||||
|
||||
|
||||
def scan_token_patterns(
|
||||
text: str,
|
||||
*,
|
||||
location: str = "body",
|
||||
safe_tokens: typing.AbstractSet[str] | None = None,
|
||||
) -> ScanResult | None:
|
||||
normalized = _normalize_text(text)
|
||||
for name, pattern in TOKEN_PATTERNS:
|
||||
for m in pattern.finditer(normalized):
|
||||
value = m.group(0)
|
||||
# A value the supervisor has approved (PRD 0062) is no longer a
|
||||
# block — keep scanning so a second, un-approved token in the
|
||||
# same request is still caught.
|
||||
if safe_tokens is not None and value in safe_tokens:
|
||||
continue
|
||||
return ScanResult(
|
||||
severity="block",
|
||||
reason=f"{name} found in {location}",
|
||||
location=location,
|
||||
context=_snippet(normalized, m.start(), m.end()),
|
||||
matched=value,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def redact_tokens(
|
||||
text: str,
|
||||
*,
|
||||
env: typing.Mapping[str, str] | None = None,
|
||||
sensitive_prefixes: tuple[str, ...] = ("EGRESS_TOKEN_",),
|
||||
) -> str:
|
||||
"""Replace token pattern matches and (if env given) provisioned secrets with REDACT."""
|
||||
for _, pattern in TOKEN_PATTERNS:
|
||||
text = pattern.sub(REDACT, text)
|
||||
if env is not None:
|
||||
for key, value in env.items():
|
||||
if any(key.startswith(p) for p in sensitive_prefixes) and value:
|
||||
for variant in _encoded_variants(value):
|
||||
text = text.replace(variant, REDACT)
|
||||
return text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Known secrets detector
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Encoded-variant cache. Provisioned secrets are stable for the life of the
|
||||
# proxy, but `_encoded_variants` is on the per-request hot path — it runs for
|
||||
# every secret on every redaction and known-secret scan (host, path, each
|
||||
# header, body). Deriving the variant set is relatively expensive (gzip +
|
||||
# nine encodings), so memoize it per distinct secret. The proxy process
|
||||
# already holds these values in `os.environ`, so caching them here adds no
|
||||
# new exposure. The cache is bounded (lru_cache maxsize) so a long-lived
|
||||
# proxy that sees rotating secrets evicts the oldest rather than growing
|
||||
# without limit; 256 comfortably covers the EGRESS_TOKEN_* set in practice.
|
||||
_VARIANT_CACHE_MAXSIZE = 256
|
||||
|
||||
|
||||
def _encoded_variants(secret: str) -> list[str]:
|
||||
"""Return the secret plus common encoded variants for exfil detection.
|
||||
|
||||
The variant set is computed once per distinct secret and cached; callers
|
||||
get a fresh list so they can't mutate the shared cached tuple."""
|
||||
return list(_compute_encoded_variants(secret))
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=_VARIANT_CACHE_MAXSIZE)
|
||||
def _compute_encoded_variants(secret: str) -> tuple[str, ...]:
|
||||
"""Derive the secret plus its encoded variants (memoized, bounded)."""
|
||||
seen: set[str] = {secret}
|
||||
variants: list[str] = [secret]
|
||||
|
||||
def _add(v: str) -> None:
|
||||
if v not in seen:
|
||||
seen.add(v)
|
||||
variants.append(v)
|
||||
|
||||
secret_bytes = secret.encode("utf-8")
|
||||
|
||||
# Standard base64 — with and without padding
|
||||
b64 = base64.b64encode(secret_bytes).decode("ascii")
|
||||
_add(b64)
|
||||
_add(b64.rstrip("="))
|
||||
|
||||
# URL-safe base64 (JWT/OAuth use -_ alphabet) — with and without padding
|
||||
b64url = base64.urlsafe_b64encode(secret_bytes).decode("ascii")
|
||||
_add(b64url)
|
||||
_add(b64url.rstrip("="))
|
||||
|
||||
# URL percent-encoding
|
||||
_add(url_quote(secret, safe=""))
|
||||
|
||||
# Hex — lowercase and uppercase
|
||||
_add(secret_bytes.hex())
|
||||
_add(secret_bytes.hex().upper())
|
||||
|
||||
# Base32 (TOTP seeds, some DNS-exfil channels)
|
||||
_add(base64.b32encode(secret_bytes).decode("ascii"))
|
||||
|
||||
# gzip + base64 (deterministic: mtime=0); recognisable by H4sI prefix
|
||||
_add(base64.b64encode(gzip.compress(secret_bytes, mtime=0)).decode("ascii"))
|
||||
|
||||
return tuple(variants)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fragmentation-resistant helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Minimum length of alnum projection for projection-based checks to run.
|
||||
# Short secrets produce too many false positives in projection space.
|
||||
_ALNUM_MIN_LEN = 8
|
||||
|
||||
# Minimum window length for the partial-substring sliding scan.
|
||||
PARTIAL_MATCH_MIN_LEN = 12
|
||||
|
||||
|
||||
def _alnum_projection(text: str) -> str:
|
||||
"""Return text with every non-alphanumeric character stripped.
|
||||
|
||||
Used for fragmentation-resistant matching: separator-injected secrets
|
||||
(spaces, hyphens, dots inserted between characters) are identical to
|
||||
their originals in alnum projection space.
|
||||
"""
|
||||
return "".join(c for c in text if c.isalnum())
|
||||
|
||||
|
||||
def _find_partial_window(secret_alnum: str, text_alnum: str, min_len: int) -> int | None:
|
||||
"""Return the earliest position in text_alnum holding a min_len-char window
|
||||
that also appears in secret_alnum, or None.
|
||||
|
||||
The secret's set of min_len-grams is small (bounded by the secret length),
|
||||
so building it once and sweeping the text a single time is O(len(text))
|
||||
rather than the O(len(secret) * len(text)) of repeated substring searches —
|
||||
which matters because this runs per provisioned secret on every request
|
||||
body. Coverage is unchanged: a hit still means at least min_len consecutive
|
||||
alphanumeric characters of the secret leaked into the text.
|
||||
"""
|
||||
if len(secret_alnum) < min_len or len(text_alnum) < min_len:
|
||||
return None
|
||||
secret_grams = {
|
||||
secret_alnum[i:i + min_len]
|
||||
for i in range(len(secret_alnum) - min_len + 1)
|
||||
}
|
||||
for pos in range(len(text_alnum) - min_len + 1):
|
||||
if text_alnum[pos:pos + min_len] in secret_grams:
|
||||
return pos
|
||||
return None
|
||||
|
||||
|
||||
def scan_known_secrets(
|
||||
text: str,
|
||||
*,
|
||||
location: str = "body",
|
||||
env: typing.Mapping[str, str] | None = None,
|
||||
sensitive_prefixes: tuple[str, ...] = ("EGRESS_TOKEN_",),
|
||||
safe_tokens: typing.AbstractSet[str] | None = None,
|
||||
) -> ScanResult | None:
|
||||
if env is None:
|
||||
return None
|
||||
|
||||
# Pre-compute alnum projection of the scan text once; reused per secret.
|
||||
text_alnum: str | None = None
|
||||
|
||||
for key, value in env.items():
|
||||
if not any(key.startswith(p) for p in sensitive_prefixes) or not value:
|
||||
continue
|
||||
|
||||
# Pass 1: exact match across encoded variants (original behaviour).
|
||||
approved_exact = False
|
||||
for variant in _encoded_variants(value):
|
||||
pos = text.find(variant)
|
||||
if pos >= 0:
|
||||
# The supervisor approves the exact encoded variant found
|
||||
# (PRD 0062); a different encoding of the same secret is a
|
||||
# fresh block.
|
||||
if safe_tokens is not None and variant in safe_tokens:
|
||||
approved_exact = True
|
||||
continue
|
||||
return ScanResult(
|
||||
severity="block",
|
||||
reason=f"provisioned secret from {key} found in {location}",
|
||||
location=location,
|
||||
context=_snippet(text, pos, pos + len(variant)),
|
||||
matched=variant,
|
||||
)
|
||||
if approved_exact:
|
||||
# Exact match was found and approved; projection passes would
|
||||
# fire on the same value, so skip them for this secret.
|
||||
continue
|
||||
|
||||
# Pass 2 & 3: fragmentation-resistant projection checks.
|
||||
secret_alnum = _alnum_projection(value)
|
||||
if len(secret_alnum) < _ALNUM_MIN_LEN:
|
||||
continue
|
||||
|
||||
if text_alnum is None:
|
||||
text_alnum = _alnum_projection(text)
|
||||
|
||||
# Pass 2: full alnum-projection exact match (catches separator injection).
|
||||
pos2 = text_alnum.find(secret_alnum)
|
||||
if pos2 >= 0:
|
||||
return ScanResult(
|
||||
severity="block",
|
||||
reason=(
|
||||
f"provisioned secret from {key} found in {location} "
|
||||
f"(fragmented match — separator injection)"
|
||||
),
|
||||
location=location,
|
||||
context=_snippet(text_alnum, pos2, pos2 + len(secret_alnum)),
|
||||
)
|
||||
|
||||
# Pass 3: sliding-window partial match (catches chunked-substring leaks).
|
||||
pos3 = _find_partial_window(secret_alnum, text_alnum, PARTIAL_MATCH_MIN_LEN)
|
||||
if pos3 is not None:
|
||||
return ScanResult(
|
||||
severity="block",
|
||||
reason=(
|
||||
f"provisioned secret from {key} found in {location} "
|
||||
f"(partial match — at least {PARTIAL_MATCH_MIN_LEN} consecutive "
|
||||
f"alphanumeric chars)"
|
||||
),
|
||||
location=location,
|
||||
context=_snippet(text_alnum, pos3, pos3 + PARTIAL_MATCH_MIN_LEN),
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entropy detector (warn-only)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Sliding window size and step for the entropy scan.
|
||||
ENTROPY_WINDOW = 64
|
||||
ENTROPY_STEP = 32
|
||||
|
||||
# Bits-per-character threshold. Random ASCII printable ≈ 6.6 bits; random
|
||||
# lowercase hex ≈ 4 bits; random base64url ≈ 6 bits. 5.5 sits above
|
||||
# typical structured data (JSON, URLs) while staying below truly random
|
||||
# content.
|
||||
ENTROPY_BLOCK_THRESHOLD = 5.5
|
||||
|
||||
|
||||
def _shannon_entropy(text: str) -> float:
|
||||
if not text:
|
||||
return 0.0
|
||||
counts = Counter(text)
|
||||
n = len(text)
|
||||
return -sum((c / n) * log2(c / n) for c in counts.values())
|
||||
|
||||
|
||||
def scan_entropy(
|
||||
text: str,
|
||||
*,
|
||||
location: str = "body",
|
||||
window: int = ENTROPY_WINDOW,
|
||||
threshold: float = ENTROPY_BLOCK_THRESHOLD,
|
||||
) -> ScanResult | None:
|
||||
"""Warn-only detector: flag windows of `window` chars with Shannon entropy
|
||||
above `threshold` bits per character.
|
||||
|
||||
Never blocks; always returns severity='warn'. Disabled by default —
|
||||
routes must opt in via dlp.outbound_detectors=['entropy'].
|
||||
"""
|
||||
if not text:
|
||||
return None
|
||||
step = max(1, window // 2)
|
||||
end = len(text)
|
||||
# Scan overlapping windows; also check the final tail if shorter than window.
|
||||
positions = list(range(0, end - window + 1, step))
|
||||
if end < window:
|
||||
positions = [0]
|
||||
elif (end - window) % step != 0:
|
||||
positions.append(end - window)
|
||||
for i in positions:
|
||||
chunk = text[i:i + window]
|
||||
if _shannon_entropy(chunk) >= threshold:
|
||||
return ScanResult(
|
||||
severity="warn",
|
||||
reason=f"high-entropy content in {location} (possible encrypted exfil)",
|
||||
location=location,
|
||||
context=_snippet(text, i, i + len(chunk)),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Naive prompt injection detector (Phase 2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
DISCLOSURE_PHRASES: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"(?i)system\s+prompt"),
|
||||
re.compile(r"(?i)my\s+instructions\s+are"),
|
||||
re.compile(r"(?i)original\s+instructions"),
|
||||
re.compile(r"(?i)secret\s+instructions"),
|
||||
re.compile(r"(?i)hidden\s+rules"),
|
||||
)
|
||||
|
||||
JAILBREAK_PHRASES: tuple[re.Pattern[str], ...] = (
|
||||
re.compile(r"(?i)ignore\s+previous"),
|
||||
re.compile(r"(?i)forget\s+everything"),
|
||||
re.compile(r"(?i)disregard\s+(?:all\s+)?(?:previous|prior)"),
|
||||
re.compile(r"(?i)pretend\s+you\s+are"),
|
||||
re.compile(r"(?i)act\s+as\s+(?:if|though)"),
|
||||
)
|
||||
|
||||
|
||||
PROXIMITY_CHARS = 500
|
||||
|
||||
|
||||
def _match_gap(a: re.Match[str], b: re.Match[str]) -> int:
|
||||
"""Character gap between two match spans; 0 when they overlap or touch."""
|
||||
return max(0, max(a.start(), b.start()) - min(a.end(), b.end()))
|
||||
|
||||
|
||||
def _closest_pair(
|
||||
a_matches: list[re.Match[str]],
|
||||
b_matches: list[re.Match[str]],
|
||||
*,
|
||||
within: int | None = None,
|
||||
) -> tuple[re.Match[str], re.Match[str]] | None:
|
||||
"""Return the (a, b) pair with the smallest character gap, or None when
|
||||
either list is empty.
|
||||
|
||||
Runs in O(n log n) sort + O(n) merge rather than the O(n*m) cross product:
|
||||
both lists are sorted by start offset and swept with a two-pointer merge,
|
||||
advancing whichever span ends first (it can only get farther from any
|
||||
later span in the other list). This matters because the inputs are
|
||||
attacker-controlled response-body matches that have already passed the
|
||||
body-size cap, so the quadratic form is a latent DoS.
|
||||
|
||||
When `within` is set, returns as soon as a pair with gap <= within is
|
||||
found: the only caller blocks on any pair inside the proximity threshold,
|
||||
so the exact global minimum past that point doesn't change the decision.
|
||||
"""
|
||||
if not a_matches or not b_matches:
|
||||
return None
|
||||
a_sorted = sorted(a_matches, key=lambda m: m.start())
|
||||
b_sorted = sorted(b_matches, key=lambda m: m.start())
|
||||
i = j = 0
|
||||
best: tuple[re.Match[str], re.Match[str]] | None = None
|
||||
best_gap: int | None = None
|
||||
while i < len(a_sorted) and j < len(b_sorted):
|
||||
a, b = a_sorted[i], b_sorted[j]
|
||||
gap = _match_gap(a, b)
|
||||
if best_gap is None or gap < best_gap:
|
||||
best_gap = gap
|
||||
best = (a, b)
|
||||
if within is not None and gap <= within:
|
||||
return best
|
||||
# Advance the span that ends first; it cannot form a closer pair with
|
||||
# any later (further-right) span from the other list.
|
||||
if a.end() <= b.end():
|
||||
i += 1
|
||||
else:
|
||||
j += 1
|
||||
return best
|
||||
|
||||
|
||||
def scan_naive_injection(text: str) -> ScanResult | None:
|
||||
location = "response body"
|
||||
disclosure_hits = [m for p in DISCLOSURE_PHRASES for m in p.finditer(text)]
|
||||
jailbreak_hits = [m for p in JAILBREAK_PHRASES for m in p.finditer(text)]
|
||||
|
||||
if disclosure_hits and jailbreak_hits:
|
||||
pair = _closest_pair(disclosure_hits, jailbreak_hits, within=PROXIMITY_CHARS)
|
||||
if pair is not None:
|
||||
dist = _match_gap(pair[0], pair[1])
|
||||
if dist <= PROXIMITY_CHARS:
|
||||
first = pair[0] if pair[0].start() <= pair[1].start() else pair[1]
|
||||
return ScanResult(
|
||||
severity="block",
|
||||
reason=(
|
||||
f"disclosure and jailbreak phrases within "
|
||||
f"{dist} chars in {location}"
|
||||
),
|
||||
location=location,
|
||||
context=_snippet(text, first.start(), first.end()),
|
||||
)
|
||||
|
||||
if disclosure_hits:
|
||||
m = disclosure_hits[0]
|
||||
return ScanResult(
|
||||
severity="warn",
|
||||
reason=f"prompt disclosure phrase detected in {location}",
|
||||
location=location,
|
||||
context=_snippet(text, m.start(), m.end()),
|
||||
)
|
||||
|
||||
if jailbreak_hits:
|
||||
m = jailbreak_hits[0]
|
||||
return ScanResult(
|
||||
severity="warn",
|
||||
reason=f"jailbreak phrase detected in {location}",
|
||||
location=location,
|
||||
context=_snippet(text, m.start(), m.end()),
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CRLF injection detector
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# URL-encoded CRLF is never legitimate in a request URL or header value.
|
||||
_CRLF_ENCODED_RE = re.compile(r"%0[dD]%0[aA]", re.ASCII)
|
||||
# Literal CRLF followed by a header-name pattern indicates header injection.
|
||||
_CRLF_HEADER_INJECT_RE = re.compile(r"\r\n[A-Za-z][A-Za-z0-9\-]+\s*:", re.ASCII)
|
||||
|
||||
|
||||
def strip_crlf(text: str) -> str:
|
||||
"""Remove URL-encoded and literal CRLF injection sequences from a request
|
||||
surface (PRD 0062 redact policy). Used to scrub the request line / headers
|
||||
so the request can be forwarded instead of hard-blocked."""
|
||||
text = _CRLF_ENCODED_RE.sub("", text)
|
||||
return _CRLF_HEADER_INJECT_RE.sub(lambda m: m.group(0)[2:], text)
|
||||
|
||||
|
||||
def scan_crlf_injection(text: str) -> ScanResult | None:
|
||||
if _CRLF_ENCODED_RE.search(text):
|
||||
return ScanResult(
|
||||
severity="block",
|
||||
reason="URL-encoded CRLF (%0d%0a) in outbound request",
|
||||
)
|
||||
if _CRLF_HEADER_INJECT_RE.search(text):
|
||||
return ScanResult(
|
||||
severity="block",
|
||||
reason="CRLF header injection pattern in outbound request",
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ENTROPY_BLOCK_THRESHOLD",
|
||||
"ENTROPY_WINDOW",
|
||||
"ENTROPY_STEP",
|
||||
"PARTIAL_MATCH_MIN_LEN",
|
||||
"REDACT",
|
||||
"SNIPPET_CONTEXT",
|
||||
"TOKEN_PATTERNS",
|
||||
"_alnum_projection",
|
||||
"_shannon_entropy",
|
||||
"redact_tokens",
|
||||
"scan_crlf_injection",
|
||||
"scan_entropy",
|
||||
"scan_known_secrets",
|
||||
"scan_naive_injection",
|
||||
"scan_token_patterns",
|
||||
"strip_crlf",
|
||||
]
|
||||
@@ -0,0 +1,808 @@
|
||||
"""mitmproxy addon entrypoint for the egress gateway (PRD 0017, PRD 0053).
|
||||
|
||||
Loaded by `mitmdump -s /app/egress_addon.py` inside the
|
||||
egress container."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import typing
|
||||
|
||||
from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=import-error
|
||||
|
||||
from bot_bottle.constants import IDENTITY_HEADER
|
||||
from bot_bottle.gateway.dlp_detectors import redact_tokens, strip_crlf
|
||||
from bot_bottle.gateway.egress_addon_core import (
|
||||
LOG_BLOCKS,
|
||||
LOG_FULL,
|
||||
DEFAULT_OUTBOUND_ON_MATCH,
|
||||
ON_MATCH_BLOCK,
|
||||
ON_MATCH_REDACT,
|
||||
Config,
|
||||
Route,
|
||||
ScanResult,
|
||||
build_inbound_scan_text,
|
||||
build_outbound_scan_text,
|
||||
build_token_allow_payload,
|
||||
decide,
|
||||
decide_git_fetch,
|
||||
is_git_fetch_request,
|
||||
is_git_push_request,
|
||||
match_route,
|
||||
resolve_client_context,
|
||||
outbound_scan_headers,
|
||||
route_to_yaml_dict,
|
||||
scan_inbound,
|
||||
scan_outbound,
|
||||
)
|
||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||
from bot_bottle.supervise_types import (
|
||||
STATUS_APPROVED,
|
||||
STATUS_MODIFIED,
|
||||
STATUSES,
|
||||
TOOL_EGRESS_TOKEN_ALLOW,
|
||||
)
|
||||
|
||||
|
||||
INTROSPECT_HOST = "_egress.local"
|
||||
|
||||
# The per-host orchestrator control plane the addon resolves every request's
|
||||
# Config against, by source IP (PRD 0070). Mandatory: the consolidated gateway
|
||||
# is the only topology now — there is no static per-bottle routes file to fall
|
||||
# back to — so an unset value is a fatal misconfiguration (see __init__).
|
||||
ORCHESTRATOR_URL_ENV = "BOT_BOTTLE_ORCHESTRATOR_URL"
|
||||
|
||||
# Per-flow key under which `request()` stashes the resolved (Config, supervise
|
||||
# slug, env) so the later `response()` and `websocket_message()` hooks scan
|
||||
# against the *calling bottle's* policy — the same one the request was decided
|
||||
# on — without a second `/resolve` per response or per WebSocket frame. A hook
|
||||
# on a flow that never resolved (no stash) fails closed to deny-all, so it's a
|
||||
# safe no-op rather than an unscanned pass.
|
||||
_FLOW_CTX_KEY = "bot_bottle_egress_ctx"
|
||||
|
||||
|
||||
def _token_from_proxy_auth(header: str) -> str:
|
||||
"""Extract the identity token (the password) from a `Proxy-Authorization:
|
||||
Basic base64(<bottle_id>:<token>)` header. Empty on any malformed value —
|
||||
the mandatory `/resolve` then fail-closes on the empty token."""
|
||||
scheme, _, encoded = header.partition(" ")
|
||||
if scheme.lower() != "basic" or not encoded:
|
||||
return ""
|
||||
try:
|
||||
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
except (binascii.Error, ValueError, UnicodeDecodeError):
|
||||
return ""
|
||||
_, _, password = decoded.partition(":")
|
||||
return password
|
||||
|
||||
# Seconds the egress proxy holds a token-blocked request open waiting for the
|
||||
# operator's supervisor decision (PRD 0062), overridable via env.
|
||||
DEFAULT_TOKEN_ALLOW_TIMEOUT_SECONDS = 300.0
|
||||
|
||||
# Maximum bytes of a response body passed to the DLP inbound scan. mitmproxy
|
||||
# buffers the full response before the hook fires; capping at scan time limits
|
||||
# the additional memory amplification from decoded text and regex match strings.
|
||||
# A cap is a security trade-off (content above the threshold is not scanned),
|
||||
# but without it a single large download OOM-kills the shared egress process
|
||||
# (issue #455). Override with EGRESS_INBOUND_SCAN_LIMIT_BYTES; set to 0 to
|
||||
# disable the cap.
|
||||
DEFAULT_INBOUND_SCAN_LIMIT_BYTES = 1 * 1024 * 1024 # 1 MiB
|
||||
# Filesystem poll cadence while awaiting the operator's response.
|
||||
TOKEN_ALLOW_POLL_INTERVAL_SECONDS = 0.5
|
||||
|
||||
# Fixed operator guidance attached to every token-allow proposal.
|
||||
_TOKEN_ALLOW_JUSTIFICATION = (
|
||||
"egress DLP blocked an outbound request carrying a detected token. "
|
||||
"Approve only if this value is a false positive or a credential this "
|
||||
"request legitimately needs; the value is then allowed for the life of "
|
||||
"this bottle's egress proxy."
|
||||
)
|
||||
|
||||
|
||||
class EgressAddon:
|
||||
# Bare annotations (no class value): __init__ sets a live PolicyResolver for
|
||||
# real runs, and every host-side test builds an addon via __new__ and sets a
|
||||
# fake resolver. Egress is resolver-only now — the per-request policy always
|
||||
# comes from the orchestrator's /resolve (PRD 0070); there is no static
|
||||
# per-bottle routes file, SIGHUP reload, or single-tenant fallback.
|
||||
_resolver: "PolicyResolver"
|
||||
# Class defaults so __new__-built addons have them (real runs get fresh
|
||||
# per-instance collections in __init__; only http_connect mutates them,
|
||||
# which request-flow tests don't exercise unless they call http_connect).
|
||||
_conn_tokens: "dict[str, str]" = {}
|
||||
_passthrough_conns: "set[str]" = set()
|
||||
_inbound_scan_limit: int = DEFAULT_INBOUND_SCAN_LIMIT_BYTES
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Resolver-only: the gateway is always multi-tenant, resolving each
|
||||
# request's policy by source IP against the orchestrator control plane
|
||||
# (PRD 0070). The URL is mandatory — without a policy source the gateway
|
||||
# must not come up (fail-closed), rather than silently allowing nothing.
|
||||
orch_url = os.environ.get(ORCHESTRATOR_URL_ENV, "").strip()
|
||||
if not orch_url:
|
||||
raise RuntimeError(
|
||||
f"{ORCHESTRATOR_URL_ENV} is required: the egress gateway "
|
||||
"resolves every request's policy from the orchestrator and has "
|
||||
"no static routes file to fall back to."
|
||||
)
|
||||
self._resolver = PolicyResolver(orch_url)
|
||||
# Tokens the operator has approved this session (PRD 0062), keyed by
|
||||
# bottle so the shared gateway keeps each bottle's safelist separate —
|
||||
# a global set would let bottle A's approved secret pass bottle B's DLP
|
||||
# scan. In-memory only (a restart re-prompts); mutated only from the
|
||||
# asyncio loop that runs the addon hooks, so no lock is needed.
|
||||
self._safe_tokens: dict[str, set[str]] = {}
|
||||
# Per-client-connection identity token captured from the CONNECT's
|
||||
# `Proxy-Authorization` (HTTPS tunnels don't repeat it on the bumped
|
||||
# inner requests). Keyed by client_conn.id; cleared on disconnect.
|
||||
self._conn_tokens: dict[str, str] = {}
|
||||
# Connections whose route carries `inspect: false` — mitmproxy tunnels
|
||||
# these without TLS interception so the client sees the server's real
|
||||
# cert. Keyed by client_conn.id; cleared on disconnect.
|
||||
self._passthrough_conns: set[str] = set()
|
||||
self._token_allow_timeout = _token_allow_timeout_from_env(os.environ)
|
||||
self._inbound_scan_limit = _inbound_scan_limit_from_env(os.environ)
|
||||
|
||||
@staticmethod
|
||||
def _supervise_available(slug: str) -> bool:
|
||||
"""Supervise is reachable for this request iff we resolved a bottle to
|
||||
attribute its proposals to (the source-IP-attributed bottle id). Empty
|
||||
→ fail closed (no queue to write to)."""
|
||||
return bool(slug)
|
||||
|
||||
def _safe_tokens_for(self, slug: str) -> set[str]:
|
||||
"""This bottle's operator-approved DLP safelist (PRD 0062), created on
|
||||
first use. Keyed by bottle so the shared gateway never leaks one
|
||||
bottle's approved token into another's scan."""
|
||||
return self._safe_tokens.setdefault(slug, set())
|
||||
|
||||
def _serve_introspection(
|
||||
self, flow: http.HTTPFlow, path: str, config: Config,
|
||||
) -> None:
|
||||
"""Serve the calling bottle's own allowlist. `config` is this flow's
|
||||
resolved policy (the same one every hook uses), so the agent sees the
|
||||
routes that actually apply to it."""
|
||||
if path == "/allowlist":
|
||||
payload = json.dumps(
|
||||
{"routes": [route_to_yaml_dict(r) for r in config.routes]},
|
||||
indent=2,
|
||||
).encode("utf-8")
|
||||
flow.response = http.Response.make(
|
||||
200, payload,
|
||||
{"Content-Type": "application/json"},
|
||||
)
|
||||
return
|
||||
flow.response = http.Response.make(
|
||||
404,
|
||||
f"egress introspection: no such endpoint {path!r}".encode(),
|
||||
{"Content-Type": "text/plain; charset=utf-8"},
|
||||
)
|
||||
|
||||
def _flow_log(self, flow: http.HTTPFlow) -> int:
|
||||
"""This flow's log level, from the policy `request()` resolved and
|
||||
stashed. The block/redact log gates were a single global in the static-
|
||||
config world; they are per bottle now, so they read it from the flow."""
|
||||
return self._flow_ctx(flow)[0].log
|
||||
|
||||
def _req_ctx(self, flow: http.HTTPFlow) -> dict[str, object]:
|
||||
# Redact with this flow's resolved env overlay (process env + the
|
||||
# bottle's /resolve tokens), so the ctx scrubs the calling bottle's
|
||||
# provisioned secrets, not just os.environ's.
|
||||
env = self._flow_ctx(flow)[2]
|
||||
return {
|
||||
"host": redact_tokens(flow.request.pretty_host, env=env),
|
||||
"method": flow.request.method,
|
||||
"path": redact_tokens(flow.request.path, env=env),
|
||||
}
|
||||
|
||||
def _block(
|
||||
self,
|
||||
flow: http.HTTPFlow,
|
||||
reason: str,
|
||||
ctx: dict[str, object] | None = None,
|
||||
) -> None:
|
||||
if self._flow_log(flow) >= LOG_BLOCKS:
|
||||
entry: dict[str, object] = {"event": "egress_block", "reason": reason}
|
||||
if ctx:
|
||||
entry.update(ctx)
|
||||
sys.stderr.write(json.dumps(entry) + "\n")
|
||||
flow.response = http.Response.make(
|
||||
403,
|
||||
reason.encode("utf-8"),
|
||||
{"Content-Type": "text/plain; charset=utf-8"},
|
||||
)
|
||||
|
||||
def _log_request(
|
||||
self, flow: http.HTTPFlow, env: "typing.Mapping[str, str]",
|
||||
) -> None:
|
||||
# `env` is the per-flow resolved overlay (process env + this bottle's
|
||||
# /resolve tokens), so the log redaction scrubs the calling bottle's
|
||||
# provisioned secrets — not just the process-level ones in os.environ.
|
||||
headers = {
|
||||
k: redact_tokens(v, env=env)
|
||||
for k, v in flow.request.headers.items()
|
||||
if k.lower() != "authorization"
|
||||
}
|
||||
body = redact_tokens(flow.request.get_text(strict=False) or "", env=env)
|
||||
sys.stderr.write(
|
||||
json.dumps({
|
||||
"event": "egress_request",
|
||||
"host": redact_tokens(flow.request.pretty_host, env=env),
|
||||
"method": flow.request.method,
|
||||
"path": redact_tokens(flow.request.path, env=env),
|
||||
"headers": headers,
|
||||
"body": body,
|
||||
})
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
def _log_response(
|
||||
self, flow: http.HTTPFlow, env: "typing.Mapping[str, str]",
|
||||
) -> None:
|
||||
# Per-flow env overlay (see _log_request): redact this bottle's tokens.
|
||||
headers = {
|
||||
k: redact_tokens(v, env=env)
|
||||
for k, v in flow.response.headers.items()
|
||||
}
|
||||
body = redact_tokens(flow.response.get_text(strict=False) or "", env=env)
|
||||
sys.stderr.write(
|
||||
json.dumps({
|
||||
"event": "egress_response",
|
||||
"host": flow.request.pretty_host,
|
||||
"status": flow.response.status_code,
|
||||
"headers": headers,
|
||||
"body": body,
|
||||
})
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
def _resolve_flow(
|
||||
self, flow: http.HTTPFlow,
|
||||
) -> "tuple[Config, str, typing.Mapping[str, str]]":
|
||||
"""The calling bottle's `(Config, supervise slug, env)`, resolved by
|
||||
source IP in one round-trip against the orchestrator — fail-closed to
|
||||
deny-all + empty slug if unattributed. `env` is the process env overlaid
|
||||
with the bottle's `/resolve` tokens, so upstream-auth injection (and DLP)
|
||||
use *this* bottle's credentials. The identity token, if the agent
|
||||
injected one, is read then stripped so it never leaks upstream."""
|
||||
conn = flow.client_conn
|
||||
client_ip = conn.peername[0] if conn and conn.peername else ""
|
||||
token = self._request_token(flow)
|
||||
config, slug, tokens = resolve_client_context(self._resolver, client_ip, token)
|
||||
env = {**os.environ, **tokens} if tokens else os.environ
|
||||
return config, slug, env
|
||||
|
||||
def _stash_flow_ctx(
|
||||
self,
|
||||
flow: http.HTTPFlow,
|
||||
config: Config,
|
||||
slug: str,
|
||||
env: "typing.Mapping[str, str]",
|
||||
) -> None:
|
||||
"""Remember the per-flow context `request()` resolved, so the later
|
||||
`response()` / `websocket_message()` hooks reuse it — scanning against
|
||||
the same bottle's policy the request was decided on, with one `/resolve`
|
||||
per flow rather than one per frame."""
|
||||
meta = getattr(flow, "metadata", None)
|
||||
if isinstance(meta, dict):
|
||||
meta[_FLOW_CTX_KEY] = (config, slug, env)
|
||||
|
||||
def _flow_ctx(
|
||||
self, flow: http.HTTPFlow,
|
||||
) -> "tuple[Config, str, typing.Mapping[str, str]]":
|
||||
"""The `(Config, supervise slug, env)` `request()` resolved for this
|
||||
flow, so a later hook scans against the calling bottle's policy. Falls
|
||||
back to deny-all (empty routes, empty slug) for a flow that never passed
|
||||
through `request()` (or a flow object without metadata) — fail-closed, so
|
||||
a DLP hook on such a flow is a safe no-op rather than an unscanned pass."""
|
||||
meta = getattr(flow, "metadata", None)
|
||||
if isinstance(meta, dict):
|
||||
ctx = meta.get(_FLOW_CTX_KEY)
|
||||
if ctx is not None:
|
||||
return ctx
|
||||
return Config(routes=()), "", os.environ
|
||||
|
||||
def _request_token(self, flow: http.HTTPFlow) -> str:
|
||||
"""The per-bottle identity token for this request, from the proxy
|
||||
credentials — the delivery mechanism (`HTTPS_PROXY=http://id:token@gw`)
|
||||
that clients honor without app changes. Plain-HTTP requests carry
|
||||
`Proxy-Authorization` directly; HTTPS bumped requests inherit the token
|
||||
captured from their tunnel's CONNECT. Read then stripped so it never
|
||||
leaks upstream (also strips the legacy header, if present)."""
|
||||
token = _token_from_proxy_auth(
|
||||
flow.request.headers.get("Proxy-Authorization", ""))
|
||||
flow.request.headers.pop("Proxy-Authorization", None)
|
||||
flow.request.headers.pop(IDENTITY_HEADER, None)
|
||||
conn = flow.client_conn
|
||||
conn_id = getattr(conn, "id", "") if conn is not None else ""
|
||||
if not token and conn_id:
|
||||
token = self._conn_tokens.get(conn_id, "")
|
||||
# Remember the token per connection so a later token-block on this flow
|
||||
# can attribute its supervise proposal by (source_ip, identity_token)
|
||||
# over RPC — plain-HTTP requests carry it here, HTTPS tunnels captured
|
||||
# it at CONNECT (http_connect); both land in `_conn_tokens`.
|
||||
if token and conn_id:
|
||||
self._conn_tokens[conn_id] = token
|
||||
return token
|
||||
|
||||
def http_connect(self, flow: http.HTTPFlow) -> None:
|
||||
"""Capture the identity token from an HTTPS tunnel's CONNECT (the inner
|
||||
bumped requests won't carry `Proxy-Authorization`), keyed by client
|
||||
connection, and strip it so it never reaches upstream.
|
||||
|
||||
For `inspect: false` routes, also resolve the policy here to make the
|
||||
allowlist decision before the TLS handshake: the tunnel is either
|
||||
blocked immediately or marked for passthrough in `_passthrough_conns`
|
||||
so `tls_clienthello` skips interception."""
|
||||
token = _token_from_proxy_auth(
|
||||
flow.request.headers.get("Proxy-Authorization", ""))
|
||||
flow.request.headers.pop("Proxy-Authorization", None)
|
||||
conn = flow.client_conn
|
||||
conn_id = getattr(conn, "id", "") if conn is not None else ""
|
||||
if conn_id:
|
||||
self._conn_tokens[conn_id] = token
|
||||
|
||||
# Resolve the policy here for all HTTPS connections and stash it so
|
||||
# request() reuses it without a second orchestrator round-trip. For
|
||||
# passthrough hosts we also make the allowlist decision now because
|
||||
# inner requests never reach request() after the TLS bypass.
|
||||
client_ip = conn.peername[0] if conn is not None and conn.peername else ""
|
||||
config, slug, env = resolve_client_context(self._resolver, client_ip, token)
|
||||
self._stash_flow_ctx(flow, config, slug, env)
|
||||
host = flow.request.pretty_host
|
||||
route = match_route(config.routes, host)
|
||||
if route is not None and not route.inspect:
|
||||
decision = decide(config.routes, host, "/", env, deny_reason=config.deny_reason)
|
||||
if decision.action == "block":
|
||||
flow.response = http.Response.make(
|
||||
403,
|
||||
decision.reason.encode("utf-8"),
|
||||
{"Content-Type": "text/plain; charset=utf-8"},
|
||||
)
|
||||
return
|
||||
if conn_id:
|
||||
self._passthrough_conns.add(conn_id)
|
||||
|
||||
def tls_clienthello(self, client_hello: typing.Any) -> None:
|
||||
"""Skip TLS interception for `inspect: false` routes so the client sees
|
||||
the server's real certificate rather than the MITM CA's leaf."""
|
||||
conn_id = getattr(client_hello.context.client, "id", "")
|
||||
if conn_id in self._passthrough_conns:
|
||||
client_hello.ignore_connection = True
|
||||
|
||||
def client_disconnected(self, client: typing.Any) -> None:
|
||||
"""Drop the per-connection token and passthrough flag when the client
|
||||
goes away."""
|
||||
conn_id = getattr(client, "id", "")
|
||||
self._conn_tokens.pop(conn_id, None)
|
||||
self._passthrough_conns.discard(conn_id)
|
||||
|
||||
async def request(self, flow: http.HTTPFlow) -> None:
|
||||
request_path, _, query = flow.request.path.partition("?")
|
||||
|
||||
# Reuse the context stashed by http_connect for HTTPS flows (one
|
||||
# orchestrator round-trip per connection). Plain-HTTP flows have no
|
||||
# prior CONNECT stash, so resolve now and stash for response/websocket.
|
||||
meta = getattr(flow, "metadata", None)
|
||||
if isinstance(meta, dict) and _FLOW_CTX_KEY in meta:
|
||||
config, slug, env = meta[_FLOW_CTX_KEY]
|
||||
self._request_token(flow) # strip identity headers; token already resolved
|
||||
else:
|
||||
config, slug, env = self._resolve_flow(flow)
|
||||
self._stash_flow_ctx(flow, config, slug, env)
|
||||
|
||||
# Introspection ("_egress.local/allowlist") reports the calling bottle's
|
||||
# own resolved routes — served after resolution so it reflects this
|
||||
# bottle's policy, not a stale global.
|
||||
if flow.request.pretty_host == INTROSPECT_HOST:
|
||||
self._serve_introspection(flow, request_path, config)
|
||||
return
|
||||
|
||||
# DLP outbound scan BEFORE stripping auth — catches tokens the
|
||||
# agent tried to smuggle in any header, path, query param, or body.
|
||||
# Hostname is included to catch DNS-tunnelling exfiltration attempts.
|
||||
# `inspect: false` routes skip scanning entirely (TLS is also not
|
||||
# intercepted for HTTPS, so this branch only fires for plain HTTP).
|
||||
route = match_route(config.routes, flow.request.pretty_host)
|
||||
if route is not None and route.inspect:
|
||||
if not await self._handle_outbound_dlp(flow, route, slug, env):
|
||||
return
|
||||
# The redact policy may have rewritten the request line; recompute
|
||||
# the path/query the git checks below rely on.
|
||||
request_path, _, query = flow.request.path.partition("?")
|
||||
|
||||
if is_git_push_request(request_path, query):
|
||||
self._block(
|
||||
flow,
|
||||
"egress: git push over HTTPS is not supported; "
|
||||
"use the bottle.git SSH path (gitleaks-scanned by "
|
||||
"git-gate's pre-receive hook).",
|
||||
ctx=self._req_ctx(flow),
|
||||
)
|
||||
return
|
||||
|
||||
if is_git_fetch_request(request_path, query):
|
||||
git_decision = decide_git_fetch(
|
||||
config.routes, flow.request.pretty_host,
|
||||
)
|
||||
if git_decision.action == "block":
|
||||
self._block(
|
||||
flow,
|
||||
git_decision.reason,
|
||||
ctx=self._req_ctx(flow),
|
||||
)
|
||||
return
|
||||
|
||||
# Strip agent-set Authorization after DLP scan so smuggled tokens
|
||||
# are caught above; the route may inject gateway-owned auth below.
|
||||
# Routes with preserve_auth=True pass the header through as-is so the
|
||||
# agent's own credentials (e.g. registry bearer tokens) reach the upstream.
|
||||
if route is None or not route.preserve_auth:
|
||||
flow.request.headers.pop("authorization", None)
|
||||
|
||||
# Build headers mapping for match evaluation
|
||||
req_headers = {k.lower(): v for k, v in flow.request.headers.items()}
|
||||
|
||||
decision = decide(
|
||||
config.routes,
|
||||
flow.request.pretty_host,
|
||||
request_path,
|
||||
env,
|
||||
request_method=flow.request.method,
|
||||
request_headers=req_headers,
|
||||
deny_reason=config.deny_reason,
|
||||
)
|
||||
|
||||
if decision.action == "block":
|
||||
self._block(flow, decision.reason, ctx=self._req_ctx(flow))
|
||||
return
|
||||
|
||||
if decision.inject_authorization is not None:
|
||||
flow.request.headers["authorization"] = decision.inject_authorization
|
||||
|
||||
if config.log >= LOG_FULL:
|
||||
self._log_request(flow, env)
|
||||
|
||||
def _block_dlp(self, flow: http.HTTPFlow, result: ScanResult) -> None:
|
||||
ctx = self._req_ctx(flow)
|
||||
if result.context:
|
||||
ctx = {**ctx, "context": result.context}
|
||||
self._block(flow, f"egress DLP: {result.reason}", ctx=ctx)
|
||||
|
||||
async def _handle_outbound_dlp(
|
||||
self,
|
||||
flow: http.HTTPFlow,
|
||||
route: Route,
|
||||
slug: str,
|
||||
env: "typing.Mapping[str, str]",
|
||||
) -> bool:
|
||||
"""Scan the outbound request and apply the route's on-match policy
|
||||
(PRD 0062). `env` is the per-bottle env overlay (process env + this
|
||||
bottle's tokens) used for DLP detection. Returns True if the request may
|
||||
be forwarded, False if a 403 response has been written to `flow`.
|
||||
|
||||
Loops so the supervise policy can re-scan after each approval — a
|
||||
second, un-approved token in the same request is still caught."""
|
||||
while True:
|
||||
request_path, _, query = flow.request.path.partition("?")
|
||||
body = flow.request.get_text(strict=False) or ""
|
||||
headers = outbound_scan_headers(route, dict(flow.request.headers))
|
||||
scan_text = build_outbound_scan_text(
|
||||
flow.request.pretty_host, request_path, query, headers, body,
|
||||
)
|
||||
# CRLF is scanned only over the request line + headers, never the
|
||||
# body (see scan_outbound) — a body is not an injection vector.
|
||||
crlf_text = build_outbound_scan_text(
|
||||
flow.request.pretty_host, request_path, query, headers, "",
|
||||
)
|
||||
result = scan_outbound(
|
||||
route, scan_text, env,
|
||||
safe_tokens=self._safe_tokens_for(slug), crlf_text=crlf_text,
|
||||
)
|
||||
if result is None or result.severity != "block":
|
||||
return True
|
||||
|
||||
policy = route.outbound_on_match or DEFAULT_OUTBOUND_ON_MATCH
|
||||
|
||||
# redact scrubs every detection (tokens and structural CRLF) and
|
||||
# forwards; it fails closed only if a match survives the scrub.
|
||||
if policy == ON_MATCH_REDACT:
|
||||
if self._redact_outbound(flow, route, env):
|
||||
if self._flow_log(flow) >= LOG_BLOCKS:
|
||||
sys.stderr.write(json.dumps({
|
||||
"event": "egress_redacted",
|
||||
"reason": f"egress DLP: {result.reason}",
|
||||
**self._req_ctx(flow),
|
||||
}) + "\n")
|
||||
return True
|
||||
self._block(
|
||||
flow,
|
||||
f"egress DLP: {result.reason}; redaction could not remove "
|
||||
"all matches (e.g. a match in the hostname)",
|
||||
ctx=self._req_ctx(flow),
|
||||
)
|
||||
return False
|
||||
|
||||
# Structural blocks (CRLF, no safelist-able value) cannot be
|
||||
# supervised — there is nothing to approve and remember — so under
|
||||
# block/supervise they are a hard 403.
|
||||
if policy == ON_MATCH_BLOCK or not result.matched:
|
||||
self._block_dlp(flow, result)
|
||||
return False
|
||||
|
||||
# supervise (default): hold the request for operator approval.
|
||||
# Fall back to a hard 403 when supervise isn't wired for the bottle.
|
||||
if not self._supervise_available(slug):
|
||||
self._block_dlp(flow, result)
|
||||
return False
|
||||
approved = await self._supervise_token_block(flow, request_path, result, slug, env)
|
||||
if not approved:
|
||||
return False # _supervise_token_block wrote the 403 response
|
||||
# loop: the approved value is now in safe_tokens; re-scan.
|
||||
|
||||
def _redact_outbound(
|
||||
self, flow: http.HTTPFlow, route: Route, env: "typing.Mapping[str, str]",
|
||||
) -> bool:
|
||||
"""Scrub detected tokens (and CRLF injection sequences) from the mutable
|
||||
request surfaces (body, headers, path/query) and re-scan. `env` is the
|
||||
per-bottle env overlay. Returns True if the request is now clean; False
|
||||
if a block-severity match remains on a surface redaction cannot rewrite
|
||||
(the hostname) so the caller fails closed."""
|
||||
body = flow.request.get_text(strict=False)
|
||||
if body:
|
||||
redacted_body = redact_tokens(body, env=env)
|
||||
if redacted_body != body:
|
||||
flow.request.text = redacted_body
|
||||
for name, value in list(flow.request.headers.items()):
|
||||
if name.lower() == "host":
|
||||
continue # routing-critical; never a legitimate token
|
||||
redacted = strip_crlf(redact_tokens(value, env=env))
|
||||
if redacted != value:
|
||||
flow.request.headers[name] = redacted
|
||||
redacted_path = strip_crlf(redact_tokens(flow.request.path, env=env))
|
||||
if redacted_path != flow.request.path:
|
||||
flow.request.path = redacted_path
|
||||
|
||||
request_path, _, query = flow.request.path.partition("?")
|
||||
new_body = flow.request.get_text(strict=False) or ""
|
||||
headers = outbound_scan_headers(route, dict(flow.request.headers))
|
||||
scan_text = build_outbound_scan_text(
|
||||
flow.request.pretty_host, request_path, query, headers, new_body,
|
||||
)
|
||||
crlf_text = build_outbound_scan_text(
|
||||
flow.request.pretty_host, request_path, query, headers, "",
|
||||
)
|
||||
result = scan_outbound(route, scan_text, env, crlf_text=crlf_text)
|
||||
return result is None or result.severity != "block"
|
||||
|
||||
async def _supervise_token_block(
|
||||
self,
|
||||
flow: http.HTTPFlow,
|
||||
request_path: str,
|
||||
result: ScanResult,
|
||||
slug: str,
|
||||
env: "typing.Mapping[str, str]",
|
||||
) -> bool:
|
||||
"""Route a token DLP block to the operator's supervisor queue and wait.
|
||||
|
||||
`slug` attributes the proposal to the calling bottle (its own queue +
|
||||
safelist) — in the shared gateway this is what keeps one bottle's
|
||||
approval from unblocking another's request. `env` is the per-bottle env
|
||||
overlay used to redact secrets from the proposal text. Returns True if
|
||||
the operator approved (the matched value is added to that bottle's
|
||||
safelist and the caller re-scans); False if the request must be blocked
|
||||
(a 403 response has been written to `flow`)."""
|
||||
host = flow.request.pretty_host
|
||||
payload = build_token_allow_payload(
|
||||
redact_tokens(host, env=env),
|
||||
flow.request.method,
|
||||
redact_tokens(request_path, env=env),
|
||||
result,
|
||||
)
|
||||
# Attribute the proposal by (source_ip, identity_token) over the control
|
||||
# plane — the data plane no longer opens bot-bottle.db (PRD 0070 / #469).
|
||||
# source_ip + token come from this bottle's connection; `slug` is kept
|
||||
# only for its own DLP safelist.
|
||||
conn = flow.client_conn
|
||||
source_ip = conn.peername[0] if conn is not None and conn.peername else ""
|
||||
token = self._conn_tokens.get(getattr(conn, "id", ""), "") if conn is not None else ""
|
||||
try:
|
||||
proposal_id = self._resolver.propose_supervise(
|
||||
source_ip, token,
|
||||
tool=TOOL_EGRESS_TOKEN_ALLOW,
|
||||
proposed_file=payload,
|
||||
justification=_TOKEN_ALLOW_JUSTIFICATION,
|
||||
)
|
||||
except PolicyResolveError as e:
|
||||
sys.stderr.write(
|
||||
f"egress: could not queue token-allow proposal: {e}; "
|
||||
"blocking request\n"
|
||||
)
|
||||
proposal_id = None
|
||||
if proposal_id is None:
|
||||
self._block(flow, f"egress DLP: {result.reason}", ctx=self._req_ctx(flow))
|
||||
return False
|
||||
|
||||
sys.stderr.write(json.dumps({
|
||||
"event": "egress_token_supervise",
|
||||
"reason": f"egress DLP: {result.reason}",
|
||||
"proposal": proposal_id,
|
||||
**self._req_ctx(flow),
|
||||
}) + "\n")
|
||||
|
||||
response = await self._await_token_response(proposal_id, source_ip, token)
|
||||
|
||||
if response is not None and response.get("status") in (
|
||||
STATUS_APPROVED, STATUS_MODIFIED,
|
||||
):
|
||||
self._safe_tokens_for(slug).add(result.matched)
|
||||
if self._flow_log(flow) >= LOG_BLOCKS:
|
||||
sys.stderr.write(json.dumps({
|
||||
"event": "egress_token_allowed",
|
||||
"reason": f"egress DLP: {result.reason}",
|
||||
"proposal": proposal_id,
|
||||
**self._req_ctx(flow),
|
||||
}) + "\n")
|
||||
return True
|
||||
|
||||
if response is None:
|
||||
reason = (
|
||||
f"egress DLP: {result.reason}; supervisor approval timed out "
|
||||
f"after {self._token_allow_timeout:g}s"
|
||||
)
|
||||
else:
|
||||
reason = f"egress DLP: {result.reason}; supervisor rejected the request"
|
||||
self._block(flow, reason, ctx=self._req_ctx(flow))
|
||||
return False
|
||||
|
||||
async def _await_token_response(
|
||||
self,
|
||||
proposal_id: str,
|
||||
source_ip: str,
|
||||
token: str,
|
||||
) -> "dict[str, object] | None":
|
||||
"""Poll the control plane for the operator's decision without blocking
|
||||
the proxy event loop. Returns the terminal `{status, ...}` payload once
|
||||
decided, or None on timeout. A transient orchestrator error (or a
|
||||
`pending`/`unknown` status) is retried until the deadline, then fails
|
||||
closed — the caller blocks the request."""
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + self._token_allow_timeout
|
||||
while True:
|
||||
try:
|
||||
result = self._resolver.poll_supervise(source_ip, token, proposal_id)
|
||||
except PolicyResolveError:
|
||||
result = None
|
||||
if result is not None and result.get("status") in STATUSES:
|
||||
return result
|
||||
if loop.time() >= deadline:
|
||||
return None
|
||||
await asyncio.sleep(TOKEN_ALLOW_POLL_INTERVAL_SECONDS)
|
||||
|
||||
def response(self, flow: http.HTTPFlow) -> None:
|
||||
"""DLP inbound scan on response headers and body, against the calling
|
||||
bottle's resolved config (`request()` stashed it — see `_flow_ctx`)."""
|
||||
config, _slug, env = self._flow_ctx(flow)
|
||||
route = match_route(config.routes, flow.request.pretty_host)
|
||||
if route is None or not route.inspect:
|
||||
return
|
||||
if flow.response is None:
|
||||
return
|
||||
if config.log >= LOG_FULL:
|
||||
self._log_response(flow, env)
|
||||
resp_headers = {k.lower(): v for k, v in flow.response.headers.items()}
|
||||
body = flow.response.get_text(strict=False) or ""
|
||||
if self._inbound_scan_limit and len(body) > self._inbound_scan_limit:
|
||||
sys.stderr.write(json.dumps({
|
||||
"event": "egress_scan_truncated",
|
||||
"host": flow.request.pretty_host,
|
||||
"body_bytes": len(body),
|
||||
"scan_limit_bytes": self._inbound_scan_limit,
|
||||
}) + "\n")
|
||||
body = body[:self._inbound_scan_limit]
|
||||
scan_text = build_inbound_scan_text(resp_headers, body)
|
||||
if not scan_text:
|
||||
return
|
||||
result = scan_inbound(route, scan_text)
|
||||
if result is None:
|
||||
return
|
||||
resp_ctx: dict[str, object] = {
|
||||
**self._req_ctx(flow),
|
||||
"response_status": flow.response.status_code,
|
||||
}
|
||||
if result.context:
|
||||
resp_ctx = {**resp_ctx, "context": result.context}
|
||||
if result.severity == "block":
|
||||
self._block(flow, f"egress DLP: {result.reason}", ctx=resp_ctx)
|
||||
elif result.severity == "warn" and config.log >= LOG_BLOCKS:
|
||||
sys.stderr.write(
|
||||
json.dumps({
|
||||
"event": "egress_warn",
|
||||
"reason": f"egress DLP: {result.reason}",
|
||||
**resp_ctx,
|
||||
})
|
||||
+ "\n"
|
||||
)
|
||||
|
||||
def websocket_message(self, flow: http.HTTPFlow) -> None:
|
||||
"""DLP scan on WebSocket frames, against the calling bottle's resolved
|
||||
config (see `_flow_ctx`). `request()` resolves and stashes the per-flow
|
||||
(config, slug, env) at the upgrade, and every frame reuses it.
|
||||
|
||||
Outbound frames (from_client) are scanned for credential leakage;
|
||||
inbound frames are scanned for prompt injection. On a block the
|
||||
entire connection is killed — there is no HTTP response surface to
|
||||
write to after the upgrade.
|
||||
"""
|
||||
if flow.websocket is None: # type: ignore[union-attr]
|
||||
return
|
||||
config, slug, env = self._flow_ctx(flow)
|
||||
route = match_route(config.routes, flow.request.pretty_host)
|
||||
if route is None or not route.inspect:
|
||||
return
|
||||
message = flow.websocket.messages[-1] # type: ignore[union-attr]
|
||||
content = message.content.decode("utf-8", errors="replace")
|
||||
if message.from_client:
|
||||
# A WebSocket data frame is not an HTTP request line, so CRLF is
|
||||
# not an injection vector here — scan only for credential leakage.
|
||||
result = scan_outbound(
|
||||
route, content, env,
|
||||
safe_tokens=self._safe_tokens_for(slug), crlf_text="",
|
||||
)
|
||||
if result is not None and result.severity == "block":
|
||||
sys.stderr.write(f"egress DLP: {result.reason}\n")
|
||||
flow.kill() # type: ignore[union-attr]
|
||||
else:
|
||||
result = scan_inbound(route, content)
|
||||
if result is not None:
|
||||
if result.severity == "block":
|
||||
sys.stderr.write(f"egress DLP: {result.reason}\n")
|
||||
flow.kill() # type: ignore[union-attr]
|
||||
elif result.severity == "warn":
|
||||
sys.stderr.write(f"egress DLP warn: {result.reason}\n")
|
||||
|
||||
|
||||
def _inbound_scan_limit_from_env(env: "os._Environ[str]") -> int:
|
||||
"""Read EGRESS_INBOUND_SCAN_LIMIT_BYTES; fall back to the default on an
|
||||
unset or invalid value. Returns 0 to disable the cap."""
|
||||
raw = env.get("EGRESS_INBOUND_SCAN_LIMIT_BYTES", "").strip()
|
||||
if not raw:
|
||||
return DEFAULT_INBOUND_SCAN_LIMIT_BYTES
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
value = -1
|
||||
if value < 0:
|
||||
sys.stderr.write(
|
||||
"egress: invalid EGRESS_INBOUND_SCAN_LIMIT_BYTES="
|
||||
f"{raw!r}; using default {DEFAULT_INBOUND_SCAN_LIMIT_BYTES}\n"
|
||||
)
|
||||
return DEFAULT_INBOUND_SCAN_LIMIT_BYTES
|
||||
return value
|
||||
|
||||
|
||||
def _token_allow_timeout_from_env(env: "os._Environ[str]") -> float:
|
||||
"""Read EGRESS_TOKEN_ALLOW_TIMEOUT_SECONDS; fall back to the default on an
|
||||
unset or invalid value (a bad value should not wedge egress at boot)."""
|
||||
raw = env.get("EGRESS_TOKEN_ALLOW_TIMEOUT_SECONDS", "").strip()
|
||||
if not raw:
|
||||
return DEFAULT_TOKEN_ALLOW_TIMEOUT_SECONDS
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
value = 0.0
|
||||
if value <= 0:
|
||||
sys.stderr.write(
|
||||
"egress: invalid EGRESS_TOKEN_ALLOW_TIMEOUT_SECONDS="
|
||||
f"{raw!r}; using default {DEFAULT_TOKEN_ALLOW_TIMEOUT_SECONDS:g}s\n"
|
||||
)
|
||||
return DEFAULT_TOKEN_ALLOW_TIMEOUT_SECONDS
|
||||
return value
|
||||
|
||||
|
||||
addons = [EgressAddon()]
|
||||
@@ -0,0 +1,954 @@
|
||||
"""Pure logic for the egress mitmproxy addon (PRD 0017, PRD 0053).
|
||||
|
||||
Split out of `egress_addon.py` so the host's unit tests can
|
||||
exercise the parse + decision functions without depending on the
|
||||
`mitmproxy` package. The companion module wraps these with the
|
||||
`mitmproxy.http.HTTPFlow` API and is loaded inside the gateway
|
||||
container.
|
||||
|
||||
Imports: stdlib + sibling package modules (`yaml_subset`,
|
||||
`egress_dlp_config`). Available in the gateway via the installed
|
||||
`bot_bottle` package (see `Dockerfile.gateway`)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import typing
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..yaml_subset import YamlSubsetError, parse_yaml_subset
|
||||
|
||||
# DLP detector-config parsing lives in a sibling module. Re-exported below
|
||||
# so existing `from egress_addon_core import ON_MATCH_*` callers keep working.
|
||||
from .egress_dlp_config import (
|
||||
DEFAULT_OUTBOUND_ON_MATCH,
|
||||
INBOUND_DETECTOR_NAMES,
|
||||
ON_MATCH_BLOCK,
|
||||
ON_MATCH_REDACT,
|
||||
ON_MATCH_SUPERVISE,
|
||||
OUTBOUND_DETECTOR_NAMES,
|
||||
OUTBOUND_ON_MATCH_VALUES,
|
||||
parse_inspect_block,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Match types (Gateway API HTTPRoute vocabulary, PRD 0053)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PATH_MATCH_TYPES = ("exact", "prefix", "regex")
|
||||
HEADER_MATCH_TYPES = ("exact", "regex")
|
||||
|
||||
VALID_METHODS = frozenset({
|
||||
"GET", "HEAD", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "TRACE",
|
||||
"CONNECT",
|
||||
})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PathMatch:
|
||||
type: str # "exact" | "prefix" | "regex"
|
||||
value: str
|
||||
compiled: re.Pattern[str] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HeaderMatch:
|
||||
name: str
|
||||
value: str
|
||||
type: str = "exact" # "exact" | "regex"
|
||||
compiled: re.Pattern[str] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MatchEntry:
|
||||
paths: tuple[PathMatch, ...] = ()
|
||||
methods: tuple[str, ...] = ()
|
||||
headers: tuple[HeaderMatch, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Route:
|
||||
host: str
|
||||
matches: tuple[MatchEntry, ...] = ()
|
||||
auth_scheme: str = ""
|
||||
token_env: str = ""
|
||||
git_fetch: bool = False
|
||||
outbound_detectors: tuple[str, ...] | None = None
|
||||
inbound_detectors: tuple[str, ...] | None = None
|
||||
# "" means unset → DEFAULT_OUTBOUND_ON_MATCH. See OUTBOUND_ON_MATCH_VALUES.
|
||||
outbound_on_match: str = ""
|
||||
preserve_auth: bool = False
|
||||
# False tunnels HTTPS without TLS interception or HTTP-level controls.
|
||||
inspect: bool = True
|
||||
|
||||
|
||||
LOG_OFF = 0 # no logging
|
||||
LOG_BLOCKS = 1 # log block/warn events with request context
|
||||
LOG_FULL = 2 # log block/warn events + full request and response bodies
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
routes: tuple[Route, ...]
|
||||
log: int = LOG_OFF
|
||||
# Why this Config is a deny-all, when it is one for a reason *other* than
|
||||
# the bottle's own policy genuinely not listing the host. A deny-all is
|
||||
# indistinguishable from "policy loaded, host not allowed" at the decision
|
||||
# point — both are simply "no matching route" — so without this the
|
||||
# operator sees `host X is not in the allowlist` and goes hunting for a
|
||||
# missing route that was never the problem. Empty for a normally-parsed
|
||||
# policy; `decide` prefers it over the allowlist wording when set.
|
||||
deny_reason: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Decision:
|
||||
action: str # "forward" or "block"
|
||||
reason: str = ""
|
||||
inject_authorization: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ScanResult:
|
||||
severity: str # "block" or "warn"
|
||||
reason: str
|
||||
location: str = "" # where the match was found, e.g. "body", "authorization header"
|
||||
context: str = "" # surrounding text with the match replaced by REDACT
|
||||
# Raw substring the detector matched. Used inside the gateway to key the
|
||||
# supervisor-approved "safe tokens" set (PRD 0062); never logged or written
|
||||
# to a proposal file. Empty for structural detectors (CRLF) that carry no
|
||||
# safelist-able value.
|
||||
matched: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _parse_path_match(idx: int, j: int, raw: object) -> PathMatch:
|
||||
label = f"route[{idx}] matches paths[{j}]"
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"{label}: must be an object")
|
||||
raw_dict: dict[str, object] = typing.cast(dict[str, object], raw)
|
||||
ptype = raw_dict.get("type", "prefix")
|
||||
if not isinstance(ptype, str) or ptype not in PATH_MATCH_TYPES:
|
||||
raise ValueError(
|
||||
f"{label}: 'type' must be one of {', '.join(PATH_MATCH_TYPES)} "
|
||||
f"(got {ptype!r})"
|
||||
)
|
||||
value = raw_dict.get("value")
|
||||
if not isinstance(value, str) or not value:
|
||||
raise ValueError(f"{label}: 'value' must be a non-empty string")
|
||||
if ptype in ("exact", "prefix") and not value.startswith("/"):
|
||||
raise ValueError(
|
||||
f"{label}: value {value!r} must start with '/' for "
|
||||
f"type {ptype!r}"
|
||||
)
|
||||
compiled: re.Pattern[str] | None = None
|
||||
if ptype == "regex":
|
||||
try:
|
||||
compiled = re.compile(value)
|
||||
except re.error as e:
|
||||
raise ValueError(
|
||||
f"{label}: regex {value!r} failed to compile: {e}"
|
||||
) from e
|
||||
for k in raw_dict:
|
||||
if k not in ("type", "value"):
|
||||
raise ValueError(f"{label}: unknown key {k!r}")
|
||||
return PathMatch(type=ptype, value=value, compiled=compiled)
|
||||
|
||||
|
||||
def _parse_header_match(idx: int, j: int, raw: object) -> HeaderMatch:
|
||||
label = f"route[{idx}] matches headers[{j}]"
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"{label}: must be an object")
|
||||
raw_dict: dict[str, object] = typing.cast(dict[str, object], raw)
|
||||
name = raw_dict.get("name")
|
||||
if not isinstance(name, str) or not name:
|
||||
raise ValueError(f"{label}: 'name' must be a non-empty string")
|
||||
value = raw_dict.get("value")
|
||||
if not isinstance(value, str):
|
||||
raise ValueError(f"{label}: 'value' must be a string")
|
||||
htype = raw_dict.get("type", "exact")
|
||||
if not isinstance(htype, str) or htype not in HEADER_MATCH_TYPES:
|
||||
raise ValueError(
|
||||
f"{label}: 'type' must be one of {', '.join(HEADER_MATCH_TYPES)} "
|
||||
f"(got {htype!r})"
|
||||
)
|
||||
compiled: re.Pattern[str] | None = None
|
||||
if htype == "regex":
|
||||
try:
|
||||
compiled = re.compile(value)
|
||||
except re.error as e:
|
||||
raise ValueError(
|
||||
f"{label}: regex {value!r} failed to compile: {e}"
|
||||
) from e
|
||||
for k in raw_dict:
|
||||
if k not in ("name", "value", "type"):
|
||||
raise ValueError(f"{label}: unknown key {k!r}")
|
||||
return HeaderMatch(name=name, value=value, type=htype, compiled=compiled)
|
||||
|
||||
|
||||
def _parse_match_entry(idx: int, k: int, raw: object) -> MatchEntry:
|
||||
label = f"route[{idx}] matches[{k}]"
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"{label}: must be an object")
|
||||
raw_dict: dict[str, object] = typing.cast(dict[str, object], raw)
|
||||
|
||||
paths: tuple[PathMatch, ...] = ()
|
||||
paths_raw = raw_dict.get("paths")
|
||||
if paths_raw is not None:
|
||||
if not isinstance(paths_raw, list):
|
||||
raise ValueError(f"{label}: 'paths' must be a list")
|
||||
paths_list = typing.cast(list[object], paths_raw)
|
||||
paths = tuple(_parse_path_match(idx, j, p) for j, p in enumerate(paths_list))
|
||||
|
||||
methods: tuple[str, ...] = ()
|
||||
methods_raw = raw_dict.get("methods")
|
||||
if methods_raw is not None:
|
||||
if not isinstance(methods_raw, list):
|
||||
raise ValueError(f"{label}: 'methods' must be a list")
|
||||
methods_list = typing.cast(list[object], methods_raw)
|
||||
normalised: list[str] = []
|
||||
for j, m in enumerate(methods_list):
|
||||
if not isinstance(m, str):
|
||||
raise ValueError(f"{label}: methods[{j}] must be a string")
|
||||
upper = m.upper()
|
||||
if upper not in VALID_METHODS:
|
||||
raise ValueError(
|
||||
f"{label}: methods[{j}] {m!r} is not a valid HTTP method"
|
||||
)
|
||||
normalised.append(upper)
|
||||
methods = tuple(normalised)
|
||||
|
||||
headers: tuple[HeaderMatch, ...] = ()
|
||||
headers_raw = raw_dict.get("headers")
|
||||
if headers_raw is not None:
|
||||
if not isinstance(headers_raw, list):
|
||||
raise ValueError(f"{label}: 'headers' must be a list")
|
||||
headers_list = typing.cast(list[object], headers_raw)
|
||||
headers = tuple(
|
||||
_parse_header_match(idx, j, h) for j, h in enumerate(headers_list)
|
||||
)
|
||||
|
||||
for key in raw_dict:
|
||||
if key not in ("paths", "methods", "headers"):
|
||||
raise ValueError(f"{label}: unknown key {key!r}")
|
||||
|
||||
return MatchEntry(paths=paths, methods=methods, headers=headers)
|
||||
|
||||
|
||||
def parse_routes(payload: object) -> tuple[Route, ...]:
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("routes payload: top-level must be an object")
|
||||
payload_dict: dict[str, object] = typing.cast(dict[str, object], payload)
|
||||
raw: object = payload_dict.get("routes")
|
||||
if not isinstance(raw, list):
|
||||
raise ValueError("routes payload: 'routes' must be a list")
|
||||
raw_list: list[object] = typing.cast(list[object], raw)
|
||||
out: list[Route] = []
|
||||
for i, r in enumerate(raw_list):
|
||||
out.append(_parse_one(i, r))
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _parse_one(idx: int, raw: object) -> Route:
|
||||
label = f"route[{idx}]"
|
||||
if not isinstance(raw, dict):
|
||||
raise ValueError(f"{label}: must be an object (got {type(raw).__name__})")
|
||||
raw_dict: dict[str, object] = typing.cast(dict[str, object], raw)
|
||||
host: object = raw_dict.get("host")
|
||||
if not isinstance(host, str) or not host:
|
||||
raise ValueError(f"{label}: 'host' must be a non-empty string")
|
||||
legacy_flat = "inspect" not in raw_dict
|
||||
inspect_raw = raw_dict.get("inspect", {})
|
||||
if inspect_raw is False:
|
||||
inspect = False
|
||||
settings: dict[str, object] = {}
|
||||
elif isinstance(inspect_raw, dict):
|
||||
inspect = True
|
||||
settings = (
|
||||
{k: v for k, v in raw_dict.items() if k != "host"}
|
||||
if legacy_flat
|
||||
else typing.cast(dict[str, object], inspect_raw)
|
||||
)
|
||||
legacy_dlp = settings.pop("dlp", None)
|
||||
if isinstance(legacy_dlp, dict):
|
||||
settings.update(typing.cast(dict[str, object], legacy_dlp))
|
||||
elif legacy_dlp is not None:
|
||||
raise ValueError(
|
||||
f"{label} ({host}): legacy 'dlp' must be an object"
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"{label} ({host}): 'inspect' must be false or an object")
|
||||
|
||||
# matches
|
||||
matches: tuple[MatchEntry, ...] = ()
|
||||
matches_raw = settings.get("matches")
|
||||
if matches_raw is not None:
|
||||
if not isinstance(matches_raw, list):
|
||||
raise ValueError(f"{label} ({host}): 'matches' must be a list")
|
||||
matches_list = typing.cast(list[object], matches_raw)
|
||||
matches = tuple(
|
||||
_parse_match_entry(idx, k, m) for k, m in enumerate(matches_list)
|
||||
)
|
||||
|
||||
# auth (unchanged wire format)
|
||||
auth_scheme: object = settings.get("auth_scheme", "")
|
||||
token_env: object = settings.get("token_env", "")
|
||||
if not isinstance(auth_scheme, str):
|
||||
raise ValueError(f"{label} ({host}): 'auth_scheme' must be a string")
|
||||
if not isinstance(token_env, str):
|
||||
raise ValueError(f"{label} ({host}): 'token_env' must be a string")
|
||||
if bool(auth_scheme) != bool(token_env):
|
||||
raise ValueError(
|
||||
f"{label} ({host}): 'auth_scheme' and 'token_env' must be both "
|
||||
f"set or both empty (got auth_scheme={auth_scheme!r}, "
|
||||
f"token_env={token_env!r})"
|
||||
)
|
||||
|
||||
# git-over-HTTPS policy
|
||||
git_fetch = False
|
||||
git_raw = settings.get("git")
|
||||
if git_raw is not None:
|
||||
if not isinstance(git_raw, dict):
|
||||
raise ValueError(f"{label} ({host}): 'git' must be an object")
|
||||
git_dict: dict[str, object] = typing.cast(dict[str, object], git_raw)
|
||||
fetch_raw = git_dict.get("fetch", False)
|
||||
if fetch_raw is True or fetch_raw is False:
|
||||
git_fetch = fetch_raw
|
||||
else:
|
||||
raise ValueError(f"{label} ({host}): 'git.fetch' must be a boolean")
|
||||
for k in git_dict:
|
||||
if k != "fetch":
|
||||
raise ValueError(
|
||||
f"{label} ({host}): git has unknown key {k!r}; "
|
||||
"accepted key is 'fetch'"
|
||||
)
|
||||
|
||||
# dlp detectors
|
||||
outbound_detectors, inbound_detectors, outbound_on_match = parse_inspect_block(
|
||||
idx, host, settings,
|
||||
)
|
||||
|
||||
preserve_auth_raw = settings.get("preserve_auth", False)
|
||||
if preserve_auth_raw is not True and preserve_auth_raw is not False:
|
||||
raise ValueError(
|
||||
f"{label} ({host}): 'preserve_auth' must be a boolean"
|
||||
)
|
||||
preserve_auth: bool = preserve_auth_raw
|
||||
|
||||
for k in settings:
|
||||
if k not in (
|
||||
"matches", "auth_scheme", "token_env", "git", "preserve_auth",
|
||||
"outbound_detectors", "inbound_detectors", "outbound_on_match",
|
||||
):
|
||||
raise ValueError(
|
||||
f"{label} ({host}): inspect has unknown key {k!r}"
|
||||
)
|
||||
for k in raw_dict:
|
||||
if not legacy_flat and k not in ("host", "inspect"):
|
||||
raise ValueError(
|
||||
f"{label} ({host}): unknown key {k!r}; accepted keys "
|
||||
f"are 'host' and 'inspect'"
|
||||
)
|
||||
|
||||
return Route(
|
||||
host=host,
|
||||
matches=matches,
|
||||
auth_scheme=auth_scheme,
|
||||
token_env=token_env,
|
||||
git_fetch=git_fetch,
|
||||
outbound_detectors=outbound_detectors,
|
||||
inbound_detectors=inbound_detectors,
|
||||
outbound_on_match=outbound_on_match,
|
||||
preserve_auth=preserve_auth,
|
||||
inspect=inspect,
|
||||
)
|
||||
|
||||
|
||||
def _path_match_to_dict(pm: PathMatch) -> dict[str, object]:
|
||||
d: dict[str, object] = {"value": pm.value}
|
||||
if pm.type != "prefix":
|
||||
d["type"] = pm.type
|
||||
return d
|
||||
|
||||
|
||||
def _header_match_to_dict(hm: HeaderMatch) -> dict[str, object]:
|
||||
d: dict[str, object] = {"name": hm.name, "value": hm.value}
|
||||
if hm.type != "exact":
|
||||
d["type"] = hm.type
|
||||
return d
|
||||
|
||||
|
||||
def _match_entry_to_dict(me: MatchEntry) -> dict[str, object]:
|
||||
d: dict[str, object] = {}
|
||||
if me.paths:
|
||||
d["paths"] = [_path_match_to_dict(p) for p in me.paths]
|
||||
if me.methods:
|
||||
d["methods"] = list(me.methods)
|
||||
if me.headers:
|
||||
d["headers"] = [_header_match_to_dict(h) for h in me.headers]
|
||||
return d
|
||||
|
||||
|
||||
def route_to_yaml_dict(r: Route) -> dict[str, object]:
|
||||
"""Serialize a Route to YAML-schema-compatible dict.
|
||||
|
||||
Uses the same field names the YAML parser accepts, so the output
|
||||
can be round-tripped directly into an `allow` or `egress-block`
|
||||
proposal without translation. Fields that are empty/default are
|
||||
omitted so the agent doesn't copy irrelevant keys."""
|
||||
d: dict[str, object] = {"host": r.host}
|
||||
if not r.inspect:
|
||||
d["inspect"] = False
|
||||
return d
|
||||
inspected: dict[str, object] = {}
|
||||
if r.auth_scheme:
|
||||
inspected["auth_scheme"] = r.auth_scheme
|
||||
inspected["token_env"] = r.token_env
|
||||
if r.matches:
|
||||
inspected["matches"] = [_match_entry_to_dict(m) for m in r.matches]
|
||||
if r.git_fetch:
|
||||
inspected["git"] = {"fetch": True}
|
||||
if r.outbound_detectors is not None:
|
||||
inspected["outbound_detectors"] = list(r.outbound_detectors)
|
||||
if r.inbound_detectors is not None:
|
||||
inspected["inbound_detectors"] = list(r.inbound_detectors)
|
||||
if r.outbound_on_match:
|
||||
inspected["outbound_on_match"] = r.outbound_on_match
|
||||
if r.preserve_auth:
|
||||
inspected["preserve_auth"] = True
|
||||
if inspected:
|
||||
d["inspect"] = inspected
|
||||
return d
|
||||
|
||||
|
||||
def parse_config(payload: object) -> "Config":
|
||||
"""Parse a full egress config payload (top-level log level + routes)."""
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("routes payload: top-level must be an object")
|
||||
payload_dict: dict[str, object] = typing.cast(dict[str, object], payload)
|
||||
|
||||
log_raw: object = payload_dict.get("log", LOG_OFF)
|
||||
if log_raw is True or log_raw is False or not isinstance(log_raw, int) \
|
||||
or log_raw not in (LOG_OFF, LOG_BLOCKS, LOG_FULL):
|
||||
raise ValueError(
|
||||
f"routes payload: 'log' must be {LOG_OFF}, {LOG_BLOCKS}, or {LOG_FULL}"
|
||||
)
|
||||
|
||||
routes = parse_routes(payload)
|
||||
return Config(routes=routes, log=log_raw)
|
||||
|
||||
|
||||
def load_config(text: str) -> "Config":
|
||||
"""Parse YAML text → Config (routes + log flag)."""
|
||||
try:
|
||||
payload = parse_yaml_subset(text)
|
||||
except YamlSubsetError as e:
|
||||
raise ValueError(f"routes payload: invalid YAML: {e}") from e
|
||||
return parse_config(payload)
|
||||
|
||||
|
||||
class PolicyResolverLike(typing.Protocol):
|
||||
"""The bit of `policy_resolver.PolicyResolver` this module needs — kept a
|
||||
Protocol so egress_addon_core stays free of that import."""
|
||||
|
||||
def resolve(self, source_ip: str, identity_token: str = ...) -> "str | None":
|
||||
...
|
||||
|
||||
|
||||
# Deny-all explanations. Each names the *actual* failure so an operator isn't
|
||||
# sent looking for a missing egress route when the bottle never had a policy
|
||||
# to begin with — the failure mode that made a bricked registration read like
|
||||
# a misconfigured allowlist.
|
||||
DENY_UNATTRIBUTED = (
|
||||
"egress: this request was not attributed to any bottle, so no egress "
|
||||
"policy applies and every host is denied. Either the bottle's registry "
|
||||
"row is missing/ambiguous (torn down, or another bottle claimed its "
|
||||
"source IP), or the request carried no matching identity token — check "
|
||||
"that the caller's proxy URL includes it. This is not an allowlist problem."
|
||||
)
|
||||
DENY_UNPARSEABLE = (
|
||||
"egress: this bottle's egress policy could not be parsed, so it is being "
|
||||
"treated as deny-all. Fix the bottle's egress.routes; every host is denied "
|
||||
"until it loads."
|
||||
)
|
||||
DENY_RESOLVER_ERROR = (
|
||||
"egress: the orchestrator could not be reached to resolve this bottle's "
|
||||
"egress policy, so every host is denied (fail-closed). Check that the "
|
||||
"control plane is up; this is not an allowlist problem."
|
||||
)
|
||||
|
||||
|
||||
def _config_from_policy(policy: "str | None") -> "Config":
|
||||
"""Parse a resolved policy blob into a Config, fail-closed: None / empty /
|
||||
unparseable all become a deny-all Config (no routes → every request
|
||||
blocked). Each deny-all carries the reason it is one, so the block message
|
||||
names the real fault instead of blaming the allowlist."""
|
||||
if not policy:
|
||||
return Config(routes=(), deny_reason=DENY_UNATTRIBUTED)
|
||||
try:
|
||||
return load_config(policy)
|
||||
except ValueError:
|
||||
return Config(routes=(), deny_reason=DENY_UNPARSEABLE)
|
||||
|
||||
|
||||
def resolve_client_config(
|
||||
resolver: PolicyResolverLike, client_ip: str, identity_token: str = ""
|
||||
) -> "Config":
|
||||
"""The calling client's egress Config, resolved from the orchestrator via
|
||||
`resolver` and parsed — **fail-closed**. An unattributed client (None), a
|
||||
resolver error, or an unparseable policy all yield a deny-all Config (no
|
||||
routes → every request blocked). A compromised, absent, or confused
|
||||
orchestrator must never *widen* a bottle's egress."""
|
||||
try:
|
||||
policy = resolver.resolve(client_ip, identity_token)
|
||||
except Exception: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
||||
return Config(routes=(), deny_reason=DENY_RESOLVER_ERROR)
|
||||
return _config_from_policy(policy)
|
||||
|
||||
|
||||
class ContextResolverLike(typing.Protocol):
|
||||
"""The bit of `policy_resolver.PolicyResolver` `resolve_client_context`
|
||||
needs — one round-trip returning policy, bottle id, and auth tokens."""
|
||||
|
||||
def resolve_policy_and_bottle_id(
|
||||
self, source_ip: str, identity_token: str = ...,
|
||||
) -> "tuple[str | None, str | None, dict[str, str]]":
|
||||
...
|
||||
|
||||
|
||||
def resolve_client_context(
|
||||
resolver: ContextResolverLike, client_ip: str, identity_token: str = "",
|
||||
) -> "tuple[Config, str, dict[str, str]]":
|
||||
"""The calling client's `(Config, bottle_id, tokens)` in one round-trip —
|
||||
**fail-closed**. The Config follows `resolve_client_config`'s deny-all
|
||||
rules; the bottle id is `""` whenever unattributed or the orchestrator
|
||||
errored (caller treats as "supervise unavailable", never another bottle's
|
||||
queue); `tokens` are the per-bottle upstream auth values the addon injects.
|
||||
One `/resolve` keys the egress policy, the supervise queue + safelist, and
|
||||
auth injection."""
|
||||
try:
|
||||
policy, bottle_id, tokens = resolver.resolve_policy_and_bottle_id(
|
||||
client_ip, identity_token,
|
||||
)
|
||||
except Exception: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
||||
return Config(routes=(), deny_reason=DENY_RESOLVER_ERROR), "", {}
|
||||
return _config_from_policy(policy), (bottle_id or ""), tokens
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Match evaluation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _path_matches(pm: PathMatch, request_path: str) -> bool:
|
||||
if pm.type == "exact":
|
||||
return request_path == pm.value
|
||||
if pm.type == "prefix":
|
||||
if request_path == pm.value:
|
||||
return True
|
||||
if not pm.value.endswith("/"):
|
||||
return request_path.startswith(pm.value + "/")
|
||||
return request_path.startswith(pm.value)
|
||||
if pm.type == "regex" and pm.compiled is not None:
|
||||
return pm.compiled.search(request_path) is not None
|
||||
return False
|
||||
|
||||
|
||||
def _entry_matches(
|
||||
entry: MatchEntry,
|
||||
request_path: str,
|
||||
request_method: str,
|
||||
request_headers: typing.Mapping[str, str],
|
||||
) -> bool:
|
||||
"""All predicates within a MatchEntry are ANDed."""
|
||||
if entry.paths:
|
||||
if not any(_path_matches(pm, request_path) for pm in entry.paths):
|
||||
return False
|
||||
if entry.methods:
|
||||
if request_method.upper() not in entry.methods:
|
||||
return False
|
||||
if entry.headers:
|
||||
for hm in entry.headers:
|
||||
header_val = request_headers.get(hm.name.lower())
|
||||
if header_val is None:
|
||||
return False
|
||||
if hm.type == "exact":
|
||||
if header_val != hm.value:
|
||||
return False
|
||||
elif hm.type == "regex" and hm.compiled is not None:
|
||||
if not hm.compiled.search(header_val):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def evaluate_matches(
|
||||
route: Route,
|
||||
request_path: str,
|
||||
request_method: str = "GET",
|
||||
request_headers: typing.Mapping[str, str] | None = None,
|
||||
) -> bool:
|
||||
"""Return True if the request matches this route's match entries.
|
||||
Empty matches tuple means all requests match (bare-pass route)."""
|
||||
if not route.matches:
|
||||
return True
|
||||
hdrs: typing.Mapping[str, str] = request_headers or {}
|
||||
return any(
|
||||
_entry_matches(entry, request_path, request_method, hdrs)
|
||||
for entry in route.matches
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Git push detection (unchanged)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def is_git_push_request(path: str, query: str) -> bool:
|
||||
if path.endswith("/git-receive-pack"):
|
||||
return True
|
||||
if path.endswith("/info/refs"):
|
||||
for pair in query.split("&"):
|
||||
k, _, v = pair.partition("=")
|
||||
if k == "service" and v == "git-receive-pack":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_git_fetch_request(path: str, query: str) -> bool:
|
||||
if path.endswith("/git-upload-pack"):
|
||||
return True
|
||||
if path.endswith("/info/refs"):
|
||||
for pair in query.split("&"):
|
||||
k, _, v = pair.partition("=")
|
||||
if k == "service" and v == "git-upload-pack":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Route lookup + decision
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def match_route(
|
||||
routes: typing.Sequence[Route],
|
||||
request_host: str,
|
||||
) -> Route | None:
|
||||
target = request_host.lower()
|
||||
for r in routes:
|
||||
if r.host.lower() == target:
|
||||
return r
|
||||
return None
|
||||
|
||||
|
||||
def decide(
|
||||
routes: typing.Sequence[Route],
|
||||
request_host: str,
|
||||
request_path: str,
|
||||
environ: typing.Mapping[str, str],
|
||||
*,
|
||||
request_method: str = "GET",
|
||||
request_headers: typing.Mapping[str, str] | None = None,
|
||||
deny_reason: str = "",
|
||||
) -> Decision:
|
||||
"""`deny_reason` is `Config.deny_reason`: when the deny-all came from a
|
||||
missing/unparseable policy rather than the bottle's own allowlist, report
|
||||
that instead of implying a route is merely absent."""
|
||||
route = match_route(routes, request_host)
|
||||
if route is None:
|
||||
return Decision(
|
||||
action="block",
|
||||
reason=deny_reason or (
|
||||
f"egress: host {request_host!r} is not in the "
|
||||
f"bottle's egress.routes allowlist. Declare a "
|
||||
f"route for it or remove the request."
|
||||
),
|
||||
)
|
||||
|
||||
if not evaluate_matches(route, request_path, request_method, request_headers):
|
||||
return Decision(
|
||||
action="block",
|
||||
reason=(
|
||||
f"egress: request {request_method} {request_path!r} "
|
||||
f"does not match any entry in matches for "
|
||||
f"{route.host!r}"
|
||||
),
|
||||
)
|
||||
|
||||
if route.auth_scheme and route.token_env:
|
||||
token = environ.get(route.token_env, "")
|
||||
if not token:
|
||||
return Decision(
|
||||
action="block",
|
||||
reason=(
|
||||
f"egress: route for {route.host!r} declared auth "
|
||||
f"but env var {route.token_env!r} is unset"
|
||||
),
|
||||
)
|
||||
return Decision(
|
||||
action="forward",
|
||||
inject_authorization=f"{route.auth_scheme} {token}",
|
||||
)
|
||||
|
||||
return Decision(action="forward")
|
||||
|
||||
|
||||
def decide_git_fetch(
|
||||
routes: typing.Sequence[Route],
|
||||
request_host: str,
|
||||
) -> Decision:
|
||||
route = match_route(routes, request_host)
|
||||
if route is not None and route.git_fetch:
|
||||
return Decision(action="forward")
|
||||
return Decision(
|
||||
action="block",
|
||||
reason=(
|
||||
"egress: git fetch/clone over HTTPS is not allowed by default; "
|
||||
"use git-gate for declared repos or set "
|
||||
"egress.routes[].git.fetch=true for explicit read-only "
|
||||
"HTTPS Git access."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DLP scan dispatch (PRD 0053)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def build_outbound_scan_text(
|
||||
host: str,
|
||||
path: str,
|
||||
query: str,
|
||||
headers: typing.Mapping[str, str],
|
||||
body: str,
|
||||
) -> str:
|
||||
"""Assemble all outbound request surfaces into one string for DLP scanning.
|
||||
|
||||
Covers hostname (DNS tunnelling), path, query params, all headers, body.
|
||||
"""
|
||||
parts: list[str] = [host, path]
|
||||
if query:
|
||||
parts.append(query)
|
||||
for name, value in headers.items():
|
||||
parts.append(f"{name}: {value}")
|
||||
if body:
|
||||
parts.append(body)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def outbound_scan_headers(
|
||||
route: Route,
|
||||
headers: typing.Mapping[str, str],
|
||||
) -> dict[str, str]:
|
||||
"""Return request headers that should be included in outbound DLP.
|
||||
|
||||
Routes that inject gateway-owned auth always strip the agent's
|
||||
Authorization header before forwarding. Scanning that header first
|
||||
creates false positives for provider clients that insist on sending
|
||||
their own bearer-shaped placeholder, while still not changing what
|
||||
reaches the upstream.
|
||||
"""
|
||||
out: dict[str, str] = {}
|
||||
skip_auth = bool(route.auth_scheme and route.token_env)
|
||||
for name, value in headers.items():
|
||||
if skip_auth and name.lower() == "authorization":
|
||||
continue
|
||||
out[name] = value
|
||||
return out
|
||||
|
||||
|
||||
def build_inbound_scan_text(
|
||||
headers: typing.Mapping[str, str],
|
||||
body: str,
|
||||
) -> str:
|
||||
"""Assemble inbound response surfaces into one string for DLP scanning.
|
||||
|
||||
Covers all response headers plus body.
|
||||
"""
|
||||
parts: list[str] = []
|
||||
for name, value in headers.items():
|
||||
parts.append(f"{name}: {value}")
|
||||
if body:
|
||||
parts.append(body)
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _detector_enabled(
|
||||
configured: tuple[str, ...] | None,
|
||||
name: str,
|
||||
) -> bool:
|
||||
"""Check if a named detector is enabled for a route direction.
|
||||
None means all enabled; empty tuple means all disabled."""
|
||||
if configured is None:
|
||||
return True
|
||||
return name in configured
|
||||
|
||||
|
||||
def scan_outbound(
|
||||
route: Route,
|
||||
body: str | bytes,
|
||||
environ: typing.Mapping[str, str],
|
||||
*,
|
||||
safe_tokens: typing.AbstractSet[str] | None = None,
|
||||
crlf_text: str | None = None,
|
||||
) -> ScanResult | None:
|
||||
if not route.inspect:
|
||||
return None
|
||||
# Lazy import to avoid circular deps and keep dlp_detectors optional
|
||||
# at import time (the gateway copies it flat alongside this file).
|
||||
try:
|
||||
from dlp_detectors import ( # type: ignore[import-not-found]
|
||||
scan_crlf_injection,
|
||||
scan_entropy,
|
||||
scan_known_secrets,
|
||||
scan_token_patterns,
|
||||
)
|
||||
except ImportError: # pragma: no cover - host-side path
|
||||
from .dlp_detectors import ( # type: ignore[import-not-found]
|
||||
scan_crlf_injection,
|
||||
scan_entropy,
|
||||
scan_known_secrets,
|
||||
scan_token_patterns,
|
||||
)
|
||||
|
||||
# Binary bodies: latin-1 is a bijective byte↔codepoint mapping that
|
||||
# preserves every byte value, so ASCII-range secret strings remain
|
||||
# findable by str.find / regex. Prefer strict UTF-8 for valid text bodies.
|
||||
if isinstance(body, bytes):
|
||||
try:
|
||||
text = body.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
text = body.decode("latin-1")
|
||||
else:
|
||||
text = body
|
||||
|
||||
# CRLF injection is only an attack in the request line + headers, never the
|
||||
# body: an HTTP body is delimited by Content-Length, so CRLF bytes there
|
||||
# cannot split the request. Scanning the body produces false positives on
|
||||
# legitimate form-encoded / multi-line content. Callers pass the
|
||||
# body-excluded surfaces as `crlf_text`; `None` falls back to the full text
|
||||
# for backward-compatible callers (host-side tests, websocket frames).
|
||||
crlf_target = text if crlf_text is None else crlf_text
|
||||
result = scan_crlf_injection(crlf_target)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
if _detector_enabled(route.outbound_detectors, "token_patterns"):
|
||||
result = scan_token_patterns(text, location="body", safe_tokens=safe_tokens)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
if _detector_enabled(route.outbound_detectors, "known_secrets"):
|
||||
# BOT_BOTTLE_SENSITIVE_PREFIXES lets operators add extra env prefixes
|
||||
# beyond EGRESS_TOKEN_* without changing the manifest schema.
|
||||
extra_raw = environ.get("BOT_BOTTLE_SENSITIVE_PREFIXES", "")
|
||||
extra = tuple(p for p in extra_raw.split(",") if p)
|
||||
sensitive_prefixes = ("EGRESS_TOKEN_",) + extra
|
||||
result = scan_known_secrets(
|
||||
text, location="body", env=environ,
|
||||
sensitive_prefixes=sensitive_prefixes, safe_tokens=safe_tokens,
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
# Entropy scanning requires explicit opt-in: it is NOT part of the
|
||||
# default "all detectors" set because it produces false positives on
|
||||
# legitimate base64 / binary payloads. Routes must list "entropy" in
|
||||
# dlp.outbound_detectors to enable it.
|
||||
if (
|
||||
route.outbound_detectors is not None
|
||||
and "entropy" in route.outbound_detectors
|
||||
):
|
||||
result = scan_entropy(text, location="body")
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def build_token_allow_payload(
|
||||
host: str,
|
||||
method: str,
|
||||
path: str,
|
||||
result: ScanResult,
|
||||
) -> str:
|
||||
"""Render the human-readable supervisor proposal body for an outbound
|
||||
token block (PRD 0062). Carries the host/method/path, the detector
|
||||
reason, and the redacted context snippet — never the raw token value."""
|
||||
lines = [
|
||||
"egress blocked an outbound request carrying a detected token",
|
||||
f"host: {host}",
|
||||
f"method: {method}",
|
||||
f"path: {path}",
|
||||
f"detector: {result.reason}",
|
||||
]
|
||||
if result.context:
|
||||
lines.append(f"context: {result.context}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def scan_inbound(
|
||||
route: Route,
|
||||
body: str | bytes,
|
||||
) -> ScanResult | None:
|
||||
if not route.inspect:
|
||||
return None
|
||||
try:
|
||||
from dlp_detectors import scan_naive_injection # type: ignore[import-not-found]
|
||||
except ImportError: # pragma: no cover - host-side path
|
||||
from .dlp_detectors import scan_naive_injection # type: ignore[import-not-found]
|
||||
|
||||
text = body if isinstance(body, str) else body.decode("utf-8", errors="replace")
|
||||
|
||||
if _detector_enabled(route.inbound_detectors, "naive_injection_detection"):
|
||||
result = scan_naive_injection(text)
|
||||
if result is not None:
|
||||
return result
|
||||
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LOG_BLOCKS",
|
||||
"route_to_yaml_dict",
|
||||
"LOG_FULL",
|
||||
"LOG_OFF",
|
||||
"ON_MATCH_BLOCK",
|
||||
"ON_MATCH_REDACT",
|
||||
"ON_MATCH_SUPERVISE",
|
||||
"OUTBOUND_ON_MATCH_VALUES",
|
||||
"DEFAULT_OUTBOUND_ON_MATCH",
|
||||
"OUTBOUND_DETECTOR_NAMES",
|
||||
"INBOUND_DETECTOR_NAMES",
|
||||
"parse_inspect_block",
|
||||
"Config",
|
||||
"Decision",
|
||||
"HeaderMatch",
|
||||
"MatchEntry",
|
||||
"PathMatch",
|
||||
"Route",
|
||||
"ScanResult",
|
||||
"build_inbound_scan_text",
|
||||
"build_outbound_scan_text",
|
||||
"build_token_allow_payload",
|
||||
"decide",
|
||||
"decide_git_fetch",
|
||||
"evaluate_matches",
|
||||
"is_git_push_request",
|
||||
"is_git_fetch_request",
|
||||
"load_config",
|
||||
"DENY_UNATTRIBUTED",
|
||||
"DENY_UNPARSEABLE",
|
||||
"DENY_RESOLVER_ERROR",
|
||||
"resolve_client_config",
|
||||
"resolve_client_context",
|
||||
"PolicyResolverLike",
|
||||
"ContextResolverLike",
|
||||
"match_route",
|
||||
"outbound_scan_headers",
|
||||
"parse_config",
|
||||
"parse_routes",
|
||||
"scan_inbound",
|
||||
"scan_outbound",
|
||||
]
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Inspection and DLP configuration parsing for egress routes.
|
||||
|
||||
A route's optional `inspect:` object names which outbound/inbound detectors run
|
||||
and what the proxy does when an outbound detector matches a token
|
||||
(`outbound_on_match`). This module owns parsing and validating that block,
|
||||
kept apart from the request-time scan/decision flow in `egress_addon_core`
|
||||
so each half reads top-to-bottom without scrolling past the other.
|
||||
|
||||
Stdlib-only; ships flat into the gateway image alongside
|
||||
`egress_addon_core.py` — see `Dockerfile.gateway`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
OUTBOUND_DETECTOR_NAMES = frozenset({"token_patterns", "known_secrets", "entropy"})
|
||||
INBOUND_DETECTOR_NAMES = frozenset({"naive_injection_detection"})
|
||||
|
||||
# Per-route policy for what the proxy does when an outbound DLP detector
|
||||
# matches a token (PRD 0062).
|
||||
ON_MATCH_BLOCK = "block" # hard 403, never overridable
|
||||
ON_MATCH_REDACT = "redact" # scrub the matched value, forward the request
|
||||
ON_MATCH_SUPERVISE = "supervise" # queue for operator approval, hold the request
|
||||
OUTBOUND_ON_MATCH_VALUES = (ON_MATCH_BLOCK, ON_MATCH_REDACT, ON_MATCH_SUPERVISE)
|
||||
# Unset resolves to supervise (fall back to block when supervise is not wired).
|
||||
DEFAULT_OUTBOUND_ON_MATCH = ON_MATCH_SUPERVISE
|
||||
|
||||
|
||||
def parse_inspect_block(
|
||||
idx: int,
|
||||
host: str,
|
||||
inspect: dict[str, object],
|
||||
) -> tuple[tuple[str, ...] | None, tuple[str, ...] | None, str]:
|
||||
"""Parse DLP settings from an inspected route."""
|
||||
label = f"route[{idx}] ({host})"
|
||||
dlp = inspect
|
||||
|
||||
def _parse_detector_field(
|
||||
field: str,
|
||||
valid_names: frozenset[str],
|
||||
) -> tuple[str, ...] | None:
|
||||
val = dlp.get(field)
|
||||
if val is None:
|
||||
return None
|
||||
if val is False:
|
||||
return ()
|
||||
if not isinstance(val, list):
|
||||
raise ValueError(
|
||||
f"{label}: inspect.{field} must be false, a list, or omitted"
|
||||
)
|
||||
items = typing.cast(list[object], val)
|
||||
names: list[str] = []
|
||||
for j, item in enumerate(items):
|
||||
if not isinstance(item, str):
|
||||
raise ValueError(
|
||||
f"{label}: inspect.{field}[{j}] must be a string"
|
||||
)
|
||||
if item not in valid_names:
|
||||
raise ValueError(
|
||||
f"{label}: inspect.{field}[{j}] {item!r} is not a valid "
|
||||
f"detector name; valid names: {', '.join(sorted(valid_names))}"
|
||||
)
|
||||
names.append(item)
|
||||
return tuple(names)
|
||||
|
||||
outbound = _parse_detector_field("outbound_detectors", OUTBOUND_DETECTOR_NAMES)
|
||||
inbound = _parse_detector_field("inbound_detectors", INBOUND_DETECTOR_NAMES)
|
||||
|
||||
on_match = ""
|
||||
on_match_raw = dlp.get("outbound_on_match")
|
||||
if on_match_raw is not None:
|
||||
if not isinstance(on_match_raw, str) or on_match_raw not in OUTBOUND_ON_MATCH_VALUES:
|
||||
raise ValueError(
|
||||
f"{label}: inspect.outbound_on_match must be one of "
|
||||
f"{', '.join(OUTBOUND_ON_MATCH_VALUES)} (got {on_match_raw!r})"
|
||||
)
|
||||
on_match = on_match_raw
|
||||
|
||||
return outbound, inbound, on_match
|
||||
@@ -0,0 +1,413 @@
|
||||
"""Gateway data-plane supervisor (PRD 0070; PRD 0024 bundle shape).
|
||||
|
||||
PID 1 inside the `bot-bottle-gateway` data-plane image. Spawns
|
||||
the configured daemons (egress, git-gate, supervise),
|
||||
forwards SIGTERM/SIGINT to each child, and propagates per-daemon
|
||||
stdout+stderr to the container log with a `[name] ` prefix.
|
||||
|
||||
Failure policy: when a child dies unexpectedly, the supervisor
|
||||
restarts it automatically and logs the restart. The gateway stays
|
||||
up; a temporary loss of one daemon (e.g. egress OOM-killed) is
|
||||
recovered without manual container recreation. The supervisor
|
||||
itself exits only when the operator sends SIGTERM/SIGINT.
|
||||
|
||||
Daemon subset is env-driven via `BOT_BOTTLE_GATEWAY_DAEMONS=egress`
|
||||
for callers that don't use git-gate or supervise. Default: all
|
||||
daemons.
|
||||
|
||||
Stdlib-only by design — adding supervisord/s6/runit for four
|
||||
daemons is heavier than this script.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import IO, Sequence
|
||||
|
||||
|
||||
# Below compose's default 10s `stop_grace_period`. After this many
|
||||
# seconds past SIGTERM, escalate to SIGKILL on any still-running
|
||||
# child.
|
||||
_GRACE_SECONDS = 8.0
|
||||
|
||||
# Tight enough that exits and signals propagate without lag; loose
|
||||
# enough that the main loop isn't a CPU hog.
|
||||
_POLL_INTERVAL = 0.1
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _DaemonSpec:
|
||||
name: str
|
||||
argv: Sequence[str]
|
||||
|
||||
|
||||
# Env-var name prefixes that carry egress-only credentials.
|
||||
# `egress_apply.py` assigns `EGRESS_TOKEN_<n>` slots that egress
|
||||
# reads to inject `Authorization` headers on configured routes;
|
||||
# no other daemon in the bundle should see these values.
|
||||
_EGRESS_ONLY_ENV_PREFIXES: tuple[str, ...] = ("EGRESS_TOKEN_",)
|
||||
_READY_GATED_DAEMONS: tuple[str, ...] = ("git-gate", "git-http")
|
||||
|
||||
# The control-plane signing key is the orchestrator's alone — verifying tokens.
|
||||
# The data-plane daemons instead hold the pre-minted `gateway` JWT they present.
|
||||
# Scoping each to its process (even in the combined infra container) keeps a
|
||||
# compromised data-plane daemon from reading the key and minting a `cli` token
|
||||
# (issue #469 review). Values match paths.CONTROL_PLANE_TOKEN_ENV /
|
||||
# CONTROL_AUTH_JWT_ENV; hardcoded here so this supervisor stays import-light.
|
||||
_SIGNING_KEY_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN"
|
||||
_GATEWAY_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT"
|
||||
|
||||
# Daemons that must be requested explicitly via BOT_BOTTLE_GATEWAY_DAEMONS
|
||||
# and are NOT started in the default (env-var-unset) case. The orchestrator
|
||||
# only runs in the combined infra container, never in a standalone gateway.
|
||||
_OPT_IN_DAEMONS: frozenset[str] = frozenset({"orchestrator"})
|
||||
|
||||
|
||||
def _env_for_daemon(name: str, base_env: dict[str, str]) -> dict[str, str]:
|
||||
"""Per-daemon env, scoped to what each process legitimately needs.
|
||||
Returns a fresh dict — callers can mutate without affecting `base_env`.
|
||||
|
||||
* `EGRESS_TOKEN_*` upstream-auth slots go to egress only.
|
||||
* the control-plane signing key goes to the orchestrator only.
|
||||
* the pre-minted `gateway` JWT goes to the data-plane daemons only (the
|
||||
orchestrator verifies tokens, it never presents one)."""
|
||||
env = dict(base_env)
|
||||
if name != "egress":
|
||||
env = {
|
||||
k: v for k, v in env.items()
|
||||
if not any(k.startswith(p) for p in _EGRESS_ONLY_ENV_PREFIXES)
|
||||
}
|
||||
if name == "orchestrator":
|
||||
env.pop(_GATEWAY_JWT_ENV, None)
|
||||
else:
|
||||
env.pop(_SIGNING_KEY_ENV, None)
|
||||
return env
|
||||
|
||||
|
||||
# The orchestrator is listed first so it starts before the gateway daemons,
|
||||
# giving the control plane a head start to accept /resolve calls. The gateway
|
||||
# daemons tolerate early /resolve failures and retry per-request.
|
||||
_DAEMONS: tuple[_DaemonSpec, ...] = (
|
||||
_DaemonSpec("orchestrator", (
|
||||
"python3", "-m", "bot_bottle.orchestrator",
|
||||
"--host", "0.0.0.0", "--port", "8099", "--broker", "stub",
|
||||
)),
|
||||
_DaemonSpec("egress", ("/bin/sh", "/app/egress-entrypoint.sh")),
|
||||
_DaemonSpec("git-gate", ("/bin/sh", "/git-gate-entrypoint.sh")),
|
||||
_DaemonSpec("git-http", ("python3", "-m", "bot_bottle.gateway.git_http_backend")),
|
||||
_DaemonSpec("supervise", ("python3", "-m", "bot_bottle.gateway.supervise_server")),
|
||||
)
|
||||
|
||||
|
||||
def _argv_for_daemon(name: str, argv: Sequence[str], env: dict[str, str]) -> list[str]:
|
||||
ready_file = env.get("BOT_BOTTLE_GIT_GATE_READY_FILE", "").strip()
|
||||
if name not in _READY_GATED_DAEMONS or not ready_file:
|
||||
return list(argv)
|
||||
return [
|
||||
"/bin/sh",
|
||||
"-c",
|
||||
"while [ ! -f \"$BOT_BOTTLE_GIT_GATE_READY_FILE\" ]; do "
|
||||
"sleep 0.1; "
|
||||
"done; "
|
||||
"exec \"$@\"",
|
||||
name,
|
||||
*argv,
|
||||
]
|
||||
|
||||
|
||||
def _selected_daemons(
|
||||
env: dict[str, str],
|
||||
all_daemons: Sequence[_DaemonSpec] | None = None,
|
||||
) -> tuple[_DaemonSpec, ...]:
|
||||
"""Filter the daemon set by the BOT_BOTTLE_GATEWAY_DAEMONS env var.
|
||||
|
||||
When the var is unset/empty, return all non-opt-in daemons (the
|
||||
standard gateway subset). Opt-in daemons (e.g. `orchestrator`) only
|
||||
run when explicitly named — they never start in a plain gateway
|
||||
container that doesn't set the env var. Unknown names are ignored.
|
||||
|
||||
`all_daemons` defaults to `_DAEMONS` resolved at call time (not at
|
||||
definition time), so tests can pass a custom list."""
|
||||
if all_daemons is None:
|
||||
all_daemons = _DAEMONS
|
||||
raw = env.get("BOT_BOTTLE_GATEWAY_DAEMONS", "").strip()
|
||||
if not raw:
|
||||
return tuple(d for d in all_daemons if d.name not in _OPT_IN_DAEMONS)
|
||||
wanted = {n.strip() for n in raw.split(",") if n.strip()}
|
||||
return tuple(d for d in all_daemons if d.name in wanted)
|
||||
|
||||
|
||||
def _log(msg: str) -> None:
|
||||
sys.stdout.write(f"gateway-init: {msg}\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def _pump(name: str, stream: IO[bytes]) -> None:
|
||||
"""Read lines from `stream`, prefix with `[name]`, write to
|
||||
stdout. Runs in its own thread per child; daemon=True so a
|
||||
blocked read doesn't keep the process alive after main exits."""
|
||||
for raw in iter(stream.readline, b""):
|
||||
line = raw.decode("utf-8", errors="replace").rstrip("\n")
|
||||
sys.stdout.write(f"[{name}] {line}\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]:
|
||||
env = _env_for_daemon(spec.name, dict(os.environ))
|
||||
proc = subprocess.Popen( # pylint: disable=consider-using-with
|
||||
_argv_for_daemon(spec.name, spec.argv, env),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
bufsize=0,
|
||||
env=env,
|
||||
)
|
||||
threading.Thread(
|
||||
target=_pump, args=(spec.name, proc.stdout), daemon=True
|
||||
).start()
|
||||
return proc
|
||||
|
||||
|
||||
class _Supervisor:
|
||||
"""Holds the running children + shutdown state. Pulled out so
|
||||
the test suite can drive it with fake commands."""
|
||||
|
||||
def __init__(self, specs: Sequence[_DaemonSpec]):
|
||||
self.specs = tuple(specs)
|
||||
self.procs: list[tuple[_DaemonSpec, subprocess.Popen[bytes]]] = []
|
||||
self.shutdown_at: float | None = None
|
||||
# Names of children that have been logged as having exited
|
||||
# so we only log each death once across watch-loop ticks.
|
||||
self._logged_dead: set[str] = set()
|
||||
# Signal handlers add daemon names here and return quickly.
|
||||
# The main watch loop drains the set, so repeated restart
|
||||
# requests for one daemon coalesce into one restart.
|
||||
self._restart_requested: set[str] = set()
|
||||
|
||||
def start_all(self) -> None:
|
||||
for spec in self.specs:
|
||||
_log(f"starting {spec.name}")
|
||||
self.procs.append((spec, _spawn(spec)))
|
||||
|
||||
def request_shutdown(self, reason: str) -> None:
|
||||
if self.shutdown_at is not None:
|
||||
return
|
||||
self.shutdown_at = time.monotonic()
|
||||
self._restart_requested.clear()
|
||||
_log(f"shutting down ({reason}); forwarding SIGTERM")
|
||||
for _, p in self.procs:
|
||||
if p.poll() is None:
|
||||
try:
|
||||
p.terminate()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
def _sigkill_all(self) -> None:
|
||||
for _, p in self.procs:
|
||||
if p.poll() is None:
|
||||
try:
|
||||
p.kill()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
def request_restart(self, daemon_name: str) -> bool:
|
||||
"""Queue a daemon restart for the main loop to process.
|
||||
|
||||
Signal handlers use this non-blocking path instead of doing
|
||||
subprocess lifecycle work directly. Requests coalesce by
|
||||
daemon name: one pending restart is enough to make the daemon
|
||||
reread the latest config from disk.
|
||||
|
||||
Returns True iff a daemon by that name is known to the
|
||||
supervisor and shutdown has not started."""
|
||||
if self.shutdown_at is not None:
|
||||
_log(f"restart {daemon_name} skipped; supervisor is shutting down")
|
||||
return False
|
||||
if not any(spec.name == daemon_name for spec, _ in self.procs):
|
||||
return False
|
||||
self._restart_requested.add(daemon_name)
|
||||
return True
|
||||
|
||||
def tick(self) -> bool:
|
||||
"""One iteration of the watch loop. Returns True when every
|
||||
child has exited and the supervisor can return.
|
||||
|
||||
A child dying unexpectedly is logged and restarted but does
|
||||
NOT initiate shutdown — see the module docstring's
|
||||
failure-policy section. Shutdown is signal-driven only."""
|
||||
restarted_children = bool(self._restart_requested)
|
||||
self._drain_restart_requests()
|
||||
|
||||
for spec, p in self.procs:
|
||||
rc = p.poll()
|
||||
if rc is None or spec.name in self._logged_dead:
|
||||
continue
|
||||
self._logged_dead.add(spec.name)
|
||||
if self.shutdown_at is None:
|
||||
_log(f"{spec.name} exited with code {rc}; scheduling restart")
|
||||
self._restart_requested.add(spec.name)
|
||||
else:
|
||||
_log(f"{spec.name} exited with code {rc}")
|
||||
|
||||
# Restart deaths discovered above before checking whether all
|
||||
# processes are done. Deferring this until the next tick would make a
|
||||
# single-daemon supervisor return True and exit with the restart still
|
||||
# queued.
|
||||
restarted_children |= bool(self._restart_requested)
|
||||
self._drain_restart_requests()
|
||||
|
||||
if self.shutdown_at is not None:
|
||||
elapsed = time.monotonic() - self.shutdown_at
|
||||
if elapsed > _GRACE_SECONDS:
|
||||
still_running = [
|
||||
spec.name for spec, p in self.procs if p.poll() is None
|
||||
]
|
||||
if still_running:
|
||||
_log(
|
||||
f"grace ({_GRACE_SECONDS:.0f}s) elapsed; SIGKILL on "
|
||||
f"{', '.join(still_running)}"
|
||||
)
|
||||
self._sigkill_all()
|
||||
|
||||
done = (
|
||||
not restarted_children
|
||||
and all(p.poll() is not None for _, p in self.procs)
|
||||
)
|
||||
if done:
|
||||
for _, p in self.procs:
|
||||
if p.stdout is not None:
|
||||
p.stdout.close()
|
||||
return done
|
||||
|
||||
def exit_code(self) -> int:
|
||||
"""Positive child failures win; otherwise report success.
|
||||
|
||||
Python represents signal-terminated children as negative
|
||||
return codes. A signal-only graceful shutdown should not leak
|
||||
that platform-specific detail into the container exit status,
|
||||
but a positive crash before shutdown should remain visible."""
|
||||
positives = [
|
||||
p.returncode for _, p in self.procs
|
||||
if p.returncode is not None and p.returncode > 0
|
||||
]
|
||||
return max(positives, default=0)
|
||||
|
||||
def _drain_restart_requests(self) -> None:
|
||||
if self.shutdown_at is not None:
|
||||
self._restart_requested.clear()
|
||||
return
|
||||
requested = tuple(sorted(self._restart_requested))
|
||||
self._restart_requested.clear()
|
||||
for daemon_name in requested:
|
||||
if self.shutdown_at is not None:
|
||||
self._restart_requested.clear()
|
||||
return
|
||||
self.restart_daemon(daemon_name)
|
||||
|
||||
def forward_signal(self, sig: int, daemon_name: str) -> bool:
|
||||
"""Forward a signal to one named child. Used by the SIGHUP
|
||||
path: egress_apply.py runs `docker kill --signal HUP
|
||||
<bundle>`, the host kernel delivers SIGHUP to PID 1 (this
|
||||
supervisor), and we relay it to mitmdump so it reloads
|
||||
its addon's routes.yaml. Returns True iff a live child by
|
||||
that name was signaled."""
|
||||
for spec, p in self.procs:
|
||||
if spec.name != daemon_name:
|
||||
continue
|
||||
if p.poll() is not None:
|
||||
_log(
|
||||
f"SIGHUP for {daemon_name} dropped; daemon "
|
||||
f"already exited (rc={p.returncode})"
|
||||
)
|
||||
return False
|
||||
try:
|
||||
p.send_signal(sig)
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
_log(f"forwarded {signal.Signals(sig).name} to {daemon_name}")
|
||||
return True
|
||||
return False
|
||||
|
||||
def restart_daemon(self, daemon_name: str, *, grace: float = 5.0) -> bool:
|
||||
"""Terminate one named child and spawn a fresh one, leaving
|
||||
the other daemons running. A daemon that has no in-process
|
||||
reload can be restarted this way after its config file changes.
|
||||
|
||||
Behavior: SIGTERM → wait up to `grace` seconds → SIGKILL if
|
||||
still alive → spawn a replacement under the same DaemonSpec.
|
||||
The `procs` slot is updated in place so subsequent
|
||||
forward_signal / shutdown calls reach the new pid.
|
||||
|
||||
Returns True iff a daemon by that name was running and a
|
||||
replacement spawned; False if no such daemon (not wired
|
||||
for this bottle)."""
|
||||
if self.shutdown_at is not None:
|
||||
_log(f"restart {daemon_name} skipped; supervisor is shutting down")
|
||||
return False
|
||||
for idx, (spec, p) in enumerate(self.procs):
|
||||
if spec.name != daemon_name:
|
||||
continue
|
||||
_log(f"restarting {daemon_name}")
|
||||
if p.poll() is None:
|
||||
try:
|
||||
p.terminate()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
try:
|
||||
p.wait(timeout=grace)
|
||||
except subprocess.TimeoutExpired:
|
||||
_log(
|
||||
f"{daemon_name} did not exit within {grace:.0f}s "
|
||||
f"of SIGTERM; SIGKILL"
|
||||
)
|
||||
try:
|
||||
p.kill()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
p.wait()
|
||||
if p.stdout is not None:
|
||||
p.stdout.close()
|
||||
self._logged_dead.discard(daemon_name)
|
||||
new_proc = _spawn(spec)
|
||||
self.procs[idx] = (spec, new_proc)
|
||||
_log(f"{daemon_name} restarted (pid {new_proc.pid})")
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
del argv # no flags yet; env-driven only
|
||||
specs = _selected_daemons(dict(os.environ))
|
||||
if not specs:
|
||||
_log("no daemons selected; nothing to do")
|
||||
return 0
|
||||
|
||||
sup = _Supervisor(specs)
|
||||
sup.start_all()
|
||||
|
||||
signal.signal(signal.SIGTERM, lambda *_: sup.request_shutdown("SIGTERM")) # type: ignore
|
||||
signal.signal(signal.SIGINT, lambda *_: sup.request_shutdown("SIGINT")) # type: ignore
|
||||
# SIGHUP reload path: egress_apply.py runs `docker kill
|
||||
# --signal HUP <bundle>` after writing routes.yaml. The kernel
|
||||
# delivers SIGHUP to PID 1 (this supervisor); forward it to
|
||||
# mitmdump so it reloads its addon.
|
||||
signal.signal(
|
||||
signal.SIGHUP,
|
||||
lambda *_: sup.forward_signal(signal.SIGHUP, "egress"), # type: ignore[misc]
|
||||
)
|
||||
|
||||
while not sup.tick():
|
||||
time.sleep(_POLL_INTERVAL)
|
||||
|
||||
rc = sup.exit_code()
|
||||
_log(f"exit {rc}")
|
||||
return rc
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,562 @@
|
||||
"""Pure host-side rendering for the per-agent git-gate (PRD 0008).
|
||||
|
||||
Builds the agent's `.gitconfig` insteadOf rewrites, the known_hosts
|
||||
line, and the entrypoint / pre-receive / access-hook scripts the gateway
|
||||
runs. No docker or forge calls — exposed for tests and reuse across
|
||||
backends. Split out of `git_gate.py` so the control surface (`GitGate`)
|
||||
and the deploy-key lifecycle (`git_gate_provision`) each read on their
|
||||
own; `git_gate` re-exports these names for API stability."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import shlex
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ..constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER
|
||||
from ..manifest import ManifestBottle, ManifestGitEntry
|
||||
|
||||
# Short network alias for git-gate inside the gateway. The
|
||||
# agent's `.gitconfig` insteadOf rewrites resolve through this name.
|
||||
GIT_GATE_HOSTNAME = "git-gate"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GitGateUpstream:
|
||||
"""One bare repo on the gate. `name` drives the bare-repo path
|
||||
(`/git/<name>.git`), the agent's URL after insteadOf rewrite
|
||||
(`git://<gate>/<name>.git`), and the per-upstream credential
|
||||
paths inside the gate (`/git-gate/creds/<name>-key` and
|
||||
`/git-gate/creds/<name>-known_hosts`).
|
||||
|
||||
`identity_file` is the host-side absolute path the gate's start
|
||||
step will docker-cp into the container. `known_host_key` is the
|
||||
KnownHostKey string from the manifest; the gate's start step
|
||||
materialises it into a known_hosts file if non-empty.
|
||||
|
||||
the gate credential paths inside the running gateway."""
|
||||
|
||||
name: str
|
||||
upstream_url: str
|
||||
upstream_host: str
|
||||
upstream_port: str
|
||||
identity_file: str
|
||||
known_host_key: str
|
||||
known_hosts_file: Path = Path()
|
||||
|
||||
def git_gate_upstreams_for_bottle(bottle: ManifestBottle) -> tuple[GitGateUpstream, ...]:
|
||||
"""Lift each `bottle.git` entry into a GitGateUpstream. Unique-Name
|
||||
validation already ran in `manifest.ManifestBottle.from_dict`."""
|
||||
return tuple(
|
||||
GitGateUpstream(
|
||||
name=e.Name,
|
||||
upstream_url=e.Upstream,
|
||||
upstream_host=e.UpstreamHost,
|
||||
upstream_port=e.UpstreamPort,
|
||||
identity_file=e.IdentityFile,
|
||||
known_host_key=e.KnownHostKey,
|
||||
)
|
||||
for e in bottle.git
|
||||
)
|
||||
|
||||
|
||||
def _gitconfig_validate_value(field: str, value: str) -> None:
|
||||
"""Raise ValueError if value contains characters that break gitconfig line syntax."""
|
||||
if "\n" in value or "\r" in value:
|
||||
raise ValueError(
|
||||
f"git-gate: {field} contains a newline, which would inject "
|
||||
f"arbitrary gitconfig keys; rejecting manifest entry"
|
||||
)
|
||||
|
||||
|
||||
def git_gate_render_gitconfig(
|
||||
entries: tuple[ManifestGitEntry, ...], gate_host: str, *, scheme: str = "git",
|
||||
identity_token: str = "",
|
||||
) -> str:
|
||||
"""Render the agent's ~/.gitconfig content for git-gate
|
||||
`insteadOf` rewrites. Pure host-side, no docker / VM;
|
||||
exposed for tests + reuse across backends.
|
||||
|
||||
`gate_host` is the part of the URL between `<scheme>://` and the
|
||||
repo path — backends differ here:
|
||||
- docker: `git-gate` (the short network alias)
|
||||
- firecracker: `<bundle_ip>:<port>` (no DNS on the
|
||||
point-to-point TAP link)
|
||||
|
||||
Empty `entries` returns an empty string so callers can no-op
|
||||
cleanly without conditional formatting at the call site."""
|
||||
if not entries:
|
||||
return ""
|
||||
out = [
|
||||
"# bot-bottle git-gate (PRD 0008): every git operation against\n",
|
||||
"# a declared upstream routes through the gate, which mirrors\n",
|
||||
"# the upstream bidirectionally (gitleaks-scanned push;\n",
|
||||
"# fetch-from-upstream-before-every-upload-pack via access-hook).\n",
|
||||
]
|
||||
# Over the smart-HTTP transport (VM backends), attach the per-bottle
|
||||
# identity token as a request header on requests to the gate, scoped to
|
||||
# its URL so it never goes to any other remote. git-http requires it (the
|
||||
# gateway's mandatory (source_ip, token) attribution). git:// (single-tenant
|
||||
# docker) carries no header — attribution there is the network alias.
|
||||
if identity_token and scheme == "http":
|
||||
_gitconfig_validate_value("identity_token", identity_token)
|
||||
out.append(f'[http "http://{gate_host}/"]\n')
|
||||
out.append(f"\textraHeader = {IDENTITY_HEADER}: {identity_token}\n")
|
||||
for entry in entries:
|
||||
_gitconfig_validate_value(f"repos[{entry.Name!r}].url", entry.Upstream)
|
||||
out.append(f'[url "{scheme}://{gate_host}/{entry.Name}.git"]\n')
|
||||
out.append(f"\tinsteadOf = {entry.Upstream}\n")
|
||||
if entry.RemoteKey and entry.RemoteKey != entry.UpstreamHost:
|
||||
port = (
|
||||
f":{entry.UpstreamPort}"
|
||||
if entry.UpstreamPort and entry.UpstreamPort != "22"
|
||||
else ""
|
||||
)
|
||||
alias = (
|
||||
f"ssh://{entry.UpstreamUser}@{entry.RemoteKey}{port}/"
|
||||
f"{entry.UpstreamPath}"
|
||||
)
|
||||
_gitconfig_validate_value(f"repos[{entry.Name!r}].url (resolved alias)", alias)
|
||||
out.append(f"\tinsteadOf = {alias}\n")
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def git_gate_known_hosts_line(host: str, port: str, key: str) -> str:
|
||||
"""Format `host[:port] key` for OpenSSH's known_hosts. Non-default
|
||||
ports use the bracketed `[host]:port` form (the form OpenSSH writes
|
||||
on disk for hosts reached via a non-22 port)."""
|
||||
if port and port != "22":
|
||||
target = f"[{host}]:{port}"
|
||||
else:
|
||||
target = host
|
||||
return f"{target} {key}\n"
|
||||
|
||||
|
||||
def _git_gate_init_repo_fn(repo_root: str, creds_dir: str) -> list[str]:
|
||||
"""The `init_repo` shell function, parameterized by the bare-repo root
|
||||
and the per-bottle creds dir. Single source of the credential-wiring
|
||||
logic, shared by the single-tenant daemon entrypoint (`/git`,
|
||||
`/git-gate/creds`) and the consolidated per-bottle provisioning
|
||||
(`/git/<bottle_id>`, `/git-gate/creds/<bottle_id>`)."""
|
||||
return [
|
||||
"init_repo() {",
|
||||
" name=$1",
|
||||
" upstream_url=$2",
|
||||
f" keyfile={creds_dir}/${{name}}-key",
|
||||
f" hostsfile={creds_dir}/${{name}}-known_hosts",
|
||||
"",
|
||||
# `|| true`: PRD 0018 chunk 3+ bind-mounts these RO from the
|
||||
# host, so chmod-syscalls fail with EROFS. The files already
|
||||
# have the right perms on the host (SSH requires 0600 to load
|
||||
# the key in the first place), so the chmod is best-effort
|
||||
# cleanup for the legacy docker-cp path where the file
|
||||
# landed at the host's umask perms.
|
||||
" chmod 600 \"$keyfile\" 2>/dev/null || true",
|
||||
" if [ -f \"$hostsfile\" ]; then",
|
||||
" chmod 600 \"$hostsfile\" 2>/dev/null || true",
|
||||
" fi",
|
||||
"",
|
||||
f" repo={repo_root}/${{name}}.git",
|
||||
" if [ ! -d \"$repo\" ]; then",
|
||||
" git init --bare \"$repo\" >/dev/null",
|
||||
# --mirror=fetch sets remote.origin.fetch = +refs/*:refs/* so a later
|
||||
# `git fetch origin` mirrors the upstream's full ref graph (heads,
|
||||
# tags, notes) into the bare repo at canonical paths. It does NOT set
|
||||
# remote.origin.mirror=true, so an explicit `git push origin
|
||||
# <ref>:<ref>` still pushes one ref.
|
||||
" git -C \"$repo\" remote add --mirror=fetch origin \"$upstream_url\"",
|
||||
" fi",
|
||||
" git -C \"$repo\" config git-gate.identityFile \"$keyfile\"",
|
||||
" git -C \"$repo\" config git-gate.knownHosts \"$hostsfile\"",
|
||||
" git -C \"$repo\" config receive.denyCurrentBranch ignore",
|
||||
" git -C \"$repo\" config receive.advertisePushOptions true",
|
||||
" git -C \"$repo\" config http.receivepack true",
|
||||
" install -m 755 /etc/git-gate/pre-receive \"$repo/hooks/pre-receive\"",
|
||||
"}",
|
||||
]
|
||||
|
||||
|
||||
def git_gate_render_entrypoint(upstreams: tuple[GitGateUpstream, ...]) -> str:
|
||||
"""Posix-sh entrypoint. One `init_repo` call per upstream, then
|
||||
`exec git daemon`. The function reads
|
||||
`/git-gate/creds/<name>-{key,known_hosts}` (bind-mounted into
|
||||
the bundle by the renderer) and wires them into each bare repo's
|
||||
config; the access-hook + pre-receive hook pick those paths up
|
||||
at fetch / push time."""
|
||||
lines = ["#!/bin/sh", "set -eu", ""]
|
||||
lines += _git_gate_init_repo_fn("/git", "/git-gate/creds")
|
||||
lines += ["", "mkdir -p /git"]
|
||||
for u in upstreams:
|
||||
lines.append(f"init_repo {shlex.quote(u.name)} {shlex.quote(u.upstream_url)}")
|
||||
lines.extend([
|
||||
"",
|
||||
"exec git daemon \\",
|
||||
" --reuseaddr \\",
|
||||
f" --timeout={GIT_GATE_TIMEOUT_SECS} \\",
|
||||
f" --init-timeout={GIT_GATE_TIMEOUT_SECS} \\",
|
||||
" --base-path=/git \\",
|
||||
" --export-all \\",
|
||||
" --enable=receive-pack \\",
|
||||
" --access-hook=/etc/git-gate/access-hook \\",
|
||||
" --verbose",
|
||||
])
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# A bottle id namespaces the consolidated gateway's repo + creds dirs; it is
|
||||
# embedded unquoted in the provisioning script, so restrict it to a shell- and
|
||||
# path-safe alphabet (registry ids are token_hex — this is defense in depth).
|
||||
_SAFE_BOTTLE_ID = re.compile(r"[A-Za-z0-9_-]+")
|
||||
|
||||
|
||||
def git_gate_render_provision(
|
||||
bottle_id: str, upstreams: tuple[GitGateUpstream, ...],
|
||||
) -> str:
|
||||
"""Posix-sh script that provisions ONE bottle's bare repos into the
|
||||
consolidated gateway (PRD 0070), under `/git/<bottle_id>/` with creds
|
||||
read from `/git-gate/creds/<bottle_id>/`. Init-only — no `git daemon`,
|
||||
since the shared gateway already serves every bottle; run inside the
|
||||
running gateway when the bottle is registered.
|
||||
|
||||
Isolating each bottle's repo root and creds dir by id is what keeps one
|
||||
bottle's push credentials out of another's repos on the shared gateway."""
|
||||
if not _SAFE_BOTTLE_ID.fullmatch(bottle_id):
|
||||
raise ValueError(f"git-gate: unsafe bottle id {bottle_id!r}")
|
||||
repo_root = f"/git/{bottle_id}"
|
||||
creds_dir = f"/git-gate/creds/{bottle_id}"
|
||||
lines = ["#!/bin/sh", "set -eu", ""]
|
||||
lines += _git_gate_init_repo_fn(repo_root, creds_dir)
|
||||
lines += ["", f"mkdir -p {shlex.quote(repo_root)}"]
|
||||
for u in upstreams:
|
||||
lines.append(f"init_repo {shlex.quote(u.name)} {shlex.quote(u.upstream_url)}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def git_gate_render_hook() -> str:
|
||||
"""The shared pre-receive hook: gitleaks-scan all incoming refs,
|
||||
then forward each accepted ref to the real upstream (`origin`)
|
||||
using the per-repo credential. Failure in either phase aborts
|
||||
the push so the agent sees a real rejection. POSIX sh.
|
||||
|
||||
Two phases (scan all, then push all) keeps a hit on ref N from
|
||||
half-pushing refs 1..N-1; both phases re-read stdin from a temp
|
||||
file because pre-receive's stdin is a one-shot stream."""
|
||||
return r"""#!/bin/sh
|
||||
# git-gate pre-receive (PRD 0008). Stdin: <old> <new> <ref> per line.
|
||||
set -u
|
||||
|
||||
refs_file=$(mktemp)
|
||||
trap 'rm -f "$refs_file"' EXIT
|
||||
cat > "$refs_file"
|
||||
|
||||
zero=0000000000000000000000000000000000000000
|
||||
|
||||
supervise_gitleaks_allow() {
|
||||
log_opts=$1
|
||||
ref=$2
|
||||
report_file=$(mktemp)
|
||||
if ! gitleaks git \
|
||||
--log-opts="$log_opts" \
|
||||
--no-banner \
|
||||
--redact \
|
||||
--ignore-gitleaks-allow \
|
||||
--report-format=json \
|
||||
--report-path="$report_file" \
|
||||
--exit-code 0 \
|
||||
1>&2; then
|
||||
rm -f "$report_file"
|
||||
echo "git-gate: gitleaks inline-suppression scan failed for $ref" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
proposal_id=$(
|
||||
PYTHONPATH="/app${PYTHONPATH:+:$PYTHONPATH}" GITLEAKS_ALLOW_REF="$ref" python3 - "$report_file" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Queue over the control-plane RPC — the git-gate data plane no longer opens
|
||||
# bot-bottle.db (PRD 0070 / issue #469). Attribution is by (source_ip,
|
||||
# identity_token), resolved server-side, so the proposal lands under the
|
||||
# calling bottle exactly as a direct write once did.
|
||||
try:
|
||||
from bot_bottle.gateway.policy_resolver import PolicyResolver, PolicyResolveError
|
||||
from bot_bottle.supervise_types import TOOL_GITLEAKS_ALLOW
|
||||
except ImportError:
|
||||
from policy_resolver import PolicyResolver, PolicyResolveError
|
||||
from supervise_types import TOOL_GITLEAKS_ALLOW
|
||||
|
||||
report_path = Path(sys.argv[1])
|
||||
source_ip = os.environ.get("SUPERVISE_SOURCE_IP", "")
|
||||
token = os.environ.get("SUPERVISE_IDENTITY_TOKEN", "")
|
||||
orch_url = os.environ.get("BOT_BOTTLE_ORCHESTRATOR_URL", "")
|
||||
if not source_ip or not orch_url:
|
||||
sys.exit(2)
|
||||
|
||||
try:
|
||||
raw = json.loads(report_path.read_text() or "[]")
|
||||
except json.JSONDecodeError:
|
||||
sys.exit(3)
|
||||
if not isinstance(raw, list):
|
||||
sys.exit(3)
|
||||
if not raw:
|
||||
sys.exit(0)
|
||||
|
||||
ref = os.environ.get("GITLEAKS_ALLOW_REF", "")
|
||||
lines = [
|
||||
"gitleaks inline suppression requires supervisor approval",
|
||||
f"ref: {ref}",
|
||||
"",
|
||||
]
|
||||
for i, finding in enumerate(raw, 1):
|
||||
if not isinstance(finding, dict):
|
||||
continue
|
||||
file_path = finding.get("File", "")
|
||||
line_no = finding.get("StartLine", finding.get("Line", ""))
|
||||
rule_id = finding.get("RuleID", "")
|
||||
commit = finding.get("Commit", "")
|
||||
line = finding.get("Line", "")
|
||||
lines.extend([
|
||||
f"finding {i}:",
|
||||
f" file: {file_path}",
|
||||
f" line: {line_no}",
|
||||
f" rule: {rule_id}",
|
||||
f" commit: {commit}",
|
||||
f" code: {line}",
|
||||
"",
|
||||
])
|
||||
|
||||
payload = "\n".join(lines).rstrip() + "\n"
|
||||
try:
|
||||
proposal_id = PolicyResolver(orch_url).propose_supervise(
|
||||
source_ip, token,
|
||||
tool=TOOL_GITLEAKS_ALLOW,
|
||||
proposed_file=payload,
|
||||
justification=(
|
||||
"git-gate found gitleaks findings hidden by # gitleaks:allow; "
|
||||
"approve only for dummy test fixtures or confirmed false positives"
|
||||
),
|
||||
)
|
||||
except PolicyResolveError:
|
||||
sys.exit(4)
|
||||
if not proposal_id:
|
||||
sys.exit(2)
|
||||
print(proposal_id)
|
||||
PY
|
||||
)
|
||||
rc=$?
|
||||
rm -f "$report_file"
|
||||
if [ "$rc" -eq 0 ] && [ -z "$proposal_id" ]; then
|
||||
return 0
|
||||
fi
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "git-gate: cannot route # gitleaks:allow finding to supervisor; refusing push" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
timeout=${SUPERVISE_GITLEAKS_ALLOW_TIMEOUT_SECONDS:-300}
|
||||
case "$timeout" in
|
||||
''|*[!0-9]*)
|
||||
echo "git-gate: invalid SUPERVISE_GITLEAKS_ALLOW_TIMEOUT_SECONDS=$timeout" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
echo "git-gate: queued # gitleaks:allow supervisor approval $proposal_id" >&2
|
||||
echo "git-gate: approve with './cli.py supervise' to continue this push" >&2
|
||||
waited=0
|
||||
while [ "$waited" -lt "$timeout" ]; do
|
||||
status=$(PYTHONPATH="/app${PYTHONPATH:+:$PYTHONPATH}" python3 - "$proposal_id" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Non-blocking poll over the control plane. A decided proposal is archived
|
||||
# server-side on read, so no separate archive step is needed here.
|
||||
try:
|
||||
from bot_bottle.gateway.policy_resolver import PolicyResolver, PolicyResolveError
|
||||
except ImportError:
|
||||
from policy_resolver import PolicyResolver, PolicyResolveError
|
||||
|
||||
source_ip = os.environ.get("SUPERVISE_SOURCE_IP", "")
|
||||
token = os.environ.get("SUPERVISE_IDENTITY_TOKEN", "")
|
||||
orch_url = os.environ.get("BOT_BOTTLE_ORCHESTRATOR_URL", "")
|
||||
try:
|
||||
result = PolicyResolver(orch_url).poll_supervise(source_ip, token, sys.argv[1])
|
||||
except PolicyResolveError:
|
||||
sys.exit(2)
|
||||
if result is None:
|
||||
sys.exit(2)
|
||||
status = result.get("status", "")
|
||||
if status in ("pending", "unknown"):
|
||||
sys.exit(2)
|
||||
print(status)
|
||||
PY
|
||||
)
|
||||
rc=$?
|
||||
if [ "$rc" -eq 2 ]; then
|
||||
status=""
|
||||
elif [ "$rc" -ne 0 ]; then
|
||||
status="invalid"
|
||||
fi
|
||||
if [ -n "$status" ]; then
|
||||
case "$status" in
|
||||
approved|modified)
|
||||
echo "git-gate: supervisor approved # gitleaks:allow for $ref" >&2
|
||||
return 0
|
||||
;;
|
||||
rejected)
|
||||
echo "git-gate: supervisor rejected # gitleaks:allow for $ref" >&2
|
||||
return 1
|
||||
;;
|
||||
*)
|
||||
echo "git-gate: invalid supervisor response for # gitleaks:allow" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
echo "git-gate: supervisor approval timed out for # gitleaks:allow; refusing push" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# Phase 1: gitleaks scan each ref's incoming commits.
|
||||
while IFS=' ' read -r old new ref; do
|
||||
[ -z "$ref" ] && continue
|
||||
[ "$new" = "$zero" ] && continue
|
||||
# Scan only the commits this push introduces — those reachable from
|
||||
# $new but not from any ref the gate already has. Everything already
|
||||
# on the gate arrived via upstream mirror-fetch or a previously
|
||||
# gitleaks-scanned push, so it's already-upstream or already-scanned;
|
||||
# re-scanning it only resurfaces historical fixture findings.
|
||||
#
|
||||
# Applies to both new refs and updates. The old existing-branch range
|
||||
# `$old..$new` walks commits reachable from the new tip but not the
|
||||
# *old branch tip*: on a rebase/force-push onto a freshly-advanced
|
||||
# main that pulls in all of main's new history (incl. the deliberate
|
||||
# sandbox-escape gitleaks fixtures), blocking the push. `--not --all`
|
||||
# excludes anything already on the gate regardless of ancestry, so it
|
||||
# is also correct for non-fast-forward pushes (a rebase can skip
|
||||
# commits off the direct path). Security-equivalent per PRD 0028's
|
||||
# analysis: the bare repo's refs come only from trusted upstream
|
||||
# mirror-fetch or gitleaks-gated pushes.
|
||||
# See PRD 0028 (open question) / issues #106, #346.
|
||||
log_opts="$new --not --all"
|
||||
echo "git-gate: gitleaks scanning $ref ($log_opts)" >&2
|
||||
if ! gitleaks git --log-opts="$log_opts" --no-banner --redact 1>&2; then
|
||||
echo "git-gate: gitleaks rejected push to $ref" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! supervise_gitleaks_allow "$log_opts" "$ref"; then
|
||||
exit 1
|
||||
fi
|
||||
done < "$refs_file"
|
||||
|
||||
# Phase 2: forward each ref to the upstream (`origin`, configured
|
||||
# in the entrypoint via `git remote add --mirror=fetch`).
|
||||
keyfile=$(git config --get git-gate.identityFile)
|
||||
hostsfile=$(git config --get git-gate.knownHosts)
|
||||
if [ ! -f "$hostsfile" ]; then
|
||||
echo "git-gate: no KnownHostKey configured for this upstream; refusing to push" >&2
|
||||
echo "git-gate: add KnownHostKey to the bottle.git entry and restart the bottle" >&2
|
||||
exit 1
|
||||
fi
|
||||
ssh_cmd="ssh -i $keyfile -o UserKnownHostsFile=$hostsfile -o StrictHostKeyChecking=yes -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=10"
|
||||
|
||||
push_option_count=${GIT_PUSH_OPTION_COUNT:-0}
|
||||
case "$push_option_count" in
|
||||
''|*[!0-9]*)
|
||||
echo "git-gate: invalid GIT_PUSH_OPTION_COUNT=$push_option_count" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
set --
|
||||
i=0
|
||||
while [ "$i" -lt "$push_option_count" ]; do
|
||||
opt=$(printenv "GIT_PUSH_OPTION_$i" || :)
|
||||
set -- "$@" --push-option="$opt"
|
||||
i=$((i + 1))
|
||||
done
|
||||
|
||||
while IFS=' ' read -r old new ref; do
|
||||
[ -z "$ref" ] && continue
|
||||
if [ "$new" = "$zero" ]; then
|
||||
refspec=":$ref"
|
||||
elif [ "$old" != "$zero" ] && ! git merge-base --is-ancestor "$old" "$new" 2>/dev/null; then
|
||||
refspec="+$new:$ref"
|
||||
else
|
||||
refspec="$new:$ref"
|
||||
fi
|
||||
echo "git-gate: forwarding $ref to origin" >&2
|
||||
if ! GIT_SSH_COMMAND="$ssh_cmd" git push "$@" origin "$refspec" 1>&2; then
|
||||
echo "git-gate: upstream push failed for $ref" >&2
|
||||
exit 1
|
||||
fi
|
||||
done < "$refs_file"
|
||||
|
||||
exit 0
|
||||
"""
|
||||
|
||||
|
||||
def git_gate_render_access_hook() -> str:
|
||||
"""`git daemon --access-hook` script. Runs before each protocol
|
||||
service; for `upload-pack` (fetch / clone / ls-remote / pull) it
|
||||
refreshes the bare repo from upstream first, so the response
|
||||
reflects upstream's current state. For other services (notably
|
||||
`receive-pack`) it returns 0 immediately and lets the existing
|
||||
pre-receive hook gate the operation. POSIX sh.
|
||||
|
||||
The hook receives:
|
||||
$1 service name (`upload-pack`, `receive-pack`, ...)
|
||||
$2 absolute path to the resolved repo
|
||||
$3 client hostname (unused)
|
||||
$4 client tcp address (unused)
|
||||
|
||||
Fail-closed on upstream errors: the agent's fetch fails too,
|
||||
so it never silently sees stale data — matches the PRD's
|
||||
'equivalent to operations against the upstream' contract."""
|
||||
return r"""#!/bin/sh
|
||||
# git-gate access-hook (PRD 0008). $1=service $2=repo $3=host $4=peer
|
||||
set -u
|
||||
service=$1
|
||||
repo_dir=$2
|
||||
|
||||
# Push path keeps its own gating in pre-receive (gitleaks +
|
||||
# forward). Only refresh-from-upstream on fetch operations.
|
||||
if [ "$service" != "upload-pack" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
keyfile=$(git -C "$repo_dir" config --get git-gate.identityFile 2>/dev/null || true)
|
||||
hostsfile=$(git -C "$repo_dir" config --get git-gate.knownHosts 2>/dev/null || true)
|
||||
if [ -z "$keyfile" ] || [ ! -f "$hostsfile" ]; then
|
||||
echo "git-gate: missing credentials for $repo_dir; refusing fetch" >&2
|
||||
exit 1
|
||||
fi
|
||||
ssh_cmd="ssh -i $keyfile -o UserKnownHostsFile=$hostsfile -o StrictHostKeyChecking=yes -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=10"
|
||||
|
||||
echo "git-gate: refreshing $repo_dir from upstream" >&2
|
||||
if ! GIT_SSH_COMMAND="$ssh_cmd" git -C "$repo_dir" fetch origin --prune >&2; then
|
||||
echo "git-gate: upstream fetch failed for $repo_dir; refusing to serve stale data" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Sync the bare repo's HEAD to upstream's HEAD on the first fetch
|
||||
# (when it still points at the `git init --bare` default of
|
||||
# refs/heads/master and upstream uses something else, the cloned
|
||||
# checkout would fail with "remote HEAD refers to nonexistent ref").
|
||||
# Costs one extra ls-remote on first fetch only; subsequent fetches
|
||||
# skip the branch. If upstream's default branch changes after the
|
||||
# gate has cached it, restart the bottle to resync.
|
||||
if ! git -C "$repo_dir" rev-parse --verify HEAD >/dev/null 2>&1; then
|
||||
upstream_head=$(GIT_SSH_COMMAND="$ssh_cmd" git -C "$repo_dir" \
|
||||
ls-remote --symref origin HEAD 2>/dev/null \
|
||||
| awk '/^ref:/ {print $2; exit}')
|
||||
if [ -n "$upstream_head" ]; then
|
||||
git -C "$repo_dir" symbolic-ref HEAD "$upstream_head" || true
|
||||
fi
|
||||
fi
|
||||
exit 0
|
||||
"""
|
||||
@@ -0,0 +1,287 @@
|
||||
"""Tiny smart-HTTP wrapper for git-gate repos.
|
||||
|
||||
Used where `git://` push traffic over a host-published Docker port can
|
||||
hang before receive-pack reaches hooks (e.g. the firecracker backend,
|
||||
where the guest reaches the gateway over the point-to-point TAP). The
|
||||
wrapper serves the same `/git/*.git` bare repos through
|
||||
`git http-backend`, so pre-receive and upstream forwarding remain the
|
||||
git-gate enforcement point.
|
||||
|
||||
One shared gateway serves every bottle (PRD 0070): each request is served
|
||||
from the calling bottle's repo namespace (`<root>/<bottle_id>`), attributed
|
||||
from the unspoofable source IP via the orchestrator. Per-repo credentials +
|
||||
hooks scope by repo directory, so isolating the *root* per bottle isolates
|
||||
its creds too. Unattributed clients — and a missing/unreachable orchestrator
|
||||
— fail closed (404). `BOT_BOTTLE_ORCHESTRATOR_URL` is mandatory: there is no
|
||||
single-tenant flat-root fallback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import typing
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from bot_bottle.constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER
|
||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||
|
||||
|
||||
DEFAULT_PORT = 9420
|
||||
|
||||
# The per-host orchestrator control plane the backend attributes each request
|
||||
# to, serving from the *calling* bottle's repo namespace selected by source IP.
|
||||
# Mandatory — the same env the egress addon requires; there is no single flat
|
||||
# repo-root fallback.
|
||||
ORCHESTRATOR_URL_ENV = "BOT_BOTTLE_ORCHESTRATOR_URL"
|
||||
|
||||
# The base under which each bottle's `<bottle_id>` repo namespace is nested.
|
||||
DEFAULT_REPO_ROOT = "/git"
|
||||
|
||||
|
||||
class ResolverLike(typing.Protocol):
|
||||
"""Structural type for the resolver `resolve_sandbox_root` needs — just
|
||||
`resolve_bottle_id`. A Protocol so tests can pass a fake without
|
||||
importing PolicyResolver (mirrors egress_addon_core.PolicyResolverLike)."""
|
||||
|
||||
def resolve_bottle_id(
|
||||
self, source_ip: str, identity_token: str = ...,
|
||||
) -> "str | None": ...
|
||||
|
||||
|
||||
def resolve_sandbox_root(
|
||||
resolver: "ResolverLike",
|
||||
base_root: Path,
|
||||
source_ip: str,
|
||||
identity_token: str = "",
|
||||
) -> Path | None:
|
||||
"""The per-sandbox repo root to serve this request from — `base_root/
|
||||
<bottle_id>`, where the sandbox is attributed from the source IP via the
|
||||
orchestrator — or None to deny (404). Fail-closed: an unattributed client, a
|
||||
resolver error, or a namespace that would escape `base_root` all deny, so one
|
||||
sandbox can never reach another's repos."""
|
||||
try:
|
||||
bottle_id = resolver.resolve_bottle_id(source_ip, identity_token)
|
||||
except PolicyResolveError:
|
||||
return None # orchestrator unreachable/errored → deny
|
||||
if not bottle_id:
|
||||
return None # unattributed → deny
|
||||
base = base_root.resolve()
|
||||
namespace = (base / bottle_id).resolve()
|
||||
if base not in namespace.parents:
|
||||
return None # bottle_id tried to escape the root → deny
|
||||
return namespace
|
||||
|
||||
# Bound memory use while still allowing ordinary git push packfiles.
|
||||
MAX_BODY_BYTES = 100 * 1024 * 1024
|
||||
|
||||
|
||||
class GitHttpHandler(BaseHTTPRequestHandler):
|
||||
server_version = "bot-bottle-git-http/1"
|
||||
|
||||
def do_GET(self) -> None:
|
||||
self._run_backend()
|
||||
|
||||
def do_POST(self) -> None:
|
||||
self._run_backend()
|
||||
|
||||
def _sandbox_root(self) -> Path | None:
|
||||
"""This request's per-sandbox repo root (the calling bottle's source-IP-
|
||||
selected `<base>/<bottle_id>` namespace), or None to deny. `GIT_PROJECT_
|
||||
ROOT` keeps git's own env-var name."""
|
||||
base = Path(os.environ.get("GIT_PROJECT_ROOT", DEFAULT_REPO_ROOT))
|
||||
resolver = getattr(self.server, "policy_resolver", None)
|
||||
if resolver is None:
|
||||
return None # server started without a resolver (misconfig) → deny
|
||||
token = self.headers.get(IDENTITY_HEADER, "")
|
||||
return resolve_sandbox_root(resolver, base, self.client_address[0], token)
|
||||
|
||||
def _run_backend(self) -> None:
|
||||
sandbox_root = self._sandbox_root()
|
||||
if sandbox_root is None:
|
||||
# Unattributed / resolver error: deny before touching any repo.
|
||||
self.send_error(404)
|
||||
return
|
||||
parsed = urlsplit(self.path)
|
||||
if self._is_upload_pack(parsed.path, parsed.query):
|
||||
repo_dir = self._repo_dir(sandbox_root, parsed.path)
|
||||
if repo_dir is None:
|
||||
self.send_error(404)
|
||||
return
|
||||
hook_path = os.environ.get(
|
||||
"GIT_GATE_ACCESS_HOOK", "/etc/git-gate/access-hook",
|
||||
)
|
||||
peer = self.client_address[0]
|
||||
try:
|
||||
hook = subprocess.run(
|
||||
[hook_path, "upload-pack", str(repo_dir), peer, peer],
|
||||
capture_output=True,
|
||||
check=False,
|
||||
timeout=GIT_GATE_TIMEOUT_SECS,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
# The access-hook couldn't be run (missing, not executable,
|
||||
# timed out, …). Fail closed with a real HTTP error rather
|
||||
# than letting the exception kill the handler thread — an
|
||||
# unhandled exception closes the socket with no response, which
|
||||
# the client sees as an opaque "empty reply from server".
|
||||
self.log_message(
|
||||
"access-hook could not run for %s: %s", parsed.path, exc,
|
||||
)
|
||||
self.send_error(503, "git-gate access-hook unavailable")
|
||||
return
|
||||
if hook.returncode != 0:
|
||||
detail = (hook.stderr or hook.stdout).decode(
|
||||
"utf-8", errors="replace",
|
||||
).rstrip()
|
||||
if detail:
|
||||
for line in detail.splitlines():
|
||||
self.log_message("access-hook denied %s: %s",
|
||||
parsed.path, line)
|
||||
else:
|
||||
self.log_message(
|
||||
"access-hook denied %s: exit=%d (no output)",
|
||||
parsed.path, hook.returncode,
|
||||
)
|
||||
self.send_response(403)
|
||||
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
||||
self.end_headers()
|
||||
self.wfile.write(hook.stderr or hook.stdout)
|
||||
return
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"GIT_PROJECT_ROOT": str(sandbox_root),
|
||||
"GIT_HTTP_EXPORT_ALL": "1",
|
||||
"REQUEST_METHOD": self.command,
|
||||
"PATH_INFO": parsed.path,
|
||||
"QUERY_STRING": parsed.query,
|
||||
"CONTENT_TYPE": self.headers.get("content-type", ""),
|
||||
"CONTENT_LENGTH": self.headers.get("content-length", "0"),
|
||||
"REMOTE_ADDR": self.client_address[0],
|
||||
"REMOTE_PORT": str(self.client_address[1]),
|
||||
"REMOTE_USER": "",
|
||||
"SERVER_NAME": self.server.server_name, # type: ignore
|
||||
"SERVER_PORT": str(self.server.server_port), # type: ignore
|
||||
"SERVER_PROTOCOL": self.request_version,
|
||||
})
|
||||
# Attribute the gitleaks-allow supervise proposal (queued by
|
||||
# receive-pack's pre-receive hook, a child of the CGI we spawn below) to
|
||||
# the calling bottle. The hook queues over the control-plane RPC (PRD
|
||||
# 0070 / issue #469), which re-resolves the bottle from these — the same
|
||||
# (source_ip, identity_token) pair that selected the namespaced root
|
||||
# above — so it can only ever queue its own proposals.
|
||||
env["SUPERVISE_SOURCE_IP"] = self.client_address[0]
|
||||
env["SUPERVISE_IDENTITY_TOKEN"] = self.headers.get(IDENTITY_HEADER, "")
|
||||
for header, variable in (
|
||||
("accept", "HTTP_ACCEPT"),
|
||||
("content-encoding", "HTTP_CONTENT_ENCODING"),
|
||||
("git-protocol", "HTTP_GIT_PROTOCOL"),
|
||||
("user-agent", "HTTP_USER_AGENT"),
|
||||
):
|
||||
value = self.headers.get(header)
|
||||
if value:
|
||||
env[variable] = value
|
||||
raw_length = self.headers.get("content-length", "0") or "0"
|
||||
try:
|
||||
length = int(raw_length)
|
||||
except ValueError:
|
||||
self.send_error(400, "Bad Content-Length")
|
||||
return
|
||||
if length < 0:
|
||||
self.send_error(400, "Negative Content-Length")
|
||||
return
|
||||
if length > MAX_BODY_BYTES:
|
||||
self.send_error(413, "Request body too large")
|
||||
return
|
||||
body = self.rfile.read(length) if length else b""
|
||||
proc = subprocess.run(
|
||||
["git", "http-backend"],
|
||||
input=body,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
timeout=GIT_GATE_TIMEOUT_SECS,
|
||||
)
|
||||
self._write_cgi_response(proc.stdout)
|
||||
|
||||
def _repo_dir(self, sandbox_root: Path, path: str) -> Path | None:
|
||||
root = sandbox_root.resolve()
|
||||
relative = path.lstrip("/").split(".git", 1)[0] + ".git"
|
||||
candidate = (root / relative).resolve()
|
||||
if root not in (candidate, *candidate.parents):
|
||||
return None
|
||||
if not candidate.is_dir():
|
||||
return None
|
||||
return candidate
|
||||
|
||||
@staticmethod
|
||||
def _is_upload_pack(path: str, query: str) -> bool:
|
||||
if path.endswith("/git-upload-pack"):
|
||||
return True
|
||||
if path.endswith("/info/refs"):
|
||||
return any(
|
||||
pair == "service=git-upload-pack"
|
||||
for pair in query.split("&")
|
||||
)
|
||||
return False
|
||||
|
||||
def _write_cgi_response(self, raw: bytes) -> None:
|
||||
head, sep, body = raw.partition(b"\r\n\r\n")
|
||||
line_sep = b"\r\n"
|
||||
if not sep:
|
||||
head, sep, body = raw.partition(b"\n\n")
|
||||
line_sep = b"\n"
|
||||
status = 200
|
||||
headers: list[tuple[str, str]] = []
|
||||
for line in head.split(line_sep):
|
||||
if not line:
|
||||
continue
|
||||
key, _, value = line.decode("latin1").partition(":")
|
||||
value = value.strip()
|
||||
if key.lower() == "status":
|
||||
try:
|
||||
status = int(value.split()[0])
|
||||
except (ValueError, IndexError):
|
||||
self.log_message(
|
||||
"malformed CGI Status header %r; using 500", value,
|
||||
)
|
||||
status = 500
|
||||
else:
|
||||
headers.append((key, value))
|
||||
self.send_response(status)
|
||||
for key, value in headers:
|
||||
self.send_header(key, value)
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format: str, *args: object) -> None: # type: ignore # noqa: A002
|
||||
sys.stdout.write(format % args + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
port = int(os.environ.get("GIT_HTTP_PORT", str(DEFAULT_PORT)))
|
||||
orch_url = os.environ.get(ORCHESTRATOR_URL_ENV, "").strip()
|
||||
if not orch_url:
|
||||
# Resolver-only: without an orchestrator the backend can't attribute a
|
||||
# request to a bottle namespace, so it must not serve (fail-closed).
|
||||
sys.stderr.write(
|
||||
f"git-http: {ORCHESTRATOR_URL_ENV} is required "
|
||||
"(no single-tenant flat-root fallback)\n"
|
||||
)
|
||||
return 1
|
||||
server = ThreadingHTTPServer(("0.0.0.0", port), GitHttpHandler)
|
||||
# Resolve each request's sandbox namespace by source IP against the
|
||||
# orchestrator control plane.
|
||||
server.policy_resolver = PolicyResolver(orch_url) # type: ignore[attr-defined]
|
||||
sys.stdout.write(f"git-http listening on 0.0.0.0:{port} (multi-tenant)\n")
|
||||
sys.stdout.flush()
|
||||
server.serve_forever()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,198 @@
|
||||
"""Gateway-side per-client policy resolver (PRD 0070).
|
||||
|
||||
The consolidated gateway serves every bottle from one process, so for each
|
||||
request it must apply the *calling* bottle's policy, selected by the source
|
||||
IP the attribution invariant makes unspoofable. This resolves that policy
|
||||
from the orchestrator's control plane (`POST /resolve`), keyed on the
|
||||
`(source_ip, identity_token)` pair.
|
||||
|
||||
**Always fresh — no cache.** The resolver is called rarely enough that a
|
||||
round-trip doesn't matter, and correctness matters more than speed: every
|
||||
resolve reflects the orchestrator's *current* view, so a revocation, a
|
||||
policy change, or a bottle teardown the orchestrator knows about is honored
|
||||
immediately rather than lingering for a cache TTL. (If this ever becomes a
|
||||
hot path, add caching with orchestrator-driven invalidation — not a blind
|
||||
TTL.)
|
||||
|
||||
**Fail-closed:** an unattributed client (the orchestrator answers `403`)
|
||||
resolves to `None`, and the caller (the egress addon, git-gate) must then
|
||||
deny — exactly as an unknown bottle should be treated. Orchestrator
|
||||
*errors* (unreachable / unexpected status) raise, so the caller can fail
|
||||
closed too rather than silently serving stale or empty policy.
|
||||
|
||||
The resolved value is the policy blob the orchestrator stores verbatim; the
|
||||
consumer parses it (e.g. the egress addon's `load_config`). This module is
|
||||
stdlib-only and free of bot-bottle imports.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
DEFAULT_TIMEOUT_SECONDS = 2.0
|
||||
|
||||
# The role-scoped `gateway` token this data plane presents on every control-plane
|
||||
# call, read from the env the launcher injects into the gateway (a pre-minted
|
||||
# signed JWT — the gateway never holds the signing key, so it can't forge a
|
||||
# higher-privilege `cli` token). Constant + env-var name are duplicated here
|
||||
# rather than imported because this module is COPYed flat into the gateway image,
|
||||
# free of bot-bottle imports — same rationale as IDENTITY_HEADER in egress_addon
|
||||
# / git_http_backend.
|
||||
CONTROL_AUTH_HEADER = "x-bot-bottle-control-auth"
|
||||
CONTROL_AUTH_JWT_ENV = "BOT_BOTTLE_CONTROL_AUTH_JWT"
|
||||
|
||||
|
||||
def _control_auth_headers() -> dict[str, str]:
|
||||
"""The auth header to send, or {} when no token is configured (an open
|
||||
control plane, e.g. Firecracker behind its nft boundary — sending nothing
|
||||
is correct there and harmlessly ignored)."""
|
||||
token = os.environ.get(CONTROL_AUTH_JWT_ENV, "").strip()
|
||||
return {CONTROL_AUTH_HEADER: token} if token else {}
|
||||
|
||||
|
||||
class PolicyResolveError(RuntimeError):
|
||||
"""The orchestrator was unreachable or returned an unexpected status —
|
||||
distinct from a clean `403` (unattributed), which returns None."""
|
||||
|
||||
|
||||
class PolicyResolver:
|
||||
"""Resolves each client's policy from the orchestrator, fresh per call."""
|
||||
|
||||
def __init__(self, base_url: str, *, timeout: float = DEFAULT_TIMEOUT_SECONDS) -> None:
|
||||
self._base = base_url.rstrip("/")
|
||||
self._timeout = timeout
|
||||
|
||||
def _post_json(self, path: str, payload: dict[str, object]) -> dict[str, object] | None:
|
||||
"""POST `payload` to control-plane `path`, returning the JSON object body
|
||||
— or None when the orchestrator answers `403` (unattributed / fail
|
||||
closed). Raises `PolicyResolveError` on an unreachable / unexpected-status
|
||||
/ malformed response so every caller can fail closed. Shared by
|
||||
`_post_resolve` and the supervise propose/poll RPCs."""
|
||||
body = json.dumps(payload).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{self._base}{path}", data=body, method="POST",
|
||||
headers={"Content-Type": "application/json", **_control_auth_headers()},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=self._timeout) as resp:
|
||||
data = json.loads(resp.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 403:
|
||||
return None # unattributed → fail closed (caller denies)
|
||||
raise PolicyResolveError(f"{path} returned HTTP {e.code}") from e
|
||||
except (urllib.error.URLError, TimeoutError, OSError, ValueError) as e:
|
||||
raise PolicyResolveError(f"{path} unreachable or malformed: {e}") from e
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
def _post_resolve(self, source_ip: str, identity_token: str) -> dict[str, object] | None:
|
||||
"""The orchestrator's `/resolve` payload for this client, or None if
|
||||
unattributed (a clean `403`). Raises `PolicyResolveError` on an
|
||||
unreachable / unexpected-status / malformed response so every caller
|
||||
can fail closed. Shared by `resolve` and `resolve_bottle_id`."""
|
||||
return self._post_json(
|
||||
"/resolve", {"source_ip": source_ip, "identity_token": identity_token},
|
||||
)
|
||||
|
||||
def propose_supervise(
|
||||
self,
|
||||
source_ip: str,
|
||||
identity_token: str,
|
||||
*,
|
||||
tool: str,
|
||||
proposed_file: str,
|
||||
justification: str,
|
||||
) -> str | None:
|
||||
"""Queue a supervise proposal on the control plane and return its
|
||||
`proposal_id`. The orchestrator attributes the proposal to the calling
|
||||
bottle by `(source_ip, identity_token)` — exactly like `/resolve` — so a
|
||||
bottle can only ever queue its *own* proposals. Returns None when the
|
||||
pair is unattributed (a clean `403`); raises `PolicyResolveError` if the
|
||||
orchestrator can't be reached, so the data-plane caller fails closed
|
||||
(blocks / refuses) rather than silently dropping the proposal."""
|
||||
payload = self._post_json("/supervise/propose", {
|
||||
"source_ip": source_ip,
|
||||
"identity_token": identity_token,
|
||||
"tool": tool,
|
||||
"proposed_file": proposed_file,
|
||||
"justification": justification,
|
||||
})
|
||||
if payload is None:
|
||||
return None
|
||||
proposal_id = payload.get("proposal_id")
|
||||
return proposal_id if isinstance(proposal_id, str) and proposal_id else None
|
||||
|
||||
def poll_supervise(
|
||||
self, source_ip: str, identity_token: str, proposal_id: str,
|
||||
) -> dict[str, object] | None:
|
||||
"""Poll a queued proposal for the operator's decision, **non-blocking**.
|
||||
Attributed by `(source_ip, identity_token)` so a bottle can only read its
|
||||
*own* proposal's response. Returns `{"status": ...}` where status is one
|
||||
of the terminal decisions (`approved`/`modified`/`rejected`, carrying
|
||||
`notes` + `final_file`), `pending` (queued, no decision yet), or
|
||||
`unknown` (no such queued proposal for this bottle). Returns None when
|
||||
unattributed; raises `PolicyResolveError` if unreachable."""
|
||||
return self._post_json("/supervise/poll", {
|
||||
"source_ip": source_ip,
|
||||
"identity_token": identity_token,
|
||||
"proposal_id": proposal_id,
|
||||
})
|
||||
|
||||
def resolve(self, source_ip: str, identity_token: str = "") -> str | None:
|
||||
"""The calling bottle's policy blob, or None if unattributed. Always
|
||||
fetches from the orchestrator so revocations / changes / teardowns
|
||||
are honored immediately.
|
||||
|
||||
`identity_token` is optional *transitionally* — omitting it resolves
|
||||
by source IP alone (the network-layer attribution, sound by
|
||||
construction on Firecracker's `/31`+nft, weaker elsewhere). The
|
||||
consolidated end state makes the token mandatory so the app-layer
|
||||
defense is always enforced, not silently degraded; that flips at the
|
||||
`/resolve` boundary once identity-token *delivery* lands (a PRD 0070
|
||||
open question — we can't require what the agent can't yet present).
|
||||
Raises `PolicyResolveError` if the orchestrator can't be reached."""
|
||||
payload = self._post_resolve(source_ip, identity_token)
|
||||
if payload is None:
|
||||
return None
|
||||
policy = payload.get("policy")
|
||||
return policy if isinstance(policy, str) else ""
|
||||
|
||||
def resolve_bottle_id(self, source_ip: str, identity_token: str = "") -> str | None:
|
||||
"""The calling bottle's id, or None if unattributed. This is the
|
||||
source-IP-keyed identity the git-gate uses to select the bottle's
|
||||
repo namespace (there is no per-bottle policy blob to parse — the
|
||||
bottle *is* the namespace). Same fail-closed contract as `resolve`."""
|
||||
payload = self._post_resolve(source_ip, identity_token)
|
||||
if payload is None:
|
||||
return None
|
||||
bottle_id = payload.get("bottle_id")
|
||||
return bottle_id if isinstance(bottle_id, str) and bottle_id else None
|
||||
|
||||
def resolve_policy_and_bottle_id(
|
||||
self, source_ip: str, identity_token: str = "",
|
||||
) -> tuple[str | None, str | None, dict[str, str]]:
|
||||
"""The policy blob, bottle id, **and per-bottle egress auth tokens** in
|
||||
a single `/resolve` — so the egress addon gets everything it needs
|
||||
(policy for routing, bottle id for the supervise queue/safelist, tokens
|
||||
to inject upstream auth) in one round-trip. Returns `(None, None, {})`
|
||||
when unattributed (a clean `403`). Raises `PolicyResolveError` if the
|
||||
orchestrator can't be reached, so the caller still fails closed."""
|
||||
payload = self._post_resolve(source_ip, identity_token)
|
||||
if payload is None:
|
||||
return None, None, {}
|
||||
policy = payload.get("policy")
|
||||
bottle_id = payload.get("bottle_id")
|
||||
raw = payload.get("tokens")
|
||||
tokens = {
|
||||
k: v for k, v in raw.items() if isinstance(k, str) and isinstance(v, str)
|
||||
} if isinstance(raw, dict) else {}
|
||||
return (
|
||||
policy if isinstance(policy, str) else "",
|
||||
bottle_id if isinstance(bottle_id, str) and bottle_id else None,
|
||||
tokens,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["PolicyResolver", "PolicyResolveError"]
|
||||
@@ -0,0 +1,806 @@
|
||||
"""Supervise daemon HTTP server (PRD 0013).
|
||||
|
||||
Per-bottle MCP server exposing tools the agent calls to propose egress
|
||||
config changes when stuck. The tools are `egress-allow`,
|
||||
`egress-block`, `list-egress-routes`, and `check-proposal`.
|
||||
|
||||
Each queued proposal tool call:
|
||||
|
||||
1. Validates the proposed file syntactically.
|
||||
2. Queues a Proposal over the control-plane RPC (`POST /supervise/propose`),
|
||||
which attributes it to the calling bottle by (source_ip, identity_token)
|
||||
and writes it to the one orchestrator-owned SQLite database. This daemon
|
||||
never opens `bot-bottle.db` itself (PRD 0070 / issue #469).
|
||||
3. Polls the control plane (`POST /supervise/poll`) for a matching Response,
|
||||
up to a short grace window (`SUPERVISE_RESPONSE_TIMEOUT_SECONDS`,
|
||||
default 30s).
|
||||
4. On a decision within the window, returns the operator's
|
||||
`{status, notes}`. On timeout, returns `status: pending` **with the
|
||||
proposal id** and leaves the proposal queued — the flow is
|
||||
non-blocking past the grace window (PRD prd-new / issue #412).
|
||||
|
||||
`check-proposal` is the non-blocking companion: given a `proposal_id`
|
||||
returned by a `pending` response, it reports the current decision
|
||||
(`pending` | `approved` | `modified` | `rejected`) without re-proposing,
|
||||
so an approval made out-of-band (e.g. a web review console) can be resumed
|
||||
without holding an HTTP request open.
|
||||
|
||||
One shared server fronts every bottle (PRD 0070) and attributes each
|
||||
proposal to the calling bottle by source IP, resolved from the orchestrator
|
||||
— an unattributed or unreachable source fails closed. BOT_BOTTLE_ORCHESTRATOR_URL
|
||||
is mandatory: there is no fixed-slug single-tenant fallback, and the queue is
|
||||
reached only over that control-plane RPC (no direct database handle).
|
||||
|
||||
Speaks MCP over HTTP+JSON-RPC. Methods handled:
|
||||
|
||||
* `initialize` — handshake; returns server info + caps.
|
||||
* `notifications/initialized` — ack-only.
|
||||
* `tools/list` — returns the tool definitions.
|
||||
* `tools/call` — validates, queues, waits out the grace
|
||||
window, returns (pending past it); or, for
|
||||
`check-proposal`, a non-blocking status poll.
|
||||
|
||||
Everything else returns JSON-RPC error -32601 (method not found).
|
||||
|
||||
The Dockerfile copies this script to /app/supervise_server.py and installs
|
||||
the bot_bottle package so its `from bot_bottle.*` imports resolve.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import socketserver
|
||||
import sys
|
||||
import time
|
||||
import typing
|
||||
from dataclasses import dataclass
|
||||
|
||||
from bot_bottle.constants import IDENTITY_HEADER
|
||||
from bot_bottle.gateway.egress_addon_core import (
|
||||
LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict,
|
||||
)
|
||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||
from bot_bottle import supervise as _sv
|
||||
|
||||
|
||||
# --- JSON-RPC / MCP plumbing ----------------------------------------------
|
||||
|
||||
|
||||
MCP_PROTOCOL_VERSION = "2024-11-05"
|
||||
SERVER_NAME = "bot-bottle-supervise"
|
||||
SERVER_VERSION = "0.1.0"
|
||||
|
||||
JSONRPC_VERSION = "2.0"
|
||||
|
||||
# JSON-RPC 2.0 standard error codes.
|
||||
ERR_PARSE = -32700
|
||||
ERR_INVALID_REQUEST = -32600
|
||||
ERR_METHOD_NOT_FOUND = -32601
|
||||
ERR_INVALID_PARAMS = -32602
|
||||
ERR_INTERNAL = -32603
|
||||
|
||||
DEFAULT_RESPONSE_TIMEOUT_SECONDS = 30.0
|
||||
# Grace-window poll cadence. 250 ms keeps the operator-visible latency low while
|
||||
# staying gentle on the shared control plane — each poll is now an HTTP round
|
||||
# trip, not a local file read, so the old 50 ms would be ~20 req/s per pending
|
||||
# proposal across every waiting agent (issue #469 review).
|
||||
MIN_RESPONSE_POLL_INTERVAL_SECONDS = 0.25
|
||||
|
||||
# The per-host orchestrator control plane the shared supervise server attributes
|
||||
# each proposal to, by source IP. Mandatory — there is no single-tenant
|
||||
# SUPERVISE_BOTTLE_SLUG fallback.
|
||||
ORCHESTRATOR_URL_ENV = "BOT_BOTTLE_ORCHESTRATOR_URL"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class JsonRpcRequest:
|
||||
method: str
|
||||
params: dict[str, object]
|
||||
id: object # None for notifications; int/str/null for requests
|
||||
is_notification: bool
|
||||
|
||||
|
||||
def parse_jsonrpc(body: bytes) -> JsonRpcRequest:
|
||||
"""Parse a single JSON-RPC 2.0 request body. Raises ValueError
|
||||
with a JSON-RPC error code attached if the shape is wrong."""
|
||||
try:
|
||||
raw = json.loads(body)
|
||||
except json.JSONDecodeError as e:
|
||||
raise _RpcClientError(ERR_PARSE, f"parse error: {e}") from e
|
||||
if not isinstance(raw, dict):
|
||||
raise _RpcClientError(ERR_INVALID_REQUEST, "request must be a JSON object")
|
||||
if raw.get("jsonrpc") != JSONRPC_VERSION:
|
||||
raise _RpcClientError(ERR_INVALID_REQUEST, "jsonrpc field must be '2.0'")
|
||||
method = raw.get("method")
|
||||
if not isinstance(method, str):
|
||||
raise _RpcClientError(ERR_INVALID_REQUEST, "method must be a string")
|
||||
params = raw.get("params", {})
|
||||
if params is None:
|
||||
params = {}
|
||||
if not isinstance(params, dict):
|
||||
raise _RpcClientError(ERR_INVALID_PARAMS, "params must be an object")
|
||||
rpc_id = raw.get("id", _NO_ID)
|
||||
is_notification = rpc_id is _NO_ID
|
||||
return JsonRpcRequest(
|
||||
method=method,
|
||||
params=params,
|
||||
id=None if is_notification else rpc_id,
|
||||
is_notification=is_notification,
|
||||
)
|
||||
|
||||
|
||||
_NO_ID = object()
|
||||
|
||||
|
||||
class _RpcError(Exception):
|
||||
"""Base class for all typed RPC errors that surface as JSON-RPC error responses."""
|
||||
def __init__(self, code: int, message: str):
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
class _RpcClientError(_RpcError):
|
||||
"""Caller sent a bad request; returned verbatim, no server-side logging."""
|
||||
|
||||
|
||||
class _RpcInternalError(_RpcError):
|
||||
"""Server-side fault; logged at ERROR with cause, always returns ERR_INTERNAL."""
|
||||
def __init__(self, message: str) -> None:
|
||||
super().__init__(ERR_INTERNAL, message)
|
||||
|
||||
|
||||
def jsonrpc_result(request_id: object, result: object) -> bytes:
|
||||
payload = {"jsonrpc": JSONRPC_VERSION, "id": request_id, "result": result}
|
||||
return (json.dumps(payload) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
def jsonrpc_error(request_id: object, code: int, message: str) -> bytes:
|
||||
payload = {
|
||||
"jsonrpc": JSONRPC_VERSION,
|
||||
"id": request_id,
|
||||
"error": {"code": code, "message": message},
|
||||
}
|
||||
return (json.dumps(payload) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
# --- Tool definitions ------------------------------------------------------
|
||||
|
||||
|
||||
# Shared by both proposal tools (egress-allow / egress-block): they take the
|
||||
# same arguments and differ only in their top-level tool description. Kept as a
|
||||
# single source of truth so the schema can't drift between the two tools.
|
||||
_ROUTES_YAML_DESCRIPTION = (
|
||||
"Full proposed /etc/egress/routes.yaml content. "
|
||||
"Each route entry accepts these keys:\n"
|
||||
" host: <hostname> (required)\n"
|
||||
" inspect: false (opaque whole-host TLS tunnel; no HTTP controls)\n"
|
||||
" inspect: (omit for inspected defaults)\n"
|
||||
" auth_scheme: Bearer|token (must pair with token_env)\n"
|
||||
" token_env: <ENV_VAR_NAME> (must pair with auth_scheme)\n"
|
||||
" matches: (optional list of match entries)\n"
|
||||
" - paths: [{type: prefix|exact|regex, value: /...}]\n"
|
||||
" methods: [GET, POST, ...]\n"
|
||||
" headers: [{name: X-Hdr, value: val, type: exact|regex}]\n"
|
||||
" git: (optional; omit to block git clone/fetch)\n"
|
||||
" fetch: true\n"
|
||||
" outbound_detectors: [token_patterns, known_secrets]\n"
|
||||
" inbound_detectors: [naive_injection_detection]\n"
|
||||
" outbound_on_match: block|redact|supervise (default supervise)\n"
|
||||
"Omit any key that should use its default. "
|
||||
"`list-egress-routes` returns routes in this same format."
|
||||
)
|
||||
|
||||
|
||||
def _proposal_input_schema() -> dict[str, object]:
|
||||
"""Build a fresh input schema for a routes.yaml proposal tool. Returns a
|
||||
new dict per call so the two tool definitions don't alias one object."""
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"routes_yaml": {
|
||||
"type": "string",
|
||||
"description": _ROUTES_YAML_DESCRIPTION,
|
||||
},
|
||||
"justification": {
|
||||
"type": "string",
|
||||
"description": "Why this egress route is needed.",
|
||||
},
|
||||
},
|
||||
"required": ["routes_yaml", "justification"],
|
||||
}
|
||||
|
||||
|
||||
TOOL_DEFINITIONS: list[dict[str, object]] = [
|
||||
{
|
||||
"name": _sv.TOOL_LIST_EGRESS_ROUTES,
|
||||
"description": (
|
||||
"List the current egress route table — the bottle's "
|
||||
"allowlist. Returns JSON with one entry per allowed host, "
|
||||
"each carrying its matches rules (if any) and whether "
|
||||
"the proxy injects Authorization for the route. Use this "
|
||||
"before composing an `egress-allow` or `egress-block` proposal so "
|
||||
"the new routes file extends the live one rather than "
|
||||
"replacing it."
|
||||
),
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": _sv.TOOL_EGRESS_ALLOW,
|
||||
"description": (
|
||||
"Request operator approval to change the bottle's egress "
|
||||
"allowlist. Pass the full proposed routes.yaml content, not "
|
||||
"just the new host, plus a justification. Use "
|
||||
"`list-egress-routes` first so the proposal preserves existing "
|
||||
"routes."
|
||||
),
|
||||
"inputSchema": _proposal_input_schema(),
|
||||
},
|
||||
{
|
||||
"name": _sv.TOOL_EGRESS_BLOCK,
|
||||
"description": (
|
||||
"Request operator approval to change the bottle's egress "
|
||||
"allowlist after a blocked outbound request. Pass the full "
|
||||
"proposed routes.yaml content plus a justification. Use "
|
||||
"`list-egress-routes` first so the proposal preserves existing "
|
||||
"routes."
|
||||
),
|
||||
"inputSchema": _proposal_input_schema(),
|
||||
},
|
||||
{
|
||||
"name": _sv.TOOL_CHECK_PROPOSAL,
|
||||
"description": (
|
||||
"Poll a previously queued proposal for the operator's decision "
|
||||
"WITHOUT blocking or re-proposing. Pass the `proposal_id` you "
|
||||
"got back when an `egress-allow`/`egress-block` call returned "
|
||||
"`status: pending`. Returns the current status: `pending` (no "
|
||||
"decision yet — poll again later), `approved`, `modified`, "
|
||||
"`rejected`, or `unknown` (no such queued proposal — wrong id, "
|
||||
"or it was already resolved and read)."
|
||||
),
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"proposal_id": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"The proposal id from a `pending` response."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["proposal_id"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
# Map each proposal tool to the input field that carries the agent's
|
||||
# payload (stored in Proposal.proposed_file).
|
||||
PROPOSED_FILE_FIELD: dict[str, str] = {
|
||||
_sv.TOOL_EGRESS_ALLOW: "routes_yaml",
|
||||
_sv.TOOL_EGRESS_BLOCK: "routes_yaml",
|
||||
}
|
||||
|
||||
|
||||
# --- Validation ------------------------------------------------------------
|
||||
|
||||
|
||||
def validate_proposed_file(tool: str, content: str) -> None:
|
||||
"""Syntactic validation. The operator is the real gate; this just
|
||||
catches obvious paste-errors / wrong-tool selections before they
|
||||
enter the queue."""
|
||||
if not content.strip():
|
||||
raise _RpcClientError(ERR_INVALID_PARAMS, f"{tool}: proposed file is empty")
|
||||
if tool in (_sv.TOOL_EGRESS_ALLOW, _sv.TOOL_EGRESS_BLOCK):
|
||||
try:
|
||||
config = load_config(content)
|
||||
except ValueError as e:
|
||||
raise _RpcClientError(
|
||||
ERR_INVALID_PARAMS,
|
||||
f"{tool}: proposed routes.yaml is not valid: {e}",
|
||||
) from e
|
||||
if config.log != LOG_OFF:
|
||||
raise _RpcClientError(
|
||||
ERR_INVALID_PARAMS,
|
||||
f"{tool}: proposed routes.yaml must not change egress logging",
|
||||
)
|
||||
else:
|
||||
raise _RpcClientError(ERR_INVALID_PARAMS, f"unknown tool {tool!r}")
|
||||
|
||||
|
||||
# --- MCP handlers ----------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ServerConfig:
|
||||
# No bottle_slug: the calling bottle is attributed per request by the
|
||||
# control plane from (source_ip, identity_token), so nothing on this daemon
|
||||
# is keyed by a single slug (PRD 0070 / issue #469).
|
||||
response_timeout_seconds: float = DEFAULT_RESPONSE_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
def handle_initialize(_params: dict[str, object]) -> dict[str, object]:
|
||||
return {
|
||||
"protocolVersion": MCP_PROTOCOL_VERSION,
|
||||
"capabilities": {"tools": {"listChanged": False}},
|
||||
"serverInfo": {"name": SERVER_NAME, "version": SERVER_VERSION},
|
||||
}
|
||||
|
||||
|
||||
def handle_tools_list(_params: dict[str, object]) -> dict[str, object]:
|
||||
return {"tools": TOOL_DEFINITIONS}
|
||||
|
||||
|
||||
def handle_tools_call(
|
||||
params: dict[str, object],
|
||||
config: ServerConfig,
|
||||
*,
|
||||
resolver: "PolicyResolver",
|
||||
source_ip: str,
|
||||
identity_token: str,
|
||||
) -> dict[str, object]:
|
||||
"""Validates the proposal, queues it over the control-plane RPC, polls for
|
||||
a Response through the grace window, and returns the result wrapped in MCP
|
||||
`content`.
|
||||
|
||||
The queue lives behind the orchestrator now — `propose_supervise` attributes
|
||||
the proposal to the calling bottle by `(source_ip, identity_token)` and
|
||||
`poll_supervise` reads only that bottle's own responses, so this daemon
|
||||
never opens `bot-bottle.db`. `list-egress-routes` never reaches here — the
|
||||
handler answers it from the calling bottle's resolved policy before
|
||||
dispatching (see `MCPHandler._dispatch`); this path is the queued,
|
||||
operator-approved `egress-allow` / `egress-block` tools."""
|
||||
name = params.get("name")
|
||||
if not isinstance(name, str):
|
||||
raise _RpcClientError(ERR_INVALID_PARAMS, "tools/call missing 'name'")
|
||||
|
||||
args_raw = params.get("arguments", {})
|
||||
if not isinstance(args_raw, dict):
|
||||
raise _RpcClientError(ERR_INVALID_PARAMS, "tools/call 'arguments' must be an object")
|
||||
|
||||
justification = args_raw.get("justification")
|
||||
if not isinstance(justification, str) or not justification.strip():
|
||||
raise _RpcClientError(
|
||||
ERR_INVALID_PARAMS,
|
||||
f"{name}: 'justification' is required and must be a non-empty string",
|
||||
)
|
||||
|
||||
if name in PROPOSED_FILE_FIELD:
|
||||
file_field = PROPOSED_FILE_FIELD[name]
|
||||
proposed_file = args_raw.get(file_field)
|
||||
if not isinstance(proposed_file, str):
|
||||
raise _RpcClientError(
|
||||
ERR_INVALID_PARAMS,
|
||||
f"{name}: '{file_field}' is required and must be a string",
|
||||
)
|
||||
validate_proposed_file(name, proposed_file)
|
||||
else:
|
||||
raise _RpcClientError(ERR_INVALID_PARAMS, f"unknown tool {name!r}")
|
||||
|
||||
proposal_id = _queue_proposal(
|
||||
resolver, source_ip, identity_token,
|
||||
tool=name, proposed_file=proposed_file, justification=justification,
|
||||
)
|
||||
sys.stderr.write(
|
||||
f"supervise: queued proposal {proposal_id} ({name}); waiting for operator...\n"
|
||||
)
|
||||
sys.stderr.flush()
|
||||
|
||||
deadline = time.monotonic() + config.response_timeout_seconds
|
||||
while time.monotonic() < deadline:
|
||||
response = _poll_response(resolver, source_ip, identity_token, proposal_id)
|
||||
if response is not None:
|
||||
return {
|
||||
"content": [{"type": "text", "text": format_response_text(response)}],
|
||||
"isError": response.status == _sv.STATUS_REJECTED,
|
||||
}
|
||||
time.sleep(MIN_RESPONSE_POLL_INTERVAL_SECONDS)
|
||||
|
||||
text = format_pending_response_text(proposal_id, config.response_timeout_seconds)
|
||||
return {
|
||||
"content": [{"type": "text", "text": text}],
|
||||
"isError": False,
|
||||
}
|
||||
|
||||
|
||||
def _queue_proposal(
|
||||
resolver: "PolicyResolver",
|
||||
source_ip: str,
|
||||
identity_token: str,
|
||||
*,
|
||||
tool: str,
|
||||
proposed_file: str,
|
||||
justification: str,
|
||||
) -> str:
|
||||
"""Queue a proposal over the control plane and return its id, mapping the
|
||||
resolver's fail-closed outcomes to internal errors so `do_POST` returns a
|
||||
clean -32603 rather than leaking the cause to the agent."""
|
||||
try:
|
||||
proposal_id = resolver.propose_supervise(
|
||||
source_ip, identity_token,
|
||||
tool=tool, proposed_file=proposed_file, justification=justification,
|
||||
)
|
||||
except PolicyResolveError as e:
|
||||
raise _RpcInternalError(f"orchestrator unreachable, cannot queue proposal: {e}") from e
|
||||
if proposal_id is None:
|
||||
raise _RpcInternalError("request source is not attributed to a bottle")
|
||||
return proposal_id
|
||||
|
||||
|
||||
def _poll_response(
|
||||
resolver: "PolicyResolver", source_ip: str, identity_token: str, proposal_id: str,
|
||||
) -> "_sv.Response | None":
|
||||
"""One non-blocking control-plane poll for `proposal_id`'s decision. Returns
|
||||
the operator's Response once decided, or None while it is still pending
|
||||
(or `unknown` — treated as still-waiting so a transient blip degrades to the
|
||||
pending-timeout path rather than a spurious error mid grace window)."""
|
||||
try:
|
||||
result = resolver.poll_supervise(source_ip, identity_token, proposal_id)
|
||||
except PolicyResolveError as e:
|
||||
raise _RpcInternalError(f"orchestrator unreachable, cannot poll proposal: {e}") from e
|
||||
if result is None:
|
||||
raise _RpcInternalError("request source is not attributed to a bottle")
|
||||
if result.get("status") in _sv.STATUSES:
|
||||
return _response_from_poll(proposal_id, result)
|
||||
return None
|
||||
|
||||
|
||||
def _response_from_poll(proposal_id: str, result: "dict[str, object]") -> "_sv.Response":
|
||||
"""Build a Response from a decided `/supervise/poll` payload."""
|
||||
final_file = result.get("final_file")
|
||||
return _sv.Response(
|
||||
proposal_id=proposal_id,
|
||||
status=str(result.get("status")),
|
||||
notes=str(result.get("notes") or ""),
|
||||
final_file=final_file if isinstance(final_file, str) else None,
|
||||
)
|
||||
|
||||
|
||||
def handle_check_proposal(
|
||||
params: dict[str, object],
|
||||
*,
|
||||
resolver: "PolicyResolver",
|
||||
source_ip: str,
|
||||
identity_token: str,
|
||||
) -> dict[str, object]:
|
||||
"""Non-blocking poll of a queued proposal's decision, by id.
|
||||
|
||||
Never creates a Proposal (so `check-proposal` isn't in `TOOLS`); it only
|
||||
reads the queue over the control-plane RPC, attributed by
|
||||
`(source_ip, identity_token)` so it can only read *this* bottle's own
|
||||
proposals. The orchestrator archives a decided proposal on read — the same
|
||||
terminal step the synchronous path triggers — so `pending` proposals stay
|
||||
visible to the operator until they're both decided *and* polled."""
|
||||
args_raw = params.get("arguments", {})
|
||||
if not isinstance(args_raw, dict):
|
||||
raise _RpcClientError(ERR_INVALID_PARAMS, "tools/call 'arguments' must be an object")
|
||||
proposal_id = args_raw.get("proposal_id")
|
||||
if not isinstance(proposal_id, str) or not proposal_id.strip():
|
||||
raise _RpcClientError(
|
||||
ERR_INVALID_PARAMS,
|
||||
"check-proposal: 'proposal_id' is required and must be a non-empty string",
|
||||
)
|
||||
proposal_id = proposal_id.strip()
|
||||
|
||||
try:
|
||||
result = resolver.poll_supervise(source_ip, identity_token, proposal_id)
|
||||
except PolicyResolveError as e:
|
||||
raise _RpcInternalError(f"orchestrator unreachable, cannot poll proposal: {e}") from e
|
||||
if result is None:
|
||||
raise _RpcInternalError("request source is not attributed to a bottle")
|
||||
status = result.get("status")
|
||||
|
||||
if status == _sv.POLL_STATUS_UNKNOWN:
|
||||
return {
|
||||
"content": [{"type": "text", "text": format_unknown_proposal_text(proposal_id)}],
|
||||
"isError": True,
|
||||
}
|
||||
if status == _sv.POLL_STATUS_PENDING:
|
||||
return {
|
||||
"content": [{"type": "text", "text": format_still_pending_text(proposal_id)}],
|
||||
"isError": False,
|
||||
}
|
||||
response = _response_from_poll(proposal_id, result)
|
||||
return {
|
||||
"content": [{"type": "text", "text": format_response_text(response)}],
|
||||
"isError": response.status == _sv.STATUS_REJECTED,
|
||||
}
|
||||
|
||||
|
||||
def format_response_text(response: "_sv.Response") -> str:
|
||||
"""Pretty-print a Response for the tool's text content. The agent
|
||||
reads the text and decides whether to retry / give up / surface."""
|
||||
lines = [f"status: {response.status}"]
|
||||
if response.notes:
|
||||
lines.append(f"notes: {response.notes}")
|
||||
if response.status == _sv.STATUS_MODIFIED and response.final_file is not None:
|
||||
lines.append("the operator modified your proposal before approving; "
|
||||
"the final config is now what's been applied")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_pending_response_text(proposal_id: str, timeout_seconds: float) -> str:
|
||||
"""Grace-window timeout: the proposal stays queued, and the agent is
|
||||
told the id so it can `check-proposal` instead of re-proposing."""
|
||||
return "\n".join([
|
||||
"status: pending",
|
||||
f"proposal_id: {proposal_id}",
|
||||
(
|
||||
f"notes: no operator decision within {timeout_seconds:g}s; the "
|
||||
"proposal remains queued. Poll it (do not re-propose) by calling "
|
||||
f"`check-proposal` with proposal_id={proposal_id!r}."
|
||||
),
|
||||
])
|
||||
|
||||
|
||||
def format_still_pending_text(proposal_id: str) -> str:
|
||||
return "\n".join([
|
||||
"status: pending",
|
||||
f"proposal_id: {proposal_id}",
|
||||
"notes: still queued; no operator decision yet. Call `check-proposal` again later.",
|
||||
])
|
||||
|
||||
|
||||
def format_unknown_proposal_text(proposal_id: str) -> str:
|
||||
return "\n".join([
|
||||
"status: unknown",
|
||||
f"proposal_id: {proposal_id}",
|
||||
(
|
||||
"notes: no queued proposal with this id for this bottle — the id "
|
||||
"may be wrong, or the proposal was already resolved and read."
|
||||
),
|
||||
])
|
||||
|
||||
|
||||
# --- HTTP transport --------------------------------------------------------
|
||||
|
||||
|
||||
# Max request body the server accepts. 1 MB is well above any realistic
|
||||
# routes.yaml proposal.
|
||||
MAX_BODY_BYTES = 1 * 1024 * 1024
|
||||
|
||||
|
||||
class MCPHandler(http.server.BaseHTTPRequestHandler):
|
||||
"""Per-request JSON-RPC handler. Each tools/call may block for
|
||||
a long time; the ThreadingMixIn on the server class ensures
|
||||
other requests can be processed concurrently."""
|
||||
|
||||
server_version = f"{SERVER_NAME}/{SERVER_VERSION}"
|
||||
|
||||
def log_message(self, format: str, *args: typing.Any) -> None: # noqa: A002
|
||||
if os.environ.get("SUPERVISE_DEBUG"):
|
||||
super().log_message(format, *args)
|
||||
|
||||
def do_GET(self) -> None:
|
||||
# /health for liveness; everything else 405. POST is the only
|
||||
# method MCP needs.
|
||||
if self.path == "/health":
|
||||
self._write_text(200, "ok\n")
|
||||
return
|
||||
self._write_text(405, "use POST for MCP requests\n")
|
||||
|
||||
def do_POST(self) -> None:
|
||||
length_header = self.headers.get("Content-Length")
|
||||
if length_header is None:
|
||||
self._write_text(411, "Content-Length required\n")
|
||||
return
|
||||
try:
|
||||
length = int(length_header)
|
||||
except ValueError:
|
||||
self._write_text(400, "invalid Content-Length\n")
|
||||
return
|
||||
if length < 0 or length > MAX_BODY_BYTES:
|
||||
self._write_text(413, "request body too large\n")
|
||||
return
|
||||
body = self.rfile.read(length)
|
||||
|
||||
try:
|
||||
req = parse_jsonrpc(body)
|
||||
except _RpcClientError as e:
|
||||
self._write_jsonrpc(jsonrpc_error(None, e.code, e.message))
|
||||
return
|
||||
|
||||
config = typing.cast("MCPServer", self.server).config
|
||||
|
||||
try:
|
||||
result = self._dispatch(req, config)
|
||||
except _RpcClientError as e:
|
||||
self._write_jsonrpc(jsonrpc_error(req.id, e.code, e.message))
|
||||
return
|
||||
except _RpcInternalError as e:
|
||||
cause = e.__cause__
|
||||
detail = f": {cause}" if cause else ""
|
||||
sys.stderr.write(f"supervise: internal error: {e.message}{detail}\n")
|
||||
sys.stderr.flush()
|
||||
self._write_jsonrpc(jsonrpc_error(req.id, ERR_INTERNAL, "internal error"))
|
||||
return
|
||||
except Exception as e: # noqa: W0718 — unexpected errors
|
||||
sys.stderr.write(f"supervise: unexpected error: {type(e).__name__}: {e}\n")
|
||||
sys.stderr.flush()
|
||||
self._write_jsonrpc(jsonrpc_error(req.id, ERR_INTERNAL, "internal error"))
|
||||
return
|
||||
|
||||
if req.is_notification:
|
||||
self._write_text(202, "")
|
||||
return
|
||||
self._write_jsonrpc(jsonrpc_result(req.id, result))
|
||||
|
||||
def _dispatch(self, req: JsonRpcRequest, config: ServerConfig) -> object:
|
||||
method = req.method
|
||||
if method == "initialize":
|
||||
return handle_initialize(req.params)
|
||||
if method == "notifications/initialized":
|
||||
return None # ack-only
|
||||
if method == "tools/list":
|
||||
return handle_tools_list(req.params)
|
||||
if method == "tools/call":
|
||||
# `list-egress-routes` is read-only introspection. The shared gateway
|
||||
# has no static route table (routes are resolved per request by
|
||||
# source IP), so answer it from the calling bottle's resolved policy.
|
||||
# Otherwise the agent sees an empty allowlist and composes an egress
|
||||
# proposal that *replaces* the live routes instead of extending them
|
||||
# — silently dropping base routes like api.anthropic.com on approval.
|
||||
if req.params.get("name") == _sv.TOOL_LIST_EGRESS_ROUTES:
|
||||
return self._resolved_routes_payload()
|
||||
resolver = self._resolver_or_fail()
|
||||
source_ip = self.client_address[0]
|
||||
token = self._identity_token()
|
||||
# `check-proposal` is a non-blocking read of the calling bottle's
|
||||
# own queue — attributed by (source_ip, identity_token) like a
|
||||
# proposal, but it never queues or blocks.
|
||||
if req.params.get("name") == _sv.TOOL_CHECK_PROPOSAL:
|
||||
return handle_check_proposal(
|
||||
req.params, resolver=resolver,
|
||||
source_ip=source_ip, identity_token=token,
|
||||
)
|
||||
# The control plane attributes the proposal to the source-IP + token
|
||||
# resolved bottle, so the one shared queue holds each bottle's
|
||||
# proposal under its own id — no slug is asserted by this daemon.
|
||||
return handle_tools_call(
|
||||
req.params, config, resolver=resolver,
|
||||
source_ip=source_ip, identity_token=token,
|
||||
)
|
||||
raise _RpcClientError(ERR_METHOD_NOT_FOUND, f"method not found: {method}")
|
||||
|
||||
def _identity_token(self) -> str:
|
||||
"""The agent's per-bottle identity token from the request header (the
|
||||
MCP client sends it via `mcp add --header`), or empty. The control plane
|
||||
requires the (source_ip, token) pair, so a missing/wrong token
|
||||
fail-closes there."""
|
||||
headers = getattr(self, "headers", None)
|
||||
return headers.get(IDENTITY_HEADER, "") if headers is not None else ""
|
||||
|
||||
def _resolver_or_fail(self) -> "PolicyResolver":
|
||||
"""This server's policy resolver. A server started without one is a
|
||||
misconfiguration, not a tenancy mode — fail closed rather than
|
||||
attribute (or list) anything."""
|
||||
resolver = getattr(self.server, "policy_resolver", None)
|
||||
if resolver is None:
|
||||
raise _RpcInternalError("supervise server has no policy resolver")
|
||||
return resolver
|
||||
|
||||
def _resolved_routes_payload(self) -> dict[str, object]:
|
||||
"""The calling bottle's live egress routes as the `list-egress-routes`
|
||||
JSON payload, resolved by (source_ip, identity token). Fail-closed: an
|
||||
unattributed source or an unreachable orchestrator yields an empty route
|
||||
list (never another bottle's), courtesy of `resolve_client_context`."""
|
||||
resolver = self._resolver_or_fail()
|
||||
conf, _slug, _tokens = resolve_client_context(
|
||||
resolver, self.client_address[0], self._identity_token(),
|
||||
)
|
||||
body = json.dumps(
|
||||
{"routes": [route_to_yaml_dict(r) for r in conf.routes]}, indent=2,
|
||||
)
|
||||
return {"content": [{"type": "text", "text": body}], "isError": False}
|
||||
|
||||
def _write_jsonrpc(self, body: bytes) -> None:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _write_text(self, status: int, body: str) -> None:
|
||||
encoded = body.encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "text/plain; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(encoded)))
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
if encoded:
|
||||
self.wfile.write(encoded)
|
||||
|
||||
|
||||
class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||||
allow_reuse_address = True
|
||||
daemon_threads = True
|
||||
config: ServerConfig = ServerConfig()
|
||||
# Set by `serve`; every proposal is attributed to the source-IP + token
|
||||
# resolved bottle by the control plane. A server without a resolver fails
|
||||
# closed per request (see `_resolver_or_fail`).
|
||||
policy_resolver: "PolicyResolver | None" = None
|
||||
|
||||
|
||||
# --- Entry point -----------------------------------------------------------
|
||||
|
||||
|
||||
def serve(
|
||||
*,
|
||||
resolver: "PolicyResolver",
|
||||
port: int = _sv.SUPERVISE_PORT,
|
||||
bind: str = "0.0.0.0",
|
||||
response_timeout_seconds: float = DEFAULT_RESPONSE_TIMEOUT_SECONDS,
|
||||
) -> typing.NoReturn:
|
||||
server = MCPServer((bind, port), MCPHandler)
|
||||
# Every request's proposal is attributed to the source-IP + token resolved
|
||||
# bottle by the control plane (see MCPHandler._dispatch).
|
||||
server.config = ServerConfig(response_timeout_seconds=response_timeout_seconds)
|
||||
server.policy_resolver = resolver
|
||||
sys.stderr.write(
|
||||
f"supervise listening on {bind}:{port}; multi-tenant; "
|
||||
f"tools: {', '.join(t['name'] for t in TOOL_DEFINITIONS)}\n" # type: ignore[arg-type]
|
||||
)
|
||||
sys.stderr.flush()
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
server.server_close()
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
del argv # config is env-only, no CLI flags
|
||||
orch_url = os.environ.get(ORCHESTRATOR_URL_ENV, "").strip()
|
||||
if not orch_url:
|
||||
# Resolver-only: without an orchestrator the server can't attribute a
|
||||
# proposal to a bottle, so it must not serve (fail-closed).
|
||||
sys.stderr.write(
|
||||
f"supervise: {ORCHESTRATOR_URL_ENV} is required "
|
||||
"(no single-tenant SUPERVISE_BOTTLE_SLUG fallback)\n"
|
||||
)
|
||||
return 2
|
||||
port = int(os.environ.get("SUPERVISE_PORT", str(_sv.SUPERVISE_PORT)))
|
||||
bind = os.environ.get("SUPERVISE_BIND", "0.0.0.0")
|
||||
try:
|
||||
response_timeout_seconds = _response_timeout_from_env(os.environ)
|
||||
except ValueError as e:
|
||||
sys.stderr.write(f"supervise: {e}\n")
|
||||
return 2
|
||||
serve(
|
||||
resolver=PolicyResolver(orch_url),
|
||||
port=port,
|
||||
bind=bind,
|
||||
response_timeout_seconds=response_timeout_seconds,
|
||||
)
|
||||
return 0 # serve() does not return
|
||||
|
||||
|
||||
def _response_timeout_from_env(env: typing.Mapping[str, str]) -> float:
|
||||
raw = env.get("SUPERVISE_RESPONSE_TIMEOUT_SECONDS", "").strip()
|
||||
if not raw:
|
||||
return DEFAULT_RESPONSE_TIMEOUT_SECONDS
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError as e:
|
||||
raise ValueError(
|
||||
"SUPERVISE_RESPONSE_TIMEOUT_SECONDS must be a positive number"
|
||||
) from e
|
||||
if value <= 0:
|
||||
raise ValueError(
|
||||
"SUPERVISE_RESPONSE_TIMEOUT_SECONDS must be a positive number"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
Reference in New Issue
Block a user