From 5ad3449e3bbb6f7f54e7c2c2113a86dbb0c7715a Mon Sep 17 00:00:00 2001 From: claude Date: Sat, 18 Jul 2026 02:31:39 +0000 Subject: [PATCH] refactor(gateway): replace flat-file import shims with installed package Install bot_bottle via pip in Dockerfile.gateway instead of COPYing individual .py files flat under /app/. This eliminates the try/except import shims in egress_addon_core, dlp_detectors, egress_addon, supervise, supervise_server, and git_http_backend that existed only to support the flat-bundle layout. Adds bot_bottle/constants.py as a single source of truth for IDENTITY_HEADER and GIT_GATE_TIMEOUT_SECS, removing the duplicated literal definitions in egress_addon.py, supervise_server.py, git_http_backend.py, and git_gate_render.py. Test files updated to match: test_supervise_server.py drops the sys.path.insert hack in favour of direct package imports; the egress_addon test shims no longer pre-populate sys.modules with a bare egress_addon_core alias. --- Dockerfile.gateway | 39 +++++++-------- bot_bottle/constants.py | 17 +++++++ bot_bottle/dlp_detectors.py | 10 ++-- bot_bottle/egress_addon.py | 31 ++---------- bot_bottle/egress_addon_core.py | 48 ++++++------------ bot_bottle/git_gate_render.py | 8 +-- bot_bottle/git_http_backend.py | 25 +--------- bot_bottle/policy_resolver.py | 3 +- bot_bottle/supervise.py | 50 ++++++------------- bot_bottle/supervise_server.py | 28 +++-------- pyproject.toml | 8 +++ tests/unit/test_egress_addon_log_redaction.py | 5 +- tests/unit/test_egress_addon_request_flow.py | 3 -- tests/unit/test_supervise_server.py | 13 ++--- 14 files changed, 98 insertions(+), 190 deletions(-) create mode 100644 bot_bottle/constants.py create mode 100644 pyproject.toml diff --git a/Dockerfile.gateway b/Dockerfile.gateway index a6cfe05..d9dbe57 100644 --- a/Dockerfile.gateway +++ b/Dockerfile.gateway @@ -16,10 +16,13 @@ # Layout: # # /usr/bin/gitleaks gitleaks binary -# /app/egress_addon.py + siblings mitmproxy addon (egress) +# /app/egress_addon.py mitmproxy addon entry point # /app/egress-entrypoint.sh mitmdump launcher -# /app/supervise_server.py + .py supervise MCP server -# /app/gateway_init.py PID 1 supervisor +# /app/supervise_server.py supervise MCP server entry point +# /app/gateway_init.py PID 1 supervisor entry point +# /app/git_http_backend.py git-http entry point +# /usr/local/lib/python*/bot_bottle/ installed package (all shared modules) +# /etc/egress/routes.yaml bind-mounted at run time # /etc/git-gate/pre-receive docker-cp'd at start time # /git-gate-entrypoint.sh docker-cp'd at start time # /git-gate/creds/* docker-cp'd at start time @@ -87,24 +90,18 @@ RUN arch="${TARGETARCH:-$(dpkg --print-architecture)}" \ && tar -xzf /tmp/gitleaks.tar.gz -C /usr/bin gitleaks \ && rm /tmp/gitleaks.tar.gz -# Project Python: addon + server modules + the init supervisor. -# Kept flat under /app/ so mitmdump's loader resolves them as -# top-level siblings (absolute imports), matching the prior -# Dockerfile.egress / Dockerfile.supervise layout. -COPY bot_bottle/egress_addon_core.py /app/egress_addon_core.py -COPY bot_bottle/egress_dlp_config.py /app/egress_dlp_config.py +# Install bot_bottle as a proper package so entry-point scripts can use +# `from bot_bottle.X import Y` absolute imports. A rename or a missing +# module is caught at pip-install time — not at container runtime. +COPY pyproject.toml /src/ +COPY bot_bottle/ /src/bot_bottle/ +RUN pip install --no-cache-dir /src/ + +# Entry-point scripts invoked by gateway_init.py. These live at /app/ +# so the supervisor, shell entrypoint, and ENTRYPOINT all reach them at +# known fixed paths. Their sibling imports all go through bot_bottle.* +# (the installed package above), not flat /app/ neighbours. COPY bot_bottle/egress_addon.py /app/egress_addon.py -COPY bot_bottle/policy_resolver.py /app/policy_resolver.py -COPY bot_bottle/dlp_detectors.py /app/dlp_detectors.py -COPY bot_bottle/yaml_subset.py /app/yaml_subset.py -COPY bot_bottle/paths.py /app/paths.py -COPY bot_bottle/migrations.py /app/migrations.py -COPY bot_bottle/db_store.py /app/db_store.py -COPY bot_bottle/supervise_types.py /app/supervise_types.py -COPY bot_bottle/queue_store.py /app/queue_store.py -COPY bot_bottle/audit_store.py /app/audit_store.py -COPY bot_bottle/store_manager.py /app/store_manager.py -COPY bot_bottle/supervise.py /app/supervise.py COPY bot_bottle/supervise_server.py /app/supervise_server.py COPY bot_bottle/gateway_init.py /app/gateway_init.py COPY bot_bottle/git_http_backend.py /app/git_http_backend.py @@ -126,8 +123,6 @@ RUN mkdir -p \ # subset the bottle uses. EXPOSE 8888 9099 9418 9420 9100 -# WORKDIR matches Dockerfile.supervise's prior layout so the -# in-app same-dir import in supervise_server.py stays deterministic. WORKDIR /app # PID 1 is the supervisor. It owns signal handling and exit-code diff --git a/bot_bottle/constants.py b/bot_bottle/constants.py new file mode 100644 index 0000000..0fbad72 --- /dev/null +++ b/bot_bottle/constants.py @@ -0,0 +1,17 @@ +"""Shared wire-protocol constants for gateway-bundled modules. + +Single source of truth for values that appear across the egress addon, +git-http backend, supervise server, and git-gate renderer. Importing +from this module instead of duplicating the literals means a rename is +a one-line change and is caught by the type checker at the import site.""" + +# App-layer identity token header. Delivered as proxy credentials +# (HTTPS_PROXY=http://:@gw) by launch; the egress +# addon reads and strips it, the supervise server and git-http backend +# read it for attribution, and none of them forward it upstream. +IDENTITY_HEADER = "x-bot-bottle-identity" + +# Shared timeout (seconds) for all git-gate subprocess and CGI calls: +# git daemon (--timeout/--init-timeout), the access-hook subprocess in +# git_http_backend, and the git http-backend CGI subprocess. +GIT_GATE_TIMEOUT_SECS = 15 diff --git a/bot_bottle/dlp_detectors.py b/bot_bottle/dlp_detectors.py index 346a147..2ca286a 100644 --- a/bot_bottle/dlp_detectors.py +++ b/bot_bottle/dlp_detectors.py @@ -3,9 +3,8 @@ Pure Python, no mitmproxy dependency. Each detector is a module-level function returning `ScanResult | None`. -Ships flat into the gateway image alongside -`egress_addon_core.py` — both this file and the package source use -the same try/except import shim pattern. +Available in the gateway via the installed `bot_bottle` package +(see `Dockerfile.gateway`). """ from __future__ import annotations @@ -20,10 +19,7 @@ from math import log2 from collections import Counter from urllib.parse import quote as url_quote -try: - from egress_addon_core import ScanResult # type: ignore[import-not-found] -except ImportError: # pragma: no cover - host-side path - from .egress_addon_core import ScanResult +from .egress_addon_core import ScanResult # --------------------------------------------------------------------------- diff --git a/bot_bottle/egress_addon.py b/bot_bottle/egress_addon.py index b2ebb43..b550dc6 100644 --- a/bot_bottle/egress_addon.py +++ b/bot_bottle/egress_addon.py @@ -15,7 +15,9 @@ import typing from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=import-error -from egress_addon_core import ( # type: ignore[import-not-found] # pylint: disable=import-error +from bot_bottle.constants import IDENTITY_HEADER +from bot_bottle.dlp_detectors import redact_tokens, strip_crlf +from bot_bottle.egress_addon_core import ( LOG_BLOCKS, LOG_FULL, DEFAULT_OUTBOUND_ON_MATCH, @@ -38,24 +40,8 @@ from egress_addon_core import ( # type: ignore[import-not-found] # pylint: dis scan_inbound, scan_outbound, ) - -try: - from dlp_detectors import redact_tokens, strip_crlf # type: ignore[import-not-found] -except ImportError: # pragma: no cover - host-side path - from bot_bottle.dlp_detectors import ( # type: ignore[import-not-found] - redact_tokens, - strip_crlf, - ) - -try: - import supervise as _sv # type: ignore[import-not-found] -except ImportError: # pragma: no cover - host-side path - from bot_bottle import supervise as _sv # type: ignore[import-not-found] - -try: - from policy_resolver import PolicyResolver # type: ignore[import-not-found] -except ImportError: # pragma: no cover - host-side path - from bot_bottle.policy_resolver import PolicyResolver +from bot_bottle import supervise as _sv +from bot_bottle.policy_resolver import PolicyResolver INTROSPECT_HOST = "_egress.local" @@ -66,13 +52,6 @@ INTROSPECT_HOST = "_egress.local" # back to — so an unset value is a fatal misconfiguration (see __init__). ORCHESTRATOR_URL_ENV = "BOT_BOTTLE_ORCHESTRATOR_URL" -# App-layer identity token. Delivered as proxy credentials -# (`HTTPS_PROXY=http://:@gw`): clients honor it as part of -# the proxy protocol without app changes, and the addon reads + strips it so -# it never leaks upstream. The legacy `x-bot-bottle-identity` request header -# is still stripped defensively (git-http uses that header on its own port). -IDENTITY_HEADER = "x-bot-bottle-identity" - # Per-flow key under which `request()` stashes the resolved (Config, supervise # slug, env) so the later `response()` and `websocket_message()` hooks scan # against the *calling bottle's* policy — the same one the request was decided diff --git a/bot_bottle/egress_addon_core.py b/bot_bottle/egress_addon_core.py index a1ee7f2..d3e2d3a 100644 --- a/bot_bottle/egress_addon_core.py +++ b/bot_bottle/egress_addon_core.py @@ -6,9 +6,9 @@ exercise the parse + decision functions without depending on the `mitmproxy.http.HTTPFlow` API and is loaded inside the gateway container. -Imports: stdlib + `yaml_subset` (which is itself stdlib-only and -ships flat into the gateway image alongside this file — -see `Dockerfile.gateway`).""" +Imports: stdlib + sibling package modules (`yaml_subset`, +`egress_dlp_config`). Available in the gateway via the installed +`bot_bottle` package (see `Dockerfile.gateway`).""" from __future__ import annotations @@ -16,36 +16,20 @@ import re import typing from dataclasses import dataclass -try: - from yaml_subset import YamlSubsetError, parse_yaml_subset # type: ignore[import-not-found] -except ImportError: # pragma: no cover - host-side path - from .yaml_subset import YamlSubsetError, parse_yaml_subset +from .yaml_subset import YamlSubsetError, parse_yaml_subset -# DLP detector-config parsing lives in a sibling module (also flat-bundled -# into the gateway — see Dockerfile.gateway). Re-exported below so existing -# `from egress_addon_core import ON_MATCH_*` callers keep working. -try: - from egress_dlp_config import ( # type: ignore[import-not-found] - DEFAULT_OUTBOUND_ON_MATCH, - INBOUND_DETECTOR_NAMES, - ON_MATCH_BLOCK, - ON_MATCH_REDACT, - ON_MATCH_SUPERVISE, - OUTBOUND_DETECTOR_NAMES, - OUTBOUND_ON_MATCH_VALUES, - parse_dlp_block, - ) -except ImportError: # pragma: no cover - host-side path - from .egress_dlp_config import ( - DEFAULT_OUTBOUND_ON_MATCH, - INBOUND_DETECTOR_NAMES, - ON_MATCH_BLOCK, - ON_MATCH_REDACT, - ON_MATCH_SUPERVISE, - OUTBOUND_DETECTOR_NAMES, - OUTBOUND_ON_MATCH_VALUES, - parse_dlp_block, - ) +# DLP detector-config parsing lives in a sibling module. Re-exported below +# so existing `from egress_addon_core import ON_MATCH_*` callers keep working. +from .egress_dlp_config import ( + DEFAULT_OUTBOUND_ON_MATCH, + INBOUND_DETECTOR_NAMES, + ON_MATCH_BLOCK, + ON_MATCH_REDACT, + ON_MATCH_SUPERVISE, + OUTBOUND_DETECTOR_NAMES, + OUTBOUND_ON_MATCH_VALUES, + parse_dlp_block, +) # --------------------------------------------------------------------------- diff --git a/bot_bottle/git_gate_render.py b/bot_bottle/git_gate_render.py index c33cb89..a7a93e8 100644 --- a/bot_bottle/git_gate_render.py +++ b/bot_bottle/git_gate_render.py @@ -14,18 +14,12 @@ import shlex from dataclasses import dataclass from pathlib import Path +from .constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER from .manifest import ManifestBottle, ManifestGitEntry # Short network alias for git-gate inside the gateway. The # agent's `.gitconfig` insteadOf rewrites resolve through this name. GIT_GATE_HOSTNAME = "git-gate" -# App-layer identity token header the agent's git sends to git-http and the -# gateway validates (mirrors egress_addon / git_http_backend IDENTITY_HEADER). -IDENTITY_HEADER = "x-bot-bottle-identity" -# Shared timeout (seconds) for all git-gate subprocess and CGI calls: -# git daemon (--timeout/--init-timeout), the access-hook subprocess in -# git_http_backend, and the git http-backend CGI subprocess. -GIT_GATE_TIMEOUT_SECS = 15 @dataclass(frozen=True) diff --git a/bot_bottle/git_http_backend.py b/bot_bottle/git_http_backend.py index ba92026..22e538e 100644 --- a/bot_bottle/git_http_backend.py +++ b/bot_bottle/git_http_backend.py @@ -26,16 +26,8 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path from urllib.parse import urlsplit -# policy_resolver ships flat alongside this file in the gateway -# image (see Dockerfile.gateway); the bot_bottle.* fallback is the -# host-side / test path. Mirrors egress_addon's import shape. -try: - from policy_resolver import ( # type: ignore[import-not-found] - PolicyResolveError, - PolicyResolver, - ) -except ImportError: # pragma: no cover - host-side path - from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver +from bot_bottle.constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER +from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver DEFAULT_PORT = 9420 @@ -46,12 +38,6 @@ DEFAULT_PORT = 9420 # repo-root fallback. ORCHESTRATOR_URL_ENV = "BOT_BOTTLE_ORCHESTRATOR_URL" -# App-layer identity token (defense-in-depth over the source-IP invariant); -# the agent injects it, the backend reads it for attribution and never -# forwards it to `git http-backend`. Mirrors egress_addon.IDENTITY_HEADER -# (duplicated, not imported: egress_addon pulls in mitmproxy). -IDENTITY_HEADER = "x-bot-bottle-identity" - # The base under which each bottle's `` repo namespace is nested. DEFAULT_REPO_ROOT = "/git" @@ -89,13 +75,6 @@ def resolve_sandbox_root( return None # bottle_id tried to escape the root → deny return namespace -# Mirrors git_gate_render.GIT_GATE_TIMEOUT_SECS. Duplicated rather than -# imported: this module ships as a flat top-level sibling in the gateway -# bundle image (see Dockerfile.gateway), not as part of the bot_bottle -# package, so `bot_bottle.git_gate` and its dependency chain aren't -# available at runtime. -GIT_GATE_TIMEOUT_SECS = 15 - # Bound memory use while still allowing ordinary git push packfiles. MAX_BODY_BYTES = 100 * 1024 * 1024 diff --git a/bot_bottle/policy_resolver.py b/bot_bottle/policy_resolver.py index e16ac12..06ed563 100644 --- a/bot_bottle/policy_resolver.py +++ b/bot_bottle/policy_resolver.py @@ -22,8 +22,7 @@ closed too rather than silently serving stale or empty policy. The resolved value is the policy blob the orchestrator stores verbatim; the consumer parses it (e.g. the egress addon's `load_config`). This module is -stdlib-only and free of bot-bottle imports so it can be COPYed flat into -the gateway. +stdlib-only and free of bot-bottle imports. """ from __future__ import annotations diff --git a/bot_bottle/supervise.py b/bot_bottle/supervise.py index df07ede..0acd593 100644 --- a/bot_bottle/supervise.py +++ b/bot_bottle/supervise.py @@ -37,40 +37,22 @@ from abc import ABC from dataclasses import dataclass from pathlib import Path -try: - from .supervise_types import ( - ACTION_OPERATOR_EDIT, - AuditEntry, - Proposal, - Response, - STATUSES, - STATUS_APPROVED, - STATUS_MODIFIED, - STATUS_REJECTED, - TOOLS, - TOOL_EGRESS_ALLOW, - TOOL_EGRESS_BLOCK, - TOOL_EGRESS_TOKEN_ALLOW, - TOOL_GITLEAKS_ALLOW, - TOOL_LIST_EGRESS_ROUTES, - ) -except ImportError: - from supervise_types import ( # type: ignore[import-not-found,no-redef] # pylint: disable=import-error,no-name-in-module - ACTION_OPERATOR_EDIT, - AuditEntry, - Proposal, - Response, - STATUSES, - STATUS_APPROVED, - STATUS_MODIFIED, - STATUS_REJECTED, - TOOLS, - TOOL_EGRESS_ALLOW, - TOOL_EGRESS_BLOCK, - TOOL_EGRESS_TOKEN_ALLOW, - TOOL_GITLEAKS_ALLOW, - TOOL_LIST_EGRESS_ROUTES, - ) +from .supervise_types import ( + ACTION_OPERATOR_EDIT, + AuditEntry, + Proposal, + Response, + STATUSES, + STATUS_APPROVED, + STATUS_MODIFIED, + STATUS_REJECTED, + TOOLS, + TOOL_EGRESS_ALLOW, + TOOL_EGRESS_BLOCK, + TOOL_EGRESS_TOKEN_ALLOW, + TOOL_GITLEAKS_ALLOW, + TOOL_LIST_EGRESS_ROUTES, +) try: diff --git a/bot_bottle/supervise_server.py b/bot_bottle/supervise_server.py index b3953d3..3cfcf02 100644 --- a/bot_bottle/supervise_server.py +++ b/bot_bottle/supervise_server.py @@ -26,9 +26,8 @@ Speaks MCP over HTTP+JSON-RPC. Methods handled: Everything else returns JSON-RPC error -32601 (method not found). -Stdlib-only. The Dockerfile copies this file + bot_bottle/supervise.py -into the image; the server imports `supervise` for the queue / Proposal -plumbing. +The Dockerfile copies this script to /app/supervise_server.py and installs +the bot_bottle package so its `from bot_bottle.*` imports resolve. """ from __future__ import annotations @@ -42,29 +41,18 @@ import time import typing from dataclasses import dataclass, replace -try: - # Same-directory imports inside the bundle container; these files are - # COPYed flat under /app by Dockerfile.gateway. - from egress_addon_core import ( - LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict, - ) - from policy_resolver import PolicyResolveError, PolicyResolver - import supervise as _sv -except ModuleNotFoundError: - # Package imports for host-side tests and tooling. - from .egress_addon_core import ( - LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict, - ) - from .policy_resolver import PolicyResolveError, PolicyResolver - from . import supervise as _sv +from bot_bottle.constants import IDENTITY_HEADER +from bot_bottle.egress_addon_core import ( + LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict, +) +from bot_bottle.policy_resolver import PolicyResolveError, PolicyResolver +from bot_bottle import supervise as _sv # --- JSON-RPC / MCP plumbing ---------------------------------------------- MCP_PROTOCOL_VERSION = "2024-11-05" -# App-layer identity token header (mirrors egress_addon / git_http_backend). -IDENTITY_HEADER = "x-bot-bottle-identity" SERVER_NAME = "bot-bottle-supervise" SERVER_VERSION = "0.1.0" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ab1a97b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.backends.legacy:build" + +[project] +name = "bot-bottle" +version = "0.0.0" +requires-python = ">=3.11" diff --git a/tests/unit/test_egress_addon_log_redaction.py b/tests/unit/test_egress_addon_log_redaction.py index fe9b7c6..8b64f67 100644 --- a/tests/unit/test_egress_addon_log_redaction.py +++ b/tests/unit/test_egress_addon_log_redaction.py @@ -18,7 +18,7 @@ from unittest.mock import patch # --------------------------------------------------------------------------- -# Gateway-import shims — must run before importing egress_addon +# mitmproxy stub — must run before importing egress_addon # --------------------------------------------------------------------------- def _ensure_shims() -> None: @@ -32,9 +32,6 @@ def _ensure_shims() -> None: setattr(_mm, "http", _mh) sys.modules["mitmproxy"] = _mm sys.modules["mitmproxy.http"] = _mh - if "egress_addon_core" not in sys.modules: - import bot_bottle.egress_addon_core as _core - sys.modules["egress_addon_core"] = _core _ensure_shims() diff --git a/tests/unit/test_egress_addon_request_flow.py b/tests/unit/test_egress_addon_request_flow.py index 33eb3d0..2d834e6 100644 --- a/tests/unit/test_egress_addon_request_flow.py +++ b/tests/unit/test_egress_addon_request_flow.py @@ -190,9 +190,6 @@ def _ensure_shims() -> None: setattr(mh, "Response", _Response) if not hasattr(mh, "HTTPFlow"): setattr(mh, "HTTPFlow", object) - if "egress_addon_core" not in sys.modules: - import bot_bottle.egress_addon_core as _core - sys.modules["egress_addon_core"] = _core _ensure_shims() diff --git a/tests/unit/test_supervise_server.py b/tests/unit/test_supervise_server.py index 64d8051..1ba0f77 100644 --- a/tests/unit/test_supervise_server.py +++ b/tests/unit/test_supervise_server.py @@ -2,7 +2,6 @@ import http.client import json -import sys import tempfile import threading import time @@ -13,15 +12,9 @@ from unittest.mock import patch from tests.unit import use_bottle_root - -# The server module loads `supervise` via same-directory import inside -# the container (Dockerfile.supervise WORKDIRs into /app). For tests -# we mirror that by injecting bot_bottle/ onto sys.path under the -# bare name `supervise`. -sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent / "bot_bottle")) -import supervise as _sv # noqa: E402 # type: ignore -import queue_store as _qs # noqa: E402 # type: ignore -import audit_store as _as # noqa: E402 # type: ignore +from bot_bottle import supervise as _sv +from bot_bottle import queue_store as _qs +from bot_bottle import audit_store as _as from bot_bottle import supervise_server # noqa: E402 from bot_bottle.supervise_server import (