refactor(gateway): split data-plane files into egress/supervisor/git_gate services
tracker-policy-pr / check-pr (pull_request) Successful in 11s
test / integration-docker (pull_request) Successful in 17s
test / unit (pull_request) Successful in 49s
lint / lint (push) Failing after 2m49s
test / integration-firecracker (pull_request) Successful in 3m35s
test / coverage (pull_request) Successful in 18s
test / publish-infra (pull_request) Has been skipped

Group the gateway's data-plane modules into three service sub-packages
mirroring the host-side trio (bot_bottle.egress / .supervisor / .git_gate):

  gateway/egress/     addon_core, addon, dlp_config, dlp_detectors
  gateway/supervisor/ server            (was supervise_server)
  gateway/git_gate/   render, http_backend

Prefix-stripped filenames now that the package namespaces them; each
sub-package has a thin docstring __init__ (no eager imports, cheap leaf
loads). The two cross-cutting files stay at the gateway root:
policy_resolver (shared per-client lookup) and gateway_init, renamed to
bootstrap now that gateway/ already namespaces it.

Updated all importers (bot_bottle + tests), the in-VM/container `-m`
launch strings, the Dockerfile.gateway addon shim + ENTRYPOINT, and the
five gateway entries in scripts/critical-modules.txt. Full unit suite
green (2243).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-24 17:00:51 -04:00
parent a446551acb
commit ce744a85c4
40 changed files with 113 additions and 90 deletions
+4 -4
View File
@@ -7,7 +7,7 @@ import base64
import gzip
import unittest
from bot_bottle.gateway.dlp_detectors import (
from bot_bottle.gateway.egress.dlp_detectors import (
ENTROPY_BLOCK_THRESHOLD,
PARTIAL_MATCH_MIN_LEN,
REDACT,
@@ -465,17 +465,17 @@ class TestMatchedAndSafeTokens(unittest.TestCase):
class TestStripCrlf(unittest.TestCase):
def test_removes_url_encoded_crlf(self):
from bot_bottle.gateway.dlp_detectors import strip_crlf
from bot_bottle.gateway.egress.dlp_detectors import strip_crlf
out = strip_crlf("next=%0d%0aX-Injected: evil")
self.assertNotRegex(out, r"%0[dD]%0[aA]")
def test_removes_literal_header_injection(self):
from bot_bottle.gateway.dlp_detectors import strip_crlf
from bot_bottle.gateway.egress.dlp_detectors import strip_crlf
out = strip_crlf("value\r\nX-Injected: evil")
self.assertIsNone(scan_crlf_injection(out))
def test_leaves_clean_text_unchanged(self):
from bot_bottle.gateway.dlp_detectors import strip_crlf
from bot_bottle.gateway.egress.dlp_detectors import strip_crlf
self.assertEqual("/api/v1/data?q=hello", strip_crlf("/api/v1/data?q=hello"))
class TestAlnumProjection(unittest.TestCase):
+11 -11
View File
@@ -344,7 +344,7 @@ class TestRenderRoutes(unittest.TestCase):
self.assertEqual([], parse_yaml_subset(rendered)["routes"])
def test_round_trip_through_addon_core(self):
from bot_bottle.gateway.egress_addon_core import load_config
from bot_bottle.gateway.egress.addon_core import load_config
b = _bottle([
{"host": "api.github.com",
"auth": {"scheme": "Bearer", "token_ref": "GH_PAT"},
@@ -363,7 +363,7 @@ class TestRenderRoutes(unittest.TestCase):
self.assertEqual("", addon_routes[2].auth_scheme)
def test_dlp_round_trips(self):
from bot_bottle.gateway.egress_addon_core import load_config
from bot_bottle.gateway.egress.addon_core import load_config
b = _bottle([{"host": "x.example", "dlp": {
"outbound_detectors": ["token_patterns"],
"inbound_detectors": False,
@@ -375,7 +375,7 @@ class TestRenderRoutes(unittest.TestCase):
self.assertEqual((), addon_routes[0].inbound_detectors)
def test_outbound_on_match_round_trips(self):
from bot_bottle.gateway.egress_addon_core import load_config
from bot_bottle.gateway.egress.addon_core import load_config
b = _bottle([{"host": "logs.example", "dlp": {
"outbound_on_match": "redact",
}}])
@@ -392,7 +392,7 @@ class TestRenderRoutes(unittest.TestCase):
self.assertNotIn("outbound_on_match", rendered)
def test_git_fetch_policy_round_trips(self):
from bot_bottle.gateway.egress_addon_core import load_config
from bot_bottle.gateway.egress.addon_core import load_config
b = _bottle([{"host": "github.com", "git": {"fetch": True}}])
routes = egress_routes_for_bottle(b)
rendered = egress_render_routes(routes)
@@ -405,7 +405,7 @@ class TestRenderRoutes(unittest.TestCase):
it, but the renderer in between dropped it — so the flag never reached
the proxy and registry pulls kept failing with "unauthorized" while
the config looked correct everywhere it was inspected."""
from bot_bottle.gateway.egress_addon_core import load_config
from bot_bottle.gateway.egress.addon_core import load_config
b = _bottle([{"host": "registry-1.docker.io", "preserve_auth": True}])
routes = egress_routes_for_bottle(b)
rendered = egress_render_routes(routes)
@@ -416,7 +416,7 @@ class TestRenderRoutes(unittest.TestCase):
b = _bottle([{"host": "x.example"}])
rendered = egress_render_routes(egress_routes_for_bottle(b))
self.assertNotIn("preserve_auth", rendered)
from bot_bottle.gateway.egress_addon_core import load_config
from bot_bottle.gateway.egress.addon_core import load_config
self.assertFalse(load_config(rendered).routes[0].preserve_auth)
def test_log_zero_omitted_from_render(self):
@@ -434,7 +434,7 @@ class TestRenderRoutes(unittest.TestCase):
self.assertTrue(rendered.startswith(f"log: {level}\n"))
def test_log_level_round_trips_to_addon_core(self):
from bot_bottle.gateway.egress_addon_core import load_config, LOG_FULL
from bot_bottle.gateway.egress.addon_core import load_config, LOG_FULL
b = _bottle([{"host": "x.example"}])
routes = egress_routes_for_bottle(b)
rendered = egress_render_routes(routes, log=LOG_FULL)
@@ -444,7 +444,7 @@ class TestRenderRoutes(unittest.TestCase):
def test_log_via_manifest_flows_to_render(self):
from bot_bottle.manifest import ManifestIndex
from bot_bottle.gateway.egress_addon_core import load_config, LOG_BLOCKS
from bot_bottle.gateway.egress.addon_core import load_config, LOG_BLOCKS
m = ManifestIndex.from_json_obj({
"bottles": {"dev": {"egress": {
"log": 1,
@@ -512,7 +512,7 @@ class TestRenderRoutesEscaping(unittest.TestCase):
self.assertEqual('Bear"er', parsed[0]["inspect"]["auth_scheme"])
def test_path_value_with_double_quote_round_trips(self):
from bot_bottle.gateway.egress_addon_core import PathMatch, MatchEntry
from bot_bottle.gateway.egress.addon_core import PathMatch, MatchEntry
routes = (EgressRoute(
host="api.example",
matches=(MatchEntry(paths=(PathMatch(type="prefix", value='/v1/"quoted"/'),)),),
@@ -521,7 +521,7 @@ class TestRenderRoutesEscaping(unittest.TestCase):
self.assertEqual('/v1/"quoted"/', parsed[0]["inspect"]["matches"][0]["paths"][0]["value"])
def test_header_value_with_double_quote_round_trips(self):
from bot_bottle.gateway.egress_addon_core import HeaderMatch, MatchEntry
from bot_bottle.gateway.egress.addon_core import HeaderMatch, MatchEntry
routes = (EgressRoute(
host="api.example",
matches=(MatchEntry(headers=(HeaderMatch(name="x-h", value='val"ue'),)),),
@@ -598,7 +598,7 @@ class TestCanaryGeneration(unittest.TestCase):
self.assertNotEqual(plan_a.canary, plan_b.canary)
def test_canary_detected_by_scan_known_secrets(self):
from bot_bottle.gateway.dlp_detectors import scan_known_secrets
from bot_bottle.gateway.egress.dlp_detectors import scan_known_secrets
plan = self._make_plan()
env = {plan.canary_env: plan.canary}
+4 -4
View File
@@ -12,7 +12,7 @@ import unittest
from pathlib import Path
from urllib.parse import urlsplit
from bot_bottle.gateway.egress_addon_core import (
from bot_bottle.gateway.egress.addon_core import (
LOG_BLOCKS,
LOG_FULL,
LOG_OFF,
@@ -1377,15 +1377,15 @@ class TestScanOutboundEnhanced(unittest.TestCase):
class TestOutboundDetectorNames(unittest.TestCase):
def test_entropy_in_outbound_detector_names(self):
from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES
from bot_bottle.gateway.egress.addon_core import OUTBOUND_DETECTOR_NAMES
self.assertIn("entropy", OUTBOUND_DETECTOR_NAMES)
def test_known_secrets_in_outbound_detector_names(self):
from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES
from bot_bottle.gateway.egress.addon_core import OUTBOUND_DETECTOR_NAMES
self.assertIn("known_secrets", OUTBOUND_DETECTOR_NAMES)
def test_token_patterns_in_outbound_detector_names(self):
from bot_bottle.gateway.egress_addon_core import OUTBOUND_DETECTOR_NAMES
from bot_bottle.gateway.egress.addon_core import OUTBOUND_DETECTOR_NAMES
self.assertIn("token_patterns", OUTBOUND_DETECTOR_NAMES)
@@ -36,7 +36,7 @@ def _ensure_shims() -> None:
_ensure_shims()
from bot_bottle.gateway.egress_addon import EgressAddon # noqa: E402 (import after shims)
from bot_bottle.gateway.egress.addon import EgressAddon # noqa: E402 (import after shims)
# ---------------------------------------------------------------------------
+4 -4
View File
@@ -194,15 +194,15 @@ def _ensure_shims() -> None:
_ensure_shims()
import bot_bottle.gateway.egress_addon as _ea_mod # noqa: E402 (after shims)
from bot_bottle.gateway.egress_addon import EgressAddon # noqa: E402 (after shims)
from bot_bottle.gateway.egress_addon import ( # noqa: E402
import bot_bottle.gateway.egress.addon as _ea_mod # noqa: E402 (after shims)
from bot_bottle.gateway.egress.addon import EgressAddon # noqa: E402 (after shims)
from bot_bottle.gateway.egress.addon import ( # noqa: E402
DEFAULT_INBOUND_SCAN_LIMIT_BYTES,
DEFAULT_TOKEN_ALLOW_TIMEOUT_SECONDS,
_inbound_scan_limit_from_env,
_token_allow_timeout_from_env,
)
from bot_bottle.gateway.egress_addon_core import ( # noqa: E402
from bot_bottle.gateway.egress.addon_core import ( # noqa: E402
Config,
LOG_BLOCKS,
LOG_FULL,
+1 -1
View File
@@ -8,7 +8,7 @@ from __future__ import annotations
import unittest
from bot_bottle.gateway.egress_addon_core import (
from bot_bottle.gateway.egress.addon_core import (
HeaderMatch,
MatchEntry,
PathMatch,
+1 -1
View File
@@ -4,7 +4,7 @@ from __future__ import annotations
import unittest
from bot_bottle.gateway.egress_addon_core import (
from bot_bottle.gateway.egress.addon_core import (
DENY_RESOLVER_ERROR,
DENY_UNATTRIBUTED,
DENY_UNPARSEABLE,
+1 -1
View File
@@ -72,7 +72,7 @@ class TestBuildInfraRootfs(unittest.TestCase):
self.assertIn("bot_bottle.orchestrator", init)
# Gateway launches via the installed package (there is no
# /app/gateway_init.py file since the daemons moved into bot_bottle).
self.assertIn("bot_bottle.gateway.gateway_init", init)
self.assertIn("bot_bottle.gateway.bootstrap", init)
self.assertIn("export PATH=", init)
# Persistent registry volume mounted at the DB dir before the CP starts.
self.assertIn("/dev/vdb", init)
+4 -4
View File
@@ -1,6 +1,6 @@
"""Unit: gateway data-plane init supervisor (PRD 0070; PRD 0024 bundle shape).
Tests both the helper functions in `bot_bottle.gateway.gateway_init`
Tests both the helper functions in `bot_bottle.gateway.bootstrap`
and the supervisor's end-to-end signal / exit-code behavior. The
end-to-end tests use real subprocesses (`sleep`, `/bin/sh -c '...'`)
short-lived, no docker required so they run under `tests/unit/`
@@ -18,7 +18,7 @@ import warnings
from pathlib import Path
from unittest.mock import patch
from bot_bottle.gateway.gateway_init import (
from bot_bottle.gateway.bootstrap import (
_DaemonSpec,
_Supervisor,
_argv_for_daemon,
@@ -489,7 +489,7 @@ class TestSupervisor(unittest.TestCase):
time.sleep(0.3) # let `trap` register
sup.request_shutdown(reason="test")
with patch("bot_bottle.gateway.gateway_init._GRACE_SECONDS", 0.3):
with patch("bot_bottle.gateway.bootstrap._GRACE_SECONDS", 0.3):
rc = self._drive(sup, max_wait_s=4.0)
# Process was SIGKILL'd → returncode -9 on POSIX.
@@ -531,7 +531,7 @@ class TestMainEndToEnd(unittest.TestCase):
helper = (
"import os, runpy, sys\n"
"from bot_bottle.gateway import gateway_init as si\n"
"from bot_bottle.gateway import bootstrap as si\n"
"si._DAEMONS = (\n"
f" si._DaemonSpec('alpha', ({SLEEP!r},'30')),\n"
f" si._DaemonSpec('beta', ({SLEEP!r},'30')),\n"
+1 -1
View File
@@ -4,7 +4,7 @@ from __future__ import annotations
import unittest
from bot_bottle.gateway.git_gate_render import (
from bot_bottle.gateway.git_gate.render import (
GitGateUpstream,
git_gate_render_entrypoint,
git_gate_render_provision,
+8 -8
View File
@@ -10,7 +10,7 @@ from pathlib import Path
from unittest import mock
from bot_bottle.git_gate import GIT_GATE_TIMEOUT_SECS
from bot_bottle.gateway.git_http_backend import GitHttpHandler, MAX_BODY_BYTES
from bot_bottle.gateway.git_gate.http_backend import GitHttpHandler, MAX_BODY_BYTES
# The git-http backend is resolver-only: every request is attributed to a
@@ -199,7 +199,7 @@ class TestGitHttpBackend(unittest.TestCase):
subprocess.CompletedProcess(["git"], 0, backend_response, b""),
]
with mock.patch(
"bot_bottle.gateway.git_http_backend.subprocess.run",
"bot_bottle.gateway.git_gate.http_backend.subprocess.run",
side_effect=calls,
) as run:
request = urllib.request.Request(
@@ -265,7 +265,7 @@ class TestGitHttpBackend(unittest.TestCase):
subprocess.CompletedProcess(["git"], 0, backend_response, b""),
]
with mock.patch(
"bot_bottle.gateway.git_http_backend.subprocess.run",
"bot_bottle.gateway.git_gate.http_backend.subprocess.run",
side_effect=calls,
) as run:
req = urllib.request.Request(
@@ -309,7 +309,7 @@ class TestGitHttpBackend(unittest.TestCase):
denial = b"git-gate: upstream fetch failed; refusing to serve stale data\n"
with mock.patch(
"bot_bottle.gateway.git_http_backend.subprocess.run",
"bot_bottle.gateway.git_gate.http_backend.subprocess.run",
return_value=subprocess.CompletedProcess(
["hook"], 1, b"", denial,
),
@@ -355,7 +355,7 @@ class TestGitHttpBackend(unittest.TestCase):
self.addCleanup(server.server_close)
with mock.patch(
"bot_bottle.gateway.git_http_backend.subprocess.run",
"bot_bottle.gateway.git_gate.http_backend.subprocess.run",
return_value=subprocess.CompletedProcess(
["hook"], 2, b"", b"",
),
@@ -402,7 +402,7 @@ class TestGitHttpBackend(unittest.TestCase):
self.addCleanup(server.server_close)
with mock.patch(
"bot_bottle.gateway.git_http_backend.subprocess.run",
"bot_bottle.gateway.git_gate.http_backend.subprocess.run",
side_effect=PermissionError(13, "Permission denied"),
):
buf = io.StringIO()
@@ -461,7 +461,7 @@ class TestMalformedStatusHeader(unittest.TestCase):
def _get_with_backend_response(self, cgi_response: bytes) -> int:
with mock.patch(
"bot_bottle.gateway.git_http_backend.subprocess.run",
"bot_bottle.gateway.git_gate.http_backend.subprocess.run",
return_value=mock.Mock(returncode=0, stdout=cgi_response),
):
req = urllib.request.Request(
@@ -545,7 +545,7 @@ class TestContentLengthBounds(unittest.TestCase):
# With a valid Content-Length the handler proceeds into
# git http-backend; that will fail (no real git repo) but the
# status won't be 400 or 413.
with mock.patch("bot_bottle.gateway.git_http_backend.subprocess.run") as run:
with mock.patch("bot_bottle.gateway.git_gate.http_backend.subprocess.run") as run:
run.return_value = mock.Mock(
returncode=0,
stdout=(
+1 -1
View File
@@ -10,7 +10,7 @@ from __future__ import annotations
import unittest
from pathlib import Path
from bot_bottle.gateway.git_http_backend import resolve_sandbox_root
from bot_bottle.gateway.git_gate.http_backend import resolve_sandbox_root
from bot_bottle.gateway.policy_resolver import PolicyResolveError
_BASE = Path("/git")
+1 -1
View File
@@ -48,7 +48,7 @@ class TestInfraRun(unittest.TestCase):
self.assertIn("bot_bottle.orchestrator", script)
# Gateway launches via the installed package (there is no
# /app/gateway_init.py file since the daemons moved into bot_bottle).
self.assertIn("bot_bottle.gateway.gateway_init", script)
self.assertIn("bot_bottle.gateway.bootstrap", script)
self.assertIn("127.0.0.1", script) # they reach each other on loopback
def test_db_is_a_container_only_volume(self) -> None:
+1 -1
View File
@@ -7,7 +7,7 @@ import unittest
from pathlib import Path
from bot_bottle.egress import EgressPlan, EgressRoute
from bot_bottle.gateway.egress_addon_core import LOG_BLOCKS, load_config
from bot_bottle.gateway.egress.addon_core import LOG_BLOCKS, load_config
from bot_bottle.orchestrator.registration import (
RegistrationInputs,
egress_policy,
+2 -2
View File
@@ -22,8 +22,8 @@ from bot_bottle.orchestrator import supervisor as _sv
from bot_bottle.orchestrator.store import queue_store as _qs
from bot_bottle.store import audit_store as _as
from bot_bottle.gateway import supervise_server # noqa: E402
from bot_bottle.gateway.supervise_server import (
from bot_bottle.gateway.supervisor import server as supervise_server # noqa: E402
from bot_bottle.gateway.supervisor.server import (
ERR_INTERNAL,
ERR_INVALID_PARAMS,
ERR_INVALID_REQUEST,