refactor(gateway): split data-plane files into egress/supervisor/git_gate services
tracker-policy-pr / check-pr (pull_request) Successful in 11s
test / integration-docker (pull_request) Successful in 17s
test / unit (pull_request) Successful in 49s
lint / lint (push) Failing after 2m49s
test / integration-firecracker (pull_request) Successful in 3m35s
test / coverage (pull_request) Successful in 18s
test / publish-infra (pull_request) Has been skipped
tracker-policy-pr / check-pr (pull_request) Successful in 11s
test / integration-docker (pull_request) Successful in 17s
test / unit (pull_request) Successful in 49s
lint / lint (push) Failing after 2m49s
test / integration-firecracker (pull_request) Successful in 3m35s
test / coverage (pull_request) Successful in 18s
test / publish-infra (pull_request) Has been skipped
Group the gateway's data-plane modules into three service sub-packages mirroring the host-side trio (bot_bottle.egress / .supervisor / .git_gate): gateway/egress/ addon_core, addon, dlp_config, dlp_detectors gateway/supervisor/ server (was supervise_server) gateway/git_gate/ render, http_backend Prefix-stripped filenames now that the package namespaces them; each sub-package has a thin docstring __init__ (no eager imports, cheap leaf loads). The two cross-cutting files stay at the gateway root: policy_resolver (shared per-client lookup) and gateway_init, renamed to bootstrap now that gateway/ already namespaces it. Updated all importers (bot_bottle + tests), the in-VM/container `-m` launch strings, the Dockerfile.gateway addon shim + ENTRYPOINT, and the five gateway entries in scripts/critical-modules.txt. Full unit suite green (2243). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -11,7 +11,7 @@ from pathlib import Path
|
||||
|
||||
from ..bottle_state import egress_state_dir
|
||||
from ..egress import EGRESS_ROUTES_FILENAME
|
||||
from ..gateway.egress_addon_core import LOG_OFF, load_config
|
||||
from ..gateway.egress.addon_core import LOG_OFF, load_config
|
||||
|
||||
|
||||
class EgressApplyError(RuntimeError):
|
||||
|
||||
@@ -542,7 +542,7 @@ BOT_BOTTLE_ROOT=/var/lib/bot-bottle BOT_BOTTLE_CONTROL_PLANE_TOKEN="$CP_KEY" pyt
|
||||
BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise \\
|
||||
BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{CONTROL_PLANE_PORT} \\
|
||||
BOT_BOTTLE_CONTROL_AUTH_JWT="$GW_JWT" \\
|
||||
python3 -m bot_bottle.gateway.gateway_init &
|
||||
python3 -m bot_bottle.gateway.bootstrap &
|
||||
|
||||
# Reap as PID 1; children are backgrounded, so `wait` blocks.
|
||||
while : ; do wait ; done
|
||||
|
||||
@@ -107,7 +107,7 @@ def _init_script(port: int) -> str:
|
||||
# control-plane RPC and never opens bot-bottle.db (PRD 0070 / #469).
|
||||
f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} "
|
||||
f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} "
|
||||
f"python3 -m bot_bottle.gateway.gateway_init ) &\n"
|
||||
f"python3 -m bot_bottle.gateway.bootstrap ) &\n"
|
||||
"while : ; do wait ; done\n"
|
||||
)
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ from ...bottle_state import (
|
||||
)
|
||||
from ...egress import Egress
|
||||
from ...git_gate import GitGate
|
||||
from ...gateway.git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT
|
||||
from ...gateway.git_gate.http_backend import DEFAULT_PORT as _GIT_HTTP_PORT
|
||||
from ...image_cache import check_stale
|
||||
from ...log import die, info, warn
|
||||
from .. import BottleImages
|
||||
|
||||
@@ -13,7 +13,7 @@ Layout:
|
||||
contract).
|
||||
|
||||
The runtime enforcement (the mitmproxy addon) lives in
|
||||
`bot_bottle.gateway.egress_addon*`. Public names are re-exported lazily via
|
||||
`bot_bottle.gateway.egress.addon*`. Public names are re-exported lazily via
|
||||
`__getattr__`, so `from bot_bottle.egress import …` keeps working and importing
|
||||
`egress.plan` (the contract's dependency) stays light.
|
||||
"""
|
||||
|
||||
@@ -11,7 +11,7 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ..gateway.egress_addon_core import Route
|
||||
from ..gateway.egress.addon_core import Route
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
routes, render the gateway's `routes.yaml`, assign per-route token slots, and
|
||||
plant the exfil canary. The service also resolves the launch-time token values
|
||||
and the agent/gateway env entries the backend injects. The runtime enforcement
|
||||
(the mitmproxy addon) lives in `bot_bottle.gateway.egress_addon*`.
|
||||
(the mitmproxy addon) lives in `bot_bottle.gateway.egress.addon*`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -14,7 +14,7 @@ import secrets
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from ..gateway.egress_addon_core import (
|
||||
from ..gateway.egress.addon_core import (
|
||||
ON_MATCH_REDACT,
|
||||
HeaderMatch as CoreHeaderMatch,
|
||||
MatchEntry as CoreMatchEntry,
|
||||
|
||||
@@ -100,8 +100,8 @@ _DAEMONS: tuple[_DaemonSpec, ...] = (
|
||||
)),
|
||||
_DaemonSpec("egress", ("/bin/sh", "/app/egress-entrypoint.sh")),
|
||||
_DaemonSpec("git-gate", ("/bin/sh", "/git-gate-entrypoint.sh")),
|
||||
_DaemonSpec("git-http", ("python3", "-m", "bot_bottle.gateway.git_http_backend")),
|
||||
_DaemonSpec("supervise", ("python3", "-m", "bot_bottle.gateway.supervise_server")),
|
||||
_DaemonSpec("git-http", ("python3", "-m", "bot_bottle.gateway.git_gate.http_backend")),
|
||||
_DaemonSpec("supervise", ("python3", "-m", "bot_bottle.gateway.supervisor.server")),
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
"""Gateway-side (data-plane) egress service: the mitmproxy addon and its
|
||||
pure decision core, DLP detectors, and route/DLP config parsing.
|
||||
|
||||
These are the long-running data-plane pieces (loaded by mitmdump inside the
|
||||
`bot-bottle-gateway` image), distinct from the host-side `bot_bottle.egress`
|
||||
service that renders routes and prepares env. Import the concrete modules
|
||||
directly (`from bot_bottle.gateway.egress.addon_core import ...`) — this
|
||||
package deliberately does no eager work so leaf imports stay cheap.
|
||||
"""
|
||||
@@ -16,8 +16,8 @@ import typing
|
||||
from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=import-error
|
||||
|
||||
from bot_bottle.constants import IDENTITY_HEADER
|
||||
from bot_bottle.gateway.dlp_detectors import redact_tokens, strip_crlf
|
||||
from bot_bottle.gateway.egress_addon_core import (
|
||||
from bot_bottle.gateway.egress.dlp_detectors import redact_tokens, strip_crlf
|
||||
from bot_bottle.gateway.egress.addon_core import (
|
||||
LOG_BLOCKS,
|
||||
LOG_FULL,
|
||||
DEFAULT_OUTBOUND_ON_MATCH,
|
||||
@@ -16,11 +16,11 @@ import re
|
||||
import typing
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..yaml_subset import YamlSubsetError, parse_yaml_subset
|
||||
from ...yaml_subset import YamlSubsetError, parse_yaml_subset
|
||||
|
||||
# DLP detector-config parsing lives in a sibling module. Re-exported below
|
||||
# so existing `from egress_addon_core import ON_MATCH_*` callers keep working.
|
||||
from .egress_dlp_config import (
|
||||
from .dlp_config import (
|
||||
DEFAULT_OUTBOUND_ON_MATCH,
|
||||
INBOUND_DETECTOR_NAMES,
|
||||
ON_MATCH_BLOCK,
|
||||
@@ -19,7 +19,7 @@ from math import log2
|
||||
from collections import Counter
|
||||
from urllib.parse import quote as url_quote
|
||||
|
||||
from .egress_addon_core import ScanResult
|
||||
from .addon_core import ScanResult
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Gateway-side (data-plane) git-gate service: pure host-side hook rendering
|
||||
(PRD 0008) and the smart-HTTP backend that fronts git-gate repos.
|
||||
|
||||
The host-side git-gate service (provisioning, preflight) lives in
|
||||
`bot_bottle.git_gate`; this is the data-plane counterpart. Import the concrete
|
||||
modules directly (`from bot_bottle.gateway.git_gate.render import ...`).
|
||||
"""
|
||||
@@ -14,8 +14,8 @@ import shlex
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ..constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER
|
||||
from ..manifest import ManifestBottle, ManifestGitEntry
|
||||
from ...constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER
|
||||
from ...manifest import ManifestBottle, ManifestGitEntry
|
||||
|
||||
# Short network alias for git-gate inside the gateway. The
|
||||
# agent's `.gitconfig` insteadOf rewrites resolve through this name.
|
||||
@@ -0,0 +1,7 @@
|
||||
"""Gateway-side (data-plane) supervise service: the supervise daemon's HTTP
|
||||
server (PRD 0013).
|
||||
|
||||
The host-side supervise control lives in `bot_bottle.orchestrator.supervisor`;
|
||||
this is the in-gateway daemon the agents reach. Import the concrete module
|
||||
directly (`from bot_bottle.gateway.supervisor.server import ...`).
|
||||
"""
|
||||
@@ -58,7 +58,7 @@ import typing
|
||||
from dataclasses import dataclass
|
||||
|
||||
from bot_bottle.constants import IDENTITY_HEADER
|
||||
from bot_bottle.gateway.egress_addon_core import (
|
||||
from bot_bottle.gateway.egress.addon_core import (
|
||||
LOG_OFF, load_config, resolve_client_context, route_to_yaml_dict,
|
||||
)
|
||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||
@@ -14,7 +14,7 @@ Layout:
|
||||
service delegates to.
|
||||
|
||||
The rendering + the in-gateway hook execution live in
|
||||
`bot_bottle.gateway.git_gate_render`. The public names are re-exported lazily
|
||||
`bot_bottle.gateway.git_gate.render`. The public names are re-exported lazily
|
||||
via `__getattr__`, so `from bot_bottle.git_gate import …` keeps working and
|
||||
importing `git_gate.plan` (the contract's dependency) doesn't drag in the
|
||||
provisioning / forge-API code.
|
||||
@@ -31,7 +31,7 @@ if TYPE_CHECKING:
|
||||
provision_git_gate_dynamic_keys,
|
||||
revoke_git_gate_provisioned_keys,
|
||||
)
|
||||
from ..gateway.git_gate_render import (
|
||||
from ..gateway.git_gate.render import (
|
||||
GIT_GATE_HOSTNAME,
|
||||
GIT_GATE_TIMEOUT_SECS,
|
||||
GitGateUpstream,
|
||||
@@ -55,17 +55,17 @@ _LAZY: dict[str, str] = {
|
||||
"revoke_git_gate_provisioned_keys": ".provision",
|
||||
"_provision_dynamic_key": ".provision",
|
||||
"_resolve_identity_file": ".provision",
|
||||
"GIT_GATE_HOSTNAME": "..gateway.git_gate_render",
|
||||
"GIT_GATE_TIMEOUT_SECS": "..gateway.git_gate_render",
|
||||
"GitGateUpstream": "..gateway.git_gate_render",
|
||||
"git_gate_upstreams_for_bottle": "..gateway.git_gate_render",
|
||||
"git_gate_render_gitconfig": "..gateway.git_gate_render",
|
||||
"git_gate_known_hosts_line": "..gateway.git_gate_render",
|
||||
"git_gate_render_entrypoint": "..gateway.git_gate_render",
|
||||
"git_gate_render_provision": "..gateway.git_gate_render",
|
||||
"git_gate_render_hook": "..gateway.git_gate_render",
|
||||
"git_gate_render_access_hook": "..gateway.git_gate_render",
|
||||
"_gitconfig_validate_value": "..gateway.git_gate_render",
|
||||
"GIT_GATE_HOSTNAME": "..gateway.git_gate.render",
|
||||
"GIT_GATE_TIMEOUT_SECS": "..gateway.git_gate.render",
|
||||
"GitGateUpstream": "..gateway.git_gate.render",
|
||||
"git_gate_upstreams_for_bottle": "..gateway.git_gate.render",
|
||||
"git_gate_render_gitconfig": "..gateway.git_gate.render",
|
||||
"git_gate_known_hosts_line": "..gateway.git_gate.render",
|
||||
"git_gate_render_entrypoint": "..gateway.git_gate.render",
|
||||
"git_gate_render_provision": "..gateway.git_gate.render",
|
||||
"git_gate_render_hook": "..gateway.git_gate.render",
|
||||
"git_gate_render_access_hook": "..gateway.git_gate.render",
|
||||
"_gitconfig_validate_value": "..gateway.git_gate.render",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ..gateway.git_gate_render import GitGateUpstream
|
||||
from ..gateway.git_gate.render import GitGateUpstream
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -17,7 +17,7 @@ from ..bottle_state import globalize_slug
|
||||
from ..errors import MissingEnvVarError
|
||||
from ..log import info
|
||||
from ..manifest import ManifestBottle, ManifestGitEntry
|
||||
from ..gateway.git_gate_render import GitGateUpstream
|
||||
from ..gateway.git_gate.render import GitGateUpstream
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .plan import GitGatePlan
|
||||
|
||||
@@ -8,7 +8,7 @@ upstream before fetches. The agent never sees the upstream credential.
|
||||
`GitGate` is the host-side service the backend drives at launch: build the plan
|
||||
(`prepare`), provision / revoke the per-upstream deploy keys, and preflight the
|
||||
upstream host keys. The rendering it emits + the in-gateway hook execution live
|
||||
in `bot_bottle.gateway.git_gate_render`.
|
||||
in `bot_bottle.gateway.git_gate.render`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -16,7 +16,7 @@ from __future__ import annotations
|
||||
from pathlib import Path
|
||||
|
||||
from ..manifest import Manifest, ManifestBottle
|
||||
from ..gateway.git_gate_render import (
|
||||
from ..gateway.git_gate.render import (
|
||||
GitGateUpstream,
|
||||
git_gate_known_hosts_line,
|
||||
git_gate_render_access_hook,
|
||||
|
||||
Reference in New Issue
Block a user