fix(supervise): get bot-bottle.db off the data plane (supervise + egress proposals over RPC) #471

Open
didericis-claude wants to merge 44 commits from fix/db-off-data-plane-469 into main
8 changed files with 176 additions and 80 deletions
Showing only changes of commit a446551acb - Show all commits
@@ -16,7 +16,7 @@ from __future__ import annotations
from typing import Any
from ...egress import egress_agent_env_entries
from ...egress import Egress
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from .bottle_plan import DockerBottlePlan
@@ -63,7 +63,7 @@ def consolidated_agent_compose(
# env (set in launch.py) and is never written to the compose file on disk.
if getattr(plan, "env_var_secret", ""):
env.append(ENV_VAR_SECRET_NAME)
env.extend(egress_agent_env_entries(plan.egress_plan))
env.extend(Egress().agent_env_entries(plan.egress_plan))
service: dict[str, Any] = {
"image": plan.image,
+2 -2
View File
@@ -37,7 +37,7 @@ from pathlib import Path
from typing import Callable, Generator
from ...agent_provider import runtime_for
from ...egress import egress_resolve_token_values
from ...egress import Egress
from ...git_gate import GitGate
from ...image_cache import check_stale
from ...log import die, info, warn
@@ -143,7 +143,7 @@ def launch(
# handed to the orchestrator (in memory) for the gateway to inject —
# the agent never sees them.
effective_env = {**os.environ, **plan.agent_provision.provisioned_env}
token_values = egress_resolve_token_values(
token_values = Egress().resolve_token_values(
plan.egress_plan.token_env_map, effective_env,
)
teardown_timeout = resolve_teardown_timeout()
+3 -6
View File
@@ -38,10 +38,7 @@ from ...bottle_state import (
git_gate_state_dir,
read_committed_image,
)
from ...egress import (
egress_agent_env_entries,
egress_resolve_token_values,
)
from ...egress import Egress
from ...git_gate import GitGate
from ...image_cache import check_stale_path
from ...log import die, info, warn
@@ -110,7 +107,7 @@ def launch(
# (in memory) for the gateway to inject — the agent never sees them.
# Attribution is by the VM's guest IP (unspoofable via /31 TAP + nft).
effective_env = {**os.environ, **plan.agent_provision.provisioned_env}
token_values = egress_resolve_token_values(
token_values = Egress().resolve_token_values(
plan.egress_plan.token_env_map, effective_env,
)
teardown_timeout = resolve_teardown_timeout()
@@ -284,7 +281,7 @@ def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str
env["MCP_SUPERVISE_URL"] = plan.agent_supervise_url
if plan.env_var_secret:
env[ENV_VAR_SECRET_NAME] = plan.env_var_secret
for entry in egress_agent_env_entries(plan.egress_plan):
for entry in Egress().agent_env_entries(plan.egress_plan):
key, _, value = entry.partition("=")
env[key] = value
env.update(plan.agent_provision.guest_env)
+3 -6
View File
@@ -44,10 +44,7 @@ from ...bottle_state import (
git_gate_state_dir,
read_committed_image,
)
from ...egress import (
egress_agent_env_entries,
egress_resolve_token_values,
)
from ...egress import Egress
from ...git_gate import GitGate
from ...gateway.git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT
from ...image_cache import check_stale
@@ -191,7 +188,7 @@ def launch(
f"{endpoint.network}"
)
effective_env = {**os.environ, **plan.agent_provision.provisioned_env}
token_values = egress_resolve_token_values(
token_values = Egress().resolve_token_values(
plan.egress_plan.token_env_map, effective_env,
)
teardown_timeout = resolve_teardown_timeout()
@@ -451,7 +448,7 @@ def _agent_env_entries(
# so the secret value never lands on argv.
for name in sorted(plan.forwarded_env.keys()):
env.append(name)
env.extend(egress_agent_env_entries(plan.egress_plan))
env.extend(Egress().agent_env_entries(plan.egress_plan))
return tuple(env)
+90
View File
@@ -0,0 +1,90 @@
"""Per-agent egress (PRD 0017).
The egress gateway is a TLS-terminating forward proxy that allow-lists a
bottle's outbound HTTP(S), scans payloads for secret exfil, and injects
per-route upstream credentials the agent never sees.
Layout:
* `service` — the `Egress` host-side service (`prepare` the launch plan,
`resolve_token_values`, `agent_env_entries`) + the route-building /
rendering functions.
* `plan` — `EgressPlan` / `EgressRoute`, the launch DTOs (in the backend
contract).
The runtime enforcement (the mitmproxy addon) lives in
`bot_bottle.gateway.egress_addon*`. Public names are re-exported lazily via
`__getattr__`, so `from bot_bottle.egress import …` keeps working and importing
`egress.plan` (the contract's dependency) stays light.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from .plan import EgressPlan, EgressRoute
from .service import (
CLAUDE_HOST_CREDENTIAL_TOKEN_REF,
CODEX_HOST_CREDENTIAL_TOKEN_REF,
EGRESS_HOSTNAME,
EGRESS_ROUTES_FILENAME,
EGRESS_ROUTES_IN_CONTAINER,
Egress,
egress_agent_env_entries,
egress_gateway_env_entries,
egress_manifest_routes,
egress_render_routes,
egress_resolve_token_values,
egress_routes_for_bottle,
egress_token_env_map,
)
_LAZY: dict[str, str] = {
"EgressPlan": ".plan",
"EgressRoute": ".plan",
"Egress": ".service",
"CLAUDE_HOST_CREDENTIAL_TOKEN_REF": ".service",
"CODEX_HOST_CREDENTIAL_TOKEN_REF": ".service",
"EGRESS_HOSTNAME": ".service",
"EGRESS_ROUTES_FILENAME": ".service",
"EGRESS_ROUTES_IN_CONTAINER": ".service",
"egress_agent_env_entries": ".service",
"egress_gateway_env_entries": ".service",
"egress_manifest_routes": ".service",
"egress_render_routes": ".service",
"egress_resolve_token_values": ".service",
"egress_routes_for_bottle": ".service",
"egress_token_env_map": ".service",
}
def __getattr__(name: str) -> Any:
src = _LAZY.get(name)
if src is None:
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
from importlib import import_module
value = getattr(import_module(src, __name__), name)
globals()[name] = value
return value
__all__ = [
"CLAUDE_HOST_CREDENTIAL_TOKEN_REF",
"CODEX_HOST_CREDENTIAL_TOKEN_REF",
"EGRESS_HOSTNAME",
"EGRESS_ROUTES_FILENAME",
"EGRESS_ROUTES_IN_CONTAINER",
"Egress",
"EgressPlan",
"EgressRoute",
"egress_manifest_routes",
"egress_render_routes",
"egress_resolve_token_values",
"egress_routes_for_bottle",
"egress_agent_env_entries",
"egress_gateway_env_entries",
"egress_token_env_map",
]
+48
View File
@@ -0,0 +1,48 @@
"""Egress launch DTOs (PRD 0017).
`EgressRoute` (the host-side extension of the addon's wire `Route`) and
`EgressPlan` (the launch plan the backend contract references). Pure value
types — the route-building / rendering logic and the `Egress` service live in
`egress.service`.
"""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from ..gateway.egress_addon_core import Route
@dataclass(frozen=True)
class EgressRoute(Route):
"""Host-side extension of the addon's `Route`.
Inherits `host`, `matches`, `auth_scheme`, and `token_env`
from `egress_addon_core.Route` — those are the fields that cross the
YAML wire into the gateway. The fields below are host-only and
are never serialised to the addon.
`token_ref` is the host env var the CLI reads at launch and forwards
into the container's environ under `token_env`.
`roles` carries the manifest route's role tuple (reserved for
future use; always empty today)."""
token_ref: str = ""
roles: tuple[str, ...] = ()
@dataclass(frozen=True)
class EgressPlan:
slug: str
routes_path: Path
routes: tuple[EgressRoute, ...]
token_env_map: dict[str, str]
internal_network: str = ""
egress_network: str = ""
mitmproxy_ca_host_path: Path = Path()
mitmproxy_ca_cert_only_host_path: Path = Path()
log: int = 0
canary: str = ""
canary_env: str = ""
@@ -1,33 +1,33 @@
"""Per-bottle egress proxy (PRD 0017, PRD 0053).
"""The `Egress` host-side service (PRD 0017).
This module defines the abstract proxy (`Egress`), its plan
dataclass (`EgressPlan`), and the resolved per-route shape
(`EgressRoute`). The gateway's start/stop lifecycle is backend-
specific and lives on concrete subclasses (see
`bot_bottle/backend/docker/egress.py`).
`Egress` builds a bottle's egress plan at launch: resolve the manifest's egress
routes, render the gateway's `routes.yaml`, assign per-route token slots, and
plant the exfil canary. The service also resolves the launch-time token values
and the agent/gateway env entries the backend injects. The runtime enforcement
(the mitmproxy addon) lives in `bot_bottle.gateway.egress_addon*`.
"""
from __future__ import annotations
import dataclasses
import secrets
from abc import ABC
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING
from .gateway.egress_addon_core import (
from ..gateway.egress_addon_core import (
ON_MATCH_REDACT,
HeaderMatch as CoreHeaderMatch,
MatchEntry as CoreMatchEntry,
PathMatch as CorePathMatch,
Route,
)
from .errors import MissingEnvVarError
from .log import die
from ..errors import MissingEnvVarError
from ..log import die
from .plan import EgressPlan, EgressRoute
if TYPE_CHECKING:
from .manifest import ManifestBottle
from ..manifest import ManifestBottle
CODEX_HOST_CREDENTIAL_TOKEN_REF = "BOT_BOTTLE_CODEX_HOST_ACCESS_TOKEN"
CLAUDE_HOST_CREDENTIAL_TOKEN_REF = "BOT_BOTTLE_CLAUDE_HOST_ACCESS_TOKEN"
@@ -82,39 +82,6 @@ def egress_agent_env_entries(plan: "EgressPlan") -> tuple[str, ...]:
return ()
@dataclass(frozen=True)
class EgressRoute(Route):
"""Host-side extension of the addon's `Route`.
Inherits `host`, `matches`, `auth_scheme`, and `token_env`
from `egress_addon_core.Route` those are the fields that cross the
YAML wire into the gateway. The fields below are host-only and
are never serialised to the addon.
`token_ref` is the host env var the CLI reads at launch and forwards
into the container's environ under `token_env`.
`roles` carries the manifest route's role tuple (reserved for
future use; always empty today)."""
token_ref: str = ""
roles: tuple[str, ...] = ()
@dataclass(frozen=True)
class EgressPlan:
slug: str
routes_path: Path
routes: tuple[EgressRoute, ...]
token_env_map: dict[str, str]
internal_network: str = ""
egress_network: str = ""
mitmproxy_ca_host_path: Path = Path()
mitmproxy_ca_cert_only_host_path: Path = Path()
log: int = 0
canary: str = ""
canary_env: str = ""
def egress_manifest_routes(
bottle: ManifestBottle,
@@ -389,7 +356,11 @@ def egress_resolve_token_values(
return out
class Egress(ABC):
class Egress:
"""The host-side egress service. The backend drives `prepare` at launch,
then `resolve_token_values` to resolve the routes' upstream credentials and
`agent_env_entries` for the agent's egress env. Stateless."""
def prepare(
self,
bottle: ManifestBottle,
@@ -416,20 +387,13 @@ class Egress(ABC):
canary_env=_random_canary_env(),
)
__all__ = [
"CLAUDE_HOST_CREDENTIAL_TOKEN_REF",
"CODEX_HOST_CREDENTIAL_TOKEN_REF",
"EGRESS_HOSTNAME",
"EGRESS_ROUTES_FILENAME",
"EGRESS_ROUTES_IN_CONTAINER",
"Egress",
"EgressPlan",
"EgressRoute",
"egress_manifest_routes",
"egress_render_routes",
"egress_resolve_token_values",
"egress_routes_for_bottle",
"egress_agent_env_entries",
"egress_gateway_env_entries",
"egress_token_env_map",
]
def resolve_token_values(
self, token_env_map: dict[str, str], host_env: dict[str, str],
) -> dict[str, str]:
"""Resolve each route's upstream credential from the host env at launch.
Raises `MissingEnvVarError` for an unset/empty referenced host var."""
return egress_resolve_token_values(token_env_map, host_env)
def agent_env_entries(self, plan: EgressPlan) -> tuple[str, ...]:
"""The agent-visible egress env entries (the exfil canary)."""
return egress_agent_env_entries(plan)
+1 -1
View File
@@ -10,7 +10,6 @@ from bot_bottle.egress import (
Egress,
EgressPlan,
EgressRoute,
_yaml_str_escape,
egress_agent_env_entries,
egress_manifest_routes,
egress_render_routes,
@@ -19,6 +18,7 @@ from bot_bottle.egress import (
egress_gateway_env_entries,
egress_token_env_map,
)
from bot_bottle.egress.service import _yaml_str_escape
from bot_bottle.errors import MissingEnvVarError
from bot_bottle.manifest import ManifestIndex
from bot_bottle.yaml_subset import parse_yaml_subset