didericis/bot-bottle
1
0
Fork 0
Code Issues 32 Pull Requests 8 Actions Packages Releases Wiki Activity

refactor(gateway): macOS gateway under the Gateway service ABC

Browse Source 
Extract the Apple-Container gateway data plane out of `MacosInfraService` into
`MacosGateway`, the macOS implementation of the shared `Gateway` service.
`connect_to_orchestrator(url, gateway_token)` runs the triple-homed gateway
container (egress + agent + control networks) carrying the mitmproxy CA and the
pre-minted `gateway` token; `address()` / `ca_cert_pem()` /
`provisioning_transport()` round out the contract.

The infra service now composes the gateway: `ensure_running` mints the
role-scoped `gateway` JWT (it holds the signing key; the gateway never does —
#469) and hands it to `gateway().connect_to_orchestrator(...)`, and
`ca_cert_pem` delegates to the service. The inlined `_ensure_gateway_container`
+ CA-read are gone. `AppleGatewayTransport` + the gateway container name/label
now live with the gateway module, breaking the old infra→provision coupling.

Splits the gateway-run + CA tests out of test_macos_infra into a dedicated
test_macos_gateway; the infra tests mock `svc.gateway`.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
didericis
2026-07-25 13:04:53 -04:00
parent 05df21f210
commit c4ccd74f9b
5 changed files with 368 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bot_bottle/backend/macos_container/gateway.py
+158 -11
View File
@@ -1,21 +1,36 @@
"""Shared network/image constants for the macOS consolidated infra container.
"""The macOS gateway data plane as an Apple container (PRD 0070).
The gateway data plane no longer runs as its own Apple container — it shares a
single per-host **infra container** with the control plane (see `infra`),
because two Apple-Container guests writing one `bot-bottle.db` over virtiofs
would race incoherent `fcntl` locks. This module holds the pieces both the
infra service and the launch/provision glue need: the network names, the
gateway image, and the network-creation helper.
`MacosGateway` is the Apple-Container implementation of the backend-neutral
`Gateway` service. It runs the gateway daemons (egress / git-http / supervise)
in a single triple-homed container and never opens `bot-bottle.db` — it reaches
the supervise queue over the control-plane RPC (#469), so it holds no signing
key, only the pre-minted `gateway` token the orchestrator hands it via
`connect_to_orchestrator`.
This module also holds the pieces the infra service + launch/provision glue
share: the network names, the gateway image, and the network-creation helper.
"""
from __future__ import annotations
import os
from ...gateway import GatewayError
from ...gateway import (
DEFAULT_CA_TIMEOUT_SECONDS,
GATEWAY_CA_CERT,
MITMPROXY_HOME,
Gateway,
GatewayError,
GatewayTransport,
)
from ...paths import (
ORCHESTRATOR_AUTH_JWT_ENV,
host_gateway_ca_dir,
)
from .. import util as backend_util
from . import util as container_mod
# The shared host-only network the infra container and every agent bottle sit
# The shared host-only network the gateway container and every agent bottle sit
# on. The agent's address here is the attribution key. Distinct from the docker
# names so both backends can coexist on one host.
GATEWAY_NETWORK = "bot-bottle-mac-gateway"
@@ -26,13 +41,19 @@ GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress"
# route to the control plane (PRD 0070 "Separating the planes").
CONTROL_NETWORK = "bot-bottle-mac-control"
# The gateway (data plane) container. The name predates the split — it was the
# combined "infra" container — and is kept so callers importing it still resolve
# the gateway (probe / reprovision attribute against it).
GATEWAY_NAME = "bot-bottle-mac-infra"
GATEWAY_LABEL = "bot-bottle-mac-infra=1"
# The gateway subset the consolidated model runs (no per-bottle git:// daemon).
GATEWAY_DAEMONS = "egress,git-http,supervise"
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
ORCHESTRATOR_IMAGE = os.environ.get(
"BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest"
)
DEFAULT_CA_TIMEOUT_SECONDS = 30.0
def ensure_networks(
network: str = GATEWAY_NETWORK,
@@ -47,13 +68,139 @@ def ensure_networks(
container_mod.create_network(control_network, internal=True)
class MacosGateway(Gateway):
"""The consolidated gateway as a single Apple container, triple-homed on the
NAT egress, host-only agent, and control networks.
Images are built by the infra service (`ensure_built` here is the ABC's
no-op); the networks are ensured there too. `connect_to_orchestrator` runs
the container carrying the mitmproxy CA + the pre-minted `gateway` token."""
def __init__(
self,
image_ref: str = GATEWAY_IMAGE,
*,
name: str = GATEWAY_NAME,
network: str = GATEWAY_NETWORK,
egress_network: str = GATEWAY_EGRESS_NETWORK,
control_network: str = CONTROL_NETWORK,
) -> None:
self.image_ref = image_ref
self.name = name
self.network = network
self.egress_network = egress_network
self.control_network = control_network
# Set by `connect_to_orchestrator`: the URL the daemons resolve policy
# against + the pre-minted `gateway` token they present. The gateway
# never mints, so it never holds the signing key (#469).
self._orchestrator_url = ""
self._gateway_token = ""
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
"""Bind the gateway to this orchestrator and (re)start it, dual-homed on
the agent + control networks, resolving policy against `orchestrator_url`
(the orchestrator's control-network address — Apple has no container
DNS) and presenting `gateway_token`."""
self._orchestrator_url = orchestrator_url
self._gateway_token = gateway_token
# Fail closed on a missing policy source or token: the resolver-only data
# plane (PRD 0070) would only crash-loop its daemons without an
# orchestrator URL, and it cannot mint the token it presents (#469).
if not self._orchestrator_url:
raise GatewayError(
"gateway requires an orchestrator URL to run "
"(resolver-only data plane; no single-tenant fallback)"
)
if not self._gateway_token:
raise GatewayError(
"gateway requires a pre-minted `gateway` token to run "
"(the orchestrator mints it; the gateway never holds the key)"
)
container_mod.force_remove_container(self.name)
argv = [
"container", "run", "--detach",
"--name", self.name,
"--label", "bot-bottle.backend=macos-container",
"--label", GATEWAY_LABEL,
# NAT egress FIRST (default route out); the host-only agent network
# is where agents reach the gateway; the control network reaches the
# orchestrator.
"--network", self.egress_network,
"--network", self.network,
"--network", self.control_network,
"--dns", container_mod.dns_server(),
# The mitmproxy CA on a host bind-mount (survives recreation +
# volume pruning — issue #450). No DB mount: the data plane never
# opens bot-bottle.db (#469).
"--mount",
container_mod.bind_mount_spec(str(host_gateway_ca_dir()), MITMPROXY_HOME),
"--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={GATEWAY_DAEMONS}",
"--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}",
# The pre-minted `gateway` JWT (never the signing key). Bare
# `--env NAME` inherits the value from run_env below.
"--env", ORCHESTRATOR_AUTH_JWT_ENV,
self.image_ref,
]
run_env = {**os.environ, ORCHESTRATOR_AUTH_JWT_ENV: self._gateway_token}
result = container_mod.run_container_argv(argv, env=run_env)
if result.returncode != 0:
raise GatewayError(
f"gateway container failed to start: "
f"{(result.stderr or '').strip() or '<no stderr>'}"
)
def is_running(self) -> bool:
return container_mod.container_is_running(self.name)
def stop(self) -> None:
"""Remove the gateway container (idempotent)."""
container_mod.force_remove_container(self.name)
def address(self) -> str:
"""The gateway's agent-network address — the proxy / git-http / supervise
target agents dial (also the source IP the gateway attributes by)."""
ip = container_mod.try_container_ipv4_on_network(self.name, self.network)
if not ip:
raise GatewayError(
f"gateway {self.name} has no address on {self.network}"
)
return ip
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
interception. Read from the gateway container; polls because mitmproxy
writes it a beat after start."""
def _fetch() -> str | None:
result = container_mod.run_container_argv(
["container", "exec", self.name, "cat", GATEWAY_CA_CERT])
return result.stdout if result.returncode == 0 and result.stdout.strip() else None
try:
return backend_util.poll_ca_cert(_fetch, timeout=timeout)
except TimeoutError as exc:
raise GatewayError(
f"gateway CA not available in {self.name} after {timeout:g}s"
) from exc
def provisioning_transport(self) -> GatewayTransport:
"""The exec/cp transport git-gate provisioning stages per-bottle repos +
deploy keys through (over the `container` CLI)."""
# Local import: gateway_provision imports GATEWAY_NAME from this module,
# so importing AppleGatewayTransport at module scope would cycle.
from .gateway_provision import AppleGatewayTransport
return AppleGatewayTransport(self.name)
__all__ = [
"GATEWAY_NETWORK",
"GATEWAY_EGRESS_NETWORK",
"CONTROL_NETWORK",
"GATEWAY_NAME",
"GATEWAY_LABEL",
"GATEWAY_DAEMONS",
"GATEWAY_IMAGE",
"ORCHESTRATOR_IMAGE",
"GatewayError",
"DEFAULT_CA_TIMEOUT_SECONDS",
"ensure_networks",
"MacosGateway",
]
bot_bottle/backend/macos_container/gateway_provision.py
+3 -3
View File
@@ -9,15 +9,15 @@ CLI's equivalents against the infra container that hosts the gateway daemons.
from __future__ import annotations
from ..docker.gateway_provision import GatewayProvisionError
from ...gateway import GatewayProvisionError
from . import util as container_mod
from .infra import INFRA_NAME
from .gateway import GATEWAY_NAME
class AppleGatewayTransport:
"""`GatewayTransport` for the gateway daemons in the Apple infra container."""
def __init__(self, gateway: str = INFRA_NAME) -> None:
def __init__(self, gateway: str = GATEWAY_NAME) -> None:
self.gateway = gateway
def exec(self, argv: list[str]) -> None:
bot_bottle/backend/macos_container/infra.py
+28 -64
View File
@@ -29,7 +29,6 @@ from dataclasses import dataclass
from pathlib import Path
from ... import log
from ...gateway import GATEWAY_CA_CERT, MITMPROXY_HOME
from ...orchestrator.lifecycle import (
DEFAULT_PORT,
DEFAULT_STARTUP_TIMEOUT_SECONDS,
@@ -38,32 +37,30 @@ from ...orchestrator.lifecycle import (
)
from ...orchestrator_auth import ROLE_GATEWAY, mint
from ...paths import (
ORCHESTRATOR_AUTH_JWT_ENV,
ORCHESTRATOR_TOKEN_ENV,
HOST_DB_FILENAME,
host_orchestrator_token,
host_gateway_ca_dir,
)
from .. import util as backend_util
from . import util as container_mod
from .gateway import (
CONTROL_NETWORK,
DEFAULT_CA_TIMEOUT_SECONDS,
GATEWAY_EGRESS_NETWORK,
GATEWAY_IMAGE,
GATEWAY_NAME,
GATEWAY_NETWORK,
GatewayError,
MacosGateway,
ORCHESTRATOR_IMAGE,
ensure_networks,
)
# The orchestrator (control plane) container + the gateway (data plane)
# container. `INFRA_NAME` is kept — now the gateway container — for callers that
# still import it (probe / reprovision attribute against the gateway).
# The orchestrator (control plane) container. `INFRA_NAME` is kept — now aliasing
# the gateway container — for callers that still import it (probe / reprovision
# attribute against the gateway).
ORCHESTRATOR_NAME = "bot-bottle-mac-orchestrator"
ORCHESTRATOR_LABEL = "bot-bottle-mac-orchestrator=1"
INFRA_NAME = "bot-bottle-mac-infra" # the gateway container
INFRA_LABEL = "bot-bottle-mac-infra=1"
INFRA_NAME = GATEWAY_NAME # the container agents attribute against is the gateway
# Container-only volume holding bot-bottle.db, mounted ONLY into the
# orchestrator. One kernel writes it (never host-shared or cross-guest).
INFRA_DB_VOLUME = "bot-bottle-mac-db"
@@ -79,9 +76,6 @@ _REPO_ROOT = Path(__file__).resolve().parents[3]
_HEALTH_POLL_SECONDS = 0.25
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
# The gateway subset the consolidated model runs (no per-bottle git:// daemon).
_GATEWAY_DAEMONS = "egress,git-http,supervise"
@dataclass(frozen=True)
class InfraEndpoint:
@@ -126,6 +120,19 @@ class MacosInfraService:
def gateway_name(self) -> str:
return self._gateway_name
def gateway(self) -> MacosGateway:
"""The data-plane gateway service, triple-homed on the egress + agent +
control networks. Cheap to reconstruct — the launch flow reads its
`address()` / CA / provisioning transport off it, and `ensure_running`
connects it to the control plane."""
return MacosGateway(
self.gateway_image,
name=self._gateway_name,
network=self.network,
egress_network=self.egress_network,
control_network=self.control_network,
)
def _resolve_orchestrator_url(self) -> str:
"""The control-plane URL (orchestrator's control-network address), or ""
while it has no address."""
@@ -190,8 +197,12 @@ class MacosInfraService:
url = self._wait_healthy(startup_timeout)
# (Re)ensure the gateway once the control plane it resolves against is
# healthy — it needs the orchestrator's control-network address.
self._ensure_gateway_container(url)
# healthy — it needs the orchestrator's control-network address. The
# orchestrator (which holds the signing key) mints the role-scoped
# `gateway` JWT and hands it to the gateway, which never sees the key
# (#469).
gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token())
self.gateway().connect_to_orchestrator(url, gateway_token)
return InfraEndpoint(orchestrator_url=url, gateway_ip=self._resolve_gateway_ip())
def _run_orchestrator_container(self, current_hash: str) -> None:
@@ -230,44 +241,6 @@ class MacosInfraService:
f"{(result.stderr or '').strip() or '<no stderr>'}"
)
def _ensure_gateway_container(self, orchestrator_url: str) -> None:
"""Start (recreate) the gateway container, dual-homed on the agent +
control networks, resolving policy against `orchestrator_url` (the
orchestrator's control-network address — Apple has no DNS)."""
container_mod.force_remove_container(self._gateway_name)
_signing_key = host_orchestrator_token()
argv = [
"container", "run", "--detach",
"--name", self._gateway_name,
"--label", "bot-bottle.backend=macos-container",
"--label", INFRA_LABEL,
# NAT egress FIRST (default route out); the host-only agent network
# is where agents reach the gateway; the control network reaches the
# orchestrator.
"--network", self.egress_network,
"--network", self.network,
"--network", self.control_network,
"--dns", container_mod.dns_server(),
# The mitmproxy CA on a host bind-mount (survives recreation +
# volume pruning — issue #450). No DB mount: the data plane never
# opens bot-bottle.db (#469).
"--mount",
container_mod.bind_mount_spec(str(host_gateway_ca_dir()), MITMPROXY_HOME),
"--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS}",
"--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={orchestrator_url}",
# The pre-minted `gateway` JWT (never the signing key). Bare
# `--env NAME` inherits the value from run_env below.
"--env", ORCHESTRATOR_AUTH_JWT_ENV,
self.gateway_image,
]
run_env = {**os.environ, ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key)}
result = container_mod.run_container_argv(argv, env=run_env)
if result.returncode != 0:
raise GatewayError(
f"gateway container failed to start: "
f"{(result.stderr or '').strip() or '<no stderr>'}"
)
def _wait_healthy(self, startup_timeout: float) -> str:
deadline = time.monotonic() + startup_timeout
while True:
@@ -284,18 +257,9 @@ class MacosInfraService:
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
interception. Read from the gateway container; polls because mitmproxy
writes it a beat after start."""
def _fetch() -> str | None:
result = container_mod.run_container_argv(
["container", "exec", self._gateway_name, "cat", GATEWAY_CA_CERT])
return result.stdout if result.returncode == 0 and result.stdout.strip() else None
try:
return backend_util.poll_ca_cert(_fetch, timeout=timeout)
except TimeoutError as exc:
raise GatewayError(
f"gateway CA not available in {self._gateway_name} after {timeout:g}s"
) from exc
interception — delegated to the gateway service (reads it out of the
gateway container, polling until mitmproxy writes it)."""
return self.gateway().ca_cert_pem(timeout=timeout)
def stop(self) -> None:
"""Remove both containers (idempotent). The DB volume persists."""