refactor(gateway): macOS gateway under the Gateway service ABC
Extract the Apple-Container gateway data plane out of `MacosInfraService` into `MacosGateway`, the macOS implementation of the shared `Gateway` service. `connect_to_orchestrator(url, gateway_token)` runs the triple-homed gateway container (egress + agent + control networks) carrying the mitmproxy CA and the pre-minted `gateway` token; `address()` / `ca_cert_pem()` / `provisioning_transport()` round out the contract. The infra service now composes the gateway: `ensure_running` mints the role-scoped `gateway` JWT (it holds the signing key; the gateway never does — #469) and hands it to `gateway().connect_to_orchestrator(...)`, and `ca_cert_pem` delegates to the service. The inlined `_ensure_gateway_container` + CA-read are gone. `AppleGatewayTransport` + the gateway container name/label now live with the gateway module, breaking the old infra→provision coupling. Splits the gateway-run + CA tests out of test_macos_infra into a dedicated test_macos_gateway; the infra tests mock `svc.gateway`. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -1,21 +1,36 @@
|
|||||||
"""Shared network/image constants for the macOS consolidated infra container.
|
"""The macOS gateway data plane as an Apple container (PRD 0070).
|
||||||
|
|
||||||
The gateway data plane no longer runs as its own Apple container — it shares a
|
`MacosGateway` is the Apple-Container implementation of the backend-neutral
|
||||||
single per-host **infra container** with the control plane (see `infra`),
|
`Gateway` service. It runs the gateway daemons (egress / git-http / supervise)
|
||||||
because two Apple-Container guests writing one `bot-bottle.db` over virtiofs
|
in a single triple-homed container and never opens `bot-bottle.db` — it reaches
|
||||||
would race incoherent `fcntl` locks. This module holds the pieces both the
|
the supervise queue over the control-plane RPC (#469), so it holds no signing
|
||||||
infra service and the launch/provision glue need: the network names, the
|
key, only the pre-minted `gateway` token the orchestrator hands it via
|
||||||
gateway image, and the network-creation helper.
|
`connect_to_orchestrator`.
|
||||||
|
|
||||||
|
This module also holds the pieces the infra service + launch/provision glue
|
||||||
|
share: the network names, the gateway image, and the network-creation helper.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from ...gateway import GatewayError
|
from ...gateway import (
|
||||||
|
DEFAULT_CA_TIMEOUT_SECONDS,
|
||||||
|
GATEWAY_CA_CERT,
|
||||||
|
MITMPROXY_HOME,
|
||||||
|
Gateway,
|
||||||
|
GatewayError,
|
||||||
|
GatewayTransport,
|
||||||
|
)
|
||||||
|
from ...paths import (
|
||||||
|
ORCHESTRATOR_AUTH_JWT_ENV,
|
||||||
|
host_gateway_ca_dir,
|
||||||
|
)
|
||||||
|
from .. import util as backend_util
|
||||||
from . import util as container_mod
|
from . import util as container_mod
|
||||||
|
|
||||||
# The shared host-only network the infra container and every agent bottle sit
|
# The shared host-only network the gateway container and every agent bottle sit
|
||||||
# on. The agent's address here is the attribution key. Distinct from the docker
|
# on. The agent's address here is the attribution key. Distinct from the docker
|
||||||
# names so both backends can coexist on one host.
|
# names so both backends can coexist on one host.
|
||||||
GATEWAY_NETWORK = "bot-bottle-mac-gateway"
|
GATEWAY_NETWORK = "bot-bottle-mac-gateway"
|
||||||
@@ -26,13 +41,19 @@ GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress"
|
|||||||
# route to the control plane (PRD 0070 "Separating the planes").
|
# route to the control plane (PRD 0070 "Separating the planes").
|
||||||
CONTROL_NETWORK = "bot-bottle-mac-control"
|
CONTROL_NETWORK = "bot-bottle-mac-control"
|
||||||
|
|
||||||
|
# The gateway (data plane) container. The name predates the split — it was the
|
||||||
|
# combined "infra" container — and is kept so callers importing it still resolve
|
||||||
|
# the gateway (probe / reprovision attribute against it).
|
||||||
|
GATEWAY_NAME = "bot-bottle-mac-infra"
|
||||||
|
GATEWAY_LABEL = "bot-bottle-mac-infra=1"
|
||||||
|
# The gateway subset the consolidated model runs (no per-bottle git:// daemon).
|
||||||
|
GATEWAY_DAEMONS = "egress,git-http,supervise"
|
||||||
|
|
||||||
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
||||||
ORCHESTRATOR_IMAGE = os.environ.get(
|
ORCHESTRATOR_IMAGE = os.environ.get(
|
||||||
"BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest"
|
"BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest"
|
||||||
)
|
)
|
||||||
|
|
||||||
DEFAULT_CA_TIMEOUT_SECONDS = 30.0
|
|
||||||
|
|
||||||
|
|
||||||
def ensure_networks(
|
def ensure_networks(
|
||||||
network: str = GATEWAY_NETWORK,
|
network: str = GATEWAY_NETWORK,
|
||||||
@@ -47,13 +68,139 @@ def ensure_networks(
|
|||||||
container_mod.create_network(control_network, internal=True)
|
container_mod.create_network(control_network, internal=True)
|
||||||
|
|
||||||
|
|
||||||
|
class MacosGateway(Gateway):
|
||||||
|
"""The consolidated gateway as a single Apple container, triple-homed on the
|
||||||
|
NAT egress, host-only agent, and control networks.
|
||||||
|
|
||||||
|
Images are built by the infra service (`ensure_built` here is the ABC's
|
||||||
|
no-op); the networks are ensured there too. `connect_to_orchestrator` runs
|
||||||
|
the container carrying the mitmproxy CA + the pre-minted `gateway` token."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
image_ref: str = GATEWAY_IMAGE,
|
||||||
|
*,
|
||||||
|
name: str = GATEWAY_NAME,
|
||||||
|
network: str = GATEWAY_NETWORK,
|
||||||
|
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
||||||
|
control_network: str = CONTROL_NETWORK,
|
||||||
|
) -> None:
|
||||||
|
self.image_ref = image_ref
|
||||||
|
self.name = name
|
||||||
|
self.network = network
|
||||||
|
self.egress_network = egress_network
|
||||||
|
self.control_network = control_network
|
||||||
|
# Set by `connect_to_orchestrator`: the URL the daemons resolve policy
|
||||||
|
# against + the pre-minted `gateway` token they present. The gateway
|
||||||
|
# never mints, so it never holds the signing key (#469).
|
||||||
|
self._orchestrator_url = ""
|
||||||
|
self._gateway_token = ""
|
||||||
|
|
||||||
|
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
|
||||||
|
"""Bind the gateway to this orchestrator and (re)start it, dual-homed on
|
||||||
|
the agent + control networks, resolving policy against `orchestrator_url`
|
||||||
|
(the orchestrator's control-network address — Apple has no container
|
||||||
|
DNS) and presenting `gateway_token`."""
|
||||||
|
self._orchestrator_url = orchestrator_url
|
||||||
|
self._gateway_token = gateway_token
|
||||||
|
# Fail closed on a missing policy source or token: the resolver-only data
|
||||||
|
# plane (PRD 0070) would only crash-loop its daemons without an
|
||||||
|
# orchestrator URL, and it cannot mint the token it presents (#469).
|
||||||
|
if not self._orchestrator_url:
|
||||||
|
raise GatewayError(
|
||||||
|
"gateway requires an orchestrator URL to run "
|
||||||
|
"(resolver-only data plane; no single-tenant fallback)"
|
||||||
|
)
|
||||||
|
if not self._gateway_token:
|
||||||
|
raise GatewayError(
|
||||||
|
"gateway requires a pre-minted `gateway` token to run "
|
||||||
|
"(the orchestrator mints it; the gateway never holds the key)"
|
||||||
|
)
|
||||||
|
container_mod.force_remove_container(self.name)
|
||||||
|
argv = [
|
||||||
|
"container", "run", "--detach",
|
||||||
|
"--name", self.name,
|
||||||
|
"--label", "bot-bottle.backend=macos-container",
|
||||||
|
"--label", GATEWAY_LABEL,
|
||||||
|
# NAT egress FIRST (default route out); the host-only agent network
|
||||||
|
# is where agents reach the gateway; the control network reaches the
|
||||||
|
# orchestrator.
|
||||||
|
"--network", self.egress_network,
|
||||||
|
"--network", self.network,
|
||||||
|
"--network", self.control_network,
|
||||||
|
"--dns", container_mod.dns_server(),
|
||||||
|
# The mitmproxy CA on a host bind-mount (survives recreation +
|
||||||
|
# volume pruning — issue #450). No DB mount: the data plane never
|
||||||
|
# opens bot-bottle.db (#469).
|
||||||
|
"--mount",
|
||||||
|
container_mod.bind_mount_spec(str(host_gateway_ca_dir()), MITMPROXY_HOME),
|
||||||
|
"--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={GATEWAY_DAEMONS}",
|
||||||
|
"--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}",
|
||||||
|
# The pre-minted `gateway` JWT (never the signing key). Bare
|
||||||
|
# `--env NAME` inherits the value from run_env below.
|
||||||
|
"--env", ORCHESTRATOR_AUTH_JWT_ENV,
|
||||||
|
self.image_ref,
|
||||||
|
]
|
||||||
|
run_env = {**os.environ, ORCHESTRATOR_AUTH_JWT_ENV: self._gateway_token}
|
||||||
|
result = container_mod.run_container_argv(argv, env=run_env)
|
||||||
|
if result.returncode != 0:
|
||||||
|
raise GatewayError(
|
||||||
|
f"gateway container failed to start: "
|
||||||
|
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def is_running(self) -> bool:
|
||||||
|
return container_mod.container_is_running(self.name)
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
"""Remove the gateway container (idempotent)."""
|
||||||
|
container_mod.force_remove_container(self.name)
|
||||||
|
|
||||||
|
def address(self) -> str:
|
||||||
|
"""The gateway's agent-network address — the proxy / git-http / supervise
|
||||||
|
target agents dial (also the source IP the gateway attributes by)."""
|
||||||
|
ip = container_mod.try_container_ipv4_on_network(self.name, self.network)
|
||||||
|
if not ip:
|
||||||
|
raise GatewayError(
|
||||||
|
f"gateway {self.name} has no address on {self.network}"
|
||||||
|
)
|
||||||
|
return ip
|
||||||
|
|
||||||
|
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
||||||
|
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
|
||||||
|
interception. Read from the gateway container; polls because mitmproxy
|
||||||
|
writes it a beat after start."""
|
||||||
|
def _fetch() -> str | None:
|
||||||
|
result = container_mod.run_container_argv(
|
||||||
|
["container", "exec", self.name, "cat", GATEWAY_CA_CERT])
|
||||||
|
return result.stdout if result.returncode == 0 and result.stdout.strip() else None
|
||||||
|
try:
|
||||||
|
return backend_util.poll_ca_cert(_fetch, timeout=timeout)
|
||||||
|
except TimeoutError as exc:
|
||||||
|
raise GatewayError(
|
||||||
|
f"gateway CA not available in {self.name} after {timeout:g}s"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def provisioning_transport(self) -> GatewayTransport:
|
||||||
|
"""The exec/cp transport git-gate provisioning stages per-bottle repos +
|
||||||
|
deploy keys through (over the `container` CLI)."""
|
||||||
|
# Local import: gateway_provision imports GATEWAY_NAME from this module,
|
||||||
|
# so importing AppleGatewayTransport at module scope would cycle.
|
||||||
|
from .gateway_provision import AppleGatewayTransport
|
||||||
|
return AppleGatewayTransport(self.name)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"GATEWAY_NETWORK",
|
"GATEWAY_NETWORK",
|
||||||
"GATEWAY_EGRESS_NETWORK",
|
"GATEWAY_EGRESS_NETWORK",
|
||||||
"CONTROL_NETWORK",
|
"CONTROL_NETWORK",
|
||||||
|
"GATEWAY_NAME",
|
||||||
|
"GATEWAY_LABEL",
|
||||||
|
"GATEWAY_DAEMONS",
|
||||||
"GATEWAY_IMAGE",
|
"GATEWAY_IMAGE",
|
||||||
"ORCHESTRATOR_IMAGE",
|
"ORCHESTRATOR_IMAGE",
|
||||||
"GatewayError",
|
"GatewayError",
|
||||||
"DEFAULT_CA_TIMEOUT_SECONDS",
|
"DEFAULT_CA_TIMEOUT_SECONDS",
|
||||||
"ensure_networks",
|
"ensure_networks",
|
||||||
|
"MacosGateway",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -9,15 +9,15 @@ CLI's equivalents against the infra container that hosts the gateway daemons.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from ..docker.gateway_provision import GatewayProvisionError
|
from ...gateway import GatewayProvisionError
|
||||||
from . import util as container_mod
|
from . import util as container_mod
|
||||||
from .infra import INFRA_NAME
|
from .gateway import GATEWAY_NAME
|
||||||
|
|
||||||
|
|
||||||
class AppleGatewayTransport:
|
class AppleGatewayTransport:
|
||||||
"""`GatewayTransport` for the gateway daemons in the Apple infra container."""
|
"""`GatewayTransport` for the gateway daemons in the Apple infra container."""
|
||||||
|
|
||||||
def __init__(self, gateway: str = INFRA_NAME) -> None:
|
def __init__(self, gateway: str = GATEWAY_NAME) -> None:
|
||||||
self.gateway = gateway
|
self.gateway = gateway
|
||||||
|
|
||||||
def exec(self, argv: list[str]) -> None:
|
def exec(self, argv: list[str]) -> None:
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ from dataclasses import dataclass
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ... import log
|
from ... import log
|
||||||
from ...gateway import GATEWAY_CA_CERT, MITMPROXY_HOME
|
|
||||||
from ...orchestrator.lifecycle import (
|
from ...orchestrator.lifecycle import (
|
||||||
DEFAULT_PORT,
|
DEFAULT_PORT,
|
||||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||||
@@ -38,32 +37,30 @@ from ...orchestrator.lifecycle import (
|
|||||||
)
|
)
|
||||||
from ...orchestrator_auth import ROLE_GATEWAY, mint
|
from ...orchestrator_auth import ROLE_GATEWAY, mint
|
||||||
from ...paths import (
|
from ...paths import (
|
||||||
ORCHESTRATOR_AUTH_JWT_ENV,
|
|
||||||
ORCHESTRATOR_TOKEN_ENV,
|
ORCHESTRATOR_TOKEN_ENV,
|
||||||
HOST_DB_FILENAME,
|
HOST_DB_FILENAME,
|
||||||
host_orchestrator_token,
|
host_orchestrator_token,
|
||||||
host_gateway_ca_dir,
|
|
||||||
)
|
)
|
||||||
from .. import util as backend_util
|
|
||||||
from . import util as container_mod
|
from . import util as container_mod
|
||||||
from .gateway import (
|
from .gateway import (
|
||||||
CONTROL_NETWORK,
|
CONTROL_NETWORK,
|
||||||
DEFAULT_CA_TIMEOUT_SECONDS,
|
DEFAULT_CA_TIMEOUT_SECONDS,
|
||||||
GATEWAY_EGRESS_NETWORK,
|
GATEWAY_EGRESS_NETWORK,
|
||||||
GATEWAY_IMAGE,
|
GATEWAY_IMAGE,
|
||||||
|
GATEWAY_NAME,
|
||||||
GATEWAY_NETWORK,
|
GATEWAY_NETWORK,
|
||||||
GatewayError,
|
GatewayError,
|
||||||
|
MacosGateway,
|
||||||
ORCHESTRATOR_IMAGE,
|
ORCHESTRATOR_IMAGE,
|
||||||
ensure_networks,
|
ensure_networks,
|
||||||
)
|
)
|
||||||
|
|
||||||
# The orchestrator (control plane) container + the gateway (data plane)
|
# The orchestrator (control plane) container. `INFRA_NAME` is kept — now aliasing
|
||||||
# container. `INFRA_NAME` is kept — now the gateway container — for callers that
|
# the gateway container — for callers that still import it (probe / reprovision
|
||||||
# still import it (probe / reprovision attribute against the gateway).
|
# attribute against the gateway).
|
||||||
ORCHESTRATOR_NAME = "bot-bottle-mac-orchestrator"
|
ORCHESTRATOR_NAME = "bot-bottle-mac-orchestrator"
|
||||||
ORCHESTRATOR_LABEL = "bot-bottle-mac-orchestrator=1"
|
ORCHESTRATOR_LABEL = "bot-bottle-mac-orchestrator=1"
|
||||||
INFRA_NAME = "bot-bottle-mac-infra" # the gateway container
|
INFRA_NAME = GATEWAY_NAME # the container agents attribute against is the gateway
|
||||||
INFRA_LABEL = "bot-bottle-mac-infra=1"
|
|
||||||
# Container-only volume holding bot-bottle.db, mounted ONLY into the
|
# Container-only volume holding bot-bottle.db, mounted ONLY into the
|
||||||
# orchestrator. One kernel writes it (never host-shared or cross-guest).
|
# orchestrator. One kernel writes it (never host-shared or cross-guest).
|
||||||
INFRA_DB_VOLUME = "bot-bottle-mac-db"
|
INFRA_DB_VOLUME = "bot-bottle-mac-db"
|
||||||
@@ -79,9 +76,6 @@ _REPO_ROOT = Path(__file__).resolve().parents[3]
|
|||||||
_HEALTH_POLL_SECONDS = 0.25
|
_HEALTH_POLL_SECONDS = 0.25
|
||||||
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
|
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
|
||||||
|
|
||||||
# The gateway subset the consolidated model runs (no per-bottle git:// daemon).
|
|
||||||
_GATEWAY_DAEMONS = "egress,git-http,supervise"
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class InfraEndpoint:
|
class InfraEndpoint:
|
||||||
@@ -126,6 +120,19 @@ class MacosInfraService:
|
|||||||
def gateway_name(self) -> str:
|
def gateway_name(self) -> str:
|
||||||
return self._gateway_name
|
return self._gateway_name
|
||||||
|
|
||||||
|
def gateway(self) -> MacosGateway:
|
||||||
|
"""The data-plane gateway service, triple-homed on the egress + agent +
|
||||||
|
control networks. Cheap to reconstruct — the launch flow reads its
|
||||||
|
`address()` / CA / provisioning transport off it, and `ensure_running`
|
||||||
|
connects it to the control plane."""
|
||||||
|
return MacosGateway(
|
||||||
|
self.gateway_image,
|
||||||
|
name=self._gateway_name,
|
||||||
|
network=self.network,
|
||||||
|
egress_network=self.egress_network,
|
||||||
|
control_network=self.control_network,
|
||||||
|
)
|
||||||
|
|
||||||
def _resolve_orchestrator_url(self) -> str:
|
def _resolve_orchestrator_url(self) -> str:
|
||||||
"""The control-plane URL (orchestrator's control-network address), or ""
|
"""The control-plane URL (orchestrator's control-network address), or ""
|
||||||
while it has no address."""
|
while it has no address."""
|
||||||
@@ -190,8 +197,12 @@ class MacosInfraService:
|
|||||||
url = self._wait_healthy(startup_timeout)
|
url = self._wait_healthy(startup_timeout)
|
||||||
|
|
||||||
# (Re)ensure the gateway once the control plane it resolves against is
|
# (Re)ensure the gateway once the control plane it resolves against is
|
||||||
# healthy — it needs the orchestrator's control-network address.
|
# healthy — it needs the orchestrator's control-network address. The
|
||||||
self._ensure_gateway_container(url)
|
# orchestrator (which holds the signing key) mints the role-scoped
|
||||||
|
# `gateway` JWT and hands it to the gateway, which never sees the key
|
||||||
|
# (#469).
|
||||||
|
gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token())
|
||||||
|
self.gateway().connect_to_orchestrator(url, gateway_token)
|
||||||
return InfraEndpoint(orchestrator_url=url, gateway_ip=self._resolve_gateway_ip())
|
return InfraEndpoint(orchestrator_url=url, gateway_ip=self._resolve_gateway_ip())
|
||||||
|
|
||||||
def _run_orchestrator_container(self, current_hash: str) -> None:
|
def _run_orchestrator_container(self, current_hash: str) -> None:
|
||||||
@@ -230,44 +241,6 @@ class MacosInfraService:
|
|||||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||||
)
|
)
|
||||||
|
|
||||||
def _ensure_gateway_container(self, orchestrator_url: str) -> None:
|
|
||||||
"""Start (recreate) the gateway container, dual-homed on the agent +
|
|
||||||
control networks, resolving policy against `orchestrator_url` (the
|
|
||||||
orchestrator's control-network address — Apple has no DNS)."""
|
|
||||||
container_mod.force_remove_container(self._gateway_name)
|
|
||||||
_signing_key = host_orchestrator_token()
|
|
||||||
argv = [
|
|
||||||
"container", "run", "--detach",
|
|
||||||
"--name", self._gateway_name,
|
|
||||||
"--label", "bot-bottle.backend=macos-container",
|
|
||||||
"--label", INFRA_LABEL,
|
|
||||||
# NAT egress FIRST (default route out); the host-only agent network
|
|
||||||
# is where agents reach the gateway; the control network reaches the
|
|
||||||
# orchestrator.
|
|
||||||
"--network", self.egress_network,
|
|
||||||
"--network", self.network,
|
|
||||||
"--network", self.control_network,
|
|
||||||
"--dns", container_mod.dns_server(),
|
|
||||||
# The mitmproxy CA on a host bind-mount (survives recreation +
|
|
||||||
# volume pruning — issue #450). No DB mount: the data plane never
|
|
||||||
# opens bot-bottle.db (#469).
|
|
||||||
"--mount",
|
|
||||||
container_mod.bind_mount_spec(str(host_gateway_ca_dir()), MITMPROXY_HOME),
|
|
||||||
"--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS}",
|
|
||||||
"--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={orchestrator_url}",
|
|
||||||
# The pre-minted `gateway` JWT (never the signing key). Bare
|
|
||||||
# `--env NAME` inherits the value from run_env below.
|
|
||||||
"--env", ORCHESTRATOR_AUTH_JWT_ENV,
|
|
||||||
self.gateway_image,
|
|
||||||
]
|
|
||||||
run_env = {**os.environ, ORCHESTRATOR_AUTH_JWT_ENV: mint(ROLE_GATEWAY, _signing_key)}
|
|
||||||
result = container_mod.run_container_argv(argv, env=run_env)
|
|
||||||
if result.returncode != 0:
|
|
||||||
raise GatewayError(
|
|
||||||
f"gateway container failed to start: "
|
|
||||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
|
||||||
)
|
|
||||||
|
|
||||||
def _wait_healthy(self, startup_timeout: float) -> str:
|
def _wait_healthy(self, startup_timeout: float) -> str:
|
||||||
deadline = time.monotonic() + startup_timeout
|
deadline = time.monotonic() + startup_timeout
|
||||||
while True:
|
while True:
|
||||||
@@ -284,18 +257,9 @@ class MacosInfraService:
|
|||||||
|
|
||||||
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
||||||
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
|
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
|
||||||
interception. Read from the gateway container; polls because mitmproxy
|
interception — delegated to the gateway service (reads it out of the
|
||||||
writes it a beat after start."""
|
gateway container, polling until mitmproxy writes it)."""
|
||||||
def _fetch() -> str | None:
|
return self.gateway().ca_cert_pem(timeout=timeout)
|
||||||
result = container_mod.run_container_argv(
|
|
||||||
["container", "exec", self._gateway_name, "cat", GATEWAY_CA_CERT])
|
|
||||||
return result.stdout if result.returncode == 0 and result.stdout.strip() else None
|
|
||||||
try:
|
|
||||||
return backend_util.poll_ca_cert(_fetch, timeout=timeout)
|
|
||||||
except TimeoutError as exc:
|
|
||||||
raise GatewayError(
|
|
||||||
f"gateway CA not available in {self._gateway_name} after {timeout:g}s"
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
def stop(self) -> None:
|
def stop(self) -> None:
|
||||||
"""Remove both containers (idempotent). The DB volume persists."""
|
"""Remove both containers (idempotent). The DB volume persists."""
|
||||||
|
|||||||
@@ -0,0 +1,147 @@
|
|||||||
|
"""Unit: the macOS gateway data plane as an Apple container (PRD 0070)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
|
from bot_bottle.backend.macos_container.gateway import (
|
||||||
|
GATEWAY_NAME,
|
||||||
|
GatewayError,
|
||||||
|
MacosGateway,
|
||||||
|
)
|
||||||
|
|
||||||
|
_GW = "bot_bottle.backend.macos_container.gateway"
|
||||||
|
|
||||||
|
_ORCH_URL = "http://192.168.128.2:8099"
|
||||||
|
_TOKEN = "gw-jwt-token"
|
||||||
|
_CA_PEM = "-----BEGIN CERTIFICATE-----\nMII...\n-----END CERTIFICATE-----\n"
|
||||||
|
|
||||||
|
|
||||||
|
def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock:
|
||||||
|
return Mock(returncode=returncode, stdout=stdout, stderr=stderr)
|
||||||
|
|
||||||
|
|
||||||
|
def _spec(src: str, tgt: str, readonly: bool = False) -> str:
|
||||||
|
return f"type=bind,source={src},target={tgt}" + (",readonly" if readonly else "")
|
||||||
|
|
||||||
|
|
||||||
|
class TestMacosGatewayConnect(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.gw = MacosGateway("bot-bottle-gateway:latest")
|
||||||
|
|
||||||
|
def _connect(self, url: str = _ORCH_URL, token: str = _TOKEN) -> Mock:
|
||||||
|
run = Mock(return_value=_proc())
|
||||||
|
with patch(f"{_GW}.container_mod") as mod, \
|
||||||
|
patch(f"{_GW}.host_gateway_ca_dir", return_value=Path("/ca")):
|
||||||
|
mod.dns_server.return_value = "1.1.1.1"
|
||||||
|
mod.bind_mount_spec.side_effect = _spec
|
||||||
|
mod.run_container_argv = run
|
||||||
|
self.gw.connect_to_orchestrator(url, token)
|
||||||
|
return run
|
||||||
|
|
||||||
|
def test_default_name(self) -> None:
|
||||||
|
self.assertEqual(GATEWAY_NAME, self.gw.name)
|
||||||
|
|
||||||
|
def test_refuses_without_orchestrator_url(self) -> None:
|
||||||
|
with patch(f"{_GW}.container_mod") as mod:
|
||||||
|
with self.assertRaises(GatewayError):
|
||||||
|
self.gw.connect_to_orchestrator("", _TOKEN)
|
||||||
|
mod.run_container_argv.assert_not_called()
|
||||||
|
|
||||||
|
def test_refuses_without_gateway_token(self) -> None:
|
||||||
|
with patch(f"{_GW}.container_mod") as mod:
|
||||||
|
with self.assertRaises(GatewayError):
|
||||||
|
self.gw.connect_to_orchestrator(_ORCH_URL, "")
|
||||||
|
mod.run_container_argv.assert_not_called()
|
||||||
|
|
||||||
|
def test_triple_homed(self) -> None:
|
||||||
|
argv = self._connect().call_args.args[0]
|
||||||
|
nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
||||||
|
self.assertEqual(
|
||||||
|
["bot-bottle-mac-egress", "bot-bottle-mac-gateway", "bot-bottle-mac-control"],
|
||||||
|
nets,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_resolves_orchestrator_and_holds_ca(self) -> None:
|
||||||
|
argv = self._connect().call_args.args[0]
|
||||||
|
self.assertIn(f"BOT_BOTTLE_ORCHESTRATOR_URL={_ORCH_URL}", argv)
|
||||||
|
self.assertIn("BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", argv)
|
||||||
|
self.assertIn("bot-bottle-gateway:latest", argv)
|
||||||
|
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
|
||||||
|
self.assertTrue([m for m in mounts if "target=/home/mitmproxy/.mitmproxy" in m])
|
||||||
|
|
||||||
|
def test_injects_the_pre_minted_gateway_token(self) -> None:
|
||||||
|
# The gateway presents the token it was handed — it never mints (holds no
|
||||||
|
# signing key). The value rides the env, not argv.
|
||||||
|
from bot_bottle.paths import ORCHESTRATOR_AUTH_JWT_ENV
|
||||||
|
|
||||||
|
run = self._connect()
|
||||||
|
env = run.call_args.kwargs["env"]
|
||||||
|
self.assertEqual(_TOKEN, env[ORCHESTRATOR_AUTH_JWT_ENV])
|
||||||
|
|
||||||
|
def test_raises_on_start_failure(self) -> None:
|
||||||
|
with patch(f"{_GW}.container_mod") as mod, \
|
||||||
|
patch(f"{_GW}.host_gateway_ca_dir", return_value=Path("/ca")):
|
||||||
|
mod.dns_server.return_value = "1.1.1.1"
|
||||||
|
mod.bind_mount_spec.side_effect = _spec
|
||||||
|
mod.run_container_argv.return_value = _proc(returncode=1, stderr="boom")
|
||||||
|
with self.assertRaises(GatewayError):
|
||||||
|
self.gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
||||||
|
|
||||||
|
|
||||||
|
class TestMacosGatewayLifecycle(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.gw = MacosGateway("bot-bottle-gateway:latest")
|
||||||
|
|
||||||
|
def test_is_running_reads_container_state(self) -> None:
|
||||||
|
with patch(f"{_GW}.container_mod") as mod:
|
||||||
|
mod.container_is_running.return_value = True
|
||||||
|
self.assertTrue(self.gw.is_running())
|
||||||
|
with patch(f"{_GW}.container_mod") as mod:
|
||||||
|
mod.container_is_running.return_value = False
|
||||||
|
self.assertFalse(self.gw.is_running())
|
||||||
|
|
||||||
|
def test_stop_removes_the_container(self) -> None:
|
||||||
|
with patch(f"{_GW}.container_mod") as mod:
|
||||||
|
self.gw.stop()
|
||||||
|
mod.force_remove_container.assert_called_once_with(self.gw.name)
|
||||||
|
|
||||||
|
def test_address_reads_agent_network_ip(self) -> None:
|
||||||
|
with patch(f"{_GW}.container_mod") as mod:
|
||||||
|
mod.try_container_ipv4_on_network.return_value = "192.168.128.3"
|
||||||
|
self.assertEqual("192.168.128.3", self.gw.address())
|
||||||
|
mod.try_container_ipv4_on_network.assert_called_once_with(
|
||||||
|
self.gw.name, self.gw.network)
|
||||||
|
|
||||||
|
def test_address_raises_when_unassigned(self) -> None:
|
||||||
|
with patch(f"{_GW}.container_mod") as mod:
|
||||||
|
mod.try_container_ipv4_on_network.return_value = ""
|
||||||
|
with self.assertRaises(GatewayError):
|
||||||
|
self.gw.address()
|
||||||
|
|
||||||
|
def test_ca_cert_pem_reads_from_the_gateway_container(self) -> None:
|
||||||
|
with patch(f"{_GW}.container_mod") as mod:
|
||||||
|
mod.run_container_argv.return_value = _proc(stdout=_CA_PEM)
|
||||||
|
self.assertEqual(_CA_PEM, self.gw.ca_cert_pem())
|
||||||
|
argv = mod.run_container_argv.call_args.args[0]
|
||||||
|
self.assertEqual(["container", "exec", self.gw.name, "cat"], argv[:4])
|
||||||
|
|
||||||
|
def test_ca_cert_pem_raises_when_absent(self) -> None:
|
||||||
|
with patch(f"{_GW}.container_mod") as mod:
|
||||||
|
mod.run_container_argv.return_value = _proc(returncode=1, stderr="no file")
|
||||||
|
with self.assertRaises(GatewayError):
|
||||||
|
self.gw.ca_cert_pem(timeout=0)
|
||||||
|
|
||||||
|
def test_provisioning_transport_targets_the_gateway_container(self) -> None:
|
||||||
|
from bot_bottle.backend.macos_container.gateway_provision import (
|
||||||
|
AppleGatewayTransport,
|
||||||
|
)
|
||||||
|
transport = self.gw.provisioning_transport()
|
||||||
|
assert isinstance(transport, AppleGatewayTransport)
|
||||||
|
self.assertEqual(self.gw.name, transport.gateway)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -4,7 +4,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import MagicMock, Mock, patch
|
||||||
|
|
||||||
from bot_bottle.backend.macos_container.infra import (
|
from bot_bottle.backend.macos_container.infra import (
|
||||||
INFRA_DB_VOLUME,
|
INFRA_DB_VOLUME,
|
||||||
@@ -73,63 +73,48 @@ class TestOrchestratorRun(unittest.TestCase):
|
|||||||
svc._run_orchestrator_container("h1")
|
svc._run_orchestrator_container("h1")
|
||||||
|
|
||||||
|
|
||||||
class TestGatewayRun(unittest.TestCase):
|
class TestInfraEnsureRunning(unittest.TestCase):
|
||||||
def _run(self, svc: MacosInfraService, url: str = "http://10.0.0.5:8099") -> list[str]:
|
"""Isolate the orchestrator-container logic; the gateway is brought up by
|
||||||
run = Mock(return_value=_ok())
|
`MacosGateway` (its own module, tested separately), so mock `svc.gateway`.
|
||||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
`ensure_running` mints the `gateway` token, so `mint` /
|
||||||
patch(f"{_INFRA}.host_orchestrator_token", return_value="k"), \
|
`host_orchestrator_token` are stubbed."""
|
||||||
patch(f"{_INFRA}.mint", return_value="jwt"):
|
|
||||||
mod.dns_server.return_value = "1.1.1.1"
|
|
||||||
mod.bind_mount_spec.side_effect = _spec
|
|
||||||
mod.run_container_argv = run
|
|
||||||
svc._ensure_gateway_container(url)
|
|
||||||
return run.call_args.args[0]
|
|
||||||
|
|
||||||
def test_gateway_is_triple_homed(self) -> None:
|
def _patches(self, svc: MacosInfraService):
|
||||||
argv = self._run(MacosInfraService(repo_root=Path("/r")))
|
gw = MagicMock()
|
||||||
nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
return gw, (
|
||||||
self.assertEqual(
|
patch.object(svc, "ensure_built"),
|
||||||
["bot-bottle-mac-egress", "bot-bottle-mac-gateway", "bot-bottle-mac-control"],
|
patch.object(svc, "gateway", return_value=gw),
|
||||||
nets,
|
patch(f"{_INFRA}.host_orchestrator_token", return_value="k"),
|
||||||
|
patch(f"{_INFRA}.mint", return_value="jwt"),
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_gateway_resolves_orchestrator_by_url_and_holds_ca(self) -> None:
|
|
||||||
argv = self._run(MacosInfraService(repo_root=Path("/r")), "http://10.0.0.5:8099")
|
|
||||||
self.assertIn("BOT_BOTTLE_ORCHESTRATOR_URL=http://10.0.0.5:8099", argv)
|
|
||||||
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
|
|
||||||
self.assertTrue([m for m in mounts if "target=/home/mitmproxy/.mitmproxy" in m])
|
|
||||||
self.assertIn("bot-bottle-gateway:latest", argv)
|
|
||||||
self.assertIn(
|
|
||||||
"BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise", argv)
|
|
||||||
|
|
||||||
|
|
||||||
class TestInfraEnsureRunning(unittest.TestCase):
|
|
||||||
def test_current_healthy_orchestrator_left_alone(self) -> None:
|
def test_current_healthy_orchestrator_left_alone(self) -> None:
|
||||||
svc = MacosInfraService(repo_root=Path("/r"))
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
run = Mock()
|
run = Mock()
|
||||||
|
gw, extra = self._patches(svc)
|
||||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
||||||
patch.object(svc, "ensure_built"), \
|
|
||||||
patch.object(svc, "_run_orchestrator_container", run), \
|
patch.object(svc, "_run_orchestrator_container", run), \
|
||||||
patch.object(svc, "_ensure_gateway_container"), \
|
patch.object(svc, "is_healthy", return_value=True), \
|
||||||
patch.object(svc, "is_healthy", return_value=True):
|
extra[0], extra[1], extra[2], extra[3]:
|
||||||
mod.container_is_running.return_value = True
|
mod.container_is_running.return_value = True
|
||||||
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
||||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||||
endpoint = svc.ensure_running()
|
endpoint = svc.ensure_running()
|
||||||
run.assert_not_called()
|
run.assert_not_called()
|
||||||
|
gw.connect_to_orchestrator.assert_called_once()
|
||||||
self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url)
|
self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url)
|
||||||
self.assertEqual("192.168.128.2", endpoint.gateway_ip)
|
self.assertEqual("192.168.128.2", endpoint.gateway_ip)
|
||||||
|
|
||||||
def test_changed_source_recreates_orchestrator(self) -> None:
|
def test_changed_source_recreates_orchestrator(self) -> None:
|
||||||
svc = MacosInfraService(repo_root=Path("/r"))
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
run = Mock()
|
run = Mock()
|
||||||
|
_gw, extra = self._patches(svc)
|
||||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
patch(f"{_INFRA}.source_hash", return_value="h2"), \
|
patch(f"{_INFRA}.source_hash", return_value="h2"), \
|
||||||
patch.object(svc, "ensure_built"), \
|
|
||||||
patch.object(svc, "_run_orchestrator_container", run), \
|
patch.object(svc, "_run_orchestrator_container", run), \
|
||||||
patch.object(svc, "_ensure_gateway_container"), \
|
patch.object(svc, "is_healthy", return_value=True), \
|
||||||
patch.object(svc, "is_healthy", return_value=True):
|
extra[0], extra[1], extra[2], extra[3]:
|
||||||
mod.container_is_running.return_value = True
|
mod.container_is_running.return_value = True
|
||||||
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
||||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||||
@@ -139,12 +124,12 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
|||||||
def test_wedged_but_current_orchestrator_recreated(self) -> None:
|
def test_wedged_but_current_orchestrator_recreated(self) -> None:
|
||||||
svc = MacosInfraService(repo_root=Path("/r"))
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
run = Mock()
|
run = Mock()
|
||||||
|
_gw, extra = self._patches(svc)
|
||||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
||||||
patch.object(svc, "ensure_built"), \
|
|
||||||
patch.object(svc, "_run_orchestrator_container", run), \
|
patch.object(svc, "_run_orchestrator_container", run), \
|
||||||
patch.object(svc, "_ensure_gateway_container"), \
|
patch.object(svc, "is_healthy", Mock(side_effect=[False, True])), \
|
||||||
patch.object(svc, "is_healthy", Mock(side_effect=[False, True])):
|
extra[0], extra[1], extra[2], extra[3]:
|
||||||
mod.container_is_running.return_value = True
|
mod.container_is_running.return_value = True
|
||||||
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
||||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||||
@@ -153,12 +138,12 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
|||||||
|
|
||||||
def test_never_healthy_raises(self) -> None:
|
def test_never_healthy_raises(self) -> None:
|
||||||
svc = MacosInfraService(repo_root=Path("/r"))
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
|
_gw, extra = self._patches(svc)
|
||||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||||
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
||||||
patch.object(svc, "ensure_built"), \
|
|
||||||
patch.object(svc, "_run_orchestrator_container"), \
|
patch.object(svc, "_run_orchestrator_container"), \
|
||||||
patch.object(svc, "_ensure_gateway_container"), \
|
patch.object(svc, "is_healthy", return_value=False), \
|
||||||
patch.object(svc, "is_healthy", return_value=False):
|
extra[0], extra[1], extra[2], extra[3]:
|
||||||
mod.container_is_running.return_value = False
|
mod.container_is_running.return_value = False
|
||||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||||
with self.assertRaises(OrchestratorStartError):
|
with self.assertRaises(OrchestratorStartError):
|
||||||
@@ -166,22 +151,14 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
|||||||
|
|
||||||
|
|
||||||
class TestCaCertPem(unittest.TestCase):
|
class TestCaCertPem(unittest.TestCase):
|
||||||
def test_reads_ca_out_of_the_gateway_container(self) -> None:
|
def test_delegates_to_the_gateway_service(self) -> None:
|
||||||
svc = MacosInfraService(repo_root=Path("/r"))
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
with patch(f"{_INFRA}.container_mod") as mod:
|
gw = MagicMock()
|
||||||
mod.run_container_argv.return_value = _ok("-----BEGIN CERTIFICATE-----\n")
|
gw.ca_cert_pem.return_value = "-----BEGIN CERTIFICATE-----\n"
|
||||||
pem = svc.ca_cert_pem()
|
with patch.object(svc, "gateway", return_value=gw):
|
||||||
|
pem = svc.ca_cert_pem(timeout=5)
|
||||||
self.assertTrue(pem.startswith("-----BEGIN CERTIFICATE-----"))
|
self.assertTrue(pem.startswith("-----BEGIN CERTIFICATE-----"))
|
||||||
argv = mod.run_container_argv.call_args.args[0]
|
gw.ca_cert_pem.assert_called_once_with(timeout=5)
|
||||||
self.assertEqual(["container", "exec", "bot-bottle-mac-infra", "cat"], argv[:4])
|
|
||||||
|
|
||||||
def test_raises_gateway_error_when_cert_never_appears(self) -> None:
|
|
||||||
from bot_bottle.backend.macos_container.gateway import GatewayError
|
|
||||||
svc = MacosInfraService(repo_root=Path("/r"))
|
|
||||||
with patch(f"{_INFRA}.container_mod") as mod:
|
|
||||||
mod.run_container_argv.return_value = _fail()
|
|
||||||
with self.assertRaises(GatewayError):
|
|
||||||
svc.ca_cert_pem(timeout=0)
|
|
||||||
|
|
||||||
|
|
||||||
class TestProbeOrchestrator(unittest.TestCase):
|
class TestProbeOrchestrator(unittest.TestCase):
|
||||||
|
|||||||
Reference in New Issue
Block a user