refactor(gateway): move DockerGateway to the backend layer, drop the dead standalone-gateway path
lint / lint (push) Successful in 54s
lint / lint (push) Successful in 54s
The `DockerGateway` container-lifecycle impl now lives in `backend/docker/gateway.py` (the shape backend gateway classes will share); `orchestrator/gateway.py` keeps only the backend-neutral pieces (the `Gateway` ABC, constants, `rotate_gateway_ca`). Delete the standalone-gateway path it was the only consumer of. `--gateway` on `python -m bot_bottle.orchestrator` was invoked nowhere — the production docker flow runs the gateway data plane inside the combined `bot-bottle-infra` container via `OrchestratorService`, never this class. Removing it takes with it `Orchestrator.ensure_gateway()` and the `gateway` ctor arg; `gateway_status()` becomes a stub reporting `configured: false` so the documented `GET /gateway` control-plane route keeps its contract. Fix a latent bug the move surfaced: `launch.py` read the shared gateway CA via `docker exec bot-bottle-orch-gateway`, a container the consolidated flow never creates — so the agent CA install was reaching a nonexistent name. Read it from `bot-bottle-infra` (INFRA_NAME) instead. Repoint the affected tests to the new module; drop the unit test + fake that covered the deleted standalone wiring. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,189 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ...control_auth import ROLE_GATEWAY, mint
|
||||||
|
from ...docker_cmd import run_docker
|
||||||
|
from ...paths import (
|
||||||
|
CONTROL_AUTH_JWT_ENV,
|
||||||
|
host_control_plane_token,
|
||||||
|
host_gateway_ca_dir,
|
||||||
|
)
|
||||||
|
from ...orchestrator.gateway import (
|
||||||
|
Gateway, GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK, GATEWAY_DOCKERFILE,
|
||||||
|
REPO_ROOT, GATEWAY_LABEL, MITMPROXY_HOME, DEFAULT_CA_TIMEOUT_SECONDS,
|
||||||
|
CA_POLL_SECONDS, GATEWAY_CA_CERT, GatewayError
|
||||||
|
)
|
||||||
|
|
||||||
|
class DockerGateway(Gateway):
|
||||||
|
"""The consolidated gateway as a single, fixed-name Docker container.
|
||||||
|
|
||||||
|
`image_ref` defaults to the gateway data-plane image; `ensure_built`
|
||||||
|
builds it from `Dockerfile.gateway` when it's missing. (Note: slice 5
|
||||||
|
builds + launches the bundle container; wiring its per-bottle,
|
||||||
|
source-IP-keyed config is a later slice — see PRD 0070.)"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
image_ref: str = GATEWAY_IMAGE,
|
||||||
|
*,
|
||||||
|
name: str = GATEWAY_NAME,
|
||||||
|
network: str = GATEWAY_NETWORK,
|
||||||
|
orchestrator_url: str = "",
|
||||||
|
build_context: Path | None = None,
|
||||||
|
dockerfile: str | None = GATEWAY_DOCKERFILE,
|
||||||
|
host_port_bindings: tuple[int, ...] = (),
|
||||||
|
) -> None:
|
||||||
|
self.image_ref = image_ref
|
||||||
|
self.name = name
|
||||||
|
self.network = network
|
||||||
|
# The control-plane URL the gateway's data plane resolves per bottle
|
||||||
|
# against — reached by container name over docker DNS on the shared
|
||||||
|
# network (container↔container, no host firewall). Mandatory to *run*
|
||||||
|
# the gateway (see `ensure_running`); empty is tolerated only for the
|
||||||
|
# construct-then-read-CA path (`ca_cert_pem` on an already-running
|
||||||
|
# container), which never launches a container.
|
||||||
|
self._orchestrator_url = orchestrator_url
|
||||||
|
self._build_context = build_context or REPO_ROOT
|
||||||
|
self._dockerfile = dockerfile
|
||||||
|
# Ports published on the host (0.0.0.0). Used by the Firecracker
|
||||||
|
# backend's dev-harness gateway so VMs can reach it via their TAP link;
|
||||||
|
# Docker's DNAT + the nft `ct status dnat accept` rule handle the rest.
|
||||||
|
self._host_port_bindings = host_port_bindings
|
||||||
|
|
||||||
|
def image_exists(self) -> bool:
|
||||||
|
return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0
|
||||||
|
|
||||||
|
def ensure_built(self) -> None:
|
||||||
|
"""Build the bundle image from its Dockerfile, **cache-aware** — cheap
|
||||||
|
(a cache check) when nothing changed, a real rebuild when the flat
|
||||||
|
sources (egress addon / git-http / policy_resolver / supervise) moved.
|
||||||
|
|
||||||
|
This deliberately builds every time rather than build-if-missing: the
|
||||||
|
per-bottle model kept the image fresh via compose's `build:` on up, and
|
||||||
|
a stale image silently runs the OLD single-tenant daemons. No-op only
|
||||||
|
when no dockerfile is configured (a pre-pulled image). BOT_BOTTLE_NO_CACHE
|
||||||
|
forces a full rebuild (parity with `start --no-cache`)."""
|
||||||
|
if self._dockerfile is None:
|
||||||
|
return
|
||||||
|
argv = ["docker", "build", "-t", self.image_ref,
|
||||||
|
"-f", str(self._build_context / self._dockerfile),
|
||||||
|
str(self._build_context)]
|
||||||
|
if os.environ.get("BOT_BOTTLE_NO_CACHE"):
|
||||||
|
argv.insert(2, "--no-cache")
|
||||||
|
proc = run_docker(argv)
|
||||||
|
if proc.returncode != 0:
|
||||||
|
raise GatewayError(f"gateway image build failed: {proc.stderr.strip()}")
|
||||||
|
|
||||||
|
def is_running(self) -> bool:
|
||||||
|
proc = run_docker([
|
||||||
|
"docker", "ps",
|
||||||
|
"--filter", f"name=^{self.name}$",
|
||||||
|
"--filter", "status=running",
|
||||||
|
"--format", "{{.Names}}",
|
||||||
|
])
|
||||||
|
return self.name in proc.stdout.split()
|
||||||
|
|
||||||
|
def _running_image_is_current(self) -> bool:
|
||||||
|
"""True iff the running gateway was created from the *current*
|
||||||
|
`image_ref`. When `ensure_built` rebuilds the image (a source change),
|
||||||
|
the running container is still the OLD image running the OLD flat
|
||||||
|
daemons — so this is how a rebuild actually takes effect: a mismatch
|
||||||
|
means recreate."""
|
||||||
|
running = run_docker(["docker", "inspect", "--format", "{{.Image}}", self.name])
|
||||||
|
current = run_docker(["docker", "image", "inspect", "--format", "{{.Id}}", self.image_ref])
|
||||||
|
if running.returncode != 0 or current.returncode != 0:
|
||||||
|
return True # can't compare → don't churn a working container
|
||||||
|
return running.stdout.strip() == current.stdout.strip()
|
||||||
|
|
||||||
|
def _ensure_network(self) -> None:
|
||||||
|
"""Create the shared gateway network if it doesn't exist. Idempotent —
|
||||||
|
a concurrent create loses harmlessly (the loser sees 'already exists').
|
||||||
|
Docker picks the subnet; the launcher reads it back to allocate IPs."""
|
||||||
|
if run_docker(["docker", "network", "inspect", self.network]).returncode == 0:
|
||||||
|
return
|
||||||
|
proc = run_docker(["docker", "network", "create", self.network])
|
||||||
|
if proc.returncode != 0 and "already exists" not in proc.stderr:
|
||||||
|
raise GatewayError(
|
||||||
|
f"gateway network {self.network} failed to create: {proc.stderr.strip()}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def ensure_running(self) -> None:
|
||||||
|
# Fail closed on a missing policy source. The data-plane daemons are
|
||||||
|
# resolver-only now (PRD 0070) — without an orchestrator URL egress
|
||||||
|
# raises, git-http exits 1, and supervise exits 2 — so launching a
|
||||||
|
# gateway without one would only crash-loop its daemons. Refuse here so
|
||||||
|
# the misconfiguration surfaces as a clear error, not a broken container.
|
||||||
|
if not self._orchestrator_url:
|
||||||
|
raise GatewayError(
|
||||||
|
"gateway requires an orchestrator URL to run "
|
||||||
|
"(resolver-only data plane; no single-tenant fallback)"
|
||||||
|
)
|
||||||
|
# Recreate when the running container's image is stale (a rebuild),
|
||||||
|
# so source changes to the gateway's flat daemons take effect — not
|
||||||
|
# just when the container is absent.
|
||||||
|
if self.is_running() and self._running_image_is_current():
|
||||||
|
return
|
||||||
|
self._ensure_network()
|
||||||
|
# Clear any stale (stopped OR outdated-image) container holding the
|
||||||
|
# fixed name, then start fresh. `rm --force` on an absent name is a
|
||||||
|
# tolerated no-op.
|
||||||
|
run_docker(["docker", "rm", "--force", self.name])
|
||||||
|
argv = [
|
||||||
|
"docker", "run", "--detach",
|
||||||
|
"--name", self.name,
|
||||||
|
"--label", GATEWAY_LABEL,
|
||||||
|
"--network", self.network,
|
||||||
|
# Persist the self-generated CA on the host so it survives both
|
||||||
|
# container recreation AND docker volume pruning (agents trust it)
|
||||||
|
# — see host_gateway_ca_dir / issue #450.
|
||||||
|
"--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}",
|
||||||
|
# No DB mount: the data plane (egress / supervise / git-gate) reaches
|
||||||
|
# the supervise queue over the control-plane RPC and never opens
|
||||||
|
# bot-bottle.db, so the gateway container gets no file handle on it
|
||||||
|
# (PRD 0070 / issue #469).
|
||||||
|
]
|
||||||
|
for port in self._host_port_bindings:
|
||||||
|
argv += ["--publish", f"0.0.0.0:{port}:{port}"]
|
||||||
|
run_env = dict(os.environ)
|
||||||
|
# The gateway's egress / git / supervise daemons resolve source-IP ->
|
||||||
|
# policy against the control plane per request (guaranteed non-empty by
|
||||||
|
# the check above).
|
||||||
|
argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"]
|
||||||
|
# ...presenting a role-scoped `gateway` token (a signed JWT minted from
|
||||||
|
# the host signing key) on those calls. The gateway never receives the
|
||||||
|
# signing key — only this pre-minted token, which it cannot rewrite into
|
||||||
|
# a `cli` token, so a compromised data-plane process can't drive the
|
||||||
|
# operator routes (issue #469 review). Bare `--env NAME` keeps the value
|
||||||
|
# off argv / `docker inspect`; only the gateway (not the agent) is given it.
|
||||||
|
argv += ["--env", CONTROL_AUTH_JWT_ENV]
|
||||||
|
run_env[CONTROL_AUTH_JWT_ENV] = mint(ROLE_GATEWAY, host_control_plane_token())
|
||||||
|
argv.append(self.image_ref)
|
||||||
|
proc = run_docker(argv, env=run_env)
|
||||||
|
if proc.returncode != 0:
|
||||||
|
raise GatewayError(f"gateway failed to start: {proc.stderr.strip()}")
|
||||||
|
|
||||||
|
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
||||||
|
"""The gateway's CA certificate (PEM) that agents install to trust its
|
||||||
|
TLS interception. mitmproxy generates it a moment after the container
|
||||||
|
starts, so this **polls** for it (up to `timeout`) rather than assuming
|
||||||
|
it's already there on a fresh gateway — raising only if it never
|
||||||
|
appears."""
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
while True:
|
||||||
|
proc = run_docker(["docker", "exec", self.name, "cat", GATEWAY_CA_CERT])
|
||||||
|
if proc.returncode == 0 and proc.stdout.strip():
|
||||||
|
return proc.stdout
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
raise GatewayError(
|
||||||
|
f"gateway CA cert not available after {timeout:g}s: "
|
||||||
|
f"{proc.stderr.strip() or 'empty'}"
|
||||||
|
)
|
||||||
|
time.sleep(CA_POLL_SECONDS)
|
||||||
|
|
||||||
|
def stop(self) -> None:
|
||||||
|
proc = run_docker(["docker", "rm", "--force", self.name])
|
||||||
|
if proc.returncode != 0 and "No such container" not in proc.stderr:
|
||||||
|
raise GatewayError(f"gateway failed to stop: {proc.stderr.strip()}")
|
||||||
@@ -66,7 +66,8 @@ from .compose import (
|
|||||||
from .consolidated_compose import consolidated_agent_compose
|
from .consolidated_compose import consolidated_agent_compose
|
||||||
from ...orchestrator.config_store import resolve_teardown_timeout
|
from ...orchestrator.config_store import resolve_teardown_timeout
|
||||||
from .consolidated_launch import launch_consolidated, teardown_consolidated
|
from .consolidated_launch import launch_consolidated, teardown_consolidated
|
||||||
from ...orchestrator.gateway import DockerGateway
|
from ...orchestrator.lifecycle import INFRA_NAME
|
||||||
|
from .gateway import DockerGateway
|
||||||
|
|
||||||
|
|
||||||
# Where the repo root lives, for `docker build` context. Computed once.
|
# Where the repo root lives, for `docker build` context. Computed once.
|
||||||
@@ -159,11 +160,14 @@ def launch(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Step 4: install the SHARED gateway CA into the agent (replaces the
|
# Step 4: install the SHARED gateway CA into the agent (replaces the
|
||||||
# per-bottle CA) — read it out of the running gateway.
|
# per-bottle CA) — read it out of the running gateway. The consolidated
|
||||||
|
# flow runs the gateway data plane inside the per-host infra container
|
||||||
|
# (INFRA_NAME), so read the CA from there, not the legacy standalone
|
||||||
|
# gateway container name.
|
||||||
ca_dir = egress_state_dir(plan.slug) / "gateway-ca"
|
ca_dir = egress_state_dir(plan.slug) / "gateway-ca"
|
||||||
ca_dir.mkdir(parents=True, exist_ok=True)
|
ca_dir.mkdir(parents=True, exist_ok=True)
|
||||||
ca_file = ca_dir / "gateway-ca.pem"
|
ca_file = ca_dir / "gateway-ca.pem"
|
||||||
ca_file.write_text(DockerGateway(network=ctx.network).ca_cert_pem())
|
ca_file.write_text(DockerGateway(name=INFRA_NAME).ca_cert_pem())
|
||||||
egress_plan = dataclasses.replace(
|
egress_plan = dataclasses.replace(
|
||||||
plan.egress_plan,
|
plan.egress_plan,
|
||||||
mitmproxy_ca_host_path=ca_file,
|
mitmproxy_ca_host_path=ca_file,
|
||||||
|
|||||||
@@ -10,10 +10,10 @@ backend-neutral "consolidation core" that needs no VM packaging:
|
|||||||
* `broker` — the signed, structured launch-request contract + a
|
* `broker` — the signed, structured launch-request contract + a
|
||||||
`LaunchBroker` (stub for the harness) that verifies
|
`LaunchBroker` (stub for the harness) that verifies
|
||||||
provenance before acting.
|
provenance before acting.
|
||||||
* `gateway` — the consolidated per-host gateway: a `Gateway`
|
* `gateway` — the consolidated per-host gateway: the `Gateway`
|
||||||
lifecycle contract (idempotent singleton) + a
|
lifecycle contract (idempotent singleton). One
|
||||||
`DockerGateway` impl. One gateway shared by all
|
gateway shared by all bottles instead of one per
|
||||||
bottles instead of one per bottle.
|
bottle; backend impls live under `backend/*/gateway`.
|
||||||
* `service` — the `Orchestrator`: owns the registry, brokers the
|
* `service` — the `Orchestrator`: owns the registry, brokers the
|
||||||
launch lifecycle (launch/teardown), manages the
|
launch lifecycle (launch/teardown), manages the
|
||||||
shared gateway, attributes.
|
shared gateway, attributes.
|
||||||
@@ -38,7 +38,7 @@ from .broker import (
|
|||||||
verify_request,
|
verify_request,
|
||||||
)
|
)
|
||||||
from .docker_broker import DockerBroker, DockerBrokerError
|
from .docker_broker import DockerBroker, DockerBrokerError
|
||||||
from .gateway import DockerGateway, Gateway, GatewayError
|
from .gateway import Gateway, GatewayError
|
||||||
from .service import Orchestrator
|
from .service import Orchestrator
|
||||||
from .control_plane import ControlPlaneServer, dispatch, make_server
|
from .control_plane import ControlPlaneServer, dispatch, make_server
|
||||||
|
|
||||||
@@ -53,7 +53,6 @@ __all__ = [
|
|||||||
"DockerBroker",
|
"DockerBroker",
|
||||||
"DockerBrokerError",
|
"DockerBrokerError",
|
||||||
"Gateway",
|
"Gateway",
|
||||||
"DockerGateway",
|
|
||||||
"GatewayError",
|
"GatewayError",
|
||||||
"sign_request",
|
"sign_request",
|
||||||
"verify_request",
|
"verify_request",
|
||||||
|
|||||||
@@ -22,7 +22,6 @@ from .control_plane import make_server
|
|||||||
from .docker_broker import DockerBroker
|
from .docker_broker import DockerBroker
|
||||||
from .registry import RegistryStore, default_db_path
|
from .registry import RegistryStore, default_db_path
|
||||||
from .service import Orchestrator
|
from .service import Orchestrator
|
||||||
from .gateway import DockerGateway, Gateway
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
def main(argv: list[str] | None = None) -> int:
|
||||||
@@ -38,10 +37,6 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
"--broker", choices=("stub", "docker"), default="stub",
|
"--broker", choices=("stub", "docker"), default="stub",
|
||||||
help="launch broker: 'stub' records requests; 'docker' runs containers",
|
help="launch broker: 'stub' records requests; 'docker' runs containers",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
|
||||||
"--gateway", action="store_true",
|
|
||||||
help="run one consolidated per-host gateway (build-if-missing)",
|
|
||||||
)
|
|
||||||
args = parser.parse_args(argv)
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
registry = RegistryStore(args.db)
|
registry = RegistryStore(args.db)
|
||||||
@@ -57,20 +52,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
# anything; 'docker' runs real containers (firecracker drops in later).
|
# anything; 'docker' runs real containers (firecracker drops in later).
|
||||||
secret = secrets.token_bytes(32)
|
secret = secrets.token_bytes(32)
|
||||||
broker: LaunchBroker = DockerBroker(secret) if args.broker == "docker" else StubBroker(secret)
|
broker: LaunchBroker = DockerBroker(secret) if args.broker == "docker" else StubBroker(secret)
|
||||||
# Standalone `--gateway` (not the consolidated flow, where the host
|
orchestrator = Orchestrator(registry, broker, secret)
|
||||||
# lifecycle runs the gateway). The gateway resolves against this same
|
|
||||||
# process; the URL is only reachable when they share a docker network.
|
|
||||||
gateway: Gateway | None = (
|
|
||||||
DockerGateway(orchestrator_url=f"http://{args.host}:{args.port}")
|
|
||||||
if args.gateway else None
|
|
||||||
)
|
|
||||||
orchestrator = Orchestrator(registry, broker, secret, gateway)
|
|
||||||
|
|
||||||
# One persistent per-host gateway, shared by every bottle: build the
|
|
||||||
# bundle image if missing, then bring the singleton up (idempotent).
|
|
||||||
if gateway is not None:
|
|
||||||
orchestrator.ensure_gateway()
|
|
||||||
log.info("consolidated gateway ensured", context={"name": gateway.name})
|
|
||||||
|
|
||||||
server = make_server(orchestrator, host=args.host, port=args.port)
|
server = make_server(orchestrator, host=args.host, port=args.port)
|
||||||
bound_host, bound_port = server.server_address[0], server.server_address[1]
|
bound_host, bound_port = server.server_address[0], server.server_address[1]
|
||||||
|
|||||||
@@ -7,8 +7,9 @@ because the attribution invariant (source IP + identity token, see
|
|||||||
so per-bottle policy lives in one long-lived process keyed on who's calling.
|
so per-bottle policy lives in one long-lived process keyed on who's calling.
|
||||||
|
|
||||||
`Gateway` is the backend-neutral lifecycle contract (mirrors `LaunchBroker`):
|
`Gateway` is the backend-neutral lifecycle contract (mirrors `LaunchBroker`):
|
||||||
ensure the single instance is up, report it, tear it down. `DockerGateway`
|
ensure the single instance is up, report it, tear it down. The docker
|
||||||
is the docker implementation; a firecracker gateway VM slots in later.
|
implementation (`DockerGateway`) lives in `backend/docker/gateway.py`; a
|
||||||
|
firecracker gateway VM slots in later.
|
||||||
|
|
||||||
The defining behaviour is **idempotent singleton**: `ensure_running` starts
|
The defining behaviour is **idempotent singleton**: `ensure_running` starts
|
||||||
the instance if absent and is a no-op if it's already up, so N bottle
|
the instance if absent and is a no-op if it's already up, so N bottle
|
||||||
@@ -19,20 +20,13 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import abc
|
import abc
|
||||||
import os
|
import os
|
||||||
import time
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ..control_auth import ROLE_GATEWAY, mint
|
from ..paths import host_gateway_ca_dir
|
||||||
from ..docker_cmd import run_docker
|
|
||||||
from ..paths import (
|
|
||||||
CONTROL_AUTH_JWT_ENV,
|
|
||||||
host_control_plane_token,
|
|
||||||
host_gateway_ca_dir,
|
|
||||||
)
|
|
||||||
|
|
||||||
# The gateway's mitmproxy writes its CA a beat after the container starts, so
|
# The gateway's mitmproxy writes its CA a beat after the container starts, so
|
||||||
# reads poll for it rather than assuming it's there on a fresh launch.
|
# reads poll for it rather than assuming it's there on a fresh launch.
|
||||||
_CA_POLL_SECONDS = 0.5
|
CA_POLL_SECONDS = 0.5
|
||||||
DEFAULT_CA_TIMEOUT_SECONDS = 30.0
|
DEFAULT_CA_TIMEOUT_SECONDS = 30.0
|
||||||
|
|
||||||
GATEWAY_NAME = "bot-bottle-orch-gateway"
|
GATEWAY_NAME = "bot-bottle-orch-gateway"
|
||||||
@@ -66,7 +60,7 @@ GATEWAY_CA_GLOB = "mitmproxy-ca*"
|
|||||||
# that lands. Env override matches the backend's BOT_BOTTLE_GATEWAY_IMAGE.
|
# that lands. Env override matches the backend's BOT_BOTTLE_GATEWAY_IMAGE.
|
||||||
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
||||||
GATEWAY_DOCKERFILE = "Dockerfile.gateway"
|
GATEWAY_DOCKERFILE = "Dockerfile.gateway"
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]:
|
def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]:
|
||||||
@@ -118,180 +112,8 @@ class Gateway(abc.ABC):
|
|||||||
"""Remove the gateway. Idempotent — absent is success."""
|
"""Remove the gateway. Idempotent — absent is success."""
|
||||||
|
|
||||||
|
|
||||||
class DockerGateway(Gateway):
|
|
||||||
"""The consolidated gateway as a single, fixed-name Docker container.
|
|
||||||
|
|
||||||
`image_ref` defaults to the gateway data-plane image; `ensure_built`
|
|
||||||
builds it from `Dockerfile.gateway` when it's missing. (Note: slice 5
|
|
||||||
builds + launches the bundle container; wiring its per-bottle,
|
|
||||||
source-IP-keyed config is a later slice — see PRD 0070.)"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
image_ref: str = GATEWAY_IMAGE,
|
|
||||||
*,
|
|
||||||
name: str = GATEWAY_NAME,
|
|
||||||
network: str = GATEWAY_NETWORK,
|
|
||||||
orchestrator_url: str = "",
|
|
||||||
build_context: Path | None = None,
|
|
||||||
dockerfile: str | None = GATEWAY_DOCKERFILE,
|
|
||||||
host_port_bindings: tuple[int, ...] = (),
|
|
||||||
) -> None:
|
|
||||||
self.image_ref = image_ref
|
|
||||||
self.name = name
|
|
||||||
self.network = network
|
|
||||||
# The control-plane URL the gateway's data plane resolves per bottle
|
|
||||||
# against — reached by container name over docker DNS on the shared
|
|
||||||
# network (container↔container, no host firewall). Mandatory to *run*
|
|
||||||
# the gateway (see `ensure_running`); empty is tolerated only for the
|
|
||||||
# construct-then-read-CA path (`ca_cert_pem` on an already-running
|
|
||||||
# container), which never launches a container.
|
|
||||||
self._orchestrator_url = orchestrator_url
|
|
||||||
self._build_context = build_context or _REPO_ROOT
|
|
||||||
self._dockerfile = dockerfile
|
|
||||||
# Ports published on the host (0.0.0.0). Used by the Firecracker
|
|
||||||
# backend's dev-harness gateway so VMs can reach it via their TAP link;
|
|
||||||
# Docker's DNAT + the nft `ct status dnat accept` rule handle the rest.
|
|
||||||
self._host_port_bindings = host_port_bindings
|
|
||||||
|
|
||||||
def image_exists(self) -> bool:
|
|
||||||
return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0
|
|
||||||
|
|
||||||
def ensure_built(self) -> None:
|
|
||||||
"""Build the bundle image from its Dockerfile, **cache-aware** — cheap
|
|
||||||
(a cache check) when nothing changed, a real rebuild when the flat
|
|
||||||
sources (egress addon / git-http / policy_resolver / supervise) moved.
|
|
||||||
|
|
||||||
This deliberately builds every time rather than build-if-missing: the
|
|
||||||
per-bottle model kept the image fresh via compose's `build:` on up, and
|
|
||||||
a stale image silently runs the OLD single-tenant daemons. No-op only
|
|
||||||
when no dockerfile is configured (a pre-pulled image). BOT_BOTTLE_NO_CACHE
|
|
||||||
forces a full rebuild (parity with `start --no-cache`)."""
|
|
||||||
if self._dockerfile is None:
|
|
||||||
return
|
|
||||||
argv = ["docker", "build", "-t", self.image_ref,
|
|
||||||
"-f", str(self._build_context / self._dockerfile),
|
|
||||||
str(self._build_context)]
|
|
||||||
if os.environ.get("BOT_BOTTLE_NO_CACHE"):
|
|
||||||
argv.insert(2, "--no-cache")
|
|
||||||
proc = run_docker(argv)
|
|
||||||
if proc.returncode != 0:
|
|
||||||
raise GatewayError(f"gateway image build failed: {proc.stderr.strip()}")
|
|
||||||
|
|
||||||
def is_running(self) -> bool:
|
|
||||||
proc = run_docker([
|
|
||||||
"docker", "ps",
|
|
||||||
"--filter", f"name=^{self.name}$",
|
|
||||||
"--filter", "status=running",
|
|
||||||
"--format", "{{.Names}}",
|
|
||||||
])
|
|
||||||
return self.name in proc.stdout.split()
|
|
||||||
|
|
||||||
def _running_image_is_current(self) -> bool:
|
|
||||||
"""True iff the running gateway was created from the *current*
|
|
||||||
`image_ref`. When `ensure_built` rebuilds the image (a source change),
|
|
||||||
the running container is still the OLD image running the OLD flat
|
|
||||||
daemons — so this is how a rebuild actually takes effect: a mismatch
|
|
||||||
means recreate."""
|
|
||||||
running = run_docker(["docker", "inspect", "--format", "{{.Image}}", self.name])
|
|
||||||
current = run_docker(["docker", "image", "inspect", "--format", "{{.Id}}", self.image_ref])
|
|
||||||
if running.returncode != 0 or current.returncode != 0:
|
|
||||||
return True # can't compare → don't churn a working container
|
|
||||||
return running.stdout.strip() == current.stdout.strip()
|
|
||||||
|
|
||||||
def _ensure_network(self) -> None:
|
|
||||||
"""Create the shared gateway network if it doesn't exist. Idempotent —
|
|
||||||
a concurrent create loses harmlessly (the loser sees 'already exists').
|
|
||||||
Docker picks the subnet; the launcher reads it back to allocate IPs."""
|
|
||||||
if run_docker(["docker", "network", "inspect", self.network]).returncode == 0:
|
|
||||||
return
|
|
||||||
proc = run_docker(["docker", "network", "create", self.network])
|
|
||||||
if proc.returncode != 0 and "already exists" not in proc.stderr:
|
|
||||||
raise GatewayError(
|
|
||||||
f"gateway network {self.network} failed to create: {proc.stderr.strip()}"
|
|
||||||
)
|
|
||||||
|
|
||||||
def ensure_running(self) -> None:
|
|
||||||
# Fail closed on a missing policy source. The data-plane daemons are
|
|
||||||
# resolver-only now (PRD 0070) — without an orchestrator URL egress
|
|
||||||
# raises, git-http exits 1, and supervise exits 2 — so launching a
|
|
||||||
# gateway without one would only crash-loop its daemons. Refuse here so
|
|
||||||
# the misconfiguration surfaces as a clear error, not a broken container.
|
|
||||||
if not self._orchestrator_url:
|
|
||||||
raise GatewayError(
|
|
||||||
"gateway requires an orchestrator URL to run "
|
|
||||||
"(resolver-only data plane; no single-tenant fallback)"
|
|
||||||
)
|
|
||||||
# Recreate when the running container's image is stale (a rebuild),
|
|
||||||
# so source changes to the gateway's flat daemons take effect — not
|
|
||||||
# just when the container is absent.
|
|
||||||
if self.is_running() and self._running_image_is_current():
|
|
||||||
return
|
|
||||||
self._ensure_network()
|
|
||||||
# Clear any stale (stopped OR outdated-image) container holding the
|
|
||||||
# fixed name, then start fresh. `rm --force` on an absent name is a
|
|
||||||
# tolerated no-op.
|
|
||||||
run_docker(["docker", "rm", "--force", self.name])
|
|
||||||
argv = [
|
|
||||||
"docker", "run", "--detach",
|
|
||||||
"--name", self.name,
|
|
||||||
"--label", GATEWAY_LABEL,
|
|
||||||
"--network", self.network,
|
|
||||||
# Persist the self-generated CA on the host so it survives both
|
|
||||||
# container recreation AND docker volume pruning (agents trust it)
|
|
||||||
# — see host_gateway_ca_dir / issue #450.
|
|
||||||
"--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}",
|
|
||||||
# No DB mount: the data plane (egress / supervise / git-gate) reaches
|
|
||||||
# the supervise queue over the control-plane RPC and never opens
|
|
||||||
# bot-bottle.db, so the gateway container gets no file handle on it
|
|
||||||
# (PRD 0070 / issue #469).
|
|
||||||
]
|
|
||||||
for port in self._host_port_bindings:
|
|
||||||
argv += ["--publish", f"0.0.0.0:{port}:{port}"]
|
|
||||||
run_env = dict(os.environ)
|
|
||||||
# The gateway's egress / git / supervise daemons resolve source-IP ->
|
|
||||||
# policy against the control plane per request (guaranteed non-empty by
|
|
||||||
# the check above).
|
|
||||||
argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"]
|
|
||||||
# ...presenting a role-scoped `gateway` token (a signed JWT minted from
|
|
||||||
# the host signing key) on those calls. The gateway never receives the
|
|
||||||
# signing key — only this pre-minted token, which it cannot rewrite into
|
|
||||||
# a `cli` token, so a compromised data-plane process can't drive the
|
|
||||||
# operator routes (issue #469 review). Bare `--env NAME` keeps the value
|
|
||||||
# off argv / `docker inspect`; only the gateway (not the agent) is given it.
|
|
||||||
argv += ["--env", CONTROL_AUTH_JWT_ENV]
|
|
||||||
run_env[CONTROL_AUTH_JWT_ENV] = mint(ROLE_GATEWAY, host_control_plane_token())
|
|
||||||
argv.append(self.image_ref)
|
|
||||||
proc = run_docker(argv, env=run_env)
|
|
||||||
if proc.returncode != 0:
|
|
||||||
raise GatewayError(f"gateway failed to start: {proc.stderr.strip()}")
|
|
||||||
|
|
||||||
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
|
||||||
"""The gateway's CA certificate (PEM) that agents install to trust its
|
|
||||||
TLS interception. mitmproxy generates it a moment after the container
|
|
||||||
starts, so this **polls** for it (up to `timeout`) rather than assuming
|
|
||||||
it's already there on a fresh gateway — raising only if it never
|
|
||||||
appears."""
|
|
||||||
deadline = time.monotonic() + timeout
|
|
||||||
while True:
|
|
||||||
proc = run_docker(["docker", "exec", self.name, "cat", GATEWAY_CA_CERT])
|
|
||||||
if proc.returncode == 0 and proc.stdout.strip():
|
|
||||||
return proc.stdout
|
|
||||||
if time.monotonic() >= deadline:
|
|
||||||
raise GatewayError(
|
|
||||||
f"gateway CA cert not available after {timeout:g}s: "
|
|
||||||
f"{proc.stderr.strip() or 'empty'}"
|
|
||||||
)
|
|
||||||
time.sleep(_CA_POLL_SECONDS)
|
|
||||||
|
|
||||||
def stop(self) -> None:
|
|
||||||
proc = run_docker(["docker", "rm", "--force", self.name])
|
|
||||||
if proc.returncode != 0 and "No such container" not in proc.stderr:
|
|
||||||
raise GatewayError(f"gateway failed to stop: {proc.stderr.strip()}")
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"Gateway", "DockerGateway", "GatewayError", "rotate_gateway_ca",
|
"Gateway", "GatewayError", "rotate_gateway_ca",
|
||||||
"GATEWAY_NAME", "GATEWAY_LABEL", "GATEWAY_IMAGE", "GATEWAY_NETWORK",
|
"GATEWAY_NAME", "GATEWAY_LABEL", "GATEWAY_IMAGE", "GATEWAY_NETWORK",
|
||||||
"GATEWAY_CA_CERT", "GATEWAY_CA_GLOB",
|
"GATEWAY_CA_CERT", "GATEWAY_CA_GLOB",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -27,7 +27,6 @@ from datetime import datetime, timezone
|
|||||||
|
|
||||||
from .broker import LaunchBroker, LaunchRequest, sign_request
|
from .broker import LaunchBroker, LaunchRequest, sign_request
|
||||||
from .registry import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore
|
from .registry import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore
|
||||||
from .gateway import Gateway
|
|
||||||
from ..supervise import (
|
from ..supervise import (
|
||||||
AuditEntry,
|
AuditEntry,
|
||||||
COMPONENT_FOR_TOOL,
|
COMPONENT_FOR_TOOL,
|
||||||
@@ -72,12 +71,10 @@ class Orchestrator:
|
|||||||
registry: RegistryStore,
|
registry: RegistryStore,
|
||||||
broker: LaunchBroker,
|
broker: LaunchBroker,
|
||||||
sign_secret: bytes,
|
sign_secret: bytes,
|
||||||
gateway: Gateway | None = None,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
self.registry = registry
|
self.registry = registry
|
||||||
self._broker = broker
|
self._broker = broker
|
||||||
self._secret = sign_secret
|
self._secret = sign_secret
|
||||||
self._gateway = gateway
|
|
||||||
# Per-bottle egress auth tokens (env_name -> value), keyed by bottle_id.
|
# Per-bottle egress auth tokens (env_name -> value), keyed by bottle_id.
|
||||||
# Held **in memory only** — never written to the registry DB — so the
|
# Held **in memory only** — never written to the registry DB — so the
|
||||||
# gateway can inject each bottle's upstream credential without secrets
|
# gateway can inject each bottle's upstream credential without secrets
|
||||||
@@ -383,22 +380,15 @@ class Orchestrator:
|
|||||||
|
|
||||||
# --- consolidated gateway ----------------------------------------------
|
# --- consolidated gateway ----------------------------------------------
|
||||||
|
|
||||||
def ensure_gateway(self) -> None:
|
|
||||||
"""Ensure the single per-host gateway is built and up (idempotent).
|
|
||||||
No-op when no gateway is configured."""
|
|
||||||
if self._gateway is not None:
|
|
||||||
self._gateway.ensure_built()
|
|
||||||
self._gateway.ensure_running()
|
|
||||||
|
|
||||||
def gateway_status(self) -> dict[str, object]:
|
def gateway_status(self) -> dict[str, object]:
|
||||||
"""Report the shared gateway for the control plane / console."""
|
"""Report the shared gateway for the control plane / console.
|
||||||
if self._gateway is None:
|
|
||||||
|
The orchestrator no longer owns a standalone gateway lifecycle — the
|
||||||
|
consolidated flow runs the gateway data plane inside the per-host infra
|
||||||
|
container/VM (see `backend/*/gateway`), so this reports `configured:
|
||||||
|
false`. Retained for the documented `GET /gateway` control-plane
|
||||||
|
contract."""
|
||||||
return {"configured": False}
|
return {"configured": False}
|
||||||
return {
|
|
||||||
"configured": True,
|
|
||||||
"name": self._gateway.name,
|
|
||||||
"running": self._gateway.is_running(),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["Orchestrator"]
|
__all__ = ["Orchestrator"]
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import secrets
|
|||||||
import subprocess
|
import subprocess
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from bot_bottle.orchestrator.gateway import DockerGateway
|
from bot_bottle.backend.docker.gateway import DockerGateway
|
||||||
from tests._docker import skip_unless_docker
|
from tests._docker import skip_unless_docker
|
||||||
|
|
||||||
IMAGE = "busybox"
|
IMAGE = "busybox"
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from __future__ import annotations
|
|||||||
import subprocess
|
import subprocess
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from bot_bottle.orchestrator.gateway import DockerGateway
|
from bot_bottle.backend.docker.gateway import DockerGateway
|
||||||
from tests._docker import skip_unless_docker
|
from tests._docker import skip_unless_docker
|
||||||
|
|
||||||
IMAGE = "busybox"
|
IMAGE = "busybox"
|
||||||
|
|||||||
@@ -7,10 +7,10 @@ import unittest
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
|
from bot_bottle.backend.docker.gateway import DockerGateway
|
||||||
from bot_bottle.orchestrator.gateway import (
|
from bot_bottle.orchestrator.gateway import (
|
||||||
GATEWAY_CA_CERT,
|
GATEWAY_CA_CERT,
|
||||||
GATEWAY_NAME,
|
GATEWAY_NAME,
|
||||||
DockerGateway,
|
|
||||||
GatewayError,
|
GatewayError,
|
||||||
rotate_gateway_ca,
|
rotate_gateway_ca,
|
||||||
)
|
)
|
||||||
@@ -20,7 +20,7 @@ from tests.unit import use_bottle_root
|
|||||||
|
|
||||||
_CA_PEM = "-----BEGIN CERTIFICATE-----\nMII...\n-----END CERTIFICATE-----\n"
|
_CA_PEM = "-----BEGIN CERTIFICATE-----\nMII...\n-----END CERTIFICATE-----\n"
|
||||||
|
|
||||||
_RUN_DOCKER = "bot_bottle.orchestrator.gateway.run_docker"
|
_RUN_DOCKER = "bot_bottle.backend.docker.gateway.run_docker"
|
||||||
|
|
||||||
|
|
||||||
def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock:
|
def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock:
|
||||||
@@ -162,7 +162,7 @@ class TestDockerGateway(unittest.TestCase):
|
|||||||
# First read: CA not there yet; second read: present.
|
# First read: CA not there yet; second read: present.
|
||||||
seq = [_proc(returncode=1, stderr="No such file"), _proc(stdout=_CA_PEM)]
|
seq = [_proc(returncode=1, stderr="No such file"), _proc(stdout=_CA_PEM)]
|
||||||
with patch(_RUN_DOCKER, side_effect=seq), \
|
with patch(_RUN_DOCKER, side_effect=seq), \
|
||||||
patch("bot_bottle.orchestrator.gateway.time.sleep"):
|
patch("bot_bottle.backend.docker.gateway.time.sleep"):
|
||||||
self.assertEqual(_CA_PEM, self.sc.ca_cert_pem(timeout=5))
|
self.assertEqual(_CA_PEM, self.sc.ca_cert_pem(timeout=5))
|
||||||
|
|
||||||
def test_ensure_running_reuses_existing_network(self) -> None:
|
def test_ensure_running_reuses_existing_network(self) -> None:
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ from unittest.mock import patch
|
|||||||
from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBroker
|
from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBroker
|
||||||
from bot_bottle.orchestrator.registry import RegistryStore
|
from bot_bottle.orchestrator.registry import RegistryStore
|
||||||
from bot_bottle.orchestrator.service import Orchestrator
|
from bot_bottle.orchestrator.service import Orchestrator
|
||||||
from bot_bottle.orchestrator.gateway import Gateway
|
|
||||||
from bot_bottle.orchestrator.secret_store import new_env_var_secret
|
from bot_bottle.orchestrator.secret_store import new_env_var_secret
|
||||||
from bot_bottle.store_manager import StoreManager
|
from bot_bottle.store_manager import StoreManager
|
||||||
from bot_bottle.supervise import (
|
from bot_bottle.supervise import (
|
||||||
@@ -38,29 +37,6 @@ class _FailingBroker(LaunchBroker):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class _FakeGateway(Gateway):
|
|
||||||
"""In-memory gateway for wiring tests."""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self.name = "fake-gateway"
|
|
||||||
self.ensured = 0
|
|
||||||
self.built = 0
|
|
||||||
self._running = False
|
|
||||||
|
|
||||||
def ensure_built(self) -> None:
|
|
||||||
self.built += 1
|
|
||||||
|
|
||||||
def ensure_running(self) -> None:
|
|
||||||
self.ensured += 1
|
|
||||||
self._running = True
|
|
||||||
|
|
||||||
def is_running(self) -> bool:
|
|
||||||
return self._running
|
|
||||||
|
|
||||||
def stop(self) -> None:
|
|
||||||
self._running = False
|
|
||||||
|
|
||||||
|
|
||||||
class TestOrchestrator(unittest.TestCase):
|
class TestOrchestrator(unittest.TestCase):
|
||||||
def setUp(self) -> None:
|
def setUp(self) -> None:
|
||||||
self._tmp = tempfile.TemporaryDirectory()
|
self._tmp = tempfile.TemporaryDirectory()
|
||||||
@@ -168,24 +144,11 @@ class TestOrchestrator(unittest.TestCase):
|
|||||||
orch.launch_bottle("10.243.0.9")
|
orch.launch_bottle("10.243.0.9")
|
||||||
self.assertEqual([], self.store.all()) # no orphan
|
self.assertEqual([], self.store.all()) # no orphan
|
||||||
|
|
||||||
def test_gateway_unconfigured_by_default(self) -> None:
|
def test_gateway_status_reports_unconfigured(self) -> None:
|
||||||
|
# The orchestrator no longer owns a standalone gateway lifecycle; the
|
||||||
|
# consolidated flow runs the gateway data plane in the per-host infra
|
||||||
|
# container/VM. `GET /gateway` therefore always reports unconfigured.
|
||||||
self.assertEqual({"configured": False}, self.orch.gateway_status())
|
self.assertEqual({"configured": False}, self.orch.gateway_status())
|
||||||
self.orch.ensure_gateway() # no-op, must not raise
|
|
||||||
|
|
||||||
def test_gateway_wired_and_ensured(self) -> None:
|
|
||||||
sc = _FakeGateway()
|
|
||||||
orch = Orchestrator(self.store, self.broker, self.secret, gateway=sc)
|
|
||||||
self.assertEqual(
|
|
||||||
{"configured": True, "name": "fake-gateway", "running": False},
|
|
||||||
orch.gateway_status(),
|
|
||||||
)
|
|
||||||
orch.ensure_gateway()
|
|
||||||
self.assertEqual(1, sc.built) # ensure_gateway builds first,
|
|
||||||
self.assertEqual(1, sc.ensured) # then runs
|
|
||||||
self.assertEqual(
|
|
||||||
{"configured": True, "name": "fake-gateway", "running": True},
|
|
||||||
orch.gateway_status(),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestOrchestratorSupervise(unittest.TestCase):
|
class TestOrchestratorSupervise(unittest.TestCase):
|
||||||
|
|||||||
Reference in New Issue
Block a user