Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4b17e6d683 | |||
| 7a48ea2b0c | |||
| ec953ceda7 | |||
| ed0f95f445 | |||
| 794e4e662d | |||
| f2fe1f9b2d |
@@ -102,20 +102,6 @@ jobs:
|
|||||||
python3 --version
|
python3 --version
|
||||||
python3 cli.py backend status --backend=docker
|
python3 cli.py backend status --backend=docker
|
||||||
|
|
||||||
- name: Preflight — clear any leftover poisoned gateway network
|
|
||||||
run: |
|
|
||||||
# The gateway network has a fixed name and persists across jobs on
|
|
||||||
# this shared runner. A pre-fix or concurrent launch can leave it with
|
|
||||||
# a malformed IPv6 subnet that trips docker's own ParseAddr in
|
|
||||||
# `network inspect` (see PR #515); the code now self-heals it, but the
|
|
||||||
# heal can't run if `network inspect` is what's broken on some daemon
|
|
||||||
# versions. Drop the network here so this run recreates it IPv4-only.
|
|
||||||
# Remove the attached gateway container first (else `network rm` fails
|
|
||||||
# on active endpoints); both are recreated by ensure_running. Harmless
|
|
||||||
# when absent.
|
|
||||||
docker rm --force bot-bottle-orch-gateway 2>/dev/null || true
|
|
||||||
docker network rm bot-bottle-gateway 2>/dev/null || true
|
|
||||||
|
|
||||||
- name: Run integration tests (docker) with coverage
|
- name: Run integration tests (docker) with coverage
|
||||||
env:
|
env:
|
||||||
BOT_BOTTLE_BACKEND: docker
|
BOT_BOTTLE_BACKEND: docker
|
||||||
@@ -298,7 +284,7 @@ jobs:
|
|||||||
- name: Combined coverage (unit + docker integration)
|
- name: Combined coverage (unit + docker integration)
|
||||||
run: PYTHON=python3 bash scripts/coverage.sh aggregate critical
|
run: PYTHON=python3 bash scripts/coverage.sh aggregate critical
|
||||||
|
|
||||||
- name: Diff-coverage gate (changed lines >= 80%)
|
- name: Diff-coverage gate (changed lines >= 90%)
|
||||||
run: |
|
run: |
|
||||||
git fetch --no-tags origin main:refs/remotes/origin/main
|
git fetch --no-tags origin main:refs/remotes/origin/main
|
||||||
python3 scripts/diff_coverage.py --base origin/main --min 80
|
python3 scripts/diff_coverage.py --base origin/main --min 90
|
||||||
|
|||||||
@@ -30,7 +30,6 @@ if TYPE_CHECKING:
|
|||||||
BottleImages,
|
BottleImages,
|
||||||
BottlePlan,
|
BottlePlan,
|
||||||
BottleSpec,
|
BottleSpec,
|
||||||
EnumerationError,
|
|
||||||
ExecResult,
|
ExecResult,
|
||||||
)
|
)
|
||||||
from .selection import (
|
from .selection import (
|
||||||
@@ -60,7 +59,6 @@ _LAZY_MODULES: dict[str, str] = {
|
|||||||
"BottleImages": "base",
|
"BottleImages": "base",
|
||||||
"BottleBackend": "base",
|
"BottleBackend": "base",
|
||||||
"BackendStatus": "base",
|
"BackendStatus": "base",
|
||||||
"EnumerationError": "base",
|
|
||||||
"get_bottle_backend": "selection",
|
"get_bottle_backend": "selection",
|
||||||
"known_backend_names": "selection",
|
"known_backend_names": "selection",
|
||||||
"has_backend": "selection",
|
"has_backend": "selection",
|
||||||
@@ -102,7 +100,6 @@ __all__ = [
|
|||||||
"BottlePlan",
|
"BottlePlan",
|
||||||
"BottleSpec",
|
"BottleSpec",
|
||||||
"ExecResult",
|
"ExecResult",
|
||||||
"EnumerationError",
|
|
||||||
"CommitCancelled",
|
"CommitCancelled",
|
||||||
"Freezer",
|
"Freezer",
|
||||||
"get_freezer",
|
"get_freezer",
|
||||||
|
|||||||
@@ -42,10 +42,6 @@ class BackendStatus(enum.IntEnum):
|
|||||||
READY = 0
|
READY = 0
|
||||||
|
|
||||||
|
|
||||||
class EnumerationError(RuntimeError):
|
|
||||||
"""A backend could not produce an authoritative live-resource snapshot."""
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class BottleSpec:
|
class BottleSpec:
|
||||||
"""CLI-supplied intent. Backend-agnostic — each backend's prepare
|
"""CLI-supplied intent. Backend-agnostic — each backend's prepare
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ from pathlib import Path
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from ...log import die, warn
|
from ...log import die, warn
|
||||||
from ..base import EnumerationError
|
|
||||||
|
|
||||||
|
|
||||||
# --- Lifecycle helpers (PRD 0018 chunk 3) ----------------------------------
|
# --- Lifecycle helpers (PRD 0018 chunk 3) ----------------------------------
|
||||||
@@ -53,20 +52,19 @@ def slug_from_compose_project(project: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def list_compose_projects(
|
def list_compose_projects(
|
||||||
*,
|
*, include_stopped: bool = True, warn_on_error: bool = True,
|
||||||
include_stopped: bool = True,
|
|
||||||
warn_on_error: bool = True,
|
|
||||||
raise_on_error: bool = False,
|
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
"""All compose project names starting with `bot-bottle-`.
|
"""All compose project names starting with `bot-bottle-`.
|
||||||
`include_stopped=True` (default) runs `docker compose ls --all`
|
`include_stopped=True` (default) runs `docker compose ls --all`
|
||||||
so exited projects appear too; pass False to get only projects
|
so exited projects appear too; pass False to get only projects
|
||||||
with at least one running container.
|
with at least one running container.
|
||||||
|
|
||||||
Best-effort callers get ``[]`` on Docker errors or malformed output.
|
Returns [] on docker daemon errors or malformed output rather
|
||||||
Enumeration callers pass ``raise_on_error=True`` so a failed query is not
|
than raising — callers should treat the empty list as "no
|
||||||
reported as an authoritative empty result.
|
projects discoverable", not "no projects exist". `warn_on_error`
|
||||||
"""
|
stays true for explicit operator commands like cleanup, but active
|
||||||
|
discovery paths set it false so dashboard refreshes don't spam
|
||||||
|
stderr while Docker Desktop is stopped."""
|
||||||
argv = ["docker", "compose", "ls", "--format", "json"]
|
argv = ["docker", "compose", "ls", "--format", "json"]
|
||||||
if include_stopped:
|
if include_stopped:
|
||||||
argv.insert(3, "--all")
|
argv.insert(3, "--all")
|
||||||
@@ -74,27 +72,19 @@ def list_compose_projects(
|
|||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
argv, capture_output=True, text=True, check=False,
|
argv, capture_output=True, text=True, check=False,
|
||||||
)
|
)
|
||||||
except FileNotFoundError as exc:
|
except FileNotFoundError:
|
||||||
if raise_on_error:
|
# docker binary not on PATH — same shape as a daemon-down
|
||||||
raise EnumerationError(
|
# error from the caller's POV: no projects discoverable.
|
||||||
"docker compose ls failed: docker not found"
|
|
||||||
) from exc
|
|
||||||
return []
|
return []
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
message = f"docker compose ls failed: {result.stderr.strip()}"
|
|
||||||
if raise_on_error:
|
|
||||||
raise EnumerationError(message)
|
|
||||||
if warn_on_error:
|
if warn_on_error:
|
||||||
warn(message)
|
warn(f"docker compose ls failed: {result.stderr.strip()}")
|
||||||
return []
|
return []
|
||||||
try:
|
try:
|
||||||
projects = json.loads(result.stdout or "[]")
|
projects = json.loads(result.stdout or "[]")
|
||||||
except json.JSONDecodeError as e:
|
except json.JSONDecodeError as e:
|
||||||
message = f"docker compose ls returned malformed JSON: {e}"
|
|
||||||
if raise_on_error:
|
|
||||||
raise EnumerationError(message) from e
|
|
||||||
if warn_on_error:
|
if warn_on_error:
|
||||||
warn(message)
|
warn(f"docker compose ls returned malformed JSON: {e}")
|
||||||
return []
|
return []
|
||||||
names: list[str] = []
|
names: list[str] = []
|
||||||
for p in projects:
|
for p in projects:
|
||||||
@@ -107,10 +97,7 @@ def list_compose_projects(
|
|||||||
|
|
||||||
|
|
||||||
def list_active_slugs(
|
def list_active_slugs(
|
||||||
*,
|
*, include_stopped: bool = False, warn_on_error: bool = True,
|
||||||
include_stopped: bool = False,
|
|
||||||
warn_on_error: bool = True,
|
|
||||||
raise_on_error: bool = False,
|
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
"""Slugs (project name minus prefix) of currently-running
|
"""Slugs (project name minus prefix) of currently-running
|
||||||
bottles. Used by the dashboard's operator-edit verbs to choose
|
bottles. Used by the dashboard's operator-edit verbs to choose
|
||||||
@@ -121,7 +108,6 @@ def list_active_slugs(
|
|||||||
for p in list_compose_projects(
|
for p in list_compose_projects(
|
||||||
include_stopped=include_stopped,
|
include_stopped=include_stopped,
|
||||||
warn_on_error=warn_on_error,
|
warn_on_error=warn_on_error,
|
||||||
raise_on_error=raise_on_error,
|
|
||||||
)
|
)
|
||||||
) if slug
|
) if slug
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -68,11 +68,6 @@ def _network_container_ips(network: str) -> list[str]:
|
|||||||
"docker", "network", "inspect", "--format",
|
"docker", "network", "inspect", "--format",
|
||||||
"{{range .Containers}}{{.IPv4Address}} {{end}}", network,
|
"{{range .Containers}}{{.IPv4Address}} {{end}}", network,
|
||||||
])
|
])
|
||||||
if proc.returncode != 0:
|
|
||||||
detail = proc.stderr.strip() or f"exit {proc.returncode}"
|
|
||||||
raise ConsolidatedLaunchError(
|
|
||||||
f"could not inspect addresses on gateway network {network}: {detail}"
|
|
||||||
)
|
|
||||||
ips: list[str] = []
|
ips: list[str] = []
|
||||||
for entry in proc.stdout.split():
|
for entry in proc.stdout.split():
|
||||||
ips.append(entry.split("/", 1)[0])
|
ips.append(entry.split("/", 1)[0])
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
"""Active-agent enumeration for the docker backend.
|
"""Active-agent enumeration for the docker backend.
|
||||||
|
|
||||||
Returns `ActiveAgent` records the CLI `active` command and the
|
Returns `ActiveAgent` records the CLI `active` command and the
|
||||||
dashboard agents pane consume. Docker query failures raise rather
|
dashboard agents pane consume. Empty when docker isn't reachable
|
||||||
than masquerading as an authoritative empty result.
|
— gated by `has_backend('docker')` at the cross-backend caller
|
||||||
|
so this module trusts that docker is available when called.
|
||||||
|
|
||||||
The parser (`_parse_services_by_project`) is exposed for direct
|
The parser (`_parse_services_by_project`) is exposed for direct
|
||||||
unit testing; the docker `docker ps` invocation is in
|
unit testing; the docker `docker ps` invocation is in
|
||||||
@@ -12,18 +13,17 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
from .. import ActiveAgent, EnumerationError
|
from .. import ActiveAgent
|
||||||
from ...bottle_state import read_metadata
|
from ...bottle_state import read_metadata
|
||||||
from .compose import compose_project_name, list_active_slugs
|
from .compose import compose_project_name, list_active_slugs
|
||||||
|
|
||||||
|
|
||||||
def enumerate_active() -> list[ActiveAgent]:
|
def enumerate_active() -> list[ActiveAgent]:
|
||||||
"""All currently-running docker-backed agents."""
|
"""All currently-running docker-backed agents. Caller is
|
||||||
slugs = list_active_slugs(
|
responsible for gating on `has_backend('docker')` if it
|
||||||
include_stopped=False,
|
matters; if docker is missing the `docker ps` call below
|
||||||
warn_on_error=False,
|
returns an empty list silently."""
|
||||||
raise_on_error=True,
|
slugs = list_active_slugs(include_stopped=False, warn_on_error=False)
|
||||||
)
|
|
||||||
if not slugs:
|
if not slugs:
|
||||||
return []
|
return []
|
||||||
services_by_project = _query_services_by_project()
|
services_by_project = _query_services_by_project()
|
||||||
@@ -74,8 +74,8 @@ def _query_services_by_project() -> dict[str, set[str]]:
|
|||||||
],
|
],
|
||||||
capture_output=True, text=True, check=False,
|
capture_output=True, text=True, check=False,
|
||||||
)
|
)
|
||||||
except FileNotFoundError as exc:
|
except FileNotFoundError:
|
||||||
raise EnumerationError("docker ps failed: docker not found") from exc
|
return {}
|
||||||
if r.returncode != 0:
|
if r.returncode != 0:
|
||||||
raise EnumerationError(f"docker ps failed: {r.stderr.strip()}")
|
return {}
|
||||||
return _parse_services_by_project(r.stdout or "")
|
return _parse_services_by_project(r.stdout or "")
|
||||||
|
|||||||
@@ -140,43 +140,12 @@ class DockerGateway(Gateway):
|
|||||||
marker = inspected.stdout.strip()
|
marker = inspected.stdout.strip()
|
||||||
if marker in {"", self._subnet}:
|
if marker in {"", self._subnet}:
|
||||||
return
|
return
|
||||||
# Inspectable but mislabelled: the stale auto-IPAM network created
|
if inspected.returncode == 0:
|
||||||
# by older releases. Replace it below.
|
# Migrate the stale auto-IPAM network created by older releases.
|
||||||
stale = True
|
# Removing the fixed gateway is safe here: this launch recreates it.
|
||||||
else:
|
|
||||||
# inspect failed. Classify by stderr — do NOT assume "not absent"
|
|
||||||
# implies "poisoned": a transient daemon/API error, permission
|
|
||||||
# failure, timeout, or bad context also fails here, and destroying
|
|
||||||
# the shared gateway on that guess would tear the network out from
|
|
||||||
# under every live bottle.
|
|
||||||
err = inspected.stderr.lower()
|
|
||||||
if "no such network" in err or "not found" in err:
|
|
||||||
# Absent: nothing to replace — create it below.
|
|
||||||
stale = False
|
|
||||||
elif "parseaddr" in err:
|
|
||||||
# Present but poisoned. A daemon that default-enables IPv6
|
|
||||||
# attaches an fdd0::/64 subnet whose `::1/64` gateway trips
|
|
||||||
# docker's own netip.ParseAddr in `network inspect`/`ls`, so the
|
|
||||||
# command exits non-zero with that signature. A fixed release
|
|
||||||
# never *creates* such a network, but one can survive on a
|
|
||||||
# shared host from an older or concurrent launch — and
|
|
||||||
# `--ipv6=false` alone can't heal it, since the create below only
|
|
||||||
# no-ops on "already exists". Force-replace it so later reads
|
|
||||||
# (e.g. `_network_cidr` pinning a source IP) stop failing.
|
|
||||||
stale = True
|
|
||||||
else:
|
|
||||||
# Unrecognized failure: no evidence the network is malformed.
|
|
||||||
# Surface it rather than mutate shared state on a guess.
|
|
||||||
raise GatewayError(
|
|
||||||
f"gateway network {self.network} could not be inspected: "
|
|
||||||
f"{inspected.stderr.strip()}"
|
|
||||||
)
|
|
||||||
if stale:
|
|
||||||
# Migrate the stale/poisoned network. Removing the fixed gateway is
|
|
||||||
# safe here: this launch recreates it.
|
|
||||||
run_docker(["docker", "rm", "--force", self.name])
|
run_docker(["docker", "rm", "--force", self.name])
|
||||||
removed = run_docker(["docker", "network", "rm", self.network])
|
removed = run_docker(["docker", "network", "rm", self.network])
|
||||||
if removed.returncode != 0 and "no such network" not in removed.stderr.lower():
|
if removed.returncode != 0:
|
||||||
raise GatewayError(
|
raise GatewayError(
|
||||||
f"gateway network {self.network} needs explicit subnet "
|
f"gateway network {self.network} needs explicit subnet "
|
||||||
f"{self._subnet} but could not be replaced: "
|
f"{self._subnet} but could not be replaced: "
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ import subprocess
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ...log import info
|
from ...log import info
|
||||||
from .. import EnumerationError
|
|
||||||
from . import util
|
from . import util
|
||||||
from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan
|
from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan
|
||||||
|
|
||||||
@@ -63,23 +62,12 @@ def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
|
|||||||
* ``orphan_pids`` — firecracker pids whose run dir no longer exists
|
* ``orphan_pids`` — firecracker pids whose run dir no longer exists
|
||||||
(a lingering VMM to kill).
|
(a lingering VMM to kill).
|
||||||
"""
|
"""
|
||||||
try:
|
result = subprocess.run(
|
||||||
result = subprocess.run(
|
["pgrep", "-a", "firecracker"],
|
||||||
["pgrep", "-a", "firecracker"],
|
capture_output=True, text=True, check=False,
|
||||||
capture_output=True, text=True, check=False,
|
)
|
||||||
)
|
|
||||||
except OSError as exc:
|
|
||||||
raise EnumerationError(
|
|
||||||
f"could not enumerate Firecracker processes: {exc}"
|
|
||||||
) from exc
|
|
||||||
if result.returncode == 1:
|
|
||||||
# pgrep's documented "no processes matched" result.
|
|
||||||
return set(), []
|
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
detail = (result.stderr or "").strip() or f"exit {result.returncode}"
|
return set(), []
|
||||||
raise EnumerationError(
|
|
||||||
f"could not enumerate Firecracker processes: {detail}"
|
|
||||||
)
|
|
||||||
live: set[str] = set()
|
live: set[str] = set()
|
||||||
orphan_pids: list[int] = []
|
orphan_pids: list[int] = []
|
||||||
for line in result.stdout.splitlines():
|
for line in result.stdout.splitlines():
|
||||||
|
|||||||
@@ -1,32 +1,14 @@
|
|||||||
"""Active-agent enumeration for the Firecracker backend.
|
"""Active-agent enumeration for the Firecracker backend.
|
||||||
|
|
||||||
Running bottles are the Firecracker processes whose ``--config-file`` points
|
The backend is disabled during the companion-container removal (#385) — it can't
|
||||||
at an existing per-bottle run directory. The same authoritative process scan
|
launch bottles, so there are none to enumerate. Real enumeration returns
|
||||||
protects cleanup from deleting live VMs; operational scan failures propagate
|
with the backend's consolidated relaunch (#354).
|
||||||
as ``EnumerationError`` instead of masquerading as an empty host.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from ...bottle_state import read_metadata
|
|
||||||
from .. import ActiveAgent
|
from .. import ActiveAgent
|
||||||
from .cleanup import live_run_dirs
|
|
||||||
|
|
||||||
|
|
||||||
def enumerate_active() -> list[ActiveAgent]:
|
def enumerate_active() -> list[ActiveAgent]:
|
||||||
out: list[ActiveAgent] = []
|
return []
|
||||||
for run_dir in live_run_dirs():
|
|
||||||
slug = run_dir.name
|
|
||||||
metadata = read_metadata(slug)
|
|
||||||
out.append(ActiveAgent(
|
|
||||||
backend_name="firecracker",
|
|
||||||
slug=slug,
|
|
||||||
agent_name=metadata.agent_name if metadata else "?",
|
|
||||||
started_at=metadata.started_at if metadata else "",
|
|
||||||
# Firecracker uses the shared gateway, so there are no
|
|
||||||
# per-bottle gateway service containers to report.
|
|
||||||
services=(),
|
|
||||||
label=metadata.label if metadata else "",
|
|
||||||
color=metadata.color if metadata else "",
|
|
||||||
))
|
|
||||||
return out
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
|||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
from ...bottle_state import read_metadata
|
from ...bottle_state import read_metadata
|
||||||
from .. import ActiveAgent, EnumerationError
|
from .. import ActiveAgent
|
||||||
from .infra import INFRA_NAME, ORCHESTRATOR_NAME
|
from .infra import INFRA_NAME, ORCHESTRATOR_NAME
|
||||||
|
|
||||||
# The name every agent container carries: `bot-bottle-<slug>`. Exported
|
# The name every agent container carries: `bot-bottle-<slug>`. Exported
|
||||||
@@ -20,18 +20,17 @@ CONTAINER_NAME_PREFIX = "bot-bottle-"
|
|||||||
_INFRA_NAMES = frozenset({INFRA_NAME, ORCHESTRATOR_NAME})
|
_INFRA_NAMES = frozenset({INFRA_NAME, ORCHESTRATOR_NAME})
|
||||||
|
|
||||||
|
|
||||||
|
class EnumerationError(RuntimeError):
|
||||||
|
"""container list failed; the resulting live set is not authoritative."""
|
||||||
|
|
||||||
|
|
||||||
def enumerate_active() -> list[ActiveAgent]:
|
def enumerate_active() -> list[ActiveAgent]:
|
||||||
try:
|
result = subprocess.run(
|
||||||
result = subprocess.run(
|
["container", "list", "--quiet"],
|
||||||
["container", "list", "--quiet"],
|
capture_output=True,
|
||||||
capture_output=True,
|
text=True,
|
||||||
text=True,
|
check=False,
|
||||||
check=False,
|
)
|
||||||
)
|
|
||||||
except FileNotFoundError as exc:
|
|
||||||
raise EnumerationError(
|
|
||||||
"container list failed: container CLI not found"
|
|
||||||
) from exc
|
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
raise EnumerationError(
|
raise EnumerationError(
|
||||||
f"container list failed: "
|
f"container list failed: "
|
||||||
|
|||||||
@@ -389,9 +389,19 @@ class EgressAddon:
|
|||||||
self._passthrough_conns.discard(conn_id)
|
self._passthrough_conns.discard(conn_id)
|
||||||
|
|
||||||
async def request(self, flow: http.HTTPFlow) -> None:
|
async def request(self, flow: http.HTTPFlow) -> None:
|
||||||
config, slug, env = self._request_context(flow)
|
|
||||||
request_path, _, query = flow.request.path.partition("?")
|
request_path, _, query = flow.request.path.partition("?")
|
||||||
|
|
||||||
|
# Reuse the context stashed by http_connect for HTTPS flows (one
|
||||||
|
# orchestrator round-trip per connection). Plain-HTTP flows have no
|
||||||
|
# prior CONNECT stash, so resolve now and stash for response/websocket.
|
||||||
|
meta = getattr(flow, "metadata", None)
|
||||||
|
if isinstance(meta, dict) and _FLOW_CTX_KEY in meta:
|
||||||
|
config, slug, env = meta[_FLOW_CTX_KEY]
|
||||||
|
self._request_token(flow) # strip identity headers; token already resolved
|
||||||
|
else:
|
||||||
|
config, slug, env = self._resolve_flow(flow)
|
||||||
|
self._stash_flow_ctx(flow, config, slug, env)
|
||||||
|
|
||||||
# Introspection ("_egress.local/allowlist") reports the calling bottle's
|
# Introspection ("_egress.local/allowlist") reports the calling bottle's
|
||||||
# own resolved routes — served after resolution so it reflects this
|
# own resolved routes — served after resolution so it reflects this
|
||||||
# bottle's policy, not a stale global.
|
# bottle's policy, not a stale global.
|
||||||
@@ -412,29 +422,6 @@ class EgressAddon:
|
|||||||
# the path/query the git checks below rely on.
|
# the path/query the git checks below rely on.
|
||||||
request_path, _, query = flow.request.path.partition("?")
|
request_path, _, query = flow.request.path.partition("?")
|
||||||
|
|
||||||
if not self._allow_git_request(flow, config, request_path, query):
|
|
||||||
return
|
|
||||||
|
|
||||||
self._apply_route_policy(flow, config, route, request_path, env)
|
|
||||||
|
|
||||||
def _request_context(
|
|
||||||
self, flow: http.HTTPFlow,
|
|
||||||
) -> tuple[Config, str, "typing.Mapping[str, str]"]:
|
|
||||||
"""Resolve one bottle context, reusing the HTTPS CONNECT snapshot."""
|
|
||||||
meta = getattr(flow, "metadata", None)
|
|
||||||
if isinstance(meta, dict) and _FLOW_CTX_KEY in meta:
|
|
||||||
config, slug, env = meta[_FLOW_CTX_KEY]
|
|
||||||
self._request_token(flow)
|
|
||||||
return config, slug, env
|
|
||||||
config, slug, env = self._resolve_flow(flow)
|
|
||||||
self._stash_flow_ctx(flow, config, slug, env)
|
|
||||||
return config, slug, env
|
|
||||||
|
|
||||||
def _allow_git_request(
|
|
||||||
self, flow: http.HTTPFlow, config: Config,
|
|
||||||
request_path: str, query: str,
|
|
||||||
) -> bool:
|
|
||||||
"""Apply the HTTPS Git push/fetch boundary before general routing."""
|
|
||||||
if is_git_push_request(request_path, query):
|
if is_git_push_request(request_path, query):
|
||||||
self._block(
|
self._block(
|
||||||
flow,
|
flow,
|
||||||
@@ -443,20 +430,20 @@ class EgressAddon:
|
|||||||
"git-gate's pre-receive hook).",
|
"git-gate's pre-receive hook).",
|
||||||
ctx=self._req_ctx(flow),
|
ctx=self._req_ctx(flow),
|
||||||
)
|
)
|
||||||
return False
|
return
|
||||||
if not is_git_fetch_request(request_path, query):
|
|
||||||
return True
|
if is_git_fetch_request(request_path, query):
|
||||||
git_decision = decide_git_fetch(config.routes, flow.request.pretty_host)
|
git_decision = decide_git_fetch(
|
||||||
if git_decision.action != "block":
|
config.routes, flow.request.pretty_host,
|
||||||
return True
|
)
|
||||||
self._block(flow, git_decision.reason, ctx=self._req_ctx(flow))
|
if git_decision.action == "block":
|
||||||
return False
|
self._block(
|
||||||
|
flow,
|
||||||
|
git_decision.reason,
|
||||||
|
ctx=self._req_ctx(flow),
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
def _apply_route_policy(
|
|
||||||
self, flow: http.HTTPFlow, config: Config, route: Route | None,
|
|
||||||
request_path: str, env: "typing.Mapping[str, str]",
|
|
||||||
) -> None:
|
|
||||||
"""Strip agent auth, evaluate the route, then inject gateway auth."""
|
|
||||||
# Strip agent-set Authorization after DLP scan so smuggled tokens
|
# Strip agent-set Authorization after DLP scan so smuggled tokens
|
||||||
# are caught above; the route may inject gateway-owned auth below.
|
# are caught above; the route may inject gateway-owned auth below.
|
||||||
# Routes with preserve_auth=True pass the header through as-is so the
|
# Routes with preserve_auth=True pass the header through as-is so the
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
"""Run the orchestrator control plane as a plain process (PRD 0070 dev-harness).
|
"""Run the orchestrator control plane as a plain process (PRD 0070 dev-harness).
|
||||||
|
|
||||||
BOT_BOTTLE_ORCHESTRATOR_TOKEN=<signing-key> \
|
python -m bot_bottle.orchestrator [--host H] [--port P] [--db PATH]
|
||||||
python -m bot_bottle.orchestrator [--host H] [--port P] [--db PATH]
|
|
||||||
|
|
||||||
The PRD sequences the orchestrator as a plain-process dev-harness first, so
|
The PRD sequences the orchestrator as a plain-process dev-harness first, so
|
||||||
the consolidation core (registry + attribution + HTTP control plane + live
|
the consolidation core (registry + attribution + HTTP control plane + live
|
||||||
@@ -17,13 +16,15 @@ import secrets
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from .. import log
|
from .. import log
|
||||||
from ..trust_domain import CONTROL_PLANE
|
|
||||||
from .broker import LaunchBroker, StubBroker
|
|
||||||
from .docker_broker import DockerBroker
|
|
||||||
from .server import make_server
|
|
||||||
from .service import OrchestratorCore
|
|
||||||
from .store.store_manager import StoreManager
|
from .store.store_manager import StoreManager
|
||||||
|
from ..paths import LAUNCH_BROKER_KEY_ENV
|
||||||
|
from .broker import StubBroker, SubmitBroker
|
||||||
|
from .broker_client import BrokerClient
|
||||||
|
from .host_server import DEFAULT_PORT, broker_secret
|
||||||
|
from .server import make_server
|
||||||
|
from .docker_broker import DockerBroker
|
||||||
from .store.registry_store import RegistryStore, default_db_path
|
from .store.registry_store import RegistryStore, default_db_path
|
||||||
|
from .service import OrchestratorCore
|
||||||
|
|
||||||
|
|
||||||
def main(argv: list[str] | None = None) -> int:
|
def main(argv: list[str] | None = None) -> int:
|
||||||
@@ -36,15 +37,15 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
help=f"registry DB path (default: {default_db_path()})",
|
help=f"registry DB path (default: {default_db_path()})",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--broker", choices=("stub", "docker"), default="stub",
|
"--broker", choices=("stub", "docker", "http"), default="stub",
|
||||||
help="launch broker: 'stub' records requests; 'docker' runs containers",
|
help="launch broker: 'stub' records requests; 'docker' runs containers "
|
||||||
|
"in-process; 'http' relays signed requests to a host control server",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--host-controller-url", default=f"http://127.0.0.1:{DEFAULT_PORT}",
|
||||||
|
help="host control server URL (used only with --broker http)",
|
||||||
)
|
)
|
||||||
args = parser.parse_args(argv)
|
args = parser.parse_args(argv)
|
||||||
if not CONTROL_PLANE.key_from_env():
|
|
||||||
log.die(
|
|
||||||
f"{CONTROL_PLANE.key_env} is required; refusing to start the "
|
|
||||||
"orchestrator without caller authentication"
|
|
||||||
)
|
|
||||||
|
|
||||||
registry = RegistryStore(args.db)
|
registry = RegistryStore(args.db)
|
||||||
registry.migrate()
|
registry.migrate()
|
||||||
@@ -54,11 +55,27 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
# operator reaches it over HTTP (never a second, disconnected DB).
|
# operator reaches it over HTTP (never a second, disconnected DB).
|
||||||
StoreManager(registry.db_path).migrate()
|
StoreManager(registry.db_path).migrate()
|
||||||
|
|
||||||
# An ephemeral signing secret ties the orchestrator (signer) to its
|
# A signing secret ties the orchestrator (signer) to its broker (verifier).
|
||||||
# broker (verifier). 'stub' records launches instead of starting
|
# 'stub' records launches instead of starting anything; 'docker' runs real
|
||||||
# anything; 'docker' runs real containers (firecracker drops in later).
|
# containers in-process; 'http' relays signed requests to a separate host
|
||||||
secret = secrets.token_bytes(32)
|
# control server, which verifies and launches. For 'stub'/'docker' the secret
|
||||||
broker: LaunchBroker = DockerBroker(secret) if args.broker == "docker" else StubBroker(secret)
|
# is ephemeral (signer and verifier share this process). For 'http' it must be
|
||||||
|
# the SAME key the host controller holds — and this process is the *guest*
|
||||||
|
# (signer), so it must be given that key by injection, NOT mint its own
|
||||||
|
# process-local one (which would diverge from the host's and 401 every launch).
|
||||||
|
broker: SubmitBroker
|
||||||
|
if args.broker == "http":
|
||||||
|
secret = broker_secret() # env-injected only; no host-file fallback here
|
||||||
|
if secret is None:
|
||||||
|
parser.error(
|
||||||
|
f"--broker http requires the launch-broker key injected as "
|
||||||
|
f"${LAUNCH_BROKER_KEY_ENV} (the host controller owns/mints it); the "
|
||||||
|
"orchestrator must not mint its own or it would diverge from the host's"
|
||||||
|
)
|
||||||
|
broker = BrokerClient(args.host_controller_url)
|
||||||
|
else:
|
||||||
|
secret = secrets.token_bytes(32)
|
||||||
|
broker = DockerBroker(secret) if args.broker == "docker" else StubBroker(secret)
|
||||||
orchestrator = OrchestratorCore(registry, broker, secret)
|
orchestrator = OrchestratorCore(registry, broker, secret)
|
||||||
|
|
||||||
server = make_server(orchestrator, host=args.host, port=args.port)
|
server = make_server(orchestrator, host=args.host, port=args.port)
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ import json
|
|||||||
import secrets
|
import secrets
|
||||||
import time
|
import time
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
_JWT_HEADER = {"alg": "HS256", "typ": "JWT"}
|
_JWT_HEADER = {"alg": "HS256", "typ": "JWT"}
|
||||||
_ALLOWED_OPS = ("launch", "teardown")
|
_ALLOWED_OPS = ("launch", "teardown")
|
||||||
@@ -37,7 +38,21 @@ _ALLOWED_OPS = ("launch", "teardown")
|
|||||||
class BrokerAuthError(Exception):
|
class BrokerAuthError(Exception):
|
||||||
"""A broker request failed provenance or schema verification —
|
"""A broker request failed provenance or schema verification —
|
||||||
bad/absent signature, malformed token, or a payload that doesn't match
|
bad/absent signature, malformed token, or a payload that doesn't match
|
||||||
the fixed launch-request shape. Fail-closed: the broker must not act."""
|
the fixed launch-request shape. Fail-closed: the broker must not act.
|
||||||
|
|
||||||
|
A **definite** negative: nothing was launched, so a caller may safely roll
|
||||||
|
back as if the op never happened."""
|
||||||
|
|
||||||
|
|
||||||
|
class BrokerUnavailableError(Exception):
|
||||||
|
"""A brokered request could not be carried to a verdict: the broker (or the
|
||||||
|
wire to it) was unreachable, timed out, or dropped the response.
|
||||||
|
|
||||||
|
Crucially **ambiguous** — unlike `BrokerAuthError`, the op MAY already have
|
||||||
|
taken effect on the backend before the response was lost, so a caller must
|
||||||
|
NOT assume it did nothing (e.g. must not roll a registry row back as if no
|
||||||
|
launch happened, which would orphan a running container). Only the in-process
|
||||||
|
brokers never raise this; the out-of-process `BrokerClient` does."""
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -123,6 +138,16 @@ def verify_request(token: str, secret: bytes) -> LaunchRequest:
|
|||||||
|
|
||||||
# --- the broker itself ------------------------------------------------------
|
# --- the broker itself ------------------------------------------------------
|
||||||
|
|
||||||
|
class SubmitBroker(Protocol):
|
||||||
|
"""The single method `OrchestratorCore` depends on: verify a signed token and
|
||||||
|
perform its op, returning the verified request. Both the in-process
|
||||||
|
`LaunchBroker` and the out-of-process `BrokerClient` (which relays the token
|
||||||
|
to the host control server) satisfy it structurally, so the core is unchanged
|
||||||
|
whether the backend is local or a real host service."""
|
||||||
|
|
||||||
|
def submit(self, token: str) -> LaunchRequest: ...
|
||||||
|
|
||||||
|
|
||||||
class LaunchBroker(abc.ABC):
|
class LaunchBroker(abc.ABC):
|
||||||
"""Verifies a signed request came from the orchestrator, then performs
|
"""Verifies a signed request came from the orchestrator, then performs
|
||||||
the backend-native launch/teardown. Subclasses implement `_launch` /
|
the backend-native launch/teardown. Subclasses implement `_launch` /
|
||||||
@@ -168,7 +193,9 @@ class StubBroker(LaunchBroker):
|
|||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"BrokerAuthError",
|
"BrokerAuthError",
|
||||||
|
"BrokerUnavailableError",
|
||||||
"LaunchRequest",
|
"LaunchRequest",
|
||||||
|
"SubmitBroker",
|
||||||
"LaunchBroker",
|
"LaunchBroker",
|
||||||
"StubBroker",
|
"StubBroker",
|
||||||
"sign_request",
|
"sign_request",
|
||||||
|
|||||||
@@ -0,0 +1,126 @@
|
|||||||
|
"""Orchestrator-side broker transport (issue #468, chunk 1).
|
||||||
|
|
||||||
|
The signer's half of the launch-broker transport gap. `BrokerClient` satisfies
|
||||||
|
the exact `submit(token)` contract `OrchestratorCore` already depends on (see
|
||||||
|
`broker.SubmitBroker`), but instead of verifying and launching in-process it POSTs
|
||||||
|
the signed token to the host control server over HTTP (stdlib `urllib`, like
|
||||||
|
`orchestrator/client.py`). Because it is drop-in for that interface, wiring a real
|
||||||
|
out-of-process backend does not change the core: it still signs a request and
|
||||||
|
calls `submit()`; only the wire is new.
|
||||||
|
|
||||||
|
A provenance/schema rejection from the host controller (HTTP 401) is re-raised as
|
||||||
|
the same `BrokerAuthError` the in-process broker raises, so the launch path's
|
||||||
|
rollback-on-failure (`OrchestratorCore.launch_bottle`) behaves identically whether
|
||||||
|
the broker is local or remote.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
from .broker import BrokerAuthError, BrokerUnavailableError, LaunchRequest
|
||||||
|
|
||||||
|
DEFAULT_TIMEOUT_SECONDS = 5.0
|
||||||
|
|
||||||
|
|
||||||
|
class BrokerClientError(RuntimeError):
|
||||||
|
"""The host control server *responded*, but with an unexpected status other
|
||||||
|
than the fail-closed 401 (which surfaces as `BrokerAuthError`) — e.g. a 502
|
||||||
|
backend failure or a malformed body. A definite negative: the host processed
|
||||||
|
the request and it did not launch. (A *no-response* failure — unreachable /
|
||||||
|
timeout / dropped — is the ambiguous `BrokerUnavailableError` instead.)"""
|
||||||
|
|
||||||
|
|
||||||
|
class BrokerClient:
|
||||||
|
"""Drop-in `submit(token)` that relays a signed request to the host control
|
||||||
|
server. Holds no secret — provenance rides entirely in the signed token, so a
|
||||||
|
caller that can reach this client still cannot forge a launch."""
|
||||||
|
|
||||||
|
def __init__(self, base_url: str, *, timeout: float = DEFAULT_TIMEOUT_SECONDS) -> None:
|
||||||
|
self._base = base_url.rstrip("/")
|
||||||
|
self._timeout = timeout
|
||||||
|
|
||||||
|
def submit(self, token: str) -> LaunchRequest:
|
||||||
|
"""POST the signed token to the host controller and return the request it
|
||||||
|
verified and acted on.
|
||||||
|
|
||||||
|
Raises `BrokerAuthError` on a fail-closed 401 (bad provenance/schema —
|
||||||
|
the same exception the in-process broker raises); `BrokerClientError` if
|
||||||
|
the host *responds* with any other non-success status or a malformed
|
||||||
|
body (a definite negative); or `BrokerUnavailableError` if no response is
|
||||||
|
obtained (unreachable / timeout / dropped) — the **ambiguous** case, where
|
||||||
|
the host may already have acted, so the caller must not roll back."""
|
||||||
|
data = json.dumps({"token": token}).encode()
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{self._base}/broker", data=data, method="POST",
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=self._timeout) as resp:
|
||||||
|
return _request_from(_json_object(resp.read()))
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
detail = _error_detail(e)
|
||||||
|
if e.code == 401:
|
||||||
|
raise BrokerAuthError(
|
||||||
|
detail or "host controller rejected the request"
|
||||||
|
) from e
|
||||||
|
raise BrokerClientError(
|
||||||
|
f"POST /broker: HTTP {e.code} {detail}".rstrip()
|
||||||
|
) from e
|
||||||
|
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
||||||
|
# No usable response — unreachable, timed out, or the connection
|
||||||
|
# dropped mid-exchange. Ambiguous: the request may already have
|
||||||
|
# launched the bottle, so this is NOT a definite failure.
|
||||||
|
raise BrokerUnavailableError(f"POST /broker: {e}") from e
|
||||||
|
|
||||||
|
|
||||||
|
def _json_object(raw: bytes) -> dict[str, object]:
|
||||||
|
"""Parse a JSON object, tolerating an empty or malformed body (→ {}), like
|
||||||
|
the orchestrator client — a bad body becomes a clean 'missing field' error
|
||||||
|
downstream rather than an opaque JSON crash."""
|
||||||
|
if not raw:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
obj = json.loads(raw)
|
||||||
|
except ValueError:
|
||||||
|
return {}
|
||||||
|
return obj if isinstance(obj, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def _error_detail(e: urllib.error.HTTPError) -> str:
|
||||||
|
"""The `error` string from a structured error response, best-effort — an
|
||||||
|
error body may be absent or unreadable, in which case there is no detail."""
|
||||||
|
try:
|
||||||
|
detail = _json_object(e.read()).get("error", "")
|
||||||
|
except Exception: # noqa: BLE001 — the error body is advisory only
|
||||||
|
return ""
|
||||||
|
return detail if isinstance(detail, str) else ""
|
||||||
|
|
||||||
|
|
||||||
|
def _request_from(payload: dict[str, object]) -> LaunchRequest:
|
||||||
|
"""Reconstruct the verified `LaunchRequest` the controller echoed, so the
|
||||||
|
returned value matches the in-process broker's (which returns the request it
|
||||||
|
acted on). A missing op/bottle_id means a malformed response."""
|
||||||
|
op = payload.get("op")
|
||||||
|
bottle_id = payload.get("bottle_id")
|
||||||
|
if not isinstance(op, str) or not isinstance(bottle_id, str) or not bottle_id:
|
||||||
|
raise BrokerClientError("host controller response missing op/bottle_id")
|
||||||
|
source_ip = payload.get("source_ip")
|
||||||
|
image_ref = payload.get("image_ref")
|
||||||
|
slot = payload.get("slot")
|
||||||
|
return LaunchRequest(
|
||||||
|
op=op,
|
||||||
|
bottle_id=bottle_id,
|
||||||
|
source_ip=source_ip if isinstance(source_ip, str) else "",
|
||||||
|
image_ref=image_ref if isinstance(image_ref, str) else "",
|
||||||
|
slot=slot if isinstance(slot, int) and not isinstance(slot, bool) else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"BrokerClient",
|
||||||
|
"BrokerClientError",
|
||||||
|
"DEFAULT_TIMEOUT_SECONDS",
|
||||||
|
]
|
||||||
@@ -0,0 +1,287 @@
|
|||||||
|
"""Host control server (issue #468) — the launch broker as a real host service.
|
||||||
|
|
||||||
|
Chunk 1 of the host-control-server stack closes the **transport** gap the PRD
|
||||||
|
opens with: today `LaunchBroker.submit(token)` is an in-process method call from
|
||||||
|
`OrchestratorCore`, and a real host service needs it reachable over the wire.
|
||||||
|
This module is that service — the single privileged host component — reached over
|
||||||
|
**HTTP** (the universal transport 0070 chose), mirroring the orchestrator control
|
||||||
|
plane's shape (`orchestrator/server.py`): a pure `dispatch()` for socket-free
|
||||||
|
testing, wrapped by a thin stdlib `http.server` adapter.
|
||||||
|
|
||||||
|
GET /health -> 200 {"status": "ok"}
|
||||||
|
POST /broker -> 200 {"op", "bottle_id", "source_ip", "image_ref", "slot"}
|
||||||
|
400 (bad body) | 401 (bad provenance/schema) | 502 (backend)
|
||||||
|
body: {"token": "<signed launch/teardown JWT>"}
|
||||||
|
|
||||||
|
Only the **signed token** crosses the wire; the server holds the shared HS256
|
||||||
|
secret and a real `LaunchBroker` (e.g. `DockerBroker`) and runs the existing
|
||||||
|
`verify_request` + `_launch`/`_teardown` path behind the endpoint, so nothing
|
||||||
|
free-form ever reaches it. Provenance/schema failures are fail-closed 401s that
|
||||||
|
never touch the backend (`LaunchBroker.submit` verifies before acting), and a
|
||||||
|
backend launch failure is a 502 the caller must surface — neither takes the
|
||||||
|
controller down.
|
||||||
|
|
||||||
|
The signed launch token *is* the endpoint's authentication (its provenance is the
|
||||||
|
whole point of the JWS), so `/broker` needs no separate caller credential; the
|
||||||
|
host controller's own lifecycle endpoints, which do, arrive with the `host`-role
|
||||||
|
tokens of the separate `HOST_CONTROLLER` trust domain in a later chunk.
|
||||||
|
|
||||||
|
The shared signing secret is the durable **launch-broker `TrustDomain` key**
|
||||||
|
(#468/#476): a host-canonical key file minted 0600 on first use, provisioned to
|
||||||
|
the orchestrator (signer) and this server (verifier). A backend launcher injects
|
||||||
|
it via `$BOT_BOTTLE_LAUNCH_BROKER_KEY`; a host-side dev-harness process reads the
|
||||||
|
key file directly. Durability is the point — a restarted orchestrator re-verifies
|
||||||
|
against the same key, so re-adoption works.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import http.server
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import socketserver
|
||||||
|
import sys
|
||||||
|
import typing
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
from .. import log
|
||||||
|
from ..paths import LAUNCH_BROKER_KEY_ENV
|
||||||
|
from ..trust_domain import LAUNCH_BROKER
|
||||||
|
from .broker import BrokerAuthError, LaunchBroker
|
||||||
|
from .docker_broker import DockerBroker
|
||||||
|
|
||||||
|
# JSON body payload type (parsed request / rendered response).
|
||||||
|
Json = dict[str, object]
|
||||||
|
|
||||||
|
# Default host-controller port. Distinct from the orchestrator control plane
|
||||||
|
# (8099) — a separate privileged component listening on its own socket.
|
||||||
|
DEFAULT_PORT = 8091
|
||||||
|
|
||||||
|
# Cap on the request body. A signed broker request is tiny, so rejecting anything
|
||||||
|
# larger *before reading it* keeps a caller that can merely reach the socket (no
|
||||||
|
# signed token needed) from exhausting memory or a handler thread with a huge
|
||||||
|
# Content-Length — the signed token, not mere reachability, is the authority.
|
||||||
|
MAX_BODY_BYTES = 64 * 1024
|
||||||
|
|
||||||
|
# Per-request socket timeout, bounding how long a stalled / slow-loris caller can
|
||||||
|
# hold a handler thread on this privileged listener.
|
||||||
|
REQUEST_TIMEOUT_SECONDS = 15
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_json_object(body: bytes) -> Json:
|
||||||
|
"""Parse a JSON object body. Raises ValueError for non-objects / bad JSON."""
|
||||||
|
if not body:
|
||||||
|
return {}
|
||||||
|
obj = json.loads(body) # raises json.JSONDecodeError (a ValueError)
|
||||||
|
if not isinstance(obj, dict):
|
||||||
|
raise ValueError("request body must be a JSON object")
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|
||||||
|
def broker_secret(
|
||||||
|
environ: typing.Mapping[str, str] | None = None, *, allow_host_file: bool = False,
|
||||||
|
) -> bytes | None:
|
||||||
|
"""The shared launch-broker HS256 secret, as this process should use it.
|
||||||
|
|
||||||
|
Always prefers the key injected into this process's env
|
||||||
|
(`$BOT_BOTTLE_LAUNCH_BROKER_KEY`). `allow_host_file` decides the fallback when
|
||||||
|
it is absent, and the distinction is a security boundary:
|
||||||
|
|
||||||
|
- **Host-side** processes — the host controller and the host dev-harness — pass
|
||||||
|
``allow_host_file=True`` to read (minting on first use) the durable host key
|
||||||
|
file (``bot_bottle_root()/launch-broker-key``) they legitimately own.
|
||||||
|
- The **guest orchestrator** (``--broker http``) keeps the default ``False``.
|
||||||
|
It runs inside a container/VM whose ``bot_bottle_root()`` is process-local,
|
||||||
|
so minting a file there would silently create a key UNRELATED to the host
|
||||||
|
controller's — startup would succeed but every launch would be rejected 401.
|
||||||
|
It must instead be *given* the key by its launcher, and fail closed (None)
|
||||||
|
if it wasn't, rather than diverge.
|
||||||
|
|
||||||
|
None when no key is available (a guest with no injection, or an unwritable
|
||||||
|
host root)."""
|
||||||
|
key = LAUNCH_BROKER.key_from_env(environ)
|
||||||
|
if not key and allow_host_file:
|
||||||
|
try:
|
||||||
|
key = LAUNCH_BROKER.signing_key() # host-canonical, minted on first use
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
return key.encode("utf-8") if key else None
|
||||||
|
|
||||||
|
|
||||||
|
def dispatch( # pylint: disable=too-many-return-statements
|
||||||
|
broker: LaunchBroker, method: str, path: str, body: bytes,
|
||||||
|
) -> tuple[int, Json]:
|
||||||
|
"""Route one host-control request to a (status, payload) pair. Pure — the
|
||||||
|
only side effect is the broker's own backend launch — so routing is testable
|
||||||
|
without a socket.
|
||||||
|
|
||||||
|
Total by design: a provenance/schema failure becomes 401 and a backend launch
|
||||||
|
failure becomes 502 rather than raising, so one bad request can neither act
|
||||||
|
on the backend nor take the controller down for the next caller."""
|
||||||
|
route = urlsplit(path).path.rstrip("/") or "/"
|
||||||
|
|
||||||
|
if method == "GET" and route == "/health":
|
||||||
|
return 200, {"status": "ok"}
|
||||||
|
|
||||||
|
if method == "POST" and route == "/broker":
|
||||||
|
try:
|
||||||
|
data = _parse_json_object(body)
|
||||||
|
except ValueError as e:
|
||||||
|
return 400, {"error": f"invalid JSON: {e}"}
|
||||||
|
token = data.get("token")
|
||||||
|
if not isinstance(token, str) or not token:
|
||||||
|
return 400, {"error": "token (string) is required"}
|
||||||
|
try:
|
||||||
|
req = broker.submit(token)
|
||||||
|
except BrokerAuthError as e:
|
||||||
|
# Fail-closed: bad signature, malformed token, or off-schema payload.
|
||||||
|
# `submit` verifies before acting, so nothing was launched.
|
||||||
|
return 401, {"error": f"broker auth failed: {e}"}
|
||||||
|
except Exception as e: # noqa: BLE001 — a backend launch failure (docker
|
||||||
|
# down, image gone) is operational, not a control-plane bug; the
|
||||||
|
# caller must see it as a distinct 502, and the server must stay up.
|
||||||
|
return 502, {"error": f"backend launch failed: {e}"}
|
||||||
|
return 200, {
|
||||||
|
"op": req.op,
|
||||||
|
"bottle_id": req.bottle_id,
|
||||||
|
"source_ip": req.source_ip,
|
||||||
|
"image_ref": req.image_ref,
|
||||||
|
"slot": req.slot,
|
||||||
|
}
|
||||||
|
|
||||||
|
return 404, {"error": "not found"}
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(http.server.BaseHTTPRequestHandler):
|
||||||
|
"""Thin stdlib adapter: read the body, call `dispatch`, write JSON."""
|
||||||
|
|
||||||
|
# Socket timeout per request (applied by StreamRequestHandler.setup) so a
|
||||||
|
# stalled caller can't pin a handler thread on this privileged listener.
|
||||||
|
timeout = REQUEST_TIMEOUT_SECONDS
|
||||||
|
|
||||||
|
# Quiet by default; opt back into stdlib access logging with
|
||||||
|
# BOT_BOTTLE_HOST_CONTROLLER_DEBUG (the controller has its own logging).
|
||||||
|
def log_message(self, format: str, *args: typing.Any) -> None: # noqa: A002
|
||||||
|
if os.environ.get("BOT_BOTTLE_HOST_CONTROLLER_DEBUG"):
|
||||||
|
super().log_message(format, *args)
|
||||||
|
|
||||||
|
def _serve(self, method: str) -> None:
|
||||||
|
"""Read the request body (bounded), dispatch it, and write the JSON
|
||||||
|
reply. A dispatch that raises (it shouldn't — dispatch is total) still
|
||||||
|
returns a 500 rather than dropping the connection."""
|
||||||
|
server = self.server
|
||||||
|
assert isinstance(server, HostControlServer)
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get("Content-Length") or 0)
|
||||||
|
except ValueError:
|
||||||
|
self._reply(400, {"error": "invalid Content-Length"})
|
||||||
|
return
|
||||||
|
if length < 0 or length > MAX_BODY_BYTES:
|
||||||
|
# Reject before reading: nothing legitimate is this big, so an
|
||||||
|
# oversized declared length is a bug or a resource-exhaustion attempt.
|
||||||
|
self._reply(413, {"error": "request body too large"})
|
||||||
|
return
|
||||||
|
body = self.rfile.read(length) if length > 0 else b""
|
||||||
|
try:
|
||||||
|
status, payload = dispatch(server.broker, method, self.path, body)
|
||||||
|
except Exception as e: # noqa: BLE001 — the controller must stay up
|
||||||
|
sys.stderr.write(f"host controller: {method} {self.path} failed: {e!r}\n")
|
||||||
|
sys.stderr.flush()
|
||||||
|
status, payload = 500, {"error": f"internal error: {e}"}
|
||||||
|
self._reply(status, payload)
|
||||||
|
|
||||||
|
def _reply(self, status: int, payload: typing.Mapping[str, object]) -> None:
|
||||||
|
"""Write one JSON response with an explicit Content-Length."""
|
||||||
|
data = json.dumps(payload).encode()
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(data)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(data)
|
||||||
|
|
||||||
|
def do_GET(self) -> None:
|
||||||
|
self._serve("GET")
|
||||||
|
|
||||||
|
def do_POST(self) -> None:
|
||||||
|
self._serve("POST")
|
||||||
|
|
||||||
|
|
||||||
|
class HostControlServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||||||
|
"""Threading HTTP server that carries the launch broker for its handlers.
|
||||||
|
|
||||||
|
The broker holds the shared signing secret and performs the backend-native
|
||||||
|
launch/teardown; the server itself keeps no secret of its own — provenance
|
||||||
|
rides entirely in each request's signed token."""
|
||||||
|
|
||||||
|
daemon_threads = True
|
||||||
|
allow_reuse_address = True
|
||||||
|
|
||||||
|
def __init__(self, address: tuple[str, int], broker: LaunchBroker) -> None:
|
||||||
|
self.broker = broker
|
||||||
|
super().__init__(address, Handler)
|
||||||
|
|
||||||
|
|
||||||
|
def make_host_server(
|
||||||
|
broker: LaunchBroker, host: str = "127.0.0.1", port: int = DEFAULT_PORT
|
||||||
|
) -> HostControlServer:
|
||||||
|
"""Build (but do not start) a host control server. `port=0` binds an
|
||||||
|
ephemeral port — read `server.server_address` for the actual one."""
|
||||||
|
return HostControlServer((host, port), broker)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
"""Run the host control server as a plain process (dev-harness).
|
||||||
|
|
||||||
|
python -m bot_bottle.orchestrator.host_server [--host H] [--port P]
|
||||||
|
|
||||||
|
Fail-closed: without the launch-broker key the server can verify no request's
|
||||||
|
provenance, so it refuses to start rather than run a launcher that accepts
|
||||||
|
unsigned input. As the host-side owner of the key, it may mint/read the host
|
||||||
|
key file (`allow_host_file=True`)."""
|
||||||
|
parser = argparse.ArgumentParser(prog="bot_bottle.orchestrator.host_server")
|
||||||
|
parser.add_argument("--host", default="127.0.0.1", help="bind address")
|
||||||
|
parser.add_argument("--port", type=int, default=DEFAULT_PORT, help="bind port (0 = ephemeral)")
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
|
secret = broker_secret(allow_host_file=True)
|
||||||
|
if secret is None:
|
||||||
|
sys.stderr.write(
|
||||||
|
f"host controller: refusing to start without the launch-broker key "
|
||||||
|
f"(${LAUNCH_BROKER_KEY_ENV}, or a writable host root to mint it) — it "
|
||||||
|
"could verify no request's provenance and would relay unsigned "
|
||||||
|
"launches to the backend\n"
|
||||||
|
)
|
||||||
|
sys.stderr.flush()
|
||||||
|
return 2
|
||||||
|
|
||||||
|
broker = DockerBroker(secret)
|
||||||
|
server = make_host_server(broker, host=args.host, port=args.port)
|
||||||
|
bound_host, bound_port = server.server_address[0], server.server_address[1]
|
||||||
|
log.info(
|
||||||
|
"host control server listening",
|
||||||
|
context={"host": bound_host, "port": bound_port},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
server.serve_forever()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
log.info("host controller shutting down")
|
||||||
|
finally:
|
||||||
|
server.server_close()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"dispatch",
|
||||||
|
"Handler",
|
||||||
|
"HostControlServer",
|
||||||
|
"make_host_server",
|
||||||
|
"broker_secret",
|
||||||
|
"main",
|
||||||
|
"Json",
|
||||||
|
"DEFAULT_PORT",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -59,10 +59,8 @@ import http.server
|
|||||||
import json
|
import json
|
||||||
import math
|
import math
|
||||||
import os
|
import os
|
||||||
import socket
|
|
||||||
import socketserver
|
import socketserver
|
||||||
import sys
|
import sys
|
||||||
import threading
|
|
||||||
import typing
|
import typing
|
||||||
from urllib.parse import urlsplit
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
@@ -82,9 +80,6 @@ Json = dict[str, object]
|
|||||||
# token at all, and a compromised gateway holds only `gateway` — neither can
|
# token at all, and a compromised gateway holds only `gateway` — neither can
|
||||||
# drive the operator routes (approve proposals, rewrite policy, read tokens).
|
# drive the operator routes (approve proposals, rewrite policy, read tokens).
|
||||||
ORCHESTRATOR_AUTH_HEADER = "x-bot-bottle-orchestrator-auth"
|
ORCHESTRATOR_AUTH_HEADER = "x-bot-bottle-orchestrator-auth"
|
||||||
MAX_BODY_BYTES = 1 * 1024 * 1024
|
|
||||||
REQUEST_TIMEOUT_SECONDS = 10.0
|
|
||||||
MAX_REQUEST_THREADS = 32
|
|
||||||
|
|
||||||
# The routes the data plane (role `gateway`) is allowed to reach — exactly the
|
# The routes the data plane (role `gateway`) is allowed to reach — exactly the
|
||||||
# per-request lookups PolicyResolver makes. Every other authenticated route is
|
# per-request lookups PolicyResolver makes. Every other authenticated route is
|
||||||
@@ -121,8 +116,9 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches
|
|||||||
no I/O beyond the orchestrator — so it is fully testable without a socket.
|
no I/O beyond the orchestrator — so it is fully testable without a socket.
|
||||||
|
|
||||||
`role` is the caller's verified control-plane role (`gateway` or `cli`), or
|
`role` is the caller's verified control-plane role (`gateway` or `cli`), or
|
||||||
None for an unauthenticated request. Every route except `GET /health`
|
None for an unauthenticated request; an open-mode server (no signing key
|
||||||
requires a role: a missing role is 401, and a role that
|
configured — see `OrchestratorServer`) passes `cli`. Every route except
|
||||||
|
`GET /health` requires a role: a missing role is 401, and a role that
|
||||||
doesn't cover the route is 403 — so a `gateway` data-plane token can reach
|
doesn't cover the route is 403 — so a `gateway` data-plane token can reach
|
||||||
`/resolve` + `/supervise/{propose,poll}` but not the operator routes
|
`/resolve` + `/supervise/{propose,poll}` but not the operator routes
|
||||||
(rewrite policy, read injected tokens, approve its own supervise proposals).
|
(rewrite policy, read injected tokens, approve its own supervise proposals).
|
||||||
@@ -376,33 +372,10 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
|||||||
plane down for the caller."""
|
plane down for the caller."""
|
||||||
server = self.server
|
server = self.server
|
||||||
assert isinstance(server, OrchestratorServer)
|
assert isinstance(server, OrchestratorServer)
|
||||||
|
length = int(self.headers.get("Content-Length") or 0)
|
||||||
|
body = self.rfile.read(length) if length > 0 else b""
|
||||||
role = server.role_for(self.headers.get(ORCHESTRATOR_AUTH_HEADER, ""))
|
role = server.role_for(self.headers.get(ORCHESTRATOR_AUTH_HEADER, ""))
|
||||||
route = urlsplit(self.path).path.rstrip("/") or "/"
|
|
||||||
if not (method == "GET" and route == "/health") and role is None:
|
|
||||||
self._write_json(
|
|
||||||
401, {"error": "control-plane authentication required"},
|
|
||||||
)
|
|
||||||
return
|
|
||||||
length_header = self.headers.get("Content-Length")
|
|
||||||
try:
|
try:
|
||||||
length = int(length_header) if length_header is not None else 0
|
|
||||||
except ValueError:
|
|
||||||
self._write_json(400, {"error": "invalid Content-Length"})
|
|
||||||
return
|
|
||||||
if length < 0:
|
|
||||||
self._write_json(400, {"error": "invalid Content-Length"})
|
|
||||||
return
|
|
||||||
if length > MAX_BODY_BYTES:
|
|
||||||
self._write_json(413, {"error": "request body too large"})
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
body = self.rfile.read(length) if length else b""
|
|
||||||
except (TimeoutError, socket.timeout):
|
|
||||||
self._write_json(408, {"error": "request body read timed out"})
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
status: int
|
|
||||||
payload: Json
|
|
||||||
status, payload = dispatch(
|
status, payload = dispatch(
|
||||||
server.orchestrator, method, self.path, body, role=role)
|
server.orchestrator, method, self.path, body, role=role)
|
||||||
except Exception as e: # noqa: BLE001 — the control plane must stay up
|
except Exception as e: # noqa: BLE001 — the control plane must stay up
|
||||||
@@ -415,9 +388,6 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
|||||||
)
|
)
|
||||||
sys.stderr.flush()
|
sys.stderr.flush()
|
||||||
status, payload = 500, {"error": "internal error"}
|
status, payload = 500, {"error": "internal error"}
|
||||||
self._write_json(status, payload)
|
|
||||||
|
|
||||||
def _write_json(self, status: int, payload: Json) -> None:
|
|
||||||
data = json.dumps(payload).encode()
|
data = json.dumps(payload).encode()
|
||||||
self.send_response(status)
|
self.send_response(status)
|
||||||
self.send_header("Content-Type", "application/json")
|
self.send_header("Content-Type", "application/json")
|
||||||
@@ -444,80 +414,51 @@ class OrchestratorServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
|||||||
Holds the per-host control-plane *signing key* (from
|
Holds the per-host control-plane *signing key* (from
|
||||||
`$BOT_BOTTLE_ORCHESTRATOR_TOKEN`, injected by the launcher into the
|
`$BOT_BOTTLE_ORCHESTRATOR_TOKEN`, injected by the launcher into the
|
||||||
orchestrator process only) and verifies each request's role-scoped token
|
orchestrator process only) and verifies each request's role-scoped token
|
||||||
against it. Every route but `/health` requires a valid token whose role
|
against it. When a key is set, every route but `/health` requires a valid
|
||||||
covers the route. Construction fails when the key is absent so a new or
|
token whose role covers the route; when it is unset the server runs **open**
|
||||||
misconfigured launcher cannot accidentally expose an open control plane."""
|
(full `cli` access) and says so loudly at startup — a fail-visible fallback
|
||||||
|
for tests and any backend that hasn't wired the key yet (e.g. Firecracker,
|
||||||
|
whose nft boundary already blocks agents from the control-plane port)."""
|
||||||
|
|
||||||
daemon_threads = True
|
daemon_threads = True
|
||||||
allow_reuse_address = True
|
allow_reuse_address = True
|
||||||
|
|
||||||
def __init__(
|
def __init__(self, address: tuple[str, int], orchestrator: OrchestratorCore) -> None:
|
||||||
self,
|
|
||||||
address: tuple[str, int],
|
|
||||||
orchestrator: OrchestratorCore,
|
|
||||||
*,
|
|
||||||
signing_key: str,
|
|
||||||
) -> None:
|
|
||||||
self.orchestrator = orchestrator
|
self.orchestrator = orchestrator
|
||||||
self._signing_key = signing_key.strip()
|
# The control-plane trust domain's signing key, as injected into THIS
|
||||||
|
# (the owning) process by the launcher (#476). Unset → open mode below.
|
||||||
|
self._signing_key = CONTROL_PLANE.key_from_env()
|
||||||
if not self._signing_key:
|
if not self._signing_key:
|
||||||
raise ValueError(
|
sys.stderr.write(
|
||||||
"orchestrator control-plane signing key is required; "
|
"orchestrator: WARNING — no control-plane signing key "
|
||||||
"refusing to start without caller authentication"
|
f"(${CONTROL_PLANE.key_env}); running WITHOUT caller "
|
||||||
|
"authentication. Any client that can reach this port can drive "
|
||||||
|
"it. Backends that put the control plane on an agent-reachable "
|
||||||
|
"network MUST set this.\n"
|
||||||
)
|
)
|
||||||
self._request_slots = threading.BoundedSemaphore(MAX_REQUEST_THREADS)
|
sys.stderr.flush()
|
||||||
super().__init__(address, Handler)
|
super().__init__(address, Handler)
|
||||||
|
|
||||||
def get_request(self) -> tuple[socket.socket, typing.Any]:
|
|
||||||
request, client_address = super().get_request()
|
|
||||||
request.settimeout(REQUEST_TIMEOUT_SECONDS)
|
|
||||||
return request, client_address
|
|
||||||
|
|
||||||
def process_request(
|
|
||||||
self, request: typing.Any, client_address: typing.Any,
|
|
||||||
) -> None:
|
|
||||||
# Bound concurrency before ThreadingMixIn creates a worker. Backpressure
|
|
||||||
# stays in the accept loop instead of allocating an unbounded thread per
|
|
||||||
# slow or malicious connection.
|
|
||||||
self._request_slots.acquire()
|
|
||||||
try:
|
|
||||||
super().process_request(request, client_address)
|
|
||||||
except BaseException:
|
|
||||||
self._request_slots.release()
|
|
||||||
raise
|
|
||||||
|
|
||||||
def process_request_thread(
|
|
||||||
self, request: typing.Any, client_address: typing.Any,
|
|
||||||
) -> None:
|
|
||||||
try:
|
|
||||||
super().process_request_thread(request, client_address)
|
|
||||||
finally:
|
|
||||||
self._request_slots.release()
|
|
||||||
|
|
||||||
def role_for(self, presented: str) -> str | None:
|
def role_for(self, presented: str) -> str | None:
|
||||||
"""The verified caller role, or None for a missing/invalid token."""
|
"""The role the request is authorized as, or None if unauthenticated.
|
||||||
|
Open mode (no signing key) grants full `cli` access — the fail-visible
|
||||||
|
fallback. Otherwise verify the presented signed token; a missing/invalid
|
||||||
|
token yields None (→ 401), a valid one yields its `gateway`/`cli`
|
||||||
|
role (→ per-route 401/403 in `dispatch`)."""
|
||||||
|
if not self._signing_key:
|
||||||
|
return ROLE_CLI
|
||||||
return CONTROL_PLANE.verify(presented, self._signing_key)
|
return CONTROL_PLANE.verify(presented, self._signing_key)
|
||||||
|
|
||||||
|
|
||||||
def make_server(
|
def make_server(
|
||||||
orchestrator: OrchestratorCore,
|
orchestrator: OrchestratorCore, host: str = "127.0.0.1", port: int = 0
|
||||||
host: str = "127.0.0.1",
|
|
||||||
port: int = 0,
|
|
||||||
*,
|
|
||||||
signing_key: str | None = None,
|
|
||||||
) -> OrchestratorServer:
|
) -> OrchestratorServer:
|
||||||
"""Build an authenticated control-plane server.
|
"""Build (but do not start) a control-plane server. `port=0` binds an
|
||||||
|
ephemeral port — read `server.server_address` for the actual one."""
|
||||||
``signing_key=None`` reads the owning process's injected environment.
|
return OrchestratorServer((host, port), orchestrator)
|
||||||
Empty or missing keys are rejected by :class:`OrchestratorServer`.
|
|
||||||
"""
|
|
||||||
key = CONTROL_PLANE.key_from_env() if signing_key is None else signing_key
|
|
||||||
return OrchestratorServer(
|
|
||||||
(host, port), orchestrator, signing_key=key,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"dispatch", "Handler", "OrchestratorServer", "make_server", "Json",
|
"dispatch", "Handler", "OrchestratorServer", "make_server", "Json",
|
||||||
"ORCHESTRATOR_AUTH_HEADER", "MAX_BODY_BYTES",
|
"ORCHESTRATOR_AUTH_HEADER",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ import json
|
|||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from .broker import LaunchBroker, LaunchRequest, sign_request
|
from .broker import BrokerUnavailableError, LaunchRequest, SubmitBroker, sign_request
|
||||||
from .store.registry_store import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore
|
from .store.registry_store import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore
|
||||||
from .supervisor import (
|
from .supervisor import (
|
||||||
AuditEntry,
|
AuditEntry,
|
||||||
@@ -62,7 +62,7 @@ class OrchestratorCore:
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
registry: RegistryStore,
|
registry: RegistryStore,
|
||||||
broker: LaunchBroker,
|
broker: SubmitBroker,
|
||||||
sign_secret: bytes,
|
sign_secret: bytes,
|
||||||
supervisor: Supervisor | None = None,
|
supervisor: Supervisor | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -111,14 +111,23 @@ class OrchestratorCore:
|
|||||||
image_ref=image_ref,
|
image_ref=image_ref,
|
||||||
slot=slot,
|
slot=slot,
|
||||||
)
|
)
|
||||||
launched = False
|
|
||||||
try:
|
try:
|
||||||
self._broker.submit(sign_request(req, self._secret))
|
self._broker.submit(sign_request(req, self._secret))
|
||||||
launched = True
|
except BrokerUnavailableError:
|
||||||
finally:
|
# Ambiguous delivery failure (timeout / dropped response): the broker
|
||||||
if not launched:
|
# may already have launched the bottle before the response was lost.
|
||||||
self.registry.deregister(rec.bottle_id)
|
# Do NOT deregister — that would orphan a running container with no
|
||||||
self._tokens.pop(rec.bottle_id, None)
|
# registry row (reconcile reaps rows, never containers). Keep the row
|
||||||
|
# so reconcile reaps it iff the bottle is not actually live; surface
|
||||||
|
# the error so the caller knows the launch is unconfirmed.
|
||||||
|
raise
|
||||||
|
except Exception:
|
||||||
|
# A definite failure — a fail-closed rejection, a backend launch
|
||||||
|
# error, or the host reporting it did not launch: nothing is running,
|
||||||
|
# so roll the registry entry back to leave no orphan.
|
||||||
|
self.registry.deregister(rec.bottle_id)
|
||||||
|
self._tokens.pop(rec.bottle_id, None)
|
||||||
|
raise
|
||||||
return rec
|
return rec
|
||||||
|
|
||||||
def teardown_bottle(self, bottle_id: str) -> bool:
|
def teardown_bottle(self, bottle_id: str) -> bool:
|
||||||
@@ -366,23 +375,15 @@ class OrchestratorCore:
|
|||||||
value with *env_var_secret*, and restores ``_tokens[bottle_id]``.
|
value with *env_var_secret*, and restores ``_tokens[bottle_id]``.
|
||||||
Returns True on success, False when no stored secrets exist for this
|
Returns True on success, False when no stored secrets exist for this
|
||||||
bottle or decryption fails (wrong key / corrupt data)."""
|
bottle or decryption fails (wrong key / corrupt data)."""
|
||||||
from .store.secret_store import decrypt_value, encrypt_value, is_legacy_blob
|
from .store.secret_store import decrypt_value
|
||||||
encrypted = self.registry.get_agent_secrets(bottle_id)
|
encrypted = self.registry.get_agent_secrets(bottle_id)
|
||||||
if not encrypted:
|
if not encrypted:
|
||||||
return False
|
return False
|
||||||
try:
|
try:
|
||||||
decrypted = {
|
self._tokens[bottle_id] = {k: decrypt_value(env_var_secret, v)
|
||||||
k: decrypt_value(env_var_secret, v) for k, v in encrypted.items()
|
for k, v in encrypted.items()}
|
||||||
}
|
|
||||||
except ValueError:
|
except ValueError:
|
||||||
return False
|
return False
|
||||||
self._tokens[bottle_id] = decrypted
|
|
||||||
if any(is_legacy_blob(value) for value in encrypted.values()):
|
|
||||||
migrated = {
|
|
||||||
key: encrypt_value(env_var_secret, value)
|
|
||||||
for key, value in decrypted.items()
|
|
||||||
}
|
|
||||||
self.registry.store_agent_secrets(bottle_id, migrated)
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# --- consolidated gateway ----------------------------------------------
|
# --- consolidated gateway ----------------------------------------------
|
||||||
|
|||||||
@@ -12,18 +12,22 @@ reattachment path reads ENV_VAR_SECRET from the running agent container via
|
|||||||
``POST /bottles/<id>/reprovision_gateway``; the orchestrator decrypts the
|
``POST /bottles/<id>/reprovision_gateway``; the orchestrator decrypts the
|
||||||
stored rows and re-populates ``_tokens``.
|
stored rows and re-populates ``_tokens``.
|
||||||
|
|
||||||
Encryption scheme: encrypt-then-MAC using independent HMAC-SHA256-derived
|
Encryption scheme: HMAC-SHA256 used as a PRF in CTR mode, **authenticated**
|
||||||
encryption and authentication subkeys (stdlib-only, no external deps). Each
|
encrypt-then-MAC (stdlib-only, no external deps). Each value is encrypted
|
||||||
value is encrypted independently. New output blobs are:
|
independently. The output blob is ``nonce (16 bytes) || ciphertext || tag
|
||||||
|
(32 bytes)`` encoded as URL-safe base64 (no padding).
|
||||||
``version || nonce (16 bytes) || ciphertext || tag (32 bytes)``
|
|
||||||
|
|
||||||
encoded as URL-safe base64 (no padding). The version marker lets the reader
|
|
||||||
accept legacy ``nonce || ciphertext`` rows long enough to rewrite them in the
|
|
||||||
authenticated format after a successful reprovision.
|
|
||||||
|
|
||||||
keystream_block_i = HMAC-SHA256(key, nonce || i.to_bytes(4, "big"))
|
keystream_block_i = HMAC-SHA256(key, nonce || i.to_bytes(4, "big"))
|
||||||
ciphertext_i = plaintext_i XOR keystream_block_i[:len(plaintext_i)]
|
ciphertext_i = plaintext_i XOR keystream_block_i[:len(plaintext_i)]
|
||||||
|
mac_key = HMAC-SHA256(key, "bottled-secret-mac-v1")
|
||||||
|
tag = HMAC-SHA256(mac_key, nonce || ciphertext)
|
||||||
|
|
||||||
|
The tag is what makes a **wrong key deterministically detectable**: without it,
|
||||||
|
CTR decryption with the wrong key yields garbage that only fails when it isn't
|
||||||
|
valid UTF-8 (so ``reprovision`` would sometimes "succeed" with a wrong
|
||||||
|
ENV_VAR_SECRET and inject garbage egress tokens). The MAC key is derived from
|
||||||
|
the ENV_VAR_SECRET by a domain-separated HMAC so the same key never both
|
||||||
|
generates the keystream and signs the tag with the same message shape.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -35,9 +39,8 @@ import secrets
|
|||||||
|
|
||||||
_KEY_BYTES = 32 # 256-bit key from ENV_VAR_SECRET
|
_KEY_BYTES = 32 # 256-bit key from ENV_VAR_SECRET
|
||||||
_NONCE_BYTES = 16 # 128-bit random nonce per encrypt call
|
_NONCE_BYTES = 16 # 128-bit random nonce per encrypt call
|
||||||
|
_TAG_BYTES = 32 # HMAC-SHA256 authentication tag
|
||||||
_BLOCK = 32 # HMAC-SHA256 output width == one keystream block
|
_BLOCK = 32 # HMAC-SHA256 output width == one keystream block
|
||||||
_TAG_BYTES = 32
|
|
||||||
_VERSION = b"BBSE1"
|
|
||||||
|
|
||||||
# Env-var name the agent container receives at startup.
|
# Env-var name the agent container receives at startup.
|
||||||
ENV_VAR_SECRET_NAME = "ENV_VAR_SECRET"
|
ENV_VAR_SECRET_NAME = "ENV_VAR_SECRET"
|
||||||
@@ -49,13 +52,7 @@ def new_env_var_secret() -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _b64dec(s: str) -> bytes:
|
def _b64dec(s: str) -> bytes:
|
||||||
return base64.b64decode(
|
return base64.urlsafe_b64decode(s + "=" * (-len(s) % 4))
|
||||||
s + "=" * (-len(s) % 4), altchars=b"-_", validate=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _subkey(key: bytes, purpose: bytes) -> bytes:
|
|
||||||
return hmac.new(key, b"bot-bottle-secret-store:" + purpose, hashlib.sha256).digest()
|
|
||||||
|
|
||||||
|
|
||||||
def _keystream(key: bytes, nonce: bytes, block_index: int) -> bytes:
|
def _keystream(key: bytes, nonce: bytes, block_index: int) -> bytes:
|
||||||
@@ -64,93 +61,58 @@ def _keystream(key: bytes, nonce: bytes, block_index: int) -> bytes:
|
|||||||
).digest()
|
).digest()
|
||||||
|
|
||||||
|
|
||||||
|
def _tag(key: bytes, nonce: bytes, ciphertext: bytes) -> bytes:
|
||||||
|
"""The authentication tag over ``nonce || ciphertext``, keyed by a MAC
|
||||||
|
subkey domain-separated from the keystream key."""
|
||||||
|
mac_key = hmac.new(key, b"bottled-secret-mac-v1", hashlib.sha256).digest()
|
||||||
|
return hmac.new(mac_key, nonce + ciphertext, hashlib.sha256).digest()
|
||||||
|
|
||||||
|
|
||||||
|
def _ctr(key: bytes, nonce: bytes, data: bytes) -> bytes:
|
||||||
|
"""CTR keystream XOR — its own inverse, so it both encrypts and decrypts."""
|
||||||
|
out = bytearray()
|
||||||
|
for i in range(0, len(data), _BLOCK):
|
||||||
|
chunk = data[i : i + _BLOCK]
|
||||||
|
ks = _keystream(key, nonce, i)[: len(chunk)]
|
||||||
|
out.extend(b ^ k for b, k in zip(chunk, ks))
|
||||||
|
return bytes(out)
|
||||||
|
|
||||||
|
|
||||||
def encrypt_value(secret_b64: str, plaintext: str) -> str:
|
def encrypt_value(secret_b64: str, plaintext: str) -> str:
|
||||||
"""Encrypt a single string value with *secret_b64* (the ENV_VAR_SECRET).
|
"""Encrypt a single string value with *secret_b64* (the ENV_VAR_SECRET).
|
||||||
|
|
||||||
Returns a URL-safe base64 authenticated blob suitable for
|
Returns a URL-safe base64 blob ``nonce || ciphertext || tag`` suitable for
|
||||||
the ``bottled_agent_secrets.value`` column."""
|
the ``bottled_agent_secrets.value`` column."""
|
||||||
key = _b64dec(secret_b64)
|
key = _b64dec(secret_b64)
|
||||||
encryption_key = _subkey(key, b"encryption")
|
|
||||||
authentication_key = _subkey(key, b"authentication")
|
|
||||||
pt = plaintext.encode()
|
|
||||||
nonce = secrets.token_bytes(_NONCE_BYTES)
|
nonce = secrets.token_bytes(_NONCE_BYTES)
|
||||||
ct = bytearray()
|
ct = _ctr(key, nonce, plaintext.encode())
|
||||||
for i in range(0, len(pt), _BLOCK):
|
tag = _tag(key, nonce, ct)
|
||||||
chunk = pt[i : i + _BLOCK]
|
return base64.urlsafe_b64encode(nonce + ct + tag).rstrip(b"=").decode()
|
||||||
ks = _keystream(encryption_key, nonce, i // _BLOCK)[: len(chunk)]
|
|
||||||
ct.extend(p ^ k for p, k in zip(chunk, ks))
|
|
||||||
authenticated = _VERSION + nonce + bytes(ct)
|
|
||||||
tag = hmac.new(authentication_key, authenticated, hashlib.sha256).digest()
|
|
||||||
return base64.urlsafe_b64encode(authenticated + tag).rstrip(b"=").decode()
|
|
||||||
|
|
||||||
|
|
||||||
def is_legacy_blob(blob_b64: str) -> bool:
|
|
||||||
"""Whether *blob_b64* uses the pre-authentication storage format."""
|
|
||||||
try:
|
|
||||||
return not _b64dec(blob_b64).startswith(_VERSION)
|
|
||||||
except (ValueError, TypeError):
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _decrypt_legacy(key: bytes, blob: bytes) -> str:
|
|
||||||
"""Read the original ``nonce || ciphertext`` format for migration only."""
|
|
||||||
if len(blob) < _NONCE_BYTES:
|
|
||||||
raise ValueError("ciphertext blob too short")
|
|
||||||
nonce, ciphertext = blob[:_NONCE_BYTES], blob[_NONCE_BYTES:]
|
|
||||||
pt = bytearray()
|
|
||||||
# The legacy format used the byte offset as the PRF counter.
|
|
||||||
for i in range(0, len(ciphertext), _BLOCK):
|
|
||||||
chunk = ciphertext[i : i + _BLOCK]
|
|
||||||
ks = _keystream(key, nonce, i)[: len(chunk)]
|
|
||||||
pt.extend(c ^ k for c, k in zip(chunk, ks))
|
|
||||||
try:
|
|
||||||
return bytes(pt).decode()
|
|
||||||
except UnicodeDecodeError as exc:
|
|
||||||
raise ValueError(f"decryption produced non-UTF-8 output: {exc}") from exc
|
|
||||||
|
|
||||||
|
|
||||||
def decrypt_value(secret_b64: str, blob_b64: str) -> str:
|
def decrypt_value(secret_b64: str, blob_b64: str) -> str:
|
||||||
"""Decrypt a blob produced by :func:`encrypt_value`.
|
"""Decrypt a blob produced by :func:`encrypt_value`.
|
||||||
|
|
||||||
Returns the original plaintext string. Raises ``ValueError`` for malformed
|
Returns the original plaintext string. Raises ``ValueError`` for malformed
|
||||||
input, authentication failure, or a key mismatch. Legacy unauthenticated
|
input, a **wrong key**, or a tampered ciphertext — all caught by the
|
||||||
rows remain readable so callers can migrate them immediately."""
|
authentication tag before any plaintext is returned, so a wrong
|
||||||
|
ENV_VAR_SECRET is rejected deterministically (never a garbage token)."""
|
||||||
key = _b64dec(secret_b64)
|
key = _b64dec(secret_b64)
|
||||||
try:
|
try:
|
||||||
blob = _b64dec(blob_b64)
|
blob = _b64dec(blob_b64)
|
||||||
except (ValueError, TypeError) as exc:
|
except Exception as exc:
|
||||||
raise ValueError(f"invalid ciphertext blob: {exc}") from exc
|
raise ValueError(f"invalid ciphertext blob: {exc}") from exc
|
||||||
if not blob.startswith(_VERSION):
|
if len(blob) < _NONCE_BYTES + _TAG_BYTES:
|
||||||
return _decrypt_legacy(key, blob)
|
|
||||||
minimum = len(_VERSION) + _NONCE_BYTES + _TAG_BYTES
|
|
||||||
if len(blob) < minimum:
|
|
||||||
raise ValueError("ciphertext blob too short")
|
raise ValueError("ciphertext blob too short")
|
||||||
authenticated, supplied_tag = blob[:-_TAG_BYTES], blob[-_TAG_BYTES:]
|
nonce = blob[:_NONCE_BYTES]
|
||||||
authentication_key = _subkey(key, b"authentication")
|
tag = blob[-_TAG_BYTES:]
|
||||||
expected_tag = hmac.new(
|
ciphertext = blob[_NONCE_BYTES:-_TAG_BYTES]
|
||||||
authentication_key, authenticated, hashlib.sha256,
|
if not hmac.compare_digest(tag, _tag(key, nonce, ciphertext)):
|
||||||
).digest()
|
raise ValueError("ciphertext failed authentication (wrong key or tampered)")
|
||||||
if not hmac.compare_digest(supplied_tag, expected_tag):
|
|
||||||
raise ValueError("ciphertext authentication failed")
|
|
||||||
nonce_start = len(_VERSION)
|
|
||||||
nonce = blob[nonce_start : nonce_start + _NONCE_BYTES]
|
|
||||||
ciphertext = blob[nonce_start + _NONCE_BYTES : -_TAG_BYTES]
|
|
||||||
encryption_key = _subkey(key, b"encryption")
|
|
||||||
pt = bytearray()
|
|
||||||
for i in range(0, len(ciphertext), _BLOCK):
|
|
||||||
chunk = ciphertext[i : i + _BLOCK]
|
|
||||||
ks = _keystream(encryption_key, nonce, i // _BLOCK)[: len(chunk)]
|
|
||||||
pt.extend(c ^ k for c, k in zip(chunk, ks))
|
|
||||||
try:
|
try:
|
||||||
return bytes(pt).decode()
|
return _ctr(key, nonce, ciphertext).decode()
|
||||||
except UnicodeDecodeError as exc:
|
except UnicodeDecodeError as exc: # pragma: no cover - authenticated, so unreachable
|
||||||
raise ValueError(f"decryption produced non-UTF-8 output (wrong key?): {exc}") from exc
|
raise ValueError(f"decryption produced non-UTF-8 output: {exc}") from exc
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = ["ENV_VAR_SECRET_NAME", "new_env_var_secret", "encrypt_value", "decrypt_value"]
|
||||||
"ENV_VAR_SECRET_NAME",
|
|
||||||
"new_env_var_secret",
|
|
||||||
"encrypt_value",
|
|
||||||
"decrypt_value",
|
|
||||||
"is_legacy_blob",
|
|
||||||
]
|
|
||||||
|
|||||||
@@ -36,6 +36,13 @@ ROLE_GATEWAY = "gateway"
|
|||||||
ROLE_CLI = "cli"
|
ROLE_CLI = "cli"
|
||||||
ROLES: frozenset[str] = frozenset({ROLE_GATEWAY, ROLE_CLI})
|
ROLES: frozenset[str] = frozenset({ROLE_GATEWAY, ROLE_CLI})
|
||||||
|
|
||||||
|
# The host controller's own lifecycle role (#468). Deliberately OUTSIDE `ROLES`:
|
||||||
|
# it belongs to a separate trust domain (`HOST_CONTROLLER`) signed by a key the
|
||||||
|
# orchestrator never holds, so the orchestrator's control-plane key can neither
|
||||||
|
# mint nor accept it — the orchestrator must not be able to forge the credential
|
||||||
|
# used to start and stop it.
|
||||||
|
ROLE_HOST = "host"
|
||||||
|
|
||||||
_ALG = "HS256"
|
_ALG = "HS256"
|
||||||
|
|
||||||
|
|
||||||
@@ -103,4 +110,4 @@ def verify(token: str, secret: str, *, roles: frozenset[str] = ROLES) -> str | N
|
|||||||
return role if isinstance(role, str) and role in roles else None
|
return role if isinstance(role, str) and role in roles else None
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["ROLE_GATEWAY", "ROLE_CLI", "ROLES", "mint", "verify"]
|
__all__ = ["ROLE_GATEWAY", "ROLE_CLI", "ROLE_HOST", "ROLES", "mint", "verify"]
|
||||||
|
|||||||
@@ -47,6 +47,22 @@ ORCHESTRATOR_TOKEN_ENV = "BOT_BOTTLE_ORCHESTRATOR_TOKEN"
|
|||||||
# cannot forge a higher-privilege `cli` token (issue #469 review).
|
# cannot forge a higher-privilege `cli` token (issue #469 review).
|
||||||
ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT"
|
ORCHESTRATOR_AUTH_JWT_ENV = "BOT_BOTTLE_ORCHESTRATOR_AUTH_JWT"
|
||||||
|
|
||||||
|
# The durable launch-broker signing key: the HS256 secret the orchestrator
|
||||||
|
# (signer) and the host control server (verifier) share to sign/verify launch
|
||||||
|
# requests (#468). A host-canonical key file (minted 0600 on first use) so it
|
||||||
|
# survives orchestrator restarts — re-adoption re-verifies against the same key —
|
||||||
|
# instead of the ephemeral per-process secret of the in-process broker.
|
||||||
|
LAUNCH_BROKER_KEY_FILENAME = "launch-broker-key"
|
||||||
|
LAUNCH_BROKER_KEY_ENV = "BOT_BOTTLE_LAUNCH_BROKER_KEY"
|
||||||
|
# The host controller's OWN key, for its lifecycle endpoints (the direct
|
||||||
|
# cli -> host controller path that starts/stops the orchestrator). Separate from
|
||||||
|
# the launch-broker key and never held by the orchestrator: the controller starts
|
||||||
|
# and stops the orchestrator, so the orchestrator must not be able to mint the
|
||||||
|
# credentials used to drive it (#468/#476).
|
||||||
|
HOST_CONTROLLER_KEY_FILENAME = "host-controller-key"
|
||||||
|
HOST_CONTROLLER_KEY_ENV = "BOT_BOTTLE_HOST_CONTROLLER_KEY"
|
||||||
|
HOST_CONTROLLER_AUTH_JWT_ENV = "BOT_BOTTLE_HOST_CONTROLLER_AUTH_JWT"
|
||||||
|
|
||||||
# The host directory holding the gateway's persistent mitmproxy CA. Bind-mounted
|
# The host directory holding the gateway's persistent mitmproxy CA. Bind-mounted
|
||||||
# into the infra/gateway container at mitmproxy's confdir so the self-generated
|
# into the infra/gateway container at mitmproxy's confdir so the self-generated
|
||||||
# CA survives container recreation — every agent installs this one CA to trust
|
# CA survives container recreation — every agent installs this one CA to trust
|
||||||
@@ -142,6 +158,11 @@ __all__ = [
|
|||||||
"ORCHESTRATOR_TOKEN_FILENAME",
|
"ORCHESTRATOR_TOKEN_FILENAME",
|
||||||
"ORCHESTRATOR_TOKEN_ENV",
|
"ORCHESTRATOR_TOKEN_ENV",
|
||||||
"ORCHESTRATOR_AUTH_JWT_ENV",
|
"ORCHESTRATOR_AUTH_JWT_ENV",
|
||||||
|
"LAUNCH_BROKER_KEY_FILENAME",
|
||||||
|
"LAUNCH_BROKER_KEY_ENV",
|
||||||
|
"HOST_CONTROLLER_KEY_FILENAME",
|
||||||
|
"HOST_CONTROLLER_KEY_ENV",
|
||||||
|
"HOST_CONTROLLER_AUTH_JWT_ENV",
|
||||||
"GATEWAY_CA_DIRNAME",
|
"GATEWAY_CA_DIRNAME",
|
||||||
"bot_bottle_root",
|
"bot_bottle_root",
|
||||||
"host_db_path",
|
"host_db_path",
|
||||||
|
|||||||
@@ -29,8 +29,13 @@ from collections.abc import Mapping
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from . import orchestrator_auth
|
from . import orchestrator_auth
|
||||||
from .orchestrator_auth import ROLE_GATEWAY
|
from .orchestrator_auth import ROLE_GATEWAY, ROLE_HOST
|
||||||
from .paths import (
|
from .paths import (
|
||||||
|
HOST_CONTROLLER_AUTH_JWT_ENV,
|
||||||
|
HOST_CONTROLLER_KEY_ENV,
|
||||||
|
HOST_CONTROLLER_KEY_FILENAME,
|
||||||
|
LAUNCH_BROKER_KEY_ENV,
|
||||||
|
LAUNCH_BROKER_KEY_FILENAME,
|
||||||
ORCHESTRATOR_AUTH_JWT_ENV,
|
ORCHESTRATOR_AUTH_JWT_ENV,
|
||||||
ORCHESTRATOR_TOKEN_ENV,
|
ORCHESTRATOR_TOKEN_ENV,
|
||||||
ORCHESTRATOR_TOKEN_FILENAME,
|
ORCHESTRATOR_TOKEN_FILENAME,
|
||||||
@@ -40,7 +45,7 @@ from .paths import (
|
|||||||
|
|
||||||
class ProvisioningError(RuntimeError):
|
class ProvisioningError(RuntimeError):
|
||||||
"""A control-plane auth invariant would be violated (e.g. starting the
|
"""A control-plane auth invariant would be violated (e.g. starting the
|
||||||
orchestrator without its signing key)."""
|
orchestrator without its signing key — which would run OPEN)."""
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -67,8 +72,9 @@ class TrustDomain:
|
|||||||
|
|
||||||
def key_from_env(self, environ: Mapping[str, str] | None = None) -> str:
|
def key_from_env(self, environ: Mapping[str, str] | None = None) -> str:
|
||||||
"""The signing key as the owning process sees it — read from `key_env`
|
"""The signing key as the owning process sees it — read from `key_env`
|
||||||
(default `os.environ`). ``""`` when unset; owning services reject that
|
(default `os.environ`). "" when unset; the caller decides whether that is
|
||||||
value rather than start without authentication."""
|
fatal (`ControlPlaneProvisioning`) or the open-mode fallback
|
||||||
|
(`OrchestratorServer`)."""
|
||||||
env = os.environ if environ is None else environ
|
env = os.environ if environ is None else environ
|
||||||
return env.get(self.key_env, "").strip()
|
return env.get(self.key_env, "").strip()
|
||||||
|
|
||||||
@@ -98,6 +104,40 @@ CONTROL_PLANE = TrustDomain(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# The launch-broker domain (#468): durable key material for the broker's own
|
||||||
|
# signed launch requests (`broker.py`'s HS256 launch JWT), shared by the
|
||||||
|
# orchestrator (signer) and the host control server (verifier). Unlike
|
||||||
|
# `CONTROL_PLANE` it mints no role tokens — the broker's provenance is the launch
|
||||||
|
# JWT, not a role token — so its `roles` set is empty and it is used only as a
|
||||||
|
# provider of durable, host-canonical key material (`signing_key` / `key_from_env`).
|
||||||
|
# The durability is the point: the key survives orchestrator restarts, so a
|
||||||
|
# restarted orchestrator re-verifies against the same key instead of the
|
||||||
|
# ephemeral per-process secret the in-process broker used.
|
||||||
|
LAUNCH_BROKER = TrustDomain(
|
||||||
|
name="launch-broker",
|
||||||
|
key_filename=LAUNCH_BROKER_KEY_FILENAME,
|
||||||
|
roles=frozenset(),
|
||||||
|
key_env=LAUNCH_BROKER_KEY_ENV,
|
||||||
|
token_env="",
|
||||||
|
)
|
||||||
|
|
||||||
|
# The host controller's own domain (#468) — the SECOND domain #476 reserves. Its
|
||||||
|
# key, which the orchestrator never holds, signs the `host`-role tokens the CLI
|
||||||
|
# presents on the host controller's lifecycle endpoints (start / restart / status
|
||||||
|
# of the orchestrator itself). Keeping it separate from `CONTROL_PLANE` is the
|
||||||
|
# whole point: the host controller starts and stops the orchestrator, so the
|
||||||
|
# orchestrator must not be able to mint the credentials used to drive it. (The
|
||||||
|
# lifecycle endpoints themselves arrive in a later chunk; the domain is
|
||||||
|
# established here alongside the durable launch-broker key.)
|
||||||
|
HOST_CONTROLLER = TrustDomain(
|
||||||
|
name="host-controller",
|
||||||
|
key_filename=HOST_CONTROLLER_KEY_FILENAME,
|
||||||
|
roles=frozenset({ROLE_HOST}),
|
||||||
|
key_env=HOST_CONTROLLER_KEY_ENV,
|
||||||
|
token_env=HOST_CONTROLLER_AUTH_JWT_ENV,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class ControlPlaneProvisioning:
|
class ControlPlaneProvisioning:
|
||||||
"""The one seam every backend launcher uses to provision control-plane auth,
|
"""The one seam every backend launcher uses to provision control-plane auth,
|
||||||
@@ -131,9 +171,56 @@ class ControlPlaneProvisioning:
|
|||||||
return self.domain.mint(ROLE_GATEWAY)
|
return self.domain.mint(ROLE_GATEWAY)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class LaunchBrokerProvisioning:
|
||||||
|
"""The seam that provisions the host-side launch broker's durable keys (#468),
|
||||||
|
the counterpart to `ControlPlaneProvisioning`. Both the orchestrator (signer)
|
||||||
|
and the host control server (verifier) receive the SAME launch-broker key
|
||||||
|
(carry it in `broker_domain.key_env`); the host controller ALSO receives its
|
||||||
|
own lifecycle key (`controller_domain.key_env`) the orchestrator never holds.
|
||||||
|
|
||||||
|
Fail-closed like the control-plane seam: minting returns "" only if the host
|
||||||
|
root is unwritable, and an empty launch-broker key would leave the verifier
|
||||||
|
unable to authenticate any launch — so we raise rather than hand back a key
|
||||||
|
that would make the host controller reject (or, if a caller defaulted it,
|
||||||
|
accept) unsigned input."""
|
||||||
|
|
||||||
|
broker_domain: TrustDomain = LAUNCH_BROKER
|
||||||
|
controller_domain: TrustDomain = HOST_CONTROLLER
|
||||||
|
|
||||||
|
def broker_key(self) -> str:
|
||||||
|
"""The durable launch-broker key both the orchestrator and the host
|
||||||
|
control server must receive (in `broker_domain.key_env`). Raises rather
|
||||||
|
than return ""."""
|
||||||
|
key = self.broker_domain.signing_key()
|
||||||
|
if not key:
|
||||||
|
raise ProvisioningError(
|
||||||
|
f"refusing to provision the {self.broker_domain.name} broker "
|
||||||
|
"without a signing key: the host controller could then verify no "
|
||||||
|
"launch request's provenance"
|
||||||
|
)
|
||||||
|
return key
|
||||||
|
|
||||||
|
def controller_key(self) -> str:
|
||||||
|
"""The host controller's own lifecycle key — provisioned ONLY to the host
|
||||||
|
controller (in `controller_domain.key_env`), never to the orchestrator, so
|
||||||
|
the orchestrator cannot mint the `host`-role tokens that start and stop
|
||||||
|
it. Raises rather than return ""."""
|
||||||
|
key = self.controller_domain.signing_key()
|
||||||
|
if not key:
|
||||||
|
raise ProvisioningError(
|
||||||
|
f"refusing to provision the {self.controller_domain.name} without "
|
||||||
|
"a signing key: its lifecycle endpoints would authenticate no one"
|
||||||
|
)
|
||||||
|
return key
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"ProvisioningError",
|
"ProvisioningError",
|
||||||
"TrustDomain",
|
"TrustDomain",
|
||||||
"CONTROL_PLANE",
|
"CONTROL_PLANE",
|
||||||
|
"LAUNCH_BROKER",
|
||||||
|
"HOST_CONTROLLER",
|
||||||
"ControlPlaneProvisioning",
|
"ControlPlaneProvisioning",
|
||||||
|
"LaunchBrokerProvisioning",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -3,10 +3,6 @@
|
|||||||
- **Status:** Accepted
|
- **Status:** Accepted
|
||||||
- **Date:** 2026-06-25
|
- **Date:** 2026-06-25
|
||||||
- **Deciders:** didericis
|
- **Deciders:** didericis
|
||||||
- **Revised:** 2026-07-27 — thresholds relaxed (critical minimum 90→85%,
|
|
||||||
diff-coverage gate 90→80%) to cut low-value test churn on changed lines.
|
|
||||||
The risk-weighting structure and the "global is informational" rule are
|
|
||||||
unchanged.
|
|
||||||
|
|
||||||
## Context
|
## Context
|
||||||
|
|
||||||
@@ -38,7 +34,7 @@ a regression (Goodhart's law).
|
|||||||
Coverage is **risk-weighted**, measured over the **combined unit +
|
Coverage is **risk-weighted**, measured over the **combined unit +
|
||||||
integration** suites, with three rules:
|
integration** suites, with three rules:
|
||||||
|
|
||||||
1. **Critical modules must remain ≥ 85%.** The curated security/logic core
|
1. **Critical modules must remain ≥ 90%.** The curated security/logic core
|
||||||
covers the host and gateway egress policy, manifest trust boundary,
|
covers the host and gateway egress policy, manifest trust boundary,
|
||||||
git-gate enforcement, supervise protocol/server, YAML parser, and bottle
|
git-gate enforcement, supervise protocol/server, YAML parser, and bottle
|
||||||
state. The concrete module list lives in `scripts/critical-modules.txt`;
|
state. The concrete module list lives in `scripts/critical-modules.txt`;
|
||||||
@@ -59,7 +55,7 @@ integration** suites, with three rules:
|
|||||||
|
|
||||||
The forward-looking guard is a **diff-coverage gate**
|
The forward-looking guard is a **diff-coverage gate**
|
||||||
(`scripts/diff_coverage.py`): new/changed executable lines on a branch
|
(`scripts/diff_coverage.py`): new/changed executable lines on a branch
|
||||||
must be ≥ 80% covered. This catches regressions where they are
|
must be ≥ 90% covered. This catches regressions where they are
|
||||||
introduced without forcing a back-fill crusade through legacy glue. The
|
introduced without forcing a back-fill crusade through legacy glue. The
|
||||||
gate skips lines in omitted files (there is no coverage data for them),
|
gate skips lines in omitted files (there is no coverage data for them),
|
||||||
so the omit list cannot launder *new* logic into the dark: anything that
|
so the omit list cannot launder *new* logic into the dark: anything that
|
||||||
|
|||||||
@@ -56,7 +56,8 @@ key.
|
|||||||
- Rewriting the HMAC primitive: `orchestrator_auth.mint/verify` gain an optional
|
- Rewriting the HMAC primitive: `orchestrator_auth.mint/verify` gain an optional
|
||||||
`roles=` arg (default unchanged) so a key can carry a different role set;
|
`roles=` arg (default unchanged) so a key can carry a different role set;
|
||||||
nothing else changes.
|
nothing else changes.
|
||||||
- Network topology or the plane split (#469).
|
- Network topology, the plane split (#469), or the server's open-mode fallback
|
||||||
|
for tests.
|
||||||
|
|
||||||
## Design
|
## Design
|
||||||
|
|
||||||
|
|||||||
@@ -1,197 +0,0 @@
|
|||||||
# PRD 0082: Authoritative failure boundaries
|
|
||||||
|
|
||||||
- **Status:** Draft
|
|
||||||
- **Author:** codex
|
|
||||||
- **Created:** 2026-07-27
|
|
||||||
- **Issue:** #444
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
Make every security- or lifecycle-sensitive snapshot distinguish authoritative
|
|
||||||
empty state from unavailable state, and make every destructive or
|
|
||||||
resource-consuming boundary revalidate the assumptions it acts on. This
|
|
||||||
finishes the focused quality work begun under #444 without broad rewrites:
|
|
||||||
cleanup cannot act on stale identities, policy introspection cannot publish a
|
|
||||||
fabricated empty policy, gateway servers bound untrusted work, and daemon
|
|
||||||
shutdown does not emit uncaught background-thread failures. Shared
|
|
||||||
control-plane storage and gateway credential provisioning also enforce their
|
|
||||||
filesystem security contract before sensitive data is written.
|
|
||||||
|
|
||||||
## Problem
|
|
||||||
|
|
||||||
Several paths are individually fail-closed but compose into unsafe or
|
|
||||||
misleading behavior:
|
|
||||||
|
|
||||||
1. `cleanup` prepares a plan, waits indefinitely for operator confirmation,
|
|
||||||
then kills stored PIDs and removes stored paths without checking that those
|
|
||||||
identities still describe the same orphan. A PID may be reused or a run
|
|
||||||
directory may become active during the prompt.
|
|
||||||
2. The supervisor reuses egress's deny-all fallback for
|
|
||||||
`list-egress-routes`. Deny-all is correct for enforcement, but presenting it
|
|
||||||
as a successful empty route table can cause a later replace-all proposal to
|
|
||||||
discard live routes.
|
|
||||||
3. The supervisor and Git HTTP services accept bounded declared body sizes but
|
|
||||||
use blocking reads and unbounded request threads. An untrusted bottle can
|
|
||||||
exhaust the shared gateway with slow or parallel requests.
|
|
||||||
4. macOS cleanup enumerates containers and networks independently and treats a
|
|
||||||
failed query as an empty class, so a partial snapshot can still become a
|
|
||||||
destructive plan.
|
|
||||||
5. Gateway log-pump threads race stream closure during shutdown and emit
|
|
||||||
uncaught exceptions even when shutdown otherwise succeeds.
|
|
||||||
6. Firecracker discovers VMs through whitespace-split `pgrep -a` output.
|
|
||||||
A configured cache path containing spaces can hide a live VM from the
|
|
||||||
snapshot and make its run directory appear orphaned.
|
|
||||||
7. Docker cleanup asks compose for its project snapshot in best-effort mode.
|
|
||||||
A transient query failure can therefore become an empty stopped-project
|
|
||||||
set and authorize deletion of associated state directories.
|
|
||||||
8. Firecracker artifact downloads and registry publication have no network
|
|
||||||
deadline, so an unresponsive registry can hold setup or release work
|
|
||||||
indefinitely.
|
|
||||||
9. Authenticated secret blobs select the unauthenticated legacy decoder when
|
|
||||||
their in-band version prefix is changed, allowing storage tampering to
|
|
||||||
bypass tag verification.
|
|
||||||
10. Cleanup executes the entire post-confirmation snapshot rather than the
|
|
||||||
intersection with what the operator saw, and mutation failures are not
|
|
||||||
reflected in the command result.
|
|
||||||
11. Git smart-HTTP can retain sixteen 100 MiB request bodies concurrently,
|
|
||||||
cleanup mutations have no subprocess deadline, and Firecracker signalling
|
|
||||||
failures bypass shared mutation accounting.
|
|
||||||
12. SQLite creates the shared control-plane database before its mode is
|
|
||||||
restricted, then suppresses permission-repair failures. Gateway transports
|
|
||||||
also differ in whether copied deploy-key modes are preserved.
|
|
||||||
|
|
||||||
These are one design problem: state used to authorize deletion, replacement,
|
|
||||||
or resource allocation must be authoritative at the point of use.
|
|
||||||
|
|
||||||
## Goals / Success Criteria
|
|
||||||
|
|
||||||
- Cleanup never signals a PID or recursively deletes a path solely because it
|
|
||||||
appeared in a pre-confirmation snapshot.
|
|
||||||
- Firecracker cleanup proves immediately before action that a PID is still the
|
|
||||||
same Firecracker process and that a run directory is still orphaned.
|
|
||||||
- Firecracker process discovery reads NUL-delimited argv from `/proc`; paths
|
|
||||||
are never reconstructed from whitespace-delimited process listings.
|
|
||||||
- All backend cleanup discovery primitives raise a typed enumeration error on
|
|
||||||
operational failure. No backend may independently continue from a partial
|
|
||||||
snapshot.
|
|
||||||
- Shared cleanup control flow lives in the backend layer; concrete backends
|
|
||||||
override resource-specific discovery and validation primitives rather than
|
|
||||||
each implementing a bespoke failure policy.
|
|
||||||
- `list-egress-routes` returns an MCP error when attribution or policy
|
|
||||||
resolution is unavailable. A genuine, authoritatively resolved empty policy
|
|
||||||
remains a successful empty list.
|
|
||||||
- Supervisor and Git HTTP request bodies have total read deadlines, and each
|
|
||||||
service bounds concurrent request work. Limits apply to authenticated
|
|
||||||
callers because bottles themselves are untrusted.
|
|
||||||
- Gateway child-output pumping treats expected stream closure during shutdown
|
|
||||||
as completion while preserving diagnostics for unexpected failures.
|
|
||||||
- Artifact pull, existence-check, and publication requests use explicit
|
|
||||||
network deadlines.
|
|
||||||
- Persisted secrets accept only the authenticated format. The schema migration
|
|
||||||
intentionally clears legacy rows; local agents are reprovisioned rather
|
|
||||||
than retaining a ciphertext-controlled downgrade path.
|
|
||||||
- Cleanup executes only resources present in both the displayed and current
|
|
||||||
authoritative plans, attempts every approved mutation, and returns failure
|
|
||||||
when any mutation does not complete.
|
|
||||||
- Git request bodies spool to disk behind a separate heavy-work semaphore;
|
|
||||||
cleanup commands have configurable deadlines; Firecracker signalling
|
|
||||||
failures aggregate while identity-verification uncertainty still aborts.
|
|
||||||
- The shared database directory and file are private before SQLite writes any
|
|
||||||
control-plane state; an inability to enforce those modes aborts startup.
|
|
||||||
- Gateway credential directories and files receive explicit private modes
|
|
||||||
inside the gateway, independent of Docker, Apple Container, or SSH copy
|
|
||||||
semantics.
|
|
||||||
- Unit tests cover PID/path reuse, partial backend enumeration, transient
|
|
||||||
policy resolution failure, slow bodies, concurrency saturation, and stream
|
|
||||||
closure races.
|
|
||||||
|
|
||||||
## Non-goals
|
|
||||||
|
|
||||||
- Further decomposition solely to reduce module line counts.
|
|
||||||
- Replacing gateway stdlib HTTP services with a web framework.
|
|
||||||
- Changing egress matching, DLP decisions, proposal semantics, or backend
|
|
||||||
launch behavior beyond the synchronization required for safe cleanup.
|
|
||||||
- Making cleanup silently skip uncertain resources. Uncertainty is an
|
|
||||||
operator-visible failure.
|
|
||||||
|
|
||||||
## Design
|
|
||||||
|
|
||||||
### Shared backend control flow
|
|
||||||
|
|
||||||
Follow the backend architecture rule used by gateway attachment: shared
|
|
||||||
behavior lives above concrete backends; subclasses provide primitives, not
|
|
||||||
control flow.
|
|
||||||
|
|
||||||
Cleanup remains previewable, but confirmation authorizes a *new authoritative
|
|
||||||
evaluation*, not blind execution of the displayed object. The shared flow:
|
|
||||||
|
|
||||||
1. asks each available backend for a preview;
|
|
||||||
2. displays the union and asks for confirmation;
|
|
||||||
3. refreshes each non-empty backend plan;
|
|
||||||
4. validates destructive identities immediately before action;
|
|
||||||
5. aborts loudly if the refreshed plan or any identity cannot be proven safe.
|
|
||||||
|
|
||||||
Backend-specific primitives define how to identify a resource. Firecracker
|
|
||||||
uses process start identity plus canonical config/run paths; container
|
|
||||||
backends use authoritative CLI queries and stable resource names/labels.
|
|
||||||
|
|
||||||
Container engines expose destructive name-based commands without a portable
|
|
||||||
compare-and-delete operation. Cleanup therefore refreshes after confirmation
|
|
||||||
and requires every discovery query to succeed, minimizing but not claiming to
|
|
||||||
eliminate the final name-reuse race. A future engine-specific stable-ID
|
|
||||||
primitive may close that residual window without moving control flow back
|
|
||||||
into each backend.
|
|
||||||
|
|
||||||
### Enforcement state versus introspection state
|
|
||||||
|
|
||||||
Egress enforcement retains its deny-all fallback because uncertainty must not
|
|
||||||
grant network access. Supervisor introspection uses a strict resolver path:
|
|
||||||
unattributed callers and resolver failures become typed MCP errors, while a
|
|
||||||
successfully resolved policy containing zero routes returns `routes: []`.
|
|
||||||
|
|
||||||
### Gateway resource boundaries
|
|
||||||
|
|
||||||
Both stdlib servers set a per-connection body deadline before reading and use a
|
|
||||||
bounded request executor or semaphore. Saturated capacity fails quickly with a
|
|
||||||
service-unavailable response. Existing size caps remain independent:
|
|
||||||
supervisor proposals retain the 1 MiB cap and Git pack requests retain their
|
|
||||||
larger protocol-appropriate cap.
|
|
||||||
|
|
||||||
### Shutdown diagnostics
|
|
||||||
|
|
||||||
The gateway output pump catches only stream-closure exceptions expected after
|
|
||||||
the supervisor closes child pipes. Other I/O failures remain visible and are
|
|
||||||
reported through the supervisor's normal diagnostic channel.
|
|
||||||
|
|
||||||
### Shared filesystem security
|
|
||||||
|
|
||||||
The common SQLite store owns database creation for every backend. It creates
|
|
||||||
the parent directory and an empty database with private modes before opening
|
|
||||||
SQLite, repairs existing modes, verifies the resulting state, and propagates
|
|
||||||
every enforcement failure. Backend launchers do not duplicate this policy.
|
|
||||||
|
|
||||||
The backend-neutral gateway provisioner likewise applies directory and file
|
|
||||||
modes after transport copies complete. This avoids relying on copy behavior
|
|
||||||
that differs among Docker, Apple Container, and Firecracker's SSH transport.
|
|
||||||
|
|
||||||
## Implementation chunks
|
|
||||||
|
|
||||||
1. Existing fail-closed security and backend enumeration fixes.
|
|
||||||
2. FastAPI orchestrator transport and bounded control-plane bodies.
|
|
||||||
3. Egress request-policy and outbound-DLP pipeline extraction.
|
|
||||||
4. Supervisor MCP dispatch extraction.
|
|
||||||
5. Shared cleanup refresh/revalidation plus authoritative macOS discovery.
|
|
||||||
6. Strict supervisor introspection and bounded supervisor/Git HTTP work.
|
|
||||||
7. Gateway shutdown log-pump closure handling.
|
|
||||||
8. Lossless Firecracker process identities, authoritative Docker cleanup
|
|
||||||
queries, and bounded Firecracker artifact transfers.
|
|
||||||
9. Mandatory authenticated secret storage, shared cleanup-plan intersection
|
|
||||||
and mutation accounting, and contained Git backend process failures.
|
|
||||||
10. Disk-spooled and separately bounded Git bodies, cleanup command deadlines,
|
|
||||||
and classified Firecracker signalling failures.
|
|
||||||
11. Fail-closed shared database creation and backend-neutral gateway credential
|
|
||||||
permissions.
|
|
||||||
|
|
||||||
## Open questions
|
|
||||||
|
|
||||||
None.
|
|
||||||
@@ -0,0 +1,273 @@
|
|||||||
|
# PRD prd-new: Host control server
|
||||||
|
|
||||||
|
- **Status:** Draft
|
||||||
|
- **Author:** Claude
|
||||||
|
- **Created:** 2026-07-26
|
||||||
|
- **Issue:** #468
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
Promote the in-process launch broker into a standalone **host control
|
||||||
|
server**: the single privileged component on the host. Both the CLI and the
|
||||||
|
orchestrator drive it over HTTP; it brokers agent launches, owns the
|
||||||
|
orchestrator's own lifecycle, and is the sole writer of host-durable state (the
|
||||||
|
tamper-evident audit record). This closes the three gaps between today's
|
||||||
|
well-formed broker *contract* ([`orchestrator/broker.py`](../../bot_bottle/orchestrator/broker.py))
|
||||||
|
and a real out-of-process service — transport, durable provisioned secret,
|
||||||
|
and a disciplined op vocabulary — and splits host state by
|
||||||
|
owner and lifetime. The prize: **the CLI no longer needs the Docker socket**,
|
||||||
|
which is what finally lets a dedicated Gitea runner user drop the
|
||||||
|
root-equivalent `docker` group (PRD 0070, "Relationship to other work").
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
Container launches run directly from a short-lived CLI process against the
|
||||||
|
Docker socket. That socket is root-equivalent, so every host that launches
|
||||||
|
bottles hands root to whoever invokes the CLI — including a CI runner user we
|
||||||
|
want to keep unprivileged. PRD 0070 already argues for replacing the fat socket
|
||||||
|
with a **thin, structured, auditable** launch broker, and the contract for that
|
||||||
|
broker exists and is tested in-process. But it is *only* in-process:
|
||||||
|
`LaunchBroker.submit(token)` is a method call from
|
||||||
|
`OrchestratorCore.launch_bottle` ([`service.py:116`](../../bot_bottle/orchestrator/service.py)),
|
||||||
|
and `DockerBroker` is on no production path — every backend starts the
|
||||||
|
orchestrator with `--broker stub` ([`__main__.py:54`](../../bot_bottle/orchestrator/__main__.py)).
|
||||||
|
|
||||||
|
Three gaps stand between that scaffold and a host service:
|
||||||
|
|
||||||
|
1. **No transport.** `submit` is an in-process call. A real service needs a
|
||||||
|
`BrokerClient` that POSTs the signed token and a host-side HTTP server that
|
||||||
|
verifies and acts.
|
||||||
|
2. **The signing secret is ephemeral and self-generated.**
|
||||||
|
[`__main__.py:53`](../../bot_bottle/orchestrator/__main__.py) does
|
||||||
|
`secrets.token_bytes(32)` and hands the *same value* to signer and verifier —
|
||||||
|
viable only because they share a process. A separate daemon needs the secret
|
||||||
|
provisioned out of band and durable across orchestrator restarts.
|
||||||
|
3. **The op vocabulary is `launch` / `teardown` only.** Everything else
|
||||||
|
host-privileged still lives in the CLI, so the schema has to grow — carefully,
|
||||||
|
since PRD 0070's security argument rests on "structured requests only, static
|
||||||
|
flags + ids."
|
||||||
|
|
||||||
|
Separately, host state has no clear owner. `OrchestratorCore.reconcile` takes
|
||||||
|
`live_source_ips` as a parameter *only because the orchestrator cannot see the
|
||||||
|
backend* ([`service.py:137`](../../bot_bottle/orchestrator/service.py)); the
|
||||||
|
egress traffic log is written to the container's stderr; and there is no durable,
|
||||||
|
tamper-evident home for the audit record that survives orchestrator destruction.
|
||||||
|
|
||||||
|
## Goals / Success Criteria
|
||||||
|
|
||||||
|
- A standalone host control server that the CLI and orchestrator reach over
|
||||||
|
**HTTP**, with three entry paths working end to end:
|
||||||
|
- `web console -(iroh)-> orchestrator -(http)-> host controller -> launch`
|
||||||
|
- `cli -(http)-> orchestrator -(http)-> host controller -> launch`
|
||||||
|
- `cli -(http)-> host controller` — start / restart / status of the
|
||||||
|
orchestrator **itself** (the bootstrap/recovery path #391 targets).
|
||||||
|
- The launch op is expressed as a **signed JWT of static flags + ids only**,
|
||||||
|
verified against a closed schema.
|
||||||
|
- The signing secret is **provisioned out of band and durable** across
|
||||||
|
orchestrator restarts (a `TrustDomain` per #476, with a key the orchestrator
|
||||||
|
never holds for the host controller's *own* endpoints).
|
||||||
|
- Host-privileged operations move off the CLI to the control server; **the CLI
|
||||||
|
no longer opens the Docker socket** for bottle operations.
|
||||||
|
- `Orchestrator.reconcile` no longer takes `live_source_ips` — live-bottle
|
||||||
|
enumeration becomes an internal control-server call.
|
||||||
|
- Host-durable state lands as an **append-only, hash-chained JSONL** audit log
|
||||||
|
owned solely by the host controller; operational state stays SQLite owned
|
||||||
|
solely by the orchestrator.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- **Removing standing privilege.** This converts on-demand privilege (a CLI the
|
||||||
|
user invokes) into standing privilege (a daemon under launchd/systemd). The
|
||||||
|
win is that the privilege is *narrower* (structured requests vs. a raw socket),
|
||||||
|
not that it disappears. "Always running" is an accepted new property.
|
||||||
|
- **Asymmetric signing.** We stay HS256 — see Design / "Signing stays
|
||||||
|
symmetric."
|
||||||
|
- **Integrity against a live compromised orchestrator.** Host-location of the
|
||||||
|
audit log does not buy this: the orchestrator makes the decisions being audited
|
||||||
|
and can forge or omit entries wherever the file lives. An off-box copy is the
|
||||||
|
answer, tracked separately.
|
||||||
|
- **A single unified DB for all state.** Impossible over a guest-kernel share
|
||||||
|
(SQLite locking is not coherent); state is split by owner and lifetime instead.
|
||||||
|
- **The generic `SecretProvider` (#355)** and **remote terminal design (#478)** —
|
||||||
|
both ride the same door but are their own work.
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
### Topology
|
||||||
|
|
||||||
|
The host controller is the sole privileged component. The orchestrator becomes a
|
||||||
|
client of it for launches, and the CLI becomes a client of it for *both* bottle
|
||||||
|
operations (indirectly, through the orchestrator) and orchestrator lifecycle
|
||||||
|
(directly, for bootstrap/recovery — startup can't route through the thing being
|
||||||
|
started).
|
||||||
|
|
||||||
|
```
|
||||||
|
web console ─(iroh)─▶ orchestrator ─┐
|
||||||
|
├─(http, signed JWT)─▶ host controller ─▶ launch
|
||||||
|
cli ────────(http)──▶ orchestrator ─┘
|
||||||
|
cli ────────(http, bearer)──────────────────────────────▶ host controller (orchestrator lifecycle)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Transport: `BrokerClient` + host server
|
||||||
|
|
||||||
|
`LaunchBroker.submit(token)` keeps its exact signature and semantics; only the
|
||||||
|
*wire* changes. A new `BrokerClient` implements the same submit contract by
|
||||||
|
POSTing the signed token to the host controller (stdlib `urllib`, like the
|
||||||
|
existing [`orchestrator/client.py`](../../bot_bottle/orchestrator/client.py)),
|
||||||
|
and the host controller's launch handler is the existing `verify_request` +
|
||||||
|
`_launch`/`_teardown` path, now reached over HTTP instead of a method call. The
|
||||||
|
in-process `StubBroker` stays for the dev-harness and tests; `DockerBroker`'s
|
||||||
|
`_launch`/`_teardown` bodies move behind the server unchanged. Because the client
|
||||||
|
satisfies the same interface `OrchestratorCore` already depends on, the core does
|
||||||
|
not change to gain a real backend.
|
||||||
|
|
||||||
|
### Signing stays symmetric (HS256)
|
||||||
|
|
||||||
|
PRD 0070 nominally specifies asymmetric; the code is HS256 and we keep it.
|
||||||
|
Asymmetric matters when the verifier is *less* privileged than the signer — here
|
||||||
|
it is the reverse: the host controller (verifier) is strictly more privileged
|
||||||
|
than the orchestrator (signer), and a controller that could forge orchestrator
|
||||||
|
requests gains nothing, since it is already the component that launches. Staying
|
||||||
|
symmetric also honors the no-runtime-deps policy (stdlib has no Ed25519). This
|
||||||
|
matches the reasoning already inlined in `broker.py`'s module docstring.
|
||||||
|
|
||||||
|
### Replay protection is out of scope (tracked in #494)
|
||||||
|
|
||||||
|
Once the launch token travels over a wire, a captured token could be replayed —
|
||||||
|
`sign_request` already emits `jti`/`iat` but `verify_request` reads neither, so
|
||||||
|
there is no expiry window or `jti` cache today. Enforcing that (an `iat` window +
|
||||||
|
a self-trimming `jti` cache) is a pure in-process change that lands independently
|
||||||
|
of this work, and it is deferred to **#494** rather than gating the MVP of the
|
||||||
|
host control server. Nothing here depends on it; it can merge before or after.
|
||||||
|
|
||||||
|
### Op vocabulary and the "ids + static flags" rule (gap 3)
|
||||||
|
|
||||||
|
Each op moved off the CLI widens the privileged surface, so growth is governed by
|
||||||
|
one explicit rule, enforced in `verify_request`'s schema check:
|
||||||
|
|
||||||
|
> A broker op carries **only ids and enumerated static flags** — a bottle id, a
|
||||||
|
> pool slot, a **content-addressed** image ref chosen from a fixed set, an op
|
||||||
|
> name from a closed vocabulary. Never a free-form path, argv, command, or
|
||||||
|
> caller-supplied filesystem location. If an operation cannot be expressed that
|
||||||
|
> way, it does not become a broker op.
|
||||||
|
|
||||||
|
Operations that fit and move off the CLI (all today in
|
||||||
|
`backend/*/consolidated_launch.py`, driven by a short-lived CLI process):
|
||||||
|
|
||||||
|
| Op | What it does | Fits the rule because |
|
||||||
|
|---|---|---|
|
||||||
|
| `launch` / `teardown` | existing | ids + slot + image ref |
|
||||||
|
| `orchestrator.ensure_running` | start the infra container | no arguments |
|
||||||
|
| `orchestrator.{start,restart,status}` | lifecycle (the #391 path) | no arguments |
|
||||||
|
| `list_live` | enumerate running bottles for reconcile | no arguments; returns ids/IPs |
|
||||||
|
| `allocate_ip` | `next_free_ip` over `_network_container_ips` | no arguments; returns an IP |
|
||||||
|
| `provision_git_gate` | `cp`/`exec` a per-bottle deploy key into the gateway | bottle id + key handle, no path |
|
||||||
|
| `reprovision` | `docker exec printenv <ENV_VAR_SECRET>` on a live agent | bottle id + secret *name* |
|
||||||
|
|
||||||
|
Image **builds** stay with the orchestrator for v1 (PRD 0070 §Memory: builds run
|
||||||
|
control-plane-side; a dedicated slim build unit is later, #468-adjacent), so no
|
||||||
|
`build` broker op is added here.
|
||||||
|
|
||||||
|
With `list_live` as an internal control-server call, `Orchestrator.reconcile`'s
|
||||||
|
`live_source_ips` parameter goes away — the tell PRD 0070 called out that the
|
||||||
|
orchestrator couldn't see the backend disappears with it.
|
||||||
|
|
||||||
|
### Secret provisioning (gap 2)
|
||||||
|
|
||||||
|
The shared HS256 secret becomes a durable, out-of-band artifact via the
|
||||||
|
**`TrustDomain`** seam (#476,
|
||||||
|
[`trust_domain.py`](../../bot_bottle/trust_domain.py)):
|
||||||
|
|
||||||
|
- The **launch-broker secret** is a `TrustDomain` whose key
|
||||||
|
(`host_signing_key(<file>)`, minted 0600 on first use, durable under
|
||||||
|
`bot_bottle_root()`) is provisioned to the orchestrator (signer) and the host
|
||||||
|
controller (verifier). Durability across orchestrator restarts is what makes
|
||||||
|
re-adoption work — a restart re-verifies against the same key.
|
||||||
|
- The **host controller's own lifecycle endpoints** (the direct `cli -> host
|
||||||
|
controller` path) get a **separate** `TrustDomain` key the orchestrator never
|
||||||
|
holds — exactly the second domain #476's PRD reserves. The orchestrator must
|
||||||
|
not be able to mint the credentials used to start and stop it.
|
||||||
|
|
||||||
|
This reuses the seam #476 landed rather than re-deriving provisioning per
|
||||||
|
backend (the PR #471 bug class).
|
||||||
|
|
||||||
|
### One daemon, structurally separate handlers (open decision 1)
|
||||||
|
|
||||||
|
The audit writer and the broker live in **one daemon** for install simplicity,
|
||||||
|
but with **no shared parsing** and **different credentials per handler**:
|
||||||
|
|
||||||
|
- the **launch** handler requires the signed launch **JWT** (provenance +
|
||||||
|
un-coercible schema);
|
||||||
|
- the **audit-append** handler takes a plain **bearer token** and writes to the
|
||||||
|
JSONL log.
|
||||||
|
|
||||||
|
This does not defend against orchestrator compromise (it holds both creds) — it
|
||||||
|
stops a bug in the boring audit path from reaching the privileged launch path.
|
||||||
|
The launcher stays small enough to audit line-by-line, per PRD 0070.
|
||||||
|
|
||||||
|
### State ownership: split by owner and lifetime
|
||||||
|
|
||||||
|
A single mounted DB is impossible — SQLite locking is not coherent across guest
|
||||||
|
kernels over a share, which is why the macOS backend already uses a container-only
|
||||||
|
volume (`INFRA_DB_VOLUME`). So state splits three ways (depends on #469, which
|
||||||
|
gets `bot-bottle.db` off the data plane first):
|
||||||
|
|
||||||
|
| Owner | State | Home | Shape |
|
||||||
|
|---|---|---|---|
|
||||||
|
| **Orchestrator** | `orchestrator_bottles` registry; `bottled_agent_secrets` (encrypted egress tokens); `supervise_proposals` / `supervise_responses` | volume nothing else mounts (generalizing the macOS design) | **SQLite** — mutable, transactional, queried |
|
||||||
|
| **Host controller** | supervise audit entries; egress traffic log (today → container stderr); host-side config | host filesystem, survives orchestrator/volume destruction | **JSONL** — append-only |
|
||||||
|
| **Gateway** | none | — | after #469 the data plane holds no DB state |
|
||||||
|
|
||||||
|
The historical record is **JSONL, not SQLite**, because it is append-only, never
|
||||||
|
updated, never transactionally queried: `O_APPEND` writes are atomic, there is no
|
||||||
|
locking protocol to get wrong, hash-chaining for tamper-evidence is cheap, and it
|
||||||
|
survives container-runtime volume pruning (the #450 lesson) and stays readable
|
||||||
|
without the orchestrator running. Both halves of "the audit record" — supervise
|
||||||
|
decisions and the egress traffic log — land in the one place.
|
||||||
|
|
||||||
|
The orchestrator is **sole mounter and sole writer** of its SQLite volume; the
|
||||||
|
host controller is **sole writer** of the JSONL log, over the authenticated
|
||||||
|
audit-append channel.
|
||||||
|
|
||||||
|
## Implementation chunks
|
||||||
|
|
||||||
|
Ordered, each independently mergeable:
|
||||||
|
|
||||||
|
1. **`BrokerClient` + host launch server** over HTTP, reusing `verify_request`
|
||||||
|
and the existing `DockerBroker` bodies. Wire `OrchestratorCore` to a
|
||||||
|
`BrokerClient` behind a flag; keep `StubBroker` for the dev-harness. Closes
|
||||||
|
gap 1.
|
||||||
|
2. **Durable secret via `TrustDomain`** — provision the launch-broker key to
|
||||||
|
signer + verifier; add the host controller's own lifecycle `TrustDomain`.
|
||||||
|
Closes gap 2.
|
||||||
|
3. **Grow the op vocabulary** one op at a time (`list_live` first — it also
|
||||||
|
removes `reconcile`'s `live_source_ips`), each behind the ids + static-flags
|
||||||
|
rule. Closes gap 3.
|
||||||
|
4. **JSONL audit log** — the host-controller-owned, hash-chained historical
|
||||||
|
record with the plain-bearer audit-append handler; redirect the egress traffic
|
||||||
|
log into it.
|
||||||
|
5. **Drop the Docker socket from the CLI** once every host-privileged op it used
|
||||||
|
is a broker op — the payoff that unblocks the unprivileged Gitea runner user.
|
||||||
|
|
||||||
|
## Open questions
|
||||||
|
|
||||||
|
1. **Schema-width rule enforcement.** The "ids + static flags" rule is stated;
|
||||||
|
should `verify_request` reject unknown claim keys outright (strict schema) to
|
||||||
|
keep the surface from drifting? Leaning yes.
|
||||||
|
2. **Audit-append back-pressure.** What the audit handler does if the JSONL sink
|
||||||
|
is unavailable (fail-closed vs. buffer) — resolve before shipping chunk 5.
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- **PRD 0070** — the contract, the launch broker, and the state tiers this
|
||||||
|
implements.
|
||||||
|
- **#469** — get `bot-bottle.db` off the data plane (lands underneath this).
|
||||||
|
- **#476** ([`prd-new-control-plane-auth-provisioning`](prd-new-control-plane-auth-provisioning.md))
|
||||||
|
— the `TrustDomain` seam this plugs the host controller's key into.
|
||||||
|
- **#391** — backend-agnostic orchestrator restart (the bootstrap path).
|
||||||
|
- **#494** — enforce broker replay protection (`iat` window + `jti` cache); split
|
||||||
|
out of this PRD as an independent in-process change.
|
||||||
|
- **#386** — prebuilt images from the Gitea OCI registry (the fixed image set the
|
||||||
|
broker validates against).
|
||||||
|
- **#355** — generic `SecretProvider`.
|
||||||
|
- **#478** — remote terminal design.
|
||||||
+5
-5
@@ -13,7 +13,7 @@
|
|||||||
# are re-executed; no KVM or Docker dependency.
|
# are re-executed; no KVM or Docker dependency.
|
||||||
#
|
#
|
||||||
# Pass "critical" as the last argument in either mode to also report just the
|
# Pass "critical" as the last argument in either mode to also report just the
|
||||||
# critical modules (ADR 0004 target: 85%).
|
# critical modules (ADR 0004 target: 90%).
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
cd "$(dirname "$0")/.."
|
cd "$(dirname "$0")/.."
|
||||||
@@ -34,8 +34,8 @@ if [ "${1:-}" = "aggregate" ]; then
|
|||||||
"$PY" -m coverage report -m
|
"$PY" -m coverage report -m
|
||||||
|
|
||||||
if [ "${2:-}" = "critical" ]; then
|
if [ "${2:-}" = "critical" ]; then
|
||||||
echo "== critical modules (ADR 0004 minimum: 85%) ==" >&2
|
echo "== critical modules (ADR 0004 minimum: 90%) ==" >&2
|
||||||
"$PY" -m coverage report --include="$CRITICAL" --fail-under=85
|
"$PY" -m coverage report --include="$CRITICAL" --fail-under=90
|
||||||
fi
|
fi
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
@@ -55,6 +55,6 @@ echo "== combined report ==" >&2
|
|||||||
"$PY" -m coverage report -m
|
"$PY" -m coverage report -m
|
||||||
|
|
||||||
if [ "${1:-}" = "critical" ]; then
|
if [ "${1:-}" = "critical" ]; then
|
||||||
echo "== critical modules (ADR 0004 minimum: 85%) ==" >&2
|
echo "== critical modules (ADR 0004 minimum: 90%) ==" >&2
|
||||||
"$PY" -m coverage report --include="$CRITICAL" --fail-under=85
|
"$PY" -m coverage report --include="$CRITICAL" --fail-under=90
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
# Critical security/logic core held to the >=85% coverage bar by
|
# Critical security/logic core held to the >=90% coverage bar by
|
||||||
# docs/decisions/0004-coverage-policy.md.
|
# docs/decisions/0004-coverage-policy.md.
|
||||||
#
|
#
|
||||||
# SINGLE SOURCE OF TRUTH: scripts/coverage.sh (the `critical` report) and
|
# SINGLE SOURCE OF TRUTH: scripts/coverage.sh (the `critical` report) and
|
||||||
|
|||||||
@@ -13,8 +13,8 @@ policy.
|
|||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
scripts/coverage.sh # produce .coverage first
|
scripts/coverage.sh # produce .coverage first
|
||||||
python3 scripts/diff_coverage.py # gate against origin/main, min 80%
|
python3 scripts/diff_coverage.py # gate against origin/main, min 90%
|
||||||
python3 scripts/diff_coverage.py --base main --min 75
|
python3 scripts/diff_coverage.py --base main --min 85
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -74,7 +74,7 @@ def main() -> int:
|
|||||||
ap = argparse.ArgumentParser()
|
ap = argparse.ArgumentParser()
|
||||||
ap.add_argument("--base", default="origin/main",
|
ap.add_argument("--base", default="origin/main",
|
||||||
help="git ref to diff against (default: origin/main)")
|
help="git ref to diff against (default: origin/main)")
|
||||||
ap.add_argument("--min", type=float, default=80.0,
|
ap.add_argument("--min", type=float, default=90.0,
|
||||||
help="minimum %% of changed executable lines covered")
|
help="minimum %% of changed executable lines covered")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ from bot_bottle.backend.docker.compose import (
|
|||||||
list_compose_projects,
|
list_compose_projects,
|
||||||
slug_from_compose_project,
|
slug_from_compose_project,
|
||||||
)
|
)
|
||||||
from bot_bottle.backend import EnumerationError
|
|
||||||
|
|
||||||
|
|
||||||
class TestProjectNaming(unittest.TestCase):
|
class TestProjectNaming(unittest.TestCase):
|
||||||
@@ -70,19 +69,6 @@ class TestComposeProjectListing(unittest.TestCase):
|
|||||||
self.assertEqual([], list_active_slugs(warn_on_error=False))
|
self.assertEqual([], list_active_slugs(warn_on_error=False))
|
||||||
warn.assert_not_called()
|
warn.assert_not_called()
|
||||||
|
|
||||||
def test_compose_ls_error_can_be_raised_for_enumeration(self):
|
|
||||||
with mock.patch(
|
|
||||||
"bot_bottle.backend.docker.compose.subprocess.run",
|
|
||||||
return_value=subprocess.CompletedProcess(
|
|
||||||
args=["docker"], returncode=1, stdout="", stderr="no daemon",
|
|
||||||
),
|
|
||||||
):
|
|
||||||
with self.assertRaisesRegex(EnumerationError, "no daemon"):
|
|
||||||
list_active_slugs(
|
|
||||||
warn_on_error=False,
|
|
||||||
raise_on_error=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -7,8 +7,6 @@ from pathlib import Path
|
|||||||
from unittest.mock import MagicMock, Mock, patch
|
from unittest.mock import MagicMock, Mock, patch
|
||||||
|
|
||||||
from bot_bottle.backend.docker.consolidated_launch import (
|
from bot_bottle.backend.docker.consolidated_launch import (
|
||||||
ConsolidatedLaunchError,
|
|
||||||
_network_container_ips,
|
|
||||||
launch_consolidated,
|
launch_consolidated,
|
||||||
deprovision_consolidated,
|
deprovision_consolidated,
|
||||||
)
|
)
|
||||||
@@ -88,16 +86,6 @@ class TestLaunchConsolidated(unittest.TestCase):
|
|||||||
client.teardown_bottle.assert_called_once_with("b1") # no orphan left
|
client.teardown_bottle.assert_called_once_with("b1") # no orphan left
|
||||||
|
|
||||||
|
|
||||||
class TestNetworkContainerIps(unittest.TestCase):
|
|
||||||
def test_fails_closed_when_network_inspection_fails(self) -> None:
|
|
||||||
result = Mock(returncode=1, stdout="", stderr="daemon unavailable")
|
|
||||||
with (
|
|
||||||
patch(f"{_MOD}.run_docker", return_value=result),
|
|
||||||
self.assertRaisesRegex(ConsolidatedLaunchError, "daemon unavailable"),
|
|
||||||
):
|
|
||||||
_network_container_ips("bot-bottle-gateway")
|
|
||||||
|
|
||||||
|
|
||||||
class TestTeardownConsolidated(unittest.TestCase):
|
class TestTeardownConsolidated(unittest.TestCase):
|
||||||
def test_deregisters_and_deprovisions(self) -> None:
|
def test_deregisters_and_deprovisions(self) -> None:
|
||||||
client = Mock()
|
client = Mock()
|
||||||
|
|||||||
@@ -19,16 +19,13 @@ of issue #77 — the dashboard now delegates to this layer.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import subprocess
|
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
from tests.unit import use_bottle_root
|
from tests.unit import use_bottle_root
|
||||||
from bot_bottle import bottle_state
|
from bot_bottle import bottle_state
|
||||||
from bot_bottle.backend.docker import enumerate as _enumerate
|
from bot_bottle.backend.docker import enumerate as _enumerate
|
||||||
from bot_bottle.backend import EnumerationError
|
|
||||||
|
|
||||||
|
|
||||||
class TestParseServicesByProject(unittest.TestCase):
|
class TestParseServicesByProject(unittest.TestCase):
|
||||||
@@ -75,18 +72,6 @@ class TestParseServicesByProject(unittest.TestCase):
|
|||||||
self.assertEqual({"bot-bottle-dev-abc": {"egress"}}, out)
|
self.assertEqual({"bot-bottle-dev-abc": {"egress"}}, out)
|
||||||
|
|
||||||
|
|
||||||
class TestQueryServicesByProject(unittest.TestCase):
|
|
||||||
def test_docker_ps_failure_is_not_an_empty_result(self):
|
|
||||||
with patch(
|
|
||||||
"bot_bottle.backend.docker.enumerate.subprocess.run",
|
|
||||||
return_value=subprocess.CompletedProcess(
|
|
||||||
args=["docker"], returncode=1, stdout="", stderr="daemon down",
|
|
||||||
),
|
|
||||||
):
|
|
||||||
with self.assertRaisesRegex(EnumerationError, "daemon down"):
|
|
||||||
_enumerate._query_services_by_project()
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeHomeMixin:
|
class _FakeHomeMixin:
|
||||||
def _setup_fake_home(self) -> None:
|
def _setup_fake_home(self) -> None:
|
||||||
self._tmp = tempfile.TemporaryDirectory(prefix="enum-active.")
|
self._tmp = tempfile.TemporaryDirectory(prefix="enum-active.")
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ import unittest
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from bot_bottle.backend import EnumerationError
|
|
||||||
from bot_bottle.backend.firecracker import cleanup as fc_cleanup
|
from bot_bottle.backend.firecracker import cleanup as fc_cleanup
|
||||||
from bot_bottle.backend.firecracker.bottle_cleanup_plan import (
|
from bot_bottle.backend.firecracker.bottle_cleanup_plan import (
|
||||||
FirecrackerBottleCleanupPlan,
|
FirecrackerBottleCleanupPlan,
|
||||||
@@ -59,37 +58,10 @@ class TestProcessScan(unittest.TestCase):
|
|||||||
self.assertEqual({str(run_root / "live-a")}, live)
|
self.assertEqual({str(run_root / "live-a")}, live)
|
||||||
self.assertEqual([222], orphan_pids)
|
self.assertEqual([222], orphan_pids)
|
||||||
|
|
||||||
def test_scan_empty_when_pgrep_finds_no_processes(self):
|
def test_scan_empty_when_pgrep_fails(self):
|
||||||
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(returncode=1)):
|
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(returncode=1)):
|
||||||
self.assertEqual((set(), []), fc_cleanup._scan_processes(Path("/x")))
|
self.assertEqual((set(), []), fc_cleanup._scan_processes(Path("/x")))
|
||||||
|
|
||||||
def test_scan_raises_when_pgrep_errors(self):
|
|
||||||
proc = subprocess.CompletedProcess(
|
|
||||||
[], 2, stdout="", stderr="invalid process expression",
|
|
||||||
)
|
|
||||||
with (
|
|
||||||
patch.object(fc_cleanup.subprocess, "run", return_value=proc),
|
|
||||||
self.assertRaisesRegex(EnumerationError, "invalid process expression"),
|
|
||||||
):
|
|
||||||
fc_cleanup._scan_processes(Path("/x"))
|
|
||||||
|
|
||||||
def test_prepare_cleanup_does_not_plan_deletions_when_scan_errors(self):
|
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
|
||||||
run_root = Path(tmp)
|
|
||||||
live = run_root / "live-a"
|
|
||||||
live.mkdir()
|
|
||||||
with (
|
|
||||||
patch.object(fc_cleanup, "_run_root", return_value=run_root),
|
|
||||||
patch.object(
|
|
||||||
fc_cleanup.subprocess,
|
|
||||||
"run",
|
|
||||||
return_value=_proc(returncode=2),
|
|
||||||
),
|
|
||||||
self.assertRaises(EnumerationError),
|
|
||||||
):
|
|
||||||
fc_cleanup.prepare_cleanup()
|
|
||||||
self.assertTrue(live.is_dir())
|
|
||||||
|
|
||||||
def test_live_run_dirs_returns_paths_in_stable_order(self):
|
def test_live_run_dirs_returns_paths_in_stable_order(self):
|
||||||
with patch.object(fc_cleanup, "_run_root", return_value=Path("/run")), \
|
with patch.object(fc_cleanup, "_run_root", return_value=Path("/run")), \
|
||||||
patch.object(fc_cleanup, "_scan_processes",
|
patch.object(fc_cleanup, "_scan_processes",
|
||||||
|
|||||||
@@ -1,64 +0,0 @@
|
|||||||
"""Unit tests for Firecracker active-agent enumeration."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import unittest
|
|
||||||
from pathlib import Path
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
from bot_bottle.backend import EnumerationError
|
|
||||||
from bot_bottle.backend.firecracker import enumerate as fc_enumerate
|
|
||||||
from bot_bottle.bottle_state import BottleMetadata
|
|
||||||
|
|
||||||
|
|
||||||
class TestEnumerateActive(unittest.TestCase):
|
|
||||||
def test_maps_live_run_dirs_to_active_agents(self) -> None:
|
|
||||||
metadata = BottleMetadata(
|
|
||||||
identity="dev-a",
|
|
||||||
agent_name="claude",
|
|
||||||
cwd="",
|
|
||||||
copy_cwd=False,
|
|
||||||
started_at="2026-07-26T12:00:00Z",
|
|
||||||
label="review",
|
|
||||||
color="blue",
|
|
||||||
)
|
|
||||||
with (
|
|
||||||
patch.object(
|
|
||||||
fc_enumerate, "live_run_dirs",
|
|
||||||
return_value=(Path("/cache/run/dev-a"),),
|
|
||||||
),
|
|
||||||
patch.object(fc_enumerate, "read_metadata", return_value=metadata),
|
|
||||||
):
|
|
||||||
agents = fc_enumerate.enumerate_active()
|
|
||||||
self.assertEqual(1, len(agents))
|
|
||||||
self.assertEqual("firecracker", agents[0].backend_name)
|
|
||||||
self.assertEqual("dev-a", agents[0].slug)
|
|
||||||
self.assertEqual("claude", agents[0].agent_name)
|
|
||||||
self.assertEqual("review", agents[0].label)
|
|
||||||
self.assertEqual((), agents[0].services)
|
|
||||||
|
|
||||||
def test_missing_metadata_uses_safe_defaults(self) -> None:
|
|
||||||
with (
|
|
||||||
patch.object(
|
|
||||||
fc_enumerate, "live_run_dirs",
|
|
||||||
return_value=(Path("/cache/run/dev-a"),),
|
|
||||||
),
|
|
||||||
patch.object(fc_enumerate, "read_metadata", return_value=None),
|
|
||||||
):
|
|
||||||
agent = fc_enumerate.enumerate_active()[0]
|
|
||||||
self.assertEqual("?", agent.agent_name)
|
|
||||||
self.assertEqual("", agent.started_at)
|
|
||||||
|
|
||||||
def test_process_scan_failure_propagates(self) -> None:
|
|
||||||
with (
|
|
||||||
patch.object(
|
|
||||||
fc_enumerate, "live_run_dirs",
|
|
||||||
side_effect=EnumerationError("pgrep failed"),
|
|
||||||
),
|
|
||||||
self.assertRaisesRegex(EnumerationError, "pgrep failed"),
|
|
||||||
):
|
|
||||||
fc_enumerate.enumerate_active()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -5,7 +5,6 @@ from __future__ import annotations
|
|||||||
import unittest
|
import unittest
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from bot_bottle.backend import EnumerationError
|
|
||||||
from bot_bottle.backend.macos_container import cleanup, enumerate as enum_mod
|
from bot_bottle.backend.macos_container import cleanup, enumerate as enum_mod
|
||||||
from bot_bottle.backend.macos_container.bottle_cleanup_plan import (
|
from bot_bottle.backend.macos_container.bottle_cleanup_plan import (
|
||||||
MacosContainerBottleCleanupPlan,
|
MacosContainerBottleCleanupPlan,
|
||||||
@@ -70,18 +69,10 @@ class TestMacosContainerEnumerate(unittest.TestCase):
|
|||||||
self.assertEqual(["dev-abc"], [a.slug for a in agents])
|
self.assertEqual(["dev-abc"], [a.slug for a in agents])
|
||||||
|
|
||||||
def test_raises_when_the_cli_fails(self):
|
def test_raises_when_the_cli_fails(self):
|
||||||
|
from bot_bottle.backend.macos_container.enumerate import EnumerationError
|
||||||
with self.assertRaises(EnumerationError):
|
with self.assertRaises(EnumerationError):
|
||||||
self._enumerate("", returncode=1)
|
self._enumerate("", returncode=1)
|
||||||
|
|
||||||
def test_raises_typed_error_when_cli_is_missing(self):
|
|
||||||
with (
|
|
||||||
patch.object(
|
|
||||||
enum_mod.subprocess, "run", side_effect=FileNotFoundError,
|
|
||||||
),
|
|
||||||
self.assertRaisesRegex(EnumerationError, "CLI not found"),
|
|
||||||
):
|
|
||||||
enum_mod.enumerate_active()
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -0,0 +1,117 @@
|
|||||||
|
"""Unit: orchestrator-side broker client (issue #468, chunk 1). HTTP mocked."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import unittest
|
||||||
|
import urllib.error
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from bot_bottle.orchestrator.broker import (
|
||||||
|
BrokerAuthError,
|
||||||
|
BrokerUnavailableError,
|
||||||
|
LaunchRequest,
|
||||||
|
)
|
||||||
|
from bot_bottle.orchestrator.broker_client import BrokerClient, BrokerClientError
|
||||||
|
|
||||||
|
_URLOPEN = "bot_bottle.orchestrator.broker_client.urllib.request.urlopen"
|
||||||
|
|
||||||
|
|
||||||
|
def _resp(payload: object) -> MagicMock:
|
||||||
|
m = MagicMock()
|
||||||
|
m.__enter__.return_value.read.return_value = json.dumps(payload).encode()
|
||||||
|
return m
|
||||||
|
|
||||||
|
|
||||||
|
def _http_error(code: int, payload: object = None) -> urllib.error.HTTPError:
|
||||||
|
body = json.dumps(payload).encode() if payload is not None else b""
|
||||||
|
return urllib.error.HTTPError(
|
||||||
|
"http://host/broker", code, "err", {}, io.BytesIO(body)) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
class TestSubmit(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.c = BrokerClient("http://host:8091")
|
||||||
|
|
||||||
|
def test_returns_the_verified_request(self) -> None:
|
||||||
|
echo = {
|
||||||
|
"op": "launch", "bottle_id": "b1", "source_ip": "10.0.0.1",
|
||||||
|
"image_ref": "img", "slot": 3,
|
||||||
|
}
|
||||||
|
with patch(_URLOPEN, return_value=_resp(echo)):
|
||||||
|
got = self.c.submit("tok")
|
||||||
|
self.assertEqual(
|
||||||
|
LaunchRequest(op="launch", bottle_id="b1", source_ip="10.0.0.1",
|
||||||
|
image_ref="img", slot=3),
|
||||||
|
got,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_posts_token_to_broker_endpoint(self) -> None:
|
||||||
|
with patch(_URLOPEN, return_value=_resp({"op": "teardown", "bottle_id": "b1"})) as m:
|
||||||
|
self.c.submit("signed-token")
|
||||||
|
request = m.call_args.args[0]
|
||||||
|
self.assertEqual("POST", request.get_method())
|
||||||
|
self.assertTrue(request.full_url.endswith("/broker"))
|
||||||
|
self.assertEqual({"token": "signed-token"}, json.loads(request.data))
|
||||||
|
|
||||||
|
def test_401_raises_broker_auth_error(self) -> None:
|
||||||
|
# A fail-closed provenance/schema rejection surfaces as the SAME exception
|
||||||
|
# the in-process broker raises, so the launch path's rollback is identical.
|
||||||
|
with patch(_URLOPEN, side_effect=_http_error(401, {"error": "bad signature"})):
|
||||||
|
with self.assertRaises(BrokerAuthError):
|
||||||
|
self.c.submit("forged")
|
||||||
|
|
||||||
|
def test_502_is_a_definite_client_error(self) -> None:
|
||||||
|
# The host responded — it processed the request and did not launch, so a
|
||||||
|
# definite BrokerClientError (the caller may safely roll back).
|
||||||
|
with patch(_URLOPEN, side_effect=_http_error(502, {"error": "docker down"})):
|
||||||
|
with self.assertRaises(BrokerClientError):
|
||||||
|
self.c.submit("tok")
|
||||||
|
|
||||||
|
def test_unreachable_is_ambiguous_unavailable(self) -> None:
|
||||||
|
# No response at all — the request may already have launched, so the
|
||||||
|
# AMBIGUOUS BrokerUnavailableError (the caller must NOT roll back).
|
||||||
|
with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
|
||||||
|
with self.assertRaises(BrokerUnavailableError):
|
||||||
|
self.c.submit("tok")
|
||||||
|
|
||||||
|
def test_timeout_is_ambiguous_unavailable(self) -> None:
|
||||||
|
# A dropped/late response after the request was sent is the exact orphan
|
||||||
|
# risk: the host may have launched. Must be ambiguous, not a definite fail.
|
||||||
|
with patch(_URLOPEN, side_effect=TimeoutError("read timed out")):
|
||||||
|
with self.assertRaises(BrokerUnavailableError):
|
||||||
|
self.c.submit("tok")
|
||||||
|
|
||||||
|
def test_malformed_success_body_raises(self) -> None:
|
||||||
|
with patch(_URLOPEN, return_value=_resp({"op": "launch"})): # missing bottle_id
|
||||||
|
with self.assertRaises(BrokerClientError):
|
||||||
|
self.c.submit("tok")
|
||||||
|
|
||||||
|
def test_empty_error_body_is_tolerated(self) -> None:
|
||||||
|
# An error with no readable JSON body still classifies by status code.
|
||||||
|
with patch(_URLOPEN, side_effect=_http_error(401)):
|
||||||
|
with self.assertRaises(BrokerAuthError):
|
||||||
|
self.c.submit("forged")
|
||||||
|
|
||||||
|
def test_non_json_success_body_raises(self) -> None:
|
||||||
|
# A 200 whose body isn't JSON is tolerated into {} then fails the
|
||||||
|
# missing-field check — a definite client error, not a crash.
|
||||||
|
m = MagicMock()
|
||||||
|
m.__enter__.return_value.read.return_value = b"not json at all"
|
||||||
|
with patch(_URLOPEN, return_value=m):
|
||||||
|
with self.assertRaises(BrokerClientError):
|
||||||
|
self.c.submit("tok")
|
||||||
|
|
||||||
|
def test_unreadable_error_body_is_tolerated(self) -> None:
|
||||||
|
# An HTTPError whose body can't be read (fp=None) still classifies by
|
||||||
|
# status — the error detail is best-effort.
|
||||||
|
err = urllib.error.HTTPError(
|
||||||
|
"http://host/broker", 502, "err", {}, None) # type: ignore[arg-type]
|
||||||
|
with patch(_URLOPEN, side_effect=err):
|
||||||
|
with self.assertRaises(BrokerClientError):
|
||||||
|
self.c.submit("tok")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -248,88 +248,6 @@ class TestDockerGateway(unittest.TestCase):
|
|||||||
calls,
|
calls,
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_ensure_running_replaces_poisoned_ipv6_network(self) -> None:
|
|
||||||
# A daemon that default-enables IPv6 leaves the gateway network with a
|
|
||||||
# malformed fdd0::/64 gateway, so `docker network inspect` exits
|
|
||||||
# non-zero with a ParseAddr error (not "No such network"). `--ipv6=false`
|
|
||||||
# can't heal an already-poisoned network — the create just no-ops on
|
|
||||||
# "already exists" — so _ensure_network must force-remove and recreate
|
|
||||||
# it, else every later subnet read keeps failing.
|
|
||||||
calls: list[list[str]] = []
|
|
||||||
|
|
||||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
|
||||||
calls.append(argv)
|
|
||||||
if argv[:2] == ["docker", "ps"]:
|
|
||||||
return _proc(stdout="")
|
|
||||||
if argv[:3] == ["docker", "network", "inspect"]:
|
|
||||||
return _proc(
|
|
||||||
returncode=1,
|
|
||||||
stderr='ParseAddr("fdd0:0:0:4::1/64"): unexpected character, '
|
|
||||||
'want colon (at "/64")',
|
|
||||||
)
|
|
||||||
return _proc()
|
|
||||||
|
|
||||||
with patch(_RUN_DOCKER, side_effect=fake):
|
|
||||||
self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
|
||||||
self.assertIn(["docker", "rm", "--force", self.sc.name], calls)
|
|
||||||
self.assertIn(["docker", "network", "rm", self.sc.network], calls)
|
|
||||||
creates = [c for c in calls if c[:3] == ["docker", "network", "create"]]
|
|
||||||
self.assertEqual(
|
|
||||||
[[
|
|
||||||
"docker", "network", "create",
|
|
||||||
"--ipv6=false",
|
|
||||||
"--subnet", DEFAULT_GATEWAY_SUBNET,
|
|
||||||
"--label",
|
|
||||||
f"bot-bottle.gateway-subnet={DEFAULT_GATEWAY_SUBNET}",
|
|
||||||
self.sc.network,
|
|
||||||
]],
|
|
||||||
creates,
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_ensure_running_creates_network_when_inspect_reports_absent(self) -> None:
|
|
||||||
# The absent case (inspect fails with "No such network") must NOT try to
|
|
||||||
# remove anything — it just creates. Guards the poisoned-vs-absent split.
|
|
||||||
calls: list[list[str]] = []
|
|
||||||
|
|
||||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
|
||||||
calls.append(argv)
|
|
||||||
if argv[:3] == ["docker", "network", "inspect"]:
|
|
||||||
return _proc(returncode=1, stderr="Error: No such network: x")
|
|
||||||
return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc()
|
|
||||||
|
|
||||||
with patch(_RUN_DOCKER, side_effect=fake):
|
|
||||||
self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
|
||||||
self.assertNotIn(["docker", "network", "rm", self.sc.network], calls)
|
|
||||||
creates = [c for c in calls if c[:3] == ["docker", "network", "create"]]
|
|
||||||
self.assertEqual(1, len(creates))
|
|
||||||
|
|
||||||
def test_ensure_running_does_not_destroy_on_generic_inspect_error(self) -> None:
|
|
||||||
# A generic inspect failure (daemon hiccup, permission, timeout) is NOT
|
|
||||||
# evidence of a poisoned network. Only the ParseAddr poison signature may
|
|
||||||
# take the destructive heal path; anything else must surface as an error
|
|
||||||
# without tearing down a possibly-healthy shared gateway.
|
|
||||||
calls: list[list[str]] = []
|
|
||||||
|
|
||||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
|
||||||
calls.append(argv)
|
|
||||||
if argv[:2] == ["docker", "ps"]:
|
|
||||||
return _proc(stdout="")
|
|
||||||
if argv[:3] == ["docker", "network", "inspect"]:
|
|
||||||
return _proc(
|
|
||||||
returncode=1,
|
|
||||||
stderr="Cannot connect to the Docker daemon at unix:///var/run/docker.sock",
|
|
||||||
)
|
|
||||||
return _proc()
|
|
||||||
|
|
||||||
with patch(_RUN_DOCKER, side_effect=fake):
|
|
||||||
with self.assertRaises(GatewayError):
|
|
||||||
self.sc.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
|
||||||
# No mutation of the shared gateway: neither the container nor the
|
|
||||||
# network is removed, and nothing is recreated.
|
|
||||||
self.assertNotIn(["docker", "network", "rm", self.sc.network], calls)
|
|
||||||
self.assertFalse(any(c[:3] == ["docker", "rm", "--force"] for c in calls))
|
|
||||||
self.assertEqual([], [c for c in calls if c[:3] == ["docker", "network", "create"]])
|
|
||||||
|
|
||||||
def test_ca_cert_pem_reads_from_container(self) -> None:
|
def test_ca_cert_pem_reads_from_container(self) -> None:
|
||||||
with patch(_RUN_DOCKER, return_value=_proc(stdout=_CA_PEM)) as m:
|
with patch(_RUN_DOCKER, return_value=_proc(stdout=_CA_PEM)) as m:
|
||||||
self.assertEqual(_CA_PEM, self.sc.ca_cert_pem())
|
self.assertEqual(_CA_PEM, self.sc.ca_cert_pem())
|
||||||
|
|||||||
@@ -0,0 +1,306 @@
|
|||||||
|
"""Unit tests for the host control server (issue #468, chunk 1).
|
||||||
|
|
||||||
|
Mostly exercises the pure `dispatch()` (socket-free, like the orchestrator
|
||||||
|
server tests), plus a real-socket round-trip through `BrokerClient` that proves
|
||||||
|
the full sign -> POST -> verify -> act seam over HTTP.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import http.client
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import secrets
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import typing
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from bot_bottle.orchestrator.broker import (
|
||||||
|
BrokerAuthError,
|
||||||
|
LaunchBroker,
|
||||||
|
LaunchRequest,
|
||||||
|
StubBroker,
|
||||||
|
sign_request,
|
||||||
|
)
|
||||||
|
from bot_bottle.orchestrator.broker_client import BrokerClient
|
||||||
|
from bot_bottle.orchestrator.host_server import (
|
||||||
|
MAX_BODY_BYTES,
|
||||||
|
Handler,
|
||||||
|
HostControlServer,
|
||||||
|
broker_secret,
|
||||||
|
dispatch,
|
||||||
|
main,
|
||||||
|
make_host_server,
|
||||||
|
)
|
||||||
|
from bot_bottle.paths import LAUNCH_BROKER_KEY_ENV
|
||||||
|
|
||||||
|
|
||||||
|
def _body(obj: object) -> bytes:
|
||||||
|
return json.dumps(obj).encode()
|
||||||
|
|
||||||
|
|
||||||
|
class _RaisingBroker(LaunchBroker):
|
||||||
|
"""A broker whose backend launch always fails — exercises the 502 path (an
|
||||||
|
operational backend failure, distinct from a fail-closed provenance 401)."""
|
||||||
|
|
||||||
|
def _launch(self, req: LaunchRequest) -> None:
|
||||||
|
raise RuntimeError("docker down")
|
||||||
|
|
||||||
|
def _teardown(self, req: LaunchRequest) -> None:
|
||||||
|
raise RuntimeError("docker down")
|
||||||
|
|
||||||
|
|
||||||
|
class TestDispatch(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.secret = secrets.token_bytes(16)
|
||||||
|
self.broker = StubBroker(self.secret)
|
||||||
|
|
||||||
|
def _token(self, **kwargs: object) -> str:
|
||||||
|
return sign_request(LaunchRequest(**kwargs), self.secret) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
def test_health(self) -> None:
|
||||||
|
status, payload = dispatch(self.broker, "GET", "/health", b"")
|
||||||
|
self.assertEqual(200, status)
|
||||||
|
self.assertEqual("ok", payload["status"])
|
||||||
|
|
||||||
|
def test_broker_launch_verifies_and_acts(self) -> None:
|
||||||
|
token = self._token(
|
||||||
|
op="launch", bottle_id="b1", source_ip="10.243.0.1",
|
||||||
|
image_ref="img", slot=2,
|
||||||
|
)
|
||||||
|
status, payload = dispatch(self.broker, "POST", "/broker", _body({"token": token}))
|
||||||
|
self.assertEqual(200, status)
|
||||||
|
self.assertEqual("launch", payload["op"])
|
||||||
|
self.assertEqual("b1", payload["bottle_id"])
|
||||||
|
self.assertEqual("img", payload["image_ref"])
|
||||||
|
self.assertEqual(2, payload["slot"])
|
||||||
|
self.assertEqual(["b1"], [r.bottle_id for r in self.broker.launched])
|
||||||
|
|
||||||
|
def test_broker_teardown_acts(self) -> None:
|
||||||
|
token = self._token(op="teardown", bottle_id="b1")
|
||||||
|
status, _ = dispatch(self.broker, "POST", "/broker", _body({"token": token}))
|
||||||
|
self.assertEqual(200, status)
|
||||||
|
self.assertEqual(["b1"], [r.bottle_id for r in self.broker.torn_down])
|
||||||
|
|
||||||
|
def test_forged_token_is_401_and_nothing_acted(self) -> None:
|
||||||
|
forged = sign_request(
|
||||||
|
LaunchRequest(op="launch", bottle_id="b1"), secrets.token_bytes(16))
|
||||||
|
status, payload = dispatch(self.broker, "POST", "/broker", _body({"token": forged}))
|
||||||
|
self.assertEqual(401, status)
|
||||||
|
self.assertIn("broker auth failed", str(payload["error"]))
|
||||||
|
self.assertEqual([], self.broker.launched) # fail-closed: never launched
|
||||||
|
|
||||||
|
def test_backend_failure_is_502(self) -> None:
|
||||||
|
broker = _RaisingBroker(self.secret)
|
||||||
|
token = self._token(op="launch", bottle_id="b1", image_ref="img")
|
||||||
|
status, payload = dispatch(broker, "POST", "/broker", _body({"token": token}))
|
||||||
|
self.assertEqual(502, status)
|
||||||
|
self.assertIn("backend launch failed", str(payload["error"]))
|
||||||
|
|
||||||
|
def test_missing_token_is_400(self) -> None:
|
||||||
|
status, _ = dispatch(self.broker, "POST", "/broker", _body({}))
|
||||||
|
self.assertEqual(400, status)
|
||||||
|
|
||||||
|
def test_bad_json_is_400(self) -> None:
|
||||||
|
status, _ = dispatch(self.broker, "POST", "/broker", b"{not json")
|
||||||
|
self.assertEqual(400, status)
|
||||||
|
|
||||||
|
def test_empty_body_is_missing_token_400(self) -> None:
|
||||||
|
# Empty body parses to {} (no token) → 400, never reaching the broker.
|
||||||
|
status, _ = dispatch(self.broker, "POST", "/broker", b"")
|
||||||
|
self.assertEqual(400, status)
|
||||||
|
self.assertEqual([], self.broker.launched)
|
||||||
|
|
||||||
|
def test_non_object_body_is_400(self) -> None:
|
||||||
|
status, _ = dispatch(self.broker, "POST", "/broker", b"[1, 2]")
|
||||||
|
self.assertEqual(400, status)
|
||||||
|
|
||||||
|
def test_unknown_route_404(self) -> None:
|
||||||
|
status, _ = dispatch(self.broker, "GET", "/nope", b"")
|
||||||
|
self.assertEqual(404, status)
|
||||||
|
|
||||||
|
def test_trailing_slash_normalized(self) -> None:
|
||||||
|
status, _ = dispatch(self.broker, "GET", "/health/", b"")
|
||||||
|
self.assertEqual(200, status)
|
||||||
|
|
||||||
|
|
||||||
|
class TestBrokerSecret(unittest.TestCase):
|
||||||
|
"""The durable launch-broker key (#468/#476): prefer the env-injected key,
|
||||||
|
else the durable host key file, so signer and verifier resolve the same one."""
|
||||||
|
|
||||||
|
def test_reads_injected_key_from_env(self) -> None:
|
||||||
|
# The injected key is honoured regardless of allow_host_file — both the
|
||||||
|
# host controller and the guest orchestrator take an injected key.
|
||||||
|
self.assertEqual(
|
||||||
|
b"injected-key", broker_secret({LAUNCH_BROKER_KEY_ENV: "injected-key"}))
|
||||||
|
self.assertEqual(
|
||||||
|
b"injected-key",
|
||||||
|
broker_secret({LAUNCH_BROKER_KEY_ENV: "injected-key"}, allow_host_file=True))
|
||||||
|
|
||||||
|
def test_guest_without_injection_fails_closed(self) -> None:
|
||||||
|
# The default (guest orchestrator): no env key and NO host-file fallback,
|
||||||
|
# so it returns None rather than mint a divergent process-local key.
|
||||||
|
self.assertIsNone(broker_secret({}))
|
||||||
|
|
||||||
|
def test_host_side_falls_back_to_the_durable_key_file(self) -> None:
|
||||||
|
# allow_host_file=True (host controller / dev-harness): mint/read the
|
||||||
|
# durable host key file, the same key on every call (restart re-adoption).
|
||||||
|
with tempfile.TemporaryDirectory() as root:
|
||||||
|
with patch.dict("os.environ", {"BOT_BOTTLE_ROOT": root}, clear=False):
|
||||||
|
os.environ.pop(LAUNCH_BROKER_KEY_ENV, None)
|
||||||
|
first = broker_secret(allow_host_file=True)
|
||||||
|
second = broker_secret(allow_host_file=True)
|
||||||
|
self.assertTrue(first)
|
||||||
|
self.assertEqual(first, second)
|
||||||
|
|
||||||
|
|
||||||
|
class TestSeamRoundTrip(unittest.TestCase):
|
||||||
|
"""The whole point of chunk 1: a request signed by the orchestrator side is
|
||||||
|
POSTed to a real host control server, verified there, and acted on — over
|
||||||
|
HTTP, not an in-process call."""
|
||||||
|
|
||||||
|
def _serve(self, broker: LaunchBroker) -> BrokerClient:
|
||||||
|
server = make_host_server(broker, "127.0.0.1", 0)
|
||||||
|
self.addCleanup(server.server_close)
|
||||||
|
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||||
|
self.addCleanup(server.shutdown)
|
||||||
|
host, port = server.server_address[0], server.server_address[1]
|
||||||
|
return BrokerClient(f"http://{host}:{port}")
|
||||||
|
|
||||||
|
def test_sign_post_verify_act_over_http(self) -> None:
|
||||||
|
secret = secrets.token_bytes(16)
|
||||||
|
broker = StubBroker(secret)
|
||||||
|
client = self._serve(broker)
|
||||||
|
req = LaunchRequest(
|
||||||
|
op="launch", bottle_id="b1", source_ip="10.0.0.1", image_ref="img", slot=1)
|
||||||
|
got = client.submit(sign_request(req, secret))
|
||||||
|
self.assertEqual(req, got) # the controller echoes the verified request
|
||||||
|
self.assertEqual(["b1"], [r.bottle_id for r in broker.launched])
|
||||||
|
|
||||||
|
def test_forged_token_raises_broker_auth_error_over_http(self) -> None:
|
||||||
|
secret = secrets.token_bytes(16)
|
||||||
|
broker = StubBroker(secret)
|
||||||
|
client = self._serve(broker)
|
||||||
|
forged = sign_request(
|
||||||
|
LaunchRequest(op="launch", bottle_id="b1"), secrets.token_bytes(16))
|
||||||
|
with self.assertRaises(BrokerAuthError):
|
||||||
|
client.submit(forged)
|
||||||
|
self.assertEqual([], broker.launched) # fail-closed across the wire
|
||||||
|
|
||||||
|
|
||||||
|
class TestRequestLimits(unittest.TestCase):
|
||||||
|
"""The privileged listener must not let a caller that can merely reach the
|
||||||
|
socket (no signed token) exhaust it via an oversized declared body — and it
|
||||||
|
rejects on the Content-Length *header*, before reading the body."""
|
||||||
|
|
||||||
|
def _addr(self) -> tuple[str, int]:
|
||||||
|
self.broker = StubBroker(secrets.token_bytes(16))
|
||||||
|
server = make_host_server(self.broker, "127.0.0.1", 0)
|
||||||
|
self.addCleanup(server.server_close)
|
||||||
|
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||||
|
self.addCleanup(server.shutdown)
|
||||||
|
host, port = server.server_address[:2]
|
||||||
|
return typing.cast(str, host), port
|
||||||
|
|
||||||
|
def test_oversized_content_length_is_rejected_before_reading(self) -> None:
|
||||||
|
host, port = self._addr()
|
||||||
|
conn = http.client.HTTPConnection(host, port, timeout=5)
|
||||||
|
self.addCleanup(conn.close)
|
||||||
|
# Declare an oversized body but send only a sliver: the server must reject
|
||||||
|
# on the header before reading, so the caller gets a clean, deterministic
|
||||||
|
# 413 (no large unread body to race a connection reset).
|
||||||
|
conn.putrequest("POST", "/broker", skip_accept_encoding=True)
|
||||||
|
conn.putheader("Content-Type", "application/json")
|
||||||
|
conn.putheader("Content-Length", str(MAX_BODY_BYTES + 1))
|
||||||
|
conn.endheaders()
|
||||||
|
conn.send(b"{}") # far short of the declared length; never read
|
||||||
|
resp = conn.getresponse()
|
||||||
|
self.assertEqual(413, resp.status)
|
||||||
|
self.assertEqual([], self.broker.launched) # never reached the broker
|
||||||
|
|
||||||
|
|
||||||
|
class TestServeUnit(unittest.TestCase):
|
||||||
|
"""Drive `Handler._serve` directly (no socket). The real per-request handler
|
||||||
|
runs in a daemon thread whose coverage/trace data is lost, so the
|
||||||
|
bounded-body and error paths are exercised here in the main thread instead."""
|
||||||
|
|
||||||
|
def _handler(self, broker: LaunchBroker, headers: dict[str, str],
|
||||||
|
body: bytes = b"") -> tuple[Handler, MagicMock]:
|
||||||
|
server = HostControlServer.__new__(HostControlServer)
|
||||||
|
server.broker = broker
|
||||||
|
h = Handler.__new__(Handler)
|
||||||
|
h.server = server
|
||||||
|
h.headers = headers # type: ignore[assignment] — dict is a valid .get() stand-in
|
||||||
|
h.path = "/broker"
|
||||||
|
h.rfile = io.BytesIO(body)
|
||||||
|
h.wfile = io.BytesIO()
|
||||||
|
send_response = MagicMock()
|
||||||
|
h.send_response = send_response # type: ignore[method-assign]
|
||||||
|
h.send_header = MagicMock() # type: ignore[method-assign]
|
||||||
|
h.end_headers = MagicMock() # type: ignore[method-assign]
|
||||||
|
return h, send_response
|
||||||
|
|
||||||
|
def test_oversized_content_length_is_413(self) -> None:
|
||||||
|
broker = StubBroker(secrets.token_bytes(16))
|
||||||
|
h, send_response = self._handler(broker, {"Content-Length": str(MAX_BODY_BYTES + 1)})
|
||||||
|
h.do_POST() # exercises do_POST -> _serve
|
||||||
|
send_response.assert_called_once_with(413)
|
||||||
|
self.assertEqual([], broker.launched) # rejected before the broker
|
||||||
|
|
||||||
|
def test_invalid_content_length_is_400(self) -> None:
|
||||||
|
h, send_response = self._handler(StubBroker(secrets.token_bytes(16)),
|
||||||
|
{"Content-Length": "not-a-number"})
|
||||||
|
h._serve("POST")
|
||||||
|
send_response.assert_called_once_with(400)
|
||||||
|
|
||||||
|
def test_valid_request_dispatches_200(self) -> None:
|
||||||
|
secret = secrets.token_bytes(16)
|
||||||
|
broker = StubBroker(secret)
|
||||||
|
body = _body({"token": sign_request(
|
||||||
|
LaunchRequest(op="teardown", bottle_id="b1"), secret)})
|
||||||
|
h, send_response = self._handler(broker, {"Content-Length": str(len(body))}, body)
|
||||||
|
h._serve("POST")
|
||||||
|
send_response.assert_called_once_with(200)
|
||||||
|
self.assertEqual(["b1"], [r.bottle_id for r in broker.torn_down])
|
||||||
|
|
||||||
|
def test_dispatch_exception_becomes_500(self) -> None:
|
||||||
|
# dispatch is total, but the handler still guards it: a raised dispatch
|
||||||
|
# returns 500 rather than dropping the connection.
|
||||||
|
h, send_response = self._handler(
|
||||||
|
StubBroker(secrets.token_bytes(16)), {"Content-Length": "0"})
|
||||||
|
with patch("bot_bottle.orchestrator.host_server.dispatch",
|
||||||
|
side_effect=RuntimeError("boom")):
|
||||||
|
h._serve("POST")
|
||||||
|
send_response.assert_called_once_with(500)
|
||||||
|
|
||||||
|
def test_health_over_do_get(self) -> None:
|
||||||
|
h, send_response = self._handler(StubBroker(secrets.token_bytes(16)), {})
|
||||||
|
h.path = "/health"
|
||||||
|
h.do_GET()
|
||||||
|
send_response.assert_called_once_with(200)
|
||||||
|
|
||||||
|
|
||||||
|
class TestMain(unittest.TestCase):
|
||||||
|
def test_fail_closed_without_secret(self) -> None:
|
||||||
|
with patch("bot_bottle.orchestrator.host_server.broker_secret",
|
||||||
|
return_value=None):
|
||||||
|
self.assertEqual(2, main(["--port", "0"]))
|
||||||
|
|
||||||
|
def test_serves_then_shuts_down_cleanly(self) -> None:
|
||||||
|
fake = MagicMock()
|
||||||
|
fake.server_address = ("127.0.0.1", 0)
|
||||||
|
fake.serve_forever.side_effect = KeyboardInterrupt
|
||||||
|
with patch("bot_bottle.orchestrator.host_server.broker_secret",
|
||||||
|
return_value=b"k"), \
|
||||||
|
patch("bot_bottle.orchestrator.host_server.make_host_server",
|
||||||
|
return_value=fake):
|
||||||
|
self.assertEqual(0, main(["--port", "0"]))
|
||||||
|
fake.serve_forever.assert_called_once()
|
||||||
|
fake.server_close.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
"""Unit: the orchestrator dev-harness entrypoint (`python -m bot_bottle.orchestrator`).
|
||||||
|
|
||||||
|
Exercises broker selection (stub / docker / http) and the fail-closed http path,
|
||||||
|
patching `make_server` so the serve loop returns instead of blocking.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import secrets
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from bot_bottle.orchestrator.__main__ import main
|
||||||
|
from bot_bottle.paths import LAUNCH_BROKER_KEY_ENV
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_server() -> MagicMock:
|
||||||
|
fake = MagicMock()
|
||||||
|
fake.server_address = ("127.0.0.1", 0)
|
||||||
|
# Break out of serve_forever immediately, exercising the try/finally.
|
||||||
|
fake.serve_forever.side_effect = KeyboardInterrupt
|
||||||
|
return fake
|
||||||
|
|
||||||
|
|
||||||
|
class TestMain(unittest.TestCase):
|
||||||
|
def _run(self, broker: str, env: dict[str, str] | None = None) -> tuple[int, MagicMock]:
|
||||||
|
fake = _fake_server()
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
argv = ["--db", str(Path(d) / "r.db"), "--port", "0", "--broker", broker]
|
||||||
|
with patch("bot_bottle.orchestrator.__main__.make_server", return_value=fake), \
|
||||||
|
patch.dict("os.environ", env or {}, clear=False):
|
||||||
|
if env is None:
|
||||||
|
os.environ.pop(LAUNCH_BROKER_KEY_ENV, None)
|
||||||
|
rc = main(argv)
|
||||||
|
return rc, fake
|
||||||
|
|
||||||
|
def test_stub_broker_serves_and_closes(self) -> None:
|
||||||
|
rc, fake = self._run("stub")
|
||||||
|
self.assertEqual(0, rc)
|
||||||
|
fake.serve_forever.assert_called_once()
|
||||||
|
fake.server_close.assert_called_once()
|
||||||
|
|
||||||
|
def test_docker_broker_serves(self) -> None:
|
||||||
|
rc, _ = self._run("docker")
|
||||||
|
self.assertEqual(0, rc)
|
||||||
|
|
||||||
|
def test_http_broker_with_injected_key_serves(self) -> None:
|
||||||
|
# The guest orchestrator takes the launch-broker key by injection.
|
||||||
|
rc, _ = self._run(
|
||||||
|
"http", env={LAUNCH_BROKER_KEY_ENV: secrets.token_urlsafe(16)})
|
||||||
|
self.assertEqual(0, rc)
|
||||||
|
|
||||||
|
def test_http_broker_without_injected_key_exits(self) -> None:
|
||||||
|
# Fail-closed: no host-file fallback for the guest, so a missing injected
|
||||||
|
# key is a usage error rather than a silently-minted divergent key.
|
||||||
|
with tempfile.TemporaryDirectory() as d:
|
||||||
|
with patch.dict("os.environ", {}, clear=False):
|
||||||
|
os.environ.pop(LAUNCH_BROKER_KEY_ENV, None)
|
||||||
|
with self.assertRaises(SystemExit):
|
||||||
|
main(["--db", str(Path(d) / "r.db"), "--broker", "http"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -3,12 +3,11 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
import hashlib
|
|
||||||
import hmac
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from bot_bottle.orchestrator.store.secret_store import (
|
from bot_bottle.orchestrator.store.secret_store import (
|
||||||
ENV_VAR_SECRET_NAME,
|
ENV_VAR_SECRET_NAME,
|
||||||
|
_NONCE_BYTES,
|
||||||
decrypt_value,
|
decrypt_value,
|
||||||
encrypt_value,
|
encrypt_value,
|
||||||
new_env_var_secret,
|
new_env_var_secret,
|
||||||
@@ -68,35 +67,27 @@ class TestDecryptErrors(unittest.TestCase):
|
|||||||
def setUp(self) -> None:
|
def setUp(self) -> None:
|
||||||
self.secret = new_env_var_secret()
|
self.secret = new_env_var_secret()
|
||||||
|
|
||||||
def test_wrong_key_raises_value_error(self) -> None:
|
def test_wrong_key_always_raises_value_error(self) -> None:
|
||||||
ct = encrypt_value(self.secret, "secret-token")
|
# Deterministic: the authentication tag rejects a wrong key every time,
|
||||||
other_key = new_env_var_secret()
|
# so reprovision can never inject a garbage token. Repeat across many
|
||||||
with self.assertRaisesRegex(ValueError, "authentication failed"):
|
# random keys (the old unauthenticated scheme let ~5% through when the
|
||||||
decrypt_value(other_key, ct)
|
# garbage happened to decode as valid UTF-8).
|
||||||
|
for _ in range(200):
|
||||||
|
ct = encrypt_value(self.secret, "secret-token")
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
decrypt_value(new_env_var_secret(), ct)
|
||||||
|
|
||||||
def test_tampered_ciphertext_raises_value_error(self) -> None:
|
def test_tampered_ciphertext_raises_value_error(self) -> None:
|
||||||
raw = bytearray(base64.urlsafe_b64decode(
|
ct = encrypt_value(self.secret, "secret-token")
|
||||||
encrypt_value(self.secret, "secret-token") + "=="
|
raw = bytearray(base64.urlsafe_b64decode(ct + "=" * (-len(ct) % 4)))
|
||||||
))
|
raw[_NONCE_BYTES] ^= 0x01 # flip a bit in the ciphertext body → tag mismatch
|
||||||
raw[22] ^= 1
|
tampered = base64.urlsafe_b64encode(bytes(raw)).rstrip(b"=").decode()
|
||||||
tampered = base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
|
with self.assertRaises(ValueError):
|
||||||
with self.assertRaisesRegex(ValueError, "authentication failed"):
|
|
||||||
decrypt_value(self.secret, tampered)
|
decrypt_value(self.secret, tampered)
|
||||||
|
|
||||||
def test_reads_legacy_ciphertext_for_migration(self) -> None:
|
|
||||||
key = base64.urlsafe_b64decode(self.secret + "==")
|
|
||||||
nonce = b"0123456789abcdef"
|
|
||||||
plaintext = b"legacy-token"
|
|
||||||
stream = hmac.new(
|
|
||||||
key, nonce + (0).to_bytes(4, "big"), hashlib.sha256,
|
|
||||||
).digest()
|
|
||||||
ciphertext = bytes(p ^ k for p, k in zip(plaintext, stream))
|
|
||||||
legacy = base64.urlsafe_b64encode(nonce + ciphertext).rstrip(b"=").decode()
|
|
||||||
self.assertEqual("legacy-token", decrypt_value(self.secret, legacy))
|
|
||||||
|
|
||||||
def test_truncated_blob_raises_value_error(self) -> None:
|
def test_truncated_blob_raises_value_error(self) -> None:
|
||||||
with self.assertRaises(ValueError):
|
with self.assertRaises(ValueError):
|
||||||
decrypt_value(self.secret, "dG9vc2hvcnQ") # "tooshort" — under 16 nonce bytes
|
decrypt_value(self.secret, "dG9vc2hvcnQ") # "tooshort" — under nonce+tag
|
||||||
|
|
||||||
def test_invalid_base64_raises_value_error(self) -> None:
|
def test_invalid_base64_raises_value_error(self) -> None:
|
||||||
with self.assertRaises(ValueError):
|
with self.assertRaises(ValueError):
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ server tests), plus one real-socket round-trip to prove the handler wiring.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
import http.client
|
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
import secrets
|
import secrets
|
||||||
@@ -23,7 +22,7 @@ from unittest.mock import MagicMock, patch
|
|||||||
|
|
||||||
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint
|
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, mint
|
||||||
from bot_bottle.orchestrator.broker import StubBroker
|
from bot_bottle.orchestrator.broker import StubBroker
|
||||||
from bot_bottle.orchestrator.server import MAX_BODY_BYTES, dispatch, make_server
|
from bot_bottle.orchestrator.server import dispatch, make_server
|
||||||
from bot_bottle.orchestrator.store.registry_store import BottleRecord, RegistryStore
|
from bot_bottle.orchestrator.store.registry_store import BottleRecord, RegistryStore
|
||||||
from bot_bottle.orchestrator.service import OrchestratorCore
|
from bot_bottle.orchestrator.service import OrchestratorCore
|
||||||
from bot_bottle.orchestrator.store.store_manager import StoreManager
|
from bot_bottle.orchestrator.store.store_manager import StoreManager
|
||||||
@@ -252,51 +251,11 @@ class TestDispatch(unittest.TestCase):
|
|||||||
|
|
||||||
|
|
||||||
class TestServerRoundTrip(unittest.TestCase):
|
class TestServerRoundTrip(unittest.TestCase):
|
||||||
def _raw_status(self, content_length: str, *, authenticated: bool = True) -> int:
|
|
||||||
tmp = tempfile.TemporaryDirectory()
|
|
||||||
self.addCleanup(tmp.cleanup)
|
|
||||||
key = "request-limits-key"
|
|
||||||
server = make_server(
|
|
||||||
_orchestrator(Path(tmp.name) / "r.db"),
|
|
||||||
"127.0.0.1", 0, signing_key=key,
|
|
||||||
)
|
|
||||||
self.addCleanup(server.server_close)
|
|
||||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
|
||||||
self.addCleanup(server.shutdown)
|
|
||||||
conn = http.client.HTTPConnection(
|
|
||||||
str(server.server_address[0]), server.server_address[1], timeout=5,
|
|
||||||
)
|
|
||||||
self.addCleanup(conn.close)
|
|
||||||
conn.putrequest("POST", "/bottles")
|
|
||||||
conn.putheader("Content-Length", content_length)
|
|
||||||
if authenticated:
|
|
||||||
conn.putheader(
|
|
||||||
"x-bot-bottle-orchestrator-auth", mint(ROLE_CLI, key),
|
|
||||||
)
|
|
||||||
conn.endheaders()
|
|
||||||
return conn.getresponse().status
|
|
||||||
|
|
||||||
def test_rejects_malformed_content_length(self) -> None:
|
|
||||||
self.assertEqual(400, self._raw_status("not-a-number"))
|
|
||||||
|
|
||||||
def test_rejects_oversized_body_without_reading_it(self) -> None:
|
|
||||||
self.assertEqual(413, self._raw_status(str(MAX_BODY_BYTES + 1)))
|
|
||||||
|
|
||||||
def test_rejects_unauthenticated_request_before_reading_body(self) -> None:
|
|
||||||
self.assertEqual(
|
|
||||||
401,
|
|
||||||
self._raw_status(str(MAX_BODY_BYTES), authenticated=False),
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_http_register_health_attribute(self) -> None:
|
def test_http_register_health_attribute(self) -> None:
|
||||||
tmp = tempfile.TemporaryDirectory()
|
tmp = tempfile.TemporaryDirectory()
|
||||||
self.addCleanup(tmp.cleanup)
|
self.addCleanup(tmp.cleanup)
|
||||||
orch = _orchestrator(Path(tmp.name) / "r.db")
|
orch = _orchestrator(Path(tmp.name) / "r.db")
|
||||||
signing_key = "round-trip-key"
|
server = make_server(orch, "127.0.0.1", 0)
|
||||||
auth = mint(ROLE_CLI, signing_key)
|
|
||||||
server = make_server(
|
|
||||||
orch, "127.0.0.1", 0, signing_key=signing_key,
|
|
||||||
)
|
|
||||||
self.addCleanup(server.server_close)
|
self.addCleanup(server.server_close)
|
||||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||||
thread.start()
|
thread.start()
|
||||||
@@ -308,10 +267,7 @@ class TestServerRoundTrip(unittest.TestCase):
|
|||||||
reg = json.load(urllib.request.urlopen(
|
reg = json.load(urllib.request.urlopen(
|
||||||
urllib.request.Request(
|
urllib.request.Request(
|
||||||
f"{base}/bottles", data=_body({"source_ip": "10.243.0.7"}),
|
f"{base}/bottles", data=_body({"source_ip": "10.243.0.7"}),
|
||||||
method="POST", headers={
|
method="POST", headers={"Content-Type": "application/json"},
|
||||||
"Content-Type": "application/json",
|
|
||||||
"x-bot-bottle-orchestrator-auth": auth,
|
|
||||||
},
|
|
||||||
), timeout=5,
|
), timeout=5,
|
||||||
))
|
))
|
||||||
self.assertTrue(reg["bottle_id"])
|
self.assertTrue(reg["bottle_id"])
|
||||||
@@ -323,10 +279,7 @@ class TestServerRoundTrip(unittest.TestCase):
|
|||||||
urllib.request.Request(
|
urllib.request.Request(
|
||||||
f"{base}/attribute",
|
f"{base}/attribute",
|
||||||
data=_body({"source_ip": "10.243.0.7", "identity_token": reg["identity_token"]}),
|
data=_body({"source_ip": "10.243.0.7", "identity_token": reg["identity_token"]}),
|
||||||
method="POST", headers={
|
method="POST", headers={"Content-Type": "application/json"},
|
||||||
"Content-Type": "application/json",
|
|
||||||
"x-bot-bottle-orchestrator-auth": auth,
|
|
||||||
},
|
|
||||||
), timeout=5,
|
), timeout=5,
|
||||||
))
|
))
|
||||||
self.assertEqual(reg["bottle_id"], attr["bottle_id"])
|
self.assertEqual(reg["bottle_id"], attr["bottle_id"])
|
||||||
@@ -334,25 +287,15 @@ class TestServerRoundTrip(unittest.TestCase):
|
|||||||
def test_internal_failure_is_contextual_but_redacted(self) -> None:
|
def test_internal_failure_is_contextual_but_redacted(self) -> None:
|
||||||
orch = MagicMock()
|
orch = MagicMock()
|
||||||
orch.registry.all.side_effect = RuntimeError("SENSITIVE request value")
|
orch.registry.all.side_effect = RuntimeError("SENSITIVE request value")
|
||||||
signing_key = "failure-path-key"
|
|
||||||
auth = mint(ROLE_CLI, signing_key)
|
|
||||||
with patch("sys.stderr", io.StringIO()) as stderr:
|
with patch("sys.stderr", io.StringIO()) as stderr:
|
||||||
server = make_server(
|
server = make_server(orch, "127.0.0.1", 0)
|
||||||
orch, "127.0.0.1", 0, signing_key=signing_key,
|
|
||||||
)
|
|
||||||
self.addCleanup(server.server_close)
|
self.addCleanup(server.server_close)
|
||||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||||
thread.start()
|
thread.start()
|
||||||
self.addCleanup(server.shutdown)
|
self.addCleanup(server.shutdown)
|
||||||
host, port = server.server_address[0], server.server_address[1]
|
host, port = server.server_address[0], server.server_address[1]
|
||||||
with self.assertRaises(urllib.error.HTTPError) as raised:
|
with self.assertRaises(urllib.error.HTTPError) as raised:
|
||||||
urllib.request.urlopen(
|
urllib.request.urlopen(f"http://{host}:{port}/bottles", timeout=5)
|
||||||
urllib.request.Request(
|
|
||||||
f"http://{host}:{port}/bottles",
|
|
||||||
headers={"x-bot-bottle-orchestrator-auth": auth},
|
|
||||||
),
|
|
||||||
timeout=5,
|
|
||||||
)
|
|
||||||
payload = json.loads(raised.exception.read())
|
payload = json.loads(raised.exception.read())
|
||||||
output = stderr.getvalue()
|
output = stderr.getvalue()
|
||||||
self.assertEqual({"error": "internal error"}, payload)
|
self.assertEqual({"error": "internal error"}, payload)
|
||||||
@@ -426,9 +369,8 @@ class TestOrchestratorAuth(unittest.TestCase):
|
|||||||
self.assertIsNotNone(self.orch.registry.get(rec.bottle_id))
|
self.assertIsNotNone(self.orch.registry.get(rec.bottle_id))
|
||||||
|
|
||||||
def _server_with_key(self, signing_key: str):
|
def _server_with_key(self, signing_key: str):
|
||||||
server = make_server(
|
with patch.dict("os.environ", {"BOT_BOTTLE_ORCHESTRATOR_TOKEN": signing_key}):
|
||||||
self.orch, "127.0.0.1", 0, signing_key=signing_key,
|
server = make_server(self.orch, "127.0.0.1", 0)
|
||||||
)
|
|
||||||
self.addCleanup(server.server_close)
|
self.addCleanup(server.server_close)
|
||||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||||
self.addCleanup(server.shutdown)
|
self.addCleanup(server.shutdown)
|
||||||
@@ -457,10 +399,16 @@ class TestOrchestratorAuth(unittest.TestCase):
|
|||||||
self.assertEqual(403, self._status(f"{base}/bottles", header=gateway_tok))
|
self.assertEqual(403, self._status(f"{base}/bottles", header=gateway_tok))
|
||||||
self.assertEqual(200, self._status(f"{base}/bottles", header=cli_tok))
|
self.assertEqual(200, self._status(f"{base}/bottles", header=cli_tok))
|
||||||
|
|
||||||
def test_unconfigured_server_refuses_to_start(self) -> None:
|
def test_unconfigured_server_runs_open(self) -> None:
|
||||||
with patch.dict("os.environ", {}, clear=True):
|
"""No signing key set (tests / nft-protected Firecracker): open mode
|
||||||
with self.assertRaisesRegex(ValueError, "signing key is required"):
|
grants full cli access, so existing round-trip behavior is unchanged."""
|
||||||
make_server(self.orch, "127.0.0.1", 0)
|
with patch.dict("os.environ", {}, clear=False):
|
||||||
|
import os
|
||||||
|
os.environ.pop("BOT_BOTTLE_ORCHESTRATOR_TOKEN", None)
|
||||||
|
server = make_server(self.orch, "127.0.0.1", 0)
|
||||||
|
self.addCleanup(server.server_close)
|
||||||
|
self.assertEqual(ROLE_CLI, server.role_for(""))
|
||||||
|
self.assertEqual(ROLE_CLI, server.role_for("anything"))
|
||||||
|
|
||||||
|
|
||||||
class TestDispatchSupervise(unittest.TestCase):
|
class TestDispatchSupervise(unittest.TestCase):
|
||||||
|
|||||||
@@ -11,7 +11,12 @@ from contextlib import closing
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from bot_bottle.orchestrator.broker import LaunchBroker, LaunchRequest, StubBroker
|
from bot_bottle.orchestrator.broker import (
|
||||||
|
BrokerUnavailableError,
|
||||||
|
LaunchBroker,
|
||||||
|
LaunchRequest,
|
||||||
|
StubBroker,
|
||||||
|
)
|
||||||
from bot_bottle.orchestrator.store.registry_store import RegistryStore
|
from bot_bottle.orchestrator.store.registry_store import RegistryStore
|
||||||
from bot_bottle.orchestrator.service import OrchestratorCore
|
from bot_bottle.orchestrator.service import OrchestratorCore
|
||||||
from bot_bottle.orchestrator.store.secret_store import new_env_var_secret
|
from bot_bottle.orchestrator.store.secret_store import new_env_var_secret
|
||||||
@@ -25,8 +30,8 @@ from bot_bottle.orchestrator.supervisor import (
|
|||||||
|
|
||||||
|
|
||||||
class _FailingBroker(LaunchBroker):
|
class _FailingBroker(LaunchBroker):
|
||||||
"""Verifies the token like any broker, then fails the launch — to
|
"""Verifies the token like any broker, then fails the launch *definitely* —
|
||||||
exercise the orchestrator's registry rollback."""
|
to exercise the orchestrator's registry rollback."""
|
||||||
|
|
||||||
def _launch(self, req: LaunchRequest) -> None:
|
def _launch(self, req: LaunchRequest) -> None:
|
||||||
raise RuntimeError("launch failed")
|
raise RuntimeError("launch failed")
|
||||||
@@ -35,6 +40,18 @@ class _FailingBroker(LaunchBroker):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class _UnavailableBroker(LaunchBroker):
|
||||||
|
"""Verifies the token, then raises the *ambiguous* BrokerUnavailableError —
|
||||||
|
the host may already have launched — so the orchestrator must KEEP the
|
||||||
|
registry row rather than orphan a running container."""
|
||||||
|
|
||||||
|
def _launch(self, req: LaunchRequest) -> None:
|
||||||
|
raise BrokerUnavailableError("delivery dropped after send")
|
||||||
|
|
||||||
|
def _teardown(self, req: LaunchRequest) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class TestOrchestrator(unittest.TestCase):
|
class TestOrchestrator(unittest.TestCase):
|
||||||
def setUp(self) -> None:
|
def setUp(self) -> None:
|
||||||
self._tmp = tempfile.TemporaryDirectory()
|
self._tmp = tempfile.TemporaryDirectory()
|
||||||
@@ -144,11 +161,20 @@ class TestOrchestrator(unittest.TestCase):
|
|||||||
self.assertIsNotNone(self.orch.resolve("10.243.0.3", rec.identity_token))
|
self.assertIsNotNone(self.orch.resolve("10.243.0.3", rec.identity_token))
|
||||||
self.assertIsNone(self.orch.resolve("10.243.0.3", "wrong-token"))
|
self.assertIsNone(self.orch.resolve("10.243.0.3", "wrong-token"))
|
||||||
|
|
||||||
def test_launch_rolls_back_registry_on_broker_failure(self) -> None:
|
def test_launch_rolls_back_registry_on_definite_broker_failure(self) -> None:
|
||||||
orch = OrchestratorCore(self.store, _FailingBroker(self.secret), self.secret)
|
orch = OrchestratorCore(self.store, _FailingBroker(self.secret), self.secret)
|
||||||
with self.assertRaises(RuntimeError):
|
with self.assertRaises(RuntimeError):
|
||||||
orch.launch_bottle("10.243.0.9")
|
orch.launch_bottle("10.243.0.9")
|
||||||
self.assertEqual([], self.store.all()) # no orphan
|
self.assertEqual([], self.store.all()) # no orphan row
|
||||||
|
|
||||||
|
def test_launch_keeps_registry_on_ambiguous_broker_failure(self) -> None:
|
||||||
|
# The host may already have launched the bottle before the response was
|
||||||
|
# lost, so deregistering would orphan a running container with no row.
|
||||||
|
# The row is kept for reconcile to reap iff the bottle is not live.
|
||||||
|
orch = OrchestratorCore(self.store, _UnavailableBroker(self.secret), self.secret)
|
||||||
|
with self.assertRaises(BrokerUnavailableError):
|
||||||
|
orch.launch_bottle("10.243.0.9")
|
||||||
|
self.assertEqual(1, len(self.store.all())) # row survives — no orphan container
|
||||||
|
|
||||||
def test_gateway_status_reports_unconfigured(self) -> None:
|
def test_gateway_status_reports_unconfigured(self) -> None:
|
||||||
# The orchestrator no longer owns a standalone gateway lifecycle; the
|
# The orchestrator no longer owns a standalone gateway lifecycle; the
|
||||||
|
|||||||
@@ -6,10 +6,13 @@ import unittest
|
|||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from bot_bottle import orchestrator_auth
|
from bot_bottle import orchestrator_auth
|
||||||
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY
|
from bot_bottle.orchestrator_auth import ROLE_CLI, ROLE_GATEWAY, ROLE_HOST
|
||||||
from bot_bottle.trust_domain import (
|
from bot_bottle.trust_domain import (
|
||||||
CONTROL_PLANE,
|
CONTROL_PLANE,
|
||||||
|
HOST_CONTROLLER,
|
||||||
|
LAUNCH_BROKER,
|
||||||
ControlPlaneProvisioning,
|
ControlPlaneProvisioning,
|
||||||
|
LaunchBrokerProvisioning,
|
||||||
ProvisioningError,
|
ProvisioningError,
|
||||||
TrustDomain,
|
TrustDomain,
|
||||||
)
|
)
|
||||||
@@ -78,7 +81,8 @@ class TestControlPlaneProvisioning(unittest.TestCase):
|
|||||||
self.assertEqual("key", prov.orchestrator_key())
|
self.assertEqual("key", prov.orchestrator_key())
|
||||||
|
|
||||||
def test_orchestrator_key_fail_closes_when_empty(self) -> None:
|
def test_orchestrator_key_fail_closes_when_empty(self) -> None:
|
||||||
# Invariant 4: the orchestrator must never start without a key. There is no
|
# Invariant 4: the orchestrator must never start without a key — it would
|
||||||
|
# run OPEN and grant every caller that reaches it full `cli`. There is no
|
||||||
# topology opt-out: a separate host does not stop the gateway (or any
|
# topology opt-out: a separate host does not stop the gateway (or any
|
||||||
# other caller) from reaching the control-plane listener.
|
# other caller) from reaching the control-plane listener.
|
||||||
prov = ControlPlaneProvisioning()
|
prov = ControlPlaneProvisioning()
|
||||||
@@ -100,5 +104,73 @@ class TestControlPlaneProvisioning(unittest.TestCase):
|
|||||||
self.assertNotEqual(ROLE_CLI, CONTROL_PLANE.verify(tok, "k"))
|
self.assertNotEqual(ROLE_CLI, CONTROL_PLANE.verify(tok, "k"))
|
||||||
|
|
||||||
|
|
||||||
|
class TestLaunchBrokerAndHostControllerDomains(unittest.TestCase):
|
||||||
|
"""The real #468 domains: the launch-broker key (shared by orchestrator +
|
||||||
|
host controller) and the host controller's own lifecycle key."""
|
||||||
|
|
||||||
|
def test_launch_broker_mints_no_role_tokens(self) -> None:
|
||||||
|
# Empty role set — it provides durable key material for the broker's own
|
||||||
|
# launch JWT, not orchestrator_auth role tokens.
|
||||||
|
self.assertEqual(frozenset(), LAUNCH_BROKER.roles)
|
||||||
|
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
LAUNCH_BROKER.mint(ROLE_CLI)
|
||||||
|
|
||||||
|
def test_host_controller_signs_host_role_only(self) -> None:
|
||||||
|
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||||
|
tok = HOST_CONTROLLER.mint(ROLE_HOST)
|
||||||
|
self.assertEqual(ROLE_HOST, HOST_CONTROLLER.verify(tok, "k"))
|
||||||
|
# A control-plane `cli` token (the orchestrator's key) never verifies as a
|
||||||
|
# host-controller role — the orchestrator can't forge lifecycle creds.
|
||||||
|
cli_tok = orchestrator_auth.mint(ROLE_CLI, "k")
|
||||||
|
self.assertIsNone(HOST_CONTROLLER.verify(cli_tok, "k"))
|
||||||
|
|
||||||
|
def test_control_plane_cannot_mint_the_host_role(self) -> None:
|
||||||
|
# `host` is outside the control-plane role set on purpose.
|
||||||
|
with patch("bot_bottle.trust_domain.host_signing_key", return_value="k"):
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
CONTROL_PLANE.mint(ROLE_HOST)
|
||||||
|
|
||||||
|
def test_the_three_domains_use_distinct_keys_and_env_vars(self) -> None:
|
||||||
|
self.assertEqual(3, len({
|
||||||
|
CONTROL_PLANE.key_filename,
|
||||||
|
LAUNCH_BROKER.key_filename,
|
||||||
|
HOST_CONTROLLER.key_filename,
|
||||||
|
}))
|
||||||
|
self.assertEqual(3, len({
|
||||||
|
CONTROL_PLANE.key_env, LAUNCH_BROKER.key_env, HOST_CONTROLLER.key_env,
|
||||||
|
}))
|
||||||
|
|
||||||
|
|
||||||
|
class TestLaunchBrokerProvisioning(unittest.TestCase):
|
||||||
|
def test_broker_key_returns_the_durable_key(self) -> None:
|
||||||
|
prov = LaunchBrokerProvisioning()
|
||||||
|
with patch("bot_bottle.trust_domain.host_signing_key", return_value="bk"):
|
||||||
|
self.assertEqual("bk", prov.broker_key())
|
||||||
|
|
||||||
|
def test_broker_key_fail_closes_when_empty(self) -> None:
|
||||||
|
# An empty key would leave the host controller unable to verify any
|
||||||
|
# launch — fail-closed rather than hand back a useless/dangerous key.
|
||||||
|
prov = LaunchBrokerProvisioning()
|
||||||
|
with patch("bot_bottle.trust_domain.host_signing_key", return_value=""):
|
||||||
|
with self.assertRaises(ProvisioningError):
|
||||||
|
prov.broker_key()
|
||||||
|
|
||||||
|
def test_controller_key_is_distinct_from_the_broker_key(self) -> None:
|
||||||
|
# The orchestrator holds the broker key but NEVER the controller key.
|
||||||
|
prov = LaunchBrokerProvisioning()
|
||||||
|
keys = {"launch-broker-key": "bk", "host-controller-key": "ck"}
|
||||||
|
with patch("bot_bottle.trust_domain.host_signing_key",
|
||||||
|
side_effect=keys.__getitem__):
|
||||||
|
self.assertEqual("bk", prov.broker_key())
|
||||||
|
self.assertEqual("ck", prov.controller_key())
|
||||||
|
|
||||||
|
def test_controller_key_fail_closes_when_empty(self) -> None:
|
||||||
|
prov = LaunchBrokerProvisioning()
|
||||||
|
with patch("bot_bottle.trust_domain.host_signing_key", return_value=""):
|
||||||
|
with self.assertRaises(ProvisioningError):
|
||||||
|
prov.controller_key()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user