Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f242733d15 | |||
| fd295d4c14 | |||
| f33566941b |
@@ -28,6 +28,7 @@ import subprocess
|
||||
|
||||
from ...paths import bot_bottle_root
|
||||
from ...log import info, warn
|
||||
from .. import EnumerationError
|
||||
from . import util as docker_mod
|
||||
from .bottle_cleanup_plan import DockerBottleCleanupPlan
|
||||
from ...bottle_state import bottle_state_dir, is_preserved
|
||||
@@ -36,15 +37,17 @@ from .compose import COMPOSE_PROJECT_PREFIX, list_compose_projects
|
||||
|
||||
def _list_prefixed_containers() -> list[str]:
|
||||
"""All bot-bottle-prefixed containers, running or stopped."""
|
||||
result = subprocess.run(
|
||||
["docker", "ps", "-a",
|
||||
"--filter", f"name=^{COMPOSE_PROJECT_PREFIX}",
|
||||
"--format", "{{.Names}}\t{{.Label \"com.docker.compose.project\"}}"],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "ps", "-a",
|
||||
"--filter", f"name=^{COMPOSE_PROJECT_PREFIX}",
|
||||
"--format", "{{.Names}}\t{{.Label \"com.docker.compose.project\"}}"],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
except OSError as exc:
|
||||
raise EnumerationError(f"docker ps failed: {exc}") from exc
|
||||
if result.returncode != 0:
|
||||
warn(f"docker ps failed: {result.stderr.strip()}")
|
||||
return []
|
||||
raise EnumerationError(f"docker ps failed: {result.stderr.strip()}")
|
||||
out: list[str] = []
|
||||
for line in (result.stdout or "").splitlines():
|
||||
if not line:
|
||||
@@ -63,15 +66,19 @@ def _list_prefixed_networks() -> list[str]:
|
||||
to a compose project. Compose-managed networks have a
|
||||
`com.docker.compose.project` label; bare ones (from pre-compose
|
||||
code paths) don't."""
|
||||
result = subprocess.run(
|
||||
["docker", "network", "ls",
|
||||
"--filter", f"name={COMPOSE_PROJECT_PREFIX}",
|
||||
"--format", "{{.Name}}\t{{.Label \"com.docker.compose.project\"}}"],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["docker", "network", "ls",
|
||||
"--filter", f"name={COMPOSE_PROJECT_PREFIX}",
|
||||
"--format", "{{.Name}}\t{{.Label \"com.docker.compose.project\"}}"],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
except OSError as exc:
|
||||
raise EnumerationError(f"docker network ls failed: {exc}") from exc
|
||||
if result.returncode != 0:
|
||||
warn(f"docker network ls failed: {result.stderr.strip()}")
|
||||
return []
|
||||
raise EnumerationError(
|
||||
f"docker network ls failed: {result.stderr.strip()}"
|
||||
)
|
||||
out: list[str] = []
|
||||
for line in (result.stdout or "").splitlines():
|
||||
if not line:
|
||||
@@ -120,7 +127,10 @@ def prepare_cleanup() -> DockerBottleCleanupPlan:
|
||||
`enumerate_active_agents()` so the orphan-state-dir bucket
|
||||
doesn't include slugs whose non-docker bottle is still up."""
|
||||
docker_mod.require_docker()
|
||||
projects = list_compose_projects()
|
||||
projects = list_compose_projects(
|
||||
warn_on_error=False,
|
||||
raise_on_error=True,
|
||||
)
|
||||
project_set = set(projects)
|
||||
# Late import to avoid a circular at module-load time —
|
||||
# the backend package's __init__ imports this module.
|
||||
|
||||
@@ -11,10 +11,9 @@ Reaps *orphans* only — resources with no live VM behind them:
|
||||
— a VMM left lingering after its dir was removed.
|
||||
|
||||
A run dir with a *live* firecracker process is a running bottle and is
|
||||
left strictly alone: it is neither killed nor removed. (The backend's
|
||||
`enumerate_active` registry is still a stub — #354 — so a live process
|
||||
is the only reliable "this bottle is in use" signal we have. Once the
|
||||
registry lands, registry-orphaned-but-running VMs can be reaped too.)
|
||||
left strictly alone: it is neither killed nor removed. Active-agent
|
||||
enumeration uses this same process snapshot, so cleanup and generic
|
||||
backend consumers agree about which bottles are running.
|
||||
|
||||
TAP slots free themselves (the flock drops when the launcher exits), so
|
||||
there is nothing to reclaim there.
|
||||
@@ -22,6 +21,7 @@ there is nothing to reclaim there.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
@@ -38,7 +38,7 @@ def _run_root() -> Path:
|
||||
return util.cache_dir() / "run"
|
||||
|
||||
|
||||
def _run_dir_of(cmd: str, run_root: Path) -> Path | None:
|
||||
def _run_dir_of(args: Sequence[str], run_root: Path) -> Path | None:
|
||||
"""The bottle run dir a firecracker cmdline belongs to, or None.
|
||||
|
||||
A bottle VM is launched with `--config-file <run_root>/<slug>/config.json`,
|
||||
@@ -46,15 +46,35 @@ def _run_dir_of(cmd: str, run_root: Path) -> Path | None:
|
||||
the run root. Anything else (a builder VM, the infra VM elsewhere) is
|
||||
not ours to reap here.
|
||||
"""
|
||||
toks = cmd.split()
|
||||
for i, tok in enumerate(toks):
|
||||
if tok == "--config-file" and i + 1 < len(toks):
|
||||
parent = Path(toks[i + 1]).parent
|
||||
for i, arg in enumerate(args):
|
||||
if arg == "--config-file" and i + 1 < len(args):
|
||||
parent = Path(args[i + 1]).parent
|
||||
if parent.parent == run_root:
|
||||
return parent
|
||||
return None
|
||||
|
||||
|
||||
def _decode_cmdline(raw: bytes) -> tuple[str, ...]:
|
||||
"""Decode Linux's NUL-delimited argv without losing embedded spaces."""
|
||||
return tuple(
|
||||
value.decode(errors="surrogateescape")
|
||||
for value in raw.split(b"\0") if value
|
||||
)
|
||||
|
||||
|
||||
def _process_args(pid: int) -> tuple[str, ...] | None:
|
||||
"""Read one process's lossless argv, or None when it exited meanwhile."""
|
||||
try:
|
||||
raw = Path(f"/proc/{pid}/cmdline").read_bytes()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except OSError as exc:
|
||||
raise EnumerationError(
|
||||
f"could not inspect Firecracker pid {pid}: {exc}"
|
||||
) from exc
|
||||
return _decode_cmdline(raw)
|
||||
|
||||
|
||||
def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
|
||||
"""Inspect running firecracker VMs under ``run_root``.
|
||||
|
||||
@@ -65,7 +85,7 @@ def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
|
||||
"""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-a", "firecracker"],
|
||||
["pgrep", "firecracker"],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
except OSError as exc:
|
||||
@@ -83,14 +103,14 @@ def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
|
||||
live: set[str] = set()
|
||||
orphan_pids: list[int] = []
|
||||
for line in result.stdout.splitlines():
|
||||
parts = line.split(None, 1)
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
try:
|
||||
pid = int(parts[0])
|
||||
pid = int(line.strip())
|
||||
except ValueError:
|
||||
continue
|
||||
run_dir = _run_dir_of(parts[1], run_root)
|
||||
args = _process_args(pid)
|
||||
if args is None:
|
||||
continue
|
||||
run_dir = _run_dir_of(args, run_root)
|
||||
if run_dir is None:
|
||||
continue
|
||||
if run_dir.is_dir():
|
||||
@@ -157,8 +177,8 @@ def _terminate_orphan(pid: int, run_root: Path) -> None:
|
||||
raise EnumerationError(
|
||||
f"could not revalidate Firecracker pid {pid}: {exc}"
|
||||
) from exc
|
||||
command = raw.replace(b"\0", b" ").decode(errors="replace")
|
||||
run_dir = _run_dir_of(command, run_root)
|
||||
args = _decode_cmdline(raw)
|
||||
run_dir = _run_dir_of(args, run_root)
|
||||
if run_dir is None or run_dir.is_dir():
|
||||
return
|
||||
info(f"kill firecracker VM pid {pid}")
|
||||
|
||||
@@ -41,6 +41,8 @@ from ... import resources
|
||||
from ...log import die, info
|
||||
from . import util
|
||||
|
||||
ARTIFACT_HTTP_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
# Bump if the on-disk artifact *format* changes (compression, layout) so a new
|
||||
# scheme can't collide with a cached/published artifact of the old one.
|
||||
_ARTIFACT_FORMAT = "1"
|
||||
@@ -164,7 +166,9 @@ def _download(url: str, dest: Path) -> None:
|
||||
"""Stream `url` to `dest` (atomic via a `.part` sibling)."""
|
||||
tmp = dest.with_suffix(dest.suffix + ".part")
|
||||
try:
|
||||
with urllib.request.urlopen(_open(url)) as resp, open(tmp, "wb") as out:
|
||||
with urllib.request.urlopen(
|
||||
_open(url), timeout=ARTIFACT_HTTP_TIMEOUT_SECONDS,
|
||||
) as resp, open(tmp, "wb") as out:
|
||||
shutil.copyfileobj(resp, out, _CHUNK)
|
||||
except urllib.error.HTTPError as e:
|
||||
tmp.unlink(missing_ok=True)
|
||||
|
||||
@@ -34,6 +34,7 @@ from pathlib import Path
|
||||
from . import infra_artifact, infra_vm, util
|
||||
|
||||
_CHUNK = 1 << 20
|
||||
_REGISTRY_HTTP_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
_GZ_NAME = "rootfs.ext4.gz"
|
||||
_SHA_NAME = "rootfs.ext4.gz.sha256"
|
||||
@@ -91,7 +92,9 @@ def _put(url: str, body: "bytes | Path", token: str) -> None:
|
||||
req.add_header("Authorization", f"token {token}")
|
||||
req.add_header("Content-Type", "application/octet-stream")
|
||||
try:
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
with urllib.request.urlopen(
|
||||
req, timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
|
||||
) as resp:
|
||||
print(f" uploaded {url} (HTTP {resp.status})")
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 409:
|
||||
@@ -112,7 +115,9 @@ def _delete(url: str, token: str) -> None:
|
||||
if token:
|
||||
req.add_header("Authorization", f"token {token}")
|
||||
try:
|
||||
with urllib.request.urlopen(req):
|
||||
with urllib.request.urlopen(
|
||||
req, timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
|
||||
):
|
||||
pass
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 404:
|
||||
@@ -151,7 +156,10 @@ def _try_download_published(role: str, role_dir: Path) -> str | None:
|
||||
version = _role_version(role)
|
||||
sha_url = infra_artifact.artifact_url(version, _SHA_NAME, role=role)
|
||||
try:
|
||||
with urllib.request.urlopen(infra_artifact._open(sha_url)):
|
||||
with urllib.request.urlopen(
|
||||
infra_artifact._open(sha_url),
|
||||
timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
|
||||
):
|
||||
pass
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
@@ -195,7 +203,10 @@ def _publish_bundle(role: str, role_dir: Path, token: str) -> str:
|
||||
# present, a re-publish is a no-op. Otherwise clear any partial upload left
|
||||
# by an interrupted prior attempt and upload the complete set.
|
||||
try:
|
||||
with urllib.request.urlopen(infra_artifact._open(sha_url)) as resp:
|
||||
with urllib.request.urlopen(
|
||||
infra_artifact._open(sha_url),
|
||||
timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
|
||||
) as resp:
|
||||
remote_sha = resp.read().decode("utf-8").split()[0].strip().lower()
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 404:
|
||||
|
||||
@@ -136,10 +136,18 @@ def _pump(name: str, stream: IO[bytes]) -> None:
|
||||
"""Read lines from `stream`, prefix with `[name]`, write to
|
||||
stdout. Runs in its own thread per child; daemon=True so a
|
||||
blocked read doesn't keep the process alive after main exits."""
|
||||
for raw in iter(stream.readline, b""):
|
||||
line = raw.decode("utf-8", errors="replace").rstrip("\n")
|
||||
sys.stdout.write(f"[{name}] {line}\n")
|
||||
sys.stdout.flush()
|
||||
try:
|
||||
for raw in iter(stream.readline, b""):
|
||||
line = raw.decode("utf-8", errors="replace").rstrip("\n")
|
||||
sys.stdout.write(f"[{name}] {line}\n")
|
||||
sys.stdout.flush()
|
||||
except (OSError, ValueError) as exc:
|
||||
# The manager closes a dead child's pipe after wait() and before a
|
||||
# restart. A pump can be between readline calls at that exact moment;
|
||||
# closed-stream errors are normal completion, not uncaught thread
|
||||
# failures. Preserve genuinely unexpected I/O diagnostics.
|
||||
if not stream.closed:
|
||||
_log(f"{name} output pump stopped: {type(exc).__name__}: {exc}")
|
||||
|
||||
|
||||
def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]:
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Shared resource boundaries for gateway stdlib HTTP services."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.server
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Protocol
|
||||
|
||||
|
||||
class Readable(Protocol):
|
||||
def read(self, size: int = -1, /) -> bytes: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BodyReadError(Exception):
|
||||
status: int
|
||||
message: str
|
||||
|
||||
|
||||
def read_declared_body(
|
||||
stream: Readable,
|
||||
connection: socket.socket,
|
||||
raw_length: str | None,
|
||||
*,
|
||||
maximum: int,
|
||||
timeout_seconds: float,
|
||||
require_length: bool,
|
||||
) -> bytes:
|
||||
"""Validate and read exactly one declared body under a read deadline."""
|
||||
if raw_length is None:
|
||||
if require_length:
|
||||
raise BodyReadError(411, "Content-Length required")
|
||||
raw_length = "0"
|
||||
try:
|
||||
length = int(raw_length)
|
||||
except ValueError as exc:
|
||||
raise BodyReadError(400, "invalid Content-Length") from exc
|
||||
if length < 0:
|
||||
raise BodyReadError(400, "invalid Content-Length")
|
||||
if length > maximum:
|
||||
raise BodyReadError(413, "request body too large")
|
||||
previous_timeout = connection.gettimeout()
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
chunks: list[bytes] = []
|
||||
remaining = length
|
||||
try:
|
||||
while remaining:
|
||||
timeout = deadline - time.monotonic()
|
||||
if timeout <= 0:
|
||||
raise BodyReadError(408, "request body read timed out")
|
||||
connection.settimeout(timeout)
|
||||
chunk = stream.read(min(remaining, 64 * 1024))
|
||||
if not chunk:
|
||||
raise BodyReadError(400, "incomplete request body")
|
||||
chunks.append(chunk)
|
||||
remaining -= len(chunk)
|
||||
except TimeoutError as exc:
|
||||
raise BodyReadError(408, "request body read timed out") from exc
|
||||
finally:
|
||||
connection.settimeout(previous_timeout)
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
class BoundedThreadingHTTPServer(http.server.ThreadingHTTPServer):
|
||||
"""ThreadingHTTPServer with a hard cap on in-flight request threads."""
|
||||
|
||||
daemon_threads = True
|
||||
|
||||
def __init__( # pylint: disable=consider-using-with
|
||||
self, *args, max_workers: int = 32, **kwargs, # type: ignore[no-untyped-def]
|
||||
):
|
||||
if max_workers < 1:
|
||||
raise ValueError("max_workers must be positive")
|
||||
self._request_slots = threading.BoundedSemaphore(max_workers)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def process_request(
|
||||
self, request: Any, client_address: Any,
|
||||
) -> None:
|
||||
if not self._request_slots.acquire( # pylint: disable=consider-using-with
|
||||
blocking=False,
|
||||
):
|
||||
try:
|
||||
request.sendall(
|
||||
b"HTTP/1.1 503 Service Unavailable\r\n"
|
||||
b"Content-Length: 0\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
finally:
|
||||
self.shutdown_request(request)
|
||||
return
|
||||
try:
|
||||
super().process_request(request, client_address)
|
||||
except BaseException:
|
||||
self._request_slots.release()
|
||||
raise
|
||||
|
||||
def process_request_thread(
|
||||
self, request: Any, client_address: Any,
|
||||
) -> None:
|
||||
try:
|
||||
super().process_request_thread(request, client_address)
|
||||
finally:
|
||||
self._request_slots.release()
|
||||
|
||||
|
||||
__all__ = ["BodyReadError", "BoundedThreadingHTTPServer", "read_declared_body"]
|
||||
@@ -22,11 +22,16 @@ import os
|
||||
import subprocess
|
||||
import sys
|
||||
import typing
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from http.server import BaseHTTPRequestHandler
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from bot_bottle.constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER
|
||||
from bot_bottle.gateway.bounded_http import (
|
||||
BodyReadError,
|
||||
BoundedThreadingHTTPServer,
|
||||
read_declared_body,
|
||||
)
|
||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||
|
||||
|
||||
@@ -77,6 +82,8 @@ def resolve_sandbox_root(
|
||||
|
||||
# Bound memory use while still allowing ordinary git push packfiles.
|
||||
MAX_BODY_BYTES = 100 * 1024 * 1024
|
||||
REQUEST_BODY_TIMEOUT_SECONDS = 30.0
|
||||
MAX_REQUEST_WORKERS = 16
|
||||
|
||||
|
||||
class GitHttpHandler(BaseHTTPRequestHandler):
|
||||
@@ -184,19 +191,18 @@ class GitHttpHandler(BaseHTTPRequestHandler):
|
||||
value = self.headers.get(header)
|
||||
if value:
|
||||
env[variable] = value
|
||||
raw_length = self.headers.get("content-length", "0") or "0"
|
||||
try:
|
||||
length = int(raw_length)
|
||||
except ValueError:
|
||||
self.send_error(400, "Bad Content-Length")
|
||||
body = read_declared_body(
|
||||
self.rfile,
|
||||
self.connection,
|
||||
self.headers.get("content-length"),
|
||||
maximum=MAX_BODY_BYTES,
|
||||
timeout_seconds=REQUEST_BODY_TIMEOUT_SECONDS,
|
||||
require_length=False,
|
||||
)
|
||||
except BodyReadError as exc:
|
||||
self.send_error(exc.status, exc.message)
|
||||
return
|
||||
if length < 0:
|
||||
self.send_error(400, "Negative Content-Length")
|
||||
return
|
||||
if length > MAX_BODY_BYTES:
|
||||
self.send_error(413, "Request body too large")
|
||||
return
|
||||
body = self.rfile.read(length) if length else b""
|
||||
proc = subprocess.run(
|
||||
["git", "http-backend"],
|
||||
input=body,
|
||||
@@ -273,7 +279,9 @@ def main() -> int:
|
||||
"(no single-tenant flat-root fallback)\n"
|
||||
)
|
||||
return 1
|
||||
server = ThreadingHTTPServer(("0.0.0.0", port), GitHttpHandler)
|
||||
server = BoundedThreadingHTTPServer(
|
||||
("0.0.0.0", port), GitHttpHandler, max_workers=MAX_REQUEST_WORKERS,
|
||||
)
|
||||
# Resolve each request's sandbox namespace by source IP against the
|
||||
# orchestrator control plane.
|
||||
server.policy_resolver = PolicyResolver(orch_url) # type: ignore[attr-defined]
|
||||
|
||||
@@ -6,9 +6,8 @@ import json
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Protocol
|
||||
|
||||
from bot_bottle.gateway.egress.context import resolve_client_context
|
||||
from bot_bottle.gateway.egress.schema import route_to_yaml_dict
|
||||
from bot_bottle.gateway.policy_resolver import PolicyResolver
|
||||
from bot_bottle.gateway.egress.schema import load_config, route_to_yaml_dict
|
||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||
from bot_bottle.supervisor import types as _sv
|
||||
|
||||
|
||||
@@ -24,6 +23,10 @@ class MethodNotFoundError(Exception):
|
||||
"""Raised when a JSON-RPC method has no MCP handler."""
|
||||
|
||||
|
||||
class RouteResolutionError(Exception):
|
||||
"""The caller's live route table could not be resolved authoritatively."""
|
||||
|
||||
|
||||
Handler = Callable[[dict[str, object]], object]
|
||||
|
||||
|
||||
@@ -60,10 +63,19 @@ def resolved_routes_payload(
|
||||
source_ip: str,
|
||||
identity_token: str,
|
||||
) -> dict[str, object]:
|
||||
"""Render the calling bottle's routes, failing closed to an empty list."""
|
||||
config, _slug, _tokens = resolve_client_context(
|
||||
resolver, source_ip, identity_token,
|
||||
)
|
||||
"""Render an authoritatively resolved route table for the calling bottle."""
|
||||
try:
|
||||
policy, bottle_id, _tokens = resolver.resolve_policy_and_bottle_id(
|
||||
source_ip, identity_token,
|
||||
)
|
||||
except PolicyResolveError as exc:
|
||||
raise RouteResolutionError("orchestrator unavailable") from exc
|
||||
if not bottle_id:
|
||||
raise RouteResolutionError("request source is not attributed to a bottle")
|
||||
try:
|
||||
config = load_config(policy or "")
|
||||
except ValueError as exc:
|
||||
raise RouteResolutionError("resolved policy is invalid") from exc
|
||||
body = json.dumps(
|
||||
{"routes": [route_to_yaml_dict(route) for route in config.routes]},
|
||||
indent=2,
|
||||
@@ -74,6 +86,7 @@ def resolved_routes_payload(
|
||||
__all__ = [
|
||||
"Handlers",
|
||||
"MethodNotFoundError",
|
||||
"RouteResolutionError",
|
||||
"dispatch",
|
||||
"resolved_routes_payload",
|
||||
]
|
||||
|
||||
@@ -51,19 +51,24 @@ from __future__ import annotations
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import socketserver
|
||||
import sys
|
||||
import time
|
||||
import typing
|
||||
from dataclasses import dataclass
|
||||
|
||||
from bot_bottle.constants import IDENTITY_HEADER
|
||||
from bot_bottle.gateway.bounded_http import (
|
||||
BodyReadError,
|
||||
BoundedThreadingHTTPServer,
|
||||
read_declared_body,
|
||||
)
|
||||
from bot_bottle.gateway.egress.schema import load_config
|
||||
from bot_bottle.gateway.egress.types import LOG_OFF
|
||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||
from bot_bottle.gateway.supervisor.mcp_dispatch import (
|
||||
Handlers as DispatchHandlers,
|
||||
MethodNotFoundError,
|
||||
RouteResolutionError,
|
||||
dispatch,
|
||||
resolved_routes_payload,
|
||||
)
|
||||
@@ -570,6 +575,8 @@ def format_unknown_proposal_text(proposal_id: str) -> str:
|
||||
# Max request body the server accepts. 1 MB is well above any realistic
|
||||
# routes.yaml proposal.
|
||||
MAX_BODY_BYTES = 1 * 1024 * 1024
|
||||
REQUEST_BODY_TIMEOUT_SECONDS = 10.0
|
||||
MAX_REQUEST_WORKERS = 32
|
||||
|
||||
|
||||
class MCPHandler(http.server.BaseHTTPRequestHandler):
|
||||
@@ -592,19 +599,18 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
|
||||
self._write_text(405, "use POST for MCP requests\n")
|
||||
|
||||
def do_POST(self) -> None:
|
||||
length_header = self.headers.get("Content-Length")
|
||||
if length_header is None:
|
||||
self._write_text(411, "Content-Length required\n")
|
||||
return
|
||||
try:
|
||||
length = int(length_header)
|
||||
except ValueError:
|
||||
self._write_text(400, "invalid Content-Length\n")
|
||||
body = read_declared_body(
|
||||
self.rfile,
|
||||
self.connection,
|
||||
self.headers.get("Content-Length"),
|
||||
maximum=MAX_BODY_BYTES,
|
||||
timeout_seconds=REQUEST_BODY_TIMEOUT_SECONDS,
|
||||
require_length=True,
|
||||
)
|
||||
except BodyReadError as exc:
|
||||
self._write_text(exc.status, exc.message + "\n")
|
||||
return
|
||||
if length < 0 or length > MAX_BODY_BYTES:
|
||||
self._write_text(413, "request body too large\n")
|
||||
return
|
||||
body = self.rfile.read(length)
|
||||
|
||||
try:
|
||||
req = parse_jsonrpc(body)
|
||||
@@ -660,20 +666,25 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
|
||||
identity_token=self._identity_token(),
|
||||
)
|
||||
|
||||
return dispatch(
|
||||
req,
|
||||
DispatchHandlers(
|
||||
initialize=handle_initialize,
|
||||
tools_list=handle_tools_list,
|
||||
list_routes=lambda _params: resolved_routes_payload(
|
||||
def list_routes(_params: dict[str, object]) -> object:
|
||||
try:
|
||||
return resolved_routes_payload(
|
||||
self._resolver_or_fail(),
|
||||
self.client_address[0],
|
||||
self._identity_token(),
|
||||
),
|
||||
check_proposal=check,
|
||||
propose=propose,
|
||||
),
|
||||
)
|
||||
)
|
||||
except RouteResolutionError as exc:
|
||||
raise _RpcInternalError(
|
||||
f"could not resolve live egress routes: {exc}"
|
||||
) from exc
|
||||
|
||||
return dispatch(req, DispatchHandlers(
|
||||
initialize=handle_initialize,
|
||||
tools_list=handle_tools_list,
|
||||
list_routes=list_routes,
|
||||
check_proposal=check,
|
||||
propose=propose,
|
||||
))
|
||||
|
||||
def _identity_token(self) -> str:
|
||||
"""The agent's per-bottle identity token from the request header (the
|
||||
@@ -711,7 +722,7 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
|
||||
self.wfile.write(encoded)
|
||||
|
||||
|
||||
class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||||
class MCPServer(BoundedThreadingHTTPServer):
|
||||
allow_reuse_address = True
|
||||
daemon_threads = True
|
||||
config: ServerConfig = ServerConfig()
|
||||
@@ -720,6 +731,9 @@ class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||||
# closed per request (see `_resolver_or_fail`).
|
||||
policy_resolver: "PolicyResolver | None" = None
|
||||
|
||||
def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def]
|
||||
super().__init__(*args, max_workers=MAX_REQUEST_WORKERS, **kwargs)
|
||||
|
||||
|
||||
# --- Entry point -----------------------------------------------------------
|
||||
|
||||
|
||||
@@ -14,9 +14,12 @@ from __future__ import annotations
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from tests.unit import use_bottle_root
|
||||
from bot_bottle import bottle_state
|
||||
from bot_bottle.backend import EnumerationError
|
||||
from bot_bottle.backend.docker import cleanup
|
||||
from bot_bottle.backend.docker.cleanup import _list_orphan_state_dirs
|
||||
|
||||
|
||||
@@ -116,5 +119,37 @@ class TestOrphanStateDirs(_FakeHomeMixin, unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestAuthoritativeDiscovery(unittest.TestCase):
|
||||
def test_prepare_requires_authoritative_compose_projects(self):
|
||||
with patch.object(cleanup.docker_mod, "require_docker"), \
|
||||
patch.object(
|
||||
cleanup, "list_compose_projects",
|
||||
side_effect=EnumerationError("compose unavailable"),
|
||||
) as projects, self.assertRaisesRegex(
|
||||
EnumerationError, "compose unavailable",
|
||||
):
|
||||
cleanup.prepare_cleanup()
|
||||
projects.assert_called_once_with(
|
||||
warn_on_error=False,
|
||||
raise_on_error=True,
|
||||
)
|
||||
|
||||
def test_container_query_failure_raises(self):
|
||||
failed = cleanup.subprocess.CompletedProcess(
|
||||
[], 1, stdout="", stderr="daemon unavailable",
|
||||
)
|
||||
with patch.object(cleanup.subprocess, "run", return_value=failed), \
|
||||
self.assertRaisesRegex(EnumerationError, "daemon unavailable"):
|
||||
cleanup._list_prefixed_containers()
|
||||
|
||||
def test_network_query_failure_raises(self):
|
||||
failed = cleanup.subprocess.CompletedProcess(
|
||||
[], 1, stdout="", stderr="daemon unavailable",
|
||||
)
|
||||
with patch.object(cleanup.subprocess, "run", return_value=failed), \
|
||||
self.assertRaisesRegex(EnumerationError, "daemon unavailable"):
|
||||
cleanup._list_prefixed_networks()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -32,15 +32,19 @@ class TestProcessScan(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
Path("/cache/run/dev-a"),
|
||||
fc_cleanup._run_dir_of(
|
||||
f"firecracker --config-file {run_root}/dev-a/config.json", run_root
|
||||
("firecracker", "--config-file",
|
||||
f"{run_root}/dev-a/config.json"), run_root
|
||||
),
|
||||
)
|
||||
# infra/builder VMs elsewhere, or nested paths, are not ours.
|
||||
self.assertIsNone(
|
||||
fc_cleanup._run_dir_of("firecracker --config-file /elsewhere/config.json", run_root)
|
||||
fc_cleanup._run_dir_of(
|
||||
("firecracker", "--config-file", "/elsewhere/config.json"),
|
||||
run_root,
|
||||
)
|
||||
)
|
||||
self.assertIsNone(
|
||||
fc_cleanup._run_dir_of("firecracker --no-config", run_root)
|
||||
fc_cleanup._run_dir_of(("firecracker", "--no-config"), run_root)
|
||||
)
|
||||
|
||||
def test_scan_splits_live_dirs_from_orphan_pids(self):
|
||||
@@ -48,17 +52,40 @@ class TestProcessScan(unittest.TestCase):
|
||||
run_root = Path(tmp)
|
||||
(run_root / "live-a").mkdir() # dir present -> live VM, protected
|
||||
# "gone-b" dir intentionally absent -> lingering VMM, orphan pid
|
||||
out = (
|
||||
f"111 firecracker --config-file {run_root}/live-a/config.json\n"
|
||||
f"222 firecracker --config-file {run_root}/gone-b/config.json\n"
|
||||
"333 firecracker --config-file /elsewhere/config.json\n"
|
||||
"notanint firecracker --config-file x\n"
|
||||
)
|
||||
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
|
||||
args = {
|
||||
111: ("firecracker", "--config-file",
|
||||
f"{run_root}/live-a/config.json"),
|
||||
222: ("firecracker", "--config-file",
|
||||
f"{run_root}/gone-b/config.json"),
|
||||
333: ("firecracker", "--config-file", "/elsewhere/config.json"),
|
||||
}
|
||||
with patch.object(
|
||||
fc_cleanup.subprocess, "run",
|
||||
return_value=_proc("111\n222\n333\nnotanint\n"),
|
||||
), patch.object(
|
||||
fc_cleanup, "_process_args", side_effect=args.get,
|
||||
):
|
||||
live, orphan_pids = fc_cleanup._scan_processes(run_root)
|
||||
self.assertEqual({str(run_root / "live-a")}, live)
|
||||
self.assertEqual([222], orphan_pids)
|
||||
|
||||
def test_scan_preserves_spaces_in_config_path(self):
|
||||
with tempfile.TemporaryDirectory(prefix="fc cache ") as tmp:
|
||||
run_root = Path(tmp)
|
||||
live = run_root / "live bottle"
|
||||
live.mkdir()
|
||||
with patch.object(
|
||||
fc_cleanup.subprocess, "run", return_value=_proc("111\n"),
|
||||
), patch.object(
|
||||
fc_cleanup, "_process_args",
|
||||
return_value=("firecracker", "--config-file",
|
||||
str(live / "config.json")),
|
||||
):
|
||||
self.assertEqual(
|
||||
({str(live)}, []),
|
||||
fc_cleanup._scan_processes(run_root),
|
||||
)
|
||||
|
||||
def test_scan_empty_when_pgrep_finds_no_processes(self):
|
||||
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(returncode=1)):
|
||||
self.assertEqual((set(), []), fc_cleanup._scan_processes(Path("/x")))
|
||||
@@ -117,9 +144,14 @@ class TestProcessScan(unittest.TestCase):
|
||||
run_root = Path(tmp)
|
||||
(run_root / "live-a").mkdir()
|
||||
(run_root / "dead-b").mkdir()
|
||||
out = f"111 firecracker --config-file {run_root}/live-a/config.json\n"
|
||||
with patch.object(fc_cleanup, "_run_root", return_value=run_root), \
|
||||
patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
|
||||
patch.object(fc_cleanup.subprocess, "run",
|
||||
return_value=_proc("111\n")), \
|
||||
patch.object(
|
||||
fc_cleanup, "_process_args",
|
||||
return_value=("firecracker", "--config-file",
|
||||
str(run_root / "live-a/config.json")),
|
||||
):
|
||||
plan = fc_cleanup.prepare_cleanup()
|
||||
self.assertEqual((), plan.vm_pids)
|
||||
self.assertEqual((str(run_root / "dead-b"),), plan.run_dirs)
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Unit tests for shared gateway stdlib HTTP resource boundaries."""
|
||||
# pylint: disable=protected-access
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import socket
|
||||
import unittest
|
||||
|
||||
from bot_bottle.gateway.bounded_http import (
|
||||
BodyReadError,
|
||||
BoundedThreadingHTTPServer,
|
||||
read_declared_body,
|
||||
)
|
||||
|
||||
|
||||
class _Handler:
|
||||
pass
|
||||
|
||||
|
||||
class _TimeoutStream:
|
||||
def read(self, _size: int = -1, /) -> bytes:
|
||||
raise TimeoutError
|
||||
|
||||
|
||||
class TestDeclaredBody(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.left, self.right = socket.socketpair()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.left.close()
|
||||
self.right.close()
|
||||
|
||||
def test_rejects_incomplete_body(self) -> None:
|
||||
with self.assertRaisesRegex(BodyReadError, "incomplete"):
|
||||
read_declared_body(
|
||||
io.BytesIO(b"short"),
|
||||
self.left,
|
||||
"10",
|
||||
maximum=100,
|
||||
timeout_seconds=1,
|
||||
require_length=True,
|
||||
)
|
||||
|
||||
def test_maps_read_timeout(self) -> None:
|
||||
with self.assertRaises(BodyReadError) as caught:
|
||||
read_declared_body(
|
||||
_TimeoutStream(),
|
||||
self.left,
|
||||
"1",
|
||||
maximum=100,
|
||||
timeout_seconds=1,
|
||||
require_length=True,
|
||||
)
|
||||
self.assertEqual(408, caught.exception.status)
|
||||
|
||||
|
||||
class TestBoundedServer(unittest.TestCase):
|
||||
def test_saturated_server_rejects_without_spawning_thread(self) -> None:
|
||||
client, peer = socket.socketpair()
|
||||
with BoundedThreadingHTTPServer(
|
||||
("127.0.0.1", 0), _Handler, max_workers=1, # type: ignore[arg-type]
|
||||
) as server:
|
||||
with client, peer:
|
||||
# Directly reserve the only slot to model an in-flight handler.
|
||||
self.assertTrue(
|
||||
server._request_slots.acquire( # pylint: disable=consider-using-with
|
||||
blocking=False,
|
||||
),
|
||||
)
|
||||
try:
|
||||
server.process_request(client, ("127.0.0.1", 1))
|
||||
self.assertIn(b"503 Service Unavailable", peer.recv(1024))
|
||||
finally:
|
||||
server._request_slots.release()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -23,6 +23,7 @@ from bot_bottle.gateway.bootstrap import (
|
||||
_DaemonManager,
|
||||
_argv_for_daemon,
|
||||
_env_for_daemon,
|
||||
_pump,
|
||||
_selected_daemons,
|
||||
)
|
||||
from tests._bin import SLEEP
|
||||
@@ -565,5 +566,26 @@ class TestMainEndToEnd(unittest.TestCase):
|
||||
self.assertIn("no daemons selected", out)
|
||||
|
||||
|
||||
class _FailingStream:
|
||||
def __init__(self, *, closed: bool) -> None:
|
||||
self.closed = closed
|
||||
|
||||
def readline(self) -> bytes:
|
||||
raise ValueError("I/O operation on closed file")
|
||||
|
||||
|
||||
class TestOutputPump(unittest.TestCase):
|
||||
def test_closed_stream_race_is_normal_completion(self) -> None:
|
||||
with patch("bot_bottle.gateway.bootstrap._log") as log:
|
||||
_pump("egress", _FailingStream(closed=True)) # type: ignore[arg-type]
|
||||
log.assert_not_called()
|
||||
|
||||
def test_unexpected_io_failure_is_logged(self) -> None:
|
||||
with patch("bot_bottle.gateway.bootstrap._log") as log:
|
||||
_pump("egress", _FailingStream(closed=False)) # type: ignore[arg-type]
|
||||
log.assert_called_once()
|
||||
self.assertIn("output pump stopped", log.call_args.args[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -284,6 +284,18 @@ class TestEnsureArtifact(_CacheMixin):
|
||||
ia.ensure_artifact_gz(version, role=_ROLE)
|
||||
self.assertEqual(first_calls, len(net.calls))
|
||||
|
||||
def test_download_uses_network_deadline(self) -> None:
|
||||
response = mock.MagicMock()
|
||||
response.__enter__.return_value = io.BytesIO(b"payload")
|
||||
with tempfile.TemporaryDirectory() as d, mock.patch.object(
|
||||
ia.urllib.request, "urlopen", return_value=response,
|
||||
) as urlopen:
|
||||
ia._download("https://registry/artifact", Path(d) / "artifact")
|
||||
self.assertEqual(
|
||||
ia.ARTIFACT_HTTP_TIMEOUT_SECONDS,
|
||||
urlopen.call_args.kwargs["timeout"],
|
||||
)
|
||||
|
||||
def test_checksum_mismatch_fails_closed(self) -> None:
|
||||
version = "beefbeefbeefbeef"
|
||||
gz = _gz(b"payload")
|
||||
|
||||
@@ -49,6 +49,16 @@ class TestPut(unittest.TestCase):
|
||||
self.assertNotIsInstance(req.data, (bytes, bytearray))
|
||||
self.assertEqual(str(len(payload)), req.get_header("Content-length"))
|
||||
|
||||
def test_put_uses_network_deadline(self) -> None:
|
||||
with mock.patch.object(
|
||||
pub.urllib.request, "urlopen", return_value=_Resp(),
|
||||
) as urlopen:
|
||||
pub._put("https://reg/pkg", b"payload", token="")
|
||||
self.assertEqual(
|
||||
pub._REGISTRY_HTTP_TIMEOUT_SECONDS,
|
||||
urlopen.call_args.kwargs["timeout"],
|
||||
)
|
||||
|
||||
def test_small_bytes_body_still_works(self) -> None:
|
||||
captured: list[urllib.request.Request] = []
|
||||
|
||||
|
||||
@@ -717,18 +717,26 @@ class TestResolvedRoutesPayload(unittest.TestCase):
|
||||
hosts = {r["host"] for r in data["routes"]}
|
||||
self.assertEqual({"api.anthropic.com", "www.google.com"}, hosts)
|
||||
|
||||
def test_orchestrator_error_fails_closed_to_empty(self) -> None:
|
||||
# resolve_client_context swallows resolver errors → deny-all (empty),
|
||||
# never another bottle's routes.
|
||||
def test_orchestrator_error_is_not_reported_as_empty_policy(self) -> None:
|
||||
with self.assertRaises(supervise_server.RouteResolutionError):
|
||||
resolved_routes_payload(
|
||||
typing.cast(
|
||||
supervise_server.PolicyResolver,
|
||||
_FakeSuperviseResolver(raises=True),
|
||||
),
|
||||
_SRC,
|
||||
_TOK,
|
||||
)
|
||||
|
||||
def test_authoritative_empty_policy_remains_successful(self) -> None:
|
||||
payload = resolved_routes_payload(
|
||||
typing.cast(
|
||||
supervise_server.PolicyResolver,
|
||||
_FakeSuperviseResolver(raises=True),
|
||||
_FakeSuperviseResolver(bottle_id="b1", policy="routes: []\n"),
|
||||
),
|
||||
_SRC,
|
||||
_TOK,
|
||||
)
|
||||
assert payload is not None
|
||||
data = json.loads(payload["content"][0]["text"]) # type: ignore[index]
|
||||
self.assertEqual([], data["routes"])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user