Compare commits

..

3 Commits

Author SHA1 Message Date
didericis-codex f242733d15 fix(cleanup): use authoritative resource identities
test / integration-docker (pull_request) Has been cancelled
test / image-input-builds (pull_request) Successful in 53s
test / unit (pull_request) Successful in 59s
test / coverage (pull_request) Has been skipped
tracker-policy-pr / check-pr (pull_request) Failing after 12m23s
2026-07-27 04:46:00 +00:00
didericis-codex fd295d4c14 fix(gateway): contain output pump shutdown races
test / image-input-builds (pull_request) Failing after 12m49s
test / unit (pull_request) Has started running
test / coverage (pull_request) Blocked by required conditions
test / integration-docker (pull_request) Blocked by required conditions
tracker-policy-pr / check-pr (pull_request) Successful in 13s
2026-07-27 04:46:00 +00:00
didericis-codex f33566941b fix(gateway): bound stdlib HTTP request work
test / image-input-builds (pull_request) Successful in 1m2s
test / unit (pull_request) Has started running
test / coverage (pull_request) Blocked by required conditions
test / integration-docker (pull_request) Blocked by required conditions
tracker-policy-pr / check-pr (pull_request) Failing after 12m29s
2026-07-27 04:46:00 +00:00
16 changed files with 499 additions and 104 deletions
+27 -17
View File
@@ -28,6 +28,7 @@ import subprocess
from ...paths import bot_bottle_root
from ...log import info, warn
from .. import EnumerationError
from . import util as docker_mod
from .bottle_cleanup_plan import DockerBottleCleanupPlan
from ...bottle_state import bottle_state_dir, is_preserved
@@ -36,15 +37,17 @@ from .compose import COMPOSE_PROJECT_PREFIX, list_compose_projects
def _list_prefixed_containers() -> list[str]:
"""All bot-bottle-prefixed containers, running or stopped."""
result = subprocess.run(
["docker", "ps", "-a",
"--filter", f"name=^{COMPOSE_PROJECT_PREFIX}",
"--format", "{{.Names}}\t{{.Label \"com.docker.compose.project\"}}"],
capture_output=True, text=True, check=False,
)
try:
result = subprocess.run(
["docker", "ps", "-a",
"--filter", f"name=^{COMPOSE_PROJECT_PREFIX}",
"--format", "{{.Names}}\t{{.Label \"com.docker.compose.project\"}}"],
capture_output=True, text=True, check=False,
)
except OSError as exc:
raise EnumerationError(f"docker ps failed: {exc}") from exc
if result.returncode != 0:
warn(f"docker ps failed: {result.stderr.strip()}")
return []
raise EnumerationError(f"docker ps failed: {result.stderr.strip()}")
out: list[str] = []
for line in (result.stdout or "").splitlines():
if not line:
@@ -63,15 +66,19 @@ def _list_prefixed_networks() -> list[str]:
to a compose project. Compose-managed networks have a
`com.docker.compose.project` label; bare ones (from pre-compose
code paths) don't."""
result = subprocess.run(
["docker", "network", "ls",
"--filter", f"name={COMPOSE_PROJECT_PREFIX}",
"--format", "{{.Name}}\t{{.Label \"com.docker.compose.project\"}}"],
capture_output=True, text=True, check=False,
)
try:
result = subprocess.run(
["docker", "network", "ls",
"--filter", f"name={COMPOSE_PROJECT_PREFIX}",
"--format", "{{.Name}}\t{{.Label \"com.docker.compose.project\"}}"],
capture_output=True, text=True, check=False,
)
except OSError as exc:
raise EnumerationError(f"docker network ls failed: {exc}") from exc
if result.returncode != 0:
warn(f"docker network ls failed: {result.stderr.strip()}")
return []
raise EnumerationError(
f"docker network ls failed: {result.stderr.strip()}"
)
out: list[str] = []
for line in (result.stdout or "").splitlines():
if not line:
@@ -120,7 +127,10 @@ def prepare_cleanup() -> DockerBottleCleanupPlan:
`enumerate_active_agents()` so the orphan-state-dir bucket
doesn't include slugs whose non-docker bottle is still up."""
docker_mod.require_docker()
projects = list_compose_projects()
projects = list_compose_projects(
warn_on_error=False,
raise_on_error=True,
)
project_set = set(projects)
# Late import to avoid a circular at module-load time —
# the backend package's __init__ imports this module.
+37 -17
View File
@@ -11,10 +11,9 @@ Reaps *orphans* only — resources with no live VM behind them:
— a VMM left lingering after its dir was removed.
A run dir with a *live* firecracker process is a running bottle and is
left strictly alone: it is neither killed nor removed. (The backend's
`enumerate_active` registry is still a stub — #354 — so a live process
is the only reliable "this bottle is in use" signal we have. Once the
registry lands, registry-orphaned-but-running VMs can be reaped too.)
left strictly alone: it is neither killed nor removed. Active-agent
enumeration uses this same process snapshot, so cleanup and generic
backend consumers agree about which bottles are running.
TAP slots free themselves (the flock drops when the launcher exits), so
there is nothing to reclaim there.
@@ -22,6 +21,7 @@ there is nothing to reclaim there.
from __future__ import annotations
from collections.abc import Sequence
import os
import shutil
import signal
@@ -38,7 +38,7 @@ def _run_root() -> Path:
return util.cache_dir() / "run"
def _run_dir_of(cmd: str, run_root: Path) -> Path | None:
def _run_dir_of(args: Sequence[str], run_root: Path) -> Path | None:
"""The bottle run dir a firecracker cmdline belongs to, or None.
A bottle VM is launched with `--config-file <run_root>/<slug>/config.json`,
@@ -46,15 +46,35 @@ def _run_dir_of(cmd: str, run_root: Path) -> Path | None:
the run root. Anything else (a builder VM, the infra VM elsewhere) is
not ours to reap here.
"""
toks = cmd.split()
for i, tok in enumerate(toks):
if tok == "--config-file" and i + 1 < len(toks):
parent = Path(toks[i + 1]).parent
for i, arg in enumerate(args):
if arg == "--config-file" and i + 1 < len(args):
parent = Path(args[i + 1]).parent
if parent.parent == run_root:
return parent
return None
def _decode_cmdline(raw: bytes) -> tuple[str, ...]:
"""Decode Linux's NUL-delimited argv without losing embedded spaces."""
return tuple(
value.decode(errors="surrogateescape")
for value in raw.split(b"\0") if value
)
def _process_args(pid: int) -> tuple[str, ...] | None:
"""Read one process's lossless argv, or None when it exited meanwhile."""
try:
raw = Path(f"/proc/{pid}/cmdline").read_bytes()
except FileNotFoundError:
return None
except OSError as exc:
raise EnumerationError(
f"could not inspect Firecracker pid {pid}: {exc}"
) from exc
return _decode_cmdline(raw)
def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
"""Inspect running firecracker VMs under ``run_root``.
@@ -65,7 +85,7 @@ def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
"""
try:
result = subprocess.run(
["pgrep", "-a", "firecracker"],
["pgrep", "firecracker"],
capture_output=True, text=True, check=False,
)
except OSError as exc:
@@ -83,14 +103,14 @@ def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
live: set[str] = set()
orphan_pids: list[int] = []
for line in result.stdout.splitlines():
parts = line.split(None, 1)
if len(parts) != 2:
continue
try:
pid = int(parts[0])
pid = int(line.strip())
except ValueError:
continue
run_dir = _run_dir_of(parts[1], run_root)
args = _process_args(pid)
if args is None:
continue
run_dir = _run_dir_of(args, run_root)
if run_dir is None:
continue
if run_dir.is_dir():
@@ -157,8 +177,8 @@ def _terminate_orphan(pid: int, run_root: Path) -> None:
raise EnumerationError(
f"could not revalidate Firecracker pid {pid}: {exc}"
) from exc
command = raw.replace(b"\0", b" ").decode(errors="replace")
run_dir = _run_dir_of(command, run_root)
args = _decode_cmdline(raw)
run_dir = _run_dir_of(args, run_root)
if run_dir is None or run_dir.is_dir():
return
info(f"kill firecracker VM pid {pid}")
@@ -41,6 +41,8 @@ from ... import resources
from ...log import die, info
from . import util
ARTIFACT_HTTP_TIMEOUT_SECONDS = 30.0
# Bump if the on-disk artifact *format* changes (compression, layout) so a new
# scheme can't collide with a cached/published artifact of the old one.
_ARTIFACT_FORMAT = "1"
@@ -164,7 +166,9 @@ def _download(url: str, dest: Path) -> None:
"""Stream `url` to `dest` (atomic via a `.part` sibling)."""
tmp = dest.with_suffix(dest.suffix + ".part")
try:
with urllib.request.urlopen(_open(url)) as resp, open(tmp, "wb") as out:
with urllib.request.urlopen(
_open(url), timeout=ARTIFACT_HTTP_TIMEOUT_SECONDS,
) as resp, open(tmp, "wb") as out:
shutil.copyfileobj(resp, out, _CHUNK)
except urllib.error.HTTPError as e:
tmp.unlink(missing_ok=True)
@@ -34,6 +34,7 @@ from pathlib import Path
from . import infra_artifact, infra_vm, util
_CHUNK = 1 << 20
_REGISTRY_HTTP_TIMEOUT_SECONDS = 30.0
_GZ_NAME = "rootfs.ext4.gz"
_SHA_NAME = "rootfs.ext4.gz.sha256"
@@ -91,7 +92,9 @@ def _put(url: str, body: "bytes | Path", token: str) -> None:
req.add_header("Authorization", f"token {token}")
req.add_header("Content-Type", "application/octet-stream")
try:
with urllib.request.urlopen(req) as resp:
with urllib.request.urlopen(
req, timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
) as resp:
print(f" uploaded {url} (HTTP {resp.status})")
except urllib.error.HTTPError as e:
if e.code == 409:
@@ -112,7 +115,9 @@ def _delete(url: str, token: str) -> None:
if token:
req.add_header("Authorization", f"token {token}")
try:
with urllib.request.urlopen(req):
with urllib.request.urlopen(
req, timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
):
pass
except urllib.error.HTTPError as e:
if e.code != 404:
@@ -151,7 +156,10 @@ def _try_download_published(role: str, role_dir: Path) -> str | None:
version = _role_version(role)
sha_url = infra_artifact.artifact_url(version, _SHA_NAME, role=role)
try:
with urllib.request.urlopen(infra_artifact._open(sha_url)):
with urllib.request.urlopen(
infra_artifact._open(sha_url),
timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
):
pass
except urllib.error.HTTPError as e:
if e.code == 404:
@@ -195,7 +203,10 @@ def _publish_bundle(role: str, role_dir: Path, token: str) -> str:
# present, a re-publish is a no-op. Otherwise clear any partial upload left
# by an interrupted prior attempt and upload the complete set.
try:
with urllib.request.urlopen(infra_artifact._open(sha_url)) as resp:
with urllib.request.urlopen(
infra_artifact._open(sha_url),
timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
) as resp:
remote_sha = resp.read().decode("utf-8").split()[0].strip().lower()
except urllib.error.HTTPError as e:
if e.code != 404:
+12 -4
View File
@@ -136,10 +136,18 @@ def _pump(name: str, stream: IO[bytes]) -> None:
"""Read lines from `stream`, prefix with `[name]`, write to
stdout. Runs in its own thread per child; daemon=True so a
blocked read doesn't keep the process alive after main exits."""
for raw in iter(stream.readline, b""):
line = raw.decode("utf-8", errors="replace").rstrip("\n")
sys.stdout.write(f"[{name}] {line}\n")
sys.stdout.flush()
try:
for raw in iter(stream.readline, b""):
line = raw.decode("utf-8", errors="replace").rstrip("\n")
sys.stdout.write(f"[{name}] {line}\n")
sys.stdout.flush()
except (OSError, ValueError) as exc:
# The manager closes a dead child's pipe after wait() and before a
# restart. A pump can be between readline calls at that exact moment;
# closed-stream errors are normal completion, not uncaught thread
# failures. Preserve genuinely unexpected I/O diagnostics.
if not stream.closed:
_log(f"{name} output pump stopped: {type(exc).__name__}: {exc}")
def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]:
+109
View File
@@ -0,0 +1,109 @@
"""Shared resource boundaries for gateway stdlib HTTP services."""
from __future__ import annotations
import http.server
import socket
import threading
import time
from dataclasses import dataclass
from typing import Any, Protocol
class Readable(Protocol):
def read(self, size: int = -1, /) -> bytes: ...
@dataclass(frozen=True)
class BodyReadError(Exception):
status: int
message: str
def read_declared_body(
stream: Readable,
connection: socket.socket,
raw_length: str | None,
*,
maximum: int,
timeout_seconds: float,
require_length: bool,
) -> bytes:
"""Validate and read exactly one declared body under a read deadline."""
if raw_length is None:
if require_length:
raise BodyReadError(411, "Content-Length required")
raw_length = "0"
try:
length = int(raw_length)
except ValueError as exc:
raise BodyReadError(400, "invalid Content-Length") from exc
if length < 0:
raise BodyReadError(400, "invalid Content-Length")
if length > maximum:
raise BodyReadError(413, "request body too large")
previous_timeout = connection.gettimeout()
deadline = time.monotonic() + timeout_seconds
chunks: list[bytes] = []
remaining = length
try:
while remaining:
timeout = deadline - time.monotonic()
if timeout <= 0:
raise BodyReadError(408, "request body read timed out")
connection.settimeout(timeout)
chunk = stream.read(min(remaining, 64 * 1024))
if not chunk:
raise BodyReadError(400, "incomplete request body")
chunks.append(chunk)
remaining -= len(chunk)
except TimeoutError as exc:
raise BodyReadError(408, "request body read timed out") from exc
finally:
connection.settimeout(previous_timeout)
return b"".join(chunks)
class BoundedThreadingHTTPServer(http.server.ThreadingHTTPServer):
"""ThreadingHTTPServer with a hard cap on in-flight request threads."""
daemon_threads = True
def __init__( # pylint: disable=consider-using-with
self, *args, max_workers: int = 32, **kwargs, # type: ignore[no-untyped-def]
):
if max_workers < 1:
raise ValueError("max_workers must be positive")
self._request_slots = threading.BoundedSemaphore(max_workers)
super().__init__(*args, **kwargs)
def process_request(
self, request: Any, client_address: Any,
) -> None:
if not self._request_slots.acquire( # pylint: disable=consider-using-with
blocking=False,
):
try:
request.sendall(
b"HTTP/1.1 503 Service Unavailable\r\n"
b"Content-Length: 0\r\nConnection: close\r\n\r\n"
)
finally:
self.shutdown_request(request)
return
try:
super().process_request(request, client_address)
except BaseException:
self._request_slots.release()
raise
def process_request_thread(
self, request: Any, client_address: Any,
) -> None:
try:
super().process_request_thread(request, client_address)
finally:
self._request_slots.release()
__all__ = ["BodyReadError", "BoundedThreadingHTTPServer", "read_declared_body"]
+21 -13
View File
@@ -22,11 +22,16 @@ import os
import subprocess
import sys
import typing
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from http.server import BaseHTTPRequestHandler
from pathlib import Path
from urllib.parse import urlsplit
from bot_bottle.constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER
from bot_bottle.gateway.bounded_http import (
BodyReadError,
BoundedThreadingHTTPServer,
read_declared_body,
)
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
@@ -77,6 +82,8 @@ def resolve_sandbox_root(
# Bound memory use while still allowing ordinary git push packfiles.
MAX_BODY_BYTES = 100 * 1024 * 1024
REQUEST_BODY_TIMEOUT_SECONDS = 30.0
MAX_REQUEST_WORKERS = 16
class GitHttpHandler(BaseHTTPRequestHandler):
@@ -184,19 +191,18 @@ class GitHttpHandler(BaseHTTPRequestHandler):
value = self.headers.get(header)
if value:
env[variable] = value
raw_length = self.headers.get("content-length", "0") or "0"
try:
length = int(raw_length)
except ValueError:
self.send_error(400, "Bad Content-Length")
body = read_declared_body(
self.rfile,
self.connection,
self.headers.get("content-length"),
maximum=MAX_BODY_BYTES,
timeout_seconds=REQUEST_BODY_TIMEOUT_SECONDS,
require_length=False,
)
except BodyReadError as exc:
self.send_error(exc.status, exc.message)
return
if length < 0:
self.send_error(400, "Negative Content-Length")
return
if length > MAX_BODY_BYTES:
self.send_error(413, "Request body too large")
return
body = self.rfile.read(length) if length else b""
proc = subprocess.run(
["git", "http-backend"],
input=body,
@@ -273,7 +279,9 @@ def main() -> int:
"(no single-tenant flat-root fallback)\n"
)
return 1
server = ThreadingHTTPServer(("0.0.0.0", port), GitHttpHandler)
server = BoundedThreadingHTTPServer(
("0.0.0.0", port), GitHttpHandler, max_workers=MAX_REQUEST_WORKERS,
)
# Resolve each request's sandbox namespace by source IP against the
# orchestrator control plane.
server.policy_resolver = PolicyResolver(orch_url) # type: ignore[attr-defined]
+20 -7
View File
@@ -6,9 +6,8 @@ import json
from dataclasses import dataclass
from typing import Callable, Protocol
from bot_bottle.gateway.egress.context import resolve_client_context
from bot_bottle.gateway.egress.schema import route_to_yaml_dict
from bot_bottle.gateway.policy_resolver import PolicyResolver
from bot_bottle.gateway.egress.schema import load_config, route_to_yaml_dict
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
from bot_bottle.supervisor import types as _sv
@@ -24,6 +23,10 @@ class MethodNotFoundError(Exception):
"""Raised when a JSON-RPC method has no MCP handler."""
class RouteResolutionError(Exception):
"""The caller's live route table could not be resolved authoritatively."""
Handler = Callable[[dict[str, object]], object]
@@ -60,10 +63,19 @@ def resolved_routes_payload(
source_ip: str,
identity_token: str,
) -> dict[str, object]:
"""Render the calling bottle's routes, failing closed to an empty list."""
config, _slug, _tokens = resolve_client_context(
resolver, source_ip, identity_token,
)
"""Render an authoritatively resolved route table for the calling bottle."""
try:
policy, bottle_id, _tokens = resolver.resolve_policy_and_bottle_id(
source_ip, identity_token,
)
except PolicyResolveError as exc:
raise RouteResolutionError("orchestrator unavailable") from exc
if not bottle_id:
raise RouteResolutionError("request source is not attributed to a bottle")
try:
config = load_config(policy or "")
except ValueError as exc:
raise RouteResolutionError("resolved policy is invalid") from exc
body = json.dumps(
{"routes": [route_to_yaml_dict(route) for route in config.routes]},
indent=2,
@@ -74,6 +86,7 @@ def resolved_routes_payload(
__all__ = [
"Handlers",
"MethodNotFoundError",
"RouteResolutionError",
"dispatch",
"resolved_routes_payload",
]
+38 -24
View File
@@ -51,19 +51,24 @@ from __future__ import annotations
import http.server
import json
import os
import socketserver
import sys
import time
import typing
from dataclasses import dataclass
from bot_bottle.constants import IDENTITY_HEADER
from bot_bottle.gateway.bounded_http import (
BodyReadError,
BoundedThreadingHTTPServer,
read_declared_body,
)
from bot_bottle.gateway.egress.schema import load_config
from bot_bottle.gateway.egress.types import LOG_OFF
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
from bot_bottle.gateway.supervisor.mcp_dispatch import (
Handlers as DispatchHandlers,
MethodNotFoundError,
RouteResolutionError,
dispatch,
resolved_routes_payload,
)
@@ -570,6 +575,8 @@ def format_unknown_proposal_text(proposal_id: str) -> str:
# Max request body the server accepts. 1 MB is well above any realistic
# routes.yaml proposal.
MAX_BODY_BYTES = 1 * 1024 * 1024
REQUEST_BODY_TIMEOUT_SECONDS = 10.0
MAX_REQUEST_WORKERS = 32
class MCPHandler(http.server.BaseHTTPRequestHandler):
@@ -592,19 +599,18 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
self._write_text(405, "use POST for MCP requests\n")
def do_POST(self) -> None:
length_header = self.headers.get("Content-Length")
if length_header is None:
self._write_text(411, "Content-Length required\n")
return
try:
length = int(length_header)
except ValueError:
self._write_text(400, "invalid Content-Length\n")
body = read_declared_body(
self.rfile,
self.connection,
self.headers.get("Content-Length"),
maximum=MAX_BODY_BYTES,
timeout_seconds=REQUEST_BODY_TIMEOUT_SECONDS,
require_length=True,
)
except BodyReadError as exc:
self._write_text(exc.status, exc.message + "\n")
return
if length < 0 or length > MAX_BODY_BYTES:
self._write_text(413, "request body too large\n")
return
body = self.rfile.read(length)
try:
req = parse_jsonrpc(body)
@@ -660,20 +666,25 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
identity_token=self._identity_token(),
)
return dispatch(
req,
DispatchHandlers(
initialize=handle_initialize,
tools_list=handle_tools_list,
list_routes=lambda _params: resolved_routes_payload(
def list_routes(_params: dict[str, object]) -> object:
try:
return resolved_routes_payload(
self._resolver_or_fail(),
self.client_address[0],
self._identity_token(),
),
check_proposal=check,
propose=propose,
),
)
)
except RouteResolutionError as exc:
raise _RpcInternalError(
f"could not resolve live egress routes: {exc}"
) from exc
return dispatch(req, DispatchHandlers(
initialize=handle_initialize,
tools_list=handle_tools_list,
list_routes=list_routes,
check_proposal=check,
propose=propose,
))
def _identity_token(self) -> str:
"""The agent's per-bottle identity token from the request header (the
@@ -711,7 +722,7 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
self.wfile.write(encoded)
class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
class MCPServer(BoundedThreadingHTTPServer):
allow_reuse_address = True
daemon_threads = True
config: ServerConfig = ServerConfig()
@@ -720,6 +731,9 @@ class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
# closed per request (see `_resolver_or_fail`).
policy_resolver: "PolicyResolver | None" = None
def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def]
super().__init__(*args, max_workers=MAX_REQUEST_WORKERS, **kwargs)
# --- Entry point -----------------------------------------------------------
+35
View File
@@ -14,9 +14,12 @@ from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from tests.unit import use_bottle_root
from bot_bottle import bottle_state
from bot_bottle.backend import EnumerationError
from bot_bottle.backend.docker import cleanup
from bot_bottle.backend.docker.cleanup import _list_orphan_state_dirs
@@ -116,5 +119,37 @@ class TestOrphanStateDirs(_FakeHomeMixin, unittest.TestCase):
)
class TestAuthoritativeDiscovery(unittest.TestCase):
def test_prepare_requires_authoritative_compose_projects(self):
with patch.object(cleanup.docker_mod, "require_docker"), \
patch.object(
cleanup, "list_compose_projects",
side_effect=EnumerationError("compose unavailable"),
) as projects, self.assertRaisesRegex(
EnumerationError, "compose unavailable",
):
cleanup.prepare_cleanup()
projects.assert_called_once_with(
warn_on_error=False,
raise_on_error=True,
)
def test_container_query_failure_raises(self):
failed = cleanup.subprocess.CompletedProcess(
[], 1, stdout="", stderr="daemon unavailable",
)
with patch.object(cleanup.subprocess, "run", return_value=failed), \
self.assertRaisesRegex(EnumerationError, "daemon unavailable"):
cleanup._list_prefixed_containers()
def test_network_query_failure_raises(self):
failed = cleanup.subprocess.CompletedProcess(
[], 1, stdout="", stderr="daemon unavailable",
)
with patch.object(cleanup.subprocess, "run", return_value=failed), \
self.assertRaisesRegex(EnumerationError, "daemon unavailable"):
cleanup._list_prefixed_networks()
if __name__ == "__main__":
unittest.main()
+44 -12
View File
@@ -32,15 +32,19 @@ class TestProcessScan(unittest.TestCase):
self.assertEqual(
Path("/cache/run/dev-a"),
fc_cleanup._run_dir_of(
f"firecracker --config-file {run_root}/dev-a/config.json", run_root
("firecracker", "--config-file",
f"{run_root}/dev-a/config.json"), run_root
),
)
# infra/builder VMs elsewhere, or nested paths, are not ours.
self.assertIsNone(
fc_cleanup._run_dir_of("firecracker --config-file /elsewhere/config.json", run_root)
fc_cleanup._run_dir_of(
("firecracker", "--config-file", "/elsewhere/config.json"),
run_root,
)
)
self.assertIsNone(
fc_cleanup._run_dir_of("firecracker --no-config", run_root)
fc_cleanup._run_dir_of(("firecracker", "--no-config"), run_root)
)
def test_scan_splits_live_dirs_from_orphan_pids(self):
@@ -48,17 +52,40 @@ class TestProcessScan(unittest.TestCase):
run_root = Path(tmp)
(run_root / "live-a").mkdir() # dir present -> live VM, protected
# "gone-b" dir intentionally absent -> lingering VMM, orphan pid
out = (
f"111 firecracker --config-file {run_root}/live-a/config.json\n"
f"222 firecracker --config-file {run_root}/gone-b/config.json\n"
"333 firecracker --config-file /elsewhere/config.json\n"
"notanint firecracker --config-file x\n"
)
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
args = {
111: ("firecracker", "--config-file",
f"{run_root}/live-a/config.json"),
222: ("firecracker", "--config-file",
f"{run_root}/gone-b/config.json"),
333: ("firecracker", "--config-file", "/elsewhere/config.json"),
}
with patch.object(
fc_cleanup.subprocess, "run",
return_value=_proc("111\n222\n333\nnotanint\n"),
), patch.object(
fc_cleanup, "_process_args", side_effect=args.get,
):
live, orphan_pids = fc_cleanup._scan_processes(run_root)
self.assertEqual({str(run_root / "live-a")}, live)
self.assertEqual([222], orphan_pids)
def test_scan_preserves_spaces_in_config_path(self):
with tempfile.TemporaryDirectory(prefix="fc cache ") as tmp:
run_root = Path(tmp)
live = run_root / "live bottle"
live.mkdir()
with patch.object(
fc_cleanup.subprocess, "run", return_value=_proc("111\n"),
), patch.object(
fc_cleanup, "_process_args",
return_value=("firecracker", "--config-file",
str(live / "config.json")),
):
self.assertEqual(
({str(live)}, []),
fc_cleanup._scan_processes(run_root),
)
def test_scan_empty_when_pgrep_finds_no_processes(self):
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(returncode=1)):
self.assertEqual((set(), []), fc_cleanup._scan_processes(Path("/x")))
@@ -117,9 +144,14 @@ class TestProcessScan(unittest.TestCase):
run_root = Path(tmp)
(run_root / "live-a").mkdir()
(run_root / "dead-b").mkdir()
out = f"111 firecracker --config-file {run_root}/live-a/config.json\n"
with patch.object(fc_cleanup, "_run_root", return_value=run_root), \
patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
patch.object(fc_cleanup.subprocess, "run",
return_value=_proc("111\n")), \
patch.object(
fc_cleanup, "_process_args",
return_value=("firecracker", "--config-file",
str(run_root / "live-a/config.json")),
):
plan = fc_cleanup.prepare_cleanup()
self.assertEqual((), plan.vm_pids)
self.assertEqual((str(run_root / "dead-b"),), plan.run_dirs)
+79
View File
@@ -0,0 +1,79 @@
"""Unit tests for shared gateway stdlib HTTP resource boundaries."""
# pylint: disable=protected-access
from __future__ import annotations
import io
import socket
import unittest
from bot_bottle.gateway.bounded_http import (
BodyReadError,
BoundedThreadingHTTPServer,
read_declared_body,
)
class _Handler:
pass
class _TimeoutStream:
def read(self, _size: int = -1, /) -> bytes:
raise TimeoutError
class TestDeclaredBody(unittest.TestCase):
def setUp(self) -> None:
self.left, self.right = socket.socketpair()
def tearDown(self) -> None:
self.left.close()
self.right.close()
def test_rejects_incomplete_body(self) -> None:
with self.assertRaisesRegex(BodyReadError, "incomplete"):
read_declared_body(
io.BytesIO(b"short"),
self.left,
"10",
maximum=100,
timeout_seconds=1,
require_length=True,
)
def test_maps_read_timeout(self) -> None:
with self.assertRaises(BodyReadError) as caught:
read_declared_body(
_TimeoutStream(),
self.left,
"1",
maximum=100,
timeout_seconds=1,
require_length=True,
)
self.assertEqual(408, caught.exception.status)
class TestBoundedServer(unittest.TestCase):
def test_saturated_server_rejects_without_spawning_thread(self) -> None:
client, peer = socket.socketpair()
with BoundedThreadingHTTPServer(
("127.0.0.1", 0), _Handler, max_workers=1, # type: ignore[arg-type]
) as server:
with client, peer:
# Directly reserve the only slot to model an in-flight handler.
self.assertTrue(
server._request_slots.acquire( # pylint: disable=consider-using-with
blocking=False,
),
)
try:
server.process_request(client, ("127.0.0.1", 1))
self.assertIn(b"503 Service Unavailable", peer.recv(1024))
finally:
server._request_slots.release()
if __name__ == "__main__":
unittest.main()
+22
View File
@@ -23,6 +23,7 @@ from bot_bottle.gateway.bootstrap import (
_DaemonManager,
_argv_for_daemon,
_env_for_daemon,
_pump,
_selected_daemons,
)
from tests._bin import SLEEP
@@ -565,5 +566,26 @@ class TestMainEndToEnd(unittest.TestCase):
self.assertIn("no daemons selected", out)
class _FailingStream:
def __init__(self, *, closed: bool) -> None:
self.closed = closed
def readline(self) -> bytes:
raise ValueError("I/O operation on closed file")
class TestOutputPump(unittest.TestCase):
def test_closed_stream_race_is_normal_completion(self) -> None:
with patch("bot_bottle.gateway.bootstrap._log") as log:
_pump("egress", _FailingStream(closed=True)) # type: ignore[arg-type]
log.assert_not_called()
def test_unexpected_io_failure_is_logged(self) -> None:
with patch("bot_bottle.gateway.bootstrap._log") as log:
_pump("egress", _FailingStream(closed=False)) # type: ignore[arg-type]
log.assert_called_once()
self.assertIn("output pump stopped", log.call_args.args[0])
if __name__ == "__main__":
unittest.main()
+12
View File
@@ -284,6 +284,18 @@ class TestEnsureArtifact(_CacheMixin):
ia.ensure_artifact_gz(version, role=_ROLE)
self.assertEqual(first_calls, len(net.calls))
def test_download_uses_network_deadline(self) -> None:
response = mock.MagicMock()
response.__enter__.return_value = io.BytesIO(b"payload")
with tempfile.TemporaryDirectory() as d, mock.patch.object(
ia.urllib.request, "urlopen", return_value=response,
) as urlopen:
ia._download("https://registry/artifact", Path(d) / "artifact")
self.assertEqual(
ia.ARTIFACT_HTTP_TIMEOUT_SECONDS,
urlopen.call_args.kwargs["timeout"],
)
def test_checksum_mismatch_fails_closed(self) -> None:
version = "beefbeefbeefbeef"
gz = _gz(b"payload")
+10
View File
@@ -49,6 +49,16 @@ class TestPut(unittest.TestCase):
self.assertNotIsInstance(req.data, (bytes, bytearray))
self.assertEqual(str(len(payload)), req.get_header("Content-length"))
def test_put_uses_network_deadline(self) -> None:
with mock.patch.object(
pub.urllib.request, "urlopen", return_value=_Resp(),
) as urlopen:
pub._put("https://reg/pkg", b"payload", token="")
self.assertEqual(
pub._REGISTRY_HTTP_TIMEOUT_SECONDS,
urlopen.call_args.kwargs["timeout"],
)
def test_small_bytes_body_still_works(self) -> None:
captured: list[urllib.request.Request] = []
+13 -5
View File
@@ -717,18 +717,26 @@ class TestResolvedRoutesPayload(unittest.TestCase):
hosts = {r["host"] for r in data["routes"]}
self.assertEqual({"api.anthropic.com", "www.google.com"}, hosts)
def test_orchestrator_error_fails_closed_to_empty(self) -> None:
# resolve_client_context swallows resolver errors → deny-all (empty),
# never another bottle's routes.
def test_orchestrator_error_is_not_reported_as_empty_policy(self) -> None:
with self.assertRaises(supervise_server.RouteResolutionError):
resolved_routes_payload(
typing.cast(
supervise_server.PolicyResolver,
_FakeSuperviseResolver(raises=True),
),
_SRC,
_TOK,
)
def test_authoritative_empty_policy_remains_successful(self) -> None:
payload = resolved_routes_payload(
typing.cast(
supervise_server.PolicyResolver,
_FakeSuperviseResolver(raises=True),
_FakeSuperviseResolver(bottle_id="b1", policy="routes: []\n"),
),
_SRC,
_TOK,
)
assert payload is not None
data = json.loads(payload["content"][0]["text"]) # type: ignore[index]
self.assertEqual([], data["routes"])