Compare commits

..

4 Commits

Author SHA1 Message Date
didericis-codex f242733d15 fix(cleanup): use authoritative resource identities
test / integration-docker (pull_request) Has been cancelled
test / image-input-builds (pull_request) Successful in 53s
test / unit (pull_request) Successful in 59s
test / coverage (pull_request) Has been skipped
tracker-policy-pr / check-pr (pull_request) Failing after 12m23s
2026-07-27 04:46:00 +00:00
didericis-codex fd295d4c14 fix(gateway): contain output pump shutdown races
test / image-input-builds (pull_request) Failing after 12m49s
test / unit (pull_request) Has started running
test / coverage (pull_request) Blocked by required conditions
test / integration-docker (pull_request) Blocked by required conditions
tracker-policy-pr / check-pr (pull_request) Successful in 13s
2026-07-27 04:46:00 +00:00
didericis-codex f33566941b fix(gateway): bound stdlib HTTP request work
test / image-input-builds (pull_request) Successful in 1m2s
test / unit (pull_request) Has started running
test / coverage (pull_request) Blocked by required conditions
test / integration-docker (pull_request) Blocked by required conditions
tracker-policy-pr / check-pr (pull_request) Failing after 12m29s
2026-07-27 04:46:00 +00:00
didericis-codex c7c3a79028 fix(cleanup): revalidate destructive backend plans
test / image-input-builds (pull_request) Failing after 13m11s
test / unit (pull_request) Has started running
test / coverage (pull_request) Blocked by required conditions
test / integration-docker (pull_request) Blocked by required conditions
tracker-policy-pr / check-pr (pull_request) Successful in 13s
2026-07-27 04:45:59 +00:00
22 changed files with 673 additions and 145 deletions
+27 -17
View File
@@ -28,6 +28,7 @@ import subprocess
from ...paths import bot_bottle_root from ...paths import bot_bottle_root
from ...log import info, warn from ...log import info, warn
from .. import EnumerationError
from . import util as docker_mod from . import util as docker_mod
from .bottle_cleanup_plan import DockerBottleCleanupPlan from .bottle_cleanup_plan import DockerBottleCleanupPlan
from ...bottle_state import bottle_state_dir, is_preserved from ...bottle_state import bottle_state_dir, is_preserved
@@ -36,15 +37,17 @@ from .compose import COMPOSE_PROJECT_PREFIX, list_compose_projects
def _list_prefixed_containers() -> list[str]: def _list_prefixed_containers() -> list[str]:
"""All bot-bottle-prefixed containers, running or stopped.""" """All bot-bottle-prefixed containers, running or stopped."""
result = subprocess.run( try:
["docker", "ps", "-a", result = subprocess.run(
"--filter", f"name=^{COMPOSE_PROJECT_PREFIX}", ["docker", "ps", "-a",
"--format", "{{.Names}}\t{{.Label \"com.docker.compose.project\"}}"], "--filter", f"name=^{COMPOSE_PROJECT_PREFIX}",
capture_output=True, text=True, check=False, "--format", "{{.Names}}\t{{.Label \"com.docker.compose.project\"}}"],
) capture_output=True, text=True, check=False,
)
except OSError as exc:
raise EnumerationError(f"docker ps failed: {exc}") from exc
if result.returncode != 0: if result.returncode != 0:
warn(f"docker ps failed: {result.stderr.strip()}") raise EnumerationError(f"docker ps failed: {result.stderr.strip()}")
return []
out: list[str] = [] out: list[str] = []
for line in (result.stdout or "").splitlines(): for line in (result.stdout or "").splitlines():
if not line: if not line:
@@ -63,15 +66,19 @@ def _list_prefixed_networks() -> list[str]:
to a compose project. Compose-managed networks have a to a compose project. Compose-managed networks have a
`com.docker.compose.project` label; bare ones (from pre-compose `com.docker.compose.project` label; bare ones (from pre-compose
code paths) don't.""" code paths) don't."""
result = subprocess.run( try:
["docker", "network", "ls", result = subprocess.run(
"--filter", f"name={COMPOSE_PROJECT_PREFIX}", ["docker", "network", "ls",
"--format", "{{.Name}}\t{{.Label \"com.docker.compose.project\"}}"], "--filter", f"name={COMPOSE_PROJECT_PREFIX}",
capture_output=True, text=True, check=False, "--format", "{{.Name}}\t{{.Label \"com.docker.compose.project\"}}"],
) capture_output=True, text=True, check=False,
)
except OSError as exc:
raise EnumerationError(f"docker network ls failed: {exc}") from exc
if result.returncode != 0: if result.returncode != 0:
warn(f"docker network ls failed: {result.stderr.strip()}") raise EnumerationError(
return [] f"docker network ls failed: {result.stderr.strip()}"
)
out: list[str] = [] out: list[str] = []
for line in (result.stdout or "").splitlines(): for line in (result.stdout or "").splitlines():
if not line: if not line:
@@ -120,7 +127,10 @@ def prepare_cleanup() -> DockerBottleCleanupPlan:
`enumerate_active_agents()` so the orphan-state-dir bucket `enumerate_active_agents()` so the orphan-state-dir bucket
doesn't include slugs whose non-docker bottle is still up.""" doesn't include slugs whose non-docker bottle is still up."""
docker_mod.require_docker() docker_mod.require_docker()
projects = list_compose_projects() projects = list_compose_projects(
warn_on_error=False,
raise_on_error=True,
)
project_set = set(projects) project_set = set(projects)
# Late import to avoid a circular at module-load time — # Late import to avoid a circular at module-load time —
# the backend package's __init__ imports this module. # the backend package's __init__ imports this module.
+74 -24
View File
@@ -11,10 +11,9 @@ Reaps *orphans* only — resources with no live VM behind them:
— a VMM left lingering after its dir was removed. — a VMM left lingering after its dir was removed.
A run dir with a *live* firecracker process is a running bottle and is A run dir with a *live* firecracker process is a running bottle and is
left strictly alone: it is neither killed nor removed. (The backend's left strictly alone: it is neither killed nor removed. Active-agent
`enumerate_active` registry is still a stub — #354 — so a live process enumeration uses this same process snapshot, so cleanup and generic
is the only reliable "this bottle is in use" signal we have. Once the backend consumers agree about which bottles are running.
registry lands, registry-orphaned-but-running VMs can be reaped too.)
TAP slots free themselves (the flock drops when the launcher exits), so TAP slots free themselves (the flock drops when the launcher exits), so
there is nothing to reclaim there. there is nothing to reclaim there.
@@ -22,6 +21,7 @@ there is nothing to reclaim there.
from __future__ import annotations from __future__ import annotations
from collections.abc import Sequence
import os import os
import shutil import shutil
import signal import signal
@@ -30,7 +30,7 @@ from pathlib import Path
from ...log import info from ...log import info
from .. import EnumerationError from .. import EnumerationError
from . import util from . import lifecycle_lock, util
from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan
@@ -38,7 +38,7 @@ def _run_root() -> Path:
return util.cache_dir() / "run" return util.cache_dir() / "run"
def _run_dir_of(cmd: str, run_root: Path) -> Path | None: def _run_dir_of(args: Sequence[str], run_root: Path) -> Path | None:
"""The bottle run dir a firecracker cmdline belongs to, or None. """The bottle run dir a firecracker cmdline belongs to, or None.
A bottle VM is launched with `--config-file <run_root>/<slug>/config.json`, A bottle VM is launched with `--config-file <run_root>/<slug>/config.json`,
@@ -46,15 +46,35 @@ def _run_dir_of(cmd: str, run_root: Path) -> Path | None:
the run root. Anything else (a builder VM, the infra VM elsewhere) is the run root. Anything else (a builder VM, the infra VM elsewhere) is
not ours to reap here. not ours to reap here.
""" """
toks = cmd.split() for i, arg in enumerate(args):
for i, tok in enumerate(toks): if arg == "--config-file" and i + 1 < len(args):
if tok == "--config-file" and i + 1 < len(toks): parent = Path(args[i + 1]).parent
parent = Path(toks[i + 1]).parent
if parent.parent == run_root: if parent.parent == run_root:
return parent return parent
return None return None
def _decode_cmdline(raw: bytes) -> tuple[str, ...]:
"""Decode Linux's NUL-delimited argv without losing embedded spaces."""
return tuple(
value.decode(errors="surrogateescape")
for value in raw.split(b"\0") if value
)
def _process_args(pid: int) -> tuple[str, ...] | None:
"""Read one process's lossless argv, or None when it exited meanwhile."""
try:
raw = Path(f"/proc/{pid}/cmdline").read_bytes()
except FileNotFoundError:
return None
except OSError as exc:
raise EnumerationError(
f"could not inspect Firecracker pid {pid}: {exc}"
) from exc
return _decode_cmdline(raw)
def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]: def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
"""Inspect running firecracker VMs under ``run_root``. """Inspect running firecracker VMs under ``run_root``.
@@ -65,7 +85,7 @@ def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
""" """
try: try:
result = subprocess.run( result = subprocess.run(
["pgrep", "-a", "firecracker"], ["pgrep", "firecracker"],
capture_output=True, text=True, check=False, capture_output=True, text=True, check=False,
) )
except OSError as exc: except OSError as exc:
@@ -83,14 +103,14 @@ def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
live: set[str] = set() live: set[str] = set()
orphan_pids: list[int] = [] orphan_pids: list[int] = []
for line in result.stdout.splitlines(): for line in result.stdout.splitlines():
parts = line.split(None, 1)
if len(parts) != 2:
continue
try: try:
pid = int(parts[0]) pid = int(line.strip())
except ValueError: except ValueError:
continue continue
run_dir = _run_dir_of(parts[1], run_root) args = _process_args(pid)
if args is None:
continue
run_dir = _run_dir_of(args, run_root)
if run_dir is None: if run_dir is None:
continue continue
if run_dir.is_dir(): if run_dir.is_dir():
@@ -126,12 +146,42 @@ def prepare_cleanup() -> FirecrackerBottleCleanupPlan:
def cleanup(plan: FirecrackerBottleCleanupPlan) -> None: def cleanup(plan: FirecrackerBottleCleanupPlan) -> None:
for pid in plan.vm_pids: """Revalidate the preview under the launch lock, then remove its survivors."""
info(f"kill firecracker VM pid {pid}") with lifecycle_lock.hold():
fresh = prepare_cleanup()
approved_pids = set(plan.vm_pids).intersection(fresh.vm_pids)
approved_dirs = set(plan.run_dirs).intersection(fresh.run_dirs)
for pid in sorted(approved_pids):
_terminate_orphan(pid, _run_root())
for path in sorted(approved_dirs):
info(f"rm -rf {path}")
shutil.rmtree(path, ignore_errors=True)
def _terminate_orphan(pid: int, run_root: Path) -> None:
"""Signal exactly the process identity that still owns an orphan config."""
try:
pidfd = os.pidfd_open(pid)
except ProcessLookupError:
return
except OSError as exc:
raise EnumerationError(
f"could not pin Firecracker pid {pid} for cleanup: {exc}"
) from exc
try:
try: try:
os.kill(pid, signal.SIGTERM) raw = Path(f"/proc/{pid}/cmdline").read_bytes()
except ProcessLookupError: except FileNotFoundError:
pass return
for path in plan.run_dirs: except OSError as exc:
info(f"rm -rf {path}") raise EnumerationError(
shutil.rmtree(path, ignore_errors=True) f"could not revalidate Firecracker pid {pid}: {exc}"
) from exc
args = _decode_cmdline(raw)
run_dir = _run_dir_of(args, run_root)
if run_dir is None or run_dir.is_dir():
return
info(f"kill firecracker VM pid {pid}")
signal.pidfd_send_signal(pidfd, signal.SIGTERM)
finally:
os.close(pidfd)
@@ -41,6 +41,8 @@ from ... import resources
from ...log import die, info from ...log import die, info
from . import util from . import util
ARTIFACT_HTTP_TIMEOUT_SECONDS = 30.0
# Bump if the on-disk artifact *format* changes (compression, layout) so a new # Bump if the on-disk artifact *format* changes (compression, layout) so a new
# scheme can't collide with a cached/published artifact of the old one. # scheme can't collide with a cached/published artifact of the old one.
_ARTIFACT_FORMAT = "1" _ARTIFACT_FORMAT = "1"
@@ -164,7 +166,9 @@ def _download(url: str, dest: Path) -> None:
"""Stream `url` to `dest` (atomic via a `.part` sibling).""" """Stream `url` to `dest` (atomic via a `.part` sibling)."""
tmp = dest.with_suffix(dest.suffix + ".part") tmp = dest.with_suffix(dest.suffix + ".part")
try: try:
with urllib.request.urlopen(_open(url)) as resp, open(tmp, "wb") as out: with urllib.request.urlopen(
_open(url), timeout=ARTIFACT_HTTP_TIMEOUT_SECONDS,
) as resp, open(tmp, "wb") as out:
shutil.copyfileobj(resp, out, _CHUNK) shutil.copyfileobj(resp, out, _CHUNK)
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
tmp.unlink(missing_ok=True) tmp.unlink(missing_ok=True)
+23 -19
View File
@@ -46,7 +46,7 @@ from ...log import die, info, warn
from ...supervisor.types import SUPERVISE_PORT from ...supervisor.types import SUPERVISE_PORT
from ..docker.egress import EGRESS_PORT from ..docker.egress import EGRESS_PORT
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from . import firecracker_vm, image_builder, isolation_probe, netpool, util from . import firecracker_vm, image_builder, isolation_probe, lifecycle_lock, netpool, util
from .bottle import FirecrackerBottle from .bottle import FirecrackerBottle
from .bottle_plan import FirecrackerBottlePlan from .bottle_plan import FirecrackerBottlePlan
from ...orchestrator.store.config_store import resolve_teardown_timeout from ...orchestrator.store.config_store import resolve_teardown_timeout
@@ -164,25 +164,29 @@ def launch(
) )
# Step 6: build the per-bottle rootfs + SSH key, then boot. # Step 6: build the per-bottle rootfs + SSH key, then boot.
run_dir = util.cache_dir() / "run" / plan.slug # Cleanup takes the same lock while refreshing its process snapshot.
run_dir.mkdir(parents=True, exist_ok=True) # Hold it until the VMM exists so a newly-created run dir can never be
# Remove the run dir on teardown so the per-bottle rootfs.ext4 (~1G) # mistaken for an orphan in the build-before-boot window.
# doesn't leak. Registered before vm.terminate below so it runs *after* with lifecycle_lock.hold():
# it (ExitStack is LIFO): the VM is gone before we rm its rootfs. run_dir = util.cache_dir() / "run" / plan.slug
stack.callback(lambda: shutil.rmtree(run_dir, ignore_errors=True)) run_dir.mkdir(parents=True, exist_ok=True)
rootfs = run_dir / "rootfs.ext4" # Remove the run dir on teardown so the per-bottle rootfs.ext4 (~1G)
util.build_rootfs_ext4(agent_base, rootfs) # doesn't leak. Registered before vm.terminate below so it runs
private_key, pubkey = util.generate_keypair(run_dir) # *after* it (ExitStack is LIFO).
stack.callback(lambda: shutil.rmtree(run_dir, ignore_errors=True))
rootfs = run_dir / "rootfs.ext4"
util.build_rootfs_ext4(agent_base, rootfs)
private_key, pubkey = util.generate_keypair(run_dir)
vm = firecracker_vm.boot( vm = firecracker_vm.boot(
name=plan.container_name, name=plan.container_name,
rootfs=rootfs, rootfs=rootfs,
tap=slot.iface, tap=slot.iface,
guest_ip=slot.guest_ip, guest_ip=slot.guest_ip,
host_ip=slot.host_ip, host_ip=slot.host_ip,
pubkey=pubkey, pubkey=pubkey,
run_dir=run_dir, run_dir=run_dir,
) )
stack.callback(vm.terminate) stack.callback(vm.terminate)
firecracker_vm.wait_for_ssh(vm, private_key) firecracker_vm.wait_for_ssh(vm, private_key)
persist_env_var_secret(private_key, slot.guest_ip, ctx.env_var_secret) persist_env_var_secret(private_key, slot.guest_ip, ctx.env_var_secret)
@@ -0,0 +1,30 @@
"""Serialize Firecracker run-directory creation with orphan cleanup."""
from __future__ import annotations
import fcntl
from contextlib import contextmanager
from pathlib import Path
from typing import Generator
from . import util
def _lock_path() -> Path:
return util.cache_dir() / "run.lifecycle.lock"
@contextmanager
def hold() -> Generator[None]:
"""Exclude cleanup while a launch directory lacks a visible VMM."""
path = _lock_path()
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("a", encoding="utf-8") as handle:
fcntl.flock(handle, fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(handle, fcntl.LOCK_UN)
__all__ = ["hold"]
@@ -34,6 +34,7 @@ from pathlib import Path
from . import infra_artifact, infra_vm, util from . import infra_artifact, infra_vm, util
_CHUNK = 1 << 20 _CHUNK = 1 << 20
_REGISTRY_HTTP_TIMEOUT_SECONDS = 30.0
_GZ_NAME = "rootfs.ext4.gz" _GZ_NAME = "rootfs.ext4.gz"
_SHA_NAME = "rootfs.ext4.gz.sha256" _SHA_NAME = "rootfs.ext4.gz.sha256"
@@ -91,7 +92,9 @@ def _put(url: str, body: "bytes | Path", token: str) -> None:
req.add_header("Authorization", f"token {token}") req.add_header("Authorization", f"token {token}")
req.add_header("Content-Type", "application/octet-stream") req.add_header("Content-Type", "application/octet-stream")
try: try:
with urllib.request.urlopen(req) as resp: with urllib.request.urlopen(
req, timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
) as resp:
print(f" uploaded {url} (HTTP {resp.status})") print(f" uploaded {url} (HTTP {resp.status})")
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
if e.code == 409: if e.code == 409:
@@ -112,7 +115,9 @@ def _delete(url: str, token: str) -> None:
if token: if token:
req.add_header("Authorization", f"token {token}") req.add_header("Authorization", f"token {token}")
try: try:
with urllib.request.urlopen(req): with urllib.request.urlopen(
req, timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
):
pass pass
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
if e.code != 404: if e.code != 404:
@@ -151,7 +156,10 @@ def _try_download_published(role: str, role_dir: Path) -> str | None:
version = _role_version(role) version = _role_version(role)
sha_url = infra_artifact.artifact_url(version, _SHA_NAME, role=role) sha_url = infra_artifact.artifact_url(version, _SHA_NAME, role=role)
try: try:
with urllib.request.urlopen(infra_artifact._open(sha_url)): with urllib.request.urlopen(
infra_artifact._open(sha_url),
timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
):
pass pass
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
if e.code == 404: if e.code == 404:
@@ -195,7 +203,10 @@ def _publish_bundle(role: str, role_dir: Path, token: str) -> str:
# present, a re-publish is a no-op. Otherwise clear any partial upload left # present, a re-publish is a no-op. Otherwise clear any partial upload left
# by an interrupted prior attempt and upload the complete set. # by an interrupted prior attempt and upload the complete set.
try: try:
with urllib.request.urlopen(infra_artifact._open(sha_url)) as resp: with urllib.request.urlopen(
infra_artifact._open(sha_url),
timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
) as resp:
remote_sha = resp.read().decode("utf-8").split()[0].strip().lower() remote_sha = resp.read().decode("utf-8").split()[0].strip().lower()
except urllib.error.HTTPError as e: except urllib.error.HTTPError as e:
if e.code != 404: if e.code != 404:
@@ -4,7 +4,8 @@ from __future__ import annotations
import subprocess import subprocess
from ...log import info, warn from .. import EnumerationError
from ...log import info
from . import util as container_mod from . import util as container_mod
from .bottle_cleanup_plan import MacosContainerBottleCleanupPlan from .bottle_cleanup_plan import MacosContainerBottleCleanupPlan
@@ -19,8 +20,8 @@ def _list_prefixed_containers() -> list[str]:
check=False, check=False,
) )
if result.returncode != 0: if result.returncode != 0:
warn(f"container list failed: {result.stderr.strip()}") detail = result.stderr.strip() or f"exit {result.returncode}"
return [] raise EnumerationError(f"container list failed: {detail}")
return sorted( return sorted(
name for name in (line.strip() for line in result.stdout.splitlines()) name for name in (line.strip() for line in result.stdout.splitlines())
if name.startswith(_PREFIX) if name.startswith(_PREFIX)
@@ -35,7 +36,8 @@ def _list_prefixed_networks() -> list[str]:
check=False, check=False,
) )
if result.returncode != 0: if result.returncode != 0:
return [] detail = result.stderr.strip() or f"exit {result.returncode}"
raise EnumerationError(f"container network list failed: {detail}")
return sorted( return sorted(
name for name in (line.strip() for line in result.stdout.splitlines()) name for name in (line.strip() for line in result.stdout.splitlines())
if name.startswith(_PREFIX) if name.startswith(_PREFIX)
+5 -1
View File
@@ -52,7 +52,11 @@ def cmd_cleanup(_argv: list[str]) -> int:
info("cleanup: skipped") info("cleanup: skipped")
return 0 return 0
for name, backend, plan in prepared: # Confirmation authorizes a fresh authoritative snapshot, not blind use of
# identities that may have changed while the operator reviewed the preview.
refreshed = [(name, backend, backend.prepare_cleanup())
for name, backend, _plan in prepared]
for name, backend, plan in refreshed:
if plan.empty: if plan.empty:
continue continue
backend.cleanup(plan) backend.cleanup(plan)
+12 -4
View File
@@ -136,10 +136,18 @@ def _pump(name: str, stream: IO[bytes]) -> None:
"""Read lines from `stream`, prefix with `[name]`, write to """Read lines from `stream`, prefix with `[name]`, write to
stdout. Runs in its own thread per child; daemon=True so a stdout. Runs in its own thread per child; daemon=True so a
blocked read doesn't keep the process alive after main exits.""" blocked read doesn't keep the process alive after main exits."""
for raw in iter(stream.readline, b""): try:
line = raw.decode("utf-8", errors="replace").rstrip("\n") for raw in iter(stream.readline, b""):
sys.stdout.write(f"[{name}] {line}\n") line = raw.decode("utf-8", errors="replace").rstrip("\n")
sys.stdout.flush() sys.stdout.write(f"[{name}] {line}\n")
sys.stdout.flush()
except (OSError, ValueError) as exc:
# The manager closes a dead child's pipe after wait() and before a
# restart. A pump can be between readline calls at that exact moment;
# closed-stream errors are normal completion, not uncaught thread
# failures. Preserve genuinely unexpected I/O diagnostics.
if not stream.closed:
_log(f"{name} output pump stopped: {type(exc).__name__}: {exc}")
def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]: def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]:
+109
View File
@@ -0,0 +1,109 @@
"""Shared resource boundaries for gateway stdlib HTTP services."""
from __future__ import annotations
import http.server
import socket
import threading
import time
from dataclasses import dataclass
from typing import Any, Protocol
class Readable(Protocol):
def read(self, size: int = -1, /) -> bytes: ...
@dataclass(frozen=True)
class BodyReadError(Exception):
status: int
message: str
def read_declared_body(
stream: Readable,
connection: socket.socket,
raw_length: str | None,
*,
maximum: int,
timeout_seconds: float,
require_length: bool,
) -> bytes:
"""Validate and read exactly one declared body under a read deadline."""
if raw_length is None:
if require_length:
raise BodyReadError(411, "Content-Length required")
raw_length = "0"
try:
length = int(raw_length)
except ValueError as exc:
raise BodyReadError(400, "invalid Content-Length") from exc
if length < 0:
raise BodyReadError(400, "invalid Content-Length")
if length > maximum:
raise BodyReadError(413, "request body too large")
previous_timeout = connection.gettimeout()
deadline = time.monotonic() + timeout_seconds
chunks: list[bytes] = []
remaining = length
try:
while remaining:
timeout = deadline - time.monotonic()
if timeout <= 0:
raise BodyReadError(408, "request body read timed out")
connection.settimeout(timeout)
chunk = stream.read(min(remaining, 64 * 1024))
if not chunk:
raise BodyReadError(400, "incomplete request body")
chunks.append(chunk)
remaining -= len(chunk)
except TimeoutError as exc:
raise BodyReadError(408, "request body read timed out") from exc
finally:
connection.settimeout(previous_timeout)
return b"".join(chunks)
class BoundedThreadingHTTPServer(http.server.ThreadingHTTPServer):
"""ThreadingHTTPServer with a hard cap on in-flight request threads."""
daemon_threads = True
def __init__( # pylint: disable=consider-using-with
self, *args, max_workers: int = 32, **kwargs, # type: ignore[no-untyped-def]
):
if max_workers < 1:
raise ValueError("max_workers must be positive")
self._request_slots = threading.BoundedSemaphore(max_workers)
super().__init__(*args, **kwargs)
def process_request(
self, request: Any, client_address: Any,
) -> None:
if not self._request_slots.acquire( # pylint: disable=consider-using-with
blocking=False,
):
try:
request.sendall(
b"HTTP/1.1 503 Service Unavailable\r\n"
b"Content-Length: 0\r\nConnection: close\r\n\r\n"
)
finally:
self.shutdown_request(request)
return
try:
super().process_request(request, client_address)
except BaseException:
self._request_slots.release()
raise
def process_request_thread(
self, request: Any, client_address: Any,
) -> None:
try:
super().process_request_thread(request, client_address)
finally:
self._request_slots.release()
__all__ = ["BodyReadError", "BoundedThreadingHTTPServer", "read_declared_body"]
+21 -13
View File
@@ -22,11 +22,16 @@ import os
import subprocess import subprocess
import sys import sys
import typing import typing
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from http.server import BaseHTTPRequestHandler
from pathlib import Path from pathlib import Path
from urllib.parse import urlsplit from urllib.parse import urlsplit
from bot_bottle.constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER from bot_bottle.constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER
from bot_bottle.gateway.bounded_http import (
BodyReadError,
BoundedThreadingHTTPServer,
read_declared_body,
)
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
@@ -77,6 +82,8 @@ def resolve_sandbox_root(
# Bound memory use while still allowing ordinary git push packfiles. # Bound memory use while still allowing ordinary git push packfiles.
MAX_BODY_BYTES = 100 * 1024 * 1024 MAX_BODY_BYTES = 100 * 1024 * 1024
REQUEST_BODY_TIMEOUT_SECONDS = 30.0
MAX_REQUEST_WORKERS = 16
class GitHttpHandler(BaseHTTPRequestHandler): class GitHttpHandler(BaseHTTPRequestHandler):
@@ -184,19 +191,18 @@ class GitHttpHandler(BaseHTTPRequestHandler):
value = self.headers.get(header) value = self.headers.get(header)
if value: if value:
env[variable] = value env[variable] = value
raw_length = self.headers.get("content-length", "0") or "0"
try: try:
length = int(raw_length) body = read_declared_body(
except ValueError: self.rfile,
self.send_error(400, "Bad Content-Length") self.connection,
self.headers.get("content-length"),
maximum=MAX_BODY_BYTES,
timeout_seconds=REQUEST_BODY_TIMEOUT_SECONDS,
require_length=False,
)
except BodyReadError as exc:
self.send_error(exc.status, exc.message)
return return
if length < 0:
self.send_error(400, "Negative Content-Length")
return
if length > MAX_BODY_BYTES:
self.send_error(413, "Request body too large")
return
body = self.rfile.read(length) if length else b""
proc = subprocess.run( proc = subprocess.run(
["git", "http-backend"], ["git", "http-backend"],
input=body, input=body,
@@ -273,7 +279,9 @@ def main() -> int:
"(no single-tenant flat-root fallback)\n" "(no single-tenant flat-root fallback)\n"
) )
return 1 return 1
server = ThreadingHTTPServer(("0.0.0.0", port), GitHttpHandler) server = BoundedThreadingHTTPServer(
("0.0.0.0", port), GitHttpHandler, max_workers=MAX_REQUEST_WORKERS,
)
# Resolve each request's sandbox namespace by source IP against the # Resolve each request's sandbox namespace by source IP against the
# orchestrator control plane. # orchestrator control plane.
server.policy_resolver = PolicyResolver(orch_url) # type: ignore[attr-defined] server.policy_resolver = PolicyResolver(orch_url) # type: ignore[attr-defined]
+20 -7
View File
@@ -6,9 +6,8 @@ import json
from dataclasses import dataclass from dataclasses import dataclass
from typing import Callable, Protocol from typing import Callable, Protocol
from bot_bottle.gateway.egress.context import resolve_client_context from bot_bottle.gateway.egress.schema import load_config, route_to_yaml_dict
from bot_bottle.gateway.egress.schema import route_to_yaml_dict from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
from bot_bottle.gateway.policy_resolver import PolicyResolver
from bot_bottle.supervisor import types as _sv from bot_bottle.supervisor import types as _sv
@@ -24,6 +23,10 @@ class MethodNotFoundError(Exception):
"""Raised when a JSON-RPC method has no MCP handler.""" """Raised when a JSON-RPC method has no MCP handler."""
class RouteResolutionError(Exception):
"""The caller's live route table could not be resolved authoritatively."""
Handler = Callable[[dict[str, object]], object] Handler = Callable[[dict[str, object]], object]
@@ -60,10 +63,19 @@ def resolved_routes_payload(
source_ip: str, source_ip: str,
identity_token: str, identity_token: str,
) -> dict[str, object]: ) -> dict[str, object]:
"""Render the calling bottle's routes, failing closed to an empty list.""" """Render an authoritatively resolved route table for the calling bottle."""
config, _slug, _tokens = resolve_client_context( try:
resolver, source_ip, identity_token, policy, bottle_id, _tokens = resolver.resolve_policy_and_bottle_id(
) source_ip, identity_token,
)
except PolicyResolveError as exc:
raise RouteResolutionError("orchestrator unavailable") from exc
if not bottle_id:
raise RouteResolutionError("request source is not attributed to a bottle")
try:
config = load_config(policy or "")
except ValueError as exc:
raise RouteResolutionError("resolved policy is invalid") from exc
body = json.dumps( body = json.dumps(
{"routes": [route_to_yaml_dict(route) for route in config.routes]}, {"routes": [route_to_yaml_dict(route) for route in config.routes]},
indent=2, indent=2,
@@ -74,6 +86,7 @@ def resolved_routes_payload(
__all__ = [ __all__ = [
"Handlers", "Handlers",
"MethodNotFoundError", "MethodNotFoundError",
"RouteResolutionError",
"dispatch", "dispatch",
"resolved_routes_payload", "resolved_routes_payload",
] ]
+38 -24
View File
@@ -51,19 +51,24 @@ from __future__ import annotations
import http.server import http.server
import json import json
import os import os
import socketserver
import sys import sys
import time import time
import typing import typing
from dataclasses import dataclass from dataclasses import dataclass
from bot_bottle.constants import IDENTITY_HEADER from bot_bottle.constants import IDENTITY_HEADER
from bot_bottle.gateway.bounded_http import (
BodyReadError,
BoundedThreadingHTTPServer,
read_declared_body,
)
from bot_bottle.gateway.egress.schema import load_config from bot_bottle.gateway.egress.schema import load_config
from bot_bottle.gateway.egress.types import LOG_OFF from bot_bottle.gateway.egress.types import LOG_OFF
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
from bot_bottle.gateway.supervisor.mcp_dispatch import ( from bot_bottle.gateway.supervisor.mcp_dispatch import (
Handlers as DispatchHandlers, Handlers as DispatchHandlers,
MethodNotFoundError, MethodNotFoundError,
RouteResolutionError,
dispatch, dispatch,
resolved_routes_payload, resolved_routes_payload,
) )
@@ -570,6 +575,8 @@ def format_unknown_proposal_text(proposal_id: str) -> str:
# Max request body the server accepts. 1 MB is well above any realistic # Max request body the server accepts. 1 MB is well above any realistic
# routes.yaml proposal. # routes.yaml proposal.
MAX_BODY_BYTES = 1 * 1024 * 1024 MAX_BODY_BYTES = 1 * 1024 * 1024
REQUEST_BODY_TIMEOUT_SECONDS = 10.0
MAX_REQUEST_WORKERS = 32
class MCPHandler(http.server.BaseHTTPRequestHandler): class MCPHandler(http.server.BaseHTTPRequestHandler):
@@ -592,19 +599,18 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
self._write_text(405, "use POST for MCP requests\n") self._write_text(405, "use POST for MCP requests\n")
def do_POST(self) -> None: def do_POST(self) -> None:
length_header = self.headers.get("Content-Length")
if length_header is None:
self._write_text(411, "Content-Length required\n")
return
try: try:
length = int(length_header) body = read_declared_body(
except ValueError: self.rfile,
self._write_text(400, "invalid Content-Length\n") self.connection,
self.headers.get("Content-Length"),
maximum=MAX_BODY_BYTES,
timeout_seconds=REQUEST_BODY_TIMEOUT_SECONDS,
require_length=True,
)
except BodyReadError as exc:
self._write_text(exc.status, exc.message + "\n")
return return
if length < 0 or length > MAX_BODY_BYTES:
self._write_text(413, "request body too large\n")
return
body = self.rfile.read(length)
try: try:
req = parse_jsonrpc(body) req = parse_jsonrpc(body)
@@ -660,20 +666,25 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
identity_token=self._identity_token(), identity_token=self._identity_token(),
) )
return dispatch( def list_routes(_params: dict[str, object]) -> object:
req, try:
DispatchHandlers( return resolved_routes_payload(
initialize=handle_initialize,
tools_list=handle_tools_list,
list_routes=lambda _params: resolved_routes_payload(
self._resolver_or_fail(), self._resolver_or_fail(),
self.client_address[0], self.client_address[0],
self._identity_token(), self._identity_token(),
), )
check_proposal=check, except RouteResolutionError as exc:
propose=propose, raise _RpcInternalError(
), f"could not resolve live egress routes: {exc}"
) ) from exc
return dispatch(req, DispatchHandlers(
initialize=handle_initialize,
tools_list=handle_tools_list,
list_routes=list_routes,
check_proposal=check,
propose=propose,
))
def _identity_token(self) -> str: def _identity_token(self) -> str:
"""The agent's per-bottle identity token from the request header (the """The agent's per-bottle identity token from the request header (the
@@ -711,7 +722,7 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
self.wfile.write(encoded) self.wfile.write(encoded)
class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer): class MCPServer(BoundedThreadingHTTPServer):
allow_reuse_address = True allow_reuse_address = True
daemon_threads = True daemon_threads = True
config: ServerConfig = ServerConfig() config: ServerConfig = ServerConfig()
@@ -720,6 +731,9 @@ class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
# closed per request (see `_resolver_or_fail`). # closed per request (see `_resolver_or_fail`).
policy_resolver: "PolicyResolver | None" = None policy_resolver: "PolicyResolver | None" = None
def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def]
super().__init__(*args, max_workers=MAX_REQUEST_WORKERS, **kwargs)
# --- Entry point ----------------------------------------------------------- # --- Entry point -----------------------------------------------------------
+22 -3
View File
@@ -41,8 +41,8 @@ class TestCmdCleanup(unittest.TestCase):
): ):
self.assertEqual(0, cmd.cmd_cleanup([])) self.assertEqual(0, cmd.cmd_cleanup([]))
docker.prepare_cleanup.assert_called_once() self.assertEqual(2, docker.prepare_cleanup.call_count)
fc.prepare_cleanup.assert_called_once() self.assertEqual(2, fc.prepare_cleanup.call_count)
docker.cleanup.assert_called_once_with(docker_plan) docker.cleanup.assert_called_once_with(docker_plan)
fc.cleanup.assert_called_once_with(fc_plan) fc.cleanup.assert_called_once_with(fc_plan)
@@ -68,7 +68,7 @@ class TestCmdCleanup(unittest.TestCase):
): ):
self.assertEqual(0, cmd.cmd_cleanup([])) self.assertEqual(0, cmd.cmd_cleanup([]))
docker.prepare_cleanup.assert_called_once() self.assertEqual(2, docker.prepare_cleanup.call_count)
docker.cleanup.assert_called_once_with(docker_plan) docker.cleanup.assert_called_once_with(docker_plan)
macos.prepare_cleanup.assert_not_called() macos.prepare_cleanup.assert_not_called()
@@ -135,6 +135,25 @@ class TestCmdCleanup(unittest.TestCase):
docker.cleanup.assert_called_once_with(docker_plan) docker.cleanup.assert_called_once_with(docker_plan)
fc.cleanup.assert_not_called() fc.cleanup.assert_not_called()
def test_executes_refreshed_plan_after_confirmation(self):
backend = MagicMock()
preview = MagicMock(empty=False)
refreshed = MagicMock(empty=False)
backend.prepare_cleanup.side_effect = [preview, refreshed]
with patch.object(
cmd, "known_backend_names", return_value=("firecracker",),
), patch.object(
cmd, "get_bottle_backend", return_value=backend,
), patch.object(
cmd, "has_backend", return_value=True,
), patch.object(
cmd, "_prompt_yes", return_value=True,
):
self.assertEqual(0, cmd.cmd_cleanup([]))
backend.cleanup.assert_called_once_with(refreshed)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+35
View File
@@ -14,9 +14,12 @@ from __future__ import annotations
import tempfile import tempfile
import unittest import unittest
from pathlib import Path from pathlib import Path
from unittest.mock import patch
from tests.unit import use_bottle_root from tests.unit import use_bottle_root
from bot_bottle import bottle_state from bot_bottle import bottle_state
from bot_bottle.backend import EnumerationError
from bot_bottle.backend.docker import cleanup
from bot_bottle.backend.docker.cleanup import _list_orphan_state_dirs from bot_bottle.backend.docker.cleanup import _list_orphan_state_dirs
@@ -116,5 +119,37 @@ class TestOrphanStateDirs(_FakeHomeMixin, unittest.TestCase):
) )
class TestAuthoritativeDiscovery(unittest.TestCase):
def test_prepare_requires_authoritative_compose_projects(self):
with patch.object(cleanup.docker_mod, "require_docker"), \
patch.object(
cleanup, "list_compose_projects",
side_effect=EnumerationError("compose unavailable"),
) as projects, self.assertRaisesRegex(
EnumerationError, "compose unavailable",
):
cleanup.prepare_cleanup()
projects.assert_called_once_with(
warn_on_error=False,
raise_on_error=True,
)
def test_container_query_failure_raises(self):
failed = cleanup.subprocess.CompletedProcess(
[], 1, stdout="", stderr="daemon unavailable",
)
with patch.object(cleanup.subprocess, "run", return_value=failed), \
self.assertRaisesRegex(EnumerationError, "daemon unavailable"):
cleanup._list_prefixed_containers()
def test_network_query_failure_raises(self):
failed = cleanup.subprocess.CompletedProcess(
[], 1, stdout="", stderr="daemon unavailable",
)
with patch.object(cleanup.subprocess, "run", return_value=failed), \
self.assertRaisesRegex(EnumerationError, "daemon unavailable"):
cleanup._list_prefixed_networks()
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+79 -19
View File
@@ -32,15 +32,19 @@ class TestProcessScan(unittest.TestCase):
self.assertEqual( self.assertEqual(
Path("/cache/run/dev-a"), Path("/cache/run/dev-a"),
fc_cleanup._run_dir_of( fc_cleanup._run_dir_of(
f"firecracker --config-file {run_root}/dev-a/config.json", run_root ("firecracker", "--config-file",
f"{run_root}/dev-a/config.json"), run_root
), ),
) )
# infra/builder VMs elsewhere, or nested paths, are not ours. # infra/builder VMs elsewhere, or nested paths, are not ours.
self.assertIsNone( self.assertIsNone(
fc_cleanup._run_dir_of("firecracker --config-file /elsewhere/config.json", run_root) fc_cleanup._run_dir_of(
("firecracker", "--config-file", "/elsewhere/config.json"),
run_root,
)
) )
self.assertIsNone( self.assertIsNone(
fc_cleanup._run_dir_of("firecracker --no-config", run_root) fc_cleanup._run_dir_of(("firecracker", "--no-config"), run_root)
) )
def test_scan_splits_live_dirs_from_orphan_pids(self): def test_scan_splits_live_dirs_from_orphan_pids(self):
@@ -48,17 +52,40 @@ class TestProcessScan(unittest.TestCase):
run_root = Path(tmp) run_root = Path(tmp)
(run_root / "live-a").mkdir() # dir present -> live VM, protected (run_root / "live-a").mkdir() # dir present -> live VM, protected
# "gone-b" dir intentionally absent -> lingering VMM, orphan pid # "gone-b" dir intentionally absent -> lingering VMM, orphan pid
out = ( args = {
f"111 firecracker --config-file {run_root}/live-a/config.json\n" 111: ("firecracker", "--config-file",
f"222 firecracker --config-file {run_root}/gone-b/config.json\n" f"{run_root}/live-a/config.json"),
"333 firecracker --config-file /elsewhere/config.json\n" 222: ("firecracker", "--config-file",
"notanint firecracker --config-file x\n" f"{run_root}/gone-b/config.json"),
) 333: ("firecracker", "--config-file", "/elsewhere/config.json"),
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)): }
with patch.object(
fc_cleanup.subprocess, "run",
return_value=_proc("111\n222\n333\nnotanint\n"),
), patch.object(
fc_cleanup, "_process_args", side_effect=args.get,
):
live, orphan_pids = fc_cleanup._scan_processes(run_root) live, orphan_pids = fc_cleanup._scan_processes(run_root)
self.assertEqual({str(run_root / "live-a")}, live) self.assertEqual({str(run_root / "live-a")}, live)
self.assertEqual([222], orphan_pids) self.assertEqual([222], orphan_pids)
def test_scan_preserves_spaces_in_config_path(self):
with tempfile.TemporaryDirectory(prefix="fc cache ") as tmp:
run_root = Path(tmp)
live = run_root / "live bottle"
live.mkdir()
with patch.object(
fc_cleanup.subprocess, "run", return_value=_proc("111\n"),
), patch.object(
fc_cleanup, "_process_args",
return_value=("firecracker", "--config-file",
str(live / "config.json")),
):
self.assertEqual(
({str(live)}, []),
fc_cleanup._scan_processes(run_root),
)
def test_scan_empty_when_pgrep_finds_no_processes(self): def test_scan_empty_when_pgrep_finds_no_processes(self):
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(returncode=1)): with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(returncode=1)):
self.assertEqual((set(), []), fc_cleanup._scan_processes(Path("/x"))) self.assertEqual((set(), []), fc_cleanup._scan_processes(Path("/x")))
@@ -117,9 +144,14 @@ class TestProcessScan(unittest.TestCase):
run_root = Path(tmp) run_root = Path(tmp)
(run_root / "live-a").mkdir() (run_root / "live-a").mkdir()
(run_root / "dead-b").mkdir() (run_root / "dead-b").mkdir()
out = f"111 firecracker --config-file {run_root}/live-a/config.json\n"
with patch.object(fc_cleanup, "_run_root", return_value=run_root), \ with patch.object(fc_cleanup, "_run_root", return_value=run_root), \
patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)): patch.object(fc_cleanup.subprocess, "run",
return_value=_proc("111\n")), \
patch.object(
fc_cleanup, "_process_args",
return_value=("firecracker", "--config-file",
str(run_root / "live-a/config.json")),
):
plan = fc_cleanup.prepare_cleanup() plan = fc_cleanup.prepare_cleanup()
self.assertEqual((), plan.vm_pids) self.assertEqual((), plan.vm_pids)
self.assertEqual((str(run_root / "dead-b"),), plan.run_dirs) self.assertEqual((str(run_root / "dead-b"),), plan.run_dirs)
@@ -132,18 +164,46 @@ class TestCleanupRemoval(unittest.TestCase):
vm_pids=(101,), vm_pids=(101,),
run_dirs=("/run/dev-x",), run_dirs=("/run/dev-x",),
) )
with patch.object(fc_cleanup.os, "kill") as kill, \ with patch.object(fc_cleanup, "prepare_cleanup", return_value=plan), \
patch.object(fc_cleanup, "_run_root", return_value=Path("/run")), \
patch.object(fc_cleanup, "_terminate_orphan") as terminate, \
patch.object(fc_cleanup.shutil, "rmtree") as rmtree, \ patch.object(fc_cleanup.shutil, "rmtree") as rmtree, \
patch.object(fc_cleanup, "info"): patch.object(fc_cleanup, "info"):
fc_cleanup.cleanup(plan) fc_cleanup.cleanup(plan)
kill.assert_called_once() terminate.assert_called_once_with(101, Path("/run"))
rmtree.assert_called_once_with("/run/dev-x", ignore_errors=True) rmtree.assert_called_once_with("/run/dev-x", ignore_errors=True)
def test_cleanup_tolerates_dead_pid(self): def test_cleanup_skips_resources_no_longer_in_refreshed_plan(self):
plan = FirecrackerBottleCleanupPlan(vm_pids=(999,)) preview = FirecrackerBottleCleanupPlan(
with patch.object(fc_cleanup.os, "kill", side_effect=ProcessLookupError), \ vm_pids=(999,), run_dirs=("/run/reused",),
patch.object(fc_cleanup, "info"): )
fc_cleanup.cleanup(plan) # must not raise with patch.object(
fc_cleanup, "prepare_cleanup",
return_value=FirecrackerBottleCleanupPlan(),
), patch.object(fc_cleanup, "_terminate_orphan") as terminate, \
patch.object(fc_cleanup.shutil, "rmtree") as rmtree:
fc_cleanup.cleanup(preview)
terminate.assert_not_called()
rmtree.assert_not_called()
def test_pidfd_prevents_pid_reuse_from_signalling_unrelated_process(self):
with patch.object(fc_cleanup.os, "pidfd_open", return_value=7), \
patch.object(
fc_cleanup.Path, "read_bytes",
return_value=b"/usr/bin/python\0worker.py\0",
), patch.object(fc_cleanup.signal, "pidfd_send_signal") as send, \
patch.object(fc_cleanup.os, "close"):
fc_cleanup._terminate_orphan(101, Path("/run"))
send.assert_not_called()
def test_pidfd_signals_revalidated_orphan(self):
command = b"firecracker\0--config-file\0/run/gone/config.json\0"
with patch.object(fc_cleanup.os, "pidfd_open", return_value=7), \
patch.object(fc_cleanup.Path, "read_bytes", return_value=command), \
patch.object(fc_cleanup.signal, "pidfd_send_signal") as send, \
patch.object(fc_cleanup.os, "close"), patch.object(fc_cleanup, "info"):
fc_cleanup._terminate_orphan(101, Path("/run"))
send.assert_called_once_with(7, fc_cleanup.signal.SIGTERM)
class TestCleanupPlan(unittest.TestCase): class TestCleanupPlan(unittest.TestCase):
+79
View File
@@ -0,0 +1,79 @@
"""Unit tests for shared gateway stdlib HTTP resource boundaries."""
# pylint: disable=protected-access
from __future__ import annotations
import io
import socket
import unittest
from bot_bottle.gateway.bounded_http import (
BodyReadError,
BoundedThreadingHTTPServer,
read_declared_body,
)
class _Handler:
pass
class _TimeoutStream:
def read(self, _size: int = -1, /) -> bytes:
raise TimeoutError
class TestDeclaredBody(unittest.TestCase):
def setUp(self) -> None:
self.left, self.right = socket.socketpair()
def tearDown(self) -> None:
self.left.close()
self.right.close()
def test_rejects_incomplete_body(self) -> None:
with self.assertRaisesRegex(BodyReadError, "incomplete"):
read_declared_body(
io.BytesIO(b"short"),
self.left,
"10",
maximum=100,
timeout_seconds=1,
require_length=True,
)
def test_maps_read_timeout(self) -> None:
with self.assertRaises(BodyReadError) as caught:
read_declared_body(
_TimeoutStream(),
self.left,
"1",
maximum=100,
timeout_seconds=1,
require_length=True,
)
self.assertEqual(408, caught.exception.status)
class TestBoundedServer(unittest.TestCase):
def test_saturated_server_rejects_without_spawning_thread(self) -> None:
client, peer = socket.socketpair()
with BoundedThreadingHTTPServer(
("127.0.0.1", 0), _Handler, max_workers=1, # type: ignore[arg-type]
) as server:
with client, peer:
# Directly reserve the only slot to model an in-flight handler.
self.assertTrue(
server._request_slots.acquire( # pylint: disable=consider-using-with
blocking=False,
),
)
try:
server.process_request(client, ("127.0.0.1", 1))
self.assertIn(b"503 Service Unavailable", peer.recv(1024))
finally:
server._request_slots.release()
if __name__ == "__main__":
unittest.main()
+22
View File
@@ -23,6 +23,7 @@ from bot_bottle.gateway.bootstrap import (
_DaemonManager, _DaemonManager,
_argv_for_daemon, _argv_for_daemon,
_env_for_daemon, _env_for_daemon,
_pump,
_selected_daemons, _selected_daemons,
) )
from tests._bin import SLEEP from tests._bin import SLEEP
@@ -565,5 +566,26 @@ class TestMainEndToEnd(unittest.TestCase):
self.assertIn("no daemons selected", out) self.assertIn("no daemons selected", out)
class _FailingStream:
def __init__(self, *, closed: bool) -> None:
self.closed = closed
def readline(self) -> bytes:
raise ValueError("I/O operation on closed file")
class TestOutputPump(unittest.TestCase):
def test_closed_stream_race_is_normal_completion(self) -> None:
with patch("bot_bottle.gateway.bootstrap._log") as log:
_pump("egress", _FailingStream(closed=True)) # type: ignore[arg-type]
log.assert_not_called()
def test_unexpected_io_failure_is_logged(self) -> None:
with patch("bot_bottle.gateway.bootstrap._log") as log:
_pump("egress", _FailingStream(closed=False)) # type: ignore[arg-type]
log.assert_called_once()
self.assertIn("output pump stopped", log.call_args.args[0])
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+12
View File
@@ -284,6 +284,18 @@ class TestEnsureArtifact(_CacheMixin):
ia.ensure_artifact_gz(version, role=_ROLE) ia.ensure_artifact_gz(version, role=_ROLE)
self.assertEqual(first_calls, len(net.calls)) self.assertEqual(first_calls, len(net.calls))
def test_download_uses_network_deadline(self) -> None:
response = mock.MagicMock()
response.__enter__.return_value = io.BytesIO(b"payload")
with tempfile.TemporaryDirectory() as d, mock.patch.object(
ia.urllib.request, "urlopen", return_value=response,
) as urlopen:
ia._download("https://registry/artifact", Path(d) / "artifact")
self.assertEqual(
ia.ARTIFACT_HTTP_TIMEOUT_SECONDS,
urlopen.call_args.kwargs["timeout"],
)
def test_checksum_mismatch_fails_closed(self) -> None: def test_checksum_mismatch_fails_closed(self) -> None:
version = "beefbeefbeefbeef" version = "beefbeefbeefbeef"
gz = _gz(b"payload") gz = _gz(b"payload")
@@ -42,6 +42,22 @@ class TestMacosContainerCleanup(unittest.TestCase):
run.call_args_list[1].args[0], run.call_args_list[1].args[0],
) )
def test_container_enumeration_failure_aborts(self):
completed = cleanup.subprocess.CompletedProcess(
args=[], returncode=1, stdout="", stderr="service unavailable",
)
with patch.object(cleanup.subprocess, "run", return_value=completed), \
self.assertRaisesRegex(EnumerationError, "service unavailable"):
cleanup._list_prefixed_containers()
def test_network_enumeration_failure_aborts(self):
completed = cleanup.subprocess.CompletedProcess(
args=[], returncode=1, stdout="", stderr="service unavailable",
)
with patch.object(cleanup.subprocess, "run", return_value=completed), \
self.assertRaisesRegex(EnumerationError, "service unavailable"):
cleanup._list_prefixed_networks()
class TestMacosContainerEnumerate(unittest.TestCase): class TestMacosContainerEnumerate(unittest.TestCase):
"""The backend launches bottles again (PRD 0070), so enumeration is real """The backend launches bottles again (PRD 0070), so enumeration is real
+10
View File
@@ -49,6 +49,16 @@ class TestPut(unittest.TestCase):
self.assertNotIsInstance(req.data, (bytes, bytearray)) self.assertNotIsInstance(req.data, (bytes, bytearray))
self.assertEqual(str(len(payload)), req.get_header("Content-length")) self.assertEqual(str(len(payload)), req.get_header("Content-length"))
def test_put_uses_network_deadline(self) -> None:
with mock.patch.object(
pub.urllib.request, "urlopen", return_value=_Resp(),
) as urlopen:
pub._put("https://reg/pkg", b"payload", token="")
self.assertEqual(
pub._REGISTRY_HTTP_TIMEOUT_SECONDS,
urlopen.call_args.kwargs["timeout"],
)
def test_small_bytes_body_still_works(self) -> None: def test_small_bytes_body_still_works(self) -> None:
captured: list[urllib.request.Request] = [] captured: list[urllib.request.Request] = []
+13 -5
View File
@@ -717,18 +717,26 @@ class TestResolvedRoutesPayload(unittest.TestCase):
hosts = {r["host"] for r in data["routes"]} hosts = {r["host"] for r in data["routes"]}
self.assertEqual({"api.anthropic.com", "www.google.com"}, hosts) self.assertEqual({"api.anthropic.com", "www.google.com"}, hosts)
def test_orchestrator_error_fails_closed_to_empty(self) -> None: def test_orchestrator_error_is_not_reported_as_empty_policy(self) -> None:
# resolve_client_context swallows resolver errors → deny-all (empty), with self.assertRaises(supervise_server.RouteResolutionError):
# never another bottle's routes. resolved_routes_payload(
typing.cast(
supervise_server.PolicyResolver,
_FakeSuperviseResolver(raises=True),
),
_SRC,
_TOK,
)
def test_authoritative_empty_policy_remains_successful(self) -> None:
payload = resolved_routes_payload( payload = resolved_routes_payload(
typing.cast( typing.cast(
supervise_server.PolicyResolver, supervise_server.PolicyResolver,
_FakeSuperviseResolver(raises=True), _FakeSuperviseResolver(bottle_id="b1", policy="routes: []\n"),
), ),
_SRC, _SRC,
_TOK, _TOK,
) )
assert payload is not None
data = json.loads(payload["content"][0]["text"]) # type: ignore[index] data = json.loads(payload["content"][0]["text"]) # type: ignore[index]
self.assertEqual([], data["routes"]) self.assertEqual([], data["routes"])