Compare commits

...

3 Commits

Author SHA1 Message Date
didericis-codex fd295d4c14 fix(gateway): contain output pump shutdown races
test / image-input-builds (pull_request) Failing after 12m49s
test / unit (pull_request) Has started running
test / coverage (pull_request) Blocked by required conditions
test / integration-docker (pull_request) Blocked by required conditions
tracker-policy-pr / check-pr (pull_request) Successful in 13s
2026-07-27 04:46:00 +00:00
didericis-codex f33566941b fix(gateway): bound stdlib HTTP request work
test / image-input-builds (pull_request) Successful in 1m2s
test / unit (pull_request) Has started running
test / coverage (pull_request) Blocked by required conditions
test / integration-docker (pull_request) Blocked by required conditions
tracker-policy-pr / check-pr (pull_request) Failing after 12m29s
2026-07-27 04:46:00 +00:00
didericis-codex c7c3a79028 fix(cleanup): revalidate destructive backend plans
test / image-input-builds (pull_request) Failing after 13m11s
test / unit (pull_request) Has started running
test / coverage (pull_request) Blocked by required conditions
test / integration-docker (pull_request) Blocked by required conditions
tracker-policy-pr / check-pr (pull_request) Successful in 13s
2026-07-27 04:45:59 +00:00
16 changed files with 490 additions and 96 deletions
+38 -8
View File
@@ -30,7 +30,7 @@ from pathlib import Path
from ...log import info from ...log import info
from .. import EnumerationError from .. import EnumerationError
from . import util from . import lifecycle_lock, util
from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan
@@ -126,12 +126,42 @@ def prepare_cleanup() -> FirecrackerBottleCleanupPlan:
def cleanup(plan: FirecrackerBottleCleanupPlan) -> None: def cleanup(plan: FirecrackerBottleCleanupPlan) -> None:
for pid in plan.vm_pids: """Revalidate the preview under the launch lock, then remove its survivors."""
info(f"kill firecracker VM pid {pid}") with lifecycle_lock.hold():
try: fresh = prepare_cleanup()
os.kill(pid, signal.SIGTERM) approved_pids = set(plan.vm_pids).intersection(fresh.vm_pids)
except ProcessLookupError: approved_dirs = set(plan.run_dirs).intersection(fresh.run_dirs)
pass for pid in sorted(approved_pids):
for path in plan.run_dirs: _terminate_orphan(pid, _run_root())
for path in sorted(approved_dirs):
info(f"rm -rf {path}") info(f"rm -rf {path}")
shutil.rmtree(path, ignore_errors=True) shutil.rmtree(path, ignore_errors=True)
def _terminate_orphan(pid: int, run_root: Path) -> None:
"""Signal exactly the process identity that still owns an orphan config."""
try:
pidfd = os.pidfd_open(pid)
except ProcessLookupError:
return
except OSError as exc:
raise EnumerationError(
f"could not pin Firecracker pid {pid} for cleanup: {exc}"
) from exc
try:
try:
raw = Path(f"/proc/{pid}/cmdline").read_bytes()
except FileNotFoundError:
return
except OSError as exc:
raise EnumerationError(
f"could not revalidate Firecracker pid {pid}: {exc}"
) from exc
command = raw.replace(b"\0", b" ").decode(errors="replace")
run_dir = _run_dir_of(command, run_root)
if run_dir is None or run_dir.is_dir():
return
info(f"kill firecracker VM pid {pid}")
signal.pidfd_send_signal(pidfd, signal.SIGTERM)
finally:
os.close(pidfd)
+7 -3
View File
@@ -46,7 +46,7 @@ from ...log import die, info, warn
from ...supervisor.types import SUPERVISE_PORT from ...supervisor.types import SUPERVISE_PORT
from ..docker.egress import EGRESS_PORT from ..docker.egress import EGRESS_PORT
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from . import firecracker_vm, image_builder, isolation_probe, netpool, util from . import firecracker_vm, image_builder, isolation_probe, lifecycle_lock, netpool, util
from .bottle import FirecrackerBottle from .bottle import FirecrackerBottle
from .bottle_plan import FirecrackerBottlePlan from .bottle_plan import FirecrackerBottlePlan
from ...orchestrator.store.config_store import resolve_teardown_timeout from ...orchestrator.store.config_store import resolve_teardown_timeout
@@ -164,11 +164,15 @@ def launch(
) )
# Step 6: build the per-bottle rootfs + SSH key, then boot. # Step 6: build the per-bottle rootfs + SSH key, then boot.
# Cleanup takes the same lock while refreshing its process snapshot.
# Hold it until the VMM exists so a newly-created run dir can never be
# mistaken for an orphan in the build-before-boot window.
with lifecycle_lock.hold():
run_dir = util.cache_dir() / "run" / plan.slug run_dir = util.cache_dir() / "run" / plan.slug
run_dir.mkdir(parents=True, exist_ok=True) run_dir.mkdir(parents=True, exist_ok=True)
# Remove the run dir on teardown so the per-bottle rootfs.ext4 (~1G) # Remove the run dir on teardown so the per-bottle rootfs.ext4 (~1G)
# doesn't leak. Registered before vm.terminate below so it runs *after* # doesn't leak. Registered before vm.terminate below so it runs
# it (ExitStack is LIFO): the VM is gone before we rm its rootfs. # *after* it (ExitStack is LIFO).
stack.callback(lambda: shutil.rmtree(run_dir, ignore_errors=True)) stack.callback(lambda: shutil.rmtree(run_dir, ignore_errors=True))
rootfs = run_dir / "rootfs.ext4" rootfs = run_dir / "rootfs.ext4"
util.build_rootfs_ext4(agent_base, rootfs) util.build_rootfs_ext4(agent_base, rootfs)
@@ -0,0 +1,30 @@
"""Serialize Firecracker run-directory creation with orphan cleanup."""
from __future__ import annotations
import fcntl
from contextlib import contextmanager
from pathlib import Path
from typing import Generator
from . import util
def _lock_path() -> Path:
return util.cache_dir() / "run.lifecycle.lock"
@contextmanager
def hold() -> Generator[None]:
"""Exclude cleanup while a launch directory lacks a visible VMM."""
path = _lock_path()
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("a", encoding="utf-8") as handle:
fcntl.flock(handle, fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(handle, fcntl.LOCK_UN)
__all__ = ["hold"]
@@ -4,7 +4,8 @@ from __future__ import annotations
import subprocess import subprocess
from ...log import info, warn from .. import EnumerationError
from ...log import info
from . import util as container_mod from . import util as container_mod
from .bottle_cleanup_plan import MacosContainerBottleCleanupPlan from .bottle_cleanup_plan import MacosContainerBottleCleanupPlan
@@ -19,8 +20,8 @@ def _list_prefixed_containers() -> list[str]:
check=False, check=False,
) )
if result.returncode != 0: if result.returncode != 0:
warn(f"container list failed: {result.stderr.strip()}") detail = result.stderr.strip() or f"exit {result.returncode}"
return [] raise EnumerationError(f"container list failed: {detail}")
return sorted( return sorted(
name for name in (line.strip() for line in result.stdout.splitlines()) name for name in (line.strip() for line in result.stdout.splitlines())
if name.startswith(_PREFIX) if name.startswith(_PREFIX)
@@ -35,7 +36,8 @@ def _list_prefixed_networks() -> list[str]:
check=False, check=False,
) )
if result.returncode != 0: if result.returncode != 0:
return [] detail = result.stderr.strip() or f"exit {result.returncode}"
raise EnumerationError(f"container network list failed: {detail}")
return sorted( return sorted(
name for name in (line.strip() for line in result.stdout.splitlines()) name for name in (line.strip() for line in result.stdout.splitlines())
if name.startswith(_PREFIX) if name.startswith(_PREFIX)
+5 -1
View File
@@ -52,7 +52,11 @@ def cmd_cleanup(_argv: list[str]) -> int:
info("cleanup: skipped") info("cleanup: skipped")
return 0 return 0
for name, backend, plan in prepared: # Confirmation authorizes a fresh authoritative snapshot, not blind use of
# identities that may have changed while the operator reviewed the preview.
refreshed = [(name, backend, backend.prepare_cleanup())
for name, backend, _plan in prepared]
for name, backend, plan in refreshed:
if plan.empty: if plan.empty:
continue continue
backend.cleanup(plan) backend.cleanup(plan)
+8
View File
@@ -136,10 +136,18 @@ def _pump(name: str, stream: IO[bytes]) -> None:
"""Read lines from `stream`, prefix with `[name]`, write to """Read lines from `stream`, prefix with `[name]`, write to
stdout. Runs in its own thread per child; daemon=True so a stdout. Runs in its own thread per child; daemon=True so a
blocked read doesn't keep the process alive after main exits.""" blocked read doesn't keep the process alive after main exits."""
try:
for raw in iter(stream.readline, b""): for raw in iter(stream.readline, b""):
line = raw.decode("utf-8", errors="replace").rstrip("\n") line = raw.decode("utf-8", errors="replace").rstrip("\n")
sys.stdout.write(f"[{name}] {line}\n") sys.stdout.write(f"[{name}] {line}\n")
sys.stdout.flush() sys.stdout.flush()
except (OSError, ValueError) as exc:
# The manager closes a dead child's pipe after wait() and before a
# restart. A pump can be between readline calls at that exact moment;
# closed-stream errors are normal completion, not uncaught thread
# failures. Preserve genuinely unexpected I/O diagnostics.
if not stream.closed:
_log(f"{name} output pump stopped: {type(exc).__name__}: {exc}")
def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]: def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]:
+109
View File
@@ -0,0 +1,109 @@
"""Shared resource boundaries for gateway stdlib HTTP services."""
from __future__ import annotations
import http.server
import socket
import threading
import time
from dataclasses import dataclass
from typing import Any, Protocol
class Readable(Protocol):
def read(self, size: int = -1, /) -> bytes: ...
@dataclass(frozen=True)
class BodyReadError(Exception):
status: int
message: str
def read_declared_body(
stream: Readable,
connection: socket.socket,
raw_length: str | None,
*,
maximum: int,
timeout_seconds: float,
require_length: bool,
) -> bytes:
"""Validate and read exactly one declared body under a read deadline."""
if raw_length is None:
if require_length:
raise BodyReadError(411, "Content-Length required")
raw_length = "0"
try:
length = int(raw_length)
except ValueError as exc:
raise BodyReadError(400, "invalid Content-Length") from exc
if length < 0:
raise BodyReadError(400, "invalid Content-Length")
if length > maximum:
raise BodyReadError(413, "request body too large")
previous_timeout = connection.gettimeout()
deadline = time.monotonic() + timeout_seconds
chunks: list[bytes] = []
remaining = length
try:
while remaining:
timeout = deadline - time.monotonic()
if timeout <= 0:
raise BodyReadError(408, "request body read timed out")
connection.settimeout(timeout)
chunk = stream.read(min(remaining, 64 * 1024))
if not chunk:
raise BodyReadError(400, "incomplete request body")
chunks.append(chunk)
remaining -= len(chunk)
except TimeoutError as exc:
raise BodyReadError(408, "request body read timed out") from exc
finally:
connection.settimeout(previous_timeout)
return b"".join(chunks)
class BoundedThreadingHTTPServer(http.server.ThreadingHTTPServer):
"""ThreadingHTTPServer with a hard cap on in-flight request threads."""
daemon_threads = True
def __init__( # pylint: disable=consider-using-with
self, *args, max_workers: int = 32, **kwargs, # type: ignore[no-untyped-def]
):
if max_workers < 1:
raise ValueError("max_workers must be positive")
self._request_slots = threading.BoundedSemaphore(max_workers)
super().__init__(*args, **kwargs)
def process_request(
self, request: Any, client_address: Any,
) -> None:
if not self._request_slots.acquire( # pylint: disable=consider-using-with
blocking=False,
):
try:
request.sendall(
b"HTTP/1.1 503 Service Unavailable\r\n"
b"Content-Length: 0\r\nConnection: close\r\n\r\n"
)
finally:
self.shutdown_request(request)
return
try:
super().process_request(request, client_address)
except BaseException:
self._request_slots.release()
raise
def process_request_thread(
self, request: Any, client_address: Any,
) -> None:
try:
super().process_request_thread(request, client_address)
finally:
self._request_slots.release()
__all__ = ["BodyReadError", "BoundedThreadingHTTPServer", "read_declared_body"]
+21 -13
View File
@@ -22,11 +22,16 @@ import os
import subprocess import subprocess
import sys import sys
import typing import typing
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from http.server import BaseHTTPRequestHandler
from pathlib import Path from pathlib import Path
from urllib.parse import urlsplit from urllib.parse import urlsplit
from bot_bottle.constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER from bot_bottle.constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER
from bot_bottle.gateway.bounded_http import (
BodyReadError,
BoundedThreadingHTTPServer,
read_declared_body,
)
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
@@ -77,6 +82,8 @@ def resolve_sandbox_root(
# Bound memory use while still allowing ordinary git push packfiles. # Bound memory use while still allowing ordinary git push packfiles.
MAX_BODY_BYTES = 100 * 1024 * 1024 MAX_BODY_BYTES = 100 * 1024 * 1024
REQUEST_BODY_TIMEOUT_SECONDS = 30.0
MAX_REQUEST_WORKERS = 16
class GitHttpHandler(BaseHTTPRequestHandler): class GitHttpHandler(BaseHTTPRequestHandler):
@@ -184,19 +191,18 @@ class GitHttpHandler(BaseHTTPRequestHandler):
value = self.headers.get(header) value = self.headers.get(header)
if value: if value:
env[variable] = value env[variable] = value
raw_length = self.headers.get("content-length", "0") or "0"
try: try:
length = int(raw_length) body = read_declared_body(
except ValueError: self.rfile,
self.send_error(400, "Bad Content-Length") self.connection,
self.headers.get("content-length"),
maximum=MAX_BODY_BYTES,
timeout_seconds=REQUEST_BODY_TIMEOUT_SECONDS,
require_length=False,
)
except BodyReadError as exc:
self.send_error(exc.status, exc.message)
return return
if length < 0:
self.send_error(400, "Negative Content-Length")
return
if length > MAX_BODY_BYTES:
self.send_error(413, "Request body too large")
return
body = self.rfile.read(length) if length else b""
proc = subprocess.run( proc = subprocess.run(
["git", "http-backend"], ["git", "http-backend"],
input=body, input=body,
@@ -273,7 +279,9 @@ def main() -> int:
"(no single-tenant flat-root fallback)\n" "(no single-tenant flat-root fallback)\n"
) )
return 1 return 1
server = ThreadingHTTPServer(("0.0.0.0", port), GitHttpHandler) server = BoundedThreadingHTTPServer(
("0.0.0.0", port), GitHttpHandler, max_workers=MAX_REQUEST_WORKERS,
)
# Resolve each request's sandbox namespace by source IP against the # Resolve each request's sandbox namespace by source IP against the
# orchestrator control plane. # orchestrator control plane.
server.policy_resolver = PolicyResolver(orch_url) # type: ignore[attr-defined] server.policy_resolver = PolicyResolver(orch_url) # type: ignore[attr-defined]
+19 -6
View File
@@ -6,9 +6,8 @@ import json
from dataclasses import dataclass from dataclasses import dataclass
from typing import Callable, Protocol from typing import Callable, Protocol
from bot_bottle.gateway.egress.context import resolve_client_context from bot_bottle.gateway.egress.schema import load_config, route_to_yaml_dict
from bot_bottle.gateway.egress.schema import route_to_yaml_dict from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
from bot_bottle.gateway.policy_resolver import PolicyResolver
from bot_bottle.supervisor import types as _sv from bot_bottle.supervisor import types as _sv
@@ -24,6 +23,10 @@ class MethodNotFoundError(Exception):
"""Raised when a JSON-RPC method has no MCP handler.""" """Raised when a JSON-RPC method has no MCP handler."""
class RouteResolutionError(Exception):
"""The caller's live route table could not be resolved authoritatively."""
Handler = Callable[[dict[str, object]], object] Handler = Callable[[dict[str, object]], object]
@@ -60,10 +63,19 @@ def resolved_routes_payload(
source_ip: str, source_ip: str,
identity_token: str, identity_token: str,
) -> dict[str, object]: ) -> dict[str, object]:
"""Render the calling bottle's routes, failing closed to an empty list.""" """Render an authoritatively resolved route table for the calling bottle."""
config, _slug, _tokens = resolve_client_context( try:
resolver, source_ip, identity_token, policy, bottle_id, _tokens = resolver.resolve_policy_and_bottle_id(
source_ip, identity_token,
) )
except PolicyResolveError as exc:
raise RouteResolutionError("orchestrator unavailable") from exc
if not bottle_id:
raise RouteResolutionError("request source is not attributed to a bottle")
try:
config = load_config(policy or "")
except ValueError as exc:
raise RouteResolutionError("resolved policy is invalid") from exc
body = json.dumps( body = json.dumps(
{"routes": [route_to_yaml_dict(route) for route in config.routes]}, {"routes": [route_to_yaml_dict(route) for route in config.routes]},
indent=2, indent=2,
@@ -74,6 +86,7 @@ def resolved_routes_payload(
__all__ = [ __all__ = [
"Handlers", "Handlers",
"MethodNotFoundError", "MethodNotFoundError",
"RouteResolutionError",
"dispatch", "dispatch",
"resolved_routes_payload", "resolved_routes_payload",
] ]
+36 -22
View File
@@ -51,19 +51,24 @@ from __future__ import annotations
import http.server import http.server
import json import json
import os import os
import socketserver
import sys import sys
import time import time
import typing import typing
from dataclasses import dataclass from dataclasses import dataclass
from bot_bottle.constants import IDENTITY_HEADER from bot_bottle.constants import IDENTITY_HEADER
from bot_bottle.gateway.bounded_http import (
BodyReadError,
BoundedThreadingHTTPServer,
read_declared_body,
)
from bot_bottle.gateway.egress.schema import load_config from bot_bottle.gateway.egress.schema import load_config
from bot_bottle.gateway.egress.types import LOG_OFF from bot_bottle.gateway.egress.types import LOG_OFF
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
from bot_bottle.gateway.supervisor.mcp_dispatch import ( from bot_bottle.gateway.supervisor.mcp_dispatch import (
Handlers as DispatchHandlers, Handlers as DispatchHandlers,
MethodNotFoundError, MethodNotFoundError,
RouteResolutionError,
dispatch, dispatch,
resolved_routes_payload, resolved_routes_payload,
) )
@@ -570,6 +575,8 @@ def format_unknown_proposal_text(proposal_id: str) -> str:
# Max request body the server accepts. 1 MB is well above any realistic # Max request body the server accepts. 1 MB is well above any realistic
# routes.yaml proposal. # routes.yaml proposal.
MAX_BODY_BYTES = 1 * 1024 * 1024 MAX_BODY_BYTES = 1 * 1024 * 1024
REQUEST_BODY_TIMEOUT_SECONDS = 10.0
MAX_REQUEST_WORKERS = 32
class MCPHandler(http.server.BaseHTTPRequestHandler): class MCPHandler(http.server.BaseHTTPRequestHandler):
@@ -592,19 +599,18 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
self._write_text(405, "use POST for MCP requests\n") self._write_text(405, "use POST for MCP requests\n")
def do_POST(self) -> None: def do_POST(self) -> None:
length_header = self.headers.get("Content-Length")
if length_header is None:
self._write_text(411, "Content-Length required\n")
return
try: try:
length = int(length_header) body = read_declared_body(
except ValueError: self.rfile,
self._write_text(400, "invalid Content-Length\n") self.connection,
self.headers.get("Content-Length"),
maximum=MAX_BODY_BYTES,
timeout_seconds=REQUEST_BODY_TIMEOUT_SECONDS,
require_length=True,
)
except BodyReadError as exc:
self._write_text(exc.status, exc.message + "\n")
return return
if length < 0 or length > MAX_BODY_BYTES:
self._write_text(413, "request body too large\n")
return
body = self.rfile.read(length)
try: try:
req = parse_jsonrpc(body) req = parse_jsonrpc(body)
@@ -660,20 +666,25 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
identity_token=self._identity_token(), identity_token=self._identity_token(),
) )
return dispatch( def list_routes(_params: dict[str, object]) -> object:
req, try:
DispatchHandlers( return resolved_routes_payload(
initialize=handle_initialize,
tools_list=handle_tools_list,
list_routes=lambda _params: resolved_routes_payload(
self._resolver_or_fail(), self._resolver_or_fail(),
self.client_address[0], self.client_address[0],
self._identity_token(), self._identity_token(),
), )
except RouteResolutionError as exc:
raise _RpcInternalError(
f"could not resolve live egress routes: {exc}"
) from exc
return dispatch(req, DispatchHandlers(
initialize=handle_initialize,
tools_list=handle_tools_list,
list_routes=list_routes,
check_proposal=check, check_proposal=check,
propose=propose, propose=propose,
), ))
)
def _identity_token(self) -> str: def _identity_token(self) -> str:
"""The agent's per-bottle identity token from the request header (the """The agent's per-bottle identity token from the request header (the
@@ -711,7 +722,7 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
self.wfile.write(encoded) self.wfile.write(encoded)
class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer): class MCPServer(BoundedThreadingHTTPServer):
allow_reuse_address = True allow_reuse_address = True
daemon_threads = True daemon_threads = True
config: ServerConfig = ServerConfig() config: ServerConfig = ServerConfig()
@@ -720,6 +731,9 @@ class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
# closed per request (see `_resolver_or_fail`). # closed per request (see `_resolver_or_fail`).
policy_resolver: "PolicyResolver | None" = None policy_resolver: "PolicyResolver | None" = None
def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def]
super().__init__(*args, max_workers=MAX_REQUEST_WORKERS, **kwargs)
# --- Entry point ----------------------------------------------------------- # --- Entry point -----------------------------------------------------------
+22 -3
View File
@@ -41,8 +41,8 @@ class TestCmdCleanup(unittest.TestCase):
): ):
self.assertEqual(0, cmd.cmd_cleanup([])) self.assertEqual(0, cmd.cmd_cleanup([]))
docker.prepare_cleanup.assert_called_once() self.assertEqual(2, docker.prepare_cleanup.call_count)
fc.prepare_cleanup.assert_called_once() self.assertEqual(2, fc.prepare_cleanup.call_count)
docker.cleanup.assert_called_once_with(docker_plan) docker.cleanup.assert_called_once_with(docker_plan)
fc.cleanup.assert_called_once_with(fc_plan) fc.cleanup.assert_called_once_with(fc_plan)
@@ -68,7 +68,7 @@ class TestCmdCleanup(unittest.TestCase):
): ):
self.assertEqual(0, cmd.cmd_cleanup([])) self.assertEqual(0, cmd.cmd_cleanup([]))
docker.prepare_cleanup.assert_called_once() self.assertEqual(2, docker.prepare_cleanup.call_count)
docker.cleanup.assert_called_once_with(docker_plan) docker.cleanup.assert_called_once_with(docker_plan)
macos.prepare_cleanup.assert_not_called() macos.prepare_cleanup.assert_not_called()
@@ -135,6 +135,25 @@ class TestCmdCleanup(unittest.TestCase):
docker.cleanup.assert_called_once_with(docker_plan) docker.cleanup.assert_called_once_with(docker_plan)
fc.cleanup.assert_not_called() fc.cleanup.assert_not_called()
def test_executes_refreshed_plan_after_confirmation(self):
backend = MagicMock()
preview = MagicMock(empty=False)
refreshed = MagicMock(empty=False)
backend.prepare_cleanup.side_effect = [preview, refreshed]
with patch.object(
cmd, "known_backend_names", return_value=("firecracker",),
), patch.object(
cmd, "get_bottle_backend", return_value=backend,
), patch.object(
cmd, "has_backend", return_value=True,
), patch.object(
cmd, "_prompt_yes", return_value=True,
):
self.assertEqual(0, cmd.cmd_cleanup([]))
backend.cleanup.assert_called_once_with(refreshed)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+35 -7
View File
@@ -132,18 +132,46 @@ class TestCleanupRemoval(unittest.TestCase):
vm_pids=(101,), vm_pids=(101,),
run_dirs=("/run/dev-x",), run_dirs=("/run/dev-x",),
) )
with patch.object(fc_cleanup.os, "kill") as kill, \ with patch.object(fc_cleanup, "prepare_cleanup", return_value=plan), \
patch.object(fc_cleanup, "_run_root", return_value=Path("/run")), \
patch.object(fc_cleanup, "_terminate_orphan") as terminate, \
patch.object(fc_cleanup.shutil, "rmtree") as rmtree, \ patch.object(fc_cleanup.shutil, "rmtree") as rmtree, \
patch.object(fc_cleanup, "info"): patch.object(fc_cleanup, "info"):
fc_cleanup.cleanup(plan) fc_cleanup.cleanup(plan)
kill.assert_called_once() terminate.assert_called_once_with(101, Path("/run"))
rmtree.assert_called_once_with("/run/dev-x", ignore_errors=True) rmtree.assert_called_once_with("/run/dev-x", ignore_errors=True)
def test_cleanup_tolerates_dead_pid(self): def test_cleanup_skips_resources_no_longer_in_refreshed_plan(self):
plan = FirecrackerBottleCleanupPlan(vm_pids=(999,)) preview = FirecrackerBottleCleanupPlan(
with patch.object(fc_cleanup.os, "kill", side_effect=ProcessLookupError), \ vm_pids=(999,), run_dirs=("/run/reused",),
patch.object(fc_cleanup, "info"): )
fc_cleanup.cleanup(plan) # must not raise with patch.object(
fc_cleanup, "prepare_cleanup",
return_value=FirecrackerBottleCleanupPlan(),
), patch.object(fc_cleanup, "_terminate_orphan") as terminate, \
patch.object(fc_cleanup.shutil, "rmtree") as rmtree:
fc_cleanup.cleanup(preview)
terminate.assert_not_called()
rmtree.assert_not_called()
def test_pidfd_prevents_pid_reuse_from_signalling_unrelated_process(self):
with patch.object(fc_cleanup.os, "pidfd_open", return_value=7), \
patch.object(
fc_cleanup.Path, "read_bytes",
return_value=b"/usr/bin/python\0worker.py\0",
), patch.object(fc_cleanup.signal, "pidfd_send_signal") as send, \
patch.object(fc_cleanup.os, "close"):
fc_cleanup._terminate_orphan(101, Path("/run"))
send.assert_not_called()
def test_pidfd_signals_revalidated_orphan(self):
command = b"firecracker\0--config-file\0/run/gone/config.json\0"
with patch.object(fc_cleanup.os, "pidfd_open", return_value=7), \
patch.object(fc_cleanup.Path, "read_bytes", return_value=command), \
patch.object(fc_cleanup.signal, "pidfd_send_signal") as send, \
patch.object(fc_cleanup.os, "close"), patch.object(fc_cleanup, "info"):
fc_cleanup._terminate_orphan(101, Path("/run"))
send.assert_called_once_with(7, fc_cleanup.signal.SIGTERM)
class TestCleanupPlan(unittest.TestCase): class TestCleanupPlan(unittest.TestCase):
+79
View File
@@ -0,0 +1,79 @@
"""Unit tests for shared gateway stdlib HTTP resource boundaries."""
# pylint: disable=protected-access
from __future__ import annotations
import io
import socket
import unittest
from bot_bottle.gateway.bounded_http import (
BodyReadError,
BoundedThreadingHTTPServer,
read_declared_body,
)
class _Handler:
pass
class _TimeoutStream:
def read(self, _size: int = -1, /) -> bytes:
raise TimeoutError
class TestDeclaredBody(unittest.TestCase):
def setUp(self) -> None:
self.left, self.right = socket.socketpair()
def tearDown(self) -> None:
self.left.close()
self.right.close()
def test_rejects_incomplete_body(self) -> None:
with self.assertRaisesRegex(BodyReadError, "incomplete"):
read_declared_body(
io.BytesIO(b"short"),
self.left,
"10",
maximum=100,
timeout_seconds=1,
require_length=True,
)
def test_maps_read_timeout(self) -> None:
with self.assertRaises(BodyReadError) as caught:
read_declared_body(
_TimeoutStream(),
self.left,
"1",
maximum=100,
timeout_seconds=1,
require_length=True,
)
self.assertEqual(408, caught.exception.status)
class TestBoundedServer(unittest.TestCase):
def test_saturated_server_rejects_without_spawning_thread(self) -> None:
client, peer = socket.socketpair()
with BoundedThreadingHTTPServer(
("127.0.0.1", 0), _Handler, max_workers=1, # type: ignore[arg-type]
) as server:
with client, peer:
# Directly reserve the only slot to model an in-flight handler.
self.assertTrue(
server._request_slots.acquire( # pylint: disable=consider-using-with
blocking=False,
),
)
try:
server.process_request(client, ("127.0.0.1", 1))
self.assertIn(b"503 Service Unavailable", peer.recv(1024))
finally:
server._request_slots.release()
if __name__ == "__main__":
unittest.main()
+22
View File
@@ -23,6 +23,7 @@ from bot_bottle.gateway.bootstrap import (
_DaemonManager, _DaemonManager,
_argv_for_daemon, _argv_for_daemon,
_env_for_daemon, _env_for_daemon,
_pump,
_selected_daemons, _selected_daemons,
) )
from tests._bin import SLEEP from tests._bin import SLEEP
@@ -565,5 +566,26 @@ class TestMainEndToEnd(unittest.TestCase):
self.assertIn("no daemons selected", out) self.assertIn("no daemons selected", out)
class _FailingStream:
def __init__(self, *, closed: bool) -> None:
self.closed = closed
def readline(self) -> bytes:
raise ValueError("I/O operation on closed file")
class TestOutputPump(unittest.TestCase):
def test_closed_stream_race_is_normal_completion(self) -> None:
with patch("bot_bottle.gateway.bootstrap._log") as log:
_pump("egress", _FailingStream(closed=True)) # type: ignore[arg-type]
log.assert_not_called()
def test_unexpected_io_failure_is_logged(self) -> None:
with patch("bot_bottle.gateway.bootstrap._log") as log:
_pump("egress", _FailingStream(closed=False)) # type: ignore[arg-type]
log.assert_called_once()
self.assertIn("output pump stopped", log.call_args.args[0])
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
@@ -42,6 +42,22 @@ class TestMacosContainerCleanup(unittest.TestCase):
run.call_args_list[1].args[0], run.call_args_list[1].args[0],
) )
def test_container_enumeration_failure_aborts(self):
completed = cleanup.subprocess.CompletedProcess(
args=[], returncode=1, stdout="", stderr="service unavailable",
)
with patch.object(cleanup.subprocess, "run", return_value=completed), \
self.assertRaisesRegex(EnumerationError, "service unavailable"):
cleanup._list_prefixed_containers()
def test_network_enumeration_failure_aborts(self):
completed = cleanup.subprocess.CompletedProcess(
args=[], returncode=1, stdout="", stderr="service unavailable",
)
with patch.object(cleanup.subprocess, "run", return_value=completed), \
self.assertRaisesRegex(EnumerationError, "service unavailable"):
cleanup._list_prefixed_networks()
class TestMacosContainerEnumerate(unittest.TestCase): class TestMacosContainerEnumerate(unittest.TestCase):
"""The backend launches bottles again (PRD 0070), so enumeration is real """The backend launches bottles again (PRD 0070), so enumeration is real
+13 -5
View File
@@ -717,10 +717,9 @@ class TestResolvedRoutesPayload(unittest.TestCase):
hosts = {r["host"] for r in data["routes"]} hosts = {r["host"] for r in data["routes"]}
self.assertEqual({"api.anthropic.com", "www.google.com"}, hosts) self.assertEqual({"api.anthropic.com", "www.google.com"}, hosts)
def test_orchestrator_error_fails_closed_to_empty(self) -> None: def test_orchestrator_error_is_not_reported_as_empty_policy(self) -> None:
# resolve_client_context swallows resolver errors → deny-all (empty), with self.assertRaises(supervise_server.RouteResolutionError):
# never another bottle's routes. resolved_routes_payload(
payload = resolved_routes_payload(
typing.cast( typing.cast(
supervise_server.PolicyResolver, supervise_server.PolicyResolver,
_FakeSuperviseResolver(raises=True), _FakeSuperviseResolver(raises=True),
@@ -728,7 +727,16 @@ class TestResolvedRoutesPayload(unittest.TestCase):
_SRC, _SRC,
_TOK, _TOK,
) )
assert payload is not None
def test_authoritative_empty_policy_remains_successful(self) -> None:
payload = resolved_routes_payload(
typing.cast(
supervise_server.PolicyResolver,
_FakeSuperviseResolver(bottle_id="b1", policy="routes: []\n"),
),
_SRC,
_TOK,
)
data = json.loads(payload["content"][0]["text"]) # type: ignore[index] data = json.loads(payload["content"][0]["text"]) # type: ignore[index]
self.assertEqual([], data["routes"]) self.assertEqual([], data["routes"])