Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 95cbd0e1c8 | |||
| f242733d15 | |||
| fd295d4c14 | |||
| f33566941b | |||
| c7c3a79028 | |||
| bb1776a858 | |||
| a24fe0264d | |||
| 105538d3a6 |
@@ -172,6 +172,10 @@ class BottleCleanupPlan(ABC):
|
|||||||
"""True iff there is nothing to clean up; the CLI uses this to
|
"""True iff there is nothing to clean up; the CLI uses this to
|
||||||
short-circuit before showing the y/N."""
|
short-circuit before showing the y/N."""
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def intersect(self, current: "BottleCleanupPlan") -> "BottleCleanupPlan":
|
||||||
|
"""Resources both displayed to the operator and currently removable."""
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class ExecResult:
|
class ExecResult:
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
"""Shared destructive-cleanup execution and failure accounting."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
class CleanupError(RuntimeError):
|
||||||
|
"""One or more approved cleanup mutations did not complete."""
|
||||||
|
|
||||||
|
|
||||||
|
class CleanupFailures:
|
||||||
|
"""Attempt every approved mutation, then fail with complete diagnostics."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self._messages: list[str] = []
|
||||||
|
|
||||||
|
def run(self, argv: Sequence[str], description: str) -> None:
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
list(argv), capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
except OSError as exc:
|
||||||
|
self._messages.append(f"{description}: {exc}")
|
||||||
|
return
|
||||||
|
if result.returncode != 0:
|
||||||
|
detail = (result.stderr or result.stdout).strip()
|
||||||
|
self._messages.append(
|
||||||
|
f"{description}: {detail or f'exit {result.returncode}'}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def remove_tree(self, path: Path, description: str) -> None:
|
||||||
|
try:
|
||||||
|
shutil.rmtree(path)
|
||||||
|
except FileNotFoundError:
|
||||||
|
return
|
||||||
|
except OSError as exc:
|
||||||
|
self._messages.append(f"{description}: {exc}")
|
||||||
|
|
||||||
|
def raise_if_any(self) -> None:
|
||||||
|
if self._messages:
|
||||||
|
raise CleanupError("; ".join(self._messages))
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["CleanupError", "CleanupFailures"]
|
||||||
@@ -46,6 +46,22 @@ class DockerBottleCleanupPlan(BottleCleanupPlan):
|
|||||||
and not self.orphan_state_dirs
|
and not self.orphan_state_dirs
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def intersect(self, current: BottleCleanupPlan) -> "DockerBottleCleanupPlan":
|
||||||
|
if not isinstance(current, DockerBottleCleanupPlan):
|
||||||
|
raise TypeError("cleanup plans must have the same backend type")
|
||||||
|
return DockerBottleCleanupPlan(
|
||||||
|
projects=tuple(x for x in self.projects if x in current.projects),
|
||||||
|
stray_containers=tuple(
|
||||||
|
x for x in self.stray_containers if x in current.stray_containers
|
||||||
|
),
|
||||||
|
stray_networks=tuple(
|
||||||
|
x for x in self.stray_networks if x in current.stray_networks
|
||||||
|
),
|
||||||
|
orphan_state_dirs=tuple(
|
||||||
|
x for x in self.orphan_state_dirs if x in current.orphan_state_dirs
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
def print(self) -> None:
|
def print(self) -> None:
|
||||||
print(file=sys.stderr)
|
print(file=sys.stderr)
|
||||||
for name in self.projects:
|
for name in self.projects:
|
||||||
|
|||||||
@@ -23,11 +23,12 @@ Active-agent enumeration lives in `backend/docker/enumerate.py`.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
from ...paths import bot_bottle_root
|
from ...paths import bot_bottle_root
|
||||||
from ...log import info, warn
|
from ...log import info
|
||||||
|
from .. import EnumerationError
|
||||||
|
from ..cleanup_control import CleanupFailures
|
||||||
from . import util as docker_mod
|
from . import util as docker_mod
|
||||||
from .bottle_cleanup_plan import DockerBottleCleanupPlan
|
from .bottle_cleanup_plan import DockerBottleCleanupPlan
|
||||||
from ...bottle_state import bottle_state_dir, is_preserved
|
from ...bottle_state import bottle_state_dir, is_preserved
|
||||||
@@ -36,15 +37,17 @@ from .compose import COMPOSE_PROJECT_PREFIX, list_compose_projects
|
|||||||
|
|
||||||
def _list_prefixed_containers() -> list[str]:
|
def _list_prefixed_containers() -> list[str]:
|
||||||
"""All bot-bottle-prefixed containers, running or stopped."""
|
"""All bot-bottle-prefixed containers, running or stopped."""
|
||||||
|
try:
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["docker", "ps", "-a",
|
["docker", "ps", "-a",
|
||||||
"--filter", f"name=^{COMPOSE_PROJECT_PREFIX}",
|
"--filter", f"name=^{COMPOSE_PROJECT_PREFIX}",
|
||||||
"--format", "{{.Names}}\t{{.Label \"com.docker.compose.project\"}}"],
|
"--format", "{{.Names}}\t{{.Label \"com.docker.compose.project\"}}"],
|
||||||
capture_output=True, text=True, check=False,
|
capture_output=True, text=True, check=False,
|
||||||
)
|
)
|
||||||
|
except OSError as exc:
|
||||||
|
raise EnumerationError(f"docker ps failed: {exc}") from exc
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
warn(f"docker ps failed: {result.stderr.strip()}")
|
raise EnumerationError(f"docker ps failed: {result.stderr.strip()}")
|
||||||
return []
|
|
||||||
out: list[str] = []
|
out: list[str] = []
|
||||||
for line in (result.stdout or "").splitlines():
|
for line in (result.stdout or "").splitlines():
|
||||||
if not line:
|
if not line:
|
||||||
@@ -63,15 +66,19 @@ def _list_prefixed_networks() -> list[str]:
|
|||||||
to a compose project. Compose-managed networks have a
|
to a compose project. Compose-managed networks have a
|
||||||
`com.docker.compose.project` label; bare ones (from pre-compose
|
`com.docker.compose.project` label; bare ones (from pre-compose
|
||||||
code paths) don't."""
|
code paths) don't."""
|
||||||
|
try:
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["docker", "network", "ls",
|
["docker", "network", "ls",
|
||||||
"--filter", f"name={COMPOSE_PROJECT_PREFIX}",
|
"--filter", f"name={COMPOSE_PROJECT_PREFIX}",
|
||||||
"--format", "{{.Name}}\t{{.Label \"com.docker.compose.project\"}}"],
|
"--format", "{{.Name}}\t{{.Label \"com.docker.compose.project\"}}"],
|
||||||
capture_output=True, text=True, check=False,
|
capture_output=True, text=True, check=False,
|
||||||
)
|
)
|
||||||
|
except OSError as exc:
|
||||||
|
raise EnumerationError(f"docker network ls failed: {exc}") from exc
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
warn(f"docker network ls failed: {result.stderr.strip()}")
|
raise EnumerationError(
|
||||||
return []
|
f"docker network ls failed: {result.stderr.strip()}"
|
||||||
|
)
|
||||||
out: list[str] = []
|
out: list[str] = []
|
||||||
for line in (result.stdout or "").splitlines():
|
for line in (result.stdout or "").splitlines():
|
||||||
if not line:
|
if not line:
|
||||||
@@ -120,7 +127,10 @@ def prepare_cleanup() -> DockerBottleCleanupPlan:
|
|||||||
`enumerate_active_agents()` so the orphan-state-dir bucket
|
`enumerate_active_agents()` so the orphan-state-dir bucket
|
||||||
doesn't include slugs whose non-docker bottle is still up."""
|
doesn't include slugs whose non-docker bottle is still up."""
|
||||||
docker_mod.require_docker()
|
docker_mod.require_docker()
|
||||||
projects = list_compose_projects()
|
projects = list_compose_projects(
|
||||||
|
warn_on_error=False,
|
||||||
|
raise_on_error=True,
|
||||||
|
)
|
||||||
project_set = set(projects)
|
project_set = set(projects)
|
||||||
# Late import to avoid a circular at module-load time —
|
# Late import to avoid a circular at module-load time —
|
||||||
# the backend package's __init__ imports this module.
|
# the backend package's __init__ imports this module.
|
||||||
@@ -140,40 +150,30 @@ def cleanup(plan: DockerBottleCleanupPlan) -> None:
|
|||||||
"""Remove everything in the plan. Projects first (whose `compose
|
"""Remove everything in the plan. Projects first (whose `compose
|
||||||
down` reaps their containers + networks atomically), then stray
|
down` reaps their containers + networks atomically), then stray
|
||||||
legacy resources, then orphan state dirs."""
|
legacy resources, then orphan state dirs."""
|
||||||
|
failures = CleanupFailures()
|
||||||
for project in plan.projects:
|
for project in plan.projects:
|
||||||
info(f"docker compose down ({project})")
|
info(f"docker compose down ({project})")
|
||||||
result = subprocess.run(
|
failures.run(
|
||||||
["docker", "compose", "-p", project, "down", "--volumes"],
|
["docker", "compose", "-p", project, "down", "--volumes"],
|
||||||
capture_output=True, text=True, check=False,
|
f"docker compose down failed for {project}",
|
||||||
)
|
|
||||||
if result.returncode != 0:
|
|
||||||
warn(
|
|
||||||
f"compose down failed for {project}: "
|
|
||||||
f"{result.stderr.strip()}"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
for name in plan.stray_containers:
|
for name in plan.stray_containers:
|
||||||
info(f"removing stray container {name}")
|
info(f"removing stray container {name}")
|
||||||
subprocess.run(
|
failures.run(
|
||||||
["docker", "rm", "-f", name],
|
["docker", "rm", "-f", name],
|
||||||
stdout=subprocess.DEVNULL,
|
f"removing stray container {name}",
|
||||||
stderr=subprocess.DEVNULL,
|
|
||||||
check=False,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
for name in plan.stray_networks:
|
for name in plan.stray_networks:
|
||||||
info(f"removing stray network {name}")
|
info(f"removing stray network {name}")
|
||||||
subprocess.run(
|
failures.run(
|
||||||
["docker", "network", "rm", name],
|
["docker", "network", "rm", name],
|
||||||
stdout=subprocess.DEVNULL,
|
f"removing stray network {name}",
|
||||||
stderr=subprocess.DEVNULL,
|
|
||||||
check=False,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
for identity in plan.orphan_state_dirs:
|
for identity in plan.orphan_state_dirs:
|
||||||
path = bottle_state_dir(identity)
|
path = bottle_state_dir(identity)
|
||||||
info(f"removing orphan state dir {path}")
|
info(f"removing orphan state dir {path}")
|
||||||
try:
|
failures.remove_tree(path, f"removing orphan state dir {path}")
|
||||||
shutil.rmtree(path, ignore_errors=True)
|
failures.raise_if_any()
|
||||||
except OSError as e:
|
|
||||||
warn(f"failed to remove {path}: {e}")
|
|
||||||
|
|||||||
@@ -27,3 +27,11 @@ class FirecrackerBottleCleanupPlan(BottleCleanupPlan):
|
|||||||
@property
|
@property
|
||||||
def empty(self) -> bool:
|
def empty(self) -> bool:
|
||||||
return not (self.vm_pids or self.run_dirs)
|
return not (self.vm_pids or self.run_dirs)
|
||||||
|
|
||||||
|
def intersect(self, current: BottleCleanupPlan) -> "FirecrackerBottleCleanupPlan":
|
||||||
|
if not isinstance(current, FirecrackerBottleCleanupPlan):
|
||||||
|
raise TypeError("cleanup plans must have the same backend type")
|
||||||
|
return FirecrackerBottleCleanupPlan(
|
||||||
|
vm_pids=tuple(x for x in self.vm_pids if x in current.vm_pids),
|
||||||
|
run_dirs=tuple(x for x in self.run_dirs if x in current.run_dirs),
|
||||||
|
)
|
||||||
|
|||||||
@@ -11,10 +11,9 @@ Reaps *orphans* only — resources with no live VM behind them:
|
|||||||
— a VMM left lingering after its dir was removed.
|
— a VMM left lingering after its dir was removed.
|
||||||
|
|
||||||
A run dir with a *live* firecracker process is a running bottle and is
|
A run dir with a *live* firecracker process is a running bottle and is
|
||||||
left strictly alone: it is neither killed nor removed. (The backend's
|
left strictly alone: it is neither killed nor removed. Active-agent
|
||||||
`enumerate_active` registry is still a stub — #354 — so a live process
|
enumeration uses this same process snapshot, so cleanup and generic
|
||||||
is the only reliable "this bottle is in use" signal we have. Once the
|
backend consumers agree about which bottles are running.
|
||||||
registry lands, registry-orphaned-but-running VMs can be reaped too.)
|
|
||||||
|
|
||||||
TAP slots free themselves (the flock drops when the launcher exits), so
|
TAP slots free themselves (the flock drops when the launcher exits), so
|
||||||
there is nothing to reclaim there.
|
there is nothing to reclaim there.
|
||||||
@@ -22,15 +21,16 @@ there is nothing to reclaim there.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
import os
|
import os
|
||||||
import shutil
|
|
||||||
import signal
|
import signal
|
||||||
import subprocess
|
import subprocess
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ...log import info
|
from ...log import info
|
||||||
from .. import EnumerationError
|
from .. import EnumerationError
|
||||||
from . import util
|
from ..cleanup_control import CleanupFailures
|
||||||
|
from . import lifecycle_lock, util
|
||||||
from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan
|
from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan
|
||||||
|
|
||||||
|
|
||||||
@@ -38,7 +38,7 @@ def _run_root() -> Path:
|
|||||||
return util.cache_dir() / "run"
|
return util.cache_dir() / "run"
|
||||||
|
|
||||||
|
|
||||||
def _run_dir_of(cmd: str, run_root: Path) -> Path | None:
|
def _run_dir_of(args: Sequence[str], run_root: Path) -> Path | None:
|
||||||
"""The bottle run dir a firecracker cmdline belongs to, or None.
|
"""The bottle run dir a firecracker cmdline belongs to, or None.
|
||||||
|
|
||||||
A bottle VM is launched with `--config-file <run_root>/<slug>/config.json`,
|
A bottle VM is launched with `--config-file <run_root>/<slug>/config.json`,
|
||||||
@@ -46,15 +46,35 @@ def _run_dir_of(cmd: str, run_root: Path) -> Path | None:
|
|||||||
the run root. Anything else (a builder VM, the infra VM elsewhere) is
|
the run root. Anything else (a builder VM, the infra VM elsewhere) is
|
||||||
not ours to reap here.
|
not ours to reap here.
|
||||||
"""
|
"""
|
||||||
toks = cmd.split()
|
for i, arg in enumerate(args):
|
||||||
for i, tok in enumerate(toks):
|
if arg == "--config-file" and i + 1 < len(args):
|
||||||
if tok == "--config-file" and i + 1 < len(toks):
|
parent = Path(args[i + 1]).parent
|
||||||
parent = Path(toks[i + 1]).parent
|
|
||||||
if parent.parent == run_root:
|
if parent.parent == run_root:
|
||||||
return parent
|
return parent
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_cmdline(raw: bytes) -> tuple[str, ...]:
|
||||||
|
"""Decode Linux's NUL-delimited argv without losing embedded spaces."""
|
||||||
|
return tuple(
|
||||||
|
value.decode(errors="surrogateescape")
|
||||||
|
for value in raw.split(b"\0") if value
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _process_args(pid: int) -> tuple[str, ...] | None:
|
||||||
|
"""Read one process's lossless argv, or None when it exited meanwhile."""
|
||||||
|
try:
|
||||||
|
raw = Path(f"/proc/{pid}/cmdline").read_bytes()
|
||||||
|
except FileNotFoundError:
|
||||||
|
return None
|
||||||
|
except OSError as exc:
|
||||||
|
raise EnumerationError(
|
||||||
|
f"could not inspect Firecracker pid {pid}: {exc}"
|
||||||
|
) from exc
|
||||||
|
return _decode_cmdline(raw)
|
||||||
|
|
||||||
|
|
||||||
def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
|
def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
|
||||||
"""Inspect running firecracker VMs under ``run_root``.
|
"""Inspect running firecracker VMs under ``run_root``.
|
||||||
|
|
||||||
@@ -65,7 +85,7 @@ def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
|
|||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["pgrep", "-a", "firecracker"],
|
["pgrep", "firecracker"],
|
||||||
capture_output=True, text=True, check=False,
|
capture_output=True, text=True, check=False,
|
||||||
)
|
)
|
||||||
except OSError as exc:
|
except OSError as exc:
|
||||||
@@ -83,14 +103,14 @@ def _scan_processes(run_root: Path) -> tuple[set[str], list[int]]:
|
|||||||
live: set[str] = set()
|
live: set[str] = set()
|
||||||
orphan_pids: list[int] = []
|
orphan_pids: list[int] = []
|
||||||
for line in result.stdout.splitlines():
|
for line in result.stdout.splitlines():
|
||||||
parts = line.split(None, 1)
|
|
||||||
if len(parts) != 2:
|
|
||||||
continue
|
|
||||||
try:
|
try:
|
||||||
pid = int(parts[0])
|
pid = int(line.strip())
|
||||||
except ValueError:
|
except ValueError:
|
||||||
continue
|
continue
|
||||||
run_dir = _run_dir_of(parts[1], run_root)
|
args = _process_args(pid)
|
||||||
|
if args is None:
|
||||||
|
continue
|
||||||
|
run_dir = _run_dir_of(args, run_root)
|
||||||
if run_dir is None:
|
if run_dir is None:
|
||||||
continue
|
continue
|
||||||
if run_dir.is_dir():
|
if run_dir.is_dir():
|
||||||
@@ -126,12 +146,44 @@ def prepare_cleanup() -> FirecrackerBottleCleanupPlan:
|
|||||||
|
|
||||||
|
|
||||||
def cleanup(plan: FirecrackerBottleCleanupPlan) -> None:
|
def cleanup(plan: FirecrackerBottleCleanupPlan) -> None:
|
||||||
for pid in plan.vm_pids:
|
"""Revalidate the preview under the launch lock, then remove its survivors."""
|
||||||
info(f"kill firecracker VM pid {pid}")
|
with lifecycle_lock.hold():
|
||||||
try:
|
fresh = prepare_cleanup()
|
||||||
os.kill(pid, signal.SIGTERM)
|
approved_pids = set(plan.vm_pids).intersection(fresh.vm_pids)
|
||||||
except ProcessLookupError:
|
approved_dirs = set(plan.run_dirs).intersection(fresh.run_dirs)
|
||||||
pass
|
failures = CleanupFailures()
|
||||||
for path in plan.run_dirs:
|
for pid in sorted(approved_pids):
|
||||||
|
_terminate_orphan(pid, _run_root())
|
||||||
|
for path in sorted(approved_dirs):
|
||||||
info(f"rm -rf {path}")
|
info(f"rm -rf {path}")
|
||||||
shutil.rmtree(path, ignore_errors=True)
|
failures.remove_tree(Path(path), f"removing Firecracker run dir {path}")
|
||||||
|
failures.raise_if_any()
|
||||||
|
|
||||||
|
|
||||||
|
def _terminate_orphan(pid: int, run_root: Path) -> None:
|
||||||
|
"""Signal exactly the process identity that still owns an orphan config."""
|
||||||
|
try:
|
||||||
|
pidfd = os.pidfd_open(pid)
|
||||||
|
except ProcessLookupError:
|
||||||
|
return
|
||||||
|
except OSError as exc:
|
||||||
|
raise EnumerationError(
|
||||||
|
f"could not pin Firecracker pid {pid} for cleanup: {exc}"
|
||||||
|
) from exc
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
raw = Path(f"/proc/{pid}/cmdline").read_bytes()
|
||||||
|
except FileNotFoundError:
|
||||||
|
return
|
||||||
|
except OSError as exc:
|
||||||
|
raise EnumerationError(
|
||||||
|
f"could not revalidate Firecracker pid {pid}: {exc}"
|
||||||
|
) from exc
|
||||||
|
args = _decode_cmdline(raw)
|
||||||
|
run_dir = _run_dir_of(args, run_root)
|
||||||
|
if run_dir is None or run_dir.is_dir():
|
||||||
|
return
|
||||||
|
info(f"kill firecracker VM pid {pid}")
|
||||||
|
signal.pidfd_send_signal(pidfd, signal.SIGTERM)
|
||||||
|
finally:
|
||||||
|
os.close(pidfd)
|
||||||
|
|||||||
@@ -41,6 +41,8 @@ from ... import resources
|
|||||||
from ...log import die, info
|
from ...log import die, info
|
||||||
from . import util
|
from . import util
|
||||||
|
|
||||||
|
ARTIFACT_HTTP_TIMEOUT_SECONDS = 30.0
|
||||||
|
|
||||||
# Bump if the on-disk artifact *format* changes (compression, layout) so a new
|
# Bump if the on-disk artifact *format* changes (compression, layout) so a new
|
||||||
# scheme can't collide with a cached/published artifact of the old one.
|
# scheme can't collide with a cached/published artifact of the old one.
|
||||||
_ARTIFACT_FORMAT = "1"
|
_ARTIFACT_FORMAT = "1"
|
||||||
@@ -164,7 +166,9 @@ def _download(url: str, dest: Path) -> None:
|
|||||||
"""Stream `url` to `dest` (atomic via a `.part` sibling)."""
|
"""Stream `url` to `dest` (atomic via a `.part` sibling)."""
|
||||||
tmp = dest.with_suffix(dest.suffix + ".part")
|
tmp = dest.with_suffix(dest.suffix + ".part")
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(_open(url)) as resp, open(tmp, "wb") as out:
|
with urllib.request.urlopen(
|
||||||
|
_open(url), timeout=ARTIFACT_HTTP_TIMEOUT_SECONDS,
|
||||||
|
) as resp, open(tmp, "wb") as out:
|
||||||
shutil.copyfileobj(resp, out, _CHUNK)
|
shutil.copyfileobj(resp, out, _CHUNK)
|
||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
tmp.unlink(missing_ok=True)
|
tmp.unlink(missing_ok=True)
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ from ...log import die, info, warn
|
|||||||
from ...supervisor.types import SUPERVISE_PORT
|
from ...supervisor.types import SUPERVISE_PORT
|
||||||
from ..docker.egress import EGRESS_PORT
|
from ..docker.egress import EGRESS_PORT
|
||||||
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||||
from . import firecracker_vm, image_builder, isolation_probe, netpool, util
|
from . import firecracker_vm, image_builder, isolation_probe, lifecycle_lock, netpool, util
|
||||||
from .bottle import FirecrackerBottle
|
from .bottle import FirecrackerBottle
|
||||||
from .bottle_plan import FirecrackerBottlePlan
|
from .bottle_plan import FirecrackerBottlePlan
|
||||||
from ...orchestrator.store.config_store import resolve_teardown_timeout
|
from ...orchestrator.store.config_store import resolve_teardown_timeout
|
||||||
@@ -164,11 +164,15 @@ def launch(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Step 6: build the per-bottle rootfs + SSH key, then boot.
|
# Step 6: build the per-bottle rootfs + SSH key, then boot.
|
||||||
|
# Cleanup takes the same lock while refreshing its process snapshot.
|
||||||
|
# Hold it until the VMM exists so a newly-created run dir can never be
|
||||||
|
# mistaken for an orphan in the build-before-boot window.
|
||||||
|
with lifecycle_lock.hold():
|
||||||
run_dir = util.cache_dir() / "run" / plan.slug
|
run_dir = util.cache_dir() / "run" / plan.slug
|
||||||
run_dir.mkdir(parents=True, exist_ok=True)
|
run_dir.mkdir(parents=True, exist_ok=True)
|
||||||
# Remove the run dir on teardown so the per-bottle rootfs.ext4 (~1G)
|
# Remove the run dir on teardown so the per-bottle rootfs.ext4 (~1G)
|
||||||
# doesn't leak. Registered before vm.terminate below so it runs *after*
|
# doesn't leak. Registered before vm.terminate below so it runs
|
||||||
# it (ExitStack is LIFO): the VM is gone before we rm its rootfs.
|
# *after* it (ExitStack is LIFO).
|
||||||
stack.callback(lambda: shutil.rmtree(run_dir, ignore_errors=True))
|
stack.callback(lambda: shutil.rmtree(run_dir, ignore_errors=True))
|
||||||
rootfs = run_dir / "rootfs.ext4"
|
rootfs = run_dir / "rootfs.ext4"
|
||||||
util.build_rootfs_ext4(agent_base, rootfs)
|
util.build_rootfs_ext4(agent_base, rootfs)
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
"""Serialize Firecracker run-directory creation with orphan cleanup."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import fcntl
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Generator
|
||||||
|
|
||||||
|
from . import util
|
||||||
|
|
||||||
|
|
||||||
|
def _lock_path() -> Path:
|
||||||
|
return util.cache_dir() / "run.lifecycle.lock"
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def hold() -> Generator[None]:
|
||||||
|
"""Exclude cleanup while a launch directory lacks a visible VMM."""
|
||||||
|
path = _lock_path()
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
with path.open("a", encoding="utf-8") as handle:
|
||||||
|
fcntl.flock(handle, fcntl.LOCK_EX)
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
fcntl.flock(handle, fcntl.LOCK_UN)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["hold"]
|
||||||
@@ -34,6 +34,7 @@ from pathlib import Path
|
|||||||
from . import infra_artifact, infra_vm, util
|
from . import infra_artifact, infra_vm, util
|
||||||
|
|
||||||
_CHUNK = 1 << 20
|
_CHUNK = 1 << 20
|
||||||
|
_REGISTRY_HTTP_TIMEOUT_SECONDS = 30.0
|
||||||
|
|
||||||
_GZ_NAME = "rootfs.ext4.gz"
|
_GZ_NAME = "rootfs.ext4.gz"
|
||||||
_SHA_NAME = "rootfs.ext4.gz.sha256"
|
_SHA_NAME = "rootfs.ext4.gz.sha256"
|
||||||
@@ -91,7 +92,9 @@ def _put(url: str, body: "bytes | Path", token: str) -> None:
|
|||||||
req.add_header("Authorization", f"token {token}")
|
req.add_header("Authorization", f"token {token}")
|
||||||
req.add_header("Content-Type", "application/octet-stream")
|
req.add_header("Content-Type", "application/octet-stream")
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(req) as resp:
|
with urllib.request.urlopen(
|
||||||
|
req, timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
|
||||||
|
) as resp:
|
||||||
print(f" uploaded {url} (HTTP {resp.status})")
|
print(f" uploaded {url} (HTTP {resp.status})")
|
||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
if e.code == 409:
|
if e.code == 409:
|
||||||
@@ -112,7 +115,9 @@ def _delete(url: str, token: str) -> None:
|
|||||||
if token:
|
if token:
|
||||||
req.add_header("Authorization", f"token {token}")
|
req.add_header("Authorization", f"token {token}")
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(req):
|
with urllib.request.urlopen(
|
||||||
|
req, timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
|
||||||
|
):
|
||||||
pass
|
pass
|
||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
if e.code != 404:
|
if e.code != 404:
|
||||||
@@ -151,7 +156,10 @@ def _try_download_published(role: str, role_dir: Path) -> str | None:
|
|||||||
version = _role_version(role)
|
version = _role_version(role)
|
||||||
sha_url = infra_artifact.artifact_url(version, _SHA_NAME, role=role)
|
sha_url = infra_artifact.artifact_url(version, _SHA_NAME, role=role)
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(infra_artifact._open(sha_url)):
|
with urllib.request.urlopen(
|
||||||
|
infra_artifact._open(sha_url),
|
||||||
|
timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
|
||||||
|
):
|
||||||
pass
|
pass
|
||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
if e.code == 404:
|
if e.code == 404:
|
||||||
@@ -195,7 +203,10 @@ def _publish_bundle(role: str, role_dir: Path, token: str) -> str:
|
|||||||
# present, a re-publish is a no-op. Otherwise clear any partial upload left
|
# present, a re-publish is a no-op. Otherwise clear any partial upload left
|
||||||
# by an interrupted prior attempt and upload the complete set.
|
# by an interrupted prior attempt and upload the complete set.
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(infra_artifact._open(sha_url)) as resp:
|
with urllib.request.urlopen(
|
||||||
|
infra_artifact._open(sha_url),
|
||||||
|
timeout=_REGISTRY_HTTP_TIMEOUT_SECONDS,
|
||||||
|
) as resp:
|
||||||
remote_sha = resp.read().decode("utf-8").split()[0].strip().lower()
|
remote_sha = resp.read().decode("utf-8").split()[0].strip().lower()
|
||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
if e.code != 404:
|
if e.code != 404:
|
||||||
|
|||||||
@@ -25,3 +25,11 @@ class MacosContainerBottleCleanupPlan(BottleCleanupPlan):
|
|||||||
@property
|
@property
|
||||||
def empty(self) -> bool:
|
def empty(self) -> bool:
|
||||||
return not self.containers and not self.networks
|
return not self.containers and not self.networks
|
||||||
|
|
||||||
|
def intersect(self, current: BottleCleanupPlan) -> "MacosContainerBottleCleanupPlan":
|
||||||
|
if not isinstance(current, MacosContainerBottleCleanupPlan):
|
||||||
|
raise TypeError("cleanup plans must have the same backend type")
|
||||||
|
return MacosContainerBottleCleanupPlan(
|
||||||
|
containers=tuple(x for x in self.containers if x in current.containers),
|
||||||
|
networks=tuple(x for x in self.networks if x in current.networks),
|
||||||
|
)
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
from ...log import info, warn
|
from .. import EnumerationError
|
||||||
|
from ..cleanup_control import CleanupFailures
|
||||||
|
from ...log import info
|
||||||
from . import util as container_mod
|
from . import util as container_mod
|
||||||
from .bottle_cleanup_plan import MacosContainerBottleCleanupPlan
|
from .bottle_cleanup_plan import MacosContainerBottleCleanupPlan
|
||||||
|
|
||||||
@@ -19,8 +21,8 @@ def _list_prefixed_containers() -> list[str]:
|
|||||||
check=False,
|
check=False,
|
||||||
)
|
)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
warn(f"container list failed: {result.stderr.strip()}")
|
detail = result.stderr.strip() or f"exit {result.returncode}"
|
||||||
return []
|
raise EnumerationError(f"container list failed: {detail}")
|
||||||
return sorted(
|
return sorted(
|
||||||
name for name in (line.strip() for line in result.stdout.splitlines())
|
name for name in (line.strip() for line in result.stdout.splitlines())
|
||||||
if name.startswith(_PREFIX)
|
if name.startswith(_PREFIX)
|
||||||
@@ -35,7 +37,8 @@ def _list_prefixed_networks() -> list[str]:
|
|||||||
check=False,
|
check=False,
|
||||||
)
|
)
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
return []
|
detail = result.stderr.strip() or f"exit {result.returncode}"
|
||||||
|
raise EnumerationError(f"container network list failed: {detail}")
|
||||||
return sorted(
|
return sorted(
|
||||||
name for name in (line.strip() for line in result.stdout.splitlines())
|
name for name in (line.strip() for line in result.stdout.splitlines())
|
||||||
if name.startswith(_PREFIX)
|
if name.startswith(_PREFIX)
|
||||||
@@ -51,19 +54,17 @@ def prepare_cleanup() -> MacosContainerBottleCleanupPlan:
|
|||||||
|
|
||||||
|
|
||||||
def cleanup(plan: MacosContainerBottleCleanupPlan) -> None:
|
def cleanup(plan: MacosContainerBottleCleanupPlan) -> None:
|
||||||
|
failures = CleanupFailures()
|
||||||
for name in plan.containers:
|
for name in plan.containers:
|
||||||
info(f"container delete --force {name}")
|
info(f"container delete --force {name}")
|
||||||
subprocess.run(
|
failures.run(
|
||||||
["container", "delete", "--force", name],
|
["container", "delete", "--force", name],
|
||||||
stdout=subprocess.DEVNULL,
|
f"deleting container {name}",
|
||||||
stderr=subprocess.DEVNULL,
|
|
||||||
check=False,
|
|
||||||
)
|
)
|
||||||
for name in plan.networks:
|
for name in plan.networks:
|
||||||
info(f"container network delete {name}")
|
info(f"container network delete {name}")
|
||||||
subprocess.run(
|
failures.run(
|
||||||
["container", "network", "delete", name],
|
["container", "network", "delete", name],
|
||||||
stdout=subprocess.DEVNULL,
|
f"deleting network {name}",
|
||||||
stderr=subprocess.DEVNULL,
|
|
||||||
check=False,
|
|
||||||
)
|
)
|
||||||
|
failures.raise_if_any()
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ from __future__ import annotations
|
|||||||
import sys
|
import sys
|
||||||
|
|
||||||
from ...backend import get_bottle_backend, has_backend, known_backend_names
|
from ...backend import get_bottle_backend, has_backend, known_backend_names
|
||||||
|
from ...backend.cleanup_control import CleanupError
|
||||||
from ...log import info
|
from ...log import info
|
||||||
from ...util import read_tty_line
|
from ...util import read_tty_line
|
||||||
|
|
||||||
@@ -52,10 +53,20 @@ def cmd_cleanup(_argv: list[str]) -> int:
|
|||||||
info("cleanup: skipped")
|
info("cleanup: skipped")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
for name, backend, plan in prepared:
|
# Confirmation authorizes a fresh authoritative snapshot, not blind use of
|
||||||
if plan.empty:
|
# identities that may have changed while the operator reviewed the preview.
|
||||||
|
failures: list[str] = []
|
||||||
|
for name, backend, displayed in prepared:
|
||||||
|
current = backend.prepare_cleanup()
|
||||||
|
approved = displayed.intersect(current)
|
||||||
|
if approved.empty:
|
||||||
continue
|
continue
|
||||||
backend.cleanup(plan)
|
try:
|
||||||
|
backend.cleanup(approved)
|
||||||
|
except CleanupError as exc:
|
||||||
|
failures.append(f"{name}: {exc}")
|
||||||
|
if failures:
|
||||||
|
raise CleanupError("cleanup incomplete: " + "; ".join(failures))
|
||||||
info("cleanup: done")
|
info("cleanup: done")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|||||||
@@ -136,10 +136,18 @@ def _pump(name: str, stream: IO[bytes]) -> None:
|
|||||||
"""Read lines from `stream`, prefix with `[name]`, write to
|
"""Read lines from `stream`, prefix with `[name]`, write to
|
||||||
stdout. Runs in its own thread per child; daemon=True so a
|
stdout. Runs in its own thread per child; daemon=True so a
|
||||||
blocked read doesn't keep the process alive after main exits."""
|
blocked read doesn't keep the process alive after main exits."""
|
||||||
|
try:
|
||||||
for raw in iter(stream.readline, b""):
|
for raw in iter(stream.readline, b""):
|
||||||
line = raw.decode("utf-8", errors="replace").rstrip("\n")
|
line = raw.decode("utf-8", errors="replace").rstrip("\n")
|
||||||
sys.stdout.write(f"[{name}] {line}\n")
|
sys.stdout.write(f"[{name}] {line}\n")
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
|
except (OSError, ValueError) as exc:
|
||||||
|
# The manager closes a dead child's pipe after wait() and before a
|
||||||
|
# restart. A pump can be between readline calls at that exact moment;
|
||||||
|
# closed-stream errors are normal completion, not uncaught thread
|
||||||
|
# failures. Preserve genuinely unexpected I/O diagnostics.
|
||||||
|
if not stream.closed:
|
||||||
|
_log(f"{name} output pump stopped: {type(exc).__name__}: {exc}")
|
||||||
|
|
||||||
|
|
||||||
def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]:
|
def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]:
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
"""Shared resource boundaries for gateway stdlib HTTP services."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import http.server
|
||||||
|
import socket
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any, Protocol
|
||||||
|
|
||||||
|
|
||||||
|
class Readable(Protocol):
|
||||||
|
def read(self, size: int = -1, /) -> bytes: ...
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class BodyReadError(Exception):
|
||||||
|
status: int
|
||||||
|
message: str
|
||||||
|
|
||||||
|
|
||||||
|
def read_declared_body(
|
||||||
|
stream: Readable,
|
||||||
|
connection: socket.socket,
|
||||||
|
raw_length: str | None,
|
||||||
|
*,
|
||||||
|
maximum: int,
|
||||||
|
timeout_seconds: float,
|
||||||
|
require_length: bool,
|
||||||
|
) -> bytes:
|
||||||
|
"""Validate and read exactly one declared body under a read deadline."""
|
||||||
|
if raw_length is None:
|
||||||
|
if require_length:
|
||||||
|
raise BodyReadError(411, "Content-Length required")
|
||||||
|
raw_length = "0"
|
||||||
|
try:
|
||||||
|
length = int(raw_length)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise BodyReadError(400, "invalid Content-Length") from exc
|
||||||
|
if length < 0:
|
||||||
|
raise BodyReadError(400, "invalid Content-Length")
|
||||||
|
if length > maximum:
|
||||||
|
raise BodyReadError(413, "request body too large")
|
||||||
|
previous_timeout = connection.gettimeout()
|
||||||
|
deadline = time.monotonic() + timeout_seconds
|
||||||
|
chunks: list[bytes] = []
|
||||||
|
remaining = length
|
||||||
|
try:
|
||||||
|
while remaining:
|
||||||
|
timeout = deadline - time.monotonic()
|
||||||
|
if timeout <= 0:
|
||||||
|
raise BodyReadError(408, "request body read timed out")
|
||||||
|
connection.settimeout(timeout)
|
||||||
|
chunk = stream.read(min(remaining, 64 * 1024))
|
||||||
|
if not chunk:
|
||||||
|
raise BodyReadError(400, "incomplete request body")
|
||||||
|
chunks.append(chunk)
|
||||||
|
remaining -= len(chunk)
|
||||||
|
except TimeoutError as exc:
|
||||||
|
raise BodyReadError(408, "request body read timed out") from exc
|
||||||
|
finally:
|
||||||
|
connection.settimeout(previous_timeout)
|
||||||
|
return b"".join(chunks)
|
||||||
|
|
||||||
|
|
||||||
|
class BoundedThreadingHTTPServer(http.server.ThreadingHTTPServer):
|
||||||
|
"""ThreadingHTTPServer with a hard cap on in-flight request threads."""
|
||||||
|
|
||||||
|
daemon_threads = True
|
||||||
|
|
||||||
|
def __init__( # pylint: disable=consider-using-with
|
||||||
|
self, *args, max_workers: int = 32, **kwargs, # type: ignore[no-untyped-def]
|
||||||
|
):
|
||||||
|
if max_workers < 1:
|
||||||
|
raise ValueError("max_workers must be positive")
|
||||||
|
self._request_slots = threading.BoundedSemaphore(max_workers)
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
def process_request(
|
||||||
|
self, request: Any, client_address: Any,
|
||||||
|
) -> None:
|
||||||
|
if not self._request_slots.acquire( # pylint: disable=consider-using-with
|
||||||
|
blocking=False,
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
request.sendall(
|
||||||
|
b"HTTP/1.1 503 Service Unavailable\r\n"
|
||||||
|
b"Content-Length: 0\r\nConnection: close\r\n\r\n"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
self.shutdown_request(request)
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
super().process_request(request, client_address)
|
||||||
|
except BaseException:
|
||||||
|
self._request_slots.release()
|
||||||
|
raise
|
||||||
|
|
||||||
|
def process_request_thread(
|
||||||
|
self, request: Any, client_address: Any,
|
||||||
|
) -> None:
|
||||||
|
try:
|
||||||
|
super().process_request_thread(request, client_address)
|
||||||
|
finally:
|
||||||
|
self._request_slots.release()
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["BodyReadError", "BoundedThreadingHTTPServer", "read_declared_body"]
|
||||||
@@ -16,7 +16,7 @@ import typing
|
|||||||
from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=import-error
|
from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=import-error
|
||||||
|
|
||||||
from bot_bottle.constants import IDENTITY_HEADER
|
from bot_bottle.constants import IDENTITY_HEADER
|
||||||
from bot_bottle.gateway.egress.dlp_detectors import redact_tokens, strip_crlf
|
from bot_bottle.gateway.egress.dlp_detectors import redact_tokens
|
||||||
from bot_bottle.gateway.egress.dlp_config import (
|
from bot_bottle.gateway.egress.dlp_config import (
|
||||||
DEFAULT_OUTBOUND_ON_MATCH,
|
DEFAULT_OUTBOUND_ON_MATCH,
|
||||||
ON_MATCH_BLOCK,
|
ON_MATCH_BLOCK,
|
||||||
@@ -25,19 +25,19 @@ from bot_bottle.gateway.egress.dlp_config import (
|
|||||||
from bot_bottle.gateway.egress.context import resolve_client_context
|
from bot_bottle.gateway.egress.context import resolve_client_context
|
||||||
from bot_bottle.gateway.egress.dlp import (
|
from bot_bottle.gateway.egress.dlp import (
|
||||||
build_inbound_scan_text,
|
build_inbound_scan_text,
|
||||||
build_outbound_scan_text,
|
|
||||||
build_token_allow_payload,
|
build_token_allow_payload,
|
||||||
outbound_scan_headers,
|
|
||||||
scan_inbound,
|
scan_inbound,
|
||||||
scan_outbound,
|
scan_outbound,
|
||||||
)
|
)
|
||||||
|
from bot_bottle.gateway.egress.outbound_pipeline import redact_request, scan_request
|
||||||
from bot_bottle.gateway.egress.matching import (
|
from bot_bottle.gateway.egress.matching import (
|
||||||
decide,
|
decide,
|
||||||
decide_git_fetch,
|
|
||||||
is_git_fetch_request,
|
|
||||||
is_git_push_request,
|
|
||||||
match_route,
|
match_route,
|
||||||
)
|
)
|
||||||
|
from bot_bottle.gateway.egress.request_pipeline import (
|
||||||
|
evaluate_route_policy,
|
||||||
|
git_block_reason,
|
||||||
|
)
|
||||||
from bot_bottle.gateway.egress.schema import route_to_yaml_dict
|
from bot_bottle.gateway.egress.schema import route_to_yaml_dict
|
||||||
from bot_bottle.gateway.egress.types import (
|
from bot_bottle.gateway.egress.types import (
|
||||||
LOG_BLOCKS,
|
LOG_BLOCKS,
|
||||||
@@ -435,21 +435,12 @@ class EgressAddon:
|
|||||||
request_path: str, query: str,
|
request_path: str, query: str,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Apply the HTTPS Git push/fetch boundary before general routing."""
|
"""Apply the HTTPS Git push/fetch boundary before general routing."""
|
||||||
if is_git_push_request(request_path, query):
|
reason = git_block_reason(
|
||||||
self._block(
|
config.routes, flow.request.pretty_host, request_path, query,
|
||||||
flow,
|
|
||||||
"egress: git push over HTTPS is not supported; "
|
|
||||||
"use the bottle.git SSH path (gitleaks-scanned by "
|
|
||||||
"git-gate's pre-receive hook).",
|
|
||||||
ctx=self._req_ctx(flow),
|
|
||||||
)
|
)
|
||||||
return False
|
if not reason:
|
||||||
if not is_git_fetch_request(request_path, query):
|
|
||||||
return True
|
return True
|
||||||
git_decision = decide_git_fetch(config.routes, flow.request.pretty_host)
|
self._block(flow, reason, ctx=self._req_ctx(flow))
|
||||||
if git_decision.action != "block":
|
|
||||||
return True
|
|
||||||
self._block(flow, git_decision.reason, ctx=self._req_ctx(flow))
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def _apply_route_policy(
|
def _apply_route_policy(
|
||||||
@@ -461,30 +452,26 @@ class EgressAddon:
|
|||||||
# are caught above; the route may inject gateway-owned auth below.
|
# are caught above; the route may inject gateway-owned auth below.
|
||||||
# Routes with preserve_auth=True pass the header through as-is so the
|
# Routes with preserve_auth=True pass the header through as-is so the
|
||||||
# agent's own credentials (e.g. registry bearer tokens) reach the upstream.
|
# agent's own credentials (e.g. registry bearer tokens) reach the upstream.
|
||||||
if route is None or not route.preserve_auth:
|
result = evaluate_route_policy(
|
||||||
|
config,
|
||||||
|
route,
|
||||||
|
host=flow.request.pretty_host,
|
||||||
|
request_path=request_path,
|
||||||
|
method=flow.request.method,
|
||||||
|
headers=dict(flow.request.headers),
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
if result.strip_authorization:
|
||||||
flow.request.headers.pop("authorization", None)
|
flow.request.headers.pop("authorization", None)
|
||||||
|
|
||||||
# Build headers mapping for match evaluation
|
if result.block_reason:
|
||||||
req_headers = {k.lower(): v for k, v in flow.request.headers.items()}
|
self._block(flow, result.block_reason, ctx=self._req_ctx(flow))
|
||||||
|
|
||||||
decision = decide(
|
|
||||||
config.routes,
|
|
||||||
flow.request.pretty_host,
|
|
||||||
request_path,
|
|
||||||
env,
|
|
||||||
request_method=flow.request.method,
|
|
||||||
request_headers=req_headers,
|
|
||||||
deny_reason=config.deny_reason,
|
|
||||||
)
|
|
||||||
|
|
||||||
if decision.action == "block":
|
|
||||||
self._block(flow, decision.reason, ctx=self._req_ctx(flow))
|
|
||||||
return
|
return
|
||||||
|
|
||||||
if decision.inject_authorization is not None:
|
if result.inject_authorization is not None:
|
||||||
flow.request.headers["authorization"] = decision.inject_authorization
|
flow.request.headers["authorization"] = result.inject_authorization
|
||||||
|
|
||||||
if config.log >= LOG_FULL:
|
if result.log_request:
|
||||||
self._log_request(flow, env)
|
self._log_request(flow, env)
|
||||||
|
|
||||||
def _block_dlp(self, flow: http.HTTPFlow, result: ScanResult) -> None:
|
def _block_dlp(self, flow: http.HTTPFlow, result: ScanResult) -> None:
|
||||||
@@ -508,20 +495,12 @@ class EgressAddon:
|
|||||||
Loops so the supervise policy can re-scan after each approval — a
|
Loops so the supervise policy can re-scan after each approval — a
|
||||||
second, un-approved token in the same request is still caught."""
|
second, un-approved token in the same request is still caught."""
|
||||||
while True:
|
while True:
|
||||||
request_path, _, query = flow.request.path.partition("?")
|
request_path, _, _ = flow.request.path.partition("?")
|
||||||
body = flow.request.get_text(strict=False) or ""
|
result = scan_request(
|
||||||
headers = outbound_scan_headers(route, dict(flow.request.headers))
|
flow.request,
|
||||||
scan_text = build_outbound_scan_text(
|
route,
|
||||||
flow.request.pretty_host, request_path, query, headers, body,
|
env,
|
||||||
)
|
safe_tokens=self._safe_tokens_for(slug),
|
||||||
# CRLF is scanned only over the request line + headers, never the
|
|
||||||
# body (see scan_outbound) — a body is not an injection vector.
|
|
||||||
crlf_text = build_outbound_scan_text(
|
|
||||||
flow.request.pretty_host, request_path, query, headers, "",
|
|
||||||
)
|
|
||||||
result = scan_outbound(
|
|
||||||
route, scan_text, env,
|
|
||||||
safe_tokens=self._safe_tokens_for(slug), crlf_text=crlf_text,
|
|
||||||
)
|
)
|
||||||
if result is None or result.severity != "block":
|
if result is None or result.severity != "block":
|
||||||
return True
|
return True
|
||||||
@@ -531,7 +510,7 @@ class EgressAddon:
|
|||||||
# redact scrubs every detection (tokens and structural CRLF) and
|
# redact scrubs every detection (tokens and structural CRLF) and
|
||||||
# forwards; it fails closed only if a match survives the scrub.
|
# forwards; it fails closed only if a match survives the scrub.
|
||||||
if policy == ON_MATCH_REDACT:
|
if policy == ON_MATCH_REDACT:
|
||||||
if self._redact_outbound(flow, route, env):
|
if redact_request(flow.request, route, env):
|
||||||
if self._flow_log(flow) >= LOG_BLOCKS:
|
if self._flow_log(flow) >= LOG_BLOCKS:
|
||||||
sys.stderr.write(json.dumps({
|
sys.stderr.write(json.dumps({
|
||||||
"event": "egress_redacted",
|
"event": "egress_redacted",
|
||||||
@@ -564,41 +543,6 @@ class EgressAddon:
|
|||||||
return False # _supervise_token_block wrote the 403 response
|
return False # _supervise_token_block wrote the 403 response
|
||||||
# loop: the approved value is now in safe_tokens; re-scan.
|
# loop: the approved value is now in safe_tokens; re-scan.
|
||||||
|
|
||||||
def _redact_outbound(
|
|
||||||
self, flow: http.HTTPFlow, route: Route, env: "typing.Mapping[str, str]",
|
|
||||||
) -> bool:
|
|
||||||
"""Scrub detected tokens (and CRLF injection sequences) from the mutable
|
|
||||||
request surfaces (body, headers, path/query) and re-scan. `env` is the
|
|
||||||
per-bottle env overlay. Returns True if the request is now clean; False
|
|
||||||
if a block-severity match remains on a surface redaction cannot rewrite
|
|
||||||
(the hostname) so the caller fails closed."""
|
|
||||||
body = flow.request.get_text(strict=False)
|
|
||||||
if body:
|
|
||||||
redacted_body = redact_tokens(body, env=env)
|
|
||||||
if redacted_body != body:
|
|
||||||
flow.request.text = redacted_body
|
|
||||||
for name, value in list(flow.request.headers.items()):
|
|
||||||
if name.lower() == "host":
|
|
||||||
continue # routing-critical; never a legitimate token
|
|
||||||
redacted = strip_crlf(redact_tokens(value, env=env))
|
|
||||||
if redacted != value:
|
|
||||||
flow.request.headers[name] = redacted
|
|
||||||
redacted_path = strip_crlf(redact_tokens(flow.request.path, env=env))
|
|
||||||
if redacted_path != flow.request.path:
|
|
||||||
flow.request.path = redacted_path
|
|
||||||
|
|
||||||
request_path, _, query = flow.request.path.partition("?")
|
|
||||||
new_body = flow.request.get_text(strict=False) or ""
|
|
||||||
headers = outbound_scan_headers(route, dict(flow.request.headers))
|
|
||||||
scan_text = build_outbound_scan_text(
|
|
||||||
flow.request.pretty_host, request_path, query, headers, new_body,
|
|
||||||
)
|
|
||||||
crlf_text = build_outbound_scan_text(
|
|
||||||
flow.request.pretty_host, request_path, query, headers, "",
|
|
||||||
)
|
|
||||||
result = scan_outbound(route, scan_text, env, crlf_text=crlf_text)
|
|
||||||
return result is None or result.severity != "block"
|
|
||||||
|
|
||||||
async def _supervise_token_block(
|
async def _supervise_token_block(
|
||||||
self,
|
self,
|
||||||
flow: http.HTTPFlow,
|
flow: http.HTTPFlow,
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
"""Outbound DLP request scanning and redaction for the egress pipeline."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import ItemsView, Mapping, Protocol
|
||||||
|
|
||||||
|
from .dlp import (
|
||||||
|
build_outbound_scan_text,
|
||||||
|
outbound_scan_headers,
|
||||||
|
scan_outbound,
|
||||||
|
)
|
||||||
|
from .dlp_detectors import redact_tokens, strip_crlf
|
||||||
|
from .types import Route, ScanResult
|
||||||
|
|
||||||
|
|
||||||
|
class MutableHeaders(Protocol):
|
||||||
|
def items(self) -> ItemsView[str, str]: ...
|
||||||
|
def __getitem__(self, name: str, /) -> str: ...
|
||||||
|
def __setitem__(self, name: str, value: str, /) -> None: ...
|
||||||
|
|
||||||
|
|
||||||
|
class MutableRequest(Protocol):
|
||||||
|
pretty_host: str
|
||||||
|
path: str
|
||||||
|
headers: MutableHeaders
|
||||||
|
text: str
|
||||||
|
|
||||||
|
def get_text(self, strict: bool = False) -> str | None: ...
|
||||||
|
|
||||||
|
|
||||||
|
def scan_request(
|
||||||
|
request: MutableRequest,
|
||||||
|
route: Route,
|
||||||
|
env: Mapping[str, str],
|
||||||
|
*,
|
||||||
|
safe_tokens: set[str] | None = None,
|
||||||
|
) -> ScanResult | None:
|
||||||
|
"""Scan all mutable outbound request surfaces in their canonical order."""
|
||||||
|
request_path, _, query = request.path.partition("?")
|
||||||
|
headers = outbound_scan_headers(route, dict(request.headers.items()))
|
||||||
|
body = request.get_text(strict=False) or ""
|
||||||
|
scan_text = build_outbound_scan_text(
|
||||||
|
request.pretty_host, request_path, query, headers, body,
|
||||||
|
)
|
||||||
|
# Bodies cannot alter HTTP framing, so CRLF detection is deliberately
|
||||||
|
# restricted to the request line and headers.
|
||||||
|
crlf_text = build_outbound_scan_text(
|
||||||
|
request.pretty_host, request_path, query, headers, "",
|
||||||
|
)
|
||||||
|
return scan_outbound(
|
||||||
|
route,
|
||||||
|
scan_text,
|
||||||
|
env,
|
||||||
|
safe_tokens=safe_tokens,
|
||||||
|
crlf_text=crlf_text,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def redact_request(
|
||||||
|
request: MutableRequest,
|
||||||
|
route: Route,
|
||||||
|
env: Mapping[str, str],
|
||||||
|
) -> bool:
|
||||||
|
"""Redact mutable request surfaces and return whether the result is clean."""
|
||||||
|
body = request.get_text(strict=False)
|
||||||
|
if body:
|
||||||
|
redacted_body = redact_tokens(body, env=env)
|
||||||
|
if redacted_body != body:
|
||||||
|
request.text = redacted_body
|
||||||
|
for name, value in list(request.headers.items()):
|
||||||
|
if name.lower() == "host":
|
||||||
|
continue
|
||||||
|
redacted = strip_crlf(redact_tokens(value, env=env))
|
||||||
|
if redacted != value:
|
||||||
|
request.headers[name] = redacted
|
||||||
|
redacted_path = strip_crlf(redact_tokens(request.path, env=env))
|
||||||
|
if redacted_path != request.path:
|
||||||
|
request.path = redacted_path
|
||||||
|
result = scan_request(request, route, env)
|
||||||
|
return result is None or result.severity != "block"
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["MutableRequest", "redact_request", "scan_request"]
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
"""Framework-neutral request policy stages for the egress adapter.
|
||||||
|
|
||||||
|
The mitmproxy addon owns flow mutation and response construction. This module
|
||||||
|
owns the ordered Git and route-policy decisions so those rules remain directly
|
||||||
|
testable without a live proxy flow.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Mapping, Sequence
|
||||||
|
|
||||||
|
from .matching import (
|
||||||
|
decide,
|
||||||
|
decide_git_fetch,
|
||||||
|
is_git_fetch_request,
|
||||||
|
is_git_push_request,
|
||||||
|
)
|
||||||
|
from .types import LOG_FULL, Config, Route
|
||||||
|
|
||||||
|
GIT_PUSH_BLOCK_REASON = (
|
||||||
|
"egress: git push over HTTPS is not supported; "
|
||||||
|
"use the bottle.git SSH path (gitleaks-scanned by "
|
||||||
|
"git-gate's pre-receive hook)."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RoutePolicyResult:
|
||||||
|
"""The flow mutations and outcome produced by general route policy."""
|
||||||
|
|
||||||
|
block_reason: str = ""
|
||||||
|
strip_authorization: bool = False
|
||||||
|
inject_authorization: str | None = None
|
||||||
|
log_request: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
def git_block_reason(
|
||||||
|
routes: Sequence[Route],
|
||||||
|
host: str,
|
||||||
|
request_path: str,
|
||||||
|
query: str,
|
||||||
|
) -> str:
|
||||||
|
"""Return the HTTPS Git policy denial, or ``""`` when allowed."""
|
||||||
|
if is_git_push_request(request_path, query):
|
||||||
|
return GIT_PUSH_BLOCK_REASON
|
||||||
|
if not is_git_fetch_request(request_path, query):
|
||||||
|
return ""
|
||||||
|
decision = decide_git_fetch(routes, host)
|
||||||
|
return decision.reason if decision.action == "block" else ""
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_route_policy(
|
||||||
|
config: Config,
|
||||||
|
route: Route | None,
|
||||||
|
*,
|
||||||
|
host: str,
|
||||||
|
request_path: str,
|
||||||
|
method: str,
|
||||||
|
headers: Mapping[str, str],
|
||||||
|
env: Mapping[str, str],
|
||||||
|
) -> RoutePolicyResult:
|
||||||
|
"""Evaluate authorization stripping, matching, injection, and logging."""
|
||||||
|
strip_authorization = route is None or not route.preserve_auth
|
||||||
|
effective_headers = {
|
||||||
|
name.lower(): value
|
||||||
|
for name, value in headers.items()
|
||||||
|
if not (strip_authorization and name.lower() == "authorization")
|
||||||
|
}
|
||||||
|
decision = decide(
|
||||||
|
config.routes,
|
||||||
|
host,
|
||||||
|
request_path,
|
||||||
|
env,
|
||||||
|
request_method=method,
|
||||||
|
request_headers=effective_headers,
|
||||||
|
deny_reason=config.deny_reason,
|
||||||
|
)
|
||||||
|
return RoutePolicyResult(
|
||||||
|
block_reason=decision.reason if decision.action == "block" else "",
|
||||||
|
strip_authorization=strip_authorization,
|
||||||
|
inject_authorization=decision.inject_authorization,
|
||||||
|
log_request=config.log >= LOG_FULL,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"GIT_PUSH_BLOCK_REASON",
|
||||||
|
"RoutePolicyResult",
|
||||||
|
"evaluate_route_policy",
|
||||||
|
"git_block_reason",
|
||||||
|
]
|
||||||
@@ -22,11 +22,16 @@ import os
|
|||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import typing
|
import typing
|
||||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
from http.server import BaseHTTPRequestHandler
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from urllib.parse import urlsplit
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
from bot_bottle.constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER
|
from bot_bottle.constants import GIT_GATE_TIMEOUT_SECS, IDENTITY_HEADER
|
||||||
|
from bot_bottle.gateway.bounded_http import (
|
||||||
|
BodyReadError,
|
||||||
|
BoundedThreadingHTTPServer,
|
||||||
|
read_declared_body,
|
||||||
|
)
|
||||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||||
|
|
||||||
|
|
||||||
@@ -77,6 +82,8 @@ def resolve_sandbox_root(
|
|||||||
|
|
||||||
# Bound memory use while still allowing ordinary git push packfiles.
|
# Bound memory use while still allowing ordinary git push packfiles.
|
||||||
MAX_BODY_BYTES = 100 * 1024 * 1024
|
MAX_BODY_BYTES = 100 * 1024 * 1024
|
||||||
|
REQUEST_BODY_TIMEOUT_SECONDS = 30.0
|
||||||
|
MAX_REQUEST_WORKERS = 16
|
||||||
|
|
||||||
|
|
||||||
class GitHttpHandler(BaseHTTPRequestHandler):
|
class GitHttpHandler(BaseHTTPRequestHandler):
|
||||||
@@ -184,19 +191,19 @@ class GitHttpHandler(BaseHTTPRequestHandler):
|
|||||||
value = self.headers.get(header)
|
value = self.headers.get(header)
|
||||||
if value:
|
if value:
|
||||||
env[variable] = value
|
env[variable] = value
|
||||||
raw_length = self.headers.get("content-length", "0") or "0"
|
|
||||||
try:
|
try:
|
||||||
length = int(raw_length)
|
body = read_declared_body(
|
||||||
except ValueError:
|
self.rfile,
|
||||||
self.send_error(400, "Bad Content-Length")
|
self.connection,
|
||||||
|
self.headers.get("content-length"),
|
||||||
|
maximum=MAX_BODY_BYTES,
|
||||||
|
timeout_seconds=REQUEST_BODY_TIMEOUT_SECONDS,
|
||||||
|
require_length=False,
|
||||||
|
)
|
||||||
|
except BodyReadError as exc:
|
||||||
|
self.send_error(exc.status, exc.message)
|
||||||
return
|
return
|
||||||
if length < 0:
|
try:
|
||||||
self.send_error(400, "Negative Content-Length")
|
|
||||||
return
|
|
||||||
if length > MAX_BODY_BYTES:
|
|
||||||
self.send_error(413, "Request body too large")
|
|
||||||
return
|
|
||||||
body = self.rfile.read(length) if length else b""
|
|
||||||
proc = subprocess.run(
|
proc = subprocess.run(
|
||||||
["git", "http-backend"],
|
["git", "http-backend"],
|
||||||
input=body,
|
input=body,
|
||||||
@@ -205,6 +212,10 @@ class GitHttpHandler(BaseHTTPRequestHandler):
|
|||||||
check=False,
|
check=False,
|
||||||
timeout=GIT_GATE_TIMEOUT_SECS,
|
timeout=GIT_GATE_TIMEOUT_SECS,
|
||||||
)
|
)
|
||||||
|
except (OSError, subprocess.SubprocessError) as exc:
|
||||||
|
self.log_message("git http-backend unavailable: %s", exc)
|
||||||
|
self.send_error(503, "git backend unavailable")
|
||||||
|
return
|
||||||
self._write_cgi_response(proc.stdout)
|
self._write_cgi_response(proc.stdout)
|
||||||
|
|
||||||
def _repo_dir(self, sandbox_root: Path, path: str) -> Path | None:
|
def _repo_dir(self, sandbox_root: Path, path: str) -> Path | None:
|
||||||
@@ -273,7 +284,9 @@ def main() -> int:
|
|||||||
"(no single-tenant flat-root fallback)\n"
|
"(no single-tenant flat-root fallback)\n"
|
||||||
)
|
)
|
||||||
return 1
|
return 1
|
||||||
server = ThreadingHTTPServer(("0.0.0.0", port), GitHttpHandler)
|
server = BoundedThreadingHTTPServer(
|
||||||
|
("0.0.0.0", port), GitHttpHandler, max_workers=MAX_REQUEST_WORKERS,
|
||||||
|
)
|
||||||
# Resolve each request's sandbox namespace by source IP against the
|
# Resolve each request's sandbox namespace by source IP against the
|
||||||
# orchestrator control plane.
|
# orchestrator control plane.
|
||||||
server.policy_resolver = PolicyResolver(orch_url) # type: ignore[attr-defined]
|
server.policy_resolver = PolicyResolver(orch_url) # type: ignore[attr-defined]
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
"""Framework-neutral MCP method and tool dispatch."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Callable, Protocol
|
||||||
|
|
||||||
|
from bot_bottle.gateway.egress.schema import load_config, route_to_yaml_dict
|
||||||
|
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||||
|
from bot_bottle.supervisor import types as _sv
|
||||||
|
|
||||||
|
|
||||||
|
class Request(Protocol):
|
||||||
|
@property
|
||||||
|
def method(self) -> str: ...
|
||||||
|
|
||||||
|
@property
|
||||||
|
def params(self) -> dict[str, object]: ...
|
||||||
|
|
||||||
|
|
||||||
|
class MethodNotFoundError(Exception):
|
||||||
|
"""Raised when a JSON-RPC method has no MCP handler."""
|
||||||
|
|
||||||
|
|
||||||
|
class RouteResolutionError(Exception):
|
||||||
|
"""The caller's live route table could not be resolved authoritatively."""
|
||||||
|
|
||||||
|
|
||||||
|
Handler = Callable[[dict[str, object]], object]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Handlers:
|
||||||
|
initialize: Handler
|
||||||
|
tools_list: Handler
|
||||||
|
list_routes: Handler
|
||||||
|
check_proposal: Handler
|
||||||
|
propose: Handler
|
||||||
|
|
||||||
|
|
||||||
|
def dispatch(request: Request, handlers: Handlers) -> object:
|
||||||
|
"""Route one parsed request without depending on the HTTP server."""
|
||||||
|
if request.method == "initialize":
|
||||||
|
return handlers.initialize(request.params)
|
||||||
|
if request.method == "notifications/initialized":
|
||||||
|
return None
|
||||||
|
if request.method == "tools/list":
|
||||||
|
return handlers.tools_list(request.params)
|
||||||
|
if request.method != "tools/call":
|
||||||
|
raise MethodNotFoundError(request.method)
|
||||||
|
|
||||||
|
tool = request.params.get("name")
|
||||||
|
if tool == _sv.TOOL_LIST_EGRESS_ROUTES:
|
||||||
|
return handlers.list_routes(request.params)
|
||||||
|
if tool == _sv.TOOL_CHECK_PROPOSAL:
|
||||||
|
return handlers.check_proposal(request.params)
|
||||||
|
return handlers.propose(request.params)
|
||||||
|
|
||||||
|
|
||||||
|
def resolved_routes_payload(
|
||||||
|
resolver: PolicyResolver,
|
||||||
|
source_ip: str,
|
||||||
|
identity_token: str,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
"""Render an authoritatively resolved route table for the calling bottle."""
|
||||||
|
try:
|
||||||
|
policy, bottle_id, _tokens = resolver.resolve_policy_and_bottle_id(
|
||||||
|
source_ip, identity_token,
|
||||||
|
)
|
||||||
|
except PolicyResolveError as exc:
|
||||||
|
raise RouteResolutionError("orchestrator unavailable") from exc
|
||||||
|
if not bottle_id:
|
||||||
|
raise RouteResolutionError("request source is not attributed to a bottle")
|
||||||
|
try:
|
||||||
|
config = load_config(policy or "")
|
||||||
|
except ValueError as exc:
|
||||||
|
raise RouteResolutionError("resolved policy is invalid") from exc
|
||||||
|
body = json.dumps(
|
||||||
|
{"routes": [route_to_yaml_dict(route) for route in config.routes]},
|
||||||
|
indent=2,
|
||||||
|
)
|
||||||
|
return {"content": [{"type": "text", "text": body}], "isError": False}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"Handlers",
|
||||||
|
"MethodNotFoundError",
|
||||||
|
"RouteResolutionError",
|
||||||
|
"dispatch",
|
||||||
|
"resolved_routes_payload",
|
||||||
|
]
|
||||||
@@ -51,17 +51,27 @@ from __future__ import annotations
|
|||||||
import http.server
|
import http.server
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import socketserver
|
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import typing
|
import typing
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from bot_bottle.constants import IDENTITY_HEADER
|
from bot_bottle.constants import IDENTITY_HEADER
|
||||||
from bot_bottle.gateway.egress.context import resolve_client_context
|
from bot_bottle.gateway.bounded_http import (
|
||||||
from bot_bottle.gateway.egress.schema import load_config, route_to_yaml_dict
|
BodyReadError,
|
||||||
|
BoundedThreadingHTTPServer,
|
||||||
|
read_declared_body,
|
||||||
|
)
|
||||||
|
from bot_bottle.gateway.egress.schema import load_config
|
||||||
from bot_bottle.gateway.egress.types import LOG_OFF
|
from bot_bottle.gateway.egress.types import LOG_OFF
|
||||||
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
from bot_bottle.gateway.policy_resolver import PolicyResolveError, PolicyResolver
|
||||||
|
from bot_bottle.gateway.supervisor.mcp_dispatch import (
|
||||||
|
Handlers as DispatchHandlers,
|
||||||
|
MethodNotFoundError,
|
||||||
|
RouteResolutionError,
|
||||||
|
dispatch,
|
||||||
|
resolved_routes_payload,
|
||||||
|
)
|
||||||
from bot_bottle.supervisor import types as _sv
|
from bot_bottle.supervisor import types as _sv
|
||||||
|
|
||||||
|
|
||||||
@@ -565,6 +575,8 @@ def format_unknown_proposal_text(proposal_id: str) -> str:
|
|||||||
# Max request body the server accepts. 1 MB is well above any realistic
|
# Max request body the server accepts. 1 MB is well above any realistic
|
||||||
# routes.yaml proposal.
|
# routes.yaml proposal.
|
||||||
MAX_BODY_BYTES = 1 * 1024 * 1024
|
MAX_BODY_BYTES = 1 * 1024 * 1024
|
||||||
|
REQUEST_BODY_TIMEOUT_SECONDS = 10.0
|
||||||
|
MAX_REQUEST_WORKERS = 32
|
||||||
|
|
||||||
|
|
||||||
class MCPHandler(http.server.BaseHTTPRequestHandler):
|
class MCPHandler(http.server.BaseHTTPRequestHandler):
|
||||||
@@ -587,19 +599,18 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
self._write_text(405, "use POST for MCP requests\n")
|
self._write_text(405, "use POST for MCP requests\n")
|
||||||
|
|
||||||
def do_POST(self) -> None:
|
def do_POST(self) -> None:
|
||||||
length_header = self.headers.get("Content-Length")
|
|
||||||
if length_header is None:
|
|
||||||
self._write_text(411, "Content-Length required\n")
|
|
||||||
return
|
|
||||||
try:
|
try:
|
||||||
length = int(length_header)
|
body = read_declared_body(
|
||||||
except ValueError:
|
self.rfile,
|
||||||
self._write_text(400, "invalid Content-Length\n")
|
self.connection,
|
||||||
|
self.headers.get("Content-Length"),
|
||||||
|
maximum=MAX_BODY_BYTES,
|
||||||
|
timeout_seconds=REQUEST_BODY_TIMEOUT_SECONDS,
|
||||||
|
require_length=True,
|
||||||
|
)
|
||||||
|
except BodyReadError as exc:
|
||||||
|
self._write_text(exc.status, exc.message + "\n")
|
||||||
return
|
return
|
||||||
if length < 0 or length > MAX_BODY_BYTES:
|
|
||||||
self._write_text(413, "request body too large\n")
|
|
||||||
return
|
|
||||||
body = self.rfile.read(length)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
req = parse_jsonrpc(body)
|
req = parse_jsonrpc(body)
|
||||||
@@ -611,6 +622,11 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
result = self._dispatch(req, config)
|
result = self._dispatch(req, config)
|
||||||
|
except MethodNotFoundError as e:
|
||||||
|
self._write_jsonrpc(
|
||||||
|
jsonrpc_error(req.id, ERR_METHOD_NOT_FOUND, f"method not found: {e}"),
|
||||||
|
)
|
||||||
|
return
|
||||||
except _RpcClientError as e:
|
except _RpcClientError as e:
|
||||||
self._write_jsonrpc(jsonrpc_error(req.id, e.code, e.message))
|
self._write_jsonrpc(jsonrpc_error(req.id, e.code, e.message))
|
||||||
return
|
return
|
||||||
@@ -633,41 +649,42 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
self._write_jsonrpc(jsonrpc_result(req.id, result))
|
self._write_jsonrpc(jsonrpc_result(req.id, result))
|
||||||
|
|
||||||
def _dispatch(self, req: JsonRpcRequest, config: ServerConfig) -> object:
|
def _dispatch(self, req: JsonRpcRequest, config: ServerConfig) -> object:
|
||||||
method = req.method
|
def check(params: dict[str, object]) -> object:
|
||||||
if method == "initialize":
|
|
||||||
return handle_initialize(req.params)
|
|
||||||
if method == "notifications/initialized":
|
|
||||||
return None # ack-only
|
|
||||||
if method == "tools/list":
|
|
||||||
return handle_tools_list(req.params)
|
|
||||||
if method == "tools/call":
|
|
||||||
# `list-egress-routes` is read-only introspection. The shared gateway
|
|
||||||
# has no static route table (routes are resolved per request by
|
|
||||||
# source IP), so answer it from the calling bottle's resolved policy.
|
|
||||||
# Otherwise the agent sees an empty allowlist and composes an egress
|
|
||||||
# proposal that *replaces* the live routes instead of extending them
|
|
||||||
# — silently dropping base routes like api.anthropic.com on approval.
|
|
||||||
if req.params.get("name") == _sv.TOOL_LIST_EGRESS_ROUTES:
|
|
||||||
return self._resolved_routes_payload()
|
|
||||||
resolver = self._resolver_or_fail()
|
|
||||||
source_ip = self.client_address[0]
|
|
||||||
token = self._identity_token()
|
|
||||||
# `check-proposal` is a non-blocking read of the calling bottle's
|
|
||||||
# own queue — attributed by (source_ip, identity_token) like a
|
|
||||||
# proposal, but it never queues or blocks.
|
|
||||||
if req.params.get("name") == _sv.TOOL_CHECK_PROPOSAL:
|
|
||||||
return handle_check_proposal(
|
return handle_check_proposal(
|
||||||
req.params, resolver=resolver,
|
params,
|
||||||
source_ip=source_ip, identity_token=token,
|
resolver=self._resolver_or_fail(),
|
||||||
|
source_ip=self.client_address[0],
|
||||||
|
identity_token=self._identity_token(),
|
||||||
)
|
)
|
||||||
# The control plane attributes the proposal to the source-IP + token
|
|
||||||
# resolved bottle, so the one shared queue holds each bottle's
|
def propose(params: dict[str, object]) -> object:
|
||||||
# proposal under its own id — no slug is asserted by this daemon.
|
|
||||||
return handle_tools_call(
|
return handle_tools_call(
|
||||||
req.params, config, resolver=resolver,
|
params,
|
||||||
source_ip=source_ip, identity_token=token,
|
config,
|
||||||
|
resolver=self._resolver_or_fail(),
|
||||||
|
source_ip=self.client_address[0],
|
||||||
|
identity_token=self._identity_token(),
|
||||||
)
|
)
|
||||||
raise _RpcClientError(ERR_METHOD_NOT_FOUND, f"method not found: {method}")
|
|
||||||
|
def list_routes(_params: dict[str, object]) -> object:
|
||||||
|
try:
|
||||||
|
return resolved_routes_payload(
|
||||||
|
self._resolver_or_fail(),
|
||||||
|
self.client_address[0],
|
||||||
|
self._identity_token(),
|
||||||
|
)
|
||||||
|
except RouteResolutionError as exc:
|
||||||
|
raise _RpcInternalError(
|
||||||
|
f"could not resolve live egress routes: {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
return dispatch(req, DispatchHandlers(
|
||||||
|
initialize=handle_initialize,
|
||||||
|
tools_list=handle_tools_list,
|
||||||
|
list_routes=list_routes,
|
||||||
|
check_proposal=check,
|
||||||
|
propose=propose,
|
||||||
|
))
|
||||||
|
|
||||||
def _identity_token(self) -> str:
|
def _identity_token(self) -> str:
|
||||||
"""The agent's per-bottle identity token from the request header (the
|
"""The agent's per-bottle identity token from the request header (the
|
||||||
@@ -686,20 +703,6 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
raise _RpcInternalError("supervise server has no policy resolver")
|
raise _RpcInternalError("supervise server has no policy resolver")
|
||||||
return resolver
|
return resolver
|
||||||
|
|
||||||
def _resolved_routes_payload(self) -> dict[str, object]:
|
|
||||||
"""The calling bottle's live egress routes as the `list-egress-routes`
|
|
||||||
JSON payload, resolved by (source_ip, identity token). Fail-closed: an
|
|
||||||
unattributed source or an unreachable orchestrator yields an empty route
|
|
||||||
list (never another bottle's), courtesy of `resolve_client_context`."""
|
|
||||||
resolver = self._resolver_or_fail()
|
|
||||||
conf, _slug, _tokens = resolve_client_context(
|
|
||||||
resolver, self.client_address[0], self._identity_token(),
|
|
||||||
)
|
|
||||||
body = json.dumps(
|
|
||||||
{"routes": [route_to_yaml_dict(r) for r in conf.routes]}, indent=2,
|
|
||||||
)
|
|
||||||
return {"content": [{"type": "text", "text": body}], "isError": False}
|
|
||||||
|
|
||||||
def _write_jsonrpc(self, body: bytes) -> None:
|
def _write_jsonrpc(self, body: bytes) -> None:
|
||||||
self.send_response(200)
|
self.send_response(200)
|
||||||
self.send_header("Content-Type", "application/json")
|
self.send_header("Content-Type", "application/json")
|
||||||
@@ -719,7 +722,7 @@ class MCPHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
self.wfile.write(encoded)
|
self.wfile.write(encoded)
|
||||||
|
|
||||||
|
|
||||||
class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
class MCPServer(BoundedThreadingHTTPServer):
|
||||||
allow_reuse_address = True
|
allow_reuse_address = True
|
||||||
daemon_threads = True
|
daemon_threads = True
|
||||||
config: ServerConfig = ServerConfig()
|
config: ServerConfig = ServerConfig()
|
||||||
@@ -728,6 +731,9 @@ class MCPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
|||||||
# closed per request (see `_resolver_or_fail`).
|
# closed per request (see `_resolver_or_fail`).
|
||||||
policy_resolver: "PolicyResolver | None" = None
|
policy_resolver: "PolicyResolver | None" = None
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs): # type: ignore[no-untyped-def]
|
||||||
|
super().__init__(*args, max_workers=MAX_REQUEST_WORKERS, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
# --- Entry point -----------------------------------------------------------
|
# --- Entry point -----------------------------------------------------------
|
||||||
|
|
||||||
|
|||||||
@@ -366,7 +366,7 @@ class OrchestratorCore:
|
|||||||
value with *env_var_secret*, and restores ``_tokens[bottle_id]``.
|
value with *env_var_secret*, and restores ``_tokens[bottle_id]``.
|
||||||
Returns True on success, False when no stored secrets exist for this
|
Returns True on success, False when no stored secrets exist for this
|
||||||
bottle or decryption fails (wrong key / corrupt data)."""
|
bottle or decryption fails (wrong key / corrupt data)."""
|
||||||
from .store.secret_store import decrypt_value, encrypt_value, is_legacy_blob
|
from .store.secret_store import decrypt_value
|
||||||
encrypted = self.registry.get_agent_secrets(bottle_id)
|
encrypted = self.registry.get_agent_secrets(bottle_id)
|
||||||
if not encrypted:
|
if not encrypted:
|
||||||
return False
|
return False
|
||||||
@@ -377,12 +377,6 @@ class OrchestratorCore:
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
return False
|
return False
|
||||||
self._tokens[bottle_id] = decrypted
|
self._tokens[bottle_id] = decrypted
|
||||||
if any(is_legacy_blob(value) for value in encrypted.values()):
|
|
||||||
migrated = {
|
|
||||||
key: encrypt_value(env_var_secret, value)
|
|
||||||
for key, value in decrypted.items()
|
|
||||||
}
|
|
||||||
self.registry.store_agent_secrets(bottle_id, migrated)
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
# --- consolidated gateway ----------------------------------------------
|
# --- consolidated gateway ----------------------------------------------
|
||||||
|
|||||||
@@ -129,6 +129,10 @@ _MIGRATIONS = TableMigrations(
|
|||||||
# v5 — index for fast per-bottle lookups and bulk DELETE on teardown.
|
# v5 — index for fast per-bottle lookups and bulk DELETE on teardown.
|
||||||
"CREATE INDEX IF NOT EXISTS idx_bottled_agent_secrets_id "
|
"CREATE INDEX IF NOT EXISTS idx_bottled_agent_secrets_id "
|
||||||
"ON bottled_agent_secrets (bottled_agent_id, type)",
|
"ON bottled_agent_secrets (bottled_agent_id, type)",
|
||||||
|
# v6 — unauthenticated legacy ciphertext must never be selected by
|
||||||
|
# attacker-controlled blob contents. Existing local agents are
|
||||||
|
# intentionally reprovisioned instead of retaining downgrade support.
|
||||||
|
"DELETE FROM bottled_agent_secrets",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -18,9 +18,9 @@ value is encrypted independently. New output blobs are:
|
|||||||
|
|
||||||
``version || nonce (16 bytes) || ciphertext || tag (32 bytes)``
|
``version || nonce (16 bytes) || ciphertext || tag (32 bytes)``
|
||||||
|
|
||||||
encoded as URL-safe base64 (no padding). The version marker lets the reader
|
encoded as URL-safe base64 (no padding). Unversioned legacy ciphertext is
|
||||||
accept legacy ``nonce || ciphertext`` rows long enough to rewrite them in the
|
rejected; the registry migration clears those rows rather than allowing blob
|
||||||
authenticated format after a successful reprovision.
|
contents to select an unauthenticated decoder.
|
||||||
|
|
||||||
keystream_block_i = HMAC-SHA256(key, nonce || i.to_bytes(4, "big"))
|
keystream_block_i = HMAC-SHA256(key, nonce || i.to_bytes(4, "big"))
|
||||||
ciphertext_i = plaintext_i XOR keystream_block_i[:len(plaintext_i)]
|
ciphertext_i = plaintext_i XOR keystream_block_i[:len(plaintext_i)]
|
||||||
@@ -84,44 +84,18 @@ def encrypt_value(secret_b64: str, plaintext: str) -> str:
|
|||||||
return base64.urlsafe_b64encode(authenticated + tag).rstrip(b"=").decode()
|
return base64.urlsafe_b64encode(authenticated + tag).rstrip(b"=").decode()
|
||||||
|
|
||||||
|
|
||||||
def is_legacy_blob(blob_b64: str) -> bool:
|
|
||||||
"""Whether *blob_b64* uses the pre-authentication storage format."""
|
|
||||||
try:
|
|
||||||
return not _b64dec(blob_b64).startswith(_VERSION)
|
|
||||||
except (ValueError, TypeError):
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def _decrypt_legacy(key: bytes, blob: bytes) -> str:
|
|
||||||
"""Read the original ``nonce || ciphertext`` format for migration only."""
|
|
||||||
if len(blob) < _NONCE_BYTES:
|
|
||||||
raise ValueError("ciphertext blob too short")
|
|
||||||
nonce, ciphertext = blob[:_NONCE_BYTES], blob[_NONCE_BYTES:]
|
|
||||||
pt = bytearray()
|
|
||||||
# The legacy format used the byte offset as the PRF counter.
|
|
||||||
for i in range(0, len(ciphertext), _BLOCK):
|
|
||||||
chunk = ciphertext[i : i + _BLOCK]
|
|
||||||
ks = _keystream(key, nonce, i)[: len(chunk)]
|
|
||||||
pt.extend(c ^ k for c, k in zip(chunk, ks))
|
|
||||||
try:
|
|
||||||
return bytes(pt).decode()
|
|
||||||
except UnicodeDecodeError as exc:
|
|
||||||
raise ValueError(f"decryption produced non-UTF-8 output: {exc}") from exc
|
|
||||||
|
|
||||||
|
|
||||||
def decrypt_value(secret_b64: str, blob_b64: str) -> str:
|
def decrypt_value(secret_b64: str, blob_b64: str) -> str:
|
||||||
"""Decrypt a blob produced by :func:`encrypt_value`.
|
"""Decrypt a blob produced by :func:`encrypt_value`.
|
||||||
|
|
||||||
Returns the original plaintext string. Raises ``ValueError`` for malformed
|
Returns the original plaintext string. Raises ``ValueError`` for malformed
|
||||||
input, authentication failure, or a key mismatch. Legacy unauthenticated
|
input, authentication failure, or a key mismatch."""
|
||||||
rows remain readable so callers can migrate them immediately."""
|
|
||||||
key = _b64dec(secret_b64)
|
key = _b64dec(secret_b64)
|
||||||
try:
|
try:
|
||||||
blob = _b64dec(blob_b64)
|
blob = _b64dec(blob_b64)
|
||||||
except (ValueError, TypeError) as exc:
|
except (ValueError, TypeError) as exc:
|
||||||
raise ValueError(f"invalid ciphertext blob: {exc}") from exc
|
raise ValueError(f"invalid ciphertext blob: {exc}") from exc
|
||||||
if not blob.startswith(_VERSION):
|
if not blob.startswith(_VERSION):
|
||||||
return _decrypt_legacy(key, blob)
|
raise ValueError("unsupported ciphertext format")
|
||||||
minimum = len(_VERSION) + _NONCE_BYTES + _TAG_BYTES
|
minimum = len(_VERSION) + _NONCE_BYTES + _TAG_BYTES
|
||||||
if len(blob) < minimum:
|
if len(blob) < minimum:
|
||||||
raise ValueError("ciphertext blob too short")
|
raise ValueError("ciphertext blob too short")
|
||||||
@@ -152,5 +126,4 @@ __all__ = [
|
|||||||
"new_env_var_secret",
|
"new_env_var_secret",
|
||||||
"encrypt_value",
|
"encrypt_value",
|
||||||
"decrypt_value",
|
"decrypt_value",
|
||||||
"is_legacy_blob",
|
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from bot_bottle.cli.commands import cleanup as cmd
|
|||||||
def _make_backend(empty: bool = True):
|
def _make_backend(empty: bool = True):
|
||||||
backend = MagicMock()
|
backend = MagicMock()
|
||||||
plan = MagicMock(empty=empty)
|
plan = MagicMock(empty=empty)
|
||||||
|
plan.intersect.return_value = plan
|
||||||
backend.prepare_cleanup.return_value = plan
|
backend.prepare_cleanup.return_value = plan
|
||||||
backend.cleanup = MagicMock()
|
backend.cleanup = MagicMock()
|
||||||
return backend, plan
|
return backend, plan
|
||||||
@@ -41,8 +42,8 @@ class TestCmdCleanup(unittest.TestCase):
|
|||||||
):
|
):
|
||||||
self.assertEqual(0, cmd.cmd_cleanup([]))
|
self.assertEqual(0, cmd.cmd_cleanup([]))
|
||||||
|
|
||||||
docker.prepare_cleanup.assert_called_once()
|
self.assertEqual(2, docker.prepare_cleanup.call_count)
|
||||||
fc.prepare_cleanup.assert_called_once()
|
self.assertEqual(2, fc.prepare_cleanup.call_count)
|
||||||
docker.cleanup.assert_called_once_with(docker_plan)
|
docker.cleanup.assert_called_once_with(docker_plan)
|
||||||
fc.cleanup.assert_called_once_with(fc_plan)
|
fc.cleanup.assert_called_once_with(fc_plan)
|
||||||
|
|
||||||
@@ -68,7 +69,7 @@ class TestCmdCleanup(unittest.TestCase):
|
|||||||
):
|
):
|
||||||
self.assertEqual(0, cmd.cmd_cleanup([]))
|
self.assertEqual(0, cmd.cmd_cleanup([]))
|
||||||
|
|
||||||
docker.prepare_cleanup.assert_called_once()
|
self.assertEqual(2, docker.prepare_cleanup.call_count)
|
||||||
docker.cleanup.assert_called_once_with(docker_plan)
|
docker.cleanup.assert_called_once_with(docker_plan)
|
||||||
macos.prepare_cleanup.assert_not_called()
|
macos.prepare_cleanup.assert_not_called()
|
||||||
|
|
||||||
@@ -135,6 +136,28 @@ class TestCmdCleanup(unittest.TestCase):
|
|||||||
docker.cleanup.assert_called_once_with(docker_plan)
|
docker.cleanup.assert_called_once_with(docker_plan)
|
||||||
fc.cleanup.assert_not_called()
|
fc.cleanup.assert_not_called()
|
||||||
|
|
||||||
|
def test_executes_only_displayed_resources_still_current(self):
|
||||||
|
backend = MagicMock()
|
||||||
|
preview = MagicMock(empty=False)
|
||||||
|
refreshed = MagicMock(empty=False)
|
||||||
|
approved = MagicMock(empty=False)
|
||||||
|
preview.intersect.return_value = approved
|
||||||
|
backend.prepare_cleanup.side_effect = [preview, refreshed]
|
||||||
|
|
||||||
|
with patch.object(
|
||||||
|
cmd, "known_backend_names", return_value=("firecracker",),
|
||||||
|
), patch.object(
|
||||||
|
cmd, "get_bottle_backend", return_value=backend,
|
||||||
|
), patch.object(
|
||||||
|
cmd, "has_backend", return_value=True,
|
||||||
|
), patch.object(
|
||||||
|
cmd, "_prompt_yes", return_value=True,
|
||||||
|
):
|
||||||
|
self.assertEqual(0, cmd.cmd_cleanup([]))
|
||||||
|
|
||||||
|
preview.intersect.assert_called_once_with(refreshed)
|
||||||
|
backend.cleanup.assert_called_once_with(approved)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -14,9 +14,12 @@ from __future__ import annotations
|
|||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
from tests.unit import use_bottle_root
|
from tests.unit import use_bottle_root
|
||||||
from bot_bottle import bottle_state
|
from bot_bottle import bottle_state
|
||||||
|
from bot_bottle.backend import EnumerationError
|
||||||
|
from bot_bottle.backend.docker import cleanup
|
||||||
from bot_bottle.backend.docker.cleanup import _list_orphan_state_dirs
|
from bot_bottle.backend.docker.cleanup import _list_orphan_state_dirs
|
||||||
|
|
||||||
|
|
||||||
@@ -116,5 +119,37 @@ class TestOrphanStateDirs(_FakeHomeMixin, unittest.TestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestAuthoritativeDiscovery(unittest.TestCase):
|
||||||
|
def test_prepare_requires_authoritative_compose_projects(self):
|
||||||
|
with patch.object(cleanup.docker_mod, "require_docker"), \
|
||||||
|
patch.object(
|
||||||
|
cleanup, "list_compose_projects",
|
||||||
|
side_effect=EnumerationError("compose unavailable"),
|
||||||
|
) as projects, self.assertRaisesRegex(
|
||||||
|
EnumerationError, "compose unavailable",
|
||||||
|
):
|
||||||
|
cleanup.prepare_cleanup()
|
||||||
|
projects.assert_called_once_with(
|
||||||
|
warn_on_error=False,
|
||||||
|
raise_on_error=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_container_query_failure_raises(self):
|
||||||
|
failed = cleanup.subprocess.CompletedProcess(
|
||||||
|
[], 1, stdout="", stderr="daemon unavailable",
|
||||||
|
)
|
||||||
|
with patch.object(cleanup.subprocess, "run", return_value=failed), \
|
||||||
|
self.assertRaisesRegex(EnumerationError, "daemon unavailable"):
|
||||||
|
cleanup._list_prefixed_containers()
|
||||||
|
|
||||||
|
def test_network_query_failure_raises(self):
|
||||||
|
failed = cleanup.subprocess.CompletedProcess(
|
||||||
|
[], 1, stdout="", stderr="daemon unavailable",
|
||||||
|
)
|
||||||
|
with patch.object(cleanup.subprocess, "run", return_value=failed), \
|
||||||
|
self.assertRaisesRegex(EnumerationError, "daemon unavailable"):
|
||||||
|
cleanup._list_prefixed_networks()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
"""Unit tests for framework-neutral outbound DLP request stages."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from bot_bottle.gateway.egress.outbound_pipeline import (
|
||||||
|
MutableHeaders,
|
||||||
|
redact_request,
|
||||||
|
scan_request,
|
||||||
|
)
|
||||||
|
from bot_bottle.gateway.egress.types import Route
|
||||||
|
|
||||||
|
|
||||||
|
class _Headers(dict[str, str]):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class _Request:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
host: str = "api.example.com",
|
||||||
|
path: str = "/v1/messages",
|
||||||
|
headers: dict[str, str] | None = None,
|
||||||
|
body: str = "",
|
||||||
|
) -> None:
|
||||||
|
self.pretty_host = host
|
||||||
|
self.path = path
|
||||||
|
self.headers: MutableHeaders = _Headers(headers or {})
|
||||||
|
self.text = body
|
||||||
|
|
||||||
|
def get_text(self, strict: bool = False) -> str | None:
|
||||||
|
del strict
|
||||||
|
return self.text
|
||||||
|
|
||||||
|
|
||||||
|
class TestOutboundScan(unittest.TestCase):
|
||||||
|
def test_detects_secret_in_body(self) -> None:
|
||||||
|
request = _Request(body="token=sk-" + "a" * 48)
|
||||||
|
|
||||||
|
result = scan_request(request, Route(host="api.example.com"), {})
|
||||||
|
|
||||||
|
self.assertIsNotNone(result)
|
||||||
|
self.assertEqual("block", result.severity if result else None)
|
||||||
|
|
||||||
|
def test_safe_token_is_ignored(self) -> None:
|
||||||
|
token = "sk-" + "a" * 48
|
||||||
|
request = _Request(body=f"token={token}")
|
||||||
|
|
||||||
|
result = scan_request(
|
||||||
|
request,
|
||||||
|
Route(host="api.example.com"),
|
||||||
|
{},
|
||||||
|
safe_tokens={token},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIsNone(result)
|
||||||
|
|
||||||
|
|
||||||
|
class TestOutboundRedaction(unittest.TestCase):
|
||||||
|
def test_redacts_body_header_and_path_but_preserves_host(self) -> None:
|
||||||
|
token = "sk-" + "a" * 48
|
||||||
|
request = _Request(
|
||||||
|
path=f"/v1/messages?key={token}",
|
||||||
|
headers={"Host": "api.example.com", "X-Token": token + "\r\nInjected: yes"},
|
||||||
|
body=f"token={token}",
|
||||||
|
)
|
||||||
|
|
||||||
|
clean = redact_request(request, Route(host="api.example.com"), {})
|
||||||
|
|
||||||
|
self.assertTrue(clean)
|
||||||
|
self.assertNotIn(token, request.path)
|
||||||
|
self.assertNotIn(token, request.headers["X-Token"])
|
||||||
|
self.assertNotIn("\r", request.headers["X-Token"])
|
||||||
|
self.assertNotIn(token, request.text)
|
||||||
|
self.assertEqual("api.example.com", request.headers["Host"])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
"""Unit tests for framework-neutral egress request policy stages."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from bot_bottle.gateway.egress.request_pipeline import (
|
||||||
|
GIT_PUSH_BLOCK_REASON,
|
||||||
|
evaluate_route_policy,
|
||||||
|
git_block_reason,
|
||||||
|
)
|
||||||
|
from bot_bottle.gateway.egress.types import Config, LOG_FULL, Route
|
||||||
|
|
||||||
|
|
||||||
|
class TestGitPolicy(unittest.TestCase):
|
||||||
|
def test_push_is_always_blocked(self) -> None:
|
||||||
|
reason = git_block_reason(
|
||||||
|
(), "git.example.com", "/repo.git/git-receive-pack", "",
|
||||||
|
)
|
||||||
|
self.assertEqual(GIT_PUSH_BLOCK_REASON, reason)
|
||||||
|
|
||||||
|
def test_fetch_requires_route_opt_in(self) -> None:
|
||||||
|
path = "/repo.git/git-upload-pack"
|
||||||
|
blocked = git_block_reason((), "git.example.com", path, "")
|
||||||
|
allowed = git_block_reason(
|
||||||
|
(Route(host="git.example.com", git_fetch=True),),
|
||||||
|
"git.example.com",
|
||||||
|
path,
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
self.assertTrue(blocked)
|
||||||
|
self.assertEqual("", allowed)
|
||||||
|
|
||||||
|
def test_non_git_request_is_not_decided_here(self) -> None:
|
||||||
|
self.assertEqual(
|
||||||
|
"",
|
||||||
|
git_block_reason((), "api.example.com", "/v1/messages", ""),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRoutePolicy(unittest.TestCase):
|
||||||
|
def test_strips_agent_auth_and_injects_gateway_auth(self) -> None:
|
||||||
|
route = Route(
|
||||||
|
host="api.example.com",
|
||||||
|
auth_scheme="Bearer",
|
||||||
|
token_env="API_TOKEN",
|
||||||
|
)
|
||||||
|
result = evaluate_route_policy(
|
||||||
|
Config(routes=(route,)),
|
||||||
|
route,
|
||||||
|
host="api.example.com",
|
||||||
|
request_path="/v1/messages",
|
||||||
|
method="POST",
|
||||||
|
headers={"Authorization": "agent-secret"},
|
||||||
|
env={"API_TOKEN": "gateway-secret"},
|
||||||
|
)
|
||||||
|
self.assertTrue(result.strip_authorization)
|
||||||
|
self.assertEqual("Bearer gateway-secret", result.inject_authorization)
|
||||||
|
self.assertFalse(result.block_reason)
|
||||||
|
|
||||||
|
def test_preserved_auth_participates_in_matching(self) -> None:
|
||||||
|
route = Route(host="registry.example.com", preserve_auth=True)
|
||||||
|
result = evaluate_route_policy(
|
||||||
|
Config(routes=(route,), log=LOG_FULL),
|
||||||
|
route,
|
||||||
|
host="registry.example.com",
|
||||||
|
request_path="/v2/",
|
||||||
|
method="GET",
|
||||||
|
headers={"Authorization": "Bearer agent-token"},
|
||||||
|
env={},
|
||||||
|
)
|
||||||
|
self.assertFalse(result.strip_authorization)
|
||||||
|
self.assertTrue(result.log_request)
|
||||||
|
|
||||||
|
def test_missing_route_fails_closed(self) -> None:
|
||||||
|
result = evaluate_route_policy(
|
||||||
|
Config(routes=(), deny_reason="not allowed"),
|
||||||
|
None,
|
||||||
|
host="blocked.example.com",
|
||||||
|
request_path="/",
|
||||||
|
method="GET",
|
||||||
|
headers={},
|
||||||
|
env={},
|
||||||
|
)
|
||||||
|
self.assertEqual("not allowed", result.block_reason)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -32,15 +32,19 @@ class TestProcessScan(unittest.TestCase):
|
|||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
Path("/cache/run/dev-a"),
|
Path("/cache/run/dev-a"),
|
||||||
fc_cleanup._run_dir_of(
|
fc_cleanup._run_dir_of(
|
||||||
f"firecracker --config-file {run_root}/dev-a/config.json", run_root
|
("firecracker", "--config-file",
|
||||||
|
f"{run_root}/dev-a/config.json"), run_root
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
# infra/builder VMs elsewhere, or nested paths, are not ours.
|
# infra/builder VMs elsewhere, or nested paths, are not ours.
|
||||||
self.assertIsNone(
|
self.assertIsNone(
|
||||||
fc_cleanup._run_dir_of("firecracker --config-file /elsewhere/config.json", run_root)
|
fc_cleanup._run_dir_of(
|
||||||
|
("firecracker", "--config-file", "/elsewhere/config.json"),
|
||||||
|
run_root,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
self.assertIsNone(
|
self.assertIsNone(
|
||||||
fc_cleanup._run_dir_of("firecracker --no-config", run_root)
|
fc_cleanup._run_dir_of(("firecracker", "--no-config"), run_root)
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_scan_splits_live_dirs_from_orphan_pids(self):
|
def test_scan_splits_live_dirs_from_orphan_pids(self):
|
||||||
@@ -48,17 +52,40 @@ class TestProcessScan(unittest.TestCase):
|
|||||||
run_root = Path(tmp)
|
run_root = Path(tmp)
|
||||||
(run_root / "live-a").mkdir() # dir present -> live VM, protected
|
(run_root / "live-a").mkdir() # dir present -> live VM, protected
|
||||||
# "gone-b" dir intentionally absent -> lingering VMM, orphan pid
|
# "gone-b" dir intentionally absent -> lingering VMM, orphan pid
|
||||||
out = (
|
args = {
|
||||||
f"111 firecracker --config-file {run_root}/live-a/config.json\n"
|
111: ("firecracker", "--config-file",
|
||||||
f"222 firecracker --config-file {run_root}/gone-b/config.json\n"
|
f"{run_root}/live-a/config.json"),
|
||||||
"333 firecracker --config-file /elsewhere/config.json\n"
|
222: ("firecracker", "--config-file",
|
||||||
"notanint firecracker --config-file x\n"
|
f"{run_root}/gone-b/config.json"),
|
||||||
)
|
333: ("firecracker", "--config-file", "/elsewhere/config.json"),
|
||||||
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
|
}
|
||||||
|
with patch.object(
|
||||||
|
fc_cleanup.subprocess, "run",
|
||||||
|
return_value=_proc("111\n222\n333\nnotanint\n"),
|
||||||
|
), patch.object(
|
||||||
|
fc_cleanup, "_process_args", side_effect=args.get,
|
||||||
|
):
|
||||||
live, orphan_pids = fc_cleanup._scan_processes(run_root)
|
live, orphan_pids = fc_cleanup._scan_processes(run_root)
|
||||||
self.assertEqual({str(run_root / "live-a")}, live)
|
self.assertEqual({str(run_root / "live-a")}, live)
|
||||||
self.assertEqual([222], orphan_pids)
|
self.assertEqual([222], orphan_pids)
|
||||||
|
|
||||||
|
def test_scan_preserves_spaces_in_config_path(self):
|
||||||
|
with tempfile.TemporaryDirectory(prefix="fc cache ") as tmp:
|
||||||
|
run_root = Path(tmp)
|
||||||
|
live = run_root / "live bottle"
|
||||||
|
live.mkdir()
|
||||||
|
with patch.object(
|
||||||
|
fc_cleanup.subprocess, "run", return_value=_proc("111\n"),
|
||||||
|
), patch.object(
|
||||||
|
fc_cleanup, "_process_args",
|
||||||
|
return_value=("firecracker", "--config-file",
|
||||||
|
str(live / "config.json")),
|
||||||
|
):
|
||||||
|
self.assertEqual(
|
||||||
|
({str(live)}, []),
|
||||||
|
fc_cleanup._scan_processes(run_root),
|
||||||
|
)
|
||||||
|
|
||||||
def test_scan_empty_when_pgrep_finds_no_processes(self):
|
def test_scan_empty_when_pgrep_finds_no_processes(self):
|
||||||
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(returncode=1)):
|
with patch.object(fc_cleanup.subprocess, "run", return_value=_proc(returncode=1)):
|
||||||
self.assertEqual((set(), []), fc_cleanup._scan_processes(Path("/x")))
|
self.assertEqual((set(), []), fc_cleanup._scan_processes(Path("/x")))
|
||||||
@@ -117,9 +144,14 @@ class TestProcessScan(unittest.TestCase):
|
|||||||
run_root = Path(tmp)
|
run_root = Path(tmp)
|
||||||
(run_root / "live-a").mkdir()
|
(run_root / "live-a").mkdir()
|
||||||
(run_root / "dead-b").mkdir()
|
(run_root / "dead-b").mkdir()
|
||||||
out = f"111 firecracker --config-file {run_root}/live-a/config.json\n"
|
|
||||||
with patch.object(fc_cleanup, "_run_root", return_value=run_root), \
|
with patch.object(fc_cleanup, "_run_root", return_value=run_root), \
|
||||||
patch.object(fc_cleanup.subprocess, "run", return_value=_proc(out)):
|
patch.object(fc_cleanup.subprocess, "run",
|
||||||
|
return_value=_proc("111\n")), \
|
||||||
|
patch.object(
|
||||||
|
fc_cleanup, "_process_args",
|
||||||
|
return_value=("firecracker", "--config-file",
|
||||||
|
str(run_root / "live-a/config.json")),
|
||||||
|
):
|
||||||
plan = fc_cleanup.prepare_cleanup()
|
plan = fc_cleanup.prepare_cleanup()
|
||||||
self.assertEqual((), plan.vm_pids)
|
self.assertEqual((), plan.vm_pids)
|
||||||
self.assertEqual((str(run_root / "dead-b"),), plan.run_dirs)
|
self.assertEqual((str(run_root / "dead-b"),), plan.run_dirs)
|
||||||
@@ -132,18 +164,46 @@ class TestCleanupRemoval(unittest.TestCase):
|
|||||||
vm_pids=(101,),
|
vm_pids=(101,),
|
||||||
run_dirs=("/run/dev-x",),
|
run_dirs=("/run/dev-x",),
|
||||||
)
|
)
|
||||||
with patch.object(fc_cleanup.os, "kill") as kill, \
|
with patch.object(fc_cleanup, "prepare_cleanup", return_value=plan), \
|
||||||
patch.object(fc_cleanup.shutil, "rmtree") as rmtree, \
|
patch.object(fc_cleanup, "_run_root", return_value=Path("/run")), \
|
||||||
|
patch.object(fc_cleanup, "_terminate_orphan") as terminate, \
|
||||||
|
patch("bot_bottle.backend.cleanup_control.shutil.rmtree") as rmtree, \
|
||||||
patch.object(fc_cleanup, "info"):
|
patch.object(fc_cleanup, "info"):
|
||||||
fc_cleanup.cleanup(plan)
|
fc_cleanup.cleanup(plan)
|
||||||
kill.assert_called_once()
|
terminate.assert_called_once_with(101, Path("/run"))
|
||||||
rmtree.assert_called_once_with("/run/dev-x", ignore_errors=True)
|
rmtree.assert_called_once_with(Path("/run/dev-x"))
|
||||||
|
|
||||||
def test_cleanup_tolerates_dead_pid(self):
|
def test_cleanup_skips_resources_no_longer_in_refreshed_plan(self):
|
||||||
plan = FirecrackerBottleCleanupPlan(vm_pids=(999,))
|
preview = FirecrackerBottleCleanupPlan(
|
||||||
with patch.object(fc_cleanup.os, "kill", side_effect=ProcessLookupError), \
|
vm_pids=(999,), run_dirs=("/run/reused",),
|
||||||
patch.object(fc_cleanup, "info"):
|
)
|
||||||
fc_cleanup.cleanup(plan) # must not raise
|
with patch.object(
|
||||||
|
fc_cleanup, "prepare_cleanup",
|
||||||
|
return_value=FirecrackerBottleCleanupPlan(),
|
||||||
|
), patch.object(fc_cleanup, "_terminate_orphan") as terminate, \
|
||||||
|
patch("bot_bottle.backend.cleanup_control.shutil.rmtree") as rmtree:
|
||||||
|
fc_cleanup.cleanup(preview)
|
||||||
|
terminate.assert_not_called()
|
||||||
|
rmtree.assert_not_called()
|
||||||
|
|
||||||
|
def test_pidfd_prevents_pid_reuse_from_signalling_unrelated_process(self):
|
||||||
|
with patch.object(fc_cleanup.os, "pidfd_open", return_value=7), \
|
||||||
|
patch.object(
|
||||||
|
fc_cleanup.Path, "read_bytes",
|
||||||
|
return_value=b"/usr/bin/python\0worker.py\0",
|
||||||
|
), patch.object(fc_cleanup.signal, "pidfd_send_signal") as send, \
|
||||||
|
patch.object(fc_cleanup.os, "close"):
|
||||||
|
fc_cleanup._terminate_orphan(101, Path("/run"))
|
||||||
|
send.assert_not_called()
|
||||||
|
|
||||||
|
def test_pidfd_signals_revalidated_orphan(self):
|
||||||
|
command = b"firecracker\0--config-file\0/run/gone/config.json\0"
|
||||||
|
with patch.object(fc_cleanup.os, "pidfd_open", return_value=7), \
|
||||||
|
patch.object(fc_cleanup.Path, "read_bytes", return_value=command), \
|
||||||
|
patch.object(fc_cleanup.signal, "pidfd_send_signal") as send, \
|
||||||
|
patch.object(fc_cleanup.os, "close"), patch.object(fc_cleanup, "info"):
|
||||||
|
fc_cleanup._terminate_orphan(101, Path("/run"))
|
||||||
|
send.assert_called_once_with(7, fc_cleanup.signal.SIGTERM)
|
||||||
|
|
||||||
|
|
||||||
class TestCleanupPlan(unittest.TestCase):
|
class TestCleanupPlan(unittest.TestCase):
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
"""Unit tests for shared gateway stdlib HTTP resource boundaries."""
|
||||||
|
# pylint: disable=protected-access
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import io
|
||||||
|
import socket
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from bot_bottle.gateway.bounded_http import (
|
||||||
|
BodyReadError,
|
||||||
|
BoundedThreadingHTTPServer,
|
||||||
|
read_declared_body,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _Handler:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class _TimeoutStream:
|
||||||
|
def read(self, _size: int = -1, /) -> bytes:
|
||||||
|
raise TimeoutError
|
||||||
|
|
||||||
|
|
||||||
|
class TestDeclaredBody(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.left, self.right = socket.socketpair()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.left.close()
|
||||||
|
self.right.close()
|
||||||
|
|
||||||
|
def test_rejects_incomplete_body(self) -> None:
|
||||||
|
with self.assertRaisesRegex(BodyReadError, "incomplete"):
|
||||||
|
read_declared_body(
|
||||||
|
io.BytesIO(b"short"),
|
||||||
|
self.left,
|
||||||
|
"10",
|
||||||
|
maximum=100,
|
||||||
|
timeout_seconds=1,
|
||||||
|
require_length=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_maps_read_timeout(self) -> None:
|
||||||
|
with self.assertRaises(BodyReadError) as caught:
|
||||||
|
read_declared_body(
|
||||||
|
_TimeoutStream(),
|
||||||
|
self.left,
|
||||||
|
"1",
|
||||||
|
maximum=100,
|
||||||
|
timeout_seconds=1,
|
||||||
|
require_length=True,
|
||||||
|
)
|
||||||
|
self.assertEqual(408, caught.exception.status)
|
||||||
|
|
||||||
|
|
||||||
|
class TestBoundedServer(unittest.TestCase):
|
||||||
|
def test_saturated_server_rejects_without_spawning_thread(self) -> None:
|
||||||
|
client, peer = socket.socketpair()
|
||||||
|
with BoundedThreadingHTTPServer(
|
||||||
|
("127.0.0.1", 0), _Handler, max_workers=1, # type: ignore[arg-type]
|
||||||
|
) as server:
|
||||||
|
with client, peer:
|
||||||
|
# Directly reserve the only slot to model an in-flight handler.
|
||||||
|
self.assertTrue(
|
||||||
|
server._request_slots.acquire( # pylint: disable=consider-using-with
|
||||||
|
blocking=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
server.process_request(client, ("127.0.0.1", 1))
|
||||||
|
self.assertIn(b"503 Service Unavailable", peer.recv(1024))
|
||||||
|
finally:
|
||||||
|
server._request_slots.release()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -23,6 +23,7 @@ from bot_bottle.gateway.bootstrap import (
|
|||||||
_DaemonManager,
|
_DaemonManager,
|
||||||
_argv_for_daemon,
|
_argv_for_daemon,
|
||||||
_env_for_daemon,
|
_env_for_daemon,
|
||||||
|
_pump,
|
||||||
_selected_daemons,
|
_selected_daemons,
|
||||||
)
|
)
|
||||||
from tests._bin import SLEEP
|
from tests._bin import SLEEP
|
||||||
@@ -565,5 +566,26 @@ class TestMainEndToEnd(unittest.TestCase):
|
|||||||
self.assertIn("no daemons selected", out)
|
self.assertIn("no daemons selected", out)
|
||||||
|
|
||||||
|
|
||||||
|
class _FailingStream:
|
||||||
|
def __init__(self, *, closed: bool) -> None:
|
||||||
|
self.closed = closed
|
||||||
|
|
||||||
|
def readline(self) -> bytes:
|
||||||
|
raise ValueError("I/O operation on closed file")
|
||||||
|
|
||||||
|
|
||||||
|
class TestOutputPump(unittest.TestCase):
|
||||||
|
def test_closed_stream_race_is_normal_completion(self) -> None:
|
||||||
|
with patch("bot_bottle.gateway.bootstrap._log") as log:
|
||||||
|
_pump("egress", _FailingStream(closed=True)) # type: ignore[arg-type]
|
||||||
|
log.assert_not_called()
|
||||||
|
|
||||||
|
def test_unexpected_io_failure_is_logged(self) -> None:
|
||||||
|
with patch("bot_bottle.gateway.bootstrap._log") as log:
|
||||||
|
_pump("egress", _FailingStream(closed=False)) # type: ignore[arg-type]
|
||||||
|
log.assert_called_once()
|
||||||
|
self.assertIn("output pump stopped", log.call_args.args[0])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -486,6 +486,19 @@ class TestMalformedStatusHeader(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
self.assertEqual(500, status)
|
self.assertEqual(500, status)
|
||||||
|
|
||||||
|
def test_backend_timeout_returns_503(self):
|
||||||
|
with mock.patch(
|
||||||
|
"bot_bottle.gateway.git_gate.http_backend.subprocess.run",
|
||||||
|
side_effect=subprocess.TimeoutExpired(["git", "http-backend"], 1),
|
||||||
|
):
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"http://127.0.0.1:{self._port}/repo.git/info/refs",
|
||||||
|
method="GET",
|
||||||
|
)
|
||||||
|
with self.assertRaises(urllib.error.HTTPError) as raised:
|
||||||
|
urllib.request.urlopen(req, timeout=3)
|
||||||
|
self.assertEqual(503, raised.exception.code)
|
||||||
|
|
||||||
|
|
||||||
class TestContentLengthBounds(unittest.TestCase):
|
class TestContentLengthBounds(unittest.TestCase):
|
||||||
"""PRD 0041: malformed or oversized Content-Length is rejected before
|
"""PRD 0041: malformed or oversized Content-Length is rejected before
|
||||||
|
|||||||
@@ -284,6 +284,18 @@ class TestEnsureArtifact(_CacheMixin):
|
|||||||
ia.ensure_artifact_gz(version, role=_ROLE)
|
ia.ensure_artifact_gz(version, role=_ROLE)
|
||||||
self.assertEqual(first_calls, len(net.calls))
|
self.assertEqual(first_calls, len(net.calls))
|
||||||
|
|
||||||
|
def test_download_uses_network_deadline(self) -> None:
|
||||||
|
response = mock.MagicMock()
|
||||||
|
response.__enter__.return_value = io.BytesIO(b"payload")
|
||||||
|
with tempfile.TemporaryDirectory() as d, mock.patch.object(
|
||||||
|
ia.urllib.request, "urlopen", return_value=response,
|
||||||
|
) as urlopen:
|
||||||
|
ia._download("https://registry/artifact", Path(d) / "artifact")
|
||||||
|
self.assertEqual(
|
||||||
|
ia.ARTIFACT_HTTP_TIMEOUT_SECONDS,
|
||||||
|
urlopen.call_args.kwargs["timeout"],
|
||||||
|
)
|
||||||
|
|
||||||
def test_checksum_mismatch_fails_closed(self) -> None:
|
def test_checksum_mismatch_fails_closed(self) -> None:
|
||||||
version = "beefbeefbeefbeef"
|
version = "beefbeefbeefbeef"
|
||||||
gz = _gz(b"payload")
|
gz = _gz(b"payload")
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import unittest
|
|||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from bot_bottle.backend import EnumerationError
|
from bot_bottle.backend import EnumerationError
|
||||||
|
from bot_bottle.backend.cleanup_control import CleanupError
|
||||||
from bot_bottle.backend.macos_container import cleanup, enumerate as enum_mod
|
from bot_bottle.backend.macos_container import cleanup, enumerate as enum_mod
|
||||||
from bot_bottle.backend.macos_container.bottle_cleanup_plan import (
|
from bot_bottle.backend.macos_container.bottle_cleanup_plan import (
|
||||||
MacosContainerBottleCleanupPlan,
|
MacosContainerBottleCleanupPlan,
|
||||||
@@ -31,7 +32,10 @@ class TestMacosContainerCleanup(unittest.TestCase):
|
|||||||
containers=("bot-bottle-a",),
|
containers=("bot-bottle-a",),
|
||||||
networks=("bot-bottle-net-a",),
|
networks=("bot-bottle-net-a",),
|
||||||
)
|
)
|
||||||
with patch.object(cleanup.subprocess, "run") as run:
|
completed = cleanup.subprocess.CompletedProcess(
|
||||||
|
args=[], returncode=0, stdout="", stderr="",
|
||||||
|
)
|
||||||
|
with patch.object(cleanup.subprocess, "run", return_value=completed) as run:
|
||||||
cleanup.cleanup(plan)
|
cleanup.cleanup(plan)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
["container", "delete", "--force", "bot-bottle-a"],
|
["container", "delete", "--force", "bot-bottle-a"],
|
||||||
@@ -42,6 +46,34 @@ class TestMacosContainerCleanup(unittest.TestCase):
|
|||||||
run.call_args_list[1].args[0],
|
run.call_args_list[1].args[0],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_cleanup_attempts_all_resources_then_raises(self):
|
||||||
|
plan = MacosContainerBottleCleanupPlan(
|
||||||
|
containers=("bot-bottle-a",), networks=("bot-bottle-net-a",),
|
||||||
|
)
|
||||||
|
failed = cleanup.subprocess.CompletedProcess(
|
||||||
|
args=[], returncode=1, stdout="", stderr="unavailable",
|
||||||
|
)
|
||||||
|
with patch.object(cleanup.subprocess, "run", return_value=failed) as run, \
|
||||||
|
self.assertRaisesRegex(CleanupError, "unavailable"):
|
||||||
|
cleanup.cleanup(plan)
|
||||||
|
self.assertEqual(2, run.call_count)
|
||||||
|
|
||||||
|
def test_container_enumeration_failure_aborts(self):
|
||||||
|
completed = cleanup.subprocess.CompletedProcess(
|
||||||
|
args=[], returncode=1, stdout="", stderr="service unavailable",
|
||||||
|
)
|
||||||
|
with patch.object(cleanup.subprocess, "run", return_value=completed), \
|
||||||
|
self.assertRaisesRegex(EnumerationError, "service unavailable"):
|
||||||
|
cleanup._list_prefixed_containers()
|
||||||
|
|
||||||
|
def test_network_enumeration_failure_aborts(self):
|
||||||
|
completed = cleanup.subprocess.CompletedProcess(
|
||||||
|
args=[], returncode=1, stdout="", stderr="service unavailable",
|
||||||
|
)
|
||||||
|
with patch.object(cleanup.subprocess, "run", return_value=completed), \
|
||||||
|
self.assertRaisesRegex(EnumerationError, "service unavailable"):
|
||||||
|
cleanup._list_prefixed_networks()
|
||||||
|
|
||||||
|
|
||||||
class TestMacosContainerEnumerate(unittest.TestCase):
|
class TestMacosContainerEnumerate(unittest.TestCase):
|
||||||
"""The backend launches bottles again (PRD 0070), so enumeration is real
|
"""The backend launches bottles again (PRD 0070), so enumeration is real
|
||||||
|
|||||||
@@ -224,6 +224,17 @@ class TestAgentSecrets(unittest.TestCase):
|
|||||||
reopened = RegistryStore(self.db)
|
reopened = RegistryStore(self.db)
|
||||||
self.assertEqual({"K": "v"}, reopened.get_agent_secrets("bottle-1"))
|
self.assertEqual({"K": "v"}, reopened.get_agent_secrets("bottle-1"))
|
||||||
|
|
||||||
|
def test_v6_migration_clears_legacy_secret_rows(self) -> None:
|
||||||
|
self.store.store_agent_secrets("bottle-1", {"K": "legacy"})
|
||||||
|
with closing(sqlite3.connect(self.db)) as conn:
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE schema_versions SET version = 5 "
|
||||||
|
"WHERE module = 'orchestrator_registry'"
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
self.store.migrate()
|
||||||
|
self.assertEqual({}, self.store.get_agent_secrets("bottle-1"))
|
||||||
|
|
||||||
|
|
||||||
class TestReapAbsent(unittest.TestCase):
|
class TestReapAbsent(unittest.TestCase):
|
||||||
"""`reap_absent` — the self-heal for rows whose bottle is gone.
|
"""`reap_absent` — the self-heal for rows whose bottle is gone.
|
||||||
|
|||||||
@@ -3,8 +3,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
import hashlib
|
|
||||||
import hmac
|
|
||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from bot_bottle.orchestrator.store.secret_store import (
|
from bot_bottle.orchestrator.store.secret_store import (
|
||||||
@@ -83,16 +81,21 @@ class TestDecryptErrors(unittest.TestCase):
|
|||||||
with self.assertRaisesRegex(ValueError, "authentication failed"):
|
with self.assertRaisesRegex(ValueError, "authentication failed"):
|
||||||
decrypt_value(self.secret, tampered)
|
decrypt_value(self.secret, tampered)
|
||||||
|
|
||||||
def test_reads_legacy_ciphertext_for_migration(self) -> None:
|
def test_rejects_legacy_ciphertext(self) -> None:
|
||||||
key = base64.urlsafe_b64decode(self.secret + "==")
|
legacy = base64.urlsafe_b64encode(
|
||||||
nonce = b"0123456789abcdef"
|
b"0123456789abcdeflegacy-token",
|
||||||
plaintext = b"legacy-token"
|
).rstrip(b"=").decode()
|
||||||
stream = hmac.new(
|
with self.assertRaisesRegex(ValueError, "unsupported ciphertext format"):
|
||||||
key, nonce + (0).to_bytes(4, "big"), hashlib.sha256,
|
decrypt_value(self.secret, legacy)
|
||||||
).digest()
|
|
||||||
ciphertext = bytes(p ^ k for p, k in zip(plaintext, stream))
|
def test_rejects_authenticated_blob_with_changed_version(self) -> None:
|
||||||
legacy = base64.urlsafe_b64encode(nonce + ciphertext).rstrip(b"=").decode()
|
raw = bytearray(base64.urlsafe_b64decode(
|
||||||
self.assertEqual("legacy-token", decrypt_value(self.secret, legacy))
|
encrypt_value(self.secret, "secret-token") + "=="
|
||||||
|
))
|
||||||
|
raw[0] ^= 1
|
||||||
|
downgraded = base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
|
||||||
|
with self.assertRaisesRegex(ValueError, "unsupported ciphertext format"):
|
||||||
|
decrypt_value(self.secret, downgraded)
|
||||||
|
|
||||||
def test_truncated_blob_raises_value_error(self) -> None:
|
def test_truncated_blob_raises_value_error(self) -> None:
|
||||||
with self.assertRaises(ValueError):
|
with self.assertRaises(ValueError):
|
||||||
|
|||||||
@@ -49,6 +49,16 @@ class TestPut(unittest.TestCase):
|
|||||||
self.assertNotIsInstance(req.data, (bytes, bytearray))
|
self.assertNotIsInstance(req.data, (bytes, bytearray))
|
||||||
self.assertEqual(str(len(payload)), req.get_header("Content-length"))
|
self.assertEqual(str(len(payload)), req.get_header("Content-length"))
|
||||||
|
|
||||||
|
def test_put_uses_network_deadline(self) -> None:
|
||||||
|
with mock.patch.object(
|
||||||
|
pub.urllib.request, "urlopen", return_value=_Resp(),
|
||||||
|
) as urlopen:
|
||||||
|
pub._put("https://reg/pkg", b"payload", token="")
|
||||||
|
self.assertEqual(
|
||||||
|
pub._REGISTRY_HTTP_TIMEOUT_SECONDS,
|
||||||
|
urlopen.call_args.kwargs["timeout"],
|
||||||
|
)
|
||||||
|
|
||||||
def test_small_bytes_body_still_works(self) -> None:
|
def test_small_bytes_body_still_works(self) -> None:
|
||||||
captured: list[urllib.request.Request] = []
|
captured: list[urllib.request.Request] = []
|
||||||
|
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import tempfile
|
|||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import types
|
import types
|
||||||
|
import typing
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -47,6 +48,7 @@ from bot_bottle.gateway.supervisor.server import (
|
|||||||
jsonrpc_error,
|
jsonrpc_error,
|
||||||
jsonrpc_result,
|
jsonrpc_result,
|
||||||
parse_jsonrpc,
|
parse_jsonrpc,
|
||||||
|
resolved_routes_payload,
|
||||||
validate_proposed_file,
|
validate_proposed_file,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -701,22 +703,40 @@ class TestResolvedRoutesPayload(unittest.TestCase):
|
|||||||
" - host: api.anthropic.com\n"
|
" - host: api.anthropic.com\n"
|
||||||
" - host: www.google.com\n"
|
" - host: www.google.com\n"
|
||||||
)
|
)
|
||||||
payload = _handler(
|
payload = resolved_routes_payload(
|
||||||
_FakeSuperviseResolver(bottle_id="b1", policy=policy)
|
typing.cast(
|
||||||
)._resolved_routes_payload()
|
supervise_server.PolicyResolver,
|
||||||
|
_FakeSuperviseResolver(bottle_id="b1", policy=policy),
|
||||||
|
),
|
||||||
|
_SRC,
|
||||||
|
_TOK,
|
||||||
|
)
|
||||||
assert payload is not None
|
assert payload is not None
|
||||||
self.assertFalse(payload["isError"]) # type: ignore[index]
|
self.assertFalse(payload["isError"]) # type: ignore[index]
|
||||||
data = json.loads(payload["content"][0]["text"]) # type: ignore[index]
|
data = json.loads(payload["content"][0]["text"]) # type: ignore[index]
|
||||||
hosts = {r["host"] for r in data["routes"]}
|
hosts = {r["host"] for r in data["routes"]}
|
||||||
self.assertEqual({"api.anthropic.com", "www.google.com"}, hosts)
|
self.assertEqual({"api.anthropic.com", "www.google.com"}, hosts)
|
||||||
|
|
||||||
def test_orchestrator_error_fails_closed_to_empty(self) -> None:
|
def test_orchestrator_error_is_not_reported_as_empty_policy(self) -> None:
|
||||||
# resolve_client_context swallows resolver errors → deny-all (empty),
|
with self.assertRaises(supervise_server.RouteResolutionError):
|
||||||
# never another bottle's routes.
|
resolved_routes_payload(
|
||||||
payload = _handler(
|
typing.cast(
|
||||||
_FakeSuperviseResolver(raises=True)
|
supervise_server.PolicyResolver,
|
||||||
)._resolved_routes_payload()
|
_FakeSuperviseResolver(raises=True),
|
||||||
assert payload is not None
|
),
|
||||||
|
_SRC,
|
||||||
|
_TOK,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_authoritative_empty_policy_remains_successful(self) -> None:
|
||||||
|
payload = resolved_routes_payload(
|
||||||
|
typing.cast(
|
||||||
|
supervise_server.PolicyResolver,
|
||||||
|
_FakeSuperviseResolver(bottle_id="b1", policy="routes: []\n"),
|
||||||
|
),
|
||||||
|
_SRC,
|
||||||
|
_TOK,
|
||||||
|
)
|
||||||
data = json.loads(payload["content"][0]["text"]) # type: ignore[index]
|
data = json.loads(payload["content"][0]["text"]) # type: ignore[index]
|
||||||
self.assertEqual([], data["routes"])
|
self.assertEqual([], data["routes"])
|
||||||
|
|
||||||
@@ -724,7 +744,13 @@ class TestResolvedRoutesPayload(unittest.TestCase):
|
|||||||
# A server without a resolver is a misconfig, not a mode: raise rather
|
# A server without a resolver is a misconfig, not a mode: raise rather
|
||||||
# than list anything.
|
# than list anything.
|
||||||
with self.assertRaises(_RpcInternalError):
|
with self.assertRaises(_RpcInternalError):
|
||||||
_handler(None)._resolved_routes_payload()
|
_handler(None)._dispatch(
|
||||||
|
parse_jsonrpc(
|
||||||
|
b'{"jsonrpc":"2.0","id":1,"method":"tools/call",'
|
||||||
|
b'"params":{"name":"list-egress-routes"}}',
|
||||||
|
),
|
||||||
|
ServerConfig(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestNonBlockingSupervise(unittest.TestCase):
|
class TestNonBlockingSupervise(unittest.TestCase):
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
"""Unit tests for framework-neutral supervisor MCP dispatch."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from bot_bottle.gateway.supervisor.mcp_dispatch import (
|
||||||
|
Handlers,
|
||||||
|
MethodNotFoundError,
|
||||||
|
dispatch,
|
||||||
|
)
|
||||||
|
from bot_bottle.supervisor import types as _sv
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class _Request:
|
||||||
|
method: str
|
||||||
|
params: dict[str, object]
|
||||||
|
|
||||||
|
|
||||||
|
class TestDispatch(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.calls: list[str] = []
|
||||||
|
|
||||||
|
def handler(name: str):
|
||||||
|
def call(_params: dict[str, object]) -> str:
|
||||||
|
self.calls.append(name)
|
||||||
|
return name
|
||||||
|
|
||||||
|
return call
|
||||||
|
|
||||||
|
self.handlers = Handlers(
|
||||||
|
initialize=handler("initialize"),
|
||||||
|
tools_list=handler("tools_list"),
|
||||||
|
list_routes=handler("list_routes"),
|
||||||
|
check_proposal=handler("check_proposal"),
|
||||||
|
propose=handler("propose"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def request(self, method: str, **params: object) -> _Request:
|
||||||
|
return _Request(method=method, params=params)
|
||||||
|
|
||||||
|
def test_routes_protocol_methods(self) -> None:
|
||||||
|
self.assertEqual(
|
||||||
|
"initialize", dispatch(self.request("initialize"), self.handlers),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
"tools_list", dispatch(self.request("tools/list"), self.handlers),
|
||||||
|
)
|
||||||
|
self.assertIsNone(
|
||||||
|
dispatch(self.request("notifications/initialized"), self.handlers),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_routes_each_tool_class(self) -> None:
|
||||||
|
self.assertEqual(
|
||||||
|
"list_routes",
|
||||||
|
dispatch(
|
||||||
|
self.request("tools/call", name=_sv.TOOL_LIST_EGRESS_ROUTES),
|
||||||
|
self.handlers,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
"check_proposal",
|
||||||
|
dispatch(
|
||||||
|
self.request("tools/call", name=_sv.TOOL_CHECK_PROPOSAL),
|
||||||
|
self.handlers,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
"propose",
|
||||||
|
dispatch(self.request("tools/call", name=_sv.TOOL_EGRESS_ALLOW), self.handlers),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_unknown_method_is_typed(self) -> None:
|
||||||
|
with self.assertRaisesRegex(MethodNotFoundError, "unknown"):
|
||||||
|
dispatch(self.request("unknown"), self.handlers)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user