Compare commits
51 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b05775b581 | |||
| 2a2669b69f | |||
| 8ae6561b33 | |||
| 03eacd9f57 | |||
| c839cee319 | |||
| ea75fab3b4 | |||
| 13a8bdd7ca | |||
| 3318bdce28 | |||
| b2f61053ad | |||
| 485d101430 | |||
| e97366dad5 | |||
| 8e7f5c9572 | |||
| 69db629e16 | |||
| 96f75599a1 | |||
| 3062fc9230 | |||
| 404369a3e0 | |||
| 327e756eef | |||
| 444924d95d | |||
| 118d81533b | |||
| 353b1ad984 | |||
| 40ad2364ab | |||
| 9e5d00c6fa | |||
| 0c158c5718 | |||
| fe25ad7623 | |||
| 6570e9f044 | |||
| 39c823d3c0 | |||
| 8a44537f7e | |||
| 72ee1f77da | |||
| 00418a2834 | |||
| 2cbb178f88 | |||
| ed9a19bb38 | |||
| c5ae93c8ba | |||
| 186b905dff | |||
| 3519b924cb | |||
| c77f578fb4 | |||
| bc52c3525c | |||
| b70f3228ca | |||
| f7b3d6aaca | |||
| b05f245299 | |||
| 8d4e7f2582 | |||
| a0e17d56de | |||
| 934c1617b9 | |||
| bd964c0278 | |||
| 93756f2a8b | |||
| d9843fa41e | |||
| ae6c8b13d9 | |||
| 13a8d66c9b | |||
| c46d392b44 | |||
| 2a473eb404 | |||
| 4d89ffce8b | |||
| 822cff48ad |
@@ -5,7 +5,7 @@
|
|||||||
# bot-bottle
|
# bot-bottle
|
||||||
|
|
||||||
[](https://gitea.dideric.is/didericis/bot-bottle/actions?workflow=test.yml)
|
[](https://gitea.dideric.is/didericis/bot-bottle/actions?workflow=test.yml)
|
||||||
[](https://coverage.readthedocs.io/)
|
[](https://coverage.readthedocs.io/)
|
||||||
[](https://gitea.dideric.is/didericis/bot-bottle/src/branch/main/docs/decisions/0004-coverage-policy.md)
|
[](https://gitea.dideric.is/didericis/bot-bottle/src/branch/main/docs/decisions/0004-coverage-policy.md)
|
||||||
|
|
||||||
**Problem:** Developer wants to run a coding agent without supervision, but they don't want a prompt injected or misbehaving agent wrecking their environment or exfiltrating sensitive data.
|
**Problem:** Developer wants to run a coding agent without supervision, but they don't want a prompt injected or misbehaving agent wrecking their environment or exfiltrating sensitive data.
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ from abc import ABC, abstractmethod
|
|||||||
from contextlib import AbstractContextManager
|
from contextlib import AbstractContextManager
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Generic, Sequence, TypeVar
|
from typing import TYPE_CHECKING, Any, Generic, Sequence, TypeVar
|
||||||
|
|
||||||
from ..agent_provider import AgentProvisionPlan, get_provider, build_agent_provision_plan
|
from ..agent_provider import AgentProvisionPlan, get_provider, build_agent_provision_plan
|
||||||
from ..egress import EgressPlan
|
from ..egress import EgressPlan
|
||||||
@@ -54,6 +54,9 @@ from ..workspace import WorkspacePlan, workspace_plan
|
|||||||
from .print_util import print_multi, visible_agent_env_names
|
from .print_util import print_multi, visible_agent_env_names
|
||||||
from .util import host_skill_dir
|
from .util import host_skill_dir
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from .freeze import CommitCancelled, Freezer, get_freezer
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class BottleSpec:
|
class BottleSpec:
|
||||||
@@ -572,28 +575,63 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]):
|
|||||||
Not called by the launch path or the test suite."""
|
Not called by the launch path or the test suite."""
|
||||||
|
|
||||||
|
|
||||||
# Import concrete backend classes AFTER the base types are defined, so
|
# _backends is None until the first call to _get_backends(), at which
|
||||||
# each backend module can pull BottleSpec / BottlePlan / BottleBackend
|
# point all three concrete backend classes are imported and instantiated.
|
||||||
# via `from . import ...` without hitting a partially-initialized module.
|
# Keeping the imports out of module scope means that importing any
|
||||||
from .docker import DockerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position
|
# backend sub-module (e.g. `backend.docker.util`) no longer drags the
|
||||||
from .firecracker import FirecrackerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position
|
# firecracker and macos-container implementations into memory.
|
||||||
from .macos_container import MacosContainerBottleBackend # noqa: E402 # pylint: disable=wrong-import-position
|
#
|
||||||
|
# Tests may replace _backends with a {name: fake} dict via patch.object;
|
||||||
# Freezer is imported after the backend classes for the same reason:
|
# _get_backends() returns the current module-level value as-is when it
|
||||||
# Freezer.commit_slug constructs ActiveAgent, which must be fully
|
# is not None, so test fakes take effect without triggering real imports.
|
||||||
# defined first.
|
_backends: dict[str, BottleBackend[Any, Any]] | None = None
|
||||||
from .freeze import CommitCancelled, Freezer, get_freezer # noqa: E402 # pylint: disable=wrong-import-position
|
|
||||||
|
|
||||||
|
|
||||||
# The dict is heterogeneous: each value is a BottleBackend specialized
|
def _get_backends() -> dict[str, BottleBackend[Any, Any]]:
|
||||||
# over its own plan type. Concrete plan types are erased here because
|
"""Return the registry of all backend instances, loading lazily on first call."""
|
||||||
# the registry is selected at runtime and the CLI only needs the
|
global _backends # pylint: disable=global-statement
|
||||||
# unparameterized methods (prepare → plan → launch(plan), cleanup, etc.).
|
if _backends is None:
|
||||||
_BACKENDS: dict[str, BottleBackend[Any, Any]] = {
|
from .docker import DockerBottleBackend
|
||||||
"docker": DockerBottleBackend(),
|
from .firecracker import FirecrackerBottleBackend
|
||||||
"firecracker": FirecrackerBottleBackend(),
|
from .macos_container import MacosContainerBottleBackend
|
||||||
"macos-container": MacosContainerBottleBackend(),
|
_backends = {
|
||||||
}
|
"docker": DockerBottleBackend(),
|
||||||
|
"firecracker": FirecrackerBottleBackend(),
|
||||||
|
"macos-container": MacosContainerBottleBackend(),
|
||||||
|
}
|
||||||
|
return _backends
|
||||||
|
|
||||||
|
|
||||||
|
def __getattr__(name: str) -> Any:
|
||||||
|
"""Lazily surface concrete backend classes and freeze symbols at the
|
||||||
|
package level so existing `from bot_bottle.backend import X` and
|
||||||
|
`patch.object(backend_mod, X, ...)` call-sites keep working without
|
||||||
|
forcing an import of every backend at module-init time."""
|
||||||
|
if name == "DockerBottleBackend":
|
||||||
|
from .docker import DockerBottleBackend
|
||||||
|
globals()[name] = DockerBottleBackend
|
||||||
|
return DockerBottleBackend
|
||||||
|
if name == "FirecrackerBottleBackend":
|
||||||
|
from .firecracker import FirecrackerBottleBackend
|
||||||
|
globals()[name] = FirecrackerBottleBackend
|
||||||
|
return FirecrackerBottleBackend
|
||||||
|
if name == "MacosContainerBottleBackend":
|
||||||
|
from .macos_container import MacosContainerBottleBackend
|
||||||
|
globals()[name] = MacosContainerBottleBackend
|
||||||
|
return MacosContainerBottleBackend
|
||||||
|
if name == "CommitCancelled":
|
||||||
|
from .freeze import CommitCancelled
|
||||||
|
globals()[name] = CommitCancelled
|
||||||
|
return CommitCancelled
|
||||||
|
if name == "Freezer":
|
||||||
|
from .freeze import Freezer
|
||||||
|
globals()[name] = Freezer
|
||||||
|
return Freezer
|
||||||
|
if name == "get_freezer":
|
||||||
|
from .freeze import get_freezer
|
||||||
|
globals()[name] = get_freezer
|
||||||
|
return get_freezer
|
||||||
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||||
|
|
||||||
|
|
||||||
def get_bottle_backend(
|
def get_bottle_backend(
|
||||||
@@ -611,10 +649,11 @@ def get_bottle_backend(
|
|||||||
Dies with a pointer at the known backends if the chosen name
|
Dies with a pointer at the known backends if the chosen name
|
||||||
isn't implemented."""
|
isn't implemented."""
|
||||||
resolved = name or os.environ.get("BOT_BOTTLE_BACKEND") or _default_backend_name()
|
resolved = name or os.environ.get("BOT_BOTTLE_BACKEND") or _default_backend_name()
|
||||||
if resolved not in _BACKENDS:
|
backends = _get_backends()
|
||||||
known = ", ".join(sorted(_BACKENDS))
|
if resolved not in backends:
|
||||||
|
known = ", ".join(sorted(backends))
|
||||||
die(f"unknown backend {resolved!r}; known backends: {known}")
|
die(f"unknown backend {resolved!r}; known backends: {known}")
|
||||||
return _BACKENDS[resolved]
|
return backends[resolved]
|
||||||
|
|
||||||
|
|
||||||
def _default_backend_name() -> str:
|
def _default_backend_name() -> str:
|
||||||
@@ -624,16 +663,17 @@ def _default_backend_name() -> str:
|
|||||||
# `firecracker` binary isn't installed yet: selecting it here routes
|
# `firecracker` binary isn't installed yet: selecting it here routes
|
||||||
# start through firecracker's preflight, which prints an install
|
# start through firecracker's preflight, which prints an install
|
||||||
# pointer, instead of silently falling back to docker.
|
# pointer, instead of silently falling back to docker.
|
||||||
|
from .firecracker import FirecrackerBottleBackend
|
||||||
if FirecrackerBottleBackend.is_host_capable():
|
if FirecrackerBottleBackend.is_host_capable():
|
||||||
return "firecracker"
|
return "firecracker"
|
||||||
return "docker"
|
return "docker"
|
||||||
|
|
||||||
|
|
||||||
def known_backend_names() -> tuple[str, ...]:
|
def known_backend_names() -> tuple[str, ...]:
|
||||||
"""Sorted tuple of all backend keys in `_BACKENDS`. Used by
|
"""Sorted tuple of all backend keys in `_get_backends()`. Used by
|
||||||
argparse (`--backend` choices) and the dashboard's backend
|
argparse (`--backend` choices) and the dashboard's backend
|
||||||
picker."""
|
picker."""
|
||||||
return tuple(sorted(_BACKENDS))
|
return tuple(sorted(_get_backends()))
|
||||||
|
|
||||||
|
|
||||||
def has_backend(name: str) -> bool:
|
def has_backend(name: str) -> bool:
|
||||||
@@ -645,9 +685,10 @@ def has_backend(name: str) -> bool:
|
|||||||
|
|
||||||
Returns False for unknown names so callers can pass
|
Returns False for unknown names so callers can pass
|
||||||
arbitrary input without separate validation."""
|
arbitrary input without separate validation."""
|
||||||
if name not in _BACKENDS:
|
backends = _get_backends()
|
||||||
|
if name not in backends:
|
||||||
return False
|
return False
|
||||||
return _BACKENDS[name].is_available()
|
return backends[name].is_available()
|
||||||
|
|
||||||
|
|
||||||
def enumerate_active_agents() -> list[ActiveAgent]:
|
def enumerate_active_agents() -> list[ActiveAgent]:
|
||||||
@@ -663,10 +704,11 @@ def enumerate_active_agents() -> list[ActiveAgent]:
|
|||||||
deterministic tiebreaker. Agents with missing metadata
|
deterministic tiebreaker. Agents with missing metadata
|
||||||
(`started_at == ""`) sort first."""
|
(`started_at == ""`) sort first."""
|
||||||
out: list[ActiveAgent] = []
|
out: list[ActiveAgent] = []
|
||||||
for name in known_backend_names():
|
backends = _get_backends()
|
||||||
if not has_backend(name):
|
for name in sorted(backends):
|
||||||
|
if not backends[name].is_available():
|
||||||
continue
|
continue
|
||||||
out.extend(_BACKENDS[name].enumerate_active())
|
out.extend(backends[name].enumerate_active())
|
||||||
out.sort(key=lambda a: (a.started_at, a.slug))
|
out.sort(key=lambda a: (a.started_at, a.slug))
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|||||||
@@ -98,6 +98,7 @@ def launch_consolidated(
|
|||||||
git_gate_plan: GitGatePlan,
|
git_gate_plan: GitGatePlan,
|
||||||
*,
|
*,
|
||||||
image_ref: str = "",
|
image_ref: str = "",
|
||||||
|
tokens: dict[str, str] | None = None,
|
||||||
service: OrchestratorService | None = None,
|
service: OrchestratorService | None = None,
|
||||||
gateway_name: str = GATEWAY_NAME,
|
gateway_name: str = GATEWAY_NAME,
|
||||||
network: str = GATEWAY_NETWORK,
|
network: str = GATEWAY_NETWORK,
|
||||||
@@ -116,7 +117,8 @@ def launch_consolidated(
|
|||||||
|
|
||||||
inputs = registration_inputs(egress_plan)
|
inputs = registration_inputs(egress_plan)
|
||||||
reg = client.register_bottle(
|
reg = client.register_bottle(
|
||||||
source_ip, image_ref=image_ref, policy=inputs.policy, metadata=inputs.metadata,
|
source_ip, image_ref=image_ref, policy=inputs.policy,
|
||||||
|
metadata=inputs.metadata, tokens=tokens,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
provision_git_gate(gateway_name, reg.bottle_id, git_gate_plan)
|
provision_git_gate(gateway_name, reg.bottle_id, git_gate_plan)
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ from contextlib import ExitStack, contextmanager
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Callable, Generator
|
from typing import Callable, Generator
|
||||||
|
|
||||||
|
from ...egress import egress_resolve_token_values
|
||||||
from ...git_gate import (
|
from ...git_gate import (
|
||||||
provision_git_gate_dynamic_keys,
|
provision_git_gate_dynamic_keys,
|
||||||
revoke_git_gate_provisioned_keys,
|
revoke_git_gate_provisioned_keys,
|
||||||
@@ -120,8 +121,17 @@ def launch(
|
|||||||
|
|
||||||
# Step 3: register on the orchestrator + provision this bottle's
|
# Step 3: register on the orchestrator + provision this bottle's
|
||||||
# git-gate state into the shared gateway; get the agent's attach
|
# git-gate state into the shared gateway; get the agent's attach
|
||||||
# context (pinned source IP, gateway address, shared network).
|
# context (pinned source IP, gateway address, shared network). The
|
||||||
ctx = launch_consolidated(plan.egress_plan, git_gate_plan, image_ref=plan.image)
|
# per-bottle egress auth tokens are resolved from the host env now and
|
||||||
|
# handed to the orchestrator (in memory) for the gateway to inject —
|
||||||
|
# the agent never sees them.
|
||||||
|
effective_env = {**os.environ, **plan.agent_provision.provisioned_env}
|
||||||
|
token_values = egress_resolve_token_values(
|
||||||
|
plan.egress_plan.token_env_map, effective_env,
|
||||||
|
)
|
||||||
|
ctx = launch_consolidated(
|
||||||
|
plan.egress_plan, git_gate_plan, image_ref=plan.image, tokens=token_values,
|
||||||
|
)
|
||||||
stack.callback(
|
stack.callback(
|
||||||
teardown_consolidated, ctx.bottle_id, orchestrator_url=ctx.orchestrator_url,
|
teardown_consolidated, ctx.bottle_id, orchestrator_url=ctx.orchestrator_url,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -7,8 +7,9 @@ from __future__ import annotations
|
|||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
from typing import Iterable, Iterator
|
from typing import Iterator
|
||||||
|
|
||||||
|
from ...docker_cmd import run_docker
|
||||||
from ...log import die, info
|
from ...log import die, info
|
||||||
# from ...workspace import WorkspacePlan
|
# from ...workspace import WorkspacePlan
|
||||||
|
|
||||||
@@ -30,12 +31,7 @@ def container_name_candidates(base: str) -> Iterator[str]:
|
|||||||
def runsc_available() -> bool:
|
def runsc_available() -> bool:
|
||||||
"""Return True if the Docker daemon has the gVisor (`runsc`) runtime
|
"""Return True if the Docker daemon has the gVisor (`runsc`) runtime
|
||||||
registered. Called once per prepare; the result lives on the plan."""
|
registered. Called once per prepare; the result lives on the plan."""
|
||||||
r = subprocess.run(
|
r = run_docker(["docker", "info", "--format", "{{json .Runtimes}}"])
|
||||||
["docker", "info", "--format", "{{json .Runtimes}}"],
|
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
return r.returncode == 0 and "runsc" in r.stdout
|
return r.returncode == 0 and "runsc" in r.stdout
|
||||||
|
|
||||||
|
|
||||||
@@ -49,20 +45,15 @@ def require_docker() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def image_exists(ref: str) -> bool:
|
def image_exists(ref: str) -> bool:
|
||||||
return _silent_run(["docker", "image", "inspect", ref]) == 0
|
return run_docker(["docker", "image", "inspect", ref]).returncode == 0
|
||||||
|
|
||||||
|
|
||||||
def container_exists(name: str) -> bool:
|
def container_exists(name: str) -> bool:
|
||||||
"""Returns True if a container (running or stopped) with the given
|
"""Returns True if a container (running or stopped) with the given
|
||||||
name exists. Uses `docker ps -a -q -f name=^<name>$` so substring
|
name exists. Uses `docker ps -a -q -f name=^<name>$` so substring
|
||||||
matches don't false-positive."""
|
matches don't false-positive."""
|
||||||
result = subprocess.run(
|
result = run_docker(["docker", "ps", "-a", "-q", "-f", f"name=^{name}$"])
|
||||||
["docker", "ps", "-a", "-q", "-f", f"name=^{name}$"],
|
return result.returncode == 0 and bool(result.stdout.strip())
|
||||||
capture_output=True,
|
|
||||||
text=True,
|
|
||||||
check=True,
|
|
||||||
)
|
|
||||||
return bool(result.stdout.strip())
|
|
||||||
|
|
||||||
|
|
||||||
def force_remove_container(name: str) -> None:
|
def force_remove_container(name: str) -> None:
|
||||||
@@ -70,12 +61,7 @@ def force_remove_container(name: str) -> None:
|
|||||||
doesn't — and the rm itself is best-effort (errors swallowed) so
|
doesn't — and the rm itself is best-effort (errors swallowed) so
|
||||||
this is safe to register as a teardown callback."""
|
this is safe to register as a teardown callback."""
|
||||||
if container_exists(name):
|
if container_exists(name):
|
||||||
subprocess.run(
|
run_docker(["docker", "rm", "-f", name])
|
||||||
["docker", "rm", "-f", name],
|
|
||||||
stdout=subprocess.DEVNULL,
|
|
||||||
stderr=subprocess.DEVNULL,
|
|
||||||
check=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def docker_exec_root(container: str, argv: list[str]) -> None:
|
def docker_exec_root(container: str, argv: list[str]) -> None:
|
||||||
@@ -155,22 +141,10 @@ def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
|
|||||||
def commit_container(container_name: str, image_tag: str) -> None:
|
def commit_container(container_name: str, image_tag: str) -> None:
|
||||||
"""Run `docker commit <container_name> <image_tag>` to snapshot the
|
"""Run `docker commit <container_name> <image_tag>` to snapshot the
|
||||||
running container's filesystem state as a local Docker image."""
|
running container's filesystem state as a local Docker image."""
|
||||||
result = subprocess.run(
|
result = run_docker(["docker", "commit", container_name, image_tag])
|
||||||
["docker", "commit", container_name, image_tag],
|
|
||||||
capture_output=True, text=True, check=False,
|
|
||||||
)
|
|
||||||
if result.returncode != 0:
|
if result.returncode != 0:
|
||||||
die(
|
die(
|
||||||
f"docker commit {container_name!r} → {image_tag!r} failed: "
|
f"docker commit {container_name!r} → {image_tag!r} failed: "
|
||||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||||
)
|
)
|
||||||
info(f"committed {container_name!r} → {image_tag!r}")
|
info(f"committed {container_name!r} → {image_tag!r}")
|
||||||
|
|
||||||
|
|
||||||
def _silent_run(cmd: Iterable[str]) -> int:
|
|
||||||
return subprocess.run(
|
|
||||||
list(cmd),
|
|
||||||
stdout=subprocess.DEVNULL,
|
|
||||||
stderr=subprocess.DEVNULL,
|
|
||||||
check=False,
|
|
||||||
).returncode
|
|
||||||
|
|||||||
+46
-33
@@ -10,6 +10,7 @@ import json
|
|||||||
import os
|
import os
|
||||||
import signal
|
import signal
|
||||||
import sys
|
import sys
|
||||||
|
import typing
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=import-error
|
from mitmproxy import http # type: ignore[import-not-found] # pylint: disable=import-error
|
||||||
@@ -230,21 +231,27 @@ class EgressAddon:
|
|||||||
+ "\n"
|
+ "\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
def _resolve_flow(self, flow: http.HTTPFlow) -> "tuple[Config, str]":
|
def _resolve_flow(
|
||||||
"""The `(Config, supervise slug)` to apply to this request. Single-tenant
|
self, flow: http.HTTPFlow,
|
||||||
→ the static `self.config` and the env slug. Consolidated → the calling
|
) -> "tuple[Config, str, typing.Mapping[str, str]]":
|
||||||
bottle's Config and its bottle id, resolved by source IP in one
|
"""The `(Config, supervise slug, env)` to apply to this request.
|
||||||
round-trip (fail-closed to deny-all + empty slug if unattributed). The
|
Single-tenant → the static `self.config`, the env slug, and the process
|
||||||
identity token, if the agent injected one, is read then stripped so it
|
env. Consolidated → the calling bottle's Config + bottle id + auth
|
||||||
never leaks upstream. The slug keys both the proposal queue and the
|
tokens, resolved by source IP in one round-trip (fail-closed to deny-all
|
||||||
per-bottle safelist, so an approval only ever affects its own bottle."""
|
+ empty slug if unattributed); `env` is the process env overlaid with
|
||||||
|
the bottle's tokens, so upstream-auth injection (and DLP) use *this*
|
||||||
|
bottle's credentials — exactly what the per-bottle sidecar's env did.
|
||||||
|
The identity token, if the agent injected one, is read then stripped so
|
||||||
|
it never leaks upstream."""
|
||||||
if self._resolver is None:
|
if self._resolver is None:
|
||||||
return self.config, self._supervise_slug
|
return self.config, self._supervise_slug, os.environ
|
||||||
conn = flow.client_conn
|
conn = flow.client_conn
|
||||||
client_ip = conn.peername[0] if conn and conn.peername else ""
|
client_ip = conn.peername[0] if conn and conn.peername else ""
|
||||||
token = flow.request.headers.get(IDENTITY_HEADER, "")
|
token = flow.request.headers.get(IDENTITY_HEADER, "")
|
||||||
flow.request.headers.pop(IDENTITY_HEADER, None)
|
flow.request.headers.pop(IDENTITY_HEADER, None)
|
||||||
return resolve_client_context(self._resolver, client_ip, token)
|
config, slug, tokens = resolve_client_context(self._resolver, client_ip, token)
|
||||||
|
env = {**os.environ, **tokens} if tokens else os.environ
|
||||||
|
return config, slug, env
|
||||||
|
|
||||||
async def request(self, flow: http.HTTPFlow) -> None:
|
async def request(self, flow: http.HTTPFlow) -> None:
|
||||||
request_path, _, query = flow.request.path.partition("?")
|
request_path, _, query = flow.request.path.partition("?")
|
||||||
@@ -253,14 +260,14 @@ class EgressAddon:
|
|||||||
self._serve_introspection(flow, request_path)
|
self._serve_introspection(flow, request_path)
|
||||||
return
|
return
|
||||||
|
|
||||||
config, slug = self._resolve_flow(flow)
|
config, slug, env = self._resolve_flow(flow)
|
||||||
|
|
||||||
# DLP outbound scan BEFORE stripping auth — catches tokens the
|
# DLP outbound scan BEFORE stripping auth — catches tokens the
|
||||||
# agent tried to smuggle in any header, path, query param, or body.
|
# agent tried to smuggle in any header, path, query param, or body.
|
||||||
# Hostname is included to catch DNS-tunnelling exfiltration attempts.
|
# Hostname is included to catch DNS-tunnelling exfiltration attempts.
|
||||||
route = match_route(config.routes, flow.request.pretty_host)
|
route = match_route(config.routes, flow.request.pretty_host)
|
||||||
if route is not None:
|
if route is not None:
|
||||||
if not await self._handle_outbound_dlp(flow, route, slug):
|
if not await self._handle_outbound_dlp(flow, route, slug, env):
|
||||||
return
|
return
|
||||||
# The redact policy may have rewritten the request line; recompute
|
# The redact policy may have rewritten the request line; recompute
|
||||||
# the path/query the git checks below rely on.
|
# the path/query the git checks below rely on.
|
||||||
@@ -299,7 +306,7 @@ class EgressAddon:
|
|||||||
config.routes,
|
config.routes,
|
||||||
flow.request.pretty_host,
|
flow.request.pretty_host,
|
||||||
request_path,
|
request_path,
|
||||||
os.environ,
|
env,
|
||||||
request_method=flow.request.method,
|
request_method=flow.request.method,
|
||||||
request_headers=req_headers,
|
request_headers=req_headers,
|
||||||
)
|
)
|
||||||
@@ -325,10 +332,12 @@ class EgressAddon:
|
|||||||
flow: http.HTTPFlow,
|
flow: http.HTTPFlow,
|
||||||
route: Route,
|
route: Route,
|
||||||
slug: str,
|
slug: str,
|
||||||
|
env: "typing.Mapping[str, str]",
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Scan the outbound request and apply the route's on-match policy
|
"""Scan the outbound request and apply the route's on-match policy
|
||||||
(PRD 0062). Returns True if the request may be forwarded, False if a
|
(PRD 0062). `env` is the per-bottle env overlay (process env + this
|
||||||
403 response has been written to `flow`.
|
bottle's tokens) used for DLP detection. Returns True if the request may
|
||||||
|
be forwarded, False if a 403 response has been written to `flow`.
|
||||||
|
|
||||||
Loops so the supervise policy can re-scan after each approval — a
|
Loops so the supervise policy can re-scan after each approval — a
|
||||||
second, un-approved token in the same request is still caught."""
|
second, un-approved token in the same request is still caught."""
|
||||||
@@ -345,7 +354,7 @@ class EgressAddon:
|
|||||||
flow.request.pretty_host, request_path, query, headers, "",
|
flow.request.pretty_host, request_path, query, headers, "",
|
||||||
)
|
)
|
||||||
result = scan_outbound(
|
result = scan_outbound(
|
||||||
route, scan_text, os.environ,
|
route, scan_text, env,
|
||||||
safe_tokens=self._safe_tokens_for(slug), crlf_text=crlf_text,
|
safe_tokens=self._safe_tokens_for(slug), crlf_text=crlf_text,
|
||||||
)
|
)
|
||||||
if result is None or result.severity != "block":
|
if result is None or result.severity != "block":
|
||||||
@@ -356,7 +365,7 @@ class EgressAddon:
|
|||||||
# redact scrubs every detection (tokens and structural CRLF) and
|
# redact scrubs every detection (tokens and structural CRLF) and
|
||||||
# forwards; it fails closed only if a match survives the scrub.
|
# forwards; it fails closed only if a match survives the scrub.
|
||||||
if policy == ON_MATCH_REDACT:
|
if policy == ON_MATCH_REDACT:
|
||||||
if self._redact_outbound(flow, route):
|
if self._redact_outbound(flow, route, env):
|
||||||
if self.config.log >= LOG_BLOCKS:
|
if self.config.log >= LOG_BLOCKS:
|
||||||
sys.stderr.write(json.dumps({
|
sys.stderr.write(json.dumps({
|
||||||
"event": "egress_redacted",
|
"event": "egress_redacted",
|
||||||
@@ -384,29 +393,31 @@ class EgressAddon:
|
|||||||
if not self._supervise_available(slug):
|
if not self._supervise_available(slug):
|
||||||
self._block_dlp(flow, result)
|
self._block_dlp(flow, result)
|
||||||
return False
|
return False
|
||||||
approved = await self._supervise_token_block(flow, request_path, result, slug)
|
approved = await self._supervise_token_block(flow, request_path, result, slug, env)
|
||||||
if not approved:
|
if not approved:
|
||||||
return False # _supervise_token_block wrote the 403 response
|
return False # _supervise_token_block wrote the 403 response
|
||||||
# loop: the approved value is now in safe_tokens; re-scan.
|
# loop: the approved value is now in safe_tokens; re-scan.
|
||||||
|
|
||||||
def _redact_outbound(self, flow: http.HTTPFlow, route: Route) -> bool:
|
def _redact_outbound(
|
||||||
|
self, flow: http.HTTPFlow, route: Route, env: "typing.Mapping[str, str]",
|
||||||
|
) -> bool:
|
||||||
"""Scrub detected tokens (and CRLF injection sequences) from the mutable
|
"""Scrub detected tokens (and CRLF injection sequences) from the mutable
|
||||||
request surfaces (body, headers, path/query) and re-scan. Returns True
|
request surfaces (body, headers, path/query) and re-scan. `env` is the
|
||||||
if the request is now clean; False if a block-severity match remains on
|
per-bottle env overlay. Returns True if the request is now clean; False
|
||||||
a surface redaction cannot rewrite (the hostname) so the caller fails
|
if a block-severity match remains on a surface redaction cannot rewrite
|
||||||
closed."""
|
(the hostname) so the caller fails closed."""
|
||||||
body = flow.request.get_text(strict=False)
|
body = flow.request.get_text(strict=False)
|
||||||
if body:
|
if body:
|
||||||
redacted_body = redact_tokens(body, env=os.environ)
|
redacted_body = redact_tokens(body, env=env)
|
||||||
if redacted_body != body:
|
if redacted_body != body:
|
||||||
flow.request.text = redacted_body
|
flow.request.text = redacted_body
|
||||||
for name, value in list(flow.request.headers.items()):
|
for name, value in list(flow.request.headers.items()):
|
||||||
if name.lower() == "host":
|
if name.lower() == "host":
|
||||||
continue # routing-critical; never a legitimate token
|
continue # routing-critical; never a legitimate token
|
||||||
redacted = strip_crlf(redact_tokens(value, env=os.environ))
|
redacted = strip_crlf(redact_tokens(value, env=env))
|
||||||
if redacted != value:
|
if redacted != value:
|
||||||
flow.request.headers[name] = redacted
|
flow.request.headers[name] = redacted
|
||||||
redacted_path = strip_crlf(redact_tokens(flow.request.path, env=os.environ))
|
redacted_path = strip_crlf(redact_tokens(flow.request.path, env=env))
|
||||||
if redacted_path != flow.request.path:
|
if redacted_path != flow.request.path:
|
||||||
flow.request.path = redacted_path
|
flow.request.path = redacted_path
|
||||||
|
|
||||||
@@ -419,7 +430,7 @@ class EgressAddon:
|
|||||||
crlf_text = build_outbound_scan_text(
|
crlf_text = build_outbound_scan_text(
|
||||||
flow.request.pretty_host, request_path, query, headers, "",
|
flow.request.pretty_host, request_path, query, headers, "",
|
||||||
)
|
)
|
||||||
result = scan_outbound(route, scan_text, os.environ, crlf_text=crlf_text)
|
result = scan_outbound(route, scan_text, env, crlf_text=crlf_text)
|
||||||
return result is None or result.severity != "block"
|
return result is None or result.severity != "block"
|
||||||
|
|
||||||
async def _supervise_token_block(
|
async def _supervise_token_block(
|
||||||
@@ -428,20 +439,22 @@ class EgressAddon:
|
|||||||
request_path: str,
|
request_path: str,
|
||||||
result: ScanResult,
|
result: ScanResult,
|
||||||
slug: str,
|
slug: str,
|
||||||
|
env: "typing.Mapping[str, str]",
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""Route a token DLP block to the operator's supervisor queue and wait.
|
"""Route a token DLP block to the operator's supervisor queue and wait.
|
||||||
|
|
||||||
`slug` attributes the proposal to the calling bottle (its own queue +
|
`slug` attributes the proposal to the calling bottle (its own queue +
|
||||||
safelist) — in the shared gateway this is what keeps one bottle's
|
safelist) — in the shared gateway this is what keeps one bottle's
|
||||||
approval from unblocking another's request. Returns True if the operator
|
approval from unblocking another's request. `env` is the per-bottle env
|
||||||
approved (the matched value is added to that bottle's safelist and the
|
overlay used to redact secrets from the proposal text. Returns True if
|
||||||
caller re-scans); False if the request must be blocked (a 403 response
|
the operator approved (the matched value is added to that bottle's
|
||||||
has been written to `flow`)."""
|
safelist and the caller re-scans); False if the request must be blocked
|
||||||
|
(a 403 response has been written to `flow`)."""
|
||||||
host = flow.request.pretty_host
|
host = flow.request.pretty_host
|
||||||
payload = build_token_allow_payload(
|
payload = build_token_allow_payload(
|
||||||
redact_tokens(host, env=os.environ),
|
redact_tokens(host, env=env),
|
||||||
flow.request.method,
|
flow.request.method,
|
||||||
redact_tokens(request_path, env=os.environ),
|
redact_tokens(request_path, env=env),
|
||||||
result,
|
result,
|
||||||
)
|
)
|
||||||
proposal = _sv.Proposal.new(
|
proposal = _sv.Proposal.new(
|
||||||
|
|||||||
@@ -450,28 +450,31 @@ def resolve_client_config(
|
|||||||
|
|
||||||
class ContextResolverLike(typing.Protocol):
|
class ContextResolverLike(typing.Protocol):
|
||||||
"""The bit of `policy_resolver.PolicyResolver` `resolve_client_context`
|
"""The bit of `policy_resolver.PolicyResolver` `resolve_client_context`
|
||||||
needs — one round-trip returning both policy and bottle id."""
|
needs — one round-trip returning policy, bottle id, and auth tokens."""
|
||||||
|
|
||||||
def resolve_policy_and_bottle_id(
|
def resolve_policy_and_bottle_id(
|
||||||
self, source_ip: str, identity_token: str = ...,
|
self, source_ip: str, identity_token: str = ...,
|
||||||
) -> "tuple[str | None, str | None]":
|
) -> "tuple[str | None, str | None, dict[str, str]]":
|
||||||
...
|
...
|
||||||
|
|
||||||
|
|
||||||
def resolve_client_context(
|
def resolve_client_context(
|
||||||
resolver: ContextResolverLike, client_ip: str, identity_token: str = "",
|
resolver: ContextResolverLike, client_ip: str, identity_token: str = "",
|
||||||
) -> "tuple[Config, str]":
|
) -> "tuple[Config, str, dict[str, str]]":
|
||||||
"""The calling client's `(Config, bottle_id)` in one round-trip —
|
"""The calling client's `(Config, bottle_id, tokens)` in one round-trip —
|
||||||
**fail-closed**. The Config follows `resolve_client_config`'s deny-all
|
**fail-closed**. The Config follows `resolve_client_config`'s deny-all
|
||||||
rules; the bottle id is `""` whenever unattributed or the orchestrator
|
rules; the bottle id is `""` whenever unattributed or the orchestrator
|
||||||
errored, which the caller treats as "supervise unavailable for this
|
errored (caller treats as "supervise unavailable", never another bottle's
|
||||||
bottle" (never another bottle's queue). One `/resolve` keys both the
|
queue); `tokens` are the per-bottle upstream auth values the addon injects.
|
||||||
per-request egress policy and the per-bottle supervise queue + safelist."""
|
One `/resolve` keys the egress policy, the supervise queue + safelist, and
|
||||||
|
auth injection."""
|
||||||
try:
|
try:
|
||||||
policy, bottle_id = resolver.resolve_policy_and_bottle_id(client_ip, identity_token)
|
policy, bottle_id, tokens = resolver.resolve_policy_and_bottle_id(
|
||||||
|
client_ip, identity_token,
|
||||||
|
)
|
||||||
except Exception: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
except Exception: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
||||||
return Config(routes=()), "" # orchestrator unreachable/errored → deny
|
return Config(routes=()), "", {} # orchestrator unreachable/errored → deny
|
||||||
return _config_from_policy(policy), (bottle_id or "")
|
return _config_from_policy(policy), (bottle_id or ""), tokens
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -90,14 +90,18 @@ class OrchestratorClient:
|
|||||||
image_ref: str = "",
|
image_ref: str = "",
|
||||||
metadata: str = "",
|
metadata: str = "",
|
||||||
policy: str = "",
|
policy: str = "",
|
||||||
|
tokens: dict[str, str] | None = None,
|
||||||
) -> RegisteredBottle:
|
) -> RegisteredBottle:
|
||||||
"""Register a bottle and broker its launch (`POST /bottles`). Returns
|
"""Register a bottle and broker its launch (`POST /bottles`). `tokens`
|
||||||
its minted id + identity token."""
|
are the per-bottle egress auth values (env_name -> value) the
|
||||||
|
orchestrator holds in memory for the gateway to inject. Returns the
|
||||||
|
minted id + identity token."""
|
||||||
payload = self._ok("POST", "/bottles", {
|
payload = self._ok("POST", "/bottles", {
|
||||||
"source_ip": source_ip,
|
"source_ip": source_ip,
|
||||||
"image_ref": image_ref,
|
"image_ref": image_ref,
|
||||||
"metadata": metadata,
|
"metadata": metadata,
|
||||||
"policy": policy,
|
"policy": policy,
|
||||||
|
"tokens": tokens or {},
|
||||||
})
|
})
|
||||||
bottle_id = payload.get("bottle_id")
|
bottle_id = payload.get("bottle_id")
|
||||||
token = payload.get("identity_token")
|
token = payload.get("identity_token")
|
||||||
|
|||||||
@@ -81,11 +81,16 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches
|
|||||||
image_ref = data.get("image_ref")
|
image_ref = data.get("image_ref")
|
||||||
metadata = data.get("metadata")
|
metadata = data.get("metadata")
|
||||||
policy = data.get("policy")
|
policy = data.get("policy")
|
||||||
|
raw_tokens = data.get("tokens")
|
||||||
|
tokens = {
|
||||||
|
k: v for k, v in raw_tokens.items() if isinstance(k, str) and isinstance(v, str)
|
||||||
|
} if isinstance(raw_tokens, dict) else {}
|
||||||
rec = orch.launch_bottle(
|
rec = orch.launch_bottle(
|
||||||
source_ip,
|
source_ip,
|
||||||
image_ref=image_ref if isinstance(image_ref, str) else "",
|
image_ref=image_ref if isinstance(image_ref, str) else "",
|
||||||
metadata=metadata if isinstance(metadata, str) else "",
|
metadata=metadata if isinstance(metadata, str) else "",
|
||||||
policy=policy if isinstance(policy, str) else "",
|
policy=policy if isinstance(policy, str) else "",
|
||||||
|
tokens=tokens,
|
||||||
)
|
)
|
||||||
return 201, {"bottle_id": rec.bottle_id, "identity_token": rec.identity_token}
|
return 201, {"bottle_id": rec.bottle_id, "identity_token": rec.identity_token}
|
||||||
|
|
||||||
@@ -137,7 +142,13 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches
|
|||||||
rec = orch.resolve(source_ip, token if isinstance(token, str) else "")
|
rec = orch.resolve(source_ip, token if isinstance(token, str) else "")
|
||||||
if rec is None:
|
if rec is None:
|
||||||
return 403, {"error": "unattributed"}
|
return 403, {"error": "unattributed"}
|
||||||
return 200, {"bottle_id": rec.bottle_id, "policy": rec.policy}
|
# tokens are the in-memory per-bottle egress auth values the gateway
|
||||||
|
# injects; served here, never persisted.
|
||||||
|
return 200, {
|
||||||
|
"bottle_id": rec.bottle_id,
|
||||||
|
"policy": rec.policy,
|
||||||
|
"tokens": orch.tokens_for(rec.bottle_id),
|
||||||
|
}
|
||||||
|
|
||||||
return 404, {"error": "not found"}
|
return 404, {"error": "not found"}
|
||||||
|
|
||||||
|
|||||||
@@ -19,10 +19,16 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import abc
|
import abc
|
||||||
import os
|
import os
|
||||||
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ..docker_cmd import run_docker
|
from ..docker_cmd import run_docker
|
||||||
|
|
||||||
|
# The gateway's mitmproxy writes its CA a beat after the container starts, so
|
||||||
|
# reads poll for it rather than assuming it's there on a fresh launch.
|
||||||
|
_CA_POLL_SECONDS = 0.5
|
||||||
|
DEFAULT_CA_TIMEOUT_SECONDS = 30.0
|
||||||
|
|
||||||
GATEWAY_NAME = "bot-bottle-orch-gateway"
|
GATEWAY_NAME = "bot-bottle-orch-gateway"
|
||||||
GATEWAY_LABEL = "bot-bottle-orch-gateway=1"
|
GATEWAY_LABEL = "bot-bottle-orch-gateway=1"
|
||||||
# The single user-defined network the gateway and every agent bottle share.
|
# The single user-defined network the gateway and every agent bottle share.
|
||||||
@@ -138,6 +144,18 @@ class DockerGateway(Gateway):
|
|||||||
])
|
])
|
||||||
return self.name in proc.stdout.split()
|
return self.name in proc.stdout.split()
|
||||||
|
|
||||||
|
def _running_image_is_current(self) -> bool:
|
||||||
|
"""True iff the running gateway was created from the *current*
|
||||||
|
`image_ref`. When `ensure_built` rebuilds the image (a source change),
|
||||||
|
the running container is still the OLD image running the OLD flat
|
||||||
|
daemons — so this is how a rebuild actually takes effect: a mismatch
|
||||||
|
means recreate."""
|
||||||
|
running = run_docker(["docker", "inspect", "--format", "{{.Image}}", self.name])
|
||||||
|
current = run_docker(["docker", "image", "inspect", "--format", "{{.Id}}", self.image_ref])
|
||||||
|
if running.returncode != 0 or current.returncode != 0:
|
||||||
|
return True # can't compare → don't churn a working container
|
||||||
|
return running.stdout.strip() == current.stdout.strip()
|
||||||
|
|
||||||
def _ensure_network(self) -> None:
|
def _ensure_network(self) -> None:
|
||||||
"""Create the shared gateway network if it doesn't exist. Idempotent —
|
"""Create the shared gateway network if it doesn't exist. Idempotent —
|
||||||
a concurrent create loses harmlessly (the loser sees 'already exists').
|
a concurrent create loses harmlessly (the loser sees 'already exists').
|
||||||
@@ -151,11 +169,15 @@ class DockerGateway(Gateway):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def ensure_running(self) -> None:
|
def ensure_running(self) -> None:
|
||||||
if self.is_running():
|
# Recreate when the running container's image is stale (a rebuild),
|
||||||
|
# so source changes to the gateway's flat daemons take effect — not
|
||||||
|
# just when the container is absent.
|
||||||
|
if self.is_running() and self._running_image_is_current():
|
||||||
return
|
return
|
||||||
self._ensure_network()
|
self._ensure_network()
|
||||||
# Clear any stale (stopped) container holding the fixed name, then
|
# Clear any stale (stopped OR outdated-image) container holding the
|
||||||
# start fresh. `rm --force` on an absent name is a tolerated no-op.
|
# fixed name, then start fresh. `rm --force` on an absent name is a
|
||||||
|
# tolerated no-op.
|
||||||
run_docker(["docker", "rm", "--force", self.name])
|
run_docker(["docker", "rm", "--force", self.name])
|
||||||
argv = [
|
argv = [
|
||||||
"docker", "run", "--detach",
|
"docker", "run", "--detach",
|
||||||
@@ -175,17 +197,23 @@ class DockerGateway(Gateway):
|
|||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
raise GatewayError(f"gateway failed to start: {proc.stderr.strip()}")
|
raise GatewayError(f"gateway failed to start: {proc.stderr.strip()}")
|
||||||
|
|
||||||
def ca_cert_pem(self) -> str:
|
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
||||||
"""The gateway's CA certificate (PEM) that agents install to trust its
|
"""The gateway's CA certificate (PEM) that agents install to trust its
|
||||||
TLS interception. Read from the running container; raises if the
|
TLS interception. mitmproxy generates it a moment after the container
|
||||||
gateway hasn't generated it yet (it's created on first mitmproxy
|
starts, so this **polls** for it (up to `timeout`) rather than assuming
|
||||||
start)."""
|
it's already there on a fresh gateway — raising only if it never
|
||||||
proc = run_docker(["docker", "exec", self.name, "cat", GATEWAY_CA_CERT])
|
appears."""
|
||||||
if proc.returncode != 0 or not proc.stdout.strip():
|
deadline = time.monotonic() + timeout
|
||||||
raise GatewayError(
|
while True:
|
||||||
f"gateway CA cert not available: {proc.stderr.strip() or 'empty'}"
|
proc = run_docker(["docker", "exec", self.name, "cat", GATEWAY_CA_CERT])
|
||||||
)
|
if proc.returncode == 0 and proc.stdout.strip():
|
||||||
return proc.stdout
|
return proc.stdout
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
raise GatewayError(
|
||||||
|
f"gateway CA cert not available after {timeout:g}s: "
|
||||||
|
f"{proc.stderr.strip() or 'empty'}"
|
||||||
|
)
|
||||||
|
time.sleep(_CA_POLL_SECONDS)
|
||||||
|
|
||||||
def stop(self) -> None:
|
def stop(self) -> None:
|
||||||
proc = run_docker(["docker", "rm", "--force", self.name])
|
proc = run_docker(["docker", "rm", "--force", self.name])
|
||||||
|
|||||||
@@ -127,13 +127,17 @@ class OrchestratorService:
|
|||||||
gateway are left untouched. Raises `OrchestratorStartError` on
|
gateway are left untouched. Raises `OrchestratorStartError` on
|
||||||
timeout."""
|
timeout."""
|
||||||
gateway = self._gateway()
|
gateway = self._gateway()
|
||||||
gateway.ensure_built() # build the bundle image if missing (both use it)
|
gateway.ensure_built() # rebuild the bundle image on a source change
|
||||||
if self.is_healthy():
|
gateway.ensure_running() # creates the shared network + (re)starts gateway
|
||||||
gateway.ensure_running() # make sure the gateway is up too
|
|
||||||
return self.url
|
|
||||||
|
|
||||||
|
# Always (re)create the orchestrator container. It runs the repo's code
|
||||||
|
# bind-mounted, but the Python process loaded that code at startup and
|
||||||
|
# won't reload — so reusing a healthy-but-stale container would keep
|
||||||
|
# running OLD control-plane code (e.g. dropping the tokens field). Cheap
|
||||||
|
# (~seconds); the registry DB persists and the current launch
|
||||||
|
# re-registers its own in-memory state. (The dedicated orchestrator
|
||||||
|
# image follow-up replaces this with image-staleness detection.)
|
||||||
log.info("starting orchestrator container", context={"name": ORCHESTRATOR_NAME})
|
log.info("starting orchestrator container", context={"name": ORCHESTRATOR_NAME})
|
||||||
gateway.ensure_running() # creates the shared network + gateway
|
|
||||||
self._run_orchestrator_container()
|
self._run_orchestrator_container()
|
||||||
|
|
||||||
deadline = time.monotonic() + startup_timeout
|
deadline = time.monotonic() + startup_timeout
|
||||||
|
|||||||
@@ -149,7 +149,15 @@ class RegistryStore(DbStore):
|
|||||||
policy: str = "",
|
policy: str = "",
|
||||||
) -> BottleRecord:
|
) -> BottleRecord:
|
||||||
"""Register (or replace) a bottle. Mints a bottle_id / identity token
|
"""Register (or replace) a bottle. Mints a bottle_id / identity token
|
||||||
when not supplied. Live — this is the control-plane reload path."""
|
when not supplied. Live — this is the control-plane reload path.
|
||||||
|
|
||||||
|
Supersedes any *other* active bottle at the same source IP first:
|
||||||
|
`by_source_ip` fail-closes on ambiguity (>1 active row → None), so a
|
||||||
|
leftover row at a reused IP — from an untorn-down bottle, crash
|
||||||
|
recovery, or a persisted DB — would otherwise *brick* the new bottle's
|
||||||
|
attribution. The new registration is authoritative for its IP; the
|
||||||
|
stale rows go. INSERT OR REPLACE handles a same-`bottle_id` reload in
|
||||||
|
place (its own row is exempt from the sweep)."""
|
||||||
rec = BottleRecord(
|
rec = BottleRecord(
|
||||||
bottle_id=bottle_id or secrets.token_hex(8),
|
bottle_id=bottle_id or secrets.token_hex(8),
|
||||||
source_ip=source_ip,
|
source_ip=source_ip,
|
||||||
@@ -160,6 +168,11 @@ class RegistryStore(DbStore):
|
|||||||
policy=policy,
|
policy=policy,
|
||||||
)
|
)
|
||||||
with self._connect() as conn:
|
with self._connect() as conn:
|
||||||
|
conn.execute(
|
||||||
|
"DELETE FROM orchestrator_bottles "
|
||||||
|
"WHERE source_ip = ? AND state = 'active' AND bottle_id != ?",
|
||||||
|
(rec.source_ip, rec.bottle_id),
|
||||||
|
)
|
||||||
conn.execute(
|
conn.execute(
|
||||||
"INSERT OR REPLACE INTO orchestrator_bottles "
|
"INSERT OR REPLACE INTO orchestrator_bottles "
|
||||||
"(bottle_id, source_ip, identity_token, state, created_at, metadata, policy) "
|
"(bottle_id, source_ip, identity_token, state, created_at, metadata, policy) "
|
||||||
|
|||||||
@@ -38,6 +38,12 @@ class Orchestrator:
|
|||||||
self._broker = broker
|
self._broker = broker
|
||||||
self._secret = sign_secret
|
self._secret = sign_secret
|
||||||
self._gateway = gateway
|
self._gateway = gateway
|
||||||
|
# Per-bottle egress auth tokens (env_name -> value), keyed by bottle_id.
|
||||||
|
# Held **in memory only** — never written to the registry DB — so the
|
||||||
|
# gateway can inject each bottle's upstream credential without secrets
|
||||||
|
# at rest. Lost on restart (re-launch re-registers them); the future
|
||||||
|
# SecretProvider (#355) replaces this with per-request minting.
|
||||||
|
self._tokens: dict[str, dict[str, str]] = {}
|
||||||
|
|
||||||
def launch_bottle(
|
def launch_bottle(
|
||||||
self,
|
self,
|
||||||
@@ -47,11 +53,14 @@ class Orchestrator:
|
|||||||
slot: int | None = None,
|
slot: int | None = None,
|
||||||
metadata: str = "",
|
metadata: str = "",
|
||||||
policy: str = "",
|
policy: str = "",
|
||||||
|
tokens: dict[str, str] | None = None,
|
||||||
) -> BottleRecord:
|
) -> BottleRecord:
|
||||||
"""Register a bottle (with its gateway policy) and broker its launch.
|
"""Register a bottle (with its gateway policy + in-memory egress auth
|
||||||
Rolls the registry entry back if the launch doesn't take, so a
|
tokens) and broker its launch. Rolls the registry entry back if the
|
||||||
failure leaves no orphan."""
|
launch doesn't take, so a failure leaves no orphan."""
|
||||||
rec = self.registry.register(source_ip, metadata=metadata, policy=policy)
|
rec = self.registry.register(source_ip, metadata=metadata, policy=policy)
|
||||||
|
if tokens:
|
||||||
|
self._tokens[rec.bottle_id] = dict(tokens)
|
||||||
req = LaunchRequest(
|
req = LaunchRequest(
|
||||||
op="launch",
|
op="launch",
|
||||||
bottle_id=rec.bottle_id,
|
bottle_id=rec.bottle_id,
|
||||||
@@ -66,6 +75,7 @@ class Orchestrator:
|
|||||||
finally:
|
finally:
|
||||||
if not launched:
|
if not launched:
|
||||||
self.registry.deregister(rec.bottle_id)
|
self.registry.deregister(rec.bottle_id)
|
||||||
|
self._tokens.pop(rec.bottle_id, None)
|
||||||
return rec
|
return rec
|
||||||
|
|
||||||
def teardown_bottle(self, bottle_id: str) -> bool:
|
def teardown_bottle(self, bottle_id: str) -> bool:
|
||||||
@@ -76,8 +86,15 @@ class Orchestrator:
|
|||||||
req = LaunchRequest(op="teardown", bottle_id=bottle_id, source_ip=rec.source_ip)
|
req = LaunchRequest(op="teardown", bottle_id=bottle_id, source_ip=rec.source_ip)
|
||||||
self._broker.submit(sign_request(req, self._secret))
|
self._broker.submit(sign_request(req, self._secret))
|
||||||
self.registry.deregister(bottle_id)
|
self.registry.deregister(bottle_id)
|
||||||
|
self._tokens.pop(bottle_id, None)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
def tokens_for(self, bottle_id: str) -> dict[str, str]:
|
||||||
|
"""The bottle's in-memory egress auth tokens (env_name -> value), or
|
||||||
|
empty. The gateway injects these per request; they are never
|
||||||
|
persisted."""
|
||||||
|
return dict(self._tokens.get(bottle_id, {}))
|
||||||
|
|
||||||
def attribute(self, source_ip: str, identity_token: str) -> BottleRecord | None:
|
def attribute(self, source_ip: str, identity_token: str) -> BottleRecord | None:
|
||||||
"""Fail-closed attribution (delegates to the registry)."""
|
"""Fail-closed attribution (delegates to the registry)."""
|
||||||
return self.registry.attribute(source_ip, identity_token)
|
return self.registry.attribute(source_ip, identity_token)
|
||||||
|
|||||||
@@ -102,21 +102,26 @@ class PolicyResolver:
|
|||||||
|
|
||||||
def resolve_policy_and_bottle_id(
|
def resolve_policy_and_bottle_id(
|
||||||
self, source_ip: str, identity_token: str = "",
|
self, source_ip: str, identity_token: str = "",
|
||||||
) -> tuple[str | None, str | None]:
|
) -> tuple[str | None, str | None, dict[str, str]]:
|
||||||
"""Both the policy blob and the bottle id in a single `/resolve` — so
|
"""The policy blob, bottle id, **and per-bottle egress auth tokens** in
|
||||||
a caller that needs each (the egress addon: policy for routing, bottle
|
a single `/resolve` — so the egress addon gets everything it needs
|
||||||
id to key the calling bottle's supervise queue + safelist) makes one
|
(policy for routing, bottle id for the supervise queue/safelist, tokens
|
||||||
round-trip, not two. Returns `(None, None)` when unattributed (a clean
|
to inject upstream auth) in one round-trip. Returns `(None, None, {})`
|
||||||
`403`). Raises `PolicyResolveError` if the orchestrator can't be
|
when unattributed (a clean `403`). Raises `PolicyResolveError` if the
|
||||||
reached, so the caller still fails closed."""
|
orchestrator can't be reached, so the caller still fails closed."""
|
||||||
payload = self._post_resolve(source_ip, identity_token)
|
payload = self._post_resolve(source_ip, identity_token)
|
||||||
if payload is None:
|
if payload is None:
|
||||||
return None, None
|
return None, None, {}
|
||||||
policy = payload.get("policy")
|
policy = payload.get("policy")
|
||||||
bottle_id = payload.get("bottle_id")
|
bottle_id = payload.get("bottle_id")
|
||||||
|
raw = payload.get("tokens")
|
||||||
|
tokens = {
|
||||||
|
k: v for k, v in raw.items() if isinstance(k, str) and isinstance(v, str)
|
||||||
|
} if isinstance(raw, dict) else {}
|
||||||
return (
|
return (
|
||||||
policy if isinstance(policy, str) else "",
|
policy if isinstance(policy, str) else "",
|
||||||
bottle_id if isinstance(bottle_id, str) and bottle_id else None,
|
bottle_id if isinstance(bottle_id, str) and bottle_id else None,
|
||||||
|
tokens,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,25 @@ the biggest credential risk).
|
|||||||
|
|
||||||
## Summary
|
## Summary
|
||||||
|
|
||||||
|
> **Status — implemented (2026-07-14).** This note's recommendation
|
||||||
|
> shipped, so the "today" / "in-flight" tenses below are now historical
|
||||||
|
> (kept as the reasoning that led here). Current reality: bot-bottle no
|
||||||
|
> longer injects raw provider tokens into the agent. The agent's environ
|
||||||
|
> carries only a **placeholder** (e.g.
|
||||||
|
> `CLAUDE_CODE_OAUTH_TOKEN=egress-placeholder`); the real token is held
|
||||||
|
> in the egress sidecar and injected as the upstream `Authorization`
|
||||||
|
> header. Rather than adding a *separate* in-container reverse proxy (as
|
||||||
|
> proposed below), credential injection was folded into the **existing
|
||||||
|
> pipelock/mitmproxy egress firewall**: an `EgressRoute` carries
|
||||||
|
> `auth_scheme` + `token_ref`, the host token is forwarded into the
|
||||||
|
> sidecar addon under an `EGRESS_TOKEN_N` slot, and the addon rewrites
|
||||||
|
> `Authorization` on matching routes — **one MITM, not two** (this
|
||||||
|
> resolves the Infisical-doubling concern noted later). The same
|
||||||
|
> mechanism now covers Codex and Pi provider tokens and git-host PATs.
|
||||||
|
> A `printenv` inside the bottle yields the placeholder, not the secret.
|
||||||
|
> For current wiring see `bot_bottle/egress.py` and
|
||||||
|
> `bot_bottle/contrib/*/agent_provider.py`.
|
||||||
|
|
||||||
Today every bot-bottle agent gets `CLAUDE_CODE_OAUTH_TOKEN` (and
|
Today every bot-bottle agent gets `CLAUDE_CODE_OAUTH_TOKEN` (and
|
||||||
any `bottle.env` secrets like a Gitea PAT) injected as env vars,
|
any `bottle.env` secrets like a Gitea PAT) injected as env vars,
|
||||||
which means the agent process can read them with `printenv` or
|
which means the agent process can read them with `printenv` or
|
||||||
@@ -95,7 +114,9 @@ The remaining credible designs reduce to three:
|
|||||||
|
|
||||||
### Anthropic / Claude Code
|
### Anthropic / Claude Code
|
||||||
|
|
||||||
**Today's wiring** (`bot_bottle/cli/start.py`): the host's
|
**Original wiring** (pre-gateway; **superseded 2026-07-14** — see the
|
||||||
|
Status note in the Summary; the token now lives in the egress sidecar
|
||||||
|
and the agent gets a placeholder): the host's
|
||||||
`BOT_BOTTLE_CLAUDE_OAUTH_TOKEN` is forwarded into the bottle as
|
`BOT_BOTTLE_CLAUDE_OAUTH_TOKEN` is forwarded into the bottle as
|
||||||
`CLAUDE_CODE_OAUTH_TOKEN` via `docker run -e CLAUDE_CODE_OAUTH_TOKEN`
|
`CLAUDE_CODE_OAUTH_TOKEN` via `docker run -e CLAUDE_CODE_OAUTH_TOKEN`
|
||||||
(no `=value`, so the value never lands on argv — good). Inside the
|
(no `=value`, so the value never lands on argv — good). Inside the
|
||||||
@@ -332,6 +353,12 @@ exposed.
|
|||||||
|
|
||||||
## Recommended path forward
|
## Recommended path forward
|
||||||
|
|
||||||
|
> **Mostly implemented (2026-07-14).** Steps 1–3 shipped — the token
|
||||||
|
> injection moved into the egress sidecar (folded into pipelock rather
|
||||||
|
> than a standalone reverse proxy) and generalized to Codex/Pi/git-host
|
||||||
|
> tokens. Steps 4–5 (issuance-side scope narrowing, per-route allowlists)
|
||||||
|
> remain good hygiene. See the Status note in the Summary.
|
||||||
|
|
||||||
In priority order:
|
In priority order:
|
||||||
|
|
||||||
1. **In-container reverse proxy holding `CLAUDE_CODE_OAUTH_TOKEN`.**
|
1. **In-container reverse proxy holding `CLAUDE_CODE_OAUTH_TOKEN`.**
|
||||||
|
|||||||
@@ -103,7 +103,19 @@ manifest merge.
|
|||||||
has no equivalent. (3) *Trust posture:* OneCLI's managed tier reintroduces a
|
has no equivalent. (3) *Trust posture:* OneCLI's managed tier reintroduces a
|
||||||
third-party credential custodian, whereas bot-bottle's OSS-runtime +
|
third-party credential custodian, whereas bot-bottle's OSS-runtime +
|
||||||
paid-control-plane split keeps custody inside the operator's own boundary —
|
paid-control-plane split keeps custody inside the operator's own boundary —
|
||||||
the stronger story for the security-minded self-hoster. **Tactical read:**
|
the stronger story for the security-minded self-hoster. (4) *Runs inside your
|
||||||
|
network boundary — local/internal reach.* Because bot-bottle executes the
|
||||||
|
agent on your own host (homelab, corporate LAN, a Tailnet) and egress is a
|
||||||
|
manifest field, giving an agent *scoped* access to **internal** resources — a
|
||||||
|
private Gitea, a LAN database, a Tailscale node — is just another egress-route
|
||||||
|
line, not a networking project (the same move an operator already makes to
|
||||||
|
reach their Tailscale services). OneCLI's OSS core can self-host too, but it's
|
||||||
|
a credential *broker* for outbound API calls, not an agent runtime, and its
|
||||||
|
managed tier + 50+ integrations are oriented at public SaaS — it doesn't put
|
||||||
|
the agent behind your firewall for you. This is a reach advantage, distinct
|
||||||
|
from the isolation ones above, and it's a wedge cloud-first agent products
|
||||||
|
(Devin, Copilot Workspace, OneCLI Cloud) structurally can't match. **Tactical
|
||||||
|
read:**
|
||||||
adopt OneCLI's OSS core for the credential slice if building is undesirable
|
adopt OneCLI's OSS core for the credential slice if building is undesirable
|
||||||
(it's mature now); don't build atop its managed tier (competitor, not
|
(it's mature now); don't build atop its managed tier (competitor, not
|
||||||
dependency); re-position bot-bottle on isolation + fleet + self-hosted custody
|
dependency); re-position bot-bottle on isolation + fleet + self-hosted custody
|
||||||
|
|||||||
@@ -0,0 +1,161 @@
|
|||||||
|
"""Integration: two bottles share one gateway; source-IP attribution keeps
|
||||||
|
their egress tokens *and* allowlists isolated (PRD 0070 multi-tenancy).
|
||||||
|
|
||||||
|
The whole premise of the consolidation is one shared gateway for every
|
||||||
|
bottle, isolated only by the unspoofable source IP. A single-bottle test
|
||||||
|
can't catch cross-bottle token bleed or an allowlist leak — this brings up
|
||||||
|
the real orchestrator + gateway and drives two bottles through the one
|
||||||
|
gateway to prove each gets exactly its own token and its own routes.
|
||||||
|
|
||||||
|
Gated on a reachable Docker daemon and builds the bundle image, so it runs
|
||||||
|
with the other docker integration tests, not in unit CI. Uses the real
|
||||||
|
fixed gateway/orchestrator names (that's what it's testing), so it can't run
|
||||||
|
concurrently with a live per-host gateway; it points the orchestrator at a
|
||||||
|
throwaway BOT_BOTTLE_ROOT for a clean registry and tears everything down.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from bot_bottle.backend.docker.consolidated_launch import (
|
||||||
|
_network_cidr,
|
||||||
|
_network_container_ips,
|
||||||
|
)
|
||||||
|
from bot_bottle.backend.docker.egress import EGRESS_PORT
|
||||||
|
from bot_bottle.backend.docker.gateway_net import next_free_ip
|
||||||
|
from bot_bottle.orchestrator.client import OrchestratorClient
|
||||||
|
from bot_bottle.orchestrator.gateway import GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK
|
||||||
|
from bot_bottle.orchestrator.lifecycle import OrchestratorService
|
||||||
|
from tests._docker import skip_unless_docker
|
||||||
|
|
||||||
|
# One upstream reachable under two names; reflects the Authorization header so
|
||||||
|
# the probe can see exactly what the gateway injected (or didn't).
|
||||||
|
_ECHO_NAME = "bot-bottle-mtitest-echo"
|
||||||
|
_ECHO_ALIASES = ("echo-shared", "echo-bonly")
|
||||||
|
_ECHO_SRC = (
|
||||||
|
"from http.server import BaseHTTPRequestHandler, HTTPServer\n"
|
||||||
|
"class H(BaseHTTPRequestHandler):\n"
|
||||||
|
" def do_GET(s):\n"
|
||||||
|
" a=s.headers.get('Authorization','NONE'); s.send_response(200); s.end_headers()\n"
|
||||||
|
" s.wfile.write(('AUTH='+a).encode())\n"
|
||||||
|
" def log_message(s,*a): pass\n"
|
||||||
|
"HTTPServer(('0.0.0.0',80),H).serve_forever()\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
# A: echo-shared only, authed. B: echo-shared authed + echo-bonly open.
|
||||||
|
_POLICY_A = (
|
||||||
|
'routes:\n'
|
||||||
|
' - host: echo-shared\n'
|
||||||
|
' auth_scheme: "Bearer"\n'
|
||||||
|
' token_env: "EGRESS_TOKEN_0"\n'
|
||||||
|
)
|
||||||
|
_POLICY_B = _POLICY_A + ' - host: echo-bonly\n'
|
||||||
|
_TOKEN_A = "sk-AAA-alice"
|
||||||
|
_TOKEN_B = "sk-BBB-bob"
|
||||||
|
|
||||||
|
# Client probe: open http://<host>/ through the gateway proxy from a container
|
||||||
|
# pinned to the bottle's source IP, printing "<status> <body>". Uses only the
|
||||||
|
# bundle image's python3 — no dependency on the agent image.
|
||||||
|
_PROBE_SRC = (
|
||||||
|
"import sys,urllib.request,urllib.error\n"
|
||||||
|
"op=urllib.request.build_opener(urllib.request.ProxyHandler({'http':sys.argv[1]}))\n"
|
||||||
|
"try:\n"
|
||||||
|
" r=op.open('http://'+sys.argv[2]+'/',timeout=8)\n"
|
||||||
|
" sys.stdout.write(str(r.status)+' '+r.read().decode())\n"
|
||||||
|
"except urllib.error.HTTPError as e:\n"
|
||||||
|
" sys.stdout.write(str(e.code)+' '+e.read().decode())\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@skip_unless_docker()
|
||||||
|
@unittest.skipIf(
|
||||||
|
os.environ.get("GITEA_ACTIONS") == "true",
|
||||||
|
"skipped under act_runner: the orchestrator container bind-mounts the repo "
|
||||||
|
"path into a container on the socket-shared host daemon, which can't see the "
|
||||||
|
"runner's /workspace — same host-bind-mount constraint as the other "
|
||||||
|
"bottle-bringup integration tests",
|
||||||
|
)
|
||||||
|
class TestMultitenantIsolation(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self._tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self._tmp.cleanup)
|
||||||
|
# Throwaway root → a clean registry DB, independent of the host's.
|
||||||
|
self.svc = OrchestratorService(host_root=Path(self._tmp.name))
|
||||||
|
self.addCleanup(self._teardown_docker)
|
||||||
|
# ensure_running builds the bundle image (slow on a cold cache) and
|
||||||
|
# brings up the shared network + gateway + orchestrator.
|
||||||
|
url = self.svc.ensure_running(startup_timeout=300)
|
||||||
|
self.client = OrchestratorClient(url)
|
||||||
|
self.gw_ip = self._container_ip(GATEWAY_NAME)
|
||||||
|
self._run_echo()
|
||||||
|
|
||||||
|
def _teardown_docker(self) -> None:
|
||||||
|
subprocess.run(["docker", "rm", "--force", _ECHO_NAME],
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
|
||||||
|
self.svc.stop()
|
||||||
|
subprocess.run(["docker", "network", "rm", GATEWAY_NETWORK],
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
|
||||||
|
# The orchestrator container wrote the registry DB as root into the
|
||||||
|
# throwaway root; chown it back so the (non-root) tempdir cleanup can
|
||||||
|
# remove it.
|
||||||
|
subprocess.run(
|
||||||
|
["docker", "run", "--rm", "-v", f"{self._tmp.name}:/r",
|
||||||
|
"--entrypoint", "chown", GATEWAY_IMAGE, "-R",
|
||||||
|
f"{os.getuid()}:{os.getgid()}", "/r"],
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _container_ip(name: str) -> str:
|
||||||
|
proc = subprocess.run(
|
||||||
|
["docker", "inspect", "-f",
|
||||||
|
'{{(index .NetworkSettings.Networks "' + GATEWAY_NETWORK + '").IPAddress}}', name],
|
||||||
|
stdout=subprocess.PIPE, text=True, check=True,
|
||||||
|
)
|
||||||
|
return proc.stdout.strip()
|
||||||
|
|
||||||
|
def _run_echo(self) -> None:
|
||||||
|
argv = ["docker", "run", "--detach", "--name", _ECHO_NAME,
|
||||||
|
"--network", GATEWAY_NETWORK]
|
||||||
|
for alias in _ECHO_ALIASES:
|
||||||
|
argv += ["--network-alias", alias]
|
||||||
|
argv += ["--entrypoint", "python3", GATEWAY_IMAGE, "-c", _ECHO_SRC]
|
||||||
|
proc = subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||||
|
text=True, check=False)
|
||||||
|
self.assertEqual(0, proc.returncode, f"echo upstream failed: {proc.stderr}")
|
||||||
|
|
||||||
|
def _free_ip(self, extra_taken: list[str]) -> str:
|
||||||
|
taken = _network_container_ips(GATEWAY_NETWORK) + extra_taken
|
||||||
|
return next_free_ip(_network_cidr(GATEWAY_NETWORK), taken)
|
||||||
|
|
||||||
|
def _probe(self, source_ip: str, host: str) -> str:
|
||||||
|
proc = subprocess.run(
|
||||||
|
["docker", "run", "--rm", "--network", GATEWAY_NETWORK, "--ip", source_ip,
|
||||||
|
"--entrypoint", "python3", GATEWAY_IMAGE, "-c", _PROBE_SRC,
|
||||||
|
f"http://{self.gw_ip}:{EGRESS_PORT}", host],
|
||||||
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False, timeout=90,
|
||||||
|
)
|
||||||
|
return proc.stdout.strip()
|
||||||
|
|
||||||
|
def test_two_bottles_share_gateway_with_isolated_tokens_and_allowlists(self) -> None:
|
||||||
|
ip_a = self._free_ip([])
|
||||||
|
ip_b = self._free_ip([ip_a])
|
||||||
|
self.client.register_bottle(ip_a, policy=_POLICY_A, tokens={"EGRESS_TOKEN_0": _TOKEN_A})
|
||||||
|
self.client.register_bottle(ip_b, policy=_POLICY_B, tokens={"EGRESS_TOKEN_0": _TOKEN_B})
|
||||||
|
|
||||||
|
# Each bottle gets its OWN token injected on the shared route — no bleed.
|
||||||
|
self.assertEqual(f"200 AUTH=Bearer {_TOKEN_A}", self._probe(ip_a, "echo-shared"))
|
||||||
|
self.assertEqual(f"200 AUTH=Bearer {_TOKEN_B}", self._probe(ip_b, "echo-shared"))
|
||||||
|
|
||||||
|
# Allowlist is per-bottle: echo-bonly is only in B's policy.
|
||||||
|
self.assertTrue(self._probe(ip_a, "echo-bonly").startswith("403"), # fail-closed for A
|
||||||
|
"A reached a host outside its allowlist")
|
||||||
|
self.assertEqual("200 AUTH=NONE", self._probe(ip_b, "echo-bonly")) # allowed, unauthed for B
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -40,7 +40,7 @@ class TestGetBottleBackend(unittest.TestCase):
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
with patch.dict(os.environ, {}, clear=True), \
|
with patch.dict(os.environ, {}, clear=True), \
|
||||||
patch.object(backend_mod, "_BACKENDS", {
|
patch.object(backend_mod, "_backends", {
|
||||||
"macos-container": _FakeBackend(),
|
"macos-container": _FakeBackend(),
|
||||||
"docker": _FakeBackend(),
|
"docker": _FakeBackend(),
|
||||||
}):
|
}):
|
||||||
@@ -61,7 +61,7 @@ class TestGetBottleBackend(unittest.TestCase):
|
|||||||
with patch.dict(os.environ, {}, clear=True), \
|
with patch.dict(os.environ, {}, clear=True), \
|
||||||
patch.object(backend_mod.FirecrackerBottleBackend,
|
patch.object(backend_mod.FirecrackerBottleBackend,
|
||||||
"is_host_capable", classmethod(lambda cls: False)), \
|
"is_host_capable", classmethod(lambda cls: False)), \
|
||||||
patch.object(backend_mod, "_BACKENDS", {
|
patch.object(backend_mod, "_backends", {
|
||||||
"macos-container": _FakeBackend("macos-container", False),
|
"macos-container": _FakeBackend("macos-container", False),
|
||||||
"docker": _FakeBackend("docker", True),
|
"docker": _FakeBackend("docker", True),
|
||||||
}):
|
}):
|
||||||
@@ -83,7 +83,7 @@ class TestGetBottleBackend(unittest.TestCase):
|
|||||||
with patch.dict(os.environ, {}, clear=True), \
|
with patch.dict(os.environ, {}, clear=True), \
|
||||||
patch.object(backend_mod.FirecrackerBottleBackend,
|
patch.object(backend_mod.FirecrackerBottleBackend,
|
||||||
"is_host_capable", classmethod(lambda cls: True)), \
|
"is_host_capable", classmethod(lambda cls: True)), \
|
||||||
patch.object(backend_mod, "_BACKENDS", {
|
patch.object(backend_mod, "_backends", {
|
||||||
"macos-container": _FakeBackend("macos-container", False),
|
"macos-container": _FakeBackend("macos-container", False),
|
||||||
"firecracker": _FakeBackend("firecracker", False),
|
"firecracker": _FakeBackend("firecracker", False),
|
||||||
"docker": _FakeBackend("docker", True),
|
"docker": _FakeBackend("docker", True),
|
||||||
@@ -133,7 +133,7 @@ class TestEnumerateActiveAgents(unittest.TestCase):
|
|||||||
return self._items
|
return self._items
|
||||||
|
|
||||||
with patch.object(
|
with patch.object(
|
||||||
backend_mod, "_BACKENDS",
|
backend_mod, "_backends",
|
||||||
{"docker": _FakeBackend([a]), "firecracker": _FakeBackend([b])},
|
{"docker": _FakeBackend([a]), "firecracker": _FakeBackend([b])},
|
||||||
):
|
):
|
||||||
self.assertEqual([a, b], enumerate_active_agents())
|
self.assertEqual([a, b], enumerate_active_agents())
|
||||||
@@ -167,7 +167,7 @@ class TestEnumerateActiveAgents(unittest.TestCase):
|
|||||||
return self._items
|
return self._items
|
||||||
|
|
||||||
with patch.object(
|
with patch.object(
|
||||||
backend_mod, "_BACKENDS",
|
backend_mod, "_backends",
|
||||||
{
|
{
|
||||||
"docker": _FakeBackend([newer, tie_b]),
|
"docker": _FakeBackend([newer, tie_b]),
|
||||||
"firecracker": _FakeBackend([missing_metadata, tie_a]),
|
"firecracker": _FakeBackend([missing_metadata, tie_a]),
|
||||||
@@ -187,7 +187,7 @@ class TestEnumerateActiveAgents(unittest.TestCase):
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
with patch.object(
|
with patch.object(
|
||||||
backend_mod, "_BACKENDS",
|
backend_mod, "_backends",
|
||||||
{"docker": _FakeBackend(), "firecracker": _FakeBackend()},
|
{"docker": _FakeBackend(), "firecracker": _FakeBackend()},
|
||||||
):
|
):
|
||||||
self.assertEqual([], enumerate_active_agents())
|
self.assertEqual([], enumerate_active_agents())
|
||||||
@@ -218,7 +218,7 @@ class TestEnumerateActiveAgents(unittest.TestCase):
|
|||||||
return self._items
|
return self._items
|
||||||
|
|
||||||
with patch.object(
|
with patch.object(
|
||||||
backend_mod, "_BACKENDS",
|
backend_mod, "_backends",
|
||||||
{
|
{
|
||||||
"docker": _FakeBackend([present], available=True),
|
"docker": _FakeBackend([present], available=True),
|
||||||
"firecracker": _FakeBackend([hidden], available=False),
|
"firecracker": _FakeBackend([hidden], available=False),
|
||||||
@@ -234,7 +234,7 @@ class TestHasBackend(unittest.TestCase):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
with patch.object(
|
with patch.object(
|
||||||
backend_mod, "_BACKENDS", {"docker": _FakeBackend()},
|
backend_mod, "_backends", {"docker": _FakeBackend()},
|
||||||
):
|
):
|
||||||
from bot_bottle.backend import has_backend
|
from bot_bottle.backend import has_backend
|
||||||
self.assertFalse(has_backend("docker"))
|
self.assertFalse(has_backend("docker"))
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ def _fail(stderr: str = "boom") -> subprocess.CompletedProcess: # type: ignore
|
|||||||
class TestCommitContainer(unittest.TestCase):
|
class TestCommitContainer(unittest.TestCase):
|
||||||
def test_runs_docker_commit(self):
|
def test_runs_docker_commit(self):
|
||||||
with patch.object(
|
with patch.object(
|
||||||
docker_mod.subprocess, "run", return_value=_ok(),
|
docker_mod, "run_docker", return_value=_ok(),
|
||||||
) as run, patch.object(docker_mod, "info"):
|
) as run, patch.object(docker_mod, "info"):
|
||||||
docker_mod.commit_container(
|
docker_mod.commit_container(
|
||||||
"bot-bottle-dev-abc12",
|
"bot-bottle-dev-abc12",
|
||||||
@@ -47,7 +47,7 @@ class TestCommitContainer(unittest.TestCase):
|
|||||||
|
|
||||||
def test_dies_on_docker_commit_failure(self):
|
def test_dies_on_docker_commit_failure(self):
|
||||||
with patch.object(
|
with patch.object(
|
||||||
docker_mod.subprocess, "run", return_value=_fail("No such container"),
|
docker_mod, "run_docker", return_value=_fail("No such container"),
|
||||||
), patch.object(
|
), patch.object(
|
||||||
docker_mod, "die", side_effect=SystemExit("die"),
|
docker_mod, "die", side_effect=SystemExit("die"),
|
||||||
) as die:
|
) as die:
|
||||||
@@ -58,7 +58,7 @@ class TestCommitContainer(unittest.TestCase):
|
|||||||
|
|
||||||
def test_die_message_includes_image_tag(self):
|
def test_die_message_includes_image_tag(self):
|
||||||
with patch.object(
|
with patch.object(
|
||||||
docker_mod.subprocess, "run", return_value=_fail("boom"),
|
docker_mod, "run_docker", return_value=_fail("boom"),
|
||||||
), patch.object(
|
), patch.object(
|
||||||
docker_mod, "die", side_effect=SystemExit("die"),
|
docker_mod, "die", side_effect=SystemExit("die"),
|
||||||
) as die:
|
) as die:
|
||||||
|
|||||||
@@ -233,16 +233,19 @@ class _CtxResolver:
|
|||||||
"""Fake orchestrator resolver: maps source IP -> bottle id, and grants the
|
"""Fake orchestrator resolver: maps source IP -> bottle id, and grants the
|
||||||
same allow-list to any attributed bottle (unattributed -> deny)."""
|
same allow-list to any attributed bottle (unattributed -> deny)."""
|
||||||
|
|
||||||
def __init__(self, ip_to_bottle: dict[str, str]) -> None:
|
def __init__(
|
||||||
|
self, ip_to_bottle: dict[str, str], tokens: dict[str, str] | None = None,
|
||||||
|
) -> None:
|
||||||
self._map = ip_to_bottle
|
self._map = ip_to_bottle
|
||||||
|
self._tokens = tokens or {}
|
||||||
|
|
||||||
def resolve_policy_and_bottle_id(
|
def resolve_policy_and_bottle_id(
|
||||||
self, source_ip: str, identity_token: str = "",
|
self, source_ip: str, identity_token: str = "",
|
||||||
) -> tuple[str | None, str | None]:
|
) -> tuple[str | None, str | None, dict[str, str]]:
|
||||||
del identity_token
|
del identity_token
|
||||||
bottle_id = self._map.get(source_ip)
|
bottle_id = self._map.get(source_ip)
|
||||||
policy = "routes:\n - host: api.example.com\n" if bottle_id else None
|
policy = "routes:\n - host: api.example.com\n" if bottle_id else None
|
||||||
return policy, bottle_id
|
return policy, bottle_id, (self._tokens if bottle_id else {})
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
@@ -803,6 +806,46 @@ class TestSuperviseMultiTenant(unittest.TestCase):
|
|||||||
_run_request(addon, flow)
|
_run_request(addon, flow)
|
||||||
self.assertEqual(["bottle-b"], seen) # proposal keyed by the resolved bottle
|
self.assertEqual(["bottle-b"], seen) # proposal keyed by the resolved bottle
|
||||||
|
|
||||||
|
def test_auth_token_injected_from_resolved_tokens(self) -> None:
|
||||||
|
# The bottle's upstream token comes from /resolve (in-memory on the
|
||||||
|
# orchestrator), NOT the gateway's env — the request is forwarded with
|
||||||
|
# Authorization injected. This is the token-injection cut-over fix.
|
||||||
|
policy = (
|
||||||
|
'routes:\n - host: api.example.com\n'
|
||||||
|
' auth_scheme: "Bearer"\n token_env: "EGRESS_TOKEN_0"\n'
|
||||||
|
)
|
||||||
|
|
||||||
|
class _AuthResolver:
|
||||||
|
def resolve_policy_and_bottle_id(self, ip: str, identity_token: str = ""):
|
||||||
|
del ip, identity_token
|
||||||
|
return policy, "bottle-a", {"EGRESS_TOKEN_0": "sk-secret"}
|
||||||
|
|
||||||
|
addon = _addon(Config(routes=()))
|
||||||
|
addon._resolver = cast(Any, _AuthResolver())
|
||||||
|
flow = _with_client_ip(_Flow(_Request(host="api.example.com", method="GET")), "10.0.0.1")
|
||||||
|
_run_request(addon, flow)
|
||||||
|
self.assertIsNone(flow.response) # forwarded, not blocked
|
||||||
|
self.assertEqual("Bearer sk-secret", flow.request.headers.get("authorization"))
|
||||||
|
|
||||||
|
def test_authed_route_without_token_is_blocked(self) -> None:
|
||||||
|
# No token resolved for the bottle → the authed route can't inject, so
|
||||||
|
# the request is blocked (the error the cut-over originally produced).
|
||||||
|
policy = (
|
||||||
|
'routes:\n - host: api.example.com\n'
|
||||||
|
' auth_scheme: "Bearer"\n token_env: "EGRESS_TOKEN_0"\n'
|
||||||
|
)
|
||||||
|
|
||||||
|
class _NoTokenResolver:
|
||||||
|
def resolve_policy_and_bottle_id(self, ip: str, identity_token: str = ""):
|
||||||
|
del ip, identity_token
|
||||||
|
return policy, "bottle-a", {} # no tokens
|
||||||
|
|
||||||
|
addon = _addon(Config(routes=()))
|
||||||
|
addon._resolver = cast(Any, _NoTokenResolver())
|
||||||
|
flow = _with_client_ip(_Flow(_Request(host="api.example.com", method="GET")), "10.0.0.1")
|
||||||
|
_run_request(addon, flow)
|
||||||
|
self.assertIsNotNone(flow.response) # blocked — token unset
|
||||||
|
|
||||||
def test_unattributed_source_ip_cannot_supervise(self) -> None:
|
def test_unattributed_source_ip_cannot_supervise(self) -> None:
|
||||||
addon = self._consolidated_addon()
|
addon = self._consolidated_addon()
|
||||||
# 10.9.9.9 is not in the resolver map -> deny-all config, empty slug.
|
# 10.9.9.9 is not in the resolver map -> deny-all config, empty slug.
|
||||||
|
|||||||
@@ -51,45 +51,55 @@ class TestResolveClientConfig(unittest.TestCase):
|
|||||||
|
|
||||||
class _FakeContextResolver:
|
class _FakeContextResolver:
|
||||||
def __init__(
|
def __init__(
|
||||||
self, policy: str | None = None, bottle_id: str | None = None, raises: bool = False,
|
self, policy: str | None = None, bottle_id: str | None = None,
|
||||||
|
raises: bool = False, tokens: dict[str, str] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._policy = policy
|
self._policy = policy
|
||||||
self._bottle_id = bottle_id
|
self._bottle_id = bottle_id
|
||||||
self._raises = raises
|
self._raises = raises
|
||||||
|
self._tokens = tokens or {}
|
||||||
|
|
||||||
def resolve_policy_and_bottle_id(
|
def resolve_policy_and_bottle_id(
|
||||||
self, source_ip: str, identity_token: str = "",
|
self, source_ip: str, identity_token: str = "",
|
||||||
) -> tuple[str | None, str | None]:
|
) -> tuple[str | None, str | None, dict[str, str]]:
|
||||||
if self._raises:
|
if self._raises:
|
||||||
raise PolicyResolveError("orchestrator down")
|
raise PolicyResolveError("orchestrator down")
|
||||||
return self._policy, self._bottle_id
|
return self._policy, self._bottle_id, self._tokens
|
||||||
|
|
||||||
|
|
||||||
class TestResolveClientContext(unittest.TestCase):
|
class TestResolveClientContext(unittest.TestCase):
|
||||||
def test_returns_config_and_bottle_id(self) -> None:
|
def test_returns_config_bottle_id_and_tokens(self) -> None:
|
||||||
cfg, slug = resolve_client_context(
|
cfg, slug, tokens = resolve_client_context(
|
||||||
_FakeContextResolver(policy="routes:\n - host: example.com\n", bottle_id="b1"),
|
_FakeContextResolver(
|
||||||
|
policy="routes:\n - host: example.com\n", bottle_id="b1",
|
||||||
|
tokens={"EGRESS_TOKEN_0": "sekret"},
|
||||||
|
),
|
||||||
"10.243.0.1",
|
"10.243.0.1",
|
||||||
)
|
)
|
||||||
self.assertEqual(("example.com",), tuple(r.host for r in cfg.routes))
|
self.assertEqual(("example.com",), tuple(r.host for r in cfg.routes))
|
||||||
self.assertEqual("b1", slug)
|
self.assertEqual("b1", slug)
|
||||||
|
self.assertEqual({"EGRESS_TOKEN_0": "sekret"}, tokens) # for auth injection
|
||||||
|
|
||||||
def test_unattributed_denies_and_empty_slug(self) -> None:
|
def test_unattributed_denies_and_empty_slug(self) -> None:
|
||||||
cfg, slug = resolve_client_context(
|
cfg, slug, tokens = resolve_client_context(
|
||||||
_FakeContextResolver(policy=None, bottle_id=None), "10.243.0.9",
|
_FakeContextResolver(policy=None, bottle_id=None), "10.243.0.9",
|
||||||
)
|
)
|
||||||
self.assertEqual((), cfg.routes)
|
self.assertEqual((), cfg.routes)
|
||||||
self.assertEqual("", slug) # no bottle → supervise unavailable
|
self.assertEqual("", slug) # no bottle → supervise unavailable
|
||||||
|
self.assertEqual({}, tokens)
|
||||||
|
|
||||||
def test_resolver_error_denies_and_empty_slug(self) -> None:
|
def test_resolver_error_denies_empty_slug_no_tokens(self) -> None:
|
||||||
cfg, slug = resolve_client_context(_FakeContextResolver(raises=True), "10.243.0.1")
|
cfg, slug, tokens = resolve_client_context(
|
||||||
|
_FakeContextResolver(raises=True), "10.243.0.1",
|
||||||
|
)
|
||||||
self.assertEqual((), cfg.routes)
|
self.assertEqual((), cfg.routes)
|
||||||
self.assertEqual("", slug)
|
self.assertEqual("", slug)
|
||||||
|
self.assertEqual({}, tokens)
|
||||||
|
|
||||||
def test_unparseable_policy_denies_but_keeps_slug(self) -> None:
|
def test_unparseable_policy_denies_but_keeps_slug(self) -> None:
|
||||||
# A bad policy denies egress, but the bottle is still attributed (its
|
# A bad policy denies egress, but the bottle is still attributed (its
|
||||||
# supervise queue is keyed by the id, independent of route parsing).
|
# supervise queue is keyed by the id, independent of route parsing).
|
||||||
cfg, slug = resolve_client_context(
|
cfg, slug, _tokens = resolve_client_context(
|
||||||
_FakeContextResolver(policy="routes: notalist\n", bottle_id="b2"), "10.243.0.1",
|
_FakeContextResolver(policy="routes: notalist\n", bottle_id="b2"), "10.243.0.1",
|
||||||
)
|
)
|
||||||
self.assertEqual((), cfg.routes)
|
self.assertEqual((), cfg.routes)
|
||||||
|
|||||||
@@ -35,11 +35,43 @@ class TestDockerGateway(unittest.TestCase):
|
|||||||
with patch(_RUN_DOCKER, return_value=_proc(stdout="")):
|
with patch(_RUN_DOCKER, return_value=_proc(stdout="")):
|
||||||
self.assertFalse(self.sc.is_running())
|
self.assertFalse(self.sc.is_running())
|
||||||
|
|
||||||
def test_ensure_running_is_noop_when_already_up(self) -> None:
|
def test_ensure_running_noop_when_up_and_image_current(self) -> None:
|
||||||
with patch(_RUN_DOCKER, return_value=_proc(stdout=self.sc.name)) as m:
|
calls: list[list[str]] = []
|
||||||
|
|
||||||
|
def fake(argv: list[str]) -> Mock:
|
||||||
|
calls.append(argv)
|
||||||
|
if argv[:2] == ["docker", "ps"]:
|
||||||
|
return _proc(stdout=self.sc.name) # running
|
||||||
|
if argv[:3] == ["docker", "image", "inspect"]:
|
||||||
|
return _proc(stdout="img-A") # current image id
|
||||||
|
if argv[:2] == ["docker", "inspect"]:
|
||||||
|
return _proc(stdout="img-A") # container's image (same)
|
||||||
|
return _proc()
|
||||||
|
|
||||||
|
with patch(_RUN_DOCKER, side_effect=fake):
|
||||||
self.sc.ensure_running()
|
self.sc.ensure_running()
|
||||||
# Only the is_running() ps probe — no rm / run.
|
self.assertEqual([], [c for c in calls if c[:2] == ["docker", "run"]])
|
||||||
self.assertEqual(1, m.call_count)
|
self.assertEqual([], [c for c in calls if c[:2] == ["docker", "rm"]])
|
||||||
|
|
||||||
|
def test_ensure_running_recreates_when_image_is_stale(self) -> None:
|
||||||
|
# Running, but the container was built from an OLD image → recreate so
|
||||||
|
# a rebuild's new flat daemons take effect.
|
||||||
|
calls: list[list[str]] = []
|
||||||
|
|
||||||
|
def fake(argv: list[str]) -> Mock:
|
||||||
|
calls.append(argv)
|
||||||
|
if argv[:2] == ["docker", "ps"]:
|
||||||
|
return _proc(stdout=self.sc.name)
|
||||||
|
if argv[:3] == ["docker", "image", "inspect"]:
|
||||||
|
return _proc(stdout="img-NEW")
|
||||||
|
if argv[:2] == ["docker", "inspect"]:
|
||||||
|
return _proc(stdout="img-OLD")
|
||||||
|
return _proc()
|
||||||
|
|
||||||
|
with patch(_RUN_DOCKER, side_effect=fake):
|
||||||
|
self.sc.ensure_running()
|
||||||
|
self.assertEqual(1, len([c for c in calls if c[:2] == ["docker", "run"]]))
|
||||||
|
self.assertTrue(any(c[:2] == ["docker", "rm"] for c in calls))
|
||||||
|
|
||||||
def test_ensure_running_starts_the_singleton_when_absent(self) -> None:
|
def test_ensure_running_starts_the_singleton_when_absent(self) -> None:
|
||||||
calls: list[list[str]] = []
|
calls: list[list[str]] = []
|
||||||
@@ -81,9 +113,17 @@ class TestDockerGateway(unittest.TestCase):
|
|||||||
self.assertEqual(["docker", "exec", self.sc.name, "cat", GATEWAY_CA_CERT], argv)
|
self.assertEqual(["docker", "exec", self.sc.name, "cat", GATEWAY_CA_CERT], argv)
|
||||||
|
|
||||||
def test_ca_cert_pem_raises_when_absent(self) -> None:
|
def test_ca_cert_pem_raises_when_absent(self) -> None:
|
||||||
|
# timeout=0 → one probe then give up (no polling delay in the test).
|
||||||
with patch(_RUN_DOCKER, return_value=_proc(returncode=1, stderr="No such file")):
|
with patch(_RUN_DOCKER, return_value=_proc(returncode=1, stderr="No such file")):
|
||||||
with self.assertRaises(GatewayError):
|
with self.assertRaises(GatewayError):
|
||||||
self.sc.ca_cert_pem()
|
self.sc.ca_cert_pem(timeout=0)
|
||||||
|
|
||||||
|
def test_ca_cert_pem_polls_until_mitmproxy_writes_it(self) -> None:
|
||||||
|
# First read: CA not there yet; second read: present.
|
||||||
|
seq = [_proc(returncode=1, stderr="No such file"), _proc(stdout=_CA_PEM)]
|
||||||
|
with patch(_RUN_DOCKER, side_effect=seq), \
|
||||||
|
patch("bot_bottle.orchestrator.gateway.time.sleep"):
|
||||||
|
self.assertEqual(_CA_PEM, self.sc.ca_cert_pem(timeout=5))
|
||||||
|
|
||||||
def test_ensure_running_reuses_existing_network(self) -> None:
|
def test_ensure_running_reuses_existing_network(self) -> None:
|
||||||
calls: list[list[str]] = []
|
calls: list[list[str]] = []
|
||||||
|
|||||||
@@ -46,13 +46,18 @@ class TestOrchestratorService(unittest.TestCase):
|
|||||||
with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
|
with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
|
||||||
self.assertFalse(self.svc.is_healthy())
|
self.assertFalse(self.svc.is_healthy())
|
||||||
|
|
||||||
def test_ensure_running_healthy_still_ensures_gateway_but_not_orchestrator(self) -> None:
|
def test_ensure_running_always_recreates_orchestrator(self) -> None:
|
||||||
|
# Even when a control plane is already healthy, the orchestrator is
|
||||||
|
# recreated so bind-mounted code changes take effect (its process
|
||||||
|
# won't reload). The gateway is ensured too.
|
||||||
|
run = Mock(return_value=Mock(returncode=0, stderr=""))
|
||||||
with patch(_URLOPEN, return_value=_health(200)), \
|
with patch(_URLOPEN, return_value=_health(200)), \
|
||||||
patch(_GATEWAY) as gw_cls, patch(_RUN) as run:
|
patch(_GATEWAY) as gw_cls, patch(_RUN, run), patch(_SLEEP):
|
||||||
self.assertEqual(self.svc.url, self.svc.ensure_running())
|
self.assertEqual(self.svc.url, self.svc.ensure_running())
|
||||||
gw = gw_cls.return_value
|
gw_cls.return_value.ensure_running.assert_called() # gateway kept up
|
||||||
gw.ensure_running.assert_called() # gateway kept up
|
runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]]
|
||||||
run.assert_not_called() # no orchestrator container run
|
self.assertEqual(1, len(runs)) # orchestrator recreated
|
||||||
|
self.assertIn(ORCHESTRATOR_NAME, runs[0])
|
||||||
|
|
||||||
def test_ensure_running_starts_orchestrator_container_when_absent(self) -> None:
|
def test_ensure_running_starts_orchestrator_container_when_absent(self) -> None:
|
||||||
run = Mock(return_value=Mock(returncode=0, stderr=""))
|
run = Mock(return_value=Mock(returncode=0, stderr=""))
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlite3
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -23,6 +24,20 @@ class TestRegistryStore(unittest.TestCase):
|
|||||||
def tearDown(self) -> None:
|
def tearDown(self) -> None:
|
||||||
self._tmp.cleanup()
|
self._tmp.cleanup()
|
||||||
|
|
||||||
|
def _force_active_row(self, source_ip: str, bottle_id: str) -> None:
|
||||||
|
"""Insert a raw active row, bypassing `register`'s same-IP supersede —
|
||||||
|
the only way to manufacture the ambiguity the fail-closed guard exists
|
||||||
|
for (crash-orphaned / externally-injected duplicate)."""
|
||||||
|
conn = sqlite3.connect(self.db)
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO orchestrator_bottles "
|
||||||
|
"(bottle_id, source_ip, identity_token, state, created_at, metadata, policy) "
|
||||||
|
"VALUES (?, ?, ?, 'active', 0.0, '', '')",
|
||||||
|
(bottle_id, source_ip, "tok-" + bottle_id),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
def test_register_mints_id_and_token(self) -> None:
|
def test_register_mints_id_and_token(self) -> None:
|
||||||
rec = self.store.register("10.243.0.1")
|
rec = self.store.register("10.243.0.1")
|
||||||
self.assertTrue(rec.bottle_id)
|
self.assertTrue(rec.bottle_id)
|
||||||
@@ -76,11 +91,29 @@ class TestRegistryStore(unittest.TestCase):
|
|||||||
|
|
||||||
def test_attribute_ambiguous_ip_denied(self) -> None:
|
def test_attribute_ambiguous_ip_denied(self) -> None:
|
||||||
# Two active bottles on one source IP is a misconfiguration — deny
|
# Two active bottles on one source IP is a misconfiguration — deny
|
||||||
# rather than guess (fail-closed), even with a valid token.
|
# rather than guess (fail-closed), even with a valid token. (Forced
|
||||||
|
# directly: `register` now supersedes, so this can only arise from an
|
||||||
|
# orphaned/injected row.)
|
||||||
a = self.store.register("10.243.0.1", bottle_id="a")
|
a = self.store.register("10.243.0.1", bottle_id="a")
|
||||||
self.store.register("10.243.0.1", bottle_id="b")
|
self._force_active_row("10.243.0.1", "b")
|
||||||
self.assertIsNone(self.store.attribute("10.243.0.1", a.identity_token))
|
self.assertIsNone(self.store.attribute("10.243.0.1", a.identity_token))
|
||||||
|
|
||||||
|
def test_reregister_same_source_ip_supersedes(self) -> None:
|
||||||
|
# Re-registering a source IP (reused IP / crash recovery / persisted
|
||||||
|
# DB) must supersede the prior active bottle, not add a second active
|
||||||
|
# row that would fail-closed the whole IP to 403.
|
||||||
|
a = self.store.register("10.243.0.1", bottle_id="a", policy="A")
|
||||||
|
b = self.store.register("10.243.0.1", bottle_id="b", policy="B")
|
||||||
|
# exactly one active row at that IP, and it's the new bottle
|
||||||
|
got = self.store.by_source_ip("10.243.0.1")
|
||||||
|
assert got is not None
|
||||||
|
self.assertEqual("b", got.bottle_id)
|
||||||
|
self.assertEqual("B", got.policy)
|
||||||
|
# the superseded bottle is gone and its token no longer attributes
|
||||||
|
self.assertIsNone(self.store.get("a"))
|
||||||
|
self.assertIsNone(self.store.attribute("10.243.0.1", a.identity_token))
|
||||||
|
self.assertIsNotNone(self.store.attribute("10.243.0.1", b.identity_token))
|
||||||
|
|
||||||
def test_state_persists_across_reopen(self) -> None:
|
def test_state_persists_across_reopen(self) -> None:
|
||||||
rec = self.store.register("10.243.0.1")
|
rec = self.store.register("10.243.0.1")
|
||||||
reopened = RegistryStore(self.db)
|
reopened = RegistryStore(self.db)
|
||||||
@@ -130,7 +163,7 @@ class TestRegistryStore(unittest.TestCase):
|
|||||||
def test_by_source_ip_unknown_or_ambiguous_denied(self) -> None:
|
def test_by_source_ip_unknown_or_ambiguous_denied(self) -> None:
|
||||||
self.assertIsNone(self.store.by_source_ip("10.243.9.9")) # unknown
|
self.assertIsNone(self.store.by_source_ip("10.243.9.9")) # unknown
|
||||||
self.store.register("10.243.0.1", bottle_id="a")
|
self.store.register("10.243.0.1", bottle_id="a")
|
||||||
self.store.register("10.243.0.1", bottle_id="b")
|
self._force_active_row("10.243.0.1", "b") # orphaned duplicate
|
||||||
self.assertIsNone(self.store.by_source_ip("10.243.0.1")) # ambiguous
|
self.assertIsNone(self.store.by_source_ip("10.243.0.1")) # ambiguous
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -92,6 +92,18 @@ class TestOrchestrator(unittest.TestCase):
|
|||||||
assert got is not None
|
assert got is not None
|
||||||
self.assertEqual('{"routes":[]}', got.policy)
|
self.assertEqual('{"routes":[]}', got.policy)
|
||||||
|
|
||||||
|
def test_tokens_held_in_memory_and_cleared_on_teardown(self) -> None:
|
||||||
|
rec = self.orch.launch_bottle("10.243.0.5", tokens={"EGRESS_TOKEN_0": "sk"})
|
||||||
|
self.assertEqual({"EGRESS_TOKEN_0": "sk"}, self.orch.tokens_for(rec.bottle_id))
|
||||||
|
# not persisted in the registry (redacted record has no tokens field)
|
||||||
|
self.assertNotIn("tokens", rec.redacted())
|
||||||
|
self.orch.teardown_bottle(rec.bottle_id)
|
||||||
|
self.assertEqual({}, self.orch.tokens_for(rec.bottle_id))
|
||||||
|
|
||||||
|
def test_tokens_default_empty(self) -> None:
|
||||||
|
rec = self.orch.launch_bottle("10.243.0.6")
|
||||||
|
self.assertEqual({}, self.orch.tokens_for(rec.bottle_id))
|
||||||
|
|
||||||
def test_set_policy_live_reload(self) -> None:
|
def test_set_policy_live_reload(self) -> None:
|
||||||
rec = self.orch.launch_bottle("10.243.0.3")
|
rec = self.orch.launch_bottle("10.243.0.3")
|
||||||
self.assertTrue(self.orch.set_policy(rec.bottle_id, '{"x":1}'))
|
self.assertTrue(self.orch.set_policy(rec.bottle_id, '{"x":1}'))
|
||||||
|
|||||||
@@ -89,13 +89,17 @@ class TestPolicyResolver(unittest.TestCase):
|
|||||||
self.r.resolve_bottle_id("10.243.0.1", "tok")
|
self.r.resolve_bottle_id("10.243.0.1", "tok")
|
||||||
|
|
||||||
def test_resolve_policy_and_bottle_id_one_call(self) -> None:
|
def test_resolve_policy_and_bottle_id_one_call(self) -> None:
|
||||||
with patch(_URLOPEN, return_value=_resp({"bottle_id": "b1", "policy": "P"})) as m:
|
payload = {"bottle_id": "b1", "policy": "P", "tokens": {"EGRESS_TOKEN_0": "s"}}
|
||||||
self.assertEqual(("P", "b1"), self.r.resolve_policy_and_bottle_id("10.243.0.1", "t"))
|
with patch(_URLOPEN, return_value=_resp(payload)) as m:
|
||||||
self.assertEqual(1, m.call_count) # both from a single /resolve
|
self.assertEqual(
|
||||||
|
("P", "b1", {"EGRESS_TOKEN_0": "s"}),
|
||||||
|
self.r.resolve_policy_and_bottle_id("10.243.0.1", "t"),
|
||||||
|
)
|
||||||
|
self.assertEqual(1, m.call_count) # policy + id + tokens from a single /resolve
|
||||||
|
|
||||||
def test_resolve_policy_and_bottle_id_403_is_none_none(self) -> None:
|
def test_resolve_policy_and_bottle_id_403_is_none_none_empty(self) -> None:
|
||||||
with patch(_URLOPEN, side_effect=_http_error(403)):
|
with patch(_URLOPEN, side_effect=_http_error(403)):
|
||||||
self.assertEqual((None, None), self.r.resolve_policy_and_bottle_id("10.243.0.9"))
|
self.assertEqual((None, None, {}), self.r.resolve_policy_and_bottle_id("10.243.0.9"))
|
||||||
|
|
||||||
def test_resolve_policy_and_bottle_id_error_raises(self) -> None:
|
def test_resolve_policy_and_bottle_id_error_raises(self) -> None:
|
||||||
with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
|
with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
|
||||||
|
|||||||
Reference in New Issue
Block a user