Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 19164da2d2 | |||
| 5550bb75ad | |||
| abd3fedcea | |||
| b679e3f6ca | |||
| b8941c101a | |||
| 92be0a4d90 | |||
| 05e99eeaf1 | |||
| 00f32a3890 |
@@ -45,15 +45,10 @@ def teardown_consolidated(
|
||||
transport: GatewayTransport,
|
||||
*,
|
||||
orchestrator_url: str,
|
||||
timeout: float | None = None,
|
||||
) -> None:
|
||||
"""Deregister the bottle and remove its git-gate state. Both steps are
|
||||
idempotent so this is safe from a cleanup trap."""
|
||||
from ..orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
|
||||
OrchestratorClient(
|
||||
orchestrator_url,
|
||||
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
|
||||
).teardown_bottle(bottle_id)
|
||||
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
|
||||
deprovision_git_gate(transport, bottle_id)
|
||||
|
||||
|
||||
|
||||
@@ -122,11 +122,10 @@ def launch_consolidated(
|
||||
|
||||
def teardown_consolidated(
|
||||
bottle_id: str, *, orchestrator_url: str, infra_name: str = INFRA_NAME,
|
||||
timeout: float | None = None,
|
||||
) -> None:
|
||||
"""Deregister the bottle and remove its git-gate state. Idempotent."""
|
||||
_teardown_util(bottle_id, DockerGatewayTransport(infra_name),
|
||||
orchestrator_url=orchestrator_url, timeout=timeout)
|
||||
orchestrator_url=orchestrator_url)
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
||||
@@ -62,7 +62,6 @@ from .compose import (
|
||||
write_compose_file,
|
||||
)
|
||||
from .consolidated_compose import consolidated_agent_compose
|
||||
from ...orchestrator.config_store import resolve_teardown_timeout
|
||||
from .consolidated_launch import launch_consolidated, teardown_consolidated
|
||||
from ...orchestrator.gateway import DockerGateway
|
||||
|
||||
@@ -134,14 +133,11 @@ def launch(
|
||||
token_values = egress_resolve_token_values(
|
||||
plan.egress_plan.token_env_map, effective_env,
|
||||
)
|
||||
teardown_timeout = resolve_teardown_timeout()
|
||||
ctx = launch_consolidated(
|
||||
plan.egress_plan, git_gate_plan, image_ref=plan.image, tokens=token_values,
|
||||
)
|
||||
stack.callback(
|
||||
teardown_consolidated, ctx.bottle_id,
|
||||
orchestrator_url=ctx.orchestrator_url,
|
||||
timeout=teardown_timeout,
|
||||
teardown_consolidated, ctx.bottle_id, orchestrator_url=ctx.orchestrator_url,
|
||||
)
|
||||
|
||||
# Step 4: install the SHARED gateway CA into the agent (replaces the
|
||||
|
||||
@@ -83,15 +83,12 @@ def launch_consolidated(
|
||||
)
|
||||
|
||||
|
||||
def teardown_consolidated(
|
||||
bottle_id: str, *, orchestrator_url: str, timeout: float | None = None,
|
||||
) -> None:
|
||||
def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None:
|
||||
"""Deregister the bottle and remove its git-gate state from the gateway
|
||||
VM. Both steps are idempotent so this is safe from a cleanup trap. Does
|
||||
NOT stop the infra VM — it's a persistent per-host singleton shared by
|
||||
every bottle."""
|
||||
_teardown_util(bottle_id, infra_vm.gateway_transport(),
|
||||
orchestrator_url=orchestrator_url, timeout=timeout)
|
||||
_teardown_util(bottle_id, infra_vm.gateway_transport(), orchestrator_url=orchestrator_url)
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
||||
@@ -52,7 +52,6 @@ from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||
from . import firecracker_vm, image_builder, isolation_probe, netpool, util
|
||||
from .bottle import FirecrackerBottle
|
||||
from .bottle_plan import FirecrackerBottlePlan
|
||||
from ...orchestrator.config_store import resolve_teardown_timeout
|
||||
from .consolidated_launch import (
|
||||
launch_consolidated,
|
||||
teardown_consolidated,
|
||||
@@ -113,7 +112,6 @@ def launch(
|
||||
token_values = egress_resolve_token_values(
|
||||
plan.egress_plan.token_env_map, effective_env,
|
||||
)
|
||||
teardown_timeout = resolve_teardown_timeout()
|
||||
ctx = launch_consolidated(
|
||||
plan.egress_plan, git_gate_plan,
|
||||
guest_ip=slot.guest_ip,
|
||||
@@ -123,7 +121,6 @@ def launch(
|
||||
stack.callback(
|
||||
teardown_consolidated, ctx.bottle_id,
|
||||
orchestrator_url=ctx.orchestrator_url,
|
||||
timeout=teardown_timeout,
|
||||
)
|
||||
|
||||
# Step 5: install the SHARED gateway CA (replaces the per-bottle CA).
|
||||
|
||||
@@ -68,14 +68,9 @@ class MacosContainerBottle(Bottle):
|
||||
# reaches the agent (PRD 0070): registration mints it *after* the
|
||||
# container exists — its source IP is the registration key and Apple
|
||||
# Container assigns that by DHCP — so it cannot be in the run-time env
|
||||
# the way docker's compose spec does it.
|
||||
#
|
||||
# `container exec --env` does NOT override a run-time value — it
|
||||
# appends, leaving duplicate entries in the agent's `environ` whose
|
||||
# resolution is runtime-specific (Node last-wins, Rust first-wins). So
|
||||
# nothing here may rely on superseding: the proxy vars are supplied
|
||||
# *only* at exec time and are deliberately absent from the run-time
|
||||
# env. See `launch._agent_env_entries`.
|
||||
# the way docker's compose spec does it. `container exec --env` wins
|
||||
# over the run-time value, so the token-bearing proxy URL set here
|
||||
# supersedes the token-less one baked in at launch.
|
||||
self._exec_env = dict(exec_env or {})
|
||||
self._closed = False
|
||||
|
||||
|
||||
@@ -36,11 +36,8 @@ from dataclasses import dataclass
|
||||
|
||||
from ...egress import EgressPlan
|
||||
from ...git_gate import GitGatePlan
|
||||
from ...log import info
|
||||
from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
|
||||
from ...orchestrator.client import OrchestratorClient
|
||||
from ..consolidated_util import provision_bottle, teardown_consolidated as _teardown_util
|
||||
from . import util as container_mod
|
||||
from .enumerate import CONTAINER_NAME_PREFIX, EnumerationError, enumerate_active
|
||||
from .gateway import GATEWAY_NETWORK
|
||||
from .gateway_provision import AppleGatewayTransport
|
||||
from .infra import MacosInfraService, OrchestratorStartError
|
||||
@@ -91,32 +88,6 @@ def ensure_gateway(
|
||||
)
|
||||
|
||||
|
||||
def live_source_ips(network: str) -> list[str]:
|
||||
"""Every running agent container's address on `network`.
|
||||
|
||||
The reconciliation input: the orchestrator lives inside the infra
|
||||
container and cannot enumerate the host's containers, so the host has to
|
||||
tell it which bottles are actually up. Containers that have not been
|
||||
assigned an address yet contribute nothing — the reap's grace window, not
|
||||
this list, is what protects an in-flight launch.
|
||||
|
||||
Raises `EnumerationError` when the live set cannot be determined
|
||||
authoritatively: either the container listing fails or any individual
|
||||
inspect fails. Callers must skip reconciliation in that case to avoid
|
||||
unregistering healthy bottles."""
|
||||
ips: list[str] = []
|
||||
for agent in enumerate_active():
|
||||
name = f"{CONTAINER_NAME_PREFIX}{agent.slug}"
|
||||
ip = container_mod.inspect_container_network_ip(name, network)
|
||||
if ip is None:
|
||||
raise EnumerationError(
|
||||
f"container inspect {name!r} failed; live set is not authoritative"
|
||||
)
|
||||
if ip:
|
||||
ips.append(ip)
|
||||
return ips
|
||||
|
||||
|
||||
def register_agent(
|
||||
egress_plan: EgressPlan,
|
||||
git_gate_plan: GitGatePlan,
|
||||
@@ -131,16 +102,6 @@ def register_agent(
|
||||
container — it is the attribution key the gateway resolves policy by.
|
||||
Raises on failure; the caller tears down."""
|
||||
client = OrchestratorClient(endpoint.orchestrator_url)
|
||||
# Self-heal before registering: a launcher that died hard (SIGKILL, closed
|
||||
# terminal, host sleep) never ran its teardown callback, leaving an active
|
||||
# row with no container. vmnet recycles addresses, so such a row can
|
||||
# collide with this bottle's — and `by_source_ip` fail-closes on ambiguity,
|
||||
# which would resolve no policy at all and deny every host. Best-effort: a
|
||||
# reconciliation failure must not block an otherwise-fine launch.
|
||||
try:
|
||||
client.reconcile(live_source_ips(endpoint.network))
|
||||
except (OrchestratorClientError, EnumerationError) as e:
|
||||
info(f"registry reconciliation skipped: {e}")
|
||||
reg = provision_bottle(
|
||||
client, source_ip, egress_plan, git_gate_plan, AppleGatewayTransport(),
|
||||
image_ref=image_ref, tokens=tokens,
|
||||
@@ -155,21 +116,17 @@ def register_agent(
|
||||
)
|
||||
|
||||
|
||||
def teardown_consolidated(
|
||||
bottle_id: str, *, orchestrator_url: str, timeout: float | None = None,
|
||||
) -> None:
|
||||
def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None:
|
||||
"""Deregister the bottle and remove its git-gate state from the gateway.
|
||||
Both steps are idempotent so this is safe from a cleanup trap. Does NOT
|
||||
stop the gateway — it's a persistent per-host singleton."""
|
||||
_teardown_util(bottle_id, AppleGatewayTransport(),
|
||||
orchestrator_url=orchestrator_url, timeout=timeout)
|
||||
_teardown_util(bottle_id, AppleGatewayTransport(), orchestrator_url=orchestrator_url)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GatewayEndpoint",
|
||||
"LaunchContext",
|
||||
"ensure_gateway",
|
||||
"live_source_ips",
|
||||
"register_agent",
|
||||
"teardown_consolidated",
|
||||
"ConsolidatedLaunchError",
|
||||
|
||||
@@ -8,21 +8,13 @@ from ...bottle_state import read_metadata
|
||||
from .. import ActiveAgent
|
||||
from .infra import INFRA_NAME
|
||||
|
||||
# The name every agent container carries: `bot-bottle-<slug>`. Exported
|
||||
# because callers that act on a running bottle (gateway-host rewrites,
|
||||
# registry reconciliation) have to map an enumerated slug back to a
|
||||
# container name.
|
||||
CONTAINER_NAME_PREFIX = "bot-bottle-"
|
||||
_PREFIX = "bot-bottle-"
|
||||
# The shared per-host infra container carries the same prefix as agent
|
||||
# containers but is infrastructure, not a bottle — one control plane + gateway
|
||||
# serves every agent, so listing it as an agent would invent one per host.
|
||||
_INFRA_NAMES = frozenset({INFRA_NAME})
|
||||
|
||||
|
||||
class EnumerationError(RuntimeError):
|
||||
"""container list failed; the resulting live set is not authoritative."""
|
||||
|
||||
|
||||
def enumerate_active() -> list[ActiveAgent]:
|
||||
result = subprocess.run(
|
||||
["container", "list", "--quiet"],
|
||||
@@ -31,15 +23,12 @@ def enumerate_active() -> list[ActiveAgent]:
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise EnumerationError(
|
||||
f"container list failed: "
|
||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||
)
|
||||
return []
|
||||
out: list[ActiveAgent] = []
|
||||
for name in sorted(line.strip() for line in result.stdout.splitlines()):
|
||||
if not name.startswith(CONTAINER_NAME_PREFIX) or name in _INFRA_NAMES:
|
||||
if not name.startswith(_PREFIX) or name in _INFRA_NAMES:
|
||||
continue
|
||||
slug = name[len(CONTAINER_NAME_PREFIX):]
|
||||
slug = name[len(_PREFIX):]
|
||||
metadata = read_metadata(slug)
|
||||
out.append(ActiveAgent(
|
||||
backend_name="macos-container",
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
"""Stable gateway name for macOS agents, via each bottle's `/etc/hosts`.
|
||||
|
||||
The shared gateway's address is assigned by vmnet's DHCP and changes whenever
|
||||
the infra container is recreated — a source-hash bump, an image upgrade, a
|
||||
crash. Every agent-facing URL (egress proxy, git-http, supervise) embeds that
|
||||
address, and the proxy URL reaches the agent as **process environment** at
|
||||
`container exec` time. A running process's `environ` cannot be rewritten from
|
||||
outside, so a moved gateway used to strand every running bottle permanently:
|
||||
not degraded, unreachable, until the bottle was relaunched and its session
|
||||
thrown away.
|
||||
|
||||
So the agent never learns the address. It is given a stable *name*
|
||||
(`GATEWAY_HOSTNAME`) in every URL, resolved through its own `/etc/hosts`.
|
||||
Unlike `environ`, that is a file — it can be rewritten inside a container that
|
||||
is already running, so a gateway that comes back at a new address is picked up
|
||||
by live bottles instead of orphaning them.
|
||||
|
||||
Apple Container 1.0 offers no container-name DNS on a user network (the only
|
||||
nameserver an agent sees is vmnet's, which does not know container names) and
|
||||
`container run` has no `--add-host`, so the entry is written by exec after the
|
||||
container starts.
|
||||
|
||||
Writing it needs root, and the agent runs as `node`: the agent therefore
|
||||
cannot repoint its own gateway name, while the host (which drives `container
|
||||
exec --user root`) can. That asymmetry is deliberate — keep it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ...log import warn
|
||||
from . import util as container_mod
|
||||
from .enumerate import CONTAINER_NAME_PREFIX, enumerate_active
|
||||
|
||||
# The name every agent-facing gateway URL uses. Must not collide with a real
|
||||
# DNS name the agent might resolve; it is bottle-local by construction.
|
||||
GATEWAY_HOSTNAME = "bot-bottle-gateway"
|
||||
|
||||
# Marker so the rewrite is idempotent and only ever touches our own line —
|
||||
# the rest of /etc/hosts (localhost, the container's own name) is preserved.
|
||||
_MARKER = "# bot-bottle gateway"
|
||||
|
||||
|
||||
def _rewrite_script(gateway_ip: str) -> str:
|
||||
"""A shell one-liner that replaces our managed line in `/etc/hosts`.
|
||||
|
||||
Rewrites in place via a temp file + `cat` rather than `mv`, so the file
|
||||
keeps its original inode, ownership, and mode — a bind-mounted or
|
||||
pre-created `/etc/hosts` must not be replaced by a root-owned 0644 copy
|
||||
that the runtime then refuses to update.
|
||||
"""
|
||||
return (
|
||||
"set -e; "
|
||||
f"grep -v '{_MARKER}' /etc/hosts > /tmp/.bb-hosts || true; "
|
||||
f"printf '%s %s %s\\n' '{gateway_ip}' '{GATEWAY_HOSTNAME}' "
|
||||
f"'{_MARKER}' >> /tmp/.bb-hosts; "
|
||||
"cat /tmp/.bb-hosts > /etc/hosts; "
|
||||
"rm -f /tmp/.bb-hosts"
|
||||
)
|
||||
|
||||
|
||||
def set_gateway_host(container_name: str, gateway_ip: str) -> None:
|
||||
"""Point `GATEWAY_HOSTNAME` at `gateway_ip` inside one running container.
|
||||
|
||||
Must run before the agent is exec'd: the agent's proxy URL names the
|
||||
gateway, so the entry has to exist for its first connection. Idempotent —
|
||||
re-running with the same address is a no-op in effect.
|
||||
"""
|
||||
container_mod.exec_container_as_root(
|
||||
container_name, ["sh", "-c", _rewrite_script(gateway_ip)],
|
||||
)
|
||||
|
||||
|
||||
def refresh_gateway_host(gateway_ip: str) -> list[str]:
|
||||
"""Re-point every running bottle at the current gateway address.
|
||||
|
||||
Called once the shared gateway is known to be up, so a bottle stranded by
|
||||
an earlier gateway restart re-attaches instead of needing a relaunch.
|
||||
Returns the containers updated.
|
||||
|
||||
Best-effort per bottle: one container that refuses the write (already
|
||||
exiting, say) must not stop the others from being repaired, and must not
|
||||
fail the launch that triggered the sweep.
|
||||
"""
|
||||
updated: list[str] = []
|
||||
for agent in enumerate_active():
|
||||
name = f"{CONTAINER_NAME_PREFIX}{agent.slug}"
|
||||
try:
|
||||
set_gateway_host(name, gateway_ip)
|
||||
updated.append(name)
|
||||
# One bad bottle must not stop the sweep, so this is deliberately broad.
|
||||
except Exception as e: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
||||
warn(f"could not re-point {name} at the gateway: {e}")
|
||||
return updated
|
||||
|
||||
|
||||
__all__ = ["GATEWAY_HOSTNAME", "set_gateway_host", "refresh_gateway_host"]
|
||||
@@ -59,13 +59,7 @@ from ..docker.egress import EGRESS_PORT
|
||||
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||
from . import util as container_mod
|
||||
from .bottle import MacosContainerBottle
|
||||
from .gateway_hosts import (
|
||||
GATEWAY_HOSTNAME,
|
||||
refresh_gateway_host,
|
||||
set_gateway_host,
|
||||
)
|
||||
from .bottle_plan import MacosContainerBottlePlan
|
||||
from ...orchestrator.config_store import resolve_teardown_timeout
|
||||
from .consolidated_launch import (
|
||||
GatewayEndpoint,
|
||||
ensure_gateway,
|
||||
@@ -106,11 +100,6 @@ def launch(
|
||||
# Step 1: the per-host singletons. Must precede the agent run — its
|
||||
# proxy env needs the gateway's address at `container run` time.
|
||||
endpoint = ensure_gateway()
|
||||
# The gateway's address may have changed since these bottles launched
|
||||
# (any infra recreate re-runs DHCP). They name the gateway rather than
|
||||
# address it, so re-pointing /etc/hosts re-attaches them in place
|
||||
# instead of leaving them stranded until relaunch.
|
||||
refresh_gateway_host(endpoint.gateway_ip)
|
||||
|
||||
# Step 2: mint this bottle's deploy keys, then point it at the SHARED
|
||||
# gateway's CA + git-http/supervise ports.
|
||||
@@ -128,9 +117,6 @@ def launch(
|
||||
# attribution key; `--cap-drop CAP_NET_RAW` at run is what makes it
|
||||
# unforgeable. Poll: `container run --detach` can return before vmnet's
|
||||
# DHCP has assigned the address.
|
||||
# Resolve the gateway name before anything execs: every agent-facing
|
||||
# URL uses it, so the entry must exist for the first connection.
|
||||
set_gateway_host(plan.container_name, endpoint.gateway_ip)
|
||||
source_ip = container_mod.wait_container_ipv4_on_network(
|
||||
plan.container_name, endpoint.network,
|
||||
)
|
||||
@@ -143,7 +129,6 @@ def launch(
|
||||
token_values = egress_resolve_token_values(
|
||||
plan.egress_plan.token_env_map, effective_env,
|
||||
)
|
||||
teardown_timeout = resolve_teardown_timeout()
|
||||
ctx = register_agent(
|
||||
plan.egress_plan,
|
||||
plan.git_gate_plan,
|
||||
@@ -155,7 +140,6 @@ def launch(
|
||||
stack.callback(
|
||||
teardown_consolidated, ctx.bottle_id,
|
||||
orchestrator_url=ctx.orchestrator_url,
|
||||
timeout=teardown_timeout,
|
||||
)
|
||||
info(
|
||||
f"agent {plan.container_name} registered "
|
||||
@@ -247,20 +231,13 @@ def _stamp_agent_urls(
|
||||
) -> MacosContainerBottlePlan:
|
||||
"""Point the agent's git-gate insteadOf rewrites + supervise MCP at the
|
||||
shared gateway's ports. Both bypass the egress proxy (NO_PROXY covers the
|
||||
gateway name).
|
||||
|
||||
Addressed by `GATEWAY_HOSTNAME`, never by IP: these URLs are baked into
|
||||
the agent's gitconfig and MCP config at provision time, so an address here
|
||||
would strand the bottle the moment the gateway moved. The name is resolved
|
||||
per connection through `/etc/hosts`, which stays rewritable while the
|
||||
bottle runs."""
|
||||
del endpoint # addressed by name; the address reaches the bottle via /etc/hosts
|
||||
gateway address)."""
|
||||
git_gate_url = (
|
||||
f"http://{GATEWAY_HOSTNAME}:{_GIT_HTTP_PORT}"
|
||||
f"http://{endpoint.gateway_ip}:{_GIT_HTTP_PORT}"
|
||||
if plan.git_gate_plan.upstreams else ""
|
||||
)
|
||||
supervise_url = (
|
||||
f"http://{GATEWAY_HOSTNAME}:{SUPERVISE_PORT}/"
|
||||
f"http://{endpoint.gateway_ip}:{SUPERVISE_PORT}/"
|
||||
if plan.supervise_plan is not None else ""
|
||||
)
|
||||
return dataclasses.replace(
|
||||
@@ -270,43 +247,30 @@ def _stamp_agent_urls(
|
||||
)
|
||||
|
||||
|
||||
def _proxy_url(identity_token: str = "") -> str:
|
||||
def _proxy_url(gateway_ip: str, identity_token: str = "") -> str:
|
||||
"""The agent's egress proxy URL. The identity token rides as proxy
|
||||
credentials — the gateway reads Proxy-Authorization, resolves the
|
||||
(source_ip, token) pair against the control plane, and strips it before
|
||||
upstream. Without a valid pair `/resolve` denies the request (#366).
|
||||
|
||||
Names the gateway rather than addressing it: this URL reaches the agent as
|
||||
process environment, which cannot be rewritten once the agent is running,
|
||||
so an address baked here is unfixable if the gateway moves."""
|
||||
upstream. Without a valid pair `/resolve` denies the request (#366)."""
|
||||
cred = f"bottle:{identity_token}@" if identity_token else ""
|
||||
return f"http://{cred}{GATEWAY_HOSTNAME}:{EGRESS_PORT}"
|
||||
return f"http://{cred}{gateway_ip}:{EGRESS_PORT}"
|
||||
|
||||
|
||||
def _no_proxy() -> str:
|
||||
def _no_proxy(gateway_ip: str) -> str:
|
||||
# git-http + supervise live on the gateway and must NOT go through the
|
||||
# egress proxy — the agent reaches them directly by name. Deliberately
|
||||
# address-free: NO_PROXY is baked into the run-time env and is therefore
|
||||
# just as unfixable as the proxy URL if the gateway moves.
|
||||
return f"localhost,127.0.0.1,{GATEWAY_HOSTNAME}"
|
||||
# egress proxy — the agent reaches them directly by its address.
|
||||
return f"localhost,127.0.0.1,{gateway_ip}"
|
||||
|
||||
|
||||
def _identity_proxy_env(
|
||||
endpoint: GatewayEndpoint, identity_token: str,
|
||||
) -> dict[str, str]:
|
||||
"""The token-bearing proxy env applied at `container exec` — the only way
|
||||
to get the token in, since it does not exist until after the container
|
||||
runs (registration keys on the DHCP-assigned address).
|
||||
|
||||
This is the *sole* source of `*_PROXY` for the agent. It deliberately does
|
||||
not rely on overriding a run-time value: `container exec --env` appends
|
||||
rather than replaces, so a run-time `HTTPS_PROXY` would survive alongside
|
||||
this one and first-wins runtimes would read the wrong entry. See
|
||||
`_agent_env_entries`."""
|
||||
"""The token-bearing proxy env applied at `container exec`. It supersedes
|
||||
the token-less run-time value (exec `--env` wins), which is the only way to
|
||||
get the token in: it does not exist until after the container runs."""
|
||||
if not identity_token:
|
||||
return {}
|
||||
del endpoint # the gateway is named, not addressed
|
||||
url = _proxy_url(identity_token)
|
||||
url = _proxy_url(endpoint.gateway_ip, identity_token)
|
||||
return {
|
||||
"HTTPS_PROXY": url, "HTTP_PROXY": url,
|
||||
"https_proxy": url, "http_proxy": url,
|
||||
@@ -353,23 +317,16 @@ def _agent_run_argv(
|
||||
def _agent_env_entries(
|
||||
plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint,
|
||||
) -> tuple[str, ...]:
|
||||
# No `*_PROXY` here on purpose. The token-bearing URL is applied at
|
||||
# `container exec` (`_identity_proxy_env`), and Apple's `container exec
|
||||
# --env` **appends** to the run-time environment rather than replacing it:
|
||||
# setting a token-less value here leaves two `HTTPS_PROXY` entries in the
|
||||
# agent's `environ`, token-less first. Which one a runtime reads is then
|
||||
# pure luck — Node takes the last (and worked), Rust's `std::env::var`
|
||||
# takes the first, so Codex proxied without its identity token and
|
||||
# `/resolve` fail-closed on every request.
|
||||
#
|
||||
# A token-less proxy URL has no legitimate consumer anyway: the init
|
||||
# process is `sleep` and everything that egresses arrives via exec. Its
|
||||
# only value was a tidy 403 for unattributed callers, which is not worth
|
||||
# silently dropping attribution for. Without it a process that egresses
|
||||
# before the exec-time env still fails closed — the agent network is
|
||||
# host-only, so there is no route off it except the gateway.
|
||||
no_proxy = _no_proxy()
|
||||
# Token-less at run time — the token does not exist yet (see
|
||||
# `_identity_proxy_env`). Anything egressing before the exec-time override
|
||||
# is denied by `/resolve`, which is the safe direction.
|
||||
proxy_url = _proxy_url(endpoint.gateway_ip)
|
||||
no_proxy = _no_proxy(endpoint.gateway_ip)
|
||||
env = [
|
||||
f"HTTPS_PROXY={proxy_url}",
|
||||
f"HTTP_PROXY={proxy_url}",
|
||||
f"https_proxy={proxy_url}",
|
||||
f"http_proxy={proxy_url}",
|
||||
f"NO_PROXY={no_proxy}",
|
||||
f"no_proxy={no_proxy}",
|
||||
f"NODE_EXTRA_CA_CERTS={AGENT_CA_PATH}",
|
||||
|
||||
@@ -360,21 +360,6 @@ def exec_container(name: str, argv: list[str]) -> None:
|
||||
)
|
||||
|
||||
|
||||
def exec_container_as_root(name: str, argv: list[str]) -> None:
|
||||
"""`exec_container`, but as uid 0 inside the container.
|
||||
|
||||
For host-driven maintenance the agent itself must not be able to perform —
|
||||
rewriting `/etc/hosts` to point the gateway name at an address. The agent
|
||||
runs as `node`, so it cannot repoint its own gateway; the host can.
|
||||
"""
|
||||
result = _run_container_op([_CONTAINER, "exec", "--user", "root", name, *argv])
|
||||
if result.returncode != 0:
|
||||
die(
|
||||
f"container exec (root) in {name} failed: "
|
||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||
)
|
||||
|
||||
|
||||
def _run_container_op(cmd: list[str]) -> subprocess.CompletedProcess[str]:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
@@ -572,41 +557,6 @@ def try_container_ipv4_on_network(name: str, network: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def inspect_container_network_ip(name: str, network: str) -> str | None:
|
||||
"""IP of `name` on `network`, distinguishing inspect failure from "not yet".
|
||||
|
||||
Returns:
|
||||
- the IP string when the container has one on `network`
|
||||
- "" when inspect succeeds but no address is assigned yet (in-flight DHCP)
|
||||
- None when the inspect command itself fails (authoritative list impossible)
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[_CONTAINER, "inspect", name],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(result.stdout or "[]")
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if isinstance(data, list):
|
||||
data = data[0] if data else {}
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
status = data.get("status")
|
||||
networks = status.get("networks") if isinstance(status, dict) else None
|
||||
if not isinstance(networks, list):
|
||||
return ""
|
||||
for entry in networks:
|
||||
if not isinstance(entry, dict) or entry.get("network") != network:
|
||||
continue
|
||||
raw = entry.get("ipv4Address")
|
||||
if isinstance(raw, str) and raw:
|
||||
return raw.split("/", 1)[0]
|
||||
return ""
|
||||
|
||||
|
||||
def wait_container_ipv4_on_network(
|
||||
name: str, network: str, *, timeout: float = 15.0, poll: float = 0.25,
|
||||
) -> str:
|
||||
|
||||
@@ -379,7 +379,6 @@ class EgressAddon:
|
||||
env,
|
||||
request_method=flow.request.method,
|
||||
request_headers=req_headers,
|
||||
deny_reason=config.deny_reason,
|
||||
)
|
||||
|
||||
if decision.action == "block":
|
||||
|
||||
@@ -89,14 +89,6 @@ LOG_FULL = 2 # log block/warn events + full request and response bodies
|
||||
class Config:
|
||||
routes: tuple[Route, ...]
|
||||
log: int = LOG_OFF
|
||||
# Why this Config is a deny-all, when it is one for a reason *other* than
|
||||
# the bottle's own policy genuinely not listing the host. A deny-all is
|
||||
# indistinguishable from "policy loaded, host not allowed" at the decision
|
||||
# point — both are simply "no matching route" — so without this the
|
||||
# operator sees `host X is not in the allowlist` and goes hunting for a
|
||||
# missing route that was never the problem. Empty for a normally-parsed
|
||||
# policy; `decide` prefers it over the allowlist wording when set.
|
||||
deny_reason: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -413,40 +405,16 @@ class PolicyResolverLike(typing.Protocol):
|
||||
...
|
||||
|
||||
|
||||
# Deny-all explanations. Each names the *actual* failure so an operator isn't
|
||||
# sent looking for a missing egress route when the bottle never had a policy
|
||||
# to begin with — the failure mode that made a bricked registration read like
|
||||
# a misconfigured allowlist.
|
||||
DENY_UNATTRIBUTED = (
|
||||
"egress: this request was not attributed to any bottle, so no egress "
|
||||
"policy applies and every host is denied. Either the bottle's registry "
|
||||
"row is missing/ambiguous (torn down, or another bottle claimed its "
|
||||
"source IP), or the request carried no matching identity token — check "
|
||||
"that the caller's proxy URL includes it. This is not an allowlist problem."
|
||||
)
|
||||
DENY_UNPARSEABLE = (
|
||||
"egress: this bottle's egress policy could not be parsed, so it is being "
|
||||
"treated as deny-all. Fix the bottle's egress.routes; every host is denied "
|
||||
"until it loads."
|
||||
)
|
||||
DENY_RESOLVER_ERROR = (
|
||||
"egress: the orchestrator could not be reached to resolve this bottle's "
|
||||
"egress policy, so every host is denied (fail-closed). Check that the "
|
||||
"control plane is up; this is not an allowlist problem."
|
||||
)
|
||||
|
||||
|
||||
def _config_from_policy(policy: "str | None") -> "Config":
|
||||
"""Parse a resolved policy blob into a Config, fail-closed: None / empty /
|
||||
unparseable all become a deny-all Config (no routes → every request
|
||||
blocked). Each deny-all carries the reason it is one, so the block message
|
||||
names the real fault instead of blaming the allowlist."""
|
||||
blocked)."""
|
||||
if not policy:
|
||||
return Config(routes=(), deny_reason=DENY_UNATTRIBUTED)
|
||||
return Config(routes=()) # unattributed or empty → deny-all
|
||||
try:
|
||||
return load_config(policy)
|
||||
except ValueError:
|
||||
return Config(routes=(), deny_reason=DENY_UNPARSEABLE)
|
||||
return Config(routes=()) # unparseable policy → deny
|
||||
|
||||
|
||||
def resolve_client_config(
|
||||
@@ -460,7 +428,7 @@ def resolve_client_config(
|
||||
try:
|
||||
policy = resolver.resolve(client_ip, identity_token)
|
||||
except Exception: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
||||
return Config(routes=(), deny_reason=DENY_RESOLVER_ERROR)
|
||||
return Config(routes=()) # orchestrator unreachable/errored → deny
|
||||
return _config_from_policy(policy)
|
||||
|
||||
|
||||
@@ -489,7 +457,7 @@ def resolve_client_context(
|
||||
client_ip, identity_token,
|
||||
)
|
||||
except Exception: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
||||
return Config(routes=(), deny_reason=DENY_RESOLVER_ERROR), "", {}
|
||||
return Config(routes=()), "", {} # orchestrator unreachable/errored → deny
|
||||
return _config_from_policy(policy), (bottle_id or ""), tokens
|
||||
|
||||
|
||||
@@ -604,16 +572,12 @@ def decide(
|
||||
*,
|
||||
request_method: str = "GET",
|
||||
request_headers: typing.Mapping[str, str] | None = None,
|
||||
deny_reason: str = "",
|
||||
) -> Decision:
|
||||
"""`deny_reason` is `Config.deny_reason`: when the deny-all came from a
|
||||
missing/unparseable policy rather than the bottle's own allowlist, report
|
||||
that instead of implying a route is merely absent."""
|
||||
route = match_route(routes, request_host)
|
||||
if route is None:
|
||||
return Decision(
|
||||
action="block",
|
||||
reason=deny_reason or (
|
||||
reason=(
|
||||
f"egress: host {request_host!r} is not in the "
|
||||
f"bottle's egress.routes allowlist. Declare a "
|
||||
f"route for it or remove the request."
|
||||
@@ -888,9 +852,6 @@ __all__ = [
|
||||
"is_git_push_request",
|
||||
"is_git_fetch_request",
|
||||
"load_config",
|
||||
"DENY_UNATTRIBUTED",
|
||||
"DENY_UNPARSEABLE",
|
||||
"DENY_RESOLVER_ERROR",
|
||||
"resolve_client_config",
|
||||
"resolve_client_context",
|
||||
"PolicyResolverLike",
|
||||
|
||||
@@ -15,7 +15,6 @@ from __future__ import annotations
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..paths import host_control_plane_token
|
||||
@@ -148,20 +147,6 @@ class OrchestratorClient:
|
||||
raise OrchestratorClientError(f"teardown {bottle_id}: HTTP {status}")
|
||||
return True
|
||||
|
||||
def reconcile(
|
||||
self, live_source_ips: Iterable[str], *, grace_seconds: float | None = None,
|
||||
) -> list[str]:
|
||||
"""Drop registry rows for bottles that are no longer running
|
||||
(`POST /reconcile`), returning the reaped bottle ids. `live_source_ips`
|
||||
is the caller's enumeration of its live bottles — the orchestrator
|
||||
can't see the backend from inside the infra container."""
|
||||
body: dict[str, object] = {"live_source_ips": list(live_source_ips)}
|
||||
if grace_seconds is not None:
|
||||
body["grace_seconds"] = grace_seconds
|
||||
payload = self._ok("POST", "/reconcile", body)
|
||||
reaped = payload.get("reaped")
|
||||
return [r for r in reaped if isinstance(r, str)] if isinstance(reaped, list) else []
|
||||
|
||||
def set_policy(self, bottle_id: str, policy: str) -> bool:
|
||||
"""Live-reload a bottle's policy (`PUT /bottles/<id>/policy`). False on
|
||||
404 (unknown bottle)."""
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
"""Per-host orchestrator configuration store (settings in bot-bottle.db).
|
||||
|
||||
Co-tenants the shared `bot-bottle.db` via the `DbStore` framework. Settings
|
||||
are readable by the host launch path directly (no HTTP round-trip to the
|
||||
orchestrator), so they take effect even before the orchestrator is reachable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from ..db_store import DbStore
|
||||
from ..migrations import TableMigrations
|
||||
from ..paths import host_db_path
|
||||
|
||||
TEARDOWN_TIMEOUT_ENV = "BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS"
|
||||
DEFAULT_TEARDOWN_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
_MIGRATIONS = TableMigrations(
|
||||
"orchestrator_config",
|
||||
[
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS orchestrator_config (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
teardown_timeout_seconds REAL
|
||||
)
|
||||
""",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class OrchestratorConfigStore(DbStore):
|
||||
"""Orchestrator settings in the shared host DB."""
|
||||
|
||||
def __init__(self, db_path: Path | None = None) -> None:
|
||||
super().__init__(db_path or host_db_path(), _MIGRATIONS)
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
conn = super()._connect()
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
return conn
|
||||
|
||||
def get_teardown_timeout_seconds(self) -> float | None:
|
||||
"""Return the configured teardown timeout, or None if not set."""
|
||||
try:
|
||||
with self._connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT teardown_timeout_seconds FROM orchestrator_config WHERE id = 1"
|
||||
).fetchone()
|
||||
except sqlite3.OperationalError:
|
||||
return None
|
||||
return row["teardown_timeout_seconds"] if row else None
|
||||
|
||||
def set_teardown_timeout_seconds(self, value: float) -> None:
|
||||
"""Persist the teardown timeout."""
|
||||
with self._connection() as conn:
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO orchestrator_config"
|
||||
" (id, teardown_timeout_seconds) VALUES (1, ?)",
|
||||
(value,),
|
||||
)
|
||||
self._chmod()
|
||||
|
||||
def delete_teardown_timeout_seconds(self) -> bool:
|
||||
"""Clear the stored teardown timeout. Returns True if a value existed."""
|
||||
with self._connection() as conn:
|
||||
cur = conn.execute(
|
||||
"UPDATE orchestrator_config SET teardown_timeout_seconds = NULL"
|
||||
" WHERE id = 1 AND teardown_timeout_seconds IS NOT NULL"
|
||||
)
|
||||
return cur.rowcount > 0
|
||||
|
||||
|
||||
def resolve_teardown_timeout(db_path: Path | None = None) -> float:
|
||||
"""Return the teardown timeout to use, in priority order:
|
||||
|
||||
1. ``BOT_BOTTLE_ORCHESTRATOR_TEARDOWN_TIMEOUT_SECONDS`` env var
|
||||
2. ``teardown_timeout_seconds`` in the orchestrator config DB
|
||||
3. ``DEFAULT_TEARDOWN_TIMEOUT_SECONDS`` (30 s)
|
||||
"""
|
||||
raw = os.environ.get(TEARDOWN_TIMEOUT_ENV, "").strip()
|
||||
if raw:
|
||||
try:
|
||||
value = float(raw)
|
||||
if value > 0:
|
||||
return value
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
store = OrchestratorConfigStore(db_path)
|
||||
if not store.is_migrated():
|
||||
store.migrate()
|
||||
db_value = store.get_teardown_timeout_seconds()
|
||||
if db_value is not None and db_value > 0:
|
||||
return db_value
|
||||
|
||||
return DEFAULT_TEARDOWN_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
__all__ = [
|
||||
"OrchestratorConfigStore",
|
||||
"resolve_teardown_timeout",
|
||||
"TEARDOWN_TIMEOUT_ENV",
|
||||
"DEFAULT_TEARDOWN_TIMEOUT_SECONDS",
|
||||
]
|
||||
@@ -13,9 +13,6 @@ vsock / unix-socket portability caveats):
|
||||
PUT /bottles/<bottle_id>/policy -> 200 {"updated": true} | 404 (live reload)
|
||||
body: {"policy"}
|
||||
DELETE /bottles/<bottle_id> -> 200 {"torn_down": true} | 404 (teardown)
|
||||
POST /reconcile -> 200 {"reaped": [bottle_id, ...]}
|
||||
body: {"live_source_ips": [...],
|
||||
["grace_seconds"]}
|
||||
POST /attribute -> 200 {"bottle_id"} | 403
|
||||
POST /resolve -> 200 {"bottle_id","policy"} | 403
|
||||
body: {"source_ip","identity_token"}
|
||||
@@ -144,27 +141,6 @@ def dispatch( # pylint: disable=too-many-return-statements,too-many-branches
|
||||
return 200, {"torn_down": True}
|
||||
return 404, {"error": "no such bottle"}
|
||||
|
||||
if method == "POST" and route == "/reconcile":
|
||||
# Host-driven self-heal: the caller enumerates its live bottles (only
|
||||
# the host can see the backend) and the orchestrator drops rows for
|
||||
# every other active bottle. Trusted-caller only — an agent that could
|
||||
# reach this would be able to unregister its neighbours.
|
||||
try:
|
||||
data = _parse_json_object(body)
|
||||
except ValueError as e:
|
||||
return 400, {"error": f"invalid JSON: {e}"}
|
||||
raw_ips = data.get("live_source_ips")
|
||||
if not isinstance(raw_ips, list):
|
||||
return 400, {"error": "live_source_ips (list of strings) is required"}
|
||||
live = [ip for ip in raw_ips if isinstance(ip, str) and ip]
|
||||
grace = data.get("grace_seconds")
|
||||
kwargs = (
|
||||
{"grace_seconds": float(grace)}
|
||||
if isinstance(grace, (int, float)) and not isinstance(grace, bool)
|
||||
else {}
|
||||
)
|
||||
return 200, {"reaped": orch.reconcile(live, **kwargs)}
|
||||
|
||||
if method == "POST" and route == "/attribute":
|
||||
try:
|
||||
data = _parse_json_object(body)
|
||||
|
||||
@@ -32,7 +32,6 @@ import hmac
|
||||
import secrets
|
||||
import sqlite3
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
@@ -43,12 +42,6 @@ from ..paths import host_db_path
|
||||
# 256 bits of urandom, URL-safe — unguessable per-bottle identity token.
|
||||
IDENTITY_TOKEN_BYTES = 32
|
||||
|
||||
# How recently a row must have been registered to be exempt from
|
||||
# `reap_absent`. Covers the window between `container run` and the address
|
||||
# becoming visible to another launch's enumeration, so reconciliation never
|
||||
# reaps a bottle that is still coming up.
|
||||
DEFAULT_REAP_GRACE_SECONDS = 120.0
|
||||
|
||||
|
||||
def new_identity_token() -> str:
|
||||
"""A fresh per-bottle identity token (PRD 0070 attribution defence)."""
|
||||
@@ -232,70 +225,6 @@ class RegistryStore(DbStore):
|
||||
).fetchall()
|
||||
return [_row_to_record(r) for r in rows]
|
||||
|
||||
def reap_absent(
|
||||
self,
|
||||
live_source_ips: Iterable[str],
|
||||
*,
|
||||
grace_seconds: float = DEFAULT_REAP_GRACE_SECONDS,
|
||||
now: float | None = None,
|
||||
) -> list[BottleRecord]:
|
||||
"""Delete active rows whose source IP is not held by a live bottle.
|
||||
|
||||
A row only ever leaves the registry two ways: an explicit
|
||||
`teardown_bottle` (the launcher's cleanup callback) or the supersede
|
||||
sweep in `register`. Neither runs when the launching CLI dies hard —
|
||||
SIGKILL, a closed terminal, a host sleep/crash — so the row outlives
|
||||
its container. That orphan is not inert: source IPs are recycled by
|
||||
the backend's DHCP, and `by_source_ip` fail-closes on ambiguity, so a
|
||||
leftover row at a reused address can brick the *next* bottle that
|
||||
lands on it (no policy resolved -> every host denied, reported to the
|
||||
agent as "not in the allowlist"). Reconciling against the live set at
|
||||
launch keeps the registry from accumulating those landmines.
|
||||
|
||||
Restores the invariant the data plane needs: **at most one active row
|
||||
per live address, and none at all for a dead one.** Two cases, because
|
||||
a dead bottle's address may already have been handed to a live one:
|
||||
|
||||
* no live bottle holds the address — every row there is an orphan;
|
||||
* a live bottle holds it but several rows claim it — the newest
|
||||
registration is authoritative and the rest are orphans, the same
|
||||
rule `register`'s same-IP supersede sweep applies. Without this
|
||||
second case a recycled address stays ambiguous, which is exactly
|
||||
the state that resolves no policy.
|
||||
|
||||
`grace_seconds` protects an in-flight launch: registration happens
|
||||
moments after `container run`, and a concurrent launch's address may
|
||||
not be visible to the caller's enumeration yet. Rows younger than the
|
||||
grace window are never reaped, so reconciliation can't race a bottle
|
||||
that is still coming up. Returns the deleted records."""
|
||||
live = {ip for ip in live_source_ips if ip}
|
||||
cutoff = (time.time() if now is None else now) - grace_seconds
|
||||
with self._connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM orchestrator_bottles WHERE state = 'active'",
|
||||
).fetchall()
|
||||
by_ip: dict[str, list[BottleRecord]] = {}
|
||||
for row in rows:
|
||||
rec = _row_to_record(row)
|
||||
by_ip.setdefault(rec.source_ip, []).append(rec)
|
||||
candidates: list[BottleRecord] = []
|
||||
for ip, recs in by_ip.items():
|
||||
if ip not in live:
|
||||
candidates.extend(recs)
|
||||
continue
|
||||
# Keep the newest claim on a live address; supersede the rest.
|
||||
recs.sort(key=lambda r: r.created_at)
|
||||
candidates.extend(recs[:-1])
|
||||
doomed = [r for r in candidates if r.created_at <= cutoff]
|
||||
for rec in doomed:
|
||||
conn.execute(
|
||||
"DELETE FROM orchestrator_bottles WHERE bottle_id = ?",
|
||||
(rec.bottle_id,),
|
||||
)
|
||||
if doomed:
|
||||
self._chmod()
|
||||
return doomed
|
||||
|
||||
def by_source_ip(self, source_ip: str) -> BottleRecord | None:
|
||||
"""Network-layer attribution: the single active bottle at this source
|
||||
IP, or None if unknown or ambiguous (more than one — a
|
||||
@@ -333,5 +262,4 @@ __all__ = [
|
||||
"new_identity_token",
|
||||
"default_db_path",
|
||||
"IDENTITY_TOKEN_BYTES",
|
||||
"DEFAULT_REAP_GRACE_SECONDS",
|
||||
]
|
||||
|
||||
@@ -13,20 +13,15 @@ Launch lifecycle:
|
||||
and returns the record. If the broker rejects/fails, the registry entry
|
||||
is rolled back so a failed launch leaves no orphan.
|
||||
* `teardown_bottle` sends a signed teardown request, then deregisters.
|
||||
* `reconcile` sweeps rows whose bottle is no longer running — the
|
||||
self-heal for the teardown paths that never got to run (a hard-killed
|
||||
launcher), since an orphan row at a recycled source IP bricks the next
|
||||
bottle that lands on it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from .broker import LaunchBroker, LaunchRequest, sign_request
|
||||
from .registry import DEFAULT_REAP_GRACE_SECONDS, BottleRecord, RegistryStore
|
||||
from .registry import BottleRecord, RegistryStore
|
||||
from .gateway import Gateway
|
||||
from ..supervise import (
|
||||
AuditEntry,
|
||||
@@ -122,30 +117,6 @@ class Orchestrator:
|
||||
self._tokens.pop(bottle_id, None)
|
||||
return True
|
||||
|
||||
def reconcile(
|
||||
self,
|
||||
live_source_ips: Iterable[str],
|
||||
*,
|
||||
grace_seconds: float = DEFAULT_REAP_GRACE_SECONDS,
|
||||
) -> list[str]:
|
||||
"""Drop registry rows for bottles that are no longer running, and
|
||||
forget their in-memory egress tokens. Returns the reaped bottle ids.
|
||||
|
||||
The caller supplies the live set because only the host can enumerate
|
||||
its own containers — the orchestrator runs *inside* the infra
|
||||
container and has no view of the backend. Deliberately does not
|
||||
broker a teardown: the container is already gone, so there is nothing
|
||||
to stop, and a broker error must not stop the sweep from clearing
|
||||
the row that would otherwise brick the next bottle at that address.
|
||||
|
||||
See `RegistryStore.reap_absent` for why orphans accumulate and why
|
||||
they are harmful rather than merely untidy."""
|
||||
reaped = self.registry.reap_absent(
|
||||
live_source_ips, grace_seconds=grace_seconds)
|
||||
for rec in reaped:
|
||||
self._tokens.pop(rec.bottle_id, None)
|
||||
return [rec.bottle_id for rec in reaped]
|
||||
|
||||
def tokens_for(self, bottle_id: str) -> dict[str, str]:
|
||||
"""The bottle's in-memory egress auth tokens (env_name -> value), or
|
||||
empty. The gateway injects these per request; they are never
|
||||
|
||||
@@ -4,14 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from bot_bottle.egress_addon_core import (
|
||||
DENY_RESOLVER_ERROR,
|
||||
DENY_UNATTRIBUTED,
|
||||
DENY_UNPARSEABLE,
|
||||
decide,
|
||||
resolve_client_config,
|
||||
resolve_client_context,
|
||||
)
|
||||
from bot_bottle.egress_addon_core import resolve_client_config, resolve_client_context
|
||||
from bot_bottle.policy_resolver import PolicyResolveError
|
||||
|
||||
|
||||
@@ -115,55 +108,3 @@ class TestResolveClientContext(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestDenyReasonNamesTheRealFault(unittest.TestCase):
|
||||
"""A deny-all must not masquerade as a missing allowlist entry.
|
||||
|
||||
Regression: an unregistered bottle resolves no policy, so *every* host is
|
||||
denied — but the block message said `host X is not in the allowlist`,
|
||||
which reads as a config problem and sends the operator hunting for a route
|
||||
that was never missing. The structural reason wins over that wording.
|
||||
"""
|
||||
|
||||
def _reason(self, resolver: object, host: str = "chatgpt.com") -> str:
|
||||
cfg = resolve_client_config(resolver, "10.243.0.1") # type: ignore[arg-type]
|
||||
return decide(cfg.routes, host, "/v1/x", {}, deny_reason=cfg.deny_reason).reason
|
||||
|
||||
def test_unattributed_says_unattributed_not_allowlist(self) -> None:
|
||||
reason = self._reason(_FakeResolver(result=None))
|
||||
self.assertEqual(DENY_UNATTRIBUTED, reason)
|
||||
# The misleading claim is the one that must be gone: the host was
|
||||
# never "not in the allowlist" — there was no allowlist at all.
|
||||
self.assertNotIn("is not in the bottle's egress.routes allowlist", reason)
|
||||
# Both causes must be named. `/resolve` fail-closes on a missing row
|
||||
# *and* on a token mismatch, and the message pointing only at the row
|
||||
# sent us hunting for a deregistered bottle that was registered fine.
|
||||
self.assertIn("registry row", reason)
|
||||
self.assertIn("identity token", reason)
|
||||
|
||||
def test_resolver_error_says_orchestrator_unreachable(self) -> None:
|
||||
self.assertEqual(DENY_RESOLVER_ERROR, self._reason(_FakeResolver(raises=True)))
|
||||
|
||||
def test_unparseable_policy_says_so(self) -> None:
|
||||
self.assertEqual(
|
||||
DENY_UNPARSEABLE, self._reason(_FakeResolver(result="routes: notalist\n")))
|
||||
|
||||
def test_a_real_allowlist_miss_keeps_the_allowlist_wording(self) -> None:
|
||||
"""The message only changes for structural deny-alls — a loaded policy
|
||||
that genuinely lacks the host still points at the allowlist."""
|
||||
reason = self._reason(_FakeResolver(result='routes:\n - host: "api.example.com"\n'))
|
||||
self.assertIn("is not in the bottle's egress.routes allowlist", reason)
|
||||
self.assertIn("chatgpt.com", reason)
|
||||
|
||||
def test_allowed_host_is_still_forwarded(self) -> None:
|
||||
cfg = resolve_client_config(
|
||||
_FakeResolver(result='routes:\n - host: "api.example.com"\n'), "10.243.0.1")
|
||||
decision = decide(
|
||||
cfg.routes, "api.example.com", "/v1/x", {}, deny_reason=cfg.deny_reason)
|
||||
self.assertEqual("forward", decision.action)
|
||||
|
||||
def test_a_parsed_policy_carries_no_deny_reason(self) -> None:
|
||||
cfg = resolve_client_config(
|
||||
_FakeResolver(result='routes:\n - host: "api.example.com"\n'), "10.243.0.1")
|
||||
self.assertEqual("", cfg.deny_reason)
|
||||
|
||||
@@ -88,8 +88,7 @@ class TestRegisterAgent(unittest.TestCase):
|
||||
*, source_ip: str = "192.168.128.9",
|
||||
):
|
||||
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
||||
patch(f"{_UTIL}.provision_git_gate", provision or Mock()), \
|
||||
patch(f"{_MOD}.live_source_ips", return_value=[]):
|
||||
patch(f"{_UTIL}.provision_git_gate", provision or Mock()):
|
||||
return register_agent(
|
||||
_egress_plan(), _git_plan(),
|
||||
source_ip=source_ip, endpoint=_endpoint(), image_ref="img:1",
|
||||
@@ -136,104 +135,3 @@ class TestTeardown(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestLiveSourceIps(unittest.TestCase):
|
||||
"""The reconciliation input: the host enumerates its own bottles because
|
||||
the orchestrator, inside the infra container, cannot see the backend."""
|
||||
|
||||
def _agents(self, *slugs: str) -> list[Mock]:
|
||||
return [Mock(slug=s) for s in slugs]
|
||||
|
||||
def test_maps_slugs_to_container_addresses(self) -> None:
|
||||
from bot_bottle.backend.macos_container.consolidated_launch import live_source_ips
|
||||
with patch(f"{_MOD}.enumerate_active", return_value=self._agents("a", "b")), \
|
||||
patch(f"{_MOD}.container_mod.inspect_container_network_ip",
|
||||
side_effect=["10.0.0.1", "10.0.0.2"]) as ip:
|
||||
got = live_source_ips("net0")
|
||||
self.assertEqual(["10.0.0.1", "10.0.0.2"], got)
|
||||
self.assertEqual("bot-bottle-a", ip.call_args_list[0].args[0])
|
||||
|
||||
def test_containers_without_an_address_are_skipped(self) -> None:
|
||||
"""A container that hasn't been given a DHCP address yet contributes
|
||||
nothing — the reap's grace window, not this list, protects it."""
|
||||
from bot_bottle.backend.macos_container.consolidated_launch import live_source_ips
|
||||
with patch(f"{_MOD}.enumerate_active", return_value=self._agents("a", "b")), \
|
||||
patch(f"{_MOD}.container_mod.inspect_container_network_ip",
|
||||
side_effect=["", "10.0.0.2"]):
|
||||
self.assertEqual(["10.0.0.2"], live_source_ips("net0"))
|
||||
|
||||
def test_container_list_failure_raises(self) -> None:
|
||||
"""If container list fails, the live set is not authoritative and
|
||||
reconciliation must be skipped."""
|
||||
from bot_bottle.backend.macos_container.consolidated_launch import live_source_ips
|
||||
from bot_bottle.backend.macos_container.enumerate import EnumerationError
|
||||
with patch(f"{_MOD}.enumerate_active",
|
||||
side_effect=EnumerationError("container list failed")):
|
||||
with self.assertRaises(EnumerationError):
|
||||
live_source_ips("net0")
|
||||
|
||||
def test_per_container_inspect_failure_raises(self) -> None:
|
||||
"""If any individual inspect fails, the live set is not authoritative."""
|
||||
from bot_bottle.backend.macos_container.consolidated_launch import live_source_ips
|
||||
from bot_bottle.backend.macos_container.enumerate import EnumerationError
|
||||
with patch(f"{_MOD}.enumerate_active", return_value=self._agents("a", "b")), \
|
||||
patch(f"{_MOD}.container_mod.inspect_container_network_ip",
|
||||
side_effect=["10.0.0.1", None]):
|
||||
with self.assertRaises(EnumerationError):
|
||||
live_source_ips("net0")
|
||||
|
||||
|
||||
class TestRegisterAgentReconciles(unittest.TestCase):
|
||||
"""Registration self-heals the registry first: an orphan row at a recycled
|
||||
address makes attribution ambiguous, which resolves no policy at all and
|
||||
denies every host for the bottle being launched."""
|
||||
|
||||
def _register(self, client: Mock) -> None:
|
||||
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
||||
patch(f"{_UTIL}.provision_git_gate"), \
|
||||
patch(f"{_MOD}.live_source_ips", return_value=["10.0.0.7"]):
|
||||
register_agent(
|
||||
_egress_plan(), _git_plan(),
|
||||
source_ip="10.0.0.7", endpoint=_endpoint(),
|
||||
)
|
||||
|
||||
def test_reconciles_before_registering(self) -> None:
|
||||
client = _client()
|
||||
calls: list[str] = []
|
||||
|
||||
def _reconcile(*_args: object, **_kwargs: object) -> list[str]:
|
||||
calls.append("reconcile")
|
||||
return []
|
||||
|
||||
def _register_bottle(*_args: object, **_kwargs: object) -> RegisteredBottle:
|
||||
calls.append("register")
|
||||
return RegisteredBottle("b1", "tok")
|
||||
|
||||
client.reconcile.side_effect = _reconcile
|
||||
client.register_bottle.side_effect = _register_bottle
|
||||
self._register(client)
|
||||
self.assertEqual(["reconcile", "register"], calls)
|
||||
client.reconcile.assert_called_once_with(["10.0.0.7"])
|
||||
|
||||
def test_a_reconcile_failure_does_not_block_the_launch(self) -> None:
|
||||
from bot_bottle.orchestrator.client import OrchestratorClientError
|
||||
client = _client()
|
||||
client.reconcile.side_effect = OrchestratorClientError("unreachable")
|
||||
self._register(client)
|
||||
client.register_bottle.assert_called_once()
|
||||
|
||||
def test_enumeration_error_does_not_block_the_launch(self) -> None:
|
||||
"""A partial container listing must not abort the launch — skip
|
||||
reconciliation and proceed, just as with an unreachable orchestrator."""
|
||||
from bot_bottle.backend.macos_container.enumerate import EnumerationError
|
||||
client = _client()
|
||||
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
||||
patch(f"{_UTIL}.provision_git_gate"), \
|
||||
patch(f"{_MOD}.live_source_ips",
|
||||
side_effect=EnumerationError("container list failed")):
|
||||
register_agent(
|
||||
_egress_plan(), _git_plan(),
|
||||
source_ip="10.0.0.7", endpoint=_endpoint(),
|
||||
)
|
||||
client.register_bottle.assert_called_once()
|
||||
|
||||
@@ -66,10 +66,8 @@ class TestMacosContainerEnumerate(unittest.TestCase):
|
||||
agents = self._enumerate("bot-bottle-mac-infra\nbot-bottle-dev-abc\n")
|
||||
self.assertEqual(["dev-abc"], [a.slug for a in agents])
|
||||
|
||||
def test_raises_when_the_cli_fails(self):
|
||||
from bot_bottle.backend.macos_container.enumerate import EnumerationError
|
||||
with self.assertRaises(EnumerationError):
|
||||
self._enumerate("", returncode=1)
|
||||
def test_empty_when_the_cli_fails(self):
|
||||
self.assertEqual([], self._enumerate("", returncode=1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -17,10 +17,10 @@ from unittest.mock import patch
|
||||
from bot_bottle.backend.macos_container.bottle import MacosContainerBottle
|
||||
from bot_bottle.backend.macos_container.bottle_plan import MacosContainerBottlePlan
|
||||
from bot_bottle.backend.macos_container.consolidated_launch import GatewayEndpoint
|
||||
from bot_bottle.backend.macos_container.gateway_hosts import GATEWAY_HOSTNAME
|
||||
from bot_bottle.backend.macos_container.launch import (
|
||||
_agent_run_argv,
|
||||
_identity_proxy_env,
|
||||
_proxy_url,
|
||||
)
|
||||
from bot_bottle.manifest import ManifestIndex
|
||||
|
||||
@@ -104,38 +104,19 @@ class TestAgentRunArgv(unittest.TestCase):
|
||||
read back after start."""
|
||||
self.assertNotIn("--ip", self.argv)
|
||||
|
||||
def test_run_time_env_sets_no_proxy_vars_at_all(self) -> None:
|
||||
"""Regression: the proxy vars must exist *only* in the exec-time env.
|
||||
|
||||
`container exec --env` appends rather than replaces, so a token-less
|
||||
`HTTPS_PROXY` here would survive next to the token-bearing one and
|
||||
leave two entries in the agent's `environ`. Resolution is then
|
||||
runtime-specific — Node reads the last, Rust's `std::env::var` reads
|
||||
the first — so Codex proxied without its identity token and every
|
||||
request fail-closed at `/resolve`.
|
||||
"""
|
||||
for entry in self.argv:
|
||||
self.assertFalse(
|
||||
entry.upper().startswith(("HTTP_PROXY=", "HTTPS_PROXY=")),
|
||||
f"run-time env must not set a proxy var, got {entry!r}",
|
||||
)
|
||||
|
||||
def test_run_time_env_carries_no_identity_token(self) -> None:
|
||||
"""The token is minted by registration, which happens after this run,
|
||||
so it cannot be here under any name."""
|
||||
self.assertNotIn("bottle:", " ".join(self.argv))
|
||||
def test_run_time_proxy_carries_no_identity_token(self) -> None:
|
||||
"""The token is minted by registration, which happens after this run —
|
||||
so it cannot be here. `/resolve` denies the token-less pair (#366),
|
||||
which is the safe direction; the real value arrives at exec time."""
|
||||
joined = " ".join(self.argv)
|
||||
self.assertIn(f"HTTP_PROXY={_proxy_url('192.168.128.3')}", joined)
|
||||
self.assertNotIn("bottle:", joined)
|
||||
|
||||
def test_gateway_bypasses_the_proxy(self) -> None:
|
||||
"""git-http + supervise live on the gateway and must be reached
|
||||
directly, not through its own egress proxy."""
|
||||
entry = next(a for a in self.argv if a.startswith("NO_PROXY="))
|
||||
self.assertIn(GATEWAY_HOSTNAME, entry)
|
||||
|
||||
def test_no_proxy_names_the_gateway_and_never_addresses_it(self) -> None:
|
||||
"""NO_PROXY is baked into the run-time env, so an address here is as
|
||||
unfixable as the proxy URL if the gateway moves."""
|
||||
entry = next(a for a in self.argv if a.startswith("NO_PROXY="))
|
||||
self.assertNotIn("192.168.128.3", entry)
|
||||
self.assertIn("192.168.128.3", entry)
|
||||
|
||||
def test_forwarded_secrets_stay_off_argv(self) -> None:
|
||||
"""Bare name → inherited from the run process env, so the value never
|
||||
@@ -153,7 +134,7 @@ class TestIdentityTokenDelivery(unittest.TestCase):
|
||||
def test_exec_env_carries_the_token_as_proxy_credentials(self) -> None:
|
||||
env = _identity_proxy_env(_endpoint(), "s3cret")
|
||||
self.assertEqual(
|
||||
f"http://bottle:s3cret@{GATEWAY_HOSTNAME}:9099", env["HTTP_PROXY"],
|
||||
"http://bottle:s3cret@192.168.128.3:9099", env["HTTP_PROXY"],
|
||||
)
|
||||
self.assertEqual(env["HTTP_PROXY"], env["https_proxy"])
|
||||
|
||||
@@ -221,7 +202,7 @@ class TestPlanIdentityToken(unittest.TestCase):
|
||||
self.assertIn("HTTP_PROXY", argv)
|
||||
self.assertNotIn("s3cret", " ".join(argv))
|
||||
self.assertEqual(
|
||||
f"http://bottle:s3cret@{GATEWAY_HOSTNAME}:9099", kwargs["env"]["HTTP_PROXY"],
|
||||
"http://bottle:s3cret@192.168.128.3:9099", kwargs["env"]["HTTP_PROXY"],
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -272,30 +272,6 @@ resolver #2
|
||||
),
|
||||
)
|
||||
|
||||
def test_exec_container_as_root_selects_root_user(self):
|
||||
completed = util.subprocess.CompletedProcess(
|
||||
args=[], returncode=0, stdout="", stderr="",
|
||||
)
|
||||
with patch.object(util, "_run_container_op", return_value=completed) as run:
|
||||
util.exec_container_as_root("bot-bottle-demo", ["true"])
|
||||
|
||||
run.assert_called_once_with([
|
||||
"container", "exec", "--user", "root", "bot-bottle-demo", "true",
|
||||
])
|
||||
|
||||
def test_exec_container_as_root_reports_failure(self):
|
||||
failed = util.subprocess.CompletedProcess(
|
||||
args=[], returncode=1, stdout="", stderr="permission denied\n",
|
||||
)
|
||||
with patch.object(util, "_run_container_op", return_value=failed), \
|
||||
patch.object(util, "die", side_effect=SystemExit("die")) as die:
|
||||
with self.assertRaises(SystemExit):
|
||||
util.exec_container_as_root("bot-bottle-demo", ["true"])
|
||||
|
||||
die.assert_called_once_with(
|
||||
"container exec (root) in bot-bottle-demo failed: permission denied",
|
||||
)
|
||||
|
||||
|
||||
def _completed(stdout: str, returncode: int = 0):
|
||||
return util.subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr="")
|
||||
@@ -334,50 +310,6 @@ class TestInspectDigests(unittest.TestCase):
|
||||
self.assertEqual({}, util.container_env("x"))
|
||||
|
||||
|
||||
class TestInspectContainerNetworkIp(unittest.TestCase):
|
||||
"""inspect_container_network_ip must distinguish inspect failure (None)
|
||||
from 'no DHCP address yet' (""), which is the invariant live_source_ips
|
||||
relies on to skip reconciliation on partial snapshots."""
|
||||
|
||||
_NETWORK = "bot-bottle-mac-gateway"
|
||||
|
||||
def _inspect(self, stdout: str, returncode: int = 0) -> str | None:
|
||||
cp = util.subprocess.CompletedProcess(
|
||||
args=[], returncode=returncode, stdout=stdout, stderr="",
|
||||
)
|
||||
with patch.object(util.subprocess, "run", return_value=cp):
|
||||
return util.inspect_container_network_ip("bot-bottle-abc", self._NETWORK)
|
||||
|
||||
def _entry(self, ip: str = "192.168.128.5") -> str:
|
||||
return (
|
||||
f'[{{"status":{{"networks":['
|
||||
f'{{"network":"{self._NETWORK}","ipv4Address":"{ip}"}}'
|
||||
f']}}}}]'
|
||||
)
|
||||
|
||||
def test_returns_ip_when_inspect_succeeds(self) -> None:
|
||||
self.assertEqual("192.168.128.5", self._inspect(self._entry()))
|
||||
|
||||
def test_strips_cidr_prefix(self) -> None:
|
||||
self.assertEqual("192.168.128.5", self._inspect(self._entry("192.168.128.5/24")))
|
||||
|
||||
def test_returns_empty_string_when_no_address_assigned_yet(self) -> None:
|
||||
no_ip = f'[{{"status":{{"networks":[{{"network":"{self._NETWORK}","ipv4Address":""}}]}}}}]'
|
||||
self.assertEqual("", self._inspect(no_ip))
|
||||
|
||||
def test_returns_empty_string_when_network_list_absent(self) -> None:
|
||||
self.assertEqual("", self._inspect('[{"status":{}}]'))
|
||||
|
||||
def test_returns_none_on_nonzero_exit(self) -> None:
|
||||
self.assertIsNone(self._inspect("", returncode=1))
|
||||
|
||||
def test_returns_none_on_malformed_json(self) -> None:
|
||||
self.assertIsNone(self._inspect("not-json"))
|
||||
|
||||
def test_returns_none_on_unexpected_json_shape(self) -> None:
|
||||
self.assertIsNone(self._inspect("null"))
|
||||
|
||||
|
||||
class TestWaitContainerIpv4(unittest.TestCase):
|
||||
def test_returns_address_once_dhcp_assigns_it(self):
|
||||
with patch.object(util, "try_container_ipv4_on_network", side_effect=["", "", "192.168.128.4"]), \
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
"""Unit: stable gateway name via each bottle's /etc/hosts (issue #443).
|
||||
|
||||
The gateway's address moves whenever the infra container is recreated. Agents
|
||||
name it instead of addressing it, and the name resolves through `/etc/hosts` —
|
||||
a file, so it stays rewritable while the bottle runs, unlike the `environ` the
|
||||
proxy URL is delivered in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.backend.macos_container.gateway_hosts import (
|
||||
GATEWAY_HOSTNAME,
|
||||
refresh_gateway_host,
|
||||
set_gateway_host,
|
||||
)
|
||||
|
||||
_MOD = "bot_bottle.backend.macos_container.gateway_hosts"
|
||||
|
||||
|
||||
class TestSetGatewayHost(unittest.TestCase):
|
||||
def _script(self, exec_root: object) -> str:
|
||||
argv = exec_root.call_args.args[1] # type: ignore[attr-defined]
|
||||
self.assertEqual(["sh", "-c"], argv[:2])
|
||||
return argv[2]
|
||||
|
||||
def test_writes_the_address_against_the_stable_name(self) -> None:
|
||||
with patch(f"{_MOD}.container_mod.exec_container_as_root") as ex:
|
||||
set_gateway_host("bot-bottle-demo", "192.168.128.19")
|
||||
self.assertEqual("bot-bottle-demo", ex.call_args.args[0])
|
||||
script = self._script(ex)
|
||||
self.assertIn("192.168.128.19", script)
|
||||
self.assertIn(GATEWAY_HOSTNAME, script)
|
||||
|
||||
def test_runs_as_root_so_the_agent_cannot_repoint_itself(self) -> None:
|
||||
"""The agent runs as `node`. If it could rewrite /etc/hosts it could
|
||||
aim its own gateway name elsewhere, so the write must go through the
|
||||
root-only helper."""
|
||||
with patch(f"{_MOD}.container_mod.exec_container_as_root") as ex:
|
||||
set_gateway_host("bot-bottle-demo", "10.0.0.1")
|
||||
ex.assert_called_once()
|
||||
|
||||
def test_is_idempotent_by_removing_its_own_line_first(self) -> None:
|
||||
"""Re-pointing must replace the managed entry, not append a second one
|
||||
— two entries for the same name would resolve by luck of ordering."""
|
||||
with patch(f"{_MOD}.container_mod.exec_container_as_root") as ex:
|
||||
set_gateway_host("bot-bottle-demo", "10.0.0.1")
|
||||
self.assertIn("grep -v", self._script(ex))
|
||||
|
||||
def test_preserves_the_rest_of_the_hosts_file(self) -> None:
|
||||
"""localhost and the container's own name must survive the rewrite."""
|
||||
with patch(f"{_MOD}.container_mod.exec_container_as_root") as ex:
|
||||
set_gateway_host("bot-bottle-demo", "10.0.0.1")
|
||||
script = self._script(ex)
|
||||
# Filter-and-append, never a truncating write of just our line.
|
||||
self.assertIn("/etc/hosts >", script)
|
||||
self.assertIn(">> /tmp/.bb-hosts", script)
|
||||
|
||||
def test_keeps_the_original_inode(self) -> None:
|
||||
"""`cat >` rather than `mv`: a pre-created /etc/hosts must keep its
|
||||
ownership and mode, not be replaced by a root-owned copy."""
|
||||
script = None
|
||||
with patch(f"{_MOD}.container_mod.exec_container_as_root") as ex:
|
||||
set_gateway_host("bot-bottle-demo", "10.0.0.1")
|
||||
script = self._script(ex)
|
||||
self.assertIn("cat /tmp/.bb-hosts > /etc/hosts", script)
|
||||
self.assertNotIn("mv ", script)
|
||||
|
||||
|
||||
class TestRefreshGatewayHost(unittest.TestCase):
|
||||
"""The re-attach sweep: bottles stranded by an earlier gateway restart get
|
||||
re-pointed in place instead of needing a relaunch."""
|
||||
|
||||
def _agents(self, *slugs: str) -> list[SimpleNamespace]:
|
||||
return [SimpleNamespace(slug=s) for s in slugs]
|
||||
|
||||
def test_repoints_every_running_bottle(self) -> None:
|
||||
with patch(f"{_MOD}.enumerate_active", return_value=self._agents("a", "b")), \
|
||||
patch(f"{_MOD}.set_gateway_host") as setter:
|
||||
updated = refresh_gateway_host("192.168.128.19")
|
||||
self.assertEqual(["bot-bottle-a", "bot-bottle-b"], updated)
|
||||
self.assertEqual(
|
||||
[("bot-bottle-a", "192.168.128.19"), ("bot-bottle-b", "192.168.128.19")],
|
||||
[c.args for c in setter.call_args_list],
|
||||
)
|
||||
|
||||
def test_one_failing_bottle_does_not_stop_the_sweep(self) -> None:
|
||||
"""A container that is already exiting must not block the repair of
|
||||
its neighbours, nor fail the launch that triggered the sweep."""
|
||||
def _flaky(name: str, _ip: str) -> None:
|
||||
if name == "bot-bottle-a":
|
||||
raise RuntimeError("container is exiting")
|
||||
|
||||
with patch(f"{_MOD}.enumerate_active", return_value=self._agents("a", "b")), \
|
||||
patch(f"{_MOD}.set_gateway_host", side_effect=_flaky), \
|
||||
patch(f"{_MOD}.warn") as warn:
|
||||
updated = refresh_gateway_host("10.0.0.1")
|
||||
self.assertEqual(["bot-bottle-b"], updated)
|
||||
warn.assert_called_once()
|
||||
|
||||
def test_no_running_bottles_is_a_clean_no_op(self) -> None:
|
||||
with patch(f"{_MOD}.enumerate_active", return_value=[]), \
|
||||
patch(f"{_MOD}.set_gateway_host") as setter:
|
||||
self.assertEqual([], refresh_gateway_host("10.0.0.1"))
|
||||
setter.assert_not_called()
|
||||
|
||||
|
||||
|
||||
class TestLaunchWiring(unittest.TestCase):
|
||||
"""Ordering matters: the name must resolve before anything execs, and the
|
||||
stranded-bottle sweep must run once the gateway is known to be up."""
|
||||
|
||||
def test_launch_sets_the_host_entry_before_reading_the_source_ip(self) -> None:
|
||||
"""The agent's every URL names the gateway, so the entry has to exist
|
||||
before the first connection — which means before the agent execs."""
|
||||
import inspect
|
||||
|
||||
from bot_bottle.backend.macos_container import launch
|
||||
|
||||
src = inspect.getsource(launch)
|
||||
set_at = src.index("set_gateway_host(plan.container_name")
|
||||
exec_at = src.index("wait_container_ipv4_on_network")
|
||||
self.assertLess(set_at, exec_at)
|
||||
|
||||
def test_launch_refreshes_stranded_bottles_after_ensure_gateway(self) -> None:
|
||||
import inspect
|
||||
|
||||
from bot_bottle.backend.macos_container import launch
|
||||
|
||||
src = inspect.getsource(launch)
|
||||
ensure_at = src.index("endpoint = ensure_gateway()")
|
||||
refresh_at = src.index("refresh_gateway_host(endpoint.gateway_ip)")
|
||||
self.assertLess(ensure_at, refresh_at)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -104,32 +104,3 @@ class TestHealthAndPolicy(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestReconcile(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.c = OrchestratorClient("http://orch:8080")
|
||||
|
||||
def test_posts_live_ips_and_returns_reaped(self) -> None:
|
||||
with patch(_URLOPEN, return_value=_resp(200, {"reaped": ["b1", "b2"]})) as m:
|
||||
got = self.c.reconcile(["10.0.0.2", "10.0.0.3"])
|
||||
self.assertEqual(["b1", "b2"], got)
|
||||
sent = json.loads(m.call_args.args[0].data)
|
||||
self.assertEqual(["10.0.0.2", "10.0.0.3"], sent["live_source_ips"])
|
||||
self.assertNotIn("grace_seconds", sent) # omitted -> server default
|
||||
|
||||
def test_grace_seconds_is_forwarded_when_given(self) -> None:
|
||||
with patch(_URLOPEN, return_value=_resp(200, {"reaped": []})) as m:
|
||||
self.c.reconcile([], grace_seconds=30)
|
||||
self.assertEqual(30, json.loads(m.call_args.args[0].data)["grace_seconds"])
|
||||
|
||||
def test_malformed_reaped_is_tolerated(self) -> None:
|
||||
with patch(_URLOPEN, return_value=_resp(200, {"reaped": ["ok", 5, None]})):
|
||||
self.assertEqual(["ok"], self.c.reconcile([]))
|
||||
with patch(_URLOPEN, return_value=_resp(200, {})):
|
||||
self.assertEqual([], self.c.reconcile([]))
|
||||
|
||||
def test_error_status_raises(self) -> None:
|
||||
with patch(_URLOPEN, side_effect=_http_error(500)):
|
||||
with self.assertRaises(OrchestratorClientError):
|
||||
self.c.reconcile([])
|
||||
|
||||
@@ -1,147 +0,0 @@
|
||||
"""Unit: OrchestratorConfigStore and resolve_teardown_timeout.
|
||||
|
||||
Also verifies the lifecycle ordering invariant: resolve_teardown_timeout()
|
||||
must be called before launch_consolidated() / register_agent() so that a
|
||||
resolver failure cannot leave an orphaned registration with no teardown
|
||||
callback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
from bot_bottle.orchestrator.config_store import (
|
||||
DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
|
||||
TEARDOWN_TIMEOUT_ENV,
|
||||
OrchestratorConfigStore,
|
||||
resolve_teardown_timeout,
|
||||
)
|
||||
|
||||
|
||||
class TestOrchestratorConfigStore(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.db = Path(self._tmp.name) / "test.db"
|
||||
self.store = OrchestratorConfigStore(self.db)
|
||||
self.store.migrate()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def test_get_returns_none_when_not_set(self) -> None:
|
||||
self.assertIsNone(self.store.get_teardown_timeout_seconds())
|
||||
|
||||
def test_set_and_get_roundtrip(self) -> None:
|
||||
self.store.set_teardown_timeout_seconds(42.5)
|
||||
self.assertEqual(42.5, self.store.get_teardown_timeout_seconds())
|
||||
|
||||
def test_set_overwrites_existing_value(self) -> None:
|
||||
self.store.set_teardown_timeout_seconds(10.0)
|
||||
self.store.set_teardown_timeout_seconds(20.0)
|
||||
self.assertEqual(20.0, self.store.get_teardown_timeout_seconds())
|
||||
|
||||
def test_delete_clears_value_and_returns_true(self) -> None:
|
||||
self.store.set_teardown_timeout_seconds(30.0)
|
||||
deleted = self.store.delete_teardown_timeout_seconds()
|
||||
self.assertTrue(deleted)
|
||||
self.assertIsNone(self.store.get_teardown_timeout_seconds())
|
||||
|
||||
def test_delete_absent_returns_false(self) -> None:
|
||||
self.assertFalse(self.store.delete_teardown_timeout_seconds())
|
||||
|
||||
def test_is_migrated_true_after_migrate(self) -> None:
|
||||
self.assertTrue(self.store.is_migrated())
|
||||
|
||||
def test_is_migrated_false_before_migrate(self) -> None:
|
||||
store = OrchestratorConfigStore(Path(self._tmp.name) / "new.db")
|
||||
self.assertFalse(store.is_migrated())
|
||||
|
||||
|
||||
class TestResolveTeardownTimeout(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.db = Path(self._tmp.name) / "cfg.db"
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
os.environ.pop(TEARDOWN_TIMEOUT_ENV, None)
|
||||
|
||||
def test_returns_default_when_nothing_configured(self) -> None:
|
||||
self.assertEqual(
|
||||
DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
|
||||
resolve_teardown_timeout(self.db),
|
||||
)
|
||||
|
||||
def test_env_var_overrides_default(self) -> None:
|
||||
os.environ[TEARDOWN_TIMEOUT_ENV] = "99"
|
||||
self.assertEqual(99.0, resolve_teardown_timeout(self.db))
|
||||
|
||||
def test_env_var_overrides_db_value(self) -> None:
|
||||
store = OrchestratorConfigStore(self.db)
|
||||
store.migrate()
|
||||
store.set_teardown_timeout_seconds(55.0)
|
||||
os.environ[TEARDOWN_TIMEOUT_ENV] = "77"
|
||||
self.assertEqual(77.0, resolve_teardown_timeout(self.db))
|
||||
|
||||
def test_db_value_overrides_default(self) -> None:
|
||||
store = OrchestratorConfigStore(self.db)
|
||||
store.migrate()
|
||||
store.set_teardown_timeout_seconds(42.0)
|
||||
self.assertEqual(42.0, resolve_teardown_timeout(self.db))
|
||||
|
||||
def test_invalid_env_var_falls_through_to_default(self) -> None:
|
||||
os.environ[TEARDOWN_TIMEOUT_ENV] = "not-a-number"
|
||||
self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, resolve_teardown_timeout(self.db))
|
||||
|
||||
def test_non_positive_env_var_falls_through_to_default(self) -> None:
|
||||
os.environ[TEARDOWN_TIMEOUT_ENV] = "0"
|
||||
self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, resolve_teardown_timeout(self.db))
|
||||
|
||||
def test_non_positive_db_value_falls_through_to_default(self) -> None:
|
||||
store = OrchestratorConfigStore(self.db)
|
||||
store.migrate()
|
||||
store.set_teardown_timeout_seconds(0.0)
|
||||
self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, resolve_teardown_timeout(self.db))
|
||||
|
||||
def test_migrates_db_on_first_call(self) -> None:
|
||||
result = resolve_teardown_timeout(self.db)
|
||||
self.assertEqual(DEFAULT_TEARDOWN_TIMEOUT_SECONDS, result)
|
||||
self.assertTrue(OrchestratorConfigStore(self.db).is_migrated())
|
||||
|
||||
|
||||
class TestTeardownTimeoutResolvedBeforeRegistration(unittest.TestCase):
|
||||
"""Ordering invariant: if resolve_teardown_timeout() raises, the bottle
|
||||
must not yet be registered — no orphaned state can result."""
|
||||
|
||||
def _src(self, module: ModuleType) -> str:
|
||||
return inspect.getsource(module)
|
||||
|
||||
def test_docker_resolves_timeout_before_launch_consolidated(self) -> None:
|
||||
from bot_bottle.backend.docker import launch
|
||||
src = self._src(launch)
|
||||
resolve_at = src.index("teardown_timeout = resolve_teardown_timeout()")
|
||||
launch_at = src.index("ctx = launch_consolidated(")
|
||||
self.assertLess(resolve_at, launch_at)
|
||||
|
||||
def test_firecracker_resolves_timeout_before_launch_consolidated(self) -> None:
|
||||
from bot_bottle.backend.firecracker import launch
|
||||
src = self._src(launch)
|
||||
resolve_at = src.index("teardown_timeout = resolve_teardown_timeout()")
|
||||
launch_at = src.index("ctx = launch_consolidated(")
|
||||
self.assertLess(resolve_at, launch_at)
|
||||
|
||||
def test_macos_resolves_timeout_before_register_agent(self) -> None:
|
||||
from bot_bottle.backend.macos_container import launch
|
||||
src = self._src(launch)
|
||||
resolve_at = src.index("teardown_timeout = resolve_teardown_timeout()")
|
||||
register_at = src.index("ctx = register_agent(")
|
||||
self.assertLess(resolve_at, register_at)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -8,13 +8,11 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import secrets
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -266,7 +264,6 @@ class TestControlPlaneAuth(unittest.TestCase):
|
||||
("POST", "/bottles", _body({"source_ip": "10.0.0.1"})),
|
||||
("PUT", "/bottles/x/policy", _body({"policy": "routes: []"})),
|
||||
("DELETE", "/bottles/x", b""),
|
||||
("POST", "/reconcile", _body({"live_source_ips": []})),
|
||||
("POST", "/resolve", _body({"source_ip": "10.0.0.1", "identity_token": "t"})),
|
||||
("POST", "/attribute", _body({"source_ip": "10.0.0.1", "identity_token": "t"})),
|
||||
("GET", "/supervise/proposals", b""),
|
||||
@@ -390,56 +387,3 @@ class TestDispatchSupervise(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestReconcileRoute(unittest.TestCase):
|
||||
"""`POST /reconcile` — the host tells the orchestrator which bottles are
|
||||
actually up, since the orchestrator can't see the backend from inside the
|
||||
infra container."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.orch = _orchestrator(Path(self._tmp.name) / "r.db")
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def _old(self, source_ip: str) -> str:
|
||||
rec = self.orch.registry.register(source_ip)
|
||||
with closing(sqlite3.connect(self.orch.registry.db_path)) as conn:
|
||||
conn.execute(
|
||||
"UPDATE orchestrator_bottles SET created_at = 0.0 WHERE bottle_id = ?",
|
||||
(rec.bottle_id,))
|
||||
conn.commit()
|
||||
return rec.bottle_id
|
||||
|
||||
def test_reaps_absent_and_reports_ids(self) -> None:
|
||||
dead = self._old("10.0.0.1")
|
||||
alive = self._old("10.0.0.2")
|
||||
status, payload = dispatch(
|
||||
self.orch, "POST", "/reconcile", _body({"live_source_ips": ["10.0.0.2"]}))
|
||||
self.assertEqual(200, status)
|
||||
self.assertEqual([dead], payload["reaped"])
|
||||
self.assertIsNone(self.orch.registry.get(dead))
|
||||
self.assertIsNotNone(self.orch.registry.get(alive))
|
||||
|
||||
def test_missing_live_source_ips_is_400(self) -> None:
|
||||
status, _ = dispatch(self.orch, "POST", "/reconcile", _body({}))
|
||||
self.assertEqual(400, status)
|
||||
|
||||
def test_grace_seconds_is_honoured(self) -> None:
|
||||
"""A grace window wide enough to cover the row protects it."""
|
||||
self.orch.registry.register("10.0.0.3")
|
||||
status, payload = dispatch(
|
||||
self.orch, "POST", "/reconcile",
|
||||
_body({"live_source_ips": [], "grace_seconds": 3600}))
|
||||
self.assertEqual(200, status)
|
||||
self.assertEqual([], payload["reaped"])
|
||||
|
||||
def test_non_string_entries_are_ignored(self) -> None:
|
||||
dead = self._old("10.0.0.4")
|
||||
status, payload = dispatch(
|
||||
self.orch, "POST", "/reconcile",
|
||||
_body({"live_source_ips": [None, 7, "10.0.0.9"]}))
|
||||
self.assertEqual(200, status)
|
||||
self.assertEqual([dead], payload["reaped"])
|
||||
|
||||
@@ -4,9 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
|
||||
from bot_bottle.orchestrator.registry import (
|
||||
@@ -171,81 +169,3 @@ class TestRegistryStore(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestReapAbsent(unittest.TestCase):
|
||||
"""`reap_absent` — the self-heal for rows whose bottle is gone.
|
||||
|
||||
An orphan is not merely untidy: source IPs get recycled, and
|
||||
`by_source_ip` fail-closes on ambiguity, so a leftover row at a reused
|
||||
address resolves *no* policy for the next bottle that lands there and
|
||||
every host it asks for is denied.
|
||||
"""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.db = Path(self._tmp.name) / "registry.db"
|
||||
self.store = RegistryStore(self.db)
|
||||
self.store.migrate()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def _aged(self, source_ip: str, *, age: float) -> BottleRecord:
|
||||
"""Register a bottle and backdate it past the grace window."""
|
||||
rec = self.store.register(source_ip)
|
||||
with closing(sqlite3.connect(self.db)) as conn:
|
||||
conn.execute(
|
||||
"UPDATE orchestrator_bottles SET created_at = ? WHERE bottle_id = ?",
|
||||
(time.time() - age, rec.bottle_id),
|
||||
)
|
||||
conn.commit()
|
||||
return rec
|
||||
|
||||
def test_reaps_row_with_no_live_container(self) -> None:
|
||||
gone = self._aged("10.243.0.9", age=600)
|
||||
reaped = self.store.reap_absent([])
|
||||
self.assertEqual([gone.bottle_id], [r.bottle_id for r in reaped])
|
||||
self.assertIsNone(self.store.get(gone.bottle_id))
|
||||
|
||||
def test_keeps_row_whose_ip_is_live(self) -> None:
|
||||
alive = self._aged("10.243.0.9", age=600)
|
||||
self.assertEqual([], self.store.reap_absent(["10.243.0.9"]))
|
||||
self.assertIsNotNone(self.store.get(alive.bottle_id))
|
||||
|
||||
def test_grace_window_protects_an_in_flight_launch(self) -> None:
|
||||
"""A bottle registered moments ago is never reaped, even though the
|
||||
caller's enumeration didn't see its address yet."""
|
||||
fresh = self.store.register("10.243.0.10")
|
||||
self.assertEqual([], self.store.reap_absent([]))
|
||||
self.assertIsNotNone(self.store.get(fresh.bottle_id))
|
||||
|
||||
def test_reaping_the_orphan_unbricks_the_reused_address(self) -> None:
|
||||
"""The regression this exists for: an orphan at an address that vmnet
|
||||
later hands to a new bottle makes `by_source_ip` ambiguous, so the new
|
||||
bottle resolves no policy at all."""
|
||||
orphan = self._aged("10.243.0.11", age=600)
|
||||
# A new bottle lands on the recycled address. Force the row in directly
|
||||
# so `register`'s own supersede sweep doesn't mask the ambiguity.
|
||||
with closing(sqlite3.connect(self.db)) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO orchestrator_bottles "
|
||||
"(bottle_id, source_ip, identity_token, state, created_at, metadata, policy) "
|
||||
"VALUES ('newbottle', '10.243.0.11', 'tok-new', 'active', ?, '', 'routes: []')",
|
||||
(time.time(),),
|
||||
)
|
||||
conn.commit()
|
||||
self.assertIsNone(self.store.by_source_ip("10.243.0.11")) # bricked
|
||||
|
||||
reaped = self.store.reap_absent(["10.243.0.11"], grace_seconds=60)
|
||||
self.assertEqual([orphan.bottle_id], [r.bottle_id for r in reaped])
|
||||
rec = self.store.by_source_ip("10.243.0.11")
|
||||
assert rec is not None
|
||||
self.assertEqual("newbottle", rec.bottle_id)
|
||||
|
||||
def test_ignores_empty_ips_in_the_live_set(self) -> None:
|
||||
gone = self._aged("10.243.0.12", age=600)
|
||||
self.assertEqual(
|
||||
[gone.bottle_id],
|
||||
[r.bottle_id for r in self.store.reap_absent(["", "10.243.0.99"])],
|
||||
)
|
||||
|
||||
@@ -4,10 +4,8 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import secrets
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -306,55 +304,3 @@ class TestOrchestratorSupervise(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestOrchestratorReconcile(unittest.TestCase):
|
||||
"""`reconcile` — drop rows for bottles that are no longer running."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.secret = secrets.token_bytes(16)
|
||||
self.db = Path(self._tmp.name) / "r.db"
|
||||
self.store = RegistryStore(self.db)
|
||||
self.store.migrate()
|
||||
self.broker = StubBroker(self.secret)
|
||||
self.orch = Orchestrator(self.store, self.broker, self.secret)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def _age_all(self, seconds: float) -> None:
|
||||
"""Backdate every row past the reap grace window."""
|
||||
with closing(sqlite3.connect(self.db)) as conn:
|
||||
conn.execute(
|
||||
"UPDATE orchestrator_bottles SET created_at = created_at - ?", (seconds,))
|
||||
conn.commit()
|
||||
|
||||
def test_reaps_dead_bottle_and_forgets_its_tokens(self) -> None:
|
||||
dead = self.orch.launch_bottle("10.243.0.1", tokens={"EGRESS_TOKEN_0": "s3cret"})
|
||||
live = self.orch.launch_bottle("10.243.0.2", tokens={"EGRESS_TOKEN_0": "keep"})
|
||||
self._age_all(600)
|
||||
|
||||
self.assertEqual([dead.bottle_id], self.orch.reconcile(["10.243.0.2"]))
|
||||
self.assertIsNone(self.store.get(dead.bottle_id))
|
||||
self.assertIsNotNone(self.store.get(live.bottle_id))
|
||||
# The in-memory egress credential goes with the row.
|
||||
self.assertEqual({}, self.orch.tokens_for(dead.bottle_id))
|
||||
self.assertEqual({"EGRESS_TOKEN_0": "keep"}, self.orch.tokens_for(live.bottle_id))
|
||||
|
||||
def test_reconcile_does_not_broker_a_teardown(self) -> None:
|
||||
"""The container is already gone — there is nothing to stop, and a
|
||||
broker error must not stop the sweep clearing the row."""
|
||||
self.orch.launch_bottle("10.243.0.1")
|
||||
self._age_all(600)
|
||||
self.broker.launched.clear()
|
||||
self.orch.reconcile([])
|
||||
self.assertEqual([], self.broker.torn_down)
|
||||
|
||||
def test_reconcile_keeps_everything_when_all_are_live(self) -> None:
|
||||
a = self.orch.launch_bottle("10.243.0.1")
|
||||
b = self.orch.launch_bottle("10.243.0.2")
|
||||
self._age_all(600)
|
||||
self.assertEqual([], self.orch.reconcile(["10.243.0.1", "10.243.0.2"]))
|
||||
self.assertIsNotNone(self.store.get(a.bottle_id))
|
||||
self.assertIsNotNone(self.store.get(b.bottle_id))
|
||||
|
||||
Reference in New Issue
Block a user