Compare commits
6 Commits
965ee67c46
...
a5910696a5
| Author | SHA1 | Date | |
|---|---|---|---|
| a5910696a5 | |||
| c69642e568 | |||
| dfc693e0b6 | |||
| 39d47b8108 | |||
| d0a0ce8d60 | |||
| 5c08701983 |
@@ -1,9 +1,12 @@
|
||||
"""FirecrackerFreezer — snapshot a running microVM to a Docker image.
|
||||
"""FirecrackerFreezer — snapshot a running microVM to a rootfs tar.
|
||||
|
||||
The VM is live and can't be block-copied safely, so — like the macOS
|
||||
backend — we stream the guest root filesystem out over the control
|
||||
channel (SSH here) and rebuild an image from it. The bottle keeps
|
||||
running after the snapshot.
|
||||
channel (SSH here). Unlike the other backends this needs no Docker: the
|
||||
tar *is* the resumable artifact. `resume` extracts it and rebuilds a
|
||||
fresh per-bottle ext4 with `mke2fs -d` (see `util.build_committed_rootfs_dir`
|
||||
and `launch._build_agent_base`). The bottle keeps running after the
|
||||
snapshot.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,9 +14,9 @@ from __future__ import annotations
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from ...bottle_state import committed_rootfs_path
|
||||
from ...log import die, info
|
||||
from .. import ActiveAgent
|
||||
from ..freeze import Freezer
|
||||
@@ -30,14 +33,13 @@ class FirecrackerFreezer(Freezer):
|
||||
if not private_key.is_file() or not guest_ip:
|
||||
die(f"cannot freeze {agent.slug}: run dir {run_dir} is missing the "
|
||||
f"SSH key or VM config (is the bottle still running?)")
|
||||
image_tag = f"bot-bottle-committed-{agent.slug}:latest"
|
||||
_commit_via_ssh(private_key, guest_ip, image_tag)
|
||||
info(f"committed {agent.slug} -> {image_tag!r}")
|
||||
return image_tag
|
||||
tar_path = committed_rootfs_path(agent.slug)
|
||||
_commit_rootfs_via_ssh(private_key, guest_ip, tar_path)
|
||||
info(f"committed {agent.slug} -> {tar_path}")
|
||||
return str(tar_path)
|
||||
|
||||
def _export_hint(self, slug: str, image_ref: str) -> None:
|
||||
info(f"to export for migration: docker image save {image_ref} "
|
||||
f"-o {slug}.tar")
|
||||
info(f"to export for migration: cp {image_ref} {slug}.tar")
|
||||
|
||||
|
||||
def _guest_ip_from_config(config_path: Path) -> str:
|
||||
@@ -53,24 +55,36 @@ def _guest_ip_from_config(config_path: Path) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def _commit_via_ssh(private_key: Path, guest_ip: str, image_tag: str) -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="bot-bottle-fc-commit.") as tmp:
|
||||
rootfs_tar = os.path.join(tmp, "rootfs.tar")
|
||||
ssh = util.ssh_base_argv(private_key, guest_ip)
|
||||
with open(rootfs_tar, "wb") as tar_out:
|
||||
result = subprocess.run(
|
||||
[*ssh, "--", "tar", "--create", "--one-file-system",
|
||||
"--exclude=./proc", "--exclude=./sys", "--exclude=./dev",
|
||||
"--exclude=./run", "--file=-", "--directory=/", "."],
|
||||
stdout=tar_out, stderr=subprocess.PIPE, check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
die(f"ssh tar for {guest_ip} failed: "
|
||||
f"{(result.stderr or b'').decode().strip() or '<no stderr>'}")
|
||||
with open(os.path.join(tmp, "Dockerfile"), "w", encoding="utf-8") as f:
|
||||
f.write("FROM scratch\nADD rootfs.tar /\nUSER node\nWORKDIR /home/node\n")
|
||||
build = subprocess.run(
|
||||
["docker", "build", "-t", image_tag, tmp], check=False,
|
||||
def _commit_rootfs_via_ssh(private_key: Path, guest_ip: str, tar_path: Path) -> None:
|
||||
"""Stream the guest rootfs out over SSH into `tar_path`. Excludes the
|
||||
virtual/live mounts (proc/sys/dev/run) — resume recreates those empty
|
||||
mount points. Written to a `.partial` sibling and renamed on success so
|
||||
a failed freeze never leaves a truncated artifact in its place."""
|
||||
tar_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
partial = tar_path.with_name(tar_path.name + ".partial")
|
||||
ssh = util.ssh_base_argv(private_key, guest_ip)
|
||||
# The snapshot can contain the bottle's private workspace, so keep it
|
||||
# owner-only (0600) for the whole stream. The `os.open` mode only applies
|
||||
# on *creation*, so unlink any leftover partial (a prior interrupted run
|
||||
# could have left it world-readable, or something could swap in a symlink
|
||||
# at this predictable name) and exclusively recreate it — O_EXCL|O_NOFOLLOW
|
||||
# — then fchmod immediately so umask can't loosen it. Re-assert after the
|
||||
# rename too (os.replace carries the source mode, but be explicit).
|
||||
partial.unlink(missing_ok=True)
|
||||
fd = os.open(
|
||||
partial, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600
|
||||
)
|
||||
os.fchmod(fd, 0o600)
|
||||
with os.fdopen(fd, "wb") as tar_out:
|
||||
result = subprocess.run(
|
||||
[*ssh, "--", "tar", "--create", "--one-file-system",
|
||||
"--exclude=./proc", "--exclude=./sys", "--exclude=./dev",
|
||||
"--exclude=./run", "--file=-", "--directory=/", "."],
|
||||
stdout=tar_out, stderr=subprocess.PIPE, check=False,
|
||||
)
|
||||
if build.returncode != 0:
|
||||
die(f"docker build for {image_tag!r} failed")
|
||||
if result.returncode != 0:
|
||||
partial.unlink(missing_ok=True)
|
||||
die(f"ssh tar for {guest_ip} failed: "
|
||||
f"{(result.stderr or b'').decode().strip() or '<no stderr>'}")
|
||||
os.replace(partial, tar_path)
|
||||
os.chmod(tar_path, 0o600)
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"""Launch flow for the Firecracker backend (PRD 0070, consolidated).
|
||||
|
||||
Per bottle:
|
||||
1. build the agent image (docker), export it to a cached ext4 rootfs;
|
||||
1. build the agent rootfs in a builder VM (buildah, no host docker), or
|
||||
resume a frozen bottle from its committed rootfs tar; cache the ext4;
|
||||
2. ensure the per-host orchestrator + shared gateway are up;
|
||||
3. claim a free TAP pool slot (rootless flock);
|
||||
4. register the bottle on the orchestrator by the VM's guest IP (the
|
||||
@@ -31,6 +32,7 @@ from typing import Callable, Generator
|
||||
|
||||
from ...agent_provider import runtime_for
|
||||
from ...bottle_state import (
|
||||
committed_rootfs_path,
|
||||
egress_state_dir,
|
||||
git_gate_state_dir,
|
||||
read_committed_image,
|
||||
@@ -45,7 +47,6 @@ from ...git_gate import (
|
||||
)
|
||||
from ...log import info, warn
|
||||
from ...supervise import SUPERVISE_PORT
|
||||
from ..docker import util as docker_mod
|
||||
from ..docker.egress import EGRESS_PORT
|
||||
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||
from . import firecracker_vm, image_builder, isolation_probe, netpool, util
|
||||
@@ -210,16 +211,13 @@ def _build_agent_base(
|
||||
) -> tuple[FirecrackerBottlePlan, Path]:
|
||||
"""Produce the agent's base rootfs dir. Primary path: build the Dockerfile
|
||||
inside a Firecracker builder VM (buildah, no host docker), smoke-testing
|
||||
the image before export. A committed snapshot (freeze/migrate) is still
|
||||
exported via the host docker path until that is ported too."""
|
||||
the image before export. A committed snapshot (freeze/migrate) is resumed
|
||||
directly from the rootfs tar the freezer wrote — no host docker either."""
|
||||
committed = read_committed_image(plan.slug)
|
||||
if committed and docker_mod.image_exists(committed):
|
||||
info(f"using committed image {committed!r}")
|
||||
plan = dataclasses.replace(
|
||||
plan,
|
||||
agent_provision=dataclasses.replace(plan.agent_provision, image=committed),
|
||||
)
|
||||
return plan, util.build_base_rootfs_dir(committed)
|
||||
committed_tar = committed_rootfs_path(plan.slug)
|
||||
if committed and committed_tar.is_file():
|
||||
info(f"resuming from committed rootfs {committed_tar}")
|
||||
return plan, util.build_committed_rootfs_dir(committed_tar)
|
||||
base = image_builder.build_agent_rootfs_dir(
|
||||
Path(plan.dockerfile_path),
|
||||
image_tag=plan.image,
|
||||
|
||||
@@ -14,6 +14,7 @@ and `./cli.py backend setup --backend=firecracker`.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
@@ -212,15 +213,80 @@ def build_base_rootfs_dir(
|
||||
return base
|
||||
|
||||
|
||||
def build_committed_rootfs_dir(tar_path: Path) -> Path:
|
||||
"""Prepare a base rootfs dir from a frozen-bottle snapshot tar (the
|
||||
freeze/resume path — no Docker). Extracts the snapshot, recreates the
|
||||
virtual mount points the freezer excluded, and injects the guest init +
|
||||
static dropbear, mirroring `build_base_rootfs_dir` but sourced from a tar
|
||||
we control rather than a Docker image.
|
||||
|
||||
Cached under the rootfs cache, keyed by the tar's size+mtime so a
|
||||
re-freeze re-extracts but repeated resumes of the same snapshot don't.
|
||||
Returns the prepared directory (read as the `mke2fs -d` source)."""
|
||||
st = tar_path.stat()
|
||||
fingerprint = hashlib.sha256(
|
||||
f"{tar_path}:{st.st_size}:{st.st_mtime_ns}".encode()
|
||||
).hexdigest()[:16]
|
||||
base = cache_dir() / "rootfs" / f"committed-{fingerprint}"
|
||||
ready = base / ".bb-ready"
|
||||
if ready.is_file():
|
||||
return base
|
||||
|
||||
if base.exists():
|
||||
shutil.rmtree(base, ignore_errors=True)
|
||||
base.mkdir(parents=True)
|
||||
|
||||
info(f"extracting committed rootfs {tar_path} -> {base}")
|
||||
result = subprocess.run(
|
||||
["tar", "-x", "-f", str(tar_path), "-C", str(base)],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
die(f"extracting committed rootfs {tar_path} failed: "
|
||||
f"{result.stderr.strip() or '<no stderr>'}")
|
||||
|
||||
# The freezer excludes the live/virtual filesystems from the snapshot;
|
||||
# recreate them as empty mount points so the guest init can mount
|
||||
# proc/sys/dev and dropbear has a writable /run.
|
||||
for mount_point in ("proc", "sys", "dev", "run"):
|
||||
(base / mount_point).mkdir(mode=0o755, exist_ok=True)
|
||||
|
||||
inject_guest_boot(base)
|
||||
ready.write_text("ok\n")
|
||||
return base
|
||||
|
||||
|
||||
def inject_guest_boot(rootfs: Path, init_script: str | None = None) -> None:
|
||||
"""Drop the static dropbear and the PID-1 init into the rootfs.
|
||||
`init_script` defaults to the SSH-only agent init; the infra VM
|
||||
passes its own (control plane + gateway) init."""
|
||||
shutil.copy2(dropbear_path(), rootfs / "bb-dropbear")
|
||||
os.chmod(rootfs / "bb-dropbear", 0o755)
|
||||
init = rootfs / "bb-init"
|
||||
init.write_text(init_script or _GUEST_INIT)
|
||||
os.chmod(init, 0o755)
|
||||
passes its own (control plane + gateway) init.
|
||||
|
||||
A committed snapshot is guest-controlled, so `bb-dropbear`/`bb-init`
|
||||
may already exist as symlinks aimed at a host file (e.g. bb-init ->
|
||||
~/.bashrc). Replace whatever is there and create the files with
|
||||
O_EXCL|O_NOFOLLOW so the write always lands a fresh regular file in
|
||||
the staging tree and never follows a planted symlink out of it."""
|
||||
_write_staged_file(rootfs / "bb-dropbear", dropbear_path().read_bytes())
|
||||
_write_staged_file(rootfs / "bb-init", (init_script or _GUEST_INIT).encode())
|
||||
|
||||
|
||||
def _write_staged_file(path: Path, data: bytes) -> None:
|
||||
"""Write `data` to `path` (mode 0755) as a fresh regular file inside a
|
||||
staging rootfs, replacing any pre-existing entry without following a
|
||||
symlink at `path`. Fails closed on anything unexpected there."""
|
||||
if path.is_symlink() or path.exists():
|
||||
if path.is_dir() and not path.is_symlink():
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
path.unlink()
|
||||
fd = os.open(
|
||||
path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o755
|
||||
)
|
||||
try:
|
||||
os.write(fd, data)
|
||||
finally:
|
||||
os.close(fd)
|
||||
os.chmod(path, 0o755)
|
||||
|
||||
|
||||
def build_rootfs_ext4(base_dir: Path, out_path: Path, *, slack_mib: int = 1024) -> None:
|
||||
|
||||
@@ -52,6 +52,7 @@ class MacosContainerBottle(Bottle):
|
||||
terminal_title: str = "",
|
||||
terminal_color: str = "",
|
||||
agent_workdir: str = "/home/node",
|
||||
exec_env: dict[str, str] | None = None,
|
||||
):
|
||||
self.name = container
|
||||
self._teardown = teardown
|
||||
@@ -62,6 +63,15 @@ class MacosContainerBottle(Bottle):
|
||||
self.terminal_color = terminal_color
|
||||
self.agent_provider_template = agent_provider_template
|
||||
self.agent_workdir = agent_workdir
|
||||
# Env applied to the agent process at `container exec` time, on top of
|
||||
# what the container was run with. This is how the identity token
|
||||
# reaches the agent (PRD 0070): registration mints it *after* the
|
||||
# container exists — its source IP is the registration key and Apple
|
||||
# Container assigns that by DHCP — so it cannot be in the run-time env
|
||||
# the way docker's compose spec does it. `container exec --env` wins
|
||||
# over the run-time value, so the token-bearing proxy URL set here
|
||||
# supersedes the token-less one baked in at launch.
|
||||
self._exec_env = dict(exec_env or {})
|
||||
self._closed = False
|
||||
|
||||
def agent_argv(self, argv: list[str], *, tty: bool = True) -> list[str]:
|
||||
@@ -74,6 +84,12 @@ class MacosContainerBottle(Bottle):
|
||||
)
|
||||
)
|
||||
container_exec = ["container", "exec"]
|
||||
# Bare env names, same rule as the terminal hints below: the value
|
||||
# stays in the child env `exec_agent` builds and never reaches argv —
|
||||
# the proxy URL here carries the identity token, which `ps` would
|
||||
# otherwise expose to every process on the host.
|
||||
for name in sorted(self._exec_env):
|
||||
container_exec.extend(["--env", name])
|
||||
if tty:
|
||||
container_exec.extend(["--interactive", "--tty"])
|
||||
# Forward terminal capability hints so TUIs can enable modified-key
|
||||
@@ -94,21 +110,33 @@ class MacosContainerBottle(Bottle):
|
||||
|
||||
def exec_agent(self, argv: list[str], *, tty: bool = True) -> int:
|
||||
agent_argv = self.agent_argv(argv, tty=tty)
|
||||
# The values behind the bare `--env` names in `agent_argv`. `sh -lc`
|
||||
# below is in this process tree, so the child env reaches `container
|
||||
# exec` either way.
|
||||
env = {**os.environ, **self._exec_env} if self._exec_env else None
|
||||
script = (
|
||||
exec_shell_script(agent_argv, self.terminal_title, self.terminal_color)
|
||||
if tty else None
|
||||
)
|
||||
if script is None:
|
||||
return subprocess.run(agent_argv, check=False).returncode
|
||||
return subprocess.run(["sh", "-lc", script], check=False).returncode
|
||||
return subprocess.run(agent_argv, env=env, check=False).returncode
|
||||
return subprocess.run(["sh", "-lc", script], env=env, check=False).returncode
|
||||
|
||||
def exec(self, script: str, *, user: str = "node") -> ExecResult:
|
||||
# Carry the same exec env the agent gets: provisioning steps run
|
||||
# through here, and a provider whose provision step fetches anything
|
||||
# would egress without the identity token and be denied by /resolve.
|
||||
# Bare `--env NAME` again, so the token stays off argv.
|
||||
argv = ["container", "exec", "--user", user, "--interactive"]
|
||||
for name in sorted(self._exec_env):
|
||||
argv.extend(["--env", name])
|
||||
argv.extend([self.name, "sh", "-s"])
|
||||
result = subprocess.run(
|
||||
["container", "exec", "--user", user, "--interactive",
|
||||
self.name, "sh", "-s"],
|
||||
argv,
|
||||
input=script,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env={**os.environ, **self._exec_env} if self._exec_env else None,
|
||||
check=False,
|
||||
)
|
||||
return ExecResult(
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Consolidated bottle launch sequence for the macOS backend (PRD 0070).
|
||||
|
||||
The docker backend allocates a free address, pins the agent to it with
|
||||
`--ip`, registers it, *then* starts the agent — registration precedes launch
|
||||
because the pinned address is known up front.
|
||||
|
||||
**Apple Container 1.0.0 has no `--ip`.** The `--network` flag takes only
|
||||
`<name>[,mac=…][,mtu=…]`; the address is assigned by vmnet's DHCP and is
|
||||
knowable only once the container is running. So the macOS order inverts:
|
||||
|
||||
ensure_gateway() -> caller starts the agent -> register_agent(source_ip)
|
||||
|
||||
That is why this module exposes two functions where docker has one — the
|
||||
caller has to start the agent in between. `ensure_gateway` runs first because
|
||||
the agent's proxy env needs the gateway's address at `container run` time; the
|
||||
agent's *own* address (the attribution key) only exists afterwards.
|
||||
|
||||
The consequence for the identity token: it is minted by registration, i.e.
|
||||
*after* the agent container exists, so it cannot be baked into the run-time
|
||||
env the way docker's compose spec does. It is delivered at `container exec`
|
||||
time instead — see `bottle.MacosContainerBottle`.
|
||||
|
||||
That delivery is load-bearing, not a nicety: `/resolve` requires a matching
|
||||
`(source_ip, identity_token)` pair and fail-closes with no source-IP-only
|
||||
fallback (#366). So egress that does not carry the token is denied — which is
|
||||
the safe direction, and is why the agent's init process is a bare `sleep` and
|
||||
every real command arrives through `container exec`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ...egress import EgressPlan
|
||||
from ...git_gate import GitGatePlan
|
||||
from ...orchestrator.client import OrchestratorClient
|
||||
from ...orchestrator.registration import registration_inputs
|
||||
from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate
|
||||
from .gateway import GATEWAY_NETWORK
|
||||
from .gateway_provision import AppleGatewayTransport
|
||||
from .orchestrator_service import MacosOrchestratorService, OrchestratorStartError
|
||||
|
||||
|
||||
class ConsolidatedLaunchError(RuntimeError):
|
||||
"""The consolidated register/provision sequence could not complete."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GatewayEndpoint:
|
||||
"""What the agent `container run` needs to reach the shared gateway."""
|
||||
|
||||
orchestrator_url: str
|
||||
gateway_ip: str # the gateway's address — the agent's proxy target
|
||||
gateway_ca_pem: str # the shared CA the provisioner installs
|
||||
network: str # the shared host-only network to attach to
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LaunchContext:
|
||||
"""What the running agent needs once it has been registered."""
|
||||
|
||||
bottle_id: str
|
||||
identity_token: str
|
||||
source_ip: str # the agent's DHCP-assigned address (attribution key)
|
||||
gateway_ip: str
|
||||
network: str
|
||||
orchestrator_url: str
|
||||
|
||||
|
||||
def ensure_gateway(
|
||||
*, service: MacosOrchestratorService | None = None,
|
||||
) -> GatewayEndpoint:
|
||||
"""Ensure the orchestrator control plane + shared gateway are up, and
|
||||
report how to reach them. Idempotent — both are per-host singletons, so N
|
||||
bottle launches share the one pair. Call before starting the agent
|
||||
container: the agent's proxy env needs `gateway_ip` at run time."""
|
||||
service = service or MacosOrchestratorService()
|
||||
url = service.ensure_running()
|
||||
gateway = service.gateway(url)
|
||||
return GatewayEndpoint(
|
||||
orchestrator_url=url,
|
||||
gateway_ip=gateway.ip_on_shared_network(),
|
||||
gateway_ca_pem=gateway.ca_cert_pem(),
|
||||
network=service.network,
|
||||
)
|
||||
|
||||
|
||||
def register_agent(
|
||||
egress_plan: EgressPlan,
|
||||
git_gate_plan: GitGatePlan,
|
||||
*,
|
||||
source_ip: str,
|
||||
endpoint: GatewayEndpoint,
|
||||
image_ref: str = "",
|
||||
tokens: dict[str, str] | None = None,
|
||||
) -> LaunchContext:
|
||||
"""Register the (already running) agent by its address and provision its
|
||||
git-gate state into the gateway. `source_ip` must be read from the live
|
||||
container — it is the attribution key the gateway resolves policy by.
|
||||
Raises on failure; the caller tears down."""
|
||||
client = OrchestratorClient(endpoint.orchestrator_url)
|
||||
inputs = registration_inputs(egress_plan)
|
||||
reg = client.register_bottle(
|
||||
source_ip, image_ref=image_ref, policy=inputs.policy,
|
||||
metadata=inputs.metadata, tokens=tokens,
|
||||
)
|
||||
try:
|
||||
provision_git_gate(AppleGatewayTransport(), reg.bottle_id, git_gate_plan)
|
||||
except Exception:
|
||||
# Roll the registration back so a provisioning failure leaves no orphan.
|
||||
client.teardown_bottle(reg.bottle_id)
|
||||
raise
|
||||
return LaunchContext(
|
||||
bottle_id=reg.bottle_id,
|
||||
identity_token=reg.identity_token,
|
||||
source_ip=source_ip,
|
||||
gateway_ip=endpoint.gateway_ip,
|
||||
network=endpoint.network,
|
||||
orchestrator_url=endpoint.orchestrator_url,
|
||||
)
|
||||
|
||||
|
||||
def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None:
|
||||
"""Deregister the bottle and remove its git-gate state from the gateway.
|
||||
Both steps are idempotent so this is safe from a cleanup trap. Does NOT
|
||||
stop the gateway — it's a persistent per-host singleton."""
|
||||
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
|
||||
deprovision_git_gate(AppleGatewayTransport(), bottle_id)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GatewayEndpoint",
|
||||
"LaunchContext",
|
||||
"ensure_gateway",
|
||||
"register_agent",
|
||||
"teardown_consolidated",
|
||||
"ConsolidatedLaunchError",
|
||||
"OrchestratorStartError",
|
||||
"GATEWAY_NETWORK",
|
||||
]
|
||||
@@ -1,9 +1,11 @@
|
||||
"""Host-side egress route-apply for the macos-container backend.
|
||||
|
||||
The per-bottle companion container this used to signal (`container kill
|
||||
--signal HUP <container>`) was removed in the companion-container removal (#385),
|
||||
along with the disabled macOS launch path. Fails closed until the macOS
|
||||
backend grows the consolidated gateway.
|
||||
--signal HUP <container>`) was removed in the companion-container removal
|
||||
(#385). In the consolidated model the shared gateway resolves egress policy
|
||||
per-request against the orchestrator rather than reloading a per-bottle routes
|
||||
file, so the live per-bottle reload is not supported here and fails closed
|
||||
until the gateway-side apply lands — same posture as the docker backend.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -16,8 +18,8 @@ class MacOSContainerEgressApplicator(EgressApplicator):
|
||||
del slug
|
||||
raise EgressApplyError(
|
||||
"live egress route-apply was removed with the per-bottle "
|
||||
"companion container (#385); the macos-container backend is "
|
||||
"disabled until it uses the consolidated gateway."
|
||||
"companion container (#385); route changes will flow through "
|
||||
"the consolidated gateway in a follow-up."
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,14 +1,43 @@
|
||||
"""Active-agent enumeration for the macOS Apple Container backend.
|
||||
|
||||
The backend is disabled during the companion-container removal (#385) — it can't
|
||||
launch bottles, so there are none to enumerate. Enumeration returns when
|
||||
the backend grows the consolidated gateway.
|
||||
"""
|
||||
"""Active-agent enumeration for the macOS Apple Container backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
|
||||
from ...bottle_state import read_metadata
|
||||
from .. import ActiveAgent
|
||||
from .gateway import GATEWAY_NAME
|
||||
from .orchestrator_service import ORCHESTRATOR_NAME
|
||||
|
||||
_PREFIX = "bot-bottle-"
|
||||
# The shared per-host singletons carry the same prefix as agent containers but
|
||||
# are infrastructure, not bottles — one gateway and one control plane serve
|
||||
# every agent, so listing them as agents would invent one per host.
|
||||
_INFRA_NAMES = frozenset({GATEWAY_NAME, ORCHESTRATOR_NAME})
|
||||
|
||||
|
||||
def enumerate_active() -> list[ActiveAgent]:
|
||||
return []
|
||||
result = subprocess.run(
|
||||
["container", "list", "--quiet"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return []
|
||||
out: list[ActiveAgent] = []
|
||||
for name in sorted(line.strip() for line in result.stdout.splitlines()):
|
||||
if not name.startswith(_PREFIX) or name in _INFRA_NAMES:
|
||||
continue
|
||||
slug = name[len(_PREFIX):]
|
||||
metadata = read_metadata(slug)
|
||||
out.append(ActiveAgent(
|
||||
backend_name="macos-container",
|
||||
slug=slug,
|
||||
agent_name=metadata.agent_name if metadata else "?",
|
||||
started_at=metadata.started_at if metadata else "",
|
||||
services=(),
|
||||
label=metadata.label if metadata else "",
|
||||
color=metadata.color if metadata else "",
|
||||
))
|
||||
return out
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
"""The consolidated per-host gateway as an Apple container (PRD 0070).
|
||||
|
||||
The macOS counterpart of `orchestrator.gateway.DockerGateway`: one persistent
|
||||
gateway per host, shared by every bottle, attributing each request to a bottle
|
||||
by its source IP on the shared host-only network.
|
||||
|
||||
Two Apple Container 1.0.0 constraints shape this and make it *not* a
|
||||
transliteration of the docker gateway:
|
||||
|
||||
- **No container DNS.** Containers cannot resolve each other by name (the
|
||||
host-only network's resolver refuses the query), so the gateway reaches the
|
||||
control plane by **IP**, not by name as the docker gateway does. The
|
||||
orchestrator must therefore be started *before* the gateway — see
|
||||
`orchestrator_service`.
|
||||
- **Networks are fixed at `container run`.** There is no `network connect`,
|
||||
so a network cannot be attached to a running container. The gateway must sit
|
||||
on one shared, up-front network for the lifetime of the process; per-bottle
|
||||
networks would mean restarting the gateway on every launch, which defeats the
|
||||
consolidation.
|
||||
|
||||
The gateway is dual-homed, **NAT network first**: Apple Container makes the
|
||||
first `--network` the default route, so the egress network must lead or the
|
||||
gateway has no route to the internet.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from ...orchestrator.gateway import (
|
||||
GATEWAY_CA_CERT,
|
||||
GATEWAY_DOCKERFILE,
|
||||
MITMPROXY_HOME,
|
||||
Gateway,
|
||||
GatewayError,
|
||||
)
|
||||
from ...paths import host_db_path
|
||||
from ...supervise import DB_PATH_IN_CONTAINER
|
||||
from . import util as container_mod
|
||||
|
||||
# Distinct from the docker gateway's names so both backends' gateways can
|
||||
# coexist on one host (a macOS host can run the docker backend too).
|
||||
GATEWAY_NAME = "bot-bottle-mac-gateway"
|
||||
# The shared host-only network the gateway and every agent bottle sit on. The
|
||||
# agent's address here is the attribution key.
|
||||
GATEWAY_NETWORK = "bot-bottle-mac-gateway"
|
||||
# The NAT network that gives the gateway (and only the gateway) a route out.
|
||||
GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress"
|
||||
|
||||
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
_SUPERVISE_DB_DIR_IN_CONTAINER = os.path.dirname(DB_PATH_IN_CONTAINER)
|
||||
|
||||
# mitmproxy writes its CA a beat after start; reads poll rather than assume.
|
||||
_CA_POLL_SECONDS = 0.5
|
||||
DEFAULT_CA_TIMEOUT_SECONDS = 30.0
|
||||
|
||||
|
||||
def gateway_ca_dir() -> Path:
|
||||
"""Host dir bind-mounted as mitmproxy's home, keeping the gateway's
|
||||
self-generated CA **stable across container recreation** — every agent
|
||||
installs this one CA to trust the shared gateway's TLS interception, so it
|
||||
must not rotate when the gateway restarts.
|
||||
|
||||
The docker gateway uses a named volume for this; a plain host dir is the
|
||||
same guarantee with fewer moving parts, and it lets `ca_cert_pem` read the
|
||||
PEM straight off the host instead of shelling into the container."""
|
||||
path = host_db_path().parent / "mac-gateway-ca"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def ensure_networks(
|
||||
network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK,
|
||||
) -> None:
|
||||
"""Create the shared host-only network + the gateway's NAT network.
|
||||
Idempotent — `create_network` tolerates 'already exists'.
|
||||
|
||||
Module-level rather than a gateway method because the **orchestrator**
|
||||
needs the shared network too, and it starts first (Apple has no container
|
||||
DNS, so the gateway must be handed the control plane's IP). Both callers
|
||||
ensure the networks; whoever runs first wins."""
|
||||
container_mod.create_network(egress_network)
|
||||
container_mod.create_network(network, internal=True)
|
||||
|
||||
|
||||
def _host_db_dir() -> str:
|
||||
db_dir = host_db_path().parent
|
||||
db_dir.mkdir(parents=True, exist_ok=True)
|
||||
return str(db_dir)
|
||||
|
||||
|
||||
def _mount(source: str, target: str, *, readonly: bool = False) -> str:
|
||||
spec = f"type=bind,source={source},target={target}"
|
||||
if readonly:
|
||||
spec += ",readonly"
|
||||
return spec
|
||||
|
||||
|
||||
class AppleGateway(Gateway):
|
||||
"""The consolidated gateway as a single, fixed-name Apple container."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_ref: str = GATEWAY_IMAGE,
|
||||
*,
|
||||
name: str = GATEWAY_NAME,
|
||||
network: str = GATEWAY_NETWORK,
|
||||
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
||||
orchestrator_url: str = "",
|
||||
build_context: Path | None = None,
|
||||
dockerfile: str | None = GATEWAY_DOCKERFILE,
|
||||
) -> None:
|
||||
self.image_ref = image_ref
|
||||
self.name = name
|
||||
self.network = network
|
||||
self.egress_network = egress_network
|
||||
# Reached by IP (no container DNS on Apple) — the caller resolves the
|
||||
# orchestrator's address before constructing this. Empty → single-tenant.
|
||||
self._orchestrator_url = orchestrator_url
|
||||
self._build_context = build_context or _REPO_ROOT
|
||||
self._dockerfile = dockerfile
|
||||
|
||||
def ensure_built(self) -> None:
|
||||
"""Build the gateway data-plane image from its Dockerfile. Builds every
|
||||
time (cache-aware, so it's cheap when nothing changed): a stale image
|
||||
silently runs the OLD single-tenant daemons. Mirrors `DockerGateway`."""
|
||||
if self._dockerfile is None:
|
||||
return
|
||||
container_mod.build_image(
|
||||
self.image_ref, str(self._build_context), dockerfile=self._dockerfile,
|
||||
)
|
||||
|
||||
def is_running(self) -> bool:
|
||||
return container_mod.container_is_running(self.name)
|
||||
|
||||
def _running_image_is_current(self) -> bool:
|
||||
"""True iff the running gateway was created from the *current*
|
||||
`image_ref`. `ensure_built` rebuilding the image is not enough on its
|
||||
own — the running container still holds the OLD image, so this
|
||||
mismatch check is what makes a rebuild take effect."""
|
||||
running = container_mod.container_image_digest(self.name)
|
||||
current = container_mod.image_digest(self.image_ref)
|
||||
if not running or not current:
|
||||
return True # can't compare → don't churn a working container
|
||||
return running == current
|
||||
|
||||
def _running_control_plane_is_current(self) -> bool:
|
||||
"""True iff the running gateway points at the control plane we would
|
||||
pass today.
|
||||
|
||||
Docker gets this for free — it hands the gateway a container *name*,
|
||||
which survives the orchestrator being recreated. Apple has no container
|
||||
DNS, so the URL is an **IP baked into the gateway's env at run time**,
|
||||
and a recreated orchestrator can come back on a different DHCP address.
|
||||
Without this check the gateway would keep pointing at the old address
|
||||
and every `/resolve` would fail — denying egress for *every* bottle on
|
||||
the host until something else happened to recreate the gateway."""
|
||||
if not self._orchestrator_url:
|
||||
return True
|
||||
env = container_mod.container_env(self.name)
|
||||
if not env:
|
||||
return True # can't compare → don't churn a working container
|
||||
return env.get("BOT_BOTTLE_ORCHESTRATOR_URL") == self._orchestrator_url
|
||||
|
||||
def ensure_running(self) -> None:
|
||||
if (self.is_running()
|
||||
and self._running_image_is_current()
|
||||
and self._running_control_plane_is_current()):
|
||||
return
|
||||
ensure_networks(self.network, self.egress_network)
|
||||
container_mod.force_remove_container(self.name)
|
||||
argv = [
|
||||
"container", "run", "--detach",
|
||||
"--name", self.name,
|
||||
"--label", "bot-bottle.backend=macos-container",
|
||||
"--label", "bot-bottle-mac-gateway=1",
|
||||
# NAT network FIRST: Apple Container takes the first --network as
|
||||
# the default route, so this ordering is what gives the gateway a
|
||||
# route out. Reversing it silently blackholes egress.
|
||||
"--network", self.egress_network,
|
||||
"--network", self.network,
|
||||
# The NAT gateway routes but does not resolve, so DNS is explicit.
|
||||
"--dns", container_mod.dns_server(),
|
||||
"--mount", _mount(str(gateway_ca_dir()), MITMPROXY_HOME),
|
||||
"--mount", _mount(_host_db_dir(), _SUPERVISE_DB_DIR_IN_CONTAINER),
|
||||
"--env", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}",
|
||||
]
|
||||
if self._orchestrator_url:
|
||||
# Makes the data plane multi-tenant: each request resolves
|
||||
# source-IP → policy against the control plane.
|
||||
argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"]
|
||||
argv.append(self.image_ref)
|
||||
result = container_mod.run_container_argv(argv)
|
||||
if result.returncode != 0:
|
||||
raise GatewayError(
|
||||
f"gateway failed to start: "
|
||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||
)
|
||||
|
||||
def ip_on_shared_network(self) -> str:
|
||||
"""The gateway's address on the shared host-only network — what agents
|
||||
point their proxy / git-http / supervise URLs at."""
|
||||
return container_mod.container_ipv4_on_network(self.name, self.network)
|
||||
|
||||
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
||||
"""The gateway's CA certificate (PEM) that agents install to trust its
|
||||
TLS interception. Polls: mitmproxy generates it a moment after start.
|
||||
Read from the host bind-mount, so no exec into the container."""
|
||||
ca_path = gateway_ca_dir() / os.path.basename(GATEWAY_CA_CERT)
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
try:
|
||||
pem = ca_path.read_text()
|
||||
if pem.strip():
|
||||
return pem
|
||||
except OSError:
|
||||
pass
|
||||
if time.monotonic() >= deadline:
|
||||
raise GatewayError(
|
||||
f"gateway CA cert not available at {ca_path} after {timeout:g}s"
|
||||
)
|
||||
time.sleep(_CA_POLL_SECONDS)
|
||||
|
||||
def stop(self) -> None:
|
||||
container_mod.force_remove_container(self.name)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AppleGateway",
|
||||
"GatewayError",
|
||||
"GATEWAY_NAME",
|
||||
"GATEWAY_NETWORK",
|
||||
"GATEWAY_EGRESS_NETWORK",
|
||||
"GATEWAY_IMAGE",
|
||||
"gateway_ca_dir",
|
||||
]
|
||||
@@ -0,0 +1,44 @@
|
||||
"""`GatewayTransport` for the Apple gateway container (PRD 0070).
|
||||
|
||||
The provisioning *logic* (per-bottle creds dirs, namespaced repo init) is
|
||||
backend-neutral and lives in `backend.docker.gateway_provision`; this is only
|
||||
the transport — how files and commands reach the running gateway. Docker uses
|
||||
`docker exec`/`docker cp` and Firecracker uses SSH; Apple uses the `container`
|
||||
CLI's equivalents.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..docker.gateway_provision import GatewayProvisionError
|
||||
from . import util as container_mod
|
||||
from .gateway import GATEWAY_NAME
|
||||
|
||||
|
||||
class AppleGatewayTransport:
|
||||
"""`GatewayTransport` for the gateway as an Apple container."""
|
||||
|
||||
def __init__(self, gateway: str = GATEWAY_NAME) -> None:
|
||||
self.gateway = gateway
|
||||
|
||||
def exec(self, argv: list[str]) -> None:
|
||||
result = container_mod.run_container_argv(
|
||||
["container", "exec", self.gateway, *argv]
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise GatewayProvisionError(
|
||||
f"gateway exec {argv!r} failed: "
|
||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||
)
|
||||
|
||||
def cp_into(self, src: str, dest: str) -> None:
|
||||
result = container_mod.run_container_argv(
|
||||
["container", "cp", src, f"{self.gateway}:{dest}"]
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise GatewayProvisionError(
|
||||
f"gateway cp {src} -> {dest} failed: "
|
||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["AppleGatewayTransport", "GatewayProvisionError"]
|
||||
@@ -1,24 +1,74 @@
|
||||
"""Launch flow for the macOS Apple Container backend — disabled (#385).
|
||||
"""Launch flow for the macOS Apple Container backend (PRD 0070).
|
||||
|
||||
This backend launched a per-bottle companion container (the egress /
|
||||
git-gate / supervise data plane) alongside the agent container, with the
|
||||
agent's proxy env pointed at the companion's host-only IP. That
|
||||
per-bottle-companion architecture was removed in the companion-container removal;
|
||||
the macOS backend will be re-enabled once it grows the consolidated
|
||||
per-host gateway the docker backend already uses.
|
||||
The agent container attaches to the **shared host-only gateway network** and
|
||||
proxies egress through the one per-host gateway, replacing the per-bottle
|
||||
companion container removed in #385.
|
||||
|
||||
Until then, launching a macOS bottle fails closed. `prepare` / `status`
|
||||
/ cleanup still work.
|
||||
The order differs from docker's, forced by Apple Container 1.0.0 having no
|
||||
`--ip` (see `consolidated_launch`): the agent is started *before* it is
|
||||
registered, because its DHCP-assigned address — the attribution key — does not
|
||||
exist until then.
|
||||
|
||||
gateway up -> run agent -> read its IP -> register it -> provision
|
||||
|
||||
Two things follow from that inversion:
|
||||
|
||||
- The **identity token** is minted by registration and so cannot be in the
|
||||
agent's run-time env; it rides the proxy URL applied at `container exec`
|
||||
time (`bottle.MacosContainerBottle`). `/resolve` requires it (#366), so
|
||||
egress without it is denied — hence the bare `sleep` init: every real agent
|
||||
command goes through exec and therefore carries the token.
|
||||
- The agent is run with `--cap-drop CAP_NET_RAW`. Apple Container grants
|
||||
NET_RAW by default, which would let an agent open a raw socket and forge a
|
||||
neighbour's source address on the shared segment. NET_ADMIN is already
|
||||
absent (the agent cannot change its own address or route), so dropping
|
||||
NET_RAW is what closes the source-address half of PRD 0070's invariant:
|
||||
"a packet's source address, as seen by the orchestrator, provably identifies
|
||||
the originating bottle." The identity token is the other half — an attacker
|
||||
would need to forge the address *and* steal the token — but the invariant is
|
||||
a stated precondition of consolidation, so it is enforced on its own terms
|
||||
rather than left to the token.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from contextlib import contextmanager
|
||||
import dataclasses
|
||||
import os
|
||||
import subprocess
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Callable, Generator
|
||||
|
||||
from ...log import die
|
||||
from ...bottle_state import (
|
||||
egress_state_dir,
|
||||
git_gate_state_dir,
|
||||
read_committed_image,
|
||||
)
|
||||
from ...egress import (
|
||||
egress_agent_env_entries,
|
||||
egress_resolve_token_values,
|
||||
)
|
||||
from ...git_gate import (
|
||||
provision_git_gate_dynamic_keys,
|
||||
revoke_git_gate_provisioned_keys,
|
||||
)
|
||||
from ...log import die, info, warn
|
||||
from ...supervise import SUPERVISE_PORT
|
||||
from ..docker.egress import EGRESS_PORT
|
||||
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||
from . import util as container_mod
|
||||
from .bottle import MacosContainerBottle
|
||||
from .bottle_plan import MacosContainerBottlePlan
|
||||
from .consolidated_launch import (
|
||||
GatewayEndpoint,
|
||||
ensure_gateway,
|
||||
register_agent,
|
||||
teardown_consolidated,
|
||||
)
|
||||
|
||||
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
|
||||
_AGENT_SLEEP_SECONDS = "2147483647"
|
||||
_GIT_HTTP_PORT = 9420
|
||||
|
||||
|
||||
@contextmanager
|
||||
@@ -27,13 +77,260 @@ def launch(
|
||||
*,
|
||||
provision: Callable[[MacosContainerBottlePlan, "MacosContainerBottle"], str | None],
|
||||
) -> Generator[MacosContainerBottle, None, None]:
|
||||
"""Fail closed: the macOS backend is disabled until it grows the
|
||||
consolidated per-host gateway (the companion-container path it used
|
||||
was removed in #385)."""
|
||||
del plan, provision
|
||||
die(
|
||||
"the macos-container backend is temporarily disabled during the "
|
||||
"companion-container removal (#385); it will return once it uses "
|
||||
"the consolidated gateway. Use --backend=docker for now."
|
||||
"""Build, run, register, provision, and yield an Apple Container bottle on
|
||||
the shared per-host gateway."""
|
||||
stack = ExitStack()
|
||||
bottle_for_revoke = plan.manifest.bottle
|
||||
git_gate_dir_for_revoke = git_gate_state_dir(plan.slug)
|
||||
|
||||
def teardown() -> None:
|
||||
teardown_exc: BaseException | None = None
|
||||
try:
|
||||
stack.close()
|
||||
except BaseException as exc: # noqa: W0718 - teardown must continue
|
||||
teardown_exc = exc
|
||||
warn(f"macos-container teardown failed: {exc!r}")
|
||||
revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke)
|
||||
if teardown_exc is not None:
|
||||
raise teardown_exc
|
||||
|
||||
try:
|
||||
plan = _build_images(plan)
|
||||
|
||||
# Step 1: the per-host singletons. Must precede the agent run — its
|
||||
# proxy env needs the gateway's address at `container run` time.
|
||||
endpoint = ensure_gateway()
|
||||
|
||||
# Step 2: mint this bottle's deploy keys, then point it at the SHARED
|
||||
# gateway's CA + git-http/supervise ports.
|
||||
plan = _provision_git_gate_keys(plan)
|
||||
plan = _install_gateway_ca(plan, endpoint)
|
||||
plan = _stamp_agent_urls(plan, endpoint)
|
||||
|
||||
# Step 3: run the agent. It has no identity token yet — registration
|
||||
# needs the address this run assigns.
|
||||
container_mod.force_remove_container(plan.container_name)
|
||||
_start_agent(plan, endpoint)
|
||||
stack.callback(container_mod.force_remove_container, plan.container_name)
|
||||
|
||||
# Step 4: read the assigned address and register by it. This is the
|
||||
# attribution key; `--cap-drop CAP_NET_RAW` at run is what makes it
|
||||
# unforgeable.
|
||||
source_ip = container_mod.container_ipv4_on_network(
|
||||
plan.container_name, endpoint.network,
|
||||
)
|
||||
effective_env = {**os.environ, **plan.agent_provision.provisioned_env}
|
||||
token_values = egress_resolve_token_values(
|
||||
plan.egress_plan.token_env_map, effective_env,
|
||||
)
|
||||
ctx = register_agent(
|
||||
plan.egress_plan,
|
||||
plan.git_gate_plan,
|
||||
source_ip=source_ip,
|
||||
endpoint=endpoint,
|
||||
image_ref=plan.image,
|
||||
tokens=token_values,
|
||||
)
|
||||
stack.callback(
|
||||
teardown_consolidated, ctx.bottle_id,
|
||||
orchestrator_url=ctx.orchestrator_url,
|
||||
)
|
||||
info(
|
||||
f"agent {plan.container_name} registered "
|
||||
f"(gateway {endpoint.gateway_ip}, ip {source_ip})"
|
||||
)
|
||||
|
||||
bottle = MacosContainerBottle(
|
||||
plan.container_name,
|
||||
teardown,
|
||||
None,
|
||||
agent_command=plan.agent_command,
|
||||
agent_prompt_mode=plan.agent_prompt_mode,
|
||||
agent_provider_template=plan.agent_provider_template,
|
||||
terminal_title=(
|
||||
f"{plan.spec.label} ({plan.spec.agent_name})"
|
||||
if plan.spec.label else plan.spec.agent_name
|
||||
),
|
||||
terminal_color=plan.spec.color,
|
||||
agent_workdir=plan.workspace_plan.workdir,
|
||||
exec_env=_identity_proxy_env(endpoint, ctx.identity_token),
|
||||
)
|
||||
bottle.prompt_path = provision(plan, bottle)
|
||||
|
||||
yield bottle
|
||||
finally:
|
||||
teardown()
|
||||
|
||||
|
||||
def _build_images(plan: MacosContainerBottlePlan) -> MacosContainerBottlePlan:
|
||||
"""Build the agent image. The gateway's own image is built by
|
||||
`ensure_gateway` — it belongs to the shared singleton, not to a bottle."""
|
||||
committed = read_committed_image(plan.slug)
|
||||
if committed and container_mod.image_exists(committed):
|
||||
info(f"using committed image {committed!r}")
|
||||
return dataclasses.replace(
|
||||
plan,
|
||||
agent_provision=dataclasses.replace(
|
||||
plan.agent_provision, image=committed,
|
||||
),
|
||||
)
|
||||
container_mod.build_image(
|
||||
plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path,
|
||||
)
|
||||
yield # unreachable — `die` raises; keeps this a generator/contextmanager
|
||||
return plan
|
||||
|
||||
|
||||
def _provision_git_gate_keys(
|
||||
plan: MacosContainerBottlePlan,
|
||||
) -> MacosContainerBottlePlan:
|
||||
if not plan.git_gate_plan.upstreams:
|
||||
return plan
|
||||
git_gate_plan = provision_git_gate_dynamic_keys(
|
||||
plan.manifest.bottle,
|
||||
plan.git_gate_plan,
|
||||
git_gate_state_dir(plan.slug),
|
||||
)
|
||||
return dataclasses.replace(plan, git_gate_plan=git_gate_plan)
|
||||
|
||||
|
||||
def _install_gateway_ca(
|
||||
plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint,
|
||||
) -> MacosContainerBottlePlan:
|
||||
"""Stage the SHARED gateway's CA for the provisioner to install, replacing
|
||||
the per-bottle CA the companion container used to mint. Every bottle on
|
||||
this host trusts this one CA."""
|
||||
ca_dir = egress_state_dir(plan.slug) / "gateway-ca"
|
||||
ca_dir.mkdir(parents=True, exist_ok=True)
|
||||
ca_file = ca_dir / "gateway-ca.pem"
|
||||
ca_file.write_text(endpoint.gateway_ca_pem)
|
||||
egress_plan = dataclasses.replace(
|
||||
plan.egress_plan,
|
||||
mitmproxy_ca_host_path=ca_file,
|
||||
mitmproxy_ca_cert_only_host_path=ca_file,
|
||||
)
|
||||
return dataclasses.replace(plan, egress_plan=egress_plan)
|
||||
|
||||
|
||||
def _stamp_agent_urls(
|
||||
plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint,
|
||||
) -> MacosContainerBottlePlan:
|
||||
"""Point the agent's git-gate insteadOf rewrites + supervise MCP at the
|
||||
shared gateway's ports. Both bypass the egress proxy (NO_PROXY covers the
|
||||
gateway address)."""
|
||||
git_gate_url = (
|
||||
f"http://{endpoint.gateway_ip}:{_GIT_HTTP_PORT}"
|
||||
if plan.git_gate_plan.upstreams else ""
|
||||
)
|
||||
supervise_url = (
|
||||
f"http://{endpoint.gateway_ip}:{SUPERVISE_PORT}/"
|
||||
if plan.supervise_plan is not None else ""
|
||||
)
|
||||
return dataclasses.replace(
|
||||
plan,
|
||||
agent_proxy_url=f"http://{endpoint.gateway_ip}:{EGRESS_PORT}",
|
||||
agent_git_gate_url=git_gate_url,
|
||||
agent_supervise_url=supervise_url,
|
||||
)
|
||||
|
||||
|
||||
def _proxy_url(gateway_ip: str, identity_token: str = "") -> str:
|
||||
"""The agent's egress proxy URL. The identity token rides as proxy
|
||||
credentials — the gateway reads Proxy-Authorization, resolves the
|
||||
(source_ip, token) pair against the control plane, and strips it before
|
||||
upstream. Without a valid pair `/resolve` denies the request (#366)."""
|
||||
cred = f"bottle:{identity_token}@" if identity_token else ""
|
||||
return f"http://{cred}{gateway_ip}:{EGRESS_PORT}"
|
||||
|
||||
|
||||
def _no_proxy(gateway_ip: str) -> str:
|
||||
# git-http + supervise live on the gateway and must NOT go through the
|
||||
# egress proxy — the agent reaches them directly by its address.
|
||||
return f"localhost,127.0.0.1,{gateway_ip}"
|
||||
|
||||
|
||||
def _identity_proxy_env(
|
||||
endpoint: GatewayEndpoint, identity_token: str,
|
||||
) -> dict[str, str]:
|
||||
"""The token-bearing proxy env applied at `container exec`. It supersedes
|
||||
the token-less run-time value (exec `--env` wins), which is the only way to
|
||||
get the token in: it does not exist until after the container runs."""
|
||||
if not identity_token:
|
||||
return {}
|
||||
url = _proxy_url(endpoint.gateway_ip, identity_token)
|
||||
return {
|
||||
"HTTPS_PROXY": url, "HTTP_PROXY": url,
|
||||
"https_proxy": url, "http_proxy": url,
|
||||
}
|
||||
|
||||
|
||||
def _start_agent(plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint) -> None:
|
||||
argv = _agent_run_argv(plan, endpoint)
|
||||
env = {**os.environ, **plan.forwarded_env}
|
||||
info(f"container run agent {plan.container_name}")
|
||||
result = subprocess.run(
|
||||
argv, capture_output=True, text=True, env=env, check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
die(
|
||||
f"container run for agent {plan.container_name} failed: "
|
||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||
)
|
||||
|
||||
|
||||
def _agent_run_argv(
|
||||
plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint,
|
||||
) -> list[str]:
|
||||
argv = [
|
||||
"container", "run",
|
||||
"--name", plan.container_name,
|
||||
"--detach",
|
||||
"--label", "bot-bottle.backend=macos-container",
|
||||
"--network", endpoint.network,
|
||||
# The attribution invariant: without NET_RAW the agent cannot open a
|
||||
# raw socket, so it cannot source-IP-spoof its neighbours on the shared
|
||||
# segment. NET_ADMIN is not granted by default, so its address and
|
||||
# route are already fixed. See the module docstring.
|
||||
"--cap-drop", "CAP_NET_RAW",
|
||||
]
|
||||
for entry in _agent_env_entries(plan, endpoint):
|
||||
argv += ["--env", entry]
|
||||
# The init process is a no-op: every agent command arrives via
|
||||
# `container exec`, which is also how the identity token gets in.
|
||||
argv += [plan.image, "sleep", _AGENT_SLEEP_SECONDS]
|
||||
return argv
|
||||
|
||||
|
||||
def _agent_env_entries(
|
||||
plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint,
|
||||
) -> tuple[str, ...]:
|
||||
# Token-less at run time — the token does not exist yet (see
|
||||
# `_identity_proxy_env`). Anything egressing before the exec-time override
|
||||
# is denied by `/resolve`, which is the safe direction.
|
||||
proxy_url = _proxy_url(endpoint.gateway_ip)
|
||||
no_proxy = _no_proxy(endpoint.gateway_ip)
|
||||
env = [
|
||||
f"HTTPS_PROXY={proxy_url}",
|
||||
f"HTTP_PROXY={proxy_url}",
|
||||
f"https_proxy={proxy_url}",
|
||||
f"http_proxy={proxy_url}",
|
||||
f"NO_PROXY={no_proxy}",
|
||||
f"no_proxy={no_proxy}",
|
||||
f"NODE_EXTRA_CA_CERTS={AGENT_CA_PATH}",
|
||||
f"SSL_CERT_FILE={AGENT_CA_BUNDLE}",
|
||||
f"REQUESTS_CA_BUNDLE={AGENT_CA_BUNDLE}",
|
||||
]
|
||||
if plan.agent_git_gate_url:
|
||||
env.append(f"GIT_GATE_URL={plan.agent_git_gate_url}")
|
||||
if plan.agent_supervise_url:
|
||||
env.append(f"MCP_SUPERVISE_URL={plan.agent_supervise_url}")
|
||||
for name, value in sorted(plan.agent_provision.guest_env.items()):
|
||||
env.append(f"{name}={value}")
|
||||
# Forwarded vars: bare name → inherits from the `container run` process env
|
||||
# so the secret value never lands on argv.
|
||||
for name in sorted(plan.forwarded_env.keys()):
|
||||
env.append(name)
|
||||
env.extend(egress_agent_env_entries(plan.egress_plan))
|
||||
return tuple(env)
|
||||
|
||||
|
||||
__all__ = ["launch"]
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
"""Orchestrator + gateway lifecycle for the macOS backend (PRD 0070).
|
||||
|
||||
The macOS counterpart of `orchestrator.lifecycle.OrchestratorService`. Same
|
||||
shape — an idempotent singleton that brings up the control plane and the shared
|
||||
gateway and hands back the control-plane URL — but the startup **order is
|
||||
reversed**, and the reason is a real Apple Container constraint rather than a
|
||||
stylistic choice:
|
||||
|
||||
- Docker starts the gateway first and lets it find the control plane by
|
||||
*container name* over docker DNS.
|
||||
- Apple Container 1.0.0 has **no container DNS** (see `gateway`), so the
|
||||
gateway can only be handed an **IP**. That IP does not exist until the
|
||||
orchestrator container is running — hence: orchestrator first, read its
|
||||
address, then start the gateway pointed at it.
|
||||
|
||||
The second difference: no published port. Apple Container puts every container
|
||||
on a host-reachable address, and the host can reach the host-only network
|
||||
directly, so the host CLI and the gateway use the **same** URL — the
|
||||
orchestrator's address on the shared network. Docker needs a
|
||||
`--publish 127.0.0.1:…` hop plus a separate `internal_url` for the same job.
|
||||
|
||||
Like the docker service this runs the **register-only broker**: the backend
|
||||
launches agent containers, so the control plane needs no privileged socket.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from ... import log
|
||||
from ...orchestrator.lifecycle import (
|
||||
DEFAULT_PORT,
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
ORCHESTRATOR_DOCKERFILE,
|
||||
ORCHESTRATOR_IMAGE,
|
||||
ORCHESTRATOR_SOURCE_HASH_LABEL,
|
||||
OrchestratorStartError,
|
||||
source_hash,
|
||||
)
|
||||
from ...paths import bot_bottle_root
|
||||
from . import util as container_mod
|
||||
from .gateway import (
|
||||
GATEWAY_EGRESS_NETWORK,
|
||||
GATEWAY_IMAGE,
|
||||
GATEWAY_NETWORK,
|
||||
AppleGateway,
|
||||
ensure_networks,
|
||||
)
|
||||
|
||||
# Distinct from the docker backend's container name so both can run on one host.
|
||||
ORCHESTRATOR_NAME = "bot-bottle-mac-orchestrator"
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
_APP_DIR = "/app"
|
||||
_ROOT_IN_CONTAINER = "/bot-bottle-root"
|
||||
|
||||
_HEALTH_POLL_SECONDS = 0.25
|
||||
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
|
||||
|
||||
|
||||
def _mount(source: str, target: str, *, readonly: bool = False) -> str:
|
||||
spec = f"type=bind,source={source},target={target}"
|
||||
if readonly:
|
||||
spec += ",readonly"
|
||||
return spec
|
||||
|
||||
|
||||
class MacosOrchestratorService:
|
||||
"""Manages the orchestrator control-plane container + the shared Apple
|
||||
gateway. Callers only need `ensure_running()`, which returns the URL."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
port: int = DEFAULT_PORT,
|
||||
network: str = GATEWAY_NETWORK,
|
||||
egress_network: str = GATEWAY_EGRESS_NETWORK,
|
||||
image: str = ORCHESTRATOR_IMAGE,
|
||||
gateway_image: str = GATEWAY_IMAGE,
|
||||
repo_root: Path = _REPO_ROOT,
|
||||
host_root: Path | None = None,
|
||||
orchestrator_name: str = ORCHESTRATOR_NAME,
|
||||
) -> None:
|
||||
self.port = port
|
||||
self.network = network
|
||||
self.egress_network = egress_network
|
||||
self.image = image
|
||||
self._gateway_image = gateway_image
|
||||
self._repo_root = repo_root
|
||||
self._host_root = host_root or bot_bottle_root()
|
||||
self._orchestrator_name = orchestrator_name
|
||||
# Resolved once the container is up — there is no name to fall back on.
|
||||
self._url = ""
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
"""The control-plane URL, or "" before the container is up. One URL for
|
||||
both the host CLI and the gateway (see the module docstring)."""
|
||||
return self._url
|
||||
|
||||
def _resolve_url(self) -> str:
|
||||
"""The control-plane URL, or "" while the container has no address."""
|
||||
ip = container_mod.try_container_ipv4_on_network(
|
||||
self._orchestrator_name, self.network,
|
||||
)
|
||||
return f"http://{ip}:{self.port}" if ip else ""
|
||||
|
||||
def is_healthy(
|
||||
self, url: str = "", *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS,
|
||||
) -> bool:
|
||||
target = url or self._url
|
||||
if not target:
|
||||
return False
|
||||
try:
|
||||
with urllib.request.urlopen(f"{target}/health", timeout=timeout) as resp:
|
||||
return resp.status == 200
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
return False
|
||||
|
||||
def _ensure_orchestrator_image(self) -> None:
|
||||
"""Build the lean control-plane image when missing. Build-if-missing,
|
||||
not build-every-time: the control plane bind-mounts its source, so a
|
||||
code change is caught by the source-hash recreate, not a rebuild."""
|
||||
if container_mod.image_exists(self.image):
|
||||
return
|
||||
container_mod.build_image(
|
||||
self.image, str(self._repo_root), dockerfile=ORCHESTRATOR_DOCKERFILE,
|
||||
)
|
||||
|
||||
def _orchestrator_source_current(self, current_hash: str) -> bool:
|
||||
"""True iff the running orchestrator was created from the *current*
|
||||
bind-mounted source. The process loaded that code at startup and won't
|
||||
reload it, so a stale container would keep serving OLD control-plane
|
||||
code."""
|
||||
if not container_mod.container_is_running(self._orchestrator_name):
|
||||
return False
|
||||
data = container_mod.inspect_container(self._orchestrator_name)
|
||||
config = data.get("configuration")
|
||||
labels = config.get("labels") if isinstance(config, dict) else None
|
||||
if not isinstance(labels, dict):
|
||||
return True # can't compare → don't churn a working container
|
||||
return labels.get(ORCHESTRATOR_SOURCE_HASH_LABEL) == current_hash
|
||||
|
||||
def _run_orchestrator_container(self, current_hash: str) -> None:
|
||||
# The orchestrator is the first thing on the shared network, so it —
|
||||
# not the gateway — is what has to bring the network into existence.
|
||||
ensure_networks(self.network, self.egress_network)
|
||||
container_mod.force_remove_container(self._orchestrator_name)
|
||||
argv = [
|
||||
"container", "run", "--detach",
|
||||
"--name", self._orchestrator_name,
|
||||
"--label", "bot-bottle.backend=macos-container",
|
||||
"--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current_hash}",
|
||||
# Host-only network only: the control plane needs no route out, and
|
||||
# the host reaches it here directly (no --publish needed).
|
||||
"--network", self.network,
|
||||
"--mount", _mount(str(self._repo_root), _APP_DIR, readonly=True),
|
||||
"--workdir", _APP_DIR,
|
||||
# Persist the registry DB on the host (sole-owner: only the
|
||||
# orchestrator opens bot-bottle.db).
|
||||
"--mount", _mount(str(self._host_root), _ROOT_IN_CONTAINER),
|
||||
"--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}",
|
||||
"--entrypoint", "python3",
|
||||
self.image,
|
||||
"-m", "bot_bottle.orchestrator",
|
||||
"--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub",
|
||||
]
|
||||
result = container_mod.run_container_argv(argv)
|
||||
if result.returncode != 0:
|
||||
raise OrchestratorStartError(
|
||||
f"orchestrator container failed to start: "
|
||||
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||
)
|
||||
|
||||
def gateway(self, orchestrator_url: str) -> AppleGateway:
|
||||
"""The shared gateway, pointed at the control plane at
|
||||
`orchestrator_url` (by IP — Apple has no container DNS)."""
|
||||
return AppleGateway(
|
||||
self._gateway_image,
|
||||
network=self.network,
|
||||
egress_network=self.egress_network,
|
||||
orchestrator_url=orchestrator_url,
|
||||
)
|
||||
|
||||
def ensure_running(
|
||||
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
) -> str:
|
||||
"""Ensure the control plane + shared gateway are up; return the
|
||||
control-plane URL. Idempotent — a healthy control plane running current
|
||||
code and a running gateway are left untouched."""
|
||||
current_hash = source_hash(self._repo_root)
|
||||
if not self._orchestrator_source_current(current_hash):
|
||||
self._ensure_orchestrator_image()
|
||||
log.info(
|
||||
"starting orchestrator container",
|
||||
context={"name": self._orchestrator_name},
|
||||
)
|
||||
self._run_orchestrator_container(current_hash)
|
||||
|
||||
url = self._wait_healthy(startup_timeout)
|
||||
self._url = url
|
||||
|
||||
# Gateway second: it can only reach the control plane by IP, which does
|
||||
# not exist until the orchestrator container is up (see the docstring).
|
||||
gateway = self.gateway(url)
|
||||
gateway.ensure_built()
|
||||
gateway.ensure_running()
|
||||
return url
|
||||
|
||||
def _wait_healthy(self, startup_timeout: float) -> str:
|
||||
"""Poll until the control plane answers /health, resolving its address
|
||||
each time: the container is up before it has an IP, and it has an IP
|
||||
before the server binds."""
|
||||
deadline = time.monotonic() + startup_timeout
|
||||
while True:
|
||||
url = self._resolve_url()
|
||||
if url and self.is_healthy(url):
|
||||
log.info("orchestrator healthy", context={"url": url})
|
||||
return url
|
||||
if time.monotonic() >= deadline:
|
||||
raise OrchestratorStartError(
|
||||
f"orchestrator did not become healthy within "
|
||||
f"{startup_timeout:g}s"
|
||||
)
|
||||
time.sleep(_HEALTH_POLL_SECONDS)
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Remove the orchestrator + gateway containers (idempotent)."""
|
||||
container_mod.force_remove_container(self._orchestrator_name)
|
||||
self.gateway("").stop()
|
||||
self._url = ""
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MacosOrchestratorService",
|
||||
"OrchestratorStartError",
|
||||
"ORCHESTRATOR_NAME",
|
||||
]
|
||||
@@ -437,22 +437,131 @@ def inspect_container(name: str) -> dict[str, object]:
|
||||
|
||||
|
||||
def container_ipv4_on_network(name: str, network: str) -> str:
|
||||
data = inspect_container(name)
|
||||
"""The container's IPv4 address on `network`. Fatal if absent — callers
|
||||
that can tolerate "not yet" want `try_container_ipv4_on_network`."""
|
||||
ip = try_container_ipv4_on_network(name, network)
|
||||
if not ip:
|
||||
die(f"container {name} has no IPv4 address on {network}")
|
||||
return ip
|
||||
|
||||
|
||||
def run_container_argv(argv: list[str]) -> subprocess.CompletedProcess[str]:
|
||||
"""Run a `container` command, returning the result for the caller to
|
||||
interpret. Unlike the `die`-on-failure helpers above, this lets callers
|
||||
that raise their own typed errors (the gateway / orchestrator lifecycle)
|
||||
keep control of the failure path."""
|
||||
return subprocess.run(argv, capture_output=True, text=True, check=False)
|
||||
|
||||
|
||||
def _normalize_digest(value: str) -> str:
|
||||
return value.split(":", 1)[1] if ":" in value else value
|
||||
|
||||
|
||||
def image_digest(ref: str) -> str:
|
||||
"""The digest of image `ref`, or "" if it can't be read. Empty is a
|
||||
'don't know' signal — callers treat it as "don't churn a working
|
||||
container" rather than as a mismatch."""
|
||||
result = run_container_argv([_CONTAINER, "image", "inspect", ref])
|
||||
if result.returncode != 0:
|
||||
return ""
|
||||
try:
|
||||
data = json.loads(result.stdout or "{}")
|
||||
except json.JSONDecodeError:
|
||||
return ""
|
||||
if isinstance(data, list) and data:
|
||||
data = data[0]
|
||||
if not isinstance(data, dict):
|
||||
return ""
|
||||
config = data.get("configuration")
|
||||
if isinstance(config, dict):
|
||||
descriptor = config.get("descriptor")
|
||||
if isinstance(descriptor, dict) and descriptor.get("digest"):
|
||||
return _normalize_digest(str(descriptor["digest"]))
|
||||
value = data.get("id")
|
||||
return _normalize_digest(str(value)) if value else ""
|
||||
|
||||
|
||||
def container_image_digest(name: str) -> str:
|
||||
"""The digest of the image container `name` was created from, or "" if it
|
||||
can't be read. Compare with `image_digest(ref)` to tell whether a running
|
||||
container predates an image rebuild."""
|
||||
result = run_container_argv([_CONTAINER, "inspect", name])
|
||||
if result.returncode != 0:
|
||||
return ""
|
||||
try:
|
||||
data = json.loads(result.stdout or "[]")
|
||||
except json.JSONDecodeError:
|
||||
return ""
|
||||
if isinstance(data, list) and data:
|
||||
data = data[0]
|
||||
if not isinstance(data, dict):
|
||||
return ""
|
||||
config = data.get("configuration")
|
||||
if not isinstance(config, dict):
|
||||
return ""
|
||||
image = config.get("image")
|
||||
if not isinstance(image, dict):
|
||||
return ""
|
||||
descriptor = image.get("descriptor")
|
||||
if isinstance(descriptor, dict) and descriptor.get("digest"):
|
||||
return _normalize_digest(str(descriptor["digest"]))
|
||||
return ""
|
||||
|
||||
|
||||
def container_env(name: str) -> dict[str, str]:
|
||||
"""The env container `name` was started with, or {} if unreadable. Lets a
|
||||
caller tell whether a running container's baked-in configuration still
|
||||
matches what it would pass today."""
|
||||
result = run_container_argv([_CONTAINER, "inspect", name])
|
||||
if result.returncode != 0:
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(result.stdout or "[]")
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
if isinstance(data, list) and data:
|
||||
data = data[0]
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
config = data.get("configuration")
|
||||
init = config.get("initProcess") if isinstance(config, dict) else None
|
||||
entries = init.get("environment") if isinstance(init, dict) else None
|
||||
if not isinstance(entries, list):
|
||||
return {}
|
||||
env: dict[str, str] = {}
|
||||
for entry in entries:
|
||||
if isinstance(entry, str) and "=" in entry:
|
||||
key, value = entry.split("=", 1)
|
||||
env[key] = value
|
||||
return env
|
||||
|
||||
|
||||
def try_container_ipv4_on_network(name: str, network: str) -> str:
|
||||
"""`container_ipv4_on_network` without the fatal exit: "" when the address
|
||||
isn't readable yet. For pollers — a container is created before it has an
|
||||
address, so "not yet" is an expected state there, not an error."""
|
||||
result = run_container_argv([_CONTAINER, "inspect", name])
|
||||
if result.returncode != 0:
|
||||
return ""
|
||||
try:
|
||||
data = json.loads(result.stdout or "[]")
|
||||
except json.JSONDecodeError:
|
||||
return ""
|
||||
if isinstance(data, list) and data:
|
||||
data = data[0]
|
||||
if not isinstance(data, dict):
|
||||
return ""
|
||||
status = data.get("status")
|
||||
networks = status.get("networks") if isinstance(status, dict) else None
|
||||
if not isinstance(networks, list):
|
||||
die(f"container inspect {name} did not include status.networks")
|
||||
return ""
|
||||
for entry in networks:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
if entry.get("network") != network:
|
||||
if not isinstance(entry, dict) or entry.get("network") != network:
|
||||
continue
|
||||
raw = entry.get("ipv4Address")
|
||||
if not isinstance(raw, str) or not raw:
|
||||
die(f"container {name} has no IPv4 address on {network}")
|
||||
return raw.split("/", 1)[0]
|
||||
die(f"container {name} is not attached to network {network}")
|
||||
raise AssertionError("unreachable")
|
||||
if isinstance(raw, str) and raw:
|
||||
return raw.split("/", 1)[0]
|
||||
return ""
|
||||
|
||||
|
||||
def image_id(ref: str) -> str:
|
||||
|
||||
@@ -44,6 +44,7 @@ from .paths import bot_bottle_root
|
||||
_STATE_SUBDIR = "state"
|
||||
_PER_BOTTLE_DOCKERFILE_NAME = "Dockerfile"
|
||||
_COMMITTED_IMAGE_NAME = "committed-image"
|
||||
_COMMITTED_ROOTFS_NAME = "committed-rootfs.tar"
|
||||
_TRANSCRIPT_SUBDIR = "transcript"
|
||||
# Per-daemon scratch subdirs. PRD 0018 chunk 2: bind-mount sources
|
||||
# live here so chunk 3's `docker compose up` can find them at stable
|
||||
@@ -200,6 +201,15 @@ def committed_image_path(identity: str) -> Path:
|
||||
return bottle_state_dir(identity) / _COMMITTED_IMAGE_NAME
|
||||
|
||||
|
||||
def committed_rootfs_path(identity: str) -> Path:
|
||||
"""Where the Firecracker freezer stores a snapshot of the bottle's
|
||||
guest rootfs (a plain tar). This is the resumable/migratable artifact
|
||||
the Firecracker backend boots from — no Docker image involved. The
|
||||
matching `committed-image` state file records that a snapshot exists
|
||||
(and its path); `resume` boots from this tar when both are present."""
|
||||
return bottle_state_dir(identity) / _COMMITTED_ROOTFS_NAME
|
||||
|
||||
|
||||
def write_committed_image(identity: str, image_tag: str) -> Path:
|
||||
"""Persist the committed image tag for `identity`. The next
|
||||
`cli.py resume <identity>` will boot from this image instead of
|
||||
@@ -354,6 +364,7 @@ __all__ = [
|
||||
"cleanup_state",
|
||||
"clear_preserve_marker",
|
||||
"committed_image_path",
|
||||
"committed_rootfs_path",
|
||||
"egress_state_dir",
|
||||
"git_gate_state_dir",
|
||||
"is_preserved",
|
||||
|
||||
@@ -41,7 +41,7 @@ ORCHESTRATOR_IMAGE = os.environ.get(
|
||||
ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator"
|
||||
# Baked onto the container as a label so `ensure_running` can tell whether the
|
||||
# running process is executing the *current* bind-mounted source — see
|
||||
# `_source_hash`.
|
||||
# `source_hash`.
|
||||
ORCHESTRATOR_SOURCE_HASH_LABEL = "bot-bottle-orchestrator-source-hash"
|
||||
|
||||
# The repo root is bind-mounted into the control-plane container so
|
||||
@@ -60,7 +60,7 @@ class OrchestratorStartError(RuntimeError):
|
||||
"""The orchestrator container did not become healthy within the timeout."""
|
||||
|
||||
|
||||
def _source_hash(repo_root: Path) -> str:
|
||||
def source_hash(repo_root: Path) -> str:
|
||||
"""Content hash of the orchestrator's bind-mounted Python source (the
|
||||
`bot_bottle` package the control-plane process imports). This only
|
||||
changes when the code that would actually run inside the container
|
||||
@@ -134,17 +134,17 @@ class OrchestratorService:
|
||||
proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"])
|
||||
return name in proc.stdout.split()
|
||||
|
||||
def _run_orchestrator_container(self, source_hash: str) -> None:
|
||||
def _run_orchestrator_container(self, current_hash: str) -> None:
|
||||
"""Start the control-plane container (idempotent: clears a stale
|
||||
fixed-name container first). Register-only broker → no docker socket.
|
||||
Labels the container with `source_hash` so a later `ensure_running`
|
||||
can detect a real code change (see `_source_hash`)."""
|
||||
Labels the container with `current_hash` so a later `ensure_running`
|
||||
can detect a real code change (see `source_hash`)."""
|
||||
run_docker(["docker", "rm", "--force", self._orchestrator_name])
|
||||
proc = run_docker([
|
||||
"docker", "run", "--detach",
|
||||
"--name", self._orchestrator_name,
|
||||
"--label", self._orchestrator_label,
|
||||
"--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={source_hash}",
|
||||
"--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current_hash}",
|
||||
"--network", self.network,
|
||||
# Host CLI reaches the control plane here; bound to loopback so it
|
||||
# is not exposed on the host's external interfaces.
|
||||
@@ -224,7 +224,7 @@ class OrchestratorService:
|
||||
# launch (the prior behaviour) would drop every other active
|
||||
# bottle's in-memory egress tokens each time a new bottle starts,
|
||||
# since the orchestrator process holds them only in memory (#381).
|
||||
current_hash = _source_hash(self._repo_root)
|
||||
current_hash = source_hash(self._repo_root)
|
||||
if self.is_healthy() and self._orchestrator_source_current(current_hash):
|
||||
return self.url
|
||||
|
||||
|
||||
@@ -358,3 +358,111 @@ the Apple Container-specific constraints directly:
|
||||
|
||||
Do not implement the backend as a direct clone of Docker Compose
|
||||
service aliases. That assumption failed in this run.
|
||||
|
||||
## Addendum: consolidated-gateway findings (2026-07-17, PRD 0070)
|
||||
|
||||
Re-tested on Apple Container 1.0.0 while porting the backend to the
|
||||
per-host consolidated gateway (#351). The two-network shape above still
|
||||
holds; these are the additional constraints that shaped the port, each
|
||||
verified against the live CLI on this host.
|
||||
|
||||
### No static IP for a container
|
||||
|
||||
`container run --network` accepts only
|
||||
`<name>[,mac=XX:XX:XX:XX:XX:XX][,mtu=VALUE]`. There is no `--ip`. The
|
||||
address comes from vmnet's DHCP and is knowable only after the container
|
||||
is running:
|
||||
|
||||
```console
|
||||
$ container run --name a --network bb-net --detach alpine sleep 900
|
||||
$ container inspect a | jq -r '.[0].status.networks[0].ipv4Address'
|
||||
192.168.128.3/24
|
||||
```
|
||||
|
||||
Consequence: the docker backend's "allocate a free IP -> pin it with
|
||||
`--ip` -> register -> launch" order cannot be reproduced. macOS inverts
|
||||
it to "launch -> read the assigned address -> register". The identity
|
||||
token therefore cannot be in the agent's run-time env (registration mints
|
||||
it after the container exists) and is delivered at `container exec` time.
|
||||
|
||||
### Networks are fixed at run time
|
||||
|
||||
There is no `container network connect`; `container network` exposes only
|
||||
`create`, `delete`, `list`, `inspect`, `prune`. A network cannot be
|
||||
attached to a running container, so a *persistent* shared gateway rules
|
||||
out per-bottle networks — they would force a gateway restart per launch.
|
||||
One shared host-only network, created up front, is the only shape that
|
||||
keeps the gateway a singleton.
|
||||
|
||||
### No container DNS
|
||||
|
||||
Containers cannot resolve each other by name; the host-only network's
|
||||
resolver refuses the query:
|
||||
|
||||
```console
|
||||
$ container exec agent nslookup gw
|
||||
;; connection timed out; no servers could be reached
|
||||
$ container exec agent cat /etc/resolv.conf
|
||||
nameserver 192.168.128.1
|
||||
```
|
||||
|
||||
Consequence: the gateway is handed the control plane's **IP**, not a
|
||||
container name as on docker. That forces the startup order
|
||||
orchestrator -> read its address -> gateway.
|
||||
|
||||
### The host can reach the host-only network directly
|
||||
|
||||
```console
|
||||
$ container inspect c | jq -r '.[0].status.networks[0].ipv4Address'
|
||||
192.168.128.2/24
|
||||
$ curl -s http://192.168.128.2:8099/i
|
||||
ok
|
||||
```
|
||||
|
||||
So no `--publish` hop is needed: the host CLI and the gateway use the
|
||||
same control-plane URL. Docker needs `--publish 127.0.0.1:...` plus a
|
||||
separate internal URL for the same job.
|
||||
|
||||
### CAP_NET_RAW is granted by default — and matters for attribution
|
||||
|
||||
Apple grants NET_RAW but not NET_ADMIN. The agent therefore cannot change
|
||||
its own address or route:
|
||||
|
||||
```console
|
||||
$ container exec agent ip addr add 192.168.128.99/24 dev eth0
|
||||
ip: RTNETLINK answers: Operation not permitted
|
||||
$ container exec agent ip route replace default via 192.168.128.2 dev eth0
|
||||
ip: RTNETLINK answers: Operation not permitted
|
||||
$ container exec agent grep CapEff /proc/self/status
|
||||
CapEff: 00000000a80425fb # bit 13 (NET_RAW) set, bit 12 (NET_ADMIN) clear
|
||||
```
|
||||
|
||||
But NET_RAW permits raw sockets, i.e. source-address forgery against
|
||||
neighbours on the shared segment — directly against PRD 0070's invariant
|
||||
("a packet's source address, as seen by the orchestrator, provably
|
||||
identifies the originating bottle"). `--cap-drop CAP_NET_RAW` closes it:
|
||||
|
||||
```console
|
||||
$ container run --cap-drop CAP_NET_RAW ... alpine
|
||||
$ container exec nr grep CapEff /proc/self/status
|
||||
CapEff: 00000000a80405fb # bit 13 cleared
|
||||
$ container exec nr ping -c1 192.168.128.2
|
||||
ping: permission denied (are you root?)
|
||||
```
|
||||
|
||||
The agent is run with `--cap-drop CAP_NET_RAW` for this reason.
|
||||
|
||||
### `container exec` inherits run-time env, and `--env` overrides it
|
||||
|
||||
```console
|
||||
$ container run --name e --env FOO=from_run --detach alpine sleep 120
|
||||
$ container exec e sh -c 'echo $FOO'
|
||||
from_run
|
||||
$ container exec --env FOO=from_exec e sh -c 'echo $FOO'
|
||||
from_exec
|
||||
```
|
||||
|
||||
This is what makes exec-time identity-token delivery work: the token-less
|
||||
proxy URL baked in at launch is superseded by the token-bearing one at
|
||||
exec. Bare `--env NAME` (inherit from the parent process) keeps the token
|
||||
value off argv.
|
||||
|
||||
@@ -205,25 +205,29 @@ class TestFirecrackerFreezer(_FakeHomeMixin, unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_snapshots_running_vm_without_stopping(self):
|
||||
"""Commit should tar the running guest rootfs over SSH, not stop it."""
|
||||
"""Commit should tar the running guest rootfs over SSH into the
|
||||
committed-rootfs artifact (no Docker), not stop the VM."""
|
||||
slug = "dev-abc12"
|
||||
self._write_meta(slug)
|
||||
self._stage_run_dir(slug)
|
||||
freezer = FirecrackerFreezer()
|
||||
agent = _make_agent(slug, "firecracker")
|
||||
|
||||
with patch("bot_bottle.backend.firecracker.freezer._commit_via_ssh") as mock_commit, \
|
||||
commit_fn = "bot_bottle.backend.firecracker.freezer._commit_rootfs_via_ssh"
|
||||
with patch(commit_fn) as mock_commit, \
|
||||
patch("bot_bottle.backend.freeze.info"), \
|
||||
patch("bot_bottle.backend.firecracker.freezer.info"):
|
||||
freezer.commit(agent)
|
||||
|
||||
image_tag = f"bot-bottle-committed-{slug}:latest"
|
||||
tar_path = bottle_state.committed_rootfs_path(slug)
|
||||
self.assertEqual(1, mock_commit.call_count)
|
||||
# (private_key, guest_ip, image_tag) — guest_ip parsed from config.
|
||||
# (private_key, guest_ip, tar_path) — guest_ip parsed from config.
|
||||
args = mock_commit.call_args.args
|
||||
self.assertEqual("100.64.0.1", args[1])
|
||||
self.assertEqual(image_tag, args[2])
|
||||
self.assertEqual(image_tag, bottle_state.read_committed_image(slug))
|
||||
self.assertEqual(tar_path, args[2])
|
||||
# The committed-image state records the artifact path; resume boots
|
||||
# from the tar rather than a Docker image.
|
||||
self.assertEqual(str(tar_path), bottle_state.read_committed_image(slug))
|
||||
self.assertTrue(bottle_state.is_preserved(slug))
|
||||
|
||||
|
||||
|
||||
@@ -6,10 +6,13 @@ branches. Mock subprocess/os so nothing needs KVM or a live VM.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.backend.firecracker import firecracker_vm, freezer, netpool, util
|
||||
@@ -128,5 +131,172 @@ class TestRequireFirecracker(unittest.TestCase):
|
||||
util.require_firecracker()
|
||||
|
||||
|
||||
class TestBuildCommittedRootfsDir(unittest.TestCase):
|
||||
"""Resume prepares the base rootfs dir from the freezer's snapshot tar
|
||||
with no Docker: extract, recreate the excluded mount points, inject the
|
||||
guest boot bits."""
|
||||
|
||||
def _make_tar(self, tmp: Path) -> Path:
|
||||
import tarfile
|
||||
|
||||
src = tmp / "src"
|
||||
(src / "home" / "node").mkdir(parents=True)
|
||||
(src / "home" / "node" / "hello").write_text("hi")
|
||||
tar_path = tmp / "rootfs.tar"
|
||||
with tarfile.open(tar_path, "w") as tar:
|
||||
tar.add(src, arcname=".")
|
||||
return tar_path
|
||||
|
||||
def test_extracts_recreates_mountpoints_and_injects_boot(self):
|
||||
with tempfile.TemporaryDirectory(prefix="fc-committed.") as d:
|
||||
tmp = Path(d)
|
||||
tar_path = self._make_tar(tmp)
|
||||
# Stand in for the static dropbear that inject_guest_boot copies.
|
||||
dropbear = tmp / "dropbear"
|
||||
dropbear.write_text("#!/bin/true\n")
|
||||
cache = tmp / "cache"
|
||||
cache.mkdir()
|
||||
|
||||
with patch.object(util, "cache_dir", return_value=cache), \
|
||||
patch.object(util, "dropbear_path", return_value=dropbear), \
|
||||
patch.object(util, "info"):
|
||||
base = util.build_committed_rootfs_dir(tar_path)
|
||||
|
||||
self.assertEqual("hi", (base / "home" / "node" / "hello").read_text())
|
||||
for mount_point in ("proc", "sys", "dev", "run"):
|
||||
self.assertTrue((base / mount_point).is_dir(),
|
||||
f"missing recreated mount point /{mount_point}")
|
||||
self.assertTrue((base / "bb-dropbear").is_file())
|
||||
self.assertTrue((base / "bb-init").is_file())
|
||||
self.assertTrue((base / ".bb-ready").is_file())
|
||||
|
||||
def test_caches_on_repeat_and_reextracts_after_refreeze(self):
|
||||
with tempfile.TemporaryDirectory(prefix="fc-committed.") as d:
|
||||
tmp = Path(d)
|
||||
tar_path = self._make_tar(tmp)
|
||||
dropbear = tmp / "dropbear"
|
||||
dropbear.write_text("#!/bin/true\n")
|
||||
cache = tmp / "cache"
|
||||
cache.mkdir()
|
||||
|
||||
ctx = [
|
||||
patch.object(util, "cache_dir", return_value=cache),
|
||||
patch.object(util, "dropbear_path", return_value=dropbear),
|
||||
patch.object(util, "info"),
|
||||
]
|
||||
for c in ctx:
|
||||
c.start()
|
||||
self.addCleanup(lambda: [c.stop() for c in ctx])
|
||||
|
||||
real_run = subprocess.run
|
||||
calls = {"n": 0}
|
||||
|
||||
def counting_run(argv: list[str], *a: Any, **k: Any) -> Any:
|
||||
if argv and argv[0] == "tar":
|
||||
calls["n"] += 1
|
||||
return real_run(argv, *a, **k)
|
||||
|
||||
with patch.object(util.subprocess, "run", side_effect=counting_run):
|
||||
first = util.build_committed_rootfs_dir(tar_path)
|
||||
second = util.build_committed_rootfs_dir(tar_path)
|
||||
self.assertEqual(first, second)
|
||||
self.assertEqual(1, calls["n"]) # cached — no re-extract
|
||||
|
||||
# A re-freeze rewrites the tar; a new size/mtime -> new cache
|
||||
# key -> re-extract. Force a distinct mtime so the test isn't
|
||||
# at the mercy of filesystem timestamp granularity.
|
||||
import tarfile
|
||||
extra = tmp / "extra"
|
||||
extra.mkdir()
|
||||
(extra / "note").write_text("v2")
|
||||
with tarfile.open(tar_path, "w") as tar:
|
||||
tar.add(extra, arcname=".")
|
||||
st = tar_path.stat()
|
||||
os.utime(tar_path, ns=(st.st_atime_ns, st.st_mtime_ns + 1_000_000_000))
|
||||
|
||||
third = util.build_committed_rootfs_dir(tar_path)
|
||||
self.assertNotEqual(first, third)
|
||||
self.assertEqual(2, calls["n"])
|
||||
|
||||
|
||||
class TestInjectGuestBootSymlinkSafe(unittest.TestCase):
|
||||
"""A committed snapshot is guest-controlled: inject_guest_boot must not
|
||||
follow a planted symlink and overwrite a host file during resume."""
|
||||
|
||||
def test_planted_symlink_does_not_escape_staging_tree(self):
|
||||
with tempfile.TemporaryDirectory(prefix="fc-inject.") as d:
|
||||
tmp = Path(d)
|
||||
dropbear = tmp / "dropbear"
|
||||
dropbear.write_bytes(b"DROPBEAR")
|
||||
# A host file the malicious snapshot tries to clobber.
|
||||
victim = tmp / "victim"
|
||||
victim.write_text("original")
|
||||
|
||||
rootfs = tmp / "rootfs"
|
||||
rootfs.mkdir()
|
||||
# The snapshot planted bb-init/bb-dropbear as symlinks to it.
|
||||
(rootfs / "bb-init").symlink_to(victim)
|
||||
(rootfs / "bb-dropbear").symlink_to(victim)
|
||||
|
||||
with patch.object(util, "dropbear_path", return_value=dropbear):
|
||||
util.inject_guest_boot(rootfs, init_script="#!/bin/sh\nreal\n")
|
||||
|
||||
# Host file untouched; the staged paths are fresh regular files.
|
||||
self.assertEqual("original", victim.read_text())
|
||||
self.assertFalse((rootfs / "bb-init").is_symlink())
|
||||
self.assertFalse((rootfs / "bb-dropbear").is_symlink())
|
||||
self.assertEqual("#!/bin/sh\nreal\n", (rootfs / "bb-init").read_text())
|
||||
self.assertEqual(b"DROPBEAR", (rootfs / "bb-dropbear").read_bytes())
|
||||
|
||||
|
||||
class TestCommitRootfsPermissions(unittest.TestCase):
|
||||
"""The snapshot tar can hold the bottle's private workspace, so the
|
||||
freezer must write it owner-only (0600)."""
|
||||
|
||||
def _commit(self, tar_path: Path) -> int:
|
||||
"""Run _commit_rootfs_via_ssh with a stubbed ssh|tar pipe; return the
|
||||
mode of the open partial observed mid-stream (from subprocess.run)."""
|
||||
key = tar_path.parent.parent / "key"
|
||||
key.write_text("K")
|
||||
seen: dict[str, int] = {}
|
||||
|
||||
def fake_run(argv: list[str], *a: Any, **k: Any) -> Any:
|
||||
out = k["stdout"]
|
||||
seen["mode"] = stat.S_IMODE(os.fstat(out.fileno()).st_mode)
|
||||
out.write(b"TARDATA")
|
||||
return subprocess.CompletedProcess(argv, 0, b"", b"")
|
||||
|
||||
with patch.object(freezer.util, "ssh_base_argv", return_value=["ssh"]), \
|
||||
patch.object(freezer.subprocess, "run", side_effect=fake_run):
|
||||
freezer._commit_rootfs_via_ssh(key, "10.0.0.1", tar_path)
|
||||
return seen["mode"]
|
||||
|
||||
def test_snapshot_created_owner_only(self):
|
||||
with tempfile.TemporaryDirectory(prefix="fc-freeze.") as d:
|
||||
tar_path = Path(d) / "state" / "committed-rootfs.tar"
|
||||
tar_path.parent.mkdir()
|
||||
stream_mode = self._commit(tar_path)
|
||||
|
||||
self.assertEqual(0o600, stream_mode) # private during the stream
|
||||
self.assertEqual(b"TARDATA", tar_path.read_bytes())
|
||||
self.assertEqual(0o600, stat.S_IMODE(tar_path.stat().st_mode))
|
||||
|
||||
def test_leftover_world_readable_partial_is_recreated_private(self):
|
||||
"""A partial left 0644 by an interrupted prior run must not keep the
|
||||
new snapshot world-readable while it streams."""
|
||||
with tempfile.TemporaryDirectory(prefix="fc-freeze.") as d:
|
||||
tar_path = Path(d) / "state" / "committed-rootfs.tar"
|
||||
tar_path.parent.mkdir()
|
||||
partial = tar_path.with_name(tar_path.name + ".partial")
|
||||
partial.write_bytes(b"stale")
|
||||
os.chmod(partial, 0o644)
|
||||
|
||||
stream_mode = self._commit(tar_path)
|
||||
|
||||
self.assertEqual(0o600, stream_mode)
|
||||
self.assertEqual(b"TARDATA", tar_path.read_bytes())
|
||||
self.assertEqual(0o600, stat.S_IMODE(tar_path.stat().st_mode))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Unit: macOS consolidated launch — register-after-start (PRD 0070)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
from bot_bottle.backend.macos_container.consolidated_launch import (
|
||||
GatewayEndpoint,
|
||||
ensure_gateway,
|
||||
register_agent,
|
||||
teardown_consolidated,
|
||||
)
|
||||
from bot_bottle.egress import EgressPlan, EgressRoute
|
||||
from bot_bottle.git_gate import GitGatePlan
|
||||
from bot_bottle.orchestrator.client import RegisteredBottle
|
||||
|
||||
_MOD = "bot_bottle.backend.macos_container.consolidated_launch"
|
||||
|
||||
|
||||
def _egress_plan() -> EgressPlan:
|
||||
return EgressPlan(
|
||||
slug="demo", routes_path=Path("/x"),
|
||||
routes=(EgressRoute(host="api.example.com"),), token_env_map={},
|
||||
)
|
||||
|
||||
|
||||
def _git_plan() -> GitGatePlan:
|
||||
return GitGatePlan(
|
||||
slug="demo", entrypoint_script=Path(), hook_script=Path(),
|
||||
access_hook_script=Path(), upstreams=(),
|
||||
)
|
||||
|
||||
|
||||
def _endpoint() -> GatewayEndpoint:
|
||||
return GatewayEndpoint(
|
||||
orchestrator_url="http://192.168.128.2:8099",
|
||||
gateway_ip="192.168.128.3",
|
||||
gateway_ca_pem="-----BEGIN CERTIFICATE-----\n",
|
||||
network="bot-bottle-mac-gateway",
|
||||
)
|
||||
|
||||
|
||||
def _client() -> Mock:
|
||||
c = Mock()
|
||||
c.register_bottle.return_value = RegisteredBottle("b1", "tok")
|
||||
return c
|
||||
|
||||
|
||||
class TestEnsureGateway(unittest.TestCase):
|
||||
def _run(self, service: MagicMock) -> GatewayEndpoint:
|
||||
with patch(f"{_MOD}.MacosOrchestratorService", return_value=service):
|
||||
return ensure_gateway()
|
||||
|
||||
def _service(self) -> MagicMock:
|
||||
service = MagicMock()
|
||||
service.ensure_running.return_value = "http://192.168.128.2:8099"
|
||||
service.network = "bot-bottle-mac-gateway"
|
||||
service.gateway.return_value.ip_on_shared_network.return_value = "192.168.128.3"
|
||||
service.gateway.return_value.ca_cert_pem.return_value = "PEM"
|
||||
return service
|
||||
|
||||
def test_reports_gateway_endpoint(self) -> None:
|
||||
endpoint = self._run(self._service())
|
||||
self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url)
|
||||
self.assertEqual("192.168.128.3", endpoint.gateway_ip)
|
||||
self.assertEqual("PEM", endpoint.gateway_ca_pem)
|
||||
self.assertEqual("bot-bottle-mac-gateway", endpoint.network)
|
||||
|
||||
def test_gateway_is_pointed_at_the_resolved_control_plane(self) -> None:
|
||||
"""Apple has no container DNS, so the gateway must be handed the
|
||||
control plane's *resolved URL* rather than a container name."""
|
||||
service = self._service()
|
||||
self._run(service)
|
||||
service.gateway.assert_called_with("http://192.168.128.2:8099")
|
||||
|
||||
|
||||
class TestRegisterAgent(unittest.TestCase):
|
||||
def _run(
|
||||
self, client: Mock, provision: Mock | None = None,
|
||||
*, source_ip: str = "192.168.128.9",
|
||||
):
|
||||
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
||||
patch(f"{_MOD}.provision_git_gate", provision or Mock()):
|
||||
return register_agent(
|
||||
_egress_plan(), _git_plan(),
|
||||
source_ip=source_ip, endpoint=_endpoint(), image_ref="img:1",
|
||||
)
|
||||
|
||||
def test_registers_by_the_address_read_from_the_live_container(self) -> None:
|
||||
"""The attribution key is the DHCP-assigned address the caller read
|
||||
back — there is no --ip to pin it up front."""
|
||||
client = _client()
|
||||
ctx = self._run(client, source_ip="192.168.128.9")
|
||||
self.assertEqual("192.168.128.9", ctx.source_ip)
|
||||
self.assertEqual("192.168.128.9", client.register_bottle.call_args.args[0])
|
||||
|
||||
def test_returns_identity_token_and_bottle_id(self) -> None:
|
||||
ctx = self._run(_client())
|
||||
self.assertEqual("b1", ctx.bottle_id)
|
||||
self.assertEqual("tok", ctx.identity_token)
|
||||
|
||||
def test_provisions_git_gate_for_the_registered_bottle(self) -> None:
|
||||
provision = Mock()
|
||||
self._run(_client(), provision)
|
||||
self.assertEqual("b1", provision.call_args.args[1])
|
||||
|
||||
def test_rolls_registration_back_when_provisioning_fails(self) -> None:
|
||||
"""A provisioning failure must not leave an orphan registration
|
||||
holding the source IP."""
|
||||
client = _client()
|
||||
provision = Mock(side_effect=RuntimeError("boom"))
|
||||
with self.assertRaises(RuntimeError):
|
||||
self._run(client, provision)
|
||||
client.teardown_bottle.assert_called_once_with("b1")
|
||||
|
||||
|
||||
class TestTeardown(unittest.TestCase):
|
||||
def test_deregisters_and_deprovisions(self) -> None:
|
||||
client = Mock()
|
||||
deprovision = Mock()
|
||||
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
||||
patch(f"{_MOD}.deprovision_git_gate", deprovision):
|
||||
teardown_consolidated("b1", orchestrator_url="http://o:8099")
|
||||
client.teardown_bottle.assert_called_once_with("b1")
|
||||
self.assertEqual("b1", deprovision.call_args.args[1])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -43,10 +43,34 @@ class TestMacosContainerCleanup(unittest.TestCase):
|
||||
|
||||
|
||||
class TestMacosContainerEnumerate(unittest.TestCase):
|
||||
def test_enumerate_active_is_empty_while_disabled(self):
|
||||
# The macOS backend is disabled during the companion-container removal cleanup
|
||||
# (#385); it launches nothing, so there is nothing to enumerate.
|
||||
self.assertEqual([], enum_mod.enumerate_active())
|
||||
"""The backend launches bottles again (PRD 0070), so enumeration is real
|
||||
rather than the disabled-era stub. These must not shell out: `container`
|
||||
does not exist on the Linux CI host."""
|
||||
|
||||
def _enumerate(self, stdout: str, returncode: int = 0):
|
||||
completed = enum_mod.subprocess.CompletedProcess(
|
||||
args=[], returncode=returncode, stdout=stdout, stderr="",
|
||||
)
|
||||
with patch.object(enum_mod.subprocess, "run", return_value=completed), \
|
||||
patch.object(enum_mod, "read_metadata", return_value=None):
|
||||
return enum_mod.enumerate_active()
|
||||
|
||||
def test_lists_agent_containers_by_slug(self):
|
||||
agents = self._enumerate("bot-bottle-dev-abc\nunrelated\n")
|
||||
self.assertEqual(["dev-abc"], [a.slug for a in agents])
|
||||
self.assertEqual(["macos-container"], [a.backend_name for a in agents])
|
||||
|
||||
def test_excludes_the_shared_singletons(self):
|
||||
"""The gateway and control plane share the bot-bottle- prefix but are
|
||||
infrastructure — listing them would invent an agent per host."""
|
||||
agents = self._enumerate(
|
||||
"bot-bottle-mac-gateway\nbot-bottle-mac-orchestrator\n"
|
||||
"bot-bottle-dev-abc\n"
|
||||
)
|
||||
self.assertEqual(["dev-abc"], [a.slug for a in agents])
|
||||
|
||||
def test_empty_when_the_cli_fails(self):
|
||||
self.assertEqual([], self._enumerate("", returncode=1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Unit: macOS agent run wiring — the attribution invariant + token delivery
|
||||
(PRD 0070).
|
||||
|
||||
Replaces the argv coverage from the per-bottle companion-container era
|
||||
(`test_macos_container_launch.py`, removed with that architecture in #385).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.backend.macos_container.bottle import MacosContainerBottle
|
||||
from bot_bottle.backend.macos_container.bottle_plan import MacosContainerBottlePlan
|
||||
from bot_bottle.backend.macos_container.consolidated_launch import GatewayEndpoint
|
||||
from bot_bottle.backend.macos_container.launch import (
|
||||
_agent_run_argv,
|
||||
_identity_proxy_env,
|
||||
_proxy_url,
|
||||
)
|
||||
from bot_bottle.manifest import ManifestIndex
|
||||
|
||||
_BOTTLE = "bot_bottle.backend.macos_container.bottle"
|
||||
|
||||
_MANIFEST = ManifestIndex.from_json_obj({
|
||||
"bottles": {"dev": {}},
|
||||
"agents": {"demo": {"skills": [], "prompt": "", "bottle": "dev"}},
|
||||
}).load_for_agent("demo")
|
||||
|
||||
|
||||
def _endpoint() -> GatewayEndpoint:
|
||||
return GatewayEndpoint(
|
||||
orchestrator_url="http://192.168.128.2:8099",
|
||||
gateway_ip="192.168.128.3",
|
||||
gateway_ca_pem="PEM",
|
||||
network="bot-bottle-mac-gateway",
|
||||
)
|
||||
|
||||
|
||||
def _plan(
|
||||
stage_dir: Path,
|
||||
*,
|
||||
agent_git_gate_url: str = "",
|
||||
agent_supervise_url: str = "",
|
||||
) -> MacosContainerBottlePlan:
|
||||
routes_path = stage_dir / "routes.yaml"
|
||||
routes_path.write_text("routes: []\n", encoding="utf-8")
|
||||
ca_path = stage_dir / "gateway-ca.pem"
|
||||
ca_path.write_text("ca\n", encoding="utf-8")
|
||||
egress_plan = SimpleNamespace(
|
||||
mitmproxy_ca_host_path=ca_path,
|
||||
routes_path=routes_path,
|
||||
routes=("route",),
|
||||
token_env_map={"EGRESS_TOKEN_0": "HOST_TOKEN"},
|
||||
canary="",
|
||||
canary_env="",
|
||||
)
|
||||
return cast(MacosContainerBottlePlan, SimpleNamespace(
|
||||
spec=SimpleNamespace(),
|
||||
manifest=_MANIFEST,
|
||||
stage_dir=stage_dir,
|
||||
slug="dev-abc",
|
||||
container_name="bot-bottle-dev-abc",
|
||||
image="bot-bottle-agent:latest",
|
||||
forwarded_env={"OAUTH_TOKEN": "host-value"},
|
||||
egress_plan=egress_plan,
|
||||
git_gate_plan=SimpleNamespace(upstreams=()),
|
||||
supervise_plan=None,
|
||||
agent_provision=SimpleNamespace(
|
||||
guest_env={"LITERAL": "value"},
|
||||
provisioned_env={},
|
||||
),
|
||||
agent_git_gate_url=agent_git_gate_url,
|
||||
agent_supervise_url=agent_supervise_url,
|
||||
))
|
||||
|
||||
|
||||
class TestAgentRunArgv(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.argv = _agent_run_argv(_plan(Path(self._tmp.name)), _endpoint())
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def test_drops_net_raw(self) -> None:
|
||||
"""The attribution invariant: Apple grants CAP_NET_RAW by default,
|
||||
which would let an agent forge a neighbour's source address with a raw
|
||||
socket and be attributed as that bottle."""
|
||||
self.assertIn("--cap-drop", self.argv)
|
||||
self.assertEqual("CAP_NET_RAW", self.argv[self.argv.index("--cap-drop") + 1])
|
||||
|
||||
def test_attaches_to_the_shared_gateway_network(self) -> None:
|
||||
self.assertEqual(
|
||||
"bot-bottle-mac-gateway", self.argv[self.argv.index("--network") + 1],
|
||||
)
|
||||
|
||||
def test_never_pins_an_ip(self) -> None:
|
||||
"""Apple Container 1.0.0 has no --ip: the address is DHCP-assigned and
|
||||
read back after start."""
|
||||
self.assertNotIn("--ip", self.argv)
|
||||
|
||||
def test_run_time_proxy_carries_no_identity_token(self) -> None:
|
||||
"""The token is minted by registration, which happens after this run —
|
||||
so it cannot be here. `/resolve` denies the token-less pair (#366),
|
||||
which is the safe direction; the real value arrives at exec time."""
|
||||
joined = " ".join(self.argv)
|
||||
self.assertIn(f"HTTP_PROXY={_proxy_url('192.168.128.3')}", joined)
|
||||
self.assertNotIn("bottle:", joined)
|
||||
|
||||
def test_gateway_bypasses_the_proxy(self) -> None:
|
||||
"""git-http + supervise live on the gateway and must be reached
|
||||
directly, not through its own egress proxy."""
|
||||
entry = next(a for a in self.argv if a.startswith("NO_PROXY="))
|
||||
self.assertIn("192.168.128.3", entry)
|
||||
|
||||
def test_forwarded_secrets_stay_off_argv(self) -> None:
|
||||
"""Bare name → inherited from the run process env, so the value never
|
||||
lands on the command line."""
|
||||
self.assertIn("OAUTH_TOKEN", self.argv)
|
||||
self.assertNotIn("host-value", " ".join(self.argv))
|
||||
|
||||
def test_agent_init_is_a_no_op(self) -> None:
|
||||
"""Every agent command arrives via `container exec`; the init process
|
||||
just holds the container open."""
|
||||
self.assertEqual("sleep", self.argv[-2])
|
||||
|
||||
|
||||
class TestIdentityTokenDelivery(unittest.TestCase):
|
||||
def test_exec_env_carries_the_token_as_proxy_credentials(self) -> None:
|
||||
env = _identity_proxy_env(_endpoint(), "s3cret")
|
||||
self.assertEqual(
|
||||
"http://bottle:s3cret@192.168.128.3:9099", env["HTTP_PROXY"],
|
||||
)
|
||||
self.assertEqual(env["HTTP_PROXY"], env["https_proxy"])
|
||||
|
||||
def test_no_token_means_no_override(self) -> None:
|
||||
self.assertEqual({}, _identity_proxy_env(_endpoint(), ""))
|
||||
|
||||
def test_token_value_never_reaches_argv(self) -> None:
|
||||
"""`ps` is world-readable: the token rides the child env behind a bare
|
||||
`--env` name, never the command line."""
|
||||
bottle = MacosContainerBottle(
|
||||
"bot-bottle-demo", lambda: None, None,
|
||||
exec_env=_identity_proxy_env(_endpoint(), "s3cret"),
|
||||
)
|
||||
argv = bottle.agent_argv(["--help"], tty=False)
|
||||
self.assertNotIn("s3cret", " ".join(argv))
|
||||
self.assertIn("HTTP_PROXY", argv)
|
||||
self.assertEqual("--env", argv[argv.index("HTTP_PROXY") - 1])
|
||||
|
||||
def test_bottle_without_exec_env_is_unchanged(self) -> None:
|
||||
bottle = MacosContainerBottle("bot-bottle-demo", lambda: None, None)
|
||||
argv = bottle.agent_argv(["--help"], tty=False)
|
||||
self.assertNotIn("--env", argv)
|
||||
|
||||
def test_provisioning_exec_also_carries_the_token(self) -> None:
|
||||
"""`provision` runs through `exec`; a provider whose provision step
|
||||
fetches anything would otherwise egress token-less and be denied."""
|
||||
bottle = MacosContainerBottle(
|
||||
"bot-bottle-demo", lambda: None, None,
|
||||
exec_env=_identity_proxy_env(_endpoint(), "s3cret"),
|
||||
)
|
||||
with patch(f"{_BOTTLE}.subprocess.run") as run:
|
||||
run.return_value = SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
bottle.exec("echo hi")
|
||||
argv, kwargs = run.call_args.args[0], run.call_args.kwargs
|
||||
self.assertIn("HTTP_PROXY", argv)
|
||||
self.assertNotIn("s3cret", " ".join(argv))
|
||||
self.assertEqual(
|
||||
"http://bottle:s3cret@192.168.128.3:9099", kwargs["env"]["HTTP_PROXY"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,275 @@
|
||||
"""Unit: the Apple gateway + orchestrator lifecycle (PRD 0070)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from bot_bottle.backend.macos_container.gateway import AppleGateway, GatewayError
|
||||
from bot_bottle.backend.macos_container.orchestrator_service import (
|
||||
MacosOrchestratorService,
|
||||
OrchestratorStartError,
|
||||
)
|
||||
|
||||
_GW = "bot_bottle.backend.macos_container.gateway"
|
||||
_ORCH = "bot_bottle.backend.macos_container.orchestrator_service"
|
||||
|
||||
|
||||
def _ok(stdout: str = "") -> Mock:
|
||||
return Mock(returncode=0, stdout=stdout, stderr="")
|
||||
|
||||
|
||||
def _fail(stderr: str = "boom") -> Mock:
|
||||
return Mock(returncode=1, stdout="", stderr=stderr)
|
||||
|
||||
|
||||
class TestAppleGatewayRun(unittest.TestCase):
|
||||
def _argv(self, run: Mock) -> list[str]:
|
||||
return run.call_args.args[0]
|
||||
|
||||
def _start(self, run: Mock) -> None:
|
||||
with patch(f"{_GW}.container_mod") as mod:
|
||||
mod.container_is_running.return_value = False
|
||||
mod.dns_server.return_value = "1.1.1.1"
|
||||
mod.run_container_argv = run
|
||||
AppleGateway(orchestrator_url="http://192.168.128.2:8099").ensure_running()
|
||||
|
||||
def test_nat_network_precedes_the_host_only_network(self) -> None:
|
||||
"""Apple Container makes the FIRST --network the default route, so the
|
||||
NAT network must lead or the gateway has no route out."""
|
||||
run = Mock(return_value=_ok())
|
||||
self._start(run)
|
||||
argv = self._argv(run)
|
||||
networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
||||
self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], networks)
|
||||
|
||||
def test_control_plane_url_is_passed_for_multi_tenancy(self) -> None:
|
||||
run = Mock(return_value=_ok())
|
||||
self._start(run)
|
||||
self.assertIn(
|
||||
"BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.2:8099", self._argv(run),
|
||||
)
|
||||
|
||||
def test_dns_is_explicit(self) -> None:
|
||||
"""The NAT gateway routes but does not resolve."""
|
||||
run = Mock(return_value=_ok())
|
||||
self._start(run)
|
||||
argv = self._argv(run)
|
||||
self.assertEqual("1.1.1.1", argv[argv.index("--dns") + 1])
|
||||
|
||||
def test_start_failure_raises(self) -> None:
|
||||
with self.assertRaises(GatewayError):
|
||||
self._start(Mock(return_value=_fail()))
|
||||
|
||||
def test_running_current_gateway_is_left_alone(self) -> None:
|
||||
"""Idempotent singleton: N launches must not restart the gateway and
|
||||
drop every other bottle's data plane."""
|
||||
run = Mock(return_value=_ok())
|
||||
with patch(f"{_GW}.container_mod") as mod:
|
||||
mod.container_is_running.return_value = True
|
||||
mod.container_image_digest.return_value = "abc"
|
||||
mod.image_digest.return_value = "abc"
|
||||
mod.run_container_argv = run
|
||||
AppleGateway().ensure_running()
|
||||
run.assert_not_called()
|
||||
|
||||
def test_stale_image_forces_a_recreate(self) -> None:
|
||||
"""A rebuilt image only takes effect if the running container is
|
||||
replaced — otherwise it keeps serving the OLD daemons."""
|
||||
run = Mock(return_value=_ok())
|
||||
with patch(f"{_GW}.container_mod") as mod:
|
||||
mod.container_is_running.return_value = True
|
||||
mod.container_image_digest.return_value = "old"
|
||||
mod.image_digest.return_value = "new"
|
||||
mod.dns_server.return_value = "1.1.1.1"
|
||||
mod.run_container_argv = run
|
||||
AppleGateway().ensure_running()
|
||||
run.assert_called_once()
|
||||
|
||||
def test_unreadable_digest_does_not_churn(self) -> None:
|
||||
run = Mock(return_value=_ok())
|
||||
with patch(f"{_GW}.container_mod") as mod:
|
||||
mod.container_is_running.return_value = True
|
||||
mod.container_image_digest.return_value = ""
|
||||
mod.image_digest.return_value = ""
|
||||
mod.run_container_argv = run
|
||||
AppleGateway().ensure_running()
|
||||
run.assert_not_called()
|
||||
|
||||
def _start_with_running_env(self, env: dict[str, str], url: str) -> Mock:
|
||||
run = Mock(return_value=_ok())
|
||||
with patch(f"{_GW}.container_mod") as mod:
|
||||
mod.container_is_running.return_value = True
|
||||
mod.container_image_digest.return_value = "abc"
|
||||
mod.image_digest.return_value = "abc"
|
||||
mod.container_env.return_value = env
|
||||
mod.dns_server.return_value = "1.1.1.1"
|
||||
mod.run_container_argv = run
|
||||
AppleGateway(orchestrator_url=url).ensure_running()
|
||||
return run
|
||||
|
||||
def test_moved_control_plane_forces_a_recreate(self) -> None:
|
||||
"""Docker hands the gateway a container *name*, stable across an
|
||||
orchestrator recreate. Apple has no DNS, so the URL is an IP baked into
|
||||
the gateway's env — if the orchestrator comes back on a new address and
|
||||
the gateway isn't recreated, every /resolve fails and every bottle on
|
||||
the host loses egress."""
|
||||
run = self._start_with_running_env(
|
||||
{"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"},
|
||||
"http://192.168.128.7:8099",
|
||||
)
|
||||
run.assert_called_once()
|
||||
self.assertIn(
|
||||
"BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.7:8099",
|
||||
run.call_args.args[0],
|
||||
)
|
||||
|
||||
def test_unmoved_control_plane_does_not_churn(self) -> None:
|
||||
run = self._start_with_running_env(
|
||||
{"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"},
|
||||
"http://192.168.128.2:8099",
|
||||
)
|
||||
run.assert_not_called()
|
||||
|
||||
def test_unreadable_env_does_not_churn(self) -> None:
|
||||
run = self._start_with_running_env({}, "http://192.168.128.2:8099")
|
||||
run.assert_not_called()
|
||||
|
||||
|
||||
class TestMacosOrchestratorService(unittest.TestCase):
|
||||
def test_orchestrator_starts_before_the_gateway(self) -> None:
|
||||
"""Apple has no container DNS, so the gateway can only be handed the
|
||||
control plane's IP — which does not exist until it is running. This
|
||||
ordering is the whole reason the macOS service diverges from docker's."""
|
||||
order: list[str] = []
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
gateway = Mock()
|
||||
gateway.ensure_running.side_effect = lambda: order.append("gateway")
|
||||
|
||||
def _record_orchestrator(_hash: str) -> None:
|
||||
order.append("orchestrator")
|
||||
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
||||
patch.object(svc, "_run_orchestrator_container",
|
||||
side_effect=_record_orchestrator), \
|
||||
patch.object(svc, "gateway", return_value=gateway), \
|
||||
patch.object(svc, "is_healthy", return_value=True):
|
||||
mod.container_is_running.return_value = False
|
||||
mod.image_exists.return_value = True
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
url = svc.ensure_running()
|
||||
self.assertEqual(["orchestrator", "gateway"], order)
|
||||
self.assertEqual("http://192.168.128.2:8099", url)
|
||||
|
||||
def test_gateway_is_handed_the_resolved_url(self) -> None:
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
||||
patch.object(svc, "_run_orchestrator_container"), \
|
||||
patch.object(svc, "gateway") as gw, \
|
||||
patch.object(svc, "is_healthy", return_value=True):
|
||||
mod.container_is_running.return_value = False
|
||||
mod.image_exists.return_value = True
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
svc.ensure_running()
|
||||
gw.assert_called_with("http://192.168.128.2:8099")
|
||||
|
||||
def test_current_source_leaves_a_healthy_orchestrator_alone(self) -> None:
|
||||
"""Recreating on every launch would drop every other live bottle's
|
||||
in-memory egress tokens (#381)."""
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
run = Mock()
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
||||
patch.object(svc, "_run_orchestrator_container", run), \
|
||||
patch.object(svc, "gateway"), \
|
||||
patch.object(svc, "is_healthy", return_value=True):
|
||||
mod.container_is_running.return_value = True
|
||||
mod.inspect_container.return_value = {
|
||||
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
|
||||
}
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
svc.ensure_running()
|
||||
run.assert_not_called()
|
||||
|
||||
def test_changed_source_recreates_the_orchestrator(self) -> None:
|
||||
"""The control-plane process loaded its bind-mounted source at startup
|
||||
and won't reload it."""
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
run = Mock()
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.source_hash", return_value="h2"), \
|
||||
patch.object(svc, "_run_orchestrator_container", run), \
|
||||
patch.object(svc, "gateway"), \
|
||||
patch.object(svc, "is_healthy", return_value=True):
|
||||
mod.container_is_running.return_value = True
|
||||
mod.inspect_container.return_value = {
|
||||
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
|
||||
}
|
||||
mod.image_exists.return_value = True
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
svc.ensure_running()
|
||||
run.assert_called_once()
|
||||
|
||||
def test_never_healthy_raises(self) -> None:
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
||||
patch.object(svc, "_run_orchestrator_container"), \
|
||||
patch.object(svc, "is_healthy", return_value=False):
|
||||
mod.container_is_running.return_value = False
|
||||
mod.image_exists.return_value = True
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
with self.assertRaises(OrchestratorStartError):
|
||||
svc.ensure_running(startup_timeout=0.01)
|
||||
|
||||
def test_control_plane_needs_no_route_out(self) -> None:
|
||||
"""The orchestrator sits only on the host-only network: the host
|
||||
reaches it there directly, so there is no --publish and no NAT leg."""
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
run = Mock(return_value=_ok())
|
||||
# `ensure_networks` lives in the gateway module and shells out to the
|
||||
# `container` CLI, which does not exist on the Linux CI host — patch it
|
||||
# here, not gateway.container_mod, since it is called through this
|
||||
# module's imported name.
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.ensure_networks"):
|
||||
mod.run_container_argv = run
|
||||
svc._run_orchestrator_container("h1")
|
||||
argv = run.call_args.args[0]
|
||||
networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
||||
self.assertEqual(["bot-bottle-mac-gateway"], networks)
|
||||
self.assertNotIn("--publish", argv)
|
||||
|
||||
def test_networks_exist_before_the_orchestrator_runs(self) -> None:
|
||||
"""The orchestrator is the first container on the shared network, so it
|
||||
has to create it — the gateway that used to do so now starts second."""
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
order: list[str] = []
|
||||
|
||||
def _networks(*_args: str) -> None:
|
||||
order.append("networks")
|
||||
|
||||
def _run(*_args: list[str]) -> Mock:
|
||||
order.append("run")
|
||||
return _ok()
|
||||
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.ensure_networks", side_effect=_networks):
|
||||
mod.run_container_argv = Mock(side_effect=_run)
|
||||
svc._run_orchestrator_container("h1")
|
||||
self.assertEqual(["networks", "run"], order)
|
||||
|
||||
def test_orchestrator_start_failure_raises(self) -> None:
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.ensure_networks"):
|
||||
mod.run_container_argv = Mock(return_value=_fail())
|
||||
with self.assertRaises(OrchestratorStartError):
|
||||
svc._run_orchestrator_container("h1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -14,7 +14,7 @@ from bot_bottle.orchestrator.lifecycle import (
|
||||
ORCHESTRATOR_SOURCE_HASH_LABEL,
|
||||
OrchestratorService,
|
||||
OrchestratorStartError,
|
||||
_source_hash,
|
||||
source_hash,
|
||||
)
|
||||
from tests.unit import use_bottle_root
|
||||
|
||||
@@ -57,7 +57,7 @@ class TestOrchestratorService(unittest.TestCase):
|
||||
# A healthy control plane already running the *current* bind-mounted
|
||||
# source is left alone — recreating it on every launch would drop
|
||||
# every other active bottle's in-memory egress tokens (#381).
|
||||
current = _source_hash(self.svc._repo_root)
|
||||
current = source_hash(self.svc._repo_root)
|
||||
calls: list[list[str]] = []
|
||||
|
||||
def fake(argv: list[str]) -> Mock:
|
||||
@@ -98,7 +98,7 @@ class TestOrchestratorService(unittest.TestCase):
|
||||
self.assertEqual(1, len(runs))
|
||||
self.assertIn(ORCHESTRATOR_NAME, runs[0])
|
||||
# the fresh container is labeled with the current hash, not the stale one
|
||||
current = _source_hash(self.svc._repo_root)
|
||||
current = source_hash(self.svc._repo_root)
|
||||
self.assertIn(f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current}", runs[0])
|
||||
|
||||
def test_ensure_running_starts_orchestrator_container_when_absent(self) -> None:
|
||||
|
||||
Reference in New Issue
Block a user