Compare commits

..

2 Commits

Author SHA1 Message Date
didericis a73aba935f fix(firecracker): read only BOT_BOTTLE_INFRA_ARTIFACT_TOKEN for the artifact
lint / lint (push) Successful in 2m19s
test / unit (pull_request) Successful in 1m21s
test / integration (pull_request) Successful in 24s
test / coverage (pull_request) Successful in 1m24s
Drop the fallback to the general-purpose BOT_BOTTLE_CLAUDE_GITEA_TOKEN so
the artifact pull/publish uses a dedicated, package-scoped token that can
be granted (or revoked) independently of the general Gitea token.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UoEZHDjv84ChoZbozQERhJ
2026-07-16 23:31:17 -04:00
didericis 948504bde2 feat(firecracker): pull the infra rootfs as a prebuilt artifact (PRD 0069 Stage 2)
lint / lint (push) Successful in 2m25s
test / unit (pull_request) Successful in 1m22s
test / integration (pull_request) Successful in 26s
test / coverage (pull_request) Successful in 1m23s
Stage 2 of the docker-free Firecracker backend (#348): stop building the
fixed infra image on the launch host. The infra VM's rootfs is host- and
bottle-agnostic (authorized_keys + guest IP ride the kernel cmdline, not the
rootfs), so it's built once off-host and published as a versioned, ready-to-
boot ext4; the launch host downloads + verifies + boots it — no Docker, no
image tooling, just HTTP + gunzip.

- infra_artifact.py: version = content hash of the rootfs inputs (the shipped
  bot_bottle package + the three Dockerfiles + the init), so a launch host
  pulls the artifact matching its code and a content change can't silently
  boot a stale rootfs. Pull + sha256-verify (fail-closed) + gunzip from a
  Gitea generic package; base/owner/token configurable, default this Gitea.
- infra_vm.ensure_built/boot default to the pull path; BOT_BOTTLE_INFRA_BUILD=
  local keeps the docker build-from-source path for iterating on Dockerfiles.
- publish_infra.py: the off-host half — builds the images with Docker, mke2fs
  the rootfs (with buildah slack), gzips, and PUTs it to the generic package.

Rollout note: default=pull means a launch 404s until an artifact is published;
until the Gitea packages endpoint is enabled + an artifact published, use
BOT_BOTTLE_INFRA_BUILD=local. Freeze/migrate's remaining docker use is a
separate PR.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UoEZHDjv84ChoZbozQERhJ
2026-07-16 23:21:17 -04:00
45 changed files with 200 additions and 2936 deletions
+30 -44
View File
@@ -1,12 +1,9 @@
"""FirecrackerFreezer — snapshot a running microVM to a rootfs tar. """FirecrackerFreezer — snapshot a running microVM to a Docker image.
The VM is live and can't be block-copied safely, so — like the macOS The VM is live and can't be block-copied safely, so — like the macOS
backend — we stream the guest root filesystem out over the control backend — we stream the guest root filesystem out over the control
channel (SSH here). Unlike the other backends this needs no Docker: the channel (SSH here) and rebuild an image from it. The bottle keeps
tar *is* the resumable artifact. `resume` extracts it and rebuilds a running after the snapshot.
fresh per-bottle ext4 with `mke2fs -d` (see `util.build_committed_rootfs_dir`
and `launch._build_agent_base`). The bottle keeps running after the
snapshot.
""" """
from __future__ import annotations from __future__ import annotations
@@ -14,9 +11,9 @@ from __future__ import annotations
import json import json
import os import os
import subprocess import subprocess
import tempfile
from pathlib import Path from pathlib import Path
from ...bottle_state import committed_rootfs_path
from ...log import die, info from ...log import die, info
from .. import ActiveAgent from .. import ActiveAgent
from ..freeze import Freezer from ..freeze import Freezer
@@ -33,13 +30,14 @@ class FirecrackerFreezer(Freezer):
if not private_key.is_file() or not guest_ip: if not private_key.is_file() or not guest_ip:
die(f"cannot freeze {agent.slug}: run dir {run_dir} is missing the " die(f"cannot freeze {agent.slug}: run dir {run_dir} is missing the "
f"SSH key or VM config (is the bottle still running?)") f"SSH key or VM config (is the bottle still running?)")
tar_path = committed_rootfs_path(agent.slug) image_tag = f"bot-bottle-committed-{agent.slug}:latest"
_commit_rootfs_via_ssh(private_key, guest_ip, tar_path) _commit_via_ssh(private_key, guest_ip, image_tag)
info(f"committed {agent.slug} -> {tar_path}") info(f"committed {agent.slug} -> {image_tag!r}")
return str(tar_path) return image_tag
def _export_hint(self, slug: str, image_ref: str) -> None: def _export_hint(self, slug: str, image_ref: str) -> None:
info(f"to export for migration: cp {image_ref} {slug}.tar") info(f"to export for migration: docker image save {image_ref} "
f"-o {slug}.tar")
def _guest_ip_from_config(config_path: Path) -> str: def _guest_ip_from_config(config_path: Path) -> str:
@@ -55,36 +53,24 @@ def _guest_ip_from_config(config_path: Path) -> str:
return "" return ""
def _commit_rootfs_via_ssh(private_key: Path, guest_ip: str, tar_path: Path) -> None: def _commit_via_ssh(private_key: Path, guest_ip: str, image_tag: str) -> None:
"""Stream the guest rootfs out over SSH into `tar_path`. Excludes the with tempfile.TemporaryDirectory(prefix="bot-bottle-fc-commit.") as tmp:
virtual/live mounts (proc/sys/dev/run) — resume recreates those empty rootfs_tar = os.path.join(tmp, "rootfs.tar")
mount points. Written to a `.partial` sibling and renamed on success so ssh = util.ssh_base_argv(private_key, guest_ip)
a failed freeze never leaves a truncated artifact in its place.""" with open(rootfs_tar, "wb") as tar_out:
tar_path.parent.mkdir(parents=True, exist_ok=True) result = subprocess.run(
partial = tar_path.with_name(tar_path.name + ".partial") [*ssh, "--", "tar", "--create", "--one-file-system",
ssh = util.ssh_base_argv(private_key, guest_ip) "--exclude=./proc", "--exclude=./sys", "--exclude=./dev",
# The snapshot can contain the bottle's private workspace, so keep it "--exclude=./run", "--file=-", "--directory=/", "."],
# owner-only (0600) for the whole stream. The `os.open` mode only applies stdout=tar_out, stderr=subprocess.PIPE, check=False,
# on *creation*, so unlink any leftover partial (a prior interrupted run )
# could have left it world-readable, or something could swap in a symlink if result.returncode != 0:
# at this predictable name) and exclusively recreate it — O_EXCL|O_NOFOLLOW die(f"ssh tar for {guest_ip} failed: "
# — then fchmod immediately so umask can't loosen it. Re-assert after the f"{(result.stderr or b'').decode().strip() or '<no stderr>'}")
# rename too (os.replace carries the source mode, but be explicit). with open(os.path.join(tmp, "Dockerfile"), "w", encoding="utf-8") as f:
partial.unlink(missing_ok=True) f.write("FROM scratch\nADD rootfs.tar /\nUSER node\nWORKDIR /home/node\n")
fd = os.open( build = subprocess.run(
partial, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600 ["docker", "build", "-t", image_tag, tmp], check=False,
)
os.fchmod(fd, 0o600)
with os.fdopen(fd, "wb") as tar_out:
result = subprocess.run(
[*ssh, "--", "tar", "--create", "--one-file-system",
"--exclude=./proc", "--exclude=./sys", "--exclude=./dev",
"--exclude=./run", "--file=-", "--directory=/", "."],
stdout=tar_out, stderr=subprocess.PIPE, check=False,
) )
if result.returncode != 0: if build.returncode != 0:
partial.unlink(missing_ok=True) die(f"docker build for {image_tag!r} failed")
die(f"ssh tar for {guest_ip} failed: "
f"{(result.stderr or b'').decode().strip() or '<no stderr>'}")
os.replace(partial, tar_path)
os.chmod(tar_path, 0o600)
@@ -45,7 +45,7 @@ _DOCKERFILES = ("Dockerfile.orchestrator", "Dockerfile.gateway", "Dockerfile.inf
_DEFAULT_BASE = "https://gitea.dideric.is" _DEFAULT_BASE = "https://gitea.dideric.is"
_DEFAULT_OWNER = "didericis" _DEFAULT_OWNER = "didericis"
_PACKAGE = "bot-bottle-firecracker-infra" _PACKAGE = "bot-bottle-infra"
# Streaming copy chunk for the (hundreds-of-MB) download. # Streaming copy chunk for the (hundreds-of-MB) download.
_CHUNK = 1 << 20 _CHUNK = 1 << 20
@@ -57,34 +57,24 @@ def local_build_requested() -> bool:
return os.environ.get("BOT_BOTTLE_INFRA_BUILD", "").strip().lower() == "local" return os.environ.get("BOT_BOTTLE_INFRA_BUILD", "").strip().lower() == "local"
def infra_artifact_version(init_script: str, *, repo_root: Path = _REPO_ROOT) -> str: def infra_artifact_version(init_script: str) -> str:
"""Content hash (16 hex) of everything baked into the infra rootfs: the """Content hash (16 hex) of everything baked into the infra rootfs: the
whole shipped `bot_bottle` package, the three fixed Dockerfiles, and the whole shipped `bot_bottle` package (the infra image `COPY`s it wholesale),
guest init. Deterministic across the publish host and the launch host when the three fixed Dockerfiles, and the guest init. Deterministic across the
both run the same checkout, so the tag the launch host pulls is exactly the publish host and the launch host when both run the same checkout, so the
tag publish produced. tag the launch host pulls is exactly the tag publish produced."""
The package is `COPY bot_bottle /app/bot_bottle`'d wholesale into the image,
so hash *every* regular file under it — not just `*.py`. Non-Python inputs
(e.g. `egress_entrypoint.sh`, `netpool.defaults.env`) are baked in too, and
a change to one must bump the version or a launch host could boot a stale
rootfs whose code differs from its checkout. `__pycache__`/`.pyc` are the
only exclusions — build artifacts, never copied."""
h = hashlib.sha256() h = hashlib.sha256()
h.update(f"format={_ARTIFACT_FORMAT}\n".encode()) h.update(f"format={_ARTIFACT_FORMAT}\n".encode())
pkg = repo_root / "bot_bottle" pkg = _REPO_ROOT / "bot_bottle"
for path in sorted(pkg.rglob("*")): for path in sorted(pkg.rglob("*.py")):
if not path.is_file(): if "__pycache__" in path.parts:
continue continue
if "__pycache__" in path.parts or path.suffix == ".pyc": h.update(str(path.relative_to(_REPO_ROOT)).encode())
continue
h.update(str(path.relative_to(repo_root)).encode())
h.update(b"\0")
h.update(path.read_bytes()) h.update(path.read_bytes())
for name in _DOCKERFILES: for name in _DOCKERFILES:
p = _REPO_ROOT / name
h.update(name.encode()) h.update(name.encode())
h.update(b"\0") h.update(p.read_bytes())
h.update((repo_root / name).read_bytes())
h.update(b"init\0") h.update(b"init\0")
h.update(init_script.encode()) h.update(init_script.encode())
return h.hexdigest()[:16] return h.hexdigest()[:16]
+11 -9
View File
@@ -1,8 +1,7 @@
"""Launch flow for the Firecracker backend (PRD 0070, consolidated). """Launch flow for the Firecracker backend (PRD 0070, consolidated).
Per bottle: Per bottle:
1. build the agent rootfs in a builder VM (buildah, no host docker), or 1. build the agent image (docker), export it to a cached ext4 rootfs;
resume a frozen bottle from its committed rootfs tar; cache the ext4;
2. ensure the per-host orchestrator + shared gateway are up; 2. ensure the per-host orchestrator + shared gateway are up;
3. claim a free TAP pool slot (rootless flock); 3. claim a free TAP pool slot (rootless flock);
4. register the bottle on the orchestrator by the VM's guest IP (the 4. register the bottle on the orchestrator by the VM's guest IP (the
@@ -32,7 +31,6 @@ from typing import Callable, Generator
from ...agent_provider import runtime_for from ...agent_provider import runtime_for
from ...bottle_state import ( from ...bottle_state import (
committed_rootfs_path,
egress_state_dir, egress_state_dir,
git_gate_state_dir, git_gate_state_dir,
read_committed_image, read_committed_image,
@@ -47,6 +45,7 @@ from ...git_gate import (
) )
from ...log import info, warn from ...log import info, warn
from ...supervise import SUPERVISE_PORT from ...supervise import SUPERVISE_PORT
from ..docker import util as docker_mod
from ..docker.egress import EGRESS_PORT from ..docker.egress import EGRESS_PORT
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from . import firecracker_vm, image_builder, isolation_probe, netpool, util from . import firecracker_vm, image_builder, isolation_probe, netpool, util
@@ -211,13 +210,16 @@ def _build_agent_base(
) -> tuple[FirecrackerBottlePlan, Path]: ) -> tuple[FirecrackerBottlePlan, Path]:
"""Produce the agent's base rootfs dir. Primary path: build the Dockerfile """Produce the agent's base rootfs dir. Primary path: build the Dockerfile
inside a Firecracker builder VM (buildah, no host docker), smoke-testing inside a Firecracker builder VM (buildah, no host docker), smoke-testing
the image before export. A committed snapshot (freeze/migrate) is resumed the image before export. A committed snapshot (freeze/migrate) is still
directly from the rootfs tar the freezer wrote — no host docker either.""" exported via the host docker path until that is ported too."""
committed = read_committed_image(plan.slug) committed = read_committed_image(plan.slug)
committed_tar = committed_rootfs_path(plan.slug) if committed and docker_mod.image_exists(committed):
if committed and committed_tar.is_file(): info(f"using committed image {committed!r}")
info(f"resuming from committed rootfs {committed_tar}") plan = dataclasses.replace(
return plan, util.build_committed_rootfs_dir(committed_tar) plan,
agent_provision=dataclasses.replace(plan.agent_provision, image=committed),
)
return plan, util.build_base_rootfs_dir(committed)
base = image_builder.build_agent_rootfs_dir( base = image_builder.build_agent_rootfs_dir(
Path(plan.dockerfile_path), Path(plan.dockerfile_path),
image_tag=plan.image, image_tag=plan.image,
@@ -5,7 +5,7 @@ never on the launch host. It runs the same pipeline the launch host used to run
locally — `docker build` the three fixed images, export to a rootfs dir, inject locally — `docker build` the three fixed images, export to a rootfs dir, inject
the guest boot, `mke2fs` to an ext4 with the buildah build slack — then gzips the guest boot, `mke2fs` to an ext4 with the buildah build slack — then gzips
the ext4 and PUTs it (plus a `.sha256`) to the ext4 and PUTs it (plus a `.sha256`) to
`…/api/packages/<owner>/generic/bot-bottle-firecracker-infra/<version>/`. `…/api/packages/<owner>/generic/bot-bottle-infra/<version>/`.
The `<version>` is `infra_artifact.infra_artifact_version(...)`, the content The `<version>` is `infra_artifact.infra_artifact_version(...)`, the content
hash of the rootfs inputs, so a launch host at the same code checkout resolves hash of the rootfs inputs, so a launch host at the same code checkout resolves
@@ -33,18 +33,6 @@ from . import infra_artifact, infra_vm, util
_CHUNK = 1 << 20 _CHUNK = 1 << 20
# A human-readable description shipped alongside the artifact — generic packages
# have no description field, so this file *is* the description on the package
# page. Uploaded on every publish so it never goes stale.
_ABOUT_NAME = "about.txt"
_ABOUT_TEXT = (
"bot-bottle infra rootfs for the Firecracker backend (PRD 0069 Stage 2, "
"#348): the per-host infra VM (orchestrator control plane + gateway + "
"buildah). Prebuilt off-host, gzip ext4; the launch host downloads + "
"sha256-verifies + boots it, no host Docker. The version tag is a content "
"hash of the rootfs inputs. Files: rootfs.ext4.gz + rootfs.ext4.gz.sha256.\n"
)
def _gzip(src: Path, dest: Path) -> None: def _gzip(src: Path, dest: Path) -> None:
with open(src, "rb") as fh, gzip.open(dest, "wb") as out: with open(src, "rb") as fh, gzip.open(dest, "wb") as out:
@@ -59,22 +47,8 @@ def _sha256(path: Path) -> str:
return h.hexdigest() return h.hexdigest()
def _put(url: str, body: "bytes | Path", token: str) -> None: def _put(url: str, body: bytes, token: str) -> None:
"""PUT `body` (raw bytes, or a Path streamed from disk) to `url`. The rootfs req = urllib.request.Request(url, data=body, method="PUT")
is hundreds of MB, so it is passed as a Path and streamed — `urlopen` reads
the open file in blocks rather than materializing it in memory (with an
explicit Content-Length, which Gitea requires and which also stops urllib
from `len()`-ing a non-bytes body)."""
handle = None
if isinstance(body, Path):
length = body.stat().st_size
handle = open(body, "rb")
data: object = handle
else:
length = len(body)
data = body
req = urllib.request.Request(url, data=data, method="PUT") # type: ignore[arg-type]
req.add_header("Content-Length", str(length))
if token: if token:
req.add_header("Authorization", f"token {token}") req.add_header("Authorization", f"token {token}")
req.add_header("Content-Type", "application/octet-stream") req.add_header("Content-Type", "application/octet-stream")
@@ -90,9 +64,6 @@ def _put(url: str, body: "bytes | Path", token: str) -> None:
raise SystemExit(f"upload failed (HTTP {e.code}): {url}\n{e.read().decode(errors='replace')}") raise SystemExit(f"upload failed (HTTP {e.code}): {url}\n{e.read().decode(errors='replace')}")
except urllib.error.URLError as e: except urllib.error.URLError as e:
raise SystemExit(f"registry unreachable: {url} ({e.reason})") raise SystemExit(f"registry unreachable: {url} ({e.reason})")
finally:
if handle is not None:
handle.close()
def _delete(url: str, token: str) -> None: def _delete(url: str, token: str) -> None:
@@ -150,17 +121,14 @@ def main(argv: list[str] | None = None) -> int:
version, gz, sha = build_artifact(Path(tmp)) version, gz, sha = build_artifact(Path(tmp))
gz_url = infra_artifact.artifact_url(version, gz.name) gz_url = infra_artifact.artifact_url(version, gz.name)
sha_url = infra_artifact.artifact_url(version, sha.name) sha_url = infra_artifact.artifact_url(version, sha.name)
about_url = infra_artifact.artifact_url(version, _ABOUT_NAME)
if args.dry_run: if args.dry_run:
print(f"dry-run: would upload -> {gz_url}") print(f"dry-run: would upload -> {gz_url}")
return 0 return 0
if args.force: if args.force:
_delete(gz_url, token) _delete(gz_url, token)
_delete(sha_url, token) _delete(sha_url, token)
_delete(about_url, token) _put(gz_url, gz.read_bytes(), token)
_put(gz_url, gz, token) # streamed from disk (hundreds of MB) _put(sha_url, sha.read_bytes(), token)
_put(sha_url, sha.read_bytes(), token) # tiny, in-memory is fine
_put(about_url, _ABOUT_TEXT.encode(), token) # package description
print(f"published infra rootfs {version}") print(f"published infra rootfs {version}")
return 0 return 0
+6 -72
View File
@@ -14,7 +14,6 @@ and `./cli.py backend setup --backend=firecracker`.
from __future__ import annotations from __future__ import annotations
import hashlib
import os import os
import platform import platform
import shutil import shutil
@@ -213,80 +212,15 @@ def build_base_rootfs_dir(
return base return base
def build_committed_rootfs_dir(tar_path: Path) -> Path:
"""Prepare a base rootfs dir from a frozen-bottle snapshot tar (the
freeze/resume path — no Docker). Extracts the snapshot, recreates the
virtual mount points the freezer excluded, and injects the guest init +
static dropbear, mirroring `build_base_rootfs_dir` but sourced from a tar
we control rather than a Docker image.
Cached under the rootfs cache, keyed by the tar's size+mtime so a
re-freeze re-extracts but repeated resumes of the same snapshot don't.
Returns the prepared directory (read as the `mke2fs -d` source)."""
st = tar_path.stat()
fingerprint = hashlib.sha256(
f"{tar_path}:{st.st_size}:{st.st_mtime_ns}".encode()
).hexdigest()[:16]
base = cache_dir() / "rootfs" / f"committed-{fingerprint}"
ready = base / ".bb-ready"
if ready.is_file():
return base
if base.exists():
shutil.rmtree(base, ignore_errors=True)
base.mkdir(parents=True)
info(f"extracting committed rootfs {tar_path} -> {base}")
result = subprocess.run(
["tar", "-x", "-f", str(tar_path), "-C", str(base)],
capture_output=True, text=True, check=False,
)
if result.returncode != 0:
die(f"extracting committed rootfs {tar_path} failed: "
f"{result.stderr.strip() or '<no stderr>'}")
# The freezer excludes the live/virtual filesystems from the snapshot;
# recreate them as empty mount points so the guest init can mount
# proc/sys/dev and dropbear has a writable /run.
for mount_point in ("proc", "sys", "dev", "run"):
(base / mount_point).mkdir(mode=0o755, exist_ok=True)
inject_guest_boot(base)
ready.write_text("ok\n")
return base
def inject_guest_boot(rootfs: Path, init_script: str | None = None) -> None: def inject_guest_boot(rootfs: Path, init_script: str | None = None) -> None:
"""Drop the static dropbear and the PID-1 init into the rootfs. """Drop the static dropbear and the PID-1 init into the rootfs.
`init_script` defaults to the SSH-only agent init; the infra VM `init_script` defaults to the SSH-only agent init; the infra VM
passes its own (control plane + gateway) init. passes its own (control plane + gateway) init."""
shutil.copy2(dropbear_path(), rootfs / "bb-dropbear")
A committed snapshot is guest-controlled, so `bb-dropbear`/`bb-init` os.chmod(rootfs / "bb-dropbear", 0o755)
may already exist as symlinks aimed at a host file (e.g. bb-init -> init = rootfs / "bb-init"
~/.bashrc). Replace whatever is there and create the files with init.write_text(init_script or _GUEST_INIT)
O_EXCL|O_NOFOLLOW so the write always lands a fresh regular file in os.chmod(init, 0o755)
the staging tree and never follows a planted symlink out of it."""
_write_staged_file(rootfs / "bb-dropbear", dropbear_path().read_bytes())
_write_staged_file(rootfs / "bb-init", (init_script or _GUEST_INIT).encode())
def _write_staged_file(path: Path, data: bytes) -> None:
"""Write `data` to `path` (mode 0755) as a fresh regular file inside a
staging rootfs, replacing any pre-existing entry without following a
symlink at `path`. Fails closed on anything unexpected there."""
if path.is_symlink() or path.exists():
if path.is_dir() and not path.is_symlink():
shutil.rmtree(path)
else:
path.unlink()
fd = os.open(
path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o755
)
try:
os.write(fd, data)
finally:
os.close(fd)
os.chmod(path, 0o755)
def build_rootfs_ext4(base_dir: Path, out_path: Path, *, slack_mib: int = 1024) -> None: def build_rootfs_ext4(base_dir: Path, out_path: Path, *, slack_mib: int = 1024) -> None:
@@ -89,14 +89,6 @@ class MacosContainerBottleBackend(
with _launch.launch(plan, provision=self.provision) as bottle: with _launch.launch(plan, provision=self.provision) as bottle:
yield bottle yield bottle
def ensure_orchestrator(self) -> str:
"""Bring up the per-host infra container (control plane + gateway) and
return its control-plane URL — the on-demand entry point operator tools
(`supervise`) call when no control plane is running yet. Mirrors
firecracker's infra-VM bring-up."""
from .infra import MacosInfraService
return MacosInfraService().ensure_running().control_plane_url
def prepare_cleanup(self) -> MacosContainerBottleCleanupPlan: def prepare_cleanup(self) -> MacosContainerBottleCleanupPlan:
return _cleanup.prepare_cleanup() return _cleanup.prepare_cleanup()
+4 -32
View File
@@ -52,7 +52,6 @@ class MacosContainerBottle(Bottle):
terminal_title: str = "", terminal_title: str = "",
terminal_color: str = "", terminal_color: str = "",
agent_workdir: str = "/home/node", agent_workdir: str = "/home/node",
exec_env: dict[str, str] | None = None,
): ):
self.name = container self.name = container
self._teardown = teardown self._teardown = teardown
@@ -63,15 +62,6 @@ class MacosContainerBottle(Bottle):
self.terminal_color = terminal_color self.terminal_color = terminal_color
self.agent_provider_template = agent_provider_template self.agent_provider_template = agent_provider_template
self.agent_workdir = agent_workdir self.agent_workdir = agent_workdir
# Env applied to the agent process at `container exec` time, on top of
# what the container was run with. This is how the identity token
# reaches the agent (PRD 0070): registration mints it *after* the
# container exists — its source IP is the registration key and Apple
# Container assigns that by DHCP — so it cannot be in the run-time env
# the way docker's compose spec does it. `container exec --env` wins
# over the run-time value, so the token-bearing proxy URL set here
# supersedes the token-less one baked in at launch.
self._exec_env = dict(exec_env or {})
self._closed = False self._closed = False
def agent_argv(self, argv: list[str], *, tty: bool = True) -> list[str]: def agent_argv(self, argv: list[str], *, tty: bool = True) -> list[str]:
@@ -84,12 +74,6 @@ class MacosContainerBottle(Bottle):
) )
) )
container_exec = ["container", "exec"] container_exec = ["container", "exec"]
# Bare env names, same rule as the terminal hints below: the value
# stays in the child env `exec_agent` builds and never reaches argv —
# the proxy URL here carries the identity token, which `ps` would
# otherwise expose to every process on the host.
for name in sorted(self._exec_env):
container_exec.extend(["--env", name])
if tty: if tty:
container_exec.extend(["--interactive", "--tty"]) container_exec.extend(["--interactive", "--tty"])
# Forward terminal capability hints so TUIs can enable modified-key # Forward terminal capability hints so TUIs can enable modified-key
@@ -110,33 +94,21 @@ class MacosContainerBottle(Bottle):
def exec_agent(self, argv: list[str], *, tty: bool = True) -> int: def exec_agent(self, argv: list[str], *, tty: bool = True) -> int:
agent_argv = self.agent_argv(argv, tty=tty) agent_argv = self.agent_argv(argv, tty=tty)
# The values behind the bare `--env` names in `agent_argv`. `sh -lc`
# below is in this process tree, so the child env reaches `container
# exec` either way.
env = {**os.environ, **self._exec_env} if self._exec_env else None
script = ( script = (
exec_shell_script(agent_argv, self.terminal_title, self.terminal_color) exec_shell_script(agent_argv, self.terminal_title, self.terminal_color)
if tty else None if tty else None
) )
if script is None: if script is None:
return subprocess.run(agent_argv, env=env, check=False).returncode return subprocess.run(agent_argv, check=False).returncode
return subprocess.run(["sh", "-lc", script], env=env, check=False).returncode return subprocess.run(["sh", "-lc", script], check=False).returncode
def exec(self, script: str, *, user: str = "node") -> ExecResult: def exec(self, script: str, *, user: str = "node") -> ExecResult:
# Carry the same exec env the agent gets: provisioning steps run
# through here, and a provider whose provision step fetches anything
# would egress without the identity token and be denied by /resolve.
# Bare `--env NAME` again, so the token stays off argv.
argv = ["container", "exec", "--user", user, "--interactive"]
for name in sorted(self._exec_env):
argv.extend(["--env", name])
argv.extend([self.name, "sh", "-s"])
result = subprocess.run( result = subprocess.run(
argv, ["container", "exec", "--user", user, "--interactive",
self.name, "sh", "-s"],
input=script, input=script,
capture_output=True, capture_output=True,
text=True, text=True,
env={**os.environ, **self._exec_env} if self._exec_env else None,
check=False, check=False,
) )
return ExecResult( return ExecResult(
@@ -13,13 +13,9 @@ from .. import BottlePlan
class MacosContainerBottlePlan(BottlePlan): class MacosContainerBottlePlan(BottlePlan):
slug: str slug: str
forwarded_env: dict[str, str] = field(repr=False) forwarded_env: dict[str, str] = field(repr=False)
agent_proxy_url: str = ""
agent_git_gate_url: str = "" agent_git_gate_url: str = ""
agent_supervise_url: str = "" agent_supervise_url: str = ""
# Read by provision-time consumers (git extraHeader, supervise MCP header)
# via getattr(plan, "identity_token", ""); stamped in launch after the
# bottle is registered. See launch.py's stamp for why it lives here and not
# only in the exec-time proxy env.
identity_token: str = ""
@property @property
def container_name(self) -> str: def container_name(self) -> str:
@@ -1,144 +0,0 @@
"""Consolidated bottle launch sequence for the macOS backend (PRD 0070).
The docker backend allocates a free address, pins the agent to it with
`--ip`, registers it, *then* starts the agent — registration precedes launch
because the pinned address is known up front.
**Apple Container 1.0.0 has no `--ip`.** The `--network` flag takes only
`<name>[,mac=…][,mtu=…]`; the address is assigned by vmnet's DHCP and is
knowable only once the container is running. So the macOS order inverts:
ensure_gateway() -> caller starts the agent -> register_agent(source_ip)
That is why this module exposes two functions where docker has one — the
caller has to start the agent in between. `ensure_gateway` runs first because
the agent's proxy env needs the gateway's address at `container run` time; the
agent's *own* address (the attribution key) only exists afterwards.
The control plane and the gateway are one **infra container** here (see
`infra`), so `gateway_ip` and the control-plane host are the same address.
The consequence for the identity token: it is minted by registration, i.e.
*after* the agent container exists, so it cannot be baked into the run-time
env the way docker's compose spec does. It is delivered at `container exec`
time instead — see `bottle.MacosContainerBottle`.
That delivery is load-bearing, not a nicety: `/resolve` requires a matching
`(source_ip, identity_token)` pair and fail-closes with no source-IP-only
fallback (#366). So egress that does not carry the token is denied — which is
the safe direction, and is why the agent's init process is a bare `sleep` and
every real command arrives through `container exec`.
"""
from __future__ import annotations
from dataclasses import dataclass
from ...egress import EgressPlan
from ...git_gate import GitGatePlan
from ...orchestrator.client import OrchestratorClient
from ...orchestrator.registration import registration_inputs
from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate
from .gateway import GATEWAY_NETWORK
from .gateway_provision import AppleGatewayTransport
from .infra import MacosInfraService, OrchestratorStartError
class ConsolidatedLaunchError(RuntimeError):
"""The consolidated register/provision sequence could not complete."""
@dataclass(frozen=True)
class GatewayEndpoint:
"""What the agent `container run` needs to reach the shared gateway (the
infra container). `gateway_ip` is that container's host-only address, the
same host the control-plane URL points at."""
orchestrator_url: str
gateway_ip: str # the gateway's address — the agent's proxy target
gateway_ca_pem: str # the shared CA the provisioner installs
network: str # the shared host-only network to attach to
@dataclass(frozen=True)
class LaunchContext:
"""What the running agent needs once it has been registered."""
bottle_id: str
identity_token: str
source_ip: str # the agent's DHCP-assigned address (attribution key)
gateway_ip: str
network: str
orchestrator_url: str
def ensure_gateway(
*, service: MacosInfraService | None = None,
) -> GatewayEndpoint:
"""Ensure the per-host infra container (control plane + gateway) is up and
report how to reach it. Idempotent — one singleton, so N bottle launches
share it. Call before starting the agent container: the agent's proxy env
needs `gateway_ip` at run time."""
service = service or MacosInfraService()
infra = service.ensure_running()
return GatewayEndpoint(
orchestrator_url=infra.control_plane_url,
gateway_ip=infra.gateway_ip,
gateway_ca_pem=service.ca_cert_pem(),
network=service.network,
)
def register_agent(
egress_plan: EgressPlan,
git_gate_plan: GitGatePlan,
*,
source_ip: str,
endpoint: GatewayEndpoint,
image_ref: str = "",
tokens: dict[str, str] | None = None,
) -> LaunchContext:
"""Register the (already running) agent by its address and provision its
git-gate state into the gateway. `source_ip` must be read from the live
container — it is the attribution key the gateway resolves policy by.
Raises on failure; the caller tears down."""
client = OrchestratorClient(endpoint.orchestrator_url)
inputs = registration_inputs(egress_plan)
reg = client.register_bottle(
source_ip, image_ref=image_ref, policy=inputs.policy,
metadata=inputs.metadata, tokens=tokens,
)
try:
provision_git_gate(AppleGatewayTransport(), reg.bottle_id, git_gate_plan)
except Exception:
# Roll the registration back so a provisioning failure leaves no orphan.
client.teardown_bottle(reg.bottle_id)
raise
return LaunchContext(
bottle_id=reg.bottle_id,
identity_token=reg.identity_token,
source_ip=source_ip,
gateway_ip=endpoint.gateway_ip,
network=endpoint.network,
orchestrator_url=endpoint.orchestrator_url,
)
def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None:
"""Deregister the bottle and remove its git-gate state from the gateway.
Both steps are idempotent so this is safe from a cleanup trap. Does NOT
stop the gateway — it's a persistent per-host singleton."""
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
deprovision_git_gate(AppleGatewayTransport(), bottle_id)
__all__ = [
"GatewayEndpoint",
"LaunchContext",
"ensure_gateway",
"register_agent",
"teardown_consolidated",
"ConsolidatedLaunchError",
"OrchestratorStartError",
"GATEWAY_NETWORK",
]
@@ -1,11 +1,9 @@
"""Host-side egress route-apply for the macos-container backend. """Host-side egress route-apply for the macos-container backend.
The per-bottle companion container this used to signal (`container kill The per-bottle companion container this used to signal (`container kill
--signal HUP <container>`) was removed in the companion-container removal --signal HUP <container>`) was removed in the companion-container removal (#385),
(#385). In the consolidated model the shared gateway resolves egress policy along with the disabled macOS launch path. Fails closed until the macOS
per-request against the orchestrator rather than reloading a per-bottle routes backend grows the consolidated gateway.
file, so the live per-bottle reload is not supported here and fails closed
until the gateway-side apply lands — same posture as the docker backend.
""" """
from __future__ import annotations from __future__ import annotations
@@ -18,8 +16,8 @@ class MacOSContainerEgressApplicator(EgressApplicator):
del slug del slug
raise EgressApplyError( raise EgressApplyError(
"live egress route-apply was removed with the per-bottle " "live egress route-apply was removed with the per-bottle "
"companion container (#385); route changes will flow through " "companion container (#385); the macos-container backend is "
"the consolidated gateway in a follow-up." "disabled until it uses the consolidated gateway."
) )
@@ -1,42 +1,14 @@
"""Active-agent enumeration for the macOS Apple Container backend.""" """Active-agent enumeration for the macOS Apple Container backend.
The backend is disabled during the companion-container removal (#385) — it can't
launch bottles, so there are none to enumerate. Enumeration returns when
the backend grows the consolidated gateway.
"""
from __future__ import annotations from __future__ import annotations
import subprocess
from ...bottle_state import read_metadata
from .. import ActiveAgent from .. import ActiveAgent
from .infra import INFRA_NAME
_PREFIX = "bot-bottle-"
# The shared per-host infra container carries the same prefix as agent
# containers but is infrastructure, not a bottle — one control plane + gateway
# serves every agent, so listing it as an agent would invent one per host.
_INFRA_NAMES = frozenset({INFRA_NAME})
def enumerate_active() -> list[ActiveAgent]: def enumerate_active() -> list[ActiveAgent]:
result = subprocess.run( return []
["container", "list", "--quiet"],
capture_output=True,
text=True,
check=False,
)
if result.returncode != 0:
return []
out: list[ActiveAgent] = []
for name in sorted(line.strip() for line in result.stdout.splitlines()):
if not name.startswith(_PREFIX) or name in _INFRA_NAMES:
continue
slug = name[len(_PREFIX):]
metadata = read_metadata(slug)
out.append(ActiveAgent(
backend_name="macos-container",
slug=slug,
agent_name=metadata.agent_name if metadata else "?",
started_at=metadata.started_at if metadata else "",
services=(),
label=metadata.label if metadata else "",
color=metadata.color if metadata else "",
))
return out
@@ -1,46 +0,0 @@
"""Shared network/image constants for the macOS consolidated infra container.
The gateway data plane no longer runs as its own Apple container — it shares a
single per-host **infra container** with the control plane (see `infra`),
because two Apple-Container guests writing one `bot-bottle.db` over virtiofs
would race incoherent `fcntl` locks. This module holds the pieces both the
infra service and the launch/provision glue need: the network names, the
gateway image, and the network-creation helper.
"""
from __future__ import annotations
import os
from ...orchestrator.gateway import GatewayError
from . import util as container_mod
# The shared host-only network the infra container and every agent bottle sit
# on. The agent's address here is the attribution key. Distinct from the docker
# names so both backends can coexist on one host.
GATEWAY_NETWORK = "bot-bottle-mac-gateway"
# The NAT network that gives the infra container (and only it) a route out.
GATEWAY_EGRESS_NETWORK = "bot-bottle-mac-egress"
GATEWAY_IMAGE = os.environ.get("BOT_BOTTLE_GATEWAY_IMAGE", "bot-bottle-gateway:latest")
DEFAULT_CA_TIMEOUT_SECONDS = 30.0
def ensure_networks(
network: str = GATEWAY_NETWORK, egress_network: str = GATEWAY_EGRESS_NETWORK,
) -> None:
"""Create the shared host-only network + the NAT egress network. Idempotent
— `create_network` tolerates 'already exists'."""
container_mod.create_network(egress_network)
container_mod.create_network(network, internal=True)
__all__ = [
"GATEWAY_NETWORK",
"GATEWAY_EGRESS_NETWORK",
"GATEWAY_IMAGE",
"GatewayError",
"DEFAULT_CA_TIMEOUT_SECONDS",
"ensure_networks",
]
@@ -1,44 +0,0 @@
"""`GatewayTransport` for the Apple infra container (PRD 0070).
The provisioning *logic* (per-bottle creds dirs, namespaced repo init) is
backend-neutral and lives in `backend.docker.gateway_provision`; this is only
the transport — how files and commands reach the running gateway. Docker uses
`docker exec`/`docker cp` and Firecracker uses SSH; Apple uses the `container`
CLI's equivalents against the infra container that hosts the gateway daemons.
"""
from __future__ import annotations
from ..docker.gateway_provision import GatewayProvisionError
from . import util as container_mod
from .infra import INFRA_NAME
class AppleGatewayTransport:
"""`GatewayTransport` for the gateway daemons in the Apple infra container."""
def __init__(self, gateway: str = INFRA_NAME) -> None:
self.gateway = gateway
def exec(self, argv: list[str]) -> None:
result = container_mod.run_container_argv(
["container", "exec", self.gateway, *argv]
)
if result.returncode != 0:
raise GatewayProvisionError(
f"gateway exec {argv!r} failed: "
f"{(result.stderr or '').strip() or '<no stderr>'}"
)
def cp_into(self, src: str, dest: str) -> None:
result = container_mod.run_container_argv(
["container", "cp", src, f"{self.gateway}:{dest}"]
)
if result.returncode != 0:
raise GatewayProvisionError(
f"gateway cp {src} -> {dest} failed: "
f"{(result.stderr or '').strip() or '<no stderr>'}"
)
__all__ = ["AppleGatewayTransport", "GatewayProvisionError"]
-305
View File
@@ -1,305 +0,0 @@
"""The per-host infra container for the macOS backend (PRD 0070).
A single persistent Apple container that runs BOTH the orchestrator control
plane and the gateway data plane — the macOS analogue of the Firecracker infra
VM (`backend/firecracker/infra_vm.py`), not the docker backend's two separate
containers.
Why one container, not two: Apple Containers are lightweight VMs, each with its
own kernel. The docker backend runs the orchestrator and gateway as two
containers safely because they share the host kernel, so their concurrent
writes to the one `bot-bottle.db` (the orchestrator's registry + the gateway
supervise daemon's queue) are serialized by coherent `fcntl` locks. Across two
*guest* kernels sharing a virtiofs-mounted DB those locks are not coherent, and
concurrent writers can corrupt the file. Firecracker solved this by putting
both services in one guest with the DB on a device only that guest mounts; this
does the same with Apple primitives.
Two consequences fall out of the single container, both simplifications:
- **No DNS dance.** The control plane and the gateway daemons reach each other
over `127.0.0.1`, so nothing depends on Apple's (absent) container DNS and
there is no orchestrator-before-gateway ordering to get right.
- **The DB is never host-shared.** It lives on a container-only volume, so no
host process opens the live file. The host CLI reaches registry + supervise
state through the control-plane HTTP surface (`cli/supervise.py` already uses
`OrchestratorClient`), exactly as it does for firecracker.
The control-plane source is bind-mounted (like the docker orchestrator), so a
code change takes effect on the next launch without an image rebuild; the
gateway daemons are baked in the gateway image and rebuild through its own
digest check.
"""
from __future__ import annotations
import os
import time
import urllib.error
import urllib.request
from dataclasses import dataclass
from pathlib import Path
from ... import log
from ...orchestrator.gateway import GATEWAY_CA_CERT
from ...orchestrator.lifecycle import (
DEFAULT_PORT,
DEFAULT_STARTUP_TIMEOUT_SECONDS,
OrchestratorStartError,
source_hash,
)
from ...paths import (
CONTROL_PLANE_TOKEN_ENV,
HOST_DB_FILENAME,
host_control_plane_token,
)
from . import util as container_mod
from .gateway import (
DEFAULT_CA_TIMEOUT_SECONDS,
GATEWAY_EGRESS_NETWORK,
GATEWAY_IMAGE,
GATEWAY_NETWORK,
GatewayError,
ensure_networks,
)
# The one per-host infra container: control plane + gateway data plane.
INFRA_NAME = "bot-bottle-mac-infra"
INFRA_LABEL = "bot-bottle-mac-infra=1"
# Container-only volume holding bot-bottle.db. No host bind-mount, so the DB is
# written by exactly one kernel (this container's). Survives recreation.
INFRA_DB_VOLUME = "bot-bottle-mac-db"
# BOT_BOTTLE_ROOT inside the container; host_db_path() resolves the DB to
# <root>/db/<filename> and the supervise daemon writes the same file.
_DB_ROOT_IN_CONTAINER = "/var/lib/bot-bottle"
_DB_PATH_IN_CONTAINER = f"{_DB_ROOT_IN_CONTAINER}/db/{HOST_DB_FILENAME}"
_SRC_IN_CONTAINER = "/bot-bottle-src"
_REPO_ROOT = Path(__file__).resolve().parents[3]
_HEALTH_POLL_SECONDS = 0.25
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
_CA_POLL_SECONDS = 0.5
# The gateway subset the consolidated model runs (no per-bottle git:// daemon).
_GATEWAY_DAEMONS = "egress,git-http,supervise"
def _init_script(port: int) -> str:
"""PID-1 init: start the control plane and the gateway daemons, both in
this container, reaching each other over loopback. Backgrounded so `wait`
reaps as PID 1. No `set -e` — a transient daemon failure must not kill the
whole container (gateway_init applies the same 'stay up' policy)."""
return (
"export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n"
f"mkdir -p $(dirname {_DB_PATH_IN_CONTAINER})\n"
# Control plane, from the bind-mounted source (stdlib-only package).
f"( cd {_SRC_IN_CONTAINER} && BOT_BOTTLE_ROOT={_DB_ROOT_IN_CONTAINER} "
f"python3 -m bot_bottle.orchestrator --host 0.0.0.0 --port {port} "
"--broker stub ) &\n"
# Gateway data plane, multi-tenant against the local control plane.
f"( cd /app && BOT_BOTTLE_GATEWAY_DAEMONS={_GATEWAY_DAEMONS} "
f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{port} "
f"SUPERVISE_DB_PATH={_DB_PATH_IN_CONTAINER} python3 /app/gateway_init.py ) &\n"
"while : ; do wait ; done\n"
)
@dataclass(frozen=True)
class InfraEndpoint:
"""How to reach the running infra container. The control plane and the
gateway are the same container, so one address serves both."""
control_plane_url: str # http://<infra ip>:8099 — host CLI + registration
gateway_ip: str # same container; agents' proxy / git-http / MCP target
class MacosInfraService:
"""Manages the single per-host infra container. Callers use
`ensure_running()` (returns the endpoint) and `ca_cert_pem()`."""
def __init__(
self,
*,
port: int = DEFAULT_PORT,
network: str = GATEWAY_NETWORK,
egress_network: str = GATEWAY_EGRESS_NETWORK,
image: str = GATEWAY_IMAGE,
repo_root: Path = _REPO_ROOT,
name: str = INFRA_NAME,
db_volume: str = INFRA_DB_VOLUME,
) -> None:
self.port = port
self.network = network
self.egress_network = egress_network
self.image = image
self._repo_root = repo_root
self._name = name
self._db_volume = db_volume
def _resolve_url(self) -> str:
"""The control-plane URL, or "" while the container has no address."""
ip = container_mod.try_container_ipv4_on_network(self._name, self.network)
return f"http://{ip}:{self.port}" if ip else ""
def is_healthy(
self, url: str, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS,
) -> bool:
if not url:
return False
try:
with urllib.request.urlopen(f"{url}/health", timeout=timeout) as resp:
return resp.status == 200
except (urllib.error.URLError, TimeoutError, OSError):
return False
def _source_current(self, current_hash: str) -> bool:
"""True iff the running infra container was created from the current
bind-mounted control-plane source. The control-plane process loads that
code at startup and won't reload it, so a stale container keeps serving
OLD code."""
if not container_mod.container_is_running(self._name):
return False
env = container_mod.container_env(self._name)
if not env:
return True # can't compare → don't churn a working container
return env.get("BOT_BOTTLE_SOURCE_HASH") == current_hash
def _running_healthy_endpoint(self, current_hash: str) -> InfraEndpoint | None:
"""The endpoint if the running container is BOTH source-current and
answering /health, else None (→ recreate). Health, not just the source
label, is what lets a wedged-but-current container self-heal instead of
being polled to death forever."""
if not self._source_current(current_hash):
return None
url = self._resolve_url()
if url and self.is_healthy(url):
return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url))
return None
def ensure_built(self) -> None:
"""Ensure the gateway data-plane image exists. The control-plane source
is bind-mounted, not baked, so only the gateway image needs building."""
container_mod.build_image(
self.image, str(self._repo_root), dockerfile="Dockerfile.gateway",
)
def ensure_running(
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
) -> InfraEndpoint:
"""Ensure the single infra container is up; return how to reach it.
Idempotent per-host singleton — a healthy container on current source
is left untouched, so N launches share the one control plane + gateway.
Raises `OrchestratorStartError` on startup timeout."""
current_hash = source_hash(self._repo_root)
endpoint = self._running_healthy_endpoint(current_hash)
if endpoint is not None:
return endpoint
self.ensure_built()
log.info("starting infra container", context={"name": self._name})
self._run_container(current_hash)
return self._wait_healthy(startup_timeout)
def _run_container(self, current_hash: str) -> None:
ensure_networks(self.network, self.egress_network)
container_mod.force_remove_container(self._name)
argv = [
"container", "run", "--detach",
"--name", self._name,
"--label", "bot-bottle.backend=macos-container",
"--label", INFRA_LABEL,
# NAT network FIRST so the gateway's egress has a default route;
# the host-only network is where agents (and the host CLI) reach it.
"--network", self.egress_network,
"--network", self.network,
"--dns", container_mod.dns_server(),
# Container-only DB volume: one kernel writes bot-bottle.db, never
# shared with the host or another guest.
"--volume", f"{self._db_volume}:{_DB_ROOT_IN_CONTAINER}",
# Bind-mount the control-plane source (read-only); a code change
# takes effect on relaunch with no image rebuild.
"--mount",
container_mod.bind_mount_spec(
str(self._repo_root), _SRC_IN_CONTAINER, readonly=True),
# Baked onto the container so `_source_current` can detect a real
# control-plane code change and recreate.
"--env", f"BOT_BOTTLE_SOURCE_HASH={current_hash}",
# The control-plane secret, for BOTH the control plane (to require
# it) and the gateway's PolicyResolver (to present it) — they share
# this one container. Bare `--env NAME` inherits the value from the
# run process below, so the secret never lands on argv or in
# `container inspect`'s command line. The agent runs in a SEPARATE
# container that is never given this var, which is the whole point.
"--env", CONTROL_PLANE_TOKEN_ENV,
"--entrypoint", "sh",
self.image,
"-c", _init_script(self.port),
]
run_env = {**os.environ, CONTROL_PLANE_TOKEN_ENV: host_control_plane_token()}
result = container_mod.run_container_argv(argv, env=run_env)
if result.returncode != 0:
raise OrchestratorStartError(
f"infra container failed to start: "
f"{(result.stderr or '').strip() or '<no stderr>'}"
)
def _wait_healthy(self, startup_timeout: float) -> InfraEndpoint:
deadline = time.monotonic() + startup_timeout
while True:
url = self._resolve_url()
if url and self.is_healthy(url):
log.info("infra container healthy", context={"url": url})
return InfraEndpoint(control_plane_url=url, gateway_ip=_ip_of(url))
if time.monotonic() >= deadline:
raise OrchestratorStartError(
f"infra container did not become healthy within "
f"{startup_timeout:g}s"
)
time.sleep(_HEALTH_POLL_SECONDS)
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
interception. Read out of the container (the CA lives on a
container-internal path, not a host mount); polls because mitmproxy
writes it a beat after start."""
deadline = time.monotonic() + timeout
while True:
result = container_mod.run_container_argv(
["container", "exec", self._name, "cat", GATEWAY_CA_CERT])
if result.returncode == 0 and result.stdout.strip():
return result.stdout
if time.monotonic() >= deadline:
raise GatewayError(
f"gateway CA not available in {self._name} after {timeout:g}s: "
f"{(result.stderr or '').strip() or 'empty'}"
)
time.sleep(_CA_POLL_SECONDS)
def stop(self) -> None:
"""Remove the infra container (idempotent). The DB volume persists."""
container_mod.force_remove_container(self._name)
def _ip_of(url: str) -> str:
"""The host from an http://host:port URL."""
return url.split("://", 1)[-1].rsplit(":", 1)[0]
def probe_control_plane_url(port: int = DEFAULT_PORT) -> str:
"""The running infra container's control-plane URL, or "" if it isn't up.
Used by host-side control-plane discovery (`discover_orchestrator_url`);
safe to call on any host — returns "" when the container or the `container`
CLI isn't present."""
ip = container_mod.try_container_ipv4_on_network(INFRA_NAME, GATEWAY_NETWORK)
return f"http://{ip}:{port}" if ip else ""
__all__ = [
"MacosInfraService",
"InfraEndpoint",
"OrchestratorStartError",
"GatewayError",
"INFRA_NAME",
"INFRA_DB_VOLUME",
]
+20 -331
View File
@@ -1,74 +1,24 @@
"""Launch flow for the macOS Apple Container backend (PRD 0070). """Launch flow for the macOS Apple Container backend — disabled (#385).
The agent container attaches to the **shared host-only gateway network** and This backend launched a per-bottle companion container (the egress /
proxies egress through the one per-host gateway, replacing the per-bottle git-gate / supervise data plane) alongside the agent container, with the
companion container removed in #385. agent's proxy env pointed at the companion's host-only IP. That
per-bottle-companion architecture was removed in the companion-container removal;
the macOS backend will be re-enabled once it grows the consolidated
per-host gateway the docker backend already uses.
The order differs from docker's, forced by Apple Container 1.0.0 having no Until then, launching a macOS bottle fails closed. `prepare` / `status`
`--ip` (see `consolidated_launch`): the agent is started *before* it is / cleanup still work.
registered, because its DHCP-assigned address — the attribution key — does not
exist until then.
gateway up -> run agent -> read its IP -> register it -> provision
Two things follow from that inversion:
- The **identity token** is minted by registration and so cannot be in the
agent's run-time env; it rides the proxy URL applied at `container exec`
time (`bottle.MacosContainerBottle`). `/resolve` requires it (#366), so
egress without it is denied — hence the bare `sleep` init: every real agent
command goes through exec and therefore carries the token.
- The agent is run with `--cap-drop CAP_NET_RAW`. Apple Container grants
NET_RAW by default, which would let an agent open a raw socket and forge a
neighbour's source address on the shared segment. NET_ADMIN is already
absent (the agent cannot change its own address or route), so dropping
NET_RAW is what closes the source-address half of PRD 0070's invariant:
"a packet's source address, as seen by the orchestrator, provably identifies
the originating bottle." The identity token is the other half — an attacker
would need to forge the address *and* steal the token — but the invariant is
a stated precondition of consolidation, so it is enforced on its own terms
rather than left to the token.
""" """
from __future__ import annotations from __future__ import annotations
import dataclasses from contextlib import contextmanager
import os
import subprocess
from contextlib import ExitStack, contextmanager
from pathlib import Path
from typing import Callable, Generator from typing import Callable, Generator
from ...bottle_state import ( from ...log import die
egress_state_dir,
git_gate_state_dir,
read_committed_image,
)
from ...egress import (
egress_agent_env_entries,
egress_resolve_token_values,
)
from ...git_gate import (
provision_git_gate_dynamic_keys,
revoke_git_gate_provisioned_keys,
)
from ...git_http_backend import DEFAULT_PORT as _GIT_HTTP_PORT
from ...log import die, info, warn
from ...supervise import SUPERVISE_PORT
from ..docker.egress import EGRESS_PORT
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from . import util as container_mod
from .bottle import MacosContainerBottle from .bottle import MacosContainerBottle
from .bottle_plan import MacosContainerBottlePlan from .bottle_plan import MacosContainerBottlePlan
from .consolidated_launch import (
GatewayEndpoint,
ensure_gateway,
register_agent,
teardown_consolidated,
)
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
_AGENT_SLEEP_SECONDS = "2147483647"
@contextmanager @contextmanager
@@ -77,274 +27,13 @@ def launch(
*, *,
provision: Callable[[MacosContainerBottlePlan, "MacosContainerBottle"], str | None], provision: Callable[[MacosContainerBottlePlan, "MacosContainerBottle"], str | None],
) -> Generator[MacosContainerBottle, None, None]: ) -> Generator[MacosContainerBottle, None, None]:
"""Build, run, register, provision, and yield an Apple Container bottle on """Fail closed: the macOS backend is disabled until it grows the
the shared per-host gateway.""" consolidated per-host gateway (the companion-container path it used
stack = ExitStack() was removed in #385)."""
bottle_for_revoke = plan.manifest.bottle del plan, provision
git_gate_dir_for_revoke = git_gate_state_dir(plan.slug) die(
"the macos-container backend is temporarily disabled during the "
def teardown() -> None: "companion-container removal (#385); it will return once it uses "
teardown_exc: BaseException | None = None "the consolidated gateway. Use --backend=docker for now."
try:
stack.close()
except BaseException as exc: # noqa: W0718 - teardown must continue
teardown_exc = exc
warn(f"macos-container teardown failed: {exc!r}")
revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke)
if teardown_exc is not None:
raise teardown_exc
try:
plan = _build_images(plan)
# Step 1: the per-host singletons. Must precede the agent run — its
# proxy env needs the gateway's address at `container run` time.
endpoint = ensure_gateway()
# Step 2: mint this bottle's deploy keys, then point it at the SHARED
# gateway's CA + git-http/supervise ports.
plan = _provision_git_gate_keys(plan)
plan = _install_gateway_ca(plan, endpoint)
plan = _stamp_agent_urls(plan, endpoint)
# Step 3: run the agent. It has no identity token yet — registration
# needs the address this run assigns.
container_mod.force_remove_container(plan.container_name)
_start_agent(plan, endpoint)
stack.callback(container_mod.force_remove_container, plan.container_name)
# Step 4: read the assigned address and register by it. This is the
# attribution key; `--cap-drop CAP_NET_RAW` at run is what makes it
# unforgeable. Poll: `container run --detach` can return before vmnet's
# DHCP has assigned the address.
source_ip = container_mod.wait_container_ipv4_on_network(
plan.container_name, endpoint.network,
)
if not source_ip:
die(
f"agent {plan.container_name} never got an address on "
f"{endpoint.network}"
)
effective_env = {**os.environ, **plan.agent_provision.provisioned_env}
token_values = egress_resolve_token_values(
plan.egress_plan.token_env_map, effective_env,
)
ctx = register_agent(
plan.egress_plan,
plan.git_gate_plan,
source_ip=source_ip,
endpoint=endpoint,
image_ref=plan.image,
tokens=token_values,
)
stack.callback(
teardown_consolidated, ctx.bottle_id,
orchestrator_url=ctx.orchestrator_url,
)
info(
f"agent {plan.container_name} registered "
f"(gateway {endpoint.gateway_ip}, ip {source_ip})"
)
# Stamp the token onto the plan so provision-time consumers can read it,
# not only the exec-time egress proxy. git-gate's gitconfig extraHeader
# and the supervise MCP --header both reach the gateway on NO_PROXY (they
# bypass the egress proxy that carries the token), so without this the
# gateway's /resolve fail-closes and every git fetch/push and supervise
# call from the bottle is denied. Registration already produced the
# token above, so — unlike the run-time env — the plan CAN carry it.
plan = dataclasses.replace(plan, identity_token=ctx.identity_token)
bottle = MacosContainerBottle(
plan.container_name,
teardown,
None,
agent_command=plan.agent_command,
agent_prompt_mode=plan.agent_prompt_mode,
agent_provider_template=plan.agent_provider_template,
terminal_title=(
f"{plan.spec.label} ({plan.spec.agent_name})"
if plan.spec.label else plan.spec.agent_name
),
terminal_color=plan.spec.color,
agent_workdir=plan.workspace_plan.workdir,
exec_env=_identity_proxy_env(endpoint, ctx.identity_token),
)
bottle.prompt_path = provision(plan, bottle)
yield bottle
finally:
teardown()
def _build_images(plan: MacosContainerBottlePlan) -> MacosContainerBottlePlan:
"""Build the agent image. The gateway's own image is built by
`ensure_gateway` — it belongs to the shared singleton, not to a bottle."""
committed = read_committed_image(plan.slug)
if committed and container_mod.image_exists(committed):
info(f"using committed image {committed!r}")
return dataclasses.replace(
plan,
agent_provision=dataclasses.replace(
plan.agent_provision, image=committed,
),
)
container_mod.build_image(
plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path,
) )
return plan yield # unreachable — `die` raises; keeps this a generator/contextmanager
def _provision_git_gate_keys(
plan: MacosContainerBottlePlan,
) -> MacosContainerBottlePlan:
if not plan.git_gate_plan.upstreams:
return plan
git_gate_plan = provision_git_gate_dynamic_keys(
plan.manifest.bottle,
plan.git_gate_plan,
git_gate_state_dir(plan.slug),
)
return dataclasses.replace(plan, git_gate_plan=git_gate_plan)
def _install_gateway_ca(
plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint,
) -> MacosContainerBottlePlan:
"""Stage the SHARED gateway's CA for the provisioner to install, replacing
the per-bottle CA the companion container used to mint. Every bottle on
this host trusts this one CA."""
ca_dir = egress_state_dir(plan.slug) / "gateway-ca"
ca_dir.mkdir(parents=True, exist_ok=True)
ca_file = ca_dir / "gateway-ca.pem"
ca_file.write_text(endpoint.gateway_ca_pem)
egress_plan = dataclasses.replace(
plan.egress_plan,
mitmproxy_ca_host_path=ca_file,
mitmproxy_ca_cert_only_host_path=ca_file,
)
return dataclasses.replace(plan, egress_plan=egress_plan)
def _stamp_agent_urls(
plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint,
) -> MacosContainerBottlePlan:
"""Point the agent's git-gate insteadOf rewrites + supervise MCP at the
shared gateway's ports. Both bypass the egress proxy (NO_PROXY covers the
gateway address)."""
git_gate_url = (
f"http://{endpoint.gateway_ip}:{_GIT_HTTP_PORT}"
if plan.git_gate_plan.upstreams else ""
)
supervise_url = (
f"http://{endpoint.gateway_ip}:{SUPERVISE_PORT}/"
if plan.supervise_plan is not None else ""
)
return dataclasses.replace(
plan,
agent_git_gate_url=git_gate_url,
agent_supervise_url=supervise_url,
)
def _proxy_url(gateway_ip: str, identity_token: str = "") -> str:
"""The agent's egress proxy URL. The identity token rides as proxy
credentials — the gateway reads Proxy-Authorization, resolves the
(source_ip, token) pair against the control plane, and strips it before
upstream. Without a valid pair `/resolve` denies the request (#366)."""
cred = f"bottle:{identity_token}@" if identity_token else ""
return f"http://{cred}{gateway_ip}:{EGRESS_PORT}"
def _no_proxy(gateway_ip: str) -> str:
# git-http + supervise live on the gateway and must NOT go through the
# egress proxy — the agent reaches them directly by its address.
return f"localhost,127.0.0.1,{gateway_ip}"
def _identity_proxy_env(
endpoint: GatewayEndpoint, identity_token: str,
) -> dict[str, str]:
"""The token-bearing proxy env applied at `container exec`. It supersedes
the token-less run-time value (exec `--env` wins), which is the only way to
get the token in: it does not exist until after the container runs."""
if not identity_token:
return {}
url = _proxy_url(endpoint.gateway_ip, identity_token)
return {
"HTTPS_PROXY": url, "HTTP_PROXY": url,
"https_proxy": url, "http_proxy": url,
}
def _start_agent(plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint) -> None:
argv = _agent_run_argv(plan, endpoint)
env = {**os.environ, **plan.forwarded_env}
info(f"container run agent {plan.container_name}")
result = subprocess.run(
argv, capture_output=True, text=True, env=env, check=False,
)
if result.returncode != 0:
die(
f"container run for agent {plan.container_name} failed: "
f"{(result.stderr or '').strip() or '<no stderr>'}"
)
def _agent_run_argv(
plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint,
) -> list[str]:
argv = [
"container", "run",
"--name", plan.container_name,
"--detach",
"--label", "bot-bottle.backend=macos-container",
"--network", endpoint.network,
# The attribution invariant: without NET_RAW the agent cannot open a
# raw socket, so it cannot source-IP-spoof its neighbours on the shared
# segment. NET_ADMIN is not granted by default, so its address and
# route are already fixed. See the module docstring.
"--cap-drop", "CAP_NET_RAW",
]
for entry in _agent_env_entries(plan, endpoint):
argv += ["--env", entry]
# The init process is a no-op: every agent command arrives via
# `container exec`, which is also how the identity token gets in.
argv += [plan.image, "sleep", _AGENT_SLEEP_SECONDS]
return argv
def _agent_env_entries(
plan: MacosContainerBottlePlan, endpoint: GatewayEndpoint,
) -> tuple[str, ...]:
# Token-less at run time — the token does not exist yet (see
# `_identity_proxy_env`). Anything egressing before the exec-time override
# is denied by `/resolve`, which is the safe direction.
proxy_url = _proxy_url(endpoint.gateway_ip)
no_proxy = _no_proxy(endpoint.gateway_ip)
env = [
f"HTTPS_PROXY={proxy_url}",
f"HTTP_PROXY={proxy_url}",
f"https_proxy={proxy_url}",
f"http_proxy={proxy_url}",
f"NO_PROXY={no_proxy}",
f"no_proxy={no_proxy}",
f"NODE_EXTRA_CA_CERTS={AGENT_CA_PATH}",
f"SSL_CERT_FILE={AGENT_CA_BUNDLE}",
f"REQUESTS_CA_BUNDLE={AGENT_CA_BUNDLE}",
]
if plan.agent_git_gate_url:
env.append(f"GIT_GATE_URL={plan.agent_git_gate_url}")
if plan.agent_supervise_url:
env.append(f"MCP_SUPERVISE_URL={plan.agent_supervise_url}")
for name, value in sorted(plan.agent_provision.guest_env.items()):
env.append(f"{name}={value}")
# Forwarded vars: bare name → inherits from the `container run` process env
# so the secret value never lands on argv.
for name in sorted(plan.forwarded_env.keys()):
env.append(name)
env.extend(egress_agent_env_entries(plan.egress_plan))
return tuple(env)
__all__ = ["launch"]
+11 -134
View File
@@ -437,145 +437,22 @@ def inspect_container(name: str) -> dict[str, object]:
def container_ipv4_on_network(name: str, network: str) -> str: def container_ipv4_on_network(name: str, network: str) -> str:
"""The container's IPv4 address on `network`. Fatal if absent — callers data = inspect_container(name)
that can tolerate "not yet" want `try_container_ipv4_on_network`.""" status = data.get("status")
ip = try_container_ipv4_on_network(name, network)
if not ip:
die(f"container {name} has no IPv4 address on {network}")
return ip
def run_container_argv(
argv: list[str], *, env: dict[str, str] | None = None,
) -> subprocess.CompletedProcess[str]:
"""Run a `container` command, returning the result for the caller to
interpret. Unlike the `die`-on-failure helpers above, this lets callers
that raise their own typed errors (the gateway / orchestrator lifecycle)
keep control of the failure path.
`env` sets the child process environment — used to hand a secret to a bare
`--env NAME` flag (Apple's "just key → inherit from host" form) so the
value is inherited from this process, never written onto argv or into
`container inspect`'s recorded command line."""
return subprocess.run(
argv, capture_output=True, text=True, check=False, env=env)
def bind_mount_spec(source: str, target: str, *, readonly: bool = False) -> str:
"""A `container run --mount` bind spec. One definition so the gateway and
orchestrator emit an identical string — a divergence here would silently
break one backend's mounts while the other kept working."""
spec = f"type=bind,source={source},target={target}"
if readonly:
spec += ",readonly"
return spec
def _normalize_digest(value: str) -> str:
return value.split(":", 1)[1] if ":" in value else value
def _inspect_first(argv: list[str]) -> dict[str, object]:
"""Run an inspect command and return its first JSON object, or {} on any
failure (non-zero exit, malformed JSON, unexpected shape). {} is the shared
'don't know' signal all the non-fatal inspect readers below build on — a
caller comparing against it treats it as 'leave the working container
alone', never as a mismatch."""
result = run_container_argv(argv)
if result.returncode != 0:
return {}
try:
data = json.loads(result.stdout or "[]")
except json.JSONDecodeError:
return {}
if isinstance(data, list):
data = data[0] if data else {}
return data if isinstance(data, dict) else {}
def _descriptor_digest(node: object) -> str:
"""The normalized digest under a `{... "descriptor": {"digest": ...}}`
node, or "". Both the image and container inspect shapes nest the image's
identity this way, so the digest readers stay symmetric — a difference
between them is what would spuriously recreate a container."""
if not isinstance(node, dict):
return ""
descriptor = node.get("descriptor")
if isinstance(descriptor, dict) and descriptor.get("digest"):
return _normalize_digest(str(descriptor["digest"]))
return ""
def image_digest(ref: str) -> str:
"""The digest of image `ref`, or "" if it can't be read. Reads exactly the
field `container_image_digest` reads (`configuration.descriptor.digest`) so
the two are comparable; "" means 'don't know' → callers don't churn."""
data = _inspect_first([_CONTAINER, "image", "inspect", ref])
return _descriptor_digest(data.get("configuration"))
def container_image_digest(name: str) -> str:
"""The digest of the image container `name` was created from, or "" if it
can't be read. Compare with `image_digest(ref)` to tell whether a running
container predates an image rebuild."""
config = _inspect_first([_CONTAINER, "inspect", name]).get("configuration")
image = config.get("image") if isinstance(config, dict) else None
return _descriptor_digest(image)
def container_env(name: str) -> dict[str, str]:
"""The env container `name` was started with, or {} if unreadable. Lets a
caller tell whether a running container's baked-in configuration still
matches what it would pass today."""
config = _inspect_first([_CONTAINER, "inspect", name]).get("configuration")
init = config.get("initProcess") if isinstance(config, dict) else None
entries = init.get("environment") if isinstance(init, dict) else None
if not isinstance(entries, list):
return {}
env: dict[str, str] = {}
for entry in entries:
if isinstance(entry, str) and "=" in entry:
key, value = entry.split("=", 1)
env[key] = value
return env
def try_container_ipv4_on_network(name: str, network: str) -> str:
"""`container_ipv4_on_network` without the fatal exit: "" when the address
isn't readable yet. For pollers — a container is created before it has an
address, so "not yet" is an expected state there, not an error."""
status = _inspect_first([_CONTAINER, "inspect", name]).get("status")
networks = status.get("networks") if isinstance(status, dict) else None networks = status.get("networks") if isinstance(status, dict) else None
if not isinstance(networks, list): if not isinstance(networks, list):
return "" die(f"container inspect {name} did not include status.networks")
for entry in networks: for entry in networks:
if not isinstance(entry, dict) or entry.get("network") != network: if not isinstance(entry, dict):
continue
if entry.get("network") != network:
continue continue
raw = entry.get("ipv4Address") raw = entry.get("ipv4Address")
if isinstance(raw, str) and raw: if not isinstance(raw, str) or not raw:
return raw.split("/", 1)[0] die(f"container {name} has no IPv4 address on {network}")
return "" return raw.split("/", 1)[0]
die(f"container {name} is not attached to network {network}")
raise AssertionError("unreachable")
def wait_container_ipv4_on_network(
name: str, network: str, *, timeout: float = 15.0, poll: float = 0.25,
) -> str:
"""Poll for the container's DHCP-assigned address on `network`, returning
it once available or "" on timeout.
Apple Container has no `--ip`: `container run --detach` can return before
vmnet's DHCP has populated `status.networks[].ipv4Address`, so a bare read
right after start races the assignment. Callers that need the address (the
attribution key, the gateway's proxy target) poll through here instead of
the fatal `container_ipv4_on_network`."""
deadline = time.monotonic() + timeout
while True:
ip = try_container_ipv4_on_network(name, network)
if ip:
return ip
if time.monotonic() >= deadline:
return ""
time.sleep(poll)
def image_id(ref: str) -> str: def image_id(ref: str) -> str:
-21
View File
@@ -31,7 +31,6 @@ from __future__ import annotations
import dataclasses import dataclasses
import json import json
import secrets import secrets
import socket
import string import string
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
@@ -44,7 +43,6 @@ from .paths import bot_bottle_root
_STATE_SUBDIR = "state" _STATE_SUBDIR = "state"
_PER_BOTTLE_DOCKERFILE_NAME = "Dockerfile" _PER_BOTTLE_DOCKERFILE_NAME = "Dockerfile"
_COMMITTED_IMAGE_NAME = "committed-image" _COMMITTED_IMAGE_NAME = "committed-image"
_COMMITTED_ROOTFS_NAME = "committed-rootfs.tar"
_TRANSCRIPT_SUBDIR = "transcript" _TRANSCRIPT_SUBDIR = "transcript"
# Per-daemon scratch subdirs. PRD 0018 chunk 2: bind-mount sources # Per-daemon scratch subdirs. PRD 0018 chunk 2: bind-mount sources
# live here so chunk 3's `docker compose up` can find them at stable # live here so chunk 3's `docker compose up` can find them at stable
@@ -89,14 +87,6 @@ def bottle_identity(agent_name: str) -> str:
return f"{slug}-{suffix}" return f"{slug}-{suffix}"
def globalize_slug(slug: str) -> str:
"""Return a globally-unique slug qualified with the current hostname.
Assumes slug is a value returned from mint_slug. Use wherever a slug
must be unique across hosts (e.g. deploy-key titles)."""
return f"{socket.gethostname()}-{slug}"
@dataclass(frozen=True) @dataclass(frozen=True)
class BottleMetadata: class BottleMetadata:
"""Persistent record of how a bottle was launched, written at """Persistent record of how a bottle was launched, written at
@@ -201,15 +191,6 @@ def committed_image_path(identity: str) -> Path:
return bottle_state_dir(identity) / _COMMITTED_IMAGE_NAME return bottle_state_dir(identity) / _COMMITTED_IMAGE_NAME
def committed_rootfs_path(identity: str) -> Path:
"""Where the Firecracker freezer stores a snapshot of the bottle's
guest rootfs (a plain tar). This is the resumable/migratable artifact
the Firecracker backend boots from — no Docker image involved. The
matching `committed-image` state file records that a snapshot exists
(and its path); `resume` boots from this tar when both are present."""
return bottle_state_dir(identity) / _COMMITTED_ROOTFS_NAME
def write_committed_image(identity: str, image_tag: str) -> Path: def write_committed_image(identity: str, image_tag: str) -> Path:
"""Persist the committed image tag for `identity`. The next """Persist the committed image tag for `identity`. The next
`cli.py resume <identity>` will boot from this image instead of `cli.py resume <identity>` will boot from this image instead of
@@ -359,12 +340,10 @@ __all__ = [
"BottleMetadata", "BottleMetadata",
"agent_state_dir", "agent_state_dir",
"bottle_identity", "bottle_identity",
"globalize_slug",
"bottle_state_dir", "bottle_state_dir",
"cleanup_state", "cleanup_state",
"clear_preserve_marker", "clear_preserve_marker",
"committed_image_path", "committed_image_path",
"committed_rootfs_path",
"egress_state_dir", "egress_state_dir",
"git_gate_state_dir", "git_gate_state_dir",
"is_preserved", "is_preserved",
+3 -10
View File
@@ -14,20 +14,13 @@ from __future__ import annotations
import subprocess import subprocess
def run_docker( def run_docker(argv: list[str]) -> subprocess.CompletedProcess[str]:
argv: list[str], *, env: dict[str, str] | None = None,
) -> subprocess.CompletedProcess[str]:
"""Run a `docker` command, capturing stdout/stderr as text. Never raises """Run a `docker` command, capturing stdout/stderr as text. Never raises
on a non-zero exit — callers inspect `returncode` / `stderr` so they can on a non-zero exit — callers inspect `returncode` / `stderr` so they can
stay fail-closed or tolerate idempotent no-ops (e.g. removing an stay fail-closed or tolerate idempotent no-ops (e.g. removing an
already-absent container). already-absent container)."""
`env` sets the child process environment — used to hand a secret to a bare
`--env NAME` flag (docker inherits its value from this process) so the
value never lands on argv or in `docker inspect`'s recorded command line."""
return subprocess.run( return subprocess.run(
argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False,
check=False, env=env,
) )
+21 -76
View File
@@ -78,16 +78,6 @@ ORCHESTRATOR_URL_ENV = "BOT_BOTTLE_ORCHESTRATOR_URL"
# is still stripped defensively (git-http uses that header on its own port). # is still stripped defensively (git-http uses that header on its own port).
IDENTITY_HEADER = "x-bot-bottle-identity" IDENTITY_HEADER = "x-bot-bottle-identity"
# Per-flow key under which `request()` stashes the resolved (Config, supervise
# slug, env) so the later `response()` and `websocket_message()` hooks scan
# against the *calling bottle's* policy. In the consolidated (multi-tenant)
# gateway the static `self.config` is empty — every request's real policy comes
# from the per-request `/resolve` — so a hook that fell back to `self.config`
# would find no route and silently skip its DLP scan (fail-open). Resolving once
# at the request and reusing it also avoids a `/resolve` round-trip per response
# and per WebSocket frame.
_FLOW_CTX_KEY = "bot_bottle_egress_ctx"
def _token_from_proxy_auth(header: str) -> str: def _token_from_proxy_auth(header: str) -> str:
"""Extract the identity token (the password) from a `Proxy-Authorization: """Extract the identity token (the password) from a `Proxy-Authorization:
@@ -233,39 +223,31 @@ class EgressAddon:
{"Content-Type": "text/plain; charset=utf-8"}, {"Content-Type": "text/plain; charset=utf-8"},
) )
def _log_request( def _log_request(self, flow: http.HTTPFlow) -> None:
self, flow: http.HTTPFlow, env: "typing.Mapping[str, str]",
) -> None:
# `env` is the per-flow resolved overlay (process env + this bottle's
# /resolve tokens), so the log redaction scrubs the calling bottle's
# provisioned secrets — not just the process-level ones in os.environ.
headers = { headers = {
k: redact_tokens(v, env=env) k: redact_tokens(v, env=os.environ)
for k, v in flow.request.headers.items() for k, v in flow.request.headers.items()
if k.lower() != "authorization" if k.lower() != "authorization"
} }
body = redact_tokens(flow.request.get_text(strict=False) or "", env=env) body = redact_tokens(flow.request.get_text(strict=False) or "", env=os.environ)
sys.stderr.write( sys.stderr.write(
json.dumps({ json.dumps({
"event": "egress_request", "event": "egress_request",
"host": redact_tokens(flow.request.pretty_host, env=env), "host": redact_tokens(flow.request.pretty_host, env=os.environ),
"method": flow.request.method, "method": flow.request.method,
"path": redact_tokens(flow.request.path, env=env), "path": redact_tokens(flow.request.path, env=os.environ),
"headers": headers, "headers": headers,
"body": body, "body": body,
}) })
+ "\n" + "\n"
) )
def _log_response( def _log_response(self, flow: http.HTTPFlow) -> None:
self, flow: http.HTTPFlow, env: "typing.Mapping[str, str]",
) -> None:
# Per-flow env overlay (see _log_request): redact this bottle's tokens.
headers = { headers = {
k: redact_tokens(v, env=env) k: redact_tokens(v, env=os.environ)
for k, v in flow.response.headers.items() for k, v in flow.response.headers.items()
} }
body = redact_tokens(flow.response.get_text(strict=False) or "", env=env) body = redact_tokens(flow.response.get_text(strict=False) or "", env=os.environ)
sys.stderr.write( sys.stderr.write(
json.dumps({ json.dumps({
"event": "egress_response", "event": "egress_response",
@@ -298,36 +280,6 @@ class EgressAddon:
env = {**os.environ, **tokens} if tokens else os.environ env = {**os.environ, **tokens} if tokens else os.environ
return config, slug, env return config, slug, env
def _stash_flow_ctx(
self,
flow: http.HTTPFlow,
config: Config,
slug: str,
env: "typing.Mapping[str, str]",
) -> None:
"""Remember the per-flow context `request()` resolved, so the later
`response()` / `websocket_message()` hooks reuse it — scanning against
the same bottle's policy the request was decided on, with one `/resolve`
per flow rather than one per frame."""
meta = getattr(flow, "metadata", None)
if isinstance(meta, dict):
meta[_FLOW_CTX_KEY] = (config, slug, env)
def _flow_ctx(
self, flow: http.HTTPFlow,
) -> "tuple[Config, str, typing.Mapping[str, str]]":
"""The `(Config, supervise slug, env)` `request()` resolved for this
flow, so a later hook scans against the calling bottle's policy — not the
empty static config the consolidated gateway carries. Falls back to the
single-tenant static values for a flow that never passed through
`request()` (or a flow object without metadata)."""
meta = getattr(flow, "metadata", None)
if isinstance(meta, dict):
ctx = meta.get(_FLOW_CTX_KEY)
if ctx is not None:
return ctx
return self.config, self._supervise_slug, os.environ
def _request_token(self, flow: http.HTTPFlow) -> str: def _request_token(self, flow: http.HTTPFlow) -> str:
"""The per-bottle identity token for this request, from the proxy """The per-bottle identity token for this request, from the proxy
credentials — the delivery mechanism (`HTTPS_PROXY=http://id:token@gw`) credentials — the delivery mechanism (`HTTPS_PROXY=http://id:token@gw`)
@@ -367,9 +319,6 @@ class EgressAddon:
return return
config, slug, env = self._resolve_flow(flow) config, slug, env = self._resolve_flow(flow)
# Stash for the response / websocket hooks so their DLP scans use this
# bottle's resolved policy, not the empty static config (see _flow_ctx).
self._stash_flow_ctx(flow, config, slug, env)
# DLP outbound scan BEFORE stripping auth — catches tokens the # DLP outbound scan BEFORE stripping auth — catches tokens the
# agent tried to smuggle in any header, path, query param, or body. # agent tried to smuggle in any header, path, query param, or body.
@@ -428,7 +377,7 @@ class EgressAddon:
flow.request.headers["authorization"] = decision.inject_authorization flow.request.headers["authorization"] = decision.inject_authorization
if config.log >= LOG_FULL: if config.log >= LOG_FULL:
self._log_request(flow, env) self._log_request(flow)
def _block_dlp(self, flow: http.HTTPFlow, result: ScanResult) -> None: def _block_dlp(self, flow: http.HTTPFlow, result: ScanResult) -> None:
ctx = self._req_ctx(flow) ctx = self._req_ctx(flow)
@@ -637,17 +586,14 @@ class EgressAddon:
await asyncio.sleep(TOKEN_ALLOW_POLL_INTERVAL_SECONDS) await asyncio.sleep(TOKEN_ALLOW_POLL_INTERVAL_SECONDS)
def response(self, flow: http.HTTPFlow) -> None: def response(self, flow: http.HTTPFlow) -> None:
"""DLP inbound scan on response headers and body, against the calling """DLP inbound scan on response headers and body."""
bottle's resolved config (multi-tenant) or the static config route = match_route(self.config.routes, flow.request.pretty_host)
(single-tenant) — see `_flow_ctx`."""
config, _slug, env = self._flow_ctx(flow)
route = match_route(config.routes, flow.request.pretty_host)
if route is None: if route is None:
return return
if flow.response is None: if flow.response is None:
return return
if config.log >= LOG_FULL: if self.config.log >= LOG_FULL:
self._log_response(flow, env) self._log_response(flow)
resp_headers = {k.lower(): v for k, v in flow.response.headers.items()} resp_headers = {k.lower(): v for k, v in flow.response.headers.items()}
body = flow.response.get_text(strict=False) or "" body = flow.response.get_text(strict=False) or ""
scan_text = build_inbound_scan_text(resp_headers, body) scan_text = build_inbound_scan_text(resp_headers, body)
@@ -664,7 +610,7 @@ class EgressAddon:
resp_ctx = {**resp_ctx, "context": result.context} resp_ctx = {**resp_ctx, "context": result.context}
if result.severity == "block": if result.severity == "block":
self._block(flow, f"egress DLP: {result.reason}", ctx=resp_ctx) self._block(flow, f"egress DLP: {result.reason}", ctx=resp_ctx)
elif result.severity == "warn" and config.log >= LOG_BLOCKS: elif result.severity == "warn" and self.config.log >= LOG_BLOCKS:
sys.stderr.write( sys.stderr.write(
json.dumps({ json.dumps({
"event": "egress_warn", "event": "egress_warn",
@@ -675,10 +621,7 @@ class EgressAddon:
) )
def websocket_message(self, flow: http.HTTPFlow) -> None: def websocket_message(self, flow: http.HTTPFlow) -> None:
"""DLP scan on WebSocket frames, against the calling bottle's resolved """DLP scan on WebSocket frames.
config (see `_flow_ctx`). `request()` resolves and stashes the per-flow
(config, slug, env) at the upgrade, so both the multi-tenant and
single-tenant gateways scan here.
Outbound frames (from_client) are scanned for credential leakage; Outbound frames (from_client) are scanned for credential leakage;
inbound frames are scanned for prompt injection. On a block the inbound frames are scanned for prompt injection. On a block the
@@ -687,8 +630,10 @@ class EgressAddon:
""" """
if flow.websocket is None: # type: ignore[union-attr] if flow.websocket is None: # type: ignore[union-attr]
return return
config, slug, env = self._flow_ctx(flow) # WebSocket DLP runs against the static config only (single-tenant); in
route = match_route(config.routes, flow.request.pretty_host) # the consolidated gateway self.config has no routes, so this is inert
# until websocket routing is made source-IP-aware (a separate slice).
route = match_route(self.config.routes, flow.request.pretty_host)
if route is None: if route is None:
return return
message = flow.websocket.messages[-1] # type: ignore[union-attr] message = flow.websocket.messages[-1] # type: ignore[union-attr]
@@ -697,8 +642,8 @@ class EgressAddon:
# A WebSocket data frame is not an HTTP request line, so CRLF is # A WebSocket data frame is not an HTTP request line, so CRLF is
# not an injection vector here — scan only for credential leakage. # not an injection vector here — scan only for credential leakage.
result = scan_outbound( result = scan_outbound(
route, content, env, route, content, os.environ,
safe_tokens=self._safe_tokens_for(slug), crlf_text="", safe_tokens=self._safe_tokens_for(self._supervise_slug), crlf_text="",
) )
if result is not None and result.severity == "block": if result is not None and result.severity == "block":
sys.stderr.write(f"egress DLP: {result.reason}\n") sys.stderr.write(f"egress DLP: {result.reason}\n")
+1 -2
View File
@@ -13,7 +13,6 @@ import dataclasses
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from .bottle_state import globalize_slug
from .errors import MissingEnvVarError from .errors import MissingEnvVarError
from .log import info from .log import info
from .manifest import ManifestBottle, ManifestGitEntry from .manifest import ManifestBottle, ManifestGitEntry
@@ -47,7 +46,7 @@ def _provision_dynamic_key(
owner_repo = entry.UpstreamPath owner_repo = entry.UpstreamPath
if owner_repo.endswith(".git"): if owner_repo.endswith(".git"):
owner_repo = owner_repo[:-4] owner_repo = owner_repo[:-4]
title = f"bot-bottle:{globalize_slug(slug)}:{entry.Name}" title = f"bot-bottle:{slug}:{entry.Name}"
info(f"provisioning deploy key for git-gate.repos[{entry.Name!r}]") info(f"provisioning deploy key for git-gate.repos[{entry.Name!r}]")
key_id, private_key_bytes = provisioner.create(owner_repo, title) key_id, private_key_bytes = provisioner.create(owner_repo, title)
+2 -37
View File
@@ -17,22 +17,9 @@ import urllib.error
import urllib.request import urllib.request
from dataclasses import dataclass from dataclasses import dataclass
from ..paths import host_control_plane_token
from .control_plane import CONTROL_AUTH_HEADER
DEFAULT_TIMEOUT_SECONDS = 5.0 DEFAULT_TIMEOUT_SECONDS = 5.0
def _host_auth_token() -> str:
"""The per-host control-plane secret, or "" if it can't be read. "" means
'send no auth header' — correct against an open (unconfigured) control
plane, and harmlessly rejected by a secured one."""
try:
return host_control_plane_token()
except OSError:
return ""
class OrchestratorClientError(RuntimeError): class OrchestratorClientError(RuntimeError):
"""A control-plane call failed (unreachable, or an unexpected status).""" """A control-plane call failed (unreachable, or an unexpected status)."""
@@ -47,24 +34,11 @@ class RegisteredBottle:
class OrchestratorClient: class OrchestratorClient:
"""Trusted host-side client for the orchestrator control plane. """Trusted host-side client for the orchestrator control plane."""
Presents the per-host control-plane secret on every call (the header the def __init__(self, base_url: str, *, timeout: float = DEFAULT_TIMEOUT_SECONDS) -> None:
control plane requires on all routes but `/health`). The secret is read
from the host file — this client only ever runs host-side (CLI, launcher,
discovery), so it can read what an agent can't. `auth_token` is overridable
for tests; the default reads the host file, minting it on first use."""
def __init__(
self,
base_url: str,
*,
timeout: float = DEFAULT_TIMEOUT_SECONDS,
auth_token: str | None = None,
) -> None:
self._base = base_url.rstrip("/") self._base = base_url.rstrip("/")
self._timeout = timeout self._timeout = timeout
self._auth_token = auth_token if auth_token is not None else _host_auth_token()
def _request( def _request(
self, method: str, path: str, body: dict[str, object] | None = None, self, method: str, path: str, body: dict[str, object] | None = None,
@@ -75,8 +49,6 @@ class OrchestratorClient:
callers can treat 404 as a meaningful "no such bottle".""" callers can treat 404 as a meaningful "no such bottle"."""
data = json.dumps(body).encode() if body is not None else None data = json.dumps(body).encode() if body is not None else None
headers = {"Content-Type": "application/json"} if data is not None else {} headers = {"Content-Type": "application/json"} if data is not None else {}
if self._auth_token:
headers[CONTROL_AUTH_HEADER] = self._auth_token
req = urllib.request.Request( req = urllib.request.Request(
f"{self._base}{path}", data=data, method=method, headers=headers, f"{self._base}{path}", data=data, method=method, headers=headers,
) )
@@ -214,13 +186,6 @@ def discover_orchestrator_url(*, timeout: float = 2.0) -> str:
f"http://{netpool.orch_slot().guest_ip}:{CONTROL_PLANE_PORT}") f"http://{netpool.orch_slot().guest_ip}:{CONTROL_PLANE_PORT}")
except Exception: # noqa: BLE001 — backend optional / not firecracker except Exception: # noqa: BLE001 — backend optional / not firecracker
pass pass
try: # macOS: infra container control plane on its host-only address
from ..backend.macos_container.infra import probe_control_plane_url
url = probe_control_plane_url()
if url:
candidates.append(url)
except Exception: # noqa: BLE001 — backend optional / not macOS
pass
for url in candidates: for url in candidates:
if OrchestratorClient(url, timeout=timeout).health(): if OrchestratorClient(url, timeout=timeout).health():
return url return url
+5 -59
View File
@@ -34,7 +34,6 @@ returned only once, to the caller that launches the bottle.
from __future__ import annotations from __future__ import annotations
import hmac
import http.server import http.server
import json import json
import os import os
@@ -43,20 +42,11 @@ import sys
import typing import typing
from urllib.parse import urlsplit from urllib.parse import urlsplit
from ..paths import CONTROL_PLANE_TOKEN_ENV
from .service import Orchestrator from .service import Orchestrator
# JSON body payload type (parsed request / rendered response). # JSON body payload type (parsed request / rendered response).
Json = dict[str, object] Json = dict[str, object]
# The request header carrying the per-host control-plane secret. Every route
# except `GET /health` requires it (see `dispatch`). The trusted callers hold
# the secret (the gateway's PolicyResolver, the host CLI's OrchestratorClient);
# an agent that can merely *reach* the port cannot present it, so it can't
# enumerate bottles, rewrite policy, read injected upstream tokens, or approve
# its own supervise proposals.
CONTROL_AUTH_HEADER = "x-bot-bottle-control-auth"
def _parse_json_object(body: bytes) -> Json: def _parse_json_object(body: bytes) -> Json:
"""Parse a JSON object body. Raises ValueError for non-objects / bad JSON.""" """Parse a JSON object body. Raises ValueError for non-objects / bad JSON."""
@@ -69,29 +59,15 @@ def _parse_json_object(body: bytes) -> Json:
def dispatch( # pylint: disable=too-many-return-statements,too-many-branches def dispatch( # pylint: disable=too-many-return-statements,too-many-branches
orch: Orchestrator, method: str, path: str, body: bytes, *, authorized: bool = True, orch: Orchestrator, method: str, path: str, body: bytes
) -> tuple[int, Json]: ) -> tuple[int, Json]:
"""Route one control-plane request to a (status, payload) pair. Pure — """Route one control-plane request to a (status, payload) pair. Pure —
no I/O beyond the orchestrator so it is fully testable without a socket. no I/O beyond the orchestrator so it is fully testable without a socket."""
`authorized` is whether the request presented the control-plane secret (or
no secret is configured see `ControlPlaneServer`). Every route except
`GET /health` requires it: the source-IP + identity-token checks inside
`/resolve` and `/attribute` authenticate the *bottle* a request is about,
not the *caller*, so without this gate any agent that can reach the port
could rewrite another bottle's policy, read the injected upstream tokens,
or approve its own supervise proposals. Defaults True so unit tests of the
routing logic don't have to thread it through."""
route = urlsplit(path).path.rstrip("/") or "/" route = urlsplit(path).path.rstrip("/") or "/"
if method == "GET" and route == "/health": if method == "GET" and route == "/health":
return 200, {"status": "ok"} return 200, {"status": "ok"}
if not authorized:
# Everything below is a trusted-caller operation. Deny before touching
# the registry / broker / supervise store.
return 401, {"error": "control-plane authentication required"}
if method == "GET" and route == "/gateway": if method == "GET" and route == "/gateway":
return 200, orch.gateway_status() return 200, orch.gateway_status()
@@ -233,10 +209,8 @@ class Handler(http.server.BaseHTTPRequestHandler):
assert isinstance(server, ControlPlaneServer) assert isinstance(server, ControlPlaneServer)
length = int(self.headers.get("Content-Length") or 0) length = int(self.headers.get("Content-Length") or 0)
body = self.rfile.read(length) if length > 0 else b"" body = self.rfile.read(length) if length > 0 else b""
authorized = server.is_authorized(self.headers.get(CONTROL_AUTH_HEADER, ""))
try: try:
status, payload = dispatch( status, payload = dispatch(server.orchestrator, method, self.path, body)
server.orchestrator, method, self.path, body, authorized=authorized)
except Exception as e: # noqa: BLE001 — the control plane must stay up except Exception as e: # noqa: BLE001 — the control plane must stay up
sys.stderr.write(f"orchestrator: {method} {self.path} failed: {e!r}\n") sys.stderr.write(f"orchestrator: {method} {self.path} failed: {e!r}\n")
sys.stderr.flush() sys.stderr.flush()
@@ -262,40 +236,15 @@ class Handler(http.server.BaseHTTPRequestHandler):
class ControlPlaneServer(socketserver.ThreadingMixIn, http.server.HTTPServer): class ControlPlaneServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
"""Threading HTTP server that carries the orchestrator for its handlers. """Threading HTTP server that carries the orchestrator for its handlers."""
Holds the per-host control-plane secret (from `$BOT_BOTTLE_CONTROL_PLANE_TOKEN`,
injected by the launcher into this container only). When a secret is set,
every route but `/health` requires it; when it is unset the server runs
**open** and says so loudly at startup a fail-visible fallback for tests
and any backend that hasn't wired the secret yet (e.g. Firecracker, whose
nft boundary already blocks agents from the control-plane port)."""
daemon_threads = True daemon_threads = True
allow_reuse_address = True allow_reuse_address = True
def __init__(self, address: tuple[str, int], orchestrator: Orchestrator) -> None: def __init__(self, address: tuple[str, int], orchestrator: Orchestrator) -> None:
self.orchestrator = orchestrator self.orchestrator = orchestrator
self._auth_token = os.environ.get(CONTROL_PLANE_TOKEN_ENV, "").strip()
if not self._auth_token:
sys.stderr.write(
"orchestrator: WARNING — no control-plane secret "
f"(${CONTROL_PLANE_TOKEN_ENV}); running WITHOUT caller "
"authentication. Any client that can reach this port can drive "
"it. Backends that put the control plane on an agent-reachable "
"network MUST set this.\n"
)
sys.stderr.flush()
super().__init__(address, Handler) super().__init__(address, Handler)
def is_authorized(self, presented: str) -> bool:
"""True iff the request may proceed past `/health`: either no secret is
configured (open mode) or the presented header matches it. Constant-time
compare so a wrong token leaks nothing timing-wise."""
if not self._auth_token:
return True
return hmac.compare_digest(presented, self._auth_token)
def make_server( def make_server(
orchestrator: Orchestrator, host: str = "127.0.0.1", port: int = 0 orchestrator: Orchestrator, host: str = "127.0.0.1", port: int = 0
@@ -305,7 +254,4 @@ def make_server(
return ControlPlaneServer((host, port), orchestrator) return ControlPlaneServer((host, port), orchestrator)
__all__ = [ __all__ = ["dispatch", "Handler", "ControlPlaneServer", "make_server", "Json"]
"dispatch", "Handler", "ControlPlaneServer", "make_server", "Json",
"CONTROL_AUTH_HEADER",
]
+2 -13
View File
@@ -23,11 +23,7 @@ import time
from pathlib import Path from pathlib import Path
from ..docker_cmd import run_docker from ..docker_cmd import run_docker
from ..paths import ( from ..paths import host_db_path
CONTROL_PLANE_TOKEN_ENV,
host_control_plane_token,
host_db_path,
)
from ..supervise import DB_PATH_IN_CONTAINER from ..supervise import DB_PATH_IN_CONTAINER
# The host DB dir is bind-mounted here so the gateway's supervise daemon # The host DB dir is bind-mounted here so the gateway's supervise daemon
@@ -219,19 +215,12 @@ class DockerGateway(Gateway):
] ]
for port in self._host_port_bindings: for port in self._host_port_bindings:
argv += ["--publish", f"0.0.0.0:{port}:{port}"] argv += ["--publish", f"0.0.0.0:{port}:{port}"]
run_env = dict(os.environ)
if self._orchestrator_url: if self._orchestrator_url:
# Makes the gateway's egress / git / supervise daemons multi-tenant: # Makes the gateway's egress / git / supervise daemons multi-tenant:
# each request resolves source-IP -> policy against the control plane. # each request resolves source-IP -> policy against the control plane.
argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"] argv += ["--env", f"BOT_BOTTLE_ORCHESTRATOR_URL={self._orchestrator_url}"]
# ...and presents the control-plane secret on those /resolve calls
# (the control plane requires it). Bare `--env NAME` keeps the value
# off argv / `docker inspect`; only the gateway (not the agent) is
# given it. Only needed in multi-tenant mode, where /resolve is used.
argv += ["--env", CONTROL_PLANE_TOKEN_ENV]
run_env[CONTROL_PLANE_TOKEN_ENV] = host_control_plane_token()
argv.append(self.image_ref) argv.append(self.image_ref)
proc = run_docker(argv, env=run_env) proc = run_docker(argv)
if proc.returncode != 0: if proc.returncode != 0:
raise GatewayError(f"gateway failed to start: {proc.stderr.strip()}") raise GatewayError(f"gateway failed to start: {proc.stderr.strip()}")
+14 -30
View File
@@ -25,8 +25,8 @@ from pathlib import Path
from .. import log from .. import log
from ..docker_cmd import run_docker from ..docker_cmd import run_docker
from ..paths import CONTROL_PLANE_TOKEN_ENV, bot_bottle_root, host_control_plane_token from ..paths import bot_bottle_root
from .gateway import GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK, DockerGateway, GatewayError from .gateway import GATEWAY_IMAGE, GATEWAY_NETWORK, DockerGateway, GatewayError
DEFAULT_PORT = 8099 DEFAULT_PORT = 8099
ORCHESTRATOR_NAME = "bot-bottle-orchestrator" ORCHESTRATOR_NAME = "bot-bottle-orchestrator"
@@ -41,7 +41,7 @@ ORCHESTRATOR_IMAGE = os.environ.get(
ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator" ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator"
# Baked onto the container as a label so `ensure_running` can tell whether the # Baked onto the container as a label so `ensure_running` can tell whether the
# running process is executing the *current* bind-mounted source — see # running process is executing the *current* bind-mounted source — see
# `source_hash`. # `_source_hash`.
ORCHESTRATOR_SOURCE_HASH_LABEL = "bot-bottle-orchestrator-source-hash" ORCHESTRATOR_SOURCE_HASH_LABEL = "bot-bottle-orchestrator-source-hash"
# The repo root is bind-mounted into the control-plane container so # The repo root is bind-mounted into the control-plane container so
@@ -60,7 +60,7 @@ class OrchestratorStartError(RuntimeError):
"""The orchestrator container did not become healthy within the timeout.""" """The orchestrator container did not become healthy within the timeout."""
def source_hash(repo_root: Path) -> str: def _source_hash(repo_root: Path) -> str:
"""Content hash of the orchestrator's bind-mounted Python source (the """Content hash of the orchestrator's bind-mounted Python source (the
`bot_bottle` package the control-plane process imports). This only `bot_bottle` package the control-plane process imports). This only
changes when the code that would actually run inside the container changes when the code that would actually run inside the container
@@ -83,11 +83,8 @@ class OrchestratorService:
`orchestrator_name` / `orchestrator_label` let backends run independent `orchestrator_name` / `orchestrator_label` let backends run independent
orchestrators on the same host without name collisions (e.g. the orchestrators on the same host without name collisions (e.g. the
Firecracker backend uses `bot-bottle-fc-orchestrator` alongside the Docker Firecracker backend uses `bot-bottle-fc-orchestrator` alongside the Docker
backend's `bot-bottle-orchestrator`); `gateway_name` gives the paired backend's `bot-bottle-orchestrator`). Subclass and override `_gateway()`
gateway container the same treatment (e.g. isolated integration tests to supply a backend-specific gateway variant."""
that can't share the production `GATEWAY_NAME` singleton). Subclass and
override `_gateway()` for anything `_gateway_image`/`gateway_name` can't
express (a genuinely backend-specific gateway variant)."""
def __init__( def __init__(
self, self,
@@ -96,7 +93,6 @@ class OrchestratorService:
network: str = GATEWAY_NETWORK, network: str = GATEWAY_NETWORK,
image: str = ORCHESTRATOR_IMAGE, image: str = ORCHESTRATOR_IMAGE,
gateway_image: str = GATEWAY_IMAGE, gateway_image: str = GATEWAY_IMAGE,
gateway_name: str = GATEWAY_NAME,
repo_root: Path = _REPO_ROOT, repo_root: Path = _REPO_ROOT,
host_root: Path | None = None, host_root: Path | None = None,
orchestrator_name: str = ORCHESTRATOR_NAME, orchestrator_name: str = ORCHESTRATOR_NAME,
@@ -110,7 +106,6 @@ class OrchestratorService:
# were one conflated image before the split. # were one conflated image before the split.
self.image = image self.image = image
self._gateway_image = gateway_image self._gateway_image = gateway_image
self._gateway_name = gateway_name
self._repo_root = repo_root self._repo_root = repo_root
self._host_root = host_root or bot_bottle_root() self._host_root = host_root or bot_bottle_root()
self._orchestrator_name = orchestrator_name self._orchestrator_name = orchestrator_name
@@ -139,24 +134,20 @@ class OrchestratorService:
proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"]) proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"])
return name in proc.stdout.split() return name in proc.stdout.split()
def _run_orchestrator_container(self, current_hash: str) -> None: def _run_orchestrator_container(self, source_hash: str) -> None:
"""Start the control-plane container (idempotent: clears a stale """Start the control-plane container (idempotent: clears a stale
fixed-name container first). Register-only broker no docker socket. fixed-name container first). Register-only broker no docker socket.
Labels the container with `current_hash` so a later `ensure_running` Labels the container with `source_hash` so a later `ensure_running`
can detect a real code change (see `source_hash`).""" can detect a real code change (see `_source_hash`)."""
run_docker(["docker", "rm", "--force", self._orchestrator_name]) run_docker(["docker", "rm", "--force", self._orchestrator_name])
proc = run_docker([ proc = run_docker([
"docker", "run", "--detach", "docker", "run", "--detach",
"--name", self._orchestrator_name, "--name", self._orchestrator_name,
"--label", self._orchestrator_label, "--label", self._orchestrator_label,
"--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current_hash}", "--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={source_hash}",
"--network", self.network, "--network", self.network,
# Host CLI reaches the control plane here; bound to loopback so it # Host CLI reaches the control plane here; bound to loopback so it
# is not exposed on the host's external interfaces. NOTE: the # is not exposed on the host's external interfaces.
# container is still on `self.network` (the shared gateway network),
# so agents can reach it by container IP — which is exactly why the
# control plane requires the secret below rather than trusting the
# network boundary.
"--publish", f"127.0.0.1:{self.port}:{self.port}", "--publish", f"127.0.0.1:{self.port}:{self.port}",
"--volume", f"{self._repo_root}:{_APP_DIR}:ro", "--volume", f"{self._repo_root}:{_APP_DIR}:ro",
"--workdir", _APP_DIR, "--workdir", _APP_DIR,
@@ -164,15 +155,11 @@ class OrchestratorService:
# orchestrator opens bot-bottle.db). # orchestrator opens bot-bottle.db).
"--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}", "--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}",
"--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}", "--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}",
# The control-plane secret it requires on every route but /health.
# Bare `--env NAME` → docker inherits the value from the run env
# below, so the secret never lands on argv / `docker inspect`.
"--env", CONTROL_PLANE_TOKEN_ENV,
"--entrypoint", "python3", "--entrypoint", "python3",
self.image, self.image,
"-m", "bot_bottle.orchestrator", "-m", "bot_bottle.orchestrator",
"--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub", "--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub",
], env={**os.environ, CONTROL_PLANE_TOKEN_ENV: host_control_plane_token()}) ])
if proc.returncode != 0: if proc.returncode != 0:
raise OrchestratorStartError( raise OrchestratorStartError(
f"orchestrator container failed to start: {proc.stderr.strip()}" f"orchestrator container failed to start: {proc.stderr.strip()}"
@@ -180,10 +167,7 @@ class OrchestratorService:
def _gateway(self) -> DockerGateway: def _gateway(self) -> DockerGateway:
return DockerGateway( return DockerGateway(
self._gateway_image, self._gateway_image, network=self.network, orchestrator_url=self.internal_url
name=self._gateway_name,
network=self.network,
orchestrator_url=self.internal_url,
) )
def _ensure_orchestrator_image(self) -> None: def _ensure_orchestrator_image(self) -> None:
@@ -240,7 +224,7 @@ class OrchestratorService:
# launch (the prior behaviour) would drop every other active # launch (the prior behaviour) would drop every other active
# bottle's in-memory egress tokens each time a new bottle starts, # bottle's in-memory egress tokens each time a new bottle starts,
# since the orchestrator process holds them only in memory (#381). # since the orchestrator process holds them only in memory (#381).
current_hash = source_hash(self._repo_root) current_hash = _source_hash(self._repo_root)
if self.is_healthy() and self._orchestrator_source_current(current_hash): if self.is_healthy() and self._orchestrator_source_current(current_hash):
return self.url return self.url
+1 -59
View File
@@ -16,8 +16,6 @@ layer (and to COPY flat into the gateway).
from __future__ import annotations from __future__ import annotations
import os import os
import secrets
import stat
from pathlib import Path from pathlib import Path
# The single shared host state DB. All bot-bottle SQLite stores (supervise # The single shared host state DB. All bot-bottle SQLite stores (supervise
@@ -25,14 +23,6 @@ from pathlib import Path
# TableMigrations schema_key namespaces each store's tables. # TableMigrations schema_key namespaces each store's tables.
HOST_DB_FILENAME = "bot-bottle.db" HOST_DB_FILENAME = "bot-bottle.db"
# The per-host control-plane secret file, and the env var the launchers inject
# its value into. The control plane requires this secret on every mutating /
# reading route (see orchestrator/control_plane.py); it is held only by the
# trusted callers (control plane, gateway, host CLI) and never handed to an
# agent, so an agent that can reach the control-plane port still can't drive it.
CONTROL_PLANE_TOKEN_FILENAME = "control-plane-token"
CONTROL_PLANE_TOKEN_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN"
def bot_bottle_root() -> Path: def bot_bottle_root() -> Path:
"""The app data root — `$BOT_BOTTLE_ROOT` if set, else `~/.bot-bottle`.""" """The app data root — `$BOT_BOTTLE_ROOT` if set, else `~/.bot-bottle`."""
@@ -50,52 +40,4 @@ def host_db_path() -> Path:
return bot_bottle_root() / "db" / HOST_DB_FILENAME return bot_bottle_root() / "db" / HOST_DB_FILENAME
def host_db_dir() -> Path: __all__ = ["HOST_DB_FILENAME", "bot_bottle_root", "host_db_path"]
"""The directory holding the shared host state DB, created if missing.
Backends bind-mount this into their gateway so the supervise daemon writes
to the one DB the orchestrator (and the operator over HTTP) reads."""
db_dir = host_db_path().parent
db_dir.mkdir(parents=True, exist_ok=True)
return db_dir
def host_control_plane_token() -> str:
"""The per-host control-plane secret, minted (256-bit, url-safe) and
persisted 0600 on first use, then reused.
This is the shared secret the launchers inject into the control-plane and
gateway containers and that the host CLI presents on every call. It is a
*host* artifact the file lives under the root the agent never mounts, and
the env var is set only on the trusted containers so reading it here is
safe on the host launch path but the value never reaches a bottle."""
path = bot_bottle_root() / CONTROL_PLANE_TOKEN_FILENAME
try:
existing = path.read_text().strip()
if existing:
return existing
except OSError:
pass
path.parent.mkdir(parents=True, exist_ok=True)
token = secrets.token_urlsafe(32)
# Create 0600 up front (O_EXCL loses a concurrent race harmlessly — we
# re-read the winner's token below) so the secret is never briefly world-
# readable between write and chmod.
try:
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
except FileExistsError:
return path.read_text().strip()
with os.fdopen(fd, "w") as f:
f.write(token)
os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
return token
__all__ = [
"HOST_DB_FILENAME",
"CONTROL_PLANE_TOKEN_FILENAME",
"CONTROL_PLANE_TOKEN_ENV",
"bot_bottle_root",
"host_db_path",
"host_db_dir",
"host_control_plane_token",
]
+1 -19
View File
@@ -29,29 +29,11 @@ the gateway.
from __future__ import annotations from __future__ import annotations
import json import json
import os
import urllib.error import urllib.error
import urllib.request import urllib.request
DEFAULT_TIMEOUT_SECONDS = 2.0 DEFAULT_TIMEOUT_SECONDS = 2.0
# The control-plane secret this gateway presents on every /resolve call, read
# from the env the launcher injects into the gateway container. The control
# plane requires it (orchestrator/control_plane.py). Constant + env-var name are
# duplicated here rather than imported because this module is COPYed flat into
# the gateway image, free of bot-bottle imports — same rationale as
# IDENTITY_HEADER in egress_addon / git_http_backend.
CONTROL_AUTH_HEADER = "x-bot-bottle-control-auth"
CONTROL_PLANE_TOKEN_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN"
def _control_auth_headers() -> dict[str, str]:
"""The auth header to send, or {} when no secret is configured (an open
control plane, e.g. Firecracker behind its nft boundary sending nothing
is correct there and harmlessly ignored)."""
token = os.environ.get(CONTROL_PLANE_TOKEN_ENV, "").strip()
return {CONTROL_AUTH_HEADER: token} if token else {}
class PolicyResolveError(RuntimeError): class PolicyResolveError(RuntimeError):
"""The orchestrator was unreachable or returned an unexpected status — """The orchestrator was unreachable or returned an unexpected status —
@@ -75,7 +57,7 @@ class PolicyResolver:
).encode() ).encode()
req = urllib.request.Request( req = urllib.request.Request(
f"{self._base}/resolve", data=body, method="POST", f"{self._base}/resolve", data=body, method="POST",
headers={"Content-Type": "application/json", **_control_auth_headers()}, headers={"Content-Type": "application/json"},
) )
try: try:
with urllib.request.urlopen(req, timeout=self._timeout) as resp: with urllib.request.urlopen(req, timeout=self._timeout) as resp:
@@ -112,12 +112,12 @@ bottle-agnostic**: the per-boot bits (authorized_keys, guest IP) arrive on the
published ext4 boots on any launch host. published ext4 boots on any launch host.
- **Artifact.** `rootfs.ext4` + a `rootfs.ext4.sha256`, published as a Gitea - **Artifact.** `rootfs.ext4` + a `rootfs.ext4.sha256`, published as a Gitea
**generic package** (`bot-bottle-firecracker-infra/<tag>`) — generic packages take **generic package** (`bot-bottle-infra/<tag>`) — generic packages take
arbitrary large binaries (no attachment size cap / file-type allowlist that arbitrary large binaries (no attachment size cap / file-type allowlist that
release attachments impose). The matching `vmlinux` kernel can ship the same release attachments impose). The matching `vmlinux` kernel can ship the same
way, so the whole VM is fetchable. way, so the whole VM is fetchable.
- **Pull.** The launch host `GET`s - **Pull.** The launch host `GET`s
`…/api/packages/<owner>/generic/bot-bottle-firecracker-infra/<tag>/rootfs.ext4` (+ `…/api/packages/<owner>/generic/bot-bottle-infra/<tag>/rootfs.ext4` (+
`.sha256`) for its pinned tag, verifies the checksum, caches it under the `.sha256`) for its pinned tag, verifies the checksum, caches it under the
tag, and attaches it as the infra VM's root disk. Host prerequisite is an tag, and attaches it as the infra VM's root disk. Host prerequisite is an
HTTP client — nothing else. Public packages need no auth to pull; a token HTTP client — nothing else. Public packages need no auth to pull; a token
@@ -358,111 +358,3 @@ the Apple Container-specific constraints directly:
Do not implement the backend as a direct clone of Docker Compose Do not implement the backend as a direct clone of Docker Compose
service aliases. That assumption failed in this run. service aliases. That assumption failed in this run.
## Addendum: consolidated-gateway findings (2026-07-17, PRD 0070)
Re-tested on Apple Container 1.0.0 while porting the backend to the
per-host consolidated gateway (#351). The two-network shape above still
holds; these are the additional constraints that shaped the port, each
verified against the live CLI on this host.
### No static IP for a container
`container run --network` accepts only
`<name>[,mac=XX:XX:XX:XX:XX:XX][,mtu=VALUE]`. There is no `--ip`. The
address comes from vmnet's DHCP and is knowable only after the container
is running:
```console
$ container run --name a --network bb-net --detach alpine sleep 900
$ container inspect a | jq -r '.[0].status.networks[0].ipv4Address'
192.168.128.3/24
```
Consequence: the docker backend's "allocate a free IP -> pin it with
`--ip` -> register -> launch" order cannot be reproduced. macOS inverts
it to "launch -> read the assigned address -> register". The identity
token therefore cannot be in the agent's run-time env (registration mints
it after the container exists) and is delivered at `container exec` time.
### Networks are fixed at run time
There is no `container network connect`; `container network` exposes only
`create`, `delete`, `list`, `inspect`, `prune`. A network cannot be
attached to a running container, so a *persistent* shared gateway rules
out per-bottle networks — they would force a gateway restart per launch.
One shared host-only network, created up front, is the only shape that
keeps the gateway a singleton.
### No container DNS
Containers cannot resolve each other by name; the host-only network's
resolver refuses the query:
```console
$ container exec agent nslookup gw
;; connection timed out; no servers could be reached
$ container exec agent cat /etc/resolv.conf
nameserver 192.168.128.1
```
Consequence: the gateway is handed the control plane's **IP**, not a
container name as on docker. That forces the startup order
orchestrator -> read its address -> gateway.
### The host can reach the host-only network directly
```console
$ container inspect c | jq -r '.[0].status.networks[0].ipv4Address'
192.168.128.2/24
$ curl -s http://192.168.128.2:8099/i
ok
```
So no `--publish` hop is needed: the host CLI and the gateway use the
same control-plane URL. Docker needs `--publish 127.0.0.1:...` plus a
separate internal URL for the same job.
### CAP_NET_RAW is granted by default — and matters for attribution
Apple grants NET_RAW but not NET_ADMIN. The agent therefore cannot change
its own address or route:
```console
$ container exec agent ip addr add 192.168.128.99/24 dev eth0
ip: RTNETLINK answers: Operation not permitted
$ container exec agent ip route replace default via 192.168.128.2 dev eth0
ip: RTNETLINK answers: Operation not permitted
$ container exec agent grep CapEff /proc/self/status
CapEff: 00000000a80425fb # bit 13 (NET_RAW) set, bit 12 (NET_ADMIN) clear
```
But NET_RAW permits raw sockets, i.e. source-address forgery against
neighbours on the shared segment — directly against PRD 0070's invariant
("a packet's source address, as seen by the orchestrator, provably
identifies the originating bottle"). `--cap-drop CAP_NET_RAW` closes it:
```console
$ container run --cap-drop CAP_NET_RAW ... alpine
$ container exec nr grep CapEff /proc/self/status
CapEff: 00000000a80405fb # bit 13 cleared
$ container exec nr ping -c1 192.168.128.2
ping: permission denied (are you root?)
```
The agent is run with `--cap-drop CAP_NET_RAW` for this reason.
### `container exec` inherits run-time env, and `--env` overrides it
```console
$ container run --name e --env FOO=from_run --detach alpine sleep 120
$ container exec e sh -c 'echo $FOO'
from_run
$ container exec --env FOO=from_exec e sh -c 'echo $FOO'
from_exec
```
This is what makes exec-time identity-token delivery work: the token-less
proxy URL baked in at launch is superseded by the token-bearing one at
exec. Bare `--env NAME` (inherit from the parent process) keeps the token
value off argv.
@@ -1,155 +0,0 @@
"""Integration: the Docker orchestrator's control plane enforces the
per-host auth secret against a real container (issue #400).
Unit tests exercise `dispatch()` in-process, socket-free. This starts the
actual orchestrator + gateway as Docker containers and drives the real HTTP
server over its published loopback port, the same path an agent sharing the
gateway network or the trusted host CLI would use.
Gated on a reachable Docker daemon. Uses unique container/network names,
test-only image tags (never the production `:latest` ones, so a rebuild
here can't make `_running_image_is_current()` see a real host's running
gateway as stale and force-recreate it), and a throwaway `BOT_BOTTLE_ROOT`
so a run never touches or collides with a real per-host orchestrator or
gateway. The whole stack is brought up once for the class (`setUpClass`),
not per test method every test here is a read-only check against the
same running control plane.
"""
from __future__ import annotations
import os
import secrets
import subprocess
import tempfile
import unittest
from pathlib import Path
from bot_bottle.orchestrator.client import OrchestratorClient
from bot_bottle.orchestrator.lifecycle import OrchestratorService
from bot_bottle.paths import host_control_plane_token
from tests._docker import skip_unless_docker
# Fixed (not per-run-suffixed) so repeated runs reuse the same layer-cached
# image instead of leaking a new dangling tag on every invocation.
_TEST_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:itest"
_TEST_GATEWAY_IMAGE = "bot-bottle-gateway:itest"
@skip_unless_docker()
@unittest.skipIf(
os.environ.get("GITEA_ACTIONS") == "true",
"skipped under act_runner: the orchestrator container bind-mounts the repo "
"path into a container on the socket-shared host daemon, which can't see the "
"runner's /workspace — same host-bind-mount constraint as the other "
"bottle-bringup integration tests",
)
class TestDockerControlPlaneAuthIntegration(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
suffix = secrets.token_hex(4)
cls._tmp = tempfile.TemporaryDirectory() # pylint: disable=consider-using-with
cls.addClassCleanup(cls._tmp.cleanup)
# host_control_plane_token() — both the token read below and the one
# OrchestratorService injects into the container's env — resolves its
# path via the *ambient* BOT_BOTTLE_ROOT env var, not the host_root
# kwarg passed to the constructor (that kwarg only controls the DB
# bind-mount destination). Without pointing the env var at the same
# throwaway dir, this "isolated" test would read/write the developer's
# real ~/.bot-bottle/control-plane-token.
previous_root = os.environ.get("BOT_BOTTLE_ROOT")
def _restore_root() -> None:
if previous_root is None:
os.environ.pop("BOT_BOTTLE_ROOT", None)
else:
os.environ["BOT_BOTTLE_ROOT"] = previous_root
os.environ["BOT_BOTTLE_ROOT"] = cls._tmp.name
cls.addClassCleanup(_restore_root)
orchestrator_name = f"bot-bottle-orch-itest-{suffix}"
gateway_name = f"bot-bottle-gw-itest-{suffix}"
network = f"bot-bottle-net-itest-{suffix}"
host_root = Path(cls._tmp.name)
cls.addClassCleanup(
cls._teardown_docker, orchestrator_name, gateway_name, network, host_root
)
cls.svc = OrchestratorService(
orchestrator_name=orchestrator_name,
gateway_name=gateway_name,
network=network,
image=_TEST_ORCHESTRATOR_IMAGE,
gateway_image=_TEST_GATEWAY_IMAGE,
port=20000 + secrets.randbelow(10000),
host_root=host_root,
)
cls.svc.ensure_running()
cls.token = host_control_plane_token()
@staticmethod
def _teardown_docker(
orchestrator_name: str, gateway_name: str, network: str, host_root: Path
) -> None:
subprocess.run(
["docker", "rm", "--force", orchestrator_name, gateway_name],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
)
subprocess.run(
["docker", "network", "rm", network],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
)
# The orchestrator container (no USER directive) wrote the registry
# DB as root into the throwaway host_root; chown it back so the
# (non-root) tempdir cleanup can remove it. Same workaround
# test_multitenant_isolation.py uses for the identical bind mount.
subprocess.run(
["docker", "run", "--rm", "-v", f"{host_root}:/r",
"--entrypoint", "chown", _TEST_GATEWAY_IMAGE, "-R",
f"{os.getuid()}:{os.getgid()}", "/r"],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
)
def _request(
self, method: str, path: str, *, token: str = ""
) -> tuple[int, dict[str, object]]:
# Reuses the real host-side client's request/response handling rather
# than hand-rolling urllib here; _request (not one of the named
# wrapper methods) is what exposes raw status codes for arbitrary
# paths/tokens, which is exactly what these auth-boundary tests need.
client = OrchestratorClient(self.svc.url, auth_token=token)
return client._request(method, path) # pylint: disable=protected-access
def test_health_is_open_without_a_token(self) -> None:
status, payload = self._request("GET", "/health")
self.assertEqual(200, status)
self.assertEqual("ok", payload["status"])
def test_bottles_rejects_a_caller_with_no_token(self) -> None:
"""The enumeration attack from issue #400: an agent sharing the
gateway network could list every bottle + its policy with no
credential at all."""
status, _ = self._request("GET", "/bottles")
self.assertEqual(401, status)
def test_bottles_rejects_a_wrong_token(self) -> None:
status, _ = self._request("GET", "/bottles", token="not-the-real-secret")
self.assertEqual(401, status)
def test_bottles_accepts_the_real_token(self) -> None:
status, payload = self._request("GET", "/bottles", token=self.token)
self.assertEqual(200, status)
self.assertEqual([], payload["bottles"])
def test_resolve_rejects_a_caller_with_no_token(self) -> None:
"""The credential-lift attack from issue #400: an agent could POST
/resolve directly and read back the upstream tokens it's never meant
to see."""
status, _ = self._request("POST", "/resolve")
self.assertEqual(401, status)
if __name__ == "__main__":
unittest.main()
+6 -10
View File
@@ -205,29 +205,25 @@ class TestFirecrackerFreezer(_FakeHomeMixin, unittest.TestCase):
) )
def test_snapshots_running_vm_without_stopping(self): def test_snapshots_running_vm_without_stopping(self):
"""Commit should tar the running guest rootfs over SSH into the """Commit should tar the running guest rootfs over SSH, not stop it."""
committed-rootfs artifact (no Docker), not stop the VM."""
slug = "dev-abc12" slug = "dev-abc12"
self._write_meta(slug) self._write_meta(slug)
self._stage_run_dir(slug) self._stage_run_dir(slug)
freezer = FirecrackerFreezer() freezer = FirecrackerFreezer()
agent = _make_agent(slug, "firecracker") agent = _make_agent(slug, "firecracker")
commit_fn = "bot_bottle.backend.firecracker.freezer._commit_rootfs_via_ssh" with patch("bot_bottle.backend.firecracker.freezer._commit_via_ssh") as mock_commit, \
with patch(commit_fn) as mock_commit, \
patch("bot_bottle.backend.freeze.info"), \ patch("bot_bottle.backend.freeze.info"), \
patch("bot_bottle.backend.firecracker.freezer.info"): patch("bot_bottle.backend.firecracker.freezer.info"):
freezer.commit(agent) freezer.commit(agent)
tar_path = bottle_state.committed_rootfs_path(slug) image_tag = f"bot-bottle-committed-{slug}:latest"
self.assertEqual(1, mock_commit.call_count) self.assertEqual(1, mock_commit.call_count)
# (private_key, guest_ip, tar_path) — guest_ip parsed from config. # (private_key, guest_ip, image_tag) — guest_ip parsed from config.
args = mock_commit.call_args.args args = mock_commit.call_args.args
self.assertEqual("100.64.0.1", args[1]) self.assertEqual("100.64.0.1", args[1])
self.assertEqual(tar_path, args[2]) self.assertEqual(image_tag, args[2])
# The committed-image state records the artifact path; resume boots self.assertEqual(image_tag, bottle_state.read_committed_image(slug))
# from the tar rather than a Docker image.
self.assertEqual(str(tar_path), bottle_state.read_committed_image(slug))
self.assertTrue(bottle_state.is_preserved(slug)) self.assertTrue(bottle_state.is_preserved(slug))
+5 -10
View File
@@ -247,7 +247,7 @@ class TestHasBackend(unittest.TestCase):
class TestEnsureOrchestrator(unittest.TestCase): class TestEnsureOrchestrator(unittest.TestCase):
"""The backend-agnostic orchestrator bring-up entry point. Docker starts """The backend-agnostic orchestrator bring-up entry point. Docker starts
the orchestrator + gateway containers; firecracker boots the infra VM; the orchestrator + gateway containers; firecracker boots the infra VM;
macos-container starts the infra container.""" backends without one (macos-container) die with a pointer."""
def test_docker_delegates_to_orchestrator_service(self): def test_docker_delegates_to_orchestrator_service(self):
b = get_bottle_backend("docker") b = get_bottle_backend("docker")
@@ -272,16 +272,11 @@ class TestEnsureOrchestrator(unittest.TestCase):
url = b.ensure_orchestrator() url = b.ensure_orchestrator()
self.assertEqual(url, "http://10.243.255.1:8099") self.assertEqual(url, "http://10.243.255.1:8099")
def test_macos_delegates_to_infra_container(self): def test_macos_default_dies(self):
from bot_bottle.log import Die
b = get_bottle_backend("macos-container") b = get_bottle_backend("macos-container")
with patch( with self.assertRaises(Die):
"bot_bottle.backend.macos_container.infra.MacosInfraService" b.ensure_orchestrator()
) as service_cls:
service_cls.return_value.ensure_running.return_value.control_plane_url = (
"http://192.168.128.2:8099"
)
url = b.ensure_orchestrator()
self.assertEqual(url, "http://192.168.128.2:8099")
if __name__ == "__main__": if __name__ == "__main__":
@@ -8,7 +8,6 @@ real mitmproxy package."""
from __future__ import annotations from __future__ import annotations
import json import json
import os
import sys import sys
import types import types
import unittest import unittest
@@ -108,14 +107,14 @@ class _Flow:
def _log_request(addon: EgressAddon, flow: _Flow) -> dict[str, Any]: def _log_request(addon: EgressAddon, flow: _Flow) -> dict[str, Any]:
buf = StringIO() buf = StringIO()
with patch("sys.stderr", buf): with patch("sys.stderr", buf):
addon._log_request(flow, os.environ) # type: ignore[arg-type] addon._log_request(flow) # type: ignore[arg-type]
return json.loads(buf.getvalue()) return json.loads(buf.getvalue())
def _log_response(addon: EgressAddon, flow: _Flow) -> dict[str, Any]: def _log_response(addon: EgressAddon, flow: _Flow) -> dict[str, Any]:
buf = StringIO() buf = StringIO()
with patch("sys.stderr", buf): with patch("sys.stderr", buf):
addon._log_response(flow, os.environ) # type: ignore[arg-type] addon._log_response(flow) # type: ignore[arg-type]
return json.loads(buf.getvalue()) return json.loads(buf.getvalue())
@@ -141,10 +141,6 @@ class _Flow:
self.response = response self.response = response
self.websocket: Any = None self.websocket: Any = None
self.killed = False self.killed = False
# mitmproxy flows carry a per-flow `metadata` dict for addon use; the
# egress addon stashes the resolved (config, slug, env) there in
# request() so the response/websocket hooks reuse it.
self.metadata: dict[str, Any] = {}
def kill(self) -> None: def kill(self) -> None:
self.killed = True self.killed = True
@@ -863,91 +859,5 @@ class TestSuperviseMultiTenant(unittest.TestCase):
self.assertNotIn(_OPENAI_KEY, addon._safe_tokens_for("")) self.assertNotIn(_OPENAI_KEY, addon._safe_tokens_for(""))
class TestMultiTenantInboundDlp(unittest.TestCase):
"""Consolidated gateway: the response + websocket DLP hooks must scan
against the *calling bottle's* config, resolved by source IP in request()
and reused here. The static `self.config` is empty in this mode, so before
the flow-context stash these hooks silently skipped every scan (fail-open).
"""
def _consolidated_addon(self) -> EgressAddon:
addon = _addon(Config(routes=())) # empty static config, as in prod
addon._resolver = cast(Any, _CtxResolver({"10.0.0.1": "bottle-a"}))
return addon
def test_response_injection_blocked_after_request_resolves(self) -> None:
addon = self._consolidated_addon()
flow = _with_client_ip(_Flow(_Request(host="api.example.com")), "10.0.0.1")
_run_request(addon, flow) # resolves + stashes bottle-a's allowlist
self.assertIsNone(flow.response) # request forwarded
flow.response = _Response(200, content=_INJECTION_BLOCK)
addon.response(flow) # type: ignore[arg-type]
assert flow.response is not None
# Empty static config would have found no route and left this 200.
self.assertEqual(403, flow.response.status_code)
def test_websocket_outbound_token_killed_after_request_resolves(self) -> None:
addon = self._consolidated_addon()
flow = _with_client_ip(_Flow(_Request(host="api.example.com")), "10.0.0.1")
_run_request(addon, flow) # the ws upgrade resolves + stashes the config
flow.websocket = _WebSocketData(
[_Message(f"k={_OPENAI_KEY}".encode(), from_client=True)]
)
addon.websocket_message(flow) # type: ignore[arg-type]
self.assertTrue(flow.killed) # scanned against bottle-a's route now
def test_websocket_inbound_injection_killed_after_request_resolves(self) -> None:
addon = self._consolidated_addon()
flow = _with_client_ip(_Flow(_Request(host="api.example.com")), "10.0.0.1")
_run_request(addon, flow)
flow.websocket = _WebSocketData(
[_Message(_INJECTION_BLOCK.encode(), from_client=False)]
)
addon.websocket_message(flow) # type: ignore[arg-type]
self.assertTrue(flow.killed)
def test_response_log_redacts_per_bottle_resolve_token(self) -> None:
# LOG_FULL response logging now runs in multi-tenant mode, so it must
# scrub the calling bottle's /resolve token — which lives only in the
# resolved env overlay, never in the gateway's os.environ. A
# non-token-shaped secret is caught only via that env, so os.environ
# redaction (the pre-fix behaviour) would leak it into the log.
secret = "bottle-a-provisioned-secret-value"
policy = "log: 2\nroutes:\n - host: api.example.com\n"
class _TokenResolver:
def resolve_policy_and_bottle_id(
self, ip: str, identity_token: str = "",
) -> tuple[str | None, str | None, dict[str, str]]:
del ip, identity_token
return policy, "bottle-a", {"EGRESS_TOKEN_0": secret}
addon = _addon(Config(routes=()))
addon._resolver = cast(Any, _TokenResolver())
flow = _with_client_ip(_Flow(_Request(host="api.example.com")), "10.0.0.1")
_run_request(addon, flow)
flow.response = _Response(200, content=f"echo {secret} back")
buf = StringIO()
with patch("sys.stderr", buf):
addon.response(flow) # type: ignore[arg-type]
logged = buf.getvalue()
self.assertIn("egress_response", logged) # LOG_FULL logged the response
self.assertNotIn(secret, logged) # redacted via the resolved env overlay
def test_unattributed_flow_has_no_route_so_frame_passes(self) -> None:
# An unattributed source resolves to a deny-all (empty) config, so the
# request is blocked at the upgrade and any later frame has no route to
# scan against — it passes rather than being attributed to a bottle.
addon = self._consolidated_addon()
flow = _with_client_ip(_Flow(_Request(host="api.example.com")), "10.9.9.9")
_run_request(addon, flow)
self.assertIsNotNone(flow.response) # blocked at the upgrade
flow.websocket = _WebSocketData(
[_Message(f"k={_OPENAI_KEY}".encode(), from_client=True)]
)
addon.websocket_message(flow) # type: ignore[arg-type]
self.assertFalse(flow.killed)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
-170
View File
@@ -6,13 +6,10 @@ branches. Mock subprocess/os so nothing needs KVM or a live VM.
from __future__ import annotations from __future__ import annotations
import json import json
import os
import stat
import subprocess import subprocess
import tempfile import tempfile
import unittest import unittest
from pathlib import Path from pathlib import Path
from typing import Any
from unittest.mock import patch from unittest.mock import patch
from bot_bottle.backend.firecracker import firecracker_vm, freezer, netpool, util from bot_bottle.backend.firecracker import firecracker_vm, freezer, netpool, util
@@ -131,172 +128,5 @@ class TestRequireFirecracker(unittest.TestCase):
util.require_firecracker() util.require_firecracker()
class TestBuildCommittedRootfsDir(unittest.TestCase):
"""Resume prepares the base rootfs dir from the freezer's snapshot tar
with no Docker: extract, recreate the excluded mount points, inject the
guest boot bits."""
def _make_tar(self, tmp: Path) -> Path:
import tarfile
src = tmp / "src"
(src / "home" / "node").mkdir(parents=True)
(src / "home" / "node" / "hello").write_text("hi")
tar_path = tmp / "rootfs.tar"
with tarfile.open(tar_path, "w") as tar:
tar.add(src, arcname=".")
return tar_path
def test_extracts_recreates_mountpoints_and_injects_boot(self):
with tempfile.TemporaryDirectory(prefix="fc-committed.") as d:
tmp = Path(d)
tar_path = self._make_tar(tmp)
# Stand in for the static dropbear that inject_guest_boot copies.
dropbear = tmp / "dropbear"
dropbear.write_text("#!/bin/true\n")
cache = tmp / "cache"
cache.mkdir()
with patch.object(util, "cache_dir", return_value=cache), \
patch.object(util, "dropbear_path", return_value=dropbear), \
patch.object(util, "info"):
base = util.build_committed_rootfs_dir(tar_path)
self.assertEqual("hi", (base / "home" / "node" / "hello").read_text())
for mount_point in ("proc", "sys", "dev", "run"):
self.assertTrue((base / mount_point).is_dir(),
f"missing recreated mount point /{mount_point}")
self.assertTrue((base / "bb-dropbear").is_file())
self.assertTrue((base / "bb-init").is_file())
self.assertTrue((base / ".bb-ready").is_file())
def test_caches_on_repeat_and_reextracts_after_refreeze(self):
with tempfile.TemporaryDirectory(prefix="fc-committed.") as d:
tmp = Path(d)
tar_path = self._make_tar(tmp)
dropbear = tmp / "dropbear"
dropbear.write_text("#!/bin/true\n")
cache = tmp / "cache"
cache.mkdir()
ctx = [
patch.object(util, "cache_dir", return_value=cache),
patch.object(util, "dropbear_path", return_value=dropbear),
patch.object(util, "info"),
]
for c in ctx:
c.start()
self.addCleanup(lambda: [c.stop() for c in ctx])
real_run = subprocess.run
calls = {"n": 0}
def counting_run(argv: list[str], *a: Any, **k: Any) -> Any:
if argv and argv[0] == "tar":
calls["n"] += 1
return real_run(argv, *a, **k)
with patch.object(util.subprocess, "run", side_effect=counting_run):
first = util.build_committed_rootfs_dir(tar_path)
second = util.build_committed_rootfs_dir(tar_path)
self.assertEqual(first, second)
self.assertEqual(1, calls["n"]) # cached — no re-extract
# A re-freeze rewrites the tar; a new size/mtime -> new cache
# key -> re-extract. Force a distinct mtime so the test isn't
# at the mercy of filesystem timestamp granularity.
import tarfile
extra = tmp / "extra"
extra.mkdir()
(extra / "note").write_text("v2")
with tarfile.open(tar_path, "w") as tar:
tar.add(extra, arcname=".")
st = tar_path.stat()
os.utime(tar_path, ns=(st.st_atime_ns, st.st_mtime_ns + 1_000_000_000))
third = util.build_committed_rootfs_dir(tar_path)
self.assertNotEqual(first, third)
self.assertEqual(2, calls["n"])
class TestInjectGuestBootSymlinkSafe(unittest.TestCase):
"""A committed snapshot is guest-controlled: inject_guest_boot must not
follow a planted symlink and overwrite a host file during resume."""
def test_planted_symlink_does_not_escape_staging_tree(self):
with tempfile.TemporaryDirectory(prefix="fc-inject.") as d:
tmp = Path(d)
dropbear = tmp / "dropbear"
dropbear.write_bytes(b"DROPBEAR")
# A host file the malicious snapshot tries to clobber.
victim = tmp / "victim"
victim.write_text("original")
rootfs = tmp / "rootfs"
rootfs.mkdir()
# The snapshot planted bb-init/bb-dropbear as symlinks to it.
(rootfs / "bb-init").symlink_to(victim)
(rootfs / "bb-dropbear").symlink_to(victim)
with patch.object(util, "dropbear_path", return_value=dropbear):
util.inject_guest_boot(rootfs, init_script="#!/bin/sh\nreal\n")
# Host file untouched; the staged paths are fresh regular files.
self.assertEqual("original", victim.read_text())
self.assertFalse((rootfs / "bb-init").is_symlink())
self.assertFalse((rootfs / "bb-dropbear").is_symlink())
self.assertEqual("#!/bin/sh\nreal\n", (rootfs / "bb-init").read_text())
self.assertEqual(b"DROPBEAR", (rootfs / "bb-dropbear").read_bytes())
class TestCommitRootfsPermissions(unittest.TestCase):
"""The snapshot tar can hold the bottle's private workspace, so the
freezer must write it owner-only (0600)."""
def _commit(self, tar_path: Path) -> int:
"""Run _commit_rootfs_via_ssh with a stubbed ssh|tar pipe; return the
mode of the open partial observed mid-stream (from subprocess.run)."""
key = tar_path.parent.parent / "key"
key.write_text("K")
seen: dict[str, int] = {}
def fake_run(argv: list[str], *a: Any, **k: Any) -> Any:
out = k["stdout"]
seen["mode"] = stat.S_IMODE(os.fstat(out.fileno()).st_mode)
out.write(b"TARDATA")
return subprocess.CompletedProcess(argv, 0, b"", b"")
with patch.object(freezer.util, "ssh_base_argv", return_value=["ssh"]), \
patch.object(freezer.subprocess, "run", side_effect=fake_run):
freezer._commit_rootfs_via_ssh(key, "10.0.0.1", tar_path)
return seen["mode"]
def test_snapshot_created_owner_only(self):
with tempfile.TemporaryDirectory(prefix="fc-freeze.") as d:
tar_path = Path(d) / "state" / "committed-rootfs.tar"
tar_path.parent.mkdir()
stream_mode = self._commit(tar_path)
self.assertEqual(0o600, stream_mode) # private during the stream
self.assertEqual(b"TARDATA", tar_path.read_bytes())
self.assertEqual(0o600, stat.S_IMODE(tar_path.stat().st_mode))
def test_leftover_world_readable_partial_is_recreated_private(self):
"""A partial left 0644 by an interrupted prior run must not keep the
new snapshot world-readable while it streams."""
with tempfile.TemporaryDirectory(prefix="fc-freeze.") as d:
tar_path = Path(d) / "state" / "committed-rootfs.tar"
tar_path.parent.mkdir()
partial = tar_path.with_name(tar_path.name + ".partial")
partial.write_bytes(b"stale")
os.chmod(partial, 0o644)
stream_mode = self._commit(tar_path)
self.assertEqual(0o600, stream_mode)
self.assertEqual(b"TARDATA", tar_path.read_bytes())
self.assertEqual(0o600, stat.S_IMODE(tar_path.stat().st_mode))
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+2 -4
View File
@@ -6,7 +6,6 @@ Covers the pure `git_gate_render_gitconfig` renderer and the dynamic
from __future__ import annotations from __future__ import annotations
import socket
import tempfile import tempfile
import types import types
import unittest import unittest
@@ -127,9 +126,8 @@ class TestProvisionDynamicKey(unittest.TestCase):
self.assertEqual(b"PRIVATE-KEY-BYTES", key_file.read_bytes()) self.assertEqual(b"PRIVATE-KEY-BYTES", key_file.read_bytes())
id_file = Path(d) / "repo-deploy-key-id" id_file = Path(d) / "repo-deploy-key-id"
self.assertEqual("kid123", id_file.read_text()) self.assertEqual("kid123", id_file.read_text())
# owner_repo had .git stripped; title carries globalize_slug(slug) + name # owner_repo had .git stripped; title carries slug + name
hostname = socket.gethostname() self.assertEqual([("o/r", "bot-bottle:myslug:repo")], fake.created)
self.assertEqual([("o/r", f"bot-bottle:{hostname}-myslug:repo")], fake.created)
def test_missing_token_raises(self) -> None: def test_missing_token_raises(self) -> None:
with tempfile.TemporaryDirectory() as d, \ with tempfile.TemporaryDirectory() as d, \
+1 -39
View File
@@ -79,44 +79,6 @@ class TestVersion(unittest.TestCase):
ia.infra_artifact_version("a"), ia.infra_artifact_version("b")) ia.infra_artifact_version("a"), ia.infra_artifact_version("b"))
class TestVersionInputs(unittest.TestCase):
"""The hash must cover *every* file baked into the rootfs, not just `*.py`
(`COPY bot_bottle` is wholesale) else a non-Python change (e.g. the egress
entrypoint shell script) leaves the version unchanged and a launch host
boots a rootfs whose code differs from its checkout."""
def _fake_repo(self, root: Path) -> None:
pkg = root / "bot_bottle"
pkg.mkdir()
(pkg / "app.py").write_text("print('hi')\n")
(pkg / "egress_entrypoint.sh").write_text("#!/bin/sh\nexec mitmdump\n")
(pkg / "netpool.defaults.env").write_text("FOO=1\n")
for name in ("Dockerfile.orchestrator", "Dockerfile.gateway", "Dockerfile.infra"):
(root / name).write_text(f"FROM scratch # {name}\n")
def test_non_python_file_change_bumps_version(self) -> None:
with tempfile.TemporaryDirectory() as d:
root = Path(d)
self._fake_repo(root)
before = ia.infra_artifact_version("init", repo_root=root)
(root / "bot_bottle" / "egress_entrypoint.sh").write_text(
"#!/bin/sh\nexec mitmdump --different\n")
after = ia.infra_artifact_version("init", repo_root=root)
self.assertNotEqual(before, after)
def test_pyc_and_pycache_ignored(self) -> None:
with tempfile.TemporaryDirectory() as d:
root = Path(d)
self._fake_repo(root)
before = ia.infra_artifact_version("init", repo_root=root)
cache = root / "bot_bottle" / "__pycache__"
cache.mkdir()
(cache / "app.cpython-312.pyc").write_bytes(b"\x00bytecode")
(root / "bot_bottle" / "app.pyc").write_bytes(b"\x00bytecode")
after = ia.infra_artifact_version("init", repo_root=root)
self.assertEqual(before, after)
class TestEnsureArtifact(_CacheMixin): class TestEnsureArtifact(_CacheMixin):
def test_downloads_verifies_and_caches(self) -> None: def test_downloads_verifies_and_caches(self) -> None:
version = "deadbeef00000000" version = "deadbeef00000000"
@@ -169,7 +131,7 @@ class TestConfig(unittest.TestCase):
url = ia.artifact_url("v1", "rootfs.ext4.gz") url = ia.artifact_url("v1", "rootfs.ext4.gz")
self.assertEqual( self.assertEqual(
"https://mirror.example/api/packages/acme/generic/" "https://mirror.example/api/packages/acme/generic/"
"bot-bottle-firecracker-infra/v1/rootfs.ext4.gz", url) "bot-bottle-infra/v1/rootfs.ext4.gz", url)
def test_local_build_flag(self) -> None: def test_local_build_flag(self) -> None:
with mock.patch.dict(os.environ, {"BOT_BOTTLE_INFRA_BUILD": "local"}): with mock.patch.dict(os.environ, {"BOT_BOTTLE_INFRA_BUILD": "local"}):
@@ -1,136 +0,0 @@
"""Unit: macOS consolidated launch — register-after-start (PRD 0070)."""
from __future__ import annotations
import unittest
from pathlib import Path
from unittest.mock import MagicMock, Mock, patch
from bot_bottle.backend.macos_container.consolidated_launch import (
GatewayEndpoint,
ensure_gateway,
register_agent,
teardown_consolidated,
)
from bot_bottle.egress import EgressPlan, EgressRoute
from bot_bottle.git_gate import GitGatePlan
from bot_bottle.orchestrator.client import RegisteredBottle
_MOD = "bot_bottle.backend.macos_container.consolidated_launch"
def _egress_plan() -> EgressPlan:
return EgressPlan(
slug="demo", routes_path=Path("/x"),
routes=(EgressRoute(host="api.example.com"),), token_env_map={},
)
def _git_plan() -> GitGatePlan:
return GitGatePlan(
slug="demo", entrypoint_script=Path(), hook_script=Path(),
access_hook_script=Path(), upstreams=(),
)
def _endpoint() -> GatewayEndpoint:
return GatewayEndpoint(
orchestrator_url="http://192.168.128.2:8099",
gateway_ip="192.168.128.3",
gateway_ca_pem="-----BEGIN CERTIFICATE-----\n",
network="bot-bottle-mac-gateway",
)
def _client() -> Mock:
c = Mock()
c.register_bottle.return_value = RegisteredBottle("b1", "tok")
return c
class TestEnsureGateway(unittest.TestCase):
def _run(self, service: MagicMock) -> GatewayEndpoint:
with patch(f"{_MOD}.MacosInfraService", return_value=service):
return ensure_gateway()
def _service(self) -> MagicMock:
from bot_bottle.backend.macos_container.infra import InfraEndpoint
service = MagicMock()
service.ensure_running.return_value = InfraEndpoint(
control_plane_url="http://192.168.128.2:8099",
gateway_ip="192.168.128.2",
)
service.network = "bot-bottle-mac-gateway"
service.ca_cert_pem.return_value = "PEM"
return service
def test_reports_gateway_endpoint(self) -> None:
endpoint = self._run(self._service())
self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url)
self.assertEqual("192.168.128.2", endpoint.gateway_ip)
self.assertEqual("PEM", endpoint.gateway_ca_pem)
self.assertEqual("bot-bottle-mac-gateway", endpoint.network)
def test_control_plane_and_gateway_share_one_address(self) -> None:
"""One infra container hosts both, so the gateway IP and the
control-plane host are the same."""
endpoint = self._run(self._service())
self.assertEqual(
endpoint.gateway_ip,
endpoint.orchestrator_url.split("://")[1].split(":")[0],
)
class TestRegisterAgent(unittest.TestCase):
def _run(
self, client: Mock, provision: Mock | None = None,
*, source_ip: str = "192.168.128.9",
):
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
patch(f"{_MOD}.provision_git_gate", provision or Mock()):
return register_agent(
_egress_plan(), _git_plan(),
source_ip=source_ip, endpoint=_endpoint(), image_ref="img:1",
)
def test_registers_by_the_address_read_from_the_live_container(self) -> None:
"""The attribution key is the DHCP-assigned address the caller read
back there is no --ip to pin it up front."""
client = _client()
ctx = self._run(client, source_ip="192.168.128.9")
self.assertEqual("192.168.128.9", ctx.source_ip)
self.assertEqual("192.168.128.9", client.register_bottle.call_args.args[0])
def test_returns_identity_token_and_bottle_id(self) -> None:
ctx = self._run(_client())
self.assertEqual("b1", ctx.bottle_id)
self.assertEqual("tok", ctx.identity_token)
def test_provisions_git_gate_for_the_registered_bottle(self) -> None:
provision = Mock()
self._run(_client(), provision)
self.assertEqual("b1", provision.call_args.args[1])
def test_rolls_registration_back_when_provisioning_fails(self) -> None:
"""A provisioning failure must not leave an orphan registration
holding the source IP."""
client = _client()
provision = Mock(side_effect=RuntimeError("boom"))
with self.assertRaises(RuntimeError):
self._run(client, provision)
client.teardown_bottle.assert_called_once_with("b1")
class TestTeardown(unittest.TestCase):
def test_deregisters_and_deprovisions(self) -> None:
client = Mock()
deprovision = Mock()
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
patch(f"{_MOD}.deprovision_git_gate", deprovision):
teardown_consolidated("b1", orchestrator_url="http://o:8099")
client.teardown_bottle.assert_called_once_with("b1")
self.assertEqual("b1", deprovision.call_args.args[1])
if __name__ == "__main__":
unittest.main()
+4 -25
View File
@@ -43,31 +43,10 @@ class TestMacosContainerCleanup(unittest.TestCase):
class TestMacosContainerEnumerate(unittest.TestCase): class TestMacosContainerEnumerate(unittest.TestCase):
"""The backend launches bottles again (PRD 0070), so enumeration is real def test_enumerate_active_is_empty_while_disabled(self):
rather than the disabled-era stub. These must not shell out: `container` # The macOS backend is disabled during the companion-container removal cleanup
does not exist on the Linux CI host.""" # (#385); it launches nothing, so there is nothing to enumerate.
self.assertEqual([], enum_mod.enumerate_active())
def _enumerate(self, stdout: str, returncode: int = 0):
completed = enum_mod.subprocess.CompletedProcess(
args=[], returncode=returncode, stdout=stdout, stderr="",
)
with patch.object(enum_mod.subprocess, "run", return_value=completed), \
patch.object(enum_mod, "read_metadata", return_value=None):
return enum_mod.enumerate_active()
def test_lists_agent_containers_by_slug(self):
agents = self._enumerate("bot-bottle-dev-abc\nunrelated\n")
self.assertEqual(["dev-abc"], [a.slug for a in agents])
self.assertEqual(["macos-container"], [a.backend_name for a in agents])
def test_excludes_the_infra_singleton(self):
"""The infra container shares the bot-bottle- prefix but is
infrastructure listing it would invent an agent per host."""
agents = self._enumerate("bot-bottle-mac-infra\nbot-bottle-dev-abc\n")
self.assertEqual(["dev-abc"], [a.slug for a in agents])
def test_empty_when_the_cli_fails(self):
self.assertEqual([], self._enumerate("", returncode=1))
if __name__ == "__main__": if __name__ == "__main__":
@@ -1,210 +0,0 @@
"""Unit: macOS agent run wiring — the attribution invariant + token delivery
(PRD 0070).
Replaces the argv coverage from the per-bottle companion-container era
(`test_macos_container_launch.py`, removed with that architecture in #385).
"""
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from typing import cast
from unittest.mock import patch
from bot_bottle.backend.macos_container.bottle import MacosContainerBottle
from bot_bottle.backend.macos_container.bottle_plan import MacosContainerBottlePlan
from bot_bottle.backend.macos_container.consolidated_launch import GatewayEndpoint
from bot_bottle.backend.macos_container.launch import (
_agent_run_argv,
_identity_proxy_env,
_proxy_url,
)
from bot_bottle.manifest import ManifestIndex
_BOTTLE = "bot_bottle.backend.macos_container.bottle"
_MANIFEST = ManifestIndex.from_json_obj({
"bottles": {"dev": {}},
"agents": {"demo": {"skills": [], "prompt": "", "bottle": "dev"}},
}).load_for_agent("demo")
def _endpoint() -> GatewayEndpoint:
return GatewayEndpoint(
orchestrator_url="http://192.168.128.2:8099",
gateway_ip="192.168.128.3",
gateway_ca_pem="PEM",
network="bot-bottle-mac-gateway",
)
def _plan(
stage_dir: Path,
*,
agent_git_gate_url: str = "",
agent_supervise_url: str = "",
) -> MacosContainerBottlePlan:
routes_path = stage_dir / "routes.yaml"
routes_path.write_text("routes: []\n", encoding="utf-8")
ca_path = stage_dir / "gateway-ca.pem"
ca_path.write_text("ca\n", encoding="utf-8")
egress_plan = SimpleNamespace(
mitmproxy_ca_host_path=ca_path,
routes_path=routes_path,
routes=("route",),
token_env_map={"EGRESS_TOKEN_0": "HOST_TOKEN"},
canary="",
canary_env="",
)
return cast(MacosContainerBottlePlan, SimpleNamespace(
spec=SimpleNamespace(),
manifest=_MANIFEST,
stage_dir=stage_dir,
slug="dev-abc",
container_name="bot-bottle-dev-abc",
image="bot-bottle-agent:latest",
forwarded_env={"OAUTH_TOKEN": "host-value"},
egress_plan=egress_plan,
git_gate_plan=SimpleNamespace(upstreams=()),
supervise_plan=None,
agent_provision=SimpleNamespace(
guest_env={"LITERAL": "value"},
provisioned_env={},
),
agent_git_gate_url=agent_git_gate_url,
agent_supervise_url=agent_supervise_url,
))
class TestAgentRunArgv(unittest.TestCase):
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.argv = _agent_run_argv(_plan(Path(self._tmp.name)), _endpoint())
def tearDown(self) -> None:
self._tmp.cleanup()
def test_drops_net_raw(self) -> None:
"""The attribution invariant: Apple grants CAP_NET_RAW by default,
which would let an agent forge a neighbour's source address with a raw
socket and be attributed as that bottle."""
self.assertIn("--cap-drop", self.argv)
self.assertEqual("CAP_NET_RAW", self.argv[self.argv.index("--cap-drop") + 1])
def test_attaches_to_the_shared_gateway_network(self) -> None:
self.assertEqual(
"bot-bottle-mac-gateway", self.argv[self.argv.index("--network") + 1],
)
def test_never_pins_an_ip(self) -> None:
"""Apple Container 1.0.0 has no --ip: the address is DHCP-assigned and
read back after start."""
self.assertNotIn("--ip", self.argv)
def test_run_time_proxy_carries_no_identity_token(self) -> None:
"""The token is minted by registration, which happens after this run —
so it cannot be here. `/resolve` denies the token-less pair (#366),
which is the safe direction; the real value arrives at exec time."""
joined = " ".join(self.argv)
self.assertIn(f"HTTP_PROXY={_proxy_url('192.168.128.3')}", joined)
self.assertNotIn("bottle:", joined)
def test_gateway_bypasses_the_proxy(self) -> None:
"""git-http + supervise live on the gateway and must be reached
directly, not through its own egress proxy."""
entry = next(a for a in self.argv if a.startswith("NO_PROXY="))
self.assertIn("192.168.128.3", entry)
def test_forwarded_secrets_stay_off_argv(self) -> None:
"""Bare name → inherited from the run process env, so the value never
lands on the command line."""
self.assertIn("OAUTH_TOKEN", self.argv)
self.assertNotIn("host-value", " ".join(self.argv))
def test_agent_init_is_a_no_op(self) -> None:
"""Every agent command arrives via `container exec`; the init process
just holds the container open."""
self.assertEqual("sleep", self.argv[-2])
class TestIdentityTokenDelivery(unittest.TestCase):
def test_exec_env_carries_the_token_as_proxy_credentials(self) -> None:
env = _identity_proxy_env(_endpoint(), "s3cret")
self.assertEqual(
"http://bottle:s3cret@192.168.128.3:9099", env["HTTP_PROXY"],
)
self.assertEqual(env["HTTP_PROXY"], env["https_proxy"])
def test_no_token_means_no_override(self) -> None:
self.assertEqual({}, _identity_proxy_env(_endpoint(), ""))
def test_token_value_never_reaches_argv(self) -> None:
"""`ps` is world-readable: the token rides the child env behind a bare
`--env` name, never the command line."""
bottle = MacosContainerBottle(
"bot-bottle-demo", lambda: None, None,
exec_env=_identity_proxy_env(_endpoint(), "s3cret"),
)
argv = bottle.agent_argv(["--help"], tty=False)
self.assertNotIn("s3cret", " ".join(argv))
self.assertIn("HTTP_PROXY", argv)
self.assertEqual("--env", argv[argv.index("HTTP_PROXY") - 1])
def test_bottle_without_exec_env_is_unchanged(self) -> None:
bottle = MacosContainerBottle("bot-bottle-demo", lambda: None, None)
argv = bottle.agent_argv(["--help"], tty=False)
self.assertNotIn("--env", argv)
class TestPlanIdentityToken(unittest.TestCase):
"""git-gate's gitconfig extraHeader and the supervise MCP --header read
`getattr(plan, "identity_token", "")` at provision time and both bypass the
egress proxy (NO_PROXY), so the exec-time proxy token never reaches them
the plan must carry the token or /resolve fail-closes and git + supervise
are denied on macOS."""
def test_macos_plan_has_the_identity_token_field(self) -> None:
from dataclasses import fields
from bot_bottle.backend.macos_container.bottle_plan import (
MacosContainerBottlePlan,
)
names = {f.name for f in fields(MacosContainerBottlePlan)}
self.assertIn("identity_token", names)
def test_matches_the_docker_plan(self) -> None:
"""Both consolidated backends must expose identity_token so a shared
provision-time consumer degrades to neither backend silently."""
from dataclasses import fields
from bot_bottle.backend.docker.bottle_plan import DockerBottlePlan
from bot_bottle.backend.macos_container.bottle_plan import (
MacosContainerBottlePlan,
)
docker = {f.name for f in fields(DockerBottlePlan)}
macos = {f.name for f in fields(MacosContainerBottlePlan)}
self.assertIn("identity_token", docker & macos)
def test_provisioning_exec_also_carries_the_token(self) -> None:
"""`provision` runs through `exec`; a provider whose provision step
fetches anything would otherwise egress token-less and be denied."""
bottle = MacosContainerBottle(
"bot-bottle-demo", lambda: None, None,
exec_env=_identity_proxy_env(_endpoint(), "s3cret"),
)
with patch(f"{_BOTTLE}.subprocess.run") as run:
run.return_value = SimpleNamespace(returncode=0, stdout="", stderr="")
bottle.exec("echo hi")
argv, kwargs = run.call_args.args[0], run.call_args.kwargs
self.assertIn("HTTP_PROXY", argv)
self.assertNotIn("s3cret", " ".join(argv))
self.assertEqual(
"http://bottle:s3cret@192.168.128.3:9099", kwargs["env"]["HTTP_PROXY"],
)
if __name__ == "__main__":
unittest.main()
-50
View File
@@ -273,55 +273,5 @@ resolver #2
) )
def _completed(stdout: str, returncode: int = 0):
return util.subprocess.CompletedProcess(args=[], returncode=returncode, stdout=stdout, stderr="")
class TestInspectDigests(unittest.TestCase):
"""image_digest and container_image_digest must read the SAME field shape
(a `descriptor.digest`) so a running container and its image are
comparable. An asymmetry recreates the gateway on every launch."""
_IMAGE = '{"configuration": {"descriptor": {"digest": "sha256:abc123"}}, "id": "abc123"}'
_CONTAINER = '[{"configuration": {"image": {"descriptor": {"digest": "sha256:abc123"}}}}]'
def test_image_and_container_digests_agree_for_the_same_image(self):
with patch.object(util, "run_container_argv") as run:
run.return_value = _completed(self._IMAGE)
img = util.image_digest("bot-bottle-gateway:latest")
run.return_value = _completed(self._CONTAINER)
ctr = util.container_image_digest("bot-bottle-mac-gateway")
self.assertEqual("abc123", img)
self.assertEqual(img, ctr)
def test_image_digest_never_falls_back_to_a_tag_or_id(self):
"""The old `id` fallback could yield a value container_image_digest
can't produce, permanently mismatching. Missing descriptor → '' (don't
churn), never a stray id/tag."""
with patch.object(util, "run_container_argv") as run:
run.return_value = _completed('{"id": "bot-bottle-gateway:latest"}')
self.assertEqual("", util.image_digest("bot-bottle-gateway:latest"))
def test_unreadable_inspect_returns_empty(self):
with patch.object(util, "run_container_argv") as run:
run.return_value = _completed("", returncode=1)
self.assertEqual("", util.image_digest("x"))
self.assertEqual("", util.container_image_digest("x"))
self.assertEqual({}, util.container_env("x"))
class TestWaitContainerIpv4(unittest.TestCase):
def test_returns_address_once_dhcp_assigns_it(self):
with patch.object(util, "try_container_ipv4_on_network", side_effect=["", "", "192.168.128.4"]), \
patch.object(util.time, "sleep"):
ip = util.wait_container_ipv4_on_network("c", "net", timeout=5, poll=0)
self.assertEqual("192.168.128.4", ip)
def test_returns_empty_on_timeout(self):
with patch.object(util, "try_container_ipv4_on_network", return_value=""), \
patch.object(util.time, "sleep"):
self.assertEqual("", util.wait_container_ipv4_on_network("c", "net", timeout=-1))
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
-168
View File
@@ -1,168 +0,0 @@
"""Unit: the single macOS infra container (control plane + gateway, PRD 0070)."""
from __future__ import annotations
import unittest
from pathlib import Path
from unittest.mock import Mock, patch
from bot_bottle.backend.macos_container.infra import (
INFRA_DB_VOLUME,
MacosInfraService,
OrchestratorStartError,
probe_control_plane_url,
)
_INFRA = "bot_bottle.backend.macos_container.infra"
def _ok(stdout: str = "") -> Mock:
return Mock(returncode=0, stdout=stdout, stderr="")
def _fail(stderr: str = "boom") -> Mock:
return Mock(returncode=1, stdout="", stderr=stderr)
class TestInfraRun(unittest.TestCase):
def _run_container(self, svc: MacosInfraService) -> list[str]:
run = Mock(return_value=_ok())
def _spec(src: str, tgt: str, readonly: bool = False) -> str:
return f"type=bind,source={src},target={tgt}" + (
",readonly" if readonly else "")
with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.ensure_networks"):
mod.dns_server.return_value = "1.1.1.1"
mod.bind_mount_spec.side_effect = _spec
mod.run_container_argv = run
svc._run_container("h1")
return run.call_args.args[0]
def test_single_container_runs_both_processes(self) -> None:
"""The whole point: one container starts the control plane AND the
gateway daemons, so one kernel owns the DB."""
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
script = argv[-1]
self.assertIn("bot_bottle.orchestrator", script)
self.assertIn("gateway_init.py", script)
self.assertIn("127.0.0.1", script) # they reach each other on loopback
def test_db_is_a_container_only_volume(self) -> None:
"""No host bind-mount of the DB — a named volume only this container
mounts, so the DB is never written by two kernels."""
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
vols = [argv[i + 1] for i, a in enumerate(argv) if a == "--volume"]
self.assertTrue(any(v.startswith(f"{INFRA_DB_VOLUME}:") for v in vols))
# The repo source is bind-mounted read-only; the DB is not a bind mount.
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
self.assertTrue(all("bot-bottle.db" not in m for m in mounts))
def test_nat_network_precedes_the_host_only_network(self) -> None:
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], nets)
def test_source_hash_is_labelled_for_recreate(self) -> None:
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", argv)
def test_start_failure_raises(self) -> None:
svc = MacosInfraService(repo_root=Path("/r"))
with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.ensure_networks"):
mod.dns_server.return_value = "1.1.1.1"
mod.bind_mount_spec.return_value = "m"
mod.run_container_argv = Mock(return_value=_fail())
with self.assertRaises(OrchestratorStartError):
svc._run_container("h1")
class TestInfraEnsureRunning(unittest.TestCase):
def test_current_healthy_container_is_left_alone(self) -> None:
"""Idempotent singleton: N launches must not churn the infra container
and drop every live bottle's control plane."""
svc = MacosInfraService(repo_root=Path("/r"))
run = Mock()
with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.source_hash", return_value="h1"), \
patch.object(svc, "_run_container", run), \
patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = True
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
endpoint = svc.ensure_running()
run.assert_not_called()
self.assertEqual("http://192.168.128.2:8099", endpoint.control_plane_url)
self.assertEqual("192.168.128.2", endpoint.gateway_ip)
def test_changed_source_recreates(self) -> None:
svc = MacosInfraService(repo_root=Path("/r"))
run = Mock()
with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.source_hash", return_value="h2"), \
patch.object(svc, "ensure_built"), \
patch.object(svc, "_run_container", run), \
patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = True
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
svc.ensure_running()
run.assert_called_once()
def test_wedged_but_current_container_is_recreated(self) -> None:
"""Current source but a dead HTTP server must be recreated, not polled
to death forever health, not just the source label, gates reuse."""
svc = MacosInfraService(repo_root=Path("/r"))
run = Mock()
health = Mock(side_effect=[False, True])
with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.source_hash", return_value="h1"), \
patch.object(svc, "ensure_built"), \
patch.object(svc, "_run_container", run), \
patch.object(svc, "is_healthy", health):
mod.container_is_running.return_value = True
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
svc.ensure_running()
run.assert_called_once()
def test_never_healthy_raises(self) -> None:
svc = MacosInfraService(repo_root=Path("/r"))
with patch(f"{_INFRA}.container_mod") as mod, \
patch(f"{_INFRA}.source_hash", return_value="h1"), \
patch.object(svc, "ensure_built"), \
patch.object(svc, "_run_container"), \
patch.object(svc, "is_healthy", return_value=False):
mod.container_is_running.return_value = False
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
with self.assertRaises(OrchestratorStartError):
svc.ensure_running(startup_timeout=0.01)
class TestCaCertPem(unittest.TestCase):
def test_reads_ca_out_of_the_container(self) -> None:
svc = MacosInfraService(repo_root=Path("/r"))
with patch(f"{_INFRA}.container_mod") as mod:
mod.run_container_argv.return_value = _ok("-----BEGIN CERTIFICATE-----\n")
pem = svc.ca_cert_pem()
self.assertTrue(pem.startswith("-----BEGIN CERTIFICATE-----"))
argv = mod.run_container_argv.call_args.args[0]
self.assertEqual(["container", "exec", "bot-bottle-mac-infra", "cat"], argv[:4])
class TestProbeControlPlane(unittest.TestCase):
def test_returns_url_when_running(self) -> None:
with patch(f"{_INFRA}.container_mod") as mod:
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
self.assertEqual("http://192.168.128.2:8099", probe_control_plane_url())
def test_empty_when_absent(self) -> None:
with patch(f"{_INFRA}.container_mod") as mod:
mod.try_container_ipv4_on_network.return_value = ""
self.assertEqual("", probe_control_plane_url())
if __name__ == "__main__":
unittest.main()
@@ -11,7 +11,6 @@ import secrets
import tempfile import tempfile
import threading import threading
import unittest import unittest
import urllib.error
import urllib.request import urllib.request
from pathlib import Path from pathlib import Path
from unittest.mock import patch from unittest.mock import patch
@@ -244,82 +243,6 @@ class TestServerRoundTrip(unittest.TestCase):
self.assertEqual(reg["bottle_id"], attr["bottle_id"]) self.assertEqual(reg["bottle_id"], attr["bottle_id"])
class TestControlPlaneAuth(unittest.TestCase):
"""The per-host control-plane secret (issue #400): every route but /health
is a trusted-caller op an agent must not be able to drive just because it
can reach the port."""
def setUp(self) -> None:
self._tmp = tempfile.TemporaryDirectory()
self.addCleanup(self._tmp.cleanup)
self.orch = _orchestrator(Path(self._tmp.name) / "r.db")
def test_health_is_public_even_unauthorized(self) -> None:
status, _ = dispatch(self.orch, "GET", "/health", b"", authorized=False)
self.assertEqual(200, status)
def test_unauthorized_denies_every_other_route(self) -> None:
for method, path, body in [
("GET", "/bottles", b""),
("POST", "/bottles", _body({"source_ip": "10.0.0.1"})),
("PUT", "/bottles/x/policy", _body({"policy": "routes: []"})),
("DELETE", "/bottles/x", b""),
("POST", "/resolve", _body({"source_ip": "10.0.0.1", "identity_token": "t"})),
("POST", "/attribute", _body({"source_ip": "10.0.0.1", "identity_token": "t"})),
("GET", "/supervise/proposals", b""),
("POST", "/supervise/respond", _body({"proposal_id": "p", "bottle_slug": "s", "decision": "approve"})),
]:
status, _ = dispatch(self.orch, method, path, body, authorized=False)
self.assertEqual(401, status, f"{method} {path} should be 401 unauthorized")
def test_deny_happens_before_the_registry_is_touched(self) -> None:
"""An unauthorized DELETE must not tear a bottle down. 401, and the
bottle is still there."""
rec = self.orch.registry.register("10.0.0.9", policy="", metadata="")
status, _ = dispatch(
self.orch, "DELETE", f"/bottles/{rec.bottle_id}", b"", authorized=False)
self.assertEqual(401, status)
self.assertIsNotNone(self.orch.registry.get(rec.bottle_id))
def _server_with_secret(self, secret: str):
with patch.dict("os.environ", {"BOT_BOTTLE_CONTROL_PLANE_TOKEN": secret}):
server = make_server(self.orch, "127.0.0.1", 0)
self.addCleanup(server.server_close)
threading.Thread(target=server.serve_forever, daemon=True).start()
self.addCleanup(server.shutdown)
host, port = server.server_address[0], server.server_address[1]
return f"http://{host}:{port}"
def _status(self, url: str, *, header: str | None = None) -> int:
req = urllib.request.Request(url)
if header is not None:
req.add_header("x-bot-bottle-control-auth", header)
try:
return urllib.request.urlopen(req, timeout=5).status
except urllib.error.HTTPError as e:
return e.code
def test_configured_server_enforces_the_header_over_http(self) -> None:
base = self._server_with_secret("s3cret-admin")
# /health is public — no header needed.
self.assertEqual(200, self._status(f"{base}/health"))
# /bottles requires the secret.
self.assertEqual(401, self._status(f"{base}/bottles"))
self.assertEqual(401, self._status(f"{base}/bottles", header="wrong"))
self.assertEqual(200, self._status(f"{base}/bottles", header="s3cret-admin"))
def test_unconfigured_server_runs_open(self) -> None:
"""No secret set (tests / nft-protected Firecracker): open mode, so the
existing round-trip and unit behavior are unchanged."""
with patch.dict("os.environ", {}, clear=False):
import os
os.environ.pop("BOT_BOTTLE_CONTROL_PLANE_TOKEN", None)
server = make_server(self.orch, "127.0.0.1", 0)
self.addCleanup(server.server_close)
self.assertTrue(server.is_authorized(""))
self.assertTrue(server.is_authorized("anything"))
class TestDispatchSupervise(unittest.TestCase): class TestDispatchSupervise(unittest.TestCase):
"""The /supervise/* routes over the pure dispatch().""" """The /supervise/* routes over the pure dispatch()."""
+8 -8
View File
@@ -38,7 +38,7 @@ class TestDockerGateway(unittest.TestCase):
def test_ensure_running_noop_when_up_and_image_current(self) -> None: def test_ensure_running_noop_when_up_and_image_current(self) -> None:
calls: list[list[str]] = [] calls: list[list[str]] = []
def fake(argv: list[str], **_kw: object) -> Mock: def fake(argv: list[str]) -> Mock:
calls.append(argv) calls.append(argv)
if argv[:2] == ["docker", "ps"]: if argv[:2] == ["docker", "ps"]:
return _proc(stdout=self.sc.name) # running return _proc(stdout=self.sc.name) # running
@@ -58,7 +58,7 @@ class TestDockerGateway(unittest.TestCase):
# a rebuild's new flat daemons take effect. # a rebuild's new flat daemons take effect.
calls: list[list[str]] = [] calls: list[list[str]] = []
def fake(argv: list[str], **_kw: object) -> Mock: def fake(argv: list[str]) -> Mock:
calls.append(argv) calls.append(argv)
if argv[:2] == ["docker", "ps"]: if argv[:2] == ["docker", "ps"]:
return _proc(stdout=self.sc.name) return _proc(stdout=self.sc.name)
@@ -76,7 +76,7 @@ class TestDockerGateway(unittest.TestCase):
def test_ensure_running_starts_the_singleton_when_absent(self) -> None: def test_ensure_running_starts_the_singleton_when_absent(self) -> None:
calls: list[list[str]] = [] calls: list[list[str]] = []
def fake(argv: list[str], **_kw: object) -> Mock: def fake(argv: list[str]) -> Mock:
calls.append(argv) calls.append(argv)
return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc() return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc()
@@ -102,7 +102,7 @@ class TestDockerGateway(unittest.TestCase):
def test_ensure_running_creates_network_when_missing(self) -> None: def test_ensure_running_creates_network_when_missing(self) -> None:
calls: list[list[str]] = [] calls: list[list[str]] = []
def fake(argv: list[str], **_kw: object) -> Mock: def fake(argv: list[str]) -> Mock:
calls.append(argv) calls.append(argv)
if argv[:3] == ["docker", "network", "inspect"]: if argv[:3] == ["docker", "network", "inspect"]:
return _proc(returncode=1, stderr="No such network") return _proc(returncode=1, stderr="No such network")
@@ -135,7 +135,7 @@ class TestDockerGateway(unittest.TestCase):
def test_ensure_running_reuses_existing_network(self) -> None: def test_ensure_running_reuses_existing_network(self) -> None:
calls: list[list[str]] = [] calls: list[list[str]] = []
def fake(argv: list[str], **_kw: object) -> Mock: def fake(argv: list[str]) -> Mock:
calls.append(argv) calls.append(argv)
return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc() return _proc(stdout="") if argv[:2] == ["docker", "ps"] else _proc()
@@ -144,7 +144,7 @@ class TestDockerGateway(unittest.TestCase):
self.assertEqual([], [c for c in calls if c[:3] == ["docker", "network", "create"]]) self.assertEqual([], [c for c in calls if c[:3] == ["docker", "network", "create"]])
def test_ensure_running_raises_on_docker_failure(self) -> None: def test_ensure_running_raises_on_docker_failure(self) -> None:
def fake(argv: list[str], **_kw: object) -> Mock: def fake(argv: list[str]) -> Mock:
if argv[:2] == ["docker", "ps"]: if argv[:2] == ["docker", "ps"]:
return _proc(stdout="") return _proc(stdout="")
if argv[:2] == ["docker", "run"]: if argv[:2] == ["docker", "run"]:
@@ -181,7 +181,7 @@ class TestDockerGatewayBuild(unittest.TestCase):
# build-if-missing silently ran a stale single-tenant image. # build-if-missing silently ran a stale single-tenant image.
calls: list[list[str]] = [] calls: list[list[str]] = []
def rec(argv: list[str], **_kw: object) -> Mock: def rec(argv: list[str]) -> Mock:
calls.append(argv) calls.append(argv)
return _proc() # image present, build succeeds return _proc() # image present, build succeeds
@@ -196,7 +196,7 @@ class TestDockerGatewayBuild(unittest.TestCase):
def test_ensure_built_no_cache_env_forces_full_rebuild(self) -> None: def test_ensure_built_no_cache_env_forces_full_rebuild(self) -> None:
calls: list[list[str]] = [] calls: list[list[str]] = []
def rec(argv: list[str], **_kw: object) -> Mock: def rec(argv: list[str]) -> Mock:
calls.append(argv) calls.append(argv)
return _proc() return _proc()
+8 -8
View File
@@ -14,7 +14,7 @@ from bot_bottle.orchestrator.lifecycle import (
ORCHESTRATOR_SOURCE_HASH_LABEL, ORCHESTRATOR_SOURCE_HASH_LABEL,
OrchestratorService, OrchestratorService,
OrchestratorStartError, OrchestratorStartError,
source_hash, _source_hash,
) )
from tests.unit import use_bottle_root from tests.unit import use_bottle_root
@@ -57,10 +57,10 @@ class TestOrchestratorService(unittest.TestCase):
# A healthy control plane already running the *current* bind-mounted # A healthy control plane already running the *current* bind-mounted
# source is left alone — recreating it on every launch would drop # source is left alone — recreating it on every launch would drop
# every other active bottle's in-memory egress tokens (#381). # every other active bottle's in-memory egress tokens (#381).
current = source_hash(self.svc._repo_root) current = _source_hash(self.svc._repo_root)
calls: list[list[str]] = [] calls: list[list[str]] = []
def fake(argv: list[str], **_kw: object) -> Mock: def fake(argv: list[str]) -> Mock:
calls.append(argv) calls.append(argv)
if argv[:2] == ["docker", "ps"]: if argv[:2] == ["docker", "ps"]:
return _proc(stdout=ORCHESTRATOR_NAME) return _proc(stdout=ORCHESTRATOR_NAME)
@@ -83,7 +83,7 @@ class TestOrchestratorService(unittest.TestCase):
# effect, same as the gateway's image-staleness check. # effect, same as the gateway's image-staleness check.
calls: list[list[str]] = [] calls: list[list[str]] = []
def fake(argv: list[str], **_kw: object) -> Mock: def fake(argv: list[str]) -> Mock:
calls.append(argv) calls.append(argv)
if argv[:2] == ["docker", "ps"]: if argv[:2] == ["docker", "ps"]:
return _proc(stdout=ORCHESTRATOR_NAME) return _proc(stdout=ORCHESTRATOR_NAME)
@@ -98,13 +98,13 @@ class TestOrchestratorService(unittest.TestCase):
self.assertEqual(1, len(runs)) self.assertEqual(1, len(runs))
self.assertIn(ORCHESTRATOR_NAME, runs[0]) self.assertIn(ORCHESTRATOR_NAME, runs[0])
# the fresh container is labeled with the current hash, not the stale one # the fresh container is labeled with the current hash, not the stale one
current = source_hash(self.svc._repo_root) current = _source_hash(self.svc._repo_root)
self.assertIn(f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current}", runs[0]) self.assertIn(f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current}", runs[0])
def test_ensure_running_starts_orchestrator_container_when_absent(self) -> None: def test_ensure_running_starts_orchestrator_container_when_absent(self) -> None:
calls: list[list[str]] = [] calls: list[list[str]] = []
def fake(argv: list[str], **_kw: object) -> Mock: def fake(argv: list[str]) -> Mock:
calls.append(argv) calls.append(argv)
if argv[:2] == ["docker", "ps"]: if argv[:2] == ["docker", "ps"]:
return _proc(stdout="") # not running return _proc(stdout="") # not running
@@ -127,7 +127,7 @@ class TestOrchestratorService(unittest.TestCase):
# gateway data plane — built from Dockerfile.orchestrator when absent. # gateway data plane — built from Dockerfile.orchestrator when absent.
calls: list[list[str]] = [] calls: list[list[str]] = []
def fake(argv: list[str], **_kw: object) -> Mock: def fake(argv: list[str]) -> Mock:
calls.append(argv) calls.append(argv)
if argv[:2] == ["docker", "ps"]: if argv[:2] == ["docker", "ps"]:
return _proc(stdout="") # orchestrator not running return _proc(stdout="") # orchestrator not running
@@ -148,7 +148,7 @@ class TestOrchestratorService(unittest.TestCase):
def test_ensure_running_skips_orchestrator_image_build_when_present(self) -> None: def test_ensure_running_skips_orchestrator_image_build_when_present(self) -> None:
calls: list[list[str]] = [] calls: list[list[str]] = []
def fake(argv: list[str], **_kw: object) -> Mock: def fake(argv: list[str]) -> Mock:
calls.append(argv) calls.append(argv)
if argv[:2] == ["docker", "ps"]: if argv[:2] == ["docker", "ps"]:
return _proc(stdout="") return _proc(stdout="")
-62
View File
@@ -1,62 +0,0 @@
"""Unit: the infra-artifact publisher's upload path (PRD 0069 Stage 2).
The rootfs is hundreds of MB, so `_put` must stream it from disk rather than
read it into memory. Network is mocked; no Docker, no real build.
"""
from __future__ import annotations
import tempfile
import unittest
import urllib.request
from pathlib import Path
from unittest import mock
from bot_bottle.backend.firecracker import publish_infra as pub
class _Resp:
status = 201
def __enter__(self) -> "_Resp":
return self
def __exit__(self, *a: object) -> bool:
return False
class TestPut(unittest.TestCase):
def test_streams_file_body_with_content_length(self) -> None:
captured: list[urllib.request.Request] = []
def fake_urlopen(req: urllib.request.Request, *a: object, **k: object) -> _Resp:
captured.append(req)
return _Resp()
with tempfile.TemporaryDirectory() as d:
f = Path(d) / "rootfs.ext4.gz"
payload = b"x" * 4096
f.write_bytes(payload)
with mock.patch.object(pub.urllib.request, "urlopen", fake_urlopen):
pub._put("https://reg/pkg", f, token="t")
req = captured[0]
# Body is the open file object (streamed), never the bytes in memory.
self.assertTrue(hasattr(req.data, "read"))
self.assertNotIsInstance(req.data, (bytes, bytearray))
self.assertEqual(str(len(payload)), req.get_header("Content-length"))
def test_small_bytes_body_still_works(self) -> None:
captured: list[urllib.request.Request] = []
def fake_urlopen(req: urllib.request.Request, *a: object, **k: object) -> _Resp:
captured.append(req)
return _Resp()
with mock.patch.object(pub.urllib.request, "urlopen", fake_urlopen):
pub._put("https://reg/sha", b"abc123 rootfs\n", token="")
self.assertEqual(b"abc123 rootfs\n", captured[0].data)
if __name__ == "__main__":
unittest.main()