Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b25cd72fc3 | |||
| 8e43c26ab4 | |||
| 14ff4fe186 | |||
| cae1215f63 | |||
| 28766d7733 | |||
| 819f967844 | |||
| 2f45f5afec | |||
| 31a5ec2fc8 |
+13
-36
@@ -1,45 +1,22 @@
|
|||||||
# Firecracker single infra-VM image (PRD 0070 Stage B).
|
# Shared infra image: gateway data plane + orchestrator control plane.
|
||||||
#
|
#
|
||||||
# The per-host infra VM runs the orchestrator control plane, the gateway
|
# Used directly by the Docker backend (run as one `bot-bottle-infra`
|
||||||
# data plane, AND builds agent images (buildah) — all in one microVM (see
|
# container, replacing the prior two-container split). The Firecracker
|
||||||
# backend/firecracker/infra_vm.py). It composes:
|
# backend extends this via Dockerfile.infra.fc, adding buildah/crun/
|
||||||
# * FROM the gateway image (mitmproxy / git / gitleaks / supervise + the
|
# netavark for in-VM agent-image building.
|
||||||
# flat daemon modules) — now trixie-based, so buildah 1.39 is available;
|
#
|
||||||
# * `COPY --from` the orchestrator image's content (the single definition
|
# Dockerfile.orchestrator is the single definition of the orchestrator
|
||||||
# of the control-plane payload — see Dockerfile.orchestrator), so this
|
# content (the lean `bot_bottle` package on python:3.12-slim). Both this
|
||||||
# VM and the docker backend share one orchestrator definition; and
|
# image and Dockerfile.infra.fc pull it in via `COPY --from`.
|
||||||
# * buildah, installed HERE only (the docker orchestrator/gateway images
|
|
||||||
# never carry it).
|
|
||||||
#
|
#
|
||||||
# multi-`FROM` can't union two bases (that's multi-stage, not multiple
|
# multi-`FROM` can't union two bases (that's multi-stage, not multiple
|
||||||
# inheritance), so the orchestrator content is pulled in via `COPY --from`
|
# inheritance), so the orchestrator content is pulled in via `COPY --from`
|
||||||
# rather than a second base. Both images share the trixie `python:3.12-slim`
|
# rather than a second base. Both images share the trixie `python:3.12-slim`
|
||||||
# base, so the copy is clean (same python; future installed deps copy too).
|
# base, so the copy is clean (same python; future installed deps copy too).
|
||||||
#
|
|
||||||
# The docker backend keeps orchestrator + gateway as separate images; this
|
|
||||||
# combined image exists only for the Firecracker single-VM cut. Splitting a
|
|
||||||
# service back into its own VM later is a routing change, not a repackaging
|
|
||||||
# (PRD 0070's "secret concentration"; a disposable builder can boot from
|
|
||||||
# this same image on its own TAP).
|
|
||||||
FROM bot-bottle-gateway:latest
|
FROM bot-bottle-gateway:latest
|
||||||
|
|
||||||
# --- in-VM agent-image builder (PRD 0069 Stage 3) -------------------
|
# The orchestrator content, from its single definition. The gateway image
|
||||||
# The Firecracker backend builds users' agent Dockerfiles *inside this VM*
|
# already has the flat daemon modules under /app; this adds the full
|
||||||
# with buildah (rootless, daemonless) instead of on the host — no host
|
# `bot_bottle` package so `python3 -m bot_bottle.orchestrator` resolves —
|
||||||
# Docker daemon, no root-equivalent `docker` group. `crun` is the OCI
|
# used by gateway_init when BOT_BOTTLE_GATEWAY_DAEMONS includes `orchestrator`.
|
||||||
# runtime; `netavark` + `aardvark-dns` are the network backend for `FROM`
|
|
||||||
# pulls + `RUN` egress. Requires the trixie base (buildah 1.39: bookworm's
|
|
||||||
# 1.28 can't parse Dockerfile heredocs that agent images use).
|
|
||||||
RUN apt-get update \
|
|
||||||
&& apt-get install -y --no-install-recommends \
|
|
||||||
buildah crun netavark aardvark-dns \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
# vfs + chroot: buildah works as root in the bare microVM (no
|
|
||||||
# fuse-overlayfs / overlay module / subuid maps). Matches image_builder.
|
|
||||||
ENV STORAGE_DRIVER=vfs \
|
|
||||||
BUILDAH_ISOLATION=chroot
|
|
||||||
|
|
||||||
# The orchestrator content, pulled from its single definition. The gateway
|
|
||||||
# image already has the flat daemon modules under /app; this adds the full
|
|
||||||
# `bot_bottle` package so `python3 -m bot_bottle.orchestrator` resolves.
|
|
||||||
COPY --from=bot-bottle-orchestrator:latest /app/bot_bottle /app/bot_bottle
|
COPY --from=bot-bottle-orchestrator:latest /app/bot_bottle /app/bot_bottle
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
# Firecracker infra VM image (PRD 0070 Stage B).
|
||||||
|
#
|
||||||
|
# Extends the shared infra base (Dockerfile.infra: gateway + orchestrator
|
||||||
|
# control plane) with the in-VM agent-image builder. The Firecracker backend
|
||||||
|
# builds users' agent Dockerfiles *inside this VM* with buildah (rootless,
|
||||||
|
# daemonless) instead of on the host — no host Docker daemon, no
|
||||||
|
# root-equivalent `docker` group.
|
||||||
|
#
|
||||||
|
# Requires the trixie base from bot-bottle-gateway (buildah 1.39: bookworm's
|
||||||
|
# 1.28 can't parse Dockerfile heredocs that agent images use).
|
||||||
|
#
|
||||||
|
# `crun` is the OCI runtime; `netavark` + `aardvark-dns` are the network
|
||||||
|
# backend for `FROM` pulls + `RUN` egress. `vfs` + `chroot`: buildah works
|
||||||
|
# as root in the bare microVM (no fuse-overlayfs / overlay module / subuid
|
||||||
|
# maps). Matches image_builder.
|
||||||
|
FROM bot-bottle-infra:latest
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
buildah crun netavark aardvark-dns \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
ENV STORAGE_DRIVER=vfs \
|
||||||
|
BUILDAH_ISOLATION=chroot
|
||||||
@@ -75,22 +75,6 @@ On compatible macOS hosts, the default backend requires Apple's `container` CLI
|
|||||||
|
|
||||||
Use `BOT_BOTTLE_BACKEND=docker ./cli.py start <agent>` on hosts where neither Apple Container nor KVM is available and Docker is the desired backend.
|
Use `BOT_BOTTLE_BACKEND=docker ./cli.py start <agent>` on hosts where neither Apple Container nor KVM is available and Docker is the desired backend.
|
||||||
|
|
||||||
> **Experimental containers-in-bottle spike (#392):** a bottle may set
|
|
||||||
> `docker_access: true`. On the macOS backend this starts a guest-local,
|
|
||||||
> rootless **podman** service after the bottle is registered, exposing its
|
|
||||||
> Docker-compatible API socket — the agent still uses `docker` and `docker
|
|
||||||
> compose`. It does not mount Docker Desktop's socket or add outer VM
|
|
||||||
> capabilities. Rootless Docker was tried first and does not work here at
|
|
||||||
> all: Apple Container's capability bounding set omits `CAP_SYS_ADMIN`,
|
|
||||||
> which the kernel requires to write a multi-range `uid_map`. See
|
|
||||||
> [`docs/research/rootless-docker-in-apple-container-spike.md`](docs/research/rootless-docker-in-apple-container-spike.md).
|
|
||||||
>
|
|
||||||
> The tradeoff to understand before enabling it: podman avoids that
|
|
||||||
> requirement by falling back to a single-UID mapping, so nested containers
|
|
||||||
> provide **no isolation from the agent itself** — `root` inside a nested
|
|
||||||
> container is the agent user outside it. Nested containers are a build/test
|
|
||||||
> convenience, not a security boundary. The bottle remains the boundary.
|
|
||||||
|
|
||||||
### Firecracker on Linux
|
### Firecracker on Linux
|
||||||
|
|
||||||
On Linux, a KVM-capable host defaults to the Firecracker backend. It needs:
|
On Linux, a KVM-capable host defaults to the Firecracker backend. It needs:
|
||||||
|
|||||||
@@ -0,0 +1,60 @@
|
|||||||
|
"""Shared helpers for the consolidated launch sequence (PRD 0070).
|
||||||
|
|
||||||
|
Logic that was duplicated across the docker, macos_container, and
|
||||||
|
firecracker consolidated_launch modules — extracted so each backend
|
||||||
|
imports it rather than re-implementing it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from ..egress import EgressPlan
|
||||||
|
from ..git_gate import GitGatePlan
|
||||||
|
from ..orchestrator.client import OrchestratorClient
|
||||||
|
from ..orchestrator.registration import registration_inputs
|
||||||
|
from .docker.gateway_provision import GatewayTransport, deprovision_git_gate, provision_git_gate
|
||||||
|
|
||||||
|
|
||||||
|
def provision_bottle(
|
||||||
|
client: OrchestratorClient,
|
||||||
|
source_ip: str,
|
||||||
|
egress_plan: EgressPlan,
|
||||||
|
git_gate_plan: GitGatePlan,
|
||||||
|
transport: GatewayTransport,
|
||||||
|
*,
|
||||||
|
image_ref: str = "",
|
||||||
|
tokens: dict[str, str] | None = None,
|
||||||
|
):
|
||||||
|
"""Register the bottle and provision its git-gate state. Rolls back the
|
||||||
|
registration if provisioning fails so no orphan is left. Returns the
|
||||||
|
`RegisteredBottle` from the orchestrator."""
|
||||||
|
inputs = registration_inputs(egress_plan)
|
||||||
|
reg = client.register_bottle(
|
||||||
|
source_ip, image_ref=image_ref, policy=inputs.policy,
|
||||||
|
metadata=inputs.metadata, tokens=tokens,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
provision_git_gate(transport, reg.bottle_id, git_gate_plan)
|
||||||
|
except Exception:
|
||||||
|
client.teardown_bottle(reg.bottle_id)
|
||||||
|
raise
|
||||||
|
return reg
|
||||||
|
|
||||||
|
|
||||||
|
def teardown_consolidated(
|
||||||
|
bottle_id: str,
|
||||||
|
transport: GatewayTransport,
|
||||||
|
*,
|
||||||
|
orchestrator_url: str,
|
||||||
|
timeout: float | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Deregister the bottle and remove its git-gate state. Both steps are
|
||||||
|
idempotent so this is safe from a cleanup trap."""
|
||||||
|
from ..orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
|
||||||
|
OrchestratorClient(
|
||||||
|
orchestrator_url,
|
||||||
|
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
|
||||||
|
).teardown_bottle(bottle_id)
|
||||||
|
deprovision_git_gate(transport, bottle_id)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["provision_bottle", "teardown_consolidated"]
|
||||||
@@ -1,19 +1,13 @@
|
|||||||
"""Consolidated bottle launch sequence for the docker backend (PRD 0070).
|
"""Consolidated bottle launch sequence for the docker backend (PRD 0070).
|
||||||
|
|
||||||
Composes the orchestrator primitives into the register/teardown sequence that
|
Composes the orchestrator primitives into the register/teardown sequence:
|
||||||
replaces the per-bottle gateway:
|
|
||||||
|
|
||||||
1. ensure the orchestrator control plane + shared gateway are up;
|
1. ensure the single infra container (control plane + gateway) is up;
|
||||||
2. allocate the bottle a pinned source IP on the gateway network (the
|
2. allocate the bottle a pinned source IP on the gateway network;
|
||||||
attribution key), skipping the gateway's own address + live bottles;
|
3. register it and provision its git-gate repos/creds into the gateway.
|
||||||
3. register it (egress policy blob + slug metadata) → bottle id + identity
|
|
||||||
token;
|
|
||||||
4. provision its git-gate repos/creds into the running gateway.
|
|
||||||
|
|
||||||
It returns a `LaunchContext` with everything the agent container needs to
|
Returns a `LaunchContext` with everything the agent container needs to
|
||||||
attach — network, pinned IP, the gateway's address (its proxy target), the
|
attach. The agent `docker run` itself is the backend's job; this owns the
|
||||||
orchestrator URL, and the identity token. The agent `docker run` itself is
|
|
||||||
the backend's job (it owns provider provisioning); this owns the
|
|
||||||
orchestrator-facing wiring so that sequence stays testable in isolation.
|
orchestrator-facing wiring so that sequence stays testable in isolation.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -25,15 +19,12 @@ from ...docker_cmd import run_docker
|
|||||||
from ...egress import EgressPlan
|
from ...egress import EgressPlan
|
||||||
from ...git_gate import GitGatePlan
|
from ...git_gate import GitGatePlan
|
||||||
from ...orchestrator.client import OrchestratorClient
|
from ...orchestrator.client import OrchestratorClient
|
||||||
from ...orchestrator.gateway import GATEWAY_NAME, GATEWAY_NETWORK
|
from ...orchestrator.gateway import GATEWAY_NETWORK
|
||||||
from ...orchestrator.lifecycle import OrchestratorService
|
from ...orchestrator.lifecycle import INFRA_NAME, OrchestratorService
|
||||||
from ...orchestrator.registration import registration_inputs
|
from ..consolidated_util import provision_bottle
|
||||||
|
from ..consolidated_util import teardown_consolidated as _teardown_util
|
||||||
|
from .gateway_provision import DockerGatewayTransport
|
||||||
from .gateway_net import next_free_ip
|
from .gateway_net import next_free_ip
|
||||||
from .gateway_provision import (
|
|
||||||
DockerGatewayTransport,
|
|
||||||
deprovision_git_gate,
|
|
||||||
provision_git_gate,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class ConsolidatedLaunchError(RuntimeError):
|
class ConsolidatedLaunchError(RuntimeError):
|
||||||
@@ -75,24 +66,21 @@ def _container_ip(name: str, network: str) -> str:
|
|||||||
ip = proc.stdout.strip()
|
ip = proc.stdout.strip()
|
||||||
if proc.returncode != 0 or not ip:
|
if proc.returncode != 0 or not ip:
|
||||||
raise ConsolidatedLaunchError(
|
raise ConsolidatedLaunchError(
|
||||||
f"gateway {name} has no address on {network}: {proc.stderr.strip()}"
|
f"container {name} has no address on {network}: {proc.stderr.strip()}"
|
||||||
)
|
)
|
||||||
return ip
|
return ip
|
||||||
|
|
||||||
|
|
||||||
def _network_container_ips(network: str) -> list[str]:
|
def _network_container_ips(network: str) -> list[str]:
|
||||||
"""Every address currently assigned on the gateway network — the ground
|
"""Every address currently assigned on the gateway network — the ground
|
||||||
truth for "in use": the gateway + orchestrator infrastructure containers
|
truth for "in use": the infra container and every live agent. Read from
|
||||||
and every live agent. Read from the network so a new bottle can't collide
|
the network so a new bottle can't collide with anything actually attached."""
|
||||||
with anything actually attached (a registry-only view would miss the
|
|
||||||
orchestrator/gateway containers)."""
|
|
||||||
proc = run_docker([
|
proc = run_docker([
|
||||||
"docker", "network", "inspect", "--format",
|
"docker", "network", "inspect", "--format",
|
||||||
"{{range .Containers}}{{.IPv4Address}} {{end}}", network,
|
"{{range .Containers}}{{.IPv4Address}} {{end}}", network,
|
||||||
])
|
])
|
||||||
ips: list[str] = []
|
ips: list[str] = []
|
||||||
for entry in proc.stdout.split():
|
for entry in proc.stdout.split():
|
||||||
# entries look like "172.20.0.2/16" — keep the address.
|
|
||||||
ips.append(entry.split("/", 1)[0])
|
ips.append(entry.split("/", 1)[0])
|
||||||
return ips
|
return ips
|
||||||
|
|
||||||
@@ -104,33 +92,24 @@ def launch_consolidated(
|
|||||||
image_ref: str = "",
|
image_ref: str = "",
|
||||||
tokens: dict[str, str] | None = None,
|
tokens: dict[str, str] | None = None,
|
||||||
service: OrchestratorService | None = None,
|
service: OrchestratorService | None = None,
|
||||||
gateway_name: str = GATEWAY_NAME,
|
infra_name: str = INFRA_NAME,
|
||||||
network: str = GATEWAY_NETWORK,
|
network: str = GATEWAY_NETWORK,
|
||||||
) -> LaunchContext:
|
) -> LaunchContext:
|
||||||
"""Ensure the orchestrator + gateway are up, allocate + register the
|
"""Ensure the infra container is up, allocate + register the bottle, and
|
||||||
bottle, and provision its git-gate state. Returns the agent's attach
|
provision its git-gate state. Returns the agent's attach context."""
|
||||||
context. Raises `ConsolidatedLaunchError` (or the primitives' own errors)
|
|
||||||
if any step fails — the caller tears down on failure."""
|
|
||||||
service = service or OrchestratorService()
|
service = service or OrchestratorService()
|
||||||
url = service.ensure_running()
|
url = service.ensure_running()
|
||||||
client = OrchestratorClient(url)
|
client = OrchestratorClient(url)
|
||||||
|
|
||||||
cidr = _network_cidr(network)
|
cidr = _network_cidr(network)
|
||||||
gateway_ip = _container_ip(gateway_name, network)
|
gateway_ip = _container_ip(infra_name, network)
|
||||||
source_ip = next_free_ip(cidr, _network_container_ips(network))
|
source_ip = next_free_ip(cidr, _network_container_ips(network))
|
||||||
|
|
||||||
inputs = registration_inputs(egress_plan)
|
transport = DockerGatewayTransport(infra_name)
|
||||||
reg = client.register_bottle(
|
reg = provision_bottle(
|
||||||
source_ip, image_ref=image_ref, policy=inputs.policy,
|
client, source_ip, egress_plan, git_gate_plan, transport,
|
||||||
metadata=inputs.metadata, tokens=tokens,
|
image_ref=image_ref, tokens=tokens,
|
||||||
)
|
)
|
||||||
try:
|
|
||||||
provision_git_gate(
|
|
||||||
DockerGatewayTransport(gateway_name), reg.bottle_id, git_gate_plan)
|
|
||||||
except Exception:
|
|
||||||
# Roll the registration back so a provisioning failure leaves no orphan.
|
|
||||||
client.teardown_bottle(reg.bottle_id)
|
|
||||||
raise
|
|
||||||
return LaunchContext(
|
return LaunchContext(
|
||||||
bottle_id=reg.bottle_id,
|
bottle_id=reg.bottle_id,
|
||||||
identity_token=reg.identity_token,
|
identity_token=reg.identity_token,
|
||||||
@@ -142,20 +121,12 @@ def launch_consolidated(
|
|||||||
|
|
||||||
|
|
||||||
def teardown_consolidated(
|
def teardown_consolidated(
|
||||||
bottle_id: str,
|
bottle_id: str, *, orchestrator_url: str, infra_name: str = INFRA_NAME,
|
||||||
*,
|
|
||||||
orchestrator_url: str,
|
|
||||||
gateway_name: str = GATEWAY_NAME,
|
|
||||||
timeout: float | None = None,
|
timeout: float | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Deregister the bottle and remove its git-gate state from the gateway.
|
"""Deregister the bottle and remove its git-gate state. Idempotent."""
|
||||||
Both steps are idempotent so this is safe from a cleanup trap."""
|
_teardown_util(bottle_id, DockerGatewayTransport(infra_name),
|
||||||
from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
|
orchestrator_url=orchestrator_url, timeout=timeout)
|
||||||
OrchestratorClient(
|
|
||||||
orchestrator_url,
|
|
||||||
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
|
|
||||||
).teardown_bottle(bottle_id)
|
|
||||||
deprovision_git_gate(DockerGatewayTransport(gateway_name), bottle_id)
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
|||||||
@@ -33,8 +33,7 @@ from ...orchestrator.client import OrchestratorClient
|
|||||||
from ...orchestrator.lifecycle import (
|
from ...orchestrator.lifecycle import (
|
||||||
OrchestratorStartError, # re-exported so callers can catch it
|
OrchestratorStartError, # re-exported so callers can catch it
|
||||||
)
|
)
|
||||||
from ...orchestrator.registration import registration_inputs
|
from ..consolidated_util import provision_bottle, teardown_consolidated as _teardown_util
|
||||||
from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate
|
|
||||||
from . import infra_vm
|
from . import infra_vm
|
||||||
|
|
||||||
|
|
||||||
@@ -68,18 +67,11 @@ def launch_consolidated(
|
|||||||
url = infra.control_plane_url
|
url = infra.control_plane_url
|
||||||
client = OrchestratorClient(url)
|
client = OrchestratorClient(url)
|
||||||
|
|
||||||
inputs = registration_inputs(egress_plan)
|
transport = infra_vm.gateway_transport()
|
||||||
reg = client.register_bottle(
|
reg = provision_bottle(
|
||||||
guest_ip, image_ref=image_ref, policy=inputs.policy,
|
client, guest_ip, egress_plan, git_gate_plan, transport,
|
||||||
metadata=inputs.metadata, tokens=tokens,
|
image_ref=image_ref, tokens=tokens,
|
||||||
)
|
)
|
||||||
try:
|
|
||||||
provision_git_gate(
|
|
||||||
infra_vm.gateway_transport(), reg.bottle_id, git_gate_plan)
|
|
||||||
except Exception:
|
|
||||||
client.teardown_bottle(reg.bottle_id)
|
|
||||||
raise
|
|
||||||
|
|
||||||
# The shared gateway CA every agent on this host trusts for TLS
|
# The shared gateway CA every agent on this host trusts for TLS
|
||||||
# interception — fetched from the infra VM over SSH.
|
# interception — fetched from the infra VM over SSH.
|
||||||
return LaunchContext(
|
return LaunchContext(
|
||||||
@@ -98,12 +90,8 @@ def teardown_consolidated(
|
|||||||
VM. Both steps are idempotent so this is safe from a cleanup trap. Does
|
VM. Both steps are idempotent so this is safe from a cleanup trap. Does
|
||||||
NOT stop the infra VM — it's a persistent per-host singleton shared by
|
NOT stop the infra VM — it's a persistent per-host singleton shared by
|
||||||
every bottle."""
|
every bottle."""
|
||||||
from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
|
_teardown_util(bottle_id, infra_vm.gateway_transport(),
|
||||||
OrchestratorClient(
|
orchestrator_url=orchestrator_url, timeout=timeout)
|
||||||
orchestrator_url,
|
|
||||||
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
|
|
||||||
).teardown_bottle(bottle_id)
|
|
||||||
deprovision_git_gate(infra_vm.gateway_transport(), bottle_id)
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ from . import util
|
|||||||
_ARTIFACT_FORMAT = "1"
|
_ARTIFACT_FORMAT = "1"
|
||||||
|
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||||
_DOCKERFILES = ("Dockerfile.orchestrator", "Dockerfile.gateway", "Dockerfile.infra")
|
_DOCKERFILES = ("Dockerfile.orchestrator", "Dockerfile.gateway", "Dockerfile.infra", "Dockerfile.infra.fc")
|
||||||
|
|
||||||
_DEFAULT_BASE = "https://gitea.dideric.is"
|
_DEFAULT_BASE = "https://gitea.dideric.is"
|
||||||
_DEFAULT_OWNER = "didericis"
|
_DEFAULT_OWNER = "didericis"
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ from pathlib import Path
|
|||||||
from typing import Generator
|
from typing import Generator
|
||||||
|
|
||||||
from ...log import die, info
|
from ...log import die, info
|
||||||
|
from .. import util as backend_util
|
||||||
from ..docker import util as docker_mod
|
from ..docker import util as docker_mod
|
||||||
from ..docker.gateway_provision import GatewayProvisionError
|
from ..docker.gateway_provision import GatewayProvisionError
|
||||||
from . import firecracker_vm, infra_artifact, netpool, util
|
from . import firecracker_vm, infra_artifact, netpool, util
|
||||||
@@ -93,19 +94,18 @@ class InfraVm:
|
|||||||
"""The gateway's mitmproxy CA (PEM) that agents install to trust its
|
"""The gateway's mitmproxy CA (PEM) that agents install to trust its
|
||||||
TLS interception. Generated a moment after boot, so this polls over
|
TLS interception. Generated a moment after boot, so this polls over
|
||||||
SSH until it appears (mirrors DockerGateway.ca_cert_pem)."""
|
SSH until it appears (mirrors DockerGateway.ca_cert_pem)."""
|
||||||
deadline = time.monotonic() + timeout
|
def _fetch() -> str | None:
|
||||||
while True:
|
|
||||||
proc = subprocess.run(
|
proc = subprocess.run(
|
||||||
util.ssh_base_argv(self.private_key, self.guest_ip)
|
util.ssh_base_argv(self.private_key, self.guest_ip)
|
||||||
+ [f"cat {_GATEWAY_CA_PATH}"],
|
+ [f"cat {_GATEWAY_CA_PATH}"],
|
||||||
capture_output=True, text=True, timeout=15, check=False,
|
capture_output=True, text=True, timeout=15, check=False,
|
||||||
)
|
)
|
||||||
if proc.returncode == 0 and "BEGIN CERTIFICATE" in proc.stdout:
|
ok = proc.returncode == 0 and "BEGIN CERTIFICATE" in proc.stdout
|
||||||
return proc.stdout
|
return proc.stdout if ok else None
|
||||||
if time.monotonic() >= deadline:
|
try:
|
||||||
die(f"gateway CA not available after {timeout:g}s: "
|
return backend_util.poll_ca_cert(_fetch, timeout=timeout)
|
||||||
f"{proc.stderr.strip() or 'empty'}")
|
except TimeoutError as exc:
|
||||||
time.sleep(_HEALTH_POLL_SECONDS)
|
die(str(exc))
|
||||||
|
|
||||||
|
|
||||||
def ensure_built() -> None:
|
def ensure_built() -> None:
|
||||||
@@ -125,16 +125,19 @@ def ensure_built() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def build_infra_images_with_docker() -> None:
|
def build_infra_images_with_docker() -> None:
|
||||||
"""Build the three fixed images from source with host Docker: orchestrator,
|
"""Build the four fixed images from source with host Docker: orchestrator,
|
||||||
gateway, then the combined infra image (`COPY --from` orchestrator, `FROM`
|
gateway, the shared infra base (Dockerfile.infra), then the Firecracker
|
||||||
gateway). The launch host uses this only in `BOT_BOTTLE_INFRA_BUILD=local`
|
infra image (Dockerfile.infra.fc: FROM infra + buildah). The launch host
|
||||||
mode; `publish_infra` uses it off-host to produce the published artifact."""
|
uses this only in `BOT_BOTTLE_INFRA_BUILD=local` mode; `publish_infra`
|
||||||
|
uses it off-host to produce the published artifact."""
|
||||||
docker_mod.build_image(
|
docker_mod.build_image(
|
||||||
_ORCHESTRATOR_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.orchestrator")
|
_ORCHESTRATOR_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.orchestrator")
|
||||||
docker_mod.build_image(
|
docker_mod.build_image(
|
||||||
_GATEWAY_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.gateway")
|
_GATEWAY_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.gateway")
|
||||||
docker_mod.build_image(
|
docker_mod.build_image(
|
||||||
_INFRA_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.infra")
|
"bot-bottle-infra:latest", str(_REPO_ROOT), dockerfile="Dockerfile.infra")
|
||||||
|
docker_mod.build_image(
|
||||||
|
_INFRA_IMAGE, str(_REPO_ROOT), dockerfile="Dockerfile.infra.fc")
|
||||||
|
|
||||||
|
|
||||||
def build_infra_rootfs_dir() -> Path:
|
def build_infra_rootfs_dir() -> Path:
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ class MacosContainerBottlePlan(BottlePlan):
|
|||||||
# bottle is registered. See launch.py's stamp for why it lives here and not
|
# bottle is registered. See launch.py's stamp for why it lives here and not
|
||||||
# only in the exec-time proxy env.
|
# only in the exec-time proxy env.
|
||||||
identity_token: str = ""
|
identity_token: str = ""
|
||||||
docker_access: bool = False
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def container_name(self) -> str:
|
def container_name(self) -> str:
|
||||||
|
|||||||
@@ -38,8 +38,7 @@ from ...egress import EgressPlan
|
|||||||
from ...git_gate import GitGatePlan
|
from ...git_gate import GitGatePlan
|
||||||
from ...log import info
|
from ...log import info
|
||||||
from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
|
from ...orchestrator.client import OrchestratorClient, OrchestratorClientError
|
||||||
from ...orchestrator.registration import registration_inputs
|
from ..consolidated_util import provision_bottle, teardown_consolidated as _teardown_util
|
||||||
from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate
|
|
||||||
from . import util as container_mod
|
from . import util as container_mod
|
||||||
from .enumerate import CONTAINER_NAME_PREFIX, EnumerationError, enumerate_active
|
from .enumerate import CONTAINER_NAME_PREFIX, EnumerationError, enumerate_active
|
||||||
from .gateway import GATEWAY_NETWORK
|
from .gateway import GATEWAY_NETWORK
|
||||||
@@ -142,17 +141,10 @@ def register_agent(
|
|||||||
client.reconcile(live_source_ips(endpoint.network))
|
client.reconcile(live_source_ips(endpoint.network))
|
||||||
except (OrchestratorClientError, EnumerationError) as e:
|
except (OrchestratorClientError, EnumerationError) as e:
|
||||||
info(f"registry reconciliation skipped: {e}")
|
info(f"registry reconciliation skipped: {e}")
|
||||||
inputs = registration_inputs(egress_plan)
|
reg = provision_bottle(
|
||||||
reg = client.register_bottle(
|
client, source_ip, egress_plan, git_gate_plan, AppleGatewayTransport(),
|
||||||
source_ip, image_ref=image_ref, policy=inputs.policy,
|
image_ref=image_ref, tokens=tokens,
|
||||||
metadata=inputs.metadata, tokens=tokens,
|
|
||||||
)
|
)
|
||||||
try:
|
|
||||||
provision_git_gate(AppleGatewayTransport(), reg.bottle_id, git_gate_plan)
|
|
||||||
except Exception:
|
|
||||||
# Roll the registration back so a provisioning failure leaves no orphan.
|
|
||||||
client.teardown_bottle(reg.bottle_id)
|
|
||||||
raise
|
|
||||||
return LaunchContext(
|
return LaunchContext(
|
||||||
bottle_id=reg.bottle_id,
|
bottle_id=reg.bottle_id,
|
||||||
identity_token=reg.identity_token,
|
identity_token=reg.identity_token,
|
||||||
@@ -169,12 +161,8 @@ def teardown_consolidated(
|
|||||||
"""Deregister the bottle and remove its git-gate state from the gateway.
|
"""Deregister the bottle and remove its git-gate state from the gateway.
|
||||||
Both steps are idempotent so this is safe from a cleanup trap. Does NOT
|
Both steps are idempotent so this is safe from a cleanup trap. Does NOT
|
||||||
stop the gateway — it's a persistent per-host singleton."""
|
stop the gateway — it's a persistent per-host singleton."""
|
||||||
from ...orchestrator.config_store import DEFAULT_TEARDOWN_TIMEOUT_SECONDS
|
_teardown_util(bottle_id, AppleGatewayTransport(),
|
||||||
OrchestratorClient(
|
orchestrator_url=orchestrator_url, timeout=timeout)
|
||||||
orchestrator_url,
|
|
||||||
timeout=timeout if timeout is not None else DEFAULT_TEARDOWN_TIMEOUT_SECONDS,
|
|
||||||
).teardown_bottle(bottle_id)
|
|
||||||
deprovision_git_gate(AppleGatewayTransport(), bottle_id)
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
|||||||
@@ -53,6 +53,7 @@ from ...paths import (
|
|||||||
HOST_DB_FILENAME,
|
HOST_DB_FILENAME,
|
||||||
host_control_plane_token,
|
host_control_plane_token,
|
||||||
)
|
)
|
||||||
|
from .. import util as backend_util
|
||||||
from . import util as container_mod
|
from . import util as container_mod
|
||||||
from .gateway import (
|
from .gateway import (
|
||||||
DEFAULT_CA_TIMEOUT_SECONDS,
|
DEFAULT_CA_TIMEOUT_SECONDS,
|
||||||
@@ -263,18 +264,16 @@ class MacosInfraService:
|
|||||||
interception. Read out of the container (the CA lives on a
|
interception. Read out of the container (the CA lives on a
|
||||||
container-internal path, not a host mount); polls because mitmproxy
|
container-internal path, not a host mount); polls because mitmproxy
|
||||||
writes it a beat after start."""
|
writes it a beat after start."""
|
||||||
deadline = time.monotonic() + timeout
|
def _fetch() -> str | None:
|
||||||
while True:
|
|
||||||
result = container_mod.run_container_argv(
|
result = container_mod.run_container_argv(
|
||||||
["container", "exec", self._name, "cat", GATEWAY_CA_CERT])
|
["container", "exec", self._name, "cat", GATEWAY_CA_CERT])
|
||||||
if result.returncode == 0 and result.stdout.strip():
|
return result.stdout if result.returncode == 0 and result.stdout.strip() else None
|
||||||
return result.stdout
|
try:
|
||||||
if time.monotonic() >= deadline:
|
return backend_util.poll_ca_cert(_fetch, timeout=timeout)
|
||||||
raise GatewayError(
|
except TimeoutError as exc:
|
||||||
f"gateway CA not available in {self._name} after {timeout:g}s: "
|
raise GatewayError(
|
||||||
f"{(result.stderr or '').strip() or 'empty'}"
|
f"gateway CA not available in {self._name} after {timeout:g}s"
|
||||||
)
|
) from exc
|
||||||
time.sleep(_CA_POLL_SECONDS)
|
|
||||||
|
|
||||||
def stop(self) -> None:
|
def stop(self) -> None:
|
||||||
"""Remove the infra container (idempotent). The DB volume persists."""
|
"""Remove the infra container (idempotent). The DB volume persists."""
|
||||||
|
|||||||
@@ -64,7 +64,6 @@ from .gateway_hosts import (
|
|||||||
refresh_gateway_host,
|
refresh_gateway_host,
|
||||||
set_gateway_host,
|
set_gateway_host,
|
||||||
)
|
)
|
||||||
from . import rootless_podman
|
|
||||||
from .bottle_plan import MacosContainerBottlePlan
|
from .bottle_plan import MacosContainerBottlePlan
|
||||||
from ...orchestrator.config_store import resolve_teardown_timeout
|
from ...orchestrator.config_store import resolve_teardown_timeout
|
||||||
from .consolidated_launch import (
|
from .consolidated_launch import (
|
||||||
@@ -172,10 +171,6 @@ def launch(
|
|||||||
# token above, so — unlike the run-time env — the plan CAN carry it.
|
# token above, so — unlike the run-time env — the plan CAN carry it.
|
||||||
plan = dataclasses.replace(plan, identity_token=ctx.identity_token)
|
plan = dataclasses.replace(plan, identity_token=ctx.identity_token)
|
||||||
|
|
||||||
exec_env = {
|
|
||||||
**_identity_proxy_env(endpoint, ctx.identity_token),
|
|
||||||
**rootless_podman.guest_env(plan.docker_access),
|
|
||||||
}
|
|
||||||
bottle = MacosContainerBottle(
|
bottle = MacosContainerBottle(
|
||||||
plan.container_name,
|
plan.container_name,
|
||||||
teardown,
|
teardown,
|
||||||
@@ -189,16 +184,10 @@ def launch(
|
|||||||
),
|
),
|
||||||
terminal_color=plan.spec.color,
|
terminal_color=plan.spec.color,
|
||||||
agent_workdir=plan.workspace_plan.workdir,
|
agent_workdir=plan.workspace_plan.workdir,
|
||||||
exec_env=exec_env,
|
exec_env=_identity_proxy_env(endpoint, ctx.identity_token),
|
||||||
)
|
)
|
||||||
bottle.prompt_path = provision(plan, bottle)
|
bottle.prompt_path = provision(plan, bottle)
|
||||||
|
|
||||||
if plan.docker_access:
|
|
||||||
rootless_podman.prepare_guest_devices(
|
|
||||||
plan.container_name, container_mod.exec_container_as_root,
|
|
||||||
)
|
|
||||||
rootless_podman.start(bottle)
|
|
||||||
|
|
||||||
yield bottle
|
yield bottle
|
||||||
finally:
|
finally:
|
||||||
teardown()
|
teardown()
|
||||||
@@ -210,22 +199,15 @@ def _build_images(plan: MacosContainerBottlePlan) -> MacosContainerBottlePlan:
|
|||||||
committed = read_committed_image(plan.slug)
|
committed = read_committed_image(plan.slug)
|
||||||
if committed and container_mod.image_exists(committed):
|
if committed and container_mod.image_exists(committed):
|
||||||
info(f"using committed image {committed!r}")
|
info(f"using committed image {committed!r}")
|
||||||
plan = dataclasses.replace(
|
return dataclasses.replace(
|
||||||
plan,
|
plan,
|
||||||
agent_provision=dataclasses.replace(
|
agent_provision=dataclasses.replace(
|
||||||
plan.agent_provision, image=committed,
|
plan.agent_provision, image=committed,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
else:
|
container_mod.build_image(
|
||||||
container_mod.build_image(
|
plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path,
|
||||||
plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path,
|
)
|
||||||
)
|
|
||||||
if plan.docker_access:
|
|
||||||
image = rootless_podman.build_image(plan.image, container_mod.build_image)
|
|
||||||
plan = dataclasses.replace(
|
|
||||||
plan,
|
|
||||||
agent_provision=dataclasses.replace(plan.agent_provision, image=image),
|
|
||||||
)
|
|
||||||
return plan
|
return plan
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -44,5 +44,4 @@ def resolve_plan(
|
|||||||
egress_plan=egress_plan,
|
egress_plan=egress_plan,
|
||||||
supervise_plan=supervise_plan,
|
supervise_plan=supervise_plan,
|
||||||
agent_provision=agent_provision_plan,
|
agent_provision=agent_provision_plan,
|
||||||
docker_access=manifest.bottle.docker_access,
|
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,89 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
uid="$(id -u)"
|
|
||||||
if [ "$uid" -eq 0 ]; then
|
|
||||||
echo "refusing to run rootless podman as root" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
for command in podman docker fuse-overlayfs slirp4netns; do
|
|
||||||
command -v "$command" >/dev/null 2>&1 || {
|
|
||||||
echo "missing rootless podman prerequisite: $command" >&2
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
done
|
|
||||||
|
|
||||||
# The inverse of the rootless-Docker check, and the whole point of the podman
|
|
||||||
# variant: a subordinate range would push podman onto newuidmap, which cannot
|
|
||||||
# write a multi-range uid_map without CAP_SYS_ADMIN in this guest. An empty
|
|
||||||
# range keeps it on the single-UID self-mapping an unprivileged process may
|
|
||||||
# write itself.
|
|
||||||
if grep -q "^$(id -un):" /etc/subuid 2>/dev/null; then
|
|
||||||
echo "unexpected subordinate UID range for $(id -un): podman would" >&2
|
|
||||||
echo "require CAP_SYS_ADMIN via newuidmap in this guest" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
for device in /dev/fuse /dev/net/tun; do
|
|
||||||
[ -r "$device" ] && [ -w "$device" ] || {
|
|
||||||
echo "device $device is not readable/writable by $(id -un)" >&2
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
done
|
|
||||||
|
|
||||||
export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/bot-bottle-podman-run}"
|
|
||||||
config="$HOME/.config/containers"
|
|
||||||
mkdir -p "$XDG_RUNTIME_DIR" "$config"
|
|
||||||
chmod 700 "$XDG_RUNTIME_DIR"
|
|
||||||
|
|
||||||
# ignore_chown_errors is required, not incidental: with a single-UID mapping
|
|
||||||
# there is no second UID for image layers to be chowned to, so layers that
|
|
||||||
# record other owners would otherwise fail to extract.
|
|
||||||
cat > "$config/storage.conf" <<'CONF'
|
|
||||||
[storage]
|
|
||||||
driver="overlay"
|
|
||||||
[storage.options.overlay]
|
|
||||||
mount_program="/usr/bin/fuse-overlayfs"
|
|
||||||
ignore_chown_errors="true"
|
|
||||||
CONF
|
|
||||||
|
|
||||||
# No cgroup delegation reaches this guest, so asking podman to manage cgroups
|
|
||||||
# fails; events_logger=file avoids the journald socket that is equally absent.
|
|
||||||
cat > "$config/containers.conf" <<'CONF'
|
|
||||||
[containers]
|
|
||||||
cgroups="disabled"
|
|
||||||
[engine]
|
|
||||||
cgroup_manager="cgroupfs"
|
|
||||||
events_logger="file"
|
|
||||||
CONF
|
|
||||||
|
|
||||||
# Registry pulls egress through the bottle's proxy like everything else. The
|
|
||||||
# token-bearing proxy URL is already in the agent's environment; persisting it
|
|
||||||
# inside this disposable VM does not broaden its authority.
|
|
||||||
python3 - <<'PY'
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("https_proxy", "")
|
|
||||||
no_proxy = os.environ.get("NO_PROXY") or os.environ.get("no_proxy", "")
|
|
||||||
config = {"proxies": {"default": {
|
|
||||||
"httpProxy": proxy,
|
|
||||||
"httpsProxy": proxy,
|
|
||||||
"noProxy": no_proxy,
|
|
||||||
}}}
|
|
||||||
path = Path.home() / ".docker" / "config.json"
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_text(json.dumps(config), encoding="utf-8")
|
|
||||||
path.chmod(0o600)
|
|
||||||
PY
|
|
||||||
|
|
||||||
if docker info >/dev/null 2>&1; then
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
log=/tmp/bot-bottle-rootless-podman.log
|
|
||||||
nohup podman system service --time=0 \
|
|
||||||
"unix://$XDG_RUNTIME_DIR/podman.sock" \
|
|
||||||
>"$log" 2>&1 </dev/null &
|
|
||||||
@@ -1,132 +0,0 @@
|
|||||||
"""Experimental rootless podman bootstrap for Apple-container bottles.
|
|
||||||
|
|
||||||
The service and every nested container remain inside the existing per-bottle
|
|
||||||
VM. This module refuses to compensate for missing prerequisites with outer
|
|
||||||
capabilities, a privileged container, or a host Docker socket.
|
|
||||||
|
|
||||||
Podman is used rather than rootless Docker for one specific reason: Apple
|
|
||||||
Container's capability bounding set omits `CAP_SYS_ADMIN`, which the kernel
|
|
||||||
requires to write a multi-range `uid_map` via `newuidmap`. Rootless Docker
|
|
||||||
has no path that avoids that write. Podman does — with no subordinate UID
|
|
||||||
range configured it falls back to a single-UID self-mapping, which an
|
|
||||||
unprivileged process may write itself. See
|
|
||||||
`docs/research/rootless-docker-in-apple-container-spike.md`.
|
|
||||||
|
|
||||||
That fallback is why `build_image` *removes* the agent user's `/etc/subuid`
|
|
||||||
and `/etc/subgid` entries instead of adding them: their presence is precisely
|
|
||||||
what would send podman down the `newuidmap` path that cannot work here.
|
|
||||||
|
|
||||||
The agent still talks to `docker` and `docker compose`; those speak to
|
|
||||||
podman's Docker-compatible API socket, so nothing in the agent's habits
|
|
||||||
changes.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import shlex
|
|
||||||
import shutil
|
|
||||||
import tempfile
|
|
||||||
import time
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Callable
|
|
||||||
|
|
||||||
from ...log import die, info
|
|
||||||
|
|
||||||
_INIT = "/usr/local/libexec/bot-bottle/rootless-podman-init"
|
|
||||||
_RUNTIME_DIR = "/tmp/bot-bottle-podman-run"
|
|
||||||
_SOCKET = f"{_RUNTIME_DIR}/podman.sock"
|
|
||||||
_LOG = "/tmp/bot-bottle-rootless-podman.log"
|
|
||||||
READY_RETRIES = 30
|
|
||||||
|
|
||||||
# Apple Container creates both device nodes 0600 root:root, so the agent user
|
|
||||||
# cannot open them: /dev/fuse blocks the fuse-overlayfs storage driver and
|
|
||||||
# /dev/net/tun blocks slirp4netns, which rootless podman uses for the default
|
|
||||||
# bridge network that stock compose files expect. Relaxing the modes needs no
|
|
||||||
# capability the bottle does not already hold — unlike CAP_SYS_ADMIN, which is
|
|
||||||
# what killed the rootless-Docker approach.
|
|
||||||
_GUEST_DEVICES = ("/dev/fuse", "/dev/net/tun")
|
|
||||||
|
|
||||||
|
|
||||||
def build_image(
|
|
||||||
base_image: str,
|
|
||||||
build: Callable[..., None],
|
|
||||||
) -> str:
|
|
||||||
"""Layer spike-only tooling on an already-built provider image."""
|
|
||||||
image = f"{base_image}-rootless-podman"
|
|
||||||
init_script = Path(__file__).with_name("rootless-podman-init.sh")
|
|
||||||
with tempfile.TemporaryDirectory(prefix="bot-bottle-rootless-podman.") as tmp:
|
|
||||||
context = Path(tmp)
|
|
||||||
shutil.copy2(init_script, context / "rootless-podman-init.sh")
|
|
||||||
(context / "Dockerfile").write_text(
|
|
||||||
"FROM docker:28-cli AS docker_cli\n"
|
|
||||||
f"FROM {base_image}\n"
|
|
||||||
"USER root\n"
|
|
||||||
"COPY --from=docker_cli /usr/local/bin/docker /usr/local/bin/docker\n"
|
|
||||||
"COPY --from=docker_cli /usr/local/libexec/docker/cli-plugins/"
|
|
||||||
"docker-compose /usr/local/libexec/docker/cli-plugins/docker-compose\n"
|
|
||||||
"RUN apt-get update \\\n"
|
|
||||||
" && apt-get install -y --no-install-recommends podman "
|
|
||||||
"fuse-overlayfs slirp4netns uidmap \\\n"
|
|
||||||
" && rm -rf /var/lib/apt/lists/* \\\n"
|
|
||||||
# Deliberate: an empty subordinate range keeps podman on the
|
|
||||||
# single-UID mapping that needs no CAP_SYS_ADMIN. Adding ranges
|
|
||||||
# here would reintroduce the newuidmap failure this spike exists
|
|
||||||
# to route around.
|
|
||||||
" && sed -i '/^node:/d' /etc/subuid /etc/subgid\n"
|
|
||||||
"COPY rootless-podman-init.sh "
|
|
||||||
"/usr/local/libexec/bot-bottle/rootless-podman-init\n"
|
|
||||||
"RUN chmod 0755 /usr/local/libexec/bot-bottle/rootless-podman-init\n"
|
|
||||||
"USER node\n",
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
build(image, str(context), dockerfile=str(context / "Dockerfile"))
|
|
||||||
return image
|
|
||||||
|
|
||||||
|
|
||||||
def guest_env(enabled: bool) -> dict[str, str]:
|
|
||||||
"""Environment consumed by the Docker CLI inside an enabled bottle."""
|
|
||||||
if not enabled:
|
|
||||||
return {}
|
|
||||||
return {
|
|
||||||
"DOCKER_HOST": f"unix://{_SOCKET}",
|
|
||||||
"XDG_RUNTIME_DIR": _RUNTIME_DIR,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def prepare_guest_devices(container_name: str, exec_as_root: Callable[..., None]) -> None:
|
|
||||||
"""Make /dev/fuse and /dev/net/tun openable by the agent user.
|
|
||||||
|
|
||||||
Runs as root inside the bottle because the agent must not be able to
|
|
||||||
re-mode device nodes itself. No outer capability is involved.
|
|
||||||
"""
|
|
||||||
exec_as_root(
|
|
||||||
container_name,
|
|
||||||
["sh", "-c", f"chmod 0666 {' '.join(_GUEST_DEVICES)}"],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def start(bottle: object) -> None:
|
|
||||||
"""Start and verify the unprivileged service through the bottle exec API."""
|
|
||||||
info("starting experimental rootless podman service")
|
|
||||||
result = bottle.exec(shlex.quote(_INIT)) # type: ignore[attr-defined]
|
|
||||||
if result.returncode != 0:
|
|
||||||
detail = (result.stderr or result.stdout or "").strip()
|
|
||||||
die(f"rootless podman bootstrap failed: {detail or '<no output>'}")
|
|
||||||
|
|
||||||
for _ in range(READY_RETRIES):
|
|
||||||
result = bottle.exec("docker info >/dev/null 2>&1") # type: ignore[attr-defined]
|
|
||||||
if result.returncode == 0:
|
|
||||||
info("rootless podman service is ready")
|
|
||||||
return
|
|
||||||
time.sleep(0.2)
|
|
||||||
|
|
||||||
logs = bottle.exec( # type: ignore[attr-defined]
|
|
||||||
f"tail -n 80 {_LOG} 2>/dev/null || true"
|
|
||||||
)
|
|
||||||
die(
|
|
||||||
"rootless podman did not become ready without additional outer "
|
|
||||||
f"privileges:\n{(logs.stdout or logs.stderr or '<no log>').strip()}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = ["build_image", "guest_env", "prepare_guest_devices", "start"]
|
|
||||||
@@ -7,6 +7,8 @@ from __future__ import annotations
|
|||||||
import hashlib
|
import hashlib
|
||||||
import os
|
import os
|
||||||
import ssl
|
import ssl
|
||||||
|
import time
|
||||||
|
from collections.abc import Callable
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
@@ -15,6 +17,24 @@ from ..log import die, info
|
|||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from ..egress import EgressPlan
|
from ..egress import EgressPlan
|
||||||
|
|
||||||
|
_CA_POLL_INTERVAL = 0.5
|
||||||
|
|
||||||
|
|
||||||
|
def poll_ca_cert(fetch: Callable[[], str | None], *, timeout: float) -> str:
|
||||||
|
"""Poll `fetch` until it returns a non-empty PEM string or `timeout` expires.
|
||||||
|
|
||||||
|
`fetch` should return the PEM on success and `None` (or empty string) when
|
||||||
|
the cert is not yet available. Raises `TimeoutError` if the cert never
|
||||||
|
appears within `timeout` seconds."""
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
while True:
|
||||||
|
result = fetch()
|
||||||
|
if result:
|
||||||
|
return result
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
raise TimeoutError(f"CA cert not available after {timeout:g}s")
|
||||||
|
time.sleep(_CA_POLL_INTERVAL)
|
||||||
|
|
||||||
|
|
||||||
# Debian-family CA layout, shared by every backend (all guest images
|
# Debian-family CA layout, shared by every backend (all guest images
|
||||||
# are Debian-family). AGENT_CA_PATH is the source path that
|
# are Debian-family). AGENT_CA_PATH is the source path that
|
||||||
|
|||||||
+35
-15
@@ -61,6 +61,11 @@ class _DaemonSpec:
|
|||||||
_EGRESS_ONLY_ENV_PREFIXES: tuple[str, ...] = ("EGRESS_TOKEN_",)
|
_EGRESS_ONLY_ENV_PREFIXES: tuple[str, ...] = ("EGRESS_TOKEN_",)
|
||||||
_READY_GATED_DAEMONS: tuple[str, ...] = ("git-gate", "git-http")
|
_READY_GATED_DAEMONS: tuple[str, ...] = ("git-gate", "git-http")
|
||||||
|
|
||||||
|
# Daemons that must be requested explicitly via BOT_BOTTLE_GATEWAY_DAEMONS
|
||||||
|
# and are NOT started in the default (env-var-unset) case. The orchestrator
|
||||||
|
# only runs in the combined infra container, never in a standalone gateway.
|
||||||
|
_OPT_IN_DAEMONS: frozenset[str] = frozenset({"orchestrator"})
|
||||||
|
|
||||||
|
|
||||||
def _env_for_daemon(name: str, base_env: dict[str, str]) -> dict[str, str]:
|
def _env_for_daemon(name: str, base_env: dict[str, str]) -> dict[str, str]:
|
||||||
"""Egress sees the full bundle env. Everyone else gets a copy
|
"""Egress sees the full bundle env. Everyone else gets a copy
|
||||||
@@ -75,7 +80,14 @@ def _env_for_daemon(name: str, base_env: dict[str, str]) -> dict[str, str]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# The orchestrator is listed first so it starts before the gateway daemons,
|
||||||
|
# giving the control plane a head start to accept /resolve calls. The gateway
|
||||||
|
# daemons tolerate early /resolve failures and retry per-request.
|
||||||
_DAEMONS: tuple[_DaemonSpec, ...] = (
|
_DAEMONS: tuple[_DaemonSpec, ...] = (
|
||||||
|
_DaemonSpec("orchestrator", (
|
||||||
|
"python3", "-m", "bot_bottle.orchestrator",
|
||||||
|
"--host", "0.0.0.0", "--port", "8099", "--broker", "stub",
|
||||||
|
)),
|
||||||
_DaemonSpec("egress", ("/bin/sh", "/app/egress-entrypoint.sh")),
|
_DaemonSpec("egress", ("/bin/sh", "/app/egress-entrypoint.sh")),
|
||||||
_DaemonSpec("git-gate", ("/bin/sh", "/git-gate-entrypoint.sh")),
|
_DaemonSpec("git-gate", ("/bin/sh", "/git-gate-entrypoint.sh")),
|
||||||
_DaemonSpec("git-http", ("python3", "-m", "bot_bottle.git_http_backend")),
|
_DaemonSpec("git-http", ("python3", "-m", "bot_bottle.git_http_backend")),
|
||||||
@@ -103,18 +115,20 @@ def _selected_daemons(
|
|||||||
env: dict[str, str],
|
env: dict[str, str],
|
||||||
all_daemons: Sequence[_DaemonSpec] | None = None,
|
all_daemons: Sequence[_DaemonSpec] | None = None,
|
||||||
) -> tuple[_DaemonSpec, ...]:
|
) -> tuple[_DaemonSpec, ...]:
|
||||||
"""Filter the daemon set by the BOT_BOTTLE_GATEWAY_DAEMONS env
|
"""Filter the daemon set by the BOT_BOTTLE_GATEWAY_DAEMONS env var.
|
||||||
var. Unknown names in the list are ignored — the caller is the
|
|
||||||
source of truth for which daemons are wired.
|
|
||||||
|
|
||||||
`all_daemons` defaults to `_DAEMONS` resolved at call time (not
|
When the var is unset/empty, return all non-opt-in daemons (the
|
||||||
at definition time), so tests can monkey-patch the module-level
|
standard gateway subset). Opt-in daemons (e.g. `orchestrator`) only
|
||||||
`_DAEMONS` and have the new value take effect."""
|
run when explicitly named — they never start in a plain gateway
|
||||||
|
container that doesn't set the env var. Unknown names are ignored.
|
||||||
|
|
||||||
|
`all_daemons` defaults to `_DAEMONS` resolved at call time (not at
|
||||||
|
definition time), so tests can pass a custom list."""
|
||||||
if all_daemons is None:
|
if all_daemons is None:
|
||||||
all_daemons = _DAEMONS
|
all_daemons = _DAEMONS
|
||||||
raw = env.get("BOT_BOTTLE_GATEWAY_DAEMONS", "").strip()
|
raw = env.get("BOT_BOTTLE_GATEWAY_DAEMONS", "").strip()
|
||||||
if not raw:
|
if not raw:
|
||||||
return tuple(all_daemons)
|
return tuple(d for d in all_daemons if d.name not in _OPT_IN_DAEMONS)
|
||||||
wanted = {n.strip() for n in raw.split(",") if n.strip()}
|
wanted = {n.strip() for n in raw.split(",") if n.strip()}
|
||||||
return tuple(d for d in all_daemons if d.name in wanted)
|
return tuple(d for d in all_daemons if d.name in wanted)
|
||||||
|
|
||||||
@@ -136,7 +150,7 @@ def _pump(name: str, stream: IO[bytes]) -> None:
|
|||||||
|
|
||||||
def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]:
|
def _spawn(spec: _DaemonSpec) -> subprocess.Popen[bytes]:
|
||||||
env = _env_for_daemon(spec.name, dict(os.environ))
|
env = _env_for_daemon(spec.name, dict(os.environ))
|
||||||
proc = subprocess.Popen(
|
proc = subprocess.Popen( # pylint: disable=consider-using-with
|
||||||
_argv_for_daemon(spec.name, spec.argv, env),
|
_argv_for_daemon(spec.name, spec.argv, env),
|
||||||
stdout=subprocess.PIPE,
|
stdout=subprocess.PIPE,
|
||||||
stderr=subprocess.STDOUT,
|
stderr=subprocess.STDOUT,
|
||||||
@@ -183,6 +197,14 @@ class _Supervisor:
|
|||||||
except ProcessLookupError:
|
except ProcessLookupError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def _sigkill_all(self) -> None:
|
||||||
|
for _, p in self.procs:
|
||||||
|
if p.poll() is None:
|
||||||
|
try:
|
||||||
|
p.kill()
|
||||||
|
except ProcessLookupError:
|
||||||
|
pass
|
||||||
|
|
||||||
def request_restart(self, daemon_name: str) -> bool:
|
def request_restart(self, daemon_name: str) -> bool:
|
||||||
"""Queue a daemon restart for the main loop to process.
|
"""Queue a daemon restart for the main loop to process.
|
||||||
|
|
||||||
@@ -235,12 +257,7 @@ class _Supervisor:
|
|||||||
f"grace ({_GRACE_SECONDS:.0f}s) elapsed; SIGKILL on "
|
f"grace ({_GRACE_SECONDS:.0f}s) elapsed; SIGKILL on "
|
||||||
f"{', '.join(still_running)}"
|
f"{', '.join(still_running)}"
|
||||||
)
|
)
|
||||||
for _, p in self.procs:
|
self._sigkill_all()
|
||||||
if p.poll() is None:
|
|
||||||
try:
|
|
||||||
p.kill()
|
|
||||||
except ProcessLookupError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
done = all(p.poll() is not None for _, p in self.procs)
|
done = all(p.poll() is not None for _, p in self.procs)
|
||||||
if done:
|
if done:
|
||||||
@@ -361,7 +378,10 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||||||
# --signal HUP <bundle>` after writing routes.yaml. The kernel
|
# --signal HUP <bundle>` after writing routes.yaml. The kernel
|
||||||
# delivers SIGHUP to PID 1 (this supervisor); forward it to
|
# delivers SIGHUP to PID 1 (this supervisor); forward it to
|
||||||
# mitmdump so it reloads its addon.
|
# mitmdump so it reloads its addon.
|
||||||
signal.signal(signal.SIGHUP, lambda *_: sup.forward_signal(signal.SIGHUP, "egress")) # type: ignore
|
signal.signal(
|
||||||
|
signal.SIGHUP,
|
||||||
|
lambda *_: sup.forward_signal(signal.SIGHUP, "egress"), # type: ignore[misc]
|
||||||
|
)
|
||||||
|
|
||||||
while not sup.tick():
|
while not sup.tick():
|
||||||
time.sleep(_POLL_INTERVAL)
|
time.sleep(_POLL_INTERVAL)
|
||||||
|
|||||||
@@ -44,9 +44,6 @@ class ManifestBottle:
|
|||||||
# daemon that exposes egress MCP tools to the agent. Set
|
# daemon that exposes egress MCP tools to the agent. Set
|
||||||
# `supervise: false` to skip the gateway.
|
# `supervise: false` to skip the gateway.
|
||||||
supervise: bool = True
|
supervise: bool = True
|
||||||
# Experimental guest-local container engine (issue #392). Backends must
|
|
||||||
# implement this without granting access to a host/shared daemon.
|
|
||||||
docker_access: bool = False
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_dict(cls, name: str, raw: object) -> "ManifestBottle":
|
def from_dict(cls, name: str, raw: object) -> "ManifestBottle":
|
||||||
@@ -126,15 +123,7 @@ class ManifestBottle:
|
|||||||
f"(was {type(supervise_raw).__name__})"
|
f"(was {type(supervise_raw).__name__})"
|
||||||
)
|
)
|
||||||
|
|
||||||
docker_access_raw = d.get("docker_access", False)
|
|
||||||
if not isinstance(docker_access_raw, bool):
|
|
||||||
raise ManifestError(
|
|
||||||
f"bottle '{name}' docker_access must be a boolean "
|
|
||||||
f"(was {type(docker_access_raw).__name__})"
|
|
||||||
)
|
|
||||||
|
|
||||||
return cls(
|
return cls(
|
||||||
env=env, agent_provider=agent_provider, git=git,
|
env=env, agent_provider=agent_provider, git=git,
|
||||||
git_user=git_user, egress=egress, supervise=supervise_raw,
|
git_user=git_user, egress=egress, supervise=supervise_raw,
|
||||||
docker_access=docker_access_raw,
|
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -54,7 +54,6 @@ def _merge_two_bottles_runtime(base: "ManifestBottle", override: "ManifestBottle
|
|||||||
git_user=merged_git_user,
|
git_user=merged_git_user,
|
||||||
egress=merged_egress,
|
egress=merged_egress,
|
||||||
supervise=override.supervise,
|
supervise=override.supervise,
|
||||||
docker_access=override.docker_access,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -207,7 +206,6 @@ def _fold_two_bottles(
|
|||||||
git_user=merged_git_user,
|
git_user=merged_git_user,
|
||||||
egress=merged_egress,
|
egress=merged_egress,
|
||||||
supervise=later.supervise,
|
supervise=later.supervise,
|
||||||
docker_access=later.docker_access,
|
|
||||||
), merged_repos_raw
|
), merged_repos_raw
|
||||||
|
|
||||||
|
|
||||||
@@ -268,11 +266,6 @@ def _merge_bottles(
|
|||||||
merged_supervise = (
|
merged_supervise = (
|
||||||
child.supervise if "supervise" in child_raw else parent.supervise
|
child.supervise if "supervise" in child_raw else parent.supervise
|
||||||
)
|
)
|
||||||
merged_docker_access = (
|
|
||||||
child.docker_access
|
|
||||||
if "docker_access" in child_raw
|
|
||||||
else parent.docker_access
|
|
||||||
)
|
|
||||||
validate_egress_routes(name, merged_egress.routes)
|
validate_egress_routes(name, merged_egress.routes)
|
||||||
|
|
||||||
return ManifestBottle(
|
return ManifestBottle(
|
||||||
@@ -282,7 +275,6 @@ def _merge_bottles(
|
|||||||
git_user=merged_git_user,
|
git_user=merged_git_user,
|
||||||
egress=merged_egress,
|
egress=merged_egress,
|
||||||
supervise=merged_supervise,
|
supervise=merged_supervise,
|
||||||
docker_access=merged_docker_access,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -16,10 +16,7 @@ _FILENAME_RX = re.compile(r"^[a-z][a-z0-9-]*$")
|
|||||||
# sets dies with a "did you mean" pointer: typos should not silently
|
# sets dies with a "did you mean" pointer: typos should not silently
|
||||||
# ghost into an empty config.
|
# ghost into an empty config.
|
||||||
BOTTLE_KEYS = frozenset(
|
BOTTLE_KEYS = frozenset(
|
||||||
{
|
{"env", "extends", "agent_provider", "git-gate", "egress", "supervise"}
|
||||||
"env", "extends", "agent_provider", "git-gate", "egress", "supervise",
|
|
||||||
"docker_access",
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
AGENT_KEYS_REQUIRED: frozenset[str] = frozenset()
|
AGENT_KEYS_REQUIRED: frozenset[str] = frozenset()
|
||||||
AGENT_KEYS_OPTIONAL = frozenset({"bottle", "skills", "git-gate"})
|
AGENT_KEYS_OPTIONAL = frozenset({"bottle", "skills", "git-gate"})
|
||||||
|
|||||||
@@ -1,17 +1,15 @@
|
|||||||
"""Orchestrator + gateway lifecycle (PRD 0070, docker slice).
|
"""Orchestrator + gateway lifecycle (PRD 0070, docker slice).
|
||||||
|
|
||||||
Runs the orchestrator control plane **as a container** on the shared gateway
|
Runs both the orchestrator control plane and the gateway data plane inside
|
||||||
network, alongside the gateway container. This is the PRD's "virtualize the
|
a single `bot-bottle-infra` container on the shared gateway network —
|
||||||
orchestrator": container↔container between the gateway and the orchestrator
|
matching the structure already used by the macOS and Firecracker backends.
|
||||||
avoids the host firewall (which drops container→host traffic), and the gateway
|
`gateway_init` is PID 1 and supervises both; the infra container is an
|
||||||
reaches the control plane by container name over docker DNS. The host CLI
|
idempotent per-host singleton.
|
||||||
reaches it via a published loopback port.
|
|
||||||
|
|
||||||
The orchestrator runs with the **register-only broker** — the *backend*
|
The combined container replaces the prior two-container split
|
||||||
launches agent containers (compose), so the orchestrator needs no docker
|
(bot-bottle-orchestrator + bot-bottle-orch-gateway). The host CLI reaches
|
||||||
socket. That keeps this control-plane container unprivileged; the host manages
|
the control plane via a published loopback port; gateway daemons reach it
|
||||||
both containers. `ensure_running` is an idempotent singleton (fixed container
|
over 127.0.0.1 (same container).
|
||||||
names + the published port).
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -26,49 +24,70 @@ from pathlib import Path
|
|||||||
from .. import log
|
from .. import log
|
||||||
from ..docker_cmd import run_docker
|
from ..docker_cmd import run_docker
|
||||||
from ..paths import CONTROL_PLANE_TOKEN_ENV, bot_bottle_root, host_control_plane_token
|
from ..paths import CONTROL_PLANE_TOKEN_ENV, bot_bottle_root, host_control_plane_token
|
||||||
from .gateway import GATEWAY_IMAGE, GATEWAY_NAME, GATEWAY_NETWORK, DockerGateway, GatewayError
|
from ..supervise import DB_PATH_IN_CONTAINER
|
||||||
|
from .gateway import (
|
||||||
|
GATEWAY_CA_VOLUME,
|
||||||
|
GATEWAY_DOCKERFILE,
|
||||||
|
GATEWAY_IMAGE,
|
||||||
|
GATEWAY_NETWORK,
|
||||||
|
GatewayError,
|
||||||
|
MITMPROXY_HOME,
|
||||||
|
_host_db_dir,
|
||||||
|
)
|
||||||
|
|
||||||
DEFAULT_PORT = 8099
|
DEFAULT_PORT = 8099
|
||||||
ORCHESTRATOR_NAME = "bot-bottle-orchestrator"
|
DEFAULT_STARTUP_TIMEOUT_SECONDS = 45.0
|
||||||
ORCHESTRATOR_LABEL = "bot-bottle-orchestrator=1"
|
|
||||||
# The control-plane's own runtime image — lean (python + the stdlib-only
|
INFRA_NAME = "bot-bottle-infra"
|
||||||
# `bot_bottle` package, bind-mounted at run time), distinct from the heavy
|
INFRA_LABEL = "bot-bottle-infra=1"
|
||||||
# gateway data-plane image it used to borrow (#384). Env override for
|
# The combined infra image: gateway data plane + orchestrator content.
|
||||||
# operators pinning a published build.
|
# Built from Dockerfile.infra (FROM gateway + COPY --from orchestrator).
|
||||||
|
INFRA_IMAGE = os.environ.get("BOT_BOTTLE_INFRA_IMAGE", "bot-bottle-infra:latest")
|
||||||
|
INFRA_DOCKERFILE = "Dockerfile.infra"
|
||||||
|
# Baked as a container label so `ensure_running` can detect whether the
|
||||||
|
# running container is executing the current bind-mounted source.
|
||||||
|
INFRA_SOURCE_HASH_LABEL = "bot-bottle-infra-source-hash"
|
||||||
|
|
||||||
|
# Orchestrator image: the single canonical definition of the control-plane
|
||||||
|
# content (lean: python:3.12-slim + bot_bottle package, no mitmproxy/git).
|
||||||
|
# Used as a build intermediate: `Dockerfile.infra` COPY --from this image.
|
||||||
ORCHESTRATOR_IMAGE = os.environ.get(
|
ORCHESTRATOR_IMAGE = os.environ.get(
|
||||||
"BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest"
|
"BOT_BOTTLE_ORCHESTRATOR_IMAGE", "bot-bottle-orchestrator:latest"
|
||||||
)
|
)
|
||||||
ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator"
|
ORCHESTRATOR_DOCKERFILE = "Dockerfile.orchestrator"
|
||||||
# Baked onto the container as a label so `ensure_running` can tell whether the
|
|
||||||
# running process is executing the *current* bind-mounted source — see
|
|
||||||
# `source_hash`.
|
|
||||||
ORCHESTRATOR_SOURCE_HASH_LABEL = "bot-bottle-orchestrator-source-hash"
|
|
||||||
|
|
||||||
# The repo root is bind-mounted into the control-plane container so
|
# The gateway daemons + orchestrator the infra container runs.
|
||||||
# `python -m bot_bottle.orchestrator` resolves the package (the orchestrator
|
# BOT_BOTTLE_GATEWAY_DAEMONS listing `orchestrator` opts it in to
|
||||||
# is stdlib-only, so the lean orchestrator image's python is enough).
|
# gateway_init's supervise tree (see gateway_init._OPT_IN_DAEMONS).
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
_INFRA_DAEMONS = "egress,git-http,supervise,orchestrator"
|
||||||
_APP_DIR = "/app"
|
|
||||||
|
# The bind-mount path for the live control-plane source inside the
|
||||||
|
# container. Separate from /app so the gateway's baked scripts
|
||||||
|
# (egress_addon.py, egress-entrypoint.sh) are not overlaid.
|
||||||
|
_SRC_IN_CONTAINER = "/bot-bottle-src"
|
||||||
|
# Bot-bottle host-root bind-mount inside the container (DB + state).
|
||||||
_ROOT_IN_CONTAINER = "/bot-bottle-root"
|
_ROOT_IN_CONTAINER = "/bot-bottle-root"
|
||||||
|
|
||||||
|
# The supervise daemon writes proposals into the host DB directory.
|
||||||
|
_SUPERVISE_DB_DIR_IN_CONTAINER = os.path.dirname(DB_PATH_IN_CONTAINER)
|
||||||
|
|
||||||
_HEALTH_POLL_SECONDS = 0.25
|
_HEALTH_POLL_SECONDS = 0.25
|
||||||
DEFAULT_STARTUP_TIMEOUT_SECONDS = 45.0
|
|
||||||
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
|
_HEALTH_REQUEST_TIMEOUT_SECONDS = 1.0
|
||||||
|
|
||||||
|
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
class OrchestratorStartError(RuntimeError):
|
class OrchestratorStartError(RuntimeError):
|
||||||
"""The orchestrator container did not become healthy within the timeout."""
|
"""The infra container did not become healthy within the timeout."""
|
||||||
|
|
||||||
|
|
||||||
def source_hash(repo_root: Path) -> str:
|
def source_hash(repo_root: Path) -> str:
|
||||||
"""Content hash of the orchestrator's bind-mounted Python source (the
|
"""Content hash of the orchestrator's bind-mounted Python source (the
|
||||||
`bot_bottle` package the control-plane process imports). This only
|
`bot_bottle` package the control-plane process imports). Changes only
|
||||||
changes when the code that would actually run inside the container
|
when the code that would actually run changes — `ensure_running`
|
||||||
changes — `ensure_running` recreates the container on a mismatch and
|
recreates the container on a mismatch so a code change takes effect,
|
||||||
otherwise leaves a healthy one alone, so a bottle launch that isn't
|
but leaves a healthy up-to-date container alone to preserve in-memory
|
||||||
accompanied by a code change doesn't restart the process and drop every
|
egress tokens."""
|
||||||
*other* active bottle's in-memory egress tokens (`Orchestrator._tokens`
|
|
||||||
in `service.py`, never persisted to disk by design)."""
|
|
||||||
h = hashlib.sha256()
|
h = hashlib.sha256()
|
||||||
for path in sorted((repo_root / "bot_bottle").rglob("*.py")):
|
for path in sorted((repo_root / "bot_bottle").rglob("*.py")):
|
||||||
h.update(str(path.relative_to(repo_root)).encode())
|
h.update(str(path.relative_to(repo_root)).encode())
|
||||||
@@ -77,57 +96,37 @@ def source_hash(repo_root: Path) -> str:
|
|||||||
|
|
||||||
|
|
||||||
class OrchestratorService:
|
class OrchestratorService:
|
||||||
"""Manages the orchestrator control-plane container + the shared gateway.
|
"""Manages the single per-host infra container (control plane + gateway).
|
||||||
Callers only need `ensure_running()` + `url`.
|
Callers only need `ensure_running()` + `url`.
|
||||||
|
|
||||||
`orchestrator_name` / `orchestrator_label` let backends run independent
|
`infra_name` / `infra_label` let backends run independent infra containers
|
||||||
orchestrators on the same host without name collisions (e.g. the
|
on the same host without name collisions (e.g. isolated integration tests
|
||||||
Firecracker backend uses `bot-bottle-fc-orchestrator` alongside the Docker
|
that can't share the production INFRA_NAME singleton)."""
|
||||||
backend's `bot-bottle-orchestrator`); `gateway_name` gives the paired
|
|
||||||
gateway container the same treatment (e.g. isolated integration tests
|
|
||||||
that can't share the production `GATEWAY_NAME` singleton). Subclass and
|
|
||||||
override `_gateway()` for anything `_gateway_image`/`gateway_name` can't
|
|
||||||
express (a genuinely backend-specific gateway variant)."""
|
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
port: int = DEFAULT_PORT,
|
port: int = DEFAULT_PORT,
|
||||||
network: str = GATEWAY_NETWORK,
|
network: str = GATEWAY_NETWORK,
|
||||||
image: str = ORCHESTRATOR_IMAGE,
|
image: str = INFRA_IMAGE,
|
||||||
gateway_image: str = GATEWAY_IMAGE,
|
|
||||||
gateway_name: str = GATEWAY_NAME,
|
|
||||||
repo_root: Path = _REPO_ROOT,
|
repo_root: Path = _REPO_ROOT,
|
||||||
host_root: Path | None = None,
|
host_root: Path | None = None,
|
||||||
orchestrator_name: str = ORCHESTRATOR_NAME,
|
infra_name: str = INFRA_NAME,
|
||||||
orchestrator_label: str = ORCHESTRATOR_LABEL,
|
infra_label: str = INFRA_LABEL,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.port = port
|
self.port = port
|
||||||
self.network = network
|
self.network = network
|
||||||
# Two distinct images (#384): `image` is the lean control-plane
|
|
||||||
# runtime this container runs; `_gateway_image` is the heavy egress /
|
|
||||||
# git-gate / supervise data plane the gateway container runs. They
|
|
||||||
# were one conflated image before the split.
|
|
||||||
self.image = image
|
self.image = image
|
||||||
self._gateway_image = gateway_image
|
|
||||||
self._gateway_name = gateway_name
|
|
||||||
self._repo_root = repo_root
|
self._repo_root = repo_root
|
||||||
self._host_root = host_root or bot_bottle_root()
|
self._host_root = host_root or bot_bottle_root()
|
||||||
self._orchestrator_name = orchestrator_name
|
self._infra_name = infra_name
|
||||||
self._orchestrator_label = orchestrator_label
|
self._infra_label = infra_label
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def url(self) -> str:
|
def url(self) -> str:
|
||||||
"""Host-side control-plane URL (published loopback port)."""
|
"""Host-side control-plane URL (published loopback port)."""
|
||||||
return f"http://127.0.0.1:{self.port}"
|
return f"http://127.0.0.1:{self.port}"
|
||||||
|
|
||||||
@property
|
|
||||||
def internal_url(self) -> str:
|
|
||||||
"""Control-plane URL as the gateway container reaches it — by name over
|
|
||||||
docker DNS on the shared network. This is the gateway's
|
|
||||||
BOT_BOTTLE_ORCHESTRATOR_URL."""
|
|
||||||
return f"http://{self._orchestrator_name}:{self.port}"
|
|
||||||
|
|
||||||
def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool:
|
def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool:
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(f"{self.url}/health", timeout=timeout) as resp:
|
with urllib.request.urlopen(f"{self.url}/health", timeout=timeout) as resp:
|
||||||
@@ -139,139 +138,129 @@ class OrchestratorService:
|
|||||||
proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"])
|
proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"])
|
||||||
return name in proc.stdout.split()
|
return name in proc.stdout.split()
|
||||||
|
|
||||||
def _run_orchestrator_container(self, current_hash: str) -> None:
|
def _infra_source_current(self, current_hash: str) -> bool:
|
||||||
"""Start the control-plane container (idempotent: clears a stale
|
"""True iff the running infra container was started from the current
|
||||||
fixed-name container first). Register-only broker → no docker socket.
|
bind-mounted source. Mirrors the macOS backend's `_source_current`."""
|
||||||
Labels the container with `current_hash` so a later `ensure_running`
|
if not self._container_running(self._infra_name):
|
||||||
can detect a real code change (see `source_hash`)."""
|
|
||||||
run_docker(["docker", "rm", "--force", self._orchestrator_name])
|
|
||||||
proc = run_docker([
|
|
||||||
"docker", "run", "--detach",
|
|
||||||
"--name", self._orchestrator_name,
|
|
||||||
"--label", self._orchestrator_label,
|
|
||||||
"--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current_hash}",
|
|
||||||
"--network", self.network,
|
|
||||||
# Host CLI reaches the control plane here; bound to loopback so it
|
|
||||||
# is not exposed on the host's external interfaces. NOTE: the
|
|
||||||
# container is still on `self.network` (the shared gateway network),
|
|
||||||
# so agents can reach it by container IP — which is exactly why the
|
|
||||||
# control plane requires the secret below rather than trusting the
|
|
||||||
# network boundary.
|
|
||||||
"--publish", f"127.0.0.1:{self.port}:{self.port}",
|
|
||||||
"--volume", f"{self._repo_root}:{_APP_DIR}:ro",
|
|
||||||
"--workdir", _APP_DIR,
|
|
||||||
# Persist the registry DB on the host (sole-owner: only the
|
|
||||||
# orchestrator opens bot-bottle.db).
|
|
||||||
"--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}",
|
|
||||||
"--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}",
|
|
||||||
# The control-plane secret it requires on every route but /health.
|
|
||||||
# Bare `--env NAME` → docker inherits the value from the run env
|
|
||||||
# below, so the secret never lands on argv / `docker inspect`.
|
|
||||||
"--env", CONTROL_PLANE_TOKEN_ENV,
|
|
||||||
"--entrypoint", "python3",
|
|
||||||
self.image,
|
|
||||||
"-m", "bot_bottle.orchestrator",
|
|
||||||
"--host", "0.0.0.0", "--port", str(self.port), "--broker", "stub",
|
|
||||||
], env={**os.environ, CONTROL_PLANE_TOKEN_ENV: host_control_plane_token()})
|
|
||||||
if proc.returncode != 0:
|
|
||||||
raise OrchestratorStartError(
|
|
||||||
f"orchestrator container failed to start: {proc.stderr.strip()}"
|
|
||||||
)
|
|
||||||
|
|
||||||
def _gateway(self) -> DockerGateway:
|
|
||||||
return DockerGateway(
|
|
||||||
self._gateway_image,
|
|
||||||
name=self._gateway_name,
|
|
||||||
network=self.network,
|
|
||||||
orchestrator_url=self.internal_url,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _ensure_orchestrator_image(self) -> None:
|
|
||||||
"""Build the lean control-plane image from `Dockerfile.orchestrator`
|
|
||||||
when it's missing (#384). Cheap — a `FROM python:*-slim` base with no
|
|
||||||
deps to install, so the layer cache makes rebuilds a no-op. Unlike the
|
|
||||||
gateway image this is build-if-missing, not build-every-time: the
|
|
||||||
control plane bind-mounts its source, so a code change is caught by the
|
|
||||||
source-hash recreate (below), not by an image rebuild."""
|
|
||||||
if run_docker(["docker", "image", "inspect", self.image]).returncode == 0:
|
|
||||||
return
|
|
||||||
argv = ["docker", "build", "-t", self.image,
|
|
||||||
"-f", str(self._repo_root / ORCHESTRATOR_DOCKERFILE),
|
|
||||||
str(self._repo_root)]
|
|
||||||
if os.environ.get("BOT_BOTTLE_NO_CACHE"):
|
|
||||||
argv.insert(2, "--no-cache")
|
|
||||||
proc = run_docker(argv)
|
|
||||||
if proc.returncode != 0:
|
|
||||||
raise GatewayError(
|
|
||||||
f"orchestrator image build failed: {proc.stderr.strip()}"
|
|
||||||
)
|
|
||||||
|
|
||||||
def _orchestrator_source_current(self, current_hash: str) -> bool:
|
|
||||||
"""True iff the running orchestrator container was created from the
|
|
||||||
*current* bind-mounted source. Mirrors `DockerGateway`'s
|
|
||||||
image-staleness check, but by content hash rather than image id since
|
|
||||||
the orchestrator runs bind-mounted source, not a built image."""
|
|
||||||
if not self._container_running(self._orchestrator_name):
|
|
||||||
return False
|
return False
|
||||||
proc = run_docker([
|
proc = run_docker([
|
||||||
"docker", "inspect", "--format",
|
"docker", "inspect", "--format",
|
||||||
"{{ index .Config.Labels \"" + ORCHESTRATOR_SOURCE_HASH_LABEL + "\" }}",
|
"{{ index .Config.Labels \"" + INFRA_SOURCE_HASH_LABEL + "\" }}",
|
||||||
self._orchestrator_name,
|
self._infra_name,
|
||||||
])
|
])
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
return True # can't compare -> don't churn a working container
|
return True # can't compare → don't churn a working container
|
||||||
return proc.stdout.strip() == current_hash
|
return proc.stdout.strip() == current_hash
|
||||||
|
|
||||||
|
def _ensure_network(self) -> None:
|
||||||
|
if run_docker(["docker", "network", "inspect", self.network]).returncode == 0:
|
||||||
|
return
|
||||||
|
proc = run_docker(["docker", "network", "create", self.network])
|
||||||
|
if proc.returncode != 0 and "already exists" not in proc.stderr:
|
||||||
|
raise GatewayError(
|
||||||
|
f"gateway network {self.network} failed to create: {proc.stderr.strip()}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _build_images(self) -> None:
|
||||||
|
"""Build the gateway base, the orchestrator intermediate, then the
|
||||||
|
infra image. All are cache-aware: a no-op when nothing changed."""
|
||||||
|
for tag, dockerfile in (
|
||||||
|
(GATEWAY_IMAGE, GATEWAY_DOCKERFILE),
|
||||||
|
(ORCHESTRATOR_IMAGE, ORCHESTRATOR_DOCKERFILE),
|
||||||
|
(self.image, INFRA_DOCKERFILE),
|
||||||
|
):
|
||||||
|
argv = ["docker", "build", "-t", tag,
|
||||||
|
"-f", str(self._repo_root / dockerfile),
|
||||||
|
str(self._repo_root)]
|
||||||
|
if os.environ.get("BOT_BOTTLE_NO_CACHE"):
|
||||||
|
argv.insert(2, "--no-cache")
|
||||||
|
proc = run_docker(argv)
|
||||||
|
if proc.returncode != 0:
|
||||||
|
raise GatewayError(f"{dockerfile} build failed: {proc.stderr.strip()}")
|
||||||
|
|
||||||
|
def _run_infra_container(self, current_hash: str) -> None:
|
||||||
|
"""Start the combined infra container (idempotent: clears a stale
|
||||||
|
fixed-name container first). Labels the container with `current_hash`
|
||||||
|
so a later `ensure_running` can detect a real code change."""
|
||||||
|
self._ensure_network()
|
||||||
|
run_docker(["docker", "rm", "--force", self._infra_name])
|
||||||
|
proc = run_docker([
|
||||||
|
"docker", "run", "--detach",
|
||||||
|
"--name", self._infra_name,
|
||||||
|
"--label", self._infra_label,
|
||||||
|
"--label", f"{INFRA_SOURCE_HASH_LABEL}={current_hash}",
|
||||||
|
"--network", self.network,
|
||||||
|
# Host CLI reaches the control plane here (loopback only).
|
||||||
|
# gateway_init always starts the orchestrator on DEFAULT_PORT (8099)
|
||||||
|
# inside the container; self.port is the host-side published port.
|
||||||
|
"--publish", f"127.0.0.1:{self.port}:{DEFAULT_PORT}",
|
||||||
|
# Persist the mitmproxy CA so it survives container recreation.
|
||||||
|
"--volume", f"{GATEWAY_CA_VOLUME}:{MITMPROXY_HOME}",
|
||||||
|
# Shared supervise DB (same file the operator reads over HTTP).
|
||||||
|
"--volume", f"{_host_db_dir()}:{_SUPERVISE_DB_DIR_IN_CONTAINER}",
|
||||||
|
"--env", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}",
|
||||||
|
# Live control-plane source, mounted to a path that does not
|
||||||
|
# overlay the gateway's baked /app scripts.
|
||||||
|
"--volume", f"{self._repo_root}:{_SRC_IN_CONTAINER}:ro",
|
||||||
|
# PYTHONPATH lets the orchestrator (and other Python daemons)
|
||||||
|
# import the live source ahead of the installed package.
|
||||||
|
"--env", f"PYTHONPATH={_SRC_IN_CONTAINER}",
|
||||||
|
# Orchestrator registry DB on the host (sole writer: control plane).
|
||||||
|
"--volume", f"{self._host_root}:{_ROOT_IN_CONTAINER}",
|
||||||
|
"--env", f"BOT_BOTTLE_ROOT={_ROOT_IN_CONTAINER}",
|
||||||
|
# Control-plane secret: required by the orchestrator (to enforce)
|
||||||
|
# and by the gateway daemons (to present on /resolve calls).
|
||||||
|
"--env", CONTROL_PLANE_TOKEN_ENV,
|
||||||
|
# Gateway daemons reach the orchestrator over loopback at its
|
||||||
|
# fixed internal port (DEFAULT_PORT), independent of self.port.
|
||||||
|
"--env", f"BOT_BOTTLE_ORCHESTRATOR_URL=http://127.0.0.1:{DEFAULT_PORT}",
|
||||||
|
# Opt the orchestrator into gateway_init's supervise tree.
|
||||||
|
"--env", f"BOT_BOTTLE_GATEWAY_DAEMONS={_INFRA_DAEMONS}",
|
||||||
|
self.image,
|
||||||
|
], env={**os.environ, CONTROL_PLANE_TOKEN_ENV: host_control_plane_token()})
|
||||||
|
if proc.returncode != 0:
|
||||||
|
raise OrchestratorStartError(
|
||||||
|
f"infra container failed to start: {proc.stderr.strip()}"
|
||||||
|
)
|
||||||
|
|
||||||
def ensure_running(
|
def ensure_running(
|
||||||
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Ensure the control plane + shared gateway are up; return the host
|
"""Ensure the infra container (control plane + gateway) is up; return
|
||||||
control-plane URL. Idempotent — a healthy control plane running
|
the host control-plane URL. Idempotent — a healthy container on current
|
||||||
current code and a running gateway are left untouched. Raises
|
source is left untouched. Raises `OrchestratorStartError` on timeout."""
|
||||||
`OrchestratorStartError` on timeout."""
|
self._build_images()
|
||||||
gateway = self._gateway()
|
|
||||||
gateway.ensure_built() # rebuild the bundle image on a source change
|
|
||||||
gateway.ensure_running() # creates the shared network + (re)starts gateway
|
|
||||||
|
|
||||||
# Recreate the orchestrator container only when its bind-mounted
|
|
||||||
# source has actually changed since it started — its Python process
|
|
||||||
# loaded that code at startup and won't reload, so a stale container
|
|
||||||
# would keep running OLD control-plane code. Recreating on *every*
|
|
||||||
# launch (the prior behaviour) would drop every other active
|
|
||||||
# bottle's in-memory egress tokens each time a new bottle starts,
|
|
||||||
# since the orchestrator process holds them only in memory (#381).
|
|
||||||
current_hash = source_hash(self._repo_root)
|
current_hash = source_hash(self._repo_root)
|
||||||
if self.is_healthy() and self._orchestrator_source_current(current_hash):
|
if self.is_healthy() and self._infra_source_current(current_hash):
|
||||||
return self.url
|
return self.url
|
||||||
|
|
||||||
self._ensure_orchestrator_image()
|
log.info("starting infra container", context={"name": self._infra_name})
|
||||||
log.info(
|
self._run_infra_container(current_hash)
|
||||||
"starting orchestrator container",
|
|
||||||
context={"name": self._orchestrator_name},
|
|
||||||
)
|
|
||||||
self._run_orchestrator_container(current_hash)
|
|
||||||
|
|
||||||
deadline = time.monotonic() + startup_timeout
|
deadline = time.monotonic() + startup_timeout
|
||||||
while time.monotonic() < deadline:
|
while time.monotonic() < deadline:
|
||||||
if self.is_healthy():
|
if self.is_healthy():
|
||||||
log.info("orchestrator healthy", context={"url": self.url})
|
log.info("infra container healthy", context={"url": self.url})
|
||||||
return self.url
|
return self.url
|
||||||
time.sleep(_HEALTH_POLL_SECONDS)
|
time.sleep(_HEALTH_POLL_SECONDS)
|
||||||
raise OrchestratorStartError(
|
raise OrchestratorStartError(
|
||||||
f"orchestrator at {self.url} did not become healthy within {startup_timeout:g}s"
|
f"infra container at {self.url} did not become healthy within {startup_timeout:g}s"
|
||||||
)
|
)
|
||||||
|
|
||||||
def stop(self) -> None:
|
def stop(self) -> None:
|
||||||
"""Remove the orchestrator + gateway containers (idempotent)."""
|
"""Remove the infra container (idempotent)."""
|
||||||
run_docker(["docker", "rm", "--force", self._orchestrator_name])
|
run_docker(["docker", "rm", "--force", self._infra_name])
|
||||||
self._gateway().stop()
|
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"OrchestratorService",
|
"OrchestratorService",
|
||||||
"OrchestratorStartError",
|
"OrchestratorStartError",
|
||||||
"ORCHESTRATOR_NAME",
|
"INFRA_NAME",
|
||||||
|
"INFRA_IMAGE",
|
||||||
|
"INFRA_SOURCE_HASH_LABEL",
|
||||||
"ORCHESTRATOR_IMAGE",
|
"ORCHESTRATOR_IMAGE",
|
||||||
"DEFAULT_PORT",
|
"DEFAULT_PORT",
|
||||||
"DEFAULT_STARTUP_TIMEOUT_SECONDS",
|
"DEFAULT_STARTUP_TIMEOUT_SECONDS",
|
||||||
|
"source_hash",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -0,0 +1,156 @@
|
|||||||
|
# PRD prd-new: Consolidate infra backend for Docker
|
||||||
|
|
||||||
|
- **Status:** Active
|
||||||
|
- **Author:** Claude
|
||||||
|
- **Created:** 2026-07-20
|
||||||
|
- **Issue:** #431
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
The Docker backend runs two containers — `bot-bottle-orch-gateway` (gateway
|
||||||
|
data plane) and `bot-bottle-orchestrator` (control plane) — where the
|
||||||
|
macOS and Firecracker backends already run a single combined infra
|
||||||
|
unit. This PRD collapses Docker to the same model: one `bot-bottle-infra`
|
||||||
|
container running both processes under the `gateway_init` supervise tree, a
|
||||||
|
restructured `Dockerfile.infra` as the shared gateway+orchestrator base,
|
||||||
|
and a handful of extracted shared utilities (CA cert polling, teardown
|
||||||
|
sequence, launch skeleton) that are currently duplicated across all three
|
||||||
|
`consolidated_launch.py` files.
|
||||||
|
|
||||||
|
## Goals / success criteria
|
||||||
|
|
||||||
|
- Docker backend starts exactly one infra container instead of two.
|
||||||
|
- `Dockerfile.infra` is the shared base image (gateway + orchestrator, no
|
||||||
|
buildah); the Firecracker image layers buildah on top of it.
|
||||||
|
- The orchestrator process runs under the `gateway_init` supervise tree
|
||||||
|
inside the combined container (one PID-1, one restart/health surface).
|
||||||
|
- CA cert polling, the teardown sequence, and the shared launch skeleton
|
||||||
|
(ensure-infra → register → provision → return context) live in a single
|
||||||
|
shared module; all three backends import from it.
|
||||||
|
- No functional change to macOS or Firecracker launch paths.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- Changing per-bottle isolation — agents stay one-VM/container-each.
|
||||||
|
- Consolidating transport implementations (`DockerGatewayTransport`,
|
||||||
|
`AppleGatewayTransport`, `SshGatewayTransport`) — these are already the
|
||||||
|
right abstraction boundary.
|
||||||
|
- macOS DHCP-inversion of registration order — irreducible backend
|
||||||
|
difference, stays as-is.
|
||||||
|
- Any changes to the orchestrator RPC protocol or the attribution model.
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
### Dockerfile restructuring
|
||||||
|
|
||||||
|
**Current shape:**
|
||||||
|
|
||||||
|
- `Dockerfile.gateway` — data plane (mitmproxy, gitleaks, git, openssh,
|
||||||
|
supervise daemons)
|
||||||
|
- `Dockerfile.orchestrator` — control plane (python:3.12-slim + bot_bottle
|
||||||
|
package; stdlib-only, no third-party deps)
|
||||||
|
- `Dockerfile.infra` — Firecracker only: `FROM bot-bottle-gateway` +
|
||||||
|
buildah + `COPY --from bot-bottle-orchestrator`
|
||||||
|
|
||||||
|
**New shape:**
|
||||||
|
|
||||||
|
- `Dockerfile.gateway` — unchanged
|
||||||
|
- `Dockerfile.orchestrator` — unchanged (single definition of orchestrator
|
||||||
|
content; both Docker infra and Firecracker infra `COPY --from` it)
|
||||||
|
- `Dockerfile.infra` — **shared base**: `FROM bot-bottle-gateway` + `COPY
|
||||||
|
--from bot-bottle-orchestrator` (no buildah — Docker infra image)
|
||||||
|
- `Dockerfile.infra.fc` — Firecracker only: `FROM bot-bottle-infra` +
|
||||||
|
buildah/crun/netavark/aardvark-dns (layered on the shared base, same net
|
||||||
|
result as today)
|
||||||
|
|
||||||
|
The comment in `Dockerfile.infra` that says "the docker backend keeps
|
||||||
|
orchestrator + gateway as separate images; this combined image exists only
|
||||||
|
for the Firecracker single-VM cut" is removed.
|
||||||
|
|
||||||
|
### Orchestrator in the supervise tree
|
||||||
|
|
||||||
|
`gateway_init` already supervises the data-plane daemons (egress, git-http,
|
||||||
|
supervise-MCP). The orchestrator control plane is added as another supervised
|
||||||
|
process: `python3 -m bot_bottle.orchestrator --host 0.0.0.0 --port <port>
|
||||||
|
--broker stub`.
|
||||||
|
|
||||||
|
The orchestrator source is bind-mounted (`/app` → repo root, as today) so
|
||||||
|
dev live-reload still works. `source_hash`-based container recreation in
|
||||||
|
`OrchestratorService.ensure_running` continues to apply — a code change
|
||||||
|
recreates the combined infra container, which bounces both gateway and
|
||||||
|
orchestrator. This is acceptable: the docker backend is a dev/legacy target
|
||||||
|
where in-flight egress connections across a code deploy are not a hard
|
||||||
|
requirement.
|
||||||
|
|
||||||
|
### `OrchestratorService` changes
|
||||||
|
|
||||||
|
`OrchestratorService` currently starts two containers in sequence: gateway
|
||||||
|
first (`DockerGateway.ensure_running`), then orchestrator. After this PRD:
|
||||||
|
|
||||||
|
- Single `docker run` of `bot-bottle-infra:latest`
|
||||||
|
- Container name: `bot-bottle-infra` (replaces `bot-bottle-orch-gateway` +
|
||||||
|
`bot-bottle-orchestrator`)
|
||||||
|
- Published ports: `127.0.0.1:{host_port}:8099` for the control plane
|
||||||
|
(`gateway_init` listens on a fixed internal port 8099; the caller-chosen
|
||||||
|
host port maps to it)
|
||||||
|
- Bind mounts: repo root + host root (same as today)
|
||||||
|
- `DockerGateway` becomes an implementation detail of `OrchestratorService`
|
||||||
|
rather than a separately started container; the gateway image name
|
||||||
|
(`GATEWAY_IMAGE`) is no longer referenced at runtime, only at build time
|
||||||
|
for the `Dockerfile.infra` base
|
||||||
|
|
||||||
|
The `_gateway()` / `ensure_running` two-step in `OrchestratorService` is
|
||||||
|
replaced by a single `_run_infra_container()`.
|
||||||
|
|
||||||
|
### Shared backend utilities
|
||||||
|
|
||||||
|
Three items are duplicated across
|
||||||
|
`backend/docker/consolidated_launch.py`,
|
||||||
|
`backend/macos_container/consolidated_launch.py`, and
|
||||||
|
`backend/firecracker/consolidated_launch.py`:
|
||||||
|
|
||||||
|
1. **CA cert polling loop** — `deadline = time.monotonic() + timeout; while
|
||||||
|
...: try fetch CA; sleep` — extracted to
|
||||||
|
`backend/consolidated_util.py:poll_ca_cert(transport, *, timeout)`.
|
||||||
|
|
||||||
|
2. **Teardown sequence** — `OrchestratorClient(url).teardown_bottle(id)` +
|
||||||
|
`deprovision_git_gate(transport, id)` — extracted to
|
||||||
|
`backend/consolidated_util.py:teardown_consolidated(url, transport,
|
||||||
|
bottle_id)`.
|
||||||
|
|
||||||
|
3. **Launch skeleton** — all three follow: ensure-infra → allocate/register
|
||||||
|
→ provision git-gate → fetch CA cert → return launch context. The macOS
|
||||||
|
inversion (agent starts before registration, source IP from DHCP) is the
|
||||||
|
only deviation. Extract a shared `_provision_bottle(transport, bottle_id,
|
||||||
|
plan, orchestrator_url)` helper covering the register → provision →
|
||||||
|
return-token steps; the backends keep their own `launch_consolidated`
|
||||||
|
wrappers for the before/after (infra-ensure + agent-start + IP
|
||||||
|
allocation), calling the shared helper.
|
||||||
|
|
||||||
|
The new `backend/consolidated_util.py` module holds only backend-neutral,
|
||||||
|
transport-agnostic logic. All three backends import from it.
|
||||||
|
|
||||||
|
## Implementation chunks
|
||||||
|
|
||||||
|
1. **(this PR)** Dockerfile restructuring: rename current `Dockerfile.infra`
|
||||||
|
content to `Dockerfile.infra.fc`; write new `Dockerfile.infra` as
|
||||||
|
gateway+orchestrator base. Update Firecracker image-build references from
|
||||||
|
`Dockerfile.infra` → `Dockerfile.infra.fc`.
|
||||||
|
|
||||||
|
2. Add orchestrator process to `gateway_init` supervise tree.
|
||||||
|
|
||||||
|
3. Collapse `OrchestratorService` to a single-container start; rename
|
||||||
|
container from `bot-bottle-orch-gateway`/`bot-bottle-orchestrator` →
|
||||||
|
`bot-bottle-infra`; update image name constant.
|
||||||
|
|
||||||
|
4. Extract `backend/consolidated_util.py` with `poll_ca_cert`,
|
||||||
|
`teardown_consolidated`, and `_provision_bottle`; update all three
|
||||||
|
`consolidated_launch.py` files to import from it.
|
||||||
|
|
||||||
|
5. Update tests that reference the old container names or two-container
|
||||||
|
startup sequence.
|
||||||
|
|
||||||
|
## Open questions
|
||||||
|
|
||||||
|
None — the supervise-tree approach and shared Dockerfile layering were
|
||||||
|
confirmed in issue #431.
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
# Egress proxy OOMs on large downloads
|
|
||||||
|
|
||||||
Found on 2026-07-21 while running the rootless-podman spike
|
|
||||||
(`docs/research/rootless-docker-in-apple-container-spike.md`). Recorded
|
|
||||||
rather than fixed — the fix is a security-relevant decision, not a
|
|
||||||
mechanical patch.
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
A single large HTTPS download through the gateway kills the egress
|
|
||||||
proxy. `mitmdump` buffers whole response bodies so the DLP detectors can
|
|
||||||
scan them, grows past the gateway container's memory limit, and is
|
|
||||||
OOM-killed by the cgroup. Nothing restarts it.
|
|
||||||
|
|
||||||
Two properties make this worse than a failed download:
|
|
||||||
|
|
||||||
- **The gateway is a per-host singleton.** Every bottle shares it, so
|
|
||||||
one bottle's download takes egress away from all of them.
|
|
||||||
- **There is no restart on death.** The gateway supervisor is
|
|
||||||
`while : ; do wait ; done`; a killed daemon stays dead until the infra
|
|
||||||
container is recreated.
|
|
||||||
|
|
||||||
So ordinary agent activity — pulling a container image, downloading a
|
|
||||||
model or dataset, fetching a large tarball — is a denial of service
|
|
||||||
against every other bottle on the host. No malice required, though it is
|
|
||||||
trivially reachable on purpose.
|
|
||||||
|
|
||||||
## Evidence
|
|
||||||
|
|
||||||
Triggered by `docker compose up` pulling `quay.io/fedora/python-312`
|
|
||||||
(two layers, ~82MB and ~83MB) inside a bottle. The pull itself
|
|
||||||
succeeded; the *next* request failed:
|
|
||||||
|
|
||||||
```
|
|
||||||
initializing source docker://quay.io/fedora/python-312:latest:
|
|
||||||
pinging container registry quay.io: Get "https://quay.io/v2/":
|
|
||||||
proxyconnect tcp: dial tcp 192.168.128.39:9099: connect: connection refused
|
|
||||||
```
|
|
||||||
|
|
||||||
From the gateway's `dmesg`:
|
|
||||||
|
|
||||||
```
|
|
||||||
python3 invoked oom-killer: gfp_mask=0x100cca(GFP_HIGHUSER_MOVABLE), order=0
|
|
||||||
oom-kill:constraint=CONSTRAINT_MEMCG,
|
|
||||||
oom_memcg=/container/bot-bottle-mac-infra,
|
|
||||||
task_memcg=/container/bot-bottle-mac-infra,task=mitmdump,pid=118
|
|
||||||
Memory cgroup out of memory: Killed process 118 (mitmdump)
|
|
||||||
total-vm:1391936kB, anon-rss:997768kB
|
|
||||||
```
|
|
||||||
|
|
||||||
~1GB RSS against a 1024MB container. Note the amplification: ~165MB of
|
|
||||||
layers produced ~1GB of resident memory, so the buffering is several
|
|
||||||
copies deep (encoded body, decoded body, and the text conversion the
|
|
||||||
regex detectors scan).
|
|
||||||
|
|
||||||
Afterwards the gateway container was still running and healthy-looking —
|
|
||||||
orchestrator, supervise, and git-http all alive — with no `mitmdump`
|
|
||||||
process at all, and it stayed that way until the container was
|
|
||||||
recreated. A liveness check on the container would not have caught this.
|
|
||||||
|
|
||||||
## Reproduction
|
|
||||||
|
|
||||||
1. Launch any bottle with an egress route to a host serving a large file.
|
|
||||||
2. Download >~150MB over HTTPS through the proxy.
|
|
||||||
3. `dmesg | grep -i oom` inside `bot-bottle-mac-infra`, and note that no
|
|
||||||
`mitmdump` process remains.
|
|
||||||
|
|
||||||
Beware a false negative when checking: truncating the process listing
|
|
||||||
(`cut -c1-45`) cuts before the binary name, because `mitmdump` runs as
|
|
||||||
`/usr/local/bin/python3.12 /usr/local/bin/mitmdump …`.
|
|
||||||
|
|
||||||
## Fix options, not yet chosen
|
|
||||||
|
|
||||||
1. **Restart dead daemons.** Smallest change and strictly an
|
|
||||||
improvement: an OOM then degrades one download instead of removing
|
|
||||||
egress for every bottle. Does not stop the OOM.
|
|
||||||
2. **Cap the scanned body size.** Above a threshold, stop buffering —
|
|
||||||
either skip the scan or stream it. This is the root-cause fix and a
|
|
||||||
security decision: a size threshold is exactly the hole an exfiltrator
|
|
||||||
would aim for, so "skip above N" trades a DoS for a covert channel.
|
|
||||||
Streaming with a bounded window keeps coverage, at more complexity.
|
|
||||||
3. **Raise the gateway's memory limit.** Moves the threshold; does not
|
|
||||||
remove it.
|
|
||||||
|
|
||||||
Worth noting that (1) and (2) are complementary — the restart gap is
|
|
||||||
worth closing regardless of how the memory behaviour is resolved.
|
|
||||||
@@ -1,353 +0,0 @@
|
|||||||
# Rootless Docker inside Apple Container bottles
|
|
||||||
|
|
||||||
Spike branch: `spike/rootless-docker-macos` (`a4d8461`)
|
|
||||||
|
|
||||||
## Summary
|
|
||||||
|
|
||||||
**Negative result.** Rootless Docker cannot run inside an Apple
|
|
||||||
Container bottle without granting the bottle `CAP_SYS_ADMIN`. This is a
|
|
||||||
kernel constraint on writing multi-range `uid_map`, not a packaging gap
|
|
||||||
we can close with a better init script, a different base image, or more
|
|
||||||
careful `/etc/subuid` handling.
|
|
||||||
|
|
||||||
The spike was built on the premise — stated in
|
|
||||||
`bot_bottle/backend/macos_container/rootless_docker.py` — that it would
|
|
||||||
*"deliberately refuse to compensate for missing prerequisites with outer
|
|
||||||
capabilities, a privileged container, or a host Docker socket."* That
|
|
||||||
premise is exactly what the experiment falsified. The two ways forward
|
|
||||||
are to abandon the premise (add `CAP_SYS_ADMIN` to the bottle, and with
|
|
||||||
it most of the isolation the bottle exists to provide) or to abandon
|
|
||||||
rootless Docker.
|
|
||||||
|
|
||||||
Recommendation: abandon rootless Docker. Podman does not have this
|
|
||||||
problem — see [Podman is not blocked by
|
|
||||||
this](#podman-is-not-blocked-by-this) below.
|
|
||||||
|
|
||||||
## Local environment
|
|
||||||
|
|
||||||
Tested on 2026-07-21:
|
|
||||||
|
|
||||||
```console
|
|
||||||
$ sw_vers
|
|
||||||
ProductName: macOS
|
|
||||||
ProductVersion: 26.5.1
|
|
||||||
BuildVersion: 25F80
|
|
||||||
|
|
||||||
$ container --version
|
|
||||||
container CLI version 1.0.0 (build: release, commit: ee848e3)
|
|
||||||
|
|
||||||
$ uname -a # inside the bottle
|
|
||||||
Linux ... 6.18.15 #1 SMP Tue Mar 17 01:36:53 UTC 2026 aarch64 GNU/Linux
|
|
||||||
```
|
|
||||||
|
|
||||||
## The failure
|
|
||||||
|
|
||||||
`tests/integration/test_macos_rootless_docker_spike.py` builds the
|
|
||||||
image, launches the bottle, and dies in `rootless_docker.start`:
|
|
||||||
|
|
||||||
```
|
|
||||||
+ exec rootlesskit --net=slirp4netns --mtu=65520 ... dockerd-rootless.sh
|
|
||||||
[rootlesskit:parent] error: failed to setup UID/GID map:
|
|
||||||
newuidmap 1100 [0 1000 1 1 100000 65536] failed:
|
|
||||||
newuidmap: write to uid_map failed: Operation not permitted
|
|
||||||
```
|
|
||||||
|
|
||||||
## Why it fails
|
|
||||||
|
|
||||||
Every prerequisite you would normally suspect is present and correct in
|
|
||||||
the guest:
|
|
||||||
|
|
||||||
| Check | Result |
|
|
||||||
| --- | --- |
|
|
||||||
| `/usr/bin/newuidmap` | `-rwsr-xr-x root root` — setuid bit intact, survived the OCI export |
|
|
||||||
| `/` mount options | `rw,relatime` — **not** `nosuid` |
|
|
||||||
| `NoNewPrivs` | `0` |
|
|
||||||
| `Seccomp` | `0`, no filters |
|
|
||||||
| `/etc/subuid`, `/etc/subgid` | `node:100000:65536` in both |
|
|
||||||
| user namespace | `user:[4026531837]`, identical to pid 1 — the *initial* userns |
|
|
||||||
| `unshare -U -r true` | succeeds |
|
|
||||||
| `/proc/sys/user/max_user_namespaces` | `4505` |
|
|
||||||
|
|
||||||
The one thing that is missing is in the capability bounding set that
|
|
||||||
Apple Container gives the container:
|
|
||||||
|
|
||||||
```
|
|
||||||
CapBnd: 00000000a80425fb
|
|
||||||
= chown, dac_override, fowner, fsetid, kill, setgid, setuid, setpcap,
|
|
||||||
net_bind_service, net_raw, sys_chroot, mknod, audit_write, setfcap
|
|
||||||
```
|
|
||||||
|
|
||||||
No `CAP_SYS_ADMIN`. That is the whole story, and the chain is:
|
|
||||||
|
|
||||||
1. The kernel's `map_write()` gates writing a `uid_map` on
|
|
||||||
`file_ns_capable(file, ns, CAP_SYS_ADMIN)` — capability over the
|
|
||||||
**new** user namespace, evaluated against the credentials that opened
|
|
||||||
`/proc/<pid>/uid_map`.
|
|
||||||
2. `newuidmap` is setuid-root, so it runs with euid 0 — but its
|
|
||||||
capability sets are clamped by the bounding set, which has no
|
|
||||||
`CAP_SYS_ADMIN`.
|
|
||||||
3. `cap_capable()` has a shortcut that grants *all* capabilities when
|
|
||||||
the caller's userns is the new namespace's parent **and**
|
|
||||||
`ns->owner == cred->euid`. It does not apply: the namespace was
|
|
||||||
created by `node` (uid 1000) while `newuidmap` runs as euid 0.
|
|
||||||
4. So the check falls through to the effective-set test in the initial
|
|
||||||
userns, which fails. `EPERM`.
|
|
||||||
|
|
||||||
Note that the single-line unprivileged path (`unshare -U -r`) works
|
|
||||||
precisely because it does not go through `newuidmap` and does not need
|
|
||||||
`CAP_SYS_ADMIN`. Only the multi-range subuid mapping that rootless
|
|
||||||
Docker requires does.
|
|
||||||
|
|
||||||
This is the same constraint that makes upstream's `dind-rootless` image
|
|
||||||
require `--privileged`. It is not specific to Apple Container, except
|
|
||||||
that Apple Container gives us no bounding set that includes
|
|
||||||
`CAP_SYS_ADMIN` by default.
|
|
||||||
|
|
||||||
## It does work with the capability — which is the point
|
|
||||||
|
|
||||||
Adding the capability clears the failure immediately, and exposes one
|
|
||||||
further, much smaller blocker: `/dev/net/tun` exists (the kernel has
|
|
||||||
tun; `/proc/misc` lists `200 tun`) but Apple Container creates it
|
|
||||||
`crw------- root root`, so uid 1000 cannot open it and `slirp4netns`
|
|
||||||
fails with `open: Permission denied`. A `chmod 0666 /dev/net/tun` as
|
|
||||||
root inside the bottle fixes that, and needs no capability beyond what
|
|
||||||
the bottle already has.
|
|
||||||
|
|
||||||
With both applied by hand, the daemon comes up completely:
|
|
||||||
|
|
||||||
```console
|
|
||||||
$ container run --rm -u root --cap-add CAP_SYS_ADMIN \
|
|
||||||
bot-bottle-claude:latest-rootless-docker sh -c '...'
|
|
||||||
Server Version: 20.10.24+dfsg1
|
|
||||||
Storage Driver: fuse-overlayfs
|
|
||||||
Cgroup Driver: none
|
|
||||||
Cgroup Version: 2
|
|
||||||
API listen on /tmp/rt/docker.sock
|
|
||||||
```
|
|
||||||
|
|
||||||
So `rootless-docker-init.sh` and `rootless_docker.py` are *correct*.
|
|
||||||
The spike did not fail on a bug. It failed on its own premise.
|
|
||||||
|
|
||||||
Two secondary findings from that run, relevant if anyone revisits this:
|
|
||||||
|
|
||||||
- Debian's `docker.io` package pins Docker **20.10** (EOL), not the 28.x
|
|
||||||
implied by the `docker:28-cli` compose plugin the image copies in.
|
|
||||||
- `Cgroup Driver: none` — no resource limits on nested containers.
|
|
||||||
|
|
||||||
## Why we should not just add the capability
|
|
||||||
|
|
||||||
`CAP_SYS_ADMIN` is close to a superset of "root" in practical terms —
|
|
||||||
mount, `pivot_root`, namespace manipulation, and a long tail of
|
|
||||||
subsystem-specific powers. Granting it to the agent bottle would
|
|
||||||
undercut the containment argument the rest of the backend is built
|
|
||||||
around, including the deliberately narrow choices immediately adjacent
|
|
||||||
to it in `launch.py` (`--cap-drop CAP_NET_RAW`, no `NET_ADMIN`, a
|
|
||||||
host-only agent network). Trading all of that for nested `docker
|
|
||||||
compose` is a bad exchange.
|
|
||||||
|
|
||||||
## Podman is not blocked by this
|
|
||||||
|
|
||||||
Sanity-checked on the same host, same kernel, same runtime, so the
|
|
||||||
comparison is apples to apples:
|
|
||||||
|
|
||||||
| Scenario | Result |
|
|
||||||
| --- | --- |
|
|
||||||
| Podman rootless, `/etc/subuid` populated | **Fails identically** — `newuidmap: write to uid_map failed: Operation not permitted` |
|
|
||||||
| Podman rootless, no subuid ranges, `--network=host` | **Works**, no added capabilities |
|
|
||||||
| Podman rootless, no subuid ranges, default netns, `/dev/net/tun` at `0600` | Fails — `slirp4netns: open("/dev/net/tun"): Permission denied` |
|
|
||||||
| Podman rootless, no subuid ranges, default netns, `/dev/net/tun` at `0666` | **Works**, no added capabilities |
|
|
||||||
|
|
||||||
The difference is that podman degrades gracefully when no subuid range
|
|
||||||
is available: it falls back to a single-UID self-mapping, which an
|
|
||||||
unprivileged process may write itself, so `newuidmap` is never invoked
|
|
||||||
and `CAP_SYS_ADMIN` is never needed. Docker's rootless mode has no
|
|
||||||
equivalent fallback.
|
|
||||||
|
|
||||||
The cost of that fallback is real and should be weighed before building
|
|
||||||
on it: with a single-UID mapping, every UID inside a nested container
|
|
||||||
collapses onto the bottle's own uid 1000. There is no UID separation
|
|
||||||
between the agent and anything it runs — `root` in a nested container is
|
|
||||||
the agent user outside it. It also requires `ignore_chown_errors` on the
|
|
||||||
storage driver. Whether that is acceptable depends on whether the bottle
|
|
||||||
boundary (which is unchanged) or the nested-container boundary (which is
|
|
||||||
effectively nil) is the one we are relying on.
|
|
||||||
|
|
||||||
## What the podman spike then needed
|
|
||||||
|
|
||||||
The podman implementation that replaced the Docker one on this branch
|
|
||||||
turned up two more device-node blockers of the same shape as
|
|
||||||
`/dev/net/tun` — Apple Container creates the node, but 0600 root:root:
|
|
||||||
|
|
||||||
- **`/dev/fuse`** — blocks the `fuse-overlayfs` storage driver
|
|
||||||
(`fuse: failed to open /dev/fuse: Permission denied`). Without it the
|
|
||||||
only working driver is `vfs`, which copies whole layers per container.
|
|
||||||
- **`/dev/net/tun`** — blocks `slirp4netns`, which rootless podman uses
|
|
||||||
for the default bridge network.
|
|
||||||
|
|
||||||
Both are fixed by `chmod 0666` as root inside the bottle, which needs no
|
|
||||||
capability the bottle does not already hold. This is categorically
|
|
||||||
different from the `CAP_SYS_ADMIN` requirement: it is a permission on a
|
|
||||||
node that already exists, not an outer privilege grant.
|
|
||||||
|
|
||||||
One design note worth recording: the agent-facing surface stays `docker`
|
|
||||||
and `docker compose`, pointed at podman's Docker-compatible API socket
|
|
||||||
via `DOCKER_HOST`. Setting `netns="host"` in `containers.conf` does *not*
|
|
||||||
propagate through that compat API — stock `docker run` and compose files
|
|
||||||
request bridge networking explicitly — so slirp4netns (and therefore the
|
|
||||||
`/dev/net/tun` chmod) is required for ordinary compose files to work at
|
|
||||||
all. Host networking remains available per-workload via
|
|
||||||
`--network=host`.
|
|
||||||
|
|
||||||
Verified working in a bottle with zero added capabilities: fuse-overlayfs
|
|
||||||
storage, the compat API socket, `docker run` on both bridge and host
|
|
||||||
networking, and published ports.
|
|
||||||
|
|
||||||
### Nested pulls collide with our own egress DLP
|
|
||||||
|
|
||||||
The first live run got podman up and `docker compose` running, then
|
|
||||||
failed on the image pull:
|
|
||||||
|
|
||||||
```
|
|
||||||
web Pulling
|
|
||||||
initializing source docker://python:3.12-alpine: reading manifest ...
|
|
||||||
StatusCode: 403, egress DLP: Generic Bearer JWT found in body
|
|
||||||
```
|
|
||||||
|
|
||||||
This is bot-bottle's own egress scanner, not a podman problem. The
|
|
||||||
Docker registry auth flow carries a bearer JWT *by protocol*, and the
|
|
||||||
`token_patterns` detector's `Generic Bearer JWT` rule
|
|
||||||
(`Bearer\s+[A-Za-z0-9._\-]{50,}`) matches it on every pull. Any bottle
|
|
||||||
that pulls images will hit this.
|
|
||||||
|
|
||||||
The fix is per-route detector scoping, which the egress config already
|
|
||||||
supports — drop `token_patterns` on the registry hosts and keep
|
|
||||||
`known_secrets`:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"host": "registry-1.docker.io",
|
|
||||||
"dlp": {"outbound_detectors": ["known_secrets"]}}
|
|
||||||
```
|
|
||||||
|
|
||||||
That is the right trade rather than a grudging one: `known_secrets`
|
|
||||||
matches the bottle's *actual* credential values, so real exfil through a
|
|
||||||
registry host is still caught. `token_patterns` on a registry route only
|
|
||||||
ever produces protocol noise.
|
|
||||||
|
|
||||||
Worth generalising later: any manifest enabling `docker_access` needs
|
|
||||||
this on its registry routes, so it probably belongs in a shared
|
|
||||||
registry-route snippet rather than being copy-pasted per bottle.
|
|
||||||
|
|
||||||
### And then registry auth collides with the Authorization strip
|
|
||||||
|
|
||||||
With DLP scoped, the pull failed differently: `unauthorized:
|
|
||||||
authentication required`. This one is architectural.
|
|
||||||
|
|
||||||
`egress_addon.py` strips agent-set `Authorization` unconditionally
|
|
||||||
before forwarding — deliberately, so an agent cannot smuggle a
|
|
||||||
credential out in a header the DLP detectors don't recognise. A route
|
|
||||||
may carry gateway-injected auth instead, but only from a *static* token
|
|
||||||
in an env var (`auth_scheme` + `token_env`).
|
|
||||||
|
|
||||||
Docker registry auth doesn't fit that shape. The client fetches a
|
|
||||||
short-lived, per-repository-scope bearer token from `auth.docker.io` and
|
|
||||||
presents it to `registry-1.docker.io`. There is no static token to
|
|
||||||
inject, and the token the client legitimately obtained is stripped.
|
|
||||||
|
|
||||||
Measured inside a bottle, by hand:
|
|
||||||
|
|
||||||
| Step | Result |
|
|
||||||
| --- | --- |
|
|
||||||
| Fetch token from `auth.docker.io` | 200, 5409-byte token body |
|
|
||||||
| Manifest request **with** that valid token | 401 |
|
|
||||||
| Manifest request with **no** Authorization | 401 — identical |
|
|
||||||
|
|
||||||
A valid token behaves exactly like sending none, which is direct
|
|
||||||
evidence the header never arrives. Any nested-container workflow that
|
|
||||||
pulls from a registry is blocked on this, so it is not a detail that can
|
|
||||||
be deferred: pulling base images is most of what nested containers are
|
|
||||||
for.
|
|
||||||
|
|
||||||
### Registries that skip the token dance work today
|
|
||||||
|
|
||||||
Not every registry needs the stripped header. Measured directly:
|
|
||||||
|
|
||||||
| Registry | Manifest request with no `Authorization` |
|
|
||||||
| --- | --- |
|
|
||||||
| `quay.io` | 200 |
|
|
||||||
| `mcr.microsoft.com` | 200 |
|
|
||||||
| `registry.k8s.io` | 307 (redirect, no auth) |
|
|
||||||
| `ghcr.io` | 401 |
|
|
||||||
| `registry-1.docker.io` | 401 |
|
|
||||||
|
|
||||||
So "just add the registry to the bottle config" genuinely works — for
|
|
||||||
quay, MCR, registry.k8s.io, or any unauthenticated internal registry.
|
|
||||||
Docker Hub and GHCR are the ones that need the strip resolved. The
|
|
||||||
acceptance test uses quay for exactly this reason.
|
|
||||||
|
|
||||||
Resolving it for Docker Hub means picking one of:
|
|
||||||
|
|
||||||
1. **Per-route opt-in to preserve client Authorization.** Smallest
|
|
||||||
change. Note the compounding effect on exactly these routes: the DLP
|
|
||||||
scoping above already removed `token_patterns` there, so a
|
|
||||||
preserved-auth registry route is one where the agent may send bearer
|
|
||||||
tokens that neither the strip nor the pattern detector inspects.
|
|
||||||
`known_secrets` still applies, so the bottle's real credentials are
|
|
||||||
still caught.
|
|
||||||
2. **A registry-aware gateway** that performs the token dance itself and
|
|
||||||
injects the result. Preserves the invariant fully; materially more
|
|
||||||
work, and it makes the gateway speak a specific registry protocol.
|
|
||||||
3. **Pre-seed images at provision time** (host-side `container image
|
|
||||||
save` into podman storage), so bottles never pull at runtime.
|
|
||||||
Preserves the invariant, and limits nested containers to
|
|
||||||
pre-approved images — which fits the custody positioning, at the cost
|
|
||||||
of no ad-hoc `docker pull`.
|
|
||||||
4. **Stop.** Nested containers are not supported on this backend.
|
|
||||||
|
|
||||||
### Podman 4.3.1 silently swallows container exit codes
|
|
||||||
|
|
||||||
Debian bookworm — which the current agent base image is built on —
|
|
||||||
ships podman 4.3.1. Through its Docker-compatible API, `docker run`
|
|
||||||
returns 0 no matter what the container did:
|
|
||||||
|
|
||||||
| Command | podman 4.3.1 | podman 5.4.2 |
|
|
||||||
| --- | --- | --- |
|
|
||||||
| `docker run … sh -c 'exit 7'` (compat API) | **0** | 7 |
|
|
||||||
| `docker run … sh -c 'exit 0'` (compat API) | 0 | 0 |
|
|
||||||
| `podman run … sh -c 'exit 7'` (native) | 7 | 7 |
|
|
||||||
|
|
||||||
This is worse than a broken feature: every failing command an agent runs
|
|
||||||
via `docker run` reports success. A test suite, a build step, or a CI
|
|
||||||
script inside a bottle would pass while failing. It also silently
|
|
||||||
defeated the acceptance test's egress-containment assertion, which is
|
|
||||||
why that assertion now checks an in-band marker rather than an exit
|
|
||||||
code.
|
|
||||||
|
|
||||||
Podman 5.4.2 (Debian trixie) fixes it, but needs two packages that
|
|
||||||
bookworm's podman does not: `passt` (podman 5's default network tool)
|
|
||||||
and `nftables` (netavark shells out to `nft`; without it every run fails
|
|
||||||
with `unable to upgrade to tcp, received 500`). With both installed,
|
|
||||||
exit codes propagate correctly and the compat API behaves.
|
|
||||||
|
|
||||||
The open question this leaves is where podman 5 comes from, since the
|
|
||||||
agent base is bookworm-based:
|
|
||||||
|
|
||||||
1. **Move the agent images to Debian trixie.** Trixie is current stable.
|
|
||||||
Correct, and the blast radius is every bottle, not just this feature.
|
|
||||||
2. **Drop the compat socket and use podman natively** (`podman-docker`
|
|
||||||
provides a `docker` shim; compose comes from `podman-compose`).
|
|
||||||
Native podman propagates exit codes correctly even on 4.3.1. Contained
|
|
||||||
to this feature, at the cost of `docker compose` becoming
|
|
||||||
`docker-compose`/`podman-compose`.
|
|
||||||
3. **Ship bookworm's podman 4.3.1 with the compat socket** — not viable.
|
|
||||||
Silent false success is a correctness bug agents cannot see.
|
|
||||||
|
|
||||||
## Recommendation
|
|
||||||
|
|
||||||
1. Do not revive rootless Docker on this backend. This document is the
|
|
||||||
record of why.
|
|
||||||
2. Nested containers, if wanted, come from podman under the
|
|
||||||
single-mapping constraint — with the explicit understanding that the
|
|
||||||
nested-container boundary carries no security weight. `root` in a
|
|
||||||
nested container is the agent user outside it.
|
|
||||||
3. Nested containers are therefore a build/test convenience. The bottle
|
|
||||||
remains the security boundary, exactly as it was.
|
|
||||||
@@ -1,154 +0,0 @@
|
|||||||
"""Live-Mac acceptance spike for guest-local rootless podman (issue #392).
|
|
||||||
|
|
||||||
Run explicitly on an Apple Silicon/macOS 26 host:
|
|
||||||
|
|
||||||
BOT_BOTTLE_ROOTLESS_PODMAN_SPIKE=1 \
|
|
||||||
python3 -m unittest tests.integration.test_macos_rootless_podman_spike -v
|
|
||||||
|
|
||||||
The opt-in is deliberate: ordinary Linux CI cannot execute Apple Container.
|
|
||||||
|
|
||||||
Podman rather than Docker because Apple Container's capability bounding set
|
|
||||||
omits CAP_SYS_ADMIN; see
|
|
||||||
docs/research/rootless-docker-in-apple-container-spike.md. The agent-facing
|
|
||||||
surface is still `docker` and `docker compose`, which talk to podman's
|
|
||||||
Docker-compatible API socket.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import os
|
|
||||||
import platform
|
|
||||||
import shutil
|
|
||||||
import tempfile
|
|
||||||
import unittest
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from bot_bottle.backend import BottleSpec, get_bottle_backend
|
|
||||||
from bot_bottle.manifest import ManifestIndex
|
|
||||||
|
|
||||||
|
|
||||||
@unittest.skipUnless(
|
|
||||||
platform.system() == "Darwin"
|
|
||||||
and os.environ.get("BOT_BOTTLE_ROOTLESS_PODMAN_SPIKE") == "1",
|
|
||||||
"requires an explicit live-Mac rootless-podman spike run",
|
|
||||||
)
|
|
||||||
class TestMacosRootlessPodmanSpike(unittest.TestCase):
|
|
||||||
def test_compose_stays_inside_registered_bottle(self) -> None:
|
|
||||||
workspace = Path(tempfile.mkdtemp(prefix="rootless-podman-spike."))
|
|
||||||
stage = Path(tempfile.mkdtemp(prefix="rootless-podman-stage."))
|
|
||||||
try:
|
|
||||||
(workspace / "index.html").write_text("bottle-compose-ok\n")
|
|
||||||
(workspace / "compose.yaml").write_text(
|
|
||||||
"services:\n"
|
|
||||||
" web:\n"
|
|
||||||
" image: quay.io/prometheus/busybox\n"
|
|
||||||
" working_dir: /workspace\n"
|
|
||||||
" command: httpd -f -p 8000 -h /workspace\n"
|
|
||||||
" volumes: ['.:/workspace']\n"
|
|
||||||
" ports: ['18080:8000']\n",
|
|
||||||
encoding="utf-8",
|
|
||||||
)
|
|
||||||
manifest = ManifestIndex.from_json_obj({
|
|
||||||
"bottles": {"dev": {
|
|
||||||
"docker_access": True,
|
|
||||||
# A deliberately tiny image. Pulling a ~165MB one
|
|
||||||
# OOM-kills the shared egress proxy, which buffers whole
|
|
||||||
# response bodies to scan them — a real defect, but a
|
|
||||||
# separate one from what this test covers. See the
|
|
||||||
# research note.
|
|
||||||
#
|
|
||||||
# quay.io deliberately, not Docker Hub: the egress proxy
|
|
||||||
# strips agent-set Authorization (so an agent cannot
|
|
||||||
# smuggle a credential out in a header), and Docker Hub
|
|
||||||
# requires a client-fetched, per-scope bearer token that
|
|
||||||
# the strip therefore removes. quay serves manifests with
|
|
||||||
# no Authorization at all, so a plain route is enough.
|
|
||||||
#
|
|
||||||
# token_patterns is still scoped off: registry traffic
|
|
||||||
# carries bearer JWTs by protocol and trips the generic
|
|
||||||
# rule. known_secrets stays on — it matches the bottle's
|
|
||||||
# own credentials, which is the detector that catches
|
|
||||||
# real exfil.
|
|
||||||
"egress": {"routes": [
|
|
||||||
{"host": "quay.io", "dlp": {
|
|
||||||
"outbound_detectors": ["known_secrets"],
|
|
||||||
}},
|
|
||||||
{"host": "cdn01.quay.io", "dlp": {
|
|
||||||
"outbound_detectors": ["known_secrets"],
|
|
||||||
}},
|
|
||||||
]},
|
|
||||||
}},
|
|
||||||
"agents": {"spike": {
|
|
||||||
"bottle": "dev", "skills": [], "prompt": "",
|
|
||||||
}},
|
|
||||||
})
|
|
||||||
spec = BottleSpec(
|
|
||||||
manifest=manifest,
|
|
||||||
agent_name="spike",
|
|
||||||
copy_cwd=True,
|
|
||||||
user_cwd=str(workspace),
|
|
||||||
)
|
|
||||||
backend = get_bottle_backend("macos-container")
|
|
||||||
plan = backend.prepare(spec, stage_dir=stage)
|
|
||||||
with backend.launch(plan) as bottle:
|
|
||||||
workdir = plan.workspace_plan.workdir
|
|
||||||
checks = (
|
|
||||||
"docker info >/dev/null && docker compose version && "
|
|
||||||
f"cd {workdir} && docker compose up -d --wait && "
|
|
||||||
"curl --fail --silent http://127.0.0.1:18080/ | "
|
|
||||||
"grep -q bottle-compose-ok"
|
|
||||||
)
|
|
||||||
result = bottle.exec(checks)
|
|
||||||
self.assertEqual(
|
|
||||||
0, result.returncode,
|
|
||||||
f"stdout={result.stdout!r}\nstderr={result.stderr!r}",
|
|
||||||
)
|
|
||||||
# podman's compat API reports rootlessness through its own
|
|
||||||
# native endpoint; the Docker-shaped SecurityOptions field does
|
|
||||||
# not carry it.
|
|
||||||
inspect = bottle.exec(
|
|
||||||
"podman info --format '{{.Host.Security.Rootless}}'"
|
|
||||||
)
|
|
||||||
self.assertIn("true", inspect.stdout.lower())
|
|
||||||
self.assertEqual(
|
|
||||||
0,
|
|
||||||
bottle.exec(
|
|
||||||
"test \"$(id -u)\" -ne 0"
|
|
||||||
).returncode,
|
|
||||||
"the podman service must not be running as bottle root",
|
|
||||||
)
|
|
||||||
self.assertNotEqual(
|
|
||||||
0,
|
|
||||||
bottle.exec("test -S /var/run/docker.sock").returncode,
|
|
||||||
"spike must never expose a host/rootful Docker socket",
|
|
||||||
)
|
|
||||||
# Asserted on an in-band marker, not on `docker run`'s exit
|
|
||||||
# code: podman 4.3.1's Docker-compat API swallows the
|
|
||||||
# container's status and returns 0 for everything, so an
|
|
||||||
# exit-code assertion here passes whether egress was blocked
|
|
||||||
# or wide open. That silent false pass is worse than no check
|
|
||||||
# at all, and it is exactly this check — the one proving a
|
|
||||||
# nested container cannot escape the egress path.
|
|
||||||
#
|
|
||||||
# busybox ships wget, so a failure here means egress was
|
|
||||||
# refused rather than the binary being absent.
|
|
||||||
direct = bottle.exec(
|
|
||||||
"docker run --rm --env HTTP_PROXY= --env HTTPS_PROXY= "
|
|
||||||
"--env http_proxy= --env https_proxy= "
|
|
||||||
"quay.io/prometheus/busybox sh -c "
|
|
||||||
"'wget -T 4 -qO- https://evil.example.com/ "
|
|
||||||
"&& echo ESCAPED || echo CONTAINED'"
|
|
||||||
)
|
|
||||||
self.assertIn(
|
|
||||||
"CONTAINED", direct.stdout,
|
|
||||||
"an inner container obtained direct, unproxied egress: "
|
|
||||||
f"stdout={direct.stdout!r} stderr={direct.stderr!r}",
|
|
||||||
)
|
|
||||||
self.assertNotIn("ESCAPED", direct.stdout)
|
|
||||||
finally:
|
|
||||||
shutil.rmtree(workspace, ignore_errors=True)
|
|
||||||
shutil.rmtree(stage, ignore_errors=True)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -34,6 +34,7 @@ from tests._docker import skip_unless_docker
|
|||||||
# image instead of leaking a new dangling tag on every invocation.
|
# image instead of leaking a new dangling tag on every invocation.
|
||||||
_TEST_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:itest"
|
_TEST_ORCHESTRATOR_IMAGE = "bot-bottle-orchestrator:itest"
|
||||||
_TEST_GATEWAY_IMAGE = "bot-bottle-gateway:itest"
|
_TEST_GATEWAY_IMAGE = "bot-bottle-gateway:itest"
|
||||||
|
_TEST_INFRA_IMAGE = "bot-bottle-infra:itest"
|
||||||
|
|
||||||
|
|
||||||
@skip_unless_docker()
|
@skip_unless_docker()
|
||||||
@@ -69,20 +70,17 @@ class TestDockerControlPlaneAuthIntegration(unittest.TestCase):
|
|||||||
os.environ["BOT_BOTTLE_ROOT"] = cls._tmp.name
|
os.environ["BOT_BOTTLE_ROOT"] = cls._tmp.name
|
||||||
cls.addClassCleanup(_restore_root)
|
cls.addClassCleanup(_restore_root)
|
||||||
|
|
||||||
orchestrator_name = f"bot-bottle-orch-itest-{suffix}"
|
infra_name = f"bot-bottle-infra-itest-{suffix}"
|
||||||
gateway_name = f"bot-bottle-gw-itest-{suffix}"
|
|
||||||
network = f"bot-bottle-net-itest-{suffix}"
|
network = f"bot-bottle-net-itest-{suffix}"
|
||||||
host_root = Path(cls._tmp.name)
|
host_root = Path(cls._tmp.name)
|
||||||
cls.addClassCleanup(
|
cls.addClassCleanup(
|
||||||
cls._teardown_docker, orchestrator_name, gateway_name, network, host_root
|
cls._teardown_docker, infra_name, network, host_root
|
||||||
)
|
)
|
||||||
|
|
||||||
cls.svc = OrchestratorService(
|
cls.svc = OrchestratorService(
|
||||||
orchestrator_name=orchestrator_name,
|
infra_name=infra_name,
|
||||||
gateway_name=gateway_name,
|
|
||||||
network=network,
|
network=network,
|
||||||
image=_TEST_ORCHESTRATOR_IMAGE,
|
image=_TEST_INFRA_IMAGE,
|
||||||
gateway_image=_TEST_GATEWAY_IMAGE,
|
|
||||||
port=20000 + secrets.randbelow(10000),
|
port=20000 + secrets.randbelow(10000),
|
||||||
host_root=host_root,
|
host_root=host_root,
|
||||||
)
|
)
|
||||||
@@ -91,23 +89,23 @@ class TestDockerControlPlaneAuthIntegration(unittest.TestCase):
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _teardown_docker(
|
def _teardown_docker(
|
||||||
orchestrator_name: str, gateway_name: str, network: str, host_root: Path
|
infra_name: str, network: str, host_root: Path
|
||||||
) -> None:
|
) -> None:
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["docker", "rm", "--force", orchestrator_name, gateway_name],
|
["docker", "rm", "--force", infra_name],
|
||||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
||||||
)
|
)
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["docker", "network", "rm", network],
|
["docker", "network", "rm", network],
|
||||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
||||||
)
|
)
|
||||||
# The orchestrator container (no USER directive) wrote the registry
|
# The infra container (no USER directive) wrote the registry
|
||||||
# DB as root into the throwaway host_root; chown it back so the
|
# DB as root into the throwaway host_root; chown it back so the
|
||||||
# (non-root) tempdir cleanup can remove it. Same workaround
|
# (non-root) tempdir cleanup can remove it. Same workaround
|
||||||
# test_multitenant_isolation.py uses for the identical bind mount.
|
# test_multitenant_isolation.py uses for the identical bind mount.
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["docker", "run", "--rm", "-v", f"{host_root}:/r",
|
["docker", "run", "--rm", "-v", f"{host_root}:/r",
|
||||||
"--entrypoint", "chown", _TEST_GATEWAY_IMAGE, "-R",
|
"--entrypoint", "chown", _TEST_INFRA_IMAGE, "-R",
|
||||||
f"{os.getuid()}:{os.getgid()}", "/r"],
|
f"{os.getuid()}:{os.getgid()}", "/r"],
|
||||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
"""Unit: shared cross-backend helpers in backend/util.py."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from bot_bottle.backend import util as backend_util
|
||||||
|
|
||||||
|
|
||||||
|
class TestPollCaCert(unittest.TestCase):
|
||||||
|
def test_returns_pem_on_first_success(self) -> None:
|
||||||
|
result = backend_util.poll_ca_cert(lambda: "PEM", timeout=1.0)
|
||||||
|
self.assertEqual("PEM", result)
|
||||||
|
|
||||||
|
def test_raises_timeout_error_when_cert_never_appears(self) -> None:
|
||||||
|
with self.assertRaises(TimeoutError):
|
||||||
|
backend_util.poll_ca_cert(lambda: None, timeout=0.0)
|
||||||
|
|
||||||
|
def test_polls_until_cert_appears(self) -> None:
|
||||||
|
responses = iter([None, None, "-----BEGIN CERTIFICATE-----\n"])
|
||||||
|
with patch("bot_bottle.backend.util.time.sleep") as mock_sleep:
|
||||||
|
result = backend_util.poll_ca_cert(lambda: next(responses), timeout=5.0)
|
||||||
|
self.assertTrue(result.startswith("-----BEGIN CERTIFICATE-----"))
|
||||||
|
self.assertEqual(2, mock_sleep.call_count)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -15,6 +15,7 @@ from bot_bottle.git_gate import GitGatePlan
|
|||||||
from bot_bottle.orchestrator.client import RegisteredBottle
|
from bot_bottle.orchestrator.client import RegisteredBottle
|
||||||
|
|
||||||
_MOD = "bot_bottle.backend.docker.consolidated_launch"
|
_MOD = "bot_bottle.backend.docker.consolidated_launch"
|
||||||
|
_UTIL = "bot_bottle.backend.consolidated_util"
|
||||||
|
|
||||||
|
|
||||||
def _egress_plan() -> EgressPlan:
|
def _egress_plan() -> EgressPlan:
|
||||||
@@ -49,7 +50,7 @@ class TestLaunchConsolidated(unittest.TestCase):
|
|||||||
patch(f"{_MOD}._container_ip", return_value="172.18.0.2"), \
|
patch(f"{_MOD}._container_ip", return_value="172.18.0.2"), \
|
||||||
patch(f"{_MOD}._network_container_ips", return_value=list(on_network)), \
|
patch(f"{_MOD}._network_container_ips", return_value=list(on_network)), \
|
||||||
patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
||||||
patch(f"{_MOD}.provision_git_gate", provision or Mock()):
|
patch(f"{_UTIL}.provision_git_gate", provision or Mock()):
|
||||||
return launch_consolidated(_egress_plan(), _git_plan(), service=service)
|
return launch_consolidated(_egress_plan(), _git_plan(), service=service)
|
||||||
|
|
||||||
def test_allocates_ip_registers_and_provisions(self) -> None:
|
def test_allocates_ip_registers_and_provisions(self) -> None:
|
||||||
@@ -84,8 +85,8 @@ class TestLaunchConsolidated(unittest.TestCase):
|
|||||||
class TestTeardownConsolidated(unittest.TestCase):
|
class TestTeardownConsolidated(unittest.TestCase):
|
||||||
def test_deregisters_and_deprovisions(self) -> None:
|
def test_deregisters_and_deprovisions(self) -> None:
|
||||||
client = Mock()
|
client = Mock()
|
||||||
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
with patch(f"{_UTIL}.OrchestratorClient", return_value=client), \
|
||||||
patch(f"{_MOD}.deprovision_git_gate") as deprov:
|
patch(f"{_UTIL}.deprovision_git_gate") as deprov:
|
||||||
teardown_consolidated("b1", orchestrator_url="http://orch:8080")
|
teardown_consolidated("b1", orchestrator_url="http://orch:8080")
|
||||||
client.teardown_bottle.assert_called_once_with("b1")
|
client.teardown_bottle.assert_called_once_with("b1")
|
||||||
deprov.assert_called_once()
|
deprov.assert_called_once()
|
||||||
|
|||||||
@@ -71,6 +71,16 @@ class TestSshGatewayTransport(unittest.TestCase):
|
|||||||
t.exec(["mkdir", "-p", "/git-gate"])
|
t.exec(["mkdir", "-p", "/git-gate"])
|
||||||
|
|
||||||
|
|
||||||
|
class TestGatewayCaPem(unittest.TestCase):
|
||||||
|
def test_dies_when_cert_never_appears(self) -> None:
|
||||||
|
from subprocess import CompletedProcess
|
||||||
|
infra = infra_vm.InfraVm(vm=None, guest_ip="10.0.0.1", private_key=Path("/k"))
|
||||||
|
with patch.object(infra_vm.subprocess, "run",
|
||||||
|
return_value=CompletedProcess([], 1, stdout="", stderr="")), \
|
||||||
|
self.assertRaises(SystemExit):
|
||||||
|
infra.gateway_ca_pem(timeout=0)
|
||||||
|
|
||||||
|
|
||||||
class TestRegistryVolume(unittest.TestCase):
|
class TestRegistryVolume(unittest.TestCase):
|
||||||
def test_reuses_existing_volume(self):
|
def test_reuses_existing_volume(self):
|
||||||
import tempfile
|
import tempfile
|
||||||
|
|||||||
@@ -96,7 +96,7 @@ class TestVersionInputs(unittest.TestCase):
|
|||||||
(pkg / "app.py").write_text("print('hi')\n")
|
(pkg / "app.py").write_text("print('hi')\n")
|
||||||
(pkg / "egress_entrypoint.sh").write_text("#!/bin/sh\nexec mitmdump\n")
|
(pkg / "egress_entrypoint.sh").write_text("#!/bin/sh\nexec mitmdump\n")
|
||||||
(pkg / "netpool.defaults.env").write_text("FOO=1\n")
|
(pkg / "netpool.defaults.env").write_text("FOO=1\n")
|
||||||
for name in ("Dockerfile.orchestrator", "Dockerfile.gateway", "Dockerfile.infra"):
|
for name in ("Dockerfile.orchestrator", "Dockerfile.gateway", "Dockerfile.infra", "Dockerfile.infra.fc"):
|
||||||
(root / name).write_text(f"FROM scratch # {name}\n")
|
(root / name).write_text(f"FROM scratch # {name}\n")
|
||||||
(root / "pyproject.toml").write_text("[project]\nname = 'bot-bottle'\n")
|
(root / "pyproject.toml").write_text("[project]\nname = 'bot-bottle'\n")
|
||||||
|
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from bot_bottle.git_gate import GitGatePlan
|
|||||||
from bot_bottle.orchestrator.client import RegisteredBottle
|
from bot_bottle.orchestrator.client import RegisteredBottle
|
||||||
|
|
||||||
_MOD = "bot_bottle.backend.macos_container.consolidated_launch"
|
_MOD = "bot_bottle.backend.macos_container.consolidated_launch"
|
||||||
|
_UTIL = "bot_bottle.backend.consolidated_util"
|
||||||
|
|
||||||
|
|
||||||
def _egress_plan() -> EgressPlan:
|
def _egress_plan() -> EgressPlan:
|
||||||
@@ -87,7 +88,7 @@ class TestRegisterAgent(unittest.TestCase):
|
|||||||
*, source_ip: str = "192.168.128.9",
|
*, source_ip: str = "192.168.128.9",
|
||||||
):
|
):
|
||||||
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
||||||
patch(f"{_MOD}.provision_git_gate", provision or Mock()), \
|
patch(f"{_UTIL}.provision_git_gate", provision or Mock()), \
|
||||||
patch(f"{_MOD}.live_source_ips", return_value=[]):
|
patch(f"{_MOD}.live_source_ips", return_value=[]):
|
||||||
return register_agent(
|
return register_agent(
|
||||||
_egress_plan(), _git_plan(),
|
_egress_plan(), _git_plan(),
|
||||||
@@ -126,8 +127,8 @@ class TestTeardown(unittest.TestCase):
|
|||||||
def test_deregisters_and_deprovisions(self) -> None:
|
def test_deregisters_and_deprovisions(self) -> None:
|
||||||
client = Mock()
|
client = Mock()
|
||||||
deprovision = Mock()
|
deprovision = Mock()
|
||||||
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
with patch(f"{_UTIL}.OrchestratorClient", return_value=client), \
|
||||||
patch(f"{_MOD}.deprovision_git_gate", deprovision):
|
patch(f"{_UTIL}.deprovision_git_gate", deprovision):
|
||||||
teardown_consolidated("b1", orchestrator_url="http://o:8099")
|
teardown_consolidated("b1", orchestrator_url="http://o:8099")
|
||||||
client.teardown_bottle.assert_called_once_with("b1")
|
client.teardown_bottle.assert_called_once_with("b1")
|
||||||
self.assertEqual("b1", deprovision.call_args.args[1])
|
self.assertEqual("b1", deprovision.call_args.args[1])
|
||||||
@@ -190,7 +191,7 @@ class TestRegisterAgentReconciles(unittest.TestCase):
|
|||||||
|
|
||||||
def _register(self, client: Mock) -> None:
|
def _register(self, client: Mock) -> None:
|
||||||
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
||||||
patch(f"{_MOD}.provision_git_gate"), \
|
patch(f"{_UTIL}.provision_git_gate"), \
|
||||||
patch(f"{_MOD}.live_source_ips", return_value=["10.0.0.7"]):
|
patch(f"{_MOD}.live_source_ips", return_value=["10.0.0.7"]):
|
||||||
register_agent(
|
register_agent(
|
||||||
_egress_plan(), _git_plan(),
|
_egress_plan(), _git_plan(),
|
||||||
@@ -228,7 +229,7 @@ class TestRegisterAgentReconciles(unittest.TestCase):
|
|||||||
from bot_bottle.backend.macos_container.enumerate import EnumerationError
|
from bot_bottle.backend.macos_container.enumerate import EnumerationError
|
||||||
client = _client()
|
client = _client()
|
||||||
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
with patch(f"{_MOD}.OrchestratorClient", return_value=client), \
|
||||||
patch(f"{_MOD}.provision_git_gate"), \
|
patch(f"{_UTIL}.provision_git_gate"), \
|
||||||
patch(f"{_MOD}.live_source_ips",
|
patch(f"{_MOD}.live_source_ips",
|
||||||
side_effect=EnumerationError("container list failed")):
|
side_effect=EnumerationError("container list failed")):
|
||||||
register_agent(
|
register_agent(
|
||||||
|
|||||||
@@ -22,7 +22,6 @@ from bot_bottle.backend.macos_container.launch import (
|
|||||||
_agent_run_argv,
|
_agent_run_argv,
|
||||||
_identity_proxy_env,
|
_identity_proxy_env,
|
||||||
)
|
)
|
||||||
from bot_bottle.backend.macos_container.rootless_podman import guest_env
|
|
||||||
from bot_bottle.manifest import ManifestIndex
|
from bot_bottle.manifest import ManifestIndex
|
||||||
|
|
||||||
_BOTTLE = "bot_bottle.backend.macos_container.bottle"
|
_BOTTLE = "bot_bottle.backend.macos_container.bottle"
|
||||||
@@ -77,7 +76,6 @@ def _plan(
|
|||||||
),
|
),
|
||||||
agent_git_gate_url=agent_git_gate_url,
|
agent_git_gate_url=agent_git_gate_url,
|
||||||
agent_supervise_url=agent_supervise_url,
|
agent_supervise_url=agent_supervise_url,
|
||||||
docker_access=False,
|
|
||||||
))
|
))
|
||||||
|
|
||||||
|
|
||||||
@@ -180,18 +178,6 @@ class TestIdentityTokenDelivery(unittest.TestCase):
|
|||||||
self.assertNotIn("--env", argv)
|
self.assertNotIn("--env", argv)
|
||||||
|
|
||||||
|
|
||||||
class TestRootlessPodmanEnvironment(unittest.TestCase):
|
|
||||||
def test_disabled_bottle_gets_no_docker_environment(self) -> None:
|
|
||||||
self.assertEqual({}, guest_env(False))
|
|
||||||
|
|
||||||
def test_enabled_bottle_uses_only_guest_local_socket(self) -> None:
|
|
||||||
env = guest_env(True)
|
|
||||||
self.assertEqual(
|
|
||||||
"unix:///tmp/bot-bottle-podman-run/podman.sock", env["DOCKER_HOST"],
|
|
||||||
)
|
|
||||||
self.assertNotIn("/var/run/docker.sock", " ".join(env.values()))
|
|
||||||
|
|
||||||
|
|
||||||
class TestPlanIdentityToken(unittest.TestCase):
|
class TestPlanIdentityToken(unittest.TestCase):
|
||||||
"""git-gate's gitconfig extraHeader and the supervise MCP --header read
|
"""git-gate's gitconfig extraHeader and the supervise MCP --header read
|
||||||
`getattr(plan, "identity_token", "")` at provision time and both bypass the
|
`getattr(plan, "identity_token", "")` at provision time and both bypass the
|
||||||
|
|||||||
@@ -153,6 +153,14 @@ class TestCaCertPem(unittest.TestCase):
|
|||||||
argv = mod.run_container_argv.call_args.args[0]
|
argv = mod.run_container_argv.call_args.args[0]
|
||||||
self.assertEqual(["container", "exec", "bot-bottle-mac-infra", "cat"], argv[:4])
|
self.assertEqual(["container", "exec", "bot-bottle-mac-infra", "cat"], argv[:4])
|
||||||
|
|
||||||
|
def test_raises_gateway_error_when_cert_never_appears(self) -> None:
|
||||||
|
from bot_bottle.backend.macos_container.gateway import GatewayError
|
||||||
|
svc = MacosInfraService(repo_root=Path("/r"))
|
||||||
|
with patch(f"{_INFRA}.container_mod") as mod:
|
||||||
|
mod.run_container_argv.return_value = _fail()
|
||||||
|
with self.assertRaises(GatewayError):
|
||||||
|
svc.ca_cert_pem(timeout=0)
|
||||||
|
|
||||||
|
|
||||||
class TestProbeControlPlane(unittest.TestCase):
|
class TestProbeControlPlane(unittest.TestCase):
|
||||||
def test_returns_url_when_running(self) -> None:
|
def test_returns_url_when_running(self) -> None:
|
||||||
|
|||||||
@@ -1,147 +0,0 @@
|
|||||||
"""Unit coverage for the fail-closed macOS rootless-podman spike."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import unittest
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from pathlib import Path
|
|
||||||
from types import SimpleNamespace
|
|
||||||
from typing import cast
|
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
from bot_bottle.backend.macos_container import rootless_podman
|
|
||||||
from bot_bottle.backend.macos_container import launch as launch_mod
|
|
||||||
from bot_bottle.backend.macos_container.bottle_plan import MacosContainerBottlePlan
|
|
||||||
|
|
||||||
|
|
||||||
class _Bottle:
|
|
||||||
def __init__(self, results: list[SimpleNamespace]) -> None:
|
|
||||||
self.results = results
|
|
||||||
self.commands: list[str] = []
|
|
||||||
|
|
||||||
def exec(self, command: str) -> SimpleNamespace:
|
|
||||||
self.commands.append(command)
|
|
||||||
return self.results.pop(0)
|
|
||||||
|
|
||||||
|
|
||||||
def _result(returncode: int, *, stdout: str = "", stderr: str = "") -> SimpleNamespace:
|
|
||||||
return SimpleNamespace(returncode=returncode, stdout=stdout, stderr=stderr)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class _AgentProvision:
|
|
||||||
image: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class _Plan:
|
|
||||||
slug: str
|
|
||||||
image: str
|
|
||||||
dockerfile_path: str
|
|
||||||
docker_access: bool
|
|
||||||
agent_provision: _AgentProvision
|
|
||||||
|
|
||||||
|
|
||||||
class TestRootlessPodmanStart(unittest.TestCase):
|
|
||||||
def test_bootstraps_then_waits_for_guest_local_service(self) -> None:
|
|
||||||
bottle = _Bottle([_result(0), _result(1), _result(0)])
|
|
||||||
with patch.object(rootless_podman.time, "sleep"):
|
|
||||||
rootless_podman.start(bottle)
|
|
||||||
self.assertIn("rootless-podman-init", bottle.commands[0])
|
|
||||||
self.assertEqual(2, bottle.commands.count("docker info >/dev/null 2>&1"))
|
|
||||||
|
|
||||||
def test_bootstrap_failure_is_fatal_without_privilege_fallback(self) -> None:
|
|
||||||
bottle = _Bottle([_result(1, stderr="slirp4netns missing")])
|
|
||||||
with patch.object(rootless_podman, "die", side_effect=RuntimeError) as die:
|
|
||||||
with self.assertRaises(RuntimeError):
|
|
||||||
rootless_podman.start(bottle)
|
|
||||||
self.assertIn("slirp4netns missing", die.call_args.args[0])
|
|
||||||
self.assertEqual(1, len(bottle.commands))
|
|
||||||
|
|
||||||
def test_timeout_reports_guest_log(self) -> None:
|
|
||||||
bottle = _Bottle(
|
|
||||||
[_result(0)]
|
|
||||||
+ [_result(1) for _ in range(rootless_podman.READY_RETRIES)]
|
|
||||||
+ [_result(0, stdout="operation not permitted")]
|
|
||||||
)
|
|
||||||
with patch.object(rootless_podman.time, "sleep"), \
|
|
||||||
patch.object(rootless_podman, "die", side_effect=RuntimeError) as die:
|
|
||||||
with self.assertRaises(RuntimeError):
|
|
||||||
rootless_podman.start(bottle)
|
|
||||||
self.assertIn("operation not permitted", die.call_args.args[0])
|
|
||||||
|
|
||||||
|
|
||||||
class TestRootlessPodmanDevices(unittest.TestCase):
|
|
||||||
def test_relaxes_only_the_two_blocked_device_nodes_as_root(self) -> None:
|
|
||||||
calls: list[tuple[str, list[str]]] = []
|
|
||||||
rootless_podman.prepare_guest_devices(
|
|
||||||
"bottle-1", lambda name, argv: calls.append((name, argv)),
|
|
||||||
)
|
|
||||||
self.assertEqual(1, len(calls))
|
|
||||||
name, argv = calls[0]
|
|
||||||
self.assertEqual("bottle-1", name)
|
|
||||||
self.assertIn("chmod 0666 /dev/fuse /dev/net/tun", argv[-1])
|
|
||||||
|
|
||||||
|
|
||||||
class TestRootlessPodmanImage(unittest.TestCase):
|
|
||||||
def test_layers_tooling_without_changing_base_image(self) -> None:
|
|
||||||
calls: list[tuple[str, str, str]] = []
|
|
||||||
|
|
||||||
def build(image: str, context: str, *, dockerfile: str) -> None:
|
|
||||||
calls.append((image, context, dockerfile))
|
|
||||||
text = Path(dockerfile).read_text(encoding="utf-8")
|
|
||||||
self.assertIn("FROM agent:base", text)
|
|
||||||
self.assertIn("podman fuse-overlayfs slirp4netns uidmap", text)
|
|
||||||
self.assertIn("USER node", text)
|
|
||||||
self.assertTrue((Path(context) / "rootless-podman-init.sh").is_file())
|
|
||||||
|
|
||||||
image = rootless_podman.build_image("agent:base", build)
|
|
||||||
self.assertEqual("agent:base-rootless-podman", image)
|
|
||||||
self.assertEqual("agent:base-rootless-podman", calls[0][0])
|
|
||||||
|
|
||||||
def test_strips_subordinate_ranges_so_podman_avoids_newuidmap(self) -> None:
|
|
||||||
"""The single-UID fallback is the entire reason podman works here.
|
|
||||||
|
|
||||||
A subordinate range would send podman down the newuidmap path, which
|
|
||||||
cannot write a multi-range uid_map without CAP_SYS_ADMIN in an Apple
|
|
||||||
Container guest — the failure that killed the rootless-Docker spike.
|
|
||||||
"""
|
|
||||||
seen: list[str] = []
|
|
||||||
|
|
||||||
def build(image: str, context: str, *, dockerfile: str) -> None:
|
|
||||||
seen.append(Path(dockerfile).read_text(encoding="utf-8"))
|
|
||||||
|
|
||||||
rootless_podman.build_image("agent:base", build)
|
|
||||||
text = seen[0]
|
|
||||||
self.assertIn("sed -i '/^node:/d' /etc/subuid /etc/subgid", text)
|
|
||||||
self.assertNotIn("subuid", text.replace(
|
|
||||||
"sed -i '/^node:/d' /etc/subuid /etc/subgid", "",
|
|
||||||
))
|
|
||||||
|
|
||||||
def test_launch_builds_base_then_rootless_variant(self) -> None:
|
|
||||||
plan = cast(MacosContainerBottlePlan, cast(object, _Plan(
|
|
||||||
slug="dev-abc",
|
|
||||||
image="agent:base",
|
|
||||||
dockerfile_path="/repo/Dockerfile",
|
|
||||||
docker_access=True,
|
|
||||||
agent_provision=_AgentProvision(image="agent:base"),
|
|
||||||
)))
|
|
||||||
with patch.object(launch_mod, "read_committed_image", return_value=None), \
|
|
||||||
patch.object(launch_mod.container_mod, "build_image") as build, \
|
|
||||||
patch.object(
|
|
||||||
launch_mod.rootless_podman,
|
|
||||||
"build_image",
|
|
||||||
return_value="agent:base-rootless-podman",
|
|
||||||
) as build_rootless:
|
|
||||||
result = launch_mod._build_images(plan) # pylint: disable=protected-access
|
|
||||||
|
|
||||||
build.assert_called_once_with(
|
|
||||||
"agent:base", launch_mod._REPO_DIR, # pylint: disable=protected-access
|
|
||||||
dockerfile="/repo/Dockerfile",
|
|
||||||
)
|
|
||||||
build_rootless.assert_called_once_with("agent:base", build)
|
|
||||||
self.assertEqual("agent:base-rootless-podman", result.agent_provision.image)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -56,12 +56,6 @@ class TestMergeBottlesRuntime(unittest.TestCase):
|
|||||||
result = merge_bottles_runtime([base, override])
|
result = merge_bottles_runtime([base, override])
|
||||||
self.assertFalse(result.supervise)
|
self.assertFalse(result.supervise)
|
||||||
|
|
||||||
def test_docker_access_later_wins(self):
|
|
||||||
result = merge_bottles_runtime([
|
|
||||||
_bottle(docker_access=False), _bottle(docker_access=True),
|
|
||||||
])
|
|
||||||
self.assertTrue(result.docker_access)
|
|
||||||
|
|
||||||
def test_three_bottles_merged_left_to_right(self):
|
def test_three_bottles_merged_left_to_right(self):
|
||||||
b1 = _bottle(env={"A": "1", "B": "1", "C": "1"})
|
b1 = _bottle(env={"A": "1", "B": "1", "C": "1"})
|
||||||
b2 = _bottle(env={"B": "2", "C": "2"})
|
b2 = _bottle(env={"B": "2", "C": "2"})
|
||||||
|
|||||||
@@ -44,20 +44,13 @@ class TestBottleValidation(unittest.TestCase):
|
|||||||
with self.assertRaises(ManifestError):
|
with self.assertRaises(ManifestError):
|
||||||
ManifestBottle.from_dict("b", {"supervise": "yes"})
|
ManifestBottle.from_dict("b", {"supervise": "yes"})
|
||||||
|
|
||||||
def test_docker_access_not_bool(self) -> None:
|
|
||||||
with self.assertRaises(ManifestError):
|
|
||||||
ManifestBottle.from_dict("b", {"docker_access": "yes"})
|
|
||||||
|
|
||||||
def test_removed_runtime_field(self) -> None:
|
def test_removed_runtime_field(self) -> None:
|
||||||
with self.assertRaises(ManifestError):
|
with self.assertRaises(ManifestError):
|
||||||
ManifestBottle.from_dict("b", {"runtime": "runsc"})
|
ManifestBottle.from_dict("b", {"runtime": "runsc"})
|
||||||
|
|
||||||
def test_valid_minimal(self) -> None:
|
def test_valid_minimal(self) -> None:
|
||||||
b = ManifestBottle.from_dict(
|
b = ManifestBottle.from_dict("b", {"supervise": False, "env": {"X": "1"}})
|
||||||
"b", {"supervise": False, "docker_access": True, "env": {"X": "1"}},
|
|
||||||
)
|
|
||||||
self.assertFalse(b.supervise)
|
self.assertFalse(b.supervise)
|
||||||
self.assertTrue(b.docker_access)
|
|
||||||
self.assertEqual({"X": "1"}, dict(b.env))
|
self.assertEqual({"X": "1"}, dict(b.env))
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Unit: orchestrator+gateway container lifecycle — idempotent singleton (PRD 0070)."""
|
"""Unit: infra container lifecycle — idempotent singleton (PRD 0070)."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -8,10 +8,10 @@ import urllib.error
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import MagicMock, Mock, patch
|
from unittest.mock import MagicMock, Mock, patch
|
||||||
|
|
||||||
|
from bot_bottle.orchestrator.gateway import GatewayError
|
||||||
from bot_bottle.orchestrator.lifecycle import (
|
from bot_bottle.orchestrator.lifecycle import (
|
||||||
ORCHESTRATOR_IMAGE,
|
INFRA_NAME,
|
||||||
ORCHESTRATOR_NAME,
|
INFRA_SOURCE_HASH_LABEL,
|
||||||
ORCHESTRATOR_SOURCE_HASH_LABEL,
|
|
||||||
OrchestratorService,
|
OrchestratorService,
|
||||||
OrchestratorStartError,
|
OrchestratorStartError,
|
||||||
source_hash,
|
source_hash,
|
||||||
@@ -20,7 +20,6 @@ from tests.unit import use_bottle_root
|
|||||||
|
|
||||||
_URLOPEN = "bot_bottle.orchestrator.lifecycle.urllib.request.urlopen"
|
_URLOPEN = "bot_bottle.orchestrator.lifecycle.urllib.request.urlopen"
|
||||||
_RUN = "bot_bottle.orchestrator.lifecycle.run_docker"
|
_RUN = "bot_bottle.orchestrator.lifecycle.run_docker"
|
||||||
_GATEWAY = "bot_bottle.orchestrator.lifecycle.DockerGateway"
|
|
||||||
_SLEEP = "bot_bottle.orchestrator.lifecycle.time.sleep"
|
_SLEEP = "bot_bottle.orchestrator.lifecycle.time.sleep"
|
||||||
_MONOTONIC = "bot_bottle.orchestrator.lifecycle.time.monotonic"
|
_MONOTONIC = "bot_bottle.orchestrator.lifecycle.time.monotonic"
|
||||||
|
|
||||||
@@ -42,10 +41,8 @@ class TestOrchestratorService(unittest.TestCase):
|
|||||||
self.addCleanup(use_bottle_root(Path(self._tmp.name)))
|
self.addCleanup(use_bottle_root(Path(self._tmp.name)))
|
||||||
self.svc = OrchestratorService(port=8099)
|
self.svc = OrchestratorService(port=8099)
|
||||||
|
|
||||||
def test_urls(self) -> None:
|
def test_url(self) -> None:
|
||||||
self.assertEqual("http://127.0.0.1:8099", self.svc.url)
|
self.assertEqual("http://127.0.0.1:8099", self.svc.url)
|
||||||
# The gateway reaches the control plane by container name over docker DNS.
|
|
||||||
self.assertEqual(f"http://{ORCHESTRATOR_NAME}:8099", self.svc.internal_url)
|
|
||||||
|
|
||||||
def test_is_healthy(self) -> None:
|
def test_is_healthy(self) -> None:
|
||||||
with patch(_URLOPEN, return_value=_health(200)):
|
with patch(_URLOPEN, return_value=_health(200)):
|
||||||
@@ -54,126 +51,169 @@ class TestOrchestratorService(unittest.TestCase):
|
|||||||
self.assertFalse(self.svc.is_healthy())
|
self.assertFalse(self.svc.is_healthy())
|
||||||
|
|
||||||
def test_ensure_running_noop_when_healthy_and_source_unchanged(self) -> None:
|
def test_ensure_running_noop_when_healthy_and_source_unchanged(self) -> None:
|
||||||
# A healthy control plane already running the *current* bind-mounted
|
# A healthy container on current source is left alone — recreating it
|
||||||
# source is left alone — recreating it on every launch would drop
|
# on every launch drops in-memory egress tokens (#381).
|
||||||
# every other active bottle's in-memory egress tokens (#381).
|
|
||||||
current = source_hash(self.svc._repo_root)
|
current = source_hash(self.svc._repo_root)
|
||||||
calls: list[list[str]] = []
|
calls: list[list[str]] = []
|
||||||
|
|
||||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||||
calls.append(argv)
|
calls.append(argv)
|
||||||
if argv[:2] == ["docker", "ps"]:
|
if argv[:2] == ["docker", "ps"]:
|
||||||
return _proc(stdout=ORCHESTRATOR_NAME)
|
return _proc(stdout=INFRA_NAME)
|
||||||
if argv[:2] == ["docker", "inspect"]:
|
if argv[:2] == ["docker", "inspect"]:
|
||||||
return _proc(stdout=current)
|
return _proc(stdout=current)
|
||||||
return _proc()
|
return _proc()
|
||||||
|
|
||||||
with patch(_URLOPEN, return_value=_health(200)), \
|
with patch(_URLOPEN, return_value=_health(200)), \
|
||||||
patch(_GATEWAY) as gw_cls, patch(_RUN, side_effect=fake), patch(_SLEEP):
|
patch(_RUN, side_effect=fake), patch(_SLEEP):
|
||||||
self.assertEqual(self.svc.url, self.svc.ensure_running())
|
self.assertEqual(self.svc.url, self.svc.ensure_running())
|
||||||
gw_cls.return_value.ensure_running.assert_called() # gateway kept up
|
|
||||||
runs = [c for c in calls if c[:2] == ["docker", "run"]]
|
runs = [c for c in calls if c[:2] == ["docker", "run"]]
|
||||||
rms = [c for c in calls if c[:3] == ["docker", "rm", "--force"] and ORCHESTRATOR_NAME in c]
|
rms = [c for c in calls if c[:3] == ["docker", "rm", "--force"] and INFRA_NAME in c]
|
||||||
self.assertEqual([], runs) # not recreated
|
self.assertEqual([], runs)
|
||||||
self.assertEqual([], rms)
|
self.assertEqual([], rms)
|
||||||
|
|
||||||
def test_ensure_running_recreates_when_source_changed(self) -> None:
|
def test_ensure_running_recreates_when_source_changed(self) -> None:
|
||||||
# Healthy, but the running container's label doesn't match the
|
|
||||||
# current source hash (a real code change) — recreate so it takes
|
|
||||||
# effect, same as the gateway's image-staleness check.
|
|
||||||
calls: list[list[str]] = []
|
calls: list[list[str]] = []
|
||||||
|
|
||||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||||
calls.append(argv)
|
calls.append(argv)
|
||||||
if argv[:2] == ["docker", "ps"]:
|
if argv[:2] == ["docker", "ps"]:
|
||||||
return _proc(stdout=ORCHESTRATOR_NAME)
|
return _proc(stdout=INFRA_NAME)
|
||||||
if argv[:2] == ["docker", "inspect"]:
|
if argv[:2] == ["docker", "inspect"]:
|
||||||
return _proc(stdout="stale-hash")
|
return _proc(stdout="stale-hash")
|
||||||
return _proc()
|
return _proc()
|
||||||
|
|
||||||
with patch(_URLOPEN, return_value=_health(200)), \
|
with patch(_URLOPEN, return_value=_health(200)), \
|
||||||
patch(_GATEWAY), patch(_RUN, side_effect=fake), patch(_SLEEP):
|
patch(_RUN, side_effect=fake), patch(_SLEEP):
|
||||||
self.assertEqual(self.svc.url, self.svc.ensure_running())
|
self.assertEqual(self.svc.url, self.svc.ensure_running())
|
||||||
runs = [c for c in calls if c[:2] == ["docker", "run"]]
|
runs = [c for c in calls if c[:2] == ["docker", "run"]]
|
||||||
self.assertEqual(1, len(runs))
|
self.assertEqual(1, len(runs))
|
||||||
self.assertIn(ORCHESTRATOR_NAME, runs[0])
|
self.assertIn(INFRA_NAME, runs[0])
|
||||||
# the fresh container is labeled with the current hash, not the stale one
|
|
||||||
current = source_hash(self.svc._repo_root)
|
current = source_hash(self.svc._repo_root)
|
||||||
self.assertIn(f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current}", runs[0])
|
self.assertIn(f"{INFRA_SOURCE_HASH_LABEL}={current}", runs[0])
|
||||||
|
|
||||||
def test_ensure_running_starts_orchestrator_container_when_absent(self) -> None:
|
def test_ensure_running_starts_infra_container_when_absent(self) -> None:
|
||||||
calls: list[list[str]] = []
|
|
||||||
|
|
||||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
|
||||||
calls.append(argv)
|
|
||||||
if argv[:2] == ["docker", "ps"]:
|
|
||||||
return _proc(stdout="") # not running
|
|
||||||
return _proc()
|
|
||||||
|
|
||||||
with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
|
||||||
patch(_GATEWAY), patch(_RUN, side_effect=fake), patch(_SLEEP):
|
|
||||||
self.assertEqual(self.svc.url, self.svc.ensure_running())
|
|
||||||
runs = [c for c in calls if c[:2] == ["docker", "run"]]
|
|
||||||
self.assertEqual(1, len(runs))
|
|
||||||
argv = runs[0]
|
|
||||||
self.assertIn(ORCHESTRATOR_NAME, argv)
|
|
||||||
self.assertIn("--broker", argv)
|
|
||||||
self.assertEqual("stub", argv[argv.index("--broker") + 1]) # register-only, no socket
|
|
||||||
self.assertIn("bot_bottle.orchestrator", argv)
|
|
||||||
self.assertEqual("127.0.0.1:8099:8099", argv[argv.index("--publish") + 1])
|
|
||||||
|
|
||||||
def test_ensure_running_builds_lean_orchestrator_image_when_missing(self) -> None:
|
|
||||||
# The control plane runs its own lean image (#384), distinct from the
|
|
||||||
# gateway data plane — built from Dockerfile.orchestrator when absent.
|
|
||||||
calls: list[list[str]] = []
|
|
||||||
|
|
||||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
|
||||||
calls.append(argv)
|
|
||||||
if argv[:2] == ["docker", "ps"]:
|
|
||||||
return _proc(stdout="") # orchestrator not running
|
|
||||||
if argv[:3] == ["docker", "image", "inspect"]:
|
|
||||||
return _proc(returncode=1) # image absent -> build
|
|
||||||
return _proc()
|
|
||||||
|
|
||||||
with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
|
||||||
patch(_GATEWAY), patch(_RUN, side_effect=fake), patch(_SLEEP):
|
|
||||||
self.svc.ensure_running()
|
|
||||||
builds = [c for c in calls if c[:2] == ["docker", "build"]]
|
|
||||||
self.assertEqual(1, len(builds))
|
|
||||||
self.assertIn(ORCHESTRATOR_IMAGE, builds[0])
|
|
||||||
self.assertTrue(any(a.endswith("Dockerfile.orchestrator") for a in builds[0]))
|
|
||||||
# It is NOT the gateway image/dockerfile — the split is the point.
|
|
||||||
self.assertFalse(any("Dockerfile.gateway" in a for a in builds[0]))
|
|
||||||
|
|
||||||
def test_ensure_running_skips_orchestrator_image_build_when_present(self) -> None:
|
|
||||||
calls: list[list[str]] = []
|
calls: list[list[str]] = []
|
||||||
|
|
||||||
def fake(argv: list[str], **_kw: object) -> Mock:
|
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||||
calls.append(argv)
|
calls.append(argv)
|
||||||
if argv[:2] == ["docker", "ps"]:
|
if argv[:2] == ["docker", "ps"]:
|
||||||
return _proc(stdout="")
|
return _proc(stdout="")
|
||||||
if argv[:3] == ["docker", "image", "inspect"]:
|
|
||||||
return _proc(returncode=0) # image present -> no build
|
|
||||||
return _proc()
|
return _proc()
|
||||||
|
|
||||||
with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
||||||
patch(_GATEWAY), patch(_RUN, side_effect=fake), patch(_SLEEP):
|
patch(_RUN, side_effect=fake), patch(_SLEEP):
|
||||||
|
self.assertEqual(self.svc.url, self.svc.ensure_running())
|
||||||
|
runs = [c for c in calls if c[:2] == ["docker", "run"]]
|
||||||
|
self.assertEqual(1, len(runs))
|
||||||
|
argv = runs[0]
|
||||||
|
self.assertIn(INFRA_NAME, argv)
|
||||||
|
# Published on loopback — not exposed on external interfaces.
|
||||||
|
self.assertEqual("127.0.0.1:8099:8099", argv[argv.index("--publish") + 1])
|
||||||
|
# Both processes in one container — no separate entrypoint override.
|
||||||
|
self.assertNotIn("--entrypoint", argv)
|
||||||
|
# Gateway daemons + orchestrator explicitly opted in.
|
||||||
|
daemons_flag = "BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise,orchestrator"
|
||||||
|
self.assertIn("orchestrator", argv[argv.index(daemons_flag)])
|
||||||
|
|
||||||
|
def test_ensure_running_builds_all_images(self) -> None:
|
||||||
|
calls: list[list[str]] = []
|
||||||
|
|
||||||
|
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||||
|
calls.append(argv)
|
||||||
|
if argv[:2] == ["docker", "ps"]:
|
||||||
|
return _proc(stdout="")
|
||||||
|
return _proc()
|
||||||
|
|
||||||
|
with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
||||||
|
patch(_RUN, side_effect=fake), patch(_SLEEP):
|
||||||
self.svc.ensure_running()
|
self.svc.ensure_running()
|
||||||
self.assertEqual([], [c for c in calls if c[:2] == ["docker", "build"]])
|
builds = [c for c in calls if c[:2] == ["docker", "build"]]
|
||||||
|
# Gateway base + orchestrator intermediate + infra image — all three built.
|
||||||
|
self.assertEqual(3, len(builds))
|
||||||
|
dockerfiles = [next(a for a in b if "Dockerfile" in a) for b in builds]
|
||||||
|
self.assertIn("Dockerfile.gateway", dockerfiles[0])
|
||||||
|
self.assertIn("Dockerfile.orchestrator", dockerfiles[1])
|
||||||
|
self.assertIn("Dockerfile.infra", dockerfiles[2])
|
||||||
|
# All three images are distinct.
|
||||||
|
tags = [b[b.index("-t") + 1] for b in builds]
|
||||||
|
self.assertEqual(3, len(set(tags)))
|
||||||
|
|
||||||
|
def test_publish_maps_host_port_to_fixed_internal_port(self) -> None:
|
||||||
|
"""A non-default self.port is published to the fixed internal port 8099,
|
||||||
|
not to self.port:self.port — the orchestrator always listens on 8099."""
|
||||||
|
calls: list[list[str]] = []
|
||||||
|
|
||||||
|
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||||
|
calls.append(argv)
|
||||||
|
if argv[:2] == ["docker", "ps"]:
|
||||||
|
return _proc(stdout="")
|
||||||
|
return _proc()
|
||||||
|
|
||||||
|
svc = OrchestratorService(port=20001)
|
||||||
|
with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
||||||
|
patch(_RUN, side_effect=fake), patch(_SLEEP):
|
||||||
|
svc.ensure_running()
|
||||||
|
runs = [c for c in calls if c[:2] == ["docker", "run"]]
|
||||||
|
argv = runs[0]
|
||||||
|
self.assertEqual("127.0.0.1:20001:8099", argv[argv.index("--publish") + 1])
|
||||||
|
orch_url = next(a for a in argv if "BOT_BOTTLE_ORCHESTRATOR_URL" in a)
|
||||||
|
self.assertIn(":8099", orch_url)
|
||||||
|
|
||||||
def test_ensure_running_raises_on_timeout(self) -> None:
|
def test_ensure_running_raises_on_timeout(self) -> None:
|
||||||
with patch(_URLOPEN, side_effect=urllib.error.URLError("down")), \
|
with patch(_URLOPEN, side_effect=urllib.error.URLError("down")), \
|
||||||
patch(_GATEWAY), patch(_RUN, return_value=Mock(returncode=0, stderr="")), \
|
patch(_RUN, return_value=Mock(returncode=0, stdout="", stderr="")), \
|
||||||
patch(_SLEEP), patch(_MONOTONIC, side_effect=[0.0, 0.5, 2.0]):
|
patch(_SLEEP), patch(_MONOTONIC, side_effect=[0.0, 0.5, 2.0]):
|
||||||
with self.assertRaises(OrchestratorStartError):
|
with self.assertRaises(OrchestratorStartError):
|
||||||
self.svc.ensure_running(startup_timeout=1.0)
|
self.svc.ensure_running(startup_timeout=1.0)
|
||||||
|
|
||||||
def test_stop_removes_orchestrator_and_gateway(self) -> None:
|
def test_noop_when_healthy_and_inspect_fails(self) -> None:
|
||||||
with patch(_RUN) as run, patch(_GATEWAY) as gw_cls:
|
"""If docker inspect fails (e.g. docker daemon hiccup), leave the
|
||||||
|
working container alone rather than churning it."""
|
||||||
|
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||||
|
if argv[:2] == ["docker", "ps"]:
|
||||||
|
return _proc(stdout=INFRA_NAME)
|
||||||
|
if argv[:2] == ["docker", "inspect"]:
|
||||||
|
return _proc(returncode=1, stderr="daemon error")
|
||||||
|
return _proc()
|
||||||
|
|
||||||
|
with patch(_URLOPEN, return_value=_health(200)), \
|
||||||
|
patch(_RUN, side_effect=fake), patch(_SLEEP):
|
||||||
|
self.svc.ensure_running()
|
||||||
|
# no docker run — the working container was left alone
|
||||||
|
|
||||||
|
def test_build_failure_raises(self) -> None:
|
||||||
|
with patch(_URLOPEN, side_effect=urllib.error.URLError("down")), \
|
||||||
|
patch(_RUN, return_value=_proc(returncode=1, stderr="no space left on device")):
|
||||||
|
with self.assertRaises(GatewayError):
|
||||||
|
self.svc.ensure_running()
|
||||||
|
|
||||||
|
def test_ensure_network_creates_if_missing(self) -> None:
|
||||||
|
"""If the gateway network doesn't exist yet, create it."""
|
||||||
|
calls: list[list[str]] = []
|
||||||
|
|
||||||
|
def fake(argv: list[str], **_kw: object) -> Mock:
|
||||||
|
calls.append(argv)
|
||||||
|
if argv[:3] == ["docker", "network", "inspect"]:
|
||||||
|
return _proc(returncode=1, stderr="not found")
|
||||||
|
if argv[:2] == ["docker", "ps"]:
|
||||||
|
return _proc(stdout="")
|
||||||
|
return _proc()
|
||||||
|
|
||||||
|
with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
|
||||||
|
patch(_RUN, side_effect=fake), patch(_SLEEP):
|
||||||
|
self.svc.ensure_running()
|
||||||
|
creates = [c for c in calls if c[:3] == ["docker", "network", "create"]]
|
||||||
|
self.assertEqual(1, len(creates))
|
||||||
|
|
||||||
|
def test_stop_removes_infra_container(self) -> None:
|
||||||
|
with patch(_RUN) as run:
|
||||||
self.svc.stop()
|
self.svc.stop()
|
||||||
rms = [c.args[0] for c in run.call_args_list if c.args[0][:3] == ["docker", "rm", "--force"]]
|
rms = [
|
||||||
self.assertTrue(any(ORCHESTRATOR_NAME in a for a in rms))
|
c.args[0] for c in run.call_args_list
|
||||||
gw_cls.return_value.stop.assert_called_once()
|
if c.args[0][:3] == ["docker", "rm", "--force"]
|
||||||
|
]
|
||||||
|
self.assertTrue(any(INFRA_NAME in a for a in rms))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
Reference in New Issue
Block a user