Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 95981ea9d3 | |||
| bab44dbe97 | |||
| 0a1f949937 | |||
| 64ca39ddd4 | |||
| 3c426f45f5 | |||
| 1d3e9e859c |
+33
-6
@@ -1,9 +1,10 @@
|
|||||||
# Orchestrator control-plane image (PRD 0070, #384).
|
# Orchestrator control-plane image (PRD 0070, #384) + in-VM agent-image
|
||||||
|
# builder (PRD 0069 Stage 3).
|
||||||
#
|
#
|
||||||
# The per-host orchestrator runs `python3 -m bot_bottle.orchestrator`.
|
# The per-host orchestrator runs `python3 -m bot_bottle.orchestrator`.
|
||||||
# The `bot_bottle` package is **stdlib-only** by design, so the control
|
# The `bot_bottle` package is **stdlib-only** by design, so the control
|
||||||
# plane needs nothing but a Python runtime — none of the gateway's
|
# plane itself needs nothing but a Python runtime — none of the
|
||||||
# mitmproxy / git / gitleaks payload (that is the separate
|
# gateway's mitmproxy / git / gitleaks payload (that is the separate
|
||||||
# `bot-bottle-gateway` image, Dockerfile.gateway). Splitting them keeps
|
# `bot-bottle-gateway` image, Dockerfile.gateway). Splitting them keeps
|
||||||
# the secret-dense control plane (it concentrates every bottle's egress
|
# the secret-dense control plane (it concentrates every bottle's egress
|
||||||
# tokens — see PRD 0070's "secret concentration") on a minimal
|
# tokens — see PRD 0070's "secret concentration") on a minimal
|
||||||
@@ -17,9 +18,35 @@
|
|||||||
|
|
||||||
FROM python:3.12-slim
|
FROM python:3.12-slim
|
||||||
|
|
||||||
# No third-party deps to install — stdlib only. Kept as an explicit,
|
# --- in-VM agent-image builder (PRD 0069 Stage 3) -------------------
|
||||||
# self-documenting stage so a future confinement step (baking the
|
# The Firecracker backend removes the host Docker daemon by building
|
||||||
# package, dropping the bind mount) has an obvious home.
|
# users' agent Dockerfiles *inside the orchestrator VM* with buildah
|
||||||
|
# (rootless, daemonless) instead of on the host. The host then needs no
|
||||||
|
# Docker daemon and no root-equivalent `docker` group; an untrusted
|
||||||
|
# Dockerfile builds inside the confined orchestrator VM, never on the
|
||||||
|
# host — strictly more isolated than host `docker build`.
|
||||||
|
#
|
||||||
|
# `git` lets buildah resolve git-context builds and lets the backend
|
||||||
|
# clone/pull; `ca-certificates` is needed for `FROM` pulls over TLS.
|
||||||
|
# buildah's runtime helpers (stripped by --no-install-recommends, so
|
||||||
|
# listed explicitly): `crun` is the OCI runtime that executes build
|
||||||
|
# steps; `netavark` + `aardvark-dns` are the network backend buildah
|
||||||
|
# uses to give `FROM` pulls and `RUN` steps network access.
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
buildah crun netavark aardvark-dns git ca-certificates \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# vfs storage + chroot isolation: buildah then needs neither
|
||||||
|
# fuse-overlayfs / an overlay kernel module nor configured subuid maps
|
||||||
|
# (newuidmap/newgidmap), so it works unconditionally as root in a
|
||||||
|
# minimal microVM rootfs. vfs copies layers rather than stacking them —
|
||||||
|
# slower and heavier than overlay; a build-cache optimization (overlay
|
||||||
|
# where available, a persistent cache disk) is tracked for later.
|
||||||
|
ENV STORAGE_DRIVER=vfs \
|
||||||
|
BUILDAH_ISOLATION=chroot
|
||||||
|
|
||||||
|
# No third-party *Python* deps — the control plane stays stdlib only.
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
# Documentation only; lifecycle.py overrides the entrypoint to
|
# Documentation only; lifecycle.py overrides the entrypoint to
|
||||||
|
|||||||
@@ -61,6 +61,13 @@ class AgentProviderRuntime:
|
|||||||
prompt_mode: PromptMode
|
prompt_mode: PromptMode
|
||||||
bypass_args: tuple[str, ...]
|
bypass_args: tuple[str, ...]
|
||||||
resume_args: tuple[str, ...]
|
resume_args: tuple[str, ...]
|
||||||
|
# argv run inside a throwaway container of a freshly built agent
|
||||||
|
# image, right after `build_image()`, to catch a build that
|
||||||
|
# exited 0 but produced a broken CLI (e.g. an npm
|
||||||
|
# optionalDependencies fetch for a platform-native binary that
|
||||||
|
# silently no-ops on a transient failure). Empty tuple skips the
|
||||||
|
# check — not every provider has opted in yet.
|
||||||
|
smoke_test: tuple[str, ...] = ()
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ from contextlib import ExitStack, contextmanager
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Callable, Generator
|
from typing import Callable, Generator
|
||||||
|
|
||||||
|
from ...agent_provider import runtime_for
|
||||||
from ...egress import egress_resolve_token_values
|
from ...egress import egress_resolve_token_values
|
||||||
from ...git_gate import (
|
from ...git_gate import (
|
||||||
provision_git_gate_dynamic_keys,
|
provision_git_gate_dynamic_keys,
|
||||||
@@ -110,6 +111,9 @@ def launch(
|
|||||||
plan.image, _REPO_DIR,
|
plan.image, _REPO_DIR,
|
||||||
dockerfile=plan.dockerfile_path,
|
dockerfile=plan.dockerfile_path,
|
||||||
)
|
)
|
||||||
|
docker_mod.verify_agent_image(
|
||||||
|
plan.image, runtime_for(plan.agent_provider_template).smoke_test,
|
||||||
|
)
|
||||||
|
|
||||||
# Step 2: mint the git-gate dynamic (gitea) deploy keys, if any, before
|
# Step 2: mint the git-gate dynamic (gitea) deploy keys, if any, before
|
||||||
# provisioning the bottle's repos into the shared gateway.
|
# provisioning the bottle's repos into the shared gateway.
|
||||||
|
|||||||
@@ -4,11 +4,13 @@ existence, and building images."""
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
from typing import Iterable, Iterator
|
from typing import Iterable, Iterator
|
||||||
|
|
||||||
|
from ...docker_cmd import run_docker
|
||||||
from ...log import die, info
|
from ...log import die, info
|
||||||
# from ...workspace import WorkspacePlan
|
# from ...workspace import WorkspacePlan
|
||||||
|
|
||||||
@@ -88,6 +90,29 @@ def docker_exec_root(container: str, argv: list[str]) -> None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def docker_exec(container: str, argv: list[str], *, user: str = "") -> None:
|
||||||
|
"""Run `docker exec` in the named container, dying with the
|
||||||
|
command's own stderr on failure. Pass `user=\"0\"` to run as root."""
|
||||||
|
cmd = ["docker", "exec"]
|
||||||
|
if user:
|
||||||
|
cmd += ["-u", user]
|
||||||
|
cmd += [container, *argv]
|
||||||
|
result = run_docker(cmd)
|
||||||
|
if result.returncode != 0:
|
||||||
|
die(
|
||||||
|
f"docker exec in {container} failed: "
|
||||||
|
f"{(result.stderr or '').strip() or '<no stderr>'}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def docker_cp(src: str, dest: str) -> None:
|
||||||
|
"""Run `docker cp`, dying with the command's own stderr on failure."""
|
||||||
|
result = run_docker(["docker", "cp", src, dest])
|
||||||
|
if result.returncode != 0:
|
||||||
|
die(f"docker cp {src} -> {dest} failed: "
|
||||||
|
f"{(result.stderr or '').strip() or '<no stderr>'}")
|
||||||
|
|
||||||
|
|
||||||
_SLUG_RE = re.compile(r"[^a-z0-9]+")
|
_SLUG_RE = re.compile(r"[^a-z0-9]+")
|
||||||
|
|
||||||
|
|
||||||
@@ -108,15 +133,40 @@ def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
|
|||||||
|
|
||||||
`dockerfile` is an optional path (relative to `context`, or
|
`dockerfile` is an optional path (relative to `context`, or
|
||||||
absolute) for callers that need to build from a non-default
|
absolute) for callers that need to build from a non-default
|
||||||
Dockerfile in the same context — e.g. `Dockerfile.git-gate`."""
|
Dockerfile in the same context — e.g. `Dockerfile.git-gate`.
|
||||||
|
|
||||||
|
Set `BOT_BOTTLE_NO_CACHE=1` (the `start --no-cache` flag) to force
|
||||||
|
`--no-cache`. The npm/curl installers some provider Dockerfiles
|
||||||
|
shell out to can silently no-op on a transient network failure —
|
||||||
|
e.g. an `optionalDependencies` fetch for a platform-native binary —
|
||||||
|
and Docker will then cache that broken layer indefinitely."""
|
||||||
info(f"building image {ref} from {context} (layer cache keeps repeat builds fast)")
|
info(f"building image {ref} from {context} (layer cache keeps repeat builds fast)")
|
||||||
args = ["docker", "build", "-t", ref]
|
args = ["docker", "build", "-t", ref]
|
||||||
|
if os.environ.get("BOT_BOTTLE_NO_CACHE") == "1":
|
||||||
|
args.append("--no-cache")
|
||||||
if dockerfile:
|
if dockerfile:
|
||||||
args.extend(["-f", dockerfile])
|
args.extend(["-f", dockerfile])
|
||||||
args.append(context)
|
args.append(context)
|
||||||
subprocess.run(args, check=True)
|
subprocess.run(args, check=True)
|
||||||
|
|
||||||
|
|
||||||
|
def verify_agent_image(image: str, argv: tuple[str, ...]) -> None:
|
||||||
|
"""Run `argv` inside a throwaway container of a freshly built agent
|
||||||
|
image and die loudly if it fails, instead of shipping an image
|
||||||
|
whose CLI only breaks at first real use. No-op when the provider
|
||||||
|
hasn't declared a smoke test (`AgentProviderRuntime.smoke_test`)."""
|
||||||
|
if not argv:
|
||||||
|
return
|
||||||
|
result = run_docker(["docker", "run", "--rm", "--entrypoint", argv[0], image, *argv[1:]])
|
||||||
|
if result.returncode != 0:
|
||||||
|
detail = (result.stderr or result.stdout or "").strip()
|
||||||
|
die(
|
||||||
|
f"agent image {image!r} failed its post-build smoke test "
|
||||||
|
f"({' '.join(argv)}): {detail}\n"
|
||||||
|
f"Try rebuilding from scratch: bot-bottle start --no-cache"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# def build_image_with_cwd(
|
# def build_image_with_cwd(
|
||||||
# derived: str,
|
# derived: str,
|
||||||
# base: str,
|
# base: str,
|
||||||
|
|||||||
@@ -0,0 +1,164 @@
|
|||||||
|
"""Consolidated bottle launch sequence for the Firecracker backend (PRD 0070).
|
||||||
|
|
||||||
|
Mirrors bot_bottle.backend.docker.consolidated_launch but wired for
|
||||||
|
Firecracker's TAP-based network topology instead of a shared Docker bridge.
|
||||||
|
|
||||||
|
The per-bottle sidecar bundle (one `docker run` per bottle, published on the
|
||||||
|
slot's host-side TAP IP) is replaced by a single persistent gateway that
|
||||||
|
every Firecracker VM shares. The gateway runs as a Docker container in the
|
||||||
|
dev-harness (a Firecracker VM is stage B per PRD 0070), with its ports
|
||||||
|
published on the host (`0.0.0.0:PORT`). VMs reach it at their slot's
|
||||||
|
host-side TAP IP because Docker's iptables PREROUTING DNAT redirects
|
||||||
|
port 9099/9100/9420 traffic to the gateway container — a path the nft
|
||||||
|
isolation table already allows via `ct status dnat accept` in the forward
|
||||||
|
chain.
|
||||||
|
|
||||||
|
Attribution is by the VM's guest IP, which is unspoofable by construction:
|
||||||
|
the /31 point-to-point TAP topology + the `bot_bottle_fc` nft table ensure
|
||||||
|
that only the expected VM can source-IP that address.
|
||||||
|
|
||||||
|
Sequence:
|
||||||
|
1. ensure the Firecracker-flavoured orchestrator + gateway are up;
|
||||||
|
2. register the bottle by its guest IP (attribution key) → bottle id +
|
||||||
|
identity token;
|
||||||
|
3. provision its git-gate repos/creds into the running gateway;
|
||||||
|
4. fetch the shared gateway CA for the provisioner to install in the rootfs.
|
||||||
|
|
||||||
|
The TAP slot allocation, rootfs build, and VM boot are the caller's job.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from ...egress import EgressPlan
|
||||||
|
from ...git_gate import GitGatePlan
|
||||||
|
from ...orchestrator.client import OrchestratorClient
|
||||||
|
from ...orchestrator.gateway import (
|
||||||
|
GATEWAY_NETWORK,
|
||||||
|
DockerGateway,
|
||||||
|
)
|
||||||
|
from ...orchestrator.lifecycle import (
|
||||||
|
OrchestratorService,
|
||||||
|
OrchestratorStartError, # re-exported so callers can catch it
|
||||||
|
)
|
||||||
|
from ...orchestrator.registration import registration_inputs
|
||||||
|
from ..docker.egress import EGRESS_PORT
|
||||||
|
from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate
|
||||||
|
from ...supervise import SUPERVISE_PORT
|
||||||
|
|
||||||
|
_GIT_HTTP_PORT = 9420
|
||||||
|
|
||||||
|
# Separate names from the Docker gateway so both backends can coexist on one
|
||||||
|
# host (and for clarity in `docker ps` output).
|
||||||
|
_FC_GATEWAY_NAME = "bot-bottle-fc-gateway"
|
||||||
|
_FC_ORCHESTRATOR_NAME = "bot-bottle-fc-orchestrator"
|
||||||
|
_FC_ORCHESTRATOR_LABEL = "bot-bottle-fc-orchestrator=1"
|
||||||
|
# Ports the gateway publishes on the host so Firecracker VMs can reach it
|
||||||
|
# via their TAP link. Docker's PREROUTING DNAT + nft's `ct status dnat
|
||||||
|
# accept` in the forward chain route the traffic.
|
||||||
|
_FC_GATEWAY_HOST_PORTS = (EGRESS_PORT, SUPERVISE_PORT, _GIT_HTTP_PORT)
|
||||||
|
|
||||||
|
|
||||||
|
class ConsolidatedLaunchError(RuntimeError):
|
||||||
|
"""The consolidated register/provision sequence could not complete."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class LaunchContext:
|
||||||
|
"""What the Firecracker launch needs from the consolidated sequence."""
|
||||||
|
|
||||||
|
bottle_id: str
|
||||||
|
identity_token: str
|
||||||
|
source_ip: str # the VM's guest IP — the attribution key
|
||||||
|
gateway_ca_pem: str # the shared gateway CA the provisioner installs
|
||||||
|
orchestrator_url: str
|
||||||
|
|
||||||
|
|
||||||
|
class _FirecrackerOrchestratorService(OrchestratorService):
|
||||||
|
"""Dev-harness orchestrator for the Firecracker backend.
|
||||||
|
|
||||||
|
Uses a gateway that publishes its ports on the host so Firecracker VMs can
|
||||||
|
reach it via their TAP link. The gateway and orchestrator containers use
|
||||||
|
`*-fc-*` names so both backends can run independently on the same host.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, **kwargs: object) -> None:
|
||||||
|
super().__init__(
|
||||||
|
orchestrator_name=_FC_ORCHESTRATOR_NAME,
|
||||||
|
orchestrator_label=_FC_ORCHESTRATOR_LABEL,
|
||||||
|
**kwargs, # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
|
||||||
|
def _gateway(self) -> DockerGateway:
|
||||||
|
# The heavy data-plane image (#384 split it from the lean control-plane
|
||||||
|
# `image` this service's orchestrator container runs).
|
||||||
|
return DockerGateway(
|
||||||
|
self._gateway_image,
|
||||||
|
name=_FC_GATEWAY_NAME,
|
||||||
|
network=self.network,
|
||||||
|
orchestrator_url=self.internal_url,
|
||||||
|
host_port_bindings=_FC_GATEWAY_HOST_PORTS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def launch_consolidated(
|
||||||
|
egress_plan: EgressPlan,
|
||||||
|
git_gate_plan: GitGatePlan,
|
||||||
|
*,
|
||||||
|
guest_ip: str,
|
||||||
|
image_ref: str = "",
|
||||||
|
tokens: dict[str, str] | None = None,
|
||||||
|
service: OrchestratorService | None = None,
|
||||||
|
gateway_name: str = _FC_GATEWAY_NAME,
|
||||||
|
) -> LaunchContext:
|
||||||
|
"""Ensure the orchestrator + Firecracker gateway are up, register the
|
||||||
|
bottle by its guest IP, and provision its git-gate state. Returns the
|
||||||
|
context the VM launch needs. Raises `ConsolidatedLaunchError` (or the
|
||||||
|
primitives' own errors) on failure — the caller tears down on failure."""
|
||||||
|
service = service or _FirecrackerOrchestratorService()
|
||||||
|
url = service.ensure_running()
|
||||||
|
client = OrchestratorClient(url)
|
||||||
|
|
||||||
|
inputs = registration_inputs(egress_plan)
|
||||||
|
reg = client.register_bottle(
|
||||||
|
guest_ip, image_ref=image_ref, policy=inputs.policy,
|
||||||
|
metadata=inputs.metadata, tokens=tokens,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
provision_git_gate(gateway_name, reg.bottle_id, git_gate_plan)
|
||||||
|
except Exception:
|
||||||
|
client.teardown_bottle(reg.bottle_id)
|
||||||
|
raise
|
||||||
|
|
||||||
|
# Fetch the shared gateway CA here so the caller can install it in the
|
||||||
|
# rootfs (the same CA every agent on this host trusts for TLS interception).
|
||||||
|
gateway_ca_pem = DockerGateway(
|
||||||
|
name=gateway_name, network=GATEWAY_NETWORK,
|
||||||
|
).ca_cert_pem()
|
||||||
|
|
||||||
|
return LaunchContext(
|
||||||
|
bottle_id=reg.bottle_id,
|
||||||
|
identity_token=reg.identity_token,
|
||||||
|
source_ip=guest_ip,
|
||||||
|
gateway_ca_pem=gateway_ca_pem,
|
||||||
|
orchestrator_url=url,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def teardown_consolidated(
|
||||||
|
bottle_id: str, *, orchestrator_url: str, gateway_name: str = _FC_GATEWAY_NAME,
|
||||||
|
) -> None:
|
||||||
|
"""Deregister the bottle and remove its git-gate state from the gateway.
|
||||||
|
Both steps are idempotent so this is safe from a cleanup trap."""
|
||||||
|
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
|
||||||
|
deprovision_git_gate(gateway_name, bottle_id)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"LaunchContext",
|
||||||
|
"launch_consolidated",
|
||||||
|
"teardown_consolidated",
|
||||||
|
"ConsolidatedLaunchError",
|
||||||
|
"OrchestratorStartError",
|
||||||
|
]
|
||||||
@@ -1,25 +1,64 @@
|
|||||||
"""Launch flow for the Firecracker backend — temporarily disabled (#385).
|
"""Launch flow for the Firecracker backend (PRD 0070, consolidated).
|
||||||
|
|
||||||
The firecracker backend launched a per-bottle companion container (the
|
Per bottle:
|
||||||
egress / git-gate / supervise data plane) alongside each microVM. That
|
1. build the agent image (docker), export it to a cached ext4 rootfs;
|
||||||
per-bottle-companion architecture was removed in the companion-container removal;
|
2. ensure the per-host orchestrator + shared gateway are up;
|
||||||
firecracker's replacement — the consolidated per-host gateway — lands in
|
3. claim a free TAP pool slot (rootless flock);
|
||||||
its own cutover (#354).
|
4. register the bottle on the orchestrator by the VM's guest IP (the
|
||||||
|
attribution key) and provision its git-gate state into the gateway;
|
||||||
|
5. boot the microVM on that TAP; wait for SSH;
|
||||||
|
6. provision (shared gateway CA, prompt, skills, workspace, git, supervise)
|
||||||
|
over SSH.
|
||||||
|
|
||||||
Until that lands, launching a firecracker bottle fails closed rather than
|
The per-bottle Docker sidecar bundle is gone. The shared gateway handles
|
||||||
silently running the removed path. `prepare` / `status` / cleanup still
|
egress / git-gate / supervise for every VM; Docker's PREROUTING DNAT routes
|
||||||
work, so `backend status --backend=firecracker` and orphan cleanup are
|
the VMs' traffic to it, and the nft table's `ct status dnat accept` rule
|
||||||
unaffected.
|
in the forward chain lets it pass. The VM still sends to `host_tap_ip:PORT`
|
||||||
|
— the address its world is, by nft design, limited to.
|
||||||
|
|
||||||
|
Isolation is enforced by the operator-provisioned nft table (checked
|
||||||
|
fail-closed in preflight): a VM reaches only the sidecar (DNAT'd from
|
||||||
|
the host TAP IP) and nothing else.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from contextlib import contextmanager
|
import dataclasses
|
||||||
|
import os
|
||||||
|
from contextlib import ExitStack, contextmanager
|
||||||
|
from pathlib import Path
|
||||||
from typing import Callable, Generator
|
from typing import Callable, Generator
|
||||||
|
|
||||||
from ...log import die
|
from ...agent_provider import runtime_for
|
||||||
|
from ...bottle_state import (
|
||||||
|
egress_state_dir,
|
||||||
|
git_gate_state_dir,
|
||||||
|
read_committed_image,
|
||||||
|
)
|
||||||
|
from ...egress import (
|
||||||
|
egress_agent_env_entries,
|
||||||
|
egress_resolve_token_values,
|
||||||
|
)
|
||||||
|
from ...git_gate import (
|
||||||
|
provision_git_gate_dynamic_keys,
|
||||||
|
revoke_git_gate_provisioned_keys,
|
||||||
|
)
|
||||||
|
from ...log import info, warn
|
||||||
|
from ...supervise import SUPERVISE_PORT
|
||||||
|
from ..docker import util as docker_mod
|
||||||
|
from ..docker.egress import EGRESS_PORT
|
||||||
|
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||||
|
from . import firecracker_vm, isolation_probe, netpool, util
|
||||||
from .bottle import FirecrackerBottle
|
from .bottle import FirecrackerBottle
|
||||||
from .bottle_plan import FirecrackerBottlePlan
|
from .bottle_plan import FirecrackerBottlePlan
|
||||||
|
from .consolidated_launch import (
|
||||||
|
launch_consolidated,
|
||||||
|
teardown_consolidated,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
|
||||||
|
_GIT_HTTP_PORT = 9420
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
@@ -28,12 +67,181 @@ def launch(
|
|||||||
*,
|
*,
|
||||||
provision: Callable[[FirecrackerBottlePlan, "FirecrackerBottle"], str | None],
|
provision: Callable[[FirecrackerBottlePlan, "FirecrackerBottle"], str | None],
|
||||||
) -> Generator[FirecrackerBottle, None, None]:
|
) -> Generator[FirecrackerBottle, None, None]:
|
||||||
"""Fail closed: the firecracker backend is disabled while its
|
"""Build, launch, and provision a Firecracker bottle via the consolidated
|
||||||
consolidated (gateway-backed) launch is built in #354."""
|
orchestrator. Teardown on exit."""
|
||||||
del plan, provision
|
stack = ExitStack()
|
||||||
die(
|
bottle_for_revoke = plan.manifest.bottle
|
||||||
"the firecracker backend is temporarily disabled during the "
|
git_gate_dir_for_revoke = git_gate_state_dir(plan.slug)
|
||||||
"companion-container removal (#385); its consolidated relaunch "
|
|
||||||
"lands in #354. Use --backend=docker for now."
|
def teardown() -> None:
|
||||||
|
teardown_exc: BaseException | None = None
|
||||||
|
try:
|
||||||
|
stack.close()
|
||||||
|
except BaseException as exc: # noqa: W0718 - teardown must continue
|
||||||
|
teardown_exc = exc
|
||||||
|
warn(f"firecracker teardown failed: {exc!r}")
|
||||||
|
revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke)
|
||||||
|
if teardown_exc is not None:
|
||||||
|
raise teardown_exc
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Step 1: agent image. The sidecar bundle image is built by the
|
||||||
|
# orchestrator service (ensure_running → ensure_built); we only
|
||||||
|
# build the agent image here. Use a committed snapshot when available.
|
||||||
|
plan = _build_agent_image(plan)
|
||||||
|
|
||||||
|
# Step 2: mint the git-gate dynamic (gitea) deploy keys, if any.
|
||||||
|
git_gate_plan = plan.git_gate_plan
|
||||||
|
if git_gate_plan.upstreams:
|
||||||
|
git_gate_plan = provision_git_gate_dynamic_keys(
|
||||||
|
plan.manifest.bottle, git_gate_plan, git_gate_state_dir(plan.slug),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 3: claim a TAP slot; the flock is held until teardown.
|
||||||
|
slot, lock = netpool.allocate(plan.slug)
|
||||||
|
stack.callback(lock.close)
|
||||||
|
info(f"firecracker slot {slot.iface}: host={slot.host_ip} "
|
||||||
|
f"guest={slot.guest_ip}")
|
||||||
|
|
||||||
|
# Step 4: register on the orchestrator + provision this bottle's
|
||||||
|
# git-gate state into the shared gateway. The per-bottle egress tokens
|
||||||
|
# are resolved from the host env now and handed to the orchestrator
|
||||||
|
# (in memory) for the gateway to inject — the agent never sees them.
|
||||||
|
# Attribution is by the VM's guest IP (unspoofable via /31 TAP + nft).
|
||||||
|
effective_env = {**os.environ, **plan.agent_provision.provisioned_env}
|
||||||
|
token_values = egress_resolve_token_values(
|
||||||
|
plan.egress_plan.token_env_map, effective_env,
|
||||||
|
)
|
||||||
|
ctx = launch_consolidated(
|
||||||
|
plan.egress_plan, git_gate_plan,
|
||||||
|
guest_ip=slot.guest_ip,
|
||||||
|
image_ref=plan.image,
|
||||||
|
tokens=token_values,
|
||||||
|
)
|
||||||
|
stack.callback(
|
||||||
|
teardown_consolidated, ctx.bottle_id,
|
||||||
|
orchestrator_url=ctx.orchestrator_url,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 5: install the SHARED gateway CA (replaces the per-bottle CA).
|
||||||
|
# Write it to a stable host path so the provisioner can copy it over SSH.
|
||||||
|
ca_dir = egress_state_dir(plan.slug) / "gateway-ca"
|
||||||
|
ca_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
ca_file = ca_dir / "gateway-ca.pem"
|
||||||
|
ca_file.write_text(ctx.gateway_ca_pem)
|
||||||
|
egress_plan = dataclasses.replace(
|
||||||
|
plan.egress_plan,
|
||||||
|
mitmproxy_ca_host_path=ca_file,
|
||||||
|
mitmproxy_ca_cert_only_host_path=ca_file,
|
||||||
|
)
|
||||||
|
# Point the agent's git-gate insteadOf rewrites and supervise MCP URL
|
||||||
|
# at the shared gateway (reached at the slot's host TAP IP — the VM
|
||||||
|
# sends there and Docker DNAT routes to the gateway container).
|
||||||
|
git_gate_url = (
|
||||||
|
f"http://{slot.host_ip}:{_GIT_HTTP_PORT}" if git_gate_plan.upstreams else ""
|
||||||
|
)
|
||||||
|
supervise_url = (
|
||||||
|
f"http://{slot.host_ip}:{SUPERVISE_PORT}/"
|
||||||
|
if plan.supervise_plan is not None else ""
|
||||||
|
)
|
||||||
|
plan = dataclasses.replace(
|
||||||
|
plan,
|
||||||
|
git_gate_plan=git_gate_plan,
|
||||||
|
egress_plan=egress_plan,
|
||||||
|
agent_proxy_url=f"http://{slot.host_ip}:{EGRESS_PORT}",
|
||||||
|
agent_git_gate_url=git_gate_url,
|
||||||
|
agent_supervise_url=supervise_url,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 6: build the per-bottle rootfs + SSH key, then boot.
|
||||||
|
base_dir = util.build_base_rootfs_dir(plan.image)
|
||||||
|
run_dir = util.cache_dir() / "run" / plan.slug
|
||||||
|
run_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
rootfs = run_dir / "rootfs.ext4"
|
||||||
|
util.build_rootfs_ext4(base_dir, rootfs)
|
||||||
|
private_key, pubkey = util.generate_keypair(run_dir)
|
||||||
|
|
||||||
|
vm = firecracker_vm.boot(
|
||||||
|
name=plan.container_name,
|
||||||
|
rootfs=rootfs,
|
||||||
|
tap=slot.iface,
|
||||||
|
guest_ip=slot.guest_ip,
|
||||||
|
host_ip=slot.host_ip,
|
||||||
|
pubkey=pubkey,
|
||||||
|
run_dir=run_dir,
|
||||||
|
)
|
||||||
|
stack.callback(vm.terminate)
|
||||||
|
firecracker_vm.wait_for_ssh(vm, private_key)
|
||||||
|
|
||||||
|
# Authoritative fail-closed egress-boundary check, before the agent
|
||||||
|
# runs: prove the VM cannot reach the host directly.
|
||||||
|
isolation_probe.verify_isolation(private_key, slot.guest_ip)
|
||||||
|
|
||||||
|
bottle = FirecrackerBottle(
|
||||||
|
plan.container_name,
|
||||||
|
private_key=private_key,
|
||||||
|
guest_ip=slot.guest_ip,
|
||||||
|
guest_env=_agent_guest_env(plan, slot.host_ip),
|
||||||
|
agent_command=plan.agent_command,
|
||||||
|
agent_prompt_mode=plan.agent_prompt_mode,
|
||||||
|
agent_provider_template=plan.agent_provider_template,
|
||||||
|
terminal_title=(
|
||||||
|
f"{plan.spec.label} ({plan.spec.agent_name})"
|
||||||
|
if plan.spec.label else plan.spec.agent_name
|
||||||
|
),
|
||||||
|
terminal_color=plan.spec.color,
|
||||||
|
agent_workdir=plan.workspace_plan.workdir,
|
||||||
|
)
|
||||||
|
bottle.prompt_path = provision(plan, bottle)
|
||||||
|
|
||||||
|
yield bottle
|
||||||
|
finally:
|
||||||
|
teardown()
|
||||||
|
|
||||||
|
|
||||||
|
def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
|
||||||
|
committed = read_committed_image(plan.slug)
|
||||||
|
if committed and docker_mod.image_exists(committed):
|
||||||
|
info(f"using committed image {committed!r}")
|
||||||
|
return dataclasses.replace(
|
||||||
|
plan,
|
||||||
|
agent_provision=dataclasses.replace(plan.agent_provision, image=committed),
|
||||||
|
)
|
||||||
|
docker_mod.build_image(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path)
|
||||||
|
docker_mod.verify_agent_image(
|
||||||
|
plan.image, runtime_for(plan.agent_provider_template).smoke_test,
|
||||||
)
|
)
|
||||||
yield # unreachable — `die` raises; keeps this a generator/contextmanager
|
return plan
|
||||||
|
|
||||||
|
|
||||||
|
# --- agent guest env -------------------------------------------------
|
||||||
|
|
||||||
|
def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str]:
|
||||||
|
"""Env injected into every agent/exec call over SSH. The VM has no
|
||||||
|
baked process env (it just runs init), so the proxy/CA/git/supervise
|
||||||
|
wiring is applied per-invocation."""
|
||||||
|
proxy_url = f"http://{host_ip}:{EGRESS_PORT}"
|
||||||
|
no_proxy = f"localhost,127.0.0.1,{host_ip}"
|
||||||
|
env: dict[str, str] = {
|
||||||
|
"HTTPS_PROXY": proxy_url, "HTTP_PROXY": proxy_url,
|
||||||
|
"https_proxy": proxy_url, "http_proxy": proxy_url,
|
||||||
|
"NO_PROXY": no_proxy, "no_proxy": no_proxy,
|
||||||
|
"NODE_EXTRA_CA_CERTS": AGENT_CA_PATH,
|
||||||
|
"SSL_CERT_FILE": AGENT_CA_BUNDLE,
|
||||||
|
"REQUESTS_CA_BUNDLE": AGENT_CA_BUNDLE,
|
||||||
|
}
|
||||||
|
if plan.agent_git_gate_url:
|
||||||
|
env["GIT_GATE_URL"] = plan.agent_git_gate_url
|
||||||
|
if plan.agent_supervise_url:
|
||||||
|
env["MCP_SUPERVISE_URL"] = plan.agent_supervise_url
|
||||||
|
for entry in egress_agent_env_entries(plan.egress_plan):
|
||||||
|
key, _, value = entry.partition("=")
|
||||||
|
env[key] = value
|
||||||
|
env.update(plan.agent_provision.guest_env)
|
||||||
|
# Forwarded (bare-name) env: resolve host values now, since the VM
|
||||||
|
# can't inherit them from a `docker run --env NAME`.
|
||||||
|
for name in plan.forwarded_env:
|
||||||
|
value = os.environ.get(name)
|
||||||
|
if value is not None:
|
||||||
|
env[name] = value
|
||||||
|
return env
|
||||||
|
|||||||
@@ -15,3 +15,11 @@ BOT_BOTTLE_FC_POOL_SIZE=8
|
|||||||
BOT_BOTTLE_FC_IP_BASE=10.243.0.0
|
BOT_BOTTLE_FC_IP_BASE=10.243.0.0
|
||||||
BOT_BOTTLE_FC_IFACE_PREFIX=bbfc
|
BOT_BOTTLE_FC_IFACE_PREFIX=bbfc
|
||||||
BOT_BOTTLE_FC_NFT_TABLE=bot_bottle_fc
|
BOT_BOTTLE_FC_NFT_TABLE=bot_bottle_fc
|
||||||
|
# The orchestrator/gateway VM's own TAP — a dedicated link OUTSIDE the
|
||||||
|
# bbfc* agent pool. Unlike agent VMs (which reach only their gateway),
|
||||||
|
# the orchestrator is trusted infra that needs real NAT'd internet
|
||||||
|
# egress: to FROM-pull + apt/npm during in-VM agent-image builds
|
||||||
|
# (buildah) and to forward agent egress upstream (Stage B gateway). Its
|
||||||
|
# /31 is the top of the IP_BASE /16 (host x.y.255.0, guest x.y.255.1),
|
||||||
|
# clear of the pool near the bottom of the block.
|
||||||
|
BOT_BOTTLE_FC_ORCH_IFACE=bborch0
|
||||||
|
|||||||
@@ -79,6 +79,12 @@ def _cfg(key: str) -> str:
|
|||||||
IFACE_PREFIX = _cfg("BOT_BOTTLE_FC_IFACE_PREFIX")
|
IFACE_PREFIX = _cfg("BOT_BOTTLE_FC_IFACE_PREFIX")
|
||||||
NFT_TABLE = _cfg("BOT_BOTTLE_FC_NFT_TABLE")
|
NFT_TABLE = _cfg("BOT_BOTTLE_FC_NFT_TABLE")
|
||||||
|
|
||||||
|
# The orchestrator/gateway VM's dedicated TAP — outside the bbfc* agent
|
||||||
|
# pool and, unlike it, NAT'd to the internet (see `orch_slot`). The
|
||||||
|
# orchestrator is trusted infra: it builds agent images in-VM (buildah
|
||||||
|
# needs to FROM-pull + apt/npm) and forwards agent egress upstream.
|
||||||
|
ORCH_IFACE = _cfg("BOT_BOTTLE_FC_ORCH_IFACE")
|
||||||
|
|
||||||
|
|
||||||
def pool_size() -> int:
|
def pool_size() -> int:
|
||||||
return int(_cfg("BOT_BOTTLE_FC_POOL_SIZE"))
|
return int(_cfg("BOT_BOTTLE_FC_POOL_SIZE"))
|
||||||
@@ -123,6 +129,25 @@ def all_slots() -> list[Slot]:
|
|||||||
return [slot(i) for i in range(pool_size())]
|
return [slot(i) for i in range(pool_size())]
|
||||||
|
|
||||||
|
|
||||||
|
def orch_slot() -> Slot:
|
||||||
|
"""The orchestrator/gateway VM's dedicated link — its own TAP
|
||||||
|
(`ORCH_IFACE`) on a /31 at the TOP of the IP_BASE /16 (host
|
||||||
|
x.y.255.0, guest x.y.255.1), well clear of the agent pool near the
|
||||||
|
bottom of the block. Unlike a pool `Slot`, this link is NAT'd out to
|
||||||
|
the internet by the setup (the orchestrator is trusted infra), so it
|
||||||
|
is deliberately *not* one of the isolated `bbfc*` slots.
|
||||||
|
|
||||||
|
`index` is -1 (sentinel: not a pool index)."""
|
||||||
|
base16 = int(ipaddress.IPv4Address(ip_base())) & 0xFFFF0000
|
||||||
|
host = base16 + 0xFF00
|
||||||
|
return Slot(
|
||||||
|
index=-1,
|
||||||
|
iface=ORCH_IFACE,
|
||||||
|
host_ip=str(ipaddress.IPv4Address(host)),
|
||||||
|
guest_ip=str(ipaddress.IPv4Address(host + 1)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# --- fail-closed verification ---------------------------------------
|
# --- fail-closed verification ---------------------------------------
|
||||||
|
|
||||||
def _run_ok(argv: list[str]) -> bool:
|
def _run_ok(argv: list[str]) -> bool:
|
||||||
|
|||||||
@@ -60,13 +60,21 @@ def dns_server() -> str:
|
|||||||
|
|
||||||
|
|
||||||
def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
|
def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
|
||||||
"""Build an OCI image with Apple's BuildKit-backed `container build`."""
|
"""Build an OCI image with Apple's BuildKit-backed `container build`.
|
||||||
|
|
||||||
|
Set `BOT_BOTTLE_NO_CACHE=1` (the `start --no-cache` flag) to force
|
||||||
|
`--no-cache`. The npm/curl installers some provider Dockerfiles
|
||||||
|
shell out to can silently no-op on a transient network failure —
|
||||||
|
e.g. an `optionalDependencies` fetch for a platform-native binary —
|
||||||
|
and the builder will then cache that broken layer indefinitely."""
|
||||||
info(
|
info(
|
||||||
f"building image {ref} from {context} with Apple Container "
|
f"building image {ref} from {context} with Apple Container "
|
||||||
"(layer cache keeps repeat builds fast)"
|
"(layer cache keeps repeat builds fast)"
|
||||||
)
|
)
|
||||||
_ensure_builder_dns()
|
_ensure_builder_dns()
|
||||||
args = [_CONTAINER, "build", "-t", ref, "--dns", dns_server()]
|
args = [_CONTAINER, "build", "-t", ref, "--dns", dns_server()]
|
||||||
|
if os.environ.get("BOT_BOTTLE_NO_CACHE") == "1":
|
||||||
|
args.append("--no-cache")
|
||||||
if dockerfile:
|
if dockerfile:
|
||||||
# `container build` resolves -f relative to the current working
|
# `container build` resolves -f relative to the current working
|
||||||
# directory, not the build context. Anchor a relative Dockerfile to
|
# directory, not the build context. Anchor a relative Dockerfile to
|
||||||
@@ -78,6 +86,28 @@ def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
|
|||||||
subprocess.run(args, check=True)
|
subprocess.run(args, check=True)
|
||||||
|
|
||||||
|
|
||||||
|
def verify_agent_image(image: str, argv: tuple[str, ...]) -> None:
|
||||||
|
"""Run `argv` inside a throwaway container of a freshly built agent
|
||||||
|
image and die loudly if it fails, instead of shipping an image
|
||||||
|
whose CLI only breaks at first real use. No-op when the provider
|
||||||
|
hasn't declared a smoke test (`AgentProviderRuntime.smoke_test`)."""
|
||||||
|
if not argv:
|
||||||
|
return
|
||||||
|
result = subprocess.run(
|
||||||
|
[_CONTAINER, "run", "--rm", "--entrypoint", argv[0], image, *argv[1:]],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
detail = (result.stderr or result.stdout or "").strip()
|
||||||
|
die(
|
||||||
|
f"agent image {image!r} failed its post-build smoke test "
|
||||||
|
f"({' '.join(argv)}): {detail}\n"
|
||||||
|
f"Try rebuilding from scratch: bot-bottle start --no-cache"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def commit_container(container_name: str, image_tag: str) -> None:
|
def commit_container(container_name: str, image_tag: str) -> None:
|
||||||
"""Snapshot a running Apple Container as a local image.
|
"""Snapshot a running Apple Container as a local image.
|
||||||
|
|
||||||
|
|||||||
@@ -46,6 +46,17 @@ def cmd_start(argv: list[str]) -> int:
|
|||||||
parser = argparse.ArgumentParser(prog=f"{PROG} start", add_help=True)
|
parser = argparse.ArgumentParser(prog=f"{PROG} start", add_help=True)
|
||||||
parser.add_argument("--dry-run", action="store_true")
|
parser.add_argument("--dry-run", action="store_true")
|
||||||
parser.add_argument("--cwd", action="store_true", help="copy host cwd into the running bottle")
|
parser.add_argument("--cwd", action="store_true", help="copy host cwd into the running bottle")
|
||||||
|
parser.add_argument(
|
||||||
|
"--no-cache",
|
||||||
|
action="store_true",
|
||||||
|
help=(
|
||||||
|
"rebuild agent/sidecar images from scratch, bypassing the "
|
||||||
|
"build layer cache. Use when an image looks broken after a "
|
||||||
|
"dependency bump — e.g. an installer's optionalDependencies "
|
||||||
|
"fetch silently no-op'd on a transient failure and got baked "
|
||||||
|
"into a cached layer."
|
||||||
|
),
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--backend",
|
"--backend",
|
||||||
choices=known_backend_names(),
|
choices=known_backend_names(),
|
||||||
@@ -97,6 +108,11 @@ def cmd_start(argv: list[str]) -> int:
|
|||||||
args = parser.parse_args(argv)
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
dry_run = args.dry_run or os.environ.get("BOT_BOTTLE_DRY_RUN") == "1"
|
dry_run = args.dry_run or os.environ.get("BOT_BOTTLE_DRY_RUN") == "1"
|
||||||
|
if args.no_cache or os.environ.get("BOT_BOTTLE_NO_CACHE") == "1":
|
||||||
|
# Read by build_image() in each backend's util module — set here
|
||||||
|
# so both the interactive and --headless paths pick it up without
|
||||||
|
# threading a no_cache field through every backend's plan dataclass.
|
||||||
|
os.environ["BOT_BOTTLE_NO_CACHE"] = "1"
|
||||||
|
|
||||||
manifest = ManifestIndex.resolve(USER_CWD)
|
manifest = ManifestIndex.resolve(USER_CWD)
|
||||||
backend_name: str | None = args.backend
|
backend_name: str | None = args.backend
|
||||||
|
|||||||
@@ -91,6 +91,7 @@ _RUNTIME = AgentProviderRuntime(
|
|||||||
prompt_mode="append_file",
|
prompt_mode="append_file",
|
||||||
bypass_args=("--dangerously-skip-permissions",),
|
bypass_args=("--dangerously-skip-permissions",),
|
||||||
resume_args=("--continue",),
|
resume_args=("--continue",),
|
||||||
|
smoke_test=("claude", "--version"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ _RUNTIME = AgentProviderRuntime(
|
|||||||
prompt_mode="read_prompt_file",
|
prompt_mode="read_prompt_file",
|
||||||
bypass_args=("--dangerously-bypass-approvals-and-sandbox",),
|
bypass_args=("--dangerously-bypass-approvals-and-sandbox",),
|
||||||
resume_args=("resume", "--last"),
|
resume_args=("resume", "--last"),
|
||||||
|
smoke_test=(_CODEX_CLI, "--version"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -100,6 +100,7 @@ class DockerGateway(Gateway):
|
|||||||
orchestrator_url: str = "",
|
orchestrator_url: str = "",
|
||||||
build_context: Path | None = None,
|
build_context: Path | None = None,
|
||||||
dockerfile: str | None = GATEWAY_DOCKERFILE,
|
dockerfile: str | None = GATEWAY_DOCKERFILE,
|
||||||
|
host_port_bindings: tuple[int, ...] = (),
|
||||||
) -> None:
|
) -> None:
|
||||||
self.image_ref = image_ref
|
self.image_ref = image_ref
|
||||||
self.name = name
|
self.name = name
|
||||||
@@ -110,6 +111,10 @@ class DockerGateway(Gateway):
|
|||||||
self._orchestrator_url = orchestrator_url
|
self._orchestrator_url = orchestrator_url
|
||||||
self._build_context = build_context or _REPO_ROOT
|
self._build_context = build_context or _REPO_ROOT
|
||||||
self._dockerfile = dockerfile
|
self._dockerfile = dockerfile
|
||||||
|
# Ports published on the host (0.0.0.0). Used by the Firecracker
|
||||||
|
# backend's dev-harness gateway so VMs can reach it via their TAP link;
|
||||||
|
# Docker's DNAT + the nft `ct status dnat accept` rule handle the rest.
|
||||||
|
self._host_port_bindings = host_port_bindings
|
||||||
|
|
||||||
def image_exists(self) -> bool:
|
def image_exists(self) -> bool:
|
||||||
return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0
|
return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0
|
||||||
@@ -188,6 +193,8 @@ class DockerGateway(Gateway):
|
|||||||
# trust it) — see GATEWAY_CA_VOLUME.
|
# trust it) — see GATEWAY_CA_VOLUME.
|
||||||
"--volume", f"{GATEWAY_CA_VOLUME}:{MITMPROXY_HOME}",
|
"--volume", f"{GATEWAY_CA_VOLUME}:{MITMPROXY_HOME}",
|
||||||
]
|
]
|
||||||
|
for port in self._host_port_bindings:
|
||||||
|
argv += ["--publish", f"0.0.0.0:{port}:{port}"]
|
||||||
if self._orchestrator_url:
|
if self._orchestrator_url:
|
||||||
# Makes the gateway's egress / git / supervise daemons multi-tenant:
|
# Makes the gateway's egress / git / supervise daemons multi-tenant:
|
||||||
# each request resolves source-IP -> policy against the control plane.
|
# each request resolves source-IP -> policy against the control plane.
|
||||||
|
|||||||
@@ -78,7 +78,13 @@ def _source_hash(repo_root: Path) -> str:
|
|||||||
|
|
||||||
class OrchestratorService:
|
class OrchestratorService:
|
||||||
"""Manages the orchestrator control-plane container + the shared gateway.
|
"""Manages the orchestrator control-plane container + the shared gateway.
|
||||||
Callers only need `ensure_running()` + `url`."""
|
Callers only need `ensure_running()` + `url`.
|
||||||
|
|
||||||
|
`orchestrator_name` / `orchestrator_label` let backends run independent
|
||||||
|
orchestrators on the same host without name collisions (e.g. the
|
||||||
|
Firecracker backend uses `bot-bottle-fc-orchestrator` alongside the Docker
|
||||||
|
backend's `bot-bottle-orchestrator`). Subclass and override `_gateway()`
|
||||||
|
to supply a backend-specific gateway variant."""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -89,6 +95,8 @@ class OrchestratorService:
|
|||||||
gateway_image: str = GATEWAY_IMAGE,
|
gateway_image: str = GATEWAY_IMAGE,
|
||||||
repo_root: Path = _REPO_ROOT,
|
repo_root: Path = _REPO_ROOT,
|
||||||
host_root: Path | None = None,
|
host_root: Path | None = None,
|
||||||
|
orchestrator_name: str = ORCHESTRATOR_NAME,
|
||||||
|
orchestrator_label: str = ORCHESTRATOR_LABEL,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.port = port
|
self.port = port
|
||||||
self.network = network
|
self.network = network
|
||||||
@@ -100,6 +108,8 @@ class OrchestratorService:
|
|||||||
self._gateway_image = gateway_image
|
self._gateway_image = gateway_image
|
||||||
self._repo_root = repo_root
|
self._repo_root = repo_root
|
||||||
self._host_root = host_root or bot_bottle_root()
|
self._host_root = host_root or bot_bottle_root()
|
||||||
|
self._orchestrator_name = orchestrator_name
|
||||||
|
self._orchestrator_label = orchestrator_label
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def url(self) -> str:
|
def url(self) -> str:
|
||||||
@@ -111,7 +121,7 @@ class OrchestratorService:
|
|||||||
"""Control-plane URL as the gateway container reaches it — by name over
|
"""Control-plane URL as the gateway container reaches it — by name over
|
||||||
docker DNS on the shared network. This is the gateway's
|
docker DNS on the shared network. This is the gateway's
|
||||||
BOT_BOTTLE_ORCHESTRATOR_URL."""
|
BOT_BOTTLE_ORCHESTRATOR_URL."""
|
||||||
return f"http://{ORCHESTRATOR_NAME}:{self.port}"
|
return f"http://{self._orchestrator_name}:{self.port}"
|
||||||
|
|
||||||
def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool:
|
def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool:
|
||||||
try:
|
try:
|
||||||
@@ -129,11 +139,11 @@ class OrchestratorService:
|
|||||||
fixed-name container first). Register-only broker → no docker socket.
|
fixed-name container first). Register-only broker → no docker socket.
|
||||||
Labels the container with `source_hash` so a later `ensure_running`
|
Labels the container with `source_hash` so a later `ensure_running`
|
||||||
can detect a real code change (see `_source_hash`)."""
|
can detect a real code change (see `_source_hash`)."""
|
||||||
run_docker(["docker", "rm", "--force", ORCHESTRATOR_NAME])
|
run_docker(["docker", "rm", "--force", self._orchestrator_name])
|
||||||
proc = run_docker([
|
proc = run_docker([
|
||||||
"docker", "run", "--detach",
|
"docker", "run", "--detach",
|
||||||
"--name", ORCHESTRATOR_NAME,
|
"--name", self._orchestrator_name,
|
||||||
"--label", ORCHESTRATOR_LABEL,
|
"--label", self._orchestrator_label,
|
||||||
"--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={source_hash}",
|
"--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={source_hash}",
|
||||||
"--network", self.network,
|
"--network", self.network,
|
||||||
# Host CLI reaches the control plane here; bound to loopback so it
|
# Host CLI reaches the control plane here; bound to loopback so it
|
||||||
@@ -185,12 +195,12 @@ class OrchestratorService:
|
|||||||
*current* bind-mounted source. Mirrors `DockerGateway`'s
|
*current* bind-mounted source. Mirrors `DockerGateway`'s
|
||||||
image-staleness check, but by content hash rather than image id since
|
image-staleness check, but by content hash rather than image id since
|
||||||
the orchestrator runs bind-mounted source, not a built image."""
|
the orchestrator runs bind-mounted source, not a built image."""
|
||||||
if not self._container_running(ORCHESTRATOR_NAME):
|
if not self._container_running(self._orchestrator_name):
|
||||||
return False
|
return False
|
||||||
proc = run_docker([
|
proc = run_docker([
|
||||||
"docker", "inspect", "--format",
|
"docker", "inspect", "--format",
|
||||||
"{{ index .Config.Labels \"" + ORCHESTRATOR_SOURCE_HASH_LABEL + "\" }}",
|
"{{ index .Config.Labels \"" + ORCHESTRATOR_SOURCE_HASH_LABEL + "\" }}",
|
||||||
ORCHESTRATOR_NAME,
|
self._orchestrator_name,
|
||||||
])
|
])
|
||||||
if proc.returncode != 0:
|
if proc.returncode != 0:
|
||||||
return True # can't compare -> don't churn a working container
|
return True # can't compare -> don't churn a working container
|
||||||
@@ -219,7 +229,10 @@ class OrchestratorService:
|
|||||||
return self.url
|
return self.url
|
||||||
|
|
||||||
self._ensure_orchestrator_image()
|
self._ensure_orchestrator_image()
|
||||||
log.info("starting orchestrator container", context={"name": ORCHESTRATOR_NAME})
|
log.info(
|
||||||
|
"starting orchestrator container",
|
||||||
|
context={"name": self._orchestrator_name},
|
||||||
|
)
|
||||||
self._run_orchestrator_container(current_hash)
|
self._run_orchestrator_container(current_hash)
|
||||||
|
|
||||||
deadline = time.monotonic() + startup_timeout
|
deadline = time.monotonic() + startup_timeout
|
||||||
@@ -234,7 +247,7 @@ class OrchestratorService:
|
|||||||
|
|
||||||
def stop(self) -> None:
|
def stop(self) -> None:
|
||||||
"""Remove the orchestrator + gateway containers (idempotent)."""
|
"""Remove the orchestrator + gateway containers (idempotent)."""
|
||||||
run_docker(["docker", "rm", "--force", ORCHESTRATOR_NAME])
|
run_docker(["docker", "rm", "--force", self._orchestrator_name])
|
||||||
self._gateway().stop()
|
self._gateway().stop()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -49,10 +49,12 @@ let
|
|||||||
# /31 alignment == an even final octet (only bit 0 matters for base+2i).
|
# /31 alignment == an even final octet (only bit 0 matters for base+2i).
|
||||||
lastOctet = lib.toInt (lib.last (lib.splitString "." cfg.ipBase));
|
lastOctet = lib.toInt (lib.last (lib.splitString "." cfg.ipBase));
|
||||||
|
|
||||||
# The script needs ip/nft/sysctl + the usual coreutils. It gets every
|
# The script needs ip/nft/sysctl + the usual coreutils, plus iptables
|
||||||
# pool value via the unit's Environment=, so it never reads the shared
|
# for the orchestrator link's DOCKER-USER accept (best-effort; skipped
|
||||||
# defaults file (which isn't beside it once copied to the store).
|
# when Docker is absent). It gets every pool value via the unit's
|
||||||
runtimePath = with pkgs; [ iproute2 nftables procps coreutils gnused ];
|
# Environment=, so it never reads the shared defaults file (which isn't
|
||||||
|
# beside it once copied to the store).
|
||||||
|
runtimePath = with pkgs; [ iproute2 nftables iptables procps coreutils gnused ];
|
||||||
|
|
||||||
ownEnv =
|
ownEnv =
|
||||||
if cfg.group != null
|
if cfg.group != null
|
||||||
@@ -64,6 +66,7 @@ let
|
|||||||
BOT_BOTTLE_FC_IP_BASE = cfg.ipBase;
|
BOT_BOTTLE_FC_IP_BASE = cfg.ipBase;
|
||||||
BOT_BOTTLE_FC_IFACE_PREFIX = cfg.ifacePrefix;
|
BOT_BOTTLE_FC_IFACE_PREFIX = cfg.ifacePrefix;
|
||||||
BOT_BOTTLE_FC_NFT_TABLE = cfg.tableName;
|
BOT_BOTTLE_FC_NFT_TABLE = cfg.tableName;
|
||||||
|
BOT_BOTTLE_FC_ORCH_IFACE = cfg.orchIface;
|
||||||
} // ownEnv;
|
} // ownEnv;
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
@@ -127,6 +130,19 @@ in
|
|||||||
description = "nftables table name for the isolation boundary. Must match netpool.NFT_TABLE.";
|
description = "nftables table name for the isolation boundary. Must match netpool.NFT_TABLE.";
|
||||||
};
|
};
|
||||||
|
|
||||||
|
orchIface = lib.mkOption {
|
||||||
|
type = lib.types.str;
|
||||||
|
default = defaults.BOT_BOTTLE_FC_ORCH_IFACE;
|
||||||
|
defaultText = lib.literalMD "the shared `netpool.defaults.env` value";
|
||||||
|
description = ''
|
||||||
|
TAP name for the orchestrator/gateway VM's dedicated link. Unlike
|
||||||
|
the isolated bbfc* agent pool, this link is NAT'd to the internet
|
||||||
|
(the orchestrator is trusted infra that builds agent images in-VM
|
||||||
|
and forwards agent egress upstream). Must match
|
||||||
|
BOT_BOTTLE_FC_ORCH_IFACE / netpool.ORCH_IFACE.
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
writeEnvFile = lib.mkOption {
|
writeEnvFile = lib.mkOption {
|
||||||
type = lib.types.bool;
|
type = lib.types.bool;
|
||||||
default = false;
|
default = false;
|
||||||
@@ -176,6 +192,7 @@ in
|
|||||||
BOT_BOTTLE_FC_IP_BASE=${cfg.ipBase}
|
BOT_BOTTLE_FC_IP_BASE=${cfg.ipBase}
|
||||||
BOT_BOTTLE_FC_IFACE_PREFIX=${cfg.ifacePrefix}
|
BOT_BOTTLE_FC_IFACE_PREFIX=${cfg.ifacePrefix}
|
||||||
BOT_BOTTLE_FC_NFT_TABLE=${cfg.tableName}
|
BOT_BOTTLE_FC_NFT_TABLE=${cfg.tableName}
|
||||||
|
BOT_BOTTLE_FC_ORCH_IFACE=${cfg.orchIface}
|
||||||
'';
|
'';
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -63,12 +63,13 @@ POOL_SIZE="${BOT_BOTTLE_FC_POOL_SIZE:-$(_default BOT_BOTTLE_FC_POOL_SIZE)}"
|
|||||||
IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-$(_default BOT_BOTTLE_FC_IP_BASE)}"
|
IP_BASE="${BOT_BOTTLE_FC_IP_BASE:-$(_default BOT_BOTTLE_FC_IP_BASE)}"
|
||||||
PREFIX="${BOT_BOTTLE_FC_IFACE_PREFIX:-$(_default BOT_BOTTLE_FC_IFACE_PREFIX)}"
|
PREFIX="${BOT_BOTTLE_FC_IFACE_PREFIX:-$(_default BOT_BOTTLE_FC_IFACE_PREFIX)}"
|
||||||
TABLE="${BOT_BOTTLE_FC_NFT_TABLE:-$(_default BOT_BOTTLE_FC_NFT_TABLE)}"
|
TABLE="${BOT_BOTTLE_FC_NFT_TABLE:-$(_default BOT_BOTTLE_FC_NFT_TABLE)}"
|
||||||
|
ORCH_IFACE="${BOT_BOTTLE_FC_ORCH_IFACE:-$(_default BOT_BOTTLE_FC_ORCH_IFACE)}"
|
||||||
OWNER="${BOT_BOTTLE_FC_OWNER:-${SUDO_USER:-$USER}}"
|
OWNER="${BOT_BOTTLE_FC_OWNER:-${SUDO_USER:-$USER}}"
|
||||||
GROUP="${BOT_BOTTLE_FC_GROUP:-}"
|
GROUP="${BOT_BOTTLE_FC_GROUP:-}"
|
||||||
|
|
||||||
# Fail loudly rather than provisioning a half/empty range if a value
|
# Fail loudly rather than provisioning a half/empty range if a value
|
||||||
# resolved to nothing (env unset AND the shared file unreadable).
|
# resolved to nothing (env unset AND the shared file unreadable).
|
||||||
for _v in POOL_SIZE IP_BASE PREFIX TABLE; do
|
for _v in POOL_SIZE IP_BASE PREFIX TABLE ORCH_IFACE; do
|
||||||
[ -n "${!_v}" ] || { echo "error: $_v unresolved (set BOT_BOTTLE_FC_* or fix $_DEFAULTS)" >&2; exit 1; }
|
[ -n "${!_v}" ] || { echo "error: $_v unresolved (set BOT_BOTTLE_FC_* or fix $_DEFAULTS)" >&2; exit 1; }
|
||||||
done
|
done
|
||||||
|
|
||||||
@@ -89,6 +90,13 @@ host_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 )); }
|
|||||||
guest_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 + 1 )); }
|
guest_ip() { _int_to_ip $(( $(_ip_to_int "$IP_BASE") + 2*$1 + 1 )); }
|
||||||
iface() { echo "${PREFIX}$1"; }
|
iface() { echo "${PREFIX}$1"; }
|
||||||
|
|
||||||
|
# Orchestrator/gateway VM link: a /31 at the TOP of the IP_BASE /16
|
||||||
|
# (host x.y.255.0, guest x.y.255.1), well clear of the agent pool near
|
||||||
|
# the bottom of the block. Must match netpool.py:orch_slot().
|
||||||
|
_orch_base() { echo $(( ($(_ip_to_int "$IP_BASE") & 0xFFFF0000) + 0xFF00 )); }
|
||||||
|
orch_host() { _int_to_ip "$(_orch_base)"; }
|
||||||
|
orch_guest() { _int_to_ip $(( $(_orch_base) + 1 )); }
|
||||||
|
|
||||||
require_root() {
|
require_root() {
|
||||||
if [ "$(id -u)" -ne 0 ]; then
|
if [ "$(id -u)" -ne 0 ]; then
|
||||||
echo "error: '$1' needs root (run under sudo)" >&2
|
echo "error: '$1' needs root (run under sudo)" >&2
|
||||||
@@ -125,8 +133,19 @@ cmd_up() {
|
|||||||
echo " $dev host=$host guest=$(guest_ip "$i") $own_desc"
|
echo " $dev host=$host guest=$(guest_ip "$i") $own_desc"
|
||||||
done
|
done
|
||||||
|
|
||||||
|
# The orchestrator/gateway VM's dedicated link. Same rootless-open
|
||||||
|
# ownership as the pool, but NAT'd to the internet (below) — it is
|
||||||
|
# trusted infra, not an isolated agent slot.
|
||||||
|
ip link show "$ORCH_IFACE" >/dev/null 2>&1 \
|
||||||
|
|| ip tuntap add dev "$ORCH_IFACE" mode tap "${own_args[@]}"
|
||||||
|
ip addr replace "$(orch_host)/31" dev "$ORCH_IFACE"
|
||||||
|
ip link set "$ORCH_IFACE" up
|
||||||
|
echo " $ORCH_IFACE host=$(orch_host) guest=$(orch_guest) (NAT'd egress) $own_desc"
|
||||||
|
|
||||||
_install_nft
|
_install_nft
|
||||||
echo "nftables table inet $TABLE installed (fail-closed boundary)"
|
echo "nftables table inet $TABLE installed (fail-closed boundary)"
|
||||||
|
_install_orch_egress
|
||||||
|
echo "orchestrator egress installed ($ORCH_IFACE -> NAT out)"
|
||||||
echo "done."
|
echo "done."
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -161,8 +180,63 @@ table inet $TABLE {
|
|||||||
EOF
|
EOF
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Give the orchestrator/gateway VM real internet egress (agent VMs get
|
||||||
|
# none — that's the isolation table above). Three parts, because the
|
||||||
|
# path must work both during bootstrap (Docker still present) and after
|
||||||
|
# Docker is removed:
|
||||||
|
# * masquerade — SNAT the orch guest /31 out the host uplink so its
|
||||||
|
# RFC-1918 address can reach the internet.
|
||||||
|
# * nft forward — accept the orch link's forward path (load-bearing
|
||||||
|
# on a pure-nft host whose FORWARD policy drops; a
|
||||||
|
# harmless no-op where forwarding is already open).
|
||||||
|
# It never drops, so it can't weaken the isolation
|
||||||
|
# table's agent drops.
|
||||||
|
# * DOCKER-USER — during bootstrap Docker's FORWARD chain policy is
|
||||||
|
# DROP; its sanctioned DOCKER-USER hook is the only
|
||||||
|
# place a user ACCEPT survives. Best-effort + guarded
|
||||||
|
# (skipped once Docker is gone).
|
||||||
|
_install_orch_egress() {
|
||||||
|
nft -f - <<EOF
|
||||||
|
table inet ${TABLE}_nat {
|
||||||
|
chain forward {
|
||||||
|
type filter hook forward priority -10; policy accept;
|
||||||
|
iifname "$ORCH_IFACE" accept
|
||||||
|
oifname "$ORCH_IFACE" ct state established,related accept
|
||||||
|
}
|
||||||
|
chain postrouting {
|
||||||
|
type nat hook postrouting priority 100; policy accept;
|
||||||
|
ip saddr $(orch_guest) oifname != "$ORCH_IFACE" masquerade
|
||||||
|
}
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
_docker_user_orch add
|
||||||
|
}
|
||||||
|
|
||||||
|
# Insert (add) or delete (del) the DOCKER-USER ACCEPT rules for the
|
||||||
|
# orchestrator link, idempotently, only when the chain exists.
|
||||||
|
_docker_user_orch() {
|
||||||
|
local op="$1" flag
|
||||||
|
command -v iptables >/dev/null 2>&1 || return 0
|
||||||
|
iptables -t filter -L DOCKER-USER >/dev/null 2>&1 || return 0
|
||||||
|
for flag in "-i" "-o"; do
|
||||||
|
if [ "$op" = add ]; then
|
||||||
|
iptables -C DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT 2>/dev/null \
|
||||||
|
|| iptables -I DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT
|
||||||
|
else
|
||||||
|
iptables -D DOCKER-USER "$flag" "$ORCH_IFACE" -j ACCEPT 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
cmd_down() {
|
cmd_down() {
|
||||||
require_root down
|
require_root down
|
||||||
|
_docker_user_orch del
|
||||||
|
nft delete table inet "${TABLE}_nat" 2>/dev/null || true
|
||||||
|
if ip link show "$ORCH_IFACE" >/dev/null 2>&1; then
|
||||||
|
ip link set "$ORCH_IFACE" down 2>/dev/null || true
|
||||||
|
ip tuntap del dev "$ORCH_IFACE" mode tap 2>/dev/null || true
|
||||||
|
echo " removed $ORCH_IFACE"
|
||||||
|
fi
|
||||||
nft delete table inet "$TABLE" 2>/dev/null || true
|
nft delete table inet "$TABLE" 2>/dev/null || true
|
||||||
for i in $(seq 0 $((POOL_SIZE-1))); do
|
for i in $(seq 0 $((POOL_SIZE-1))); do
|
||||||
local dev ; dev="$(iface "$i")"
|
local dev ; dev="$(iface "$i")"
|
||||||
@@ -178,6 +252,8 @@ cmd_down() {
|
|||||||
cmd_status() {
|
cmd_status() {
|
||||||
echo "table inet $TABLE:"
|
echo "table inet $TABLE:"
|
||||||
nft list table inet "$TABLE" 2>/dev/null || echo " (absent)"
|
nft list table inet "$TABLE" 2>/dev/null || echo " (absent)"
|
||||||
|
echo "table inet ${TABLE}_nat (orchestrator egress):"
|
||||||
|
nft list table inet "${TABLE}_nat" 2>/dev/null || echo " (absent)"
|
||||||
echo "taps:"
|
echo "taps:"
|
||||||
for i in $(seq 0 $((POOL_SIZE-1))); do
|
for i in $(seq 0 $((POOL_SIZE-1))); do
|
||||||
local dev ; dev="$(iface "$i")"
|
local dev ; dev="$(iface "$i")"
|
||||||
@@ -185,6 +261,9 @@ cmd_status() {
|
|||||||
ip -brief addr show "$dev" | sed 's/^/ /'
|
ip -brief addr show "$dev" | sed 's/^/ /'
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
if ip -brief addr show "$ORCH_IFACE" >/dev/null 2>&1; then
|
||||||
|
ip -brief addr show "$ORCH_IFACE" | sed 's/^/ /'
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
|
|||||||
@@ -91,6 +91,7 @@ class TestLaunchCommittedImage(unittest.TestCase):
|
|||||||
with mock.patch.object(launch_mod, "read_committed_image", return_value=committed_tag), \
|
with mock.patch.object(launch_mod, "read_committed_image", return_value=committed_tag), \
|
||||||
mock.patch.object(launch_mod.docker_mod, "image_exists", return_value=image_present), \
|
mock.patch.object(launch_mod.docker_mod, "image_exists", return_value=image_present), \
|
||||||
mock.patch.object(launch_mod.docker_mod, "build_image", side_effect=_build), \
|
mock.patch.object(launch_mod.docker_mod, "build_image", side_effect=_build), \
|
||||||
|
mock.patch.object(launch_mod.docker_mod, "verify_agent_image"), \
|
||||||
mock.patch.object(launch_mod, "launch_consolidated", return_value=_CTX), \
|
mock.patch.object(launch_mod, "launch_consolidated", return_value=_CTX), \
|
||||||
mock.patch.object(launch_mod, "teardown_consolidated"), \
|
mock.patch.object(launch_mod, "teardown_consolidated"), \
|
||||||
mock.patch.object(launch_mod, "DockerGateway", return_value=gw), \
|
mock.patch.object(launch_mod, "DockerGateway", return_value=gw), \
|
||||||
|
|||||||
@@ -94,6 +94,7 @@ class TestTeardownWarning(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
with mock.patch.object(launch_mod.docker_mod, "build_image"), \
|
with mock.patch.object(launch_mod.docker_mod, "build_image"), \
|
||||||
|
mock.patch.object(launch_mod.docker_mod, "verify_agent_image"), \
|
||||||
mock.patch.object(launch_mod, "launch_consolidated", return_value=ctx), \
|
mock.patch.object(launch_mod, "launch_consolidated", return_value=ctx), \
|
||||||
mock.patch.object(launch_mod, "teardown_consolidated"), \
|
mock.patch.object(launch_mod, "teardown_consolidated"), \
|
||||||
mock.patch.object(launch_mod, "DockerGateway", return_value=gw), \
|
mock.patch.object(launch_mod, "DockerGateway", return_value=gw), \
|
||||||
|
|||||||
@@ -64,6 +64,21 @@ class TestNetpoolProbes(unittest.TestCase):
|
|||||||
patch.object(netpool, "tap_present", side_effect=[True, False]):
|
patch.object(netpool, "tap_present", side_effect=[True, False]):
|
||||||
self.assertEqual(["bbfc1"], netpool.missing_taps())
|
self.assertEqual(["bbfc1"], netpool.missing_taps())
|
||||||
|
|
||||||
|
def test_orch_slot_is_top_of_ip_base_16(self):
|
||||||
|
# Dedicated orchestrator link: /31 at the top of the IP_BASE /16,
|
||||||
|
# clear of the pool (bottom of the block). Must match the shell
|
||||||
|
# script's orch_host()/orch_guest() and be a non-pool index.
|
||||||
|
with patch.dict("os.environ", {"BOT_BOTTLE_FC_IP_BASE": "10.243.0.0"}):
|
||||||
|
s = netpool.orch_slot()
|
||||||
|
self.assertEqual(netpool.ORCH_IFACE, s.iface)
|
||||||
|
self.assertEqual("10.243.255.0", s.host_ip)
|
||||||
|
self.assertEqual("10.243.255.1", s.guest_ip)
|
||||||
|
self.assertEqual(-1, s.index)
|
||||||
|
# Never collides with a pool slot's guest address.
|
||||||
|
with patch.dict("os.environ", {"BOT_BOTTLE_FC_IP_BASE": "10.243.0.0"}):
|
||||||
|
pool_guests = {sl.guest_ip for sl in netpool.all_slots()}
|
||||||
|
self.assertNotIn(s.guest_ip, pool_guests)
|
||||||
|
|
||||||
|
|
||||||
class TestConsoleTail(unittest.TestCase):
|
class TestConsoleTail(unittest.TestCase):
|
||||||
def test_reads_tail(self):
|
def test_reads_tail(self):
|
||||||
|
|||||||
Reference in New Issue
Block a user