feat(firecracker): implement consolidated orchestrator launch (PRD 0070)
Replace the per-bottle Docker sidecar bundle with the shared per-host orchestrator + gateway, mirroring what the Docker backend already has. - Add `bot_bottle/backend/firecracker/consolidated_launch.py`: `_FirecrackerOrchestratorService` (subclasses `OrchestratorService`, overrides `_gateway()` to return a `DockerGateway` with host port bindings so Firecracker VMs can reach it via their TAP link); `launch_consolidated()` registers the bottle by guest IP (attribution key), provisions git-gate into the shared gateway, and returns the shared CA + orchestrator URL for teardown; `teardown_consolidated()` deregisters and cleans up. - Rewrite `bot_bottle/backend/firecracker/launch.py`: removes the per-bottle sidecar bundle (`_start_sidecar_bundle`, `_stage_git_gate`, etc.) and `_mint_certs`; wires `launch_consolidated()` instead. The VM still sends to `host_tap_ip:PORT` — Docker's PREROUTING DNAT + the nft `ct status dnat accept` rule in the forward chain route the traffic to the shared gateway container. - Extend `DockerGateway` with `host_port_bindings` so the Firecracker gateway publishes its ports on the host (`0.0.0.0:PORT`). - Parameterise `OrchestratorService` with `orchestrator_name` / `orchestrator_label` so Docker and Firecracker orchestrators can coexist on the same host (`bot-bottle-orchestrator` vs `bot-bottle-fc-orchestrator`). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,162 @@
|
|||||||
|
"""Consolidated bottle launch sequence for the Firecracker backend (PRD 0070).
|
||||||
|
|
||||||
|
Mirrors bot_bottle.backend.docker.consolidated_launch but wired for
|
||||||
|
Firecracker's TAP-based network topology instead of a shared Docker bridge.
|
||||||
|
|
||||||
|
The per-bottle sidecar bundle (one `docker run` per bottle, published on the
|
||||||
|
slot's host-side TAP IP) is replaced by a single persistent gateway that
|
||||||
|
every Firecracker VM shares. The gateway runs as a Docker container in the
|
||||||
|
dev-harness (a Firecracker VM is stage B per PRD 0070), with its ports
|
||||||
|
published on the host (`0.0.0.0:PORT`). VMs reach it at their slot's
|
||||||
|
host-side TAP IP because Docker's iptables PREROUTING DNAT redirects
|
||||||
|
port 9099/9100/9420 traffic to the gateway container — a path the nft
|
||||||
|
isolation table already allows via `ct status dnat accept` in the forward
|
||||||
|
chain.
|
||||||
|
|
||||||
|
Attribution is by the VM's guest IP, which is unspoofable by construction:
|
||||||
|
the /31 point-to-point TAP topology + the `bot_bottle_fc` nft table ensure
|
||||||
|
that only the expected VM can source-IP that address.
|
||||||
|
|
||||||
|
Sequence:
|
||||||
|
1. ensure the Firecracker-flavoured orchestrator + gateway are up;
|
||||||
|
2. register the bottle by its guest IP (attribution key) → bottle id +
|
||||||
|
identity token;
|
||||||
|
3. provision its git-gate repos/creds into the running gateway;
|
||||||
|
4. fetch the shared gateway CA for the provisioner to install in the rootfs.
|
||||||
|
|
||||||
|
The TAP slot allocation, rootfs build, and VM boot are the caller's job.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from ...egress import EgressPlan
|
||||||
|
from ...git_gate import GitGatePlan
|
||||||
|
from ...orchestrator.client import OrchestratorClient
|
||||||
|
from ...orchestrator.gateway import (
|
||||||
|
GATEWAY_NETWORK,
|
||||||
|
DockerGateway,
|
||||||
|
)
|
||||||
|
from ...orchestrator.lifecycle import (
|
||||||
|
OrchestratorService,
|
||||||
|
OrchestratorStartError, # re-exported so callers can catch it
|
||||||
|
)
|
||||||
|
from ...orchestrator.registration import registration_inputs
|
||||||
|
from ..docker.egress import EGRESS_PORT
|
||||||
|
from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate
|
||||||
|
from ...supervise import SUPERVISE_PORT
|
||||||
|
|
||||||
|
_GIT_HTTP_PORT = 9420
|
||||||
|
|
||||||
|
# Separate names from the Docker gateway so both backends can coexist on one
|
||||||
|
# host (and for clarity in `docker ps` output).
|
||||||
|
_FC_GATEWAY_NAME = "bot-bottle-fc-gateway"
|
||||||
|
_FC_ORCHESTRATOR_NAME = "bot-bottle-fc-orchestrator"
|
||||||
|
_FC_ORCHESTRATOR_LABEL = "bot-bottle-fc-orchestrator=1"
|
||||||
|
# Ports the gateway publishes on the host so Firecracker VMs can reach it
|
||||||
|
# via their TAP link. Docker's PREROUTING DNAT + nft's `ct status dnat
|
||||||
|
# accept` in the forward chain route the traffic.
|
||||||
|
_FC_GATEWAY_HOST_PORTS = (EGRESS_PORT, SUPERVISE_PORT, _GIT_HTTP_PORT)
|
||||||
|
|
||||||
|
|
||||||
|
class ConsolidatedLaunchError(RuntimeError):
|
||||||
|
"""The consolidated register/provision sequence could not complete."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class LaunchContext:
|
||||||
|
"""What the Firecracker launch needs from the consolidated sequence."""
|
||||||
|
|
||||||
|
bottle_id: str
|
||||||
|
identity_token: str
|
||||||
|
source_ip: str # the VM's guest IP — the attribution key
|
||||||
|
gateway_ca_pem: str # the shared gateway CA the provisioner installs
|
||||||
|
orchestrator_url: str
|
||||||
|
|
||||||
|
|
||||||
|
class _FirecrackerOrchestratorService(OrchestratorService):
|
||||||
|
"""Dev-harness orchestrator for the Firecracker backend.
|
||||||
|
|
||||||
|
Uses a gateway that publishes its ports on the host so Firecracker VMs can
|
||||||
|
reach it via their TAP link. The gateway and orchestrator containers use
|
||||||
|
`*-fc-*` names so both backends can run independently on the same host.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, **kwargs: object) -> None:
|
||||||
|
super().__init__(
|
||||||
|
orchestrator_name=_FC_ORCHESTRATOR_NAME,
|
||||||
|
orchestrator_label=_FC_ORCHESTRATOR_LABEL,
|
||||||
|
**kwargs, # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
|
||||||
|
def _gateway(self) -> DockerGateway:
|
||||||
|
return DockerGateway(
|
||||||
|
self.image,
|
||||||
|
name=_FC_GATEWAY_NAME,
|
||||||
|
network=self.network,
|
||||||
|
orchestrator_url=self.internal_url,
|
||||||
|
host_port_bindings=_FC_GATEWAY_HOST_PORTS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def launch_consolidated(
|
||||||
|
egress_plan: EgressPlan,
|
||||||
|
git_gate_plan: GitGatePlan,
|
||||||
|
*,
|
||||||
|
guest_ip: str,
|
||||||
|
image_ref: str = "",
|
||||||
|
tokens: dict[str, str] | None = None,
|
||||||
|
service: OrchestratorService | None = None,
|
||||||
|
gateway_name: str = _FC_GATEWAY_NAME,
|
||||||
|
) -> LaunchContext:
|
||||||
|
"""Ensure the orchestrator + Firecracker gateway are up, register the
|
||||||
|
bottle by its guest IP, and provision its git-gate state. Returns the
|
||||||
|
context the VM launch needs. Raises `ConsolidatedLaunchError` (or the
|
||||||
|
primitives' own errors) on failure — the caller tears down on failure."""
|
||||||
|
service = service or _FirecrackerOrchestratorService()
|
||||||
|
url = service.ensure_running()
|
||||||
|
client = OrchestratorClient(url)
|
||||||
|
|
||||||
|
inputs = registration_inputs(egress_plan)
|
||||||
|
reg = client.register_bottle(
|
||||||
|
guest_ip, image_ref=image_ref, policy=inputs.policy,
|
||||||
|
metadata=inputs.metadata, tokens=tokens,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
provision_git_gate(gateway_name, reg.bottle_id, git_gate_plan)
|
||||||
|
except Exception:
|
||||||
|
client.teardown_bottle(reg.bottle_id)
|
||||||
|
raise
|
||||||
|
|
||||||
|
# Fetch the shared gateway CA here so the caller can install it in the
|
||||||
|
# rootfs (the same CA every agent on this host trusts for TLS interception).
|
||||||
|
gateway_ca_pem = DockerGateway(
|
||||||
|
name=gateway_name, network=GATEWAY_NETWORK,
|
||||||
|
).ca_cert_pem()
|
||||||
|
|
||||||
|
return LaunchContext(
|
||||||
|
bottle_id=reg.bottle_id,
|
||||||
|
identity_token=reg.identity_token,
|
||||||
|
source_ip=guest_ip,
|
||||||
|
gateway_ca_pem=gateway_ca_pem,
|
||||||
|
orchestrator_url=url,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def teardown_consolidated(
|
||||||
|
bottle_id: str, *, orchestrator_url: str, gateway_name: str = _FC_GATEWAY_NAME,
|
||||||
|
) -> None:
|
||||||
|
"""Deregister the bottle and remove its git-gate state from the gateway.
|
||||||
|
Both steps are idempotent so this is safe from a cleanup trap."""
|
||||||
|
OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id)
|
||||||
|
deprovision_git_gate(gateway_name, bottle_id)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"LaunchContext",
|
||||||
|
"launch_consolidated",
|
||||||
|
"teardown_consolidated",
|
||||||
|
"ConsolidatedLaunchError",
|
||||||
|
"OrchestratorStartError",
|
||||||
|
]
|
||||||
@@ -1,25 +1,30 @@
|
|||||||
"""Launch flow for the Firecracker backend.
|
"""Launch flow for the Firecracker backend (PRD 0070, consolidated).
|
||||||
|
|
||||||
Per bottle:
|
Per bottle:
|
||||||
1. mint the egress CA, build the agent image (docker), export it to a
|
1. build the agent image (docker), export it to a cached ext4 rootfs;
|
||||||
cached ext4 rootfs;
|
2. ensure the per-host orchestrator + shared gateway are up;
|
||||||
2. claim a free TAP pool slot (rootless flock);
|
3. claim a free TAP pool slot (rootless flock);
|
||||||
3. bring up the Docker sidecar bundle, publishing egress / git-gate /
|
4. register the bottle on the orchestrator by the VM's guest IP (the
|
||||||
supervise on the slot's host-side TAP IP at fixed ports;
|
attribution key) and provision its git-gate state into the gateway;
|
||||||
4. boot the microVM on that TAP; wait for SSH;
|
5. boot the microVM on that TAP; wait for SSH;
|
||||||
5. provision (CA, prompt, skills, workspace, git, supervise) over SSH.
|
6. provision (shared gateway CA, prompt, skills, workspace, git, supervise)
|
||||||
|
over SSH.
|
||||||
|
|
||||||
|
The per-bottle Docker sidecar bundle is gone. The shared gateway handles
|
||||||
|
egress / git-gate / supervise for every VM; Docker's PREROUTING DNAT routes
|
||||||
|
the VMs' traffic to it, and the nft table's `ct status dnat accept` rule
|
||||||
|
in the forward chain lets it pass. The VM still sends to `host_tap_ip:PORT`
|
||||||
|
— the address its world is, by nft design, limited to.
|
||||||
|
|
||||||
Isolation is enforced by the operator-provisioned nft table (checked
|
Isolation is enforced by the operator-provisioned nft table (checked
|
||||||
fail-closed in preflight): a VM reaches only its sidecar (DNAT'd from
|
fail-closed in preflight): a VM reaches only the sidecar (DNAT'd from
|
||||||
the host TAP IP) and nothing else. The agent's HTTPS_PROXY therefore
|
the host TAP IP) and nothing else.
|
||||||
points at `http://<host_tap_ip>:9099`, its only route to the world.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import dataclasses
|
import dataclasses
|
||||||
import os
|
import os
|
||||||
import subprocess
|
|
||||||
from contextlib import ExitStack, contextmanager
|
from contextlib import ExitStack, contextmanager
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Callable, Generator
|
from typing import Callable, Generator
|
||||||
@@ -31,47 +36,29 @@ from ...bottle_state import (
|
|||||||
read_committed_image,
|
read_committed_image,
|
||||||
)
|
)
|
||||||
from ...egress import (
|
from ...egress import (
|
||||||
EGRESS_ROUTES_IN_CONTAINER,
|
|
||||||
egress_agent_env_entries,
|
egress_agent_env_entries,
|
||||||
egress_resolve_token_values,
|
egress_resolve_token_values,
|
||||||
egress_sidecar_env_entries,
|
|
||||||
)
|
)
|
||||||
from ...git_gate import (
|
from ...git_gate import (
|
||||||
provision_git_gate_dynamic_keys,
|
provision_git_gate_dynamic_keys,
|
||||||
revoke_git_gate_provisioned_keys,
|
revoke_git_gate_provisioned_keys,
|
||||||
)
|
)
|
||||||
from ...log import die, info, warn
|
from ...log import info, warn
|
||||||
from ...supervise import DB_PATH_IN_CONTAINER, SUPERVISE_PORT
|
from ...supervise import SUPERVISE_PORT
|
||||||
from ...util import expand_tilde
|
|
||||||
from ..docker import util as docker_mod
|
from ..docker import util as docker_mod
|
||||||
from ..docker.egress import (
|
from ..docker.egress import EGRESS_PORT
|
||||||
EGRESS_CA_IN_CONTAINER,
|
|
||||||
EGRESS_PORT,
|
|
||||||
egress_tls_init,
|
|
||||||
)
|
|
||||||
from ..docker.git_gate import (
|
|
||||||
GIT_GATE_ACCESS_HOOK_IN_CONTAINER,
|
|
||||||
GIT_GATE_CREDS_DIR_IN_CONTAINER,
|
|
||||||
GIT_GATE_ENTRYPOINT_IN_CONTAINER,
|
|
||||||
GIT_GATE_HOOK_IN_CONTAINER,
|
|
||||||
)
|
|
||||||
from ..docker.sidecar_bundle import (
|
|
||||||
SIDECAR_BUNDLE_DOCKERFILE,
|
|
||||||
SIDECAR_BUNDLE_IMAGE,
|
|
||||||
)
|
|
||||||
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
|
||||||
from . import firecracker_vm, isolation_probe, netpool, util
|
from . import firecracker_vm, isolation_probe, netpool, util
|
||||||
from .bottle import FirecrackerBottle
|
from .bottle import FirecrackerBottle
|
||||||
from .bottle_plan import FirecrackerBottlePlan
|
from .bottle_plan import FirecrackerBottlePlan
|
||||||
|
from .consolidated_launch import (
|
||||||
|
launch_consolidated,
|
||||||
|
teardown_consolidated,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
|
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
|
||||||
_GIT_HTTP_PORT = 9420
|
_GIT_HTTP_PORT = 9420
|
||||||
_GIT_GATE_READY_FILE = "/run/git-gate/ready"
|
|
||||||
|
|
||||||
|
|
||||||
def sidecar_container_name(slug: str) -> str:
|
|
||||||
return f"bot-bottle-sidecars-{slug}"
|
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
@@ -80,6 +67,8 @@ def launch(
|
|||||||
*,
|
*,
|
||||||
provision: Callable[[FirecrackerBottlePlan, "FirecrackerBottle"], str | None],
|
provision: Callable[[FirecrackerBottlePlan, "FirecrackerBottle"], str | None],
|
||||||
) -> Generator[FirecrackerBottle, None, None]:
|
) -> Generator[FirecrackerBottle, None, None]:
|
||||||
|
"""Build, launch, and provision a Firecracker bottle via the consolidated
|
||||||
|
orchestrator. Teardown on exit."""
|
||||||
stack = ExitStack()
|
stack = ExitStack()
|
||||||
bottle_for_revoke = plan.manifest.bottle
|
bottle_for_revoke = plan.manifest.bottle
|
||||||
git_gate_dir_for_revoke = git_gate_state_dir(plan.slug)
|
git_gate_dir_for_revoke = git_gate_state_dir(plan.slug)
|
||||||
@@ -96,26 +85,75 @@ def launch(
|
|||||||
raise teardown_exc
|
raise teardown_exc
|
||||||
|
|
||||||
try:
|
try:
|
||||||
plan = _mint_certs(plan)
|
# Step 1: agent image. The sidecar bundle image is built by the
|
||||||
|
# orchestrator service (ensure_running → ensure_built); we only
|
||||||
|
# build the agent image here. Use a committed snapshot when available.
|
||||||
plan = _build_agent_image(plan)
|
plan = _build_agent_image(plan)
|
||||||
|
|
||||||
# Claim a TAP slot; the flock is held until teardown closes it.
|
# Step 2: mint the git-gate dynamic (gitea) deploy keys, if any.
|
||||||
|
git_gate_plan = plan.git_gate_plan
|
||||||
|
if git_gate_plan.upstreams:
|
||||||
|
git_gate_plan = provision_git_gate_dynamic_keys(
|
||||||
|
plan.manifest.bottle, git_gate_plan, git_gate_state_dir(plan.slug),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 3: claim a TAP slot; the flock is held until teardown.
|
||||||
slot, lock = netpool.allocate(plan.slug)
|
slot, lock = netpool.allocate(plan.slug)
|
||||||
stack.callback(lock.close)
|
stack.callback(lock.close)
|
||||||
info(f"firecracker slot {slot.iface}: host={slot.host_ip} "
|
info(f"firecracker slot {slot.iface}: host={slot.host_ip} "
|
||||||
f"guest={slot.guest_ip}")
|
f"guest={slot.guest_ip}")
|
||||||
|
|
||||||
plan = _provision_git_gate_keys(plan)
|
# Step 4: register on the orchestrator + provision this bottle's
|
||||||
|
# git-gate state into the shared gateway. The per-bottle egress tokens
|
||||||
|
# are resolved from the host env now and handed to the orchestrator
|
||||||
|
# (in memory) for the gateway to inject — the agent never sees them.
|
||||||
|
# Attribution is by the VM's guest IP (unspoofable via /31 TAP + nft).
|
||||||
|
effective_env = {**os.environ, **plan.agent_provision.provisioned_env}
|
||||||
|
token_values = egress_resolve_token_values(
|
||||||
|
plan.egress_plan.token_env_map, effective_env,
|
||||||
|
)
|
||||||
|
ctx = launch_consolidated(
|
||||||
|
plan.egress_plan, git_gate_plan,
|
||||||
|
guest_ip=slot.guest_ip,
|
||||||
|
image_ref=plan.image,
|
||||||
|
tokens=token_values,
|
||||||
|
)
|
||||||
|
stack.callback(
|
||||||
|
teardown_consolidated, ctx.bottle_id,
|
||||||
|
orchestrator_url=ctx.orchestrator_url,
|
||||||
|
)
|
||||||
|
|
||||||
sidecar_name = sidecar_container_name(plan.slug)
|
# Step 5: install the SHARED gateway CA (replaces the per-bottle CA).
|
||||||
docker_mod.force_remove_container(sidecar_name)
|
# Write it to a stable host path so the provisioner can copy it over SSH.
|
||||||
_start_sidecar_bundle(plan, sidecar_name, slot.host_ip)
|
ca_dir = egress_state_dir(plan.slug) / "gateway-ca"
|
||||||
stack.callback(docker_mod.force_remove_container, sidecar_name)
|
ca_dir.mkdir(parents=True, exist_ok=True)
|
||||||
_stage_git_gate(plan, sidecar_name)
|
ca_file = ca_dir / "gateway-ca.pem"
|
||||||
|
ca_file.write_text(ctx.gateway_ca_pem)
|
||||||
|
egress_plan = dataclasses.replace(
|
||||||
|
plan.egress_plan,
|
||||||
|
mitmproxy_ca_host_path=ca_file,
|
||||||
|
mitmproxy_ca_cert_only_host_path=ca_file,
|
||||||
|
)
|
||||||
|
# Point the agent's git-gate insteadOf rewrites and supervise MCP URL
|
||||||
|
# at the shared gateway (reached at the slot's host TAP IP — the VM
|
||||||
|
# sends there and Docker DNAT routes to the gateway container).
|
||||||
|
git_gate_url = (
|
||||||
|
f"http://{slot.host_ip}:{_GIT_HTTP_PORT}" if git_gate_plan.upstreams else ""
|
||||||
|
)
|
||||||
|
supervise_url = (
|
||||||
|
f"http://{slot.host_ip}:{SUPERVISE_PORT}/"
|
||||||
|
if plan.supervise_plan is not None else ""
|
||||||
|
)
|
||||||
|
plan = dataclasses.replace(
|
||||||
|
plan,
|
||||||
|
git_gate_plan=git_gate_plan,
|
||||||
|
egress_plan=egress_plan,
|
||||||
|
agent_proxy_url=f"http://{slot.host_ip}:{EGRESS_PORT}",
|
||||||
|
agent_git_gate_url=git_gate_url,
|
||||||
|
agent_supervise_url=supervise_url,
|
||||||
|
)
|
||||||
|
|
||||||
plan = _stamp_agent_urls(plan, slot.host_ip)
|
# Step 6: build the per-bottle rootfs + SSH key, then boot.
|
||||||
|
|
||||||
# Build the per-bottle rootfs + SSH key, then boot.
|
|
||||||
base_dir = util.build_base_rootfs_dir(plan.image)
|
base_dir = util.build_base_rootfs_dir(plan.image)
|
||||||
run_dir = util.cache_dir() / "run" / plan.slug
|
run_dir = util.cache_dir() / "run" / plan.slug
|
||||||
run_dir.mkdir(parents=True, exist_ok=True)
|
run_dir.mkdir(parents=True, exist_ok=True)
|
||||||
@@ -135,8 +173,8 @@ def launch(
|
|||||||
stack.callback(vm.terminate)
|
stack.callback(vm.terminate)
|
||||||
firecracker_vm.wait_for_ssh(vm, private_key)
|
firecracker_vm.wait_for_ssh(vm, private_key)
|
||||||
|
|
||||||
# Authoritative fail-closed egress-boundary check, before the
|
# Authoritative fail-closed egress-boundary check, before the agent
|
||||||
# agent runs: prove the VM cannot reach the host directly.
|
# runs: prove the VM cannot reach the host directly.
|
||||||
isolation_probe.verify_isolation(private_key, slot.guest_ip)
|
isolation_probe.verify_isolation(private_key, slot.guest_ip)
|
||||||
|
|
||||||
bottle = FirecrackerBottle(
|
bottle = FirecrackerBottle(
|
||||||
@@ -161,18 +199,7 @@ def launch(
|
|||||||
teardown()
|
teardown()
|
||||||
|
|
||||||
|
|
||||||
def _mint_certs(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
|
|
||||||
egress_ca_host, egress_ca_cert_only = egress_tls_init(egress_state_dir(plan.slug))
|
|
||||||
egress_plan = dataclasses.replace(
|
|
||||||
plan.egress_plan,
|
|
||||||
mitmproxy_ca_host_path=egress_ca_host,
|
|
||||||
mitmproxy_ca_cert_only_host_path=egress_ca_cert_only,
|
|
||||||
)
|
|
||||||
return dataclasses.replace(plan, egress_plan=egress_plan)
|
|
||||||
|
|
||||||
|
|
||||||
def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
|
def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
|
||||||
docker_mod.build_image(SIDECAR_BUNDLE_IMAGE, _REPO_DIR, dockerfile=SIDECAR_BUNDLE_DOCKERFILE)
|
|
||||||
committed = read_committed_image(plan.slug)
|
committed = read_committed_image(plan.slug)
|
||||||
if committed and docker_mod.image_exists(committed):
|
if committed and docker_mod.image_exists(committed):
|
||||||
info(f"using committed image {committed!r}")
|
info(f"using committed image {committed!r}")
|
||||||
@@ -187,148 +214,6 @@ def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
|
|||||||
return plan
|
return plan
|
||||||
|
|
||||||
|
|
||||||
def _provision_git_gate_keys(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
|
|
||||||
if not plan.git_gate_plan.upstreams:
|
|
||||||
return plan
|
|
||||||
git_gate_plan = provision_git_gate_dynamic_keys(
|
|
||||||
plan.manifest.bottle, plan.git_gate_plan, git_gate_state_dir(plan.slug),
|
|
||||||
)
|
|
||||||
return dataclasses.replace(plan, git_gate_plan=git_gate_plan)
|
|
||||||
|
|
||||||
|
|
||||||
def _stamp_agent_urls(
|
|
||||||
plan: FirecrackerBottlePlan, host_ip: str,
|
|
||||||
) -> FirecrackerBottlePlan:
|
|
||||||
proxy_url = f"http://{host_ip}:{EGRESS_PORT}"
|
|
||||||
supervise_url = (
|
|
||||||
f"http://{host_ip}:{SUPERVISE_PORT}/" if plan.supervise_plan is not None else ""
|
|
||||||
)
|
|
||||||
git_gate_url = (
|
|
||||||
f"http://{host_ip}:{_GIT_HTTP_PORT}" if plan.git_gate_plan.upstreams else ""
|
|
||||||
)
|
|
||||||
return dataclasses.replace(
|
|
||||||
plan,
|
|
||||||
agent_proxy_url=proxy_url,
|
|
||||||
agent_git_gate_url=git_gate_url,
|
|
||||||
agent_supervise_url=supervise_url,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# --- sidecar bundle (Docker) ----------------------------------------
|
|
||||||
|
|
||||||
def _start_sidecar_bundle(
|
|
||||||
plan: FirecrackerBottlePlan, sidecar_name: str, host_ip: str,
|
|
||||||
) -> None:
|
|
||||||
argv = ["docker", "run", "--name", sidecar_name, "--detach", "--rm",
|
|
||||||
"-e", f"BOT_BOTTLE_SIDECAR_DAEMONS={','.join(_sidecar_daemons(plan))}"]
|
|
||||||
for entry in _sidecar_env_entries(plan):
|
|
||||||
argv += ["-e", entry]
|
|
||||||
for host_path, container_path, read_only in _sidecar_mounts(plan):
|
|
||||||
argv += ["-v", f"{host_path}:{container_path}{':ro' if read_only else ''}"]
|
|
||||||
# Publish on the slot's host TAP IP at fixed ports — each bottle
|
|
||||||
# has a distinct host_ip, so fixed ports never collide, and the VM
|
|
||||||
# reaches them at a stable, well-known address (its only route out).
|
|
||||||
for port in _sidecar_ports(plan):
|
|
||||||
argv += ["-p", f"{host_ip}:{port}:{port}"]
|
|
||||||
argv.append(SIDECAR_BUNDLE_IMAGE)
|
|
||||||
|
|
||||||
effective_env = {**dict(os.environ), **plan.agent_provision.provisioned_env}
|
|
||||||
token_values = egress_resolve_token_values(
|
|
||||||
plan.egress_plan.token_env_map, effective_env,
|
|
||||||
)
|
|
||||||
env = {**os.environ, **token_values}
|
|
||||||
info(f"docker run sidecar bundle {sidecar_name} (published on {host_ip})")
|
|
||||||
result = subprocess.run(argv, capture_output=True, text=True, env=env, check=False)
|
|
||||||
if result.returncode != 0:
|
|
||||||
die(f"docker run for sidecar bundle {sidecar_name} failed: "
|
|
||||||
f"{(result.stderr or '').strip() or '<no stderr>'}")
|
|
||||||
|
|
||||||
|
|
||||||
def _sidecar_daemons(plan: FirecrackerBottlePlan) -> tuple[str, ...]:
|
|
||||||
daemons = ["egress"]
|
|
||||||
if plan.git_gate_plan.upstreams:
|
|
||||||
daemons += ["git-gate", "git-http"]
|
|
||||||
if plan.supervise_plan is not None:
|
|
||||||
daemons.append("supervise")
|
|
||||||
return tuple(daemons)
|
|
||||||
|
|
||||||
|
|
||||||
def _sidecar_ports(plan: FirecrackerBottlePlan) -> tuple[int, ...]:
|
|
||||||
ports = [EGRESS_PORT]
|
|
||||||
if plan.git_gate_plan.upstreams:
|
|
||||||
ports.append(_GIT_HTTP_PORT)
|
|
||||||
if plan.supervise_plan is not None:
|
|
||||||
ports.append(SUPERVISE_PORT)
|
|
||||||
return tuple(ports)
|
|
||||||
|
|
||||||
|
|
||||||
def _sidecar_env_entries(plan: FirecrackerBottlePlan) -> tuple[str, ...]:
|
|
||||||
env: list[str] = list(egress_sidecar_env_entries(plan.egress_plan))
|
|
||||||
if plan.git_gate_plan.upstreams:
|
|
||||||
env.append(f"BOT_BOTTLE_GIT_GATE_READY_FILE={_GIT_GATE_READY_FILE}")
|
|
||||||
if plan.supervise_plan is not None:
|
|
||||||
env += [
|
|
||||||
f"SUPERVISE_BOTTLE_SLUG={plan.slug}",
|
|
||||||
f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}",
|
|
||||||
f"SUPERVISE_PORT={SUPERVISE_PORT}",
|
|
||||||
]
|
|
||||||
return tuple(env)
|
|
||||||
|
|
||||||
|
|
||||||
def _sidecar_mounts(
|
|
||||||
plan: FirecrackerBottlePlan,
|
|
||||||
) -> tuple[tuple[str, str, bool], ...]:
|
|
||||||
mounts: list[tuple[str, str, bool]] = []
|
|
||||||
ep = plan.egress_plan
|
|
||||||
mounts.append((str(ep.mitmproxy_ca_host_path.parent),
|
|
||||||
str(Path(EGRESS_CA_IN_CONTAINER).parent), False))
|
|
||||||
if ep.routes:
|
|
||||||
mounts.append((str(ep.routes_path.parent),
|
|
||||||
str(Path(EGRESS_ROUTES_IN_CONTAINER).parent), True))
|
|
||||||
sp = plan.supervise_plan
|
|
||||||
if sp is not None:
|
|
||||||
mounts.append((str(sp.db_path.parent),
|
|
||||||
str(Path(DB_PATH_IN_CONTAINER).parent), False))
|
|
||||||
return tuple(mounts)
|
|
||||||
|
|
||||||
|
|
||||||
def _stage_git_gate(plan: FirecrackerBottlePlan, sidecar_name: str) -> None:
|
|
||||||
gp = plan.git_gate_plan
|
|
||||||
if not gp.upstreams:
|
|
||||||
return
|
|
||||||
docker_mod.docker_exec(sidecar_name, [
|
|
||||||
"mkdir", "-p",
|
|
||||||
str(Path(GIT_GATE_HOOK_IN_CONTAINER).parent),
|
|
||||||
GIT_GATE_CREDS_DIR_IN_CONTAINER, "/git",
|
|
||||||
str(Path(_GIT_GATE_READY_FILE).parent),
|
|
||||||
])
|
|
||||||
for host_path, container_path in _git_gate_files(plan):
|
|
||||||
docker_mod.docker_cp(host_path, f"{sidecar_name}:{container_path}")
|
|
||||||
docker_mod.docker_exec(sidecar_name, [
|
|
||||||
"sh", "-c",
|
|
||||||
f"chmod 755 {GIT_GATE_ENTRYPOINT_IN_CONTAINER} "
|
|
||||||
f"{GIT_GATE_HOOK_IN_CONTAINER} {GIT_GATE_ACCESS_HOOK_IN_CONTAINER} && "
|
|
||||||
f"chmod 600 {GIT_GATE_CREDS_DIR_IN_CONTAINER}/* && "
|
|
||||||
f"touch {_GIT_GATE_READY_FILE}",
|
|
||||||
])
|
|
||||||
|
|
||||||
|
|
||||||
def _git_gate_files(plan: FirecrackerBottlePlan) -> tuple[tuple[str, str], ...]:
|
|
||||||
gp = plan.git_gate_plan
|
|
||||||
files: list[tuple[str, str]] = [
|
|
||||||
(str(gp.entrypoint_script), GIT_GATE_ENTRYPOINT_IN_CONTAINER),
|
|
||||||
(str(gp.hook_script), GIT_GATE_HOOK_IN_CONTAINER),
|
|
||||||
(str(gp.access_hook_script), GIT_GATE_ACCESS_HOOK_IN_CONTAINER),
|
|
||||||
]
|
|
||||||
for upstream in gp.upstreams:
|
|
||||||
files.append((expand_tilde(upstream.identity_file),
|
|
||||||
f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{upstream.name}-key"))
|
|
||||||
if upstream.known_hosts_file:
|
|
||||||
files.append((str(upstream.known_hosts_file),
|
|
||||||
f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{upstream.name}-known_hosts"))
|
|
||||||
return tuple(files)
|
|
||||||
|
|
||||||
|
|
||||||
# --- agent guest env -------------------------------------------------
|
# --- agent guest env -------------------------------------------------
|
||||||
|
|
||||||
def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str]:
|
def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str]:
|
||||||
|
|||||||
@@ -100,6 +100,7 @@ class DockerGateway(Gateway):
|
|||||||
orchestrator_url: str = "",
|
orchestrator_url: str = "",
|
||||||
build_context: Path | None = None,
|
build_context: Path | None = None,
|
||||||
dockerfile: str | None = GATEWAY_DOCKERFILE,
|
dockerfile: str | None = GATEWAY_DOCKERFILE,
|
||||||
|
host_port_bindings: tuple[int, ...] = (),
|
||||||
) -> None:
|
) -> None:
|
||||||
self.image_ref = image_ref
|
self.image_ref = image_ref
|
||||||
self.name = name
|
self.name = name
|
||||||
@@ -110,6 +111,10 @@ class DockerGateway(Gateway):
|
|||||||
self._orchestrator_url = orchestrator_url
|
self._orchestrator_url = orchestrator_url
|
||||||
self._build_context = build_context or _REPO_ROOT
|
self._build_context = build_context or _REPO_ROOT
|
||||||
self._dockerfile = dockerfile
|
self._dockerfile = dockerfile
|
||||||
|
# Ports published on the host (0.0.0.0). Used by the Firecracker
|
||||||
|
# backend's dev-harness gateway so VMs can reach it via their TAP link;
|
||||||
|
# Docker's DNAT + the nft `ct status dnat accept` rule handle the rest.
|
||||||
|
self._host_port_bindings = host_port_bindings
|
||||||
|
|
||||||
def image_exists(self) -> bool:
|
def image_exists(self) -> bool:
|
||||||
return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0
|
return run_docker(["docker", "image", "inspect", self.image_ref]).returncode == 0
|
||||||
@@ -188,6 +193,8 @@ class DockerGateway(Gateway):
|
|||||||
# trust it) — see GATEWAY_CA_VOLUME.
|
# trust it) — see GATEWAY_CA_VOLUME.
|
||||||
"--volume", f"{GATEWAY_CA_VOLUME}:{MITMPROXY_HOME}",
|
"--volume", f"{GATEWAY_CA_VOLUME}:{MITMPROXY_HOME}",
|
||||||
]
|
]
|
||||||
|
for port in self._host_port_bindings:
|
||||||
|
argv += ["--publish", f"0.0.0.0:{port}:{port}"]
|
||||||
if self._orchestrator_url:
|
if self._orchestrator_url:
|
||||||
# Makes the gateway's egress / git / supervise daemons multi-tenant:
|
# Makes the gateway's egress / git / supervise daemons multi-tenant:
|
||||||
# each request resolves source-IP -> policy against the control plane.
|
# each request resolves source-IP -> policy against the control plane.
|
||||||
|
|||||||
@@ -48,7 +48,13 @@ class OrchestratorStartError(RuntimeError):
|
|||||||
|
|
||||||
class OrchestratorService:
|
class OrchestratorService:
|
||||||
"""Manages the orchestrator control-plane container + the shared gateway.
|
"""Manages the orchestrator control-plane container + the shared gateway.
|
||||||
Callers only need `ensure_running()` + `url`."""
|
Callers only need `ensure_running()` + `url`.
|
||||||
|
|
||||||
|
`orchestrator_name` / `orchestrator_label` let backends run independent
|
||||||
|
orchestrators on the same host without name collisions (e.g. the
|
||||||
|
Firecracker backend uses `bot-bottle-fc-orchestrator` alongside the Docker
|
||||||
|
backend's `bot-bottle-orchestrator`). Subclass and override `_gateway()`
|
||||||
|
to supply a backend-specific gateway variant."""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -58,12 +64,16 @@ class OrchestratorService:
|
|||||||
image: str = GATEWAY_IMAGE,
|
image: str = GATEWAY_IMAGE,
|
||||||
repo_root: Path = _REPO_ROOT,
|
repo_root: Path = _REPO_ROOT,
|
||||||
host_root: Path | None = None,
|
host_root: Path | None = None,
|
||||||
|
orchestrator_name: str = ORCHESTRATOR_NAME,
|
||||||
|
orchestrator_label: str = ORCHESTRATOR_LABEL,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.port = port
|
self.port = port
|
||||||
self.network = network
|
self.network = network
|
||||||
self.image = image
|
self.image = image
|
||||||
self._repo_root = repo_root
|
self._repo_root = repo_root
|
||||||
self._host_root = host_root or bot_bottle_root()
|
self._host_root = host_root or bot_bottle_root()
|
||||||
|
self._orchestrator_name = orchestrator_name
|
||||||
|
self._orchestrator_label = orchestrator_label
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def url(self) -> str:
|
def url(self) -> str:
|
||||||
@@ -75,7 +85,7 @@ class OrchestratorService:
|
|||||||
"""Control-plane URL as the gateway container reaches it — by name over
|
"""Control-plane URL as the gateway container reaches it — by name over
|
||||||
docker DNS on the shared network. This is the gateway's
|
docker DNS on the shared network. This is the gateway's
|
||||||
BOT_BOTTLE_ORCHESTRATOR_URL."""
|
BOT_BOTTLE_ORCHESTRATOR_URL."""
|
||||||
return f"http://{ORCHESTRATOR_NAME}:{self.port}"
|
return f"http://{self._orchestrator_name}:{self.port}"
|
||||||
|
|
||||||
def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool:
|
def is_healthy(self, *, timeout: float = _HEALTH_REQUEST_TIMEOUT_SECONDS) -> bool:
|
||||||
try:
|
try:
|
||||||
@@ -91,11 +101,11 @@ class OrchestratorService:
|
|||||||
def _run_orchestrator_container(self) -> None:
|
def _run_orchestrator_container(self) -> None:
|
||||||
"""Start the control-plane container (idempotent: clears a stale
|
"""Start the control-plane container (idempotent: clears a stale
|
||||||
fixed-name container first). Register-only broker → no docker socket."""
|
fixed-name container first). Register-only broker → no docker socket."""
|
||||||
run_docker(["docker", "rm", "--force", ORCHESTRATOR_NAME])
|
run_docker(["docker", "rm", "--force", self._orchestrator_name])
|
||||||
proc = run_docker([
|
proc = run_docker([
|
||||||
"docker", "run", "--detach",
|
"docker", "run", "--detach",
|
||||||
"--name", ORCHESTRATOR_NAME,
|
"--name", self._orchestrator_name,
|
||||||
"--label", ORCHESTRATOR_LABEL,
|
"--label", self._orchestrator_label,
|
||||||
"--network", self.network,
|
"--network", self.network,
|
||||||
# Host CLI reaches the control plane here; bound to loopback so it
|
# Host CLI reaches the control plane here; bound to loopback so it
|
||||||
# is not exposed on the host's external interfaces.
|
# is not exposed on the host's external interfaces.
|
||||||
@@ -137,7 +147,7 @@ class OrchestratorService:
|
|||||||
# (~seconds); the registry DB persists and the current launch
|
# (~seconds); the registry DB persists and the current launch
|
||||||
# re-registers its own in-memory state. (The dedicated orchestrator
|
# re-registers its own in-memory state. (The dedicated orchestrator
|
||||||
# image follow-up replaces this with image-staleness detection.)
|
# image follow-up replaces this with image-staleness detection.)
|
||||||
log.info("starting orchestrator container", context={"name": ORCHESTRATOR_NAME})
|
log.info("starting orchestrator container", context={"name": self._orchestrator_name})
|
||||||
self._run_orchestrator_container()
|
self._run_orchestrator_container()
|
||||||
|
|
||||||
deadline = time.monotonic() + startup_timeout
|
deadline = time.monotonic() + startup_timeout
|
||||||
@@ -152,7 +162,7 @@ class OrchestratorService:
|
|||||||
|
|
||||||
def stop(self) -> None:
|
def stop(self) -> None:
|
||||||
"""Remove the orchestrator + gateway containers (idempotent)."""
|
"""Remove the orchestrator + gateway containers (idempotent)."""
|
||||||
run_docker(["docker", "rm", "--force", ORCHESTRATOR_NAME])
|
run_docker(["docker", "rm", "--force", self._orchestrator_name])
|
||||||
self._gateway().stop()
|
self._gateway().stop()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -88,7 +88,6 @@ class TestLaunchCommittedImage(unittest.TestCase):
|
|||||||
del ctx, dockerfile
|
del ctx, dockerfile
|
||||||
built.append(image)
|
built.append(image)
|
||||||
|
|
||||||
<<<<<<< HEAD
|
|
||||||
with mock.patch.object(launch_mod, "read_committed_image", return_value=committed_tag), \
|
with mock.patch.object(launch_mod, "read_committed_image", return_value=committed_tag), \
|
||||||
mock.patch.object(launch_mod.docker_mod, "image_exists", return_value=image_present), \
|
mock.patch.object(launch_mod.docker_mod, "image_exists", return_value=image_present), \
|
||||||
mock.patch.object(launch_mod.docker_mod, "build_image", side_effect=_build), \
|
mock.patch.object(launch_mod.docker_mod, "build_image", side_effect=_build), \
|
||||||
|
|||||||
Reference in New Issue
Block a user