didericis/bot-bottle
1
0
Fork 0
Code Issues 25 Pull Requests 9 Actions Packages Projects Releases Wiki Activity

feat(firecracker): implement consolidated orchestrator launch (PRD 0070)
lint / lint (push) Successful in 2m6s
Details
test / unit (pull_request) Successful in 1m6s
Details
test / integration (pull_request) Successful in 22s
Details
test / coverage (pull_request) Successful in 1m17s
Details

Browse Source 
Replace the per-bottle Docker sidecar bundle with the shared per-host
orchestrator + gateway, mirroring what the Docker backend already has.

- Add `bot_bottle/backend/firecracker/consolidated_launch.py`:
  `_FirecrackerOrchestratorService` (subclasses `OrchestratorService`,
  overrides `_gateway()` to return a `DockerGateway` with host port
  bindings so Firecracker VMs can reach it via their TAP link);
  `launch_consolidated()` registers the bottle by guest IP (attribution
  key), provisions git-gate into the shared gateway, and returns the
  shared CA + orchestrator URL for teardown; `teardown_consolidated()`
  deregisters and cleans up.

- Rewrite `bot_bottle/backend/firecracker/launch.py`: removes the
  per-bottle sidecar bundle (`_start_sidecar_bundle`, `_stage_git_gate`,
  etc.) and `_mint_certs`; wires `launch_consolidated()` instead. The VM
  still sends to `host_tap_ip:PORT` — Docker's PREROUTING DNAT + the nft
  `ct status dnat accept` rule in the forward chain route the traffic to
  the shared gateway container.

- Extend `DockerGateway` with `host_port_bindings` so the Firecracker
  gateway publishes its ports on the host (`0.0.0.0:PORT`).

- Parameterise `OrchestratorService` with `orchestrator_name` /
  `orchestrator_label` so Docker and Firecracker orchestrators can
  coexist on the same host (`bot-bottle-orchestrator` vs
  `bot-bottle-fc-orchestrator`).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
didericis (claude)
2026-07-14 07:57:01 +00:00
parent 4376720e58
commit f47957d847
5 changed files with 274 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bot_bottle/backend/firecracker/launch.py
+88 -203
View File
@@ -1,25 +1,30 @@
"""Launch flow for the Firecracker backend.
"""Launch flow for the Firecracker backend (PRD 0070, consolidated).
Per bottle:
1. mint the egress CA, build the agent image (docker), export it to a
cached ext4 rootfs;
2. claim a free TAP pool slot (rootless flock);
3. bring up the Docker sidecar bundle, publishing egress / git-gate /
supervise on the slot's host-side TAP IP at fixed ports;
4. boot the microVM on that TAP; wait for SSH;
5. provision (CA, prompt, skills, workspace, git, supervise) over SSH.
1. build the agent image (docker), export it to a cached ext4 rootfs;
2. ensure the per-host orchestrator + shared gateway are up;
3. claim a free TAP pool slot (rootless flock);
4. register the bottle on the orchestrator by the VM's guest IP (the
attribution key) and provision its git-gate state into the gateway;
5. boot the microVM on that TAP; wait for SSH;
6. provision (shared gateway CA, prompt, skills, workspace, git, supervise)
over SSH.
The per-bottle Docker sidecar bundle is gone. The shared gateway handles
egress / git-gate / supervise for every VM; Docker's PREROUTING DNAT routes
the VMs' traffic to it, and the nft table's `ct status dnat accept` rule
in the forward chain lets it pass. The VM still sends to `host_tap_ip:PORT`
— the address its world is, by nft design, limited to.
Isolation is enforced by the operator-provisioned nft table (checked
fail-closed in preflight): a VM reaches only its sidecar (DNAT'd from
the host TAP IP) and nothing else. The agent's HTTPS_PROXY therefore
points at `http://<host_tap_ip>:9099`, its only route to the world.
fail-closed in preflight): a VM reaches only the sidecar (DNAT'd from
the host TAP IP) and nothing else.
"""
from __future__ import annotations
import dataclasses
import os
import subprocess
from contextlib import ExitStack, contextmanager
from pathlib import Path
from typing import Callable, Generator
@@ -31,47 +36,29 @@ from ...bottle_state import (
read_committed_image,
)
from ...egress import (
EGRESS_ROUTES_IN_CONTAINER,
egress_agent_env_entries,
egress_resolve_token_values,
egress_sidecar_env_entries,
)
from ...git_gate import (
provision_git_gate_dynamic_keys,
revoke_git_gate_provisioned_keys,
)
from ...log import die, info, warn
from ...supervise import DB_PATH_IN_CONTAINER, SUPERVISE_PORT
from ...util import expand_tilde
from ...log import info, warn
from ...supervise import SUPERVISE_PORT
from ..docker import util as docker_mod
from ..docker.egress import (
EGRESS_CA_IN_CONTAINER,
EGRESS_PORT,
egress_tls_init,
)
from ..docker.git_gate import (
GIT_GATE_ACCESS_HOOK_IN_CONTAINER,
GIT_GATE_CREDS_DIR_IN_CONTAINER,
GIT_GATE_ENTRYPOINT_IN_CONTAINER,
GIT_GATE_HOOK_IN_CONTAINER,
)
from ..docker.sidecar_bundle import (
SIDECAR_BUNDLE_DOCKERFILE,
SIDECAR_BUNDLE_IMAGE,
)
from ..docker.egress import EGRESS_PORT
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from . import firecracker_vm, isolation_probe, netpool, util
from .bottle import FirecrackerBottle
from .bottle_plan import FirecrackerBottlePlan
from .consolidated_launch import (
launch_consolidated,
teardown_consolidated,
)
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
_GIT_HTTP_PORT = 9420
_GIT_GATE_READY_FILE = "/run/git-gate/ready"
def sidecar_container_name(slug: str) -> str:
return f"bot-bottle-sidecars-{slug}"
@contextmanager
@@ -80,6 +67,8 @@ def launch(
*,
provision: Callable[[FirecrackerBottlePlan, "FirecrackerBottle"], str | None],
) -> Generator[FirecrackerBottle, None, None]:
"""Build, launch, and provision a Firecracker bottle via the consolidated
orchestrator. Teardown on exit."""
stack = ExitStack()
bottle_for_revoke = plan.manifest.bottle
git_gate_dir_for_revoke = git_gate_state_dir(plan.slug)
@@ -96,26 +85,75 @@ def launch(
raise teardown_exc
try:
plan = _mint_certs(plan)
# Step 1: agent image. The sidecar bundle image is built by the
# orchestrator service (ensure_running → ensure_built); we only
# build the agent image here. Use a committed snapshot when available.
plan = _build_agent_image(plan)
# Claim a TAP slot; the flock is held until teardown closes it.
# Step 2: mint the git-gate dynamic (gitea) deploy keys, if any.
git_gate_plan = plan.git_gate_plan
if git_gate_plan.upstreams:
git_gate_plan = provision_git_gate_dynamic_keys(
plan.manifest.bottle, git_gate_plan, git_gate_state_dir(plan.slug),
)
# Step 3: claim a TAP slot; the flock is held until teardown.
slot, lock = netpool.allocate(plan.slug)
stack.callback(lock.close)
info(f"firecracker slot {slot.iface}: host={slot.host_ip} "
f"guest={slot.guest_ip}")
plan = _provision_git_gate_keys(plan)
# Step 4: register on the orchestrator + provision this bottle's
# git-gate state into the shared gateway. The per-bottle egress tokens
# are resolved from the host env now and handed to the orchestrator
# (in memory) for the gateway to inject — the agent never sees them.
# Attribution is by the VM's guest IP (unspoofable via /31 TAP + nft).
effective_env = {**os.environ, **plan.agent_provision.provisioned_env}
token_values = egress_resolve_token_values(
plan.egress_plan.token_env_map, effective_env,
)
ctx = launch_consolidated(
plan.egress_plan, git_gate_plan,
guest_ip=slot.guest_ip,
image_ref=plan.image,
tokens=token_values,
)
stack.callback(
teardown_consolidated, ctx.bottle_id,
orchestrator_url=ctx.orchestrator_url,
)
sidecar_name = sidecar_container_name(plan.slug)
docker_mod.force_remove_container(sidecar_name)
_start_sidecar_bundle(plan, sidecar_name, slot.host_ip)
stack.callback(docker_mod.force_remove_container, sidecar_name)
_stage_git_gate(plan, sidecar_name)
# Step 5: install the SHARED gateway CA (replaces the per-bottle CA).
# Write it to a stable host path so the provisioner can copy it over SSH.
ca_dir = egress_state_dir(plan.slug) / "gateway-ca"
ca_dir.mkdir(parents=True, exist_ok=True)
ca_file = ca_dir / "gateway-ca.pem"
ca_file.write_text(ctx.gateway_ca_pem)
egress_plan = dataclasses.replace(
plan.egress_plan,
mitmproxy_ca_host_path=ca_file,
mitmproxy_ca_cert_only_host_path=ca_file,
)
# Point the agent's git-gate insteadOf rewrites and supervise MCP URL
# at the shared gateway (reached at the slot's host TAP IP — the VM
# sends there and Docker DNAT routes to the gateway container).
git_gate_url = (
f"http://{slot.host_ip}:{_GIT_HTTP_PORT}" if git_gate_plan.upstreams else ""
)
supervise_url = (
f"http://{slot.host_ip}:{SUPERVISE_PORT}/"
if plan.supervise_plan is not None else ""
)
plan = dataclasses.replace(
plan,
git_gate_plan=git_gate_plan,
egress_plan=egress_plan,
agent_proxy_url=f"http://{slot.host_ip}:{EGRESS_PORT}",
agent_git_gate_url=git_gate_url,
agent_supervise_url=supervise_url,
)
plan = _stamp_agent_urls(plan, slot.host_ip)
# Build the per-bottle rootfs + SSH key, then boot.
# Step 6: build the per-bottle rootfs + SSH key, then boot.
base_dir = util.build_base_rootfs_dir(plan.image)
run_dir = util.cache_dir() / "run" / plan.slug
run_dir.mkdir(parents=True, exist_ok=True)
@@ -135,8 +173,8 @@ def launch(
stack.callback(vm.terminate)
firecracker_vm.wait_for_ssh(vm, private_key)
# Authoritative fail-closed egress-boundary check, before the
# agent runs: prove the VM cannot reach the host directly.
# Authoritative fail-closed egress-boundary check, before the agent
# runs: prove the VM cannot reach the host directly.
isolation_probe.verify_isolation(private_key, slot.guest_ip)
bottle = FirecrackerBottle(
@@ -161,18 +199,7 @@ def launch(
teardown()
def _mint_certs(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
egress_ca_host, egress_ca_cert_only = egress_tls_init(egress_state_dir(plan.slug))
egress_plan = dataclasses.replace(
plan.egress_plan,
mitmproxy_ca_host_path=egress_ca_host,
mitmproxy_ca_cert_only_host_path=egress_ca_cert_only,
)
return dataclasses.replace(plan, egress_plan=egress_plan)
def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
docker_mod.build_image(SIDECAR_BUNDLE_IMAGE, _REPO_DIR, dockerfile=SIDECAR_BUNDLE_DOCKERFILE)
committed = read_committed_image(plan.slug)
if committed and docker_mod.image_exists(committed):
info(f"using committed image {committed!r}")
@@ -187,148 +214,6 @@ def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
return plan
def _provision_git_gate_keys(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
if not plan.git_gate_plan.upstreams:
return plan
git_gate_plan = provision_git_gate_dynamic_keys(
plan.manifest.bottle, plan.git_gate_plan, git_gate_state_dir(plan.slug),
)
return dataclasses.replace(plan, git_gate_plan=git_gate_plan)
def _stamp_agent_urls(
plan: FirecrackerBottlePlan, host_ip: str,
) -> FirecrackerBottlePlan:
proxy_url = f"http://{host_ip}:{EGRESS_PORT}"
supervise_url = (
f"http://{host_ip}:{SUPERVISE_PORT}/" if plan.supervise_plan is not None else ""
)
git_gate_url = (
f"http://{host_ip}:{_GIT_HTTP_PORT}" if plan.git_gate_plan.upstreams else ""
)
return dataclasses.replace(
plan,
agent_proxy_url=proxy_url,
agent_git_gate_url=git_gate_url,
agent_supervise_url=supervise_url,
)
# --- sidecar bundle (Docker) ----------------------------------------
def _start_sidecar_bundle(
plan: FirecrackerBottlePlan, sidecar_name: str, host_ip: str,
) -> None:
argv = ["docker", "run", "--name", sidecar_name, "--detach", "--rm",
"-e", f"BOT_BOTTLE_SIDECAR_DAEMONS={','.join(_sidecar_daemons(plan))}"]
for entry in _sidecar_env_entries(plan):
argv += ["-e", entry]
for host_path, container_path, read_only in _sidecar_mounts(plan):
argv += ["-v", f"{host_path}:{container_path}{':ro' if read_only else ''}"]
# Publish on the slot's host TAP IP at fixed ports — each bottle
# has a distinct host_ip, so fixed ports never collide, and the VM
# reaches them at a stable, well-known address (its only route out).
for port in _sidecar_ports(plan):
argv += ["-p", f"{host_ip}:{port}:{port}"]
argv.append(SIDECAR_BUNDLE_IMAGE)
effective_env = {**dict(os.environ), **plan.agent_provision.provisioned_env}
token_values = egress_resolve_token_values(
plan.egress_plan.token_env_map, effective_env,
)
env = {**os.environ, **token_values}
info(f"docker run sidecar bundle {sidecar_name} (published on {host_ip})")
result = subprocess.run(argv, capture_output=True, text=True, env=env, check=False)
if result.returncode != 0:
die(f"docker run for sidecar bundle {sidecar_name} failed: "
f"{(result.stderr or '').strip() or '<no stderr>'}")
def _sidecar_daemons(plan: FirecrackerBottlePlan) -> tuple[str, ...]:
daemons = ["egress"]
if plan.git_gate_plan.upstreams:
daemons += ["git-gate", "git-http"]
if plan.supervise_plan is not None:
daemons.append("supervise")
return tuple(daemons)
def _sidecar_ports(plan: FirecrackerBottlePlan) -> tuple[int, ...]:
ports = [EGRESS_PORT]
if plan.git_gate_plan.upstreams:
ports.append(_GIT_HTTP_PORT)
if plan.supervise_plan is not None:
ports.append(SUPERVISE_PORT)
return tuple(ports)
def _sidecar_env_entries(plan: FirecrackerBottlePlan) -> tuple[str, ...]:
env: list[str] = list(egress_sidecar_env_entries(plan.egress_plan))
if plan.git_gate_plan.upstreams:
env.append(f"BOT_BOTTLE_GIT_GATE_READY_FILE={_GIT_GATE_READY_FILE}")
if plan.supervise_plan is not None:
env += [
f"SUPERVISE_BOTTLE_SLUG={plan.slug}",
f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}",
f"SUPERVISE_PORT={SUPERVISE_PORT}",
]
return tuple(env)
def _sidecar_mounts(
plan: FirecrackerBottlePlan,
) -> tuple[tuple[str, str, bool], ...]:
mounts: list[tuple[str, str, bool]] = []
ep = plan.egress_plan
mounts.append((str(ep.mitmproxy_ca_host_path.parent),
str(Path(EGRESS_CA_IN_CONTAINER).parent), False))
if ep.routes:
mounts.append((str(ep.routes_path.parent),
str(Path(EGRESS_ROUTES_IN_CONTAINER).parent), True))
sp = plan.supervise_plan
if sp is not None:
mounts.append((str(sp.db_path.parent),
str(Path(DB_PATH_IN_CONTAINER).parent), False))
return tuple(mounts)
def _stage_git_gate(plan: FirecrackerBottlePlan, sidecar_name: str) -> None:
gp = plan.git_gate_plan
if not gp.upstreams:
return
docker_mod.docker_exec(sidecar_name, [
"mkdir", "-p",
str(Path(GIT_GATE_HOOK_IN_CONTAINER).parent),
GIT_GATE_CREDS_DIR_IN_CONTAINER, "/git",
str(Path(_GIT_GATE_READY_FILE).parent),
])
for host_path, container_path in _git_gate_files(plan):
docker_mod.docker_cp(host_path, f"{sidecar_name}:{container_path}")
docker_mod.docker_exec(sidecar_name, [
"sh", "-c",
f"chmod 755 {GIT_GATE_ENTRYPOINT_IN_CONTAINER} "
f"{GIT_GATE_HOOK_IN_CONTAINER} {GIT_GATE_ACCESS_HOOK_IN_CONTAINER} && "
f"chmod 600 {GIT_GATE_CREDS_DIR_IN_CONTAINER}/* && "
f"touch {_GIT_GATE_READY_FILE}",
])
def _git_gate_files(plan: FirecrackerBottlePlan) -> tuple[tuple[str, str], ...]:
gp = plan.git_gate_plan
files: list[tuple[str, str]] = [
(str(gp.entrypoint_script), GIT_GATE_ENTRYPOINT_IN_CONTAINER),
(str(gp.hook_script), GIT_GATE_HOOK_IN_CONTAINER),
(str(gp.access_hook_script), GIT_GATE_ACCESS_HOOK_IN_CONTAINER),
]
for upstream in gp.upstreams:
files.append((expand_tilde(upstream.identity_file),
f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{upstream.name}-key"))
if upstream.known_hosts_file:
files.append((str(upstream.known_hosts_file),
f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{upstream.name}-known_hosts"))
return tuple(files)
# --- agent guest env -------------------------------------------------
def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str]: