didericis/bot-bottle
1
0
Fork 0
Code Issues 25 Pull Requests 9 Actions Packages Projects Releases Wiki Activity

refactor(de-sidecar): remove the per-bottle companion-container architecture

Browse Source 
The per-agent companion container (the egress/git-gate/supervise data
plane run once per bottle) is the pre-consolidation architecture. Remove
it and disable the backends that still depend on it, per the #385 thread.

- Delete `backend/docker/sidecar_bundle.py`; docker's live path uses the
  consolidated shared gateway, not a per-bottle bundle.
- Disable the firecracker and macos-container backends: their `launch()`
  fails closed (they launched a per-bottle companion; firecracker's
  consolidated relaunch is #354, macos follows). Their `enumerate` return
  empty and `cleanup` drop the companion-container discovery (firecracker
  keeps VMM/run-dir cleanup).
- Fail-close both backends' `egress_apply` reload (it signalled the
  per-bottle container); consolidated egress policy resolves per-request
  against the orchestrator, so gateway-side apply is a follow-up.
- Rename `egress_sidecar_env_entries` → `egress_gateway_env_entries`,
  `SIDECAR_PORTS` → `GATEWAY_PORTS`.
- Move the shared DockerBottlePlan fixture to `tests/unit/_docker_bottle_plan.py`;
  delete tests for the removed launch paths; update cleanup/egress-apply tests.

Docker consolidated launch verified end-to-end (multitenant isolation
integration test passes). macos/firecracker are intentionally disabled.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck
This commit is contained in:
didericis
2026-07-14 17:02:08 -04:00
parent c10d1cb6e0
commit 77948ef56c
22 changed files with 259 additions and 1950 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bot_bottle/backend/firecracker/bottle_cleanup_plan.py
+3 -6
View File
@@ -10,10 +10,9 @@ from .. import BottleCleanupPlan
@dataclass(frozen=True)
class FirecrackerBottleCleanupPlan(BottleCleanupPlan):
# PIDs of orphaned firecracker VMM processes and the sidecar
# containers left behind by previous bottles.
# PIDs of orphaned firecracker VMM processes and the per-bottle run
# dirs left behind by previous bottles.
vm_pids: tuple[int, ...] = ()
containers: tuple[str, ...] = ()
run_dirs: tuple[str, ...] = ()
def print(self) -> None:
@@ -22,11 +21,9 @@ class FirecrackerBottleCleanupPlan(BottleCleanupPlan):
return
for pid in self.vm_pids:
info(f"firecracker VM process: pid {pid}")
for name in self.containers:
info(f"firecracker sidecar container: {name}")
for path in self.run_dirs:
info(f"firecracker run dir: {path}")
@property
def empty(self) -> bool:
return not (self.vm_pids or self.containers or self.run_dirs)
return not (self.vm_pids or self.run_dirs)
bot_bottle/backend/firecracker/cleanup.py
+2 -23
View File
@@ -1,9 +1,8 @@
"""Cleanup for the Firecracker backend.
Orphans are: firecracker VMM processes whose config lives under our run
dir, the `bot-bottle-sidecars-*` containers, and the per-bottle run
dirs. TAP slots free themselves (the flock drops when the launcher
exits), so there is nothing to reclaim there.
dir, and the per-bottle run dirs. TAP slots free themselves (the flock
drops when the launcher exits), so there is nothing to reclaim there.
"""
from __future__ import annotations
@@ -18,8 +17,6 @@ from ...log import info
from . import util
from .bottle_cleanup_plan import FirecrackerBottleCleanupPlan
_SIDECAR_PREFIX = "bot-bottle-sidecars-"
def _run_root() -> Path:
return util.cache_dir() / "run"
@@ -46,17 +43,6 @@ def _orphan_vm_pids() -> list[int]:
return pids
def _sidecar_containers() -> list[str]:
result = subprocess.run(
["docker", "ps", "-a", "--format", "{{.Names}}",
"--filter", f"name={_SIDECAR_PREFIX}"],
capture_output=True, text=True, check=False,
)
if result.returncode != 0:
return []
return sorted(n.strip() for n in result.stdout.splitlines() if n.strip())
def _run_dirs() -> list[str]:
run_root = _run_root()
if not run_root.is_dir():
@@ -67,7 +53,6 @@ def _run_dirs() -> list[str]:
def prepare_cleanup() -> FirecrackerBottleCleanupPlan:
return FirecrackerBottleCleanupPlan(
vm_pids=tuple(_orphan_vm_pids()),
containers=tuple(_sidecar_containers()),
run_dirs=tuple(_run_dirs()),
)
@@ -79,12 +64,6 @@ def cleanup(plan: FirecrackerBottleCleanupPlan) -> None:
os.kill(pid, signal.SIGTERM)
except ProcessLookupError:
pass
for name in plan.containers:
info(f"docker rm -f {name}")
subprocess.run(
["docker", "rm", "-f", name],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
)
for path in plan.run_dirs:
info(f"rm -rf {path}")
shutil.rmtree(path, ignore_errors=True)
bot_bottle/backend/firecracker/enumerate.py
+4 -33
View File
@@ -1,43 +1,14 @@
"""Active-agent enumeration for the Firecracker backend.
The agent runs in a VM (no container to list), so a live bottle is
identified by its running sidecar container `bot-bottle-sidecars-<slug>`
— the same discovery-by-prefix the other backends use.
The backend is disabled during the de-sidecar cleanup (#385) — it can't
launch bottles, so there are none to enumerate. Real enumeration returns
with the backend's consolidated relaunch (#354).
"""
from __future__ import annotations
import subprocess
from ...bottle_state import read_metadata
from .. import ActiveAgent
_SIDECAR_PREFIX = "bot-bottle-sidecars-"
def enumerate_active() -> list[ActiveAgent]:
result = subprocess.run(
["docker", "ps", "--format", "{{.Names}}",
"--filter", f"name={_SIDECAR_PREFIX}"],
capture_output=True, text=True, check=False,
)
if result.returncode != 0:
return []
out: list[ActiveAgent] = []
for name in sorted(n.strip() for n in result.stdout.splitlines() if n.strip()):
slug = name[len(_SIDECAR_PREFIX):]
metadata = read_metadata(slug)
if metadata is None or metadata.backend != "firecracker":
# Skip sidecars owned by another backend (docker shares the
# container-name prefix).
continue
out.append(ActiveAgent(
backend_name="firecracker",
slug=slug,
agent_name=metadata.agent_name,
started_at=metadata.started_at,
services=(),
label=metadata.label,
color=metadata.color,
))
return out
return []
bot_bottle/backend/firecracker/launch.py
+20 -385
View File
@@ -1,404 +1,39 @@
"""Launch flow for the Firecracker backend.
"""Launch flow for the Firecracker backend — temporarily disabled (#385).
Per bottle:
1. mint the egress CA, build the agent image (docker), export it to a
cached ext4 rootfs;
2. claim a free TAP pool slot (rootless flock);
3. bring up the Docker sidecar bundle, publishing egress / git-gate /
supervise on the slot's host-side TAP IP at fixed ports;
4. boot the microVM on that TAP; wait for SSH;
5. provision (CA, prompt, skills, workspace, git, supervise) over SSH.
The firecracker backend launched a per-bottle companion container (the
egress / git-gate / supervise data plane) alongside each microVM. That
per-bottle-companion architecture was removed in the de-sidecar cleanup;
firecracker's replacement — the consolidated per-host gateway — lands in
its own cutover (#354).
Isolation is enforced by the operator-provisioned nft table (checked
fail-closed in preflight): a VM reaches only its sidecar (DNAT'd from
the host TAP IP) and nothing else. The agent's HTTPS_PROXY therefore
points at `http://<host_tap_ip>:9099`, its only route to the world.
Until that lands, launching a firecracker bottle fails closed rather than
silently running the removed path. `prepare` / `status` / cleanup still
work, so `backend status --backend=firecracker` and orphan cleanup are
unaffected.
"""
from __future__ import annotations
import dataclasses
import os
import subprocess
from contextlib import ExitStack, contextmanager
from pathlib import Path
from contextlib import contextmanager
from typing import Callable, Generator
from ...bottle_state import (
egress_state_dir,
git_gate_state_dir,
read_committed_image,
)
from ...egress import (
EGRESS_ROUTES_IN_CONTAINER,
egress_agent_env_entries,
egress_resolve_token_values,
egress_sidecar_env_entries,
)
from ...git_gate import (
provision_git_gate_dynamic_keys,
revoke_git_gate_provisioned_keys,
)
from ...log import die, info, warn
from ...supervise import DB_PATH_IN_CONTAINER, SUPERVISE_PORT
from ...util import expand_tilde
from ..docker.egress import (
EGRESS_CA_IN_CONTAINER,
EGRESS_PORT,
egress_tls_init,
)
from ..docker.git_gate import (
GIT_GATE_ACCESS_HOOK_IN_CONTAINER,
GIT_GATE_CREDS_DIR_IN_CONTAINER,
GIT_GATE_ENTRYPOINT_IN_CONTAINER,
GIT_GATE_HOOK_IN_CONTAINER,
)
from ..docker.sidecar_bundle import (
SIDECAR_BUNDLE_DOCKERFILE,
SIDECAR_BUNDLE_IMAGE,
)
from ..util import AGENT_CA_BUNDLE, AGENT_CA_PATH
from . import firecracker_vm, isolation_probe, netpool, util
from ...log import die
from .bottle import FirecrackerBottle
from .bottle_plan import FirecrackerBottlePlan
_REPO_DIR = str(Path(__file__).resolve().parent.parent.parent.parent)
_GIT_HTTP_PORT = 9420
_GIT_GATE_READY_FILE = "/run/git-gate/ready"
def sidecar_container_name(slug: str) -> str:
return f"bot-bottle-sidecars-{slug}"
@contextmanager
def launch(
plan: FirecrackerBottlePlan,
*,
provision: Callable[[FirecrackerBottlePlan, "FirecrackerBottle"], str | None],
) -> Generator[FirecrackerBottle, None, None]:
stack = ExitStack()
bottle_for_revoke = plan.manifest.bottle
git_gate_dir_for_revoke = git_gate_state_dir(plan.slug)
def teardown() -> None:
teardown_exc: BaseException | None = None
try:
stack.close()
except BaseException as exc: # noqa: W0718 - teardown must continue
teardown_exc = exc
warn(f"firecracker teardown failed: {exc!r}")
revoke_git_gate_provisioned_keys(bottle_for_revoke, git_gate_dir_for_revoke)
if teardown_exc is not None:
raise teardown_exc
try:
plan = _mint_certs(plan)
plan = _build_agent_image(plan)
# Claim a TAP slot; the flock is held until teardown closes it.
slot, lock = netpool.allocate(plan.slug)
stack.callback(lock.close)
info(f"firecracker slot {slot.iface}: host={slot.host_ip} "
f"guest={slot.guest_ip}")
plan = _provision_git_gate_keys(plan)
sidecar_name = sidecar_container_name(plan.slug)
_force_remove_container(sidecar_name)
_start_sidecar_bundle(plan, sidecar_name, slot.host_ip)
stack.callback(_force_remove_container, sidecar_name)
_stage_git_gate(plan, sidecar_name)
plan = _stamp_agent_urls(plan, slot.host_ip)
# Build the per-bottle rootfs + SSH key, then boot.
base_dir = util.build_base_rootfs_dir(plan.image)
run_dir = util.cache_dir() / "run" / plan.slug
run_dir.mkdir(parents=True, exist_ok=True)
rootfs = run_dir / "rootfs.ext4"
util.build_rootfs_ext4(base_dir, rootfs)
private_key, pubkey = util.generate_keypair(run_dir)
vm = firecracker_vm.boot(
name=plan.container_name,
rootfs=rootfs,
tap=slot.iface,
guest_ip=slot.guest_ip,
host_ip=slot.host_ip,
pubkey=pubkey,
run_dir=run_dir,
)
stack.callback(vm.terminate)
firecracker_vm.wait_for_ssh(vm, private_key)
# Authoritative fail-closed egress-boundary check, before the
# agent runs: prove the VM cannot reach the host directly.
isolation_probe.verify_isolation(private_key, slot.guest_ip)
bottle = FirecrackerBottle(
plan.container_name,
private_key=private_key,
guest_ip=slot.guest_ip,
guest_env=_agent_guest_env(plan, slot.host_ip),
agent_command=plan.agent_command,
agent_prompt_mode=plan.agent_prompt_mode,
agent_provider_template=plan.agent_provider_template,
terminal_title=(
f"{plan.spec.label} ({plan.spec.agent_name})"
if plan.spec.label else plan.spec.agent_name
),
terminal_color=plan.spec.color,
agent_workdir=plan.workspace_plan.workdir,
)
bottle.prompt_path = provision(plan, bottle)
yield bottle
finally:
teardown()
def _mint_certs(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
egress_ca_host, egress_ca_cert_only = egress_tls_init(egress_state_dir(plan.slug))
egress_plan = dataclasses.replace(
plan.egress_plan,
mitmproxy_ca_host_path=egress_ca_host,
mitmproxy_ca_cert_only_host_path=egress_ca_cert_only,
"""Fail closed: the firecracker backend is disabled while its
consolidated (gateway-backed) launch is built in #354."""
del plan, provision
die(
"the firecracker backend is temporarily disabled during the "
"companion-container removal (#385); its consolidated relaunch "
"lands in #354. Use --backend=docker for now."
)
return dataclasses.replace(plan, egress_plan=egress_plan)
def _build_agent_image(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
_docker_build(SIDECAR_BUNDLE_IMAGE, _REPO_DIR, dockerfile=SIDECAR_BUNDLE_DOCKERFILE)
committed = read_committed_image(plan.slug)
if committed and _image_exists(committed):
info(f"using committed image {committed!r}")
return dataclasses.replace(
plan,
agent_provision=dataclasses.replace(plan.agent_provision, image=committed),
)
_docker_build(plan.image, _REPO_DIR, dockerfile=plan.dockerfile_path)
return plan
def _provision_git_gate_keys(plan: FirecrackerBottlePlan) -> FirecrackerBottlePlan:
if not plan.git_gate_plan.upstreams:
return plan
git_gate_plan = provision_git_gate_dynamic_keys(
plan.manifest.bottle, plan.git_gate_plan, git_gate_state_dir(plan.slug),
)
return dataclasses.replace(plan, git_gate_plan=git_gate_plan)
def _stamp_agent_urls(
plan: FirecrackerBottlePlan, host_ip: str,
) -> FirecrackerBottlePlan:
proxy_url = f"http://{host_ip}:{EGRESS_PORT}"
supervise_url = (
f"http://{host_ip}:{SUPERVISE_PORT}/" if plan.supervise_plan is not None else ""
)
git_gate_url = (
f"http://{host_ip}:{_GIT_HTTP_PORT}" if plan.git_gate_plan.upstreams else ""
)
return dataclasses.replace(
plan,
agent_proxy_url=proxy_url,
agent_git_gate_url=git_gate_url,
agent_supervise_url=supervise_url,
)
# --- sidecar bundle (Docker) ----------------------------------------
def _start_sidecar_bundle(
plan: FirecrackerBottlePlan, sidecar_name: str, host_ip: str,
) -> None:
argv = ["docker", "run", "--name", sidecar_name, "--detach", "--rm",
"-e", f"BOT_BOTTLE_SIDECAR_DAEMONS={','.join(_sidecar_daemons(plan))}"]
for entry in _sidecar_env_entries(plan):
argv += ["-e", entry]
for host_path, container_path, read_only in _sidecar_mounts(plan):
argv += ["-v", f"{host_path}:{container_path}{':ro' if read_only else ''}"]
# Publish on the slot's host TAP IP at fixed ports — each bottle
# has a distinct host_ip, so fixed ports never collide, and the VM
# reaches them at a stable, well-known address (its only route out).
for port in _sidecar_ports(plan):
argv += ["-p", f"{host_ip}:{port}:{port}"]
argv.append(SIDECAR_BUNDLE_IMAGE)
effective_env = {**dict(os.environ), **plan.agent_provision.provisioned_env}
token_values = egress_resolve_token_values(
plan.egress_plan.token_env_map, effective_env,
)
env = {**os.environ, **token_values}
info(f"docker run sidecar bundle {sidecar_name} (published on {host_ip})")
result = subprocess.run(argv, capture_output=True, text=True, env=env, check=False)
if result.returncode != 0:
die(f"docker run for sidecar bundle {sidecar_name} failed: "
f"{(result.stderr or '').strip() or '<no stderr>'}")
def _sidecar_daemons(plan: FirecrackerBottlePlan) -> tuple[str, ...]:
daemons = ["egress"]
if plan.git_gate_plan.upstreams:
daemons += ["git-gate", "git-http"]
if plan.supervise_plan is not None:
daemons.append("supervise")
return tuple(daemons)
def _sidecar_ports(plan: FirecrackerBottlePlan) -> tuple[int, ...]:
ports = [EGRESS_PORT]
if plan.git_gate_plan.upstreams:
ports.append(_GIT_HTTP_PORT)
if plan.supervise_plan is not None:
ports.append(SUPERVISE_PORT)
return tuple(ports)
def _sidecar_env_entries(plan: FirecrackerBottlePlan) -> tuple[str, ...]:
env: list[str] = list(egress_sidecar_env_entries(plan.egress_plan))
if plan.git_gate_plan.upstreams:
env.append(f"BOT_BOTTLE_GIT_GATE_READY_FILE={_GIT_GATE_READY_FILE}")
if plan.supervise_plan is not None:
env += [
f"SUPERVISE_BOTTLE_SLUG={plan.slug}",
f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}",
f"SUPERVISE_PORT={SUPERVISE_PORT}",
]
return tuple(env)
def _sidecar_mounts(
plan: FirecrackerBottlePlan,
) -> tuple[tuple[str, str, bool], ...]:
mounts: list[tuple[str, str, bool]] = []
ep = plan.egress_plan
mounts.append((str(ep.mitmproxy_ca_host_path.parent),
str(Path(EGRESS_CA_IN_CONTAINER).parent), False))
if ep.routes:
mounts.append((str(ep.routes_path.parent),
str(Path(EGRESS_ROUTES_IN_CONTAINER).parent), True))
sp = plan.supervise_plan
if sp is not None:
mounts.append((str(sp.db_path.parent),
str(Path(DB_PATH_IN_CONTAINER).parent), False))
return tuple(mounts)
def _stage_git_gate(plan: FirecrackerBottlePlan, sidecar_name: str) -> None:
gp = plan.git_gate_plan
if not gp.upstreams:
return
_docker_exec(sidecar_name, [
"mkdir", "-p",
str(Path(GIT_GATE_HOOK_IN_CONTAINER).parent),
GIT_GATE_CREDS_DIR_IN_CONTAINER, "/git",
str(Path(_GIT_GATE_READY_FILE).parent),
])
for host_path, container_path in _git_gate_files(plan):
_docker_cp(host_path, f"{sidecar_name}:{container_path}")
_docker_exec(sidecar_name, [
"sh", "-c",
f"chmod 755 {GIT_GATE_ENTRYPOINT_IN_CONTAINER} "
f"{GIT_GATE_HOOK_IN_CONTAINER} {GIT_GATE_ACCESS_HOOK_IN_CONTAINER} && "
f"chmod 600 {GIT_GATE_CREDS_DIR_IN_CONTAINER}/* && "
f"touch {_GIT_GATE_READY_FILE}",
])
def _git_gate_files(plan: FirecrackerBottlePlan) -> tuple[tuple[str, str], ...]:
gp = plan.git_gate_plan
files: list[tuple[str, str]] = [
(str(gp.entrypoint_script), GIT_GATE_ENTRYPOINT_IN_CONTAINER),
(str(gp.hook_script), GIT_GATE_HOOK_IN_CONTAINER),
(str(gp.access_hook_script), GIT_GATE_ACCESS_HOOK_IN_CONTAINER),
]
for upstream in gp.upstreams:
files.append((expand_tilde(upstream.identity_file),
f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{upstream.name}-key"))
if upstream.known_hosts_file:
files.append((str(upstream.known_hosts_file),
f"{GIT_GATE_CREDS_DIR_IN_CONTAINER}/{upstream.name}-known_hosts"))
return tuple(files)
# --- agent guest env -------------------------------------------------
def _agent_guest_env(plan: FirecrackerBottlePlan, host_ip: str) -> dict[str, str]:
"""Env injected into every agent/exec call over SSH. The VM has no
baked process env (it just runs init), so the proxy/CA/git/supervise
wiring is applied per-invocation."""
proxy_url = f"http://{host_ip}:{EGRESS_PORT}"
no_proxy = f"localhost,127.0.0.1,{host_ip}"
env: dict[str, str] = {
"HTTPS_PROXY": proxy_url, "HTTP_PROXY": proxy_url,
"https_proxy": proxy_url, "http_proxy": proxy_url,
"NO_PROXY": no_proxy, "no_proxy": no_proxy,
"NODE_EXTRA_CA_CERTS": AGENT_CA_PATH,
"SSL_CERT_FILE": AGENT_CA_BUNDLE,
"REQUESTS_CA_BUNDLE": AGENT_CA_BUNDLE,
}
if plan.agent_git_gate_url:
env["GIT_GATE_URL"] = plan.agent_git_gate_url
if plan.agent_supervise_url:
env["MCP_SUPERVISE_URL"] = plan.agent_supervise_url
for entry in egress_agent_env_entries(plan.egress_plan):
key, _, value = entry.partition("=")
env[key] = value
env.update(plan.agent_provision.guest_env)
# Forwarded (bare-name) env: resolve host values now, since the VM
# can't inherit them from a `docker run --env NAME`.
for name in plan.forwarded_env:
value = os.environ.get(name)
if value is not None:
env[name] = value
return env
# --- docker helpers --------------------------------------------------
def _docker_build(ref: str, context: str, *, dockerfile: str = "") -> None:
info(f"docker build {ref}")
args = ["docker", "build", "-t", ref]
if dockerfile:
if not os.path.isabs(dockerfile):
dockerfile = os.path.join(context, dockerfile)
args += ["-f", dockerfile]
args.append(context)
result = subprocess.run(args, check=False)
if result.returncode != 0:
die(f"docker build for {ref!r} failed")
def _image_exists(ref: str) -> bool:
return subprocess.run(
["docker", "image", "inspect", ref],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
).returncode == 0
def _force_remove_container(name: str) -> None:
subprocess.run(
["docker", "rm", "-f", name],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False,
)
def _docker_exec(name: str, argv: list[str]) -> None:
result = subprocess.run(
["docker", "exec", name, *argv], capture_output=True, text=True, check=False,
)
if result.returncode != 0:
die(f"docker exec in {name} failed: "
f"{(result.stderr or '').strip() or '<no stderr>'}")
def _docker_cp(host_path: str, dest: str) -> None:
result = subprocess.run(
["docker", "cp", host_path, dest], capture_output=True, text=True, check=False,
)
if result.returncode != 0:
die(f"docker cp {host_path} -> {dest} failed: "
f"{(result.stderr or '').strip() or '<no stderr>'}")
yield # unreachable — `die` raises; keeps this a generator/contextmanager
bot_bottle/backend/firecracker/netpool.py
+1 -1
View File
@@ -91,7 +91,7 @@ def ip_base() -> str:
# Sidecar ports the VM reaches at its host-side TAP IP. Kept in sync
# with the backend constants (egress 9099, supervise 9100, git-http
# 9420); rendered into the setup output for operator visibility.
SIDECAR_PORTS = (9099, 9100, 9420)
GATEWAY_PORTS = (9099, 9100, 9420)
@dataclass(frozen=True)