refactor(backend): InfraService ABC + FirecrackerInfraService

Add a backend-neutral `InfraService` ABC (backend/infra_service.py) so all three
backends present the same composer contract: `orchestrator()` / `gateway()`
accessors, `ensure_running(*, startup_timeout) -> str` (the host control-plane
URL), `stop()`, plus concrete `url()` / `is_healthy()` that delegate to the
orchestrator. `DockerInfraService` and `MacosInfraService` now subclass it.

Give firecracker the missing class: `FirecrackerInfraService`
(backend/firecracker/infra.py) wraps the `infra_vm` substrate as the pair
coordinator (adopt-or-boot-both under the singleton flock). `infra_vm` keeps only
the plane-agnostic substrate; its `ensure_running` / `_adopt` / `InfraEndpoint`
move to the class, and the coordinator helpers it now calls cross-module go
public (`singleton_lock` / `expected_version` / `adoptable` /
`record_booted_version`).

Unify the return type on the way: `ensure_running` returns the URL string
everywhere (was `str` for docker, two different `InfraEndpoint`s for
macOS/firecracker), and those `InfraEndpoint`s are deleted — the services are the
source of truth (callers read `gateway().address()` / `orchestrator().url()`).
docker's `url` property becomes the inherited `url()`. backend.py /
consolidated_launch / image_builder + tests updated.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-25 16:10:21 -04:00
parent cb2d778a8f
commit afb92ca155
19 changed files with 337 additions and 309 deletions
+2 -9
View File
@@ -40,6 +40,7 @@ from ...gateway import (
GATEWAY_NAME,
GATEWAY_NETWORK,
)
from ..infra_service import InfraService
from ...orchestrator.lifecycle import (
DEFAULT_PORT,
DEFAULT_STARTUP_TIMEOUT_SECONDS,
@@ -53,7 +54,7 @@ INFRA_NAME = GATEWAY_NAME # the container agents attribute against is the gatew
_REPO_ROOT = Path(__file__).resolve().parents[3]
class DockerInfraService:
class DockerInfraService(InfraService):
"""Composes the per-host control plane + gateway as two containers.
`orchestrator_name` / `gateway_name` let callers run independent pairs on
@@ -91,14 +92,6 @@ class DockerInfraService:
against (gateway IP, git-gate provisioning transport, CA fetch)."""
return self._gateway_name
@property
def url(self) -> str:
"""Host-side control-plane URL (the orchestrator's published loopback)."""
return self.orchestrator().url()
def is_healthy(self) -> bool:
return self.orchestrator().is_healthy()
def orchestrator(self) -> DockerOrchestrator:
"""The control-plane service. Cheap to reconstruct — `ensure_built`
builds its image, `ensure_running` brings it up, and the launch flow
+2 -2
View File
@@ -123,5 +123,5 @@ class FirecrackerBottleBackend(
return plan.agent_supervise_url
def ensure_orchestrator(self) -> str:
from . import infra_vm
return infra_vm.ensure_running().orchestrator_url
from .infra import FirecrackerInfraService
return FirecrackerInfraService().ensure_running()
@@ -41,8 +41,9 @@ from ...orchestrator.lifecycle import (
from ...orchestrator.reprovision import reprovision_bottles
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
from ..provision_bottle import deprovision_bottle, provision_bottle
from . import cleanup, infra_vm, util
from . import cleanup, util
from .gateway import FirecrackerGateway
from .infra import FirecrackerInfraService
_ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret"
@@ -122,13 +123,13 @@ def launch_consolidated(
"""Ensure the infra VM is up, register the bottle by its guest IP, and
provision its git-gate state into the gateway VM. Returns the context the
agent-VM launch needs. Raises on failure — the caller tears down."""
infra = infra_vm.ensure_running()
url = infra.orchestrator_url
service = FirecrackerInfraService()
url = service.ensure_running()
client = OrchestratorClient(url)
_reprovision_running_bottles(client)
# Read the gateway's provisioning transport + CA off the Gateway service.
gateway = infra.gateway
gateway = service.gateway()
transport = gateway.provisioning_transport()
reg = provision_bottle(
client, guest_ip, egress_plan, git_gate_plan, transport,
+8 -9
View File
@@ -7,14 +7,13 @@ fetching the mitmproxy CA, and the SSH exec/cp provisioning transport. The
gateway VM never opens `bot-bottle.db` (#469), so it holds no signing key —
only the token the host hands it via `connect_to_orchestrator`.
What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM
substrate the orchestrator VM also uses — booting a VM from a per-plane rootfs
(`boot_vm`), the stable SSH keypair, the secret-push retry loop, and the PID
lifecycle — plus the pair coordinator (`ensure_running`: orchestrator-first
health gate, singleton lock, adoption/version marker). The guest-side gateway
daemon startup lives in the gateway rootfs's guest init (`_gateway_init` in
`infra_vm`), so it can't live in a host-side method either. These are the shared
seam.
The plane-agnostic VM substrate the orchestrator VM also uses stays in
`infra_vm` — booting a VM from a per-plane rootfs (`boot_vm`), the stable SSH
keypair, the secret-push retry loop, the PID lifecycle, and the
adoption/version helpers. The guest-side gateway daemon startup lives in the
gateway rootfs's guest init (`_gateway_init` in `infra_vm`), so it can't live in
a host-side method either. The pair coordinator (orchestrator-first, under a
singleton flock) is `FirecrackerInfraService` (`infra.py`).
"""
from __future__ import annotations
@@ -112,7 +111,7 @@ class FirecrackerGateway(Gateway):
def stop(self) -> None:
"""Stop only the gateway VM (idempotent — absent is success). A dead
gateway already fails `infra_vm._adoptable`, so no version marker to
gateway already fails `infra_vm.adoptable`, so no version marker to
clear here."""
infra_vm._kill_pidfile(infra_vm._gw_dir())
infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True)
@@ -26,7 +26,8 @@ from pathlib import Path
from typing import Generator
from ...log import die, info
from . import infra_vm, util
from . import util
from .infra import FirecrackerInfraService
# vfs + chroot: buildah works as root in the microVM (no fuse-overlayfs /
# overlay module / subuid maps). `--isolation` is a build/run-only flag;
@@ -125,8 +126,9 @@ def _build_in_infra(
orchestrator VM (the build-capable control plane), smoke test the image,
and stream its rootfs into `base`. The infra VMs persist; only the
per-build container/image/context are cleaned up."""
orchestrator = infra_vm.ensure_running().orchestrator
key, ip = orchestrator.ssh_target()
service = FirecrackerInfraService()
service.ensure_running()
key, ip = service.orchestrator().ssh_target()
tag = f"bot-bottle-agent-build-{digest}"
ctx = f"/tmp/agent-build-{digest}"
smoke_ctr, export_ctr = f"{tag}-smoke", f"{tag}-export"
+80
View File
@@ -0,0 +1,80 @@
"""The per-host infra service for the Firecracker backend (PRD 0070).
`FirecrackerInfraService` is the Firecracker `InfraService`: it composes the
orchestrator + gateway microVMs as an idempotent per-host singleton pair over
the shared `infra_vm` substrate (boot primitives, the singleton flock, the
adopt/version machinery). Unlike the docker/macOS services it takes no
per-instance config — the firecracker pair is a **hard** per-host singleton,
pinned to the fixed netpool links (`orch_slot()` / `gw_slot()`), host cache
paths, and a `booted-version` marker, so two pairs can't coexist on one host.
"""
from __future__ import annotations
from ...log import info
from ..infra_service import InfraService
from ...orchestrator.lifecycle import DEFAULT_STARTUP_TIMEOUT_SECONDS
from . import infra_vm
from .gateway import FirecrackerGateway
from .orchestrator import FirecrackerOrchestrator
class FirecrackerInfraService(InfraService):
"""Bring up the orchestrator + gateway microVM pair as a per-host singleton."""
def orchestrator(self) -> FirecrackerOrchestrator:
"""The control-plane service (adopts the running orchestrator VM by its
fixed link; no live handle needed for URL / SSH / health)."""
return FirecrackerOrchestrator()
def gateway(self) -> FirecrackerGateway:
"""The data-plane service (adopts the running gateway VM by its fixed
link; CA fetch / provisioning go through the stable key)."""
return FirecrackerGateway()
def ensure_running(
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
) -> str:
"""Idempotent per-host singleton pair. Adopt the running VMs if the
orchestrator's control plane is healthy AND the gateway VM is alive AND
both booted the CURRENT code version (a prior launcher booted them — they
outlive short-lived `start` processes); otherwise clear any stale VMs and
boot a fresh pair (orchestrator first, then the gateway that resolves
policy against it). Returns the host control-plane URL.
Concurrency-safe: the cold stop/build/boot path is serialized by a host
flock, so two simultaneous first launches don't both boot on the same
links/PIDs. The healthy fast-path takes no lock."""
key = infra_vm._infra_dir() / "id_ed25519"
orchestrator = self.orchestrator()
url = orchestrator.url()
want = infra_vm.expected_version()
if infra_vm.adoptable(key, url, want):
info(f"adopting running infra VMs (orchestrator at {url})")
return url
with infra_vm.singleton_lock():
# Re-check under the lock: another launcher may have booted them while
# we waited (double-checked, so we adopt not re-boot).
if infra_vm.adoptable(key, url, want):
info(f"adopting running infra VMs (orchestrator at {url})")
return url
# Clear stale/hung/OUTDATED VMs holding either link before booting fresh.
infra_vm.stop()
infra_vm.ensure_built()
# Orchestrator first — the gateway daemons reach the control plane at
# startup. Each service owns its own boot; the orchestrator (which
# holds the signing key) mints the role-scoped `gateway` JWT for the
# gateway, which never sees the key (#469).
orchestrator.ensure_running(startup_timeout=startup_timeout)
self.gateway().connect_to_orchestrator(
orchestrator.gateway_url(), orchestrator.mint_gateway_token())
infra_vm.record_booted_version(want)
return url
def stop(self) -> None:
"""Stop both infra VMs (idempotent) and drop the version marker."""
infra_vm.stop()
__all__ = ["FirecrackerInfraService"]
+5 -87
View File
@@ -40,16 +40,12 @@ import urllib.request
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING, Generator
from typing import Generator
from ...log import die, info
from ..docker import util as docker_mod
from . import firecracker_vm, infra_artifact, netpool, util
if TYPE_CHECKING:
from .gateway import FirecrackerGateway
from .orchestrator import FirecrackerOrchestrator
# The orchestrator VM's signing-key path on its persistent /dev/vdb volume
# (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds this
# file with the host-canonical key AFTER boot (over SSH), so the VM verifies
@@ -89,7 +85,6 @@ GIT_HTTP_PORT = 9420
# now; routing DNS through a filtered path is a later refinement.
_INFRA_RESOLVER = "1.1.1.1"
_CA_TIMEOUT_SECONDS = 30.0
# How long the launcher retries pushing a secret while the guest's SSH comes
# up. Below the init's own wait window, so a failed push dies here first.
_SECRET_PUSH_TIMEOUT_SECONDS = 30.0
@@ -108,24 +103,6 @@ class InfraVm:
vm: firecracker_vm.VmHandle | None = None
@dataclass
class InfraEndpoint:
"""How to reach the running per-host pair. `orchestrator` is the control-plane
`Orchestrator` service (host CLI + registration + in-VM builds); `gateway` is
the data-plane `Gateway` service (agent egress / git-http / supervise, CA
fetch, git-gate provisioning). Mirrors the docker/macOS `InfraEndpoint`."""
orchestrator: "FirecrackerOrchestrator"
gateway: "FirecrackerGateway"
@property
def orchestrator_url(self) -> str:
return self.orchestrator.url()
def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str:
return self.gateway.ca_cert_pem(timeout=timeout)
def role_init(role: str) -> str:
"""The guest PID-1 init for `role` (each per-plane rootfs bakes only its
own — no `bb_role` branch, since the rootfs *is* the role)."""
@@ -177,67 +154,8 @@ def build_rootfs_dir(role: str) -> Path:
)
def ensure_running() -> InfraEndpoint:
"""Idempotent per-host singleton pair. Adopt the running VMs if the
orchestrator's control plane is healthy AND the gateway VM is alive AND
both booted the CURRENT code version (a prior launcher booted them — they
outlive short-lived `start` processes); otherwise clear any stale VMs and
boot a fresh pair (orchestrator first, then the gateway that resolves
policy against it). Returns handles usable for builds / CA fetch / git-gate
provisioning.
Concurrency-safe: the cold stop/build/boot path is serialized by a host
flock, so two simultaneous first launches don't both boot on the same
links/PIDs. The healthy fast-path takes no lock."""
# Local imports: the orchestrator + gateway services import this module for
# the shared VM substrate, so importing them at module scope would cycle.
from .orchestrator import FirecrackerOrchestrator
from .gateway import FirecrackerGateway
key = _infra_dir() / "id_ed25519"
orchestrator = FirecrackerOrchestrator()
url = orchestrator.url()
want = _expected_version()
if _adoptable(key, url, want):
info(f"adopting running infra VMs (orchestrator at {url})")
return _adopt(key)
with _singleton_lock():
# Re-check under the lock: another launcher may have booted them while
# we waited for the lock (double-checked, so we adopt not re-boot).
if _adoptable(key, url, want):
info(f"adopting running infra VMs (orchestrator at {url})")
return _adopt(key)
# Clear stale/hung/OUTDATED VMs holding either link before booting fresh.
stop()
ensure_built()
# Orchestrator first — the gateway daemons reach the control plane at
# startup. Each service owns its own boot; the orchestrator (which holds
# the signing key) mints the role-scoped `gateway` JWT for the gateway,
# which never sees the key (#469).
orchestrator.ensure_running()
gateway = FirecrackerGateway()
gateway.connect_to_orchestrator(
orchestrator.gateway_url(), orchestrator.mint_gateway_token())
_record_booted_version(want)
return InfraEndpoint(orchestrator=orchestrator, gateway=gateway)
def _adopt(key: Path) -> "InfraEndpoint":
"""Build the service handles for the already-running pair from the stable key
+ the fixed links (no live VM handle — teardown / CA fetch go through the PID
files + stable key)."""
from .orchestrator import FirecrackerOrchestrator
from .gateway import FirecrackerGateway
return InfraEndpoint(
orchestrator=FirecrackerOrchestrator(),
gateway=FirecrackerGateway(
InfraVm(guest_ip=netpool.gw_slot().guest_ip, private_key=key)),
)
@contextmanager
def _singleton_lock() -> Generator[None, None, None]:
def singleton_lock() -> Generator[None, None, None]:
"""Host-level exclusive lock serializing the infra pair's cold create path
(`stop`/`ensure_built`/`boot`). flock auto-releases if the launcher
crashes, so the lock is never leaked."""
@@ -336,13 +254,13 @@ def _version_file() -> Path:
return _infra_dir() / "booted-version"
def _expected_version() -> str:
def expected_version() -> str:
"""The combined marker for the running pair: both per-plane artifact
versions, so a change to either rootfs dislodges the adopted pair."""
return " ".join(f"{role}={_role_version(role)}" for role in infra_artifact.ROLES)
def _adoptable(key: Path, url: str, want: str) -> bool:
def adoptable(key: Path, url: str, want: str) -> bool:
"""Adopt the running pair only if it booted from the CURRENT version, the
orchestrator's control plane is healthy, and the gateway VM is still alive.
A missing/mismatched marker means a prior launcher booted an older infra
@@ -361,7 +279,7 @@ def _adoptable(key: Path, url: str, want: str) -> bool:
return _pidfile_alive(_gw_dir())
def _record_booted_version(version: str) -> None:
def record_booted_version(version: str) -> None:
_version_file().write_text(version + "\n", encoding="utf-8")
@@ -7,12 +7,11 @@ volume (/dev/vdb — the sole opener of `bot-bottle.db`), seeding the host-canon
signing key over SSH, and waiting for `/health`. The host CLI reaches it at its
guest IP, and so does the gateway (`bb_orch` cmdline), so `url()` == `gateway_url()`.
What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM
substrate the gateway VM also uses — booting a VM from a per-plane rootfs
(`boot_vm`), the stable SSH keypair, the secret-push retry, the PID lifecycle —
plus the pair coordinator (`ensure_running`: adopt-or-boot-both under a singleton
lock with a combined version marker) and the per-plane guest inits (`role_init`).
Those are the shared seam.
The plane-agnostic VM substrate the gateway VM also uses stays in `infra_vm` —
booting a VM from a per-plane rootfs (`boot_vm`), the stable SSH keypair, the
secret-push retry, the PID lifecycle, the adoption/version helpers, and the
per-plane guest inits (`role_init`). The pair coordinator (adopt-or-boot-both
under a singleton flock) is `FirecrackerInfraService` (`infra.py`).
"""
from __future__ import annotations
@@ -90,7 +89,7 @@ class FirecrackerOrchestrator(Orchestrator):
def stop(self) -> None:
"""Stop only the orchestrator VM (idempotent). A dead orchestrator fails
`infra_vm._adoptable`'s health check, so no version marker to clear."""
`infra_vm.adoptable`'s health check, so no version marker to clear."""
infra_vm._kill_pidfile(infra_vm._orch_dir())
infra_vm._pid_file(infra_vm._orch_dir()).unlink(missing_ok=True)
+58
View File
@@ -0,0 +1,58 @@
"""The per-host infra service contract (PRD 0070).
`InfraService` is the backend-neutral composer of the per-host **pair**: the
`Orchestrator` (control plane) + `Gateway` (data plane) services, brought up as
an idempotent per-host singleton. One concrete impl per backend
(`backend/*/infra.py`), mirroring how `Orchestrator` and `Gateway` each have a
per-backend impl.
The two backend-specific accessors (`orchestrator()` / `gateway()`) are the
source of truth for how to reach each plane; the convenience `url()` /
`is_healthy()` just delegate to the orchestrator.
"""
from __future__ import annotations
import abc
from ..gateway import Gateway
from ..orchestrator.lifecycle import DEFAULT_STARTUP_TIMEOUT_SECONDS, Orchestrator
class InfraService(abc.ABC):
"""Compose + bring up the per-host orchestrator + gateway pair.
Backend-neutral: docker/macOS run the pair as two containers, firecracker as
two microVMs. Callers read reach-info off `orchestrator()` / `gateway()`."""
@abc.abstractmethod
def orchestrator(self) -> Orchestrator:
"""The control-plane service. Cheap to reconstruct."""
@abc.abstractmethod
def gateway(self) -> Gateway:
"""The data-plane service. Cheap to reconstruct."""
@abc.abstractmethod
def ensure_running(
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
) -> str:
"""Bring the orchestrator + gateway pair up (idempotent per-host
singleton — a healthy, current pair is left running). Returns the host
control-plane URL. Raises `OrchestratorStartError` on control-plane
startup timeout."""
@abc.abstractmethod
def stop(self) -> None:
"""Remove both the orchestrator and the gateway. Idempotent."""
def url(self) -> str:
"""The host-facing control-plane URL the CLI reaches (the orchestrator's)."""
return self.orchestrator().url()
def is_healthy(self) -> bool:
"""True iff the control plane answers `/health`."""
return self.orchestrator().is_healthy()
__all__ = ["InfraService"]
@@ -106,7 +106,7 @@ class MacosContainerBottleBackend(
(`supervise`) call when no control plane is running yet. Mirrors
firecracker's infra bring-up."""
from .infra import MacosInfraService
return MacosInfraService().ensure_running().orchestrator_url
return MacosInfraService().ensure_running()
def prepare_cleanup(self) -> MacosContainerBottleCleanupPlan:
return _cleanup.prepare_cleanup()
@@ -87,10 +87,10 @@ def ensure_gateway(
launches share it. Call before starting the agent container: the agent's
proxy env needs `gateway_ip` at run time."""
service = service or MacosInfraService()
infra = service.ensure_running()
orchestrator_url = service.ensure_running()
endpoint = GatewayEndpoint(
orchestrator_url=infra.orchestrator_url,
gateway_ip=infra.gateway_ip,
orchestrator_url=orchestrator_url,
gateway_ip=service.gateway().address(),
gateway_ca_pem=service.ca_cert_pem(),
network=service.network,
)
+9 -27
View File
@@ -22,7 +22,6 @@ so they have no route to the control plane (the L3 block, not just the JWT).
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from ...orchestrator.lifecycle import (
@@ -30,6 +29,7 @@ from ...orchestrator.lifecycle import (
DEFAULT_STARTUP_TIMEOUT_SECONDS,
OrchestratorStartError,
)
from ..infra_service import InfraService
from . import util as container_mod
from .gateway import (
CONTROL_NETWORK,
@@ -59,19 +59,10 @@ INFRA_DB_VOLUME = ORCHESTRATOR_DB_VOLUME
_REPO_ROOT = Path(__file__).resolve().parents[3]
@dataclass(frozen=True)
class InfraEndpoint:
"""How to reach the running pair. `orchestrator_url` is the orchestrator's
control-network address (host CLI + registration); `gateway_ip` is the
gateway's agent-network address (proxy / git-http / MCP target)."""
orchestrator_url: str
gateway_ip: str
class MacosInfraService:
class MacosInfraService(InfraService):
"""Composes the per-host orchestrator + gateway containers. Callers use
`ensure_running()` (returns the endpoint) and `ca_cert_pem()`."""
`ensure_running()` (returns the control-plane URL), the `orchestrator()` /
`gateway()` accessors, and `ca_cert_pem()`."""
def __init__(
self,
@@ -129,20 +120,12 @@ class MacosInfraService:
repo_root=self._repo_root,
)
def _resolve_gateway_ip(self) -> str:
"""The gateway's agent-network address, or "" while it has none."""
return container_mod.try_container_ipv4_on_network(
self._gateway_name, self.network)
def is_healthy(self) -> bool:
return self.orchestrator().is_healthy()
def ensure_running(
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
) -> InfraEndpoint:
"""Ensure the orchestrator + gateway containers are up; return how to
reach them. Idempotent per-host singleton — a healthy orchestrator on
current source is left untouched. Raises `OrchestratorStartError` on
) -> str:
"""Ensure the orchestrator + gateway containers are up; return the host
control-plane URL. Idempotent per-host singleton — a healthy orchestrator
on current source is left untouched. Raises `OrchestratorStartError` on
control-plane startup timeout."""
# The networks (host-only agent + NAT egress + host-only control) are a
# shared concern — create them before either plane comes up.
@@ -162,7 +145,7 @@ class MacosInfraService:
# (#469).
url = orchestrator.url()
gateway.connect_to_orchestrator(url, orchestrator.mint_gateway_token())
return InfraEndpoint(orchestrator_url=url, gateway_ip=self._resolve_gateway_ip())
return url
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
@@ -178,7 +161,6 @@ class MacosInfraService:
__all__ = [
"MacosInfraService",
"InfraEndpoint",
"OrchestratorStartError",
"GatewayError",
"ORCHESTRATOR_NAME",