didericis/bot-bottle
1
0
Fork 0
Code Issues 34 Pull Requests 9 Actions 6 Packages Releases Wiki Activity

refactor(firecracker): orchestrator under the Orchestrator service ABC

Browse Source 
Pull the control plane's host-side logic out of `infra_vm` into
`FirecrackerOrchestrator` (backend/firecracker/orchestrator.py): booting the
orchestrator microVM on its link with the persistent registry volume (/dev/vdb),
seeding the host-canonical signing key over SSH, waiting for /health, and the
buildah SSH target. `url()` == `gateway_url()` (the gateway resolves the
orchestrator by guest IP via bb_orch). This completes the trio — docker, macOS,
firecracker — behind both the Gateway and Orchestrator ABCs.

`infra_vm` stays the pair coordinator + shared substrate: `ensure_running` now
mints nothing itself — it constructs both services (lazy import to break the
cycle), calls `orchestrator.ensure_running()` first, then
`gateway.connect_to_orchestrator(orch.gateway_url(), orch.mint_gateway_token())`.
The plane-agnostic boot primitives graduate to public API (`_boot_vm`/
`_push_secret` -> `boot_vm`/`push_secret`) now that they're consumed only across
modules; `InfraVm` loses its `orchestrator_url` (moved to the service) and
`InfraEndpoint.orchestrator` is now the `FirecrackerOrchestrator` service.

Container-lifecycle tests split into test_firecracker_orchestrator; the infra_vm
tests keep the substrate + pair-coordinator coverage.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
didericis
2026-07-25 14:54:28 -04:00
parent 5465379654
commit 8d583789d2
7 changed files with 386 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bot_bottle/backend/firecracker/gateway.py
+3 -3
View File
@@ -9,7 +9,7 @@ only the token the host hands it via `connect_to_orchestrator`.
What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM
substrate the orchestrator VM shares — booting a VM from the shared rootfs substrate the orchestrator VM shares — booting a VM from the shared rootfs
(`_boot_vm`), the stable SSH keypair, the secret-push retry loop, and the PID (`boot_vm`), the stable SSH keypair, the secret-push retry loop, and the PID
lifecycle — plus the pair coordinator (`ensure_running`: orchestrator-first lifecycle — plus the pair coordinator (`ensure_running`: orchestrator-first
health gate, singleton lock, adoption/version marker). The guest-side gateway health gate, singleton lock, adoption/version marker). The guest-side gateway
daemon startup lives in the shared `bb_role=gateway` init branch baked into the daemon startup lives in the shared `bb_role=gateway` init branch baked into the
@@ -96,12 +96,12 @@ class FirecrackerGateway(Gateway):
) )
# Boot on the gateway link from the shared rootfs (bb_role=gateway), then # Boot on the gateway link from the shared rootfs (bb_role=gateway), then
# push the token the init waits for before starting the data plane. # push the token the init waits for before starting the data plane.
vm = infra_vm._boot_vm( vm = infra_vm.boot_vm(
name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(), name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(),
role="gateway", mem_mib=_GW_MEM_MIB, role="gateway", mem_mib=_GW_MEM_MIB,
extra_boot_args=f"bb_orch={orchestrator_guest_ip}", extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
) )
infra_vm._push_secret( infra_vm.push_secret(
vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH, vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH,
"the gateway JWT to the gateway VM (its data plane will not start)", "the gateway JWT to the gateway VM (its data plane will not start)",
) )
bot_bottle/backend/firecracker/image_builder.py
+1 -1
View File
@@ -126,7 +126,7 @@ def _build_in_infra(
and stream its rootfs into `base`. The infra VMs persist; only the and stream its rootfs into `base`. The infra VMs persist; only the
per-build container/image/context are cleaned up.""" per-build container/image/context are cleaned up."""
orchestrator = infra_vm.ensure_running().orchestrator orchestrator = infra_vm.ensure_running().orchestrator
key, ip = orchestrator.private_key, orchestrator.guest_ip key, ip = orchestrator.ssh_target()
tag = f"bot-bottle-agent-build-{digest}" tag = f"bot-bottle-agent-build-{digest}"
ctx = f"/tmp/agent-build-{digest}" ctx = f"/tmp/agent-build-{digest}"
smoke_ctr, export_ctr = f"{tag}-smoke", f"{tag}-export" smoke_ctr, export_ctr = f"{tag}-smoke", f"{tag}-export"
bot_bottle/backend/firecracker/infra_vm.py
+28 -122
View File
@@ -43,13 +43,12 @@ from pathlib import Path
from typing import TYPE_CHECKING, Generator from typing import TYPE_CHECKING, Generator
from ...log import die, info from ...log import die, info
from ...orchestrator_auth import ROLE_GATEWAY, mint
from ...paths import host_orchestrator_token
from ..docker import util as docker_mod from ..docker import util as docker_mod
from . import firecracker_vm, infra_artifact, netpool, util from . import firecracker_vm, infra_artifact, netpool, util
if TYPE_CHECKING: if TYPE_CHECKING:
from .gateway import FirecrackerGateway from .gateway import FirecrackerGateway
from .orchestrator import FirecrackerOrchestrator
# The orchestrator VM's signing-key path on its persistent /dev/vdb volume # The orchestrator VM's signing-key path on its persistent /dev/vdb volume
# (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds this # (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds this
@@ -76,19 +75,12 @@ ORCHESTRATOR_PORT = 8099
EGRESS_PORT = 9099 EGRESS_PORT = 9099
SUPERVISE_PORT = 9100 SUPERVISE_PORT = 9100
GIT_HTTP_PORT = 9420 GIT_HTTP_PORT = 9420
# Memory ceiling (fixed at boot, demand-paged — no balloon reclaim). The
# orchestrator keeps the build headroom (buildah's 2-4 GB working set during
# in-VM agent builds); the gateway's slimmer ceiling is set by the gateway
# service — PRD 0070 "Memory: fixed ceilings".
_ORCH_MEM_MIB = 4096
# The infra VMs make direct upstream connections (gateway egress, and buildah # The infra VMs make direct upstream connections (gateway egress, and buildah
# during builds), and the kernel `ip=` cmdline sets no resolver. Public for # during builds), and the kernel `ip=` cmdline sets no resolver. Public for
# now; routing DNS through a filtered path is a later refinement. # now; routing DNS through a filtered path is a later refinement.
_INFRA_RESOLVER = "1.1.1.1" _INFRA_RESOLVER = "1.1.1.1"
_HEALTH_TIMEOUT_SECONDS = 45.0
_HEALTH_POLL_SECONDS = 0.5
_CA_TIMEOUT_SECONDS = 30.0 _CA_TIMEOUT_SECONDS = 30.0
# How long the launcher retries pushing a secret while the guest's SSH comes # How long the launcher retries pushing a secret while the guest's SSH comes
# up. Below the init's own wait window, so a failed push dies here first. # up. Below the init's own wait window, so a failed push dies here first.
@@ -107,24 +99,20 @@ class InfraVm:
private_key: Path private_key: Path
vm: firecracker_vm.VmHandle | None = None vm: firecracker_vm.VmHandle | None = None
@property
def orchestrator_url(self) -> str:
return f"http://{self.guest_ip}:{ORCHESTRATOR_PORT}"
@dataclass @dataclass
class InfraEndpoint: class InfraEndpoint:
"""How to reach the running per-host pair. `orchestrator` is the control """How to reach the running per-host pair. `orchestrator` is the control-plane
plane (host CLI + registration + in-VM builds); `gateway` is the data-plane `Orchestrator` service (host CLI + registration + in-VM builds); `gateway` is
`Gateway` service (agent egress / git-http / supervise, CA fetch, git-gate the data-plane `Gateway` service (agent egress / git-http / supervise, CA
provisioning). Mirrors the docker/macOS `InfraEndpoint` shape.""" fetch, git-gate provisioning). Mirrors the docker/macOS `InfraEndpoint`."""
orchestrator: InfraVm orchestrator: "FirecrackerOrchestrator"
gateway: "FirecrackerGateway" gateway: "FirecrackerGateway"
@property @property
def orchestrator_url(self) -> str: def orchestrator_url(self) -> str:
return self.orchestrator.orchestrator_url return self.orchestrator.url()
def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str: def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str:
return self.gateway.ca_cert_pem(timeout=timeout) return self.gateway.ca_cert_pem(timeout=timeout)
@@ -186,8 +174,14 @@ def ensure_running() -> InfraEndpoint:
Concurrency-safe: the cold stop/build/boot path is serialized by a host Concurrency-safe: the cold stop/build/boot path is serialized by a host
flock, so two simultaneous first launches don't both boot on the same flock, so two simultaneous first launches don't both boot on the same
links/PIDs. The healthy fast-path takes no lock.""" links/PIDs. The healthy fast-path takes no lock."""
# Local imports: the orchestrator + gateway services import this module for
# the shared VM substrate, so importing them at module scope would cycle.
from .orchestrator import FirecrackerOrchestrator
from .gateway import FirecrackerGateway
key = _infra_dir() / "id_ed25519" key = _infra_dir() / "id_ed25519"
url = f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}" orchestrator = FirecrackerOrchestrator()
url = orchestrator.url()
want = _expected_version() want = _expected_version()
if _adoptable(key, url, want): if _adoptable(key, url, want):
info(f"adopting running infra VMs (orchestrator at {url})") info(f"adopting running infra VMs (orchestrator at {url})")
@@ -202,29 +196,26 @@ def ensure_running() -> InfraEndpoint:
# Clear stale/hung/OUTDATED VMs holding either link before booting fresh. # Clear stale/hung/OUTDATED VMs holding either link before booting fresh.
stop() stop()
ensure_built() ensure_built()
orchestrator = boot_orchestrator() # Orchestrator first — the gateway daemons reach the control plane at
wait_for_health(orchestrator) # startup. Each service owns its own boot; the orchestrator (which holds
# The orchestrator holds the signing key; mint the role-scoped `gateway` # the signing key) mints the role-scoped `gateway` JWT for the gateway,
# JWT on the host and hand it to the gateway service, which never sees # which never sees the key (#469).
# the key (#469). The gateway owns its own boot — see FirecrackerGateway. orchestrator.ensure_running()
# Local import: the gateway service imports this module for the shared VM
# substrate, so importing it at module scope would cycle.
from .gateway import FirecrackerGateway
gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token())
gateway = FirecrackerGateway() gateway = FirecrackerGateway()
gateway.connect_to_orchestrator(orchestrator.orchestrator_url, gateway_token) gateway.connect_to_orchestrator(
orchestrator.gateway_url(), orchestrator.mint_gateway_token())
_record_booted_version(want) _record_booted_version(want)
return InfraEndpoint(orchestrator=orchestrator, gateway=gateway) return InfraEndpoint(orchestrator=orchestrator, gateway=gateway)
def _adopt(key: Path) -> InfraEndpoint: def _adopt(key: Path) -> "InfraEndpoint":
"""Build handles to the already-running pair from the stable key + the """Build the service handles for the already-running pair from the stable key
fixed links (no live VM handle — teardown / CA fetch go through the PID + the fixed links (no live VM handle — teardown / CA fetch go through the PID
files + stable key).""" files + stable key)."""
from .orchestrator import FirecrackerOrchestrator
from .gateway import FirecrackerGateway from .gateway import FirecrackerGateway
return InfraEndpoint( return InfraEndpoint(
orchestrator=InfraVm( orchestrator=FirecrackerOrchestrator(),
guest_ip=netpool.orch_slot().guest_ip, private_key=key),
gateway=FirecrackerGateway( gateway=FirecrackerGateway(
InfraVm(guest_ip=netpool.gw_slot().guest_ip, private_key=key)), InfraVm(guest_ip=netpool.gw_slot().guest_ip, private_key=key)),
) )
@@ -259,24 +250,7 @@ def stop() -> None:
_version_file().unlink(missing_ok=True) _version_file().unlink(missing_ok=True)
def boot_orchestrator() -> InfraVm: def boot_vm(
"""Boot the orchestrator (control-plane) VM (detached, so it outlives the
launcher) on the orchestrator link, and seed its signing key. Prefer
`ensure_running`."""
slot = netpool.orch_slot()
orch = _boot_vm(
name="bot-bottle-orchestrator", slot=slot, run_dir=_orch_dir(),
role="orchestrator", mem_mib=_ORCH_MEM_MIB,
data_drive=_ensure_registry_volume(),
)
# Push the host-canonical signing key over SSH (the init waits for it
# before starting the control plane). The host token file stays the single
# source of truth, so a co-running Docker/macOS control plane keeps working.
_push_signing_key(orch)
return orch
def _boot_vm(
*, *,
name: str, name: str,
slot: netpool.Slot, slot: netpool.Slot,
@@ -376,50 +350,7 @@ def _record_booted_version(version: str) -> None:
_version_file().write_text(version + "\n", encoding="utf-8") _version_file().write_text(version + "\n", encoding="utf-8")
# The registry "volume": a host-side ext4 file attached to the orchestrator VM def push_secret(infra: InfraVm, secret: str, dest: str, what: str) -> None:
# as a second virtio-block device (guest /dev/vdb), mounted at the control
# plane's DB dir. It outlives the ephemeral rootfs, so the bottle registry
# survives an orchestrator-VM restart — the firecracker analogue of a docker
# volume. It is a plain ext4 file: `sudo mount -o loop <path>` on the host
# (with the VM stopped) to inspect bot-bottle.db directly. The gateway VM has
# no such volume — the data plane never opens the DB (#469).
_REGISTRY_SIZE = "512M"
def registry_volume_path() -> Path:
return _infra_dir() / "registry.ext4"
def _ensure_registry_volume() -> Path:
"""Create the empty ext4 registry volume on first use; reuse it after."""
vol = registry_volume_path()
if vol.exists():
return vol
info(f"creating infra registry volume {vol} ({_REGISTRY_SIZE})")
proc = subprocess.run(
["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _REGISTRY_SIZE],
capture_output=True, text=True, check=False,
)
if proc.returncode != 0:
vol.unlink(missing_ok=True)
die(f"creating registry volume failed: {proc.stderr.strip()}")
return vol
def _push_signing_key(infra: InfraVm) -> None:
"""Push the host-canonical signing key into the freshly booted orchestrator
VM over SSH (atomic write), so its control plane verifies tokens with the
same key the host CLI signs from. Retries until the guest's SSH is up (it
comes up before the init waits for this file); dies if it never lands,
since the VM then refuses to start its control plane rather than run OPEN."""
_push_secret(
infra, host_orchestrator_token(), _GUEST_SIGNING_KEY_PATH,
"the control-plane signing key to the orchestrator VM "
"(its control plane will not start)",
)
def _push_secret(infra: InfraVm, secret: str, dest: str, what: str) -> None:
"""Pipe `secret` to an atomic write of `dest` in the guest over SSH, """Pipe `secret` to an atomic write of `dest` in the guest over SSH,
retrying while the guest's SSH comes up; die (naming `what`) if it never retrying while the guest's SSH comes up; die (naming `what`) if it never
succeeds. Bare-pipe input keeps the value off argv.""" succeeds. Bare-pipe input keeps the value off argv."""
@@ -525,31 +456,6 @@ def _health_ok(url: str) -> bool:
return False return False
def wait_for_health(
infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS,
) -> None:
"""Poll the orchestrator's /health until it answers 200 or the deadline
passes. Dies (with the console tail) if the VMM exits early."""
url = f"{infra.orchestrator_url}/health"
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if infra.vm is not None and not infra.vm.is_alive():
die(f"orchestrator VM exited during boot (rc={infra.vm.process.returncode}).\n"
f"{firecracker_vm._console_tail(infra.vm.console_log)}")
try:
with urllib.request.urlopen(url, timeout=1.0) as resp:
if resp.status == 200:
info(f"orchestrator control plane healthy at {infra.orchestrator_url}")
return
except (urllib.error.URLError, TimeoutError, OSError):
pass
time.sleep(_HEALTH_POLL_SECONDS)
tail = (firecracker_vm._console_tail(infra.vm.console_log)
if infra.vm is not None else "")
die(f"orchestrator control plane at {url} did not become healthy within "
f"{timeout:.0f}s.\n{tail}")
def _infra_init() -> str: def _infra_init() -> str:
"""PID-1 init for the infra VMs. Shared setup (pseudo-filesystems, PATH, """PID-1 init for the infra VMs. Shared setup (pseudo-filesystems, PATH,
resolver, debug SSH), then a `bb_role` cmdline branch selecting the plane: resolver, debug SSH), then a `bb_role` cmdline branch selecting the plane:
bot_bottle/backend/firecracker/orchestrator.py
+159
View File
@@ -0,0 +1,159 @@
"""The Firecracker orchestrator (control plane) as a microVM (PRD 0070).
`FirecrackerOrchestrator` is the Firecracker implementation of the backend-neutral
`Orchestrator` service, and owns the control plane's host-side logic directly:
booting the orchestrator microVM on its NAT'd link with the persistent registry
volume (/dev/vdb — the sole opener of `bot-bottle.db`), seeding the host-canonical
signing key over SSH, and waiting for `/health`. The host CLI reaches it at its
guest IP, and so does the gateway (`bb_orch` cmdline), so `url()` == `gateway_url()`.
What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM
substrate the gateway VM also shares — booting a VM from the shared rootfs
(`boot_vm`), the stable SSH keypair, the secret-push retry, the PID lifecycle —
plus the pair coordinator (`ensure_running`: adopt-or-boot-both under a singleton
lock with a shared version marker) and the single shared `bb_role`-branched init
baked into the one published rootfs. Those are the shared seam.
"""
from __future__ import annotations
import subprocess
import time
from pathlib import Path
from ...log import die, info
from ...paths import host_orchestrator_token
from ...orchestrator.lifecycle import (
DEFAULT_STARTUP_TIMEOUT_SECONDS,
Orchestrator,
)
from . import firecracker_vm, infra_vm, netpool
from .infra_vm import ORCHESTRATOR_PORT
# The orchestrator microVM's name (the `bb_role=orchestrator` boot tag / run
# dir). Fixed per host — one control-plane VM.
ORCHESTRATOR_NAME = "bot-bottle-orchestrator"
# Memory ceiling (fixed at boot, demand-paged). The orchestrator keeps the build
# headroom (buildah's 2-4 GB working set during in-VM agent builds) — PRD 0070
# "Memory: fixed ceilings".
_ORCH_MEM_MIB = 4096
_HEALTH_POLL_SECONDS = 0.5
# The registry "volume": a host-side ext4 file attached to the orchestrator VM as
# a second virtio-block device (guest /dev/vdb), mounted at the control plane's DB
# dir. It outlives the ephemeral rootfs, so the bottle registry survives an
# orchestrator-VM restart — the firecracker analogue of a docker volume. A plain
# ext4 file: `sudo mount -o loop <path>` on the host (VM stopped) to inspect
# bot-bottle.db. The gateway VM has no such volume — the data plane never opens
# the DB (#469).
_REGISTRY_SIZE = "512M"
def registry_volume_path() -> Path:
return infra_vm._infra_dir() / "registry.ext4"
class FirecrackerOrchestrator(Orchestrator):
"""The control plane as a Firecracker microVM on the orchestrator link.
The shared rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's
`ensure_built` no-op here); `ensure_running` boots the VM, seeds the signing
key, and blocks until `/health` answers."""
name = ORCHESTRATOR_NAME
def __init__(self, vm: infra_vm.InfraVm | None = None) -> None:
# The live VM handle when this process booted it; None when adapting an
# already-running orchestrator (health/URL go through the fixed link).
self._vm = vm
def url(self) -> str:
"""The orchestrator VM's guest IP URL — where the host CLI reaches the
control plane."""
return f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}"
def gateway_url(self) -> str:
"""Same guest IP the host uses — the gateway resolves the orchestrator at
`bb_orch=<guest_ip>` off its cmdline."""
return self.url()
def ssh_target(self) -> tuple[Path, str]:
"""(private key, guest IP) for SSHing into the orchestrator VM. In-VM
agent-image builds (buildah) run here — the build host drives them over
SSH with the stable infra key."""
return infra_vm._infra_dir() / "id_ed25519", netpool.orch_slot().guest_ip
def is_running(self) -> bool:
return infra_vm._pidfile_alive(infra_vm._orch_dir())
def stop(self) -> None:
"""Stop only the orchestrator VM (idempotent). A dead orchestrator fails
`infra_vm._adoptable`'s health check, so no version marker to clear."""
infra_vm._kill_pidfile(infra_vm._orch_dir())
infra_vm._pid_file(infra_vm._orch_dir()).unlink(missing_ok=True)
def ensure_running(
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
) -> None:
"""Boot the orchestrator VM on its link with the persistent registry
volume, seed the signing key over SSH, and block until `/health` answers.
Idempotent — a live, healthy control plane is left alone (the pair
coordinator otherwise `stop()`s it before calling, so this boots fresh)."""
if self.is_running() and self.is_healthy():
return
vm = infra_vm.boot_vm(
name=ORCHESTRATOR_NAME, slot=netpool.orch_slot(),
run_dir=infra_vm._orch_dir(), role="orchestrator", mem_mib=_ORCH_MEM_MIB,
data_drive=self._ensure_registry_volume(),
)
# Push the host-canonical signing key (the init waits for it before
# starting the control plane). The host token file stays the single
# source of truth, so a co-running docker/macOS control plane keeps
# working; the guest verifies tokens with the same key the CLI signs from.
infra_vm.push_secret(
vm, host_orchestrator_token(), infra_vm._GUEST_SIGNING_KEY_PATH,
"the control-plane signing key to the orchestrator VM "
"(its control plane will not start)",
)
self._vm = vm
self._wait_for_health(vm, timeout=startup_timeout)
def _wait_for_health(
self, vm: infra_vm.InfraVm, *, timeout: float,
) -> None:
"""Poll `/health` until it answers 200 or the deadline passes. Dies (with
the console tail) if the VMM exits early."""
url = f"{self.url()}/health"
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if vm.vm is not None and not vm.vm.is_alive():
die(f"orchestrator VM exited during boot (rc={vm.vm.process.returncode}).\n"
f"{firecracker_vm._console_tail(vm.vm.console_log)}")
if self.is_healthy():
info(f"orchestrator control plane healthy at {self.url()}")
return
time.sleep(_HEALTH_POLL_SECONDS)
tail = (firecracker_vm._console_tail(vm.vm.console_log)
if vm.vm is not None else "")
die(f"orchestrator control plane at {url} did not become healthy within "
f"{timeout:.0f}s.\n{tail}")
def _ensure_registry_volume(self) -> Path:
"""Create the empty ext4 registry volume on first use; reuse it after."""
vol = registry_volume_path()
if vol.exists():
return vol
info(f"creating infra registry volume {vol} ({_REGISTRY_SIZE})")
proc = subprocess.run(
["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _REGISTRY_SIZE],
capture_output=True, text=True, check=False,
)
if proc.returncode != 0:
vol.unlink(missing_ok=True)
die(f"creating registry volume failed: {proc.stderr.strip()}")
return vol
__all__ = ["FirecrackerOrchestrator", "ORCHESTRATOR_NAME", "registry_volume_path"]
tests/unit/test_firecracker_gateway.py
+6 -6
View File
@@ -1,7 +1,7 @@
"""Unit: the Firecracker gateway data plane as a microVM (PRD 0070). """Unit: the Firecracker gateway data plane as a microVM (PRD 0070).
The gateway service owns its host-side logic directly: booting the gateway VM The gateway service owns its host-side logic directly: booting the gateway VM
(via the shared `infra_vm._boot_vm` substrate), seeding the pre-minted token, (via the shared `infra_vm.boot_vm` substrate), seeding the pre-minted token,
fetching the CA over SSH, and the SSH exec/cp provisioning transport. fetching the CA over SSH, and the SSH exec/cp provisioning transport.
""" """
@@ -37,21 +37,21 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
def test_refuses_without_orchestrator_url(self) -> None: def test_refuses_without_orchestrator_url(self) -> None:
gw = FirecrackerGateway() gw = FirecrackerGateway()
with patch.object(infra_vm, "_boot_vm") as boot: with patch.object(infra_vm, "boot_vm") as boot:
with self.assertRaises(GatewayError): with self.assertRaises(GatewayError):
gw.connect_to_orchestrator("", _TOKEN) gw.connect_to_orchestrator("", _TOKEN)
boot.assert_not_called() boot.assert_not_called()
def test_refuses_without_gateway_token(self) -> None: def test_refuses_without_gateway_token(self) -> None:
gw = FirecrackerGateway() gw = FirecrackerGateway()
with patch.object(infra_vm, "_boot_vm") as boot: with patch.object(infra_vm, "boot_vm") as boot:
with self.assertRaises(GatewayError): with self.assertRaises(GatewayError):
gw.connect_to_orchestrator(_ORCH_URL, "") gw.connect_to_orchestrator(_ORCH_URL, "")
boot.assert_not_called() boot.assert_not_called()
def test_refuses_a_url_without_a_host(self) -> None: def test_refuses_a_url_without_a_host(self) -> None:
gw = FirecrackerGateway() gw = FirecrackerGateway()
with patch.object(infra_vm, "_boot_vm") as boot: with patch.object(infra_vm, "boot_vm") as boot:
with self.assertRaises(GatewayError): with self.assertRaises(GatewayError):
gw.connect_to_orchestrator("http://:8099", _TOKEN) gw.connect_to_orchestrator("http://:8099", _TOKEN)
boot.assert_not_called() boot.assert_not_called()
@@ -59,8 +59,8 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None: def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None:
gw = FirecrackerGateway() gw = FirecrackerGateway()
booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k")) booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))
with patch.object(infra_vm, "_boot_vm", return_value=booted) as boot, \ with patch.object(infra_vm, "boot_vm", return_value=booted) as boot, \
patch.object(infra_vm, "_push_secret") as push: patch.object(infra_vm, "push_secret") as push:
gw.connect_to_orchestrator(_ORCH_URL, _TOKEN) gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
# Booted on the gateway link with the gateway role; the orchestrator's # Booted on the gateway link with the gateway role; the orchestrator's
# guest IP (parsed off the URL) rides the cmdline as bb_orch. # guest IP (parsed off the URL) rides the cmdline as bb_orch.
tests/unit/test_firecracker_infra_vm.py
+43 -149
View File
@@ -15,65 +15,51 @@ from unittest.mock import MagicMock, patch
from bot_bottle.backend.firecracker import infra_vm from bot_bottle.backend.firecracker import infra_vm
class TestPushSecrets(unittest.TestCase): class TestPushSecret(unittest.TestCase):
"""`_push_signing_key` (orchestrator) seeds the guest's secret over SSH via """`push_secret` is the shared substrate both services seed a guest secret
`_push_secret`. The host token file stays the single source of truth, never over SSH with (atomic write, retry while SSH comes up). The per-plane pushes
clobbered per-backend (#469). The gateway's own JWT push lives with the live with each service (orchestrator: signing key; gateway: JWT)."""
gateway service (test_firecracker_gateway)."""
def _infra(self) -> infra_vm.InfraVm: def _infra(self) -> infra_vm.InfraVm:
return infra_vm.InfraVm(guest_ip="10.0.0.1", private_key=Path("/k")) return infra_vm.InfraVm(guest_ip="10.0.0.1", private_key=Path("/k"))
def test_signing_key_written_atomically(self): def test_written_atomically(self):
proc = MagicMock(returncode=0, stderr="") proc = MagicMock(returncode=0, stderr="")
with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \ with patch.object(infra_vm.subprocess, "run", return_value=proc) as run:
patch.object(infra_vm.subprocess, "run", return_value=proc) as run: infra_vm.push_secret(self._infra(), "secret", "/dest", "the thing")
infra_vm._push_signing_key(self._infra()) self.assertEqual("secret", run.call_args.kwargs["input"])
# Piped the key to an atomic write of the guest token path.
self.assertEqual("host-key", run.call_args.kwargs["input"])
remote_cmd = run.call_args.args[0][-1] remote_cmd = run.call_args.args[0][-1]
self.assertIn(f"cat > {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp", remote_cmd) self.assertIn("cat > /dest.tmp", remote_cmd)
self.assertIn(f"mv {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp " self.assertIn("mv /dest.tmp /dest", remote_cmd)
f"{infra_vm._GUEST_SIGNING_KEY_PATH}", remote_cmd)
def test_dies_when_push_never_succeeds(self): def test_dies_when_push_never_succeeds(self):
proc = MagicMock(returncode=255, stderr="ssh: connect refused") proc = MagicMock(returncode=255, stderr="ssh: connect refused")
with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \ with patch.object(infra_vm, "_SECRET_PUSH_TIMEOUT_SECONDS", 0.05), \
patch.object(infra_vm, "_SECRET_PUSH_TIMEOUT_SECONDS", 0.05), \
patch.object(infra_vm, "_SECRET_PUSH_POLL_SECONDS", 0.0), \ patch.object(infra_vm, "_SECRET_PUSH_POLL_SECONDS", 0.0), \
patch.object(infra_vm.subprocess, "run", return_value=proc), \ patch.object(infra_vm.subprocess, "run", return_value=proc), \
patch.object(infra_vm, "die", side_effect=SystemExit) as die: patch.object(infra_vm, "die", side_effect=SystemExit) as die:
with self.assertRaises(SystemExit): with self.assertRaises(SystemExit):
infra_vm._push_signing_key(self._infra()) infra_vm.push_secret(self._infra(), "secret", "/dest", "the thing")
die.assert_called_once() die.assert_called_once()
class TestOrchestratorUrl(unittest.TestCase):
def test_url_uses_guest_ip_and_port(self):
infra = infra_vm.InfraVm(
vm=MagicMock(), guest_ip="10.243.255.1", private_key=Path("/k"))
self.assertEqual(
f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}",
infra.orchestrator_url,
)
class TestInfraEndpoint(unittest.TestCase): class TestInfraEndpoint(unittest.TestCase):
"""The endpoint mirrors docker/macOS: it exposes the orchestrator's URL and """The endpoint mirrors docker/macOS: it exposes the orchestrator service's
delegates the CA fetch to the gateway service.""" URL and delegates the CA fetch to the gateway service."""
def _endpoint(self) -> infra_vm.InfraEndpoint: def _endpoint(self) -> infra_vm.InfraEndpoint:
from bot_bottle.backend.firecracker.gateway import FirecrackerGateway from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator
return infra_vm.InfraEndpoint( return infra_vm.InfraEndpoint(
orchestrator=infra_vm.InfraVm(guest_ip="10.243.255.1", private_key=Path("/k")), orchestrator=FirecrackerOrchestrator(),
gateway=FirecrackerGateway( gateway=FirecrackerGateway(
infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))), infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))),
) )
def test_orchestrator_url_is_the_orchestrator_vm(self): def test_orchestrator_url_is_the_orchestrator_service(self):
ep = self._endpoint() ep = self._endpoint()
self.assertEqual( ip = infra_vm.netpool.orch_slot().guest_ip
f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}", ep.orchestrator_url) self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", ep.orchestrator_url)
def test_gateway_ca_pem_delegates_to_the_gateway_service(self): def test_gateway_ca_pem_delegates_to_the_gateway_service(self):
ep = self._endpoint() ep = self._endpoint()
@@ -122,64 +108,6 @@ class TestBuildInfraRootfs(unittest.TestCase):
self.assertIn("BOT_BOTTLE_ORCHESTRATOR_URL=http://$ORCH:", init) self.assertIn("BOT_BOTTLE_ORCHESTRATOR_URL=http://$ORCH:", init)
class TestBootRole(unittest.TestCase):
"""`boot_orchestrator` boots the shared rootfs with the `bb_role=orchestrator`
cmdline, the orchestrator memory ceiling + registry volume, and seeds the
signing key. (The gateway's boot lives in the gateway service —
test_firecracker_gateway.)"""
def _boot_env(self):
vm = MagicMock()
vm.process.pid = 4242
return vm
def test_orchestrator_role_mem_and_registry(self):
import tempfile
vm = self._boot_env()
with tempfile.TemporaryDirectory() as td, \
patch.object(infra_vm.netpool, "tap_present", return_value=True), \
patch.object(infra_vm.infra_artifact, "local_build_requested", return_value=False), \
patch.object(infra_vm.infra_artifact, "materialize_ext4"), \
patch.object(infra_vm.infra_artifact, "infra_artifact_version", return_value="v"), \
patch.object(infra_vm, "_stable_keypair", return_value=(Path("/k"), "pub")), \
patch.object(infra_vm, "_ensure_registry_volume", return_value=Path("/reg")), \
patch.object(infra_vm, "_push_signing_key") as push, \
patch.object(infra_vm.firecracker_vm, "boot", return_value=vm) as boot, \
patch.object(infra_vm, "_orch_dir", return_value=Path(td)):
infra_vm.boot_orchestrator()
kw = boot.call_args.kwargs
self.assertIn("bb_role=orchestrator", kw["extra_boot_args"])
self.assertEqual(infra_vm._ORCH_MEM_MIB, kw["mem_mib"])
self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP
push.assert_called_once() # signing key seeded
class TestRegistryVolume(unittest.TestCase):
def test_reuses_existing_volume(self):
import tempfile
with tempfile.TemporaryDirectory() as td:
vol = Path(td) / "registry.ext4"
vol.write_bytes(b"") # already present
with patch.object(infra_vm, "registry_volume_path", return_value=vol), \
patch.object(infra_vm.subprocess, "run") as run:
out = infra_vm._ensure_registry_volume()
run.assert_not_called() # no mke2fs when it exists
self.assertEqual(vol, out)
def test_creates_volume_when_missing(self):
import tempfile
from subprocess import CompletedProcess
with tempfile.TemporaryDirectory() as td:
vol = Path(td) / "registry.ext4"
with patch.object(infra_vm, "registry_volume_path", return_value=vol), \
patch.object(infra_vm.subprocess, "run",
return_value=CompletedProcess([], 0)) as run:
infra_vm._ensure_registry_volume()
argv = run.call_args.args[0]
self.assertIn("mke2fs", argv)
self.assertIn(str(vol), argv)
class TestEnsureBuilt(unittest.TestCase): class TestEnsureBuilt(unittest.TestCase):
def test_default_pulls_artifact_without_docker(self): def test_default_pulls_artifact_without_docker(self):
# PRD 0069 Stage 2: the launch host pulls the prebuilt rootfs; no Docker. # PRD 0069 Stage 2: the launch host pulls the prebuilt rootfs; no Docker.
@@ -204,30 +132,10 @@ class TestEnsureBuilt(unittest.TestCase):
self.assertIn(infra_vm._GATEWAY_IMAGE, tags[:-1]) self.assertIn(infra_vm._GATEWAY_IMAGE, tags[:-1])
class TestWaitForHealth(unittest.TestCase): # The orchestrator + gateway services are imported lazily inside ensure_running
def _infra(self, alive: bool = True) -> infra_vm.InfraVm: # / _adopt (to break the infra_vm <-> service import cycle), so patch them at
vm = MagicMock() # their home modules.
vm.is_alive.return_value = alive _ORCH_CLS = "bot_bottle.backend.firecracker.orchestrator.FirecrackerOrchestrator"
return infra_vm.InfraVm(vm=vm, guest_ip="10.0.0.1", private_key=Path("/k"))
def test_returns_on_200(self):
infra = self._infra()
cm = MagicMock()
cm.__enter__.return_value.status = 200
with patch.object(infra_vm.urllib.request, "urlopen", return_value=cm):
infra_vm.wait_for_health(infra, timeout=5) # must not raise
def test_dies_when_vm_exits(self):
infra = self._infra(alive=False)
assert infra.vm is not None # narrow for the type checker (it's a mock)
infra.vm.process.returncode = 1
with patch.object(infra_vm.firecracker_vm, "_console_tail", return_value=""), \
self.assertRaises(SystemExit):
infra_vm.wait_for_health(infra, timeout=5)
# The gateway service is imported lazily inside ensure_running / _adopt (to
# break the infra_vm <-> gateway import cycle), so patch it at its home module.
_GW_CLS = "bot_bottle.backend.firecracker.gateway.FirecrackerGateway" _GW_CLS = "bot_bottle.backend.firecracker.gateway.FirecrackerGateway"
@@ -236,6 +144,7 @@ class TestEnsureRunningSingleton(unittest.TestCase):
# Healthy control plane + live gateway + existing key + matching version # Healthy control plane + live gateway + existing key + matching version
# marker -> adopt (no boot). # marker -> adopt (no boot).
from bot_bottle.backend.firecracker.gateway import FirecrackerGateway from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator
import tempfile import tempfile
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
d = Path(td) d = Path(td)
@@ -244,14 +153,13 @@ class TestEnsureRunningSingleton(unittest.TestCase):
with patch.object(infra_vm, "_infra_dir", return_value=d), \ with patch.object(infra_vm, "_infra_dir", return_value=d), \
patch.object(infra_vm, "_expected_version", return_value="v-current"), \ patch.object(infra_vm, "_expected_version", return_value="v-current"), \
patch.object(infra_vm, "_health_ok", return_value=True), \ patch.object(infra_vm, "_health_ok", return_value=True), \
patch.object(infra_vm, "_pidfile_alive", return_value=True), \ patch.object(infra_vm, "_pidfile_alive", return_value=True):
patch.object(infra_vm, "boot_orchestrator") as boot_o:
ep = infra_vm.ensure_running() ep = infra_vm.ensure_running()
boot_o.assert_not_called() # Adopted service handles addressing the two distinct infra links.
self.assertIsNone(ep.orchestrator.vm) self.assertIsInstance(ep.orchestrator, FirecrackerOrchestrator)
# The orchestrator handle + the adopted gateway service address the two self.assertEqual(
# distinct infra links. infra_vm.netpool.orch_slot().guest_ip,
self.assertEqual(infra_vm.netpool.orch_slot().guest_ip, ep.orchestrator.guest_ip) ep.orchestrator_url.split("://")[1].split(":")[0])
self.assertIsInstance(ep.gateway, FirecrackerGateway) self.assertIsInstance(ep.gateway, FirecrackerGateway)
self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, ep.gateway.address()) self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, ep.gateway.address())
@@ -269,18 +177,15 @@ class TestEnsureRunningSingleton(unittest.TestCase):
patch.object(infra_vm, "_pidfile_alive", return_value=True), \ patch.object(infra_vm, "_pidfile_alive", return_value=True), \
patch.object(infra_vm, "stop") as stop, \ patch.object(infra_vm, "stop") as stop, \
patch.object(infra_vm, "ensure_built"), \ patch.object(infra_vm, "ensure_built"), \
patch.object(infra_vm, "wait_for_health"), \ patch(_ORCH_CLS) as orch_cls, \
patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \
patch.object(infra_vm, "mint", return_value="gw.jwt"), \
patch.object(infra_vm, "boot_orchestrator") as boot_o, \
patch(_GW_CLS) as gw_cls: patch(_GW_CLS) as gw_cls:
boot_o.return_value = infra_vm.InfraVm( orch_cls.return_value.gateway_url.return_value = "http://10.243.255.1:8099"
guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) orch_cls.return_value.mint_gateway_token.return_value = "gw.jwt"
infra_vm.ensure_running() infra_vm.ensure_running()
stop.assert_called_once() # dislodge the outdated pair stop.assert_called_once() # dislodge the outdated pair
boot_o.assert_called_once() orch_cls.return_value.ensure_running.assert_called_once()
# The gateway service is bound to the orchestrator's URL, carrying # The gateway service is bound to the orchestrator's gateway URL,
# the host-minted `gateway` token. # carrying the orchestrator-minted `gateway` token.
gw_cls.return_value.connect_to_orchestrator.assert_called_once_with( gw_cls.return_value.connect_to_orchestrator.assert_called_once_with(
"http://10.243.255.1:8099", "gw.jwt") "http://10.243.255.1:8099", "gw.jwt")
# The fresh boot records the current version for the next launcher. # The fresh boot records the current version for the next launcher.
@@ -300,16 +205,11 @@ class TestEnsureRunningSingleton(unittest.TestCase):
patch.object(infra_vm, "_pidfile_alive", return_value=False), \ patch.object(infra_vm, "_pidfile_alive", return_value=False), \
patch.object(infra_vm, "stop") as stop, \ patch.object(infra_vm, "stop") as stop, \
patch.object(infra_vm, "ensure_built"), \ patch.object(infra_vm, "ensure_built"), \
patch.object(infra_vm, "wait_for_health"), \ patch(_ORCH_CLS) as orch_cls, \
patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \
patch.object(infra_vm, "mint", return_value="gw.jwt"), \
patch.object(infra_vm, "boot_orchestrator") as boot_o, \
patch(_GW_CLS) as gw_cls: patch(_GW_CLS) as gw_cls:
boot_o.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
infra_vm.ensure_running() infra_vm.ensure_running()
stop.assert_called_once() stop.assert_called_once()
boot_o.assert_called_once() orch_cls.return_value.ensure_running.assert_called_once()
gw_cls.return_value.connect_to_orchestrator.assert_called_once() gw_cls.return_value.connect_to_orchestrator.assert_called_once()
def test_boots_both_when_unhealthy(self): def test_boots_both_when_unhealthy(self):
@@ -320,20 +220,14 @@ class TestEnsureRunningSingleton(unittest.TestCase):
patch.object(infra_vm, "_health_ok", return_value=False), \ patch.object(infra_vm, "_health_ok", return_value=False), \
patch.object(infra_vm, "stop") as stop, \ patch.object(infra_vm, "stop") as stop, \
patch.object(infra_vm, "ensure_built") as built, \ patch.object(infra_vm, "ensure_built") as built, \
patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \ patch(_ORCH_CLS) as orch_cls, \
patch.object(infra_vm, "mint", return_value="gw.jwt"), \ patch(_GW_CLS) as gw_cls:
patch.object(infra_vm, "boot_orchestrator") as boot_o, \
patch(_GW_CLS) as gw_cls, \
patch.object(infra_vm, "wait_for_health") as wait:
boot_o.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
infra_vm.ensure_running() infra_vm.ensure_running()
stop.assert_called_once() # clear stale VMs first stop.assert_called_once() # clear stale VMs first
built.assert_called_once() built.assert_called_once()
# Orchestrator boots + becomes healthy BEFORE the gateway is connected # Orchestrator is brought up BEFORE the gateway is connected (its daemons
# (its daemons reach the control plane at startup). # reach the control plane at startup).
boot_o.assert_called_once() orch_cls.return_value.ensure_running.assert_called_once()
wait.assert_called_once()
gw_cls.return_value.connect_to_orchestrator.assert_called_once() gw_cls.return_value.connect_to_orchestrator.assert_called_once()
tests/unit/test_firecracker_orchestrator.py
+146
View File
@@ -0,0 +1,146 @@
"""Unit: the Firecracker orchestrator (control plane) microVM lifecycle (PRD 0070).
The control plane's host-side logic — boot on the orchestrator link with the
registry volume, seed the signing key, wait for /health — lives in
`FirecrackerOrchestrator`, composing the shared `infra_vm` VM substrate.
"""
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from unittest.mock import MagicMock, patch
from bot_bottle.backend.firecracker import infra_vm
from bot_bottle.backend.firecracker.orchestrator import (
ORCHESTRATOR_NAME,
FirecrackerOrchestrator,
registry_volume_path,
)
from bot_bottle.backend.firecracker.infra_vm import ORCHESTRATOR_PORT
_ORCH = "bot_bottle.backend.firecracker.orchestrator"
class TestUrls(unittest.TestCase):
def test_url_and_gateway_url_are_the_orch_link_guest_ip(self) -> None:
orch = FirecrackerOrchestrator()
ip = infra_vm.netpool.orch_slot().guest_ip
self.assertEqual(f"http://{ip}:{ORCHESTRATOR_PORT}", orch.url())
self.assertEqual(orch.url(), orch.gateway_url())
def test_ssh_target_is_stable_key_and_orch_guest_ip(self) -> None:
orch = FirecrackerOrchestrator()
with patch.object(infra_vm, "_infra_dir", return_value=Path("/infra")):
key, ip = orch.ssh_target()
self.assertEqual(Path("/infra/id_ed25519"), key)
self.assertEqual(infra_vm.netpool.orch_slot().guest_ip, ip)
class TestEnsureRunning(unittest.TestCase):
def test_boots_orchestrator_role_with_mem_and_registry_then_pushes_key(self) -> None:
orch = FirecrackerOrchestrator()
vm = infra_vm.InfraVm(guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
with patch.object(infra_vm, "boot_vm", return_value=vm) as boot, \
patch.object(infra_vm, "push_secret") as push, \
patch(f"{_ORCH}.host_orchestrator_token", return_value="host-key"), \
patch.object(orch, "is_running", return_value=False), \
patch.object(orch, "_ensure_registry_volume", return_value=Path("/reg")), \
patch.object(orch, "_wait_for_health"):
orch.ensure_running()
kw = boot.call_args.kwargs
self.assertEqual("orchestrator", kw["role"])
self.assertEqual(4096, kw["mem_mib"]) # orchestrator keeps build headroom
self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP
# The host-canonical signing key is pushed to the guest signing-key path.
self.assertEqual("host-key", push.call_args.args[1])
self.assertEqual(infra_vm._GUEST_SIGNING_KEY_PATH, push.call_args.args[2])
def test_noop_when_running_and_healthy(self) -> None:
orch = FirecrackerOrchestrator()
with patch.object(orch, "is_running", return_value=True), \
patch.object(orch, "is_healthy", return_value=True), \
patch.object(infra_vm, "boot_vm") as boot:
orch.ensure_running()
boot.assert_not_called()
class TestWaitForHealth(unittest.TestCase):
def _vm(self, alive: bool = True) -> infra_vm.InfraVm:
h = MagicMock()
h.is_alive.return_value = alive
return infra_vm.InfraVm(vm=h, guest_ip="10.0.0.1", private_key=Path("/k"))
def test_returns_when_healthy(self) -> None:
orch = FirecrackerOrchestrator()
with patch.object(orch, "is_healthy", return_value=True):
orch._wait_for_health(self._vm(), timeout=5) # must not raise
def test_dies_when_vm_exits(self) -> None:
orch = FirecrackerOrchestrator()
vm = self._vm(alive=False)
assert vm.vm is not None
vm.vm.process.returncode = 1
with patch(f"{_ORCH}.firecracker_vm._console_tail", return_value=""), \
patch(f"{_ORCH}.die", side_effect=SystemExit) as die:
with self.assertRaises(SystemExit):
orch._wait_for_health(vm, timeout=5)
die.assert_called_once()
class TestRegistryVolume(unittest.TestCase):
def test_reuses_existing_volume(self) -> None:
orch = FirecrackerOrchestrator()
with tempfile.TemporaryDirectory() as td:
vol = Path(td) / "registry.ext4"
vol.write_bytes(b"") # already present
with patch(f"{_ORCH}.registry_volume_path", return_value=vol), \
patch(f"{_ORCH}.subprocess.run") as run:
out = orch._ensure_registry_volume()
run.assert_not_called() # no mke2fs when it exists
self.assertEqual(vol, out)
def test_creates_volume_when_missing(self) -> None:
from subprocess import CompletedProcess
orch = FirecrackerOrchestrator()
with tempfile.TemporaryDirectory() as td:
vol = Path(td) / "registry.ext4"
with patch(f"{_ORCH}.registry_volume_path", return_value=vol), \
patch(f"{_ORCH}.subprocess.run",
return_value=CompletedProcess([], 0)) as run:
orch._ensure_registry_volume()
argv = run.call_args.args[0]
self.assertIn("mke2fs", argv)
self.assertIn(str(vol), argv)
def test_registry_volume_path_under_infra_dir(self) -> None:
with patch.object(infra_vm, "_infra_dir", return_value=Path("/infra")):
self.assertEqual(Path("/infra/registry.ext4"), registry_volume_path())
class TestLifecycle(unittest.TestCase):
def test_is_running_reads_the_orch_pidfile(self) -> None:
orch = FirecrackerOrchestrator()
with patch.object(infra_vm, "_pidfile_alive", return_value=True) as alive, \
patch.object(infra_vm, "_orch_dir", return_value=Path("/orch")):
self.assertTrue(orch.is_running())
alive.assert_called_once_with(Path("/orch"))
def test_stop_kills_only_the_orch_pidfile(self) -> None:
orch = FirecrackerOrchestrator()
with tempfile.TemporaryDirectory() as td:
d = Path(td)
(d / "vm.pid").write_text("123")
with patch.object(infra_vm, "_orch_dir", return_value=d), \
patch.object(infra_vm, "_kill_pidfile") as kill:
orch.stop()
kill.assert_called_once_with(d)
self.assertFalse((d / "vm.pid").exists())
def test_name_is_the_orchestrator_vm(self) -> None:
self.assertEqual(ORCHESTRATOR_NAME, FirecrackerOrchestrator().name)
if __name__ == "__main__":
unittest.main()