refactor(firecracker): orchestrator under the Orchestrator service ABC
Pull the control plane's host-side logic out of `infra_vm` into `FirecrackerOrchestrator` (backend/firecracker/orchestrator.py): booting the orchestrator microVM on its link with the persistent registry volume (/dev/vdb), seeding the host-canonical signing key over SSH, waiting for /health, and the buildah SSH target. `url()` == `gateway_url()` (the gateway resolves the orchestrator by guest IP via bb_orch). This completes the trio — docker, macOS, firecracker — behind both the Gateway and Orchestrator ABCs. `infra_vm` stays the pair coordinator + shared substrate: `ensure_running` now mints nothing itself — it constructs both services (lazy import to break the cycle), calls `orchestrator.ensure_running()` first, then `gateway.connect_to_orchestrator(orch.gateway_url(), orch.mint_gateway_token())`. The plane-agnostic boot primitives graduate to public API (`_boot_vm`/ `_push_secret` -> `boot_vm`/`push_secret`) now that they're consumed only across modules; `InfraVm` loses its `orchestrator_url` (moved to the service) and `InfraEndpoint.orchestrator` is now the `FirecrackerOrchestrator` service. Container-lifecycle tests split into test_firecracker_orchestrator; the infra_vm tests keep the substrate + pair-coordinator coverage. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -9,7 +9,7 @@ only the token the host hands it via `connect_to_orchestrator`.
|
||||
|
||||
What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM
|
||||
substrate the orchestrator VM shares — booting a VM from the shared rootfs
|
||||
(`_boot_vm`), the stable SSH keypair, the secret-push retry loop, and the PID
|
||||
(`boot_vm`), the stable SSH keypair, the secret-push retry loop, and the PID
|
||||
lifecycle — plus the pair coordinator (`ensure_running`: orchestrator-first
|
||||
health gate, singleton lock, adoption/version marker). The guest-side gateway
|
||||
daemon startup lives in the shared `bb_role=gateway` init branch baked into the
|
||||
@@ -96,12 +96,12 @@ class FirecrackerGateway(Gateway):
|
||||
)
|
||||
# Boot on the gateway link from the shared rootfs (bb_role=gateway), then
|
||||
# push the token the init waits for before starting the data plane.
|
||||
vm = infra_vm._boot_vm(
|
||||
vm = infra_vm.boot_vm(
|
||||
name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(),
|
||||
role="gateway", mem_mib=_GW_MEM_MIB,
|
||||
extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
|
||||
)
|
||||
infra_vm._push_secret(
|
||||
infra_vm.push_secret(
|
||||
vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH,
|
||||
"the gateway JWT to the gateway VM (its data plane will not start)",
|
||||
)
|
||||
|
||||
@@ -126,7 +126,7 @@ def _build_in_infra(
|
||||
and stream its rootfs into `base`. The infra VMs persist; only the
|
||||
per-build container/image/context are cleaned up."""
|
||||
orchestrator = infra_vm.ensure_running().orchestrator
|
||||
key, ip = orchestrator.private_key, orchestrator.guest_ip
|
||||
key, ip = orchestrator.ssh_target()
|
||||
tag = f"bot-bottle-agent-build-{digest}"
|
||||
ctx = f"/tmp/agent-build-{digest}"
|
||||
smoke_ctr, export_ctr = f"{tag}-smoke", f"{tag}-export"
|
||||
|
||||
@@ -43,13 +43,12 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Generator
|
||||
|
||||
from ...log import die, info
|
||||
from ...orchestrator_auth import ROLE_GATEWAY, mint
|
||||
from ...paths import host_orchestrator_token
|
||||
from ..docker import util as docker_mod
|
||||
from . import firecracker_vm, infra_artifact, netpool, util
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .gateway import FirecrackerGateway
|
||||
from .orchestrator import FirecrackerOrchestrator
|
||||
|
||||
# The orchestrator VM's signing-key path on its persistent /dev/vdb volume
|
||||
# (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds this
|
||||
@@ -76,19 +75,12 @@ ORCHESTRATOR_PORT = 8099
|
||||
EGRESS_PORT = 9099
|
||||
SUPERVISE_PORT = 9100
|
||||
GIT_HTTP_PORT = 9420
|
||||
# Memory ceiling (fixed at boot, demand-paged — no balloon reclaim). The
|
||||
# orchestrator keeps the build headroom (buildah's 2-4 GB working set during
|
||||
# in-VM agent builds); the gateway's slimmer ceiling is set by the gateway
|
||||
# service — PRD 0070 "Memory: fixed ceilings".
|
||||
_ORCH_MEM_MIB = 4096
|
||||
|
||||
# The infra VMs make direct upstream connections (gateway egress, and buildah
|
||||
# during builds), and the kernel `ip=` cmdline sets no resolver. Public for
|
||||
# now; routing DNS through a filtered path is a later refinement.
|
||||
_INFRA_RESOLVER = "1.1.1.1"
|
||||
|
||||
_HEALTH_TIMEOUT_SECONDS = 45.0
|
||||
_HEALTH_POLL_SECONDS = 0.5
|
||||
_CA_TIMEOUT_SECONDS = 30.0
|
||||
# How long the launcher retries pushing a secret while the guest's SSH comes
|
||||
# up. Below the init's own wait window, so a failed push dies here first.
|
||||
@@ -107,24 +99,20 @@ class InfraVm:
|
||||
private_key: Path
|
||||
vm: firecracker_vm.VmHandle | None = None
|
||||
|
||||
@property
|
||||
def orchestrator_url(self) -> str:
|
||||
return f"http://{self.guest_ip}:{ORCHESTRATOR_PORT}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class InfraEndpoint:
|
||||
"""How to reach the running per-host pair. `orchestrator` is the control
|
||||
plane (host CLI + registration + in-VM builds); `gateway` is the data-plane
|
||||
`Gateway` service (agent egress / git-http / supervise, CA fetch, git-gate
|
||||
provisioning). Mirrors the docker/macOS `InfraEndpoint` shape."""
|
||||
"""How to reach the running per-host pair. `orchestrator` is the control-plane
|
||||
`Orchestrator` service (host CLI + registration + in-VM builds); `gateway` is
|
||||
the data-plane `Gateway` service (agent egress / git-http / supervise, CA
|
||||
fetch, git-gate provisioning). Mirrors the docker/macOS `InfraEndpoint`."""
|
||||
|
||||
orchestrator: InfraVm
|
||||
orchestrator: "FirecrackerOrchestrator"
|
||||
gateway: "FirecrackerGateway"
|
||||
|
||||
@property
|
||||
def orchestrator_url(self) -> str:
|
||||
return self.orchestrator.orchestrator_url
|
||||
return self.orchestrator.url()
|
||||
|
||||
def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str:
|
||||
return self.gateway.ca_cert_pem(timeout=timeout)
|
||||
@@ -186,8 +174,14 @@ def ensure_running() -> InfraEndpoint:
|
||||
Concurrency-safe: the cold stop/build/boot path is serialized by a host
|
||||
flock, so two simultaneous first launches don't both boot on the same
|
||||
links/PIDs. The healthy fast-path takes no lock."""
|
||||
# Local imports: the orchestrator + gateway services import this module for
|
||||
# the shared VM substrate, so importing them at module scope would cycle.
|
||||
from .orchestrator import FirecrackerOrchestrator
|
||||
from .gateway import FirecrackerGateway
|
||||
|
||||
key = _infra_dir() / "id_ed25519"
|
||||
url = f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}"
|
||||
orchestrator = FirecrackerOrchestrator()
|
||||
url = orchestrator.url()
|
||||
want = _expected_version()
|
||||
if _adoptable(key, url, want):
|
||||
info(f"adopting running infra VMs (orchestrator at {url})")
|
||||
@@ -202,29 +196,26 @@ def ensure_running() -> InfraEndpoint:
|
||||
# Clear stale/hung/OUTDATED VMs holding either link before booting fresh.
|
||||
stop()
|
||||
ensure_built()
|
||||
orchestrator = boot_orchestrator()
|
||||
wait_for_health(orchestrator)
|
||||
# The orchestrator holds the signing key; mint the role-scoped `gateway`
|
||||
# JWT on the host and hand it to the gateway service, which never sees
|
||||
# the key (#469). The gateway owns its own boot — see FirecrackerGateway.
|
||||
# Local import: the gateway service imports this module for the shared VM
|
||||
# substrate, so importing it at module scope would cycle.
|
||||
from .gateway import FirecrackerGateway
|
||||
gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token())
|
||||
# Orchestrator first — the gateway daemons reach the control plane at
|
||||
# startup. Each service owns its own boot; the orchestrator (which holds
|
||||
# the signing key) mints the role-scoped `gateway` JWT for the gateway,
|
||||
# which never sees the key (#469).
|
||||
orchestrator.ensure_running()
|
||||
gateway = FirecrackerGateway()
|
||||
gateway.connect_to_orchestrator(orchestrator.orchestrator_url, gateway_token)
|
||||
gateway.connect_to_orchestrator(
|
||||
orchestrator.gateway_url(), orchestrator.mint_gateway_token())
|
||||
_record_booted_version(want)
|
||||
return InfraEndpoint(orchestrator=orchestrator, gateway=gateway)
|
||||
|
||||
|
||||
def _adopt(key: Path) -> InfraEndpoint:
|
||||
"""Build handles to the already-running pair from the stable key + the
|
||||
fixed links (no live VM handle — teardown / CA fetch go through the PID
|
||||
def _adopt(key: Path) -> "InfraEndpoint":
|
||||
"""Build the service handles for the already-running pair from the stable key
|
||||
+ the fixed links (no live VM handle — teardown / CA fetch go through the PID
|
||||
files + stable key)."""
|
||||
from .orchestrator import FirecrackerOrchestrator
|
||||
from .gateway import FirecrackerGateway
|
||||
return InfraEndpoint(
|
||||
orchestrator=InfraVm(
|
||||
guest_ip=netpool.orch_slot().guest_ip, private_key=key),
|
||||
orchestrator=FirecrackerOrchestrator(),
|
||||
gateway=FirecrackerGateway(
|
||||
InfraVm(guest_ip=netpool.gw_slot().guest_ip, private_key=key)),
|
||||
)
|
||||
@@ -259,24 +250,7 @@ def stop() -> None:
|
||||
_version_file().unlink(missing_ok=True)
|
||||
|
||||
|
||||
def boot_orchestrator() -> InfraVm:
|
||||
"""Boot the orchestrator (control-plane) VM (detached, so it outlives the
|
||||
launcher) on the orchestrator link, and seed its signing key. Prefer
|
||||
`ensure_running`."""
|
||||
slot = netpool.orch_slot()
|
||||
orch = _boot_vm(
|
||||
name="bot-bottle-orchestrator", slot=slot, run_dir=_orch_dir(),
|
||||
role="orchestrator", mem_mib=_ORCH_MEM_MIB,
|
||||
data_drive=_ensure_registry_volume(),
|
||||
)
|
||||
# Push the host-canonical signing key over SSH (the init waits for it
|
||||
# before starting the control plane). The host token file stays the single
|
||||
# source of truth, so a co-running Docker/macOS control plane keeps working.
|
||||
_push_signing_key(orch)
|
||||
return orch
|
||||
|
||||
|
||||
def _boot_vm(
|
||||
def boot_vm(
|
||||
*,
|
||||
name: str,
|
||||
slot: netpool.Slot,
|
||||
@@ -376,50 +350,7 @@ def _record_booted_version(version: str) -> None:
|
||||
_version_file().write_text(version + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
# The registry "volume": a host-side ext4 file attached to the orchestrator VM
|
||||
# as a second virtio-block device (guest /dev/vdb), mounted at the control
|
||||
# plane's DB dir. It outlives the ephemeral rootfs, so the bottle registry
|
||||
# survives an orchestrator-VM restart — the firecracker analogue of a docker
|
||||
# volume. It is a plain ext4 file: `sudo mount -o loop <path>` on the host
|
||||
# (with the VM stopped) to inspect bot-bottle.db directly. The gateway VM has
|
||||
# no such volume — the data plane never opens the DB (#469).
|
||||
_REGISTRY_SIZE = "512M"
|
||||
|
||||
|
||||
def registry_volume_path() -> Path:
|
||||
return _infra_dir() / "registry.ext4"
|
||||
|
||||
|
||||
def _ensure_registry_volume() -> Path:
|
||||
"""Create the empty ext4 registry volume on first use; reuse it after."""
|
||||
vol = registry_volume_path()
|
||||
if vol.exists():
|
||||
return vol
|
||||
info(f"creating infra registry volume {vol} ({_REGISTRY_SIZE})")
|
||||
proc = subprocess.run(
|
||||
["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _REGISTRY_SIZE],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
vol.unlink(missing_ok=True)
|
||||
die(f"creating registry volume failed: {proc.stderr.strip()}")
|
||||
return vol
|
||||
|
||||
|
||||
def _push_signing_key(infra: InfraVm) -> None:
|
||||
"""Push the host-canonical signing key into the freshly booted orchestrator
|
||||
VM over SSH (atomic write), so its control plane verifies tokens with the
|
||||
same key the host CLI signs from. Retries until the guest's SSH is up (it
|
||||
comes up before the init waits for this file); dies if it never lands,
|
||||
since the VM then refuses to start its control plane rather than run OPEN."""
|
||||
_push_secret(
|
||||
infra, host_orchestrator_token(), _GUEST_SIGNING_KEY_PATH,
|
||||
"the control-plane signing key to the orchestrator VM "
|
||||
"(its control plane will not start)",
|
||||
)
|
||||
|
||||
|
||||
def _push_secret(infra: InfraVm, secret: str, dest: str, what: str) -> None:
|
||||
def push_secret(infra: InfraVm, secret: str, dest: str, what: str) -> None:
|
||||
"""Pipe `secret` to an atomic write of `dest` in the guest over SSH,
|
||||
retrying while the guest's SSH comes up; die (naming `what`) if it never
|
||||
succeeds. Bare-pipe input keeps the value off argv."""
|
||||
@@ -525,31 +456,6 @@ def _health_ok(url: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def wait_for_health(
|
||||
infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS,
|
||||
) -> None:
|
||||
"""Poll the orchestrator's /health until it answers 200 or the deadline
|
||||
passes. Dies (with the console tail) if the VMM exits early."""
|
||||
url = f"{infra.orchestrator_url}/health"
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if infra.vm is not None and not infra.vm.is_alive():
|
||||
die(f"orchestrator VM exited during boot (rc={infra.vm.process.returncode}).\n"
|
||||
f"{firecracker_vm._console_tail(infra.vm.console_log)}")
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=1.0) as resp:
|
||||
if resp.status == 200:
|
||||
info(f"orchestrator control plane healthy at {infra.orchestrator_url}")
|
||||
return
|
||||
except (urllib.error.URLError, TimeoutError, OSError):
|
||||
pass
|
||||
time.sleep(_HEALTH_POLL_SECONDS)
|
||||
tail = (firecracker_vm._console_tail(infra.vm.console_log)
|
||||
if infra.vm is not None else "")
|
||||
die(f"orchestrator control plane at {url} did not become healthy within "
|
||||
f"{timeout:.0f}s.\n{tail}")
|
||||
|
||||
|
||||
def _infra_init() -> str:
|
||||
"""PID-1 init for the infra VMs. Shared setup (pseudo-filesystems, PATH,
|
||||
resolver, debug SSH), then a `bb_role` cmdline branch selecting the plane:
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
"""The Firecracker orchestrator (control plane) as a microVM (PRD 0070).
|
||||
|
||||
`FirecrackerOrchestrator` is the Firecracker implementation of the backend-neutral
|
||||
`Orchestrator` service, and owns the control plane's host-side logic directly:
|
||||
booting the orchestrator microVM on its NAT'd link with the persistent registry
|
||||
volume (/dev/vdb — the sole opener of `bot-bottle.db`), seeding the host-canonical
|
||||
signing key over SSH, and waiting for `/health`. The host CLI reaches it at its
|
||||
guest IP, and so does the gateway (`bb_orch` cmdline), so `url()` == `gateway_url()`.
|
||||
|
||||
What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM
|
||||
substrate the gateway VM also shares — booting a VM from the shared rootfs
|
||||
(`boot_vm`), the stable SSH keypair, the secret-push retry, the PID lifecycle —
|
||||
plus the pair coordinator (`ensure_running`: adopt-or-boot-both under a singleton
|
||||
lock with a shared version marker) and the single shared `bb_role`-branched init
|
||||
baked into the one published rootfs. Those are the shared seam.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from ...log import die, info
|
||||
from ...paths import host_orchestrator_token
|
||||
from ...orchestrator.lifecycle import (
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
Orchestrator,
|
||||
)
|
||||
from . import firecracker_vm, infra_vm, netpool
|
||||
from .infra_vm import ORCHESTRATOR_PORT
|
||||
|
||||
# The orchestrator microVM's name (the `bb_role=orchestrator` boot tag / run
|
||||
# dir). Fixed per host — one control-plane VM.
|
||||
ORCHESTRATOR_NAME = "bot-bottle-orchestrator"
|
||||
|
||||
# Memory ceiling (fixed at boot, demand-paged). The orchestrator keeps the build
|
||||
# headroom (buildah's 2-4 GB working set during in-VM agent builds) — PRD 0070
|
||||
# "Memory: fixed ceilings".
|
||||
_ORCH_MEM_MIB = 4096
|
||||
|
||||
_HEALTH_POLL_SECONDS = 0.5
|
||||
|
||||
# The registry "volume": a host-side ext4 file attached to the orchestrator VM as
|
||||
# a second virtio-block device (guest /dev/vdb), mounted at the control plane's DB
|
||||
# dir. It outlives the ephemeral rootfs, so the bottle registry survives an
|
||||
# orchestrator-VM restart — the firecracker analogue of a docker volume. A plain
|
||||
# ext4 file: `sudo mount -o loop <path>` on the host (VM stopped) to inspect
|
||||
# bot-bottle.db. The gateway VM has no such volume — the data plane never opens
|
||||
# the DB (#469).
|
||||
_REGISTRY_SIZE = "512M"
|
||||
|
||||
|
||||
def registry_volume_path() -> Path:
|
||||
return infra_vm._infra_dir() / "registry.ext4"
|
||||
|
||||
|
||||
class FirecrackerOrchestrator(Orchestrator):
|
||||
"""The control plane as a Firecracker microVM on the orchestrator link.
|
||||
|
||||
The shared rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's
|
||||
`ensure_built` no-op here); `ensure_running` boots the VM, seeds the signing
|
||||
key, and blocks until `/health` answers."""
|
||||
|
||||
name = ORCHESTRATOR_NAME
|
||||
|
||||
def __init__(self, vm: infra_vm.InfraVm | None = None) -> None:
|
||||
# The live VM handle when this process booted it; None when adapting an
|
||||
# already-running orchestrator (health/URL go through the fixed link).
|
||||
self._vm = vm
|
||||
|
||||
def url(self) -> str:
|
||||
"""The orchestrator VM's guest IP URL — where the host CLI reaches the
|
||||
control plane."""
|
||||
return f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}"
|
||||
|
||||
def gateway_url(self) -> str:
|
||||
"""Same guest IP the host uses — the gateway resolves the orchestrator at
|
||||
`bb_orch=<guest_ip>` off its cmdline."""
|
||||
return self.url()
|
||||
|
||||
def ssh_target(self) -> tuple[Path, str]:
|
||||
"""(private key, guest IP) for SSHing into the orchestrator VM. In-VM
|
||||
agent-image builds (buildah) run here — the build host drives them over
|
||||
SSH with the stable infra key."""
|
||||
return infra_vm._infra_dir() / "id_ed25519", netpool.orch_slot().guest_ip
|
||||
|
||||
def is_running(self) -> bool:
|
||||
return infra_vm._pidfile_alive(infra_vm._orch_dir())
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop only the orchestrator VM (idempotent). A dead orchestrator fails
|
||||
`infra_vm._adoptable`'s health check, so no version marker to clear."""
|
||||
infra_vm._kill_pidfile(infra_vm._orch_dir())
|
||||
infra_vm._pid_file(infra_vm._orch_dir()).unlink(missing_ok=True)
|
||||
|
||||
def ensure_running(
|
||||
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
) -> None:
|
||||
"""Boot the orchestrator VM on its link with the persistent registry
|
||||
volume, seed the signing key over SSH, and block until `/health` answers.
|
||||
Idempotent — a live, healthy control plane is left alone (the pair
|
||||
coordinator otherwise `stop()`s it before calling, so this boots fresh)."""
|
||||
if self.is_running() and self.is_healthy():
|
||||
return
|
||||
vm = infra_vm.boot_vm(
|
||||
name=ORCHESTRATOR_NAME, slot=netpool.orch_slot(),
|
||||
run_dir=infra_vm._orch_dir(), role="orchestrator", mem_mib=_ORCH_MEM_MIB,
|
||||
data_drive=self._ensure_registry_volume(),
|
||||
)
|
||||
# Push the host-canonical signing key (the init waits for it before
|
||||
# starting the control plane). The host token file stays the single
|
||||
# source of truth, so a co-running docker/macOS control plane keeps
|
||||
# working; the guest verifies tokens with the same key the CLI signs from.
|
||||
infra_vm.push_secret(
|
||||
vm, host_orchestrator_token(), infra_vm._GUEST_SIGNING_KEY_PATH,
|
||||
"the control-plane signing key to the orchestrator VM "
|
||||
"(its control plane will not start)",
|
||||
)
|
||||
self._vm = vm
|
||||
self._wait_for_health(vm, timeout=startup_timeout)
|
||||
|
||||
def _wait_for_health(
|
||||
self, vm: infra_vm.InfraVm, *, timeout: float,
|
||||
) -> None:
|
||||
"""Poll `/health` until it answers 200 or the deadline passes. Dies (with
|
||||
the console tail) if the VMM exits early."""
|
||||
url = f"{self.url()}/health"
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if vm.vm is not None and not vm.vm.is_alive():
|
||||
die(f"orchestrator VM exited during boot (rc={vm.vm.process.returncode}).\n"
|
||||
f"{firecracker_vm._console_tail(vm.vm.console_log)}")
|
||||
if self.is_healthy():
|
||||
info(f"orchestrator control plane healthy at {self.url()}")
|
||||
return
|
||||
time.sleep(_HEALTH_POLL_SECONDS)
|
||||
tail = (firecracker_vm._console_tail(vm.vm.console_log)
|
||||
if vm.vm is not None else "")
|
||||
die(f"orchestrator control plane at {url} did not become healthy within "
|
||||
f"{timeout:.0f}s.\n{tail}")
|
||||
|
||||
def _ensure_registry_volume(self) -> Path:
|
||||
"""Create the empty ext4 registry volume on first use; reuse it after."""
|
||||
vol = registry_volume_path()
|
||||
if vol.exists():
|
||||
return vol
|
||||
info(f"creating infra registry volume {vol} ({_REGISTRY_SIZE})")
|
||||
proc = subprocess.run(
|
||||
["mke2fs", "-q", "-t", "ext4", "-F", str(vol), _REGISTRY_SIZE],
|
||||
capture_output=True, text=True, check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
vol.unlink(missing_ok=True)
|
||||
die(f"creating registry volume failed: {proc.stderr.strip()}")
|
||||
return vol
|
||||
|
||||
|
||||
__all__ = ["FirecrackerOrchestrator", "ORCHESTRATOR_NAME", "registry_volume_path"]
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Unit: the Firecracker gateway data plane as a microVM (PRD 0070).
|
||||
|
||||
The gateway service owns its host-side logic directly: booting the gateway VM
|
||||
(via the shared `infra_vm._boot_vm` substrate), seeding the pre-minted token,
|
||||
(via the shared `infra_vm.boot_vm` substrate), seeding the pre-minted token,
|
||||
fetching the CA over SSH, and the SSH exec/cp provisioning transport.
|
||||
"""
|
||||
|
||||
@@ -37,21 +37,21 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
|
||||
|
||||
def test_refuses_without_orchestrator_url(self) -> None:
|
||||
gw = FirecrackerGateway()
|
||||
with patch.object(infra_vm, "_boot_vm") as boot:
|
||||
with patch.object(infra_vm, "boot_vm") as boot:
|
||||
with self.assertRaises(GatewayError):
|
||||
gw.connect_to_orchestrator("", _TOKEN)
|
||||
boot.assert_not_called()
|
||||
|
||||
def test_refuses_without_gateway_token(self) -> None:
|
||||
gw = FirecrackerGateway()
|
||||
with patch.object(infra_vm, "_boot_vm") as boot:
|
||||
with patch.object(infra_vm, "boot_vm") as boot:
|
||||
with self.assertRaises(GatewayError):
|
||||
gw.connect_to_orchestrator(_ORCH_URL, "")
|
||||
boot.assert_not_called()
|
||||
|
||||
def test_refuses_a_url_without_a_host(self) -> None:
|
||||
gw = FirecrackerGateway()
|
||||
with patch.object(infra_vm, "_boot_vm") as boot:
|
||||
with patch.object(infra_vm, "boot_vm") as boot:
|
||||
with self.assertRaises(GatewayError):
|
||||
gw.connect_to_orchestrator("http://:8099", _TOKEN)
|
||||
boot.assert_not_called()
|
||||
@@ -59,8 +59,8 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
|
||||
def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None:
|
||||
gw = FirecrackerGateway()
|
||||
booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))
|
||||
with patch.object(infra_vm, "_boot_vm", return_value=booted) as boot, \
|
||||
patch.object(infra_vm, "_push_secret") as push:
|
||||
with patch.object(infra_vm, "boot_vm", return_value=booted) as boot, \
|
||||
patch.object(infra_vm, "push_secret") as push:
|
||||
gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
|
||||
# Booted on the gateway link with the gateway role; the orchestrator's
|
||||
# guest IP (parsed off the URL) rides the cmdline as bb_orch.
|
||||
|
||||
@@ -15,65 +15,51 @@ from unittest.mock import MagicMock, patch
|
||||
from bot_bottle.backend.firecracker import infra_vm
|
||||
|
||||
|
||||
class TestPushSecrets(unittest.TestCase):
|
||||
"""`_push_signing_key` (orchestrator) seeds the guest's secret over SSH via
|
||||
`_push_secret`. The host token file stays the single source of truth, never
|
||||
clobbered per-backend (#469). The gateway's own JWT push lives with the
|
||||
gateway service (test_firecracker_gateway)."""
|
||||
class TestPushSecret(unittest.TestCase):
|
||||
"""`push_secret` is the shared substrate both services seed a guest secret
|
||||
over SSH with (atomic write, retry while SSH comes up). The per-plane pushes
|
||||
live with each service (orchestrator: signing key; gateway: JWT)."""
|
||||
|
||||
def _infra(self) -> infra_vm.InfraVm:
|
||||
return infra_vm.InfraVm(guest_ip="10.0.0.1", private_key=Path("/k"))
|
||||
|
||||
def test_signing_key_written_atomically(self):
|
||||
def test_written_atomically(self):
|
||||
proc = MagicMock(returncode=0, stderr="")
|
||||
with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \
|
||||
patch.object(infra_vm.subprocess, "run", return_value=proc) as run:
|
||||
infra_vm._push_signing_key(self._infra())
|
||||
# Piped the key to an atomic write of the guest token path.
|
||||
self.assertEqual("host-key", run.call_args.kwargs["input"])
|
||||
with patch.object(infra_vm.subprocess, "run", return_value=proc) as run:
|
||||
infra_vm.push_secret(self._infra(), "secret", "/dest", "the thing")
|
||||
self.assertEqual("secret", run.call_args.kwargs["input"])
|
||||
remote_cmd = run.call_args.args[0][-1]
|
||||
self.assertIn(f"cat > {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp", remote_cmd)
|
||||
self.assertIn(f"mv {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp "
|
||||
f"{infra_vm._GUEST_SIGNING_KEY_PATH}", remote_cmd)
|
||||
self.assertIn("cat > /dest.tmp", remote_cmd)
|
||||
self.assertIn("mv /dest.tmp /dest", remote_cmd)
|
||||
|
||||
def test_dies_when_push_never_succeeds(self):
|
||||
proc = MagicMock(returncode=255, stderr="ssh: connect refused")
|
||||
with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \
|
||||
patch.object(infra_vm, "_SECRET_PUSH_TIMEOUT_SECONDS", 0.05), \
|
||||
with patch.object(infra_vm, "_SECRET_PUSH_TIMEOUT_SECONDS", 0.05), \
|
||||
patch.object(infra_vm, "_SECRET_PUSH_POLL_SECONDS", 0.0), \
|
||||
patch.object(infra_vm.subprocess, "run", return_value=proc), \
|
||||
patch.object(infra_vm, "die", side_effect=SystemExit) as die:
|
||||
with self.assertRaises(SystemExit):
|
||||
infra_vm._push_signing_key(self._infra())
|
||||
infra_vm.push_secret(self._infra(), "secret", "/dest", "the thing")
|
||||
die.assert_called_once()
|
||||
|
||||
|
||||
class TestOrchestratorUrl(unittest.TestCase):
|
||||
def test_url_uses_guest_ip_and_port(self):
|
||||
infra = infra_vm.InfraVm(
|
||||
vm=MagicMock(), guest_ip="10.243.255.1", private_key=Path("/k"))
|
||||
self.assertEqual(
|
||||
f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}",
|
||||
infra.orchestrator_url,
|
||||
)
|
||||
|
||||
|
||||
class TestInfraEndpoint(unittest.TestCase):
|
||||
"""The endpoint mirrors docker/macOS: it exposes the orchestrator's URL and
|
||||
delegates the CA fetch to the gateway service."""
|
||||
"""The endpoint mirrors docker/macOS: it exposes the orchestrator service's
|
||||
URL and delegates the CA fetch to the gateway service."""
|
||||
|
||||
def _endpoint(self) -> infra_vm.InfraEndpoint:
|
||||
from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
|
||||
from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator
|
||||
return infra_vm.InfraEndpoint(
|
||||
orchestrator=infra_vm.InfraVm(guest_ip="10.243.255.1", private_key=Path("/k")),
|
||||
orchestrator=FirecrackerOrchestrator(),
|
||||
gateway=FirecrackerGateway(
|
||||
infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))),
|
||||
)
|
||||
|
||||
def test_orchestrator_url_is_the_orchestrator_vm(self):
|
||||
def test_orchestrator_url_is_the_orchestrator_service(self):
|
||||
ep = self._endpoint()
|
||||
self.assertEqual(
|
||||
f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}", ep.orchestrator_url)
|
||||
ip = infra_vm.netpool.orch_slot().guest_ip
|
||||
self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", ep.orchestrator_url)
|
||||
|
||||
def test_gateway_ca_pem_delegates_to_the_gateway_service(self):
|
||||
ep = self._endpoint()
|
||||
@@ -122,64 +108,6 @@ class TestBuildInfraRootfs(unittest.TestCase):
|
||||
self.assertIn("BOT_BOTTLE_ORCHESTRATOR_URL=http://$ORCH:", init)
|
||||
|
||||
|
||||
class TestBootRole(unittest.TestCase):
|
||||
"""`boot_orchestrator` boots the shared rootfs with the `bb_role=orchestrator`
|
||||
cmdline, the orchestrator memory ceiling + registry volume, and seeds the
|
||||
signing key. (The gateway's boot lives in the gateway service —
|
||||
test_firecracker_gateway.)"""
|
||||
|
||||
def _boot_env(self):
|
||||
vm = MagicMock()
|
||||
vm.process.pid = 4242
|
||||
return vm
|
||||
|
||||
def test_orchestrator_role_mem_and_registry(self):
|
||||
import tempfile
|
||||
vm = self._boot_env()
|
||||
with tempfile.TemporaryDirectory() as td, \
|
||||
patch.object(infra_vm.netpool, "tap_present", return_value=True), \
|
||||
patch.object(infra_vm.infra_artifact, "local_build_requested", return_value=False), \
|
||||
patch.object(infra_vm.infra_artifact, "materialize_ext4"), \
|
||||
patch.object(infra_vm.infra_artifact, "infra_artifact_version", return_value="v"), \
|
||||
patch.object(infra_vm, "_stable_keypair", return_value=(Path("/k"), "pub")), \
|
||||
patch.object(infra_vm, "_ensure_registry_volume", return_value=Path("/reg")), \
|
||||
patch.object(infra_vm, "_push_signing_key") as push, \
|
||||
patch.object(infra_vm.firecracker_vm, "boot", return_value=vm) as boot, \
|
||||
patch.object(infra_vm, "_orch_dir", return_value=Path(td)):
|
||||
infra_vm.boot_orchestrator()
|
||||
kw = boot.call_args.kwargs
|
||||
self.assertIn("bb_role=orchestrator", kw["extra_boot_args"])
|
||||
self.assertEqual(infra_vm._ORCH_MEM_MIB, kw["mem_mib"])
|
||||
self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP
|
||||
push.assert_called_once() # signing key seeded
|
||||
|
||||
|
||||
class TestRegistryVolume(unittest.TestCase):
|
||||
def test_reuses_existing_volume(self):
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
vol = Path(td) / "registry.ext4"
|
||||
vol.write_bytes(b"") # already present
|
||||
with patch.object(infra_vm, "registry_volume_path", return_value=vol), \
|
||||
patch.object(infra_vm.subprocess, "run") as run:
|
||||
out = infra_vm._ensure_registry_volume()
|
||||
run.assert_not_called() # no mke2fs when it exists
|
||||
self.assertEqual(vol, out)
|
||||
|
||||
def test_creates_volume_when_missing(self):
|
||||
import tempfile
|
||||
from subprocess import CompletedProcess
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
vol = Path(td) / "registry.ext4"
|
||||
with patch.object(infra_vm, "registry_volume_path", return_value=vol), \
|
||||
patch.object(infra_vm.subprocess, "run",
|
||||
return_value=CompletedProcess([], 0)) as run:
|
||||
infra_vm._ensure_registry_volume()
|
||||
argv = run.call_args.args[0]
|
||||
self.assertIn("mke2fs", argv)
|
||||
self.assertIn(str(vol), argv)
|
||||
|
||||
|
||||
class TestEnsureBuilt(unittest.TestCase):
|
||||
def test_default_pulls_artifact_without_docker(self):
|
||||
# PRD 0069 Stage 2: the launch host pulls the prebuilt rootfs; no Docker.
|
||||
@@ -204,30 +132,10 @@ class TestEnsureBuilt(unittest.TestCase):
|
||||
self.assertIn(infra_vm._GATEWAY_IMAGE, tags[:-1])
|
||||
|
||||
|
||||
class TestWaitForHealth(unittest.TestCase):
|
||||
def _infra(self, alive: bool = True) -> infra_vm.InfraVm:
|
||||
vm = MagicMock()
|
||||
vm.is_alive.return_value = alive
|
||||
return infra_vm.InfraVm(vm=vm, guest_ip="10.0.0.1", private_key=Path("/k"))
|
||||
|
||||
def test_returns_on_200(self):
|
||||
infra = self._infra()
|
||||
cm = MagicMock()
|
||||
cm.__enter__.return_value.status = 200
|
||||
with patch.object(infra_vm.urllib.request, "urlopen", return_value=cm):
|
||||
infra_vm.wait_for_health(infra, timeout=5) # must not raise
|
||||
|
||||
def test_dies_when_vm_exits(self):
|
||||
infra = self._infra(alive=False)
|
||||
assert infra.vm is not None # narrow for the type checker (it's a mock)
|
||||
infra.vm.process.returncode = 1
|
||||
with patch.object(infra_vm.firecracker_vm, "_console_tail", return_value=""), \
|
||||
self.assertRaises(SystemExit):
|
||||
infra_vm.wait_for_health(infra, timeout=5)
|
||||
|
||||
|
||||
# The gateway service is imported lazily inside ensure_running / _adopt (to
|
||||
# break the infra_vm <-> gateway import cycle), so patch it at its home module.
|
||||
# The orchestrator + gateway services are imported lazily inside ensure_running
|
||||
# / _adopt (to break the infra_vm <-> service import cycle), so patch them at
|
||||
# their home modules.
|
||||
_ORCH_CLS = "bot_bottle.backend.firecracker.orchestrator.FirecrackerOrchestrator"
|
||||
_GW_CLS = "bot_bottle.backend.firecracker.gateway.FirecrackerGateway"
|
||||
|
||||
|
||||
@@ -236,6 +144,7 @@ class TestEnsureRunningSingleton(unittest.TestCase):
|
||||
# Healthy control plane + live gateway + existing key + matching version
|
||||
# marker -> adopt (no boot).
|
||||
from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
|
||||
from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
d = Path(td)
|
||||
@@ -244,14 +153,13 @@ class TestEnsureRunningSingleton(unittest.TestCase):
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=d), \
|
||||
patch.object(infra_vm, "_expected_version", return_value="v-current"), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=True), \
|
||||
patch.object(infra_vm, "boot_orchestrator") as boot_o:
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=True):
|
||||
ep = infra_vm.ensure_running()
|
||||
boot_o.assert_not_called()
|
||||
self.assertIsNone(ep.orchestrator.vm)
|
||||
# The orchestrator handle + the adopted gateway service address the two
|
||||
# distinct infra links.
|
||||
self.assertEqual(infra_vm.netpool.orch_slot().guest_ip, ep.orchestrator.guest_ip)
|
||||
# Adopted service handles addressing the two distinct infra links.
|
||||
self.assertIsInstance(ep.orchestrator, FirecrackerOrchestrator)
|
||||
self.assertEqual(
|
||||
infra_vm.netpool.orch_slot().guest_ip,
|
||||
ep.orchestrator_url.split("://")[1].split(":")[0])
|
||||
self.assertIsInstance(ep.gateway, FirecrackerGateway)
|
||||
self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, ep.gateway.address())
|
||||
|
||||
@@ -269,18 +177,15 @@ class TestEnsureRunningSingleton(unittest.TestCase):
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=True), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built"), \
|
||||
patch.object(infra_vm, "wait_for_health"), \
|
||||
patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \
|
||||
patch.object(infra_vm, "mint", return_value="gw.jwt"), \
|
||||
patch.object(infra_vm, "boot_orchestrator") as boot_o, \
|
||||
patch(_ORCH_CLS) as orch_cls, \
|
||||
patch(_GW_CLS) as gw_cls:
|
||||
boot_o.return_value = infra_vm.InfraVm(
|
||||
guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
|
||||
orch_cls.return_value.gateway_url.return_value = "http://10.243.255.1:8099"
|
||||
orch_cls.return_value.mint_gateway_token.return_value = "gw.jwt"
|
||||
infra_vm.ensure_running()
|
||||
stop.assert_called_once() # dislodge the outdated pair
|
||||
boot_o.assert_called_once()
|
||||
# The gateway service is bound to the orchestrator's URL, carrying
|
||||
# the host-minted `gateway` token.
|
||||
orch_cls.return_value.ensure_running.assert_called_once()
|
||||
# The gateway service is bound to the orchestrator's gateway URL,
|
||||
# carrying the orchestrator-minted `gateway` token.
|
||||
gw_cls.return_value.connect_to_orchestrator.assert_called_once_with(
|
||||
"http://10.243.255.1:8099", "gw.jwt")
|
||||
# The fresh boot records the current version for the next launcher.
|
||||
@@ -300,16 +205,11 @@ class TestEnsureRunningSingleton(unittest.TestCase):
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=False), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built"), \
|
||||
patch.object(infra_vm, "wait_for_health"), \
|
||||
patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \
|
||||
patch.object(infra_vm, "mint", return_value="gw.jwt"), \
|
||||
patch.object(infra_vm, "boot_orchestrator") as boot_o, \
|
||||
patch(_ORCH_CLS) as orch_cls, \
|
||||
patch(_GW_CLS) as gw_cls:
|
||||
boot_o.return_value = infra_vm.InfraVm(
|
||||
guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
|
||||
infra_vm.ensure_running()
|
||||
stop.assert_called_once()
|
||||
boot_o.assert_called_once()
|
||||
orch_cls.return_value.ensure_running.assert_called_once()
|
||||
gw_cls.return_value.connect_to_orchestrator.assert_called_once()
|
||||
|
||||
def test_boots_both_when_unhealthy(self):
|
||||
@@ -320,20 +220,14 @@ class TestEnsureRunningSingleton(unittest.TestCase):
|
||||
patch.object(infra_vm, "_health_ok", return_value=False), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built") as built, \
|
||||
patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \
|
||||
patch.object(infra_vm, "mint", return_value="gw.jwt"), \
|
||||
patch.object(infra_vm, "boot_orchestrator") as boot_o, \
|
||||
patch(_GW_CLS) as gw_cls, \
|
||||
patch.object(infra_vm, "wait_for_health") as wait:
|
||||
boot_o.return_value = infra_vm.InfraVm(
|
||||
guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
|
||||
patch(_ORCH_CLS) as orch_cls, \
|
||||
patch(_GW_CLS) as gw_cls:
|
||||
infra_vm.ensure_running()
|
||||
stop.assert_called_once() # clear stale VMs first
|
||||
built.assert_called_once()
|
||||
# Orchestrator boots + becomes healthy BEFORE the gateway is connected
|
||||
# (its daemons reach the control plane at startup).
|
||||
boot_o.assert_called_once()
|
||||
wait.assert_called_once()
|
||||
# Orchestrator is brought up BEFORE the gateway is connected (its daemons
|
||||
# reach the control plane at startup).
|
||||
orch_cls.return_value.ensure_running.assert_called_once()
|
||||
gw_cls.return_value.connect_to_orchestrator.assert_called_once()
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Unit: the Firecracker orchestrator (control plane) microVM lifecycle (PRD 0070).
|
||||
|
||||
The control plane's host-side logic — boot on the orchestrator link with the
|
||||
registry volume, seed the signing key, wait for /health — lives in
|
||||
`FirecrackerOrchestrator`, composing the shared `infra_vm` VM substrate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bot_bottle.backend.firecracker import infra_vm
|
||||
from bot_bottle.backend.firecracker.orchestrator import (
|
||||
ORCHESTRATOR_NAME,
|
||||
FirecrackerOrchestrator,
|
||||
registry_volume_path,
|
||||
)
|
||||
from bot_bottle.backend.firecracker.infra_vm import ORCHESTRATOR_PORT
|
||||
|
||||
_ORCH = "bot_bottle.backend.firecracker.orchestrator"
|
||||
|
||||
|
||||
class TestUrls(unittest.TestCase):
|
||||
def test_url_and_gateway_url_are_the_orch_link_guest_ip(self) -> None:
|
||||
orch = FirecrackerOrchestrator()
|
||||
ip = infra_vm.netpool.orch_slot().guest_ip
|
||||
self.assertEqual(f"http://{ip}:{ORCHESTRATOR_PORT}", orch.url())
|
||||
self.assertEqual(orch.url(), orch.gateway_url())
|
||||
|
||||
def test_ssh_target_is_stable_key_and_orch_guest_ip(self) -> None:
|
||||
orch = FirecrackerOrchestrator()
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=Path("/infra")):
|
||||
key, ip = orch.ssh_target()
|
||||
self.assertEqual(Path("/infra/id_ed25519"), key)
|
||||
self.assertEqual(infra_vm.netpool.orch_slot().guest_ip, ip)
|
||||
|
||||
|
||||
class TestEnsureRunning(unittest.TestCase):
|
||||
def test_boots_orchestrator_role_with_mem_and_registry_then_pushes_key(self) -> None:
|
||||
orch = FirecrackerOrchestrator()
|
||||
vm = infra_vm.InfraVm(guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
|
||||
with patch.object(infra_vm, "boot_vm", return_value=vm) as boot, \
|
||||
patch.object(infra_vm, "push_secret") as push, \
|
||||
patch(f"{_ORCH}.host_orchestrator_token", return_value="host-key"), \
|
||||
patch.object(orch, "is_running", return_value=False), \
|
||||
patch.object(orch, "_ensure_registry_volume", return_value=Path("/reg")), \
|
||||
patch.object(orch, "_wait_for_health"):
|
||||
orch.ensure_running()
|
||||
kw = boot.call_args.kwargs
|
||||
self.assertEqual("orchestrator", kw["role"])
|
||||
self.assertEqual(4096, kw["mem_mib"]) # orchestrator keeps build headroom
|
||||
self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP
|
||||
# The host-canonical signing key is pushed to the guest signing-key path.
|
||||
self.assertEqual("host-key", push.call_args.args[1])
|
||||
self.assertEqual(infra_vm._GUEST_SIGNING_KEY_PATH, push.call_args.args[2])
|
||||
|
||||
def test_noop_when_running_and_healthy(self) -> None:
|
||||
orch = FirecrackerOrchestrator()
|
||||
with patch.object(orch, "is_running", return_value=True), \
|
||||
patch.object(orch, "is_healthy", return_value=True), \
|
||||
patch.object(infra_vm, "boot_vm") as boot:
|
||||
orch.ensure_running()
|
||||
boot.assert_not_called()
|
||||
|
||||
|
||||
class TestWaitForHealth(unittest.TestCase):
|
||||
def _vm(self, alive: bool = True) -> infra_vm.InfraVm:
|
||||
h = MagicMock()
|
||||
h.is_alive.return_value = alive
|
||||
return infra_vm.InfraVm(vm=h, guest_ip="10.0.0.1", private_key=Path("/k"))
|
||||
|
||||
def test_returns_when_healthy(self) -> None:
|
||||
orch = FirecrackerOrchestrator()
|
||||
with patch.object(orch, "is_healthy", return_value=True):
|
||||
orch._wait_for_health(self._vm(), timeout=5) # must not raise
|
||||
|
||||
def test_dies_when_vm_exits(self) -> None:
|
||||
orch = FirecrackerOrchestrator()
|
||||
vm = self._vm(alive=False)
|
||||
assert vm.vm is not None
|
||||
vm.vm.process.returncode = 1
|
||||
with patch(f"{_ORCH}.firecracker_vm._console_tail", return_value=""), \
|
||||
patch(f"{_ORCH}.die", side_effect=SystemExit) as die:
|
||||
with self.assertRaises(SystemExit):
|
||||
orch._wait_for_health(vm, timeout=5)
|
||||
die.assert_called_once()
|
||||
|
||||
|
||||
class TestRegistryVolume(unittest.TestCase):
|
||||
def test_reuses_existing_volume(self) -> None:
|
||||
orch = FirecrackerOrchestrator()
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
vol = Path(td) / "registry.ext4"
|
||||
vol.write_bytes(b"") # already present
|
||||
with patch(f"{_ORCH}.registry_volume_path", return_value=vol), \
|
||||
patch(f"{_ORCH}.subprocess.run") as run:
|
||||
out = orch._ensure_registry_volume()
|
||||
run.assert_not_called() # no mke2fs when it exists
|
||||
self.assertEqual(vol, out)
|
||||
|
||||
def test_creates_volume_when_missing(self) -> None:
|
||||
from subprocess import CompletedProcess
|
||||
orch = FirecrackerOrchestrator()
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
vol = Path(td) / "registry.ext4"
|
||||
with patch(f"{_ORCH}.registry_volume_path", return_value=vol), \
|
||||
patch(f"{_ORCH}.subprocess.run",
|
||||
return_value=CompletedProcess([], 0)) as run:
|
||||
orch._ensure_registry_volume()
|
||||
argv = run.call_args.args[0]
|
||||
self.assertIn("mke2fs", argv)
|
||||
self.assertIn(str(vol), argv)
|
||||
|
||||
def test_registry_volume_path_under_infra_dir(self) -> None:
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=Path("/infra")):
|
||||
self.assertEqual(Path("/infra/registry.ext4"), registry_volume_path())
|
||||
|
||||
|
||||
class TestLifecycle(unittest.TestCase):
|
||||
def test_is_running_reads_the_orch_pidfile(self) -> None:
|
||||
orch = FirecrackerOrchestrator()
|
||||
with patch.object(infra_vm, "_pidfile_alive", return_value=True) as alive, \
|
||||
patch.object(infra_vm, "_orch_dir", return_value=Path("/orch")):
|
||||
self.assertTrue(orch.is_running())
|
||||
alive.assert_called_once_with(Path("/orch"))
|
||||
|
||||
def test_stop_kills_only_the_orch_pidfile(self) -> None:
|
||||
orch = FirecrackerOrchestrator()
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
d = Path(td)
|
||||
(d / "vm.pid").write_text("123")
|
||||
with patch.object(infra_vm, "_orch_dir", return_value=d), \
|
||||
patch.object(infra_vm, "_kill_pidfile") as kill:
|
||||
orch.stop()
|
||||
kill.assert_called_once_with(d)
|
||||
self.assertFalse((d / "vm.pid").exists())
|
||||
|
||||
def test_name_is_the_orchestrator_vm(self) -> None:
|
||||
self.assertEqual(ORCHESTRATOR_NAME, FirecrackerOrchestrator().name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user