refactor(backend): InfraService ABC + FirecrackerInfraService
Add a backend-neutral `InfraService` ABC (backend/infra_service.py) so all three backends present the same composer contract: `orchestrator()` / `gateway()` accessors, `ensure_running(*, startup_timeout) -> str` (the host control-plane URL), `stop()`, plus concrete `url()` / `is_healthy()` that delegate to the orchestrator. `DockerInfraService` and `MacosInfraService` now subclass it. Give firecracker the missing class: `FirecrackerInfraService` (backend/firecracker/infra.py) wraps the `infra_vm` substrate as the pair coordinator (adopt-or-boot-both under the singleton flock). `infra_vm` keeps only the plane-agnostic substrate; its `ensure_running` / `_adopt` / `InfraEndpoint` move to the class, and the coordinator helpers it now calls cross-module go public (`singleton_lock` / `expected_version` / `adoptable` / `record_booted_version`). Unify the return type on the way: `ensure_running` returns the URL string everywhere (was `str` for docker, two different `InfraEndpoint`s for macOS/firecracker), and those `InfraEndpoint`s are deleted — the services are the source of truth (callers read `gateway().address()` / `orchestrator().url()`). docker's `url` property becomes the inherited `url()`. backend.py / consolidated_launch / image_builder + tests updated. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -40,6 +40,7 @@ from ...gateway import (
|
||||
GATEWAY_NAME,
|
||||
GATEWAY_NETWORK,
|
||||
)
|
||||
from ..infra_service import InfraService
|
||||
from ...orchestrator.lifecycle import (
|
||||
DEFAULT_PORT,
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
@@ -53,7 +54,7 @@ INFRA_NAME = GATEWAY_NAME # the container agents attribute against is the gatew
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
class DockerInfraService:
|
||||
class DockerInfraService(InfraService):
|
||||
"""Composes the per-host control plane + gateway as two containers.
|
||||
|
||||
`orchestrator_name` / `gateway_name` let callers run independent pairs on
|
||||
@@ -91,14 +92,6 @@ class DockerInfraService:
|
||||
against (gateway IP, git-gate provisioning transport, CA fetch)."""
|
||||
return self._gateway_name
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
"""Host-side control-plane URL (the orchestrator's published loopback)."""
|
||||
return self.orchestrator().url()
|
||||
|
||||
def is_healthy(self) -> bool:
|
||||
return self.orchestrator().is_healthy()
|
||||
|
||||
def orchestrator(self) -> DockerOrchestrator:
|
||||
"""The control-plane service. Cheap to reconstruct — `ensure_built`
|
||||
builds its image, `ensure_running` brings it up, and the launch flow
|
||||
|
||||
@@ -123,5 +123,5 @@ class FirecrackerBottleBackend(
|
||||
return plan.agent_supervise_url
|
||||
|
||||
def ensure_orchestrator(self) -> str:
|
||||
from . import infra_vm
|
||||
return infra_vm.ensure_running().orchestrator_url
|
||||
from .infra import FirecrackerInfraService
|
||||
return FirecrackerInfraService().ensure_running()
|
||||
|
||||
@@ -41,8 +41,9 @@ from ...orchestrator.lifecycle import (
|
||||
from ...orchestrator.reprovision import reprovision_bottles
|
||||
from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME
|
||||
from ..provision_bottle import deprovision_bottle, provision_bottle
|
||||
from . import cleanup, infra_vm, util
|
||||
from . import cleanup, util
|
||||
from .gateway import FirecrackerGateway
|
||||
from .infra import FirecrackerInfraService
|
||||
|
||||
_ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret"
|
||||
|
||||
@@ -122,13 +123,13 @@ def launch_consolidated(
|
||||
"""Ensure the infra VM is up, register the bottle by its guest IP, and
|
||||
provision its git-gate state into the gateway VM. Returns the context the
|
||||
agent-VM launch needs. Raises on failure — the caller tears down."""
|
||||
infra = infra_vm.ensure_running()
|
||||
url = infra.orchestrator_url
|
||||
service = FirecrackerInfraService()
|
||||
url = service.ensure_running()
|
||||
client = OrchestratorClient(url)
|
||||
_reprovision_running_bottles(client)
|
||||
|
||||
# Read the gateway's provisioning transport + CA off the Gateway service.
|
||||
gateway = infra.gateway
|
||||
gateway = service.gateway()
|
||||
transport = gateway.provisioning_transport()
|
||||
reg = provision_bottle(
|
||||
client, guest_ip, egress_plan, git_gate_plan, transport,
|
||||
|
||||
@@ -7,14 +7,13 @@ fetching the mitmproxy CA, and the SSH exec/cp provisioning transport. The
|
||||
gateway VM never opens `bot-bottle.db` (#469), so it holds no signing key —
|
||||
only the token the host hands it via `connect_to_orchestrator`.
|
||||
|
||||
What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM
|
||||
substrate the orchestrator VM also uses — booting a VM from a per-plane rootfs
|
||||
(`boot_vm`), the stable SSH keypair, the secret-push retry loop, and the PID
|
||||
lifecycle — plus the pair coordinator (`ensure_running`: orchestrator-first
|
||||
health gate, singleton lock, adoption/version marker). The guest-side gateway
|
||||
daemon startup lives in the gateway rootfs's guest init (`_gateway_init` in
|
||||
`infra_vm`), so it can't live in a host-side method either. These are the shared
|
||||
seam.
|
||||
The plane-agnostic VM substrate the orchestrator VM also uses stays in
|
||||
`infra_vm` — booting a VM from a per-plane rootfs (`boot_vm`), the stable SSH
|
||||
keypair, the secret-push retry loop, the PID lifecycle, and the
|
||||
adoption/version helpers. The guest-side gateway daemon startup lives in the
|
||||
gateway rootfs's guest init (`_gateway_init` in `infra_vm`), so it can't live in
|
||||
a host-side method either. The pair coordinator (orchestrator-first, under a
|
||||
singleton flock) is `FirecrackerInfraService` (`infra.py`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -112,7 +111,7 @@ class FirecrackerGateway(Gateway):
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop only the gateway VM (idempotent — absent is success). A dead
|
||||
gateway already fails `infra_vm._adoptable`, so no version marker to
|
||||
gateway already fails `infra_vm.adoptable`, so no version marker to
|
||||
clear here."""
|
||||
infra_vm._kill_pidfile(infra_vm._gw_dir())
|
||||
infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True)
|
||||
|
||||
@@ -26,7 +26,8 @@ from pathlib import Path
|
||||
from typing import Generator
|
||||
|
||||
from ...log import die, info
|
||||
from . import infra_vm, util
|
||||
from . import util
|
||||
from .infra import FirecrackerInfraService
|
||||
|
||||
# vfs + chroot: buildah works as root in the microVM (no fuse-overlayfs /
|
||||
# overlay module / subuid maps). `--isolation` is a build/run-only flag;
|
||||
@@ -125,8 +126,9 @@ def _build_in_infra(
|
||||
orchestrator VM (the build-capable control plane), smoke test the image,
|
||||
and stream its rootfs into `base`. The infra VMs persist; only the
|
||||
per-build container/image/context are cleaned up."""
|
||||
orchestrator = infra_vm.ensure_running().orchestrator
|
||||
key, ip = orchestrator.ssh_target()
|
||||
service = FirecrackerInfraService()
|
||||
service.ensure_running()
|
||||
key, ip = service.orchestrator().ssh_target()
|
||||
tag = f"bot-bottle-agent-build-{digest}"
|
||||
ctx = f"/tmp/agent-build-{digest}"
|
||||
smoke_ctr, export_ctr = f"{tag}-smoke", f"{tag}-export"
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
"""The per-host infra service for the Firecracker backend (PRD 0070).
|
||||
|
||||
`FirecrackerInfraService` is the Firecracker `InfraService`: it composes the
|
||||
orchestrator + gateway microVMs as an idempotent per-host singleton pair over
|
||||
the shared `infra_vm` substrate (boot primitives, the singleton flock, the
|
||||
adopt/version machinery). Unlike the docker/macOS services it takes no
|
||||
per-instance config — the firecracker pair is a **hard** per-host singleton,
|
||||
pinned to the fixed netpool links (`orch_slot()` / `gw_slot()`), host cache
|
||||
paths, and a `booted-version` marker, so two pairs can't coexist on one host.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ...log import info
|
||||
from ..infra_service import InfraService
|
||||
from ...orchestrator.lifecycle import DEFAULT_STARTUP_TIMEOUT_SECONDS
|
||||
from . import infra_vm
|
||||
from .gateway import FirecrackerGateway
|
||||
from .orchestrator import FirecrackerOrchestrator
|
||||
|
||||
|
||||
class FirecrackerInfraService(InfraService):
|
||||
"""Bring up the orchestrator + gateway microVM pair as a per-host singleton."""
|
||||
|
||||
def orchestrator(self) -> FirecrackerOrchestrator:
|
||||
"""The control-plane service (adopts the running orchestrator VM by its
|
||||
fixed link; no live handle needed for URL / SSH / health)."""
|
||||
return FirecrackerOrchestrator()
|
||||
|
||||
def gateway(self) -> FirecrackerGateway:
|
||||
"""The data-plane service (adopts the running gateway VM by its fixed
|
||||
link; CA fetch / provisioning go through the stable key)."""
|
||||
return FirecrackerGateway()
|
||||
|
||||
def ensure_running(
|
||||
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
) -> str:
|
||||
"""Idempotent per-host singleton pair. Adopt the running VMs if the
|
||||
orchestrator's control plane is healthy AND the gateway VM is alive AND
|
||||
both booted the CURRENT code version (a prior launcher booted them — they
|
||||
outlive short-lived `start` processes); otherwise clear any stale VMs and
|
||||
boot a fresh pair (orchestrator first, then the gateway that resolves
|
||||
policy against it). Returns the host control-plane URL.
|
||||
|
||||
Concurrency-safe: the cold stop/build/boot path is serialized by a host
|
||||
flock, so two simultaneous first launches don't both boot on the same
|
||||
links/PIDs. The healthy fast-path takes no lock."""
|
||||
key = infra_vm._infra_dir() / "id_ed25519"
|
||||
orchestrator = self.orchestrator()
|
||||
url = orchestrator.url()
|
||||
want = infra_vm.expected_version()
|
||||
if infra_vm.adoptable(key, url, want):
|
||||
info(f"adopting running infra VMs (orchestrator at {url})")
|
||||
return url
|
||||
|
||||
with infra_vm.singleton_lock():
|
||||
# Re-check under the lock: another launcher may have booted them while
|
||||
# we waited (double-checked, so we adopt not re-boot).
|
||||
if infra_vm.adoptable(key, url, want):
|
||||
info(f"adopting running infra VMs (orchestrator at {url})")
|
||||
return url
|
||||
# Clear stale/hung/OUTDATED VMs holding either link before booting fresh.
|
||||
infra_vm.stop()
|
||||
infra_vm.ensure_built()
|
||||
# Orchestrator first — the gateway daemons reach the control plane at
|
||||
# startup. Each service owns its own boot; the orchestrator (which
|
||||
# holds the signing key) mints the role-scoped `gateway` JWT for the
|
||||
# gateway, which never sees the key (#469).
|
||||
orchestrator.ensure_running(startup_timeout=startup_timeout)
|
||||
self.gateway().connect_to_orchestrator(
|
||||
orchestrator.gateway_url(), orchestrator.mint_gateway_token())
|
||||
infra_vm.record_booted_version(want)
|
||||
return url
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop both infra VMs (idempotent) and drop the version marker."""
|
||||
infra_vm.stop()
|
||||
|
||||
|
||||
__all__ = ["FirecrackerInfraService"]
|
||||
@@ -40,16 +40,12 @@ import urllib.request
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Generator
|
||||
from typing import Generator
|
||||
|
||||
from ...log import die, info
|
||||
from ..docker import util as docker_mod
|
||||
from . import firecracker_vm, infra_artifact, netpool, util
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .gateway import FirecrackerGateway
|
||||
from .orchestrator import FirecrackerOrchestrator
|
||||
|
||||
# The orchestrator VM's signing-key path on its persistent /dev/vdb volume
|
||||
# (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds this
|
||||
# file with the host-canonical key AFTER boot (over SSH), so the VM verifies
|
||||
@@ -89,7 +85,6 @@ GIT_HTTP_PORT = 9420
|
||||
# now; routing DNS through a filtered path is a later refinement.
|
||||
_INFRA_RESOLVER = "1.1.1.1"
|
||||
|
||||
_CA_TIMEOUT_SECONDS = 30.0
|
||||
# How long the launcher retries pushing a secret while the guest's SSH comes
|
||||
# up. Below the init's own wait window, so a failed push dies here first.
|
||||
_SECRET_PUSH_TIMEOUT_SECONDS = 30.0
|
||||
@@ -108,24 +103,6 @@ class InfraVm:
|
||||
vm: firecracker_vm.VmHandle | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class InfraEndpoint:
|
||||
"""How to reach the running per-host pair. `orchestrator` is the control-plane
|
||||
`Orchestrator` service (host CLI + registration + in-VM builds); `gateway` is
|
||||
the data-plane `Gateway` service (agent egress / git-http / supervise, CA
|
||||
fetch, git-gate provisioning). Mirrors the docker/macOS `InfraEndpoint`."""
|
||||
|
||||
orchestrator: "FirecrackerOrchestrator"
|
||||
gateway: "FirecrackerGateway"
|
||||
|
||||
@property
|
||||
def orchestrator_url(self) -> str:
|
||||
return self.orchestrator.url()
|
||||
|
||||
def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str:
|
||||
return self.gateway.ca_cert_pem(timeout=timeout)
|
||||
|
||||
|
||||
def role_init(role: str) -> str:
|
||||
"""The guest PID-1 init for `role` (each per-plane rootfs bakes only its
|
||||
own — no `bb_role` branch, since the rootfs *is* the role)."""
|
||||
@@ -177,67 +154,8 @@ def build_rootfs_dir(role: str) -> Path:
|
||||
)
|
||||
|
||||
|
||||
def ensure_running() -> InfraEndpoint:
|
||||
"""Idempotent per-host singleton pair. Adopt the running VMs if the
|
||||
orchestrator's control plane is healthy AND the gateway VM is alive AND
|
||||
both booted the CURRENT code version (a prior launcher booted them — they
|
||||
outlive short-lived `start` processes); otherwise clear any stale VMs and
|
||||
boot a fresh pair (orchestrator first, then the gateway that resolves
|
||||
policy against it). Returns handles usable for builds / CA fetch / git-gate
|
||||
provisioning.
|
||||
|
||||
Concurrency-safe: the cold stop/build/boot path is serialized by a host
|
||||
flock, so two simultaneous first launches don't both boot on the same
|
||||
links/PIDs. The healthy fast-path takes no lock."""
|
||||
# Local imports: the orchestrator + gateway services import this module for
|
||||
# the shared VM substrate, so importing them at module scope would cycle.
|
||||
from .orchestrator import FirecrackerOrchestrator
|
||||
from .gateway import FirecrackerGateway
|
||||
|
||||
key = _infra_dir() / "id_ed25519"
|
||||
orchestrator = FirecrackerOrchestrator()
|
||||
url = orchestrator.url()
|
||||
want = _expected_version()
|
||||
if _adoptable(key, url, want):
|
||||
info(f"adopting running infra VMs (orchestrator at {url})")
|
||||
return _adopt(key)
|
||||
|
||||
with _singleton_lock():
|
||||
# Re-check under the lock: another launcher may have booted them while
|
||||
# we waited for the lock (double-checked, so we adopt not re-boot).
|
||||
if _adoptable(key, url, want):
|
||||
info(f"adopting running infra VMs (orchestrator at {url})")
|
||||
return _adopt(key)
|
||||
# Clear stale/hung/OUTDATED VMs holding either link before booting fresh.
|
||||
stop()
|
||||
ensure_built()
|
||||
# Orchestrator first — the gateway daemons reach the control plane at
|
||||
# startup. Each service owns its own boot; the orchestrator (which holds
|
||||
# the signing key) mints the role-scoped `gateway` JWT for the gateway,
|
||||
# which never sees the key (#469).
|
||||
orchestrator.ensure_running()
|
||||
gateway = FirecrackerGateway()
|
||||
gateway.connect_to_orchestrator(
|
||||
orchestrator.gateway_url(), orchestrator.mint_gateway_token())
|
||||
_record_booted_version(want)
|
||||
return InfraEndpoint(orchestrator=orchestrator, gateway=gateway)
|
||||
|
||||
|
||||
def _adopt(key: Path) -> "InfraEndpoint":
|
||||
"""Build the service handles for the already-running pair from the stable key
|
||||
+ the fixed links (no live VM handle — teardown / CA fetch go through the PID
|
||||
files + stable key)."""
|
||||
from .orchestrator import FirecrackerOrchestrator
|
||||
from .gateway import FirecrackerGateway
|
||||
return InfraEndpoint(
|
||||
orchestrator=FirecrackerOrchestrator(),
|
||||
gateway=FirecrackerGateway(
|
||||
InfraVm(guest_ip=netpool.gw_slot().guest_ip, private_key=key)),
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _singleton_lock() -> Generator[None, None, None]:
|
||||
def singleton_lock() -> Generator[None, None, None]:
|
||||
"""Host-level exclusive lock serializing the infra pair's cold create path
|
||||
(`stop`/`ensure_built`/`boot`). flock auto-releases if the launcher
|
||||
crashes, so the lock is never leaked."""
|
||||
@@ -336,13 +254,13 @@ def _version_file() -> Path:
|
||||
return _infra_dir() / "booted-version"
|
||||
|
||||
|
||||
def _expected_version() -> str:
|
||||
def expected_version() -> str:
|
||||
"""The combined marker for the running pair: both per-plane artifact
|
||||
versions, so a change to either rootfs dislodges the adopted pair."""
|
||||
return " ".join(f"{role}={_role_version(role)}" for role in infra_artifact.ROLES)
|
||||
|
||||
|
||||
def _adoptable(key: Path, url: str, want: str) -> bool:
|
||||
def adoptable(key: Path, url: str, want: str) -> bool:
|
||||
"""Adopt the running pair only if it booted from the CURRENT version, the
|
||||
orchestrator's control plane is healthy, and the gateway VM is still alive.
|
||||
A missing/mismatched marker means a prior launcher booted an older infra
|
||||
@@ -361,7 +279,7 @@ def _adoptable(key: Path, url: str, want: str) -> bool:
|
||||
return _pidfile_alive(_gw_dir())
|
||||
|
||||
|
||||
def _record_booted_version(version: str) -> None:
|
||||
def record_booted_version(version: str) -> None:
|
||||
_version_file().write_text(version + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
|
||||
@@ -7,12 +7,11 @@ volume (/dev/vdb — the sole opener of `bot-bottle.db`), seeding the host-canon
|
||||
signing key over SSH, and waiting for `/health`. The host CLI reaches it at its
|
||||
guest IP, and so does the gateway (`bb_orch` cmdline), so `url()` == `gateway_url()`.
|
||||
|
||||
What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM
|
||||
substrate the gateway VM also uses — booting a VM from a per-plane rootfs
|
||||
(`boot_vm`), the stable SSH keypair, the secret-push retry, the PID lifecycle —
|
||||
plus the pair coordinator (`ensure_running`: adopt-or-boot-both under a singleton
|
||||
lock with a combined version marker) and the per-plane guest inits (`role_init`).
|
||||
Those are the shared seam.
|
||||
The plane-agnostic VM substrate the gateway VM also uses stays in `infra_vm` —
|
||||
booting a VM from a per-plane rootfs (`boot_vm`), the stable SSH keypair, the
|
||||
secret-push retry, the PID lifecycle, the adoption/version helpers, and the
|
||||
per-plane guest inits (`role_init`). The pair coordinator (adopt-or-boot-both
|
||||
under a singleton flock) is `FirecrackerInfraService` (`infra.py`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -90,7 +89,7 @@ class FirecrackerOrchestrator(Orchestrator):
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop only the orchestrator VM (idempotent). A dead orchestrator fails
|
||||
`infra_vm._adoptable`'s health check, so no version marker to clear."""
|
||||
`infra_vm.adoptable`'s health check, so no version marker to clear."""
|
||||
infra_vm._kill_pidfile(infra_vm._orch_dir())
|
||||
infra_vm._pid_file(infra_vm._orch_dir()).unlink(missing_ok=True)
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
"""The per-host infra service contract (PRD 0070).
|
||||
|
||||
`InfraService` is the backend-neutral composer of the per-host **pair**: the
|
||||
`Orchestrator` (control plane) + `Gateway` (data plane) services, brought up as
|
||||
an idempotent per-host singleton. One concrete impl per backend
|
||||
(`backend/*/infra.py`), mirroring how `Orchestrator` and `Gateway` each have a
|
||||
per-backend impl.
|
||||
|
||||
The two backend-specific accessors (`orchestrator()` / `gateway()`) are the
|
||||
source of truth for how to reach each plane; the convenience `url()` /
|
||||
`is_healthy()` just delegate to the orchestrator.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
|
||||
from ..gateway import Gateway
|
||||
from ..orchestrator.lifecycle import DEFAULT_STARTUP_TIMEOUT_SECONDS, Orchestrator
|
||||
|
||||
|
||||
class InfraService(abc.ABC):
|
||||
"""Compose + bring up the per-host orchestrator + gateway pair.
|
||||
|
||||
Backend-neutral: docker/macOS run the pair as two containers, firecracker as
|
||||
two microVMs. Callers read reach-info off `orchestrator()` / `gateway()`."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def orchestrator(self) -> Orchestrator:
|
||||
"""The control-plane service. Cheap to reconstruct."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def gateway(self) -> Gateway:
|
||||
"""The data-plane service. Cheap to reconstruct."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def ensure_running(
|
||||
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
) -> str:
|
||||
"""Bring the orchestrator + gateway pair up (idempotent per-host
|
||||
singleton — a healthy, current pair is left running). Returns the host
|
||||
control-plane URL. Raises `OrchestratorStartError` on control-plane
|
||||
startup timeout."""
|
||||
|
||||
@abc.abstractmethod
|
||||
def stop(self) -> None:
|
||||
"""Remove both the orchestrator and the gateway. Idempotent."""
|
||||
|
||||
def url(self) -> str:
|
||||
"""The host-facing control-plane URL the CLI reaches (the orchestrator's)."""
|
||||
return self.orchestrator().url()
|
||||
|
||||
def is_healthy(self) -> bool:
|
||||
"""True iff the control plane answers `/health`."""
|
||||
return self.orchestrator().is_healthy()
|
||||
|
||||
|
||||
__all__ = ["InfraService"]
|
||||
@@ -106,7 +106,7 @@ class MacosContainerBottleBackend(
|
||||
(`supervise`) call when no control plane is running yet. Mirrors
|
||||
firecracker's infra bring-up."""
|
||||
from .infra import MacosInfraService
|
||||
return MacosInfraService().ensure_running().orchestrator_url
|
||||
return MacosInfraService().ensure_running()
|
||||
|
||||
def prepare_cleanup(self) -> MacosContainerBottleCleanupPlan:
|
||||
return _cleanup.prepare_cleanup()
|
||||
|
||||
@@ -87,10 +87,10 @@ def ensure_gateway(
|
||||
launches share it. Call before starting the agent container: the agent's
|
||||
proxy env needs `gateway_ip` at run time."""
|
||||
service = service or MacosInfraService()
|
||||
infra = service.ensure_running()
|
||||
orchestrator_url = service.ensure_running()
|
||||
endpoint = GatewayEndpoint(
|
||||
orchestrator_url=infra.orchestrator_url,
|
||||
gateway_ip=infra.gateway_ip,
|
||||
orchestrator_url=orchestrator_url,
|
||||
gateway_ip=service.gateway().address(),
|
||||
gateway_ca_pem=service.ca_cert_pem(),
|
||||
network=service.network,
|
||||
)
|
||||
|
||||
@@ -22,7 +22,6 @@ so they have no route to the control plane (the L3 block, not just the JWT).
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ...orchestrator.lifecycle import (
|
||||
@@ -30,6 +29,7 @@ from ...orchestrator.lifecycle import (
|
||||
DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
OrchestratorStartError,
|
||||
)
|
||||
from ..infra_service import InfraService
|
||||
from . import util as container_mod
|
||||
from .gateway import (
|
||||
CONTROL_NETWORK,
|
||||
@@ -59,19 +59,10 @@ INFRA_DB_VOLUME = ORCHESTRATOR_DB_VOLUME
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InfraEndpoint:
|
||||
"""How to reach the running pair. `orchestrator_url` is the orchestrator's
|
||||
control-network address (host CLI + registration); `gateway_ip` is the
|
||||
gateway's agent-network address (proxy / git-http / MCP target)."""
|
||||
|
||||
orchestrator_url: str
|
||||
gateway_ip: str
|
||||
|
||||
|
||||
class MacosInfraService:
|
||||
class MacosInfraService(InfraService):
|
||||
"""Composes the per-host orchestrator + gateway containers. Callers use
|
||||
`ensure_running()` (returns the endpoint) and `ca_cert_pem()`."""
|
||||
`ensure_running()` (returns the control-plane URL), the `orchestrator()` /
|
||||
`gateway()` accessors, and `ca_cert_pem()`."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -129,20 +120,12 @@ class MacosInfraService:
|
||||
repo_root=self._repo_root,
|
||||
)
|
||||
|
||||
def _resolve_gateway_ip(self) -> str:
|
||||
"""The gateway's agent-network address, or "" while it has none."""
|
||||
return container_mod.try_container_ipv4_on_network(
|
||||
self._gateway_name, self.network)
|
||||
|
||||
def is_healthy(self) -> bool:
|
||||
return self.orchestrator().is_healthy()
|
||||
|
||||
def ensure_running(
|
||||
self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
|
||||
) -> InfraEndpoint:
|
||||
"""Ensure the orchestrator + gateway containers are up; return how to
|
||||
reach them. Idempotent per-host singleton — a healthy orchestrator on
|
||||
current source is left untouched. Raises `OrchestratorStartError` on
|
||||
) -> str:
|
||||
"""Ensure the orchestrator + gateway containers are up; return the host
|
||||
control-plane URL. Idempotent per-host singleton — a healthy orchestrator
|
||||
on current source is left untouched. Raises `OrchestratorStartError` on
|
||||
control-plane startup timeout."""
|
||||
# The networks (host-only agent + NAT egress + host-only control) are a
|
||||
# shared concern — create them before either plane comes up.
|
||||
@@ -162,7 +145,7 @@ class MacosInfraService:
|
||||
# (#469).
|
||||
url = orchestrator.url()
|
||||
gateway.connect_to_orchestrator(url, orchestrator.mint_gateway_token())
|
||||
return InfraEndpoint(orchestrator_url=url, gateway_ip=self._resolve_gateway_ip())
|
||||
return url
|
||||
|
||||
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
||||
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
|
||||
@@ -178,7 +161,6 @@ class MacosInfraService:
|
||||
|
||||
__all__ = [
|
||||
"MacosInfraService",
|
||||
"InfraEndpoint",
|
||||
"OrchestratorStartError",
|
||||
"GatewayError",
|
||||
"ORCHESTRATOR_NAME",
|
||||
|
||||
@@ -130,7 +130,7 @@ class TestDockerOrchestratorAuthIntegration(unittest.TestCase):
|
||||
# than hand-rolling urllib here; _request (not one of the named
|
||||
# wrapper methods) is what exposes raw status codes for arbitrary
|
||||
# paths/tokens, which is exactly what these auth-boundary tests need.
|
||||
client = OrchestratorClient(self.svc.url, auth_token=token)
|
||||
client = OrchestratorClient(self.svc.url(), auth_token=token)
|
||||
return client._request(method, path) # pylint: disable=protected-access
|
||||
|
||||
def test_health_is_open_without_a_token(self) -> None:
|
||||
|
||||
@@ -442,8 +442,7 @@ class TestIsBackendReady(unittest.TestCase):
|
||||
|
||||
class TestEnsureOrchestrator(unittest.TestCase):
|
||||
"""The backend-agnostic orchestrator bring-up entry point. Docker starts
|
||||
the orchestrator + gateway containers; firecracker boots the infra VM;
|
||||
macos-container starts the infra container."""
|
||||
each backend's InfraService brings up its orchestrator + gateway pair."""
|
||||
|
||||
def test_docker_delegates_to_infra_service(self):
|
||||
b = get_bottle_backend("docker")
|
||||
@@ -457,23 +456,23 @@ class TestEnsureOrchestrator(unittest.TestCase):
|
||||
self.assertEqual(url, "http://127.0.0.1:8099")
|
||||
service_cls.return_value.ensure_running.assert_called_once_with()
|
||||
|
||||
def test_firecracker_delegates_to_infra_vm(self):
|
||||
def test_firecracker_delegates_to_infra_service(self):
|
||||
b = get_bottle_backend("firecracker")
|
||||
with patch(
|
||||
"bot_bottle.backend.firecracker.infra_vm.ensure_running"
|
||||
) as ensure_running:
|
||||
ensure_running.return_value.orchestrator_url = (
|
||||
"bot_bottle.backend.firecracker.infra.FirecrackerInfraService"
|
||||
) as service_cls:
|
||||
service_cls.return_value.ensure_running.return_value = (
|
||||
"http://10.243.255.1:8099"
|
||||
)
|
||||
url = b.ensure_orchestrator()
|
||||
self.assertEqual(url, "http://10.243.255.1:8099")
|
||||
|
||||
def test_macos_delegates_to_infra_container(self):
|
||||
def test_macos_delegates_to_infra_service(self):
|
||||
b = get_bottle_backend("macos-container")
|
||||
with patch(
|
||||
"bot_bottle.backend.macos_container.infra.MacosInfraService"
|
||||
) as service_cls:
|
||||
service_cls.return_value.ensure_running.return_value.orchestrator_url = (
|
||||
service_cls.return_value.ensure_running.return_value = (
|
||||
"http://192.168.128.2:8099"
|
||||
)
|
||||
url = b.ensure_orchestrator()
|
||||
|
||||
@@ -37,7 +37,7 @@ class TestDockerInfraService(unittest.TestCase):
|
||||
self.assertEqual(GATEWAY_NAME, self.svc.gateway_name)
|
||||
|
||||
def test_url_delegates_to_the_orchestrator(self) -> None:
|
||||
self.assertEqual("http://127.0.0.1:8099", self.svc.url)
|
||||
self.assertEqual("http://127.0.0.1:8099", self.svc.url())
|
||||
|
||||
def test_is_healthy_delegates_to_the_orchestrator(self) -> None:
|
||||
self.orch.is_healthy.return_value = True
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Unit: FirecrackerInfraService composes the orchestrator + gateway microVM pair.
|
||||
|
||||
The two-VM singleton lifecycle (adopt-or-boot-both under a flock, the version
|
||||
marker) that must hold without a live VM. The VM substrate + boot primitives are
|
||||
tested in test_firecracker_infra_vm; the orchestrator/gateway services in their
|
||||
own modules. These isolate the *composition*."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from bot_bottle.backend.firecracker import infra_vm
|
||||
from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
|
||||
from bot_bottle.backend.firecracker.infra import FirecrackerInfraService
|
||||
from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator
|
||||
|
||||
# The service imports the classes into its own module namespace, so patch them
|
||||
# there (not at their home modules).
|
||||
_ORCH_CLS = "bot_bottle.backend.firecracker.infra.FirecrackerOrchestrator"
|
||||
_GW_CLS = "bot_bottle.backend.firecracker.infra.FirecrackerGateway"
|
||||
|
||||
|
||||
class TestAccessors(unittest.TestCase):
|
||||
def test_orchestrator_and_gateway_services(self) -> None:
|
||||
svc = FirecrackerInfraService()
|
||||
self.assertIsInstance(svc.orchestrator(), FirecrackerOrchestrator)
|
||||
self.assertIsInstance(svc.gateway(), FirecrackerGateway)
|
||||
|
||||
def test_stop_stops_both_vms(self) -> None:
|
||||
with patch.object(infra_vm, "stop") as stop:
|
||||
FirecrackerInfraService().stop()
|
||||
stop.assert_called_once()
|
||||
|
||||
def test_url_and_is_healthy_delegate_to_the_orchestrator(self) -> None:
|
||||
svc = FirecrackerInfraService()
|
||||
ip = infra_vm.netpool.orch_slot().guest_ip
|
||||
self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", svc.url())
|
||||
|
||||
|
||||
class TestEnsureRunning(unittest.TestCase):
|
||||
def test_adopts_when_healthy_alive_and_version_matches(self):
|
||||
# Healthy control plane + live gateway + existing key + matching version
|
||||
# marker -> adopt (no stop/build/boot); returns the orchestrator URL.
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
d = Path(td)
|
||||
(d / "id_ed25519").write_text("k")
|
||||
(d / "booted-version").write_text("v-current\n")
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=d), \
|
||||
patch.object(infra_vm, "expected_version", return_value="v-current"), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=True), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built") as built:
|
||||
url = FirecrackerInfraService().ensure_running()
|
||||
stop.assert_not_called()
|
||||
built.assert_not_called()
|
||||
ip = infra_vm.netpool.orch_slot().guest_ip
|
||||
self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", url)
|
||||
|
||||
def test_reboots_both_when_version_stale(self):
|
||||
# Healthy control plane but the running pair booted an OLDER image
|
||||
# (marker mismatch) -> reboot rather than adopt stale code.
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
d = Path(td)
|
||||
(d / "id_ed25519").write_text("k")
|
||||
(d / "booted-version").write_text("v-old\n")
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=d), \
|
||||
patch.object(infra_vm, "expected_version", return_value="v-current"), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=True), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built"), \
|
||||
patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
|
||||
orch_cls.return_value.gateway_url.return_value = "http://10.243.255.1:8099"
|
||||
orch_cls.return_value.mint_gateway_token.return_value = "gw.jwt"
|
||||
FirecrackerInfraService().ensure_running()
|
||||
stop.assert_called_once() # dislodge the outdated pair
|
||||
orch_cls.return_value.ensure_running.assert_called_once()
|
||||
# The gateway service is bound to the orchestrator's gateway URL,
|
||||
# carrying the orchestrator-minted `gateway` token.
|
||||
gw_cls.return_value.connect_to_orchestrator.assert_called_once_with(
|
||||
"http://10.243.255.1:8099", "gw.jwt")
|
||||
# The fresh boot records the current version for the next launcher.
|
||||
self.assertEqual("v-current\n", (d / "booted-version").read_text())
|
||||
|
||||
def test_reboots_when_gateway_dead(self):
|
||||
# Orchestrator healthy + marker current, but the gateway VM is gone ->
|
||||
# the pair is half-down, so reboot both rather than adopt partial state.
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
d = Path(td)
|
||||
(d / "id_ed25519").write_text("k")
|
||||
(d / "booted-version").write_text("v-current\n")
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=d), \
|
||||
patch.object(infra_vm, "expected_version", return_value="v-current"), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=False), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built"), \
|
||||
patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
|
||||
FirecrackerInfraService().ensure_running()
|
||||
stop.assert_called_once()
|
||||
orch_cls.return_value.ensure_running.assert_called_once()
|
||||
gw_cls.return_value.connect_to_orchestrator.assert_called_once()
|
||||
|
||||
def test_boots_both_when_no_running_pair(self):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=Path(td)), \
|
||||
patch.object(infra_vm, "expected_version", return_value="v-current"), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=False), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built") as built, \
|
||||
patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls:
|
||||
FirecrackerInfraService().ensure_running()
|
||||
stop.assert_called_once() # clear stale VMs first
|
||||
built.assert_called_once()
|
||||
# Orchestrator is brought up BEFORE the gateway is connected (its daemons
|
||||
# reach the control plane at startup).
|
||||
orch_cls.return_value.ensure_running.assert_called_once()
|
||||
gw_cls.return_value.connect_to_orchestrator.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,8 +1,8 @@
|
||||
"""Unit tests for the Firecracker infra VMs (orchestrator + gateway boot).
|
||||
"""Unit tests for the Firecracker infra-VM substrate (boot primitives + inits).
|
||||
|
||||
The KVM boot / HTTP reachability is integration-tested on a KVM host; here we
|
||||
cover the URL shape, the shared-rootfs role wiring, the secret-push decisions,
|
||||
and the two-VM singleton lifecycle that must hold without a VM.
|
||||
cover the per-plane role inits, rootfs build, the secret-push retry, and the
|
||||
pidfile/adoption helpers. The pair coordinator is tested in test_firecracker_infra.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -43,31 +43,6 @@ class TestPushSecret(unittest.TestCase):
|
||||
die.assert_called_once()
|
||||
|
||||
|
||||
class TestInfraEndpoint(unittest.TestCase):
|
||||
"""The endpoint mirrors docker/macOS: it exposes the orchestrator service's
|
||||
URL and delegates the CA fetch to the gateway service."""
|
||||
|
||||
def _endpoint(self) -> infra_vm.InfraEndpoint:
|
||||
from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
|
||||
from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator
|
||||
return infra_vm.InfraEndpoint(
|
||||
orchestrator=FirecrackerOrchestrator(),
|
||||
gateway=FirecrackerGateway(
|
||||
infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))),
|
||||
)
|
||||
|
||||
def test_orchestrator_url_is_the_orchestrator_service(self):
|
||||
ep = self._endpoint()
|
||||
ip = infra_vm.netpool.orch_slot().guest_ip
|
||||
self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", ep.orchestrator_url)
|
||||
|
||||
def test_gateway_ca_pem_delegates_to_the_gateway_service(self):
|
||||
ep = self._endpoint()
|
||||
with patch.object(ep.gateway, "ca_cert_pem", return_value="PEM") as ca:
|
||||
self.assertEqual("PEM", ep.gateway_ca_pem(timeout=1))
|
||||
ca.assert_called_once_with(timeout=1)
|
||||
|
||||
|
||||
class TestRoleInits(unittest.TestCase):
|
||||
def test_orchestrator_init_starts_only_the_control_plane(self):
|
||||
init = infra_vm.role_init("orchestrator")
|
||||
@@ -143,105 +118,6 @@ class TestEnsureBuilt(unittest.TestCase):
|
||||
tags.index(infra_vm._ORCHESTRATOR_FC_IMAGE))
|
||||
|
||||
|
||||
# The orchestrator + gateway services are imported lazily inside ensure_running
|
||||
# / _adopt (to break the infra_vm <-> service import cycle), so patch them at
|
||||
# their home modules.
|
||||
_ORCH_CLS = "bot_bottle.backend.firecracker.orchestrator.FirecrackerOrchestrator"
|
||||
_GW_CLS = "bot_bottle.backend.firecracker.gateway.FirecrackerGateway"
|
||||
|
||||
|
||||
class TestEnsureRunningSingleton(unittest.TestCase):
|
||||
def test_adopts_when_healthy_alive_and_version_matches(self):
|
||||
# Healthy control plane + live gateway + existing key + matching version
|
||||
# marker -> adopt (no boot).
|
||||
from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
|
||||
from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
d = Path(td)
|
||||
(d / "id_ed25519").write_text("k")
|
||||
(d / "booted-version").write_text("v-current\n")
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=d), \
|
||||
patch.object(infra_vm, "_expected_version", return_value="v-current"), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=True):
|
||||
ep = infra_vm.ensure_running()
|
||||
# Adopted service handles addressing the two distinct infra links.
|
||||
self.assertIsInstance(ep.orchestrator, FirecrackerOrchestrator)
|
||||
self.assertEqual(
|
||||
infra_vm.netpool.orch_slot().guest_ip,
|
||||
ep.orchestrator_url.split("://")[1].split(":")[0])
|
||||
self.assertIsInstance(ep.gateway, FirecrackerGateway)
|
||||
self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, ep.gateway.address())
|
||||
|
||||
def test_reboots_both_when_version_stale(self):
|
||||
# Healthy control plane but the running pair booted an OLDER image
|
||||
# (marker mismatch) -> reboot rather than adopt stale code.
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
d = Path(td)
|
||||
(d / "id_ed25519").write_text("k")
|
||||
(d / "booted-version").write_text("v-old\n")
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=d), \
|
||||
patch.object(infra_vm, "_expected_version", return_value="v-current"), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=True), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built"), \
|
||||
patch(_ORCH_CLS) as orch_cls, \
|
||||
patch(_GW_CLS) as gw_cls:
|
||||
orch_cls.return_value.gateway_url.return_value = "http://10.243.255.1:8099"
|
||||
orch_cls.return_value.mint_gateway_token.return_value = "gw.jwt"
|
||||
infra_vm.ensure_running()
|
||||
stop.assert_called_once() # dislodge the outdated pair
|
||||
orch_cls.return_value.ensure_running.assert_called_once()
|
||||
# The gateway service is bound to the orchestrator's gateway URL,
|
||||
# carrying the orchestrator-minted `gateway` token.
|
||||
gw_cls.return_value.connect_to_orchestrator.assert_called_once_with(
|
||||
"http://10.243.255.1:8099", "gw.jwt")
|
||||
# The fresh boot records the current version for the next launcher.
|
||||
self.assertEqual("v-current\n", (d / "booted-version").read_text())
|
||||
|
||||
def test_reboots_when_gateway_dead(self):
|
||||
# Orchestrator healthy + marker current, but the gateway VM is gone ->
|
||||
# the pair is half-down, so reboot both rather than adopt partial state.
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
d = Path(td)
|
||||
(d / "id_ed25519").write_text("k")
|
||||
(d / "booted-version").write_text("v-current\n")
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=d), \
|
||||
patch.object(infra_vm, "_expected_version", return_value="v-current"), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=False), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built"), \
|
||||
patch(_ORCH_CLS) as orch_cls, \
|
||||
patch(_GW_CLS) as gw_cls:
|
||||
infra_vm.ensure_running()
|
||||
stop.assert_called_once()
|
||||
orch_cls.return_value.ensure_running.assert_called_once()
|
||||
gw_cls.return_value.connect_to_orchestrator.assert_called_once()
|
||||
|
||||
def test_boots_both_when_unhealthy(self):
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=Path(td)), \
|
||||
patch.object(infra_vm, "_expected_version", return_value="v-current"), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=False), \
|
||||
patch.object(infra_vm, "stop") as stop, \
|
||||
patch.object(infra_vm, "ensure_built") as built, \
|
||||
patch(_ORCH_CLS) as orch_cls, \
|
||||
patch(_GW_CLS) as gw_cls:
|
||||
infra_vm.ensure_running()
|
||||
stop.assert_called_once() # clear stale VMs first
|
||||
built.assert_called_once()
|
||||
# Orchestrator is brought up BEFORE the gateway is connected (its daemons
|
||||
# reach the control plane at startup).
|
||||
orch_cls.return_value.ensure_running.assert_called_once()
|
||||
gw_cls.return_value.connect_to_orchestrator.assert_called_once()
|
||||
|
||||
|
||||
class TestStop(unittest.TestCase):
|
||||
def test_stops_both_vms_and_drops_marker(self):
|
||||
import tempfile
|
||||
@@ -306,21 +182,21 @@ class TestAdoptable(unittest.TestCase):
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=d), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=True):
|
||||
self.assertTrue(infra_vm._adoptable(d / "id_ed25519", "u", "v1"))
|
||||
self.assertTrue(infra_vm.adoptable(d / "id_ed25519", "u", "v1"))
|
||||
|
||||
def test_false_when_key_missing(self):
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
d = self._dir(td, key=False, version="v1")
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=d):
|
||||
self.assertFalse(infra_vm._adoptable(d / "id_ed25519", "u", "v1"))
|
||||
self.assertFalse(infra_vm.adoptable(d / "id_ed25519", "u", "v1"))
|
||||
|
||||
def test_false_when_no_version_marker(self):
|
||||
import tempfile
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
d = self._dir(td) # key present, no booted-version
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=d):
|
||||
self.assertFalse(infra_vm._adoptable(d / "id_ed25519", "u", "v1"))
|
||||
self.assertFalse(infra_vm.adoptable(d / "id_ed25519", "u", "v1"))
|
||||
|
||||
def test_false_when_version_mismatch(self):
|
||||
import tempfile
|
||||
@@ -329,7 +205,7 @@ class TestAdoptable(unittest.TestCase):
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=d), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=True):
|
||||
self.assertFalse(infra_vm._adoptable(d / "id_ed25519", "u", "v1"))
|
||||
self.assertFalse(infra_vm.adoptable(d / "id_ed25519", "u", "v1"))
|
||||
|
||||
def test_false_when_gateway_dead(self):
|
||||
import tempfile
|
||||
@@ -338,7 +214,7 @@ class TestAdoptable(unittest.TestCase):
|
||||
with patch.object(infra_vm, "_infra_dir", return_value=d), \
|
||||
patch.object(infra_vm, "_health_ok", return_value=True), \
|
||||
patch.object(infra_vm, "_pidfile_alive", return_value=False):
|
||||
self.assertFalse(infra_vm._adoptable(d / "id_ed25519", "u", "v1"))
|
||||
self.assertFalse(infra_vm.adoptable(d / "id_ed25519", "u", "v1"))
|
||||
|
||||
|
||||
class TestKillInfraFirecrackers(unittest.TestCase):
|
||||
|
||||
@@ -55,14 +55,13 @@ class TestEnsureGateway(unittest.TestCase):
|
||||
return ensure_gateway()
|
||||
|
||||
def _service(self) -> MagicMock:
|
||||
from bot_bottle.backend.macos_container.infra import InfraEndpoint
|
||||
service = MagicMock()
|
||||
# Two containers now: the orchestrator on the control network, the
|
||||
# gateway on the agent network — distinct addresses.
|
||||
service.ensure_running.return_value = InfraEndpoint(
|
||||
orchestrator_url="http://192.168.128.2:8099",
|
||||
gateway_ip="192.168.128.3",
|
||||
)
|
||||
# gateway on the agent network — distinct addresses. `ensure_running`
|
||||
# returns the control-plane URL; the gateway address comes off the
|
||||
# gateway service.
|
||||
service.ensure_running.return_value = "http://192.168.128.2:8099"
|
||||
service.gateway.return_value.address.return_value = "192.168.128.3"
|
||||
service.network = "bot-bottle-mac-gateway"
|
||||
service.ca_cert_pem.return_value = "PEM"
|
||||
return service
|
||||
|
||||
@@ -30,21 +30,17 @@ class TestInfraEnsureRunning(unittest.TestCase):
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
def test_composes_networks_builds_and_brings_up_orchestrator_first(self) -> None:
|
||||
with patch(f"{_INFRA}.ensure_networks") as nets, \
|
||||
patch(f"{_INFRA}.container_mod") as mod:
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.3"
|
||||
endpoint = self.svc.ensure_running()
|
||||
with patch(f"{_INFRA}.ensure_networks") as nets:
|
||||
url = self.svc.ensure_running()
|
||||
nets.assert_called_once()
|
||||
self.orch.ensure_built.assert_called_once()
|
||||
self.gw.ensure_built.assert_called_once()
|
||||
self.orch.ensure_running.assert_called_once()
|
||||
self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url)
|
||||
self.assertEqual("192.168.128.3", endpoint.gateway_ip)
|
||||
# Returns the host control-plane URL (the InfraService contract).
|
||||
self.assertEqual("http://192.168.128.2:8099", url)
|
||||
|
||||
def test_connects_gateway_with_orchestrator_url_and_minted_token(self) -> None:
|
||||
with patch(f"{_INFRA}.ensure_networks"), \
|
||||
patch(f"{_INFRA}.container_mod") as mod:
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.3"
|
||||
with patch(f"{_INFRA}.ensure_networks"):
|
||||
self.svc.ensure_running()
|
||||
self.gw.connect_to_orchestrator.assert_called_once_with(
|
||||
"http://192.168.128.2:8099", "gw.jwt")
|
||||
|
||||
Reference in New Issue
Block a user