diff --git a/bot_bottle/backend/docker/infra.py b/bot_bottle/backend/docker/infra.py index cc2d792d..d2e201bd 100644 --- a/bot_bottle/backend/docker/infra.py +++ b/bot_bottle/backend/docker/infra.py @@ -40,6 +40,7 @@ from ...gateway import ( GATEWAY_NAME, GATEWAY_NETWORK, ) +from ..infra_service import InfraService from ...orchestrator.lifecycle import ( DEFAULT_PORT, DEFAULT_STARTUP_TIMEOUT_SECONDS, @@ -53,7 +54,7 @@ INFRA_NAME = GATEWAY_NAME # the container agents attribute against is the gatew _REPO_ROOT = Path(__file__).resolve().parents[3] -class DockerInfraService: +class DockerInfraService(InfraService): """Composes the per-host control plane + gateway as two containers. `orchestrator_name` / `gateway_name` let callers run independent pairs on @@ -91,14 +92,6 @@ class DockerInfraService: against (gateway IP, git-gate provisioning transport, CA fetch).""" return self._gateway_name - @property - def url(self) -> str: - """Host-side control-plane URL (the orchestrator's published loopback).""" - return self.orchestrator().url() - - def is_healthy(self) -> bool: - return self.orchestrator().is_healthy() - def orchestrator(self) -> DockerOrchestrator: """The control-plane service. Cheap to reconstruct — `ensure_built` builds its image, `ensure_running` brings it up, and the launch flow diff --git a/bot_bottle/backend/firecracker/backend.py b/bot_bottle/backend/firecracker/backend.py index 7586a7a7..2cbdf456 100644 --- a/bot_bottle/backend/firecracker/backend.py +++ b/bot_bottle/backend/firecracker/backend.py @@ -123,5 +123,5 @@ class FirecrackerBottleBackend( return plan.agent_supervise_url def ensure_orchestrator(self) -> str: - from . import infra_vm - return infra_vm.ensure_running().orchestrator_url + from .infra import FirecrackerInfraService + return FirecrackerInfraService().ensure_running() diff --git a/bot_bottle/backend/firecracker/consolidated_launch.py b/bot_bottle/backend/firecracker/consolidated_launch.py index 3f8ce72b..05cf3cba 100644 --- a/bot_bottle/backend/firecracker/consolidated_launch.py +++ b/bot_bottle/backend/firecracker/consolidated_launch.py @@ -41,8 +41,9 @@ from ...orchestrator.lifecycle import ( from ...orchestrator.reprovision import reprovision_bottles from ...orchestrator.store.secret_store import ENV_VAR_SECRET_NAME from ..provision_bottle import deprovision_bottle, provision_bottle -from . import cleanup, infra_vm, util +from . import cleanup, util from .gateway import FirecrackerGateway +from .infra import FirecrackerInfraService _ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret" @@ -122,13 +123,13 @@ def launch_consolidated( """Ensure the infra VM is up, register the bottle by its guest IP, and provision its git-gate state into the gateway VM. Returns the context the agent-VM launch needs. Raises on failure — the caller tears down.""" - infra = infra_vm.ensure_running() - url = infra.orchestrator_url + service = FirecrackerInfraService() + url = service.ensure_running() client = OrchestratorClient(url) _reprovision_running_bottles(client) # Read the gateway's provisioning transport + CA off the Gateway service. - gateway = infra.gateway + gateway = service.gateway() transport = gateway.provisioning_transport() reg = provision_bottle( client, guest_ip, egress_plan, git_gate_plan, transport, diff --git a/bot_bottle/backend/firecracker/gateway.py b/bot_bottle/backend/firecracker/gateway.py index 2ef1e551..2c727fa1 100644 --- a/bot_bottle/backend/firecracker/gateway.py +++ b/bot_bottle/backend/firecracker/gateway.py @@ -7,14 +7,13 @@ fetching the mitmproxy CA, and the SSH exec/cp provisioning transport. The gateway VM never opens `bot-bottle.db` (#469), so it holds no signing key — only the token the host hands it via `connect_to_orchestrator`. -What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM -substrate the orchestrator VM also uses — booting a VM from a per-plane rootfs -(`boot_vm`), the stable SSH keypair, the secret-push retry loop, and the PID -lifecycle — plus the pair coordinator (`ensure_running`: orchestrator-first -health gate, singleton lock, adoption/version marker). The guest-side gateway -daemon startup lives in the gateway rootfs's guest init (`_gateway_init` in -`infra_vm`), so it can't live in a host-side method either. These are the shared -seam. +The plane-agnostic VM substrate the orchestrator VM also uses stays in +`infra_vm` — booting a VM from a per-plane rootfs (`boot_vm`), the stable SSH +keypair, the secret-push retry loop, the PID lifecycle, and the +adoption/version helpers. The guest-side gateway daemon startup lives in the +gateway rootfs's guest init (`_gateway_init` in `infra_vm`), so it can't live in +a host-side method either. The pair coordinator (orchestrator-first, under a +singleton flock) is `FirecrackerInfraService` (`infra.py`). """ from __future__ import annotations @@ -112,7 +111,7 @@ class FirecrackerGateway(Gateway): def stop(self) -> None: """Stop only the gateway VM (idempotent — absent is success). A dead - gateway already fails `infra_vm._adoptable`, so no version marker to + gateway already fails `infra_vm.adoptable`, so no version marker to clear here.""" infra_vm._kill_pidfile(infra_vm._gw_dir()) infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True) diff --git a/bot_bottle/backend/firecracker/image_builder.py b/bot_bottle/backend/firecracker/image_builder.py index 3f250ae6..77da0ace 100644 --- a/bot_bottle/backend/firecracker/image_builder.py +++ b/bot_bottle/backend/firecracker/image_builder.py @@ -26,7 +26,8 @@ from pathlib import Path from typing import Generator from ...log import die, info -from . import infra_vm, util +from . import util +from .infra import FirecrackerInfraService # vfs + chroot: buildah works as root in the microVM (no fuse-overlayfs / # overlay module / subuid maps). `--isolation` is a build/run-only flag; @@ -125,8 +126,9 @@ def _build_in_infra( orchestrator VM (the build-capable control plane), smoke test the image, and stream its rootfs into `base`. The infra VMs persist; only the per-build container/image/context are cleaned up.""" - orchestrator = infra_vm.ensure_running().orchestrator - key, ip = orchestrator.ssh_target() + service = FirecrackerInfraService() + service.ensure_running() + key, ip = service.orchestrator().ssh_target() tag = f"bot-bottle-agent-build-{digest}" ctx = f"/tmp/agent-build-{digest}" smoke_ctr, export_ctr = f"{tag}-smoke", f"{tag}-export" diff --git a/bot_bottle/backend/firecracker/infra.py b/bot_bottle/backend/firecracker/infra.py new file mode 100644 index 00000000..327a0018 --- /dev/null +++ b/bot_bottle/backend/firecracker/infra.py @@ -0,0 +1,80 @@ +"""The per-host infra service for the Firecracker backend (PRD 0070). + +`FirecrackerInfraService` is the Firecracker `InfraService`: it composes the +orchestrator + gateway microVMs as an idempotent per-host singleton pair over +the shared `infra_vm` substrate (boot primitives, the singleton flock, the +adopt/version machinery). Unlike the docker/macOS services it takes no +per-instance config — the firecracker pair is a **hard** per-host singleton, +pinned to the fixed netpool links (`orch_slot()` / `gw_slot()`), host cache +paths, and a `booted-version` marker, so two pairs can't coexist on one host. +""" + +from __future__ import annotations + +from ...log import info +from ..infra_service import InfraService +from ...orchestrator.lifecycle import DEFAULT_STARTUP_TIMEOUT_SECONDS +from . import infra_vm +from .gateway import FirecrackerGateway +from .orchestrator import FirecrackerOrchestrator + + +class FirecrackerInfraService(InfraService): + """Bring up the orchestrator + gateway microVM pair as a per-host singleton.""" + + def orchestrator(self) -> FirecrackerOrchestrator: + """The control-plane service (adopts the running orchestrator VM by its + fixed link; no live handle needed for URL / SSH / health).""" + return FirecrackerOrchestrator() + + def gateway(self) -> FirecrackerGateway: + """The data-plane service (adopts the running gateway VM by its fixed + link; CA fetch / provisioning go through the stable key).""" + return FirecrackerGateway() + + def ensure_running( + self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, + ) -> str: + """Idempotent per-host singleton pair. Adopt the running VMs if the + orchestrator's control plane is healthy AND the gateway VM is alive AND + both booted the CURRENT code version (a prior launcher booted them — they + outlive short-lived `start` processes); otherwise clear any stale VMs and + boot a fresh pair (orchestrator first, then the gateway that resolves + policy against it). Returns the host control-plane URL. + + Concurrency-safe: the cold stop/build/boot path is serialized by a host + flock, so two simultaneous first launches don't both boot on the same + links/PIDs. The healthy fast-path takes no lock.""" + key = infra_vm._infra_dir() / "id_ed25519" + orchestrator = self.orchestrator() + url = orchestrator.url() + want = infra_vm.expected_version() + if infra_vm.adoptable(key, url, want): + info(f"adopting running infra VMs (orchestrator at {url})") + return url + + with infra_vm.singleton_lock(): + # Re-check under the lock: another launcher may have booted them while + # we waited (double-checked, so we adopt not re-boot). + if infra_vm.adoptable(key, url, want): + info(f"adopting running infra VMs (orchestrator at {url})") + return url + # Clear stale/hung/OUTDATED VMs holding either link before booting fresh. + infra_vm.stop() + infra_vm.ensure_built() + # Orchestrator first — the gateway daemons reach the control plane at + # startup. Each service owns its own boot; the orchestrator (which + # holds the signing key) mints the role-scoped `gateway` JWT for the + # gateway, which never sees the key (#469). + orchestrator.ensure_running(startup_timeout=startup_timeout) + self.gateway().connect_to_orchestrator( + orchestrator.gateway_url(), orchestrator.mint_gateway_token()) + infra_vm.record_booted_version(want) + return url + + def stop(self) -> None: + """Stop both infra VMs (idempotent) and drop the version marker.""" + infra_vm.stop() + + +__all__ = ["FirecrackerInfraService"] diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index 9eddfb65..3686464f 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -40,16 +40,12 @@ import urllib.request from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Generator +from typing import Generator from ...log import die, info from ..docker import util as docker_mod from . import firecracker_vm, infra_artifact, netpool, util -if TYPE_CHECKING: - from .gateway import FirecrackerGateway - from .orchestrator import FirecrackerOrchestrator - # The orchestrator VM's signing-key path on its persistent /dev/vdb volume # (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds this # file with the host-canonical key AFTER boot (over SSH), so the VM verifies @@ -89,7 +85,6 @@ GIT_HTTP_PORT = 9420 # now; routing DNS through a filtered path is a later refinement. _INFRA_RESOLVER = "1.1.1.1" -_CA_TIMEOUT_SECONDS = 30.0 # How long the launcher retries pushing a secret while the guest's SSH comes # up. Below the init's own wait window, so a failed push dies here first. _SECRET_PUSH_TIMEOUT_SECONDS = 30.0 @@ -108,24 +103,6 @@ class InfraVm: vm: firecracker_vm.VmHandle | None = None -@dataclass -class InfraEndpoint: - """How to reach the running per-host pair. `orchestrator` is the control-plane - `Orchestrator` service (host CLI + registration + in-VM builds); `gateway` is - the data-plane `Gateway` service (agent egress / git-http / supervise, CA - fetch, git-gate provisioning). Mirrors the docker/macOS `InfraEndpoint`.""" - - orchestrator: "FirecrackerOrchestrator" - gateway: "FirecrackerGateway" - - @property - def orchestrator_url(self) -> str: - return self.orchestrator.url() - - def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str: - return self.gateway.ca_cert_pem(timeout=timeout) - - def role_init(role: str) -> str: """The guest PID-1 init for `role` (each per-plane rootfs bakes only its own — no `bb_role` branch, since the rootfs *is* the role).""" @@ -177,67 +154,8 @@ def build_rootfs_dir(role: str) -> Path: ) -def ensure_running() -> InfraEndpoint: - """Idempotent per-host singleton pair. Adopt the running VMs if the - orchestrator's control plane is healthy AND the gateway VM is alive AND - both booted the CURRENT code version (a prior launcher booted them — they - outlive short-lived `start` processes); otherwise clear any stale VMs and - boot a fresh pair (orchestrator first, then the gateway that resolves - policy against it). Returns handles usable for builds / CA fetch / git-gate - provisioning. - - Concurrency-safe: the cold stop/build/boot path is serialized by a host - flock, so two simultaneous first launches don't both boot on the same - links/PIDs. The healthy fast-path takes no lock.""" - # Local imports: the orchestrator + gateway services import this module for - # the shared VM substrate, so importing them at module scope would cycle. - from .orchestrator import FirecrackerOrchestrator - from .gateway import FirecrackerGateway - - key = _infra_dir() / "id_ed25519" - orchestrator = FirecrackerOrchestrator() - url = orchestrator.url() - want = _expected_version() - if _adoptable(key, url, want): - info(f"adopting running infra VMs (orchestrator at {url})") - return _adopt(key) - - with _singleton_lock(): - # Re-check under the lock: another launcher may have booted them while - # we waited for the lock (double-checked, so we adopt not re-boot). - if _adoptable(key, url, want): - info(f"adopting running infra VMs (orchestrator at {url})") - return _adopt(key) - # Clear stale/hung/OUTDATED VMs holding either link before booting fresh. - stop() - ensure_built() - # Orchestrator first — the gateway daemons reach the control plane at - # startup. Each service owns its own boot; the orchestrator (which holds - # the signing key) mints the role-scoped `gateway` JWT for the gateway, - # which never sees the key (#469). - orchestrator.ensure_running() - gateway = FirecrackerGateway() - gateway.connect_to_orchestrator( - orchestrator.gateway_url(), orchestrator.mint_gateway_token()) - _record_booted_version(want) - return InfraEndpoint(orchestrator=orchestrator, gateway=gateway) - - -def _adopt(key: Path) -> "InfraEndpoint": - """Build the service handles for the already-running pair from the stable key - + the fixed links (no live VM handle — teardown / CA fetch go through the PID - files + stable key).""" - from .orchestrator import FirecrackerOrchestrator - from .gateway import FirecrackerGateway - return InfraEndpoint( - orchestrator=FirecrackerOrchestrator(), - gateway=FirecrackerGateway( - InfraVm(guest_ip=netpool.gw_slot().guest_ip, private_key=key)), - ) - - @contextmanager -def _singleton_lock() -> Generator[None, None, None]: +def singleton_lock() -> Generator[None, None, None]: """Host-level exclusive lock serializing the infra pair's cold create path (`stop`/`ensure_built`/`boot`). flock auto-releases if the launcher crashes, so the lock is never leaked.""" @@ -336,13 +254,13 @@ def _version_file() -> Path: return _infra_dir() / "booted-version" -def _expected_version() -> str: +def expected_version() -> str: """The combined marker for the running pair: both per-plane artifact versions, so a change to either rootfs dislodges the adopted pair.""" return " ".join(f"{role}={_role_version(role)}" for role in infra_artifact.ROLES) -def _adoptable(key: Path, url: str, want: str) -> bool: +def adoptable(key: Path, url: str, want: str) -> bool: """Adopt the running pair only if it booted from the CURRENT version, the orchestrator's control plane is healthy, and the gateway VM is still alive. A missing/mismatched marker means a prior launcher booted an older infra @@ -361,7 +279,7 @@ def _adoptable(key: Path, url: str, want: str) -> bool: return _pidfile_alive(_gw_dir()) -def _record_booted_version(version: str) -> None: +def record_booted_version(version: str) -> None: _version_file().write_text(version + "\n", encoding="utf-8") diff --git a/bot_bottle/backend/firecracker/orchestrator.py b/bot_bottle/backend/firecracker/orchestrator.py index 4ed2d452..4b79808f 100644 --- a/bot_bottle/backend/firecracker/orchestrator.py +++ b/bot_bottle/backend/firecracker/orchestrator.py @@ -7,12 +7,11 @@ volume (/dev/vdb — the sole opener of `bot-bottle.db`), seeding the host-canon signing key over SSH, and waiting for `/health`. The host CLI reaches it at its guest IP, and so does the gateway (`bb_orch` cmdline), so `url()` == `gateway_url()`. -What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM -substrate the gateway VM also uses — booting a VM from a per-plane rootfs -(`boot_vm`), the stable SSH keypair, the secret-push retry, the PID lifecycle — -plus the pair coordinator (`ensure_running`: adopt-or-boot-both under a singleton -lock with a combined version marker) and the per-plane guest inits (`role_init`). -Those are the shared seam. +The plane-agnostic VM substrate the gateway VM also uses stays in `infra_vm` — +booting a VM from a per-plane rootfs (`boot_vm`), the stable SSH keypair, the +secret-push retry, the PID lifecycle, the adoption/version helpers, and the +per-plane guest inits (`role_init`). The pair coordinator (adopt-or-boot-both +under a singleton flock) is `FirecrackerInfraService` (`infra.py`). """ from __future__ import annotations @@ -90,7 +89,7 @@ class FirecrackerOrchestrator(Orchestrator): def stop(self) -> None: """Stop only the orchestrator VM (idempotent). A dead orchestrator fails - `infra_vm._adoptable`'s health check, so no version marker to clear.""" + `infra_vm.adoptable`'s health check, so no version marker to clear.""" infra_vm._kill_pidfile(infra_vm._orch_dir()) infra_vm._pid_file(infra_vm._orch_dir()).unlink(missing_ok=True) diff --git a/bot_bottle/backend/infra_service.py b/bot_bottle/backend/infra_service.py new file mode 100644 index 00000000..b35dbc45 --- /dev/null +++ b/bot_bottle/backend/infra_service.py @@ -0,0 +1,58 @@ +"""The per-host infra service contract (PRD 0070). + +`InfraService` is the backend-neutral composer of the per-host **pair**: the +`Orchestrator` (control plane) + `Gateway` (data plane) services, brought up as +an idempotent per-host singleton. One concrete impl per backend +(`backend/*/infra.py`), mirroring how `Orchestrator` and `Gateway` each have a +per-backend impl. + +The two backend-specific accessors (`orchestrator()` / `gateway()`) are the +source of truth for how to reach each plane; the convenience `url()` / +`is_healthy()` just delegate to the orchestrator. +""" + +from __future__ import annotations + +import abc + +from ..gateway import Gateway +from ..orchestrator.lifecycle import DEFAULT_STARTUP_TIMEOUT_SECONDS, Orchestrator + + +class InfraService(abc.ABC): + """Compose + bring up the per-host orchestrator + gateway pair. + + Backend-neutral: docker/macOS run the pair as two containers, firecracker as + two microVMs. Callers read reach-info off `orchestrator()` / `gateway()`.""" + + @abc.abstractmethod + def orchestrator(self) -> Orchestrator: + """The control-plane service. Cheap to reconstruct.""" + + @abc.abstractmethod + def gateway(self) -> Gateway: + """The data-plane service. Cheap to reconstruct.""" + + @abc.abstractmethod + def ensure_running( + self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, + ) -> str: + """Bring the orchestrator + gateway pair up (idempotent per-host + singleton — a healthy, current pair is left running). Returns the host + control-plane URL. Raises `OrchestratorStartError` on control-plane + startup timeout.""" + + @abc.abstractmethod + def stop(self) -> None: + """Remove both the orchestrator and the gateway. Idempotent.""" + + def url(self) -> str: + """The host-facing control-plane URL the CLI reaches (the orchestrator's).""" + return self.orchestrator().url() + + def is_healthy(self) -> bool: + """True iff the control plane answers `/health`.""" + return self.orchestrator().is_healthy() + + +__all__ = ["InfraService"] diff --git a/bot_bottle/backend/macos_container/backend.py b/bot_bottle/backend/macos_container/backend.py index ba09e5a0..f5a7d7d5 100644 --- a/bot_bottle/backend/macos_container/backend.py +++ b/bot_bottle/backend/macos_container/backend.py @@ -106,7 +106,7 @@ class MacosContainerBottleBackend( (`supervise`) call when no control plane is running yet. Mirrors firecracker's infra bring-up.""" from .infra import MacosInfraService - return MacosInfraService().ensure_running().orchestrator_url + return MacosInfraService().ensure_running() def prepare_cleanup(self) -> MacosContainerBottleCleanupPlan: return _cleanup.prepare_cleanup() diff --git a/bot_bottle/backend/macos_container/consolidated_launch.py b/bot_bottle/backend/macos_container/consolidated_launch.py index e67ba418..c8b8b6c4 100644 --- a/bot_bottle/backend/macos_container/consolidated_launch.py +++ b/bot_bottle/backend/macos_container/consolidated_launch.py @@ -87,10 +87,10 @@ def ensure_gateway( launches share it. Call before starting the agent container: the agent's proxy env needs `gateway_ip` at run time.""" service = service or MacosInfraService() - infra = service.ensure_running() + orchestrator_url = service.ensure_running() endpoint = GatewayEndpoint( - orchestrator_url=infra.orchestrator_url, - gateway_ip=infra.gateway_ip, + orchestrator_url=orchestrator_url, + gateway_ip=service.gateway().address(), gateway_ca_pem=service.ca_cert_pem(), network=service.network, ) diff --git a/bot_bottle/backend/macos_container/infra.py b/bot_bottle/backend/macos_container/infra.py index ea20858f..c9bac13b 100644 --- a/bot_bottle/backend/macos_container/infra.py +++ b/bot_bottle/backend/macos_container/infra.py @@ -22,7 +22,6 @@ so they have no route to the control plane (the L3 block, not just the JWT). from __future__ import annotations -from dataclasses import dataclass from pathlib import Path from ...orchestrator.lifecycle import ( @@ -30,6 +29,7 @@ from ...orchestrator.lifecycle import ( DEFAULT_STARTUP_TIMEOUT_SECONDS, OrchestratorStartError, ) +from ..infra_service import InfraService from . import util as container_mod from .gateway import ( CONTROL_NETWORK, @@ -59,19 +59,10 @@ INFRA_DB_VOLUME = ORCHESTRATOR_DB_VOLUME _REPO_ROOT = Path(__file__).resolve().parents[3] -@dataclass(frozen=True) -class InfraEndpoint: - """How to reach the running pair. `orchestrator_url` is the orchestrator's - control-network address (host CLI + registration); `gateway_ip` is the - gateway's agent-network address (proxy / git-http / MCP target).""" - - orchestrator_url: str - gateway_ip: str - - -class MacosInfraService: +class MacosInfraService(InfraService): """Composes the per-host orchestrator + gateway containers. Callers use - `ensure_running()` (returns the endpoint) and `ca_cert_pem()`.""" + `ensure_running()` (returns the control-plane URL), the `orchestrator()` / + `gateway()` accessors, and `ca_cert_pem()`.""" def __init__( self, @@ -129,20 +120,12 @@ class MacosInfraService: repo_root=self._repo_root, ) - def _resolve_gateway_ip(self) -> str: - """The gateway's agent-network address, or "" while it has none.""" - return container_mod.try_container_ipv4_on_network( - self._gateway_name, self.network) - - def is_healthy(self) -> bool: - return self.orchestrator().is_healthy() - def ensure_running( self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, - ) -> InfraEndpoint: - """Ensure the orchestrator + gateway containers are up; return how to - reach them. Idempotent per-host singleton — a healthy orchestrator on - current source is left untouched. Raises `OrchestratorStartError` on + ) -> str: + """Ensure the orchestrator + gateway containers are up; return the host + control-plane URL. Idempotent per-host singleton — a healthy orchestrator + on current source is left untouched. Raises `OrchestratorStartError` on control-plane startup timeout.""" # The networks (host-only agent + NAT egress + host-only control) are a # shared concern — create them before either plane comes up. @@ -162,7 +145,7 @@ class MacosInfraService: # (#469). url = orchestrator.url() gateway.connect_to_orchestrator(url, orchestrator.mint_gateway_token()) - return InfraEndpoint(orchestrator_url=url, gateway_ip=self._resolve_gateway_ip()) + return url def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: """The gateway's mitmproxy CA (PEM) agents install to trust its TLS @@ -178,7 +161,6 @@ class MacosInfraService: __all__ = [ "MacosInfraService", - "InfraEndpoint", "OrchestratorStartError", "GatewayError", "ORCHESTRATOR_NAME", diff --git a/tests/integration/test_orchestrator_docker_auth.py b/tests/integration/test_orchestrator_docker_auth.py index 40dae8b1..23f4d8dc 100644 --- a/tests/integration/test_orchestrator_docker_auth.py +++ b/tests/integration/test_orchestrator_docker_auth.py @@ -130,7 +130,7 @@ class TestDockerOrchestratorAuthIntegration(unittest.TestCase): # than hand-rolling urllib here; _request (not one of the named # wrapper methods) is what exposes raw status codes for arbitrary # paths/tokens, which is exactly what these auth-boundary tests need. - client = OrchestratorClient(self.svc.url, auth_token=token) + client = OrchestratorClient(self.svc.url(), auth_token=token) return client._request(method, path) # pylint: disable=protected-access def test_health_is_open_without_a_token(self) -> None: diff --git a/tests/unit/test_backend_selection.py b/tests/unit/test_backend_selection.py index 9df4e5d2..af24f0f3 100644 --- a/tests/unit/test_backend_selection.py +++ b/tests/unit/test_backend_selection.py @@ -442,8 +442,7 @@ class TestIsBackendReady(unittest.TestCase): class TestEnsureOrchestrator(unittest.TestCase): """The backend-agnostic orchestrator bring-up entry point. Docker starts - the orchestrator + gateway containers; firecracker boots the infra VM; - macos-container starts the infra container.""" + each backend's InfraService brings up its orchestrator + gateway pair.""" def test_docker_delegates_to_infra_service(self): b = get_bottle_backend("docker") @@ -457,23 +456,23 @@ class TestEnsureOrchestrator(unittest.TestCase): self.assertEqual(url, "http://127.0.0.1:8099") service_cls.return_value.ensure_running.assert_called_once_with() - def test_firecracker_delegates_to_infra_vm(self): + def test_firecracker_delegates_to_infra_service(self): b = get_bottle_backend("firecracker") with patch( - "bot_bottle.backend.firecracker.infra_vm.ensure_running" - ) as ensure_running: - ensure_running.return_value.orchestrator_url = ( + "bot_bottle.backend.firecracker.infra.FirecrackerInfraService" + ) as service_cls: + service_cls.return_value.ensure_running.return_value = ( "http://10.243.255.1:8099" ) url = b.ensure_orchestrator() self.assertEqual(url, "http://10.243.255.1:8099") - def test_macos_delegates_to_infra_container(self): + def test_macos_delegates_to_infra_service(self): b = get_bottle_backend("macos-container") with patch( "bot_bottle.backend.macos_container.infra.MacosInfraService" ) as service_cls: - service_cls.return_value.ensure_running.return_value.orchestrator_url = ( + service_cls.return_value.ensure_running.return_value = ( "http://192.168.128.2:8099" ) url = b.ensure_orchestrator() diff --git a/tests/unit/test_docker_infra.py b/tests/unit/test_docker_infra.py index 339befb5..e55a0353 100644 --- a/tests/unit/test_docker_infra.py +++ b/tests/unit/test_docker_infra.py @@ -37,7 +37,7 @@ class TestDockerInfraService(unittest.TestCase): self.assertEqual(GATEWAY_NAME, self.svc.gateway_name) def test_url_delegates_to_the_orchestrator(self) -> None: - self.assertEqual("http://127.0.0.1:8099", self.svc.url) + self.assertEqual("http://127.0.0.1:8099", self.svc.url()) def test_is_healthy_delegates_to_the_orchestrator(self) -> None: self.orch.is_healthy.return_value = True diff --git a/tests/unit/test_firecracker_infra.py b/tests/unit/test_firecracker_infra.py new file mode 100644 index 00000000..f7ee8905 --- /dev/null +++ b/tests/unit/test_firecracker_infra.py @@ -0,0 +1,126 @@ +"""Unit: FirecrackerInfraService composes the orchestrator + gateway microVM pair. + +The two-VM singleton lifecycle (adopt-or-boot-both under a flock, the version +marker) that must hold without a live VM. The VM substrate + boot primitives are +tested in test_firecracker_infra_vm; the orchestrator/gateway services in their +own modules. These isolate the *composition*.""" + +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from bot_bottle.backend.firecracker import infra_vm +from bot_bottle.backend.firecracker.gateway import FirecrackerGateway +from bot_bottle.backend.firecracker.infra import FirecrackerInfraService +from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator + +# The service imports the classes into its own module namespace, so patch them +# there (not at their home modules). +_ORCH_CLS = "bot_bottle.backend.firecracker.infra.FirecrackerOrchestrator" +_GW_CLS = "bot_bottle.backend.firecracker.infra.FirecrackerGateway" + + +class TestAccessors(unittest.TestCase): + def test_orchestrator_and_gateway_services(self) -> None: + svc = FirecrackerInfraService() + self.assertIsInstance(svc.orchestrator(), FirecrackerOrchestrator) + self.assertIsInstance(svc.gateway(), FirecrackerGateway) + + def test_stop_stops_both_vms(self) -> None: + with patch.object(infra_vm, "stop") as stop: + FirecrackerInfraService().stop() + stop.assert_called_once() + + def test_url_and_is_healthy_delegate_to_the_orchestrator(self) -> None: + svc = FirecrackerInfraService() + ip = infra_vm.netpool.orch_slot().guest_ip + self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", svc.url()) + + +class TestEnsureRunning(unittest.TestCase): + def test_adopts_when_healthy_alive_and_version_matches(self): + # Healthy control plane + live gateway + existing key + matching version + # marker -> adopt (no stop/build/boot); returns the orchestrator URL. + with tempfile.TemporaryDirectory() as td: + d = Path(td) + (d / "id_ed25519").write_text("k") + (d / "booted-version").write_text("v-current\n") + with patch.object(infra_vm, "_infra_dir", return_value=d), \ + patch.object(infra_vm, "expected_version", return_value="v-current"), \ + patch.object(infra_vm, "_health_ok", return_value=True), \ + patch.object(infra_vm, "_pidfile_alive", return_value=True), \ + patch.object(infra_vm, "stop") as stop, \ + patch.object(infra_vm, "ensure_built") as built: + url = FirecrackerInfraService().ensure_running() + stop.assert_not_called() + built.assert_not_called() + ip = infra_vm.netpool.orch_slot().guest_ip + self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", url) + + def test_reboots_both_when_version_stale(self): + # Healthy control plane but the running pair booted an OLDER image + # (marker mismatch) -> reboot rather than adopt stale code. + with tempfile.TemporaryDirectory() as td: + d = Path(td) + (d / "id_ed25519").write_text("k") + (d / "booted-version").write_text("v-old\n") + with patch.object(infra_vm, "_infra_dir", return_value=d), \ + patch.object(infra_vm, "expected_version", return_value="v-current"), \ + patch.object(infra_vm, "_health_ok", return_value=True), \ + patch.object(infra_vm, "_pidfile_alive", return_value=True), \ + patch.object(infra_vm, "stop") as stop, \ + patch.object(infra_vm, "ensure_built"), \ + patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls: + orch_cls.return_value.gateway_url.return_value = "http://10.243.255.1:8099" + orch_cls.return_value.mint_gateway_token.return_value = "gw.jwt" + FirecrackerInfraService().ensure_running() + stop.assert_called_once() # dislodge the outdated pair + orch_cls.return_value.ensure_running.assert_called_once() + # The gateway service is bound to the orchestrator's gateway URL, + # carrying the orchestrator-minted `gateway` token. + gw_cls.return_value.connect_to_orchestrator.assert_called_once_with( + "http://10.243.255.1:8099", "gw.jwt") + # The fresh boot records the current version for the next launcher. + self.assertEqual("v-current\n", (d / "booted-version").read_text()) + + def test_reboots_when_gateway_dead(self): + # Orchestrator healthy + marker current, but the gateway VM is gone -> + # the pair is half-down, so reboot both rather than adopt partial state. + with tempfile.TemporaryDirectory() as td: + d = Path(td) + (d / "id_ed25519").write_text("k") + (d / "booted-version").write_text("v-current\n") + with patch.object(infra_vm, "_infra_dir", return_value=d), \ + patch.object(infra_vm, "expected_version", return_value="v-current"), \ + patch.object(infra_vm, "_health_ok", return_value=True), \ + patch.object(infra_vm, "_pidfile_alive", return_value=False), \ + patch.object(infra_vm, "stop") as stop, \ + patch.object(infra_vm, "ensure_built"), \ + patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls: + FirecrackerInfraService().ensure_running() + stop.assert_called_once() + orch_cls.return_value.ensure_running.assert_called_once() + gw_cls.return_value.connect_to_orchestrator.assert_called_once() + + def test_boots_both_when_no_running_pair(self): + with tempfile.TemporaryDirectory() as td: + with patch.object(infra_vm, "_infra_dir", return_value=Path(td)), \ + patch.object(infra_vm, "expected_version", return_value="v-current"), \ + patch.object(infra_vm, "_health_ok", return_value=False), \ + patch.object(infra_vm, "stop") as stop, \ + patch.object(infra_vm, "ensure_built") as built, \ + patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls: + FirecrackerInfraService().ensure_running() + stop.assert_called_once() # clear stale VMs first + built.assert_called_once() + # Orchestrator is brought up BEFORE the gateway is connected (its daemons + # reach the control plane at startup). + orch_cls.return_value.ensure_running.assert_called_once() + gw_cls.return_value.connect_to_orchestrator.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_firecracker_infra_vm.py b/tests/unit/test_firecracker_infra_vm.py index 20f78ed0..600c2aee 100644 --- a/tests/unit/test_firecracker_infra_vm.py +++ b/tests/unit/test_firecracker_infra_vm.py @@ -1,8 +1,8 @@ -"""Unit tests for the Firecracker infra VMs (orchestrator + gateway boot). +"""Unit tests for the Firecracker infra-VM substrate (boot primitives + inits). The KVM boot / HTTP reachability is integration-tested on a KVM host; here we -cover the URL shape, the shared-rootfs role wiring, the secret-push decisions, -and the two-VM singleton lifecycle that must hold without a VM. +cover the per-plane role inits, rootfs build, the secret-push retry, and the +pidfile/adoption helpers. The pair coordinator is tested in test_firecracker_infra. """ from __future__ import annotations @@ -43,31 +43,6 @@ class TestPushSecret(unittest.TestCase): die.assert_called_once() -class TestInfraEndpoint(unittest.TestCase): - """The endpoint mirrors docker/macOS: it exposes the orchestrator service's - URL and delegates the CA fetch to the gateway service.""" - - def _endpoint(self) -> infra_vm.InfraEndpoint: - from bot_bottle.backend.firecracker.gateway import FirecrackerGateway - from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator - return infra_vm.InfraEndpoint( - orchestrator=FirecrackerOrchestrator(), - gateway=FirecrackerGateway( - infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))), - ) - - def test_orchestrator_url_is_the_orchestrator_service(self): - ep = self._endpoint() - ip = infra_vm.netpool.orch_slot().guest_ip - self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", ep.orchestrator_url) - - def test_gateway_ca_pem_delegates_to_the_gateway_service(self): - ep = self._endpoint() - with patch.object(ep.gateway, "ca_cert_pem", return_value="PEM") as ca: - self.assertEqual("PEM", ep.gateway_ca_pem(timeout=1)) - ca.assert_called_once_with(timeout=1) - - class TestRoleInits(unittest.TestCase): def test_orchestrator_init_starts_only_the_control_plane(self): init = infra_vm.role_init("orchestrator") @@ -143,105 +118,6 @@ class TestEnsureBuilt(unittest.TestCase): tags.index(infra_vm._ORCHESTRATOR_FC_IMAGE)) -# The orchestrator + gateway services are imported lazily inside ensure_running -# / _adopt (to break the infra_vm <-> service import cycle), so patch them at -# their home modules. -_ORCH_CLS = "bot_bottle.backend.firecracker.orchestrator.FirecrackerOrchestrator" -_GW_CLS = "bot_bottle.backend.firecracker.gateway.FirecrackerGateway" - - -class TestEnsureRunningSingleton(unittest.TestCase): - def test_adopts_when_healthy_alive_and_version_matches(self): - # Healthy control plane + live gateway + existing key + matching version - # marker -> adopt (no boot). - from bot_bottle.backend.firecracker.gateway import FirecrackerGateway - from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator - import tempfile - with tempfile.TemporaryDirectory() as td: - d = Path(td) - (d / "id_ed25519").write_text("k") - (d / "booted-version").write_text("v-current\n") - with patch.object(infra_vm, "_infra_dir", return_value=d), \ - patch.object(infra_vm, "_expected_version", return_value="v-current"), \ - patch.object(infra_vm, "_health_ok", return_value=True), \ - patch.object(infra_vm, "_pidfile_alive", return_value=True): - ep = infra_vm.ensure_running() - # Adopted service handles addressing the two distinct infra links. - self.assertIsInstance(ep.orchestrator, FirecrackerOrchestrator) - self.assertEqual( - infra_vm.netpool.orch_slot().guest_ip, - ep.orchestrator_url.split("://")[1].split(":")[0]) - self.assertIsInstance(ep.gateway, FirecrackerGateway) - self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, ep.gateway.address()) - - def test_reboots_both_when_version_stale(self): - # Healthy control plane but the running pair booted an OLDER image - # (marker mismatch) -> reboot rather than adopt stale code. - import tempfile - with tempfile.TemporaryDirectory() as td: - d = Path(td) - (d / "id_ed25519").write_text("k") - (d / "booted-version").write_text("v-old\n") - with patch.object(infra_vm, "_infra_dir", return_value=d), \ - patch.object(infra_vm, "_expected_version", return_value="v-current"), \ - patch.object(infra_vm, "_health_ok", return_value=True), \ - patch.object(infra_vm, "_pidfile_alive", return_value=True), \ - patch.object(infra_vm, "stop") as stop, \ - patch.object(infra_vm, "ensure_built"), \ - patch(_ORCH_CLS) as orch_cls, \ - patch(_GW_CLS) as gw_cls: - orch_cls.return_value.gateway_url.return_value = "http://10.243.255.1:8099" - orch_cls.return_value.mint_gateway_token.return_value = "gw.jwt" - infra_vm.ensure_running() - stop.assert_called_once() # dislodge the outdated pair - orch_cls.return_value.ensure_running.assert_called_once() - # The gateway service is bound to the orchestrator's gateway URL, - # carrying the orchestrator-minted `gateway` token. - gw_cls.return_value.connect_to_orchestrator.assert_called_once_with( - "http://10.243.255.1:8099", "gw.jwt") - # The fresh boot records the current version for the next launcher. - self.assertEqual("v-current\n", (d / "booted-version").read_text()) - - def test_reboots_when_gateway_dead(self): - # Orchestrator healthy + marker current, but the gateway VM is gone -> - # the pair is half-down, so reboot both rather than adopt partial state. - import tempfile - with tempfile.TemporaryDirectory() as td: - d = Path(td) - (d / "id_ed25519").write_text("k") - (d / "booted-version").write_text("v-current\n") - with patch.object(infra_vm, "_infra_dir", return_value=d), \ - patch.object(infra_vm, "_expected_version", return_value="v-current"), \ - patch.object(infra_vm, "_health_ok", return_value=True), \ - patch.object(infra_vm, "_pidfile_alive", return_value=False), \ - patch.object(infra_vm, "stop") as stop, \ - patch.object(infra_vm, "ensure_built"), \ - patch(_ORCH_CLS) as orch_cls, \ - patch(_GW_CLS) as gw_cls: - infra_vm.ensure_running() - stop.assert_called_once() - orch_cls.return_value.ensure_running.assert_called_once() - gw_cls.return_value.connect_to_orchestrator.assert_called_once() - - def test_boots_both_when_unhealthy(self): - import tempfile - with tempfile.TemporaryDirectory() as td: - with patch.object(infra_vm, "_infra_dir", return_value=Path(td)), \ - patch.object(infra_vm, "_expected_version", return_value="v-current"), \ - patch.object(infra_vm, "_health_ok", return_value=False), \ - patch.object(infra_vm, "stop") as stop, \ - patch.object(infra_vm, "ensure_built") as built, \ - patch(_ORCH_CLS) as orch_cls, \ - patch(_GW_CLS) as gw_cls: - infra_vm.ensure_running() - stop.assert_called_once() # clear stale VMs first - built.assert_called_once() - # Orchestrator is brought up BEFORE the gateway is connected (its daemons - # reach the control plane at startup). - orch_cls.return_value.ensure_running.assert_called_once() - gw_cls.return_value.connect_to_orchestrator.assert_called_once() - - class TestStop(unittest.TestCase): def test_stops_both_vms_and_drops_marker(self): import tempfile @@ -306,21 +182,21 @@ class TestAdoptable(unittest.TestCase): with patch.object(infra_vm, "_infra_dir", return_value=d), \ patch.object(infra_vm, "_health_ok", return_value=True), \ patch.object(infra_vm, "_pidfile_alive", return_value=True): - self.assertTrue(infra_vm._adoptable(d / "id_ed25519", "u", "v1")) + self.assertTrue(infra_vm.adoptable(d / "id_ed25519", "u", "v1")) def test_false_when_key_missing(self): import tempfile with tempfile.TemporaryDirectory() as td: d = self._dir(td, key=False, version="v1") with patch.object(infra_vm, "_infra_dir", return_value=d): - self.assertFalse(infra_vm._adoptable(d / "id_ed25519", "u", "v1")) + self.assertFalse(infra_vm.adoptable(d / "id_ed25519", "u", "v1")) def test_false_when_no_version_marker(self): import tempfile with tempfile.TemporaryDirectory() as td: d = self._dir(td) # key present, no booted-version with patch.object(infra_vm, "_infra_dir", return_value=d): - self.assertFalse(infra_vm._adoptable(d / "id_ed25519", "u", "v1")) + self.assertFalse(infra_vm.adoptable(d / "id_ed25519", "u", "v1")) def test_false_when_version_mismatch(self): import tempfile @@ -329,7 +205,7 @@ class TestAdoptable(unittest.TestCase): with patch.object(infra_vm, "_infra_dir", return_value=d), \ patch.object(infra_vm, "_health_ok", return_value=True), \ patch.object(infra_vm, "_pidfile_alive", return_value=True): - self.assertFalse(infra_vm._adoptable(d / "id_ed25519", "u", "v1")) + self.assertFalse(infra_vm.adoptable(d / "id_ed25519", "u", "v1")) def test_false_when_gateway_dead(self): import tempfile @@ -338,7 +214,7 @@ class TestAdoptable(unittest.TestCase): with patch.object(infra_vm, "_infra_dir", return_value=d), \ patch.object(infra_vm, "_health_ok", return_value=True), \ patch.object(infra_vm, "_pidfile_alive", return_value=False): - self.assertFalse(infra_vm._adoptable(d / "id_ed25519", "u", "v1")) + self.assertFalse(infra_vm.adoptable(d / "id_ed25519", "u", "v1")) class TestKillInfraFirecrackers(unittest.TestCase): diff --git a/tests/unit/test_macos_consolidated_launch.py b/tests/unit/test_macos_consolidated_launch.py index 93747cff..c340f1b7 100644 --- a/tests/unit/test_macos_consolidated_launch.py +++ b/tests/unit/test_macos_consolidated_launch.py @@ -55,14 +55,13 @@ class TestEnsureGateway(unittest.TestCase): return ensure_gateway() def _service(self) -> MagicMock: - from bot_bottle.backend.macos_container.infra import InfraEndpoint service = MagicMock() # Two containers now: the orchestrator on the control network, the - # gateway on the agent network — distinct addresses. - service.ensure_running.return_value = InfraEndpoint( - orchestrator_url="http://192.168.128.2:8099", - gateway_ip="192.168.128.3", - ) + # gateway on the agent network — distinct addresses. `ensure_running` + # returns the control-plane URL; the gateway address comes off the + # gateway service. + service.ensure_running.return_value = "http://192.168.128.2:8099" + service.gateway.return_value.address.return_value = "192.168.128.3" service.network = "bot-bottle-mac-gateway" service.ca_cert_pem.return_value = "PEM" return service diff --git a/tests/unit/test_macos_infra.py b/tests/unit/test_macos_infra.py index 06ef02cc..a5b1e655 100644 --- a/tests/unit/test_macos_infra.py +++ b/tests/unit/test_macos_infra.py @@ -30,21 +30,17 @@ class TestInfraEnsureRunning(unittest.TestCase): self.addCleanup(p.stop) def test_composes_networks_builds_and_brings_up_orchestrator_first(self) -> None: - with patch(f"{_INFRA}.ensure_networks") as nets, \ - patch(f"{_INFRA}.container_mod") as mod: - mod.try_container_ipv4_on_network.return_value = "192.168.128.3" - endpoint = self.svc.ensure_running() + with patch(f"{_INFRA}.ensure_networks") as nets: + url = self.svc.ensure_running() nets.assert_called_once() self.orch.ensure_built.assert_called_once() self.gw.ensure_built.assert_called_once() self.orch.ensure_running.assert_called_once() - self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url) - self.assertEqual("192.168.128.3", endpoint.gateway_ip) + # Returns the host control-plane URL (the InfraService contract). + self.assertEqual("http://192.168.128.2:8099", url) def test_connects_gateway_with_orchestrator_url_and_minted_token(self) -> None: - with patch(f"{_INFRA}.ensure_networks"), \ - patch(f"{_INFRA}.container_mod") as mod: - mod.try_container_ipv4_on_network.return_value = "192.168.128.3" + with patch(f"{_INFRA}.ensure_networks"): self.svc.ensure_running() self.gw.connect_to_orchestrator.assert_called_once_with( "http://192.168.128.2:8099", "gw.jwt")