didericis/bot-bottle
1
0
Fork 0
Code Issues 30 Pull Requests 5 Actions Packages Releases Wiki Activity

refactor(firecracker): move gateway logic into the gateway service

Browse Source 
Pull the gateway's host-side logic out of `infra_vm` and into
`FirecrackerGateway`'s own methods, so the gateway is self-contained and
`infra_vm` shrinks toward being just the pair coordinator (a step toward
removing it). Now living in the gateway service: the gateway VM boot +
`bb_orch` cmdline, the pre-minted JWT push, the mitmproxy CA fetch over SSH,
the `SshGatewayTransport` exec/cp, and the lifecycle predicates
(is_running/stop/address). `ensure_running` / `_adopt` construct the gateway
and call `connect_to_orchestrator` (lazy import to break the cycle); the
InfraEndpoint's gateway is now the `FirecrackerGateway` service.

What deliberately stays shared in `infra_vm` (documented at the top of
gateway.py): the plane-agnostic VM substrate the orchestrator VM also needs
(`_boot_vm`, the stable SSH keypair, the secret-push retry, PID lifecycle), the
pair coordinator (`ensure_running`: orchestrator-first health gate + singleton
lock + adoption/version marker), and the single shared `bb_role`-branched init
baked into the one published rootfs both VMs boot — the guest-side gateway
startup can't live in a host method. These are the seam that moves to a neutral
module when the Orchestrator service lands and `infra_vm` dissolves.

CA fetch now raises GatewayError (was die/SystemExit) to match the ABC.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
didericis
2026-07-25 13:34:36 -04:00
parent d54c559a3a
commit 5bf9f052b9
5 changed files with 262 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bot_bottle/backend/firecracker/consolidated_launch.py
+2 -3
View File
@@ -130,9 +130,8 @@ def launch_consolidated(
client = OrchestratorClient(url) client = OrchestratorClient(url)
_reprovision_running_bottles(client) _reprovision_running_bottles(client)
# Read the gateway's provisioning transport + CA off the Gateway service # Read the gateway's provisioning transport + CA off the Gateway service.
# (adapting the running gateway VM). gateway = infra.gateway
gateway = FirecrackerGateway(infra.gateway)
transport = gateway.provisioning_transport() transport = gateway.provisioning_transport()
reg = provision_bottle( reg = provision_bottle(
client, guest_ip, egress_plan, git_gate_plan, transport, client, guest_ip, egress_plan, git_gate_plan, transport,
bot_bottle/backend/firecracker/gateway.py
+122 -25
View File
@@ -1,38 +1,100 @@
"""The Firecracker gateway data plane as a microVM (PRD 0070). """The Firecracker gateway data plane as a microVM (PRD 0070).
`FirecrackerGateway` is the Firecracker implementation of the backend-neutral `FirecrackerGateway` is the Firecracker implementation of the backend-neutral
`Gateway` service. The gateway here is a whole microVM — the slim data-plane `Gateway` service, and owns the gateway's host-side logic directly: booting the
unit (mitmproxy TLS bump + DLP + git-http) booted on its own NAT'd link; agent gateway microVM on its NAT'd link, seeding the pre-minted `gateway` token,
VMs' gateway-port traffic is DNAT'd to it and it never opens `bot-bottle.db` fetching the mitmproxy CA, and the SSH exec/cp provisioning transport. The
(#469), so it holds no signing key, only the pre-minted `gateway` token the gateway VM never opens `bot-bottle.db` (#469), so it holds no signing key —
host hands it via `connect_to_orchestrator`. only the token the host hands it via `connect_to_orchestrator`.
The VM lifecycle primitives (boot, SSH secret push, netpool links, nft, PID What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM
files) stay in `infra_vm`, which drives the orchestrator+gateway pair as a substrate the orchestrator VM shares — booting a VM from the shared rootfs
singleton; this adapter composes them into the uniform `Gateway` surface. (`_boot_vm`), the stable SSH keypair, the secret-push retry loop, and the PID
lifecycle — plus the pair coordinator (`ensure_running`: orchestrator-first
health gate, singleton lock, adoption/version marker). The guest-side gateway
daemon startup lives in the shared `bb_role=gateway` init branch baked into the
one published rootfs both VMs boot, so it can't live in a host-side method
either. These are the shared seam that moves to a neutral module when the
Orchestrator service lands and `infra_vm` is dissolved.
""" """
from __future__ import annotations from __future__ import annotations
import os
import shlex
import stat
import subprocess
from pathlib import Path
from urllib.parse import urlparse from urllib.parse import urlparse
from ...gateway import ( from ...gateway import (
DEFAULT_CA_TIMEOUT_SECONDS, DEFAULT_CA_TIMEOUT_SECONDS,
Gateway, Gateway,
GatewayError, GatewayError,
GatewayProvisionError,
GatewayTransport, GatewayTransport,
) )
from . import infra_vm from .. import util as backend_util
from . import infra_vm, netpool, util
# The gateway microVM's name (the `bb_role=gateway` boot tag / run dir). Fixed # The gateway microVM's name (the `bb_role=gateway` boot tag / run dir). Fixed
# per host — one gateway VM, shared by every agent VM. # per host — one gateway VM, shared by every agent VM.
GATEWAY_NAME = "bot-bottle-gateway" GATEWAY_NAME = "bot-bottle-gateway"
# The gateway VM's slim memory ceiling (buildah is present on the shared rootfs
# but unused on the data plane) — PRD 0070 "Memory: fixed ceilings".
_GW_MEM_MIB = 2048
# The pre-minted `gateway` JWT path in the guest. The *shared* init (both planes
# boot one rootfs) waits for it before starting the data plane, so its canonical
# definition lives with that init in `infra_vm`; imported here for the push so
# the load-bearing path isn't duplicated.
_GUEST_GATEWAY_JWT_PATH = infra_vm._GUEST_GATEWAY_JWT_PATH
# mitmproxy writes its CA here a beat after start; agents install it to trust
# the gateway's TLS interception. Host-side only (SSH cat), so it lives here.
_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem"
_CA_FETCH_TIMEOUT_SECONDS = 15.0
class SshGatewayTransport:
"""`GatewayTransport` for the gateway running in the gateway VM — the docker
exec/cp equivalents over SSH (dropbear + the stable infra key)."""
def __init__(self, private_key: Path, guest_ip: str) -> None:
self._key = private_key
self._ip = guest_ip
def exec(self, argv: list[str]) -> None:
proc = subprocess.run(
util.ssh_base_argv(self._key, self._ip) + [shlex.join(argv)],
capture_output=True, text=True, timeout=60, check=False,
)
if proc.returncode != 0:
raise GatewayProvisionError(
f"infra gateway exec {argv!r} failed: {proc.stderr.strip()}")
def cp_into(self, src: str, dest: str) -> None:
# Preserve the source mode (docker cp does): the access-hook is staged
# 0700 and git-http execs it directly — a plain `cat >` would land it
# 0644 and the exec fails with EACCES; keys stay 0600.
mode = stat.S_IMODE(os.stat(src).st_mode)
q = shlex.quote(dest)
proc = subprocess.run(
util.ssh_base_argv(self._key, self._ip)
+ [f"cat > {q} && chmod {mode:o} {q}"],
input=Path(src).read_bytes(), capture_output=True, timeout=30, check=False,
)
if proc.returncode != 0:
raise GatewayProvisionError(
f"infra gateway cp {src} -> {dest} failed: "
f"{proc.stderr.decode(errors='replace').strip()}")
class FirecrackerGateway(Gateway): class FirecrackerGateway(Gateway):
"""The consolidated gateway as a Firecracker microVM on the gateway link. """The consolidated gateway as a Firecracker microVM on the gateway link.
The rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's The shared rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's
`ensure_built` no-op here); `connect_to_orchestrator` boots the VM resolving `ensure_built` no-op here); `connect_to_orchestrator` boots the VM resolving
policy against the orchestrator and seeds the pre-minted `gateway` token.""" policy against the orchestrator and seeds the pre-minted `gateway` token."""
@@ -47,10 +109,12 @@ class FirecrackerGateway(Gateway):
self._gateway_token = "" self._gateway_token = ""
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None: def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
"""Boot the gateway VM resolving policy against `orchestrator_url` and """Boot the gateway VM on its link resolving policy against
seed `gateway_token`. The orchestrator's guest IP is parsed from the URL `orchestrator_url`, then seed `gateway_token` over SSH. The orchestrator's
and passed on the cmdline, so the baked init stays IP-independent. Boot guest IP is parsed from the URL and passed on the cmdline (`bb_orch=`), so
the orchestrator first — the gateway daemons reach it at startup.""" the baked init stays IP-independent. Boot the orchestrator first — the
gateway daemons reach it at startup. The gateway never mints, so it holds
no signing key, only this token (#469)."""
self._orchestrator_url = orchestrator_url self._orchestrator_url = orchestrator_url
self._gateway_token = gateway_token self._gateway_token = gateway_token
if not self._orchestrator_url: if not self._orchestrator_url:
@@ -68,31 +132,64 @@ class FirecrackerGateway(Gateway):
raise GatewayError( raise GatewayError(
f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}" f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}"
) )
self._vm = infra_vm.boot_gateway(orchestrator_guest_ip, self._gateway_token) # Boot on the gateway link from the shared rootfs (bb_role=gateway), then
# push the token the init waits for before starting the data plane.
vm = infra_vm._boot_vm(
name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(),
role="gateway", mem_mib=_GW_MEM_MIB,
extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
)
infra_vm._push_secret(
vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH,
"the gateway JWT to the gateway VM (its data plane will not start)",
)
self._vm = vm
def is_running(self) -> bool: def is_running(self) -> bool:
return infra_vm.gateway_running() return infra_vm._pidfile_alive(infra_vm._gw_dir())
def stop(self) -> None: def stop(self) -> None:
"""Stop only the gateway VM (idempotent).""" """Stop only the gateway VM (idempotent — absent is success). A dead
infra_vm.stop_gateway() gateway already fails `infra_vm._adoptable`, so no version marker to
clear here."""
infra_vm._kill_pidfile(infra_vm._gw_dir())
infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True)
def address(self) -> str: def address(self) -> str:
"""The gateway VM's guest IP — the agent-facing target agent VMs' """The gateway VM's guest IP — the agent-facing target agent VMs'
gateway-port traffic is DNAT'd to.""" gateway-port traffic is DNAT'd to."""
return infra_vm.gateway_guest_ip() return netpool.gw_slot().guest_ip
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str: def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS """The gateway's mitmproxy CA (PEM) agents install to trust its TLS
interception. Fetched from the gateway VM over SSH; polls because interception. Fetched from the gateway VM over SSH; polls because
mitmproxy writes it a beat after boot.""" mitmproxy writes it a beat after boot. Works from any later launcher —
vm = self._vm or infra_vm.adopted_gateway_vm() falls back to the stable key + fixed link when this process didn't boot
return vm.gateway_ca_pem(timeout=timeout) the VM."""
key = self._vm.private_key if self._vm else infra_vm._infra_dir() / "id_ed25519"
ip = self._vm.guest_ip if self._vm else netpool.gw_slot().guest_ip
def _fetch() -> str | None:
proc = subprocess.run(
util.ssh_base_argv(key, ip) + [f"cat {_GATEWAY_CA_PATH}"],
capture_output=True, text=True,
timeout=_CA_FETCH_TIMEOUT_SECONDS, check=False,
)
ok = proc.returncode == 0 and "BEGIN CERTIFICATE" in proc.stdout
return proc.stdout if ok else None
try:
return backend_util.poll_ca_cert(_fetch, timeout=timeout)
except TimeoutError as exc:
raise GatewayError(str(exc)) from exc
def provisioning_transport(self) -> GatewayTransport: def provisioning_transport(self) -> GatewayTransport:
"""The exec/cp transport git-gate provisioning stages per-bottle repos + """The exec/cp transport git-gate provisioning stages per-bottle repos +
deploy keys through (over SSH to the gateway VM).""" deploy keys through (over SSH to the gateway VM). Needs no live handle —
return infra_vm.gateway_transport() built from the stable key + the gateway link's guest IP, so teardown can
use it too."""
return SshGatewayTransport(
infra_vm._infra_dir() / "id_ed25519", netpool.gw_slot().guest_ip)
__all__ = ["FirecrackerGateway", "GATEWAY_NAME"] __all__ = ["FirecrackerGateway", "SshGatewayTransport", "GATEWAY_NAME"]
bot_bottle/backend/firecracker/infra_vm.py
+23 -143
View File
@@ -32,9 +32,7 @@ from __future__ import annotations
import fcntl import fcntl
import hashlib import hashlib
import os import os
import shlex
import signal import signal
import stat
import subprocess import subprocess
import time import time
import urllib.error import urllib.error
@@ -42,16 +40,17 @@ import urllib.request
from contextlib import contextmanager from contextlib import contextmanager
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Generator from typing import TYPE_CHECKING, Generator
from ...log import die, info from ...log import die, info
from ...orchestrator_auth import ROLE_GATEWAY, mint from ...orchestrator_auth import ROLE_GATEWAY, mint
from ...paths import host_orchestrator_token from ...paths import host_orchestrator_token
from .. import util as backend_util
from ..docker import util as docker_mod from ..docker import util as docker_mod
from ...gateway import GatewayProvisionError
from . import firecracker_vm, infra_artifact, netpool, util from . import firecracker_vm, infra_artifact, netpool, util
if TYPE_CHECKING:
from .gateway import FirecrackerGateway
# The orchestrator VM's signing-key path on its persistent /dev/vdb volume # The orchestrator VM's signing-key path on its persistent /dev/vdb volume
# (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds this # (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds this
# file with the host-canonical key AFTER boot (over SSH), so the VM verifies # file with the host-canonical key AFTER boot (over SSH), so the VM verifies
@@ -77,16 +76,11 @@ ORCHESTRATOR_PORT = 8099
EGRESS_PORT = 9099 EGRESS_PORT = 9099
SUPERVISE_PORT = 9100 SUPERVISE_PORT = 9100
GIT_HTTP_PORT = 9420 GIT_HTTP_PORT = 9420
# mitmproxy writes its CA here a beat after start; agents install it to trust # Memory ceiling (fixed at boot, demand-paged — no balloon reclaim). The
# the gateway's TLS interception.
_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem"
# Memory ceilings (fixed at boot, demand-paged — no balloon reclaim). The
# orchestrator keeps the build headroom (buildah's 2-4 GB working set during # orchestrator keeps the build headroom (buildah's 2-4 GB working set during
# in-VM agent builds); the gateway is the slim unit (mitmproxy TLS bump + DLP # in-VM agent builds); the gateway's slimmer ceiling is set by the gateway
# body buffering, ~1 GB) — PRD 0070 "Memory: fixed ceilings". # service — PRD 0070 "Memory: fixed ceilings".
_ORCH_MEM_MIB = 4096 _ORCH_MEM_MIB = 4096
_GW_MEM_MIB = 2048
# The infra VMs make direct upstream connections (gateway egress, and buildah # The infra VMs make direct upstream connections (gateway egress, and buildah
# during builds), and the kernel `ip=` cmdline sets no resolver. Public for # during builds), and the kernel `ip=` cmdline sets no resolver. Public for
@@ -117,41 +111,23 @@ class InfraVm:
def orchestrator_url(self) -> str: def orchestrator_url(self) -> str:
return f"http://{self.guest_ip}:{ORCHESTRATOR_PORT}" return f"http://{self.guest_ip}:{ORCHESTRATOR_PORT}"
def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str:
"""The gateway's mitmproxy CA (PEM) that agents install to trust its
TLS interception. Generated a moment after boot, so this polls over
SSH until it appears (mirrors DockerGateway.ca_cert_pem). Call on the
gateway VM handle."""
def _fetch() -> str | None:
proc = subprocess.run(
util.ssh_base_argv(self.private_key, self.guest_ip)
+ [f"cat {_GATEWAY_CA_PATH}"],
capture_output=True, text=True, timeout=15, check=False,
)
ok = proc.returncode == 0 and "BEGIN CERTIFICATE" in proc.stdout
return proc.stdout if ok else None
try:
return backend_util.poll_ca_cert(_fetch, timeout=timeout)
except TimeoutError as exc:
die(str(exc))
@dataclass @dataclass
class InfraEndpoint: class InfraEndpoint:
"""How to reach the running per-host pair. `orchestrator` is the control """How to reach the running per-host pair. `orchestrator` is the control
plane (host CLI + registration + in-VM builds); `gateway` is the data plane (host CLI + registration + in-VM builds); `gateway` is the data-plane
plane (agent egress / git-http / supervise, CA fetch, git-gate `Gateway` service (agent egress / git-http / supervise, CA fetch, git-gate
provisioning). Mirrors the docker/macOS `InfraEndpoint` shape.""" provisioning). Mirrors the docker/macOS `InfraEndpoint` shape."""
orchestrator: InfraVm orchestrator: InfraVm
gateway: InfraVm gateway: "FirecrackerGateway"
@property @property
def orchestrator_url(self) -> str: def orchestrator_url(self) -> str:
return self.orchestrator.orchestrator_url return self.orchestrator.orchestrator_url
def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str: def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str:
return self.gateway.gateway_ca_pem(timeout=timeout) return self.gateway.ca_cert_pem(timeout=timeout)
def ensure_built() -> None: def ensure_built() -> None:
@@ -229,22 +205,28 @@ def ensure_running() -> InfraEndpoint:
orchestrator = boot_orchestrator() orchestrator = boot_orchestrator()
wait_for_health(orchestrator) wait_for_health(orchestrator)
# The orchestrator holds the signing key; mint the role-scoped `gateway` # The orchestrator holds the signing key; mint the role-scoped `gateway`
# JWT on the host and hand it to the gateway VM, which never sees the key # JWT on the host and hand it to the gateway service, which never sees
# (#469). # the key (#469). The gateway owns its own boot — see FirecrackerGateway.
# Local import: the gateway service imports this module for the shared VM
# substrate, so importing it at module scope would cycle.
from .gateway import FirecrackerGateway
gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token()) gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token())
gateway = boot_gateway(orchestrator.guest_ip, gateway_token) gateway = FirecrackerGateway()
gateway.connect_to_orchestrator(orchestrator.orchestrator_url, gateway_token)
_record_booted_version(want) _record_booted_version(want)
return InfraEndpoint(orchestrator=orchestrator, gateway=gateway) return InfraEndpoint(orchestrator=orchestrator, gateway=gateway)
def _adopt(key: Path) -> InfraEndpoint: def _adopt(key: Path) -> InfraEndpoint:
"""Build handles to the already-running pair from the stable key + the """Build handles to the already-running pair from the stable key + the
fixed links (vm=None — teardown then goes through the PID files).""" fixed links (no live VM handle — teardown / CA fetch go through the PID
files + stable key)."""
from .gateway import FirecrackerGateway
return InfraEndpoint( return InfraEndpoint(
orchestrator=InfraVm( orchestrator=InfraVm(
guest_ip=netpool.orch_slot().guest_ip, private_key=key), guest_ip=netpool.orch_slot().guest_ip, private_key=key),
gateway=InfraVm( gateway=FirecrackerGateway(
guest_ip=netpool.gw_slot().guest_ip, private_key=key), InfraVm(guest_ip=netpool.gw_slot().guest_ip, private_key=key)),
) )
@@ -294,25 +276,6 @@ def boot_orchestrator() -> InfraVm:
return orch return orch
def boot_gateway(orchestrator_guest_ip: str, gateway_token: str) -> InfraVm:
"""Boot the gateway (data-plane) VM (detached) on the gateway link, and
seed the pre-minted `gateway` JWT `gateway_token`. Its daemons resolve
policy against the orchestrator at `orchestrator_guest_ip:8099` (passed on
the cmdline, so the baked init stays IP-independent). Boot the orchestrator
first — the gateway daemons reach it at startup. The gateway never holds the
signing key, only this token (#469). Prefer `ensure_running`."""
slot = netpool.gw_slot()
gateway = _boot_vm(
name="bot-bottle-gateway", slot=slot, run_dir=_gw_dir(),
role="gateway", mem_mib=_GW_MEM_MIB,
extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
)
# Push the pre-minted `gateway` JWT (never the signing key — issue #469).
# The init waits for it before starting the data plane.
_push_gateway_jwt(gateway, gateway_token)
return gateway
def _boot_vm( def _boot_vm(
*, *,
name: str, name: str,
@@ -456,18 +419,6 @@ def _push_signing_key(infra: InfraVm) -> None:
) )
def _push_gateway_jwt(infra: InfraVm, gateway_token: str) -> None:
"""Push the pre-minted `gateway` JWT into the freshly booted gateway VM
over SSH (atomic write). Minted on the HOST from the canonical signing key
so the data plane never holds the key itself (issue #469); the gateway
daemons present it to the orchestrator. Retries until SSH is up; dies if it
never lands, since the VM then refuses to start the data plane."""
_push_secret(
infra, gateway_token, _GUEST_GATEWAY_JWT_PATH,
"the gateway JWT to the gateway VM (its data plane will not start)",
)
def _push_secret(infra: InfraVm, secret: str, dest: str, what: str) -> None: def _push_secret(infra: InfraVm, secret: str, dest: str, what: str) -> None:
"""Pipe `secret` to an atomic write of `dest` in the guest over SSH, """Pipe `secret` to an atomic write of `dest` in the guest over SSH,
retrying while the guest's SSH comes up; die (naming `what`) if it never retrying while the guest's SSH comes up; die (naming `what`) if it never
@@ -574,77 +525,6 @@ def _health_ok(url: str) -> bool:
return False return False
class SshGatewayTransport:
"""`GatewayTransport` for the gateway running in the gateway VM — the
docker exec/cp equivalents over SSH (dropbear + the stable infra key)."""
def __init__(self, private_key: Path, guest_ip: str) -> None:
self._key = private_key
self._ip = guest_ip
def exec(self, argv: list[str]) -> None:
proc = subprocess.run(
util.ssh_base_argv(self._key, self._ip) + [shlex.join(argv)],
capture_output=True, text=True, timeout=60, check=False,
)
if proc.returncode != 0:
raise GatewayProvisionError(
f"infra gateway exec {argv!r} failed: {proc.stderr.strip()}")
def cp_into(self, src: str, dest: str) -> None:
# Preserve the source mode (docker cp does): the access-hook is staged
# 0700 and git-http execs it directly — a plain `cat >` would land it
# 0644 and the exec fails with EACCES; keys stay 0600.
mode = stat.S_IMODE(os.stat(src).st_mode)
q = shlex.quote(dest)
proc = subprocess.run(
util.ssh_base_argv(self._key, self._ip)
+ [f"cat > {q} && chmod {mode:o} {q}"],
input=Path(src).read_bytes(), capture_output=True, timeout=30, check=False,
)
if proc.returncode != 0:
raise GatewayProvisionError(
f"infra gateway cp {src} -> {dest} failed: "
f"{proc.stderr.decode(errors='replace').strip()}")
def gateway_transport() -> SshGatewayTransport:
"""git-gate provisioning transport for the gateway VM, built from the
stable key + the gateway link's guest IP. Needs no live VM handle, so
teardown can use it too."""
return SshGatewayTransport(
_infra_dir() / "id_ed25519", netpool.gw_slot().guest_ip)
def gateway_guest_ip() -> str:
"""The gateway VM's fixed guest IP on its link — the agent-facing address
agent VMs' gateway-port traffic is DNAT'd to (PRD 0070)."""
return netpool.gw_slot().guest_ip
def gateway_running() -> bool:
"""True iff the gateway VM is still a live VMM."""
return _pidfile_alive(_gw_dir())
def stop_gateway() -> None:
"""Stop only the gateway VM (idempotent — absent is success). Drops the pair
version marker so a half-stopped pair is never adopted (`_adoptable`)."""
_kill_pidfile(_gw_dir())
_pid_file(_gw_dir()).unlink(missing_ok=True)
_version_file().unlink(missing_ok=True)
def adopted_gateway_vm() -> InfraVm:
"""A handle to the already-running gateway VM from the stable key + the
fixed gateway link (vm=None — CA fetch / provisioning work from any later
launcher, not only the process that booted it)."""
return InfraVm(
guest_ip=netpool.gw_slot().guest_ip,
private_key=_infra_dir() / "id_ed25519",
)
def wait_for_health( def wait_for_health(
infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS, infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS,
) -> None: ) -> None:
tests/unit/test_firecracker_gateway.py
+78 -37
View File
@@ -1,17 +1,26 @@
"""Unit: the Firecracker gateway data plane as a microVM (PRD 0070).""" """Unit: the Firecracker gateway data plane as a microVM (PRD 0070).
The gateway service owns its host-side logic directly: booting the gateway VM
(via the shared `infra_vm._boot_vm` substrate), seeding the pre-minted token,
fetching the CA over SSH, and the SSH exec/cp provisioning transport.
"""
from __future__ import annotations from __future__ import annotations
import os
import tempfile
import unittest import unittest
from pathlib import Path from pathlib import Path
from unittest.mock import MagicMock, patch from subprocess import CompletedProcess
from unittest.mock import patch
from bot_bottle.backend.firecracker import infra_vm from bot_bottle.backend.firecracker import infra_vm
from bot_bottle.backend.firecracker.gateway import ( from bot_bottle.backend.firecracker.gateway import (
GATEWAY_NAME, GATEWAY_NAME,
FirecrackerGateway, FirecrackerGateway,
SshGatewayTransport,
) )
from bot_bottle.gateway import GatewayError from bot_bottle.gateway import GatewayError, GatewayProvisionError
_GW = "bot_bottle.backend.firecracker.gateway" _GW = "bot_bottle.backend.firecracker.gateway"
@@ -25,76 +34,108 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
def test_refuses_without_orchestrator_url(self) -> None: def test_refuses_without_orchestrator_url(self) -> None:
gw = FirecrackerGateway() gw = FirecrackerGateway()
with patch.object(infra_vm, "boot_gateway") as boot: with patch.object(infra_vm, "_boot_vm") as boot:
with self.assertRaises(GatewayError): with self.assertRaises(GatewayError):
gw.connect_to_orchestrator("", _TOKEN) gw.connect_to_orchestrator("", _TOKEN)
boot.assert_not_called() boot.assert_not_called()
def test_refuses_without_gateway_token(self) -> None: def test_refuses_without_gateway_token(self) -> None:
gw = FirecrackerGateway() gw = FirecrackerGateway()
with patch.object(infra_vm, "boot_gateway") as boot: with patch.object(infra_vm, "_boot_vm") as boot:
with self.assertRaises(GatewayError): with self.assertRaises(GatewayError):
gw.connect_to_orchestrator(_ORCH_URL, "") gw.connect_to_orchestrator(_ORCH_URL, "")
boot.assert_not_called() boot.assert_not_called()
def test_refuses_a_url_without_a_host(self) -> None: def test_refuses_a_url_without_a_host(self) -> None:
gw = FirecrackerGateway() gw = FirecrackerGateway()
with patch.object(infra_vm, "boot_gateway") as boot: with patch.object(infra_vm, "_boot_vm") as boot:
with self.assertRaises(GatewayError): with self.assertRaises(GatewayError):
gw.connect_to_orchestrator("http://:8099", _TOKEN) gw.connect_to_orchestrator("http://:8099", _TOKEN)
boot.assert_not_called() boot.assert_not_called()
def test_boots_the_gateway_vm_against_the_orchestrator_guest_ip(self) -> None: def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None:
gw = FirecrackerGateway() gw = FirecrackerGateway()
booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k")) booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))
with patch.object(infra_vm, "boot_gateway", return_value=booted) as boot: with patch.object(infra_vm, "_boot_vm", return_value=booted) as boot, \
patch.object(infra_vm, "_push_secret") as push:
gw.connect_to_orchestrator(_ORCH_URL, _TOKEN) gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
# The orchestrator guest IP is parsed off the URL; the token is threaded # Booted on the gateway link with the gateway role; the orchestrator's
# through unchanged (the gateway never mints — #469). # guest IP (parsed off the URL) rides the cmdline as bb_orch.
boot.assert_called_once_with("10.243.255.1", _TOKEN) kw = boot.call_args.kwargs
self.assertEqual("gateway", kw["role"])
self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"])
self.assertIsNone(kw.get("data_drive")) # data plane never opens the DB
# The host-minted token (never the key — #469) is pushed to the guest.
push.assert_called_once()
self.assertEqual(_TOKEN, push.call_args.args[1])
class TestFirecrackerGatewaySurface(unittest.TestCase): class TestFirecrackerGatewaySurface(unittest.TestCase):
def test_is_running_reads_the_gateway_pidfile(self) -> None: def test_is_running_reads_the_gateway_pidfile(self) -> None:
gw = FirecrackerGateway() gw = FirecrackerGateway()
with patch.object(infra_vm, "gateway_running", return_value=True): with patch.object(infra_vm, "_pidfile_alive", return_value=True) as alive, \
patch.object(infra_vm, "_gw_dir", return_value=Path("/gw")):
self.assertTrue(gw.is_running()) self.assertTrue(gw.is_running())
with patch.object(infra_vm, "gateway_running", return_value=False): alive.assert_called_once_with(Path("/gw"))
self.assertFalse(gw.is_running())
def test_stop_stops_only_the_gateway_vm(self) -> None: def test_stop_kills_only_the_gateway_pidfile(self) -> None:
gw = FirecrackerGateway() gw = FirecrackerGateway()
with patch.object(infra_vm, "stop_gateway") as stop: with tempfile.TemporaryDirectory() as td:
gw.stop() d = Path(td)
stop.assert_called_once_with() (d / "vm.pid").write_text("123")
with patch.object(infra_vm, "_gw_dir", return_value=d), \
patch.object(infra_vm, "_kill_pidfile") as kill:
gw.stop()
kill.assert_called_once_with(d)
self.assertFalse((d / "vm.pid").exists()) # pidfile cleared
def test_address_is_the_gateway_link_guest_ip(self) -> None: def test_address_is_the_gateway_link_guest_ip(self) -> None:
gw = FirecrackerGateway() gw = FirecrackerGateway()
with patch.object(infra_vm, "gateway_guest_ip", return_value="10.243.255.3"): self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, gw.address())
self.assertEqual("10.243.255.3", gw.address())
def test_ca_cert_pem_uses_the_live_handle_when_present(self) -> None: def test_ca_cert_pem_reads_over_ssh_from_the_handle(self) -> None:
vm = MagicMock() vm = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))
vm.gateway_ca_pem.return_value = "PEM"
gw = FirecrackerGateway(vm) gw = FirecrackerGateway(vm)
with patch.object(infra_vm, "adopted_gateway_vm") as adopt: pem = "-----BEGIN CERTIFICATE-----\nx\n-----END CERTIFICATE-----\n"
self.assertEqual("PEM", gw.ca_cert_pem(timeout=1)) with patch.object(infra_vm.subprocess, "run",
adopt.assert_not_called() return_value=CompletedProcess([], 0, stdout=pem)) as run:
vm.gateway_ca_pem.assert_called_once_with(timeout=1) self.assertEqual(pem, gw.ca_cert_pem(timeout=5))
# cat the CA path over SSH to the gateway VM's guest IP.
argv = run.call_args.args[0]
self.assertTrue(any("10.243.255.3" in a for a in argv))
self.assertIn("cat /home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem", argv)
def test_ca_cert_pem_adopts_the_running_gateway_when_no_handle(self) -> None: def test_ca_cert_pem_raises_gateway_error_when_absent(self) -> None:
vm = MagicMock()
vm.gateway_ca_pem.return_value = "PEM"
gw = FirecrackerGateway() gw = FirecrackerGateway()
with patch.object(infra_vm, "adopted_gateway_vm", return_value=vm) as adopt: with patch.object(infra_vm.subprocess, "run",
self.assertEqual("PEM", gw.ca_cert_pem()) return_value=CompletedProcess([], 1, stdout="", stderr="")):
adopt.assert_called_once_with() with self.assertRaises(GatewayError):
gw.ca_cert_pem(timeout=0)
def test_provisioning_transport_targets_the_gateway_vm(self) -> None: def test_provisioning_transport_targets_the_gateway_link(self) -> None:
gw = FirecrackerGateway() gw = FirecrackerGateway()
sentinel = object() transport = gw.provisioning_transport()
with patch.object(infra_vm, "gateway_transport", return_value=sentinel): assert isinstance(transport, SshGatewayTransport)
self.assertIs(sentinel, gw.provisioning_transport()) self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, transport._ip)
class TestSshGatewayTransport(unittest.TestCase):
def test_cp_into_preserves_source_mode(self) -> None:
with tempfile.NamedTemporaryFile() as f:
os.chmod(f.name, 0o700) # like the staged access-hook
t = SshGatewayTransport(Path("/k"), "10.0.0.1")
with patch(f"{_GW}.subprocess.run",
return_value=CompletedProcess([], 0)) as run:
t.cp_into(f.name, "/etc/git-gate/access-hook")
remote_cmd = run.call_args.args[0][-1]
self.assertIn("chmod 700", remote_cmd) # exec bit preserved over SSH
def test_exec_raises_on_failure(self) -> None:
t = SshGatewayTransport(Path("/k"), "10.0.0.1")
with patch(f"{_GW}.subprocess.run",
return_value=CompletedProcess([], 1, stderr="nope")), \
self.assertRaises(GatewayProvisionError):
t.exec(["mkdir", "-p", "/git-gate"])
if __name__ == "__main__": if __name__ == "__main__":
tests/unit/test_firecracker_infra_vm.py
+37 -108
View File
@@ -16,10 +16,10 @@ from bot_bottle.backend.firecracker import infra_vm
class TestPushSecrets(unittest.TestCase): class TestPushSecrets(unittest.TestCase):
"""`_push_signing_key` (orchestrator) and `_push_gateway_jwt` (gateway) """`_push_signing_key` (orchestrator) seeds the guest's secret over SSH via
seed their guest's secret over SSH via `_push_secret`. The host token file `_push_secret`. The host token file stays the single source of truth, never
stays the single source of truth, never clobbered per-backend (#469); the clobbered per-backend (#469). The gateway's own JWT push lives with the
gateway gets a pre-minted `gateway` JWT, never the signing key.""" gateway service (test_firecracker_gateway)."""
def _infra(self) -> infra_vm.InfraVm: def _infra(self) -> infra_vm.InfraVm:
return infra_vm.InfraVm(guest_ip="10.0.0.1", private_key=Path("/k")) return infra_vm.InfraVm(guest_ip="10.0.0.1", private_key=Path("/k"))
@@ -36,17 +36,6 @@ class TestPushSecrets(unittest.TestCase):
self.assertIn(f"mv {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp " self.assertIn(f"mv {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp "
f"{infra_vm._GUEST_SIGNING_KEY_PATH}", remote_cmd) f"{infra_vm._GUEST_SIGNING_KEY_PATH}", remote_cmd)
def test_gateway_jwt_is_the_pre_minted_token_not_the_key(self):
# The host mints the `gateway` JWT and hands it in; `_push_gateway_jwt`
# only writes it. It is the JWT (never the key itself) that lands in the
# gateway VM.
proc = MagicMock(returncode=0, stderr="")
with patch.object(infra_vm.subprocess, "run", return_value=proc) as run:
infra_vm._push_gateway_jwt(self._infra(), "gw.jwt")
self.assertEqual("gw.jwt", run.call_args.kwargs["input"])
remote_cmd = run.call_args.args[0][-1]
self.assertIn(f"cat > {infra_vm._GUEST_GATEWAY_JWT_PATH}.tmp", remote_cmd)
def test_dies_when_push_never_succeeds(self): def test_dies_when_push_never_succeeds(self):
proc = MagicMock(returncode=255, stderr="ssh: connect refused") proc = MagicMock(returncode=255, stderr="ssh: connect refused")
with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \ with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \
@@ -71,12 +60,14 @@ class TestOrchestratorUrl(unittest.TestCase):
class TestInfraEndpoint(unittest.TestCase): class TestInfraEndpoint(unittest.TestCase):
"""The endpoint mirrors docker/macOS: it exposes the orchestrator's URL and """The endpoint mirrors docker/macOS: it exposes the orchestrator's URL and
delegates the CA fetch to the gateway VM handle.""" delegates the CA fetch to the gateway service."""
def _endpoint(self) -> infra_vm.InfraEndpoint: def _endpoint(self) -> infra_vm.InfraEndpoint:
from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
return infra_vm.InfraEndpoint( return infra_vm.InfraEndpoint(
orchestrator=infra_vm.InfraVm(guest_ip="10.243.255.1", private_key=Path("/k")), orchestrator=infra_vm.InfraVm(guest_ip="10.243.255.1", private_key=Path("/k")),
gateway=infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k")), gateway=FirecrackerGateway(
infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))),
) )
def test_orchestrator_url_is_the_orchestrator_vm(self): def test_orchestrator_url_is_the_orchestrator_vm(self):
@@ -84,9 +75,9 @@ class TestInfraEndpoint(unittest.TestCase):
self.assertEqual( self.assertEqual(
f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}", ep.orchestrator_url) f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}", ep.orchestrator_url)
def test_gateway_ca_pem_delegates_to_the_gateway_vm(self): def test_gateway_ca_pem_delegates_to_the_gateway_service(self):
ep = self._endpoint() ep = self._endpoint()
with patch.object(ep.gateway, "gateway_ca_pem", return_value="PEM") as ca: with patch.object(ep.gateway, "ca_cert_pem", return_value="PEM") as ca:
self.assertEqual("PEM", ep.gateway_ca_pem(timeout=1)) self.assertEqual("PEM", ep.gateway_ca_pem(timeout=1))
ca.assert_called_once_with(timeout=1) ca.assert_called_once_with(timeout=1)
@@ -132,9 +123,10 @@ class TestBuildInfraRootfs(unittest.TestCase):
class TestBootRole(unittest.TestCase): class TestBootRole(unittest.TestCase):
"""`boot_orchestrator` / `boot_gateway` boot the shared rootfs with the """`boot_orchestrator` boots the shared rootfs with the `bb_role=orchestrator`
right `bb_role` cmdline, memory ceiling, and (orchestrator-only) registry cmdline, the orchestrator memory ceiling + registry volume, and seeds the
volume; each seeds its own secret.""" signing key. (The gateway's boot lives in the gateway service —
test_firecracker_gateway.)"""
def _boot_env(self): def _boot_env(self):
vm = MagicMock() vm = MagicMock()
@@ -161,67 +153,6 @@ class TestBootRole(unittest.TestCase):
self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP
push.assert_called_once() # signing key seeded push.assert_called_once() # signing key seeded
def test_gateway_role_carries_orch_addr_and_no_registry(self):
import tempfile
vm = self._boot_env()
with tempfile.TemporaryDirectory() as td, \
patch.object(infra_vm.netpool, "tap_present", return_value=True), \
patch.object(infra_vm.infra_artifact, "local_build_requested", return_value=False), \
patch.object(infra_vm.infra_artifact, "materialize_ext4"), \
patch.object(infra_vm.infra_artifact, "infra_artifact_version", return_value="v"), \
patch.object(infra_vm, "_stable_keypair", return_value=(Path("/k"), "pub")), \
patch.object(infra_vm, "_push_gateway_jwt") as push, \
patch.object(infra_vm.firecracker_vm, "boot", return_value=vm) as boot, \
patch.object(infra_vm, "_gw_dir", return_value=Path(td)):
infra_vm.boot_gateway("10.243.255.1", "gw.jwt")
kw = boot.call_args.kwargs
self.assertIn("bb_role=gateway", kw["extra_boot_args"])
self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"]) # reaches the CP
self.assertEqual(infra_vm._GW_MEM_MIB, kw["mem_mib"])
self.assertIsNone(kw["data_drive"]) # data plane never opens the DB
# The host-minted token is threaded through to the JWT push.
self.assertEqual("gw.jwt", push.call_args.args[1])
class TestSshGatewayTransport(unittest.TestCase):
def test_cp_into_preserves_source_mode(self):
import tempfile
from subprocess import CompletedProcess
with tempfile.NamedTemporaryFile() as f:
os.chmod(f.name, 0o700) # like the staged access-hook
t = infra_vm.SshGatewayTransport(Path("/k"), "10.0.0.1")
with patch.object(infra_vm.subprocess, "run",
return_value=CompletedProcess([], 0)) as run:
t.cp_into(f.name, "/etc/git-gate/access-hook")
remote_cmd = run.call_args.args[0][-1]
self.assertIn("chmod 700", remote_cmd) # exec bit preserved over SSH
def test_exec_raises_on_failure(self):
from subprocess import CompletedProcess
t = infra_vm.SshGatewayTransport(Path("/k"), "10.0.0.1")
with patch.object(infra_vm.subprocess, "run",
return_value=CompletedProcess([], 1, stderr="nope")), \
self.assertRaises(infra_vm.GatewayProvisionError):
t.exec(["mkdir", "-p", "/git-gate"])
class TestGatewayTransportTargetsGatewayVm(unittest.TestCase):
def test_transport_uses_the_gateway_link_guest_ip(self):
# git-gate provisioning goes to the gateway VM (gw_slot), not the
# orchestrator, now that the planes are split.
t = infra_vm.gateway_transport()
self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, t._ip)
class TestGatewayCaPem(unittest.TestCase):
def test_dies_when_cert_never_appears(self) -> None:
from subprocess import CompletedProcess
gw = infra_vm.InfraVm(vm=None, guest_ip="10.0.0.1", private_key=Path("/k"))
with patch.object(infra_vm.subprocess, "run",
return_value=CompletedProcess([], 1, stdout="", stderr="")), \
self.assertRaises(SystemExit):
gw.gateway_ca_pem(timeout=0)
class TestRegistryVolume(unittest.TestCase): class TestRegistryVolume(unittest.TestCase):
def test_reuses_existing_volume(self): def test_reuses_existing_volume(self):
@@ -295,10 +226,16 @@ class TestWaitForHealth(unittest.TestCase):
infra_vm.wait_for_health(infra, timeout=5) infra_vm.wait_for_health(infra, timeout=5)
# The gateway service is imported lazily inside ensure_running / _adopt (to
# break the infra_vm <-> gateway import cycle), so patch it at its home module.
_GW_CLS = "bot_bottle.backend.firecracker.gateway.FirecrackerGateway"
class TestEnsureRunningSingleton(unittest.TestCase): class TestEnsureRunningSingleton(unittest.TestCase):
def test_adopts_when_healthy_alive_and_version_matches(self): def test_adopts_when_healthy_alive_and_version_matches(self):
# Healthy control plane + live gateway + existing key + matching version # Healthy control plane + live gateway + existing key + matching version
# marker -> adopt (no boot); both VM handles have vm=None. # marker -> adopt (no boot).
from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
import tempfile import tempfile
with tempfile.TemporaryDirectory() as td: with tempfile.TemporaryDirectory() as td:
d = Path(td) d = Path(td)
@@ -308,16 +245,15 @@ class TestEnsureRunningSingleton(unittest.TestCase):
patch.object(infra_vm, "_expected_version", return_value="v-current"), \ patch.object(infra_vm, "_expected_version", return_value="v-current"), \
patch.object(infra_vm, "_health_ok", return_value=True), \ patch.object(infra_vm, "_health_ok", return_value=True), \
patch.object(infra_vm, "_pidfile_alive", return_value=True), \ patch.object(infra_vm, "_pidfile_alive", return_value=True), \
patch.object(infra_vm, "boot_orchestrator") as boot_o, \ patch.object(infra_vm, "boot_orchestrator") as boot_o:
patch.object(infra_vm, "boot_gateway") as boot_g:
ep = infra_vm.ensure_running() ep = infra_vm.ensure_running()
boot_o.assert_not_called() boot_o.assert_not_called()
boot_g.assert_not_called()
self.assertIsNone(ep.orchestrator.vm) self.assertIsNone(ep.orchestrator.vm)
self.assertIsNone(ep.gateway.vm) # The orchestrator handle + the adopted gateway service address the two
# The two handles address the two distinct infra links. # distinct infra links.
self.assertEqual(infra_vm.netpool.orch_slot().guest_ip, ep.orchestrator.guest_ip) self.assertEqual(infra_vm.netpool.orch_slot().guest_ip, ep.orchestrator.guest_ip)
self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, ep.gateway.guest_ip) self.assertIsInstance(ep.gateway, FirecrackerGateway)
self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, ep.gateway.address())
def test_reboots_both_when_version_stale(self): def test_reboots_both_when_version_stale(self):
# Healthy control plane but the running pair booted an OLDER image # Healthy control plane but the running pair booted an OLDER image
@@ -337,19 +273,16 @@ class TestEnsureRunningSingleton(unittest.TestCase):
patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \ patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \
patch.object(infra_vm, "mint", return_value="gw.jwt"), \ patch.object(infra_vm, "mint", return_value="gw.jwt"), \
patch.object(infra_vm, "boot_orchestrator") as boot_o, \ patch.object(infra_vm, "boot_orchestrator") as boot_o, \
patch.object(infra_vm, "boot_gateway") as boot_g: patch(_GW_CLS) as gw_cls:
boot_o.return_value = infra_vm.InfraVm( boot_o.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
boot_g.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.3", private_key=Path("/k"), vm=MagicMock())
infra_vm.ensure_running() infra_vm.ensure_running()
stop.assert_called_once() # dislodge the outdated pair stop.assert_called_once() # dislodge the outdated pair
boot_o.assert_called_once() boot_o.assert_called_once()
boot_g.assert_called_once() # The gateway service is bound to the orchestrator's URL, carrying
# The gateway is booted pointing at the orchestrator's guest IP, # the host-minted `gateway` token.
# carrying the host-minted `gateway` token. gw_cls.return_value.connect_to_orchestrator.assert_called_once_with(
self.assertEqual("10.243.255.1", boot_g.call_args.args[0]) "http://10.243.255.1:8099", "gw.jwt")
self.assertEqual("gw.jwt", boot_g.call_args.args[1])
# The fresh boot records the current version for the next launcher. # The fresh boot records the current version for the next launcher.
self.assertEqual("v-current\n", (d / "booted-version").read_text()) self.assertEqual("v-current\n", (d / "booted-version").read_text())
@@ -371,15 +304,13 @@ class TestEnsureRunningSingleton(unittest.TestCase):
patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \ patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \
patch.object(infra_vm, "mint", return_value="gw.jwt"), \ patch.object(infra_vm, "mint", return_value="gw.jwt"), \
patch.object(infra_vm, "boot_orchestrator") as boot_o, \ patch.object(infra_vm, "boot_orchestrator") as boot_o, \
patch.object(infra_vm, "boot_gateway") as boot_g: patch(_GW_CLS) as gw_cls:
boot_o.return_value = infra_vm.InfraVm( boot_o.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
boot_g.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.3", private_key=Path("/k"), vm=MagicMock())
infra_vm.ensure_running() infra_vm.ensure_running()
stop.assert_called_once() stop.assert_called_once()
boot_o.assert_called_once() boot_o.assert_called_once()
boot_g.assert_called_once() gw_cls.return_value.connect_to_orchestrator.assert_called_once()
def test_boots_both_when_unhealthy(self): def test_boots_both_when_unhealthy(self):
import tempfile import tempfile
@@ -392,20 +323,18 @@ class TestEnsureRunningSingleton(unittest.TestCase):
patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \ patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \
patch.object(infra_vm, "mint", return_value="gw.jwt"), \ patch.object(infra_vm, "mint", return_value="gw.jwt"), \
patch.object(infra_vm, "boot_orchestrator") as boot_o, \ patch.object(infra_vm, "boot_orchestrator") as boot_o, \
patch.object(infra_vm, "boot_gateway") as boot_g, \ patch(_GW_CLS) as gw_cls, \
patch.object(infra_vm, "wait_for_health") as wait: patch.object(infra_vm, "wait_for_health") as wait:
boot_o.return_value = infra_vm.InfraVm( boot_o.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock()) guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
boot_g.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.3", private_key=Path("/k"), vm=MagicMock())
infra_vm.ensure_running() infra_vm.ensure_running()
stop.assert_called_once() # clear stale VMs first stop.assert_called_once() # clear stale VMs first
built.assert_called_once() built.assert_called_once()
# Orchestrator boots + becomes healthy BEFORE the gateway (its daemons # Orchestrator boots + becomes healthy BEFORE the gateway is connected
# reach the control plane at startup). # (its daemons reach the control plane at startup).
boot_o.assert_called_once() boot_o.assert_called_once()
wait.assert_called_once() wait.assert_called_once()
boot_g.assert_called_once() gw_cls.return_value.connect_to_orchestrator.assert_called_once()
class TestStop(unittest.TestCase): class TestStop(unittest.TestCase):