refactor(firecracker): move gateway logic into the gateway service

Pull the gateway's host-side logic out of `infra_vm` and into
`FirecrackerGateway`'s own methods, so the gateway is self-contained and
`infra_vm` shrinks toward being just the pair coordinator (a step toward
removing it). Now living in the gateway service: the gateway VM boot +
`bb_orch` cmdline, the pre-minted JWT push, the mitmproxy CA fetch over SSH,
the `SshGatewayTransport` exec/cp, and the lifecycle predicates
(is_running/stop/address). `ensure_running` / `_adopt` construct the gateway
and call `connect_to_orchestrator` (lazy import to break the cycle); the
InfraEndpoint's gateway is now the `FirecrackerGateway` service.

What deliberately stays shared in `infra_vm` (documented at the top of
gateway.py): the plane-agnostic VM substrate the orchestrator VM also needs
(`_boot_vm`, the stable SSH keypair, the secret-push retry, PID lifecycle), the
pair coordinator (`ensure_running`: orchestrator-first health gate + singleton
lock + adoption/version marker), and the single shared `bb_role`-branched init
baked into the one published rootfs both VMs boot — the guest-side gateway
startup can't live in a host method. These are the seam that moves to a neutral
module when the Orchestrator service lands and `infra_vm` dissolves.

CA fetch now raises GatewayError (was die/SystemExit) to match the ABC.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-25 13:34:36 -04:00
parent d54c559a3a
commit 5bf9f052b9
5 changed files with 262 additions and 316 deletions
@@ -130,9 +130,8 @@ def launch_consolidated(
client = OrchestratorClient(url)
_reprovision_running_bottles(client)
# Read the gateway's provisioning transport + CA off the Gateway service
# (adapting the running gateway VM).
gateway = FirecrackerGateway(infra.gateway)
# Read the gateway's provisioning transport + CA off the Gateway service.
gateway = infra.gateway
transport = gateway.provisioning_transport()
reg = provision_bottle(
client, guest_ip, egress_plan, git_gate_plan, transport,
+122 -25
View File
@@ -1,38 +1,100 @@
"""The Firecracker gateway data plane as a microVM (PRD 0070).
`FirecrackerGateway` is the Firecracker implementation of the backend-neutral
`Gateway` service. The gateway here is a whole microVM — the slim data-plane
unit (mitmproxy TLS bump + DLP + git-http) booted on its own NAT'd link; agent
VMs' gateway-port traffic is DNAT'd to it and it never opens `bot-bottle.db`
(#469), so it holds no signing key, only the pre-minted `gateway` token the
host hands it via `connect_to_orchestrator`.
`Gateway` service, and owns the gateway's host-side logic directly: booting the
gateway microVM on its NAT'd link, seeding the pre-minted `gateway` token,
fetching the mitmproxy CA, and the SSH exec/cp provisioning transport. The
gateway VM never opens `bot-bottle.db` (#469), so it holds no signing key —
only the token the host hands it via `connect_to_orchestrator`.
The VM lifecycle primitives (boot, SSH secret push, netpool links, nft, PID
files) stay in `infra_vm`, which drives the orchestrator+gateway pair as a
singleton; this adapter composes them into the uniform `Gateway` surface.
What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM
substrate the orchestrator VM shares — booting a VM from the shared rootfs
(`_boot_vm`), the stable SSH keypair, the secret-push retry loop, and the PID
lifecycle — plus the pair coordinator (`ensure_running`: orchestrator-first
health gate, singleton lock, adoption/version marker). The guest-side gateway
daemon startup lives in the shared `bb_role=gateway` init branch baked into the
one published rootfs both VMs boot, so it can't live in a host-side method
either. These are the shared seam that moves to a neutral module when the
Orchestrator service lands and `infra_vm` is dissolved.
"""
from __future__ import annotations
import os
import shlex
import stat
import subprocess
from pathlib import Path
from urllib.parse import urlparse
from ...gateway import (
DEFAULT_CA_TIMEOUT_SECONDS,
Gateway,
GatewayError,
GatewayProvisionError,
GatewayTransport,
)
from . import infra_vm
from .. import util as backend_util
from . import infra_vm, netpool, util
# The gateway microVM's name (the `bb_role=gateway` boot tag / run dir). Fixed
# per host — one gateway VM, shared by every agent VM.
GATEWAY_NAME = "bot-bottle-gateway"
# The gateway VM's slim memory ceiling (buildah is present on the shared rootfs
# but unused on the data plane) — PRD 0070 "Memory: fixed ceilings".
_GW_MEM_MIB = 2048
# The pre-minted `gateway` JWT path in the guest. The *shared* init (both planes
# boot one rootfs) waits for it before starting the data plane, so its canonical
# definition lives with that init in `infra_vm`; imported here for the push so
# the load-bearing path isn't duplicated.
_GUEST_GATEWAY_JWT_PATH = infra_vm._GUEST_GATEWAY_JWT_PATH
# mitmproxy writes its CA here a beat after start; agents install it to trust
# the gateway's TLS interception. Host-side only (SSH cat), so it lives here.
_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem"
_CA_FETCH_TIMEOUT_SECONDS = 15.0
class SshGatewayTransport:
"""`GatewayTransport` for the gateway running in the gateway VM — the docker
exec/cp equivalents over SSH (dropbear + the stable infra key)."""
def __init__(self, private_key: Path, guest_ip: str) -> None:
self._key = private_key
self._ip = guest_ip
def exec(self, argv: list[str]) -> None:
proc = subprocess.run(
util.ssh_base_argv(self._key, self._ip) + [shlex.join(argv)],
capture_output=True, text=True, timeout=60, check=False,
)
if proc.returncode != 0:
raise GatewayProvisionError(
f"infra gateway exec {argv!r} failed: {proc.stderr.strip()}")
def cp_into(self, src: str, dest: str) -> None:
# Preserve the source mode (docker cp does): the access-hook is staged
# 0700 and git-http execs it directly — a plain `cat >` would land it
# 0644 and the exec fails with EACCES; keys stay 0600.
mode = stat.S_IMODE(os.stat(src).st_mode)
q = shlex.quote(dest)
proc = subprocess.run(
util.ssh_base_argv(self._key, self._ip)
+ [f"cat > {q} && chmod {mode:o} {q}"],
input=Path(src).read_bytes(), capture_output=True, timeout=30, check=False,
)
if proc.returncode != 0:
raise GatewayProvisionError(
f"infra gateway cp {src} -> {dest} failed: "
f"{proc.stderr.decode(errors='replace').strip()}")
class FirecrackerGateway(Gateway):
"""The consolidated gateway as a Firecracker microVM on the gateway link.
The rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's
The shared rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's
`ensure_built` no-op here); `connect_to_orchestrator` boots the VM resolving
policy against the orchestrator and seeds the pre-minted `gateway` token."""
@@ -47,10 +109,12 @@ class FirecrackerGateway(Gateway):
self._gateway_token = ""
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
"""Boot the gateway VM resolving policy against `orchestrator_url` and
seed `gateway_token`. The orchestrator's guest IP is parsed from the URL
and passed on the cmdline, so the baked init stays IP-independent. Boot
the orchestrator first — the gateway daemons reach it at startup."""
"""Boot the gateway VM on its link resolving policy against
`orchestrator_url`, then seed `gateway_token` over SSH. The orchestrator's
guest IP is parsed from the URL and passed on the cmdline (`bb_orch=`), so
the baked init stays IP-independent. Boot the orchestrator first — the
gateway daemons reach it at startup. The gateway never mints, so it holds
no signing key, only this token (#469)."""
self._orchestrator_url = orchestrator_url
self._gateway_token = gateway_token
if not self._orchestrator_url:
@@ -68,31 +132,64 @@ class FirecrackerGateway(Gateway):
raise GatewayError(
f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}"
)
self._vm = infra_vm.boot_gateway(orchestrator_guest_ip, self._gateway_token)
# Boot on the gateway link from the shared rootfs (bb_role=gateway), then
# push the token the init waits for before starting the data plane.
vm = infra_vm._boot_vm(
name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(),
role="gateway", mem_mib=_GW_MEM_MIB,
extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
)
infra_vm._push_secret(
vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH,
"the gateway JWT to the gateway VM (its data plane will not start)",
)
self._vm = vm
def is_running(self) -> bool:
return infra_vm.gateway_running()
return infra_vm._pidfile_alive(infra_vm._gw_dir())
def stop(self) -> None:
"""Stop only the gateway VM (idempotent)."""
infra_vm.stop_gateway()
"""Stop only the gateway VM (idempotent — absent is success). A dead
gateway already fails `infra_vm._adoptable`, so no version marker to
clear here."""
infra_vm._kill_pidfile(infra_vm._gw_dir())
infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True)
def address(self) -> str:
"""The gateway VM's guest IP — the agent-facing target agent VMs'
gateway-port traffic is DNAT'd to."""
return infra_vm.gateway_guest_ip()
return netpool.gw_slot().guest_ip
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
interception. Fetched from the gateway VM over SSH; polls because
mitmproxy writes it a beat after boot."""
vm = self._vm or infra_vm.adopted_gateway_vm()
return vm.gateway_ca_pem(timeout=timeout)
mitmproxy writes it a beat after boot. Works from any later launcher —
falls back to the stable key + fixed link when this process didn't boot
the VM."""
key = self._vm.private_key if self._vm else infra_vm._infra_dir() / "id_ed25519"
ip = self._vm.guest_ip if self._vm else netpool.gw_slot().guest_ip
def _fetch() -> str | None:
proc = subprocess.run(
util.ssh_base_argv(key, ip) + [f"cat {_GATEWAY_CA_PATH}"],
capture_output=True, text=True,
timeout=_CA_FETCH_TIMEOUT_SECONDS, check=False,
)
ok = proc.returncode == 0 and "BEGIN CERTIFICATE" in proc.stdout
return proc.stdout if ok else None
try:
return backend_util.poll_ca_cert(_fetch, timeout=timeout)
except TimeoutError as exc:
raise GatewayError(str(exc)) from exc
def provisioning_transport(self) -> GatewayTransport:
"""The exec/cp transport git-gate provisioning stages per-bottle repos +
deploy keys through (over SSH to the gateway VM)."""
return infra_vm.gateway_transport()
deploy keys through (over SSH to the gateway VM). Needs no live handle —
built from the stable key + the gateway link's guest IP, so teardown can
use it too."""
return SshGatewayTransport(
infra_vm._infra_dir() / "id_ed25519", netpool.gw_slot().guest_ip)
__all__ = ["FirecrackerGateway", "GATEWAY_NAME"]
__all__ = ["FirecrackerGateway", "SshGatewayTransport", "GATEWAY_NAME"]
+23 -143
View File
@@ -32,9 +32,7 @@ from __future__ import annotations
import fcntl
import hashlib
import os
import shlex
import signal
import stat
import subprocess
import time
import urllib.error
@@ -42,16 +40,17 @@ import urllib.request
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import Generator
from typing import TYPE_CHECKING, Generator
from ...log import die, info
from ...orchestrator_auth import ROLE_GATEWAY, mint
from ...paths import host_orchestrator_token
from .. import util as backend_util
from ..docker import util as docker_mod
from ...gateway import GatewayProvisionError
from . import firecracker_vm, infra_artifact, netpool, util
if TYPE_CHECKING:
from .gateway import FirecrackerGateway
# The orchestrator VM's signing-key path on its persistent /dev/vdb volume
# (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds this
# file with the host-canonical key AFTER boot (over SSH), so the VM verifies
@@ -77,16 +76,11 @@ ORCHESTRATOR_PORT = 8099
EGRESS_PORT = 9099
SUPERVISE_PORT = 9100
GIT_HTTP_PORT = 9420
# mitmproxy writes its CA here a beat after start; agents install it to trust
# the gateway's TLS interception.
_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem"
# Memory ceilings (fixed at boot, demand-paged — no balloon reclaim). The
# Memory ceiling (fixed at boot, demand-paged — no balloon reclaim). The
# orchestrator keeps the build headroom (buildah's 2-4 GB working set during
# in-VM agent builds); the gateway is the slim unit (mitmproxy TLS bump + DLP
# body buffering, ~1 GB) — PRD 0070 "Memory: fixed ceilings".
# in-VM agent builds); the gateway's slimmer ceiling is set by the gateway
# service — PRD 0070 "Memory: fixed ceilings".
_ORCH_MEM_MIB = 4096
_GW_MEM_MIB = 2048
# The infra VMs make direct upstream connections (gateway egress, and buildah
# during builds), and the kernel `ip=` cmdline sets no resolver. Public for
@@ -117,41 +111,23 @@ class InfraVm:
def orchestrator_url(self) -> str:
return f"http://{self.guest_ip}:{ORCHESTRATOR_PORT}"
def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str:
"""The gateway's mitmproxy CA (PEM) that agents install to trust its
TLS interception. Generated a moment after boot, so this polls over
SSH until it appears (mirrors DockerGateway.ca_cert_pem). Call on the
gateway VM handle."""
def _fetch() -> str | None:
proc = subprocess.run(
util.ssh_base_argv(self.private_key, self.guest_ip)
+ [f"cat {_GATEWAY_CA_PATH}"],
capture_output=True, text=True, timeout=15, check=False,
)
ok = proc.returncode == 0 and "BEGIN CERTIFICATE" in proc.stdout
return proc.stdout if ok else None
try:
return backend_util.poll_ca_cert(_fetch, timeout=timeout)
except TimeoutError as exc:
die(str(exc))
@dataclass
class InfraEndpoint:
"""How to reach the running per-host pair. `orchestrator` is the control
plane (host CLI + registration + in-VM builds); `gateway` is the data
plane (agent egress / git-http / supervise, CA fetch, git-gate
plane (host CLI + registration + in-VM builds); `gateway` is the data-plane
`Gateway` service (agent egress / git-http / supervise, CA fetch, git-gate
provisioning). Mirrors the docker/macOS `InfraEndpoint` shape."""
orchestrator: InfraVm
gateway: InfraVm
gateway: "FirecrackerGateway"
@property
def orchestrator_url(self) -> str:
return self.orchestrator.orchestrator_url
def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str:
return self.gateway.gateway_ca_pem(timeout=timeout)
return self.gateway.ca_cert_pem(timeout=timeout)
def ensure_built() -> None:
@@ -229,22 +205,28 @@ def ensure_running() -> InfraEndpoint:
orchestrator = boot_orchestrator()
wait_for_health(orchestrator)
# The orchestrator holds the signing key; mint the role-scoped `gateway`
# JWT on the host and hand it to the gateway VM, which never sees the key
# (#469).
# JWT on the host and hand it to the gateway service, which never sees
# the key (#469). The gateway owns its own boot — see FirecrackerGateway.
# Local import: the gateway service imports this module for the shared VM
# substrate, so importing it at module scope would cycle.
from .gateway import FirecrackerGateway
gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token())
gateway = boot_gateway(orchestrator.guest_ip, gateway_token)
gateway = FirecrackerGateway()
gateway.connect_to_orchestrator(orchestrator.orchestrator_url, gateway_token)
_record_booted_version(want)
return InfraEndpoint(orchestrator=orchestrator, gateway=gateway)
def _adopt(key: Path) -> InfraEndpoint:
"""Build handles to the already-running pair from the stable key + the
fixed links (vm=None — teardown then goes through the PID files)."""
fixed links (no live VM handle — teardown / CA fetch go through the PID
files + stable key)."""
from .gateway import FirecrackerGateway
return InfraEndpoint(
orchestrator=InfraVm(
guest_ip=netpool.orch_slot().guest_ip, private_key=key),
gateway=InfraVm(
guest_ip=netpool.gw_slot().guest_ip, private_key=key),
gateway=FirecrackerGateway(
InfraVm(guest_ip=netpool.gw_slot().guest_ip, private_key=key)),
)
@@ -294,25 +276,6 @@ def boot_orchestrator() -> InfraVm:
return orch
def boot_gateway(orchestrator_guest_ip: str, gateway_token: str) -> InfraVm:
"""Boot the gateway (data-plane) VM (detached) on the gateway link, and
seed the pre-minted `gateway` JWT `gateway_token`. Its daemons resolve
policy against the orchestrator at `orchestrator_guest_ip:8099` (passed on
the cmdline, so the baked init stays IP-independent). Boot the orchestrator
first — the gateway daemons reach it at startup. The gateway never holds the
signing key, only this token (#469). Prefer `ensure_running`."""
slot = netpool.gw_slot()
gateway = _boot_vm(
name="bot-bottle-gateway", slot=slot, run_dir=_gw_dir(),
role="gateway", mem_mib=_GW_MEM_MIB,
extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
)
# Push the pre-minted `gateway` JWT (never the signing key — issue #469).
# The init waits for it before starting the data plane.
_push_gateway_jwt(gateway, gateway_token)
return gateway
def _boot_vm(
*,
name: str,
@@ -456,18 +419,6 @@ def _push_signing_key(infra: InfraVm) -> None:
)
def _push_gateway_jwt(infra: InfraVm, gateway_token: str) -> None:
"""Push the pre-minted `gateway` JWT into the freshly booted gateway VM
over SSH (atomic write). Minted on the HOST from the canonical signing key
so the data plane never holds the key itself (issue #469); the gateway
daemons present it to the orchestrator. Retries until SSH is up; dies if it
never lands, since the VM then refuses to start the data plane."""
_push_secret(
infra, gateway_token, _GUEST_GATEWAY_JWT_PATH,
"the gateway JWT to the gateway VM (its data plane will not start)",
)
def _push_secret(infra: InfraVm, secret: str, dest: str, what: str) -> None:
"""Pipe `secret` to an atomic write of `dest` in the guest over SSH,
retrying while the guest's SSH comes up; die (naming `what`) if it never
@@ -574,77 +525,6 @@ def _health_ok(url: str) -> bool:
return False
class SshGatewayTransport:
"""`GatewayTransport` for the gateway running in the gateway VM — the
docker exec/cp equivalents over SSH (dropbear + the stable infra key)."""
def __init__(self, private_key: Path, guest_ip: str) -> None:
self._key = private_key
self._ip = guest_ip
def exec(self, argv: list[str]) -> None:
proc = subprocess.run(
util.ssh_base_argv(self._key, self._ip) + [shlex.join(argv)],
capture_output=True, text=True, timeout=60, check=False,
)
if proc.returncode != 0:
raise GatewayProvisionError(
f"infra gateway exec {argv!r} failed: {proc.stderr.strip()}")
def cp_into(self, src: str, dest: str) -> None:
# Preserve the source mode (docker cp does): the access-hook is staged
# 0700 and git-http execs it directly — a plain `cat >` would land it
# 0644 and the exec fails with EACCES; keys stay 0600.
mode = stat.S_IMODE(os.stat(src).st_mode)
q = shlex.quote(dest)
proc = subprocess.run(
util.ssh_base_argv(self._key, self._ip)
+ [f"cat > {q} && chmod {mode:o} {q}"],
input=Path(src).read_bytes(), capture_output=True, timeout=30, check=False,
)
if proc.returncode != 0:
raise GatewayProvisionError(
f"infra gateway cp {src} -> {dest} failed: "
f"{proc.stderr.decode(errors='replace').strip()}")
def gateway_transport() -> SshGatewayTransport:
"""git-gate provisioning transport for the gateway VM, built from the
stable key + the gateway link's guest IP. Needs no live VM handle, so
teardown can use it too."""
return SshGatewayTransport(
_infra_dir() / "id_ed25519", netpool.gw_slot().guest_ip)
def gateway_guest_ip() -> str:
"""The gateway VM's fixed guest IP on its link — the agent-facing address
agent VMs' gateway-port traffic is DNAT'd to (PRD 0070)."""
return netpool.gw_slot().guest_ip
def gateway_running() -> bool:
"""True iff the gateway VM is still a live VMM."""
return _pidfile_alive(_gw_dir())
def stop_gateway() -> None:
"""Stop only the gateway VM (idempotent — absent is success). Drops the pair
version marker so a half-stopped pair is never adopted (`_adoptable`)."""
_kill_pidfile(_gw_dir())
_pid_file(_gw_dir()).unlink(missing_ok=True)
_version_file().unlink(missing_ok=True)
def adopted_gateway_vm() -> InfraVm:
"""A handle to the already-running gateway VM from the stable key + the
fixed gateway link (vm=None — CA fetch / provisioning work from any later
launcher, not only the process that booted it)."""
return InfraVm(
guest_ip=netpool.gw_slot().guest_ip,
private_key=_infra_dir() / "id_ed25519",
)
def wait_for_health(
infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS,
) -> None:
+78 -37
View File
@@ -1,17 +1,26 @@
"""Unit: the Firecracker gateway data plane as a microVM (PRD 0070)."""
"""Unit: the Firecracker gateway data plane as a microVM (PRD 0070).
The gateway service owns its host-side logic directly: booting the gateway VM
(via the shared `infra_vm._boot_vm` substrate), seeding the pre-minted token,
fetching the CA over SSH, and the SSH exec/cp provisioning transport.
"""
from __future__ import annotations
import os
import tempfile
import unittest
from pathlib import Path
from unittest.mock import MagicMock, patch
from subprocess import CompletedProcess
from unittest.mock import patch
from bot_bottle.backend.firecracker import infra_vm
from bot_bottle.backend.firecracker.gateway import (
GATEWAY_NAME,
FirecrackerGateway,
SshGatewayTransport,
)
from bot_bottle.gateway import GatewayError
from bot_bottle.gateway import GatewayError, GatewayProvisionError
_GW = "bot_bottle.backend.firecracker.gateway"
@@ -25,76 +34,108 @@ class TestFirecrackerGatewayConnect(unittest.TestCase):
def test_refuses_without_orchestrator_url(self) -> None:
gw = FirecrackerGateway()
with patch.object(infra_vm, "boot_gateway") as boot:
with patch.object(infra_vm, "_boot_vm") as boot:
with self.assertRaises(GatewayError):
gw.connect_to_orchestrator("", _TOKEN)
boot.assert_not_called()
def test_refuses_without_gateway_token(self) -> None:
gw = FirecrackerGateway()
with patch.object(infra_vm, "boot_gateway") as boot:
with patch.object(infra_vm, "_boot_vm") as boot:
with self.assertRaises(GatewayError):
gw.connect_to_orchestrator(_ORCH_URL, "")
boot.assert_not_called()
def test_refuses_a_url_without_a_host(self) -> None:
gw = FirecrackerGateway()
with patch.object(infra_vm, "boot_gateway") as boot:
with patch.object(infra_vm, "_boot_vm") as boot:
with self.assertRaises(GatewayError):
gw.connect_to_orchestrator("http://:8099", _TOKEN)
boot.assert_not_called()
def test_boots_the_gateway_vm_against_the_orchestrator_guest_ip(self) -> None:
def test_boots_the_gateway_vm_and_seeds_the_token(self) -> None:
gw = FirecrackerGateway()
booted = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))
with patch.object(infra_vm, "boot_gateway", return_value=booted) as boot:
with patch.object(infra_vm, "_boot_vm", return_value=booted) as boot, \
patch.object(infra_vm, "_push_secret") as push:
gw.connect_to_orchestrator(_ORCH_URL, _TOKEN)
# The orchestrator guest IP is parsed off the URL; the token is threaded
# through unchanged (the gateway never mints — #469).
boot.assert_called_once_with("10.243.255.1", _TOKEN)
# Booted on the gateway link with the gateway role; the orchestrator's
# guest IP (parsed off the URL) rides the cmdline as bb_orch.
kw = boot.call_args.kwargs
self.assertEqual("gateway", kw["role"])
self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"])
self.assertIsNone(kw.get("data_drive")) # data plane never opens the DB
# The host-minted token (never the key — #469) is pushed to the guest.
push.assert_called_once()
self.assertEqual(_TOKEN, push.call_args.args[1])
class TestFirecrackerGatewaySurface(unittest.TestCase):
def test_is_running_reads_the_gateway_pidfile(self) -> None:
gw = FirecrackerGateway()
with patch.object(infra_vm, "gateway_running", return_value=True):
with patch.object(infra_vm, "_pidfile_alive", return_value=True) as alive, \
patch.object(infra_vm, "_gw_dir", return_value=Path("/gw")):
self.assertTrue(gw.is_running())
with patch.object(infra_vm, "gateway_running", return_value=False):
self.assertFalse(gw.is_running())
alive.assert_called_once_with(Path("/gw"))
def test_stop_stops_only_the_gateway_vm(self) -> None:
def test_stop_kills_only_the_gateway_pidfile(self) -> None:
gw = FirecrackerGateway()
with patch.object(infra_vm, "stop_gateway") as stop:
gw.stop()
stop.assert_called_once_with()
with tempfile.TemporaryDirectory() as td:
d = Path(td)
(d / "vm.pid").write_text("123")
with patch.object(infra_vm, "_gw_dir", return_value=d), \
patch.object(infra_vm, "_kill_pidfile") as kill:
gw.stop()
kill.assert_called_once_with(d)
self.assertFalse((d / "vm.pid").exists()) # pidfile cleared
def test_address_is_the_gateway_link_guest_ip(self) -> None:
gw = FirecrackerGateway()
with patch.object(infra_vm, "gateway_guest_ip", return_value="10.243.255.3"):
self.assertEqual("10.243.255.3", gw.address())
self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, gw.address())
def test_ca_cert_pem_uses_the_live_handle_when_present(self) -> None:
vm = MagicMock()
vm.gateway_ca_pem.return_value = "PEM"
def test_ca_cert_pem_reads_over_ssh_from_the_handle(self) -> None:
vm = infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))
gw = FirecrackerGateway(vm)
with patch.object(infra_vm, "adopted_gateway_vm") as adopt:
self.assertEqual("PEM", gw.ca_cert_pem(timeout=1))
adopt.assert_not_called()
vm.gateway_ca_pem.assert_called_once_with(timeout=1)
pem = "-----BEGIN CERTIFICATE-----\nx\n-----END CERTIFICATE-----\n"
with patch.object(infra_vm.subprocess, "run",
return_value=CompletedProcess([], 0, stdout=pem)) as run:
self.assertEqual(pem, gw.ca_cert_pem(timeout=5))
# cat the CA path over SSH to the gateway VM's guest IP.
argv = run.call_args.args[0]
self.assertTrue(any("10.243.255.3" in a for a in argv))
self.assertIn("cat /home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem", argv)
def test_ca_cert_pem_adopts_the_running_gateway_when_no_handle(self) -> None:
vm = MagicMock()
vm.gateway_ca_pem.return_value = "PEM"
def test_ca_cert_pem_raises_gateway_error_when_absent(self) -> None:
gw = FirecrackerGateway()
with patch.object(infra_vm, "adopted_gateway_vm", return_value=vm) as adopt:
self.assertEqual("PEM", gw.ca_cert_pem())
adopt.assert_called_once_with()
with patch.object(infra_vm.subprocess, "run",
return_value=CompletedProcess([], 1, stdout="", stderr="")):
with self.assertRaises(GatewayError):
gw.ca_cert_pem(timeout=0)
def test_provisioning_transport_targets_the_gateway_vm(self) -> None:
def test_provisioning_transport_targets_the_gateway_link(self) -> None:
gw = FirecrackerGateway()
sentinel = object()
with patch.object(infra_vm, "gateway_transport", return_value=sentinel):
self.assertIs(sentinel, gw.provisioning_transport())
transport = gw.provisioning_transport()
assert isinstance(transport, SshGatewayTransport)
self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, transport._ip)
class TestSshGatewayTransport(unittest.TestCase):
def test_cp_into_preserves_source_mode(self) -> None:
with tempfile.NamedTemporaryFile() as f:
os.chmod(f.name, 0o700) # like the staged access-hook
t = SshGatewayTransport(Path("/k"), "10.0.0.1")
with patch(f"{_GW}.subprocess.run",
return_value=CompletedProcess([], 0)) as run:
t.cp_into(f.name, "/etc/git-gate/access-hook")
remote_cmd = run.call_args.args[0][-1]
self.assertIn("chmod 700", remote_cmd) # exec bit preserved over SSH
def test_exec_raises_on_failure(self) -> None:
t = SshGatewayTransport(Path("/k"), "10.0.0.1")
with patch(f"{_GW}.subprocess.run",
return_value=CompletedProcess([], 1, stderr="nope")), \
self.assertRaises(GatewayProvisionError):
t.exec(["mkdir", "-p", "/git-gate"])
if __name__ == "__main__":
+37 -108
View File
@@ -16,10 +16,10 @@ from bot_bottle.backend.firecracker import infra_vm
class TestPushSecrets(unittest.TestCase):
"""`_push_signing_key` (orchestrator) and `_push_gateway_jwt` (gateway)
seed their guest's secret over SSH via `_push_secret`. The host token file
stays the single source of truth, never clobbered per-backend (#469); the
gateway gets a pre-minted `gateway` JWT, never the signing key."""
"""`_push_signing_key` (orchestrator) seeds the guest's secret over SSH via
`_push_secret`. The host token file stays the single source of truth, never
clobbered per-backend (#469). The gateway's own JWT push lives with the
gateway service (test_firecracker_gateway)."""
def _infra(self) -> infra_vm.InfraVm:
return infra_vm.InfraVm(guest_ip="10.0.0.1", private_key=Path("/k"))
@@ -36,17 +36,6 @@ class TestPushSecrets(unittest.TestCase):
self.assertIn(f"mv {infra_vm._GUEST_SIGNING_KEY_PATH}.tmp "
f"{infra_vm._GUEST_SIGNING_KEY_PATH}", remote_cmd)
def test_gateway_jwt_is_the_pre_minted_token_not_the_key(self):
# The host mints the `gateway` JWT and hands it in; `_push_gateway_jwt`
# only writes it. It is the JWT (never the key itself) that lands in the
# gateway VM.
proc = MagicMock(returncode=0, stderr="")
with patch.object(infra_vm.subprocess, "run", return_value=proc) as run:
infra_vm._push_gateway_jwt(self._infra(), "gw.jwt")
self.assertEqual("gw.jwt", run.call_args.kwargs["input"])
remote_cmd = run.call_args.args[0][-1]
self.assertIn(f"cat > {infra_vm._GUEST_GATEWAY_JWT_PATH}.tmp", remote_cmd)
def test_dies_when_push_never_succeeds(self):
proc = MagicMock(returncode=255, stderr="ssh: connect refused")
with patch.object(infra_vm, "host_orchestrator_token", return_value="host-key"), \
@@ -71,12 +60,14 @@ class TestOrchestratorUrl(unittest.TestCase):
class TestInfraEndpoint(unittest.TestCase):
"""The endpoint mirrors docker/macOS: it exposes the orchestrator's URL and
delegates the CA fetch to the gateway VM handle."""
delegates the CA fetch to the gateway service."""
def _endpoint(self) -> infra_vm.InfraEndpoint:
from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
return infra_vm.InfraEndpoint(
orchestrator=infra_vm.InfraVm(guest_ip="10.243.255.1", private_key=Path("/k")),
gateway=infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k")),
gateway=FirecrackerGateway(
infra_vm.InfraVm(guest_ip="10.243.255.3", private_key=Path("/k"))),
)
def test_orchestrator_url_is_the_orchestrator_vm(self):
@@ -84,9 +75,9 @@ class TestInfraEndpoint(unittest.TestCase):
self.assertEqual(
f"http://10.243.255.1:{infra_vm.ORCHESTRATOR_PORT}", ep.orchestrator_url)
def test_gateway_ca_pem_delegates_to_the_gateway_vm(self):
def test_gateway_ca_pem_delegates_to_the_gateway_service(self):
ep = self._endpoint()
with patch.object(ep.gateway, "gateway_ca_pem", return_value="PEM") as ca:
with patch.object(ep.gateway, "ca_cert_pem", return_value="PEM") as ca:
self.assertEqual("PEM", ep.gateway_ca_pem(timeout=1))
ca.assert_called_once_with(timeout=1)
@@ -132,9 +123,10 @@ class TestBuildInfraRootfs(unittest.TestCase):
class TestBootRole(unittest.TestCase):
"""`boot_orchestrator` / `boot_gateway` boot the shared rootfs with the
right `bb_role` cmdline, memory ceiling, and (orchestrator-only) registry
volume; each seeds its own secret."""
"""`boot_orchestrator` boots the shared rootfs with the `bb_role=orchestrator`
cmdline, the orchestrator memory ceiling + registry volume, and seeds the
signing key. (The gateway's boot lives in the gateway service —
test_firecracker_gateway.)"""
def _boot_env(self):
vm = MagicMock()
@@ -161,67 +153,6 @@ class TestBootRole(unittest.TestCase):
self.assertEqual(Path("/reg"), kw["data_drive"]) # DB volume on the CP
push.assert_called_once() # signing key seeded
def test_gateway_role_carries_orch_addr_and_no_registry(self):
import tempfile
vm = self._boot_env()
with tempfile.TemporaryDirectory() as td, \
patch.object(infra_vm.netpool, "tap_present", return_value=True), \
patch.object(infra_vm.infra_artifact, "local_build_requested", return_value=False), \
patch.object(infra_vm.infra_artifact, "materialize_ext4"), \
patch.object(infra_vm.infra_artifact, "infra_artifact_version", return_value="v"), \
patch.object(infra_vm, "_stable_keypair", return_value=(Path("/k"), "pub")), \
patch.object(infra_vm, "_push_gateway_jwt") as push, \
patch.object(infra_vm.firecracker_vm, "boot", return_value=vm) as boot, \
patch.object(infra_vm, "_gw_dir", return_value=Path(td)):
infra_vm.boot_gateway("10.243.255.1", "gw.jwt")
kw = boot.call_args.kwargs
self.assertIn("bb_role=gateway", kw["extra_boot_args"])
self.assertIn("bb_orch=10.243.255.1", kw["extra_boot_args"]) # reaches the CP
self.assertEqual(infra_vm._GW_MEM_MIB, kw["mem_mib"])
self.assertIsNone(kw["data_drive"]) # data plane never opens the DB
# The host-minted token is threaded through to the JWT push.
self.assertEqual("gw.jwt", push.call_args.args[1])
class TestSshGatewayTransport(unittest.TestCase):
def test_cp_into_preserves_source_mode(self):
import tempfile
from subprocess import CompletedProcess
with tempfile.NamedTemporaryFile() as f:
os.chmod(f.name, 0o700) # like the staged access-hook
t = infra_vm.SshGatewayTransport(Path("/k"), "10.0.0.1")
with patch.object(infra_vm.subprocess, "run",
return_value=CompletedProcess([], 0)) as run:
t.cp_into(f.name, "/etc/git-gate/access-hook")
remote_cmd = run.call_args.args[0][-1]
self.assertIn("chmod 700", remote_cmd) # exec bit preserved over SSH
def test_exec_raises_on_failure(self):
from subprocess import CompletedProcess
t = infra_vm.SshGatewayTransport(Path("/k"), "10.0.0.1")
with patch.object(infra_vm.subprocess, "run",
return_value=CompletedProcess([], 1, stderr="nope")), \
self.assertRaises(infra_vm.GatewayProvisionError):
t.exec(["mkdir", "-p", "/git-gate"])
class TestGatewayTransportTargetsGatewayVm(unittest.TestCase):
def test_transport_uses_the_gateway_link_guest_ip(self):
# git-gate provisioning goes to the gateway VM (gw_slot), not the
# orchestrator, now that the planes are split.
t = infra_vm.gateway_transport()
self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, t._ip)
class TestGatewayCaPem(unittest.TestCase):
def test_dies_when_cert_never_appears(self) -> None:
from subprocess import CompletedProcess
gw = infra_vm.InfraVm(vm=None, guest_ip="10.0.0.1", private_key=Path("/k"))
with patch.object(infra_vm.subprocess, "run",
return_value=CompletedProcess([], 1, stdout="", stderr="")), \
self.assertRaises(SystemExit):
gw.gateway_ca_pem(timeout=0)
class TestRegistryVolume(unittest.TestCase):
def test_reuses_existing_volume(self):
@@ -295,10 +226,16 @@ class TestWaitForHealth(unittest.TestCase):
infra_vm.wait_for_health(infra, timeout=5)
# The gateway service is imported lazily inside ensure_running / _adopt (to
# break the infra_vm <-> gateway import cycle), so patch it at its home module.
_GW_CLS = "bot_bottle.backend.firecracker.gateway.FirecrackerGateway"
class TestEnsureRunningSingleton(unittest.TestCase):
def test_adopts_when_healthy_alive_and_version_matches(self):
# Healthy control plane + live gateway + existing key + matching version
# marker -> adopt (no boot); both VM handles have vm=None.
# marker -> adopt (no boot).
from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
import tempfile
with tempfile.TemporaryDirectory() as td:
d = Path(td)
@@ -308,16 +245,15 @@ class TestEnsureRunningSingleton(unittest.TestCase):
patch.object(infra_vm, "_expected_version", return_value="v-current"), \
patch.object(infra_vm, "_health_ok", return_value=True), \
patch.object(infra_vm, "_pidfile_alive", return_value=True), \
patch.object(infra_vm, "boot_orchestrator") as boot_o, \
patch.object(infra_vm, "boot_gateway") as boot_g:
patch.object(infra_vm, "boot_orchestrator") as boot_o:
ep = infra_vm.ensure_running()
boot_o.assert_not_called()
boot_g.assert_not_called()
self.assertIsNone(ep.orchestrator.vm)
self.assertIsNone(ep.gateway.vm)
# The two handles address the two distinct infra links.
# The orchestrator handle + the adopted gateway service address the two
# distinct infra links.
self.assertEqual(infra_vm.netpool.orch_slot().guest_ip, ep.orchestrator.guest_ip)
self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, ep.gateway.guest_ip)
self.assertIsInstance(ep.gateway, FirecrackerGateway)
self.assertEqual(infra_vm.netpool.gw_slot().guest_ip, ep.gateway.address())
def test_reboots_both_when_version_stale(self):
# Healthy control plane but the running pair booted an OLDER image
@@ -337,19 +273,16 @@ class TestEnsureRunningSingleton(unittest.TestCase):
patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \
patch.object(infra_vm, "mint", return_value="gw.jwt"), \
patch.object(infra_vm, "boot_orchestrator") as boot_o, \
patch.object(infra_vm, "boot_gateway") as boot_g:
patch(_GW_CLS) as gw_cls:
boot_o.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
boot_g.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.3", private_key=Path("/k"), vm=MagicMock())
infra_vm.ensure_running()
stop.assert_called_once() # dislodge the outdated pair
boot_o.assert_called_once()
boot_g.assert_called_once()
# The gateway is booted pointing at the orchestrator's guest IP,
# carrying the host-minted `gateway` token.
self.assertEqual("10.243.255.1", boot_g.call_args.args[0])
self.assertEqual("gw.jwt", boot_g.call_args.args[1])
# The gateway service is bound to the orchestrator's URL, carrying
# the host-minted `gateway` token.
gw_cls.return_value.connect_to_orchestrator.assert_called_once_with(
"http://10.243.255.1:8099", "gw.jwt")
# The fresh boot records the current version for the next launcher.
self.assertEqual("v-current\n", (d / "booted-version").read_text())
@@ -371,15 +304,13 @@ class TestEnsureRunningSingleton(unittest.TestCase):
patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \
patch.object(infra_vm, "mint", return_value="gw.jwt"), \
patch.object(infra_vm, "boot_orchestrator") as boot_o, \
patch.object(infra_vm, "boot_gateway") as boot_g:
patch(_GW_CLS) as gw_cls:
boot_o.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
boot_g.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.3", private_key=Path("/k"), vm=MagicMock())
infra_vm.ensure_running()
stop.assert_called_once()
boot_o.assert_called_once()
boot_g.assert_called_once()
gw_cls.return_value.connect_to_orchestrator.assert_called_once()
def test_boots_both_when_unhealthy(self):
import tempfile
@@ -392,20 +323,18 @@ class TestEnsureRunningSingleton(unittest.TestCase):
patch.object(infra_vm, "host_orchestrator_token", return_value="k"), \
patch.object(infra_vm, "mint", return_value="gw.jwt"), \
patch.object(infra_vm, "boot_orchestrator") as boot_o, \
patch.object(infra_vm, "boot_gateway") as boot_g, \
patch(_GW_CLS) as gw_cls, \
patch.object(infra_vm, "wait_for_health") as wait:
boot_o.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.1", private_key=Path("/k"), vm=MagicMock())
boot_g.return_value = infra_vm.InfraVm(
guest_ip="10.243.255.3", private_key=Path("/k"), vm=MagicMock())
infra_vm.ensure_running()
stop.assert_called_once() # clear stale VMs first
built.assert_called_once()
# Orchestrator boots + becomes healthy BEFORE the gateway (its daemons
# reach the control plane at startup).
# Orchestrator boots + becomes healthy BEFORE the gateway is connected
# (its daemons reach the control plane at startup).
boot_o.assert_called_once()
wait.assert_called_once()
boot_g.assert_called_once()
gw_cls.return_value.connect_to_orchestrator.assert_called_once()
class TestStop(unittest.TestCase):