refactor(gateway): Firecracker gateway under the Gateway service ABC
Add `FirecrackerGateway`, the Firecracker implementation of the shared `Gateway` service — completing the trio (docker, macOS, firecracker) behind one uniform contract. The gateway here is a whole microVM, so the adapter composes `infra_vm`'s VM primitives (boot, SSH secret push, netpool links) into the ABC surface: `connect_to_orchestrator(url, token)` parses the orchestrator guest IP off the URL and boots the gateway VM seeding the pre-minted token; `address()` is the gateway link's guest IP; `ca_cert_pem()` fetches over SSH (adapting the running VM when this process didn't boot it); `provisioning_transport()` is the SSH exec/cp transport. Trust-model alignment with the other backends: minting moves out of `_push_gateway_jwt` to the host composition point — `ensure_running` mints the role-scoped `gateway` JWT and threads it through `boot_gateway(orch_ip, token)`, so the push step only writes the token the gateway presents (#469). `infra_vm` keeps driving the orchestrator+gateway pair as a singleton and gains gateway-only helpers (`gateway_running`/`stop_gateway`/`gateway_guest_ip`/ `adopted_gateway_vm`); the consolidated launch reads the gateway's transport + CA off the service. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -45,6 +45,7 @@ from ..consolidated_util import (
|
||||
teardown_consolidated as _teardown_util,
|
||||
)
|
||||
from . import cleanup, infra_vm, util
|
||||
from .gateway import FirecrackerGateway
|
||||
|
||||
_ENV_VAR_SECRET_PATH = "/run/bot-bottle/env-var-secret"
|
||||
|
||||
@@ -129,18 +130,21 @@ def launch_consolidated(
|
||||
client = OrchestratorClient(url)
|
||||
_reprovision_running_bottles(client)
|
||||
|
||||
transport = infra_vm.gateway_transport()
|
||||
# Read the gateway's provisioning transport + CA off the Gateway service
|
||||
# (adapting the running gateway VM).
|
||||
gateway = FirecrackerGateway(infra.gateway)
|
||||
transport = gateway.provisioning_transport()
|
||||
reg = provision_bottle(
|
||||
client, guest_ip, egress_plan, git_gate_plan, transport,
|
||||
image_ref=image_ref, tokens=tokens,
|
||||
)
|
||||
# The shared gateway CA every agent on this host trusts for TLS
|
||||
# interception — fetched from the infra VM over SSH.
|
||||
# interception — fetched from the gateway VM over SSH.
|
||||
return LaunchContext(
|
||||
bottle_id=reg.bottle_id,
|
||||
identity_token=reg.identity_token,
|
||||
source_ip=guest_ip,
|
||||
gateway_ca_pem=infra.gateway_ca_pem(),
|
||||
gateway_ca_pem=gateway.ca_cert_pem(),
|
||||
orchestrator_url=url,
|
||||
env_var_secret=reg.env_var_secret,
|
||||
)
|
||||
@@ -153,7 +157,7 @@ def teardown_consolidated(
|
||||
VM. Both steps are idempotent so this is safe from a cleanup trap. Does
|
||||
NOT stop the infra VMs — they're persistent per-host singletons shared by
|
||||
every bottle."""
|
||||
_teardown_util(bottle_id, infra_vm.gateway_transport(),
|
||||
_teardown_util(bottle_id, FirecrackerGateway().provisioning_transport(),
|
||||
orchestrator_url=orchestrator_url, timeout=timeout)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
"""The Firecracker gateway data plane as a microVM (PRD 0070).
|
||||
|
||||
`FirecrackerGateway` is the Firecracker implementation of the backend-neutral
|
||||
`Gateway` service. The gateway here is a whole microVM — the slim data-plane
|
||||
unit (mitmproxy TLS bump + DLP + git-http) booted on its own NAT'd link; agent
|
||||
VMs' gateway-port traffic is DNAT'd to it and it never opens `bot-bottle.db`
|
||||
(#469), so it holds no signing key, only the pre-minted `gateway` token the
|
||||
host hands it via `connect_to_orchestrator`.
|
||||
|
||||
The VM lifecycle primitives (boot, SSH secret push, netpool links, nft, PID
|
||||
files) stay in `infra_vm`, which drives the orchestrator+gateway pair as a
|
||||
singleton; this adapter composes them into the uniform `Gateway` surface.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ...gateway import (
|
||||
DEFAULT_CA_TIMEOUT_SECONDS,
|
||||
Gateway,
|
||||
GatewayError,
|
||||
GatewayTransport,
|
||||
)
|
||||
from . import infra_vm
|
||||
|
||||
# The gateway microVM's name (the `bb_role=gateway` boot tag / run dir). Fixed
|
||||
# per host — one gateway VM, shared by every agent VM.
|
||||
GATEWAY_NAME = "bot-bottle-gateway"
|
||||
|
||||
|
||||
class FirecrackerGateway(Gateway):
|
||||
"""The consolidated gateway as a Firecracker microVM on the gateway link.
|
||||
|
||||
The rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's
|
||||
`ensure_built` no-op here); `connect_to_orchestrator` boots the VM resolving
|
||||
policy against the orchestrator and seeds the pre-minted `gateway` token."""
|
||||
|
||||
name = GATEWAY_NAME
|
||||
|
||||
def __init__(self, vm: infra_vm.InfraVm | None = None) -> None:
|
||||
# The live VM handle when this process booted it; None when adapting an
|
||||
# already-running gateway (CA fetch / provisioning go through the stable
|
||||
# key + fixed link, so no live handle is needed).
|
||||
self._vm = vm
|
||||
self._orchestrator_url = ""
|
||||
self._gateway_token = ""
|
||||
|
||||
def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
|
||||
"""Boot the gateway VM resolving policy against `orchestrator_url` and
|
||||
seed `gateway_token`. The orchestrator's guest IP is parsed from the URL
|
||||
and passed on the cmdline, so the baked init stays IP-independent. Boot
|
||||
the orchestrator first — the gateway daemons reach it at startup."""
|
||||
self._orchestrator_url = orchestrator_url
|
||||
self._gateway_token = gateway_token
|
||||
if not self._orchestrator_url:
|
||||
raise GatewayError(
|
||||
"gateway requires an orchestrator URL to run "
|
||||
"(resolver-only data plane; no single-tenant fallback)"
|
||||
)
|
||||
if not self._gateway_token:
|
||||
raise GatewayError(
|
||||
"gateway requires a pre-minted `gateway` token to run "
|
||||
"(the orchestrator mints it; the gateway never holds the key)"
|
||||
)
|
||||
orchestrator_guest_ip = urlparse(self._orchestrator_url).hostname or ""
|
||||
if not orchestrator_guest_ip:
|
||||
raise GatewayError(
|
||||
f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}"
|
||||
)
|
||||
self._vm = infra_vm.boot_gateway(orchestrator_guest_ip, self._gateway_token)
|
||||
|
||||
def is_running(self) -> bool:
|
||||
return infra_vm.gateway_running()
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop only the gateway VM (idempotent)."""
|
||||
infra_vm.stop_gateway()
|
||||
|
||||
def address(self) -> str:
|
||||
"""The gateway VM's guest IP — the agent-facing target agent VMs'
|
||||
gateway-port traffic is DNAT'd to."""
|
||||
return infra_vm.gateway_guest_ip()
|
||||
|
||||
def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
|
||||
"""The gateway's mitmproxy CA (PEM) agents install to trust its TLS
|
||||
interception. Fetched from the gateway VM over SSH; polls because
|
||||
mitmproxy writes it a beat after boot."""
|
||||
vm = self._vm or infra_vm.adopted_gateway_vm()
|
||||
return vm.gateway_ca_pem(timeout=timeout)
|
||||
|
||||
def provisioning_transport(self) -> GatewayTransport:
|
||||
"""The exec/cp transport git-gate provisioning stages per-bottle repos +
|
||||
deploy keys through (over SSH to the gateway VM)."""
|
||||
return infra_vm.gateway_transport()
|
||||
|
||||
|
||||
__all__ = ["FirecrackerGateway", "GATEWAY_NAME"]
|
||||
@@ -49,7 +49,7 @@ from ...orchestrator_auth import ROLE_GATEWAY, mint
|
||||
from ...paths import host_orchestrator_token
|
||||
from .. import util as backend_util
|
||||
from ..docker import util as docker_mod
|
||||
from ..docker.gateway_provision import GatewayProvisionError
|
||||
from ...gateway import GatewayProvisionError
|
||||
from . import firecracker_vm, infra_artifact, netpool, util
|
||||
|
||||
# The orchestrator VM's signing-key path on its persistent /dev/vdb volume
|
||||
@@ -228,7 +228,11 @@ def ensure_running() -> InfraEndpoint:
|
||||
ensure_built()
|
||||
orchestrator = boot_orchestrator()
|
||||
wait_for_health(orchestrator)
|
||||
gateway = boot_gateway(orchestrator.guest_ip)
|
||||
# The orchestrator holds the signing key; mint the role-scoped `gateway`
|
||||
# JWT on the host and hand it to the gateway VM, which never sees the key
|
||||
# (#469).
|
||||
gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token())
|
||||
gateway = boot_gateway(orchestrator.guest_ip, gateway_token)
|
||||
_record_booted_version(want)
|
||||
return InfraEndpoint(orchestrator=orchestrator, gateway=gateway)
|
||||
|
||||
@@ -290,12 +294,13 @@ def boot_orchestrator() -> InfraVm:
|
||||
return orch
|
||||
|
||||
|
||||
def boot_gateway(orchestrator_guest_ip: str) -> InfraVm:
|
||||
def boot_gateway(orchestrator_guest_ip: str, gateway_token: str) -> InfraVm:
|
||||
"""Boot the gateway (data-plane) VM (detached) on the gateway link, and
|
||||
seed its pre-minted `gateway` JWT. Its daemons resolve policy against the
|
||||
orchestrator at `orchestrator_guest_ip:8099` (passed on the cmdline, so the
|
||||
baked init stays IP-independent). Boot the orchestrator first — the gateway
|
||||
daemons reach it at startup. Prefer `ensure_running`."""
|
||||
seed the pre-minted `gateway` JWT `gateway_token`. Its daemons resolve
|
||||
policy against the orchestrator at `orchestrator_guest_ip:8099` (passed on
|
||||
the cmdline, so the baked init stays IP-independent). Boot the orchestrator
|
||||
first — the gateway daemons reach it at startup. The gateway never holds the
|
||||
signing key, only this token (#469). Prefer `ensure_running`."""
|
||||
slot = netpool.gw_slot()
|
||||
gateway = _boot_vm(
|
||||
name="bot-bottle-gateway", slot=slot, run_dir=_gw_dir(),
|
||||
@@ -304,7 +309,7 @@ def boot_gateway(orchestrator_guest_ip: str) -> InfraVm:
|
||||
)
|
||||
# Push the pre-minted `gateway` JWT (never the signing key — issue #469).
|
||||
# The init waits for it before starting the data plane.
|
||||
_push_gateway_jwt(gateway)
|
||||
_push_gateway_jwt(gateway, gateway_token)
|
||||
return gateway
|
||||
|
||||
|
||||
@@ -451,15 +456,14 @@ def _push_signing_key(infra: InfraVm) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _push_gateway_jwt(infra: InfraVm) -> None:
|
||||
def _push_gateway_jwt(infra: InfraVm, gateway_token: str) -> None:
|
||||
"""Push the pre-minted `gateway` JWT into the freshly booted gateway VM
|
||||
over SSH (atomic write). Minted on the HOST from the canonical signing key
|
||||
so the data plane never holds the key itself (issue #469); the gateway
|
||||
daemons present it to the orchestrator. Retries until SSH is up; dies if it
|
||||
never lands, since the VM then refuses to start the data plane."""
|
||||
_push_secret(
|
||||
infra, mint(ROLE_GATEWAY, host_orchestrator_token()),
|
||||
_GUEST_GATEWAY_JWT_PATH,
|
||||
infra, gateway_token, _GUEST_GATEWAY_JWT_PATH,
|
||||
"the gateway JWT to the gateway VM (its data plane will not start)",
|
||||
)
|
||||
|
||||
@@ -612,6 +616,35 @@ def gateway_transport() -> SshGatewayTransport:
|
||||
_infra_dir() / "id_ed25519", netpool.gw_slot().guest_ip)
|
||||
|
||||
|
||||
def gateway_guest_ip() -> str:
|
||||
"""The gateway VM's fixed guest IP on its link — the agent-facing address
|
||||
agent VMs' gateway-port traffic is DNAT'd to (PRD 0070)."""
|
||||
return netpool.gw_slot().guest_ip
|
||||
|
||||
|
||||
def gateway_running() -> bool:
|
||||
"""True iff the gateway VM is still a live VMM."""
|
||||
return _pidfile_alive(_gw_dir())
|
||||
|
||||
|
||||
def stop_gateway() -> None:
|
||||
"""Stop only the gateway VM (idempotent — absent is success). Drops the pair
|
||||
version marker so a half-stopped pair is never adopted (`_adoptable`)."""
|
||||
_kill_pidfile(_gw_dir())
|
||||
_pid_file(_gw_dir()).unlink(missing_ok=True)
|
||||
_version_file().unlink(missing_ok=True)
|
||||
|
||||
|
||||
def adopted_gateway_vm() -> InfraVm:
|
||||
"""A handle to the already-running gateway VM from the stable key + the
|
||||
fixed gateway link (vm=None — CA fetch / provisioning work from any later
|
||||
launcher, not only the process that booted it)."""
|
||||
return InfraVm(
|
||||
guest_ip=netpool.gw_slot().guest_ip,
|
||||
private_key=_infra_dir() / "id_ed25519",
|
||||
)
|
||||
|
||||
|
||||
def wait_for_health(
|
||||
infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS,
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user