refactor(firecracker): move gateway logic into the gateway service

Pull the gateway's host-side logic out of `infra_vm` and into
`FirecrackerGateway`'s own methods, so the gateway is self-contained and
`infra_vm` shrinks toward being just the pair coordinator (a step toward
removing it). Now living in the gateway service: the gateway VM boot +
`bb_orch` cmdline, the pre-minted JWT push, the mitmproxy CA fetch over SSH,
the `SshGatewayTransport` exec/cp, and the lifecycle predicates
(is_running/stop/address). `ensure_running` / `_adopt` construct the gateway
and call `connect_to_orchestrator` (lazy import to break the cycle); the
InfraEndpoint's gateway is now the `FirecrackerGateway` service.

What deliberately stays shared in `infra_vm` (documented at the top of
gateway.py): the plane-agnostic VM substrate the orchestrator VM also needs
(`_boot_vm`, the stable SSH keypair, the secret-push retry, PID lifecycle), the
pair coordinator (`ensure_running`: orchestrator-first health gate + singleton
lock + adoption/version marker), and the single shared `bb_role`-branched init
baked into the one published rootfs both VMs boot — the guest-side gateway
startup can't live in a host method. These are the seam that moves to a neutral
module when the Orchestrator service lands and `infra_vm` dissolves.

CA fetch now raises GatewayError (was die/SystemExit) to match the ABC.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

bot_bottle/backend/firecracker/consolidated_launch.py

@@ -130,9 +130,8 @@ def launch_consolidated(
     client = OrchestratorClient(url)
     _reprovision_running_bottles(client)
 
-    # Read the gateway's provisioning transport + CA off the Gateway service
-    # (adapting the running gateway VM).
-    gateway = FirecrackerGateway(infra.gateway)
+    # Read the gateway's provisioning transport + CA off the Gateway service.
+    gateway = infra.gateway
     transport = gateway.provisioning_transport()
     reg = provision_bottle(
         client, guest_ip, egress_plan, git_gate_plan, transport,

bot_bottle/backend/firecracker/gateway.py

@@ -1,38 +1,100 @@
 """The Firecracker gateway data plane as a microVM (PRD 0070).
 
 `FirecrackerGateway` is the Firecracker implementation of the backend-neutral
-`Gateway` service. The gateway here is a whole microVM — the slim data-plane
-unit (mitmproxy TLS bump + DLP + git-http) booted on its own NAT'd link; agent
-VMs' gateway-port traffic is DNAT'd to it and it never opens `bot-bottle.db`
-(#469), so it holds no signing key, only the pre-minted `gateway` token the
-host hands it via `connect_to_orchestrator`.
+`Gateway` service, and owns the gateway's host-side logic directly: booting the
+gateway microVM on its NAT'd link, seeding the pre-minted `gateway` token,
+fetching the mitmproxy CA, and the SSH exec/cp provisioning transport. The
+gateway VM never opens `bot-bottle.db` (#469), so it holds no signing key —
+only the token the host hands it via `connect_to_orchestrator`.
 
-The VM lifecycle primitives (boot, SSH secret push, netpool links, nft, PID
-files) stay in `infra_vm`, which drives the orchestrator+gateway pair as a
-singleton; this adapter composes them into the uniform `Gateway` surface.
+What deliberately stays in `infra_vm` (not moved here): the plane-agnostic VM
+substrate the orchestrator VM shares — booting a VM from the shared rootfs
+(`_boot_vm`), the stable SSH keypair, the secret-push retry loop, and the PID
+lifecycle — plus the pair coordinator (`ensure_running`: orchestrator-first
+health gate, singleton lock, adoption/version marker). The guest-side gateway
+daemon startup lives in the shared `bb_role=gateway` init branch baked into the
+one published rootfs both VMs boot, so it can't live in a host-side method
+either. These are the shared seam that moves to a neutral module when the
+Orchestrator service lands and `infra_vm` is dissolved.
 """
 
 from __future__ import annotations
 
+import os
+import shlex
+import stat
+import subprocess
+from pathlib import Path
 from urllib.parse import urlparse
 
 from ...gateway import (
     DEFAULT_CA_TIMEOUT_SECONDS,
     Gateway,
     GatewayError,
+    GatewayProvisionError,
     GatewayTransport,
 )
-from . import infra_vm
+from .. import util as backend_util
+from . import infra_vm, netpool, util
 
 # The gateway microVM's name (the `bb_role=gateway` boot tag / run dir). Fixed
 # per host — one gateway VM, shared by every agent VM.
 GATEWAY_NAME = "bot-bottle-gateway"
 
+# The gateway VM's slim memory ceiling (buildah is present on the shared rootfs
+# but unused on the data plane) — PRD 0070 "Memory: fixed ceilings".
+_GW_MEM_MIB = 2048
+
+# The pre-minted `gateway` JWT path in the guest. The *shared* init (both planes
+# boot one rootfs) waits for it before starting the data plane, so its canonical
+# definition lives with that init in `infra_vm`; imported here for the push so
+# the load-bearing path isn't duplicated.
+_GUEST_GATEWAY_JWT_PATH = infra_vm._GUEST_GATEWAY_JWT_PATH
+# mitmproxy writes its CA here a beat after start; agents install it to trust
+# the gateway's TLS interception. Host-side only (SSH cat), so it lives here.
+_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem"
+
+_CA_FETCH_TIMEOUT_SECONDS = 15.0
+
+
+class SshGatewayTransport:
+    """`GatewayTransport` for the gateway running in the gateway VM — the docker
+    exec/cp equivalents over SSH (dropbear + the stable infra key)."""
+
+    def __init__(self, private_key: Path, guest_ip: str) -> None:
+        self._key = private_key
+        self._ip = guest_ip
+
+    def exec(self, argv: list[str]) -> None:
+        proc = subprocess.run(
+            util.ssh_base_argv(self._key, self._ip) + [shlex.join(argv)],
+            capture_output=True, text=True, timeout=60, check=False,
+        )
+        if proc.returncode != 0:
+            raise GatewayProvisionError(
+                f"infra gateway exec {argv!r} failed: {proc.stderr.strip()}")
+
+    def cp_into(self, src: str, dest: str) -> None:
+        # Preserve the source mode (docker cp does): the access-hook is staged
+        # 0700 and git-http execs it directly — a plain `cat >` would land it
+        # 0644 and the exec fails with EACCES; keys stay 0600.
+        mode = stat.S_IMODE(os.stat(src).st_mode)
+        q = shlex.quote(dest)
+        proc = subprocess.run(
+            util.ssh_base_argv(self._key, self._ip)
+            + [f"cat > {q} && chmod {mode:o} {q}"],
+            input=Path(src).read_bytes(), capture_output=True, timeout=30, check=False,
+        )
+        if proc.returncode != 0:
+            raise GatewayProvisionError(
+                f"infra gateway cp {src} -> {dest} failed: "
+                f"{proc.stderr.decode(errors='replace').strip()}")
+
 
 class FirecrackerGateway(Gateway):
     """The consolidated gateway as a Firecracker microVM on the gateway link.
 
-    The rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's
+    The shared rootfs is built/downloaded by `infra_vm.ensure_built` (the ABC's
     `ensure_built` no-op here); `connect_to_orchestrator` boots the VM resolving
     policy against the orchestrator and seeds the pre-minted `gateway` token."""
 
@@ -47,10 +109,12 @@ class FirecrackerGateway(Gateway):
         self._gateway_token = ""
 
     def connect_to_orchestrator(self, orchestrator_url: str, gateway_token: str) -> None:
-        """Boot the gateway VM resolving policy against `orchestrator_url` and
-        seed `gateway_token`. The orchestrator's guest IP is parsed from the URL
-        and passed on the cmdline, so the baked init stays IP-independent. Boot
-        the orchestrator first — the gateway daemons reach it at startup."""
+        """Boot the gateway VM on its link resolving policy against
+        `orchestrator_url`, then seed `gateway_token` over SSH. The orchestrator's
+        guest IP is parsed from the URL and passed on the cmdline (`bb_orch=`), so
+        the baked init stays IP-independent. Boot the orchestrator first — the
+        gateway daemons reach it at startup. The gateway never mints, so it holds
+        no signing key, only this token (#469)."""
         self._orchestrator_url = orchestrator_url
         self._gateway_token = gateway_token
         if not self._orchestrator_url:
@@ -68,31 +132,64 @@ class FirecrackerGateway(Gateway):
             raise GatewayError(
                 f"cannot resolve orchestrator guest IP from {self._orchestrator_url!r}"
             )
-        self._vm = infra_vm.boot_gateway(orchestrator_guest_ip, self._gateway_token)
+        # Boot on the gateway link from the shared rootfs (bb_role=gateway), then
+        # push the token the init waits for before starting the data plane.
+        vm = infra_vm._boot_vm(
+            name=GATEWAY_NAME, slot=netpool.gw_slot(), run_dir=infra_vm._gw_dir(),
+            role="gateway", mem_mib=_GW_MEM_MIB,
+            extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
+        )
+        infra_vm._push_secret(
+            vm, self._gateway_token, _GUEST_GATEWAY_JWT_PATH,
+            "the gateway JWT to the gateway VM (its data plane will not start)",
+        )
+        self._vm = vm
 
     def is_running(self) -> bool:
-        return infra_vm.gateway_running()
+        return infra_vm._pidfile_alive(infra_vm._gw_dir())
 
     def stop(self) -> None:
-        """Stop only the gateway VM (idempotent)."""
-        infra_vm.stop_gateway()
+        """Stop only the gateway VM (idempotent — absent is success). A dead
+        gateway already fails `infra_vm._adoptable`, so no version marker to
+        clear here."""
+        infra_vm._kill_pidfile(infra_vm._gw_dir())
+        infra_vm._pid_file(infra_vm._gw_dir()).unlink(missing_ok=True)
 
     def address(self) -> str:
         """The gateway VM's guest IP — the agent-facing target agent VMs'
         gateway-port traffic is DNAT'd to."""
-        return infra_vm.gateway_guest_ip()
+        return netpool.gw_slot().guest_ip
 
     def ca_cert_pem(self, *, timeout: float = DEFAULT_CA_TIMEOUT_SECONDS) -> str:
         """The gateway's mitmproxy CA (PEM) agents install to trust its TLS
         interception. Fetched from the gateway VM over SSH; polls because
-        mitmproxy writes it a beat after boot."""
-        vm = self._vm or infra_vm.adopted_gateway_vm()
-        return vm.gateway_ca_pem(timeout=timeout)
+        mitmproxy writes it a beat after boot. Works from any later launcher —
+        falls back to the stable key + fixed link when this process didn't boot
+        the VM."""
+        key = self._vm.private_key if self._vm else infra_vm._infra_dir() / "id_ed25519"
+        ip = self._vm.guest_ip if self._vm else netpool.gw_slot().guest_ip
+
+        def _fetch() -> str | None:
+            proc = subprocess.run(
+                util.ssh_base_argv(key, ip) + [f"cat {_GATEWAY_CA_PATH}"],
+                capture_output=True, text=True,
+                timeout=_CA_FETCH_TIMEOUT_SECONDS, check=False,
+            )
+            ok = proc.returncode == 0 and "BEGIN CERTIFICATE" in proc.stdout
+            return proc.stdout if ok else None
+
+        try:
+            return backend_util.poll_ca_cert(_fetch, timeout=timeout)
+        except TimeoutError as exc:
+            raise GatewayError(str(exc)) from exc
 
     def provisioning_transport(self) -> GatewayTransport:
         """The exec/cp transport git-gate provisioning stages per-bottle repos +
-        deploy keys through (over SSH to the gateway VM)."""
-        return infra_vm.gateway_transport()
+        deploy keys through (over SSH to the gateway VM). Needs no live handle —
+        built from the stable key + the gateway link's guest IP, so teardown can
+        use it too."""
+        return SshGatewayTransport(
+            infra_vm._infra_dir() / "id_ed25519", netpool.gw_slot().guest_ip)
 
 
-__all__ = ["FirecrackerGateway", "GATEWAY_NAME"]
+__all__ = ["FirecrackerGateway", "SshGatewayTransport", "GATEWAY_NAME"]

bot_bottle/backend/firecracker/infra_vm.py

@@ -32,9 +32,7 @@ from __future__ import annotations
 import fcntl
 import hashlib
 import os
-import shlex
 import signal
-import stat
 import subprocess
 import time
 import urllib.error
@@ -42,16 +40,17 @@ import urllib.request
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Generator
+from typing import TYPE_CHECKING, Generator
 
 from ...log import die, info
 from ...orchestrator_auth import ROLE_GATEWAY, mint
 from ...paths import host_orchestrator_token
-from .. import util as backend_util
 from ..docker import util as docker_mod
-from ...gateway import GatewayProvisionError
 from . import firecracker_vm, infra_artifact, netpool, util
 
+if TYPE_CHECKING:
+    from .gateway import FirecrackerGateway
+
 # The orchestrator VM's signing-key path on its persistent /dev/vdb volume
 # (mounted at BOT_BOTTLE_ROOT=/var/lib/bot-bottle). The launcher seeds this
 # file with the host-canonical key AFTER boot (over SSH), so the VM verifies
@@ -77,16 +76,11 @@ ORCHESTRATOR_PORT = 8099
 EGRESS_PORT = 9099
 SUPERVISE_PORT = 9100
 GIT_HTTP_PORT = 9420
-# mitmproxy writes its CA here a beat after start; agents install it to trust
-# the gateway's TLS interception.
-_GATEWAY_CA_PATH = "/home/mitmproxy/.mitmproxy/mitmproxy-ca-cert.pem"
-
-# Memory ceilings (fixed at boot, demand-paged — no balloon reclaim). The
+# Memory ceiling (fixed at boot, demand-paged — no balloon reclaim). The
 # orchestrator keeps the build headroom (buildah's 2-4 GB working set during
-# in-VM agent builds); the gateway is the slim unit (mitmproxy TLS bump + DLP
-# body buffering, ~1 GB) — PRD 0070 "Memory: fixed ceilings".
+# in-VM agent builds); the gateway's slimmer ceiling is set by the gateway
+# service — PRD 0070 "Memory: fixed ceilings".
 _ORCH_MEM_MIB = 4096
-_GW_MEM_MIB = 2048
 
 # The infra VMs make direct upstream connections (gateway egress, and buildah
 # during builds), and the kernel `ip=` cmdline sets no resolver. Public for
@@ -117,41 +111,23 @@ class InfraVm:
     def orchestrator_url(self) -> str:
         return f"http://{self.guest_ip}:{ORCHESTRATOR_PORT}"
 
-    def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str:
-        """The gateway's mitmproxy CA (PEM) that agents install to trust its
-        TLS interception. Generated a moment after boot, so this polls over
-        SSH until it appears (mirrors DockerGateway.ca_cert_pem). Call on the
-        gateway VM handle."""
-        def _fetch() -> str | None:
-            proc = subprocess.run(
-                util.ssh_base_argv(self.private_key, self.guest_ip)
-                + [f"cat {_GATEWAY_CA_PATH}"],
-                capture_output=True, text=True, timeout=15, check=False,
-            )
-            ok = proc.returncode == 0 and "BEGIN CERTIFICATE" in proc.stdout
-            return proc.stdout if ok else None
-        try:
-            return backend_util.poll_ca_cert(_fetch, timeout=timeout)
-        except TimeoutError as exc:
-            die(str(exc))
-
 
 @dataclass
 class InfraEndpoint:
     """How to reach the running per-host pair. `orchestrator` is the control
-    plane (host CLI + registration + in-VM builds); `gateway` is the data
-    plane (agent egress / git-http / supervise, CA fetch, git-gate
+    plane (host CLI + registration + in-VM builds); `gateway` is the data-plane
+    `Gateway` service (agent egress / git-http / supervise, CA fetch, git-gate
     provisioning). Mirrors the docker/macOS `InfraEndpoint` shape."""
 
     orchestrator: InfraVm
-    gateway: InfraVm
+    gateway: "FirecrackerGateway"
 
     @property
     def orchestrator_url(self) -> str:
         return self.orchestrator.orchestrator_url
 
     def gateway_ca_pem(self, *, timeout: float = _CA_TIMEOUT_SECONDS) -> str:
-        return self.gateway.gateway_ca_pem(timeout=timeout)
+        return self.gateway.ca_cert_pem(timeout=timeout)
 
 
 def ensure_built() -> None:
@@ -229,22 +205,28 @@ def ensure_running() -> InfraEndpoint:
         orchestrator = boot_orchestrator()
         wait_for_health(orchestrator)
         # The orchestrator holds the signing key; mint the role-scoped `gateway`
-        # JWT on the host and hand it to the gateway VM, which never sees the key
-        # (#469).
+        # JWT on the host and hand it to the gateway service, which never sees
+        # the key (#469). The gateway owns its own boot — see FirecrackerGateway.
+        # Local import: the gateway service imports this module for the shared VM
+        # substrate, so importing it at module scope would cycle.
+        from .gateway import FirecrackerGateway
         gateway_token = mint(ROLE_GATEWAY, host_orchestrator_token())
-        gateway = boot_gateway(orchestrator.guest_ip, gateway_token)
+        gateway = FirecrackerGateway()
+        gateway.connect_to_orchestrator(orchestrator.orchestrator_url, gateway_token)
         _record_booted_version(want)
         return InfraEndpoint(orchestrator=orchestrator, gateway=gateway)
 
 
 def _adopt(key: Path) -> InfraEndpoint:
     """Build handles to the already-running pair from the stable key + the
-    fixed links (vm=None — teardown then goes through the PID files)."""
+    fixed links (no live VM handle — teardown / CA fetch go through the PID
+    files + stable key)."""
+    from .gateway import FirecrackerGateway
     return InfraEndpoint(
         orchestrator=InfraVm(
             guest_ip=netpool.orch_slot().guest_ip, private_key=key),
-        gateway=InfraVm(
-            guest_ip=netpool.gw_slot().guest_ip, private_key=key),
+        gateway=FirecrackerGateway(
+            InfraVm(guest_ip=netpool.gw_slot().guest_ip, private_key=key)),
     )
 
 
@@ -294,25 +276,6 @@ def boot_orchestrator() -> InfraVm:
     return orch
 
 
-def boot_gateway(orchestrator_guest_ip: str, gateway_token: str) -> InfraVm:
-    """Boot the gateway (data-plane) VM (detached) on the gateway link, and
-    seed the pre-minted `gateway` JWT `gateway_token`. Its daemons resolve
-    policy against the orchestrator at `orchestrator_guest_ip:8099` (passed on
-    the cmdline, so the baked init stays IP-independent). Boot the orchestrator
-    first — the gateway daemons reach it at startup. The gateway never holds the
-    signing key, only this token (#469). Prefer `ensure_running`."""
-    slot = netpool.gw_slot()
-    gateway = _boot_vm(
-        name="bot-bottle-gateway", slot=slot, run_dir=_gw_dir(),
-        role="gateway", mem_mib=_GW_MEM_MIB,
-        extra_boot_args=f"bb_orch={orchestrator_guest_ip}",
-    )
-    # Push the pre-minted `gateway` JWT (never the signing key — issue #469).
-    # The init waits for it before starting the data plane.
-    _push_gateway_jwt(gateway, gateway_token)
-    return gateway
-
-
 def _boot_vm(
     *,
     name: str,
@@ -456,18 +419,6 @@ def _push_signing_key(infra: InfraVm) -> None:
     )
 
 
-def _push_gateway_jwt(infra: InfraVm, gateway_token: str) -> None:
-    """Push the pre-minted `gateway` JWT into the freshly booted gateway VM
-    over SSH (atomic write). Minted on the HOST from the canonical signing key
-    so the data plane never holds the key itself (issue #469); the gateway
-    daemons present it to the orchestrator. Retries until SSH is up; dies if it
-    never lands, since the VM then refuses to start the data plane."""
-    _push_secret(
-        infra, gateway_token, _GUEST_GATEWAY_JWT_PATH,
-        "the gateway JWT to the gateway VM (its data plane will not start)",
-    )
-
-
 def _push_secret(infra: InfraVm, secret: str, dest: str, what: str) -> None:
     """Pipe `secret` to an atomic write of `dest` in the guest over SSH,
     retrying while the guest's SSH comes up; die (naming `what`) if it never
@@ -574,77 +525,6 @@ def _health_ok(url: str) -> bool:
         return False
 
 
-class SshGatewayTransport:
-    """`GatewayTransport` for the gateway running in the gateway VM — the
-    docker exec/cp equivalents over SSH (dropbear + the stable infra key)."""
-
-    def __init__(self, private_key: Path, guest_ip: str) -> None:
-        self._key = private_key
-        self._ip = guest_ip
-
-    def exec(self, argv: list[str]) -> None:
-        proc = subprocess.run(
-            util.ssh_base_argv(self._key, self._ip) + [shlex.join(argv)],
-            capture_output=True, text=True, timeout=60, check=False,
-        )
-        if proc.returncode != 0:
-            raise GatewayProvisionError(
-                f"infra gateway exec {argv!r} failed: {proc.stderr.strip()}")
-
-    def cp_into(self, src: str, dest: str) -> None:
-        # Preserve the source mode (docker cp does): the access-hook is staged
-        # 0700 and git-http execs it directly — a plain `cat >` would land it
-        # 0644 and the exec fails with EACCES; keys stay 0600.
-        mode = stat.S_IMODE(os.stat(src).st_mode)
-        q = shlex.quote(dest)
-        proc = subprocess.run(
-            util.ssh_base_argv(self._key, self._ip)
-            + [f"cat > {q} && chmod {mode:o} {q}"],
-            input=Path(src).read_bytes(), capture_output=True, timeout=30, check=False,
-        )
-        if proc.returncode != 0:
-            raise GatewayProvisionError(
-                f"infra gateway cp {src} -> {dest} failed: "
-                f"{proc.stderr.decode(errors='replace').strip()}")
-
-
-def gateway_transport() -> SshGatewayTransport:
-    """git-gate provisioning transport for the gateway VM, built from the
-    stable key + the gateway link's guest IP. Needs no live VM handle, so
-    teardown can use it too."""
-    return SshGatewayTransport(
-        _infra_dir() / "id_ed25519", netpool.gw_slot().guest_ip)
-
-
-def gateway_guest_ip() -> str:
-    """The gateway VM's fixed guest IP on its link — the agent-facing address
-    agent VMs' gateway-port traffic is DNAT'd to (PRD 0070)."""
-    return netpool.gw_slot().guest_ip
-
-
-def gateway_running() -> bool:
-    """True iff the gateway VM is still a live VMM."""
-    return _pidfile_alive(_gw_dir())
-
-
-def stop_gateway() -> None:
-    """Stop only the gateway VM (idempotent — absent is success). Drops the pair
-    version marker so a half-stopped pair is never adopted (`_adoptable`)."""
-    _kill_pidfile(_gw_dir())
-    _pid_file(_gw_dir()).unlink(missing_ok=True)
-    _version_file().unlink(missing_ok=True)
-
-
-def adopted_gateway_vm() -> InfraVm:
-    """A handle to the already-running gateway VM from the stable key + the
-    fixed gateway link (vm=None — CA fetch / provisioning work from any later
-    launcher, not only the process that booted it)."""
-    return InfraVm(
-        guest_ip=netpool.gw_slot().guest_ip,
-        private_key=_infra_dir() / "id_ed25519",
-    )
-
-
 def wait_for_health(
     infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS,
 ) -> None: