diff --git a/bot_bottle/backend/firecracker/consolidated_launch.py b/bot_bottle/backend/firecracker/consolidated_launch.py index 0609fac..899676b 100644 --- a/bot_bottle/backend/firecracker/consolidated_launch.py +++ b/bot_bottle/backend/firecracker/consolidated_launch.py @@ -1,27 +1,23 @@ -"""Consolidated bottle launch sequence for the Firecracker backend (PRD 0070). +"""Consolidated bottle launch sequence for the Firecracker backend +(PRD 0070, Stage B). -Mirrors bot_bottle.backend.docker.consolidated_launch but wired for -Firecracker's TAP-based network topology instead of a shared Docker bridge. +The shared gateway + orchestrator control plane run in a single persistent +per-host **infra VM** (`infra_vm.py`), not Docker containers. Agent VMs reach +the gateway's egress / supervise / git-http ports at the infra VM via a +PREROUTING DNAT on their own host-side TAP IP (see +`scripts/firecracker-netpool.sh`), and the host CLI reaches the control plane +over HTTP at the infra VM's guest IP. -The per-bottle sidecar bundle (one `docker run` per bottle, published on the -slot's host-side TAP IP) is replaced by a single persistent gateway that -every Firecracker VM shares. The gateway runs as a Docker container in the -dev-harness (a Firecracker VM is stage B per PRD 0070), with its ports -published on the host (`0.0.0.0:PORT`). VMs reach it at their slot's -host-side TAP IP because Docker's iptables PREROUTING DNAT redirects -port 9099/9100/9420 traffic to the gateway container — a path the nft -isolation table already allows via `ct status dnat accept` in the forward -chain. - -Attribution is by the VM's guest IP, which is unspoofable by construction: -the /31 point-to-point TAP topology + the `bot_bottle_fc` nft table ensure -that only the expected VM can source-IP that address. +Attribution is by the agent VM's guest IP, unspoofable by construction: the +/31 point-to-point TAP + the `bot_bottle_fc` nft table ensure only the +expected VM can source-IP that address. Sequence: - 1. ensure the Firecracker-flavoured orchestrator + gateway are up; + 1. ensure the infra VM (control plane + gateway) is up (a singleton — a + prior launcher may already have booted it); 2. register the bottle by its guest IP (attribution key) → bottle id + identity token; - 3. provision its git-gate repos/creds into the running gateway; + 3. provision its git-gate repos/creds into the gateway VM (over SSH); 4. fetch the shared gateway CA for the provisioner to install in the rootfs. The TAP slot allocation, rootfs build, and VM boot are the caller's job. @@ -34,34 +30,12 @@ from dataclasses import dataclass from ...egress import EgressPlan from ...git_gate import GitGatePlan from ...orchestrator.client import OrchestratorClient -from ...orchestrator.gateway import ( - GATEWAY_NETWORK, - DockerGateway, -) from ...orchestrator.lifecycle import ( - OrchestratorService, OrchestratorStartError, # re-exported so callers can catch it ) from ...orchestrator.registration import registration_inputs -from ..docker.egress import EGRESS_PORT -from ..docker.gateway_provision import ( - DockerGatewayTransport, - deprovision_git_gate, - provision_git_gate, -) -from ...supervise import SUPERVISE_PORT - -_GIT_HTTP_PORT = 9420 - -# Separate names from the Docker gateway so both backends can coexist on one -# host (and for clarity in `docker ps` output). -_FC_GATEWAY_NAME = "bot-bottle-fc-gateway" -_FC_ORCHESTRATOR_NAME = "bot-bottle-fc-orchestrator" -_FC_ORCHESTRATOR_LABEL = "bot-bottle-fc-orchestrator=1" -# Ports the gateway publishes on the host so Firecracker VMs can reach it -# via their TAP link. Docker's PREROUTING DNAT + nft's `ct status dnat -# accept` in the forward chain route the traffic. -_FC_GATEWAY_HOST_PORTS = (EGRESS_PORT, SUPERVISE_PORT, _GIT_HTTP_PORT) +from ..docker.gateway_provision import deprovision_git_gate, provision_git_gate +from . import infra_vm class ConsolidatedLaunchError(RuntimeError): @@ -79,33 +53,6 @@ class LaunchContext: orchestrator_url: str -class _FirecrackerOrchestratorService(OrchestratorService): - """Dev-harness orchestrator for the Firecracker backend. - - Uses a gateway that publishes its ports on the host so Firecracker VMs can - reach it via their TAP link. The gateway and orchestrator containers use - `*-fc-*` names so both backends can run independently on the same host. - """ - - def __init__(self, **kwargs: object) -> None: - super().__init__( - orchestrator_name=_FC_ORCHESTRATOR_NAME, - orchestrator_label=_FC_ORCHESTRATOR_LABEL, - **kwargs, # type: ignore[arg-type] - ) - - def _gateway(self) -> DockerGateway: - # The heavy data-plane image (#384 split it from the lean control-plane - # `image` this service's orchestrator container runs). - return DockerGateway( - self._gateway_image, - name=_FC_GATEWAY_NAME, - network=self.network, - orchestrator_url=self.internal_url, - host_port_bindings=_FC_GATEWAY_HOST_PORTS, - ) - - def launch_consolidated( egress_plan: EgressPlan, git_gate_plan: GitGatePlan, @@ -113,15 +60,12 @@ def launch_consolidated( guest_ip: str, image_ref: str = "", tokens: dict[str, str] | None = None, - service: OrchestratorService | None = None, - gateway_name: str = _FC_GATEWAY_NAME, ) -> LaunchContext: - """Ensure the orchestrator + Firecracker gateway are up, register the - bottle by its guest IP, and provision its git-gate state. Returns the - context the VM launch needs. Raises `ConsolidatedLaunchError` (or the - primitives' own errors) on failure — the caller tears down on failure.""" - service = service or _FirecrackerOrchestratorService() - url = service.ensure_running() + """Ensure the infra VM is up, register the bottle by its guest IP, and + provision its git-gate state into the gateway VM. Returns the context the + agent-VM launch needs. Raises on failure — the caller tears down.""" + infra = infra_vm.ensure_running() + url = infra.control_plane_url client = OrchestratorClient(url) inputs = registration_inputs(egress_plan) @@ -131,33 +75,29 @@ def launch_consolidated( ) try: provision_git_gate( - DockerGatewayTransport(gateway_name), reg.bottle_id, git_gate_plan) + infra_vm.gateway_transport(), reg.bottle_id, git_gate_plan) except Exception: client.teardown_bottle(reg.bottle_id) raise - # Fetch the shared gateway CA here so the caller can install it in the - # rootfs (the same CA every agent on this host trusts for TLS interception). - gateway_ca_pem = DockerGateway( - name=gateway_name, network=GATEWAY_NETWORK, - ).ca_cert_pem() - + # The shared gateway CA every agent on this host trusts for TLS + # interception — fetched from the infra VM over SSH. return LaunchContext( bottle_id=reg.bottle_id, identity_token=reg.identity_token, source_ip=guest_ip, - gateway_ca_pem=gateway_ca_pem, + gateway_ca_pem=infra.gateway_ca_pem(), orchestrator_url=url, ) -def teardown_consolidated( - bottle_id: str, *, orchestrator_url: str, gateway_name: str = _FC_GATEWAY_NAME, -) -> None: - """Deregister the bottle and remove its git-gate state from the gateway. - Both steps are idempotent so this is safe from a cleanup trap.""" +def teardown_consolidated(bottle_id: str, *, orchestrator_url: str) -> None: + """Deregister the bottle and remove its git-gate state from the gateway + VM. Both steps are idempotent so this is safe from a cleanup trap. Does + NOT stop the infra VM — it's a persistent per-host singleton shared by + every bottle.""" OrchestratorClient(orchestrator_url).teardown_bottle(bottle_id) - deprovision_git_gate(DockerGatewayTransport(gateway_name), bottle_id) + deprovision_git_gate(infra_vm.gateway_transport(), bottle_id) __all__ = [ diff --git a/bot_bottle/backend/firecracker/infra_vm.py b/bot_bottle/backend/firecracker/infra_vm.py index c3df64a..b6c0c50 100644 --- a/bot_bottle/backend/firecracker/infra_vm.py +++ b/bot_bottle/backend/firecracker/infra_vm.py @@ -18,6 +18,7 @@ surface. from __future__ import annotations import os +import shlex import signal import subprocess import time @@ -28,6 +29,7 @@ from pathlib import Path from ...log import die, info from ..docker import util as docker_mod +from ..docker.gateway_provision import GatewayProvisionError from . import firecracker_vm, netpool, util # The single infra-VM image: gateway data plane + baked control-plane source @@ -228,6 +230,42 @@ def _health_ok(url: str) -> bool: return False +class SshGatewayTransport: + """`GatewayTransport` for the gateway running in the infra VM — the docker + exec/cp equivalents over SSH (dropbear + the stable infra key).""" + + def __init__(self, private_key: Path, guest_ip: str) -> None: + self._key = private_key + self._ip = guest_ip + + def exec(self, argv: list[str]) -> None: + proc = subprocess.run( + util.ssh_base_argv(self._key, self._ip) + [shlex.join(argv)], + capture_output=True, text=True, timeout=60, check=False, + ) + if proc.returncode != 0: + raise GatewayProvisionError( + f"infra gateway exec {argv!r} failed: {proc.stderr.strip()}") + + def cp_into(self, src: str, dest: str) -> None: + proc = subprocess.run( + util.ssh_base_argv(self._key, self._ip) + [f"cat > {shlex.quote(dest)}"], + input=Path(src).read_bytes(), capture_output=True, timeout=30, check=False, + ) + if proc.returncode != 0: + raise GatewayProvisionError( + f"infra gateway cp {src} -> {dest} failed: " + f"{proc.stderr.decode(errors='replace').strip()}") + + +def gateway_transport() -> SshGatewayTransport: + """git-gate provisioning transport for the gateway in the infra VM, built + from the stable key + the orchestrator link's guest IP. Needs no live VM + handle, so teardown can use it too.""" + return SshGatewayTransport( + _infra_dir() / "id_ed25519", netpool.orch_slot().guest_ip) + + def wait_for_health( infra: InfraVm, *, timeout: float = _HEALTH_TIMEOUT_SECONDS, ) -> None: