didericis-claude
d066e4032b
prd-number-check / require-numbered-prds (pull_request) Successful in 10s
tracker-policy-pr / check-pr (pull_request) Successful in 16s
lint / lint (push) Failing after 1m1s
test / unit (pull_request) Successful in 56s
test / image-input-builds (pull_request) Successful in 1m0s
test / integration-docker (pull_request) Failing after 3m7s
test / coverage (pull_request) Has been skipped
On a gateway cold boot, reconcile every live agent VM against the fresh gateway: push the new CA into each agent's trust store, re-provision git-gate repos/creds from the persisted upstreams snapshot, and restore egress tokens. Per-bottle failures are logged and skipped rather than aborting the whole reconcile. New: reconcile.py — attach_bottled_agents_to_gateway, _push_ca, _reprovision_git_gate, _guest_ip_from_config. New: git_gate/provision.py writes upstreams.json after key provisioning so the bring-up reconcile can reconstruct the upstream table without the manifest. Wired into FirecrackerInfraService.ensure_running() cold-boot path; base.py BottleBackend gets a no-op default. Old _reprovision_running_bottles removed from consolidated_launch.py (superseded by reconcile.py). Tests migrated and extended.
158 lines
8.5 KiB
Python
158 lines
8.5 KiB
Python
"""Unit: FirecrackerInfraService composes the orchestrator + gateway microVM pair.
|
|
|
|
The two-VM singleton lifecycle (adopt-or-boot-both under a flock, the version
|
|
marker) that must hold without a live VM. The VM substrate + boot primitives are
|
|
tested in test_firecracker_infra_vm; the orchestrator/gateway services in their
|
|
own modules. These isolate the *composition*."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from bot_bottle.backend.firecracker import infra_vm
|
|
from bot_bottle.backend.firecracker.gateway import FirecrackerGateway
|
|
from bot_bottle.backend.firecracker.infra import FirecrackerInfraService
|
|
from bot_bottle.backend.firecracker.orchestrator import FirecrackerOrchestrator
|
|
|
|
# The service imports the classes into its own module namespace, so patch them
|
|
# there (not at their home modules).
|
|
_ORCH_CLS = "bot_bottle.backend.firecracker.infra.FirecrackerOrchestrator"
|
|
_GW_CLS = "bot_bottle.backend.firecracker.infra.FirecrackerGateway"
|
|
_RECONCILE = "bot_bottle.backend.firecracker.infra.attach_bottled_agents_to_gateway"
|
|
|
|
|
|
class TestAccessors(unittest.TestCase):
|
|
def test_orchestrator_and_gateway_services(self) -> None:
|
|
svc = FirecrackerInfraService()
|
|
self.assertIsInstance(svc.orchestrator(), FirecrackerOrchestrator)
|
|
self.assertIsInstance(svc.gateway(), FirecrackerGateway)
|
|
|
|
def test_stop_stops_both_vms(self) -> None:
|
|
with patch.object(infra_vm, "stop") as stop:
|
|
FirecrackerInfraService().stop()
|
|
stop.assert_called_once()
|
|
|
|
def test_url_and_is_healthy_delegate_to_the_orchestrator(self) -> None:
|
|
svc = FirecrackerInfraService()
|
|
ip = infra_vm.netpool.orch_slot().guest_ip
|
|
self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", svc.url())
|
|
|
|
|
|
class TestEnsureRunning(unittest.TestCase):
|
|
def test_adopts_when_healthy_alive_and_version_matches(self):
|
|
# Healthy control plane + live gateway + existing key + matching version
|
|
# marker -> adopt (no stop/build/boot); returns the orchestrator URL.
|
|
with tempfile.TemporaryDirectory() as td:
|
|
d = Path(td)
|
|
(d / "id_ed25519").write_text("k")
|
|
(d / "booted-version").write_text("v-current\n")
|
|
with patch.object(infra_vm, "_infra_dir", return_value=d), \
|
|
patch.object(infra_vm, "expected_version", return_value="v-current"), \
|
|
patch.object(infra_vm, "_health_ok", return_value=True), \
|
|
patch.object(infra_vm, "_pidfile_alive", return_value=True), \
|
|
patch.object(infra_vm, "stop") as stop, \
|
|
patch.object(infra_vm, "ensure_built") as built, \
|
|
patch(_RECONCILE) as reconcile:
|
|
url = FirecrackerInfraService().ensure_running()
|
|
stop.assert_not_called()
|
|
built.assert_not_called()
|
|
# Adopt path: no reconcile — gateway state is intact.
|
|
reconcile.assert_not_called()
|
|
ip = infra_vm.netpool.orch_slot().guest_ip
|
|
self.assertEqual(f"http://{ip}:{infra_vm.ORCHESTRATOR_PORT}", url)
|
|
|
|
def test_reboots_both_when_version_stale(self):
|
|
# Healthy control plane but the running pair booted an OLDER image
|
|
# (marker mismatch) -> reboot rather than adopt stale code.
|
|
with tempfile.TemporaryDirectory() as td:
|
|
d = Path(td)
|
|
(d / "id_ed25519").write_text("k")
|
|
(d / "booted-version").write_text("v-old\n")
|
|
with patch.object(infra_vm, "_infra_dir", return_value=d), \
|
|
patch.object(infra_vm, "expected_version", return_value="v-current"), \
|
|
patch.object(infra_vm, "_health_ok", return_value=True), \
|
|
patch.object(infra_vm, "_pidfile_alive", return_value=True), \
|
|
patch.object(infra_vm, "stop") as stop, \
|
|
patch.object(infra_vm, "ensure_built"), \
|
|
patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
|
|
patch(_RECONCILE) as reconcile:
|
|
orch_cls.return_value.gateway_url.return_value = "http://10.243.255.1:8099"
|
|
orch_cls.return_value.mint_gateway_token.return_value = "gw.jwt"
|
|
FirecrackerInfraService().ensure_running()
|
|
stop.assert_called_once() # dislodge the outdated pair
|
|
orch_cls.return_value.ensure_running.assert_called_once()
|
|
# The gateway service is bound to the orchestrator's gateway URL,
|
|
# carrying the orchestrator-minted `gateway` token.
|
|
gw_cls.return_value.connect_to_orchestrator.assert_called_once_with(
|
|
"http://10.243.255.1:8099", "gw.jwt")
|
|
# The fresh boot records the current version for the next launcher.
|
|
self.assertEqual("v-current\n", (d / "booted-version").read_text())
|
|
# Cold boot: reconcile fires after gateway is up.
|
|
reconcile.assert_called_once()
|
|
|
|
def test_reboots_when_gateway_dead(self):
|
|
# Orchestrator healthy + marker current, but the gateway VM is gone ->
|
|
# the pair is half-down, so reboot both rather than adopt partial state.
|
|
with tempfile.TemporaryDirectory() as td:
|
|
d = Path(td)
|
|
(d / "id_ed25519").write_text("k")
|
|
(d / "booted-version").write_text("v-current\n")
|
|
with patch.object(infra_vm, "_infra_dir", return_value=d), \
|
|
patch.object(infra_vm, "expected_version", return_value="v-current"), \
|
|
patch.object(infra_vm, "_health_ok", return_value=True), \
|
|
patch.object(infra_vm, "_pidfile_alive", return_value=False), \
|
|
patch.object(infra_vm, "stop") as stop, \
|
|
patch.object(infra_vm, "ensure_built"), \
|
|
patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
|
|
patch(_RECONCILE) as reconcile:
|
|
FirecrackerInfraService().ensure_running()
|
|
stop.assert_called_once()
|
|
orch_cls.return_value.ensure_running.assert_called_once()
|
|
gw_cls.return_value.connect_to_orchestrator.assert_called_once()
|
|
reconcile.assert_called_once()
|
|
|
|
def test_boots_both_when_no_running_pair(self):
|
|
with tempfile.TemporaryDirectory() as td:
|
|
with patch.object(infra_vm, "_infra_dir", return_value=Path(td)), \
|
|
patch.object(infra_vm, "expected_version", return_value="v-current"), \
|
|
patch.object(infra_vm, "_health_ok", return_value=False), \
|
|
patch.object(infra_vm, "stop") as stop, \
|
|
patch.object(infra_vm, "ensure_built") as built, \
|
|
patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
|
|
patch(_RECONCILE) as reconcile:
|
|
FirecrackerInfraService().ensure_running()
|
|
stop.assert_called_once() # clear stale VMs first
|
|
built.assert_called_once()
|
|
# Orchestrator is brought up BEFORE the gateway is connected (its daemons
|
|
# reach the control plane at startup).
|
|
orch_cls.return_value.ensure_running.assert_called_once()
|
|
gw_cls.return_value.connect_to_orchestrator.assert_called_once()
|
|
# Cold boot: reconcile fires to restore CA, git-gate, and egress tokens.
|
|
reconcile.assert_called_once()
|
|
|
|
def test_reconcile_receives_url_and_fresh_gateway(self):
|
|
# reconcile gets the orchestrator URL and the gateway service (not the
|
|
# class). Verify the args to ensure it can reach the right endpoints.
|
|
with tempfile.TemporaryDirectory() as td:
|
|
with patch.object(infra_vm, "_infra_dir", return_value=Path(td)), \
|
|
patch.object(infra_vm, "expected_version", return_value="v-current"), \
|
|
patch.object(infra_vm, "_health_ok", return_value=False), \
|
|
patch.object(infra_vm, "stop"), \
|
|
patch.object(infra_vm, "ensure_built"), \
|
|
patch(_ORCH_CLS) as orch_cls, patch(_GW_CLS) as gw_cls, \
|
|
patch(_RECONCILE) as reconcile:
|
|
orch_cls.return_value.url.return_value = "http://10.243.255.1:8099"
|
|
FirecrackerInfraService().ensure_running()
|
|
call_args = reconcile.call_args
|
|
# First arg: the orchestrator's host URL.
|
|
self.assertEqual("http://10.243.255.1:8099", call_args.args[0])
|
|
# Second arg: the gateway service instance (not the class).
|
|
self.assertIs(gw_cls.return_value, call_args.args[1])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|