didericis-claude
de80aafb1f
Add `attach_bottled_agents_to_gateway()` as an `@abc.abstractmethod` on `BottleBackend` and implement the first reconcile step across all three backends: on a gateway cold boot, replace every already-running agent's trusted CA with the freshly-minted gateway CA. Reconciling running bottles is a backend responsibility (only the backend can enumerate its agents and reach them — firecracker over SSH, docker/macOS over exec/cp), so making it an abstract method keeps the fix cross-backend by construction. The gateway rootfs is ephemeral, so a rebuild mints a new CA that every running bottle distrusts (SSL verification fails — #510). This reconcile is what distributes the fresh CA, so a routine gateway rebuild now doubles as a free CA rotation. Per-bottle steps are best-effort: one unreachable or malformed bottle is logged and skipped, never blocking the others. Trigger — `Gateway.connect_to_orchestrator` now returns a cold-boot bool (True when it actually (re)brought the gateway up, False when a healthy current gateway was left untouched); each infra `ensure_running` gates the reconcile on it, so it fires exactly on cold boot and never on an adopt. Git-gate re-provision and egress-token restore fold into this same reconcile on the same trigger in follow-up PRs (#516). Refs #516. Addresses #510. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
285 lines
14 KiB
Python
285 lines
14 KiB
Python
"""Backend-agnostic and backend-specific encrypted-secret recovery tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import Mock, patch
|
|
|
|
from bot_bottle.orchestrator.reprovision import reprovision_bottles
|
|
from bot_bottle.backend.firecracker import consolidated_launch as fc
|
|
from bot_bottle.backend.macos_container import consolidated_launch as mac
|
|
from bot_bottle.backend.docker import consolidated_launch as docker
|
|
from bot_bottle.orchestrator.client import OrchestratorClientError
|
|
|
|
|
|
def _proc(returncode: int = 0, stdout: str = "", stderr: str = ""):
|
|
return subprocess.CompletedProcess([], returncode, stdout=stdout, stderr=stderr)
|
|
|
|
|
|
class TestSharedReprovision(unittest.TestCase):
|
|
def test_joins_registry_records_by_source_ip(self) -> None:
|
|
client = Mock()
|
|
client.list_bottles.return_value = [
|
|
{"bottle_id": "b1", "source_ip": "10.0.0.1"},
|
|
{"bottle_id": "b2", "source_ip": "10.0.0.2"},
|
|
{"bottle_id": 3, "source_ip": "10.0.0.3"},
|
|
]
|
|
client.reprovision_gateway.side_effect = [True, False]
|
|
count = reprovision_bottles(
|
|
client, {"10.0.0.1": " key-1\n", "10.0.0.2": "key-2"},
|
|
)
|
|
self.assertEqual(1, count)
|
|
self.assertEqual(
|
|
[("b1", "key-1"), ("b2", "key-2")],
|
|
[call.args for call in client.reprovision_gateway.call_args_list],
|
|
)
|
|
|
|
def test_one_failure_does_not_block_other_bottles(self) -> None:
|
|
client = Mock()
|
|
client.list_bottles.return_value = [
|
|
{"bottle_id": "b1", "source_ip": "10.0.0.1"},
|
|
{"bottle_id": "b2", "source_ip": "10.0.0.2"},
|
|
]
|
|
client.reprovision_gateway.side_effect = [
|
|
OrchestratorClientError("bad key"), True,
|
|
]
|
|
with patch("bot_bottle.orchestrator.reprovision.debug") as debug:
|
|
count = reprovision_bottles(
|
|
client, {"10.0.0.1": "key-1", "10.0.0.2": "key-2"},
|
|
)
|
|
self.assertEqual(1, count)
|
|
self.assertEqual("b1", debug.call_args.kwargs["context"]["bottle_id"])
|
|
self.assertNotIn("bad key", repr(debug.call_args))
|
|
|
|
|
|
class TestMacosReprovision(unittest.TestCase):
|
|
def test_reads_configured_container_env_and_reprovisions(self) -> None:
|
|
endpoint = mac.GatewayEndpoint("http://orch", "10.0.0.9", "PEM", "net")
|
|
agent = SimpleNamespace(slug="demo")
|
|
client = Mock()
|
|
with patch.object(mac, "enumerate_active", return_value=[agent]), \
|
|
patch.object(mac.container_mod, "inspect_container_network_ip",
|
|
return_value="10.0.0.1"), \
|
|
patch.object(mac.container_mod, "read_container_env", return_value="key"), \
|
|
patch.object(mac, "OrchestratorClient", return_value=client), \
|
|
patch.object(mac, "reprovision_bottles", return_value=1) as restore, \
|
|
patch.object(mac, "info"):
|
|
mac._reprovision_running_bottles(endpoint)
|
|
restore.assert_called_once_with(client, {"10.0.0.1": "key"})
|
|
|
|
def test_enumeration_failure_is_best_effort(self) -> None:
|
|
endpoint = mac.GatewayEndpoint("http://orch", "10.0.0.9", "PEM", "net")
|
|
with patch.object(mac, "enumerate_active",
|
|
side_effect=mac.EnumerationError("failed")), \
|
|
patch.object(mac, "info") as info:
|
|
mac._reprovision_running_bottles(endpoint)
|
|
self.assertIn("skipped", info.call_args.args[0])
|
|
|
|
|
|
class TestDockerReprovision(unittest.TestCase):
|
|
def test_maps_network_containers_to_keys(self) -> None:
|
|
# The gateway container (excluded — it's on the data network but isn't
|
|
# an agent) + one agent + a malformed line.
|
|
inspect = _proc(stdout=(
|
|
"bot-bottle-orch-gateway 172.18.0.2/16\n"
|
|
"bot-bottle-a 172.18.0.3/16\n"
|
|
"malformed\n"
|
|
))
|
|
key = _proc(stdout="secret\n")
|
|
client = Mock()
|
|
with patch.object(docker, "OrchestratorClient", return_value=client), \
|
|
patch.object(docker, "run_docker", side_effect=[inspect, key]), \
|
|
patch.object(docker, "reprovision_bottles", return_value=1) as restore, \
|
|
patch.object(docker.log, "info"):
|
|
docker._reprovision_running_bottles("http://orch")
|
|
restore.assert_called_once_with(client, {"172.18.0.3": "secret"})
|
|
|
|
def test_missing_docker_is_best_effort(self) -> None:
|
|
with patch.object(docker, "run_docker", side_effect=FileNotFoundError("docker")), \
|
|
patch.object(docker.log, "info") as info:
|
|
docker._reprovision_running_bottles("http://orch")
|
|
self.assertIn("skipped", info.call_args.args[0])
|
|
|
|
|
|
class TestFirecrackerReprovision(unittest.TestCase):
|
|
def _run_dir(self, root: Path, ip: str = "10.243.0.3") -> Path:
|
|
run_dir = root / "demo"
|
|
run_dir.mkdir()
|
|
(run_dir / "bottle_id_ed25519").write_text("key")
|
|
(run_dir / "config.json").write_text(json.dumps({
|
|
"boot-source": {"boot_args": f"root=/dev/vda ip={ip}::gw:mask::eth0:off"}
|
|
}))
|
|
return run_dir
|
|
|
|
def test_extracts_guest_ip_from_config(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
run_dir = self._run_dir(Path(tmp))
|
|
self.assertEqual("10.243.0.3", fc._guest_ip_from_config(run_dir / "config.json"))
|
|
self.assertEqual("", fc._guest_ip_from_config(run_dir / "missing.json"))
|
|
|
|
def test_persists_key_over_stdin_not_argv(self) -> None:
|
|
with patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
|
|
patch.object(fc.subprocess, "run", return_value=_proc()) as run:
|
|
fc.persist_env_var_secret(Path("/key"), "10.0.0.1", "super-secret")
|
|
self.assertEqual("super-secret", run.call_args.kwargs["input"])
|
|
self.assertNotIn("super-secret", " ".join(run.call_args.args[0]))
|
|
|
|
def test_persist_failure_is_fatal_to_launch(self) -> None:
|
|
with patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
|
|
patch.object(fc.subprocess, "run", return_value=_proc(1, stderr="denied")):
|
|
with self.assertRaisesRegex(fc.ConsolidatedLaunchError, "denied"):
|
|
fc.persist_env_var_secret(Path("/key"), "10.0.0.1", "secret")
|
|
|
|
def test_reads_live_vm_key_and_reprovisions(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
run_dir = self._run_dir(Path(tmp))
|
|
client = Mock()
|
|
with patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \
|
|
patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
|
|
patch.object(fc.subprocess, "run", return_value=_proc(stdout="secret\n")), \
|
|
patch.object(fc, "reprovision_bottles", return_value=1) as restore, \
|
|
patch.object(fc, "info"):
|
|
fc._reprovision_running_bottles(client)
|
|
restore.assert_called_once_with(client, {"10.243.0.3": "secret"})
|
|
|
|
def test_unreadable_vm_is_skipped(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
run_dir = self._run_dir(Path(tmp))
|
|
client = Mock()
|
|
with patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \
|
|
patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
|
|
patch.object(fc.subprocess, "run", return_value=_proc(1)), \
|
|
patch.object(fc, "reprovision_bottles", return_value=0) as restore:
|
|
fc._reprovision_running_bottles(client)
|
|
restore.assert_called_once_with(client, {})
|
|
|
|
|
|
class TestFirecrackerCaReconcile(unittest.TestCase):
|
|
"""PRD 0081: attach_bottled_agents_to_gateway pushes the current gateway CA
|
|
into every running agent VM over SSH."""
|
|
|
|
def _run_dir(self, root: Path, ip: str = "10.243.0.3") -> Path:
|
|
run_dir = root / "demo"
|
|
run_dir.mkdir()
|
|
(run_dir / "bottle_id_ed25519").write_text("key")
|
|
(run_dir / "config.json").write_text(json.dumps({
|
|
"boot-source": {"boot_args": f"root=/dev/vda ip={ip}::gw:mask::eth0:off"}
|
|
}))
|
|
return run_dir
|
|
|
|
def test_pushes_current_ca_over_stdin_and_rebuilds_trust_store(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
run_dir = self._run_dir(Path(tmp))
|
|
gw = Mock()
|
|
gw.ca_cert_pem.return_value = "PEM-DATA"
|
|
with patch.object(fc, "FirecrackerGateway", return_value=gw), \
|
|
patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \
|
|
patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
|
|
patch.object(fc.subprocess, "run", return_value=_proc()) as run, \
|
|
patch.object(fc, "info"):
|
|
fc.attach_bottled_agents_to_gateway()
|
|
# The cert goes over stdin (off argv), and the trust store is rebuilt.
|
|
self.assertEqual("PEM-DATA", run.call_args.kwargs["input"])
|
|
script = run.call_args.args[0][-1]
|
|
self.assertIn(fc.AGENT_CA_PATH, script)
|
|
self.assertIn("update-ca-certificates", script)
|
|
|
|
def test_unreachable_vm_is_skipped_not_fatal(self) -> None:
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
run_dir = self._run_dir(Path(tmp))
|
|
gw = Mock()
|
|
gw.ca_cert_pem.return_value = "PEM"
|
|
with patch.object(fc, "FirecrackerGateway", return_value=gw), \
|
|
patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \
|
|
patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
|
|
patch.object(fc.subprocess, "run", return_value=_proc(1, stderr="down")), \
|
|
patch.object(fc, "info") as info:
|
|
fc.attach_bottled_agents_to_gateway() # does not raise
|
|
self.assertTrue(any("skipped" in c.args[0] for c in info.call_args_list))
|
|
|
|
def test_missing_gateway_ca_skips_the_whole_reconcile(self) -> None:
|
|
gw = Mock()
|
|
gw.ca_cert_pem.side_effect = fc.GatewayError("no CA yet")
|
|
with patch.object(fc, "FirecrackerGateway", return_value=gw), \
|
|
patch.object(fc.cleanup, "live_run_dirs") as live, \
|
|
patch.object(fc, "info"):
|
|
fc.attach_bottled_agents_to_gateway()
|
|
live.assert_not_called() # bailed before enumerating agents
|
|
|
|
|
|
class TestDockerCaReconcile(unittest.TestCase):
|
|
def test_pushes_ca_to_each_agent_container_excluding_gateway(self) -> None:
|
|
# network inspect lists the gateway (excluded) + one agent + a blank line.
|
|
inspect = _proc(stdout="bot-bottle-orch-gateway\nbot-bottle-a\n\n")
|
|
with patch.object(
|
|
docker, "run_docker",
|
|
side_effect=[inspect, _proc(), _proc(), _proc()],
|
|
) as run, patch.object(docker.log, "info"):
|
|
docker.attach_bottled_agents_to_gateway(
|
|
"PEM", gateway_name="bot-bottle-orch-gateway")
|
|
argvs = [c.args[0] for c in run.call_args_list]
|
|
# First call enumerates; the agent then gets mkdir + cp + exec — never
|
|
# the gateway container.
|
|
self.assertNotIn(
|
|
"bot-bottle-orch-gateway",
|
|
" ".join(tok for a in argvs[1:] for tok in a),
|
|
)
|
|
cp = next(a for a in argvs if a[:2] == ["docker", "cp"])
|
|
self.assertEqual(f"bot-bottle-a:{docker.AGENT_CA_PATH}", cp[-1])
|
|
exec_argv = next(a for a in argvs if "update-ca-certificates" in a[-1])
|
|
self.assertIn(docker.AGENT_CA_PATH, exec_argv[-1])
|
|
|
|
def test_one_container_failure_does_not_block_others(self) -> None:
|
|
inspect = _proc(stdout="a\nb\n")
|
|
# a: mkdir fails (skip); b: mkdir/cp/exec all succeed.
|
|
with patch.object(
|
|
docker, "run_docker",
|
|
side_effect=[inspect, _proc(1, stderr="gone"), _proc(), _proc(), _proc()],
|
|
), patch.object(docker.log, "info") as info:
|
|
docker.attach_bottled_agents_to_gateway("PEM", gateway_name="gw")
|
|
self.assertTrue(
|
|
any("skipped for a" in str(c.args[0]) for c in info.call_args_list)
|
|
)
|
|
|
|
def test_fetches_ca_from_gateway_when_not_supplied(self) -> None:
|
|
svc = Mock()
|
|
svc.gateway.return_value.ca_cert_pem.return_value = "FETCHED"
|
|
with patch.object(docker, "DockerInfraService", return_value=svc), \
|
|
patch.object(docker, "run_docker", return_value=_proc(stdout="")), \
|
|
patch.object(docker.log, "info"):
|
|
docker.attach_bottled_agents_to_gateway(network="net", gateway_name="gw")
|
|
svc.gateway.return_value.ca_cert_pem.assert_called_once()
|
|
|
|
|
|
class TestMacosCaReconcile(unittest.TestCase):
|
|
def test_pushes_ca_to_each_running_agent_container(self) -> None:
|
|
agent = SimpleNamespace(slug="demo")
|
|
with patch.object(mac, "enumerate_active", return_value=[agent]), \
|
|
patch.object(mac.container_mod, "run_container_argv",
|
|
return_value=_proc()) as run, \
|
|
patch.object(mac, "info"):
|
|
mac.attach_bottled_agents_to_gateway("PEM")
|
|
argvs = [c.args[0] for c in run.call_args_list]
|
|
name = f"{mac.CONTAINER_NAME_PREFIX}demo"
|
|
self.assertTrue(any(name in a for a in argvs))
|
|
self.assertTrue(
|
|
any("update-ca-certificates" in a[-1] for a in argvs if a[-2:-1] == ["-c"])
|
|
)
|
|
self.assertTrue(any(mac.AGENT_CA_PATH in " ".join(a) for a in argvs))
|
|
|
|
def test_enumeration_failure_is_best_effort(self) -> None:
|
|
with patch.object(mac, "enumerate_active",
|
|
side_effect=mac.EnumerationError("failed")), \
|
|
patch.object(mac, "info") as info:
|
|
mac.attach_bottled_agents_to_gateway("PEM") # does not raise
|
|
self.assertIn("skipped", info.call_args.args[0])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|