feat(backend): reconcile running bottles' CA on gateway bring-up (0081)

Add `attach_bottled_agents_to_gateway()` as an `@abc.abstractmethod` on `BottleBackend` and implement the first reconcile step across all three backends: on a gateway cold boot, replace every already-running agent's trusted CA with the freshly-minted gateway CA. Reconciling running bottles is a backend responsibility (only the backend can enumerate its agents and reach them — firecracker over SSH, docker/macOS over exec/cp), so making it an abstract method keeps the fix cross-backend by construction. The gateway rootfs is ephemeral, so a rebuild mints a new CA that every running bottle distrusts (SSL verification fails — #510). This reconcile is what distributes the fresh CA, so a routine gateway rebuild now doubles as a free CA rotation. Per-bottle steps are best-effort: one unreachable or malformed bottle is logged and skipped, never blocking the others. Trigger — `Gateway.connect_to_orchestrator` now returns a cold-boot bool (True when it actually (re)brought the gateway up, False when a healthy current gateway was left untouched); each infra `ensure_running` gates the reconcile on it, so it fires exactly on cold boot and never on an adopt. Git-gate re-provision and egress-token restore fold into this same reconcile on the same trigger in follow-up PRs (#516). Refs #516. Addresses #510. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-26 23:54:27 +00:00
parent 314b30c013
commit 36f594b770
21 changed files with 488 additions and 15 deletions
@@ -159,5 +159,126 @@ class TestFirecrackerReprovision(unittest.TestCase):
        restore.assert_called_once_with(client, {})


+class TestFirecrackerCaReconcile(unittest.TestCase):
+    """PRD 0081: attach_bottled_agents_to_gateway pushes the current gateway CA
+    into every running agent VM over SSH."""
+
+    def _run_dir(self, root: Path, ip: str = "10.243.0.3") -> Path:
+        run_dir = root / "demo"
+        run_dir.mkdir()
+        (run_dir / "bottle_id_ed25519").write_text("key")
+        (run_dir / "config.json").write_text(json.dumps({
+            "boot-source": {"boot_args": f"root=/dev/vda ip={ip}::gw:mask::eth0:off"}
+        }))
+        return run_dir
+
+    def test_pushes_current_ca_over_stdin_and_rebuilds_trust_store(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            run_dir = self._run_dir(Path(tmp))
+            gw = Mock()
+            gw.ca_cert_pem.return_value = "PEM-DATA"
+            with patch.object(fc, "FirecrackerGateway", return_value=gw), \
+                 patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \
+                 patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
+                 patch.object(fc.subprocess, "run", return_value=_proc()) as run, \
+                 patch.object(fc, "info"):
+                fc.attach_bottled_agents_to_gateway()
+        # The cert goes over stdin (off argv), and the trust store is rebuilt.
+        self.assertEqual("PEM-DATA", run.call_args.kwargs["input"])
+        script = run.call_args.args[0][-1]
+        self.assertIn(fc.AGENT_CA_PATH, script)
+        self.assertIn("update-ca-certificates", script)
+
+    def test_unreachable_vm_is_skipped_not_fatal(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            run_dir = self._run_dir(Path(tmp))
+            gw = Mock()
+            gw.ca_cert_pem.return_value = "PEM"
+            with patch.object(fc, "FirecrackerGateway", return_value=gw), \
+                 patch.object(fc.cleanup, "live_run_dirs", return_value=(run_dir,)), \
+                 patch.object(fc.util, "ssh_base_argv", return_value=["ssh", "guest"]), \
+                 patch.object(fc.subprocess, "run", return_value=_proc(1, stderr="down")), \
+                 patch.object(fc, "info") as info:
+                fc.attach_bottled_agents_to_gateway()  # does not raise
+        self.assertTrue(any("skipped" in c.args[0] for c in info.call_args_list))
+
+    def test_missing_gateway_ca_skips_the_whole_reconcile(self) -> None:
+        gw = Mock()
+        gw.ca_cert_pem.side_effect = fc.GatewayError("no CA yet")
+        with patch.object(fc, "FirecrackerGateway", return_value=gw), \
+             patch.object(fc.cleanup, "live_run_dirs") as live, \
+             patch.object(fc, "info"):
+            fc.attach_bottled_agents_to_gateway()
+        live.assert_not_called()  # bailed before enumerating agents
+
+
+class TestDockerCaReconcile(unittest.TestCase):
+    def test_pushes_ca_to_each_agent_container_excluding_gateway(self) -> None:
+        # network inspect lists the gateway (excluded) + one agent + a blank line.
+        inspect = _proc(stdout="bot-bottle-orch-gateway\nbot-bottle-a\n\n")
+        with patch.object(
+            docker, "run_docker",
+            side_effect=[inspect, _proc(), _proc(), _proc()],
+        ) as run, patch.object(docker.log, "info"):
+            docker.attach_bottled_agents_to_gateway(
+                "PEM", gateway_name="bot-bottle-orch-gateway")
+        argvs = [c.args[0] for c in run.call_args_list]
+        # First call enumerates; the agent then gets mkdir + cp + exec — never
+        # the gateway container.
+        self.assertNotIn(
+            "bot-bottle-orch-gateway",
+            " ".join(tok for a in argvs[1:] for tok in a),
+        )
+        cp = next(a for a in argvs if a[:2] == ["docker", "cp"])
+        self.assertEqual(f"bot-bottle-a:{docker.AGENT_CA_PATH}", cp[-1])
+        exec_argv = next(a for a in argvs if "update-ca-certificates" in a[-1])
+        self.assertIn(docker.AGENT_CA_PATH, exec_argv[-1])
+
+    def test_one_container_failure_does_not_block_others(self) -> None:
+        inspect = _proc(stdout="a\nb\n")
+        # a: mkdir fails (skip); b: mkdir/cp/exec all succeed.
+        with patch.object(
+            docker, "run_docker",
+            side_effect=[inspect, _proc(1, stderr="gone"), _proc(), _proc(), _proc()],
+        ), patch.object(docker.log, "info") as info:
+            docker.attach_bottled_agents_to_gateway("PEM", gateway_name="gw")
+        self.assertTrue(
+            any("skipped for a" in str(c.args[0]) for c in info.call_args_list)
+        )
+
+    def test_fetches_ca_from_gateway_when_not_supplied(self) -> None:
+        svc = Mock()
+        svc.gateway.return_value.ca_cert_pem.return_value = "FETCHED"
+        with patch.object(docker, "DockerInfraService", return_value=svc), \
+             patch.object(docker, "run_docker", return_value=_proc(stdout="")), \
+             patch.object(docker.log, "info"):
+            docker.attach_bottled_agents_to_gateway(network="net", gateway_name="gw")
+        svc.gateway.return_value.ca_cert_pem.assert_called_once()
+
+
+class TestMacosCaReconcile(unittest.TestCase):
+    def test_pushes_ca_to_each_running_agent_container(self) -> None:
+        agent = SimpleNamespace(slug="demo")
+        with patch.object(mac, "enumerate_active", return_value=[agent]), \
+             patch.object(mac.container_mod, "run_container_argv",
+                          return_value=_proc()) as run, \
+             patch.object(mac, "info"):
+            mac.attach_bottled_agents_to_gateway("PEM")
+        argvs = [c.args[0] for c in run.call_args_list]
+        name = f"{mac.CONTAINER_NAME_PREFIX}demo"
+        self.assertTrue(any(name in a for a in argvs))
+        self.assertTrue(
+            any("update-ca-certificates" in a[-1] for a in argvs if a[-2:-1] == ["-c"])
+        )
+        self.assertTrue(any(mac.AGENT_CA_PATH in " ".join(a) for a in argvs))
+
+    def test_enumeration_failure_is_best_effort(self) -> None:
+        with patch.object(mac, "enumerate_active",
+                          side_effect=mac.EnumerationError("failed")), \
+             patch.object(mac, "info") as info:
+            mac.attach_bottled_agents_to_gateway("PEM")  # does not raise
+        self.assertIn("skipped", info.call_args.args[0])
+
+
 if __name__ == "__main__":
    unittest.main()