fix(orchestrator): reap registry rows whose bottle is no longer running
A registry row only ever left the registry two ways: an explicit teardown_bottle (the launcher's cleanup callback) or the same-IP supersede sweep in register(). Neither runs when the launching CLI dies hard, so the row outlives its container. That orphan is not inert. Source IPs are recycled by the backend's DHCP and by_source_ip fail-closes on ambiguity, so a leftover row at a reused address resolves no policy at all for the next bottle that lands there — and a bottle with no policy denies every host, which surfaces to the agent as "host X is not in the allowlist" for hosts that were never the problem. Add reap_absent/reconcile and call it from the macOS launch path before registering, so each launch self-heals the registry. Restores the invariant the data plane needs: at most one active row per live address, and none for a dead one. The second half matters as much as the first — when several rows claim a *live* address the newest wins and the rest are swept, otherwise a recycled address stays ambiguous, which is exactly the bricked state. The host supplies the live set because the orchestrator runs inside the infra container and cannot see the backend. A grace window exempts rows younger than it, so reconciliation cannot race a bottle still coming up, and a reconcile failure is logged rather than blocking an otherwise-fine launch. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -4,8 +4,10 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import secrets
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import closing
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -304,3 +306,55 @@ class TestOrchestratorSupervise(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestOrchestratorReconcile(unittest.TestCase):
|
||||
"""`reconcile` — drop rows for bottles that are no longer running."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self._tmp = tempfile.TemporaryDirectory()
|
||||
self.secret = secrets.token_bytes(16)
|
||||
self.db = Path(self._tmp.name) / "r.db"
|
||||
self.store = RegistryStore(self.db)
|
||||
self.store.migrate()
|
||||
self.broker = StubBroker(self.secret)
|
||||
self.orch = Orchestrator(self.store, self.broker, self.secret)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self._tmp.cleanup()
|
||||
|
||||
def _age_all(self, seconds: float) -> None:
|
||||
"""Backdate every row past the reap grace window."""
|
||||
with closing(sqlite3.connect(self.db)) as conn:
|
||||
conn.execute(
|
||||
"UPDATE orchestrator_bottles SET created_at = created_at - ?", (seconds,))
|
||||
conn.commit()
|
||||
|
||||
def test_reaps_dead_bottle_and_forgets_its_tokens(self) -> None:
|
||||
dead = self.orch.launch_bottle("10.243.0.1", tokens={"EGRESS_TOKEN_0": "s3cret"})
|
||||
live = self.orch.launch_bottle("10.243.0.2", tokens={"EGRESS_TOKEN_0": "keep"})
|
||||
self._age_all(600)
|
||||
|
||||
self.assertEqual([dead.bottle_id], self.orch.reconcile(["10.243.0.2"]))
|
||||
self.assertIsNone(self.store.get(dead.bottle_id))
|
||||
self.assertIsNotNone(self.store.get(live.bottle_id))
|
||||
# The in-memory egress credential goes with the row.
|
||||
self.assertEqual({}, self.orch.tokens_for(dead.bottle_id))
|
||||
self.assertEqual({"EGRESS_TOKEN_0": "keep"}, self.orch.tokens_for(live.bottle_id))
|
||||
|
||||
def test_reconcile_does_not_broker_a_teardown(self) -> None:
|
||||
"""The container is already gone — there is nothing to stop, and a
|
||||
broker error must not stop the sweep clearing the row."""
|
||||
self.orch.launch_bottle("10.243.0.1")
|
||||
self._age_all(600)
|
||||
self.broker.launched.clear()
|
||||
self.orch.reconcile([])
|
||||
self.assertEqual([], self.broker.torn_down)
|
||||
|
||||
def test_reconcile_keeps_everything_when_all_are_live(self) -> None:
|
||||
a = self.orch.launch_bottle("10.243.0.1")
|
||||
b = self.orch.launch_bottle("10.243.0.2")
|
||||
self._age_all(600)
|
||||
self.assertEqual([], self.orch.reconcile(["10.243.0.1", "10.243.0.2"]))
|
||||
self.assertIsNotNone(self.store.get(a.bottle_id))
|
||||
self.assertIsNotNone(self.store.get(b.bottle_id))
|
||||
|
||||
Reference in New Issue
Block a user