fix(orchestrator): recreate the orchestrator container on ensure_running

The orchestrator runs the repo's code bind-mounted, but the Python process
loads it at startup and never reloads — and ensure_running reused a
healthy-but-stale container. So after a code change (e.g. the token fix), the
running orchestrator kept executing OLD control-plane code that dropped the
new /bottles 'tokens' field, and egress auth injection stayed broken no matter
how many times the gateway image was rebuilt.

ensure_running now always recreates the orchestrator container (cheap; the
registry DB persists and the current launch re-registers its in-memory
state). Combined with the gateway's image-staleness recreate, a fresh 'start'
now runs current code end to end.

Validated live: register an authed route + in-memory token -> a client at the
bottle IP curling through the gateway proxy receives the injected
'Authorization: Bearer <token>' header.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck
This commit is contained in:
2026-07-14 02:16:06 -04:00
parent 3318bdce28
commit 13a8bdd7ca
2 changed files with 19 additions and 10 deletions
+10 -5
View File
@@ -46,13 +46,18 @@ class TestOrchestratorService(unittest.TestCase):
with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
self.assertFalse(self.svc.is_healthy())
def test_ensure_running_healthy_still_ensures_gateway_but_not_orchestrator(self) -> None:
def test_ensure_running_always_recreates_orchestrator(self) -> None:
# Even when a control plane is already healthy, the orchestrator is
# recreated so bind-mounted code changes take effect (its process
# won't reload). The gateway is ensured too.
run = Mock(return_value=Mock(returncode=0, stderr=""))
with patch(_URLOPEN, return_value=_health(200)), \
patch(_GATEWAY) as gw_cls, patch(_RUN) as run:
patch(_GATEWAY) as gw_cls, patch(_RUN, run), patch(_SLEEP):
self.assertEqual(self.svc.url, self.svc.ensure_running())
gw = gw_cls.return_value
gw.ensure_running.assert_called() # gateway kept up
run.assert_not_called() # no orchestrator container run
gw_cls.return_value.ensure_running.assert_called() # gateway kept up
runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]]
self.assertEqual(1, len(runs)) # orchestrator recreated
self.assertIn(ORCHESTRATOR_NAME, runs[0])
def test_ensure_running_starts_orchestrator_container_when_absent(self) -> None:
run = Mock(return_value=Mock(returncode=0, stderr=""))