From 13a8bdd7ca2ad2e64a8e9dccbb90bd0c07382821 Mon Sep 17 00:00:00 2001 From: didericis Date: Tue, 14 Jul 2026 02:16:06 -0400 Subject: [PATCH] fix(orchestrator): recreate the orchestrator container on ensure_running MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The orchestrator runs the repo's code bind-mounted, but the Python process loads it at startup and never reloads — and ensure_running reused a healthy-but-stale container. So after a code change (e.g. the token fix), the running orchestrator kept executing OLD control-plane code that dropped the new /bottles 'tokens' field, and egress auth injection stayed broken no matter how many times the gateway image was rebuilt. ensure_running now always recreates the orchestrator container (cheap; the registry DB persists and the current launch re-registers its in-memory state). Combined with the gateway's image-staleness recreate, a fresh 'start' now runs current code end to end. Validated live: register an authed route + in-memory token -> a client at the bottle IP curling through the gateway proxy receives the injected 'Authorization: Bearer ' header. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck --- bot_bottle/orchestrator/lifecycle.py | 14 +++++++++----- tests/unit/test_orchestrator_lifecycle.py | 15 ++++++++++----- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/bot_bottle/orchestrator/lifecycle.py b/bot_bottle/orchestrator/lifecycle.py index 0fffbf5..3fa7153 100644 --- a/bot_bottle/orchestrator/lifecycle.py +++ b/bot_bottle/orchestrator/lifecycle.py @@ -127,13 +127,17 @@ class OrchestratorService: gateway are left untouched. Raises `OrchestratorStartError` on timeout.""" gateway = self._gateway() - gateway.ensure_built() # build the bundle image if missing (both use it) - if self.is_healthy(): - gateway.ensure_running() # make sure the gateway is up too - return self.url + gateway.ensure_built() # rebuild the bundle image on a source change + gateway.ensure_running() # creates the shared network + (re)starts gateway + # Always (re)create the orchestrator container. It runs the repo's code + # bind-mounted, but the Python process loaded that code at startup and + # won't reload — so reusing a healthy-but-stale container would keep + # running OLD control-plane code (e.g. dropping the tokens field). Cheap + # (~seconds); the registry DB persists and the current launch + # re-registers its own in-memory state. (The dedicated orchestrator + # image follow-up replaces this with image-staleness detection.) log.info("starting orchestrator container", context={"name": ORCHESTRATOR_NAME}) - gateway.ensure_running() # creates the shared network + gateway self._run_orchestrator_container() deadline = time.monotonic() + startup_timeout diff --git a/tests/unit/test_orchestrator_lifecycle.py b/tests/unit/test_orchestrator_lifecycle.py index 359e18a..4eeccba 100644 --- a/tests/unit/test_orchestrator_lifecycle.py +++ b/tests/unit/test_orchestrator_lifecycle.py @@ -46,13 +46,18 @@ class TestOrchestratorService(unittest.TestCase): with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")): self.assertFalse(self.svc.is_healthy()) - def test_ensure_running_healthy_still_ensures_gateway_but_not_orchestrator(self) -> None: + def test_ensure_running_always_recreates_orchestrator(self) -> None: + # Even when a control plane is already healthy, the orchestrator is + # recreated so bind-mounted code changes take effect (its process + # won't reload). The gateway is ensured too. + run = Mock(return_value=Mock(returncode=0, stderr="")) with patch(_URLOPEN, return_value=_health(200)), \ - patch(_GATEWAY) as gw_cls, patch(_RUN) as run: + patch(_GATEWAY) as gw_cls, patch(_RUN, run), patch(_SLEEP): self.assertEqual(self.svc.url, self.svc.ensure_running()) - gw = gw_cls.return_value - gw.ensure_running.assert_called() # gateway kept up - run.assert_not_called() # no orchestrator container run + gw_cls.return_value.ensure_running.assert_called() # gateway kept up + runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]] + self.assertEqual(1, len(runs)) # orchestrator recreated + self.assertIn(ORCHESTRATOR_NAME, runs[0]) def test_ensure_running_starts_orchestrator_container_when_absent(self) -> None: run = Mock(return_value=Mock(returncode=0, stderr=""))