fix(orchestrator): only recreate the orchestrator on a real source change

ensure_running() unconditionally recreated the bot-bottle-orchestrator
container on every bottle launch, added so bind-mounted code changes take
effect (the process never reloads). But the orchestrator's per-bottle
egress auth tokens live only in its process memory (Orchestrator._tokens),
by design never persisted to disk. So starting a second bottle recreated
the container and silently dropped the FIRST bottle's already-injected
token -- its next authed egress call 403s with "env var EGRESS_TOKEN_0 is
unset", and it needs a full restart to recover.

Now the orchestrator container is labeled with a content hash of its
bind-mounted bot_bottle source, mirroring the gateway's existing
image-staleness check. ensure_running only recreates on a hash mismatch
(a real code change), so a bottle launch that isn't accompanied by a code
change leaves a healthy orchestrator -- and every other active bottle's
in-memory tokens -- alone.

Fixes #381.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

bot_bottle/backend/__init__.py

@@ -40,7 +40,7 @@ from abc import ABC, abstractmethod
 from contextlib import AbstractContextManager
 from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Generic, Sequence, TypeVar
+from typing import Any, Generic, Sequence, TypeVar
 
 from ..agent_provider import AgentProvisionPlan, get_provider, build_agent_provision_plan
 from ..egress import EgressPlan
@@ -54,9 +54,6 @@ from ..workspace import WorkspacePlan, workspace_plan
 from .print_util import print_multi, visible_agent_env_names
 from .util import host_skill_dir
 
-if TYPE_CHECKING:
-    from .freeze import CommitCancelled, Freezer, get_freezer
-
 
 @dataclass(frozen=True)
 class BottleSpec:
@@ -575,63 +572,28 @@ class BottleBackend(ABC, Generic[PlanT, CleanupT]):
         Not called by the launch path or the test suite."""
 
 
-# _backends is None until the first call to _get_backends(), at which
-# point all three concrete backend classes are imported and instantiated.
-# Keeping the imports out of module scope means that importing any
-# backend sub-module (e.g. `backend.docker.util`) no longer drags the
-# firecracker and macos-container implementations into memory.
-#
-# Tests may replace _backends with a {name: fake} dict via patch.object;
-# _get_backends() returns the current module-level value as-is when it
-# is not None, so test fakes take effect without triggering real imports.
-_backends: dict[str, BottleBackend[Any, Any]] | None = None
+# Import concrete backend classes AFTER the base types are defined, so
+# each backend module can pull BottleSpec / BottlePlan / BottleBackend
+# via `from . import ...` without hitting a partially-initialized module.
+from .docker import DockerBottleBackend  # noqa: E402  # pylint: disable=wrong-import-position
+from .firecracker import FirecrackerBottleBackend  # noqa: E402  # pylint: disable=wrong-import-position
+from .macos_container import MacosContainerBottleBackend  # noqa: E402  # pylint: disable=wrong-import-position
+
+# Freezer is imported after the backend classes for the same reason:
+# Freezer.commit_slug constructs ActiveAgent, which must be fully
+# defined first.
+from .freeze import CommitCancelled, Freezer, get_freezer  # noqa: E402  # pylint: disable=wrong-import-position
 
 
-def _get_backends() -> dict[str, BottleBackend[Any, Any]]:
-    """Return the registry of all backend instances, loading lazily on first call."""
-    global _backends  # pylint: disable=global-statement
-    if _backends is None:
-        from .docker import DockerBottleBackend
-        from .firecracker import FirecrackerBottleBackend
-        from .macos_container import MacosContainerBottleBackend
-        _backends = {
-            "docker": DockerBottleBackend(),
-            "firecracker": FirecrackerBottleBackend(),
-            "macos-container": MacosContainerBottleBackend(),
-        }
-    return _backends
-
-
-def __getattr__(name: str) -> Any:
-    """Lazily surface concrete backend classes and freeze symbols at the
-    package level so existing `from bot_bottle.backend import X` and
-    `patch.object(backend_mod, X, ...)` call-sites keep working without
-    forcing an import of every backend at module-init time."""
-    if name == "DockerBottleBackend":
-        from .docker import DockerBottleBackend
-        globals()[name] = DockerBottleBackend
-        return DockerBottleBackend
-    if name == "FirecrackerBottleBackend":
-        from .firecracker import FirecrackerBottleBackend
-        globals()[name] = FirecrackerBottleBackend
-        return FirecrackerBottleBackend
-    if name == "MacosContainerBottleBackend":
-        from .macos_container import MacosContainerBottleBackend
-        globals()[name] = MacosContainerBottleBackend
-        return MacosContainerBottleBackend
-    if name == "CommitCancelled":
-        from .freeze import CommitCancelled
-        globals()[name] = CommitCancelled
-        return CommitCancelled
-    if name == "Freezer":
-        from .freeze import Freezer
-        globals()[name] = Freezer
-        return Freezer
-    if name == "get_freezer":
-        from .freeze import get_freezer
-        globals()[name] = get_freezer
-        return get_freezer
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+# The dict is heterogeneous: each value is a BottleBackend specialized
+# over its own plan type. Concrete plan types are erased here because
+# the registry is selected at runtime and the CLI only needs the
+# unparameterized methods (prepare → plan → launch(plan), cleanup, etc.).
+_BACKENDS: dict[str, BottleBackend[Any, Any]] = {
+    "docker": DockerBottleBackend(),
+    "firecracker": FirecrackerBottleBackend(),
+    "macos-container": MacosContainerBottleBackend(),
+}
 
 
 def get_bottle_backend(
@@ -649,11 +611,10 @@ def get_bottle_backend(
     Dies with a pointer at the known backends if the chosen name
     isn't implemented."""
     resolved = name or os.environ.get("BOT_BOTTLE_BACKEND") or _default_backend_name()
-    backends = _get_backends()
-    if resolved not in backends:
-        known = ", ".join(sorted(backends))
+    if resolved not in _BACKENDS:
+        known = ", ".join(sorted(_BACKENDS))
         die(f"unknown backend {resolved!r}; known backends: {known}")
-    return backends[resolved]
+    return _BACKENDS[resolved]
 
 
 def _default_backend_name() -> str:
@@ -663,17 +624,16 @@ def _default_backend_name() -> str:
     # `firecracker` binary isn't installed yet: selecting it here routes
     # start through firecracker's preflight, which prints an install
     # pointer, instead of silently falling back to docker.
-    from .firecracker import FirecrackerBottleBackend
     if FirecrackerBottleBackend.is_host_capable():
         return "firecracker"
     return "docker"
 
 
 def known_backend_names() -> tuple[str, ...]:
-    """Sorted tuple of all backend keys in `_get_backends()`. Used by
+    """Sorted tuple of all backend keys in `_BACKENDS`. Used by
     argparse (`--backend` choices) and the dashboard's backend
     picker."""
-    return tuple(sorted(_get_backends()))
+    return tuple(sorted(_BACKENDS))
 
 
 def has_backend(name: str) -> bool:
@@ -685,10 +645,9 @@ def has_backend(name: str) -> bool:
 
     Returns False for unknown names so callers can pass
     arbitrary input without separate validation."""
-    backends = _get_backends()
-    if name not in backends:
+    if name not in _BACKENDS:
         return False
-    return backends[name].is_available()
+    return _BACKENDS[name].is_available()
 
 
 def enumerate_active_agents() -> list[ActiveAgent]:
@@ -704,11 +663,10 @@ def enumerate_active_agents() -> list[ActiveAgent]:
     deterministic tiebreaker. Agents with missing metadata
     (`started_at == ""`) sort first."""
     out: list[ActiveAgent] = []
-    backends = _get_backends()
-    for name in sorted(backends):
-        if not backends[name].is_available():
+    for name in known_backend_names():
+        if not has_backend(name):
             continue
-        out.extend(backends[name].enumerate_active())
+        out.extend(_BACKENDS[name].enumerate_active())
     out.sort(key=lambda a: (a.started_at, a.slug))
     return out
 

bot_bottle/backend/docker/util.py

@@ -7,9 +7,8 @@ from __future__ import annotations
 import re
 import shutil
 import subprocess
-from typing import Iterator
+from typing import Iterable, Iterator
 
-from ...docker_cmd import run_docker
 from ...log import die, info
 # from ...workspace import WorkspacePlan
 
@@ -31,7 +30,12 @@ def container_name_candidates(base: str) -> Iterator[str]:
 def runsc_available() -> bool:
     """Return True if the Docker daemon has the gVisor (`runsc`) runtime
     registered. Called once per prepare; the result lives on the plan."""
-    r = run_docker(["docker", "info", "--format", "{{json .Runtimes}}"])
+    r = subprocess.run(
+        ["docker", "info", "--format", "{{json .Runtimes}}"],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
     return r.returncode == 0 and "runsc" in r.stdout
 
 
@@ -45,15 +49,20 @@ def require_docker() -> None:
 
 
 def image_exists(ref: str) -> bool:
-    return run_docker(["docker", "image", "inspect", ref]).returncode == 0
+    return _silent_run(["docker", "image", "inspect", ref]) == 0
 
 
 def container_exists(name: str) -> bool:
     """Returns True if a container (running or stopped) with the given
     name exists. Uses `docker ps -a -q -f name=^<name>$` so substring
     matches don't false-positive."""
-    result = run_docker(["docker", "ps", "-a", "-q", "-f", f"name=^{name}$"])
-    return result.returncode == 0 and bool(result.stdout.strip())
+    result = subprocess.run(
+        ["docker", "ps", "-a", "-q", "-f", f"name=^{name}$"],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return bool(result.stdout.strip())
 
 
 def force_remove_container(name: str) -> None:
@@ -61,7 +70,12 @@ def force_remove_container(name: str) -> None:
     doesn't — and the rm itself is best-effort (errors swallowed) so
     this is safe to register as a teardown callback."""
     if container_exists(name):
-        run_docker(["docker", "rm", "-f", name])
+        subprocess.run(
+            ["docker", "rm", "-f", name],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            check=False,
+        )
 
 
 def docker_exec_root(container: str, argv: list[str]) -> None:
@@ -141,10 +155,22 @@ def build_image(ref: str, context: str, *, dockerfile: str = "") -> None:
 def commit_container(container_name: str, image_tag: str) -> None:
     """Run `docker commit <container_name> <image_tag>` to snapshot the
     running container's filesystem state as a local Docker image."""
-    result = run_docker(["docker", "commit", container_name, image_tag])
+    result = subprocess.run(
+        ["docker", "commit", container_name, image_tag],
+        capture_output=True, text=True, check=False,
+    )
     if result.returncode != 0:
         die(
             f"docker commit {container_name!r} → {image_tag!r} failed: "
             f"{(result.stderr or '').strip() or '<no stderr>'}"
         )
     info(f"committed {container_name!r} → {image_tag!r}")
+
+
+def _silent_run(cmd: Iterable[str]) -> int:
+    return subprocess.run(
+        list(cmd),
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        check=False,
+    ).returncode

bot_bottle/orchestrator/lifecycle.py

@@ -16,6 +16,7 @@ names + the published port).
 
 from __future__ import annotations
 
+import hashlib
 import time
 import urllib.error
 import urllib.request
@@ -29,6 +30,10 @@ from .gateway import GATEWAY_IMAGE, GATEWAY_NETWORK, DockerGateway
 DEFAULT_PORT = 8099
 ORCHESTRATOR_NAME = "bot-bottle-orchestrator"
 ORCHESTRATOR_LABEL = "bot-bottle-orchestrator=1"
+# Baked onto the container as a label so `ensure_running` can tell whether the
+# running process is executing the *current* bind-mounted source — see
+# `_source_hash`.
+ORCHESTRATOR_SOURCE_HASH_LABEL = "bot-bottle-orchestrator-source-hash"
 
 # The repo root is bind-mounted into the control-plane container so
 # `python -m bot_bottle.orchestrator` resolves the package (the orchestrator
@@ -46,6 +51,22 @@ class OrchestratorStartError(RuntimeError):
     """The orchestrator container did not become healthy within the timeout."""
 
 
+def _source_hash(repo_root: Path) -> str:
+    """Content hash of the orchestrator's bind-mounted Python source (the
+    `bot_bottle` package the control-plane process imports). This only
+    changes when the code that would actually run inside the container
+    changes — `ensure_running` recreates the container on a mismatch and
+    otherwise leaves a healthy one alone, so a bottle launch that isn't
+    accompanied by a code change doesn't restart the process and drop every
+    *other* active bottle's in-memory egress tokens (`Orchestrator._tokens`
+    in `service.py`, never persisted to disk by design)."""
+    h = hashlib.sha256()
+    for path in sorted((repo_root / "bot_bottle").rglob("*.py")):
+        h.update(str(path.relative_to(repo_root)).encode())
+        h.update(path.read_bytes())
+    return h.hexdigest()
+
+
 class OrchestratorService:
     """Manages the orchestrator control-plane container + the shared gateway.
     Callers only need `ensure_running()` + `url`."""
@@ -88,14 +109,17 @@ class OrchestratorService:
         proc = run_docker(["docker", "ps", "--filter", f"name=^/{name}$", "--format", "{{.Names}}"])
         return name in proc.stdout.split()
 
-    def _run_orchestrator_container(self) -> None:
+    def _run_orchestrator_container(self, source_hash: str) -> None:
         """Start the control-plane container (idempotent: clears a stale
-        fixed-name container first). Register-only broker → no docker socket."""
+        fixed-name container first). Register-only broker → no docker socket.
+        Labels the container with `source_hash` so a later `ensure_running`
+        can detect a real code change (see `_source_hash`)."""
         run_docker(["docker", "rm", "--force", ORCHESTRATOR_NAME])
         proc = run_docker([
             "docker", "run", "--detach",
             "--name", ORCHESTRATOR_NAME,
             "--label", ORCHESTRATOR_LABEL,
+            "--label", f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={source_hash}",
             "--network", self.network,
             # Host CLI reaches the control plane here; bound to loopback so it
             # is not exposed on the host's external interfaces.
@@ -119,26 +143,46 @@ class OrchestratorService:
     def _gateway(self) -> DockerGateway:
         return DockerGateway(self.image, network=self.network, orchestrator_url=self.internal_url)
 
+    def _orchestrator_source_current(self, current_hash: str) -> bool:
+        """True iff the running orchestrator container was created from the
+        *current* bind-mounted source. Mirrors `DockerGateway`'s
+        image-staleness check, but by content hash rather than image id since
+        the orchestrator runs bind-mounted source, not a built image."""
+        if not self._container_running(ORCHESTRATOR_NAME):
+            return False
+        proc = run_docker([
+            "docker", "inspect", "--format",
+            "{{ index .Config.Labels \"" + ORCHESTRATOR_SOURCE_HASH_LABEL + "\" }}",
+            ORCHESTRATOR_NAME,
+        ])
+        if proc.returncode != 0:
+            return True  # can't compare -> don't churn a working container
+        return proc.stdout.strip() == current_hash
+
     def ensure_running(
         self, *, startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,
     ) -> str:
         """Ensure the control plane + shared gateway are up; return the host
-        control-plane URL. Idempotent — a healthy control plane and a running
-        gateway are left untouched. Raises `OrchestratorStartError` on
-        timeout."""
+        control-plane URL. Idempotent — a healthy control plane running
+        current code and a running gateway are left untouched. Raises
+        `OrchestratorStartError` on timeout."""
         gateway = self._gateway()
         gateway.ensure_built()  # rebuild the bundle image on a source change
         gateway.ensure_running()  # creates the shared network + (re)starts gateway
 
-        # Always (re)create the orchestrator container. It runs the repo's code
-        # bind-mounted, but the Python process loaded that code at startup and
-        # won't reload — so reusing a healthy-but-stale container would keep
-        # running OLD control-plane code (e.g. dropping the tokens field). Cheap
-        # (~seconds); the registry DB persists and the current launch
-        # re-registers its own in-memory state. (The dedicated orchestrator
-        # image follow-up replaces this with image-staleness detection.)
+        # Recreate the orchestrator container only when its bind-mounted
+        # source has actually changed since it started — its Python process
+        # loaded that code at startup and won't reload, so a stale container
+        # would keep running OLD control-plane code. Recreating on *every*
+        # launch (the prior behaviour) would drop every other active
+        # bottle's in-memory egress tokens each time a new bottle starts,
+        # since the orchestrator process holds them only in memory (#381).
+        current_hash = _source_hash(self._repo_root)
+        if self.is_healthy() and self._orchestrator_source_current(current_hash):
+            return self.url
+
         log.info("starting orchestrator container", context={"name": ORCHESTRATOR_NAME})
-        self._run_orchestrator_container()
+        self._run_orchestrator_container(current_hash)
 
         deadline = time.monotonic() + startup_timeout
         while time.monotonic() < deadline:

tests/unit/test_backend_selection.py

@@ -40,7 +40,7 @@ class TestGetBottleBackend(unittest.TestCase):
                 return True
 
         with patch.dict(os.environ, {}, clear=True), \
-             patch.object(backend_mod, "_backends", {
+             patch.object(backend_mod, "_BACKENDS", {
                  "macos-container": _FakeBackend(),
                  "docker": _FakeBackend(),
              }):
@@ -61,7 +61,7 @@ class TestGetBottleBackend(unittest.TestCase):
         with patch.dict(os.environ, {}, clear=True), \
              patch.object(backend_mod.FirecrackerBottleBackend,
                           "is_host_capable", classmethod(lambda cls: False)), \
-             patch.object(backend_mod, "_backends", {
+             patch.object(backend_mod, "_BACKENDS", {
                  "macos-container": _FakeBackend("macos-container", False),
                  "docker": _FakeBackend("docker", True),
              }):
@@ -83,7 +83,7 @@ class TestGetBottleBackend(unittest.TestCase):
         with patch.dict(os.environ, {}, clear=True), \
              patch.object(backend_mod.FirecrackerBottleBackend,
                           "is_host_capable", classmethod(lambda cls: True)), \
-             patch.object(backend_mod, "_backends", {
+             patch.object(backend_mod, "_BACKENDS", {
                  "macos-container": _FakeBackend("macos-container", False),
                  "firecracker": _FakeBackend("firecracker", False),
                  "docker": _FakeBackend("docker", True),
@@ -133,7 +133,7 @@ class TestEnumerateActiveAgents(unittest.TestCase):
                 return self._items
 
         with patch.object(
-            backend_mod, "_backends",
+            backend_mod, "_BACKENDS",
             {"docker": _FakeBackend([a]), "firecracker": _FakeBackend([b])},
         ):
             self.assertEqual([a, b], enumerate_active_agents())
@@ -167,7 +167,7 @@ class TestEnumerateActiveAgents(unittest.TestCase):
                 return self._items
 
         with patch.object(
-            backend_mod, "_backends",
+            backend_mod, "_BACKENDS",
             {
                 "docker": _FakeBackend([newer, tie_b]),
                 "firecracker": _FakeBackend([missing_metadata, tie_a]),
@@ -187,7 +187,7 @@ class TestEnumerateActiveAgents(unittest.TestCase):
                 return []
 
         with patch.object(
-            backend_mod, "_backends",
+            backend_mod, "_BACKENDS",
             {"docker": _FakeBackend(), "firecracker": _FakeBackend()},
         ):
             self.assertEqual([], enumerate_active_agents())
@@ -218,7 +218,7 @@ class TestEnumerateActiveAgents(unittest.TestCase):
                 return self._items
 
         with patch.object(
-            backend_mod, "_backends",
+            backend_mod, "_BACKENDS",
             {
                 "docker": _FakeBackend([present], available=True),
                 "firecracker": _FakeBackend([hidden], available=False),
@@ -234,7 +234,7 @@ class TestHasBackend(unittest.TestCase):
                 return False
 
         with patch.object(
-            backend_mod, "_backends", {"docker": _FakeBackend()},
+            backend_mod, "_BACKENDS", {"docker": _FakeBackend()},
         ):
             from bot_bottle.backend import has_backend
             self.assertFalse(has_backend("docker"))

tests/unit/test_docker_util_image.py

@@ -29,7 +29,7 @@ def _fail(stderr: str = "boom") -> subprocess.CompletedProcess:  # type: ignore
 class TestCommitContainer(unittest.TestCase):
     def test_runs_docker_commit(self):
         with patch.object(
-            docker_mod, "run_docker", return_value=_ok(),
+            docker_mod.subprocess, "run", return_value=_ok(),
         ) as run, patch.object(docker_mod, "info"):
             docker_mod.commit_container(
                 "bot-bottle-dev-abc12",
@@ -47,7 +47,7 @@ class TestCommitContainer(unittest.TestCase):
 
     def test_dies_on_docker_commit_failure(self):
         with patch.object(
-            docker_mod, "run_docker", return_value=_fail("No such container"),
+            docker_mod.subprocess, "run", return_value=_fail("No such container"),
         ), patch.object(
             docker_mod, "die", side_effect=SystemExit("die"),
         ) as die:
@@ -58,7 +58,7 @@ class TestCommitContainer(unittest.TestCase):
 
     def test_die_message_includes_image_tag(self):
         with patch.object(
-            docker_mod, "run_docker", return_value=_fail("boom"),
+            docker_mod.subprocess, "run", return_value=_fail("boom"),
         ), patch.object(
             docker_mod, "die", side_effect=SystemExit("die"),
         ) as die:

tests/unit/test_orchestrator_lifecycle.py

@@ -10,8 +10,10 @@ from unittest.mock import MagicMock, Mock, patch
 
 from bot_bottle.orchestrator.lifecycle import (
     ORCHESTRATOR_NAME,
+    ORCHESTRATOR_SOURCE_HASH_LABEL,
     OrchestratorService,
     OrchestratorStartError,
+    _source_hash,
 )
 from tests.unit import use_bottle_root
 
@@ -28,6 +30,10 @@ def _health(status: int) -> MagicMock:
     return m
 
 
+def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock:
+    return Mock(returncode=returncode, stdout=stdout, stderr=stderr)
+
+
 class TestOrchestratorService(unittest.TestCase):
     def setUp(self) -> None:
         self._tmp = tempfile.TemporaryDirectory()
@@ -46,25 +52,67 @@ class TestOrchestratorService(unittest.TestCase):
         with patch(_URLOPEN, side_effect=urllib.error.URLError("refused")):
             self.assertFalse(self.svc.is_healthy())
 
-    def test_ensure_running_always_recreates_orchestrator(self) -> None:
-        # Even when a control plane is already healthy, the orchestrator is
-        # recreated so bind-mounted code changes take effect (its process
-        # won't reload). The gateway is ensured too.
-        run = Mock(return_value=Mock(returncode=0, stderr=""))
+    def test_ensure_running_noop_when_healthy_and_source_unchanged(self) -> None:
+        # A healthy control plane already running the *current* bind-mounted
+        # source is left alone — recreating it on every launch would drop
+        # every other active bottle's in-memory egress tokens (#381).
+        current = _source_hash(self.svc._repo_root)
+        calls: list[list[str]] = []
+
+        def fake(argv: list[str]) -> Mock:
+            calls.append(argv)
+            if argv[:2] == ["docker", "ps"]:
+                return _proc(stdout=ORCHESTRATOR_NAME)
+            if argv[:2] == ["docker", "inspect"]:
+                return _proc(stdout=current)
+            return _proc()
+
         with patch(_URLOPEN, return_value=_health(200)), \
-                patch(_GATEWAY) as gw_cls, patch(_RUN, run), patch(_SLEEP):
+                patch(_GATEWAY) as gw_cls, patch(_RUN, side_effect=fake), patch(_SLEEP):
             self.assertEqual(self.svc.url, self.svc.ensure_running())
         gw_cls.return_value.ensure_running.assert_called()   # gateway kept up
-        runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]]
-        self.assertEqual(1, len(runs))                       # orchestrator recreated
+        runs = [c for c in calls if c[:2] == ["docker", "run"]]
+        rms = [c for c in calls if c[:3] == ["docker", "rm", "--force"] and ORCHESTRATOR_NAME in c]
+        self.assertEqual([], runs)   # not recreated
+        self.assertEqual([], rms)
+
+    def test_ensure_running_recreates_when_source_changed(self) -> None:
+        # Healthy, but the running container's label doesn't match the
+        # current source hash (a real code change) — recreate so it takes
+        # effect, same as the gateway's image-staleness check.
+        calls: list[list[str]] = []
+
+        def fake(argv: list[str]) -> Mock:
+            calls.append(argv)
+            if argv[:2] == ["docker", "ps"]:
+                return _proc(stdout=ORCHESTRATOR_NAME)
+            if argv[:2] == ["docker", "inspect"]:
+                return _proc(stdout="stale-hash")
+            return _proc()
+
+        with patch(_URLOPEN, return_value=_health(200)), \
+                patch(_GATEWAY), patch(_RUN, side_effect=fake), patch(_SLEEP):
+            self.assertEqual(self.svc.url, self.svc.ensure_running())
+        runs = [c for c in calls if c[:2] == ["docker", "run"]]
+        self.assertEqual(1, len(runs))
         self.assertIn(ORCHESTRATOR_NAME, runs[0])
+        # the fresh container is labeled with the current hash, not the stale one
+        current = _source_hash(self.svc._repo_root)
+        self.assertIn(f"{ORCHESTRATOR_SOURCE_HASH_LABEL}={current}", runs[0])
 
     def test_ensure_running_starts_orchestrator_container_when_absent(self) -> None:
-        run = Mock(return_value=Mock(returncode=0, stderr=""))
+        calls: list[list[str]] = []
+
+        def fake(argv: list[str]) -> Mock:
+            calls.append(argv)
+            if argv[:2] == ["docker", "ps"]:
+                return _proc(stdout="")   # not running
+            return _proc()
+
         with patch(_URLOPEN, side_effect=[urllib.error.URLError("down"), _health(200)]), \
-                patch(_GATEWAY), patch(_RUN, run), patch(_SLEEP):
+                patch(_GATEWAY), patch(_RUN, side_effect=fake), patch(_SLEEP):
             self.assertEqual(self.svc.url, self.svc.ensure_running())
-        runs = [c.args[0] for c in run.call_args_list if c.args[0][:2] == ["docker", "run"]]
+        runs = [c for c in calls if c[:2] == ["docker", "run"]]
         self.assertEqual(1, len(runs))
         argv = runs[0]
         self.assertIn(ORCHESTRATOR_NAME, argv)