refactor(macos): one infra container (control plane + gateway), fixes shared-DB races
Adopts the firecracker infra-VM pattern for macOS: the orchestrator control plane and the gateway data plane now run in a SINGLE Apple container instead of two. Apple Containers are lightweight VMs with separate kernels, so the prior two-container design had both guests writing one bot-bottle.db over virtiofs, where fcntl locks are not coherent across kernels — concurrent writes (the orchestrator's registry vs the gateway supervise daemon's queue) could corrupt it. One container = one kernel = coherent locking. The DB moves onto a container-only Apple volume (bot-bottle-mac-db), never bind-mounted from the host, so no host process opens the live file either. The host CLI already reaches registry + supervise state over the control-plane HTTP surface (cli/supervise.py uses OrchestratorClient), exactly as firecracker's VM-only DB requires. Two simplifications fall out of the single container: - No DNS dance: the control plane and gateway daemons reach each other over 127.0.0.1, so the orchestrator-before-gateway ordering (a workaround for Apple having no container DNS) is gone, along with the moved-IP recreate logic it needed. - Net -243 lines. Mechanics: the infra container runs from the gateway image with the control-plane source bind-mounted read-only (like the docker orchestrator, so a code change needs no rebuild) and a small sh -c init that starts both processes (mirrors firecracker's _infra_init). Also implements the macOS backend's ensure_orchestrator() and adds it to discover_orchestrator_url, so operator tools (supervise) can bring up / find the control plane on demand — previously the macOS backend died with "no orchestrator control plane". Verified end-to-end on real Apple Container 1.0.0: the single infra container comes up healthy (one address for control plane + gateway), both processes run, the DB is written on the container-only volume, host-side supervise works over HTTP, and a registered agent gets 200 for an allowed host / 403 for a denied one. 1824 unit tests pass with `container` absent (CI parity), pyright clean, pylint 9.89. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -247,7 +247,7 @@ class TestHasBackend(unittest.TestCase):
|
||||
class TestEnsureOrchestrator(unittest.TestCase):
|
||||
"""The backend-agnostic orchestrator bring-up entry point. Docker starts
|
||||
the orchestrator + gateway containers; firecracker boots the infra VM;
|
||||
backends without one (macos-container) die with a pointer."""
|
||||
macos-container starts the infra container."""
|
||||
|
||||
def test_docker_delegates_to_orchestrator_service(self):
|
||||
b = get_bottle_backend("docker")
|
||||
@@ -272,11 +272,16 @@ class TestEnsureOrchestrator(unittest.TestCase):
|
||||
url = b.ensure_orchestrator()
|
||||
self.assertEqual(url, "http://10.243.255.1:8099")
|
||||
|
||||
def test_macos_default_dies(self):
|
||||
from bot_bottle.log import Die
|
||||
def test_macos_delegates_to_infra_container(self):
|
||||
b = get_bottle_backend("macos-container")
|
||||
with self.assertRaises(Die):
|
||||
b.ensure_orchestrator()
|
||||
with patch(
|
||||
"bot_bottle.backend.macos_container.infra.MacosInfraService"
|
||||
) as service_cls:
|
||||
service_cls.return_value.ensure_running.return_value.control_plane_url = (
|
||||
"http://192.168.128.2:8099"
|
||||
)
|
||||
url = b.ensure_orchestrator()
|
||||
self.assertEqual(url, "http://192.168.128.2:8099")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -50,30 +50,35 @@ def _client() -> Mock:
|
||||
|
||||
class TestEnsureGateway(unittest.TestCase):
|
||||
def _run(self, service: MagicMock) -> GatewayEndpoint:
|
||||
with patch(f"{_MOD}.MacosOrchestratorService", return_value=service):
|
||||
with patch(f"{_MOD}.MacosInfraService", return_value=service):
|
||||
return ensure_gateway()
|
||||
|
||||
def _service(self) -> MagicMock:
|
||||
from bot_bottle.backend.macos_container.infra import InfraEndpoint
|
||||
service = MagicMock()
|
||||
service.ensure_running.return_value = "http://192.168.128.2:8099"
|
||||
service.ensure_running.return_value = InfraEndpoint(
|
||||
control_plane_url="http://192.168.128.2:8099",
|
||||
gateway_ip="192.168.128.2",
|
||||
)
|
||||
service.network = "bot-bottle-mac-gateway"
|
||||
service.gateway.return_value.ip_on_shared_network.return_value = "192.168.128.3"
|
||||
service.gateway.return_value.ca_cert_pem.return_value = "PEM"
|
||||
service.ca_cert_pem.return_value = "PEM"
|
||||
return service
|
||||
|
||||
def test_reports_gateway_endpoint(self) -> None:
|
||||
endpoint = self._run(self._service())
|
||||
self.assertEqual("http://192.168.128.2:8099", endpoint.orchestrator_url)
|
||||
self.assertEqual("192.168.128.3", endpoint.gateway_ip)
|
||||
self.assertEqual("192.168.128.2", endpoint.gateway_ip)
|
||||
self.assertEqual("PEM", endpoint.gateway_ca_pem)
|
||||
self.assertEqual("bot-bottle-mac-gateway", endpoint.network)
|
||||
|
||||
def test_gateway_is_pointed_at_the_resolved_control_plane(self) -> None:
|
||||
"""Apple has no container DNS, so the gateway must be handed the
|
||||
control plane's *resolved URL* rather than a container name."""
|
||||
service = self._service()
|
||||
self._run(service)
|
||||
service.gateway.assert_called_with("http://192.168.128.2:8099")
|
||||
def test_control_plane_and_gateway_share_one_address(self) -> None:
|
||||
"""One infra container hosts both, so the gateway IP and the
|
||||
control-plane host are the same."""
|
||||
endpoint = self._run(self._service())
|
||||
self.assertEqual(
|
||||
endpoint.gateway_ip,
|
||||
endpoint.orchestrator_url.split("://")[1].split(":")[0],
|
||||
)
|
||||
|
||||
|
||||
class TestRegisterAgent(unittest.TestCase):
|
||||
|
||||
@@ -60,13 +60,10 @@ class TestMacosContainerEnumerate(unittest.TestCase):
|
||||
self.assertEqual(["dev-abc"], [a.slug for a in agents])
|
||||
self.assertEqual(["macos-container"], [a.backend_name for a in agents])
|
||||
|
||||
def test_excludes_the_shared_singletons(self):
|
||||
"""The gateway and control plane share the bot-bottle- prefix but are
|
||||
infrastructure — listing them would invent an agent per host."""
|
||||
agents = self._enumerate(
|
||||
"bot-bottle-mac-gateway\nbot-bottle-mac-orchestrator\n"
|
||||
"bot-bottle-dev-abc\n"
|
||||
)
|
||||
def test_excludes_the_infra_singleton(self):
|
||||
"""The infra container shares the bot-bottle- prefix but is
|
||||
infrastructure — listing it would invent an agent per host."""
|
||||
agents = self._enumerate("bot-bottle-mac-infra\nbot-bottle-dev-abc\n")
|
||||
self.assertEqual(["dev-abc"], [a.slug for a in agents])
|
||||
|
||||
def test_empty_when_the_cli_fails(self):
|
||||
|
||||
@@ -1,297 +0,0 @@
|
||||
"""Unit: the Apple gateway + orchestrator lifecycle (PRD 0070)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from bot_bottle.backend.macos_container.gateway import AppleGateway, GatewayError
|
||||
from bot_bottle.backend.macos_container.orchestrator_service import (
|
||||
MacosOrchestratorService,
|
||||
OrchestratorStartError,
|
||||
)
|
||||
|
||||
_GW = "bot_bottle.backend.macos_container.gateway"
|
||||
_ORCH = "bot_bottle.backend.macos_container.orchestrator_service"
|
||||
|
||||
|
||||
def _ok(stdout: str = "") -> Mock:
|
||||
return Mock(returncode=0, stdout=stdout, stderr="")
|
||||
|
||||
|
||||
def _fail(stderr: str = "boom") -> Mock:
|
||||
return Mock(returncode=1, stdout="", stderr=stderr)
|
||||
|
||||
|
||||
class TestAppleGatewayRun(unittest.TestCase):
|
||||
def _argv(self, run: Mock) -> list[str]:
|
||||
return run.call_args.args[0]
|
||||
|
||||
def _start(self, run: Mock) -> None:
|
||||
with patch(f"{_GW}.container_mod") as mod:
|
||||
mod.container_is_running.return_value = False
|
||||
mod.dns_server.return_value = "1.1.1.1"
|
||||
mod.run_container_argv = run
|
||||
AppleGateway(orchestrator_url="http://192.168.128.2:8099").ensure_running()
|
||||
|
||||
def test_nat_network_precedes_the_host_only_network(self) -> None:
|
||||
"""Apple Container makes the FIRST --network the default route, so the
|
||||
NAT network must lead or the gateway has no route out."""
|
||||
run = Mock(return_value=_ok())
|
||||
self._start(run)
|
||||
argv = self._argv(run)
|
||||
networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
||||
self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], networks)
|
||||
|
||||
def test_control_plane_url_is_passed_for_multi_tenancy(self) -> None:
|
||||
run = Mock(return_value=_ok())
|
||||
self._start(run)
|
||||
self.assertIn(
|
||||
"BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.2:8099", self._argv(run),
|
||||
)
|
||||
|
||||
def test_dns_is_explicit(self) -> None:
|
||||
"""The NAT gateway routes but does not resolve."""
|
||||
run = Mock(return_value=_ok())
|
||||
self._start(run)
|
||||
argv = self._argv(run)
|
||||
self.assertEqual("1.1.1.1", argv[argv.index("--dns") + 1])
|
||||
|
||||
def test_start_failure_raises(self) -> None:
|
||||
with self.assertRaises(GatewayError):
|
||||
self._start(Mock(return_value=_fail()))
|
||||
|
||||
def test_running_current_gateway_is_left_alone(self) -> None:
|
||||
"""Idempotent singleton: N launches must not restart the gateway and
|
||||
drop every other bottle's data plane."""
|
||||
run = Mock(return_value=_ok())
|
||||
with patch(f"{_GW}.container_mod") as mod:
|
||||
mod.container_is_running.return_value = True
|
||||
mod.container_image_digest.return_value = "abc"
|
||||
mod.image_digest.return_value = "abc"
|
||||
mod.run_container_argv = run
|
||||
AppleGateway().ensure_running()
|
||||
run.assert_not_called()
|
||||
|
||||
def test_stale_image_forces_a_recreate(self) -> None:
|
||||
"""A rebuilt image only takes effect if the running container is
|
||||
replaced — otherwise it keeps serving the OLD daemons."""
|
||||
run = Mock(return_value=_ok())
|
||||
with patch(f"{_GW}.container_mod") as mod:
|
||||
mod.container_is_running.return_value = True
|
||||
mod.container_image_digest.return_value = "old"
|
||||
mod.image_digest.return_value = "new"
|
||||
mod.dns_server.return_value = "1.1.1.1"
|
||||
mod.run_container_argv = run
|
||||
AppleGateway().ensure_running()
|
||||
run.assert_called_once()
|
||||
|
||||
def test_unreadable_digest_does_not_churn(self) -> None:
|
||||
run = Mock(return_value=_ok())
|
||||
with patch(f"{_GW}.container_mod") as mod:
|
||||
mod.container_is_running.return_value = True
|
||||
mod.container_image_digest.return_value = ""
|
||||
mod.image_digest.return_value = ""
|
||||
mod.run_container_argv = run
|
||||
AppleGateway().ensure_running()
|
||||
run.assert_not_called()
|
||||
|
||||
def _start_with_running_env(self, env: dict[str, str], url: str) -> Mock:
|
||||
run = Mock(return_value=_ok())
|
||||
with patch(f"{_GW}.container_mod") as mod:
|
||||
mod.container_is_running.return_value = True
|
||||
mod.container_image_digest.return_value = "abc"
|
||||
mod.image_digest.return_value = "abc"
|
||||
mod.container_env.return_value = env
|
||||
mod.dns_server.return_value = "1.1.1.1"
|
||||
mod.run_container_argv = run
|
||||
AppleGateway(orchestrator_url=url).ensure_running()
|
||||
return run
|
||||
|
||||
def test_moved_control_plane_forces_a_recreate(self) -> None:
|
||||
"""Docker hands the gateway a container *name*, stable across an
|
||||
orchestrator recreate. Apple has no DNS, so the URL is an IP baked into
|
||||
the gateway's env — if the orchestrator comes back on a new address and
|
||||
the gateway isn't recreated, every /resolve fails and every bottle on
|
||||
the host loses egress."""
|
||||
run = self._start_with_running_env(
|
||||
{"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"},
|
||||
"http://192.168.128.7:8099",
|
||||
)
|
||||
run.assert_called_once()
|
||||
self.assertIn(
|
||||
"BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.7:8099",
|
||||
run.call_args.args[0],
|
||||
)
|
||||
|
||||
def test_unmoved_control_plane_does_not_churn(self) -> None:
|
||||
run = self._start_with_running_env(
|
||||
{"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"},
|
||||
"http://192.168.128.2:8099",
|
||||
)
|
||||
run.assert_not_called()
|
||||
|
||||
def test_unreadable_env_does_not_churn(self) -> None:
|
||||
run = self._start_with_running_env({}, "http://192.168.128.2:8099")
|
||||
run.assert_not_called()
|
||||
|
||||
|
||||
class TestMacosOrchestratorService(unittest.TestCase):
|
||||
def test_orchestrator_starts_before_the_gateway(self) -> None:
|
||||
"""Apple has no container DNS, so the gateway can only be handed the
|
||||
control plane's IP — which does not exist until it is running. This
|
||||
ordering is the whole reason the macOS service diverges from docker's."""
|
||||
order: list[str] = []
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
gateway = Mock()
|
||||
gateway.ensure_running.side_effect = lambda: order.append("gateway")
|
||||
|
||||
def _record_orchestrator(_hash: str) -> None:
|
||||
order.append("orchestrator")
|
||||
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
||||
patch.object(svc, "_run_orchestrator_container",
|
||||
side_effect=_record_orchestrator), \
|
||||
patch.object(svc, "gateway", return_value=gateway), \
|
||||
patch.object(svc, "is_healthy", return_value=True):
|
||||
mod.container_is_running.return_value = False
|
||||
mod.image_exists.return_value = True
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
url = svc.ensure_running()
|
||||
self.assertEqual(["orchestrator", "gateway"], order)
|
||||
self.assertEqual("http://192.168.128.2:8099", url)
|
||||
|
||||
def test_gateway_is_handed_the_resolved_url(self) -> None:
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
||||
patch.object(svc, "_run_orchestrator_container"), \
|
||||
patch.object(svc, "gateway") as gw, \
|
||||
patch.object(svc, "is_healthy", return_value=True):
|
||||
mod.container_is_running.return_value = False
|
||||
mod.image_exists.return_value = True
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
svc.ensure_running()
|
||||
gw.assert_called_with("http://192.168.128.2:8099")
|
||||
|
||||
def test_current_source_leaves_a_healthy_orchestrator_alone(self) -> None:
|
||||
"""Recreating on every launch would drop every other live bottle's
|
||||
in-memory egress tokens (#381)."""
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
run = Mock()
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
||||
patch.object(svc, "_run_orchestrator_container", run), \
|
||||
patch.object(svc, "gateway"), \
|
||||
patch.object(svc, "is_healthy", return_value=True):
|
||||
mod.container_is_running.return_value = True
|
||||
mod.inspect_container.return_value = {
|
||||
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
|
||||
}
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
svc.ensure_running()
|
||||
run.assert_not_called()
|
||||
|
||||
def test_wedged_but_current_orchestrator_is_recreated(self) -> None:
|
||||
"""A container running the current code but whose HTTP server is dead
|
||||
must be recreated, not left alone and polled to death forever. Checking
|
||||
the source-hash label alone (without health) would strand the host."""
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
run = Mock()
|
||||
# Unhealthy on the pre-check, healthy once recreated + waited.
|
||||
health = Mock(side_effect=[False, True])
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
||||
patch.object(svc, "_run_orchestrator_container", run), \
|
||||
patch.object(svc, "_ensure_orchestrator_image"), \
|
||||
patch.object(svc, "gateway"), \
|
||||
patch.object(svc, "is_healthy", health):
|
||||
mod.container_is_running.return_value = True
|
||||
mod.inspect_container.return_value = {
|
||||
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
|
||||
}
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
svc.ensure_running()
|
||||
run.assert_called_once()
|
||||
|
||||
def test_changed_source_recreates_the_orchestrator(self) -> None:
|
||||
"""The control-plane process loaded its bind-mounted source at startup
|
||||
and won't reload it."""
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
run = Mock()
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.source_hash", return_value="h2"), \
|
||||
patch.object(svc, "_run_orchestrator_container", run), \
|
||||
patch.object(svc, "gateway"), \
|
||||
patch.object(svc, "is_healthy", return_value=True):
|
||||
mod.container_is_running.return_value = True
|
||||
mod.inspect_container.return_value = {
|
||||
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
|
||||
}
|
||||
mod.image_exists.return_value = True
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
svc.ensure_running()
|
||||
run.assert_called_once()
|
||||
|
||||
def test_never_healthy_raises(self) -> None:
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.source_hash", return_value="h1"), \
|
||||
patch.object(svc, "_run_orchestrator_container"), \
|
||||
patch.object(svc, "is_healthy", return_value=False):
|
||||
mod.container_is_running.return_value = False
|
||||
mod.image_exists.return_value = True
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
with self.assertRaises(OrchestratorStartError):
|
||||
svc.ensure_running(startup_timeout=0.01)
|
||||
|
||||
def test_control_plane_needs_no_route_out(self) -> None:
|
||||
"""The orchestrator sits only on the host-only network: the host
|
||||
reaches it there directly, so there is no --publish and no NAT leg."""
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
run = Mock(return_value=_ok())
|
||||
# `ensure_networks` lives in the gateway module and shells out to the
|
||||
# `container` CLI, which does not exist on the Linux CI host — patch it
|
||||
# here, not gateway.container_mod, since it is called through this
|
||||
# module's imported name.
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.ensure_networks"):
|
||||
mod.run_container_argv = run
|
||||
svc._run_orchestrator_container("h1")
|
||||
argv = run.call_args.args[0]
|
||||
networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
||||
self.assertEqual(["bot-bottle-mac-gateway"], networks)
|
||||
self.assertNotIn("--publish", argv)
|
||||
|
||||
def test_networks_exist_before_the_orchestrator_runs(self) -> None:
|
||||
"""The orchestrator is the first container on the shared network, so it
|
||||
has to create it — the gateway that used to do so now starts second."""
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
order: list[str] = []
|
||||
|
||||
def _networks(*_args: str) -> None:
|
||||
order.append("networks")
|
||||
|
||||
def _run(*_args: list[str]) -> Mock:
|
||||
order.append("run")
|
||||
return _ok()
|
||||
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.ensure_networks", side_effect=_networks):
|
||||
mod.run_container_argv = Mock(side_effect=_run)
|
||||
svc._run_orchestrator_container("h1")
|
||||
self.assertEqual(["networks", "run"], order)
|
||||
|
||||
def test_orchestrator_start_failure_raises(self) -> None:
|
||||
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
|
||||
with patch(f"{_ORCH}.container_mod") as mod, \
|
||||
patch(f"{_ORCH}.ensure_networks"):
|
||||
mod.run_container_argv = Mock(return_value=_fail())
|
||||
with self.assertRaises(OrchestratorStartError):
|
||||
svc._run_orchestrator_container("h1")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,168 @@
|
||||
"""Unit: the single macOS infra container (control plane + gateway, PRD 0070)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from bot_bottle.backend.macos_container.infra import (
|
||||
INFRA_DB_VOLUME,
|
||||
MacosInfraService,
|
||||
OrchestratorStartError,
|
||||
probe_control_plane_url,
|
||||
)
|
||||
|
||||
_INFRA = "bot_bottle.backend.macos_container.infra"
|
||||
|
||||
|
||||
def _ok(stdout: str = "") -> Mock:
|
||||
return Mock(returncode=0, stdout=stdout, stderr="")
|
||||
|
||||
|
||||
def _fail(stderr: str = "boom") -> Mock:
|
||||
return Mock(returncode=1, stdout="", stderr=stderr)
|
||||
|
||||
|
||||
class TestInfraRun(unittest.TestCase):
|
||||
def _run_container(self, svc: MacosInfraService) -> list[str]:
|
||||
run = Mock(return_value=_ok())
|
||||
|
||||
def _spec(src: str, tgt: str, readonly: bool = False) -> str:
|
||||
return f"type=bind,source={src},target={tgt}" + (
|
||||
",readonly" if readonly else "")
|
||||
|
||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||
patch(f"{_INFRA}.ensure_networks"):
|
||||
mod.dns_server.return_value = "1.1.1.1"
|
||||
mod.bind_mount_spec.side_effect = _spec
|
||||
mod.run_container_argv = run
|
||||
svc._run_container("h1")
|
||||
return run.call_args.args[0]
|
||||
|
||||
def test_single_container_runs_both_processes(self) -> None:
|
||||
"""The whole point: one container starts the control plane AND the
|
||||
gateway daemons, so one kernel owns the DB."""
|
||||
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
||||
script = argv[-1]
|
||||
self.assertIn("bot_bottle.orchestrator", script)
|
||||
self.assertIn("gateway_init.py", script)
|
||||
self.assertIn("127.0.0.1", script) # they reach each other on loopback
|
||||
|
||||
def test_db_is_a_container_only_volume(self) -> None:
|
||||
"""No host bind-mount of the DB — a named volume only this container
|
||||
mounts, so the DB is never written by two kernels."""
|
||||
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
||||
vols = [argv[i + 1] for i, a in enumerate(argv) if a == "--volume"]
|
||||
self.assertTrue(any(v.startswith(f"{INFRA_DB_VOLUME}:") for v in vols))
|
||||
# The repo source is bind-mounted read-only; the DB is not a bind mount.
|
||||
mounts = [argv[i + 1] for i, a in enumerate(argv) if a == "--mount"]
|
||||
self.assertTrue(all("bot-bottle.db" not in m for m in mounts))
|
||||
|
||||
def test_nat_network_precedes_the_host_only_network(self) -> None:
|
||||
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
||||
nets = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
|
||||
self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], nets)
|
||||
|
||||
def test_source_hash_is_labelled_for_recreate(self) -> None:
|
||||
argv = self._run_container(MacosInfraService(repo_root=Path("/r")))
|
||||
self.assertIn("BOT_BOTTLE_SOURCE_HASH=h1", argv)
|
||||
|
||||
def test_start_failure_raises(self) -> None:
|
||||
svc = MacosInfraService(repo_root=Path("/r"))
|
||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||
patch(f"{_INFRA}.ensure_networks"):
|
||||
mod.dns_server.return_value = "1.1.1.1"
|
||||
mod.bind_mount_spec.return_value = "m"
|
||||
mod.run_container_argv = Mock(return_value=_fail())
|
||||
with self.assertRaises(OrchestratorStartError):
|
||||
svc._run_container("h1")
|
||||
|
||||
|
||||
class TestInfraEnsureRunning(unittest.TestCase):
|
||||
def test_current_healthy_container_is_left_alone(self) -> None:
|
||||
"""Idempotent singleton: N launches must not churn the infra container
|
||||
and drop every live bottle's control plane."""
|
||||
svc = MacosInfraService(repo_root=Path("/r"))
|
||||
run = Mock()
|
||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
||||
patch.object(svc, "_run_container", run), \
|
||||
patch.object(svc, "is_healthy", return_value=True):
|
||||
mod.container_is_running.return_value = True
|
||||
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
endpoint = svc.ensure_running()
|
||||
run.assert_not_called()
|
||||
self.assertEqual("http://192.168.128.2:8099", endpoint.control_plane_url)
|
||||
self.assertEqual("192.168.128.2", endpoint.gateway_ip)
|
||||
|
||||
def test_changed_source_recreates(self) -> None:
|
||||
svc = MacosInfraService(repo_root=Path("/r"))
|
||||
run = Mock()
|
||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||
patch(f"{_INFRA}.source_hash", return_value="h2"), \
|
||||
patch.object(svc, "ensure_built"), \
|
||||
patch.object(svc, "_run_container", run), \
|
||||
patch.object(svc, "is_healthy", return_value=True):
|
||||
mod.container_is_running.return_value = True
|
||||
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
svc.ensure_running()
|
||||
run.assert_called_once()
|
||||
|
||||
def test_wedged_but_current_container_is_recreated(self) -> None:
|
||||
"""Current source but a dead HTTP server must be recreated, not polled
|
||||
to death forever — health, not just the source label, gates reuse."""
|
||||
svc = MacosInfraService(repo_root=Path("/r"))
|
||||
run = Mock()
|
||||
health = Mock(side_effect=[False, True])
|
||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
||||
patch.object(svc, "ensure_built"), \
|
||||
patch.object(svc, "_run_container", run), \
|
||||
patch.object(svc, "is_healthy", health):
|
||||
mod.container_is_running.return_value = True
|
||||
mod.container_env.return_value = {"BOT_BOTTLE_SOURCE_HASH": "h1"}
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
svc.ensure_running()
|
||||
run.assert_called_once()
|
||||
|
||||
def test_never_healthy_raises(self) -> None:
|
||||
svc = MacosInfraService(repo_root=Path("/r"))
|
||||
with patch(f"{_INFRA}.container_mod") as mod, \
|
||||
patch(f"{_INFRA}.source_hash", return_value="h1"), \
|
||||
patch.object(svc, "ensure_built"), \
|
||||
patch.object(svc, "_run_container"), \
|
||||
patch.object(svc, "is_healthy", return_value=False):
|
||||
mod.container_is_running.return_value = False
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
with self.assertRaises(OrchestratorStartError):
|
||||
svc.ensure_running(startup_timeout=0.01)
|
||||
|
||||
|
||||
class TestCaCertPem(unittest.TestCase):
|
||||
def test_reads_ca_out_of_the_container(self) -> None:
|
||||
svc = MacosInfraService(repo_root=Path("/r"))
|
||||
with patch(f"{_INFRA}.container_mod") as mod:
|
||||
mod.run_container_argv.return_value = _ok("-----BEGIN CERTIFICATE-----\n")
|
||||
pem = svc.ca_cert_pem()
|
||||
self.assertTrue(pem.startswith("-----BEGIN CERTIFICATE-----"))
|
||||
argv = mod.run_container_argv.call_args.args[0]
|
||||
self.assertEqual(["container", "exec", "bot-bottle-mac-infra", "cat"], argv[:4])
|
||||
|
||||
|
||||
class TestProbeControlPlane(unittest.TestCase):
|
||||
def test_returns_url_when_running(self) -> None:
|
||||
with patch(f"{_INFRA}.container_mod") as mod:
|
||||
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
|
||||
self.assertEqual("http://192.168.128.2:8099", probe_control_plane_url())
|
||||
|
||||
def test_empty_when_absent(self) -> None:
|
||||
with patch(f"{_INFRA}.container_mod") as mod:
|
||||
mod.try_container_ipv4_on_network.return_value = ""
|
||||
self.assertEqual("", probe_control_plane_url())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user