didericis/bot-bottle
1
0
Fork 0
Code Issues 32 Pull Requests 8 Actions Packages Releases Wiki Activity

refactor(macos): one infra container (control plane + gateway), fixes shared-DB races
lint / lint (push) Successful in 2m15s
Details
test / unit (pull_request) Successful in 1m16s
Details
test / integration (pull_request) Successful in 23s
Details
test / coverage (pull_request) Successful in 1m17s
Details

Browse Source 
Adopts the firecracker infra-VM pattern for macOS: the orchestrator control
plane and the gateway data plane now run in a SINGLE Apple container instead of
two. Apple Containers are lightweight VMs with separate kernels, so the prior
two-container design had both guests writing one bot-bottle.db over virtiofs,
where fcntl locks are not coherent across kernels — concurrent writes (the
orchestrator's registry vs the gateway supervise daemon's queue) could corrupt
it. One container = one kernel = coherent locking.

The DB moves onto a container-only Apple volume (bot-bottle-mac-db), never
bind-mounted from the host, so no host process opens the live file either. The
host CLI already reaches registry + supervise state over the control-plane HTTP
surface (cli/supervise.py uses OrchestratorClient), exactly as firecracker's
VM-only DB requires.

Two simplifications fall out of the single container:
- No DNS dance: the control plane and gateway daemons reach each other over
  127.0.0.1, so the orchestrator-before-gateway ordering (a workaround for
  Apple having no container DNS) is gone, along with the moved-IP recreate
  logic it needed.
- Net -243 lines.

Mechanics: the infra container runs from the gateway image with the
control-plane source bind-mounted read-only (like the docker orchestrator, so a
code change needs no rebuild) and a small sh -c init that starts both processes
(mirrors firecracker's _infra_init). Also implements the macOS backend's
ensure_orchestrator() and adds it to discover_orchestrator_url, so operator
tools (supervise) can bring up / find the control plane on demand — previously
the macOS backend died with "no orchestrator control plane".

Verified end-to-end on real Apple Container 1.0.0: the single infra container
comes up healthy (one address for control plane + gateway), both processes run,
the DB is written on the container-only volume, host-side supervise works over
HTTP, and a registered agent gets 200 for an allowed host / 403 for a denied
one. 1824 unit tests pass with `container` absent (CI parity), pyright clean,
pylint 9.89.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
didericis
2026-07-17 04:14:14 -04:00
parent e24b62b6b9
commit 4a607ad098
13 changed files with 549 additions and 792 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_macos_gateway.py
-297
View File
@@ -1,297 +0,0 @@
"""Unit: the Apple gateway + orchestrator lifecycle (PRD 0070)."""
from __future__ import annotations
import unittest
from pathlib import Path
from unittest.mock import Mock, patch
from bot_bottle.backend.macos_container.gateway import AppleGateway, GatewayError
from bot_bottle.backend.macos_container.orchestrator_service import (
MacosOrchestratorService,
OrchestratorStartError,
)
_GW = "bot_bottle.backend.macos_container.gateway"
_ORCH = "bot_bottle.backend.macos_container.orchestrator_service"
def _ok(stdout: str = "") -> Mock:
return Mock(returncode=0, stdout=stdout, stderr="")
def _fail(stderr: str = "boom") -> Mock:
return Mock(returncode=1, stdout="", stderr=stderr)
class TestAppleGatewayRun(unittest.TestCase):
def _argv(self, run: Mock) -> list[str]:
return run.call_args.args[0]
def _start(self, run: Mock) -> None:
with patch(f"{_GW}.container_mod") as mod:
mod.container_is_running.return_value = False
mod.dns_server.return_value = "1.1.1.1"
mod.run_container_argv = run
AppleGateway(orchestrator_url="http://192.168.128.2:8099").ensure_running()
def test_nat_network_precedes_the_host_only_network(self) -> None:
"""Apple Container makes the FIRST --network the default route, so the
NAT network must lead or the gateway has no route out."""
run = Mock(return_value=_ok())
self._start(run)
argv = self._argv(run)
networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
self.assertEqual(["bot-bottle-mac-egress", "bot-bottle-mac-gateway"], networks)
def test_control_plane_url_is_passed_for_multi_tenancy(self) -> None:
run = Mock(return_value=_ok())
self._start(run)
self.assertIn(
"BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.2:8099", self._argv(run),
)
def test_dns_is_explicit(self) -> None:
"""The NAT gateway routes but does not resolve."""
run = Mock(return_value=_ok())
self._start(run)
argv = self._argv(run)
self.assertEqual("1.1.1.1", argv[argv.index("--dns") + 1])
def test_start_failure_raises(self) -> None:
with self.assertRaises(GatewayError):
self._start(Mock(return_value=_fail()))
def test_running_current_gateway_is_left_alone(self) -> None:
"""Idempotent singleton: N launches must not restart the gateway and
drop every other bottle's data plane."""
run = Mock(return_value=_ok())
with patch(f"{_GW}.container_mod") as mod:
mod.container_is_running.return_value = True
mod.container_image_digest.return_value = "abc"
mod.image_digest.return_value = "abc"
mod.run_container_argv = run
AppleGateway().ensure_running()
run.assert_not_called()
def test_stale_image_forces_a_recreate(self) -> None:
"""A rebuilt image only takes effect if the running container is
replaced — otherwise it keeps serving the OLD daemons."""
run = Mock(return_value=_ok())
with patch(f"{_GW}.container_mod") as mod:
mod.container_is_running.return_value = True
mod.container_image_digest.return_value = "old"
mod.image_digest.return_value = "new"
mod.dns_server.return_value = "1.1.1.1"
mod.run_container_argv = run
AppleGateway().ensure_running()
run.assert_called_once()
def test_unreadable_digest_does_not_churn(self) -> None:
run = Mock(return_value=_ok())
with patch(f"{_GW}.container_mod") as mod:
mod.container_is_running.return_value = True
mod.container_image_digest.return_value = ""
mod.image_digest.return_value = ""
mod.run_container_argv = run
AppleGateway().ensure_running()
run.assert_not_called()
def _start_with_running_env(self, env: dict[str, str], url: str) -> Mock:
run = Mock(return_value=_ok())
with patch(f"{_GW}.container_mod") as mod:
mod.container_is_running.return_value = True
mod.container_image_digest.return_value = "abc"
mod.image_digest.return_value = "abc"
mod.container_env.return_value = env
mod.dns_server.return_value = "1.1.1.1"
mod.run_container_argv = run
AppleGateway(orchestrator_url=url).ensure_running()
return run
def test_moved_control_plane_forces_a_recreate(self) -> None:
"""Docker hands the gateway a container *name*, stable across an
orchestrator recreate. Apple has no DNS, so the URL is an IP baked into
the gateway's env — if the orchestrator comes back on a new address and
the gateway isn't recreated, every /resolve fails and every bottle on
the host loses egress."""
run = self._start_with_running_env(
{"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"},
"http://192.168.128.7:8099",
)
run.assert_called_once()
self.assertIn(
"BOT_BOTTLE_ORCHESTRATOR_URL=http://192.168.128.7:8099",
run.call_args.args[0],
)
def test_unmoved_control_plane_does_not_churn(self) -> None:
run = self._start_with_running_env(
{"BOT_BOTTLE_ORCHESTRATOR_URL": "http://192.168.128.2:8099"},
"http://192.168.128.2:8099",
)
run.assert_not_called()
def test_unreadable_env_does_not_churn(self) -> None:
run = self._start_with_running_env({}, "http://192.168.128.2:8099")
run.assert_not_called()
class TestMacosOrchestratorService(unittest.TestCase):
def test_orchestrator_starts_before_the_gateway(self) -> None:
"""Apple has no container DNS, so the gateway can only be handed the
control plane's IP — which does not exist until it is running. This
ordering is the whole reason the macOS service diverges from docker's."""
order: list[str] = []
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
gateway = Mock()
gateway.ensure_running.side_effect = lambda: order.append("gateway")
def _record_orchestrator(_hash: str) -> None:
order.append("orchestrator")
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.source_hash", return_value="h1"), \
patch.object(svc, "_run_orchestrator_container",
side_effect=_record_orchestrator), \
patch.object(svc, "gateway", return_value=gateway), \
patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = False
mod.image_exists.return_value = True
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
url = svc.ensure_running()
self.assertEqual(["orchestrator", "gateway"], order)
self.assertEqual("http://192.168.128.2:8099", url)
def test_gateway_is_handed_the_resolved_url(self) -> None:
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.source_hash", return_value="h1"), \
patch.object(svc, "_run_orchestrator_container"), \
patch.object(svc, "gateway") as gw, \
patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = False
mod.image_exists.return_value = True
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
svc.ensure_running()
gw.assert_called_with("http://192.168.128.2:8099")
def test_current_source_leaves_a_healthy_orchestrator_alone(self) -> None:
"""Recreating on every launch would drop every other live bottle's
in-memory egress tokens (#381)."""
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
run = Mock()
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.source_hash", return_value="h1"), \
patch.object(svc, "_run_orchestrator_container", run), \
patch.object(svc, "gateway"), \
patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = True
mod.inspect_container.return_value = {
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
}
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
svc.ensure_running()
run.assert_not_called()
def test_wedged_but_current_orchestrator_is_recreated(self) -> None:
"""A container running the current code but whose HTTP server is dead
must be recreated, not left alone and polled to death forever. Checking
the source-hash label alone (without health) would strand the host."""
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
run = Mock()
# Unhealthy on the pre-check, healthy once recreated + waited.
health = Mock(side_effect=[False, True])
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.source_hash", return_value="h1"), \
patch.object(svc, "_run_orchestrator_container", run), \
patch.object(svc, "_ensure_orchestrator_image"), \
patch.object(svc, "gateway"), \
patch.object(svc, "is_healthy", health):
mod.container_is_running.return_value = True
mod.inspect_container.return_value = {
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
}
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
svc.ensure_running()
run.assert_called_once()
def test_changed_source_recreates_the_orchestrator(self) -> None:
"""The control-plane process loaded its bind-mounted source at startup
and won't reload it."""
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
run = Mock()
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.source_hash", return_value="h2"), \
patch.object(svc, "_run_orchestrator_container", run), \
patch.object(svc, "gateway"), \
patch.object(svc, "is_healthy", return_value=True):
mod.container_is_running.return_value = True
mod.inspect_container.return_value = {
"configuration": {"labels": {"bot-bottle-orchestrator-source-hash": "h1"}}
}
mod.image_exists.return_value = True
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
svc.ensure_running()
run.assert_called_once()
def test_never_healthy_raises(self) -> None:
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.source_hash", return_value="h1"), \
patch.object(svc, "_run_orchestrator_container"), \
patch.object(svc, "is_healthy", return_value=False):
mod.container_is_running.return_value = False
mod.image_exists.return_value = True
mod.try_container_ipv4_on_network.return_value = "192.168.128.2"
with self.assertRaises(OrchestratorStartError):
svc.ensure_running(startup_timeout=0.01)
def test_control_plane_needs_no_route_out(self) -> None:
"""The orchestrator sits only on the host-only network: the host
reaches it there directly, so there is no --publish and no NAT leg."""
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
run = Mock(return_value=_ok())
# `ensure_networks` lives in the gateway module and shells out to the
# `container` CLI, which does not exist on the Linux CI host — patch it
# here, not gateway.container_mod, since it is called through this
# module's imported name.
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.ensure_networks"):
mod.run_container_argv = run
svc._run_orchestrator_container("h1")
argv = run.call_args.args[0]
networks = [argv[i + 1] for i, a in enumerate(argv) if a == "--network"]
self.assertEqual(["bot-bottle-mac-gateway"], networks)
self.assertNotIn("--publish", argv)
def test_networks_exist_before_the_orchestrator_runs(self) -> None:
"""The orchestrator is the first container on the shared network, so it
has to create it — the gateway that used to do so now starts second."""
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
order: list[str] = []
def _networks(*_args: str) -> None:
order.append("networks")
def _run(*_args: list[str]) -> Mock:
order.append("run")
return _ok()
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.ensure_networks", side_effect=_networks):
mod.run_container_argv = Mock(side_effect=_run)
svc._run_orchestrator_container("h1")
self.assertEqual(["networks", "run"], order)
def test_orchestrator_start_failure_raises(self) -> None:
svc = MacosOrchestratorService(repo_root=Path("/r"), host_root=Path("/h"))
with patch(f"{_ORCH}.container_mod") as mod, \
patch(f"{_ORCH}.ensure_networks"):
mod.run_container_argv = Mock(return_value=_fail())
with self.assertRaises(OrchestratorStartError):
svc._run_orchestrator_container("h1")
if __name__ == "__main__":
unittest.main()