diff --git a/bot_bottle/orchestrator/gateway.py b/bot_bottle/orchestrator/gateway.py index 32a6344..741e20e 100644 --- a/bot_bottle/orchestrator/gateway.py +++ b/bot_bottle/orchestrator/gateway.py @@ -27,6 +27,7 @@ from ..paths import ( CONTROL_PLANE_TOKEN_ENV, host_control_plane_token, host_db_path, + host_gateway_ca_dir, ) from ..supervise import DB_PATH_IN_CONTAINER @@ -48,14 +49,23 @@ GATEWAY_LABEL = "bot-bottle-orch-gateway=1" # the source IP the gateway attributes by is the address on this network. GATEWAY_NETWORK = "bot-bottle-gateway" -# mitmproxy's CA dir in the bundle. A persistent named volume here keeps the -# gateway's self-generated CA STABLE across container recreation — every agent -# installs this one CA to trust the shared gateway's TLS interception, so it -# must not rotate when the gateway restarts. +# mitmproxy's CA dir in the bundle. The host's gateway-CA dir (see +# `host_gateway_ca_dir`) is bind-mounted here so the gateway's self-generated +# CA stays STABLE across container recreation — every agent installs this one +# CA to trust the shared gateway's TLS interception, so it must not rotate when +# the gateway restarts. A host bind-mount rather than a named volume: a named +# volume is silently wiped by `docker volume prune`, minting a fresh CA that +# breaks every running bottle (issue #450). MITMPROXY_HOME = "/home/mitmproxy/.mitmproxy" -GATEWAY_CA_VOLUME = "bot-bottle-gateway-mitmproxy" GATEWAY_CA_CERT = f"{MITMPROXY_HOME}/mitmproxy-ca-cert.pem" +# The CA material mitmproxy writes into its confdir. mitmproxy reuses these on +# startup when present and generates them only on first run, so persisting them +# is what makes the CA stable; deleting them (see `rotate_gateway_ca`) forces a +# fresh CA on the next start. `mitmproxy-ca.pem` (cert + private key) is the +# signing identity; the rest are derived encodings agents/clients consume. +GATEWAY_CA_GLOB = "mitmproxy-ca*" + # The gateway data-plane image + its Dockerfile. Kept as a local constant # rather than imported from the backend layer, which would drag # the whole backend layer into the lean orchestrator (see #359); unify when @@ -73,6 +83,26 @@ def _host_db_dir() -> str: return str(db_dir) +def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]: + """Delete the persisted mitmproxy CA so the next gateway start mints a + fresh one — the explicit, deliberate CA-rollover path (issue #450). + + Persistence keeps the CA stable across restarts precisely because mitmproxy + reuses the on-disk CA; rotation is therefore just removing that material. + Returns the files removed (empty when there was no CA yet); idempotent. + + This only clears the on-disk CA. It does NOT stop the running gateway (whose + mitmproxy still holds the old CA in memory) or re-provision agents — the + caller recreates the gateway to mint the new CA and re-attaches bottles. + `rotate-ca` on the orchestrator CLI wires those steps together.""" + ca_dir = ca_dir if ca_dir is not None else host_gateway_ca_dir() + removed: list[Path] = [] + for path in sorted(ca_dir.glob(GATEWAY_CA_GLOB)): + path.unlink() + removed.append(path) + return removed + + class GatewayError(Exception): """The shared gateway failed to build/start/stop (non-zero `docker` exit).""" @@ -221,9 +251,10 @@ class DockerGateway(Gateway): "--name", self.name, "--label", GATEWAY_LABEL, "--network", self.network, - # Persist the self-generated CA so it survives restarts (agents - # trust it) — see GATEWAY_CA_VOLUME. - "--volume", f"{GATEWAY_CA_VOLUME}:{MITMPROXY_HOME}", + # Persist the self-generated CA on the host so it survives both + # container recreation AND docker volume pruning (agents trust it) + # — see host_gateway_ca_dir / issue #450. + "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", # Share the one host DB: the supervise daemon queues proposals # into the same file the orchestrator (and the operator, over # HTTP) reads — no second, disconnected DB in the container. @@ -272,7 +303,7 @@ class DockerGateway(Gateway): __all__ = [ - "Gateway", "DockerGateway", "GatewayError", + "Gateway", "DockerGateway", "GatewayError", "rotate_gateway_ca", "GATEWAY_NAME", "GATEWAY_LABEL", "GATEWAY_IMAGE", "GATEWAY_NETWORK", - "GATEWAY_CA_VOLUME", "GATEWAY_CA_CERT", + "GATEWAY_CA_CERT", "GATEWAY_CA_GLOB", ] diff --git a/bot_bottle/orchestrator/lifecycle.py b/bot_bottle/orchestrator/lifecycle.py index 5cfd973..cc3d82a 100644 --- a/bot_bottle/orchestrator/lifecycle.py +++ b/bot_bottle/orchestrator/lifecycle.py @@ -23,10 +23,14 @@ from pathlib import Path from .. import log from ..docker_cmd import run_docker -from ..paths import CONTROL_PLANE_TOKEN_ENV, bot_bottle_root, host_control_plane_token +from ..paths import ( + CONTROL_PLANE_TOKEN_ENV, + bot_bottle_root, + host_control_plane_token, + host_gateway_ca_dir, +) from ..supervise import DB_PATH_IN_CONTAINER from .gateway import ( - GATEWAY_CA_VOLUME, GATEWAY_DOCKERFILE, GATEWAY_IMAGE, GATEWAY_NETWORK, @@ -194,8 +198,10 @@ class OrchestratorService: # gateway_init always starts the orchestrator on DEFAULT_PORT (8099) # inside the container; self.port is the host-side published port. "--publish", f"127.0.0.1:{self.port}:{DEFAULT_PORT}", - # Persist the mitmproxy CA so it survives container recreation. - "--volume", f"{GATEWAY_CA_VOLUME}:{MITMPROXY_HOME}", + # Persist the mitmproxy CA on the host so it survives container + # recreation AND docker volume pruning (issue #450): every agent + # trusts this one CA, so a fresh one would break all running bottles. + "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}", # Shared supervise DB (same file the operator reads over HTTP). "--volume", f"{_host_db_dir()}:{_SUPERVISE_DB_DIR_IN_CONTAINER}", "--env", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}", diff --git a/bot_bottle/orchestrator/rotate_ca.py b/bot_bottle/orchestrator/rotate_ca.py new file mode 100644 index 0000000..5495c77 --- /dev/null +++ b/bot_bottle/orchestrator/rotate_ca.py @@ -0,0 +1,62 @@ +"""Rotate the shared gateway's mitmproxy CA (issue #450). + + python -m bot_bottle.orchestrator.rotate_ca + +A deliberate CA rollover has two halves: drop the *persisted* CA so a fresh one +is minted, and drop the *running* gateway so its mitmproxy (which holds the old +CA in memory) is replaced. This one-shot command does both: + + 1. Delete the persisted CA under the host gateway-CA dir — the next gateway + start generates a new one (mitmproxy reuses an existing CA, generates only + when absent). + 2. Force-remove the infra / standalone-gateway containers so the stale + in-memory CA is gone; the next bottle launch's idempotent `ensure_running` + brings the gateway back up and mints the fresh CA. + +It does NOT re-provision the new CA into already-running bottles — those must be +re-attached so they install the new trust anchor. Rotation is thus an explicit, +operator-driven action with a brief egress interruption, not an automatic one. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +from ..docker_cmd import run_docker +from ..paths import host_gateway_ca_dir +from .gateway import GATEWAY_NAME, rotate_gateway_ca +from .lifecycle import INFRA_NAME + +# The containers whose mitmproxy would still be serving the old CA from memory: +# the consolidated infra container and the standalone per-host gateway. +_GATEWAY_CONTAINERS = (INFRA_NAME, GATEWAY_NAME) + + +def _out(msg: str) -> None: + sys.stdout.write(f"rotate-ca: {msg}\n") + + +def main(argv: list[str] | None = None) -> int: + del argv # no flags — a single deliberate action + ca_dir: Path = host_gateway_ca_dir() + removed = rotate_gateway_ca(ca_dir) + if removed: + _out(f"removed {len(removed)} CA file(s) from {ca_dir}") + else: + _out(f"no persisted CA under {ca_dir}; a fresh one is minted on next start") + + # Drop any running gateway so its in-memory (now-stale) CA is replaced on + # the next launch. `rm --force` on an absent name is a tolerated no-op. + for name in _GATEWAY_CONTAINERS: + proc = run_docker(["docker", "rm", "--force", name]) + if proc.returncode == 0 and proc.stdout.strip(): + _out(f"removed running container {name}") + + _out("done — the next bottle launch remints the CA; re-attach bottles to " + "install the new trust anchor") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bot_bottle/paths.py b/bot_bottle/paths.py index d7e1e42..74c24de 100644 --- a/bot_bottle/paths.py +++ b/bot_bottle/paths.py @@ -33,6 +33,13 @@ HOST_DB_FILENAME = "bot-bottle.db" CONTROL_PLANE_TOKEN_FILENAME = "control-plane-token" CONTROL_PLANE_TOKEN_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN" +# The host directory holding the gateway's persistent mitmproxy CA. Bind-mounted +# into the infra/gateway container at mitmproxy's confdir so the self-generated +# CA survives container recreation — every agent installs this one CA to trust +# the shared gateway's TLS interception, so it must not rotate on restart. See +# host_gateway_ca_dir() for why this is a host bind-mount, not a named volume. +GATEWAY_CA_DIRNAME = "gateway-ca" + def bot_bottle_root() -> Path: """The app data root — `$BOT_BOTTLE_ROOT` if set, else `~/.bot-bottle`.""" @@ -59,6 +66,23 @@ def host_db_dir() -> Path: return db_dir +def host_gateway_ca_dir() -> Path: + """The directory holding the gateway's persistent mitmproxy CA, created if + missing. Backends bind-mount this into the infra/gateway container at + mitmproxy's confdir so the CA persists across container recreation. + + A host bind-mount under the app-data root — deliberately NOT a Docker + named volume. A named volume survives `docker rm` but is silently wiped by + `docker volume prune` / `docker system prune --volumes` during routine host + maintenance; the gateway then mints a fresh CA that every already-running + bottle distrusts, failing the TLS handshake even after it reconnects to the + moved gateway (issue #450). A path under the root docker never prunes it, + and it stays directly inspectable + rotatable from the host.""" + ca_dir = bot_bottle_root() / GATEWAY_CA_DIRNAME + ca_dir.mkdir(parents=True, exist_ok=True) + return ca_dir + + def host_control_plane_token() -> str: """The per-host control-plane secret, minted (256-bit, url-safe) and persisted 0600 on first use, then reused. @@ -94,8 +118,10 @@ __all__ = [ "HOST_DB_FILENAME", "CONTROL_PLANE_TOKEN_FILENAME", "CONTROL_PLANE_TOKEN_ENV", + "GATEWAY_CA_DIRNAME", "bot_bottle_root", "host_db_path", "host_db_dir", + "host_gateway_ca_dir", "host_control_plane_token", ] diff --git a/docs/prds/0070-per-host-orchestrator.md b/docs/prds/0070-per-host-orchestrator.md index 8bb0a87..ee5a8f9 100644 --- a/docs/prds/0070-per-host-orchestrator.md +++ b/docs/prds/0070-per-host-orchestrator.md @@ -312,6 +312,35 @@ reaches over the RPC rather than a shared mount into the VM. WAL on the shared DB is therefore a deliberate, tested future change — not enabled ad hoc. `sqlite3` itself is stdlib, so "the host needs SQLite" is a non-cost. +### Gateway CA: host-resident, like the DB + +The shared gateway bumps TLS with a self-generated mitmproxy CA, and **every +bottle installs that CA** into its trust store to accept the bumped leaves. So +the CA is durable per-host state with the same rule as the DB: it must outlive +any single gateway container, or a restart mints a fresh CA that every +already-running bottle distrusts — the TLS handshake then fails even after the +bottle re-resolves and reconnects to the moved gateway (issue #450, a +re-attachment blocker distinct from #443/#445). + +The CA lives on the **host filesystem** at `bot_bottle_root()/gateway-ca` +(`host_gateway_ca_dir()`), bind-mounted into the container at mitmproxy's +confdir. This is deliberately a host bind-mount, **not a Docker named volume**: +a named volume survives `docker rm` but is silently wiped by +`docker volume prune` / `docker system prune --volumes` during routine host +maintenance, which is exactly how the ephemeral-CA symptom shows up in +practice. A path under the app-data root is never pruned by docker, and stays +directly inspectable and rotatable from the host. mitmproxy reuses an existing +CA and generates one only on first run, so the bind-mount alone gives +"adopt-existing, generate-on-first-run" for free. + +**Deliberate rollover** is the explicit inverse: `rotate_gateway_ca()` removes +the persisted CA material so the next start remints it, and the +`python -m bot_bottle.orchestrator.rotate_ca` one-shot wires that together with +dropping the running gateway container (whose mitmproxy still holds the old CA +in memory). Rotation does not auto-re-provision the new CA into running bottles +— those re-attach to install the new anchor — so it is an operator action with +a brief egress interruption, never an implicit one. + ## Sequencing Jump straight to the **virtualized** end state (not a host-daemon stepping diff --git a/tests/unit/test_orchestrator_gateway.py b/tests/unit/test_orchestrator_gateway.py index 61ba8e0..a413e66 100644 --- a/tests/unit/test_orchestrator_gateway.py +++ b/tests/unit/test_orchestrator_gateway.py @@ -2,7 +2,9 @@ from __future__ import annotations +import tempfile import unittest +from pathlib import Path from unittest.mock import Mock, patch from bot_bottle.orchestrator.gateway import ( @@ -10,7 +12,10 @@ from bot_bottle.orchestrator.gateway import ( GATEWAY_NAME, DockerGateway, GatewayError, + rotate_gateway_ca, ) +from bot_bottle.paths import GATEWAY_CA_DIRNAME, host_gateway_ca_dir +from tests.unit import use_bottle_root _CA_PEM = "-----BEGIN CERTIFICATE-----\nMII...\n-----END CERTIFICATE-----\n" @@ -27,6 +32,11 @@ _ORCH_URL = "http://orchestrator:9000" class TestDockerGateway(unittest.TestCase): def setUp(self) -> None: + # Redirect the app-data root so ensure_running's host-dir mkdirs (CA + + # DB) land in a throwaway dir, not the real ~/.bot-bottle. + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.addCleanup(use_bottle_root(Path(self._tmp.name))) # Resolver-only data plane (PRD 0070): running the gateway requires an # orchestrator URL, so the fixture supplies one. self.sc = DockerGateway("bot-bottle-gateway:latest", orchestrator_url=_ORCH_URL) @@ -103,8 +113,17 @@ class TestDockerGateway(unittest.TestCase): self.assertIn("bot-bottle-gateway:latest", runs[0]) # Runs on the shared gateway network so agents can reach it by IP. self.assertEqual(self.sc.network, runs[0][runs[0].index("--network") + 1]) - # Persists its CA on a named volume so agents keep trusting it. - self.assertTrue(any("mitmproxy" in a for a in runs[0])) + # Persists its CA on a HOST bind-mount (not a docker named volume, which + # `docker volume prune` would wipe — issue #450) so agents keep trusting + # it across restarts. The mount source is the host gateway-CA dir. + ca_mounts = [ + a for a in runs[0] + if a.endswith(":/home/mitmproxy/.mitmproxy") + ] + self.assertEqual(1, len(ca_mounts)) + src = ca_mounts[0].rsplit(":", 1)[0] + self.assertTrue(src.endswith("/" + GATEWAY_CA_DIRNAME), src) + self.assertTrue(Path(src).is_absolute(), src) # Shares the ONE host DB: the supervise daemon queues into the same # file the orchestrator + operator (over HTTP) use. self.assertTrue(any( @@ -234,5 +253,49 @@ class TestDockerGatewayBuild(unittest.TestCase): self.sc.ensure_built() +class TestRotateGatewayCa(unittest.TestCase): + """rotate_gateway_ca clears the persisted CA so the next start remints it.""" + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.addCleanup(use_bottle_root(Path(self._tmp.name))) + + def _seed_ca(self) -> Path: + ca_dir = host_gateway_ca_dir() + # A representative mitmproxy confdir: the CA identity + derived encodings, + # plus one non-CA file that rotation must leave untouched. + for name in ( + "mitmproxy-ca.pem", + "mitmproxy-ca-cert.pem", + "mitmproxy-ca-cert.cer", + "mitmproxy-ca-cert.p12", + ): + (ca_dir / name).write_text("x") + (ca_dir / "combined-trust.pem").write_text("keep") + return ca_dir + + def test_removes_ca_material_only(self) -> None: + ca_dir = self._seed_ca() + removed = rotate_gateway_ca(ca_dir) + self.assertEqual(4, len(removed)) + self.assertTrue(all(p.name.startswith("mitmproxy-ca") for p in removed)) + # The CA files are gone; the non-CA trust bundle survives. + self.assertEqual( + {"combined-trust.pem"}, {p.name for p in ca_dir.iterdir()} + ) + + def test_defaults_to_host_ca_dir(self) -> None: + self._seed_ca() + removed = rotate_gateway_ca() # no arg → host_gateway_ca_dir() + self.assertTrue(removed) + self.assertEqual( + [], list(host_gateway_ca_dir().glob("mitmproxy-ca*")) + ) + + def test_idempotent_when_no_ca(self) -> None: + self.assertEqual([], rotate_gateway_ca(host_gateway_ca_dir())) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_orchestrator_lifecycle.py b/tests/unit/test_orchestrator_lifecycle.py index 6665a33..2672853 100644 --- a/tests/unit/test_orchestrator_lifecycle.py +++ b/tests/unit/test_orchestrator_lifecycle.py @@ -16,6 +16,7 @@ from bot_bottle.orchestrator.lifecycle import ( OrchestratorStartError, source_hash, ) +from bot_bottle.paths import GATEWAY_CA_DIRNAME from tests.unit import use_bottle_root _URLOPEN = "bot_bottle.orchestrator.lifecycle.urllib.request.urlopen" @@ -115,6 +116,14 @@ class TestOrchestratorService(unittest.TestCase): # Gateway daemons + orchestrator explicitly opted in. daemons_flag = "BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise,orchestrator" self.assertIn("orchestrator", argv[argv.index(daemons_flag)]) + # The mitmproxy CA persists on a HOST bind-mount under the app-data root + # (not a docker named volume `docker volume prune` would wipe — #450), so + # a restarted infra container keeps the CA every running bottle trusts. + ca_mounts = [a for a in argv if a.endswith(":/home/mitmproxy/.mitmproxy")] + self.assertEqual(1, len(ca_mounts)) + src = ca_mounts[0].rsplit(":", 1)[0] + self.assertTrue(src.startswith(self._tmp.name), src) + self.assertTrue(src.endswith("/" + GATEWAY_CA_DIRNAME), src) def test_ensure_running_builds_all_images(self) -> None: calls: list[list[str]] = [] diff --git a/tests/unit/test_orchestrator_rotate_ca.py b/tests/unit/test_orchestrator_rotate_ca.py new file mode 100644 index 0000000..22199b8 --- /dev/null +++ b/tests/unit/test_orchestrator_rotate_ca.py @@ -0,0 +1,59 @@ +"""Unit: the `rotate_ca` one-shot CLI (issue #450). Docker mocked.""" + +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +from bot_bottle.orchestrator import rotate_ca +from bot_bottle.orchestrator.gateway import GATEWAY_NAME +from bot_bottle.orchestrator.lifecycle import INFRA_NAME +from bot_bottle.paths import host_gateway_ca_dir +from tests.unit import use_bottle_root + +_RUN = "bot_bottle.orchestrator.rotate_ca.run_docker" + + +def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock: + return Mock(returncode=returncode, stdout=stdout, stderr=stderr) + + +class TestRotateCaCli(unittest.TestCase): + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.addCleanup(use_bottle_root(Path(self._tmp.name))) + + def test_clears_ca_and_drops_gateway_containers(self) -> None: + ca_dir = host_gateway_ca_dir() + (ca_dir / "mitmproxy-ca.pem").write_text("x") + (ca_dir / "mitmproxy-ca-cert.pem").write_text("x") + + calls: list[list[str]] = [] + + def fake(argv: list[str], **_kw: object) -> Mock: + calls.append(argv) + # Report a removed container name so the CLI logs it. + return _proc(stdout=argv[-1]) + + with patch(_RUN, side_effect=fake): + self.assertEqual(0, rotate_ca.main([])) + + # Persisted CA is gone → next start remints it. + self.assertEqual([], list(ca_dir.glob("mitmproxy-ca*"))) + # Both the infra container and the standalone gateway are force-removed + # so no mitmproxy keeps serving the old CA from memory. + removed = {c[-1] for c in calls if c[:3] == ["docker", "rm", "--force"]} + self.assertEqual({INFRA_NAME, GATEWAY_NAME}, removed) + + def test_succeeds_with_no_persisted_ca(self) -> None: + with patch(_RUN, return_value=_proc()) as m: + self.assertEqual(0, rotate_ca.main([])) + # Still tears down any running gateway even when there was no CA on disk. + self.assertTrue(m.called) + + +if __name__ == "__main__": + unittest.main()