fix(gateway): persist mitmproxy CA on the host, not a named volume (#450)

The shared gateway self-generates a mitmproxy CA that every bottle installs
to trust its TLS interception. It was persisted on a Docker named volume,
which survives `docker rm` but is silently wiped by `docker volume prune` /
`docker system prune --volumes` during routine host maintenance. When that
happens the gateway mints a fresh CA on restart, and every already-running
bottle fails the TLS handshake even after it re-resolves and reconnects to
the moved gateway — a re-attachment blocker distinct from #443/#445.

Move CA persistence to a host bind-mount under the app-data root
(`bot_bottle_root()/gateway-ca`, via `host_gateway_ca_dir()`), mirroring how
the shared DB and control-plane token already live on the host. Docker never
prunes a path under the root, and it stays inspectable + rotatable from the
host. mitmproxy already adopts an existing CA and generates one only on first
run, so the bind-mount gives adopt-existing/generate-on-first-run for free.

Add an explicit rollover path: `rotate_gateway_ca()` clears the persisted CA
so the next start remints it, and `python -m bot_bottle.orchestrator.rotate_ca`
wires that together with dropping the running gateway container (whose
mitmproxy still holds the old CA in memory). Rotation stays an operator action
— it doesn't auto-re-provision running bottles, which re-attach to pick up the
new anchor.

Scope: the Docker infra/gateway path (the "infra container" in the report).
The macOS (`container`-only volume) and Firecracker (VM-attached ext4) backends
persist the CA differently and are unaffected.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

bot_bottle/orchestrator/gateway.py

@@ -27,6 +27,7 @@ from ..paths import (
     CONTROL_PLANE_TOKEN_ENV,
     host_control_plane_token,
     host_db_path,
+    host_gateway_ca_dir,
 )
 from ..supervise import DB_PATH_IN_CONTAINER
 
@@ -48,14 +49,23 @@ GATEWAY_LABEL = "bot-bottle-orch-gateway=1"
 # the source IP the gateway attributes by is the address on this network.
 GATEWAY_NETWORK = "bot-bottle-gateway"
 
-# mitmproxy's CA dir in the bundle. A persistent named volume here keeps the
-# gateway's self-generated CA STABLE across container recreation — every agent
-# installs this one CA to trust the shared gateway's TLS interception, so it
-# must not rotate when the gateway restarts.
+# mitmproxy's CA dir in the bundle. The host's gateway-CA dir (see
+# `host_gateway_ca_dir`) is bind-mounted here so the gateway's self-generated
+# CA stays STABLE across container recreation — every agent installs this one
+# CA to trust the shared gateway's TLS interception, so it must not rotate when
+# the gateway restarts. A host bind-mount rather than a named volume: a named
+# volume is silently wiped by `docker volume prune`, minting a fresh CA that
+# breaks every running bottle (issue #450).
 MITMPROXY_HOME = "/home/mitmproxy/.mitmproxy"
-GATEWAY_CA_VOLUME = "bot-bottle-gateway-mitmproxy"
 GATEWAY_CA_CERT = f"{MITMPROXY_HOME}/mitmproxy-ca-cert.pem"
 
+# The CA material mitmproxy writes into its confdir. mitmproxy reuses these on
+# startup when present and generates them only on first run, so persisting them
+# is what makes the CA stable; deleting them (see `rotate_gateway_ca`) forces a
+# fresh CA on the next start. `mitmproxy-ca.pem` (cert + private key) is the
+# signing identity; the rest are derived encodings agents/clients consume.
+GATEWAY_CA_GLOB = "mitmproxy-ca*"
+
 # The gateway data-plane image + its Dockerfile. Kept as a local constant
 # rather than imported from the backend layer, which would drag
 # the whole backend layer into the lean orchestrator (see #359); unify when
@@ -73,6 +83,26 @@ def _host_db_dir() -> str:
     return str(db_dir)
 
 
+def rotate_gateway_ca(ca_dir: Path | None = None) -> list[Path]:
+    """Delete the persisted mitmproxy CA so the next gateway start mints a
+    fresh one — the explicit, deliberate CA-rollover path (issue #450).
+
+    Persistence keeps the CA stable across restarts precisely because mitmproxy
+    reuses the on-disk CA; rotation is therefore just removing that material.
+    Returns the files removed (empty when there was no CA yet); idempotent.
+
+    This only clears the on-disk CA. It does NOT stop the running gateway (whose
+    mitmproxy still holds the old CA in memory) or re-provision agents — the
+    caller recreates the gateway to mint the new CA and re-attaches bottles.
+    `rotate-ca` on the orchestrator CLI wires those steps together."""
+    ca_dir = ca_dir if ca_dir is not None else host_gateway_ca_dir()
+    removed: list[Path] = []
+    for path in sorted(ca_dir.glob(GATEWAY_CA_GLOB)):
+        path.unlink()
+        removed.append(path)
+    return removed
+
+
 class GatewayError(Exception):
     """The shared gateway failed to build/start/stop (non-zero `docker` exit)."""
 
@@ -221,9 +251,10 @@ class DockerGateway(Gateway):
             "--name", self.name,
             "--label", GATEWAY_LABEL,
             "--network", self.network,
-            # Persist the self-generated CA so it survives restarts (agents
-            # trust it) — see GATEWAY_CA_VOLUME.
-            "--volume", f"{GATEWAY_CA_VOLUME}:{MITMPROXY_HOME}",
+            # Persist the self-generated CA on the host so it survives both
+            # container recreation AND docker volume pruning (agents trust it)
+            # — see host_gateway_ca_dir / issue #450.
+            "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}",
             # Share the one host DB: the supervise daemon queues proposals
             # into the same file the orchestrator (and the operator, over
             # HTTP) reads — no second, disconnected DB in the container.
@@ -272,7 +303,7 @@ class DockerGateway(Gateway):
 
 
 __all__ = [
-    "Gateway", "DockerGateway", "GatewayError",
+    "Gateway", "DockerGateway", "GatewayError", "rotate_gateway_ca",
     "GATEWAY_NAME", "GATEWAY_LABEL", "GATEWAY_IMAGE", "GATEWAY_NETWORK",
-    "GATEWAY_CA_VOLUME", "GATEWAY_CA_CERT",
+    "GATEWAY_CA_CERT", "GATEWAY_CA_GLOB",
 ]

bot_bottle/orchestrator/lifecycle.py

@@ -23,10 +23,14 @@ from pathlib import Path
 
 from .. import log
 from ..docker_cmd import run_docker
-from ..paths import CONTROL_PLANE_TOKEN_ENV, bot_bottle_root, host_control_plane_token
+from ..paths import (
+    CONTROL_PLANE_TOKEN_ENV,
+    bot_bottle_root,
+    host_control_plane_token,
+    host_gateway_ca_dir,
+)
 from ..supervise import DB_PATH_IN_CONTAINER
 from .gateway import (
-    GATEWAY_CA_VOLUME,
     GATEWAY_DOCKERFILE,
     GATEWAY_IMAGE,
     GATEWAY_NETWORK,
@@ -194,8 +198,10 @@ class OrchestratorService:
             # gateway_init always starts the orchestrator on DEFAULT_PORT (8099)
             # inside the container; self.port is the host-side published port.
             "--publish", f"127.0.0.1:{self.port}:{DEFAULT_PORT}",
-            # Persist the mitmproxy CA so it survives container recreation.
-            "--volume", f"{GATEWAY_CA_VOLUME}:{MITMPROXY_HOME}",
+            # Persist the mitmproxy CA on the host so it survives container
+            # recreation AND docker volume pruning (issue #450): every agent
+            # trusts this one CA, so a fresh one would break all running bottles.
+            "--volume", f"{host_gateway_ca_dir()}:{MITMPROXY_HOME}",
             # Shared supervise DB (same file the operator reads over HTTP).
             "--volume", f"{_host_db_dir()}:{_SUPERVISE_DB_DIR_IN_CONTAINER}",
             "--env", f"SUPERVISE_DB_PATH={DB_PATH_IN_CONTAINER}",

bot_bottle/orchestrator/rotate_ca.py

@@ -0,0 +1,62 @@
+"""Rotate the shared gateway's mitmproxy CA (issue #450).
+
+    python -m bot_bottle.orchestrator.rotate_ca
+
+A deliberate CA rollover has two halves: drop the *persisted* CA so a fresh one
+is minted, and drop the *running* gateway so its mitmproxy (which holds the old
+CA in memory) is replaced. This one-shot command does both:
+
+  1. Delete the persisted CA under the host gateway-CA dir — the next gateway
+     start generates a new one (mitmproxy reuses an existing CA, generates only
+     when absent).
+  2. Force-remove the infra / standalone-gateway containers so the stale
+     in-memory CA is gone; the next bottle launch's idempotent `ensure_running`
+     brings the gateway back up and mints the fresh CA.
+
+It does NOT re-provision the new CA into already-running bottles — those must be
+re-attached so they install the new trust anchor. Rotation is thus an explicit,
+operator-driven action with a brief egress interruption, not an automatic one.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from ..docker_cmd import run_docker
+from ..paths import host_gateway_ca_dir
+from .gateway import GATEWAY_NAME, rotate_gateway_ca
+from .lifecycle import INFRA_NAME
+
+# The containers whose mitmproxy would still be serving the old CA from memory:
+# the consolidated infra container and the standalone per-host gateway.
+_GATEWAY_CONTAINERS = (INFRA_NAME, GATEWAY_NAME)
+
+
+def _out(msg: str) -> None:
+    sys.stdout.write(f"rotate-ca: {msg}\n")
+
+
+def main(argv: list[str] | None = None) -> int:
+    del argv  # no flags — a single deliberate action
+    ca_dir: Path = host_gateway_ca_dir()
+    removed = rotate_gateway_ca(ca_dir)
+    if removed:
+        _out(f"removed {len(removed)} CA file(s) from {ca_dir}")
+    else:
+        _out(f"no persisted CA under {ca_dir}; a fresh one is minted on next start")
+
+    # Drop any running gateway so its in-memory (now-stale) CA is replaced on
+    # the next launch. `rm --force` on an absent name is a tolerated no-op.
+    for name in _GATEWAY_CONTAINERS:
+        proc = run_docker(["docker", "rm", "--force", name])
+        if proc.returncode == 0 and proc.stdout.strip():
+            _out(f"removed running container {name}")
+
+    _out("done — the next bottle launch remints the CA; re-attach bottles to "
+         "install the new trust anchor")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

bot_bottle/paths.py

@@ -33,6 +33,13 @@ HOST_DB_FILENAME = "bot-bottle.db"
 CONTROL_PLANE_TOKEN_FILENAME = "control-plane-token"
 CONTROL_PLANE_TOKEN_ENV = "BOT_BOTTLE_CONTROL_PLANE_TOKEN"
 
+# The host directory holding the gateway's persistent mitmproxy CA. Bind-mounted
+# into the infra/gateway container at mitmproxy's confdir so the self-generated
+# CA survives container recreation — every agent installs this one CA to trust
+# the shared gateway's TLS interception, so it must not rotate on restart. See
+# host_gateway_ca_dir() for why this is a host bind-mount, not a named volume.
+GATEWAY_CA_DIRNAME = "gateway-ca"
+
 
 def bot_bottle_root() -> Path:
     """The app data root — `$BOT_BOTTLE_ROOT` if set, else `~/.bot-bottle`."""
@@ -59,6 +66,23 @@ def host_db_dir() -> Path:
     return db_dir
 
 
+def host_gateway_ca_dir() -> Path:
+    """The directory holding the gateway's persistent mitmproxy CA, created if
+    missing. Backends bind-mount this into the infra/gateway container at
+    mitmproxy's confdir so the CA persists across container recreation.
+
+    A host bind-mount under the app-data root — deliberately NOT a Docker
+    named volume. A named volume survives `docker rm` but is silently wiped by
+    `docker volume prune` / `docker system prune --volumes` during routine host
+    maintenance; the gateway then mints a fresh CA that every already-running
+    bottle distrusts, failing the TLS handshake even after it reconnects to the
+    moved gateway (issue #450). A path under the root docker never prunes it,
+    and it stays directly inspectable + rotatable from the host."""
+    ca_dir = bot_bottle_root() / GATEWAY_CA_DIRNAME
+    ca_dir.mkdir(parents=True, exist_ok=True)
+    return ca_dir
+
+
 def host_control_plane_token() -> str:
     """The per-host control-plane secret, minted (256-bit, url-safe) and
     persisted 0600 on first use, then reused.
@@ -94,8 +118,10 @@ __all__ = [
     "HOST_DB_FILENAME",
     "CONTROL_PLANE_TOKEN_FILENAME",
     "CONTROL_PLANE_TOKEN_ENV",
+    "GATEWAY_CA_DIRNAME",
     "bot_bottle_root",
     "host_db_path",
     "host_db_dir",
+    "host_gateway_ca_dir",
     "host_control_plane_token",
 ]

docs/prds/0070-per-host-orchestrator.md

@@ -312,6 +312,35 @@ reaches over the RPC rather than a shared mount into the VM. WAL on the
 shared DB is therefore a deliberate, tested future change — not enabled ad
 hoc. `sqlite3` itself is stdlib, so "the host needs SQLite" is a non-cost.
 
+### Gateway CA: host-resident, like the DB
+
+The shared gateway bumps TLS with a self-generated mitmproxy CA, and **every
+bottle installs that CA** into its trust store to accept the bumped leaves. So
+the CA is durable per-host state with the same rule as the DB: it must outlive
+any single gateway container, or a restart mints a fresh CA that every
+already-running bottle distrusts — the TLS handshake then fails even after the
+bottle re-resolves and reconnects to the moved gateway (issue #450, a
+re-attachment blocker distinct from #443/#445).
+
+The CA lives on the **host filesystem** at `bot_bottle_root()/gateway-ca`
+(`host_gateway_ca_dir()`), bind-mounted into the container at mitmproxy's
+confdir. This is deliberately a host bind-mount, **not a Docker named volume**:
+a named volume survives `docker rm` but is silently wiped by
+`docker volume prune` / `docker system prune --volumes` during routine host
+maintenance, which is exactly how the ephemeral-CA symptom shows up in
+practice. A path under the app-data root is never pruned by docker, and stays
+directly inspectable and rotatable from the host. mitmproxy reuses an existing
+CA and generates one only on first run, so the bind-mount alone gives
+"adopt-existing, generate-on-first-run" for free.
+
+**Deliberate rollover** is the explicit inverse: `rotate_gateway_ca()` removes
+the persisted CA material so the next start remints it, and the
+`python -m bot_bottle.orchestrator.rotate_ca` one-shot wires that together with
+dropping the running gateway container (whose mitmproxy still holds the old CA
+in memory). Rotation does not auto-re-provision the new CA into running bottles
+— those re-attach to install the new anchor — so it is an operator action with
+a brief egress interruption, never an implicit one.
+
 ## Sequencing
 
 Jump straight to the **virtualized** end state (not a host-daemon stepping

tests/unit/test_orchestrator_gateway.py

@@ -2,7 +2,9 @@
 
 from __future__ import annotations
 
+import tempfile
 import unittest
+from pathlib import Path
 from unittest.mock import Mock, patch
 
 from bot_bottle.orchestrator.gateway import (
@@ -10,7 +12,10 @@ from bot_bottle.orchestrator.gateway import (
     GATEWAY_NAME,
     DockerGateway,
     GatewayError,
+    rotate_gateway_ca,
 )
+from bot_bottle.paths import GATEWAY_CA_DIRNAME, host_gateway_ca_dir
+from tests.unit import use_bottle_root
 
 
 _CA_PEM = "-----BEGIN CERTIFICATE-----\nMII...\n-----END CERTIFICATE-----\n"
@@ -27,6 +32,11 @@ _ORCH_URL = "http://orchestrator:9000"
 
 class TestDockerGateway(unittest.TestCase):
     def setUp(self) -> None:
+        # Redirect the app-data root so ensure_running's host-dir mkdirs (CA +
+        # DB) land in a throwaway dir, not the real ~/.bot-bottle.
+        self._tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+        self.addCleanup(use_bottle_root(Path(self._tmp.name)))
         # Resolver-only data plane (PRD 0070): running the gateway requires an
         # orchestrator URL, so the fixture supplies one.
         self.sc = DockerGateway("bot-bottle-gateway:latest", orchestrator_url=_ORCH_URL)
@@ -103,8 +113,17 @@ class TestDockerGateway(unittest.TestCase):
         self.assertIn("bot-bottle-gateway:latest", runs[0])
         # Runs on the shared gateway network so agents can reach it by IP.
         self.assertEqual(self.sc.network, runs[0][runs[0].index("--network") + 1])
-        # Persists its CA on a named volume so agents keep trusting it.
-        self.assertTrue(any("mitmproxy" in a for a in runs[0]))
+        # Persists its CA on a HOST bind-mount (not a docker named volume, which
+        # `docker volume prune` would wipe — issue #450) so agents keep trusting
+        # it across restarts. The mount source is the host gateway-CA dir.
+        ca_mounts = [
+            a for a in runs[0]
+            if a.endswith(":/home/mitmproxy/.mitmproxy")
+        ]
+        self.assertEqual(1, len(ca_mounts))
+        src = ca_mounts[0].rsplit(":", 1)[0]
+        self.assertTrue(src.endswith("/" + GATEWAY_CA_DIRNAME), src)
+        self.assertTrue(Path(src).is_absolute(), src)
         # Shares the ONE host DB: the supervise daemon queues into the same
         # file the orchestrator + operator (over HTTP) use.
         self.assertTrue(any(
@@ -234,5 +253,49 @@ class TestDockerGatewayBuild(unittest.TestCase):
                 self.sc.ensure_built()
 
 
+class TestRotateGatewayCa(unittest.TestCase):
+    """rotate_gateway_ca clears the persisted CA so the next start remints it."""
+
+    def setUp(self) -> None:
+        self._tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+        self.addCleanup(use_bottle_root(Path(self._tmp.name)))
+
+    def _seed_ca(self) -> Path:
+        ca_dir = host_gateway_ca_dir()
+        # A representative mitmproxy confdir: the CA identity + derived encodings,
+        # plus one non-CA file that rotation must leave untouched.
+        for name in (
+            "mitmproxy-ca.pem",
+            "mitmproxy-ca-cert.pem",
+            "mitmproxy-ca-cert.cer",
+            "mitmproxy-ca-cert.p12",
+        ):
+            (ca_dir / name).write_text("x")
+        (ca_dir / "combined-trust.pem").write_text("keep")
+        return ca_dir
+
+    def test_removes_ca_material_only(self) -> None:
+        ca_dir = self._seed_ca()
+        removed = rotate_gateway_ca(ca_dir)
+        self.assertEqual(4, len(removed))
+        self.assertTrue(all(p.name.startswith("mitmproxy-ca") for p in removed))
+        # The CA files are gone; the non-CA trust bundle survives.
+        self.assertEqual(
+            {"combined-trust.pem"}, {p.name for p in ca_dir.iterdir()}
+        )
+
+    def test_defaults_to_host_ca_dir(self) -> None:
+        self._seed_ca()
+        removed = rotate_gateway_ca()  # no arg → host_gateway_ca_dir()
+        self.assertTrue(removed)
+        self.assertEqual(
+            [], list(host_gateway_ca_dir().glob("mitmproxy-ca*"))
+        )
+
+    def test_idempotent_when_no_ca(self) -> None:
+        self.assertEqual([], rotate_gateway_ca(host_gateway_ca_dir()))
+
+
 if __name__ == "__main__":
     unittest.main()

tests/unit/test_orchestrator_lifecycle.py

@@ -16,6 +16,7 @@ from bot_bottle.orchestrator.lifecycle import (
     OrchestratorStartError,
     source_hash,
 )
+from bot_bottle.paths import GATEWAY_CA_DIRNAME
 from tests.unit import use_bottle_root
 
 _URLOPEN = "bot_bottle.orchestrator.lifecycle.urllib.request.urlopen"
@@ -115,6 +116,14 @@ class TestOrchestratorService(unittest.TestCase):
         # Gateway daemons + orchestrator explicitly opted in.
         daemons_flag = "BOT_BOTTLE_GATEWAY_DAEMONS=egress,git-http,supervise,orchestrator"
         self.assertIn("orchestrator", argv[argv.index(daemons_flag)])
+        # The mitmproxy CA persists on a HOST bind-mount under the app-data root
+        # (not a docker named volume `docker volume prune` would wipe — #450), so
+        # a restarted infra container keeps the CA every running bottle trusts.
+        ca_mounts = [a for a in argv if a.endswith(":/home/mitmproxy/.mitmproxy")]
+        self.assertEqual(1, len(ca_mounts))
+        src = ca_mounts[0].rsplit(":", 1)[0]
+        self.assertTrue(src.startswith(self._tmp.name), src)
+        self.assertTrue(src.endswith("/" + GATEWAY_CA_DIRNAME), src)
 
     def test_ensure_running_builds_all_images(self) -> None:
         calls: list[list[str]] = []

tests/unit/test_orchestrator_rotate_ca.py

@@ -0,0 +1,59 @@
+"""Unit: the `rotate_ca` one-shot CLI (issue #450). Docker mocked."""
+
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+from bot_bottle.orchestrator import rotate_ca
+from bot_bottle.orchestrator.gateway import GATEWAY_NAME
+from bot_bottle.orchestrator.lifecycle import INFRA_NAME
+from bot_bottle.paths import host_gateway_ca_dir
+from tests.unit import use_bottle_root
+
+_RUN = "bot_bottle.orchestrator.rotate_ca.run_docker"
+
+
+def _proc(returncode: int = 0, stdout: str = "", stderr: str = "") -> Mock:
+    return Mock(returncode=returncode, stdout=stdout, stderr=stderr)
+
+
+class TestRotateCaCli(unittest.TestCase):
+    def setUp(self) -> None:
+        self._tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+        self.addCleanup(use_bottle_root(Path(self._tmp.name)))
+
+    def test_clears_ca_and_drops_gateway_containers(self) -> None:
+        ca_dir = host_gateway_ca_dir()
+        (ca_dir / "mitmproxy-ca.pem").write_text("x")
+        (ca_dir / "mitmproxy-ca-cert.pem").write_text("x")
+
+        calls: list[list[str]] = []
+
+        def fake(argv: list[str], **_kw: object) -> Mock:
+            calls.append(argv)
+            # Report a removed container name so the CLI logs it.
+            return _proc(stdout=argv[-1])
+
+        with patch(_RUN, side_effect=fake):
+            self.assertEqual(0, rotate_ca.main([]))
+
+        # Persisted CA is gone → next start remints it.
+        self.assertEqual([], list(ca_dir.glob("mitmproxy-ca*")))
+        # Both the infra container and the standalone gateway are force-removed
+        # so no mitmproxy keeps serving the old CA from memory.
+        removed = {c[-1] for c in calls if c[:3] == ["docker", "rm", "--force"]}
+        self.assertEqual({INFRA_NAME, GATEWAY_NAME}, removed)
+
+    def test_succeeds_with_no_persisted_ca(self) -> None:
+        with patch(_RUN, return_value=_proc()) as m:
+            self.assertEqual(0, rotate_ca.main([]))
+        # Still tears down any running gateway even when there was no CA on disk.
+        self.assertTrue(m.called)
+
+
+if __name__ == "__main__":
+    unittest.main()