feat(orchestrator): slice 4 — consolidated per-host sidecar (#352)

The core consolidation win: one persistent sidecar per host, shared by
every bottle, instead of a sidecar bundle per bottle. Safe to share
because the attribution invariant (source IP + identity token) lets the
sidecar map each request to the right bottle.

  * orchestrator/sidecar.py — a backend-neutral `Sidecar` lifecycle
    contract (mirrors LaunchBroker) + a `DockerSidecar` impl. The defining
    behaviour is idempotent singleton: `ensure_running` starts the instance
    if absent and is a no-op if it's already up, so N launches never spawn
    N sidecars; `stop` is idempotent.
  * orchestrator/dockerutil.py — a shared `run_docker` helper; DockerBroker
    now uses it too (DRY with slice 3).
  * service.py — the Orchestrator holds an optional `Sidecar`, exposes
    `ensure_sidecar()` + `sidecar_status()`.
  * control_plane.py — `GET /sidecar` reports it; __main__ gains
    `--sidecar-image` and ensures the single sidecar on startup.

Tests: unit (docker mocked) — is_running, ensure idempotent (no-op when up,
starts when absent), failure raises, stop idempotent; Orchestrator sidecar
wiring/status; control-plane /sidecar; integration (gated) — ensure is a
real idempotent singleton (one container after two ensures), stop removes.
Full suite green (only pre-existing /bin/sleep errors); integration
verified locally against real docker.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WBMWTEtQdJ4W5UrWuLHCck

bot_bottle/orchestrator/__init__.py

@@ -10,10 +10,15 @@ backend-neutral "consolidation core" that needs no VM packaging:
   * `broker`        — the signed, structured launch-request contract + a
                       `LaunchBroker` (stub for the harness) that verifies
                       provenance before acting.
+  * `sidecar`       — the consolidated per-host sidecar: a `Sidecar`
+                      lifecycle contract (idempotent singleton) + a
+                      `DockerSidecar` impl. One sidecar shared by all
+                      bottles instead of one per bottle.
   * `service`       — the `Orchestrator`: owns the registry, brokers the
-                      launch lifecycle (launch/teardown), attributes.
+                      launch lifecycle (launch/teardown), manages the
+                      shared sidecar, attributes.
   * `control_plane` — the HTTP control-plane RPC (launch / teardown /
-                      list / attribute / health).
+                      list / attribute / sidecar / health).
 
 The actual backend-native launch (a real docker/firecracker broker) and
 the egress/git/supervise data plane land in later slices once this core is
@@ -33,6 +38,7 @@ from .broker import (
     verify_request,
 )
 from .docker_broker import DockerBroker, DockerBrokerError
+from .sidecar import DockerSidecar, Sidecar, SidecarError
 from .service import Orchestrator
 from .control_plane import ControlPlaneServer, dispatch, make_server
 
@@ -46,6 +52,9 @@ __all__ = [
     "StubBroker",
     "DockerBroker",
     "DockerBrokerError",
+    "Sidecar",
+    "DockerSidecar",
+    "SidecarError",
     "sign_request",
     "verify_request",
     "Orchestrator",

bot_bottle/orchestrator/__main__.py

@@ -21,6 +21,7 @@ from .control_plane import make_server
 from .docker_broker import DockerBroker
 from .registry import RegistryStore, default_db_path
 from .service import Orchestrator
+from .sidecar import DockerSidecar, Sidecar
 
 
 def main(argv: list[str] | None = None) -> int:
@@ -36,6 +37,10 @@ def main(argv: list[str] | None = None) -> int:
         "--broker", choices=("stub", "docker"), default="stub",
         help="launch broker: 'stub' records requests; 'docker' runs containers",
     )
+    parser.add_argument(
+        "--sidecar-image", default=None,
+        help="if set, run one consolidated per-host sidecar from this image",
+    )
     args = parser.parse_args(argv)
 
     registry = RegistryStore(args.db)
@@ -46,7 +51,13 @@ def main(argv: list[str] | None = None) -> int:
     # anything; 'docker' runs real containers (firecracker drops in later).
     secret = secrets.token_bytes(32)
     broker: LaunchBroker = DockerBroker(secret) if args.broker == "docker" else StubBroker(secret)
-    orchestrator = Orchestrator(registry, broker, secret)
+    sidecar: Sidecar | None = DockerSidecar(args.sidecar_image) if args.sidecar_image else None
+    orchestrator = Orchestrator(registry, broker, secret, sidecar)
+
+    # One persistent per-host sidecar, shared by every bottle (idempotent).
+    if sidecar is not None:
+        orchestrator.ensure_sidecar()
+        log.info("consolidated sidecar ensured", context={"name": sidecar.name})
 
     server = make_server(orchestrator, host=args.host, port=args.port)
     bound_host, bound_port = server.server_address[0], server.server_address[1]

bot_bottle/orchestrator/control_plane.py

@@ -5,6 +5,7 @@ The backend-agnostic control-plane RPC (CLI / console -> orchestrator) over
 vsock / unix-socket portability caveats):
 
     GET    /health              -> 200 {"status": "ok"}
+    GET    /sidecar             -> 200 {"configured", ["name", "running"]}
     GET    /bottles             -> 200 {"bottles": [ <redacted record>, ... ]}
     POST   /bottles             -> 201 {"bottle_id", "identity_token"}    (launch)
                                    body: {"source_ip", ["image_ref"], ["metadata"]}
@@ -58,6 +59,9 @@ def dispatch(  # pylint: disable=too-many-return-statements
     if method == "GET" and route == "/health":
         return 200, {"status": "ok"}
 
+    if method == "GET" and route == "/sidecar":
+        return 200, orch.sidecar_status()
+
     if method == "GET" and route == "/bottles":
         return 200, {"bottles": [r.redacted() for r in orch.registry.all()]}
 

bot_bottle/orchestrator/docker_broker.py

@@ -16,6 +16,7 @@ from __future__ import annotations
 import subprocess
 
 from .broker import LaunchBroker, LaunchRequest
+from .dockerutil import run_docker
 
 CONTAINER_PREFIX = "bot-bottle-orch-"
 BOTTLE_ID_LABEL = "bot-bottle-bottle-id"
@@ -51,10 +52,7 @@ class DockerBroker(LaunchBroker):
     Provenance + schema verification are inherited from `LaunchBroker`."""
 
     def _docker(self, argv: list[str]) -> subprocess.CompletedProcess[str]:
-        return subprocess.run(
-            argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-            text=True, check=False,
-        )
+        return run_docker(argv)
 
     def _launch(self, req: LaunchRequest) -> None:
         if not req.image_ref:

bot_bottle/orchestrator/dockerutil.py

@@ -0,0 +1,19 @@
+"""Shared `docker` subprocess helper for the orchestrator's docker-backed
+components (the launch broker and the consolidated sidecar)."""
+
+from __future__ import annotations
+
+import subprocess
+
+
+def run_docker(argv: list[str]) -> subprocess.CompletedProcess[str]:
+    """Run a `docker` command, capturing stdout/stderr as text. Never raises
+    on a non-zero exit — callers inspect `returncode` / `stderr` so they can
+    stay fail-closed or tolerate idempotent no-ops (e.g. removing an
+    already-absent container)."""
+    return subprocess.run(
+        argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=False,
+    )
+
+
+__all__ = ["run_docker"]

bot_bottle/orchestrator/service.py

@@ -19,17 +19,25 @@ from __future__ import annotations
 
 from .broker import LaunchBroker, LaunchRequest, sign_request
 from .registry import BottleRecord, RegistryStore
+from .sidecar import Sidecar
 
 
 class Orchestrator:
-    """Owns the registry + brokers launches. Backend-neutral."""
+    """Owns the registry + brokers launches, and manages the single
+    consolidated per-host sidecar. Backend-neutral (broker and sidecar
+    abstract the backend-native pieces)."""
 
     def __init__(
-        self, registry: RegistryStore, broker: LaunchBroker, sign_secret: bytes
+        self,
+        registry: RegistryStore,
+        broker: LaunchBroker,
+        sign_secret: bytes,
+        sidecar: Sidecar | None = None,
     ) -> None:
         self.registry = registry
         self._broker = broker
         self._secret = sign_secret
+        self._sidecar = sidecar
 
     def launch_bottle(
         self,
@@ -72,5 +80,23 @@ class Orchestrator:
         """Fail-closed attribution (delegates to the registry)."""
         return self.registry.attribute(source_ip, identity_token)
 
+    # --- consolidated sidecar ----------------------------------------------
+
+    def ensure_sidecar(self) -> None:
+        """Bring the single per-host sidecar up (idempotent). No-op when no
+        sidecar is configured."""
+        if self._sidecar is not None:
+            self._sidecar.ensure_running()
+
+    def sidecar_status(self) -> dict[str, object]:
+        """Report the shared sidecar for the control plane / console."""
+        if self._sidecar is None:
+            return {"configured": False}
+        return {
+            "configured": True,
+            "name": self._sidecar.name,
+            "running": self._sidecar.is_running(),
+        }
+
 
 __all__ = ["Orchestrator"]

bot_bottle/orchestrator/sidecar.py

@@ -0,0 +1,88 @@
+"""The consolidated per-host sidecar (PRD 0070).
+
+The core consolidation win: **one** persistent sidecar per host, shared by
+every bottle, instead of a sidecar bundle per bottle. It's safe to share
+because the attribution invariant (source IP + identity token, see
+`registry`) lets the sidecar attribute each request to the right bottle —
+so per-bottle policy lives in one long-lived process keyed on who's calling.
+
+`Sidecar` is the backend-neutral lifecycle contract (mirrors `LaunchBroker`):
+ensure the single instance is up, report it, tear it down. `DockerSidecar`
+is the docker implementation; a firecracker sidecar VM slots in later.
+
+The defining behaviour is **idempotent singleton**: `ensure_running` starts
+the instance if absent and is a no-op if it's already up, so N bottle
+launches never spawn N sidecars.
+"""
+
+from __future__ import annotations
+
+import abc
+
+from .dockerutil import run_docker
+
+SIDECAR_NAME = "bot-bottle-orch-sidecar"
+SIDECAR_LABEL = "bot-bottle-orch-sidecar=1"
+
+
+class SidecarError(Exception):
+    """The shared sidecar failed to start/stop (non-zero `docker` exit)."""
+
+
+class Sidecar(abc.ABC):
+    """Lifecycle of the single per-host sidecar. Backend-neutral."""
+
+    name: str
+
+    @abc.abstractmethod
+    def ensure_running(self) -> None:
+        """Start the sidecar if it isn't already up. Idempotent: a no-op
+        when it's already running (that's the whole point — one per host)."""
+
+    @abc.abstractmethod
+    def is_running(self) -> bool:
+        """True iff the sidecar instance is currently up."""
+
+    @abc.abstractmethod
+    def stop(self) -> None:
+        """Remove the sidecar. Idempotent — absent is success."""
+
+
+class DockerSidecar(Sidecar):
+    """The consolidated sidecar as a single, fixed-name Docker container."""
+
+    def __init__(self, image_ref: str, *, name: str = SIDECAR_NAME) -> None:
+        self.image_ref = image_ref
+        self.name = name
+
+    def is_running(self) -> bool:
+        proc = run_docker([
+            "docker", "ps",
+            "--filter", f"name=^{self.name}$",
+            "--filter", "status=running",
+            "--format", "{{.Names}}",
+        ])
+        return self.name in proc.stdout.split()
+
+    def ensure_running(self) -> None:
+        if self.is_running():
+            return
+        # Clear any stale (stopped) container holding the fixed name, then
+        # start fresh. `rm --force` on an absent name is a tolerated no-op.
+        run_docker(["docker", "rm", "--force", self.name])
+        proc = run_docker([
+            "docker", "run", "--detach",
+            "--name", self.name,
+            "--label", SIDECAR_LABEL,
+            self.image_ref,
+        ])
+        if proc.returncode != 0:
+            raise SidecarError(f"sidecar failed to start: {proc.stderr.strip()}")
+
+    def stop(self) -> None:
+        proc = run_docker(["docker", "rm", "--force", self.name])
+        if proc.returncode != 0 and "No such container" not in proc.stderr:
+            raise SidecarError(f"sidecar failed to stop: {proc.stderr.strip()}")
+
+
+__all__ = ["Sidecar", "DockerSidecar", "SidecarError", "SIDECAR_NAME", "SIDECAR_LABEL"]