fix(diagnostics): make optional failures observable and safe

bot_bottle/orchestrator/client.py

@@ -18,6 +18,7 @@ import urllib.request
 from collections.abc import Iterable
 from dataclasses import dataclass
 
+from ..log import debug
 from ..orchestrator_auth import ROLE_CLI
 from ..trust_domain import CONTROL_PLANE
 from .server import ORCHESTRATOR_AUTH_HEADER
@@ -53,6 +54,23 @@ class RegisteredBottle:
     env_var_secret: str = ""
 
 
+@dataclass(frozen=True)
+class BackendProbeFailure:
+    """Safe diagnostic for an optional backend discovery probe."""
+
+    backend: str
+    error_type: str
+
+
+def _probe_failure(backend: str, exc: BaseException) -> BackendProbeFailure:
+    failure = BackendProbeFailure(backend, type(exc).__name__)
+    debug(
+        "orchestrator discovery probe unavailable",
+        context={"backend": failure.backend, "error_type": failure.error_type},
+    )
+    return failure
+
+
 class OrchestratorClient:
     """Trusted host-side client for the orchestrator control plane.
 
@@ -245,32 +263,41 @@ def discover_orchestrator_url(*, timeout: float = 2.0) -> str:
     orchestrator TAP. Returns the first that answers `/health`; raises if none
     do (no orchestrator up — launch a bottle first)."""
     candidates: list[str] = []
+    failures: list[BackendProbeFailure] = []
     try:  # docker: loopback-published control plane
         from .lifecycle import DEFAULT_PORT as _DOCKER_PORT
         candidates.append(f"http://127.0.0.1:{_DOCKER_PORT}")
-    except Exception:  # noqa: BLE001 — backend optional
+    except Exception as exc:  # noqa: BLE001 — backend optional
+        failures.append(_probe_failure("docker", exc))
         candidates.append("http://127.0.0.1:8099")
     try:  # firecracker: infra VM control plane on the orchestrator TAP
         from ..backend.firecracker import netpool
         from ..backend.firecracker.infra_vm import ORCHESTRATOR_PORT
         candidates.append(
             f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}")
-    except Exception:  # noqa: BLE001 — backend optional / not firecracker
-        pass
+    except Exception as exc:  # noqa: BLE001 — backend optional / not firecracker
+        failures.append(_probe_failure("firecracker", exc))
     try:  # macOS: orchestrator container on its host-only address
         from ..backend.macos_container.infra import probe_orchestrator_url
         url = probe_orchestrator_url()
         if url:
             candidates.append(url)
-    except Exception:  # noqa: BLE001 — backend optional / not macOS
-        pass
+    except Exception as exc:  # noqa: BLE001 — backend optional / not macOS
+        failures.append(_probe_failure("macos-container", exc))
     for url in candidates:
         if OrchestratorClient(url, timeout=timeout).health():
             return url
+    detail = ""
+    if failures:
+        detail = "; optional probes unavailable: " + ", ".join(
+            f"{failure.backend} ({failure.error_type})" for failure in failures
+        )
     raise OrchestratorClientError(
         "no running orchestrator control plane found (tried "
         + ", ".join(candidates)
-        + "); launch a bottle first"
+        + ")"
+        + detail
+        + "; launch a bottle first"
     )
 
 

bot_bottle/orchestrator/reprovision.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from ..log import debug
 from .client import OrchestratorClient, OrchestratorClientError
 
 
@@ -27,7 +28,14 @@ def reprovision_bottles(
         try:
             if client.reprovision_gateway(bottle_id, secret):
                 restored += 1
-        except OrchestratorClientError:
+        except OrchestratorClientError as exc:
+            debug(
+                "gateway secret reprovision failed; continuing with other bottles",
+                context={
+                    "bottle_id": bottle_id,
+                    "error_type": type(exc).__name__,
+                },
+            )
             continue
     return restored
 

bot_bottle/orchestrator/server.py

@@ -379,9 +379,15 @@ class Handler(http.server.BaseHTTPRequestHandler):
             status, payload = dispatch(
                 server.orchestrator, method, self.path, body, role=role)
         except Exception as e:  # noqa: BLE001 — the control plane must stay up
-            sys.stderr.write(f"orchestrator: {method} {self.path} failed: {e!r}\n")
+            # Do not echo exception messages to the caller or logs: broker and
+            # persistence exceptions can contain request data. The operation,
+            # route, and exception type are enough to correlate a traceback.
+            sys.stderr.write(
+                f"orchestrator: {method} {self.path} failed "
+                f"[error_type={type(e).__name__}]\n"
+            )
             sys.stderr.flush()
-            status, payload = 500, {"error": f"internal error: {e}"}
+            status, payload = 500, {"error": "internal error"}
         data = json.dumps(payload).encode()
         self.send_response(status)
         self.send_header("Content-Type", "application/json")