fix(diagnostics): make optional failures observable and safe

This commit is contained in:
2026-07-26 06:31:38 +00:00
parent e29b79d517
commit 22dde95561
12 changed files with 180 additions and 46 deletions
+18 -18
View File
@@ -368,7 +368,7 @@ def _main_loop(stdscr: "curses._CursesWindow") -> None: # type: ignore # pragm
elif key in (curses.KEY_UP, ord("k")):
selected = max(selected - 1, 0)
elif key in (curses.KEY_ENTER, 10, 13):
_detail_view(stdscr, qp, green_attr=green_attr)
status_line = _detail_view(stdscr, qp, green_attr=green_attr)
elif key == ord("a"):
try:
status_line = _approve_from_tui(stdscr, qp)
@@ -456,7 +456,7 @@ def _detail_view(
qp: QueuedProposal,
*,
green_attr: int = 0,
) -> None: # pragma: no cover
) -> str: # pragma: no cover
"""Render the full proposal. Scrollable. Press q to return."""
lines = _detail_lines(qp, green_attr=green_attr)
offset = 0
@@ -473,7 +473,7 @@ def _detail_view(
stdscr.refresh()
key = stdscr.getch()
if key in (ord("q"), 27):
return
return ""
if key in (curses.KEY_DOWN, ord("j")):
offset = min(offset + 1, max(0, len(lines) - 1))
elif key in (curses.KEY_UP, ord("k")):
@@ -484,28 +484,28 @@ def _detail_view(
offset = max(0, len(lines) - 1)
elif key == ord("a"):
try:
_approve_from_tui(stdscr, qp)
except ApplyError:
pass
return
return _approve_from_tui(stdscr, qp)
except ApplyError as exc:
return f"apply failed: {exc}"
elif key == ord("m"):
if qp.proposal.tool in _REPORT_ONLY_TOOLS:
return
return f"modify unavailable for {qp.proposal.tool}"
edited = _modify(stdscr, qp)
if edited is not None:
try:
_approve_from_tui(
stdscr, qp, final_file=edited,
notes="operator modified before approving",
)
except ApplyError:
pass
return
if edited is None:
return "modify aborted (no change)"
try:
return _approve_from_tui(
stdscr, qp, final_file=edited,
notes="operator modified before approving",
)
except ApplyError as exc:
return f"apply failed: {exc}"
elif key == ord("r"):
reason = _prompt(stdscr, "reject reason: ")
if reason:
reject(qp, reason=reason)
return
return f"rejected {qp.proposal.tool} for [{qp.label}]"
return "reject aborted (empty reason)"
def _modify(stdscr: "curses._CursesWindow", qp: QueuedProposal) -> str | None: # type: ignore # pragma: no cover
+32 -6
View File
@@ -16,6 +16,8 @@ import os
import sys
from typing import Any, Optional
from ..log import debug
def filter_multiselect(
items: list[str],
@@ -42,7 +44,11 @@ def filter_multiselect(
try:
tty_fd = open(tty_path, "r+b", buffering=0)
except OSError:
except OSError as exc:
debug(
"multi-select unavailable; treating it as cancellation",
context={"error_type": type(exc).__name__, "tty": tty_path},
)
return None
try:
@@ -73,7 +79,11 @@ def filter_select(
try:
tty_fd = open(tty_path, "r+b", buffering=0)
except OSError:
except OSError as exc:
debug(
"filter-select unavailable; treating it as cancellation",
context={"error_type": type(exc).__name__, "tty": tty_path},
)
return None
try:
@@ -129,7 +139,11 @@ def _run_picker(items: list[str], *, title: str, tty_fd: int) -> Optional[str]:
curses.nocbreak()
curses.echo()
curses.endwin()
except Exception: # noqa: W0718 — curses can raise many error types
except Exception as exc: # noqa: W0718 — curses can raise many error types
debug(
"filter-select display failed; treating it as cancellation",
context={"error_type": type(exc).__name__},
)
return None
finally:
sys.__stdin__ = orig_stdin # type: ignore[assignment]
@@ -292,7 +306,11 @@ def _run_multiselect(
curses.nocbreak()
curses.echo()
curses.endwin()
except Exception: # noqa: W0718
except Exception as exc: # noqa: W0718
debug(
"multi-select display failed; treating it as cancellation",
context={"error_type": type(exc).__name__},
)
return None
finally:
sys.__stdin__ = orig_stdin # type: ignore[assignment]
@@ -558,13 +576,21 @@ def name_color_modal(
"""
try:
tty_fd = open(tty_path, "r+b", buffering=0) # pylint: disable=consider-using-with
except OSError:
except OSError as exc:
debug(
"name/color picker unavailable; using defaults",
context={"error_type": type(exc).__name__, "tty": tty_path},
)
return default_label, ""
try:
fd_dup = os.dup(tty_fd.fileno())
return _run_name_color(default_label, tty_fd=fd_dup, disclaimer=disclaimer)
except Exception: # noqa: BLE001 # pylint: disable=broad-exception-caught
except Exception as exc: # noqa: BLE001 # pylint: disable=broad-exception-caught
debug(
"name/color picker failed; using defaults",
context={"error_type": type(exc).__name__},
)
return default_label, ""
finally:
tty_fd.close()
+11 -2
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import typing
from ...log import debug
from .types import Config
@@ -52,7 +53,11 @@ def resolve_client_config(
) -> Config:
try:
policy = resolver.resolve(client_ip, identity_token)
except Exception: # noqa: BLE001 - a policy lookup failure must deny
except Exception as exc: # noqa: BLE001 - a policy lookup failure must deny
debug(
"egress policy resolution failed; applying deny-all",
context={"error_type": type(exc).__name__},
)
return Config(routes=(), deny_reason=DENY_RESOLVER_ERROR)
return _config_from_policy(policy)
@@ -63,6 +68,10 @@ def resolve_client_context(
try:
policy, bottle_id, tokens = resolver.resolve_policy_and_bottle_id(
client_ip, identity_token)
except Exception: # noqa: BLE001 - a policy lookup failure must deny
except Exception as exc: # noqa: BLE001 - a policy lookup failure must deny
debug(
"egress context resolution failed; applying deny-all",
context={"error_type": type(exc).__name__},
)
return Config(routes=(), deny_reason=DENY_RESOLVER_ERROR), "", {}
return _config_from_policy(policy), (bottle_id or ""), tokens
+33 -6
View File
@@ -18,6 +18,7 @@ import urllib.request
from collections.abc import Iterable
from dataclasses import dataclass
from ..log import debug
from ..orchestrator_auth import ROLE_CLI
from ..trust_domain import CONTROL_PLANE
from .server import ORCHESTRATOR_AUTH_HEADER
@@ -53,6 +54,23 @@ class RegisteredBottle:
env_var_secret: str = ""
@dataclass(frozen=True)
class BackendProbeFailure:
"""Safe diagnostic for an optional backend discovery probe."""
backend: str
error_type: str
def _probe_failure(backend: str, exc: BaseException) -> BackendProbeFailure:
failure = BackendProbeFailure(backend, type(exc).__name__)
debug(
"orchestrator discovery probe unavailable",
context={"backend": failure.backend, "error_type": failure.error_type},
)
return failure
class OrchestratorClient:
"""Trusted host-side client for the orchestrator control plane.
@@ -245,32 +263,41 @@ def discover_orchestrator_url(*, timeout: float = 2.0) -> str:
orchestrator TAP. Returns the first that answers `/health`; raises if none
do (no orchestrator up — launch a bottle first)."""
candidates: list[str] = []
failures: list[BackendProbeFailure] = []
try: # docker: loopback-published control plane
from .lifecycle import DEFAULT_PORT as _DOCKER_PORT
candidates.append(f"http://127.0.0.1:{_DOCKER_PORT}")
except Exception: # noqa: BLE001 — backend optional
except Exception as exc: # noqa: BLE001 — backend optional
failures.append(_probe_failure("docker", exc))
candidates.append("http://127.0.0.1:8099")
try: # firecracker: infra VM control plane on the orchestrator TAP
from ..backend.firecracker import netpool
from ..backend.firecracker.infra_vm import ORCHESTRATOR_PORT
candidates.append(
f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}")
except Exception: # noqa: BLE001 — backend optional / not firecracker
pass
except Exception as exc: # noqa: BLE001 — backend optional / not firecracker
failures.append(_probe_failure("firecracker", exc))
try: # macOS: orchestrator container on its host-only address
from ..backend.macos_container.infra import probe_orchestrator_url
url = probe_orchestrator_url()
if url:
candidates.append(url)
except Exception: # noqa: BLE001 — backend optional / not macOS
pass
except Exception as exc: # noqa: BLE001 — backend optional / not macOS
failures.append(_probe_failure("macos-container", exc))
for url in candidates:
if OrchestratorClient(url, timeout=timeout).health():
return url
detail = ""
if failures:
detail = "; optional probes unavailable: " + ", ".join(
f"{failure.backend} ({failure.error_type})" for failure in failures
)
raise OrchestratorClientError(
"no running orchestrator control plane found (tried "
+ ", ".join(candidates)
+ "); launch a bottle first"
+ ")"
+ detail
+ "; launch a bottle first"
)
+9 -1
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
from ..log import debug
from .client import OrchestratorClient, OrchestratorClientError
@@ -27,7 +28,14 @@ def reprovision_bottles(
try:
if client.reprovision_gateway(bottle_id, secret):
restored += 1
except OrchestratorClientError:
except OrchestratorClientError as exc:
debug(
"gateway secret reprovision failed; continuing with other bottles",
context={
"bottle_id": bottle_id,
"error_type": type(exc).__name__,
},
)
continue
return restored
+8 -2
View File
@@ -379,9 +379,15 @@ class Handler(http.server.BaseHTTPRequestHandler):
status, payload = dispatch(
server.orchestrator, method, self.path, body, role=role)
except Exception as e: # noqa: BLE001 — the control plane must stay up
sys.stderr.write(f"orchestrator: {method} {self.path} failed: {e!r}\n")
# Do not echo exception messages to the caller or logs: broker and
# persistence exceptions can contain request data. The operation,
# route, and exception type are enough to correlate a traceback.
sys.stderr.write(
f"orchestrator: {method} {self.path} failed "
f"[error_type={type(e).__name__}]\n"
)
sys.stderr.flush()
status, payload = 500, {"error": f"internal error: {e}"}
status, payload = 500, {"error": "internal error"}
data = json.dumps(payload).encode()
self.send_response(status)
self.send_header("Content-Type", "application/json")