fix(diagnostics): make optional failures observable and safe
This commit is contained in:
@@ -368,7 +368,7 @@ def _main_loop(stdscr: "curses._CursesWindow") -> None: # type: ignore # pragm
|
||||
elif key in (curses.KEY_UP, ord("k")):
|
||||
selected = max(selected - 1, 0)
|
||||
elif key in (curses.KEY_ENTER, 10, 13):
|
||||
_detail_view(stdscr, qp, green_attr=green_attr)
|
||||
status_line = _detail_view(stdscr, qp, green_attr=green_attr)
|
||||
elif key == ord("a"):
|
||||
try:
|
||||
status_line = _approve_from_tui(stdscr, qp)
|
||||
@@ -456,7 +456,7 @@ def _detail_view(
|
||||
qp: QueuedProposal,
|
||||
*,
|
||||
green_attr: int = 0,
|
||||
) -> None: # pragma: no cover
|
||||
) -> str: # pragma: no cover
|
||||
"""Render the full proposal. Scrollable. Press q to return."""
|
||||
lines = _detail_lines(qp, green_attr=green_attr)
|
||||
offset = 0
|
||||
@@ -473,7 +473,7 @@ def _detail_view(
|
||||
stdscr.refresh()
|
||||
key = stdscr.getch()
|
||||
if key in (ord("q"), 27):
|
||||
return
|
||||
return ""
|
||||
if key in (curses.KEY_DOWN, ord("j")):
|
||||
offset = min(offset + 1, max(0, len(lines) - 1))
|
||||
elif key in (curses.KEY_UP, ord("k")):
|
||||
@@ -484,28 +484,28 @@ def _detail_view(
|
||||
offset = max(0, len(lines) - 1)
|
||||
elif key == ord("a"):
|
||||
try:
|
||||
_approve_from_tui(stdscr, qp)
|
||||
except ApplyError:
|
||||
pass
|
||||
return
|
||||
return _approve_from_tui(stdscr, qp)
|
||||
except ApplyError as exc:
|
||||
return f"apply failed: {exc}"
|
||||
elif key == ord("m"):
|
||||
if qp.proposal.tool in _REPORT_ONLY_TOOLS:
|
||||
return
|
||||
return f"modify unavailable for {qp.proposal.tool}"
|
||||
edited = _modify(stdscr, qp)
|
||||
if edited is not None:
|
||||
try:
|
||||
_approve_from_tui(
|
||||
stdscr, qp, final_file=edited,
|
||||
notes="operator modified before approving",
|
||||
)
|
||||
except ApplyError:
|
||||
pass
|
||||
return
|
||||
if edited is None:
|
||||
return "modify aborted (no change)"
|
||||
try:
|
||||
return _approve_from_tui(
|
||||
stdscr, qp, final_file=edited,
|
||||
notes="operator modified before approving",
|
||||
)
|
||||
except ApplyError as exc:
|
||||
return f"apply failed: {exc}"
|
||||
elif key == ord("r"):
|
||||
reason = _prompt(stdscr, "reject reason: ")
|
||||
if reason:
|
||||
reject(qp, reason=reason)
|
||||
return
|
||||
return f"rejected {qp.proposal.tool} for [{qp.label}]"
|
||||
return "reject aborted (empty reason)"
|
||||
|
||||
|
||||
def _modify(stdscr: "curses._CursesWindow", qp: QueuedProposal) -> str | None: # type: ignore # pragma: no cover
|
||||
|
||||
+32
-6
@@ -16,6 +16,8 @@ import os
|
||||
import sys
|
||||
from typing import Any, Optional
|
||||
|
||||
from ..log import debug
|
||||
|
||||
|
||||
def filter_multiselect(
|
||||
items: list[str],
|
||||
@@ -42,7 +44,11 @@ def filter_multiselect(
|
||||
|
||||
try:
|
||||
tty_fd = open(tty_path, "r+b", buffering=0)
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
debug(
|
||||
"multi-select unavailable; treating it as cancellation",
|
||||
context={"error_type": type(exc).__name__, "tty": tty_path},
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
@@ -73,7 +79,11 @@ def filter_select(
|
||||
|
||||
try:
|
||||
tty_fd = open(tty_path, "r+b", buffering=0)
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
debug(
|
||||
"filter-select unavailable; treating it as cancellation",
|
||||
context={"error_type": type(exc).__name__, "tty": tty_path},
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
@@ -129,7 +139,11 @@ def _run_picker(items: list[str], *, title: str, tty_fd: int) -> Optional[str]:
|
||||
curses.nocbreak()
|
||||
curses.echo()
|
||||
curses.endwin()
|
||||
except Exception: # noqa: W0718 — curses can raise many error types
|
||||
except Exception as exc: # noqa: W0718 — curses can raise many error types
|
||||
debug(
|
||||
"filter-select display failed; treating it as cancellation",
|
||||
context={"error_type": type(exc).__name__},
|
||||
)
|
||||
return None
|
||||
finally:
|
||||
sys.__stdin__ = orig_stdin # type: ignore[assignment]
|
||||
@@ -292,7 +306,11 @@ def _run_multiselect(
|
||||
curses.nocbreak()
|
||||
curses.echo()
|
||||
curses.endwin()
|
||||
except Exception: # noqa: W0718
|
||||
except Exception as exc: # noqa: W0718
|
||||
debug(
|
||||
"multi-select display failed; treating it as cancellation",
|
||||
context={"error_type": type(exc).__name__},
|
||||
)
|
||||
return None
|
||||
finally:
|
||||
sys.__stdin__ = orig_stdin # type: ignore[assignment]
|
||||
@@ -558,13 +576,21 @@ def name_color_modal(
|
||||
"""
|
||||
try:
|
||||
tty_fd = open(tty_path, "r+b", buffering=0) # pylint: disable=consider-using-with
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
debug(
|
||||
"name/color picker unavailable; using defaults",
|
||||
context={"error_type": type(exc).__name__, "tty": tty_path},
|
||||
)
|
||||
return default_label, ""
|
||||
|
||||
try:
|
||||
fd_dup = os.dup(tty_fd.fileno())
|
||||
return _run_name_color(default_label, tty_fd=fd_dup, disclaimer=disclaimer)
|
||||
except Exception: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
||||
except Exception as exc: # noqa: BLE001 # pylint: disable=broad-exception-caught
|
||||
debug(
|
||||
"name/color picker failed; using defaults",
|
||||
context={"error_type": type(exc).__name__},
|
||||
)
|
||||
return default_label, ""
|
||||
finally:
|
||||
tty_fd.close()
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
from ...log import debug
|
||||
from .types import Config
|
||||
|
||||
|
||||
@@ -52,7 +53,11 @@ def resolve_client_config(
|
||||
) -> Config:
|
||||
try:
|
||||
policy = resolver.resolve(client_ip, identity_token)
|
||||
except Exception: # noqa: BLE001 - a policy lookup failure must deny
|
||||
except Exception as exc: # noqa: BLE001 - a policy lookup failure must deny
|
||||
debug(
|
||||
"egress policy resolution failed; applying deny-all",
|
||||
context={"error_type": type(exc).__name__},
|
||||
)
|
||||
return Config(routes=(), deny_reason=DENY_RESOLVER_ERROR)
|
||||
return _config_from_policy(policy)
|
||||
|
||||
@@ -63,6 +68,10 @@ def resolve_client_context(
|
||||
try:
|
||||
policy, bottle_id, tokens = resolver.resolve_policy_and_bottle_id(
|
||||
client_ip, identity_token)
|
||||
except Exception: # noqa: BLE001 - a policy lookup failure must deny
|
||||
except Exception as exc: # noqa: BLE001 - a policy lookup failure must deny
|
||||
debug(
|
||||
"egress context resolution failed; applying deny-all",
|
||||
context={"error_type": type(exc).__name__},
|
||||
)
|
||||
return Config(routes=(), deny_reason=DENY_RESOLVER_ERROR), "", {}
|
||||
return _config_from_policy(policy), (bottle_id or ""), tokens
|
||||
|
||||
@@ -18,6 +18,7 @@ import urllib.request
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from ..log import debug
|
||||
from ..orchestrator_auth import ROLE_CLI
|
||||
from ..trust_domain import CONTROL_PLANE
|
||||
from .server import ORCHESTRATOR_AUTH_HEADER
|
||||
@@ -53,6 +54,23 @@ class RegisteredBottle:
|
||||
env_var_secret: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BackendProbeFailure:
|
||||
"""Safe diagnostic for an optional backend discovery probe."""
|
||||
|
||||
backend: str
|
||||
error_type: str
|
||||
|
||||
|
||||
def _probe_failure(backend: str, exc: BaseException) -> BackendProbeFailure:
|
||||
failure = BackendProbeFailure(backend, type(exc).__name__)
|
||||
debug(
|
||||
"orchestrator discovery probe unavailable",
|
||||
context={"backend": failure.backend, "error_type": failure.error_type},
|
||||
)
|
||||
return failure
|
||||
|
||||
|
||||
class OrchestratorClient:
|
||||
"""Trusted host-side client for the orchestrator control plane.
|
||||
|
||||
@@ -245,32 +263,41 @@ def discover_orchestrator_url(*, timeout: float = 2.0) -> str:
|
||||
orchestrator TAP. Returns the first that answers `/health`; raises if none
|
||||
do (no orchestrator up — launch a bottle first)."""
|
||||
candidates: list[str] = []
|
||||
failures: list[BackendProbeFailure] = []
|
||||
try: # docker: loopback-published control plane
|
||||
from .lifecycle import DEFAULT_PORT as _DOCKER_PORT
|
||||
candidates.append(f"http://127.0.0.1:{_DOCKER_PORT}")
|
||||
except Exception: # noqa: BLE001 — backend optional
|
||||
except Exception as exc: # noqa: BLE001 — backend optional
|
||||
failures.append(_probe_failure("docker", exc))
|
||||
candidates.append("http://127.0.0.1:8099")
|
||||
try: # firecracker: infra VM control plane on the orchestrator TAP
|
||||
from ..backend.firecracker import netpool
|
||||
from ..backend.firecracker.infra_vm import ORCHESTRATOR_PORT
|
||||
candidates.append(
|
||||
f"http://{netpool.orch_slot().guest_ip}:{ORCHESTRATOR_PORT}")
|
||||
except Exception: # noqa: BLE001 — backend optional / not firecracker
|
||||
pass
|
||||
except Exception as exc: # noqa: BLE001 — backend optional / not firecracker
|
||||
failures.append(_probe_failure("firecracker", exc))
|
||||
try: # macOS: orchestrator container on its host-only address
|
||||
from ..backend.macos_container.infra import probe_orchestrator_url
|
||||
url = probe_orchestrator_url()
|
||||
if url:
|
||||
candidates.append(url)
|
||||
except Exception: # noqa: BLE001 — backend optional / not macOS
|
||||
pass
|
||||
except Exception as exc: # noqa: BLE001 — backend optional / not macOS
|
||||
failures.append(_probe_failure("macos-container", exc))
|
||||
for url in candidates:
|
||||
if OrchestratorClient(url, timeout=timeout).health():
|
||||
return url
|
||||
detail = ""
|
||||
if failures:
|
||||
detail = "; optional probes unavailable: " + ", ".join(
|
||||
f"{failure.backend} ({failure.error_type})" for failure in failures
|
||||
)
|
||||
raise OrchestratorClientError(
|
||||
"no running orchestrator control plane found (tried "
|
||||
+ ", ".join(candidates)
|
||||
+ "); launch a bottle first"
|
||||
+ ")"
|
||||
+ detail
|
||||
+ "; launch a bottle first"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..log import debug
|
||||
from .client import OrchestratorClient, OrchestratorClientError
|
||||
|
||||
|
||||
@@ -27,7 +28,14 @@ def reprovision_bottles(
|
||||
try:
|
||||
if client.reprovision_gateway(bottle_id, secret):
|
||||
restored += 1
|
||||
except OrchestratorClientError:
|
||||
except OrchestratorClientError as exc:
|
||||
debug(
|
||||
"gateway secret reprovision failed; continuing with other bottles",
|
||||
context={
|
||||
"bottle_id": bottle_id,
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
)
|
||||
continue
|
||||
return restored
|
||||
|
||||
|
||||
@@ -379,9 +379,15 @@ class Handler(http.server.BaseHTTPRequestHandler):
|
||||
status, payload = dispatch(
|
||||
server.orchestrator, method, self.path, body, role=role)
|
||||
except Exception as e: # noqa: BLE001 — the control plane must stay up
|
||||
sys.stderr.write(f"orchestrator: {method} {self.path} failed: {e!r}\n")
|
||||
# Do not echo exception messages to the caller or logs: broker and
|
||||
# persistence exceptions can contain request data. The operation,
|
||||
# route, and exception type are enough to correlate a traceback.
|
||||
sys.stderr.write(
|
||||
f"orchestrator: {method} {self.path} failed "
|
||||
f"[error_type={type(e).__name__}]\n"
|
||||
)
|
||||
sys.stderr.flush()
|
||||
status, payload = 500, {"error": f"internal error: {e}"}
|
||||
status, payload = 500, {"error": "internal error"}
|
||||
data = json.dumps(payload).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
|
||||
Reference in New Issue
Block a user