fix(gateway): restart dead children before completion check
lint / lint (push) Successful in 58s

This commit is contained in:
2026-07-23 18:55:27 -04:00
parent 44122e2728
commit e0423b9b3c
+16 -6
View File
@@ -9,8 +9,7 @@ Failure policy: when a child dies unexpectedly, the supervisor
restarts it automatically and logs the restart. The gateway stays restarts it automatically and logs the restart. The gateway stays
up; a temporary loss of one daemon (e.g. egress OOM-killed) is up; a temporary loss of one daemon (e.g. egress OOM-killed) is
recovered without manual container recreation. The supervisor recovered without manual container recreation. The supervisor
itself exits only when (a) the operator sends SIGTERM/SIGINT, or itself exits only when the operator sends SIGTERM/SIGINT.
(b) every child has died.
Daemon subset is env-driven via `BOT_BOTTLE_GATEWAY_DAEMONS=egress` Daemon subset is env-driven via `BOT_BOTTLE_GATEWAY_DAEMONS=egress`
for callers that don't use git-gate or supervise. Default: all for callers that don't use git-gate or supervise. Default: all
@@ -221,9 +220,10 @@ class _Supervisor:
"""One iteration of the watch loop. Returns True when every """One iteration of the watch loop. Returns True when every
child has exited and the supervisor can return. child has exited and the supervisor can return.
A child dying unexpectedly is logged but does NOT initiate A child dying unexpectedly is logged and restarted but does
shutdown — see the module docstring's failure-policy NOT initiate shutdown — see the module docstring's
section. Shutdown is signal-driven only.""" failure-policy section. Shutdown is signal-driven only."""
restarted_children = bool(self._restart_requested)
self._drain_restart_requests() self._drain_restart_requests()
for spec, p in self.procs: for spec, p in self.procs:
@@ -237,6 +237,13 @@ class _Supervisor:
else: else:
_log(f"{spec.name} exited with code {rc}") _log(f"{spec.name} exited with code {rc}")
# Restart deaths discovered above before checking whether all
# processes are done. Deferring this until the next tick would make a
# single-daemon supervisor return True and exit with the restart still
# queued.
restarted_children |= bool(self._restart_requested)
self._drain_restart_requests()
if self.shutdown_at is not None: if self.shutdown_at is not None:
elapsed = time.monotonic() - self.shutdown_at elapsed = time.monotonic() - self.shutdown_at
if elapsed > _GRACE_SECONDS: if elapsed > _GRACE_SECONDS:
@@ -250,7 +257,10 @@ class _Supervisor:
) )
self._sigkill_all() self._sigkill_all()
done = all(p.poll() is not None for _, p in self.procs) done = (
not restarted_children
and all(p.poll() is not None for _, p in self.procs)
)
if done: if done:
for _, p in self.procs: for _, p in self.procs:
if p.stdout is not None: if p.stdout is not None: