fix(pi_rpc): detect a dead pi process immediately, not at the turn deadline

When pi died mid-turn, _read_stdout saw EOF and logged it, but nothing woke
_consume -- it sat on the event queue until asyncio.timeout(budget) fired at the
turn deadline (180 s). A dead process at second 3 was therefore reported as "no
reply" at second 180, and the fallback that launched a fresh pi hid the death
entirely. To the user this read as a hard hang.

_read_stdout now enqueues a "_process_exited" sentinel on EOF, and _consume
raises PiRpcError on it, so the failure path (same-session fallback model, then
rotation, then a fresh process) starts within milliseconds.

Smoke-tested: SIGKILL the child during a command and the client reports it in
0.0 s instead of the full budget, with no orphan left behind.
This commit is contained in:
Kai
2026-08-28 21:17:21 -07:00
parent 6de252fe5b
commit 943f631966
2 changed files with 42 additions and 2 deletions
+14 -1
View File
@@ -445,7 +445,13 @@ class PiRpcClient:
await result await result
except Exception: except Exception:
LOG.exception("on_event callback failed") LOG.exception("on_event callback failed")
LOG.warning("pi RPC stdout closed") # EOF: the child died or closed its pipe. Without a sentinel, _consume
# would sit on _events.get() until the turn deadline -- a process that
# exited at second 3 would be reported as "no reply" at second 180, and a
# fallback that launches a fresh pi would hide the death entirely. Put a
# sentinel so the waiter fails immediately instead.
LOG.warning("pi RPC stdout closed pid=%s rc=%s", self.pid, self._process.returncode)
await self._events.put({"type": "_process_exited"})
async def _read_stderr(self) -> None: async def _read_stderr(self) -> None:
assert self._process and self._process.stderr assert self._process and self._process.stderr
@@ -528,6 +534,13 @@ class PiRpcClient:
event = await self._events.get() event = await self._events.get()
kind = event.get("type") kind = event.get("type")
if kind == "_process_exited":
rc = self._process.returncode if self._process else None
raise PiRpcError(
f"pi process exited (pid={self.pid}, returncode={rc}) while "
f"awaiting {request_id}; stderr tail: {' | '.join(self._stderr_tail[-3:])}"
)
if kind == "response" and event.get("id") == request_id: if kind == "response" and event.get("id") == request_id:
response = event response = event
if not wait_for_settled or not event.get("success"): if not wait_for_settled or not event.get("success"):
+28 -1
View File
@@ -10,6 +10,8 @@ from __future__ import annotations
import asyncio import asyncio
import os import os
import signal
import os
import subprocess import subprocess
import sys import sys
from pathlib import Path from pathlib import Path
@@ -19,7 +21,7 @@ from dataclasses import replace # noqa: E402
import pathlib # noqa: E402 import pathlib # noqa: E402
import tempfile # noqa: E402 import tempfile # noqa: E402
from pi_rpc import PiLaunchConfig, PiRpcClient # noqa: E402 from pi_rpc import PiLaunchConfig, PiRpcClient, PiRpcError # noqa: E402
REPO = Path(__file__).resolve().parents[4] REPO = Path(__file__).resolve().parents[4]
HARNESS = REPO / "docs" / "evidence" / "probe-harness" HARNESS = REPO / "docs" / "evidence" / "probe-harness"
@@ -107,6 +109,31 @@ async def main() -> int:
alive = subprocess.run(["ps", "-p", str(pid)], capture_output=True).returncode == 0 alive = subprocess.run(["ps", "-p", str(pid)], capture_output=True).returncode == 0
failures += not check("no orphan process", not alive) failures += not check("no orphan process", not alive)
# A process that dies mid-command must be reported immediately, not at the
# turn deadline. Before the sentinel this sat on the event queue for the full
# 180 s budget; a fallback that launches a fresh pi then hid the death.
print("== process exit is detected immediately ==")
client2 = PiRpcClient(cfg)
await client2.start()
import time as _time
try:
os.killpg(os.getpgid(client2.pid), signal.SIGKILL)
started = _time.monotonic()
try:
await asyncio.wait_for(client2.get_session_stats(), timeout=10)
failures += not check("dead process reported", False, "command succeeded after SIGKILL")
except PiRpcError:
elapsed = _time.monotonic() - started
failures += not check("dead process reported promptly", elapsed < 10,
f"{elapsed:.1f}s (must be well under the 180 s budget)")
finally:
try:
await client2.stop()
except Exception:
pass
failures += not check("no orphan from killed client", not client2.running)
print() print()
print("RESULT:", "all checks passed" if failures == 0 else f"{failures} check(s) failed") print("RESULT:", "all checks passed" if failures == 0 else f"{failures} check(s) failed")
return 1 if failures else 0 return 1 if failures else 0