diff --git a/shared/lib/py/pi_rpc.py b/shared/lib/py/pi_rpc.py index 7e2aed3..a7d8c8f 100644 --- a/shared/lib/py/pi_rpc.py +++ b/shared/lib/py/pi_rpc.py @@ -445,7 +445,13 @@ class PiRpcClient: await result except Exception: LOG.exception("on_event callback failed") - LOG.warning("pi RPC stdout closed") + # EOF: the child died or closed its pipe. Without a sentinel, _consume + # would sit on _events.get() until the turn deadline -- a process that + # exited at second 3 would be reported as "no reply" at second 180, and a + # fallback that launches a fresh pi would hide the death entirely. Put a + # sentinel so the waiter fails immediately instead. + LOG.warning("pi RPC stdout closed pid=%s rc=%s", self.pid, self._process.returncode) + await self._events.put({"type": "_process_exited"}) async def _read_stderr(self) -> None: assert self._process and self._process.stderr @@ -528,6 +534,13 @@ class PiRpcClient: event = await self._events.get() kind = event.get("type") + if kind == "_process_exited": + rc = self._process.returncode if self._process else None + raise PiRpcError( + f"pi process exited (pid={self.pid}, returncode={rc}) while " + f"awaiting {request_id}; stderr tail: {' | '.join(self._stderr_tail[-3:])}" + ) + if kind == "response" and event.get("id") == request_id: response = event if not wait_for_settled or not event.get("success"): diff --git a/shared/lib/py/tests/test_pi_rpc_smoke.py b/shared/lib/py/tests/test_pi_rpc_smoke.py index 2346071..09155d6 100644 --- a/shared/lib/py/tests/test_pi_rpc_smoke.py +++ b/shared/lib/py/tests/test_pi_rpc_smoke.py @@ -10,6 +10,8 @@ from __future__ import annotations import asyncio import os +import signal +import os import subprocess import sys from pathlib import Path @@ -19,7 +21,7 @@ from dataclasses import replace # noqa: E402 import pathlib # noqa: E402 import tempfile # noqa: E402 -from pi_rpc import PiLaunchConfig, PiRpcClient # noqa: E402 +from pi_rpc import PiLaunchConfig, PiRpcClient, PiRpcError # noqa: E402 REPO = Path(__file__).resolve().parents[4] HARNESS = REPO / "docs" / "evidence" / "probe-harness" @@ -107,6 +109,31 @@ async def main() -> int: alive = subprocess.run(["ps", "-p", str(pid)], capture_output=True).returncode == 0 failures += not check("no orphan process", not alive) + # A process that dies mid-command must be reported immediately, not at the + # turn deadline. Before the sentinel this sat on the event queue for the full + # 180 s budget; a fallback that launches a fresh pi then hid the death. + print("== process exit is detected immediately ==") + client2 = PiRpcClient(cfg) + await client2.start() + import time as _time + try: + os.killpg(os.getpgid(client2.pid), signal.SIGKILL) + started = _time.monotonic() + try: + await asyncio.wait_for(client2.get_session_stats(), timeout=10) + failures += not check("dead process reported", False, "command succeeded after SIGKILL") + except PiRpcError: + elapsed = _time.monotonic() - started + failures += not check("dead process reported promptly", elapsed < 10, + f"{elapsed:.1f}s (must be well under the 180 s budget)") + finally: + try: + await client2.stop() + except Exception: + pass + + failures += not check("no orphan from killed client", not client2.running) + print() print("RESULT:", "all checks passed" if failures == 0 else f"{failures} check(s) failed") return 1 if failures else 0