fix(pi_rpc): detect a dead pi process immediately, not at the turn deadline

When pi died mid-turn, _read_stdout saw EOF and logged it, but nothing woke
_consume -- it sat on the event queue until asyncio.timeout(budget) fired at the
turn deadline (180 s). A dead process at second 3 was therefore reported as "no
reply" at second 180, and the fallback that launched a fresh pi hid the death
entirely. To the user this read as a hard hang.

_read_stdout now enqueues a "_process_exited" sentinel on EOF, and _consume
raises PiRpcError on it, so the failure path (same-session fallback model, then
rotation, then a fresh process) starts within milliseconds.

Smoke-tested: SIGKILL the child during a command and the client reports it in
0.0 s instead of the full budget, with no orphan left behind.
This commit is contained in:
Kai
2026-08-28 21:17:21 -07:00
parent 6de252fe5b
commit 943f631966
2 changed files with 42 additions and 2 deletions
+28 -1
View File
@@ -10,6 +10,8 @@ from __future__ import annotations
import asyncio
import os
import signal
import os
import subprocess
import sys
from pathlib import Path
@@ -19,7 +21,7 @@ from dataclasses import replace # noqa: E402
import pathlib # noqa: E402
import tempfile # noqa: E402
from pi_rpc import PiLaunchConfig, PiRpcClient # noqa: E402
from pi_rpc import PiLaunchConfig, PiRpcClient, PiRpcError # noqa: E402
REPO = Path(__file__).resolve().parents[4]
HARNESS = REPO / "docs" / "evidence" / "probe-harness"
@@ -107,6 +109,31 @@ async def main() -> int:
alive = subprocess.run(["ps", "-p", str(pid)], capture_output=True).returncode == 0
failures += not check("no orphan process", not alive)
# A process that dies mid-command must be reported immediately, not at the
# turn deadline. Before the sentinel this sat on the event queue for the full
# 180 s budget; a fallback that launches a fresh pi then hid the death.
print("== process exit is detected immediately ==")
client2 = PiRpcClient(cfg)
await client2.start()
import time as _time
try:
os.killpg(os.getpgid(client2.pid), signal.SIGKILL)
started = _time.monotonic()
try:
await asyncio.wait_for(client2.get_session_stats(), timeout=10)
failures += not check("dead process reported", False, "command succeeded after SIGKILL")
except PiRpcError:
elapsed = _time.monotonic() - started
failures += not check("dead process reported promptly", elapsed < 10,
f"{elapsed:.1f}s (must be well under the 180 s budget)")
finally:
try:
await client2.stop()
except Exception:
pass
failures += not check("no orphan from killed client", not client2.running)
print()
print("RESULT:", "all checks passed" if failures == 0 else f"{failures} check(s) failed")
return 1 if failures else 0