fix(pi_rpc): detect a dead pi process immediately, not at the turn deadline
When pi died mid-turn, _read_stdout saw EOF and logged it, but nothing woke _consume -- it sat on the event queue until asyncio.timeout(budget) fired at the turn deadline (180 s). A dead process at second 3 was therefore reported as "no reply" at second 180, and the fallback that launched a fresh pi hid the death entirely. To the user this read as a hard hang. _read_stdout now enqueues a "_process_exited" sentinel on EOF, and _consume raises PiRpcError on it, so the failure path (same-session fallback model, then rotation, then a fresh process) starts within milliseconds. Smoke-tested: SIGKILL the child during a command and the client reports it in 0.0 s instead of the full budget, with no orphan left behind.
This commit is contained in:
+14
-1
@@ -445,7 +445,13 @@ class PiRpcClient:
|
|||||||
await result
|
await result
|
||||||
except Exception:
|
except Exception:
|
||||||
LOG.exception("on_event callback failed")
|
LOG.exception("on_event callback failed")
|
||||||
LOG.warning("pi RPC stdout closed")
|
# EOF: the child died or closed its pipe. Without a sentinel, _consume
|
||||||
|
# would sit on _events.get() until the turn deadline -- a process that
|
||||||
|
# exited at second 3 would be reported as "no reply" at second 180, and a
|
||||||
|
# fallback that launches a fresh pi would hide the death entirely. Put a
|
||||||
|
# sentinel so the waiter fails immediately instead.
|
||||||
|
LOG.warning("pi RPC stdout closed pid=%s rc=%s", self.pid, self._process.returncode)
|
||||||
|
await self._events.put({"type": "_process_exited"})
|
||||||
|
|
||||||
async def _read_stderr(self) -> None:
|
async def _read_stderr(self) -> None:
|
||||||
assert self._process and self._process.stderr
|
assert self._process and self._process.stderr
|
||||||
@@ -528,6 +534,13 @@ class PiRpcClient:
|
|||||||
event = await self._events.get()
|
event = await self._events.get()
|
||||||
kind = event.get("type")
|
kind = event.get("type")
|
||||||
|
|
||||||
|
if kind == "_process_exited":
|
||||||
|
rc = self._process.returncode if self._process else None
|
||||||
|
raise PiRpcError(
|
||||||
|
f"pi process exited (pid={self.pid}, returncode={rc}) while "
|
||||||
|
f"awaiting {request_id}; stderr tail: {' | '.join(self._stderr_tail[-3:])}"
|
||||||
|
)
|
||||||
|
|
||||||
if kind == "response" and event.get("id") == request_id:
|
if kind == "response" and event.get("id") == request_id:
|
||||||
response = event
|
response = event
|
||||||
if not wait_for_settled or not event.get("success"):
|
if not wait_for_settled or not event.get("success"):
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
|
import signal
|
||||||
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -19,7 +21,7 @@ from dataclasses import replace # noqa: E402
|
|||||||
import pathlib # noqa: E402
|
import pathlib # noqa: E402
|
||||||
import tempfile # noqa: E402
|
import tempfile # noqa: E402
|
||||||
|
|
||||||
from pi_rpc import PiLaunchConfig, PiRpcClient # noqa: E402
|
from pi_rpc import PiLaunchConfig, PiRpcClient, PiRpcError # noqa: E402
|
||||||
|
|
||||||
REPO = Path(__file__).resolve().parents[4]
|
REPO = Path(__file__).resolve().parents[4]
|
||||||
HARNESS = REPO / "docs" / "evidence" / "probe-harness"
|
HARNESS = REPO / "docs" / "evidence" / "probe-harness"
|
||||||
@@ -107,6 +109,31 @@ async def main() -> int:
|
|||||||
alive = subprocess.run(["ps", "-p", str(pid)], capture_output=True).returncode == 0
|
alive = subprocess.run(["ps", "-p", str(pid)], capture_output=True).returncode == 0
|
||||||
failures += not check("no orphan process", not alive)
|
failures += not check("no orphan process", not alive)
|
||||||
|
|
||||||
|
# A process that dies mid-command must be reported immediately, not at the
|
||||||
|
# turn deadline. Before the sentinel this sat on the event queue for the full
|
||||||
|
# 180 s budget; a fallback that launches a fresh pi then hid the death.
|
||||||
|
print("== process exit is detected immediately ==")
|
||||||
|
client2 = PiRpcClient(cfg)
|
||||||
|
await client2.start()
|
||||||
|
import time as _time
|
||||||
|
try:
|
||||||
|
os.killpg(os.getpgid(client2.pid), signal.SIGKILL)
|
||||||
|
started = _time.monotonic()
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(client2.get_session_stats(), timeout=10)
|
||||||
|
failures += not check("dead process reported", False, "command succeeded after SIGKILL")
|
||||||
|
except PiRpcError:
|
||||||
|
elapsed = _time.monotonic() - started
|
||||||
|
failures += not check("dead process reported promptly", elapsed < 10,
|
||||||
|
f"{elapsed:.1f}s (must be well under the 180 s budget)")
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
await client2.stop()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
failures += not check("no orphan from killed client", not client2.running)
|
||||||
|
|
||||||
print()
|
print()
|
||||||
print("RESULT:", "all checks passed" if failures == 0 else f"{failures} check(s) failed")
|
print("RESULT:", "all checks passed" if failures == 0 else f"{failures} check(s) failed")
|
||||||
return 1 if failures else 0
|
return 1 if failures else 0
|
||||||
|
|||||||
Reference in New Issue
Block a user