test(unraid): add offline benchmark and 10 minute NDI soak

2026-08-15 09:52:38 -07:00
parent 38230f55a0
commit f5e71edc7f
5 changed files with 118 additions and 2 deletions
@@ -0,0 +1,41 @@
+import json
+import pathlib
+import statistics
+import time
+import urllib.request
+
+from websockets.sync.client import connect
+
+token = pathlib.Path("/run/secrets/api-token").read_text().strip()
+headers = {"Authorization": "Bearer " + token}
+started = time.time(); deadline = started + 600
+frame_ids = []; latencies = []; failures = []; sessions = set(); health_samples = []
+while time.time() < deadline:
+    try:
+        with connect("ws://127.0.0.1:18120/v1/poses", additional_headers=headers, open_timeout=10) as ws:
+            while time.time() < deadline:
+                message = json.loads(ws.recv(timeout=10))
+                frame_ids.append(message["frame_id"]); sessions.add(message["session_id"])
+                latencies.append(message["timing"]["source_to_sent_ms"])
+                if len(frame_ids) % 100 == 0:
+                    req = urllib.request.Request("http://127.0.0.1:18120/v1/health", headers=headers)
+                    health_samples.append(json.load(urllib.request.urlopen(req, timeout=5)))
+    except Exception as exc:
+        failures.append(f"{type(exc).__name__}: {str(exc)[:120]}"); time.sleep(1)
+
+
+def pct(values, p):
+    values = sorted(values); return values[min(len(values)-1, max(0, int(len(values)*p)-1))] if values else None
+
+
+report = {"duration_seconds": round(time.time()-started, 3), "messages": len(frame_ids),
+          "first_frame_id": frame_ids[0] if frame_ids else None, "last_frame_id": frame_ids[-1] if frame_ids else None,
+          "strictly_increasing": all(b > a for a, b in zip(frame_ids, frame_ids[1:])),
+          "session_ids": sorted(sessions), "ws_failures": failures,
+          "source_to_sent_ms": {"p50": pct(latencies,.50), "p95": pct(latencies,.95), "p99": pct(latencies,.99)},
+          "health_samples": len(health_samples), "all_ready": all(x["ready"] for x in health_samples),
+          "max_gpu_memory_used_mb": max((x["gpu"]["memory_used_mb"] for x in health_samples), default=None),
+          "max_gpu_temperature_c": max((x["gpu"]["temperature_c"] for x in health_samples), default=None),
+          "frames_or_crops_saved": False}
+pathlib.Path("/validation/soak-10m.json").write_text(json.dumps(report, indent=2))
+