test(unraid): add offline benchmark and 10 minute NDI soak

2026-08-15 09:52:38 -07:00
parent 38230f55a0
commit f5e71edc7f
5 changed files with 118 additions and 2 deletions
@@ -21,6 +21,7 @@ from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
 from .core import BroadcastHub, LatestQueue
 from .models import Models
 from .ndi import NDIReceiver
+from .offline_benchmark import run_offline_benchmark

 logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s %(name)s %(message)s")
 LOG = logging.getLogger("tail2")
@@ -145,7 +146,9 @@ async def poses(ws: WebSocket):
 def inference_worker(loop: asyncio.AbstractEventLoop) -> None:
    threshold = float(os.getenv("DETECTION_THRESHOLD", ".35")); max_people = int(os.getenv("MAX_PEOPLE", "4"))
    try:
-        models.load(); state.models_ready = True; LOG.info("official detector and pose checkpoints loaded")
+        models.load()
+        run_offline_benchmark(models, threshold, max_people)
+        state.models_ready = True; LOG.info("official detector and pose checkpoints loaded")
    except Exception as exc:
        state.inference_fault = True; state.error_code = "MODEL_LOAD_FAILED"
        state.error_detail = f"{type(exc).__name__}: {str(exc)[:240]}"; LOG.exception("model loading failed"); return
@@ -30,6 +30,7 @@ class Models:
        from mmdet.apis import init_detector
        from mmpose.apis import init_model
        self.detector = init_detector(self.det_config, self.det_checkpoint, device="cuda:0")
+        self.detector.test_cfg.nms.iou_threshold = float(os.getenv("NMS_IOU_THRESHOLD", ".60"))
        self.pose = init_model(self.pose_config, self.pose_checkpoint, device="cuda:0")

    @torch.inference_mode()
@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import json
+import os
+import statistics
+import time
+import urllib.request
+
+import cv2
+import numpy as np
+
+DEMO_URL = "https://raw.githubusercontent.com/open-mmlab/mmpose/v1.3.2/tests/data/coco/000000000785.jpg"
+
+
+def percentile(values: list[float], p: float) -> float:
+    values = sorted(values)
+    return values[min(len(values) - 1, max(0, int(np.ceil(len(values) * p)) - 1))]
+
+
+def run_offline_benchmark(models, threshold: float, max_people: int) -> None:
+    path = "/logs/offline-benchmark.json"
+    if os.path.exists(path):
+        return
+    try:
+        encoded = np.frombuffer(urllib.request.urlopen(DEMO_URL, timeout=30).read(), dtype=np.uint8)
+        one = cv2.imdecode(encoded, cv2.IMREAD_COLOR)
+        if one is None:
+            raise RuntimeError("official demo image decode failed")
+        images = {1: one, 2: np.concatenate([one, one], axis=1),
+                  4: np.concatenate([np.concatenate([one, one], axis=1)] * 2, axis=0)}
+        for _ in range(50):
+            models.infer(one, threshold, max_people)
+        report = {"source_url": DEMO_URL, "warmup_iterations": 50, "measured_iterations": 60,
+                  "image_retained": False, "profiles": {}}
+        for requested, image in images.items():
+            samples = []; counts = []
+            for _ in range(60):
+                started = time.perf_counter_ns()
+                people, det_ms, pose_ms = models.infer(image, threshold, max_people)
+                samples.append((time.perf_counter_ns() - started) / 1e6); counts.append(len(people))
+            report["profiles"][str(requested)] = {
+                "requested_people": requested, "detected_people_min": min(counts), "detected_people_max": max(counts),
+                "p50_ms": round(percentile(samples, .50), 3), "p95_ms": round(percentile(samples, .95), 3),
+                "p99_ms": round(percentile(samples, .99), 3), "fps_from_p50": round(1000 / percentile(samples, .50), 3)}
+        with open(path, "x", encoding="utf-8") as fh:
+            json.dump(report, fh, indent=2)
+    except Exception as exc:
+        with open(path, "w", encoding="utf-8") as fh:
+            json.dump({"error": f"{type(exc).__name__}: {str(exc)[:240]}", "source_url": DEMO_URL,
+                       "image_retained": False}, fh, indent=2)
+
@@ -0,0 +1,41 @@
+import json
+import pathlib
+import statistics
+import time
+import urllib.request
+
+from websockets.sync.client import connect
+
+token = pathlib.Path("/run/secrets/api-token").read_text().strip()
+headers = {"Authorization": "Bearer " + token}
+started = time.time(); deadline = started + 600
+frame_ids = []; latencies = []; failures = []; sessions = set(); health_samples = []
+while time.time() < deadline:
+    try:
+        with connect("ws://127.0.0.1:18120/v1/poses", additional_headers=headers, open_timeout=10) as ws:
+            while time.time() < deadline:
+                message = json.loads(ws.recv(timeout=10))
+                frame_ids.append(message["frame_id"]); sessions.add(message["session_id"])
+                latencies.append(message["timing"]["source_to_sent_ms"])
+                if len(frame_ids) % 100 == 0:
+                    req = urllib.request.Request("http://127.0.0.1:18120/v1/health", headers=headers)
+                    health_samples.append(json.load(urllib.request.urlopen(req, timeout=5)))
+    except Exception as exc:
+        failures.append(f"{type(exc).__name__}: {str(exc)[:120]}"); time.sleep(1)
+
+
+def pct(values, p):
+    values = sorted(values); return values[min(len(values)-1, max(0, int(len(values)*p)-1))] if values else None
+
+
+report = {"duration_seconds": round(time.time()-started, 3), "messages": len(frame_ids),
+          "first_frame_id": frame_ids[0] if frame_ids else None, "last_frame_id": frame_ids[-1] if frame_ids else None,
+          "strictly_increasing": all(b > a for a, b in zip(frame_ids, frame_ids[1:])),
+          "session_ids": sorted(sessions), "ws_failures": failures,
+          "source_to_sent_ms": {"p50": pct(latencies,.50), "p95": pct(latencies,.95), "p99": pct(latencies,.99)},
+          "health_samples": len(health_samples), "all_ready": all(x["ready"] for x in health_samples),
+          "max_gpu_memory_used_mb": max((x["gpu"]["memory_used_mb"] for x in health_samples), default=None),
+          "max_gpu_temperature_c": max((x["gpu"]["temperature_c"] for x in health_samples), default=None),
+          "frames_or_crops_saved": False}
+pathlib.Path("/validation/soak-10m.json").write_text(json.dumps(report, indent=2))
+
@@ -96,7 +96,7 @@ services:
      POSE_CONFIG: /models/mmpose/configs/wholebody_2d_keypoint/rtmpose/cocktail14/rtmw-l_8xb1024-270e_cocktail14-256x192.py
      POSE_CHECKPOINT: /models/rtmw-dw-x-l_simcc-cocktail14_270e-256x192-20231122.pth
    healthcheck:
-      test: ["CMD", "python", "-c", "import pathlib,urllib.request; t=pathlib.Path('/run/secrets/api-token').read_text().strip(); r=urllib.request.Request('http://127.0.0.1:18120/v1/health',headers={'Authorization':'Bearer '+t}); raise SystemExit(0 if urllib.request.urlopen(r,timeout=3).status==200 else 1)"]
+      test: ["CMD", "python", "-c", "import json,pathlib,urllib.request; t=pathlib.Path('/run/secrets/api-token').read_text().strip(); r=urllib.request.Request('http://127.0.0.1:18120/v1/health',headers={'Authorization':'Bearer '+t}); raise SystemExit(0 if json.load(urllib.request.urlopen(r,timeout=3))['ready'] else 1)"]
      interval: 15s
      timeout: 5s
      retries: 6
@@ -128,3 +128,23 @@ services:
        nvidia-smi --query-gpu=name,driver_version,memory.total,memory.used,temperature.gpu --format=csv,noheader > /validation/gpu.csv
        sha256sum /models/*.pth > /validation/model-sha256.txt
        chmod 0666 /validation/*.txt /validation/*.json /validation/*.csv
+
+  soak:
+    image: tail2-pose-server:0.1.0
+    container_name: tail2-pose-soak
+    network_mode: host
+    restart: "no"
+    entrypoint: ["/bin/bash", "-lc"]
+    depends_on:
+      tail2-pose-server:
+        condition: service_healthy
+    volumes:
+      - /mnt/user/appdata/tail2-pose-server/app:/app:ro
+      - /mnt/user/appdata/tail2-pose-server/secrets/api-token:/run/secrets/api-token:ro
+      - ./validation:/validation
+    command:
+      - |
+        set -euo pipefail
+        chmod -R a+rwX /validation
+        test -s /validation/soak-10m.json || python /app/tail2/soak.py
+        chmod 0666 /validation/soak-10m.json