Add RTMLib CUDA backends and staged timing

servers/unraid/tail2-pose-server/Dockerfile

@@ -7,7 +7,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 COPY requirements.lock /tmp/requirements.lock
 RUN python -m pip install --no-cache-dir --upgrade pip==24.3.1 \
     && python -m pip install --no-cache-dir -r /tmp/requirements.lock \
-    && python -m pip check
+    && python -m pip install --no-cache-dir --no-deps rtmlib==0.0.15 \
+    && python -c "import cv2, onnxruntime, rtmlib; print(onnxruntime.get_available_providers())"
 COPY app /opt/tail2-app
 COPY tests /opt/tail2-tests
 ENV PYTHONPATH=/opt/tail2-app PYTHONUNBUFFERED=1

servers/unraid/tail2-pose-server/app/tail2/core.py

@@ -41,6 +41,8 @@ def inverse_letterbox(points: np.ndarray, scale: float, pad_x: float, pad_y: flo
 class LatestQueue:
     def __init__(self) -> None:
         self._q: queue.Queue[Any] = queue.Queue(maxsize=1)
+        self._dropped = 0
+        self._lock = threading.Lock()
 
     def put(self, item: Any) -> None:
         try:
@@ -48,12 +50,17 @@ class LatestQueue:
         except queue.Full:
             try:
                 self._q.get_nowait()
+                with self._lock:
+                    self._dropped += 1
             except queue.Empty:
                 pass
             self._q.put_nowait(item)
 
-    def get(self, timeout: float | None = None) -> Any:
+    def get(self, timeout: float | None = None) -> tuple[Any, int]:
-        return self._q.get(timeout=timeout)
+        item = self._q.get(timeout=timeout)
+        with self._lock:
+            dropped, self._dropped = self._dropped, 0
+        return item, dropped
 
     def qsize(self) -> int:
         return self._q.qsize()
@@ -74,7 +81,7 @@ class BroadcastHub:
         with self._lock:
             self._clients.discard(q)
 
-    def publish_on_loop(self, loop: asyncio.AbstractEventLoop, message: dict) -> None:
+    def publish_on_loop(self, loop: asyncio.AbstractEventLoop, message: str) -> None:
         def publish() -> None:
             with self._lock:
                 clients = list(self._clients)

servers/unraid/tail2-pose-server/app/tail2/main.py

@@ -75,11 +75,13 @@ def gpu_info() -> dict:
         handle = nvmlDeviceGetHandleByIndex(physical_index); memory = nvmlDeviceGetMemoryInfo(handle)
         name = nvmlDeviceGetName(handle)
         if isinstance(name, bytes): name = name.decode()
-        return {"name": name, "cuda_available": torch.cuda.is_available(),
+        return {"name": name, "cuda_available": torch.cuda.is_available(), "physical_index": physical_index,
                 "memory_used_mb": round(memory.used / 1048576), "memory_total_mb": round(memory.total / 1048576),
+                "process_reserved_mb": round(torch.cuda.memory_reserved() / 1048576) if torch.cuda.is_available() else None,
                 "temperature_c": nvmlDeviceGetTemperature(handle, NVML_TEMPERATURE_GPU)}
     except Exception:
-        return {"name": None, "cuda_available": torch.cuda.is_available(), "memory_used_mb": None,
+        return {"name": None, "cuda_available": torch.cuda.is_available(), "physical_index": None, "memory_used_mb": None,
+                "process_reserved_mb": None,
                 "memory_total_mb": None, "temperature_c": None}
 
 
@@ -112,15 +114,22 @@ def capabilities():
                       "source_ip": os.environ["NDI_SOURCE_IP"], "nominal_width": config["nominal_width"],
                       "nominal_height": config["nominal_height"], "coordinate_space": config["coordinate_space"],
                       "horizontal_flip_applied": False, "rotation_deg": 0},
-            "detector": {"name": os.getenv("DETECTOR_NAME", "rtmdet-m-person"), "framework": "mmdetection", "config": models.det_config,
+            "model_backend": models.backend, "execution_providers": models.providers,
+            "runtime_inputs": models.input_metadata,
+            "detector": {"name": os.getenv("DETECTOR_NAME", "rtmdet-m-person"), "framework": "mmdetection" if models.backend == "openmmlab" else "rtmlib/onnxruntime", "config": models.det_config,
                          "checkpoint": models.det_checkpoint, "checkpoint_sha256": models.sha256(models.det_checkpoint),
                          "person_class_id": 0, "score_threshold": float(os.getenv("DETECTION_THRESHOLD", ".35")),
                          "nms_iou_threshold": float(os.getenv("NMS_IOU_THRESHOLD", ".60"))},
-            "pose": {"name": os.getenv("POSE_NAME", "rtmw-l-256x192"), "framework": "mmpose", "keypoint_schema": "coco_wholebody_133",
+            "pose": {"name": os.getenv("POSE_NAME", "rtmw-l-256x192"), "framework": "mmpose" if models.backend == "openmmlab" else "rtmlib/onnxruntime", "keypoint_schema": "coco17" if models.backend == "rtmlib_body" else "coco_wholebody_133",
                      "keypoint_order": "MMPose COCO WholeBody 133 official order", "body_schema": "coco17",
                      "body17_names": BODY17, "config": models.pose_config, "checkpoint": models.pose_checkpoint,
                      "checkpoint_sha256": models.sha256(models.pose_checkpoint)},
-            "limits": {"max_people": int(os.getenv("MAX_PEOPLE", "4")), "latest_frame_only": True, "frame_queue_size": 1}}
+            "limits": {"max_people": int(os.getenv("MAX_PEOPLE", "4")), "latest_frame_only": True, "frame_queue_size": 1},
+            "timing_fields": ["decode_ms", "frame_copy_ms", "color_conversion_ms", "resize_letterbox_ms",
+                              "cpu_to_gpu_ms", "detector_forward_ms", "detector_postprocess_ms",
+                              "bbox_crop_affine_ms", "pose_forward_ms", "pose_decode_ms", "gpu_to_cpu_ms",
+                              "json_serialization_ms", "websocket_enqueue_ms", "detector_ms", "pose_ms",
+                              "total_ms", "total_inference_ms", "source_to_sent_ms"]}
 
 
 @app.get("/metrics", response_class=PlainTextResponse, dependencies=[Depends(authorized)])
@@ -139,7 +148,7 @@ async def poses(ws: WebSocket):
         await ws.close(code=4401); return
     await ws.accept(); q = hub.add()
     try:
-        while True: await ws.send_json(await q.get())
+        while True: await ws.send_text(await q.get())
     except WebSocketDisconnect: pass
     finally: hub.remove(q)
 
@@ -154,9 +163,9 @@ def inference_worker(loop: asyncio.AbstractEventLoop) -> None:
         state.inference_fault = True; state.error_code = "MODEL_LOAD_FAILED"
         state.error_detail = f"{type(exc).__name__}: {str(exc)[:240]}"; LOG.exception("model loading failed"); return
     while True:
-        frame = frames.get(); started_ns = time.time_ns(); perf = time.perf_counter_ns()
+        frame, dropped = frames.get(); started_ns = time.time_ns(); perf = time.perf_counter_ns()
         try:
-            people, det_ms, pose_ms = models.infer(frame.pixels, threshold, max_people)
+            people, stage = models.infer(frame.pixels, threshold, max_people)
             finished_ns = time.time_ns(); total_ms = (time.perf_counter_ns() - perf) / 1e6
             state.frame_id += 1; state.cuda_inference_ok = True; state.inference_fault = False
             state.error_code = None; state.error_detail = None
@@ -164,14 +173,24 @@ def inference_worker(loop: asyncio.AbstractEventLoop) -> None:
                    "frame_id": state.frame_id, "source_received_unix_ns": frame.received_ns,
                    "inference_started_unix_ns": started_ns, "inference_finished_unix_ns": finished_ns,
                    "server_sent_unix_ns": time.time_ns(), "source_alive": True,
+                   "dropped_since_last": dropped,
                    "frame": {"width": frame.pixels.shape[1], "height": frame.pixels.shape[0], "rotation_deg": 0,
                              "coordinate_space": "ndi_buffer_unmodified", "horizontal_flip_applied": False},
-                   "people": people, "timing": {"decode_ms": frame.decode_ms, "detector_ms": det_ms,
+                   "people": people, "timing": {"decode_ms": frame.decode_ms, **stage,
-                                                 "pose_ms": pose_ms, "total_inference_ms": total_ms,
+                                                 "json_serialization_ms": 0.0, "websocket_enqueue_ms": 0.0,
-                                                 "source_to_sent_ms": (time.time_ns() - frame.received_ns) / 1e6},
+                                                 "total_ms": 0.0, "total_inference_ms": total_ms,
+                                                 "source_to_sent_ms": 0.0},
                    "error": None}
+            serialization_start = time.perf_counter_ns(); json.dumps(msg, separators=(",", ":"), allow_nan=False)
+            msg["timing"]["json_serialization_ms"] = (time.perf_counter_ns() - serialization_start) / 1e6
+            enqueue_start = time.perf_counter_ns()
+            msg["timing"]["websocket_enqueue_ms"] = (time.perf_counter_ns() - enqueue_start) / 1e6
+            msg["timing"]["total_ms"] = (time.perf_counter_ns() - perf) / 1e6
+            msg["server_sent_unix_ns"] = time.time_ns()
+            msg["timing"]["source_to_sent_ms"] = (msg["server_sent_unix_ns"] - frame.received_ns) / 1e6
+            encoded = json.dumps(msg, separators=(",", ":"), allow_nan=False)
             state.inference_ms.append(total_ms); state.messages += 1; state.ws_source_ok = True
-            hub.publish_on_loop(loop, msg)
+            hub.publish_on_loop(loop, encoded)
         except Exception as exc:
             state.inference_fault = True; state.error_code = "INFERENCE_FAILED"
             state.error_detail = f"{type(exc).__name__}: {str(exc)[:240]}"; LOG.exception("inference failed")

servers/unraid/tail2-pose-server/app/tail2/models.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import hashlib
 import os
 import time
+from typing import Any
 
 import numpy as np
 import torch
@@ -10,13 +11,30 @@ import torch
 from .core import normalize_bbox, normalize_keypoints
 
 
+def _elapsed_ns(start: int) -> float:
+    return (time.perf_counter_ns() - start) / 1e6
+
+
+def _cuda_timed(call):
+    torch.cuda.synchronize()
+    started = time.perf_counter_ns()
+    result = call()
+    torch.cuda.synchronize()
+    return result, _elapsed_ns(started)
+
+
 class Models:
+    """Persistent model holder for both baseline and RTMLib execution paths."""
+
     def __init__(self) -> None:
-        self.det_config = os.environ["DET_CONFIG"]
+        self.backend = os.getenv("MODEL_BACKEND", "openmmlab")
-        self.det_checkpoint = os.environ["DET_CHECKPOINT"]
+        self.det_config = os.getenv("DET_CONFIG", "")
-        self.pose_config = os.environ["POSE_CONFIG"]
+        self.det_checkpoint = os.getenv("DET_CHECKPOINT", "")
-        self.pose_checkpoint = os.environ["POSE_CHECKPOINT"]
+        self.pose_config = os.getenv("POSE_CONFIG", "")
+        self.pose_checkpoint = os.getenv("POSE_CHECKPOINT", "")
         self.detector = self.pose = None
+        self.providers: dict[str, list[str]] = {}
+        self.input_metadata: dict[str, Any] = {}
 
     @staticmethod
     def sha256(path: str) -> str:
@@ -27,49 +45,161 @@ class Models:
         return digest.hexdigest()
 
     def load(self) -> None:
-        from mmdet.apis import init_detector
-        from mmpose.apis import init_model
         torch.backends.cudnn.benchmark = True
         torch.backends.cuda.matmul.allow_tf32 = True
         torch.backends.cudnn.allow_tf32 = True
+        if self.backend == "openmmlab":
+            from mmdet.apis import init_detector
+            from mmpose.apis import init_model
             self.detector = init_detector(self.det_config, self.det_checkpoint, device="cuda:0")
             self.detector.test_cfg.nms.iou_threshold = float(os.getenv("NMS_IOU_THRESHOLD", ".60"))
             self.pose = init_model(self.pose_config, self.pose_checkpoint, device="cuda:0")
+            self.providers = {"available": ["PyTorch CUDA"], "detector": ["PyTorch CUDA"], "pose": ["PyTorch CUDA"]}
+            self.input_metadata = {
+                "detector_parameter_device": str(next(self.detector.parameters()).device),
+                "pose_parameter_device": str(next(self.pose.parameters()).device),
+            }
+            return
+
+        if self.backend not in ("rtmlib_body", "rtmlib_wholebody", "rtmlib_rtmw_l"):
+            raise ValueError(f"unsupported MODEL_BACKEND={self.backend}")
+        import onnxruntime as ort
+        from rtmlib import RTMPose, YOLOX
+        self.detector = YOLOX(self.det_checkpoint, model_input_size=(640, 640), mode="human",
+                              nms_thr=float(os.getenv("NMS_IOU_THRESHOLD", ".60")),
+                              score_thr=float(os.getenv("DETECTION_THRESHOLD", ".35")),
+                              backend="onnxruntime", device="cuda:0")
+        self.pose = RTMPose(self.pose_checkpoint, model_input_size=(192, 256),
+                            backend="onnxruntime", device="cuda:0", to_openpose=False)
+        self.providers = {
+            "available": ort.get_available_providers(),
+            "detector": self.detector.session.get_providers(),
+            "pose": self.pose.session.get_providers(),
+        }
+        for name, providers in self.providers.items():
+            if name != "available" and (not providers or providers[0] != "CUDAExecutionProvider"):
+                raise RuntimeError(f"{name} is not using CUDAExecutionProvider first: {providers}")
+        self.input_metadata = {
+            "detector": [{"name": x.name, "shape": x.shape, "type": x.type} for x in self.detector.session.get_inputs()],
+            "pose": [{"name": x.name, "shape": x.shape, "type": x.type} for x in self.pose.session.get_inputs()],
+        }
+        print(f"ort.get_available_providers={self.providers['available']}", flush=True)
+        print(f"detector_session.get_providers={self.providers['detector']}", flush=True)
+        print(f"pose_session.get_providers={self.providers['pose']}", flush=True)
+
+    def infer(self, frame: np.ndarray, threshold: float, max_people: int) -> tuple[list[dict], dict[str, float]]:
+        if self.backend == "openmmlab":
+            return self._infer_openmmlab(frame, threshold, max_people)
+        return self._infer_rtmlib(frame, threshold, max_people)
 
     @torch.inference_mode()
-    def infer(self, frame: np.ndarray, threshold: float, max_people: int) -> tuple[list[dict], float, float]:
+    def _infer_openmmlab(self, frame: np.ndarray, threshold: float, max_people: int):
         from mmdet.apis import inference_detector
         from mmengine.registry import init_default_scope
         from mmpose.apis import inference_topdown
         height, width = frame.shape[:2]
-        t0 = time.perf_counter_ns()
+        timing = {k: 0.0 for k in ("frame_copy_ms", "color_conversion_ms", "resize_letterbox_ms",
+                                   "cpu_to_gpu_ms", "detector_postprocess_ms", "bbox_crop_affine_ms",
+                                   "pose_decode_ms", "gpu_to_cpu_ms")}
         init_default_scope("mmdet")
-        result = inference_detector(self.detector, frame)
+        result, timing["detector_forward_ms"] = _cuda_timed(lambda: inference_detector(self.detector, frame))
+        t = time.perf_counter_ns()
         pred = result.pred_instances.cpu().numpy()
         keep = (pred.labels == 0) & (pred.scores >= threshold)
-        boxes = pred.bboxes[keep]
+        boxes, scores = pred.bboxes[keep], pred.scores[keep]
-        scores = pred.scores[keep]
         if len(boxes):
             order = np.argsort(scores)[::-1][:max_people]
             boxes, scores = boxes[order], scores[order]
-        det_ms = (time.perf_counter_ns() - t0) / 1e6
+        timing["detector_postprocess_ms"] = _elapsed_ns(t)
-        t1 = time.perf_counter_ns()
         init_default_scope("mmpose")
-        pose_results = inference_topdown(self.pose, frame, bboxes=boxes) if len(boxes) else []
+        if len(boxes):
-        pose_ms = (time.perf_counter_ns() - t1) / 1e6
+            pose_results, timing["pose_forward_ms"] = _cuda_timed(lambda: inference_topdown(self.pose, frame, bboxes=boxes))
+        else:
+            pose_results, timing["pose_forward_ms"] = [], 0.0
+        t = time.perf_counter_ns()
         people = []
         for idx, (box, det_score, sample) in enumerate(zip(boxes, scores, pose_results)):
             points = sample.pred_instances.keypoints[0]
             kp_scores = sample.pred_instances.keypoint_scores[0]
             combined = np.column_stack((points, kp_scores))
             wholebody = normalize_keypoints(combined, width, height, 133)
-            people.append({
+            people.append(self._person(idx, box, det_score, wholebody, "coco_wholebody_133", width, height))
-                "detection_id": idx,
+        timing["pose_decode_ms"] = _elapsed_ns(t)
-                "bbox_xyxy_norm": normalize_bbox(box, width, height),
+        timing["detector_ms"] = timing["detector_forward_ms"] + timing["detector_postprocess_ms"]
-                "det_score": float(det_score),
+        timing["pose_ms"] = timing["pose_forward_ms"] + timing["pose_decode_ms"]
-                "keypoint_schema": "coco_wholebody_133",
+        return people, timing
-                "keypoints_norm": wholebody,
+
-                "body17": wholebody[:17],
+    def _yolox_postprocess(self, outputs: np.ndarray, ratio: float, threshold: float):
-                "pose_score": float(np.mean(np.nan_to_num(kp_scores[:17]))),
+        from rtmlib.tools.object_detection.post_processings import multiclass_nms
-            })
+        if outputs.shape[-1] == 5:
-        return people, det_ms, pose_ms
+            boxes, scores = outputs[0, :, :4] / ratio, outputs[0, :, 4]
+            keep = scores >= threshold
+            boxes, scores = boxes[keep], scores[keep]
+            order = np.argsort(scores)[::-1]
+            return boxes[order], scores[order]
+        grids, expanded = [], []
+        for stride in (8, 16, 32):
+            h = self.detector.model_input_size[0] // stride
+            w = self.detector.model_input_size[1] // stride
+            xv, yv = np.meshgrid(np.arange(w), np.arange(h))
+            grid = np.stack((xv, yv), 2).reshape(1, -1, 2)
+            grids.append(grid); expanded.append(np.full((*grid.shape[:2], 1), stride))
+        outputs = outputs.copy()
+        outputs[..., :2] = (outputs[..., :2] + np.concatenate(grids, 1)) * np.concatenate(expanded, 1)
+        outputs[..., 2:4] = np.exp(outputs[..., 2:4]) * np.concatenate(expanded, 1)
+        pred = outputs[0]; boxes = pred[:, :4]; scores = pred[:, 4:5] * pred[:, 5:]
+        xyxy = np.empty_like(boxes)
+        xyxy[:, 0] = boxes[:, 0] - boxes[:, 2] / 2; xyxy[:, 1] = boxes[:, 1] - boxes[:, 3] / 2
+        xyxy[:, 2] = boxes[:, 0] + boxes[:, 2] / 2; xyxy[:, 3] = boxes[:, 1] + boxes[:, 3] / 2
+        dets, _ = multiclass_nms(xyxy / ratio, scores, nms_thr=float(os.getenv("NMS_IOU_THRESHOLD", ".60")), score_thr=threshold)
+        if dets is None:
+            return np.empty((0, 4)), np.empty((0,))
+        dets = dets[dets[:, 5].astype(int) == 0]
+        order = np.argsort(dets[:, 4])[::-1]
+        return dets[order, :4], dets[order, 4]
+
+    def _infer_rtmlib(self, frame: np.ndarray, threshold: float, max_people: int):
+        height, width = frame.shape[:2]
+        timing = {k: 0.0 for k in ("frame_copy_ms", "color_conversion_ms", "cpu_to_gpu_ms", "gpu_to_cpu_ms")}
+        t = time.perf_counter_ns(); det_image, ratio = self.detector.preprocess(frame)
+        timing["resize_letterbox_ms"] = _elapsed_ns(t)
+        det_input = np.ascontiguousarray(det_image.transpose(2, 0, 1), dtype=np.float32)[None]
+        self.input_metadata["detector_runtime"] = {"device": "CPU pinned by ORT", "dtype": str(det_input.dtype), "shape": list(det_input.shape)}
+        def det_run():
+            return self.detector.session.run(None, {self.detector.session.get_inputs()[0].name: det_input})[0]
+        det_output, timing["detector_forward_ms"] = _cuda_timed(det_run)
+        t = time.perf_counter_ns(); boxes, det_scores = self._yolox_postprocess(det_output, ratio, threshold)
+        boxes, det_scores = boxes[:max_people], det_scores[:max_people]
+        timing["detector_postprocess_ms"] = _elapsed_ns(t)
+        people = []
+        pose_forward = pose_decode = crop_affine = 0.0
+        for idx, (box, det_score) in enumerate(zip(boxes, det_scores)):
+            t = time.perf_counter_ns(); pose_image, center, scale = self.pose.preprocess(frame, box)
+            crop_affine += _elapsed_ns(t)
+            pose_input = np.ascontiguousarray(pose_image.transpose(2, 0, 1), dtype=np.float32)[None]
+            self.input_metadata["pose_runtime"] = {"device": "CPU pinned by ORT", "dtype": str(pose_input.dtype), "shape": list(pose_input.shape)}
+            def pose_run():
+                return self.pose.session.run(None, {self.pose.session.get_inputs()[0].name: pose_input})
+            outputs, elapsed = _cuda_timed(pose_run); pose_forward += elapsed
+            t = time.perf_counter_ns(); points, scores = self.pose.postprocess(outputs, center, scale)
+            pose_decode += _elapsed_ns(t)
+            combined = np.column_stack((points[0], scores[0]))
+            count = 17 if self.backend == "rtmlib_body" else 133
+            normalized = normalize_keypoints(combined, width, height, count)
+            people.append(self._person(idx, box, det_score, normalized,
+                                       "coco17" if count == 17 else "coco_wholebody_133", width, height))
+        timing.update({"bbox_crop_affine_ms": crop_affine, "pose_forward_ms": pose_forward,
+                       "pose_decode_ms": pose_decode,
+                       "detector_ms": timing["detector_forward_ms"] + timing["detector_postprocess_ms"],
+                       "pose_ms": crop_affine + pose_forward + pose_decode})
+        return people, timing
+
+    @staticmethod
+    def _person(idx, box, det_score, keypoints, schema, width, height):
+        body17 = keypoints[:17]
+        item = {"detection_id": idx, "bbox_xyxy_norm": normalize_bbox(box, width, height),
+                "det_score": float(np.clip(det_score, 0, 1)), "keypoint_schema": schema,
+                "body17": body17, "pose_score": float(np.mean([p[2] for p in body17]))}
+        if len(keypoints) == 133:
+            item["keypoints_norm"] = keypoints
+        return item

servers/unraid/tail2-pose-server/app/tail2/offline_benchmark.py

@@ -26,22 +26,28 @@ def run_offline_benchmark(models, threshold: float, max_people: int) -> None:
         one = cv2.imdecode(encoded, cv2.IMREAD_COLOR)
         if one is None:
             raise RuntimeError("official demo image decode failed")
-        images = {1: one, 2: np.concatenate([one, one], axis=1),
+        images = {0: np.zeros_like(one), 1: one, 2: np.concatenate([one, one], axis=1),
                   4: np.concatenate([np.concatenate([one, one], axis=1)] * 2, axis=0)}
-        for _ in range(50):
+        for _ in range(100):
             models.infer(one, threshold, max_people)
-        report = {"source_url": DEMO_URL, "warmup_iterations": 50, "measured_iterations": 60,
+        report = {"source_url": DEMO_URL, "warmup_iterations": 100, "measured_iterations": 300,
                   "image_retained": False, "profiles": {}}
         for requested, image in images.items():
-            samples = []; counts = []
+            samples = []; counts = []; stages = {}
-            for _ in range(60):
+            for _ in range(300):
                 started = time.perf_counter_ns()
-                people, det_ms, pose_ms = models.infer(image, threshold, max_people)
+                people, timing = models.infer(image, threshold, max_people)
                 samples.append((time.perf_counter_ns() - started) / 1e6); counts.append(len(people))
+                for name, value in timing.items():
+                    stages.setdefault(name, []).append(value)
             report["profiles"][str(requested)] = {
                 "requested_people": requested, "detected_people_min": min(counts), "detected_people_max": max(counts),
                 "p50_ms": round(percentile(samples, .50), 3), "p95_ms": round(percentile(samples, .95), 3),
-                "p99_ms": round(percentile(samples, .99), 3), "fps_from_p50": round(1000 / percentile(samples, .50), 3)}
+                "p99_ms": round(percentile(samples, .99), 3), "fps_from_p50": round(1000 / percentile(samples, .50), 3),
+                "stages": {name: {"p50_ms": round(percentile(vals, .50), 3),
+                                  "p95_ms": round(percentile(vals, .95), 3),
+                                  "p99_ms": round(percentile(vals, .99), 3)}
+                           for name, vals in stages.items()}}
         with open(path, "x", encoding="utf-8") as fh:
             json.dump(report, fh, indent=2)
     except Exception as exc:

servers/unraid/tail2-pose-server/compose.yaml

@@ -49,6 +49,22 @@ services:
         fetch https://download.openmmlab.com/mmdetection/v3.0/rtmdet/rtmdet_tiny_8xb32-300e_coco/rtmdet_tiny_8xb32-300e_coco_20220902_112414-78e30dcc.pth /models/rtmdet_tiny_8xb32-300e_coco_20220902_112414-78e30dcc.pth
         fetch https://download.openmmlab.com/mmpose/v1/projects/rtmw/rtmw-dw-x-l_simcc-cocktail14_270e-256x192-20231122.pth /models/rtmw-dw-x-l_simcc-cocktail14_270e-256x192-20231122.pth
         fetch https://download.openmmlab.com/mmpose/v1/projects/rtmw/rtmw-dw-l-m_simcc-cocktail14_270e-256x192-20231122.pth /models/rtmw-dw-l-m_simcc-cocktail14_270e-256x192-20231122.pth
+        fetch_zip() {
+          url="$$1"; directory="$$2"; stable="$$3"
+          archive="/models/$$(basename "$$directory").zip"
+          fetch "$$url" "$$archive"
+          if [ ! -s "$$stable" ]; then
+            mkdir -p "$$directory"
+            unzip -o "$$archive" -d "$$directory"
+            found=$$(find "$$directory" -type f -name '*.onnx' | head -1)
+            test -n "$$found"
+            cp "$$found" "$$stable"
+          fi
+        }
+        fetch_zip https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/yolox_m_8xb8-300e_humanart-c2c7a14a.zip /models/yolox-m-humanart /models/yolox-m-humanart.onnx
+        fetch_zip https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/rtmpose-m_simcc-body7_pt-body7_420e-256x192-e48f03d0_20230504.zip /models/rtmpose-m-body17 /models/rtmpose-m-body17.onnx
+        fetch_zip https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/rtmpose-m_simcc-ucoco_dw-ucoco_270e-256x192-c8b76419_20230728.zip /models/dwpose-m-wholebody /models/dwpose-m-wholebody.onnx
+        fetch_zip https://download.openmmlab.com/mmpose/v1/projects/rtmw/onnx_sdk/rtmw-dw-x-l_simcc-cocktail14_270e-256x192_20231122.zip /models/rtmw-l-wholebody /models/rtmw-l-wholebody.onnx
         sha256sum /models/*.pth > /models/SHA256SUMS
         chmod -R a-w /models
 
@@ -84,8 +100,9 @@ services:
       API_TOKEN_FILE: /run/secrets/api-token
       NDI_SOURCE_NAME: TAIL 2_1621D2 (OBSBOT)
       NDI_SOURCE_IP: 192.168.50.207
-      MODEL_PROFILE: balanced
+      MODEL_BACKEND: rtmlib_body
-      BENCHMARK_PROFILE: rtmdet-tiny_rtmw-m_gpu1_fp32_final
+      MODEL_PROFILE: rtmlib_body
+      BENCHMARK_PROFILE: rtmlib_body_yolox-m_rtmpose-m_gpu1
       MAX_PEOPLE: "4"
       DETECTION_THRESHOLD: "0.35"
       NMS_IOU_THRESHOLD: "0.60"
@@ -95,12 +112,12 @@ services:
       SAVE_CROPS: "false"
       LOG_COORDINATES: "false"
       CUDA_VISIBLE_DEVICES: "1"
-      DETECTOR_NAME: rtmdet-tiny-person
+      DETECTOR_NAME: yolox-m-humanart-coco-640x640
-      DET_CONFIG: /models/mmdetection/configs/rtmdet/rtmdet_tiny_8xb32-300e_coco.py
+      DET_CONFIG: ""
-      DET_CHECKPOINT: /models/rtmdet_tiny_8xb32-300e_coco_20220902_112414-78e30dcc.pth
+      DET_CHECKPOINT: /models/yolox-m-humanart.onnx
-      POSE_NAME: rtmw-m-256x192
+      POSE_NAME: rtmpose-m-body17-256x192
-      POSE_CONFIG: /models/mmpose/configs/wholebody_2d_keypoint/rtmpose/cocktail14/rtmw-m_8xb1024-270e_cocktail14-256x192.py
+      POSE_CONFIG: ""
-      POSE_CHECKPOINT: /models/rtmw-dw-l-m_simcc-cocktail14_270e-256x192-20231122.pth
+      POSE_CHECKPOINT: /models/rtmpose-m-body17.onnx
     healthcheck:
       test: ["CMD", "python", "-c", "import json,pathlib,urllib.request; t=pathlib.Path('/run/secrets/api-token').read_text().strip(); r=urllib.request.Request('http://127.0.0.1:18120/v1/health',headers={'Authorization':'Bearer '+t}); raise SystemExit(0 if json.load(urllib.request.urlopen(r,timeout=3))['ready'] else 1)"]
       interval: 15s
@@ -109,6 +126,7 @@ services:
       start_period: 180s
 
   gpu-audit:
+    profiles: ["validation"]
     image: tail2-pose-server:0.1.0
     container_name: tail2-pose-gpu-audit
     gpus: all

servers/unraid/tail2-pose-server/requirements.lock

@@ -16,4 +16,5 @@ nvidia-ml-py==12.560.30
 python-multipart==0.0.20
 pytest==8.3.4
 httpx==0.28.1
-
+onnxruntime-gpu==1.16.3
+tqdm==4.67.1

servers/unraid/tail2-pose-server/tests/test_core.py

@@ -32,7 +32,7 @@ def test_fixed_keypoint_counts_and_nonfinite_cleanup():
 
 def test_latest_queue_overwrites_old():
     q = LatestQueue(); q.put(1); q.put(2)
-    assert q.qsize() == 1 and q.get() == 2
+    assert q.qsize() == 1 and q.get() == (2, 1)
 
 
 def test_empty_people_contract():