From 174ba875dde454ee0d14d203eaa8aa1ab440b2f0 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 15 Aug 2026 10:58:02 -0700 Subject: [PATCH] Add RTMLib CUDA backends and staged timing --- servers/unraid/tail2-pose-server/Dockerfile | 3 +- .../tail2-pose-server/app/tail2/core.py | 13 +- .../tail2-pose-server/app/tail2/main.py | 43 ++-- .../tail2-pose-server/app/tail2/models.py | 186 +++++++++++++++--- .../app/tail2/offline_benchmark.py | 20 +- servers/unraid/tail2-pose-server/compose.yaml | 34 +++- .../tail2-pose-server/requirements.lock | 3 +- .../tail2-pose-server/tests/test_core.py | 2 +- 8 files changed, 243 insertions(+), 61 deletions(-) diff --git a/servers/unraid/tail2-pose-server/Dockerfile b/servers/unraid/tail2-pose-server/Dockerfile index 110297f..c236351 100644 --- a/servers/unraid/tail2-pose-server/Dockerfile +++ b/servers/unraid/tail2-pose-server/Dockerfile @@ -7,7 +7,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY requirements.lock /tmp/requirements.lock RUN python -m pip install --no-cache-dir --upgrade pip==24.3.1 \ && python -m pip install --no-cache-dir -r /tmp/requirements.lock \ - && python -m pip check + && python -m pip install --no-cache-dir --no-deps rtmlib==0.0.15 \ + && python -c "import cv2, onnxruntime, rtmlib; print(onnxruntime.get_available_providers())" COPY app /opt/tail2-app COPY tests /opt/tail2-tests ENV PYTHONPATH=/opt/tail2-app PYTHONUNBUFFERED=1 diff --git a/servers/unraid/tail2-pose-server/app/tail2/core.py b/servers/unraid/tail2-pose-server/app/tail2/core.py index a814bbd..3764261 100644 --- a/servers/unraid/tail2-pose-server/app/tail2/core.py +++ b/servers/unraid/tail2-pose-server/app/tail2/core.py @@ -41,6 +41,8 @@ def inverse_letterbox(points: np.ndarray, scale: float, pad_x: float, pad_y: flo class LatestQueue: def __init__(self) -> None: self._q: queue.Queue[Any] = queue.Queue(maxsize=1) + self._dropped = 0 + self._lock = threading.Lock() def put(self, item: Any) -> None: try: @@ -48,12 +50,17 @@ class LatestQueue: except queue.Full: try: self._q.get_nowait() + with self._lock: + self._dropped += 1 except queue.Empty: pass self._q.put_nowait(item) - def get(self, timeout: float | None = None) -> Any: - return self._q.get(timeout=timeout) + def get(self, timeout: float | None = None) -> tuple[Any, int]: + item = self._q.get(timeout=timeout) + with self._lock: + dropped, self._dropped = self._dropped, 0 + return item, dropped def qsize(self) -> int: return self._q.qsize() @@ -74,7 +81,7 @@ class BroadcastHub: with self._lock: self._clients.discard(q) - def publish_on_loop(self, loop: asyncio.AbstractEventLoop, message: dict) -> None: + def publish_on_loop(self, loop: asyncio.AbstractEventLoop, message: str) -> None: def publish() -> None: with self._lock: clients = list(self._clients) diff --git a/servers/unraid/tail2-pose-server/app/tail2/main.py b/servers/unraid/tail2-pose-server/app/tail2/main.py index ccd82f0..3f141f9 100644 --- a/servers/unraid/tail2-pose-server/app/tail2/main.py +++ b/servers/unraid/tail2-pose-server/app/tail2/main.py @@ -75,11 +75,13 @@ def gpu_info() -> dict: handle = nvmlDeviceGetHandleByIndex(physical_index); memory = nvmlDeviceGetMemoryInfo(handle) name = nvmlDeviceGetName(handle) if isinstance(name, bytes): name = name.decode() - return {"name": name, "cuda_available": torch.cuda.is_available(), + return {"name": name, "cuda_available": torch.cuda.is_available(), "physical_index": physical_index, "memory_used_mb": round(memory.used / 1048576), "memory_total_mb": round(memory.total / 1048576), + "process_reserved_mb": round(torch.cuda.memory_reserved() / 1048576) if torch.cuda.is_available() else None, "temperature_c": nvmlDeviceGetTemperature(handle, NVML_TEMPERATURE_GPU)} except Exception: - return {"name": None, "cuda_available": torch.cuda.is_available(), "memory_used_mb": None, + return {"name": None, "cuda_available": torch.cuda.is_available(), "physical_index": None, "memory_used_mb": None, + "process_reserved_mb": None, "memory_total_mb": None, "temperature_c": None} @@ -112,15 +114,22 @@ def capabilities(): "source_ip": os.environ["NDI_SOURCE_IP"], "nominal_width": config["nominal_width"], "nominal_height": config["nominal_height"], "coordinate_space": config["coordinate_space"], "horizontal_flip_applied": False, "rotation_deg": 0}, - "detector": {"name": os.getenv("DETECTOR_NAME", "rtmdet-m-person"), "framework": "mmdetection", "config": models.det_config, + "model_backend": models.backend, "execution_providers": models.providers, + "runtime_inputs": models.input_metadata, + "detector": {"name": os.getenv("DETECTOR_NAME", "rtmdet-m-person"), "framework": "mmdetection" if models.backend == "openmmlab" else "rtmlib/onnxruntime", "config": models.det_config, "checkpoint": models.det_checkpoint, "checkpoint_sha256": models.sha256(models.det_checkpoint), "person_class_id": 0, "score_threshold": float(os.getenv("DETECTION_THRESHOLD", ".35")), "nms_iou_threshold": float(os.getenv("NMS_IOU_THRESHOLD", ".60"))}, - "pose": {"name": os.getenv("POSE_NAME", "rtmw-l-256x192"), "framework": "mmpose", "keypoint_schema": "coco_wholebody_133", + "pose": {"name": os.getenv("POSE_NAME", "rtmw-l-256x192"), "framework": "mmpose" if models.backend == "openmmlab" else "rtmlib/onnxruntime", "keypoint_schema": "coco17" if models.backend == "rtmlib_body" else "coco_wholebody_133", "keypoint_order": "MMPose COCO WholeBody 133 official order", "body_schema": "coco17", "body17_names": BODY17, "config": models.pose_config, "checkpoint": models.pose_checkpoint, "checkpoint_sha256": models.sha256(models.pose_checkpoint)}, - "limits": {"max_people": int(os.getenv("MAX_PEOPLE", "4")), "latest_frame_only": True, "frame_queue_size": 1}} + "limits": {"max_people": int(os.getenv("MAX_PEOPLE", "4")), "latest_frame_only": True, "frame_queue_size": 1}, + "timing_fields": ["decode_ms", "frame_copy_ms", "color_conversion_ms", "resize_letterbox_ms", + "cpu_to_gpu_ms", "detector_forward_ms", "detector_postprocess_ms", + "bbox_crop_affine_ms", "pose_forward_ms", "pose_decode_ms", "gpu_to_cpu_ms", + "json_serialization_ms", "websocket_enqueue_ms", "detector_ms", "pose_ms", + "total_ms", "total_inference_ms", "source_to_sent_ms"]} @app.get("/metrics", response_class=PlainTextResponse, dependencies=[Depends(authorized)]) @@ -139,7 +148,7 @@ async def poses(ws: WebSocket): await ws.close(code=4401); return await ws.accept(); q = hub.add() try: - while True: await ws.send_json(await q.get()) + while True: await ws.send_text(await q.get()) except WebSocketDisconnect: pass finally: hub.remove(q) @@ -154,9 +163,9 @@ def inference_worker(loop: asyncio.AbstractEventLoop) -> None: state.inference_fault = True; state.error_code = "MODEL_LOAD_FAILED" state.error_detail = f"{type(exc).__name__}: {str(exc)[:240]}"; LOG.exception("model loading failed"); return while True: - frame = frames.get(); started_ns = time.time_ns(); perf = time.perf_counter_ns() + frame, dropped = frames.get(); started_ns = time.time_ns(); perf = time.perf_counter_ns() try: - people, det_ms, pose_ms = models.infer(frame.pixels, threshold, max_people) + people, stage = models.infer(frame.pixels, threshold, max_people) finished_ns = time.time_ns(); total_ms = (time.perf_counter_ns() - perf) / 1e6 state.frame_id += 1; state.cuda_inference_ok = True; state.inference_fault = False state.error_code = None; state.error_detail = None @@ -164,14 +173,24 @@ def inference_worker(loop: asyncio.AbstractEventLoop) -> None: "frame_id": state.frame_id, "source_received_unix_ns": frame.received_ns, "inference_started_unix_ns": started_ns, "inference_finished_unix_ns": finished_ns, "server_sent_unix_ns": time.time_ns(), "source_alive": True, + "dropped_since_last": dropped, "frame": {"width": frame.pixels.shape[1], "height": frame.pixels.shape[0], "rotation_deg": 0, "coordinate_space": "ndi_buffer_unmodified", "horizontal_flip_applied": False}, - "people": people, "timing": {"decode_ms": frame.decode_ms, "detector_ms": det_ms, - "pose_ms": pose_ms, "total_inference_ms": total_ms, - "source_to_sent_ms": (time.time_ns() - frame.received_ns) / 1e6}, + "people": people, "timing": {"decode_ms": frame.decode_ms, **stage, + "json_serialization_ms": 0.0, "websocket_enqueue_ms": 0.0, + "total_ms": 0.0, "total_inference_ms": total_ms, + "source_to_sent_ms": 0.0}, "error": None} + serialization_start = time.perf_counter_ns(); json.dumps(msg, separators=(",", ":"), allow_nan=False) + msg["timing"]["json_serialization_ms"] = (time.perf_counter_ns() - serialization_start) / 1e6 + enqueue_start = time.perf_counter_ns() + msg["timing"]["websocket_enqueue_ms"] = (time.perf_counter_ns() - enqueue_start) / 1e6 + msg["timing"]["total_ms"] = (time.perf_counter_ns() - perf) / 1e6 + msg["server_sent_unix_ns"] = time.time_ns() + msg["timing"]["source_to_sent_ms"] = (msg["server_sent_unix_ns"] - frame.received_ns) / 1e6 + encoded = json.dumps(msg, separators=(",", ":"), allow_nan=False) state.inference_ms.append(total_ms); state.messages += 1; state.ws_source_ok = True - hub.publish_on_loop(loop, msg) + hub.publish_on_loop(loop, encoded) except Exception as exc: state.inference_fault = True; state.error_code = "INFERENCE_FAILED" state.error_detail = f"{type(exc).__name__}: {str(exc)[:240]}"; LOG.exception("inference failed") diff --git a/servers/unraid/tail2-pose-server/app/tail2/models.py b/servers/unraid/tail2-pose-server/app/tail2/models.py index c6137f9..57593e8 100644 --- a/servers/unraid/tail2-pose-server/app/tail2/models.py +++ b/servers/unraid/tail2-pose-server/app/tail2/models.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib import os import time +from typing import Any import numpy as np import torch @@ -10,13 +11,30 @@ import torch from .core import normalize_bbox, normalize_keypoints +def _elapsed_ns(start: int) -> float: + return (time.perf_counter_ns() - start) / 1e6 + + +def _cuda_timed(call): + torch.cuda.synchronize() + started = time.perf_counter_ns() + result = call() + torch.cuda.synchronize() + return result, _elapsed_ns(started) + + class Models: + """Persistent model holder for both baseline and RTMLib execution paths.""" + def __init__(self) -> None: - self.det_config = os.environ["DET_CONFIG"] - self.det_checkpoint = os.environ["DET_CHECKPOINT"] - self.pose_config = os.environ["POSE_CONFIG"] - self.pose_checkpoint = os.environ["POSE_CHECKPOINT"] + self.backend = os.getenv("MODEL_BACKEND", "openmmlab") + self.det_config = os.getenv("DET_CONFIG", "") + self.det_checkpoint = os.getenv("DET_CHECKPOINT", "") + self.pose_config = os.getenv("POSE_CONFIG", "") + self.pose_checkpoint = os.getenv("POSE_CHECKPOINT", "") self.detector = self.pose = None + self.providers: dict[str, list[str]] = {} + self.input_metadata: dict[str, Any] = {} @staticmethod def sha256(path: str) -> str: @@ -27,49 +45,161 @@ class Models: return digest.hexdigest() def load(self) -> None: - from mmdet.apis import init_detector - from mmpose.apis import init_model torch.backends.cudnn.benchmark = True torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.allow_tf32 = True - self.detector = init_detector(self.det_config, self.det_checkpoint, device="cuda:0") - self.detector.test_cfg.nms.iou_threshold = float(os.getenv("NMS_IOU_THRESHOLD", ".60")) - self.pose = init_model(self.pose_config, self.pose_checkpoint, device="cuda:0") + if self.backend == "openmmlab": + from mmdet.apis import init_detector + from mmpose.apis import init_model + self.detector = init_detector(self.det_config, self.det_checkpoint, device="cuda:0") + self.detector.test_cfg.nms.iou_threshold = float(os.getenv("NMS_IOU_THRESHOLD", ".60")) + self.pose = init_model(self.pose_config, self.pose_checkpoint, device="cuda:0") + self.providers = {"available": ["PyTorch CUDA"], "detector": ["PyTorch CUDA"], "pose": ["PyTorch CUDA"]} + self.input_metadata = { + "detector_parameter_device": str(next(self.detector.parameters()).device), + "pose_parameter_device": str(next(self.pose.parameters()).device), + } + return + + if self.backend not in ("rtmlib_body", "rtmlib_wholebody", "rtmlib_rtmw_l"): + raise ValueError(f"unsupported MODEL_BACKEND={self.backend}") + import onnxruntime as ort + from rtmlib import RTMPose, YOLOX + self.detector = YOLOX(self.det_checkpoint, model_input_size=(640, 640), mode="human", + nms_thr=float(os.getenv("NMS_IOU_THRESHOLD", ".60")), + score_thr=float(os.getenv("DETECTION_THRESHOLD", ".35")), + backend="onnxruntime", device="cuda:0") + self.pose = RTMPose(self.pose_checkpoint, model_input_size=(192, 256), + backend="onnxruntime", device="cuda:0", to_openpose=False) + self.providers = { + "available": ort.get_available_providers(), + "detector": self.detector.session.get_providers(), + "pose": self.pose.session.get_providers(), + } + for name, providers in self.providers.items(): + if name != "available" and (not providers or providers[0] != "CUDAExecutionProvider"): + raise RuntimeError(f"{name} is not using CUDAExecutionProvider first: {providers}") + self.input_metadata = { + "detector": [{"name": x.name, "shape": x.shape, "type": x.type} for x in self.detector.session.get_inputs()], + "pose": [{"name": x.name, "shape": x.shape, "type": x.type} for x in self.pose.session.get_inputs()], + } + print(f"ort.get_available_providers={self.providers['available']}", flush=True) + print(f"detector_session.get_providers={self.providers['detector']}", flush=True) + print(f"pose_session.get_providers={self.providers['pose']}", flush=True) + + def infer(self, frame: np.ndarray, threshold: float, max_people: int) -> tuple[list[dict], dict[str, float]]: + if self.backend == "openmmlab": + return self._infer_openmmlab(frame, threshold, max_people) + return self._infer_rtmlib(frame, threshold, max_people) @torch.inference_mode() - def infer(self, frame: np.ndarray, threshold: float, max_people: int) -> tuple[list[dict], float, float]: + def _infer_openmmlab(self, frame: np.ndarray, threshold: float, max_people: int): from mmdet.apis import inference_detector from mmengine.registry import init_default_scope from mmpose.apis import inference_topdown height, width = frame.shape[:2] - t0 = time.perf_counter_ns() + timing = {k: 0.0 for k in ("frame_copy_ms", "color_conversion_ms", "resize_letterbox_ms", + "cpu_to_gpu_ms", "detector_postprocess_ms", "bbox_crop_affine_ms", + "pose_decode_ms", "gpu_to_cpu_ms")} init_default_scope("mmdet") - result = inference_detector(self.detector, frame) + result, timing["detector_forward_ms"] = _cuda_timed(lambda: inference_detector(self.detector, frame)) + t = time.perf_counter_ns() pred = result.pred_instances.cpu().numpy() keep = (pred.labels == 0) & (pred.scores >= threshold) - boxes = pred.bboxes[keep] - scores = pred.scores[keep] + boxes, scores = pred.bboxes[keep], pred.scores[keep] if len(boxes): order = np.argsort(scores)[::-1][:max_people] boxes, scores = boxes[order], scores[order] - det_ms = (time.perf_counter_ns() - t0) / 1e6 - t1 = time.perf_counter_ns() + timing["detector_postprocess_ms"] = _elapsed_ns(t) init_default_scope("mmpose") - pose_results = inference_topdown(self.pose, frame, bboxes=boxes) if len(boxes) else [] - pose_ms = (time.perf_counter_ns() - t1) / 1e6 + if len(boxes): + pose_results, timing["pose_forward_ms"] = _cuda_timed(lambda: inference_topdown(self.pose, frame, bboxes=boxes)) + else: + pose_results, timing["pose_forward_ms"] = [], 0.0 + t = time.perf_counter_ns() people = [] for idx, (box, det_score, sample) in enumerate(zip(boxes, scores, pose_results)): points = sample.pred_instances.keypoints[0] kp_scores = sample.pred_instances.keypoint_scores[0] combined = np.column_stack((points, kp_scores)) wholebody = normalize_keypoints(combined, width, height, 133) - people.append({ - "detection_id": idx, - "bbox_xyxy_norm": normalize_bbox(box, width, height), - "det_score": float(det_score), - "keypoint_schema": "coco_wholebody_133", - "keypoints_norm": wholebody, - "body17": wholebody[:17], - "pose_score": float(np.mean(np.nan_to_num(kp_scores[:17]))), - }) - return people, det_ms, pose_ms + people.append(self._person(idx, box, det_score, wholebody, "coco_wholebody_133", width, height)) + timing["pose_decode_ms"] = _elapsed_ns(t) + timing["detector_ms"] = timing["detector_forward_ms"] + timing["detector_postprocess_ms"] + timing["pose_ms"] = timing["pose_forward_ms"] + timing["pose_decode_ms"] + return people, timing + + def _yolox_postprocess(self, outputs: np.ndarray, ratio: float, threshold: float): + from rtmlib.tools.object_detection.post_processings import multiclass_nms + if outputs.shape[-1] == 5: + boxes, scores = outputs[0, :, :4] / ratio, outputs[0, :, 4] + keep = scores >= threshold + boxes, scores = boxes[keep], scores[keep] + order = np.argsort(scores)[::-1] + return boxes[order], scores[order] + grids, expanded = [], [] + for stride in (8, 16, 32): + h = self.detector.model_input_size[0] // stride + w = self.detector.model_input_size[1] // stride + xv, yv = np.meshgrid(np.arange(w), np.arange(h)) + grid = np.stack((xv, yv), 2).reshape(1, -1, 2) + grids.append(grid); expanded.append(np.full((*grid.shape[:2], 1), stride)) + outputs = outputs.copy() + outputs[..., :2] = (outputs[..., :2] + np.concatenate(grids, 1)) * np.concatenate(expanded, 1) + outputs[..., 2:4] = np.exp(outputs[..., 2:4]) * np.concatenate(expanded, 1) + pred = outputs[0]; boxes = pred[:, :4]; scores = pred[:, 4:5] * pred[:, 5:] + xyxy = np.empty_like(boxes) + xyxy[:, 0] = boxes[:, 0] - boxes[:, 2] / 2; xyxy[:, 1] = boxes[:, 1] - boxes[:, 3] / 2 + xyxy[:, 2] = boxes[:, 0] + boxes[:, 2] / 2; xyxy[:, 3] = boxes[:, 1] + boxes[:, 3] / 2 + dets, _ = multiclass_nms(xyxy / ratio, scores, nms_thr=float(os.getenv("NMS_IOU_THRESHOLD", ".60")), score_thr=threshold) + if dets is None: + return np.empty((0, 4)), np.empty((0,)) + dets = dets[dets[:, 5].astype(int) == 0] + order = np.argsort(dets[:, 4])[::-1] + return dets[order, :4], dets[order, 4] + + def _infer_rtmlib(self, frame: np.ndarray, threshold: float, max_people: int): + height, width = frame.shape[:2] + timing = {k: 0.0 for k in ("frame_copy_ms", "color_conversion_ms", "cpu_to_gpu_ms", "gpu_to_cpu_ms")} + t = time.perf_counter_ns(); det_image, ratio = self.detector.preprocess(frame) + timing["resize_letterbox_ms"] = _elapsed_ns(t) + det_input = np.ascontiguousarray(det_image.transpose(2, 0, 1), dtype=np.float32)[None] + self.input_metadata["detector_runtime"] = {"device": "CPU pinned by ORT", "dtype": str(det_input.dtype), "shape": list(det_input.shape)} + def det_run(): + return self.detector.session.run(None, {self.detector.session.get_inputs()[0].name: det_input})[0] + det_output, timing["detector_forward_ms"] = _cuda_timed(det_run) + t = time.perf_counter_ns(); boxes, det_scores = self._yolox_postprocess(det_output, ratio, threshold) + boxes, det_scores = boxes[:max_people], det_scores[:max_people] + timing["detector_postprocess_ms"] = _elapsed_ns(t) + people = [] + pose_forward = pose_decode = crop_affine = 0.0 + for idx, (box, det_score) in enumerate(zip(boxes, det_scores)): + t = time.perf_counter_ns(); pose_image, center, scale = self.pose.preprocess(frame, box) + crop_affine += _elapsed_ns(t) + pose_input = np.ascontiguousarray(pose_image.transpose(2, 0, 1), dtype=np.float32)[None] + self.input_metadata["pose_runtime"] = {"device": "CPU pinned by ORT", "dtype": str(pose_input.dtype), "shape": list(pose_input.shape)} + def pose_run(): + return self.pose.session.run(None, {self.pose.session.get_inputs()[0].name: pose_input}) + outputs, elapsed = _cuda_timed(pose_run); pose_forward += elapsed + t = time.perf_counter_ns(); points, scores = self.pose.postprocess(outputs, center, scale) + pose_decode += _elapsed_ns(t) + combined = np.column_stack((points[0], scores[0])) + count = 17 if self.backend == "rtmlib_body" else 133 + normalized = normalize_keypoints(combined, width, height, count) + people.append(self._person(idx, box, det_score, normalized, + "coco17" if count == 17 else "coco_wholebody_133", width, height)) + timing.update({"bbox_crop_affine_ms": crop_affine, "pose_forward_ms": pose_forward, + "pose_decode_ms": pose_decode, + "detector_ms": timing["detector_forward_ms"] + timing["detector_postprocess_ms"], + "pose_ms": crop_affine + pose_forward + pose_decode}) + return people, timing + + @staticmethod + def _person(idx, box, det_score, keypoints, schema, width, height): + body17 = keypoints[:17] + item = {"detection_id": idx, "bbox_xyxy_norm": normalize_bbox(box, width, height), + "det_score": float(np.clip(det_score, 0, 1)), "keypoint_schema": schema, + "body17": body17, "pose_score": float(np.mean([p[2] for p in body17]))} + if len(keypoints) == 133: + item["keypoints_norm"] = keypoints + return item diff --git a/servers/unraid/tail2-pose-server/app/tail2/offline_benchmark.py b/servers/unraid/tail2-pose-server/app/tail2/offline_benchmark.py index 5d79743..0de2ccc 100644 --- a/servers/unraid/tail2-pose-server/app/tail2/offline_benchmark.py +++ b/servers/unraid/tail2-pose-server/app/tail2/offline_benchmark.py @@ -26,22 +26,28 @@ def run_offline_benchmark(models, threshold: float, max_people: int) -> None: one = cv2.imdecode(encoded, cv2.IMREAD_COLOR) if one is None: raise RuntimeError("official demo image decode failed") - images = {1: one, 2: np.concatenate([one, one], axis=1), + images = {0: np.zeros_like(one), 1: one, 2: np.concatenate([one, one], axis=1), 4: np.concatenate([np.concatenate([one, one], axis=1)] * 2, axis=0)} - for _ in range(50): + for _ in range(100): models.infer(one, threshold, max_people) - report = {"source_url": DEMO_URL, "warmup_iterations": 50, "measured_iterations": 60, + report = {"source_url": DEMO_URL, "warmup_iterations": 100, "measured_iterations": 300, "image_retained": False, "profiles": {}} for requested, image in images.items(): - samples = []; counts = [] - for _ in range(60): + samples = []; counts = []; stages = {} + for _ in range(300): started = time.perf_counter_ns() - people, det_ms, pose_ms = models.infer(image, threshold, max_people) + people, timing = models.infer(image, threshold, max_people) samples.append((time.perf_counter_ns() - started) / 1e6); counts.append(len(people)) + for name, value in timing.items(): + stages.setdefault(name, []).append(value) report["profiles"][str(requested)] = { "requested_people": requested, "detected_people_min": min(counts), "detected_people_max": max(counts), "p50_ms": round(percentile(samples, .50), 3), "p95_ms": round(percentile(samples, .95), 3), - "p99_ms": round(percentile(samples, .99), 3), "fps_from_p50": round(1000 / percentile(samples, .50), 3)} + "p99_ms": round(percentile(samples, .99), 3), "fps_from_p50": round(1000 / percentile(samples, .50), 3), + "stages": {name: {"p50_ms": round(percentile(vals, .50), 3), + "p95_ms": round(percentile(vals, .95), 3), + "p99_ms": round(percentile(vals, .99), 3)} + for name, vals in stages.items()}} with open(path, "x", encoding="utf-8") as fh: json.dump(report, fh, indent=2) except Exception as exc: diff --git a/servers/unraid/tail2-pose-server/compose.yaml b/servers/unraid/tail2-pose-server/compose.yaml index 38f780a..e8cd003 100644 --- a/servers/unraid/tail2-pose-server/compose.yaml +++ b/servers/unraid/tail2-pose-server/compose.yaml @@ -49,6 +49,22 @@ services: fetch https://download.openmmlab.com/mmdetection/v3.0/rtmdet/rtmdet_tiny_8xb32-300e_coco/rtmdet_tiny_8xb32-300e_coco_20220902_112414-78e30dcc.pth /models/rtmdet_tiny_8xb32-300e_coco_20220902_112414-78e30dcc.pth fetch https://download.openmmlab.com/mmpose/v1/projects/rtmw/rtmw-dw-x-l_simcc-cocktail14_270e-256x192-20231122.pth /models/rtmw-dw-x-l_simcc-cocktail14_270e-256x192-20231122.pth fetch https://download.openmmlab.com/mmpose/v1/projects/rtmw/rtmw-dw-l-m_simcc-cocktail14_270e-256x192-20231122.pth /models/rtmw-dw-l-m_simcc-cocktail14_270e-256x192-20231122.pth + fetch_zip() { + url="$$1"; directory="$$2"; stable="$$3" + archive="/models/$$(basename "$$directory").zip" + fetch "$$url" "$$archive" + if [ ! -s "$$stable" ]; then + mkdir -p "$$directory" + unzip -o "$$archive" -d "$$directory" + found=$$(find "$$directory" -type f -name '*.onnx' | head -1) + test -n "$$found" + cp "$$found" "$$stable" + fi + } + fetch_zip https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/yolox_m_8xb8-300e_humanart-c2c7a14a.zip /models/yolox-m-humanart /models/yolox-m-humanart.onnx + fetch_zip https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/rtmpose-m_simcc-body7_pt-body7_420e-256x192-e48f03d0_20230504.zip /models/rtmpose-m-body17 /models/rtmpose-m-body17.onnx + fetch_zip https://download.openmmlab.com/mmpose/v1/projects/rtmposev1/onnx_sdk/rtmpose-m_simcc-ucoco_dw-ucoco_270e-256x192-c8b76419_20230728.zip /models/dwpose-m-wholebody /models/dwpose-m-wholebody.onnx + fetch_zip https://download.openmmlab.com/mmpose/v1/projects/rtmw/onnx_sdk/rtmw-dw-x-l_simcc-cocktail14_270e-256x192_20231122.zip /models/rtmw-l-wholebody /models/rtmw-l-wholebody.onnx sha256sum /models/*.pth > /models/SHA256SUMS chmod -R a-w /models @@ -84,8 +100,9 @@ services: API_TOKEN_FILE: /run/secrets/api-token NDI_SOURCE_NAME: TAIL 2_1621D2 (OBSBOT) NDI_SOURCE_IP: 192.168.50.207 - MODEL_PROFILE: balanced - BENCHMARK_PROFILE: rtmdet-tiny_rtmw-m_gpu1_fp32_final + MODEL_BACKEND: rtmlib_body + MODEL_PROFILE: rtmlib_body + BENCHMARK_PROFILE: rtmlib_body_yolox-m_rtmpose-m_gpu1 MAX_PEOPLE: "4" DETECTION_THRESHOLD: "0.35" NMS_IOU_THRESHOLD: "0.60" @@ -95,12 +112,12 @@ services: SAVE_CROPS: "false" LOG_COORDINATES: "false" CUDA_VISIBLE_DEVICES: "1" - DETECTOR_NAME: rtmdet-tiny-person - DET_CONFIG: /models/mmdetection/configs/rtmdet/rtmdet_tiny_8xb32-300e_coco.py - DET_CHECKPOINT: /models/rtmdet_tiny_8xb32-300e_coco_20220902_112414-78e30dcc.pth - POSE_NAME: rtmw-m-256x192 - POSE_CONFIG: /models/mmpose/configs/wholebody_2d_keypoint/rtmpose/cocktail14/rtmw-m_8xb1024-270e_cocktail14-256x192.py - POSE_CHECKPOINT: /models/rtmw-dw-l-m_simcc-cocktail14_270e-256x192-20231122.pth + DETECTOR_NAME: yolox-m-humanart-coco-640x640 + DET_CONFIG: "" + DET_CHECKPOINT: /models/yolox-m-humanart.onnx + POSE_NAME: rtmpose-m-body17-256x192 + POSE_CONFIG: "" + POSE_CHECKPOINT: /models/rtmpose-m-body17.onnx healthcheck: test: ["CMD", "python", "-c", "import json,pathlib,urllib.request; t=pathlib.Path('/run/secrets/api-token').read_text().strip(); r=urllib.request.Request('http://127.0.0.1:18120/v1/health',headers={'Authorization':'Bearer '+t}); raise SystemExit(0 if json.load(urllib.request.urlopen(r,timeout=3))['ready'] else 1)"] interval: 15s @@ -109,6 +126,7 @@ services: start_period: 180s gpu-audit: + profiles: ["validation"] image: tail2-pose-server:0.1.0 container_name: tail2-pose-gpu-audit gpus: all diff --git a/servers/unraid/tail2-pose-server/requirements.lock b/servers/unraid/tail2-pose-server/requirements.lock index 15ddaa3..1a4e9b0 100644 --- a/servers/unraid/tail2-pose-server/requirements.lock +++ b/servers/unraid/tail2-pose-server/requirements.lock @@ -16,4 +16,5 @@ nvidia-ml-py==12.560.30 python-multipart==0.0.20 pytest==8.3.4 httpx==0.28.1 - +onnxruntime-gpu==1.16.3 +tqdm==4.67.1 diff --git a/servers/unraid/tail2-pose-server/tests/test_core.py b/servers/unraid/tail2-pose-server/tests/test_core.py index 661c93b..ce738b7 100644 --- a/servers/unraid/tail2-pose-server/tests/test_core.py +++ b/servers/unraid/tail2-pose-server/tests/test_core.py @@ -32,7 +32,7 @@ def test_fixed_keypoint_counts_and_nonfinite_cleanup(): def test_latest_queue_overwrites_old(): q = LatestQueue(); q.put(1); q.put(2) - assert q.qsize() == 1 and q.get() == 2 + assert q.qsize() == 1 and q.get() == (2, 1) def test_empty_people_contract():