kai/infra-docker-configs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add RTMLib CUDA backends and staged timing

Browse Source
This commit is contained in:
Codex
2026-08-15 10:58:02 -07:00
parent dba60667cb
commit 174ba875dd
8 changed files with 243 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
servers/unraid/tail2-pose-server/app/tail2/models.py
+158 -28
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import hashlib
import os
import time
from typing import Any
import numpy as np
import torch
@@ -10,13 +11,30 @@ import torch
from .core import normalize_bbox, normalize_keypoints
def _elapsed_ns(start: int) -> float:
return (time.perf_counter_ns() - start) / 1e6
def _cuda_timed(call):
torch.cuda.synchronize()
started = time.perf_counter_ns()
result = call()
torch.cuda.synchronize()
return result, _elapsed_ns(started)
class Models:
"""Persistent model holder for both baseline and RTMLib execution paths."""
def __init__(self) -> None:
self.det_config = os.environ["DET_CONFIG"]
self.det_checkpoint = os.environ["DET_CHECKPOINT"]
self.pose_config = os.environ["POSE_CONFIG"]
self.pose_checkpoint = os.environ["POSE_CHECKPOINT"]
self.backend = os.getenv("MODEL_BACKEND", "openmmlab")
self.det_config = os.getenv("DET_CONFIG", "")
self.det_checkpoint = os.getenv("DET_CHECKPOINT", "")
self.pose_config = os.getenv("POSE_CONFIG", "")
self.pose_checkpoint = os.getenv("POSE_CHECKPOINT", "")
self.detector = self.pose = None
self.providers: dict[str, list[str]] = {}
self.input_metadata: dict[str, Any] = {}
@staticmethod
def sha256(path: str) -> str:
@@ -27,49 +45,161 @@ class Models:
return digest.hexdigest()
def load(self) -> None:
from mmdet.apis import init_detector
from mmpose.apis import init_model
torch.backends.cudnn.benchmark = True
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
self.detector = init_detector(self.det_config, self.det_checkpoint, device="cuda:0")
self.detector.test_cfg.nms.iou_threshold = float(os.getenv("NMS_IOU_THRESHOLD", ".60"))
self.pose = init_model(self.pose_config, self.pose_checkpoint, device="cuda:0")
if self.backend == "openmmlab":
from mmdet.apis import init_detector
from mmpose.apis import init_model
self.detector = init_detector(self.det_config, self.det_checkpoint, device="cuda:0")
self.detector.test_cfg.nms.iou_threshold = float(os.getenv("NMS_IOU_THRESHOLD", ".60"))
self.pose = init_model(self.pose_config, self.pose_checkpoint, device="cuda:0")
self.providers = {"available": ["PyTorch CUDA"], "detector": ["PyTorch CUDA"], "pose": ["PyTorch CUDA"]}
self.input_metadata = {
"detector_parameter_device": str(next(self.detector.parameters()).device),
"pose_parameter_device": str(next(self.pose.parameters()).device),
}
return
if self.backend not in ("rtmlib_body", "rtmlib_wholebody", "rtmlib_rtmw_l"):
raise ValueError(f"unsupported MODEL_BACKEND={self.backend}")
import onnxruntime as ort
from rtmlib import RTMPose, YOLOX
self.detector = YOLOX(self.det_checkpoint, model_input_size=(640, 640), mode="human",
nms_thr=float(os.getenv("NMS_IOU_THRESHOLD", ".60")),
score_thr=float(os.getenv("DETECTION_THRESHOLD", ".35")),
backend="onnxruntime", device="cuda:0")
self.pose = RTMPose(self.pose_checkpoint, model_input_size=(192, 256),
backend="onnxruntime", device="cuda:0", to_openpose=False)
self.providers = {
"available": ort.get_available_providers(),
"detector": self.detector.session.get_providers(),
"pose": self.pose.session.get_providers(),
}
for name, providers in self.providers.items():
if name != "available" and (not providers or providers[0] != "CUDAExecutionProvider"):
raise RuntimeError(f"{name} is not using CUDAExecutionProvider first: {providers}")
self.input_metadata = {
"detector": [{"name": x.name, "shape": x.shape, "type": x.type} for x in self.detector.session.get_inputs()],
"pose": [{"name": x.name, "shape": x.shape, "type": x.type} for x in self.pose.session.get_inputs()],
}
print(f"ort.get_available_providers={self.providers['available']}", flush=True)
print(f"detector_session.get_providers={self.providers['detector']}", flush=True)
print(f"pose_session.get_providers={self.providers['pose']}", flush=True)
def infer(self, frame: np.ndarray, threshold: float, max_people: int) -> tuple[list[dict], dict[str, float]]:
if self.backend == "openmmlab":
return self._infer_openmmlab(frame, threshold, max_people)
return self._infer_rtmlib(frame, threshold, max_people)
@torch.inference_mode()
def infer(self, frame: np.ndarray, threshold: float, max_people: int) -> tuple[list[dict], float, float]:
def _infer_openmmlab(self, frame: np.ndarray, threshold: float, max_people: int):
from mmdet.apis import inference_detector
from mmengine.registry import init_default_scope
from mmpose.apis import inference_topdown
height, width = frame.shape[:2]
t0 = time.perf_counter_ns()
timing = {k: 0.0 for k in ("frame_copy_ms", "color_conversion_ms", "resize_letterbox_ms",
"cpu_to_gpu_ms", "detector_postprocess_ms", "bbox_crop_affine_ms",
"pose_decode_ms", "gpu_to_cpu_ms")}
init_default_scope("mmdet")
result = inference_detector(self.detector, frame)
result, timing["detector_forward_ms"] = _cuda_timed(lambda: inference_detector(self.detector, frame))
t = time.perf_counter_ns()
pred = result.pred_instances.cpu().numpy()
keep = (pred.labels == 0) & (pred.scores >= threshold)
boxes = pred.bboxes[keep]
scores = pred.scores[keep]
boxes, scores = pred.bboxes[keep], pred.scores[keep]
if len(boxes):
order = np.argsort(scores)[::-1][:max_people]
boxes, scores = boxes[order], scores[order]
det_ms = (time.perf_counter_ns() - t0) / 1e6
t1 = time.perf_counter_ns()
timing["detector_postprocess_ms"] = _elapsed_ns(t)
init_default_scope("mmpose")
pose_results = inference_topdown(self.pose, frame, bboxes=boxes) if len(boxes) else []
pose_ms = (time.perf_counter_ns() - t1) / 1e6
if len(boxes):
pose_results, timing["pose_forward_ms"] = _cuda_timed(lambda: inference_topdown(self.pose, frame, bboxes=boxes))
else:
pose_results, timing["pose_forward_ms"] = [], 0.0
t = time.perf_counter_ns()
people = []
for idx, (box, det_score, sample) in enumerate(zip(boxes, scores, pose_results)):
points = sample.pred_instances.keypoints[0]
kp_scores = sample.pred_instances.keypoint_scores[0]
combined = np.column_stack((points, kp_scores))
wholebody = normalize_keypoints(combined, width, height, 133)
people.append({
"detection_id": idx,
"bbox_xyxy_norm": normalize_bbox(box, width, height),
"det_score": float(det_score),
"keypoint_schema": "coco_wholebody_133",
"keypoints_norm": wholebody,
"body17": wholebody[:17],
"pose_score": float(np.mean(np.nan_to_num(kp_scores[:17]))),
})
return people, det_ms, pose_ms
people.append(self._person(idx, box, det_score, wholebody, "coco_wholebody_133", width, height))
timing["pose_decode_ms"] = _elapsed_ns(t)
timing["detector_ms"] = timing["detector_forward_ms"] + timing["detector_postprocess_ms"]
timing["pose_ms"] = timing["pose_forward_ms"] + timing["pose_decode_ms"]
return people, timing
def _yolox_postprocess(self, outputs: np.ndarray, ratio: float, threshold: float):
from rtmlib.tools.object_detection.post_processings import multiclass_nms
if outputs.shape[-1] == 5:
boxes, scores = outputs[0, :, :4] / ratio, outputs[0, :, 4]
keep = scores >= threshold
boxes, scores = boxes[keep], scores[keep]
order = np.argsort(scores)[::-1]
return boxes[order], scores[order]
grids, expanded = [], []
for stride in (8, 16, 32):
h = self.detector.model_input_size[0] // stride
w = self.detector.model_input_size[1] // stride
xv, yv = np.meshgrid(np.arange(w), np.arange(h))
grid = np.stack((xv, yv), 2).reshape(1, -1, 2)
grids.append(grid); expanded.append(np.full((*grid.shape[:2], 1), stride))
outputs = outputs.copy()
outputs[..., :2] = (outputs[..., :2] + np.concatenate(grids, 1)) * np.concatenate(expanded, 1)
outputs[..., 2:4] = np.exp(outputs[..., 2:4]) * np.concatenate(expanded, 1)
pred = outputs[0]; boxes = pred[:, :4]; scores = pred[:, 4:5] * pred[:, 5:]
xyxy = np.empty_like(boxes)
xyxy[:, 0] = boxes[:, 0] - boxes[:, 2] / 2; xyxy[:, 1] = boxes[:, 1] - boxes[:, 3] / 2
xyxy[:, 2] = boxes[:, 0] + boxes[:, 2] / 2; xyxy[:, 3] = boxes[:, 1] + boxes[:, 3] / 2
dets, _ = multiclass_nms(xyxy / ratio, scores, nms_thr=float(os.getenv("NMS_IOU_THRESHOLD", ".60")), score_thr=threshold)
if dets is None:
return np.empty((0, 4)), np.empty((0,))
dets = dets[dets[:, 5].astype(int) == 0]
order = np.argsort(dets[:, 4])[::-1]
return dets[order, :4], dets[order, 4]
def _infer_rtmlib(self, frame: np.ndarray, threshold: float, max_people: int):
height, width = frame.shape[:2]
timing = {k: 0.0 for k in ("frame_copy_ms", "color_conversion_ms", "cpu_to_gpu_ms", "gpu_to_cpu_ms")}
t = time.perf_counter_ns(); det_image, ratio = self.detector.preprocess(frame)
timing["resize_letterbox_ms"] = _elapsed_ns(t)
det_input = np.ascontiguousarray(det_image.transpose(2, 0, 1), dtype=np.float32)[None]
self.input_metadata["detector_runtime"] = {"device": "CPU pinned by ORT", "dtype": str(det_input.dtype), "shape": list(det_input.shape)}
def det_run():
return self.detector.session.run(None, {self.detector.session.get_inputs()[0].name: det_input})[0]
det_output, timing["detector_forward_ms"] = _cuda_timed(det_run)
t = time.perf_counter_ns(); boxes, det_scores = self._yolox_postprocess(det_output, ratio, threshold)
boxes, det_scores = boxes[:max_people], det_scores[:max_people]
timing["detector_postprocess_ms"] = _elapsed_ns(t)
people = []
pose_forward = pose_decode = crop_affine = 0.0
for idx, (box, det_score) in enumerate(zip(boxes, det_scores)):
t = time.perf_counter_ns(); pose_image, center, scale = self.pose.preprocess(frame, box)
crop_affine += _elapsed_ns(t)
pose_input = np.ascontiguousarray(pose_image.transpose(2, 0, 1), dtype=np.float32)[None]
self.input_metadata["pose_runtime"] = {"device": "CPU pinned by ORT", "dtype": str(pose_input.dtype), "shape": list(pose_input.shape)}
def pose_run():
return self.pose.session.run(None, {self.pose.session.get_inputs()[0].name: pose_input})
outputs, elapsed = _cuda_timed(pose_run); pose_forward += elapsed
t = time.perf_counter_ns(); points, scores = self.pose.postprocess(outputs, center, scale)
pose_decode += _elapsed_ns(t)
combined = np.column_stack((points[0], scores[0]))
count = 17 if self.backend == "rtmlib_body" else 133
normalized = normalize_keypoints(combined, width, height, count)
people.append(self._person(idx, box, det_score, normalized,
"coco17" if count == 17 else "coco_wholebody_133", width, height))
timing.update({"bbox_crop_affine_ms": crop_affine, "pose_forward_ms": pose_forward,
"pose_decode_ms": pose_decode,
"detector_ms": timing["detector_forward_ms"] + timing["detector_postprocess_ms"],
"pose_ms": crop_affine + pose_forward + pose_decode})
return people, timing
@staticmethod
def _person(idx, box, det_score, keypoints, schema, width, height):
body17 = keypoints[:17]
item = {"detection_id": idx, "bbox_xyxy_norm": normalize_bbox(box, width, height),
"det_score": float(np.clip(det_score, 0, 1)), "keypoint_schema": schema,
"body17": body17, "pose_score": float(np.mean([p[2] for p in body17]))}
if len(keypoints) == 133:
item["keypoints_norm"] = keypoints
return item