Separate ORT transfer timing with IO binding

servers/unraid/tail2-pose-server/app/tail2/models.py

@@ -24,6 +24,20 @@ def _cuda_timed(call):
     return result, _elapsed_ns(started)
 
 
+def _ort_iobinding_run(session, input_array: np.ndarray):
+    """Measure H2D, graph execution, and D2H as separate synchronized stages."""
+    import onnxruntime as ort
+    device_input, h2d_ms = _cuda_timed(lambda: ort.OrtValue.ortvalue_from_numpy(input_array, "cuda", 0))
+    binding = session.io_binding()
+    binding.bind_ortvalue_input(session.get_inputs()[0].name, device_input)
+    for output in session.get_outputs():
+        binding.bind_output(output.name, "cuda", 0)
+    _, forward_ms = _cuda_timed(lambda: session.run_with_iobinding(binding))
+    device_outputs = binding.get_outputs()
+    outputs, d2h_ms = _cuda_timed(lambda: [value.numpy() for value in device_outputs])
+    return outputs, h2d_ms, forward_ms, d2h_ms
+
+
 class Models:
     """Persistent model holder for both baseline and RTMLib execution paths."""
 
@@ -167,9 +181,8 @@ class Models:
         timing["resize_letterbox_ms"] = _elapsed_ns(t)
         det_input = np.ascontiguousarray(det_image.transpose(2, 0, 1), dtype=np.float32)[None]
         self.input_metadata["detector_runtime"] = {"device": "CPU pinned by ORT", "dtype": str(det_input.dtype), "shape": list(det_input.shape)}
-        def det_run():
+        det_outputs, timing["cpu_to_gpu_ms"], timing["detector_forward_ms"], timing["gpu_to_cpu_ms"] = _ort_iobinding_run(self.detector.session, det_input)
-            return self.detector.session.run(None, {self.detector.session.get_inputs()[0].name: det_input})[0]
+        det_output = det_outputs[0]
-        det_output, timing["detector_forward_ms"] = _cuda_timed(det_run)
         t = time.perf_counter_ns(); boxes, det_scores = self._yolox_postprocess(det_output, ratio, threshold)
         boxes, det_scores = boxes[:max_people], det_scores[:max_people]
         timing["detector_postprocess_ms"] = _elapsed_ns(t)
@@ -180,9 +193,8 @@ class Models:
             crop_affine += _elapsed_ns(t)
             pose_input = np.ascontiguousarray(pose_image.transpose(2, 0, 1), dtype=np.float32)[None]
             self.input_metadata["pose_runtime"] = {"device": "CPU pinned by ORT", "dtype": str(pose_input.dtype), "shape": list(pose_input.shape)}
-            def pose_run():
+            outputs, h2d, elapsed, d2h = _ort_iobinding_run(self.pose.session, pose_input)
-                return self.pose.session.run(None, {self.pose.session.get_inputs()[0].name: pose_input})
+            timing["cpu_to_gpu_ms"] += h2d; timing["gpu_to_cpu_ms"] += d2h; pose_forward += elapsed
-            outputs, elapsed = _cuda_timed(pose_run); pose_forward += elapsed
             t = time.perf_counter_ns(); points, scores = self.pose.postprocess(outputs, center, scale)
             pose_decode += _elapsed_ns(t)
             combined = np.column_stack((points[0], scores[0]))

servers/unraid/tail2-pose-server/compose.yaml

@@ -183,8 +183,8 @@ services:
       PYTHONPATH: /app
       CUDA_VISIBLE_DEVICES: "1"
       MODEL_BACKEND: rtmlib_body
-      BENCHMARK_PROFILE: rtmlib_body_yolox-m_rtmpose-m_gpu1_threads2
+      BENCHMARK_PROFILE: rtmlib_body_yolox-m_rtmpose-m_gpu1_threads2_iobinding
-      BENCHMARK_OUTPUT: /validation/benchmark-rtmlib-body.json
+      BENCHMARK_OUTPUT: /validation/benchmark-rtmlib-body-iobinding.json
       DETECTION_THRESHOLD: "0.35"
       NMS_IOU_THRESHOLD: "0.60"
       MAX_PEOPLE: "4"