From f27f9e42a8adecdb370c2ef638f0308aafa9cea3 Mon Sep 17 00:00:00 2001
From: Codex <codex@local>
Date: Sat, 15 Aug 2026 11:22:51 -0700
Subject: [PATCH] Separate ORT transfer timing with IO binding

---
 .../tail2-pose-server/app/tail2/models.py     | 24 ++++++++++++++-----
 servers/unraid/tail2-pose-server/compose.yaml |  4 ++--
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/servers/unraid/tail2-pose-server/app/tail2/models.py b/servers/unraid/tail2-pose-server/app/tail2/models.py
index 19adaa1..851d025 100644
--- a/servers/unraid/tail2-pose-server/app/tail2/models.py
+++ b/servers/unraid/tail2-pose-server/app/tail2/models.py
@@ -24,6 +24,20 @@ def _cuda_timed(call):
     return result, _elapsed_ns(started)
 
 
+def _ort_iobinding_run(session, input_array: np.ndarray):
+    """Measure H2D, graph execution, and D2H as separate synchronized stages."""
+    import onnxruntime as ort
+    device_input, h2d_ms = _cuda_timed(lambda: ort.OrtValue.ortvalue_from_numpy(input_array, "cuda", 0))
+    binding = session.io_binding()
+    binding.bind_ortvalue_input(session.get_inputs()[0].name, device_input)
+    for output in session.get_outputs():
+        binding.bind_output(output.name, "cuda", 0)
+    _, forward_ms = _cuda_timed(lambda: session.run_with_iobinding(binding))
+    device_outputs = binding.get_outputs()
+    outputs, d2h_ms = _cuda_timed(lambda: [value.numpy() for value in device_outputs])
+    return outputs, h2d_ms, forward_ms, d2h_ms
+
+
 class Models:
     """Persistent model holder for both baseline and RTMLib execution paths."""
 
@@ -167,9 +181,8 @@ class Models:
         timing["resize_letterbox_ms"] = _elapsed_ns(t)
         det_input = np.ascontiguousarray(det_image.transpose(2, 0, 1), dtype=np.float32)[None]
         self.input_metadata["detector_runtime"] = {"device": "CPU pinned by ORT", "dtype": str(det_input.dtype), "shape": list(det_input.shape)}
-        def det_run():
-            return self.detector.session.run(None, {self.detector.session.get_inputs()[0].name: det_input})[0]
-        det_output, timing["detector_forward_ms"] = _cuda_timed(det_run)
+        det_outputs, timing["cpu_to_gpu_ms"], timing["detector_forward_ms"], timing["gpu_to_cpu_ms"] = _ort_iobinding_run(self.detector.session, det_input)
+        det_output = det_outputs[0]
         t = time.perf_counter_ns(); boxes, det_scores = self._yolox_postprocess(det_output, ratio, threshold)
         boxes, det_scores = boxes[:max_people], det_scores[:max_people]
         timing["detector_postprocess_ms"] = _elapsed_ns(t)
@@ -180,9 +193,8 @@ class Models:
             crop_affine += _elapsed_ns(t)
             pose_input = np.ascontiguousarray(pose_image.transpose(2, 0, 1), dtype=np.float32)[None]
             self.input_metadata["pose_runtime"] = {"device": "CPU pinned by ORT", "dtype": str(pose_input.dtype), "shape": list(pose_input.shape)}
-            def pose_run():
-                return self.pose.session.run(None, {self.pose.session.get_inputs()[0].name: pose_input})
-            outputs, elapsed = _cuda_timed(pose_run); pose_forward += elapsed
+            outputs, h2d, elapsed, d2h = _ort_iobinding_run(self.pose.session, pose_input)
+            timing["cpu_to_gpu_ms"] += h2d; timing["gpu_to_cpu_ms"] += d2h; pose_forward += elapsed
             t = time.perf_counter_ns(); points, scores = self.pose.postprocess(outputs, center, scale)
             pose_decode += _elapsed_ns(t)
             combined = np.column_stack((points[0], scores[0]))
diff --git a/servers/unraid/tail2-pose-server/compose.yaml b/servers/unraid/tail2-pose-server/compose.yaml
index 2a5544c..d714d05 100644
--- a/servers/unraid/tail2-pose-server/compose.yaml
+++ b/servers/unraid/tail2-pose-server/compose.yaml
@@ -183,8 +183,8 @@ services:
       PYTHONPATH: /app
       CUDA_VISIBLE_DEVICES: "1"
       MODEL_BACKEND: rtmlib_body
-      BENCHMARK_PROFILE: rtmlib_body_yolox-m_rtmpose-m_gpu1_threads2
-      BENCHMARK_OUTPUT: /validation/benchmark-rtmlib-body.json
+      BENCHMARK_PROFILE: rtmlib_body_yolox-m_rtmpose-m_gpu1_threads2_iobinding
+      BENCHMARK_OUTPUT: /validation/benchmark-rtmlib-body-iobinding.json
       DETECTION_THRESHOLD: "0.35"
       NMS_IOU_THRESHOLD: "0.60"
       MAX_PEOPLE: "4"