diff --git a/servers/unraid/tail2-pose-server/app/tail2/main.py b/servers/unraid/tail2-pose-server/app/tail2/main.py index b463490..73b9174 100644 --- a/servers/unraid/tail2-pose-server/app/tail2/main.py +++ b/servers/unraid/tail2-pose-server/app/tail2/main.py @@ -21,6 +21,7 @@ from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo, from .core import BroadcastHub, LatestQueue from .models import Models from .ndi import NDIReceiver +from .offline_benchmark import run_offline_benchmark logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s %(name)s %(message)s") LOG = logging.getLogger("tail2") @@ -145,7 +146,9 @@ async def poses(ws: WebSocket): def inference_worker(loop: asyncio.AbstractEventLoop) -> None: threshold = float(os.getenv("DETECTION_THRESHOLD", ".35")); max_people = int(os.getenv("MAX_PEOPLE", "4")) try: - models.load(); state.models_ready = True; LOG.info("official detector and pose checkpoints loaded") + models.load() + run_offline_benchmark(models, threshold, max_people) + state.models_ready = True; LOG.info("official detector and pose checkpoints loaded") except Exception as exc: state.inference_fault = True; state.error_code = "MODEL_LOAD_FAILED" state.error_detail = f"{type(exc).__name__}: {str(exc)[:240]}"; LOG.exception("model loading failed"); return diff --git a/servers/unraid/tail2-pose-server/app/tail2/models.py b/servers/unraid/tail2-pose-server/app/tail2/models.py index 8a2dbc9..3ead6c3 100644 --- a/servers/unraid/tail2-pose-server/app/tail2/models.py +++ b/servers/unraid/tail2-pose-server/app/tail2/models.py @@ -30,6 +30,7 @@ class Models: from mmdet.apis import init_detector from mmpose.apis import init_model self.detector = init_detector(self.det_config, self.det_checkpoint, device="cuda:0") + self.detector.test_cfg.nms.iou_threshold = float(os.getenv("NMS_IOU_THRESHOLD", ".60")) self.pose = init_model(self.pose_config, self.pose_checkpoint, device="cuda:0") @torch.inference_mode() diff --git a/servers/unraid/tail2-pose-server/app/tail2/offline_benchmark.py b/servers/unraid/tail2-pose-server/app/tail2/offline_benchmark.py new file mode 100644 index 0000000..c48b65f --- /dev/null +++ b/servers/unraid/tail2-pose-server/app/tail2/offline_benchmark.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import json +import os +import statistics +import time +import urllib.request + +import cv2 +import numpy as np + +DEMO_URL = "https://raw.githubusercontent.com/open-mmlab/mmpose/v1.3.2/tests/data/coco/000000000785.jpg" + + +def percentile(values: list[float], p: float) -> float: + values = sorted(values) + return values[min(len(values) - 1, max(0, int(np.ceil(len(values) * p)) - 1))] + + +def run_offline_benchmark(models, threshold: float, max_people: int) -> None: + path = "/logs/offline-benchmark.json" + if os.path.exists(path): + return + try: + encoded = np.frombuffer(urllib.request.urlopen(DEMO_URL, timeout=30).read(), dtype=np.uint8) + one = cv2.imdecode(encoded, cv2.IMREAD_COLOR) + if one is None: + raise RuntimeError("official demo image decode failed") + images = {1: one, 2: np.concatenate([one, one], axis=1), + 4: np.concatenate([np.concatenate([one, one], axis=1)] * 2, axis=0)} + for _ in range(50): + models.infer(one, threshold, max_people) + report = {"source_url": DEMO_URL, "warmup_iterations": 50, "measured_iterations": 60, + "image_retained": False, "profiles": {}} + for requested, image in images.items(): + samples = []; counts = [] + for _ in range(60): + started = time.perf_counter_ns() + people, det_ms, pose_ms = models.infer(image, threshold, max_people) + samples.append((time.perf_counter_ns() - started) / 1e6); counts.append(len(people)) + report["profiles"][str(requested)] = { + "requested_people": requested, "detected_people_min": min(counts), "detected_people_max": max(counts), + "p50_ms": round(percentile(samples, .50), 3), "p95_ms": round(percentile(samples, .95), 3), + "p99_ms": round(percentile(samples, .99), 3), "fps_from_p50": round(1000 / percentile(samples, .50), 3)} + with open(path, "x", encoding="utf-8") as fh: + json.dump(report, fh, indent=2) + except Exception as exc: + with open(path, "w", encoding="utf-8") as fh: + json.dump({"error": f"{type(exc).__name__}: {str(exc)[:240]}", "source_url": DEMO_URL, + "image_retained": False}, fh, indent=2) + diff --git a/servers/unraid/tail2-pose-server/app/tail2/soak.py b/servers/unraid/tail2-pose-server/app/tail2/soak.py new file mode 100644 index 0000000..de7ef36 --- /dev/null +++ b/servers/unraid/tail2-pose-server/app/tail2/soak.py @@ -0,0 +1,41 @@ +import json +import pathlib +import statistics +import time +import urllib.request + +from websockets.sync.client import connect + +token = pathlib.Path("/run/secrets/api-token").read_text().strip() +headers = {"Authorization": "Bearer " + token} +started = time.time(); deadline = started + 600 +frame_ids = []; latencies = []; failures = []; sessions = set(); health_samples = [] +while time.time() < deadline: + try: + with connect("ws://127.0.0.1:18120/v1/poses", additional_headers=headers, open_timeout=10) as ws: + while time.time() < deadline: + message = json.loads(ws.recv(timeout=10)) + frame_ids.append(message["frame_id"]); sessions.add(message["session_id"]) + latencies.append(message["timing"]["source_to_sent_ms"]) + if len(frame_ids) % 100 == 0: + req = urllib.request.Request("http://127.0.0.1:18120/v1/health", headers=headers) + health_samples.append(json.load(urllib.request.urlopen(req, timeout=5))) + except Exception as exc: + failures.append(f"{type(exc).__name__}: {str(exc)[:120]}"); time.sleep(1) + + +def pct(values, p): + values = sorted(values); return values[min(len(values)-1, max(0, int(len(values)*p)-1))] if values else None + + +report = {"duration_seconds": round(time.time()-started, 3), "messages": len(frame_ids), + "first_frame_id": frame_ids[0] if frame_ids else None, "last_frame_id": frame_ids[-1] if frame_ids else None, + "strictly_increasing": all(b > a for a, b in zip(frame_ids, frame_ids[1:])), + "session_ids": sorted(sessions), "ws_failures": failures, + "source_to_sent_ms": {"p50": pct(latencies,.50), "p95": pct(latencies,.95), "p99": pct(latencies,.99)}, + "health_samples": len(health_samples), "all_ready": all(x["ready"] for x in health_samples), + "max_gpu_memory_used_mb": max((x["gpu"]["memory_used_mb"] for x in health_samples), default=None), + "max_gpu_temperature_c": max((x["gpu"]["temperature_c"] for x in health_samples), default=None), + "frames_or_crops_saved": False} +pathlib.Path("/validation/soak-10m.json").write_text(json.dumps(report, indent=2)) + diff --git a/servers/unraid/tail2-pose-server/compose.yaml b/servers/unraid/tail2-pose-server/compose.yaml index f5d4ae5..40cba63 100644 --- a/servers/unraid/tail2-pose-server/compose.yaml +++ b/servers/unraid/tail2-pose-server/compose.yaml @@ -96,7 +96,7 @@ services: POSE_CONFIG: /models/mmpose/configs/wholebody_2d_keypoint/rtmpose/cocktail14/rtmw-l_8xb1024-270e_cocktail14-256x192.py POSE_CHECKPOINT: /models/rtmw-dw-x-l_simcc-cocktail14_270e-256x192-20231122.pth healthcheck: - test: ["CMD", "python", "-c", "import pathlib,urllib.request; t=pathlib.Path('/run/secrets/api-token').read_text().strip(); r=urllib.request.Request('http://127.0.0.1:18120/v1/health',headers={'Authorization':'Bearer '+t}); raise SystemExit(0 if urllib.request.urlopen(r,timeout=3).status==200 else 1)"] + test: ["CMD", "python", "-c", "import json,pathlib,urllib.request; t=pathlib.Path('/run/secrets/api-token').read_text().strip(); r=urllib.request.Request('http://127.0.0.1:18120/v1/health',headers={'Authorization':'Bearer '+t}); raise SystemExit(0 if json.load(urllib.request.urlopen(r,timeout=3))['ready'] else 1)"] interval: 15s timeout: 5s retries: 6 @@ -128,3 +128,23 @@ services: nvidia-smi --query-gpu=name,driver_version,memory.total,memory.used,temperature.gpu --format=csv,noheader > /validation/gpu.csv sha256sum /models/*.pth > /validation/model-sha256.txt chmod 0666 /validation/*.txt /validation/*.json /validation/*.csv + + soak: + image: tail2-pose-server:0.1.0 + container_name: tail2-pose-soak + network_mode: host + restart: "no" + entrypoint: ["/bin/bash", "-lc"] + depends_on: + tail2-pose-server: + condition: service_healthy + volumes: + - /mnt/user/appdata/tail2-pose-server/app:/app:ro + - /mnt/user/appdata/tail2-pose-server/secrets/api-token:/run/secrets/api-token:ro + - ./validation:/validation + command: + - | + set -euo pipefail + chmod -R a+rwX /validation + test -s /validation/soak-10m.json || python /app/tail2/soak.py + chmod 0666 /validation/soak-10m.json