diff --git a/servers/unraid/tail2-pose-server/compose.yaml b/servers/unraid/tail2-pose-server/compose.yaml index 7d0c49a..38f780a 100644 --- a/servers/unraid/tail2-pose-server/compose.yaml +++ b/servers/unraid/tail2-pose-server/compose.yaml @@ -108,6 +108,41 @@ services: retries: 6 start_period: 180s + gpu-audit: + image: tail2-pose-server:0.1.0 + container_name: tail2-pose-gpu-audit + gpus: all + restart: "no" + volumes: + - /:/host:ro + - ./validation:/validation + entrypoint: ["/bin/bash", "-lc"] + command: + - | + set -euo pipefail + out=/validation/gpu-audit + mkdir -p "$$out" + nvidia-smi --query-gpu=index,uuid,pstate,utilization.gpu,utilization.memory,memory.used,memory.total,power.draw,clocks.current.sm,temperature.gpu --format=csv,noheader,nounits > "$$out/snapshot.csv" + nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_gpu_memory --format=csv,noheader,nounits > "$$out/processes.csv" + { + echo 'timestamp_utc,index,uuid,utilization_gpu_pct,utilization_memory_pct,memory_used_mib,pstate,power_draw_w,clocks_sm_mhz,temperature_c' + for i in $$(seq 1 60); do + ts=$$(date -u +%FT%T.%3NZ) + nvidia-smi --query-gpu=index,uuid,utilization.gpu,utilization.memory,memory.used,pstate,power.draw,clocks.current.sm,temperature.gpu --format=csv,noheader,nounits | sed "s/^/$$ts,/" + sleep 1 + done + } > "$$out/utilization-60s.csv" + { + echo 'pid,comm,cgroup' + tail -n +1 "$$out/processes.csv" | while IFS=, read -r uuid pid name mem; do + pid=$$(echo "$$pid" | tr -d ' ') + comm=$$(cat "/host/proc/$$pid/comm" 2>/dev/null || echo unavailable) + cgroup=$$(tr '\n' ';' < "/host/proc/$$pid/cgroup" 2>/dev/null || true) + printf '%s,%s,%s\n' "$$pid" "$$comm" "$$cgroup" + done + } > "$$out/process-map.csv" + chmod -R a+rX,u+w "$$out" + validate: profiles: ["validation"] image: tail2-pose-server:0.1.0