Add one-shot GPU contention audit

This commit is contained in:
Codex
2026-08-15 10:52:08 -07:00
parent 7d4cf43a9c
commit dba60667cb
@@ -108,6 +108,41 @@ services:
retries: 6
start_period: 180s
gpu-audit:
image: tail2-pose-server:0.1.0
container_name: tail2-pose-gpu-audit
gpus: all
restart: "no"
volumes:
- /:/host:ro
- ./validation:/validation
entrypoint: ["/bin/bash", "-lc"]
command:
- |
set -euo pipefail
out=/validation/gpu-audit
mkdir -p "$$out"
nvidia-smi --query-gpu=index,uuid,pstate,utilization.gpu,utilization.memory,memory.used,memory.total,power.draw,clocks.current.sm,temperature.gpu --format=csv,noheader,nounits > "$$out/snapshot.csv"
nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_gpu_memory --format=csv,noheader,nounits > "$$out/processes.csv"
{
echo 'timestamp_utc,index,uuid,utilization_gpu_pct,utilization_memory_pct,memory_used_mib,pstate,power_draw_w,clocks_sm_mhz,temperature_c'
for i in $$(seq 1 60); do
ts=$$(date -u +%FT%T.%3NZ)
nvidia-smi --query-gpu=index,uuid,utilization.gpu,utilization.memory,memory.used,pstate,power.draw,clocks.current.sm,temperature.gpu --format=csv,noheader,nounits | sed "s/^/$$ts,/"
sleep 1
done
} > "$$out/utilization-60s.csv"
{
echo 'pid,comm,cgroup'
tail -n +1 "$$out/processes.csv" | while IFS=, read -r uuid pid name mem; do
pid=$$(echo "$$pid" | tr -d ' ')
comm=$$(cat "/host/proc/$$pid/comm" 2>/dev/null || echo unavailable)
cgroup=$$(tr '\n' ';' < "/host/proc/$$pid/cgroup" 2>/dev/null || true)
printf '%s,%s,%s\n' "$$pid" "$$comm" "$$cgroup"
done
} > "$$out/process-map.csv"
chmod -R a+rX,u+w "$$out"
validate:
profiles: ["validation"]
image: tail2-pose-server:0.1.0