infra-docker-configs/servers/unraid/supermemory-poc/stability.sh

#!/bin/sh
set -eu

data_dir=/var/lib/supermemory
sentinel="$data_dir/.stability-gemini-paid-v1.ok"
base_url="${SUPERMEMORY_URL:-http://127.0.0.1:6767}"
container_tag="${SUPERMEMORY_CONTAINER_TAG:-supermemory_stability_paid_v1}"

if [ -f "$sentinel" ]; then
  printf '%s\n' '{"stability":"already-passed","suite":"gemini-paid-v1"}'
  exit 0
fi

api_key=$(cat "$data_dir/api-key")
auth_header="Authorization: Bearer $api_key"
run_id="paid-v1-$(date +%s)"
submitted=0
add_failures=0
processing_failures=0
documents_failures=0
hybrid_failures=0
metadata_failures=0

case_values() {
  case "$1" in
    1) content="海棠计划的下一步是核对原始报告并记录出处。"; query="海棠计划下一步是什么？"; expected="核对原始报告" ;;
    2) content="Project Juniper rollback owner is Mira Chen."; query="Who owns the Juniper rollback?"; expected="Mira Chen" ;;
    3) content="Orion 项目的 maintenance window is Tuesday 03:00 UTC."; query="Orion 的维护窗口是什么时候？"; expected="Tuesday" ;;
    4) content="The synthetic 2026 Q4 budget cap is 42000 credits."; query="What is the synthetic Q4 budget cap?"; expected="42000" ;;
    5) content="仅在本基准中，状态报告应使用 amber labels。"; query="本基准的状态报告应该用什么标签？"; expected="amber" ;;
  esac
}

for case_id in 1 2 3 4 5; do
  case_values "$case_id"
  marker="supermemory-stability-$run_id-$case_id"
  if curl --fail --silent --show-error --max-time 60 \
    -H "$auth_header" \
    -H 'Content-Type: application/json' \
    --data "{\"content\":\"$marker. $content\",\"containerTag\":\"$container_tag\",\"customId\":\"$marker\",\"metadata\":{\"source\":\"poc-stability\",\"suite\":\"gemini-paid-v1\",\"case_id\":\"$case_id\",\"status\":\"temporary\"}}" \
    "$base_url/v3/documents" >"/tmp/add-$case_id.json"; then
    document_id=$(sed -n 's/.*"id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "/tmp/add-$case_id.json" | head -n 1)
    if [ -n "$document_id" ]; then
      printf '%s\n' "$document_id" >"/tmp/id-$case_id"
      submitted=$((submitted + 1))
    else
      add_failures=$((add_failures + 1))
    fi
  else
    add_failures=$((add_failures + 1))
  fi
done

for _ in $(seq 1 300); do
  terminal=0
  for case_id in 1 2 3 4 5; do
    [ -f "/tmp/id-$case_id" ] || continue
    if [ -f "/tmp/done-$case_id" ] || [ -f "/tmp/failed-$case_id" ]; then
      terminal=$((terminal + 1))
      continue
    fi
    document_id=$(cat "/tmp/id-$case_id")
    if curl --fail --silent --show-error --max-time 30 \
      -H "$auth_header" \
      "$base_url/v3/documents/$document_id" >"/tmp/document-$case_id.json"; then
      if grep -Eq '"status"[[:space:]]*:[[:space:]]*"done"' "/tmp/document-$case_id.json"; then
        : >"/tmp/done-$case_id"
        terminal=$((terminal + 1))
      elif grep -Eq '"status"[[:space:]]*:[[:space:]]*"failed"' "/tmp/document-$case_id.json"; then
        : >"/tmp/failed-$case_id"
        terminal=$((terminal + 1))
        processing_failures=$((processing_failures + 1))
      fi
    fi
  done
  [ "$terminal" -eq "$submitted" ] && break
  sleep 2
done

for case_id in 1 2 3 4 5; do
  [ -f "/tmp/id-$case_id" ] || continue
  if [ ! -f "/tmp/done-$case_id" ]; then
    if [ ! -f "/tmp/failed-$case_id" ]; then
      processing_failures=$((processing_failures + 1))
    fi
    continue
  fi

  case_values "$case_id"
  marker="supermemory-stability-$run_id-$case_id"
  if curl --fail --silent --show-error --max-time 30 \
    -H "$auth_header" \
    -H 'Content-Type: application/json' \
    --data "{\"q\":\"$marker\",\"containerTag\":\"$container_tag\",\"searchMode\":\"documents\",\"limit\":5}" \
    "$base_url/v4/search" >"/tmp/documents-$case_id.json"; then
    if ! grep -q "$marker" "/tmp/documents-$case_id.json"; then
      documents_failures=$((documents_failures + 1))
    fi
  else
    documents_failures=$((documents_failures + 1))
  fi
  if [ -f "/tmp/documents-$case_id.json" ] && ! grep -q 'poc-stability' "/tmp/documents-$case_id.json"; then
    metadata_failures=$((metadata_failures + 1))
  fi

  if curl --fail --silent --show-error --max-time 30 \
    -H "$auth_header" \
    -H 'Content-Type: application/json' \
    --data "{\"q\":\"$query\",\"containerTag\":\"$container_tag\",\"searchMode\":\"hybrid\",\"limit\":5}" \
    "$base_url/v4/search" >"/tmp/hybrid-$case_id.json"; then
    if ! grep -q "$expected" "/tmp/hybrid-$case_id.json"; then
      hybrid_failures=$((hybrid_failures + 1))
    fi
  else
    hybrid_failures=$((hybrid_failures + 1))
  fi
done

conversation_http=$(curl --silent --show-error --max-time 60 \
  --output /tmp/stability-conversation.json \
  --write-out '%{http_code}' \
  -H "$auth_header" \
  -H 'Content-Type: application/json' \
  --data "{\"conversationId\":\"supermemory-stability-$run_id-conversation\",\"containerTags\":[\"$container_tag\"],\"messages\":[{\"role\":\"user\",\"content\":\"Synthetic session question.\"},{\"role\":\"assistant\",\"content\":\"Synthetic session answer.\"}],\"metadata\":{\"source\":\"poc-stability\",\"status\":\"temporary\"}}" \
  "$base_url/v4/conversations") || conversation_http=transport-error

printf '{"stability":"result","suite":"gemini-paid-v1","attempted":5,"submitted":%s,"add_failures":%s,"processing_failures":%s,"documents_failures":%s,"hybrid_failures":%s,"metadata_failures":%s,"conversation_http":"%s"}\n' \
  "$submitted" "$add_failures" "$processing_failures" "$documents_failures" "$hybrid_failures" "$metadata_failures" "$conversation_http"

if [ "$submitted" -eq 5 ] && [ "$add_failures" -eq 0 ] && [ "$processing_failures" -eq 0 ] && \
   [ "$documents_failures" -eq 0 ] && [ "$hybrid_failures" -eq 0 ] && [ "$metadata_failures" -eq 0 ]; then
  touch "$sentinel"
  exit 0
fi
exit 70