#!/bin/sh set -eu data_dir=/var/lib/supermemory sentinel="$data_dir/.stability-gemini-paid-v1.ok" base_url="${SUPERMEMORY_URL:-http://127.0.0.1:6767}" container_tag="${SUPERMEMORY_CONTAINER_TAG:-supermemory_stability_paid_v1}" if [ -f "$sentinel" ]; then printf '%s\n' '{"stability":"already-passed","suite":"gemini-paid-v1"}' exit 0 fi api_key=$(cat "$data_dir/api-key") auth_header="Authorization: Bearer $api_key" run_id="paid-v1-$(date +%s)" submitted=0 add_failures=0 processing_failures=0 documents_failures=0 hybrid_failures=0 metadata_failures=0 case_values() { case "$1" in 1) content="海棠计划的下一步是核对原始报告并记录出处。"; query="海棠计划下一步是什么?"; expected="核对原始报告" ;; 2) content="Project Juniper rollback owner is Mira Chen."; query="Who owns the Juniper rollback?"; expected="Mira Chen" ;; 3) content="Orion 项目的 maintenance window is Tuesday 03:00 UTC."; query="Orion 的维护窗口是什么时候?"; expected="Tuesday" ;; 4) content="The synthetic 2026 Q4 budget cap is 42000 credits."; query="What is the synthetic Q4 budget cap?"; expected="42000" ;; 5) content="仅在本基准中,状态报告应使用 amber labels。"; query="本基准的状态报告应该用什么标签?"; expected="amber" ;; esac } for case_id in 1 2 3 4 5; do case_values "$case_id" marker="supermemory-stability-$run_id-$case_id" if curl --fail --silent --show-error --max-time 60 \ -H "$auth_header" \ -H 'Content-Type: application/json' \ --data "{\"content\":\"$marker. $content\",\"containerTag\":\"$container_tag\",\"customId\":\"$marker\",\"metadata\":{\"source\":\"poc-stability\",\"suite\":\"gemini-paid-v1\",\"case_id\":\"$case_id\",\"status\":\"temporary\"}}" \ "$base_url/v3/documents" >"/tmp/add-$case_id.json"; then document_id=$(sed -n 's/.*"id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "/tmp/add-$case_id.json" | head -n 1) if [ -n "$document_id" ]; then printf '%s\n' "$document_id" >"/tmp/id-$case_id" submitted=$((submitted + 1)) else add_failures=$((add_failures + 1)) fi else add_failures=$((add_failures + 1)) fi done for _ in $(seq 1 300); do terminal=0 for case_id in 1 2 3 4 5; do [ -f "/tmp/id-$case_id" ] || continue if [ -f "/tmp/done-$case_id" ] || [ -f "/tmp/failed-$case_id" ]; then terminal=$((terminal + 1)) continue fi document_id=$(cat "/tmp/id-$case_id") if curl --fail --silent --show-error --max-time 30 \ -H "$auth_header" \ "$base_url/v3/documents/$document_id" >"/tmp/document-$case_id.json"; then if grep -Eq '"status"[[:space:]]*:[[:space:]]*"done"' "/tmp/document-$case_id.json"; then : >"/tmp/done-$case_id" terminal=$((terminal + 1)) elif grep -Eq '"status"[[:space:]]*:[[:space:]]*"failed"' "/tmp/document-$case_id.json"; then : >"/tmp/failed-$case_id" terminal=$((terminal + 1)) processing_failures=$((processing_failures + 1)) fi fi done [ "$terminal" -eq "$submitted" ] && break sleep 2 done for case_id in 1 2 3 4 5; do [ -f "/tmp/id-$case_id" ] || continue if [ ! -f "/tmp/done-$case_id" ]; then if [ ! -f "/tmp/failed-$case_id" ]; then processing_failures=$((processing_failures + 1)) fi continue fi case_values "$case_id" marker="supermemory-stability-$run_id-$case_id" if curl --fail --silent --show-error --max-time 30 \ -H "$auth_header" \ -H 'Content-Type: application/json' \ --data "{\"q\":\"$marker\",\"containerTag\":\"$container_tag\",\"searchMode\":\"documents\",\"limit\":5}" \ "$base_url/v4/search" >"/tmp/documents-$case_id.json"; then if ! grep -q "$marker" "/tmp/documents-$case_id.json"; then documents_failures=$((documents_failures + 1)) fi else documents_failures=$((documents_failures + 1)) fi if [ -f "/tmp/documents-$case_id.json" ] && ! grep -q 'poc-stability' "/tmp/documents-$case_id.json"; then metadata_failures=$((metadata_failures + 1)) fi if curl --fail --silent --show-error --max-time 30 \ -H "$auth_header" \ -H 'Content-Type: application/json' \ --data "{\"q\":\"$query\",\"containerTag\":\"$container_tag\",\"searchMode\":\"hybrid\",\"limit\":5}" \ "$base_url/v4/search" >"/tmp/hybrid-$case_id.json"; then if ! grep -q "$expected" "/tmp/hybrid-$case_id.json"; then hybrid_failures=$((hybrid_failures + 1)) fi else hybrid_failures=$((hybrid_failures + 1)) fi done conversation_http=$(curl --silent --show-error --max-time 60 \ --output /tmp/stability-conversation.json \ --write-out '%{http_code}' \ -H "$auth_header" \ -H 'Content-Type: application/json' \ --data "{\"conversationId\":\"supermemory-stability-$run_id-conversation\",\"containerTags\":[\"$container_tag\"],\"messages\":[{\"role\":\"user\",\"content\":\"Synthetic session question.\"},{\"role\":\"assistant\",\"content\":\"Synthetic session answer.\"}],\"metadata\":{\"source\":\"poc-stability\",\"status\":\"temporary\"}}" \ "$base_url/v4/conversations") || conversation_http=transport-error printf '{"stability":"result","suite":"gemini-paid-v1","attempted":5,"submitted":%s,"add_failures":%s,"processing_failures":%s,"documents_failures":%s,"hybrid_failures":%s,"metadata_failures":%s,"conversation_http":"%s"}\n' \ "$submitted" "$add_failures" "$processing_failures" "$documents_failures" "$hybrid_failures" "$metadata_failures" "$conversation_http" if [ "$submitted" -eq 5 ] && [ "$add_failures" -eq 0 ] && [ "$processing_failures" -eq 0 ] && \ [ "$documents_failures" -eq 0 ] && [ "$hybrid_failures" -eq 0 ] && [ "$metadata_failures" -eq 0 ]; then touch "$sentinel" exit 0 fi exit 70