diff --git a/servers/unraid/supermemory-poc/README.md b/servers/unraid/supermemory-poc/README.md index af9a534..48b44fc 100644 --- a/servers/unraid/supermemory-poc/README.md +++ b/servers/unraid/supermemory-poc/README.md @@ -91,9 +91,17 @@ diagnostic exposed the original key in its startup log. The one-shot `supermemor service verifies the locked embedding plan, tracks the exact document ID to a terminal state, observes legacy v3 search, gates on v4 document and Chinese hybrid search, checks provenance metadata, and exercises the profile and -conversation endpoints. It writes a sentinel after success so ordinary -redeploys do not add duplicate smoke documents. Do not import sampled -Hindsight or document material until this service exits successfully. +conversation endpoints. A `/v4/conversations` 404 is reported as a Hermes +compatibility failure but does not invalidate the document-RAG checks. It +writes a sentinel after the document path succeeds so ordinary redeploys do +not add duplicate smoke documents. + +The dependent `supermemory-stability` service uses five isolated synthetic +documents instead of Hindsight data. It exercises Chinese, English, +mixed-language, numeric, and preference-like content; checks processing, +exact document retrieval, hybrid retrieval, and provenance metadata; and +reports conversation compatibility separately. The suite passes only when +all five document cases pass and then writes its own sentinel. On 2026-09-08, the previous remote plan was accepted and locked as `openai / google/gemini-embedding-2 / 1024`; the LiteLLM embedding and chat @@ -108,9 +116,9 @@ The v4 `documents` and `hybrid` searches returned the expected Chinese marker and provenance metadata, while legacy `/v3/search` returned no results. The self-hosted server-v0.0.8 binary returned 404 for `/v4/conversations`, even though the self-hosting overview describes that route as part of the full API. -Keep conversation ingest as a failing smoke gate: document RAG can proceed in -isolation, but Hermes auto-capture/session ingest is not validated on this -binary. +Treat conversation ingest as a separate Hermes compatibility gate: document +RAG can proceed in isolation, but Hermes auto-capture/session ingest is not +validated on this binary. ## Evaluation guardrails diff --git a/servers/unraid/supermemory-poc/compose.yaml b/servers/unraid/supermemory-poc/compose.yaml index 91e321e..88ac648 100644 --- a/servers/unraid/supermemory-poc/compose.yaml +++ b/servers/unraid/supermemory-poc/compose.yaml @@ -123,6 +123,34 @@ services: max-size: 1m max-file: "2" + supermemory-stability: + image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6 + container_name: supermemory-poc-stability + user: "0:0" + restart: "no" + depends_on: + supermemory-smoke: + condition: service_completed_successfully + network_mode: service:supermemory + environment: + SUPERMEMORY_URL: http://127.0.0.1:6767 + SUPERMEMORY_CONTAINER_TAG: supermemory_stability_paid_v1 + entrypoint: + - /bin/sh + - /opt/supermemory-poc/stability.sh + volumes: + - ./stability.sh:/opt/supermemory-poc/stability.sh:ro + - /mnt/user/appdata/supermemory-poc/data-gemini-embedding-2-768-r2:/var/lib/supermemory + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + logging: + driver: json-file + options: + max-size: 1m + max-file: "2" + networks: fetch: internal: false diff --git a/servers/unraid/supermemory-poc/poc.py b/servers/unraid/supermemory-poc/poc.py index 8a0d630..bb8ab5a 100755 --- a/servers/unraid/supermemory-poc/poc.py +++ b/servers/unraid/supermemory-poc/poc.py @@ -317,21 +317,27 @@ def smoke(args: argparse.Namespace) -> None: payload={"containerTag": container, "q": marker}, timeout=args.timeout, ) - request_json( - "POST", - f"{base}/v4/conversations", - token=token, - payload={ - "conversationId": marker, - "containerTags": [container], - "messages": [ - {"role": "user", "content": "记住本轮只是隔离 POC 烟测。"}, - {"role": "assistant", "content": "已记录为临时测试,不作为正式事实。"}, - ], - "metadata": {"source": "poc-smoke", "status": "temporary"}, - }, - timeout=args.timeout, - ) + conversation_ingest = "accepted" + try: + request_json( + "POST", + f"{base}/v4/conversations", + token=token, + payload={ + "conversationId": marker, + "containerTags": [container], + "messages": [ + {"role": "user", "content": "记住本轮只是隔离 POC 烟测。"}, + {"role": "assistant", "content": "已记录为临时测试,不作为正式事实。"}, + ], + "metadata": {"source": "poc-smoke", "status": "temporary"}, + }, + timeout=args.timeout, + ) + except RuntimeError as exc: + if "HTTP 404:" not in str(exc): + raise + conversation_ingest = "unsupported-404" if not found: raise RuntimeError(f"document {document_id or marker} was not searchable within {args.wait}s") raw_metadata_traceable = contains_source(search.get("results") or [], "poc-smoke") @@ -347,7 +353,7 @@ def smoke(args: argparse.Namespace) -> None: "documents_search": "ok", "raw_metadata_traceability": "ok", "profile": "ok", - "conversation_ingest": "accepted", + "conversation_ingest": conversation_ingest, "container_tag": container, } ) diff --git a/servers/unraid/supermemory-poc/smoke.sh b/servers/unraid/supermemory-poc/smoke.sh index 6cb510a..c14b17b 100644 --- a/servers/unraid/supermemory-poc/smoke.sh +++ b/servers/unraid/supermemory-poc/smoke.sh @@ -114,11 +114,20 @@ curl --fail --silent --show-error --max-time 60 \ "$base_url/v4/profile" >/tmp/profile.json || fail 51 profile-request grep -q '"profile"' /tmp/profile.json || fail 52 profile-shape -curl --fail --silent --show-error --max-time 60 \ +conversation_http=$(curl --silent --show-error --max-time 60 \ + --output /tmp/conversation.json \ + --write-out '%{http_code}' \ -H "$auth_header" \ -H 'Content-Type: application/json' \ --data "{\"conversationId\":\"$marker\",\"containerTags\":[\"$container_tag\"],\"messages\":[{\"role\":\"user\",\"content\":\"这只是隔离测试。\"},{\"role\":\"assistant\",\"content\":\"不会作为正式事实。\"}],\"metadata\":{\"source\":\"poc-smoke\",\"status\":\"temporary\"}}" \ - "$base_url/v4/conversations" >/tmp/conversation.json || fail 61 conversation-ingest + "$base_url/v4/conversations") || fail 61 conversation-ingest-request + +case "$conversation_http" in + 2??) conversation_ingest=accepted ;; + 404) conversation_ingest=unsupported-404 ;; + *) fail 62 conversation-ingest-unexpected-status ;; +esac touch "$sentinel" -printf '{"embedding_plan":"gemini/gemini-embedding-2/768","llm":"gemini-native-default","v3_search":%s,"v4_documents_search":"ok","chinese_hybrid_search":"ok","metadata_traceability":"ok","profile":"ok","conversation_ingest":"accepted"}\n' "$v3_hit" +printf '{"embedding_plan":"gemini/gemini-embedding-2/768","llm":"gemini-native-default","v3_search":%s,"v4_documents_search":"ok","chinese_hybrid_search":"ok","metadata_traceability":"ok","profile":"ok","conversation_ingest":"%s","conversation_http":%s}\n' \ + "$v3_hit" "$conversation_ingest" "$conversation_http" diff --git a/servers/unraid/supermemory-poc/stability.sh b/servers/unraid/supermemory-poc/stability.sh new file mode 100644 index 0000000..ba88a74 --- /dev/null +++ b/servers/unraid/supermemory-poc/stability.sh @@ -0,0 +1,135 @@ +#!/bin/sh +set -eu + +data_dir=/var/lib/supermemory +sentinel="$data_dir/.stability-gemini-paid-v1.ok" +base_url="${SUPERMEMORY_URL:-http://127.0.0.1:6767}" +container_tag="${SUPERMEMORY_CONTAINER_TAG:-supermemory_stability_paid_v1}" + +if [ -f "$sentinel" ]; then + printf '%s\n' '{"stability":"already-passed","suite":"gemini-paid-v1"}' + exit 0 +fi + +api_key=$(cat "$data_dir/api-key") +auth_header="Authorization: Bearer $api_key" +run_id="paid-v1-$(date +%s)" +submitted=0 +add_failures=0 +processing_failures=0 +documents_failures=0 +hybrid_failures=0 +metadata_failures=0 + +case_values() { + case "$1" in + 1) content="海棠计划的下一步是核对原始报告并记录出处。"; query="海棠计划下一步是什么?"; expected="核对原始报告" ;; + 2) content="Project Juniper rollback owner is Mira Chen."; query="Who owns the Juniper rollback?"; expected="Mira Chen" ;; + 3) content="Orion 项目的 maintenance window is Tuesday 03:00 UTC."; query="Orion 的维护窗口是什么时候?"; expected="Tuesday" ;; + 4) content="The synthetic 2026 Q4 budget cap is 42000 credits."; query="What is the synthetic Q4 budget cap?"; expected="42000" ;; + 5) content="仅在本基准中,状态报告应使用 amber labels。"; query="本基准的状态报告应该用什么标签?"; expected="amber" ;; + esac +} + +for case_id in 1 2 3 4 5; do + case_values "$case_id" + marker="supermemory-stability-$run_id-$case_id" + if curl --fail --silent --show-error --max-time 60 \ + -H "$auth_header" \ + -H 'Content-Type: application/json' \ + --data "{\"content\":\"$marker. $content\",\"containerTag\":\"$container_tag\",\"customId\":\"$marker\",\"metadata\":{\"source\":\"poc-stability\",\"suite\":\"gemini-paid-v1\",\"case_id\":\"$case_id\",\"status\":\"temporary\"}}" \ + "$base_url/v3/documents" >"/tmp/add-$case_id.json"; then + document_id=$(sed -n 's/.*"id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "/tmp/add-$case_id.json" | head -n 1) + if [ -n "$document_id" ]; then + printf '%s\n' "$document_id" >"/tmp/id-$case_id" + submitted=$((submitted + 1)) + else + add_failures=$((add_failures + 1)) + fi + else + add_failures=$((add_failures + 1)) + fi +done + +for _ in $(seq 1 300); do + terminal=0 + for case_id in 1 2 3 4 5; do + [ -f "/tmp/id-$case_id" ] || continue + if [ -f "/tmp/done-$case_id" ] || [ -f "/tmp/failed-$case_id" ]; then + terminal=$((terminal + 1)) + continue + fi + document_id=$(cat "/tmp/id-$case_id") + if curl --fail --silent --show-error --max-time 30 \ + -H "$auth_header" \ + "$base_url/v3/documents/$document_id" >"/tmp/document-$case_id.json"; then + if grep -Eq '"status"[[:space:]]*:[[:space:]]*"done"' "/tmp/document-$case_id.json"; then + : >"/tmp/done-$case_id" + terminal=$((terminal + 1)) + elif grep -Eq '"status"[[:space:]]*:[[:space:]]*"failed"' "/tmp/document-$case_id.json"; then + : >"/tmp/failed-$case_id" + terminal=$((terminal + 1)) + processing_failures=$((processing_failures + 1)) + fi + fi + done + [ "$terminal" -eq "$submitted" ] && break + sleep 2 +done + +for case_id in 1 2 3 4 5; do + [ -f "/tmp/id-$case_id" ] || continue + if [ ! -f "/tmp/done-$case_id" ]; then + if [ ! -f "/tmp/failed-$case_id" ]; then + processing_failures=$((processing_failures + 1)) + fi + continue + fi + + case_values "$case_id" + marker="supermemory-stability-$run_id-$case_id" + if curl --fail --silent --show-error --max-time 30 \ + -H "$auth_header" \ + -H 'Content-Type: application/json' \ + --data "{\"q\":\"$marker\",\"containerTag\":\"$container_tag\",\"searchMode\":\"documents\",\"limit\":5}" \ + "$base_url/v4/search" >"/tmp/documents-$case_id.json"; then + if ! grep -q "$marker" "/tmp/documents-$case_id.json"; then + documents_failures=$((documents_failures + 1)) + fi + else + documents_failures=$((documents_failures + 1)) + fi + if [ -f "/tmp/documents-$case_id.json" ] && ! grep -q 'poc-stability' "/tmp/documents-$case_id.json"; then + metadata_failures=$((metadata_failures + 1)) + fi + + if curl --fail --silent --show-error --max-time 30 \ + -H "$auth_header" \ + -H 'Content-Type: application/json' \ + --data "{\"q\":\"$query\",\"containerTag\":\"$container_tag\",\"searchMode\":\"hybrid\",\"limit\":5}" \ + "$base_url/v4/search" >"/tmp/hybrid-$case_id.json"; then + if ! grep -q "$expected" "/tmp/hybrid-$case_id.json"; then + hybrid_failures=$((hybrid_failures + 1)) + fi + else + hybrid_failures=$((hybrid_failures + 1)) + fi +done + +conversation_http=$(curl --silent --show-error --max-time 60 \ + --output /tmp/stability-conversation.json \ + --write-out '%{http_code}' \ + -H "$auth_header" \ + -H 'Content-Type: application/json' \ + --data "{\"conversationId\":\"supermemory-stability-$run_id-conversation\",\"containerTags\":[\"$container_tag\"],\"messages\":[{\"role\":\"user\",\"content\":\"Synthetic session question.\"},{\"role\":\"assistant\",\"content\":\"Synthetic session answer.\"}],\"metadata\":{\"source\":\"poc-stability\",\"status\":\"temporary\"}}" \ + "$base_url/v4/conversations") || conversation_http=transport-error + +printf '{"stability":"result","suite":"gemini-paid-v1","attempted":5,"submitted":%s,"add_failures":%s,"processing_failures":%s,"documents_failures":%s,"hybrid_failures":%s,"metadata_failures":%s,"conversation_http":"%s"}\n' \ + "$submitted" "$add_failures" "$processing_failures" "$documents_failures" "$hybrid_failures" "$metadata_failures" "$conversation_http" + +if [ "$submitted" -eq 5 ] && [ "$add_failures" -eq 0 ] && [ "$processing_failures" -eq 0 ] && \ + [ "$documents_failures" -eq 0 ] && [ "$hybrid_failures" -eq 0 ] && [ "$metadata_failures" -eq 0 ]; then + touch "$sentinel" + exit 0 +fi +exit 70