separate Supermemory RAG stability gate

servers/unraid/supermemory-poc/README.md

@@ -91,9 +91,17 @@ diagnostic exposed the original key in its startup log. The one-shot `supermemor
 service verifies the locked embedding plan, tracks the exact document ID to a
 terminal state, observes legacy v3 search, gates on v4 document and Chinese
 hybrid search, checks provenance metadata, and exercises the profile and
-conversation endpoints. It writes a sentinel after success so ordinary
+conversation endpoints. A `/v4/conversations` 404 is reported as a Hermes
-redeploys do not add duplicate smoke documents. Do not import sampled
+compatibility failure but does not invalidate the document-RAG checks. It
-Hindsight or document material until this service exits successfully.
+writes a sentinel after the document path succeeds so ordinary redeploys do
+not add duplicate smoke documents.
+
+The dependent `supermemory-stability` service uses five isolated synthetic
+documents instead of Hindsight data. It exercises Chinese, English,
+mixed-language, numeric, and preference-like content; checks processing,
+exact document retrieval, hybrid retrieval, and provenance metadata; and
+reports conversation compatibility separately. The suite passes only when
+all five document cases pass and then writes its own sentinel.
 
 On 2026-09-08, the previous remote plan was accepted and locked as
 `openai / google/gemini-embedding-2 / 1024`; the LiteLLM embedding and chat
@@ -108,9 +116,9 @@ The v4 `documents` and `hybrid` searches returned the expected Chinese marker
 and provenance metadata, while legacy `/v3/search` returned no results. The
 self-hosted server-v0.0.8 binary returned 404 for `/v4/conversations`, even
 though the self-hosting overview describes that route as part of the full API.
-Keep conversation ingest as a failing smoke gate: document RAG can proceed in
+Treat conversation ingest as a separate Hermes compatibility gate: document
-isolation, but Hermes auto-capture/session ingest is not validated on this
+RAG can proceed in isolation, but Hermes auto-capture/session ingest is not
-binary.
+validated on this binary.
 
 ## Evaluation guardrails
 

servers/unraid/supermemory-poc/compose.yaml

@@ -123,6 +123,34 @@ services:
         max-size: 1m
         max-file: "2"
 
+  supermemory-stability:
+    image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6
+    container_name: supermemory-poc-stability
+    user: "0:0"
+    restart: "no"
+    depends_on:
+      supermemory-smoke:
+        condition: service_completed_successfully
+    network_mode: service:supermemory
+    environment:
+      SUPERMEMORY_URL: http://127.0.0.1:6767
+      SUPERMEMORY_CONTAINER_TAG: supermemory_stability_paid_v1
+    entrypoint:
+      - /bin/sh
+      - /opt/supermemory-poc/stability.sh
+    volumes:
+      - ./stability.sh:/opt/supermemory-poc/stability.sh:ro
+      - /mnt/user/appdata/supermemory-poc/data-gemini-embedding-2-768-r2:/var/lib/supermemory
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    logging:
+      driver: json-file
+      options:
+        max-size: 1m
+        max-file: "2"
+
 networks:
   fetch:
     internal: false

servers/unraid/supermemory-poc/poc.py

@@ -317,21 +317,27 @@ def smoke(args: argparse.Namespace) -> None:
         payload={"containerTag": container, "q": marker},
         timeout=args.timeout,
     )
-    request_json(
+    conversation_ingest = "accepted"
-        "POST",
+    try:
-        f"{base}/v4/conversations",
+        request_json(
-        token=token,
+            "POST",
-        payload={
+            f"{base}/v4/conversations",
-            "conversationId": marker,
+            token=token,
-            "containerTags": [container],
+            payload={
-            "messages": [
+                "conversationId": marker,
-                {"role": "user", "content": "记住本轮只是隔离 POC 烟测。"},
+                "containerTags": [container],
-                {"role": "assistant", "content": "已记录为临时测试，不作为正式事实。"},
+                "messages": [
-            ],
+                    {"role": "user", "content": "记住本轮只是隔离 POC 烟测。"},
-            "metadata": {"source": "poc-smoke", "status": "temporary"},
+                    {"role": "assistant", "content": "已记录为临时测试，不作为正式事实。"},
-        },
+                ],
-        timeout=args.timeout,
+                "metadata": {"source": "poc-smoke", "status": "temporary"},
-    )
+            },
+            timeout=args.timeout,
+        )
+    except RuntimeError as exc:
+        if "HTTP 404:" not in str(exc):
+            raise
+        conversation_ingest = "unsupported-404"
     if not found:
         raise RuntimeError(f"document {document_id or marker} was not searchable within {args.wait}s")
     raw_metadata_traceable = contains_source(search.get("results") or [], "poc-smoke")
@@ -347,7 +353,7 @@ def smoke(args: argparse.Namespace) -> None:
                 "documents_search": "ok",
                 "raw_metadata_traceability": "ok",
                 "profile": "ok",
-                "conversation_ingest": "accepted",
+                "conversation_ingest": conversation_ingest,
                 "container_tag": container,
             }
         )

servers/unraid/supermemory-poc/smoke.sh

@@ -114,11 +114,20 @@ curl --fail --silent --show-error --max-time 60 \
   "$base_url/v4/profile" >/tmp/profile.json || fail 51 profile-request
 grep -q '"profile"' /tmp/profile.json || fail 52 profile-shape
 
-curl --fail --silent --show-error --max-time 60 \
+conversation_http=$(curl --silent --show-error --max-time 60 \
+  --output /tmp/conversation.json \
+  --write-out '%{http_code}' \
   -H "$auth_header" \
   -H 'Content-Type: application/json' \
   --data "{\"conversationId\":\"$marker\",\"containerTags\":[\"$container_tag\"],\"messages\":[{\"role\":\"user\",\"content\":\"这只是隔离测试。\"},{\"role\":\"assistant\",\"content\":\"不会作为正式事实。\"}],\"metadata\":{\"source\":\"poc-smoke\",\"status\":\"temporary\"}}" \
-  "$base_url/v4/conversations" >/tmp/conversation.json || fail 61 conversation-ingest
+  "$base_url/v4/conversations") || fail 61 conversation-ingest-request
+
+case "$conversation_http" in
+  2??) conversation_ingest=accepted ;;
+  404) conversation_ingest=unsupported-404 ;;
+  *) fail 62 conversation-ingest-unexpected-status ;;
+esac
 
 touch "$sentinel"
-printf '{"embedding_plan":"gemini/gemini-embedding-2/768","llm":"gemini-native-default","v3_search":%s,"v4_documents_search":"ok","chinese_hybrid_search":"ok","metadata_traceability":"ok","profile":"ok","conversation_ingest":"accepted"}\n' "$v3_hit"
+printf '{"embedding_plan":"gemini/gemini-embedding-2/768","llm":"gemini-native-default","v3_search":%s,"v4_documents_search":"ok","chinese_hybrid_search":"ok","metadata_traceability":"ok","profile":"ok","conversation_ingest":"%s","conversation_http":%s}\n' \
+  "$v3_hit" "$conversation_ingest" "$conversation_http"

servers/unraid/supermemory-poc/stability.sh

@@ -0,0 +1,135 @@
+#!/bin/sh
+set -eu
+
+data_dir=/var/lib/supermemory
+sentinel="$data_dir/.stability-gemini-paid-v1.ok"
+base_url="${SUPERMEMORY_URL:-http://127.0.0.1:6767}"
+container_tag="${SUPERMEMORY_CONTAINER_TAG:-supermemory_stability_paid_v1}"
+
+if [ -f "$sentinel" ]; then
+  printf '%s\n' '{"stability":"already-passed","suite":"gemini-paid-v1"}'
+  exit 0
+fi
+
+api_key=$(cat "$data_dir/api-key")
+auth_header="Authorization: Bearer $api_key"
+run_id="paid-v1-$(date +%s)"
+submitted=0
+add_failures=0
+processing_failures=0
+documents_failures=0
+hybrid_failures=0
+metadata_failures=0
+
+case_values() {
+  case "$1" in
+    1) content="海棠计划的下一步是核对原始报告并记录出处。"; query="海棠计划下一步是什么？"; expected="核对原始报告" ;;
+    2) content="Project Juniper rollback owner is Mira Chen."; query="Who owns the Juniper rollback?"; expected="Mira Chen" ;;
+    3) content="Orion 项目的 maintenance window is Tuesday 03:00 UTC."; query="Orion 的维护窗口是什么时候？"; expected="Tuesday" ;;
+    4) content="The synthetic 2026 Q4 budget cap is 42000 credits."; query="What is the synthetic Q4 budget cap?"; expected="42000" ;;
+    5) content="仅在本基准中，状态报告应使用 amber labels。"; query="本基准的状态报告应该用什么标签？"; expected="amber" ;;
+  esac
+}
+
+for case_id in 1 2 3 4 5; do
+  case_values "$case_id"
+  marker="supermemory-stability-$run_id-$case_id"
+  if curl --fail --silent --show-error --max-time 60 \
+    -H "$auth_header" \
+    -H 'Content-Type: application/json' \
+    --data "{\"content\":\"$marker. $content\",\"containerTag\":\"$container_tag\",\"customId\":\"$marker\",\"metadata\":{\"source\":\"poc-stability\",\"suite\":\"gemini-paid-v1\",\"case_id\":\"$case_id\",\"status\":\"temporary\"}}" \
+    "$base_url/v3/documents" >"/tmp/add-$case_id.json"; then
+    document_id=$(sed -n 's/.*"id"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "/tmp/add-$case_id.json" | head -n 1)
+    if [ -n "$document_id" ]; then
+      printf '%s\n' "$document_id" >"/tmp/id-$case_id"
+      submitted=$((submitted + 1))
+    else
+      add_failures=$((add_failures + 1))
+    fi
+  else
+    add_failures=$((add_failures + 1))
+  fi
+done
+
+for _ in $(seq 1 300); do
+  terminal=0
+  for case_id in 1 2 3 4 5; do
+    [ -f "/tmp/id-$case_id" ] || continue
+    if [ -f "/tmp/done-$case_id" ] || [ -f "/tmp/failed-$case_id" ]; then
+      terminal=$((terminal + 1))
+      continue
+    fi
+    document_id=$(cat "/tmp/id-$case_id")
+    if curl --fail --silent --show-error --max-time 30 \
+      -H "$auth_header" \
+      "$base_url/v3/documents/$document_id" >"/tmp/document-$case_id.json"; then
+      if grep -Eq '"status"[[:space:]]*:[[:space:]]*"done"' "/tmp/document-$case_id.json"; then
+        : >"/tmp/done-$case_id"
+        terminal=$((terminal + 1))
+      elif grep -Eq '"status"[[:space:]]*:[[:space:]]*"failed"' "/tmp/document-$case_id.json"; then
+        : >"/tmp/failed-$case_id"
+        terminal=$((terminal + 1))
+        processing_failures=$((processing_failures + 1))
+      fi
+    fi
+  done
+  [ "$terminal" -eq "$submitted" ] && break
+  sleep 2
+done
+
+for case_id in 1 2 3 4 5; do
+  [ -f "/tmp/id-$case_id" ] || continue
+  if [ ! -f "/tmp/done-$case_id" ]; then
+    if [ ! -f "/tmp/failed-$case_id" ]; then
+      processing_failures=$((processing_failures + 1))
+    fi
+    continue
+  fi
+
+  case_values "$case_id"
+  marker="supermemory-stability-$run_id-$case_id"
+  if curl --fail --silent --show-error --max-time 30 \
+    -H "$auth_header" \
+    -H 'Content-Type: application/json' \
+    --data "{\"q\":\"$marker\",\"containerTag\":\"$container_tag\",\"searchMode\":\"documents\",\"limit\":5}" \
+    "$base_url/v4/search" >"/tmp/documents-$case_id.json"; then
+    if ! grep -q "$marker" "/tmp/documents-$case_id.json"; then
+      documents_failures=$((documents_failures + 1))
+    fi
+  else
+    documents_failures=$((documents_failures + 1))
+  fi
+  if [ -f "/tmp/documents-$case_id.json" ] && ! grep -q 'poc-stability' "/tmp/documents-$case_id.json"; then
+    metadata_failures=$((metadata_failures + 1))
+  fi
+
+  if curl --fail --silent --show-error --max-time 30 \
+    -H "$auth_header" \
+    -H 'Content-Type: application/json' \
+    --data "{\"q\":\"$query\",\"containerTag\":\"$container_tag\",\"searchMode\":\"hybrid\",\"limit\":5}" \
+    "$base_url/v4/search" >"/tmp/hybrid-$case_id.json"; then
+    if ! grep -q "$expected" "/tmp/hybrid-$case_id.json"; then
+      hybrid_failures=$((hybrid_failures + 1))
+    fi
+  else
+    hybrid_failures=$((hybrid_failures + 1))
+  fi
+done
+
+conversation_http=$(curl --silent --show-error --max-time 60 \
+  --output /tmp/stability-conversation.json \
+  --write-out '%{http_code}' \
+  -H "$auth_header" \
+  -H 'Content-Type: application/json' \
+  --data "{\"conversationId\":\"supermemory-stability-$run_id-conversation\",\"containerTags\":[\"$container_tag\"],\"messages\":[{\"role\":\"user\",\"content\":\"Synthetic session question.\"},{\"role\":\"assistant\",\"content\":\"Synthetic session answer.\"}],\"metadata\":{\"source\":\"poc-stability\",\"status\":\"temporary\"}}" \
+  "$base_url/v4/conversations") || conversation_http=transport-error
+
+printf '{"stability":"result","suite":"gemini-paid-v1","attempted":5,"submitted":%s,"add_failures":%s,"processing_failures":%s,"documents_failures":%s,"hybrid_failures":%s,"metadata_failures":%s,"conversation_http":"%s"}\n' \
+  "$submitted" "$add_failures" "$processing_failures" "$documents_failures" "$hybrid_failures" "$metadata_failures" "$conversation_http"
+
+if [ "$submitted" -eq 5 ] && [ "$add_failures" -eq 0 ] && [ "$processing_failures" -eq 0 ] && \
+   [ "$documents_failures" -eq 0 ] && [ "$hybrid_failures" -eq 0 ] && [ "$metadata_failures" -eq 0 ]; then
+  touch "$sentinel"
+  exit 0
+fi
+exit 70