diff --git a/servers/unraid/supermemory-poc/.env.example b/servers/unraid/supermemory-poc/.env.example new file mode 100644 index 0000000..f8b820b --- /dev/null +++ b/servers/unraid/supermemory-poc/.env.example @@ -0,0 +1,8 @@ +# Set only in the unRaid Arcane Project environment. Never commit the real file. +SUPERMEMORY_IPV4_ADDRESS= + +# Extraction/summarization LLM. OPENAI_BASE_URL may point to an approved +# OpenAI-compatible endpoint; keep it blank for the OpenAI API. +OPENAI_API_KEY= +OPENAI_BASE_URL= +OPENAI_MODEL=gpt-5.1 diff --git a/servers/unraid/supermemory-poc/.gitignore b/servers/unraid/supermemory-poc/.gitignore new file mode 100644 index 0000000..b992cb1 --- /dev/null +++ b/servers/unraid/supermemory-poc/.gitignore @@ -0,0 +1,4 @@ +.env +artifacts/ +*.jsonl +*.csv diff --git a/servers/unraid/supermemory-poc/README.md b/servers/unraid/supermemory-poc/README.md new file mode 100644 index 0000000..eecbd84 --- /dev/null +++ b/servers/unraid/supermemory-poc/README.md @@ -0,0 +1,122 @@ +# Supermemory POC on unRaid + +This project is an isolated evaluation service. It does not replace the +production Hindsight service or change the default Hermes profile. + +## Fixed deployment choices + +- Supermemory Local `server-v0.0.8`, Linux x64 binary, verified by SHA-256. +- One self-contained server; state, auth material, uploaded files, and model + cache live under `/mnt/user/appdata/supermemory-poc/data`. +- A dedicated address on Docker `br0`; port `6767` is not published on the + unRaid host address. +- Local multilingual `Xenova/bge-m3` embeddings at 1024 dimensions. +- One embedding worker and ingest concurrency 1. This avoids treating known + concurrent-local-embedding instability as a retrieval-quality result. +- Extraction/summarization uses an operator-supplied OpenAI-compatible LLM. +- Resource ceiling: 4 CPUs and 8 GiB RAM. Supermemory Local keeps its corpus + in memory, so RSS must be watched as the sample grows. + +Do not change the embedding model or dimensions in place. Use a fresh data +directory and re-ingest when comparing a different embedding plan. + +## Before deployment + +1. Confirm the chosen `SUPERMEMORY_IPV4_ADDRESS` is absent from both Arcane's + `br0` attachments and the LAN neighbor/DHCP tables. +2. Add `SUPERMEMORY_IPV4_ADDRESS`, `OPENAI_API_KEY`, and optional model/base URL + overrides to this Project's Arcane environment. Do not put secrets in Git, + Compose, activity notes, or chat. +3. Confirm `/mnt/user/appdata` has room for the 298 MiB server binary, the + multilingual model cache, data, and rollback copy. +4. Confirm no other build, image pull, backup, or migration is active on the + unRaid Environment. + +The `supermemory-fetch` init service downloads the exact release asset once, +checks its SHA-256, and stores it in the POC appdata directory. Subsequent +starts only verify the existing binary. No custom image build is required. + +## First boot and authentication + +The server creates its client bearer token at: + +`/mnt/user/appdata/supermemory-poc/data/api-key` + +Read it through an authorized unRaid/Arcane console without printing it into +logs. Save it only as `SUPERMEMORY_API_KEY` in the `supermemory-lab` Hermes +profile. The extraction LLM key and Supermemory client key are different. + +The current candidate address is `192.168.50.13`: it is not attached to an +Arcane-managed `br0` container and did not answer the pre-deployment neighbor +or ICMP probes. Confirm it is outside the DHCP pool or reserve it before the +Project is started. + +Expected Hermes profile file `$HERMES_HOME/supermemory.json` (also provided as +`hermes-supermemory.json.example`): + +```json +{ + "base_url": "http://:6767", + "container_tag": "hermes_supermemory_lab", + "auto_recall": true, + "auto_capture": true, + "max_recall_results": 10, + "profile_frequency": 50, + "capture_mode": "all", + "search_mode": "hybrid", + "api_timeout": 10.0 +} +``` + +Run `poc.py smoke` before importing any sampled production material. It checks +the v3 document path plus the v4 search, profile, and conversation endpoints +used by Hermes. + +## Evaluation guardrails + +- Score raw Supermemory API retrieval separately from the Hermes answer. In + Hermes Agent v0.21.1, the provider reads result metadata but omits it from + automatic prefetch text and from the explicit search tool response. A raw + hit with correct `source`/`original_id` followed by an uncited Hermes answer + is an adapter attribution gap, not a retrieval miss. +- Do not run the fixed scored question set with automatic writes enabled. The + default Hindsight profile has `auto_retain: true`, and this lab profile has + `auto_capture: true`; earlier test answers could leak into later questions. + Use a read-only Hindsight snapshot/profile and set Supermemory + `auto_capture: false` for the retrieval benchmark. Re-enable capture only for + the separately scored session-experience phase. +- Hermes built-in `MEMORY.md` and `USER.md` remain active independently of the + external provider. Record their presence and size for each arm so built-in + context is not credited to Hindsight or Supermemory. +- Self-hosted Supermemory has no managed Google Drive, Gmail, Notion, or + OneDrive connectors. This POC evaluates explicit file/document ingestion, + not hosted connector sync. + +## Staged data flow + +1. `poc.py sample-hindsight` reads active observations from the supported + Hindsight 0.8.4 list API and writes a local JSONL artifact. It never mutates + the source bank. +2. Manually review that artifact for scope and sensitive content. +3. `poc.py ingest-jsonl` writes the approved rows serially to the isolated + Supermemory container using stable custom IDs and provenance metadata. +4. `poc.py ingest-files` uploads a small, separately reviewed + DEVONthink/Obsidian manifest. Keep the original UUID/path in metadata and + treat the source system as authoritative. The default per-file cap is 50 + MiB so an accidental library-wide import fails closed. +5. Run the same questions against the default and `supermemory-lab` profiles; + record groundedness, source traceability, latency, tokens, and cost. + +Artifacts are ignored by Git. Do not put exported observations or document +content in this configuration repository. + +Hermes sanitizes container tags by replacing hyphens with underscores. Keep +the canonical `hermes_supermemory_lab` spelling in direct API imports so they +land in the same container queried by the profile. + +## Rollback + +Stop/down only this Project. Keep `/mnt/user/appdata/supermemory-poc` for later +inspection, or archive it before any separately approved deletion. The default +Hermes profile and Hindsight service require no rollback because this project +does not modify them. diff --git a/servers/unraid/supermemory-poc/ab-questions.jsonl.example b/servers/unraid/supermemory-poc/ab-questions.jsonl.example new file mode 100644 index 0000000..0325ec2 --- /dev/null +++ b/servers/unraid/supermemory-poc/ab-questions.jsonl.example @@ -0,0 +1,5 @@ +{"id":"memory-01","class":"user_project_memory","question":"我们为什么把 Hermes 的 Hindsight recall_max_tokens 设在 768 左右？请说明决定和依据。","expected_sources":["hindsight"]} +{"id":"memory-02","class":"user_project_memory","question":"当前 memory provider 的部署方式、bank 和自动召回/保留策略是什么？","expected_sources":["hindsight"]} +{"id":"document-01","class":"cross_document","question":"根据指定文档集合，概括核心结论，并逐条给出可追溯来源。","expected_sources":["devonthink","obsidian"]} +{"id":"document-02","class":"cross_document","question":"这些材料在时间线、数字或结论上有哪些冲突？引用各自来源。","expected_sources":["devonthink","obsidian"]} +{"id":"mixed-01","class":"mixed","question":"结合我过去对长期记忆的判断和指定文档证据，给出下一步建议；区分历史偏好、当前事实和你的推断。","expected_sources":["hindsight","devonthink","obsidian"]} diff --git a/servers/unraid/supermemory-poc/compose.yaml b/servers/unraid/supermemory-poc/compose.yaml new file mode 100644 index 0000000..83f0f6e --- /dev/null +++ b/servers/unraid/supermemory-poc/compose.yaml @@ -0,0 +1,94 @@ +services: + supermemory-fetch: + image: curlimages/curl:8.16.0@sha256:463eaf6072688fe96ac64fa623fe73e1dbe25d8ad6c34404a669ad3ce1f104b6 + container_name: supermemory-poc-fetch + user: "0:0" + restart: "no" + environment: + SUPERMEMORY_BINARY_URL: https://github.com/supermemoryai/supermemory/releases/download/server-v0.0.8/supermemory-server-linux-x64 + SUPERMEMORY_BINARY_SHA256: 87f32433d0179be80bb9d8a1bafbac65af4128324342a27ecb8bd1a77b5506f3 + entrypoint: + - /bin/sh + - -ec + command: + - | + target=/opt/supermemory/bin/supermemory-server + checksum="$${SUPERMEMORY_BINARY_SHA256} $${target}" + mkdir -p /opt/supermemory/bin + if [ -f "$${target}" ] && echo "$${checksum}" | sha256sum -c - >/dev/null 2>&1; then + exit 0 + fi + tmp="$${target}.download" + rm -f "$${tmp}" + curl --fail --location --retry 3 --output "$${tmp}" "$${SUPERMEMORY_BINARY_URL}" + echo "$${SUPERMEMORY_BINARY_SHA256} $${tmp}" | sha256sum -c - + chmod 0755 "$${tmp}" + mv "$${tmp}" "$${target}" + volumes: + - /mnt/user/appdata/supermemory-poc/bin:/opt/supermemory/bin + networks: + - fetch + security_opt: + - no-new-privileges:true + + supermemory: + image: debian:bookworm-slim@sha256:88200866dfff7ea7f5cbcb6ec7c8a701889efe6fe859fe64d6990e4b07ea4171 + platform: linux/amd64 + container_name: supermemory-poc + restart: unless-stopped + depends_on: + supermemory-fetch: + condition: service_completed_successfully + environment: + PORT: "6767" + SUPERMEMORY_DATA_DIR: /var/lib/supermemory + SUPERMEMORY_DISABLE_TELEMETRY: "1" + SUPERMEMORY_EMBEDDING_PROVIDER: local + SUPERMEMORY_EMBEDDING_MODEL: Xenova/bge-m3 + SUPERMEMORY_EMBEDDING_DIMENSIONS: "1024" + SUPERMEMORY_LOCAL_EMBEDDING_POOL_SIZE: "1" + SUPERMEMORY_LOCAL_EMBEDDING_WASM_THREADS: "1" + SUPERMEMORY_LOCAL_EMBEDDING_BATCH_SIZE: "4" + SUPERMEMORY_EMBEDDING_RAM_LIMIT: 2gb + SUPERMEMORY_INGEST_CONCURRENCY: "1" + OPENAI_API_KEY: ${OPENAI_API_KEY:?set OPENAI_API_KEY in Arcane environment} + OPENAI_BASE_URL: ${OPENAI_BASE_URL:-} + OPENAI_MODEL: ${OPENAI_MODEL:-gpt-5.1} + entrypoint: + - /bin/bash + - -ec + command: + - | + umask 077 + exec /opt/supermemory/bin/supermemory-server + healthcheck: + test: ["CMD-SHELL", "bash -ec ': >/dev/tcp/127.0.0.1/6767'"] + interval: 15s + timeout: 3s + retries: 20 + start_period: 120s + stop_grace_period: 60s + mem_limit: 8g + cpus: 4 + volumes: + - /mnt/user/appdata/supermemory-poc/bin:/opt/supermemory/bin:ro + - /mnt/user/appdata/supermemory-poc/data:/var/lib/supermemory + networks: + br0: + ipv4_address: ${SUPERMEMORY_IPV4_ADDRESS:?set an unused LAN address in Arcane environment} + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + logging: + driver: json-file + options: + max-size: 10m + max-file: "3" + +networks: + fetch: + internal: false + br0: + external: true + name: br0 diff --git a/servers/unraid/supermemory-poc/documents-manifest.jsonl.example b/servers/unraid/supermemory-poc/documents-manifest.jsonl.example new file mode 100644 index 0000000..c8ebb05 --- /dev/null +++ b/servers/unraid/supermemory-poc/documents-manifest.jsonl.example @@ -0,0 +1,2 @@ +{"path":"/absolute/path/to/reviewed-note.md","source":"obsidian","original_id":"obsidian:reviewed-note","original_path":"Projects/reviewed-note.md","date":"2026-09-01","confidence":"reviewed","status":"authoritative-copy"} +{"path":"/absolute/path/to/reviewed-report.pdf","source":"devonthink","original_id":"DEVONthink-UUID-HERE","original_path":"DEVONthink/Research/reviewed-report.pdf","date":"2026-08-15","confidence":"reviewed","status":"source-authoritative"} diff --git a/servers/unraid/supermemory-poc/hermes-supermemory.json.example b/servers/unraid/supermemory-poc/hermes-supermemory.json.example new file mode 100644 index 0000000..ea69159 --- /dev/null +++ b/servers/unraid/supermemory-poc/hermes-supermemory.json.example @@ -0,0 +1,11 @@ +{ + "api_timeout": 10.0, + "auto_capture": true, + "auto_recall": true, + "base_url": "http://192.168.50.13:6767", + "capture_mode": "all", + "container_tag": "hermes_supermemory_lab", + "max_recall_results": 10, + "profile_frequency": 50, + "search_mode": "hybrid" +} diff --git a/servers/unraid/supermemory-poc/poc.py b/servers/unraid/supermemory-poc/poc.py new file mode 100755 index 0000000..7157c6b --- /dev/null +++ b/servers/unraid/supermemory-poc/poc.py @@ -0,0 +1,399 @@ +#!/usr/bin/env python3 +"""Safe helpers for the isolated Hindsight vs Supermemory POC.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import mimetypes +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +def request_json( + method: str, + url: str, + *, + token: str = "", + payload: dict[str, Any] | None = None, + timeout: float = 30.0, +) -> Any: + headers = {"Accept": "application/json"} + data = None + if token: + headers["Authorization"] = f"Bearer {token}" + if payload is not None: + headers["Content-Type"] = "application/json" + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + req = urllib.request.Request(url, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + body = response.read() + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace")[:300] + raise RuntimeError(f"{method} {url}: HTTP {exc.code}: {detail}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"{method} {url}: {exc.reason}") from exc + return json.loads(body) if body else {} + + +def require_token(env_name: str) -> str: + token = os.environ.get(env_name, "").strip() + if not token: + raise RuntimeError(f"{env_name} is not set") + return token + + +def request_multipart_json( + url: str, + *, + token: str, + fields: dict[str, str], + file_path: Path, + timeout: float, +) -> Any: + boundary = f"supermemory-poc-{uuid.uuid4().hex}" + chunks: list[bytes] = [] + for name, value in fields.items(): + chunks.extend( + [ + f"--{boundary}\r\n".encode(), + f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode(), + value.encode("utf-8"), + b"\r\n", + ] + ) + filename = file_path.name.replace('"', "_") + mime_type = mimetypes.guess_type(filename)[0] or "application/octet-stream" + chunks.extend( + [ + f"--{boundary}\r\n".encode(), + f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n'.encode(), + f"Content-Type: {mime_type}\r\n\r\n".encode(), + file_path.read_bytes(), + b"\r\n", + f"--{boundary}--\r\n".encode(), + ] + ) + req = urllib.request.Request( + url, + data=b"".join(chunks), + headers={ + "Accept": "application/json", + "Authorization": f"Bearer {token}", + "Content-Type": f"multipart/form-data; boundary={boundary}", + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as response: + body = response.read() + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace")[:300] + raise RuntimeError(f"POST {url}: HTTP {exc.code}: {detail}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"POST {url}: {exc.reason}") from exc + return json.loads(body) if body else {} + + +def sample_hindsight(args: argparse.Namespace) -> None: + base = args.hindsight_url.rstrip("/") + bank = urllib.parse.quote(args.bank_id, safe="") + list_url = f"{base}/v1/default/banks/{bank}/memories/list" + first_query = urllib.parse.urlencode( + {"type": "observation", "state": "valid", "limit": 1, "offset": 0} + ) + first = request_json("GET", f"{list_url}?{first_query}", timeout=args.timeout) + total = int(first.get("total") or 0) + if total < 1: + raise RuntimeError("Hindsight returned no active observations") + + wanted = min(args.limit, total) + page_size = min(args.page_size, wanted) + page_count = max(1, (wanted + page_size - 1) // page_size) + max_offset = max(0, total - page_size) + offsets = ( + [0] + if page_count == 1 + else [round(index * max_offset / (page_count - 1)) for index in range(page_count)] + ) + + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + imported_at = datetime.now(timezone.utc).isoformat() + for offset in offsets: + query = urllib.parse.urlencode( + { + "type": "observation", + "state": "valid", + "limit": page_size, + "offset": offset, + } + ) + response = request_json("GET", f"{list_url}?{query}", timeout=args.timeout) + for item in response.get("items") or []: + item_id = str(item.get("id") or "").strip() + text = str(item.get("text") or "").strip() + if not item_id or not text or item_id in seen: + continue + seen.add(item_id) + rows.append( + { + "content": text, + "custom_id": f"hindsight:{args.bank_id}:{item_id}", + "metadata": { + "source": "hindsight", + "original_id": item_id, + "bank_id": args.bank_id, + "date": item.get("date"), + "mentioned_at": item.get("mentioned_at"), + "occurred_start": item.get("occurred_start"), + "occurred_end": item.get("occurred_end"), + "confidence": None, + "status": item.get("state") or "valid", + "fact_type": item.get("fact_type"), + "proof_count": item.get("proof_count"), + "tags": item.get("tags") or [], + "imported_at": imported_at, + }, + } + ) + if len(rows) >= wanted: + break + if len(rows) >= wanted: + break + + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + with output.open("x", encoding="utf-8") as handle: + for row in rows: + handle.write(json.dumps(row, ensure_ascii=False) + "\n") + output.chmod(0o600) + print(json.dumps({"sampled": len(rows), "source_total": total, "output": str(output)})) + + +def ingest_jsonl(args: argparse.Namespace) -> None: + token = require_token(args.api_key_env) + base = args.supermemory_url.rstrip("/") + succeeded = 0 + failed = 0 + for line_number, raw in enumerate(Path(args.input).read_text(encoding="utf-8").splitlines(), 1): + if not raw.strip(): + continue + row = json.loads(raw) + payload = { + "content": str(row["content"]), + "containerTag": args.container_tag, + "customId": str(row.get("custom_id") or f"poc-line:{line_number}"), + "metadata": row.get("metadata") or {}, + } + try: + request_json( + "POST", + f"{base}/v3/documents", + token=token, + payload=payload, + timeout=args.timeout, + ) + succeeded += 1 + except RuntimeError as exc: + failed += 1 + print(f"line {line_number}: {exc}", file=sys.stderr) + if not args.keep_going: + raise + time.sleep(args.delay) + print(json.dumps({"accepted": succeeded, "failed": failed, "container_tag": args.container_tag})) + + +def ingest_files(args: argparse.Namespace) -> None: + token = require_token(args.api_key_env) + base = args.supermemory_url.rstrip("/") + accepted = 0 + max_bytes = args.max_file_mib * 1024 * 1024 + for line_number, raw in enumerate(Path(args.manifest).read_text(encoding="utf-8").splitlines(), 1): + if not raw.strip(): + continue + row = json.loads(raw) + path = Path(str(row["path"])).expanduser().resolve() + if not path.is_file(): + raise RuntimeError(f"manifest line {line_number}: file does not exist: {path}") + size = path.stat().st_size + if size > max_bytes: + raise RuntimeError( + f"manifest line {line_number}: {path} is {size} bytes, above the {args.max_file_mib} MiB cap" + ) + source = str(row.get("source") or "").strip().lower() + if source not in {"devonthink", "obsidian"}: + raise RuntimeError(f"manifest line {line_number}: source must be devonthink or obsidian") + original_id = str(row.get("original_id") or "").strip() + original_path = str(row.get("original_path") or path) + stable_input = f"{source}\0{original_id}\0{original_path}".encode("utf-8") + custom_id = str(row.get("custom_id") or f"{source}:{hashlib.sha256(stable_input).hexdigest()[:32]}") + metadata = { + "source": source, + "original_id": original_id, + "original_path": original_path, + "date": str(row.get("date") or ""), + "imported_at": datetime.now(timezone.utc).isoformat(), + "confidence": str(row.get("confidence") or ""), + "status": str(row.get("status") or "reviewed"), + } + response = request_multipart_json( + f"{base}/v3/documents/file", + token=token, + fields={ + "containerTag": args.container_tag, + "customId": custom_id, + "filepath": original_path, + "metadata": json.dumps(metadata, ensure_ascii=False), + }, + file_path=path, + timeout=args.timeout, + ) + accepted += 1 + print(json.dumps({"line": line_number, "id": response.get("id"), "status": response.get("status")})) + time.sleep(args.delay) + print(json.dumps({"accepted": accepted, "container_tag": args.container_tag})) + + +def smoke(args: argparse.Namespace) -> None: + token = require_token(args.api_key_env) + base = args.supermemory_url.rstrip("/") + marker = f"supermemory-poc-smoke-{int(time.time())}" + container = args.container_tag + add = request_json( + "POST", + f"{base}/v3/documents", + token=token, + payload={ + "content": f"{marker}: 中文跨文档检索烟测；authoritative source is the POC harness.", + "containerTag": container, + "customId": marker, + "metadata": {"source": "poc-smoke", "status": "temporary"}, + }, + timeout=args.timeout, + ) + document_id = str(add.get("id") or "") + deadline = time.monotonic() + args.wait + found = False + while time.monotonic() < deadline: + search = request_json( + "POST", + f"{base}/v4/search", + token=token, + payload={ + "q": marker, + "containerTag": container, + "searchMode": "documents", + "limit": 5, + }, + timeout=args.timeout, + ) + found = bool(search.get("results")) + if found: + break + time.sleep(2) + profile = request_json( + "POST", + f"{base}/v4/profile", + token=token, + payload={"containerTag": container, "q": marker}, + timeout=args.timeout, + ) + request_json( + "POST", + f"{base}/v4/conversations", + token=token, + payload={ + "conversationId": marker, + "containerTags": [container], + "messages": [ + {"role": "user", "content": "记住本轮只是隔离 POC 烟测。"}, + {"role": "assistant", "content": "已记录为临时测试，不作为正式事实。"}, + ], + "metadata": {"source": "poc-smoke", "status": "temporary"}, + }, + timeout=args.timeout, + ) + if not found: + raise RuntimeError(f"document {document_id or marker} was not searchable within {args.wait}s") + profile_shape_ok = isinstance(profile, dict) and "profile" in profile + if not profile_shape_ok: + raise RuntimeError("v4 profile response is missing the profile field expected by Hermes") + print( + json.dumps( + { + "document_id": document_id, + "documents_search": "ok", + "profile": "ok", + "conversation_ingest": "accepted", + "container_tag": container, + } + ) + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + + sample = commands.add_parser("sample-hindsight", help="Read a stratified observation sample") + sample.add_argument("--hindsight-url", default="http://192.168.50.100:8888") + sample.add_argument("--bank-id", default="personal-context") + sample.add_argument("--limit", type=int, default=60) + sample.add_argument("--page-size", type=int, default=10) + sample.add_argument("--timeout", type=float, default=30.0) + sample.add_argument("--output", required=True) + sample.set_defaults(func=sample_hindsight) + + ingest = commands.add_parser("ingest-jsonl", help="Serially ingest reviewed JSONL rows") + ingest.add_argument("--supermemory-url", required=True) + ingest.add_argument("--container-tag", default="hermes_supermemory_lab") + ingest.add_argument("--api-key-env", default="SUPERMEMORY_API_KEY") + ingest.add_argument("--input", required=True) + ingest.add_argument("--delay", type=float, default=0.5) + ingest.add_argument("--timeout", type=float, default=60.0) + ingest.add_argument("--keep-going", action="store_true") + ingest.set_defaults(func=ingest_jsonl) + + files = commands.add_parser("ingest-files", help="Upload a reviewed DEVONthink/Obsidian manifest") + files.add_argument("--supermemory-url", required=True) + files.add_argument("--container-tag", default="hermes_supermemory_lab") + files.add_argument("--api-key-env", default="SUPERMEMORY_API_KEY") + files.add_argument("--manifest", required=True) + files.add_argument("--max-file-mib", type=int, default=50) + files.add_argument("--delay", type=float, default=1.0) + files.add_argument("--timeout", type=float, default=120.0) + files.set_defaults(func=ingest_files) + + check = commands.add_parser("smoke", help="Exercise the API paths used by Hermes") + check.add_argument("--supermemory-url", required=True) + check.add_argument("--container-tag", default="hermes_supermemory_lab") + check.add_argument("--api-key-env", default="SUPERMEMORY_API_KEY") + check.add_argument("--wait", type=float, default=120.0) + check.add_argument("--timeout", type=float, default=60.0) + check.set_defaults(func=smoke) + return parser + + +def main() -> int: + try: + args = build_parser().parse_args() + args.func(args) + return 0 + except (KeyError, ValueError, OSError, RuntimeError, json.JSONDecodeError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/servers/unraid/supermemory-poc/scorecard.csv.example b/servers/unraid/supermemory-poc/scorecard.csv.example new file mode 100644 index 0000000..f7deeac --- /dev/null +++ b/servers/unraid/supermemory-poc/scorecard.csv.example @@ -0,0 +1 @@ +run_id,question_id,arm,profile,raw_retrieval_hit_0_1,raw_metadata_traceable_0_1,answer_correctness_1_5,answer_source_traceability_0_2,preference_pollution_0_2,chinese_quality_1_5,chunk_quality_1_5,latency_ms,input_tokens,output_tokens,returned_context_tokens,estimated_cost_usd,notes