deploy local retrieval models for Hindsight

2026-09-18 00:29:33 -07:00
parent 26d56f7565
commit 2f8bf0e944
4 changed files with 132 additions and 9 deletions
@@ -60,16 +60,25 @@ HINDSIGHT_API_REFLECT_LLM_MODEL=anthropic/claude-sonnet-5
 HINDSIGHT_API_REFLECT_LLM_MAX_CONCURRENT=1
 HINDSIGHT_API_REFLECT_LLM_TIMEOUT=600

-# OpenRouter embeddings. BGE-M3 emits 1024-dimensional multilingual vectors.
+# Local TEI embeddings. BGE-M3 emits 1024-dimensional multilingual vectors.
+# OpenRouter settings remain available for a manual rollback; Hindsight does not
+# provide a native embedding-provider fallback chain.
 HINDSIGHT_API_OPENROUTER_API_KEY=replace-with-openrouter-key
-HINDSIGHT_API_EMBEDDINGS_PROVIDER=openrouter
+HINDSIGHT_API_EMBEDDINGS_PROVIDER=tei
+HINDSIGHT_API_EMBEDDINGS_TEI_URL=http://192.168.50.100:18083
 HINDSIGHT_API_EMBEDDINGS_OPENROUTER_MODEL=baai/bge-m3
 HINDSIGHT_API_EMBEDDINGS_OPENAI_BATCH_SIZE=32
 HINDSIGHT_API_EMBEDDINGS_MAX_INPUT_TOKENS=8192

-# OpenRouter reranker, with a fail-open RRF fallback.
-HINDSIGHT_API_RERANKER_PROVIDER=openrouter
-HINDSIGHT_API_RERANKER_OPENROUTER_MODEL=cohere/rerank-4-fast
-HINDSIGHT_API_RERANKER_OPENROUTER_TIMEOUT=30
+# Local TEI reranker, then OpenRouter, then fail-open RRF.
+HINDSIGHT_API_RERANKER_PROVIDER=tei
+HINDSIGHT_API_RERANKER_TEI_URL=http://192.168.50.100:18084
+HINDSIGHT_API_RERANKER_TEI_BATCH_SIZE=32
+HINDSIGHT_API_RERANKER_TEI_MAX_CONCURRENT=4
+HINDSIGHT_API_RERANKER_TEI_HTTP_TIMEOUT=30
 HINDSIGHT_API_RERANKER_MAX_CANDIDATES=100
-HINDSIGHT_API_RERANKER_1_PROVIDER=rrf
+HINDSIGHT_API_RERANKER_1_PROVIDER=openrouter
+HINDSIGHT_API_RERANKER_1_OPENROUTER_API_KEY=replace-with-openrouter-key
+HINDSIGHT_API_RERANKER_1_OPENROUTER_MODEL=cohere/rerank-4-fast
+HINDSIGHT_API_RERANKER_1_OPENROUTER_TIMEOUT=30
+HINDSIGHT_API_RERANKER_2_PROVIDER=rrf
@@ -7,8 +7,8 @@

 - Hindsight `0.9.2-slim`；
 - PostgreSQL 18 + pgvector；
- OpenRouter `baai/bge-m3` embedding；
- OpenRouter `cohere/rerank-4-fast` reranker，失败时回退到 RRF；
+- unRaid GPU0 上的本地 TEI `BAAI/bge-m3` embedding；OpenRouter 同模型保留为手动回滚；
+- unRaid GPU0 上的本地 TEI `Alibaba-NLP/gte-multilingual-reranker-base` reranker，失败时依次回退 OpenRouter `cohere/rerank-4-fast` 与 RRF；
 - ZenMux Luna 用于 Retain；
 - ZenMux Grok 4.6 用于 Consolidation；
 - ZenMux Sonnet 5 用于 Reflect。
@@ -0,0 +1,17 @@
+# Hindsight local retrieval models
+
+Arcane-managed TEI services on unRaid GPU0 for the production Hindsight VM.
+
+| Service | Model | LAN endpoint |
+| --- | --- | --- |
+| Embedding | `BAAI/bge-m3` | `http://192.168.50.100:18083` |
+| Reranker | `Alibaba-NLP/gte-multilingual-reranker-base` | `http://192.168.50.100:18084` |
+
+The BGE-M3 cache is reused from the retired unRaid Hindsight stack. The GTE
+reranker has a separate cache. Both services use physical GPU0 and conservative
+TEI concurrency/tokenizer settings.
+
+Hindsight uses the local embedding directly. Hindsight has no native embedding
+provider fallback chain, so OpenRouter BGE-M3 remains a manual rollback rather
+than an automatic fallback. Reranking uses local TEI first, then OpenRouter
+`cohere/rerank-4-fast`, then RRF.
@@ -0,0 +1,97 @@
+services:
+  embedding:
+    image: ghcr.io/huggingface/text-embeddings-inference@sha256:a7d82dfef16c3bf1a95e93f5b226f358312512dbb0d585b48c3cf886f9d470a9
+    pull_policy: never
+    container_name: hindsight-embedding-local
+    restart: unless-stopped
+    command:
+      - --model-id
+      - BAAI/bge-m3
+      - --dtype
+      - float16
+      - --max-input-length
+      - "8192"
+      - --max-client-batch-size
+      - "32"
+      - --max-concurrent-requests
+      - "64"
+      - --max-batch-tokens
+      - "8192"
+      - --tokenization-workers
+      - "4"
+      - --auto-truncate
+      - --served-model-name
+      - BAAI/bge-m3
+    environment:
+      HF_HUB_CACHE: /data
+      TZ: America/Los_Angeles
+    ports:
+      - "192.168.50.100:18083:80"
+    volumes:
+      - /mnt/cache-appdata/ai-models/hindsight/models/embedding:/data
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              device_ids: ["0"]
+              capabilities: [gpu]
+    healthcheck:
+      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:80/health"]
+      interval: 30s
+      timeout: 10s
+      start_period: 120s
+      retries: 6
+    labels:
+      com.getarcaneapp.arcane.icon: huggingface
+      net.unraid.docker.icon: https://github.com/huggingface.png?size=128
+      net.unraid.docker.webui: http://[IP]:[PORT:18083]/docs
+
+  reranker:
+    image: ghcr.io/huggingface/text-embeddings-inference@sha256:a7d82dfef16c3bf1a95e93f5b226f358312512dbb0d585b48c3cf886f9d470a9
+    pull_policy: never
+    container_name: hindsight-reranker-local
+    restart: unless-stopped
+    command:
+      - --model-id
+      - Alibaba-NLP/gte-multilingual-reranker-base
+      - --dtype
+      - float16
+      - --max-input-length
+      - "8192"
+      - --max-client-batch-size
+      - "32"
+      - --max-concurrent-requests
+      - "32"
+      - --max-batch-tokens
+      - "8192"
+      - --tokenization-workers
+      - "4"
+      - --auto-truncate
+      - --served-model-name
+      - Alibaba-NLP/gte-multilingual-reranker-base
+    environment:
+      HF_HUB_CACHE: /data
+      TZ: America/Los_Angeles
+    ports:
+      - "192.168.50.100:18084:80"
+    volumes:
+      - /mnt/cache-appdata/ai-models/hindsight/models/gte-reranker:/data
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              device_ids: ["0"]
+              capabilities: [gpu]
+    healthcheck:
+      test: ["CMD", "curl", "-fsS", "http://127.0.0.1:80/health"]
+      interval: 30s
+      timeout: 10s
+      start_period: 180s
+      retries: 6
+    labels:
+      com.getarcaneapp.arcane.icon: huggingface
+      net.unraid.docker.icon: https://github.com/huggingface.png?size=128
+      net.unraid.docker.webui: http://[IP]:[PORT:18084]/docs
+