v0.6-wip: build_report \u7edf\u4e00\u5165\u53e3 + build_glossary \u672f\u8bed\u6838\u67e5

\u7ee7\u7eed\u89e3\u51b3\u7528\u6237\u53cd\u9988\u7684 PDF \u95ee\u9898\u3002 scripts/build_report.py\uff08\u65b0\u589e\uff09\uff1a - \u5355\u4e00\u5165\u53e3\u540c\u65f6\u51fa PDF + DOCX - \u6587\u4ef6\u540d\u81ea\u52a8\u4ece manifest.report_title \u751f\u6210\uff08\u89e3\u51b3 "final.pdf" \u6CDB\u540d\u95EE\u9898\uff09 - pandoc --from=markdown-tex_math_dollars \u4fee\u590d DOCX \u751f\u6210\u65f6\u7684 $ \u8bef\u89e3 - \u81ea\u52a8\u5bfb\u627e phase2/sources.jsonl \u4f5c\u4e3a\u53c2\u8003\u6587\u732e\u5f15\u6587\u6e90 scripts/lib/search_client.py\uff08\u65b0\u589e\uff09\uff1a - Exa \u4e3b\u529b + Tavily fallback \u7684\u7edf\u4e00\u63a5\u53e3 - \u5173\u952e\u4fee\u590d\uff1atrust_env=False \u7ed5\u5f00\u7cfb\u7edf socks5 \u4ee3\u7406 \uff08Clash on macOS \u5c0a httpx TLS \u63e1\u624b\u5728 CONNECT \u540e EOF\uff09 scripts/build_glossary.py\uff08\u65b0\u589e\uff09\uff1a - \u7528\u7684\u4e92\u65b9\u5f0f\u89e3\u51b3\u4e86\u7528\u6237\u53cd\u9988 #6\uff1a\u672f\u8bed\u7ffb\u8bd1\u4e0d\u4e13\u4e1a / \u4e8b\u5b9e\u9519\u8bef - ThreadPoolExecutor \u5e76\u53d1\uff08\u9ed8\u8ba4 6 worker\uff09\uff0c\u6bcf\u4e2a\u672f\u8bed\u72ec\u7acb\uff1a Search \u2192 Top-3 snippet \u2192 Haiku \u5224\u5b9a \u2192 \u8fd4\u56de {zh, en_full, confidence, issue} - \u5b9e\u6d4b\u6210\u529f\u8bc6\u522b "Maywavee" \u4e3a "Mabwell" \u7684\u62fc\u5199\u9519\u8bef\u5e76\u6807\u51fa issue - \u65ad\u70b9\u7eed\u4f20\uff08\u5df2\u6807 verified_at \u7684\u9ed8\u8ba4\u8df3\u8fc7\uff09 - Haiku \u6210\u672c\u6781\u4f4e\uff083 \u4e2a\u672f\u8bed\u8c03\u7528 \u2248 0.01 \u7f8e\u5206\uff09 - \u652f\u6301 --extra terms.txt \u8865\u5145\u7ffb\u8bd1\u9636\u6bb5\u672a\u6536\u5165\u7684\u672f\u8bed scripts/prompts/glossary_system.txt\uff08\u65b0\u589e\uff09\uff1a - Haiku \u6838\u67e5\u672f\u8bed\u7684 prompt\uff0c\u660e\u786e\u5224\u5b9a\u7ef4\u5ea6\u548c JSON \u8f93\u51fa\u683c\u5f0f Co-authored-by: User <human>
2026-04-22 13:12:01 +08:00
parent a86010e9a7
commit d3fde1cbb8
6 changed files with 1761 additions and 310 deletions
@@ -0,0 +1,177 @@
+"""通用搜索客户端（Exa 优先，Tavily fallback）。
+
+为 build_glossary.py 这类术语核查场景服务。
+
+关键设计：
+- `trust_env=False` 绕开系统 socks 代理（Clash on macOS 配 socks5 时 httpx 会 TLS EOF）
+- Exa 优先：LinkedIn / 官网 / 百度百科返回质量最高
+- 遇到配额问题自动降级到 Tavily 或返回 empty
+- 不做深度 crawl，只要摘要
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+
+@dataclass
+class SearchHit:
+    title: str
+    url: str
+    snippet: str
+
+
+class SearchError(RuntimeError):
+    pass
+
+
+class ExaClient:
+    def __init__(self, api_key: str | None = None, timeout: float = 30.0) -> None:
+        self.api_key = api_key or os.environ.get("EXA_API_KEY")
+        if not self.api_key:
+            raise SearchError("EXA_API_KEY not set")
+        # trust_env=False 关键：不吃系统代理，避免 TLS EOF
+        self._client = httpx.Client(trust_env=False, timeout=timeout)
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> "ExaClient":
+        return self
+
+    def __exit__(self, *_args: Any) -> None:
+        self.close()
+
+    def search(self, query: str, *, num_results: int = 5) -> list[SearchHit]:
+        body = {
+            "query": query,
+            "numResults": num_results,
+            "type": "auto",
+            "contents": {"text": {"maxCharacters": 800}},
+        }
+        r = self._client.post(
+            "https://api.exa.ai/search",
+            json=body,
+            headers={"x-api-key": self.api_key, "Content-Type": "application/json"},
+        )
+        if r.status_code != 200:
+            raise SearchError(f"Exa HTTP {r.status_code}: {r.text[:200]}")
+        data = r.json()
+        out: list[SearchHit] = []
+        for item in data.get("results", [])[:num_results]:
+            out.append(
+                SearchHit(
+                    title=(item.get("title") or "")[:200],
+                    url=item.get("url") or "",
+                    snippet=(item.get("text") or item.get("snippet") or "")[:600],
+                )
+            )
+        return out
+
+
+class TavilyClient:
+    def __init__(self, api_key: str | None = None, timeout: float = 30.0) -> None:
+        self.api_key = api_key or os.environ.get("TAVILY_API_KEY")
+        if not self.api_key:
+            raise SearchError("TAVILY_API_KEY not set")
+        self._client = httpx.Client(trust_env=False, timeout=timeout)
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> "TavilyClient":
+        return self
+
+    def __exit__(self, *_args: Any) -> None:
+        self.close()
+
+    def search(self, query: str, *, num_results: int = 5) -> list[SearchHit]:
+        body = {
+            "api_key": self.api_key,
+            "query": query,
+            "search_depth": "basic",
+            "max_results": num_results,
+            "include_answer": False,
+            "include_raw_content": False,
+        }
+        r = self._client.post("https://api.tavily.com/search", json=body)
+        if r.status_code != 200:
+            raise SearchError(f"Tavily HTTP {r.status_code}: {r.text[:200]}")
+        data = r.json()
+        out: list[SearchHit] = []
+        for item in data.get("results", [])[:num_results]:
+            out.append(
+                SearchHit(
+                    title=(item.get("title") or "")[:200],
+                    url=item.get("url") or "",
+                    snippet=(item.get("content") or "")[:600],
+                )
+            )
+        return out
+
+
+class SearchClient:
+    """统一搜索门面：先用 Exa，失败/配额问题降级 Tavily。"""
+
+    def __init__(self) -> None:
+        self._exa: ExaClient | None = None
+        self._tavily: TavilyClient | None = None
+        try:
+            self._exa = ExaClient()
+        except SearchError:
+            pass
+        try:
+            self._tavily = TavilyClient()
+        except SearchError:
+            pass
+        if not (self._exa or self._tavily):
+            raise SearchError(
+                "neither EXA_API_KEY nor TAVILY_API_KEY available"
+            )
+
+    def close(self) -> None:
+        if self._exa:
+            self._exa.close()
+        if self._tavily:
+            self._tavily.close()
+
+    def __enter__(self) -> "SearchClient":
+        return self
+
+    def __exit__(self, *_args: Any) -> None:
+        self.close()
+
+    def search(self, query: str, *, num_results: int = 5) -> list[SearchHit]:
+        # 优先 Exa
+        if self._exa:
+            try:
+                return self._exa.search(query, num_results=num_results)
+            except SearchError as e:
+                msg = str(e).lower()
+                if "exceed" in msg or "quota" in msg or "429" in msg or "402" in msg:
+                    # 降级
+                    pass
+                else:
+                    # 其它错误继续往下试
+                    pass
+        if self._tavily:
+            try:
+                return self._tavily.search(query, num_results=num_results)
+            except SearchError:
+                pass
+        return []
+
+
+if __name__ == "__main__":
+    from scripts.lib.zenmux_client import load_secrets
+    load_secrets()
+    with SearchClient() as c:
+        hits = c.search("Mabwell 迈威生物 biopharmaceutical", num_results=3)
+        for i, h in enumerate(hits, 1):
+            print(f"[{i}] {h.title[:80]}")
+            print(f"    {h.url}")
+            print(f"    {h.snippet[:160]}")