v0.12: stabilize search routing and profile-driven phase4 pipeline

scripts/lib/model_config.py

@@ -0,0 +1,81 @@
+"""Model profile loading and resolution utilities."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+DEFAULT_MODEL_CONFIG = REPO_ROOT / "configs" / "models.yaml"
+LEGACY_MODEL_CONFIG = REPO_ROOT / "configs" / "model_profiles.yaml"
+
+
+class ModelConfigError(RuntimeError):
+    pass
+
+
+def load_model_config(path: Path | None = None) -> dict[str, Any]:
+    cfg_path = path or DEFAULT_MODEL_CONFIG
+    if not cfg_path.exists() and LEGACY_MODEL_CONFIG.exists():
+        cfg_path = LEGACY_MODEL_CONFIG
+    if not cfg_path.exists():
+        raise ModelConfigError(f"model config not found: {cfg_path}")
+    try:
+        data = yaml.safe_load(cfg_path.read_text(encoding="utf-8")) or {}
+    except Exception as exc:
+        raise ModelConfigError(f"invalid YAML in {cfg_path}: {exc}") from exc
+    if not isinstance(data, dict):
+        raise ModelConfigError(f"invalid model config shape in {cfg_path}")
+    return data
+
+
+def resolve_model_profile(
+    *,
+    profile: str | None = None,
+    overrides: dict[str, str] | None = None,
+    path: Path | None = None,
+) -> dict[str, Any]:
+    cfg = load_model_config(path)
+    profiles = cfg.get("profiles") or {}
+    defaults = cfg.get("defaults") or {}
+    selected = profile or defaults.get("profile")
+    if not selected:
+        raise ModelConfigError("no model profile provided and no defaults.profile set")
+    if selected not in profiles:
+        raise ModelConfigError(f"unknown model profile: {selected}")
+
+    roles = dict((profiles[selected] or {}).get("roles") or {})
+    if defaults.get("script_models"):
+        for role, model in (defaults.get("script_models") or {}).items():
+            roles.setdefault(role, model)
+    for role, model in (overrides or {}).items():
+        roles[role] = model
+
+    return {
+        "profile": selected,
+        "description": (profiles[selected] or {}).get("description", ""),
+        "roles": roles,
+    }
+
+
+def list_model_profiles(path: Path | None = None) -> list[str]:
+    cfg = load_model_config(path)
+    profiles = cfg.get("profiles") or {}
+    return sorted(profiles.keys())
+
+
+def parse_model_overrides(items: list[str] | None) -> dict[str, str]:
+    out: dict[str, str] = {}
+    for item in items or []:
+        if "=" not in item:
+            raise ModelConfigError(f"invalid override '{item}', expected role=model")
+        role, model = item.split("=", 1)
+        role = role.strip()
+        model = model.strip()
+        if not role or not model:
+            raise ModelConfigError(f"invalid override '{item}', expected role=model")
+        out[role] = model
+    return out

scripts/lib/search_client.py

@@ -1,11 +1,12 @@
-"""通用搜索客户端（Exa 优先，Tavily fallback）。
+"""通用搜索客户端（Serper / Exa / Tavily 路由）。
 
 为 build_glossary.py 这类术语核查场景服务。
 
 关键设计：
 - `trust_env=False` 绕开系统 socks 代理（Clash on macOS 配 socks5 时 httpx 会 TLS EOF）
-- Exa 优先：LinkedIn / 官网 / 百度百科返回质量最高
-- 遇到配额问题自动降级到 Tavily 或返回 empty
+- 专利 / Scholar / News 优先 Serper，保证 Google Patents / Google Scholar 路径被真正调用
+- 通用网页 Exa 优先，Tavily fallback
+- 遇到配额问题自动降级或返回 empty
 - 不做深度 crawl，只要摘要
 """
 
@@ -125,10 +126,11 @@ class SearchClient:
     所有客户端都延迟导入 serper_client，避免没装 SERPAPI_KEY 时 import 炸。
     """
 
-    def __init__(self) -> None:
+    def __init__(self, *, strict_specialized: bool = True) -> None:
         self._exa: ExaClient | None = None
         self._tavily: TavilyClient | None = None
         self._serper = None  # 惰性实例化
+        self.strict_specialized = strict_specialized
         try:
             self._exa = ExaClient()
         except SearchError:
@@ -137,10 +139,9 @@ class SearchClient:
             self._tavily = TavilyClient()
         except SearchError:
             pass
-        if not (self._exa or self._tavily):
-            raise SearchError(
-                "neither EXA_API_KEY nor TAVILY_API_KEY available"
-            )
+        self._has_serper_key = bool(os.environ.get("SERPER_API_KEY") or os.environ.get("SERPAPI_KEY"))
+        if not (self._exa or self._tavily or self._has_serper_key):
+            raise SearchError("no search API key available: set SERPER_API_KEY, SERPAPI_KEY, EXA_API_KEY, or TAVILY_API_KEY")
 
     def _get_serper(self):
         """惰性创建 SerperClient。没 key 时返回 None。"""
@@ -190,9 +191,12 @@ class SearchClient:
             try:
                 hits = serper.patents(query, num_results=num_results)
                 return [SearchHit(h.title, h.url, h.snippet) for h in hits]
-            except Exception:
-                pass
+            except Exception as exc:
+                if self.strict_specialized:
+                    raise SearchError(f"serper patents failed: {exc}") from exc
         # 降级：通用搜索加 site 限定
+        if self.strict_specialized:
+            raise SearchError("serper unavailable for patents route; refusing silent fallback")
         return self.search(f"site:patents.google.com {query}", num_results=num_results)
 
     def scholar(
@@ -215,8 +219,11 @@ class SearchClient:
                     )
                     for h in hits
                 ]
-            except Exception:
-                pass
+            except Exception as exc:
+                if self.strict_specialized:
+                    raise SearchError(f"serper scholar failed: {exc}") from exc
+        if self.strict_specialized:
+            raise SearchError("serper unavailable for scholar route; refusing silent fallback")
         return self.search(query, num_results=num_results)
 
     def news(
@@ -239,8 +246,11 @@ class SearchClient:
                     )
                     for h in hits
                 ]
-            except Exception:
-                pass
+            except Exception as exc:
+                if self.strict_specialized:
+                    raise SearchError(f"serper news failed: {exc}") from exc
+        if self.strict_specialized:
+            raise SearchError("serper unavailable for news route; refusing silent fallback")
         return self.search(query, num_results=num_results)
 
 

scripts/lib/zenmux_client.py

@@ -141,6 +141,8 @@ class ZenMuxClient:
         temperature: float = 0.3,
         max_tokens: int = 16000,
         extra_messages: list[dict[str, str]] | None = None,
+        web_search: bool = False,
+        web_search_options: dict[str, Any] | None = None,
         tag: str = "",
     ) -> str:
         """一次非流式对话补全。
@@ -167,6 +169,8 @@ class ZenMuxClient:
             "temperature": temperature,
             "max_tokens": max_tokens,
         }
+        if web_search:
+            body["web_search_options"] = web_search_options or {}
         headers = {
             "Authorization": f"Bearer {self.api_key}",
             "Content-Type": "application/json",
@@ -233,6 +237,116 @@ class ZenMuxClient:
         self.usage.failed_calls += 1
         raise ZenMuxError(f"max retries exhausted. last error: {last_error}")
 
+    def chat_complete_with_meta(
+        self,
+        model: str,
+        system: str,
+        user: str,
+        *,
+        temperature: float = 0.3,
+        max_tokens: int = 16000,
+        extra_messages: list[dict[str, str]] | None = None,
+        web_search: bool = False,
+        web_search_options: dict[str, Any] | None = None,
+        tag: str = "",
+    ) -> dict[str, Any]:
+        """Return content plus metadata from one completion call."""
+        messages: list[dict[str, str]] = [{"role": "system", "content": system}]
+        if extra_messages:
+            messages.extend(extra_messages)
+        messages.append({"role": "user", "content": user})
+
+        body: dict[str, Any] = {
+            "model": model,
+            "messages": messages,
+            "temperature": temperature,
+            "max_tokens": max_tokens,
+        }
+        if web_search:
+            body["web_search_options"] = web_search_options or {}
+
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+        url = f"{self.base_url}/chat/completions"
+
+        last_error = ""
+        for attempt in range(MAX_RETRIES):
+            t0 = time.time()
+            try:
+                resp = self._client.post(url, json=body, headers=headers)
+                elapsed = time.time() - t0
+            except httpx.RequestError as e:
+                last_error = f"network: {e}"
+                elapsed = time.time() - t0
+                self._log({"tag": tag, "attempt": attempt, "elapsed": elapsed, "error": last_error})
+                time.sleep(2 ** attempt)
+                continue
+
+            if resp.status_code != 200:
+                retryable = resp.status_code in RETRYABLE_STATUSES
+                last_error = f"HTTP {resp.status_code}: {resp.text[:500]}"
+                self._log({
+                    "tag": tag,
+                    "attempt": attempt,
+                    "elapsed": round(elapsed, 2),
+                    "status": resp.status_code,
+                    "error": last_error,
+                    "retryable": retryable,
+                })
+                if not retryable:
+                    self.usage.failed_calls += 1
+                    raise ZenMuxError(last_error)
+                sleep_for = min(60, (2 ** attempt) + (attempt * 0.5))
+                time.sleep(sleep_for)
+                continue
+
+            try:
+                data = resp.json()
+            except Exception as e:
+                raise ZenMuxError(f"invalid JSON from zenmux: {e}; body={resp.text[:500]}")
+
+            usage = data.get("usage", {}) or {}
+            with self._usage_lock:
+                self.usage.add(model, usage)
+
+            message = ((data.get("choices") or [{}])[0].get("message") or {})
+            content = message.get("content") or ""
+            annotations = message.get("annotations") or []
+            urls: list[str] = []
+            for ann in annotations:
+                if not isinstance(ann, dict):
+                    continue
+                citation = ann.get("url_citation") or {}
+                url_item = citation.get("url")
+                if url_item:
+                    urls.append(url_item)
+            self._log({
+                "tag": tag,
+                "model": model,
+                "attempt": attempt,
+                "elapsed": round(elapsed, 2),
+                "usage": usage,
+                "out_chars": len(content),
+                "status": 200,
+                "web_search": web_search,
+                "citations": len(urls),
+            })
+            if not content.strip():
+                last_error = "empty content"
+                time.sleep(2 ** attempt)
+                continue
+            return {
+                "content": content,
+                "usage": usage,
+                "citations": urls,
+                "raw": data,
+            }
+
+        self.usage.failed_calls += 1
+        raise ZenMuxError(f"max retries exhausted. last error: {last_error}")
+
 
 def load_secrets(env_path: Path | None = None) -> None:
     """从 secrets.env 把 key 塞到 os.environ，便于脚本直接运行。