v0.12: stabilize search routing and profile-driven phase4 pipeline
This commit is contained in:
kai
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply a model profile to agent definition files.
|
||||
|
||||
Supports:
|
||||
- OpenCode YAML frontmatter agents in .opencode/agents/*.md
|
||||
- Codex TOML agents in codex_adapter_templates/codex/agents/*.toml
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.lib.model_config import (
|
||||
ModelConfigError,
|
||||
parse_model_overrides,
|
||||
resolve_model_profile,
|
||||
)
|
||||
|
||||
OPENCODE_ROLE_TO_FILE = {
|
||||
"dr_plan": ".opencode/agents/dr-plan.md",
|
||||
"dr_pm": ".opencode/agents/dr-pm.md",
|
||||
"dr_searcher": ".opencode/agents/dr-searcher.md",
|
||||
"dr_analyst": ".opencode/agents/dr-analyst.md",
|
||||
"dr_verifier": ".opencode/agents/dr-verifier.md",
|
||||
"dr_chief_editor": ".opencode/agents/dr-chief-editor.md",
|
||||
"dr_editor_in_chief": ".opencode/agents/dr-editor-in-chief.md",
|
||||
"dr_reporter": ".opencode/agents/dr-reporter.md",
|
||||
}
|
||||
|
||||
CODEX_ROLE_TO_FILE = {
|
||||
"dr_plan": "codex_adapter_templates/codex/agents/dr-plan.toml",
|
||||
"dr_pm": "codex_adapter_templates/codex/agents/dr-pm.toml",
|
||||
"dr_searcher": "codex_adapter_templates/codex/agents/dr-searcher.toml",
|
||||
"dr_analyst": "codex_adapter_templates/codex/agents/dr-analyst.toml",
|
||||
"dr_verifier": "codex_adapter_templates/codex/agents/dr-verifier.toml",
|
||||
"dr_chief_editor": "codex_adapter_templates/codex/agents/dr-chief-editor.toml",
|
||||
"dr_editor_in_chief": "codex_adapter_templates/codex/agents/dr-editor-in-chief.toml",
|
||||
"dr_reporter": "codex_adapter_templates/codex/agents/dr-reporter.toml",
|
||||
}
|
||||
|
||||
|
||||
def replace_opencode_model(path: Path, model: str) -> bool:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
new_text, count = re.subn(r"(?m)^model:\s*.+$", f"model: {model}", text, count=1)
|
||||
if count == 0:
|
||||
raise SystemExit(f"failed to locate model field: {path}")
|
||||
if new_text == text:
|
||||
return False
|
||||
path.write_text(new_text, encoding="utf-8")
|
||||
return True
|
||||
|
||||
|
||||
def replace_codex_model(path: Path, model: str) -> bool:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
new_text, count = re.subn(r'(?m)^model\s*=\s*"[^"]+"$', f'model = "{model}"', text, count=1)
|
||||
if count == 0:
|
||||
raise SystemExit(f"failed to locate model field: {path}")
|
||||
if new_text == text:
|
||||
return False
|
||||
path.write_text(new_text, encoding="utf-8")
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Apply model profile to agent files")
|
||||
parser.add_argument("--profile", required=True, help="Profile name from configs/models.yaml")
|
||||
parser.add_argument("--target", choices=["opencode", "codex", "both"], default="both")
|
||||
parser.add_argument(
|
||||
"--model-override",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="ROLE=MODEL",
|
||||
help="Override one role model, repeatable",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
resolved = resolve_model_profile(
|
||||
profile=args.profile,
|
||||
overrides=parse_model_overrides(args.model_override),
|
||||
)
|
||||
except ModelConfigError as exc:
|
||||
raise SystemExit(f"model profile resolution failed: {exc}") from exc
|
||||
|
||||
roles = resolved["roles"]
|
||||
changed: list[str] = []
|
||||
|
||||
def apply_map(mapping: dict[str, str], mode: str) -> None:
|
||||
for role, rel_path in mapping.items():
|
||||
model = roles.get(role)
|
||||
if not model:
|
||||
continue
|
||||
file_path = REPO_ROOT / rel_path
|
||||
if not file_path.exists():
|
||||
continue
|
||||
if args.dry_run:
|
||||
changed.append(f"{mode}:{rel_path} -> {model}")
|
||||
continue
|
||||
did_change = replace_opencode_model(file_path, model) if mode == "opencode" else replace_codex_model(file_path, model)
|
||||
if did_change:
|
||||
changed.append(f"{mode}:{rel_path} -> {model}")
|
||||
|
||||
if args.target in ("opencode", "both"):
|
||||
apply_map(OPENCODE_ROLE_TO_FILE, "opencode")
|
||||
if args.target in ("codex", "both"):
|
||||
apply_map(CODEX_ROLE_TO_FILE, "codex")
|
||||
|
||||
print(f"Profile applied: {resolved['profile']}")
|
||||
print(f"Target: {args.target}")
|
||||
if changed:
|
||||
print("Updated:")
|
||||
for item in changed:
|
||||
print(f" - {item}")
|
||||
else:
|
||||
print("No files changed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+124
-40
@@ -16,6 +16,17 @@ from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.lib.model_config import (
|
||||
ModelConfigError,
|
||||
list_model_profiles,
|
||||
parse_model_overrides,
|
||||
resolve_model_profile,
|
||||
)
|
||||
|
||||
|
||||
PROJECTS_DIR = REPO_ROOT / "projects"
|
||||
CODEX_COMMANDS_DIR = REPO_ROOT / ".codex" / "commands"
|
||||
CODEX_COMMAND_TEMPLATES_DIR = REPO_ROOT / "codex_adapter_templates" / "codex" / "commands"
|
||||
@@ -152,48 +163,82 @@ def cmd_glossary(args: argparse.Namespace) -> int:
|
||||
|
||||
def cmd_finalize(args: argparse.Namespace) -> int:
|
||||
project_root = resolve_project(args.project)
|
||||
steps = [
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "translate.py"),
|
||||
str(project_root),
|
||||
"--workers",
|
||||
str(args.translate_workers),
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "build_glossary.py"),
|
||||
str(project_root),
|
||||
"--workers",
|
||||
str(args.glossary_workers),
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "apply_glossary.py"),
|
||||
str(project_root),
|
||||
"--input",
|
||||
"phase4/final_zh.md",
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "polish.py"),
|
||||
str(project_root),
|
||||
"--workers",
|
||||
str(args.polish_workers),
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "build_report.py"),
|
||||
str(project_root),
|
||||
],
|
||||
try:
|
||||
resolved = resolve_model_profile(
|
||||
profile=args.model_profile,
|
||||
overrides=parse_model_overrides(args.model_override),
|
||||
)
|
||||
except ModelConfigError as exc:
|
||||
raise SystemExit(f"model profile resolution failed: {exc}") from exc
|
||||
roles = resolved["roles"]
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "phase4_pipeline.py"),
|
||||
str(project_root),
|
||||
"--translate-workers",
|
||||
str(args.translate_workers),
|
||||
"--glossary-workers",
|
||||
str(args.glossary_workers),
|
||||
"--polish-workers",
|
||||
str(args.polish_workers),
|
||||
"--translate-model",
|
||||
roles.get("translate", "anthropic/claude-sonnet-4.6"),
|
||||
"--glossary-model",
|
||||
roles.get("glossary", "anthropic/claude-haiku-4.5"),
|
||||
"--polish-model",
|
||||
roles.get("polish", "anthropic/claude-sonnet-4.6"),
|
||||
"--glossary-mode",
|
||||
args.glossary_mode,
|
||||
]
|
||||
for step in steps:
|
||||
rc = run_cmd(step, dry_run=args.dry_run)
|
||||
if rc != 0:
|
||||
return rc
|
||||
if args.dry_run:
|
||||
cmd.append("--dry-run")
|
||||
return run_cmd(cmd, dry_run=False)
|
||||
|
||||
|
||||
def cmd_models(args: argparse.Namespace) -> int:
|
||||
if args.list:
|
||||
for name in list_model_profiles():
|
||||
print(name)
|
||||
return 0
|
||||
|
||||
try:
|
||||
resolved = resolve_model_profile(
|
||||
profile=args.profile,
|
||||
overrides=parse_model_overrides(args.model_override),
|
||||
)
|
||||
except ModelConfigError as exc:
|
||||
raise SystemExit(f"model profile resolution failed: {exc}") from exc
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(resolved, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
print(f"Profile: {resolved['profile']}")
|
||||
if resolved["description"]:
|
||||
print(f"Description: {resolved['description']}")
|
||||
print("Roles:")
|
||||
for role in sorted(resolved["roles"]):
|
||||
print(f" {role}: {resolved['roles'][role]}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_apply_models(args: argparse.Namespace) -> int:
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "apply_model_profile.py"),
|
||||
"--profile",
|
||||
args.profile,
|
||||
"--target",
|
||||
args.target,
|
||||
]
|
||||
for item in args.model_override:
|
||||
cmd += ["--model-override", item]
|
||||
if args.dry_run:
|
||||
cmd.append("--dry-run")
|
||||
return run_cmd(cmd, dry_run=False)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Deep Research platform-neutral CLI")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
@@ -219,12 +264,51 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
|
||||
finalize = sub.add_parser("finalize", help="Run Phase 4 deterministic pipeline")
|
||||
finalize.add_argument("project", help="Project slug or path")
|
||||
finalize.add_argument("--translate-workers", type=int, default=4)
|
||||
finalize.add_argument("--translate-workers", type=int, default=0)
|
||||
finalize.add_argument("--glossary-workers", type=int, default=4)
|
||||
finalize.add_argument("--polish-workers", type=int, default=4)
|
||||
finalize.add_argument("--polish-workers", type=int, default=0)
|
||||
finalize.add_argument(
|
||||
"--glossary-mode",
|
||||
choices=["off", "low-confidence", "full"],
|
||||
default="low-confidence",
|
||||
)
|
||||
finalize.add_argument("--model-profile", help="Model profile name from configs/models.yaml")
|
||||
finalize.add_argument(
|
||||
"--model-override",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="ROLE=MODEL",
|
||||
help="Override one role model, repeatable",
|
||||
)
|
||||
finalize.add_argument("--dry-run", action="store_true")
|
||||
finalize.set_defaults(func=cmd_finalize)
|
||||
|
||||
models = sub.add_parser("models", help="Resolve and print model profile")
|
||||
models.add_argument("--profile", help="Profile name from configs/models.yaml")
|
||||
models.add_argument("--list", action="store_true", help="List available profiles")
|
||||
models.add_argument(
|
||||
"--model-override",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="ROLE=MODEL",
|
||||
help="Override one role model, repeatable",
|
||||
)
|
||||
models.add_argument("--json", action="store_true", help="Emit JSON")
|
||||
models.set_defaults(func=cmd_models)
|
||||
|
||||
apply_models = sub.add_parser("apply-models", help="Apply profile to agent files")
|
||||
apply_models.add_argument("--profile", required=True, help="Profile name from configs/models.yaml")
|
||||
apply_models.add_argument("--target", choices=["opencode", "codex", "both"], default="both")
|
||||
apply_models.add_argument(
|
||||
"--model-override",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="ROLE=MODEL",
|
||||
help="Override one role model, repeatable",
|
||||
)
|
||||
apply_models.add_argument("--dry-run", action="store_true")
|
||||
apply_models.set_defaults(func=cmd_apply_models)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Native web grounding wrapper via ZenMux chat completions.
|
||||
|
||||
Use this when you need reproducible, model-native web search (grounding) and
|
||||
machine-readable citations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Grounded web query via ZenMux")
|
||||
parser.add_argument("query", help="Question or search prompt")
|
||||
parser.add_argument("--model", default="google/gemini-3.1-flash-lite-preview")
|
||||
parser.add_argument("--max-tokens", type=int, default=2400)
|
||||
parser.add_argument("--temperature", type=float, default=0.2)
|
||||
parser.add_argument("--json", action="store_true", help="Emit JSON envelope")
|
||||
parser.add_argument("--log-file", help="Optional JSONL call log path")
|
||||
parser.add_argument("--system", default=(
|
||||
"You are a research assistant. Use web grounding when helpful. "
|
||||
"Return concise facts with explicit source-backed statements."
|
||||
))
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
load_secrets()
|
||||
log_file = Path(args.log_file) if args.log_file else None
|
||||
|
||||
with ZenMuxClient(log_file=log_file) as client:
|
||||
result = client.chat_complete_with_meta(
|
||||
model=args.model,
|
||||
system=args.system,
|
||||
user=args.query,
|
||||
temperature=args.temperature,
|
||||
max_tokens=args.max_tokens,
|
||||
web_search=True,
|
||||
web_search_options={},
|
||||
tag="ground",
|
||||
)
|
||||
|
||||
if args.json:
|
||||
payload = {
|
||||
"query": args.query,
|
||||
"model": args.model,
|
||||
"content": result["content"],
|
||||
"citations": result["citations"],
|
||||
"usage": result["usage"],
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(result["content"])
|
||||
if result["citations"]:
|
||||
print("\nCitations:")
|
||||
for idx, url in enumerate(result["citations"], start=1):
|
||||
print(f"{idx}. {url}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Model profile loading and resolution utilities."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_MODEL_CONFIG = REPO_ROOT / "configs" / "models.yaml"
|
||||
LEGACY_MODEL_CONFIG = REPO_ROOT / "configs" / "model_profiles.yaml"
|
||||
|
||||
|
||||
class ModelConfigError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def load_model_config(path: Path | None = None) -> dict[str, Any]:
|
||||
cfg_path = path or DEFAULT_MODEL_CONFIG
|
||||
if not cfg_path.exists() and LEGACY_MODEL_CONFIG.exists():
|
||||
cfg_path = LEGACY_MODEL_CONFIG
|
||||
if not cfg_path.exists():
|
||||
raise ModelConfigError(f"model config not found: {cfg_path}")
|
||||
try:
|
||||
data = yaml.safe_load(cfg_path.read_text(encoding="utf-8")) or {}
|
||||
except Exception as exc:
|
||||
raise ModelConfigError(f"invalid YAML in {cfg_path}: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise ModelConfigError(f"invalid model config shape in {cfg_path}")
|
||||
return data
|
||||
|
||||
|
||||
def resolve_model_profile(
|
||||
*,
|
||||
profile: str | None = None,
|
||||
overrides: dict[str, str] | None = None,
|
||||
path: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
cfg = load_model_config(path)
|
||||
profiles = cfg.get("profiles") or {}
|
||||
defaults = cfg.get("defaults") or {}
|
||||
selected = profile or defaults.get("profile")
|
||||
if not selected:
|
||||
raise ModelConfigError("no model profile provided and no defaults.profile set")
|
||||
if selected not in profiles:
|
||||
raise ModelConfigError(f"unknown model profile: {selected}")
|
||||
|
||||
roles = dict((profiles[selected] or {}).get("roles") or {})
|
||||
if defaults.get("script_models"):
|
||||
for role, model in (defaults.get("script_models") or {}).items():
|
||||
roles.setdefault(role, model)
|
||||
for role, model in (overrides or {}).items():
|
||||
roles[role] = model
|
||||
|
||||
return {
|
||||
"profile": selected,
|
||||
"description": (profiles[selected] or {}).get("description", ""),
|
||||
"roles": roles,
|
||||
}
|
||||
|
||||
|
||||
def list_model_profiles(path: Path | None = None) -> list[str]:
|
||||
cfg = load_model_config(path)
|
||||
profiles = cfg.get("profiles") or {}
|
||||
return sorted(profiles.keys())
|
||||
|
||||
|
||||
def parse_model_overrides(items: list[str] | None) -> dict[str, str]:
|
||||
out: dict[str, str] = {}
|
||||
for item in items or []:
|
||||
if "=" not in item:
|
||||
raise ModelConfigError(f"invalid override '{item}', expected role=model")
|
||||
role, model = item.split("=", 1)
|
||||
role = role.strip()
|
||||
model = model.strip()
|
||||
if not role or not model:
|
||||
raise ModelConfigError(f"invalid override '{item}', expected role=model")
|
||||
out[role] = model
|
||||
return out
|
||||
@@ -1,11 +1,12 @@
|
||||
"""通用搜索客户端（Exa 优先，Tavily fallback）。
|
||||
"""通用搜索客户端（Serper / Exa / Tavily 路由）。
|
||||
|
||||
为 build_glossary.py 这类术语核查场景服务。
|
||||
|
||||
关键设计：
|
||||
- `trust_env=False` 绕开系统 socks 代理（Clash on macOS 配 socks5 时 httpx 会 TLS EOF）
|
||||
- Exa 优先：LinkedIn / 官网 / 百度百科返回质量最高
|
||||
- 遇到配额问题自动降级到 Tavily 或返回 empty
|
||||
- 专利 / Scholar / News 优先 Serper，保证 Google Patents / Google Scholar 路径被真正调用
|
||||
- 通用网页 Exa 优先，Tavily fallback
|
||||
- 遇到配额问题自动降级或返回 empty
|
||||
- 不做深度 crawl，只要摘要
|
||||
"""
|
||||
|
||||
@@ -125,10 +126,11 @@ class SearchClient:
|
||||
所有客户端都延迟导入 serper_client，避免没装 SERPAPI_KEY 时 import 炸。
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, *, strict_specialized: bool = True) -> None:
|
||||
self._exa: ExaClient | None = None
|
||||
self._tavily: TavilyClient | None = None
|
||||
self._serper = None # 惰性实例化
|
||||
self.strict_specialized = strict_specialized
|
||||
try:
|
||||
self._exa = ExaClient()
|
||||
except SearchError:
|
||||
@@ -137,10 +139,9 @@ class SearchClient:
|
||||
self._tavily = TavilyClient()
|
||||
except SearchError:
|
||||
pass
|
||||
if not (self._exa or self._tavily):
|
||||
raise SearchError(
|
||||
"neither EXA_API_KEY nor TAVILY_API_KEY available"
|
||||
)
|
||||
self._has_serper_key = bool(os.environ.get("SERPER_API_KEY") or os.environ.get("SERPAPI_KEY"))
|
||||
if not (self._exa or self._tavily or self._has_serper_key):
|
||||
raise SearchError("no search API key available: set SERPER_API_KEY, SERPAPI_KEY, EXA_API_KEY, or TAVILY_API_KEY")
|
||||
|
||||
def _get_serper(self):
|
||||
"""惰性创建 SerperClient。没 key 时返回 None。"""
|
||||
@@ -190,9 +191,12 @@ class SearchClient:
|
||||
try:
|
||||
hits = serper.patents(query, num_results=num_results)
|
||||
return [SearchHit(h.title, h.url, h.snippet) for h in hits]
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
if self.strict_specialized:
|
||||
raise SearchError(f"serper patents failed: {exc}") from exc
|
||||
# 降级：通用搜索加 site 限定
|
||||
if self.strict_specialized:
|
||||
raise SearchError("serper unavailable for patents route; refusing silent fallback")
|
||||
return self.search(f"site:patents.google.com {query}", num_results=num_results)
|
||||
|
||||
def scholar(
|
||||
@@ -215,8 +219,11 @@ class SearchClient:
|
||||
)
|
||||
for h in hits
|
||||
]
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
if self.strict_specialized:
|
||||
raise SearchError(f"serper scholar failed: {exc}") from exc
|
||||
if self.strict_specialized:
|
||||
raise SearchError("serper unavailable for scholar route; refusing silent fallback")
|
||||
return self.search(query, num_results=num_results)
|
||||
|
||||
def news(
|
||||
@@ -239,8 +246,11 @@ class SearchClient:
|
||||
)
|
||||
for h in hits
|
||||
]
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
if self.strict_specialized:
|
||||
raise SearchError(f"serper news failed: {exc}") from exc
|
||||
if self.strict_specialized:
|
||||
raise SearchError("serper unavailable for news route; refusing silent fallback")
|
||||
return self.search(query, num_results=num_results)
|
||||
|
||||
|
||||
|
||||
@@ -141,6 +141,8 @@ class ZenMuxClient:
|
||||
temperature: float = 0.3,
|
||||
max_tokens: int = 16000,
|
||||
extra_messages: list[dict[str, str]] | None = None,
|
||||
web_search: bool = False,
|
||||
web_search_options: dict[str, Any] | None = None,
|
||||
tag: str = "",
|
||||
) -> str:
|
||||
"""一次非流式对话补全。
|
||||
@@ -167,6 +169,8 @@ class ZenMuxClient:
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
if web_search:
|
||||
body["web_search_options"] = web_search_options or {}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
@@ -233,6 +237,116 @@ class ZenMuxClient:
|
||||
self.usage.failed_calls += 1
|
||||
raise ZenMuxError(f"max retries exhausted. last error: {last_error}")
|
||||
|
||||
def chat_complete_with_meta(
|
||||
self,
|
||||
model: str,
|
||||
system: str,
|
||||
user: str,
|
||||
*,
|
||||
temperature: float = 0.3,
|
||||
max_tokens: int = 16000,
|
||||
extra_messages: list[dict[str, str]] | None = None,
|
||||
web_search: bool = False,
|
||||
web_search_options: dict[str, Any] | None = None,
|
||||
tag: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""Return content plus metadata from one completion call."""
|
||||
messages: list[dict[str, str]] = [{"role": "system", "content": system}]
|
||||
if extra_messages:
|
||||
messages.extend(extra_messages)
|
||||
messages.append({"role": "user", "content": user})
|
||||
|
||||
body: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
if web_search:
|
||||
body["web_search_options"] = web_search_options or {}
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
url = f"{self.base_url}/chat/completions"
|
||||
|
||||
last_error = ""
|
||||
for attempt in range(MAX_RETRIES):
|
||||
t0 = time.time()
|
||||
try:
|
||||
resp = self._client.post(url, json=body, headers=headers)
|
||||
elapsed = time.time() - t0
|
||||
except httpx.RequestError as e:
|
||||
last_error = f"network: {e}"
|
||||
elapsed = time.time() - t0
|
||||
self._log({"tag": tag, "attempt": attempt, "elapsed": elapsed, "error": last_error})
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
|
||||
if resp.status_code != 200:
|
||||
retryable = resp.status_code in RETRYABLE_STATUSES
|
||||
last_error = f"HTTP {resp.status_code}: {resp.text[:500]}"
|
||||
self._log({
|
||||
"tag": tag,
|
||||
"attempt": attempt,
|
||||
"elapsed": round(elapsed, 2),
|
||||
"status": resp.status_code,
|
||||
"error": last_error,
|
||||
"retryable": retryable,
|
||||
})
|
||||
if not retryable:
|
||||
self.usage.failed_calls += 1
|
||||
raise ZenMuxError(last_error)
|
||||
sleep_for = min(60, (2 ** attempt) + (attempt * 0.5))
|
||||
time.sleep(sleep_for)
|
||||
continue
|
||||
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
raise ZenMuxError(f"invalid JSON from zenmux: {e}; body={resp.text[:500]}")
|
||||
|
||||
usage = data.get("usage", {}) or {}
|
||||
with self._usage_lock:
|
||||
self.usage.add(model, usage)
|
||||
|
||||
message = ((data.get("choices") or [{}])[0].get("message") or {})
|
||||
content = message.get("content") or ""
|
||||
annotations = message.get("annotations") or []
|
||||
urls: list[str] = []
|
||||
for ann in annotations:
|
||||
if not isinstance(ann, dict):
|
||||
continue
|
||||
citation = ann.get("url_citation") or {}
|
||||
url_item = citation.get("url")
|
||||
if url_item:
|
||||
urls.append(url_item)
|
||||
self._log({
|
||||
"tag": tag,
|
||||
"model": model,
|
||||
"attempt": attempt,
|
||||
"elapsed": round(elapsed, 2),
|
||||
"usage": usage,
|
||||
"out_chars": len(content),
|
||||
"status": 200,
|
||||
"web_search": web_search,
|
||||
"citations": len(urls),
|
||||
})
|
||||
if not content.strip():
|
||||
last_error = "empty content"
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
return {
|
||||
"content": content,
|
||||
"usage": usage,
|
||||
"citations": urls,
|
||||
"raw": data,
|
||||
}
|
||||
|
||||
self.usage.failed_calls += 1
|
||||
raise ZenMuxError(f"max retries exhausted. last error: {last_error}")
|
||||
|
||||
|
||||
def load_secrets(env_path: Path | None = None) -> None:
|
||||
"""从 secrets.env 把 key 塞到 os.environ，便于脚本直接运行。
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4 replacement pipeline orchestrator.
|
||||
|
||||
Default flow:
|
||||
1) translate.py
|
||||
2) optional glossary verification (low-confidence/full/off)
|
||||
3) apply_glossary.py
|
||||
4) polish.py
|
||||
5) build_report.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.lib.markdown_chunker import split_by_headers
|
||||
|
||||
|
||||
def resolve_project(arg: str) -> Path:
|
||||
p = Path(arg)
|
||||
if p.is_dir():
|
||||
return p.resolve()
|
||||
cand = REPO_ROOT / "projects" / arg
|
||||
if cand.is_dir():
|
||||
return cand.resolve()
|
||||
raise SystemExit(f"project not found: {arg}")
|
||||
|
||||
|
||||
def run_step(cmd: list[str], *, dry_run: bool) -> int:
|
||||
print("$ " + " ".join(cmd))
|
||||
if dry_run:
|
||||
return 0
|
||||
return subprocess.run(cmd, cwd=REPO_ROOT, check=False).returncode
|
||||
|
||||
|
||||
def infer_workers(source_file: Path, fallback: int, cap: int = 8) -> int:
|
||||
if not source_file.exists():
|
||||
return fallback
|
||||
text = source_file.read_text(encoding="utf-8")
|
||||
blocks = split_by_headers(text, max_level=2)
|
||||
if not blocks:
|
||||
return fallback
|
||||
cpu_cap = max(2, min(cap, (os.cpu_count() or 4)))
|
||||
suggested = max(2, min(cpu_cap, (len(blocks) + 5) // 6))
|
||||
return max(1, suggested if fallback <= 0 else min(max(fallback, 1), cpu_cap))
|
||||
|
||||
|
||||
def low_confidence_terms(glossary_path: Path) -> list[str]:
|
||||
if not glossary_path.exists():
|
||||
return []
|
||||
try:
|
||||
glossary = json.loads(glossary_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return []
|
||||
out: list[str] = []
|
||||
for term, entry in glossary.items():
|
||||
if not isinstance(entry, dict):
|
||||
out.append(term)
|
||||
continue
|
||||
conf = str(entry.get("confidence", "")).lower()
|
||||
verified = bool(entry.get("verified_at"))
|
||||
if conf != "high" or not verified:
|
||||
out.append(term)
|
||||
return sorted(set(out))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Phase 4 replacement pipeline")
|
||||
parser.add_argument("project", help="Project slug or full path")
|
||||
parser.add_argument("--translate-model", default="anthropic/claude-sonnet-4.6")
|
||||
parser.add_argument("--glossary-model", default="anthropic/claude-haiku-4.5")
|
||||
parser.add_argument("--polish-model", default="anthropic/claude-sonnet-4.6")
|
||||
parser.add_argument("--translate-workers", type=int, default=0, help="0 means auto")
|
||||
parser.add_argument("--glossary-workers", type=int, default=4)
|
||||
parser.add_argument("--polish-workers", type=int, default=0, help="0 means auto")
|
||||
parser.add_argument(
|
||||
"--glossary-mode",
|
||||
choices=["off", "low-confidence", "full"],
|
||||
default="low-confidence",
|
||||
help="off: skip, low-confidence: verify only low-confidence terms, full: verify all",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
project_root = resolve_project(args.project)
|
||||
phase4 = project_root / "phase4"
|
||||
src_en = phase4 / "final_en.md"
|
||||
if not src_en.exists():
|
||||
raise SystemExit(f"missing source: {src_en}")
|
||||
|
||||
tw = infer_workers(src_en, args.translate_workers)
|
||||
zh = phase4 / "final_zh.md"
|
||||
pw = infer_workers(zh if zh.exists() else src_en, args.polish_workers)
|
||||
|
||||
print(f"Project: {project_root.name}")
|
||||
print(f"Translate workers: {tw} | Polish workers: {pw}")
|
||||
print(f"Glossary mode: {args.glossary_mode}")
|
||||
print()
|
||||
|
||||
t0 = time.time()
|
||||
steps: list[list[str]] = [
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "translate.py"),
|
||||
str(project_root),
|
||||
"--workers",
|
||||
str(tw),
|
||||
"--model",
|
||||
args.translate_model,
|
||||
]
|
||||
]
|
||||
|
||||
if args.glossary_mode != "off":
|
||||
gcmd = [
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "build_glossary.py"),
|
||||
str(project_root),
|
||||
"--workers",
|
||||
str(args.glossary_workers),
|
||||
"--model",
|
||||
args.glossary_model,
|
||||
]
|
||||
if args.glossary_mode == "low-confidence":
|
||||
terms = low_confidence_terms(phase4 / "glossary.json")
|
||||
if terms:
|
||||
if len(terms) > 80:
|
||||
print(f"[info] low-confidence terms={len(terms)} is large; fallback to full glossary verify")
|
||||
else:
|
||||
gcmd += ["--only", ",".join(terms)]
|
||||
else:
|
||||
print("[info] no low-confidence glossary terms found; skipping glossary step")
|
||||
gcmd = []
|
||||
if gcmd:
|
||||
steps.append(gcmd)
|
||||
|
||||
steps.extend(
|
||||
[
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "apply_glossary.py"),
|
||||
str(project_root),
|
||||
"--input",
|
||||
"phase4/final_zh.md",
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "polish.py"),
|
||||
str(project_root),
|
||||
"--workers",
|
||||
str(pw),
|
||||
"--model",
|
||||
args.polish_model,
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "build_report.py"),
|
||||
str(project_root),
|
||||
],
|
||||
]
|
||||
)
|
||||
|
||||
for cmd in steps:
|
||||
rc = run_step(cmd, dry_run=args.dry_run)
|
||||
if rc != 0:
|
||||
return rc
|
||||
|
||||
print(f"\nPhase 4 pipeline done in {time.time() - t0:.1f}s")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Unified search gateway for Deep Research agents.
|
||||
|
||||
This script is the stable project-owned entrypoint that agents should call
|
||||
instead of vendor MCP tools. MCP search remains optional, while this gateway
|
||||
keeps routing behavior reproducible across OpenCode, Codex, and future
|
||||
adapters.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.lib.search_client import SearchClient, SearchError, SearchHit
|
||||
from scripts.lib.zenmux_client import load_secrets
|
||||
|
||||
|
||||
ROUTE_HELP = {
|
||||
"general": "Exa -> Tavily generic web discovery",
|
||||
"scholar": "Serper Scholar -> generic fallback",
|
||||
"patents": "Serper Google Patents -> site:patents.google.com fallback",
|
||||
"news": "Serper News -> generic fallback",
|
||||
}
|
||||
|
||||
PROFILE_ROUTES = {
|
||||
"biomed_literature": ["scholar", "general"],
|
||||
"patent_heavy": ["patents", "general"],
|
||||
"china_market": ["news", "general"],
|
||||
"investment": ["news", "general"],
|
||||
}
|
||||
|
||||
PROFILE_QUERY_PREFIX = {
|
||||
"china_market": "(China OR Chinese OR 中国 OR 国内)",
|
||||
}
|
||||
|
||||
|
||||
def search_route(client: SearchClient, route: str, query: str, args: argparse.Namespace) -> list[SearchHit]:
|
||||
if route == "general":
|
||||
return client.search(query, num_results=args.num_results)
|
||||
if route == "scholar":
|
||||
return client.scholar(query, num_results=args.num_results, year_low=args.year_low)
|
||||
if route == "patents":
|
||||
return client.patents(query, num_results=args.num_results)
|
||||
if route == "news":
|
||||
return client.news(query, num_results=args.num_results, time_range=args.time_range)
|
||||
raise SystemExit(f"unknown route: {route}")
|
||||
|
||||
|
||||
def emit_markdown(route_hits: list[tuple[str, list[SearchHit]]], query: str) -> None:
|
||||
print(f"# Search Results: {query}")
|
||||
for route, hits in route_hits:
|
||||
print()
|
||||
print(f"## Route: {route} ({ROUTE_HELP[route]})")
|
||||
if not hits:
|
||||
print("No results.")
|
||||
continue
|
||||
for i, hit in enumerate(hits, start=1):
|
||||
print(f"{i}. {hit.title or '(untitled)'}")
|
||||
print(f" - URL: {hit.url}")
|
||||
if hit.snippet:
|
||||
print(f" - Snippet: {hit.snippet}")
|
||||
|
||||
|
||||
def emit_json(route_hits: list[tuple[str, list[SearchHit]]], query: str) -> None:
|
||||
data = {
|
||||
"query": query,
|
||||
"routes": [
|
||||
{
|
||||
"route": route,
|
||||
"route_help": ROUTE_HELP[route],
|
||||
"results": [asdict(hit) for hit in hits],
|
||||
}
|
||||
for route, hits in route_hits
|
||||
],
|
||||
}
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
def emit_trace_markdown(route_trace: list[dict[str, str]]) -> None:
|
||||
print()
|
||||
print("## Route Trace")
|
||||
for item in route_trace:
|
||||
print(f"- {item['route']}: {item['status']} ({item['detail']})")
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Deep Research search gateway")
|
||||
parser.add_argument("query", help="Search query")
|
||||
parser.add_argument(
|
||||
"--route",
|
||||
choices=sorted(ROUTE_HELP),
|
||||
default="general",
|
||||
help="Single search route to run",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profile",
|
||||
choices=sorted(PROFILE_ROUTES),
|
||||
help="Run a strategy profile instead of a single route",
|
||||
)
|
||||
parser.add_argument("--num-results", type=int, default=10)
|
||||
parser.add_argument("--year-low", type=int, help="Lower year bound for scholar searches")
|
||||
parser.add_argument("--time-range", choices=["d", "w", "m", "y"], help="Serper news time range")
|
||||
parser.add_argument("--json", action="store_true", help="Emit JSON instead of Markdown")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Show planned routes without calling APIs")
|
||||
parser.add_argument(
|
||||
"--strict-specialized",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=True,
|
||||
help="Fail fast if scholar/news/patents cannot use Serper",
|
||||
)
|
||||
parser.add_argument("--trace", action="store_true", help="Include route execution trace")
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
load_secrets()
|
||||
|
||||
routes = PROFILE_ROUTES[args.profile] if args.profile else [args.route]
|
||||
query = args.query
|
||||
if args.profile in PROFILE_QUERY_PREFIX:
|
||||
query = f"{PROFILE_QUERY_PREFIX[args.profile]} {query}"
|
||||
|
||||
if args.dry_run:
|
||||
for route in routes:
|
||||
print(f"{route}: {ROUTE_HELP[route]}")
|
||||
if query != args.query:
|
||||
print(f"query_rewritten: {query}")
|
||||
return 0
|
||||
|
||||
try:
|
||||
with SearchClient(strict_specialized=args.strict_specialized) as client:
|
||||
route_hits = []
|
||||
route_trace: list[dict[str, str]] = []
|
||||
for route in routes:
|
||||
try:
|
||||
hits = search_route(client, route, query, args)
|
||||
route_hits.append((route, hits))
|
||||
route_trace.append({"route": route, "status": "ok", "detail": f"hits={len(hits)}"})
|
||||
except SearchError as exc:
|
||||
route_hits.append((route, []))
|
||||
route_trace.append({"route": route, "status": "failed", "detail": str(exc)})
|
||||
if route != "general":
|
||||
continue
|
||||
raise
|
||||
except SearchError as exc:
|
||||
raise SystemExit(f"search failed: {exc}") from exc
|
||||
|
||||
if args.json:
|
||||
data = {
|
||||
"query": query,
|
||||
"original_query": args.query,
|
||||
"strict_specialized": args.strict_specialized,
|
||||
"routes": [
|
||||
{
|
||||
"route": route,
|
||||
"route_help": ROUTE_HELP[route],
|
||||
"results": [asdict(hit) for hit in hits],
|
||||
}
|
||||
for route, hits in route_hits
|
||||
],
|
||||
"trace": route_trace if args.trace else [],
|
||||
}
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
emit_markdown(route_hits, query)
|
||||
if args.trace:
|
||||
emit_trace_markdown(route_trace)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sprint 5 regression checks for v0.12 changes.
|
||||
|
||||
Checks are non-destructive and default to dry-run behavior.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def run(cmd: list[str]) -> tuple[int, str]:
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
cwd=REPO_ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
out = (proc.stdout or "") + (proc.stderr or "")
|
||||
return proc.returncode, out
|
||||
|
||||
|
||||
def check(name: str, cmd: list[str], must_contain: list[str] | None = None) -> bool:
|
||||
print(f"[check] {name}")
|
||||
print(" $ " + " ".join(cmd))
|
||||
rc, out = run(cmd)
|
||||
if rc != 0:
|
||||
print(f" FAIL: exit={rc}")
|
||||
if out.strip():
|
||||
print(" output:")
|
||||
print(" " + out.strip().replace("\n", "\n "))
|
||||
return False
|
||||
for token in must_contain or []:
|
||||
if token not in out:
|
||||
print(f" FAIL: missing token '{token}'")
|
||||
return False
|
||||
print(" PASS")
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run Sprint 5 regression checks")
|
||||
parser.add_argument("project", help="Project slug or path for finalize dry-run")
|
||||
args = parser.parse_args()
|
||||
|
||||
checks = [
|
||||
(
|
||||
"model profiles list",
|
||||
["uv", "run", "python", "scripts/dr.py", "models", "--list"],
|
||||
["medium", "premium", "simple"],
|
||||
),
|
||||
(
|
||||
"search gateway dry-run",
|
||||
[
|
||||
"uv",
|
||||
"run",
|
||||
"python",
|
||||
"scripts/search.py",
|
||||
"GLP-1 obesity",
|
||||
"--profile",
|
||||
"china_market",
|
||||
"--dry-run",
|
||||
],
|
||||
["news:", "general:", "query_rewritten:"],
|
||||
),
|
||||
(
|
||||
"phase4 finalize dry-run",
|
||||
[
|
||||
"uv",
|
||||
"run",
|
||||
"python",
|
||||
"scripts/dr.py",
|
||||
"finalize",
|
||||
args.project,
|
||||
"--model-profile",
|
||||
"medium",
|
||||
"--dry-run",
|
||||
],
|
||||
["Phase 4 pipeline done"],
|
||||
),
|
||||
]
|
||||
|
||||
ok = True
|
||||
for name, cmd, tokens in checks:
|
||||
ok = check(name, cmd, tokens) and ok
|
||||
|
||||
if not ok:
|
||||
print("\nSprint 5 regression: FAILED")
|
||||
return 1
|
||||
print("\nSprint 5 regression: PASSED")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user