v0.20.7 restructure source and platform workspaces

This commit is contained in:
Deep Research System
2026-05-07 21:13:21 +08:00
parent b3050260b2
commit a4adeb591f
172 changed files with 214 additions and 543 deletions
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env bash
# Activate Deep Research environment (venv + secrets)
# Usage: cd <project-root> && source scripts/activate.sh
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
echo "ERROR: use 'source', not 'bash':"
echo " source scripts/activate.sh"
exit 1
fi
# Project root = directory containing pyproject.toml
# Walk up from $PWD until we find it (or give up at /)
_find_root() {
local dir="${PWD}"
while [[ "${dir}" != "/" ]]; do
[[ -f "${dir}/pyproject.toml" ]] && echo "${dir}" && return 0
dir="$(dirname "${dir}")"
done
return 1
}
_DR_ROOT="$(_find_root 2>/dev/null)"
if [[ -z "${_DR_ROOT}" ]]; then
echo "ERROR: pyproject.toml not found in ${PWD} or any parent."
echo " cd to the project root first, then: source scripts/activate.sh"
return 1
fi
# activate venv
if [[ -f "${_DR_ROOT}/.venv/bin/activate" ]]; then
source "${_DR_ROOT}/.venv/bin/activate"
echo "venv: active ($(python --version 2>&1))"
else
echo "venv: not found - run: bash ${_DR_ROOT}/scripts/setup.sh"
fi
# load secrets
if [[ -f "${_DR_ROOT}/secrets.env" ]]; then
set -a
source "${_DR_ROOT}/secrets.env"
set +a
if [[ -n "${ZENMUX_API_KEY:-}" ]]; then
echo "secrets: loaded (ZENMUX_API_KEY=${ZENMUX_API_KEY:0:12}...)"
else
echo "secrets: loaded but ZENMUX_API_KEY is empty"
fi
else
echo "secrets: missing - run: cp secrets.env.example secrets.env"
fi
# npm proxy fix: unset empty npm_config_proxy so npm inherits system http_proxy
# without this, opencode crashes with "proxy.url must be a non-empty string"
# when dynamically loading npm packages like @ai-sdk/anthropic
if [[ -z "${npm_config_proxy:-}" ]]; then
unset npm_config_proxy
fi
if [[ -z "${npm_config_https_proxy:-}" ]]; then
unset npm_config_https_proxy
fi
unset -f _find_root
unset _DR_ROOT
+260
View File
@@ -0,0 +1,260 @@
#!/usr/bin/env python3
"""根据 glossary 的核查结果,在 final_zh.md 上做精确的文本替换。
原理:
- build_glossary.py 会给每条术语标 `zh`(正确中文)和 `issue`(有发现问题)
- 本脚本扫描所有 `issue` 非空且 `confidence in {high, medium}` 的条目
- 对这些条目,在正文中把「当前错误译名」替换为「正确译名」
- 同时处理英文拼写错误(例如 Maywavee → Mabwell
规则:
- 安全第一:只做字面替换,不做上下文改写
- 明确可见:每一处替换都打印出来,方便 diff
- 幂等:多次跑结果一致
- 支持 --dry-run 预览
用法:
uv run python scripts/apply_glossary.py <project_slug>
uv run python scripts/apply_glossary.py <project_slug> --dry-run
uv run python scripts/apply_glossary.py <project_slug> --min-confidence high
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import dataclass
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
@dataclass
class Correction:
"""一条要应用的修正。"""
kind: str # "en" (英文拼写) / "zh" (中文译名)
wrong: str # 当前文中的错误写法
correct: str # 正确写法
term_key: str # glossary 里这条的 key(人工 debug 用)
confidence: str # high / medium / low
reason: str # 为什么要改(从 issue 字段提取)
_ISSUE_WRONG_EN_RE = re.compile(
r"(?:'|\"|term\s*)([A-Z][A-Za-z0-9_\- ]{2,40})['\"]*\s*(?:为|是)[^。]*?"
r"(?:拼写错误|拼写有误|应为|正确英文名为|正确为|正确拼写为)",
)
_CORRECT_EN_RE = re.compile(
r"(?:正确英文名为|正确为|正确拼写为|应为|正确写法为)\s*['\"]?([A-Z][A-Za-z0-9_\- ]{2,40})['\"]?",
)
_WRONG_ZH_RE = re.compile(
r"current_zh\s*['\"]?([\u4e00-\u9fff][^'\"。,;]{1,40})['\"]?\s*(?:为|是|存在)[^。]*?(?:错误|错译|误译|不准确|应为|应该)",
)
def resolve_project(arg: str) -> Path:
p = Path(arg)
if p.is_dir():
return p
cand = Path.cwd() / "projects" / arg
if cand.is_dir():
return cand
raise SystemExit(f"project not found: {arg}")
def load_glossary(path: Path) -> dict:
return json.loads(path.read_text(encoding="utf-8"))
# 中文替换的保守性:只对"公司名、机构名、产品名"类这种具体化的词做替换
# 规避常见缩写词在不同上下文有不同含义的情况(如 PDE = 磷酸二酯酶 OR 允许日暴露量)
_AMBIGUOUS_ABBREVS = {
"PDE", "ASGPR", "BLA", "IND", "NDA", "CMC", "API", "QC", "QA",
"ADC", "CRO", "CDMO", "CMO", "GMP", "PK", "PD", "TRL",
"FDA", "EMA", "NMPA", "ICH", "WHO", "CFDA", "CDE",
"LNP", "RNP", "AAV", "mRNA", "RNAi", "siRNA", "ASO",
"DNA", "RNA", "cDNA", "dsRNA", "ssRNA",
"OTP", "HCV", "HBV", "HPV", "HIV",
}
def _looks_like_proper_noun(term_key: str) -> bool:
"""判断这条术语是否是"专有名词"(公司/机构/产品/药物名)。"""
# 全大写 2-4 字符缩写 → 视作通用缩写,跳过(歧义风险高)
if term_key.upper() == term_key and 2 <= len(term_key) <= 5:
return False
if term_key in _AMBIGUOUS_ABBREVS:
return False
# 其他情况:视作专有名词
return True
def extract_corrections(glossary: dict, min_conf: str) -> list[Correction]:
"""从 glossary 抽出可应用的修正。
保守策略:
- 英文拼写错误(term_key 本身错):高信度直接修
- 中文译名错误(issue 里说 current_zh 错):仅对"专有名词"类(公司/产品/机构)修
- 缩写/通用术语(PDE/ASGPR 等):不自动修,避免上下文歧义
"""
rank = {"high": 3, "medium": 2, "low": 1}
threshold = rank.get(min_conf, 2)
corrections: list[Correction] = []
for term_key, entry in glossary.items():
if not isinstance(entry, dict):
continue
conf = entry.get("confidence", "low")
if rank.get(conf, 0) < threshold:
continue
issue = entry.get("issue", "")
zh = entry.get("zh", "")
en_full = entry.get("en_full", "")
if not issue:
continue
# 1. 英文拼写错误:key 本身错了,en_full 是对的
if (
en_full
and term_key != en_full
and re.search(r"拼写错误|拼写有误|spelled|should be", issue, re.IGNORECASE)
and term_key[0].isupper() # 通常是公司/产品名
):
# 排除太短的(容易误伤)或包含空格的原始 key
if len(term_key) >= 4 and len(en_full) >= 4:
corrections.append(
Correction(
kind="en",
wrong=term_key,
correct=en_full.split("(")[0].strip(), # 去掉括号内的法人全称
term_key=term_key,
confidence=conf,
reason=issue[:200],
)
)
# 2. 中文译名错误:只对"专有名词"修(排除缩写/通用术语的歧义风险)
if not _looks_like_proper_noun(term_key):
continue
m = _WRONG_ZH_RE.search(issue)
if m and zh:
wrong_zh = m.group(1).strip()
if wrong_zh != zh and len(wrong_zh) >= 2:
corrections.append(
Correction(
kind="zh",
wrong=wrong_zh,
correct=zh,
term_key=term_key,
confidence=conf,
reason=issue[:200],
)
)
return corrections
def apply_corrections(text: str, corrections: list[Correction]) -> tuple[str, list[tuple[Correction, int]]]:
"""返回 (新文本, [(correction, 替换次数)])。"""
results: list[tuple[Correction, int]] = []
new_text = text
for c in corrections:
# 英文术语用 \b 边界;中文直接替换
if c.kind == "en":
pattern = r"\b" + re.escape(c.wrong) + r"\b"
new_text, count = re.subn(pattern, c.correct, new_text)
else:
count = new_text.count(c.wrong)
if count:
new_text = new_text.replace(c.wrong, c.correct)
results.append((c, count))
return new_text, results
def main() -> int:
parser = argparse.ArgumentParser(description="用 glossary 修正正文术语")
parser.add_argument("project", help="项目 slug 或路径")
parser.add_argument(
"--input", default="phase4/final_zh.md",
help="待修正的 Markdown",
)
parser.add_argument(
"--output", default=None,
help="输出路径(默认原地覆盖)",
)
parser.add_argument(
"--glossary", default="phase4/glossary.json",
help="glossary.json 路径",
)
parser.add_argument(
"--min-confidence",
choices=["high", "medium", "low"],
default="medium",
help="只应用 >= 此置信度的修正(默认 medium)",
)
parser.add_argument(
"--dry-run", action="store_true",
help="只打印会发生的修改,不实际写入",
)
args = parser.parse_args()
project_root = resolve_project(args.project)
input_path = project_root / args.input
output_path = project_root / (args.output or args.input)
glossary_path = project_root / args.glossary
if not input_path.exists():
raise SystemExit(f"输入文件不存在:{input_path}")
if not glossary_path.exists():
raise SystemExit(f"glossary 不存在:{glossary_path}")
glossary = load_glossary(glossary_path)
corrections = extract_corrections(glossary, args.min_confidence)
print(f"Project: {project_root.name}")
print(f"Input: {input_path.relative_to(project_root)}")
print(f"Output: {output_path.relative_to(project_root)}")
print(f"Glossary: {glossary_path.relative_to(project_root)} ({len(glossary)} terms)")
print(f"Min confidence: {args.min_confidence}")
print(f"候选修正: {len(corrections)}")
print()
if not corrections:
print("没有需要应用的修正。")
return 0
original = input_path.read_text(encoding="utf-8")
new_text, results = apply_corrections(original, corrections)
applied = [(c, n) for c, n in results if n > 0]
skipped = [(c, n) for c, n in results if n == 0]
print(f"=== 已应用 {len(applied)} 条修正(共替换 {sum(n for _, n in applied)} 处)===")
for c, n in sorted(applied, key=lambda x: -x[1]):
marker = "EN" if c.kind == "en" else "ZH"
print(f" [{marker}][{c.confidence}] {c.wrong!r}{c.correct!r} (×{n})")
print(f" 原因: {c.reason[:120]}")
if skipped:
print(f"\n=== 跳过 {len(skipped)} 条(正文未出现该错误写法)===")
for c, _ in skipped[:10]:
marker = "EN" if c.kind == "en" else "ZH"
print(f" [{marker}] {c.wrong!r}{c.correct!r} (0 hit)")
if len(skipped) > 10:
print(f" …还有 {len(skipped) - 10}")
if args.dry_run:
print("\n(--dry-run:未写入)")
return 0
output_path.write_text(new_text, encoding="utf-8")
delta = len(new_text) - len(original)
print(f"\n✓ 已写入 {output_path.relative_to(project_root)} (字符变化 {delta:+d})")
return 0
if __name__ == "__main__":
sys.exit(main())
+129
View File
@@ -0,0 +1,129 @@
#!/usr/bin/env python3
"""Apply a model profile to agent definition files.
Supports:
- OpenCode YAML frontmatter agents in .opencode/agents/*.md
- Codex TOML agents in codex_adapter_templates/codex/agents/*.toml
"""
from __future__ import annotations
import argparse
import re
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from scripts.lib.model_config import (
ModelConfigError,
parse_model_overrides,
resolve_model_profile,
)
OPENCODE_ROLE_TO_FILE = {
"dr_plan": ".opencode/agents/dr-plan.md",
"dr_pm": ".opencode/agents/dr-pm.md",
"dr_searcher": ".opencode/agents/dr-searcher.md",
"dr_analyst": ".opencode/agents/dr-analyst.md",
"dr_verifier": ".opencode/agents/dr-verifier.md",
"dr_chief_editor": ".opencode/agents/dr-chief-editor.md",
"dr_editor_in_chief": ".opencode/agents/dr-editor-in-chief.md",
"dr_reporter": ".opencode/agents/dr-reporter.md",
}
CODEX_ROLE_TO_FILE = {
"dr_plan": "codex_adapter_templates/codex/agents/dr-plan.toml",
"dr_pm": "codex_adapter_templates/codex/agents/dr-pm.toml",
"dr_searcher": "codex_adapter_templates/codex/agents/dr-searcher.toml",
"dr_analyst": "codex_adapter_templates/codex/agents/dr-analyst.toml",
"dr_verifier": "codex_adapter_templates/codex/agents/dr-verifier.toml",
"dr_chief_editor": "codex_adapter_templates/codex/agents/dr-chief-editor.toml",
"dr_editor_in_chief": "codex_adapter_templates/codex/agents/dr-editor-in-chief.toml",
"dr_reporter": "codex_adapter_templates/codex/agents/dr-reporter.toml",
}
def replace_opencode_model(path: Path, model: str) -> bool:
text = path.read_text(encoding="utf-8")
new_text, count = re.subn(r"(?m)^model:\s*.+$", f"model: {model}", text, count=1)
if count == 0:
raise SystemExit(f"failed to locate model field: {path}")
if new_text == text:
return False
path.write_text(new_text, encoding="utf-8")
return True
def replace_codex_model(path: Path, model: str) -> bool:
text = path.read_text(encoding="utf-8")
new_text, count = re.subn(r'(?m)^model\s*=\s*"[^"]+"$', f'model = "{model}"', text, count=1)
if count == 0:
raise SystemExit(f"failed to locate model field: {path}")
if new_text == text:
return False
path.write_text(new_text, encoding="utf-8")
return True
def main() -> int:
parser = argparse.ArgumentParser(description="Apply model profile to agent files")
parser.add_argument("--profile", required=True, help="Profile name from configs/models.yaml")
parser.add_argument("--target", choices=["opencode", "codex", "both"], default="both")
parser.add_argument(
"--model-override",
action="append",
default=[],
metavar="ROLE=MODEL",
help="Override one role model, repeatable",
)
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
try:
resolved = resolve_model_profile(
profile=args.profile,
overrides=parse_model_overrides(args.model_override),
)
except ModelConfigError as exc:
raise SystemExit(f"model profile resolution failed: {exc}") from exc
roles = resolved["roles"]
changed: list[str] = []
def apply_map(mapping: dict[str, str], mode: str) -> None:
for role, rel_path in mapping.items():
model = roles.get(role)
if not model:
continue
file_path = REPO_ROOT / rel_path
if not file_path.exists():
continue
if args.dry_run:
changed.append(f"{mode}:{rel_path} -> {model}")
continue
did_change = replace_opencode_model(file_path, model) if mode == "opencode" else replace_codex_model(file_path, model)
if did_change:
changed.append(f"{mode}:{rel_path} -> {model}")
if args.target in ("opencode", "both"):
apply_map(OPENCODE_ROLE_TO_FILE, "opencode")
if args.target in ("codex", "both"):
apply_map(CODEX_ROLE_TO_FILE, "codex")
print(f"Profile applied: {resolved['profile']}")
print(f"Target: {args.target}")
if changed:
print("Updated:")
for item in changed:
print(f" - {item}")
else:
print("No files changed.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+332
View File
@@ -0,0 +1,332 @@
#!/usr/bin/env python3
"""双语术语表事实核查脚本。
输入:
- <project>/phase4/glossary.json(来自 translate.py 累积的初版术语表)
- 可选:--extra terms.txt(每行一个英文术语,补充进来一起核查)
流程(每个术语独立可并行):
1. 用 SearchClientTavily > Exa > Brave)搜一次(query = "<term> <domain hint>"
2. 把 top 3-5 snippet 喂给 Haiku,让模型返回 {zh, en_full, confidence, issue}
3. 合并回 glossary,字段扩展:
{
"Mabwell": {
"zh": "迈威生物",
"en_full": "Mabwell (Shanghai) Bioscience Co., Ltd.",
"confidence": "high",
"issue": "...",
"verified_at": "2026-04-22",
"sources": ["https://mabwell.com/", ...]
}
}
用法:
uv run python scripts/build_glossary.py <project_slug>
uv run python scripts/build_glossary.py <project_slug> --workers 4
uv run python scripts/build_glossary.py <project_slug> --only "Mabwell,Maywavee"
uv run python scripts/build_glossary.py <project_slug> --force
断点续传:已核查过的条目(有 verified_at 字段)默认跳过;--force 全部重跑。
"""
from __future__ import annotations
import argparse
import concurrent.futures
import datetime as dt
import json
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from scripts.lib.search_client import SearchClient, SearchError
from scripts.lib.zenmux_client import ZenMuxClient, ZenMuxError, load_secrets
DEFAULT_MODEL = "anthropic/claude-haiku-4.5"
PROMPT_FILE = Path(__file__).parent / "prompts" / "glossary_system.txt"
def resolve_project(arg: str) -> Path:
p = Path(arg)
if p.is_dir():
return p
cand = Path.cwd() / "projects" / arg
if cand.is_dir():
return cand
raise SystemExit(f"project not found: {arg}")
def load_glossary(path: Path) -> dict:
if not path.exists():
return {}
return json.loads(path.read_text(encoding="utf-8"))
def save_glossary(path: Path, glossary: dict) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(glossary, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
def migrate_legacy_entry(value) -> dict:
"""旧版 glossary 里 value 是字符串;升级为对象格式。"""
if isinstance(value, str):
return {"zh": value}
if isinstance(value, dict):
return value
return {"zh": str(value)}
def build_query(term: str, domain: str) -> str:
if domain:
return f"{term} {domain} 中文名"
return term
def parse_json_line(text: str) -> dict:
"""模型返回的单行 JSON。容忍前后额外字符。"""
s = text.strip()
l = s.find("{")
r = s.rfind("}")
if l == -1 or r == -1:
raise ValueError(f"no JSON object: {text[:200]}")
obj = json.loads(s[l : r + 1])
if not isinstance(obj, dict):
raise ValueError("top-level JSON not object")
return obj
def build_user_prompt(term: str, domain: str, current_zh: str, hits: list) -> str:
hits_text = "\n\n".join(
f"[{i+1}] {h.title}\n URL: {h.url}\n {h.snippet[:500]}"
for i, h in enumerate(hits[:5])
)
if not hits_text:
hits_text = "(无搜索结果)"
return (
f"term: {term}\n"
f"domain: {domain or '(未指定)'}\n"
f"current_zh: {current_zh or '(空)'}\n\n"
f"search_hits:\n{hits_text}\n"
)
def verify_term(
term: str,
current_zh: str,
domain: str,
search_client: SearchClient,
llm_client: ZenMuxClient,
*,
model: str,
system_prompt: str,
) -> dict:
try:
hits = search_client.search(
build_query(term, domain),
num_results=4,
)
except SearchError as e:
return {
"zh": current_zh or "",
"en_full": term,
"confidence": "low",
"issue": f"搜索失败:{e}",
"sources": [],
}
user = build_user_prompt(term, domain, current_zh, hits)
try:
raw = llm_client.chat_complete(
model=model,
system=system_prompt,
user=user,
temperature=0.1,
max_tokens=800,
tag=f"glossary:{term[:30]}",
)
obj = parse_json_line(raw)
except (ZenMuxError, ValueError) as e:
return {
"zh": current_zh or "",
"en_full": term,
"confidence": "low",
"issue": f"LLM 核查失败:{e}",
"sources": [h.url for h in hits[:3]],
}
# 规范化输出
return {
"zh": str(obj.get("zh", current_zh or "")),
"en_full": str(obj.get("en_full", term)),
"confidence": str(obj.get("confidence", "low")),
"issue": str(obj.get("issue", "")),
"sources": [h.url for h in hits[:3]],
}
def main() -> int:
parser = argparse.ArgumentParser(description="双语术语表事实核查(Haiku + Exa)")
parser.add_argument("project", help="项目 slug 或完整路径")
parser.add_argument(
"--input",
default="phase4/glossary.json",
help="初版术语表路径(相对项目根)",
)
parser.add_argument(
"--output",
default="phase4/glossary.json",
help="输出路径(默认原地覆盖)",
)
parser.add_argument("--model", default=DEFAULT_MODEL, help="LLM 模型")
parser.add_argument(
"--workers", type=int, default=4, help="并发度(默认 4;网络不稳时建议降到 3)"
)
parser.add_argument(
"--force", action="store_true",
help="忽略已核查状态,全部重跑",
)
parser.add_argument(
"--only",
default=None,
help="只核查指定术语(逗号分隔,大小写敏感)",
)
parser.add_argument(
"--extra",
default=None,
help="每行一个英文术语的文本文件,补充进术语表一起核查",
)
parser.add_argument(
"--domain",
default=None,
help="术语领域提示(默认自动读 manifest.topic",
)
args = parser.parse_args()
load_secrets()
project_root = resolve_project(args.project)
input_path = project_root / args.input
output_path = project_root / args.output
manifest_path = project_root / "manifest.json"
domain = args.domain
if not domain and manifest_path.exists():
m = json.loads(manifest_path.read_text(encoding="utf-8"))
domain = m.get("topic") or m.get("report_title") or ""
glossary = load_glossary(input_path)
# migration
for k, v in list(glossary.items()):
glossary[k] = migrate_legacy_entry(v)
if args.extra:
extra_path = Path(args.extra)
if not extra_path.exists():
raise SystemExit(f"--extra 文件不存在:{extra_path}")
for line in extra_path.read_text(encoding="utf-8").splitlines():
term = line.strip()
if term and term not in glossary:
glossary[term] = {"zh": ""}
only_terms: set[str] | None = None
if args.only:
only_terms = {t.strip() for t in args.only.split(",") if t.strip()}
system_prompt = PROMPT_FILE.read_text(encoding="utf-8")
# 筛选需要核查的
todo: list[str] = []
for term, entry in glossary.items():
if only_terms is not None and term not in only_terms:
continue
if not args.force and entry.get("verified_at"):
continue
todo.append(term)
print(f"Project: {project_root.name}")
print(f"Domain hint: {domain or '(none)'}")
print(f"Glossary size: {len(glossary)} | to verify: {len(todo)} | workers: {args.workers}")
print(f"Model: {args.model}")
if not todo:
print(" 没有需要核查的条目(使用 --force 强制重跑)")
save_glossary(output_path, glossary)
return 0
today = dt.date.today().isoformat()
logs_dir = project_root / "phase4" / "logs"
log_file = logs_dir / "glossary.jsonl"
start = time.time()
done_count = 0
failed: list[str] = []
# 并发执行:SearchClient/ZenMuxClient 都是 thread-safehttpx.Client 支持)
with SearchClient() as search_client, ZenMuxClient(log_file=log_file) as llm_client:
def worker(term: str) -> tuple[str, dict]:
current_zh = glossary.get(term, {}).get("zh", "")
result = verify_term(
term,
current_zh,
domain or "",
search_client,
llm_client,
model=args.model,
system_prompt=system_prompt,
)
return term, result
with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool:
futures = {pool.submit(worker, term): term for term in todo}
for fut in concurrent.futures.as_completed(futures):
term = futures[fut]
try:
_term, result = fut.result()
except Exception as e:
print(f" [FAIL] {term}: {e}")
failed.append(term)
continue
# 合并到 glossary
old = glossary.get(term, {})
old.update(result)
old["verified_at"] = today
glossary[term] = old
done_count += 1
conf = result.get("confidence", "?")
issue = result.get("issue", "")
zh = result.get("zh") or "(保留英文)"
marker = {"high": "", "medium": "~", "low": "?"}.get(conf, " ")
issue_str = f"{issue[:80]}" if issue else ""
print(f" [{marker}] {term:<35}{zh}{issue_str}")
# 阶段性存盘,避免中途挂掉丢数据
if done_count % 10 == 0:
save_glossary(output_path, glossary)
save_glossary(output_path, glossary)
elapsed = time.time() - start
print(f"\n完成:{done_count}/{len(todo)}(失败 {len(failed)},耗时 {elapsed:.1f}s")
print(f"术语表:{output_path.relative_to(project_root)}")
print(llm_client.usage.summary())
# 高警示项汇总
issues = [
(k, v) for k, v in glossary.items()
if v.get("issue") and v.get("confidence") != "high"
]
if issues:
print(f"\n⚠ 低置信度或带问题的条目({len(issues)} 条):")
for k, v in issues[:15]:
print(f" - {k:<30} [{v.get('confidence','?')}] {v.get('issue','')[:100]}")
if len(issues) > 15:
print(f" … 还有 {len(issues) - 15} 条,见 {output_path.name}")
return 0
if __name__ == "__main__":
sys.exit(main())
+477
View File
@@ -0,0 +1,477 @@
#!/usr/bin/env python3
"""Phase 4 成稿阶段:统一入口。
从 final_zh_polished.md(或指定的 Markdown+ manifest.json 生成:
- <title>.pdf PDFReportLab 或 Quarto/xelatex
- <title>.docx Pandoc 出 DOCX
- <title>-en.pdf 如果存在 final_en.md 也一并出英文版(可选)
文件名来自 manifest.report_title(去掉非法字符),不再用 "final.pdf" 这种通用名。
用法:
uv run python scripts/build_report.py <project_slug>
# 使用 Quarto/xelatex 引擎(推荐,更好的中文+宽表支持):
uv run python scripts/build_report.py <project_slug> --engine quarto
# 只生成 PDF
uv run python scripts/build_report.py <project_slug> --no-docx
# 从自定义 md 生成:
uv run python scripts/build_report.py <project_slug> --input phase4/final_zh.md
环境依赖:
- reportlab, pypandoc, 思源字体(bash .opencode/templates/fonts/download-fonts.sh
- pandoc 可执行文件在 PATH
- Quarto(可选,--engine quarto 时需要):https://quarto.org/docs/get-started/
安装后运行:quarto install tinytex
"""
from __future__ import annotations
import argparse
import json
import re
import shutil
import subprocess
import sys
import textwrap
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from scripts.lib.zenmux_client import load_secrets # noqa: F401 (为一致性)
from scripts.reporting.fonts import resolve_quarto_fonts
from scripts.reporting.references import build_references_block
REPO_ROOT = Path(__file__).resolve().parent.parent
PDF_TEMPLATE = REPO_ROOT / ".opencode" / "templates" / "report-template.py"
DEFAULT_FONTS_DIR = REPO_ROOT / ".opencode" / "templates" / "fonts"
def resolve_project(arg: str) -> Path:
p = Path(arg)
if p.is_dir():
return p.resolve()
cand = REPO_ROOT / "projects" / arg
if cand.is_dir():
return cand.resolve()
raise SystemExit(f"project not found: {arg}")
# 文件系统对文件名的常见限制:Windows 更严格,按最小公倍数来
_FS_ILLEGAL_RE = re.compile(r'[\\/:*?"<>|\r\n\t]+')
_WHITESPACE_RE = re.compile(r"\s+")
def sanitize_filename(name: str, max_len: int = 120) -> str:
"""把报告标题变成可跨 OS 使用的文件名。"""
if not name:
return "report"
# 去掉非法字符
cleaned = _FS_ILLEGAL_RE.sub(" ", name)
# 合并空白
cleaned = _WHITESPACE_RE.sub(" ", cleaned).strip()
# 首尾 . 空格 . Windows 要求)
cleaned = cleaned.strip(". ").strip()
if len(cleaned) > max_len:
cleaned = cleaned[:max_len].rstrip()
return cleaned or "report"
def load_manifest(path: Path) -> dict:
return json.loads(path.read_text(encoding="utf-8"))
def determine_input_md(project_root: Path, preferred: str | None) -> Path:
"""决定用哪个 Markdown 出稿。
优先级:--input 指定 > final_zh_polished.md > final_zh.md > final_en.md
"""
if preferred:
p = project_root / preferred
if not p.exists():
raise SystemExit(f"指定的 --input 不存在:{p}")
return p
for candidate in (
"phase4/final_zh_polished.md",
"phase4/final_zh.md",
"phase4/final_en.md",
):
p = project_root / candidate
if p.exists():
return p
raise SystemExit(
f"找不到任何 Markdown 源。项目根:{project_root}\n"
"先跑 translate.py / polish.py 或使用 --input 指定路径。"
)
def build_pdf(
md_path: Path,
manifest_path: Path,
output_pdf: Path,
fonts_dir: Path,
sources_path: Path | None,
) -> None:
"""调用 report-template.py 生成 PDF。"""
cmd = [
sys.executable,
str(PDF_TEMPLATE),
"--input", str(md_path),
"--manifest", str(manifest_path),
"--output", str(output_pdf),
"--fonts-dir", str(fonts_dir),
]
if sources_path and sources_path.exists():
cmd += ["--sources", str(sources_path)]
print(f"\n→ 生成 PDF{output_pdf.name}")
result = subprocess.run(cmd, check=False)
if result.returncode != 0:
raise SystemExit(f"PDF 生成失败,返回码 {result.returncode}")
def _detect_wide_tables(md_text: str, min_cols: int = 8) -> list[tuple[int, int]]:
"""返回所有列数 >= min_cols 的 Markdown 表格的 (start_line, end_line) 区间(0-based)。"""
lines = md_text.split("\n")
ranges = []
i = 0
while i < len(lines):
line = lines[i]
if line.startswith("|") and line.count("|") - 1 >= min_cols:
# Possible table header — next line should be separator
if i + 1 < len(lines) and re.match(r"^\|[\s\-:|]+\|", lines[i + 1]):
start = i
j = i + 2
while j < len(lines) and lines[j].strip().startswith("|"):
j += 1
ranges.append((start, j))
i = j
continue
i += 1
return ranges
def prepare_qmd(
md_path: Path,
manifest: dict,
output_qmd: Path,
fonts_dir: Path,
sources_path: Path | None,
wide_table_cols: int = 8,
) -> None:
"""将普通 Markdown 转换为带 Quarto front matter 的 .qmd 文件。
主要处理:
1. 插入 YAML front matter(标题、字体、页面设置等)
2. 用 {.landscape} div 包裹列数 >= wide_table_cols 的宽表
3. 将 [TOC will be generated...] 占位符替换为真实 TOC 指令
4. 将 [REFERENCES will be filled...] 占位符替换为参考文献内容
"""
title = manifest.get("report_title", "报告")
subtitle = manifest.get("report_subtitle", "")
date = manifest.get("date", "")
# 决定字体名称:Quarto/xelatex 使用系统字体 family name。
fonts = resolve_quarto_fonts(fonts_dir)
main_font = fonts.main_font
sans_font = fonts.sans_font
# Write LaTeX header file for CJK font setup.
# Using a separate .tex file avoids YAML escape issues with backslashes.
mf = main_font # "Source Han Serif CN"
sf = sans_font # "Source Han Sans CN"
front_matter = textwrap.dedent(f"""\
---
title: "{title}"
subtitle: "{subtitle}"
date: "{date}"
lang: zh
format:
pdf:
pdf-engine: xelatex
CJKmainfont: "{mf}"
mainfont: "{mf}"
mainfontoptions:
- BoldFont={mf}
- ItalicFont={mf}
- BoldItalicFont={mf}
CJKoptions:
- BoldFont={mf}
- ItalicFont={mf}
- BoldItalicFont={mf}
sansfont: "{sf}"
sansfontoptions:
- BoldFont={sf}
- ItalicFont={sf}
- BoldItalicFont={sf}
monofont: "Liberation Mono"
papersize: a4
documentclass: scrartcl
classoption:
- DIV=11
- headinclude
toc: true
toc-depth: 2
toc-title: "目录"
number-sections: false
colorlinks: true
linkcolor: NavyBlue
urlcolor: NavyBlue
geometry:
- top=25mm
- bottom=25mm
- left=25mm
- right=20mm
pdf-engine-opts:
- "-stack-size=32768"
- "-extra-mem-top=2000000"
include-in-header:
- file: _preamble.tex
---
""")
md_text = md_path.read_text(encoding="utf-8")
# Remove existing YAML front matter if any (between first two ---)
if md_text.startswith("---"):
end = md_text.find("\n---", 3)
if end != -1:
md_text = md_text[end + 4:].lstrip("\n")
# Quarto already renders the title from YAML; drop a duplicated leading H1.
md_text = re.sub(
rf"^#\s+{re.escape(title)}\s*\n+",
"",
md_text,
count=1,
)
# Replace TOC placeholder
md_text = re.sub(
r"\[TOC will be generated.*?\]",
"", # Quarto handles TOC via front matter
md_text,
)
# Replace REFERENCES placeholder with actual references from sources.jsonl
ref_block = build_references_block(sources_path, md_text)
md_text = re.sub(
r"\[REFERENCES will be filled.*?\]",
ref_block,
md_text,
)
# Wrap wide tables in {.landscape} divs
lines = md_text.split("\n")
wide_ranges = _detect_wide_tables(md_text, min_cols=wide_table_cols)
if wide_ranges:
# Insert landscape wrappers from bottom up (so line numbers stay valid)
for start, end in reversed(wide_ranges):
lines.insert(end, "\n:::")
lines.insert(start, "::: {.landscape}\n")
md_text = "\n".join(lines)
# Write LaTeX preamble file (table + landscape support)
preamble_tex = output_qmd.parent / "_preamble.tex"
preamble_tex.write_text(
"\\usepackage{longtable}\n"
"\\usepackage{booktabs}\n"
"\\usepackage{array}\n"
"\\usepackage{xcolor}\n"
"\\usepackage{titlesec}\n"
"\\definecolor{DRBlue}{HTML}{1E3A8A}\n"
"\\definecolor{DRSlate}{HTML}{374151}\n"
"\\definecolor{DRMuted}{HTML}{6B7280}\n"
# Use lscape instead of pdflscape to avoid \LS@makefcolumn recursion
# which exhausts TeX param_size on large longtables.
# lscape rotates content without changing page media box (reader must rotate).
"\\usepackage{lscape}\n"
"\\setlength{\\LTpre}{6pt}\n"
"\\setlength{\\LTpost}{6pt}\n"
"\\setlength{\\tabcolsep}{3pt}\n"
"\\linespread{1.18}\n"
"\\setlength{\\parindent}{2em}\n"
"\\setlength{\\parskip}{0.25em}\n"
"\\newcommand{\\sectionbreak}{\\clearpage}\n"
"\\titleformat{\\section}[display]\n"
" {\\centering\\Large\\bfseries\\sffamily\\color{DRBlue}}\n"
" {}{0pt}{}\n"
"\\titlespacing*{\\section}{0pt}{0pt}{1.1em}\n"
"\\titleformat{\\subsection}\n"
" {\\large\\bfseries\\sffamily\\color{DRBlue}}\n"
" {}{0pt}{}\n"
"\\titlespacing*{\\subsection}{0pt}{1.1em}{0.45em}\n"
"\\titleformat{\\subsubsection}\n"
" {\\normalsize\\bfseries\\sffamily\\color{DRSlate}}\n"
" {}{0pt}{}\n"
"\\titlespacing*{\\subsubsection}{0pt}{0.9em}{0.35em}\n"
"\\renewcommand{\\contentsname}{目录}\n"
"\\setcounter{tocdepth}{2}\n",
encoding="utf-8",
)
output_qmd.write_text(front_matter + md_text, encoding="utf-8")
print(f" .qmd prepared: {output_qmd.name} ({len(wide_ranges)} landscape table(s))")
def build_pdf_quarto(
md_path: Path,
manifest: dict,
output_pdf: Path,
fonts_dir: Path,
sources_path: Path | None,
) -> None:
"""使用 Quarto + xelatex 生成 PDF。"""
if not shutil.which("quarto"):
raise SystemExit(
"quarto 命令未找到。请先安装 Quartohttps://quarto.org/docs/get-started/\n"
"安装后运行:quarto install tinytex"
)
# Prepare .qmd in the same dir as output_pdf
qmd_path = output_pdf.parent / (output_pdf.stem + ".qmd")
prepare_qmd(md_path, manifest, qmd_path, fonts_dir, sources_path)
print(f"\n→ 生成 PDFQuarto/xelatex):{output_pdf.name}")
cmd = [
"quarto", "render", str(qmd_path),
"--to", "pdf",
"--output", output_pdf.name,
]
result = subprocess.run(cmd, cwd=str(output_pdf.parent), check=False)
if result.returncode != 0:
raise SystemExit(f"Quarto PDF 生成失败,返回码 {result.returncode}")
# Clean up auxiliary files Quarto leaves behind
for ext in (".tex", ".log", ".aux", ".toc", ".out", "-files"):
candidate = output_pdf.parent / (output_pdf.stem + ext)
if candidate.exists():
candidate.unlink(missing_ok=True)
def build_docx(md_path: Path, output_docx: Path, title: str) -> None:
"""用 pandoc 生成 DOCX。"""
if not shutil.which("pandoc"):
print(f" ⚠ pandoc 不在 PATH,跳过 DOCX 生成", file=sys.stderr)
return
print(f"\n→ 生成 DOCX{output_docx.name}")
# 关闭 tex_math_dollars/tex_math_single_backslash 防止文中 "$100" "$10^6" 被当数学公式
cmd = [
"pandoc",
str(md_path),
"-o", str(output_docx),
"--from=markdown-tex_math_dollars-tex_math_single_backslash-raw_tex",
"--to=docx",
"--standalone",
"-M", f"title={title}",
"--wrap=preserve",
]
# reference-doc 如果存在就用
ref_doc = REPO_ROOT / ".opencode" / "templates" / "reference.docx"
if ref_doc.exists():
cmd += ["--reference-doc", str(ref_doc)]
result = subprocess.run(cmd, check=False)
if result.returncode != 0:
print(f" ⚠ DOCX 生成失败(返回码 {result.returncode}),但不阻断流程", file=sys.stderr)
def main() -> int:
parser = argparse.ArgumentParser(description="Phase 4 成稿(PDF + DOCX")
parser.add_argument("project", help="项目 slug 或完整路径")
parser.add_argument(
"--input",
default=None,
help="Markdown 源(默认自动寻找 phase4/final_zh_polished.md",
)
parser.add_argument(
"--output-dir",
default="phase4",
help="输出目录(相对项目根,默认 phase4)",
)
parser.add_argument(
"--fonts-dir",
default=str(DEFAULT_FONTS_DIR),
help="字体目录",
)
parser.add_argument(
"--sources",
default=None,
help="sources.jsonl 路径(默认 phase2/sources.jsonl",
)
parser.add_argument("--no-docx", action="store_true", help="跳过 DOCX 生成")
parser.add_argument("--no-pdf", action="store_true", help="跳过 PDF 生成")
parser.add_argument(
"--engine",
choices=["reportlab", "quarto"],
default="reportlab",
help="PDF 渲染引擎:reportlab(默认,Python 原生)或 quartoxelatex,更好的中文+宽表支持)",
)
parser.add_argument(
"--basename",
default=None,
help="文件名 stem(不带扩展名),默认从 manifest.report_title 生成",
)
args = parser.parse_args()
project_root = resolve_project(args.project)
manifest_path = project_root / "manifest.json"
if not manifest_path.exists():
raise SystemExit(f"manifest.json not found: {manifest_path}")
manifest = load_manifest(manifest_path)
md_path = determine_input_md(project_root, args.input)
title = manifest.get("report_title") or manifest.get("topic") or "Deep Research Report"
basename = args.basename or sanitize_filename(title)
output_dir = project_root / args.output_dir
output_dir.mkdir(parents=True, exist_ok=True)
pdf_path = output_dir / f"{basename}.pdf"
docx_path = output_dir / f"{basename}.docx"
fonts_dir = Path(args.fonts_dir)
if not fonts_dir.is_absolute():
fonts_dir = REPO_ROOT / fonts_dir
sources_path: Path | None = None
if args.sources:
sources_path = Path(args.sources)
else:
default_src = project_root / "phase2" / "sources.jsonl"
if default_src.exists():
sources_path = default_src
print("========== Phase 4 成稿 ==========")
print(f"项目: {project_root.name}")
print(f"Markdown {md_path.relative_to(project_root)}")
print(f"标题: {title}")
print(f"输出名: {basename}")
print(f"字体目录: {fonts_dir}")
print(f"Sources {sources_path if sources_path else '(缺失)'}")
if not args.no_pdf:
if args.engine == "quarto":
build_pdf_quarto(md_path, manifest, pdf_path, fonts_dir, sources_path)
else:
build_pdf(md_path, manifest_path, pdf_path, fonts_dir, sources_path)
if not args.no_docx:
build_docx(md_path, docx_path, title)
print("\n========== 完成 ==========")
if pdf_path.exists():
print(f" PDF: {pdf_path.relative_to(project_root)} ({pdf_path.stat().st_size // 1024} KB)")
if docx_path.exists():
print(f" DOCX: {docx_path.relative_to(project_root)} ({docx_path.stat().st_size // 1024} KB)")
return 0
if __name__ == "__main__":
sys.exit(main())
+146
View File
@@ -0,0 +1,146 @@
#!/usr/bin/env python3
"""引文完整性核查。
检查 final_zh_polished.md(或其它正文)中的 [src_xxx] 引用与 sources.jsonl 是否一致:
- 孤立引用(正文有但 sources.jsonl 无):需要 dr-analyst 补信源或删这处引用
- 孤岛信源(sources.jsonl 有但正文无):被 polish 或润色误删了上下文,或 dr-analyst 收集了
但没用上
- emoji 扫描:正文里不该有 emoji
- 编号格式:检查 src_xxx 是否符合规范
用法:
uv run python scripts/check_citations.py <project_slug>
uv run python scripts/check_citations.py <project_slug> --md phase4/final_zh.md
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from pathlib import Path
EMOJI_RE = re.compile(
r"[\U0001F000-\U0001FFFF]" # Supplementary Plane emoji
r"|[\u2700-\u27BF]" # Dingbats (✅ ❌)
r"|[\u2600-\u26FF]" # Misc symbols (⭐ ⚠ ☀)
r"|[\u2B00-\u2BFF]" # Misc symbols and arrows
)
# 允许的符号(字体支持)
ALLOWED_SYMBOLS = {
"", "×", "", "", "", "", "", "",
"", "", "", "",
}
SRC_ID_RE = re.compile(r"\[(src_[A-Za-z0-9_\-]+(?:\s*,\s*src_[A-Za-z0-9_\-]+)*)\]")
def resolve_project(arg: str) -> Path:
p = Path(arg)
if p.is_dir():
return p
cand = Path.cwd() / "projects" / arg
if cand.is_dir():
return cand
raise SystemExit(f"project not found: {arg}")
def main() -> int:
parser = argparse.ArgumentParser(description="引文完整性核查")
parser.add_argument("project", help="项目 slug 或路径")
parser.add_argument("--md", default="phase4/final_zh_polished.md")
parser.add_argument("--sources", default="phase2/sources.jsonl")
args = parser.parse_args()
project = resolve_project(args.project)
md_path = project / args.md
src_path = project / args.sources
if not md_path.exists():
raise SystemExit(f"找不到正文:{md_path}")
text = md_path.read_text(encoding="utf-8")
# 1. 收集正文引用(保序去重)
cited: list[str] = []
cited_set: set[str] = set()
for m in SRC_ID_RE.finditer(text):
for sid in m.group(1).split(","):
sid = sid.strip()
if sid and sid not in cited_set:
cited_set.add(sid)
cited.append(sid)
# 2. 收集 sources.jsonl 中的 ID
sources: dict[str, dict] = {}
if src_path.exists():
for line in src_path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line:
continue
try:
r = json.loads(line)
if sid := r.get("id"):
sources[sid] = r
except Exception:
continue
orphan_cites = [s for s in cited if s not in sources]
island_sources = [s for s in sources if s not in cited_set]
matched = [s for s in cited if s in sources]
print(f"=== 引文统计 ===")
print(f" 正文引用(独立 ID):{len(cited_set)}")
print(f" sources.jsonl 收录:{len(sources)}")
print(f" 匹配:{len(matched)}")
print(f" 孤立引用(正文有 sources 无):{len(orphan_cites)}")
print(f" 孤岛信源(sources 有正文无):{len(island_sources)}")
if orphan_cites:
print(f"\n=== 孤立引用(前 20 条)===")
for s in orphan_cites[:20]:
print(f" {s}")
if len(orphan_cites) > 20:
print(f" ...还有 {len(orphan_cites) - 20}")
print(f"\n 处理建议:")
print(f" (a) 如果是 dr-analyst 编造的占位符 → 在正文中删除该引用")
print(f" (b) 如果是信源未收录 → 补到 sources.jsonl")
if island_sources:
print(f"\n=== 孤岛信源(前 20 条)===")
for s in island_sources[:20]:
print(f" {s}{sources[s].get('title', '')[:80]}")
if len(island_sources) > 20:
print(f" ...还有 {len(island_sources) - 20}")
print(f"\n 处理建议:")
print(f" (a) 如果是 polish 阶段误删了使用该信源的段落 → 检查 polish diff")
print(f" (b) 如果是收集多余信源 → 可以保留(build_references 会自动忽略)")
# 3. Emoji 扫描
emoji_hits = []
for m in EMOJI_RE.finditer(text):
c = m.group()
if c not in ALLOWED_SYMBOLS:
line = text[:m.start()].count("\n") + 1
emoji_hits.append((line, c))
if emoji_hits:
print(f"\n=== ⚠ 发现 {len(emoji_hits)} 个 emoji(不允许出现在正文)===")
seen = {}
for line, c in emoji_hits:
seen.setdefault(c, []).append(line)
for c, lines in seen.items():
print(f" U+{ord(c):04X} {c!r}{lines[:5]} 行 等 {len(lines)}")
print(f" 建议用 python3 替换:sed -i '' 's/{list(seen.keys())[0]}//g' {md_path}")
# 返回码:有问题返回非零便于 CI 使用
if orphan_cites or emoji_hits:
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
+291
View File
@@ -0,0 +1,291 @@
#!/usr/bin/env python3
"""Deploy platform adapter templates outside the repository checkout."""
from __future__ import annotations
import argparse
import os
import shutil
import sys
from dataclasses import dataclass, field
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from scripts.runtime.skills import SkillRegistry
CODEX_TEMPLATE = REPO_ROOT / "platform_adapters" / "codex"
ANTIGRAVITY_TEMPLATE = REPO_ROOT / "platform_adapters" / "antigravity" / "agent"
ANTIGRAVITY_RULES_DIR = ANTIGRAVITY_TEMPLATE / "rules"
ANTIGRAVITY_WORKFLOWS_DIR = ANTIGRAVITY_TEMPLATE / "workflows"
ANTIGRAVITY_AGENTS_FILE = ANTIGRAVITY_TEMPLATE / "agents.md"
@dataclass
class DeployResult:
platform: str
target: Path
written: list[Path] = field(default_factory=list)
skipped: list[Path] = field(default_factory=list)
planned: list[Path] = field(default_factory=list)
backups: list[Path] = field(default_factory=list)
def default_codex_home(
*,
env: dict[str, str] | None = None,
user_home: Path | None = None,
) -> Path:
values = os.environ if env is None else env
if values.get("CODEX_HOME"):
return Path(values["CODEX_HOME"]).expanduser()
home = Path.home() if user_home is None else user_home
return home / ".codex"
def copy_tree_contents(
src: Path,
dst: Path,
*,
force: bool,
dry_run: bool = False,
backup_existing: bool = True,
exclude: set[Path] | None = None,
) -> DeployResult:
if not src.exists():
raise FileNotFoundError(f"adapter template source not found: {src}")
result = DeployResult(platform="copy", target=dst)
excluded = exclude or set()
for item in sorted(src.rglob("*")):
rel = item.relative_to(src)
if rel in excluded:
continue
target = dst / rel
if item.is_dir():
if not dry_run:
target.mkdir(parents=True, exist_ok=True)
continue
if target.exists() and not force:
result.skipped.append(target)
continue
result.planned.append(target)
if dry_run:
continue
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists() and force and backup_existing:
backup = target.with_name(f"{target.name}.bak")
shutil.copy2(target, backup)
result.backups.append(backup)
shutil.copy2(item, target)
result.written.append(target)
return result
def _merge_results(platform: str, target: Path, parts: list[DeployResult]) -> DeployResult:
merged = DeployResult(platform=platform, target=target)
for part in parts:
merged.written.extend(part.written)
merged.skipped.extend(part.skipped)
merged.planned.extend(part.planned)
merged.backups.extend(part.backups)
return merged
def copy_registered_skills(dst: Path, *, force: bool, dry_run: bool = False) -> DeployResult:
result = DeployResult(platform="skills", target=dst)
for skill in SkillRegistry().list():
part = copy_tree_contents(skill.path.parent, dst / skill.name, force=force, dry_run=dry_run)
result.written.extend(part.written)
result.skipped.extend(part.skipped)
result.planned.extend(part.planned)
result.backups.extend(part.backups)
return result
def copy_antigravity_rules(dst: Path, *, force: bool, dry_run: bool = False) -> DeployResult:
result = DeployResult(platform="rules", target=dst)
if not ANTIGRAVITY_RULES_DIR.exists():
return result
part = copy_tree_contents(ANTIGRAVITY_RULES_DIR, dst, force=force, dry_run=dry_run)
result.written.extend(part.written)
result.skipped.extend(part.skipped)
result.planned.extend(part.planned)
result.backups.extend(part.backups)
return result
def copy_antigravity_workflows(dst: Path, *, force: bool, dry_run: bool = False) -> DeployResult:
result = DeployResult(platform="workflows", target=dst)
if not ANTIGRAVITY_WORKFLOWS_DIR.exists():
return result
part = copy_tree_contents(ANTIGRAVITY_WORKFLOWS_DIR, dst, force=force, dry_run=dry_run)
result.written.extend(part.written)
result.skipped.extend(part.skipped)
result.planned.extend(part.planned)
result.backups.extend(part.backups)
return result
def copy_antigravity_agents_file(dst: Path, *, force: bool, dry_run: bool = False) -> DeployResult:
result = DeployResult(platform="agents", target=dst)
if not ANTIGRAVITY_AGENTS_FILE.exists():
return result
target = dst / "agents.md"
if target.exists() and not force:
result.skipped.append(target)
return result
result.planned.append(target)
if dry_run:
return result
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists() and force:
backup = target.with_name("agents.md.bak")
shutil.copy2(target, backup)
result.backups.append(backup)
shutil.copy2(ANTIGRAVITY_AGENTS_FILE, target)
result.written.append(target)
return result
def deploy_codex(
*,
target: Path | None = None,
force: bool = False,
skip_skills: bool = False,
dry_run: bool = False,
include_config: bool = False,
repo_root: Path = REPO_ROOT,
) -> DeployResult:
codex_home = (target or default_codex_home()).expanduser()
template = repo_root / "platform_adapters" / "codex"
parts = [
copy_tree_contents(
template,
codex_home,
force=force,
dry_run=dry_run,
exclude=set() if include_config else {Path("config.toml")},
),
]
if not skip_skills:
parts.append(copy_registered_skills(codex_home / "skills", force=force, dry_run=dry_run))
return _merge_results("codex", codex_home, parts)
def deploy_antigravity(
*,
target: Path | None = None,
force: bool = False,
skip_skills: bool = False,
skip_agents: bool = False,
skip_rules: bool = False,
skip_workflows: bool = False,
dry_run: bool = False,
repo_root: Path = REPO_ROOT,
) -> DeployResult:
"""Deploy Antigravity workspace rules and skills into a workspace root.
This intentionally deploys to a workspace-local `.agent` directory, not
global Antigravity/Gemini settings, so existing user configuration is not
touched. Existing files are skipped unless `force=True`.
"""
workspace = (target or repo_root).expanduser()
parts: list[DeployResult] = []
if not skip_agents:
parts.append(copy_antigravity_agents_file(workspace / ".agent", force=force, dry_run=dry_run))
if not skip_skills:
parts.append(copy_registered_skills(workspace / ".agent" / "skills", force=force, dry_run=dry_run))
if not skip_rules:
parts.append(copy_antigravity_rules(workspace / ".agent" / "rules", force=force, dry_run=dry_run))
if not skip_workflows:
parts.append(copy_antigravity_workflows(workspace / ".agent" / "workflows", force=force, dry_run=dry_run))
return _merge_results("antigravity", workspace, parts)
def print_result(result: DeployResult) -> None:
action = "planned" if result.planned and not result.written else "written"
print(f"{result.platform} adapter deployment")
print(f" target: {result.target}")
print(f" files {action}: {len(result.planned if action == 'planned' else result.written)}")
print(f" files skipped: {len(result.skipped)}")
print(f" backups: {len(result.backups)}")
if result.written:
for path in result.written:
print(f" {path}")
elif result.planned:
for path in result.planned:
print(f" {path}")
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Deploy Deep Research surface adapters")
sub = parser.add_subparsers(dest="platform", required=True)
codex = sub.add_parser("codex", help="Deploy Codex adapter to CODEX_HOME or a target directory")
codex.add_argument("--target", type=Path, help="Codex home target; defaults to $CODEX_HOME or ~/.codex")
codex.add_argument("--force", action="store_true", help="overwrite existing files and create .bak backups")
codex.add_argument("--skip-skills", action="store_true", help="do not copy canonical skills into target/skills")
codex.add_argument("--include-config", action="store_true", help="also copy config.toml; off by default to avoid overwriting global Codex config")
codex.add_argument("--dry-run", action="store_true", help="show files that would be written")
antigravity = sub.add_parser("antigravity", help="Deploy Antigravity workspace rules and skills")
antigravity.add_argument("--target", type=Path, help="Workspace root; defaults to this repository")
antigravity.add_argument("--force", action="store_true", help="overwrite existing files and create .bak backups")
antigravity.add_argument("--skip-agents", action="store_true", help="do not copy role definitions into target/.agent/agents.md")
antigravity.add_argument("--skip-skills", action="store_true", help="do not copy canonical skills into target/.agent/skills")
antigravity.add_argument("--skip-rules", action="store_true", help="do not copy workspace rules into target/.agent/rules")
antigravity.add_argument("--skip-workflows", action="store_true", help="do not copy workflows into target/.agent/workflows")
antigravity.add_argument("--dry-run", action="store_true", help="show files that would be written without writing")
return parser
def main() -> int:
args = build_parser().parse_args()
if args.platform == "codex":
result = deploy_codex(
target=args.target,
force=args.force,
skip_skills=args.skip_skills,
dry_run=args.dry_run,
include_config=args.include_config,
)
print_result(result)
print()
print("Run Codex from this repository after deployment:")
if args.include_config:
print(" codex --profile deep-research")
else:
print(" codex")
print("Note: config.toml is not copied by default. Use --include-config only if you want the bundled profile.")
return 0
if args.platform == "antigravity":
result = deploy_antigravity(
target=args.target,
force=args.force,
skip_agents=args.skip_agents,
skip_skills=args.skip_skills,
skip_rules=args.skip_rules,
skip_workflows=args.skip_workflows,
dry_run=args.dry_run,
)
print_result(result)
print()
print("Open the target workspace in Antigravity and enable/mention the workspace rule if needed:")
print(" .agent/rules/deep-research-antigravity.md")
print("Workflow installed when supported by your Antigravity build:")
print(" .agent/workflows/deep-research-native.md")
print("Existing files are skipped by default. Use --force only when you want .bak backups and replacement.")
return 0
raise SystemExit(f"unsupported platform: {args.platform}")
if __name__ == "__main__":
raise SystemExit(main())
+202
View File
@@ -0,0 +1,202 @@
#!/usr/bin/env python3
"""Check and repair a Deep Research deployment checkout."""
from __future__ import annotations
import argparse
import os
import shutil
import subprocess
import sys
from pathlib import Path
try:
import tomllib
except ModuleNotFoundError: # pragma: no cover - Python < 3.11 fallback.
tomllib = None # type: ignore[assignment]
REPO_ROOT = Path(__file__).resolve().parent.parent
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from scripts.deploy_adapters import default_codex_home, deploy_codex
from scripts.runtime.skills import SkillRegistry
CODEX_TEMPLATE = REPO_ROOT / "platform_adapters" / "codex"
AGENT_SKILLS = REPO_ROOT / "platform_adapters" / "antigravity" / "agent" / "skills"
REQUIRED_PATHS = [
"AGENTS.md",
"GEMINI.md",
"README.md",
"PLAN.md",
"platform_adapters/antigravity/agent/agents.md",
"platform_adapters/antigravity/agent/rules/deep-research-antigravity.md",
"platform_adapters/antigravity/agent/workflows/deep-research-native.md",
"platform_adapters/antigravity/agent/skills/search-strategy/SKILL.md",
"platform_adapters/codex/config.toml",
"platform_adapters/codex/agents/dr-pm.toml",
"platform_adapters/codex/commands/dr-run.md",
"skills/deep-research/SKILL.md",
"skills/document-ingest/SKILL.md",
"scripts/dr.py",
"scripts/deploy_adapters.py",
"scripts/export_antigravity_workspace.py",
]
REQUIRED_ENV_KEYS = [
"ZENMUX_API_KEY",
"TAVILY_API_KEY",
"BRAVE_API_KEY",
"EXA_API_KEY",
]
def rel(path: Path) -> str:
return str(path.relative_to(REPO_ROOT))
def copy_tree_contents(src: Path, dst: Path, *, force: bool) -> list[Path]:
written: list[Path] = []
if not src.exists():
raise RuntimeError(f"source not found: {rel(src)}")
dst.mkdir(parents=True, exist_ok=True)
for item in src.rglob("*"):
target = dst / item.relative_to(src)
if item.is_dir():
target.mkdir(parents=True, exist_ok=True)
continue
if target.exists() and not force:
continue
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(item, target)
written.append(target)
return written
def parse_env(path: Path) -> dict[str, str]:
values: dict[str, str] = {}
if not path.exists():
return values
for raw in path.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
values[key.strip()] = value.strip().strip('"').strip("'")
return values
def git_tracked(paths: list[str]) -> set[str]:
proc = subprocess.run(
["git", "ls-files", *paths],
cwd=REPO_ROOT,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
check=False,
)
return set(proc.stdout.splitlines())
def check_toml(path: Path, issues: list[str]) -> None:
if not path.exists():
return
if tomllib is None:
issues.append("Python tomllib unavailable; skip TOML parse checks")
return
try:
tomllib.loads(path.read_text(encoding="utf-8"))
except Exception as exc: # noqa: BLE001 - deployment diagnostics.
issues.append(f"{rel(path)} TOML parse failed: {exc}")
def check_deployment() -> int:
issues: list[str] = []
warnings: list[str] = []
for item in REQUIRED_PATHS:
if not (REPO_ROOT / item).exists():
issues.append(f"missing required path: {item}")
tracked = git_tracked(["platform_adapters", "skills"])
legacy_tracked = git_tracked([".codex", ".opencode", ".claude", ".gemini", ".agents"])
if legacy_tracked:
warnings.append("legacy platform adapter files are still tracked; prefer Antigravity clean workspace or remove them from the branch")
for item in REQUIRED_PATHS:
if item.startswith("platform_adapters/") and item not in tracked:
warnings.append(f"not tracked by git: {item}")
skills = SkillRegistry().list()
if len(skills) < 10:
issues.append(f"expected at least 10 Codex skills, found {len(skills)}")
check_toml(CODEX_TEMPLATE / "config.toml", issues)
for path in sorted((CODEX_TEMPLATE / "agents").glob("*.toml")):
check_toml(path, issues)
codex_home = default_codex_home()
if not (codex_home / "commands" / "dr-run.md").exists():
warnings.append(f"Codex adapter not deployed to {codex_home}; run scripts/deploy_adapters.py codex")
env_values = {key: os.environ.get(key, "") for key in REQUIRED_ENV_KEYS}
env_values.update({k: v for k, v in parse_env(REPO_ROOT / "secrets.env").items() if not env_values.get(k)})
missing_env = [key for key in REQUIRED_ENV_KEYS if not env_values.get(key)]
if missing_env:
warnings.append("missing optional/required API keys for full automation: " + ", ".join(missing_env))
print("Deep Research deployment check")
print(f" repo: {REPO_ROOT}")
print(f" platform adapter files tracked: {sum(1 for p in tracked if p.startswith('platform_adapters/'))}")
print(f" legacy platform adapter files tracked: {len(legacy_tracked)}")
print(f" codex home: {codex_home}")
print(f" antigravity adapter source files tracked: {sum(1 for p in tracked if p.startswith('platform_adapters/antigravity/'))}")
print(f" codex skills: {len(skills)}")
if warnings:
print("\nWarnings:")
for item in warnings:
print(f" - {item}")
if issues:
print("\nIssues:")
for item in issues:
print(f" - {item}")
return 1
print("\nDeployment check passed.")
return 0
def repair(force: bool, codex_home: Path | None) -> int:
try:
codex_result = deploy_codex(target=codex_home, force=force)
except PermissionError as exc:
print(f"repair failed: permission denied: {exc}", file=sys.stderr)
return 1
except RuntimeError as exc:
print(f"repair failed: {exc}", file=sys.stderr)
return 1
print("Repair completed.")
print(f" Codex home files written: {len(codex_result.written)}")
print(f" Codex home: {codex_result.target}")
print(f" Antigravity skills source: {AGENT_SKILLS}")
return check_deployment()
def main() -> int:
parser = argparse.ArgumentParser(description="Check or repair Deep Research deployment files")
parser.add_argument("--repair", action="store_true", help="deploy Codex templates outside the repo and sync skills")
parser.add_argument("--force", action="store_true", help="overwrite existing files during --repair")
parser.add_argument("--codex-home", type=Path, help="Codex home target for --repair; defaults to $CODEX_HOME or ~/.codex")
args = parser.parse_args()
if args.repair:
return repair(force=args.force, codex_home=args.codex_home)
return check_deployment()
if __name__ == "__main__":
raise SystemExit(main())
+962
View File
@@ -0,0 +1,962 @@
#!/usr/bin/env python3
"""Platform-neutral Deep Research CLI for Codex and other adapters.
This CLI intentionally keeps deterministic orchestration in Python while
allowing Codex/OpenCode/Gemini/Claude Code to provide the agentic layer.
"""
from __future__ import annotations
import argparse
import json
import re
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from scripts.lib.model_config import (
ModelConfigError,
list_model_profiles,
parse_model_overrides,
resolve_model_profile,
)
from scripts.runtime.assembly import build_chapter_briefs, build_compressed_findings, run_chapter_assembly_workers
from scripts.runtime.orchestrator import create_phase2_task_cards, write_placeholder_packets
from scripts.runtime.methods import ResearchMethodRegistry
from scripts.runtime.phase1 import create_project, render_framework, write_material_brief
from scripts.runtime.review import build_phase3_critique, build_phase3_model_critique
from scripts.runtime.roles import resolve_runtime_profile
from scripts.runtime.source_cache import cache_sources
from scripts.runtime.sources import rebuild_sources_from_packets
from scripts.runtime.skills import SkillRegistry, default_adapter_skill_dirs
from scripts.runtime.tasks import TaskCard, load_task_cards, validate_packet, write_task_cards
from scripts.runtime.workers import run_packet_workers
PROJECTS_DIR = REPO_ROOT / "projects"
CODEX_COMMAND_TEMPLATES_DIR = REPO_ROOT / "codex_adapter_templates" / "codex" / "commands"
LEGACY_CODEX_COMMANDS_DIR = REPO_ROOT / ".codex" / "commands"
def resolve_project(project: str | None, *, projects_dir: Path = PROJECTS_DIR) -> Path:
if project:
p = Path(project)
if p.is_dir():
return p.resolve()
cand = projects_dir / project
if cand.is_dir():
return cand.resolve()
raise SystemExit(f"project not found: {project}")
manifests = sorted(
projects_dir.glob("*/manifest.json"),
key=lambda p: p.stat().st_mtime,
reverse=True,
)
if not manifests:
raise SystemExit("no projects found")
return manifests[0].parent.resolve()
def load_manifest(project_root: Path) -> dict:
path = project_root / "manifest.json"
if not path.exists():
raise SystemExit(f"manifest not found: {path}")
return json.loads(path.read_text(encoding="utf-8"))
def count_words(text: str) -> int:
return len(re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)*", text))
def count_chinese_chars(text: str) -> int:
return sum(1 for c in text if "\u4e00" <= c <= "\u9fff")
def file_state(path: Path) -> str:
return "yes" if path.exists() else "no"
def packet_state_counts(project_root: Path) -> dict[str, int]:
packets = sorted((project_root / "phase2" / "packets").glob("*.json"))
errors = sorted((project_root / "phase2" / "packet_errors").glob("*.json"))
counts = {
"ready": 0,
"placeholder": 0,
"invalid": 0,
"errors": 0,
"stale_errors": 0,
"total": len(packets),
}
ready_stems: set[str] = set()
for path in packets:
try:
packet = json.loads(path.read_text(encoding="utf-8"))
except Exception:
counts["invalid"] += 1
continue
has_evidence = bool(
packet.get("claims")
or packet.get("evidence_items")
or packet.get("counter_evidence")
or packet.get("source_ids")
)
if has_evidence:
counts["ready"] += 1
ready_stems.add(path.stem)
else:
counts["placeholder"] += 1
for path in errors:
if path.stem in ready_stems:
counts["stale_errors"] += 1
else:
counts["errors"] += 1
return counts
def run_cmd(cmd: list[str], *, dry_run: bool) -> int:
printable = " ".join(cmd)
print(f"$ {printable}")
if dry_run:
return 0
return subprocess.run(cmd, cwd=REPO_ROOT, check=False).returncode
def cmd_init(args: argparse.Namespace) -> int:
projects_dir = Path(args.projects_dir) if args.projects_dir else PROJECTS_DIR
project_root = create_project(
topic=args.topic,
slug=args.slug,
projects_dir=projects_dir,
method_key=args.method,
report_type=args.report_type,
model_profile=args.profile,
target_words=args.target_words,
input_materials=args.input_material,
)
print(f"Project: {project_root.name}")
print(f"Created: {project_root}")
print("Runtime: python-core-v0.20")
print("Next: run `dr.py frame <project>` to generate phase1/framework.md")
return 0
def cmd_frame(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
if args.dry_run:
manifest = load_manifest(project_root)
method = ResearchMethodRegistry().get(args.method or manifest.get("research_method"))
print(f"Project: {project_root.name}")
print(f"Would write: phase1/framework.md")
print(f"Research method: {method.key}")
print(f"Chapters: {args.chapters}")
return 0
path = render_framework(
project_root,
method_key=args.method,
chapter_count=args.chapters,
preserve_existing_outline=args.preserve_existing_outline,
)
print(f"Project: {project_root.name}")
print(f"Wrote: {path.relative_to(project_root)}")
print("Pause: review and approve the framework before Phase 2.")
return 0
def cmd_approve(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
manifest = load_manifest(project_root)
phase1 = manifest.setdefault("phase1", {})
phase1["approved"] = True
phase1["requires_user_interview"] = False
phase1["approved_at"] = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
manifest["updated_at"] = phase1["approved_at"]
(project_root / "manifest.json").write_text(
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
print(f"Project: {project_root.name}")
print("Phase 1 approved. Phase 2 research is now enabled.")
return 0
def cmd_skills(args: argparse.Namespace) -> int:
registry = SkillRegistry()
if args.skills_cmd == "list":
for name in registry.list_names():
print(name)
return 0
if args.skills_cmd == "validate":
result = registry.validate()
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0 if result["ok"] else 1
if args.skills_cmd == "sync":
targets = [Path(item) for item in args.target] if args.target else default_adapter_skill_dirs()
copied = registry.sync_to(targets, force=True)
print(f"Synced skills: {copied}")
for target in targets:
print(f" {target}")
return 0
raise SystemExit(f"unknown skills command: {args.skills_cmd}")
def cmd_methods(args: argparse.Namespace) -> int:
registry = ResearchMethodRegistry()
if args.methods_cmd == "list":
for name in registry.list_names():
method = registry.get(name)
print(f"{method.key}: {method.name}")
return 0
if args.methods_cmd == "show":
method = registry.get(args.method)
print(json.dumps(method.__dict__, ensure_ascii=False, indent=2))
return 0
raise SystemExit(f"unknown methods command: {args.methods_cmd}")
def cmd_research(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
manifest = load_manifest(project_root)
if not args.force and not (manifest.get("phase1") or {}).get("approved"):
raise SystemExit(
"Phase 1 is not approved. Review phase1/material_brief.md and phase1/framework.md, "
"then run `uv run python scripts/dr.py approve <project>` or pass --force."
)
if args.profile == "codex_native" and (args.execute_packets or args.assemble_chapters):
raise SystemExit(
"`codex_native` cannot be used for Python-core model execution: scripts/dr.py currently calls external "
"API clients, not Codex App built-in models. Use a clearly external profile such as `medium`, or run "
"Codex-native execution through the surface adapter/manual task workflow."
)
runtime = resolve_runtime_profile(
profile=args.profile,
overrides=parse_model_overrides(args.model_override),
)
if args.append_task_cards:
generated = create_phase2_task_cards(
project_root,
axes=args.axis,
dry_run=True,
)
existing_path = project_root / "phase2" / "task_cards.json"
existing_cards = load_task_cards(existing_path) if existing_path.exists() else []
seen = {card.task_id for card in existing_cards}
appended_cards = [TaskCard(**item) for item in generated if item["task_id"] not in seen]
runnable_existing_cards: list[TaskCard] = []
if args.execute_packets and args.axis:
axis_set = set(args.axis)
for card in existing_cards:
if card.topic_axis not in axis_set:
continue
packet_path = project_root / card.output_packet
try:
validate_packet(json.loads(packet_path.read_text(encoding="utf-8")))
except Exception:
runnable_existing_cards.append(card)
merged_cards = [*existing_cards, *appended_cards]
if not args.dry_run:
write_task_cards(existing_path, merged_cards)
phase2 = manifest.setdefault("phase2", {})
phase2.update(
{
"status": "in_progress",
"runtime": "python-core-v0.20",
"task_cards_path": "phase2/task_cards.json",
"task_cards_total": len(merged_cards),
"task_cards_appended": len(appended_cards),
"updated_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(),
}
)
(project_root / "manifest.json").write_text(
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
card_dicts = [card.to_dict() for card in [*appended_cards, *runnable_existing_cards]]
else:
card_dicts = create_phase2_task_cards(
project_root,
axes=args.axis,
dry_run=args.dry_run,
)
if args.execute_packets and args.dry_run:
raise SystemExit("--execute-packets cannot be combined with --dry-run")
if args.assemble_chapters and args.dry_run:
raise SystemExit("--assemble-chapters cannot be combined with --dry-run")
if args.execute_packets:
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets
from scripts.runtime.workers import ProjectSearchProvider
load_secrets()
def client_factory(_role):
return ZenMuxClient(log_file=project_root / "phase2" / "logs" / "packets.jsonl")
def search_provider_factory():
return ProjectSearchProvider(strict_specialized=not args.allow_search_fallback)
packet_count = run_packet_workers(
project_root=project_root,
cards=[TaskCard(**item) for item in card_dicts],
runtime=runtime,
client_factory=client_factory,
search_provider_factory=search_provider_factory,
workers=args.workers,
)
elif not (args.build_briefs or args.assemble_chapters):
packet_count = write_placeholder_packets(project_root, card_dicts, dry_run=args.dry_run)
else:
packet_count = len(list((project_root / "phase2" / "packets").glob("*.json")))
brief_count = 0
chapter_count = 0
compressed_count = 0
source_count = None
if args.build_briefs or args.assemble_chapters:
source_count = rebuild_sources_from_packets(project_root)
briefs = build_chapter_briefs(project_root)
brief_count = len(briefs)
compressed_findings = build_compressed_findings(project_root)
compressed_count = len(compressed_findings)
if args.assemble_chapters:
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets
load_secrets()
def chapter_client_factory(_role):
return ZenMuxClient(log_file=project_root / "phase2" / "logs" / "chapters.jsonl")
chapter_count = run_chapter_assembly_workers(
project_root=project_root,
briefs=compressed_findings,
runtime=runtime,
client_factory=chapter_client_factory,
workers=args.workers,
)
print(f"Project: {project_root.name}")
print(f"Runtime: python-core-v0.20")
print(f"Model profile: {runtime.profile}")
print(f"Workers: {args.workers}")
print(f"Task cards: {len(card_dicts)}")
print(f"Packets: {packet_count}")
if source_count is not None:
print(f"Sources rebuilt: {source_count}")
if args.build_briefs or args.assemble_chapters:
print(f"Chapter briefs: {brief_count}")
print(f"Compressed findings: {compressed_count}")
if args.assemble_chapters:
print(f"Chapter drafts: {chapter_count}")
if args.dry_run:
print("Dry run: no files written")
elif args.assemble_chapters:
print("Wrote: phase2/drafts/chXX.md")
elif args.execute_packets:
print("Wrote: phase2/task_cards.json and validated phase2/packets/*.json")
print("Next: rerun with --build-briefs to aggregate packets into chapter briefs.")
elif args.build_briefs:
print("Wrote: phase2/chapter_briefs/*.json and phase2/compressed_findings/*.json")
print("Next: rerun with --assemble-chapters to write Chinese chapter drafts.")
else:
print("Wrote: phase2/task_cards.json and phase2/packets/*.json")
print("Next: rerun with --execute-packets to fill packets via model workers.")
return 0
def cmd_run(args: argparse.Namespace) -> int:
target = args.project_or_topic
projects_dir = Path(args.projects_dir) if args.projects_dir else PROJECTS_DIR
try:
project_root = resolve_project(target, projects_dir=projects_dir)
except SystemExit:
if args.dry_run:
print(f"New topic detected: {target}")
print("Dry run: would create project and write phase1/framework.md")
return 0
project_root = create_project(
topic=target,
slug=args.slug,
projects_dir=projects_dir,
method_key=args.method,
report_type=args.report_type,
model_profile=args.profile or "medium",
target_words=args.target_words,
input_materials=args.input_material,
)
framework = render_framework(project_root, chapter_count=args.chapters)
print(f"Project: {project_root.name}")
print("Runtime: python-core-v0.20")
print(f"Created: {project_root}")
print(f"Wrote: {framework.relative_to(project_root)}")
print("Pause: review and approve the framework before Phase 2.")
return 0
print(f"Project: {project_root.name}")
print("Runtime: python-core-v0.20")
print("Next command: research")
if args.dry_run:
print("Dry run: would inspect manifest and continue from the next incomplete phase")
return 0
return cmd_research(
argparse.Namespace(
project=str(project_root),
workers=args.workers,
axis=None,
profile=args.profile,
model_override=[],
execute_packets=False,
append_task_cards=False,
allow_search_fallback=False,
build_briefs=False,
assemble_chapters=False,
force=False,
dry_run=False,
)
)
def cmd_review(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
if args.model_review:
if args.dry_run:
print(f"Project: {project_root.name}")
print("Phase 3 model review plan:")
print(f" model: {args.model}")
print(" context: phase3/review_context_opus_4_7.md")
print(" output: phase3/critique.md")
return 0
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets
load_secrets()
with ZenMuxClient(log_file=project_root / "phase3" / "logs" / "review.jsonl") as client:
path = build_phase3_model_critique(
project_root,
client=client,
model=args.model,
max_context_chars=args.max_context_chars,
)
else:
path = build_phase3_critique(project_root)
print(f"Project: {project_root.name}")
print(f"Wrote: {path.relative_to(project_root)}")
print("Pause: review critique before Phase 4.")
return 0
def cmd_status(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
manifest = load_manifest(project_root)
slug = project_root.name
drafts = sorted((project_root / "phase2" / "drafts").glob("ch*.md"))
evidence = sorted((project_root / "phase2" / "evidence").glob("ch*-evidence.md"))
task_cards = project_root / "phase2" / "task_cards.json"
research_brief = project_root / "phase1" / "research_brief.json"
compressed_findings = sorted((project_root / "phase2" / "compressed_findings").glob("ch*.json"))
packet_counts = packet_state_counts(project_root)
sources = project_root / "phase2" / "sources.jsonl"
final_en = project_root / "phase4" / "final_en.md"
final_zh = project_root / "phase4" / "final_zh.md"
final_zh_polished = project_root / "phase4" / "final_zh_polished.md"
glossary = project_root / "phase4" / "glossary.json"
en_words = count_words(final_en.read_text(encoding="utf-8")) if final_en.exists() else 0
zh_source = final_zh_polished if final_zh_polished.exists() else final_zh
zh_chars = count_chinese_chars(zh_source.read_text(encoding="utf-8")) if zh_source.exists() else 0
source_count = 0
if sources.exists():
source_count = sum(1 for line in sources.read_text(encoding="utf-8").splitlines() if line.strip())
print(f"Project: {manifest.get('topic', slug)}")
print(f"Slug: {slug}")
print(f"Title: {manifest.get('report_title', '(unset)')}")
print(f"Type: {manifest.get('type', '(unset)')}")
print()
print("Phases:")
for phase in ("phase1", "phase2", "phase3", "phase4"):
p = manifest.get(phase, {})
print(f" {phase}: {p.get('status', 'pending')} approved={p.get('approved', False)}")
print()
print("Artifacts:")
print(f" framework: {file_state(project_root / 'phase1' / 'framework.md')}")
print(f" research_brief.json: {file_state(research_brief)}")
print(f" task_cards.json: {file_state(task_cards)}")
print(
" packets: "
f"ready={packet_counts['ready']} "
f"placeholder={packet_counts['placeholder']} "
f"invalid={packet_counts['invalid']} "
f"errors={packet_counts['errors']} "
f"stale_errors={packet_counts['stale_errors']} "
f"total={packet_counts['total']}"
)
print(f" drafts: {len(drafts)}")
print(f" compressed findings: {len(compressed_findings)}")
print(f" evidence files: {len(evidence)}")
print(f" sources: {source_count}")
print(f" final_en.md: {file_state(final_en)} ({en_words:,} words)")
print(f" final_zh.md: {file_state(final_zh)}")
print(f" final_zh_polished.md: {file_state(final_zh_polished)} ({zh_chars:,} Chinese chars)")
print(f" glossary.json: {file_state(glossary)}")
return 0
def cmd_prompt(args: argparse.Namespace) -> int:
name = args.command
if not name.startswith("dr-"):
name = f"dr-{name}"
candidates = [
CODEX_COMMAND_TEMPLATES_DIR / f"{name}.md",
LEGACY_CODEX_COMMANDS_DIR / f"{name}.md",
]
path = next((candidate for candidate in candidates if candidate.exists()), None)
if path is None:
raise SystemExit(f"Codex command template not found: {candidates[0]}")
text = path.read_text(encoding="utf-8")
if args.argument:
text = text.replace("$ARGUMENTS", args.argument)
else:
text = text.replace("$ARGUMENTS", "")
print(text)
return 0
def cmd_glossary(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
cmd = [
sys.executable,
str(REPO_ROOT / "scripts" / "build_glossary.py"),
str(project_root),
"--workers",
str(args.workers),
]
if args.force:
cmd.append("--force")
if args.only:
cmd += ["--only", args.only]
if args.input:
cmd += ["--input", args.input]
if args.output:
cmd += ["--output", args.output]
return run_cmd(cmd, dry_run=args.dry_run)
def cmd_finalize(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
manifest = load_manifest(project_root)
effective_profile = args.model_profile or manifest.get("model_profile")
try:
resolved = resolve_model_profile(
profile=effective_profile,
overrides=parse_model_overrides(args.model_override),
)
except ModelConfigError as exc:
raise SystemExit(f"model profile resolution failed: {exc}") from exc
roles = resolved["roles"]
if not args.legacy_translate:
final_input = args.input
if args.number_citations and not args.dry_run:
rc = run_cmd(
[
sys.executable,
str(REPO_ROOT / "scripts" / "number_citations.py"),
str(project_root),
"--input",
args.input,
"--output",
args.numbered_output,
],
dry_run=False,
)
if rc != 0:
return rc
final_input = args.numbered_output
elif args.number_citations:
final_input = args.numbered_output
cmd: list[str] = [
sys.executable,
str(REPO_ROOT / "scripts" / "build_report.py"),
str(project_root),
"--input",
final_input,
]
if args.report_engine:
cmd += ["--engine", args.report_engine]
if args.no_docx:
cmd.append("--no-docx")
if args.no_pdf:
cmd.append("--no-pdf")
if args.dry_run:
print("Chinese-native finalize plan:")
if args.number_citations:
print(
"$ "
+ " ".join(
[
sys.executable,
str(REPO_ROOT / "scripts" / "number_citations.py"),
str(project_root),
"--input",
args.input,
"--output",
args.numbered_output,
]
)
)
print("$ " + " ".join(cmd))
if args.polish:
print(
"$ "
+ " ".join(
[
sys.executable,
str(REPO_ROOT / "scripts" / "polish.py"),
str(project_root),
"--source",
final_input,
"--workers",
str(args.polish_workers),
"--model",
roles.get("polish", "anthropic/claude-sonnet-4.6"),
]
)
)
return 0
if args.polish:
rc = run_cmd(
[
sys.executable,
str(REPO_ROOT / "scripts" / "polish.py"),
str(project_root),
"--source",
final_input,
"--workers",
str(args.polish_workers),
"--model",
roles.get("polish", "anthropic/claude-sonnet-4.6"),
],
dry_run=False,
)
if rc != 0:
return rc
return run_cmd(cmd, dry_run=False)
cmd = [
sys.executable,
str(REPO_ROOT / "scripts" / "phase4_pipeline.py"),
str(project_root),
"--translate-workers",
str(args.translate_workers),
"--glossary-workers",
str(args.glossary_workers),
"--polish-workers",
str(args.polish_workers),
"--translate-model",
roles.get("translate", "anthropic/claude-sonnet-4.6"),
"--glossary-model",
roles.get("glossary", "anthropic/claude-haiku-4.5"),
"--polish-model",
roles.get("polish", "anthropic/claude-sonnet-4.6"),
"--glossary-mode",
args.glossary_mode,
]
if args.dry_run:
cmd.append("--dry-run")
return run_cmd(cmd, dry_run=False)
def cmd_models(args: argparse.Namespace) -> int:
if args.list:
for name in list_model_profiles():
print(name)
return 0
try:
resolved = resolve_model_profile(
profile=args.profile,
overrides=parse_model_overrides(args.model_override),
)
except ModelConfigError as exc:
raise SystemExit(f"model profile resolution failed: {exc}") from exc
if args.probe:
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets, normalize_zenmux_model
load_secrets()
results = []
with ZenMuxClient() as client:
for requested_model in sorted(set(resolved["roles"].values())):
api_model = normalize_zenmux_model(requested_model)
try:
content = client.chat_complete(
model=requested_model,
system="Health check.",
user="Reply with OK only.",
temperature=0,
max_tokens=16,
tag=f"models:probe:{requested_model}",
)
results.append({
"requested_model": requested_model,
"api_model": api_model,
"ok": True,
"response": content.strip()[:80],
})
except Exception as exc: # noqa: BLE001 - probe should report every model.
results.append({
"requested_model": requested_model,
"api_model": api_model,
"ok": False,
"error": str(exc)[:500],
})
if args.json:
print(json.dumps({**resolved, "probe": results}, ensure_ascii=False, indent=2))
else:
print(f"Profile: {resolved['profile']}")
print("Model probe:")
for item in results:
status = "ok" if item["ok"] else "fail"
print(f" {status} {item['requested_model']} -> {item['api_model']}")
if not item["ok"]:
print(f" {item['error']}")
return 0 if all(item["ok"] for item in results) else 1
if args.json:
print(json.dumps(resolved, ensure_ascii=False, indent=2))
return 0
print(f"Profile: {resolved['profile']}")
if resolved["description"]:
print(f"Description: {resolved['description']}")
print("Roles:")
for role in sorted(resolved["roles"]):
print(f" {role}: {resolved['roles'][role]}")
if resolved.get("task_types"):
print("Task types:")
for task_type in sorted(resolved["task_types"]):
print(f" {task_type}: {resolved['task_types'][task_type]}")
return 0
def cmd_sources(args: argparse.Namespace) -> int:
project_root = resolve_project(args.project)
if args.sources_cmd == "cache":
if args.dry_run:
print(f"Project: {project_root.name}")
print(f"Would cache sources from: {args.sources}")
print(f"Important only: {not args.all}")
print(f"Limit: {args.limit}")
return 0
results = cache_sources(
project_root,
sources_rel=args.sources,
important_only=not args.all,
limit=args.limit,
force=args.force,
)
print(f"Project: {project_root.name}")
print(f"Cached source snapshots: {len(results)}")
print("Wrote: phase2/source_cache/md/*.md")
print("Updated: phase2/sources.jsonl")
return 0
raise SystemExit(f"unknown sources command: {args.sources_cmd}")
def cmd_apply_models(args: argparse.Namespace) -> int:
cmd = [
sys.executable,
str(REPO_ROOT / "scripts" / "apply_model_profile.py"),
"--profile",
args.profile,
"--target",
args.target,
]
for item in args.model_override:
cmd += ["--model-override", item]
if args.dry_run:
cmd.append("--dry-run")
return run_cmd(cmd, dry_run=False)
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Deep Research platform-neutral CLI")
sub = parser.add_subparsers(dest="cmd", required=True)
init = sub.add_parser("init", help="Initialize a Python-core research project")
init.add_argument("topic", help="Research topic")
init.add_argument("--slug", help="Project slug")
init.add_argument("--method", help="Research method key")
init.add_argument("--type", dest="report_type", default="research", help="Report type")
init.add_argument("--profile", default="medium", help="Model profile name from configs/models.yaml")
init.add_argument("--target-words", type=int, default=30000)
init.add_argument("--input-material", action="append", default=[], help="Path or note for user-provided material")
init.add_argument("--projects-dir", help="Override projects directory")
init.set_defaults(func=cmd_init)
approve = sub.add_parser("approve", help="Approve Phase 1 gates before Phase 2")
approve.add_argument("project", help="Project slug or path")
approve.set_defaults(func=cmd_approve)
frame = sub.add_parser("frame", help="Generate Phase 1 framework.md")
frame.add_argument("project", help="Project slug or path")
frame.add_argument("--method", help="Override research method key")
frame.add_argument("--chapters", type=int, default=10)
frame.add_argument("--preserve-existing-outline", action="store_true", help="Keep current framework chapter titles and enrich Phase 1 planning")
frame.add_argument("--dry-run", action="store_true")
frame.set_defaults(func=cmd_frame)
run = sub.add_parser("run", help="Run the platform-neutral Python-core workflow")
run.add_argument("project_or_topic", help="Project slug/path or new topic")
run.add_argument("--workers", type=int, default=6)
run.add_argument("--profile", help="Model profile name from configs/models.yaml")
run.add_argument("--slug", help="Project slug when project_or_topic is new")
run.add_argument("--method", help="Research method key when project_or_topic is new")
run.add_argument("--type", dest="report_type", default="research", help="Report type for new project")
run.add_argument("--target-words", type=int, default=30000)
run.add_argument("--chapters", type=int, default=10)
run.add_argument("--input-material", action="append", default=[])
run.add_argument("--projects-dir", help="Override projects directory")
run.add_argument("--dry-run", action="store_true")
run.set_defaults(func=cmd_run)
research = sub.add_parser("research", help="Run v0.20 task-card Phase 2")
research.add_argument("project", help="Project slug or path")
research.add_argument("--workers", type=int, default=6)
research.add_argument("--axis", action="append", help="Restrict generated task axes; repeatable")
research.add_argument("--profile", help="Model profile name from configs/models.yaml")
research.add_argument("--model-override", action="append", default=[], metavar="ROLE=MODEL", help="Override a role model for this run; repeatable")
research.add_argument("--execute-packets", action="store_true", help="Call model workers to fill evidence packets")
research.add_argument("--append-task-cards", action="store_true", help="Append newly generated task cards instead of replacing phase2/task_cards.json")
research.add_argument("--allow-search-fallback", action="store_true", help="Allow generic search fallback for specialized routes")
research.add_argument("--build-briefs", action="store_true", help="Aggregate packets into chapter briefs")
research.add_argument("--assemble-chapters", action="store_true", help="Call model workers to write Chinese chapter drafts")
research.add_argument("--force", action="store_true", help="bypass Phase 1 approval gate")
research.add_argument("--dry-run", action="store_true")
research.set_defaults(func=cmd_research)
skills = sub.add_parser("skills", help="Manage canonical skills")
skill_sub = skills.add_subparsers(dest="skills_cmd", required=True)
skill_sub.add_parser("list", help="List canonical skills").set_defaults(func=cmd_skills)
skill_sub.add_parser("validate", help="Validate canonical skills").set_defaults(func=cmd_skills)
skill_sync = skill_sub.add_parser("sync", help="Sync skills into adapter directories")
skill_sync.add_argument("--target", action="append", help="Target skill directory; repeatable")
skill_sync.set_defaults(func=cmd_skills)
methods = sub.add_parser("methods", help="List and inspect research framework methods")
method_sub = methods.add_subparsers(dest="methods_cmd", required=True)
method_sub.add_parser("list", help="List research methods").set_defaults(func=cmd_methods)
method_show = method_sub.add_parser("show", help="Show one research method")
method_show.add_argument("method")
method_show.set_defaults(func=cmd_methods)
status = sub.add_parser("status", help="Show project status")
status.add_argument("project", nargs="?", help="Project slug or path")
status.set_defaults(func=cmd_status)
review = sub.add_parser("review", help="Run Phase 3 review")
review.add_argument("project", help="Project slug or path")
review.add_argument("--model-review", action="store_true", help="Run independent model-based Phase 3 review")
review.add_argument("--model", default="zenmux-anthropic/claude-opus-4-7", help="Model for --model-review")
review.add_argument("--max-context-chars", type=int, default=650_000, help="Bounded context size for model review")
review.add_argument("--dry-run", action="store_true")
review.set_defaults(func=cmd_review)
prompt = sub.add_parser("prompt", help="Print a Codex command prompt template")
prompt.add_argument("command", help="Command name, e.g. dr-frame or frame")
prompt.add_argument("argument", nargs="?", help="Replacement for $ARGUMENTS")
prompt.set_defaults(func=cmd_prompt)
glossary = sub.add_parser("glossary", help="Run glossary verification")
glossary.add_argument("project", help="Project slug or path")
glossary.add_argument("--workers", type=int, default=4)
glossary.add_argument("--force", action="store_true")
glossary.add_argument("--only")
glossary.add_argument("--input")
glossary.add_argument("--output")
glossary.add_argument("--dry-run", action="store_true")
glossary.set_defaults(func=cmd_glossary)
finalize = sub.add_parser("finalize", help="Run Phase 4 deterministic pipeline")
finalize.add_argument("project", help="Project slug or path")
finalize.add_argument("--input", default="phase4/final_zh.md", help="Chinese Markdown source for default v0.20 finalization")
finalize.add_argument("--legacy-translate", action="store_true", help="Use legacy final_en -> translate -> polish pipeline")
finalize.add_argument("--polish", action="store_true", help="Run optional Chinese polish step before rendering")
finalize.add_argument("--number-citations", action="store_true", help="Convert [src_xxx] citations to numeric references before rendering")
finalize.add_argument("--numbered-output", default="phase4/final_zh_numbered.md", help="Output path for numeric citation Markdown")
finalize.add_argument("--report-engine", choices=["reportlab", "quarto"], default=None)
finalize.add_argument("--no-docx", action="store_true")
finalize.add_argument("--no-pdf", action="store_true")
finalize.add_argument("--translate-workers", type=int, default=0)
finalize.add_argument("--glossary-workers", type=int, default=4)
finalize.add_argument("--polish-workers", type=int, default=0)
finalize.add_argument(
"--glossary-mode",
choices=["off", "low-confidence", "full"],
default="low-confidence",
)
finalize.add_argument("--model-profile", help="Model profile name from configs/models.yaml")
finalize.add_argument(
"--model-override",
action="append",
default=[],
metavar="ROLE=MODEL",
help="Override one role model, repeatable",
)
finalize.add_argument("--dry-run", action="store_true")
finalize.set_defaults(func=cmd_finalize)
models = sub.add_parser("models", help="Resolve and print model profile")
models.add_argument("--profile", help="Profile name from configs/models.yaml")
models.add_argument("--list", action="store_true", help="List available profiles")
models.add_argument(
"--model-override",
action="append",
default=[],
metavar="ROLE=MODEL",
help="Override one role model, repeatable",
)
models.add_argument("--probe", action="store_true", help="Send tiny health checks to resolved role models")
models.add_argument("--json", action="store_true", help="Emit JSON")
models.set_defaults(func=cmd_models)
sources = sub.add_parser("sources", help="Manage source snapshots and source registry")
sources_sub = sources.add_subparsers(dest="sources_cmd", required=True)
sources_cache = sources_sub.add_parser("cache", help="Cache important sources as local Markdown snapshots")
sources_cache.add_argument("project", help="Project slug or path")
sources_cache.add_argument("--sources", default="phase2/sources.jsonl", help="Source registry path relative to project")
sources_cache.add_argument("--all", action="store_true", help="Cache all remote sources, not only important official/Tier 1 sources")
sources_cache.add_argument("--limit", type=int, help="Maximum sources to cache in this run")
sources_cache.add_argument("--force", action="store_true", help="Refetch even if cached_text_path already exists")
sources_cache.add_argument("--dry-run", action="store_true")
sources_cache.set_defaults(func=cmd_sources)
apply_models = sub.add_parser("apply-models", help="Apply profile to agent files")
apply_models.add_argument("--profile", required=True, help="Profile name from configs/models.yaml")
apply_models.add_argument("--target", choices=["opencode", "codex", "both"], default="both")
apply_models.add_argument(
"--model-override",
action="append",
default=[],
metavar="ROLE=MODEL",
help="Override one role model, repeatable",
)
apply_models.add_argument("--dry-run", action="store_true")
apply_models.set_defaults(func=cmd_apply_models)
return parser
def main() -> int:
parser = build_parser()
args = parser.parse_args()
return args.func(args)
if __name__ == "__main__":
raise SystemExit(main())
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env python3
"""Export a clean Antigravity-only workspace from a full checkout."""
from __future__ import annotations
import argparse
import shutil
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
FILES = [
"AGENTS.md",
"GEMINI.md",
"README.md",
"docs/antigravity-clean-workspace.md",
"docs/platform-adapters.md",
"docs/platform-branch-strategy.md",
"scripts/dr.py",
"scripts/deploy_adapters.py",
"scripts/update_platform_envs.py",
]
DIRS = [
"configs",
"scripts/runtime",
"scripts/reporting",
]
ANTIGRAVITY_AGENT_SRC = REPO_ROOT / "platform_adapters" / "antigravity" / "agent"
EMPTY_DIRS = [
"projects",
]
EXCLUDED_NAMES = {
"__pycache__",
".pytest_cache",
}
def copy_file(src: Path, dst: Path, *, dry_run: bool) -> None:
if dry_run:
print(f"file {src.relative_to(REPO_ROOT)} -> {dst}")
return
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dst)
def ignore_names(_: str, names: list[str]) -> set[str]:
return {name for name in names if name in EXCLUDED_NAMES}
def copy_dir(src: Path, dst: Path, *, dry_run: bool) -> None:
if dry_run:
print(f"dir {src.relative_to(REPO_ROOT)} -> {dst}")
return
if dst.exists():
shutil.rmtree(dst)
shutil.copytree(src, dst, ignore=ignore_names)
def export_workspace(target: Path, *, force: bool, dry_run: bool) -> None:
target = target.expanduser().resolve()
if target == REPO_ROOT:
raise SystemExit("target must not be the full repository root")
if target.exists() and any(target.iterdir()) and not force:
raise SystemExit(f"target is not empty: {target}; use --force to replace managed files")
if not dry_run:
target.mkdir(parents=True, exist_ok=True)
for rel in FILES:
src = REPO_ROOT / rel
if not src.exists():
raise FileNotFoundError(src)
copy_file(src, target / rel, dry_run=dry_run)
for rel in DIRS:
src = REPO_ROOT / rel
if not src.exists():
raise FileNotFoundError(src)
copy_dir(src, target / rel, dry_run=dry_run)
if not ANTIGRAVITY_AGENT_SRC.exists():
raise FileNotFoundError(ANTIGRAVITY_AGENT_SRC)
copy_dir(ANTIGRAVITY_AGENT_SRC, target / ".agent", dry_run=dry_run)
for rel in EMPTY_DIRS:
if dry_run:
print(f"dir {rel} -> {target / rel}")
else:
(target / rel).mkdir(parents=True, exist_ok=True)
keep = target / rel / ".gitkeep"
keep.touch(exist_ok=True)
if not dry_run:
print(f"exported clean Antigravity workspace: {target}")
print("open this target directory in Antigravity")
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Export a clean Antigravity-only workspace")
parser.add_argument("--target", required=True, type=Path, help="output workspace directory")
parser.add_argument("--force", action="store_true", help="replace managed files in a non-empty target")
parser.add_argument("--dry-run", action="store_true", help="show what would be copied")
return parser
def main() -> int:
args = build_parser().parse_args()
export_workspace(args.target, force=args.force, dry_run=args.dry_run)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env python3
"""Native web grounding wrapper via ZenMux chat completions.
Use this when you need reproducible, model-native web search (grounding) and
machine-readable citations.
"""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Grounded web query via ZenMux")
parser.add_argument("query", help="Question or search prompt")
parser.add_argument("--model", default="google/gemini-3.1-flash-lite-preview")
parser.add_argument("--max-tokens", type=int, default=2400)
parser.add_argument("--temperature", type=float, default=0.2)
parser.add_argument("--json", action="store_true", help="Emit JSON envelope")
parser.add_argument("--log-file", help="Optional JSONL call log path")
parser.add_argument("--system", default=(
"You are a research assistant. Use web grounding when helpful. "
"Return concise facts with explicit source-backed statements."
))
return parser
def main() -> int:
args = build_parser().parse_args()
load_secrets()
log_file = Path(args.log_file) if args.log_file else None
with ZenMuxClient(log_file=log_file) as client:
result = client.chat_complete_with_meta(
model=args.model,
system=args.system,
user=args.query,
temperature=args.temperature,
max_tokens=args.max_tokens,
web_search=True,
web_search_options={},
tag="ground",
)
if args.json:
payload = {
"query": args.query,
"model": args.model,
"content": result["content"],
"citations": result["citations"],
"usage": result["usage"],
}
print(json.dumps(payload, ensure_ascii=False, indent=2))
else:
print(result["content"])
if result["citations"]:
print("\nCitations:")
for idx, url in enumerate(result["citations"], start=1):
print(f"{idx}. {url}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""Backward-compatible wrapper for deploying the Codex adapter.
v0.20 keeps Codex adapter templates in the repository, but deploys the usable
adapter files to a Codex home outside the checkout.
"""
from __future__ import annotations
import argparse
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from scripts.deploy_adapters import deploy_codex, print_result
def main() -> int:
parser = argparse.ArgumentParser(description="Deploy Codex native adapter")
parser.add_argument("--target", type=Path, help="Codex home target; defaults to $CODEX_HOME or ~/.codex")
parser.add_argument("--force", action="store_true", help="overwrite existing files and create .bak backups")
parser.add_argument("--skip-skills", action="store_true", help="do not copy canonical skills into target/skills")
parser.add_argument("--include-config", action="store_true", help="also copy config.toml; off by default to avoid overwriting global Codex config")
parser.add_argument("--dry-run", action="store_true", help="show files that would be written")
args = parser.parse_args()
result = deploy_codex(
target=args.target,
force=args.force,
skip_skills=args.skip_skills,
include_config=args.include_config,
dry_run=args.dry_run,
)
print_result(result)
print()
print("Note: this no longer writes repository-local .codex files by default.")
print("Run Codex from this repository after deployment:")
if args.include_config:
print(" codex --profile deep-research")
else:
print(" codex")
print("Note: config.toml is not copied by default. Use --include-config only if you want the bundled profile.")
return 0
if __name__ == "__main__":
raise SystemExit(main())
View File
+150
View File
@@ -0,0 +1,150 @@
"""Markdown 切块 / 合并工具。
把一篇 Markdown 按标题层级(# / ##)切成"翻译单元""润色单元"
每个单元带稳定的 ID,便于断点续传和按需重跑。
核心约定:
- H1(`# `)是一级块,通常对应 Chapter / 封面 / 前置件
- H2(`## `)是二级块,对应一个 section 或独立前置件(Disclaimer / Executive Summary / Abstract / Glossary / References
- 没有任何标题的文件头(frontmatter 区)归到第 0 块
切块粒度默认到 H2;如果某个 H2 下的正文特别长可以再按 H3 切,但这是 polish 的事情,
translate 一般不需要。
"""
from __future__ import annotations
import hashlib
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Iterable
HEADER_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$", re.MULTILINE)
@dataclass
class MarkdownBlock:
"""一个翻译/润色单元。"""
order: int # 在原文中的顺序(0-based
level: int # 0 = frontmatter; 1/2/... = H1/H2/...
title: str # 标题原文(不含 # 号);frontmatter 为空串
anchor: str # 稳定 ID,用于断点续传(order + title hash
content: str # 完整内容(包含标题行本身,除 frontmatter 块外)
parent_order: int | None = None # H2 的父 H1 orderH1 为 None
word_count: int = 0 # 英文 word count 估算(只含 a-z
char_count: int = 0 # 字符数(含中文)
meta: dict = field(default_factory=dict)
@property
def short_title(self) -> str:
t = self.title.strip()
if len(t) <= 50:
return t
return t[:47] + "..."
def _count_words(text: str) -> int:
return len(re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)*", text))
def _stable_anchor(order: int, title: str) -> str:
"""order + title 生成稳定短 ID。同一文件改顺序不变,改标题重算。"""
h = hashlib.sha1(title.strip().encode("utf-8")).hexdigest()[:8]
return f"b{order:03d}-{h}"
def split_by_headers(text: str, max_level: int = 2) -> list[MarkdownBlock]:
"""把 Markdown 按 H1..H{max_level} 切块。
返回的 block 按出现顺序排列。第 0 块可能是 frontmatterlevel=0,无 title)。
"""
lines = text.splitlines(keepends=False)
# 先找出所有 header 位置
header_positions: list[tuple[int, int, str]] = [] # (line_idx, level, title)
in_code_block = False
for i, ln in enumerate(lines):
if ln.lstrip().startswith("```"):
in_code_block = not in_code_block
continue
if in_code_block:
continue
m = re.match(r"^(#{1,6})\s+(.+?)\s*$", ln)
if m:
level = len(m.group(1))
if level <= max_level:
header_positions.append((i, level, m.group(2)))
blocks: list[MarkdownBlock] = []
order = 0
# frontmatter:第一个 header 前的所有内容
first_header_line = header_positions[0][0] if header_positions else len(lines)
frontmatter_text = "\n".join(lines[:first_header_line]).rstrip()
if frontmatter_text.strip():
blocks.append(
MarkdownBlock(
order=order,
level=0,
title="",
anchor=_stable_anchor(order, "__frontmatter__"),
content=frontmatter_text,
word_count=_count_words(frontmatter_text),
char_count=len(frontmatter_text),
)
)
order += 1
# 为每个 header 建一个 block:内容 = 本 header 行 + 下一 header 行前所有内容
last_h1_order: int | None = None
for idx, (line_idx, level, title) in enumerate(header_positions):
end_line = (
header_positions[idx + 1][0]
if idx + 1 < len(header_positions)
else len(lines)
)
content = "\n".join(lines[line_idx:end_line]).rstrip()
block = MarkdownBlock(
order=order,
level=level,
title=title,
anchor=_stable_anchor(order, title),
content=content,
parent_order=last_h1_order if level > 1 else None,
word_count=_count_words(content),
char_count=len(content),
)
blocks.append(block)
if level == 1:
last_h1_order = order
order += 1
return blocks
def merge_blocks(blocks: Iterable[MarkdownBlock], separator: str = "\n\n") -> str:
"""按 order 拼回完整 Markdown。"""
return separator.join(b.content for b in sorted(blocks, key=lambda x: x.order))
def chapter_stem_from_title(title: str) -> str:
"""从 `# Chapter 1 — foo bar` 生成文件名 stem 如 `ch01`。找不到就 fallback。"""
m = re.search(r"chapter\s+(\d+)", title, re.IGNORECASE)
if m:
return f"ch{int(m.group(1)):02d}"
# Executive Summary / Abstract / Glossary / Disclaimer / References / Version History
slug = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")
return slug[:40] or "untitled"
def count_chinese_chars(text: str) -> int:
return sum(1 for c in text if "\u4e00" <= c <= "\u9fff")
if __name__ == "__main__":
import sys
src = Path(sys.argv[1])
for b in split_by_headers(src.read_text(encoding="utf-8")):
print(f"[{b.order:3d}] L{b.level} {b.word_count:>5}w {b.char_count:>6}c {b.short_title}")
+84
View File
@@ -0,0 +1,84 @@
"""Model profile loading and resolution utilities."""
from __future__ import annotations
from pathlib import Path
from typing import Any
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
DEFAULT_MODEL_CONFIG = REPO_ROOT / "configs" / "models.yaml"
LEGACY_MODEL_CONFIG = REPO_ROOT / "configs" / "model_profiles.yaml"
class ModelConfigError(RuntimeError):
pass
def load_model_config(path: Path | None = None) -> dict[str, Any]:
cfg_path = path or DEFAULT_MODEL_CONFIG
if not cfg_path.exists() and LEGACY_MODEL_CONFIG.exists():
cfg_path = LEGACY_MODEL_CONFIG
if not cfg_path.exists():
raise ModelConfigError(f"model config not found: {cfg_path}")
try:
data = yaml.safe_load(cfg_path.read_text(encoding="utf-8")) or {}
except Exception as exc:
raise ModelConfigError(f"invalid YAML in {cfg_path}: {exc}") from exc
if not isinstance(data, dict):
raise ModelConfigError(f"invalid model config shape in {cfg_path}")
return data
def resolve_model_profile(
*,
profile: str | None = None,
overrides: dict[str, str] | None = None,
path: Path | None = None,
) -> dict[str, Any]:
cfg = load_model_config(path)
profiles = cfg.get("profiles") or {}
defaults = cfg.get("defaults") or {}
selected = profile or defaults.get("profile")
if not selected:
raise ModelConfigError("no model profile provided and no defaults.profile set")
if selected not in profiles:
raise ModelConfigError(f"unknown model profile: {selected}")
selected_profile = profiles[selected] or {}
roles = dict(selected_profile.get("roles") or {})
task_types = dict(selected_profile.get("task_types") or defaults.get("task_types") or {})
if defaults.get("script_models"):
for role, model in (defaults.get("script_models") or {}).items():
roles.setdefault(role, model)
for role, model in (overrides or {}).items():
roles[role] = model
return {
"profile": selected,
"description": selected_profile.get("description", ""),
"roles": roles,
"task_types": task_types,
}
def list_model_profiles(path: Path | None = None) -> list[str]:
cfg = load_model_config(path)
profiles = cfg.get("profiles") or {}
return sorted(profiles.keys())
def parse_model_overrides(items: list[str] | None) -> dict[str, str]:
out: dict[str, str] = {}
for item in items or []:
if "=" not in item:
raise ModelConfigError(f"invalid override '{item}', expected role=model")
role, model = item.split("=", 1)
role = role.strip()
model = model.strip()
if not role or not model:
raise ModelConfigError(f"invalid override '{item}', expected role=model")
out[role] = model
return out
+427
View File
@@ -0,0 +1,427 @@
"""通用搜索客户端(Tavily / Exa / Brave / Serper 路由)。
为 build_glossary.py 这类术语核查场景服务。
关键设计:
- `trust_env=False` 绕开系统 socks 代理(Clash on macOS 配 socks5 时 httpx 会 TLS EOF
- 专利 / Scholar / News 优先 Serper,保证 Google Patents / Google Scholar 路径被真正调用
- 通用网页 Tavily 优先,Exa/Brave fallback
- 证据发现 Exa 优先,用 highlights/text 摘录喂给 evidence packet
- 遇到配额问题自动降级或返回 empty
- 不做深度 crawl,只要摘要
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Any
import httpx
@dataclass
class SearchHit:
title: str
url: str
snippet: str
class SearchError(RuntimeError):
pass
class ExaClient:
def __init__(self, api_key: str | None = None, timeout: float = 30.0) -> None:
self.api_key = api_key or os.environ.get("EXA_API_KEY")
if not self.api_key:
raise SearchError("EXA_API_KEY not set")
# trust_env=False 关键:不吃系统代理,避免 TLS EOF
self._client = httpx.Client(trust_env=False, timeout=timeout)
def close(self) -> None:
self._client.close()
def __enter__(self) -> "ExaClient":
return self
def __exit__(self, *_args: Any) -> None:
self.close()
def search(
self,
query: str,
*,
num_results: int = 5,
search_type: str = "auto",
category: str | None = None,
use_highlights: bool = False,
max_characters: int = 800,
) -> list[SearchHit]:
body = {
"query": query,
"numResults": num_results,
"type": search_type,
"contents": {"text": {"maxCharacters": max_characters}},
}
if category:
body["category"] = category
if use_highlights:
body["contents"]["highlights"] = {
"numSentences": 2,
"highlightsPerUrl": 3,
}
r = self._client.post(
"https://api.exa.ai/search",
json=body,
headers={"x-api-key": self.api_key, "Content-Type": "application/json"},
)
if r.status_code != 200:
raise SearchError(f"Exa HTTP {r.status_code}: {r.text[:200]}")
data = r.json()
out: list[SearchHit] = []
for item in data.get("results", [])[:num_results]:
highlights = item.get("highlights") or []
text = item.get("text") or item.get("snippet") or ""
if highlights:
text = " | ".join(str(h).strip() for h in highlights if str(h).strip())
out.append(
SearchHit(
title=(item.get("title") or "")[:200],
url=item.get("url") or "",
snippet=text[:1000],
)
)
return out
class TavilyClient:
def __init__(self, api_key: str | None = None, timeout: float = 30.0) -> None:
self.api_key = api_key or os.environ.get("TAVILY_API_KEY")
if not self.api_key:
raise SearchError("TAVILY_API_KEY not set")
self._client = httpx.Client(trust_env=False, timeout=timeout)
def close(self) -> None:
self._client.close()
def __enter__(self) -> "TavilyClient":
return self
def __exit__(self, *_args: Any) -> None:
self.close()
def search(self, query: str, *, num_results: int = 5) -> list[SearchHit]:
body = {
"api_key": self.api_key,
"query": query,
"search_depth": "basic",
"max_results": num_results,
"include_answer": False,
"include_raw_content": False,
}
r = self._client.post("https://api.tavily.com/search", json=body)
if r.status_code != 200:
raise SearchError(f"Tavily HTTP {r.status_code}: {r.text[:200]}")
data = r.json()
out: list[SearchHit] = []
for item in data.get("results", [])[:num_results]:
out.append(
SearchHit(
title=(item.get("title") or "")[:200],
url=item.get("url") or "",
snippet=(item.get("content") or "")[:600],
)
)
return out
class BraveClient:
def __init__(self, api_key: str | None = None, timeout: float = 30.0) -> None:
self.api_key = api_key or os.environ.get("BRAVE_API_KEY")
if not self.api_key:
raise SearchError("BRAVE_API_KEY not set")
self._client = httpx.Client(trust_env=False, timeout=timeout)
def close(self) -> None:
self._client.close()
def __enter__(self) -> "BraveClient":
return self
def __exit__(self, *_args: Any) -> None:
self.close()
def search(self, query: str, *, num_results: int = 5) -> list[SearchHit]:
r = self._client.get(
"https://api.search.brave.com/res/v1/web/search",
params={"q": query, "count": min(max(num_results, 1), 20)},
headers={
"X-Subscription-Token": self.api_key,
"Accept": "application/json",
},
)
if r.status_code != 200:
raise SearchError(f"Brave HTTP {r.status_code}: {r.text[:200]}")
data = r.json()
out: list[SearchHit] = []
for item in (data.get("web") or {}).get("results", [])[:num_results]:
out.append(
SearchHit(
title=(item.get("title") or "")[:200],
url=item.get("url") or "",
snippet=(item.get("description") or "")[:600],
)
)
return out
class SearchClient:
"""统一搜索门面,支持多路由:
- `search(query)`:通用网页搜索,优先 Tavily → Exa → Brave
- `evidence(query)`:证据发现,优先 Exa highlights → Tavily → Brave
- `patents(query)`:专利检索,走 SerperGoogle Patents);失败则通用搜索补刀
- `scholar(query)`:学术论文,走 Serper Scholar;失败则通用搜索补刀
- `news(query)`:新闻检索,走 Serper News;失败则通用搜索补刀
所有客户端都延迟导入 serper_client,避免没装 SERPAPI_KEY 时 import 炸。
"""
def __init__(self, *, strict_specialized: bool = True) -> None:
self._exa: ExaClient | None = None
self._tavily: TavilyClient | None = None
self._brave: BraveClient | None = None
self._serper = None # 惰性实例化
self.strict_specialized = strict_specialized
try:
self._exa = ExaClient()
except SearchError:
pass
try:
self._tavily = TavilyClient()
except SearchError:
pass
try:
self._brave = BraveClient()
except SearchError:
pass
self._has_serper_key = bool(os.environ.get("SERPER_API_KEY") or os.environ.get("SERPAPI_KEY"))
if not (self._exa or self._tavily or self._brave or self._has_serper_key):
raise SearchError("no search API key available: set SERPER_API_KEY, SERPAPI_KEY, EXA_API_KEY, TAVILY_API_KEY, or BRAVE_API_KEY")
def _get_serper(self):
"""惰性创建 SerperClient。没 key 时返回 None。"""
if self._serper is False:
return None
if self._serper is None:
try:
from scripts.lib.serper_client import SerperClient
self._serper = SerperClient()
except Exception:
self._serper = False
return None
return self._serper
def close(self) -> None:
if self._exa:
self._exa.close()
if self._tavily:
self._tavily.close()
if self._brave:
self._brave.close()
if self._serper and self._serper is not False:
self._serper.close()
def __enter__(self) -> "SearchClient":
return self
def __exit__(self, *_args: Any) -> None:
self.close()
def search(self, query: str, *, num_results: int = 5) -> list[SearchHit]:
"""通用网页搜索。Tavily 首选,Exa/Brave 备选。"""
if self._tavily:
try:
return self._tavily.search(query, num_results=num_results)
except SearchError:
pass
if self._exa:
try:
return self._exa.search(query, num_results=num_results)
except SearchError:
pass
if self._brave:
try:
return self._brave.search(query, num_results=num_results)
except SearchError:
pass
return []
def evidence(
self,
query: str,
*,
num_results: int = 10,
category: str | None = None,
) -> list[SearchHit]:
"""Evidence discovery route.
Exa is better suited for agent-facing evidence discovery because it can
return concise highlights/text per URL. Results are still candidate
sources only; downstream packets must score and trace important hits
back to original Tier 1-2 sources before making final claims.
"""
if self._exa:
try:
return self._exa.search(
query,
num_results=num_results,
search_type="auto",
category=category,
use_highlights=True,
max_characters=1200,
)
except SearchError:
pass
if self._tavily:
try:
return self._tavily.search(query, num_results=num_results)
except SearchError:
pass
if self._brave:
try:
return self._brave.search(query, num_results=num_results)
except SearchError:
pass
return []
def patents(self, query: str, *, num_results: int = 10) -> list[SearchHit]:
"""专利检索:Serper 走 Google Patents 最准。降级到通用搜索 + site 限定。"""
serper = self._get_serper()
if serper:
try:
hits = serper.patents(query, num_results=num_results)
return [SearchHit(h.title, h.url, h.snippet) for h in hits]
except Exception as exc:
if self.strict_specialized:
raise SearchError(f"serper patents failed: {exc}") from exc
# 降级:通用搜索加 site 限定
if self.strict_specialized:
raise SearchError("serper unavailable for patents route; refusing silent fallback")
return self.search(f"site:patents.google.com {query}", num_results=num_results)
def scholar(
self,
query: str,
*,
num_results: int = 10,
year_low: int | None = None,
) -> list[SearchHit]:
"""学术论文:Serper Scholar 带引用数。降级到通用搜索。"""
serper = self._get_serper()
if serper:
try:
hits = serper.scholar(query, num_results=num_results, year_low=year_low)
return [
SearchHit(
title=h.title,
url=h.url,
snippet=f"{h.snippet} | {h.source} | 引用 {h.cited_by}" if h.cited_by else h.snippet,
)
for h in hits
]
except Exception as exc:
if self.strict_specialized:
raise SearchError(f"serper scholar failed: {exc}") from exc
if self.strict_specialized:
raise SearchError("serper unavailable for scholar route; refusing silent fallback")
return self.search(query, num_results=num_results)
def news(
self,
query: str,
*,
num_results: int = 10,
time_range: str | None = None,
) -> list[SearchHit]:
"""新闻检索:Serper News。降级到通用搜索。"""
serper = self._get_serper()
if serper:
try:
hits = serper.news(query, num_results=num_results, time_range=time_range)
return [
SearchHit(
title=h.title,
url=h.url,
snippet=f"{h.snippet} | {h.source} | {h.date}" if h.date else h.snippet,
)
for h in hits
]
except Exception as exc:
if self.strict_specialized:
raise SearchError(f"serper news failed: {exc}") from exc
if self.strict_specialized:
raise SearchError("serper unavailable for news route; refusing silent fallback")
return self.search(query, num_results=num_results)
def fda(self, query: str, *, num_results: int = 10) -> list[SearchHit]:
"""FDA-focused discovery for warning letters and meeting records.
FDA enforcement examples are often more useful for GMP remediation than
generic web pages, so this route biases discovery toward warning
letters, inspection/enforcement pages, and meeting materials/minutes.
"""
def fda_only(hits: list[SearchHit]) -> list[SearchHit]:
return [hit for hit in hits if "fda.gov" in (hit.url or "").lower()]
focused_queries = [
f'site:fda.gov "Warning Letter" GMP pharmaceutical {query}',
f'site:fda.gov/inspections-compliance-enforcement-and-criminal-investigations "Warning Letter" {query}',
f'site:fda.gov "meeting materials" "pharmaceutical quality" {query}',
f'site:fda.gov "meeting minutes" FDA pharmaceutical quality {query}',
]
hits: list[SearchHit] = []
seen: set[str] = set()
per_query = max(2, min(num_results, 4))
for focused_query in focused_queries:
route_hits: list[SearchHit] = []
serper = self._get_serper()
if serper:
try:
route_hits = [
SearchHit(h.title, h.url, h.snippet)
for h in serper.search(focused_query, num_results=per_query)
]
except Exception as exc:
if self.strict_specialized:
raise SearchError(f"serper FDA search failed: {exc}") from exc
if not route_hits and not self.strict_specialized:
route_hits = self.search(focused_query, num_results=per_query)
for hit in fda_only(route_hits):
key = hit.url or hit.title
if not key or key in seen:
continue
seen.add(key)
hits.append(hit)
if len(hits) >= num_results:
return hits
return hits
if __name__ == "__main__":
from scripts.lib.zenmux_client import load_secrets
load_secrets()
with SearchClient() as c:
print("--- 通用: Mabwell 迈威生物 ---")
for h in c.search("Mabwell 迈威生物 biopharmaceutical", num_results=3):
print(f" {h.title[:80]}")
print(f" {h.url}")
print("\n--- 专利: dual-target siRNA ---")
for h in c.patents("dual-target siRNA GalNAc", num_results=3):
print(f" {h.title[:80]}")
print(f" {h.url}")
print("\n--- Scholar: dual-target RNAi 2024 ---")
for h in c.scholar("dual-target RNAi drug", num_results=3, year_low=2023):
print(f" {h.title[:80]}")
print(f" {h.url}")
+190
View File
@@ -0,0 +1,190 @@
"""Serper.dev 客户端(Google Search API 代理)。
为什么用 Serper
- 2500 次免费额度,远超 SerpAPI 的 100/月
- 支持 Google Search、Scholar、News、Images、Maps
- Google Patents 无专用 endpoint,但可用 `site:patents.google.com` 技巧
- 价格比 SerpAPI 便宜 3-5×
用途:
- 专利检索:通用 search + `site:patents.google.com`
- 学术论文:/scholar endpoint
- 新闻:/news endpoint(时效性敏感的行业动态)
httpx 客户端使用 trust_env=False 绕过系统 socks 代理(macOS Clash 会导致 TLS EOF)。
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Any, Literal
import httpx
SERPER_BASE = "https://google.serper.dev"
@dataclass
class SerperHit:
title: str
url: str
snippet: str
source: str = "" # 论文出处 / 新闻媒体
date: str = "" # 发表日期(如 scholar / news 返回的话)
cited_by: int = 0 # 学术论文的引用数(仅 scholar)
class SerperError(RuntimeError):
pass
class SerperClient:
def __init__(self, api_key: str | None = None, timeout: float = 30.0) -> None:
self.api_key = api_key or os.environ.get("SERPAPI_KEY") or os.environ.get("SERPER_API_KEY")
if not self.api_key:
raise SerperError("SERPAPI_KEY / SERPER_API_KEY not set")
self._client = httpx.Client(trust_env=False, timeout=timeout)
def close(self) -> None:
self._client.close()
def __enter__(self) -> "SerperClient":
return self
def __exit__(self, *_args: Any) -> None:
self.close()
def _post(self, path: str, body: dict) -> dict:
try:
r = self._client.post(
f"{SERPER_BASE}{path}",
json=body,
headers={
"X-API-KEY": self.api_key,
"Content-Type": "application/json",
},
)
except httpx.RequestError as e:
raise SerperError(f"network error: {e}")
if r.status_code != 200:
raise SerperError(f"HTTP {r.status_code}: {r.text[:300]}")
try:
return r.json()
except Exception as e:
raise SerperError(f"invalid JSON: {e}")
def search(
self,
query: str,
*,
num_results: int = 10,
gl: str = "us",
hl: str = "en",
) -> list[SerperHit]:
"""通用 Google 搜索。支持 site: / filetype: / 引号短语等 Google 高级语法。"""
data = self._post("/search", {
"q": query,
"num": num_results,
"gl": gl,
"hl": hl,
})
hits: list[SerperHit] = []
for item in (data.get("organic") or [])[:num_results]:
hits.append(SerperHit(
title=(item.get("title") or "")[:200],
url=item.get("link") or "",
snippet=(item.get("snippet") or "")[:600],
date=item.get("date") or "",
))
return hits
def scholar(
self,
query: str,
*,
num_results: int = 10,
year_low: int | None = None,
year_high: int | None = None,
) -> list[SerperHit]:
"""Google Scholar 搜索——学术论文首选。
返回带引用数、发表年份等元数据,权威信源识别更准确。
"""
body: dict[str, Any] = {"q": query, "num": num_results}
if year_low is not None:
body["tbs"] = f"cdr:1,cd_min:{year_low}" + (f",cd_max:{year_high}" if year_high else "")
data = self._post("/scholar", body)
hits: list[SerperHit] = []
for item in (data.get("organic") or [])[:num_results]:
hits.append(SerperHit(
title=(item.get("title") or "")[:200],
url=item.get("link") or "",
snippet=(item.get("snippet") or "")[:600],
source=(item.get("publicationInfo") or "")[:200],
year=item.get("year") or "",
cited_by=item.get("citedBy") or 0,
) if False else SerperHit(
title=(item.get("title") or "")[:200],
url=item.get("link") or "",
snippet=(item.get("snippet") or "")[:600],
source=(item.get("publicationInfo") or "")[:200],
date=str(item.get("year") or ""),
cited_by=item.get("citedBy") or 0,
))
return hits
def patents(
self,
query: str,
*,
num_results: int = 10,
) -> list[SerperHit]:
"""Google Patents 检索——用 site: 技巧走通用搜索。
serper.dev 没有专门的 patents endpoint,但 `site:patents.google.com` 效果很好。
"""
combined = f"site:patents.google.com {query}"
return self.search(combined, num_results=num_results)
def news(
self,
query: str,
*,
num_results: int = 10,
time_range: Literal["d", "w", "m", "y"] | None = None,
) -> list[SerperHit]:
"""Google News 搜索——时效敏感行业动态。
time_range: d=24h, w=7d, m=30d, y=1y
"""
body: dict[str, Any] = {"q": query, "num": num_results}
if time_range:
body["tbs"] = f"qdr:{time_range}"
data = self._post("/news", body)
hits: list[SerperHit] = []
for item in (data.get("news") or [])[:num_results]:
hits.append(SerperHit(
title=(item.get("title") or "")[:200],
url=item.get("link") or "",
snippet=(item.get("snippet") or "")[:600],
source=(item.get("source") or "")[:200],
date=item.get("date") or "",
))
return hits
if __name__ == "__main__":
from scripts.lib.zenmux_client import load_secrets
load_secrets()
with SerperClient() as c:
print("=== Patents: dual-target siRNA ===")
for h in c.patents("dual-target siRNA GalNAc conjugate", num_results=3):
print(f" {h.title[:70]}")
print(f" {h.url}")
print("\n=== Scholar: dual-target RNAi ===")
for h in c.scholar("dual-target RNAi drug 2024", num_results=3):
print(f" {h.title[:70]} [引用 {h.cited_by}] ({h.date})")
print(f" {h.url}")
print(f" 源: {h.source[:80]}")
+423
View File
@@ -0,0 +1,423 @@
"""ZenMux API 客户端。
直接 HTTP 调用 zenmux 的 OpenAI 兼容端点。为 Phase 4 的 Python 化脚本服务
translate.py / polish.py)。所有调用都走 /api/v1/chat/completions。
设计原则:
- 独立于 opencode,可直接在 CLI / CI / cron 运行
- 内置重试(指数退避)、限流、token 统计、结构化日志
- 失败快速可见:打印请求 ID,便于在 zenmux 后台对账
- 默认读 secrets.env 里的 ZENMUX_API_KEY
"""
from __future__ import annotations
import json
import os
import sys
import threading
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import httpx
DEFAULT_BASE_URL = "https://zenmux.ai/api/v1"
DEFAULT_TIMEOUT = 300.0 # 翻译/润色单次调用可能 60s+,留够余量
MAX_RETRIES = 5
RETRYABLE_STATUSES = {408, 429, 500, 502, 503, 504, 520, 524}
_ANTHROPIC_MODEL_ALIASES = {
"anthropic/claude-opus-4-7": "anthropic/claude-opus-4.7",
"anthropic/claude-opus-4-6": "anthropic/claude-opus-4.6",
"anthropic/claude-opus-4-5": "anthropic/claude-opus-4.5",
"anthropic/claude-opus-4-1": "anthropic/claude-opus-4.1",
"anthropic/claude-sonnet-4-6": "anthropic/claude-sonnet-4.6",
"anthropic/claude-sonnet-4-5": "anthropic/claude-sonnet-4.5",
"anthropic/claude-haiku-4-5": "anthropic/claude-haiku-4.5",
}
_MODELS_WITHOUT_TEMPERATURE = {
"anthropic/claude-opus-4.7",
}
def normalize_zenmux_model(model: str) -> str:
"""Convert adapter-facing model IDs to ZenMux OpenAI API model IDs."""
normalized = model.strip()
if normalized.startswith("zenmux-anthropic/"):
normalized = "anthropic/" + normalized.removeprefix("zenmux-anthropic/")
elif normalized.startswith("zenmux/"):
normalized = normalized.removeprefix("zenmux/")
return _ANTHROPIC_MODEL_ALIASES.get(normalized, normalized)
def model_accepts_temperature(model: str) -> bool:
"""Return whether the ZenMux API accepts `temperature` for this model."""
return normalize_zenmux_model(model) not in _MODELS_WITHOUT_TEMPERATURE
@dataclass
class UsageStats:
"""聚合一次脚本运行的 token 消耗。"""
prompt_tokens: int = 0
completion_tokens: int = 0
cache_creation_tokens: int = 0
cache_read_tokens: int = 0
call_count: int = 0
failed_calls: int = 0
by_model: dict[str, dict[str, int]] = field(default_factory=dict)
def add(self, model: str, usage: dict[str, Any]) -> None:
p = usage.get("prompt_tokens", 0) or 0
c = usage.get("completion_tokens", 0) or 0
cc = (
usage.get("cache_creation_input_tokens", 0)
or usage.get("prompt_tokens_details", {}).get("cached_tokens", 0)
or 0
)
cr = (
usage.get("cache_read_input_tokens", 0)
or 0
)
self.prompt_tokens += p
self.completion_tokens += c
self.cache_creation_tokens += cc
self.cache_read_tokens += cr
self.call_count += 1
m = self.by_model.setdefault(
model,
{"prompt": 0, "completion": 0, "cache_creation": 0, "cache_read": 0, "calls": 0},
)
m["prompt"] += p
m["completion"] += c
m["cache_creation"] += cc
m["cache_read"] += cr
m["calls"] += 1
def summary(self) -> str:
lines = [
f"Total calls: {self.call_count} (failed: {self.failed_calls})",
f"Prompt tokens: {self.prompt_tokens:>12,}",
f"Completion tokens: {self.completion_tokens:>12,}",
f"Cache creation: {self.cache_creation_tokens:>12,}",
f"Cache read: {self.cache_read_tokens:>12,}",
]
for model, s in self.by_model.items():
lines.append(
f" [{model}] calls={s['calls']} in={s['prompt']:,} "
f"out={s['completion']:,} cache_w={s['cache_creation']:,} "
f"cache_r={s['cache_read']:,}"
)
return "\n".join(lines)
class ZenMuxError(RuntimeError):
pass
class ZenMuxClient:
"""ZenMux 轻量客户端。
只暴露一个方法 `chat_complete()`,屏蔽 httpx 细节。
"""
def __init__(
self,
api_key: str | None = None,
base_url: str | None = None,
timeout: float = DEFAULT_TIMEOUT,
log_file: Path | None = None,
) -> None:
self.api_key = api_key or os.environ.get("ZENMUX_API_KEY")
if not self.api_key:
raise ZenMuxError(
"ZENMUX_API_KEY not set. Source secrets.env or pass api_key explicitly."
)
self.base_url = (base_url or os.environ.get("ZENMUX_BASE_URL") or DEFAULT_BASE_URL).rstrip("/")
self.timeout = timeout
self.log_file = log_file
self.usage = UsageStats()
self._client = httpx.Client(timeout=timeout)
self._log_lock = threading.Lock()
self._usage_lock = threading.Lock()
def close(self) -> None:
self._client.close()
def __enter__(self) -> "ZenMuxClient":
return self
def __exit__(self, *_args: Any) -> None:
self.close()
def _log(self, payload: dict[str, Any]) -> None:
if not self.log_file:
return
self.log_file.parent.mkdir(parents=True, exist_ok=True)
with self._log_lock:
with self.log_file.open("a", encoding="utf-8") as f:
f.write(json.dumps(payload, ensure_ascii=False) + "\n")
def chat_complete(
self,
model: str,
system: str,
user: str,
*,
temperature: float = 0.3,
max_tokens: int = 16000,
extra_messages: list[dict[str, str]] | None = None,
web_search: bool = False,
web_search_options: dict[str, Any] | None = None,
tag: str = "",
) -> str:
"""一次非流式对话补全。
Args:
model: 完整 model id,例如 `anthropic/claude-sonnet-4.6`zenmux slug 不带 `zenmux/` 前缀,因为 baseURL 已经定位到 zenmux)。
system: system prompt
user: user message
temperature, max_tokens: 常规参数
extra_messages: 插在 system 之后、user 之前的额外消息(few-shot 等)
tag: 给这次调用打标签,便于日志里识别(如 "translate:ch03"
Returns:
assistant 的纯文本内容。如失败抛 ZenMuxError。
"""
messages: list[dict[str, str]] = [{"role": "system", "content": system}]
if extra_messages:
messages.extend(extra_messages)
messages.append({"role": "user", "content": user})
api_model = normalize_zenmux_model(model)
body: dict[str, Any] = {
"model": api_model,
"messages": messages,
"max_tokens": max_tokens,
}
if model_accepts_temperature(api_model):
body["temperature"] = temperature
if web_search:
body["web_search_options"] = web_search_options or {}
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
url = f"{self.base_url}/chat/completions"
last_error: str = ""
for attempt in range(MAX_RETRIES):
t0 = time.time()
try:
resp = self._client.post(url, json=body, headers=headers)
elapsed = time.time() - t0
except httpx.RequestError as e:
last_error = f"network: {e}"
elapsed = time.time() - t0
self._log({
"tag": tag, "attempt": attempt, "elapsed": elapsed,
"error": last_error,
})
time.sleep(2 ** attempt)
continue
if resp.status_code == 200:
try:
data = resp.json()
except Exception as e:
raise ZenMuxError(f"invalid JSON from zenmux: {e}; body={resp.text[:500]}")
usage = data.get("usage", {}) or {}
with self._usage_lock:
self.usage.add(api_model, usage)
content = ""
choices = data.get("choices") or []
if choices:
msg = choices[0].get("message") or {}
content = msg.get("content") or ""
self._log({
"tag": tag, "model": api_model, "requested_model": model, "attempt": attempt,
"elapsed": round(elapsed, 2),
"usage": usage,
"out_chars": len(content),
"status": 200,
})
if not content.strip():
# zenmux 偶尔返 200 但 content 空;视作可重试
last_error = "empty content"
time.sleep(2 ** attempt)
continue
return content
# 非 200
retryable = resp.status_code in RETRYABLE_STATUSES
last_error = f"HTTP {resp.status_code}: {resp.text[:500]}"
self._log({
"tag": tag, "attempt": attempt, "elapsed": round(elapsed, 2),
"status": resp.status_code, "error": last_error,
"retryable": retryable,
})
if not retryable:
self.usage.failed_calls += 1
raise ZenMuxError(last_error)
sleep_for = min(60, (2 ** attempt) + (attempt * 0.5))
time.sleep(sleep_for)
self.usage.failed_calls += 1
raise ZenMuxError(f"max retries exhausted. last error: {last_error}")
def chat_complete_with_meta(
self,
model: str,
system: str,
user: str,
*,
temperature: float = 0.3,
max_tokens: int = 16000,
extra_messages: list[dict[str, str]] | None = None,
web_search: bool = False,
web_search_options: dict[str, Any] | None = None,
tag: str = "",
) -> dict[str, Any]:
"""Return content plus metadata from one completion call."""
messages: list[dict[str, str]] = [{"role": "system", "content": system}]
if extra_messages:
messages.extend(extra_messages)
messages.append({"role": "user", "content": user})
api_model = normalize_zenmux_model(model)
body: dict[str, Any] = {
"model": api_model,
"messages": messages,
"max_tokens": max_tokens,
}
if model_accepts_temperature(api_model):
body["temperature"] = temperature
if web_search:
body["web_search_options"] = web_search_options or {}
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
url = f"{self.base_url}/chat/completions"
last_error = ""
for attempt in range(MAX_RETRIES):
t0 = time.time()
try:
resp = self._client.post(url, json=body, headers=headers)
elapsed = time.time() - t0
except httpx.RequestError as e:
last_error = f"network: {e}"
elapsed = time.time() - t0
self._log({"tag": tag, "attempt": attempt, "elapsed": elapsed, "error": last_error})
time.sleep(2 ** attempt)
continue
if resp.status_code != 200:
retryable = resp.status_code in RETRYABLE_STATUSES
last_error = f"HTTP {resp.status_code}: {resp.text[:500]}"
self._log({
"tag": tag,
"attempt": attempt,
"elapsed": round(elapsed, 2),
"status": resp.status_code,
"error": last_error,
"retryable": retryable,
})
if not retryable:
self.usage.failed_calls += 1
raise ZenMuxError(last_error)
sleep_for = min(60, (2 ** attempt) + (attempt * 0.5))
time.sleep(sleep_for)
continue
try:
data = resp.json()
except Exception as e:
raise ZenMuxError(f"invalid JSON from zenmux: {e}; body={resp.text[:500]}")
usage = data.get("usage", {}) or {}
with self._usage_lock:
self.usage.add(api_model, usage)
message = ((data.get("choices") or [{}])[0].get("message") or {})
content = message.get("content") or ""
annotations = message.get("annotations") or []
urls: list[str] = []
for ann in annotations:
if not isinstance(ann, dict):
continue
citation = ann.get("url_citation") or {}
url_item = citation.get("url")
if url_item:
urls.append(url_item)
self._log({
"tag": tag,
"model": api_model,
"requested_model": model,
"attempt": attempt,
"elapsed": round(elapsed, 2),
"usage": usage,
"out_chars": len(content),
"status": 200,
"web_search": web_search,
"citations": len(urls),
})
if not content.strip():
last_error = "empty content"
time.sleep(2 ** attempt)
continue
return {
"content": content,
"usage": usage,
"citations": urls,
"raw": data,
}
self.usage.failed_calls += 1
raise ZenMuxError(f"max retries exhausted. last error: {last_error}")
def load_secrets(env_path: Path | None = None) -> None:
"""从 secrets.env 把 key 塞到 os.environ,便于脚本直接运行。
格式宽松:`KEY=VALUE` 每行一条,`#` 开头是注释,忽略空行。
"""
if env_path is None:
# 默认在仓库根找 secrets.env
here = Path(__file__).resolve()
for parent in [here.parent, *here.parents]:
cand = parent / "secrets.env"
if cand.exists():
env_path = cand
break
if not env_path or not env_path.exists():
return
for line in env_path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
k, _, v = line.partition("=")
k = k.strip()
v = v.strip().strip('"').strip("'")
if k and v and k not in os.environ:
os.environ[k] = v
if __name__ == "__main__":
# 冒烟测试:python -m scripts.lib.zenmux_client
load_secrets()
with ZenMuxClient() as c:
out = c.chat_complete(
model="anthropic/claude-haiku-4.5",
system="You reply in exactly one English word.",
user="Say hello.",
max_tokens=20,
tag="smoke",
)
print("reply:", out)
print(c.usage.summary(), file=sys.stderr)
+188
View File
@@ -0,0 +1,188 @@
#!/usr/bin/env python3
"""Convert Deep Research source IDs into numeric citations for final output."""
from __future__ import annotations
import argparse
import json
import re
from pathlib import Path
from typing import Any
SRC_CITE_RE = re.compile(r"\[((?:src_[A-Za-z0-9_-]+)(?:\s*,\s*src_[A-Za-z0-9_-]+)*)\]")
def load_sources(path: Path) -> dict[str, dict[str, Any]]:
sources: dict[str, dict[str, Any]] = {}
if not path.exists():
return sources
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
sid = obj.get("id") or obj.get("source_id")
if sid:
sources[str(sid)] = obj
return sources
def extract_ordered_source_ids(text: str) -> list[str]:
ordered: list[str] = []
seen: set[str] = set()
for match in SRC_CITE_RE.finditer(text):
for sid in [item.strip() for item in match.group(1).split(",")]:
if sid and sid not in seen:
seen.add(sid)
ordered.append(sid)
return ordered
def _canonical_source_key(sid: str, source: dict[str, Any] | None) -> str:
"""Return a stable de-duplication key for a source record.
Phase 2 often creates chapter-local source IDs for the same local PDF or
official guideline. Final references should cite the underlying source
once, while citation_map.json keeps the full src_id traceability.
"""
if not source:
return f"missing:{sid}"
title = re.sub(r"\s+", " ", str(source.get("title") or source.get("name") or sid)).strip().lower()
title = title.removesuffix(" ocr").removesuffix(".ocr").strip()
doi = str(source.get("doi") or "").strip().lower()
if doi:
return f"doi:{doi}"
path = str(source.get("path") or "").strip()
url = str(source.get("url") or "").strip()
if title and ("phase0/extracted/" in path or "phase0/extracted/" in url):
return f"local-material:{title}"
for field in ("url", "path"):
value = str(source.get(field) or "").strip()
if value:
return f"{field}:{value.rstrip('/').lower()}"
return f"title:{title or sid}"
def build_numeric_mapping(
ordered_ids: list[str],
sources: dict[str, dict[str, Any]],
) -> tuple[dict[str, int], list[dict[str, Any]]]:
mapping: dict[str, int] = {}
records: list[dict[str, Any]] = []
seen_keys: dict[str, int] = {}
record_by_number: dict[int, dict[str, Any]] = {}
for sid in ordered_ids:
source = sources.get(sid)
key = _canonical_source_key(sid, source)
if key in seen_keys:
number = seen_keys[key]
mapping[sid] = number
record_by_number[number].setdefault("source_ids", []).append(sid)
continue
number = len(records) + 1
seen_keys[key] = number
mapping[sid] = number
record = {
"number": number,
"source_id": sid,
"source_ids": [sid],
"source": source or {},
"dedupe_key": key,
}
records.append(record)
record_by_number[number] = record
return mapping, records
def format_reference(number: int, sid: str, source: dict[str, Any] | None) -> str:
if not source:
return f"{number}. {sid}. sources.jsonl 未找到该来源)"
authors = ", ".join(source.get("authors", [])) if source.get("authors") else ""
year = source.get("year") or source.get("date") or ""
title = source.get("title") or source.get("name") or sid
title = re.sub(r"(?i)(?:\s+OCR|\.ocr)$", "", str(title)).strip()
publisher = source.get("publisher") or source.get("venue") or source.get("source") or ""
url = source.get("url") or source.get("path") or ""
parts = [f"{number}. "]
if authors:
parts.append(f"{authors}. ")
if year:
parts.append(f"({year}). ")
parts.append(str(title))
if publisher:
parts.append(f". {publisher}")
if url:
parts.append(f". {url}")
return "".join(parts)
def convert_citations(text: str, mapping: dict[str, int]) -> str:
def repl(match: re.Match[str]) -> str:
ids = [item.strip() for item in match.group(1).split(",") if item.strip()]
nums: list[str] = []
seen: set[int] = set()
for sid in ids:
if sid not in mapping:
continue
number = mapping[sid]
if number in seen:
continue
seen.add(number)
nums.append(str(number))
return "<sup>[" + ", ".join(nums) + "]</sup>" if nums else match.group(0)
return SRC_CITE_RE.sub(repl, text)
def strip_existing_reference_section(text: str) -> str:
pattern = re.compile(r"\n##\s*(?:参考文献|参考来源清单|References)\s*\n.*\Z", re.S)
return pattern.sub("", text).rstrip() + "\n"
def number_citations(
*,
text: str,
sources: dict[str, dict[str, Any]],
) -> tuple[str, list[dict[str, Any]]]:
ordered_ids = extract_ordered_source_ids(text)
mapping, records = build_numeric_mapping(ordered_ids, sources)
body = convert_citations(strip_existing_reference_section(text), mapping).rstrip()
ref_lines = ["", "## 参考来源清单", ""]
for record in records:
ref_lines.append(format_reference(record["number"], record["source_id"], record["source"]))
return body + "\n" + "\n".join(ref_lines).rstrip() + "\n", records
def main() -> int:
parser = argparse.ArgumentParser(description="Convert [src_xxx] citations to numeric citations")
parser.add_argument("project", help="Project directory")
parser.add_argument("--input", default="phase4/final_zh.md")
parser.add_argument("--output", default="phase4/final_zh_numbered.md")
parser.add_argument("--sources", default="phase2/sources.jsonl")
parser.add_argument("--map", default="phase4/citation_map.json")
args = parser.parse_args()
project = Path(args.project)
src_path = project / args.input
out_path = project / args.output
sources_path = project / args.sources
map_path = project / args.map
if not src_path.exists():
raise SystemExit(f"input not found: {src_path}")
sources = load_sources(sources_path)
numbered, records = number_citations(text=src_path.read_text(encoding="utf-8"), sources=sources)
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text(numbered, encoding="utf-8")
map_path.parent.mkdir(parents=True, exist_ok=True)
map_path.write_text(json.dumps(records, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(f"Wrote: {out_path.relative_to(project)}")
print(f"Wrote: {map_path.relative_to(project)}")
print(f"Citations: {len(records)}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+183
View File
@@ -0,0 +1,183 @@
#!/usr/bin/env python3
"""Phase 4 replacement pipeline orchestrator.
Default flow:
1) translate.py
2) optional glossary verification (low-confidence/full/off)
3) apply_glossary.py
4) polish.py
5) build_report.py
"""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
import time
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from scripts.lib.markdown_chunker import split_by_headers
def resolve_project(arg: str) -> Path:
p = Path(arg)
if p.is_dir():
return p.resolve()
cand = REPO_ROOT / "projects" / arg
if cand.is_dir():
return cand.resolve()
raise SystemExit(f"project not found: {arg}")
def run_step(cmd: list[str], *, dry_run: bool) -> int:
print("$ " + " ".join(cmd))
if dry_run:
return 0
return subprocess.run(cmd, cwd=REPO_ROOT, check=False).returncode
def infer_workers(source_file: Path, fallback: int, cap: int = 8) -> int:
if not source_file.exists():
return fallback
text = source_file.read_text(encoding="utf-8")
blocks = split_by_headers(text, max_level=2)
if not blocks:
return fallback
cpu_cap = max(2, min(cap, (os.cpu_count() or 4)))
suggested = max(2, min(cpu_cap, (len(blocks) + 5) // 6))
return max(1, suggested if fallback <= 0 else min(max(fallback, 1), cpu_cap))
def low_confidence_terms(glossary_path: Path) -> list[str]:
if not glossary_path.exists():
return []
try:
glossary = json.loads(glossary_path.read_text(encoding="utf-8"))
except Exception:
return []
out: list[str] = []
for term, entry in glossary.items():
if not isinstance(entry, dict):
out.append(term)
continue
conf = str(entry.get("confidence", "")).lower()
verified = bool(entry.get("verified_at"))
if conf != "high" or not verified:
out.append(term)
return sorted(set(out))
def main() -> int:
parser = argparse.ArgumentParser(description="Phase 4 replacement pipeline")
parser.add_argument("project", help="Project slug or full path")
parser.add_argument("--translate-model", default="anthropic/claude-sonnet-4.6")
parser.add_argument("--glossary-model", default="anthropic/claude-haiku-4.5")
parser.add_argument("--polish-model", default="anthropic/claude-sonnet-4.6")
parser.add_argument("--translate-workers", type=int, default=0, help="0 means auto")
parser.add_argument("--glossary-workers", type=int, default=4)
parser.add_argument("--polish-workers", type=int, default=0, help="0 means auto")
parser.add_argument(
"--glossary-mode",
choices=["off", "low-confidence", "full"],
default="low-confidence",
help="off: skip, low-confidence: verify only low-confidence terms, full: verify all",
)
parser.add_argument("--dry-run", action="store_true")
args = parser.parse_args()
project_root = resolve_project(args.project)
phase4 = project_root / "phase4"
src_en = phase4 / "final_en.md"
if not src_en.exists():
raise SystemExit(f"missing source: {src_en}")
tw = infer_workers(src_en, args.translate_workers)
zh = phase4 / "final_zh.md"
pw = infer_workers(zh if zh.exists() else src_en, args.polish_workers)
print(f"Project: {project_root.name}")
print(f"Translate workers: {tw} | Polish workers: {pw}")
print(f"Glossary mode: {args.glossary_mode}")
print()
t0 = time.time()
steps: list[list[str]] = [
[
sys.executable,
str(REPO_ROOT / "scripts" / "translate.py"),
str(project_root),
"--workers",
str(tw),
"--model",
args.translate_model,
]
]
if args.glossary_mode != "off":
gcmd = [
sys.executable,
str(REPO_ROOT / "scripts" / "build_glossary.py"),
str(project_root),
"--workers",
str(args.glossary_workers),
"--model",
args.glossary_model,
]
if args.glossary_mode == "low-confidence":
terms = low_confidence_terms(phase4 / "glossary.json")
if terms:
if len(terms) > 80:
print(f"[info] low-confidence terms={len(terms)} is large; fallback to full glossary verify")
else:
gcmd += ["--only", ",".join(terms)]
else:
print("[info] no low-confidence glossary terms found; skipping glossary step")
gcmd = []
if gcmd:
steps.append(gcmd)
steps.extend(
[
[
sys.executable,
str(REPO_ROOT / "scripts" / "apply_glossary.py"),
str(project_root),
"--input",
"phase4/final_zh.md",
],
[
sys.executable,
str(REPO_ROOT / "scripts" / "polish.py"),
str(project_root),
"--workers",
str(pw),
"--model",
args.polish_model,
],
[
sys.executable,
str(REPO_ROOT / "scripts" / "build_report.py"),
str(project_root),
],
]
)
for cmd in steps:
rc = run_step(cmd, dry_run=args.dry_run)
if rc != 0:
return rc
print(f"\nPhase 4 pipeline done in {time.time() - t0:.1f}s")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+312
View File
@@ -0,0 +1,312 @@
#!/usr/bin/env python3
"""Phase 4 中文润色:按 H2 section 切块 → 逐块润色 → 拼接 → 写盘。
用法:
uv run python scripts/polish.py <project_slug>
架构跟 translate.py 一致:
- Python 做切块/循环/重试/断点续传
- LLM 只做"润色这一段",单次 output token 远低于上限
- 结果落在 phase4/zh_polished_chunks/<anchor>.md,重跑只补缺
- 最终拼接写入 phase4/final_zh_polished.md(默认覆写 final_zh.md 的副本)
区别只在:
- 输入是中文(final_zh.md),输出还是中文
- 不维护术语表(翻译阶段已经固定了)
- 会附带一份 notes.jsonl 记录模型发现的异常
"""
from __future__ import annotations
import argparse
import concurrent.futures
import json
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from scripts.lib.markdown_chunker import (
MarkdownBlock,
count_chinese_chars,
split_by_headers,
)
from scripts.lib.zenmux_client import ZenMuxClient, ZenMuxError, load_secrets
from scripts.runtime.skills import SkillRegistry
DEFAULT_MODEL = "anthropic/claude-sonnet-4.6"
MODEL_MAX_TOKENS = {
"anthropic/claude-sonnet-4.6": 32000,
"anthropic/claude-sonnet-4.5": 32000,
"anthropic/claude-opus-4.7": 32000,
"anthropic/claude-opus-4.6": 32000,
"anthropic/claude-haiku-4.5": 16000,
}
PROMPT_FILE = Path(__file__).parent / "prompts" / "polish_system.txt"
POLISH_SKILLS = ("humanizer-cn", "output-hygiene")
def build_polish_system_prompt(skill_registry: SkillRegistry | None = None) -> str:
"""Build the Phase 4 polish prompt with canonical writing skills attached."""
registry = skill_registry or SkillRegistry()
parts = [PROMPT_FILE.read_text(encoding="utf-8").rstrip()]
for skill_name in POLISH_SKILLS:
parts.append(f"# Skill: {skill_name}\n\n{registry.read(skill_name).rstrip()}")
return "\n\n".join(parts) + "\n"
def resolve_project(arg: str) -> Path:
p = Path(arg)
if p.is_dir():
return p
cand = Path.cwd() / "projects" / arg
if cand.is_dir():
return cand
raise SystemExit(f"project not found: {arg}")
def parse_polish_response(text: str) -> tuple[str, str]:
"""解析 <<<POLISHED>>>/<<<NOTES>>> 格式。返回 (polished, notes)。"""
p_start = text.find("<<<POLISHED>>>")
p_end = text.find("<<<END_POLISHED>>>")
if p_start == -1 or p_end == -1 or p_end <= p_start:
raise ValueError(f"missing <<<POLISHED>>> markers: {text[:300]}")
polished = text[p_start + len("<<<POLISHED>>>"): p_end].strip("\r\n")
n_start = text.find("<<<NOTES>>>")
n_end = text.find("<<<END_NOTES>>>")
notes = ""
if n_start != -1 and n_end != -1 and n_end > n_start:
notes = text[n_start + len("<<<NOTES>>>"): n_end].strip()
return polished, notes
def build_user_prompt(block: MarkdownBlock) -> str:
level_hint = f"H{block.level}" if block.level >= 1 else "frontmatter (无标题)"
return (
f"# 待润色的中文块(Markdown, {level_hint}\n"
"请按系统提示的规则润色下面这段中文。严格使用指定的分隔符格式输出。\n\n"
"----- BEGIN BLOCK -----\n"
f"{block.content}\n"
"----- END BLOCK -----\n"
)
def polish_block(
client: ZenMuxClient,
block: MarkdownBlock,
*,
model: str,
system_prompt: str,
temperature: float,
) -> tuple[str, str]:
user = build_user_prompt(block)
max_tok = MODEL_MAX_TOKENS.get(model, 16000)
raw = client.chat_complete(
model=model,
system=system_prompt,
user=user,
temperature=temperature,
max_tokens=max_tok,
tag=f"polish:{block.anchor}",
)
try:
polished, notes = parse_polish_response(raw)
except Exception as e:
raise RuntimeError(
f"bad response format for block {block.anchor}: {e}\nraw head: {raw[:300]}"
)
if not polished.strip():
raise RuntimeError(f"empty polished content for block {block.anchor}")
return polished.rstrip(), notes
def main() -> int:
parser = argparse.ArgumentParser(description="Phase 4 中文润色(按 H2 切块循环)")
parser.add_argument("project", help="项目 slug 或完整路径")
parser.add_argument(
"--source",
default="phase4/final_zh.md",
help="中文源(默认 phase4/final_zh.md,即 translate.py 的产物)",
)
parser.add_argument(
"--output",
default="phase4/final_zh_polished.md",
help="润色后输出(默认 phase4/final_zh_polished.md",
)
parser.add_argument("--model", default=DEFAULT_MODEL)
parser.add_argument("--temperature", type=float, default=0.4)
parser.add_argument(
"--force",
action="store_true",
help="忽略 zh_polished_chunks 缓存,强制重润",
)
parser.add_argument(
"--only",
default=None,
help="只润色指定 order(逗号分隔),例如 --only 2,7,18",
)
parser.add_argument(
"--limit",
type=int,
default=None,
help="最多润色前 N 个未缓存的块(调试用)",
)
parser.add_argument(
"--workers",
type=int,
default=4,
help="并发润色 worker 数(默认 4;设为 1 回退串行)",
)
args = parser.parse_args()
load_secrets()
project_root = resolve_project(args.project)
src_path = project_root / args.source
out_path = project_root / args.output
if not src_path.exists():
raise SystemExit(f"source not found: {src_path}. 请先跑 translate.py")
chunks_dir = project_root / "phase4" / "zh_polished_chunks"
chunks_dir.mkdir(parents=True, exist_ok=True)
logs_dir = project_root / "phase4" / "logs"
log_file = logs_dir / "polish.jsonl"
notes_file = project_root / "phase4" / "polish_notes.jsonl"
system_prompt = build_polish_system_prompt()
text = src_path.read_text(encoding="utf-8")
blocks = split_by_headers(text, max_level=2)
only_orders: set[int] | None = None
if args.only:
only_orders = {int(a.strip()) for a in args.only.split(",") if a.strip()}
total_cn = sum(count_chinese_chars(b.content) for b in blocks)
print(f"Source: {src_path.relative_to(project_root)}")
print(f"Blocks: {len(blocks)} | total Chinese chars: {total_cn:,}")
workers = max(1, args.workers)
print(f"Model: {args.model} | temperature: {args.temperature} | workers: {workers}")
print()
start = time.time()
todo: list[MarkdownBlock] = []
cached_count = 0
notes_records: list[dict] = []
for b in blocks:
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
if only_orders is not None and b.order not in only_orders:
continue
if chunk_path.exists() and not args.force:
cached_count += 1
print(f" [ok ] #{b.order:03d} {b.short_title} (cached)")
continue
if args.limit is not None and len(todo) >= args.limit:
continue
todo.append(b)
if todo:
print(f"To polish this run: {len(todo)} blocks | cached: {cached_count}")
print()
with ZenMuxClient(log_file=log_file) as client:
def run_one(b: MarkdownBlock) -> tuple[MarkdownBlock, str, str, float]:
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
t0 = time.time()
polished, notes = polish_block(
client,
b,
model=args.model,
system_prompt=system_prompt,
temperature=args.temperature,
)
chunk_path.write_text(polished + "\n", encoding="utf-8")
return b, polished, notes, time.time() - t0
if workers == 1:
for b in todo:
label = f"#{b.order:03d} L{b.level} {count_chinese_chars(b.content):>4}{b.short_title}"
print(f" [... ] {label} ", end="", flush=True)
try:
block, polished, notes, elapsed = run_one(b)
except (ZenMuxError, RuntimeError) as e:
print(f"\n [FAIL] {label}\n {e}")
continue
cn = count_chinese_chars(polished)
before_cn = count_chinese_chars(block.content)
delta = cn - before_cn
sign = "+" if delta >= 0 else ""
if notes:
notes_records.append(
{"order": block.order, "anchor": block.anchor, "title": block.short_title, "notes": notes}
)
print(f"\r [done] {label}{cn}字 ({sign}{delta}, {elapsed:4.1f}s)")
else:
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
futures = {pool.submit(run_one, b): b for b in todo}
for fut in concurrent.futures.as_completed(futures):
b = futures[fut]
label = f"#{b.order:03d} L{b.level} {count_chinese_chars(b.content):>4}{b.short_title}"
try:
block, polished, notes, elapsed = fut.result()
except (ZenMuxError, RuntimeError) as e:
print(f" [FAIL] {label}\n {e}")
continue
cn = count_chinese_chars(polished)
before_cn = count_chinese_chars(block.content)
delta = cn - before_cn
sign = "+" if delta >= 0 else ""
if notes:
notes_records.append(
{"order": block.order, "anchor": block.anchor, "title": block.short_title, "notes": notes}
)
print(f" [done] {label}{cn}字 ({sign}{delta}, {elapsed:4.1f}s)")
# 汇总
merged: list[str] = []
missing: list[str] = []
for b in blocks:
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
if not chunk_path.exists():
missing.append(f"#{b.order:03d} {b.short_title}")
continue
merged.append(chunk_path.read_text(encoding="utf-8").rstrip())
partial = only_orders is not None or args.limit is not None
if missing:
print(f"\n⚠ 缺失 {len(missing)} 块:")
for m in missing[:10]:
print(f" - {m}")
if len(missing) > 10:
print(f" ... 还有 {len(missing) - 10}")
if partial:
print("partial 模式:--only / --limit 生效,未生成 final_zh_polished.md")
else:
print("重新运行本脚本即可补润(已润的会跳过)。")
print(client.usage.summary())
return 1
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text("\n\n".join(merged) + "\n", encoding="utf-8")
if notes_records:
notes_file.write_text(
"\n".join(json.dumps(r, ensure_ascii=False) for r in notes_records) + "\n",
encoding="utf-8",
)
print(f"\n发现 {len(notes_records)} 条润色笔记 → {notes_file.relative_to(project_root)}")
final_text = out_path.read_text(encoding="utf-8")
cn = count_chinese_chars(final_text)
total_time = time.time() - start
print()
print(f"✓ 输出:{out_path.relative_to(project_root)}")
print(f" 润色前字数: {total_cn:,}")
print(f" 润色后字数: {cn:,} (变化 {cn - total_cn:+d}, {(cn/total_cn - 1)*100:+.1f}%)")
print(f" 耗时: {total_time:.1f}s")
print(client.usage.summary())
return 0
if __name__ == "__main__":
sys.exit(main())
+54
View File
@@ -0,0 +1,54 @@
你是一名生物医药行业的资深双语术语编辑。现在要根据网络搜索片段,核查并确定一个英文术语/专有名词的"中文规范译名"、"英文全称"与"可信度"。
## 你将收到
- `term`:待核查的英文术语(可能是公司名、缩写、化学品、平台名、药物通用名等)
- `domain`:研究领域(例如 "生物医药双靶点 RNAi 药物"),作为消歧背景
- `current_zh`:系统已有的中文译名(可能来自上一步翻译,也可能为空)
- `search_hits`35 条 web 搜索结果(title + url + snippet
## 你要判断
1. **中文规范译名**`zh`):
- 如果搜索结果里有权威的中文表达(公司官网、维基、百度百科、权威医药媒体),以它为准
- 如果 `current_zh` 已经正确,沿用它,避免无谓变更
- 如果无中文通用译名(小众学术术语、新兴化合物),保留英文原文,`zh` 字段置空字符串
- 对公司/机构名:必须使用工商注册的正式中文名(例如 "Mabwell → 迈威生物"、"Sirnaomics → 圣诺生物"
- 对药物:优先 INN 通用名(例如 "inclisiran → 英克司兰"
- 对缩写/技术术语:使用业内通行译名(例如 "GalNAc → N-乙酰半乳糖胺""RNAi → RNA 干扰"
2. **英文全称**`en_full`):
- 如果术语是缩写(≤ 8 字符全大写或混合大小写),给出英文全称(例如 "ASGPR → Asialoglycoprotein Receptor"
- 如果术语本身就是全称,填相同的字符串或其最规范的写法
- 如果术语是公司名,填其英文法人全称(例如 "Mabwell → Mabwell (Shanghai) Bioscience Co., Ltd."
3. **可信度**`confidence`):
- `high`:至少两条独立、高质量信源(官网/监管机构/权威媒体)一致支持
- `medium`:一条高质量信源支持,或多条一般信源一致支持
- `low`:只能推断或无法确认,建议人工复核
4. **问题/警示**`issue`):
- 如果 `current_zh` 明显错误(例如把 "Mabwell" 译成 "Maywavee"),在 `issue` 中指出错在哪里
- 如果术语有多种译法争议,在 `issue` 中简述
- 如果搜索结果完全不相关(错别字、生僻词),在 `issue` 中说"搜索无有效结果"
- 无异常则留空字符串
## 输出格式(严格)
输出**一行 JSON**,不要用代码围栏,不要加任何解释。
```
{"zh": "迈威生物", "en_full": "Mabwell (Shanghai) Bioscience Co., Ltd.", "confidence": "high", "issue": ""}
```
如果 `current_zh` 错了:
```
{"zh": "迈威生物", "en_full": "Mabwell (Shanghai) Bioscience Co., Ltd.", "confidence": "high", "issue": "current_zh 'Maywavee' 为拼写错误,正确为 Mabwell → 迈威生物"}
```
如果无需中文译名(保留英文):
```
{"zh": "", "en_full": "Phosphoramidite", "confidence": "high", "issue": ""}
```
+48
View File
@@ -0,0 +1,48 @@
你是一名顶级中文咨询报告编辑。现在要把一段由英文翻译而来的中文文本润色为**母语中文写作者**的成品。目标读者是生物医药行业的高层研究员、投资人与决策者。
## 不可违反的规则
1. **保留所有引用标注** `[src_xxx]`,位置可以微调但不得删除或改写。
2. **保留所有数字、百分比、日期、单位、化学式、药物代号、机构名称**,一字不改。
3. **保留 Markdown 结构**:输入是什么标题层级(#/##/###)输出就是什么。表格的 `|` 分隔符和列数不变。列表符号(-, *, 1.)不变。
4. **保留段落数量**:不要合并或拆分段落。每段原文对应一段输出。
5. **专有名词首次出现保持"中文(English)"格式**;如果译文里这个术语已经这样标了就别改,也不要删掉。
6. **不改变论点、结论、数据、案例**。只改语言表达。
7. **严禁使用 emoji**(✅ ❌ 🔶 🔷 ⭐ 🟢 🔴 ⚠️ 💡 📌 🔑 📊 等彩色符号)。如果原文里有 emoji,替换为字体支持的符号(✓ × ◆ ● ★ * 注 等)或直接删除。这些 emoji 在 PDF 里渲染为方框。
## 要去掉的"AI 味/翻译腔"表征
- 空泛套话:随着…不断发展、综上所述、本质上、从根本上、跃迁、赋能、落地、抓手
- 翻译腔开头:值得注意的是、众所周知、毫无疑问、不难看出
- 冗余连词开头:此外、而且、并且、再者(英文 moreover / furthermore / additionally 的直译残留)
- 介词短语套用:对于…来说、在…方面、…的话、关于…这一点
- 过度强调:非常、十分、极其、特别、尤为(没有数据支撑时)
- 长串的"的"字("X 的 Y 的 Z 的 W")改为短句
- 被动语态("被…所…"、"……得以……")尽量改主动
- 把"我们"删掉,除非真是作者第一人称立场
## 要加强的中文表达特征
- 句子节奏变化:短句与中句交替,避免一路长句
- 动词前置:中文偏好动词驱动,不要像英文那样把名词短语堆在主语
- 具体化:如果译文留下了模糊的"相关"、"一定的"、"较大的",根据上下文换成源文里的具体含义;若无依据,保留原样不硬改
- 段落内逻辑词(因此、相比之下、代价是)用得准确、用得克制
## 特殊情况
- 如果段落里有"译者注"、"TRANSLATOR_NOTE:" 之类残留,删除后自然连接上下文
- 如果出现明显的翻译错误(中文表达反了意思),修正它,但在输出的 `notes` 中记一笔
- 如果某句过于生硬又不确定原意,保守处理(小改),不要激进重写
## 输出格式(严格)
输出完全按下面的分隔符格式,前后不得有任何多余字符、说明或代码围栏。
<<<POLISHED>>>
...润色后的完整 Markdown 块,逐字复制,包括标题行...
<<<END_POLISHED>>>
<<<NOTES>>>
...最多两句话的异常说明;若无异常就留空...
<<<END_NOTES>>>
`<<<POLISHED>>>...<<<END_POLISHED>>>` 之间是原生 Markdown(无需转义)。
+44
View File
@@ -0,0 +1,44 @@
You are a senior English-to-Chinese biomedical translator and editor. You do NOT mechanically translate — you rewrite the meaning in natural, professional Chinese that reads as if a native Chinese consulting analyst wrote it from scratch.
## Absolute rules (non-negotiable)
1. Preserve every citation marker `[src_xxx]` verbatim, at roughly the same position as in the source.
2. Preserve every number, percentage, date, unit, chemical notation (e.g., 2-OMe), and drug code (e.g., ARO-DIMER-PA) exactly.
3. Preserve the Markdown structure: the input block starts with a Markdown heading at some level (one `#`, `##`, etc.) or is frontmatter; output the same heading at the same level. Do not demote / promote headings. Do not add new headings.
4. Preserve tables: translate cell text but keep `|` pipes and column count identical.
5. Preserve list formatting (`-`, `*`, `1.`) and code fences.
6. First mention of a technical term: use the format `中文(English` — but only once per block; subsequent mentions use Chinese only.
7. Company / institution names: use the established Chinese rendering if it is in common Chinese press (e.g., Merck → 默克, AstraZeneca → 阿斯利康, Alnylam → 阿尔尼拉姆). If no established rendering exists, keep the English as-is (e.g., NEB, Genovis, Codexis, Arrowhead, Argo).
8. Do NOT add commentary, introductions, or disclaimers beyond what the English says.
9. Do NOT collapse or merge consecutive paragraphs — preserve paragraph breaks.
10. Output Chinese-style punctuation inside Chinese text: `,。;:?!""()`. Keep English punctuation inside parenthetical English phrases.
11. Do NOT add separator lines (`---`) or blank lines that weren't in the source. If the source ends with `---`, keep it; if it doesn't, don't add one.
12. **NEVER use emoji** (✅ ❌ 🔶 🔷 ⭐ 🟢 🔴 ⚠️ 💡 📌 🔑 📊 etc.). If the source contains emoji, replace with plain text or punctuation equivalents (✓ × ◆ ● ★ * 注 等). These do not render in the PDF (font has no glyphs).
## Style rules (aim for native-Chinese feel)
- Break long English sentences into two or three short Chinese clauses.
- Prefer active voice; avoid translation-ese constructions like "对于...来说", "在...方面", "...的话", "值得注意的是".
- Do not use filler phrases like "随着...的不断发展", "综上所述", "从本质上说" unless the English explicitly argues that point.
- Use 的 sparingly. No "X的Y的Z的W" chains.
- Numbered lists with short items: translate tightly, do not pad with Chinese particles.
- SCQA-style paragraphs in Executive Summary stay SCQA in Chinese — translate the flow, never label S/C/Q/A.
## Glossary continuity
You will receive a JSON glossary of terms already translated in earlier blocks. Use those Chinese translations consistently. If you encounter a new term worth locking in, translate it and add it to the glossary patch.
## Output format (strict)
Output exactly the following, with no extra text before or after. No explanations. No code fences.
<<<TRANSLATION>>>
...the full translated Markdown block here, verbatim, including its heading line(s)...
<<<END_TRANSLATION>>>
<<<GLOSSARY_PATCH>>>
English term 1 || 中文译名 1
English term 2 || 中文译名 2
<<<END_GLOSSARY_PATCH>>>
Inside `<<<TRANSLATION>>>...<<<END_TRANSLATION>>>` the content is raw Markdown (no escaping needed).
Inside `<<<GLOSSARY_PATCH>>>...<<<END_GLOSSARY_PATCH>>>` each line is `English||Chinese`; leave empty if no new terms.
+2
View File
@@ -0,0 +1,2 @@
"""Report rendering helpers for the v0.20 Python core."""
+32
View File
@@ -0,0 +1,32 @@
"""Font resolution helpers for PDF rendering."""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
@dataclass(frozen=True)
class QuartoFonts:
main_font: str
sans_font: str
requires_system_fonts: bool
def resolve_quarto_fonts(fonts_dir: Path) -> QuartoFonts:
"""Resolve Quarto font names.
Quarto/xelatex currently uses installed font family names. We still accept
fonts_dir so callers can validate/report environment state consistently.
"""
expected = [
fonts_dir / "SourceHanSerifSC-Regular.otf",
fonts_dir / "SourceHanSansSC-Bold.otf",
]
requires_system_fonts = not all(path.exists() for path in expected)
return QuartoFonts(
main_font="Source Han Serif CN",
sans_font="Source Han Sans CN",
requires_system_fonts=requires_system_fonts,
)
+61
View File
@@ -0,0 +1,61 @@
"""Reference-list generation from Deep Research sources.jsonl."""
from __future__ import annotations
import json
import re
from pathlib import Path
def cited_source_keys(md_text: str) -> set[str]:
return set(re.findall(r"\[src_([a-zA-Z0-9_-]+)\]", md_text))
def build_references_block(sources_path: Path | None, md_text: str) -> str:
"""Build a compact references section for actually cited src IDs."""
if not sources_path or not sources_path.exists():
return "(参考文献列表:sources.jsonl 未找到)"
cited = cited_source_keys(md_text)
if not cited:
return ""
sources: dict[str, dict] = {}
with sources_path.open(encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
sid = obj.get("id", "")
key = sid.replace("src_", "")
if key in cited:
sources[sid] = obj
if not sources:
return ""
lines = ["## 参考文献\n"]
for sid in sorted(sources.keys()):
s = sources[sid]
authors = ", ".join(s.get("authors", [])) if s.get("authors") else ""
year = s.get("year", "")
title = s.get("title", sid)
venue = s.get("venue", "")
url = s.get("url", "")
entry = f"- **[{sid}]** "
if authors:
entry += f"{authors}. "
if year:
entry += f"({year}). "
entry += f"*{title}*"
if venue:
entry += f". {venue}"
if url:
entry += f". <{url}>"
lines.append(entry)
return "\n".join(lines)
+6
View File
@@ -0,0 +1,6 @@
"""v0.20 Python runtime core for Deep Research.
The runtime layer is intentionally platform-neutral: OpenCode, Codex, and
Claude Code should call into these modules instead of owning orchestration.
"""
+46
View File
@@ -0,0 +1,46 @@
"""Project artifact helpers shared by the Python runtime."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
REPO_ROOT = Path(__file__).resolve().parents[2]
PROJECTS_DIR = REPO_ROOT / "projects"
def resolve_project(project: str | Path) -> Path:
p = Path(project)
if p.is_dir():
return p.resolve()
candidate = PROJECTS_DIR / str(project)
if candidate.is_dir():
return candidate.resolve()
raise FileNotFoundError(f"project not found: {project}")
def load_manifest(project_root: Path) -> dict[str, Any]:
path = project_root / "manifest.json"
if not path.exists():
raise FileNotFoundError(f"manifest not found: {path}")
return json.loads(path.read_text(encoding="utf-8"))
def write_manifest(project_root: Path, manifest: dict[str, Any]) -> None:
path = project_root / "manifest.json"
path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def ensure_phase_dirs(project_root: Path) -> None:
for rel in (
"phase1",
"phase2/drafts",
"phase2/evidence",
"phase2/packets",
"phase3",
"phase4",
):
(project_root / rel).mkdir(parents=True, exist_ok=True)
+360
View File
@@ -0,0 +1,360 @@
"""Chapter brief aggregation and Chinese chapter assembly."""
from __future__ import annotations
import json
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Callable
from scripts.runtime.roles import RoleDefinition, RuntimeProfile
from scripts.runtime.skills import SkillRegistry
from scripts.runtime.tasks import load_task_cards, validate_packet
from scripts.runtime.workers import ChatClient
def validate_chapter_brief(brief: dict) -> None:
required = {
"chapter_id",
"chapter_title",
"packet_ids",
"core_claims",
"evidence_items",
"counter_evidence",
"source_ids",
"open_questions",
"assembly_notes",
}
missing = sorted(required - set(brief))
if missing:
raise ValueError(f"chapter brief missing fields: {missing}")
if not brief["chapter_id"]:
raise ValueError("chapter_id required")
if not brief["packet_ids"]:
raise ValueError("chapter brief requires at least one packet")
if not brief["core_claims"]:
raise ValueError("chapter brief requires core_claims")
if not brief["evidence_items"]:
raise ValueError("chapter brief requires evidence_items")
if not brief["counter_evidence"]:
raise ValueError("chapter brief requires counter_evidence")
def validate_compressed_finding(finding: dict) -> None:
required = {
"chapter_id",
"chapter_title",
"packet_ids",
"chapter_thesis",
"key_findings",
"evidence_landings",
"counter_evidence",
"source_ids",
"open_questions",
"writing_plan",
}
missing = sorted(required - set(finding))
if missing:
raise ValueError(f"compressed finding missing fields: {missing}")
if not finding["chapter_id"]:
raise ValueError("chapter_id required")
if not finding["packet_ids"]:
raise ValueError("compressed finding requires packet_ids")
if not finding["chapter_thesis"]:
raise ValueError("compressed finding requires chapter_thesis")
if not finding["key_findings"]:
raise ValueError("compressed finding requires key_findings")
if not finding["evidence_landings"]:
raise ValueError("compressed finding requires evidence_landings")
if not finding["counter_evidence"]:
raise ValueError("compressed finding requires counter_evidence")
def validate_chapter_markdown_citations(markdown: str, brief: dict) -> None:
if "key_findings" in brief:
validate_compressed_finding(brief)
else:
validate_chapter_brief(brief)
cited = set(re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", markdown))
allowed = set(brief.get("source_ids") or [])
unknown = sorted(cited - allowed)
if unknown:
raise ValueError(f"unknown citation ids in {brief['chapter_id']}: {unknown}")
def _chapter_title_from_id(chapter_id: str) -> str:
try:
index = int(chapter_id.replace("ch", ""))
return f"{index}"
except ValueError:
return chapter_id
def _load_source_registry(sources_path: Path, source_ids: list[str]) -> list[dict]:
wanted = set(source_ids)
if not sources_path.exists() or not wanted:
return []
rows: list[dict] = []
for line in sources_path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
row = json.loads(line)
except json.JSONDecodeError:
continue
if row.get("id") in wanted:
rows.append(row)
return rows
def _cached_source_excerpts(project_root: Path, cached_paths: list[str], *, max_sources: int = 5, max_chars: int = 1400) -> list[dict]:
excerpts: list[dict] = []
for rel in cached_paths[:max_sources]:
path = project_root / rel
if not path.exists():
continue
text = path.read_text(encoding="utf-8", errors="ignore").strip()
excerpts.append({"path": rel, "excerpt": text[:max_chars]})
return excerpts
def build_chapter_briefs(project_root: Path) -> list[dict]:
cards = load_task_cards(project_root / "phase2" / "task_cards.json")
grouped: dict[str, list[tuple[str, dict]]] = {}
skipped_packets: list[dict[str, str]] = []
for card in cards:
packet_path = project_root / card.output_packet
if not packet_path.exists():
skipped_packets.append({"task_id": card.task_id, "reason": "packet file missing"})
continue
packet = json.loads(packet_path.read_text(encoding="utf-8"))
try:
validate_packet(packet)
except Exception as exc:
skipped_packets.append({"task_id": card.task_id, "reason": str(exc)})
continue
for chapter_id in card.chapter_ids:
grouped.setdefault(chapter_id, []).append((card.task_id, packet))
briefs: list[dict] = []
out_dir = project_root / "phase2" / "chapter_briefs"
out_dir.mkdir(parents=True, exist_ok=True)
if skipped_packets:
(project_root / "phase2" / "brief_warnings.json").write_text(
json.dumps(skipped_packets, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
for chapter_id in sorted(grouped):
packet_pairs = grouped[chapter_id]
packet_ids = [item[0] for item in packet_pairs]
packets = [item[1] for item in packet_pairs]
source_ids = sorted({sid for packet in packets for sid in packet.get("source_ids", [])})
source_registry = _load_source_registry(project_root / "phase2" / "sources.jsonl", source_ids)
cached_paths = [
source["cached_text_path"]
for source in source_registry
if source.get("cached_text_path")
]
chapter_title = next((card.chapter_title for card in cards if chapter_id in card.chapter_ids and card.chapter_title), None)
brief = {
"chapter_id": chapter_id,
"chapter_title": chapter_title or _chapter_title_from_id(chapter_id),
"packet_ids": packet_ids,
"core_claims": [claim for packet in packets for claim in packet.get("claims", [])],
"evidence_items": [item for packet in packets for item in packet.get("evidence_items", [])],
"counter_evidence": [item for packet in packets for item in packet.get("counter_evidence", [])],
"source_ids": source_ids,
"cached_source_paths": cached_paths,
"cached_source_excerpts": _cached_source_excerpts(project_root, cached_paths),
"open_questions": [q for packet in packets for q in packet.get("open_questions", [])],
"assembly_notes": [
"用中文写正式章节,英文仅保留在必要的来源标题、原文摘录、DOI/URL 中。",
"避免碎片化:不要按 packet 逐段堆砌,要先提炼本章主线,再组织证据。",
"每个事实、数字和关键判断都必须保留 [src_xxx] 引用。",
"必须纳入 counter_evidence,并说明它如何影响结论置信度。",
],
}
validate_chapter_brief(brief)
(out_dir / f"{chapter_id}.json").write_text(
json.dumps(brief, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
briefs.append(brief)
return briefs
def _source_ids_from_item(item: dict) -> list[str]:
if item.get("source_ids"):
return list(item.get("source_ids") or [])
if item.get("source_id"):
return [item["source_id"]]
return []
def build_compressed_findings(project_root: Path) -> list[dict]:
"""Compress packet-level evidence into chapter-level writing inputs.
This is intentionally deterministic: it does not invent a better narrative,
but it forces a chapter-level evidence map before any model writes prose.
"""
brief_dir = project_root / "phase2" / "chapter_briefs"
if not brief_dir.exists() or not list(brief_dir.glob("ch*.json")):
briefs = build_chapter_briefs(project_root)
else:
briefs = [
json.loads(path.read_text(encoding="utf-8"))
for path in sorted(brief_dir.glob("ch*.json"))
]
out_dir = project_root / "phase2" / "compressed_findings"
out_dir.mkdir(parents=True, exist_ok=True)
findings: list[dict] = []
for brief in briefs:
validate_chapter_brief(brief)
core_claims = brief.get("core_claims") or []
evidence_items = brief.get("evidence_items") or []
first_claim = core_claims[0] if core_claims else {}
chapter_thesis = first_claim.get("claim") or f"{brief['chapter_title']} 需要以证据为中心重写。"
finding = {
"chapter_id": brief["chapter_id"],
"chapter_title": brief["chapter_title"],
"packet_ids": brief["packet_ids"],
"chapter_thesis": chapter_thesis,
"key_findings": [
{
"finding": claim.get("claim") or claim.get("summary") or str(claim),
"source_ids": _source_ids_from_item(claim),
"confidence": claim.get("confidence", "medium"),
}
for claim in core_claims
],
"evidence_landings": [
{
"evidence": item.get("summary") or item.get("finding") or item.get("quote") or str(item),
"source_ids": _source_ids_from_item(item),
"landing_hint": item.get("landing_hint", "用于支撑本章关键判断或整改动作。"),
}
for item in evidence_items
],
"counter_evidence": brief["counter_evidence"],
"source_ids": brief["source_ids"],
"cached_source_paths": brief.get("cached_source_paths", []),
"cached_source_excerpts": brief.get("cached_source_excerpts", []),
"open_questions": brief["open_questions"],
"writing_plan": [
"先写本章判断,不按 packet 顺序堆砌。",
"每个二级小节至少落下具体审计发现、法规要求、记录/参数或整改证据。",
"正文末尾必须保留“证据落点与待补证据”表。",
],
}
validate_compressed_finding(finding)
(out_dir / f"{brief['chapter_id']}.json").write_text(
json.dumps(finding, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
findings.append(finding)
return findings
def build_chapter_user_prompt(brief: dict) -> str:
return (
"请根据以下 compressed finding / chapter brief 写一章正式中文 Markdown 正文。\n"
"目标是形成一个完整章节,而不是 packet 摘要。避免碎片化,按金字塔结构组织:章首先给结论,再用证据支撑。\n"
"要求:标题必须是观点型判断;每个数字和事实保留 [src_xxx];纳入反方证据;不要出现调度元数据。\n"
"如 brief 中包含 cached_source_paths,说明这些是已抓取到本地的核心一手/权威信源快照;优先使用 packet 已摘录的原文,并在证据不足时标记需要从本地快照补摘录,不要重新联网检索。\n"
"禁止写空泛咨询腔。每个二级小节都必须至少落下 2 个具体审计发现、法规要求、SOP/记录/参数/现场观察或整改证据;不要只写原则。\n"
"正文末尾必须增加“证据落点与待补证据”小节,用表格列出:关键判断、已使用证据 source_id、已落地整改动作、仍缺证据。若证据不足,直接标注需回炉 Phase 2,不要用泛泛表述补齐。\n"
"只输出 Markdown,不要输出解释。\n\n"
f"{json.dumps(brief, ensure_ascii=False, indent=2)}"
)
class ChapterAssemblyWorker:
def __init__(
self,
*,
role: RoleDefinition,
client: ChatClient,
skill_registry: SkillRegistry | None = None,
) -> None:
self.role = role
self.client = client
self.skill_registry = skill_registry or SkillRegistry()
def _system_prompt(self) -> str:
skill_texts = []
for name in self.role.skills:
try:
skill_texts.append(f"# Skill: {name}\n\n{self.skill_registry.read(name)}")
except FileNotFoundError:
skill_texts.append(f"# Skill: {name}\n\n[missing skill: {name}]")
return (
f"{self.role.identity}\n\n"
"你是 Deep Research v0.20 的中文章节组装 worker。\n"
"你的职责是把结构化证据包收束成连贯章节,解决并发研究造成的碎片化。\n"
"不得编造来源,不得删除关键反方证据。\n\n"
+ "\n\n".join(skill_texts)
)
def write_chapter(self, *, project_root: Path, brief: dict) -> Path:
if "key_findings" in brief:
validate_compressed_finding(brief)
else:
validate_chapter_brief(brief)
markdown = self.client.chat_complete(
model=self.role.model,
system=self._system_prompt(),
user=build_chapter_user_prompt(brief),
temperature=self.role.temperature,
max_tokens=self.role.max_tokens,
tag=f"chapter:{brief['chapter_id']}",
)
validate_chapter_markdown_citations(markdown, brief)
out = project_root / "phase2" / "drafts" / f"{brief['chapter_id']}.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(markdown.rstrip() + "\n", encoding="utf-8")
return out
def _write_chapter_error(project_root: Path, brief: dict, error: Exception) -> None:
path = project_root / "phase2" / "chapter_errors" / f"{brief.get('chapter_id', 'unknown')}.json"
path.parent.mkdir(parents=True, exist_ok=True)
payload = {
"chapter_id": brief.get("chapter_id"),
"status": "failed",
"error": str(error),
}
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def run_chapter_assembly_workers(
*,
project_root: Path,
briefs: list[dict],
runtime: RuntimeProfile,
client_factory: Callable[[RoleDefinition], ChatClient],
workers: int,
) -> int:
role = runtime.role_for_task("chapter_assembly")
max_workers = max(1, min(workers, role.max_concurrency))
def run_one(brief: dict) -> tuple[dict, Path | None, Exception | None]:
try:
worker = ChapterAssemblyWorker(role=role, client=client_factory(role))
return brief, worker.write_chapter(project_root=project_root, brief=brief), None
except Exception as error:
return brief, None, error
written = 0
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = [pool.submit(run_one, brief) for brief in briefs]
for future in as_completed(futures):
brief, path, error = future.result()
if error is not None:
_write_chapter_error(project_root, brief, error)
continue
if path is None:
_write_chapter_error(project_root, brief, RuntimeError("chapter worker returned no output path"))
continue
written += 1
return written
+229
View File
@@ -0,0 +1,229 @@
"""Phase 0 user-provided material ingestion."""
from __future__ import annotations
import base64
import os
import shutil
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import requests
DEFAULT_FIRERED_OCR_ENDPOINT = "http://192.168.50.100:8001"
DEFAULT_OCR_MAX_PAGES = 50
@dataclass(frozen=True)
class OcrResult:
text: str
pages_processed: int
output_path: Path
def safe_filename(path: Path) -> str:
name = path.name.strip()
return name or "material"
def extract_pdf_text(path: Path) -> tuple[str, int, bool]:
from pypdf import PdfReader
reader = PdfReader(str(path))
chunks: list[str] = []
for index, page in enumerate(reader.pages, start=1):
text = (page.extract_text() or "").strip()
if text:
chunks.append(f"\n\n## Page {index}\n\n{text}")
combined = "".join(chunks).strip()
ocr_required = len(combined) < max(20, len(reader.pages) * 20)
return combined, len(reader.pages), ocr_required
def ocr_endpoint_from_env() -> str:
return os.environ.get("DEEP_RESEARCH_OCR_ENDPOINT", DEFAULT_FIRERED_OCR_ENDPOINT).rstrip("/")
def ocr_max_pages_from_env() -> int:
raw = os.environ.get("DEEP_RESEARCH_OCR_MAX_PAGES")
if not raw:
return DEFAULT_OCR_MAX_PAGES
try:
return max(1, int(raw))
except ValueError:
return DEFAULT_OCR_MAX_PAGES
def render_pdf_pages(pdf_path: Path, output_dir: Path, *, max_pages: int) -> list[Path]:
import fitz
pages_dir = output_dir / f"{pdf_path.stem}.ocr-pages"
pages_dir.mkdir(parents=True, exist_ok=True)
image_paths: list[Path] = []
doc = fitz.open(pdf_path)
try:
for index, page in enumerate(doc[:max_pages], start=1):
pix = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False)
image_path = pages_dir / f"page-{index:03d}.png"
pix.save(image_path)
image_paths.append(image_path)
finally:
doc.close()
return image_paths
def data_url_for_image(path: Path) -> str:
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
return f"data:image/png;base64,{encoded}"
def call_firered_ocr(image_path: Path, *, endpoint: str) -> str:
payload = {
"model": "firered-ocr",
"messages": [
{
"role": "user",
"content": [
{"type": "text", "text": "请识别图片中的全部文字,保持原有顺序,只输出文字。"},
{"type": "image_url", "image_url": {"url": data_url_for_image(image_path)}},
],
}
],
"temperature": 0,
"max_tokens": 3000,
}
response = requests.post(f"{endpoint.rstrip('/')}/v1/chat/completions", json=payload, timeout=60)
if not response.ok:
raise RuntimeError(f"{response.status_code} {response.text[:500]}")
data = response.json()
return str(data["choices"][0]["message"].get("content") or "").strip()
def ocr_pdf_with_firered(*, pdf_path: Path, output_dir: Path, endpoint: str, max_pages: int) -> OcrResult:
image_paths = render_pdf_pages(pdf_path, output_dir, max_pages=max_pages)
chunks: list[str] = []
for index, image_path in enumerate(image_paths, start=1):
text = call_firered_ocr(image_path, endpoint=endpoint)
if text:
chunks.append(f"\n\n## OCR Page {index}\n\n{text}")
combined = "".join(chunks).strip()
output_path = output_dir / f"{pdf_path.stem}.ocr.md"
body = [
f"# OCR Material: {pdf_path.name}",
"",
f"- source_path: {pdf_path}",
f"- endpoint: {endpoint}",
f"- pages_processed: {len(image_paths)}",
"",
combined or "OCR 未返回可用文本。",
"",
]
output_path.write_text("\n".join(body), encoding="utf-8")
return OcrResult(text=combined, pages_processed=len(image_paths), output_path=output_path)
def ingest_input_materials(project_root: Path, materials: list[str] | None) -> list[dict[str, Any]]:
inventory: list[dict[str, Any]] = []
if not materials:
return inventory
inputs_dir = project_root / "phase0" / "inputs"
extracted_dir = project_root / "phase0" / "extracted"
ocr_endpoint = ocr_endpoint_from_env()
ocr_max_pages = ocr_max_pages_from_env()
inputs_dir.mkdir(parents=True, exist_ok=True)
extracted_dir.mkdir(parents=True, exist_ok=True)
for raw in materials:
source = Path(raw).expanduser()
if not source.exists():
inventory.append({"kind": "note", "note": raw})
continue
copied = inputs_dir / safe_filename(source)
shutil.copy2(source, copied)
item: dict[str, Any] = {
"kind": source.suffix.lower().lstrip(".") or "file",
"source_path": str(source),
"copied_to": str(copied.relative_to(project_root)),
"size_bytes": source.stat().st_size,
}
if source.suffix.lower() == ".pdf":
text, pages, ocr_required = extract_pdf_text(source)
extracted = extracted_dir / f"{source.stem}.md"
ocr_result: OcrResult | None = None
ocr_error: str | None = None
if ocr_required:
try:
ocr_result = ocr_pdf_with_firered(
pdf_path=source,
output_dir=extracted_dir,
endpoint=ocr_endpoint,
max_pages=min(pages, ocr_max_pages),
)
if ocr_result.text:
text = "\n\n".join(part for part in [text, ocr_result.text] if part)
except Exception as exc: # noqa: BLE001 - ingestion should not block project init.
ocr_error = str(exc)
body = [
f"# Extracted Material: {source.name}",
"",
f"- source_path: {source}",
f"- copied_to: {copied.relative_to(project_root)}",
f"- pages: {pages}",
f"- ocr_required: {str(ocr_required).lower()}",
f"- ocr_status: {'completed' if ocr_result else 'failed' if ocr_error else 'not_required'}",
"",
text or "未能从 PDF 直接抽取文本;该材料可能需要 OCR。",
"",
]
if ocr_error:
body.extend(["## OCR Error", "", ocr_error, ""])
extracted.write_text("\n".join(body), encoding="utf-8")
item.update(
{
"pages": pages,
"extracted_to": str(extracted.relative_to(project_root)),
"text_chars": len(text),
"ocr_required": ocr_required,
"ocr_status": "completed" if ocr_result else "failed" if ocr_error else "not_required",
}
)
if ocr_result:
item.update(
{
"ocr_endpoint": ocr_endpoint,
"ocr_pages_processed": ocr_result.pages_processed,
"ocr_extracted_to": str(ocr_result.output_path.relative_to(project_root)),
"ocr_text_chars": len(ocr_result.text),
}
)
if ocr_error:
item["ocr_error"] = ocr_error
else:
item["ocr_required"] = source.suffix.lower() in {".png", ".jpg", ".jpeg", ".tif", ".tiff"}
inventory.append(item)
return inventory
def render_material_inventory(inventory: list[dict[str, Any]]) -> str:
if not inventory:
return "- 暂无;可通过 `--input-material` 加入审计报告、问题清单或内部记录。"
lines: list[str] = []
for item in inventory:
if item.get("kind") == "note":
lines.append(f"- 备注:{item.get('note', '')}")
continue
marker = ";需要 OCR" if item.get("ocr_required") else ""
ocr = f"OCR{item.get('ocr_status')}" if item.get("ocr_status") else ""
extracted = item.get("extracted_to")
extra = f";抽取文本:{extracted}" if extracted else ""
lines.append(
f"- {item.get('copied_to')}{item.get('kind')}{item.get('size_bytes', 0)} bytes{extra}{marker}{ocr}"
)
return "\n".join(lines)
+61
View File
@@ -0,0 +1,61 @@
"""Research method registry for Phase 1 framework selection."""
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
DEFAULT_METHOD_CONFIG = REPO_ROOT / "configs" / "research_methods.yaml"
@dataclass(frozen=True)
class ResearchMethod:
key: str
name: str
best_for: list[str]
structure_principle: str
task_axes: list[str]
framework_sections: list[str]
integrated_lanes: list[str]
class ResearchMethodRegistry:
def __init__(self, path: Path | None = None) -> None:
self.path = path or DEFAULT_METHOD_CONFIG
self._data = self._load()
def _load(self) -> dict[str, Any]:
if not self.path.exists():
raise FileNotFoundError(f"research method config not found: {self.path}")
data = yaml.safe_load(self.path.read_text(encoding="utf-8")) or {}
if not isinstance(data, dict) or "methods" not in data:
raise ValueError(f"invalid research method config: {self.path}")
return data
@property
def default_method(self) -> str:
return (self._data.get("defaults") or {}).get("method", "mckinsey_market")
def list_names(self) -> list[str]:
return sorted((self._data.get("methods") or {}).keys())
def get(self, key: str | None = None) -> ResearchMethod:
selected = key or self.default_method
methods = self._data.get("methods") or {}
if selected not in methods:
raise KeyError(f"unknown research_method: {selected}")
item = methods[selected] or {}
return ResearchMethod(
key=selected,
name=item.get("name", selected),
best_for=list(item.get("best_for") or []),
structure_principle=item.get("structure_principle", ""),
task_axes=list(item.get("task_axes") or []),
framework_sections=list(item.get("framework_sections") or []),
integrated_lanes=list(item.get("integrated_lanes") or item.get("task_axes") or []),
)
+100
View File
@@ -0,0 +1,100 @@
"""Deterministic orchestration helpers for v0.20."""
from __future__ import annotations
import json
from datetime import datetime, timezone
from pathlib import Path
from scripts.runtime.artifacts import ensure_phase_dirs, load_manifest, write_manifest
from scripts.runtime.methods import ResearchMethodRegistry
from scripts.runtime.tasks import generate_task_cards, generate_task_cards_from_research_brief, write_task_cards
def utc_now_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def create_phase2_task_cards(
project_root: Path,
*,
axes: list[str] | None = None,
dry_run: bool = False,
) -> list[dict[str, object]]:
framework = project_root / "phase1" / "framework.md"
if not framework.exists():
raise FileNotFoundError(f"framework not found: {framework}")
method_key = None
if (project_root / "manifest.json").exists():
method_key = load_manifest(project_root).get("research_method")
method = ResearchMethodRegistry().get(method_key)
research_brief_path = project_root / "phase1" / "research_brief.json"
framework_text = framework.read_text(encoding="utf-8")
if research_brief_path.exists():
research_brief = json.loads(research_brief_path.read_text(encoding="utf-8"))
if not research_brief.get("materials"):
material_inventory = load_manifest(project_root).get("material_inventory") or []
materials = []
for item in material_inventory:
rel = item.get("ocr_extracted_to") or item.get("extracted_to") or item.get("copied_to")
if rel:
materials.append({"path": rel, "role": "input_material"})
if materials:
research_brief["materials"] = materials
cards = generate_task_cards_from_research_brief(
project_root.name,
framework_text,
research_brief,
axes=axes,
method=method,
)
else:
cards = generate_task_cards(project_root.name, framework_text, axes=axes, method=method)
if not dry_run:
ensure_phase_dirs(project_root)
write_task_cards(project_root / "phase2" / "task_cards.json", cards)
manifest = load_manifest(project_root)
phase2 = manifest.setdefault("phase2", {})
phase2.update(
{
"status": "in_progress",
"runtime": "python-core-v0.20",
"research_method": method.key,
"task_cards_path": "phase2/task_cards.json",
"task_cards_total": len(cards),
"updated_at": utc_now_iso(),
}
)
write_manifest(project_root, manifest)
return [card.to_dict() for card in cards]
def write_placeholder_packets(
project_root: Path,
task_cards: list[dict[str, object]],
*,
dry_run: bool = False,
) -> int:
"""Create packet skeletons for manual/API completion.
This keeps the first v0.20 implementation deterministic and resumable; LLM
calls can later fill the same schema without changing downstream readers.
"""
count = 0
for card in task_cards:
packet_path = project_root / str(card["output_packet"])
packet = {
"task_id": card["task_id"],
"claims": [],
"evidence_items": [],
"counter_evidence": [],
"source_ids": [],
"source_quality_notes": [],
"open_questions": ["待由 Python role worker 调用模型补全。"],
"raw_quotes_or_notes": [],
}
if not dry_run:
packet_path.parent.mkdir(parents=True, exist_ok=True)
packet_path.write_text(json.dumps(packet, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
count += 1
return count
+902
View File
@@ -0,0 +1,902 @@
"""Phase 1 project initialization and framework generation."""
from __future__ import annotations
import hashlib
import json
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from scripts.runtime.artifacts import PROJECTS_DIR, ensure_phase_dirs, load_manifest, write_manifest
from scripts.runtime.materials import ingest_input_materials, render_material_inventory
from scripts.runtime.methods import ResearchMethod, ResearchMethodRegistry
from scripts.runtime.tasks import AXIS_ROUTES
def utc_now_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def slugify_topic(topic: str) -> str:
slug = re.sub(r"[^a-zA-Z0-9]+", "-", topic.lower()).strip("-")
if slug:
return slug[:80]
digest = hashlib.sha1(topic.encode("utf-8")).hexdigest()[:8]
return f"research-{digest}"
def create_project(
*,
topic: str,
slug: str | None = None,
projects_dir: Path = PROJECTS_DIR,
method_key: str | None = None,
report_type: str = "research",
model_profile: str = "medium",
target_words: int = 30000,
input_materials: list[str] | None = None,
) -> Path:
method = ResearchMethodRegistry().get(method_key)
project_slug = slug or slugify_topic(topic)
project_root = projects_dir / project_slug
if project_root.exists():
raise FileExistsError(f"project already exists: {project_root}")
ensure_phase_dirs(project_root)
(project_root / "phase0" / "inputs").mkdir(parents=True, exist_ok=True)
(project_root / "phase0" / "extracted").mkdir(parents=True, exist_ok=True)
material_inventory = ingest_input_materials(project_root, input_materials)
now = utc_now_iso()
manifest: dict[str, Any] = {
"version": "0.20.0",
"runtime": "python-core-v0.20",
"topic": topic,
"slug": project_slug,
"report_title": topic,
"type": report_type,
"work_language": "zh",
"model_profile": model_profile,
"research_method": method.key,
"target_words": target_words,
"input_materials": input_materials or [],
"material_inventory": material_inventory,
"created_at": now,
"updated_at": now,
"phase1": {
"status": "initialized",
"approved": False,
"requires_user_interview": True,
"material_brief_path": "phase1/material_brief.md",
},
"phase2": {"status": "pending"},
"phase3": {"status": "pending"},
"phase4": {"status": "pending"},
}
write_manifest(project_root, manifest)
_write_interview_seed(project_root, manifest, method)
write_material_brief(project_root, manifest, method)
return project_root
def _write_interview_seed(project_root: Path, manifest: dict[str, Any], method: ResearchMethod) -> None:
material_text = render_material_inventory(manifest.get("material_inventory") or [])
text = (
f"# Phase 1 访谈记录\n\n"
f"- 主题:{manifest['topic']}\n"
f"- 研究方法:{method.key} - {method.name}\n"
f"- 报告类型:{manifest['type']}\n"
f"- 目标字数:{manifest['target_words']}\n"
f"- 工作语言:中文主写作;检索关键词、证据摘录和来源笔记可保留英文。\n\n"
f"## 已提供材料\n\n{material_text}\n\n"
"## 后续访谈问题\n\n"
"1. 本报告最重要的决策用途是什么?\n"
"2. 是否有必须覆盖或必须排除的公司、产品、工艺、市场或法规范围?\n"
"3. 结论偏好是战略建议、风险清单、投资判断,还是执行路线图?\n"
)
(project_root / "phase1" / "interview.md").write_text(text, encoding="utf-8")
def _material_excerpt(project_root: Path, rel_path: str, *, max_chars: int = 1200) -> str:
path = project_root / rel_path
if not path.exists():
return "(未找到抽取文本)"
text = path.read_text(encoding="utf-8")
compact = "\n".join(line.rstrip() for line in text.splitlines() if line.strip())
return compact[:max_chars] + ("..." if len(compact) > max_chars else "")
def _derive_material_observations(project_root: Path, inventory: list[dict[str, Any]]) -> list[str]:
combined_parts: list[str] = []
for item in inventory:
rel = item.get("ocr_extracted_to") or item.get("extracted_to")
if rel and (project_root / rel).exists():
combined_parts.append((project_root / rel).read_text(encoding="utf-8"))
text = "\n".join(combined_parts)
checks = [
("审计范围覆盖生产管理、原液、制剂和无菌相关模块,报告需要同时处理 GMP 合规、工艺转移和运营协同,而不是只写质量体系。", ["生产管理", "原液", "制剂", "无菌"]),
("材料显示高风险项为 0、中风险项为 1,适合采用“商业化 readiness 与系统成熟度差距”而非“体系失控”作为初始假设。", ["高风险 0", "中风险1", "低风险7"]),
("商业化经验、无菌保障细节、文件要求与执行一致性是需要访谈确认的主线风险。", ["商业化经验不足", "无菌保障", "文件要求与执行一致性"]),
("工艺规程、批记录、CPP/CQA、VMPR/VMP、验证主计划等内容反复出现,说明工艺验证和商业化文件体系可能是 Phase 2 的重点证据轴。", ["CPP", "CQA", "VMPR", "VMP"]),
("温度、压差、WFI、冷却段微生物、RABS/ORABS、first air、APS 等无菌和设施细节需要映射到 EU Annex 1、NMPA GMP 和企业 SOP。", ["温度", "压差", "WFI", "APS"]),
("复盘材料包含责任人和局部答复,后续整改路线图应尽量回填 owner、期限、关闭证据和复核机制。", ["填写人", "是否已经回答完整", "整改"]),
]
observations = [message for message, needles in checks if any(needle in text for needle in needles)]
return observations or ["材料已导入但尚未形成足够结构化判断;需要先访谈确认研究用途、范围和优先级。"]
def write_material_brief(
project_root: Path,
manifest: dict[str, Any] | None = None,
method: ResearchMethod | None = None,
) -> Path:
"""Write a Phase 0/1 material brief that must be reviewed before Phase 2."""
manifest = manifest or load_manifest(project_root)
method = method or ResearchMethodRegistry().get(manifest.get("research_method"))
inventory = manifest.get("material_inventory") or []
lines = [
f"# Phase 0 材料简报:{manifest.get('topic', project_root.name)}",
"",
"status: 待用户确认",
f"research_method: {method.key}",
"",
"## 已导入材料",
"",
render_material_inventory(inventory),
"",
"## 材料初步解读",
"",
"以下内容由 Python core 从已落盘材料抽样生成,只作为访谈起点;不得直接视为最终结论。",
"",
]
lines.extend(["## 初步问题聚类(待访谈确认)", ""])
for observation in _derive_material_observations(project_root, inventory):
lines.append(f"- {observation}")
lines.append("")
lines.append("## 材料摘录")
lines.append("")
for item in inventory:
rel = item.get("ocr_extracted_to") or item.get("extracted_to")
if not rel:
continue
lines.extend(
[
f"### {Path(rel).name}",
"",
_material_excerpt(project_root, rel),
"",
]
)
lines.extend(
[
"## 建议访谈确认点",
"",
"1. 本报告的最重要用途是什么:内部整改、客户沟通、董事会决策,还是外部审计准备?",
"2. 哪些审计发现最需要优先展开:无菌保障、工艺验证、数据完整性、质量体系闭环,还是运营协同?",
"3. 是否存在必须排除或脱敏的项目、人员、客户、产品或工艺信息?",
"4. 短中长期整改的时间边界如何定义,例如 30/90/180 天,还是按临床/商业化里程碑划分?",
"5. 是否需要把 NMPA、FDA、EMA、ICH、WHO 的法规基线分别映射到整改责任人和证据包?",
"",
"## Gate",
"",
"请用户确认本材料简报与访谈问题后,再生成或批准 `phase1/framework.md` 并进入 Phase 2。",
"",
]
)
out = project_root / "phase1" / "material_brief.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("\n".join(lines), encoding="utf-8")
return out
def _axis_prompt_brief(axis: str, method: ResearchMethod) -> str:
prompts = {
"input_material_findings": "从用户材料中提取现场事实、审计发现、复盘记录和内部答复,并标注原始材料位置。",
"nmpa_fda_ema_ich_who_baseline": "把 NMPA、FDA、EMA、ICH、WHO、药典或 Annex 1 等要求转化为可核验的法规基线,并纳入 FDA warning letters 与会议材料作为执法尺度参照。",
"quality_system_gap": "把现场发现映射到质量体系流程缺口,覆盖偏差、变更、CAPA、文件、培训和数据完整性;优先检索 FDA warning letters 中同类缺陷的执法表述。",
"manufacturing_process_risk": "围绕生产工艺、设施、公用系统、CPP/CQA、验证和无菌保障识别系统性风险,并用 FDA warning letters / inspection enforcement examples 校准严重度。",
"operations_management_gap": "诊断运营管理、跨部门协同、会议机制、指标体系和交付节奏的结构性问题,并参考 FDA 会议纪要/meeting materials 中对质量治理的关注点。",
"team_capability": "识别人员能力、岗位职责、质量文化和管理梯队方面的缺口与建设路径。",
"capa_roadmap": "把差距转化为短中长期 CAPA 组合,要求绑定 owner、期限、优先级、关闭证据和复核机制。",
"verification_evidence": "定义整改完成后可被审计接受的验证证据,包括记录、报告、趋势和管理评审输入。",
"counter": "主动寻找反方证据、限制条件和可能降低严重度或改变优先级的解释,避免单向论证。",
}
return prompts.get(axis, f"按照 `{method.key}` 方法,对 {axis} 轴进行证据收集、证伪和结构化归纳。")
def _material_paths(manifest: dict[str, Any]) -> list[dict[str, str]]:
materials: list[dict[str, str]] = []
for item in manifest.get("material_inventory") or []:
rel = item.get("ocr_extracted_to") or item.get("extracted_to") or item.get("copied_to")
if rel:
materials.append({"path": rel, "role": "input_material"})
return materials
def _keywords_from_title(title: str) -> list[str]:
english = re.findall(r"[A-Za-z][A-Za-z0-9/+-]{1,}", title)
chinese_parts = re.split(r"[,、;;:\s]+|和|与|及|的|在|为|从|来自|集中|决定|需要|形成|成为|不是|而是", title)
domain_terms = [
"审计",
"商业化",
"阶段门",
"风险",
"法规",
"欧盟",
"NMPA",
"GMP",
"ICH",
"无菌",
"RABS",
"First Air",
"APS",
"灯检",
"隧道",
"原液",
"WFI",
"SCADA",
"EMS",
"CPP",
"CQA",
"PPQ",
"清洁验证",
"偏差",
"变更",
"CAPA",
"数据完整性",
"人员",
"培训",
"质量文化",
"运营",
"跨部门",
"指标",
"团队",
"CDMO",
"整改",
"owner",
]
title_terms = [term for term in domain_terms if term in title]
keywords = [item.strip() for item in [*english, *title_terms, *chinese_parts] if len(item.strip()) >= 2]
seen: set[str] = set()
unique: list[str] = []
for keyword in keywords:
if keyword not in seen:
seen.add(keyword)
unique.append(keyword)
return unique[:12]
def _material_lines_for_chapter(project_root: Path, manifest: dict[str, Any], title: str, *, limit: int = 4) -> list[str]:
keywords = _keywords_from_title(title)
candidates: list[tuple[int, int, str]] = []
order = 0
for item in manifest.get("material_inventory") or []:
rel = item.get("ocr_extracted_to") or item.get("extracted_to")
if not rel:
continue
path = project_root / rel
if not path.exists():
continue
for raw in path.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if len(line) < 8 or len(line) > 220:
continue
if line.startswith("#") or line.startswith("- source_path:") or line.startswith("- extracted_at:"):
continue
if "OCR Material:" in line:
continue
if re.match(r"^(审计对象|审计执行方|审计执行人|审计时间)[::]", line):
continue
score = sum(1 for keyword in keywords if keyword and keyword in line)
if score:
order += 1
candidates.append((score, order, f"{rel}{line}"))
candidates.sort(key=lambda item: (-item[0], item[1]))
return [line for _, _, line in candidates[:limit]]
def _minimum_evidence_for_method(method: ResearchMethod) -> dict[str, Any]:
if method.key == "gmp_quality_operations_diagnosis":
return {
"local_material_quotes": 2,
"official_regulatory_or_guideline_sources": 2,
"enforcement_or_best_practice_precedents": 1,
"counter_evidence_or_boundary_conditions": 1,
"actionable_remediation_items": 3,
}
return {
"high_quality_sources": 4,
"tier_1_2_sources": 2,
"counter_evidence_or_boundary_conditions": 1,
"decision_relevant_implications": 2,
}
def _central_thesis(manifest: dict[str, Any], method: ResearchMethod) -> str:
topic = manifest.get("topic") or manifest.get("report_title") or "本研究主题"
if method.key == "gmp_quality_operations_diagnosis":
return (
f"初始主判断:{topic} 不应只按审计风险项数量来评价,而应从商业化 readiness、"
"质量体系运行成熟度、生产工艺证据链和运营协同能力四条线同时诊断。Phase 2 必须用"
"现场材料原文、官方法规/指南、执法案例或标杆实践来证明、修正或推翻这一判断。"
)
return (
f"初始主判断:{topic} 需要先形成可被证据推翻的观点型框架,再由 Phase 2 按方法论证据线"
"逐项求证;不能把并发检索结果直接堆砌成报告。"
)
def _strategy_for_chapter(title: str, method: ResearchMethod) -> dict[str, Any]:
"""Return non-tautological Phase 1 strategy text for a chapter title."""
if method.key != "gmp_quality_operations_diagnosis":
return {
"core_question": f"本章需要判断:在什么证据条件下“{title}”成立,它会怎样改变最终决策?",
"bold_hypothesis": f"初始假设不是复述标题,而是预判“{title}”背后存在一个可被验证的因果机制;Phase 2 需要找证据支持、修正或推翻这个机制。",
"writing_claim": f"本章要把“{title}”写成一个可被证据检验的判断,而不是资料综述。",
"counter_evidence": [
"是否存在更简单的替代解释,能削弱本章主判断?",
"关键证据是否只来自单一来源或利益相关来源?",
"是否有反例显示本章判断只适用于部分场景?",
],
}
strategies = [
(
("审计", "阶段门"),
{
"core_question": "审计报告的低/中风险项计数,是否低估了白帆从临床/受托生产走向商业化标准时需要跨过的阶段门?",
"bold_hypothesis": "初始假设:白帆的硬件和文件基础总体可用,但审计材料暴露的是商业化 readiness 缺口,而不是简单的若干孤立缺陷;Phase 2 应验证这些缺口是否集中在无菌保障、工艺验证、质量闭环和运营节奏。",
"writing_claim": "本章要先把“风险项清单”翻译成管理层可决策的阶段门地图,说明哪些问题影响商业化放行、客户审计和技术转移节奏。",
"counter_evidence": [
"是否已有整改证据证明这些问题只是审计时点的临时缺口?",
"低/中风险评级是否足以说明商业化阶段门影响有限?",
"审计范围有限是否导致本章不能外推到整体体系成熟度?",
],
},
),
(
("法规", "欧盟", "NMPA", "ICH"),
{
"core_question": "如果按 EU Annex 1、NMPA GMP、ICH Q9/Q10 以及 FDA 执法尺度校准,哪些现场发现的严重度和整改优先级会发生变化?",
"bold_hypothesis": "初始假设:白帆按国内 GMP 逻辑已具备基础合规框架,但若以欧盟无菌标准和质量风险管理要求衡量,部分“低风险/建议项”会转化为体系成熟度缺口。",
"writing_claim": "本章要建立后文共用的法规基线,避免整改优先级只跟随原审计评级,而忽略国际化和商业化标准。",
"counter_evidence": [
"相关国际标准是否并不适用于当前产品阶段或委托生产边界?",
"NMPA 与欧盟/美国要求之间是否存在可接受差异?",
"是否有企业内部标准已经覆盖但审计材料未呈现?",
],
},
),
(
("无菌", "RABS", "First Air", "APS", "灯检"),
{
"core_question": "制剂线的主要无菌风险,是硬件布局不足,还是人员干预、首次气流保护、APS 覆盖和灯检标准执行证据不足?",
"bold_hypothesis": "初始假设:白帆制剂车间硬件基础并非主要短板,真正风险在于关键无菌行为和模拟验证是否能持续证明受控;Phase 2 应重点查 First Air、RABS 干预、APS 场景设计和灯检阳性样品管理。",
"writing_claim": "本章要把无菌保障从“设施看起来合规”推进到“关键操作和验证证据可被审计接受”。",
"counter_evidence": [
"现场是否已有完整视频复核、APS 覆盖和再培训有效性证据?",
"观察到的无菌动作问题是否只是个别人员或单次拍摄偏差?",
"灯检和 RABS 风险是否已有 SOP、趋势和复核记录闭环?",
],
},
),
(
("原液", "WFI", "SCADA", "EMS"),
{
"core_question": "原液和公用系统的风险是否被一次性封闭工艺掩盖,真正缺口在 WFI、SCADA/EMS、离线记录和异常升级证据链?",
"bold_hypothesis": "初始假设:一次性反应器和封闭转移降低了暴露风险,但不能自动证明系统受控;Phase 2 应验证 WFI 冷却回流、环境/压差报警、SCADA 数据和离线检测记录是否形成完整证据链。",
"writing_claim": "本章要说明原液与公用系统不是“硬件先进即可”,而是要证明关键状态、报警、数据和异常处理持续受控。",
"counter_evidence": [
"WFI、SCADA/EMS 和离线记录是否已有验证报告与趋势复核?",
"一次性系统是否已经充分降低共线和交叉污染风险?",
"被指出的公用系统风险是否只是设计建议而非实际偏差?",
],
},
),
(
("工艺", "CPP", "CQA", "PPQ", "清洁验证"),
{
"core_question": "现有 IND 阶段工艺规程和批记录,距离商业化 PPQ、控制策略和清洁验证所需证据还差在哪里?",
"bold_hypothesis": "初始假设:白帆目前的工艺文件足以支撑临床阶段执行,但不足以支撑商业化批记录、CPP/CQA 控制、PPQ 和清洁验证闭环;Phase 2 应查明哪些字段、参数和验证证据必须前置补齐。",
"writing_claim": "本章要把技术转移风险具体化为文件、参数、验证和批记录的硬门槛。",
"counter_evidence": [
"是否已有商业化模板、控制策略或 PPQ 草案未体现在审计材料中?",
"当前项目阶段是否尚不需要完整商业化批记录要求?",
"清洁验证和工艺验证是否已有主计划覆盖?",
],
},
),
(
("偏差", "变更", "CAPA", "数据完整性"),
{
"core_question": "白帆的问题是没有质量流程,还是流程之间的事件分类、升级、CAPA 有效性和数据完整性尚未形成运行闭环?",
"bold_hypothesis": "初始假设:白帆已有偏差、变更和 CAPA 的流程框架,但事件何时启动偏差、何时作为变更、如何证明 CAPA 有效,以及电子/纸质数据如何贯通,仍存在运行机制缺口。",
"writing_claim": "本章要把质量体系从“有 SOP”推进到“事件能被正确分类、调查、纠正、验证并趋势复核”。",
"counter_evidence": [
"是否有趋势分析、管理评审和 CAPA effectiveness check 证明体系已经闭环?",
"个别事件分类问题是否不足以代表体系性缺口?",
"电子系统和纸质记录之间是否已有数据完整性控制?",
],
},
),
(
("人员", "培训", "质量文化"),
{
"core_question": "培训记录齐全是否真的转化为一线无菌行为、偏差判断和质量风险意识?哪些证据能证明培训有效?",
"bold_hypothesis": "初始假设:白帆不缺培训台账,缺的是把培训结果转化为现场行为的一致性证据;如果 First Air、干预动作、事件判断和灯检执行仍需反复提醒,问题就不是“再培训一次”,而是培训有效性确认和质量文化运行机制不足。",
"writing_claim": "本章要把人员问题从“有没有培训”改写为“培训是否改变行为、降低风险、形成可复核证据”。",
"counter_evidence": [
"现场抽问、资格确认和再培训记录是否已证明人员理解到位?",
"被观察到的行为问题是否只发生在少数岗位或单次演示?",
"是否有岗位胜任力矩阵、年度复评和行为观察数据支撑人员能力?",
],
},
),
(
("运营", "跨部门", "指标", "review"),
{
"core_question": "白帆当前整改和生产准备依赖个人推动,还是已经形成跨部门例会、问题升级、指标看板和管理层复核的运营系统?",
"bold_hypothesis": "初始假设:运营短板不在于团队不努力,而在于缺少固定节奏和可视化管理系统;如果 owner、关闭证据、升级阈值和管理层 review 不稳定,整改会停留在临时协调,难以支撑商业化节奏。",
"writing_claim": "本章要说明运营管理是 GMP 风险的放大器:没有节奏、看板和升级机制,技术和质量问题会反复跨部门漂移。",
"counter_evidence": [
"是否已经存在稳定 PMO/例会/看板,只是未进入审计材料?",
"短期临时协调是否足以覆盖当前项目阶段,不需要完整运营系统?",
"owner、期限和关闭证据是否已经在复盘文件中基本清楚?",
],
},
),
(
("团队", "CDMO", "能力矩阵"),
{
"core_question": "对标成熟 CDMO,白帆最需要补齐的是人数、岗位能力,还是 QA/MSAT/工程/项目管理之间的角色分工?",
"bold_hypothesis": "初始假设:白帆的能力缺口不是简单扩编,而是商业化 CDMO 所需的角色矩阵尚未完全成型;Phase 2 应验证 QA 独立性、MSAT 工艺支持、工程保障、生产班组和 PMO 协同能力。",
"writing_claim": "本章要给出面向商业化的团队能力地图,说明哪些能力必须自建,哪些可外部支持,哪些要通过机制补齐。",
"counter_evidence": [
"现有人员是否已具备商业化经验,只是材料未体现?",
"对标 CDMO 是否会高估当前阶段所需组织复杂度?",
"是否可通过顾问、外包或客户支持临时补足能力?",
],
},
),
(
("整改", "owner", "路线图"),
{
"core_question": "哪些整改必须立即完成,哪些属于体系补强,哪些是能力建设?每项如何绑定 owner、关闭证据和复核窗口?",
"bold_hypothesis": "初始假设:如果整改只按问题清单逐条关闭,会漏掉体系性根因;更有效的路线应分为立即纠偏、90 天体系补强和中长期能力建设三层,并为每层定义关闭证据。",
"writing_claim": "本章要把诊断转化为可执行 CAPA 组合,而不是泛泛的改进建议。",
"counter_evidence": [
"是否已有整改计划足以覆盖 owner、期限、关闭证据和 QA verification",
"部分整改是否应前移或后移,避免资源过载?",
"哪些建议若缺少法规证据,不应被列为强制整改?",
],
},
),
(
("管理层", "CAPA", "总表"),
{
"core_question": "管理层应通过什么样的 CAPA 总表、法规映射表和复核节奏,持续判断整改是否真正降低风险?",
"bold_hypothesis": "初始假设:白帆需要的不只是一次性报告,而是一套管理层可追踪的整改仪表盘;否则 CAPA 关闭会变成文件动作,无法证明风险趋势下降和商业化 readiness 提升。",
"writing_claim": "本章要把报告成果固化成管理层治理工具:CAPA 总表、法规映射、证据包和复核节奏。",
"counter_evidence": [
"现有管理评审或质量例会是否已经能承担这个功能?",
"过度表格化是否会增加一线负担而不改善风险?",
"哪些指标真正能反映风险降低,而不是制造形式化 KPI?",
],
},
),
]
for needles, strategy in strategies:
if any(needle in title for needle in needles):
return strategy
return {
"core_question": f"本章需要判断“{title}”背后的真实风险、适用边界和整改优先级。",
"bold_hypothesis": f"初始假设:{title} 不是孤立问题,而是质量体系、工艺证据或运营机制中的一个可验证缺口;Phase 2 必须用材料原文和外部证据判断其严重度。",
"writing_claim": f"本章要把“{title}”转化为可执行的诊断结论和整改要求。",
"counter_evidence": [
"该问题是否已有充分整改或验证证据?",
"是否只是阶段性限制,而非系统性缺口?",
"外部标准是否适用于当前业务边界?",
],
}
def build_chapter_planning(
project_root: Path,
manifest: dict[str, Any],
method: ResearchMethod,
titles: list[str],
*,
quota: int,
) -> list[dict[str, Any]]:
"""Build hypothesis-driven chapter plans that become Phase 2 prompt context."""
lanes = list(method.integrated_lanes or method.task_axes)
minimum_evidence = _minimum_evidence_for_method(method)
plans: list[dict[str, Any]] = []
for idx, title in enumerate(titles, start=1):
chapter_id = f"ch{idx:02d}"
material_lines = _material_lines_for_chapter(project_root, manifest, title)
if not material_lines:
material_lines = ["未在材料中自动匹配到足够线索;Phase 2 必须先回读全部输入材料并补充原文摘录。"]
strategy = _strategy_for_chapter(title, method)
core_question = strategy["core_question"]
bold_hypothesis = strategy["bold_hypothesis"]
verification_plan = [
"先从允许的本地材料提取 2-4 条原文证据,保留出处和上下文。",
f"再按方法论 evidence lanes 求证:{''.join(lanes)}",
"每个核心判断至少匹配 2 个独立高质量来源;不足时降级为待验证判断。",
"主动搜索反方证据、低严重度解释、适用范围限制或替代原因。",
"输出时把证据、判断、整改/建议和待补证据分开,避免直接写成散文化正文。",
]
counter_evidence = strategy["counter_evidence"]
writing_claim = strategy["writing_claim"]
phase2_prompt_context = "\n".join(
[
f"章节:{chapter_id} {title}",
core_question,
bold_hypothesis,
"材料起点:",
*[f"- {line}" for line in material_lines],
"求证路线:",
*[f"- {item}" for item in verification_plan],
"必须寻找的反方/边界:",
*[f"- {item}" for item in counter_evidence],
f"写作主张:{writing_claim}",
f"最低证据要求:{json.dumps(minimum_evidence, ensure_ascii=False)}",
]
)
plans.append(
{
"chapter_id": chapter_id,
"title": title,
"suggested_words": quota,
"core_question": core_question,
"bold_hypothesis": bold_hypothesis,
"why_this_matters": "本章用于把 Phase1 的判断转化为 Phase2 可验证命题,并为最终报告保留清晰主线。",
"material_starting_points": material_lines,
"evidence_lanes": lanes,
"verification_plan": verification_plan,
"counter_evidence_to_seek": counter_evidence,
"writing_claim": writing_claim,
"minimum_evidence": minimum_evidence,
"phase2_prompt_context": phase2_prompt_context,
}
)
return plans
def build_research_brief_payload(
project_root: Path,
manifest: dict[str, Any],
method: ResearchMethod,
chapter_planning: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
"""Create the file-backed Phase 1 research brief used by task-card generation."""
axes = list(method.task_axes)
chapter_planning = chapter_planning or []
return {
"version": "0.21-alpha",
"topic": manifest.get("topic", project_root.name),
"research_method": method.key,
"method_name": method.name,
"work_language": "zh",
"tone": "事实型、整改导向、面向管理层和质量/生产负责人;避免空泛咨询腔。",
"central_question": f"如何基于已提供材料和权威法规/最佳实践,系统诊断“{manifest.get('topic', project_root.name)}”并形成可执行整改路线图?",
"central_thesis": _central_thesis(manifest, method),
"phase_logic": {
"phase1": "大胆假设:结合输入材料、访谈信息和初步搜索,定下主基调、章节命题和求证路线。",
"phase2": "小心求证:worker 只围绕 Phase1 命题收集、验证、证伪和补证,不自行重写研究方向。",
"phase3": "一致性审校:检查 Phase1 假设与 Phase2 证据是否自洽,指出需要回炉的章节或证据缺口。",
},
"phase2_mode": "chapter_integrated",
"success_criteria": [
"每个核心判断都能回到用户材料、权威法规、最佳实践或反方证据。",
"短中长期整改建议必须绑定优先级、责任、关闭证据和复核机制。",
"章节写作必须先收束主线,再使用 evidence packet;不得按 packet 机械拼贴。",
],
"phase2_inputs": {
"material_brief_path": "phase1/material_brief.md",
"framework_path": "phase1/framework.md",
"research_brief_path": "phase1/research_brief.json",
},
"materials": _material_paths(manifest),
"chapter_planning": chapter_planning,
"task_planning": {
"chapter_source": "phase1/framework.md",
"phase2_mode": "chapter_integrated",
"axes": axes,
"required_skills": [
"deep-research",
"search-gateway",
"search-strategy",
"source-quality",
"evidence-table",
"citation-manager",
],
"search_routes_by_axis": {axis: AXIS_ROUTES.get(axis, ["general"]) for axis in axes},
"axis_prompt_briefs": {axis: _axis_prompt_brief(axis, method) for axis in axes},
"stop_conditions": [
"每张任务卡至少形成 3 条可追溯 evidence_items,且不得编造 candidate_sources 以外来源。",
"关键 claim 不足 2 个独立 Tier 1-2 信源时,必须写入 open_questions 和证据缺口。",
"必须包含 counter_evidence;找不到反方证据时记录检索路径和限制。",
],
"fragmentation_guard": "并发 worker 只生产 evidence packet;章节主线由 compressed_findings 收束,禁止直接把 packet 堆成正文。",
},
"clarification_notes": {
"requires_user_review": True,
"questions_source": "phase1/material_brief.md",
"decision_items": [
"确认报告用途、受众和脱敏边界。",
"确认研究方法是否适配当前场景;MECE 只是可选方法之一。",
"确认任务切分和检索策略是否足以让低成本模型独立执行。",
],
},
}
def write_research_brief(
project_root: Path,
manifest: dict[str, Any] | None = None,
method: ResearchMethod | None = None,
chapter_planning: list[dict[str, Any]] | None = None,
) -> tuple[Path, Path]:
manifest = manifest or load_manifest(project_root)
method = method or ResearchMethodRegistry().get(manifest.get("research_method"))
payload = build_research_brief_payload(project_root, manifest, method, chapter_planning=chapter_planning)
json_path = project_root / "phase1" / "research_brief.json"
md_path = project_root / "phase1" / "research_brief.md"
json_path.parent.mkdir(parents=True, exist_ok=True)
hypothesis_path = project_root / "phase1" / "hypothesis_map.json"
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
hypothesis_path.write_text(json.dumps(payload.get("chapter_planning") or [], ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
lines = [
f"# Phase 1 Research Brief{payload['topic']}",
"",
f"- research_method: {payload['research_method']}",
f"- work_language: {payload['work_language']}",
f"- tone: {payload['tone']}",
f"- phase2_mode: {payload['phase2_mode']}",
"",
"## 中心问题",
"",
payload["central_question"],
"",
"## 主基调 / 大胆假设",
"",
payload["central_thesis"],
"",
"## Phase 逻辑",
"",
]
for phase_name, phase_text in payload["phase_logic"].items():
lines.append(f"- `{phase_name}`{phase_text}")
lines.extend([
"",
"## 成功标准",
"",
])
lines.extend(f"- {item}" for item in payload["success_criteria"])
if payload.get("chapter_planning"):
lines.extend(["", "## 章节命题与求证计划", ""])
for item in payload["chapter_planning"]:
lines.extend(
[
f"### {item['chapter_id']} {item['title']}",
"",
f"- 核心问题:{item['core_question']}",
f"- 大胆假设:{item['bold_hypothesis']}",
f"- 写作主张:{item['writing_claim']}",
f"- 证据线:{''.join(item['evidence_lanes'])}",
"- 材料起点:",
]
)
lines.extend(f" - {line}" for line in item["material_starting_points"])
lines.extend(["- 求证计划:"])
lines.extend(f" - {line}" for line in item["verification_plan"])
lines.extend([""])
lines.extend(["", "## 任务切分原则", ""])
planning = payload["task_planning"]
lines.append(planning["fragmentation_guard"])
lines.append("")
for axis in planning["axes"]:
routes = "".join(planning["search_routes_by_axis"].get(axis, []))
prompt = planning["axis_prompt_briefs"].get(axis, "")
lines.append(f"- `{axis}`{prompt} 检索路径:{routes}")
lines.extend(["", "## 必读 Skills", ""])
lines.extend(f"- {name}" for name in planning["required_skills"])
lines.extend(["", "## 停止条件", ""])
lines.extend(f"- {item}" for item in planning["stop_conditions"])
lines.append("")
md_path.write_text("\n".join(lines), encoding="utf-8")
return md_path, json_path
CHAPTER_TEMPLATES: dict[str, list[str]] = {
"mckinsey_market": [
"核心结论先行界定市场机会与约束",
"临床与真实世界证据决定需求天花板",
"监管路径和支付环境重塑商业化节奏",
"竞争格局正在从单点产品转向组合能力",
"专利与技术壁垒决定长期利润池",
"中国市场的准入和供给能力形成独立变量",
"资本市场预期与基本面之间存在可验证偏差",
"反方证据限定结论边界并提示回撤风险",
"战略选择应围绕资源约束排序",
"执行路线图需要把证据缺口转化为行动清单",
],
"gmp_gap_assessment": [
"监管基线决定整改范围而非企业主观偏好",
"现状差距需要按法规条款和业务流程双重定位",
"质量风险分级决定 CAPA 优先级",
"根因分析质量决定整改能否闭环",
"CAPA 设计必须绑定责任人、证据和期限",
"验证计划决定整改是否可被审计接受",
"供应商和外包管理常是系统性缺口放大器",
"数据完整性风险需要独立成章处理",
"实施路线图需要平衡停线风险与合规风险",
"管理层治理机制决定整改能否持续",
],
"cmc_process_risk": [
"工艺流程图是识别放大风险的起点",
"CQA 与 CPP 的映射决定控制策略质量",
"放大过程的失效模式集中在传质、混合和稳定性",
"分析方法和放行标准决定证据可信度",
"技术转移风险来自知识隐性化和现场差异",
"供应链约束会改变工艺控制边界",
"偏差和变更管理决定商业化后的韧性",
"监管沟通策略需要提前固化关键假设",
"反方证据限定平台工艺可复制性",
"CMC 路线图需要把风险转化为验证实验",
],
"rd_go_no_go": [
"科学假设强度决定项目是否值得进入下一阶段",
"POC 证据需要同时证明有效性和可转化性",
"安全性窗口决定适应症与人群选择",
"IP 与 FTO 风险决定商业化自由度",
"开发路径需要把关键不确定性前置验证",
"竞争窗口决定速度是否仍有战略价值",
"CMC 与临床运营能力影响真实可行性",
"反方证据决定 go/no-go 阈值",
"投资强度应与证据成熟度匹配",
"决策门槛需要形成可执行检查表",
],
"management_consulting": [
"现状诊断需要区分症状、根因和约束条件",
"能力差距决定组织改进优先级",
"流程断点揭示跨部门协作成本",
"治理结构决定决策速度和责任清晰度",
"运营模型需要匹配战略目标而非照搬标杆",
"数字化工具只有嵌入流程才产生价值",
"绩效指标需要避免局部最优",
"变革阻力本身是方案设计输入",
"路线图需要把 quick wins 与系统建设分层",
"落地机制决定咨询建议能否转化为成果",
],
"gmp_quality_operations_diagnosis": [
"从审计清单到商业化阶段门",
"用法规基线重新校准整改优先级",
"制剂无菌保障:从硬件合规到行为受控",
"原液与公用系统:封闭工艺背后的证据缺口",
"工艺文件与验证:商业化转移的硬门槛",
"质量系统闭环:偏差、变更、CAPA 与数据完整性",
"人员能力:培训有效性比培训记录更关键",
"运营节奏:从临时协调转向管理系统",
"团队建设:按 CDMO 能力矩阵补齐角色",
"整改路线图:立即纠偏、体系补强、能力建设",
"管理层看板:用 CAPA 总表驱动复核",
],
}
def _existing_chapter_titles(project_root: Path) -> list[str]:
framework_path = project_root / "phase1" / "framework.md"
if not framework_path.exists():
return []
from scripts.runtime.tasks import parse_framework_chapters
chapters = parse_framework_chapters(framework_path.read_text(encoding="utf-8"))
return [chapter.title for chapter in chapters if chapter.title]
def render_framework(
project_root: Path,
*,
method_key: str | None = None,
chapter_count: int = 10,
preserve_existing_outline: bool = False,
) -> Path:
manifest = load_manifest(project_root)
registry = ResearchMethodRegistry()
method = registry.get(method_key or manifest.get("research_method"))
if method_key:
manifest["research_method"] = method.key
existing_titles = _existing_chapter_titles(project_root) if preserve_existing_outline else []
titles = existing_titles or CHAPTER_TEMPLATES.get(method.key) or CHAPTER_TEMPLATES["mckinsey_market"]
chapter_count = max(8, min(15, chapter_count))
selected = titles[:chapter_count] if not existing_titles else titles
quota = max(800, int(manifest.get("target_words", 30000)) // len(selected))
chapter_planning = build_chapter_planning(project_root, manifest, method, selected, quota=quota)
sections = "\n".join(f"- {item}" for item in method.framework_sections)
axes = "".join(method.task_axes)
material_text = render_material_inventory(manifest.get("material_inventory") or [])
lines = [
f"# {manifest.get('report_title') or manifest['topic']}:研究框架",
"",
f"research_method: {method.key}",
f"method_name: {method.name}",
f"work_language: 中文主写作;检索关键词、证据摘录、source title、raw notes 可保留英文。",
f"target_words: {manifest.get('target_words', 30000)}",
"",
"## 方法选择",
"",
f"本项目采用 `{method.key}`,因为其结构原则是:{method.structure_principle}",
"",
"框架模块:",
sections,
"",
"Phase 2 任务轴:",
f"- {axes}",
"",
"## 输入材料与使用边界",
"",
material_text,
"",
"这些材料作为现场问题线索和内部事实起点使用;正式结论仍需结合 NMPA、FDA、EMA、ICH、WHO 等权威法规、指南和最佳实践进行验证。",
"",
"## 中心假设",
"",
_central_thesis(manifest, method),
"",
"Phase1 的职责是大胆假设:基于材料、访谈和初步搜索定下主基调、章节命题和求证路线。Phase2 的职责是小心求证:验证、证伪、补证,而不是重新发明报告方向。Phase3 则检查 Phase1 假设与 Phase2 证据是否自洽。",
"",
]
for item in chapter_planning:
lines.extend(
[
f"## 第{int(item['chapter_id'][2:])}{item['title']}",
"",
f"建议字数:约 {item['suggested_words']} 字。",
f"本章要解决的问题:{item['core_question']}",
f"大胆假设:{item['bold_hypothesis']}",
f"写作主张:{item['writing_claim']}",
f"证据线:{''.join(item['evidence_lanes'])}",
"",
"材料起点:",
*[f"- {line}" for line in item["material_starting_points"]],
"",
"求证计划:",
*[f"- {line}" for line in item["verification_plan"]],
"",
"必须寻找的反方/边界:",
*[f"- {line}" for line in item["counter_evidence_to_seek"]],
"",
f"最低证据要求:`{json.dumps(item['minimum_evidence'], ensure_ascii=False)}`",
"",
]
)
lines.extend(
[
"## 暂停点",
"",
"请先确认 `phase1/material_brief.md` 的材料解读和访谈问题,再确认本框架后进入 Phase 2。若章节逻辑、方法框架或字数配额需要调整,应先修改本文件。",
"",
"确认后运行:`uv run python scripts/dr.py approve <project>`;未批准时 `research` 默认会拒绝推进,可用 `--force` 临时覆盖。",
"",
]
)
out = project_root / "phase1" / "framework.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("\n".join(lines), encoding="utf-8")
research_brief_md, research_brief_json = write_research_brief(project_root, manifest, method, chapter_planning=chapter_planning)
manifest["phase1"] = {
"status": "completed",
"approved": False,
"framework_path": "phase1/framework.md",
"research_brief_path": str(research_brief_md.relative_to(project_root)),
"research_brief_json_path": str(research_brief_json.relative_to(project_root)),
"requires_user_interview": True,
"research_method": method.key,
"updated_at": utc_now_iso(),
}
manifest["updated_at"] = utc_now_iso()
write_manifest(project_root, manifest)
return out
+328
View File
@@ -0,0 +1,328 @@
"""Phase 3 review checks for the Python core."""
from __future__ import annotations
import json
import re
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from scripts.runtime.artifacts import load_manifest, write_manifest
from scripts.runtime.tasks import validate_packet
def utc_now_iso() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
def _source_ids_from_jsonl(path: Path) -> set[str]:
ids: set[str] = set()
if not path.exists():
return ids
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
obj = json.loads(line)
except json.JSONDecodeError:
continue
source_id = obj.get("id") or obj.get("source_id")
if source_id:
ids.add(str(source_id))
return ids
def _draft_citations(drafts: list[Path]) -> set[str]:
cited: set[str] = set()
for draft in drafts:
cited.update(re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", draft.read_text(encoding="utf-8")))
return cited
def _ready_packet_stems(project_root: Path) -> set[str]:
ready: set[str] = set()
for path in sorted((project_root / "phase2" / "packets").glob("*.json")):
try:
packet = json.loads(path.read_text(encoding="utf-8"))
validate_packet(packet)
except Exception:
continue
ready.add(path.stem)
return ready
def _read_text_if_exists(path: Path, *, max_chars: int | None = None) -> str:
if not path.exists():
return ""
text = path.read_text(encoding="utf-8", errors="ignore")
return text[:max_chars] if max_chars is not None else text
def _json_if_exists(path: Path, *, max_chars: int | None = None) -> str:
if not path.exists():
return ""
try:
data = json.loads(path.read_text(encoding="utf-8"))
text = json.dumps(data, ensure_ascii=False, indent=2)
except Exception:
text = path.read_text(encoding="utf-8", errors="ignore")
return text[:max_chars] if max_chars is not None else text
def _draft_quality_findings(drafts: list[Path]) -> list[dict[str, Any]]:
findings: list[dict[str, Any]] = []
generic_markers = [
"需要进一步完善",
"应当加强",
"持续改进",
"系统性",
"闭环管理",
"质量文化",
]
for draft in drafts:
text = draft.read_text(encoding="utf-8")
zh_chars = sum(1 for char in text if "\u4e00" <= char <= "\u9fff")
citations = re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", text)
evidence_table_present = "证据落点" in text and "待补证据" in text
if zh_chars >= 1200 and len(set(citations)) < 5:
findings.append({"severity": "P1", "message": f"{draft.name} 引用来源过少,可能未充分使用 evidence packet。"})
if zh_chars >= 1200 and not evidence_table_present:
findings.append({"severity": "P1", "message": f"{draft.name} 缺少“证据落点与待补证据”小节,难以判断 evidence 是否真正落到纸面。"})
generic_count = sum(text.count(marker) for marker in generic_markers)
if zh_chars >= 1200 and generic_count >= 18:
findings.append({"severity": "P1", "message": f"{draft.name} 泛化管理表述过多,需要回炉为具体审计发现、风险影响和整改动作。"})
return findings
def build_phase3_model_review_context(project_root: Path, *, max_chars: int = 650_000) -> str:
"""Build a structured, bounded context packet for an independent model review."""
deterministic_path = build_phase3_critique(project_root)
deterministic_copy = project_root / "phase3" / "critique_deterministic.md"
deterministic_copy.write_text(deterministic_path.read_text(encoding="utf-8"), encoding="utf-8")
manifest = load_manifest(project_root)
parts: list[str] = [
f"# Phase 3 Model Review Context: {manifest.get('topic', project_root.name)}",
"",
"## Review Contract",
"",
"- 这是给非 Codex 模型的独立总编审校上下文,不要求重写正文。",
"- 请判断 Phase2 草稿能否进入 Phase4,或必须回炉补证据/重写。",
"- 重点关注:证据是否落纸面、并发 packet 是否造成碎片化、法规/最佳实践覆盖是否足够、整改建议是否具体可执行。",
"",
"## Manifest",
"",
"```json",
json.dumps(manifest, ensure_ascii=False, indent=2),
"```",
"",
"## Deterministic Review Baseline",
"",
_read_text_if_exists(deterministic_copy),
"",
"## Phase 1 Framework",
"",
_read_text_if_exists(project_root / "phase1" / "framework.md", max_chars=50_000),
"",
"## Phase 1 Research Brief",
"",
_read_text_if_exists(project_root / "phase1" / "research_brief.md", max_chars=30_000),
"",
"## Phase 2 Brief Warnings",
"",
_json_if_exists(project_root / "phase2" / "brief_warnings.json", max_chars=30_000) or "",
"",
"## Phase 2 Packet Errors",
"",
]
errors = sorted((project_root / "phase2" / "packet_errors").glob("*.json"))
if errors:
for path in errors[:40]:
parts.extend([f"### {path.name}", "", _json_if_exists(path, max_chars=2_000), ""])
else:
parts.append("")
parts.extend(["", "## Source Registry Summary", ""])
source_lines = []
sources_path = project_root / "phase2" / "sources.jsonl"
if sources_path.exists():
for line in sources_path.read_text(encoding="utf-8").splitlines()[:260]:
if not line.strip():
continue
try:
source = json.loads(line)
except json.JSONDecodeError:
continue
source_lines.append(
"- {id} | {tier} | {title} | {url} | cached={cached}".format(
id=source.get("id", ""),
tier=source.get("tier", ""),
title=str(source.get("title", ""))[:120],
url=source.get("url", ""),
cached=source.get("cached_text_path", ""),
)
)
parts.append("\n".join(source_lines) or "")
parts.extend(["", "## Compressed Findings", ""])
for path in sorted((project_root / "phase2" / "compressed_findings").glob("ch*.json")):
parts.extend([f"### {path.name}", "", "```json", _json_if_exists(path, max_chars=35_000), "```", ""])
parts.extend(["", "## Chapter Drafts", ""])
for path in sorted((project_root / "phase2" / "drafts").glob("ch*.md")):
parts.extend([f"### {path.name}", "", _read_text_if_exists(path, max_chars=55_000), ""])
context = "\n".join(parts)
if len(context) > max_chars:
context = context[:max_chars] + "\n\n[Context truncated by max_chars; review should flag if truncation limits confidence.]\n"
out = project_root / "phase3" / "review_context_opus_4_7.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(context, encoding="utf-8")
return context
def phase3_model_review_system_prompt() -> str:
return (
"你是 Deep Research Phase 3 的独立总编审校模型,本次由 ZenMux Claude Opus 4.7 执行,用于避免 Codex/OpenAI 模型偏见。\n"
"你的任务是审校,不是润色或重写。必须用中文输出,英文仅可保留 source title、URL、法规缩写和原文短摘录。\n"
"请严格检查:1) 研究目标与 Phase1 框架是否契合;2) Phase2 并发 evidence packets 是否被章节真正吸收,还是造成碎片化;"
"3) FDA/NMPA/EMA/ICH/WHO/EU GMP 等权威来源是否足以支撑关键判断;4) 用户材料是否被正确作为起点且被权威来源交叉验证;"
"5) 运营管理与团队能力章节是否具体,不得泛泛咨询腔;6) CAPA 建议是否包含 owner、期限、关闭证据、QA verification、复核窗口和升级阈值;"
"7) 引用链和 source_id 是否可追踪;8) 是否仍有明显 AI 味、中英文混杂或空泛表达。\n\n"
"输出必须使用以下 Markdown 结构:\n"
"# Phase 3 Opus 4.7 独立审校\n"
"## 总体判定\n"
"给出:通过 / 有条件通过 / 回炉 Phase2,并说明最核心理由。\n"
"## P0/P1 阻断问题\n"
"列出必须修复的问题;每条写明章节/文件、问题、为什么阻断、建议动作。\n"
"## 章节级审校表\n"
"用表格覆盖 ch01-ch11:主线质量、证据密度、法规覆盖、整改可执行性、是否需要回炉。\n"
"## 证据与信源质量\n"
"单独评价 FDA warning letters、ICH Q9/Q10、EU GMP Annex 1、本地缓存信源、第三方低质信源的使用情况。\n"
"## 碎片化与叙事连贯性\n"
"判断并发研究是否造成割裂,并给出具体整合建议。\n"
"## Phase2 回炉任务清单\n"
"如果需要回炉,列出可执行任务卡级别的补证据/重写要求。\n"
"## Phase4 准入条件\n"
"明确进入 final 前必须满足的条件。\n"
)
def build_phase3_model_critique(
project_root: Path,
*,
client: Any,
model: str = "zenmux-anthropic/claude-opus-4-7",
max_context_chars: int = 650_000,
) -> Path:
context = build_phase3_model_review_context(project_root, max_chars=max_context_chars)
content = client.chat_complete(
model=model,
system=phase3_model_review_system_prompt(),
user=context,
temperature=0.2,
max_tokens=20_000,
tag="phase3:opus-review",
)
out = project_root / "phase3" / "critique.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(content.rstrip() + "\n", encoding="utf-8")
manifest = load_manifest(project_root)
phase3 = manifest.setdefault("phase3", {})
phase3.update(
{
"status": "completed",
"review_mode": "model",
"review_model": model,
"critique_path": "phase3/critique.md",
"context_path": "phase3/review_context_opus_4_7.md",
"deterministic_critique_path": "phase3/critique_deterministic.md",
"updated_at": utc_now_iso(),
}
)
manifest["updated_at"] = utc_now_iso()
write_manifest(project_root, manifest)
return out
def build_phase3_critique(project_root: Path) -> Path:
manifest = load_manifest(project_root)
drafts = sorted((project_root / "phase2" / "drafts").glob("ch*.md"))
packets = sorted((project_root / "phase2" / "packets").glob("*.json"))
ready_stems = _ready_packet_stems(project_root)
packet_errors = [
path for path in sorted((project_root / "phase2" / "packet_errors").glob("*.json"))
if path.stem not in ready_stems
]
chapter_errors = sorted((project_root / "phase2" / "chapter_errors").glob("*.json"))
sources = _source_ids_from_jsonl(project_root / "phase2" / "sources.jsonl")
cited = _draft_citations(drafts)
missing_sources = sorted(cited - sources) if sources else sorted(cited)
uncited_sources = sorted(sources - cited) if cited else sorted(sources)
findings: list[dict[str, Any]] = []
if not drafts:
findings.append({"severity": "P1", "message": "Phase 2 drafts 缺失,尚不能进入 Phase 4 成稿。"})
if packet_errors:
findings.append({"severity": "P1", "message": f"存在 {len(packet_errors)} 个 packet 失败,需要回炉补证据。"})
if chapter_errors:
findings.append({"severity": "P1", "message": f"存在 {len(chapter_errors)} 个章节组装失败,需要修复引用或重写该章。"})
quality_holds = manifest.get("quality_holds") or []
if quality_holds:
findings.append({"severity": "P1", "message": "存在质量暂停标记:" + ", ".join(quality_holds)})
findings.extend(_draft_quality_findings(drafts))
if missing_sources:
findings.append({"severity": "P1", "message": f"正文引用未在 sources.jsonl 中登记:{', '.join(missing_sources)}"})
if not findings:
findings.append({"severity": "P2", "message": "基础产物完整;仍需人工或大上下文模型审校逻辑链、反方证据和章节叙事。"})
lines = [
"# Phase 3 审校 critique",
"",
f"- 项目:{manifest.get('topic', project_root.name)}",
f"- 运行时:python-core-v0.20",
f"- drafts{len(drafts)}",
f"- packets{len(packets)}",
f"- sources{len(sources)}",
f"- cited_source_ids{', '.join(sorted(cited)) if cited else ''}",
"",
"## Findings",
"",
]
for item in findings:
lines.append(f"- [{item['severity']}] {item['message']}")
lines.extend(
[
"",
"## Residual Risks",
"",
"- 本 deterministic review 只做结构、引用和错误包检查;深层逻辑审校仍建议交给 `phase3_review` 角色执行。",
"- 若 sources 为空,本审校会把所有正文引用视为待登记来源。",
"",
"## Next",
"",
"- 若存在 P1,先回到 Phase 2 修复 packet/chapter 错误。",
"- 若仅有 P2,可进入 `dr.py finalize` 的中文原生成稿路径。",
"",
]
)
out = project_root / "phase3" / "critique.md"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text("\n".join(lines), encoding="utf-8")
phase3 = manifest.setdefault("phase3", {})
phase3.update(
{
"status": "completed",
"critique_path": "phase3/critique.md",
"findings_total": len(findings),
"missing_sources": missing_sources,
"uncited_sources": uncited_sources,
"updated_at": utc_now_iso(),
}
)
manifest["updated_at"] = utc_now_iso()
write_manifest(project_root, manifest)
return out
+149
View File
@@ -0,0 +1,149 @@
"""Runtime role and task-model resolution."""
from __future__ import annotations
from dataclasses import dataclass
from scripts.lib.model_config import resolve_model_profile
ROLE_DEFAULTS = {
"dr_plan": {
"skills": ["document-ingest", "search-gateway", "search-strategy", "source-quality", "length-budget", "mckinsey-method"],
"temperature": 0.4,
"max_tokens": 12000,
"max_concurrency": 1,
},
"dr_pm": {
"skills": ["length-budget", "evidence-table", "mckinsey-method"],
"temperature": 0.2,
"max_tokens": 8000,
"max_concurrency": 1,
},
"dr_searcher": {
"skills": ["search-gateway", "search-strategy", "source-quality"],
"temperature": 0.1,
"max_tokens": 6000,
"max_concurrency": 6,
},
"dr_analyst": {
"skills": ["search-gateway", "search-strategy", "source-quality", "evidence-table", "mckinsey-method"],
"temperature": 0.3,
"max_tokens": 14000,
"max_concurrency": 6,
},
"dr_verifier": {
"skills": ["search-gateway", "search-strategy", "source-quality", "evidence-table"],
"temperature": 0.2,
"max_tokens": 10000,
"max_concurrency": 4,
},
"dr_chief_editor": {
"skills": ["mckinsey-method", "evidence-table", "output-hygiene"],
"temperature": 0.2,
"max_tokens": 16000,
"max_concurrency": 1,
},
"dr_editor_in_chief": {
"skills": ["mckinsey-method", "citation-manager", "humanizer-cn", "output-hygiene"],
"temperature": 0.4,
"max_tokens": 20000,
"max_concurrency": 1,
},
"dr_reporter": {
"skills": ["pdf-reportlab", "citation-manager", "output-hygiene"],
"temperature": 0.1,
"max_tokens": 6000,
"max_concurrency": 1,
},
}
ROLE_IDENTITIES = {
"dr_plan": (
"你是 Deep Research 的 Phase1 研究架构师。你的工作不是列目录,而是先消化材料、访谈和初步搜索,"
"形成可被证伪的主判断、章节命题和求证路线。你要大胆假设,但必须给 Phase2 留下清晰的验证和推翻条件。"
),
"dr_pm": (
"你是 Deep Research 的研究项目经理。你的职责是把研究意图转化为可并发执行、可回收校验的任务,"
"控制碎片化、重复检索和上下文污染。"
),
"dr_searcher": (
"你是 Deep Research 的信源发现员。你的职责是用短英文关键词和轴向词找到高质量入口,"
"优先官方、法规、学术和一手材料;你不写结论,只交付可追溯来源。"
),
"dr_analyst": (
"你是 Deep Research 的章节证据分析师。你的职责不是写一篇像样的空泛文章,而是围绕 Phase1 命题"
"小心求证:提取材料原文、检索权威证据、寻找反方边界,并把证据整理成可审计的结构化 packet。"
),
"dr_verifier": (
"你是 Deep Research 的独立反方审校员。你的默认姿态是质疑:找证据缺口、适用边界、反例和过度推断,"
"并指出哪些结论必须降级或回炉。"
),
"dr_chief_editor": (
"你是 Deep Research 的 Phase3 总编审校。你的职责是通读 Phase1 假设与 Phase2 证据,判断二者是否自洽,"
"优先指出结构性失败、证据不足和需要回炉的章节。"
),
"dr_editor_in_chief": (
"你是 Deep Research 的终稿主编。你的职责是把已验证证据组织成客户可读的中文报告,"
"保持观点清晰、证据密实、表达克制,避免翻译腔和 AI 味。"
),
"dr_reporter": (
"你是 Deep Research 的报告制作负责人。你的职责是把已定稿内容可靠渲染为 PDF/DOCX,"
"确保引用、排版、中文字体、表格和输出卫生可交付。"
),
}
@dataclass(frozen=True)
class RoleDefinition:
name: str
model: str
skills: list[str]
temperature: float
max_tokens: int
max_concurrency: int
identity: str = ""
class RuntimeProfile:
def __init__(self, *, profile: str, roles: dict[str, RoleDefinition], task_types: dict[str, str]) -> None:
self.profile = profile
self.roles = roles
self.task_types = task_types
def role_for_task(self, task_type: str) -> RoleDefinition:
role_name = self.task_types.get(task_type)
if not role_name:
raise KeyError(f"unknown task_type: {task_type}")
if role_name not in self.roles:
raise KeyError(f"task_type {task_type} maps to missing role {role_name}")
return self.roles[role_name]
def resolve_runtime_profile(
*,
profile: str | None = None,
overrides: dict[str, str] | None = None,
) -> RuntimeProfile:
resolved = resolve_model_profile(profile=profile, overrides=overrides)
role_models = resolved["roles"]
roles: dict[str, RoleDefinition] = {}
for name, defaults in ROLE_DEFAULTS.items():
model = role_models.get(name)
if not model:
continue
roles[name] = RoleDefinition(
name=name,
model=model,
skills=list(defaults["skills"]),
temperature=float(defaults["temperature"]),
max_tokens=int(defaults["max_tokens"]),
max_concurrency=int(defaults["max_concurrency"]),
identity=ROLE_IDENTITIES.get(name, ""),
)
return RuntimeProfile(
profile=resolved["profile"],
roles=roles,
task_types=dict(resolved.get("task_types") or {}),
)
+114
View File
@@ -0,0 +1,114 @@
"""Canonical skill registry and adapter sync helpers."""
from __future__ import annotations
import shutil
from dataclasses import dataclass
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
CANONICAL_SKILLS_DIR = REPO_ROOT / "platform_adapters" / "antigravity" / "agent" / "skills"
LEGACY_AGENT_SKILLS_DIR = REPO_ROOT / ".agent" / "skills"
LEGACY_AGENTS_SKILLS_DIR = REPO_ROOT / ".agents" / "skills"
PROJECT_SKILLS_DIR = REPO_ROOT / "skills"
REQUIRED_SKILLS = {
"search-strategy",
"search-gateway",
"source-quality",
"length-budget",
"evidence-table",
"citation-manager",
"mckinsey-method",
}
@dataclass(frozen=True)
class SkillInfo:
name: str
path: Path
class SkillRegistry:
"""Reads skills from the canonical cross-adapter registry."""
def __init__(self, canonical_dir: Path | None = None) -> None:
self.canonical_dir = canonical_dir or CANONICAL_SKILLS_DIR
def roots(self) -> list[Path]:
roots = []
if self.canonical_dir == CANONICAL_SKILLS_DIR and PROJECT_SKILLS_DIR.exists():
roots.append(PROJECT_SKILLS_DIR)
roots.append(self.canonical_dir)
if self.canonical_dir == CANONICAL_SKILLS_DIR and LEGACY_AGENT_SKILLS_DIR.exists():
roots.append(LEGACY_AGENT_SKILLS_DIR)
if self.canonical_dir == CANONICAL_SKILLS_DIR and LEGACY_AGENTS_SKILLS_DIR.exists():
roots.append(LEGACY_AGENTS_SKILLS_DIR)
return roots
def list(self) -> list[SkillInfo]:
seen: set[str] = set()
out: list[SkillInfo] = []
for root in self.roots():
if not root.exists():
continue
for path in sorted(root.glob("*/SKILL.md")):
name = path.parent.name
if name in seen:
continue
seen.add(name)
out.append(SkillInfo(name=name, path=path))
return out
def list_names(self) -> list[str]:
return [item.name for item in self.list()]
def read(self, name: str) -> str:
for root in self.roots():
path = root / name / "SKILL.md"
if path.exists():
return path.read_text(encoding="utf-8")
raise FileNotFoundError(f"skill not found: {name}")
def validate(self, required: set[str] | None = None) -> dict[str, object]:
names = set(self.list_names())
required_names = required or REQUIRED_SKILLS
missing = sorted(required_names - names)
malformed: list[str] = []
for item in self.list():
text = item.path.read_text(encoding="utf-8")
if "name:" not in text[:300]:
malformed.append(item.name)
return {
"ok": not missing and not malformed,
"canonical_dir": str(self.canonical_dir),
"count": len(names),
"missing": missing,
"malformed": malformed,
}
def sync_to(self, targets: list[Path], *, force: bool = True) -> int:
"""Copy canonical skills into adapter skill directories.
Returns the number of skill directories copied across all targets.
"""
copied = 0
for target in targets:
if target.resolve() == self.canonical_dir.resolve():
continue
target.mkdir(parents=True, exist_ok=True)
for item in self.list():
dst = target / item.name
if dst.exists() and force:
shutil.rmtree(dst)
if not dst.exists():
shutil.copytree(item.path.parent, dst)
copied += 1
return copied
def default_adapter_skill_dirs() -> list[Path]:
return [
REPO_ROOT / ".opencode" / "skills",
REPO_ROOT / "platform_adapters" / "antigravity" / "agent" / "skills",
]
+230
View File
@@ -0,0 +1,230 @@
"""Cache important external sources as local Markdown snapshots."""
from __future__ import annotations
import hashlib
import json
import re
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urlparse
import httpx
from lxml import html
IMPORTANT_DOMAINS = (
"fda.gov",
"ema.europa.eu",
"nmpa.gov.cn",
"cde.org.cn",
"ich.org",
"who.int",
"edqm.eu",
"pmda.go.jp",
"ec.europa.eu",
"health.ec.europa.eu",
)
@dataclass(frozen=True)
class CacheResult:
source_id: str
url: str
cached_text_path: str
raw_path: str
status: str
chars: int
def _safe_stem(source: dict) -> str:
source_id = str(source.get("id") or "source")
digest = hashlib.sha1(str(source.get("url") or source_id).encode("utf-8")).hexdigest()[:10]
safe_id = re.sub(r"[^A-Za-z0-9_-]+", "_", source_id).strip("_") or "source"
return f"{safe_id}-{digest}"
def _domain(url: str) -> str:
return urlparse(url).netloc.lower()
def is_important_source(source: dict) -> bool:
url = str(source.get("url") or "")
if not url.startswith(("http://", "https://")):
return False
domain = _domain(url)
if any(domain.endswith(item) for item in IMPORTANT_DOMAINS):
return True
tier = str(source.get("tier") or "").lower()
if "tier 1" in tier or tier in {"1", "1.0"}:
return True
title = str(source.get("title") or "").lower()
return any(term in title for term in ("ich q9", "ich q10", "annex 1", "fda guidance", "who guideline"))
def load_sources(path: Path) -> list[dict]:
if not path.exists():
return []
rows: list[dict] = []
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
rows.append(json.loads(line))
return rows
def write_sources(path: Path, rows: list[dict]) -> None:
path.write_text("".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), encoding="utf-8")
def _response_ext(url: str, content_type: str) -> str:
lowered = url.lower()
if "pdf" in content_type or lowered.endswith(".pdf"):
return ".pdf"
if "html" in content_type or lowered.endswith((".html", ".htm", "/")):
return ".html"
return ".bin"
def _html_to_text(content: bytes) -> str:
doc = html.fromstring(content)
for bad in doc.xpath("//script|//style|//noscript"):
bad.drop_tree()
return "\n".join(line.strip() for line in doc.text_content().splitlines() if line.strip())
def _pdf_to_text(path: Path) -> str:
try:
import fitz
except Exception:
return ""
doc = fitz.open(path)
parts: list[str] = []
for index, page in enumerate(doc, start=1):
text = page.get_text("text").strip()
if text:
parts.append(f"## Page {index}\n\n{text}")
return "\n\n".join(parts)
def _bytes_to_text(*, raw_path: Path, content: bytes, content_type: str, url: str) -> str:
if raw_path.suffix == ".pdf" or "pdf" in content_type or url.lower().endswith(".pdf"):
return _pdf_to_text(raw_path)
if raw_path.suffix in {".html", ".htm"} or "html" in content_type:
return _html_to_text(content)
try:
return content.decode("utf-8")
except UnicodeDecodeError:
return content.decode("utf-8", errors="ignore")
def cache_source(
project_root: Path,
source: dict,
*,
client: httpx.Client | None = None,
force: bool = False,
timeout: float = 45.0,
) -> CacheResult:
url = str(source.get("url") or "")
if not url.startswith(("http://", "https://")):
raise ValueError(f"source URL is not remote: {url}")
cache_dir = project_root / "phase2" / "source_cache"
raw_dir = cache_dir / "raw"
text_dir = cache_dir / "md"
raw_dir.mkdir(parents=True, exist_ok=True)
text_dir.mkdir(parents=True, exist_ok=True)
stem = _safe_stem(source)
md_path = text_dir / f"{stem}.md"
if md_path.exists() and not force:
return CacheResult(
source_id=str(source.get("id") or ""),
url=url,
cached_text_path=str(md_path.relative_to(project_root)),
raw_path=str(source.get("cached_raw_path") or ""),
status="cached",
chars=len(md_path.read_text(encoding="utf-8")),
)
owns_client = client is None
http = client or httpx.Client(trust_env=False, follow_redirects=True, timeout=timeout)
try:
response = http.get(url)
response.raise_for_status()
content_type = response.headers.get("content-type", "").lower()
ext = _response_ext(str(response.url), content_type)
raw_path = raw_dir / f"{stem}{ext}"
raw_path.write_bytes(response.content)
text = _bytes_to_text(raw_path=raw_path, content=response.content, content_type=content_type, url=str(response.url))
lines = [
f"# Source Snapshot: {source.get('title') or source.get('id') or url}",
"",
f"- source_id: {source.get('id', '')}",
f"- original_url: {url}",
f"- fetched_url: {response.url}",
f"- content_type: {content_type}",
f"- raw_path: {raw_path.relative_to(project_root)}",
"",
"## Extracted Text",
"",
text.strip() or "[No extractable text. Keep raw file for manual review.]",
"",
]
md_path.write_text("\n".join(lines), encoding="utf-8")
return CacheResult(
source_id=str(source.get("id") or ""),
url=url,
cached_text_path=str(md_path.relative_to(project_root)),
raw_path=str(raw_path.relative_to(project_root)),
status="fetched",
chars=len(text),
)
finally:
if owns_client:
http.close()
def cache_sources(
project_root: Path,
*,
sources_rel: str = "phase2/sources.jsonl",
important_only: bool = True,
limit: int | None = None,
force: bool = False,
) -> list[CacheResult]:
sources_path = project_root / sources_rel
rows = load_sources(sources_path)
results: list[CacheResult] = []
selected_indexes = [
index
for index, row in enumerate(rows)
if row.get("url")
and (not row.get("cached_text_path") or force)
and (not important_only or is_important_source(row))
]
if limit is not None:
selected_indexes = selected_indexes[:limit]
with httpx.Client(trust_env=False, follow_redirects=True, timeout=45.0) as client:
for index in selected_indexes:
row = rows[index]
try:
result = cache_source(project_root, row, client=client, force=force)
except Exception as exc:
row["cache_status"] = "failed"
row["cache_error"] = str(exc)[:300]
continue
row["cached_text_path"] = result.cached_text_path
row["cached_raw_path"] = result.raw_path
row["cache_status"] = result.status
row["cached_text_chars"] = result.chars
results.append(result)
write_sources(sources_path, rows)
manifest = project_root / "phase2" / "source_cache" / "manifest.json"
manifest.parent.mkdir(parents=True, exist_ok=True)
manifest.write_text(
json.dumps([result.__dict__ for result in results], ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
return results
+83
View File
@@ -0,0 +1,83 @@
"""Source registry helpers for Phase 2 packets."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
def _source_key(source: dict[str, Any]) -> str:
return (source.get("id") or source.get("source_id") or source.get("doi") or source.get("url") or "").strip()
def append_packet_sources(sources_path: Path, packet: dict[str, Any]) -> int:
"""Append packet sources to sources.jsonl, preserving every citeable source_id."""
sources_path.parent.mkdir(parents=True, exist_ok=True)
existing: set[str] = set()
if sources_path.exists():
for line in sources_path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
existing.add(_source_key(json.loads(line)))
except json.JSONDecodeError:
continue
written = 0
with sources_path.open("a", encoding="utf-8") as f:
for source in packet.get("sources") or []:
key = _source_key(source)
if not key or key in existing:
continue
existing.add(key)
f.write(json.dumps(source, ensure_ascii=False) + "\n")
written += 1
return written
def rebuild_sources_from_packets(project_root: Path) -> int:
"""Rebuild phase2/sources.jsonl from packet-level source metadata.
The registry is keyed by source_id, not URL. Two packet sources may point to
the same URL but have different source_ids already cited in drafts; dropping
either row would break citation traceability.
"""
packets_dir = project_root / "phase2" / "packets"
sources_path = project_root / "phase2" / "sources.jsonl"
sources_path.parent.mkdir(parents=True, exist_ok=True)
existing_by_key: dict[str, dict[str, Any]] = {}
if sources_path.exists():
for line in sources_path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
row = json.loads(line)
except json.JSONDecodeError:
continue
key = _source_key(row)
if key:
existing_by_key[key] = row
seen: set[str] = set()
rows: list[dict[str, Any]] = []
for packet_path in sorted(packets_dir.glob("*.json")):
try:
packet = json.loads(packet_path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
continue
for source in packet.get("sources") or []:
if not isinstance(source, dict):
continue
key = _source_key(source)
if not key or key in seen:
continue
seen.add(key)
previous = existing_by_key.get(key, {})
rows.append({**source, **{k: v for k, v in previous.items() if k.startswith("cache") or k.startswith("cached_")}})
sources_path.write_text(
"".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows),
encoding="utf-8",
)
return len(rows)
+472
View File
@@ -0,0 +1,472 @@
"""Task-card and evidence-packet primitives for v0.20 Phase 2."""
from __future__ import annotations
import json
import re
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any
from scripts.runtime.methods import ResearchMethod
VALID_ROUTES = {"general", "evidence", "scholar", "patents", "news", "fda"}
DEFAULT_AXES = ["literature", "regulatory", "patents", "market", "counter"]
AXIS_ROUTES = {
"literature": ["scholar", "evidence", "general"],
"clinical": ["scholar", "evidence", "general"],
"regulatory": ["fda", "evidence", "general", "news"],
"patents": ["patents", "evidence", "general"],
"market": ["news", "general"],
"china": ["news", "general"],
"counter": ["fda", "scholar", "evidence", "general"],
"regulatory_gap": ["fda", "evidence", "general", "news"],
"risk_classification": ["evidence", "general", "scholar"],
"capa_design": ["evidence", "general", "news"],
"ownership_timeline": ["general"],
"verification_evidence": ["fda", "evidence", "general", "scholar"],
"process_flow": ["scholar", "evidence", "general"],
"cqa_cpp": ["scholar", "evidence", "general"],
"scale_up_risk": ["scholar", "evidence", "general"],
"control_strategy": ["scholar", "evidence", "general"],
"supply_chain": ["news", "general"],
"scientific_rationale": ["scholar", "evidence", "general"],
"poc_evidence": ["scholar", "evidence", "general"],
"ip_fto": ["patents", "evidence", "general"],
"development_path": ["scholar", "evidence", "general"],
"commercial_window": ["news", "general"],
"current_state": ["evidence", "general"],
"capability_gap": ["evidence", "general"],
"operating_model": ["evidence", "general"],
"governance": ["evidence", "general"],
"implementation_roadmap": ["evidence", "general"],
"nmpa_fda_ema_ich_who_baseline": ["fda", "evidence", "general", "news"],
"quality_system_gap": ["fda", "evidence", "general"],
"manufacturing_process_risk": ["fda", "scholar", "evidence", "general"],
"operations_management_gap": ["fda", "evidence", "general"],
"team_capability": ["evidence", "general", "news"],
"capa_roadmap": ["fda", "evidence", "general"],
"input_material_findings": ["evidence", "general"],
"fda_enforcement_precedents": ["fda"],
"chapter_integrated": ["fda", "scholar", "evidence", "general"],
}
@dataclass
class Chapter:
chapter_id: str
index: int
title: str
notes: str = ""
@dataclass
class TaskCard:
task_id: str
chapter_ids: list[str]
topic_axis: str
questions: list[str]
search_routes: list[str]
output_packet: str
chapter_title: str = ""
preferred_model_role: str = "dr_analyst"
status: str = "pending"
dependencies: list[str] = field(default_factory=list)
research_goal: str = ""
research_method: str = ""
prompt_brief: str = ""
required_skills: list[str] = field(default_factory=list)
allowed_materials: list[str] = field(default_factory=list)
expected_evidence: dict[str, Any] = field(default_factory=dict)
stop_conditions: list[str] = field(default_factory=list)
model_hint: str = ""
def to_dict(self) -> dict[str, Any]:
return asdict(self)
def parse_framework_chapters(framework_text: str) -> list[Chapter]:
"""Extract Chinese or English chapter headings from a framework markdown."""
lines = framework_text.splitlines()
chapters: list[Chapter] = []
current: Chapter | None = None
note_lines: list[str] = []
heading_re = re.compile(
r"^#{1,3}\s*(?:第\s*)?(\d{1,2})\s*(?:章|[.)、:-])?\s*(.+?)\s*$",
re.IGNORECASE,
)
english_re = re.compile(r"^#{1,3}\s*chapter\s+(\d{1,2})[:.)\s-]+(.+?)\s*$", re.IGNORECASE)
for line in lines:
match = heading_re.match(line.strip()) or english_re.match(line.strip())
if match:
if current:
current.notes = "\n".join(note_lines).strip()
chapters.append(current)
index = int(match.group(1))
title = match.group(2).strip(" #")
current = Chapter(chapter_id=f"ch{index:02d}", index=index, title=title)
note_lines = []
elif current:
note_lines.append(line)
if current:
current.notes = "\n".join(note_lines).strip()
chapters.append(current)
return chapters
def _questions_for_axis(chapter: Chapter, axis: str, method: ResearchMethod | None = None) -> list[str]:
if axis == "chapter_integrated":
lanes = "".join(method.integrated_lanes if method else [])
return [
f"围绕《{chapter.title}》形成章节级综合证据包,不再拆成孤立小轴。",
f"必须按当前 research_method 的 evidence lanes 组织证据:{lanes or '本地材料、权威来源、反方证据、可执行建议'}",
"若项目有用户材料,必须先读取本地材料证据并提取原文;再用本方法适用的权威来源交叉验证。",
"必须形成:材料/事实基线、外部权威证据、差距或机会判断、反方/限制条件、可执行建议和待补证据。",
]
questions = [
f"围绕《{chapter.title}》从 {axis} 角度提炼可证伪的核心结论。",
"至少寻找两个 Tier 1-2 来源支撑主要结论;不足时标注待验证。",
"主动检索反方证据、限制条件或失败案例。",
]
if axis in {
"nmpa_fda_ema_ich_who_baseline",
"quality_system_gap",
"manufacturing_process_risk",
"operations_management_gap",
"capa_roadmap",
"verification_evidence",
"counter",
"fda_enforcement_precedents",
}:
questions.append(
"必须检索并优先评估 FDA Warning Letters、inspection/enforcement 页面、会议纪要或 meeting materials,作为 GMP 缺陷严重度和整改优先级的佐证。"
)
return questions
def _default_required_skills(axis: str) -> list[str]:
skills = ["search-gateway", "search-strategy", "source-quality", "evidence-table"]
if axis == "counter":
skills.append("mckinsey-method")
return skills
def _default_expected_evidence(axis: str) -> dict[str, Any]:
expected = {
"min_tier_1_2_sources": 2,
"must_include_counter_evidence": True,
"must_include_source_metadata": True,
"preferred_evidence_types": [
"regulatory_or_best_practice_requirement",
"fda_warning_letter_or_meeting_record",
"site_or_material_finding",
"quantitative_fact_or_record",
"implementation_or_verification_evidence",
],
"axis": axis,
}
if axis == "chapter_integrated":
expected.update(
{
"min_local_material_evidence": 2,
"min_official_sources": 2,
"min_fda_or_regulatory_precedents": 1,
"min_capa_actions": 3,
"preferred_evidence_types": [
"local_audit_or_recap_quote",
"official_regulatory_requirement",
"fda_warning_letter_or_meeting_record",
"gap_analysis",
"capa_action_with_owner_and_verification",
"counter_evidence_or_boundary_condition",
],
}
)
return expected
def _default_stop_conditions() -> list[str]:
return [
"已形成至少 3 条可追溯 evidence_items,且每条关键 claim 有 source_id。",
"已主动记录 counter_evidence 或明确说明未找到反方证据的检索路径。",
"candidate_sources 不足以支撑结论时停止写作,并把缺口写入 open_questions。",
]
def _integrated_prompt_brief(chapter: Chapter, method: ResearchMethod | None) -> str:
lanes = "".join(method.integrated_lanes if method else [])
return (
f"本任务是《{chapter.title}》的章节级综合证据包。不要把多条窄轴 packet 机械拼贴;"
f"必须围绕当前研究方法的 lanes 一次性收束主线:{lanes or '事实材料、权威证据、反方证据、行动建议'}"
"输出必须让章节作者能直接写出判断、证据落点和可执行建议。"
)
def _task_card_for_chapter_axis(
*,
chapter: Chapter,
axis: str,
routes: list[str],
method_key: str,
required_skills: list[str] | None = None,
allowed_materials: list[str] | None = None,
prompt_brief: str | None = None,
questions: list[str] | None = None,
research_goal: str | None = None,
expected_evidence: dict[str, Any] | None = None,
stop_conditions: list[str] | None = None,
method: ResearchMethod | None = None,
) -> TaskCard:
return TaskCard(
task_id=f"{chapter.chapter_id}-{axis}",
chapter_ids=[chapter.chapter_id],
topic_axis=axis,
questions=questions or _questions_for_axis(chapter, axis, method),
search_routes=routes,
output_packet=f"phase2/packets/{chapter.chapter_id}-{axis}.json",
chapter_title=chapter.title,
preferred_model_role="dr_verifier" if axis == "counter" else "dr_analyst",
research_goal=research_goal or f"为《{chapter.title}》收集并验证 {axis} 轴证据,形成可写入章节的具体判断与证据落点。",
research_method=method_key,
prompt_brief=prompt_brief or (_integrated_prompt_brief(chapter, method) if axis == "chapter_integrated" else f"围绕《{chapter.title}》的 {axis} 轴,优先形成可证伪、可引用、可落地的证据包。"),
required_skills=required_skills or _default_required_skills(axis),
allowed_materials=allowed_materials or [],
expected_evidence=expected_evidence or _default_expected_evidence(axis),
stop_conditions=stop_conditions or _default_stop_conditions(),
model_hint="use_cross_model_verifier" if axis == "counter" else "use_cost_effective_research_worker",
)
def generate_task_cards(
slug: str,
framework_text: str,
*,
axes: list[str] | None = None,
method: ResearchMethod | None = None,
) -> list[TaskCard]:
del slug # slug is kept for call-site clarity and future namespacing.
chapters = parse_framework_chapters(framework_text)
selected_axes = axes or (method.task_axes if method else DEFAULT_AXES)
cards: list[TaskCard] = []
for chapter in chapters:
for axis in selected_axes:
routes = AXIS_ROUTES.get(axis, ["general"])
cards.append(
_task_card_for_chapter_axis(
chapter=chapter,
axis=axis,
routes=routes,
method_key=method.key if method else "",
method=method,
)
)
validate_task_cards(cards)
return cards
def generate_task_cards_from_research_brief(
slug: str,
framework_text: str,
research_brief: dict[str, Any],
*,
axes: list[str] | None = None,
method: ResearchMethod | None = None,
) -> list[TaskCard]:
del slug
chapters = parse_framework_chapters(framework_text)
planning = research_brief.get("task_planning") or {}
method_key = research_brief.get("research_method") or (method.key if method else "")
if method is None and method_key:
from scripts.runtime.methods import ResearchMethodRegistry
method = ResearchMethodRegistry().get(method_key)
phase2_mode = planning.get("phase2_mode") or research_brief.get("phase2_mode")
if axes:
selected_axes = axes
elif phase2_mode == "chapter_integrated":
selected_axes = ["chapter_integrated"]
else:
selected_axes = (method.task_axes if method else None) or list(planning.get("search_routes_by_axis") or []) or DEFAULT_AXES
routes_by_axis = planning.get("search_routes_by_axis") or {}
prompt_by_axis = planning.get("axis_prompt_briefs") or {}
base_skills = list(planning.get("required_skills") or [])
stop_conditions = list(planning.get("stop_conditions") or [])
allowed_materials = [
str(item.get("path"))
for item in research_brief.get("materials", [])
if item.get("path")
]
if not allowed_materials:
material_digest = (research_brief.get("phase1_inputs") or {}).get("material_digest")
if material_digest:
allowed_materials.append(str(material_digest))
chapter_plan_by_id = {
str(item.get("chapter_id")): item
for item in research_brief.get("chapter_planning", [])
if item.get("chapter_id")
}
cards: list[TaskCard] = []
for chapter in chapters:
for axis in selected_axes:
routes = list(routes_by_axis.get(axis) or AXIS_ROUTES.get(axis, ["general"]))
skills = base_skills or _default_required_skills(axis)
if "search-gateway" not in skills:
skills = ["search-gateway", *skills]
chapter_plan = chapter_plan_by_id.get(chapter.chapter_id) if axis == "chapter_integrated" else None
prompt_brief = prompt_by_axis.get(axis)
questions = None
research_goal = None
expected_evidence = None
card_stop_conditions = stop_conditions or None
if chapter_plan:
prompt_brief = chapter_plan.get("phase2_prompt_context") or prompt_brief
research_goal = chapter_plan.get("core_question")
questions = [
chapter_plan.get("core_question", ""),
chapter_plan.get("bold_hypothesis", ""),
"按 Phase1 求证计划逐条收集支持证据、反方证据和待补证据。",
"不得绕开 Phase1 主基调另起炉灶;若证据推翻假设,必须明确写出修正建议。",
]
questions.extend(str(item) for item in chapter_plan.get("verification_plan", []))
expected_evidence = _default_expected_evidence(axis)
expected_evidence.update(
{
"phase1_minimum_evidence": chapter_plan.get("minimum_evidence") or {},
"evidence_lanes": chapter_plan.get("evidence_lanes") or [],
"must_address_phase1_hypothesis": True,
}
)
card_stop_conditions = [
*(stop_conditions or _default_stop_conditions()),
"已经逐条回应 Phase1 的大胆假设:支持、修正或推翻,并说明依据。",
"已经把本地材料原文、外部证据、反方边界和行动建议分开记录。",
]
cards.append(
_task_card_for_chapter_axis(
chapter=chapter,
axis=axis,
routes=routes,
method_key=method_key,
required_skills=skills,
allowed_materials=allowed_materials,
prompt_brief=prompt_brief,
questions=questions,
research_goal=research_goal,
expected_evidence=expected_evidence,
stop_conditions=card_stop_conditions,
method=method,
)
)
validate_task_cards(cards)
return cards
def detect_dependency_cycles(cards: list[TaskCard]) -> None:
graph = {card.task_id: card.dependencies for card in cards}
visiting: set[str] = set()
visited: set[str] = set()
def visit(node: str) -> None:
if node in visiting:
raise ValueError(f"dependency cycle detected at {node}")
if node in visited:
return
visiting.add(node)
for dep in graph.get(node, []):
visit(dep)
visiting.remove(node)
visited.add(node)
for task_id in graph:
visit(task_id)
def validate_task_cards(cards: list[TaskCard]) -> None:
seen: set[str] = set()
for card in cards:
if not card.research_goal:
card.research_goal = f"围绕 {card.topic_axis} 轴收集并验证结构化证据。"
if not card.prompt_brief:
card.prompt_brief = f"{card.topic_axis} 轴形成证据包,避免泛泛结论。"
if not card.required_skills:
card.required_skills = _default_required_skills(card.topic_axis)
if "search-gateway" not in card.required_skills:
card.required_skills = ["search-gateway", *card.required_skills]
if not card.expected_evidence:
card.expected_evidence = _default_expected_evidence(card.topic_axis)
if not card.stop_conditions:
card.stop_conditions = _default_stop_conditions()
if card.task_id in seen:
raise ValueError(f"duplicate task_id: {card.task_id}")
seen.add(card.task_id)
if not card.chapter_ids:
raise ValueError(f"{card.task_id}: chapter_ids required")
if not card.chapter_title:
card.chapter_title = card.chapter_ids[0]
if not card.questions:
raise ValueError(f"{card.task_id}: questions required")
if not card.output_packet.endswith(".json"):
raise ValueError(f"{card.task_id}: output_packet must be json")
invalid_routes = sorted(set(card.search_routes) - VALID_ROUTES)
if invalid_routes:
raise ValueError(f"{card.task_id}: invalid search_routes {invalid_routes}")
missing_deps = sorted({dep for card in cards for dep in card.dependencies} - seen)
if missing_deps:
raise ValueError(f"unknown dependencies: {missing_deps}")
detect_dependency_cycles(cards)
def write_task_cards(path: Path, cards: list[TaskCard]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps([card.to_dict() for card in cards], ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
def load_task_cards(path: Path) -> list[TaskCard]:
data = json.loads(path.read_text(encoding="utf-8"))
cards = [TaskCard(**item) for item in data]
validate_task_cards(cards)
return cards
def validate_packet(packet: dict[str, Any]) -> None:
required = {
"task_id",
"claims",
"evidence_items",
"counter_evidence",
"source_ids",
"source_quality_notes",
"open_questions",
"raw_quotes_or_notes",
}
missing = sorted(required - set(packet))
if missing:
raise ValueError(f"packet missing fields: {missing}")
if not packet["claims"]:
raise ValueError("packet claims must not be empty")
if not packet["evidence_items"]:
raise ValueError("packet evidence_items must not be empty")
if not packet["counter_evidence"]:
raise ValueError("packet counter_evidence must not be empty")
declared = set(packet.get("source_ids") or [])
referenced: set[str] = set()
for section in ("claims", "counter_evidence"):
for item in packet.get(section) or []:
referenced.update(item.get("source_ids") or [])
for item in packet.get("evidence_items") or []:
if item.get("source_id"):
referenced.add(item["source_id"])
undeclared = sorted(referenced - declared)
if undeclared:
raise ValueError(f"packet source_ids referenced but not declared: {undeclared}")
packet_sources = packet.get("sources") or []
if not packet_sources:
raise ValueError("packet sources must not be empty")
known_source_ids = {source.get("id") for source in packet_sources}
missing_sources = sorted(declared - known_source_ids)
if missing_sources:
raise ValueError(f"packet source_ids missing source metadata: {missing_sources}")
+497
View File
@@ -0,0 +1,497 @@
"""Python role workers for task-card execution."""
from __future__ import annotations
import json
import re
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Callable, Protocol, Any
from scripts.runtime.roles import RoleDefinition, RuntimeProfile
from scripts.runtime.skills import SkillRegistry
from scripts.runtime.tasks import TaskCard, validate_packet
from scripts.runtime.sources import append_packet_sources
class ChatClient(Protocol):
def chat_complete(self, **kwargs) -> str:
...
class SearchProvider(Protocol):
def search(self, *, query: str, route: str, num_results: int) -> list[dict[str, Any]]:
...
class ProjectSearchProvider:
"""Thin adapter over the project-owned search client."""
def __init__(self, *, strict_specialized: bool = True) -> None:
from scripts.lib.search_client import SearchClient
self.client = SearchClient(strict_specialized=strict_specialized)
def search(self, *, query: str, route: str, num_results: int) -> list[dict[str, Any]]:
if route == "scholar":
hits = self.client.scholar(query, num_results=num_results, year_low=2020)
elif route == "patents":
hits = self.client.patents(query, num_results=num_results)
elif route == "news":
hits = self.client.news(query, num_results=num_results, time_range="y")
elif route == "fda":
hits = self.client.fda(query, num_results=num_results)
elif route == "evidence":
hits = self.client.evidence(query, num_results=num_results)
else:
hits = self.client.search(query, num_results=num_results)
return [
{
"title": hit.title,
"url": hit.url,
"snippet": hit.snippet,
"route": route,
}
for hit in hits
]
def close(self) -> None:
self.client.close()
def _extract_json_object(text: str) -> dict:
stripped = text.strip()
if stripped.startswith("```"):
stripped = stripped.strip("`")
if stripped.startswith("json"):
stripped = stripped[4:].strip()
start = stripped.find("{")
end = stripped.rfind("}")
if start == -1 or end == -1 or end < start:
raise ValueError("worker response does not contain a JSON object")
return json.loads(stripped[start : end + 1])
def _safe_source_stem(task_id: str) -> str:
return re.sub(r"[^a-zA-Z0-9]+", "_", task_id).strip("_").lower()
def contains_cjk(text: str) -> bool:
return any("\u4e00" <= char <= "\u9fff" for char in text)
def strip_cjk(text: str) -> str:
return re.sub(r"[\u3400-\u9fff]+", " ", text)
def validate_packet_against_allowed_context(
packet: dict,
search_context: dict[str, Any] | None,
material_context: dict[str, Any] | None,
) -> None:
"""Ensure the model did not invent source IDs or URLs beyond candidates."""
if not search_context and not material_context:
return
candidates = (search_context or {}).get("candidate_sources") or []
materials = (material_context or {}).get("materials") or []
if not candidates and not materials:
return
candidate_ids = {source.get("id") for source in candidates}
candidate_ids.update(item.get("source_id") for item in materials)
candidate_urls = {source.get("url") for source in candidates if source.get("url")}
candidate_urls.update(item.get("path") for item in materials if item.get("path"))
packet_sources = packet.get("sources") or []
unknown_ids = sorted(
source.get("id")
for source in packet_sources
if source.get("id") and source.get("id") not in candidate_ids
)
unknown_urls = sorted(
source.get("url")
for source in packet_sources
if source.get("url") and source.get("url") not in candidate_urls
)
if (candidates or materials) and not packet_sources:
raise ValueError("packet must include source metadata from candidate_sources or local materials")
if unknown_ids:
raise ValueError(f"packet sources include non-candidate source IDs: {unknown_ids}")
if unknown_urls:
raise ValueError(f"packet sources include non-candidate URLs: {unknown_urls}")
def normalize_packet_against_context(
packet: dict[str, Any],
search_context: dict[str, Any] | None,
material_context: dict[str, Any] | None,
) -> dict[str, Any]:
"""Deterministically fill schema metadata the model often omits."""
packet = dict(packet)
referenced: set[str] = set(packet.get("source_ids") or [])
for section in ("claims", "counter_evidence"):
for item in packet.get(section) or []:
referenced.update(item.get("source_ids") or [])
for item in packet.get("evidence_items") or []:
if item.get("source_id"):
referenced.add(item["source_id"])
if "source_ids" not in packet or not packet.get("source_ids"):
packet["source_ids"] = sorted(referenced)
available_sources: dict[str, dict[str, Any]] = {}
for source in (search_context or {}).get("candidate_sources") or []:
if source.get("id"):
available_sources[source["id"]] = source
for material in (material_context or {}).get("materials") or []:
source_id = material.get("source_id")
if source_id:
available_sources[source_id] = {
"id": source_id,
"title": material.get("title") or Path(material.get("path", "")).name,
"url": material.get("path") or "",
"tier": "local_material",
"score": 8,
}
existing_sources = {
source.get("id"): source
for source in packet.get("sources") or []
if source.get("id")
}
for source_id in packet.get("source_ids") or []:
if source_id not in existing_sources and source_id in available_sources:
existing_sources[source_id] = available_sources[source_id]
if existing_sources:
packet["sources"] = [existing_sources[source_id] for source_id in packet.get("source_ids", []) if source_id in existing_sources]
return packet
FDA_AXIS_TERMS = {
"nmpa_fda_ema_ich_who_baseline": "CGMP pharmaceutical quality system process validation aseptic processing data integrity",
"quality_system_gap": "CGMP CAPA deviation change control data integrity quality unit pharmaceutical",
"manufacturing_process_risk": "aseptic processing sterile drug manufacturing process validation PPQ cleaning validation water system",
"operations_management_gap": "pharmaceutical quality system quality metrics management review senior management FDA",
"capa_roadmap": "CGMP CAPA effectiveness remediation warning letter close-out pharmaceutical",
"verification_evidence": "FDA 483 response CAPA effectiveness verification EIR pharmaceutical quality",
"counter": "FDA warning letter CGMP pharmaceutical quality data integrity remediation limitations",
"fda_enforcement_precedents": "FDA warning letter CGMP pharmaceutical aseptic processing data integrity CAPA process validation",
}
FDA_CHAPTER_TERMS = {
"ch01": "commercial readiness phase gate remediation governance",
"ch02": "regulatory baseline CGMP EU GMP Annex 1 ICH Q9 ICH Q10",
"ch03": "aseptic processing RABS first air media fill visual inspection depyrogenation tunnel",
"ch04": "biologics drug substance WFI clean utilities SCADA EMS single-use system",
"ch05": "process validation master batch record CPP CQA PPQ cleaning validation technology transfer",
"ch06": "deviation change control CAPA document control training data integrity quality unit",
"ch07": "training effectiveness quality culture operator qualification human factors",
"ch08": "quality metrics management review escalation cross-functional governance operations",
"ch09": "CDMO quality organization technology transfer project governance capability matrix",
"ch10": "CAPA remediation plan effectiveness check owner due date verification evidence",
"ch11": "regulatory mapping CAPA tracker closure evidence quality assurance verification",
}
ROUTE_CHAPTER_TERMS = {
**FDA_CHAPTER_TERMS,
}
ROUTE_SUFFIX_TERMS = {
"scholar": "pharmaceutical GMP review validation risk management quality system",
"patents": "biologics manufacturing patent process formulation device",
"news": "pharmaceutical quality operations CDMO quality governance",
"evidence": "pharmaceutical GMP evidence guidance enforcement best practice quality operations",
"general": "pharmaceutical GMP best practice guidance quality operations remediation",
}
INTERNAL_QUERY_TOKENS = {
"chapter_integrated",
"input_material_findings",
}
def _compact_english_query(*parts: str, max_terms: int = 16) -> str:
text = strip_cjk(" ".join(part for part in parts if part))
text = re.sub(r"[^A-Za-z0-9./+-]+", " ", text)
terms: list[str] = []
seen: set[str] = set()
for raw in text.split():
term = raw.strip(" ./+-").lower()
if not term or term in INTERNAL_QUERY_TOKENS:
continue
key = term.casefold()
if key in seen:
continue
seen.add(key)
terms.append(term)
if len(terms) >= max_terms:
break
return " ".join(terms)
def _chapter_terms(card: TaskCard) -> str:
mapped = " ".join(ROUTE_CHAPTER_TERMS.get(chapter_id, "") for chapter_id in card.chapter_ids)
if mapped.strip():
return mapped
return strip_cjk(card.chapter_title)
def build_route_query(card: TaskCard, route: str) -> str:
"""Build short, route-aware queries instead of sending whole task cards."""
if route == "fda":
terms = FDA_AXIS_TERMS.get(card.topic_axis, "FDA warning letter CGMP pharmaceutical quality")
chapter_terms = " ".join(FDA_CHAPTER_TERMS.get(chapter_id, "") for chapter_id in card.chapter_ids)
query = f"{terms} {chapter_terms}".strip()
if contains_cjk(query):
raise ValueError(f"FDA route query must not contain Chinese text: {query}")
return query
if route == "scholar":
return _compact_english_query(_chapter_terms(card), ROUTE_SUFFIX_TERMS["scholar"])
if route == "patents":
return _compact_english_query(_chapter_terms(card), ROUTE_SUFFIX_TERMS["patents"])
if route == "news":
return _compact_english_query(_chapter_terms(card), ROUTE_SUFFIX_TERMS["news"])
if route == "evidence":
return _compact_english_query(_chapter_terms(card), ROUTE_SUFFIX_TERMS["evidence"])
return _compact_english_query(_chapter_terms(card), ROUTE_SUFFIX_TERMS["general"])
def _material_excerpt(project_root: Path | None, rel_path: str, *, max_chars: int = 6000) -> dict[str, str] | None:
if project_root is None:
return None
path = project_root / rel_path
if not path.exists() or not path.is_file():
return None
text = path.read_text(encoding="utf-8", errors="ignore")
return {
"path": rel_path,
"source_id": f"src_local_{_safe_source_stem(Path(rel_path).stem)}",
"title": Path(rel_path).name,
"excerpt": text[:max_chars],
}
def build_material_context(card: TaskCard, project_root: Path | None, *, max_chars_per_material: int = 6000) -> dict[str, Any]:
materials = []
seen: set[str] = set()
for rel in card.allowed_materials:
if rel in seen:
continue
seen.add(rel)
item = _material_excerpt(project_root, rel, max_chars=max_chars_per_material)
if item:
materials.append(item)
return {"materials": materials}
def build_search_context(
card: TaskCard,
search_provider: SearchProvider,
*,
num_results_per_route: int = 5,
) -> dict[str, Any]:
candidate_sources: list[dict[str, Any]] = []
routes_used: list[str] = []
source_stem = _safe_source_stem(card.task_id)
idx = 1
for route in card.search_routes:
routes_used.append(route)
query = build_route_query(card, route)
hits = search_provider.search(query=query, route=route, num_results=num_results_per_route)
for hit in hits:
candidate_sources.append(
{
"id": f"src_{source_stem}_{idx:03d}",
"title": hit.get("title", ""),
"url": hit.get("url", ""),
"snippet": hit.get("snippet", ""),
"route": hit.get("route", route),
"tier": "Tier 2",
"score": 6,
}
)
idx += 1
return {"routes_used": routes_used, "candidate_sources": candidate_sources}
def build_packet_user_prompt(
card: TaskCard,
search_context: dict[str, Any] | None = None,
material_context: dict[str, Any] | None = None,
) -> str:
context = search_context or {"routes_used": [], "candidate_sources": []}
materials = material_context or {"materials": []}
return (
"请根据以下 task card 产出一个证据包 JSON。\n"
"正式结论、summary、open_questions 用中文;英文原文摘录、source title、DOI/URL 可以保留英文。\n"
"必须主动包含 counter_evidence,且所有引用的 source_id 必须出现在 source_ids 中。\n\n"
"只能使用 candidate_sources 或 Local material context 中的来源,不得编造 URL、DOI、trial ID 或 source_id。\n"
"输出 JSON 必须包含 sources 字段;sources 只能来自 candidate_sources 或 Local material context。\n"
"如 Local material context 非空,必须至少提取 1 条本地材料原文证据;如果与本章无关,必须在 open_questions 说明为什么无关。\n\n"
f"{json.dumps(card.to_dict(), ensure_ascii=False, indent=2)}\n\n"
f"Local material context:\n{json.dumps(materials, ensure_ascii=False, indent=2)}\n\n"
f"Search context:\n{json.dumps(context, ensure_ascii=False, indent=2)}\n\n"
"只输出 JSON,不要输出 Markdown 解释。"
)
def build_packet_repair_prompt(
*,
card: TaskCard,
raw_response: str,
error: Exception,
search_context: dict[str, Any] | None = None,
material_context: dict[str, Any] | None = None,
) -> str:
context = search_context or {"routes_used": [], "candidate_sources": []}
materials = material_context or {"materials": []}
return (
"请修复上一次 evidence packet 输出,使其成为合法且通过 schema 校验的 JSON。\n"
"只输出 JSON 对象,不要输出 Markdown、解释或代码块。\n"
"保留中文主写作;英文只允许出现在来源标题、URL、DOI、原文摘录或检索笔记中。\n"
"不得编造 candidate_sources 或 Local material context 以外的来源、URL、DOI、trial ID 或 source_id。\n\n"
f"Schema error:\n{error}\n\n"
f"Task card:\n{json.dumps(card.to_dict(), ensure_ascii=False, indent=2)}\n\n"
f"Search context:\n{json.dumps(context, ensure_ascii=False, indent=2)}\n\n"
f"Local material context:\n{json.dumps(materials, ensure_ascii=False, indent=2)}\n\n"
f"Previous raw response:\n{raw_response[:12000]}"
)
class PacketWorker:
def __init__(
self,
*,
role: RoleDefinition,
client: ChatClient,
project_root: Path | None = None,
search_provider: SearchProvider | None = None,
skill_registry: SkillRegistry | None = None,
num_results_per_route: int = 5,
) -> None:
self.role = role
self.client = client
self.project_root = project_root
self.search_provider = search_provider
self.skill_registry = skill_registry or SkillRegistry()
self.num_results_per_route = num_results_per_route
def _system_prompt(self) -> str:
skill_texts = []
for name in self.role.skills:
try:
skill_texts.append(f"# Skill: {name}\n\n{self.skill_registry.read(name)}")
except FileNotFoundError:
skill_texts.append(f"# Skill: {name}\n\n[missing skill: {name}]")
return (
f"{self.role.identity}\n\n"
"你是 Deep Research v0.20 Python runtime 的证据包 worker。\n"
"你的唯一任务是把一个 task card 转换为结构化 evidence packet。\n"
"遵循中文主写作原则;不要写章节正文;不要编造 URL、DOI、trial ID 或 source_id。\n\n"
"搜索只能走项目 Python search gateway 或调用方提供的 search_context;不要直接使用 Tavily MCP、browser MCP、平台 web search 或任何需要用户权限确认的外部搜索工具。\n\n"
+ "\n\n".join(skill_texts)
)
def run(self, card: TaskCard) -> dict:
search_context = None
if self.search_provider:
search_context = build_search_context(
card,
self.search_provider,
num_results_per_route=self.num_results_per_route,
)
material_context = build_material_context(card, self.project_root)
raw = self.client.chat_complete(
model=self.role.model,
system=self._system_prompt(),
user=build_packet_user_prompt(card, search_context, material_context),
temperature=self.role.temperature,
max_tokens=self.role.max_tokens,
tag=f"packet:{card.task_id}",
)
try:
packet = normalize_packet_against_context(
_extract_json_object(raw),
search_context,
material_context,
)
validate_packet(packet)
validate_packet_against_allowed_context(packet, search_context, material_context)
return packet
except Exception as error:
repaired = self.client.chat_complete(
model=self.role.model,
system=self._system_prompt(),
user=build_packet_repair_prompt(
card=card,
raw_response=raw,
error=error,
search_context=search_context,
material_context=material_context,
),
temperature=0,
max_tokens=self.role.max_tokens,
tag=f"packet-repair:{card.task_id}",
)
packet = normalize_packet_against_context(
_extract_json_object(repaired),
search_context,
material_context,
)
validate_packet(packet)
validate_packet_against_allowed_context(packet, search_context, material_context)
return packet
def _write_packet_error(project_root: Path, card: TaskCard, error: Exception) -> None:
path = project_root / "phase2" / "packet_errors" / f"{card.task_id}.json"
path.parent.mkdir(parents=True, exist_ok=True)
payload = {
"task_id": card.task_id,
"status": "failed",
"error": str(error),
"output_packet": card.output_packet,
}
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def run_packet_workers(
*,
project_root: Path,
cards: list[TaskCard],
runtime: RuntimeProfile,
client_factory: Callable[[RoleDefinition], ChatClient],
search_provider_factory: Callable[[], SearchProvider] | None = None,
workers: int,
) -> int:
role = runtime.role_for_task("evidence_packet")
max_workers = max(1, min(workers, role.max_concurrency))
def run_one(card: TaskCard) -> tuple[TaskCard, dict | None, Exception | None]:
search_provider = search_provider_factory() if search_provider_factory else None
try:
worker = PacketWorker(role=role, client=client_factory(role), project_root=project_root, search_provider=search_provider)
return card, worker.run(card), None
except Exception as error:
return card, None, error
finally:
close = getattr(search_provider, "close", None)
if close:
close()
written = 0
with ThreadPoolExecutor(max_workers=max_workers) as pool:
futures = [pool.submit(run_one, card) for card in cards]
for future in as_completed(futures):
card, packet, error = future.result()
if error is not None:
_write_packet_error(project_root, card, error)
continue
if packet is None:
_write_packet_error(project_root, card, RuntimeError("packet worker returned no packet"))
continue
path = project_root / card.output_packet
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(packet, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
append_packet_sources(project_root / "phase2" / "sources.jsonl", packet)
written += 1
return written
+194
View File
@@ -0,0 +1,194 @@
#!/usr/bin/env python3
"""Unified search gateway for Deep Research agents.
This script is the stable project-owned entrypoint that agents should call
instead of vendor MCP tools. MCP search remains optional, while this gateway
keeps routing behavior reproducible across OpenCode, Codex, and future
adapters.
"""
from __future__ import annotations
import argparse
import json
import sys
from dataclasses import asdict
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from scripts.lib.search_client import SearchClient, SearchError, SearchHit
from scripts.lib.zenmux_client import load_secrets
ROUTE_HELP = {
"general": "Tavily -> Exa -> Brave generic web discovery",
"evidence": "Exa highlights -> Tavily -> Brave controlled evidence discovery",
"scholar": "Serper Scholar -> generic fallback",
"patents": "Serper Google Patents -> site:patents.google.com fallback",
"news": "Serper News -> generic fallback",
"fda": "FDA-focused discovery for warning letters, enforcement pages, and meeting materials",
}
PROFILE_ROUTES = {
"biomed_literature": ["scholar", "evidence", "general"],
"patent_heavy": ["patents", "evidence", "general"],
"china_market": ["news", "evidence", "general"],
"investment": ["news", "evidence", "general"],
}
PROFILE_QUERY_PREFIX = {
"china_market": "(China OR Chinese OR 中国 OR 国内)",
}
def search_route(client: SearchClient, route: str, query: str, args: argparse.Namespace) -> list[SearchHit]:
if route == "general":
return client.search(query, num_results=args.num_results)
if route == "evidence":
return client.evidence(query, num_results=args.num_results, category=args.exa_category)
if route == "scholar":
return client.scholar(query, num_results=args.num_results, year_low=args.year_low)
if route == "patents":
return client.patents(query, num_results=args.num_results)
if route == "news":
return client.news(query, num_results=args.num_results, time_range=args.time_range)
if route == "fda":
return client.fda(query, num_results=args.num_results)
raise SystemExit(f"unknown route: {route}")
def emit_markdown(route_hits: list[tuple[str, list[SearchHit]]], query: str) -> None:
print(f"# Search Results: {query}")
for route, hits in route_hits:
print()
print(f"## Route: {route} ({ROUTE_HELP[route]})")
if not hits:
print("No results.")
continue
for i, hit in enumerate(hits, start=1):
print(f"{i}. {hit.title or '(untitled)'}")
print(f" - URL: {hit.url}")
if hit.snippet:
print(f" - Snippet: {hit.snippet}")
def emit_json(route_hits: list[tuple[str, list[SearchHit]]], query: str) -> None:
data = {
"query": query,
"routes": [
{
"route": route,
"route_help": ROUTE_HELP[route],
"results": [asdict(hit) for hit in hits],
}
for route, hits in route_hits
],
}
print(json.dumps(data, ensure_ascii=False, indent=2))
def emit_trace_markdown(route_trace: list[dict[str, str]]) -> None:
print()
print("## Route Trace")
for item in route_trace:
print(f"- {item['route']}: {item['status']} ({item['detail']})")
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Deep Research search gateway")
parser.add_argument("query", help="Search query")
parser.add_argument(
"--route",
choices=sorted(ROUTE_HELP),
default="general",
help="Single search route to run",
)
parser.add_argument(
"--profile",
choices=sorted(PROFILE_ROUTES),
help="Run a strategy profile instead of a single route",
)
parser.add_argument("--num-results", type=int, default=10)
parser.add_argument(
"--exa-category",
choices=["research paper", "news", "company", "financial report", "github", "tweet", "personal site", "pdf"],
help="Optional Exa category for the evidence route",
)
parser.add_argument("--year-low", type=int, help="Lower year bound for scholar searches")
parser.add_argument("--time-range", choices=["d", "w", "m", "y"], help="Serper news time range")
parser.add_argument("--json", action="store_true", help="Emit JSON instead of Markdown")
parser.add_argument("--dry-run", action="store_true", help="Show planned routes without calling APIs")
parser.add_argument(
"--strict-specialized",
action=argparse.BooleanOptionalAction,
default=True,
help="Fail fast if scholar/news/patents cannot use Serper",
)
parser.add_argument("--trace", action="store_true", help="Include route execution trace")
return parser
def main() -> int:
parser = build_parser()
args = parser.parse_args()
load_secrets()
routes = PROFILE_ROUTES[args.profile] if args.profile else [args.route]
query = args.query
if args.profile in PROFILE_QUERY_PREFIX:
query = f"{PROFILE_QUERY_PREFIX[args.profile]} {query}"
if args.dry_run:
for route in routes:
print(f"{route}: {ROUTE_HELP[route]}")
if query != args.query:
print(f"query_rewritten: {query}")
return 0
try:
with SearchClient(strict_specialized=args.strict_specialized) as client:
route_hits = []
route_trace: list[dict[str, str]] = []
for route in routes:
try:
hits = search_route(client, route, query, args)
route_hits.append((route, hits))
route_trace.append({"route": route, "status": "ok", "detail": f"hits={len(hits)}"})
except SearchError as exc:
route_hits.append((route, []))
route_trace.append({"route": route, "status": "failed", "detail": str(exc)})
if route != "general":
continue
raise
except SearchError as exc:
raise SystemExit(f"search failed: {exc}") from exc
if args.json:
data = {
"query": query,
"original_query": args.query,
"strict_specialized": args.strict_specialized,
"routes": [
{
"route": route,
"route_help": ROUTE_HELP[route],
"results": [asdict(hit) for hit in hits],
}
for route, hits in route_hits
],
"trace": route_trace if args.trace else [],
}
print(json.dumps(data, ensure_ascii=False, indent=2))
else:
emit_markdown(route_hits, query)
if args.trace:
emit_trace_markdown(route_trace)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+147
View File
@@ -0,0 +1,147 @@
#!/usr/bin/env bash
# Deep Research - Environment setup (macOS + Debian/Ubuntu, using uv)
# Usage: bash scripts/setup.sh
set -euo pipefail
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
CYAN='\033[0;36m'
RST='\033[0m'
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
cd "${PROJECT_ROOT}"
echo -e "${CYAN}============================================${RST}"
echo -e "${CYAN} Deep Research - Setup (uv)${RST}"
echo -e "${CYAN} Project root: ${PROJECT_ROOT}${RST}"
echo -e "${CYAN}============================================${RST}"
# -------- OS detection --------
OS="unknown"
case "$(uname -s)" in
Darwin) OS="macos" ;;
Linux)
if [[ -f /etc/debian_version ]]; then
OS="debian"
elif [[ -f /etc/redhat-release ]]; then
OS="rhel"
else
OS="linux"
fi
;;
esac
echo -e "${YELLOW}Detected OS: ${OS}${RST}"
# -------- [1/4] uv check / install --------
echo ""
echo -e "${CYAN}[1/4] Checking uv...${RST}"
if ! command -v uv >/dev/null 2>&1; then
echo -e " ${YELLOW}uv not found. Installing...${RST}"
echo -e " ${YELLOW}(uv = fast Python package manager written in Rust, by Astral)${RST}"
echo ""
read -r -p " Auto-install uv? [Y/n] " REPLY
echo ""
if [[ ! ${REPLY} =~ ^[Nn]$ ]]; then
curl -LsSf https://astral.sh/uv/install.sh | sh
# bring uv into current PATH
[[ -f "${HOME}/.local/bin/uv" ]] && export PATH="${HOME}/.local/bin:${PATH}"
if ! command -v uv >/dev/null 2>&1; then
echo -e " ${RED}ERROR: uv still not found after install.${RST}"
echo " Add ~/.local/bin to PATH, then re-run:"
echo " export PATH=\"\${HOME}/.local/bin:\${PATH}\""
exit 1
fi
else
echo -e " ${RED}Manual install:${RST}"
echo " macOS: brew install uv"
echo " all: curl -LsSf https://astral.sh/uv/install.sh | sh"
echo " pip: pip install --user uv"
exit 1
fi
fi
UV_VER="$(uv --version 2>&1 | head -1)"
echo -e " ${GREEN}OK: ${UV_VER}${RST}"
# -------- [2/4] Python info --------
echo ""
echo -e "${CYAN}[2/4] Python check...${RST}"
if command -v python3 >/dev/null 2>&1; then
SYS_PY="$(python3 --version 2>&1)"
echo -e " ${GREEN}System: ${SYS_PY} (uv manages its own Python 3.10+ automatically)${RST}"
else
echo -e " ${YELLOW}python3 not found in PATH (uv will download Python automatically)${RST}"
fi
# -------- [3/4] uv sync --------
echo ""
echo -e "${CYAN}[3/4] Syncing Python dependencies (uv sync)...${RST}"
echo -e " ${YELLOW}First run may take a moment to download Python + packages.${RST}"
uv sync
echo -e " ${GREEN}OK: .venv/ ready${RST}"
# -------- [4/4] System binaries --------
echo ""
echo -e "${CYAN}[4/4] Checking system binaries...${RST}"
check_bin() {
local name="${1}"
local install_macos="${2}"
local install_debian="${3}"
if command -v "${name}" >/dev/null 2>&1; then
local ver
ver="$("${name}" --version 2>&1 | head -1)"
echo -e " ${GREEN}OK: ${name} - ${ver}${RST}"
else
echo -e " ${YELLOW}MISSING: ${name}${RST}"
case "${OS}" in
macos) echo " Install: ${install_macos}" ;;
debian) echo " Install: ${install_debian}" ;;
*) echo " Install ${name} for your OS" ;;
esac
fi
}
check_bin "pandoc" "brew install pandoc" "sudo apt install pandoc"
check_bin "opencode" "curl -fsSL https://opencode.ai/install | bash" "curl -fsSL https://opencode.ai/install | bash"
check_bin "curl" "pre-installed" "sudo apt install curl"
# -------- Summary --------
echo ""
echo -e "${CYAN}============================================${RST}"
echo -e "${CYAN} Setup complete${RST}"
echo -e "${CYAN}============================================${RST}"
echo ""
if [[ -f secrets.env ]]; then
echo -e "${GREEN}OK: secrets.env found${RST}"
else
echo -e "${YELLOW}TODO: create secrets.env${RST}"
echo " cp secrets.env.example secrets.env"
echo " # fill in ZENMUX_API_KEY etc."
fi
FONT_COUNT="$(find .opencode/templates/fonts -maxdepth 1 \( -name "*.otf" -o -name "*.ttf" \) 2>/dev/null | wc -l | tr -d ' ')"
if [[ "${FONT_COUNT}" -ge 7 ]]; then
echo -e "${GREEN}OK: fonts ready (${FONT_COUNT} files)${RST}"
else
echo -e "${YELLOW}TODO: download fonts (${FONT_COUNT}/7 present)${RST}"
echo " bash .opencode/templates/fonts/download-fonts.sh"
fi
echo ""
echo "Next steps:"
echo " 1. source scripts/activate.sh"
echo " 2. bash .opencode/templates/fonts/download-fonts.sh"
echo " 3. bash scripts/verify-zenmux.sh"
echo " 4. opencode"
echo ""
echo "Tips:"
echo " Run a script directly: uv run python .opencode/templates/report-template.py ..."
echo " Add a package: uv add <package>"
echo " Upgrade all: uv lock --upgrade && uv sync"
echo " Full rebuild: rm -rf .venv && bash scripts/setup.sh"
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
"""Sprint 5 regression checks for v0.12 changes.
Checks are non-destructive and default to dry-run behavior.
"""
from __future__ import annotations
import argparse
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
def run(cmd: list[str]) -> tuple[int, str]:
proc = subprocess.run(
cmd,
cwd=REPO_ROOT,
check=False,
capture_output=True,
text=True,
)
out = (proc.stdout or "") + (proc.stderr or "")
return proc.returncode, out
def check(name: str, cmd: list[str], must_contain: list[str] | None = None) -> bool:
print(f"[check] {name}")
print(" $ " + " ".join(cmd))
rc, out = run(cmd)
if rc != 0:
print(f" FAIL: exit={rc}")
if out.strip():
print(" output:")
print(" " + out.strip().replace("\n", "\n "))
return False
for token in must_contain or []:
if token not in out:
print(f" FAIL: missing token '{token}'")
return False
print(" PASS")
return True
def main() -> int:
parser = argparse.ArgumentParser(description="Run Sprint 5 regression checks")
parser.add_argument("project", help="Project slug or path for finalize dry-run")
args = parser.parse_args()
checks = [
(
"model profiles list",
["uv", "run", "python", "scripts/dr.py", "models", "--list"],
["medium", "premium", "simple"],
),
(
"search gateway dry-run",
[
"uv",
"run",
"python",
"scripts/search.py",
"GLP-1 obesity",
"--profile",
"china_market",
"--dry-run",
],
["news:", "general:", "query_rewritten:"],
),
(
"phase4 finalize dry-run",
[
"uv",
"run",
"python",
"scripts/dr.py",
"finalize",
args.project,
"--model-profile",
"medium",
"--dry-run",
],
["Phase 4 pipeline done"],
),
]
ok = True
for name, cmd, tokens in checks:
ok = check(name, cmd, tokens) and ok
if not ok:
print("\nSprint 5 regression: FAILED")
return 1
print("\nSprint 5 regression: PASSED")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+359
View File
@@ -0,0 +1,359 @@
#!/usr/bin/env python3
"""Phase 4 英译中:章节级切块 → 逐块翻译 → 拼接 → 写盘。
用法
uv run python scripts/translate.py <project_slug>
# 或:
uv run python scripts/translate.py projects/dual-target-rnai-pipeline-2026
断点续传每块翻译完立即写入 `phase4/zh_chunks/<anchor>.md`术语表 patch 在本轮结束后统一合并
重跑时已存在的块直接跳过只译缺的
设计要点
1. 切块按 H2 粒度单块一般 <600 英文词单次 API 调用远低于 Sonnet output token 上限
2. 并发翻译使用稳定术语表快照译完后统一合并 glossary patch避免多线程写冲突
3. 失败不会污染最终产物块级文件独立可重跑汇总步骤独立
4. 日志完整每次 API 调用写 `phase4/logs/translate.jsonl`
"""
from __future__ import annotations
import argparse
import concurrent.futures
import json
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from scripts.lib.markdown_chunker import (
MarkdownBlock,
count_chinese_chars,
split_by_headers,
)
from scripts.lib.zenmux_client import ZenMuxClient, ZenMuxError, load_secrets
DEFAULT_MODEL = "anthropic/claude-sonnet-4.6"
MODEL_MAX_TOKENS = {
"anthropic/claude-sonnet-4.6": 32000,
"anthropic/claude-sonnet-4.5": 32000,
"anthropic/claude-opus-4.7": 32000,
"anthropic/claude-opus-4.6": 32000,
"anthropic/claude-haiku-4.5": 16000,
}
PROMPT_FILE = Path(__file__).parent / "prompts" / "translate_system.txt"
def resolve_project(arg: str) -> Path:
p = Path(arg)
if p.is_dir():
return p
here = Path.cwd()
cand = here / "projects" / arg
if cand.is_dir():
return cand
raise SystemExit(f"project not found: {arg}")
def parse_delimited_response(text: str) -> tuple[str, dict[str, str]]:
"""解析自定义分隔符格式的响应。
期望结构
<<<TRANSLATION>>>
...markdown...
<<<END_TRANSLATION>>>
<<<GLOSSARY_PATCH>>>
English || 中文
...
<<<END_GLOSSARY_PATCH>>>
Returns:
(translation, glossary_patch)
"""
t_start = text.find("<<<TRANSLATION>>>")
t_end = text.find("<<<END_TRANSLATION>>>")
if t_start == -1 or t_end == -1 or t_end <= t_start:
raise ValueError(
f"missing <<<TRANSLATION>>> markers in response: {text[:300]}"
)
translation = text[t_start + len("<<<TRANSLATION>>>"): t_end].strip("\r\n")
g_start = text.find("<<<GLOSSARY_PATCH>>>")
g_end = text.find("<<<END_GLOSSARY_PATCH>>>")
patch: dict[str, str] = {}
if g_start != -1 and g_end != -1 and g_end > g_start:
body = text[g_start + len("<<<GLOSSARY_PATCH>>>"): g_end]
for line in body.splitlines():
line = line.strip()
if not line or "||" not in line:
continue
en, _, zh = line.partition("||")
en, zh = en.strip(), zh.strip()
if en and zh:
patch[en] = zh
return translation, patch
def load_glossary(path: Path) -> dict[str, str]:
if not path.exists():
return {}
try:
return json.loads(path.read_text(encoding="utf-8"))
except Exception:
return {}
def save_glossary(path: Path, glossary: dict[str, str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(glossary, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
def _glossary_value(value) -> str:
if isinstance(value, dict):
return str(value.get("zh") or value.get("中文") or "")
return str(value)
def build_user_prompt(block: MarkdownBlock, glossary: dict[str, str]) -> str:
glossary_lines = (
f"{en} || {_glossary_value(zh)}"
for en, zh in sorted(glossary.items())
if _glossary_value(zh)
)
glossary_hint = "\n".join(glossary_lines) or "(none yet)"
level_hint = (
f"H{block.level}" if block.level >= 1 else "frontmatter (no heading)"
)
return (
"# GLOSSARY (English || Chinese, already used earlier in the document):\n"
f"{glossary_hint}\n\n"
f"# BLOCK TO TRANSLATE (Markdown, {level_hint}):\n"
"Translate the block between the markers below into Chinese, "
"following all rules in the system prompt. Output using the exact "
"delimiter format specified.\n\n"
"----- BEGIN BLOCK -----\n"
f"{block.content}\n"
"----- END BLOCK -----\n"
)
def translate_block(
client: ZenMuxClient,
block: MarkdownBlock,
glossary: dict[str, str],
*,
model: str,
system_prompt: str,
temperature: float,
) -> tuple[str, dict[str, str]]:
user = build_user_prompt(block, glossary)
max_tok = MODEL_MAX_TOKENS.get(model, 16000)
raw = client.chat_complete(
model=model,
system=system_prompt,
user=user,
temperature=temperature,
max_tokens=max_tok,
tag=f"translate:{block.anchor}",
)
try:
translation, patch = parse_delimited_response(raw)
except Exception as e:
raise RuntimeError(
f"bad response format for block {block.anchor}: {e}\nraw head: {raw[:300]}"
)
if not translation.strip():
raise RuntimeError(f"empty translation for block {block.anchor}")
return translation.rstrip(), patch
def main() -> int:
parser = argparse.ArgumentParser(description="Phase 4 英译中(章节级切块并行翻译)")
parser.add_argument("project", help="项目 slug 或完整路径")
parser.add_argument(
"--source",
default="phase4/final_en.md",
help="英文源文件(相对项目根,默认 phase4/final_en.md",
)
parser.add_argument(
"--output",
default="phase4/final_zh.md",
help="中文输出(默认 phase4/final_zh.md",
)
parser.add_argument("--model", default=DEFAULT_MODEL, help="翻译模型")
parser.add_argument("--temperature", type=float, default=0.3)
parser.add_argument(
"--force",
action="store_true",
help="忽略已有 zh_chunks 缓存,强制重翻",
)
parser.add_argument(
"--only",
default=None,
help="只翻译指定 order(逗号分隔的整数),其余跳过(调试用),例如 --only 0,1,7",
)
parser.add_argument(
"--limit",
type=int,
default=None,
help="最多翻译前 N 个未缓存的块(调试用)",
)
parser.add_argument(
"--workers",
type=int,
default=4,
help="并发翻译 worker 数(默认 4;设为 1 回退串行)",
)
args = parser.parse_args()
load_secrets()
project_root = resolve_project(args.project)
src_path = project_root / args.source
out_path = project_root / args.output
if not src_path.exists():
raise SystemExit(f"source not found: {src_path}")
chunks_dir = project_root / "phase4" / "zh_chunks"
chunks_dir.mkdir(parents=True, exist_ok=True)
glossary_path = project_root / "phase4" / "glossary.json"
logs_dir = project_root / "phase4" / "logs"
log_file = logs_dir / "translate.jsonl"
system_prompt = PROMPT_FILE.read_text(encoding="utf-8")
text = src_path.read_text(encoding="utf-8")
blocks = split_by_headers(text, max_level=2)
glossary = load_glossary(glossary_path)
only_orders: set[int] | None = None
if args.only:
only_orders = {int(a.strip()) for a in args.only.split(",") if a.strip()}
total_en_words = sum(b.word_count for b in blocks)
print(f"Source: {src_path.relative_to(project_root)}")
print(f"Blocks: {len(blocks)} | total English words: {total_en_words:,}")
print(f"Glossary loaded: {len(glossary)} terms")
workers = max(1, args.workers)
print(f"Model: {args.model} | temperature: {args.temperature} | workers: {workers}")
print()
start = time.time()
todo: list[MarkdownBlock] = []
cached_count = 0
for b in blocks:
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
if only_orders is not None and b.order not in only_orders:
continue
if chunk_path.exists() and not args.force:
cached_count += 1
print(f" [ok ] #{b.order:03d} {b.short_title} (cached)")
continue
if args.limit is not None and len(todo) >= args.limit:
continue
todo.append(b)
if todo:
print(f"To translate this run: {len(todo)} blocks | cached: {cached_count}")
print()
successful_patches: list[dict[str, str]] = []
with ZenMuxClient(log_file=log_file) as client:
def run_one(b: MarkdownBlock) -> tuple[MarkdownBlock, str, dict[str, str], float]:
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
t0 = time.time()
translation, patch = translate_block(
client,
b,
glossary,
model=args.model,
system_prompt=system_prompt,
temperature=args.temperature,
)
chunk_path.write_text(translation + "\n", encoding="utf-8")
return b, translation, patch, time.time() - t0
if workers == 1:
for b in todo:
label = f"#{b.order:03d} L{b.level} {b.word_count:>4}w {b.short_title}"
print(f" [... ] {label} ", end="", flush=True)
try:
_block, translation, patch, elapsed = run_one(b)
except (ZenMuxError, RuntimeError) as e:
print(f"\n [FAIL] {label}\n {e}")
continue
successful_patches.append(patch)
cn = count_chinese_chars(translation)
print(f"\r [done] {label}{cn:>4}字 ({elapsed:4.1f}s, +{len(patch)} terms)")
else:
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
futures = {pool.submit(run_one, b): b for b in todo}
for fut in concurrent.futures.as_completed(futures):
b = futures[fut]
label = f"#{b.order:03d} L{b.level} {b.word_count:>4}w {b.short_title}"
try:
_block, translation, patch, elapsed = fut.result()
except (ZenMuxError, RuntimeError) as e:
print(f" [FAIL] {label}\n {e}")
continue
successful_patches.append(patch)
cn = count_chinese_chars(translation)
print(f" [done] {label}{cn:>4}字 ({elapsed:4.1f}s, +{len(patch)} terms)")
glossary_conflicts: list[tuple[str, str, str]] = []
for patch in successful_patches:
for k, v in patch.items():
if k not in glossary:
glossary[k] = v
elif _glossary_value(glossary[k]) != v:
glossary_conflicts.append((k, _glossary_value(glossary[k]), v))
if successful_patches:
save_glossary(glossary_path, glossary)
if glossary_conflicts:
print(f" [WARN] glossary patch 冲突 {len(glossary_conflicts)} 条,保留既有译法")
# 汇总:按 order 拼接所有 chunk
merged: list[str] = []
missing: list[str] = []
for b in blocks:
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
if not chunk_path.exists():
missing.append(f"#{b.order:03d} {b.short_title}")
continue
merged.append(chunk_path.read_text(encoding="utf-8").rstrip())
partial = only_orders is not None or args.limit is not None
if missing:
print(f"\n⚠ 缺失 {len(missing)} 块:")
for m in missing[:10]:
print(f" - {m}")
if len(missing) > 10:
print(f" ... 还有 {len(missing) - 10}")
if partial:
print("partial 模式:--only / --limit 生效,未生成 final_zh.md")
else:
print("重新运行本脚本即可补译(已译的会跳过)。")
print(client.usage.summary())
return 1
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text("\n\n".join(merged) + "\n", encoding="utf-8")
# 统计
final_text = out_path.read_text(encoding="utf-8")
cn = count_chinese_chars(final_text)
total_time = time.time() - start
print()
print(f"✓ 输出:{out_path.relative_to(project_root)}")
print(f" 中文字数: {cn:,}")
print(f" 英文词数源: {total_en_words:,} 膨胀率: {cn / max(total_en_words,1):.2f}×")
print(f" 耗时: {total_time:.1f}s")
print(f" 术语表: {len(glossary)} 条 → {glossary_path.relative_to(project_root)}")
print(client.usage.summary())
return 0
if __name__ == "__main__":
sys.exit(main())
+157
View File
@@ -0,0 +1,157 @@
#!/usr/bin/env python3
"""Pull the repo and rebuild isolated platform workspaces at repository root."""
from __future__ import annotations
import argparse
import shutil
import subprocess
from pathlib import Path
SRC_ROOT = Path(__file__).resolve().parent.parent
REPO_ROOT = SRC_ROOT.parent
ADAPTER_ROOT = SRC_ROOT / "platform_adapters"
COMMON_FILES = [
"AGENTS.md",
"GEMINI.md",
"CLAUDE.md",
"PLAN.md",
"docs/antigravity-clean-workspace.md",
"docs/platform-adapters.md",
"docs/platform-branch-strategy.md",
"scripts/dr.py",
"scripts/deploy_adapters.py",
"scripts/update_platform_envs.py",
"scripts/export_antigravity_workspace.py",
"pyproject.toml",
"requirements.txt",
"uv.lock",
]
COMMON_DIRS = [
"configs",
"scripts/runtime",
"scripts/reporting",
"skills",
]
def run_pull() -> None:
subprocess.run(["git", "pull", "--ff-only"], cwd=REPO_ROOT, check=True)
def reset_dir(path: Path) -> None:
if path.exists():
shutil.rmtree(path)
path.mkdir(parents=True, exist_ok=True)
def copy_file(rel: str, target_root: Path) -> None:
src = SRC_ROOT / rel
if not src.exists():
raise FileNotFoundError(src)
dst = target_root / rel
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dst)
def copy_dir(src: Path, dst: Path) -> None:
if not src.exists():
raise FileNotFoundError(src)
if dst.exists():
shutil.rmtree(dst)
shutil.copytree(src, dst, ignore=shutil.ignore_patterns("__pycache__", ".pytest_cache"))
def copy_common_workspace(target: Path) -> None:
for rel in COMMON_FILES:
copy_file(rel, target)
for rel in COMMON_DIRS:
copy_dir(SRC_ROOT / rel, target / rel)
(target / "projects").mkdir(parents=True, exist_ok=True)
(target / "projects" / ".gitkeep").touch(exist_ok=True)
def build_antigravity() -> Path:
target = REPO_ROOT / "antigravity"
reset_dir(target)
copy_common_workspace(target)
copy_dir(ADAPTER_ROOT / "antigravity" / "agent", target / ".agent")
copy_file("platform_adapters/antigravity/README.md", target)
return target
def build_codex() -> Path:
target = REPO_ROOT / "codex"
reset_dir(target)
copy_common_workspace(target)
copy_dir(ADAPTER_ROOT / "codex", target / ".codex")
copy_dir(ADAPTER_ROOT / "antigravity" / "agent" / "skills", target / ".codex" / "skills")
copy_file("platform_adapters/codex/README.md", target)
return target
def build_opencode() -> Path:
target = REPO_ROOT / "opencode"
reset_dir(target)
copy_common_workspace(target)
copy_dir(ADAPTER_ROOT / "opencode", target / ".opencode")
copy_file("platform_adapters/opencode/README.md", target)
return target
def build_claude_code() -> Path:
target = REPO_ROOT / "claude-code"
reset_dir(target)
copy_common_workspace(target)
copy_dir(ADAPTER_ROOT / "claude-code", target / ".claude")
copy_file("platform_adapters/claude-code/README.md", target)
return target
def build_gemini_cli() -> Path:
target = REPO_ROOT / "gemini-cli"
reset_dir(target)
copy_common_workspace(target)
copy_dir(ADAPTER_ROOT / "gemini-cli", target / ".gemini")
copy_file("platform_adapters/gemini-cli/README.md", target)
return target
BUILDERS = {
"antigravity": build_antigravity,
"codex": build_codex,
"opencode": build_opencode,
"claude-code": build_claude_code,
"gemini-cli": build_gemini_cli,
}
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Update isolated Deep Research platform environments")
parser.add_argument(
"--platform",
choices=["all", *BUILDERS.keys()],
default="all",
help="which platform env to rebuild",
)
parser.add_argument("--skip-pull", action="store_true", help="do not run git pull --ff-only first")
return parser
def main() -> int:
args = build_parser().parse_args()
if not args.skip_pull:
run_pull()
names = list(BUILDERS) if args.platform == "all" else [args.platform]
for name in names:
target = BUILDERS[name]()
print(f"rebuilt {name}: {target}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env python3
"""v0.20 Python-core regression checks."""
from __future__ import annotations
import json
import subprocess
import sys
import tempfile
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
def run(cmd: list[str]) -> None:
print("$ " + " ".join(cmd))
result = subprocess.run(cmd, cwd=REPO_ROOT, check=False, text=True, capture_output=True)
if result.stdout:
print(result.stdout.rstrip())
if result.stderr:
print(result.stderr.rstrip(), file=sys.stderr)
if result.returncode != 0:
raise SystemExit(result.returncode)
def make_fixture(root: Path) -> Path:
project = root / "v020-fixture"
(project / "phase4").mkdir(parents=True, exist_ok=True)
(project / "phase4" / "final_zh.md").write_text(
"# v0.20 回归测试报告\n\n正文引用占位。\n",
encoding="utf-8",
)
return project
def write_fixture_packet(project: Path) -> None:
cards = json.loads((project / "phase2" / "task_cards.json").read_text(encoding="utf-8"))
for index, card in enumerate(cards, start=1):
source_a = f"src_{index:03d}_a"
source_b = f"src_{index:03d}_b"
packet = {
"task_id": card["task_id"],
"claims": [{"claim": f"{card['task_id']} 回归证据支持章节主线", "source_ids": [source_a]}],
"evidence_items": [{"source_id": source_a, "summary": "权威来源支持该判断"}],
"counter_evidence": [{"claim": "证据仍需更多来源交叉验证", "source_ids": [source_b]}],
"source_ids": [source_a, source_b],
"source_quality_notes": [f"{source_a} Tier 1", f"{source_b} Tier 2"],
"open_questions": ["需要在真实项目中补充更多来源。"],
"raw_quotes_or_notes": ["English raw note can remain here."],
"sources": [
{"id": source_a, "title": f"Regression Source {index}A", "url": f"https://example.com/source-{index}-a", "tier": 1},
{"id": source_b, "title": f"Regression Source {index}B", "url": f"https://example.com/source-{index}-b", "tier": 2},
],
}
packet_path = project / card["output_packet"]
packet_path.write_text(json.dumps(packet, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def main() -> int:
python = sys.executable
run([python, "scripts/dr.py", "skills", "validate"])
run([python, "scripts/dr.py", "models", "--profile", "medium", "--json"])
with tempfile.TemporaryDirectory(prefix="deep-research-v020-") as tmp:
tmp_root = Path(tmp)
run(
[
python,
"scripts/dr.py",
"init",
"v0.20 fixture",
"--slug",
"v020-fixture",
"--projects-dir",
str(tmp_root),
"--method",
"mckinsey_market",
]
)
project = make_fixture(tmp_root)
run([python, "scripts/dr.py", "frame", str(project)])
assert (project / "phase1" / "research_brief.json").exists()
run([python, "scripts/dr.py", "approve", str(project)])
run([python, "scripts/dr.py", "research", str(project), "--workers", "2", "--axis", "literature", "--dry-run"])
run([python, "scripts/dr.py", "research", str(project), "--workers", "2", "--axis", "literature"])
write_fixture_packet(project)
run([python, "scripts/dr.py", "research", str(project), "--workers", "2", "--axis", "literature", "--build-briefs"])
assert (project / "phase2" / "compressed_findings" / "ch01.json").exists()
run([python, "scripts/dr.py", "review", str(project)])
run([python, "scripts/dr.py", "finalize", str(project), "--dry-run"])
print("v0.20 regression PASS")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env bash
# 验证 zenmux 两个端点 + prompt cache 是否可用
# 用法:bash scripts/verify-zenmux.sh
set -euo pipefail
# 加载密钥
if [[ -f secrets.env ]]; then
set -a; source secrets.env; set +a
fi
if [[ -z "${ZENMUX_API_KEY:-}" ]]; then
echo "❌ ZENMUX_API_KEY 未设置。请 cp secrets.env.example secrets.env 并填入 key"
exit 1
fi
echo "============================================"
echo " zenmux 端点与 prompt cache 自检"
echo "============================================"
# -------- 测试 1OpenAI 兼容端点(Gemini --------
echo ""
echo "[1/3] 测试 OpenAI 兼容端点(google/gemini-2.5-pro,隐式缓存)..."
RESP1=$(curl -sS -w "\n---HTTP:%{http_code}---\n" -X POST "https://zenmux.ai/api/v1/chat/completions" \
-H "Authorization: Bearer $ZENMUX_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"model": "google/gemini-2.5-pro",
"messages": [{"role": "user", "content": "用一句话回答:PubMed 是什么?"}],
"max_tokens": 100
}')
echo "$RESP1" | tail -5
echo ""
# -------- 测试 2Anthropic 兼容端点(Claude Haiku --------
echo "[2/3] 测试 Anthropic 兼容端点(claude-haiku-4.5..."
RESP2=$(curl -sS -w "\n---HTTP:%{http_code}---\n" -X POST "https://zenmux.ai/api/anthropic/v1/messages" \
-H "x-api-key: $ZENMUX_API_KEY" \
-H "anthropic-version: 2023-06-01" \
-H "Content-Type: application/json" \
-d '{
"model": "claude-haiku-4.5",
"max_tokens": 100,
"messages": [{"role": "user", "content": "用一句话回答:PubMed 是什么?"}]
}')
echo "$RESP2" | tail -5
echo ""
# -------- 测试 3Claude + prompt cacheSonnet --------
echo "[3/3] 测试 Claude prompt cache(连续 2 次请求,第 2 次应命中 cache..."
# 构造一个至少 1024 tokens 的 system prompt(约 3000 字符中文足够)
LONG_PROMPT=$(printf 'You are a biomedical research assistant. ' && \
for i in $(seq 1 80); do
printf 'This is test content line %d to build cacheable prefix context. ' "$i"
done)
PAYLOAD=$(cat <<EOF
{
"model": "claude-haiku-4.5",
"max_tokens": 50,
"system": [
{
"type": "text",
"text": "${LONG_PROMPT}",
"cache_control": {"type": "ephemeral"}
}
],
"messages": [{"role": "user", "content": "Say OK."}]
}
EOF
)
echo " 第 1 次调用(建 cache..."
R1=$(curl -sS -X POST "https://zenmux.ai/api/anthropic/v1/messages" \
-H "x-api-key: $ZENMUX_API_KEY" \
-H "anthropic-version: 2023-06-01" \
-H "Content-Type: application/json" \
-d "$PAYLOAD")
CACHE_CREATE=$(echo "$R1" | python3 -c "import sys, json; d=json.load(sys.stdin); print(d.get('usage',{}).get('cache_creation_input_tokens', 0))" 2>/dev/null || echo "?")
echo " cache_creation_input_tokens = $CACHE_CREATE"
sleep 2
echo " 第 2 次调用(应命中 cache..."
R2=$(curl -sS -X POST "https://zenmux.ai/api/anthropic/v1/messages" \
-H "x-api-key: $ZENMUX_API_KEY" \
-H "anthropic-version: 2023-06-01" \
-H "Content-Type: application/json" \
-d "$PAYLOAD")
CACHE_READ=$(echo "$R2" | python3 -c "import sys, json; d=json.load(sys.stdin); print(d.get('usage',{}).get('cache_read_input_tokens', 0))" 2>/dev/null || echo "?")
echo " cache_read_input_tokens = $CACHE_READ"
echo ""
echo "============================================"
echo " 结果"
echo "============================================"
if [[ "$CACHE_CREATE" != "0" && "$CACHE_CREATE" != "?" ]]; then
echo "✅ Cache 写入成功($CACHE_CREATE tokens"
else
echo "⚠️ Cache 未写入(可能是 prompt 太短或端点不支持)"
fi
if [[ "$CACHE_READ" != "0" && "$CACHE_READ" != "?" ]]; then
echo "✅ Cache 命中成功($CACHE_READ tokens)— 成本大幅降低"
else
echo "⚠️ Cache 未命中(检查 AGENTS.md §6.5 诊断步骤)"
fi
echo ""
echo "完整 usage 字段(第 2 次):"
echo "$R2" | python3 -m json.tool 2>/dev/null | grep -A 10 '"usage"' || echo "$R2"