v0.20.7 restructure source and platform workspaces
This commit is contained in:
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env bash
|
||||
# Activate Deep Research environment (venv + secrets)
|
||||
# Usage: cd <project-root> && source scripts/activate.sh
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
|
||||
echo "ERROR: use 'source', not 'bash':"
|
||||
echo " source scripts/activate.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Project root = directory containing pyproject.toml
|
||||
# Walk up from $PWD until we find it (or give up at /)
|
||||
_find_root() {
|
||||
local dir="${PWD}"
|
||||
while [[ "${dir}" != "/" ]]; do
|
||||
[[ -f "${dir}/pyproject.toml" ]] && echo "${dir}" && return 0
|
||||
dir="$(dirname "${dir}")"
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
_DR_ROOT="$(_find_root 2>/dev/null)"
|
||||
if [[ -z "${_DR_ROOT}" ]]; then
|
||||
echo "ERROR: pyproject.toml not found in ${PWD} or any parent."
|
||||
echo " cd to the project root first, then: source scripts/activate.sh"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# activate venv
|
||||
if [[ -f "${_DR_ROOT}/.venv/bin/activate" ]]; then
|
||||
source "${_DR_ROOT}/.venv/bin/activate"
|
||||
echo "venv: active ($(python --version 2>&1))"
|
||||
else
|
||||
echo "venv: not found - run: bash ${_DR_ROOT}/scripts/setup.sh"
|
||||
fi
|
||||
|
||||
# load secrets
|
||||
if [[ -f "${_DR_ROOT}/secrets.env" ]]; then
|
||||
set -a
|
||||
source "${_DR_ROOT}/secrets.env"
|
||||
set +a
|
||||
if [[ -n "${ZENMUX_API_KEY:-}" ]]; then
|
||||
echo "secrets: loaded (ZENMUX_API_KEY=${ZENMUX_API_KEY:0:12}...)"
|
||||
else
|
||||
echo "secrets: loaded but ZENMUX_API_KEY is empty"
|
||||
fi
|
||||
else
|
||||
echo "secrets: missing - run: cp secrets.env.example secrets.env"
|
||||
fi
|
||||
|
||||
# npm proxy fix: unset empty npm_config_proxy so npm inherits system http_proxy
|
||||
# without this, opencode crashes with "proxy.url must be a non-empty string"
|
||||
# when dynamically loading npm packages like @ai-sdk/anthropic
|
||||
if [[ -z "${npm_config_proxy:-}" ]]; then
|
||||
unset npm_config_proxy
|
||||
fi
|
||||
if [[ -z "${npm_config_https_proxy:-}" ]]; then
|
||||
unset npm_config_https_proxy
|
||||
fi
|
||||
|
||||
unset -f _find_root
|
||||
unset _DR_ROOT
|
||||
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python3
|
||||
"""根据 glossary 的核查结果,在 final_zh.md 上做精确的文本替换。
|
||||
|
||||
原理:
|
||||
- build_glossary.py 会给每条术语标 `zh`(正确中文)和 `issue`(有发现问题)
|
||||
- 本脚本扫描所有 `issue` 非空且 `confidence in {high, medium}` 的条目
|
||||
- 对这些条目,在正文中把「当前错误译名」替换为「正确译名」
|
||||
- 同时处理英文拼写错误(例如 Maywavee → Mabwell)
|
||||
|
||||
规则:
|
||||
- 安全第一:只做字面替换,不做上下文改写
|
||||
- 明确可见:每一处替换都打印出来,方便 diff
|
||||
- 幂等:多次跑结果一致
|
||||
- 支持 --dry-run 预览
|
||||
|
||||
用法:
|
||||
uv run python scripts/apply_glossary.py <project_slug>
|
||||
uv run python scripts/apply_glossary.py <project_slug> --dry-run
|
||||
uv run python scripts/apply_glossary.py <project_slug> --min-confidence high
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
|
||||
@dataclass
|
||||
class Correction:
|
||||
"""一条要应用的修正。"""
|
||||
|
||||
kind: str # "en" (英文拼写) / "zh" (中文译名)
|
||||
wrong: str # 当前文中的错误写法
|
||||
correct: str # 正确写法
|
||||
term_key: str # glossary 里这条的 key(人工 debug 用)
|
||||
confidence: str # high / medium / low
|
||||
reason: str # 为什么要改(从 issue 字段提取)
|
||||
|
||||
|
||||
_ISSUE_WRONG_EN_RE = re.compile(
|
||||
r"(?:'|\"|term\s*)([A-Z][A-Za-z0-9_\- ]{2,40})['\"]*\s*(?:为|是)[^。]*?"
|
||||
r"(?:拼写错误|拼写有误|应为|正确英文名为|正确为|正确拼写为)",
|
||||
)
|
||||
_CORRECT_EN_RE = re.compile(
|
||||
r"(?:正确英文名为|正确为|正确拼写为|应为|正确写法为)\s*['\"]?([A-Z][A-Za-z0-9_\- ]{2,40})['\"]?",
|
||||
)
|
||||
_WRONG_ZH_RE = re.compile(
|
||||
r"current_zh\s*['\"]?([\u4e00-\u9fff][^'\"。,;]{1,40})['\"]?\s*(?:为|是|存在)[^。]*?(?:错误|错译|误译|不准确|应为|应该)",
|
||||
)
|
||||
|
||||
|
||||
def resolve_project(arg: str) -> Path:
|
||||
p = Path(arg)
|
||||
if p.is_dir():
|
||||
return p
|
||||
cand = Path.cwd() / "projects" / arg
|
||||
if cand.is_dir():
|
||||
return cand
|
||||
raise SystemExit(f"project not found: {arg}")
|
||||
|
||||
|
||||
def load_glossary(path: Path) -> dict:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
# 中文替换的保守性:只对"公司名、机构名、产品名"类这种具体化的词做替换
|
||||
# 规避常见缩写词在不同上下文有不同含义的情况(如 PDE = 磷酸二酯酶 OR 允许日暴露量)
|
||||
_AMBIGUOUS_ABBREVS = {
|
||||
"PDE", "ASGPR", "BLA", "IND", "NDA", "CMC", "API", "QC", "QA",
|
||||
"ADC", "CRO", "CDMO", "CMO", "GMP", "PK", "PD", "TRL",
|
||||
"FDA", "EMA", "NMPA", "ICH", "WHO", "CFDA", "CDE",
|
||||
"LNP", "RNP", "AAV", "mRNA", "RNAi", "siRNA", "ASO",
|
||||
"DNA", "RNA", "cDNA", "dsRNA", "ssRNA",
|
||||
"OTP", "HCV", "HBV", "HPV", "HIV",
|
||||
}
|
||||
|
||||
|
||||
def _looks_like_proper_noun(term_key: str) -> bool:
|
||||
"""判断这条术语是否是"专有名词"(公司/机构/产品/药物名)。"""
|
||||
# 全大写 2-4 字符缩写 → 视作通用缩写,跳过(歧义风险高)
|
||||
if term_key.upper() == term_key and 2 <= len(term_key) <= 5:
|
||||
return False
|
||||
if term_key in _AMBIGUOUS_ABBREVS:
|
||||
return False
|
||||
# 其他情况:视作专有名词
|
||||
return True
|
||||
|
||||
|
||||
def extract_corrections(glossary: dict, min_conf: str) -> list[Correction]:
|
||||
"""从 glossary 抽出可应用的修正。
|
||||
|
||||
保守策略:
|
||||
- 英文拼写错误(term_key 本身错):高信度直接修
|
||||
- 中文译名错误(issue 里说 current_zh 错):仅对"专有名词"类(公司/产品/机构)修
|
||||
- 缩写/通用术语(PDE/ASGPR 等):不自动修,避免上下文歧义
|
||||
"""
|
||||
rank = {"high": 3, "medium": 2, "low": 1}
|
||||
threshold = rank.get(min_conf, 2)
|
||||
corrections: list[Correction] = []
|
||||
|
||||
for term_key, entry in glossary.items():
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
conf = entry.get("confidence", "low")
|
||||
if rank.get(conf, 0) < threshold:
|
||||
continue
|
||||
issue = entry.get("issue", "")
|
||||
zh = entry.get("zh", "")
|
||||
en_full = entry.get("en_full", "")
|
||||
if not issue:
|
||||
continue
|
||||
|
||||
# 1. 英文拼写错误:key 本身错了,en_full 是对的
|
||||
if (
|
||||
en_full
|
||||
and term_key != en_full
|
||||
and re.search(r"拼写错误|拼写有误|spelled|should be", issue, re.IGNORECASE)
|
||||
and term_key[0].isupper() # 通常是公司/产品名
|
||||
):
|
||||
# 排除太短的(容易误伤)或包含空格的原始 key
|
||||
if len(term_key) >= 4 and len(en_full) >= 4:
|
||||
corrections.append(
|
||||
Correction(
|
||||
kind="en",
|
||||
wrong=term_key,
|
||||
correct=en_full.split("(")[0].strip(), # 去掉括号内的法人全称
|
||||
term_key=term_key,
|
||||
confidence=conf,
|
||||
reason=issue[:200],
|
||||
)
|
||||
)
|
||||
|
||||
# 2. 中文译名错误:只对"专有名词"修(排除缩写/通用术语的歧义风险)
|
||||
if not _looks_like_proper_noun(term_key):
|
||||
continue
|
||||
m = _WRONG_ZH_RE.search(issue)
|
||||
if m and zh:
|
||||
wrong_zh = m.group(1).strip()
|
||||
if wrong_zh != zh and len(wrong_zh) >= 2:
|
||||
corrections.append(
|
||||
Correction(
|
||||
kind="zh",
|
||||
wrong=wrong_zh,
|
||||
correct=zh,
|
||||
term_key=term_key,
|
||||
confidence=conf,
|
||||
reason=issue[:200],
|
||||
)
|
||||
)
|
||||
|
||||
return corrections
|
||||
|
||||
|
||||
def apply_corrections(text: str, corrections: list[Correction]) -> tuple[str, list[tuple[Correction, int]]]:
|
||||
"""返回 (新文本, [(correction, 替换次数)])。"""
|
||||
results: list[tuple[Correction, int]] = []
|
||||
new_text = text
|
||||
for c in corrections:
|
||||
# 英文术语用 \b 边界;中文直接替换
|
||||
if c.kind == "en":
|
||||
pattern = r"\b" + re.escape(c.wrong) + r"\b"
|
||||
new_text, count = re.subn(pattern, c.correct, new_text)
|
||||
else:
|
||||
count = new_text.count(c.wrong)
|
||||
if count:
|
||||
new_text = new_text.replace(c.wrong, c.correct)
|
||||
results.append((c, count))
|
||||
return new_text, results
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="用 glossary 修正正文术语")
|
||||
parser.add_argument("project", help="项目 slug 或路径")
|
||||
parser.add_argument(
|
||||
"--input", default="phase4/final_zh.md",
|
||||
help="待修正的 Markdown",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output", default=None,
|
||||
help="输出路径(默认原地覆盖)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--glossary", default="phase4/glossary.json",
|
||||
help="glossary.json 路径",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--min-confidence",
|
||||
choices=["high", "medium", "low"],
|
||||
default="medium",
|
||||
help="只应用 >= 此置信度的修正(默认 medium)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run", action="store_true",
|
||||
help="只打印会发生的修改,不实际写入",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
project_root = resolve_project(args.project)
|
||||
input_path = project_root / args.input
|
||||
output_path = project_root / (args.output or args.input)
|
||||
glossary_path = project_root / args.glossary
|
||||
|
||||
if not input_path.exists():
|
||||
raise SystemExit(f"输入文件不存在:{input_path}")
|
||||
if not glossary_path.exists():
|
||||
raise SystemExit(f"glossary 不存在:{glossary_path}")
|
||||
|
||||
glossary = load_glossary(glossary_path)
|
||||
corrections = extract_corrections(glossary, args.min_confidence)
|
||||
|
||||
print(f"Project: {project_root.name}")
|
||||
print(f"Input: {input_path.relative_to(project_root)}")
|
||||
print(f"Output: {output_path.relative_to(project_root)}")
|
||||
print(f"Glossary: {glossary_path.relative_to(project_root)} ({len(glossary)} terms)")
|
||||
print(f"Min confidence: {args.min_confidence}")
|
||||
print(f"候选修正: {len(corrections)} 条")
|
||||
print()
|
||||
|
||||
if not corrections:
|
||||
print("没有需要应用的修正。")
|
||||
return 0
|
||||
|
||||
original = input_path.read_text(encoding="utf-8")
|
||||
new_text, results = apply_corrections(original, corrections)
|
||||
|
||||
applied = [(c, n) for c, n in results if n > 0]
|
||||
skipped = [(c, n) for c, n in results if n == 0]
|
||||
|
||||
print(f"=== 已应用 {len(applied)} 条修正(共替换 {sum(n for _, n in applied)} 处)===")
|
||||
for c, n in sorted(applied, key=lambda x: -x[1]):
|
||||
marker = "EN" if c.kind == "en" else "ZH"
|
||||
print(f" [{marker}][{c.confidence}] {c.wrong!r} → {c.correct!r} (×{n})")
|
||||
print(f" 原因: {c.reason[:120]}")
|
||||
|
||||
if skipped:
|
||||
print(f"\n=== 跳过 {len(skipped)} 条(正文未出现该错误写法)===")
|
||||
for c, _ in skipped[:10]:
|
||||
marker = "EN" if c.kind == "en" else "ZH"
|
||||
print(f" [{marker}] {c.wrong!r} → {c.correct!r} (0 hit)")
|
||||
if len(skipped) > 10:
|
||||
print(f" …还有 {len(skipped) - 10} 条")
|
||||
|
||||
if args.dry_run:
|
||||
print("\n(--dry-run:未写入)")
|
||||
return 0
|
||||
|
||||
output_path.write_text(new_text, encoding="utf-8")
|
||||
delta = len(new_text) - len(original)
|
||||
print(f"\n✓ 已写入 {output_path.relative_to(project_root)} (字符变化 {delta:+d})")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply a model profile to agent definition files.
|
||||
|
||||
Supports:
|
||||
- OpenCode YAML frontmatter agents in .opencode/agents/*.md
|
||||
- Codex TOML agents in codex_adapter_templates/codex/agents/*.toml
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.lib.model_config import (
|
||||
ModelConfigError,
|
||||
parse_model_overrides,
|
||||
resolve_model_profile,
|
||||
)
|
||||
|
||||
OPENCODE_ROLE_TO_FILE = {
|
||||
"dr_plan": ".opencode/agents/dr-plan.md",
|
||||
"dr_pm": ".opencode/agents/dr-pm.md",
|
||||
"dr_searcher": ".opencode/agents/dr-searcher.md",
|
||||
"dr_analyst": ".opencode/agents/dr-analyst.md",
|
||||
"dr_verifier": ".opencode/agents/dr-verifier.md",
|
||||
"dr_chief_editor": ".opencode/agents/dr-chief-editor.md",
|
||||
"dr_editor_in_chief": ".opencode/agents/dr-editor-in-chief.md",
|
||||
"dr_reporter": ".opencode/agents/dr-reporter.md",
|
||||
}
|
||||
|
||||
CODEX_ROLE_TO_FILE = {
|
||||
"dr_plan": "codex_adapter_templates/codex/agents/dr-plan.toml",
|
||||
"dr_pm": "codex_adapter_templates/codex/agents/dr-pm.toml",
|
||||
"dr_searcher": "codex_adapter_templates/codex/agents/dr-searcher.toml",
|
||||
"dr_analyst": "codex_adapter_templates/codex/agents/dr-analyst.toml",
|
||||
"dr_verifier": "codex_adapter_templates/codex/agents/dr-verifier.toml",
|
||||
"dr_chief_editor": "codex_adapter_templates/codex/agents/dr-chief-editor.toml",
|
||||
"dr_editor_in_chief": "codex_adapter_templates/codex/agents/dr-editor-in-chief.toml",
|
||||
"dr_reporter": "codex_adapter_templates/codex/agents/dr-reporter.toml",
|
||||
}
|
||||
|
||||
|
||||
def replace_opencode_model(path: Path, model: str) -> bool:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
new_text, count = re.subn(r"(?m)^model:\s*.+$", f"model: {model}", text, count=1)
|
||||
if count == 0:
|
||||
raise SystemExit(f"failed to locate model field: {path}")
|
||||
if new_text == text:
|
||||
return False
|
||||
path.write_text(new_text, encoding="utf-8")
|
||||
return True
|
||||
|
||||
|
||||
def replace_codex_model(path: Path, model: str) -> bool:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
new_text, count = re.subn(r'(?m)^model\s*=\s*"[^"]+"$', f'model = "{model}"', text, count=1)
|
||||
if count == 0:
|
||||
raise SystemExit(f"failed to locate model field: {path}")
|
||||
if new_text == text:
|
||||
return False
|
||||
path.write_text(new_text, encoding="utf-8")
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Apply model profile to agent files")
|
||||
parser.add_argument("--profile", required=True, help="Profile name from configs/models.yaml")
|
||||
parser.add_argument("--target", choices=["opencode", "codex", "both"], default="both")
|
||||
parser.add_argument(
|
||||
"--model-override",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="ROLE=MODEL",
|
||||
help="Override one role model, repeatable",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
resolved = resolve_model_profile(
|
||||
profile=args.profile,
|
||||
overrides=parse_model_overrides(args.model_override),
|
||||
)
|
||||
except ModelConfigError as exc:
|
||||
raise SystemExit(f"model profile resolution failed: {exc}") from exc
|
||||
|
||||
roles = resolved["roles"]
|
||||
changed: list[str] = []
|
||||
|
||||
def apply_map(mapping: dict[str, str], mode: str) -> None:
|
||||
for role, rel_path in mapping.items():
|
||||
model = roles.get(role)
|
||||
if not model:
|
||||
continue
|
||||
file_path = REPO_ROOT / rel_path
|
||||
if not file_path.exists():
|
||||
continue
|
||||
if args.dry_run:
|
||||
changed.append(f"{mode}:{rel_path} -> {model}")
|
||||
continue
|
||||
did_change = replace_opencode_model(file_path, model) if mode == "opencode" else replace_codex_model(file_path, model)
|
||||
if did_change:
|
||||
changed.append(f"{mode}:{rel_path} -> {model}")
|
||||
|
||||
if args.target in ("opencode", "both"):
|
||||
apply_map(OPENCODE_ROLE_TO_FILE, "opencode")
|
||||
if args.target in ("codex", "both"):
|
||||
apply_map(CODEX_ROLE_TO_FILE, "codex")
|
||||
|
||||
print(f"Profile applied: {resolved['profile']}")
|
||||
print(f"Target: {args.target}")
|
||||
if changed:
|
||||
print("Updated:")
|
||||
for item in changed:
|
||||
print(f" - {item}")
|
||||
else:
|
||||
print("No files changed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,332 @@
|
||||
#!/usr/bin/env python3
|
||||
"""双语术语表事实核查脚本。
|
||||
|
||||
输入:
|
||||
- <project>/phase4/glossary.json(来自 translate.py 累积的初版术语表)
|
||||
- 可选:--extra terms.txt(每行一个英文术语,补充进来一起核查)
|
||||
|
||||
流程(每个术语独立可并行):
|
||||
1. 用 SearchClient(Tavily > Exa > Brave)搜一次(query = "<term> <domain hint>")
|
||||
2. 把 top 3-5 snippet 喂给 Haiku,让模型返回 {zh, en_full, confidence, issue}
|
||||
3. 合并回 glossary,字段扩展:
|
||||
{
|
||||
"Mabwell": {
|
||||
"zh": "迈威生物",
|
||||
"en_full": "Mabwell (Shanghai) Bioscience Co., Ltd.",
|
||||
"confidence": "high",
|
||||
"issue": "...",
|
||||
"verified_at": "2026-04-22",
|
||||
"sources": ["https://mabwell.com/", ...]
|
||||
}
|
||||
}
|
||||
|
||||
用法:
|
||||
uv run python scripts/build_glossary.py <project_slug>
|
||||
uv run python scripts/build_glossary.py <project_slug> --workers 4
|
||||
uv run python scripts/build_glossary.py <project_slug> --only "Mabwell,Maywavee"
|
||||
uv run python scripts/build_glossary.py <project_slug> --force
|
||||
|
||||
断点续传:已核查过的条目(有 verified_at 字段)默认跳过;--force 全部重跑。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import datetime as dt
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from scripts.lib.search_client import SearchClient, SearchError
|
||||
from scripts.lib.zenmux_client import ZenMuxClient, ZenMuxError, load_secrets
|
||||
|
||||
DEFAULT_MODEL = "anthropic/claude-haiku-4.5"
|
||||
PROMPT_FILE = Path(__file__).parent / "prompts" / "glossary_system.txt"
|
||||
|
||||
|
||||
def resolve_project(arg: str) -> Path:
|
||||
p = Path(arg)
|
||||
if p.is_dir():
|
||||
return p
|
||||
cand = Path.cwd() / "projects" / arg
|
||||
if cand.is_dir():
|
||||
return cand
|
||||
raise SystemExit(f"project not found: {arg}")
|
||||
|
||||
|
||||
def load_glossary(path: Path) -> dict:
|
||||
if not path.exists():
|
||||
return {}
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def save_glossary(path: Path, glossary: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(glossary, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def migrate_legacy_entry(value) -> dict:
|
||||
"""旧版 glossary 里 value 是字符串;升级为对象格式。"""
|
||||
if isinstance(value, str):
|
||||
return {"zh": value}
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
return {"zh": str(value)}
|
||||
|
||||
|
||||
def build_query(term: str, domain: str) -> str:
|
||||
if domain:
|
||||
return f"{term} {domain} 中文名"
|
||||
return term
|
||||
|
||||
|
||||
def parse_json_line(text: str) -> dict:
|
||||
"""模型返回的单行 JSON。容忍前后额外字符。"""
|
||||
s = text.strip()
|
||||
l = s.find("{")
|
||||
r = s.rfind("}")
|
||||
if l == -1 or r == -1:
|
||||
raise ValueError(f"no JSON object: {text[:200]}")
|
||||
obj = json.loads(s[l : r + 1])
|
||||
if not isinstance(obj, dict):
|
||||
raise ValueError("top-level JSON not object")
|
||||
return obj
|
||||
|
||||
|
||||
def build_user_prompt(term: str, domain: str, current_zh: str, hits: list) -> str:
|
||||
hits_text = "\n\n".join(
|
||||
f"[{i+1}] {h.title}\n URL: {h.url}\n {h.snippet[:500]}"
|
||||
for i, h in enumerate(hits[:5])
|
||||
)
|
||||
if not hits_text:
|
||||
hits_text = "(无搜索结果)"
|
||||
return (
|
||||
f"term: {term}\n"
|
||||
f"domain: {domain or '(未指定)'}\n"
|
||||
f"current_zh: {current_zh or '(空)'}\n\n"
|
||||
f"search_hits:\n{hits_text}\n"
|
||||
)
|
||||
|
||||
|
||||
def verify_term(
|
||||
term: str,
|
||||
current_zh: str,
|
||||
domain: str,
|
||||
search_client: SearchClient,
|
||||
llm_client: ZenMuxClient,
|
||||
*,
|
||||
model: str,
|
||||
system_prompt: str,
|
||||
) -> dict:
|
||||
try:
|
||||
hits = search_client.search(
|
||||
build_query(term, domain),
|
||||
num_results=4,
|
||||
)
|
||||
except SearchError as e:
|
||||
return {
|
||||
"zh": current_zh or "",
|
||||
"en_full": term,
|
||||
"confidence": "low",
|
||||
"issue": f"搜索失败:{e}",
|
||||
"sources": [],
|
||||
}
|
||||
|
||||
user = build_user_prompt(term, domain, current_zh, hits)
|
||||
try:
|
||||
raw = llm_client.chat_complete(
|
||||
model=model,
|
||||
system=system_prompt,
|
||||
user=user,
|
||||
temperature=0.1,
|
||||
max_tokens=800,
|
||||
tag=f"glossary:{term[:30]}",
|
||||
)
|
||||
obj = parse_json_line(raw)
|
||||
except (ZenMuxError, ValueError) as e:
|
||||
return {
|
||||
"zh": current_zh or "",
|
||||
"en_full": term,
|
||||
"confidence": "low",
|
||||
"issue": f"LLM 核查失败:{e}",
|
||||
"sources": [h.url for h in hits[:3]],
|
||||
}
|
||||
|
||||
# 规范化输出
|
||||
return {
|
||||
"zh": str(obj.get("zh", current_zh or "")),
|
||||
"en_full": str(obj.get("en_full", term)),
|
||||
"confidence": str(obj.get("confidence", "low")),
|
||||
"issue": str(obj.get("issue", "")),
|
||||
"sources": [h.url for h in hits[:3]],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="双语术语表事实核查(Haiku + Exa)")
|
||||
parser.add_argument("project", help="项目 slug 或完整路径")
|
||||
parser.add_argument(
|
||||
"--input",
|
||||
default="phase4/glossary.json",
|
||||
help="初版术语表路径(相对项目根)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default="phase4/glossary.json",
|
||||
help="输出路径(默认原地覆盖)",
|
||||
)
|
||||
parser.add_argument("--model", default=DEFAULT_MODEL, help="LLM 模型")
|
||||
parser.add_argument(
|
||||
"--workers", type=int, default=4, help="并发度(默认 4;网络不稳时建议降到 3)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force", action="store_true",
|
||||
help="忽略已核查状态,全部重跑",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--only",
|
||||
default=None,
|
||||
help="只核查指定术语(逗号分隔,大小写敏感)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--extra",
|
||||
default=None,
|
||||
help="每行一个英文术语的文本文件,补充进术语表一起核查",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--domain",
|
||||
default=None,
|
||||
help="术语领域提示(默认自动读 manifest.topic)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
load_secrets()
|
||||
project_root = resolve_project(args.project)
|
||||
input_path = project_root / args.input
|
||||
output_path = project_root / args.output
|
||||
|
||||
manifest_path = project_root / "manifest.json"
|
||||
domain = args.domain
|
||||
if not domain and manifest_path.exists():
|
||||
m = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
domain = m.get("topic") or m.get("report_title") or ""
|
||||
|
||||
glossary = load_glossary(input_path)
|
||||
# migration
|
||||
for k, v in list(glossary.items()):
|
||||
glossary[k] = migrate_legacy_entry(v)
|
||||
|
||||
if args.extra:
|
||||
extra_path = Path(args.extra)
|
||||
if not extra_path.exists():
|
||||
raise SystemExit(f"--extra 文件不存在:{extra_path}")
|
||||
for line in extra_path.read_text(encoding="utf-8").splitlines():
|
||||
term = line.strip()
|
||||
if term and term not in glossary:
|
||||
glossary[term] = {"zh": ""}
|
||||
|
||||
only_terms: set[str] | None = None
|
||||
if args.only:
|
||||
only_terms = {t.strip() for t in args.only.split(",") if t.strip()}
|
||||
|
||||
system_prompt = PROMPT_FILE.read_text(encoding="utf-8")
|
||||
|
||||
# 筛选需要核查的
|
||||
todo: list[str] = []
|
||||
for term, entry in glossary.items():
|
||||
if only_terms is not None and term not in only_terms:
|
||||
continue
|
||||
if not args.force and entry.get("verified_at"):
|
||||
continue
|
||||
todo.append(term)
|
||||
|
||||
print(f"Project: {project_root.name}")
|
||||
print(f"Domain hint: {domain or '(none)'}")
|
||||
print(f"Glossary size: {len(glossary)} | to verify: {len(todo)} | workers: {args.workers}")
|
||||
print(f"Model: {args.model}")
|
||||
if not todo:
|
||||
print(" 没有需要核查的条目(使用 --force 强制重跑)")
|
||||
save_glossary(output_path, glossary)
|
||||
return 0
|
||||
|
||||
today = dt.date.today().isoformat()
|
||||
logs_dir = project_root / "phase4" / "logs"
|
||||
log_file = logs_dir / "glossary.jsonl"
|
||||
|
||||
start = time.time()
|
||||
done_count = 0
|
||||
failed: list[str] = []
|
||||
|
||||
# 并发执行:SearchClient/ZenMuxClient 都是 thread-safe(httpx.Client 支持)
|
||||
with SearchClient() as search_client, ZenMuxClient(log_file=log_file) as llm_client:
|
||||
|
||||
def worker(term: str) -> tuple[str, dict]:
|
||||
current_zh = glossary.get(term, {}).get("zh", "")
|
||||
result = verify_term(
|
||||
term,
|
||||
current_zh,
|
||||
domain or "",
|
||||
search_client,
|
||||
llm_client,
|
||||
model=args.model,
|
||||
system_prompt=system_prompt,
|
||||
)
|
||||
return term, result
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool:
|
||||
futures = {pool.submit(worker, term): term for term in todo}
|
||||
for fut in concurrent.futures.as_completed(futures):
|
||||
term = futures[fut]
|
||||
try:
|
||||
_term, result = fut.result()
|
||||
except Exception as e:
|
||||
print(f" [FAIL] {term}: {e}")
|
||||
failed.append(term)
|
||||
continue
|
||||
# 合并到 glossary
|
||||
old = glossary.get(term, {})
|
||||
old.update(result)
|
||||
old["verified_at"] = today
|
||||
glossary[term] = old
|
||||
done_count += 1
|
||||
conf = result.get("confidence", "?")
|
||||
issue = result.get("issue", "")
|
||||
zh = result.get("zh") or "(保留英文)"
|
||||
marker = {"high": "✓", "medium": "~", "low": "?"}.get(conf, " ")
|
||||
issue_str = f" ⚠ {issue[:80]}" if issue else ""
|
||||
print(f" [{marker}] {term:<35} → {zh}{issue_str}")
|
||||
# 阶段性存盘,避免中途挂掉丢数据
|
||||
if done_count % 10 == 0:
|
||||
save_glossary(output_path, glossary)
|
||||
|
||||
save_glossary(output_path, glossary)
|
||||
|
||||
elapsed = time.time() - start
|
||||
print(f"\n完成:{done_count}/{len(todo)}(失败 {len(failed)},耗时 {elapsed:.1f}s)")
|
||||
print(f"术语表:{output_path.relative_to(project_root)}")
|
||||
print(llm_client.usage.summary())
|
||||
|
||||
# 高警示项汇总
|
||||
issues = [
|
||||
(k, v) for k, v in glossary.items()
|
||||
if v.get("issue") and v.get("confidence") != "high"
|
||||
]
|
||||
if issues:
|
||||
print(f"\n⚠ 低置信度或带问题的条目({len(issues)} 条):")
|
||||
for k, v in issues[:15]:
|
||||
print(f" - {k:<30} [{v.get('confidence','?')}] {v.get('issue','')[:100]}")
|
||||
if len(issues) > 15:
|
||||
print(f" … 还有 {len(issues) - 15} 条,见 {output_path.name}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,477 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4 成稿阶段:统一入口。
|
||||
|
||||
从 final_zh_polished.md(或指定的 Markdown)+ manifest.json 生成:
|
||||
- <title>.pdf PDF(ReportLab 或 Quarto/xelatex)
|
||||
- <title>.docx Pandoc 出 DOCX
|
||||
- <title>-en.pdf 如果存在 final_en.md 也一并出英文版(可选)
|
||||
|
||||
文件名来自 manifest.report_title(去掉非法字符),不再用 "final.pdf" 这种通用名。
|
||||
|
||||
用法:
|
||||
uv run python scripts/build_report.py <project_slug>
|
||||
|
||||
# 使用 Quarto/xelatex 引擎(推荐,更好的中文+宽表支持):
|
||||
uv run python scripts/build_report.py <project_slug> --engine quarto
|
||||
|
||||
# 只生成 PDF:
|
||||
uv run python scripts/build_report.py <project_slug> --no-docx
|
||||
|
||||
# 从自定义 md 生成:
|
||||
uv run python scripts/build_report.py <project_slug> --input phase4/final_zh.md
|
||||
|
||||
环境依赖:
|
||||
- reportlab, pypandoc, 思源字体(bash .opencode/templates/fonts/download-fonts.sh)
|
||||
- pandoc 可执行文件在 PATH
|
||||
- Quarto(可选,--engine quarto 时需要):https://quarto.org/docs/get-started/
|
||||
安装后运行:quarto install tinytex
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from scripts.lib.zenmux_client import load_secrets # noqa: F401 (为一致性)
|
||||
from scripts.reporting.fonts import resolve_quarto_fonts
|
||||
from scripts.reporting.references import build_references_block
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
PDF_TEMPLATE = REPO_ROOT / ".opencode" / "templates" / "report-template.py"
|
||||
DEFAULT_FONTS_DIR = REPO_ROOT / ".opencode" / "templates" / "fonts"
|
||||
|
||||
|
||||
def resolve_project(arg: str) -> Path:
|
||||
p = Path(arg)
|
||||
if p.is_dir():
|
||||
return p.resolve()
|
||||
cand = REPO_ROOT / "projects" / arg
|
||||
if cand.is_dir():
|
||||
return cand.resolve()
|
||||
raise SystemExit(f"project not found: {arg}")
|
||||
|
||||
|
||||
# 文件系统对文件名的常见限制:Windows 更严格,按最小公倍数来
|
||||
_FS_ILLEGAL_RE = re.compile(r'[\\/:*?"<>|\r\n\t]+')
|
||||
_WHITESPACE_RE = re.compile(r"\s+")
|
||||
|
||||
|
||||
def sanitize_filename(name: str, max_len: int = 120) -> str:
|
||||
"""把报告标题变成可跨 OS 使用的文件名。"""
|
||||
if not name:
|
||||
return "report"
|
||||
# 去掉非法字符
|
||||
cleaned = _FS_ILLEGAL_RE.sub(" ", name)
|
||||
# 合并空白
|
||||
cleaned = _WHITESPACE_RE.sub(" ", cleaned).strip()
|
||||
# 首尾 . 空格 . (Windows 要求)
|
||||
cleaned = cleaned.strip(". ").strip()
|
||||
if len(cleaned) > max_len:
|
||||
cleaned = cleaned[:max_len].rstrip()
|
||||
return cleaned or "report"
|
||||
|
||||
|
||||
def load_manifest(path: Path) -> dict:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def determine_input_md(project_root: Path, preferred: str | None) -> Path:
|
||||
"""决定用哪个 Markdown 出稿。
|
||||
|
||||
优先级:--input 指定 > final_zh_polished.md > final_zh.md > final_en.md
|
||||
"""
|
||||
if preferred:
|
||||
p = project_root / preferred
|
||||
if not p.exists():
|
||||
raise SystemExit(f"指定的 --input 不存在:{p}")
|
||||
return p
|
||||
for candidate in (
|
||||
"phase4/final_zh_polished.md",
|
||||
"phase4/final_zh.md",
|
||||
"phase4/final_en.md",
|
||||
):
|
||||
p = project_root / candidate
|
||||
if p.exists():
|
||||
return p
|
||||
raise SystemExit(
|
||||
f"找不到任何 Markdown 源。项目根:{project_root}\n"
|
||||
"先跑 translate.py / polish.py 或使用 --input 指定路径。"
|
||||
)
|
||||
|
||||
|
||||
def build_pdf(
|
||||
md_path: Path,
|
||||
manifest_path: Path,
|
||||
output_pdf: Path,
|
||||
fonts_dir: Path,
|
||||
sources_path: Path | None,
|
||||
) -> None:
|
||||
"""调用 report-template.py 生成 PDF。"""
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(PDF_TEMPLATE),
|
||||
"--input", str(md_path),
|
||||
"--manifest", str(manifest_path),
|
||||
"--output", str(output_pdf),
|
||||
"--fonts-dir", str(fonts_dir),
|
||||
]
|
||||
if sources_path and sources_path.exists():
|
||||
cmd += ["--sources", str(sources_path)]
|
||||
|
||||
print(f"\n→ 生成 PDF:{output_pdf.name}")
|
||||
result = subprocess.run(cmd, check=False)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"PDF 生成失败,返回码 {result.returncode}")
|
||||
|
||||
|
||||
def _detect_wide_tables(md_text: str, min_cols: int = 8) -> list[tuple[int, int]]:
|
||||
"""返回所有列数 >= min_cols 的 Markdown 表格的 (start_line, end_line) 区间(0-based)。"""
|
||||
lines = md_text.split("\n")
|
||||
ranges = []
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
if line.startswith("|") and line.count("|") - 1 >= min_cols:
|
||||
# Possible table header — next line should be separator
|
||||
if i + 1 < len(lines) and re.match(r"^\|[\s\-:|]+\|", lines[i + 1]):
|
||||
start = i
|
||||
j = i + 2
|
||||
while j < len(lines) and lines[j].strip().startswith("|"):
|
||||
j += 1
|
||||
ranges.append((start, j))
|
||||
i = j
|
||||
continue
|
||||
i += 1
|
||||
return ranges
|
||||
|
||||
|
||||
def prepare_qmd(
|
||||
md_path: Path,
|
||||
manifest: dict,
|
||||
output_qmd: Path,
|
||||
fonts_dir: Path,
|
||||
sources_path: Path | None,
|
||||
wide_table_cols: int = 8,
|
||||
) -> None:
|
||||
"""将普通 Markdown 转换为带 Quarto front matter 的 .qmd 文件。
|
||||
|
||||
主要处理:
|
||||
1. 插入 YAML front matter(标题、字体、页面设置等)
|
||||
2. 用 {.landscape} div 包裹列数 >= wide_table_cols 的宽表
|
||||
3. 将 [TOC will be generated...] 占位符替换为真实 TOC 指令
|
||||
4. 将 [REFERENCES will be filled...] 占位符替换为参考文献内容
|
||||
"""
|
||||
title = manifest.get("report_title", "报告")
|
||||
subtitle = manifest.get("report_subtitle", "")
|
||||
date = manifest.get("date", "")
|
||||
|
||||
# 决定字体名称:Quarto/xelatex 使用系统字体 family name。
|
||||
fonts = resolve_quarto_fonts(fonts_dir)
|
||||
main_font = fonts.main_font
|
||||
sans_font = fonts.sans_font
|
||||
|
||||
# Write LaTeX header file for CJK font setup.
|
||||
# Using a separate .tex file avoids YAML escape issues with backslashes.
|
||||
mf = main_font # "Source Han Serif CN"
|
||||
sf = sans_font # "Source Han Sans CN"
|
||||
|
||||
front_matter = textwrap.dedent(f"""\
|
||||
---
|
||||
title: "{title}"
|
||||
subtitle: "{subtitle}"
|
||||
date: "{date}"
|
||||
lang: zh
|
||||
format:
|
||||
pdf:
|
||||
pdf-engine: xelatex
|
||||
CJKmainfont: "{mf}"
|
||||
mainfont: "{mf}"
|
||||
mainfontoptions:
|
||||
- BoldFont={mf}
|
||||
- ItalicFont={mf}
|
||||
- BoldItalicFont={mf}
|
||||
CJKoptions:
|
||||
- BoldFont={mf}
|
||||
- ItalicFont={mf}
|
||||
- BoldItalicFont={mf}
|
||||
sansfont: "{sf}"
|
||||
sansfontoptions:
|
||||
- BoldFont={sf}
|
||||
- ItalicFont={sf}
|
||||
- BoldItalicFont={sf}
|
||||
monofont: "Liberation Mono"
|
||||
papersize: a4
|
||||
documentclass: scrartcl
|
||||
classoption:
|
||||
- DIV=11
|
||||
- headinclude
|
||||
toc: true
|
||||
toc-depth: 2
|
||||
toc-title: "目录"
|
||||
number-sections: false
|
||||
colorlinks: true
|
||||
linkcolor: NavyBlue
|
||||
urlcolor: NavyBlue
|
||||
geometry:
|
||||
- top=25mm
|
||||
- bottom=25mm
|
||||
- left=25mm
|
||||
- right=20mm
|
||||
pdf-engine-opts:
|
||||
- "-stack-size=32768"
|
||||
- "-extra-mem-top=2000000"
|
||||
include-in-header:
|
||||
- file: _preamble.tex
|
||||
---
|
||||
|
||||
""")
|
||||
|
||||
md_text = md_path.read_text(encoding="utf-8")
|
||||
|
||||
# Remove existing YAML front matter if any (between first two ---)
|
||||
if md_text.startswith("---"):
|
||||
end = md_text.find("\n---", 3)
|
||||
if end != -1:
|
||||
md_text = md_text[end + 4:].lstrip("\n")
|
||||
|
||||
# Quarto already renders the title from YAML; drop a duplicated leading H1.
|
||||
md_text = re.sub(
|
||||
rf"^#\s+{re.escape(title)}\s*\n+",
|
||||
"",
|
||||
md_text,
|
||||
count=1,
|
||||
)
|
||||
|
||||
# Replace TOC placeholder
|
||||
md_text = re.sub(
|
||||
r"\[TOC will be generated.*?\]",
|
||||
"", # Quarto handles TOC via front matter
|
||||
md_text,
|
||||
)
|
||||
|
||||
# Replace REFERENCES placeholder with actual references from sources.jsonl
|
||||
ref_block = build_references_block(sources_path, md_text)
|
||||
md_text = re.sub(
|
||||
r"\[REFERENCES will be filled.*?\]",
|
||||
ref_block,
|
||||
md_text,
|
||||
)
|
||||
|
||||
# Wrap wide tables in {.landscape} divs
|
||||
lines = md_text.split("\n")
|
||||
wide_ranges = _detect_wide_tables(md_text, min_cols=wide_table_cols)
|
||||
|
||||
if wide_ranges:
|
||||
# Insert landscape wrappers from bottom up (so line numbers stay valid)
|
||||
for start, end in reversed(wide_ranges):
|
||||
lines.insert(end, "\n:::")
|
||||
lines.insert(start, "::: {.landscape}\n")
|
||||
|
||||
md_text = "\n".join(lines)
|
||||
|
||||
# Write LaTeX preamble file (table + landscape support)
|
||||
preamble_tex = output_qmd.parent / "_preamble.tex"
|
||||
preamble_tex.write_text(
|
||||
"\\usepackage{longtable}\n"
|
||||
"\\usepackage{booktabs}\n"
|
||||
"\\usepackage{array}\n"
|
||||
"\\usepackage{xcolor}\n"
|
||||
"\\usepackage{titlesec}\n"
|
||||
"\\definecolor{DRBlue}{HTML}{1E3A8A}\n"
|
||||
"\\definecolor{DRSlate}{HTML}{374151}\n"
|
||||
"\\definecolor{DRMuted}{HTML}{6B7280}\n"
|
||||
# Use lscape instead of pdflscape to avoid \LS@makefcolumn recursion
|
||||
# which exhausts TeX param_size on large longtables.
|
||||
# lscape rotates content without changing page media box (reader must rotate).
|
||||
"\\usepackage{lscape}\n"
|
||||
"\\setlength{\\LTpre}{6pt}\n"
|
||||
"\\setlength{\\LTpost}{6pt}\n"
|
||||
"\\setlength{\\tabcolsep}{3pt}\n"
|
||||
"\\linespread{1.18}\n"
|
||||
"\\setlength{\\parindent}{2em}\n"
|
||||
"\\setlength{\\parskip}{0.25em}\n"
|
||||
"\\newcommand{\\sectionbreak}{\\clearpage}\n"
|
||||
"\\titleformat{\\section}[display]\n"
|
||||
" {\\centering\\Large\\bfseries\\sffamily\\color{DRBlue}}\n"
|
||||
" {}{0pt}{}\n"
|
||||
"\\titlespacing*{\\section}{0pt}{0pt}{1.1em}\n"
|
||||
"\\titleformat{\\subsection}\n"
|
||||
" {\\large\\bfseries\\sffamily\\color{DRBlue}}\n"
|
||||
" {}{0pt}{}\n"
|
||||
"\\titlespacing*{\\subsection}{0pt}{1.1em}{0.45em}\n"
|
||||
"\\titleformat{\\subsubsection}\n"
|
||||
" {\\normalsize\\bfseries\\sffamily\\color{DRSlate}}\n"
|
||||
" {}{0pt}{}\n"
|
||||
"\\titlespacing*{\\subsubsection}{0pt}{0.9em}{0.35em}\n"
|
||||
"\\renewcommand{\\contentsname}{目录}\n"
|
||||
"\\setcounter{tocdepth}{2}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
output_qmd.write_text(front_matter + md_text, encoding="utf-8")
|
||||
print(f" .qmd prepared: {output_qmd.name} ({len(wide_ranges)} landscape table(s))")
|
||||
|
||||
def build_pdf_quarto(
|
||||
md_path: Path,
|
||||
manifest: dict,
|
||||
output_pdf: Path,
|
||||
fonts_dir: Path,
|
||||
sources_path: Path | None,
|
||||
) -> None:
|
||||
"""使用 Quarto + xelatex 生成 PDF。"""
|
||||
if not shutil.which("quarto"):
|
||||
raise SystemExit(
|
||||
"quarto 命令未找到。请先安装 Quarto:https://quarto.org/docs/get-started/\n"
|
||||
"安装后运行:quarto install tinytex"
|
||||
)
|
||||
|
||||
# Prepare .qmd in the same dir as output_pdf
|
||||
qmd_path = output_pdf.parent / (output_pdf.stem + ".qmd")
|
||||
prepare_qmd(md_path, manifest, qmd_path, fonts_dir, sources_path)
|
||||
|
||||
print(f"\n→ 生成 PDF(Quarto/xelatex):{output_pdf.name}")
|
||||
cmd = [
|
||||
"quarto", "render", str(qmd_path),
|
||||
"--to", "pdf",
|
||||
"--output", output_pdf.name,
|
||||
]
|
||||
result = subprocess.run(cmd, cwd=str(output_pdf.parent), check=False)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"Quarto PDF 生成失败,返回码 {result.returncode}")
|
||||
|
||||
# Clean up auxiliary files Quarto leaves behind
|
||||
for ext in (".tex", ".log", ".aux", ".toc", ".out", "-files"):
|
||||
candidate = output_pdf.parent / (output_pdf.stem + ext)
|
||||
if candidate.exists():
|
||||
candidate.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def build_docx(md_path: Path, output_docx: Path, title: str) -> None:
|
||||
"""用 pandoc 生成 DOCX。"""
|
||||
if not shutil.which("pandoc"):
|
||||
print(f" ⚠ pandoc 不在 PATH,跳过 DOCX 生成", file=sys.stderr)
|
||||
return
|
||||
|
||||
print(f"\n→ 生成 DOCX:{output_docx.name}")
|
||||
# 关闭 tex_math_dollars/tex_math_single_backslash 防止文中 "$100" "$10^6" 被当数学公式
|
||||
cmd = [
|
||||
"pandoc",
|
||||
str(md_path),
|
||||
"-o", str(output_docx),
|
||||
"--from=markdown-tex_math_dollars-tex_math_single_backslash-raw_tex",
|
||||
"--to=docx",
|
||||
"--standalone",
|
||||
"-M", f"title={title}",
|
||||
"--wrap=preserve",
|
||||
]
|
||||
# reference-doc 如果存在就用
|
||||
ref_doc = REPO_ROOT / ".opencode" / "templates" / "reference.docx"
|
||||
if ref_doc.exists():
|
||||
cmd += ["--reference-doc", str(ref_doc)]
|
||||
|
||||
result = subprocess.run(cmd, check=False)
|
||||
if result.returncode != 0:
|
||||
print(f" ⚠ DOCX 生成失败(返回码 {result.returncode}),但不阻断流程", file=sys.stderr)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Phase 4 成稿(PDF + DOCX)")
|
||||
parser.add_argument("project", help="项目 slug 或完整路径")
|
||||
parser.add_argument(
|
||||
"--input",
|
||||
default=None,
|
||||
help="Markdown 源(默认自动寻找 phase4/final_zh_polished.md)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default="phase4",
|
||||
help="输出目录(相对项目根,默认 phase4)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fonts-dir",
|
||||
default=str(DEFAULT_FONTS_DIR),
|
||||
help="字体目录",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sources",
|
||||
default=None,
|
||||
help="sources.jsonl 路径(默认 phase2/sources.jsonl)",
|
||||
)
|
||||
parser.add_argument("--no-docx", action="store_true", help="跳过 DOCX 生成")
|
||||
parser.add_argument("--no-pdf", action="store_true", help="跳过 PDF 生成")
|
||||
parser.add_argument(
|
||||
"--engine",
|
||||
choices=["reportlab", "quarto"],
|
||||
default="reportlab",
|
||||
help="PDF 渲染引擎:reportlab(默认,Python 原生)或 quarto(xelatex,更好的中文+宽表支持)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--basename",
|
||||
default=None,
|
||||
help="文件名 stem(不带扩展名),默认从 manifest.report_title 生成",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
project_root = resolve_project(args.project)
|
||||
manifest_path = project_root / "manifest.json"
|
||||
if not manifest_path.exists():
|
||||
raise SystemExit(f"manifest.json not found: {manifest_path}")
|
||||
|
||||
manifest = load_manifest(manifest_path)
|
||||
md_path = determine_input_md(project_root, args.input)
|
||||
|
||||
title = manifest.get("report_title") or manifest.get("topic") or "Deep Research Report"
|
||||
basename = args.basename or sanitize_filename(title)
|
||||
output_dir = project_root / args.output_dir
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pdf_path = output_dir / f"{basename}.pdf"
|
||||
docx_path = output_dir / f"{basename}.docx"
|
||||
|
||||
fonts_dir = Path(args.fonts_dir)
|
||||
if not fonts_dir.is_absolute():
|
||||
fonts_dir = REPO_ROOT / fonts_dir
|
||||
|
||||
sources_path: Path | None = None
|
||||
if args.sources:
|
||||
sources_path = Path(args.sources)
|
||||
else:
|
||||
default_src = project_root / "phase2" / "sources.jsonl"
|
||||
if default_src.exists():
|
||||
sources_path = default_src
|
||||
|
||||
print("========== Phase 4 成稿 ==========")
|
||||
print(f"项目: {project_root.name}")
|
||||
print(f"Markdown: {md_path.relative_to(project_root)}")
|
||||
print(f"标题: {title}")
|
||||
print(f"输出名: {basename}")
|
||||
print(f"字体目录: {fonts_dir}")
|
||||
print(f"Sources: {sources_path if sources_path else '(缺失)'}")
|
||||
|
||||
if not args.no_pdf:
|
||||
if args.engine == "quarto":
|
||||
build_pdf_quarto(md_path, manifest, pdf_path, fonts_dir, sources_path)
|
||||
else:
|
||||
build_pdf(md_path, manifest_path, pdf_path, fonts_dir, sources_path)
|
||||
if not args.no_docx:
|
||||
build_docx(md_path, docx_path, title)
|
||||
|
||||
print("\n========== 完成 ==========")
|
||||
if pdf_path.exists():
|
||||
print(f" PDF: {pdf_path.relative_to(project_root)} ({pdf_path.stat().st_size // 1024} KB)")
|
||||
if docx_path.exists():
|
||||
print(f" DOCX: {docx_path.relative_to(project_root)} ({docx_path.stat().st_size // 1024} KB)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python3
|
||||
"""引文完整性核查。
|
||||
|
||||
检查 final_zh_polished.md(或其它正文)中的 [src_xxx] 引用与 sources.jsonl 是否一致:
|
||||
- 孤立引用(正文有但 sources.jsonl 无):需要 dr-analyst 补信源或删这处引用
|
||||
- 孤岛信源(sources.jsonl 有但正文无):被 polish 或润色误删了上下文,或 dr-analyst 收集了
|
||||
但没用上
|
||||
- emoji 扫描:正文里不该有 emoji
|
||||
- 编号格式:检查 src_xxx 是否符合规范
|
||||
|
||||
用法:
|
||||
uv run python scripts/check_citations.py <project_slug>
|
||||
uv run python scripts/check_citations.py <project_slug> --md phase4/final_zh.md
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
EMOJI_RE = re.compile(
|
||||
r"[\U0001F000-\U0001FFFF]" # Supplementary Plane emoji
|
||||
r"|[\u2700-\u27BF]" # Dingbats (✅ ❌)
|
||||
r"|[\u2600-\u26FF]" # Misc symbols (⭐ ⚠ ☀)
|
||||
r"|[\u2B00-\u2BFF]" # Misc symbols and arrows
|
||||
)
|
||||
|
||||
# 允许的符号(字体支持)
|
||||
ALLOWED_SYMBOLS = {
|
||||
"✓", "×", "◆", "◇", "●", "○", "★", "※",
|
||||
"→", "←", "↑", "↓",
|
||||
}
|
||||
|
||||
SRC_ID_RE = re.compile(r"\[(src_[A-Za-z0-9_\-]+(?:\s*,\s*src_[A-Za-z0-9_\-]+)*)\]")
|
||||
|
||||
|
||||
def resolve_project(arg: str) -> Path:
|
||||
p = Path(arg)
|
||||
if p.is_dir():
|
||||
return p
|
||||
cand = Path.cwd() / "projects" / arg
|
||||
if cand.is_dir():
|
||||
return cand
|
||||
raise SystemExit(f"project not found: {arg}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="引文完整性核查")
|
||||
parser.add_argument("project", help="项目 slug 或路径")
|
||||
parser.add_argument("--md", default="phase4/final_zh_polished.md")
|
||||
parser.add_argument("--sources", default="phase2/sources.jsonl")
|
||||
args = parser.parse_args()
|
||||
|
||||
project = resolve_project(args.project)
|
||||
md_path = project / args.md
|
||||
src_path = project / args.sources
|
||||
|
||||
if not md_path.exists():
|
||||
raise SystemExit(f"找不到正文:{md_path}")
|
||||
|
||||
text = md_path.read_text(encoding="utf-8")
|
||||
|
||||
# 1. 收集正文引用(保序去重)
|
||||
cited: list[str] = []
|
||||
cited_set: set[str] = set()
|
||||
for m in SRC_ID_RE.finditer(text):
|
||||
for sid in m.group(1).split(","):
|
||||
sid = sid.strip()
|
||||
if sid and sid not in cited_set:
|
||||
cited_set.add(sid)
|
||||
cited.append(sid)
|
||||
|
||||
# 2. 收集 sources.jsonl 中的 ID
|
||||
sources: dict[str, dict] = {}
|
||||
if src_path.exists():
|
||||
for line in src_path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
r = json.loads(line)
|
||||
if sid := r.get("id"):
|
||||
sources[sid] = r
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
orphan_cites = [s for s in cited if s not in sources]
|
||||
island_sources = [s for s in sources if s not in cited_set]
|
||||
matched = [s for s in cited if s in sources]
|
||||
|
||||
print(f"=== 引文统计 ===")
|
||||
print(f" 正文引用(独立 ID):{len(cited_set)}")
|
||||
print(f" sources.jsonl 收录:{len(sources)}")
|
||||
print(f" 匹配:{len(matched)}")
|
||||
print(f" 孤立引用(正文有 sources 无):{len(orphan_cites)}")
|
||||
print(f" 孤岛信源(sources 有正文无):{len(island_sources)}")
|
||||
|
||||
if orphan_cites:
|
||||
print(f"\n=== 孤立引用(前 20 条)===")
|
||||
for s in orphan_cites[:20]:
|
||||
print(f" {s}")
|
||||
if len(orphan_cites) > 20:
|
||||
print(f" ...还有 {len(orphan_cites) - 20}")
|
||||
print(f"\n 处理建议:")
|
||||
print(f" (a) 如果是 dr-analyst 编造的占位符 → 在正文中删除该引用")
|
||||
print(f" (b) 如果是信源未收录 → 补到 sources.jsonl")
|
||||
|
||||
if island_sources:
|
||||
print(f"\n=== 孤岛信源(前 20 条)===")
|
||||
for s in island_sources[:20]:
|
||||
print(f" {s} — {sources[s].get('title', '')[:80]}")
|
||||
if len(island_sources) > 20:
|
||||
print(f" ...还有 {len(island_sources) - 20}")
|
||||
print(f"\n 处理建议:")
|
||||
print(f" (a) 如果是 polish 阶段误删了使用该信源的段落 → 检查 polish diff")
|
||||
print(f" (b) 如果是收集多余信源 → 可以保留(build_references 会自动忽略)")
|
||||
|
||||
# 3. Emoji 扫描
|
||||
emoji_hits = []
|
||||
for m in EMOJI_RE.finditer(text):
|
||||
c = m.group()
|
||||
if c not in ALLOWED_SYMBOLS:
|
||||
line = text[:m.start()].count("\n") + 1
|
||||
emoji_hits.append((line, c))
|
||||
|
||||
if emoji_hits:
|
||||
print(f"\n=== ⚠ 发现 {len(emoji_hits)} 个 emoji(不允许出现在正文)===")
|
||||
seen = {}
|
||||
for line, c in emoji_hits:
|
||||
seen.setdefault(c, []).append(line)
|
||||
for c, lines in seen.items():
|
||||
print(f" U+{ord(c):04X} {c!r} 第 {lines[:5]} 行 等 {len(lines)} 处")
|
||||
print(f" 建议用 python3 替换:sed -i '' 's/{list(seen.keys())[0]}//g' {md_path}")
|
||||
|
||||
# 返回码:有问题返回非零便于 CI 使用
|
||||
if orphan_cites or emoji_hits:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deploy platform adapter templates outside the repository checkout."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.runtime.skills import SkillRegistry
|
||||
|
||||
CODEX_TEMPLATE = REPO_ROOT / "platform_adapters" / "codex"
|
||||
ANTIGRAVITY_TEMPLATE = REPO_ROOT / "platform_adapters" / "antigravity" / "agent"
|
||||
ANTIGRAVITY_RULES_DIR = ANTIGRAVITY_TEMPLATE / "rules"
|
||||
ANTIGRAVITY_WORKFLOWS_DIR = ANTIGRAVITY_TEMPLATE / "workflows"
|
||||
ANTIGRAVITY_AGENTS_FILE = ANTIGRAVITY_TEMPLATE / "agents.md"
|
||||
|
||||
|
||||
@dataclass
|
||||
class DeployResult:
|
||||
platform: str
|
||||
target: Path
|
||||
written: list[Path] = field(default_factory=list)
|
||||
skipped: list[Path] = field(default_factory=list)
|
||||
planned: list[Path] = field(default_factory=list)
|
||||
backups: list[Path] = field(default_factory=list)
|
||||
|
||||
|
||||
def default_codex_home(
|
||||
*,
|
||||
env: dict[str, str] | None = None,
|
||||
user_home: Path | None = None,
|
||||
) -> Path:
|
||||
values = os.environ if env is None else env
|
||||
if values.get("CODEX_HOME"):
|
||||
return Path(values["CODEX_HOME"]).expanduser()
|
||||
home = Path.home() if user_home is None else user_home
|
||||
return home / ".codex"
|
||||
|
||||
|
||||
def copy_tree_contents(
|
||||
src: Path,
|
||||
dst: Path,
|
||||
*,
|
||||
force: bool,
|
||||
dry_run: bool = False,
|
||||
backup_existing: bool = True,
|
||||
exclude: set[Path] | None = None,
|
||||
) -> DeployResult:
|
||||
if not src.exists():
|
||||
raise FileNotFoundError(f"adapter template source not found: {src}")
|
||||
|
||||
result = DeployResult(platform="copy", target=dst)
|
||||
excluded = exclude or set()
|
||||
for item in sorted(src.rglob("*")):
|
||||
rel = item.relative_to(src)
|
||||
if rel in excluded:
|
||||
continue
|
||||
target = dst / rel
|
||||
if item.is_dir():
|
||||
if not dry_run:
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
continue
|
||||
|
||||
if target.exists() and not force:
|
||||
result.skipped.append(target)
|
||||
continue
|
||||
|
||||
result.planned.append(target)
|
||||
if dry_run:
|
||||
continue
|
||||
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if target.exists() and force and backup_existing:
|
||||
backup = target.with_name(f"{target.name}.bak")
|
||||
shutil.copy2(target, backup)
|
||||
result.backups.append(backup)
|
||||
shutil.copy2(item, target)
|
||||
result.written.append(target)
|
||||
return result
|
||||
|
||||
|
||||
def _merge_results(platform: str, target: Path, parts: list[DeployResult]) -> DeployResult:
|
||||
merged = DeployResult(platform=platform, target=target)
|
||||
for part in parts:
|
||||
merged.written.extend(part.written)
|
||||
merged.skipped.extend(part.skipped)
|
||||
merged.planned.extend(part.planned)
|
||||
merged.backups.extend(part.backups)
|
||||
return merged
|
||||
|
||||
|
||||
def copy_registered_skills(dst: Path, *, force: bool, dry_run: bool = False) -> DeployResult:
|
||||
result = DeployResult(platform="skills", target=dst)
|
||||
for skill in SkillRegistry().list():
|
||||
part = copy_tree_contents(skill.path.parent, dst / skill.name, force=force, dry_run=dry_run)
|
||||
result.written.extend(part.written)
|
||||
result.skipped.extend(part.skipped)
|
||||
result.planned.extend(part.planned)
|
||||
result.backups.extend(part.backups)
|
||||
return result
|
||||
|
||||
|
||||
def copy_antigravity_rules(dst: Path, *, force: bool, dry_run: bool = False) -> DeployResult:
|
||||
result = DeployResult(platform="rules", target=dst)
|
||||
if not ANTIGRAVITY_RULES_DIR.exists():
|
||||
return result
|
||||
part = copy_tree_contents(ANTIGRAVITY_RULES_DIR, dst, force=force, dry_run=dry_run)
|
||||
result.written.extend(part.written)
|
||||
result.skipped.extend(part.skipped)
|
||||
result.planned.extend(part.planned)
|
||||
result.backups.extend(part.backups)
|
||||
return result
|
||||
|
||||
|
||||
def copy_antigravity_workflows(dst: Path, *, force: bool, dry_run: bool = False) -> DeployResult:
|
||||
result = DeployResult(platform="workflows", target=dst)
|
||||
if not ANTIGRAVITY_WORKFLOWS_DIR.exists():
|
||||
return result
|
||||
part = copy_tree_contents(ANTIGRAVITY_WORKFLOWS_DIR, dst, force=force, dry_run=dry_run)
|
||||
result.written.extend(part.written)
|
||||
result.skipped.extend(part.skipped)
|
||||
result.planned.extend(part.planned)
|
||||
result.backups.extend(part.backups)
|
||||
return result
|
||||
|
||||
|
||||
def copy_antigravity_agents_file(dst: Path, *, force: bool, dry_run: bool = False) -> DeployResult:
|
||||
result = DeployResult(platform="agents", target=dst)
|
||||
if not ANTIGRAVITY_AGENTS_FILE.exists():
|
||||
return result
|
||||
target = dst / "agents.md"
|
||||
if target.exists() and not force:
|
||||
result.skipped.append(target)
|
||||
return result
|
||||
result.planned.append(target)
|
||||
if dry_run:
|
||||
return result
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if target.exists() and force:
|
||||
backup = target.with_name("agents.md.bak")
|
||||
shutil.copy2(target, backup)
|
||||
result.backups.append(backup)
|
||||
shutil.copy2(ANTIGRAVITY_AGENTS_FILE, target)
|
||||
result.written.append(target)
|
||||
return result
|
||||
|
||||
|
||||
def deploy_codex(
|
||||
*,
|
||||
target: Path | None = None,
|
||||
force: bool = False,
|
||||
skip_skills: bool = False,
|
||||
dry_run: bool = False,
|
||||
include_config: bool = False,
|
||||
repo_root: Path = REPO_ROOT,
|
||||
) -> DeployResult:
|
||||
codex_home = (target or default_codex_home()).expanduser()
|
||||
template = repo_root / "platform_adapters" / "codex"
|
||||
|
||||
parts = [
|
||||
copy_tree_contents(
|
||||
template,
|
||||
codex_home,
|
||||
force=force,
|
||||
dry_run=dry_run,
|
||||
exclude=set() if include_config else {Path("config.toml")},
|
||||
),
|
||||
]
|
||||
if not skip_skills:
|
||||
parts.append(copy_registered_skills(codex_home / "skills", force=force, dry_run=dry_run))
|
||||
return _merge_results("codex", codex_home, parts)
|
||||
|
||||
|
||||
def deploy_antigravity(
|
||||
*,
|
||||
target: Path | None = None,
|
||||
force: bool = False,
|
||||
skip_skills: bool = False,
|
||||
skip_agents: bool = False,
|
||||
skip_rules: bool = False,
|
||||
skip_workflows: bool = False,
|
||||
dry_run: bool = False,
|
||||
repo_root: Path = REPO_ROOT,
|
||||
) -> DeployResult:
|
||||
"""Deploy Antigravity workspace rules and skills into a workspace root.
|
||||
|
||||
This intentionally deploys to a workspace-local `.agent` directory, not
|
||||
global Antigravity/Gemini settings, so existing user configuration is not
|
||||
touched. Existing files are skipped unless `force=True`.
|
||||
"""
|
||||
workspace = (target or repo_root).expanduser()
|
||||
parts: list[DeployResult] = []
|
||||
if not skip_agents:
|
||||
parts.append(copy_antigravity_agents_file(workspace / ".agent", force=force, dry_run=dry_run))
|
||||
if not skip_skills:
|
||||
parts.append(copy_registered_skills(workspace / ".agent" / "skills", force=force, dry_run=dry_run))
|
||||
if not skip_rules:
|
||||
parts.append(copy_antigravity_rules(workspace / ".agent" / "rules", force=force, dry_run=dry_run))
|
||||
if not skip_workflows:
|
||||
parts.append(copy_antigravity_workflows(workspace / ".agent" / "workflows", force=force, dry_run=dry_run))
|
||||
return _merge_results("antigravity", workspace, parts)
|
||||
|
||||
|
||||
def print_result(result: DeployResult) -> None:
|
||||
action = "planned" if result.planned and not result.written else "written"
|
||||
print(f"{result.platform} adapter deployment")
|
||||
print(f" target: {result.target}")
|
||||
print(f" files {action}: {len(result.planned if action == 'planned' else result.written)}")
|
||||
print(f" files skipped: {len(result.skipped)}")
|
||||
print(f" backups: {len(result.backups)}")
|
||||
if result.written:
|
||||
for path in result.written:
|
||||
print(f" {path}")
|
||||
elif result.planned:
|
||||
for path in result.planned:
|
||||
print(f" {path}")
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Deploy Deep Research surface adapters")
|
||||
sub = parser.add_subparsers(dest="platform", required=True)
|
||||
|
||||
codex = sub.add_parser("codex", help="Deploy Codex adapter to CODEX_HOME or a target directory")
|
||||
codex.add_argument("--target", type=Path, help="Codex home target; defaults to $CODEX_HOME or ~/.codex")
|
||||
codex.add_argument("--force", action="store_true", help="overwrite existing files and create .bak backups")
|
||||
codex.add_argument("--skip-skills", action="store_true", help="do not copy canonical skills into target/skills")
|
||||
codex.add_argument("--include-config", action="store_true", help="also copy config.toml; off by default to avoid overwriting global Codex config")
|
||||
codex.add_argument("--dry-run", action="store_true", help="show files that would be written")
|
||||
|
||||
antigravity = sub.add_parser("antigravity", help="Deploy Antigravity workspace rules and skills")
|
||||
antigravity.add_argument("--target", type=Path, help="Workspace root; defaults to this repository")
|
||||
antigravity.add_argument("--force", action="store_true", help="overwrite existing files and create .bak backups")
|
||||
antigravity.add_argument("--skip-agents", action="store_true", help="do not copy role definitions into target/.agent/agents.md")
|
||||
antigravity.add_argument("--skip-skills", action="store_true", help="do not copy canonical skills into target/.agent/skills")
|
||||
antigravity.add_argument("--skip-rules", action="store_true", help="do not copy workspace rules into target/.agent/rules")
|
||||
antigravity.add_argument("--skip-workflows", action="store_true", help="do not copy workflows into target/.agent/workflows")
|
||||
antigravity.add_argument("--dry-run", action="store_true", help="show files that would be written without writing")
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
if args.platform == "codex":
|
||||
result = deploy_codex(
|
||||
target=args.target,
|
||||
force=args.force,
|
||||
skip_skills=args.skip_skills,
|
||||
dry_run=args.dry_run,
|
||||
include_config=args.include_config,
|
||||
)
|
||||
print_result(result)
|
||||
print()
|
||||
print("Run Codex from this repository after deployment:")
|
||||
if args.include_config:
|
||||
print(" codex --profile deep-research")
|
||||
else:
|
||||
print(" codex")
|
||||
print("Note: config.toml is not copied by default. Use --include-config only if you want the bundled profile.")
|
||||
return 0
|
||||
if args.platform == "antigravity":
|
||||
result = deploy_antigravity(
|
||||
target=args.target,
|
||||
force=args.force,
|
||||
skip_agents=args.skip_agents,
|
||||
skip_skills=args.skip_skills,
|
||||
skip_rules=args.skip_rules,
|
||||
skip_workflows=args.skip_workflows,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
print_result(result)
|
||||
print()
|
||||
print("Open the target workspace in Antigravity and enable/mention the workspace rule if needed:")
|
||||
print(" .agent/rules/deep-research-antigravity.md")
|
||||
print("Workflow installed when supported by your Antigravity build:")
|
||||
print(" .agent/workflows/deep-research-native.md")
|
||||
print("Existing files are skipped by default. Use --force only when you want .bak backups and replacement.")
|
||||
return 0
|
||||
raise SystemExit(f"unsupported platform: {args.platform}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check and repair a Deep Research deployment checkout."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError: # pragma: no cover - Python < 3.11 fallback.
|
||||
tomllib = None # type: ignore[assignment]
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.deploy_adapters import default_codex_home, deploy_codex
|
||||
from scripts.runtime.skills import SkillRegistry
|
||||
|
||||
CODEX_TEMPLATE = REPO_ROOT / "platform_adapters" / "codex"
|
||||
AGENT_SKILLS = REPO_ROOT / "platform_adapters" / "antigravity" / "agent" / "skills"
|
||||
|
||||
REQUIRED_PATHS = [
|
||||
"AGENTS.md",
|
||||
"GEMINI.md",
|
||||
"README.md",
|
||||
"PLAN.md",
|
||||
"platform_adapters/antigravity/agent/agents.md",
|
||||
"platform_adapters/antigravity/agent/rules/deep-research-antigravity.md",
|
||||
"platform_adapters/antigravity/agent/workflows/deep-research-native.md",
|
||||
"platform_adapters/antigravity/agent/skills/search-strategy/SKILL.md",
|
||||
"platform_adapters/codex/config.toml",
|
||||
"platform_adapters/codex/agents/dr-pm.toml",
|
||||
"platform_adapters/codex/commands/dr-run.md",
|
||||
"skills/deep-research/SKILL.md",
|
||||
"skills/document-ingest/SKILL.md",
|
||||
"scripts/dr.py",
|
||||
"scripts/deploy_adapters.py",
|
||||
"scripts/export_antigravity_workspace.py",
|
||||
]
|
||||
|
||||
REQUIRED_ENV_KEYS = [
|
||||
"ZENMUX_API_KEY",
|
||||
"TAVILY_API_KEY",
|
||||
"BRAVE_API_KEY",
|
||||
"EXA_API_KEY",
|
||||
]
|
||||
|
||||
|
||||
def rel(path: Path) -> str:
|
||||
return str(path.relative_to(REPO_ROOT))
|
||||
|
||||
|
||||
def copy_tree_contents(src: Path, dst: Path, *, force: bool) -> list[Path]:
|
||||
written: list[Path] = []
|
||||
if not src.exists():
|
||||
raise RuntimeError(f"source not found: {rel(src)}")
|
||||
dst.mkdir(parents=True, exist_ok=True)
|
||||
for item in src.rglob("*"):
|
||||
target = dst / item.relative_to(src)
|
||||
if item.is_dir():
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
continue
|
||||
if target.exists() and not force:
|
||||
continue
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(item, target)
|
||||
written.append(target)
|
||||
return written
|
||||
|
||||
|
||||
def parse_env(path: Path) -> dict[str, str]:
|
||||
values: dict[str, str] = {}
|
||||
if not path.exists():
|
||||
return values
|
||||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
values[key.strip()] = value.strip().strip('"').strip("'")
|
||||
return values
|
||||
|
||||
|
||||
def git_tracked(paths: list[str]) -> set[str]:
|
||||
proc = subprocess.run(
|
||||
["git", "ls-files", *paths],
|
||||
cwd=REPO_ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=False,
|
||||
)
|
||||
return set(proc.stdout.splitlines())
|
||||
|
||||
|
||||
def check_toml(path: Path, issues: list[str]) -> None:
|
||||
if not path.exists():
|
||||
return
|
||||
if tomllib is None:
|
||||
issues.append("Python tomllib unavailable; skip TOML parse checks")
|
||||
return
|
||||
try:
|
||||
tomllib.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception as exc: # noqa: BLE001 - deployment diagnostics.
|
||||
issues.append(f"{rel(path)} TOML parse failed: {exc}")
|
||||
|
||||
|
||||
def check_deployment() -> int:
|
||||
issues: list[str] = []
|
||||
warnings: list[str] = []
|
||||
|
||||
for item in REQUIRED_PATHS:
|
||||
if not (REPO_ROOT / item).exists():
|
||||
issues.append(f"missing required path: {item}")
|
||||
|
||||
tracked = git_tracked(["platform_adapters", "skills"])
|
||||
legacy_tracked = git_tracked([".codex", ".opencode", ".claude", ".gemini", ".agents"])
|
||||
if legacy_tracked:
|
||||
warnings.append("legacy platform adapter files are still tracked; prefer Antigravity clean workspace or remove them from the branch")
|
||||
for item in REQUIRED_PATHS:
|
||||
if item.startswith("platform_adapters/") and item not in tracked:
|
||||
warnings.append(f"not tracked by git: {item}")
|
||||
|
||||
skills = SkillRegistry().list()
|
||||
if len(skills) < 10:
|
||||
issues.append(f"expected at least 10 Codex skills, found {len(skills)}")
|
||||
|
||||
check_toml(CODEX_TEMPLATE / "config.toml", issues)
|
||||
for path in sorted((CODEX_TEMPLATE / "agents").glob("*.toml")):
|
||||
check_toml(path, issues)
|
||||
|
||||
codex_home = default_codex_home()
|
||||
if not (codex_home / "commands" / "dr-run.md").exists():
|
||||
warnings.append(f"Codex adapter not deployed to {codex_home}; run scripts/deploy_adapters.py codex")
|
||||
|
||||
env_values = {key: os.environ.get(key, "") for key in REQUIRED_ENV_KEYS}
|
||||
env_values.update({k: v for k, v in parse_env(REPO_ROOT / "secrets.env").items() if not env_values.get(k)})
|
||||
missing_env = [key for key in REQUIRED_ENV_KEYS if not env_values.get(key)]
|
||||
if missing_env:
|
||||
warnings.append("missing optional/required API keys for full automation: " + ", ".join(missing_env))
|
||||
|
||||
print("Deep Research deployment check")
|
||||
print(f" repo: {REPO_ROOT}")
|
||||
print(f" platform adapter files tracked: {sum(1 for p in tracked if p.startswith('platform_adapters/'))}")
|
||||
print(f" legacy platform adapter files tracked: {len(legacy_tracked)}")
|
||||
print(f" codex home: {codex_home}")
|
||||
print(f" antigravity adapter source files tracked: {sum(1 for p in tracked if p.startswith('platform_adapters/antigravity/'))}")
|
||||
print(f" codex skills: {len(skills)}")
|
||||
|
||||
if warnings:
|
||||
print("\nWarnings:")
|
||||
for item in warnings:
|
||||
print(f" - {item}")
|
||||
|
||||
if issues:
|
||||
print("\nIssues:")
|
||||
for item in issues:
|
||||
print(f" - {item}")
|
||||
return 1
|
||||
|
||||
print("\nDeployment check passed.")
|
||||
return 0
|
||||
|
||||
|
||||
def repair(force: bool, codex_home: Path | None) -> int:
|
||||
try:
|
||||
codex_result = deploy_codex(target=codex_home, force=force)
|
||||
except PermissionError as exc:
|
||||
print(f"repair failed: permission denied: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
except RuntimeError as exc:
|
||||
print(f"repair failed: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print("Repair completed.")
|
||||
print(f" Codex home files written: {len(codex_result.written)}")
|
||||
print(f" Codex home: {codex_result.target}")
|
||||
print(f" Antigravity skills source: {AGENT_SKILLS}")
|
||||
return check_deployment()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Check or repair Deep Research deployment files")
|
||||
parser.add_argument("--repair", action="store_true", help="deploy Codex templates outside the repo and sync skills")
|
||||
parser.add_argument("--force", action="store_true", help="overwrite existing files during --repair")
|
||||
parser.add_argument("--codex-home", type=Path, help="Codex home target for --repair; defaults to $CODEX_HOME or ~/.codex")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.repair:
|
||||
return repair(force=args.force, codex_home=args.codex_home)
|
||||
return check_deployment()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,962 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Platform-neutral Deep Research CLI for Codex and other adapters.
|
||||
|
||||
This CLI intentionally keeps deterministic orchestration in Python while
|
||||
allowing Codex/OpenCode/Gemini/Claude Code to provide the agentic layer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.lib.model_config import (
|
||||
ModelConfigError,
|
||||
list_model_profiles,
|
||||
parse_model_overrides,
|
||||
resolve_model_profile,
|
||||
)
|
||||
from scripts.runtime.assembly import build_chapter_briefs, build_compressed_findings, run_chapter_assembly_workers
|
||||
from scripts.runtime.orchestrator import create_phase2_task_cards, write_placeholder_packets
|
||||
from scripts.runtime.methods import ResearchMethodRegistry
|
||||
from scripts.runtime.phase1 import create_project, render_framework, write_material_brief
|
||||
from scripts.runtime.review import build_phase3_critique, build_phase3_model_critique
|
||||
from scripts.runtime.roles import resolve_runtime_profile
|
||||
from scripts.runtime.source_cache import cache_sources
|
||||
from scripts.runtime.sources import rebuild_sources_from_packets
|
||||
from scripts.runtime.skills import SkillRegistry, default_adapter_skill_dirs
|
||||
from scripts.runtime.tasks import TaskCard, load_task_cards, validate_packet, write_task_cards
|
||||
from scripts.runtime.workers import run_packet_workers
|
||||
|
||||
|
||||
PROJECTS_DIR = REPO_ROOT / "projects"
|
||||
CODEX_COMMAND_TEMPLATES_DIR = REPO_ROOT / "codex_adapter_templates" / "codex" / "commands"
|
||||
LEGACY_CODEX_COMMANDS_DIR = REPO_ROOT / ".codex" / "commands"
|
||||
|
||||
|
||||
def resolve_project(project: str | None, *, projects_dir: Path = PROJECTS_DIR) -> Path:
|
||||
if project:
|
||||
p = Path(project)
|
||||
if p.is_dir():
|
||||
return p.resolve()
|
||||
cand = projects_dir / project
|
||||
if cand.is_dir():
|
||||
return cand.resolve()
|
||||
raise SystemExit(f"project not found: {project}")
|
||||
|
||||
manifests = sorted(
|
||||
projects_dir.glob("*/manifest.json"),
|
||||
key=lambda p: p.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
if not manifests:
|
||||
raise SystemExit("no projects found")
|
||||
return manifests[0].parent.resolve()
|
||||
|
||||
|
||||
def load_manifest(project_root: Path) -> dict:
|
||||
path = project_root / "manifest.json"
|
||||
if not path.exists():
|
||||
raise SystemExit(f"manifest not found: {path}")
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def count_words(text: str) -> int:
|
||||
return len(re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)*", text))
|
||||
|
||||
|
||||
def count_chinese_chars(text: str) -> int:
|
||||
return sum(1 for c in text if "\u4e00" <= c <= "\u9fff")
|
||||
|
||||
|
||||
def file_state(path: Path) -> str:
|
||||
return "yes" if path.exists() else "no"
|
||||
|
||||
|
||||
def packet_state_counts(project_root: Path) -> dict[str, int]:
|
||||
packets = sorted((project_root / "phase2" / "packets").glob("*.json"))
|
||||
errors = sorted((project_root / "phase2" / "packet_errors").glob("*.json"))
|
||||
counts = {
|
||||
"ready": 0,
|
||||
"placeholder": 0,
|
||||
"invalid": 0,
|
||||
"errors": 0,
|
||||
"stale_errors": 0,
|
||||
"total": len(packets),
|
||||
}
|
||||
ready_stems: set[str] = set()
|
||||
for path in packets:
|
||||
try:
|
||||
packet = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
counts["invalid"] += 1
|
||||
continue
|
||||
has_evidence = bool(
|
||||
packet.get("claims")
|
||||
or packet.get("evidence_items")
|
||||
or packet.get("counter_evidence")
|
||||
or packet.get("source_ids")
|
||||
)
|
||||
if has_evidence:
|
||||
counts["ready"] += 1
|
||||
ready_stems.add(path.stem)
|
||||
else:
|
||||
counts["placeholder"] += 1
|
||||
for path in errors:
|
||||
if path.stem in ready_stems:
|
||||
counts["stale_errors"] += 1
|
||||
else:
|
||||
counts["errors"] += 1
|
||||
return counts
|
||||
|
||||
|
||||
def run_cmd(cmd: list[str], *, dry_run: bool) -> int:
|
||||
printable = " ".join(cmd)
|
||||
print(f"$ {printable}")
|
||||
if dry_run:
|
||||
return 0
|
||||
return subprocess.run(cmd, cwd=REPO_ROOT, check=False).returncode
|
||||
|
||||
|
||||
def cmd_init(args: argparse.Namespace) -> int:
|
||||
projects_dir = Path(args.projects_dir) if args.projects_dir else PROJECTS_DIR
|
||||
project_root = create_project(
|
||||
topic=args.topic,
|
||||
slug=args.slug,
|
||||
projects_dir=projects_dir,
|
||||
method_key=args.method,
|
||||
report_type=args.report_type,
|
||||
model_profile=args.profile,
|
||||
target_words=args.target_words,
|
||||
input_materials=args.input_material,
|
||||
)
|
||||
print(f"Project: {project_root.name}")
|
||||
print(f"Created: {project_root}")
|
||||
print("Runtime: python-core-v0.20")
|
||||
print("Next: run `dr.py frame <project>` to generate phase1/framework.md")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_frame(args: argparse.Namespace) -> int:
|
||||
project_root = resolve_project(args.project)
|
||||
if args.dry_run:
|
||||
manifest = load_manifest(project_root)
|
||||
method = ResearchMethodRegistry().get(args.method or manifest.get("research_method"))
|
||||
print(f"Project: {project_root.name}")
|
||||
print(f"Would write: phase1/framework.md")
|
||||
print(f"Research method: {method.key}")
|
||||
print(f"Chapters: {args.chapters}")
|
||||
return 0
|
||||
path = render_framework(
|
||||
project_root,
|
||||
method_key=args.method,
|
||||
chapter_count=args.chapters,
|
||||
preserve_existing_outline=args.preserve_existing_outline,
|
||||
)
|
||||
print(f"Project: {project_root.name}")
|
||||
print(f"Wrote: {path.relative_to(project_root)}")
|
||||
print("Pause: review and approve the framework before Phase 2.")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_approve(args: argparse.Namespace) -> int:
|
||||
project_root = resolve_project(args.project)
|
||||
manifest = load_manifest(project_root)
|
||||
phase1 = manifest.setdefault("phase1", {})
|
||||
phase1["approved"] = True
|
||||
phase1["requires_user_interview"] = False
|
||||
phase1["approved_at"] = datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
manifest["updated_at"] = phase1["approved_at"]
|
||||
(project_root / "manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"Project: {project_root.name}")
|
||||
print("Phase 1 approved. Phase 2 research is now enabled.")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_skills(args: argparse.Namespace) -> int:
|
||||
registry = SkillRegistry()
|
||||
if args.skills_cmd == "list":
|
||||
for name in registry.list_names():
|
||||
print(name)
|
||||
return 0
|
||||
if args.skills_cmd == "validate":
|
||||
result = registry.validate()
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0 if result["ok"] else 1
|
||||
if args.skills_cmd == "sync":
|
||||
targets = [Path(item) for item in args.target] if args.target else default_adapter_skill_dirs()
|
||||
copied = registry.sync_to(targets, force=True)
|
||||
print(f"Synced skills: {copied}")
|
||||
for target in targets:
|
||||
print(f" {target}")
|
||||
return 0
|
||||
raise SystemExit(f"unknown skills command: {args.skills_cmd}")
|
||||
|
||||
|
||||
def cmd_methods(args: argparse.Namespace) -> int:
|
||||
registry = ResearchMethodRegistry()
|
||||
if args.methods_cmd == "list":
|
||||
for name in registry.list_names():
|
||||
method = registry.get(name)
|
||||
print(f"{method.key}: {method.name}")
|
||||
return 0
|
||||
if args.methods_cmd == "show":
|
||||
method = registry.get(args.method)
|
||||
print(json.dumps(method.__dict__, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
raise SystemExit(f"unknown methods command: {args.methods_cmd}")
|
||||
|
||||
|
||||
def cmd_research(args: argparse.Namespace) -> int:
|
||||
project_root = resolve_project(args.project)
|
||||
manifest = load_manifest(project_root)
|
||||
if not args.force and not (manifest.get("phase1") or {}).get("approved"):
|
||||
raise SystemExit(
|
||||
"Phase 1 is not approved. Review phase1/material_brief.md and phase1/framework.md, "
|
||||
"then run `uv run python scripts/dr.py approve <project>` or pass --force."
|
||||
)
|
||||
if args.profile == "codex_native" and (args.execute_packets or args.assemble_chapters):
|
||||
raise SystemExit(
|
||||
"`codex_native` cannot be used for Python-core model execution: scripts/dr.py currently calls external "
|
||||
"API clients, not Codex App built-in models. Use a clearly external profile such as `medium`, or run "
|
||||
"Codex-native execution through the surface adapter/manual task workflow."
|
||||
)
|
||||
runtime = resolve_runtime_profile(
|
||||
profile=args.profile,
|
||||
overrides=parse_model_overrides(args.model_override),
|
||||
)
|
||||
if args.append_task_cards:
|
||||
generated = create_phase2_task_cards(
|
||||
project_root,
|
||||
axes=args.axis,
|
||||
dry_run=True,
|
||||
)
|
||||
existing_path = project_root / "phase2" / "task_cards.json"
|
||||
existing_cards = load_task_cards(existing_path) if existing_path.exists() else []
|
||||
seen = {card.task_id for card in existing_cards}
|
||||
appended_cards = [TaskCard(**item) for item in generated if item["task_id"] not in seen]
|
||||
runnable_existing_cards: list[TaskCard] = []
|
||||
if args.execute_packets and args.axis:
|
||||
axis_set = set(args.axis)
|
||||
for card in existing_cards:
|
||||
if card.topic_axis not in axis_set:
|
||||
continue
|
||||
packet_path = project_root / card.output_packet
|
||||
try:
|
||||
validate_packet(json.loads(packet_path.read_text(encoding="utf-8")))
|
||||
except Exception:
|
||||
runnable_existing_cards.append(card)
|
||||
merged_cards = [*existing_cards, *appended_cards]
|
||||
if not args.dry_run:
|
||||
write_task_cards(existing_path, merged_cards)
|
||||
phase2 = manifest.setdefault("phase2", {})
|
||||
phase2.update(
|
||||
{
|
||||
"status": "in_progress",
|
||||
"runtime": "python-core-v0.20",
|
||||
"task_cards_path": "phase2/task_cards.json",
|
||||
"task_cards_total": len(merged_cards),
|
||||
"task_cards_appended": len(appended_cards),
|
||||
"updated_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(),
|
||||
}
|
||||
)
|
||||
(project_root / "manifest.json").write_text(
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
card_dicts = [card.to_dict() for card in [*appended_cards, *runnable_existing_cards]]
|
||||
else:
|
||||
card_dicts = create_phase2_task_cards(
|
||||
project_root,
|
||||
axes=args.axis,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
if args.execute_packets and args.dry_run:
|
||||
raise SystemExit("--execute-packets cannot be combined with --dry-run")
|
||||
if args.assemble_chapters and args.dry_run:
|
||||
raise SystemExit("--assemble-chapters cannot be combined with --dry-run")
|
||||
if args.execute_packets:
|
||||
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets
|
||||
from scripts.runtime.workers import ProjectSearchProvider
|
||||
|
||||
load_secrets()
|
||||
|
||||
def client_factory(_role):
|
||||
return ZenMuxClient(log_file=project_root / "phase2" / "logs" / "packets.jsonl")
|
||||
|
||||
def search_provider_factory():
|
||||
return ProjectSearchProvider(strict_specialized=not args.allow_search_fallback)
|
||||
|
||||
packet_count = run_packet_workers(
|
||||
project_root=project_root,
|
||||
cards=[TaskCard(**item) for item in card_dicts],
|
||||
runtime=runtime,
|
||||
client_factory=client_factory,
|
||||
search_provider_factory=search_provider_factory,
|
||||
workers=args.workers,
|
||||
)
|
||||
elif not (args.build_briefs or args.assemble_chapters):
|
||||
packet_count = write_placeholder_packets(project_root, card_dicts, dry_run=args.dry_run)
|
||||
else:
|
||||
packet_count = len(list((project_root / "phase2" / "packets").glob("*.json")))
|
||||
brief_count = 0
|
||||
chapter_count = 0
|
||||
compressed_count = 0
|
||||
source_count = None
|
||||
if args.build_briefs or args.assemble_chapters:
|
||||
source_count = rebuild_sources_from_packets(project_root)
|
||||
briefs = build_chapter_briefs(project_root)
|
||||
brief_count = len(briefs)
|
||||
compressed_findings = build_compressed_findings(project_root)
|
||||
compressed_count = len(compressed_findings)
|
||||
if args.assemble_chapters:
|
||||
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets
|
||||
|
||||
load_secrets()
|
||||
|
||||
def chapter_client_factory(_role):
|
||||
return ZenMuxClient(log_file=project_root / "phase2" / "logs" / "chapters.jsonl")
|
||||
|
||||
chapter_count = run_chapter_assembly_workers(
|
||||
project_root=project_root,
|
||||
briefs=compressed_findings,
|
||||
runtime=runtime,
|
||||
client_factory=chapter_client_factory,
|
||||
workers=args.workers,
|
||||
)
|
||||
print(f"Project: {project_root.name}")
|
||||
print(f"Runtime: python-core-v0.20")
|
||||
print(f"Model profile: {runtime.profile}")
|
||||
print(f"Workers: {args.workers}")
|
||||
print(f"Task cards: {len(card_dicts)}")
|
||||
print(f"Packets: {packet_count}")
|
||||
if source_count is not None:
|
||||
print(f"Sources rebuilt: {source_count}")
|
||||
if args.build_briefs or args.assemble_chapters:
|
||||
print(f"Chapter briefs: {brief_count}")
|
||||
print(f"Compressed findings: {compressed_count}")
|
||||
if args.assemble_chapters:
|
||||
print(f"Chapter drafts: {chapter_count}")
|
||||
if args.dry_run:
|
||||
print("Dry run: no files written")
|
||||
elif args.assemble_chapters:
|
||||
print("Wrote: phase2/drafts/chXX.md")
|
||||
elif args.execute_packets:
|
||||
print("Wrote: phase2/task_cards.json and validated phase2/packets/*.json")
|
||||
print("Next: rerun with --build-briefs to aggregate packets into chapter briefs.")
|
||||
elif args.build_briefs:
|
||||
print("Wrote: phase2/chapter_briefs/*.json and phase2/compressed_findings/*.json")
|
||||
print("Next: rerun with --assemble-chapters to write Chinese chapter drafts.")
|
||||
else:
|
||||
print("Wrote: phase2/task_cards.json and phase2/packets/*.json")
|
||||
print("Next: rerun with --execute-packets to fill packets via model workers.")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_run(args: argparse.Namespace) -> int:
|
||||
target = args.project_or_topic
|
||||
projects_dir = Path(args.projects_dir) if args.projects_dir else PROJECTS_DIR
|
||||
try:
|
||||
project_root = resolve_project(target, projects_dir=projects_dir)
|
||||
except SystemExit:
|
||||
if args.dry_run:
|
||||
print(f"New topic detected: {target}")
|
||||
print("Dry run: would create project and write phase1/framework.md")
|
||||
return 0
|
||||
project_root = create_project(
|
||||
topic=target,
|
||||
slug=args.slug,
|
||||
projects_dir=projects_dir,
|
||||
method_key=args.method,
|
||||
report_type=args.report_type,
|
||||
model_profile=args.profile or "medium",
|
||||
target_words=args.target_words,
|
||||
input_materials=args.input_material,
|
||||
)
|
||||
framework = render_framework(project_root, chapter_count=args.chapters)
|
||||
print(f"Project: {project_root.name}")
|
||||
print("Runtime: python-core-v0.20")
|
||||
print(f"Created: {project_root}")
|
||||
print(f"Wrote: {framework.relative_to(project_root)}")
|
||||
print("Pause: review and approve the framework before Phase 2.")
|
||||
return 0
|
||||
|
||||
print(f"Project: {project_root.name}")
|
||||
print("Runtime: python-core-v0.20")
|
||||
print("Next command: research")
|
||||
if args.dry_run:
|
||||
print("Dry run: would inspect manifest and continue from the next incomplete phase")
|
||||
return 0
|
||||
return cmd_research(
|
||||
argparse.Namespace(
|
||||
project=str(project_root),
|
||||
workers=args.workers,
|
||||
axis=None,
|
||||
profile=args.profile,
|
||||
model_override=[],
|
||||
execute_packets=False,
|
||||
append_task_cards=False,
|
||||
allow_search_fallback=False,
|
||||
build_briefs=False,
|
||||
assemble_chapters=False,
|
||||
force=False,
|
||||
dry_run=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def cmd_review(args: argparse.Namespace) -> int:
|
||||
project_root = resolve_project(args.project)
|
||||
if args.model_review:
|
||||
if args.dry_run:
|
||||
print(f"Project: {project_root.name}")
|
||||
print("Phase 3 model review plan:")
|
||||
print(f" model: {args.model}")
|
||||
print(" context: phase3/review_context_opus_4_7.md")
|
||||
print(" output: phase3/critique.md")
|
||||
return 0
|
||||
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets
|
||||
|
||||
load_secrets()
|
||||
with ZenMuxClient(log_file=project_root / "phase3" / "logs" / "review.jsonl") as client:
|
||||
path = build_phase3_model_critique(
|
||||
project_root,
|
||||
client=client,
|
||||
model=args.model,
|
||||
max_context_chars=args.max_context_chars,
|
||||
)
|
||||
else:
|
||||
path = build_phase3_critique(project_root)
|
||||
print(f"Project: {project_root.name}")
|
||||
print(f"Wrote: {path.relative_to(project_root)}")
|
||||
print("Pause: review critique before Phase 4.")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_status(args: argparse.Namespace) -> int:
|
||||
project_root = resolve_project(args.project)
|
||||
manifest = load_manifest(project_root)
|
||||
slug = project_root.name
|
||||
|
||||
drafts = sorted((project_root / "phase2" / "drafts").glob("ch*.md"))
|
||||
evidence = sorted((project_root / "phase2" / "evidence").glob("ch*-evidence.md"))
|
||||
task_cards = project_root / "phase2" / "task_cards.json"
|
||||
research_brief = project_root / "phase1" / "research_brief.json"
|
||||
compressed_findings = sorted((project_root / "phase2" / "compressed_findings").glob("ch*.json"))
|
||||
packet_counts = packet_state_counts(project_root)
|
||||
sources = project_root / "phase2" / "sources.jsonl"
|
||||
final_en = project_root / "phase4" / "final_en.md"
|
||||
final_zh = project_root / "phase4" / "final_zh.md"
|
||||
final_zh_polished = project_root / "phase4" / "final_zh_polished.md"
|
||||
glossary = project_root / "phase4" / "glossary.json"
|
||||
|
||||
en_words = count_words(final_en.read_text(encoding="utf-8")) if final_en.exists() else 0
|
||||
zh_source = final_zh_polished if final_zh_polished.exists() else final_zh
|
||||
zh_chars = count_chinese_chars(zh_source.read_text(encoding="utf-8")) if zh_source.exists() else 0
|
||||
source_count = 0
|
||||
if sources.exists():
|
||||
source_count = sum(1 for line in sources.read_text(encoding="utf-8").splitlines() if line.strip())
|
||||
|
||||
print(f"Project: {manifest.get('topic', slug)}")
|
||||
print(f"Slug: {slug}")
|
||||
print(f"Title: {manifest.get('report_title', '(unset)')}")
|
||||
print(f"Type: {manifest.get('type', '(unset)')}")
|
||||
print()
|
||||
print("Phases:")
|
||||
for phase in ("phase1", "phase2", "phase3", "phase4"):
|
||||
p = manifest.get(phase, {})
|
||||
print(f" {phase}: {p.get('status', 'pending')} approved={p.get('approved', False)}")
|
||||
print()
|
||||
print("Artifacts:")
|
||||
print(f" framework: {file_state(project_root / 'phase1' / 'framework.md')}")
|
||||
print(f" research_brief.json: {file_state(research_brief)}")
|
||||
print(f" task_cards.json: {file_state(task_cards)}")
|
||||
print(
|
||||
" packets: "
|
||||
f"ready={packet_counts['ready']} "
|
||||
f"placeholder={packet_counts['placeholder']} "
|
||||
f"invalid={packet_counts['invalid']} "
|
||||
f"errors={packet_counts['errors']} "
|
||||
f"stale_errors={packet_counts['stale_errors']} "
|
||||
f"total={packet_counts['total']}"
|
||||
)
|
||||
print(f" drafts: {len(drafts)}")
|
||||
print(f" compressed findings: {len(compressed_findings)}")
|
||||
print(f" evidence files: {len(evidence)}")
|
||||
print(f" sources: {source_count}")
|
||||
print(f" final_en.md: {file_state(final_en)} ({en_words:,} words)")
|
||||
print(f" final_zh.md: {file_state(final_zh)}")
|
||||
print(f" final_zh_polished.md: {file_state(final_zh_polished)} ({zh_chars:,} Chinese chars)")
|
||||
print(f" glossary.json: {file_state(glossary)}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_prompt(args: argparse.Namespace) -> int:
|
||||
name = args.command
|
||||
if not name.startswith("dr-"):
|
||||
name = f"dr-{name}"
|
||||
candidates = [
|
||||
CODEX_COMMAND_TEMPLATES_DIR / f"{name}.md",
|
||||
LEGACY_CODEX_COMMANDS_DIR / f"{name}.md",
|
||||
]
|
||||
path = next((candidate for candidate in candidates if candidate.exists()), None)
|
||||
if path is None:
|
||||
raise SystemExit(f"Codex command template not found: {candidates[0]}")
|
||||
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if args.argument:
|
||||
text = text.replace("$ARGUMENTS", args.argument)
|
||||
else:
|
||||
text = text.replace("$ARGUMENTS", "")
|
||||
print(text)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_glossary(args: argparse.Namespace) -> int:
|
||||
project_root = resolve_project(args.project)
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "build_glossary.py"),
|
||||
str(project_root),
|
||||
"--workers",
|
||||
str(args.workers),
|
||||
]
|
||||
if args.force:
|
||||
cmd.append("--force")
|
||||
if args.only:
|
||||
cmd += ["--only", args.only]
|
||||
if args.input:
|
||||
cmd += ["--input", args.input]
|
||||
if args.output:
|
||||
cmd += ["--output", args.output]
|
||||
return run_cmd(cmd, dry_run=args.dry_run)
|
||||
|
||||
|
||||
def cmd_finalize(args: argparse.Namespace) -> int:
|
||||
project_root = resolve_project(args.project)
|
||||
manifest = load_manifest(project_root)
|
||||
effective_profile = args.model_profile or manifest.get("model_profile")
|
||||
try:
|
||||
resolved = resolve_model_profile(
|
||||
profile=effective_profile,
|
||||
overrides=parse_model_overrides(args.model_override),
|
||||
)
|
||||
except ModelConfigError as exc:
|
||||
raise SystemExit(f"model profile resolution failed: {exc}") from exc
|
||||
roles = resolved["roles"]
|
||||
|
||||
if not args.legacy_translate:
|
||||
final_input = args.input
|
||||
if args.number_citations and not args.dry_run:
|
||||
rc = run_cmd(
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "number_citations.py"),
|
||||
str(project_root),
|
||||
"--input",
|
||||
args.input,
|
||||
"--output",
|
||||
args.numbered_output,
|
||||
],
|
||||
dry_run=False,
|
||||
)
|
||||
if rc != 0:
|
||||
return rc
|
||||
final_input = args.numbered_output
|
||||
elif args.number_citations:
|
||||
final_input = args.numbered_output
|
||||
cmd: list[str] = [
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "build_report.py"),
|
||||
str(project_root),
|
||||
"--input",
|
||||
final_input,
|
||||
]
|
||||
if args.report_engine:
|
||||
cmd += ["--engine", args.report_engine]
|
||||
if args.no_docx:
|
||||
cmd.append("--no-docx")
|
||||
if args.no_pdf:
|
||||
cmd.append("--no-pdf")
|
||||
if args.dry_run:
|
||||
print("Chinese-native finalize plan:")
|
||||
if args.number_citations:
|
||||
print(
|
||||
"$ "
|
||||
+ " ".join(
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "number_citations.py"),
|
||||
str(project_root),
|
||||
"--input",
|
||||
args.input,
|
||||
"--output",
|
||||
args.numbered_output,
|
||||
]
|
||||
)
|
||||
)
|
||||
print("$ " + " ".join(cmd))
|
||||
if args.polish:
|
||||
print(
|
||||
"$ "
|
||||
+ " ".join(
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "polish.py"),
|
||||
str(project_root),
|
||||
"--source",
|
||||
final_input,
|
||||
"--workers",
|
||||
str(args.polish_workers),
|
||||
"--model",
|
||||
roles.get("polish", "anthropic/claude-sonnet-4.6"),
|
||||
]
|
||||
)
|
||||
)
|
||||
return 0
|
||||
if args.polish:
|
||||
rc = run_cmd(
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "polish.py"),
|
||||
str(project_root),
|
||||
"--source",
|
||||
final_input,
|
||||
"--workers",
|
||||
str(args.polish_workers),
|
||||
"--model",
|
||||
roles.get("polish", "anthropic/claude-sonnet-4.6"),
|
||||
],
|
||||
dry_run=False,
|
||||
)
|
||||
if rc != 0:
|
||||
return rc
|
||||
return run_cmd(cmd, dry_run=False)
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "phase4_pipeline.py"),
|
||||
str(project_root),
|
||||
"--translate-workers",
|
||||
str(args.translate_workers),
|
||||
"--glossary-workers",
|
||||
str(args.glossary_workers),
|
||||
"--polish-workers",
|
||||
str(args.polish_workers),
|
||||
"--translate-model",
|
||||
roles.get("translate", "anthropic/claude-sonnet-4.6"),
|
||||
"--glossary-model",
|
||||
roles.get("glossary", "anthropic/claude-haiku-4.5"),
|
||||
"--polish-model",
|
||||
roles.get("polish", "anthropic/claude-sonnet-4.6"),
|
||||
"--glossary-mode",
|
||||
args.glossary_mode,
|
||||
]
|
||||
if args.dry_run:
|
||||
cmd.append("--dry-run")
|
||||
return run_cmd(cmd, dry_run=False)
|
||||
|
||||
|
||||
def cmd_models(args: argparse.Namespace) -> int:
|
||||
if args.list:
|
||||
for name in list_model_profiles():
|
||||
print(name)
|
||||
return 0
|
||||
|
||||
try:
|
||||
resolved = resolve_model_profile(
|
||||
profile=args.profile,
|
||||
overrides=parse_model_overrides(args.model_override),
|
||||
)
|
||||
except ModelConfigError as exc:
|
||||
raise SystemExit(f"model profile resolution failed: {exc}") from exc
|
||||
|
||||
if args.probe:
|
||||
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets, normalize_zenmux_model
|
||||
|
||||
load_secrets()
|
||||
results = []
|
||||
with ZenMuxClient() as client:
|
||||
for requested_model in sorted(set(resolved["roles"].values())):
|
||||
api_model = normalize_zenmux_model(requested_model)
|
||||
try:
|
||||
content = client.chat_complete(
|
||||
model=requested_model,
|
||||
system="Health check.",
|
||||
user="Reply with OK only.",
|
||||
temperature=0,
|
||||
max_tokens=16,
|
||||
tag=f"models:probe:{requested_model}",
|
||||
)
|
||||
results.append({
|
||||
"requested_model": requested_model,
|
||||
"api_model": api_model,
|
||||
"ok": True,
|
||||
"response": content.strip()[:80],
|
||||
})
|
||||
except Exception as exc: # noqa: BLE001 - probe should report every model.
|
||||
results.append({
|
||||
"requested_model": requested_model,
|
||||
"api_model": api_model,
|
||||
"ok": False,
|
||||
"error": str(exc)[:500],
|
||||
})
|
||||
|
||||
if args.json:
|
||||
print(json.dumps({**resolved, "probe": results}, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(f"Profile: {resolved['profile']}")
|
||||
print("Model probe:")
|
||||
for item in results:
|
||||
status = "ok" if item["ok"] else "fail"
|
||||
print(f" {status} {item['requested_model']} -> {item['api_model']}")
|
||||
if not item["ok"]:
|
||||
print(f" {item['error']}")
|
||||
return 0 if all(item["ok"] for item in results) else 1
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(resolved, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
print(f"Profile: {resolved['profile']}")
|
||||
if resolved["description"]:
|
||||
print(f"Description: {resolved['description']}")
|
||||
print("Roles:")
|
||||
for role in sorted(resolved["roles"]):
|
||||
print(f" {role}: {resolved['roles'][role]}")
|
||||
if resolved.get("task_types"):
|
||||
print("Task types:")
|
||||
for task_type in sorted(resolved["task_types"]):
|
||||
print(f" {task_type}: {resolved['task_types'][task_type]}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_sources(args: argparse.Namespace) -> int:
|
||||
project_root = resolve_project(args.project)
|
||||
if args.sources_cmd == "cache":
|
||||
if args.dry_run:
|
||||
print(f"Project: {project_root.name}")
|
||||
print(f"Would cache sources from: {args.sources}")
|
||||
print(f"Important only: {not args.all}")
|
||||
print(f"Limit: {args.limit}")
|
||||
return 0
|
||||
results = cache_sources(
|
||||
project_root,
|
||||
sources_rel=args.sources,
|
||||
important_only=not args.all,
|
||||
limit=args.limit,
|
||||
force=args.force,
|
||||
)
|
||||
print(f"Project: {project_root.name}")
|
||||
print(f"Cached source snapshots: {len(results)}")
|
||||
print("Wrote: phase2/source_cache/md/*.md")
|
||||
print("Updated: phase2/sources.jsonl")
|
||||
return 0
|
||||
raise SystemExit(f"unknown sources command: {args.sources_cmd}")
|
||||
|
||||
|
||||
def cmd_apply_models(args: argparse.Namespace) -> int:
|
||||
cmd = [
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "apply_model_profile.py"),
|
||||
"--profile",
|
||||
args.profile,
|
||||
"--target",
|
||||
args.target,
|
||||
]
|
||||
for item in args.model_override:
|
||||
cmd += ["--model-override", item]
|
||||
if args.dry_run:
|
||||
cmd.append("--dry-run")
|
||||
return run_cmd(cmd, dry_run=False)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Deep Research platform-neutral CLI")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
init = sub.add_parser("init", help="Initialize a Python-core research project")
|
||||
init.add_argument("topic", help="Research topic")
|
||||
init.add_argument("--slug", help="Project slug")
|
||||
init.add_argument("--method", help="Research method key")
|
||||
init.add_argument("--type", dest="report_type", default="research", help="Report type")
|
||||
init.add_argument("--profile", default="medium", help="Model profile name from configs/models.yaml")
|
||||
init.add_argument("--target-words", type=int, default=30000)
|
||||
init.add_argument("--input-material", action="append", default=[], help="Path or note for user-provided material")
|
||||
init.add_argument("--projects-dir", help="Override projects directory")
|
||||
init.set_defaults(func=cmd_init)
|
||||
|
||||
approve = sub.add_parser("approve", help="Approve Phase 1 gates before Phase 2")
|
||||
approve.add_argument("project", help="Project slug or path")
|
||||
approve.set_defaults(func=cmd_approve)
|
||||
|
||||
frame = sub.add_parser("frame", help="Generate Phase 1 framework.md")
|
||||
frame.add_argument("project", help="Project slug or path")
|
||||
frame.add_argument("--method", help="Override research method key")
|
||||
frame.add_argument("--chapters", type=int, default=10)
|
||||
frame.add_argument("--preserve-existing-outline", action="store_true", help="Keep current framework chapter titles and enrich Phase 1 planning")
|
||||
frame.add_argument("--dry-run", action="store_true")
|
||||
frame.set_defaults(func=cmd_frame)
|
||||
|
||||
run = sub.add_parser("run", help="Run the platform-neutral Python-core workflow")
|
||||
run.add_argument("project_or_topic", help="Project slug/path or new topic")
|
||||
run.add_argument("--workers", type=int, default=6)
|
||||
run.add_argument("--profile", help="Model profile name from configs/models.yaml")
|
||||
run.add_argument("--slug", help="Project slug when project_or_topic is new")
|
||||
run.add_argument("--method", help="Research method key when project_or_topic is new")
|
||||
run.add_argument("--type", dest="report_type", default="research", help="Report type for new project")
|
||||
run.add_argument("--target-words", type=int, default=30000)
|
||||
run.add_argument("--chapters", type=int, default=10)
|
||||
run.add_argument("--input-material", action="append", default=[])
|
||||
run.add_argument("--projects-dir", help="Override projects directory")
|
||||
run.add_argument("--dry-run", action="store_true")
|
||||
run.set_defaults(func=cmd_run)
|
||||
|
||||
research = sub.add_parser("research", help="Run v0.20 task-card Phase 2")
|
||||
research.add_argument("project", help="Project slug or path")
|
||||
research.add_argument("--workers", type=int, default=6)
|
||||
research.add_argument("--axis", action="append", help="Restrict generated task axes; repeatable")
|
||||
research.add_argument("--profile", help="Model profile name from configs/models.yaml")
|
||||
research.add_argument("--model-override", action="append", default=[], metavar="ROLE=MODEL", help="Override a role model for this run; repeatable")
|
||||
research.add_argument("--execute-packets", action="store_true", help="Call model workers to fill evidence packets")
|
||||
research.add_argument("--append-task-cards", action="store_true", help="Append newly generated task cards instead of replacing phase2/task_cards.json")
|
||||
research.add_argument("--allow-search-fallback", action="store_true", help="Allow generic search fallback for specialized routes")
|
||||
research.add_argument("--build-briefs", action="store_true", help="Aggregate packets into chapter briefs")
|
||||
research.add_argument("--assemble-chapters", action="store_true", help="Call model workers to write Chinese chapter drafts")
|
||||
research.add_argument("--force", action="store_true", help="bypass Phase 1 approval gate")
|
||||
research.add_argument("--dry-run", action="store_true")
|
||||
research.set_defaults(func=cmd_research)
|
||||
|
||||
skills = sub.add_parser("skills", help="Manage canonical skills")
|
||||
skill_sub = skills.add_subparsers(dest="skills_cmd", required=True)
|
||||
skill_sub.add_parser("list", help="List canonical skills").set_defaults(func=cmd_skills)
|
||||
skill_sub.add_parser("validate", help="Validate canonical skills").set_defaults(func=cmd_skills)
|
||||
skill_sync = skill_sub.add_parser("sync", help="Sync skills into adapter directories")
|
||||
skill_sync.add_argument("--target", action="append", help="Target skill directory; repeatable")
|
||||
skill_sync.set_defaults(func=cmd_skills)
|
||||
|
||||
methods = sub.add_parser("methods", help="List and inspect research framework methods")
|
||||
method_sub = methods.add_subparsers(dest="methods_cmd", required=True)
|
||||
method_sub.add_parser("list", help="List research methods").set_defaults(func=cmd_methods)
|
||||
method_show = method_sub.add_parser("show", help="Show one research method")
|
||||
method_show.add_argument("method")
|
||||
method_show.set_defaults(func=cmd_methods)
|
||||
|
||||
status = sub.add_parser("status", help="Show project status")
|
||||
status.add_argument("project", nargs="?", help="Project slug or path")
|
||||
status.set_defaults(func=cmd_status)
|
||||
|
||||
review = sub.add_parser("review", help="Run Phase 3 review")
|
||||
review.add_argument("project", help="Project slug or path")
|
||||
review.add_argument("--model-review", action="store_true", help="Run independent model-based Phase 3 review")
|
||||
review.add_argument("--model", default="zenmux-anthropic/claude-opus-4-7", help="Model for --model-review")
|
||||
review.add_argument("--max-context-chars", type=int, default=650_000, help="Bounded context size for model review")
|
||||
review.add_argument("--dry-run", action="store_true")
|
||||
review.set_defaults(func=cmd_review)
|
||||
|
||||
prompt = sub.add_parser("prompt", help="Print a Codex command prompt template")
|
||||
prompt.add_argument("command", help="Command name, e.g. dr-frame or frame")
|
||||
prompt.add_argument("argument", nargs="?", help="Replacement for $ARGUMENTS")
|
||||
prompt.set_defaults(func=cmd_prompt)
|
||||
|
||||
glossary = sub.add_parser("glossary", help="Run glossary verification")
|
||||
glossary.add_argument("project", help="Project slug or path")
|
||||
glossary.add_argument("--workers", type=int, default=4)
|
||||
glossary.add_argument("--force", action="store_true")
|
||||
glossary.add_argument("--only")
|
||||
glossary.add_argument("--input")
|
||||
glossary.add_argument("--output")
|
||||
glossary.add_argument("--dry-run", action="store_true")
|
||||
glossary.set_defaults(func=cmd_glossary)
|
||||
|
||||
finalize = sub.add_parser("finalize", help="Run Phase 4 deterministic pipeline")
|
||||
finalize.add_argument("project", help="Project slug or path")
|
||||
finalize.add_argument("--input", default="phase4/final_zh.md", help="Chinese Markdown source for default v0.20 finalization")
|
||||
finalize.add_argument("--legacy-translate", action="store_true", help="Use legacy final_en -> translate -> polish pipeline")
|
||||
finalize.add_argument("--polish", action="store_true", help="Run optional Chinese polish step before rendering")
|
||||
finalize.add_argument("--number-citations", action="store_true", help="Convert [src_xxx] citations to numeric references before rendering")
|
||||
finalize.add_argument("--numbered-output", default="phase4/final_zh_numbered.md", help="Output path for numeric citation Markdown")
|
||||
finalize.add_argument("--report-engine", choices=["reportlab", "quarto"], default=None)
|
||||
finalize.add_argument("--no-docx", action="store_true")
|
||||
finalize.add_argument("--no-pdf", action="store_true")
|
||||
finalize.add_argument("--translate-workers", type=int, default=0)
|
||||
finalize.add_argument("--glossary-workers", type=int, default=4)
|
||||
finalize.add_argument("--polish-workers", type=int, default=0)
|
||||
finalize.add_argument(
|
||||
"--glossary-mode",
|
||||
choices=["off", "low-confidence", "full"],
|
||||
default="low-confidence",
|
||||
)
|
||||
finalize.add_argument("--model-profile", help="Model profile name from configs/models.yaml")
|
||||
finalize.add_argument(
|
||||
"--model-override",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="ROLE=MODEL",
|
||||
help="Override one role model, repeatable",
|
||||
)
|
||||
finalize.add_argument("--dry-run", action="store_true")
|
||||
finalize.set_defaults(func=cmd_finalize)
|
||||
|
||||
models = sub.add_parser("models", help="Resolve and print model profile")
|
||||
models.add_argument("--profile", help="Profile name from configs/models.yaml")
|
||||
models.add_argument("--list", action="store_true", help="List available profiles")
|
||||
models.add_argument(
|
||||
"--model-override",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="ROLE=MODEL",
|
||||
help="Override one role model, repeatable",
|
||||
)
|
||||
models.add_argument("--probe", action="store_true", help="Send tiny health checks to resolved role models")
|
||||
models.add_argument("--json", action="store_true", help="Emit JSON")
|
||||
models.set_defaults(func=cmd_models)
|
||||
|
||||
sources = sub.add_parser("sources", help="Manage source snapshots and source registry")
|
||||
sources_sub = sources.add_subparsers(dest="sources_cmd", required=True)
|
||||
sources_cache = sources_sub.add_parser("cache", help="Cache important sources as local Markdown snapshots")
|
||||
sources_cache.add_argument("project", help="Project slug or path")
|
||||
sources_cache.add_argument("--sources", default="phase2/sources.jsonl", help="Source registry path relative to project")
|
||||
sources_cache.add_argument("--all", action="store_true", help="Cache all remote sources, not only important official/Tier 1 sources")
|
||||
sources_cache.add_argument("--limit", type=int, help="Maximum sources to cache in this run")
|
||||
sources_cache.add_argument("--force", action="store_true", help="Refetch even if cached_text_path already exists")
|
||||
sources_cache.add_argument("--dry-run", action="store_true")
|
||||
sources_cache.set_defaults(func=cmd_sources)
|
||||
|
||||
apply_models = sub.add_parser("apply-models", help="Apply profile to agent files")
|
||||
apply_models.add_argument("--profile", required=True, help="Profile name from configs/models.yaml")
|
||||
apply_models.add_argument("--target", choices=["opencode", "codex", "both"], default="both")
|
||||
apply_models.add_argument(
|
||||
"--model-override",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="ROLE=MODEL",
|
||||
help="Override one role model, repeatable",
|
||||
)
|
||||
apply_models.add_argument("--dry-run", action="store_true")
|
||||
apply_models.set_defaults(func=cmd_apply_models)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
return args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export a clean Antigravity-only workspace from a full checkout."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
FILES = [
|
||||
"AGENTS.md",
|
||||
"GEMINI.md",
|
||||
"README.md",
|
||||
"docs/antigravity-clean-workspace.md",
|
||||
"docs/platform-adapters.md",
|
||||
"docs/platform-branch-strategy.md",
|
||||
"scripts/dr.py",
|
||||
"scripts/deploy_adapters.py",
|
||||
"scripts/update_platform_envs.py",
|
||||
]
|
||||
|
||||
DIRS = [
|
||||
"configs",
|
||||
"scripts/runtime",
|
||||
"scripts/reporting",
|
||||
]
|
||||
|
||||
ANTIGRAVITY_AGENT_SRC = REPO_ROOT / "platform_adapters" / "antigravity" / "agent"
|
||||
|
||||
EMPTY_DIRS = [
|
||||
"projects",
|
||||
]
|
||||
|
||||
EXCLUDED_NAMES = {
|
||||
"__pycache__",
|
||||
".pytest_cache",
|
||||
}
|
||||
|
||||
|
||||
def copy_file(src: Path, dst: Path, *, dry_run: bool) -> None:
|
||||
if dry_run:
|
||||
print(f"file {src.relative_to(REPO_ROOT)} -> {dst}")
|
||||
return
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dst)
|
||||
|
||||
|
||||
def ignore_names(_: str, names: list[str]) -> set[str]:
|
||||
return {name for name in names if name in EXCLUDED_NAMES}
|
||||
|
||||
|
||||
def copy_dir(src: Path, dst: Path, *, dry_run: bool) -> None:
|
||||
if dry_run:
|
||||
print(f"dir {src.relative_to(REPO_ROOT)} -> {dst}")
|
||||
return
|
||||
if dst.exists():
|
||||
shutil.rmtree(dst)
|
||||
shutil.copytree(src, dst, ignore=ignore_names)
|
||||
|
||||
|
||||
def export_workspace(target: Path, *, force: bool, dry_run: bool) -> None:
|
||||
target = target.expanduser().resolve()
|
||||
if target == REPO_ROOT:
|
||||
raise SystemExit("target must not be the full repository root")
|
||||
if target.exists() and any(target.iterdir()) and not force:
|
||||
raise SystemExit(f"target is not empty: {target}; use --force to replace managed files")
|
||||
|
||||
if not dry_run:
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for rel in FILES:
|
||||
src = REPO_ROOT / rel
|
||||
if not src.exists():
|
||||
raise FileNotFoundError(src)
|
||||
copy_file(src, target / rel, dry_run=dry_run)
|
||||
|
||||
for rel in DIRS:
|
||||
src = REPO_ROOT / rel
|
||||
if not src.exists():
|
||||
raise FileNotFoundError(src)
|
||||
copy_dir(src, target / rel, dry_run=dry_run)
|
||||
|
||||
if not ANTIGRAVITY_AGENT_SRC.exists():
|
||||
raise FileNotFoundError(ANTIGRAVITY_AGENT_SRC)
|
||||
copy_dir(ANTIGRAVITY_AGENT_SRC, target / ".agent", dry_run=dry_run)
|
||||
|
||||
for rel in EMPTY_DIRS:
|
||||
if dry_run:
|
||||
print(f"dir {rel} -> {target / rel}")
|
||||
else:
|
||||
(target / rel).mkdir(parents=True, exist_ok=True)
|
||||
keep = target / rel / ".gitkeep"
|
||||
keep.touch(exist_ok=True)
|
||||
|
||||
if not dry_run:
|
||||
print(f"exported clean Antigravity workspace: {target}")
|
||||
print("open this target directory in Antigravity")
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Export a clean Antigravity-only workspace")
|
||||
parser.add_argument("--target", required=True, type=Path, help="output workspace directory")
|
||||
parser.add_argument("--force", action="store_true", help="replace managed files in a non-empty target")
|
||||
parser.add_argument("--dry-run", action="store_true", help="show what would be copied")
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
export_workspace(args.target, force=args.force, dry_run=args.dry_run)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Native web grounding wrapper via ZenMux chat completions.
|
||||
|
||||
Use this when you need reproducible, model-native web search (grounding) and
|
||||
machine-readable citations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.lib.zenmux_client import ZenMuxClient, load_secrets
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Grounded web query via ZenMux")
|
||||
parser.add_argument("query", help="Question or search prompt")
|
||||
parser.add_argument("--model", default="google/gemini-3.1-flash-lite-preview")
|
||||
parser.add_argument("--max-tokens", type=int, default=2400)
|
||||
parser.add_argument("--temperature", type=float, default=0.2)
|
||||
parser.add_argument("--json", action="store_true", help="Emit JSON envelope")
|
||||
parser.add_argument("--log-file", help="Optional JSONL call log path")
|
||||
parser.add_argument("--system", default=(
|
||||
"You are a research assistant. Use web grounding when helpful. "
|
||||
"Return concise facts with explicit source-backed statements."
|
||||
))
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
load_secrets()
|
||||
log_file = Path(args.log_file) if args.log_file else None
|
||||
|
||||
with ZenMuxClient(log_file=log_file) as client:
|
||||
result = client.chat_complete_with_meta(
|
||||
model=args.model,
|
||||
system=args.system,
|
||||
user=args.query,
|
||||
temperature=args.temperature,
|
||||
max_tokens=args.max_tokens,
|
||||
web_search=True,
|
||||
web_search_options={},
|
||||
tag="ground",
|
||||
)
|
||||
|
||||
if args.json:
|
||||
payload = {
|
||||
"query": args.query,
|
||||
"model": args.model,
|
||||
"content": result["content"],
|
||||
"citations": result["citations"],
|
||||
"usage": result["usage"],
|
||||
}
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(result["content"])
|
||||
if result["citations"]:
|
||||
print("\nCitations:")
|
||||
for idx, url in enumerate(result["citations"], start=1):
|
||||
print(f"{idx}. {url}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Backward-compatible wrapper for deploying the Codex adapter.
|
||||
|
||||
v0.20 keeps Codex adapter templates in the repository, but deploys the usable
|
||||
adapter files to a Codex home outside the checkout.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.deploy_adapters import deploy_codex, print_result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Deploy Codex native adapter")
|
||||
parser.add_argument("--target", type=Path, help="Codex home target; defaults to $CODEX_HOME or ~/.codex")
|
||||
parser.add_argument("--force", action="store_true", help="overwrite existing files and create .bak backups")
|
||||
parser.add_argument("--skip-skills", action="store_true", help="do not copy canonical skills into target/skills")
|
||||
parser.add_argument("--include-config", action="store_true", help="also copy config.toml; off by default to avoid overwriting global Codex config")
|
||||
parser.add_argument("--dry-run", action="store_true", help="show files that would be written")
|
||||
args = parser.parse_args()
|
||||
|
||||
result = deploy_codex(
|
||||
target=args.target,
|
||||
force=args.force,
|
||||
skip_skills=args.skip_skills,
|
||||
include_config=args.include_config,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
print_result(result)
|
||||
print()
|
||||
print("Note: this no longer writes repository-local .codex files by default.")
|
||||
print("Run Codex from this repository after deployment:")
|
||||
if args.include_config:
|
||||
print(" codex --profile deep-research")
|
||||
else:
|
||||
print(" codex")
|
||||
print("Note: config.toml is not copied by default. Use --include-config only if you want the bundled profile.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Markdown 切块 / 合并工具。
|
||||
|
||||
把一篇 Markdown 按标题层级(# / ##)切成"翻译单元"或"润色单元",
|
||||
每个单元带稳定的 ID,便于断点续传和按需重跑。
|
||||
|
||||
核心约定:
|
||||
- H1(`# `)是一级块,通常对应 Chapter / 封面 / 前置件
|
||||
- H2(`## `)是二级块,对应一个 section 或独立前置件(Disclaimer / Executive Summary / Abstract / Glossary / References)
|
||||
- 没有任何标题的文件头(frontmatter 区)归到第 0 块
|
||||
|
||||
切块粒度默认到 H2;如果某个 H2 下的正文特别长可以再按 H3 切,但这是 polish 的事情,
|
||||
translate 一般不需要。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
HEADER_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$", re.MULTILINE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MarkdownBlock:
|
||||
"""一个翻译/润色单元。"""
|
||||
|
||||
order: int # 在原文中的顺序(0-based)
|
||||
level: int # 0 = frontmatter; 1/2/... = H1/H2/...
|
||||
title: str # 标题原文(不含 # 号);frontmatter 为空串
|
||||
anchor: str # 稳定 ID,用于断点续传(order + title hash)
|
||||
content: str # 完整内容(包含标题行本身,除 frontmatter 块外)
|
||||
parent_order: int | None = None # H2 的父 H1 order;H1 为 None
|
||||
word_count: int = 0 # 英文 word count 估算(只含 a-z)
|
||||
char_count: int = 0 # 字符数(含中文)
|
||||
meta: dict = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def short_title(self) -> str:
|
||||
t = self.title.strip()
|
||||
if len(t) <= 50:
|
||||
return t
|
||||
return t[:47] + "..."
|
||||
|
||||
|
||||
def _count_words(text: str) -> int:
|
||||
return len(re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)*", text))
|
||||
|
||||
|
||||
def _stable_anchor(order: int, title: str) -> str:
|
||||
"""order + title 生成稳定短 ID。同一文件改顺序不变,改标题重算。"""
|
||||
h = hashlib.sha1(title.strip().encode("utf-8")).hexdigest()[:8]
|
||||
return f"b{order:03d}-{h}"
|
||||
|
||||
|
||||
def split_by_headers(text: str, max_level: int = 2) -> list[MarkdownBlock]:
|
||||
"""把 Markdown 按 H1..H{max_level} 切块。
|
||||
|
||||
返回的 block 按出现顺序排列。第 0 块可能是 frontmatter(level=0,无 title)。
|
||||
"""
|
||||
lines = text.splitlines(keepends=False)
|
||||
# 先找出所有 header 位置
|
||||
header_positions: list[tuple[int, int, str]] = [] # (line_idx, level, title)
|
||||
in_code_block = False
|
||||
for i, ln in enumerate(lines):
|
||||
if ln.lstrip().startswith("```"):
|
||||
in_code_block = not in_code_block
|
||||
continue
|
||||
if in_code_block:
|
||||
continue
|
||||
m = re.match(r"^(#{1,6})\s+(.+?)\s*$", ln)
|
||||
if m:
|
||||
level = len(m.group(1))
|
||||
if level <= max_level:
|
||||
header_positions.append((i, level, m.group(2)))
|
||||
|
||||
blocks: list[MarkdownBlock] = []
|
||||
order = 0
|
||||
|
||||
# frontmatter:第一个 header 前的所有内容
|
||||
first_header_line = header_positions[0][0] if header_positions else len(lines)
|
||||
frontmatter_text = "\n".join(lines[:first_header_line]).rstrip()
|
||||
if frontmatter_text.strip():
|
||||
blocks.append(
|
||||
MarkdownBlock(
|
||||
order=order,
|
||||
level=0,
|
||||
title="",
|
||||
anchor=_stable_anchor(order, "__frontmatter__"),
|
||||
content=frontmatter_text,
|
||||
word_count=_count_words(frontmatter_text),
|
||||
char_count=len(frontmatter_text),
|
||||
)
|
||||
)
|
||||
order += 1
|
||||
|
||||
# 为每个 header 建一个 block:内容 = 本 header 行 + 下一 header 行前所有内容
|
||||
last_h1_order: int | None = None
|
||||
for idx, (line_idx, level, title) in enumerate(header_positions):
|
||||
end_line = (
|
||||
header_positions[idx + 1][0]
|
||||
if idx + 1 < len(header_positions)
|
||||
else len(lines)
|
||||
)
|
||||
content = "\n".join(lines[line_idx:end_line]).rstrip()
|
||||
block = MarkdownBlock(
|
||||
order=order,
|
||||
level=level,
|
||||
title=title,
|
||||
anchor=_stable_anchor(order, title),
|
||||
content=content,
|
||||
parent_order=last_h1_order if level > 1 else None,
|
||||
word_count=_count_words(content),
|
||||
char_count=len(content),
|
||||
)
|
||||
blocks.append(block)
|
||||
if level == 1:
|
||||
last_h1_order = order
|
||||
order += 1
|
||||
return blocks
|
||||
|
||||
|
||||
def merge_blocks(blocks: Iterable[MarkdownBlock], separator: str = "\n\n") -> str:
|
||||
"""按 order 拼回完整 Markdown。"""
|
||||
return separator.join(b.content for b in sorted(blocks, key=lambda x: x.order))
|
||||
|
||||
|
||||
def chapter_stem_from_title(title: str) -> str:
|
||||
"""从 `# Chapter 1 — foo bar` 生成文件名 stem 如 `ch01`。找不到就 fallback。"""
|
||||
m = re.search(r"chapter\s+(\d+)", title, re.IGNORECASE)
|
||||
if m:
|
||||
return f"ch{int(m.group(1)):02d}"
|
||||
# Executive Summary / Abstract / Glossary / Disclaimer / References / Version History
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")
|
||||
return slug[:40] or "untitled"
|
||||
|
||||
|
||||
def count_chinese_chars(text: str) -> int:
|
||||
return sum(1 for c in text if "\u4e00" <= c <= "\u9fff")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
src = Path(sys.argv[1])
|
||||
for b in split_by_headers(src.read_text(encoding="utf-8")):
|
||||
print(f"[{b.order:3d}] L{b.level} {b.word_count:>5}w {b.char_count:>6}c {b.short_title}")
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Model profile loading and resolution utilities."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_MODEL_CONFIG = REPO_ROOT / "configs" / "models.yaml"
|
||||
LEGACY_MODEL_CONFIG = REPO_ROOT / "configs" / "model_profiles.yaml"
|
||||
|
||||
|
||||
class ModelConfigError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def load_model_config(path: Path | None = None) -> dict[str, Any]:
|
||||
cfg_path = path or DEFAULT_MODEL_CONFIG
|
||||
if not cfg_path.exists() and LEGACY_MODEL_CONFIG.exists():
|
||||
cfg_path = LEGACY_MODEL_CONFIG
|
||||
if not cfg_path.exists():
|
||||
raise ModelConfigError(f"model config not found: {cfg_path}")
|
||||
try:
|
||||
data = yaml.safe_load(cfg_path.read_text(encoding="utf-8")) or {}
|
||||
except Exception as exc:
|
||||
raise ModelConfigError(f"invalid YAML in {cfg_path}: {exc}") from exc
|
||||
if not isinstance(data, dict):
|
||||
raise ModelConfigError(f"invalid model config shape in {cfg_path}")
|
||||
return data
|
||||
|
||||
|
||||
def resolve_model_profile(
|
||||
*,
|
||||
profile: str | None = None,
|
||||
overrides: dict[str, str] | None = None,
|
||||
path: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
cfg = load_model_config(path)
|
||||
profiles = cfg.get("profiles") or {}
|
||||
defaults = cfg.get("defaults") or {}
|
||||
selected = profile or defaults.get("profile")
|
||||
if not selected:
|
||||
raise ModelConfigError("no model profile provided and no defaults.profile set")
|
||||
if selected not in profiles:
|
||||
raise ModelConfigError(f"unknown model profile: {selected}")
|
||||
|
||||
selected_profile = profiles[selected] or {}
|
||||
roles = dict(selected_profile.get("roles") or {})
|
||||
task_types = dict(selected_profile.get("task_types") or defaults.get("task_types") or {})
|
||||
if defaults.get("script_models"):
|
||||
for role, model in (defaults.get("script_models") or {}).items():
|
||||
roles.setdefault(role, model)
|
||||
for role, model in (overrides or {}).items():
|
||||
roles[role] = model
|
||||
|
||||
return {
|
||||
"profile": selected,
|
||||
"description": selected_profile.get("description", ""),
|
||||
"roles": roles,
|
||||
"task_types": task_types,
|
||||
}
|
||||
|
||||
|
||||
def list_model_profiles(path: Path | None = None) -> list[str]:
|
||||
cfg = load_model_config(path)
|
||||
profiles = cfg.get("profiles") or {}
|
||||
return sorted(profiles.keys())
|
||||
|
||||
|
||||
def parse_model_overrides(items: list[str] | None) -> dict[str, str]:
|
||||
out: dict[str, str] = {}
|
||||
for item in items or []:
|
||||
if "=" not in item:
|
||||
raise ModelConfigError(f"invalid override '{item}', expected role=model")
|
||||
role, model = item.split("=", 1)
|
||||
role = role.strip()
|
||||
model = model.strip()
|
||||
if not role or not model:
|
||||
raise ModelConfigError(f"invalid override '{item}', expected role=model")
|
||||
out[role] = model
|
||||
return out
|
||||
@@ -0,0 +1,427 @@
|
||||
"""通用搜索客户端(Tavily / Exa / Brave / Serper 路由)。
|
||||
|
||||
为 build_glossary.py 这类术语核查场景服务。
|
||||
|
||||
关键设计:
|
||||
- `trust_env=False` 绕开系统 socks 代理(Clash on macOS 配 socks5 时 httpx 会 TLS EOF)
|
||||
- 专利 / Scholar / News 优先 Serper,保证 Google Patents / Google Scholar 路径被真正调用
|
||||
- 通用网页 Tavily 优先,Exa/Brave fallback
|
||||
- 证据发现 Exa 优先,用 highlights/text 摘录喂给 evidence packet
|
||||
- 遇到配额问题自动降级或返回 empty
|
||||
- 不做深度 crawl,只要摘要
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchHit:
|
||||
title: str
|
||||
url: str
|
||||
snippet: str
|
||||
|
||||
|
||||
class SearchError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class ExaClient:
|
||||
def __init__(self, api_key: str | None = None, timeout: float = 30.0) -> None:
|
||||
self.api_key = api_key or os.environ.get("EXA_API_KEY")
|
||||
if not self.api_key:
|
||||
raise SearchError("EXA_API_KEY not set")
|
||||
# trust_env=False 关键:不吃系统代理,避免 TLS EOF
|
||||
self._client = httpx.Client(trust_env=False, timeout=timeout)
|
||||
|
||||
def close(self) -> None:
|
||||
self._client.close()
|
||||
|
||||
def __enter__(self) -> "ExaClient":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: Any) -> None:
|
||||
self.close()
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
num_results: int = 5,
|
||||
search_type: str = "auto",
|
||||
category: str | None = None,
|
||||
use_highlights: bool = False,
|
||||
max_characters: int = 800,
|
||||
) -> list[SearchHit]:
|
||||
body = {
|
||||
"query": query,
|
||||
"numResults": num_results,
|
||||
"type": search_type,
|
||||
"contents": {"text": {"maxCharacters": max_characters}},
|
||||
}
|
||||
if category:
|
||||
body["category"] = category
|
||||
if use_highlights:
|
||||
body["contents"]["highlights"] = {
|
||||
"numSentences": 2,
|
||||
"highlightsPerUrl": 3,
|
||||
}
|
||||
r = self._client.post(
|
||||
"https://api.exa.ai/search",
|
||||
json=body,
|
||||
headers={"x-api-key": self.api_key, "Content-Type": "application/json"},
|
||||
)
|
||||
if r.status_code != 200:
|
||||
raise SearchError(f"Exa HTTP {r.status_code}: {r.text[:200]}")
|
||||
data = r.json()
|
||||
out: list[SearchHit] = []
|
||||
for item in data.get("results", [])[:num_results]:
|
||||
highlights = item.get("highlights") or []
|
||||
text = item.get("text") or item.get("snippet") or ""
|
||||
if highlights:
|
||||
text = " | ".join(str(h).strip() for h in highlights if str(h).strip())
|
||||
out.append(
|
||||
SearchHit(
|
||||
title=(item.get("title") or "")[:200],
|
||||
url=item.get("url") or "",
|
||||
snippet=text[:1000],
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
class TavilyClient:
|
||||
def __init__(self, api_key: str | None = None, timeout: float = 30.0) -> None:
|
||||
self.api_key = api_key or os.environ.get("TAVILY_API_KEY")
|
||||
if not self.api_key:
|
||||
raise SearchError("TAVILY_API_KEY not set")
|
||||
self._client = httpx.Client(trust_env=False, timeout=timeout)
|
||||
|
||||
def close(self) -> None:
|
||||
self._client.close()
|
||||
|
||||
def __enter__(self) -> "TavilyClient":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: Any) -> None:
|
||||
self.close()
|
||||
|
||||
def search(self, query: str, *, num_results: int = 5) -> list[SearchHit]:
|
||||
body = {
|
||||
"api_key": self.api_key,
|
||||
"query": query,
|
||||
"search_depth": "basic",
|
||||
"max_results": num_results,
|
||||
"include_answer": False,
|
||||
"include_raw_content": False,
|
||||
}
|
||||
r = self._client.post("https://api.tavily.com/search", json=body)
|
||||
if r.status_code != 200:
|
||||
raise SearchError(f"Tavily HTTP {r.status_code}: {r.text[:200]}")
|
||||
data = r.json()
|
||||
out: list[SearchHit] = []
|
||||
for item in data.get("results", [])[:num_results]:
|
||||
out.append(
|
||||
SearchHit(
|
||||
title=(item.get("title") or "")[:200],
|
||||
url=item.get("url") or "",
|
||||
snippet=(item.get("content") or "")[:600],
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
class BraveClient:
|
||||
def __init__(self, api_key: str | None = None, timeout: float = 30.0) -> None:
|
||||
self.api_key = api_key or os.environ.get("BRAVE_API_KEY")
|
||||
if not self.api_key:
|
||||
raise SearchError("BRAVE_API_KEY not set")
|
||||
self._client = httpx.Client(trust_env=False, timeout=timeout)
|
||||
|
||||
def close(self) -> None:
|
||||
self._client.close()
|
||||
|
||||
def __enter__(self) -> "BraveClient":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: Any) -> None:
|
||||
self.close()
|
||||
|
||||
def search(self, query: str, *, num_results: int = 5) -> list[SearchHit]:
|
||||
r = self._client.get(
|
||||
"https://api.search.brave.com/res/v1/web/search",
|
||||
params={"q": query, "count": min(max(num_results, 1), 20)},
|
||||
headers={
|
||||
"X-Subscription-Token": self.api_key,
|
||||
"Accept": "application/json",
|
||||
},
|
||||
)
|
||||
if r.status_code != 200:
|
||||
raise SearchError(f"Brave HTTP {r.status_code}: {r.text[:200]}")
|
||||
data = r.json()
|
||||
out: list[SearchHit] = []
|
||||
for item in (data.get("web") or {}).get("results", [])[:num_results]:
|
||||
out.append(
|
||||
SearchHit(
|
||||
title=(item.get("title") or "")[:200],
|
||||
url=item.get("url") or "",
|
||||
snippet=(item.get("description") or "")[:600],
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
class SearchClient:
|
||||
"""统一搜索门面,支持多路由:
|
||||
|
||||
- `search(query)`:通用网页搜索,优先 Tavily → Exa → Brave
|
||||
- `evidence(query)`:证据发现,优先 Exa highlights → Tavily → Brave
|
||||
- `patents(query)`:专利检索,走 Serper(Google Patents);失败则通用搜索补刀
|
||||
- `scholar(query)`:学术论文,走 Serper Scholar;失败则通用搜索补刀
|
||||
- `news(query)`:新闻检索,走 Serper News;失败则通用搜索补刀
|
||||
|
||||
所有客户端都延迟导入 serper_client,避免没装 SERPAPI_KEY 时 import 炸。
|
||||
"""
|
||||
|
||||
def __init__(self, *, strict_specialized: bool = True) -> None:
|
||||
self._exa: ExaClient | None = None
|
||||
self._tavily: TavilyClient | None = None
|
||||
self._brave: BraveClient | None = None
|
||||
self._serper = None # 惰性实例化
|
||||
self.strict_specialized = strict_specialized
|
||||
try:
|
||||
self._exa = ExaClient()
|
||||
except SearchError:
|
||||
pass
|
||||
try:
|
||||
self._tavily = TavilyClient()
|
||||
except SearchError:
|
||||
pass
|
||||
try:
|
||||
self._brave = BraveClient()
|
||||
except SearchError:
|
||||
pass
|
||||
self._has_serper_key = bool(os.environ.get("SERPER_API_KEY") or os.environ.get("SERPAPI_KEY"))
|
||||
if not (self._exa or self._tavily or self._brave or self._has_serper_key):
|
||||
raise SearchError("no search API key available: set SERPER_API_KEY, SERPAPI_KEY, EXA_API_KEY, TAVILY_API_KEY, or BRAVE_API_KEY")
|
||||
|
||||
def _get_serper(self):
|
||||
"""惰性创建 SerperClient。没 key 时返回 None。"""
|
||||
if self._serper is False:
|
||||
return None
|
||||
if self._serper is None:
|
||||
try:
|
||||
from scripts.lib.serper_client import SerperClient
|
||||
self._serper = SerperClient()
|
||||
except Exception:
|
||||
self._serper = False
|
||||
return None
|
||||
return self._serper
|
||||
|
||||
def close(self) -> None:
|
||||
if self._exa:
|
||||
self._exa.close()
|
||||
if self._tavily:
|
||||
self._tavily.close()
|
||||
if self._brave:
|
||||
self._brave.close()
|
||||
if self._serper and self._serper is not False:
|
||||
self._serper.close()
|
||||
|
||||
def __enter__(self) -> "SearchClient":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: Any) -> None:
|
||||
self.close()
|
||||
|
||||
def search(self, query: str, *, num_results: int = 5) -> list[SearchHit]:
|
||||
"""通用网页搜索。Tavily 首选,Exa/Brave 备选。"""
|
||||
if self._tavily:
|
||||
try:
|
||||
return self._tavily.search(query, num_results=num_results)
|
||||
except SearchError:
|
||||
pass
|
||||
if self._exa:
|
||||
try:
|
||||
return self._exa.search(query, num_results=num_results)
|
||||
except SearchError:
|
||||
pass
|
||||
if self._brave:
|
||||
try:
|
||||
return self._brave.search(query, num_results=num_results)
|
||||
except SearchError:
|
||||
pass
|
||||
return []
|
||||
|
||||
def evidence(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
num_results: int = 10,
|
||||
category: str | None = None,
|
||||
) -> list[SearchHit]:
|
||||
"""Evidence discovery route.
|
||||
|
||||
Exa is better suited for agent-facing evidence discovery because it can
|
||||
return concise highlights/text per URL. Results are still candidate
|
||||
sources only; downstream packets must score and trace important hits
|
||||
back to original Tier 1-2 sources before making final claims.
|
||||
"""
|
||||
if self._exa:
|
||||
try:
|
||||
return self._exa.search(
|
||||
query,
|
||||
num_results=num_results,
|
||||
search_type="auto",
|
||||
category=category,
|
||||
use_highlights=True,
|
||||
max_characters=1200,
|
||||
)
|
||||
except SearchError:
|
||||
pass
|
||||
if self._tavily:
|
||||
try:
|
||||
return self._tavily.search(query, num_results=num_results)
|
||||
except SearchError:
|
||||
pass
|
||||
if self._brave:
|
||||
try:
|
||||
return self._brave.search(query, num_results=num_results)
|
||||
except SearchError:
|
||||
pass
|
||||
return []
|
||||
|
||||
def patents(self, query: str, *, num_results: int = 10) -> list[SearchHit]:
|
||||
"""专利检索:Serper 走 Google Patents 最准。降级到通用搜索 + site 限定。"""
|
||||
serper = self._get_serper()
|
||||
if serper:
|
||||
try:
|
||||
hits = serper.patents(query, num_results=num_results)
|
||||
return [SearchHit(h.title, h.url, h.snippet) for h in hits]
|
||||
except Exception as exc:
|
||||
if self.strict_specialized:
|
||||
raise SearchError(f"serper patents failed: {exc}") from exc
|
||||
# 降级:通用搜索加 site 限定
|
||||
if self.strict_specialized:
|
||||
raise SearchError("serper unavailable for patents route; refusing silent fallback")
|
||||
return self.search(f"site:patents.google.com {query}", num_results=num_results)
|
||||
|
||||
def scholar(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
num_results: int = 10,
|
||||
year_low: int | None = None,
|
||||
) -> list[SearchHit]:
|
||||
"""学术论文:Serper Scholar 带引用数。降级到通用搜索。"""
|
||||
serper = self._get_serper()
|
||||
if serper:
|
||||
try:
|
||||
hits = serper.scholar(query, num_results=num_results, year_low=year_low)
|
||||
return [
|
||||
SearchHit(
|
||||
title=h.title,
|
||||
url=h.url,
|
||||
snippet=f"{h.snippet} | {h.source} | 引用 {h.cited_by}" if h.cited_by else h.snippet,
|
||||
)
|
||||
for h in hits
|
||||
]
|
||||
except Exception as exc:
|
||||
if self.strict_specialized:
|
||||
raise SearchError(f"serper scholar failed: {exc}") from exc
|
||||
if self.strict_specialized:
|
||||
raise SearchError("serper unavailable for scholar route; refusing silent fallback")
|
||||
return self.search(query, num_results=num_results)
|
||||
|
||||
def news(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
num_results: int = 10,
|
||||
time_range: str | None = None,
|
||||
) -> list[SearchHit]:
|
||||
"""新闻检索:Serper News。降级到通用搜索。"""
|
||||
serper = self._get_serper()
|
||||
if serper:
|
||||
try:
|
||||
hits = serper.news(query, num_results=num_results, time_range=time_range)
|
||||
return [
|
||||
SearchHit(
|
||||
title=h.title,
|
||||
url=h.url,
|
||||
snippet=f"{h.snippet} | {h.source} | {h.date}" if h.date else h.snippet,
|
||||
)
|
||||
for h in hits
|
||||
]
|
||||
except Exception as exc:
|
||||
if self.strict_specialized:
|
||||
raise SearchError(f"serper news failed: {exc}") from exc
|
||||
if self.strict_specialized:
|
||||
raise SearchError("serper unavailable for news route; refusing silent fallback")
|
||||
return self.search(query, num_results=num_results)
|
||||
|
||||
def fda(self, query: str, *, num_results: int = 10) -> list[SearchHit]:
|
||||
"""FDA-focused discovery for warning letters and meeting records.
|
||||
|
||||
FDA enforcement examples are often more useful for GMP remediation than
|
||||
generic web pages, so this route biases discovery toward warning
|
||||
letters, inspection/enforcement pages, and meeting materials/minutes.
|
||||
"""
|
||||
def fda_only(hits: list[SearchHit]) -> list[SearchHit]:
|
||||
return [hit for hit in hits if "fda.gov" in (hit.url or "").lower()]
|
||||
|
||||
focused_queries = [
|
||||
f'site:fda.gov "Warning Letter" GMP pharmaceutical {query}',
|
||||
f'site:fda.gov/inspections-compliance-enforcement-and-criminal-investigations "Warning Letter" {query}',
|
||||
f'site:fda.gov "meeting materials" "pharmaceutical quality" {query}',
|
||||
f'site:fda.gov "meeting minutes" FDA pharmaceutical quality {query}',
|
||||
]
|
||||
hits: list[SearchHit] = []
|
||||
seen: set[str] = set()
|
||||
per_query = max(2, min(num_results, 4))
|
||||
for focused_query in focused_queries:
|
||||
route_hits: list[SearchHit] = []
|
||||
serper = self._get_serper()
|
||||
if serper:
|
||||
try:
|
||||
route_hits = [
|
||||
SearchHit(h.title, h.url, h.snippet)
|
||||
for h in serper.search(focused_query, num_results=per_query)
|
||||
]
|
||||
except Exception as exc:
|
||||
if self.strict_specialized:
|
||||
raise SearchError(f"serper FDA search failed: {exc}") from exc
|
||||
if not route_hits and not self.strict_specialized:
|
||||
route_hits = self.search(focused_query, num_results=per_query)
|
||||
for hit in fda_only(route_hits):
|
||||
key = hit.url or hit.title
|
||||
if not key or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
hits.append(hit)
|
||||
if len(hits) >= num_results:
|
||||
return hits
|
||||
return hits
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from scripts.lib.zenmux_client import load_secrets
|
||||
load_secrets()
|
||||
with SearchClient() as c:
|
||||
print("--- 通用: Mabwell 迈威生物 ---")
|
||||
for h in c.search("Mabwell 迈威生物 biopharmaceutical", num_results=3):
|
||||
print(f" {h.title[:80]}")
|
||||
print(f" {h.url}")
|
||||
print("\n--- 专利: dual-target siRNA ---")
|
||||
for h in c.patents("dual-target siRNA GalNAc", num_results=3):
|
||||
print(f" {h.title[:80]}")
|
||||
print(f" {h.url}")
|
||||
print("\n--- Scholar: dual-target RNAi 2024 ---")
|
||||
for h in c.scholar("dual-target RNAi drug", num_results=3, year_low=2023):
|
||||
print(f" {h.title[:80]}")
|
||||
print(f" {h.url}")
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Serper.dev 客户端(Google Search API 代理)。
|
||||
|
||||
为什么用 Serper:
|
||||
- 2500 次免费额度,远超 SerpAPI 的 100/月
|
||||
- 支持 Google Search、Scholar、News、Images、Maps
|
||||
- Google Patents 无专用 endpoint,但可用 `site:patents.google.com` 技巧
|
||||
- 价格比 SerpAPI 便宜 3-5×
|
||||
|
||||
用途:
|
||||
- 专利检索:通用 search + `site:patents.google.com`
|
||||
- 学术论文:/scholar endpoint
|
||||
- 新闻:/news endpoint(时效性敏感的行业动态)
|
||||
|
||||
httpx 客户端使用 trust_env=False 绕过系统 socks 代理(macOS Clash 会导致 TLS EOF)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
SERPER_BASE = "https://google.serper.dev"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SerperHit:
|
||||
title: str
|
||||
url: str
|
||||
snippet: str
|
||||
source: str = "" # 论文出处 / 新闻媒体
|
||||
date: str = "" # 发表日期(如 scholar / news 返回的话)
|
||||
cited_by: int = 0 # 学术论文的引用数(仅 scholar)
|
||||
|
||||
|
||||
class SerperError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class SerperClient:
|
||||
def __init__(self, api_key: str | None = None, timeout: float = 30.0) -> None:
|
||||
self.api_key = api_key or os.environ.get("SERPAPI_KEY") or os.environ.get("SERPER_API_KEY")
|
||||
if not self.api_key:
|
||||
raise SerperError("SERPAPI_KEY / SERPER_API_KEY not set")
|
||||
self._client = httpx.Client(trust_env=False, timeout=timeout)
|
||||
|
||||
def close(self) -> None:
|
||||
self._client.close()
|
||||
|
||||
def __enter__(self) -> "SerperClient":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: Any) -> None:
|
||||
self.close()
|
||||
|
||||
def _post(self, path: str, body: dict) -> dict:
|
||||
try:
|
||||
r = self._client.post(
|
||||
f"{SERPER_BASE}{path}",
|
||||
json=body,
|
||||
headers={
|
||||
"X-API-KEY": self.api_key,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
except httpx.RequestError as e:
|
||||
raise SerperError(f"network error: {e}")
|
||||
if r.status_code != 200:
|
||||
raise SerperError(f"HTTP {r.status_code}: {r.text[:300]}")
|
||||
try:
|
||||
return r.json()
|
||||
except Exception as e:
|
||||
raise SerperError(f"invalid JSON: {e}")
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
num_results: int = 10,
|
||||
gl: str = "us",
|
||||
hl: str = "en",
|
||||
) -> list[SerperHit]:
|
||||
"""通用 Google 搜索。支持 site: / filetype: / 引号短语等 Google 高级语法。"""
|
||||
data = self._post("/search", {
|
||||
"q": query,
|
||||
"num": num_results,
|
||||
"gl": gl,
|
||||
"hl": hl,
|
||||
})
|
||||
hits: list[SerperHit] = []
|
||||
for item in (data.get("organic") or [])[:num_results]:
|
||||
hits.append(SerperHit(
|
||||
title=(item.get("title") or "")[:200],
|
||||
url=item.get("link") or "",
|
||||
snippet=(item.get("snippet") or "")[:600],
|
||||
date=item.get("date") or "",
|
||||
))
|
||||
return hits
|
||||
|
||||
def scholar(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
num_results: int = 10,
|
||||
year_low: int | None = None,
|
||||
year_high: int | None = None,
|
||||
) -> list[SerperHit]:
|
||||
"""Google Scholar 搜索——学术论文首选。
|
||||
|
||||
返回带引用数、发表年份等元数据,权威信源识别更准确。
|
||||
"""
|
||||
body: dict[str, Any] = {"q": query, "num": num_results}
|
||||
if year_low is not None:
|
||||
body["tbs"] = f"cdr:1,cd_min:{year_low}" + (f",cd_max:{year_high}" if year_high else "")
|
||||
data = self._post("/scholar", body)
|
||||
hits: list[SerperHit] = []
|
||||
for item in (data.get("organic") or [])[:num_results]:
|
||||
hits.append(SerperHit(
|
||||
title=(item.get("title") or "")[:200],
|
||||
url=item.get("link") or "",
|
||||
snippet=(item.get("snippet") or "")[:600],
|
||||
source=(item.get("publicationInfo") or "")[:200],
|
||||
year=item.get("year") or "",
|
||||
cited_by=item.get("citedBy") or 0,
|
||||
) if False else SerperHit(
|
||||
title=(item.get("title") or "")[:200],
|
||||
url=item.get("link") or "",
|
||||
snippet=(item.get("snippet") or "")[:600],
|
||||
source=(item.get("publicationInfo") or "")[:200],
|
||||
date=str(item.get("year") or ""),
|
||||
cited_by=item.get("citedBy") or 0,
|
||||
))
|
||||
return hits
|
||||
|
||||
def patents(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
num_results: int = 10,
|
||||
) -> list[SerperHit]:
|
||||
"""Google Patents 检索——用 site: 技巧走通用搜索。
|
||||
|
||||
serper.dev 没有专门的 patents endpoint,但 `site:patents.google.com` 效果很好。
|
||||
"""
|
||||
combined = f"site:patents.google.com {query}"
|
||||
return self.search(combined, num_results=num_results)
|
||||
|
||||
def news(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
num_results: int = 10,
|
||||
time_range: Literal["d", "w", "m", "y"] | None = None,
|
||||
) -> list[SerperHit]:
|
||||
"""Google News 搜索——时效敏感行业动态。
|
||||
|
||||
time_range: d=24h, w=7d, m=30d, y=1y
|
||||
"""
|
||||
body: dict[str, Any] = {"q": query, "num": num_results}
|
||||
if time_range:
|
||||
body["tbs"] = f"qdr:{time_range}"
|
||||
data = self._post("/news", body)
|
||||
hits: list[SerperHit] = []
|
||||
for item in (data.get("news") or [])[:num_results]:
|
||||
hits.append(SerperHit(
|
||||
title=(item.get("title") or "")[:200],
|
||||
url=item.get("link") or "",
|
||||
snippet=(item.get("snippet") or "")[:600],
|
||||
source=(item.get("source") or "")[:200],
|
||||
date=item.get("date") or "",
|
||||
))
|
||||
return hits
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from scripts.lib.zenmux_client import load_secrets
|
||||
load_secrets()
|
||||
with SerperClient() as c:
|
||||
print("=== Patents: dual-target siRNA ===")
|
||||
for h in c.patents("dual-target siRNA GalNAc conjugate", num_results=3):
|
||||
print(f" {h.title[:70]}")
|
||||
print(f" {h.url}")
|
||||
print("\n=== Scholar: dual-target RNAi ===")
|
||||
for h in c.scholar("dual-target RNAi drug 2024", num_results=3):
|
||||
print(f" {h.title[:70]} [引用 {h.cited_by}] ({h.date})")
|
||||
print(f" {h.url}")
|
||||
print(f" 源: {h.source[:80]}")
|
||||
@@ -0,0 +1,423 @@
|
||||
"""ZenMux API 客户端。
|
||||
|
||||
直接 HTTP 调用 zenmux 的 OpenAI 兼容端点。为 Phase 4 的 Python 化脚本服务
|
||||
(translate.py / polish.py)。所有调用都走 /api/v1/chat/completions。
|
||||
|
||||
设计原则:
|
||||
- 独立于 opencode,可直接在 CLI / CI / cron 运行
|
||||
- 内置重试(指数退避)、限流、token 统计、结构化日志
|
||||
- 失败快速可见:打印请求 ID,便于在 zenmux 后台对账
|
||||
- 默认读 secrets.env 里的 ZENMUX_API_KEY
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
DEFAULT_BASE_URL = "https://zenmux.ai/api/v1"
|
||||
DEFAULT_TIMEOUT = 300.0 # 翻译/润色单次调用可能 60s+,留够余量
|
||||
MAX_RETRIES = 5
|
||||
RETRYABLE_STATUSES = {408, 429, 500, 502, 503, 504, 520, 524}
|
||||
|
||||
|
||||
_ANTHROPIC_MODEL_ALIASES = {
|
||||
"anthropic/claude-opus-4-7": "anthropic/claude-opus-4.7",
|
||||
"anthropic/claude-opus-4-6": "anthropic/claude-opus-4.6",
|
||||
"anthropic/claude-opus-4-5": "anthropic/claude-opus-4.5",
|
||||
"anthropic/claude-opus-4-1": "anthropic/claude-opus-4.1",
|
||||
"anthropic/claude-sonnet-4-6": "anthropic/claude-sonnet-4.6",
|
||||
"anthropic/claude-sonnet-4-5": "anthropic/claude-sonnet-4.5",
|
||||
"anthropic/claude-haiku-4-5": "anthropic/claude-haiku-4.5",
|
||||
}
|
||||
_MODELS_WITHOUT_TEMPERATURE = {
|
||||
"anthropic/claude-opus-4.7",
|
||||
}
|
||||
|
||||
|
||||
def normalize_zenmux_model(model: str) -> str:
|
||||
"""Convert adapter-facing model IDs to ZenMux OpenAI API model IDs."""
|
||||
normalized = model.strip()
|
||||
if normalized.startswith("zenmux-anthropic/"):
|
||||
normalized = "anthropic/" + normalized.removeprefix("zenmux-anthropic/")
|
||||
elif normalized.startswith("zenmux/"):
|
||||
normalized = normalized.removeprefix("zenmux/")
|
||||
return _ANTHROPIC_MODEL_ALIASES.get(normalized, normalized)
|
||||
|
||||
|
||||
def model_accepts_temperature(model: str) -> bool:
|
||||
"""Return whether the ZenMux API accepts `temperature` for this model."""
|
||||
return normalize_zenmux_model(model) not in _MODELS_WITHOUT_TEMPERATURE
|
||||
|
||||
|
||||
@dataclass
|
||||
class UsageStats:
|
||||
"""聚合一次脚本运行的 token 消耗。"""
|
||||
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
cache_creation_tokens: int = 0
|
||||
cache_read_tokens: int = 0
|
||||
call_count: int = 0
|
||||
failed_calls: int = 0
|
||||
by_model: dict[str, dict[str, int]] = field(default_factory=dict)
|
||||
|
||||
def add(self, model: str, usage: dict[str, Any]) -> None:
|
||||
p = usage.get("prompt_tokens", 0) or 0
|
||||
c = usage.get("completion_tokens", 0) or 0
|
||||
cc = (
|
||||
usage.get("cache_creation_input_tokens", 0)
|
||||
or usage.get("prompt_tokens_details", {}).get("cached_tokens", 0)
|
||||
or 0
|
||||
)
|
||||
cr = (
|
||||
usage.get("cache_read_input_tokens", 0)
|
||||
or 0
|
||||
)
|
||||
self.prompt_tokens += p
|
||||
self.completion_tokens += c
|
||||
self.cache_creation_tokens += cc
|
||||
self.cache_read_tokens += cr
|
||||
self.call_count += 1
|
||||
m = self.by_model.setdefault(
|
||||
model,
|
||||
{"prompt": 0, "completion": 0, "cache_creation": 0, "cache_read": 0, "calls": 0},
|
||||
)
|
||||
m["prompt"] += p
|
||||
m["completion"] += c
|
||||
m["cache_creation"] += cc
|
||||
m["cache_read"] += cr
|
||||
m["calls"] += 1
|
||||
|
||||
def summary(self) -> str:
|
||||
lines = [
|
||||
f"Total calls: {self.call_count} (failed: {self.failed_calls})",
|
||||
f"Prompt tokens: {self.prompt_tokens:>12,}",
|
||||
f"Completion tokens: {self.completion_tokens:>12,}",
|
||||
f"Cache creation: {self.cache_creation_tokens:>12,}",
|
||||
f"Cache read: {self.cache_read_tokens:>12,}",
|
||||
]
|
||||
for model, s in self.by_model.items():
|
||||
lines.append(
|
||||
f" [{model}] calls={s['calls']} in={s['prompt']:,} "
|
||||
f"out={s['completion']:,} cache_w={s['cache_creation']:,} "
|
||||
f"cache_r={s['cache_read']:,}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class ZenMuxError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class ZenMuxClient:
|
||||
"""ZenMux 轻量客户端。
|
||||
|
||||
只暴露一个方法 `chat_complete()`,屏蔽 httpx 细节。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str | None = None,
|
||||
base_url: str | None = None,
|
||||
timeout: float = DEFAULT_TIMEOUT,
|
||||
log_file: Path | None = None,
|
||||
) -> None:
|
||||
self.api_key = api_key or os.environ.get("ZENMUX_API_KEY")
|
||||
if not self.api_key:
|
||||
raise ZenMuxError(
|
||||
"ZENMUX_API_KEY not set. Source secrets.env or pass api_key explicitly."
|
||||
)
|
||||
self.base_url = (base_url or os.environ.get("ZENMUX_BASE_URL") or DEFAULT_BASE_URL).rstrip("/")
|
||||
self.timeout = timeout
|
||||
self.log_file = log_file
|
||||
self.usage = UsageStats()
|
||||
self._client = httpx.Client(timeout=timeout)
|
||||
self._log_lock = threading.Lock()
|
||||
self._usage_lock = threading.Lock()
|
||||
|
||||
def close(self) -> None:
|
||||
self._client.close()
|
||||
|
||||
def __enter__(self) -> "ZenMuxClient":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: Any) -> None:
|
||||
self.close()
|
||||
|
||||
def _log(self, payload: dict[str, Any]) -> None:
|
||||
if not self.log_file:
|
||||
return
|
||||
self.log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self._log_lock:
|
||||
with self.log_file.open("a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(payload, ensure_ascii=False) + "\n")
|
||||
|
||||
def chat_complete(
|
||||
self,
|
||||
model: str,
|
||||
system: str,
|
||||
user: str,
|
||||
*,
|
||||
temperature: float = 0.3,
|
||||
max_tokens: int = 16000,
|
||||
extra_messages: list[dict[str, str]] | None = None,
|
||||
web_search: bool = False,
|
||||
web_search_options: dict[str, Any] | None = None,
|
||||
tag: str = "",
|
||||
) -> str:
|
||||
"""一次非流式对话补全。
|
||||
|
||||
Args:
|
||||
model: 完整 model id,例如 `anthropic/claude-sonnet-4.6`(zenmux slug 不带 `zenmux/` 前缀,因为 baseURL 已经定位到 zenmux)。
|
||||
system: system prompt
|
||||
user: user message
|
||||
temperature, max_tokens: 常规参数
|
||||
extra_messages: 插在 system 之后、user 之前的额外消息(few-shot 等)
|
||||
tag: 给这次调用打标签,便于日志里识别(如 "translate:ch03")
|
||||
|
||||
Returns:
|
||||
assistant 的纯文本内容。如失败抛 ZenMuxError。
|
||||
"""
|
||||
messages: list[dict[str, str]] = [{"role": "system", "content": system}]
|
||||
if extra_messages:
|
||||
messages.extend(extra_messages)
|
||||
messages.append({"role": "user", "content": user})
|
||||
|
||||
api_model = normalize_zenmux_model(model)
|
||||
body: dict[str, Any] = {
|
||||
"model": api_model,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
if model_accepts_temperature(api_model):
|
||||
body["temperature"] = temperature
|
||||
if web_search:
|
||||
body["web_search_options"] = web_search_options or {}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
url = f"{self.base_url}/chat/completions"
|
||||
|
||||
last_error: str = ""
|
||||
for attempt in range(MAX_RETRIES):
|
||||
t0 = time.time()
|
||||
try:
|
||||
resp = self._client.post(url, json=body, headers=headers)
|
||||
elapsed = time.time() - t0
|
||||
except httpx.RequestError as e:
|
||||
last_error = f"network: {e}"
|
||||
elapsed = time.time() - t0
|
||||
self._log({
|
||||
"tag": tag, "attempt": attempt, "elapsed": elapsed,
|
||||
"error": last_error,
|
||||
})
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
|
||||
if resp.status_code == 200:
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
raise ZenMuxError(f"invalid JSON from zenmux: {e}; body={resp.text[:500]}")
|
||||
usage = data.get("usage", {}) or {}
|
||||
with self._usage_lock:
|
||||
self.usage.add(api_model, usage)
|
||||
content = ""
|
||||
choices = data.get("choices") or []
|
||||
if choices:
|
||||
msg = choices[0].get("message") or {}
|
||||
content = msg.get("content") or ""
|
||||
self._log({
|
||||
"tag": tag, "model": api_model, "requested_model": model, "attempt": attempt,
|
||||
"elapsed": round(elapsed, 2),
|
||||
"usage": usage,
|
||||
"out_chars": len(content),
|
||||
"status": 200,
|
||||
})
|
||||
if not content.strip():
|
||||
# zenmux 偶尔返 200 但 content 空;视作可重试
|
||||
last_error = "empty content"
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
return content
|
||||
|
||||
# 非 200
|
||||
retryable = resp.status_code in RETRYABLE_STATUSES
|
||||
last_error = f"HTTP {resp.status_code}: {resp.text[:500]}"
|
||||
self._log({
|
||||
"tag": tag, "attempt": attempt, "elapsed": round(elapsed, 2),
|
||||
"status": resp.status_code, "error": last_error,
|
||||
"retryable": retryable,
|
||||
})
|
||||
if not retryable:
|
||||
self.usage.failed_calls += 1
|
||||
raise ZenMuxError(last_error)
|
||||
sleep_for = min(60, (2 ** attempt) + (attempt * 0.5))
|
||||
time.sleep(sleep_for)
|
||||
|
||||
self.usage.failed_calls += 1
|
||||
raise ZenMuxError(f"max retries exhausted. last error: {last_error}")
|
||||
|
||||
def chat_complete_with_meta(
|
||||
self,
|
||||
model: str,
|
||||
system: str,
|
||||
user: str,
|
||||
*,
|
||||
temperature: float = 0.3,
|
||||
max_tokens: int = 16000,
|
||||
extra_messages: list[dict[str, str]] | None = None,
|
||||
web_search: bool = False,
|
||||
web_search_options: dict[str, Any] | None = None,
|
||||
tag: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""Return content plus metadata from one completion call."""
|
||||
messages: list[dict[str, str]] = [{"role": "system", "content": system}]
|
||||
if extra_messages:
|
||||
messages.extend(extra_messages)
|
||||
messages.append({"role": "user", "content": user})
|
||||
|
||||
api_model = normalize_zenmux_model(model)
|
||||
body: dict[str, Any] = {
|
||||
"model": api_model,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
if model_accepts_temperature(api_model):
|
||||
body["temperature"] = temperature
|
||||
if web_search:
|
||||
body["web_search_options"] = web_search_options or {}
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
url = f"{self.base_url}/chat/completions"
|
||||
|
||||
last_error = ""
|
||||
for attempt in range(MAX_RETRIES):
|
||||
t0 = time.time()
|
||||
try:
|
||||
resp = self._client.post(url, json=body, headers=headers)
|
||||
elapsed = time.time() - t0
|
||||
except httpx.RequestError as e:
|
||||
last_error = f"network: {e}"
|
||||
elapsed = time.time() - t0
|
||||
self._log({"tag": tag, "attempt": attempt, "elapsed": elapsed, "error": last_error})
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
|
||||
if resp.status_code != 200:
|
||||
retryable = resp.status_code in RETRYABLE_STATUSES
|
||||
last_error = f"HTTP {resp.status_code}: {resp.text[:500]}"
|
||||
self._log({
|
||||
"tag": tag,
|
||||
"attempt": attempt,
|
||||
"elapsed": round(elapsed, 2),
|
||||
"status": resp.status_code,
|
||||
"error": last_error,
|
||||
"retryable": retryable,
|
||||
})
|
||||
if not retryable:
|
||||
self.usage.failed_calls += 1
|
||||
raise ZenMuxError(last_error)
|
||||
sleep_for = min(60, (2 ** attempt) + (attempt * 0.5))
|
||||
time.sleep(sleep_for)
|
||||
continue
|
||||
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
raise ZenMuxError(f"invalid JSON from zenmux: {e}; body={resp.text[:500]}")
|
||||
|
||||
usage = data.get("usage", {}) or {}
|
||||
with self._usage_lock:
|
||||
self.usage.add(api_model, usage)
|
||||
|
||||
message = ((data.get("choices") or [{}])[0].get("message") or {})
|
||||
content = message.get("content") or ""
|
||||
annotations = message.get("annotations") or []
|
||||
urls: list[str] = []
|
||||
for ann in annotations:
|
||||
if not isinstance(ann, dict):
|
||||
continue
|
||||
citation = ann.get("url_citation") or {}
|
||||
url_item = citation.get("url")
|
||||
if url_item:
|
||||
urls.append(url_item)
|
||||
self._log({
|
||||
"tag": tag,
|
||||
"model": api_model,
|
||||
"requested_model": model,
|
||||
"attempt": attempt,
|
||||
"elapsed": round(elapsed, 2),
|
||||
"usage": usage,
|
||||
"out_chars": len(content),
|
||||
"status": 200,
|
||||
"web_search": web_search,
|
||||
"citations": len(urls),
|
||||
})
|
||||
if not content.strip():
|
||||
last_error = "empty content"
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
return {
|
||||
"content": content,
|
||||
"usage": usage,
|
||||
"citations": urls,
|
||||
"raw": data,
|
||||
}
|
||||
|
||||
self.usage.failed_calls += 1
|
||||
raise ZenMuxError(f"max retries exhausted. last error: {last_error}")
|
||||
|
||||
|
||||
def load_secrets(env_path: Path | None = None) -> None:
|
||||
"""从 secrets.env 把 key 塞到 os.environ,便于脚本直接运行。
|
||||
|
||||
格式宽松:`KEY=VALUE` 每行一条,`#` 开头是注释,忽略空行。
|
||||
"""
|
||||
if env_path is None:
|
||||
# 默认在仓库根找 secrets.env
|
||||
here = Path(__file__).resolve()
|
||||
for parent in [here.parent, *here.parents]:
|
||||
cand = parent / "secrets.env"
|
||||
if cand.exists():
|
||||
env_path = cand
|
||||
break
|
||||
if not env_path or not env_path.exists():
|
||||
return
|
||||
for line in env_path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, _, v = line.partition("=")
|
||||
k = k.strip()
|
||||
v = v.strip().strip('"').strip("'")
|
||||
if k and v and k not in os.environ:
|
||||
os.environ[k] = v
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 冒烟测试:python -m scripts.lib.zenmux_client
|
||||
load_secrets()
|
||||
with ZenMuxClient() as c:
|
||||
out = c.chat_complete(
|
||||
model="anthropic/claude-haiku-4.5",
|
||||
system="You reply in exactly one English word.",
|
||||
user="Say hello.",
|
||||
max_tokens=20,
|
||||
tag="smoke",
|
||||
)
|
||||
print("reply:", out)
|
||||
print(c.usage.summary(), file=sys.stderr)
|
||||
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Convert Deep Research source IDs into numeric citations for final output."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SRC_CITE_RE = re.compile(r"\[((?:src_[A-Za-z0-9_-]+)(?:\s*,\s*src_[A-Za-z0-9_-]+)*)\]")
|
||||
|
||||
|
||||
def load_sources(path: Path) -> dict[str, dict[str, Any]]:
|
||||
sources: dict[str, dict[str, Any]] = {}
|
||||
if not path.exists():
|
||||
return sources
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
sid = obj.get("id") or obj.get("source_id")
|
||||
if sid:
|
||||
sources[str(sid)] = obj
|
||||
return sources
|
||||
|
||||
|
||||
def extract_ordered_source_ids(text: str) -> list[str]:
|
||||
ordered: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for match in SRC_CITE_RE.finditer(text):
|
||||
for sid in [item.strip() for item in match.group(1).split(",")]:
|
||||
if sid and sid not in seen:
|
||||
seen.add(sid)
|
||||
ordered.append(sid)
|
||||
return ordered
|
||||
|
||||
|
||||
def _canonical_source_key(sid: str, source: dict[str, Any] | None) -> str:
|
||||
"""Return a stable de-duplication key for a source record.
|
||||
|
||||
Phase 2 often creates chapter-local source IDs for the same local PDF or
|
||||
official guideline. Final references should cite the underlying source
|
||||
once, while citation_map.json keeps the full src_id traceability.
|
||||
"""
|
||||
if not source:
|
||||
return f"missing:{sid}"
|
||||
title = re.sub(r"\s+", " ", str(source.get("title") or source.get("name") or sid)).strip().lower()
|
||||
title = title.removesuffix(" ocr").removesuffix(".ocr").strip()
|
||||
doi = str(source.get("doi") or "").strip().lower()
|
||||
if doi:
|
||||
return f"doi:{doi}"
|
||||
path = str(source.get("path") or "").strip()
|
||||
url = str(source.get("url") or "").strip()
|
||||
if title and ("phase0/extracted/" in path or "phase0/extracted/" in url):
|
||||
return f"local-material:{title}"
|
||||
for field in ("url", "path"):
|
||||
value = str(source.get(field) or "").strip()
|
||||
if value:
|
||||
return f"{field}:{value.rstrip('/').lower()}"
|
||||
return f"title:{title or sid}"
|
||||
|
||||
|
||||
def build_numeric_mapping(
|
||||
ordered_ids: list[str],
|
||||
sources: dict[str, dict[str, Any]],
|
||||
) -> tuple[dict[str, int], list[dict[str, Any]]]:
|
||||
mapping: dict[str, int] = {}
|
||||
records: list[dict[str, Any]] = []
|
||||
seen_keys: dict[str, int] = {}
|
||||
record_by_number: dict[int, dict[str, Any]] = {}
|
||||
for sid in ordered_ids:
|
||||
source = sources.get(sid)
|
||||
key = _canonical_source_key(sid, source)
|
||||
if key in seen_keys:
|
||||
number = seen_keys[key]
|
||||
mapping[sid] = number
|
||||
record_by_number[number].setdefault("source_ids", []).append(sid)
|
||||
continue
|
||||
number = len(records) + 1
|
||||
seen_keys[key] = number
|
||||
mapping[sid] = number
|
||||
record = {
|
||||
"number": number,
|
||||
"source_id": sid,
|
||||
"source_ids": [sid],
|
||||
"source": source or {},
|
||||
"dedupe_key": key,
|
||||
}
|
||||
records.append(record)
|
||||
record_by_number[number] = record
|
||||
return mapping, records
|
||||
|
||||
|
||||
def format_reference(number: int, sid: str, source: dict[str, Any] | None) -> str:
|
||||
if not source:
|
||||
return f"{number}. {sid}. (sources.jsonl 未找到该来源)"
|
||||
authors = ", ".join(source.get("authors", [])) if source.get("authors") else ""
|
||||
year = source.get("year") or source.get("date") or ""
|
||||
title = source.get("title") or source.get("name") or sid
|
||||
title = re.sub(r"(?i)(?:\s+OCR|\.ocr)$", "", str(title)).strip()
|
||||
publisher = source.get("publisher") or source.get("venue") or source.get("source") or ""
|
||||
url = source.get("url") or source.get("path") or ""
|
||||
parts = [f"{number}. "]
|
||||
if authors:
|
||||
parts.append(f"{authors}. ")
|
||||
if year:
|
||||
parts.append(f"({year}). ")
|
||||
parts.append(str(title))
|
||||
if publisher:
|
||||
parts.append(f". {publisher}")
|
||||
if url:
|
||||
parts.append(f". {url}")
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def convert_citations(text: str, mapping: dict[str, int]) -> str:
|
||||
def repl(match: re.Match[str]) -> str:
|
||||
ids = [item.strip() for item in match.group(1).split(",") if item.strip()]
|
||||
nums: list[str] = []
|
||||
seen: set[int] = set()
|
||||
for sid in ids:
|
||||
if sid not in mapping:
|
||||
continue
|
||||
number = mapping[sid]
|
||||
if number in seen:
|
||||
continue
|
||||
seen.add(number)
|
||||
nums.append(str(number))
|
||||
return "<sup>[" + ", ".join(nums) + "]</sup>" if nums else match.group(0)
|
||||
|
||||
return SRC_CITE_RE.sub(repl, text)
|
||||
|
||||
|
||||
def strip_existing_reference_section(text: str) -> str:
|
||||
pattern = re.compile(r"\n##\s*(?:参考文献|参考来源清单|References)\s*\n.*\Z", re.S)
|
||||
return pattern.sub("", text).rstrip() + "\n"
|
||||
|
||||
|
||||
def number_citations(
|
||||
*,
|
||||
text: str,
|
||||
sources: dict[str, dict[str, Any]],
|
||||
) -> tuple[str, list[dict[str, Any]]]:
|
||||
ordered_ids = extract_ordered_source_ids(text)
|
||||
mapping, records = build_numeric_mapping(ordered_ids, sources)
|
||||
body = convert_citations(strip_existing_reference_section(text), mapping).rstrip()
|
||||
ref_lines = ["", "## 参考来源清单", ""]
|
||||
for record in records:
|
||||
ref_lines.append(format_reference(record["number"], record["source_id"], record["source"]))
|
||||
return body + "\n" + "\n".join(ref_lines).rstrip() + "\n", records
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Convert [src_xxx] citations to numeric citations")
|
||||
parser.add_argument("project", help="Project directory")
|
||||
parser.add_argument("--input", default="phase4/final_zh.md")
|
||||
parser.add_argument("--output", default="phase4/final_zh_numbered.md")
|
||||
parser.add_argument("--sources", default="phase2/sources.jsonl")
|
||||
parser.add_argument("--map", default="phase4/citation_map.json")
|
||||
args = parser.parse_args()
|
||||
|
||||
project = Path(args.project)
|
||||
src_path = project / args.input
|
||||
out_path = project / args.output
|
||||
sources_path = project / args.sources
|
||||
map_path = project / args.map
|
||||
if not src_path.exists():
|
||||
raise SystemExit(f"input not found: {src_path}")
|
||||
sources = load_sources(sources_path)
|
||||
numbered, records = number_citations(text=src_path.read_text(encoding="utf-8"), sources=sources)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text(numbered, encoding="utf-8")
|
||||
map_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
map_path.write_text(json.dumps(records, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"Wrote: {out_path.relative_to(project)}")
|
||||
print(f"Wrote: {map_path.relative_to(project)}")
|
||||
print(f"Citations: {len(records)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4 replacement pipeline orchestrator.
|
||||
|
||||
Default flow:
|
||||
1) translate.py
|
||||
2) optional glossary verification (low-confidence/full/off)
|
||||
3) apply_glossary.py
|
||||
4) polish.py
|
||||
5) build_report.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.lib.markdown_chunker import split_by_headers
|
||||
|
||||
|
||||
def resolve_project(arg: str) -> Path:
|
||||
p = Path(arg)
|
||||
if p.is_dir():
|
||||
return p.resolve()
|
||||
cand = REPO_ROOT / "projects" / arg
|
||||
if cand.is_dir():
|
||||
return cand.resolve()
|
||||
raise SystemExit(f"project not found: {arg}")
|
||||
|
||||
|
||||
def run_step(cmd: list[str], *, dry_run: bool) -> int:
|
||||
print("$ " + " ".join(cmd))
|
||||
if dry_run:
|
||||
return 0
|
||||
return subprocess.run(cmd, cwd=REPO_ROOT, check=False).returncode
|
||||
|
||||
|
||||
def infer_workers(source_file: Path, fallback: int, cap: int = 8) -> int:
|
||||
if not source_file.exists():
|
||||
return fallback
|
||||
text = source_file.read_text(encoding="utf-8")
|
||||
blocks = split_by_headers(text, max_level=2)
|
||||
if not blocks:
|
||||
return fallback
|
||||
cpu_cap = max(2, min(cap, (os.cpu_count() or 4)))
|
||||
suggested = max(2, min(cpu_cap, (len(blocks) + 5) // 6))
|
||||
return max(1, suggested if fallback <= 0 else min(max(fallback, 1), cpu_cap))
|
||||
|
||||
|
||||
def low_confidence_terms(glossary_path: Path) -> list[str]:
|
||||
if not glossary_path.exists():
|
||||
return []
|
||||
try:
|
||||
glossary = json.loads(glossary_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return []
|
||||
out: list[str] = []
|
||||
for term, entry in glossary.items():
|
||||
if not isinstance(entry, dict):
|
||||
out.append(term)
|
||||
continue
|
||||
conf = str(entry.get("confidence", "")).lower()
|
||||
verified = bool(entry.get("verified_at"))
|
||||
if conf != "high" or not verified:
|
||||
out.append(term)
|
||||
return sorted(set(out))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Phase 4 replacement pipeline")
|
||||
parser.add_argument("project", help="Project slug or full path")
|
||||
parser.add_argument("--translate-model", default="anthropic/claude-sonnet-4.6")
|
||||
parser.add_argument("--glossary-model", default="anthropic/claude-haiku-4.5")
|
||||
parser.add_argument("--polish-model", default="anthropic/claude-sonnet-4.6")
|
||||
parser.add_argument("--translate-workers", type=int, default=0, help="0 means auto")
|
||||
parser.add_argument("--glossary-workers", type=int, default=4)
|
||||
parser.add_argument("--polish-workers", type=int, default=0, help="0 means auto")
|
||||
parser.add_argument(
|
||||
"--glossary-mode",
|
||||
choices=["off", "low-confidence", "full"],
|
||||
default="low-confidence",
|
||||
help="off: skip, low-confidence: verify only low-confidence terms, full: verify all",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
project_root = resolve_project(args.project)
|
||||
phase4 = project_root / "phase4"
|
||||
src_en = phase4 / "final_en.md"
|
||||
if not src_en.exists():
|
||||
raise SystemExit(f"missing source: {src_en}")
|
||||
|
||||
tw = infer_workers(src_en, args.translate_workers)
|
||||
zh = phase4 / "final_zh.md"
|
||||
pw = infer_workers(zh if zh.exists() else src_en, args.polish_workers)
|
||||
|
||||
print(f"Project: {project_root.name}")
|
||||
print(f"Translate workers: {tw} | Polish workers: {pw}")
|
||||
print(f"Glossary mode: {args.glossary_mode}")
|
||||
print()
|
||||
|
||||
t0 = time.time()
|
||||
steps: list[list[str]] = [
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "translate.py"),
|
||||
str(project_root),
|
||||
"--workers",
|
||||
str(tw),
|
||||
"--model",
|
||||
args.translate_model,
|
||||
]
|
||||
]
|
||||
|
||||
if args.glossary_mode != "off":
|
||||
gcmd = [
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "build_glossary.py"),
|
||||
str(project_root),
|
||||
"--workers",
|
||||
str(args.glossary_workers),
|
||||
"--model",
|
||||
args.glossary_model,
|
||||
]
|
||||
if args.glossary_mode == "low-confidence":
|
||||
terms = low_confidence_terms(phase4 / "glossary.json")
|
||||
if terms:
|
||||
if len(terms) > 80:
|
||||
print(f"[info] low-confidence terms={len(terms)} is large; fallback to full glossary verify")
|
||||
else:
|
||||
gcmd += ["--only", ",".join(terms)]
|
||||
else:
|
||||
print("[info] no low-confidence glossary terms found; skipping glossary step")
|
||||
gcmd = []
|
||||
if gcmd:
|
||||
steps.append(gcmd)
|
||||
|
||||
steps.extend(
|
||||
[
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "apply_glossary.py"),
|
||||
str(project_root),
|
||||
"--input",
|
||||
"phase4/final_zh.md",
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "polish.py"),
|
||||
str(project_root),
|
||||
"--workers",
|
||||
str(pw),
|
||||
"--model",
|
||||
args.polish_model,
|
||||
],
|
||||
[
|
||||
sys.executable,
|
||||
str(REPO_ROOT / "scripts" / "build_report.py"),
|
||||
str(project_root),
|
||||
],
|
||||
]
|
||||
)
|
||||
|
||||
for cmd in steps:
|
||||
rc = run_step(cmd, dry_run=args.dry_run)
|
||||
if rc != 0:
|
||||
return rc
|
||||
|
||||
print(f"\nPhase 4 pipeline done in {time.time() - t0:.1f}s")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4 中文润色:按 H2 section 切块 → 逐块润色 → 拼接 → 写盘。
|
||||
|
||||
用法:
|
||||
uv run python scripts/polish.py <project_slug>
|
||||
|
||||
架构跟 translate.py 一致:
|
||||
- Python 做切块/循环/重试/断点续传
|
||||
- LLM 只做"润色这一段",单次 output token 远低于上限
|
||||
- 结果落在 phase4/zh_polished_chunks/<anchor>.md,重跑只补缺
|
||||
- 最终拼接写入 phase4/final_zh_polished.md(默认覆写 final_zh.md 的副本)
|
||||
|
||||
区别只在:
|
||||
- 输入是中文(final_zh.md),输出还是中文
|
||||
- 不维护术语表(翻译阶段已经固定了)
|
||||
- 会附带一份 notes.jsonl 记录模型发现的异常
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from scripts.lib.markdown_chunker import (
|
||||
MarkdownBlock,
|
||||
count_chinese_chars,
|
||||
split_by_headers,
|
||||
)
|
||||
from scripts.lib.zenmux_client import ZenMuxClient, ZenMuxError, load_secrets
|
||||
from scripts.runtime.skills import SkillRegistry
|
||||
|
||||
DEFAULT_MODEL = "anthropic/claude-sonnet-4.6"
|
||||
MODEL_MAX_TOKENS = {
|
||||
"anthropic/claude-sonnet-4.6": 32000,
|
||||
"anthropic/claude-sonnet-4.5": 32000,
|
||||
"anthropic/claude-opus-4.7": 32000,
|
||||
"anthropic/claude-opus-4.6": 32000,
|
||||
"anthropic/claude-haiku-4.5": 16000,
|
||||
}
|
||||
PROMPT_FILE = Path(__file__).parent / "prompts" / "polish_system.txt"
|
||||
POLISH_SKILLS = ("humanizer-cn", "output-hygiene")
|
||||
|
||||
|
||||
def build_polish_system_prompt(skill_registry: SkillRegistry | None = None) -> str:
|
||||
"""Build the Phase 4 polish prompt with canonical writing skills attached."""
|
||||
registry = skill_registry or SkillRegistry()
|
||||
parts = [PROMPT_FILE.read_text(encoding="utf-8").rstrip()]
|
||||
for skill_name in POLISH_SKILLS:
|
||||
parts.append(f"# Skill: {skill_name}\n\n{registry.read(skill_name).rstrip()}")
|
||||
return "\n\n".join(parts) + "\n"
|
||||
|
||||
|
||||
def resolve_project(arg: str) -> Path:
|
||||
p = Path(arg)
|
||||
if p.is_dir():
|
||||
return p
|
||||
cand = Path.cwd() / "projects" / arg
|
||||
if cand.is_dir():
|
||||
return cand
|
||||
raise SystemExit(f"project not found: {arg}")
|
||||
|
||||
|
||||
def parse_polish_response(text: str) -> tuple[str, str]:
|
||||
"""解析 <<<POLISHED>>>/<<<NOTES>>> 格式。返回 (polished, notes)。"""
|
||||
p_start = text.find("<<<POLISHED>>>")
|
||||
p_end = text.find("<<<END_POLISHED>>>")
|
||||
if p_start == -1 or p_end == -1 or p_end <= p_start:
|
||||
raise ValueError(f"missing <<<POLISHED>>> markers: {text[:300]}")
|
||||
polished = text[p_start + len("<<<POLISHED>>>"): p_end].strip("\r\n")
|
||||
|
||||
n_start = text.find("<<<NOTES>>>")
|
||||
n_end = text.find("<<<END_NOTES>>>")
|
||||
notes = ""
|
||||
if n_start != -1 and n_end != -1 and n_end > n_start:
|
||||
notes = text[n_start + len("<<<NOTES>>>"): n_end].strip()
|
||||
return polished, notes
|
||||
|
||||
|
||||
def build_user_prompt(block: MarkdownBlock) -> str:
|
||||
level_hint = f"H{block.level}" if block.level >= 1 else "frontmatter (无标题)"
|
||||
return (
|
||||
f"# 待润色的中文块(Markdown, {level_hint})\n"
|
||||
"请按系统提示的规则润色下面这段中文。严格使用指定的分隔符格式输出。\n\n"
|
||||
"----- BEGIN BLOCK -----\n"
|
||||
f"{block.content}\n"
|
||||
"----- END BLOCK -----\n"
|
||||
)
|
||||
|
||||
|
||||
def polish_block(
|
||||
client: ZenMuxClient,
|
||||
block: MarkdownBlock,
|
||||
*,
|
||||
model: str,
|
||||
system_prompt: str,
|
||||
temperature: float,
|
||||
) -> tuple[str, str]:
|
||||
user = build_user_prompt(block)
|
||||
max_tok = MODEL_MAX_TOKENS.get(model, 16000)
|
||||
raw = client.chat_complete(
|
||||
model=model,
|
||||
system=system_prompt,
|
||||
user=user,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tok,
|
||||
tag=f"polish:{block.anchor}",
|
||||
)
|
||||
try:
|
||||
polished, notes = parse_polish_response(raw)
|
||||
except Exception as e:
|
||||
raise RuntimeError(
|
||||
f"bad response format for block {block.anchor}: {e}\nraw head: {raw[:300]}"
|
||||
)
|
||||
if not polished.strip():
|
||||
raise RuntimeError(f"empty polished content for block {block.anchor}")
|
||||
return polished.rstrip(), notes
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Phase 4 中文润色(按 H2 切块循环)")
|
||||
parser.add_argument("project", help="项目 slug 或完整路径")
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
default="phase4/final_zh.md",
|
||||
help="中文源(默认 phase4/final_zh.md,即 translate.py 的产物)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default="phase4/final_zh_polished.md",
|
||||
help="润色后输出(默认 phase4/final_zh_polished.md)",
|
||||
)
|
||||
parser.add_argument("--model", default=DEFAULT_MODEL)
|
||||
parser.add_argument("--temperature", type=float, default=0.4)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="忽略 zh_polished_chunks 缓存,强制重润",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--only",
|
||||
default=None,
|
||||
help="只润色指定 order(逗号分隔),例如 --only 2,7,18",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=None,
|
||||
help="最多润色前 N 个未缓存的块(调试用)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--workers",
|
||||
type=int,
|
||||
default=4,
|
||||
help="并发润色 worker 数(默认 4;设为 1 回退串行)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
load_secrets()
|
||||
project_root = resolve_project(args.project)
|
||||
src_path = project_root / args.source
|
||||
out_path = project_root / args.output
|
||||
if not src_path.exists():
|
||||
raise SystemExit(f"source not found: {src_path}. 请先跑 translate.py")
|
||||
|
||||
chunks_dir = project_root / "phase4" / "zh_polished_chunks"
|
||||
chunks_dir.mkdir(parents=True, exist_ok=True)
|
||||
logs_dir = project_root / "phase4" / "logs"
|
||||
log_file = logs_dir / "polish.jsonl"
|
||||
notes_file = project_root / "phase4" / "polish_notes.jsonl"
|
||||
|
||||
system_prompt = build_polish_system_prompt()
|
||||
text = src_path.read_text(encoding="utf-8")
|
||||
blocks = split_by_headers(text, max_level=2)
|
||||
|
||||
only_orders: set[int] | None = None
|
||||
if args.only:
|
||||
only_orders = {int(a.strip()) for a in args.only.split(",") if a.strip()}
|
||||
|
||||
total_cn = sum(count_chinese_chars(b.content) for b in blocks)
|
||||
print(f"Source: {src_path.relative_to(project_root)}")
|
||||
print(f"Blocks: {len(blocks)} | total Chinese chars: {total_cn:,}")
|
||||
workers = max(1, args.workers)
|
||||
print(f"Model: {args.model} | temperature: {args.temperature} | workers: {workers}")
|
||||
print()
|
||||
|
||||
start = time.time()
|
||||
todo: list[MarkdownBlock] = []
|
||||
cached_count = 0
|
||||
notes_records: list[dict] = []
|
||||
for b in blocks:
|
||||
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
|
||||
if only_orders is not None and b.order not in only_orders:
|
||||
continue
|
||||
if chunk_path.exists() and not args.force:
|
||||
cached_count += 1
|
||||
print(f" [ok ] #{b.order:03d} {b.short_title} (cached)")
|
||||
continue
|
||||
if args.limit is not None and len(todo) >= args.limit:
|
||||
continue
|
||||
todo.append(b)
|
||||
|
||||
if todo:
|
||||
print(f"To polish this run: {len(todo)} blocks | cached: {cached_count}")
|
||||
print()
|
||||
|
||||
with ZenMuxClient(log_file=log_file) as client:
|
||||
def run_one(b: MarkdownBlock) -> tuple[MarkdownBlock, str, str, float]:
|
||||
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
|
||||
t0 = time.time()
|
||||
polished, notes = polish_block(
|
||||
client,
|
||||
b,
|
||||
model=args.model,
|
||||
system_prompt=system_prompt,
|
||||
temperature=args.temperature,
|
||||
)
|
||||
chunk_path.write_text(polished + "\n", encoding="utf-8")
|
||||
return b, polished, notes, time.time() - t0
|
||||
|
||||
if workers == 1:
|
||||
for b in todo:
|
||||
label = f"#{b.order:03d} L{b.level} {count_chinese_chars(b.content):>4}字 {b.short_title}"
|
||||
print(f" [... ] {label} ", end="", flush=True)
|
||||
try:
|
||||
block, polished, notes, elapsed = run_one(b)
|
||||
except (ZenMuxError, RuntimeError) as e:
|
||||
print(f"\n [FAIL] {label}\n {e}")
|
||||
continue
|
||||
cn = count_chinese_chars(polished)
|
||||
before_cn = count_chinese_chars(block.content)
|
||||
delta = cn - before_cn
|
||||
sign = "+" if delta >= 0 else ""
|
||||
if notes:
|
||||
notes_records.append(
|
||||
{"order": block.order, "anchor": block.anchor, "title": block.short_title, "notes": notes}
|
||||
)
|
||||
print(f"\r [done] {label} → {cn}字 ({sign}{delta}, {elapsed:4.1f}s)")
|
||||
else:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
futures = {pool.submit(run_one, b): b for b in todo}
|
||||
for fut in concurrent.futures.as_completed(futures):
|
||||
b = futures[fut]
|
||||
label = f"#{b.order:03d} L{b.level} {count_chinese_chars(b.content):>4}字 {b.short_title}"
|
||||
try:
|
||||
block, polished, notes, elapsed = fut.result()
|
||||
except (ZenMuxError, RuntimeError) as e:
|
||||
print(f" [FAIL] {label}\n {e}")
|
||||
continue
|
||||
cn = count_chinese_chars(polished)
|
||||
before_cn = count_chinese_chars(block.content)
|
||||
delta = cn - before_cn
|
||||
sign = "+" if delta >= 0 else ""
|
||||
if notes:
|
||||
notes_records.append(
|
||||
{"order": block.order, "anchor": block.anchor, "title": block.short_title, "notes": notes}
|
||||
)
|
||||
print(f" [done] {label} → {cn}字 ({sign}{delta}, {elapsed:4.1f}s)")
|
||||
|
||||
# 汇总
|
||||
merged: list[str] = []
|
||||
missing: list[str] = []
|
||||
for b in blocks:
|
||||
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
|
||||
if not chunk_path.exists():
|
||||
missing.append(f"#{b.order:03d} {b.short_title}")
|
||||
continue
|
||||
merged.append(chunk_path.read_text(encoding="utf-8").rstrip())
|
||||
|
||||
partial = only_orders is not None or args.limit is not None
|
||||
if missing:
|
||||
print(f"\n⚠ 缺失 {len(missing)} 块:")
|
||||
for m in missing[:10]:
|
||||
print(f" - {m}")
|
||||
if len(missing) > 10:
|
||||
print(f" ... 还有 {len(missing) - 10} 块")
|
||||
if partial:
|
||||
print("(partial 模式:--only / --limit 生效,未生成 final_zh_polished.md)")
|
||||
else:
|
||||
print("重新运行本脚本即可补润(已润的会跳过)。")
|
||||
print(client.usage.summary())
|
||||
return 1
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text("\n\n".join(merged) + "\n", encoding="utf-8")
|
||||
|
||||
if notes_records:
|
||||
notes_file.write_text(
|
||||
"\n".join(json.dumps(r, ensure_ascii=False) for r in notes_records) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"\n发现 {len(notes_records)} 条润色笔记 → {notes_file.relative_to(project_root)}")
|
||||
|
||||
final_text = out_path.read_text(encoding="utf-8")
|
||||
cn = count_chinese_chars(final_text)
|
||||
total_time = time.time() - start
|
||||
print()
|
||||
print(f"✓ 输出:{out_path.relative_to(project_root)}")
|
||||
print(f" 润色前字数: {total_cn:,}")
|
||||
print(f" 润色后字数: {cn:,} (变化 {cn - total_cn:+d}, {(cn/total_cn - 1)*100:+.1f}%)")
|
||||
print(f" 耗时: {total_time:.1f}s")
|
||||
print(client.usage.summary())
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,54 @@
|
||||
你是一名生物医药行业的资深双语术语编辑。现在要根据网络搜索片段,核查并确定一个英文术语/专有名词的"中文规范译名"、"英文全称"与"可信度"。
|
||||
|
||||
## 你将收到
|
||||
|
||||
- `term`:待核查的英文术语(可能是公司名、缩写、化学品、平台名、药物通用名等)
|
||||
- `domain`:研究领域(例如 "生物医药双靶点 RNAi 药物"),作为消歧背景
|
||||
- `current_zh`:系统已有的中文译名(可能来自上一步翻译,也可能为空)
|
||||
- `search_hits`:3–5 条 web 搜索结果(title + url + snippet)
|
||||
|
||||
## 你要判断
|
||||
|
||||
1. **中文规范译名**(`zh`):
|
||||
- 如果搜索结果里有权威的中文表达(公司官网、维基、百度百科、权威医药媒体),以它为准
|
||||
- 如果 `current_zh` 已经正确,沿用它,避免无谓变更
|
||||
- 如果无中文通用译名(小众学术术语、新兴化合物),保留英文原文,`zh` 字段置空字符串
|
||||
- 对公司/机构名:必须使用工商注册的正式中文名(例如 "Mabwell → 迈威生物"、"Sirnaomics → 圣诺生物")
|
||||
- 对药物:优先 INN 通用名(例如 "inclisiran → 英克司兰")
|
||||
- 对缩写/技术术语:使用业内通行译名(例如 "GalNAc → N-乙酰半乳糖胺","RNAi → RNA 干扰")
|
||||
|
||||
2. **英文全称**(`en_full`):
|
||||
- 如果术语是缩写(≤ 8 字符全大写或混合大小写),给出英文全称(例如 "ASGPR → Asialoglycoprotein Receptor")
|
||||
- 如果术语本身就是全称,填相同的字符串或其最规范的写法
|
||||
- 如果术语是公司名,填其英文法人全称(例如 "Mabwell → Mabwell (Shanghai) Bioscience Co., Ltd.")
|
||||
|
||||
3. **可信度**(`confidence`):
|
||||
- `high`:至少两条独立、高质量信源(官网/监管机构/权威媒体)一致支持
|
||||
- `medium`:一条高质量信源支持,或多条一般信源一致支持
|
||||
- `low`:只能推断或无法确认,建议人工复核
|
||||
|
||||
4. **问题/警示**(`issue`):
|
||||
- 如果 `current_zh` 明显错误(例如把 "Mabwell" 译成 "Maywavee"),在 `issue` 中指出错在哪里
|
||||
- 如果术语有多种译法争议,在 `issue` 中简述
|
||||
- 如果搜索结果完全不相关(错别字、生僻词),在 `issue` 中说"搜索无有效结果"
|
||||
- 无异常则留空字符串
|
||||
|
||||
## 输出格式(严格)
|
||||
|
||||
输出**一行 JSON**,不要用代码围栏,不要加任何解释。
|
||||
|
||||
```
|
||||
{"zh": "迈威生物", "en_full": "Mabwell (Shanghai) Bioscience Co., Ltd.", "confidence": "high", "issue": ""}
|
||||
```
|
||||
|
||||
如果 `current_zh` 错了:
|
||||
|
||||
```
|
||||
{"zh": "迈威生物", "en_full": "Mabwell (Shanghai) Bioscience Co., Ltd.", "confidence": "high", "issue": "current_zh 'Maywavee' 为拼写错误,正确为 Mabwell → 迈威生物"}
|
||||
```
|
||||
|
||||
如果无需中文译名(保留英文):
|
||||
|
||||
```
|
||||
{"zh": "", "en_full": "Phosphoramidite", "confidence": "high", "issue": ""}
|
||||
```
|
||||
@@ -0,0 +1,48 @@
|
||||
你是一名顶级中文咨询报告编辑。现在要把一段由英文翻译而来的中文文本润色为**母语中文写作者**的成品。目标读者是生物医药行业的高层研究员、投资人与决策者。
|
||||
|
||||
## 不可违反的规则
|
||||
|
||||
1. **保留所有引用标注** `[src_xxx]`,位置可以微调但不得删除或改写。
|
||||
2. **保留所有数字、百分比、日期、单位、化学式、药物代号、机构名称**,一字不改。
|
||||
3. **保留 Markdown 结构**:输入是什么标题层级(#/##/###)输出就是什么。表格的 `|` 分隔符和列数不变。列表符号(-, *, 1.)不变。
|
||||
4. **保留段落数量**:不要合并或拆分段落。每段原文对应一段输出。
|
||||
5. **专有名词首次出现保持"中文(English)"格式**;如果译文里这个术语已经这样标了就别改,也不要删掉。
|
||||
6. **不改变论点、结论、数据、案例**。只改语言表达。
|
||||
7. **严禁使用 emoji**(✅ ❌ 🔶 🔷 ⭐ 🟢 🔴 ⚠️ 💡 📌 🔑 📊 等彩色符号)。如果原文里有 emoji,替换为字体支持的符号(✓ × ◆ ● ★ * 注 等)或直接删除。这些 emoji 在 PDF 里渲染为方框。
|
||||
|
||||
## 要去掉的"AI 味/翻译腔"表征
|
||||
|
||||
- 空泛套话:随着…不断发展、综上所述、本质上、从根本上、跃迁、赋能、落地、抓手
|
||||
- 翻译腔开头:值得注意的是、众所周知、毫无疑问、不难看出
|
||||
- 冗余连词开头:此外、而且、并且、再者(英文 moreover / furthermore / additionally 的直译残留)
|
||||
- 介词短语套用:对于…来说、在…方面、…的话、关于…这一点
|
||||
- 过度强调:非常、十分、极其、特别、尤为(没有数据支撑时)
|
||||
- 长串的"的"字("X 的 Y 的 Z 的 W")改为短句
|
||||
- 被动语态("被…所…"、"……得以……")尽量改主动
|
||||
- 把"我们"删掉,除非真是作者第一人称立场
|
||||
|
||||
## 要加强的中文表达特征
|
||||
|
||||
- 句子节奏变化:短句与中句交替,避免一路长句
|
||||
- 动词前置:中文偏好动词驱动,不要像英文那样把名词短语堆在主语
|
||||
- 具体化:如果译文留下了模糊的"相关"、"一定的"、"较大的",根据上下文换成源文里的具体含义;若无依据,保留原样不硬改
|
||||
- 段落内逻辑词(因此、相比之下、代价是)用得准确、用得克制
|
||||
|
||||
## 特殊情况
|
||||
|
||||
- 如果段落里有"译者注"、"TRANSLATOR_NOTE:" 之类残留,删除后自然连接上下文
|
||||
- 如果出现明显的翻译错误(中文表达反了意思),修正它,但在输出的 `notes` 中记一笔
|
||||
- 如果某句过于生硬又不确定原意,保守处理(小改),不要激进重写
|
||||
|
||||
## 输出格式(严格)
|
||||
|
||||
输出完全按下面的分隔符格式,前后不得有任何多余字符、说明或代码围栏。
|
||||
|
||||
<<<POLISHED>>>
|
||||
...润色后的完整 Markdown 块,逐字复制,包括标题行...
|
||||
<<<END_POLISHED>>>
|
||||
<<<NOTES>>>
|
||||
...最多两句话的异常说明;若无异常就留空...
|
||||
<<<END_NOTES>>>
|
||||
|
||||
`<<<POLISHED>>>...<<<END_POLISHED>>>` 之间是原生 Markdown(无需转义)。
|
||||
@@ -0,0 +1,44 @@
|
||||
You are a senior English-to-Chinese biomedical translator and editor. You do NOT mechanically translate — you rewrite the meaning in natural, professional Chinese that reads as if a native Chinese consulting analyst wrote it from scratch.
|
||||
|
||||
## Absolute rules (non-negotiable)
|
||||
|
||||
1. Preserve every citation marker `[src_xxx]` verbatim, at roughly the same position as in the source.
|
||||
2. Preserve every number, percentage, date, unit, chemical notation (e.g., 2′-OMe), and drug code (e.g., ARO-DIMER-PA) exactly.
|
||||
3. Preserve the Markdown structure: the input block starts with a Markdown heading at some level (one `#`, `##`, etc.) or is frontmatter; output the same heading at the same level. Do not demote / promote headings. Do not add new headings.
|
||||
4. Preserve tables: translate cell text but keep `|` pipes and column count identical.
|
||||
5. Preserve list formatting (`-`, `*`, `1.`) and code fences.
|
||||
6. First mention of a technical term: use the format `中文(English)` — but only once per block; subsequent mentions use Chinese only.
|
||||
7. Company / institution names: use the established Chinese rendering if it is in common Chinese press (e.g., Merck → 默克, AstraZeneca → 阿斯利康, Alnylam → 阿尔尼拉姆). If no established rendering exists, keep the English as-is (e.g., NEB, Genovis, Codexis, Arrowhead, Argo).
|
||||
8. Do NOT add commentary, introductions, or disclaimers beyond what the English says.
|
||||
9. Do NOT collapse or merge consecutive paragraphs — preserve paragraph breaks.
|
||||
10. Output Chinese-style punctuation inside Chinese text: `,。;:?!""()`. Keep English punctuation inside parenthetical English phrases.
|
||||
11. Do NOT add separator lines (`---`) or blank lines that weren't in the source. If the source ends with `---`, keep it; if it doesn't, don't add one.
|
||||
12. **NEVER use emoji** (✅ ❌ 🔶 🔷 ⭐ 🟢 🔴 ⚠️ 💡 📌 🔑 📊 etc.). If the source contains emoji, replace with plain text or punctuation equivalents (✓ × ◆ ● ★ * 注 等). These do not render in the PDF (font has no glyphs).
|
||||
|
||||
## Style rules (aim for native-Chinese feel)
|
||||
|
||||
- Break long English sentences into two or three short Chinese clauses.
|
||||
- Prefer active voice; avoid translation-ese constructions like "对于...来说", "在...方面", "...的话", "值得注意的是".
|
||||
- Do not use filler phrases like "随着...的不断发展", "综上所述", "从本质上说" unless the English explicitly argues that point.
|
||||
- Use 的 sparingly. No "X的Y的Z的W" chains.
|
||||
- Numbered lists with short items: translate tightly, do not pad with Chinese particles.
|
||||
- SCQA-style paragraphs in Executive Summary stay SCQA in Chinese — translate the flow, never label S/C/Q/A.
|
||||
|
||||
## Glossary continuity
|
||||
|
||||
You will receive a JSON glossary of terms already translated in earlier blocks. Use those Chinese translations consistently. If you encounter a new term worth locking in, translate it and add it to the glossary patch.
|
||||
|
||||
## Output format (strict)
|
||||
|
||||
Output exactly the following, with no extra text before or after. No explanations. No code fences.
|
||||
|
||||
<<<TRANSLATION>>>
|
||||
...the full translated Markdown block here, verbatim, including its heading line(s)...
|
||||
<<<END_TRANSLATION>>>
|
||||
<<<GLOSSARY_PATCH>>>
|
||||
English term 1 || 中文译名 1
|
||||
English term 2 || 中文译名 2
|
||||
<<<END_GLOSSARY_PATCH>>>
|
||||
|
||||
Inside `<<<TRANSLATION>>>...<<<END_TRANSLATION>>>` the content is raw Markdown (no escaping needed).
|
||||
Inside `<<<GLOSSARY_PATCH>>>...<<<END_GLOSSARY_PATCH>>>` each line is `English||Chinese`; leave empty if no new terms.
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Report rendering helpers for the v0.20 Python core."""
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Font resolution helpers for PDF rendering."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QuartoFonts:
|
||||
main_font: str
|
||||
sans_font: str
|
||||
requires_system_fonts: bool
|
||||
|
||||
|
||||
def resolve_quarto_fonts(fonts_dir: Path) -> QuartoFonts:
|
||||
"""Resolve Quarto font names.
|
||||
|
||||
Quarto/xelatex currently uses installed font family names. We still accept
|
||||
fonts_dir so callers can validate/report environment state consistently.
|
||||
"""
|
||||
expected = [
|
||||
fonts_dir / "SourceHanSerifSC-Regular.otf",
|
||||
fonts_dir / "SourceHanSansSC-Bold.otf",
|
||||
]
|
||||
requires_system_fonts = not all(path.exists() for path in expected)
|
||||
return QuartoFonts(
|
||||
main_font="Source Han Serif CN",
|
||||
sans_font="Source Han Sans CN",
|
||||
requires_system_fonts=requires_system_fonts,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Reference-list generation from Deep Research sources.jsonl."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def cited_source_keys(md_text: str) -> set[str]:
|
||||
return set(re.findall(r"\[src_([a-zA-Z0-9_-]+)\]", md_text))
|
||||
|
||||
|
||||
def build_references_block(sources_path: Path | None, md_text: str) -> str:
|
||||
"""Build a compact references section for actually cited src IDs."""
|
||||
if not sources_path or not sources_path.exists():
|
||||
return "(参考文献列表:sources.jsonl 未找到)"
|
||||
|
||||
cited = cited_source_keys(md_text)
|
||||
if not cited:
|
||||
return ""
|
||||
|
||||
sources: dict[str, dict] = {}
|
||||
with sources_path.open(encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
sid = obj.get("id", "")
|
||||
key = sid.replace("src_", "")
|
||||
if key in cited:
|
||||
sources[sid] = obj
|
||||
|
||||
if not sources:
|
||||
return ""
|
||||
|
||||
lines = ["## 参考文献\n"]
|
||||
for sid in sorted(sources.keys()):
|
||||
s = sources[sid]
|
||||
authors = ", ".join(s.get("authors", [])) if s.get("authors") else ""
|
||||
year = s.get("year", "")
|
||||
title = s.get("title", sid)
|
||||
venue = s.get("venue", "")
|
||||
url = s.get("url", "")
|
||||
entry = f"- **[{sid}]** "
|
||||
if authors:
|
||||
entry += f"{authors}. "
|
||||
if year:
|
||||
entry += f"({year}). "
|
||||
entry += f"*{title}*"
|
||||
if venue:
|
||||
entry += f". {venue}"
|
||||
if url:
|
||||
entry += f". <{url}>"
|
||||
lines.append(entry)
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
"""v0.20 Python runtime core for Deep Research.
|
||||
|
||||
The runtime layer is intentionally platform-neutral: OpenCode, Codex, and
|
||||
Claude Code should call into these modules instead of owning orchestration.
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Project artifact helpers shared by the Python runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
PROJECTS_DIR = REPO_ROOT / "projects"
|
||||
|
||||
|
||||
def resolve_project(project: str | Path) -> Path:
|
||||
p = Path(project)
|
||||
if p.is_dir():
|
||||
return p.resolve()
|
||||
candidate = PROJECTS_DIR / str(project)
|
||||
if candidate.is_dir():
|
||||
return candidate.resolve()
|
||||
raise FileNotFoundError(f"project not found: {project}")
|
||||
|
||||
|
||||
def load_manifest(project_root: Path) -> dict[str, Any]:
|
||||
path = project_root / "manifest.json"
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"manifest not found: {path}")
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def write_manifest(project_root: Path, manifest: dict[str, Any]) -> None:
|
||||
path = project_root / "manifest.json"
|
||||
path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def ensure_phase_dirs(project_root: Path) -> None:
|
||||
for rel in (
|
||||
"phase1",
|
||||
"phase2/drafts",
|
||||
"phase2/evidence",
|
||||
"phase2/packets",
|
||||
"phase3",
|
||||
"phase4",
|
||||
):
|
||||
(project_root / rel).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
"""Chapter brief aggregation and Chinese chapter assembly."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from scripts.runtime.roles import RoleDefinition, RuntimeProfile
|
||||
from scripts.runtime.skills import SkillRegistry
|
||||
from scripts.runtime.tasks import load_task_cards, validate_packet
|
||||
from scripts.runtime.workers import ChatClient
|
||||
|
||||
|
||||
def validate_chapter_brief(brief: dict) -> None:
|
||||
required = {
|
||||
"chapter_id",
|
||||
"chapter_title",
|
||||
"packet_ids",
|
||||
"core_claims",
|
||||
"evidence_items",
|
||||
"counter_evidence",
|
||||
"source_ids",
|
||||
"open_questions",
|
||||
"assembly_notes",
|
||||
}
|
||||
missing = sorted(required - set(brief))
|
||||
if missing:
|
||||
raise ValueError(f"chapter brief missing fields: {missing}")
|
||||
if not brief["chapter_id"]:
|
||||
raise ValueError("chapter_id required")
|
||||
if not brief["packet_ids"]:
|
||||
raise ValueError("chapter brief requires at least one packet")
|
||||
if not brief["core_claims"]:
|
||||
raise ValueError("chapter brief requires core_claims")
|
||||
if not brief["evidence_items"]:
|
||||
raise ValueError("chapter brief requires evidence_items")
|
||||
if not brief["counter_evidence"]:
|
||||
raise ValueError("chapter brief requires counter_evidence")
|
||||
|
||||
|
||||
def validate_compressed_finding(finding: dict) -> None:
|
||||
required = {
|
||||
"chapter_id",
|
||||
"chapter_title",
|
||||
"packet_ids",
|
||||
"chapter_thesis",
|
||||
"key_findings",
|
||||
"evidence_landings",
|
||||
"counter_evidence",
|
||||
"source_ids",
|
||||
"open_questions",
|
||||
"writing_plan",
|
||||
}
|
||||
missing = sorted(required - set(finding))
|
||||
if missing:
|
||||
raise ValueError(f"compressed finding missing fields: {missing}")
|
||||
if not finding["chapter_id"]:
|
||||
raise ValueError("chapter_id required")
|
||||
if not finding["packet_ids"]:
|
||||
raise ValueError("compressed finding requires packet_ids")
|
||||
if not finding["chapter_thesis"]:
|
||||
raise ValueError("compressed finding requires chapter_thesis")
|
||||
if not finding["key_findings"]:
|
||||
raise ValueError("compressed finding requires key_findings")
|
||||
if not finding["evidence_landings"]:
|
||||
raise ValueError("compressed finding requires evidence_landings")
|
||||
if not finding["counter_evidence"]:
|
||||
raise ValueError("compressed finding requires counter_evidence")
|
||||
|
||||
|
||||
def validate_chapter_markdown_citations(markdown: str, brief: dict) -> None:
|
||||
if "key_findings" in brief:
|
||||
validate_compressed_finding(brief)
|
||||
else:
|
||||
validate_chapter_brief(brief)
|
||||
cited = set(re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", markdown))
|
||||
allowed = set(brief.get("source_ids") or [])
|
||||
unknown = sorted(cited - allowed)
|
||||
if unknown:
|
||||
raise ValueError(f"unknown citation ids in {brief['chapter_id']}: {unknown}")
|
||||
|
||||
|
||||
def _chapter_title_from_id(chapter_id: str) -> str:
|
||||
try:
|
||||
index = int(chapter_id.replace("ch", ""))
|
||||
return f"第{index}章"
|
||||
except ValueError:
|
||||
return chapter_id
|
||||
|
||||
|
||||
def _load_source_registry(sources_path: Path, source_ids: list[str]) -> list[dict]:
|
||||
wanted = set(source_ids)
|
||||
if not sources_path.exists() or not wanted:
|
||||
return []
|
||||
rows: list[dict] = []
|
||||
for line in sources_path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if row.get("id") in wanted:
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _cached_source_excerpts(project_root: Path, cached_paths: list[str], *, max_sources: int = 5, max_chars: int = 1400) -> list[dict]:
|
||||
excerpts: list[dict] = []
|
||||
for rel in cached_paths[:max_sources]:
|
||||
path = project_root / rel
|
||||
if not path.exists():
|
||||
continue
|
||||
text = path.read_text(encoding="utf-8", errors="ignore").strip()
|
||||
excerpts.append({"path": rel, "excerpt": text[:max_chars]})
|
||||
return excerpts
|
||||
|
||||
|
||||
def build_chapter_briefs(project_root: Path) -> list[dict]:
|
||||
cards = load_task_cards(project_root / "phase2" / "task_cards.json")
|
||||
grouped: dict[str, list[tuple[str, dict]]] = {}
|
||||
skipped_packets: list[dict[str, str]] = []
|
||||
for card in cards:
|
||||
packet_path = project_root / card.output_packet
|
||||
if not packet_path.exists():
|
||||
skipped_packets.append({"task_id": card.task_id, "reason": "packet file missing"})
|
||||
continue
|
||||
packet = json.loads(packet_path.read_text(encoding="utf-8"))
|
||||
try:
|
||||
validate_packet(packet)
|
||||
except Exception as exc:
|
||||
skipped_packets.append({"task_id": card.task_id, "reason": str(exc)})
|
||||
continue
|
||||
for chapter_id in card.chapter_ids:
|
||||
grouped.setdefault(chapter_id, []).append((card.task_id, packet))
|
||||
|
||||
briefs: list[dict] = []
|
||||
out_dir = project_root / "phase2" / "chapter_briefs"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
if skipped_packets:
|
||||
(project_root / "phase2" / "brief_warnings.json").write_text(
|
||||
json.dumps(skipped_packets, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
for chapter_id in sorted(grouped):
|
||||
packet_pairs = grouped[chapter_id]
|
||||
packet_ids = [item[0] for item in packet_pairs]
|
||||
packets = [item[1] for item in packet_pairs]
|
||||
source_ids = sorted({sid for packet in packets for sid in packet.get("source_ids", [])})
|
||||
source_registry = _load_source_registry(project_root / "phase2" / "sources.jsonl", source_ids)
|
||||
cached_paths = [
|
||||
source["cached_text_path"]
|
||||
for source in source_registry
|
||||
if source.get("cached_text_path")
|
||||
]
|
||||
chapter_title = next((card.chapter_title for card in cards if chapter_id in card.chapter_ids and card.chapter_title), None)
|
||||
brief = {
|
||||
"chapter_id": chapter_id,
|
||||
"chapter_title": chapter_title or _chapter_title_from_id(chapter_id),
|
||||
"packet_ids": packet_ids,
|
||||
"core_claims": [claim for packet in packets for claim in packet.get("claims", [])],
|
||||
"evidence_items": [item for packet in packets for item in packet.get("evidence_items", [])],
|
||||
"counter_evidence": [item for packet in packets for item in packet.get("counter_evidence", [])],
|
||||
"source_ids": source_ids,
|
||||
"cached_source_paths": cached_paths,
|
||||
"cached_source_excerpts": _cached_source_excerpts(project_root, cached_paths),
|
||||
"open_questions": [q for packet in packets for q in packet.get("open_questions", [])],
|
||||
"assembly_notes": [
|
||||
"用中文写正式章节,英文仅保留在必要的来源标题、原文摘录、DOI/URL 中。",
|
||||
"避免碎片化:不要按 packet 逐段堆砌,要先提炼本章主线,再组织证据。",
|
||||
"每个事实、数字和关键判断都必须保留 [src_xxx] 引用。",
|
||||
"必须纳入 counter_evidence,并说明它如何影响结论置信度。",
|
||||
],
|
||||
}
|
||||
validate_chapter_brief(brief)
|
||||
(out_dir / f"{chapter_id}.json").write_text(
|
||||
json.dumps(brief, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
briefs.append(brief)
|
||||
return briefs
|
||||
|
||||
|
||||
def _source_ids_from_item(item: dict) -> list[str]:
|
||||
if item.get("source_ids"):
|
||||
return list(item.get("source_ids") or [])
|
||||
if item.get("source_id"):
|
||||
return [item["source_id"]]
|
||||
return []
|
||||
|
||||
|
||||
def build_compressed_findings(project_root: Path) -> list[dict]:
|
||||
"""Compress packet-level evidence into chapter-level writing inputs.
|
||||
|
||||
This is intentionally deterministic: it does not invent a better narrative,
|
||||
but it forces a chapter-level evidence map before any model writes prose.
|
||||
"""
|
||||
brief_dir = project_root / "phase2" / "chapter_briefs"
|
||||
if not brief_dir.exists() or not list(brief_dir.glob("ch*.json")):
|
||||
briefs = build_chapter_briefs(project_root)
|
||||
else:
|
||||
briefs = [
|
||||
json.loads(path.read_text(encoding="utf-8"))
|
||||
for path in sorted(brief_dir.glob("ch*.json"))
|
||||
]
|
||||
out_dir = project_root / "phase2" / "compressed_findings"
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
findings: list[dict] = []
|
||||
for brief in briefs:
|
||||
validate_chapter_brief(brief)
|
||||
core_claims = brief.get("core_claims") or []
|
||||
evidence_items = brief.get("evidence_items") or []
|
||||
first_claim = core_claims[0] if core_claims else {}
|
||||
chapter_thesis = first_claim.get("claim") or f"{brief['chapter_title']} 需要以证据为中心重写。"
|
||||
finding = {
|
||||
"chapter_id": brief["chapter_id"],
|
||||
"chapter_title": brief["chapter_title"],
|
||||
"packet_ids": brief["packet_ids"],
|
||||
"chapter_thesis": chapter_thesis,
|
||||
"key_findings": [
|
||||
{
|
||||
"finding": claim.get("claim") or claim.get("summary") or str(claim),
|
||||
"source_ids": _source_ids_from_item(claim),
|
||||
"confidence": claim.get("confidence", "medium"),
|
||||
}
|
||||
for claim in core_claims
|
||||
],
|
||||
"evidence_landings": [
|
||||
{
|
||||
"evidence": item.get("summary") or item.get("finding") or item.get("quote") or str(item),
|
||||
"source_ids": _source_ids_from_item(item),
|
||||
"landing_hint": item.get("landing_hint", "用于支撑本章关键判断或整改动作。"),
|
||||
}
|
||||
for item in evidence_items
|
||||
],
|
||||
"counter_evidence": brief["counter_evidence"],
|
||||
"source_ids": brief["source_ids"],
|
||||
"cached_source_paths": brief.get("cached_source_paths", []),
|
||||
"cached_source_excerpts": brief.get("cached_source_excerpts", []),
|
||||
"open_questions": brief["open_questions"],
|
||||
"writing_plan": [
|
||||
"先写本章判断,不按 packet 顺序堆砌。",
|
||||
"每个二级小节至少落下具体审计发现、法规要求、记录/参数或整改证据。",
|
||||
"正文末尾必须保留“证据落点与待补证据”表。",
|
||||
],
|
||||
}
|
||||
validate_compressed_finding(finding)
|
||||
(out_dir / f"{brief['chapter_id']}.json").write_text(
|
||||
json.dumps(finding, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
findings.append(finding)
|
||||
return findings
|
||||
|
||||
|
||||
def build_chapter_user_prompt(brief: dict) -> str:
|
||||
return (
|
||||
"请根据以下 compressed finding / chapter brief 写一章正式中文 Markdown 正文。\n"
|
||||
"目标是形成一个完整章节,而不是 packet 摘要。避免碎片化,按金字塔结构组织:章首先给结论,再用证据支撑。\n"
|
||||
"要求:标题必须是观点型判断;每个数字和事实保留 [src_xxx];纳入反方证据;不要出现调度元数据。\n"
|
||||
"如 brief 中包含 cached_source_paths,说明这些是已抓取到本地的核心一手/权威信源快照;优先使用 packet 已摘录的原文,并在证据不足时标记需要从本地快照补摘录,不要重新联网检索。\n"
|
||||
"禁止写空泛咨询腔。每个二级小节都必须至少落下 2 个具体审计发现、法规要求、SOP/记录/参数/现场观察或整改证据;不要只写原则。\n"
|
||||
"正文末尾必须增加“证据落点与待补证据”小节,用表格列出:关键判断、已使用证据 source_id、已落地整改动作、仍缺证据。若证据不足,直接标注需回炉 Phase 2,不要用泛泛表述补齐。\n"
|
||||
"只输出 Markdown,不要输出解释。\n\n"
|
||||
f"{json.dumps(brief, ensure_ascii=False, indent=2)}"
|
||||
)
|
||||
|
||||
|
||||
class ChapterAssemblyWorker:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
role: RoleDefinition,
|
||||
client: ChatClient,
|
||||
skill_registry: SkillRegistry | None = None,
|
||||
) -> None:
|
||||
self.role = role
|
||||
self.client = client
|
||||
self.skill_registry = skill_registry or SkillRegistry()
|
||||
|
||||
def _system_prompt(self) -> str:
|
||||
skill_texts = []
|
||||
for name in self.role.skills:
|
||||
try:
|
||||
skill_texts.append(f"# Skill: {name}\n\n{self.skill_registry.read(name)}")
|
||||
except FileNotFoundError:
|
||||
skill_texts.append(f"# Skill: {name}\n\n[missing skill: {name}]")
|
||||
return (
|
||||
f"{self.role.identity}\n\n"
|
||||
"你是 Deep Research v0.20 的中文章节组装 worker。\n"
|
||||
"你的职责是把结构化证据包收束成连贯章节,解决并发研究造成的碎片化。\n"
|
||||
"不得编造来源,不得删除关键反方证据。\n\n"
|
||||
+ "\n\n".join(skill_texts)
|
||||
)
|
||||
|
||||
def write_chapter(self, *, project_root: Path, brief: dict) -> Path:
|
||||
if "key_findings" in brief:
|
||||
validate_compressed_finding(brief)
|
||||
else:
|
||||
validate_chapter_brief(brief)
|
||||
markdown = self.client.chat_complete(
|
||||
model=self.role.model,
|
||||
system=self._system_prompt(),
|
||||
user=build_chapter_user_prompt(brief),
|
||||
temperature=self.role.temperature,
|
||||
max_tokens=self.role.max_tokens,
|
||||
tag=f"chapter:{brief['chapter_id']}",
|
||||
)
|
||||
validate_chapter_markdown_citations(markdown, brief)
|
||||
out = project_root / "phase2" / "drafts" / f"{brief['chapter_id']}.md"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(markdown.rstrip() + "\n", encoding="utf-8")
|
||||
return out
|
||||
|
||||
|
||||
def _write_chapter_error(project_root: Path, brief: dict, error: Exception) -> None:
|
||||
path = project_root / "phase2" / "chapter_errors" / f"{brief.get('chapter_id', 'unknown')}.json"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"chapter_id": brief.get("chapter_id"),
|
||||
"status": "failed",
|
||||
"error": str(error),
|
||||
}
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def run_chapter_assembly_workers(
|
||||
*,
|
||||
project_root: Path,
|
||||
briefs: list[dict],
|
||||
runtime: RuntimeProfile,
|
||||
client_factory: Callable[[RoleDefinition], ChatClient],
|
||||
workers: int,
|
||||
) -> int:
|
||||
role = runtime.role_for_task("chapter_assembly")
|
||||
max_workers = max(1, min(workers, role.max_concurrency))
|
||||
|
||||
def run_one(brief: dict) -> tuple[dict, Path | None, Exception | None]:
|
||||
try:
|
||||
worker = ChapterAssemblyWorker(role=role, client=client_factory(role))
|
||||
return brief, worker.write_chapter(project_root=project_root, brief=brief), None
|
||||
except Exception as error:
|
||||
return brief, None, error
|
||||
|
||||
written = 0
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||
futures = [pool.submit(run_one, brief) for brief in briefs]
|
||||
for future in as_completed(futures):
|
||||
brief, path, error = future.result()
|
||||
if error is not None:
|
||||
_write_chapter_error(project_root, brief, error)
|
||||
continue
|
||||
if path is None:
|
||||
_write_chapter_error(project_root, brief, RuntimeError("chapter worker returned no output path"))
|
||||
continue
|
||||
written += 1
|
||||
return written
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Phase 0 user-provided material ingestion."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import os
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
DEFAULT_FIRERED_OCR_ENDPOINT = "http://192.168.50.100:8001"
|
||||
DEFAULT_OCR_MAX_PAGES = 50
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OcrResult:
|
||||
text: str
|
||||
pages_processed: int
|
||||
output_path: Path
|
||||
|
||||
|
||||
def safe_filename(path: Path) -> str:
|
||||
name = path.name.strip()
|
||||
return name or "material"
|
||||
|
||||
|
||||
def extract_pdf_text(path: Path) -> tuple[str, int, bool]:
|
||||
from pypdf import PdfReader
|
||||
|
||||
reader = PdfReader(str(path))
|
||||
chunks: list[str] = []
|
||||
for index, page in enumerate(reader.pages, start=1):
|
||||
text = (page.extract_text() or "").strip()
|
||||
if text:
|
||||
chunks.append(f"\n\n## Page {index}\n\n{text}")
|
||||
combined = "".join(chunks).strip()
|
||||
ocr_required = len(combined) < max(20, len(reader.pages) * 20)
|
||||
return combined, len(reader.pages), ocr_required
|
||||
|
||||
|
||||
def ocr_endpoint_from_env() -> str:
|
||||
return os.environ.get("DEEP_RESEARCH_OCR_ENDPOINT", DEFAULT_FIRERED_OCR_ENDPOINT).rstrip("/")
|
||||
|
||||
|
||||
def ocr_max_pages_from_env() -> int:
|
||||
raw = os.environ.get("DEEP_RESEARCH_OCR_MAX_PAGES")
|
||||
if not raw:
|
||||
return DEFAULT_OCR_MAX_PAGES
|
||||
try:
|
||||
return max(1, int(raw))
|
||||
except ValueError:
|
||||
return DEFAULT_OCR_MAX_PAGES
|
||||
|
||||
|
||||
def render_pdf_pages(pdf_path: Path, output_dir: Path, *, max_pages: int) -> list[Path]:
|
||||
import fitz
|
||||
|
||||
pages_dir = output_dir / f"{pdf_path.stem}.ocr-pages"
|
||||
pages_dir.mkdir(parents=True, exist_ok=True)
|
||||
image_paths: list[Path] = []
|
||||
doc = fitz.open(pdf_path)
|
||||
try:
|
||||
for index, page in enumerate(doc[:max_pages], start=1):
|
||||
pix = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False)
|
||||
image_path = pages_dir / f"page-{index:03d}.png"
|
||||
pix.save(image_path)
|
||||
image_paths.append(image_path)
|
||||
finally:
|
||||
doc.close()
|
||||
return image_paths
|
||||
|
||||
|
||||
def data_url_for_image(path: Path) -> str:
|
||||
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
|
||||
return f"data:image/png;base64,{encoded}"
|
||||
|
||||
|
||||
def call_firered_ocr(image_path: Path, *, endpoint: str) -> str:
|
||||
payload = {
|
||||
"model": "firered-ocr",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "请识别图片中的全部文字,保持原有顺序,只输出文字。"},
|
||||
{"type": "image_url", "image_url": {"url": data_url_for_image(image_path)}},
|
||||
],
|
||||
}
|
||||
],
|
||||
"temperature": 0,
|
||||
"max_tokens": 3000,
|
||||
}
|
||||
response = requests.post(f"{endpoint.rstrip('/')}/v1/chat/completions", json=payload, timeout=60)
|
||||
if not response.ok:
|
||||
raise RuntimeError(f"{response.status_code} {response.text[:500]}")
|
||||
data = response.json()
|
||||
return str(data["choices"][0]["message"].get("content") or "").strip()
|
||||
|
||||
|
||||
def ocr_pdf_with_firered(*, pdf_path: Path, output_dir: Path, endpoint: str, max_pages: int) -> OcrResult:
|
||||
image_paths = render_pdf_pages(pdf_path, output_dir, max_pages=max_pages)
|
||||
chunks: list[str] = []
|
||||
for index, image_path in enumerate(image_paths, start=1):
|
||||
text = call_firered_ocr(image_path, endpoint=endpoint)
|
||||
if text:
|
||||
chunks.append(f"\n\n## OCR Page {index}\n\n{text}")
|
||||
combined = "".join(chunks).strip()
|
||||
output_path = output_dir / f"{pdf_path.stem}.ocr.md"
|
||||
body = [
|
||||
f"# OCR Material: {pdf_path.name}",
|
||||
"",
|
||||
f"- source_path: {pdf_path}",
|
||||
f"- endpoint: {endpoint}",
|
||||
f"- pages_processed: {len(image_paths)}",
|
||||
"",
|
||||
combined or "OCR 未返回可用文本。",
|
||||
"",
|
||||
]
|
||||
output_path.write_text("\n".join(body), encoding="utf-8")
|
||||
return OcrResult(text=combined, pages_processed=len(image_paths), output_path=output_path)
|
||||
|
||||
|
||||
def ingest_input_materials(project_root: Path, materials: list[str] | None) -> list[dict[str, Any]]:
|
||||
inventory: list[dict[str, Any]] = []
|
||||
if not materials:
|
||||
return inventory
|
||||
|
||||
inputs_dir = project_root / "phase0" / "inputs"
|
||||
extracted_dir = project_root / "phase0" / "extracted"
|
||||
ocr_endpoint = ocr_endpoint_from_env()
|
||||
ocr_max_pages = ocr_max_pages_from_env()
|
||||
inputs_dir.mkdir(parents=True, exist_ok=True)
|
||||
extracted_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for raw in materials:
|
||||
source = Path(raw).expanduser()
|
||||
if not source.exists():
|
||||
inventory.append({"kind": "note", "note": raw})
|
||||
continue
|
||||
|
||||
copied = inputs_dir / safe_filename(source)
|
||||
shutil.copy2(source, copied)
|
||||
item: dict[str, Any] = {
|
||||
"kind": source.suffix.lower().lstrip(".") or "file",
|
||||
"source_path": str(source),
|
||||
"copied_to": str(copied.relative_to(project_root)),
|
||||
"size_bytes": source.stat().st_size,
|
||||
}
|
||||
|
||||
if source.suffix.lower() == ".pdf":
|
||||
text, pages, ocr_required = extract_pdf_text(source)
|
||||
extracted = extracted_dir / f"{source.stem}.md"
|
||||
ocr_result: OcrResult | None = None
|
||||
ocr_error: str | None = None
|
||||
if ocr_required:
|
||||
try:
|
||||
ocr_result = ocr_pdf_with_firered(
|
||||
pdf_path=source,
|
||||
output_dir=extracted_dir,
|
||||
endpoint=ocr_endpoint,
|
||||
max_pages=min(pages, ocr_max_pages),
|
||||
)
|
||||
if ocr_result.text:
|
||||
text = "\n\n".join(part for part in [text, ocr_result.text] if part)
|
||||
except Exception as exc: # noqa: BLE001 - ingestion should not block project init.
|
||||
ocr_error = str(exc)
|
||||
|
||||
body = [
|
||||
f"# Extracted Material: {source.name}",
|
||||
"",
|
||||
f"- source_path: {source}",
|
||||
f"- copied_to: {copied.relative_to(project_root)}",
|
||||
f"- pages: {pages}",
|
||||
f"- ocr_required: {str(ocr_required).lower()}",
|
||||
f"- ocr_status: {'completed' if ocr_result else 'failed' if ocr_error else 'not_required'}",
|
||||
"",
|
||||
text or "未能从 PDF 直接抽取文本;该材料可能需要 OCR。",
|
||||
"",
|
||||
]
|
||||
if ocr_error:
|
||||
body.extend(["## OCR Error", "", ocr_error, ""])
|
||||
extracted.write_text("\n".join(body), encoding="utf-8")
|
||||
item.update(
|
||||
{
|
||||
"pages": pages,
|
||||
"extracted_to": str(extracted.relative_to(project_root)),
|
||||
"text_chars": len(text),
|
||||
"ocr_required": ocr_required,
|
||||
"ocr_status": "completed" if ocr_result else "failed" if ocr_error else "not_required",
|
||||
}
|
||||
)
|
||||
if ocr_result:
|
||||
item.update(
|
||||
{
|
||||
"ocr_endpoint": ocr_endpoint,
|
||||
"ocr_pages_processed": ocr_result.pages_processed,
|
||||
"ocr_extracted_to": str(ocr_result.output_path.relative_to(project_root)),
|
||||
"ocr_text_chars": len(ocr_result.text),
|
||||
}
|
||||
)
|
||||
if ocr_error:
|
||||
item["ocr_error"] = ocr_error
|
||||
else:
|
||||
item["ocr_required"] = source.suffix.lower() in {".png", ".jpg", ".jpeg", ".tif", ".tiff"}
|
||||
inventory.append(item)
|
||||
|
||||
return inventory
|
||||
|
||||
|
||||
def render_material_inventory(inventory: list[dict[str, Any]]) -> str:
|
||||
if not inventory:
|
||||
return "- 暂无;可通过 `--input-material` 加入审计报告、问题清单或内部记录。"
|
||||
lines: list[str] = []
|
||||
for item in inventory:
|
||||
if item.get("kind") == "note":
|
||||
lines.append(f"- 备注:{item.get('note', '')}")
|
||||
continue
|
||||
marker = ";需要 OCR" if item.get("ocr_required") else ""
|
||||
ocr = f";OCR:{item.get('ocr_status')}" if item.get("ocr_status") else ""
|
||||
extracted = item.get("extracted_to")
|
||||
extra = f";抽取文本:{extracted}" if extracted else ""
|
||||
lines.append(
|
||||
f"- {item.get('copied_to')}({item.get('kind')},{item.get('size_bytes', 0)} bytes{extra}{marker}{ocr})"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Research method registry for Phase 1 framework selection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_METHOD_CONFIG = REPO_ROOT / "configs" / "research_methods.yaml"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResearchMethod:
|
||||
key: str
|
||||
name: str
|
||||
best_for: list[str]
|
||||
structure_principle: str
|
||||
task_axes: list[str]
|
||||
framework_sections: list[str]
|
||||
integrated_lanes: list[str]
|
||||
|
||||
|
||||
class ResearchMethodRegistry:
|
||||
def __init__(self, path: Path | None = None) -> None:
|
||||
self.path = path or DEFAULT_METHOD_CONFIG
|
||||
self._data = self._load()
|
||||
|
||||
def _load(self) -> dict[str, Any]:
|
||||
if not self.path.exists():
|
||||
raise FileNotFoundError(f"research method config not found: {self.path}")
|
||||
data = yaml.safe_load(self.path.read_text(encoding="utf-8")) or {}
|
||||
if not isinstance(data, dict) or "methods" not in data:
|
||||
raise ValueError(f"invalid research method config: {self.path}")
|
||||
return data
|
||||
|
||||
@property
|
||||
def default_method(self) -> str:
|
||||
return (self._data.get("defaults") or {}).get("method", "mckinsey_market")
|
||||
|
||||
def list_names(self) -> list[str]:
|
||||
return sorted((self._data.get("methods") or {}).keys())
|
||||
|
||||
def get(self, key: str | None = None) -> ResearchMethod:
|
||||
selected = key or self.default_method
|
||||
methods = self._data.get("methods") or {}
|
||||
if selected not in methods:
|
||||
raise KeyError(f"unknown research_method: {selected}")
|
||||
item = methods[selected] or {}
|
||||
return ResearchMethod(
|
||||
key=selected,
|
||||
name=item.get("name", selected),
|
||||
best_for=list(item.get("best_for") or []),
|
||||
structure_principle=item.get("structure_principle", ""),
|
||||
task_axes=list(item.get("task_axes") or []),
|
||||
framework_sections=list(item.get("framework_sections") or []),
|
||||
integrated_lanes=list(item.get("integrated_lanes") or item.get("task_axes") or []),
|
||||
)
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Deterministic orchestration helpers for v0.20."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.runtime.artifacts import ensure_phase_dirs, load_manifest, write_manifest
|
||||
from scripts.runtime.methods import ResearchMethodRegistry
|
||||
from scripts.runtime.tasks import generate_task_cards, generate_task_cards_from_research_brief, write_task_cards
|
||||
|
||||
|
||||
def utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def create_phase2_task_cards(
|
||||
project_root: Path,
|
||||
*,
|
||||
axes: list[str] | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> list[dict[str, object]]:
|
||||
framework = project_root / "phase1" / "framework.md"
|
||||
if not framework.exists():
|
||||
raise FileNotFoundError(f"framework not found: {framework}")
|
||||
method_key = None
|
||||
if (project_root / "manifest.json").exists():
|
||||
method_key = load_manifest(project_root).get("research_method")
|
||||
method = ResearchMethodRegistry().get(method_key)
|
||||
research_brief_path = project_root / "phase1" / "research_brief.json"
|
||||
framework_text = framework.read_text(encoding="utf-8")
|
||||
if research_brief_path.exists():
|
||||
research_brief = json.loads(research_brief_path.read_text(encoding="utf-8"))
|
||||
if not research_brief.get("materials"):
|
||||
material_inventory = load_manifest(project_root).get("material_inventory") or []
|
||||
materials = []
|
||||
for item in material_inventory:
|
||||
rel = item.get("ocr_extracted_to") or item.get("extracted_to") or item.get("copied_to")
|
||||
if rel:
|
||||
materials.append({"path": rel, "role": "input_material"})
|
||||
if materials:
|
||||
research_brief["materials"] = materials
|
||||
cards = generate_task_cards_from_research_brief(
|
||||
project_root.name,
|
||||
framework_text,
|
||||
research_brief,
|
||||
axes=axes,
|
||||
method=method,
|
||||
)
|
||||
else:
|
||||
cards = generate_task_cards(project_root.name, framework_text, axes=axes, method=method)
|
||||
if not dry_run:
|
||||
ensure_phase_dirs(project_root)
|
||||
write_task_cards(project_root / "phase2" / "task_cards.json", cards)
|
||||
manifest = load_manifest(project_root)
|
||||
phase2 = manifest.setdefault("phase2", {})
|
||||
phase2.update(
|
||||
{
|
||||
"status": "in_progress",
|
||||
"runtime": "python-core-v0.20",
|
||||
"research_method": method.key,
|
||||
"task_cards_path": "phase2/task_cards.json",
|
||||
"task_cards_total": len(cards),
|
||||
"updated_at": utc_now_iso(),
|
||||
}
|
||||
)
|
||||
write_manifest(project_root, manifest)
|
||||
return [card.to_dict() for card in cards]
|
||||
|
||||
|
||||
def write_placeholder_packets(
|
||||
project_root: Path,
|
||||
task_cards: list[dict[str, object]],
|
||||
*,
|
||||
dry_run: bool = False,
|
||||
) -> int:
|
||||
"""Create packet skeletons for manual/API completion.
|
||||
|
||||
This keeps the first v0.20 implementation deterministic and resumable; LLM
|
||||
calls can later fill the same schema without changing downstream readers.
|
||||
"""
|
||||
count = 0
|
||||
for card in task_cards:
|
||||
packet_path = project_root / str(card["output_packet"])
|
||||
packet = {
|
||||
"task_id": card["task_id"],
|
||||
"claims": [],
|
||||
"evidence_items": [],
|
||||
"counter_evidence": [],
|
||||
"source_ids": [],
|
||||
"source_quality_notes": [],
|
||||
"open_questions": ["待由 Python role worker 调用模型补全。"],
|
||||
"raw_quotes_or_notes": [],
|
||||
}
|
||||
if not dry_run:
|
||||
packet_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
packet_path.write_text(json.dumps(packet, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
count += 1
|
||||
return count
|
||||
@@ -0,0 +1,902 @@
|
||||
"""Phase 1 project initialization and framework generation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from scripts.runtime.artifacts import PROJECTS_DIR, ensure_phase_dirs, load_manifest, write_manifest
|
||||
from scripts.runtime.materials import ingest_input_materials, render_material_inventory
|
||||
from scripts.runtime.methods import ResearchMethod, ResearchMethodRegistry
|
||||
from scripts.runtime.tasks import AXIS_ROUTES
|
||||
|
||||
|
||||
def utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def slugify_topic(topic: str) -> str:
|
||||
slug = re.sub(r"[^a-zA-Z0-9]+", "-", topic.lower()).strip("-")
|
||||
if slug:
|
||||
return slug[:80]
|
||||
digest = hashlib.sha1(topic.encode("utf-8")).hexdigest()[:8]
|
||||
return f"research-{digest}"
|
||||
|
||||
|
||||
def create_project(
|
||||
*,
|
||||
topic: str,
|
||||
slug: str | None = None,
|
||||
projects_dir: Path = PROJECTS_DIR,
|
||||
method_key: str | None = None,
|
||||
report_type: str = "research",
|
||||
model_profile: str = "medium",
|
||||
target_words: int = 30000,
|
||||
input_materials: list[str] | None = None,
|
||||
) -> Path:
|
||||
method = ResearchMethodRegistry().get(method_key)
|
||||
project_slug = slug or slugify_topic(topic)
|
||||
project_root = projects_dir / project_slug
|
||||
if project_root.exists():
|
||||
raise FileExistsError(f"project already exists: {project_root}")
|
||||
ensure_phase_dirs(project_root)
|
||||
(project_root / "phase0" / "inputs").mkdir(parents=True, exist_ok=True)
|
||||
(project_root / "phase0" / "extracted").mkdir(parents=True, exist_ok=True)
|
||||
material_inventory = ingest_input_materials(project_root, input_materials)
|
||||
now = utc_now_iso()
|
||||
manifest: dict[str, Any] = {
|
||||
"version": "0.20.0",
|
||||
"runtime": "python-core-v0.20",
|
||||
"topic": topic,
|
||||
"slug": project_slug,
|
||||
"report_title": topic,
|
||||
"type": report_type,
|
||||
"work_language": "zh",
|
||||
"model_profile": model_profile,
|
||||
"research_method": method.key,
|
||||
"target_words": target_words,
|
||||
"input_materials": input_materials or [],
|
||||
"material_inventory": material_inventory,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
"phase1": {
|
||||
"status": "initialized",
|
||||
"approved": False,
|
||||
"requires_user_interview": True,
|
||||
"material_brief_path": "phase1/material_brief.md",
|
||||
},
|
||||
"phase2": {"status": "pending"},
|
||||
"phase3": {"status": "pending"},
|
||||
"phase4": {"status": "pending"},
|
||||
}
|
||||
write_manifest(project_root, manifest)
|
||||
_write_interview_seed(project_root, manifest, method)
|
||||
write_material_brief(project_root, manifest, method)
|
||||
return project_root
|
||||
|
||||
|
||||
def _write_interview_seed(project_root: Path, manifest: dict[str, Any], method: ResearchMethod) -> None:
|
||||
material_text = render_material_inventory(manifest.get("material_inventory") or [])
|
||||
text = (
|
||||
f"# Phase 1 访谈记录\n\n"
|
||||
f"- 主题:{manifest['topic']}\n"
|
||||
f"- 研究方法:{method.key} - {method.name}\n"
|
||||
f"- 报告类型:{manifest['type']}\n"
|
||||
f"- 目标字数:{manifest['target_words']}\n"
|
||||
f"- 工作语言:中文主写作;检索关键词、证据摘录和来源笔记可保留英文。\n\n"
|
||||
f"## 已提供材料\n\n{material_text}\n\n"
|
||||
"## 后续访谈问题\n\n"
|
||||
"1. 本报告最重要的决策用途是什么?\n"
|
||||
"2. 是否有必须覆盖或必须排除的公司、产品、工艺、市场或法规范围?\n"
|
||||
"3. 结论偏好是战略建议、风险清单、投资判断,还是执行路线图?\n"
|
||||
)
|
||||
(project_root / "phase1" / "interview.md").write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def _material_excerpt(project_root: Path, rel_path: str, *, max_chars: int = 1200) -> str:
|
||||
path = project_root / rel_path
|
||||
if not path.exists():
|
||||
return "(未找到抽取文本)"
|
||||
text = path.read_text(encoding="utf-8")
|
||||
compact = "\n".join(line.rstrip() for line in text.splitlines() if line.strip())
|
||||
return compact[:max_chars] + ("..." if len(compact) > max_chars else "")
|
||||
|
||||
|
||||
def _derive_material_observations(project_root: Path, inventory: list[dict[str, Any]]) -> list[str]:
|
||||
combined_parts: list[str] = []
|
||||
for item in inventory:
|
||||
rel = item.get("ocr_extracted_to") or item.get("extracted_to")
|
||||
if rel and (project_root / rel).exists():
|
||||
combined_parts.append((project_root / rel).read_text(encoding="utf-8"))
|
||||
text = "\n".join(combined_parts)
|
||||
checks = [
|
||||
("审计范围覆盖生产管理、原液、制剂和无菌相关模块,报告需要同时处理 GMP 合规、工艺转移和运营协同,而不是只写质量体系。", ["生产管理", "原液", "制剂", "无菌"]),
|
||||
("材料显示高风险项为 0、中风险项为 1,适合采用“商业化 readiness 与系统成熟度差距”而非“体系失控”作为初始假设。", ["高风险 0", "中风险1", "低风险7"]),
|
||||
("商业化经验、无菌保障细节、文件要求与执行一致性是需要访谈确认的主线风险。", ["商业化经验不足", "无菌保障", "文件要求与执行一致性"]),
|
||||
("工艺规程、批记录、CPP/CQA、VMPR/VMP、验证主计划等内容反复出现,说明工艺验证和商业化文件体系可能是 Phase 2 的重点证据轴。", ["CPP", "CQA", "VMPR", "VMP"]),
|
||||
("温度、压差、WFI、冷却段微生物、RABS/ORABS、first air、APS 等无菌和设施细节需要映射到 EU Annex 1、NMPA GMP 和企业 SOP。", ["温度", "压差", "WFI", "APS"]),
|
||||
("复盘材料包含责任人和局部答复,后续整改路线图应尽量回填 owner、期限、关闭证据和复核机制。", ["填写人", "是否已经回答完整", "整改"]),
|
||||
]
|
||||
observations = [message for message, needles in checks if any(needle in text for needle in needles)]
|
||||
return observations or ["材料已导入但尚未形成足够结构化判断;需要先访谈确认研究用途、范围和优先级。"]
|
||||
|
||||
|
||||
def write_material_brief(
|
||||
project_root: Path,
|
||||
manifest: dict[str, Any] | None = None,
|
||||
method: ResearchMethod | None = None,
|
||||
) -> Path:
|
||||
"""Write a Phase 0/1 material brief that must be reviewed before Phase 2."""
|
||||
manifest = manifest or load_manifest(project_root)
|
||||
method = method or ResearchMethodRegistry().get(manifest.get("research_method"))
|
||||
inventory = manifest.get("material_inventory") or []
|
||||
lines = [
|
||||
f"# Phase 0 材料简报:{manifest.get('topic', project_root.name)}",
|
||||
"",
|
||||
"status: 待用户确认",
|
||||
f"research_method: {method.key}",
|
||||
"",
|
||||
"## 已导入材料",
|
||||
"",
|
||||
render_material_inventory(inventory),
|
||||
"",
|
||||
"## 材料初步解读",
|
||||
"",
|
||||
"以下内容由 Python core 从已落盘材料抽样生成,只作为访谈起点;不得直接视为最终结论。",
|
||||
"",
|
||||
]
|
||||
lines.extend(["## 初步问题聚类(待访谈确认)", ""])
|
||||
for observation in _derive_material_observations(project_root, inventory):
|
||||
lines.append(f"- {observation}")
|
||||
lines.append("")
|
||||
lines.append("## 材料摘录")
|
||||
lines.append("")
|
||||
for item in inventory:
|
||||
rel = item.get("ocr_extracted_to") or item.get("extracted_to")
|
||||
if not rel:
|
||||
continue
|
||||
lines.extend(
|
||||
[
|
||||
f"### {Path(rel).name}",
|
||||
"",
|
||||
_material_excerpt(project_root, rel),
|
||||
"",
|
||||
]
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"## 建议访谈确认点",
|
||||
"",
|
||||
"1. 本报告的最重要用途是什么:内部整改、客户沟通、董事会决策,还是外部审计准备?",
|
||||
"2. 哪些审计发现最需要优先展开:无菌保障、工艺验证、数据完整性、质量体系闭环,还是运营协同?",
|
||||
"3. 是否存在必须排除或脱敏的项目、人员、客户、产品或工艺信息?",
|
||||
"4. 短中长期整改的时间边界如何定义,例如 30/90/180 天,还是按临床/商业化里程碑划分?",
|
||||
"5. 是否需要把 NMPA、FDA、EMA、ICH、WHO 的法规基线分别映射到整改责任人和证据包?",
|
||||
"",
|
||||
"## Gate",
|
||||
"",
|
||||
"请用户确认本材料简报与访谈问题后,再生成或批准 `phase1/framework.md` 并进入 Phase 2。",
|
||||
"",
|
||||
]
|
||||
)
|
||||
out = project_root / "phase1" / "material_brief.md"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text("\n".join(lines), encoding="utf-8")
|
||||
return out
|
||||
|
||||
|
||||
def _axis_prompt_brief(axis: str, method: ResearchMethod) -> str:
|
||||
prompts = {
|
||||
"input_material_findings": "从用户材料中提取现场事实、审计发现、复盘记录和内部答复,并标注原始材料位置。",
|
||||
"nmpa_fda_ema_ich_who_baseline": "把 NMPA、FDA、EMA、ICH、WHO、药典或 Annex 1 等要求转化为可核验的法规基线,并纳入 FDA warning letters 与会议材料作为执法尺度参照。",
|
||||
"quality_system_gap": "把现场发现映射到质量体系流程缺口,覆盖偏差、变更、CAPA、文件、培训和数据完整性;优先检索 FDA warning letters 中同类缺陷的执法表述。",
|
||||
"manufacturing_process_risk": "围绕生产工艺、设施、公用系统、CPP/CQA、验证和无菌保障识别系统性风险,并用 FDA warning letters / inspection enforcement examples 校准严重度。",
|
||||
"operations_management_gap": "诊断运营管理、跨部门协同、会议机制、指标体系和交付节奏的结构性问题,并参考 FDA 会议纪要/meeting materials 中对质量治理的关注点。",
|
||||
"team_capability": "识别人员能力、岗位职责、质量文化和管理梯队方面的缺口与建设路径。",
|
||||
"capa_roadmap": "把差距转化为短中长期 CAPA 组合,要求绑定 owner、期限、优先级、关闭证据和复核机制。",
|
||||
"verification_evidence": "定义整改完成后可被审计接受的验证证据,包括记录、报告、趋势和管理评审输入。",
|
||||
"counter": "主动寻找反方证据、限制条件和可能降低严重度或改变优先级的解释,避免单向论证。",
|
||||
}
|
||||
return prompts.get(axis, f"按照 `{method.key}` 方法,对 {axis} 轴进行证据收集、证伪和结构化归纳。")
|
||||
|
||||
|
||||
def _material_paths(manifest: dict[str, Any]) -> list[dict[str, str]]:
|
||||
materials: list[dict[str, str]] = []
|
||||
for item in manifest.get("material_inventory") or []:
|
||||
rel = item.get("ocr_extracted_to") or item.get("extracted_to") or item.get("copied_to")
|
||||
if rel:
|
||||
materials.append({"path": rel, "role": "input_material"})
|
||||
return materials
|
||||
|
||||
|
||||
def _keywords_from_title(title: str) -> list[str]:
|
||||
english = re.findall(r"[A-Za-z][A-Za-z0-9/+-]{1,}", title)
|
||||
chinese_parts = re.split(r"[,,、;;::\s]+|和|与|及|的|在|为|从|来自|集中|决定|需要|形成|成为|不是|而是", title)
|
||||
domain_terms = [
|
||||
"审计",
|
||||
"商业化",
|
||||
"阶段门",
|
||||
"风险",
|
||||
"法规",
|
||||
"欧盟",
|
||||
"NMPA",
|
||||
"GMP",
|
||||
"ICH",
|
||||
"无菌",
|
||||
"RABS",
|
||||
"First Air",
|
||||
"APS",
|
||||
"灯检",
|
||||
"隧道",
|
||||
"原液",
|
||||
"WFI",
|
||||
"SCADA",
|
||||
"EMS",
|
||||
"CPP",
|
||||
"CQA",
|
||||
"PPQ",
|
||||
"清洁验证",
|
||||
"偏差",
|
||||
"变更",
|
||||
"CAPA",
|
||||
"数据完整性",
|
||||
"人员",
|
||||
"培训",
|
||||
"质量文化",
|
||||
"运营",
|
||||
"跨部门",
|
||||
"指标",
|
||||
"团队",
|
||||
"CDMO",
|
||||
"整改",
|
||||
"owner",
|
||||
]
|
||||
title_terms = [term for term in domain_terms if term in title]
|
||||
keywords = [item.strip() for item in [*english, *title_terms, *chinese_parts] if len(item.strip()) >= 2]
|
||||
seen: set[str] = set()
|
||||
unique: list[str] = []
|
||||
for keyword in keywords:
|
||||
if keyword not in seen:
|
||||
seen.add(keyword)
|
||||
unique.append(keyword)
|
||||
return unique[:12]
|
||||
|
||||
|
||||
def _material_lines_for_chapter(project_root: Path, manifest: dict[str, Any], title: str, *, limit: int = 4) -> list[str]:
|
||||
keywords = _keywords_from_title(title)
|
||||
candidates: list[tuple[int, int, str]] = []
|
||||
order = 0
|
||||
for item in manifest.get("material_inventory") or []:
|
||||
rel = item.get("ocr_extracted_to") or item.get("extracted_to")
|
||||
if not rel:
|
||||
continue
|
||||
path = project_root / rel
|
||||
if not path.exists():
|
||||
continue
|
||||
for raw in path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw.strip()
|
||||
if len(line) < 8 or len(line) > 220:
|
||||
continue
|
||||
if line.startswith("#") or line.startswith("- source_path:") or line.startswith("- extracted_at:"):
|
||||
continue
|
||||
if "OCR Material:" in line:
|
||||
continue
|
||||
if re.match(r"^(审计对象|审计执行方|审计执行人|审计时间)[::]", line):
|
||||
continue
|
||||
score = sum(1 for keyword in keywords if keyword and keyword in line)
|
||||
if score:
|
||||
order += 1
|
||||
candidates.append((score, order, f"{rel}:{line}"))
|
||||
candidates.sort(key=lambda item: (-item[0], item[1]))
|
||||
return [line for _, _, line in candidates[:limit]]
|
||||
|
||||
|
||||
def _minimum_evidence_for_method(method: ResearchMethod) -> dict[str, Any]:
|
||||
if method.key == "gmp_quality_operations_diagnosis":
|
||||
return {
|
||||
"local_material_quotes": 2,
|
||||
"official_regulatory_or_guideline_sources": 2,
|
||||
"enforcement_or_best_practice_precedents": 1,
|
||||
"counter_evidence_or_boundary_conditions": 1,
|
||||
"actionable_remediation_items": 3,
|
||||
}
|
||||
return {
|
||||
"high_quality_sources": 4,
|
||||
"tier_1_2_sources": 2,
|
||||
"counter_evidence_or_boundary_conditions": 1,
|
||||
"decision_relevant_implications": 2,
|
||||
}
|
||||
|
||||
|
||||
def _central_thesis(manifest: dict[str, Any], method: ResearchMethod) -> str:
|
||||
topic = manifest.get("topic") or manifest.get("report_title") or "本研究主题"
|
||||
if method.key == "gmp_quality_operations_diagnosis":
|
||||
return (
|
||||
f"初始主判断:{topic} 不应只按审计风险项数量来评价,而应从商业化 readiness、"
|
||||
"质量体系运行成熟度、生产工艺证据链和运营协同能力四条线同时诊断。Phase 2 必须用"
|
||||
"现场材料原文、官方法规/指南、执法案例或标杆实践来证明、修正或推翻这一判断。"
|
||||
)
|
||||
return (
|
||||
f"初始主判断:{topic} 需要先形成可被证据推翻的观点型框架,再由 Phase 2 按方法论证据线"
|
||||
"逐项求证;不能把并发检索结果直接堆砌成报告。"
|
||||
)
|
||||
|
||||
|
||||
def _strategy_for_chapter(title: str, method: ResearchMethod) -> dict[str, Any]:
|
||||
"""Return non-tautological Phase 1 strategy text for a chapter title."""
|
||||
if method.key != "gmp_quality_operations_diagnosis":
|
||||
return {
|
||||
"core_question": f"本章需要判断:在什么证据条件下“{title}”成立,它会怎样改变最终决策?",
|
||||
"bold_hypothesis": f"初始假设不是复述标题,而是预判“{title}”背后存在一个可被验证的因果机制;Phase 2 需要找证据支持、修正或推翻这个机制。",
|
||||
"writing_claim": f"本章要把“{title}”写成一个可被证据检验的判断,而不是资料综述。",
|
||||
"counter_evidence": [
|
||||
"是否存在更简单的替代解释,能削弱本章主判断?",
|
||||
"关键证据是否只来自单一来源或利益相关来源?",
|
||||
"是否有反例显示本章判断只适用于部分场景?",
|
||||
],
|
||||
}
|
||||
|
||||
strategies = [
|
||||
(
|
||||
("审计", "阶段门"),
|
||||
{
|
||||
"core_question": "审计报告的低/中风险项计数,是否低估了白帆从临床/受托生产走向商业化标准时需要跨过的阶段门?",
|
||||
"bold_hypothesis": "初始假设:白帆的硬件和文件基础总体可用,但审计材料暴露的是商业化 readiness 缺口,而不是简单的若干孤立缺陷;Phase 2 应验证这些缺口是否集中在无菌保障、工艺验证、质量闭环和运营节奏。",
|
||||
"writing_claim": "本章要先把“风险项清单”翻译成管理层可决策的阶段门地图,说明哪些问题影响商业化放行、客户审计和技术转移节奏。",
|
||||
"counter_evidence": [
|
||||
"是否已有整改证据证明这些问题只是审计时点的临时缺口?",
|
||||
"低/中风险评级是否足以说明商业化阶段门影响有限?",
|
||||
"审计范围有限是否导致本章不能外推到整体体系成熟度?",
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
("法规", "欧盟", "NMPA", "ICH"),
|
||||
{
|
||||
"core_question": "如果按 EU Annex 1、NMPA GMP、ICH Q9/Q10 以及 FDA 执法尺度校准,哪些现场发现的严重度和整改优先级会发生变化?",
|
||||
"bold_hypothesis": "初始假设:白帆按国内 GMP 逻辑已具备基础合规框架,但若以欧盟无菌标准和质量风险管理要求衡量,部分“低风险/建议项”会转化为体系成熟度缺口。",
|
||||
"writing_claim": "本章要建立后文共用的法规基线,避免整改优先级只跟随原审计评级,而忽略国际化和商业化标准。",
|
||||
"counter_evidence": [
|
||||
"相关国际标准是否并不适用于当前产品阶段或委托生产边界?",
|
||||
"NMPA 与欧盟/美国要求之间是否存在可接受差异?",
|
||||
"是否有企业内部标准已经覆盖但审计材料未呈现?",
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
("无菌", "RABS", "First Air", "APS", "灯检"),
|
||||
{
|
||||
"core_question": "制剂线的主要无菌风险,是硬件布局不足,还是人员干预、首次气流保护、APS 覆盖和灯检标准执行证据不足?",
|
||||
"bold_hypothesis": "初始假设:白帆制剂车间硬件基础并非主要短板,真正风险在于关键无菌行为和模拟验证是否能持续证明受控;Phase 2 应重点查 First Air、RABS 干预、APS 场景设计和灯检阳性样品管理。",
|
||||
"writing_claim": "本章要把无菌保障从“设施看起来合规”推进到“关键操作和验证证据可被审计接受”。",
|
||||
"counter_evidence": [
|
||||
"现场是否已有完整视频复核、APS 覆盖和再培训有效性证据?",
|
||||
"观察到的无菌动作问题是否只是个别人员或单次拍摄偏差?",
|
||||
"灯检和 RABS 风险是否已有 SOP、趋势和复核记录闭环?",
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
("原液", "WFI", "SCADA", "EMS"),
|
||||
{
|
||||
"core_question": "原液和公用系统的风险是否被一次性封闭工艺掩盖,真正缺口在 WFI、SCADA/EMS、离线记录和异常升级证据链?",
|
||||
"bold_hypothesis": "初始假设:一次性反应器和封闭转移降低了暴露风险,但不能自动证明系统受控;Phase 2 应验证 WFI 冷却回流、环境/压差报警、SCADA 数据和离线检测记录是否形成完整证据链。",
|
||||
"writing_claim": "本章要说明原液与公用系统不是“硬件先进即可”,而是要证明关键状态、报警、数据和异常处理持续受控。",
|
||||
"counter_evidence": [
|
||||
"WFI、SCADA/EMS 和离线记录是否已有验证报告与趋势复核?",
|
||||
"一次性系统是否已经充分降低共线和交叉污染风险?",
|
||||
"被指出的公用系统风险是否只是设计建议而非实际偏差?",
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
("工艺", "CPP", "CQA", "PPQ", "清洁验证"),
|
||||
{
|
||||
"core_question": "现有 IND 阶段工艺规程和批记录,距离商业化 PPQ、控制策略和清洁验证所需证据还差在哪里?",
|
||||
"bold_hypothesis": "初始假设:白帆目前的工艺文件足以支撑临床阶段执行,但不足以支撑商业化批记录、CPP/CQA 控制、PPQ 和清洁验证闭环;Phase 2 应查明哪些字段、参数和验证证据必须前置补齐。",
|
||||
"writing_claim": "本章要把技术转移风险具体化为文件、参数、验证和批记录的硬门槛。",
|
||||
"counter_evidence": [
|
||||
"是否已有商业化模板、控制策略或 PPQ 草案未体现在审计材料中?",
|
||||
"当前项目阶段是否尚不需要完整商业化批记录要求?",
|
||||
"清洁验证和工艺验证是否已有主计划覆盖?",
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
("偏差", "变更", "CAPA", "数据完整性"),
|
||||
{
|
||||
"core_question": "白帆的问题是没有质量流程,还是流程之间的事件分类、升级、CAPA 有效性和数据完整性尚未形成运行闭环?",
|
||||
"bold_hypothesis": "初始假设:白帆已有偏差、变更和 CAPA 的流程框架,但事件何时启动偏差、何时作为变更、如何证明 CAPA 有效,以及电子/纸质数据如何贯通,仍存在运行机制缺口。",
|
||||
"writing_claim": "本章要把质量体系从“有 SOP”推进到“事件能被正确分类、调查、纠正、验证并趋势复核”。",
|
||||
"counter_evidence": [
|
||||
"是否有趋势分析、管理评审和 CAPA effectiveness check 证明体系已经闭环?",
|
||||
"个别事件分类问题是否不足以代表体系性缺口?",
|
||||
"电子系统和纸质记录之间是否已有数据完整性控制?",
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
("人员", "培训", "质量文化"),
|
||||
{
|
||||
"core_question": "培训记录齐全是否真的转化为一线无菌行为、偏差判断和质量风险意识?哪些证据能证明培训有效?",
|
||||
"bold_hypothesis": "初始假设:白帆不缺培训台账,缺的是把培训结果转化为现场行为的一致性证据;如果 First Air、干预动作、事件判断和灯检执行仍需反复提醒,问题就不是“再培训一次”,而是培训有效性确认和质量文化运行机制不足。",
|
||||
"writing_claim": "本章要把人员问题从“有没有培训”改写为“培训是否改变行为、降低风险、形成可复核证据”。",
|
||||
"counter_evidence": [
|
||||
"现场抽问、资格确认和再培训记录是否已证明人员理解到位?",
|
||||
"被观察到的行为问题是否只发生在少数岗位或单次演示?",
|
||||
"是否有岗位胜任力矩阵、年度复评和行为观察数据支撑人员能力?",
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
("运营", "跨部门", "指标", "review"),
|
||||
{
|
||||
"core_question": "白帆当前整改和生产准备依赖个人推动,还是已经形成跨部门例会、问题升级、指标看板和管理层复核的运营系统?",
|
||||
"bold_hypothesis": "初始假设:运营短板不在于团队不努力,而在于缺少固定节奏和可视化管理系统;如果 owner、关闭证据、升级阈值和管理层 review 不稳定,整改会停留在临时协调,难以支撑商业化节奏。",
|
||||
"writing_claim": "本章要说明运营管理是 GMP 风险的放大器:没有节奏、看板和升级机制,技术和质量问题会反复跨部门漂移。",
|
||||
"counter_evidence": [
|
||||
"是否已经存在稳定 PMO/例会/看板,只是未进入审计材料?",
|
||||
"短期临时协调是否足以覆盖当前项目阶段,不需要完整运营系统?",
|
||||
"owner、期限和关闭证据是否已经在复盘文件中基本清楚?",
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
("团队", "CDMO", "能力矩阵"),
|
||||
{
|
||||
"core_question": "对标成熟 CDMO,白帆最需要补齐的是人数、岗位能力,还是 QA/MSAT/工程/项目管理之间的角色分工?",
|
||||
"bold_hypothesis": "初始假设:白帆的能力缺口不是简单扩编,而是商业化 CDMO 所需的角色矩阵尚未完全成型;Phase 2 应验证 QA 独立性、MSAT 工艺支持、工程保障、生产班组和 PMO 协同能力。",
|
||||
"writing_claim": "本章要给出面向商业化的团队能力地图,说明哪些能力必须自建,哪些可外部支持,哪些要通过机制补齐。",
|
||||
"counter_evidence": [
|
||||
"现有人员是否已具备商业化经验,只是材料未体现?",
|
||||
"对标 CDMO 是否会高估当前阶段所需组织复杂度?",
|
||||
"是否可通过顾问、外包或客户支持临时补足能力?",
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
("整改", "owner", "路线图"),
|
||||
{
|
||||
"core_question": "哪些整改必须立即完成,哪些属于体系补强,哪些是能力建设?每项如何绑定 owner、关闭证据和复核窗口?",
|
||||
"bold_hypothesis": "初始假设:如果整改只按问题清单逐条关闭,会漏掉体系性根因;更有效的路线应分为立即纠偏、90 天体系补强和中长期能力建设三层,并为每层定义关闭证据。",
|
||||
"writing_claim": "本章要把诊断转化为可执行 CAPA 组合,而不是泛泛的改进建议。",
|
||||
"counter_evidence": [
|
||||
"是否已有整改计划足以覆盖 owner、期限、关闭证据和 QA verification?",
|
||||
"部分整改是否应前移或后移,避免资源过载?",
|
||||
"哪些建议若缺少法规证据,不应被列为强制整改?",
|
||||
],
|
||||
},
|
||||
),
|
||||
(
|
||||
("管理层", "CAPA", "总表"),
|
||||
{
|
||||
"core_question": "管理层应通过什么样的 CAPA 总表、法规映射表和复核节奏,持续判断整改是否真正降低风险?",
|
||||
"bold_hypothesis": "初始假设:白帆需要的不只是一次性报告,而是一套管理层可追踪的整改仪表盘;否则 CAPA 关闭会变成文件动作,无法证明风险趋势下降和商业化 readiness 提升。",
|
||||
"writing_claim": "本章要把报告成果固化成管理层治理工具:CAPA 总表、法规映射、证据包和复核节奏。",
|
||||
"counter_evidence": [
|
||||
"现有管理评审或质量例会是否已经能承担这个功能?",
|
||||
"过度表格化是否会增加一线负担而不改善风险?",
|
||||
"哪些指标真正能反映风险降低,而不是制造形式化 KPI?",
|
||||
],
|
||||
},
|
||||
),
|
||||
]
|
||||
for needles, strategy in strategies:
|
||||
if any(needle in title for needle in needles):
|
||||
return strategy
|
||||
return {
|
||||
"core_question": f"本章需要判断“{title}”背后的真实风险、适用边界和整改优先级。",
|
||||
"bold_hypothesis": f"初始假设:{title} 不是孤立问题,而是质量体系、工艺证据或运营机制中的一个可验证缺口;Phase 2 必须用材料原文和外部证据判断其严重度。",
|
||||
"writing_claim": f"本章要把“{title}”转化为可执行的诊断结论和整改要求。",
|
||||
"counter_evidence": [
|
||||
"该问题是否已有充分整改或验证证据?",
|
||||
"是否只是阶段性限制,而非系统性缺口?",
|
||||
"外部标准是否适用于当前业务边界?",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_chapter_planning(
|
||||
project_root: Path,
|
||||
manifest: dict[str, Any],
|
||||
method: ResearchMethod,
|
||||
titles: list[str],
|
||||
*,
|
||||
quota: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Build hypothesis-driven chapter plans that become Phase 2 prompt context."""
|
||||
lanes = list(method.integrated_lanes or method.task_axes)
|
||||
minimum_evidence = _minimum_evidence_for_method(method)
|
||||
plans: list[dict[str, Any]] = []
|
||||
for idx, title in enumerate(titles, start=1):
|
||||
chapter_id = f"ch{idx:02d}"
|
||||
material_lines = _material_lines_for_chapter(project_root, manifest, title)
|
||||
if not material_lines:
|
||||
material_lines = ["未在材料中自动匹配到足够线索;Phase 2 必须先回读全部输入材料并补充原文摘录。"]
|
||||
strategy = _strategy_for_chapter(title, method)
|
||||
core_question = strategy["core_question"]
|
||||
bold_hypothesis = strategy["bold_hypothesis"]
|
||||
verification_plan = [
|
||||
"先从允许的本地材料提取 2-4 条原文证据,保留出处和上下文。",
|
||||
f"再按方法论 evidence lanes 求证:{';'.join(lanes)}。",
|
||||
"每个核心判断至少匹配 2 个独立高质量来源;不足时降级为待验证判断。",
|
||||
"主动搜索反方证据、低严重度解释、适用范围限制或替代原因。",
|
||||
"输出时把证据、判断、整改/建议和待补证据分开,避免直接写成散文化正文。",
|
||||
]
|
||||
counter_evidence = strategy["counter_evidence"]
|
||||
writing_claim = strategy["writing_claim"]
|
||||
phase2_prompt_context = "\n".join(
|
||||
[
|
||||
f"章节:{chapter_id} {title}",
|
||||
core_question,
|
||||
bold_hypothesis,
|
||||
"材料起点:",
|
||||
*[f"- {line}" for line in material_lines],
|
||||
"求证路线:",
|
||||
*[f"- {item}" for item in verification_plan],
|
||||
"必须寻找的反方/边界:",
|
||||
*[f"- {item}" for item in counter_evidence],
|
||||
f"写作主张:{writing_claim}",
|
||||
f"最低证据要求:{json.dumps(minimum_evidence, ensure_ascii=False)}",
|
||||
]
|
||||
)
|
||||
plans.append(
|
||||
{
|
||||
"chapter_id": chapter_id,
|
||||
"title": title,
|
||||
"suggested_words": quota,
|
||||
"core_question": core_question,
|
||||
"bold_hypothesis": bold_hypothesis,
|
||||
"why_this_matters": "本章用于把 Phase1 的判断转化为 Phase2 可验证命题,并为最终报告保留清晰主线。",
|
||||
"material_starting_points": material_lines,
|
||||
"evidence_lanes": lanes,
|
||||
"verification_plan": verification_plan,
|
||||
"counter_evidence_to_seek": counter_evidence,
|
||||
"writing_claim": writing_claim,
|
||||
"minimum_evidence": minimum_evidence,
|
||||
"phase2_prompt_context": phase2_prompt_context,
|
||||
}
|
||||
)
|
||||
return plans
|
||||
|
||||
|
||||
def build_research_brief_payload(
|
||||
project_root: Path,
|
||||
manifest: dict[str, Any],
|
||||
method: ResearchMethod,
|
||||
chapter_planning: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Create the file-backed Phase 1 research brief used by task-card generation."""
|
||||
axes = list(method.task_axes)
|
||||
chapter_planning = chapter_planning or []
|
||||
return {
|
||||
"version": "0.21-alpha",
|
||||
"topic": manifest.get("topic", project_root.name),
|
||||
"research_method": method.key,
|
||||
"method_name": method.name,
|
||||
"work_language": "zh",
|
||||
"tone": "事实型、整改导向、面向管理层和质量/生产负责人;避免空泛咨询腔。",
|
||||
"central_question": f"如何基于已提供材料和权威法规/最佳实践,系统诊断“{manifest.get('topic', project_root.name)}”并形成可执行整改路线图?",
|
||||
"central_thesis": _central_thesis(manifest, method),
|
||||
"phase_logic": {
|
||||
"phase1": "大胆假设:结合输入材料、访谈信息和初步搜索,定下主基调、章节命题和求证路线。",
|
||||
"phase2": "小心求证:worker 只围绕 Phase1 命题收集、验证、证伪和补证,不自行重写研究方向。",
|
||||
"phase3": "一致性审校:检查 Phase1 假设与 Phase2 证据是否自洽,指出需要回炉的章节或证据缺口。",
|
||||
},
|
||||
"phase2_mode": "chapter_integrated",
|
||||
"success_criteria": [
|
||||
"每个核心判断都能回到用户材料、权威法规、最佳实践或反方证据。",
|
||||
"短中长期整改建议必须绑定优先级、责任、关闭证据和复核机制。",
|
||||
"章节写作必须先收束主线,再使用 evidence packet;不得按 packet 机械拼贴。",
|
||||
],
|
||||
"phase2_inputs": {
|
||||
"material_brief_path": "phase1/material_brief.md",
|
||||
"framework_path": "phase1/framework.md",
|
||||
"research_brief_path": "phase1/research_brief.json",
|
||||
},
|
||||
"materials": _material_paths(manifest),
|
||||
"chapter_planning": chapter_planning,
|
||||
"task_planning": {
|
||||
"chapter_source": "phase1/framework.md",
|
||||
"phase2_mode": "chapter_integrated",
|
||||
"axes": axes,
|
||||
"required_skills": [
|
||||
"deep-research",
|
||||
"search-gateway",
|
||||
"search-strategy",
|
||||
"source-quality",
|
||||
"evidence-table",
|
||||
"citation-manager",
|
||||
],
|
||||
"search_routes_by_axis": {axis: AXIS_ROUTES.get(axis, ["general"]) for axis in axes},
|
||||
"axis_prompt_briefs": {axis: _axis_prompt_brief(axis, method) for axis in axes},
|
||||
"stop_conditions": [
|
||||
"每张任务卡至少形成 3 条可追溯 evidence_items,且不得编造 candidate_sources 以外来源。",
|
||||
"关键 claim 不足 2 个独立 Tier 1-2 信源时,必须写入 open_questions 和证据缺口。",
|
||||
"必须包含 counter_evidence;找不到反方证据时记录检索路径和限制。",
|
||||
],
|
||||
"fragmentation_guard": "并发 worker 只生产 evidence packet;章节主线由 compressed_findings 收束,禁止直接把 packet 堆成正文。",
|
||||
},
|
||||
"clarification_notes": {
|
||||
"requires_user_review": True,
|
||||
"questions_source": "phase1/material_brief.md",
|
||||
"decision_items": [
|
||||
"确认报告用途、受众和脱敏边界。",
|
||||
"确认研究方法是否适配当前场景;MECE 只是可选方法之一。",
|
||||
"确认任务切分和检索策略是否足以让低成本模型独立执行。",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_research_brief(
|
||||
project_root: Path,
|
||||
manifest: dict[str, Any] | None = None,
|
||||
method: ResearchMethod | None = None,
|
||||
chapter_planning: list[dict[str, Any]] | None = None,
|
||||
) -> tuple[Path, Path]:
|
||||
manifest = manifest or load_manifest(project_root)
|
||||
method = method or ResearchMethodRegistry().get(manifest.get("research_method"))
|
||||
payload = build_research_brief_payload(project_root, manifest, method, chapter_planning=chapter_planning)
|
||||
json_path = project_root / "phase1" / "research_brief.json"
|
||||
md_path = project_root / "phase1" / "research_brief.md"
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
hypothesis_path = project_root / "phase1" / "hypothesis_map.json"
|
||||
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
hypothesis_path.write_text(json.dumps(payload.get("chapter_planning") or [], ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
lines = [
|
||||
f"# Phase 1 Research Brief:{payload['topic']}",
|
||||
"",
|
||||
f"- research_method: {payload['research_method']}",
|
||||
f"- work_language: {payload['work_language']}",
|
||||
f"- tone: {payload['tone']}",
|
||||
f"- phase2_mode: {payload['phase2_mode']}",
|
||||
"",
|
||||
"## 中心问题",
|
||||
"",
|
||||
payload["central_question"],
|
||||
"",
|
||||
"## 主基调 / 大胆假设",
|
||||
"",
|
||||
payload["central_thesis"],
|
||||
"",
|
||||
"## Phase 逻辑",
|
||||
"",
|
||||
]
|
||||
for phase_name, phase_text in payload["phase_logic"].items():
|
||||
lines.append(f"- `{phase_name}`:{phase_text}")
|
||||
lines.extend([
|
||||
"",
|
||||
"## 成功标准",
|
||||
"",
|
||||
])
|
||||
lines.extend(f"- {item}" for item in payload["success_criteria"])
|
||||
if payload.get("chapter_planning"):
|
||||
lines.extend(["", "## 章节命题与求证计划", ""])
|
||||
for item in payload["chapter_planning"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {item['chapter_id']} {item['title']}",
|
||||
"",
|
||||
f"- 核心问题:{item['core_question']}",
|
||||
f"- 大胆假设:{item['bold_hypothesis']}",
|
||||
f"- 写作主张:{item['writing_claim']}",
|
||||
f"- 证据线:{';'.join(item['evidence_lanes'])}",
|
||||
"- 材料起点:",
|
||||
]
|
||||
)
|
||||
lines.extend(f" - {line}" for line in item["material_starting_points"])
|
||||
lines.extend(["- 求证计划:"])
|
||||
lines.extend(f" - {line}" for line in item["verification_plan"])
|
||||
lines.extend([""])
|
||||
lines.extend(["", "## 任务切分原则", ""])
|
||||
planning = payload["task_planning"]
|
||||
lines.append(planning["fragmentation_guard"])
|
||||
lines.append("")
|
||||
for axis in planning["axes"]:
|
||||
routes = "、".join(planning["search_routes_by_axis"].get(axis, []))
|
||||
prompt = planning["axis_prompt_briefs"].get(axis, "")
|
||||
lines.append(f"- `{axis}`:{prompt} 检索路径:{routes}")
|
||||
lines.extend(["", "## 必读 Skills", ""])
|
||||
lines.extend(f"- {name}" for name in planning["required_skills"])
|
||||
lines.extend(["", "## 停止条件", ""])
|
||||
lines.extend(f"- {item}" for item in planning["stop_conditions"])
|
||||
lines.append("")
|
||||
md_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
return md_path, json_path
|
||||
|
||||
|
||||
CHAPTER_TEMPLATES: dict[str, list[str]] = {
|
||||
"mckinsey_market": [
|
||||
"核心结论先行界定市场机会与约束",
|
||||
"临床与真实世界证据决定需求天花板",
|
||||
"监管路径和支付环境重塑商业化节奏",
|
||||
"竞争格局正在从单点产品转向组合能力",
|
||||
"专利与技术壁垒决定长期利润池",
|
||||
"中国市场的准入和供给能力形成独立变量",
|
||||
"资本市场预期与基本面之间存在可验证偏差",
|
||||
"反方证据限定结论边界并提示回撤风险",
|
||||
"战略选择应围绕资源约束排序",
|
||||
"执行路线图需要把证据缺口转化为行动清单",
|
||||
],
|
||||
"gmp_gap_assessment": [
|
||||
"监管基线决定整改范围而非企业主观偏好",
|
||||
"现状差距需要按法规条款和业务流程双重定位",
|
||||
"质量风险分级决定 CAPA 优先级",
|
||||
"根因分析质量决定整改能否闭环",
|
||||
"CAPA 设计必须绑定责任人、证据和期限",
|
||||
"验证计划决定整改是否可被审计接受",
|
||||
"供应商和外包管理常是系统性缺口放大器",
|
||||
"数据完整性风险需要独立成章处理",
|
||||
"实施路线图需要平衡停线风险与合规风险",
|
||||
"管理层治理机制决定整改能否持续",
|
||||
],
|
||||
"cmc_process_risk": [
|
||||
"工艺流程图是识别放大风险的起点",
|
||||
"CQA 与 CPP 的映射决定控制策略质量",
|
||||
"放大过程的失效模式集中在传质、混合和稳定性",
|
||||
"分析方法和放行标准决定证据可信度",
|
||||
"技术转移风险来自知识隐性化和现场差异",
|
||||
"供应链约束会改变工艺控制边界",
|
||||
"偏差和变更管理决定商业化后的韧性",
|
||||
"监管沟通策略需要提前固化关键假设",
|
||||
"反方证据限定平台工艺可复制性",
|
||||
"CMC 路线图需要把风险转化为验证实验",
|
||||
],
|
||||
"rd_go_no_go": [
|
||||
"科学假设强度决定项目是否值得进入下一阶段",
|
||||
"POC 证据需要同时证明有效性和可转化性",
|
||||
"安全性窗口决定适应症与人群选择",
|
||||
"IP 与 FTO 风险决定商业化自由度",
|
||||
"开发路径需要把关键不确定性前置验证",
|
||||
"竞争窗口决定速度是否仍有战略价值",
|
||||
"CMC 与临床运营能力影响真实可行性",
|
||||
"反方证据决定 go/no-go 阈值",
|
||||
"投资强度应与证据成熟度匹配",
|
||||
"决策门槛需要形成可执行检查表",
|
||||
],
|
||||
"management_consulting": [
|
||||
"现状诊断需要区分症状、根因和约束条件",
|
||||
"能力差距决定组织改进优先级",
|
||||
"流程断点揭示跨部门协作成本",
|
||||
"治理结构决定决策速度和责任清晰度",
|
||||
"运营模型需要匹配战略目标而非照搬标杆",
|
||||
"数字化工具只有嵌入流程才产生价值",
|
||||
"绩效指标需要避免局部最优",
|
||||
"变革阻力本身是方案设计输入",
|
||||
"路线图需要把 quick wins 与系统建设分层",
|
||||
"落地机制决定咨询建议能否转化为成果",
|
||||
],
|
||||
"gmp_quality_operations_diagnosis": [
|
||||
"从审计清单到商业化阶段门",
|
||||
"用法规基线重新校准整改优先级",
|
||||
"制剂无菌保障:从硬件合规到行为受控",
|
||||
"原液与公用系统:封闭工艺背后的证据缺口",
|
||||
"工艺文件与验证:商业化转移的硬门槛",
|
||||
"质量系统闭环:偏差、变更、CAPA 与数据完整性",
|
||||
"人员能力:培训有效性比培训记录更关键",
|
||||
"运营节奏:从临时协调转向管理系统",
|
||||
"团队建设:按 CDMO 能力矩阵补齐角色",
|
||||
"整改路线图:立即纠偏、体系补强、能力建设",
|
||||
"管理层看板:用 CAPA 总表驱动复核",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _existing_chapter_titles(project_root: Path) -> list[str]:
|
||||
framework_path = project_root / "phase1" / "framework.md"
|
||||
if not framework_path.exists():
|
||||
return []
|
||||
from scripts.runtime.tasks import parse_framework_chapters
|
||||
|
||||
chapters = parse_framework_chapters(framework_path.read_text(encoding="utf-8"))
|
||||
return [chapter.title for chapter in chapters if chapter.title]
|
||||
|
||||
|
||||
def render_framework(
|
||||
project_root: Path,
|
||||
*,
|
||||
method_key: str | None = None,
|
||||
chapter_count: int = 10,
|
||||
preserve_existing_outline: bool = False,
|
||||
) -> Path:
|
||||
manifest = load_manifest(project_root)
|
||||
registry = ResearchMethodRegistry()
|
||||
method = registry.get(method_key or manifest.get("research_method"))
|
||||
if method_key:
|
||||
manifest["research_method"] = method.key
|
||||
existing_titles = _existing_chapter_titles(project_root) if preserve_existing_outline else []
|
||||
titles = existing_titles or CHAPTER_TEMPLATES.get(method.key) or CHAPTER_TEMPLATES["mckinsey_market"]
|
||||
chapter_count = max(8, min(15, chapter_count))
|
||||
selected = titles[:chapter_count] if not existing_titles else titles
|
||||
quota = max(800, int(manifest.get("target_words", 30000)) // len(selected))
|
||||
chapter_planning = build_chapter_planning(project_root, manifest, method, selected, quota=quota)
|
||||
sections = "\n".join(f"- {item}" for item in method.framework_sections)
|
||||
axes = "、".join(method.task_axes)
|
||||
material_text = render_material_inventory(manifest.get("material_inventory") or [])
|
||||
lines = [
|
||||
f"# {manifest.get('report_title') or manifest['topic']}:研究框架",
|
||||
"",
|
||||
f"research_method: {method.key}",
|
||||
f"method_name: {method.name}",
|
||||
f"work_language: 中文主写作;检索关键词、证据摘录、source title、raw notes 可保留英文。",
|
||||
f"target_words: {manifest.get('target_words', 30000)}",
|
||||
"",
|
||||
"## 方法选择",
|
||||
"",
|
||||
f"本项目采用 `{method.key}`,因为其结构原则是:{method.structure_principle}",
|
||||
"",
|
||||
"框架模块:",
|
||||
sections,
|
||||
"",
|
||||
"Phase 2 任务轴:",
|
||||
f"- {axes}",
|
||||
"",
|
||||
"## 输入材料与使用边界",
|
||||
"",
|
||||
material_text,
|
||||
"",
|
||||
"这些材料作为现场问题线索和内部事实起点使用;正式结论仍需结合 NMPA、FDA、EMA、ICH、WHO 等权威法规、指南和最佳实践进行验证。",
|
||||
"",
|
||||
"## 中心假设",
|
||||
"",
|
||||
_central_thesis(manifest, method),
|
||||
"",
|
||||
"Phase1 的职责是大胆假设:基于材料、访谈和初步搜索定下主基调、章节命题和求证路线。Phase2 的职责是小心求证:验证、证伪、补证,而不是重新发明报告方向。Phase3 则检查 Phase1 假设与 Phase2 证据是否自洽。",
|
||||
"",
|
||||
]
|
||||
for item in chapter_planning:
|
||||
lines.extend(
|
||||
[
|
||||
f"## 第{int(item['chapter_id'][2:])}章 {item['title']}",
|
||||
"",
|
||||
f"建议字数:约 {item['suggested_words']} 字。",
|
||||
f"本章要解决的问题:{item['core_question']}",
|
||||
f"大胆假设:{item['bold_hypothesis']}",
|
||||
f"写作主张:{item['writing_claim']}",
|
||||
f"证据线:{';'.join(item['evidence_lanes'])}",
|
||||
"",
|
||||
"材料起点:",
|
||||
*[f"- {line}" for line in item["material_starting_points"]],
|
||||
"",
|
||||
"求证计划:",
|
||||
*[f"- {line}" for line in item["verification_plan"]],
|
||||
"",
|
||||
"必须寻找的反方/边界:",
|
||||
*[f"- {line}" for line in item["counter_evidence_to_seek"]],
|
||||
"",
|
||||
f"最低证据要求:`{json.dumps(item['minimum_evidence'], ensure_ascii=False)}`",
|
||||
"",
|
||||
]
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"## 暂停点",
|
||||
"",
|
||||
"请先确认 `phase1/material_brief.md` 的材料解读和访谈问题,再确认本框架后进入 Phase 2。若章节逻辑、方法框架或字数配额需要调整,应先修改本文件。",
|
||||
"",
|
||||
"确认后运行:`uv run python scripts/dr.py approve <project>`;未批准时 `research` 默认会拒绝推进,可用 `--force` 临时覆盖。",
|
||||
"",
|
||||
]
|
||||
)
|
||||
out = project_root / "phase1" / "framework.md"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text("\n".join(lines), encoding="utf-8")
|
||||
research_brief_md, research_brief_json = write_research_brief(project_root, manifest, method, chapter_planning=chapter_planning)
|
||||
manifest["phase1"] = {
|
||||
"status": "completed",
|
||||
"approved": False,
|
||||
"framework_path": "phase1/framework.md",
|
||||
"research_brief_path": str(research_brief_md.relative_to(project_root)),
|
||||
"research_brief_json_path": str(research_brief_json.relative_to(project_root)),
|
||||
"requires_user_interview": True,
|
||||
"research_method": method.key,
|
||||
"updated_at": utc_now_iso(),
|
||||
}
|
||||
manifest["updated_at"] = utc_now_iso()
|
||||
write_manifest(project_root, manifest)
|
||||
return out
|
||||
@@ -0,0 +1,328 @@
|
||||
"""Phase 3 review checks for the Python core."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from scripts.runtime.artifacts import load_manifest, write_manifest
|
||||
from scripts.runtime.tasks import validate_packet
|
||||
|
||||
|
||||
def utc_now_iso() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def _source_ids_from_jsonl(path: Path) -> set[str]:
|
||||
ids: set[str] = set()
|
||||
if not path.exists():
|
||||
return ids
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
source_id = obj.get("id") or obj.get("source_id")
|
||||
if source_id:
|
||||
ids.add(str(source_id))
|
||||
return ids
|
||||
|
||||
|
||||
def _draft_citations(drafts: list[Path]) -> set[str]:
|
||||
cited: set[str] = set()
|
||||
for draft in drafts:
|
||||
cited.update(re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", draft.read_text(encoding="utf-8")))
|
||||
return cited
|
||||
|
||||
|
||||
def _ready_packet_stems(project_root: Path) -> set[str]:
|
||||
ready: set[str] = set()
|
||||
for path in sorted((project_root / "phase2" / "packets").glob("*.json")):
|
||||
try:
|
||||
packet = json.loads(path.read_text(encoding="utf-8"))
|
||||
validate_packet(packet)
|
||||
except Exception:
|
||||
continue
|
||||
ready.add(path.stem)
|
||||
return ready
|
||||
|
||||
|
||||
def _read_text_if_exists(path: Path, *, max_chars: int | None = None) -> str:
|
||||
if not path.exists():
|
||||
return ""
|
||||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||||
return text[:max_chars] if max_chars is not None else text
|
||||
|
||||
|
||||
def _json_if_exists(path: Path, *, max_chars: int | None = None) -> str:
|
||||
if not path.exists():
|
||||
return ""
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
text = json.dumps(data, ensure_ascii=False, indent=2)
|
||||
except Exception:
|
||||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||||
return text[:max_chars] if max_chars is not None else text
|
||||
|
||||
|
||||
def _draft_quality_findings(drafts: list[Path]) -> list[dict[str, Any]]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
generic_markers = [
|
||||
"需要进一步完善",
|
||||
"应当加强",
|
||||
"持续改进",
|
||||
"系统性",
|
||||
"闭环管理",
|
||||
"质量文化",
|
||||
]
|
||||
for draft in drafts:
|
||||
text = draft.read_text(encoding="utf-8")
|
||||
zh_chars = sum(1 for char in text if "\u4e00" <= char <= "\u9fff")
|
||||
citations = re.findall(r"\[(src_[A-Za-z0-9_-]+)\]", text)
|
||||
evidence_table_present = "证据落点" in text and "待补证据" in text
|
||||
if zh_chars >= 1200 and len(set(citations)) < 5:
|
||||
findings.append({"severity": "P1", "message": f"{draft.name} 引用来源过少,可能未充分使用 evidence packet。"})
|
||||
if zh_chars >= 1200 and not evidence_table_present:
|
||||
findings.append({"severity": "P1", "message": f"{draft.name} 缺少“证据落点与待补证据”小节,难以判断 evidence 是否真正落到纸面。"})
|
||||
generic_count = sum(text.count(marker) for marker in generic_markers)
|
||||
if zh_chars >= 1200 and generic_count >= 18:
|
||||
findings.append({"severity": "P1", "message": f"{draft.name} 泛化管理表述过多,需要回炉为具体审计发现、风险影响和整改动作。"})
|
||||
return findings
|
||||
|
||||
|
||||
def build_phase3_model_review_context(project_root: Path, *, max_chars: int = 650_000) -> str:
|
||||
"""Build a structured, bounded context packet for an independent model review."""
|
||||
deterministic_path = build_phase3_critique(project_root)
|
||||
deterministic_copy = project_root / "phase3" / "critique_deterministic.md"
|
||||
deterministic_copy.write_text(deterministic_path.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
|
||||
manifest = load_manifest(project_root)
|
||||
parts: list[str] = [
|
||||
f"# Phase 3 Model Review Context: {manifest.get('topic', project_root.name)}",
|
||||
"",
|
||||
"## Review Contract",
|
||||
"",
|
||||
"- 这是给非 Codex 模型的独立总编审校上下文,不要求重写正文。",
|
||||
"- 请判断 Phase2 草稿能否进入 Phase4,或必须回炉补证据/重写。",
|
||||
"- 重点关注:证据是否落纸面、并发 packet 是否造成碎片化、法规/最佳实践覆盖是否足够、整改建议是否具体可执行。",
|
||||
"",
|
||||
"## Manifest",
|
||||
"",
|
||||
"```json",
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2),
|
||||
"```",
|
||||
"",
|
||||
"## Deterministic Review Baseline",
|
||||
"",
|
||||
_read_text_if_exists(deterministic_copy),
|
||||
"",
|
||||
"## Phase 1 Framework",
|
||||
"",
|
||||
_read_text_if_exists(project_root / "phase1" / "framework.md", max_chars=50_000),
|
||||
"",
|
||||
"## Phase 1 Research Brief",
|
||||
"",
|
||||
_read_text_if_exists(project_root / "phase1" / "research_brief.md", max_chars=30_000),
|
||||
"",
|
||||
"## Phase 2 Brief Warnings",
|
||||
"",
|
||||
_json_if_exists(project_root / "phase2" / "brief_warnings.json", max_chars=30_000) or "无",
|
||||
"",
|
||||
"## Phase 2 Packet Errors",
|
||||
"",
|
||||
]
|
||||
errors = sorted((project_root / "phase2" / "packet_errors").glob("*.json"))
|
||||
if errors:
|
||||
for path in errors[:40]:
|
||||
parts.extend([f"### {path.name}", "", _json_if_exists(path, max_chars=2_000), ""])
|
||||
else:
|
||||
parts.append("无")
|
||||
|
||||
parts.extend(["", "## Source Registry Summary", ""])
|
||||
source_lines = []
|
||||
sources_path = project_root / "phase2" / "sources.jsonl"
|
||||
if sources_path.exists():
|
||||
for line in sources_path.read_text(encoding="utf-8").splitlines()[:260]:
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
source = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
source_lines.append(
|
||||
"- {id} | {tier} | {title} | {url} | cached={cached}".format(
|
||||
id=source.get("id", ""),
|
||||
tier=source.get("tier", ""),
|
||||
title=str(source.get("title", ""))[:120],
|
||||
url=source.get("url", ""),
|
||||
cached=source.get("cached_text_path", ""),
|
||||
)
|
||||
)
|
||||
parts.append("\n".join(source_lines) or "无")
|
||||
|
||||
parts.extend(["", "## Compressed Findings", ""])
|
||||
for path in sorted((project_root / "phase2" / "compressed_findings").glob("ch*.json")):
|
||||
parts.extend([f"### {path.name}", "", "```json", _json_if_exists(path, max_chars=35_000), "```", ""])
|
||||
|
||||
parts.extend(["", "## Chapter Drafts", ""])
|
||||
for path in sorted((project_root / "phase2" / "drafts").glob("ch*.md")):
|
||||
parts.extend([f"### {path.name}", "", _read_text_if_exists(path, max_chars=55_000), ""])
|
||||
|
||||
context = "\n".join(parts)
|
||||
if len(context) > max_chars:
|
||||
context = context[:max_chars] + "\n\n[Context truncated by max_chars; review should flag if truncation limits confidence.]\n"
|
||||
|
||||
out = project_root / "phase3" / "review_context_opus_4_7.md"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(context, encoding="utf-8")
|
||||
return context
|
||||
|
||||
|
||||
def phase3_model_review_system_prompt() -> str:
|
||||
return (
|
||||
"你是 Deep Research Phase 3 的独立总编审校模型,本次由 ZenMux Claude Opus 4.7 执行,用于避免 Codex/OpenAI 模型偏见。\n"
|
||||
"你的任务是审校,不是润色或重写。必须用中文输出,英文仅可保留 source title、URL、法规缩写和原文短摘录。\n"
|
||||
"请严格检查:1) 研究目标与 Phase1 框架是否契合;2) Phase2 并发 evidence packets 是否被章节真正吸收,还是造成碎片化;"
|
||||
"3) FDA/NMPA/EMA/ICH/WHO/EU GMP 等权威来源是否足以支撑关键判断;4) 用户材料是否被正确作为起点且被权威来源交叉验证;"
|
||||
"5) 运营管理与团队能力章节是否具体,不得泛泛咨询腔;6) CAPA 建议是否包含 owner、期限、关闭证据、QA verification、复核窗口和升级阈值;"
|
||||
"7) 引用链和 source_id 是否可追踪;8) 是否仍有明显 AI 味、中英文混杂或空泛表达。\n\n"
|
||||
"输出必须使用以下 Markdown 结构:\n"
|
||||
"# Phase 3 Opus 4.7 独立审校\n"
|
||||
"## 总体判定\n"
|
||||
"给出:通过 / 有条件通过 / 回炉 Phase2,并说明最核心理由。\n"
|
||||
"## P0/P1 阻断问题\n"
|
||||
"列出必须修复的问题;每条写明章节/文件、问题、为什么阻断、建议动作。\n"
|
||||
"## 章节级审校表\n"
|
||||
"用表格覆盖 ch01-ch11:主线质量、证据密度、法规覆盖、整改可执行性、是否需要回炉。\n"
|
||||
"## 证据与信源质量\n"
|
||||
"单独评价 FDA warning letters、ICH Q9/Q10、EU GMP Annex 1、本地缓存信源、第三方低质信源的使用情况。\n"
|
||||
"## 碎片化与叙事连贯性\n"
|
||||
"判断并发研究是否造成割裂,并给出具体整合建议。\n"
|
||||
"## Phase2 回炉任务清单\n"
|
||||
"如果需要回炉,列出可执行任务卡级别的补证据/重写要求。\n"
|
||||
"## Phase4 准入条件\n"
|
||||
"明确进入 final 前必须满足的条件。\n"
|
||||
)
|
||||
|
||||
|
||||
def build_phase3_model_critique(
|
||||
project_root: Path,
|
||||
*,
|
||||
client: Any,
|
||||
model: str = "zenmux-anthropic/claude-opus-4-7",
|
||||
max_context_chars: int = 650_000,
|
||||
) -> Path:
|
||||
context = build_phase3_model_review_context(project_root, max_chars=max_context_chars)
|
||||
content = client.chat_complete(
|
||||
model=model,
|
||||
system=phase3_model_review_system_prompt(),
|
||||
user=context,
|
||||
temperature=0.2,
|
||||
max_tokens=20_000,
|
||||
tag="phase3:opus-review",
|
||||
)
|
||||
out = project_root / "phase3" / "critique.md"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(content.rstrip() + "\n", encoding="utf-8")
|
||||
|
||||
manifest = load_manifest(project_root)
|
||||
phase3 = manifest.setdefault("phase3", {})
|
||||
phase3.update(
|
||||
{
|
||||
"status": "completed",
|
||||
"review_mode": "model",
|
||||
"review_model": model,
|
||||
"critique_path": "phase3/critique.md",
|
||||
"context_path": "phase3/review_context_opus_4_7.md",
|
||||
"deterministic_critique_path": "phase3/critique_deterministic.md",
|
||||
"updated_at": utc_now_iso(),
|
||||
}
|
||||
)
|
||||
manifest["updated_at"] = utc_now_iso()
|
||||
write_manifest(project_root, manifest)
|
||||
return out
|
||||
|
||||
|
||||
def build_phase3_critique(project_root: Path) -> Path:
|
||||
manifest = load_manifest(project_root)
|
||||
drafts = sorted((project_root / "phase2" / "drafts").glob("ch*.md"))
|
||||
packets = sorted((project_root / "phase2" / "packets").glob("*.json"))
|
||||
ready_stems = _ready_packet_stems(project_root)
|
||||
packet_errors = [
|
||||
path for path in sorted((project_root / "phase2" / "packet_errors").glob("*.json"))
|
||||
if path.stem not in ready_stems
|
||||
]
|
||||
chapter_errors = sorted((project_root / "phase2" / "chapter_errors").glob("*.json"))
|
||||
sources = _source_ids_from_jsonl(project_root / "phase2" / "sources.jsonl")
|
||||
cited = _draft_citations(drafts)
|
||||
missing_sources = sorted(cited - sources) if sources else sorted(cited)
|
||||
uncited_sources = sorted(sources - cited) if cited else sorted(sources)
|
||||
|
||||
findings: list[dict[str, Any]] = []
|
||||
if not drafts:
|
||||
findings.append({"severity": "P1", "message": "Phase 2 drafts 缺失,尚不能进入 Phase 4 成稿。"})
|
||||
if packet_errors:
|
||||
findings.append({"severity": "P1", "message": f"存在 {len(packet_errors)} 个 packet 失败,需要回炉补证据。"})
|
||||
if chapter_errors:
|
||||
findings.append({"severity": "P1", "message": f"存在 {len(chapter_errors)} 个章节组装失败,需要修复引用或重写该章。"})
|
||||
quality_holds = manifest.get("quality_holds") or []
|
||||
if quality_holds:
|
||||
findings.append({"severity": "P1", "message": "存在质量暂停标记:" + ", ".join(quality_holds)})
|
||||
findings.extend(_draft_quality_findings(drafts))
|
||||
if missing_sources:
|
||||
findings.append({"severity": "P1", "message": f"正文引用未在 sources.jsonl 中登记:{', '.join(missing_sources)}"})
|
||||
if not findings:
|
||||
findings.append({"severity": "P2", "message": "基础产物完整;仍需人工或大上下文模型审校逻辑链、反方证据和章节叙事。"})
|
||||
|
||||
lines = [
|
||||
"# Phase 3 审校 critique",
|
||||
"",
|
||||
f"- 项目:{manifest.get('topic', project_root.name)}",
|
||||
f"- 运行时:python-core-v0.20",
|
||||
f"- drafts:{len(drafts)}",
|
||||
f"- packets:{len(packets)}",
|
||||
f"- sources:{len(sources)}",
|
||||
f"- cited_source_ids:{', '.join(sorted(cited)) if cited else '无'}",
|
||||
"",
|
||||
"## Findings",
|
||||
"",
|
||||
]
|
||||
for item in findings:
|
||||
lines.append(f"- [{item['severity']}] {item['message']}")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Residual Risks",
|
||||
"",
|
||||
"- 本 deterministic review 只做结构、引用和错误包检查;深层逻辑审校仍建议交给 `phase3_review` 角色执行。",
|
||||
"- 若 sources 为空,本审校会把所有正文引用视为待登记来源。",
|
||||
"",
|
||||
"## Next",
|
||||
"",
|
||||
"- 若存在 P1,先回到 Phase 2 修复 packet/chapter 错误。",
|
||||
"- 若仅有 P2,可进入 `dr.py finalize` 的中文原生成稿路径。",
|
||||
"",
|
||||
]
|
||||
)
|
||||
out = project_root / "phase3" / "critique.md"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text("\n".join(lines), encoding="utf-8")
|
||||
phase3 = manifest.setdefault("phase3", {})
|
||||
phase3.update(
|
||||
{
|
||||
"status": "completed",
|
||||
"critique_path": "phase3/critique.md",
|
||||
"findings_total": len(findings),
|
||||
"missing_sources": missing_sources,
|
||||
"uncited_sources": uncited_sources,
|
||||
"updated_at": utc_now_iso(),
|
||||
}
|
||||
)
|
||||
manifest["updated_at"] = utc_now_iso()
|
||||
write_manifest(project_root, manifest)
|
||||
return out
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Runtime role and task-model resolution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from scripts.lib.model_config import resolve_model_profile
|
||||
|
||||
|
||||
ROLE_DEFAULTS = {
|
||||
"dr_plan": {
|
||||
"skills": ["document-ingest", "search-gateway", "search-strategy", "source-quality", "length-budget", "mckinsey-method"],
|
||||
"temperature": 0.4,
|
||||
"max_tokens": 12000,
|
||||
"max_concurrency": 1,
|
||||
},
|
||||
"dr_pm": {
|
||||
"skills": ["length-budget", "evidence-table", "mckinsey-method"],
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 8000,
|
||||
"max_concurrency": 1,
|
||||
},
|
||||
"dr_searcher": {
|
||||
"skills": ["search-gateway", "search-strategy", "source-quality"],
|
||||
"temperature": 0.1,
|
||||
"max_tokens": 6000,
|
||||
"max_concurrency": 6,
|
||||
},
|
||||
"dr_analyst": {
|
||||
"skills": ["search-gateway", "search-strategy", "source-quality", "evidence-table", "mckinsey-method"],
|
||||
"temperature": 0.3,
|
||||
"max_tokens": 14000,
|
||||
"max_concurrency": 6,
|
||||
},
|
||||
"dr_verifier": {
|
||||
"skills": ["search-gateway", "search-strategy", "source-quality", "evidence-table"],
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 10000,
|
||||
"max_concurrency": 4,
|
||||
},
|
||||
"dr_chief_editor": {
|
||||
"skills": ["mckinsey-method", "evidence-table", "output-hygiene"],
|
||||
"temperature": 0.2,
|
||||
"max_tokens": 16000,
|
||||
"max_concurrency": 1,
|
||||
},
|
||||
"dr_editor_in_chief": {
|
||||
"skills": ["mckinsey-method", "citation-manager", "humanizer-cn", "output-hygiene"],
|
||||
"temperature": 0.4,
|
||||
"max_tokens": 20000,
|
||||
"max_concurrency": 1,
|
||||
},
|
||||
"dr_reporter": {
|
||||
"skills": ["pdf-reportlab", "citation-manager", "output-hygiene"],
|
||||
"temperature": 0.1,
|
||||
"max_tokens": 6000,
|
||||
"max_concurrency": 1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
ROLE_IDENTITIES = {
|
||||
"dr_plan": (
|
||||
"你是 Deep Research 的 Phase1 研究架构师。你的工作不是列目录,而是先消化材料、访谈和初步搜索,"
|
||||
"形成可被证伪的主判断、章节命题和求证路线。你要大胆假设,但必须给 Phase2 留下清晰的验证和推翻条件。"
|
||||
),
|
||||
"dr_pm": (
|
||||
"你是 Deep Research 的研究项目经理。你的职责是把研究意图转化为可并发执行、可回收校验的任务,"
|
||||
"控制碎片化、重复检索和上下文污染。"
|
||||
),
|
||||
"dr_searcher": (
|
||||
"你是 Deep Research 的信源发现员。你的职责是用短英文关键词和轴向词找到高质量入口,"
|
||||
"优先官方、法规、学术和一手材料;你不写结论,只交付可追溯来源。"
|
||||
),
|
||||
"dr_analyst": (
|
||||
"你是 Deep Research 的章节证据分析师。你的职责不是写一篇像样的空泛文章,而是围绕 Phase1 命题"
|
||||
"小心求证:提取材料原文、检索权威证据、寻找反方边界,并把证据整理成可审计的结构化 packet。"
|
||||
),
|
||||
"dr_verifier": (
|
||||
"你是 Deep Research 的独立反方审校员。你的默认姿态是质疑:找证据缺口、适用边界、反例和过度推断,"
|
||||
"并指出哪些结论必须降级或回炉。"
|
||||
),
|
||||
"dr_chief_editor": (
|
||||
"你是 Deep Research 的 Phase3 总编审校。你的职责是通读 Phase1 假设与 Phase2 证据,判断二者是否自洽,"
|
||||
"优先指出结构性失败、证据不足和需要回炉的章节。"
|
||||
),
|
||||
"dr_editor_in_chief": (
|
||||
"你是 Deep Research 的终稿主编。你的职责是把已验证证据组织成客户可读的中文报告,"
|
||||
"保持观点清晰、证据密实、表达克制,避免翻译腔和 AI 味。"
|
||||
),
|
||||
"dr_reporter": (
|
||||
"你是 Deep Research 的报告制作负责人。你的职责是把已定稿内容可靠渲染为 PDF/DOCX,"
|
||||
"确保引用、排版、中文字体、表格和输出卫生可交付。"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RoleDefinition:
|
||||
name: str
|
||||
model: str
|
||||
skills: list[str]
|
||||
temperature: float
|
||||
max_tokens: int
|
||||
max_concurrency: int
|
||||
identity: str = ""
|
||||
|
||||
|
||||
class RuntimeProfile:
|
||||
def __init__(self, *, profile: str, roles: dict[str, RoleDefinition], task_types: dict[str, str]) -> None:
|
||||
self.profile = profile
|
||||
self.roles = roles
|
||||
self.task_types = task_types
|
||||
|
||||
def role_for_task(self, task_type: str) -> RoleDefinition:
|
||||
role_name = self.task_types.get(task_type)
|
||||
if not role_name:
|
||||
raise KeyError(f"unknown task_type: {task_type}")
|
||||
if role_name not in self.roles:
|
||||
raise KeyError(f"task_type {task_type} maps to missing role {role_name}")
|
||||
return self.roles[role_name]
|
||||
|
||||
|
||||
def resolve_runtime_profile(
|
||||
*,
|
||||
profile: str | None = None,
|
||||
overrides: dict[str, str] | None = None,
|
||||
) -> RuntimeProfile:
|
||||
resolved = resolve_model_profile(profile=profile, overrides=overrides)
|
||||
role_models = resolved["roles"]
|
||||
roles: dict[str, RoleDefinition] = {}
|
||||
for name, defaults in ROLE_DEFAULTS.items():
|
||||
model = role_models.get(name)
|
||||
if not model:
|
||||
continue
|
||||
roles[name] = RoleDefinition(
|
||||
name=name,
|
||||
model=model,
|
||||
skills=list(defaults["skills"]),
|
||||
temperature=float(defaults["temperature"]),
|
||||
max_tokens=int(defaults["max_tokens"]),
|
||||
max_concurrency=int(defaults["max_concurrency"]),
|
||||
identity=ROLE_IDENTITIES.get(name, ""),
|
||||
)
|
||||
return RuntimeProfile(
|
||||
profile=resolved["profile"],
|
||||
roles=roles,
|
||||
task_types=dict(resolved.get("task_types") or {}),
|
||||
)
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Canonical skill registry and adapter sync helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
CANONICAL_SKILLS_DIR = REPO_ROOT / "platform_adapters" / "antigravity" / "agent" / "skills"
|
||||
LEGACY_AGENT_SKILLS_DIR = REPO_ROOT / ".agent" / "skills"
|
||||
LEGACY_AGENTS_SKILLS_DIR = REPO_ROOT / ".agents" / "skills"
|
||||
PROJECT_SKILLS_DIR = REPO_ROOT / "skills"
|
||||
REQUIRED_SKILLS = {
|
||||
"search-strategy",
|
||||
"search-gateway",
|
||||
"source-quality",
|
||||
"length-budget",
|
||||
"evidence-table",
|
||||
"citation-manager",
|
||||
"mckinsey-method",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkillInfo:
|
||||
name: str
|
||||
path: Path
|
||||
|
||||
|
||||
class SkillRegistry:
|
||||
"""Reads skills from the canonical cross-adapter registry."""
|
||||
|
||||
def __init__(self, canonical_dir: Path | None = None) -> None:
|
||||
self.canonical_dir = canonical_dir or CANONICAL_SKILLS_DIR
|
||||
|
||||
def roots(self) -> list[Path]:
|
||||
roots = []
|
||||
if self.canonical_dir == CANONICAL_SKILLS_DIR and PROJECT_SKILLS_DIR.exists():
|
||||
roots.append(PROJECT_SKILLS_DIR)
|
||||
roots.append(self.canonical_dir)
|
||||
if self.canonical_dir == CANONICAL_SKILLS_DIR and LEGACY_AGENT_SKILLS_DIR.exists():
|
||||
roots.append(LEGACY_AGENT_SKILLS_DIR)
|
||||
if self.canonical_dir == CANONICAL_SKILLS_DIR and LEGACY_AGENTS_SKILLS_DIR.exists():
|
||||
roots.append(LEGACY_AGENTS_SKILLS_DIR)
|
||||
return roots
|
||||
|
||||
def list(self) -> list[SkillInfo]:
|
||||
seen: set[str] = set()
|
||||
out: list[SkillInfo] = []
|
||||
for root in self.roots():
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in sorted(root.glob("*/SKILL.md")):
|
||||
name = path.parent.name
|
||||
if name in seen:
|
||||
continue
|
||||
seen.add(name)
|
||||
out.append(SkillInfo(name=name, path=path))
|
||||
return out
|
||||
|
||||
def list_names(self) -> list[str]:
|
||||
return [item.name for item in self.list()]
|
||||
|
||||
def read(self, name: str) -> str:
|
||||
for root in self.roots():
|
||||
path = root / name / "SKILL.md"
|
||||
if path.exists():
|
||||
return path.read_text(encoding="utf-8")
|
||||
raise FileNotFoundError(f"skill not found: {name}")
|
||||
|
||||
def validate(self, required: set[str] | None = None) -> dict[str, object]:
|
||||
names = set(self.list_names())
|
||||
required_names = required or REQUIRED_SKILLS
|
||||
missing = sorted(required_names - names)
|
||||
malformed: list[str] = []
|
||||
for item in self.list():
|
||||
text = item.path.read_text(encoding="utf-8")
|
||||
if "name:" not in text[:300]:
|
||||
malformed.append(item.name)
|
||||
return {
|
||||
"ok": not missing and not malformed,
|
||||
"canonical_dir": str(self.canonical_dir),
|
||||
"count": len(names),
|
||||
"missing": missing,
|
||||
"malformed": malformed,
|
||||
}
|
||||
|
||||
def sync_to(self, targets: list[Path], *, force: bool = True) -> int:
|
||||
"""Copy canonical skills into adapter skill directories.
|
||||
|
||||
Returns the number of skill directories copied across all targets.
|
||||
"""
|
||||
copied = 0
|
||||
for target in targets:
|
||||
if target.resolve() == self.canonical_dir.resolve():
|
||||
continue
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
for item in self.list():
|
||||
dst = target / item.name
|
||||
if dst.exists() and force:
|
||||
shutil.rmtree(dst)
|
||||
if not dst.exists():
|
||||
shutil.copytree(item.path.parent, dst)
|
||||
copied += 1
|
||||
return copied
|
||||
|
||||
|
||||
def default_adapter_skill_dirs() -> list[Path]:
|
||||
return [
|
||||
REPO_ROOT / ".opencode" / "skills",
|
||||
REPO_ROOT / "platform_adapters" / "antigravity" / "agent" / "skills",
|
||||
]
|
||||
@@ -0,0 +1,230 @@
|
||||
"""Cache important external sources as local Markdown snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from lxml import html
|
||||
|
||||
|
||||
IMPORTANT_DOMAINS = (
|
||||
"fda.gov",
|
||||
"ema.europa.eu",
|
||||
"nmpa.gov.cn",
|
||||
"cde.org.cn",
|
||||
"ich.org",
|
||||
"who.int",
|
||||
"edqm.eu",
|
||||
"pmda.go.jp",
|
||||
"ec.europa.eu",
|
||||
"health.ec.europa.eu",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CacheResult:
|
||||
source_id: str
|
||||
url: str
|
||||
cached_text_path: str
|
||||
raw_path: str
|
||||
status: str
|
||||
chars: int
|
||||
|
||||
|
||||
def _safe_stem(source: dict) -> str:
|
||||
source_id = str(source.get("id") or "source")
|
||||
digest = hashlib.sha1(str(source.get("url") or source_id).encode("utf-8")).hexdigest()[:10]
|
||||
safe_id = re.sub(r"[^A-Za-z0-9_-]+", "_", source_id).strip("_") or "source"
|
||||
return f"{safe_id}-{digest}"
|
||||
|
||||
|
||||
def _domain(url: str) -> str:
|
||||
return urlparse(url).netloc.lower()
|
||||
|
||||
|
||||
def is_important_source(source: dict) -> bool:
|
||||
url = str(source.get("url") or "")
|
||||
if not url.startswith(("http://", "https://")):
|
||||
return False
|
||||
domain = _domain(url)
|
||||
if any(domain.endswith(item) for item in IMPORTANT_DOMAINS):
|
||||
return True
|
||||
tier = str(source.get("tier") or "").lower()
|
||||
if "tier 1" in tier or tier in {"1", "1.0"}:
|
||||
return True
|
||||
title = str(source.get("title") or "").lower()
|
||||
return any(term in title for term in ("ich q9", "ich q10", "annex 1", "fda guidance", "who guideline"))
|
||||
|
||||
|
||||
def load_sources(path: Path) -> list[dict]:
|
||||
if not path.exists():
|
||||
return []
|
||||
rows: list[dict] = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
|
||||
def write_sources(path: Path, rows: list[dict]) -> None:
|
||||
path.write_text("".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows), encoding="utf-8")
|
||||
|
||||
|
||||
def _response_ext(url: str, content_type: str) -> str:
|
||||
lowered = url.lower()
|
||||
if "pdf" in content_type or lowered.endswith(".pdf"):
|
||||
return ".pdf"
|
||||
if "html" in content_type or lowered.endswith((".html", ".htm", "/")):
|
||||
return ".html"
|
||||
return ".bin"
|
||||
|
||||
|
||||
def _html_to_text(content: bytes) -> str:
|
||||
doc = html.fromstring(content)
|
||||
for bad in doc.xpath("//script|//style|//noscript"):
|
||||
bad.drop_tree()
|
||||
return "\n".join(line.strip() for line in doc.text_content().splitlines() if line.strip())
|
||||
|
||||
|
||||
def _pdf_to_text(path: Path) -> str:
|
||||
try:
|
||||
import fitz
|
||||
except Exception:
|
||||
return ""
|
||||
doc = fitz.open(path)
|
||||
parts: list[str] = []
|
||||
for index, page in enumerate(doc, start=1):
|
||||
text = page.get_text("text").strip()
|
||||
if text:
|
||||
parts.append(f"## Page {index}\n\n{text}")
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def _bytes_to_text(*, raw_path: Path, content: bytes, content_type: str, url: str) -> str:
|
||||
if raw_path.suffix == ".pdf" or "pdf" in content_type or url.lower().endswith(".pdf"):
|
||||
return _pdf_to_text(raw_path)
|
||||
if raw_path.suffix in {".html", ".htm"} or "html" in content_type:
|
||||
return _html_to_text(content)
|
||||
try:
|
||||
return content.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
return content.decode("utf-8", errors="ignore")
|
||||
|
||||
|
||||
def cache_source(
|
||||
project_root: Path,
|
||||
source: dict,
|
||||
*,
|
||||
client: httpx.Client | None = None,
|
||||
force: bool = False,
|
||||
timeout: float = 45.0,
|
||||
) -> CacheResult:
|
||||
url = str(source.get("url") or "")
|
||||
if not url.startswith(("http://", "https://")):
|
||||
raise ValueError(f"source URL is not remote: {url}")
|
||||
cache_dir = project_root / "phase2" / "source_cache"
|
||||
raw_dir = cache_dir / "raw"
|
||||
text_dir = cache_dir / "md"
|
||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
text_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
stem = _safe_stem(source)
|
||||
md_path = text_dir / f"{stem}.md"
|
||||
if md_path.exists() and not force:
|
||||
return CacheResult(
|
||||
source_id=str(source.get("id") or ""),
|
||||
url=url,
|
||||
cached_text_path=str(md_path.relative_to(project_root)),
|
||||
raw_path=str(source.get("cached_raw_path") or ""),
|
||||
status="cached",
|
||||
chars=len(md_path.read_text(encoding="utf-8")),
|
||||
)
|
||||
|
||||
owns_client = client is None
|
||||
http = client or httpx.Client(trust_env=False, follow_redirects=True, timeout=timeout)
|
||||
try:
|
||||
response = http.get(url)
|
||||
response.raise_for_status()
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
ext = _response_ext(str(response.url), content_type)
|
||||
raw_path = raw_dir / f"{stem}{ext}"
|
||||
raw_path.write_bytes(response.content)
|
||||
text = _bytes_to_text(raw_path=raw_path, content=response.content, content_type=content_type, url=str(response.url))
|
||||
lines = [
|
||||
f"# Source Snapshot: {source.get('title') or source.get('id') or url}",
|
||||
"",
|
||||
f"- source_id: {source.get('id', '')}",
|
||||
f"- original_url: {url}",
|
||||
f"- fetched_url: {response.url}",
|
||||
f"- content_type: {content_type}",
|
||||
f"- raw_path: {raw_path.relative_to(project_root)}",
|
||||
"",
|
||||
"## Extracted Text",
|
||||
"",
|
||||
text.strip() or "[No extractable text. Keep raw file for manual review.]",
|
||||
"",
|
||||
]
|
||||
md_path.write_text("\n".join(lines), encoding="utf-8")
|
||||
return CacheResult(
|
||||
source_id=str(source.get("id") or ""),
|
||||
url=url,
|
||||
cached_text_path=str(md_path.relative_to(project_root)),
|
||||
raw_path=str(raw_path.relative_to(project_root)),
|
||||
status="fetched",
|
||||
chars=len(text),
|
||||
)
|
||||
finally:
|
||||
if owns_client:
|
||||
http.close()
|
||||
|
||||
|
||||
def cache_sources(
|
||||
project_root: Path,
|
||||
*,
|
||||
sources_rel: str = "phase2/sources.jsonl",
|
||||
important_only: bool = True,
|
||||
limit: int | None = None,
|
||||
force: bool = False,
|
||||
) -> list[CacheResult]:
|
||||
sources_path = project_root / sources_rel
|
||||
rows = load_sources(sources_path)
|
||||
results: list[CacheResult] = []
|
||||
selected_indexes = [
|
||||
index
|
||||
for index, row in enumerate(rows)
|
||||
if row.get("url")
|
||||
and (not row.get("cached_text_path") or force)
|
||||
and (not important_only or is_important_source(row))
|
||||
]
|
||||
if limit is not None:
|
||||
selected_indexes = selected_indexes[:limit]
|
||||
|
||||
with httpx.Client(trust_env=False, follow_redirects=True, timeout=45.0) as client:
|
||||
for index in selected_indexes:
|
||||
row = rows[index]
|
||||
try:
|
||||
result = cache_source(project_root, row, client=client, force=force)
|
||||
except Exception as exc:
|
||||
row["cache_status"] = "failed"
|
||||
row["cache_error"] = str(exc)[:300]
|
||||
continue
|
||||
row["cached_text_path"] = result.cached_text_path
|
||||
row["cached_raw_path"] = result.raw_path
|
||||
row["cache_status"] = result.status
|
||||
row["cached_text_chars"] = result.chars
|
||||
results.append(result)
|
||||
write_sources(sources_path, rows)
|
||||
manifest = project_root / "phase2" / "source_cache" / "manifest.json"
|
||||
manifest.parent.mkdir(parents=True, exist_ok=True)
|
||||
manifest.write_text(
|
||||
json.dumps([result.__dict__ for result in results], ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return results
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Source registry helpers for Phase 2 packets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _source_key(source: dict[str, Any]) -> str:
|
||||
return (source.get("id") or source.get("source_id") or source.get("doi") or source.get("url") or "").strip()
|
||||
|
||||
|
||||
def append_packet_sources(sources_path: Path, packet: dict[str, Any]) -> int:
|
||||
"""Append packet sources to sources.jsonl, preserving every citeable source_id."""
|
||||
sources_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
existing: set[str] = set()
|
||||
if sources_path.exists():
|
||||
for line in sources_path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
existing.add(_source_key(json.loads(line)))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
written = 0
|
||||
with sources_path.open("a", encoding="utf-8") as f:
|
||||
for source in packet.get("sources") or []:
|
||||
key = _source_key(source)
|
||||
if not key or key in existing:
|
||||
continue
|
||||
existing.add(key)
|
||||
f.write(json.dumps(source, ensure_ascii=False) + "\n")
|
||||
written += 1
|
||||
return written
|
||||
|
||||
|
||||
def rebuild_sources_from_packets(project_root: Path) -> int:
|
||||
"""Rebuild phase2/sources.jsonl from packet-level source metadata.
|
||||
|
||||
The registry is keyed by source_id, not URL. Two packet sources may point to
|
||||
the same URL but have different source_ids already cited in drafts; dropping
|
||||
either row would break citation traceability.
|
||||
"""
|
||||
packets_dir = project_root / "phase2" / "packets"
|
||||
sources_path = project_root / "phase2" / "sources.jsonl"
|
||||
sources_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
existing_by_key: dict[str, dict[str, Any]] = {}
|
||||
if sources_path.exists():
|
||||
for line in sources_path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
key = _source_key(row)
|
||||
if key:
|
||||
existing_by_key[key] = row
|
||||
seen: set[str] = set()
|
||||
rows: list[dict[str, Any]] = []
|
||||
|
||||
for packet_path in sorted(packets_dir.glob("*.json")):
|
||||
try:
|
||||
packet = json.loads(packet_path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
for source in packet.get("sources") or []:
|
||||
if not isinstance(source, dict):
|
||||
continue
|
||||
key = _source_key(source)
|
||||
if not key or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
previous = existing_by_key.get(key, {})
|
||||
rows.append({**source, **{k: v for k, v in previous.items() if k.startswith("cache") or k.startswith("cached_")}})
|
||||
|
||||
sources_path.write_text(
|
||||
"".join(json.dumps(row, ensure_ascii=False) + "\n" for row in rows),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return len(rows)
|
||||
@@ -0,0 +1,472 @@
|
||||
"""Task-card and evidence-packet primitives for v0.20 Phase 2."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from scripts.runtime.methods import ResearchMethod
|
||||
|
||||
|
||||
VALID_ROUTES = {"general", "evidence", "scholar", "patents", "news", "fda"}
|
||||
DEFAULT_AXES = ["literature", "regulatory", "patents", "market", "counter"]
|
||||
AXIS_ROUTES = {
|
||||
"literature": ["scholar", "evidence", "general"],
|
||||
"clinical": ["scholar", "evidence", "general"],
|
||||
"regulatory": ["fda", "evidence", "general", "news"],
|
||||
"patents": ["patents", "evidence", "general"],
|
||||
"market": ["news", "general"],
|
||||
"china": ["news", "general"],
|
||||
"counter": ["fda", "scholar", "evidence", "general"],
|
||||
"regulatory_gap": ["fda", "evidence", "general", "news"],
|
||||
"risk_classification": ["evidence", "general", "scholar"],
|
||||
"capa_design": ["evidence", "general", "news"],
|
||||
"ownership_timeline": ["general"],
|
||||
"verification_evidence": ["fda", "evidence", "general", "scholar"],
|
||||
"process_flow": ["scholar", "evidence", "general"],
|
||||
"cqa_cpp": ["scholar", "evidence", "general"],
|
||||
"scale_up_risk": ["scholar", "evidence", "general"],
|
||||
"control_strategy": ["scholar", "evidence", "general"],
|
||||
"supply_chain": ["news", "general"],
|
||||
"scientific_rationale": ["scholar", "evidence", "general"],
|
||||
"poc_evidence": ["scholar", "evidence", "general"],
|
||||
"ip_fto": ["patents", "evidence", "general"],
|
||||
"development_path": ["scholar", "evidence", "general"],
|
||||
"commercial_window": ["news", "general"],
|
||||
"current_state": ["evidence", "general"],
|
||||
"capability_gap": ["evidence", "general"],
|
||||
"operating_model": ["evidence", "general"],
|
||||
"governance": ["evidence", "general"],
|
||||
"implementation_roadmap": ["evidence", "general"],
|
||||
"nmpa_fda_ema_ich_who_baseline": ["fda", "evidence", "general", "news"],
|
||||
"quality_system_gap": ["fda", "evidence", "general"],
|
||||
"manufacturing_process_risk": ["fda", "scholar", "evidence", "general"],
|
||||
"operations_management_gap": ["fda", "evidence", "general"],
|
||||
"team_capability": ["evidence", "general", "news"],
|
||||
"capa_roadmap": ["fda", "evidence", "general"],
|
||||
"input_material_findings": ["evidence", "general"],
|
||||
"fda_enforcement_precedents": ["fda"],
|
||||
"chapter_integrated": ["fda", "scholar", "evidence", "general"],
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Chapter:
|
||||
chapter_id: str
|
||||
index: int
|
||||
title: str
|
||||
notes: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskCard:
|
||||
task_id: str
|
||||
chapter_ids: list[str]
|
||||
topic_axis: str
|
||||
questions: list[str]
|
||||
search_routes: list[str]
|
||||
output_packet: str
|
||||
chapter_title: str = ""
|
||||
preferred_model_role: str = "dr_analyst"
|
||||
status: str = "pending"
|
||||
dependencies: list[str] = field(default_factory=list)
|
||||
research_goal: str = ""
|
||||
research_method: str = ""
|
||||
prompt_brief: str = ""
|
||||
required_skills: list[str] = field(default_factory=list)
|
||||
allowed_materials: list[str] = field(default_factory=list)
|
||||
expected_evidence: dict[str, Any] = field(default_factory=dict)
|
||||
stop_conditions: list[str] = field(default_factory=list)
|
||||
model_hint: str = ""
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def parse_framework_chapters(framework_text: str) -> list[Chapter]:
|
||||
"""Extract Chinese or English chapter headings from a framework markdown."""
|
||||
lines = framework_text.splitlines()
|
||||
chapters: list[Chapter] = []
|
||||
current: Chapter | None = None
|
||||
note_lines: list[str] = []
|
||||
heading_re = re.compile(
|
||||
r"^#{1,3}\s*(?:第\s*)?(\d{1,2})\s*(?:章|[.)、:-])?\s*(.+?)\s*$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
english_re = re.compile(r"^#{1,3}\s*chapter\s+(\d{1,2})[:.)\s-]+(.+?)\s*$", re.IGNORECASE)
|
||||
for line in lines:
|
||||
match = heading_re.match(line.strip()) or english_re.match(line.strip())
|
||||
if match:
|
||||
if current:
|
||||
current.notes = "\n".join(note_lines).strip()
|
||||
chapters.append(current)
|
||||
index = int(match.group(1))
|
||||
title = match.group(2).strip(" #")
|
||||
current = Chapter(chapter_id=f"ch{index:02d}", index=index, title=title)
|
||||
note_lines = []
|
||||
elif current:
|
||||
note_lines.append(line)
|
||||
if current:
|
||||
current.notes = "\n".join(note_lines).strip()
|
||||
chapters.append(current)
|
||||
return chapters
|
||||
|
||||
|
||||
def _questions_for_axis(chapter: Chapter, axis: str, method: ResearchMethod | None = None) -> list[str]:
|
||||
if axis == "chapter_integrated":
|
||||
lanes = ";".join(method.integrated_lanes if method else [])
|
||||
return [
|
||||
f"围绕《{chapter.title}》形成章节级综合证据包,不再拆成孤立小轴。",
|
||||
f"必须按当前 research_method 的 evidence lanes 组织证据:{lanes or '本地材料、权威来源、反方证据、可执行建议'}。",
|
||||
"若项目有用户材料,必须先读取本地材料证据并提取原文;再用本方法适用的权威来源交叉验证。",
|
||||
"必须形成:材料/事实基线、外部权威证据、差距或机会判断、反方/限制条件、可执行建议和待补证据。",
|
||||
]
|
||||
questions = [
|
||||
f"围绕《{chapter.title}》从 {axis} 角度提炼可证伪的核心结论。",
|
||||
"至少寻找两个 Tier 1-2 来源支撑主要结论;不足时标注待验证。",
|
||||
"主动检索反方证据、限制条件或失败案例。",
|
||||
]
|
||||
if axis in {
|
||||
"nmpa_fda_ema_ich_who_baseline",
|
||||
"quality_system_gap",
|
||||
"manufacturing_process_risk",
|
||||
"operations_management_gap",
|
||||
"capa_roadmap",
|
||||
"verification_evidence",
|
||||
"counter",
|
||||
"fda_enforcement_precedents",
|
||||
}:
|
||||
questions.append(
|
||||
"必须检索并优先评估 FDA Warning Letters、inspection/enforcement 页面、会议纪要或 meeting materials,作为 GMP 缺陷严重度和整改优先级的佐证。"
|
||||
)
|
||||
return questions
|
||||
|
||||
|
||||
def _default_required_skills(axis: str) -> list[str]:
|
||||
skills = ["search-gateway", "search-strategy", "source-quality", "evidence-table"]
|
||||
if axis == "counter":
|
||||
skills.append("mckinsey-method")
|
||||
return skills
|
||||
|
||||
|
||||
def _default_expected_evidence(axis: str) -> dict[str, Any]:
|
||||
expected = {
|
||||
"min_tier_1_2_sources": 2,
|
||||
"must_include_counter_evidence": True,
|
||||
"must_include_source_metadata": True,
|
||||
"preferred_evidence_types": [
|
||||
"regulatory_or_best_practice_requirement",
|
||||
"fda_warning_letter_or_meeting_record",
|
||||
"site_or_material_finding",
|
||||
"quantitative_fact_or_record",
|
||||
"implementation_or_verification_evidence",
|
||||
],
|
||||
"axis": axis,
|
||||
}
|
||||
if axis == "chapter_integrated":
|
||||
expected.update(
|
||||
{
|
||||
"min_local_material_evidence": 2,
|
||||
"min_official_sources": 2,
|
||||
"min_fda_or_regulatory_precedents": 1,
|
||||
"min_capa_actions": 3,
|
||||
"preferred_evidence_types": [
|
||||
"local_audit_or_recap_quote",
|
||||
"official_regulatory_requirement",
|
||||
"fda_warning_letter_or_meeting_record",
|
||||
"gap_analysis",
|
||||
"capa_action_with_owner_and_verification",
|
||||
"counter_evidence_or_boundary_condition",
|
||||
],
|
||||
}
|
||||
)
|
||||
return expected
|
||||
|
||||
|
||||
def _default_stop_conditions() -> list[str]:
|
||||
return [
|
||||
"已形成至少 3 条可追溯 evidence_items,且每条关键 claim 有 source_id。",
|
||||
"已主动记录 counter_evidence 或明确说明未找到反方证据的检索路径。",
|
||||
"candidate_sources 不足以支撑结论时停止写作,并把缺口写入 open_questions。",
|
||||
]
|
||||
|
||||
|
||||
def _integrated_prompt_brief(chapter: Chapter, method: ResearchMethod | None) -> str:
|
||||
lanes = ";".join(method.integrated_lanes if method else [])
|
||||
return (
|
||||
f"本任务是《{chapter.title}》的章节级综合证据包。不要把多条窄轴 packet 机械拼贴;"
|
||||
f"必须围绕当前研究方法的 lanes 一次性收束主线:{lanes or '事实材料、权威证据、反方证据、行动建议'}。"
|
||||
"输出必须让章节作者能直接写出判断、证据落点和可执行建议。"
|
||||
)
|
||||
|
||||
|
||||
def _task_card_for_chapter_axis(
|
||||
*,
|
||||
chapter: Chapter,
|
||||
axis: str,
|
||||
routes: list[str],
|
||||
method_key: str,
|
||||
required_skills: list[str] | None = None,
|
||||
allowed_materials: list[str] | None = None,
|
||||
prompt_brief: str | None = None,
|
||||
questions: list[str] | None = None,
|
||||
research_goal: str | None = None,
|
||||
expected_evidence: dict[str, Any] | None = None,
|
||||
stop_conditions: list[str] | None = None,
|
||||
method: ResearchMethod | None = None,
|
||||
) -> TaskCard:
|
||||
return TaskCard(
|
||||
task_id=f"{chapter.chapter_id}-{axis}",
|
||||
chapter_ids=[chapter.chapter_id],
|
||||
topic_axis=axis,
|
||||
questions=questions or _questions_for_axis(chapter, axis, method),
|
||||
search_routes=routes,
|
||||
output_packet=f"phase2/packets/{chapter.chapter_id}-{axis}.json",
|
||||
chapter_title=chapter.title,
|
||||
preferred_model_role="dr_verifier" if axis == "counter" else "dr_analyst",
|
||||
research_goal=research_goal or f"为《{chapter.title}》收集并验证 {axis} 轴证据,形成可写入章节的具体判断与证据落点。",
|
||||
research_method=method_key,
|
||||
prompt_brief=prompt_brief or (_integrated_prompt_brief(chapter, method) if axis == "chapter_integrated" else f"围绕《{chapter.title}》的 {axis} 轴,优先形成可证伪、可引用、可落地的证据包。"),
|
||||
required_skills=required_skills or _default_required_skills(axis),
|
||||
allowed_materials=allowed_materials or [],
|
||||
expected_evidence=expected_evidence or _default_expected_evidence(axis),
|
||||
stop_conditions=stop_conditions or _default_stop_conditions(),
|
||||
model_hint="use_cross_model_verifier" if axis == "counter" else "use_cost_effective_research_worker",
|
||||
)
|
||||
|
||||
|
||||
def generate_task_cards(
|
||||
slug: str,
|
||||
framework_text: str,
|
||||
*,
|
||||
axes: list[str] | None = None,
|
||||
method: ResearchMethod | None = None,
|
||||
) -> list[TaskCard]:
|
||||
del slug # slug is kept for call-site clarity and future namespacing.
|
||||
chapters = parse_framework_chapters(framework_text)
|
||||
selected_axes = axes or (method.task_axes if method else DEFAULT_AXES)
|
||||
cards: list[TaskCard] = []
|
||||
for chapter in chapters:
|
||||
for axis in selected_axes:
|
||||
routes = AXIS_ROUTES.get(axis, ["general"])
|
||||
cards.append(
|
||||
_task_card_for_chapter_axis(
|
||||
chapter=chapter,
|
||||
axis=axis,
|
||||
routes=routes,
|
||||
method_key=method.key if method else "",
|
||||
method=method,
|
||||
)
|
||||
)
|
||||
validate_task_cards(cards)
|
||||
return cards
|
||||
|
||||
|
||||
def generate_task_cards_from_research_brief(
|
||||
slug: str,
|
||||
framework_text: str,
|
||||
research_brief: dict[str, Any],
|
||||
*,
|
||||
axes: list[str] | None = None,
|
||||
method: ResearchMethod | None = None,
|
||||
) -> list[TaskCard]:
|
||||
del slug
|
||||
chapters = parse_framework_chapters(framework_text)
|
||||
planning = research_brief.get("task_planning") or {}
|
||||
method_key = research_brief.get("research_method") or (method.key if method else "")
|
||||
if method is None and method_key:
|
||||
from scripts.runtime.methods import ResearchMethodRegistry
|
||||
|
||||
method = ResearchMethodRegistry().get(method_key)
|
||||
phase2_mode = planning.get("phase2_mode") or research_brief.get("phase2_mode")
|
||||
if axes:
|
||||
selected_axes = axes
|
||||
elif phase2_mode == "chapter_integrated":
|
||||
selected_axes = ["chapter_integrated"]
|
||||
else:
|
||||
selected_axes = (method.task_axes if method else None) or list(planning.get("search_routes_by_axis") or []) or DEFAULT_AXES
|
||||
routes_by_axis = planning.get("search_routes_by_axis") or {}
|
||||
prompt_by_axis = planning.get("axis_prompt_briefs") or {}
|
||||
base_skills = list(planning.get("required_skills") or [])
|
||||
stop_conditions = list(planning.get("stop_conditions") or [])
|
||||
allowed_materials = [
|
||||
str(item.get("path"))
|
||||
for item in research_brief.get("materials", [])
|
||||
if item.get("path")
|
||||
]
|
||||
if not allowed_materials:
|
||||
material_digest = (research_brief.get("phase1_inputs") or {}).get("material_digest")
|
||||
if material_digest:
|
||||
allowed_materials.append(str(material_digest))
|
||||
chapter_plan_by_id = {
|
||||
str(item.get("chapter_id")): item
|
||||
for item in research_brief.get("chapter_planning", [])
|
||||
if item.get("chapter_id")
|
||||
}
|
||||
cards: list[TaskCard] = []
|
||||
for chapter in chapters:
|
||||
for axis in selected_axes:
|
||||
routes = list(routes_by_axis.get(axis) or AXIS_ROUTES.get(axis, ["general"]))
|
||||
skills = base_skills or _default_required_skills(axis)
|
||||
if "search-gateway" not in skills:
|
||||
skills = ["search-gateway", *skills]
|
||||
chapter_plan = chapter_plan_by_id.get(chapter.chapter_id) if axis == "chapter_integrated" else None
|
||||
prompt_brief = prompt_by_axis.get(axis)
|
||||
questions = None
|
||||
research_goal = None
|
||||
expected_evidence = None
|
||||
card_stop_conditions = stop_conditions or None
|
||||
if chapter_plan:
|
||||
prompt_brief = chapter_plan.get("phase2_prompt_context") or prompt_brief
|
||||
research_goal = chapter_plan.get("core_question")
|
||||
questions = [
|
||||
chapter_plan.get("core_question", ""),
|
||||
chapter_plan.get("bold_hypothesis", ""),
|
||||
"按 Phase1 求证计划逐条收集支持证据、反方证据和待补证据。",
|
||||
"不得绕开 Phase1 主基调另起炉灶;若证据推翻假设,必须明确写出修正建议。",
|
||||
]
|
||||
questions.extend(str(item) for item in chapter_plan.get("verification_plan", []))
|
||||
expected_evidence = _default_expected_evidence(axis)
|
||||
expected_evidence.update(
|
||||
{
|
||||
"phase1_minimum_evidence": chapter_plan.get("minimum_evidence") or {},
|
||||
"evidence_lanes": chapter_plan.get("evidence_lanes") or [],
|
||||
"must_address_phase1_hypothesis": True,
|
||||
}
|
||||
)
|
||||
card_stop_conditions = [
|
||||
*(stop_conditions or _default_stop_conditions()),
|
||||
"已经逐条回应 Phase1 的大胆假设:支持、修正或推翻,并说明依据。",
|
||||
"已经把本地材料原文、外部证据、反方边界和行动建议分开记录。",
|
||||
]
|
||||
cards.append(
|
||||
_task_card_for_chapter_axis(
|
||||
chapter=chapter,
|
||||
axis=axis,
|
||||
routes=routes,
|
||||
method_key=method_key,
|
||||
required_skills=skills,
|
||||
allowed_materials=allowed_materials,
|
||||
prompt_brief=prompt_brief,
|
||||
questions=questions,
|
||||
research_goal=research_goal,
|
||||
expected_evidence=expected_evidence,
|
||||
stop_conditions=card_stop_conditions,
|
||||
method=method,
|
||||
)
|
||||
)
|
||||
validate_task_cards(cards)
|
||||
return cards
|
||||
|
||||
|
||||
def detect_dependency_cycles(cards: list[TaskCard]) -> None:
|
||||
graph = {card.task_id: card.dependencies for card in cards}
|
||||
visiting: set[str] = set()
|
||||
visited: set[str] = set()
|
||||
|
||||
def visit(node: str) -> None:
|
||||
if node in visiting:
|
||||
raise ValueError(f"dependency cycle detected at {node}")
|
||||
if node in visited:
|
||||
return
|
||||
visiting.add(node)
|
||||
for dep in graph.get(node, []):
|
||||
visit(dep)
|
||||
visiting.remove(node)
|
||||
visited.add(node)
|
||||
|
||||
for task_id in graph:
|
||||
visit(task_id)
|
||||
|
||||
|
||||
def validate_task_cards(cards: list[TaskCard]) -> None:
|
||||
seen: set[str] = set()
|
||||
for card in cards:
|
||||
if not card.research_goal:
|
||||
card.research_goal = f"围绕 {card.topic_axis} 轴收集并验证结构化证据。"
|
||||
if not card.prompt_brief:
|
||||
card.prompt_brief = f"按 {card.topic_axis} 轴形成证据包,避免泛泛结论。"
|
||||
if not card.required_skills:
|
||||
card.required_skills = _default_required_skills(card.topic_axis)
|
||||
if "search-gateway" not in card.required_skills:
|
||||
card.required_skills = ["search-gateway", *card.required_skills]
|
||||
if not card.expected_evidence:
|
||||
card.expected_evidence = _default_expected_evidence(card.topic_axis)
|
||||
if not card.stop_conditions:
|
||||
card.stop_conditions = _default_stop_conditions()
|
||||
if card.task_id in seen:
|
||||
raise ValueError(f"duplicate task_id: {card.task_id}")
|
||||
seen.add(card.task_id)
|
||||
if not card.chapter_ids:
|
||||
raise ValueError(f"{card.task_id}: chapter_ids required")
|
||||
if not card.chapter_title:
|
||||
card.chapter_title = card.chapter_ids[0]
|
||||
if not card.questions:
|
||||
raise ValueError(f"{card.task_id}: questions required")
|
||||
if not card.output_packet.endswith(".json"):
|
||||
raise ValueError(f"{card.task_id}: output_packet must be json")
|
||||
invalid_routes = sorted(set(card.search_routes) - VALID_ROUTES)
|
||||
if invalid_routes:
|
||||
raise ValueError(f"{card.task_id}: invalid search_routes {invalid_routes}")
|
||||
missing_deps = sorted({dep for card in cards for dep in card.dependencies} - seen)
|
||||
if missing_deps:
|
||||
raise ValueError(f"unknown dependencies: {missing_deps}")
|
||||
detect_dependency_cycles(cards)
|
||||
|
||||
|
||||
def write_task_cards(path: Path, cards: list[TaskCard]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps([card.to_dict() for card in cards], ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def load_task_cards(path: Path) -> list[TaskCard]:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
cards = [TaskCard(**item) for item in data]
|
||||
validate_task_cards(cards)
|
||||
return cards
|
||||
|
||||
|
||||
def validate_packet(packet: dict[str, Any]) -> None:
|
||||
required = {
|
||||
"task_id",
|
||||
"claims",
|
||||
"evidence_items",
|
||||
"counter_evidence",
|
||||
"source_ids",
|
||||
"source_quality_notes",
|
||||
"open_questions",
|
||||
"raw_quotes_or_notes",
|
||||
}
|
||||
missing = sorted(required - set(packet))
|
||||
if missing:
|
||||
raise ValueError(f"packet missing fields: {missing}")
|
||||
if not packet["claims"]:
|
||||
raise ValueError("packet claims must not be empty")
|
||||
if not packet["evidence_items"]:
|
||||
raise ValueError("packet evidence_items must not be empty")
|
||||
if not packet["counter_evidence"]:
|
||||
raise ValueError("packet counter_evidence must not be empty")
|
||||
declared = set(packet.get("source_ids") or [])
|
||||
referenced: set[str] = set()
|
||||
for section in ("claims", "counter_evidence"):
|
||||
for item in packet.get(section) or []:
|
||||
referenced.update(item.get("source_ids") or [])
|
||||
for item in packet.get("evidence_items") or []:
|
||||
if item.get("source_id"):
|
||||
referenced.add(item["source_id"])
|
||||
undeclared = sorted(referenced - declared)
|
||||
if undeclared:
|
||||
raise ValueError(f"packet source_ids referenced but not declared: {undeclared}")
|
||||
packet_sources = packet.get("sources") or []
|
||||
if not packet_sources:
|
||||
raise ValueError("packet sources must not be empty")
|
||||
known_source_ids = {source.get("id") for source in packet_sources}
|
||||
missing_sources = sorted(declared - known_source_ids)
|
||||
if missing_sources:
|
||||
raise ValueError(f"packet source_ids missing source metadata: {missing_sources}")
|
||||
@@ -0,0 +1,497 @@
|
||||
"""Python role workers for task-card execution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
from typing import Callable, Protocol, Any
|
||||
|
||||
from scripts.runtime.roles import RoleDefinition, RuntimeProfile
|
||||
from scripts.runtime.skills import SkillRegistry
|
||||
from scripts.runtime.tasks import TaskCard, validate_packet
|
||||
from scripts.runtime.sources import append_packet_sources
|
||||
|
||||
|
||||
class ChatClient(Protocol):
|
||||
def chat_complete(self, **kwargs) -> str:
|
||||
...
|
||||
|
||||
|
||||
class SearchProvider(Protocol):
|
||||
def search(self, *, query: str, route: str, num_results: int) -> list[dict[str, Any]]:
|
||||
...
|
||||
|
||||
|
||||
class ProjectSearchProvider:
|
||||
"""Thin adapter over the project-owned search client."""
|
||||
|
||||
def __init__(self, *, strict_specialized: bool = True) -> None:
|
||||
from scripts.lib.search_client import SearchClient
|
||||
|
||||
self.client = SearchClient(strict_specialized=strict_specialized)
|
||||
|
||||
def search(self, *, query: str, route: str, num_results: int) -> list[dict[str, Any]]:
|
||||
if route == "scholar":
|
||||
hits = self.client.scholar(query, num_results=num_results, year_low=2020)
|
||||
elif route == "patents":
|
||||
hits = self.client.patents(query, num_results=num_results)
|
||||
elif route == "news":
|
||||
hits = self.client.news(query, num_results=num_results, time_range="y")
|
||||
elif route == "fda":
|
||||
hits = self.client.fda(query, num_results=num_results)
|
||||
elif route == "evidence":
|
||||
hits = self.client.evidence(query, num_results=num_results)
|
||||
else:
|
||||
hits = self.client.search(query, num_results=num_results)
|
||||
return [
|
||||
{
|
||||
"title": hit.title,
|
||||
"url": hit.url,
|
||||
"snippet": hit.snippet,
|
||||
"route": route,
|
||||
}
|
||||
for hit in hits
|
||||
]
|
||||
|
||||
def close(self) -> None:
|
||||
self.client.close()
|
||||
|
||||
|
||||
def _extract_json_object(text: str) -> dict:
|
||||
stripped = text.strip()
|
||||
if stripped.startswith("```"):
|
||||
stripped = stripped.strip("`")
|
||||
if stripped.startswith("json"):
|
||||
stripped = stripped[4:].strip()
|
||||
start = stripped.find("{")
|
||||
end = stripped.rfind("}")
|
||||
if start == -1 or end == -1 or end < start:
|
||||
raise ValueError("worker response does not contain a JSON object")
|
||||
return json.loads(stripped[start : end + 1])
|
||||
|
||||
|
||||
def _safe_source_stem(task_id: str) -> str:
|
||||
return re.sub(r"[^a-zA-Z0-9]+", "_", task_id).strip("_").lower()
|
||||
|
||||
|
||||
def contains_cjk(text: str) -> bool:
|
||||
return any("\u4e00" <= char <= "\u9fff" for char in text)
|
||||
|
||||
|
||||
def strip_cjk(text: str) -> str:
|
||||
return re.sub(r"[\u3400-\u9fff]+", " ", text)
|
||||
|
||||
|
||||
def validate_packet_against_allowed_context(
|
||||
packet: dict,
|
||||
search_context: dict[str, Any] | None,
|
||||
material_context: dict[str, Any] | None,
|
||||
) -> None:
|
||||
"""Ensure the model did not invent source IDs or URLs beyond candidates."""
|
||||
if not search_context and not material_context:
|
||||
return
|
||||
candidates = (search_context or {}).get("candidate_sources") or []
|
||||
materials = (material_context or {}).get("materials") or []
|
||||
if not candidates and not materials:
|
||||
return
|
||||
candidate_ids = {source.get("id") for source in candidates}
|
||||
candidate_ids.update(item.get("source_id") for item in materials)
|
||||
candidate_urls = {source.get("url") for source in candidates if source.get("url")}
|
||||
candidate_urls.update(item.get("path") for item in materials if item.get("path"))
|
||||
packet_sources = packet.get("sources") or []
|
||||
unknown_ids = sorted(
|
||||
source.get("id")
|
||||
for source in packet_sources
|
||||
if source.get("id") and source.get("id") not in candidate_ids
|
||||
)
|
||||
unknown_urls = sorted(
|
||||
source.get("url")
|
||||
for source in packet_sources
|
||||
if source.get("url") and source.get("url") not in candidate_urls
|
||||
)
|
||||
if (candidates or materials) and not packet_sources:
|
||||
raise ValueError("packet must include source metadata from candidate_sources or local materials")
|
||||
if unknown_ids:
|
||||
raise ValueError(f"packet sources include non-candidate source IDs: {unknown_ids}")
|
||||
if unknown_urls:
|
||||
raise ValueError(f"packet sources include non-candidate URLs: {unknown_urls}")
|
||||
|
||||
|
||||
def normalize_packet_against_context(
|
||||
packet: dict[str, Any],
|
||||
search_context: dict[str, Any] | None,
|
||||
material_context: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Deterministically fill schema metadata the model often omits."""
|
||||
packet = dict(packet)
|
||||
referenced: set[str] = set(packet.get("source_ids") or [])
|
||||
for section in ("claims", "counter_evidence"):
|
||||
for item in packet.get(section) or []:
|
||||
referenced.update(item.get("source_ids") or [])
|
||||
for item in packet.get("evidence_items") or []:
|
||||
if item.get("source_id"):
|
||||
referenced.add(item["source_id"])
|
||||
if "source_ids" not in packet or not packet.get("source_ids"):
|
||||
packet["source_ids"] = sorted(referenced)
|
||||
|
||||
available_sources: dict[str, dict[str, Any]] = {}
|
||||
for source in (search_context or {}).get("candidate_sources") or []:
|
||||
if source.get("id"):
|
||||
available_sources[source["id"]] = source
|
||||
for material in (material_context or {}).get("materials") or []:
|
||||
source_id = material.get("source_id")
|
||||
if source_id:
|
||||
available_sources[source_id] = {
|
||||
"id": source_id,
|
||||
"title": material.get("title") or Path(material.get("path", "")).name,
|
||||
"url": material.get("path") or "",
|
||||
"tier": "local_material",
|
||||
"score": 8,
|
||||
}
|
||||
|
||||
existing_sources = {
|
||||
source.get("id"): source
|
||||
for source in packet.get("sources") or []
|
||||
if source.get("id")
|
||||
}
|
||||
for source_id in packet.get("source_ids") or []:
|
||||
if source_id not in existing_sources and source_id in available_sources:
|
||||
existing_sources[source_id] = available_sources[source_id]
|
||||
if existing_sources:
|
||||
packet["sources"] = [existing_sources[source_id] for source_id in packet.get("source_ids", []) if source_id in existing_sources]
|
||||
return packet
|
||||
|
||||
|
||||
FDA_AXIS_TERMS = {
|
||||
"nmpa_fda_ema_ich_who_baseline": "CGMP pharmaceutical quality system process validation aseptic processing data integrity",
|
||||
"quality_system_gap": "CGMP CAPA deviation change control data integrity quality unit pharmaceutical",
|
||||
"manufacturing_process_risk": "aseptic processing sterile drug manufacturing process validation PPQ cleaning validation water system",
|
||||
"operations_management_gap": "pharmaceutical quality system quality metrics management review senior management FDA",
|
||||
"capa_roadmap": "CGMP CAPA effectiveness remediation warning letter close-out pharmaceutical",
|
||||
"verification_evidence": "FDA 483 response CAPA effectiveness verification EIR pharmaceutical quality",
|
||||
"counter": "FDA warning letter CGMP pharmaceutical quality data integrity remediation limitations",
|
||||
"fda_enforcement_precedents": "FDA warning letter CGMP pharmaceutical aseptic processing data integrity CAPA process validation",
|
||||
}
|
||||
|
||||
|
||||
FDA_CHAPTER_TERMS = {
|
||||
"ch01": "commercial readiness phase gate remediation governance",
|
||||
"ch02": "regulatory baseline CGMP EU GMP Annex 1 ICH Q9 ICH Q10",
|
||||
"ch03": "aseptic processing RABS first air media fill visual inspection depyrogenation tunnel",
|
||||
"ch04": "biologics drug substance WFI clean utilities SCADA EMS single-use system",
|
||||
"ch05": "process validation master batch record CPP CQA PPQ cleaning validation technology transfer",
|
||||
"ch06": "deviation change control CAPA document control training data integrity quality unit",
|
||||
"ch07": "training effectiveness quality culture operator qualification human factors",
|
||||
"ch08": "quality metrics management review escalation cross-functional governance operations",
|
||||
"ch09": "CDMO quality organization technology transfer project governance capability matrix",
|
||||
"ch10": "CAPA remediation plan effectiveness check owner due date verification evidence",
|
||||
"ch11": "regulatory mapping CAPA tracker closure evidence quality assurance verification",
|
||||
}
|
||||
|
||||
|
||||
ROUTE_CHAPTER_TERMS = {
|
||||
**FDA_CHAPTER_TERMS,
|
||||
}
|
||||
|
||||
ROUTE_SUFFIX_TERMS = {
|
||||
"scholar": "pharmaceutical GMP review validation risk management quality system",
|
||||
"patents": "biologics manufacturing patent process formulation device",
|
||||
"news": "pharmaceutical quality operations CDMO quality governance",
|
||||
"evidence": "pharmaceutical GMP evidence guidance enforcement best practice quality operations",
|
||||
"general": "pharmaceutical GMP best practice guidance quality operations remediation",
|
||||
}
|
||||
|
||||
INTERNAL_QUERY_TOKENS = {
|
||||
"chapter_integrated",
|
||||
"input_material_findings",
|
||||
}
|
||||
|
||||
|
||||
def _compact_english_query(*parts: str, max_terms: int = 16) -> str:
|
||||
text = strip_cjk(" ".join(part for part in parts if part))
|
||||
text = re.sub(r"[^A-Za-z0-9./+-]+", " ", text)
|
||||
terms: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for raw in text.split():
|
||||
term = raw.strip(" ./+-").lower()
|
||||
if not term or term in INTERNAL_QUERY_TOKENS:
|
||||
continue
|
||||
key = term.casefold()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
terms.append(term)
|
||||
if len(terms) >= max_terms:
|
||||
break
|
||||
return " ".join(terms)
|
||||
|
||||
|
||||
def _chapter_terms(card: TaskCard) -> str:
|
||||
mapped = " ".join(ROUTE_CHAPTER_TERMS.get(chapter_id, "") for chapter_id in card.chapter_ids)
|
||||
if mapped.strip():
|
||||
return mapped
|
||||
return strip_cjk(card.chapter_title)
|
||||
|
||||
|
||||
def build_route_query(card: TaskCard, route: str) -> str:
|
||||
"""Build short, route-aware queries instead of sending whole task cards."""
|
||||
if route == "fda":
|
||||
terms = FDA_AXIS_TERMS.get(card.topic_axis, "FDA warning letter CGMP pharmaceutical quality")
|
||||
chapter_terms = " ".join(FDA_CHAPTER_TERMS.get(chapter_id, "") for chapter_id in card.chapter_ids)
|
||||
query = f"{terms} {chapter_terms}".strip()
|
||||
if contains_cjk(query):
|
||||
raise ValueError(f"FDA route query must not contain Chinese text: {query}")
|
||||
return query
|
||||
if route == "scholar":
|
||||
return _compact_english_query(_chapter_terms(card), ROUTE_SUFFIX_TERMS["scholar"])
|
||||
if route == "patents":
|
||||
return _compact_english_query(_chapter_terms(card), ROUTE_SUFFIX_TERMS["patents"])
|
||||
if route == "news":
|
||||
return _compact_english_query(_chapter_terms(card), ROUTE_SUFFIX_TERMS["news"])
|
||||
if route == "evidence":
|
||||
return _compact_english_query(_chapter_terms(card), ROUTE_SUFFIX_TERMS["evidence"])
|
||||
return _compact_english_query(_chapter_terms(card), ROUTE_SUFFIX_TERMS["general"])
|
||||
|
||||
|
||||
def _material_excerpt(project_root: Path | None, rel_path: str, *, max_chars: int = 6000) -> dict[str, str] | None:
|
||||
if project_root is None:
|
||||
return None
|
||||
path = project_root / rel_path
|
||||
if not path.exists() or not path.is_file():
|
||||
return None
|
||||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||||
return {
|
||||
"path": rel_path,
|
||||
"source_id": f"src_local_{_safe_source_stem(Path(rel_path).stem)}",
|
||||
"title": Path(rel_path).name,
|
||||
"excerpt": text[:max_chars],
|
||||
}
|
||||
|
||||
|
||||
def build_material_context(card: TaskCard, project_root: Path | None, *, max_chars_per_material: int = 6000) -> dict[str, Any]:
|
||||
materials = []
|
||||
seen: set[str] = set()
|
||||
for rel in card.allowed_materials:
|
||||
if rel in seen:
|
||||
continue
|
||||
seen.add(rel)
|
||||
item = _material_excerpt(project_root, rel, max_chars=max_chars_per_material)
|
||||
if item:
|
||||
materials.append(item)
|
||||
return {"materials": materials}
|
||||
|
||||
|
||||
def build_search_context(
|
||||
card: TaskCard,
|
||||
search_provider: SearchProvider,
|
||||
*,
|
||||
num_results_per_route: int = 5,
|
||||
) -> dict[str, Any]:
|
||||
candidate_sources: list[dict[str, Any]] = []
|
||||
routes_used: list[str] = []
|
||||
source_stem = _safe_source_stem(card.task_id)
|
||||
idx = 1
|
||||
for route in card.search_routes:
|
||||
routes_used.append(route)
|
||||
query = build_route_query(card, route)
|
||||
hits = search_provider.search(query=query, route=route, num_results=num_results_per_route)
|
||||
for hit in hits:
|
||||
candidate_sources.append(
|
||||
{
|
||||
"id": f"src_{source_stem}_{idx:03d}",
|
||||
"title": hit.get("title", ""),
|
||||
"url": hit.get("url", ""),
|
||||
"snippet": hit.get("snippet", ""),
|
||||
"route": hit.get("route", route),
|
||||
"tier": "Tier 2",
|
||||
"score": 6,
|
||||
}
|
||||
)
|
||||
idx += 1
|
||||
return {"routes_used": routes_used, "candidate_sources": candidate_sources}
|
||||
|
||||
|
||||
def build_packet_user_prompt(
|
||||
card: TaskCard,
|
||||
search_context: dict[str, Any] | None = None,
|
||||
material_context: dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
context = search_context or {"routes_used": [], "candidate_sources": []}
|
||||
materials = material_context or {"materials": []}
|
||||
return (
|
||||
"请根据以下 task card 产出一个证据包 JSON。\n"
|
||||
"正式结论、summary、open_questions 用中文;英文原文摘录、source title、DOI/URL 可以保留英文。\n"
|
||||
"必须主动包含 counter_evidence,且所有引用的 source_id 必须出现在 source_ids 中。\n\n"
|
||||
"只能使用 candidate_sources 或 Local material context 中的来源,不得编造 URL、DOI、trial ID 或 source_id。\n"
|
||||
"输出 JSON 必须包含 sources 字段;sources 只能来自 candidate_sources 或 Local material context。\n"
|
||||
"如 Local material context 非空,必须至少提取 1 条本地材料原文证据;如果与本章无关,必须在 open_questions 说明为什么无关。\n\n"
|
||||
f"{json.dumps(card.to_dict(), ensure_ascii=False, indent=2)}\n\n"
|
||||
f"Local material context:\n{json.dumps(materials, ensure_ascii=False, indent=2)}\n\n"
|
||||
f"Search context:\n{json.dumps(context, ensure_ascii=False, indent=2)}\n\n"
|
||||
"只输出 JSON,不要输出 Markdown 解释。"
|
||||
)
|
||||
|
||||
|
||||
def build_packet_repair_prompt(
|
||||
*,
|
||||
card: TaskCard,
|
||||
raw_response: str,
|
||||
error: Exception,
|
||||
search_context: dict[str, Any] | None = None,
|
||||
material_context: dict[str, Any] | None = None,
|
||||
) -> str:
|
||||
context = search_context or {"routes_used": [], "candidate_sources": []}
|
||||
materials = material_context or {"materials": []}
|
||||
return (
|
||||
"请修复上一次 evidence packet 输出,使其成为合法且通过 schema 校验的 JSON。\n"
|
||||
"只输出 JSON 对象,不要输出 Markdown、解释或代码块。\n"
|
||||
"保留中文主写作;英文只允许出现在来源标题、URL、DOI、原文摘录或检索笔记中。\n"
|
||||
"不得编造 candidate_sources 或 Local material context 以外的来源、URL、DOI、trial ID 或 source_id。\n\n"
|
||||
f"Schema error:\n{error}\n\n"
|
||||
f"Task card:\n{json.dumps(card.to_dict(), ensure_ascii=False, indent=2)}\n\n"
|
||||
f"Search context:\n{json.dumps(context, ensure_ascii=False, indent=2)}\n\n"
|
||||
f"Local material context:\n{json.dumps(materials, ensure_ascii=False, indent=2)}\n\n"
|
||||
f"Previous raw response:\n{raw_response[:12000]}"
|
||||
)
|
||||
|
||||
|
||||
class PacketWorker:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
role: RoleDefinition,
|
||||
client: ChatClient,
|
||||
project_root: Path | None = None,
|
||||
search_provider: SearchProvider | None = None,
|
||||
skill_registry: SkillRegistry | None = None,
|
||||
num_results_per_route: int = 5,
|
||||
) -> None:
|
||||
self.role = role
|
||||
self.client = client
|
||||
self.project_root = project_root
|
||||
self.search_provider = search_provider
|
||||
self.skill_registry = skill_registry or SkillRegistry()
|
||||
self.num_results_per_route = num_results_per_route
|
||||
|
||||
def _system_prompt(self) -> str:
|
||||
skill_texts = []
|
||||
for name in self.role.skills:
|
||||
try:
|
||||
skill_texts.append(f"# Skill: {name}\n\n{self.skill_registry.read(name)}")
|
||||
except FileNotFoundError:
|
||||
skill_texts.append(f"# Skill: {name}\n\n[missing skill: {name}]")
|
||||
return (
|
||||
f"{self.role.identity}\n\n"
|
||||
"你是 Deep Research v0.20 Python runtime 的证据包 worker。\n"
|
||||
"你的唯一任务是把一个 task card 转换为结构化 evidence packet。\n"
|
||||
"遵循中文主写作原则;不要写章节正文;不要编造 URL、DOI、trial ID 或 source_id。\n\n"
|
||||
"搜索只能走项目 Python search gateway 或调用方提供的 search_context;不要直接使用 Tavily MCP、browser MCP、平台 web search 或任何需要用户权限确认的外部搜索工具。\n\n"
|
||||
+ "\n\n".join(skill_texts)
|
||||
)
|
||||
|
||||
def run(self, card: TaskCard) -> dict:
|
||||
search_context = None
|
||||
if self.search_provider:
|
||||
search_context = build_search_context(
|
||||
card,
|
||||
self.search_provider,
|
||||
num_results_per_route=self.num_results_per_route,
|
||||
)
|
||||
material_context = build_material_context(card, self.project_root)
|
||||
raw = self.client.chat_complete(
|
||||
model=self.role.model,
|
||||
system=self._system_prompt(),
|
||||
user=build_packet_user_prompt(card, search_context, material_context),
|
||||
temperature=self.role.temperature,
|
||||
max_tokens=self.role.max_tokens,
|
||||
tag=f"packet:{card.task_id}",
|
||||
)
|
||||
try:
|
||||
packet = normalize_packet_against_context(
|
||||
_extract_json_object(raw),
|
||||
search_context,
|
||||
material_context,
|
||||
)
|
||||
validate_packet(packet)
|
||||
validate_packet_against_allowed_context(packet, search_context, material_context)
|
||||
return packet
|
||||
except Exception as error:
|
||||
repaired = self.client.chat_complete(
|
||||
model=self.role.model,
|
||||
system=self._system_prompt(),
|
||||
user=build_packet_repair_prompt(
|
||||
card=card,
|
||||
raw_response=raw,
|
||||
error=error,
|
||||
search_context=search_context,
|
||||
material_context=material_context,
|
||||
),
|
||||
temperature=0,
|
||||
max_tokens=self.role.max_tokens,
|
||||
tag=f"packet-repair:{card.task_id}",
|
||||
)
|
||||
packet = normalize_packet_against_context(
|
||||
_extract_json_object(repaired),
|
||||
search_context,
|
||||
material_context,
|
||||
)
|
||||
validate_packet(packet)
|
||||
validate_packet_against_allowed_context(packet, search_context, material_context)
|
||||
return packet
|
||||
|
||||
|
||||
def _write_packet_error(project_root: Path, card: TaskCard, error: Exception) -> None:
|
||||
path = project_root / "phase2" / "packet_errors" / f"{card.task_id}.json"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"task_id": card.task_id,
|
||||
"status": "failed",
|
||||
"error": str(error),
|
||||
"output_packet": card.output_packet,
|
||||
}
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def run_packet_workers(
|
||||
*,
|
||||
project_root: Path,
|
||||
cards: list[TaskCard],
|
||||
runtime: RuntimeProfile,
|
||||
client_factory: Callable[[RoleDefinition], ChatClient],
|
||||
search_provider_factory: Callable[[], SearchProvider] | None = None,
|
||||
workers: int,
|
||||
) -> int:
|
||||
role = runtime.role_for_task("evidence_packet")
|
||||
max_workers = max(1, min(workers, role.max_concurrency))
|
||||
|
||||
def run_one(card: TaskCard) -> tuple[TaskCard, dict | None, Exception | None]:
|
||||
search_provider = search_provider_factory() if search_provider_factory else None
|
||||
try:
|
||||
worker = PacketWorker(role=role, client=client_factory(role), project_root=project_root, search_provider=search_provider)
|
||||
return card, worker.run(card), None
|
||||
except Exception as error:
|
||||
return card, None, error
|
||||
finally:
|
||||
close = getattr(search_provider, "close", None)
|
||||
if close:
|
||||
close()
|
||||
|
||||
written = 0
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as pool:
|
||||
futures = [pool.submit(run_one, card) for card in cards]
|
||||
for future in as_completed(futures):
|
||||
card, packet, error = future.result()
|
||||
if error is not None:
|
||||
_write_packet_error(project_root, card, error)
|
||||
continue
|
||||
if packet is None:
|
||||
_write_packet_error(project_root, card, RuntimeError("packet worker returned no packet"))
|
||||
continue
|
||||
path = project_root / card.output_packet
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(packet, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
append_packet_sources(project_root / "phase2" / "sources.jsonl", packet)
|
||||
written += 1
|
||||
return written
|
||||
@@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Unified search gateway for Deep Research agents.
|
||||
|
||||
This script is the stable project-owned entrypoint that agents should call
|
||||
instead of vendor MCP tools. MCP search remains optional, while this gateway
|
||||
keeps routing behavior reproducible across OpenCode, Codex, and future
|
||||
adapters.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from scripts.lib.search_client import SearchClient, SearchError, SearchHit
|
||||
from scripts.lib.zenmux_client import load_secrets
|
||||
|
||||
|
||||
ROUTE_HELP = {
|
||||
"general": "Tavily -> Exa -> Brave generic web discovery",
|
||||
"evidence": "Exa highlights -> Tavily -> Brave controlled evidence discovery",
|
||||
"scholar": "Serper Scholar -> generic fallback",
|
||||
"patents": "Serper Google Patents -> site:patents.google.com fallback",
|
||||
"news": "Serper News -> generic fallback",
|
||||
"fda": "FDA-focused discovery for warning letters, enforcement pages, and meeting materials",
|
||||
}
|
||||
|
||||
PROFILE_ROUTES = {
|
||||
"biomed_literature": ["scholar", "evidence", "general"],
|
||||
"patent_heavy": ["patents", "evidence", "general"],
|
||||
"china_market": ["news", "evidence", "general"],
|
||||
"investment": ["news", "evidence", "general"],
|
||||
}
|
||||
|
||||
PROFILE_QUERY_PREFIX = {
|
||||
"china_market": "(China OR Chinese OR 中国 OR 国内)",
|
||||
}
|
||||
|
||||
|
||||
def search_route(client: SearchClient, route: str, query: str, args: argparse.Namespace) -> list[SearchHit]:
|
||||
if route == "general":
|
||||
return client.search(query, num_results=args.num_results)
|
||||
if route == "evidence":
|
||||
return client.evidence(query, num_results=args.num_results, category=args.exa_category)
|
||||
if route == "scholar":
|
||||
return client.scholar(query, num_results=args.num_results, year_low=args.year_low)
|
||||
if route == "patents":
|
||||
return client.patents(query, num_results=args.num_results)
|
||||
if route == "news":
|
||||
return client.news(query, num_results=args.num_results, time_range=args.time_range)
|
||||
if route == "fda":
|
||||
return client.fda(query, num_results=args.num_results)
|
||||
raise SystemExit(f"unknown route: {route}")
|
||||
|
||||
|
||||
def emit_markdown(route_hits: list[tuple[str, list[SearchHit]]], query: str) -> None:
|
||||
print(f"# Search Results: {query}")
|
||||
for route, hits in route_hits:
|
||||
print()
|
||||
print(f"## Route: {route} ({ROUTE_HELP[route]})")
|
||||
if not hits:
|
||||
print("No results.")
|
||||
continue
|
||||
for i, hit in enumerate(hits, start=1):
|
||||
print(f"{i}. {hit.title or '(untitled)'}")
|
||||
print(f" - URL: {hit.url}")
|
||||
if hit.snippet:
|
||||
print(f" - Snippet: {hit.snippet}")
|
||||
|
||||
|
||||
def emit_json(route_hits: list[tuple[str, list[SearchHit]]], query: str) -> None:
|
||||
data = {
|
||||
"query": query,
|
||||
"routes": [
|
||||
{
|
||||
"route": route,
|
||||
"route_help": ROUTE_HELP[route],
|
||||
"results": [asdict(hit) for hit in hits],
|
||||
}
|
||||
for route, hits in route_hits
|
||||
],
|
||||
}
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
def emit_trace_markdown(route_trace: list[dict[str, str]]) -> None:
|
||||
print()
|
||||
print("## Route Trace")
|
||||
for item in route_trace:
|
||||
print(f"- {item['route']}: {item['status']} ({item['detail']})")
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Deep Research search gateway")
|
||||
parser.add_argument("query", help="Search query")
|
||||
parser.add_argument(
|
||||
"--route",
|
||||
choices=sorted(ROUTE_HELP),
|
||||
default="general",
|
||||
help="Single search route to run",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profile",
|
||||
choices=sorted(PROFILE_ROUTES),
|
||||
help="Run a strategy profile instead of a single route",
|
||||
)
|
||||
parser.add_argument("--num-results", type=int, default=10)
|
||||
parser.add_argument(
|
||||
"--exa-category",
|
||||
choices=["research paper", "news", "company", "financial report", "github", "tweet", "personal site", "pdf"],
|
||||
help="Optional Exa category for the evidence route",
|
||||
)
|
||||
parser.add_argument("--year-low", type=int, help="Lower year bound for scholar searches")
|
||||
parser.add_argument("--time-range", choices=["d", "w", "m", "y"], help="Serper news time range")
|
||||
parser.add_argument("--json", action="store_true", help="Emit JSON instead of Markdown")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Show planned routes without calling APIs")
|
||||
parser.add_argument(
|
||||
"--strict-specialized",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=True,
|
||||
help="Fail fast if scholar/news/patents cannot use Serper",
|
||||
)
|
||||
parser.add_argument("--trace", action="store_true", help="Include route execution trace")
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
load_secrets()
|
||||
|
||||
routes = PROFILE_ROUTES[args.profile] if args.profile else [args.route]
|
||||
query = args.query
|
||||
if args.profile in PROFILE_QUERY_PREFIX:
|
||||
query = f"{PROFILE_QUERY_PREFIX[args.profile]} {query}"
|
||||
|
||||
if args.dry_run:
|
||||
for route in routes:
|
||||
print(f"{route}: {ROUTE_HELP[route]}")
|
||||
if query != args.query:
|
||||
print(f"query_rewritten: {query}")
|
||||
return 0
|
||||
|
||||
try:
|
||||
with SearchClient(strict_specialized=args.strict_specialized) as client:
|
||||
route_hits = []
|
||||
route_trace: list[dict[str, str]] = []
|
||||
for route in routes:
|
||||
try:
|
||||
hits = search_route(client, route, query, args)
|
||||
route_hits.append((route, hits))
|
||||
route_trace.append({"route": route, "status": "ok", "detail": f"hits={len(hits)}"})
|
||||
except SearchError as exc:
|
||||
route_hits.append((route, []))
|
||||
route_trace.append({"route": route, "status": "failed", "detail": str(exc)})
|
||||
if route != "general":
|
||||
continue
|
||||
raise
|
||||
except SearchError as exc:
|
||||
raise SystemExit(f"search failed: {exc}") from exc
|
||||
|
||||
if args.json:
|
||||
data = {
|
||||
"query": query,
|
||||
"original_query": args.query,
|
||||
"strict_specialized": args.strict_specialized,
|
||||
"routes": [
|
||||
{
|
||||
"route": route,
|
||||
"route_help": ROUTE_HELP[route],
|
||||
"results": [asdict(hit) for hit in hits],
|
||||
}
|
||||
for route, hits in route_hits
|
||||
],
|
||||
"trace": route_trace if args.trace else [],
|
||||
}
|
||||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
emit_markdown(route_hits, query)
|
||||
if args.trace:
|
||||
emit_trace_markdown(route_trace)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+147
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env bash
|
||||
# Deep Research - Environment setup (macOS + Debian/Ubuntu, using uv)
|
||||
# Usage: bash scripts/setup.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
CYAN='\033[0;36m'
|
||||
RST='\033[0m'
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
cd "${PROJECT_ROOT}"
|
||||
|
||||
echo -e "${CYAN}============================================${RST}"
|
||||
echo -e "${CYAN} Deep Research - Setup (uv)${RST}"
|
||||
echo -e "${CYAN} Project root: ${PROJECT_ROOT}${RST}"
|
||||
echo -e "${CYAN}============================================${RST}"
|
||||
|
||||
# -------- OS detection --------
|
||||
OS="unknown"
|
||||
case "$(uname -s)" in
|
||||
Darwin) OS="macos" ;;
|
||||
Linux)
|
||||
if [[ -f /etc/debian_version ]]; then
|
||||
OS="debian"
|
||||
elif [[ -f /etc/redhat-release ]]; then
|
||||
OS="rhel"
|
||||
else
|
||||
OS="linux"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
echo -e "${YELLOW}Detected OS: ${OS}${RST}"
|
||||
|
||||
# -------- [1/4] uv check / install --------
|
||||
echo ""
|
||||
echo -e "${CYAN}[1/4] Checking uv...${RST}"
|
||||
|
||||
if ! command -v uv >/dev/null 2>&1; then
|
||||
echo -e " ${YELLOW}uv not found. Installing...${RST}"
|
||||
echo -e " ${YELLOW}(uv = fast Python package manager written in Rust, by Astral)${RST}"
|
||||
echo ""
|
||||
read -r -p " Auto-install uv? [Y/n] " REPLY
|
||||
echo ""
|
||||
if [[ ! ${REPLY} =~ ^[Nn]$ ]]; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# bring uv into current PATH
|
||||
[[ -f "${HOME}/.local/bin/uv" ]] && export PATH="${HOME}/.local/bin:${PATH}"
|
||||
if ! command -v uv >/dev/null 2>&1; then
|
||||
echo -e " ${RED}ERROR: uv still not found after install.${RST}"
|
||||
echo " Add ~/.local/bin to PATH, then re-run:"
|
||||
echo " export PATH=\"\${HOME}/.local/bin:\${PATH}\""
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo -e " ${RED}Manual install:${RST}"
|
||||
echo " macOS: brew install uv"
|
||||
echo " all: curl -LsSf https://astral.sh/uv/install.sh | sh"
|
||||
echo " pip: pip install --user uv"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
UV_VER="$(uv --version 2>&1 | head -1)"
|
||||
echo -e " ${GREEN}OK: ${UV_VER}${RST}"
|
||||
|
||||
# -------- [2/4] Python info --------
|
||||
echo ""
|
||||
echo -e "${CYAN}[2/4] Python check...${RST}"
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
SYS_PY="$(python3 --version 2>&1)"
|
||||
echo -e " ${GREEN}System: ${SYS_PY} (uv manages its own Python 3.10+ automatically)${RST}"
|
||||
else
|
||||
echo -e " ${YELLOW}python3 not found in PATH (uv will download Python automatically)${RST}"
|
||||
fi
|
||||
|
||||
# -------- [3/4] uv sync --------
|
||||
echo ""
|
||||
echo -e "${CYAN}[3/4] Syncing Python dependencies (uv sync)...${RST}"
|
||||
echo -e " ${YELLOW}First run may take a moment to download Python + packages.${RST}"
|
||||
uv sync
|
||||
echo -e " ${GREEN}OK: .venv/ ready${RST}"
|
||||
|
||||
# -------- [4/4] System binaries --------
|
||||
echo ""
|
||||
echo -e "${CYAN}[4/4] Checking system binaries...${RST}"
|
||||
|
||||
check_bin() {
|
||||
local name="${1}"
|
||||
local install_macos="${2}"
|
||||
local install_debian="${3}"
|
||||
if command -v "${name}" >/dev/null 2>&1; then
|
||||
local ver
|
||||
ver="$("${name}" --version 2>&1 | head -1)"
|
||||
echo -e " ${GREEN}OK: ${name} - ${ver}${RST}"
|
||||
else
|
||||
echo -e " ${YELLOW}MISSING: ${name}${RST}"
|
||||
case "${OS}" in
|
||||
macos) echo " Install: ${install_macos}" ;;
|
||||
debian) echo " Install: ${install_debian}" ;;
|
||||
*) echo " Install ${name} for your OS" ;;
|
||||
esac
|
||||
fi
|
||||
}
|
||||
|
||||
check_bin "pandoc" "brew install pandoc" "sudo apt install pandoc"
|
||||
check_bin "opencode" "curl -fsSL https://opencode.ai/install | bash" "curl -fsSL https://opencode.ai/install | bash"
|
||||
check_bin "curl" "pre-installed" "sudo apt install curl"
|
||||
|
||||
# -------- Summary --------
|
||||
echo ""
|
||||
echo -e "${CYAN}============================================${RST}"
|
||||
echo -e "${CYAN} Setup complete${RST}"
|
||||
echo -e "${CYAN}============================================${RST}"
|
||||
echo ""
|
||||
|
||||
if [[ -f secrets.env ]]; then
|
||||
echo -e "${GREEN}OK: secrets.env found${RST}"
|
||||
else
|
||||
echo -e "${YELLOW}TODO: create secrets.env${RST}"
|
||||
echo " cp secrets.env.example secrets.env"
|
||||
echo " # fill in ZENMUX_API_KEY etc."
|
||||
fi
|
||||
|
||||
FONT_COUNT="$(find .opencode/templates/fonts -maxdepth 1 \( -name "*.otf" -o -name "*.ttf" \) 2>/dev/null | wc -l | tr -d ' ')"
|
||||
if [[ "${FONT_COUNT}" -ge 7 ]]; then
|
||||
echo -e "${GREEN}OK: fonts ready (${FONT_COUNT} files)${RST}"
|
||||
else
|
||||
echo -e "${YELLOW}TODO: download fonts (${FONT_COUNT}/7 present)${RST}"
|
||||
echo " bash .opencode/templates/fonts/download-fonts.sh"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo " 1. source scripts/activate.sh"
|
||||
echo " 2. bash .opencode/templates/fonts/download-fonts.sh"
|
||||
echo " 3. bash scripts/verify-zenmux.sh"
|
||||
echo " 4. opencode"
|
||||
echo ""
|
||||
echo "Tips:"
|
||||
echo " Run a script directly: uv run python .opencode/templates/report-template.py ..."
|
||||
echo " Add a package: uv add <package>"
|
||||
echo " Upgrade all: uv lock --upgrade && uv sync"
|
||||
echo " Full rebuild: rm -rf .venv && bash scripts/setup.sh"
|
||||
@@ -0,0 +1,102 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sprint 5 regression checks for v0.12 changes.
|
||||
|
||||
Checks are non-destructive and default to dry-run behavior.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def run(cmd: list[str]) -> tuple[int, str]:
|
||||
proc = subprocess.run(
|
||||
cmd,
|
||||
cwd=REPO_ROOT,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
out = (proc.stdout or "") + (proc.stderr or "")
|
||||
return proc.returncode, out
|
||||
|
||||
|
||||
def check(name: str, cmd: list[str], must_contain: list[str] | None = None) -> bool:
|
||||
print(f"[check] {name}")
|
||||
print(" $ " + " ".join(cmd))
|
||||
rc, out = run(cmd)
|
||||
if rc != 0:
|
||||
print(f" FAIL: exit={rc}")
|
||||
if out.strip():
|
||||
print(" output:")
|
||||
print(" " + out.strip().replace("\n", "\n "))
|
||||
return False
|
||||
for token in must_contain or []:
|
||||
if token not in out:
|
||||
print(f" FAIL: missing token '{token}'")
|
||||
return False
|
||||
print(" PASS")
|
||||
return True
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Run Sprint 5 regression checks")
|
||||
parser.add_argument("project", help="Project slug or path for finalize dry-run")
|
||||
args = parser.parse_args()
|
||||
|
||||
checks = [
|
||||
(
|
||||
"model profiles list",
|
||||
["uv", "run", "python", "scripts/dr.py", "models", "--list"],
|
||||
["medium", "premium", "simple"],
|
||||
),
|
||||
(
|
||||
"search gateway dry-run",
|
||||
[
|
||||
"uv",
|
||||
"run",
|
||||
"python",
|
||||
"scripts/search.py",
|
||||
"GLP-1 obesity",
|
||||
"--profile",
|
||||
"china_market",
|
||||
"--dry-run",
|
||||
],
|
||||
["news:", "general:", "query_rewritten:"],
|
||||
),
|
||||
(
|
||||
"phase4 finalize dry-run",
|
||||
[
|
||||
"uv",
|
||||
"run",
|
||||
"python",
|
||||
"scripts/dr.py",
|
||||
"finalize",
|
||||
args.project,
|
||||
"--model-profile",
|
||||
"medium",
|
||||
"--dry-run",
|
||||
],
|
||||
["Phase 4 pipeline done"],
|
||||
),
|
||||
]
|
||||
|
||||
ok = True
|
||||
for name, cmd, tokens in checks:
|
||||
ok = check(name, cmd, tokens) and ok
|
||||
|
||||
if not ok:
|
||||
print("\nSprint 5 regression: FAILED")
|
||||
return 1
|
||||
print("\nSprint 5 regression: PASSED")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,359 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4 英译中:章节级切块 → 逐块翻译 → 拼接 → 写盘。
|
||||
|
||||
用法:
|
||||
uv run python scripts/translate.py <project_slug>
|
||||
# 或:
|
||||
uv run python scripts/translate.py projects/dual-target-rnai-pipeline-2026
|
||||
|
||||
断点续传:每块翻译完立即写入 `phase4/zh_chunks/<anchor>.md`,术语表 patch 在本轮结束后统一合并。
|
||||
重跑时已存在的块直接跳过,只译缺的。
|
||||
|
||||
设计要点:
|
||||
1. 切块按 H2 粒度,单块一般 <600 英文词,单次 API 调用远低于 Sonnet output token 上限
|
||||
2. 并发翻译使用稳定术语表快照,译完后统一合并 glossary patch,避免多线程写冲突
|
||||
3. 失败不会污染最终产物:块级文件独立,可重跑;汇总步骤独立
|
||||
4. 日志完整:每次 API 调用写 `phase4/logs/translate.jsonl`
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from scripts.lib.markdown_chunker import (
|
||||
MarkdownBlock,
|
||||
count_chinese_chars,
|
||||
split_by_headers,
|
||||
)
|
||||
from scripts.lib.zenmux_client import ZenMuxClient, ZenMuxError, load_secrets
|
||||
|
||||
DEFAULT_MODEL = "anthropic/claude-sonnet-4.6"
|
||||
MODEL_MAX_TOKENS = {
|
||||
"anthropic/claude-sonnet-4.6": 32000,
|
||||
"anthropic/claude-sonnet-4.5": 32000,
|
||||
"anthropic/claude-opus-4.7": 32000,
|
||||
"anthropic/claude-opus-4.6": 32000,
|
||||
"anthropic/claude-haiku-4.5": 16000,
|
||||
}
|
||||
PROMPT_FILE = Path(__file__).parent / "prompts" / "translate_system.txt"
|
||||
|
||||
|
||||
def resolve_project(arg: str) -> Path:
|
||||
p = Path(arg)
|
||||
if p.is_dir():
|
||||
return p
|
||||
here = Path.cwd()
|
||||
cand = here / "projects" / arg
|
||||
if cand.is_dir():
|
||||
return cand
|
||||
raise SystemExit(f"project not found: {arg}")
|
||||
|
||||
|
||||
def parse_delimited_response(text: str) -> tuple[str, dict[str, str]]:
|
||||
"""解析自定义分隔符格式的响应。
|
||||
|
||||
期望结构:
|
||||
<<<TRANSLATION>>>
|
||||
...markdown...
|
||||
<<<END_TRANSLATION>>>
|
||||
<<<GLOSSARY_PATCH>>>
|
||||
English || 中文
|
||||
...
|
||||
<<<END_GLOSSARY_PATCH>>>
|
||||
|
||||
Returns:
|
||||
(translation, glossary_patch)
|
||||
"""
|
||||
t_start = text.find("<<<TRANSLATION>>>")
|
||||
t_end = text.find("<<<END_TRANSLATION>>>")
|
||||
if t_start == -1 or t_end == -1 or t_end <= t_start:
|
||||
raise ValueError(
|
||||
f"missing <<<TRANSLATION>>> markers in response: {text[:300]}"
|
||||
)
|
||||
translation = text[t_start + len("<<<TRANSLATION>>>"): t_end].strip("\r\n")
|
||||
|
||||
g_start = text.find("<<<GLOSSARY_PATCH>>>")
|
||||
g_end = text.find("<<<END_GLOSSARY_PATCH>>>")
|
||||
patch: dict[str, str] = {}
|
||||
if g_start != -1 and g_end != -1 and g_end > g_start:
|
||||
body = text[g_start + len("<<<GLOSSARY_PATCH>>>"): g_end]
|
||||
for line in body.splitlines():
|
||||
line = line.strip()
|
||||
if not line or "||" not in line:
|
||||
continue
|
||||
en, _, zh = line.partition("||")
|
||||
en, zh = en.strip(), zh.strip()
|
||||
if en and zh:
|
||||
patch[en] = zh
|
||||
return translation, patch
|
||||
|
||||
|
||||
def load_glossary(path: Path) -> dict[str, str]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def save_glossary(path: Path, glossary: dict[str, str]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(glossary, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _glossary_value(value) -> str:
|
||||
if isinstance(value, dict):
|
||||
return str(value.get("zh") or value.get("中文") or "")
|
||||
return str(value)
|
||||
|
||||
|
||||
def build_user_prompt(block: MarkdownBlock, glossary: dict[str, str]) -> str:
|
||||
glossary_lines = (
|
||||
f"{en} || {_glossary_value(zh)}"
|
||||
for en, zh in sorted(glossary.items())
|
||||
if _glossary_value(zh)
|
||||
)
|
||||
glossary_hint = "\n".join(glossary_lines) or "(none yet)"
|
||||
level_hint = (
|
||||
f"H{block.level}" if block.level >= 1 else "frontmatter (no heading)"
|
||||
)
|
||||
return (
|
||||
"# GLOSSARY (English || Chinese, already used earlier in the document):\n"
|
||||
f"{glossary_hint}\n\n"
|
||||
f"# BLOCK TO TRANSLATE (Markdown, {level_hint}):\n"
|
||||
"Translate the block between the markers below into Chinese, "
|
||||
"following all rules in the system prompt. Output using the exact "
|
||||
"delimiter format specified.\n\n"
|
||||
"----- BEGIN BLOCK -----\n"
|
||||
f"{block.content}\n"
|
||||
"----- END BLOCK -----\n"
|
||||
)
|
||||
|
||||
|
||||
def translate_block(
|
||||
client: ZenMuxClient,
|
||||
block: MarkdownBlock,
|
||||
glossary: dict[str, str],
|
||||
*,
|
||||
model: str,
|
||||
system_prompt: str,
|
||||
temperature: float,
|
||||
) -> tuple[str, dict[str, str]]:
|
||||
user = build_user_prompt(block, glossary)
|
||||
max_tok = MODEL_MAX_TOKENS.get(model, 16000)
|
||||
raw = client.chat_complete(
|
||||
model=model,
|
||||
system=system_prompt,
|
||||
user=user,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tok,
|
||||
tag=f"translate:{block.anchor}",
|
||||
)
|
||||
try:
|
||||
translation, patch = parse_delimited_response(raw)
|
||||
except Exception as e:
|
||||
raise RuntimeError(
|
||||
f"bad response format for block {block.anchor}: {e}\nraw head: {raw[:300]}"
|
||||
)
|
||||
if not translation.strip():
|
||||
raise RuntimeError(f"empty translation for block {block.anchor}")
|
||||
return translation.rstrip(), patch
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Phase 4 英译中(章节级切块并行翻译)")
|
||||
parser.add_argument("project", help="项目 slug 或完整路径")
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
default="phase4/final_en.md",
|
||||
help="英文源文件(相对项目根,默认 phase4/final_en.md)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default="phase4/final_zh.md",
|
||||
help="中文输出(默认 phase4/final_zh.md)",
|
||||
)
|
||||
parser.add_argument("--model", default=DEFAULT_MODEL, help="翻译模型")
|
||||
parser.add_argument("--temperature", type=float, default=0.3)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="忽略已有 zh_chunks 缓存,强制重翻",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--only",
|
||||
default=None,
|
||||
help="只翻译指定 order(逗号分隔的整数),其余跳过(调试用),例如 --only 0,1,7",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=None,
|
||||
help="最多翻译前 N 个未缓存的块(调试用)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--workers",
|
||||
type=int,
|
||||
default=4,
|
||||
help="并发翻译 worker 数(默认 4;设为 1 回退串行)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
load_secrets()
|
||||
project_root = resolve_project(args.project)
|
||||
src_path = project_root / args.source
|
||||
out_path = project_root / args.output
|
||||
if not src_path.exists():
|
||||
raise SystemExit(f"source not found: {src_path}")
|
||||
|
||||
chunks_dir = project_root / "phase4" / "zh_chunks"
|
||||
chunks_dir.mkdir(parents=True, exist_ok=True)
|
||||
glossary_path = project_root / "phase4" / "glossary.json"
|
||||
logs_dir = project_root / "phase4" / "logs"
|
||||
log_file = logs_dir / "translate.jsonl"
|
||||
|
||||
system_prompt = PROMPT_FILE.read_text(encoding="utf-8")
|
||||
text = src_path.read_text(encoding="utf-8")
|
||||
blocks = split_by_headers(text, max_level=2)
|
||||
glossary = load_glossary(glossary_path)
|
||||
|
||||
only_orders: set[int] | None = None
|
||||
if args.only:
|
||||
only_orders = {int(a.strip()) for a in args.only.split(",") if a.strip()}
|
||||
|
||||
total_en_words = sum(b.word_count for b in blocks)
|
||||
print(f"Source: {src_path.relative_to(project_root)}")
|
||||
print(f"Blocks: {len(blocks)} | total English words: {total_en_words:,}")
|
||||
print(f"Glossary loaded: {len(glossary)} terms")
|
||||
workers = max(1, args.workers)
|
||||
print(f"Model: {args.model} | temperature: {args.temperature} | workers: {workers}")
|
||||
print()
|
||||
|
||||
start = time.time()
|
||||
todo: list[MarkdownBlock] = []
|
||||
cached_count = 0
|
||||
for b in blocks:
|
||||
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
|
||||
if only_orders is not None and b.order not in only_orders:
|
||||
continue
|
||||
if chunk_path.exists() and not args.force:
|
||||
cached_count += 1
|
||||
print(f" [ok ] #{b.order:03d} {b.short_title} (cached)")
|
||||
continue
|
||||
if args.limit is not None and len(todo) >= args.limit:
|
||||
continue
|
||||
todo.append(b)
|
||||
|
||||
if todo:
|
||||
print(f"To translate this run: {len(todo)} blocks | cached: {cached_count}")
|
||||
print()
|
||||
|
||||
successful_patches: list[dict[str, str]] = []
|
||||
with ZenMuxClient(log_file=log_file) as client:
|
||||
def run_one(b: MarkdownBlock) -> tuple[MarkdownBlock, str, dict[str, str], float]:
|
||||
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
|
||||
t0 = time.time()
|
||||
translation, patch = translate_block(
|
||||
client,
|
||||
b,
|
||||
glossary,
|
||||
model=args.model,
|
||||
system_prompt=system_prompt,
|
||||
temperature=args.temperature,
|
||||
)
|
||||
chunk_path.write_text(translation + "\n", encoding="utf-8")
|
||||
return b, translation, patch, time.time() - t0
|
||||
|
||||
if workers == 1:
|
||||
for b in todo:
|
||||
label = f"#{b.order:03d} L{b.level} {b.word_count:>4}w {b.short_title}"
|
||||
print(f" [... ] {label} ", end="", flush=True)
|
||||
try:
|
||||
_block, translation, patch, elapsed = run_one(b)
|
||||
except (ZenMuxError, RuntimeError) as e:
|
||||
print(f"\n [FAIL] {label}\n {e}")
|
||||
continue
|
||||
successful_patches.append(patch)
|
||||
cn = count_chinese_chars(translation)
|
||||
print(f"\r [done] {label} → {cn:>4}字 ({elapsed:4.1f}s, +{len(patch)} terms)")
|
||||
else:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
futures = {pool.submit(run_one, b): b for b in todo}
|
||||
for fut in concurrent.futures.as_completed(futures):
|
||||
b = futures[fut]
|
||||
label = f"#{b.order:03d} L{b.level} {b.word_count:>4}w {b.short_title}"
|
||||
try:
|
||||
_block, translation, patch, elapsed = fut.result()
|
||||
except (ZenMuxError, RuntimeError) as e:
|
||||
print(f" [FAIL] {label}\n {e}")
|
||||
continue
|
||||
successful_patches.append(patch)
|
||||
cn = count_chinese_chars(translation)
|
||||
print(f" [done] {label} → {cn:>4}字 ({elapsed:4.1f}s, +{len(patch)} terms)")
|
||||
|
||||
glossary_conflicts: list[tuple[str, str, str]] = []
|
||||
for patch in successful_patches:
|
||||
for k, v in patch.items():
|
||||
if k not in glossary:
|
||||
glossary[k] = v
|
||||
elif _glossary_value(glossary[k]) != v:
|
||||
glossary_conflicts.append((k, _glossary_value(glossary[k]), v))
|
||||
if successful_patches:
|
||||
save_glossary(glossary_path, glossary)
|
||||
if glossary_conflicts:
|
||||
print(f" [WARN] glossary patch 冲突 {len(glossary_conflicts)} 条,保留既有译法")
|
||||
|
||||
# 汇总:按 order 拼接所有 chunk
|
||||
merged: list[str] = []
|
||||
missing: list[str] = []
|
||||
for b in blocks:
|
||||
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
|
||||
if not chunk_path.exists():
|
||||
missing.append(f"#{b.order:03d} {b.short_title}")
|
||||
continue
|
||||
merged.append(chunk_path.read_text(encoding="utf-8").rstrip())
|
||||
|
||||
partial = only_orders is not None or args.limit is not None
|
||||
if missing:
|
||||
print(f"\n⚠ 缺失 {len(missing)} 块:")
|
||||
for m in missing[:10]:
|
||||
print(f" - {m}")
|
||||
if len(missing) > 10:
|
||||
print(f" ... 还有 {len(missing) - 10} 块")
|
||||
if partial:
|
||||
print("(partial 模式:--only / --limit 生效,未生成 final_zh.md)")
|
||||
else:
|
||||
print("重新运行本脚本即可补译(已译的会跳过)。")
|
||||
print(client.usage.summary())
|
||||
return 1
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text("\n\n".join(merged) + "\n", encoding="utf-8")
|
||||
|
||||
# 统计
|
||||
final_text = out_path.read_text(encoding="utf-8")
|
||||
cn = count_chinese_chars(final_text)
|
||||
total_time = time.time() - start
|
||||
print()
|
||||
print(f"✓ 输出:{out_path.relative_to(project_root)}")
|
||||
print(f" 中文字数: {cn:,}")
|
||||
print(f" 英文词数源: {total_en_words:,} 膨胀率: {cn / max(total_en_words,1):.2f}×")
|
||||
print(f" 耗时: {total_time:.1f}s")
|
||||
print(f" 术语表: {len(glossary)} 条 → {glossary_path.relative_to(project_root)}")
|
||||
print(client.usage.summary())
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Pull the repo and rebuild isolated platform workspaces at repository root."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SRC_ROOT = Path(__file__).resolve().parent.parent
|
||||
REPO_ROOT = SRC_ROOT.parent
|
||||
ADAPTER_ROOT = SRC_ROOT / "platform_adapters"
|
||||
|
||||
COMMON_FILES = [
|
||||
"AGENTS.md",
|
||||
"GEMINI.md",
|
||||
"CLAUDE.md",
|
||||
"PLAN.md",
|
||||
"docs/antigravity-clean-workspace.md",
|
||||
"docs/platform-adapters.md",
|
||||
"docs/platform-branch-strategy.md",
|
||||
"scripts/dr.py",
|
||||
"scripts/deploy_adapters.py",
|
||||
"scripts/update_platform_envs.py",
|
||||
"scripts/export_antigravity_workspace.py",
|
||||
"pyproject.toml",
|
||||
"requirements.txt",
|
||||
"uv.lock",
|
||||
]
|
||||
|
||||
COMMON_DIRS = [
|
||||
"configs",
|
||||
"scripts/runtime",
|
||||
"scripts/reporting",
|
||||
"skills",
|
||||
]
|
||||
|
||||
|
||||
def run_pull() -> None:
|
||||
subprocess.run(["git", "pull", "--ff-only"], cwd=REPO_ROOT, check=True)
|
||||
|
||||
|
||||
def reset_dir(path: Path) -> None:
|
||||
if path.exists():
|
||||
shutil.rmtree(path)
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def copy_file(rel: str, target_root: Path) -> None:
|
||||
src = SRC_ROOT / rel
|
||||
if not src.exists():
|
||||
raise FileNotFoundError(src)
|
||||
dst = target_root / rel
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dst)
|
||||
|
||||
|
||||
def copy_dir(src: Path, dst: Path) -> None:
|
||||
if not src.exists():
|
||||
raise FileNotFoundError(src)
|
||||
if dst.exists():
|
||||
shutil.rmtree(dst)
|
||||
shutil.copytree(src, dst, ignore=shutil.ignore_patterns("__pycache__", ".pytest_cache"))
|
||||
|
||||
|
||||
def copy_common_workspace(target: Path) -> None:
|
||||
for rel in COMMON_FILES:
|
||||
copy_file(rel, target)
|
||||
for rel in COMMON_DIRS:
|
||||
copy_dir(SRC_ROOT / rel, target / rel)
|
||||
(target / "projects").mkdir(parents=True, exist_ok=True)
|
||||
(target / "projects" / ".gitkeep").touch(exist_ok=True)
|
||||
|
||||
|
||||
def build_antigravity() -> Path:
|
||||
target = REPO_ROOT / "antigravity"
|
||||
reset_dir(target)
|
||||
copy_common_workspace(target)
|
||||
copy_dir(ADAPTER_ROOT / "antigravity" / "agent", target / ".agent")
|
||||
copy_file("platform_adapters/antigravity/README.md", target)
|
||||
return target
|
||||
|
||||
|
||||
def build_codex() -> Path:
|
||||
target = REPO_ROOT / "codex"
|
||||
reset_dir(target)
|
||||
copy_common_workspace(target)
|
||||
copy_dir(ADAPTER_ROOT / "codex", target / ".codex")
|
||||
copy_dir(ADAPTER_ROOT / "antigravity" / "agent" / "skills", target / ".codex" / "skills")
|
||||
copy_file("platform_adapters/codex/README.md", target)
|
||||
return target
|
||||
|
||||
|
||||
def build_opencode() -> Path:
|
||||
target = REPO_ROOT / "opencode"
|
||||
reset_dir(target)
|
||||
copy_common_workspace(target)
|
||||
copy_dir(ADAPTER_ROOT / "opencode", target / ".opencode")
|
||||
copy_file("platform_adapters/opencode/README.md", target)
|
||||
return target
|
||||
|
||||
|
||||
def build_claude_code() -> Path:
|
||||
target = REPO_ROOT / "claude-code"
|
||||
reset_dir(target)
|
||||
copy_common_workspace(target)
|
||||
copy_dir(ADAPTER_ROOT / "claude-code", target / ".claude")
|
||||
copy_file("platform_adapters/claude-code/README.md", target)
|
||||
return target
|
||||
|
||||
|
||||
def build_gemini_cli() -> Path:
|
||||
target = REPO_ROOT / "gemini-cli"
|
||||
reset_dir(target)
|
||||
copy_common_workspace(target)
|
||||
copy_dir(ADAPTER_ROOT / "gemini-cli", target / ".gemini")
|
||||
copy_file("platform_adapters/gemini-cli/README.md", target)
|
||||
return target
|
||||
|
||||
|
||||
BUILDERS = {
|
||||
"antigravity": build_antigravity,
|
||||
"codex": build_codex,
|
||||
"opencode": build_opencode,
|
||||
"claude-code": build_claude_code,
|
||||
"gemini-cli": build_gemini_cli,
|
||||
}
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Update isolated Deep Research platform environments")
|
||||
parser.add_argument(
|
||||
"--platform",
|
||||
choices=["all", *BUILDERS.keys()],
|
||||
default="all",
|
||||
help="which platform env to rebuild",
|
||||
)
|
||||
parser.add_argument("--skip-pull", action="store_true", help="do not run git pull --ff-only first")
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
if not args.skip_pull:
|
||||
run_pull()
|
||||
|
||||
names = list(BUILDERS) if args.platform == "all" else [args.platform]
|
||||
for name in names:
|
||||
target = BUILDERS[name]()
|
||||
print(f"rebuilt {name}: {target}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""v0.20 Python-core regression checks."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def run(cmd: list[str]) -> None:
|
||||
print("$ " + " ".join(cmd))
|
||||
result = subprocess.run(cmd, cwd=REPO_ROOT, check=False, text=True, capture_output=True)
|
||||
if result.stdout:
|
||||
print(result.stdout.rstrip())
|
||||
if result.stderr:
|
||||
print(result.stderr.rstrip(), file=sys.stderr)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(result.returncode)
|
||||
|
||||
|
||||
def make_fixture(root: Path) -> Path:
|
||||
project = root / "v020-fixture"
|
||||
(project / "phase4").mkdir(parents=True, exist_ok=True)
|
||||
(project / "phase4" / "final_zh.md").write_text(
|
||||
"# v0.20 回归测试报告\n\n正文引用占位。\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return project
|
||||
|
||||
|
||||
def write_fixture_packet(project: Path) -> None:
|
||||
cards = json.loads((project / "phase2" / "task_cards.json").read_text(encoding="utf-8"))
|
||||
for index, card in enumerate(cards, start=1):
|
||||
source_a = f"src_{index:03d}_a"
|
||||
source_b = f"src_{index:03d}_b"
|
||||
packet = {
|
||||
"task_id": card["task_id"],
|
||||
"claims": [{"claim": f"{card['task_id']} 回归证据支持章节主线", "source_ids": [source_a]}],
|
||||
"evidence_items": [{"source_id": source_a, "summary": "权威来源支持该判断"}],
|
||||
"counter_evidence": [{"claim": "证据仍需更多来源交叉验证", "source_ids": [source_b]}],
|
||||
"source_ids": [source_a, source_b],
|
||||
"source_quality_notes": [f"{source_a} Tier 1", f"{source_b} Tier 2"],
|
||||
"open_questions": ["需要在真实项目中补充更多来源。"],
|
||||
"raw_quotes_or_notes": ["English raw note can remain here."],
|
||||
"sources": [
|
||||
{"id": source_a, "title": f"Regression Source {index}A", "url": f"https://example.com/source-{index}-a", "tier": 1},
|
||||
{"id": source_b, "title": f"Regression Source {index}B", "url": f"https://example.com/source-{index}-b", "tier": 2},
|
||||
],
|
||||
}
|
||||
packet_path = project / card["output_packet"]
|
||||
packet_path.write_text(json.dumps(packet, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
python = sys.executable
|
||||
run([python, "scripts/dr.py", "skills", "validate"])
|
||||
run([python, "scripts/dr.py", "models", "--profile", "medium", "--json"])
|
||||
with tempfile.TemporaryDirectory(prefix="deep-research-v020-") as tmp:
|
||||
tmp_root = Path(tmp)
|
||||
run(
|
||||
[
|
||||
python,
|
||||
"scripts/dr.py",
|
||||
"init",
|
||||
"v0.20 fixture",
|
||||
"--slug",
|
||||
"v020-fixture",
|
||||
"--projects-dir",
|
||||
str(tmp_root),
|
||||
"--method",
|
||||
"mckinsey_market",
|
||||
]
|
||||
)
|
||||
project = make_fixture(tmp_root)
|
||||
run([python, "scripts/dr.py", "frame", str(project)])
|
||||
assert (project / "phase1" / "research_brief.json").exists()
|
||||
run([python, "scripts/dr.py", "approve", str(project)])
|
||||
run([python, "scripts/dr.py", "research", str(project), "--workers", "2", "--axis", "literature", "--dry-run"])
|
||||
run([python, "scripts/dr.py", "research", str(project), "--workers", "2", "--axis", "literature"])
|
||||
write_fixture_packet(project)
|
||||
run([python, "scripts/dr.py", "research", str(project), "--workers", "2", "--axis", "literature", "--build-briefs"])
|
||||
assert (project / "phase2" / "compressed_findings" / "ch01.json").exists()
|
||||
run([python, "scripts/dr.py", "review", str(project)])
|
||||
run([python, "scripts/dr.py", "finalize", str(project), "--dry-run"])
|
||||
print("v0.20 regression PASS")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+113
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env bash
|
||||
# 验证 zenmux 两个端点 + prompt cache 是否可用
|
||||
# 用法:bash scripts/verify-zenmux.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# 加载密钥
|
||||
if [[ -f secrets.env ]]; then
|
||||
set -a; source secrets.env; set +a
|
||||
fi
|
||||
|
||||
if [[ -z "${ZENMUX_API_KEY:-}" ]]; then
|
||||
echo "❌ ZENMUX_API_KEY 未设置。请 cp secrets.env.example secrets.env 并填入 key"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "============================================"
|
||||
echo " zenmux 端点与 prompt cache 自检"
|
||||
echo "============================================"
|
||||
|
||||
# -------- 测试 1:OpenAI 兼容端点(Gemini) --------
|
||||
echo ""
|
||||
echo "[1/3] 测试 OpenAI 兼容端点(google/gemini-2.5-pro,隐式缓存)..."
|
||||
RESP1=$(curl -sS -w "\n---HTTP:%{http_code}---\n" -X POST "https://zenmux.ai/api/v1/chat/completions" \
|
||||
-H "Authorization: Bearer $ZENMUX_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "google/gemini-2.5-pro",
|
||||
"messages": [{"role": "user", "content": "用一句话回答:PubMed 是什么?"}],
|
||||
"max_tokens": 100
|
||||
}')
|
||||
echo "$RESP1" | tail -5
|
||||
echo ""
|
||||
|
||||
# -------- 测试 2:Anthropic 兼容端点(Claude Haiku) --------
|
||||
echo "[2/3] 测试 Anthropic 兼容端点(claude-haiku-4.5)..."
|
||||
RESP2=$(curl -sS -w "\n---HTTP:%{http_code}---\n" -X POST "https://zenmux.ai/api/anthropic/v1/messages" \
|
||||
-H "x-api-key: $ZENMUX_API_KEY" \
|
||||
-H "anthropic-version: 2023-06-01" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "claude-haiku-4.5",
|
||||
"max_tokens": 100,
|
||||
"messages": [{"role": "user", "content": "用一句话回答:PubMed 是什么?"}]
|
||||
}')
|
||||
echo "$RESP2" | tail -5
|
||||
echo ""
|
||||
|
||||
# -------- 测试 3:Claude + prompt cache(Sonnet) --------
|
||||
echo "[3/3] 测试 Claude prompt cache(连续 2 次请求,第 2 次应命中 cache)..."
|
||||
|
||||
# 构造一个至少 1024 tokens 的 system prompt(约 3000 字符中文足够)
|
||||
LONG_PROMPT=$(printf 'You are a biomedical research assistant. ' && \
|
||||
for i in $(seq 1 80); do
|
||||
printf 'This is test content line %d to build cacheable prefix context. ' "$i"
|
||||
done)
|
||||
|
||||
PAYLOAD=$(cat <<EOF
|
||||
{
|
||||
"model": "claude-haiku-4.5",
|
||||
"max_tokens": 50,
|
||||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "${LONG_PROMPT}",
|
||||
"cache_control": {"type": "ephemeral"}
|
||||
}
|
||||
],
|
||||
"messages": [{"role": "user", "content": "Say OK."}]
|
||||
}
|
||||
EOF
|
||||
)
|
||||
|
||||
echo " 第 1 次调用(建 cache)..."
|
||||
R1=$(curl -sS -X POST "https://zenmux.ai/api/anthropic/v1/messages" \
|
||||
-H "x-api-key: $ZENMUX_API_KEY" \
|
||||
-H "anthropic-version: 2023-06-01" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$PAYLOAD")
|
||||
CACHE_CREATE=$(echo "$R1" | python3 -c "import sys, json; d=json.load(sys.stdin); print(d.get('usage',{}).get('cache_creation_input_tokens', 0))" 2>/dev/null || echo "?")
|
||||
echo " cache_creation_input_tokens = $CACHE_CREATE"
|
||||
|
||||
sleep 2
|
||||
|
||||
echo " 第 2 次调用(应命中 cache)..."
|
||||
R2=$(curl -sS -X POST "https://zenmux.ai/api/anthropic/v1/messages" \
|
||||
-H "x-api-key: $ZENMUX_API_KEY" \
|
||||
-H "anthropic-version: 2023-06-01" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$PAYLOAD")
|
||||
CACHE_READ=$(echo "$R2" | python3 -c "import sys, json; d=json.load(sys.stdin); print(d.get('usage',{}).get('cache_read_input_tokens', 0))" 2>/dev/null || echo "?")
|
||||
echo " cache_read_input_tokens = $CACHE_READ"
|
||||
|
||||
echo ""
|
||||
echo "============================================"
|
||||
echo " 结果"
|
||||
echo "============================================"
|
||||
|
||||
if [[ "$CACHE_CREATE" != "0" && "$CACHE_CREATE" != "?" ]]; then
|
||||
echo "✅ Cache 写入成功($CACHE_CREATE tokens)"
|
||||
else
|
||||
echo "⚠️ Cache 未写入(可能是 prompt 太短或端点不支持)"
|
||||
fi
|
||||
|
||||
if [[ "$CACHE_READ" != "0" && "$CACHE_READ" != "?" ]]; then
|
||||
echo "✅ Cache 命中成功($CACHE_READ tokens)— 成本大幅降低"
|
||||
else
|
||||
echo "⚠️ Cache 未命中(检查 AGENTS.md §6.5 诊断步骤)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "完整 usage 字段(第 2 次):"
|
||||
echo "$R2" | python3 -m json.tool 2>/dev/null | grep -A 10 '"usage"' || echo "$R2"
|
||||
Reference in New Issue
Block a user