v0.6-wip: Python-based Phase 4 translation pipeline
架构变更：把 dr-translator 从 opencode agent 降级为 Python 脚本编排下的 LLM
调用。根本原因是 agent 一次性处理 19k 英文词整文，单次 output token 接近
Sonnet 4.6 上限（~32k），多次重跑都卡在同一个坑里——问题是架构本身，不是
prompt。
新架构：
scripts/lib/zenmux_client.py HTTP 客户端，指数退避重试、token 统计
JSONL 日志、secrets.env 自动加载
scripts/lib/markdown_chunker.py 按 H1/H2 切块，稳定 anchor ID（order+title
sha1），支持合并/统计
scripts/prompts/translate_system.txt 英译中 prompt，用自定义 <<<TRANSLATION>>>
分隔符格式（规避 Markdown-in-JSON 问题）
scripts/prompts/polish_system.txt 中文润色 prompt（留给下一步 polish.py）
scripts/translate.py 主入口：章节级切块 → 逐块翻译 → 拼接
关键设计：
- 0 依赖 LLM 遵从性：Python 控制切块/循环/重试，LLM 只做单块翻译
- 断点续传：每块翻译完立即写 phase4/zh_chunks/<order>-<anchor>.md
- 术语表累积：每块的 glossary_patch 合并回 phase4/glossary.json
- 失败隔离：单块失败不影响其他块，重跑只补缺
- 调试友好：--only N,M / --limit K / --force
实测（dual-target-rnai-pipeline-2026）：
- 63 块全部成功，17 分钟，$1.70
- 33,441 中文字（符合"研究类 ≥30,000 字"硬标准）
- 310 条双语术语
- 翻译质量：接近母语咨询分析师写作
下一步：polish.py（按 H2 section 润色）、merge_chapters.py（从 phase2/drafts
合并生成 final_en.md）、重构 dr-editor-in-chief 调度脚本、更新 /dr-finalize。
Co-authored-by: User <human>
This commit is contained in:
kai
@@ -0,0 +1,310 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4 英译中：章节级切块 → 逐块翻译 → 拼接 → 写盘。
|
||||
|
||||
用法：
|
||||
uv run python scripts/translate.py <project_slug>
|
||||
# 或：
|
||||
uv run python scripts/translate.py projects/dual-target-rnai-pipeline-2026
|
||||
|
||||
断点续传：每块翻译完立即写入 `phase4/zh_chunks/<anchor>.md` 和术语表 patch。
|
||||
重跑时已存在的块直接跳过，只译缺的。
|
||||
|
||||
设计要点：
|
||||
1. 切块按 H2 粒度，单块一般 <600 英文词，单次 API 调用远低于 Sonnet output token 上限
|
||||
2. 术语表累积式更新：每块调用传入当前已知术语，译完回写 patch，保证全文一致
|
||||
3. 失败不会污染最终产物：块级文件独立，可重跑；汇总步骤独立
|
||||
4. 日志完整：每次 API 调用写 `phase4/logs/translate.jsonl`
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from scripts.lib.markdown_chunker import (
|
||||
MarkdownBlock,
|
||||
count_chinese_chars,
|
||||
merge_blocks,
|
||||
split_by_headers,
|
||||
)
|
||||
from scripts.lib.zenmux_client import ZenMuxClient, ZenMuxError, load_secrets
|
||||
|
||||
DEFAULT_MODEL = "anthropic/claude-sonnet-4.6"
|
||||
MODEL_MAX_TOKENS = {
|
||||
"anthropic/claude-sonnet-4.6": 32000,
|
||||
"anthropic/claude-sonnet-4.5": 32000,
|
||||
"anthropic/claude-opus-4.7": 32000,
|
||||
"anthropic/claude-opus-4.6": 32000,
|
||||
"anthropic/claude-haiku-4.5": 16000,
|
||||
}
|
||||
PROMPT_FILE = Path(__file__).parent / "prompts" / "translate_system.txt"
|
||||
|
||||
|
||||
def resolve_project(arg: str) -> Path:
|
||||
p = Path(arg)
|
||||
if p.is_dir():
|
||||
return p
|
||||
here = Path.cwd()
|
||||
cand = here / "projects" / arg
|
||||
if cand.is_dir():
|
||||
return cand
|
||||
raise SystemExit(f"project not found: {arg}")
|
||||
|
||||
|
||||
def parse_delimited_response(text: str) -> tuple[str, dict[str, str]]:
|
||||
"""解析自定义分隔符格式的响应。
|
||||
|
||||
期望结构：
|
||||
<<<TRANSLATION>>>
|
||||
...markdown...
|
||||
<<<END_TRANSLATION>>>
|
||||
<<<GLOSSARY_PATCH>>>
|
||||
English || 中文
|
||||
...
|
||||
<<<END_GLOSSARY_PATCH>>>
|
||||
|
||||
Returns:
|
||||
(translation, glossary_patch)
|
||||
"""
|
||||
t_start = text.find("<<<TRANSLATION>>>")
|
||||
t_end = text.find("<<<END_TRANSLATION>>>")
|
||||
if t_start == -1 or t_end == -1 or t_end <= t_start:
|
||||
raise ValueError(
|
||||
f"missing <<<TRANSLATION>>> markers in response: {text[:300]}"
|
||||
)
|
||||
translation = text[t_start + len("<<<TRANSLATION>>>"): t_end].strip("\r\n")
|
||||
|
||||
g_start = text.find("<<<GLOSSARY_PATCH>>>")
|
||||
g_end = text.find("<<<END_GLOSSARY_PATCH>>>")
|
||||
patch: dict[str, str] = {}
|
||||
if g_start != -1 and g_end != -1 and g_end > g_start:
|
||||
body = text[g_start + len("<<<GLOSSARY_PATCH>>>"): g_end]
|
||||
for line in body.splitlines():
|
||||
line = line.strip()
|
||||
if not line or "||" not in line:
|
||||
continue
|
||||
en, _, zh = line.partition("||")
|
||||
en, zh = en.strip(), zh.strip()
|
||||
if en and zh:
|
||||
patch[en] = zh
|
||||
return translation, patch
|
||||
|
||||
|
||||
def load_glossary(path: Path) -> dict[str, str]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def save_glossary(path: Path, glossary: dict[str, str]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(glossary, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def build_user_prompt(block: MarkdownBlock, glossary: dict[str, str]) -> str:
|
||||
glossary_hint = (
|
||||
"\n".join(f"{en} || {zh}" for en, zh in sorted(glossary.items()))
|
||||
if glossary
|
||||
else "(none yet)"
|
||||
)
|
||||
level_hint = (
|
||||
f"H{block.level}" if block.level >= 1 else "frontmatter (no heading)"
|
||||
)
|
||||
return (
|
||||
"# GLOSSARY (English || Chinese, already used earlier in the document):\n"
|
||||
f"{glossary_hint}\n\n"
|
||||
f"# BLOCK TO TRANSLATE (Markdown, {level_hint}):\n"
|
||||
"Translate the block between the markers below into Chinese, "
|
||||
"following all rules in the system prompt. Output using the exact "
|
||||
"delimiter format specified.\n\n"
|
||||
"----- BEGIN BLOCK -----\n"
|
||||
f"{block.content}\n"
|
||||
"----- END BLOCK -----\n"
|
||||
)
|
||||
|
||||
|
||||
def translate_block(
|
||||
client: ZenMuxClient,
|
||||
block: MarkdownBlock,
|
||||
glossary: dict[str, str],
|
||||
*,
|
||||
model: str,
|
||||
system_prompt: str,
|
||||
temperature: float,
|
||||
) -> tuple[str, dict[str, str]]:
|
||||
user = build_user_prompt(block, glossary)
|
||||
max_tok = MODEL_MAX_TOKENS.get(model, 16000)
|
||||
raw = client.chat_complete(
|
||||
model=model,
|
||||
system=system_prompt,
|
||||
user=user,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tok,
|
||||
tag=f"translate:{block.anchor}",
|
||||
)
|
||||
try:
|
||||
translation, patch = parse_delimited_response(raw)
|
||||
except Exception as e:
|
||||
raise RuntimeError(
|
||||
f"bad response format for block {block.anchor}: {e}\nraw head: {raw[:300]}"
|
||||
)
|
||||
if not translation.strip():
|
||||
raise RuntimeError(f"empty translation for block {block.anchor}")
|
||||
return translation.rstrip(), patch
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Phase 4 英译中（章节级切块并行翻译）")
|
||||
parser.add_argument("project", help="项目 slug 或完整路径")
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
default="phase4/final_en.md",
|
||||
help="英文源文件（相对项目根，默认 phase4/final_en.md）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default="phase4/final_zh.md",
|
||||
help="中文输出（默认 phase4/final_zh.md）",
|
||||
)
|
||||
parser.add_argument("--model", default=DEFAULT_MODEL, help="翻译模型")
|
||||
parser.add_argument("--temperature", type=float, default=0.3)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="忽略已有 zh_chunks 缓存，强制重翻",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--only",
|
||||
default=None,
|
||||
help="只翻译指定 order（逗号分隔的整数），其余跳过（调试用），例如 --only 0,1,7",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=None,
|
||||
help="最多翻译前 N 个未缓存的块（调试用）",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
load_secrets()
|
||||
project_root = resolve_project(args.project)
|
||||
src_path = project_root / args.source
|
||||
out_path = project_root / args.output
|
||||
if not src_path.exists():
|
||||
raise SystemExit(f"source not found: {src_path}")
|
||||
|
||||
chunks_dir = project_root / "phase4" / "zh_chunks"
|
||||
chunks_dir.mkdir(parents=True, exist_ok=True)
|
||||
glossary_path = project_root / "phase4" / "glossary.json"
|
||||
logs_dir = project_root / "phase4" / "logs"
|
||||
log_file = logs_dir / "translate.jsonl"
|
||||
|
||||
system_prompt = PROMPT_FILE.read_text(encoding="utf-8")
|
||||
text = src_path.read_text(encoding="utf-8")
|
||||
blocks = split_by_headers(text, max_level=2)
|
||||
glossary = load_glossary(glossary_path)
|
||||
|
||||
only_orders: set[int] | None = None
|
||||
if args.only:
|
||||
only_orders = {int(a.strip()) for a in args.only.split(",") if a.strip()}
|
||||
|
||||
total_en_words = sum(b.word_count for b in blocks)
|
||||
print(f"Source: {src_path.relative_to(project_root)}")
|
||||
print(f"Blocks: {len(blocks)} | total English words: {total_en_words:,}")
|
||||
print(f"Glossary loaded: {len(glossary)} terms")
|
||||
print(f"Model: {args.model} | temperature: {args.temperature}")
|
||||
print()
|
||||
|
||||
start = time.time()
|
||||
translated_this_run = 0
|
||||
with ZenMuxClient(log_file=log_file) as client:
|
||||
for b in blocks:
|
||||
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
|
||||
if only_orders is not None and b.order not in only_orders:
|
||||
continue
|
||||
if chunk_path.exists() and not args.force:
|
||||
print(f" [ok ] #{b.order:03d} {b.short_title} (cached)")
|
||||
continue
|
||||
if args.limit is not None and translated_this_run >= args.limit:
|
||||
continue
|
||||
|
||||
label = f"#{b.order:03d} L{b.level} {b.word_count:>4}w {b.short_title}"
|
||||
print(f" [... ] {label} ", end="", flush=True)
|
||||
t0 = time.time()
|
||||
try:
|
||||
translation, patch = translate_block(
|
||||
client,
|
||||
b,
|
||||
glossary,
|
||||
model=args.model,
|
||||
system_prompt=system_prompt,
|
||||
temperature=args.temperature,
|
||||
)
|
||||
except (ZenMuxError, RuntimeError) as e:
|
||||
print(f"\n [FAIL] {label}\n {e}")
|
||||
continue
|
||||
elapsed = time.time() - t0
|
||||
|
||||
chunk_path.write_text(translation + "\n", encoding="utf-8")
|
||||
if patch:
|
||||
for k, v in patch.items():
|
||||
glossary.setdefault(k, v)
|
||||
save_glossary(glossary_path, glossary)
|
||||
cn = count_chinese_chars(translation)
|
||||
translated_this_run += 1
|
||||
print(f"\r [done] {label} → {cn:>4}字 ({elapsed:4.1f}s, +{len(patch)} terms)")
|
||||
|
||||
# 汇总：按 order 拼接所有 chunk
|
||||
merged: list[str] = []
|
||||
missing: list[str] = []
|
||||
for b in blocks:
|
||||
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
|
||||
if not chunk_path.exists():
|
||||
missing.append(f"#{b.order:03d} {b.short_title}")
|
||||
continue
|
||||
merged.append(chunk_path.read_text(encoding="utf-8").rstrip())
|
||||
|
||||
partial = only_orders is not None or args.limit is not None
|
||||
if missing:
|
||||
print(f"\n⚠ 缺失 {len(missing)} 块：")
|
||||
for m in missing[:10]:
|
||||
print(f" - {m}")
|
||||
if len(missing) > 10:
|
||||
print(f" ... 还有 {len(missing) - 10} 块")
|
||||
if partial:
|
||||
print("（partial 模式：--only / --limit 生效，未生成 final_zh.md）")
|
||||
else:
|
||||
print("重新运行本脚本即可补译（已译的会跳过）。")
|
||||
print(client.usage.summary())
|
||||
return 1
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text("\n\n".join(merged) + "\n", encoding="utf-8")
|
||||
|
||||
# 统计
|
||||
final_text = out_path.read_text(encoding="utf-8")
|
||||
cn = count_chinese_chars(final_text)
|
||||
total_time = time.time() - start
|
||||
print()
|
||||
print(f"✓ 输出：{out_path.relative_to(project_root)}")
|
||||
print(f" 中文字数: {cn:,}")
|
||||
print(f" 英文词数源: {total_en_words:,} 膨胀率: {cn / max(total_en_words,1):.2f}×")
|
||||
print(f" 耗时: {total_time:.1f}s")
|
||||
print(f" 术语表: {len(glossary)} 条 → {glossary_path.relative_to(project_root)}")
|
||||
print(client.usage.summary())
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user