v0.6-wip: Python-based Phase 4 translation pipeline
架构变更：把 dr-translator 从 opencode agent 降级为 Python 脚本编排下的 LLM
调用。根本原因是 agent 一次性处理 19k 英文词整文，单次 output token 接近
Sonnet 4.6 上限（~32k），多次重跑都卡在同一个坑里——问题是架构本身，不是
prompt。
新架构：
scripts/lib/zenmux_client.py HTTP 客户端，指数退避重试、token 统计
JSONL 日志、secrets.env 自动加载
scripts/lib/markdown_chunker.py 按 H1/H2 切块，稳定 anchor ID（order+title
sha1），支持合并/统计
scripts/prompts/translate_system.txt 英译中 prompt，用自定义 <<<TRANSLATION>>>
分隔符格式（规避 Markdown-in-JSON 问题）
scripts/prompts/polish_system.txt 中文润色 prompt（留给下一步 polish.py）
scripts/translate.py 主入口：章节级切块 → 逐块翻译 → 拼接
关键设计：
- 0 依赖 LLM 遵从性：Python 控制切块/循环/重试，LLM 只做单块翻译
- 断点续传：每块翻译完立即写 phase4/zh_chunks/<order>-<anchor>.md
- 术语表累积：每块的 glossary_patch 合并回 phase4/glossary.json
- 失败隔离：单块失败不影响其他块，重跑只补缺
- 调试友好：--only N,M / --limit K / --force
实测（dual-target-rnai-pipeline-2026）：
- 63 块全部成功，17 分钟，$1.70
- 33,441 中文字（符合"研究类 ≥30,000 字"硬标准）
- 310 条双语术语
- 翻译质量：接近母语咨询分析师写作
下一步：polish.py（按 H2 section 润色）、merge_chapters.py（从 phase2/drafts
合并生成 final_en.md）、重构 dr-editor-in-chief 调度脚本、更新 /dr-finalize。
Co-authored-by: User <human>
This commit is contained in:
kai
@@ -0,0 +1,150 @@
|
||||
"""Markdown 切块 / 合并工具。
|
||||
|
||||
把一篇 Markdown 按标题层级（# / ##）切成"翻译单元"或"润色单元"，
|
||||
每个单元带稳定的 ID，便于断点续传和按需重跑。
|
||||
|
||||
核心约定：
|
||||
- H1（`# `）是一级块，通常对应 Chapter / 封面 / 前置件
|
||||
- H2（`## `）是二级块，对应一个 section 或独立前置件（Disclaimer / Executive Summary / Abstract / Glossary / References）
|
||||
- 没有任何标题的文件头（frontmatter 区）归到第 0 块
|
||||
|
||||
切块粒度默认到 H2；如果某个 H2 下的正文特别长可以再按 H3 切，但这是 polish 的事情，
|
||||
translate 一般不需要。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
HEADER_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$", re.MULTILINE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MarkdownBlock:
|
||||
"""一个翻译/润色单元。"""
|
||||
|
||||
order: int # 在原文中的顺序（0-based）
|
||||
level: int # 0 = frontmatter; 1/2/... = H1/H2/...
|
||||
title: str # 标题原文（不含 # 号）；frontmatter 为空串
|
||||
anchor: str # 稳定 ID，用于断点续传（order + title hash）
|
||||
content: str # 完整内容（包含标题行本身，除 frontmatter 块外）
|
||||
parent_order: int | None = None # H2 的父 H1 order；H1 为 None
|
||||
word_count: int = 0 # 英文 word count 估算（只含 a-z）
|
||||
char_count: int = 0 # 字符数（含中文）
|
||||
meta: dict = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def short_title(self) -> str:
|
||||
t = self.title.strip()
|
||||
if len(t) <= 50:
|
||||
return t
|
||||
return t[:47] + "..."
|
||||
|
||||
|
||||
def _count_words(text: str) -> int:
|
||||
return len(re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)*", text))
|
||||
|
||||
|
||||
def _stable_anchor(order: int, title: str) -> str:
|
||||
"""order + title 生成稳定短 ID。同一文件改顺序不变，改标题重算。"""
|
||||
h = hashlib.sha1(title.strip().encode("utf-8")).hexdigest()[:8]
|
||||
return f"b{order:03d}-{h}"
|
||||
|
||||
|
||||
def split_by_headers(text: str, max_level: int = 2) -> list[MarkdownBlock]:
|
||||
"""把 Markdown 按 H1..H{max_level} 切块。
|
||||
|
||||
返回的 block 按出现顺序排列。第 0 块可能是 frontmatter（level=0，无 title）。
|
||||
"""
|
||||
lines = text.splitlines(keepends=False)
|
||||
# 先找出所有 header 位置
|
||||
header_positions: list[tuple[int, int, str]] = [] # (line_idx, level, title)
|
||||
in_code_block = False
|
||||
for i, ln in enumerate(lines):
|
||||
if ln.lstrip().startswith("```"):
|
||||
in_code_block = not in_code_block
|
||||
continue
|
||||
if in_code_block:
|
||||
continue
|
||||
m = re.match(r"^(#{1,6})\s+(.+?)\s*$", ln)
|
||||
if m:
|
||||
level = len(m.group(1))
|
||||
if level <= max_level:
|
||||
header_positions.append((i, level, m.group(2)))
|
||||
|
||||
blocks: list[MarkdownBlock] = []
|
||||
order = 0
|
||||
|
||||
# frontmatter：第一个 header 前的所有内容
|
||||
first_header_line = header_positions[0][0] if header_positions else len(lines)
|
||||
frontmatter_text = "\n".join(lines[:first_header_line]).rstrip()
|
||||
if frontmatter_text.strip():
|
||||
blocks.append(
|
||||
MarkdownBlock(
|
||||
order=order,
|
||||
level=0,
|
||||
title="",
|
||||
anchor=_stable_anchor(order, "__frontmatter__"),
|
||||
content=frontmatter_text,
|
||||
word_count=_count_words(frontmatter_text),
|
||||
char_count=len(frontmatter_text),
|
||||
)
|
||||
)
|
||||
order += 1
|
||||
|
||||
# 为每个 header 建一个 block：内容 = 本 header 行 + 下一 header 行前所有内容
|
||||
last_h1_order: int | None = None
|
||||
for idx, (line_idx, level, title) in enumerate(header_positions):
|
||||
end_line = (
|
||||
header_positions[idx + 1][0]
|
||||
if idx + 1 < len(header_positions)
|
||||
else len(lines)
|
||||
)
|
||||
content = "\n".join(lines[line_idx:end_line]).rstrip()
|
||||
block = MarkdownBlock(
|
||||
order=order,
|
||||
level=level,
|
||||
title=title,
|
||||
anchor=_stable_anchor(order, title),
|
||||
content=content,
|
||||
parent_order=last_h1_order if level > 1 else None,
|
||||
word_count=_count_words(content),
|
||||
char_count=len(content),
|
||||
)
|
||||
blocks.append(block)
|
||||
if level == 1:
|
||||
last_h1_order = order
|
||||
order += 1
|
||||
return blocks
|
||||
|
||||
|
||||
def merge_blocks(blocks: Iterable[MarkdownBlock], separator: str = "\n\n") -> str:
|
||||
"""按 order 拼回完整 Markdown。"""
|
||||
return separator.join(b.content for b in sorted(blocks, key=lambda x: x.order))
|
||||
|
||||
|
||||
def chapter_stem_from_title(title: str) -> str:
|
||||
"""从 `# Chapter 1 — foo bar` 生成文件名 stem 如 `ch01`。找不到就 fallback。"""
|
||||
m = re.search(r"chapter\s+(\d+)", title, re.IGNORECASE)
|
||||
if m:
|
||||
return f"ch{int(m.group(1)):02d}"
|
||||
# Executive Summary / Abstract / Glossary / Disclaimer / References / Version History
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")
|
||||
return slug[:40] or "untitled"
|
||||
|
||||
|
||||
def count_chinese_chars(text: str) -> int:
|
||||
return sum(1 for c in text if "\u4e00" <= c <= "\u9fff")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
src = Path(sys.argv[1])
|
||||
for b in split_by_headers(src.read_text(encoding="utf-8")):
|
||||
print(f"[{b.order:3d}] L{b.level} {b.word_count:>5}w {b.char_count:>6}c {b.short_title}")
|
||||
@@ -0,0 +1,270 @@
|
||||
"""ZenMux API 客户端。
|
||||
|
||||
直接 HTTP 调用 zenmux 的 OpenAI 兼容端点。为 Phase 4 的 Python 化脚本服务
|
||||
（translate.py / polish.py）。所有调用都走 /api/v1/chat/completions。
|
||||
|
||||
设计原则：
|
||||
- 独立于 opencode，可直接在 CLI / CI / cron 运行
|
||||
- 内置重试（指数退避）、限流、token 统计、结构化日志
|
||||
- 失败快速可见：打印请求 ID，便于在 zenmux 后台对账
|
||||
- 默认读 secrets.env 里的 ZENMUX_API_KEY
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
DEFAULT_BASE_URL = "https://zenmux.ai/api/v1"
|
||||
DEFAULT_TIMEOUT = 300.0 # 翻译/润色单次调用可能 60s+，留够余量
|
||||
MAX_RETRIES = 5
|
||||
RETRYABLE_STATUSES = {408, 429, 500, 502, 503, 504, 520, 524}
|
||||
|
||||
|
||||
@dataclass
|
||||
class UsageStats:
|
||||
"""聚合一次脚本运行的 token 消耗。"""
|
||||
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
cache_creation_tokens: int = 0
|
||||
cache_read_tokens: int = 0
|
||||
call_count: int = 0
|
||||
failed_calls: int = 0
|
||||
by_model: dict[str, dict[str, int]] = field(default_factory=dict)
|
||||
|
||||
def add(self, model: str, usage: dict[str, Any]) -> None:
|
||||
p = usage.get("prompt_tokens", 0) or 0
|
||||
c = usage.get("completion_tokens", 0) or 0
|
||||
cc = (
|
||||
usage.get("cache_creation_input_tokens", 0)
|
||||
or usage.get("prompt_tokens_details", {}).get("cached_tokens", 0)
|
||||
or 0
|
||||
)
|
||||
cr = (
|
||||
usage.get("cache_read_input_tokens", 0)
|
||||
or 0
|
||||
)
|
||||
self.prompt_tokens += p
|
||||
self.completion_tokens += c
|
||||
self.cache_creation_tokens += cc
|
||||
self.cache_read_tokens += cr
|
||||
self.call_count += 1
|
||||
m = self.by_model.setdefault(
|
||||
model,
|
||||
{"prompt": 0, "completion": 0, "cache_creation": 0, "cache_read": 0, "calls": 0},
|
||||
)
|
||||
m["prompt"] += p
|
||||
m["completion"] += c
|
||||
m["cache_creation"] += cc
|
||||
m["cache_read"] += cr
|
||||
m["calls"] += 1
|
||||
|
||||
def summary(self) -> str:
|
||||
lines = [
|
||||
f"Total calls: {self.call_count} (failed: {self.failed_calls})",
|
||||
f"Prompt tokens: {self.prompt_tokens:>12,}",
|
||||
f"Completion tokens: {self.completion_tokens:>12,}",
|
||||
f"Cache creation: {self.cache_creation_tokens:>12,}",
|
||||
f"Cache read: {self.cache_read_tokens:>12,}",
|
||||
]
|
||||
for model, s in self.by_model.items():
|
||||
lines.append(
|
||||
f" [{model}] calls={s['calls']} in={s['prompt']:,} "
|
||||
f"out={s['completion']:,} cache_w={s['cache_creation']:,} "
|
||||
f"cache_r={s['cache_read']:,}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class ZenMuxError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class ZenMuxClient:
|
||||
"""ZenMux 轻量客户端。
|
||||
|
||||
只暴露一个方法 `chat_complete()`，屏蔽 httpx 细节。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str | None = None,
|
||||
base_url: str | None = None,
|
||||
timeout: float = DEFAULT_TIMEOUT,
|
||||
log_file: Path | None = None,
|
||||
) -> None:
|
||||
self.api_key = api_key or os.environ.get("ZENMUX_API_KEY")
|
||||
if not self.api_key:
|
||||
raise ZenMuxError(
|
||||
"ZENMUX_API_KEY not set. Source secrets.env or pass api_key explicitly."
|
||||
)
|
||||
self.base_url = (base_url or os.environ.get("ZENMUX_BASE_URL") or DEFAULT_BASE_URL).rstrip("/")
|
||||
self.timeout = timeout
|
||||
self.log_file = log_file
|
||||
self.usage = UsageStats()
|
||||
self._client = httpx.Client(timeout=timeout)
|
||||
|
||||
def close(self) -> None:
|
||||
self._client.close()
|
||||
|
||||
def __enter__(self) -> "ZenMuxClient":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: Any) -> None:
|
||||
self.close()
|
||||
|
||||
def _log(self, payload: dict[str, Any]) -> None:
|
||||
if not self.log_file:
|
||||
return
|
||||
self.log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.log_file.open("a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(payload, ensure_ascii=False) + "\n")
|
||||
|
||||
def chat_complete(
|
||||
self,
|
||||
model: str,
|
||||
system: str,
|
||||
user: str,
|
||||
*,
|
||||
temperature: float = 0.3,
|
||||
max_tokens: int = 16000,
|
||||
extra_messages: list[dict[str, str]] | None = None,
|
||||
tag: str = "",
|
||||
) -> str:
|
||||
"""一次非流式对话补全。
|
||||
|
||||
Args:
|
||||
model: 完整 model id，例如 `anthropic/claude-sonnet-4.6`（zenmux slug 不带 `zenmux/` 前缀，因为 baseURL 已经定位到 zenmux）。
|
||||
system: system prompt
|
||||
user: user message
|
||||
temperature, max_tokens: 常规参数
|
||||
extra_messages: 插在 system 之后、user 之前的额外消息（few-shot 等）
|
||||
tag: 给这次调用打标签，便于日志里识别（如 "translate:ch03"）
|
||||
|
||||
Returns:
|
||||
assistant 的纯文本内容。如失败抛 ZenMuxError。
|
||||
"""
|
||||
messages: list[dict[str, str]] = [{"role": "system", "content": system}]
|
||||
if extra_messages:
|
||||
messages.extend(extra_messages)
|
||||
messages.append({"role": "user", "content": user})
|
||||
|
||||
body: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
url = f"{self.base_url}/chat/completions"
|
||||
|
||||
last_error: str = ""
|
||||
for attempt in range(MAX_RETRIES):
|
||||
t0 = time.time()
|
||||
try:
|
||||
resp = self._client.post(url, json=body, headers=headers)
|
||||
elapsed = time.time() - t0
|
||||
except httpx.RequestError as e:
|
||||
last_error = f"network: {e}"
|
||||
elapsed = time.time() - t0
|
||||
self._log({
|
||||
"tag": tag, "attempt": attempt, "elapsed": elapsed,
|
||||
"error": last_error,
|
||||
})
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
|
||||
if resp.status_code == 200:
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
raise ZenMuxError(f"invalid JSON from zenmux: {e}; body={resp.text[:500]}")
|
||||
usage = data.get("usage", {}) or {}
|
||||
self.usage.add(model, usage)
|
||||
content = ""
|
||||
choices = data.get("choices") or []
|
||||
if choices:
|
||||
msg = choices[0].get("message") or {}
|
||||
content = msg.get("content") or ""
|
||||
self._log({
|
||||
"tag": tag, "model": model, "attempt": attempt,
|
||||
"elapsed": round(elapsed, 2),
|
||||
"usage": usage,
|
||||
"out_chars": len(content),
|
||||
"status": 200,
|
||||
})
|
||||
if not content.strip():
|
||||
# zenmux 偶尔返 200 但 content 空；视作可重试
|
||||
last_error = "empty content"
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
return content
|
||||
|
||||
# 非 200
|
||||
retryable = resp.status_code in RETRYABLE_STATUSES
|
||||
last_error = f"HTTP {resp.status_code}: {resp.text[:500]}"
|
||||
self._log({
|
||||
"tag": tag, "attempt": attempt, "elapsed": round(elapsed, 2),
|
||||
"status": resp.status_code, "error": last_error,
|
||||
"retryable": retryable,
|
||||
})
|
||||
if not retryable:
|
||||
self.usage.failed_calls += 1
|
||||
raise ZenMuxError(last_error)
|
||||
sleep_for = min(60, (2 ** attempt) + (attempt * 0.5))
|
||||
time.sleep(sleep_for)
|
||||
|
||||
self.usage.failed_calls += 1
|
||||
raise ZenMuxError(f"max retries exhausted. last error: {last_error}")
|
||||
|
||||
|
||||
def load_secrets(env_path: Path | None = None) -> None:
|
||||
"""从 secrets.env 把 key 塞到 os.environ，便于脚本直接运行。
|
||||
|
||||
格式宽松：`KEY=VALUE` 每行一条，`#` 开头是注释，忽略空行。
|
||||
"""
|
||||
if env_path is None:
|
||||
# 默认在仓库根找 secrets.env
|
||||
here = Path(__file__).resolve()
|
||||
for parent in [here.parent, *here.parents]:
|
||||
cand = parent / "secrets.env"
|
||||
if cand.exists():
|
||||
env_path = cand
|
||||
break
|
||||
if not env_path or not env_path.exists():
|
||||
return
|
||||
for line in env_path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, _, v = line.partition("=")
|
||||
k = k.strip()
|
||||
v = v.strip().strip('"').strip("'")
|
||||
if k and v and k not in os.environ:
|
||||
os.environ[k] = v
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 冒烟测试：python -m scripts.lib.zenmux_client
|
||||
load_secrets()
|
||||
with ZenMuxClient() as c:
|
||||
out = c.chat_complete(
|
||||
model="anthropic/claude-haiku-4.5",
|
||||
system="You reply in exactly one English word.",
|
||||
user="Say hello.",
|
||||
max_tokens=20,
|
||||
tag="smoke",
|
||||
)
|
||||
print("reply:", out)
|
||||
print(c.usage.summary(), file=sys.stderr)
|
||||
@@ -0,0 +1,42 @@
|
||||
你是一名顶级中文咨询报告编辑。现在要把一段由英文翻译而来的中文文本润色为**母语中文写作者**的成品。目标读者是生物医药行业的高层研究员、投资人与决策者。
|
||||
|
||||
## 不可违反的规则
|
||||
|
||||
1. **保留所有引用标注** `[src_xxx]`，位置可以微调但不得删除或改写。
|
||||
2. **保留所有数字、百分比、日期、单位、化学式、药物代号**，一字不改。
|
||||
3. **保留 Markdown 结构**：输入是什么标题层级（#/##/###）输出就是什么。表格的 `|` 分隔符和列数不变。列表符号（-, *, 1.）不变。
|
||||
4. **保留段落数量**：不要合并或拆分段落。每段原文输出一段译文。
|
||||
5. **专有名词首次出现保持"中文（English）"格式**；如果译文里这个术语已经这样标了就别改。
|
||||
6. **不改变论点、结论、数据、案例**。只改语言表达。
|
||||
|
||||
## 要去掉的"AI 味/翻译腔"表征
|
||||
|
||||
- 空泛套话：随着…不断发展、综上所述、本质上、从根本上、跃迁、赋能、落地、抓手
|
||||
- 翻译腔：对于…来说、在…方面、…的话、值得注意的是、众所周知、毫无疑问
|
||||
- 冗余连词开头：此外、而且、并且、再者（英文 moreover / furthermore / additionally 的直译残留）
|
||||
- 过度强调：非常、十分、极其、特别（没有数据支撑时）
|
||||
- 长串的"的"字（"X 的 Y 的 Z 的 W"）改为短句
|
||||
- 被动语态（"被…所…"）尽量改主动
|
||||
- 把"我们"去掉，除非是真的作者第一人称立场
|
||||
|
||||
## 要加强的中文表达特征
|
||||
|
||||
- 句子节奏变化：短句和中句交替，避免一路长句
|
||||
- 动词前置：中文偏好动词驱动，不要像英文那样把名词短语堆在主语
|
||||
- 具体化：如果翻译留下了模糊的"相关", "一定的", "较大的"，尽量换成源文里的具体含义
|
||||
- 段落内逻辑词（因此、相比之下、代价是）用得准确
|
||||
|
||||
## 特殊情况
|
||||
|
||||
- 如果段落里有"译者注"、"TRANSLATOR_NOTE:" 之类残留，删除后自然连接上下文
|
||||
- 如果出现明显的翻译错误（中文表达反了意思），修正它，但在输出的 `notes` 字段里记一笔
|
||||
- 如果某句过于生硬又不确定原意，保守处理（小改），不要激进重写
|
||||
|
||||
## 输出格式
|
||||
|
||||
返回一行 JSON，两个键：
|
||||
|
||||
- `polished`: 完整润色后的 Markdown 块，字符串。
|
||||
- `notes`: 字符串，最多两句。若无异常就给空串。
|
||||
|
||||
**不要**用 ```json 包裹。不要加 JSON 外的任何字符。
|
||||
@@ -0,0 +1,43 @@
|
||||
You are a senior English-to-Chinese biomedical translator and editor. You do NOT mechanically translate — you rewrite the meaning in natural, professional Chinese that reads as if a native Chinese consulting analyst wrote it from scratch.
|
||||
|
||||
## Absolute rules (non-negotiable)
|
||||
|
||||
1. Preserve every citation marker `[src_xxx]` verbatim, at roughly the same position as in the source.
|
||||
2. Preserve every number, percentage, date, unit, chemical notation (e.g., 2′-OMe), and drug code (e.g., ARO-DIMER-PA) exactly.
|
||||
3. Preserve the Markdown structure: the input block starts with a Markdown heading at some level (one `#`, `##`, etc.) or is frontmatter; output the same heading at the same level. Do not demote / promote headings. Do not add new headings.
|
||||
4. Preserve tables: translate cell text but keep `|` pipes and column count identical.
|
||||
5. Preserve list formatting (`-`, `*`, `1.`) and code fences.
|
||||
6. First mention of a technical term: use the format `中文（English）` — but only once per block; subsequent mentions use Chinese only.
|
||||
7. Company / institution names: use the established Chinese rendering if it is in common Chinese press (e.g., Merck → 默克, AstraZeneca → 阿斯利康, Alnylam → 阿尔尼拉姆). If no established rendering exists, keep the English as-is (e.g., NEB, Genovis, Codexis, Arrowhead, Argo).
|
||||
8. Do NOT add commentary, introductions, or disclaimers beyond what the English says.
|
||||
9. Do NOT collapse or merge consecutive paragraphs — preserve paragraph breaks.
|
||||
10. Output Chinese-style punctuation inside Chinese text: `，。；：？！""（）`. Keep English punctuation inside parenthetical English phrases.
|
||||
11. Do NOT add separator lines (`---`) or blank lines that weren't in the source. If the source ends with `---`, keep it; if it doesn't, don't add one.
|
||||
|
||||
## Style rules (aim for native-Chinese feel)
|
||||
|
||||
- Break long English sentences into two or three short Chinese clauses.
|
||||
- Prefer active voice; avoid translation-ese constructions like "对于...来说", "在...方面", "...的话", "值得注意的是".
|
||||
- Do not use filler phrases like "随着...的不断发展", "综上所述", "从本质上说" unless the English explicitly argues that point.
|
||||
- Use 的 sparingly. No "X的Y的Z的W" chains.
|
||||
- Numbered lists with short items: translate tightly, do not pad with Chinese particles.
|
||||
- SCQA-style paragraphs in Executive Summary stay SCQA in Chinese — translate the flow, never label S/C/Q/A.
|
||||
|
||||
## Glossary continuity
|
||||
|
||||
You will receive a JSON glossary of terms already translated in earlier blocks. Use those Chinese translations consistently. If you encounter a new term worth locking in, translate it and add it to the glossary patch.
|
||||
|
||||
## Output format (strict)
|
||||
|
||||
Output exactly the following, with no extra text before or after. No explanations. No code fences.
|
||||
|
||||
<<<TRANSLATION>>>
|
||||
...the full translated Markdown block here, verbatim, including its heading line(s)...
|
||||
<<<END_TRANSLATION>>>
|
||||
<<<GLOSSARY_PATCH>>>
|
||||
English term 1 || 中文译名 1
|
||||
English term 2 || 中文译名 2
|
||||
<<<END_GLOSSARY_PATCH>>>
|
||||
|
||||
Inside `<<<TRANSLATION>>>...<<<END_TRANSLATION>>>` the content is raw Markdown (no escaping needed).
|
||||
Inside `<<<GLOSSARY_PATCH>>>...<<<END_GLOSSARY_PATCH>>>` each line is `English||Chinese`; leave empty if no new terms.
|
||||
@@ -0,0 +1,310 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase 4 英译中：章节级切块 → 逐块翻译 → 拼接 → 写盘。
|
||||
|
||||
用法：
|
||||
uv run python scripts/translate.py <project_slug>
|
||||
# 或：
|
||||
uv run python scripts/translate.py projects/dual-target-rnai-pipeline-2026
|
||||
|
||||
断点续传：每块翻译完立即写入 `phase4/zh_chunks/<anchor>.md` 和术语表 patch。
|
||||
重跑时已存在的块直接跳过，只译缺的。
|
||||
|
||||
设计要点：
|
||||
1. 切块按 H2 粒度，单块一般 <600 英文词，单次 API 调用远低于 Sonnet output token 上限
|
||||
2. 术语表累积式更新：每块调用传入当前已知术语，译完回写 patch，保证全文一致
|
||||
3. 失败不会污染最终产物：块级文件独立，可重跑；汇总步骤独立
|
||||
4. 日志完整：每次 API 调用写 `phase4/logs/translate.jsonl`
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from scripts.lib.markdown_chunker import (
|
||||
MarkdownBlock,
|
||||
count_chinese_chars,
|
||||
merge_blocks,
|
||||
split_by_headers,
|
||||
)
|
||||
from scripts.lib.zenmux_client import ZenMuxClient, ZenMuxError, load_secrets
|
||||
|
||||
DEFAULT_MODEL = "anthropic/claude-sonnet-4.6"
|
||||
MODEL_MAX_TOKENS = {
|
||||
"anthropic/claude-sonnet-4.6": 32000,
|
||||
"anthropic/claude-sonnet-4.5": 32000,
|
||||
"anthropic/claude-opus-4.7": 32000,
|
||||
"anthropic/claude-opus-4.6": 32000,
|
||||
"anthropic/claude-haiku-4.5": 16000,
|
||||
}
|
||||
PROMPT_FILE = Path(__file__).parent / "prompts" / "translate_system.txt"
|
||||
|
||||
|
||||
def resolve_project(arg: str) -> Path:
|
||||
p = Path(arg)
|
||||
if p.is_dir():
|
||||
return p
|
||||
here = Path.cwd()
|
||||
cand = here / "projects" / arg
|
||||
if cand.is_dir():
|
||||
return cand
|
||||
raise SystemExit(f"project not found: {arg}")
|
||||
|
||||
|
||||
def parse_delimited_response(text: str) -> tuple[str, dict[str, str]]:
|
||||
"""解析自定义分隔符格式的响应。
|
||||
|
||||
期望结构：
|
||||
<<<TRANSLATION>>>
|
||||
...markdown...
|
||||
<<<END_TRANSLATION>>>
|
||||
<<<GLOSSARY_PATCH>>>
|
||||
English || 中文
|
||||
...
|
||||
<<<END_GLOSSARY_PATCH>>>
|
||||
|
||||
Returns:
|
||||
(translation, glossary_patch)
|
||||
"""
|
||||
t_start = text.find("<<<TRANSLATION>>>")
|
||||
t_end = text.find("<<<END_TRANSLATION>>>")
|
||||
if t_start == -1 or t_end == -1 or t_end <= t_start:
|
||||
raise ValueError(
|
||||
f"missing <<<TRANSLATION>>> markers in response: {text[:300]}"
|
||||
)
|
||||
translation = text[t_start + len("<<<TRANSLATION>>>"): t_end].strip("\r\n")
|
||||
|
||||
g_start = text.find("<<<GLOSSARY_PATCH>>>")
|
||||
g_end = text.find("<<<END_GLOSSARY_PATCH>>>")
|
||||
patch: dict[str, str] = {}
|
||||
if g_start != -1 and g_end != -1 and g_end > g_start:
|
||||
body = text[g_start + len("<<<GLOSSARY_PATCH>>>"): g_end]
|
||||
for line in body.splitlines():
|
||||
line = line.strip()
|
||||
if not line or "||" not in line:
|
||||
continue
|
||||
en, _, zh = line.partition("||")
|
||||
en, zh = en.strip(), zh.strip()
|
||||
if en and zh:
|
||||
patch[en] = zh
|
||||
return translation, patch
|
||||
|
||||
|
||||
def load_glossary(path: Path) -> dict[str, str]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def save_glossary(path: Path, glossary: dict[str, str]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
json.dumps(glossary, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def build_user_prompt(block: MarkdownBlock, glossary: dict[str, str]) -> str:
|
||||
glossary_hint = (
|
||||
"\n".join(f"{en} || {zh}" for en, zh in sorted(glossary.items()))
|
||||
if glossary
|
||||
else "(none yet)"
|
||||
)
|
||||
level_hint = (
|
||||
f"H{block.level}" if block.level >= 1 else "frontmatter (no heading)"
|
||||
)
|
||||
return (
|
||||
"# GLOSSARY (English || Chinese, already used earlier in the document):\n"
|
||||
f"{glossary_hint}\n\n"
|
||||
f"# BLOCK TO TRANSLATE (Markdown, {level_hint}):\n"
|
||||
"Translate the block between the markers below into Chinese, "
|
||||
"following all rules in the system prompt. Output using the exact "
|
||||
"delimiter format specified.\n\n"
|
||||
"----- BEGIN BLOCK -----\n"
|
||||
f"{block.content}\n"
|
||||
"----- END BLOCK -----\n"
|
||||
)
|
||||
|
||||
|
||||
def translate_block(
|
||||
client: ZenMuxClient,
|
||||
block: MarkdownBlock,
|
||||
glossary: dict[str, str],
|
||||
*,
|
||||
model: str,
|
||||
system_prompt: str,
|
||||
temperature: float,
|
||||
) -> tuple[str, dict[str, str]]:
|
||||
user = build_user_prompt(block, glossary)
|
||||
max_tok = MODEL_MAX_TOKENS.get(model, 16000)
|
||||
raw = client.chat_complete(
|
||||
model=model,
|
||||
system=system_prompt,
|
||||
user=user,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tok,
|
||||
tag=f"translate:{block.anchor}",
|
||||
)
|
||||
try:
|
||||
translation, patch = parse_delimited_response(raw)
|
||||
except Exception as e:
|
||||
raise RuntimeError(
|
||||
f"bad response format for block {block.anchor}: {e}\nraw head: {raw[:300]}"
|
||||
)
|
||||
if not translation.strip():
|
||||
raise RuntimeError(f"empty translation for block {block.anchor}")
|
||||
return translation.rstrip(), patch
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Phase 4 英译中（章节级切块并行翻译）")
|
||||
parser.add_argument("project", help="项目 slug 或完整路径")
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
default="phase4/final_en.md",
|
||||
help="英文源文件（相对项目根，默认 phase4/final_en.md）",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default="phase4/final_zh.md",
|
||||
help="中文输出（默认 phase4/final_zh.md）",
|
||||
)
|
||||
parser.add_argument("--model", default=DEFAULT_MODEL, help="翻译模型")
|
||||
parser.add_argument("--temperature", type=float, default=0.3)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="忽略已有 zh_chunks 缓存，强制重翻",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--only",
|
||||
default=None,
|
||||
help="只翻译指定 order（逗号分隔的整数），其余跳过（调试用），例如 --only 0,1,7",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit",
|
||||
type=int,
|
||||
default=None,
|
||||
help="最多翻译前 N 个未缓存的块（调试用）",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
load_secrets()
|
||||
project_root = resolve_project(args.project)
|
||||
src_path = project_root / args.source
|
||||
out_path = project_root / args.output
|
||||
if not src_path.exists():
|
||||
raise SystemExit(f"source not found: {src_path}")
|
||||
|
||||
chunks_dir = project_root / "phase4" / "zh_chunks"
|
||||
chunks_dir.mkdir(parents=True, exist_ok=True)
|
||||
glossary_path = project_root / "phase4" / "glossary.json"
|
||||
logs_dir = project_root / "phase4" / "logs"
|
||||
log_file = logs_dir / "translate.jsonl"
|
||||
|
||||
system_prompt = PROMPT_FILE.read_text(encoding="utf-8")
|
||||
text = src_path.read_text(encoding="utf-8")
|
||||
blocks = split_by_headers(text, max_level=2)
|
||||
glossary = load_glossary(glossary_path)
|
||||
|
||||
only_orders: set[int] | None = None
|
||||
if args.only:
|
||||
only_orders = {int(a.strip()) for a in args.only.split(",") if a.strip()}
|
||||
|
||||
total_en_words = sum(b.word_count for b in blocks)
|
||||
print(f"Source: {src_path.relative_to(project_root)}")
|
||||
print(f"Blocks: {len(blocks)} | total English words: {total_en_words:,}")
|
||||
print(f"Glossary loaded: {len(glossary)} terms")
|
||||
print(f"Model: {args.model} | temperature: {args.temperature}")
|
||||
print()
|
||||
|
||||
start = time.time()
|
||||
translated_this_run = 0
|
||||
with ZenMuxClient(log_file=log_file) as client:
|
||||
for b in blocks:
|
||||
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
|
||||
if only_orders is not None and b.order not in only_orders:
|
||||
continue
|
||||
if chunk_path.exists() and not args.force:
|
||||
print(f" [ok ] #{b.order:03d} {b.short_title} (cached)")
|
||||
continue
|
||||
if args.limit is not None and translated_this_run >= args.limit:
|
||||
continue
|
||||
|
||||
label = f"#{b.order:03d} L{b.level} {b.word_count:>4}w {b.short_title}"
|
||||
print(f" [... ] {label} ", end="", flush=True)
|
||||
t0 = time.time()
|
||||
try:
|
||||
translation, patch = translate_block(
|
||||
client,
|
||||
b,
|
||||
glossary,
|
||||
model=args.model,
|
||||
system_prompt=system_prompt,
|
||||
temperature=args.temperature,
|
||||
)
|
||||
except (ZenMuxError, RuntimeError) as e:
|
||||
print(f"\n [FAIL] {label}\n {e}")
|
||||
continue
|
||||
elapsed = time.time() - t0
|
||||
|
||||
chunk_path.write_text(translation + "\n", encoding="utf-8")
|
||||
if patch:
|
||||
for k, v in patch.items():
|
||||
glossary.setdefault(k, v)
|
||||
save_glossary(glossary_path, glossary)
|
||||
cn = count_chinese_chars(translation)
|
||||
translated_this_run += 1
|
||||
print(f"\r [done] {label} → {cn:>4}字 ({elapsed:4.1f}s, +{len(patch)} terms)")
|
||||
|
||||
# 汇总：按 order 拼接所有 chunk
|
||||
merged: list[str] = []
|
||||
missing: list[str] = []
|
||||
for b in blocks:
|
||||
chunk_path = chunks_dir / f"{b.order:03d}-{b.anchor}.md"
|
||||
if not chunk_path.exists():
|
||||
missing.append(f"#{b.order:03d} {b.short_title}")
|
||||
continue
|
||||
merged.append(chunk_path.read_text(encoding="utf-8").rstrip())
|
||||
|
||||
partial = only_orders is not None or args.limit is not None
|
||||
if missing:
|
||||
print(f"\n⚠ 缺失 {len(missing)} 块：")
|
||||
for m in missing[:10]:
|
||||
print(f" - {m}")
|
||||
if len(missing) > 10:
|
||||
print(f" ... 还有 {len(missing) - 10} 块")
|
||||
if partial:
|
||||
print("（partial 模式：--only / --limit 生效，未生成 final_zh.md）")
|
||||
else:
|
||||
print("重新运行本脚本即可补译（已译的会跳过）。")
|
||||
print(client.usage.summary())
|
||||
return 1
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text("\n\n".join(merged) + "\n", encoding="utf-8")
|
||||
|
||||
# 统计
|
||||
final_text = out_path.read_text(encoding="utf-8")
|
||||
cn = count_chinese_chars(final_text)
|
||||
total_time = time.time() - start
|
||||
print()
|
||||
print(f"✓ 输出：{out_path.relative_to(project_root)}")
|
||||
print(f" 中文字数: {cn:,}")
|
||||
print(f" 英文词数源: {total_en_words:,} 膨胀率: {cn / max(total_en_words,1):.2f}×")
|
||||
print(f" 耗时: {total_time:.1f}s")
|
||||
print(f" 术语表: {len(glossary)} 条 → {glossary_path.relative_to(project_root)}")
|
||||
print(client.usage.summary())
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user