v0.6-wip: Python-based Phase 4 translation pipeline
架构变更：把 dr-translator 从 opencode agent 降级为 Python 脚本编排下的 LLM
调用。根本原因是 agent 一次性处理 19k 英文词整文，单次 output token 接近
Sonnet 4.6 上限（~32k），多次重跑都卡在同一个坑里——问题是架构本身，不是
prompt。
新架构：
scripts/lib/zenmux_client.py HTTP 客户端，指数退避重试、token 统计
JSONL 日志、secrets.env 自动加载
scripts/lib/markdown_chunker.py 按 H1/H2 切块，稳定 anchor ID（order+title
sha1），支持合并/统计
scripts/prompts/translate_system.txt 英译中 prompt，用自定义 <<<TRANSLATION>>>
分隔符格式（规避 Markdown-in-JSON 问题）
scripts/prompts/polish_system.txt 中文润色 prompt（留给下一步 polish.py）
scripts/translate.py 主入口：章节级切块 → 逐块翻译 → 拼接
关键设计：
- 0 依赖 LLM 遵从性：Python 控制切块/循环/重试，LLM 只做单块翻译
- 断点续传：每块翻译完立即写 phase4/zh_chunks/<order>-<anchor>.md
- 术语表累积：每块的 glossary_patch 合并回 phase4/glossary.json
- 失败隔离：单块失败不影响其他块，重跑只补缺
- 调试友好：--only N,M / --limit K / --force
实测（dual-target-rnai-pipeline-2026）：
- 63 块全部成功，17 分钟，$1.70
- 33,441 中文字（符合"研究类 ≥30,000 字"硬标准）
- 310 条双语术语
- 翻译质量：接近母语咨询分析师写作
下一步：polish.py（按 H2 section 润色）、merge_chapters.py（从 phase2/drafts
合并生成 final_en.md）、重构 dr-editor-in-chief 调度脚本、更新 /dr-finalize。
Co-authored-by: User <human>
This commit is contained in:
kai
@@ -0,0 +1,150 @@
|
||||
"""Markdown 切块 / 合并工具。
|
||||
|
||||
把一篇 Markdown 按标题层级（# / ##）切成"翻译单元"或"润色单元"，
|
||||
每个单元带稳定的 ID，便于断点续传和按需重跑。
|
||||
|
||||
核心约定：
|
||||
- H1（`# `）是一级块，通常对应 Chapter / 封面 / 前置件
|
||||
- H2（`## `）是二级块，对应一个 section 或独立前置件（Disclaimer / Executive Summary / Abstract / Glossary / References）
|
||||
- 没有任何标题的文件头（frontmatter 区）归到第 0 块
|
||||
|
||||
切块粒度默认到 H2；如果某个 H2 下的正文特别长可以再按 H3 切，但这是 polish 的事情，
|
||||
translate 一般不需要。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
HEADER_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$", re.MULTILINE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MarkdownBlock:
|
||||
"""一个翻译/润色单元。"""
|
||||
|
||||
order: int # 在原文中的顺序（0-based）
|
||||
level: int # 0 = frontmatter; 1/2/... = H1/H2/...
|
||||
title: str # 标题原文（不含 # 号）；frontmatter 为空串
|
||||
anchor: str # 稳定 ID，用于断点续传（order + title hash）
|
||||
content: str # 完整内容（包含标题行本身，除 frontmatter 块外）
|
||||
parent_order: int | None = None # H2 的父 H1 order；H1 为 None
|
||||
word_count: int = 0 # 英文 word count 估算（只含 a-z）
|
||||
char_count: int = 0 # 字符数（含中文）
|
||||
meta: dict = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def short_title(self) -> str:
|
||||
t = self.title.strip()
|
||||
if len(t) <= 50:
|
||||
return t
|
||||
return t[:47] + "..."
|
||||
|
||||
|
||||
def _count_words(text: str) -> int:
|
||||
return len(re.findall(r"[A-Za-z]+(?:[-'][A-Za-z]+)*", text))
|
||||
|
||||
|
||||
def _stable_anchor(order: int, title: str) -> str:
|
||||
"""order + title 生成稳定短 ID。同一文件改顺序不变，改标题重算。"""
|
||||
h = hashlib.sha1(title.strip().encode("utf-8")).hexdigest()[:8]
|
||||
return f"b{order:03d}-{h}"
|
||||
|
||||
|
||||
def split_by_headers(text: str, max_level: int = 2) -> list[MarkdownBlock]:
|
||||
"""把 Markdown 按 H1..H{max_level} 切块。
|
||||
|
||||
返回的 block 按出现顺序排列。第 0 块可能是 frontmatter（level=0，无 title）。
|
||||
"""
|
||||
lines = text.splitlines(keepends=False)
|
||||
# 先找出所有 header 位置
|
||||
header_positions: list[tuple[int, int, str]] = [] # (line_idx, level, title)
|
||||
in_code_block = False
|
||||
for i, ln in enumerate(lines):
|
||||
if ln.lstrip().startswith("```"):
|
||||
in_code_block = not in_code_block
|
||||
continue
|
||||
if in_code_block:
|
||||
continue
|
||||
m = re.match(r"^(#{1,6})\s+(.+?)\s*$", ln)
|
||||
if m:
|
||||
level = len(m.group(1))
|
||||
if level <= max_level:
|
||||
header_positions.append((i, level, m.group(2)))
|
||||
|
||||
blocks: list[MarkdownBlock] = []
|
||||
order = 0
|
||||
|
||||
# frontmatter：第一个 header 前的所有内容
|
||||
first_header_line = header_positions[0][0] if header_positions else len(lines)
|
||||
frontmatter_text = "\n".join(lines[:first_header_line]).rstrip()
|
||||
if frontmatter_text.strip():
|
||||
blocks.append(
|
||||
MarkdownBlock(
|
||||
order=order,
|
||||
level=0,
|
||||
title="",
|
||||
anchor=_stable_anchor(order, "__frontmatter__"),
|
||||
content=frontmatter_text,
|
||||
word_count=_count_words(frontmatter_text),
|
||||
char_count=len(frontmatter_text),
|
||||
)
|
||||
)
|
||||
order += 1
|
||||
|
||||
# 为每个 header 建一个 block：内容 = 本 header 行 + 下一 header 行前所有内容
|
||||
last_h1_order: int | None = None
|
||||
for idx, (line_idx, level, title) in enumerate(header_positions):
|
||||
end_line = (
|
||||
header_positions[idx + 1][0]
|
||||
if idx + 1 < len(header_positions)
|
||||
else len(lines)
|
||||
)
|
||||
content = "\n".join(lines[line_idx:end_line]).rstrip()
|
||||
block = MarkdownBlock(
|
||||
order=order,
|
||||
level=level,
|
||||
title=title,
|
||||
anchor=_stable_anchor(order, title),
|
||||
content=content,
|
||||
parent_order=last_h1_order if level > 1 else None,
|
||||
word_count=_count_words(content),
|
||||
char_count=len(content),
|
||||
)
|
||||
blocks.append(block)
|
||||
if level == 1:
|
||||
last_h1_order = order
|
||||
order += 1
|
||||
return blocks
|
||||
|
||||
|
||||
def merge_blocks(blocks: Iterable[MarkdownBlock], separator: str = "\n\n") -> str:
|
||||
"""按 order 拼回完整 Markdown。"""
|
||||
return separator.join(b.content for b in sorted(blocks, key=lambda x: x.order))
|
||||
|
||||
|
||||
def chapter_stem_from_title(title: str) -> str:
|
||||
"""从 `# Chapter 1 — foo bar` 生成文件名 stem 如 `ch01`。找不到就 fallback。"""
|
||||
m = re.search(r"chapter\s+(\d+)", title, re.IGNORECASE)
|
||||
if m:
|
||||
return f"ch{int(m.group(1)):02d}"
|
||||
# Executive Summary / Abstract / Glossary / Disclaimer / References / Version History
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")
|
||||
return slug[:40] or "untitled"
|
||||
|
||||
|
||||
def count_chinese_chars(text: str) -> int:
|
||||
return sum(1 for c in text if "\u4e00" <= c <= "\u9fff")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
src = Path(sys.argv[1])
|
||||
for b in split_by_headers(src.read_text(encoding="utf-8")):
|
||||
print(f"[{b.order:3d}] L{b.level} {b.word_count:>5}w {b.char_count:>6}c {b.short_title}")
|
||||
@@ -0,0 +1,270 @@
|
||||
"""ZenMux API 客户端。
|
||||
|
||||
直接 HTTP 调用 zenmux 的 OpenAI 兼容端点。为 Phase 4 的 Python 化脚本服务
|
||||
（translate.py / polish.py）。所有调用都走 /api/v1/chat/completions。
|
||||
|
||||
设计原则：
|
||||
- 独立于 opencode，可直接在 CLI / CI / cron 运行
|
||||
- 内置重试（指数退避）、限流、token 统计、结构化日志
|
||||
- 失败快速可见：打印请求 ID，便于在 zenmux 后台对账
|
||||
- 默认读 secrets.env 里的 ZENMUX_API_KEY
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
DEFAULT_BASE_URL = "https://zenmux.ai/api/v1"
|
||||
DEFAULT_TIMEOUT = 300.0 # 翻译/润色单次调用可能 60s+，留够余量
|
||||
MAX_RETRIES = 5
|
||||
RETRYABLE_STATUSES = {408, 429, 500, 502, 503, 504, 520, 524}
|
||||
|
||||
|
||||
@dataclass
|
||||
class UsageStats:
|
||||
"""聚合一次脚本运行的 token 消耗。"""
|
||||
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
cache_creation_tokens: int = 0
|
||||
cache_read_tokens: int = 0
|
||||
call_count: int = 0
|
||||
failed_calls: int = 0
|
||||
by_model: dict[str, dict[str, int]] = field(default_factory=dict)
|
||||
|
||||
def add(self, model: str, usage: dict[str, Any]) -> None:
|
||||
p = usage.get("prompt_tokens", 0) or 0
|
||||
c = usage.get("completion_tokens", 0) or 0
|
||||
cc = (
|
||||
usage.get("cache_creation_input_tokens", 0)
|
||||
or usage.get("prompt_tokens_details", {}).get("cached_tokens", 0)
|
||||
or 0
|
||||
)
|
||||
cr = (
|
||||
usage.get("cache_read_input_tokens", 0)
|
||||
or 0
|
||||
)
|
||||
self.prompt_tokens += p
|
||||
self.completion_tokens += c
|
||||
self.cache_creation_tokens += cc
|
||||
self.cache_read_tokens += cr
|
||||
self.call_count += 1
|
||||
m = self.by_model.setdefault(
|
||||
model,
|
||||
{"prompt": 0, "completion": 0, "cache_creation": 0, "cache_read": 0, "calls": 0},
|
||||
)
|
||||
m["prompt"] += p
|
||||
m["completion"] += c
|
||||
m["cache_creation"] += cc
|
||||
m["cache_read"] += cr
|
||||
m["calls"] += 1
|
||||
|
||||
def summary(self) -> str:
|
||||
lines = [
|
||||
f"Total calls: {self.call_count} (failed: {self.failed_calls})",
|
||||
f"Prompt tokens: {self.prompt_tokens:>12,}",
|
||||
f"Completion tokens: {self.completion_tokens:>12,}",
|
||||
f"Cache creation: {self.cache_creation_tokens:>12,}",
|
||||
f"Cache read: {self.cache_read_tokens:>12,}",
|
||||
]
|
||||
for model, s in self.by_model.items():
|
||||
lines.append(
|
||||
f" [{model}] calls={s['calls']} in={s['prompt']:,} "
|
||||
f"out={s['completion']:,} cache_w={s['cache_creation']:,} "
|
||||
f"cache_r={s['cache_read']:,}"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class ZenMuxError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class ZenMuxClient:
|
||||
"""ZenMux 轻量客户端。
|
||||
|
||||
只暴露一个方法 `chat_complete()`，屏蔽 httpx 细节。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str | None = None,
|
||||
base_url: str | None = None,
|
||||
timeout: float = DEFAULT_TIMEOUT,
|
||||
log_file: Path | None = None,
|
||||
) -> None:
|
||||
self.api_key = api_key or os.environ.get("ZENMUX_API_KEY")
|
||||
if not self.api_key:
|
||||
raise ZenMuxError(
|
||||
"ZENMUX_API_KEY not set. Source secrets.env or pass api_key explicitly."
|
||||
)
|
||||
self.base_url = (base_url or os.environ.get("ZENMUX_BASE_URL") or DEFAULT_BASE_URL).rstrip("/")
|
||||
self.timeout = timeout
|
||||
self.log_file = log_file
|
||||
self.usage = UsageStats()
|
||||
self._client = httpx.Client(timeout=timeout)
|
||||
|
||||
def close(self) -> None:
|
||||
self._client.close()
|
||||
|
||||
def __enter__(self) -> "ZenMuxClient":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args: Any) -> None:
|
||||
self.close()
|
||||
|
||||
def _log(self, payload: dict[str, Any]) -> None:
|
||||
if not self.log_file:
|
||||
return
|
||||
self.log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with self.log_file.open("a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(payload, ensure_ascii=False) + "\n")
|
||||
|
||||
def chat_complete(
|
||||
self,
|
||||
model: str,
|
||||
system: str,
|
||||
user: str,
|
||||
*,
|
||||
temperature: float = 0.3,
|
||||
max_tokens: int = 16000,
|
||||
extra_messages: list[dict[str, str]] | None = None,
|
||||
tag: str = "",
|
||||
) -> str:
|
||||
"""一次非流式对话补全。
|
||||
|
||||
Args:
|
||||
model: 完整 model id，例如 `anthropic/claude-sonnet-4.6`（zenmux slug 不带 `zenmux/` 前缀，因为 baseURL 已经定位到 zenmux）。
|
||||
system: system prompt
|
||||
user: user message
|
||||
temperature, max_tokens: 常规参数
|
||||
extra_messages: 插在 system 之后、user 之前的额外消息（few-shot 等）
|
||||
tag: 给这次调用打标签，便于日志里识别（如 "translate:ch03"）
|
||||
|
||||
Returns:
|
||||
assistant 的纯文本内容。如失败抛 ZenMuxError。
|
||||
"""
|
||||
messages: list[dict[str, str]] = [{"role": "system", "content": system}]
|
||||
if extra_messages:
|
||||
messages.extend(extra_messages)
|
||||
messages.append({"role": "user", "content": user})
|
||||
|
||||
body: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
url = f"{self.base_url}/chat/completions"
|
||||
|
||||
last_error: str = ""
|
||||
for attempt in range(MAX_RETRIES):
|
||||
t0 = time.time()
|
||||
try:
|
||||
resp = self._client.post(url, json=body, headers=headers)
|
||||
elapsed = time.time() - t0
|
||||
except httpx.RequestError as e:
|
||||
last_error = f"network: {e}"
|
||||
elapsed = time.time() - t0
|
||||
self._log({
|
||||
"tag": tag, "attempt": attempt, "elapsed": elapsed,
|
||||
"error": last_error,
|
||||
})
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
|
||||
if resp.status_code == 200:
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
raise ZenMuxError(f"invalid JSON from zenmux: {e}; body={resp.text[:500]}")
|
||||
usage = data.get("usage", {}) or {}
|
||||
self.usage.add(model, usage)
|
||||
content = ""
|
||||
choices = data.get("choices") or []
|
||||
if choices:
|
||||
msg = choices[0].get("message") or {}
|
||||
content = msg.get("content") or ""
|
||||
self._log({
|
||||
"tag": tag, "model": model, "attempt": attempt,
|
||||
"elapsed": round(elapsed, 2),
|
||||
"usage": usage,
|
||||
"out_chars": len(content),
|
||||
"status": 200,
|
||||
})
|
||||
if not content.strip():
|
||||
# zenmux 偶尔返 200 但 content 空；视作可重试
|
||||
last_error = "empty content"
|
||||
time.sleep(2 ** attempt)
|
||||
continue
|
||||
return content
|
||||
|
||||
# 非 200
|
||||
retryable = resp.status_code in RETRYABLE_STATUSES
|
||||
last_error = f"HTTP {resp.status_code}: {resp.text[:500]}"
|
||||
self._log({
|
||||
"tag": tag, "attempt": attempt, "elapsed": round(elapsed, 2),
|
||||
"status": resp.status_code, "error": last_error,
|
||||
"retryable": retryable,
|
||||
})
|
||||
if not retryable:
|
||||
self.usage.failed_calls += 1
|
||||
raise ZenMuxError(last_error)
|
||||
sleep_for = min(60, (2 ** attempt) + (attempt * 0.5))
|
||||
time.sleep(sleep_for)
|
||||
|
||||
self.usage.failed_calls += 1
|
||||
raise ZenMuxError(f"max retries exhausted. last error: {last_error}")
|
||||
|
||||
|
||||
def load_secrets(env_path: Path | None = None) -> None:
|
||||
"""从 secrets.env 把 key 塞到 os.environ，便于脚本直接运行。
|
||||
|
||||
格式宽松：`KEY=VALUE` 每行一条，`#` 开头是注释，忽略空行。
|
||||
"""
|
||||
if env_path is None:
|
||||
# 默认在仓库根找 secrets.env
|
||||
here = Path(__file__).resolve()
|
||||
for parent in [here.parent, *here.parents]:
|
||||
cand = parent / "secrets.env"
|
||||
if cand.exists():
|
||||
env_path = cand
|
||||
break
|
||||
if not env_path or not env_path.exists():
|
||||
return
|
||||
for line in env_path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
k, _, v = line.partition("=")
|
||||
k = k.strip()
|
||||
v = v.strip().strip('"').strip("'")
|
||||
if k and v and k not in os.environ:
|
||||
os.environ[k] = v
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 冒烟测试：python -m scripts.lib.zenmux_client
|
||||
load_secrets()
|
||||
with ZenMuxClient() as c:
|
||||
out = c.chat_complete(
|
||||
model="anthropic/claude-haiku-4.5",
|
||||
system="You reply in exactly one English word.",
|
||||
user="Say hello.",
|
||||
max_tokens=20,
|
||||
tag="smoke",
|
||||
)
|
||||
print("reply:", out)
|
||||
print(c.usage.summary(), file=sys.stderr)
|
||||
Reference in New Issue
Block a user