Normalize Chinese Part ordinals and support resumable complete-book translation
This commit is contained in:
Kai Tan
@@ -1,3 +1,10 @@
|
||||
## 2026-10-08 — uniform Chinese Part ordinals and full-book mode
|
||||
|
||||
- Reset publisher digit-only enlargement and color on translated Part numbers so “第四部” displays uniformly.
|
||||
- Add `--complete-book` to include notes, bibliography, index, copyright and non-spine text documents.
|
||||
- Add explicit configuration/credential file options and a Zenmux Sonnet 5.5 configuration without credentials.
|
||||
- Checkpoint whole-book terminology scans and record conflicting model proposals while retaining established translations.
|
||||
|
||||
## 2026-10-08 — clickable Part navigation
|
||||
|
||||
- Keep parent Part links directly under NAV list items so Calibre resolves their destinations.
|
||||
|
||||
@@ -34,6 +34,18 @@ cp .env.example .env
|
||||
|
||||
## 可重复 QA
|
||||
|
||||
完整翻译使用 `--complete-book`，会纳入默认跳过的尾注、参考文献、索引、版权页及非 spine 文字资源。图片中的文字仍作为原图保留。NAV、NCX 和书内目录继续复用实际章节标题译文。
|
||||
|
||||
Zenmux 的 Sonnet 5.5 配置见 `config/zenmux.yaml`。凭据使用本地 `ZENMUX_API_KEY`，或通过 `--env-file` 指定 dotenv 文件；不把密钥写入 YAML 或提交到 Git。已有 Anthropic 兼容接口的 Zenmux token 可在本地映射到此变量：
|
||||
|
||||
```bash
|
||||
export ZENMUX_API_KEY="$ANTHROPIC_AUTH_TOKEN"
|
||||
.venv/bin/python main.py 'input/Gambling Man.epub' --complete-book --bilingual \
|
||||
--config config/zenmux.yaml --output-dir output/sonnet55
|
||||
```
|
||||
|
||||
`--config` 指定独立配置，不需要修改默认模型。全书术语扫描逐批保存；中断后运行相同命令会恢复扫描及已验证译文。自动术语冲突记录在 glossary.json 的 conflicts 中，并保留已有译名，仍需语言复核。
|
||||
|
||||
```bash
|
||||
.venv/bin/pip install -r requirements-dev.txt
|
||||
.venv/bin/python -m pytest -q
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
llm:
|
||||
model: anthropic/claude-sonnet-5.5
|
||||
base_url: https://zenmux.ai/api/v1
|
||||
timeout: 600
|
||||
max_output_tokens: 32000
|
||||
requests_per_minute: 60
|
||||
concurrent_requests: 3
|
||||
translation:
|
||||
chunk_size: 24000
|
||||
validation_retries: 2
|
||||
auto_glossary: true
|
||||
glossary_scan_chars: 24000
|
||||
@@ -59,3 +59,11 @@ OCR、扫描页图文提取、版面中间结构与可点击目录的后续大
|
||||
- 11 本书再次往返检查：格式还原失败 0，新增结构问题 0。
|
||||
- Calibre 解析修复后的 Gambling Man 中文和双语样例，五个 Part 的目标均存在；在原生阅读器中实际点击第一部和第五部，均跳转到对应部标题，无错误弹窗。
|
||||
- 新样例位于 `.work/part-link-fixed/`，沿用已验证译文，仍为 55/1,582 个单元的局部翻译；旧的 `.work/navigation-final-v4/` 仅保留为此前 QA 证据。
|
||||
|
||||
## 第四轮：部号统一样式与完整书籍翻译准备
|
||||
|
||||
原书将 PART 后面的阿拉伯数字单独设为 1.5em 和黑色。翻译为“第四部”后，这个样式只作用在“四”上。回填现在标记中文部号中的数字 span，使其继承所在标题的字体、字号、颜色和基线；主标题的块级样式及原文数字不改动。Calibre 原生阅读器实际打开第四部，确认“第四部”统一为灰色同字号，主标题正常；PDF 辅助渲染也确认该样式。证据目录 `.work/part-ordinal-fixed/qa-part4/`，仍为此前 Luna 局部样例。
|
||||
|
||||
50 项测试通过，覆盖中文／双语部号、完整模式提取非 spine 尾注、Zenmux 凭据优先级，以及术语冲突记录、扫描中断和续扫。默认模式继续保留原有跳过策略；完整模式将尾注、参考文献、索引和版权页等文字纳入翻译。
|
||||
|
||||
真实 Sonnet 5.5 全书运行中发现，单批术语返回同一别名的不同译名会中止全书。现在自动扫描逐条校验建议，保留已确定译名并记录冲突，每批原子保存结果及进度。手工术语表的冲突仍严格报错。两本书的实际运行结果将在完成后单独记录；本节不代表已完成全书译文验收。
|
||||
|
||||
@@ -47,7 +47,9 @@ async def run_pipeline(args):
|
||||
ensure_directory(output_dir)
|
||||
|
||||
# Load Config
|
||||
config = load_global_config()
|
||||
if getattr(args, 'env_file', None):
|
||||
load_dotenv(args.env_file, override=True)
|
||||
config = load_global_config(getattr(args, 'config', 'config/config.yaml'))
|
||||
llm_conf = config.get("llm", {})
|
||||
trans_conf = config.get("translation", {})
|
||||
|
||||
@@ -77,7 +79,7 @@ async def run_pipeline(args):
|
||||
structure = BookStructure.load(book_structure_json)
|
||||
|
||||
# 2. Extraction
|
||||
extractor = FineGrainedExtractor()
|
||||
extractor = FineGrainedExtractor(complete_book=getattr(args, 'complete_book', False))
|
||||
manifest_entries = extractor.extract(structure)
|
||||
prepare_navigation(input_path,structure,manifest_entries)
|
||||
structure.save(work["structure"])
|
||||
@@ -112,6 +114,8 @@ async def run_pipeline(args):
|
||||
# Profiling
|
||||
profiler = BookProfiler(llm_client, cache_path=work["root"] / "profile.json")
|
||||
profile = await profiler.analyze(manifest_manager.entries)
|
||||
if getattr(args, 'complete_book', False):
|
||||
profile.style_guide += '\nTranslate every supplied unit fully, including front matter, copyright, notes and bibliography. Never omit text based on editorial preferences.'
|
||||
logger.info(f"Book Profile: {profile}")
|
||||
|
||||
# Translation
|
||||
@@ -196,6 +200,9 @@ def main():
|
||||
parser.add_argument("--output-dir", default="output", help="Directory for output files")
|
||||
parser.add_argument("--visual-qa", action="store_true", help="Render Calibre QA page images for visual inspection")
|
||||
parser.add_argument("--glossary", help="Per-book JSON terminology file")
|
||||
parser.add_argument("--config", default="config/config.yaml", help="Local YAML configuration file")
|
||||
parser.add_argument("--env-file", help="Local credential dotenv file (never committed)")
|
||||
parser.add_argument("--complete-book", action="store_true", help="Include notes, bibliography, index, copyright and non-spine text documents")
|
||||
parser.add_argument("--model", default=None, help="LLM Model to use (overrides config)")
|
||||
parser.add_argument("--bilingual", action="store_true", help="Output bilingual version (default is target language only)")
|
||||
parser.add_argument("--skip-translation", action="store_true", help="Skip LLM translation (for testing)")
|
||||
@@ -211,4 +218,3 @@ def main():
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import copy
|
||||
import re
|
||||
from src.common.html import parse_html
|
||||
from src.preprocessing.navigation import normalize_heading_html, sync_navigation
|
||||
from src.preprocessing.navigation import normalize_heading_html, sync_navigation, chinese_number
|
||||
from src.assembly.format_restorer import FormatRestorer
|
||||
from src.common.utils import setup_logger, add_spacing
|
||||
|
||||
@@ -76,6 +76,12 @@ class BackfillEngine:
|
||||
else: new_tag.attrs.pop('style', None)
|
||||
applied += 1
|
||||
fragment = parse_html(f'<div>{entry.translated_html}</div>').body.div
|
||||
record = titles.get(entry.entry_id)
|
||||
if record and record.get('role') == 'part' and record.get('number'):
|
||||
number = chinese_number(record['number'])
|
||||
for span in fragment.find_all('span'):
|
||||
if span.get_text(strip=True) == number:
|
||||
span['data-translation-ordinal'] = 'true'
|
||||
for node in list(fragment.contents):
|
||||
new_tag.append(node.extract())
|
||||
if mode == 'bilingual':
|
||||
|
||||
@@ -22,3 +22,13 @@ p.translation::first-letter {
|
||||
.translation[data-translation-role="body"] {
|
||||
text-indent: 2em !important;
|
||||
}
|
||||
/* A Chinese ordinal is one phrase, even when the publisher enlarged only 4.
|
||||
Limit this reset to the translated number; keep the original Part/title design. */
|
||||
.translation [data-translation-ordinal="true"] {
|
||||
font-family: inherit !important;
|
||||
font-size: inherit !important;
|
||||
font-weight: inherit !important;
|
||||
color: inherit !important;
|
||||
line-height: inherit !important;
|
||||
vertical-align: baseline !important;
|
||||
}
|
||||
|
||||
@@ -55,6 +55,8 @@ class ConfigLoader:
|
||||
base_url = os.getenv("OPENAI_BASE_URL")
|
||||
if base_url:
|
||||
self.config["llm"]["base_url"] = base_url
|
||||
if 'zenmux.ai' in self.config['llm'].get('base_url', '') and os.getenv('ZENMUX_API_KEY'):
|
||||
self.config['llm']['api_key'] = os.environ['ZENMUX_API_KEY']
|
||||
|
||||
return self.config
|
||||
|
||||
|
||||
@@ -29,8 +29,9 @@ class FineGrainedExtractor:
|
||||
r'toc\.x?html',
|
||||
]
|
||||
|
||||
def __init__(self, translate_toc: bool = False):
|
||||
def __init__(self, translate_toc: bool = False, complete_book: bool = False):
|
||||
self.translate_toc = translate_toc
|
||||
self.complete_book = complete_book
|
||||
self.format_extractor = FormatExtractor()
|
||||
|
||||
def extract(self, structure: BookStructure, profile: Optional[BookProfile] = None) -> List[ManifestEntry]:
|
||||
@@ -42,7 +43,10 @@ class FineGrainedExtractor:
|
||||
manifest_entries = []
|
||||
|
||||
# Iterate over spine to maintain order
|
||||
for item_id in structure.spine:
|
||||
item_ids = list(structure.spine)
|
||||
if self.complete_book:
|
||||
item_ids.extend(item_id for item_id in structure.resources if item_id not in item_ids)
|
||||
for item_id in item_ids:
|
||||
if item_id not in structure.resources:
|
||||
logger.warning(f"Item ID {item_id} in spine but not in resources.")
|
||||
continue
|
||||
@@ -97,8 +101,9 @@ class FineGrainedExtractor:
|
||||
def _classify_document(self, file_name: str) -> str:
|
||||
if not file_name: return 'core'
|
||||
fname = file_name.lower()
|
||||
if any(re.search(p, fname) for p in self.SKIP_TRANSLATION_PATTERNS): return 'skip'
|
||||
if any(re.search(p, fname) for p in self.TOC_PATTERNS): return 'toc'
|
||||
if any(re.search(p, fname) for p in self.SKIP_TRANSLATION_PATTERNS):
|
||||
return 'core' if self.complete_book else 'skip'
|
||||
return 'core'
|
||||
|
||||
def _should_translate(self, doc_type: str, is_decorative: bool, text: str = "") -> bool:
|
||||
|
||||
+36
-13
@@ -2,7 +2,11 @@
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import copy
|
||||
from pathlib import Path
|
||||
from src.common.utils import setup_logger
|
||||
|
||||
logger = setup_logger('glossary')
|
||||
|
||||
|
||||
def source_digest(entries):
|
||||
@@ -101,12 +105,16 @@ async def prepare_glossary(entries, llm, path, override=None, auto=True, chunk_c
|
||||
"""Scan the whole book before translation; never race glossary updates."""
|
||||
path = Path(path)
|
||||
digest = source_digest(entries)
|
||||
terms, conflicts, start = [], [], 0
|
||||
if override:
|
||||
return BookGlossary.load(override)
|
||||
if path.exists():
|
||||
data = json.loads(path.read_text())
|
||||
if data.get('source_digest') == digest:
|
||||
return BookGlossary(data['terms'])
|
||||
if data.get('scan_complete', True):
|
||||
return BookGlossary(data['terms'])
|
||||
if data.get('chunk_chars') == chunk_chars and data.get('model') == getattr(llm, 'model', ''):
|
||||
terms, conflicts, start = data['terms'], data.get('conflicts', []), data.get('next_chunk', 0)
|
||||
if not auto:
|
||||
return BookGlossary()
|
||||
chunks, current, size = [], [], 0
|
||||
@@ -116,8 +124,9 @@ async def prepare_glossary(entries, llm, path, override=None, auto=True, chunk_c
|
||||
current.append(entry.original_text); size += len(entry.original_text)
|
||||
if current:
|
||||
chunks.append('\n'.join(current))
|
||||
terms, conflicts = [], []
|
||||
for chunk in chunks:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
for index in range(start, len(chunks)):
|
||||
chunk = chunks[index]
|
||||
prompt = ('Extract recurring named people, organizations and specialist terminology. '
|
||||
'Resolve transliterations across Japanese/English/Chinese. Return only JSON '
|
||||
'{"terms":[{"source":"English name","target":"Chinese canonical name",'
|
||||
@@ -127,21 +136,35 @@ async def prepare_glossary(entries, llm, path, override=None, auto=True, chunk_c
|
||||
json.dumps(terms, ensure_ascii=False) + '\nExcerpt:\n' + chunk)
|
||||
response = await llm.raw_chat_completion('You are a book terminology editor.', prompt)
|
||||
response = re.sub(r'^```(?:json)?\s*|\s*```$', '', response.strip())
|
||||
proposals = BookGlossary(json.loads(response)['terms'])
|
||||
for term in proposals.terms:
|
||||
# Validate proposals individually: a contradictory model suggestion must
|
||||
# not discard the already frozen terms or stop the entire book scan.
|
||||
for proposal in json.loads(response)['terms']:
|
||||
try:
|
||||
term = BookGlossary([proposal]).terms[0]
|
||||
except (ValueError, AttributeError) as exc:
|
||||
conflicts.append({'proposal': proposal, 'reason': str(exc)})
|
||||
continue
|
||||
aliases = {a.casefold() for a in [term['source']] + term.get('aliases', [])}
|
||||
existing = next((t for t in terms if aliases & {a.casefold() for a in [t['source']] + t.get('aliases', [])}), None)
|
||||
if existing and existing['target'] != term['target']:
|
||||
conflicts.append({'existing': existing, 'proposal': term})
|
||||
continue
|
||||
candidate = copy.deepcopy(terms)
|
||||
if existing:
|
||||
existing['aliases'] = sorted(set(existing.get('aliases', []) + [term['source']] + term.get('aliases', [])) - {existing['source']})
|
||||
candidate[terms.index(existing)]['aliases'] = sorted(set(existing.get('aliases', []) + [term['source']] + term.get('aliases', [])) - {existing['source']})
|
||||
else:
|
||||
terms.append(term)
|
||||
BookGlossary(terms) # Fail on alias collisions before saving.
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temp = path.with_suffix('.tmp')
|
||||
temp.write_text(json.dumps({'source_digest': digest, 'terms': terms, 'conflicts': conflicts},
|
||||
ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
temp.replace(path)
|
||||
candidate.append(term)
|
||||
try:
|
||||
BookGlossary(candidate)
|
||||
except ValueError as exc:
|
||||
conflicts.append({'proposal': term, 'reason': str(exc)})
|
||||
continue
|
||||
terms = candidate
|
||||
temp = path.with_suffix('.tmp')
|
||||
temp.write_text(json.dumps({'source_digest': digest, 'terms': terms, 'conflicts': conflicts,
|
||||
'scan_complete': index + 1 == len(chunks), 'next_chunk': index + 1,
|
||||
'chunk_chars': chunk_chars, 'model': getattr(llm, 'model', '')},
|
||||
ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
temp.replace(path)
|
||||
logger.info('Terminology scan %s/%s: %s terms, %s conflicts recorded', index + 1, len(chunks), len(terms), len(conflicts))
|
||||
return BookGlossary(terms)
|
||||
|
||||
@@ -228,3 +228,40 @@ def test_calibre_resolves_parent_part_destination(tmp_path,mode):
|
||||
assert part['frag']=='part'
|
||||
assert part['dest_exists'] is True
|
||||
assert part['children'][0]['dest_exists'] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize('mode', ['target_only', 'bilingual'])
|
||||
def test_chinese_part_number_uses_uniform_ordinal_style(tmp_path, mode):
|
||||
from src.preprocessing.format_extractor import FormatExtractor
|
||||
source, work, structure, entries = fixture(tmp_path)
|
||||
part = next(r for r in structure.resources.values() if r.href == 'Text/part.xhtml')
|
||||
part.content = '<html><body><h1 id="part">PART <span class="c1">1</span><span class="heading_breakp">BOY GENIUS</span></h1></body></html>'
|
||||
entry = next(e for e in entries if e.element_id == 'part')
|
||||
_, text, mapping, _, _ = FormatExtractor().extract(str(BeautifulSoup(part.content, 'html.parser').h1))
|
||||
entry.original_text = text
|
||||
entry.placeholders = mapping
|
||||
entry.translated_text = text.replace('PART ', '第').replace('BOY GENIUS', '天才少年')
|
||||
# Preserve placeholder indices; insert the ordinal suffix after the number span.
|
||||
closing = 'φ' + next(k for k, v in mapping.items() if v == '</span>') + 'φ'
|
||||
entry.translated_text = entry.translated_text.replace(closing, closing + '部', 1)
|
||||
asyncio.run(RestorationEngine().restore_entries([entry], force=True))
|
||||
asyncio.run(BackfillEngine().backfill(structure, entries, mode))
|
||||
soup = BeautifulSoup(part.content, 'html.parser')
|
||||
translated = soup.find('h1', class_='translation')
|
||||
assert translated.find('span', class_='c1')['data-translation-ordinal'] == 'true'
|
||||
assert translated.find('span', class_='heading_breakp').get_text() == '天才少年'
|
||||
assert not translated.find('span', class_='heading_breakp').has_attr('data-translation-ordinal')
|
||||
if mode == 'bilingual':
|
||||
assert not soup.find('h1', id='part').find('span', class_='c1').has_attr('data-translation-ordinal')
|
||||
|
||||
|
||||
def test_complete_book_includes_backmatter_and_non_spine_documents(tmp_path):
|
||||
_, _, structure, _ = fixture(tmp_path)
|
||||
notes = next(r for r in structure.resources.values() if r.href == 'Text/notes.xhtml')
|
||||
structure.resources['supplement'] = notes.model_copy(update={'href': 'Text/footnotes.xhtml'})
|
||||
standard = FineGrainedExtractor().extract(structure)
|
||||
complete = FineGrainedExtractor(complete_book=True).extract(structure)
|
||||
assert not any(e.file_path == 'Text/footnotes.xhtml' for e in standard)
|
||||
assert any(e.file_path == 'Text/footnotes.xhtml' and 'Note text' in e.original_text for e in complete)
|
||||
assert FineGrainedExtractor(complete_book=True)._classify_document('index.xhtml') == 'core'
|
||||
assert FineGrainedExtractor(complete_book=True)._classify_document('nav.xhtml') == 'toc'
|
||||
|
||||
@@ -15,6 +15,7 @@ from src.assembly.backfiller import BackfillEngine
|
||||
from src.assembly.builder import BilingualBuilder
|
||||
from src.translation.translator_engine import Translator
|
||||
from src.translation.glossary import BookGlossary, matches
|
||||
from src.translation.glossary import prepare_glossary
|
||||
from src.translation.manifest_manager import ManifestManager
|
||||
from src.translation.llm_client import LLMClient
|
||||
from src.qa.epub_audit import audit_epub, compare_audits
|
||||
@@ -26,6 +27,43 @@ def entry(html='<p id="p1">Masayoshi Son <em>built</em> a company.</p>'):
|
||||
original_text=text, placeholders=mapping)
|
||||
|
||||
|
||||
def test_glossary_model_conflicts_are_recorded_and_scan_resumes(tmp_path):
|
||||
class Scanner:
|
||||
model = 'test-model'
|
||||
def __init__(self, fail=False): self.calls = 0; self.fail = fail
|
||||
async def raw_chat_completion(self, *args):
|
||||
self.calls += 1
|
||||
if self.fail and self.calls == 2: raise RuntimeError('network interrupted')
|
||||
return json.dumps({'terms': [
|
||||
{'source': 'Masayoshi Son', 'target': '孙正义', 'aliases': ['Masa']},
|
||||
{'source': 'Masa', 'target': '马萨', 'aliases': []}]})
|
||||
entries = [entry(), entry('<p id="p2">Masa continued.</p>')]
|
||||
path = tmp_path / 'glossary.json'
|
||||
with pytest.raises(RuntimeError):
|
||||
asyncio.run(prepare_glossary(entries, Scanner(True), path, chunk_chars=1))
|
||||
saved = json.loads(path.read_text())
|
||||
assert saved['next_chunk'] == 1 and not saved['scan_complete']
|
||||
assert saved['terms'][0]['target'] == '孙正义'
|
||||
assert saved['conflicts']
|
||||
resumed = Scanner()
|
||||
glossary = asyncio.run(prepare_glossary(entries, resumed, path, chunk_chars=1))
|
||||
assert resumed.calls == 1
|
||||
assert len(glossary.terms) == 1
|
||||
assert json.loads(path.read_text())['scan_complete']
|
||||
asyncio.run(prepare_glossary(entries, resumed, path, chunk_chars=1))
|
||||
assert resumed.calls == 1
|
||||
|
||||
|
||||
def test_zenmux_credentials_override_unrelated_openai_key(tmp_path, monkeypatch):
|
||||
from src.common.config import load_global_config
|
||||
config = tmp_path / 'zenmux.yaml'
|
||||
config.write_text('llm:\n base_url: https://zenmux.ai/api/v1\n')
|
||||
monkeypatch.setenv('ZENMUX_API_KEY', 'test-zenmux-key')
|
||||
monkeypatch.setenv('OPENAI_API_KEY', 'test-other-key')
|
||||
monkeypatch.delenv('OPENAI_BASE_URL', raising=False)
|
||||
assert load_global_config(config)['llm']['api_key'] == 'test-zenmux-key'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('html', [
|
||||
'<p>Hello <em><b>world</b></em>.</p>',
|
||||
'<p>The angle φ = 10, and φ1φ is literal mathematical text.</p>',
|
||||
|
||||
Reference in New Issue
Block a user