diff --git a/CHANGELOG.md b/CHANGELOG.md index 144ad17..a200c7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## 2026-10-08 — uniform Chinese Part ordinals and full-book mode + +- Reset publisher digit-only enlargement and color on translated Part numbers so “第四部” displays uniformly. +- Add `--complete-book` to include notes, bibliography, index, copyright and non-spine text documents. +- Add explicit configuration/credential file options and a Zenmux Sonnet 5.5 configuration without credentials. +- Checkpoint whole-book terminology scans and record conflicting model proposals while retaining established translations. + ## 2026-10-08 — clickable Part navigation - Keep parent Part links directly under NAV list items so Calibre resolves their destinations. diff --git a/README.md b/README.md index 82a976a..cabc367 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,18 @@ cp .env.example .env ## 可重复 QA +完整翻译使用 `--complete-book`,会纳入默认跳过的尾注、参考文献、索引、版权页及非 spine 文字资源。图片中的文字仍作为原图保留。NAV、NCX 和书内目录继续复用实际章节标题译文。 + +Zenmux 的 Sonnet 5.5 配置见 `config/zenmux.yaml`。凭据使用本地 `ZENMUX_API_KEY`,或通过 `--env-file` 指定 dotenv 文件;不把密钥写入 YAML 或提交到 Git。已有 Anthropic 兼容接口的 Zenmux token 可在本地映射到此变量: + +```bash +export ZENMUX_API_KEY="$ANTHROPIC_AUTH_TOKEN" +.venv/bin/python main.py 'input/Gambling Man.epub' --complete-book --bilingual \ + --config config/zenmux.yaml --output-dir output/sonnet55 +``` + +`--config` 指定独立配置,不需要修改默认模型。全书术语扫描逐批保存;中断后运行相同命令会恢复扫描及已验证译文。自动术语冲突记录在 glossary.json 的 conflicts 中,并保留已有译名,仍需语言复核。 + ```bash .venv/bin/pip install -r requirements-dev.txt .venv/bin/python -m pytest -q diff --git a/config/zenmux.yaml b/config/zenmux.yaml new file mode 100644 index 0000000..445d456 --- /dev/null +++ b/config/zenmux.yaml @@ -0,0 +1,12 @@ +llm: + model: anthropic/claude-sonnet-5.5 + base_url: https://zenmux.ai/api/v1 + timeout: 600 + max_output_tokens: 32000 + requests_per_minute: 60 + concurrent_requests: 3 +translation: + chunk_size: 24000 + validation_retries: 2 + auto_glossary: true + glossary_scan_chars: 24000 diff --git a/doc/QA_2026-10-08.md b/doc/QA_2026-10-08.md index d7375ff..9de3542 100644 --- a/doc/QA_2026-10-08.md +++ b/doc/QA_2026-10-08.md @@ -59,3 +59,11 @@ OCR、扫描页图文提取、版面中间结构与可点击目录的后续大 - 11 本书再次往返检查:格式还原失败 0,新增结构问题 0。 - Calibre 解析修复后的 Gambling Man 中文和双语样例,五个 Part 的目标均存在;在原生阅读器中实际点击第一部和第五部,均跳转到对应部标题,无错误弹窗。 - 新样例位于 `.work/part-link-fixed/`,沿用已验证译文,仍为 55/1,582 个单元的局部翻译;旧的 `.work/navigation-final-v4/` 仅保留为此前 QA 证据。 + +## 第四轮:部号统一样式与完整书籍翻译准备 + +原书将 PART 后面的阿拉伯数字单独设为 1.5em 和黑色。翻译为“第四部”后,这个样式只作用在“四”上。回填现在标记中文部号中的数字 span,使其继承所在标题的字体、字号、颜色和基线;主标题的块级样式及原文数字不改动。Calibre 原生阅读器实际打开第四部,确认“第四部”统一为灰色同字号,主标题正常;PDF 辅助渲染也确认该样式。证据目录 `.work/part-ordinal-fixed/qa-part4/`,仍为此前 Luna 局部样例。 + +50 项测试通过,覆盖中文/双语部号、完整模式提取非 spine 尾注、Zenmux 凭据优先级,以及术语冲突记录、扫描中断和续扫。默认模式继续保留原有跳过策略;完整模式将尾注、参考文献、索引和版权页等文字纳入翻译。 + +真实 Sonnet 5.5 全书运行中发现,单批术语返回同一别名的不同译名会中止全书。现在自动扫描逐条校验建议,保留已确定译名并记录冲突,每批原子保存结果及进度。手工术语表的冲突仍严格报错。两本书的实际运行结果将在完成后单独记录;本节不代表已完成全书译文验收。 diff --git a/main.py b/main.py index 65ebce8..ea1fb3f 100644 --- a/main.py +++ b/main.py @@ -47,7 +47,9 @@ async def run_pipeline(args): ensure_directory(output_dir) # Load Config - config = load_global_config() + if getattr(args, 'env_file', None): + load_dotenv(args.env_file, override=True) + config = load_global_config(getattr(args, 'config', 'config/config.yaml')) llm_conf = config.get("llm", {}) trans_conf = config.get("translation", {}) @@ -77,7 +79,7 @@ async def run_pipeline(args): structure = BookStructure.load(book_structure_json) # 2. Extraction - extractor = FineGrainedExtractor() + extractor = FineGrainedExtractor(complete_book=getattr(args, 'complete_book', False)) manifest_entries = extractor.extract(structure) prepare_navigation(input_path,structure,manifest_entries) structure.save(work["structure"]) @@ -112,6 +114,8 @@ async def run_pipeline(args): # Profiling profiler = BookProfiler(llm_client, cache_path=work["root"] / "profile.json") profile = await profiler.analyze(manifest_manager.entries) + if getattr(args, 'complete_book', False): + profile.style_guide += '\nTranslate every supplied unit fully, including front matter, copyright, notes and bibliography. Never omit text based on editorial preferences.' logger.info(f"Book Profile: {profile}") # Translation @@ -196,6 +200,9 @@ def main(): parser.add_argument("--output-dir", default="output", help="Directory for output files") parser.add_argument("--visual-qa", action="store_true", help="Render Calibre QA page images for visual inspection") parser.add_argument("--glossary", help="Per-book JSON terminology file") + parser.add_argument("--config", default="config/config.yaml", help="Local YAML configuration file") + parser.add_argument("--env-file", help="Local credential dotenv file (never committed)") + parser.add_argument("--complete-book", action="store_true", help="Include notes, bibliography, index, copyright and non-spine text documents") parser.add_argument("--model", default=None, help="LLM Model to use (overrides config)") parser.add_argument("--bilingual", action="store_true", help="Output bilingual version (default is target language only)") parser.add_argument("--skip-translation", action="store_true", help="Skip LLM translation (for testing)") @@ -211,4 +218,3 @@ def main(): if __name__ == "__main__": main() - diff --git a/src/assembly/backfiller.py b/src/assembly/backfiller.py index 922556d..41fb117 100644 --- a/src/assembly/backfiller.py +++ b/src/assembly/backfiller.py @@ -1,7 +1,7 @@ import copy import re from src.common.html import parse_html -from src.preprocessing.navigation import normalize_heading_html, sync_navigation +from src.preprocessing.navigation import normalize_heading_html, sync_navigation, chinese_number from src.assembly.format_restorer import FormatRestorer from src.common.utils import setup_logger, add_spacing @@ -76,6 +76,12 @@ class BackfillEngine: else: new_tag.attrs.pop('style', None) applied += 1 fragment = parse_html(f'
{entry.translated_html}
').body.div + record = titles.get(entry.entry_id) + if record and record.get('role') == 'part' and record.get('number'): + number = chinese_number(record['number']) + for span in fragment.find_all('span'): + if span.get_text(strip=True) == number: + span['data-translation-ordinal'] = 'true' for node in list(fragment.contents): new_tag.append(node.extract()) if mode == 'bilingual': diff --git a/src/assembly/bilingual_style.css b/src/assembly/bilingual_style.css index 4a9fd6c..e8932e7 100644 --- a/src/assembly/bilingual_style.css +++ b/src/assembly/bilingual_style.css @@ -22,3 +22,13 @@ p.translation::first-letter { .translation[data-translation-role="body"] { text-indent: 2em !important; } +/* A Chinese ordinal is one phrase, even when the publisher enlarged only 4. + Limit this reset to the translated number; keep the original Part/title design. */ +.translation [data-translation-ordinal="true"] { + font-family: inherit !important; + font-size: inherit !important; + font-weight: inherit !important; + color: inherit !important; + line-height: inherit !important; + vertical-align: baseline !important; +} diff --git a/src/common/config.py b/src/common/config.py index d843d95..a0e07ca 100644 --- a/src/common/config.py +++ b/src/common/config.py @@ -55,6 +55,8 @@ class ConfigLoader: base_url = os.getenv("OPENAI_BASE_URL") if base_url: self.config["llm"]["base_url"] = base_url + if 'zenmux.ai' in self.config['llm'].get('base_url', '') and os.getenv('ZENMUX_API_KEY'): + self.config['llm']['api_key'] = os.environ['ZENMUX_API_KEY'] return self.config diff --git a/src/preprocessing/text_extractor.py b/src/preprocessing/text_extractor.py index a390607..0b3b758 100644 --- a/src/preprocessing/text_extractor.py +++ b/src/preprocessing/text_extractor.py @@ -29,8 +29,9 @@ class FineGrainedExtractor: r'toc\.x?html', ] - def __init__(self, translate_toc: bool = False): + def __init__(self, translate_toc: bool = False, complete_book: bool = False): self.translate_toc = translate_toc + self.complete_book = complete_book self.format_extractor = FormatExtractor() def extract(self, structure: BookStructure, profile: Optional[BookProfile] = None) -> List[ManifestEntry]: @@ -42,7 +43,10 @@ class FineGrainedExtractor: manifest_entries = [] # Iterate over spine to maintain order - for item_id in structure.spine: + item_ids = list(structure.spine) + if self.complete_book: + item_ids.extend(item_id for item_id in structure.resources if item_id not in item_ids) + for item_id in item_ids: if item_id not in structure.resources: logger.warning(f"Item ID {item_id} in spine but not in resources.") continue @@ -97,8 +101,9 @@ class FineGrainedExtractor: def _classify_document(self, file_name: str) -> str: if not file_name: return 'core' fname = file_name.lower() - if any(re.search(p, fname) for p in self.SKIP_TRANSLATION_PATTERNS): return 'skip' if any(re.search(p, fname) for p in self.TOC_PATTERNS): return 'toc' + if any(re.search(p, fname) for p in self.SKIP_TRANSLATION_PATTERNS): + return 'core' if self.complete_book else 'skip' return 'core' def _should_translate(self, doc_type: str, is_decorative: bool, text: str = "") -> bool: diff --git a/src/translation/glossary.py b/src/translation/glossary.py index 1e87547..38c7076 100644 --- a/src/translation/glossary.py +++ b/src/translation/glossary.py @@ -2,7 +2,11 @@ import hashlib import json import re +import copy from pathlib import Path +from src.common.utils import setup_logger + +logger = setup_logger('glossary') def source_digest(entries): @@ -101,12 +105,16 @@ async def prepare_glossary(entries, llm, path, override=None, auto=True, chunk_c """Scan the whole book before translation; never race glossary updates.""" path = Path(path) digest = source_digest(entries) + terms, conflicts, start = [], [], 0 if override: return BookGlossary.load(override) if path.exists(): data = json.loads(path.read_text()) if data.get('source_digest') == digest: - return BookGlossary(data['terms']) + if data.get('scan_complete', True): + return BookGlossary(data['terms']) + if data.get('chunk_chars') == chunk_chars and data.get('model') == getattr(llm, 'model', ''): + terms, conflicts, start = data['terms'], data.get('conflicts', []), data.get('next_chunk', 0) if not auto: return BookGlossary() chunks, current, size = [], [], 0 @@ -116,8 +124,9 @@ async def prepare_glossary(entries, llm, path, override=None, auto=True, chunk_c current.append(entry.original_text); size += len(entry.original_text) if current: chunks.append('\n'.join(current)) - terms, conflicts = [], [] - for chunk in chunks: + path.parent.mkdir(parents=True, exist_ok=True) + for index in range(start, len(chunks)): + chunk = chunks[index] prompt = ('Extract recurring named people, organizations and specialist terminology. ' 'Resolve transliterations across Japanese/English/Chinese. Return only JSON ' '{"terms":[{"source":"English name","target":"Chinese canonical name",' @@ -127,21 +136,35 @@ async def prepare_glossary(entries, llm, path, override=None, auto=True, chunk_c json.dumps(terms, ensure_ascii=False) + '\nExcerpt:\n' + chunk) response = await llm.raw_chat_completion('You are a book terminology editor.', prompt) response = re.sub(r'^```(?:json)?\s*|\s*```$', '', response.strip()) - proposals = BookGlossary(json.loads(response)['terms']) - for term in proposals.terms: + # Validate proposals individually: a contradictory model suggestion must + # not discard the already frozen terms or stop the entire book scan. + for proposal in json.loads(response)['terms']: + try: + term = BookGlossary([proposal]).terms[0] + except (ValueError, AttributeError) as exc: + conflicts.append({'proposal': proposal, 'reason': str(exc)}) + continue aliases = {a.casefold() for a in [term['source']] + term.get('aliases', [])} existing = next((t for t in terms if aliases & {a.casefold() for a in [t['source']] + t.get('aliases', [])}), None) if existing and existing['target'] != term['target']: conflicts.append({'existing': existing, 'proposal': term}) continue + candidate = copy.deepcopy(terms) if existing: - existing['aliases'] = sorted(set(existing.get('aliases', []) + [term['source']] + term.get('aliases', [])) - {existing['source']}) + candidate[terms.index(existing)]['aliases'] = sorted(set(existing.get('aliases', []) + [term['source']] + term.get('aliases', [])) - {existing['source']}) else: - terms.append(term) - BookGlossary(terms) # Fail on alias collisions before saving. - path.parent.mkdir(parents=True, exist_ok=True) - temp = path.with_suffix('.tmp') - temp.write_text(json.dumps({'source_digest': digest, 'terms': terms, 'conflicts': conflicts}, - ensure_ascii=False, indent=2), encoding='utf-8') - temp.replace(path) + candidate.append(term) + try: + BookGlossary(candidate) + except ValueError as exc: + conflicts.append({'proposal': term, 'reason': str(exc)}) + continue + terms = candidate + temp = path.with_suffix('.tmp') + temp.write_text(json.dumps({'source_digest': digest, 'terms': terms, 'conflicts': conflicts, + 'scan_complete': index + 1 == len(chunks), 'next_chunk': index + 1, + 'chunk_chars': chunk_chars, 'model': getattr(llm, 'model', '')}, + ensure_ascii=False, indent=2), encoding='utf-8') + temp.replace(path) + logger.info('Terminology scan %s/%s: %s terms, %s conflicts recorded', index + 1, len(chunks), len(terms), len(conflicts)) return BookGlossary(terms) diff --git a/tests/test_navigation.py b/tests/test_navigation.py index f75af96..ecb4840 100644 --- a/tests/test_navigation.py +++ b/tests/test_navigation.py @@ -228,3 +228,40 @@ def test_calibre_resolves_parent_part_destination(tmp_path,mode): assert part['frag']=='part' assert part['dest_exists'] is True assert part['children'][0]['dest_exists'] is True + + +@pytest.mark.parametrize('mode', ['target_only', 'bilingual']) +def test_chinese_part_number_uses_uniform_ordinal_style(tmp_path, mode): + from src.preprocessing.format_extractor import FormatExtractor + source, work, structure, entries = fixture(tmp_path) + part = next(r for r in structure.resources.values() if r.href == 'Text/part.xhtml') + part.content = '

PART 1BOY GENIUS

' + entry = next(e for e in entries if e.element_id == 'part') + _, text, mapping, _, _ = FormatExtractor().extract(str(BeautifulSoup(part.content, 'html.parser').h1)) + entry.original_text = text + entry.placeholders = mapping + entry.translated_text = text.replace('PART ', '第').replace('BOY GENIUS', '天才少年') + # Preserve placeholder indices; insert the ordinal suffix after the number span. + closing = 'φ' + next(k for k, v in mapping.items() if v == '') + 'φ' + entry.translated_text = entry.translated_text.replace(closing, closing + '部', 1) + asyncio.run(RestorationEngine().restore_entries([entry], force=True)) + asyncio.run(BackfillEngine().backfill(structure, entries, mode)) + soup = BeautifulSoup(part.content, 'html.parser') + translated = soup.find('h1', class_='translation') + assert translated.find('span', class_='c1')['data-translation-ordinal'] == 'true' + assert translated.find('span', class_='heading_breakp').get_text() == '天才少年' + assert not translated.find('span', class_='heading_breakp').has_attr('data-translation-ordinal') + if mode == 'bilingual': + assert not soup.find('h1', id='part').find('span', class_='c1').has_attr('data-translation-ordinal') + + +def test_complete_book_includes_backmatter_and_non_spine_documents(tmp_path): + _, _, structure, _ = fixture(tmp_path) + notes = next(r for r in structure.resources.values() if r.href == 'Text/notes.xhtml') + structure.resources['supplement'] = notes.model_copy(update={'href': 'Text/footnotes.xhtml'}) + standard = FineGrainedExtractor().extract(structure) + complete = FineGrainedExtractor(complete_book=True).extract(structure) + assert not any(e.file_path == 'Text/footnotes.xhtml' for e in standard) + assert any(e.file_path == 'Text/footnotes.xhtml' and 'Note text' in e.original_text for e in complete) + assert FineGrainedExtractor(complete_book=True)._classify_document('index.xhtml') == 'core' + assert FineGrainedExtractor(complete_book=True)._classify_document('nav.xhtml') == 'toc' diff --git a/tests/test_regressions.py b/tests/test_regressions.py index 82be39c..710378b 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -15,6 +15,7 @@ from src.assembly.backfiller import BackfillEngine from src.assembly.builder import BilingualBuilder from src.translation.translator_engine import Translator from src.translation.glossary import BookGlossary, matches +from src.translation.glossary import prepare_glossary from src.translation.manifest_manager import ManifestManager from src.translation.llm_client import LLMClient from src.qa.epub_audit import audit_epub, compare_audits @@ -26,6 +27,43 @@ def entry(html='

Masayoshi Son built a company.

'): original_text=text, placeholders=mapping) +def test_glossary_model_conflicts_are_recorded_and_scan_resumes(tmp_path): + class Scanner: + model = 'test-model' + def __init__(self, fail=False): self.calls = 0; self.fail = fail + async def raw_chat_completion(self, *args): + self.calls += 1 + if self.fail and self.calls == 2: raise RuntimeError('network interrupted') + return json.dumps({'terms': [ + {'source': 'Masayoshi Son', 'target': '孙正义', 'aliases': ['Masa']}, + {'source': 'Masa', 'target': '马萨', 'aliases': []}]}) + entries = [entry(), entry('

Masa continued.

')] + path = tmp_path / 'glossary.json' + with pytest.raises(RuntimeError): + asyncio.run(prepare_glossary(entries, Scanner(True), path, chunk_chars=1)) + saved = json.loads(path.read_text()) + assert saved['next_chunk'] == 1 and not saved['scan_complete'] + assert saved['terms'][0]['target'] == '孙正义' + assert saved['conflicts'] + resumed = Scanner() + glossary = asyncio.run(prepare_glossary(entries, resumed, path, chunk_chars=1)) + assert resumed.calls == 1 + assert len(glossary.terms) == 1 + assert json.loads(path.read_text())['scan_complete'] + asyncio.run(prepare_glossary(entries, resumed, path, chunk_chars=1)) + assert resumed.calls == 1 + + +def test_zenmux_credentials_override_unrelated_openai_key(tmp_path, monkeypatch): + from src.common.config import load_global_config + config = tmp_path / 'zenmux.yaml' + config.write_text('llm:\n base_url: https://zenmux.ai/api/v1\n') + monkeypatch.setenv('ZENMUX_API_KEY', 'test-zenmux-key') + monkeypatch.setenv('OPENAI_API_KEY', 'test-other-key') + monkeypatch.delenv('OPENAI_BASE_URL', raising=False) + assert load_global_config(config)['llm']['api_key'] == 'test-zenmux-key' + + @pytest.mark.parametrize('html', [ '

Hello world.

', '

The angle φ = 10, and φ1φ is literal mathematical text.

',