Normalize Chinese Part ordinals and support resumable complete-book translation

This commit is contained in:
Kai Tan
2026-10-08 08:32:05 +08:00
parent 498b5d09ce
commit 644e93b874
12 changed files with 186 additions and 20 deletions
+37
View File
@@ -228,3 +228,40 @@ def test_calibre_resolves_parent_part_destination(tmp_path,mode):
assert part['frag']=='part'
assert part['dest_exists'] is True
assert part['children'][0]['dest_exists'] is True
@pytest.mark.parametrize('mode', ['target_only', 'bilingual'])
def test_chinese_part_number_uses_uniform_ordinal_style(tmp_path, mode):
from src.preprocessing.format_extractor import FormatExtractor
source, work, structure, entries = fixture(tmp_path)
part = next(r for r in structure.resources.values() if r.href == 'Text/part.xhtml')
part.content = '<html><body><h1 id="part">PART <span class="c1">1</span><span class="heading_breakp">BOY GENIUS</span></h1></body></html>'
entry = next(e for e in entries if e.element_id == 'part')
_, text, mapping, _, _ = FormatExtractor().extract(str(BeautifulSoup(part.content, 'html.parser').h1))
entry.original_text = text
entry.placeholders = mapping
entry.translated_text = text.replace('PART ', '第').replace('BOY GENIUS', '天才少年')
# Preserve placeholder indices; insert the ordinal suffix after the number span.
closing = 'φ' + next(k for k, v in mapping.items() if v == '</span>') + 'φ'
entry.translated_text = entry.translated_text.replace(closing, closing + '部', 1)
asyncio.run(RestorationEngine().restore_entries([entry], force=True))
asyncio.run(BackfillEngine().backfill(structure, entries, mode))
soup = BeautifulSoup(part.content, 'html.parser')
translated = soup.find('h1', class_='translation')
assert translated.find('span', class_='c1')['data-translation-ordinal'] == 'true'
assert translated.find('span', class_='heading_breakp').get_text() == '天才少年'
assert not translated.find('span', class_='heading_breakp').has_attr('data-translation-ordinal')
if mode == 'bilingual':
assert not soup.find('h1', id='part').find('span', class_='c1').has_attr('data-translation-ordinal')
def test_complete_book_includes_backmatter_and_non_spine_documents(tmp_path):
_, _, structure, _ = fixture(tmp_path)
notes = next(r for r in structure.resources.values() if r.href == 'Text/notes.xhtml')
structure.resources['supplement'] = notes.model_copy(update={'href': 'Text/footnotes.xhtml'})
standard = FineGrainedExtractor().extract(structure)
complete = FineGrainedExtractor(complete_book=True).extract(structure)
assert not any(e.file_path == 'Text/footnotes.xhtml' for e in standard)
assert any(e.file_path == 'Text/footnotes.xhtml' and 'Note text' in e.original_text for e in complete)
assert FineGrainedExtractor(complete_book=True)._classify_document('index.xhtml') == 'core'
assert FineGrainedExtractor(complete_book=True)._classify_document('nav.xhtml') == 'toc'
+38
View File
@@ -15,6 +15,7 @@ from src.assembly.backfiller import BackfillEngine
from src.assembly.builder import BilingualBuilder
from src.translation.translator_engine import Translator
from src.translation.glossary import BookGlossary, matches
from src.translation.glossary import prepare_glossary
from src.translation.manifest_manager import ManifestManager
from src.translation.llm_client import LLMClient
from src.qa.epub_audit import audit_epub, compare_audits
@@ -26,6 +27,43 @@ def entry(html='<p id="p1">Masayoshi Son <em>built</em> a company.</p>'):
original_text=text, placeholders=mapping)
def test_glossary_model_conflicts_are_recorded_and_scan_resumes(tmp_path):
class Scanner:
model = 'test-model'
def __init__(self, fail=False): self.calls = 0; self.fail = fail
async def raw_chat_completion(self, *args):
self.calls += 1
if self.fail and self.calls == 2: raise RuntimeError('network interrupted')
return json.dumps({'terms': [
{'source': 'Masayoshi Son', 'target': '孙正义', 'aliases': ['Masa']},
{'source': 'Masa', 'target': '马萨', 'aliases': []}]})
entries = [entry(), entry('<p id="p2">Masa continued.</p>')]
path = tmp_path / 'glossary.json'
with pytest.raises(RuntimeError):
asyncio.run(prepare_glossary(entries, Scanner(True), path, chunk_chars=1))
saved = json.loads(path.read_text())
assert saved['next_chunk'] == 1 and not saved['scan_complete']
assert saved['terms'][0]['target'] == '孙正义'
assert saved['conflicts']
resumed = Scanner()
glossary = asyncio.run(prepare_glossary(entries, resumed, path, chunk_chars=1))
assert resumed.calls == 1
assert len(glossary.terms) == 1
assert json.loads(path.read_text())['scan_complete']
asyncio.run(prepare_glossary(entries, resumed, path, chunk_chars=1))
assert resumed.calls == 1
def test_zenmux_credentials_override_unrelated_openai_key(tmp_path, monkeypatch):
from src.common.config import load_global_config
config = tmp_path / 'zenmux.yaml'
config.write_text('llm:\n base_url: https://zenmux.ai/api/v1\n')
monkeypatch.setenv('ZENMUX_API_KEY', 'test-zenmux-key')
monkeypatch.setenv('OPENAI_API_KEY', 'test-other-key')
monkeypatch.delenv('OPENAI_BASE_URL', raising=False)
assert load_global_config(config)['llm']['api_key'] == 'test-zenmux-key'
@pytest.mark.parametrize('html', [
'<p>Hello <em><b>world</b></em>.</p>',
'<p>The angle φ = 10, and φ1φ is literal mathematical text.</p>',