Synchronize chapter and navigation translations with Chinese prose indentation
This commit is contained in:
Kai Tan
@@ -0,0 +1,181 @@
|
||||
import asyncio
|
||||
import zipfile
|
||||
from bs4 import BeautifulSoup
|
||||
from ebooklib import epub
|
||||
from lxml import etree
|
||||
import pytest
|
||||
from src.common.data_model import BookStructure
|
||||
from src.preprocessing.epub_cleaner import EpubCleaner
|
||||
from src.preprocessing.text_extractor import FineGrainedExtractor
|
||||
from src.preprocessing.navigation import prepare_navigation, destination, chinese_number
|
||||
from src.assembly.restoration_engine import RestorationEngine
|
||||
from src.assembly.backfiller import BackfillEngine
|
||||
from src.assembly.builder import BilingualBuilder
|
||||
from src.qa.epub_audit import audit_epub, compare_audits
|
||||
|
||||
|
||||
def fixture(tmp_path):
|
||||
book = epub.EpubBook(); book.set_identifier('titles'); book.set_title('Title QA'); book.set_language('en')
|
||||
chapter = epub.EpubHtml(uid='ch', file_name='Text/ch.xhtml', title='Roots')
|
||||
chapter.content = '''<html><head></head><body><h2 id="ch1"><span class="line">1</span>
|
||||
<span class="subtitle">ROOTS</span></h2><p id="opening" class="noindent" style="text-indent:0 !important; color:red">Opening prose.</p>
|
||||
<p id="caption" class="caption">A caption.</p><blockquote><p id="quote">Quoted text.</p></blockquote>
|
||||
<p id="second">Second paragraph.<a epub:type="noteref" href="notes.xhtml#note">1</a></p></body></html>'''
|
||||
part = epub.EpubHtml(uid='part', file_name='Text/part.xhtml', title='Part 1')
|
||||
part.content = '<html><body><h1 id="part"><span>PART 1</span><br/><em>BOY GENIUS</em></h1></body></html>'
|
||||
contents = epub.EpubHtml(uid='contents', file_name='Text/Contents.xhtml', title='Contents')
|
||||
contents.content = '''<html><body><h1 id="contents">Contents</h1><ol><li><a href="part.xhtml#part">Part 1: Boy Genius</a>
|
||||
<ol><li><a href="ch.xhtml#ch1"><em>Chapter 1: Roots</em></a></li></ol></li></ol></body></html>'''
|
||||
notes = epub.EpubHtml(uid='notes', file_name='Text/notes.xhtml', title='Notes')
|
||||
notes.content = '<html><body><h1 id="notes">Notes</h1><aside epub:type="footnote" id="note"><p id="note-text">Note text. <a href="ch.xhtml#second">Back</a></p></aside></body></html>'
|
||||
for item in [chapter,part,contents,notes,epub.EpubNav(),epub.EpubNcx()]: book.add_item(item)
|
||||
book.toc = [(epub.Section('Part 1: Boy Genius', 'Text/part.xhtml#part'),
|
||||
[epub.Link('Text/ch.xhtml#ch1','Chapter 1: Roots','ch1')]),
|
||||
epub.Link('Text/notes.xhtml#notes','Notes','notes')]
|
||||
book.spine = ['nav',contents,part,chapter,notes]
|
||||
source=tmp_path/'source.epub';epub.write_epub(str(source),book)
|
||||
work=tmp_path/'source'; structure=BookStructure.load(EpubCleaner(source,work).clean())
|
||||
entries=FineGrainedExtractor().extract(structure)
|
||||
prepare_navigation(source,structure,entries)
|
||||
for e in entries:
|
||||
e.translated_text=(e.original_text.replace('ROOTS','根源').replace('PART 1','第一部')
|
||||
.replace('BOY GENIUS','天才少年').replace('Contents','目录').replace('Notes','注释')
|
||||
.replace('Opening prose.','开篇正文。').replace('Second paragraph.','第二段正文。')
|
||||
.replace('A caption.','图注。').replace('Quoted text.','引文。'))
|
||||
assert asyncio.run(RestorationEngine().restore_entries(entries))==len(entries)
|
||||
return source,work,structure,entries
|
||||
|
||||
|
||||
@pytest.mark.parametrize('mode',['bilingual','target_only'])
|
||||
def test_headings_nav_ncx_contents_share_translation_and_keep_links(tmp_path,mode):
|
||||
source,work,structure,entries=fixture(tmp_path)
|
||||
chapter_titles=[e for e in entries if e.element_id=='ch1']
|
||||
assert len(chapter_titles)==1
|
||||
assert not any('Chapter 1: Roots' in e.original_text for e in entries)
|
||||
asyncio.run(BackfillEngine().backfill(structure,entries,mode))
|
||||
# Repeat safely: no duplicate bilingual paragraph or translated TOC label.
|
||||
asyncio.run(BackfillEngine().backfill(structure,entries,mode))
|
||||
output=tmp_path/(mode+'.epub');BilingualBuilder(work,source).build(structure,output)
|
||||
assert not compare_audits(audit_epub(source),audit_epub(output))
|
||||
with zipfile.ZipFile(source) as before,zipfile.ZipFile(output) as after:
|
||||
prefix=next(n for n in before.namelist() if n.endswith('/nav.xhtml')).removesuffix('nav.xhtml')
|
||||
for relative in ['nav.xhtml','Text/Contents.xhtml']:
|
||||
name=prefix+relative
|
||||
a=BeautifulSoup(before.read(name),'html.parser');b=BeautifulSoup(after.read(name),'html.parser')
|
||||
assert [x.get('href') for x in a.find_all('a')]==[x.get('href') for x in b.find_all('a')]
|
||||
link=next(x for x in b.find_all('a') if 'ch.xhtml' in x.get('href',''))
|
||||
assert '第一章：根源' in link.get_text()
|
||||
assert link.get_text().count('根源')==1
|
||||
assert link.find_parent('ol').find_parent('li') is not None
|
||||
a=etree.fromstring(before.read(prefix+'toc.ncx'));b=etree.fromstring(after.read(prefix+'toc.ncx'))
|
||||
assert a.xpath('//*[local-name()="content"]/@src')==b.xpath('//*[local-name()="content"]/@src')
|
||||
assert a.xpath('//*[local-name()="navPoint"]/@id')==b.xpath('//*[local-name()="navPoint"]/@id')
|
||||
assert '第一章：根源' in ''.join(b.itertext())
|
||||
ch=BeautifulSoup(after.read(prefix+'Text/ch.xhtml'),'html.parser')
|
||||
title=ch.find('h2',class_='translation')
|
||||
assert title.find('span',class_='line').get_text()=='第一章'
|
||||
assert title.find('span',class_='subtitle').get_text()=='根源'
|
||||
assert title['data-translation-role']=='heading'
|
||||
for key in ['opening','second']:
|
||||
p=ch.find(attrs={'data-translation-for':key}) if mode=='bilingual' else ch.find(id=key)
|
||||
assert p['data-translation-role']=='body'
|
||||
assert 'text-indent' not in p.get('style','')
|
||||
for key in ['caption','quote']:
|
||||
p=ch.find(attrs={'data-translation-for':key}) if mode=='bilingual' else ch.find(id=key)
|
||||
assert p['data-translation-role']=='other'
|
||||
notes=BeautifulSoup(after.read(prefix+'Text/notes.xhtml'),'html.parser')
|
||||
assert notes.find(id='note') and notes.find('a',href='ch.xhtml#second')
|
||||
|
||||
|
||||
def test_uncertain_destination_translates_label_without_overwriting_prose(tmp_path):
|
||||
source,work,structure,entries=fixture(tmp_path)
|
||||
# A link to prose must not be treated as the chapter heading.
|
||||
nav=next(r for r in structure.resources.values() if r.href=='nav.xhtml')
|
||||
nav.content=nav.content.replace('Text/ch.xhtml#ch1','Text/ch.xhtml#opening')
|
||||
entries=FineGrainedExtractor().extract(structure)
|
||||
prepare_navigation(source,structure,entries)
|
||||
record=next(r for r in structure.navigation_titles if r['target'].endswith('#opening'))
|
||||
assert record['heading_id'] is None
|
||||
assert record['binding_note']=='no matching heading'
|
||||
entry=next(e for e in entries if e.entry_id==record['entry_id'])
|
||||
assert entry.file_path=='__navigation__'
|
||||
assert 'Opening prose.' in next(r for r in structure.resources.values() if r.href=='Text/ch.xhtml').content
|
||||
|
||||
|
||||
def test_destination_preserves_canonical_fragment_and_ignores_external():
|
||||
assert destination('Nav/toc.xhtml','../Text/ch%20one.xhtml#part%202')=='Text/ch one.xhtml#part 2'
|
||||
assert destination('Text/ch.xhtml','#x')=='Text/ch.xhtml#x'
|
||||
assert destination('toc.xhtml','https://example.com/') is None
|
||||
assert [chinese_number(x) for x in [1,10,11,20,101,1001]]==['一','十','十一','二十','一百零一','一千零一']
|
||||
|
||||
|
||||
@pytest.mark.parametrize('markup',[
|
||||
'<h2 id="number">1</h2><h2 id="title">The Bookseller</h2><h2 id="subtitle">The Ren Family: 1937–1968</h2>',
|
||||
'<p id="number" class="calibre1">1</p><p id="title" class="calibre2"><em>The Bookseller</em></p><p id="subtitle" class="calibre3">The Ren Family: 1937–1968</p>',
|
||||
'<div id="number"><b>CHAPTER 1</b></div><div id="title"><b>The Bookseller</b></div><div id="subtitle">The Ren Family: 1937–1968</div>',
|
||||
])
|
||||
def test_separate_number_title_subtitle_are_shared_without_merging_dom(tmp_path,markup):
|
||||
from src.preprocessing.navigation import navigation_labels
|
||||
source,work,structure,entries=fixture(tmp_path)
|
||||
chapter=next(r for r in structure.resources.values() if r.href=='Text/ch.xhtml')
|
||||
chapter.content='<html><body>'+markup+'<p id="prose">A long unrelated opening paragraph.</p></body></html>'
|
||||
nav=next(r for r in structure.resources.values() if r.href=='nav.xhtml')
|
||||
nav.content=nav.content.replace('Text/ch.xhtml#ch1','Text/ch.xhtml').replace('Chapter 1: Roots','Chapter 1: The Bookseller | The Ren Family: 1937–1968')
|
||||
entries=FineGrainedExtractor().extract(structure);prepare_navigation(source,structure,entries)
|
||||
record=next(r for r in structure.navigation_titles if r['target']=='Text/ch.xhtml')
|
||||
assert len(record['entry_ids'])==3
|
||||
for e in entries:
|
||||
if e.element_id in {'number','title','subtitle'}:
|
||||
e.translated_text=e.original_text.replace('CHAPTER 1','第1章：').replace('The Bookseller','书商').replace('The Ren Family: 1937–1968','任家：1937—1968')
|
||||
asyncio.run(RestorationEngine().restore_entries(entries))
|
||||
labels=navigation_labels(structure,entries)
|
||||
assert labels['Text/ch.xhtml']=='第一章：书商 任家：1937—1968'
|
||||
asyncio.run(BackfillEngine().backfill(structure,entries,'target_only',allow_partial=True))
|
||||
ch=BeautifulSoup(chapter.content,'html.parser')
|
||||
assert all(ch.find(id=id) for id in ['number','title','subtitle','prose'])
|
||||
assert ch.find(id='prose').get_text()=='A long unrelated opening paragraph.'
|
||||
|
||||
|
||||
def test_heading_number_with_spaces_and_colon_normalizes():
|
||||
from src.preprocessing.navigation import normalize_heading_html,number_info
|
||||
assert normalize_heading_html({'role':'chapter','number':1},'<span>第 1 章：</span><em>根源</em>')=='<span>第一章</span><em>根源</em>'
|
||||
assert number_info('Chapter Eleven: Rebels')==('chapter',11)
|
||||
assert number_info('Part Two: Coffee')==('part',2)
|
||||
|
||||
|
||||
def test_split_styled_digit_and_separate_separator():
|
||||
from src.preprocessing.navigation import normalize_heading_html
|
||||
assert normalize_heading_html({'role':'part','number':1},'第<span>1</span>部：<em>天才少年</em>')=='第<span>一</span>部<em>天才少年</em>'
|
||||
assert normalize_heading_html({'role':'chapter','number':1},'<span>第1章</span>：<em>根源</em>')=='<span>第一章</span><em>根源</em>'
|
||||
|
||||
|
||||
def test_numbered_titles_starting_with_two_or_i_do_not_lose_title_words():
|
||||
from src.preprocessing.navigation import title_identity
|
||||
assert title_identity('Chapter 22: Two Indians Are Better than One')==title_identity('22 TWO INDIANS ARE BETTER THAN ONE')
|
||||
assert title_identity('Chapter 26: I Hope You Go Bust')==title_identity('26 I HOPE YOU GO BUST')
|
||||
|
||||
|
||||
def test_manifest_reconciliation_updates_title_context(tmp_path):
|
||||
from src.common.data_model import ManifestEntry
|
||||
from src.translation.manifest_manager import ManifestManager
|
||||
manager=ManifestManager(tmp_path/'manifest.json')
|
||||
old=ManifestEntry(entry_id='x#h',file_path='x',element_id='h',original_text='Roots',context='section',translated_text='根源')
|
||||
manager.add_entries([old])
|
||||
new=old.model_copy(update={'context':'navigation-heading: Chapter 1: Roots','translated_text':None})
|
||||
manager.add_entries([new])
|
||||
assert manager.entries[0].context==new.context
|
||||
assert manager.entries[0].translated_text=='根源'
|
||||
|
||||
|
||||
def test_ncx_declared_utf16_is_parsed_without_utf8_assumptions(tmp_path):
|
||||
source,work,structure,entries=fixture(tmp_path)
|
||||
with zipfile.ZipFile(source) as z:
|
||||
items=[(i,z.read(i.filename)) for i in z.infolist()]
|
||||
converted=tmp_path/'utf16.epub'
|
||||
with zipfile.ZipFile(converted,'w') as z:
|
||||
for info,data in items:
|
||||
if info.filename.endswith('.ncx'):
|
||||
data=etree.tostring(etree.fromstring(data),encoding='utf-16',xml_declaration=True)
|
||||
z.writestr(info,data)
|
||||
prepare_navigation(converted,structure,entries)
|
||||
assert any(r['source_label']=='Chapter 1: Roots' for r in structure.navigation_titles)
|
||||
Reference in New Issue
Block a user