330 lines
21 KiB
Python
330 lines
21 KiB
Python
import asyncio
|
||
import zipfile
|
||
from bs4 import BeautifulSoup
|
||
from ebooklib import epub
|
||
from lxml import etree
|
||
import pytest
|
||
from src.common.data_model import BookStructure
|
||
from src.preprocessing.epub_cleaner import EpubCleaner
|
||
from src.preprocessing.text_extractor import FineGrainedExtractor
|
||
from src.preprocessing.navigation import prepare_navigation, destination, chinese_number
|
||
from src.assembly.restoration_engine import RestorationEngine
|
||
from src.assembly.backfiller import BackfillEngine
|
||
from src.assembly.builder import BilingualBuilder
|
||
from src.qa.epub_audit import audit_epub, compare_audits
|
||
|
||
|
||
def fixture(tmp_path):
|
||
book = epub.EpubBook(); book.set_identifier('titles'); book.set_title('Title QA'); book.set_language('en')
|
||
chapter = epub.EpubHtml(uid='ch', file_name='Text/ch.xhtml', title='Roots')
|
||
chapter.content = '''<html><head></head><body><h2 id="ch1"><span class="line">1</span>
|
||
<span class="subtitle">ROOTS</span></h2><p id="opening" class="noindent" style="text-indent:0 !important; color:red">Opening prose.</p>
|
||
<p id="caption" class="caption">A caption.</p><blockquote><p id="quote">Quoted text.</p></blockquote>
|
||
<div role="doc-credits"><p id="credit" style="text-indent:0 !important">1: Photo credit.</p></div>
|
||
<p role="doc-endnote" id="semantic-note" style="text-indent:0 !important">A semantic endnote.</p>
|
||
<p id="second">Second paragraph.<a epub:type="noteref" href="notes.xhtml#note">1</a></p></body></html>'''
|
||
part = epub.EpubHtml(uid='part', file_name='Text/part.xhtml', title='Part 1')
|
||
part.content = '<html><body><h1 id="part"><span>PART 1</span><br/><em>BOY GENIUS</em></h1></body></html>'
|
||
contents = epub.EpubHtml(uid='contents', file_name='Text/Contents.xhtml', title='Contents')
|
||
contents.content = '''<html><body><h1 id="contents">Contents</h1><ol><li><a href="part.xhtml#part">Part 1: Boy Genius</a>
|
||
<ol><li><a href="ch.xhtml#ch1"><em>Chapter 1: Roots</em></a></li></ol></li></ol></body></html>'''
|
||
notes = epub.EpubHtml(uid='notes', file_name='Text/notes.xhtml', title='Notes')
|
||
notes.content = '<html><body><h1 id="notes">Notes</h1><aside epub:type="footnote" id="note"><p id="note-text">Note text. <a href="ch.xhtml#second">Back</a></p></aside></body></html>'
|
||
for item in [chapter,part,contents,notes,epub.EpubNav(),epub.EpubNcx()]: book.add_item(item)
|
||
book.toc = [(epub.Section('Part 1: Boy Genius', 'Text/part.xhtml#part'),
|
||
[epub.Link('Text/ch.xhtml#ch1','Chapter 1: Roots','ch1')]),
|
||
epub.Link('Text/notes.xhtml#notes','Notes','notes')]
|
||
book.spine = ['nav',contents,part,chapter,notes]
|
||
source=tmp_path/'source.epub';epub.write_epub(str(source),book)
|
||
work=tmp_path/'source'; structure=BookStructure.load(EpubCleaner(source,work).clean())
|
||
entries=FineGrainedExtractor().extract(structure)
|
||
prepare_navigation(source,structure,entries)
|
||
for e in entries:
|
||
e.translated_text=(e.original_text.replace('ROOTS','根源').replace('PART 1','第一部')
|
||
.replace('BOY GENIUS','天才少年').replace('Contents','目录').replace('Notes','注释')
|
||
.replace('Opening prose.','开篇正文。').replace('Second paragraph.','第二段正文。')
|
||
.replace('A caption.','图注。').replace('Quoted text.','引文。'))
|
||
assert asyncio.run(RestorationEngine().restore_entries(entries))==len(entries)
|
||
return source,work,structure,entries
|
||
|
||
|
||
@pytest.mark.parametrize('mode',['bilingual','target_only'])
|
||
def test_headings_nav_ncx_contents_share_translation_and_keep_links(tmp_path,mode):
|
||
source,work,structure,entries=fixture(tmp_path)
|
||
chapter_titles=[e for e in entries if e.element_id=='ch1']
|
||
assert len(chapter_titles)==1
|
||
assert not any('Chapter 1: Roots' in e.original_text for e in entries)
|
||
asyncio.run(BackfillEngine().backfill(structure,entries,mode))
|
||
# Repeat safely: no duplicate bilingual paragraph or translated TOC label.
|
||
asyncio.run(BackfillEngine().backfill(structure,entries,mode))
|
||
output=tmp_path/(mode+'.epub');BilingualBuilder(work,source).build(structure,output)
|
||
assert not compare_audits(audit_epub(source),audit_epub(output))
|
||
with zipfile.ZipFile(source) as before,zipfile.ZipFile(output) as after:
|
||
prefix=next(n for n in before.namelist() if n.endswith('/nav.xhtml')).removesuffix('nav.xhtml')
|
||
for relative in ['nav.xhtml','Text/Contents.xhtml']:
|
||
name=prefix+relative
|
||
a=BeautifulSoup(before.read(name),'html.parser');b=BeautifulSoup(after.read(name),'html.parser')
|
||
assert [x.get('href') for x in a.find_all('a')]==[x.get('href') for x in b.find_all('a')]
|
||
link=next(x for x in b.find_all('a') if 'ch.xhtml' in x.get('href',''))
|
||
assert '第一章：根源' in link.get_text()
|
||
assert link.get_text().count('根源')==1
|
||
assert link.find_parent('ol').find_parent('li') is not None
|
||
if relative == 'nav.xhtml':
|
||
parent_link = b.find('a', href='Text/part.xhtml#part')
|
||
assert parent_link.parent.name == 'li'
|
||
assert parent_link.parent.find('ol', recursive=False) is not None
|
||
a=etree.fromstring(before.read(prefix+'toc.ncx'));b=etree.fromstring(after.read(prefix+'toc.ncx'))
|
||
assert a.xpath('//*[local-name()="content"]/@src')==b.xpath('//*[local-name()="content"]/@src')
|
||
assert a.xpath('//*[local-name()="navPoint"]/@id')==b.xpath('//*[local-name()="navPoint"]/@id')
|
||
assert '第一章：根源' in ''.join(b.itertext())
|
||
ch=BeautifulSoup(after.read(prefix+'Text/ch.xhtml'),'html.parser')
|
||
title=ch.find('h2',class_='translation')
|
||
assert title.find('span',class_='line').get_text()=='第一章'
|
||
assert title.find('span',class_='subtitle').get_text()=='根源'
|
||
assert title['data-translation-role']=='heading'
|
||
for key in ['opening','second']:
|
||
p=ch.find(attrs={'data-translation-for':key}) if mode=='bilingual' else ch.find(id=key)
|
||
assert p['data-translation-role']=='body'
|
||
assert 'text-indent' not in p.get('style','')
|
||
for key in ['caption','quote','credit','semantic-note']:
|
||
p=ch.find(attrs={'data-translation-for':key}) if mode=='bilingual' else ch.find(id=key)
|
||
assert p['data-translation-role']=='other'
|
||
if key in {'credit','semantic-note'}:
|
||
assert 'text-indent:0 !important' in p['style']
|
||
notes=BeautifulSoup(after.read(prefix+'Text/notes.xhtml'),'html.parser')
|
||
assert notes.find(id='note') and notes.find('a',href='ch.xhtml#second')
|
||
|
||
|
||
def test_uncertain_destination_translates_label_without_overwriting_prose(tmp_path):
|
||
source,work,structure,entries=fixture(tmp_path)
|
||
# A link to prose must not be treated as the chapter heading.
|
||
nav=next(r for r in structure.resources.values() if r.href=='nav.xhtml')
|
||
nav.content=nav.content.replace('Text/ch.xhtml#ch1','Text/ch.xhtml#opening')
|
||
entries=FineGrainedExtractor().extract(structure)
|
||
prepare_navigation(source,structure,entries)
|
||
record=next(r for r in structure.navigation_titles if r['target'].endswith('#opening'))
|
||
assert record['heading_id'] is None
|
||
assert record['binding_note']=='no matching heading'
|
||
entry=next(e for e in entries if e.entry_id==record['entry_id'])
|
||
assert entry.file_path=='__navigation__'
|
||
assert 'Opening prose.' in next(r for r in structure.resources.values() if r.href=='Text/ch.xhtml').content
|
||
|
||
|
||
def test_destination_preserves_canonical_fragment_and_ignores_external():
|
||
assert destination('Nav/toc.xhtml','../Text/ch%20one.xhtml#part%202')=='Text/ch one.xhtml#part 2'
|
||
assert destination('Text/ch.xhtml','#x')=='Text/ch.xhtml#x'
|
||
assert destination('toc.xhtml','https://example.com/') is None
|
||
assert [chinese_number(x) for x in [1,10,11,20,101,1001]]==['一','十','十一','二十','一百零一','一千零一']
|
||
|
||
|
||
@pytest.mark.parametrize('markup',[
|
||
'<h2 id="number">1</h2><h2 id="title">The Bookseller</h2><h2 id="subtitle">The Ren Family: 1937–1968</h2>',
|
||
'<p id="number" class="calibre1">1</p><p id="title" class="calibre2"><em>The Bookseller</em></p><p id="subtitle" class="calibre3">The Ren Family: 1937–1968</p>',
|
||
'<div id="number"><b>CHAPTER 1</b></div><div id="title"><b>The Bookseller</b></div><div id="subtitle">The Ren Family: 1937–1968</div>',
|
||
])
|
||
def test_separate_number_title_subtitle_are_shared_without_merging_dom(tmp_path,markup):
|
||
from src.preprocessing.navigation import navigation_labels
|
||
source,work,structure,entries=fixture(tmp_path)
|
||
chapter=next(r for r in structure.resources.values() if r.href=='Text/ch.xhtml')
|
||
chapter.content='<html><body>'+markup+'<p id="prose">A long unrelated opening paragraph.</p></body></html>'
|
||
nav=next(r for r in structure.resources.values() if r.href=='nav.xhtml')
|
||
nav.content=nav.content.replace('Text/ch.xhtml#ch1','Text/ch.xhtml').replace('Chapter 1: Roots','Chapter 1: The Bookseller | The Ren Family: 1937–1968')
|
||
entries=FineGrainedExtractor().extract(structure);prepare_navigation(source,structure,entries)
|
||
record=next(r for r in structure.navigation_titles if r['target']=='Text/ch.xhtml')
|
||
assert len(record['entry_ids'])==3
|
||
for e in entries:
|
||
if e.element_id in {'number','title','subtitle'}:
|
||
e.translated_text=e.original_text.replace('CHAPTER 1','第1章：').replace('The Bookseller','书商').replace('The Ren Family: 1937–1968','任家：1937—1968')
|
||
asyncio.run(RestorationEngine().restore_entries(entries))
|
||
labels=navigation_labels(structure,entries)
|
||
assert labels['Text/ch.xhtml']=='第一章：书商 任家：1937—1968'
|
||
asyncio.run(BackfillEngine().backfill(structure,entries,'target_only',allow_partial=True))
|
||
ch=BeautifulSoup(chapter.content,'html.parser')
|
||
assert all(ch.find(id=id) for id in ['number','title','subtitle','prose'])
|
||
assert ch.find(id='prose').get_text()=='A long unrelated opening paragraph.'
|
||
|
||
|
||
def test_heading_number_with_spaces_and_colon_normalizes():
|
||
from src.preprocessing.navigation import normalize_heading_html,number_info
|
||
assert normalize_heading_html({'role':'chapter','number':1},'<span>第 1 章：</span><em>根源</em>')=='<span>第一章</span><em>根源</em>'
|
||
assert number_info('Chapter Eleven: Rebels')==('chapter',11)
|
||
assert number_info('Part Two: Coffee')==('part',2)
|
||
|
||
|
||
def test_split_styled_digit_and_separate_separator():
|
||
from src.preprocessing.navigation import normalize_heading_html
|
||
assert normalize_heading_html({'role':'part','number':1},'第<span>1</span>部：<em>天才少年</em>')=='第<span>一</span>部<em>天才少年</em>'
|
||
assert normalize_heading_html({'role':'chapter','number':1},'<span>第1章</span>：<em>根源</em>')=='<span>第一章</span><em>根源</em>'
|
||
|
||
|
||
def test_numbered_titles_starting_with_two_or_i_do_not_lose_title_words():
|
||
from src.preprocessing.navigation import title_identity
|
||
assert title_identity('Chapter 22: Two Indians Are Better than One')==title_identity('22 TWO INDIANS ARE BETTER THAN ONE')
|
||
assert title_identity('Chapter 26: I Hope You Go Bust')==title_identity('26 I HOPE YOU GO BUST')
|
||
|
||
|
||
def test_manifest_reconciliation_updates_title_context(tmp_path):
|
||
from src.common.data_model import ManifestEntry
|
||
from src.translation.manifest_manager import ManifestManager
|
||
manager=ManifestManager(tmp_path/'manifest.json')
|
||
old=ManifestEntry(entry_id='x#h',file_path='x',element_id='h',original_text='Roots',context='section',translated_text='根源')
|
||
manager.add_entries([old])
|
||
new=old.model_copy(update={'context':'navigation-heading: Chapter 1: Roots','translated_text':None})
|
||
manager.add_entries([new])
|
||
assert manager.entries[0].context==new.context
|
||
assert manager.entries[0].translated_text=='根源'
|
||
|
||
|
||
def test_ncx_declared_utf16_is_parsed_without_utf8_assumptions(tmp_path):
|
||
source,work,structure,entries=fixture(tmp_path)
|
||
with zipfile.ZipFile(source) as z:
|
||
items=[(i,z.read(i.filename)) for i in z.infolist()]
|
||
converted=tmp_path/'utf16.epub'
|
||
with zipfile.ZipFile(converted,'w') as z:
|
||
for info,data in items:
|
||
if info.filename.endswith('.ncx'):
|
||
data=etree.tostring(etree.fromstring(data),encoding='utf-16',xml_declaration=True)
|
||
z.writestr(info,data)
|
||
prepare_navigation(converted,structure,entries)
|
||
assert any(r['source_label']=='Chapter 1: Roots' for r in structure.navigation_titles)
|
||
|
||
|
||
def test_navigation_generated_wrappers_are_repaired_but_publisher_styles_remain():
|
||
from src.common.html import parse_html, translation_targets
|
||
soup=parse_html('<nav epub:type="toc"><ol><li><span data-translation-unit="true" id="tr-old"><a href="part.xhtml"><em>Part 1</em></a></span><ol><li><a href="ch.xhtml">Chapter 1</a></li></ol></li></ol></nav>')
|
||
translation_targets(soup)
|
||
link=soup.find('a',href='part.xhtml')
|
||
assert link.parent.name=='li'
|
||
assert link.em.get_text()=='Part 1'
|
||
assert link.parent.find('ol',recursive=False)
|
||
translation_targets(soup)
|
||
assert link.parent.name=='li'
|
||
|
||
|
||
def test_prepare_navigation_upgrades_cached_wrapped_part_link(tmp_path):
|
||
source,work,structure,entries=fixture(tmp_path)
|
||
nav=next(r for r in structure.resources.values() if r.href=='nav.xhtml')
|
||
soup=BeautifulSoup(nav.content,'html.parser')
|
||
link=soup.find('a',href='Text/part.xhtml#part')
|
||
wrapper=soup.new_tag('span',attrs={'data-translation-unit':'true','id':'tr-old'})
|
||
link.wrap(wrapper);nav.content=str(soup)
|
||
prepare_navigation(source,structure,entries)
|
||
assert BeautifulSoup(nav.content,'html.parser').find('a',href='Text/part.xhtml#part').parent.name=='li'
|
||
|
||
|
||
@pytest.mark.parametrize('mode',['target_only','bilingual'])
|
||
def test_calibre_resolves_parent_part_destination(tmp_path,mode):
|
||
import json
|
||
import shutil
|
||
import subprocess
|
||
calibre=shutil.which('calibre-debug')
|
||
if not calibre: pytest.skip('Calibre integration check requires calibre-debug')
|
||
source,work,structure,entries=fixture(tmp_path)
|
||
asyncio.run(BackfillEngine().backfill(structure,entries,mode))
|
||
output=tmp_path/'reader.epub';BilingualBuilder(work,source).build(structure,output)
|
||
script=('import json,sys; from calibre.ebooks.oeb.polish.container import get_container; '
|
||
'from calibre.ebooks.oeb.polish.toc import get_toc; '
|
||
'print(json.dumps(get_toc(get_container(sys.argv[-1])).as_dict))')
|
||
result=subprocess.run([calibre,'-c',script,'--',str(output)],capture_output=True,text=True,check=True,timeout=30)
|
||
toc=json.loads(result.stdout)
|
||
part=next(n for n in toc['children'] if n['children'])
|
||
assert part['dest'].endswith('Text/part.xhtml')
|
||
assert part['frag']=='part'
|
||
assert part['dest_exists'] is True
|
||
assert part['children'][0]['dest_exists'] is True
|
||
|
||
|
||
@pytest.mark.parametrize('mode', ['target_only', 'bilingual'])
|
||
def test_chinese_part_number_uses_uniform_ordinal_style(tmp_path, mode):
|
||
from src.preprocessing.format_extractor import FormatExtractor
|
||
source, work, structure, entries = fixture(tmp_path)
|
||
part = next(r for r in structure.resources.values() if r.href == 'Text/part.xhtml')
|
||
part.content = '<html><body><h1 id="part">PART <span class="c1">1</span><span class="heading_breakp">BOY GENIUS</span></h1></body></html>'
|
||
entry = next(e for e in entries if e.element_id == 'part')
|
||
_, text, mapping, _, _ = FormatExtractor().extract(str(BeautifulSoup(part.content, 'html.parser').h1))
|
||
entry.original_text = text
|
||
entry.placeholders = mapping
|
||
entry.translated_text = text.replace('PART ', '第').replace('BOY GENIUS', '天才少年')
|
||
# Preserve placeholder indices; insert the ordinal suffix after the number span.
|
||
closing = 'φ' + next(k for k, v in mapping.items() if v == '</span>') + 'φ'
|
||
entry.translated_text = entry.translated_text.replace(closing, closing + '部', 1)
|
||
asyncio.run(RestorationEngine().restore_entries([entry], force=True))
|
||
asyncio.run(BackfillEngine().backfill(structure, entries, mode))
|
||
soup = BeautifulSoup(part.content, 'html.parser')
|
||
translated = soup.find('h1', class_='translation')
|
||
assert translated.find('span', class_='c1')['data-translation-ordinal'] == 'true'
|
||
assert translated.find('span', class_='heading_breakp').get_text() == '天才少年'
|
||
assert not translated.find('span', class_='heading_breakp').has_attr('data-translation-ordinal')
|
||
if mode == 'bilingual':
|
||
assert not soup.find('h1', id='part').find('span', class_='c1').has_attr('data-translation-ordinal')
|
||
|
||
|
||
def test_complete_book_includes_backmatter_and_non_spine_documents(tmp_path):
|
||
_, _, structure, _ = fixture(tmp_path)
|
||
notes = next(r for r in structure.resources.values() if r.href == 'Text/notes.xhtml')
|
||
structure.resources['supplement'] = notes.model_copy(update={'href': 'Text/footnotes.xhtml'})
|
||
standard = FineGrainedExtractor().extract(structure)
|
||
complete = FineGrainedExtractor(complete_book=True).extract(structure)
|
||
assert not any(e.file_path == 'Text/footnotes.xhtml' for e in standard)
|
||
assert any(e.file_path == 'Text/footnotes.xhtml' and 'Note text' in e.original_text for e in complete)
|
||
assert FineGrainedExtractor(complete_book=True)._classify_document('index.xhtml') == 'core'
|
||
assert FineGrainedExtractor(complete_book=True)._classify_document('nav.xhtml') == 'toc'
|
||
|
||
|
||
def test_number_only_part_heading_is_bound_without_matching_prose():
|
||
from src.common.html import parse_html
|
||
from src.preprocessing.navigation import _headings, normalize_heading_html
|
||
soup = parse_html('<h1 id="part">Part IV</h1><p>Unrelated epigraph.</p>')
|
||
headings, reason = _headings(soup, 'part.xhtml', 'Part IV')
|
||
assert reason is None and [h['id'] for h in headings] == ['part']
|
||
assert _headings(parse_html('<p>Part IV</p>'), 'part.xhtml', 'Part IV')[0] == []
|
||
assert normalize_heading_html({'role': 'part', 'number': 4}, '第四部分') == '第四部'
|
||
|
||
|
||
def test_nested_toc_anchor_binds_heading_and_normalizes_visible_ordinal():
|
||
from src.common.html import parse_html
|
||
from src.preprocessing.navigation import _headings, normalize_heading_html
|
||
soup = parse_html('<h2 id="ch1"><a id="sec3" href="contents.xhtml#rsec3">'
|
||
'<span class="sr-only">Chapter</span><span class="chno">1</span> '
|
||
'<span class="chaptit">Obsession</span></a></h2>'
|
||
'<p class="stars">° ° °</p><p>Unrelated prose.</p>')
|
||
headings, reason = _headings(soup, 'chapter.xhtml#sec3', 'Chapter 1. Obsession')
|
||
assert reason is None and [h['id'] for h in headings] == ['ch1']
|
||
translated = str(soup.h2.a).replace('Chapter', '第一章').replace('Obsession', '痴迷')
|
||
record = {'role': 'chapter', 'number': 1}
|
||
normalized = normalize_heading_html(record, translated)
|
||
result = parse_html(normalized)
|
||
assert result.find(class_='chno').get_text() == '第一章'
|
||
assert result.find(class_='sr-only')['aria-hidden'] == 'true'
|
||
assert result.find('a')['id'] == 'sec3'
|
||
assert result.find('a')['href'] == 'contents.xhtml#rsec3'
|
||
assert normalize_heading_html(record, normalized) == normalized
|
||
from src.common.data_model import BookMetaData, ManifestEntry
|
||
from src.preprocessing.navigation import navigation_labels
|
||
entry = ManifestEntry(entry_id='chapter.xhtml#ch1', file_path='chapter.xhtml', element_id='ch1',
|
||
original_text='Chapter 1 Obsession', translated_text='第一章 痴迷',
|
||
translated_html=normalized)
|
||
structure = BookStructure(metadata=BookMetaData(), navigation_titles=[{
|
||
**record, 'entry_id': entry.entry_id, 'target': 'chapter.xhtml#sec3'}])
|
||
assert navigation_labels(structure, [entry])['chapter.xhtml#sec3'] == '第一章：痴迷'
|
||
split = '<span class="sr-only">第</span><span class="chno">九</span>章 <span>考试帝国扩张</span>'
|
||
ninth = parse_html(normalize_heading_html({'role':'chapter','number':9}, split))
|
||
assert ninth.find(class_='chno').get_text() == '第九章'
|
||
assert '章章' not in ninth.get_text()
|
||
assert not any(str(n).strip() == '章' for n in ninth.find_all(string=True))
|
||
# A nested anchor in ordinary prose must still not become a title.
|
||
assert not _headings(parse_html('<p><a id="prose">A sentence.</a></p>'),
|
||
'chapter.xhtml#prose', 'Chapter 1. Obsession')[0]
|
||
|
||
|
||
def test_graphical_title_binds_using_alt_without_mutating_image():
|
||
from src.common.html import parse_html
|
||
from src.preprocessing.navigation import _headings
|
||
soup = parse_html('<section id="ch"><h1 id="heading">CHAPTER 1'
|
||
'<span><img src="word.jpeg" alt="Protocols."/></span>'
|
||
'<span><i>for</i> SLEEP</span></h1></section>')
|
||
before = str(soup)
|
||
headings, reason = _headings(soup, 'ch.xhtml#ch', 'Chapter 1: Protocols for Sleep')
|
||
assert reason is None and [h['id'] for h in headings] == ['heading']
|
||
assert str(soup) == before
|