Add EPUB translation skill and headless full-book QA with Luna validation
This commit is contained in:
Kai Tan
Vendored
+68
@@ -0,0 +1,68 @@
|
||||
"""Original miniature book used for Luna full-book forward testing. Pass an output directory."""
|
||||
from ebooklib import epub
|
||||
from pathlib import Path
|
||||
|
||||
import sys
|
||||
OUT = Path(sys.argv[1]) if len(sys.argv) > 1 else Path('.work/luna-fixture')
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
book = epub.EpubBook()
|
||||
book.set_identifier('urn:uuid:6a7c4302-c2ab-4e0d-b385-luna-test-2026')
|
||||
book.set_title('The Lantern Ledger: A Small Book About Big Decisions')
|
||||
book.set_language('en')
|
||||
book.add_author('Mira Vale')
|
||||
book.add_metadata('DC', 'description', 'An original miniature nonfiction-style book created for a complete EPUB translation workflow test.')
|
||||
book.add_metadata('DC', 'publisher', 'Fieldglass Press')
|
||||
book.add_metadata('DC', 'date', '2026')
|
||||
book.add_metadata('DC', 'rights', 'Copyright © 2026 Mira Vale. All rights reserved.')
|
||||
|
||||
style = '''body { font-family: serif; line-height: 1.45; } h1 { margin-top: 1.8em; } .opening { font-size: 1.08em; } .caption { font-size: .9em; font-style: italic; } .small { font-size: .85em; }'''
|
||||
css = epub.EpubItem(uid='style', file_name='style/book.css', media_type='text/css', content=style)
|
||||
book.add_item(css)
|
||||
|
||||
svg = '''<svg xmlns="http://www.w3.org/2000/svg" width="720" height="360" viewBox="0 0 720 360"><rect width="720" height="360" fill="#e8edf0"/><path d="M0 270 Q180 220 360 270 T720 250 V360 H0Z" fill="#778b84"/><path d="M95 255 L180 150 L265 255Z" fill="#34464b"/><rect x="168" y="205" width="25" height="50" fill="#f6c96a"/><circle cx="180" cy="176" r="23" fill="#f6c96a" opacity=".85"/><path d="M440 265 L525 120 L610 265Z" fill="#43565b"/><path d="M460 230 H590" stroke="#d9e0dc" stroke-width="8"/><text x="360" y="325" text-anchor="middle" font-family="serif" font-size="20" fill="#24363a">A light kept for the next decision</text></svg>'''
|
||||
img = epub.EpubItem(uid='lantern-art', file_name='images/lantern.svg', media_type='image/svg+xml', content=svg)
|
||||
book.add_item(img)
|
||||
|
||||
def page(uid, title, body, file_name=None):
|
||||
item = epub.EpubHtml(title=title, file_name=file_name or f'{uid}.xhtml', lang='en', uid=uid)
|
||||
item.content = f'''<html xmlns="http://www.w3.org/1999/xhtml"><head><title>{title}</title><link rel="stylesheet" type="text/css" href="style/book.css"/></head><body>{body}</body></html>'''
|
||||
book.add_item(item)
|
||||
return item
|
||||
|
||||
cover = page('cover', 'The Lantern Ledger', '''<h1>The Lantern Ledger</h1><p class="opening">A Small Book About Big Decisions</p><p>by Mira Vale</p><p class="small">An original translation-workflow test edition</p>''')
|
||||
intro = page('opening-note', 'Opening Note', '''<h1>Opening Note</h1><p class="opening">At dusk, a room can make a plan look smaller than it is. The table holds one lamp, three cups, and a map whose edges curl upward. Outside, rain edits the street into reflections.</p><p>This little book follows the work of deciding: how people gather evidence, choose a direction, and explain what they have chosen. Its scenes are deliberately compact, but every section belongs to the complete book.</p><p>Names and examples appear here to exercise translation details. The imagined scenes are not reports of actual meetings.</p>''')
|
||||
|
||||
p1c1 = page('ch1', 'Part I · The Signal', '''<h1>Part I</h1><p>The Signal</p><h2>Chapter 1: A Question in the Margin</h2><p class="opening">The first clue was not a number. It was a question pencilled beside a number: “What would make this true?”</p><p>At the fictional company Northstar Works, analyst Lena Park kept a ledger of claims and their sources. She did not distrust confident people; she simply wanted confidence to leave a trail.</p><p>When the team discussed a distant market, one slide showed a steep line climbing into the future. Lena wrote down the assumptions beneath it: customers would arrive, costs would fall, and the calendar would behave.</p><p>That last assumption made everyone laugh. Then they checked it.</p><p>A useful forecast, she said, is not a promise. It is a map of what must happen for a promise to become plausible.</p><p>Her margin question traveled from the meeting room to the workshop wall, where it stayed long after the slide was replaced.</p>''')
|
||||
|
||||
p1c2 = page('ch2', 'Part I · The Signal', '''<h2>Chapter 2: The Name on the Door</h2><p>In a translation exercise, a short name can carry a long shadow. A profile may introduce Masayoshi Son and later call him Masa. Those references point to one person; the shorter form is a familiar name, not a second executive.</p><p>Our fictional narrator imagines a conference hallway where an investor pauses at a hand-drawn chart. The point of the scene is not to claim that the encounter happened. It is to test whether a translator can preserve the distinction between a real public name and an invented setting.</p><p>“Keep the subject clear,” Lena tells a junior editor. “A pronoun should not make the reader guess which person is speaking.”</p><p>The editor adds a note to the style sheet: introduce the full name first, then use the established short form consistently.</p>''')
|
||||
|
||||
p2c1 = page('ch3', 'Part II · The Choice', '''<h1>Part II</h1><p>The Choice</p><h2>Chapter 3: What the Photograph Leaves Out</h2><p class="opening">A photograph can show a lamp, but it cannot show who paid the electricity bill.</p><figure><img src="images/lantern.svg" alt="A small glowing lantern stands between two dark hills."/><figcaption class="caption">Figure 1. The lantern marks a pause between what is known and what remains to be checked.</figcaption></figure><p>When a picture enters a book, its caption becomes part of the argument. It identifies what the reader sees and, just as carefully, avoids claiming what the image cannot prove.</p><p>The team labels the illustration as a conceptual drawing. No real person or office is pictured. That short sentence protects the reader from mistaking atmosphere for evidence.</p><p>Lena places the image beside a paragraph about uncertainty. The lamp does not answer the question; it makes the question easier to see.</p>''')
|
||||
|
||||
p2c2 = page('ch4', 'Part II · The Choice', '''<h2>Chapter 4: The Small Print</h2><p>A decision memo had one footnote: “The sample covers three districts, not the whole country.”<a id="callout-1" href="notes.xhtml#note-1" epub:type="noteref">1</a> Without that sentence, the chart looked broader than its evidence.</p><p>In review, Lena asked the writer to keep the note close to the claim. Readers should not have to search several pages to learn the boundary of a number.</p><p>The note did not weaken the recommendation. It made the recommendation precise enough to use.</p><p>At the bottom of the page, the editor added a return link so a reader could move from note to sentence and back again.</p>''')
|
||||
|
||||
notes = page('notes', 'Notes', '''<h1>Notes</h1><ol><li id="note-1" epub:type="footnote">The district count is invented for this original test book. <a href="ch4.xhtml#callout-1" epub:type="backlink">Return to the text.</a></li></ol>''')
|
||||
index = page('index', 'Index', '''<h1>Index</h1><p><a href="ch2.xhtml">Masayoshi Son, 8</a></p><p><a href="ch2.xhtml">Masa, 8</a></p><p><a href="ch1.xhtml">Northstar Works, 6</a></p><p><a href="ch3.xhtml">Photographs and captions, 10–11</a></p><p><a href="ch4.xhtml">Uncertainty, 12–13</a></p>''')
|
||||
rights = page('rights', 'Copyright', '''<h1>Copyright</h1><p>Copyright © 2026 Mira Vale.</p><p>All rights reserved. This original miniature book was written for software workflow evaluation. No portion is derived from an existing published book.</p><p>Illustration: “A light kept for the next decision,” created for this edition.</p><p>Published by Fieldglass Press, 2026.</p>''')
|
||||
|
||||
book.toc = [
|
||||
epub.Link('opening-note.xhtml', 'Opening Note', 'opening-note'),
|
||||
(epub.Section('Part I · The Signal'), [
|
||||
epub.Link('ch1.xhtml', 'Chapter 1: A Question in the Margin', 'ch1'),
|
||||
epub.Link('ch2.xhtml', 'Chapter 2: The Name on the Door', 'ch2'),
|
||||
]),
|
||||
(epub.Section('Part II · The Choice'), [
|
||||
epub.Link('ch3.xhtml', 'Chapter 3: What the Photograph Leaves Out', 'ch3'),
|
||||
epub.Link('ch4.xhtml', 'Chapter 4: The Small Print', 'ch4'),
|
||||
]),
|
||||
epub.Link('notes.xhtml', 'Notes', 'notes'),
|
||||
epub.Link('index.xhtml', 'Index', 'index'),
|
||||
epub.Link('rights.xhtml', 'Copyright', 'rights'),
|
||||
]
|
||||
book.spine = [cover, intro, p1c1, p1c2, p2c1, p2c2, notes, index, rights]
|
||||
book.add_item(epub.EpubNcx())
|
||||
|
||||
# The title page is the opening spine item; the illustration appears in Chapter 3.
|
||||
|
||||
outfile = OUT / 'source.epub'
|
||||
epub.write_epub(str(outfile), book)
|
||||
print(outfile)
|
||||
@@ -0,0 +1,201 @@
|
||||
import asyncio
|
||||
import json
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from ebooklib import epub
|
||||
from src.common.data_model import BookStructure
|
||||
from src.preprocessing.epub_cleaner import EpubCleaner
|
||||
from src.preprocessing.text_extractor import FineGrainedExtractor
|
||||
from src.preprocessing.navigation import prepare_navigation
|
||||
from src.assembly.restoration_engine import RestorationEngine
|
||||
from src.assembly.backfiller import BackfillEngine
|
||||
from src.assembly.builder import BilingualBuilder
|
||||
from src.qa.headless import audit_pair, book_inventory, notes_report
|
||||
|
||||
|
||||
def source_book(tmp_path, gateway=False):
|
||||
book = epub.EpubBook(); book.set_identifier('headless'); book.set_title('QA'); book.set_language('en')
|
||||
ch = epub.EpubHtml(uid='chapter', file_name='Text/ch.xhtml', title='Chapter 1')
|
||||
href = 'notes.xhtml' if gateway else 'notes.xhtml#n1'
|
||||
ch.content = '<h1 id="title">Chapter 1</h1>' + ''.join(
|
||||
f'<p id="p{i}">Text {i}.<a role="doc-noteref" href="{href}">{i}</a></p>' for i in range(4))
|
||||
notes = epub.EpubHtml(uid='notes', file_name='Text/notes.xhtml', title='Notes')
|
||||
notes.content = ('<h1>Author note</h1><p>Endnotes at <a href="https://example.test/endnotes">the website</a>.</p>' if gateway else
|
||||
'<aside epub:type="footnote" id="n1"><p id="note-text">A note. <a href="ch.xhtml#p0">Back</a></p></aside>')
|
||||
for item in [ch, notes, epub.EpubNav(), epub.EpubNcx()]: book.add_item(item)
|
||||
book.toc = [epub.Link('Text/ch.xhtml#title', 'Chapter 1', 'chapter')]; book.spine = ['nav', ch, notes]
|
||||
source = tmp_path/'source.epub'; epub.write_epub(str(source), book)
|
||||
return source
|
||||
|
||||
|
||||
def translated_book(tmp_path):
|
||||
source = source_book(tmp_path); work = tmp_path/'work'
|
||||
structure = BookStructure.load(EpubCleaner(source, work).clean())
|
||||
entries = FineGrainedExtractor(complete_book=True).extract(structure)
|
||||
prepare_navigation(source, structure, entries); structure.save(work/'book_structure.json')
|
||||
for e in entries: e.translated_text = e.original_text.replace('Text', '正文').replace('Chapter 1', '第一章').replace('A note.', '一条注释。')
|
||||
assert asyncio.run(RestorationEngine().restore_entries(entries)) == len(entries)
|
||||
manifest = work/'manifest.json'; manifest.write_text(json.dumps([e.model_dump() for e in entries]))
|
||||
asyncio.run(BackfillEngine().backfill(structure, entries, 'target_only'))
|
||||
output = tmp_path/'translated.epub'; BilingualBuilder(work, source).build(structure, output)
|
||||
return source, output, manifest, work/'book_structure.json'
|
||||
|
||||
|
||||
def test_external_gateway_is_visible_but_not_a_new_translation_fault(tmp_path):
|
||||
source = source_book(tmp_path, gateway=True)
|
||||
inventory = book_inventory(source); notes = notes_report(inventory)
|
||||
assert notes['counts'] == {'external_notes_gateway': 4}
|
||||
assert notes['findings'][0]['count'] == 4
|
||||
assert 'example.test/endnotes' in notes['references'][0]['external_urls'][0]
|
||||
report, _, _ = audit_pair(source, source)
|
||||
assert not report['new_issues']
|
||||
assert report['notes']['translated']['findings']
|
||||
assert report['visual_review'] == 'pending'
|
||||
|
||||
|
||||
def test_embedded_note_destination_is_distinguished(tmp_path):
|
||||
source = source_book(tmp_path)
|
||||
assert notes_report(book_inventory(source))['counts'] == {'embedded_note': 4}
|
||||
|
||||
|
||||
def test_full_coverage_and_backfill_resolve_opf_relative_manifest_paths(tmp_path):
|
||||
source, output, manifest, structure = translated_book(tmp_path)
|
||||
report, _, _ = audit_pair(source, output, manifest, structure)
|
||||
assert not report['new_issues']
|
||||
assert report['coverage']['scope'] == 'complete-book extraction'
|
||||
assert report['coverage']['total'] == report['coverage']['accepted']
|
||||
# A valid manifest must not prove that an untouched original was backfilled.
|
||||
bad, _, _ = audit_pair(source, source, manifest, structure)
|
||||
assert any(e['kind'] == 'translation_not_backfilled' for e in bad['new_issues'])
|
||||
|
||||
|
||||
def test_removed_manifest_unit_cannot_claim_complete_book(tmp_path):
|
||||
source, output, manifest, structure = translated_book(tmp_path)
|
||||
data = json.loads(manifest.read_text()); removed = data.pop()['entry_id']; manifest.write_text(json.dumps(data))
|
||||
report, _, _ = audit_pair(source, output, manifest, structure)
|
||||
assert removed in report['coverage']['missing_ids']
|
||||
assert any(e['kind'] == 'incomplete_or_mismatched_manifest' for e in report['new_issues'])
|
||||
|
||||
|
||||
def test_changed_note_destination_is_reported(tmp_path):
|
||||
source = source_book(tmp_path); output = tmp_path/'bad.epub'
|
||||
with zipfile.ZipFile(source) as before, zipfile.ZipFile(output, 'w') as after:
|
||||
for item in before.infolist():
|
||||
data = before.read(item.filename)
|
||||
if item.filename.endswith('Text/ch.xhtml'):
|
||||
data = data.replace(b'notes.xhtml#n1', b'ch.xhtml#title')
|
||||
after.writestr(item, data)
|
||||
report, _, _ = audit_pair(source, output)
|
||||
assert any(e['kind'] == 'new_note_destination_problem' for e in report['new_issues'])
|
||||
|
||||
|
||||
def test_stale_output_cannot_claim_new_manifest_translation(tmp_path):
|
||||
source, output, manifest, structure = translated_book(tmp_path)
|
||||
data = json.loads(manifest.read_text())
|
||||
entry = next(e for e in data if '正文' in e['translated_text'])
|
||||
entry['translated_text'] = entry['translated_text'].replace('正文', '修正后的正文')
|
||||
manifest.write_text(json.dumps(data))
|
||||
report, _, _ = audit_pair(source, output, manifest, structure)
|
||||
assert any(e['kind'] == 'translation_content_mismatch' and e['entry_id'] == entry['entry_id'] for e in report['new_issues'])
|
||||
|
||||
|
||||
def test_ncx_label_drift_is_detected_even_with_correct_nav(tmp_path):
|
||||
source, output, manifest, structure = translated_book(tmp_path)
|
||||
broken = tmp_path/'label-drift.epub'
|
||||
with zipfile.ZipFile(output) as before, zipfile.ZipFile(broken, 'w') as after:
|
||||
for item in before.infolist():
|
||||
data = before.read(item.filename)
|
||||
if item.filename.endswith('.ncx'):
|
||||
data = data.replace('第一章'.encode(), '错误标题'.encode())
|
||||
after.writestr(item, data)
|
||||
report, _, _ = audit_pair(source, broken, manifest, structure)
|
||||
assert any(e['kind'] == 'navigation_title_mismatch' for e in report['new_issues'])
|
||||
|
||||
|
||||
def test_wrapped_nav_links_are_not_treated_as_valid_reader_navigation(tmp_path):
|
||||
from lxml import etree
|
||||
source = source_book(tmp_path); broken = tmp_path/'wrapped.epub'
|
||||
with zipfile.ZipFile(source) as before, zipfile.ZipFile(broken, 'w') as after:
|
||||
for item in before.infolist():
|
||||
data = before.read(item.filename)
|
||||
if item.filename.endswith('/nav.xhtml'):
|
||||
doc = etree.fromstring(data)
|
||||
a = doc.xpath('//*[local-name()="nav"]//*[local-name()="a"]')[0]
|
||||
parent = a.getparent(); index = parent.index(a)
|
||||
parent.remove(a); wrapper = etree.Element('{http://www.w3.org/1999/xhtml}span'); wrapper.append(a); parent.insert(index, wrapper)
|
||||
data = etree.tostring(doc)
|
||||
after.writestr(item, data)
|
||||
report, _, _ = audit_pair(source, broken)
|
||||
assert any(e['kind'] == 'nav_link_not_direct_li' for e in report['new_issues'])
|
||||
|
||||
|
||||
def test_chinese_is_decoded_in_real_headless_browser(tmp_path):
|
||||
import os
|
||||
import pytest
|
||||
if not os.environ.get('PLAYWRIGHT_BROWSERS_PATH'):
|
||||
pytest.skip('Optional browser integration; set PLAYWRIGHT_BROWSERS_PATH')
|
||||
playwright = pytest.importorskip('playwright.sync_api')
|
||||
from src.qa.headless import archive_server
|
||||
source, output, _, _ = translated_book(tmp_path)
|
||||
name = next(n for n in book_inventory(output)['documents'] if n.endswith('/Text/ch.xhtml'))
|
||||
with archive_server({'translated': output}) as base, playwright.sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
try:
|
||||
page = browser.new_page(java_script_enabled=False)
|
||||
page.goto(base+'/translated/'+name)
|
||||
assert '正文' in page.locator('body').inner_text()
|
||||
assert '第一章' in page.locator('h1').inner_text()
|
||||
finally:
|
||||
browser.close()
|
||||
|
||||
|
||||
def agent_args(source, work, action='prepare', **overrides):
|
||||
from types import SimpleNamespace
|
||||
values = dict(action=action, source=source, work=work, model='agent/luna', batch_chars=1000,
|
||||
result=None, glossary=None, output_dir=None)
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def test_agent_harness_rejects_stale_results_without_mutating_cache(tmp_path):
|
||||
from scripts.agent_book import run
|
||||
import pytest
|
||||
source = source_book(tmp_path); work = tmp_path/'agent'
|
||||
asyncio.run(run(agent_args(source, work)))
|
||||
manifest = work/'manifest.json'; previous = manifest.read_bytes()
|
||||
entry = json.loads(previous)[0]; result = work/'results.json'
|
||||
result.write_text(json.dumps([{'entry_id': entry['entry_id'], 'source_sha256': 'wrong', 'translated_text': '中文。'}]))
|
||||
with pytest.raises(ValueError, match='stale'):
|
||||
asyncio.run(run(agent_args(source, work, 'apply', result=result)))
|
||||
assert manifest.read_bytes() == previous
|
||||
with pytest.raises(ValueError, match='Incomplete'):
|
||||
asyncio.run(run(agent_args(source, work, 'build')))
|
||||
assert not (work/'output').exists()
|
||||
|
||||
|
||||
def test_agent_invalid_repair_is_visible_and_preserves_valid_translation(tmp_path):
|
||||
from scripts.agent_book import run, unit_digest
|
||||
from src.common.data_model import ManifestEntry
|
||||
source = source_book(tmp_path); work = tmp_path/'agent'
|
||||
asyncio.run(run(agent_args(source, work)))
|
||||
manifest = work/'manifest.json'; data = json.loads(manifest.read_text())
|
||||
entry = next(e for e in data if e['placeholders'].get('1'))
|
||||
entry['translated_text'] = entry['original_text'].replace('Text', '正文')
|
||||
previous = entry['translated_text']; manifest.write_text(json.dumps(data))
|
||||
result = work/'results.json'; result.write_text(json.dumps([{'entry_id': entry['entry_id'],
|
||||
'source_sha256': unit_digest(ManifestEntry.model_validate(entry)), 'translated_text': '丢失注号的译文。'}]))
|
||||
assert asyncio.run(run(agent_args(source, work, 'apply', result=result))) == 1
|
||||
saved = next(e for e in json.loads(manifest.read_text()) if e['entry_id'] == entry['entry_id'])
|
||||
assert saved['translated_text'] == previous
|
||||
assert json.loads((work/'apply-report.json').read_text())['rejected']
|
||||
|
||||
|
||||
def test_agent_model_change_requires_isolated_work_directory(tmp_path):
|
||||
import pytest
|
||||
from scripts.agent_book import run
|
||||
source = source_book(tmp_path); work = tmp_path/'agent'
|
||||
asyncio.run(run(agent_args(source, work)))
|
||||
previous = (work/'manifest.json').read_bytes()
|
||||
with pytest.raises(ValueError, match='Model changed'):
|
||||
asyncio.run(run(agent_args(source, work, model='agent/other')))
|
||||
assert (work/'manifest.json').read_bytes() == previous
|
||||
Reference in New Issue
Block a user