Improve EPUB restoration, book terminology and Calibre QA workflow

scripts/build_sample.py

@@ -0,0 +1,45 @@
+"""Apply an agent-authored translation manifest for offline, reproducible QA."""
+import argparse
+import asyncio
+import json
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from src.common.data_model import BookStructure, ManifestEntry
+from src.preprocessing.epub_cleaner import EpubCleaner
+from src.preprocessing.text_extractor import FineGrainedExtractor
+from src.assembly.restoration_engine import RestorationEngine
+from src.assembly.backfiller import BackfillEngine
+from src.assembly.builder import BilingualBuilder
+
+
+async def run(args):
+    args.out.mkdir(parents=True,exist_ok=True)
+    structure=BookStructure.load(EpubCleaner(args.source,args.out/'source').clean())
+    entries=FineGrainedExtractor().extract(structure)
+    by_id={e.entry_id:e for e in entries}
+    provided=[ManifestEntry.model_validate(e) for e in json.loads(args.manifest.read_text())]
+    for sample in provided:
+        current=by_id.get(sample.entry_id)
+        if current is None or current.original_text!=sample.original_text or current.placeholders!=sample.placeholders:
+            raise ValueError(f'Sample/source mismatch: {sample.entry_id}')
+        current.translated_text=sample.translated_text
+    count=await RestorationEngine().restore_entries(entries)
+    if count!=len(provided):
+        raise ValueError('Sample has missing or invalid translations')
+    (args.out/'manifest.json').write_text(json.dumps([e.model_dump() for e in entries],ensure_ascii=False,indent=2))
+    for mode in ['bilingual','target_only']:
+        copy=structure.model_copy(deep=True)
+        await BackfillEngine().backfill(copy,entries,mode,allow_partial=True)
+        BilingualBuilder(args.out/'source',args.source).build(copy,args.out/f'{mode}-sample.epub')
+    print(f'Explicit partial sample: {count}/{len(entries)} translated units. Outputs: {args.out}')
+
+
+def main():
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('source',type=Path)
+    parser.add_argument('manifest',type=Path)
+    parser.add_argument('--out',type=Path,default=Path('.work/offline-sample'))
+    asyncio.run(run(parser.parse_args()))
+
+if __name__=='__main__': main()

scripts/corpus_regression.py

@@ -0,0 +1,38 @@
+"""Offline extraction/restoration checks across the supplied EPUB corpus."""
+import argparse
+import json
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from src.preprocessing.epub_cleaner import EpubCleaner
+from src.preprocessing.text_extractor import FineGrainedExtractor
+from src.common.data_model import BookStructure
+from src.assembly.format_restorer import FormatRestorer
+from src.assembly.builder import BilingualBuilder
+from src.qa.epub_audit import audit_epub, compare_audits
+
+
+def main():
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--input', type=Path, default=Path('input'))
+    parser.add_argument('--out', type=Path, default=Path('.work/corpus-qa'))
+    args=parser.parse_args(); rows=[]; args.out.mkdir(parents=True, exist_ok=True)
+    for source in sorted(args.input.glob('*.epub')):
+        row={'book':source.name}
+        try:
+            work=args.out/source.stem
+            structure=BookStructure.load(EpubCleaner(source,work).clean())
+            entries=FineGrainedExtractor().extract(structure)
+            bad=[e.entry_id for e in entries if not FormatRestorer().restore(e.original_text,e.placeholders,original_text=e.original_text)[1]]
+            output=BilingualBuilder(work,source).build(structure,work/'roundtrip.epub')
+            before,after=audit_epub(source),audit_epub(output)
+            row.update(entries=len(entries),format_failures=bad,source_issues=len(before['issues']),
+                       new_issues=compare_audits(before,after))
+        except Exception as exc:
+            row['error']=str(exc)
+        rows.append(row)
+        print(json.dumps({k:v for k,v in row.items() if k!='format_failures'},ensure_ascii=False),flush=True)
+        (args.out/'report.json').write_text(json.dumps(rows,ensure_ascii=False,indent=2))
+    return int(any(r.get('error') or r.get('format_failures') or r.get('new_issues') for r in rows))
+
+if __name__=='__main__': raise SystemExit(main())

scripts/visual_qa.py

@@ -0,0 +1,144 @@
+"""Repeatable EPUB structural audit + Calibre render evidence.
+
+Rendering is evidence for visual inspection, never an automatic visual pass.
+"""
+import argparse
+import json
+import os
+import posixpath
+import shutil
+import subprocess
+import sys
+import zipfile
+from pathlib import Path
+from lxml import etree
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from src.qa.epub_audit import audit_epub, compare_audits
+from src.assembly.package_writer import package_path
+from src.common.data_model import ManifestEntry
+from src.assembly.format_restorer import FormatRestorer
+
+
+def subset_epub(epub, output, chapter_hrefs):
+    with zipfile.ZipFile(epub) as source:
+        opf_path = package_path(source)
+        opf = etree.fromstring(source.read(opf_path))
+        selected = {item.get('id') for item in opf.xpath('//*[local-name()="manifest"]/*')
+                    if item.get('href') in chapter_hrefs}
+        if not selected:
+            raise ValueError('No selected chapters found in package')
+        manifest = opf.xpath('//*[local-name()="manifest"]')[0]
+        namespace = etree.QName(manifest).namespace
+        for item in list(manifest):
+            if (item.get('media-type') in {'application/xhtml+xml', 'application/x-dtbncx+xml'}
+                and item.get('id') not in selected):
+                manifest.remove(item)
+            elif item.get('properties') == 'cover-image':
+                item.attrib.pop('properties', None)
+        # Calibre includes non-spine HTML too, so remove unused documents from
+        # the sample manifest and provide a sample-only navigation document.
+        nav_href = '__qa_nav.xhtml'
+        etree.SubElement(manifest, f'{{{namespace}}}item', id='qa_nav', href=nav_href,
+                         properties='nav', **{'media-type':'application/xhtml+xml'})
+        nav = etree.Element('{http://www.w3.org/1999/xhtml}html',
+                            nsmap={None:'http://www.w3.org/1999/xhtml', 'epub':'http://www.idpf.org/2007/ops'})
+        head=etree.SubElement(nav,'{http://www.w3.org/1999/xhtml}head')
+        etree.SubElement(head,'{http://www.w3.org/1999/xhtml}title').text='QA sample'
+        body=etree.SubElement(nav,'{http://www.w3.org/1999/xhtml}body')
+        toc=etree.SubElement(body,'{http://www.w3.org/1999/xhtml}nav',
+                            **{'{http://www.idpf.org/2007/ops}type':'toc'})
+        listing=etree.SubElement(toc,'{http://www.w3.org/1999/xhtml}ol')
+        for href in sorted(chapter_hrefs):
+            li=etree.SubElement(listing,'{http://www.w3.org/1999/xhtml}li')
+            etree.SubElement(li,'{http://www.w3.org/1999/xhtml}a',href=href).text=Path(href).stem
+        spine = opf.xpath('//*[local-name()="spine"]')[0]
+        spine.attrib.pop('toc', None)
+        for ref in list(spine):
+            if ref.get('idref') not in selected:
+                spine.remove(ref)
+        for node in opf.xpath('//*[local-name()="guide"] | //*[local-name()="meta"][@name="cover"]'):
+            node.getparent().remove(node)
+        with zipfile.ZipFile(output, 'w') as target:
+            kept_documents = {posixpath.normpath(posixpath.join(posixpath.dirname(opf_path), href)) for href in chapter_hrefs}
+            for info in source.infolist():
+                if info.filename.lower().endswith(('.xhtml', '.html', '.htm', '.ncx')) and info.filename not in kept_documents:
+                    continue
+                target.writestr(info, etree.tostring(opf, encoding='utf-8', xml_declaration=True)
+                                if info.filename == opf_path else source.read(info.filename))
+            target.writestr(posixpath.join(posixpath.dirname(opf_path),nav_href),etree.tostring(nav))
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('source', type=Path)
+    parser.add_argument('translated', type=Path)
+    parser.add_argument('--manifest', type=Path)
+    parser.add_argument('--out', type=Path, default=Path('.work/visual-qa'))
+    parser.add_argument('--render', action='store_true', help='Calibre PDF + Poppler PNGs for visual review')
+    parser.add_argument('--pages', type=int, default=6)
+    parser.add_argument('--chapter', action='append', help='Exact EPUB chapter href; repeat to select several')
+    parser.add_argument('--max-chapters', type=int, default=3, help='Default visual sampling limit; 0 renders all')
+    parser.add_argument('--allow-partial', action='store_true', help='Explicitly permit sample coverage')
+    args = parser.parse_args()
+    args.out.mkdir(parents=True, exist_ok=True)
+    source, output = audit_epub(args.source), audit_epub(args.translated)
+    report = {'source': source, 'output': output, 'new_issues': compare_audits(source, output),
+              'visual_review': 'pending', 'coverage': None}
+    entries = []
+    if args.manifest:
+        entries = [ManifestEntry.model_validate(e) for e in json.loads(args.manifest.read_text())]
+        errors = []
+        for entry in entries:
+            html, ok = FormatRestorer().restore(entry.translated_text, entry.placeholders, original_text=entry.original_text)
+            if not entry.translated_text or not ok or not entry.translated_html or entry.validation_errors:
+                errors.append(entry.entry_id)
+        report['coverage'] = {'total': len(entries), 'accepted': len(entries) - len(errors), 'failed_ids': errors,
+                              'scope': 'provided manifest only'}
+        if errors and not args.allow_partial:
+            report['new_issues'].append({'kind': 'incomplete_translation', 'detail': str(len(errors))})
+    chapter_hrefs = args.chapter or list(dict.fromkeys(e.file_path for e in entries if e.translated_text))
+    if not args.chapter and args.max_chapters:
+        chapter_hrefs = chapter_hrefs[:args.max_chapters]
+    report['sampled_chapters'] = chapter_hrefs
+    if args.render:
+        for tool in ('ebook-convert', 'pdftoppm'):
+            if not shutil.which(tool):
+                raise RuntimeError(f'Missing QA dependency: {tool}')
+        for label, path in [('source', args.source), ('translated', args.translated)]:
+            sample = args.out / f'{label}-sample.epub'
+            if chapter_hrefs:
+                subset_epub(path, sample, set(chapter_hrefs))
+            else:
+                shutil.copyfile(path, sample)
+            for old in args.out.glob(label+'-*.png'):
+                old.unlink()
+            pdf = args.out / f'{label}.pdf'
+            command = ['ebook-convert', str(sample.resolve()), str(pdf.resolve()),
+                       '--pdf-serif-family', 'Songti SC', '--pdf-sans-family', 'PingFang SC',
+                       '--pdf-default-font-size', '14', '--paper-size', 'a4']
+            env = os.environ.copy()
+            env['CALIBRE_CONFIG_DIRECTORY'] = str((args.out/'calibre-config').resolve())
+            (args.out/'calibre-config').mkdir(exist_ok=True)
+            with (args.out/f'{label}-calibre.log').open('w') as log:
+                subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, check=True, timeout=240, env=env)
+            subprocess.run(['pdftoppm', '-f', '1', '-l', str(args.pages), '-scale-to', '1400', '-png',
+                            str(pdf), str(args.out/label)], check=True, timeout=120)
+        report['rendered_pages'] = sorted(p.name for p in args.out.glob('*.png'))
+    (args.out/'report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2))
+    import re
+    def pages(label):
+        return sorted(args.out.glob(label+'-*.png'), key=lambda p: int(re.search(r'-(\d+)\.png$', p.name)[1]))
+    source_pages, target_pages = pages('source'), pages('translated')
+    rows = ''
+    for index in range(max(len(source_pages),len(target_pages))):
+        for group in (source_pages,target_pages):
+            rows += (f'<figure><img src="{group[index].name}" style="width:100%"><figcaption>{group[index].name}</figcaption></figure>'
+                     if index<len(group) else '<figure><figcaption>No corresponding page</figcaption></figure>')
+    (args.out/'review.html').write_text('<!doctype html><meta charset="utf-8"><title>EPUB visual QA</title>'
+        '<h1>EPUB visual QA</h1><p>结构检查见 report.json。截图须人工/agent检查；生成截图不等于视觉通过。</p>'
+        '<main style="display:grid;grid-template-columns:1fr 1fr;gap:20px">'+rows+'</main>')
+    print(f"New structural issues: {len(report['new_issues'])}; visual review: pending; {args.out/'report.json'}")
+    return 1 if report['new_issues'] else 0
+
+if __name__ == '__main__':
+    raise SystemExit(main())