Improve EPUB restoration, book terminology and Calibre QA workflow

2026-10-08 07:25:27 +08:00
parent 40491e6386
commit caf9d8f4bb
35 changed files with 1439 additions and 959 deletions
@@ -0,0 +1,38 @@
+"""Offline extraction/restoration checks across the supplied EPUB corpus."""
+import argparse
+import json
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from src.preprocessing.epub_cleaner import EpubCleaner
+from src.preprocessing.text_extractor import FineGrainedExtractor
+from src.common.data_model import BookStructure
+from src.assembly.format_restorer import FormatRestorer
+from src.assembly.builder import BilingualBuilder
+from src.qa.epub_audit import audit_epub, compare_audits
+
+
+def main():
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--input', type=Path, default=Path('input'))
+    parser.add_argument('--out', type=Path, default=Path('.work/corpus-qa'))
+    args=parser.parse_args(); rows=[]; args.out.mkdir(parents=True, exist_ok=True)
+    for source in sorted(args.input.glob('*.epub')):
+        row={'book':source.name}
+        try:
+            work=args.out/source.stem
+            structure=BookStructure.load(EpubCleaner(source,work).clean())
+            entries=FineGrainedExtractor().extract(structure)
+            bad=[e.entry_id for e in entries if not FormatRestorer().restore(e.original_text,e.placeholders,original_text=e.original_text)[1]]
+            output=BilingualBuilder(work,source).build(structure,work/'roundtrip.epub')
+            before,after=audit_epub(source),audit_epub(output)
+            row.update(entries=len(entries),format_failures=bad,source_issues=len(before['issues']),
+                       new_issues=compare_audits(before,after))
+        except Exception as exc:
+            row['error']=str(exc)
+        rows.append(row)
+        print(json.dumps({k:v for k,v in row.items() if k!='format_failures'},ensure_ascii=False),flush=True)
+        (args.out/'report.json').write_text(json.dumps(rows,ensure_ascii=False,indent=2))
+    return int(any(r.get('error') or r.get('format_failures') or r.get('new_issues') for r in rows))
+
+if __name__=='__main__': raise SystemExit(main())