39 lines
2.0 KiB
Python
39 lines
2.0 KiB
Python
"""Offline extraction/restoration checks across the supplied EPUB corpus."""
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from src.preprocessing.epub_cleaner import EpubCleaner
|
|
from src.preprocessing.text_extractor import FineGrainedExtractor
|
|
from src.common.data_model import BookStructure
|
|
from src.assembly.format_restorer import FormatRestorer
|
|
from src.assembly.builder import BilingualBuilder
|
|
from src.qa.epub_audit import audit_epub, compare_audits
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--input', type=Path, default=Path('input'))
|
|
parser.add_argument('--out', type=Path, default=Path('.work/corpus-qa'))
|
|
args=parser.parse_args(); rows=[]; args.out.mkdir(parents=True, exist_ok=True)
|
|
for source in sorted(args.input.glob('*.epub')):
|
|
row={'book':source.name}
|
|
try:
|
|
work=args.out/source.stem
|
|
structure=BookStructure.load(EpubCleaner(source,work).clean())
|
|
entries=FineGrainedExtractor().extract(structure)
|
|
bad=[e.entry_id for e in entries if not FormatRestorer().restore(e.original_text,e.placeholders,original_text=e.original_text)[1]]
|
|
output=BilingualBuilder(work,source).build(structure,work/'roundtrip.epub')
|
|
before,after=audit_epub(source),audit_epub(output)
|
|
row.update(entries=len(entries),format_failures=bad,source_issues=len(before['issues']),
|
|
new_issues=compare_audits(before,after))
|
|
except Exception as exc:
|
|
row['error']=str(exc)
|
|
rows.append(row)
|
|
print(json.dumps({k:v for k,v in row.items() if k!='format_failures'},ensure_ascii=False),flush=True)
|
|
(args.out/'report.json').write_text(json.dumps(rows,ensure_ascii=False,indent=2))
|
|
return int(any(r.get('error') or r.get('format_failures') or r.get('new_issues') for r in rows))
|
|
|
|
if __name__=='__main__': raise SystemExit(main())
|