kai/epub_bilingual_translator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve EPUB restoration, book terminology and Calibre QA workflow

Browse Source
This commit is contained in:
Kai Tan
2026-10-08 07:25:27 +08:00
parent 40491e6386
commit caf9d8f4bb
35 changed files with 1439 additions and 959 deletions
Download Patch File Download Diff File Expand all files Collapse all files
main.py
+44 -6
View File
@@ -1,4 +1,7 @@
import argparse
import hashlib
import json
import subprocess
import sys
import os
import asyncio
@@ -11,6 +14,7 @@ from src.preprocessing.profiler import BookProfiler
from src.preprocessing.text_extractor import FineGrainedExtractor
from src.translation.manifest_manager import ManifestManager
from src.translation.translator_engine import Translator
from src.translation.glossary import prepare_glossary
from src.translation.llm_client import LLMClient
from src.assembly.restoration_engine import RestorationEngine
from src.assembly.backfiller import BackfillEngine
@@ -55,11 +59,15 @@ async def run_pipeline(args):
if not api_key:
logger.warning("OPENAI_API_KEY not found in env or config. LLM features may fail.")
llm_client = None
try:
# 1. Preprocessing - Reuse book_structure.json if exists
if work["structure"].exists() and not args.force_clean:
logger.info(f"Reusing existing book_structure: {work['structure']}")
structure = BookStructure.load(work["structure"])
if structure.source_sha256 != hashlib.sha256(input_path.read_bytes()).hexdigest():
logger.info("Source changed or old preprocessing cache; re-cleaning")
structure = BookStructure.load(EpubCleaner(input_path, work["root"]).clean())
else:
logger.info("Cleaning EPUB and generating book_structure...")
cleaner = EpubCleaner(input_path, work["root"])
@@ -86,6 +94,8 @@ async def run_pipeline(args):
model=model,
requests_per_minute=llm_conf.get("requests_per_minute", 60),
concurrent_requests=llm_conf.get("concurrent_requests", 5),
timeout=llm_conf.get("timeout", 180),
max_output_tokens=llm_conf.get("max_output_tokens", 16000),
chunk_dir=work["chunks"]
)
@@ -96,16 +106,25 @@ async def run_pipeline(args):
sys.exit(1)
# Profiling
profiler = BookProfiler(llm_client)
profiler = BookProfiler(llm_client, cache_path=work["root"] / "profile.json")
profile = await profiler.analyze(manifest_manager.entries)
logger.info(f"Book Profile: {profile}")
# Translation
target_chunk_size = trans_conf.get("chunk_size", 5000)
concurrent_reqs = llm_conf.get("concurrent_requests", 5)
translator = Translator(llm_client, chunk_size=target_chunk_size, max_concurrent=concurrent_reqs)
glossary = await prepare_glossary(
manifest_manager.entries, llm_client, work["root"] / "glossary.json",
override=getattr(args, 'glossary', None) or trans_conf.get("glossary_path"),
auto=trans_conf.get("auto_glossary", True),
chunk_chars=trans_conf.get("glossary_scan_chars", 24000))
translator = Translator(llm_client, chunk_size=target_chunk_size, max_concurrent=concurrent_reqs,
glossary=glossary, validation_retries=trans_conf.get("validation_retries", 2),
on_chunk_complete=manifest_manager.save)
await translator.translate(manifest_manager.entries, profile)
manifest_manager.save()
if any(not e.translated_text or e.validation_errors for e in manifest_manager.entries):
raise ValueError("Translation incomplete; inspect manifest validation_errors and resume")
else:
logger.info("Skipping translation step.")
@@ -118,11 +137,13 @@ async def run_pipeline(args):
restore_success = await restorer.restore_entries(manifest_manager.entries, force=force_restore)
manifest_manager.save() # Save translated_html
logger.info(f"Restoration complete. {restore_success} entries validated.")
if any(e.translated_text and not e.translated_html for e in manifest_manager.entries):
raise ValueError("Format validation failed; broken entries were not backfilled")
# 6. Backfill (Pure Injection)
logger.info("Injecting content into EPUB structure...")
backfiller = BackfillEngine() # Pure injection, no dependencies
updated_structure = await backfiller.backfill(structure, manifest_manager.entries, mode=args.mode)
updated_structure = await backfiller.backfill(structure, manifest_manager.entries, mode=args.mode, allow_partial=args.skip_translation)
# 7. Assembly - Pass original EPUB for TOC preservation
builder = BilingualBuilder(work["root"], original_epub_path=input_path)
@@ -136,11 +157,22 @@ async def run_pipeline(args):
created_epub = builder.build(updated_structure, output_path)
from src.qa.epub_audit import audit_epub, compare_audits
source_audit, output_audit = audit_epub(input_path), audit_epub(created_epub)
qa_report = {'source': source_audit, 'output': output_audit,
'new_issues': compare_audits(source_audit, output_audit), 'visual_review': 'pending'}
(work["root"] / "structural-qa.json").write_text(json.dumps(qa_report, ensure_ascii=False, indent=2))
if qa_report['new_issues']:
raise ValueError("EPUB structural QA failed; inspect structural-qa.json")
if getattr(args, 'visual_qa', False):
command = [sys.executable, str(Path(__file__).parent / 'scripts/visual_qa.py'),
str(input_path), str(created_epub), '--manifest', str(work['manifest']),
'--out', str(work['root'] / 'visual-qa'), '--render']
if args.skip_translation:
command.append('--allow-partial')
subprocess.run(command, check=True)
logger.info(f"Pipeline completed! Output: {created_epub}")
if llm_client:
await llm_client.close()
except EpubTranslatorError as e:
logger.error(f"An error occurred: {e}")
sys.exit(1)
@@ -150,10 +182,16 @@ async def run_pipeline(args):
traceback.print_exc()
sys.exit(1)
finally:
if llm_client:
await llm_client.close()
def main():
parser = argparse.ArgumentParser(description="EPUB Bilingual Translator")
parser.add_argument("input_epub", help="Path to the input EPUB file")
parser.add_argument("--output-dir", default="output", help="Directory for output files")
parser.add_argument("--visual-qa", action="store_true", help="Render Calibre QA page images for visual inspection")
parser.add_argument("--glossary", help="Per-book JSON terminology file")
parser.add_argument("--model", default=None, help="LLM Model to use (overrides config)")
parser.add_argument("--bilingual", action="store_true", help="Output bilingual version (default is target language only)")
parser.add_argument("--skip-translation", action="store_true", help="Skip LLM translation (for testing)")