Improve EPUB restoration, book terminology and Calibre QA workflow

pipeline/01_preprocess.py

@@ -1,4 +1,5 @@
 import argparse
+import hashlib
 import sys
 import os
 from pathlib import Path
@@ -33,11 +34,15 @@ def run_preprocess(args):
         if structure_path.exists() and not args.force:
             logger.info(f"Reusing existing structure: {structure_path}")
             structure = BookStructure.load(structure_path)
+            if structure.source_sha256 != hashlib.sha256(input_path.read_bytes()).hexdigest():
+                structure = BookStructure.load(EpubCleaner(input_path, work["root"]).clean())
         else:
             logger.info("Cleaning EPUB...")
             cleaner = EpubCleaner(input_path, work["root"])
             structure_path = cleaner.clean()
             structure = BookStructure.load(structure_path)
+            if structure.source_sha256 != hashlib.sha256(input_path.read_bytes()).hexdigest():
+                structure = BookStructure.load(EpubCleaner(input_path, work["root"]).clean())
 
         # 2. Extract Text
         logger.info("Extracting text segments...")

pipeline/02_translate.py

@@ -7,6 +7,7 @@ from pathlib import Path
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
 from src.translation.translator_engine import Translator
+from src.translation.glossary import prepare_glossary
 from src.translation.llm_client import LLMClient
 from src.translation.manifest_manager import ManifestManager
 from src.preprocessing.profiler import BookProfiler
@@ -60,6 +61,7 @@ async def run_translate(args):
         logger.error("OPENAI_API_KEY not found in env or config.")
         sys.exit(1)
         
+    llm = None
     try:
         # Load Manifest
         manager = ManifestManager(manifest_path)
@@ -74,23 +76,34 @@ async def run_translate(args):
             requests_per_minute=llm_conf.get("requests_per_minute", 60),
 
             concurrent_requests=llm_conf.get("concurrent_requests", 5),
+            timeout=llm_conf.get("timeout", 180),
+            max_output_tokens=llm_conf.get("max_output_tokens", 16000),
             chunk_dir=work["chunks"]
         )
         
         # Profile
-        profiler = BookProfiler(llm)
+        profiler = BookProfiler(llm, cache_path=work["root"] / "profile.json")
         profile = await profiler.analyze(manager.entries)
         logger.info(f"Book Profile: {profile.title} ({profile.genre})")
         
         # Translate
         target_chunk_size = trans_conf.get("chunk_size", 4000)
         concurrent_reqs = llm_conf.get("concurrent_requests", 5)
-        translator = Translator(llm, chunk_size=target_chunk_size, max_concurrent=concurrent_reqs)
+        glossary = await prepare_glossary(
+            manager.entries, llm, work["root"] / "glossary.json",
+            override=getattr(args, 'glossary', None) or trans_conf.get("glossary_path"),
+            auto=trans_conf.get("auto_glossary", True),
+            chunk_chars=trans_conf.get("glossary_scan_chars", 24000))
+        translator = Translator(llm, chunk_size=target_chunk_size, max_concurrent=concurrent_reqs,
+                                glossary=glossary, validation_retries=trans_conf.get("validation_retries", 2),
+                                on_chunk_complete=manager.save)
         await translator.translate(manager.entries, profile)
         
         # Save final state
         manager.save()
-        await llm.close()
+        if any(not e.translated_text or e.validation_errors for e in manager.entries):
+            raise ValueError("Translation incomplete; inspect manifest validation_errors and resume")
+
         
         logger.info("Translation complete.")
         
@@ -101,10 +114,15 @@ async def run_translate(args):
         logger.critical(f"Unexpected error: {e}")
         sys.exit(1)
 
+    finally:
+        if llm:
+            await llm.close()
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Step 2: Translation")
     parser.add_argument("--input-epub", help="Path to original EPUB (to derive book name)")
     parser.add_argument("--book-name", help="Book name (folder name in work/)")
+    parser.add_argument("--glossary", help="Per-book JSON terminology file")
     parser.add_argument("--model", default=None, help="LLM Model (overrides config)")
     
     args = parser.parse_args()

pipeline/03_assemble.py

@@ -7,6 +7,7 @@ from pathlib import Path
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
 from src.assembly.backfiller import BackfillEngine
+from src.assembly.restoration_engine import RestorationEngine
 from src.assembly.builder import BilingualBuilder
 from src.translation.manifest_manager import ManifestManager
 from src.translation.llm_client import LLMClient
@@ -65,7 +66,11 @@ async def run_assemble(args):
         
         # Backfill
         logger.info(f"Backfilling translations (Mode: {args.mode})...")
-        backfiller = BackfillEngine(llm_client=llm_client)
+        await RestorationEngine(llm_client).restore_entries(manager.entries)
+        manager.save()
+        if any(e.translated_text and not e.translated_html for e in manager.entries):
+            raise ValueError("Format validation failed; inspect manifest")
+        backfiller = BackfillEngine()
         updated_structure = await backfiller.backfill(structure, manager.entries, mode=args.mode)
         
         if llm_client:

pipeline/04_build_epub.py

@@ -44,6 +44,8 @@ async def run_build(args):
         manager = ManifestManager(manifest_path)
         manager.load()
         
+        if any(e.translated_text and not e.translated_html for e in manager.entries):
+            raise ValueError("Run format restoration before building; missing validated HTML")
         # Backfill (Pure Injection)
         logger.info(f"Injecting content (Mode: {args.mode})...")
         backfiller = BackfillEngine() # No dependencies needed