Feat: v0.12 Pipeline Separation, Spacing Fix, and Idempotent Restoration

This commit is contained in:
谭凯
2026-02-01 10:39:07 +08:00
parent 7a93c52b42
commit 8e58415173
42 changed files with 3741 additions and 110 deletions
+25 -29
View File
@@ -12,6 +12,7 @@ from src.preprocessing.text_extractor import FineGrainedExtractor
from src.translation.manifest_manager import ManifestManager
from src.translation.translator_engine import Translator
from src.translation.llm_client import LLMClient
from src.assembly.restoration_engine import RestorationEngine
from src.assembly.backfiller import BackfillEngine
from src.assembly.builder import BilingualBuilder
from src.common.data_model import BookStructure
@@ -76,11 +77,9 @@ async def run_pipeline(args):
manifest_manager.save()
# 4. Translation
if not args.skip_translation:
if not api_key:
logger.error("Cannot translate without API Key. Use --skip-translation to test pipeline.")
sys.exit(1)
# Initialize LLM Client (Centralized)
llm_client = None
if api_key:
llm_client = LLMClient(
api_key=api_key,
base_url=base_url,
@@ -89,6 +88,12 @@ async def run_pipeline(args):
concurrent_requests=llm_conf.get("concurrent_requests", 5),
chunk_dir=work["chunks"]
)
# 4. Translation
if not args.skip_translation:
if not llm_client:
logger.error("Cannot translate without API Key. Use --skip-translation to test pipeline.")
sys.exit(1)
# Profiling
profiler = BookProfiler(llm_client)
@@ -101,35 +106,25 @@ async def run_pipeline(args):
translator = Translator(llm_client, chunk_size=target_chunk_size, max_concurrent=concurrent_reqs)
await translator.translate(manifest_manager.entries, profile)
manifest_manager.save()
# Don't close here, wait until after backfill
# await llm_client.close()
pass
else:
logger.info("Skipping translation step.")
# 5. Backfill (now async + LLM repair enabled)
# Reuse existing llm_client if available, otherwise create temporary one if needed?
# In this flow, llm_client is created inside the 'if not args.skip_translation' block.
# If skip_translation is True, llm_client is undefined.
# 5. Restoration (Format + Spacing + Repair)
logger.info("Restoring format & applying spacing...")
# RestorationEngine handles Pangu spacing, FormatRestorer, and LLM Repair
restorer = RestorationEngine(llm_client)
# Default to skipping if already done, unless forced
force_restore = getattr(args, 'force_restore', False)
restore_success = await restorer.restore_entries(manifest_manager.entries, force=force_restore)
manifest_manager.save() # Save translated_html
logger.info(f"Restoration complete. {restore_success} entries validated.")
backfill_llm_client = None
should_close_client = False
if 'llm_client' in locals() and llm_client:
backfill_llm_client = llm_client
elif api_key and not args.skip_translation:
# This case shouldn't happen because if not skip, we key llm_client above.
# But if skip_translation is True, we might still want repair?
# For now, let's only enable repair if translation occurred or if we explicitly create one.
# User said: "LLM features may fail" if no key.
pass
# Initialization
backfiller = BackfillEngine(llm_client=backfill_llm_client)
# 6. Backfill (Pure Injection)
logger.info("Injecting content into EPUB structure...")
backfiller = BackfillEngine() # Pure injection, no dependencies
updated_structure = await backfiller.backfill(structure, manifest_manager.entries, mode=args.mode)
# 6. Assembly - Pass original EPUB for TOC preservation
# 7. Assembly - Pass original EPUB for TOC preservation
builder = BilingualBuilder(work["root"], original_epub_path=input_path)
if args.mode == "bilingual":
@@ -143,7 +138,7 @@ async def run_pipeline(args):
logger.info(f"Pipeline completed! Output: {created_epub}")
if 'llm_client' in locals() and llm_client:
if llm_client:
await llm_client.close()
except EpubTranslatorError as e:
@@ -162,6 +157,7 @@ def main():
parser.add_argument("--model", default=None, help="LLM Model to use (overrides config)")
parser.add_argument("--bilingual", action="store_true", help="Output bilingual version (default is target language only)")
parser.add_argument("--skip-translation", action="store_true", help="Skip LLM translation (for testing)")
parser.add_argument("--force-restore", action="store_true", help="Force re-run format restoration/repair even if translated_html exists")
parser.add_argument("--force-clean", action="store_true", help="Force re-clean EPUB even if book_structure exists")
args = parser.parse_args()