Feat: v0.12 Pipeline Separation, Spacing Fix, and Idempotent Restoration
This commit is contained in:
@@ -12,6 +12,7 @@ from src.preprocessing.text_extractor import FineGrainedExtractor
|
||||
from src.translation.manifest_manager import ManifestManager
|
||||
from src.translation.translator_engine import Translator
|
||||
from src.translation.llm_client import LLMClient
|
||||
from src.assembly.restoration_engine import RestorationEngine
|
||||
from src.assembly.backfiller import BackfillEngine
|
||||
from src.assembly.builder import BilingualBuilder
|
||||
from src.common.data_model import BookStructure
|
||||
@@ -76,11 +77,9 @@ async def run_pipeline(args):
|
||||
manifest_manager.save()
|
||||
|
||||
# 4. Translation
|
||||
if not args.skip_translation:
|
||||
if not api_key:
|
||||
logger.error("Cannot translate without API Key. Use --skip-translation to test pipeline.")
|
||||
sys.exit(1)
|
||||
|
||||
# Initialize LLM Client (Centralized)
|
||||
llm_client = None
|
||||
if api_key:
|
||||
llm_client = LLMClient(
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
@@ -89,6 +88,12 @@ async def run_pipeline(args):
|
||||
concurrent_requests=llm_conf.get("concurrent_requests", 5),
|
||||
chunk_dir=work["chunks"]
|
||||
)
|
||||
|
||||
# 4. Translation
|
||||
if not args.skip_translation:
|
||||
if not llm_client:
|
||||
logger.error("Cannot translate without API Key. Use --skip-translation to test pipeline.")
|
||||
sys.exit(1)
|
||||
|
||||
# Profiling
|
||||
profiler = BookProfiler(llm_client)
|
||||
@@ -101,35 +106,25 @@ async def run_pipeline(args):
|
||||
translator = Translator(llm_client, chunk_size=target_chunk_size, max_concurrent=concurrent_reqs)
|
||||
await translator.translate(manifest_manager.entries, profile)
|
||||
manifest_manager.save()
|
||||
|
||||
# Don't close here, wait until after backfill
|
||||
# await llm_client.close()
|
||||
pass
|
||||
else:
|
||||
logger.info("Skipping translation step.")
|
||||
|
||||
# 5. Backfill (now async + LLM repair enabled)
|
||||
# Reuse existing llm_client if available, otherwise create temporary one if needed?
|
||||
# In this flow, llm_client is created inside the 'if not args.skip_translation' block.
|
||||
# If skip_translation is True, llm_client is undefined.
|
||||
# 5. Restoration (Format + Spacing + Repair)
|
||||
logger.info("Restoring format & applying spacing...")
|
||||
# RestorationEngine handles Pangu spacing, FormatRestorer, and LLM Repair
|
||||
restorer = RestorationEngine(llm_client)
|
||||
# Default to skipping if already done, unless forced
|
||||
force_restore = getattr(args, 'force_restore', False)
|
||||
restore_success = await restorer.restore_entries(manifest_manager.entries, force=force_restore)
|
||||
manifest_manager.save() # Save translated_html
|
||||
logger.info(f"Restoration complete. {restore_success} entries validated.")
|
||||
|
||||
backfill_llm_client = None
|
||||
should_close_client = False
|
||||
|
||||
if 'llm_client' in locals() and llm_client:
|
||||
backfill_llm_client = llm_client
|
||||
elif api_key and not args.skip_translation:
|
||||
# This case shouldn't happen because if not skip, we key llm_client above.
|
||||
# But if skip_translation is True, we might still want repair?
|
||||
# For now, let's only enable repair if translation occurred or if we explicitly create one.
|
||||
# User said: "LLM features may fail" if no key.
|
||||
pass
|
||||
|
||||
# Initialization
|
||||
backfiller = BackfillEngine(llm_client=backfill_llm_client)
|
||||
# 6. Backfill (Pure Injection)
|
||||
logger.info("Injecting content into EPUB structure...")
|
||||
backfiller = BackfillEngine() # Pure injection, no dependencies
|
||||
updated_structure = await backfiller.backfill(structure, manifest_manager.entries, mode=args.mode)
|
||||
|
||||
# 6. Assembly - Pass original EPUB for TOC preservation
|
||||
# 7. Assembly - Pass original EPUB for TOC preservation
|
||||
builder = BilingualBuilder(work["root"], original_epub_path=input_path)
|
||||
|
||||
if args.mode == "bilingual":
|
||||
@@ -143,7 +138,7 @@ async def run_pipeline(args):
|
||||
|
||||
logger.info(f"Pipeline completed! Output: {created_epub}")
|
||||
|
||||
if 'llm_client' in locals() and llm_client:
|
||||
if llm_client:
|
||||
await llm_client.close()
|
||||
|
||||
except EpubTranslatorError as e:
|
||||
@@ -162,6 +157,7 @@ def main():
|
||||
parser.add_argument("--model", default=None, help="LLM Model to use (overrides config)")
|
||||
parser.add_argument("--bilingual", action="store_true", help="Output bilingual version (default is target language only)")
|
||||
parser.add_argument("--skip-translation", action="store_true", help="Skip LLM translation (for testing)")
|
||||
parser.add_argument("--force-restore", action="store_true", help="Force re-run format restoration/repair even if translated_html exists")
|
||||
parser.add_argument("--force-clean", action="store_true", help="Force re-clean EPUB even if book_structure exists")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
Reference in New Issue
Block a user