kai/epub_bilingual_translator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: v0.12 Pipeline Separation, Spacing Fix, and Idempotent Restoration

Browse Source
This commit is contained in:
谭凯
2026-02-01 10:39:07 +08:00
parent 7a93c52b42
commit 8e58415173
42 changed files with 3741 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/assembly/backfiller.py
+27 -69
View File
@@ -1,9 +1,8 @@
from pathlib import Path
from typing import List, Dict, Optional
from bs4 import BeautifulSoup
from bs4 import BeautifulSoup, NavigableString
from src.common.data_model import BookStructure, ManifestEntry
from src.assembly.format_restorer import FormatRestorer
from src.common.utils import setup_logger
logger = setup_logger("backfill_engine")
@@ -11,11 +10,13 @@ logger = setup_logger("backfill_engine")
class BackfillEngine:
"""
Applies translations back to the BookStructure.
Now simplified to only inject 'translated_html' or fallback to text.
No format restoration logic here.
"""
def __init__(self, llm_client=None):
self.restorer = FormatRestorer()
self.llm_client = llm_client
def __init__(self):
# No more Restorer or LLM Client
pass
async def backfill(self, structure: BookStructure, manifest_entries: List[ManifestEntry], mode: str = "bilingual") -> BookStructure:
"""
@@ -28,18 +29,17 @@ class BackfillEngine:
"""
logger.info(f"Backfilling with mode: {mode}")
# Index manifest by file and element ID for faster lookup
# Map: file_path -> element_id -> ManifestEntry
# Index manifest
manifest_map: Dict[str, Dict[str, ManifestEntry]] = {}
for entry in manifest_entries:
if not entry.translated_text:
continue # Skip untranslated entries
continue
if entry.file_path not in manifest_map:
manifest_map[entry.file_path] = {}
manifest_map[entry.file_path][entry.element_id] = entry
# Iterate resources in structure
# Iterate resources
for item_id, resource in structure.resources.items():
if resource.media_type != "application/xhtml+xml" or resource.href not in manifest_map:
continue
@@ -59,50 +59,29 @@ class BackfillEngine:
logger.warning(f"Element {element_id} not found in {resource.href}")
continue
# Restore formatting with improved logic
restored_html, success = self.restorer.restore(
entry.translated_text,
entry.placeholders,
context_id=element_id
)
# If restoration failed and LLM is available, try to repair
if not success and self.llm_client:
logger.info(f"Attempting LLM repair for {element_id}...")
repaired_text = await self._repair_placeholders(entry)
if repaired_text:
# Retry restoration with repaired text
repaired_html, repaired_success = self.restorer.restore(
repaired_text,
entry.placeholders,
context_id=f"{element_id}-REPAIR"
)
if repaired_success:
logger.info(f"LLM Repair successful for {element_id}")
restored_html = repaired_html
# Update entry to reflect repair (optional, but good for logs)
entry.translated_text = repaired_text
else:
logger.warning(f"LLM Repair failed validation for {element_id}")
# Determine content to inject
# Prefer translated_html (rich text), fallback to translated_text (plain text)
html_content = entry.translated_html
plain_text = entry.translated_text
# Create translated tag
new_tag = soup.new_tag(element.name)
# Parse restored HTML to get content nodes
# Use html.parser but be careful about fragments
# Wrap in div just to parse then extract children
inner_soup = BeautifulSoup(f"<div>{restored_html}</div>", 'html.parser')
# inner_soup.div shouldn't be None if restored_html exists
container = inner_soup.find('div')
if container:
for child in list(container.children):
new_tag.append(child)
if html_content:
# Parse HTML fragment
# Wrap in div to handle multiple top-level nodes
inner_soup = BeautifulSoup(f"<div>{html_content}</div>", 'html.parser')
container = inner_soup.find('div')
if container:
for child in list(container.children):
new_tag.append(child)
else:
new_tag.string = plain_text
else:
# Fallback
new_tag.string = entry.translated_text
# Fallback to plain text
new_tag.string = plain_text
# Copy attributes
# Copy classes and add 'translation'
classes = element.get('class', [])
if isinstance(classes, str):
@@ -114,6 +93,7 @@ class BackfillEngine:
if style:
new_tag['style'] = style
# Injection Strategy
if mode == "bilingual":
element.insert_after(new_tag)
else:
@@ -125,25 +105,3 @@ class BackfillEngine:
resource.content = str(soup)
return structure
async def _repair_placeholders(self, entry: ManifestEntry) -> Optional[str]:
"""Ask LLM to fix placeholders in translated text."""
try:
system_prompt = "You are a translation repair assistant."
user_prompt = f"""
The following translation has incorrect placeholders.
Please fix the placeholders in the Translated text so they match the Original text structure EXACTLY.
Do NOT change the Chinese translation content, only fix the φXφ tags.
Original: {entry.original_text}
Translated (Broken): {entry.translated_text}
Output ONLY the fixed Translated text.
"""
# Use raw completion as we don't have short ID context here
# But LLMClient has raw_chat_completion
repaired = await self.llm_client.raw_chat_completion(system_prompt, user_prompt)
return repaired.strip()
except Exception as e:
logger.error(f"LLM Repair error: {e}")
return None
src/assembly/restoration_engine.py
+106
View File
@@ -0,0 +1,106 @@
import asyncio
from typing import List, Optional
from loguru import logger
from src.common.data_model import ManifestEntry
from src.assembly.format_restorer import FormatRestorer
from src.translation.llm_client import LLMClient
from src.common.utils import add_spacing
class RestorationEngine:
"""
Handles post-translation format restoration:
1. Spacing optimization (Pangu)
2. HTML tag restoration (FormatRestorer)
3. LLM-based repair (if restoration fails)
"""
def __init__(self, llm_client: Optional[LLMClient] = None):
self.restorer = FormatRestorer()
self.llm_client = llm_client
async def restore_entries(self, entries: List[ManifestEntry], force: bool = False) -> int:
"""
Process a list of entries, updating their translated_html field.
Returns the number of successfully restored entries.
"""
success_count = 0
for entry in entries:
if not entry.translated_text:
continue
# Idempotency check: Skip if already restored (unless forced)
if entry.translated_html and not force:
success_count += 1
continue
# 1. Spacing Fix
spaced_text = add_spacing(entry.translated_text)
# 2. Format Restoration
restored_html, success = self.restorer.restore(
spaced_text,
entry.placeholders,
context_id=entry.entry_id
)
# 3. LLM Repair (if needed and available)
if not success and self.llm_client:
logger.debug(f"Attempting LLM repair for {entry.entry_id}...")
repaired_text = await self._repair_placeholders(entry, spaced_text)
if repaired_text:
# Retry with repaired text
repaired_html, repaired_success = self.restorer.restore(
repaired_text,
entry.placeholders,
context_id=f"{entry.entry_id}-REPAIR"
)
if repaired_success:
logger.info(f"LLM Repair successful for {entry.entry_id}")
restored_html = repaired_html
# We do NOT update translated_text here to preserve original LLM output?
# Actually user might want the spaced and repaired text as the "text".
# But let's keep translated_text as raw-ish, and translated_html as final.
# Update: To ensure consistency, maybe we should update translated_text?
# The implementation plan says "Save result to ManifestEntry.translated_html".
# It implicitly leaves translated_text alone or updates it?
# Let's keep translated_text as is (except maybe spacing? no, keep it raw).
# But wait, if we repair key text, next time we run, we might want to use the repaired text?
# For now, only populate translated_html.
success = True
else:
logger.warning(f"LLM Repair failed validation for {entry.entry_id}")
# 4. Save Result
# Even if validation failed, we often get a "best effort" restored_html from restorer (fallback).
# The restorer usually returns *something*.
# If completely failed (e.g. mismatch), restorer might return raw text or partial?
# FormatRestorer.restore returns (restored_str, success_bool).
# Even if success=False, restored_str is produced (often just stripping unused placeholders or keeping them raw).
entry.translated_html = restored_html
if success:
success_count += 1
return success_count
async def _repair_placeholders(self, entry: ManifestEntry, current_text: str) -> Optional[str]:
"""Ask LLM to fix placeholders."""
try:
system_prompt = "You are a translation repair assistant."
user_prompt = f"""
The following translation has incorrect placeholders.
Please fix the placeholders in the Translated text so they match the Original text structure EXACTLY.
Do NOT change the Chinese translation content, only fix the φXφ tags.
Original: {entry.original_text}
Translated (Broken): {current_text}
Output ONLY the fixed Translated text.
"""
repaired = await self.llm_client.raw_chat_completion(system_prompt, user_prompt)
return repaired.strip()
except Exception as e:
logger.error(f"LLM Repair error: {e}")
return None
src/common/data_model.py
+1
View File
@@ -10,6 +10,7 @@ class ManifestEntry(BaseModel):
original_text: str
placeholders: Dict[str, str] = Field(default_factory=dict)
translated_text: Optional[str] = None
translated_html: Optional[str] = Field(None, description="Final HTML with restored tags and formatting")
context: Optional[str] = None
class BookMetaData(BaseModel):
src/common/utils.py
+18
View File
@@ -23,3 +23,21 @@ def ensure_directory(path: Path):
"""Ensures a directory exists."""
if not path.exists():
path.mkdir(parents=True, exist_ok=True)
def add_spacing(text: str) -> str:
"""
Add space between CJK and English/Number/Symbol characters.
Simplified version of pangu.js logic.
"""
import re
if not text:
return text
# CJK followed by non-CJK
text = re.sub(r'([\u4e00-\u9fa5])([a-zA-Z0-9])', r'\1 \2', text)
# Non-CJK followed by CJK
text = re.sub(r'([a-zA-Z0-9])([\u4e00-\u9fa5])', r'\1 \2', text)
# Optional: Handle symbols like quote against CJK?
# For now, stick to user request: "中英文数字混排" (Chinese-English-Numbers)
return text