feat: Release v0.10 - Modular Architecture & External Config
- Refactor codebase into src/ (preprocessing, translation, assembly) - Add pipeline/ scripts for individual stages - Externalize configuration to config/config.yaml - Fix Cover Image preservation - Update documentation and manuals
This commit is contained in:
谭凯
@@ -1,32 +0,0 @@
|
||||
"""
|
||||
EPUB 双语翻译程序
|
||||
主要功能模块的初始化文件
|
||||
"""
|
||||
|
||||
__version__ = "0.08"
|
||||
__author__ = "Kaitan"
|
||||
|
||||
from .epub_parser import EPUBParser
|
||||
from .translator import EPUBTranslator
|
||||
from .llm_client import LLMClient as OpenRouterClient # Keep alias for compatibility
|
||||
from .llm_client import LLMClient
|
||||
from .text_processor import TextProcessor
|
||||
from .bilingual_builder import BilingualEPUBBuilder
|
||||
from .chinese_builder import ChineseEPUBBuilder
|
||||
from .format_extractor import FormatExtractor
|
||||
from .format_restorer import FormatRestorer
|
||||
from .utils import load_config, setup_logging
|
||||
|
||||
__all__ = [
|
||||
"EPUBParser",
|
||||
"EPUBTranslator",
|
||||
"LLMClient",
|
||||
"OpenRouterClient",
|
||||
"TextProcessor",
|
||||
"BilingualEPUBBuilder",
|
||||
"ChineseEPUBBuilder",
|
||||
"FormatExtractor",
|
||||
"FormatRestorer",
|
||||
"load_config",
|
||||
"setup_logging"
|
||||
]
|
||||
@@ -0,0 +1,149 @@
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from src.common.data_model import BookStructure, ManifestEntry
|
||||
from src.assembly.format_restorer import FormatRestorer
|
||||
from src.common.utils import setup_logger
|
||||
|
||||
logger = setup_logger("backfill_engine")
|
||||
|
||||
class BackfillEngine:
|
||||
"""
|
||||
Applies translations back to the BookStructure.
|
||||
"""
|
||||
|
||||
def __init__(self, llm_client=None):
|
||||
self.restorer = FormatRestorer()
|
||||
self.llm_client = llm_client
|
||||
|
||||
async def backfill(self, structure: BookStructure, manifest_entries: List[ManifestEntry], mode: str = "bilingual") -> BookStructure:
|
||||
"""
|
||||
Modifies the BookStructure in-place with translations.
|
||||
|
||||
Args:
|
||||
structure: The BookStructure (from book_structure.json).
|
||||
manifest_entries: List of translations.
|
||||
mode: 'bilingual' or 'target_only'.
|
||||
"""
|
||||
logger.info(f"Backfilling with mode: {mode}")
|
||||
|
||||
# Index manifest by file and element ID for faster lookup
|
||||
# Map: file_path -> element_id -> ManifestEntry
|
||||
manifest_map: Dict[str, Dict[str, ManifestEntry]] = {}
|
||||
for entry in manifest_entries:
|
||||
if not entry.translated_text:
|
||||
continue # Skip untranslated entries
|
||||
|
||||
if entry.file_path not in manifest_map:
|
||||
manifest_map[entry.file_path] = {}
|
||||
manifest_map[entry.file_path][entry.element_id] = entry
|
||||
|
||||
# Iterate resources in structure
|
||||
for item_id, resource in structure.resources.items():
|
||||
if resource.media_type != "application/xhtml+xml" or resource.href not in manifest_map:
|
||||
continue
|
||||
|
||||
file_entries = manifest_map[resource.href]
|
||||
if not file_entries:
|
||||
continue
|
||||
|
||||
logger.debug(f"Processing {resource.href} with {len(file_entries)} translations")
|
||||
|
||||
soup = BeautifulSoup(resource.content, 'html.parser')
|
||||
modified = False
|
||||
|
||||
for element_id, entry in file_entries.items():
|
||||
element = soup.find(id=element_id)
|
||||
if not element:
|
||||
logger.warning(f"Element {element_id} not found in {resource.href}")
|
||||
continue
|
||||
|
||||
# Restore formatting with improved logic
|
||||
restored_html, success = self.restorer.restore(
|
||||
entry.translated_text,
|
||||
entry.placeholders,
|
||||
context_id=element_id
|
||||
)
|
||||
|
||||
# If restoration failed and LLM is available, try to repair
|
||||
if not success and self.llm_client:
|
||||
logger.info(f"Attempting LLM repair for {element_id}...")
|
||||
repaired_text = await self._repair_placeholders(entry)
|
||||
if repaired_text:
|
||||
# Retry restoration with repaired text
|
||||
repaired_html, repaired_success = self.restorer.restore(
|
||||
repaired_text,
|
||||
entry.placeholders,
|
||||
context_id=f"{element_id}-REPAIR"
|
||||
)
|
||||
if repaired_success:
|
||||
logger.info(f"LLM Repair successful for {element_id}")
|
||||
restored_html = repaired_html
|
||||
# Update entry to reflect repair (optional, but good for logs)
|
||||
entry.translated_text = repaired_text
|
||||
else:
|
||||
logger.warning(f"LLM Repair failed validation for {element_id}")
|
||||
|
||||
|
||||
# Create translated tag
|
||||
new_tag = soup.new_tag(element.name)
|
||||
# Parse restored HTML to get content nodes
|
||||
# Use html.parser but be careful about fragments
|
||||
# Wrap in div just to parse then extract children
|
||||
inner_soup = BeautifulSoup(f"<div>{restored_html}</div>", 'html.parser')
|
||||
# inner_soup.div shouldn't be None if restored_html exists
|
||||
container = inner_soup.find('div')
|
||||
|
||||
if container:
|
||||
for child in list(container.children):
|
||||
new_tag.append(child)
|
||||
else:
|
||||
# Fallback
|
||||
new_tag.string = entry.translated_text
|
||||
|
||||
|
||||
# Copy classes and add 'translation'
|
||||
classes = element.get('class', [])
|
||||
if isinstance(classes, str):
|
||||
classes = classes.split()
|
||||
new_tag['class'] = classes + ['translation']
|
||||
|
||||
# Copy style
|
||||
style = element.get('style')
|
||||
if style:
|
||||
new_tag['style'] = style
|
||||
|
||||
if mode == "bilingual":
|
||||
element.insert_after(new_tag)
|
||||
else:
|
||||
element.replace_with(new_tag)
|
||||
|
||||
modified = True
|
||||
|
||||
if modified:
|
||||
resource.content = str(soup)
|
||||
|
||||
return structure
|
||||
|
||||
async def _repair_placeholders(self, entry: ManifestEntry) -> Optional[str]:
|
||||
"""Ask LLM to fix placeholders in translated text."""
|
||||
try:
|
||||
system_prompt = "You are a translation repair assistant."
|
||||
user_prompt = f"""
|
||||
The following translation has incorrect placeholders.
|
||||
Please fix the placeholders in the Translated text so they match the Original text structure EXACTLY.
|
||||
Do NOT change the Chinese translation content, only fix the φXφ tags.
|
||||
|
||||
Original: {entry.original_text}
|
||||
Translated (Broken): {entry.translated_text}
|
||||
|
||||
Output ONLY the fixed Translated text.
|
||||
"""
|
||||
# Use raw completion as we don't have short ID context here
|
||||
# But LLMClient has raw_chat_completion
|
||||
repaired = await self.llm_client.raw_chat_completion(system_prompt, user_prompt)
|
||||
return repaired.strip()
|
||||
except Exception as e:
|
||||
logger.error(f"LLM Repair error: {e}")
|
||||
return None
|
||||
@@ -0,0 +1,289 @@
|
||||
import shutil
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from ebooklib import epub
|
||||
from src.common.data_model import BookStructure
|
||||
from src.common.utils import setup_logger, ensure_directory
|
||||
|
||||
logger = setup_logger("bilingual_builder")
|
||||
|
||||
class BilingualBuilder:
|
||||
"""
|
||||
Assembles the final EPUB from BookStructure.
|
||||
|
||||
Preserves the original TOC structure by reading it from the original EPUB.
|
||||
"""
|
||||
|
||||
def __init__(self, work_dir: Path, original_epub_path: Path = None):
|
||||
self.work_dir = work_dir
|
||||
self.assets_dir = work_dir / "assets"
|
||||
self.original_epub_path = original_epub_path
|
||||
self._original_book = None
|
||||
|
||||
def _load_original_book(self):
|
||||
"""Lazy load original book for TOC extraction."""
|
||||
if self._original_book is None and self.original_epub_path and self.original_epub_path.exists():
|
||||
try:
|
||||
self._original_book = epub.read_epub(str(self.original_epub_path))
|
||||
logger.debug(f"Loaded original EPUB for TOC: {self.original_epub_path}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load original EPUB: {e}")
|
||||
return self._original_book
|
||||
|
||||
def _sanitize_toc(self, toc):
|
||||
"""
|
||||
Ensure all TOC nodes have IDs (for ebooklib compatibility).
|
||||
From v0.08 bilingual_builder.py
|
||||
"""
|
||||
result = []
|
||||
for item in toc:
|
||||
if isinstance(item, (epub.Link, epub.Section)):
|
||||
if not getattr(item, 'uid', None):
|
||||
item.uid = f"navPoint-{uuid.uuid4().hex[:8]}"
|
||||
result.append(item)
|
||||
elif isinstance(item, tuple) and len(item) == 2:
|
||||
# Handle (Section, [children]) structure
|
||||
section, children = item
|
||||
if isinstance(section, (epub.Link, epub.Section)):
|
||||
if not getattr(section, 'uid', None):
|
||||
section.uid = f"navPoint-{uuid.uuid4().hex[:8]}"
|
||||
sanitized_children = self._sanitize_toc(children)
|
||||
result.append((section, sanitized_children))
|
||||
else:
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
def _validate_and_fix_toc(self, toc, book_items):
|
||||
"""
|
||||
Recursively validate and fix TOC links.
|
||||
Removes nodes with broken links that cannot be fixed.
|
||||
"""
|
||||
fixed_toc = []
|
||||
for item in toc:
|
||||
if isinstance(item, (epub.Link, epub.Section)):
|
||||
# Check href
|
||||
href = getattr(item, 'href', '')
|
||||
if href:
|
||||
# Remove anchor for check
|
||||
clean_href = href.split('#')[0]
|
||||
# Check if item exists in book (by file_name)
|
||||
found = False
|
||||
for existing_item in book_items.values():
|
||||
if existing_item.file_name == clean_href:
|
||||
found = True
|
||||
break
|
||||
if clean_href.endswith(existing_item.file_name) or existing_item.file_name.endswith(clean_href):
|
||||
item.href = existing_item.file_name
|
||||
found = True
|
||||
break
|
||||
|
||||
if not found:
|
||||
if 'c0.xhtml' in clean_href:
|
||||
for existing_item in book_items.values():
|
||||
if 'titlepage' in existing_item.file_name or 'cover' in existing_item.file_name.lower():
|
||||
if existing_item.media_type == "application/xhtml+xml":
|
||||
logger.info(f"Fixed TOC link: {href} -> {existing_item.file_name}")
|
||||
item.href = existing_item.file_name
|
||||
found = True
|
||||
break
|
||||
|
||||
if not found:
|
||||
logger.warning(f"Removing broken TOC link: {href}")
|
||||
continue
|
||||
|
||||
if isinstance(item, tuple) and len(item) == 2:
|
||||
section, children = item
|
||||
fixed_children = self._validate_and_fix_toc(children, book_items)
|
||||
fixed_toc.append((section, fixed_children))
|
||||
else:
|
||||
fixed_toc.append(item)
|
||||
|
||||
return fixed_toc
|
||||
|
||||
def build(self, structure: BookStructure, output_path: Path) -> Path:
|
||||
"""
|
||||
Builds the EPUB file.
|
||||
Returns the path to the generated EPUB.
|
||||
"""
|
||||
logger.info(f"Building final EPUB: {output_path}")
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
book = epub.EpubBook()
|
||||
|
||||
# 1. Metadata
|
||||
book.set_identifier(structure.metadata.identifier or f"uuid-{uuid.uuid4().hex[:12]}")
|
||||
book.set_title(structure.metadata.title)
|
||||
book.set_language(structure.metadata.language)
|
||||
book.add_author(structure.metadata.author)
|
||||
|
||||
# 2. Copy TOC from original EPUB if available
|
||||
original_book = self._load_original_book()
|
||||
if original_book and hasattr(original_book, 'toc') and original_book.toc:
|
||||
book.toc = self._sanitize_toc(original_book.toc)
|
||||
logger.info("Copied TOC structure from original EPUB")
|
||||
|
||||
|
||||
|
||||
# 3. Add Resources - First pass: Collect CSS items
|
||||
items_map = {} # id -> epub_item
|
||||
css_items = [] # List of CSS EpubItem for linking
|
||||
html_items = [] # List of (item_id, EpubHtml) tuples
|
||||
|
||||
for item_id, resource in structure.resources.items():
|
||||
# Skip NCX - we'll handle it separately
|
||||
if resource.media_type == "application/x-dtbncx+xml":
|
||||
continue
|
||||
|
||||
if resource.media_type == "application/xhtml+xml":
|
||||
# HTML Item - create but don't add yet (need to add CSS links)
|
||||
item = epub.EpubHtml(
|
||||
uid=item_id,
|
||||
file_name=resource.href,
|
||||
media_type=resource.media_type,
|
||||
content=resource.content.encode('utf-8')
|
||||
)
|
||||
html_items.append((item_id, item))
|
||||
elif resource.file_path:
|
||||
# Binary/Asset Item
|
||||
asset_full_path = self.work_dir / resource.file_path
|
||||
|
||||
if not asset_full_path.exists():
|
||||
logger.warning(f"Asset missing: {asset_full_path}")
|
||||
continue
|
||||
|
||||
with open(asset_full_path, 'rb') as f:
|
||||
content = f.read()
|
||||
|
||||
# Check if this is the cover image
|
||||
if structure.metadata.cover_image_id == item_id:
|
||||
logger.info(f"Setting cover image: {item_id}")
|
||||
# Manually add cover metadata to avoid "set_cover" creating a duplicate item with hardcoded ID
|
||||
# 1. Add item normally (as EpubImage)
|
||||
item = epub.EpubImage(
|
||||
uid=item_id,
|
||||
file_name=resource.href,
|
||||
media_type=resource.media_type,
|
||||
content=content
|
||||
)
|
||||
book.add_item(item)
|
||||
items_map[item_id] = item
|
||||
|
||||
# 2. Add metadata pointing to it
|
||||
book.add_metadata(None, 'meta', item_id, {'name': 'cover', 'content': item_id})
|
||||
continue
|
||||
|
||||
if "image" in resource.media_type:
|
||||
item = epub.EpubImage(
|
||||
uid=item_id,
|
||||
file_name=resource.href,
|
||||
media_type=resource.media_type,
|
||||
content=content
|
||||
)
|
||||
else:
|
||||
item = epub.EpubItem(
|
||||
uid=item_id,
|
||||
file_name=resource.href,
|
||||
media_type=resource.media_type,
|
||||
content=content
|
||||
)
|
||||
# Track CSS items
|
||||
if resource.media_type == "text/css":
|
||||
css_items.append(item)
|
||||
|
||||
book.add_item(item)
|
||||
items_map[item_id] = item
|
||||
else:
|
||||
logger.warning(f"Skipping resource {item_id}: No content or file path.")
|
||||
continue
|
||||
|
||||
# 4. Add HTML items with CSS links
|
||||
for item_id, item in html_items:
|
||||
html_dir = Path(item.file_name).parent
|
||||
for css_item in css_items:
|
||||
css_path = Path(css_item.file_name)
|
||||
# Calculate relative path from HTML directory to CSS file
|
||||
try:
|
||||
relative_css_path = Path(css_path).relative_to(html_dir)
|
||||
except ValueError:
|
||||
# Not a subpath, calculate full relative
|
||||
# Go up from html_dir, then down to css_path
|
||||
up_count = len(html_dir.parts)
|
||||
relative_css_path = Path("/".join([".."] * up_count)) / css_path
|
||||
|
||||
item.add_link(href=str(relative_css_path), rel='stylesheet', type='text/css')
|
||||
book.add_item(item)
|
||||
items_map[item_id] = item
|
||||
|
||||
# 5. Copy missing items from original EPUB (cover, etc.)
|
||||
# This ensures TOC links don't break
|
||||
if original_book:
|
||||
added_hrefs = {item.file_name for item in items_map.values() if hasattr(item, 'file_name')}
|
||||
|
||||
for orig_item in original_book.get_items():
|
||||
orig_name = orig_item.get_name()
|
||||
|
||||
# Check for existence (exact or suffix overlap to handle EPUB/OEBPS prefixes)
|
||||
is_duplicate = False
|
||||
if orig_name in added_hrefs:
|
||||
is_duplicate = True
|
||||
else:
|
||||
for added in added_hrefs:
|
||||
if added.endswith(orig_name) or orig_name.endswith(added):
|
||||
is_duplicate = True
|
||||
break
|
||||
|
||||
if is_duplicate:
|
||||
continue
|
||||
|
||||
# Skip NCX and NAV - we generate these
|
||||
if 'ncx' in orig_name.lower() or orig_name.endswith('nav.xhtml'):
|
||||
continue
|
||||
|
||||
# Double check if it's an image we likely already processed
|
||||
if orig_item.media_type and orig_item.media_type.startswith("image/"):
|
||||
# If we didn't match via filename, verify if we missed it or if it's truly new
|
||||
# But suffix check usually catches it.
|
||||
pass
|
||||
|
||||
# Copy the item directly
|
||||
try:
|
||||
book.add_item(orig_item)
|
||||
items_map[orig_item.id] = orig_item
|
||||
logger.debug(f"Copied missing item from original: {orig_name}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to copy item {orig_name}: {e}")
|
||||
|
||||
# 6. Spine
|
||||
book.spine = []
|
||||
for item_id in structure.spine:
|
||||
if item_id in items_map:
|
||||
book.spine.append(items_map[item_id])
|
||||
else:
|
||||
logger.warning(f"Spine item {item_id} not found in resources.")
|
||||
|
||||
# Add missing spine items from original
|
||||
if original_book:
|
||||
for spine_id, _ in original_book.spine:
|
||||
if spine_id not in [i.id for i in book.spine]:
|
||||
orig_item = original_book.get_item_with_id(spine_id)
|
||||
if orig_item and orig_item.id in items_map:
|
||||
book.spine.append(items_map[orig_item.id])
|
||||
|
||||
# Validate and fix TOC
|
||||
if original_book and hasattr(book, 'toc') and book.toc:
|
||||
try:
|
||||
book.toc = self._validate_and_fix_toc(book.toc, items_map)
|
||||
logger.info("Validated and fixed TOC links")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to validate TOC: {e}")
|
||||
|
||||
# 7. Navigation - NCX and Nav
|
||||
book.add_item(epub.EpubNcx())
|
||||
book.add_item(epub.EpubNav())
|
||||
|
||||
# 7. Write
|
||||
epub.write_epub(str(output_path), book)
|
||||
logger.info(f"EPUB created successfully at {output_path}")
|
||||
|
||||
return output_path
|
||||
|
||||
@@ -1,89 +1,72 @@
|
||||
"""
|
||||
格式恢复模块 (优化版 v4)
|
||||
|
||||
负责：
|
||||
1. 解析译文中的配对占位符 (φ1φ...φ/1φ)
|
||||
2. 还原前缀和后缀标签（_prefix, _suffix）
|
||||
3. 从映射表中查找对应的 HTML 片段并替换
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Dict, Tuple, List, Optional
|
||||
from loguru import logger
|
||||
from src.common.utils import setup_logger
|
||||
|
||||
logger = setup_logger("format_restorer")
|
||||
|
||||
class FormatRestorer:
|
||||
"""
|
||||
格式恢复器 (优化版 v4)
|
||||
|
||||
支持：
|
||||
- 配对占位符 φ1φ...φ/1φ
|
||||
- 前缀/后缀自动回填 (_prefix, _suffix)
|
||||
Restores HTML formatting from placeholders.
|
||||
"""
|
||||
|
||||
# 占位符正则: φ1φ, φ/1φ, φ12φ, φ/12φ (支持配对格式)
|
||||
PLACEHOLDER_REGEX = re.compile(r'φ(/?\d+)φ')
|
||||
|
||||
def restore(self, text_with_placeholders: str, placeholder_map: Dict[str, str]) -> Tuple[str, bool]:
|
||||
def restore(self, text_with_placeholders: str, placeholder_map: Dict[str, str], context_id: str = None) -> Tuple[str, bool]:
|
||||
"""
|
||||
将带占位符的文本还原为 HTML
|
||||
|
||||
自动处理 _prefix 和 _suffix 键，以及配对占位符 φ1φ...φ/1φ
|
||||
|
||||
Returns:
|
||||
(html_string, success): 还原后的 HTML 和是否完全成功的标志
|
||||
Restores HTML from text with placeholders.
|
||||
Returns (restored_html, success).
|
||||
"""
|
||||
if not placeholder_map:
|
||||
return text_with_placeholders or "", True
|
||||
|
||||
if not text_with_placeholders:
|
||||
# 没有文本，只有前缀/后缀（如空元素）
|
||||
prefix = placeholder_map.get("_prefix", "")
|
||||
suffix = placeholder_map.get("_suffix", "")
|
||||
return prefix + suffix, True
|
||||
|
||||
# 提取前缀和后缀
|
||||
prefix = placeholder_map.get("_prefix", "")
|
||||
suffix = placeholder_map.get("_suffix", "")
|
||||
|
||||
# 创建只包含占位符键的映射（排除 _prefix, _suffix）
|
||||
inner_map = {k: v for k, v in placeholder_map.items() if not k.startswith("_")}
|
||||
|
||||
# 校验占位符
|
||||
found_ids = set(self.PLACEHOLDER_REGEX.findall(text_with_placeholders))
|
||||
expected_ids = set(inner_map.keys())
|
||||
|
||||
success = True
|
||||
context_msg = f" [ID: ...{context_id[-6:]}]" if context_id else ""
|
||||
|
||||
missing_ids = expected_ids - found_ids
|
||||
if missing_ids:
|
||||
logger.warning(f"格式还原警告: 丢失占位符 {missing_ids}")
|
||||
logger.warning(f"Restoration warning: missing placeholders {missing_ids}{context_msg}")
|
||||
success = False
|
||||
|
||||
unknown_ids = found_ids - expected_ids
|
||||
if unknown_ids:
|
||||
logger.warning(f"格式还原警告: 发现未知占位符 {unknown_ids}")
|
||||
success = False # 未知占位符也视为问题
|
||||
real_unknowns = set()
|
||||
for pid in unknown_ids:
|
||||
if pid.startswith('/') and pid[1:] in expected_ids:
|
||||
continue
|
||||
real_unknowns.add(pid)
|
||||
|
||||
if real_unknowns:
|
||||
logger.warning(f"Restoration warning: unknown placeholders {real_unknowns}{context_msg}")
|
||||
success = False
|
||||
|
||||
# 替换占位符
|
||||
def replace_match(match):
|
||||
pid = match.group(1) # 可能是 "1" 或 "/1"
|
||||
pid = match.group(1)
|
||||
if pid in inner_map:
|
||||
return inner_map[pid]
|
||||
else:
|
||||
return "" # 删除未知占位符
|
||||
return ""
|
||||
|
||||
try:
|
||||
# 还原占位符
|
||||
restored_inner = self.PLACEHOLDER_REGEX.sub(replace_match, text_with_placeholders)
|
||||
|
||||
# 添加前缀和后缀
|
||||
restored_html = prefix + restored_inner + suffix
|
||||
|
||||
return restored_html, success
|
||||
except Exception as e:
|
||||
logger.error(f"格式还原失败: {e}")
|
||||
logger.error(f"Restoration failed: {e}")
|
||||
return prefix + self._strip_placeholders(text_with_placeholders) + suffix, False
|
||||
|
||||
def _strip_placeholders(self, text: str) -> str:
|
||||
"""移除所有 φ...φ 占位符"""
|
||||
return self.PLACEHOLDER_REGEX.sub("", text)
|
||||
return self.PLACEHOLDER_REGEX.sub("", text)
|
||||
@@ -1,180 +0,0 @@
|
||||
"""
|
||||
双语 EPUB 构建器模块 - 安全的EPUB构建 (Manifest 兼容版)
|
||||
"""
|
||||
|
||||
from ebooklib import epub
|
||||
import ebooklib
|
||||
from bs4 import BeautifulSoup
|
||||
from typing import Dict, List
|
||||
from pathlib import Path
|
||||
from loguru import logger
|
||||
import uuid
|
||||
|
||||
|
||||
class BilingualEPUBBuilder:
|
||||
"""双语 EPUB 构建器"""
|
||||
|
||||
def __init__(self, original_book, config: Dict):
|
||||
self.original_book = original_book
|
||||
self.config = config
|
||||
self.output_config = config['output']
|
||||
|
||||
def create_bilingual_epub_with_mapping(self, translation_map: Dict[str, str],
|
||||
paragraph_map: Dict[str, Dict],
|
||||
output_path: str) -> str:
|
||||
"""
|
||||
创建双语 EPUB。使用 ordered_ids 确保与 Manifest 严格一致。
|
||||
"""
|
||||
try:
|
||||
new_book = epub.EpubBook()
|
||||
self._copy_metadata(new_book)
|
||||
new_book.toc = self._sanitize_toc(self.original_book.toc)
|
||||
|
||||
# 准备每个文件的有序ID列表
|
||||
file_ordered_ids = {}
|
||||
sorted_pids = sorted(paragraph_map.keys(), key=lambda x: int(x.split('_')[1]))
|
||||
for pid in sorted_pids:
|
||||
info = paragraph_map[pid]
|
||||
fname = info['file_name']
|
||||
if fname not in file_ordered_ids:
|
||||
file_ordered_ids[fname] = []
|
||||
file_ordered_ids[fname].append(pid)
|
||||
|
||||
processed_item_ids = set()
|
||||
item_map = {}
|
||||
|
||||
# 特殊处理：封面图片
|
||||
cover_id_meta = self.original_book.get_metadata('OPF', 'cover')
|
||||
if cover_id_meta:
|
||||
cover_item = self.original_book.get_item_with_id(cover_id_meta[0][0])
|
||||
if cover_item:
|
||||
new_book.add_item(cover_item)
|
||||
processed_item_ids.add(cover_item.id)
|
||||
item_map[cover_item.id] = cover_item
|
||||
|
||||
# 复制资源
|
||||
for item in self.original_book.get_items():
|
||||
if item.get_type() != ebooklib.ITEM_DOCUMENT:
|
||||
if item.id not in processed_item_ids:
|
||||
new_book.add_item(item)
|
||||
processed_item_ids.add(item.id)
|
||||
item_map[item.id] = item
|
||||
|
||||
# 重建 Spine
|
||||
new_spine = []
|
||||
for spine_id, linear in self.original_book.spine:
|
||||
item = self.original_book.get_item_with_id(spine_id)
|
||||
if not item: continue
|
||||
|
||||
if item.get_type() == ebooklib.ITEM_DOCUMENT:
|
||||
file_name = item.get_name()
|
||||
if file_name in file_ordered_ids:
|
||||
new_item = self._create_bilingual_document(
|
||||
item, file_ordered_ids[file_name], translation_map
|
||||
)
|
||||
new_item.id = item.id
|
||||
else:
|
||||
new_item = item
|
||||
|
||||
if new_item.id not in processed_item_ids:
|
||||
new_book.add_item(new_item)
|
||||
processed_item_ids.add(new_item.id)
|
||||
new_spine.append(new_item)
|
||||
else:
|
||||
if item.id in item_map:
|
||||
new_spine.append(item_map[item.id])
|
||||
|
||||
new_book.spine = new_spine
|
||||
new_book.add_item(epub.EpubNcx())
|
||||
new_book.add_item(epub.EpubNav())
|
||||
|
||||
output_file = self._generate_output_filename(output_path)
|
||||
epub.write_epub(output_file, new_book, {})
|
||||
return output_file
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"创建双语 EPUB 失败: {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
def _sanitize_toc(self, toc):
|
||||
"""确保 TOC 中的所有节点都有 ID，修复 ebooklib 读取后写入的兼容性问题"""
|
||||
for item in toc:
|
||||
if isinstance(item, (epub.Link, epub.Section)):
|
||||
if not getattr(item, 'uid', None):
|
||||
item.uid = f"navPoint-{uuid.uuid4().hex[:8]}"
|
||||
elif isinstance(item, tuple) and len(item) == 2:
|
||||
# 处理 (Section, [children]) 结构
|
||||
section, children = item
|
||||
if isinstance(section, (epub.Link, epub.Section)):
|
||||
if not getattr(section, 'uid', None):
|
||||
section.uid = f"navPoint-{uuid.uuid4().hex[:8]}"
|
||||
self._sanitize_toc(children)
|
||||
return toc
|
||||
|
||||
def _copy_metadata(self, new_book):
|
||||
try:
|
||||
for namespace, meta_dict in self.original_book.metadata.items():
|
||||
for name, values in meta_dict.items():
|
||||
for value, other in values:
|
||||
if name and hasattr(name, 'lower') and name.lower() == 'identifier': continue
|
||||
new_book.add_metadata(namespace, name, value, other)
|
||||
new_book.add_metadata('DC', 'language', 'zh-CN')
|
||||
new_book.set_identifier(f"bilingual-{uuid.uuid4().hex[:12]}")
|
||||
|
||||
cover_id_meta = self.original_book.get_metadata('OPF', 'cover')
|
||||
if cover_id_meta:
|
||||
cover_item = self.original_book.get_item_with_id(cover_id_meta[0][0])
|
||||
if cover_item:
|
||||
new_book.add_item(cover_item)
|
||||
new_book.set_cover(cover_item.get_name(), cover_item.get_content())
|
||||
except Exception as e:
|
||||
logger.error(f"元数据复制出错: {e}")
|
||||
|
||||
def _create_bilingual_document(self, original_item, ordered_ids: list, translation_map: dict):
|
||||
try:
|
||||
from .text_processor import TextProcessor
|
||||
soup = BeautifulSoup(original_item.get_content().decode('utf-8'), 'html.parser')
|
||||
self._add_style_link(soup)
|
||||
|
||||
# 使用与 TextProcessor 相同的过滤逻辑获取元素
|
||||
text_elements = TextProcessor.get_valid_text_elements(soup)
|
||||
|
||||
current_para_index = 0
|
||||
for element in text_elements:
|
||||
if TextProcessor.is_navigation_element(element): continue
|
||||
if not TextProcessor.clean_element_text(element): continue
|
||||
|
||||
if current_para_index < len(ordered_ids):
|
||||
target_id = ordered_ids[current_para_index]
|
||||
translation = translation_map.get(target_id)
|
||||
if translation:
|
||||
self._insert_translation(element, translation, soup)
|
||||
current_para_index += 1
|
||||
|
||||
new_item = epub.EpubHtml(title=original_item.title, file_name=original_item.get_name(), lang='zh-CN')
|
||||
new_item.set_content(str(soup).encode('utf-8'))
|
||||
return new_item
|
||||
except Exception as e:
|
||||
logger.error(f"创建双语文档失败 {original_item.get_name()}: {e}")
|
||||
return original_item
|
||||
|
||||
def _add_style_link(self, soup):
|
||||
head = soup.find('head')
|
||||
if head and not head.find('link', href='style/bilingual.css'):
|
||||
head.append(soup.new_tag('link', rel='stylesheet', type='text/css', href='style/bilingual.css'))
|
||||
|
||||
def _insert_translation(self, element, translation: str, soup):
|
||||
try:
|
||||
translation_p = soup.new_tag('p')
|
||||
translation_p.string = translation
|
||||
translation_p['class'] = ['translation-text', 'chinese']
|
||||
element.insert_after(translation_p)
|
||||
except Exception as e:
|
||||
logger.warning(f"插入翻译失败: {e}")
|
||||
|
||||
def _generate_output_filename(self, output_path: str) -> str:
|
||||
from .utils import sanitize_filename
|
||||
title = self.original_book.get_metadata('DC', 'title')
|
||||
clean_title = sanitize_filename(title[0][0]) if title else "bilingual_book"
|
||||
Path(output_path).mkdir(parents=True, exist_ok=True)
|
||||
return str(Path(output_path) / f"{clean_title}_bilingual.epub")
|
||||
@@ -1,84 +0,0 @@
|
||||
|
||||
'''Book Profiler Module
|
||||
|
||||
Features:
|
||||
1. Automatically extract book samples to generate Book Profile (Genre, Style, Glossary).
|
||||
'''
|
||||
|
||||
import json
|
||||
import random
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
from loguru import logger
|
||||
from .manifest_manager import ManifestManager
|
||||
from .llm_client import LLMClient
|
||||
|
||||
class BookProfiler:
|
||||
def __init__(self, config: Dict, llm_client: LLMClient):
|
||||
self.config = config
|
||||
self.llm_client = llm_client
|
||||
|
||||
def extract_sample_text(self, manifest: ManifestManager, char_limit: int = 3000) -> str:
|
||||
"""Extract sample text."""
|
||||
items = manifest.get_items()
|
||||
if not items: return ""
|
||||
|
||||
intro_text = []
|
||||
for item in items[:50]:
|
||||
if len(item.clean_text) > 50:
|
||||
intro_text.append(item.clean_text)
|
||||
|
||||
body_text = []
|
||||
body_items = [i for i in items[50:] if len(i.clean_text) > 80]
|
||||
if body_items:
|
||||
samples = random.sample(body_items, min(5, len(body_items)))
|
||||
body_text = [i.clean_text for i in samples]
|
||||
|
||||
full_text = "\n\n".join(intro_text[:5] + body_text)
|
||||
return full_text[:char_limit]
|
||||
|
||||
async def analyze_book(self, manifest: ManifestManager) -> Dict:
|
||||
"""Generate Book Profile."""
|
||||
existing_profile = manifest.data.get('metadata', {}).get('profile')
|
||||
if existing_profile:
|
||||
logger.info("Loaded existing Book Profile")
|
||||
return existing_profile
|
||||
|
||||
sample = self.extract_sample_text(manifest)
|
||||
if not sample: return {}
|
||||
|
||||
logger.info("Generating Book Profile...")
|
||||
|
||||
system_prompt = "You are a senior publishing editor. Analyze the text and output JSON."
|
||||
user_prompt = f"""
|
||||
Please analyze the following book excerpt.
|
||||
Output JSON format:
|
||||
{{
|
||||
"genre": "Genre",
|
||||
"style": "Style description",
|
||||
"audience": "Target Audience",
|
||||
"glossary": {{ "Term": "Chinese Translation" }},
|
||||
"translation_instruction": "Specific instruction for translator"
|
||||
}}
|
||||
|
||||
Excerpt:
|
||||
{sample}
|
||||
"""
|
||||
try:
|
||||
response = await self.llm_client.raw_chat_completion(system_prompt, user_prompt)
|
||||
json_str = response.strip()
|
||||
if "```json" in json_str:
|
||||
json_str = json_str.split("```json")[1].split("```")[0].strip()
|
||||
elif "```" in json_str:
|
||||
json_str = json_str.split("```")[1].split("```")[0].strip()
|
||||
|
||||
profile = json.loads(json_str)
|
||||
|
||||
if 'metadata' not in manifest.data:
|
||||
manifest.data['metadata'] = {}
|
||||
manifest.data['metadata']['profile'] = profile
|
||||
manifest.save()
|
||||
return profile
|
||||
except Exception as e:
|
||||
logger.error(f"Profile generation failed: {e}")
|
||||
return {}
|
||||
-226
@@ -1,226 +0,0 @@
|
||||
"""
|
||||
翻译缓存管理模块 - 简化版
|
||||
基于全局ID和chunk的缓存系统
|
||||
"""
|
||||
|
||||
import json
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, Optional, List
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class TranslationCache:
|
||||
"""翻译缓存管理器 - 简化版"""
|
||||
|
||||
def __init__(self, config: Dict):
|
||||
"""初始化缓存管理器"""
|
||||
self.config = config
|
||||
cache_config = config.get('cache', {})
|
||||
|
||||
self.enabled = cache_config.get('enabled', True)
|
||||
self.cache_dir = Path(cache_config.get('directory', 'cache'))
|
||||
self.max_age_days = cache_config.get('max_age_days', 30)
|
||||
|
||||
if self.enabled:
|
||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.translations_dir = self.cache_dir / 'translations'
|
||||
self.translations_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.info(f"翻译缓存已启用: {self.cache_dir}")
|
||||
|
||||
def get_chunk_translation(self, chunk: List[Dict], model: str) -> Optional[Dict[str, str]]:
|
||||
"""
|
||||
获取chunk的缓存翻译
|
||||
|
||||
Args:
|
||||
chunk: 段落列表（带global_id）
|
||||
model: 模型名称
|
||||
|
||||
Returns:
|
||||
{global_id: translation} 映射，如果不存在返回 None
|
||||
"""
|
||||
if not self.enabled:
|
||||
return None
|
||||
|
||||
try:
|
||||
cache_key = self._get_chunk_cache_key(chunk, model)
|
||||
cache_file = self._get_cache_file_path(cache_key)
|
||||
|
||||
if not cache_file.exists():
|
||||
return None
|
||||
|
||||
# 检查是否过期
|
||||
file_age = datetime.now() - datetime.fromtimestamp(cache_file.stat().st_mtime)
|
||||
if file_age > timedelta(days=self.max_age_days):
|
||||
logger.debug(f"缓存已过期: {cache_key[:8]}...")
|
||||
cache_file.unlink()
|
||||
return None
|
||||
|
||||
# 读取缓存
|
||||
with open(cache_file, 'r', encoding='utf-8') as f:
|
||||
cache_data = json.load(f)
|
||||
|
||||
# 验证缓存
|
||||
if (cache_data.get('success') and
|
||||
cache_data.get('model') == model and
|
||||
self._validate_cache_data(cache_data, chunk)):
|
||||
|
||||
logger.debug(f"缓存命中: {cache_key[:8]}... ({len(chunk)} 段落)")
|
||||
return cache_data.get('translations', {})
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"读取缓存失败: {e}")
|
||||
return None
|
||||
|
||||
def save_chunk_translation(self, chunk: List[Dict], translations: Dict[str, str],
|
||||
model: str, success: bool = True) -> None:
|
||||
"""
|
||||
保存chunk翻译到缓存
|
||||
|
||||
Args:
|
||||
chunk: 段落列表（带global_id）
|
||||
translations: {global_id: translation} 映射
|
||||
model: 模型名称
|
||||
success: 是否翻译成功
|
||||
"""
|
||||
if not self.enabled:
|
||||
return
|
||||
|
||||
try:
|
||||
cache_key = self._get_chunk_cache_key(chunk, model)
|
||||
cache_file = self._get_cache_file_path(cache_key)
|
||||
|
||||
# 构建缓存数据
|
||||
cache_data = {
|
||||
'global_ids': [p['global_id'] for p in chunk],
|
||||
'translations': translations,
|
||||
'model': model,
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'success': success,
|
||||
'paragraph_count': len(chunk),
|
||||
'cache_version': '3.0'
|
||||
}
|
||||
|
||||
with open(cache_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(cache_data, f, ensure_ascii=False, indent=2)
|
||||
|
||||
logger.debug(f"缓存已保存: {cache_key[:8]}... ({len(chunk)} 段落)")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"保存缓存失败: {e}")
|
||||
|
||||
def _get_chunk_cache_key(self, chunk: List[Dict], model: str) -> str:
|
||||
"""
|
||||
生成chunk缓存键（基于全局ID序列）
|
||||
|
||||
Args:
|
||||
chunk: 段落列表
|
||||
model: 模型名称
|
||||
|
||||
Returns:
|
||||
缓存键
|
||||
"""
|
||||
# 使用全局ID序列作为缓存键的一部分
|
||||
id_sequence = ",".join(p['global_id'] for p in chunk)
|
||||
combined = f"{id_sequence}|{model}"
|
||||
return hashlib.md5(combined.encode('utf-8')).hexdigest()
|
||||
|
||||
def _get_cache_file_path(self, cache_key: str) -> Path:
|
||||
"""获取缓存文件路径"""
|
||||
# 使用 hash 前缀分目录，避免单目录文件过多
|
||||
subdir = cache_key[:2]
|
||||
cache_subdir = self.translations_dir / subdir
|
||||
cache_subdir.mkdir(parents=True, exist_ok=True)
|
||||
return cache_subdir / f"{cache_key}.json"
|
||||
|
||||
def _validate_cache_data(self, cache_data: Dict, chunk: List[Dict]) -> bool:
|
||||
"""验证缓存数据的有效性"""
|
||||
# 检查ID序列是否匹配
|
||||
cached_ids = cache_data.get('global_ids', [])
|
||||
chunk_ids = [p['global_id'] for p in chunk]
|
||||
|
||||
if cached_ids != chunk_ids:
|
||||
logger.debug("缓存ID序列不匹配")
|
||||
return False
|
||||
|
||||
# 只要有翻译结果就认为有效，不要求数量完全匹配
|
||||
translations = cache_data.get('translations', {})
|
||||
if not translations:
|
||||
logger.debug("缓存无翻译结果")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def clear_cache(self, older_than_days: Optional[int] = None) -> int:
|
||||
"""清理缓存"""
|
||||
if not self.enabled or not self.translations_dir.exists():
|
||||
return 0
|
||||
|
||||
cleared_count = 0
|
||||
cutoff_time = None
|
||||
|
||||
if older_than_days is not None:
|
||||
cutoff_time = datetime.now() - timedelta(days=older_than_days)
|
||||
|
||||
try:
|
||||
for cache_file in self.translations_dir.rglob('*.json'):
|
||||
should_delete = False
|
||||
|
||||
if cutoff_time is None:
|
||||
should_delete = True
|
||||
else:
|
||||
file_time = datetime.fromtimestamp(cache_file.stat().st_mtime)
|
||||
should_delete = file_time < cutoff_time
|
||||
|
||||
if should_delete:
|
||||
cache_file.unlink()
|
||||
cleared_count += 1
|
||||
|
||||
# 清理空目录
|
||||
for date_dir in self.translations_dir.iterdir():
|
||||
if date_dir.is_dir() and not any(date_dir.iterdir()):
|
||||
date_dir.rmdir()
|
||||
|
||||
logger.info(f"清理了 {cleared_count} 个缓存文件")
|
||||
return cleared_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"清理缓存失败: {e}")
|
||||
return 0
|
||||
|
||||
def get_cache_stats(self) -> Dict:
|
||||
"""获取缓存统计信息"""
|
||||
if not self.enabled or not self.translations_dir.exists():
|
||||
return {'enabled': False}
|
||||
|
||||
try:
|
||||
cache_files = list(self.translations_dir.rglob('*.json'))
|
||||
total_files = len(cache_files)
|
||||
total_size = sum(f.stat().st_size for f in cache_files)
|
||||
|
||||
# 统计段落数
|
||||
total_paragraphs = 0
|
||||
for cache_file in cache_files:
|
||||
try:
|
||||
with open(cache_file, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
total_paragraphs += data.get('paragraph_count', 0)
|
||||
except:
|
||||
continue
|
||||
|
||||
return {
|
||||
'enabled': True,
|
||||
'total_files': total_files,
|
||||
'total_paragraphs': total_paragraphs,
|
||||
'total_size_mb': round(total_size / 1024 / 1024, 2),
|
||||
'cache_directory': str(self.cache_dir),
|
||||
'max_age_days': self.max_age_days
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"获取缓存统计失败: {e}")
|
||||
return {'enabled': True, 'error': str(e)}
|
||||
@@ -1,230 +0,0 @@
|
||||
"""
|
||||
纯中文 EPUB 构建器模块
|
||||
|
||||
负责：
|
||||
1. 复制原始 EPUB 结构
|
||||
2. 使用译文替换原文
|
||||
3. 调用 FormatRestorer 将译文占位符还原为 HTML 标签
|
||||
"""
|
||||
|
||||
from ebooklib import epub
|
||||
import ebooklib
|
||||
from bs4 import BeautifulSoup
|
||||
from typing import Dict, List
|
||||
from pathlib import Path
|
||||
from loguru import logger
|
||||
import uuid
|
||||
|
||||
class ChineseEPUBBuilder:
|
||||
"""纯中文 EPUB 构建器"""
|
||||
|
||||
def __init__(self, original_book, config: Dict):
|
||||
self.original_book = original_book
|
||||
self.config = config
|
||||
self.output_config = config['output']
|
||||
|
||||
def create_chinese_epub_with_mapping(self,
|
||||
items: List, # List[ManifestItem]
|
||||
output_path: str) -> str:
|
||||
"""
|
||||
创建纯中文 EPUB。
|
||||
"""
|
||||
try:
|
||||
new_book = epub.EpubBook()
|
||||
self._copy_metadata(new_book)
|
||||
new_book.toc = self._sanitize_toc(self.original_book.toc)
|
||||
|
||||
# 准备每个文件的有序项目列表
|
||||
file_items = {}
|
||||
for item in sorted(items, key=lambda x: x.global_id):
|
||||
fname = item.source_file
|
||||
if fname not in file_items:
|
||||
file_items[fname] = []
|
||||
file_items[fname].append(item)
|
||||
|
||||
processed_item_ids = set()
|
||||
item_map = {}
|
||||
|
||||
# 特殊处理：封面图片
|
||||
cover_id_meta = self.original_book.get_metadata('OPF', 'cover')
|
||||
if cover_id_meta:
|
||||
cover_item = self.original_book.get_item_with_id(cover_id_meta[0][0])
|
||||
if cover_item:
|
||||
new_book.add_item(cover_item)
|
||||
processed_item_ids.add(cover_item.id)
|
||||
item_map[cover_item.id] = cover_item
|
||||
|
||||
# 复制资源
|
||||
for item in self.original_book.get_items():
|
||||
if item.get_type() != ebooklib.ITEM_DOCUMENT:
|
||||
if item.id not in processed_item_ids:
|
||||
new_book.add_item(item)
|
||||
processed_item_ids.add(item.id)
|
||||
item_map[item.id] = item
|
||||
|
||||
# 重建 Spine
|
||||
new_spine = []
|
||||
for spine_id, linear in self.original_book.spine:
|
||||
item = self.original_book.get_item_with_id(spine_id)
|
||||
if not item: continue
|
||||
|
||||
if item.get_type() == ebooklib.ITEM_DOCUMENT:
|
||||
file_name = item.get_name()
|
||||
if file_name in file_items:
|
||||
new_item = self._create_chinese_document(
|
||||
item, file_items[file_name]
|
||||
)
|
||||
new_item.id = item.id
|
||||
else:
|
||||
new_item = item
|
||||
|
||||
if new_item.id not in processed_item_ids:
|
||||
new_book.add_item(new_item)
|
||||
processed_item_ids.add(new_item.id)
|
||||
new_spine.append(new_item)
|
||||
else:
|
||||
if item.id in item_map:
|
||||
new_spine.append(item_map[item.id])
|
||||
|
||||
new_book.spine = new_spine
|
||||
new_book.add_item(epub.EpubNcx())
|
||||
new_book.add_item(epub.EpubNav())
|
||||
|
||||
output_file = self._generate_output_filename(output_path)
|
||||
epub.write_epub(output_file, new_book, {})
|
||||
return output_file
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"创建纯中文 EPUB 失败: {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
def _sanitize_toc(self, toc):
|
||||
"""确保 TOC 中的所有节点都有 ID"""
|
||||
for item in toc:
|
||||
if isinstance(item, (epub.Link, epub.Section)):
|
||||
if not getattr(item, 'uid', None):
|
||||
item.uid = f"navPoint-{uuid.uuid4().hex[:8]}"
|
||||
elif isinstance(item, tuple) and len(item) == 2:
|
||||
section, children = item
|
||||
if isinstance(section, (epub.Link, epub.Section)):
|
||||
if not getattr(section, 'uid', None):
|
||||
section.uid = f"navPoint-{uuid.uuid4().hex[:8]}"
|
||||
self._sanitize_toc(children)
|
||||
return toc
|
||||
|
||||
def _copy_metadata(self, new_book):
|
||||
try:
|
||||
for namespace, meta_dict in self.original_book.metadata.items():
|
||||
for name, values in meta_dict.items():
|
||||
for value, other in values:
|
||||
if name and hasattr(name, 'lower') and name.lower() == 'identifier': continue
|
||||
new_book.add_metadata(namespace, name, value, other)
|
||||
new_book.add_metadata('DC', 'language', 'zh-CN')
|
||||
new_book.set_identifier(f"chinese-{uuid.uuid4().hex[:12]}")
|
||||
except Exception as e:
|
||||
logger.error(f"元数据复制出错: {e}")
|
||||
|
||||
def _create_chinese_document(self, original_item, manifest_items: list):
|
||||
try:
|
||||
from .text_processor import TextProcessor
|
||||
soup = BeautifulSoup(original_item.get_content().decode('utf-8'), 'html.parser')
|
||||
|
||||
# 获取文本元素
|
||||
text_elements = TextProcessor.get_valid_text_elements(soup)
|
||||
|
||||
current_idx = 0
|
||||
for element in text_elements:
|
||||
if not TextProcessor.clean_element_text(element): continue
|
||||
|
||||
if current_idx < len(manifest_items):
|
||||
m_item = manifest_items[current_idx]
|
||||
|
||||
# 只有当非导航元素时才尝试替换内容
|
||||
if not TextProcessor.is_navigation_element(element):
|
||||
# 检查是否是嵌套容器（需要特殊处理）
|
||||
is_nested = TextProcessor.is_nested_container(element)
|
||||
|
||||
# 优先使用带格式的翻译，降级到纯文本翻译
|
||||
if m_item.translation_with_original_html:
|
||||
if is_nested:
|
||||
self._replace_direct_content(element, m_item.translation_with_original_html)
|
||||
else:
|
||||
self._replace_content(element, m_item.translation_with_original_html)
|
||||
elif m_item.translation:
|
||||
# 降级：使用纯文本翻译（无格式）
|
||||
if is_nested:
|
||||
self._replace_direct_content(element, m_item.translation)
|
||||
else:
|
||||
self._replace_content(element, m_item.translation)
|
||||
|
||||
current_idx += 1
|
||||
|
||||
|
||||
new_item = epub.EpubHtml(title=original_item.title, file_name=original_item.get_name(), lang='zh-CN')
|
||||
new_item.set_content(str(soup).encode('utf-8'))
|
||||
return new_item
|
||||
except Exception as e:
|
||||
logger.error(f"创建中文文档失败 {original_item.get_name()}: {e}")
|
||||
return original_item
|
||||
|
||||
def _replace_content(self, element, translated_html: str):
|
||||
"""用译文替换元素的 inner_html"""
|
||||
try:
|
||||
# 将译文 HTML 字符串解析为 BeautifulSoup 对象
|
||||
new_soup = BeautifulSoup(translated_html, 'html.parser')
|
||||
# 清空原元素并填入新内容
|
||||
element.clear()
|
||||
# 重要：必须先转换为 list，否则 append 会修改 contents 导致跳过元素
|
||||
for content in list(new_soup.contents):
|
||||
element.append(content)
|
||||
except Exception as e:
|
||||
logger.error(f"替换内容失败: {e}")
|
||||
|
||||
def _replace_direct_content(self, element, translated_html: str):
|
||||
"""
|
||||
替换嵌套容器元素的直接文本内容（保留子块级元素）
|
||||
|
||||
策略：
|
||||
1. 保存所有子块级元素
|
||||
2. 清空元素内容
|
||||
3. 填入译文
|
||||
4. 在末尾追加保存的子块
|
||||
"""
|
||||
try:
|
||||
from bs4 import NavigableString
|
||||
block_tags = ['p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote', 'li', 'td']
|
||||
|
||||
# 1. 保存所有子块级元素
|
||||
saved_blocks = []
|
||||
for child in element.find_all(block_tags, recursive=False):
|
||||
# 只保存直接子元素中的块
|
||||
saved_blocks.append(child.extract())
|
||||
|
||||
# 2. 额外保存嵌套在 span 等内联元素中的块
|
||||
for inline in element.find_all(['span', 'a', 'em', 'strong'], recursive=True):
|
||||
for child in inline.find_all(block_tags, recursive=False):
|
||||
saved_blocks.append(child.extract())
|
||||
|
||||
# 3. 清空并填入译文
|
||||
new_soup = BeautifulSoup(translated_html, 'html.parser')
|
||||
element.clear()
|
||||
for content in list(new_soup.contents):
|
||||
element.append(content)
|
||||
|
||||
# 4. 追加保存的子块
|
||||
for block in saved_blocks:
|
||||
element.append(block)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"替换嵌套内容失败: {e}")
|
||||
# 降级到普通替换
|
||||
self._replace_content(element, translated_html)
|
||||
|
||||
|
||||
|
||||
def _generate_output_filename(self, output_path: str) -> str:
|
||||
from .utils import sanitize_filename
|
||||
title = self.original_book.get_metadata('DC', 'title')
|
||||
clean_title = sanitize_filename(title[0][0]) if title else "chinese_book"
|
||||
Path(output_path).mkdir(parents=True, exist_ok=True)
|
||||
return str(Path(output_path) / f"{clean_title}_chinese.epub")
|
||||
@@ -0,0 +1,71 @@
|
||||
import os
|
||||
import yaml
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from src.common.utils import setup_logger
|
||||
|
||||
logger = setup_logger("config_loader")
|
||||
|
||||
class ConfigLoader:
|
||||
def __init__(self, config_path: str = "config/config.yaml"):
|
||||
# Resolve absolute path relative to project root if needed,
|
||||
# but usually running from root so relative is fine.
|
||||
self.config_path = Path(config_path)
|
||||
self.config = self._load_defaults()
|
||||
|
||||
def _load_defaults(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"llm": {
|
||||
"model": "gpt-3.5-turbo",
|
||||
"base_url": "https://api.openai.com/v1",
|
||||
"timeout": 60,
|
||||
"requests_per_minute": 60,
|
||||
"concurrent_requests": 5
|
||||
},
|
||||
"translation": {
|
||||
"chunk_size": 4000
|
||||
}
|
||||
}
|
||||
|
||||
def load_config(self) -> Dict[str, Any]:
|
||||
# 1. Load YAML
|
||||
if self.config_path.exists():
|
||||
try:
|
||||
with open(self.config_path, 'r', encoding='utf-8') as f:
|
||||
file_config = yaml.safe_load(f)
|
||||
if file_config:
|
||||
self._deep_update(self.config, file_config)
|
||||
logger.info(f"Loaded config from {self.config_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load config file: {e}")
|
||||
else:
|
||||
logger.warning(f"Config file not found at {self.config_path}, using defaults")
|
||||
|
||||
# 2. Env overrides (Priority: Env > Config File > Defaults)
|
||||
load_dotenv()
|
||||
|
||||
# API Key is mandatory from Env (security best practice)
|
||||
api_key = os.getenv("OPENAI_API_KEY")
|
||||
if api_key:
|
||||
self.config["llm"]["api_key"] = api_key
|
||||
|
||||
# Base URL override
|
||||
base_url = os.getenv("OPENAI_BASE_URL")
|
||||
if base_url:
|
||||
self.config["llm"]["base_url"] = base_url
|
||||
|
||||
return self.config
|
||||
|
||||
def _deep_update(self, d, u):
|
||||
for k, v in u.items():
|
||||
if isinstance(v, dict):
|
||||
d[k] = self._deep_update(d.get(k, {}), v)
|
||||
else:
|
||||
d[k] = v
|
||||
return d
|
||||
|
||||
def load_global_config(path: str = "config/config.yaml") -> Dict[str, Any]:
|
||||
loader = ConfigLoader(path)
|
||||
return loader.load_config()
|
||||
@@ -0,0 +1,52 @@
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Dict, Any
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
class ManifestEntry(BaseModel):
|
||||
"""Represents a single translatable unit."""
|
||||
entry_id: str = Field(..., description="Global unique ID (e.g. file.html#paragraph_id)")
|
||||
file_path: str = Field(..., description="Internal path in EPUB")
|
||||
element_id: str = Field(..., description="HTML ID (e.g. uuid-1234)")
|
||||
original_text: str
|
||||
placeholders: Dict[str, str] = Field(default_factory=dict)
|
||||
translated_text: Optional[str] = None
|
||||
context: Optional[str] = None
|
||||
|
||||
class BookMetaData(BaseModel):
|
||||
title: str = "Unknown Title"
|
||||
author: str = "Unknown Author"
|
||||
language: str = "en"
|
||||
identifier: str = ""
|
||||
cover_image_id: Optional[str] = None
|
||||
|
||||
class ResourceItem(BaseModel):
|
||||
href: str
|
||||
media_type: str
|
||||
content: Optional[str] = None # For text/html
|
||||
file_path: Optional[str] = None # For binary/assets (relative to assets dir)
|
||||
properties: Optional[str] = None
|
||||
|
||||
class BookStructure(BaseModel):
|
||||
metadata: BookMetaData
|
||||
spine: List[str] = Field(default_factory=list, description="Ordered list of item IDs in spine")
|
||||
resources: Dict[str, ResourceItem] = Field(default_factory=dict, description="Map of item_id to ResourceItem")
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: Path) -> "BookStructure":
|
||||
"""Load BookStructure from JSON file."""
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
return cls.model_validate_json(f.read())
|
||||
|
||||
def save(self, path: Path):
|
||||
"""Save BookStructure to JSON file."""
|
||||
with open(path, 'w', encoding='utf-8') as f:
|
||||
f.write(self.model_dump_json(indent=2))
|
||||
|
||||
class BookProfile(BaseModel):
|
||||
"""Represents the profile of the book."""
|
||||
title: str
|
||||
author: str
|
||||
genre: str = "General"
|
||||
keywords: List[str] = Field(default_factory=list)
|
||||
style_guide: str = ""
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
class EpubTranslatorError(Exception):
|
||||
"""Base exception for Epub Translator."""
|
||||
pass
|
||||
|
||||
class CleaningError(EpubTranslatorError):
|
||||
pass
|
||||
|
||||
class ExtractionError(EpubTranslatorError):
|
||||
pass
|
||||
|
||||
class TranslationError(EpubTranslatorError):
|
||||
pass
|
||||
|
||||
class RestorationError(EpubTranslatorError):
|
||||
pass
|
||||
|
||||
class BuildError(EpubTranslatorError):
|
||||
pass
|
||||
@@ -0,0 +1,24 @@
|
||||
from pathlib import Path
|
||||
|
||||
def get_work_dirs(input_path: Path) -> dict:
|
||||
"""
|
||||
Get unified work directory paths for a specific book.
|
||||
|
||||
Structure:
|
||||
.work/
|
||||
├── {book_name}/
|
||||
│ ├── book_structure.json
|
||||
│ ├── manifest.json
|
||||
│ ├── assets/
|
||||
│ └── chunks/
|
||||
"""
|
||||
book_name = input_path.stem
|
||||
work_root = Path(".work") / book_name
|
||||
|
||||
return {
|
||||
"root": work_root,
|
||||
"structure": work_root / "book_structure.json",
|
||||
"manifest": work_root / "manifest.json",
|
||||
"assets": work_root / "assets",
|
||||
"chunks": work_root / "chunks",
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
def setup_logger(name: str, log_file: Path = None, level=logging.INFO):
|
||||
"""Sets up a logger with the given name."""
|
||||
logger = logging.getLogger(name)
|
||||
logger.setLevel(level)
|
||||
|
||||
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
|
||||
if log_file:
|
||||
file_handler = logging.FileHandler(log_file)
|
||||
file_handler.setFormatter(formatter)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
stream_handler = logging.StreamHandler()
|
||||
stream_handler.setFormatter(formatter)
|
||||
logger.addHandler(stream_handler)
|
||||
|
||||
return logger
|
||||
|
||||
def ensure_directory(path: Path):
|
||||
"""Ensures a directory exists."""
|
||||
if not path.exists():
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
@@ -1,175 +0,0 @@
|
||||
"""
|
||||
EPUB 解析器模块 (EPUB Parser Module)
|
||||
|
||||
该模块负责读取 EPUB 文件，提取元数据和内容项目。
|
||||
它使用 ebooklib 库来处理 EPUB 格式的底层细节。
|
||||
|
||||
Classes:
|
||||
EPUBParser: 负责 EPUB 文件的加载、元数据提取和内容项遍历。
|
||||
"""
|
||||
|
||||
import ebooklib
|
||||
from ebooklib import epub
|
||||
from bs4 import BeautifulSoup
|
||||
from typing import List, Dict, Any, Set, Optional
|
||||
from pathlib import Path
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class EPUBParser:
|
||||
"""
|
||||
EPUB 文件解析器。
|
||||
|
||||
负责加载 EPUB 文件，提取书籍元数据（如标题、作者），并提供方法来遍历和提取
|
||||
书中的文档内容（HTML/XHTML）。
|
||||
|
||||
Attributes:
|
||||
epub_path (Path): EPUB 文件的路径对象。
|
||||
book (epub.EpubBook): ebooklib 加载的书籍对象。
|
||||
metadata (Dict[str, str]): 提取的书籍元数据字典。
|
||||
"""
|
||||
|
||||
def __init__(self, epub_path: str):
|
||||
"""
|
||||
初始化 EPUB 解析器。
|
||||
|
||||
Args:
|
||||
epub_path (str): EPUB 文件的文件路径。
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: 如果指定的文件不存在。
|
||||
Exception: 如果 EPUB 文件加载失败（格式错误等）。
|
||||
"""
|
||||
self.epub_path = Path(epub_path)
|
||||
if not self.epub_path.exists():
|
||||
raise FileNotFoundError(f"EPUB 文件不存在: {epub_path}")
|
||||
|
||||
try:
|
||||
# ignore_ncx=True 是为了避免某些旧版 epub 的警告，但新版 ebooklib 可能行为不同
|
||||
# 这里直接读取，让 ebooklib 处理
|
||||
self.book = epub.read_epub(str(self.epub_path))
|
||||
logger.info(f"成功加载 EPUB: {self.epub_path.name}")
|
||||
except Exception as e:
|
||||
logger.error(f"加载 EPUB 失败: {e}")
|
||||
raise
|
||||
|
||||
self.metadata = self._extract_metadata()
|
||||
|
||||
def _extract_metadata(self) -> Dict[str, str]:
|
||||
"""
|
||||
从 EPUB 对象中提取标准元数据。
|
||||
|
||||
提取 Dublin Core (DC) 元数据，包括标题、作者和语言。
|
||||
|
||||
Returns:
|
||||
Dict[str, str]: 包含 'title', 'author', 'language' 的字典。
|
||||
如果提取失败，会使用默认值 ("Unknown", "en")。
|
||||
"""
|
||||
metadata = {}
|
||||
|
||||
try:
|
||||
# get_metadata 返回的是 (value, dict) 的列表，我们取第一个结果
|
||||
title_meta = self.book.get_metadata('DC', 'title')
|
||||
metadata['title'] = title_meta[0][0] if title_meta else "Unknown"
|
||||
|
||||
author_meta = self.book.get_metadata('DC', 'creator')
|
||||
metadata['author'] = author_meta[0][0] if author_meta else "Unknown"
|
||||
|
||||
lang_meta = self.book.get_metadata('DC', 'language')
|
||||
metadata['language'] = lang_meta[0][0] if lang_meta else "en"
|
||||
|
||||
logger.info(f"书籍: {metadata['title']} - {metadata['author']}")
|
||||
except Exception as e:
|
||||
logger.warning(f"提取元数据时出错: {e}")
|
||||
# 设置保底值
|
||||
metadata.setdefault('title', 'Unknown')
|
||||
metadata.setdefault('author', 'Unknown')
|
||||
metadata.setdefault('language', 'en')
|
||||
|
||||
return metadata
|
||||
|
||||
def get_toc(self):
|
||||
"""
|
||||
获取书籍的目录结构 (Table of Contents)
|
||||
|
||||
Returns:
|
||||
book.toc: ebooklib 的原始 TOC 结构
|
||||
"""
|
||||
return self.book.toc
|
||||
|
||||
def extract_all_content_items(self, include_files: Optional[Set[str]] = None) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
提取所有可翻译的内容项目（文档）。
|
||||
|
||||
遍历 EPUB 中的所有 Item，筛选出类型为 ITEM_DOCUMENT 的项目。
|
||||
同时会进行简单的过滤，跳过内容过短（<100字符）或看起来像非正文的文件（如 nav, toc, cover）。
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 内容项目列表。每个字典包含：
|
||||
- item (epub.EpubItem): 原始 Item 对象。
|
||||
- file_name (str): 文件名。
|
||||
- content (str): 解码后的 HTML 内容。
|
||||
- text_length (int): 纯文本长度（用于统计）。
|
||||
"""
|
||||
content_items = []
|
||||
|
||||
# 获取所有文档类型的项目
|
||||
for item in self.book.get_items():
|
||||
if item.get_type() == ebooklib.ITEM_DOCUMENT:
|
||||
# 如果指定了 include_files，则只处理其中的文件
|
||||
if include_files is not None:
|
||||
item_name = item.get_name()
|
||||
if item_name not in include_files:
|
||||
logger.debug(f"跳过未选中的文件: {item_name}")
|
||||
continue
|
||||
|
||||
try:
|
||||
# 获取内容 (bytes -> str)
|
||||
content = item.get_content().decode('utf-8')
|
||||
|
||||
# 简单的内容验证：提取纯文本检查长度
|
||||
soup = BeautifulSoup(content, 'html.parser')
|
||||
text = soup.get_text().strip()
|
||||
|
||||
# 跳过太短的内容（可能是只有图片的页面、空页面）
|
||||
if len(text) < 100:
|
||||
logger.debug(f"跳过短内容: {item.get_name()} ({len(text)} 字符)")
|
||||
continue
|
||||
|
||||
# 注意：文件名跳过逻辑已移至 TOCParser.get_skip_files()
|
||||
# 通过 include_files 参数在调用前过滤
|
||||
|
||||
content_items.append({
|
||||
'item': item,
|
||||
'file_name': item.get_name(),
|
||||
'content': content,
|
||||
'text_length': len(text)
|
||||
})
|
||||
|
||||
logger.debug(f"添加内容项: {item.get_name()} ({len(text)} 字符)")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"处理项目失败 {item.get_name()}: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"提取了 {len(content_items)} 个内容项目")
|
||||
return content_items
|
||||
|
||||
def get_book_info(self) -> Dict[str, str]:
|
||||
"""
|
||||
获取书籍的摘要信息。
|
||||
|
||||
Returns:
|
||||
Dict[str, str]: 包含文件名、标题、作者、语言和文档数量的字典。
|
||||
"""
|
||||
# 统计内容项
|
||||
document_count = sum(1 for item in self.book.get_items()
|
||||
if item.get_type() == ebooklib.ITEM_DOCUMENT)
|
||||
|
||||
return {
|
||||
'filename': self.epub_path.name,
|
||||
'title': self.metadata.get('title', 'Unknown'),
|
||||
'author': self.metadata.get('author', 'Unknown'),
|
||||
'language': self.metadata.get('language', 'en'),
|
||||
'document_count': document_count
|
||||
}
|
||||
@@ -1,249 +0,0 @@
|
||||
"""
|
||||
LLM Client Module - Generic OpenAI Compatible
|
||||
|
||||
Features:
|
||||
1. Fully configurable via config.json (base_url, headers).
|
||||
2. Mode-aware prompt building (bilingual vs chinese).
|
||||
3. Format repair capability for chinese mode.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from openai import AsyncOpenAI
|
||||
from typing import List, Dict, Optional, Any
|
||||
from loguru import logger
|
||||
import time
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||
from .manifest_manager import ManifestItem
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""Rate limiter for concurrency and RPM."""
|
||||
def __init__(self, requests_per_minute: int, concurrent_requests: int):
|
||||
self.semaphore = asyncio.Semaphore(concurrent_requests)
|
||||
self.min_interval = 60.0 / requests_per_minute if requests_per_minute > 0 else 0
|
||||
self.last_request_time = 0
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def acquire(self):
|
||||
await self.semaphore.acquire()
|
||||
async with self._lock:
|
||||
current_time = time.time()
|
||||
wait_time = self.min_interval - (current_time - self.last_request_time)
|
||||
if wait_time > 0:
|
||||
await asyncio.sleep(wait_time)
|
||||
self.last_request_time = time.time()
|
||||
|
||||
def release(self):
|
||||
self.semaphore.release()
|
||||
|
||||
|
||||
class LLMClient:
|
||||
"""Generic OpenAI-compatible API Client."""
|
||||
|
||||
def __init__(self, config: Dict):
|
||||
self.config = config
|
||||
llm_config = config["llm"]
|
||||
|
||||
api_key = llm_config.get("api_key")
|
||||
base_url = llm_config.get("base_url")
|
||||
extra_headers = llm_config.get("extra_headers", {})
|
||||
|
||||
if not api_key:
|
||||
raise ValueError("API Key is missing in config")
|
||||
|
||||
self.client = AsyncOpenAI(
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
default_headers=extra_headers
|
||||
)
|
||||
|
||||
self.models = llm_config.get("models", {"fast": "gpt-3.5-turbo", "smart": "gpt-4"})
|
||||
|
||||
self.rate_limiter = RateLimiter(
|
||||
llm_config["rate_limits"]["requests_per_minute"],
|
||||
llm_config["rate_limits"]["concurrent_requests"]
|
||||
)
|
||||
self.prompts = self._load_prompts()
|
||||
|
||||
def _load_prompts(self) -> Dict:
|
||||
try:
|
||||
with open("config/prompts.json", "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except:
|
||||
return {}
|
||||
|
||||
async def translate_chunk(self, items: List[ManifestItem], glossary: Dict = None,
|
||||
instruction: str = None, model_type: str = "fast",
|
||||
mode: str = "bilingual") -> Dict[str, str]:
|
||||
"""
|
||||
Translate a chunk of items.
|
||||
|
||||
Args:
|
||||
items: List of ManifestItem to translate
|
||||
glossary: Term dictionary
|
||||
instruction: Style guide
|
||||
model_type: "fast" or "smart"
|
||||
mode: "bilingual" or "chinese"
|
||||
"""
|
||||
if not items: return {}
|
||||
|
||||
model = self.models.get(model_type, self.models.get("fast"))
|
||||
prompt = self._build_prompt(items, mode)
|
||||
|
||||
try:
|
||||
# Build System Prompt
|
||||
base_sys_prompt = self.prompts.get("translation", {}).get("system", "You are a professional translator.")
|
||||
|
||||
# 中文模式：添加占位符保护指令
|
||||
if mode == "chinese":
|
||||
base_sys_prompt += """
|
||||
|
||||
Placeholder Instructions (CRITICAL):
|
||||
1. Text contains PAIRED placeholders: φNφ (start) and φ/Nφ (end), like HTML tags.
|
||||
2. Example: "φ1φTable Talkφ/1φ" means italic text, translate as "φ1φ桌谈φ/1φ"
|
||||
3. Single placeholders φNφ without φ/Nφ are inline elements (footnotes, formulas) - keep them in place.
|
||||
4. RULES:
|
||||
- DO NOT create new placeholder numbers that don't exist in the original
|
||||
- DO NOT remove or modify existing placeholders
|
||||
- Keep placeholders in the SAME relative position in your translation
|
||||
- If word order changes, keep placeholders with their associated text
|
||||
5. Each line starts with paragraph ID (p_xxxxx). Preserve them.
|
||||
"""
|
||||
|
||||
|
||||
if instruction:
|
||||
base_sys_prompt += f"\n\nBook Style Guide:\n{instruction}"
|
||||
|
||||
if glossary:
|
||||
glossary_text = "\n".join([f"{k} -> {v}" for k, v in glossary.items()])
|
||||
base_sys_prompt += f"\n\nTerminology:\n{glossary_text}"
|
||||
|
||||
# Strict formatting instructions
|
||||
base_sys_prompt += "\n\nRequirements:\n1. Each line MUST start with ID (p_xxxxx).\n2. DO NOT modify IDs.\n3. Return only translations."
|
||||
|
||||
raw_response = await self._make_request(model, base_sys_prompt, prompt)
|
||||
|
||||
if not raw_response:
|
||||
return {item.global_id: f"[Error - Empty Response]" for item in items}
|
||||
|
||||
return self._simple_parse(raw_response, items, mode)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Translation failed ({model}): {e}")
|
||||
return {item.global_id: f"[Error - {str(e)}]" for item in items}
|
||||
|
||||
async def repair_format(self, original_text: str, broken_translation: str) -> str:
|
||||
"""
|
||||
修复翻译格式：将占位符正确插入到译文中。
|
||||
"""
|
||||
model = self.models.get("fast")
|
||||
|
||||
system_prompt = "You are a format repair assistant. Your ONLY job is to insert placeholders into the translation."
|
||||
user_prompt = f"""
|
||||
Original Text (with placeholders):
|
||||
{original_text}
|
||||
|
||||
Translation (placeholders missing/incorrect):
|
||||
{broken_translation}
|
||||
|
||||
Task:
|
||||
Please rewrite the Translation to include ALL placeholders (φcXXXXXφ) from the Original Text.
|
||||
1. DO NOT translate again. Keep the meaning of the Translation.
|
||||
2. Place φcXXXXXφ tags exactly where they correspond to the original format (bold, italic, links).
|
||||
3. Output ONLY the fixed translation.
|
||||
"""
|
||||
try:
|
||||
return await self._make_request(model, system_prompt, user_prompt)
|
||||
except Exception as e:
|
||||
logger.error(f"Format repair failed: {e}")
|
||||
return broken_translation
|
||||
|
||||
async def raw_chat_completion(self, system_prompt: str, user_prompt: str, model_type: str = "smart") -> str:
|
||||
"""Generic chat completion (for Profiler)."""
|
||||
model = self.models.get(model_type, self.models.get("smart"))
|
||||
return await self._make_request(model, system_prompt, user_prompt)
|
||||
|
||||
def _build_prompt(self, items: List[ManifestItem], mode: str = "bilingual") -> str:
|
||||
"""构建翻译提示词"""
|
||||
lines = []
|
||||
for item in items:
|
||||
if mode == "chinese":
|
||||
# 中文模式：使用带占位符的文本和段落类型
|
||||
text = item.text_with_placeholders if item.text_with_placeholders else item.clean_text
|
||||
p_type = getattr(item, 'paragraph_type', 'body').upper()
|
||||
lines.append(f"{item.global_id} [{p_type}] {text}")
|
||||
else:
|
||||
# 双语模式：使用纯文本
|
||||
lines.append(f"{item.global_id} {item.clean_text}")
|
||||
return "\n".join(lines)
|
||||
|
||||
def _simple_parse(self, response: str, items: List[ManifestItem], mode: str = "bilingual") -> Dict[str, str]:
|
||||
"""解析 LLM 响应"""
|
||||
results = {}
|
||||
for i, item in enumerate(items):
|
||||
current_id = item.global_id
|
||||
start_idx = response.find(current_id)
|
||||
if start_idx == -1: continue
|
||||
|
||||
end_idx = len(response)
|
||||
if i + 1 < len(items):
|
||||
next_id = items[i+1].global_id
|
||||
next_found = response.find(next_id, start_idx + len(current_id))
|
||||
if next_found != -1:
|
||||
end_idx = next_found
|
||||
|
||||
content = response[start_idx:end_idx].strip()
|
||||
clean_content = content[len(current_id):].strip()
|
||||
clean_content = clean_content.lstrip(":： \t")
|
||||
|
||||
# 移除类型标记 (如 [BODY])
|
||||
if mode == "chinese":
|
||||
clean_content = re.sub(r'^\[[A-Z]+\]\s*', '', clean_content)
|
||||
|
||||
if clean_content:
|
||||
results[current_id] = clean_content
|
||||
|
||||
# Fallback: 逐行解析
|
||||
if len(results) < len(items):
|
||||
for line in response.split("\n"):
|
||||
line = line.strip()
|
||||
for item in items:
|
||||
if item.global_id not in results and line.startswith(item.global_id):
|
||||
res = line[len(item.global_id):].strip().lstrip(":： ")
|
||||
if mode == "chinese":
|
||||
res = re.sub(r'^\[[A-Z]+\]\s*', '', res)
|
||||
if res: results[item.global_id] = res
|
||||
|
||||
# 验证解析结果
|
||||
parsed_count = len(results)
|
||||
expected_count = len(items)
|
||||
if parsed_count < expected_count:
|
||||
missing_ids = [item.global_id for item in items if item.global_id not in results]
|
||||
logger.warning(f"LLM 响应解析不完整: {parsed_count}/{expected_count} (缺失: {missing_ids[:3]}...)")
|
||||
|
||||
return results
|
||||
|
||||
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))
|
||||
async def _make_request(self, model: str, system_prompt: str, user_prompt: str) -> str:
|
||||
await self.rate_limiter.acquire()
|
||||
try:
|
||||
resp = await self.client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt}
|
||||
],
|
||||
temperature=self.config['translation'].get('temperature', 0.2),
|
||||
max_tokens=8000
|
||||
)
|
||||
return resp.choices[0].message.content.strip()
|
||||
finally:
|
||||
self.rate_limiter.release()
|
||||
|
||||
async def close(self):
|
||||
await self.client.close()
|
||||
|
||||
# Alias for backward compatibility
|
||||
OpenRouterClient = LLMClient
|
||||
@@ -1,180 +0,0 @@
|
||||
"""
|
||||
Manifest 管理器模块 (Manifest Manager Module)
|
||||
|
||||
该模块是系统的单一真理源 (SSOT)。
|
||||
它记录了每一段文本的原始状态、清洗后的文本、哈希值以及翻译状态。
|
||||
所有对翻译流程的操作（提取、翻译、回填）都必须通过修改此 Manifest 进行。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import hashlib
|
||||
from typing import List, Dict, Optional, Any
|
||||
from pathlib import Path
|
||||
from loguru import logger
|
||||
from dataclasses import dataclass, asdict, field
|
||||
|
||||
@dataclass
|
||||
class ManifestItem:
|
||||
"""代表一个翻译单元（通常是一个段落）"""
|
||||
global_id: str
|
||||
source_file: str
|
||||
original_html: str
|
||||
clean_text: str
|
||||
text_hash: str
|
||||
tag: str
|
||||
tag_attrs: Dict[str, Any] = field(default_factory=dict) # 外层标签的属性 (class, style...)
|
||||
translation: Optional[str] = None
|
||||
status: str = "pending" # pending, translated, ignored, failed
|
||||
error_msg: Optional[str] = None
|
||||
model_used: Optional[str] = None # 记录使用的模型
|
||||
quality_score: Optional[int] = None # 记录质量评分
|
||||
|
||||
# === 中文模式专用字段 ===
|
||||
text_with_placeholders: str = "" # 带占位符的文本
|
||||
placeholder_map: Dict[str, str] = field(default_factory=dict) # 占位符映射表 {id: html_string}
|
||||
paragraph_type: str = "body" # 段落类型：chapter/section/subsection/epigraph/body
|
||||
translation_with_placeholders: str = "" # 带占位符的译文
|
||||
translation_with_original_html: str = "" # 还原后的最终 HTML (中文模式)
|
||||
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self):
|
||||
return asdict(self)
|
||||
|
||||
class ManifestManager:
|
||||
"""
|
||||
负责 Manifest 的生命周期管理。
|
||||
"""
|
||||
|
||||
def __init__(self, manifest_path: str):
|
||||
self.manifest_path = Path(manifest_path)
|
||||
self.data: Dict[str, Any] = {
|
||||
"book_id": "",
|
||||
"metadata": {},
|
||||
"chapter_range": {
|
||||
"start_title": None,
|
||||
"end_title": None,
|
||||
"included_files": []
|
||||
},
|
||||
"items": []
|
||||
}
|
||||
self._items_by_id: Dict[str, ManifestItem] = {}
|
||||
|
||||
def load(self) -> bool:
|
||||
"""从文件加载 Manifest。如果文件不存在则返回 False。"""
|
||||
if self.manifest_path.exists():
|
||||
try:
|
||||
with open(self.manifest_path, 'r', encoding='utf-8') as f:
|
||||
self.data = json.load(f)
|
||||
|
||||
# 重建对象映射
|
||||
self._items_by_id = {
|
||||
item['global_id']: ManifestItem(**item)
|
||||
for item in self.data["items"]
|
||||
}
|
||||
logger.info(f"成功从 {self.manifest_path} 加载 Manifest, 包含 {len(self._items_by_id)} 个项目")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"加载 Manifest 失败: {e}")
|
||||
return False
|
||||
return False
|
||||
|
||||
def save(self):
|
||||
"""将当前状态保存到 Manifest 文件。"""
|
||||
# 确保目录存在
|
||||
self.manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 同步 items 到 data 字典
|
||||
self.data["items"] = [item.to_dict() for item in self._items_by_id.values()]
|
||||
|
||||
with open(self.manifest_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(self.data, f, ensure_ascii=False, indent=2)
|
||||
# logger.debug(f"Manifest 已保存到 {self.manifest_path}")
|
||||
|
||||
def init_manifest(self, book_id: str, metadata: Dict, chapter_range: Dict = None):
|
||||
"""初始化一个新的 Manifest。"""
|
||||
self.data = {
|
||||
"book_id": book_id,
|
||||
"metadata": metadata,
|
||||
"chapter_range": chapter_range or {
|
||||
"start_title": None,
|
||||
"end_title": None,
|
||||
"included_files": []
|
||||
},
|
||||
"items": []
|
||||
}
|
||||
self._items_by_id = {}
|
||||
self.save()
|
||||
|
||||
def get_chapter_range(self) -> Dict:
|
||||
"""获取记录的章节范围"""
|
||||
return self.data.get("chapter_range", {})
|
||||
|
||||
def add_item(self, source_file: str, original_html: str, clean_text: str, tag: str, metadata: Dict = None) -> ManifestItem:
|
||||
"""添加一个新的翻译项并分配 ID。"""
|
||||
# 生成全局 ID
|
||||
new_index = len(self._items_by_id) + 1
|
||||
global_id = f"p_{new_index:05d}"
|
||||
|
||||
# 生成内容哈希 (用于排重和缓存)
|
||||
text_hash = hashlib.sha256(clean_text.encode('utf-8')).hexdigest()
|
||||
|
||||
item = ManifestItem(
|
||||
global_id=global_id,
|
||||
source_file=source_file,
|
||||
original_html=original_html,
|
||||
clean_text=clean_text,
|
||||
text_hash=text_hash,
|
||||
tag=tag,
|
||||
metadata=metadata or {}
|
||||
)
|
||||
|
||||
self._items_by_id[global_id] = item
|
||||
return item
|
||||
|
||||
def get_items(self, status: str = None, file_name: str = None) -> List[ManifestItem]:
|
||||
"""按状态或文件名查询项目。"""
|
||||
items = list(self._items_by_id.values())
|
||||
if status:
|
||||
items = [i for i in items if i.status == status]
|
||||
if file_name:
|
||||
items = [i for i in items if i.source_file == file_name]
|
||||
|
||||
# 必须按 ID 顺序返回以保证分块正确
|
||||
return sorted(items, key=lambda x: x.global_id)
|
||||
|
||||
def update_item(self, global_id: str, translation: str, status: str = "translated", error: str = None, model: str = None, score: int = None):
|
||||
"""更新翻译结果。"""
|
||||
if global_id in self._items_by_id:
|
||||
item = self._items_by_id[global_id]
|
||||
if translation is not None:
|
||||
item.translation = translation
|
||||
item.status = status
|
||||
if error:
|
||||
item.error_msg = error
|
||||
if model:
|
||||
item.model_used = model
|
||||
if score is not None:
|
||||
item.quality_score = score
|
||||
else:
|
||||
logger.warning(f"尝试更新不存在的 ID: {global_id}")
|
||||
|
||||
@property
|
||||
def stats(self) -> Dict:
|
||||
"""获取翻译进度统计。"""
|
||||
total = len(self._items_by_id)
|
||||
if total == 0: return {"progress": "0%"}
|
||||
|
||||
translated = sum(1 for i in self._items_by_id.values() if i.status == "translated")
|
||||
ignored = sum(1 for i in self._items_by_id.values() if i.status == "ignored")
|
||||
failed = sum(1 for i in self._items_by_id.values() if i.status == "failed")
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"translated": translated,
|
||||
"ignored": ignored,
|
||||
"failed": failed,
|
||||
"pending": total - translated - ignored - failed,
|
||||
"progress_percent": round((translated + ignored) / total * 100, 1)
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import shutil
|
||||
import uuid
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Set
|
||||
|
||||
import ebooklib
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
from ebooklib import epub
|
||||
|
||||
from src.common.data_model import BookStructure, BookMetaData, ResourceItem
|
||||
from src.common.exceptions import CleaningError
|
||||
from src.common.utils import setup_logger, ensure_directory
|
||||
|
||||
logger = setup_logger("epub_cleaner")
|
||||
|
||||
class EpubCleaner:
|
||||
def __init__(self, input_path: Path, work_dir: Path):
|
||||
self.input_path = input_path
|
||||
self.work_dir = work_dir
|
||||
self.assets_dir = work_dir / "assets"
|
||||
self.json_path = work_dir / "book_structure.json"
|
||||
|
||||
def clean(self) -> Path:
|
||||
"""
|
||||
Cleans the input EPUB and generates book_structure.json.
|
||||
Returns the path to the JSON file.
|
||||
"""
|
||||
logger.info(f"Starting cleanup for {self.input_path}")
|
||||
ensure_directory(self.work_dir)
|
||||
ensure_directory(self.assets_dir)
|
||||
|
||||
try:
|
||||
book = epub.read_epub(self.input_path)
|
||||
|
||||
# 1. Extract Metadata
|
||||
metadata = self._extract_metadata(book)
|
||||
|
||||
# 2. Process Resources
|
||||
resources = {}
|
||||
# Use zipfile for binary extraction to avoid ebooklib's memory overhead/decoding issues
|
||||
with zipfile.ZipFile(self.input_path, 'r') as zf:
|
||||
# Map ebooklib items to zip entries isn't straightforward directly via name
|
||||
# So we iterate ebooklib items and assume standard structure or handle content bytes
|
||||
|
||||
for item in book.get_items():
|
||||
item_id = item.get_id()
|
||||
file_name = item.get_name()
|
||||
media_type = item.get_type() # ebooklib constant
|
||||
|
||||
if media_type == ebooklib.ITEM_DOCUMENT:
|
||||
# Clean HTML
|
||||
content_str = item.get_content().decode('utf-8')
|
||||
cleaned_content = self._clean_html(content_str, file_name)
|
||||
|
||||
resources[item_id] = ResourceItem(
|
||||
href=file_name,
|
||||
media_type="application/xhtml+xml",
|
||||
content=cleaned_content
|
||||
)
|
||||
elif media_type in (ebooklib.ITEM_IMAGE, ebooklib.ITEM_STYLE, ebooklib.ITEM_FONT, ebooklib.ITEM_COVER):
|
||||
# Save asset
|
||||
|
||||
# Check if it is a cover
|
||||
if media_type == ebooklib.ITEM_COVER:
|
||||
metadata.cover_image_id = item_id
|
||||
logger.info(f"Found cover image: {item_id} ({file_name})")
|
||||
|
||||
# Preserve directory structure to avoid collisions
|
||||
asset_path = self.assets_dir / file_name
|
||||
ensure_directory(asset_path.parent)
|
||||
|
||||
# Ebooklib might change filenames, safer to use item.get_content()
|
||||
with open(asset_path, "wb") as f:
|
||||
f.write(item.get_content())
|
||||
|
||||
resources[item_id] = ResourceItem(
|
||||
href=file_name,
|
||||
media_type=self._get_media_type_str(item),
|
||||
file_path=str(asset_path.relative_to(self.work_dir))
|
||||
)
|
||||
elif media_type == ebooklib.ITEM_NAVIGATION:
|
||||
# NCX or NAV document - preserve for TOC
|
||||
content_bytes = item.get_content()
|
||||
asset_path = self.assets_dir / file_name
|
||||
ensure_directory(asset_path.parent)
|
||||
|
||||
with open(asset_path, "wb") as f:
|
||||
f.write(content_bytes)
|
||||
|
||||
# Determine media type
|
||||
if file_name.endswith('.ncx'):
|
||||
mt = "application/x-dtbncx+xml"
|
||||
else:
|
||||
mt = "application/xhtml+xml"
|
||||
|
||||
resources[item_id] = ResourceItem(
|
||||
href=file_name,
|
||||
media_type=mt,
|
||||
file_path=str(asset_path.relative_to(self.work_dir))
|
||||
)
|
||||
logger.debug(f"Preserved navigation: {file_name}")
|
||||
else:
|
||||
# Skip other items (scripts, etc.)
|
||||
pass
|
||||
|
||||
# 3. Extract Spine
|
||||
spine_ids = [item[0] for item in book.spine]
|
||||
|
||||
# 4. Construct Structure
|
||||
structure = BookStructure(
|
||||
metadata=metadata,
|
||||
spine=spine_ids,
|
||||
resources=resources
|
||||
)
|
||||
|
||||
# 5. Serialize
|
||||
with open(self.json_path, "w", encoding="utf-8") as f:
|
||||
f.write(structure.model_dump_json(indent=2))
|
||||
|
||||
logger.info(f"Cleanup finished. Structure saved to {self.json_path}")
|
||||
return self.json_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Cleaning failed: {e}")
|
||||
raise CleaningError(f"Failed to clean EPUB: {e}") from e
|
||||
|
||||
def _extract_metadata(self, book: epub.EpubBook) -> BookMetaData:
|
||||
title = book.get_metadata('DC', 'title')[0][0] if book.get_metadata('DC', 'title') else "Unknown"
|
||||
author = book.get_metadata('DC', 'creator')[0][0] if book.get_metadata('DC', 'creator') else "Unknown"
|
||||
lang = book.get_metadata('DC', 'language')[0][0] if book.get_metadata('DC', 'language') else "en"
|
||||
ident = book.get_metadata('DC', 'identifier')[0][0] if book.get_metadata('DC', 'identifier') else ""
|
||||
|
||||
return BookMetaData(
|
||||
title=str(title),
|
||||
author=str(author),
|
||||
language=str(lang),
|
||||
identifier=str(ident)
|
||||
)
|
||||
|
||||
def _get_media_type_str(self, item) -> str:
|
||||
# Helper to map ebooklib type to mime string if needed
|
||||
# ebooklib doesn't expose easy MIME string for all types directly on item object sometimes
|
||||
if hasattr(item, 'media_type'):
|
||||
return item.media_type
|
||||
return "application/octet-stream"
|
||||
|
||||
def _clean_html(self, content: str, filename: str) -> str:
|
||||
soup = BeautifulSoup(content, 'html.parser')
|
||||
|
||||
# 1. Provide IDs for structural/translatable elements
|
||||
self._ensure_element_ids(soup)
|
||||
|
||||
# 2. Flatten divs (Disable to prevent style loss)
|
||||
# self._flatten_divs(soup)
|
||||
|
||||
return str(soup)
|
||||
|
||||
def _ensure_element_ids(self, soup: BeautifulSoup):
|
||||
"""
|
||||
Injects UUIDs into p, h1-h6, li tags if they don't have an ID.
|
||||
This provides the anchor for translation backfilling.
|
||||
"""
|
||||
targets = soup.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'])
|
||||
for tag in targets:
|
||||
if not tag.has_attr('id'):
|
||||
tag['id'] = f"uuid-{uuid.uuid4()}"
|
||||
|
||||
def _flatten_divs(self, soup: BeautifulSoup):
|
||||
"""
|
||||
Converts generic divs containing only inline text/styles to p tags.
|
||||
Recursive strategies can be complex, sticking to simple heuristic from archive.
|
||||
"""
|
||||
inline_tags = {'span', 'em', 'strong', 'i', 'b', 'u', 'a', 'br', 'sub', 'sup', 'small'}
|
||||
|
||||
for div in list(soup.find_all('div')):
|
||||
# If div has no block children, convert to p
|
||||
has_block = any(
|
||||
isinstance(c, Tag) and c.name not in inline_tags
|
||||
for c in div.children
|
||||
)
|
||||
|
||||
if not has_block:
|
||||
div.name = 'p'
|
||||
@@ -1,19 +1,10 @@
|
||||
"""
|
||||
格式提取模块 (优化版 v3)
|
||||
|
||||
核心优化：
|
||||
1. 前缀/后缀标签分离：文本前后的标签不发送给 LLM，直接回填
|
||||
2. 公式检测：将数学变量/公式作为整体占位符
|
||||
3. 简化占位符：φ1φ 格式，每个段落独立编号
|
||||
"""
|
||||
|
||||
import re
|
||||
from bs4 import BeautifulSoup, Tag, NavigableString
|
||||
from typing import Tuple, Dict, Any, List, Optional
|
||||
|
||||
import html
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
from typing import Tuple, Dict, List
|
||||
|
||||
class HeadingDetector:
|
||||
"""标题与段落类型检测器"""
|
||||
"""Detects heading types and paragraph roles."""
|
||||
|
||||
CHAPTER_PATTERNS = [
|
||||
r'^(chapter|chap\.?|part)\s+([0-9]+|[ivxlc]+|[a-z])',
|
||||
@@ -76,12 +67,8 @@ class HeadingDetector:
|
||||
|
||||
class FormatExtractor:
|
||||
"""
|
||||
HTML 格式提取器 (优化版 v3)
|
||||
|
||||
核心改进：
|
||||
1. 前缀/后缀标签分离 - 不发送给 LLM，自动回填
|
||||
2. 公式元素整体替换
|
||||
3. 简化占位符格式 φ1φ, φ2φ
|
||||
HTML Format Extractor (Ported from v0.09 v3)
|
||||
Handles inline styles, formulas, and drop caps.
|
||||
"""
|
||||
|
||||
FORMULA_CHARS = re.compile(
|
||||
@@ -93,99 +80,78 @@ class FormatExtractor:
|
||||
def __init__(self):
|
||||
self.detector = HeadingDetector()
|
||||
|
||||
def extract(self, element_html: str) -> Tuple[str, str, Dict[str, str], str]:
|
||||
def extract(self, element_html: str) -> Tuple[str, str, Dict[str, str], str, List[str]]:
|
||||
"""
|
||||
提取格式信息
|
||||
Extracts format information.
|
||||
|
||||
Returns:
|
||||
clean_text: 纯文本
|
||||
text_with_placeholders: 只包含内嵌占位符的文本（不含前缀/后缀标签）
|
||||
placeholder_map: 占位符映射，包含特殊键 "_prefix" 和 "_suffix"
|
||||
paragraph_type: 段落类型
|
||||
clean_text: Pure text
|
||||
text_with_placeholders: Text with inline placeholders
|
||||
placeholder_map: Map of placeholders
|
||||
paragraph_type: Detected type
|
||||
endnote_anchors: List of detected endnote IDs
|
||||
"""
|
||||
soup = BeautifulSoup(element_html, 'html.parser')
|
||||
root = list(soup.children)[0] if list(soup.children) else soup
|
||||
|
||||
# 获取纯文本和段落类型
|
||||
clean_text = root.get_text().strip()
|
||||
clean_text = re.sub(r'\s+', ' ', clean_text)
|
||||
p_type = self.detector.detect(root, clean_text) if isinstance(root, Tag) else "body"
|
||||
|
||||
# 获取内部 HTML
|
||||
inner_html = root.decode_contents() if isinstance(root, Tag) else str(root)
|
||||
|
||||
# 智能提取（分离前缀/后缀）
|
||||
text_with_ph, local_map = self._smart_extract_v3(inner_html)
|
||||
|
||||
# === 验证完整性 ===
|
||||
# 将 text_with_placeholders 去掉占位符后与 clean_text 比较
|
||||
if text_with_ph:
|
||||
text_with_ph = re.sub(r'\s+', ' ', text_with_ph).strip()
|
||||
|
||||
# Verify integrity
|
||||
stripped_text = self._strip_placeholders(text_with_ph)
|
||||
stripped_text = re.sub(r'\s+', ' ', stripped_text).strip()
|
||||
|
||||
if not self._verify_content_integrity(clean_text, stripped_text):
|
||||
# 内容不完整，降级到简单模式
|
||||
from loguru import logger
|
||||
logger.warning(f"内容验证失败，降级处理: '{clean_text[:30]}...'")
|
||||
# 降级：不使用前缀/后缀分离，只做简单占位符处理
|
||||
# Fallback
|
||||
text_with_ph, local_map = self._fallback_extract(inner_html, clean_text)
|
||||
|
||||
return clean_text, text_with_ph, local_map, p_type
|
||||
endnote_anchors = []
|
||||
for pid, html in local_map.items():
|
||||
if pid.startswith("_"):
|
||||
continue
|
||||
if re.match(r'<(span|a)\s+id="[a-zA-Z][a-zA-Z0-9]{2,5}"\s*>\s*</\1>', html):
|
||||
endnote_anchors.append(pid)
|
||||
|
||||
return clean_text, text_with_ph, local_map, p_type, endnote_anchors
|
||||
|
||||
def _strip_placeholders(self, text: str) -> str:
|
||||
"""移除所有占位符（φXφ 和 φ/Xφ 格式）"""
|
||||
return re.sub(r'φ/?[0-9]+φ', '', text)
|
||||
|
||||
def _verify_content_integrity(self, clean_text: str, stripped_text: str) -> bool:
|
||||
"""
|
||||
验证内容完整性：比较 clean_text 和 stripped_text
|
||||
|
||||
允许一定的容差（空格差异、标点差异）
|
||||
"""
|
||||
# 标准化：移除空格和常见标点进行比较
|
||||
def normalize(s):
|
||||
s = re.sub(r'\s+', '', s) # 移除空格
|
||||
s = s.lower() # 忽略大小写
|
||||
# Unescape HTML entities first (e.g. & -> &)
|
||||
s = html.unescape(s)
|
||||
s = re.sub(r'\s+', '', s)
|
||||
s = s.lower()
|
||||
return s
|
||||
|
||||
norm_clean = normalize(clean_text)
|
||||
norm_stripped = normalize(stripped_text)
|
||||
|
||||
# 完全匹配
|
||||
if norm_clean == norm_stripped:
|
||||
return True
|
||||
|
||||
# 检查是否只是缺少少量字符（<5%）
|
||||
if len(norm_stripped) > 0:
|
||||
coverage = len(norm_stripped) / len(norm_clean) if norm_clean else 0
|
||||
if coverage >= 0.95:
|
||||
return True
|
||||
|
||||
|
||||
return False
|
||||
|
||||
def _fallback_extract(self, inner_html: str, clean_text: str) -> Tuple[str, Dict[str, str]]:
|
||||
"""
|
||||
降级提取：不分离前缀/后缀，直接返回纯文本
|
||||
"""
|
||||
return clean_text, {"_prefix": "", "_suffix": ""}
|
||||
|
||||
def _smart_extract_v3(self, inner_html: str) -> Tuple[str, Dict[str, str]]:
|
||||
"""
|
||||
智能提取 v3：分离前缀/后缀 + 合并内嵌公式块
|
||||
|
||||
核心逻辑：
|
||||
1. 分离前缀（第一个可翻译文本之前）和后缀（最后一个可翻译文本之后）
|
||||
2. 中间部分：检测"公式块"（连续标签+不可翻译文本），合并为单个占位符
|
||||
3. 只有真正需要翻译的格式标签（如斜体包裹的长文本）才拆分
|
||||
"""
|
||||
# 使用正则分割标签和文本
|
||||
parts = re.split(r'(<[^>]+>)', inner_html)
|
||||
parts = [p for p in parts if p]
|
||||
|
||||
if not parts:
|
||||
return "", {"_prefix": "", "_suffix": ""}
|
||||
|
||||
# 识别每个部分的类型
|
||||
part_types = [] # 'tag', 'translatable', 'formula', 'whitespace'
|
||||
part_types = []
|
||||
for part in parts:
|
||||
if part.startswith('<'):
|
||||
part_types.append('tag')
|
||||
@@ -196,7 +162,6 @@ class FormatExtractor:
|
||||
else:
|
||||
part_types.append('formula')
|
||||
|
||||
# 找到第一个和最后一个可翻译文本的索引
|
||||
first_trans_idx = None
|
||||
last_trans_idx = None
|
||||
for i, t in enumerate(part_types):
|
||||
@@ -206,46 +171,61 @@ class FormatExtractor:
|
||||
last_trans_idx = i
|
||||
|
||||
if first_trans_idx is None:
|
||||
# 没有可翻译文本，全部作为前缀
|
||||
return "", {"_prefix": inner_html, "_suffix": ""}
|
||||
|
||||
# 分割
|
||||
prefix_parts = parts[:first_trans_idx]
|
||||
middle_parts = parts[first_trans_idx:last_trans_idx + 1]
|
||||
middle_types = part_types[first_trans_idx:last_trans_idx + 1]
|
||||
suffix_parts = parts[last_trans_idx + 1:]
|
||||
# Prefix Separation
|
||||
safe_prefix_end = 0
|
||||
for i in range(first_trans_idx):
|
||||
if part_types[i] == 'tag':
|
||||
tag = parts[i]
|
||||
is_self_closing = tag.endswith('/>') or re.match(r'<(br|hr|img|meta|link)\b', tag, re.I)
|
||||
is_closing = tag.startswith('</')
|
||||
if is_self_closing or is_closing:
|
||||
safe_prefix_end = i + 1
|
||||
else:
|
||||
break
|
||||
elif part_types[i] == 'whitespace':
|
||||
safe_prefix_end = i + 1
|
||||
else:
|
||||
break
|
||||
|
||||
# === Drop Cap 检测 ===
|
||||
# 英文书籍常用首字母放大样式，如 <span class="dropcap">T</span>his
|
||||
# 检测：前缀末尾是格式化的单个字母，且中间部分第一个文本以小写字母开头
|
||||
# 处理：移除格式标签，将纯字母加入中间部分
|
||||
# Suffix Separation
|
||||
safe_suffix_start = len(parts)
|
||||
for i in range(len(parts) - 1, last_trans_idx, -1):
|
||||
if part_types[i] == 'tag':
|
||||
tag = parts[i]
|
||||
is_closing = tag.startswith('</')
|
||||
is_self_closing = tag.endswith('/>') or re.match(r'<(br|hr|img|meta|link)\b', tag, re.I)
|
||||
if is_closing or is_self_closing:
|
||||
safe_suffix_start = i
|
||||
else:
|
||||
break
|
||||
elif part_types[i] == 'whitespace':
|
||||
safe_suffix_start = i
|
||||
else:
|
||||
break
|
||||
|
||||
prefix_parts = parts[:safe_prefix_end]
|
||||
middle_parts = parts[safe_prefix_end:safe_suffix_start]
|
||||
middle_types = part_types[safe_prefix_end:safe_suffix_start]
|
||||
suffix_parts = parts[safe_suffix_start:]
|
||||
|
||||
# Drop Cap Check
|
||||
if prefix_parts and middle_parts:
|
||||
prefix_parts, middle_parts, middle_types = self._handle_drop_cap(
|
||||
prefix_parts, middle_parts, middle_types
|
||||
)
|
||||
|
||||
# 构建映射
|
||||
local_map = {}
|
||||
if prefix_parts:
|
||||
local_map["_prefix"] = "".join(prefix_parts)
|
||||
if suffix_parts:
|
||||
local_map["_suffix"] = "".join(suffix_parts)
|
||||
|
||||
# 前缀
|
||||
prefix_html = "".join(prefix_parts)
|
||||
if prefix_html:
|
||||
local_map["_prefix"] = prefix_html
|
||||
|
||||
# 后缀
|
||||
suffix_html = "".join(suffix_parts)
|
||||
if suffix_html:
|
||||
local_map["_suffix"] = suffix_html
|
||||
|
||||
# 中间部分处理：使用配对占位符格式
|
||||
#
|
||||
# 策略：
|
||||
# 1. 连续的 (tag|formula|whitespace) 不包含可翻译文本 → 合并为单个占位符 φ1φ
|
||||
# 2. 开始标签后接可翻译文本 → 配对格式 φ2φ文本φ/2φ
|
||||
#
|
||||
# Middle processing
|
||||
placeholder_counter = 1
|
||||
result_parts = []
|
||||
tag_stack = [] # 追踪开放标签 [(id, opening_tag), ...]
|
||||
tag_stack = []
|
||||
|
||||
i = 0
|
||||
while i < len(middle_parts):
|
||||
@@ -253,41 +233,32 @@ class FormatExtractor:
|
||||
ptype = middle_types[i]
|
||||
|
||||
if ptype == 'translatable':
|
||||
# 可翻译文本，直接保留
|
||||
result_parts.append(part)
|
||||
i += 1
|
||||
|
||||
elif ptype == 'tag':
|
||||
# 检查是开始标签还是结束标签
|
||||
is_closing = part.startswith('</')
|
||||
|
||||
if is_closing:
|
||||
# 结束标签
|
||||
if tag_stack:
|
||||
# 匹配最近的开放标签
|
||||
open_id, open_tag = tag_stack.pop()
|
||||
local_map[f"/{open_id}"] = part
|
||||
result_parts.append(f"φ/{open_id}φ")
|
||||
else:
|
||||
# 没有匹配的开放标签，作为单独占位符
|
||||
pid = str(placeholder_counter)
|
||||
placeholder_counter += 1
|
||||
local_map[pid] = part
|
||||
result_parts.append(f"φ{pid}φ")
|
||||
i += 1
|
||||
else:
|
||||
# 开始标签，检查后面是否有可翻译文本
|
||||
has_translatable_after = False
|
||||
for j in range(i + 1, len(middle_parts)):
|
||||
if middle_types[j] == 'translatable':
|
||||
has_translatable_after = True
|
||||
break
|
||||
elif middle_types[j] == 'tag' and middle_parts[j].startswith('</'):
|
||||
# 遇到结束标签但还没遇到可翻译文本
|
||||
break
|
||||
|
||||
if has_translatable_after:
|
||||
# 开始配对模式
|
||||
pid = str(placeholder_counter)
|
||||
placeholder_counter += 1
|
||||
local_map[pid] = part
|
||||
@@ -295,7 +266,6 @@ class FormatExtractor:
|
||||
tag_stack.append((pid, part))
|
||||
i += 1
|
||||
else:
|
||||
# 后面没有可翻译文本，合并为公式块
|
||||
block_parts = []
|
||||
while i < len(middle_parts) and middle_types[i] != 'translatable':
|
||||
block_parts.append(middle_parts[i])
|
||||
@@ -306,57 +276,46 @@ class FormatExtractor:
|
||||
placeholder_counter += 1
|
||||
local_map[pid] = block_html
|
||||
result_parts.append(f"φ{pid}φ")
|
||||
|
||||
elif ptype in ('formula', 'whitespace'):
|
||||
# 公式或空白，检查是否是连续块的开始
|
||||
block_parts = []
|
||||
while i < len(middle_parts) and middle_types[i] in ('formula', 'whitespace'):
|
||||
block_parts.append(middle_parts[i])
|
||||
i += 1
|
||||
if block_parts:
|
||||
block_html = "".join(block_parts)
|
||||
pid = str(placeholder_counter)
|
||||
placeholder_counter += 1
|
||||
local_map[pid] = block_html
|
||||
result_parts.append(f"φ{pid}φ")
|
||||
else:
|
||||
result_parts.append(part)
|
||||
i += 1
|
||||
|
||||
text_with_ph = "".join(result_parts)
|
||||
text_with_ph = re.sub(r'\s+', ' ', text_with_ph).strip()
|
||||
|
||||
# Merge consecutive placeholders
|
||||
def merge_match(m):
|
||||
full_match = m.group(0)
|
||||
pids = re.findall(r'φ(/?\d+)φ', full_match)
|
||||
if len(pids) <= 1:
|
||||
return full_match
|
||||
|
||||
merged_html = ""
|
||||
for pid in pids:
|
||||
if pid in local_map:
|
||||
merged_html += local_map[pid]
|
||||
del local_map[pid]
|
||||
|
||||
new_pid = pids[0] if pids[0].isdigit() else pids[0][1:]
|
||||
local_map[new_pid] = merged_html
|
||||
return f"φ{new_pid}φ"
|
||||
|
||||
text_with_ph = re.sub(r'(φ/?\d+φ)(φ/?\d+φ)+', merge_match, text_with_ph)
|
||||
|
||||
return text_with_ph, local_map
|
||||
|
||||
|
||||
def _handle_drop_cap(self, prefix_parts: List[str], middle_parts: List[str],
|
||||
middle_types: List[str]) -> Tuple[List[str], List[str], List[str]]:
|
||||
"""
|
||||
处理 Drop Cap（首字母放大）样式
|
||||
|
||||
典型模式：<span class="dropcap">T</span>his is...
|
||||
问题：前缀会包含 <span>T</span>，但 T 是 This 的一部分
|
||||
|
||||
处理：
|
||||
1. 检测前缀末尾是否为 "格式化的单个大写字母"
|
||||
2. 检测中间部分首个文本是否以小写字母开头
|
||||
3. 如果两者组合成一个单词，移除格式，将纯字母加入中间部分
|
||||
"""
|
||||
def _handle_drop_cap(self, prefix_parts: List[str], middle_parts: List[str], middle_types: List[str]):
|
||||
if not prefix_parts or not middle_parts:
|
||||
return prefix_parts, middle_parts, middle_types
|
||||
|
||||
# 提取前缀中的文本内容
|
||||
prefix_text = ""
|
||||
last_text_idx = -1
|
||||
for i, part in enumerate(prefix_parts):
|
||||
for part in prefix_parts:
|
||||
if not part.startswith('<'):
|
||||
prefix_text = part.strip()
|
||||
last_text_idx = i
|
||||
|
||||
# 检测是否为单个大写字母
|
||||
if not prefix_text or len(prefix_text) != 1 or not prefix_text.isupper():
|
||||
return prefix_parts, middle_parts, middle_types
|
||||
|
||||
# 检测中间部分第一个可翻译文本
|
||||
first_middle_text = ""
|
||||
first_middle_idx = -1
|
||||
for i, (part, ptype) in enumerate(zip(middle_parts, middle_types)):
|
||||
@@ -368,99 +327,49 @@ class FormatExtractor:
|
||||
if not first_middle_text:
|
||||
return prefix_parts, middle_parts, middle_types
|
||||
|
||||
# Drop Cap 检测条件（满足任一即可）:
|
||||
# 1. 后续文本以小写字母开头: T + his = This
|
||||
# 2. 后续文本以大写字母开头且紧连（无空格）: I + N OCTOBER = IN OCTOBER
|
||||
is_drop_cap = False
|
||||
first_char = first_middle_text[0] if first_middle_text else ''
|
||||
|
||||
if first_char.islower():
|
||||
# 条件1: This 模式
|
||||
is_drop_cap = True
|
||||
elif first_char.isupper():
|
||||
# 条件2: IN OCTOBER 模式 - 检查是否紧连（第一个字母后不应有空格）
|
||||
# 原始 HTML 中 </span>N 表示 N 紧跟在 I 后面
|
||||
if first_char.islower() or first_char.isupper():
|
||||
is_drop_cap = True
|
||||
|
||||
if not is_drop_cap:
|
||||
return prefix_parts, middle_parts, middle_types
|
||||
|
||||
# 组合检测：大写字母 + 后续文本的第一个单词
|
||||
combined = prefix_text + first_middle_text.split()[0] if first_middle_text else ""
|
||||
|
||||
# 验证：组合后是否为合理的英文单词/大写序列（至少 2 个字母）
|
||||
if len(combined) >= 2 and combined.isalpha():
|
||||
# 确认是 Drop Cap，移除格式
|
||||
# 从前缀中移除这个字母和其包裹的格式标签
|
||||
new_prefix = []
|
||||
skip_until_close = False
|
||||
found_letter = False
|
||||
if not (len(combined) >= 2 and combined.isalpha()):
|
||||
return prefix_parts, middle_parts, middle_types
|
||||
|
||||
for part in prefix_parts:
|
||||
if part.startswith('<') and not part.startswith('</'):
|
||||
# 开始标签，准备跳过
|
||||
skip_until_close = True
|
||||
elif part.startswith('</'):
|
||||
# 结束标签
|
||||
if skip_until_close:
|
||||
skip_until_close = False
|
||||
continue
|
||||
new_prefix.append(part)
|
||||
elif part.strip() == prefix_text:
|
||||
# 这是那个大写字母，跳过
|
||||
found_letter = True
|
||||
new_prefix = []
|
||||
skip_until_close = False
|
||||
found_letter = False
|
||||
|
||||
for part in prefix_parts:
|
||||
if part.startswith('<') and not part.startswith('</'):
|
||||
skip_until_close = True
|
||||
elif part.startswith('</'):
|
||||
if skip_until_close:
|
||||
skip_until_close = False
|
||||
continue
|
||||
else:
|
||||
if not skip_until_close:
|
||||
new_prefix.append(part)
|
||||
|
||||
if found_letter:
|
||||
# 将大写字母加到中间部分第一个可翻译文本的开头
|
||||
middle_parts = middle_parts.copy()
|
||||
middle_parts[first_middle_idx] = prefix_text + middle_parts[first_middle_idx]
|
||||
prefix_parts = new_prefix
|
||||
new_prefix.append(part)
|
||||
elif part.strip() == prefix_text:
|
||||
found_letter = True
|
||||
continue
|
||||
else:
|
||||
if not skip_until_close:
|
||||
new_prefix.append(part)
|
||||
|
||||
if found_letter:
|
||||
middle_parts = middle_parts.copy()
|
||||
middle_parts[first_middle_idx] = prefix_text + middle_parts[first_middle_idx]
|
||||
prefix_parts = new_prefix
|
||||
|
||||
return prefix_parts, middle_parts, middle_types
|
||||
|
||||
|
||||
def _is_translatable_text(self, text: str) -> bool:
|
||||
"""
|
||||
判断文本是否需要翻译（包含可翻译的单词）
|
||||
|
||||
条件（满足任一即可）：
|
||||
1. 包含 3 个及以上连续字母（如 "and", "Art", "War"）
|
||||
2. 包含空格分隔的多个单词（如 "and Sun Tzu's"）
|
||||
"""
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return False
|
||||
# 条件1: 3 个及以上连续字母
|
||||
if re.search(r'[a-zA-Z]{3,}', text):
|
||||
return True
|
||||
# 条件2: 包含空格的多单词文本（如 "a of"）
|
||||
if ' ' in text and re.search(r'[a-zA-Z]', text):
|
||||
return True
|
||||
if re.search(r'\d', text):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _is_formula_element(self, element: Tag, text_content: str) -> bool:
|
||||
"""判断元素是否为公式元素（应整体保留不翻译）"""
|
||||
if not text_content:
|
||||
return True
|
||||
if len(text_content) <= 3:
|
||||
return True
|
||||
if re.search(r'[a-zA-Z]{4,}', text_content):
|
||||
return False
|
||||
return bool(self.FORMULA_CHARS.match(text_content))
|
||||
|
||||
def _get_opening_tag(self, element: Tag) -> str:
|
||||
"""获取元素的开始标签（含属性）"""
|
||||
attrs_str = ""
|
||||
for key, value in element.attrs.items():
|
||||
if isinstance(value, list):
|
||||
value = " ".join(value)
|
||||
attrs_str += f' {key}="{value}"'
|
||||
return f"<{element.name}{attrs_str}>"
|
||||
|
||||
def reset(self):
|
||||
"""兼容旧接口"""
|
||||
pass
|
||||
@@ -0,0 +1,83 @@
|
||||
import json
|
||||
import random
|
||||
from typing import Dict, List
|
||||
from loguru import logger
|
||||
from src.translation.llm_client import LLMClient
|
||||
from src.translation.manifest_manager import ManifestManager
|
||||
from src.common.data_model import BookProfile
|
||||
|
||||
class BookProfiler:
|
||||
def __init__(self, llm_client: LLMClient):
|
||||
self.llm_client = llm_client
|
||||
|
||||
def extract_sample_text(self, entries: List[object], char_limit: int = 3000) -> str:
|
||||
"""Extract sample text from manifest entries."""
|
||||
if not entries: return ""
|
||||
|
||||
# Simple sampling strategy: First few + random middle
|
||||
intro_text = []
|
||||
for entry in entries[:50]:
|
||||
if len(entry.original_text) > 50:
|
||||
intro_text.append(entry.original_text)
|
||||
|
||||
body_text = []
|
||||
candidates = [e for e in entries[50:] if len(e.original_text) > 80]
|
||||
if candidates:
|
||||
samples = random.sample(candidates, min(5, len(candidates)))
|
||||
body_text = [e.original_text for e in samples]
|
||||
|
||||
# Join text
|
||||
full_text = "\n\n".join(intro_text[:5] + body_text)
|
||||
|
||||
# Strip placeholders to stop Profiler from seeing "garbage"
|
||||
# Matches φ1φ, φ/1φ, etc.
|
||||
import re
|
||||
clean_text = re.sub(r'φ.*?φ', '', full_text)
|
||||
|
||||
return clean_text[:char_limit]
|
||||
|
||||
async def analyze(self, entries: List[object]) -> BookProfile:
|
||||
"""Generate Book Profile."""
|
||||
sample = self.extract_sample_text(entries)
|
||||
if not sample:
|
||||
return BookProfile(title="Unknown", author="Unknown")
|
||||
|
||||
logger.info("Generating Book Profile from sample text...")
|
||||
|
||||
system_prompt = "You are a senior publishing editor. Analyze the text and output JSON."
|
||||
user_prompt = f"""
|
||||
Please analyze the following book excerpt.
|
||||
Output JSON format:
|
||||
{{
|
||||
"title": "Book Title",
|
||||
"author": "Author Name",
|
||||
"genre": "Genre",
|
||||
"style": "Style description",
|
||||
"keywords": ["keyword1", "keyword2"],
|
||||
"style_guide": "Specific instruction for translator"
|
||||
}}
|
||||
|
||||
Excerpt:
|
||||
{sample}
|
||||
"""
|
||||
try:
|
||||
response = await self.llm_client.raw_chat_completion(system_prompt, user_prompt)
|
||||
json_str = response.strip()
|
||||
# Basic cleanup
|
||||
if "```json" in json_str:
|
||||
json_str = json_str.split("```json")[1].split("```")[0].strip()
|
||||
elif "```" in json_str:
|
||||
json_str = json_str.split("```")[1].split("```")[0].strip()
|
||||
|
||||
data = json.loads(json_str)
|
||||
|
||||
return BookProfile(
|
||||
title=data.get("title", "Unknown"),
|
||||
author=data.get("author", "Unknown"),
|
||||
genre=data.get("genre", "General"),
|
||||
keywords=data.get("keywords", []),
|
||||
style_guide=data.get("style_guide", "")
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Profile generation failed: {e}")
|
||||
return BookProfile(title="Unknown", author="Unknown")
|
||||
@@ -0,0 +1,132 @@
|
||||
import re
|
||||
from typing import List, Dict, Any, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
from loguru import logger
|
||||
|
||||
from src.common.data_model import BookStructure, ManifestEntry, BookProfile
|
||||
from src.preprocessing.format_extractor import FormatExtractor
|
||||
from src.common.utils import setup_logger
|
||||
|
||||
logger = setup_logger("fine_grained_extractor")
|
||||
|
||||
class FineGrainedExtractor:
|
||||
"""
|
||||
Extracts translatable text segments from BookStructure.
|
||||
Uses FormatExtractor for detailed content analysis.
|
||||
"""
|
||||
|
||||
SKIP_TRANSLATION_PATTERNS = [
|
||||
r'index\.x?html',
|
||||
r'bibliography\.x?html',
|
||||
r'endnotes?\.x?html',
|
||||
r'footnotes?\.x?html',
|
||||
r'copyright\.x?html',
|
||||
]
|
||||
|
||||
TOC_PATTERNS = [
|
||||
r'nav\.x?html',
|
||||
r'toc\.x?html',
|
||||
]
|
||||
|
||||
def __init__(self, translate_toc: bool = False):
|
||||
self.translate_toc = translate_toc
|
||||
self.format_extractor = FormatExtractor()
|
||||
|
||||
def extract(self, structure: BookStructure, profile: Optional[BookProfile] = None) -> List[ManifestEntry]:
|
||||
"""
|
||||
Extracts translatable segments from the BookStructure.
|
||||
Iteration follows the spine order.
|
||||
"""
|
||||
logger.info("Starting extraction from BookStructure...")
|
||||
manifest_entries = []
|
||||
|
||||
# Iterate over spine to maintain order
|
||||
for item_id in structure.spine:
|
||||
if item_id not in structure.resources:
|
||||
logger.warning(f"Item ID {item_id} in spine but not in resources.")
|
||||
continue
|
||||
|
||||
resource = structure.resources[item_id]
|
||||
|
||||
# Only process HTML/XHTML
|
||||
if resource.media_type != "application/xhtml+xml" or not resource.content:
|
||||
continue
|
||||
|
||||
file_path = resource.href
|
||||
doc_type = self._classify_document(file_path)
|
||||
|
||||
# Parse HTML
|
||||
soup = BeautifulSoup(resource.content, 'html.parser')
|
||||
target_tags = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li']
|
||||
|
||||
for element in soup.find_all(target_tags):
|
||||
# Ensure element has ID (should have been done by Cleaner)
|
||||
element_id = element.get('id')
|
||||
if not element_id:
|
||||
logger.warning(f"Element in {file_path} missing ID, skipping: {element.name}")
|
||||
continue
|
||||
|
||||
# Check translation eligibility
|
||||
raw_text = element.get_text(separator=' ', strip=True)
|
||||
if not raw_text.strip():
|
||||
continue
|
||||
|
||||
is_decorative = self._is_decorative(raw_text)
|
||||
should_translate = self._should_translate(doc_type, is_decorative, raw_text)
|
||||
|
||||
if should_translate:
|
||||
# Extract detailed format
|
||||
outer_html = str(element)
|
||||
clean_text, text_with_ph, ph_map, p_type, _ = self.format_extractor.extract(outer_html)
|
||||
|
||||
if clean_text.strip() and text_with_ph.strip():
|
||||
entry = ManifestEntry(
|
||||
entry_id=f"{file_path}#{element_id}",
|
||||
file_path=file_path,
|
||||
element_id=element_id,
|
||||
original_text=text_with_ph,
|
||||
placeholders=ph_map,
|
||||
context=p_type
|
||||
)
|
||||
manifest_entries.append(entry)
|
||||
|
||||
logger.info(f"Extracted {len(manifest_entries)} entries in total.")
|
||||
return manifest_entries
|
||||
|
||||
def _classify_document(self, file_name: str) -> str:
|
||||
if not file_name: return 'core'
|
||||
fname = file_name.lower()
|
||||
if any(re.search(p, fname) for p in self.SKIP_TRANSLATION_PATTERNS): return 'skip'
|
||||
if any(re.search(p, fname) for p in self.TOC_PATTERNS): return 'toc'
|
||||
return 'core'
|
||||
|
||||
def _should_translate(self, doc_type: str, is_decorative: bool, text: str = "") -> bool:
|
||||
if is_decorative: return False
|
||||
if self._is_roman_numeral(text): return False
|
||||
if doc_type == 'core': return True
|
||||
if doc_type == 'toc': return self.translate_toc
|
||||
return False
|
||||
|
||||
def _is_roman_numeral(self, text: str) -> bool:
|
||||
text = text.strip().upper()
|
||||
if not text: return False
|
||||
pattern = re.compile(r"^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$")
|
||||
return bool(pattern.match(text))
|
||||
|
||||
def _is_decorative(self, text: str) -> bool:
|
||||
s = text.strip()
|
||||
if not s: return False
|
||||
if not any(c.isalnum() for c in s): return True
|
||||
if len(s) > 20: return False
|
||||
|
||||
patterns = [
|
||||
r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾]+$',
|
||||
r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾\s]+$',
|
||||
]
|
||||
for p in patterns:
|
||||
if re.match(p, s): return True
|
||||
|
||||
unique = set(s.replace(' ', ''))
|
||||
if len(unique) <= 3 and (unique & set('*-_~•◆◇■□▪▫●○◉◎⦿⦾')):
|
||||
return True
|
||||
return False
|
||||
@@ -1,87 +0,0 @@
|
||||
"""
|
||||
Quality Manager Module
|
||||
|
||||
Responsible for evaluating translation quality and deciding on re-translation.
|
||||
"""
|
||||
|
||||
import json
|
||||
import random
|
||||
from typing import List, Dict, Any, Tuple
|
||||
from loguru import logger
|
||||
from .manifest_manager import ManifestItem
|
||||
from .llm_client import LLMClient
|
||||
|
||||
class QualityManager:
|
||||
def __init__(self, config: Dict, llm_client: LLMClient):
|
||||
self.config = config
|
||||
self.llm_client = llm_client
|
||||
self.qc_config = config['translation'].get('quality_control', {})
|
||||
self.pass_score = self.qc_config.get('pass_score', 7)
|
||||
self.sample_size = self.qc_config.get('sample_size', 2)
|
||||
|
||||
async def evaluate_chunk(self, chunk: List[ManifestItem]) -> Tuple[bool, int, str]:
|
||||
"""
|
||||
Evaluate a chunk of translations.
|
||||
|
||||
Returns:
|
||||
(passed: bool, average_score: int, reason: str)
|
||||
"""
|
||||
if not self.qc_config.get('enabled', False):
|
||||
return True, 10, "QC Disabled"
|
||||
|
||||
# 1. Sample items
|
||||
# Filter for items that actually have content and translations
|
||||
valid_items = [item for item in chunk if item.translation and len(item.clean_text) > 20]
|
||||
|
||||
if not valid_items:
|
||||
return True, 10, "No valid items to sample"
|
||||
|
||||
sample_items = random.sample(valid_items, min(len(valid_items), self.sample_size))
|
||||
|
||||
# 2. Build Prompt
|
||||
prompt = self._build_evaluation_prompt(sample_items)
|
||||
|
||||
# 3. Call LLM (Smart)
|
||||
try:
|
||||
response = await self.llm_client.raw_chat_completion(
|
||||
system_prompt="You are a professional translation editor.",
|
||||
user_prompt=prompt,
|
||||
model_type="smart"
|
||||
)
|
||||
|
||||
# 4. Parse JSON
|
||||
# Clean potential markdown
|
||||
json_str = response.strip()
|
||||
if "```json" in json_str:
|
||||
json_str = json_str.split("```json")[1].split("```")[0].strip()
|
||||
elif "```" in json_str:
|
||||
json_str = json_str.split("```")[1].split("```")[0].strip()
|
||||
|
||||
result = json.loads(json_str)
|
||||
score = result.get('score', 0)
|
||||
reason = result.get('reason', 'No reason provided')
|
||||
|
||||
passed = score >= self.pass_score
|
||||
return passed, score, reason
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"QC evaluation failed: {e}")
|
||||
# If QC fails, we default to PASS to avoid blocking progress, but log it
|
||||
return True, 0, f"QC Error: {e}"
|
||||
|
||||
def _build_evaluation_prompt(self, items: List[ManifestItem]) -> str:
|
||||
content = ""
|
||||
for i, item in enumerate(items, 1):
|
||||
content += f"Item {i}:\nOriginal: {item.clean_text}\nTranslation: {item.translation}\n\n"
|
||||
|
||||
return f"""Please evaluate the following translations (English to Chinese).
|
||||
Focus on accuracy, fluency, and terminology consistency.
|
||||
|
||||
Items to evaluate:
|
||||
{content}
|
||||
|
||||
Return a JSON object with:
|
||||
- \"score\": An integer from 1 to 10 (10 being perfect).
|
||||
- \"reason\": A brief explanation of the score.
|
||||
|
||||
JSON Output:"""
|
||||
@@ -1,273 +0,0 @@
|
||||
"""
|
||||
文本处理器模块 (Text Processor Module) - Manifest 驱动版
|
||||
|
||||
该模块专注于 HTML 文档的遍历和段落提取。
|
||||
它不再维护全局状态，而是将提取的内容注册到 ManifestManager 中。
|
||||
"""
|
||||
|
||||
import re
|
||||
from bs4 import BeautifulSoup
|
||||
from typing import List, Dict, Any
|
||||
from loguru import logger
|
||||
from .manifest_manager import ManifestManager
|
||||
from .format_extractor import FormatExtractor
|
||||
|
||||
|
||||
class TextProcessor:
|
||||
"""
|
||||
负责从 HTML 中识别有效段落并进行清洗。
|
||||
"""
|
||||
|
||||
def __init__(self, config: Dict):
|
||||
"""
|
||||
Args:
|
||||
config (Dict): 全局配置。
|
||||
"""
|
||||
self.config = config
|
||||
self.chunk_size = config['translation'].get('chunk_size', 5000)
|
||||
self.format_extractor = FormatExtractor()
|
||||
|
||||
def extract_to_manifest(self, html_content: str, source_file: str, manifest: ManifestManager, mode: str = "bilingual"):
|
||||
"""
|
||||
解析 HTML 内容，并将识别出的段落注册到 Manifest 中。
|
||||
|
||||
Args:
|
||||
html_content (str): HTML 源码。
|
||||
source_file (str): 来源文件名。
|
||||
manifest (ManifestManager): 清单管理器实例。
|
||||
mode (str): 翻译模式 - "bilingual" 或 "chinese"
|
||||
"""
|
||||
try:
|
||||
soup = BeautifulSoup(html_content, 'html.parser')
|
||||
|
||||
# 1. 移除不需要的元素
|
||||
for element in soup(['script', 'style', 'meta', 'link']):
|
||||
element.decompose()
|
||||
|
||||
# 2. 获取有效的文本元素 (使用静态过滤逻辑)
|
||||
text_elements = self.get_valid_text_elements(soup)
|
||||
|
||||
# 3. 注册到 Manifest
|
||||
for element in text_elements:
|
||||
clean_text = self.clean_element_text(element)
|
||||
|
||||
# 过滤逻辑
|
||||
if not clean_text:
|
||||
continue
|
||||
|
||||
status = "pending"
|
||||
# 如果是导航元素，标记为 ignored
|
||||
if self.is_navigation_element(element):
|
||||
status = "ignored"
|
||||
|
||||
# 提取格式信息（中文模式）
|
||||
text_with_ph = ""
|
||||
placeholder_map = None
|
||||
p_type = "body"
|
||||
|
||||
# 检查是否是嵌套容器
|
||||
is_nested = self.is_nested_container(element)
|
||||
|
||||
if mode == "chinese":
|
||||
if is_nested:
|
||||
# 嵌套容器：先移除子块，再提取
|
||||
element_html = self._get_element_without_nested_blocks(element)
|
||||
clean_text, text_with_ph, placeholder_map, p_type = self.format_extractor.extract(element_html)
|
||||
else:
|
||||
# 普通元素：直接提取
|
||||
clean_text, text_with_ph, placeholder_map, p_type = self.format_extractor.extract(str(element))
|
||||
|
||||
# 注册
|
||||
item = manifest.add_item(
|
||||
source_file=source_file,
|
||||
original_html=str(element) if not is_nested else self._get_element_without_nested_blocks(element),
|
||||
clean_text=clean_text,
|
||||
tag=element.name,
|
||||
metadata={"status": status}
|
||||
)
|
||||
item.tag_attrs = element.attrs # 存储外层标签属性
|
||||
|
||||
# 更新中文模式字段
|
||||
if mode == "chinese":
|
||||
item.text_with_placeholders = text_with_ph
|
||||
item.placeholder_map = placeholder_map
|
||||
item.paragraph_type = p_type
|
||||
|
||||
# 同步更新 manifest 状态 (如果需要过滤)
|
||||
if status == "ignored":
|
||||
manifest.update_item(item.global_id, translation=None, status="ignored")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"从 {source_file} 提取段落失败: {e}")
|
||||
|
||||
# 类变量：存储需要特殊处理的元素（包含嵌套块的容器）
|
||||
_nested_container_ids = set()
|
||||
|
||||
@classmethod
|
||||
def get_valid_text_elements(cls, soup) -> List:
|
||||
"""
|
||||
获取有效的文本容器元素（返回原始元素，不是副本）。
|
||||
|
||||
对于包含嵌套块的容器元素：
|
||||
1. 仍然返回原始元素
|
||||
2. 在 _nested_container_ids 中记录其 id
|
||||
3. 这些元素在回填时需要特殊处理
|
||||
"""
|
||||
block_tags = ['p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote', 'li', 'td']
|
||||
all_candidates = soup.find_all(block_tags)
|
||||
candidate_set = set(all_candidates)
|
||||
|
||||
cls._nested_container_ids = set() # 重置
|
||||
final_elements = []
|
||||
processed_ids = set()
|
||||
|
||||
for element in all_candidates:
|
||||
elem_id = id(element)
|
||||
if elem_id in processed_ids:
|
||||
continue
|
||||
|
||||
# 找到此元素包含的子块
|
||||
nested_blocks = [d for d in element.find_all(block_tags) if d in candidate_set]
|
||||
|
||||
if not nested_blocks:
|
||||
# 叶子节点，直接添加
|
||||
final_elements.append(element)
|
||||
processed_ids.add(elem_id)
|
||||
else:
|
||||
# 容器节点，检查是否有直接文本（不在子块中）
|
||||
direct_text = cls.extract_direct_text(element)
|
||||
|
||||
if len(direct_text.strip()) > 50:
|
||||
# 有价值的直接文本，添加原始元素并标记
|
||||
final_elements.append(element)
|
||||
cls._nested_container_ids.add(elem_id)
|
||||
processed_ids.add(elem_id)
|
||||
|
||||
return final_elements
|
||||
|
||||
@staticmethod
|
||||
def extract_direct_text(element) -> str:
|
||||
"""
|
||||
提取元素的直接文本内容（不包括嵌套块级元素中的文本）
|
||||
"""
|
||||
block_tags = ['p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote', 'li', 'td']
|
||||
|
||||
# 创建副本进行分析
|
||||
from bs4 import BeautifulSoup as BS
|
||||
from bs4 import NavigableString
|
||||
|
||||
element_copy = BS(str(element), 'html.parser').find(element.name)
|
||||
if not element_copy:
|
||||
return ""
|
||||
|
||||
# 移除所有块级子元素
|
||||
for nested in element_copy.find_all(block_tags):
|
||||
nested.decompose()
|
||||
|
||||
return element_copy.get_text().strip()
|
||||
|
||||
@classmethod
|
||||
def is_nested_container(cls, element) -> bool:
|
||||
"""检查元素是否是包含嵌套块的容器（需要特殊回填处理）"""
|
||||
return id(element) in cls._nested_container_ids
|
||||
|
||||
@staticmethod
|
||||
def _get_element_without_nested_blocks(element) -> str:
|
||||
"""
|
||||
返回移除子块后的元素 HTML 字符串
|
||||
|
||||
用于嵌套容器的提取，避免子块内容被提取两次
|
||||
"""
|
||||
from bs4 import BeautifulSoup as BS
|
||||
block_tags = ['p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote', 'li', 'td']
|
||||
|
||||
element_copy = BS(str(element), 'html.parser').find(element.name)
|
||||
if not element_copy:
|
||||
return str(element)
|
||||
|
||||
# 移除所有块级子元素
|
||||
for nested in element_copy.find_all(block_tags):
|
||||
nested.decompose()
|
||||
|
||||
return str(element_copy)
|
||||
|
||||
|
||||
@staticmethod
|
||||
def clean_element_text(element) -> str:
|
||||
"""清理 HTML 元素，提取纯净的待翻译文本。"""
|
||||
element_copy = element.__copy__()
|
||||
|
||||
# 移除脚注引用等
|
||||
for tag in element_copy.find_all(['sup', 'sub']):
|
||||
tag.decompose()
|
||||
|
||||
footnote_patterns = re.compile(r'footnote|endnote|reference|note|super|sub', re.I)
|
||||
for tag in element_copy.find_all(['a', 'span', 'div'], class_=footnote_patterns):
|
||||
tag.decompose()
|
||||
|
||||
# 移除仅包含数字的 span
|
||||
for tag in element_copy.find_all('span'):
|
||||
if re.match(r'^(\[\d+\]|\(\d+\)|\d+)$', tag.get_text().strip()):
|
||||
tag.decompose()
|
||||
|
||||
text = element_copy.get_text().strip()
|
||||
# 正则清理残留引用标识 (如 sentence.2)
|
||||
text = re.sub(r'(\.|。|，|,)\s*(\[\d+\]|\d+)(?=\s|$)', r'\1', text)
|
||||
text = re.sub(r'\s+', ' ', text)
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def is_navigation_element(element) -> bool:
|
||||
"""判断是否是无翻译价值的导航、页码元素。"""
|
||||
classes = element.get('class', [])
|
||||
nav_classes = ['nav', 'navigation', 'toc', 'menu', 'header', 'footer', 'page-number']
|
||||
class_str = ' '.join(classes).lower() if isinstance(classes, list) else str(classes).lower()
|
||||
|
||||
if any(nc in class_str for nc in nav_classes):
|
||||
return True
|
||||
|
||||
# 检查父级
|
||||
parent = element.parent
|
||||
if parent:
|
||||
p_classes = parent.get('class', [])
|
||||
p_class_str = ' '.join(p_classes).lower() if isinstance(p_classes, list) else str(p_classes).lower()
|
||||
if any(nc in p_class_str for nc in nav_classes):
|
||||
return True
|
||||
return False
|
||||
|
||||
def create_chunks_from_manifest(self, manifest: ManifestManager, mode: str = "bilingual") -> List[List[Any]]:
|
||||
"""
|
||||
从 Manifest 中筛选待翻译项目并分块。
|
||||
|
||||
Args:
|
||||
manifest: ManifestManager 实例
|
||||
mode: 翻译模式 (保留参数以供将来使用)
|
||||
"""
|
||||
pending_items = manifest.get_items(status="pending")
|
||||
if not pending_items:
|
||||
return []
|
||||
|
||||
chunks = []
|
||||
current_chunk = []
|
||||
current_size = 0
|
||||
|
||||
for item in pending_items:
|
||||
# 中文模式使用带占位符的文本长度
|
||||
if mode == "chinese" and item.text_with_placeholders:
|
||||
text_len = len(item.text_with_placeholders)
|
||||
else:
|
||||
text_len = len(item.clean_text)
|
||||
|
||||
if current_size + text_len > self.chunk_size and current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
current_chunk = []
|
||||
current_size = 0
|
||||
|
||||
current_chunk.append(item)
|
||||
current_size += text_len
|
||||
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
|
||||
logger.info(f"分块完成: 共有 {len(pending_items)} 个待翻译项，分为 {len(chunks)} 个块")
|
||||
return chunks
|
||||
@@ -1,411 +0,0 @@
|
||||
"""
|
||||
TOC 解析器模块 (TOC Parser Module)
|
||||
|
||||
该模块负责从 EPUB 文件中提取目录结构，并提供章节范围选择功能。
|
||||
支持嵌套的目录结构（如 Part -> Chapter）。
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Set, Optional, Tuple
|
||||
from ebooklib import epub
|
||||
from loguru import logger
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
@dataclass
|
||||
class TOCItem:
|
||||
"""代表一个目录项"""
|
||||
index: int # 序号 (1-based)
|
||||
title: str # 章节标题
|
||||
href: str # 文件路径 (如 index_split_003.html 或 e9781668053393/xhtml/ch01.xhtml)
|
||||
file_name: str # 纯文件名 (不含锚点)
|
||||
level: int # 层级 (0=顶级, 1=子章节, 2=子子章节)
|
||||
skip_reason: str = "" # 跳过原因: 'front', 'back', 或空字符串表示不跳过
|
||||
|
||||
|
||||
# 前置部分 - 通常不需要翻译
|
||||
FRONT_MATTER_PATTERNS = [
|
||||
'cover', 'title page', 'copyright', 'contents',
|
||||
'table of contents', 'half title', 'halftitle',
|
||||
'how to use this ebook', 'copyright page'
|
||||
]
|
||||
|
||||
# 后置部分 - 通常不需要翻译
|
||||
# 注意：使用精确匹配避免误伤，如 "notes" 会匹配 "Technical Notes"
|
||||
BACK_MATTER_PATTERNS = [
|
||||
'endnotes', 'footnotes', 'bibliography',
|
||||
'references', 'index', 'about the author',
|
||||
'about the publisher', 'credits', 'appendix',
|
||||
'glossary', 'also by', 'resources for'
|
||||
]
|
||||
|
||||
# 需要精确匹配的模式（标题必须完全等于这些值）
|
||||
BACK_MATTER_EXACT = [
|
||||
'notes' # 精确匹配，避免匹配 "Technical Notes"
|
||||
]
|
||||
|
||||
|
||||
class TOCParser:
|
||||
"""
|
||||
TOC 解析器
|
||||
|
||||
从 EPUB 提取扁平化的目录列表，并支持章节范围选择。
|
||||
"""
|
||||
|
||||
def __init__(self, book: epub.EpubBook):
|
||||
self.book = book
|
||||
self._toc_items: List[TOCItem] = []
|
||||
self._parse_toc()
|
||||
self._classify_all_chapters()
|
||||
|
||||
def _parse_toc(self):
|
||||
"""解析 book.toc，构建扁平化的目录列表"""
|
||||
self._toc_items = []
|
||||
index = [0] # 使用列表以便在嵌套函数中修改
|
||||
|
||||
def traverse(toc_list, level=0):
|
||||
for item in toc_list:
|
||||
if isinstance(item, tuple):
|
||||
# 嵌套结构: (section, children)
|
||||
section, children = item
|
||||
index[0] += 1
|
||||
href = section.href if hasattr(section, 'href') else ""
|
||||
file_name = self._extract_file_name(href)
|
||||
self._toc_items.append(TOCItem(
|
||||
index=index[0],
|
||||
title=section.title if hasattr(section, 'title') else str(section),
|
||||
href=href,
|
||||
file_name=file_name,
|
||||
level=level
|
||||
))
|
||||
# 递归处理子节点
|
||||
traverse(children, level + 1)
|
||||
else:
|
||||
# 叶子节点
|
||||
index[0] += 1
|
||||
href = item.href if hasattr(item, 'href') else ""
|
||||
file_name = self._extract_file_name(href)
|
||||
self._toc_items.append(TOCItem(
|
||||
index=index[0],
|
||||
title=item.title if hasattr(item, 'title') else str(item),
|
||||
href=href,
|
||||
file_name=file_name,
|
||||
level=level
|
||||
))
|
||||
|
||||
traverse(self.book.toc)
|
||||
logger.debug(f"解析 TOC 完成，共 {len(self._toc_items)} 个章节")
|
||||
|
||||
def _classify_chapter(self, title: str) -> str:
|
||||
"""
|
||||
分类单个章节
|
||||
|
||||
Returns:
|
||||
'front': 前置部分（跳过）
|
||||
'back': 后置部分（跳过）
|
||||
'': 正文内容（保留）
|
||||
"""
|
||||
title_lower = title.lower().strip()
|
||||
|
||||
# 检查前置部分（模糊匹配）
|
||||
for pattern in FRONT_MATTER_PATTERNS:
|
||||
if pattern in title_lower or title_lower == pattern:
|
||||
return 'front'
|
||||
|
||||
# 检查后置部分（模糊匹配）
|
||||
for pattern in BACK_MATTER_PATTERNS:
|
||||
if pattern in title_lower or title_lower == pattern:
|
||||
return 'back'
|
||||
|
||||
# 检查后置部分（精确匹配）
|
||||
for pattern in BACK_MATTER_EXACT:
|
||||
if title_lower == pattern:
|
||||
return 'back'
|
||||
|
||||
return ''
|
||||
|
||||
def _classify_all_chapters(self):
|
||||
"""对所有章节进行分类"""
|
||||
for item in self._toc_items:
|
||||
item.skip_reason = self._classify_chapter(item.title)
|
||||
|
||||
# 统计跳过数量
|
||||
front_count = sum(1 for i in self._toc_items if i.skip_reason == 'front')
|
||||
back_count = sum(1 for i in self._toc_items if i.skip_reason == 'back')
|
||||
if front_count or back_count:
|
||||
logger.debug(f"章节分类: 跳过前置 {front_count} 个，跳过后置 {back_count} 个")
|
||||
|
||||
def get_skip_files(self) -> Set[str]:
|
||||
"""获取应该跳过的文件集合"""
|
||||
return {item.file_name for item in self._toc_items
|
||||
if item.skip_reason and item.file_name}
|
||||
|
||||
def get_content_files(self) -> Set[str]:
|
||||
"""获取正文内容的文件集合（排除前置和后置）"""
|
||||
return {item.file_name for item in self._toc_items
|
||||
if not item.skip_reason and item.file_name}
|
||||
|
||||
def get_spine_files(self) -> List[str]:
|
||||
"""获取 Spine 中的所有文件（按阅读顺序）"""
|
||||
spine_files = []
|
||||
for item_tuple in self.book.spine:
|
||||
item_id = item_tuple[0]
|
||||
item = self.book.get_item_with_id(item_id)
|
||||
if item:
|
||||
spine_files.append(item.get_name())
|
||||
return spine_files
|
||||
|
||||
def get_content_files_from_spine(self) -> Set[str]:
|
||||
"""
|
||||
基于 Spine 获取正文内容文件（排除前置和后置）
|
||||
|
||||
核心逻辑：
|
||||
1. 找到第一个正文章节在 Spine 中的位置
|
||||
2. 找到最后一个正文章节在 Spine 中的位置
|
||||
3. 返回这个范围内的所有 Spine 文件
|
||||
"""
|
||||
spine_files = self.get_spine_files()
|
||||
if not spine_files:
|
||||
return self.get_content_files() # 降级到 TOC 文件
|
||||
|
||||
# 获取正文和跳过的 TOC 文件
|
||||
content_toc_files = self.get_content_files()
|
||||
skip_toc_files = self.get_skip_files()
|
||||
|
||||
if not content_toc_files:
|
||||
return set(spine_files) # 没有分类信息，返回所有
|
||||
|
||||
# 在 Spine 中找到正文内容的边界
|
||||
first_content_idx = None
|
||||
last_content_idx = None
|
||||
|
||||
for idx, spine_file in enumerate(spine_files):
|
||||
if spine_file in content_toc_files:
|
||||
if first_content_idx is None:
|
||||
first_content_idx = idx
|
||||
last_content_idx = idx
|
||||
|
||||
if first_content_idx is None:
|
||||
return self.get_content_files() # 降级
|
||||
|
||||
# 收集边界内的所有 Spine 文件
|
||||
result = set()
|
||||
for idx in range(first_content_idx, last_content_idx + 1):
|
||||
spine_file = spine_files[idx]
|
||||
# 排除明确标记为跳过的文件
|
||||
if spine_file not in skip_toc_files:
|
||||
result.add(spine_file)
|
||||
|
||||
logger.debug(f"Spine 正文范围: {first_content_idx+1} ~ {last_content_idx+1}，共 {len(result)} 个文件")
|
||||
return result
|
||||
|
||||
def get_spine_range(self, start_title: str = None, end_title: str = None) -> Tuple[Set[str], List[TOCItem]]:
|
||||
"""
|
||||
基于 Spine 和 TOC 边界获取文件范围
|
||||
|
||||
与 get_file_range 的区别：
|
||||
- get_file_range: 只返回 TOC 中列出的文件
|
||||
- get_spine_range: 返回 TOC 边界之间的所有 Spine 文件
|
||||
"""
|
||||
spine_files = self.get_spine_files()
|
||||
|
||||
# 确定 TOC 边界
|
||||
start_item = self.find_by_title(start_title) if start_title else None
|
||||
end_item = self.find_by_title(end_title) if end_title else None
|
||||
|
||||
start_idx = start_item.index if start_item else 1
|
||||
end_idx = end_item.index if end_item else len(self._toc_items)
|
||||
|
||||
if start_idx > end_idx:
|
||||
start_idx, end_idx = end_idx, start_idx
|
||||
|
||||
# 获取选中的 TOC 项
|
||||
selected_items = [i for i in self._toc_items if start_idx <= i.index <= end_idx]
|
||||
selected_toc_files = {i.file_name for i in selected_items if i.file_name}
|
||||
|
||||
# 在 Spine 中找到这些文件的边界
|
||||
spine_start = None
|
||||
spine_end = None
|
||||
|
||||
for idx, spine_file in enumerate(spine_files):
|
||||
if spine_file in selected_toc_files:
|
||||
if spine_start is None:
|
||||
spine_start = idx
|
||||
spine_end = idx
|
||||
|
||||
if spine_start is None:
|
||||
# 降级到 TOC 文件
|
||||
logger.warning("无法在 Spine 中定位章节边界，使用 TOC 文件")
|
||||
return selected_toc_files, selected_items
|
||||
|
||||
# 扩展到下一个 TOC 章节之前
|
||||
# 找到 end_idx 之后的下一个 TOC 章节在 Spine 中的位置
|
||||
next_toc_file = None
|
||||
if end_idx < len(self._toc_items):
|
||||
next_toc_file = self._toc_items[end_idx].file_name # end_idx 是 1-based
|
||||
|
||||
if next_toc_file:
|
||||
for idx, spine_file in enumerate(spine_files):
|
||||
if spine_file == next_toc_file:
|
||||
spine_end = idx - 1 # 到下一章之前
|
||||
break
|
||||
|
||||
# 收集 Spine 范围内的所有文件
|
||||
result = set()
|
||||
for idx in range(spine_start, spine_end + 1):
|
||||
if idx < len(spine_files):
|
||||
result.add(spine_files[idx])
|
||||
|
||||
logger.info(f"Spine 范围: #{spine_start+1} ~ #{spine_end+1}，共 {len(result)} 个文件（TOC: {len(selected_toc_files)} 个）")
|
||||
return result, selected_items
|
||||
|
||||
def _extract_file_name(self, href: str) -> str:
|
||||
"""从 href 中提取纯文件名（去除锚点和路径前缀）"""
|
||||
if not href:
|
||||
return ""
|
||||
# 去除锚点 (#section1)
|
||||
path = href.split('#')[0]
|
||||
# 返回完整路径（可能包含子目录）
|
||||
return path
|
||||
|
||||
@property
|
||||
def items(self) -> List[TOCItem]:
|
||||
"""获取所有目录项"""
|
||||
return self._toc_items
|
||||
|
||||
def find_by_title(self, title: str, fuzzy: bool = True) -> Optional[TOCItem]:
|
||||
"""
|
||||
根据标题查找目录项
|
||||
|
||||
Args:
|
||||
title: 章节标题
|
||||
fuzzy: 是否模糊匹配（包含即可）
|
||||
|
||||
Returns:
|
||||
匹配的 TOCItem 或 None
|
||||
"""
|
||||
title_lower = title.lower().strip()
|
||||
|
||||
for item in self._toc_items:
|
||||
item_title_lower = item.title.lower().strip()
|
||||
|
||||
if fuzzy:
|
||||
# 模糊匹配：互相包含
|
||||
if title_lower in item_title_lower or item_title_lower in title_lower:
|
||||
return item
|
||||
else:
|
||||
# 精确匹配
|
||||
if item_title_lower == title_lower:
|
||||
return item
|
||||
|
||||
return None
|
||||
|
||||
def find_by_index(self, index: int) -> Optional[TOCItem]:
|
||||
"""根据序号查找目录项 (1-based)"""
|
||||
if 1 <= index <= len(self._toc_items):
|
||||
return self._toc_items[index - 1]
|
||||
return None
|
||||
|
||||
def get_file_range(self, start_title: str = None, end_title: str = None,
|
||||
start_index: int = None, end_index: int = None) -> Tuple[Set[str], List[TOCItem]]:
|
||||
"""
|
||||
获取指定范围内的文件集合
|
||||
|
||||
支持两种方式指定范围：
|
||||
1. 按标题: start_title ~ end_title
|
||||
2. 按序号: start_index ~ end_index
|
||||
|
||||
Returns:
|
||||
(文件名集合, 选中的目录项列表)
|
||||
"""
|
||||
# 确定起始位置
|
||||
start_item = None
|
||||
if start_title:
|
||||
start_item = self.find_by_title(start_title)
|
||||
if not start_item:
|
||||
logger.warning(f"未找到起始章节: {start_title}")
|
||||
elif start_index:
|
||||
start_item = self.find_by_index(start_index)
|
||||
|
||||
# 确定结束位置
|
||||
end_item = None
|
||||
if end_title:
|
||||
end_item = self.find_by_title(end_title)
|
||||
if not end_item:
|
||||
logger.warning(f"未找到结束章节: {end_title}")
|
||||
elif end_index:
|
||||
end_item = self.find_by_index(end_index)
|
||||
|
||||
# 默认值
|
||||
start_idx = start_item.index if start_item else 1
|
||||
end_idx = end_item.index if end_item else len(self._toc_items)
|
||||
|
||||
# 确保顺序正确
|
||||
if start_idx > end_idx:
|
||||
start_idx, end_idx = end_idx, start_idx
|
||||
|
||||
# 收集文件
|
||||
selected_items = []
|
||||
file_names = set()
|
||||
|
||||
for item in self._toc_items:
|
||||
if start_idx <= item.index <= end_idx:
|
||||
selected_items.append(item)
|
||||
if item.file_name:
|
||||
file_names.add(item.file_name)
|
||||
|
||||
logger.info(f"选择范围: #{start_idx} ~ #{end_idx}，共 {len(file_names)} 个文件")
|
||||
return file_names, selected_items
|
||||
|
||||
def format_toc_table(self, selected_range: Tuple[int, int] = None, show_skip: bool = True) -> str:
|
||||
"""
|
||||
格式化 TOC 为表格形式，用于终端显示
|
||||
|
||||
Args:
|
||||
selected_range: 可选的选中范围 (start_index, end_index)，用于高亮显示
|
||||
show_skip: 是否显示跳过标记
|
||||
|
||||
Returns:
|
||||
格式化的表格字符串
|
||||
"""
|
||||
if not self._toc_items:
|
||||
return "目录为空"
|
||||
|
||||
lines = []
|
||||
lines.append("")
|
||||
lines.append("=" * 75)
|
||||
lines.append(f"{'#':>4} {'状态':<6} {'章节名称':<35} {'文件'}")
|
||||
lines.append("=" * 75)
|
||||
|
||||
for item in self._toc_items:
|
||||
indent = " " * item.level
|
||||
title_display = f"{indent}{item.title}"
|
||||
if len(title_display) > 33:
|
||||
title_display = title_display[:30] + "..."
|
||||
|
||||
# 跳过状态标记
|
||||
status = ""
|
||||
if show_skip and item.skip_reason:
|
||||
status = "[SKIP]" if item.skip_reason else ""
|
||||
|
||||
# 如果在选中范围内，添加标记
|
||||
marker = ""
|
||||
if selected_range:
|
||||
start_idx, end_idx = selected_range
|
||||
if item.index == start_idx:
|
||||
marker = " ▶"
|
||||
elif item.index == end_idx:
|
||||
marker = " ◀"
|
||||
elif start_idx < item.index < end_idx:
|
||||
marker = " │"
|
||||
|
||||
lines.append(f"{item.index:>4}{marker:2} {status:<6} {title_display:<35} {item.file_name}")
|
||||
|
||||
lines.append("=" * 75)
|
||||
|
||||
# 统计摘要
|
||||
skip_count = sum(1 for i in self._toc_items if i.skip_reason)
|
||||
content_count = len(self._toc_items) - skip_count
|
||||
lines.append(f" 正文章节: {content_count} | 跳过章节: {skip_count}")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,231 @@
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
from openai import AsyncOpenAI
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||
from loguru import logger
|
||||
|
||||
from src.common.data_model import ManifestEntry
|
||||
from src.common.utils import setup_logger, ensure_directory
|
||||
|
||||
logger = setup_logger("llm_client")
|
||||
|
||||
# Default chunk save directory (can be overridden)
|
||||
DEFAULT_CHUNK_DIR = Path("tmp/chunks")
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
"""Rate limiter for concurrency and RPM."""
|
||||
def __init__(self, requests_per_minute: int, concurrent_requests: int):
|
||||
self.semaphore = asyncio.Semaphore(concurrent_requests)
|
||||
self.min_interval = 60.0 / requests_per_minute if requests_per_minute > 0 else 0
|
||||
self.last_request_time = 0
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def acquire(self):
|
||||
await self.semaphore.acquire()
|
||||
async with self._lock:
|
||||
current_time = time.time()
|
||||
wait_time = self.min_interval - (current_time - self.last_request_time)
|
||||
if wait_time > 0:
|
||||
await asyncio.sleep(wait_time)
|
||||
self.last_request_time = time.time()
|
||||
|
||||
def release(self):
|
||||
self.semaphore.release()
|
||||
|
||||
|
||||
class LLMClient:
|
||||
"""Generic OpenAI-compatible API Client with short ID strategy."""
|
||||
|
||||
def __init__(self, api_key: str, base_url: str, model: str = "gpt-3.5-turbo",
|
||||
requests_per_minute: int = 60, concurrent_requests: int = 5,
|
||||
extra_headers: Dict = None, chunk_dir: Path = None):
|
||||
|
||||
# Configure proxy client to avoid SOCKS issues and ensure connectivity
|
||||
import httpx
|
||||
import os
|
||||
|
||||
# Prefer HTTP proxy if available to avoid missing socksio support
|
||||
proxy_url = os.environ.get("http_proxy") or os.environ.get("https_proxy")
|
||||
http_client = httpx.AsyncClient(
|
||||
proxy=proxy_url,
|
||||
timeout=60.0,
|
||||
follow_redirects=True
|
||||
) if proxy_url else None
|
||||
|
||||
self.client = AsyncOpenAI(
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
default_headers=extra_headers,
|
||||
http_client=http_client
|
||||
)
|
||||
self.model = model
|
||||
|
||||
self.rate_limiter = RateLimiter(requests_per_minute, concurrent_requests)
|
||||
self.prompts = self._load_prompts()
|
||||
self._chunk_counter = 0
|
||||
|
||||
# Chunk directory for debug output
|
||||
self.chunk_dir = chunk_dir or DEFAULT_CHUNK_DIR
|
||||
ensure_directory(self.chunk_dir)
|
||||
|
||||
def _load_prompts(self) -> Dict:
|
||||
try:
|
||||
with open("config/prompts.json", "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load config/prompts.json: {e}")
|
||||
return {}
|
||||
|
||||
async def translate_chunk(self, items: List[ManifestEntry], glossary: Dict = None,
|
||||
instruction: str = None, mode: str = "bilingual") -> Dict[str, str]:
|
||||
"""
|
||||
Translate a chunk of items using short ID strategy.
|
||||
Returns: Dict[entry_id, translated_text]
|
||||
"""
|
||||
if not items: return {}
|
||||
|
||||
# Build prompt with short IDs
|
||||
id_map, prompt = self._build_prompt_with_short_ids(items)
|
||||
|
||||
try:
|
||||
# Build System Prompt
|
||||
base_sys_prompt = self.prompts.get("translation", {}).get("system",
|
||||
"You are a professional English to Chinese translator.")
|
||||
|
||||
if instruction:
|
||||
base_sys_prompt += f"\n\nBook Style Guide:\n{instruction}"
|
||||
|
||||
if glossary:
|
||||
glossary_text = "\n".join([f"{k} -> {v}" for k, v in glossary.items()])
|
||||
base_sys_prompt += f"\n\nTerminology:\n{glossary_text}"
|
||||
|
||||
# Short ID format instructions
|
||||
base_sys_prompt += """
|
||||
|
||||
Output Format:
|
||||
- Each line MUST start with #N: (keep this ID exactly as given)
|
||||
- Preserve any φXφ or φ/Xφ placeholders EXACTLY as-is
|
||||
- Only output translations, no explanations
|
||||
- Match the number of output lines to input lines"""
|
||||
|
||||
# Save chunk before translation
|
||||
chunk_id = self._save_chunk("before", prompt, base_sys_prompt)
|
||||
|
||||
logger.debug(f"Sending request to LLM (Chunk: {chunk_id}, Items: {len(items)})")
|
||||
|
||||
raw_response = await self._make_request(base_sys_prompt, prompt)
|
||||
|
||||
# Save chunk after translation
|
||||
self._save_chunk("after", raw_response, base_sys_prompt, chunk_id)
|
||||
|
||||
# Parse with short ID mapping
|
||||
results = self._parse_short_id_response(raw_response, id_map)
|
||||
return results
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Translation failed: {e}")
|
||||
return {item.entry_id: f"[Error - {str(e)}]" for item in items}
|
||||
|
||||
def _build_prompt_with_short_ids(self, items: List[ManifestEntry]) -> tuple:
|
||||
"""
|
||||
Build prompt with short IDs (#1, #2, ...).
|
||||
Returns: (id_map, prompt_text)
|
||||
"""
|
||||
id_map = {} # short_id -> entry_id
|
||||
lines = []
|
||||
|
||||
for i, item in enumerate(items, 1):
|
||||
short_id = f"#{i}"
|
||||
id_map[short_id] = item.entry_id
|
||||
|
||||
# Clean text (remove extra whitespace)
|
||||
text = re.sub(r'\s+', ' ', item.original_text).strip()
|
||||
lines.append(f"{short_id}: {text}")
|
||||
|
||||
return id_map, "\n".join(lines)
|
||||
|
||||
def _parse_short_id_response(self, response: str, id_map: Dict[str, str]) -> Dict[str, str]:
|
||||
"""
|
||||
Parse response with short ID format.
|
||||
Returns: Dict[entry_id, translated_text]
|
||||
"""
|
||||
results = {}
|
||||
|
||||
for line in response.split("\n"):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# Match #N: pattern
|
||||
# Handles both ASCII colon (:) and full-width colon (：)
|
||||
match = re.match(r'^#(\d+)[:：]\s*(.+)$', line)
|
||||
if match:
|
||||
short_id = f"#{match.group(1)}"
|
||||
content = match.group(2).strip()
|
||||
|
||||
# Recursively strip repeated IDs (e.g. "#12: #12: Text")
|
||||
while True:
|
||||
sub_match = re.match(r'^#(\d+)[:：]\s*(.+)$', content)
|
||||
if sub_match:
|
||||
# Check if the inner ID matches the outer ID, or just strip it anyway
|
||||
# Usually LLM repeats the same ID.
|
||||
content = sub_match.group(2).strip()
|
||||
else:
|
||||
break
|
||||
|
||||
if short_id in id_map:
|
||||
full_id = id_map[short_id]
|
||||
results[full_id] = content
|
||||
else:
|
||||
logger.warning(f"Unknown short ID in response: {short_id}")
|
||||
|
||||
return results
|
||||
|
||||
def _save_chunk(self, stage: str, content: str, system_prompt: str = None,
|
||||
chunk_id: str = None) -> str:
|
||||
"""Save chunk to tmp directory for debugging."""
|
||||
if chunk_id is None:
|
||||
self._chunk_counter += 1
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
chunk_id = f"{timestamp}_{self._chunk_counter:04d}"
|
||||
|
||||
filename = self.chunk_dir / f"chunk_{chunk_id}_{stage}.txt"
|
||||
|
||||
with open(filename, "w", encoding="utf-8") as f:
|
||||
if system_prompt and stage == "before":
|
||||
f.write("=== SYSTEM PROMPT ===\n")
|
||||
f.write(system_prompt)
|
||||
f.write("\n\n=== USER PROMPT ===\n")
|
||||
f.write(content)
|
||||
|
||||
logger.debug(f"Saved chunk: {filename}")
|
||||
return chunk_id
|
||||
|
||||
async def raw_chat_completion(self, system_prompt: str, user_prompt: str) -> str:
|
||||
"""Generic chat completion."""
|
||||
return await self._make_request(system_prompt, user_prompt)
|
||||
|
||||
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))
|
||||
async def _make_request(self, system_prompt: str, user_prompt: str) -> str:
|
||||
await self.rate_limiter.acquire()
|
||||
try:
|
||||
resp = await self.client.chat.completions.create(
|
||||
model=self.model,
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt}
|
||||
],
|
||||
temperature=0.3,
|
||||
)
|
||||
return resp.choices[0].message.content.strip()
|
||||
finally:
|
||||
self.rate_limiter.release()
|
||||
|
||||
async def close(self):
|
||||
await self.client.close()
|
||||
@@ -0,0 +1,74 @@
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
import json
|
||||
from src.common.data_model import ManifestEntry
|
||||
from src.common.utils import setup_logger
|
||||
|
||||
logger = setup_logger("manifest_manager")
|
||||
|
||||
class ManifestManager:
|
||||
"""
|
||||
Manages the translation manifest (Source of Truth).
|
||||
Handles persistence and state updates.
|
||||
"""
|
||||
|
||||
def __init__(self, manifest_path: Path):
|
||||
self.manifest_path = manifest_path
|
||||
self.entries: List[ManifestEntry] = []
|
||||
self._entries_map: Dict[str, ManifestEntry] = {}
|
||||
|
||||
def load(self):
|
||||
"""Loads manifest from disk if it exists."""
|
||||
if not self.manifest_path.exists():
|
||||
logger.info(f"Manifest not found at {self.manifest_path}, starting empty.")
|
||||
return
|
||||
|
||||
try:
|
||||
with open(self.manifest_path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
self.entries = [ManifestEntry.model_validate(item) for item in data]
|
||||
self._rebuild_map()
|
||||
logger.info(f"Loaded {len(self.entries)} entries from manifest.")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load manifest: {e}")
|
||||
raise
|
||||
|
||||
def save(self):
|
||||
"""Saves current state to disk."""
|
||||
try:
|
||||
# Pydantic v2: model_dump(mode='json') or just list dump
|
||||
data = [entry.model_dump(mode='json') for entry in self.entries]
|
||||
with open(self.manifest_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
logger.info(f"Saved {len(self.entries)} entries to manifest.")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save manifest: {e}")
|
||||
raise
|
||||
|
||||
def add_entries(self, new_entries: List[ManifestEntry]):
|
||||
"""
|
||||
Adds new entries to the manifest.
|
||||
If an entry with the same ID exists, it keeps the EXISTING one (to preserve translations).
|
||||
"""
|
||||
count = 0
|
||||
for entry in new_entries:
|
||||
if entry.entry_id not in self._entries_map:
|
||||
self.entries.append(entry)
|
||||
self._entries_map[entry.entry_id] = entry
|
||||
count += 1
|
||||
|
||||
if count > 0:
|
||||
logger.info(f"Added {count} new entries to manifest.")
|
||||
|
||||
def update_translation(self, entry_id: str, translation: str):
|
||||
"""Updates translation for a specific entry."""
|
||||
if entry_id in self._entries_map:
|
||||
self._entries_map[entry_id].translated_text = translation
|
||||
else:
|
||||
logger.warning(f"Attempted to update translation for unknown ID: {entry_id}")
|
||||
|
||||
def get_entry(self, entry_id: str) -> Optional[ManifestEntry]:
|
||||
return self._entries_map.get(entry_id)
|
||||
|
||||
def _rebuild_map(self):
|
||||
self._entries_map = {e.entry_id: e for e in self.entries}
|
||||
@@ -0,0 +1,286 @@
|
||||
"""
|
||||
Translator Module - Handles translation of ManifestEntry items.
|
||||
|
||||
Key features:
|
||||
- Character-based chunking (~5000 chars per chunk)
|
||||
- Chapter-aware grouping (chunks don't cross file boundaries)
|
||||
- Concurrent translation with asyncio.gather
|
||||
- Progress tracking and error handling
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from typing import List, Dict
|
||||
from collections import defaultdict
|
||||
from loguru import logger
|
||||
|
||||
from src.common.data_model import ManifestEntry, BookProfile
|
||||
from src.translation.llm_client import LLMClient
|
||||
from src.assembly.format_restorer import FormatRestorer
|
||||
from src.common.utils import setup_logger
|
||||
|
||||
logger = setup_logger("translator")
|
||||
|
||||
# Default chunk size in characters
|
||||
DEFAULT_CHUNK_SIZE = 5000
|
||||
# Maximum concurrent translations
|
||||
MAX_CONCURRENT = 5
|
||||
|
||||
|
||||
class Translator:
|
||||
"""
|
||||
Translates ManifestEntry items using LLM with chapter-aware chunking.
|
||||
Supports both sequential and concurrent translation modes.
|
||||
"""
|
||||
|
||||
def __init__(self, llm_client: LLMClient, chunk_size: int = DEFAULT_CHUNK_SIZE,
|
||||
max_concurrent: int = MAX_CONCURRENT):
|
||||
self.llm_client = llm_client
|
||||
self.chunk_size = chunk_size
|
||||
self.max_concurrent = max_concurrent
|
||||
self.restorer = FormatRestorer()
|
||||
|
||||
async def translate(self, entries: List[ManifestEntry], profile: BookProfile,
|
||||
concurrent: bool = True) -> List[ManifestEntry]:
|
||||
"""
|
||||
Translates all untranslated entries.
|
||||
|
||||
Args:
|
||||
entries: All manifest entries
|
||||
profile: Book profile with style guide
|
||||
concurrent: Use concurrent translation (default True)
|
||||
|
||||
Returns:
|
||||
The same entries list with translated_text populated
|
||||
"""
|
||||
untranslated = [e for e in entries if not e.translated_text]
|
||||
if not untranslated:
|
||||
logger.info("No new entries to translate.")
|
||||
return entries
|
||||
|
||||
logger.info(f"Found {len(untranslated)} entries to translate")
|
||||
|
||||
# Group by chapter (file_path)
|
||||
chapters = self._group_by_chapter(untranslated)
|
||||
logger.info(f"Grouped into {len(chapters)} chapters")
|
||||
|
||||
# Create all chunks
|
||||
all_chunks = []
|
||||
for file_path, chapter_entries in chapters.items():
|
||||
chapter_chunks = self._create_char_based_chunks(chapter_entries)
|
||||
for chunk in chapter_chunks:
|
||||
all_chunks.append((file_path, chunk))
|
||||
|
||||
total_chunks = len(all_chunks)
|
||||
logger.info(f"Created {total_chunks} chunks (avg ~{self.chunk_size} chars each)")
|
||||
|
||||
if concurrent:
|
||||
await self._translate_concurrent(all_chunks, profile)
|
||||
else:
|
||||
await self._translate_sequential(all_chunks, profile)
|
||||
|
||||
translated_count = sum(1 for e in entries if e.translated_text)
|
||||
logger.info(f"Translation complete: {translated_count}/{len(entries)} entries translated")
|
||||
return entries
|
||||
|
||||
async def _translate_concurrent(self, all_chunks: List, profile: BookProfile):
|
||||
"""Translate chunks concurrently with semaphore control."""
|
||||
semaphore = asyncio.Semaphore(self.max_concurrent)
|
||||
completed = [0] # Use list for mutable counter in closure
|
||||
total = len(all_chunks)
|
||||
success = [0]
|
||||
failed = [0]
|
||||
|
||||
async def translate_chunk_task(file_path: str, chunk: List[ManifestEntry], idx: int):
|
||||
async with semaphore:
|
||||
try:
|
||||
results = await self.llm_client.translate_chunk(
|
||||
chunk,
|
||||
instruction=profile.style_guide if profile else None,
|
||||
mode="bilingual"
|
||||
)
|
||||
|
||||
for entry in chunk:
|
||||
if entry.entry_id in results:
|
||||
entry.translated_text = results[entry.entry_id]
|
||||
success[0] += 1
|
||||
else:
|
||||
failed[0] += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Chunk {idx} failed: {e}")
|
||||
failed[0] += len(chunk)
|
||||
finally:
|
||||
completed[0] += 1
|
||||
if completed[0] % 5 == 0 or completed[0] == total:
|
||||
logger.info(f"Progress: {completed[0]}/{total} chunks ({success[0]} translated)")
|
||||
|
||||
tasks = [
|
||||
translate_chunk_task(file_path, chunk, i)
|
||||
for i, (file_path, chunk) in enumerate(all_chunks)
|
||||
]
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
logger.info(f"Concurrent translation: {success[0]} success, {failed[0]} failed")
|
||||
|
||||
async def _translate_sequential(self, all_chunks: List, profile: BookProfile):
|
||||
"""Translate chunks sequentially."""
|
||||
total_chunks = len(all_chunks)
|
||||
success_count = 0
|
||||
fail_count = 0
|
||||
|
||||
for idx, (file_path, chunk) in enumerate(all_chunks):
|
||||
try:
|
||||
results = await self.llm_client.translate_chunk(
|
||||
chunk,
|
||||
instruction=profile.style_guide if profile else None,
|
||||
mode="bilingual"
|
||||
)
|
||||
|
||||
for entry in chunk:
|
||||
if entry.entry_id in results:
|
||||
entry.translated_text = results[entry.entry_id]
|
||||
success_count += 1
|
||||
else:
|
||||
fail_count += 1
|
||||
logger.warning(f"Missing translation for: {entry.entry_id[-40:]}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Chunk {idx} translation failed: {e}")
|
||||
fail_count += len(chunk)
|
||||
|
||||
if (idx + 1) % 10 == 0 or idx + 1 == total_chunks:
|
||||
logger.info(f"Progress: {idx + 1}/{total_chunks} chunks ({success_count} entries translated)")
|
||||
|
||||
logger.info(f"Sequential translation: {success_count} success, {fail_count} failed")
|
||||
|
||||
|
||||
async def translate_chapter(self, entries: List[ManifestEntry], file_path: str,
|
||||
profile: BookProfile) -> Dict[str, int]:
|
||||
"""
|
||||
Translate a single chapter.
|
||||
|
||||
Args:
|
||||
entries: All entries (will filter by file_path)
|
||||
file_path: Chapter file path to translate
|
||||
profile: Book profile
|
||||
|
||||
Returns:
|
||||
Dict with 'success' and 'failed' counts
|
||||
"""
|
||||
chapter_entries = [e for e in entries if e.file_path == file_path and not e.translated_text]
|
||||
|
||||
if not chapter_entries:
|
||||
logger.info(f"Chapter {file_path} has no untranslated entries")
|
||||
return {"success": 0, "failed": 0}
|
||||
|
||||
logger.info(f"Translating chapter: {file_path} ({len(chapter_entries)} entries)")
|
||||
|
||||
chunks = self._create_char_based_chunks(chapter_entries)
|
||||
logger.info(f"Created {len(chunks)} chunks")
|
||||
|
||||
success = 0
|
||||
failed = 0
|
||||
|
||||
for i, chunk in enumerate(chunks, 1):
|
||||
chunk_chars = sum(len(e.original_text) for e in chunk)
|
||||
logger.debug(f"Chunk {i}/{len(chunks)}: {len(chunk)} entries, {chunk_chars} chars")
|
||||
|
||||
try:
|
||||
results = await self.llm_client.translate_chunk(
|
||||
chunk,
|
||||
instruction=profile.style_guide if profile else None,
|
||||
mode="bilingual"
|
||||
)
|
||||
|
||||
for entry in chunk:
|
||||
if entry.entry_id in results:
|
||||
entry.translated_text = results[entry.entry_id]
|
||||
|
||||
# Verify placeholder preservation
|
||||
if entry.placeholders:
|
||||
_, restored_ok = self.restorer.restore(
|
||||
entry.translated_text,
|
||||
entry.placeholders,
|
||||
context_id=entry.entry_id
|
||||
)
|
||||
if not restored_ok:
|
||||
logger.warning(f"Placeholder issue: {entry.entry_id[-40:]}")
|
||||
|
||||
success += 1
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Chunk {i} failed: {e}")
|
||||
failed += len(chunk)
|
||||
|
||||
logger.info(f"Chapter done: {success} success, {failed} failed")
|
||||
return {"success": success, "failed": failed}
|
||||
|
||||
def _group_by_chapter(self, entries: List[ManifestEntry]) -> Dict[str, List[ManifestEntry]]:
|
||||
"""Group entries by file_path (chapter)."""
|
||||
chapters = defaultdict(list)
|
||||
for entry in entries:
|
||||
chapters[entry.file_path].append(entry)
|
||||
return dict(chapters)
|
||||
|
||||
def _create_char_based_chunks(self, entries: List[ManifestEntry]) -> List[List[ManifestEntry]]:
|
||||
"""
|
||||
Create chunks based on character count.
|
||||
|
||||
Each chunk contains approximately self.chunk_size characters.
|
||||
Chunks never cross chapter boundaries (entries from same file only).
|
||||
"""
|
||||
chunks = []
|
||||
current_chunk = []
|
||||
current_size = 0
|
||||
|
||||
for entry in entries:
|
||||
text_len = len(entry.original_text)
|
||||
|
||||
# If adding this entry exceeds limit and we have content, start new chunk
|
||||
if current_size + text_len > self.chunk_size and current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
current_chunk = []
|
||||
current_size = 0
|
||||
|
||||
current_chunk.append(entry)
|
||||
current_size += text_len
|
||||
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
|
||||
return chunks
|
||||
|
||||
def get_chapter_stats(self, entries: List[ManifestEntry]) -> List[Dict]:
|
||||
"""
|
||||
Get statistics for each chapter.
|
||||
|
||||
Returns list of dicts with: file_path, total, translated, pending, chars
|
||||
"""
|
||||
chapters = self._group_by_chapter(entries)
|
||||
stats = []
|
||||
|
||||
for file_path, chapter_entries in sorted(chapters.items()):
|
||||
total = len(chapter_entries)
|
||||
translated = sum(1 for e in chapter_entries if e.translated_text)
|
||||
total_chars = sum(len(e.original_text) for e in chapter_entries)
|
||||
|
||||
# Get first text as title preview
|
||||
first_text = ""
|
||||
for e in chapter_entries:
|
||||
if e.original_text:
|
||||
first_text = e.original_text[:40].replace('\n', ' ')
|
||||
break
|
||||
|
||||
stats.append({
|
||||
"file_path": file_path,
|
||||
"title": first_text,
|
||||
"total": total,
|
||||
"translated": translated,
|
||||
"pending": total - translated,
|
||||
"chars": total_chars
|
||||
})
|
||||
|
||||
return stats
|
||||
|
||||
@@ -1,304 +0,0 @@
|
||||
"""
|
||||
EPUB Translator Core Module - v0.09 (TOC Selection Support)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import traceback
|
||||
from typing import List, Dict, Any
|
||||
from pathlib import Path
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TimeElapsedColumn
|
||||
|
||||
from .epub_parser import EPUBParser
|
||||
from .toc_parser import TOCParser
|
||||
from .llm_client import LLMClient
|
||||
from .text_processor import TextProcessor
|
||||
from .bilingual_builder import BilingualEPUBBuilder
|
||||
from .chinese_builder import ChineseEPUBBuilder
|
||||
from .manifest_manager import ManifestManager
|
||||
from .book_profiler import BookProfiler
|
||||
from .cache import TranslationCache
|
||||
from .format_restorer import FormatRestorer
|
||||
from .utils import add_spacing_between_cn_and_en_num
|
||||
|
||||
|
||||
class EPUBTranslator:
|
||||
|
||||
def __init__(self, config: Dict, use_cache: bool = True):
|
||||
self.config = config
|
||||
self.console = Console()
|
||||
self.use_cache = use_cache
|
||||
|
||||
self.parser = None
|
||||
self.llm_client = LLMClient(config)
|
||||
self.text_processor = TextProcessor(config)
|
||||
self.profiler = BookProfiler(config, self.llm_client)
|
||||
self.cache = TranslationCache(config) if use_cache else None
|
||||
self.restorer = FormatRestorer()
|
||||
|
||||
self.manifest_dir = Path("cache/manifests")
|
||||
self.manifest_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
async def translate_epub(self, epub_path: str, test_mode: bool = False,
|
||||
output_dir: str = None, mode: str = "bilingual",
|
||||
from_chapter: str = None, to_chapter: str = None) -> str:
|
||||
"""
|
||||
翻译 EPUB 文件
|
||||
|
||||
Args:
|
||||
epub_path: EPUB 文件路径
|
||||
test_mode: 测试模式（只翻译前几块）
|
||||
output_dir: 输出目录
|
||||
mode: 翻译模式 - "bilingual" (双语) 或 "chinese" (纯中文)
|
||||
from_chapter: 起始章节标题
|
||||
to_chapter: 结束章节标题
|
||||
"""
|
||||
try:
|
||||
epub_path = Path(epub_path)
|
||||
self.parser = EPUBParser(str(epub_path))
|
||||
|
||||
# 0. 解析 TOC 并处理章节范围
|
||||
toc_parser = TOCParser(self.parser.book)
|
||||
include_files = None
|
||||
chapter_range_info = None
|
||||
|
||||
if from_chapter or to_chapter:
|
||||
# 用户手动指定了章节范围 - 使用 Spine 范围
|
||||
include_files, selected_items = toc_parser.get_spine_range(
|
||||
start_title=from_chapter,
|
||||
end_title=to_chapter
|
||||
)
|
||||
if selected_items:
|
||||
start_title = selected_items[0].title
|
||||
end_title = selected_items[-1].title
|
||||
self.console.print(
|
||||
f"[cyan]📚 选择翻译范围: {start_title} ~ {end_title}[/cyan]"
|
||||
)
|
||||
self.console.print(
|
||||
f"[cyan] 包含 {len(include_files)} 个 HTML 文件 (基于 Spine)[/cyan]"
|
||||
)
|
||||
chapter_range_info = {
|
||||
"start_title": start_title,
|
||||
"end_title": end_title,
|
||||
"included_files": list(include_files)
|
||||
}
|
||||
else:
|
||||
# 未指定范围时，自动跳过前置/后置部分 - 使用 Spine 范围
|
||||
skip_files = toc_parser.get_skip_files()
|
||||
if skip_files:
|
||||
# 使用基于 Spine 的正文文件获取
|
||||
include_files = toc_parser.get_content_files_from_spine()
|
||||
skip_count = len(skip_files)
|
||||
content_count = len(include_files)
|
||||
self.console.print(
|
||||
f"[cyan]📚 智能跳过: {skip_count} 个非正文章节 (Cover, Copyright, Index 等)[/cyan]"
|
||||
)
|
||||
self.console.print(
|
||||
f"[cyan] 翻译正文: {content_count} 个 HTML 文件 (基于 Spine)[/cyan]"
|
||||
)
|
||||
|
||||
# 1. Manifest - 根据 mode 使用不同的 manifest 文件
|
||||
manifest_suffix = "_chinese" if mode == "chinese" else ""
|
||||
manifest_path = self.manifest_dir / f"{epub_path.stem}{manifest_suffix}_manifest.json"
|
||||
manifest = ManifestManager(str(manifest_path))
|
||||
|
||||
if not manifest.load() or not self.use_cache:
|
||||
self.console.print(f"[yellow]Initializing Manifest (Mode: {mode})...[/yellow]")
|
||||
manifest.init_manifest(
|
||||
book_id=epub_path.name,
|
||||
metadata=self.parser.get_book_info(),
|
||||
chapter_range=chapter_range_info
|
||||
)
|
||||
# 传递文件过滤参数
|
||||
content_items = self.parser.extract_all_content_items(include_files=include_files)
|
||||
for item in content_items:
|
||||
# 传递 mode 参数
|
||||
self.text_processor.extract_to_manifest(
|
||||
item['content'], item['file_name'], manifest, mode=mode
|
||||
)
|
||||
manifest.save()
|
||||
else:
|
||||
# 加载已有 Manifest，检查章节范围是否一致
|
||||
saved_range = manifest.get_chapter_range()
|
||||
if saved_range and saved_range.get("start_title"):
|
||||
self.console.print(
|
||||
f"[cyan]📚 翻译范围: {saved_range.get('start_title')} ~ {saved_range.get('end_title')}[/cyan]"
|
||||
)
|
||||
|
||||
stats = manifest.stats
|
||||
self.console.print(f"[green]Manifest loaded: {stats['total']} paragraphs[/green]")
|
||||
if stats['pending'] < stats['total'] and stats['translated'] > 0:
|
||||
self.console.print(
|
||||
f"[yellow]Detected incomplete translation task, completed {stats['translated']}/{stats['total']}, "
|
||||
f"continuing translation of remaining {stats['pending']} paragraphs...[/yellow]"
|
||||
)
|
||||
|
||||
# 2. Profile (Glossary)
|
||||
profile = {}
|
||||
if not test_mode:
|
||||
self.console.print("[yellow]Generating Book Profile...[/yellow]")
|
||||
profile = await self.profiler.analyze_book(manifest)
|
||||
self.console.print(f"Genre: {profile.get('genre')} | Style: {profile.get('style')}")
|
||||
|
||||
# 3. Translate - 传递 mode 参数
|
||||
chunks = self.text_processor.create_chunks_from_manifest(manifest, mode=mode)
|
||||
|
||||
if test_mode:
|
||||
self.console.print("[yellow]Test mode enabled: Translating only first 10 chunks...[/yellow]")
|
||||
chunks = chunks[:10]
|
||||
|
||||
if chunks:
|
||||
await self._translate_concurrently(chunks, manifest, profile, mode=mode)
|
||||
|
||||
# 4. Build - 根据 mode 选择正确的 Builder
|
||||
self.console.print(f"\n[yellow]Building {mode} EPUB...[/yellow]")
|
||||
output_path = output_dir or self.config['output']['output_dir']
|
||||
|
||||
if mode == "chinese":
|
||||
builder = ChineseEPUBBuilder(self.parser.book, self.config)
|
||||
result_file = builder.create_chinese_epub_with_mapping(
|
||||
manifest.get_items(), output_path
|
||||
)
|
||||
else:
|
||||
builder = BilingualEPUBBuilder(self.parser.book, self.config)
|
||||
translation_map = {item.global_id: item.translation for item in manifest.get_items() if item.translation}
|
||||
paragraph_map = {item.global_id: {
|
||||
"file_name": item.source_file,
|
||||
"text": item.clean_text,
|
||||
"html_element": item.original_html,
|
||||
"tag_attrs": item.tag_attrs
|
||||
} for item in manifest.get_items()}
|
||||
|
||||
result_file = builder.create_bilingual_epub_with_mapping(
|
||||
translation_map, paragraph_map, output_path
|
||||
)
|
||||
|
||||
final_stats = manifest.stats
|
||||
self.console.print(f"""
|
||||
[green]✅ Translation complete![/green]
|
||||
- Mode: {mode}
|
||||
- Total Paragraphs: {final_stats['total']}
|
||||
- Successfully Translated: {final_stats['translated']}
|
||||
- Skipped: {final_stats['ignored']}
|
||||
- Failed: {final_stats['failed']}
|
||||
- Output File: {result_file}
|
||||
""")
|
||||
return result_file
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
logger.error(f"Translation flow failed: {e}")
|
||||
raise
|
||||
|
||||
async def _translate_concurrently(self, chunks: List[List[Any]], manifest: ManifestManager,
|
||||
profile: Dict, mode: str = "bilingual"):
|
||||
total_chunks = len(chunks)
|
||||
glossary = profile.get('glossary', {})
|
||||
instruction = profile.get('translation_instruction', "")
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
TimeElapsedColumn(),
|
||||
console=self.console
|
||||
) as progress:
|
||||
task_id = progress.add_task(f"[cyan]Translating ({mode})...", total=total_chunks)
|
||||
|
||||
async def worker(chunk, idx):
|
||||
try:
|
||||
model_name = self.llm_client.models.get('fast')
|
||||
chunk_dicts = [item.to_dict() for item in chunk]
|
||||
results = None
|
||||
|
||||
if self.cache:
|
||||
results = self.cache.get_chunk_translation(chunk_dicts, model=model_name)
|
||||
|
||||
if not results:
|
||||
# 传递 mode 参数给 LLM
|
||||
results = await self.llm_client.translate_chunk(
|
||||
chunk,
|
||||
glossary=glossary,
|
||||
instruction=instruction,
|
||||
mode=mode
|
||||
)
|
||||
if self.cache and results:
|
||||
self.cache.save_chunk_translation(chunk_dicts, results, model=model_name)
|
||||
|
||||
for item in chunk:
|
||||
if item.global_id in results:
|
||||
raw_translation = results[item.global_id]
|
||||
|
||||
# 错误检测：如果翻译结果包含错误标记，视为失败
|
||||
if raw_translation.startswith("[Error") or "Error -" in raw_translation:
|
||||
logger.warning(f"Translation error for {item.global_id}: {raw_translation}")
|
||||
manifest.update_item(item.global_id, None, status="failed", error=raw_translation)
|
||||
continue
|
||||
|
||||
# 格式化翻译文本（盘古之白）
|
||||
processed_translation = add_spacing_between_cn_and_en_num(raw_translation)
|
||||
|
||||
if mode == "chinese":
|
||||
# 检查是否有内嵌占位符（排除 _prefix, _suffix）
|
||||
inner_placeholders = {k: v for k, v in item.placeholder_map.items()
|
||||
if not k.startswith("_")} if item.placeholder_map else {}
|
||||
|
||||
if not inner_placeholders:
|
||||
# 没有内嵌占位符，直接使用译文（清除 LLM 可能虚构的占位符）
|
||||
clean_translation = self.restorer._strip_placeholders(processed_translation)
|
||||
item.translation_with_placeholders = clean_translation
|
||||
# 还原时只添加前缀后缀
|
||||
restored_html, success = self.restorer.restore(
|
||||
clean_translation,
|
||||
item.placeholder_map
|
||||
)
|
||||
else:
|
||||
# 有内嵌占位符，正常还原
|
||||
item.translation_with_placeholders = processed_translation
|
||||
restored_html, success = self.restorer.restore(
|
||||
processed_translation,
|
||||
item.placeholder_map
|
||||
)
|
||||
|
||||
# 格式修复逻辑
|
||||
if not success:
|
||||
logger.warning(f"格式丢失 (ID: {item.global_id})，尝试自动修复...")
|
||||
try:
|
||||
fixed_translation = await self.llm_client.repair_format(
|
||||
item.text_with_placeholders,
|
||||
processed_translation
|
||||
)
|
||||
restored_html_2, success_2 = self.restorer.restore(
|
||||
fixed_translation,
|
||||
item.placeholder_map
|
||||
)
|
||||
|
||||
if success_2:
|
||||
logger.info(f"格式修复成功! (ID: {item.global_id})")
|
||||
restored_html = restored_html_2
|
||||
item.translation_with_placeholders = fixed_translation
|
||||
else:
|
||||
logger.error(f"格式修复失败 (ID: {item.global_id})，保留原始译文")
|
||||
except Exception as e:
|
||||
logger.error(f"修复过程出错: {e}")
|
||||
|
||||
item.translation_with_original_html = restored_html
|
||||
# translation 字段存储纯文本
|
||||
item.translation = self.restorer._strip_placeholders(item.translation_with_placeholders)
|
||||
manifest.update_item(item.global_id, item.translation)
|
||||
else:
|
||||
|
||||
# 双语模式
|
||||
item.translation = processed_translation
|
||||
manifest.update_item(item.global_id, item.translation)
|
||||
else:
|
||||
manifest.update_item(item.global_id, None, status="failed", error="Missing")
|
||||
manifest.save()
|
||||
except Exception as e:
|
||||
logger.error(f"Chunk {idx} failed: {e}")
|
||||
finally:
|
||||
progress.update(task_id, advance=1)
|
||||
|
||||
tasks = [worker(chunk, i) for i, chunk in enumerate(chunks)]
|
||||
await asyncio.gather(*tasks)
|
||||
-208
@@ -1,208 +0,0 @@
|
||||
"""
|
||||
工具函数模块
|
||||
提供配置加载、日志设置等通用功能
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any
|
||||
from loguru import logger
|
||||
import sys
|
||||
from dotenv import load_dotenv
|
||||
|
||||
|
||||
def load_config(config_path: str = "config/config.json") -> Dict[str, Any]:
|
||||
"""
|
||||
加载配置文件
|
||||
|
||||
Args:
|
||||
config_path: 配置文件路径
|
||||
|
||||
Returns:
|
||||
配置字典
|
||||
"""
|
||||
# 加载 .env 文件
|
||||
load_dotenv()
|
||||
|
||||
try:
|
||||
with open(config_path, 'r', encoding='utf-8') as f:
|
||||
config = json.load(f)
|
||||
|
||||
# 从环境变量获取 API Key
|
||||
for provider_name, provider_config in config.get('providers', {}).items():
|
||||
env_key = f"{provider_name.upper()}_API_KEY"
|
||||
if env_key in os.environ:
|
||||
provider_config['api_key'] = os.environ[env_key]
|
||||
|
||||
return config
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError(f"配置文件未找到: {config_path}")
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f"配置文件格式错误: {e}")
|
||||
|
||||
|
||||
def load_prompts(prompts_path: str = "config/prompts.json") -> Dict[str, str]:
|
||||
"""
|
||||
加载提示词模板
|
||||
|
||||
Args:
|
||||
prompts_path: 提示词文件路径
|
||||
|
||||
Returns:
|
||||
提示词字典
|
||||
"""
|
||||
try:
|
||||
with open(prompts_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError(f"提示词文件未找到: {prompts_path}")
|
||||
|
||||
|
||||
def setup_logging(config: Dict[str, Any]) -> None:
|
||||
"""
|
||||
设置日志配置
|
||||
|
||||
Args:
|
||||
config: 配置字典
|
||||
"""
|
||||
log_config = config.get('logging', {})
|
||||
|
||||
# 移除默认处理器
|
||||
logger.remove()
|
||||
|
||||
# 添加控制台输出
|
||||
logger.add(
|
||||
sys.stdout,
|
||||
level=log_config.get('level', 'INFO'),
|
||||
format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>"
|
||||
)
|
||||
|
||||
# 添加文件输出
|
||||
if 'file' in log_config:
|
||||
log_file = log_config['file']
|
||||
# 确保日志目录存在
|
||||
Path(log_file).parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.add(
|
||||
log_file,
|
||||
level=log_config.get('level', 'INFO'),
|
||||
rotation=log_config.get('rotation', '10 MB'),
|
||||
retention=log_config.get('retention', '7 days'),
|
||||
encoding='utf-8',
|
||||
format="{time:YYYY-MM-DD HH:mm:ss} | {level: <8} | {name}:{function}:{line} - {message}"
|
||||
)
|
||||
|
||||
|
||||
def ensure_output_dir(output_dir: str) -> Path:
|
||||
"""
|
||||
确保输出目录存在
|
||||
|
||||
Args:
|
||||
output_dir: 输出目录路径
|
||||
|
||||
Returns:
|
||||
输出目录的 Path 对象
|
||||
"""
|
||||
output_path = Path(output_dir)
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
return output_path
|
||||
|
||||
|
||||
def sanitize_filename(filename: str) -> str:
|
||||
"""
|
||||
清理文件名，移除非法字符
|
||||
|
||||
Args:
|
||||
filename: 原始文件名
|
||||
|
||||
Returns:
|
||||
清理后的文件名
|
||||
"""
|
||||
import re
|
||||
# 移除或替换非法字符
|
||||
filename = re.sub(r'[<>:"/\\|?*]', '_', filename)
|
||||
# 移除多余的空格和点
|
||||
filename = re.sub(r'\s+', ' ', filename).strip('. ')
|
||||
return filename
|
||||
|
||||
|
||||
def format_file_size(size_bytes: int) -> str:
|
||||
"""
|
||||
格式化文件大小显示
|
||||
|
||||
Args:
|
||||
size_bytes: 字节数
|
||||
|
||||
Returns:
|
||||
格式化的大小字符串
|
||||
"""
|
||||
if size_bytes == 0:
|
||||
return "0B"
|
||||
|
||||
size_names = ["B", "KB", "MB", "GB"]
|
||||
import math
|
||||
i = int(math.floor(math.log(size_bytes, 1024)))
|
||||
p = math.pow(1024, i)
|
||||
s = round(size_bytes / p, 2)
|
||||
return f"{s} {size_names[i]}"
|
||||
|
||||
|
||||
def estimate_tokens(text: str) -> int:
|
||||
"""
|
||||
估算文本的 token 数量
|
||||
|
||||
Args:
|
||||
text: 输入文本
|
||||
|
||||
Returns:
|
||||
估算的 token 数量
|
||||
"""
|
||||
# 简单估算：英文约 4 字符/token，中文约 1.5 字符/token
|
||||
import re
|
||||
|
||||
# 分离中英文
|
||||
chinese_chars = len(re.findall(r'[\u4e00-\u9fff]', text))
|
||||
other_chars = len(text) - chinese_chars
|
||||
|
||||
# 估算 tokens
|
||||
estimated_tokens = chinese_chars / 1.5 + other_chars / 4
|
||||
return int(estimated_tokens)
|
||||
|
||||
|
||||
def truncate_text(text: str, max_length: int = 100) -> str:
|
||||
"""
|
||||
截断文本用于显示
|
||||
|
||||
Args:
|
||||
text: 原始文本
|
||||
max_length: 最大长度
|
||||
|
||||
Returns:
|
||||
截断后的文本
|
||||
"""
|
||||
if len(text) <= max_length:
|
||||
return text
|
||||
return text[:max_length-3] + "..."
|
||||
|
||||
|
||||
def add_spacing_between_cn_and_en_num(text: str) -> str:
|
||||
"""
|
||||
在中文和英文/数字之间添加空格（盘古之白）
|
||||
|
||||
Args:
|
||||
text: 原始文本
|
||||
|
||||
Returns:
|
||||
处理后的文本
|
||||
"""
|
||||
import re
|
||||
if not text:
|
||||
return text
|
||||
|
||||
# 中文-英文/数字
|
||||
text = re.sub(r'([\u4e00-\u9fff])([a-zA-Z0-9])', r'\1 \2', text)
|
||||
# 英文/数字-中文
|
||||
text = re.sub(r'([a-zA-Z0-9])([\u4e00-\u9fff])', r'\1 \2', text)
|
||||
|
||||
return text
|
||||
Reference in New Issue
Block a user