Initial commit

2026-01-19 09:51:07 +08:00
commit 9ef82393be
174 changed files with 22285 additions and 0 deletions
@@ -0,0 +1,160 @@
+"""
+文本处理器模块 (Text Processor Module) - Manifest 驱动版
+
+该模块专注于 HTML 文档的遍历和段落提取。
+它不再维护全局状态，而是将提取的内容注册到 ManifestManager 中。
+"""
+
+import re
+from bs4 import BeautifulSoup
+from typing import List, Dict, Any
+from loguru import logger
+from .manifest_manager import ManifestManager
+
+
+class TextProcessor:
+    """
+    负责从 HTML 中识别有效段落并进行清洗。
+    """
+    
+    def __init__(self, config: Dict):
+        """
+        Args:
+            config (Dict): 全局配置。
+        """
+        self.config = config
+        self.chunk_size = config['translation'].get('chunk_size', 5000)
+        
+    def extract_to_manifest(self, html_content: str, source_file: str, manifest: ManifestManager):
+        """
+        解析 HTML 内容，并将识别出的段落注册到 Manifest 中。
+
+        Args:
+            html_content (str): HTML 源码。
+            source_file (str): 来源文件名。
+            manifest (ManifestManager): 清单管理器实例。
+        """
+        try:
+            soup = BeautifulSoup(html_content, 'html.parser')
+            
+            # 1. 移除不需要的元素
+            for element in soup(['script', 'style', 'meta', 'link']):
+                element.decompose()
+            
+            # 2. 获取有效的文本元素 (使用静态过滤逻辑)
+            text_elements = self.get_valid_text_elements(soup)
+            
+            # 3. 注册到 Manifest
+            for element in text_elements:
+                clean_text = self.clean_element_text(element)
+                
+                # 过滤逻辑
+                if not clean_text:
+                    continue
+                
+                status = "pending"
+                # 如果是导航元素，标记为 ignored
+                if self.is_navigation_element(element):
+                    status = "ignored"
+                
+                # 注册
+                item = manifest.add_item(
+                    source_file=source_file,
+                    original_html=str(element),
+                    clean_text=clean_text,
+                    tag=element.name,
+                    metadata={"status": status} # 临时传递给 manifest
+                )
+                
+                # 同步更新 manifest 状态 (如果需要过滤)
+                if status == "ignored":
+                    manifest.update_item(item.global_id, translation=None, status="ignored")
+
+        except Exception as e:
+            logger.error(f"从 {source_file} 提取段落失败: {e}")
+
+    @staticmethod
+    def get_valid_text_elements(soup) -> List:
+        """获取不含嵌套子块的叶子级文本容器元素。"""
+        tags = ['p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote', 'li', 'td']
+        all_candidates = soup.find_all(tags)
+        candidate_set = set(all_candidates)
+        
+        final_elements = []
+        for element in all_candidates:
+            # 如果包含其他候选标签，说明是容器，跳过
+            if any(d in candidate_set for d in element.find_all(tags)):
+                continue
+            final_elements.append(element)
+        return final_elements
+
+    @staticmethod
+    def clean_element_text(element) -> str:
+        """清理 HTML 元素，提取纯净的待翻译文本。"""
+        element_copy = element.__copy__()
+        
+        # 移除脚注引用等
+        for tag in element_copy.find_all(['sup', 'sub']):
+            tag.decompose()
+            
+        footnote_patterns = re.compile(r'footnote|endnote|reference|note|super|sub', re.I)
+        for tag in element_copy.find_all(['a', 'span', 'div'], class_=footnote_patterns):
+            tag.decompose()
+
+        # 移除仅包含数字的 span
+        for tag in element_copy.find_all('span'):
+            if re.match(r'^(\[\d+\]|\(\d+\)|\d+)$', tag.get_text().strip()):
+                tag.decompose()
+        
+        text = element_copy.get_text().strip()
+        # 正则清理残留引用标识 (如 sentence.2)
+        text = re.sub(r'(\.|。|，|,)\s*(\[\d+\]|\d+)(?=\s|$)', r'\1', text)
+        text = re.sub(r'\s+', ' ', text)
+        return text
+
+    @staticmethod
+    def is_navigation_element(element) -> bool:
+        """判断是否是无翻译价值的导航、页码元素。"""
+        classes = element.get('class', [])
+        nav_classes = ['nav', 'navigation', 'toc', 'menu', 'header', 'footer', 'page-number']
+        class_str = ' '.join(classes).lower() if isinstance(classes, list) else str(classes).lower()
+        
+        if any(nc in class_str for nc in nav_classes):
+            return True
+            
+        # 检查父级
+        parent = element.parent
+        if parent:
+            p_classes = parent.get('class', [])
+            p_class_str = ' '.join(p_classes).lower() if isinstance(p_classes, list) else str(p_classes).lower()
+            if any(nc in p_class_str for nc in nav_classes):
+                return True
+        return False
+
+    def create_chunks_from_manifest(self, manifest: ManifestManager) -> List[List[Any]]:
+        """
+        从 Manifest 中筛选待翻译项目并分块。
+        """
+        pending_items = manifest.get_items(status="pending")
+        if not pending_items:
+            return []
+            
+        chunks = []
+        current_chunk = []
+        current_size = 0
+        
+        for item in pending_items:
+            text_len = len(item.clean_text)
+            if current_size + text_len > self.chunk_size and current_chunk:
+                chunks.append(current_chunk)
+                current_chunk = []
+                current_size = 0
+            
+            current_chunk.append(item)
+            current_size += text_len
+            
+        if current_chunk:
+            chunks.append(current_chunk)
+            
+        logger.info(f"分块完成: 共有 {len(pending_items)} 个待翻译项，分为 {len(chunks)} 个块")
+        return chunks