""" 文本处理器模块 (Text Processor Module) - Manifest 驱动版 该模块专注于 HTML 文档的遍历和段落提取。 它不再维护全局状态,而是将提取的内容注册到 ManifestManager 中。 """ import re from bs4 import BeautifulSoup from typing import List, Dict, Any from loguru import logger from .manifest_manager import ManifestManager class TextProcessor: """ 负责从 HTML 中识别有效段落并进行清洗。 """ def __init__(self, config: Dict): """ Args: config (Dict): 全局配置。 """ self.config = config self.chunk_size = config['translation'].get('chunk_size', 5000) def extract_to_manifest(self, html_content: str, source_file: str, manifest: ManifestManager): """ 解析 HTML 内容,并将识别出的段落注册到 Manifest 中。 Args: html_content (str): HTML 源码。 source_file (str): 来源文件名。 manifest (ManifestManager): 清单管理器实例。 """ try: soup = BeautifulSoup(html_content, 'html.parser') # 1. 移除不需要的元素 for element in soup(['script', 'style', 'meta', 'link']): element.decompose() # 2. 获取有效的文本元素 (使用静态过滤逻辑) text_elements = self.get_valid_text_elements(soup) # 3. 注册到 Manifest for element in text_elements: clean_text = self.clean_element_text(element) # 过滤逻辑 if not clean_text: continue status = "pending" # 如果是导航元素,标记为 ignored if self.is_navigation_element(element): status = "ignored" # 注册 item = manifest.add_item( source_file=source_file, original_html=str(element), clean_text=clean_text, tag=element.name, metadata={"status": status} # 临时传递给 manifest ) # 同步更新 manifest 状态 (如果需要过滤) if status == "ignored": manifest.update_item(item.global_id, translation=None, status="ignored") except Exception as e: logger.error(f"从 {source_file} 提取段落失败: {e}") @staticmethod def get_valid_text_elements(soup) -> List: """获取不含嵌套子块的叶子级文本容器元素。""" tags = ['p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote', 'li', 'td'] all_candidates = soup.find_all(tags) candidate_set = set(all_candidates) final_elements = [] for element in all_candidates: # 如果包含其他候选标签,说明是容器,跳过 if any(d in candidate_set for d in element.find_all(tags)): continue final_elements.append(element) return final_elements @staticmethod def clean_element_text(element) -> str: """清理 HTML 元素,提取纯净的待翻译文本。""" element_copy = element.__copy__() # 移除脚注引用等 for tag in element_copy.find_all(['sup', 'sub']): tag.decompose() footnote_patterns = re.compile(r'footnote|endnote|reference|note|super|sub', re.I) for tag in element_copy.find_all(['a', 'span', 'div'], class_=footnote_patterns): tag.decompose() # 移除仅包含数字的 span for tag in element_copy.find_all('span'): if re.match(r'^(\[\d+\]|\(\d+\)|\d+)$', tag.get_text().strip()): tag.decompose() text = element_copy.get_text().strip() # 正则清理残留引用标识 (如 sentence.2) text = re.sub(r'(\.|。|,|,)\s*(\[\d+\]|\d+)(?=\s|$)', r'\1', text) text = re.sub(r'\s+', ' ', text) return text @staticmethod def is_navigation_element(element) -> bool: """判断是否是无翻译价值的导航、页码元素。""" classes = element.get('class', []) nav_classes = ['nav', 'navigation', 'toc', 'menu', 'header', 'footer', 'page-number'] class_str = ' '.join(classes).lower() if isinstance(classes, list) else str(classes).lower() if any(nc in class_str for nc in nav_classes): return True # 检查父级 parent = element.parent if parent: p_classes = parent.get('class', []) p_class_str = ' '.join(p_classes).lower() if isinstance(p_classes, list) else str(p_classes).lower() if any(nc in p_class_str for nc in nav_classes): return True return False def create_chunks_from_manifest(self, manifest: ManifestManager) -> List[List[Any]]: """ 从 Manifest 中筛选待翻译项目并分块。 """ pending_items = manifest.get_items(status="pending") if not pending_items: return [] chunks = [] current_chunk = [] current_size = 0 for item in pending_items: text_len = len(item.clean_text) if current_size + text_len > self.chunk_size and current_chunk: chunks.append(current_chunk) current_chunk = [] current_size = 0 current_chunk.append(item) current_size += text_len if current_chunk: chunks.append(current_chunk) logger.info(f"分块完成: 共有 {len(pending_items)} 个待翻译项,分为 {len(chunks)} 个块") return chunks