160 lines
5.9 KiB
Python
160 lines
5.9 KiB
Python
"""
|
||
文本处理器模块 (Text Processor Module) - Manifest 驱动版
|
||
|
||
该模块专注于 HTML 文档的遍历和段落提取。
|
||
它不再维护全局状态，而是将提取的内容注册到 ManifestManager 中。
|
||
"""
|
||
|
||
import re
|
||
from bs4 import BeautifulSoup
|
||
from typing import List, Dict, Any
|
||
from loguru import logger
|
||
from .manifest_manager import ManifestManager
|
||
|
||
|
||
class TextProcessor:
|
||
"""
|
||
负责从 HTML 中识别有效段落并进行清洗。
|
||
"""
|
||
|
||
def __init__(self, config: Dict):
|
||
"""
|
||
Args:
|
||
config (Dict): 全局配置。
|
||
"""
|
||
self.config = config
|
||
self.chunk_size = config['translation'].get('chunk_size', 5000)
|
||
|
||
def extract_to_manifest(self, html_content: str, source_file: str, manifest: ManifestManager):
|
||
"""
|
||
解析 HTML 内容，并将识别出的段落注册到 Manifest 中。
|
||
|
||
Args:
|
||
html_content (str): HTML 源码。
|
||
source_file (str): 来源文件名。
|
||
manifest (ManifestManager): 清单管理器实例。
|
||
"""
|
||
try:
|
||
soup = BeautifulSoup(html_content, 'html.parser')
|
||
|
||
# 1. 移除不需要的元素
|
||
for element in soup(['script', 'style', 'meta', 'link']):
|
||
element.decompose()
|
||
|
||
# 2. 获取有效的文本元素 (使用静态过滤逻辑)
|
||
text_elements = self.get_valid_text_elements(soup)
|
||
|
||
# 3. 注册到 Manifest
|
||
for element in text_elements:
|
||
clean_text = self.clean_element_text(element)
|
||
|
||
# 过滤逻辑
|
||
if not clean_text:
|
||
continue
|
||
|
||
status = "pending"
|
||
# 如果是导航元素，标记为 ignored
|
||
if self.is_navigation_element(element):
|
||
status = "ignored"
|
||
|
||
# 注册
|
||
item = manifest.add_item(
|
||
source_file=source_file,
|
||
original_html=str(element),
|
||
clean_text=clean_text,
|
||
tag=element.name,
|
||
metadata={"status": status} # 临时传递给 manifest
|
||
)
|
||
|
||
# 同步更新 manifest 状态 (如果需要过滤)
|
||
if status == "ignored":
|
||
manifest.update_item(item.global_id, translation=None, status="ignored")
|
||
|
||
except Exception as e:
|
||
logger.error(f"从 {source_file} 提取段落失败: {e}")
|
||
|
||
@staticmethod
|
||
def get_valid_text_elements(soup) -> List:
|
||
"""获取不含嵌套子块的叶子级文本容器元素。"""
|
||
tags = ['p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote', 'li', 'td']
|
||
all_candidates = soup.find_all(tags)
|
||
candidate_set = set(all_candidates)
|
||
|
||
final_elements = []
|
||
for element in all_candidates:
|
||
# 如果包含其他候选标签，说明是容器，跳过
|
||
if any(d in candidate_set for d in element.find_all(tags)):
|
||
continue
|
||
final_elements.append(element)
|
||
return final_elements
|
||
|
||
@staticmethod
|
||
def clean_element_text(element) -> str:
|
||
"""清理 HTML 元素，提取纯净的待翻译文本。"""
|
||
element_copy = element.__copy__()
|
||
|
||
# 移除脚注引用等
|
||
for tag in element_copy.find_all(['sup', 'sub']):
|
||
tag.decompose()
|
||
|
||
footnote_patterns = re.compile(r'footnote|endnote|reference|note|super|sub', re.I)
|
||
for tag in element_copy.find_all(['a', 'span', 'div'], class_=footnote_patterns):
|
||
tag.decompose()
|
||
|
||
# 移除仅包含数字的 span
|
||
for tag in element_copy.find_all('span'):
|
||
if re.match(r'^(\[\d+\]|\(\d+\)|\d+)$', tag.get_text().strip()):
|
||
tag.decompose()
|
||
|
||
text = element_copy.get_text().strip()
|
||
# 正则清理残留引用标识 (如 sentence.2)
|
||
text = re.sub(r'(\.|。|，|,)\s*(\[\d+\]|\d+)(?=\s|$)', r'\1', text)
|
||
text = re.sub(r'\s+', ' ', text)
|
||
return text
|
||
|
||
@staticmethod
|
||
def is_navigation_element(element) -> bool:
|
||
"""判断是否是无翻译价值的导航、页码元素。"""
|
||
classes = element.get('class', [])
|
||
nav_classes = ['nav', 'navigation', 'toc', 'menu', 'header', 'footer', 'page-number']
|
||
class_str = ' '.join(classes).lower() if isinstance(classes, list) else str(classes).lower()
|
||
|
||
if any(nc in class_str for nc in nav_classes):
|
||
return True
|
||
|
||
# 检查父级
|
||
parent = element.parent
|
||
if parent:
|
||
p_classes = parent.get('class', [])
|
||
p_class_str = ' '.join(p_classes).lower() if isinstance(p_classes, list) else str(p_classes).lower()
|
||
if any(nc in p_class_str for nc in nav_classes):
|
||
return True
|
||
return False
|
||
|
||
def create_chunks_from_manifest(self, manifest: ManifestManager) -> List[List[Any]]:
|
||
"""
|
||
从 Manifest 中筛选待翻译项目并分块。
|
||
"""
|
||
pending_items = manifest.get_items(status="pending")
|
||
if not pending_items:
|
||
return []
|
||
|
||
chunks = []
|
||
current_chunk = []
|
||
current_size = 0
|
||
|
||
for item in pending_items:
|
||
text_len = len(item.clean_text)
|
||
if current_size + text_len > self.chunk_size and current_chunk:
|
||
chunks.append(current_chunk)
|
||
current_chunk = []
|
||
current_size = 0
|
||
|
||
current_chunk.append(item)
|
||
current_size += text_len
|
||
|
||
if current_chunk:
|
||
chunks.append(current_chunk)
|
||
|
||
logger.info(f"分块完成: 共有 {len(pending_items)} 个待翻译项，分为 {len(chunks)} 个块")
|
||
return chunks |