344 lines
11 KiB
Python
344 lines
11 KiB
Python
"""
|
||
文本处理器模块 - 重构版
|
||
实现全局编号系统，确保段落精确对应 + 清理HTML标签
|
||
"""
|
||
|
||
import re
|
||
from bs4 import BeautifulSoup
|
||
from typing import List, Dict
|
||
from loguru import logger
|
||
|
||
|
||
class TextProcessor:
|
||
"""文本处理器 - 简化版，专注核心功能"""
|
||
|
||
def __init__(self, config: Dict):
|
||
"""
|
||
初始化文本处理器
|
||
|
||
Args:
|
||
config: 配置字典
|
||
"""
|
||
self.config = config
|
||
# 不再使用最小长度限制，只要有内容就提取
|
||
self.chunk_size = config['translation']['chunk_size']
|
||
self._global_id_counter = 0
|
||
|
||
logger.info(f"文本处理器初始化: chunk_size={self.chunk_size}, 无最小长度限制")
|
||
|
||
@staticmethod
|
||
def get_valid_text_elements(soup) -> List:
|
||
"""
|
||
获取有效的文本元素列表，自动过滤嵌套容器
|
||
(静态方法，供Builder共用，确保遍历顺序一致)
|
||
|
||
Args:
|
||
soup: BeautifulSoup对象
|
||
|
||
Returns:
|
||
过滤后的元素列表
|
||
"""
|
||
# 定义关注的标签
|
||
tags = ['p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'blockquote', 'li', 'td']
|
||
|
||
# 1. 获取所有候选元素
|
||
all_candidates = soup.find_all(tags)
|
||
|
||
# 2. 转换为集合以提高查找速度
|
||
candidate_set = set(all_candidates)
|
||
|
||
final_elements = []
|
||
|
||
for element in all_candidates:
|
||
# 3. 检查当前元素是否包含其他候选元素
|
||
# 如果包含，说明它是父容器，应该跳过，让子元素去被处理
|
||
has_candidate_children = False
|
||
|
||
# 只查找直接子级或后代中的候选标签
|
||
descendants = element.find_all(tags)
|
||
|
||
for child in descendants:
|
||
if child in candidate_set:
|
||
has_candidate_children = True
|
||
break
|
||
|
||
if has_candidate_children:
|
||
# 这是一个容器元素，跳过
|
||
continue
|
||
|
||
final_elements.append(element)
|
||
|
||
return final_elements
|
||
|
||
def extract_paragraphs_with_global_id(self, html_content: str, source_file: str = "") -> List[Dict]:
|
||
"""
|
||
提取段落并分配全局唯一ID
|
||
|
||
Args:
|
||
html_content: HTML内容
|
||
source_file: 来源文件名（用于调试）
|
||
|
||
Returns:
|
||
带全局ID的段落列表
|
||
"""
|
||
try:
|
||
soup = BeautifulSoup(html_content, 'html.parser')
|
||
paragraphs = []
|
||
|
||
# 移除不需要的元素
|
||
for element in soup(['script', 'style', 'meta', 'link']):
|
||
element.decompose()
|
||
|
||
# 获取有效的文本元素 (使用统一的过滤逻辑)
|
||
text_elements = self.get_valid_text_elements(soup)
|
||
|
||
position = 0
|
||
for element in text_elements:
|
||
# 清理文本：移除上标、下标等
|
||
clean_text = self._clean_element_text(element)
|
||
|
||
# 过滤逻辑：
|
||
# 1. 如果是导航元素，跳过
|
||
if self._is_navigation_element(element):
|
||
continue
|
||
|
||
# 2. 内容检查：只要不是空字符串，就保留
|
||
if not clean_text:
|
||
continue
|
||
|
||
# 分配全局唯一ID
|
||
global_id = self._generate_global_id()
|
||
|
||
paragraphs.append({
|
||
'global_id': global_id,
|
||
'text': clean_text,
|
||
'html_element': str(element),
|
||
'source_file': source_file,
|
||
'position': position, # 在文件中的位置（重要！）
|
||
'tag': element.name,
|
||
'length': len(clean_text)
|
||
})
|
||
|
||
position += 1
|
||
|
||
logger.info(f"从 {source_file} 提取了 {len(paragraphs)} 个段落")
|
||
return paragraphs
|
||
|
||
except Exception as e:
|
||
logger.error(f"提取段落失败: {e}")
|
||
return []
|
||
|
||
@staticmethod
|
||
def clean_element_text(element) -> str:
|
||
"""
|
||
清理元素文本：移除上标、下标、脚注等 (静态方法，供Builder共用)
|
||
|
||
Args:
|
||
element: HTML元素
|
||
|
||
Returns:
|
||
清理后的文本
|
||
"""
|
||
# 复制元素，避免修改原始DOM
|
||
element_copy = element.__copy__()
|
||
|
||
# 移除上标和下标（通常是脚注引用）
|
||
for tag in element_copy.find_all(['sup', 'sub']):
|
||
tag.decompose()
|
||
|
||
# 移除带有特定class的span/a标签 (脚注常见写法)
|
||
footnote_patterns = re.compile(r'footnote|endnote|reference|note|super|sub', re.I)
|
||
for tag in element_copy.find_all(['a', 'span', 'div'], class_=footnote_patterns):
|
||
tag.decompose()
|
||
|
||
# 移除仅包含数字或中括号数字的小型文本节点 (针对单纯文本形式的脚注 [1] 或 1)
|
||
for tag in element_copy.find_all('span'):
|
||
text = tag.get_text().strip()
|
||
# 匹配 [1], (1), 1, 12
|
||
if re.match(r'^(\[\d+\]|\(\d+\)|\d+)$', text):
|
||
tag.decompose()
|
||
|
||
# 获取清理后的文本
|
||
text = element_copy.get_text().strip()
|
||
|
||
# 额外的正则清理：移除正文末尾残留的引用标记，如 "text.[1]" 或 "text.1"
|
||
text = re.sub(r'(\.|。|，|,)\s*(\[\d+\]|\d+)(?=\s|$)', r'\1', text)
|
||
|
||
# 清理多余的空白
|
||
text = re.sub(r'\s+', ' ', text)
|
||
|
||
return text
|
||
|
||
@staticmethod
|
||
def is_navigation_element(element) -> bool:
|
||
"""
|
||
判断是否是导航元素 (静态方法，供Builder共用)
|
||
|
||
Args:
|
||
element: HTML元素
|
||
|
||
Returns:
|
||
是否是导航元素
|
||
"""
|
||
# 检查class属性
|
||
classes = element.get('class', [])
|
||
nav_classes = ['nav', 'navigation', 'toc', 'menu', 'header', 'footer', 'page-number']
|
||
|
||
# 处理 class 可能是列表或字符串的情况
|
||
if isinstance(classes, list):
|
||
class_str = ' '.join(classes).lower()
|
||
else:
|
||
class_str = str(classes).lower()
|
||
|
||
if any(nav_class in class_str for nav_class in nav_classes):
|
||
return True
|
||
|
||
# 检查父元素
|
||
parent = element.parent
|
||
if parent:
|
||
parent_classes = parent.get('class', [])
|
||
if isinstance(parent_classes, list):
|
||
parent_class_str = ' '.join(parent_classes).lower()
|
||
else:
|
||
parent_class_str = str(parent_classes).lower()
|
||
|
||
if any(nav_class in parent_class_str for nav_class in nav_classes):
|
||
return True
|
||
|
||
return False
|
||
|
||
def _clean_element_text(self, element) -> str:
|
||
"""兼容旧调用的包装器"""
|
||
return self.clean_element_text(element)
|
||
|
||
def _is_navigation_element(self, element) -> bool:
|
||
"""兼容旧调用的包装器"""
|
||
return self.is_navigation_element(element)
|
||
|
||
def _generate_global_id(self) -> str:
|
||
"""
|
||
生成全局唯一ID
|
||
|
||
Returns:
|
||
全局ID字符串，格式：p_0001
|
||
"""
|
||
self._global_id_counter += 1
|
||
return f"p_{self._global_id_counter:04d}"
|
||
|
||
def create_chunks_by_size(self, paragraphs: List[Dict]) -> List[List[Dict]]:
|
||
"""
|
||
按字符数创建chunks，不切断段落，不考虑章节边界
|
||
|
||
Args:
|
||
paragraphs: 带全局ID的段落列表
|
||
|
||
Returns:
|
||
分块的段落列表
|
||
"""
|
||
if not paragraphs:
|
||
return []
|
||
|
||
chunks = []
|
||
current_chunk = []
|
||
current_size = 0
|
||
|
||
for paragraph in paragraphs:
|
||
para_length = paragraph['length']
|
||
|
||
# 如果当前chunk加上这个段落不超过限制，就加入
|
||
if current_size + para_length <= self.chunk_size:
|
||
current_chunk.append(paragraph)
|
||
current_size += para_length
|
||
else:
|
||
# 保存当前chunk（如果有内容）
|
||
if current_chunk:
|
||
chunks.append(current_chunk)
|
||
|
||
# 开始新chunk
|
||
current_chunk = [paragraph]
|
||
current_size = para_length
|
||
|
||
# 保存最后一个chunk
|
||
if current_chunk:
|
||
chunks.append(current_chunk)
|
||
|
||
# 统计信息
|
||
total_chars = sum(p['length'] for p in paragraphs)
|
||
avg_chunk_size = total_chars / len(chunks) if chunks else 0
|
||
|
||
logger.info(f"创建了 {len(chunks)} 个chunk，"
|
||
f"总段落数: {len(paragraphs)}, "
|
||
f"平均chunk大小: {avg_chunk_size:.0f} 字符")
|
||
|
||
# 显示chunk分布
|
||
for i, chunk in enumerate(chunks, 1):
|
||
chunk_size = sum(p['length'] for p in chunk)
|
||
logger.debug(f" Chunk {i}: {len(chunk)} 段落, {chunk_size} 字符, "
|
||
f"ID范围: {chunk[0]['global_id']} - {chunk[-1]['global_id']}")
|
||
|
||
return chunks
|
||
|
||
def validate_translation(self, original: str, translation: str) -> bool:
|
||
"""
|
||
验证翻译质量
|
||
|
||
Args:
|
||
original: 原文
|
||
translation: 译文
|
||
|
||
Returns:
|
||
是否通过验证
|
||
"""
|
||
# 检查是否是失败标记
|
||
if translation.startswith('[翻译失败') or translation.startswith('[解析失败'):
|
||
return False
|
||
|
||
# 检查基本长度
|
||
if len(translation) < len(original) * 0.1:
|
||
logger.warning("翻译过短")
|
||
return False
|
||
|
||
if len(translation) > len(original) * 8:
|
||
logger.warning("翻译过长")
|
||
return False
|
||
|
||
# 检查是否包含中文
|
||
if not re.search(r'[\u4e00-\u9fff]', translation):
|
||
logger.warning("翻译不包含中文")
|
||
return False
|
||
|
||
return True
|
||
|
||
def get_statistics(self, paragraphs: List[Dict]) -> Dict:
|
||
"""
|
||
获取段落统计信息
|
||
|
||
Args:
|
||
paragraphs: 段落列表
|
||
|
||
Returns:
|
||
统计信息字典
|
||
"""
|
||
if not paragraphs:
|
||
return {}
|
||
|
||
total_chars = sum(p['length'] for p in paragraphs)
|
||
avg_length = total_chars / len(paragraphs)
|
||
|
||
# 按来源文件分组统计
|
||
by_source = {}
|
||
for p in paragraphs:
|
||
source = p['source_file']
|
||
if source not in by_source:
|
||
by_source[source] = 0
|
||
by_source[source] += 1
|
||
|
||
return {
|
||
'total_paragraphs': len(paragraphs),
|
||
'total_characters': total_chars,
|
||
'average_length': round(avg_length, 1),
|
||
'min_length': min(p['length'] for p in paragraphs),
|
||
'max_length': max(p['length'] for p in paragraphs),
|
||
'by_source_file': by_source
|
||
}
|