谭凯
7a93c52b42
- Refactor codebase into src/ (preprocessing, translation, assembly) - Add pipeline/ scripts for individual stages - Externalize configuration to config/config.yaml - Fix Cover Image preservation - Update documentation and manuals
143 lines
5.8 KiB
Python
143 lines
5.8 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
分块测试脚本
|
|
测试新的字符范围分块逻辑
|
|
"""
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# 添加 src 目录到路径
|
|
sys.path.insert(0, str(Path(__file__).parent / "src"))
|
|
|
|
from src.epub_parser import EPUBParser
|
|
from src.text_processor import TextProcessor
|
|
from src.utils import load_config
|
|
from rich.console import Console
|
|
from rich.table import Table
|
|
from rich.panel import Panel
|
|
|
|
|
|
def test_chunking_logic(epub_path: str):
|
|
"""测试分块逻辑"""
|
|
console = Console()
|
|
|
|
try:
|
|
# 加载配置
|
|
config = load_config('config/config.json')
|
|
|
|
# 初始化组件
|
|
parser = EPUBParser(epub_path)
|
|
text_processor = TextProcessor(config)
|
|
|
|
console.print(f"[bold blue]测试分块逻辑: {epub_path}[/bold blue]\n")
|
|
|
|
# 显示配置
|
|
chunk_config = config['translation']['chunk_config']
|
|
config_table = Table(title="分块配置")
|
|
config_table.add_column("参数", style="cyan")
|
|
config_table.add_column("值", style="white")
|
|
|
|
config_table.add_row("最小字符数", str(chunk_config['min_chars']))
|
|
config_table.add_row("最大字符数", str(chunk_config['max_chars']))
|
|
config_table.add_row("最大段落数", str(chunk_config['max_paragraphs']))
|
|
|
|
console.print(config_table)
|
|
|
|
# 提取内容
|
|
content_items = parser.extract_translatable_content()
|
|
|
|
console.print(f"\n[cyan]找到 {len(content_items)} 个内容项目[/cyan]")
|
|
|
|
# 测试前3个内容项目
|
|
for i, content_item in enumerate(content_items[:3], 1):
|
|
console.print(f"\n[yellow]测试项目 {i}: {content_item['title']}[/yellow]")
|
|
|
|
# 提取段落
|
|
paragraphs = text_processor.extract_paragraphs(content_item['content'])
|
|
console.print(f"提取了 {len(paragraphs)} 个段落")
|
|
|
|
# 显示段落统计
|
|
if paragraphs:
|
|
para_chars = [p['char_count'] for p in paragraphs]
|
|
para_table = Table(title="段落统计")
|
|
para_table.add_column("统计项", style="cyan")
|
|
para_table.add_column("值", style="white")
|
|
|
|
para_table.add_row("段落数量", str(len(paragraphs)))
|
|
para_table.add_row("总字符数", f"{sum(para_chars):,}")
|
|
para_table.add_row("平均字符/段", f"{sum(para_chars)/len(para_chars):.0f}")
|
|
para_table.add_row("最短段落", f"{min(para_chars)} 字符")
|
|
para_table.add_row("最长段落", f"{max(para_chars)} 字符")
|
|
|
|
console.print(para_table)
|
|
|
|
# 创建分块
|
|
chunks = text_processor.create_chunks(paragraphs)
|
|
|
|
# 显示分块结果
|
|
console.print(f"\n[green]创建了 {len(chunks)} 个翻译块[/green]")
|
|
|
|
chunk_table = Table(title="翻译块详情")
|
|
chunk_table.add_column("块号", style="cyan")
|
|
chunk_table.add_column("段落数", style="yellow")
|
|
chunk_table.add_column("字符数", style="green")
|
|
chunk_table.add_column("字符范围", style="blue")
|
|
|
|
for j, chunk in enumerate(chunks, 1):
|
|
chunk_chars = sum(p['char_count'] for p in chunk)
|
|
char_range = f"{chunk_config['min_chars']}-{chunk_config['max_chars']}"
|
|
|
|
# 检查是否在范围内
|
|
in_range = chunk_config['min_chars'] <= chunk_chars <= chunk_config['max_chars']
|
|
char_display = f"{chunk_chars:,}" + (" ✓" if in_range else " ⚠")
|
|
|
|
chunk_table.add_row(
|
|
str(j),
|
|
str(len(chunk)),
|
|
char_display,
|
|
char_range
|
|
)
|
|
|
|
console.print(chunk_table)
|
|
|
|
# 显示统计
|
|
stats = text_processor.get_chunk_stats(chunks)
|
|
if stats:
|
|
stats_table = Table(title="分块统计")
|
|
stats_table.add_column("统计项", style="cyan")
|
|
stats_table.add_column("值", style="white")
|
|
|
|
stats_table.add_row("总翻译块数", str(stats['total_chunks']))
|
|
stats_table.add_row("平均段落/块", f"{stats['avg_paragraphs_per_chunk']:.1f}")
|
|
stats_table.add_row("平均字符/块", f"{stats['avg_chars_per_chunk']:.0f}")
|
|
stats_table.add_row("字符范围", f"{stats['min_chars_per_chunk']}-{stats['max_chars_per_chunk']}")
|
|
|
|
console.print(stats_table)
|
|
|
|
# 显示示例块内容
|
|
if chunks:
|
|
example_chunk = chunks[0]
|
|
example_texts = [p['text'][:100] + "..." for p in example_chunk[:2]]
|
|
|
|
console.print(Panel(
|
|
"\n".join(f"{i+1}. {text}" for i, text in enumerate(example_texts)),
|
|
title=f"示例块内容 (块1, 前2段)",
|
|
border_style="green"
|
|
))
|
|
|
|
console.print(f"\n[bold green]分块测试完成！[/bold green]")
|
|
|
|
except Exception as e:
|
|
console.print(f"[red]测试失败: {e}[/red]")
|
|
import traceback
|
|
console.print(traceback.format_exc())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) != 2:
|
|
print("使用方法: python test_chunking.py <epub_file>")
|
|
sys.exit(1)
|
|
|
|
epub_file = sys.argv[1]
|
|
test_chunking_logic(epub_file) |