feat: Release v0.10 - Modular Architecture & External Config

- Refactor codebase into src/ (preprocessing, translation, assembly)
- Add pipeline/ scripts for individual stages
- Externalize configuration to config/config.yaml
- Fix Cover Image preservation
- Update documentation and manuals
This commit is contained in:
谭凯
2026-01-31 22:49:44 +08:00
parent 9ef82393be
commit 7a93c52b42
306 changed files with 30313 additions and 1071 deletions
+216
View File
@@ -0,0 +1,216 @@
#!/usr/bin/env python3
"""
调试和测试脚本
用于诊断 EPUB 解析问题
"""
import sys
from pathlib import Path
# 添加 src 目录到路径
sys.path.insert(0, str(Path(__file__).parent / "src"))
from src.epub_parser import EPUBParser
from src.text_processor import TextProcessor
from src.utils import load_config
from rich.console import Console
from rich.table import Table
from rich.panel import Panel
from bs4 import BeautifulSoup
import ebooklib
def debug_epub_structure(epub_path: str):
"""调试 EPUB 结构"""
console = Console()
try:
# 加载配置
config = load_config('config/config.json')
# 初始化解析器
parser = EPUBParser(epub_path)
text_processor = TextProcessor(config)
console.print(f"[bold blue]调试 EPUB 文件: {epub_path}[/bold blue]\n")
# 显示基本信息
book_info = parser.get_book_info()
info_table = Table(title="书籍信息")
info_table.add_column("属性", style="cyan")
info_table.add_column("", style="white")
for key, value in book_info.items():
info_table.add_row(key, str(value))
console.print(info_table)
# 显示目录结构
console.print("\n[bold green]目录结构分析:[/bold green]")
toc_table = Table(title="目录结构")
toc_table.add_column("类型", style="cyan")
toc_table.add_column("标题", style="white")
toc_table.add_column("状态", style="green")
# 检查各种内容类型
content_types = ['preface', 'introduction', 'prologue', 'abstract', 'epilogue', 'acknowledgments']
for content_type in content_types:
item = parser.toc_structure.get(content_type)
if item:
toc_table.add_row(content_type, item['title'], "✓ 找到")
else:
toc_table.add_row(content_type, "-", "✗ 未找到")
# 章节信息
chapters = parser.toc_structure['chapters']
toc_table.add_row("chapters", f"{len(chapters)} 个章节", "✓ 找到" if chapters else "✗ 未找到")
console.print(toc_table)
# 显示章节列表
if chapters:
console.print("\n[bold yellow]章节列表:[/bold yellow]")
chapter_table = Table()
chapter_table.add_column("序号", style="cyan")
chapter_table.add_column("标题", style="white")
chapter_table.add_column("内容长度", style="green")
for i, chapter in enumerate(chapters[:10], 1): # 只显示前10个
content = parser._extract_item_content(chapter)
content_length = len(content) if content else 0
chapter_table.add_row(str(i), chapter['title'], f"{content_length:,} 字符")
if len(chapters) > 10:
chapter_table.add_row("...", f"还有 {len(chapters) - 10} 个章节", "...")
console.print(chapter_table)
# 测试段落提取
console.print("\n[bold magenta]段落提取测试:[/bold magenta]")
# 选择第一个有内容的项目进行测试
test_content = None
test_title = ""
# 优先测试序言类内容
for content_type in ['prologue', 'preface', 'introduction', 'abstract']:
item = parser.toc_structure.get(content_type)
if item:
test_content = parser._extract_item_content(item)
test_title = f"{content_type}: {item['title']}"
break
# 如果没有序言,测试第一个章节
if not test_content and chapters:
test_content = parser._extract_item_content(chapters[0])
test_title = f"章节: {chapters[0]['title']}"
if test_content:
paragraphs = text_processor.extract_paragraphs(test_content)
console.print(f"测试内容: {test_title}")
console.print(f"原始内容长度: {len(test_content):,} 字符")
console.print(f"提取段落数: {len(paragraphs)}")
if paragraphs:
# 显示前几个段落
para_table = Table(title="段落示例")
para_table.add_column("序号", style="cyan")
para_table.add_column("类型", style="yellow")
para_table.add_column("内容预览", style="white")
para_table.add_column("长度", style="green")
for i, para in enumerate(paragraphs[:5], 1):
preview = para['text'][:100] + "..." if len(para['text']) > 100 else para['text']
para_table.add_row(
str(i),
para.get('type', 'unknown'),
preview,
str(len(para['text']))
)
console.print(para_table)
# 测试翻译块创建
chunks = text_processor.create_chunks(paragraphs, 3)
console.print(f"\n[cyan]翻译块信息:[/cyan] 创建了 {len(chunks)} 个翻译块")
if chunks:
chunk_table = Table(title="翻译块示例")
chunk_table.add_column("块号", style="cyan")
chunk_table.add_column("段落数", style="yellow")
chunk_table.add_column("总字符数", style="green")
for i, chunk in enumerate(chunks[:3], 1): # 显示前3个块
total_chars = sum(len(p['text']) for p in chunk)
chunk_table.add_row(str(i), str(len(chunk)), f"{total_chars:,}")
console.print(chunk_table)
else:
console.print("[red]未能提取到段落![/red]")
# 显示原始内容的一部分用于调试
soup = BeautifulSoup(test_content, 'html.parser')
text_content = soup.get_text()[:500]
console.print(Panel(
text_content,
title="原始文本内容(前500字符)",
border_style="red"
))
else:
console.print("[red]未找到可测试的内容![/red]")
# 显示所有 HTML 文件
console.print("\n[bold cyan]所有 HTML 文件:[/bold cyan]")
try:
html_items = list(parser.book.get_items_of_type(ebooklib.ITEM_DOCUMENT))
file_table = Table()
file_table.add_column("文件名", style="cyan")
file_table.add_column("大小", style="green")
file_table.add_column("内容预览", style="white")
for item in html_items[:10]: # 只显示前10个
try:
content = item.get_content().decode('utf-8', errors='ignore')
soup = BeautifulSoup(content, 'html.parser')
text_preview = soup.get_text()[:100].replace('\n', ' ')
file_table.add_row(
item.get_name(),
f"{len(content):,} 字符",
text_preview + "..." if len(text_preview) == 100 else text_preview
)
except Exception as e:
file_table.add_row(item.get_name(), "错误", f"读取失败: {e}")
if len(html_items) > 10:
file_table.add_row("...", f"还有 {len(html_items) - 10} 个文件", "...")
console.print(file_table)
except Exception as e:
console.print(f"[yellow]无法列出 HTML 文件: {e}[/yellow]")
# 总结
console.print(f"\n[bold green]✓ 调试完成[/bold green]")
console.print(f"[green]结论: EPUB 文件结构正常,可以进行翻译[/green]")
except Exception as e:
console.print(f"[red]调试失败: {e}[/red]")
import traceback
console.print(traceback.format_exc())
if __name__ == "__main__":
if len(sys.argv) != 2:
print("使用方法: python debug.py <epub_file>")
sys.exit(1)
epub_file = sys.argv[1]
debug_epub_structure(epub_file)
@@ -0,0 +1,132 @@
import sys
import os
from pathlib import Path
from ebooklib import epub
import ebooklib
from loguru import logger
# Add project root to path
sys.path.insert(0, str(Path(__file__).parent))
from src.utils import load_config
def compare_epubs(original_path, new_path):
print(f"🔍 Comparing EPUBs:\n Original: {original_path}\n New: {new_path}")
print("=" * 60)
if not os.path.exists(new_path):
print(f"❌ New EPUB not found: {new_path}")
return
try:
orig_book = epub.read_epub(original_path)
new_book = epub.read_epub(new_path)
except Exception as e:
print(f"❌ Error reading EPUBs: {e}")
return
# 1. Metadata Comparison
print("\n[1] Metadata Comparison")
print("-" * 60)
namespaces = ['DC', 'OPF']
for ns in namespaces:
orig_meta = orig_book.metadata.get(ns, {})
new_meta = new_book.metadata.get(ns, {})
all_keys = set(orig_meta.keys()) | set(new_meta.keys())
for key in sorted(all_keys):
orig_vals = [v[0] for v in orig_meta.get(key, [])]
new_vals = [v[0] for v in new_meta.get(key, [])]
if orig_vals != new_vals:
print(f" ⚠️ {ns}:{key} Changed:")
print(f" Orig: {orig_vals}")
print(f" New: {new_vals}")
else:
# print(f" ✅ {ns}:{key} match")
pass
# Special check for Cover
print("\n[2] Cover Image Check")
print("-" * 60)
# Check via Metadata
orig_cover_meta = orig_book.get_metadata('OPF', 'cover')
new_cover_meta = new_book.get_metadata('OPF', 'cover')
print(f" Original Cover Meta (OPF): {orig_cover_meta}")
print(f" New Cover Meta (OPF): {new_cover_meta}")
# Check via Manifest Items
orig_cover_items = [i for i in orig_book.get_items() if 'cover' in i.get_name().lower() and i.media_type.startswith('image/')]
new_cover_items = [i for i in new_book.get_items() if 'cover' in i.get_name().lower() and i.media_type.startswith('image/')]
print(f" Original Cover Image Items: {[i.get_name() for i in orig_cover_items]}")
print(f" New Cover Image Items: {[i.get_name() for i in new_cover_items]}")
# 3. Spine Comparison (Reading Order)
print("\n[3] Spine (Reading Order) Comparison")
print("-" * 60)
orig_spine_ids = [item[0] for item in orig_book.spine]
new_spine_ids = [item[0] for item in new_book.spine]
print(f" Original Spine Length: {len(orig_spine_ids)}")
print(f" New Spine Length: {len(new_spine_ids)}")
# Map IDs to Filenames for better readability
def get_filename(book, item_id):
item = book.get_item_with_id(item_id)
return item.get_name() if item else "UNKNOWN"
# Compare first few and last few
limit = 5
print(f" First {limit} items:")
for i in range(min(len(orig_spine_ids), len(new_spine_ids), limit)):
f_orig = get_filename(orig_book, orig_spine_ids[i])
f_new = get_filename(new_book, new_spine_ids[i])
status = "" if f_orig == f_new else ""
print(f" {i+1}. {status} Orig: {f_orig} | New: {f_new}")
# Check for missing items in spine
orig_filenames = set(get_filename(orig_book, i) for i in orig_spine_ids)
new_filenames = set(get_filename(new_book, i) for i in new_spine_ids)
missing_in_new = orig_filenames - new_filenames
if missing_in_new:
print(f"\n ⚠️ Missing from New Spine ({len(missing_in_new)}):")
for f in list(missing_in_new)[:10]:
print(f" - {f}")
# 4. Manifest Comparison (All Resources)
print("\n[4] Manifest (All Resources) Comparison")
print("-" * 60)
orig_manifest = {i.get_name() for i in orig_book.get_items()}
new_manifest = {i.get_name() for i in new_book.get_items()}
missing_resources = orig_manifest - new_manifest
# Filter out NCX/Nav as they might be regenerated with different names
missing_resources = {f for f in missing_resources if not f.endswith('.ncx') and 'nav' not in f.lower()}
if missing_resources:
print(f" ⚠️ Resources Missing in New Book ({len(missing_resources)}):")
for f in sorted(list(missing_resources)):
print(f" - {f}")
else:
print(" ✅ All resources preserved.")
if __name__ == "__main__":
orig_path = "input/To Explain the World The Discovery of Modern Science (H) (Steven Weinberg [Weinberg, Steven]) (Z-Library).epub"
# Escaped path from user prompt: "input/To Explain the World The Discovery of Modern Science (H) (Steven Weinberg [Weinberg, Steven]) (Z-Library).epub"
# We generated this in the previous batch test
new_path = "test_output/To Explain the World The Discovery of Modern Science (H)_bilingual.epub"
if len(sys.argv) > 2:
orig_path = sys.argv[1]
new_path = sys.argv[2]
compare_epubs(orig_path, new_path)
@@ -0,0 +1,74 @@
import sys
from pathlib import Path
from bs4 import BeautifulSoup
from loguru import logger
# Add project root to path
sys.path.insert(0, str(Path(__file__).parent))
from src.epub_parser import EPUBParser
from src.text_processor import TextProcessor
from src.utils import load_config
def debug_nested_structure(epub_path: str):
config = load_config()
parser = EPUBParser(epub_path)
content_items = parser.extract_all_content_items()
# Check just one chapter (e.g. Chapter 1)
target_item = None
for item in content_items:
if 'c01' in item['file_name']: # Chapter 1 usually
target_item = item
break
if not target_item:
target_item = content_items[2] # Fallback to 3rd item
print(f"Checking file: {target_item['file_name']}")
soup = BeautifulSoup(target_item['content'], 'html.parser')
# Simulate TextProcessor extraction logic
text_elements = soup.find_all(['p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
'blockquote', 'li', 'td'])
min_len = config['processing'].get('min_paragraph_length', 30)
extracted = []
for i, element in enumerate(text_elements):
# Clean text logic
clean_text = TextProcessor.clean_element_text(element)
is_valid = True
if len(clean_text) < min_len:
is_valid = False
if TextProcessor.is_navigation_element(element):
is_valid = False
if is_valid:
extracted.append((element, clean_text))
# Check for nesting
# If this element contains other valid extracted elements
for prev_el, prev_text in extracted[:-1]:
# Check if current element is inside previous element
if element in prev_el.descendants:
print(f"\n⚠️ NESTING DETECTED!")
print(f" Parent: <{prev_el.name}> {prev_text[:50]}...")
print(f" Child: <{element.name}> {clean_text[:50]}...")
# Check if previous element is inside current element
if prev_el in element.descendants:
print(f"\n⚠️ NESTING DETECTED!")
print(f" Parent: <{element.name}> {clean_text[:50]}...")
print(f" Child: <{prev_el.name}> {prev_text[:50]}...")
if __name__ == "__main__":
if len(sys.argv) < 2:
print("Usage: python debug_structure.py <epub_file>")
sys.exit(1)
epub_file = sys.argv[1]
debug_nested_structure(epub_file)
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
"""
快速修复和测试脚本
"""
import sys
import os
from pathlib import Path
# 添加 src 目录到路径
sys.path.insert(0, str(Path(__file__).parent / "src"))
from rich.console import Console
def check_api_key():
"""检查 API Key 设置"""
console = Console()
# 检查环境变量
env_key = os.environ.get('OPENROUTER_API_KEY')
if env_key and env_key != 'YOUR_OPENROUTER_API_KEY':
console.print(f"[green]✓ 环境变量中找到 API Key: {env_key[:10]}...[/green]")
return True
# 检查 .env 文件
env_file = Path('.env')
if env_file.exists():
with open(env_file, 'r') as f:
content = f.read()
if 'OPENROUTER_API_KEY=' in content and 'YOUR_OPENROUTER_API_KEY' not in content:
console.print("[green]✓ .env 文件中找到 API Key[/green]")
return True
# 检查配置文件
config_file = Path('config/config.json')
if config_file.exists():
import json
try:
with open(config_file, 'r') as f:
config = json.load(f)
api_key = config.get('openrouter', {}).get('api_key', '')
if api_key and api_key != 'YOUR_OPENROUTER_API_KEY':
console.print(f"[green]✓ 配置文件中找到 API Key: {api_key[:10]}...[/green]")
return True
except Exception as e:
console.print(f"[red]配置文件读取错误: {e}[/red]")
console.print("[red]✗ 未找到有效的 API Key[/red]")
console.print("\n请设置 OpenRouter API Key:")
console.print("1. 环境变量: export OPENROUTER_API_KEY='your_key'")
console.print("2. .env 文件: OPENROUTER_API_KEY=your_key")
console.print("3. 配置文件: 编辑 config/config.json")
return False
def quick_fix():
"""快速修复常见问题"""
console = Console()
console.print("[bold blue]EPUB 翻译器 - 快速修复[/bold blue]\n")
# 检查 API Key
if not check_api_key():
return False
# 检查依赖
console.print("\n[cyan]检查依赖...[/cyan]")
required_modules = [
'ebooklib', 'bs4', 'lxml', 'openai',
'aiohttp', 'pydantic', 'loguru', 'rich'
]
missing_modules = []
for module in required_modules:
try:
if module == 'bs4':
import bs4
else:
__import__(module)
console.print(f"[green]✓ {module}[/green]")
except ImportError:
console.print(f"[red]✗ {module}[/red]")
missing_modules.append(module)
if missing_modules:
console.print(f"\n[red]缺少依赖: {', '.join(missing_modules)}[/red]")
console.print("请运行: uv pip install -r requirements.txt")
return False
console.print("\n[green]✓ 所有检查通过[/green]")
return True
if __name__ == "__main__":
if quick_fix():
print("\n可以开始使用翻译器了!")
print("运行: python main.py your_book.epub --test")
else:
print("\n请先修复上述问题")
sys.exit(1)
+79
View File
@@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""
EPUB 翻译器使用示例
演示如何使用程序进行翻译
"""
import asyncio
import sys
from pathlib import Path
# 添加 src 目录到路径
sys.path.insert(0, str(Path(__file__).parent / "src"))
from src.translator import EPUBTranslator
from src.utils import load_config, setup_logging
from rich.console import Console
async def example_usage():
"""使用示例"""
console = Console()
console.print("[bold blue]EPUB 翻译器使用示例[/bold blue]")
try:
# 加载配置
config = load_config('config/config.json')
setup_logging(config)
# 初始化翻译器
translator = EPUBTranslator(config)
# 示例 EPUB 文件路径(请替换为实际文件)
epub_file = "sample_book.epub"
if not Path(epub_file).exists():
console.print(f"[yellow]示例文件 {epub_file} 不存在[/yellow]")
console.print("请将你的 EPUB 文件放在当前目录并重命名为 sample_book.epub")
return
# 1. 估算翻译成本
console.print("\n[cyan]1. 估算翻译成本...[/cyan]")
estimate = await translator.get_translation_estimate(epub_file)
if estimate:
console.print(f"总段落数: {estimate['total_paragraphs']}")
console.print(f"估算时间: {estimate['estimated_time_minutes']:.1f} 分钟")
console.print(f"估算请求数: {estimate['estimated_requests']}")
# 2. 测试翻译
console.print("\n[cyan]2. 运行测试翻译...[/cyan]")
test_result = await translator.translate_epub(epub_file, test_mode=True)
if test_result.get('status') == 'success':
console.print("[green]测试翻译成功![/green]")
else:
console.print("[red]测试翻译失败[/red]")
return
# 3. 询问是否继续完整翻译
console.print("\n[yellow]是否继续完整翻译?这可能需要一些时间和费用。[/yellow]")
response = input("输入 'yes' 继续,其他任意键退出: ")
if response.lower() == 'yes':
console.print("\n[cyan]3. 开始完整翻译...[/cyan]")
output_file = await translator.translate_epub(epub_file, test_mode=False)
console.print(f"[green]翻译完成!输出文件: {output_file}[/green]")
else:
console.print("[yellow]已取消完整翻译[/yellow]")
except Exception as e:
console.print(f"[red]示例运行失败: {e}[/red]")
finally:
await translator.llm_client.close()
if __name__ == "__main__":
asyncio.run(example_usage())