feat: Release v0.10 - Modular Architecture & External Config
- Refactor codebase into src/ (preprocessing, translation, assembly) - Add pipeline/ scripts for individual stages - Externalize configuration to config/config.yaml - Fix Cover Image preservation - Update documentation and manuals
This commit is contained in:
谭凯
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Placeholder Backfill Test Script
|
||||
Tests placeholder restoration on translated entries.
|
||||
|
||||
Usage:
|
||||
python scripts/test_backfill.py --chapter 38
|
||||
python scripts/test_backfill.py --chapter 38 --limit 10
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.manifest_manager import ManifestManager
|
||||
from src.format_restorer import FormatRestorer
|
||||
|
||||
# Configuration - Use unified .work directory
|
||||
BOOK_NAME = "Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)"
|
||||
WORK_DIR = Path(f".work/{BOOK_NAME}")
|
||||
MANIFEST_PATH = WORK_DIR / "manifest.json"
|
||||
|
||||
|
||||
def build_toc_from_manifest(manager: ManifestManager) -> list:
|
||||
"""Build TOC from manifest entries."""
|
||||
files = {}
|
||||
for entry in manager.entries:
|
||||
fp = entry.file_path
|
||||
if fp not in files:
|
||||
files[fp] = {'count': 0, 'first_text': ''}
|
||||
files[fp]['count'] += 1
|
||||
if not files[fp]['first_text'] and entry.original_text:
|
||||
files[fp]['first_text'] = entry.original_text[:40].replace('\n', ' ')
|
||||
|
||||
toc = []
|
||||
for i, (fp, info) in enumerate(sorted(files.items()), 1):
|
||||
toc.append({
|
||||
'index': i,
|
||||
'href': fp,
|
||||
'title': info['first_text'] or f"File {i}",
|
||||
'paragraphs': info['count']
|
||||
})
|
||||
return toc
|
||||
|
||||
|
||||
def get_chapter_entries(manager: ManifestManager, chapter_index: int) -> tuple:
|
||||
"""Get entries for a specific chapter by index."""
|
||||
toc = build_toc_from_manifest(manager)
|
||||
|
||||
if chapter_index < 1 or chapter_index > len(toc):
|
||||
print(f"错误: 章节编号 {chapter_index} 无效 (范围: 1-{len(toc)})")
|
||||
return None, None
|
||||
|
||||
chapter = toc[chapter_index - 1]
|
||||
href = chapter['href']
|
||||
|
||||
entries = [e for e in manager.entries if e.file_path == href]
|
||||
return chapter, entries
|
||||
|
||||
|
||||
def test_backfill(chapter: dict, entries: list, limit: int = None):
|
||||
"""Test placeholder restoration for a chapter."""
|
||||
|
||||
print(f"\n" + "=" * 70)
|
||||
print(f"占位符回填测试 - 章节 #{chapter['index']}: {chapter['title'][:40]}...")
|
||||
print("=" * 70)
|
||||
|
||||
# Filter entries with translation and placeholders
|
||||
translated = [e for e in entries if e.translated_text]
|
||||
with_placeholders = [e for e in translated if e.placeholders and len(e.placeholders) > 0]
|
||||
|
||||
print(f"\n统计:")
|
||||
print(f" 总段落: {len(entries)}")
|
||||
print(f" 已翻译: {len(translated)}")
|
||||
print(f" 有占位符: {len(with_placeholders)}")
|
||||
|
||||
if not translated:
|
||||
print("\n⚠️ 该章节没有已翻译的内容!")
|
||||
return
|
||||
|
||||
# Test restoration
|
||||
restorer = FormatRestorer()
|
||||
|
||||
success_count = 0
|
||||
fail_count = 0
|
||||
results = []
|
||||
|
||||
test_entries = with_placeholders[:limit] if limit else with_placeholders
|
||||
|
||||
print(f"\n测试 {len(test_entries)} 个带占位符的段落:")
|
||||
print("-" * 70)
|
||||
|
||||
for i, entry in enumerate(test_entries, 1):
|
||||
original = entry.original_text
|
||||
translated = entry.translated_text
|
||||
placeholders = entry.placeholders
|
||||
|
||||
# Get non-internal placeholders
|
||||
visible_ph = {k: v for k, v in placeholders.items() if not k.startswith('_')}
|
||||
|
||||
# Perform restoration
|
||||
restored, success = restorer.restore(translated, placeholders)
|
||||
|
||||
if success:
|
||||
success_count += 1
|
||||
status = "✅"
|
||||
else:
|
||||
fail_count += 1
|
||||
status = "❌"
|
||||
|
||||
results.append({
|
||||
'index': i,
|
||||
'entry_id': entry.entry_id,
|
||||
'original': original,
|
||||
'translated': translated,
|
||||
'restored': restored,
|
||||
'placeholders': visible_ph,
|
||||
'success': success
|
||||
})
|
||||
|
||||
# Print summary
|
||||
print(f"\n[{i}] {status} {entry.entry_id[-40:]}")
|
||||
print(f" 占位符: {list(visible_ph.keys())}")
|
||||
print(f" 原文: {original[:50]}...")
|
||||
print(f" 译文: {translated[:50]}...")
|
||||
|
||||
if not success:
|
||||
print(f" 还原: {restored[:50]}...")
|
||||
# Show what placeholders are missing
|
||||
missing = []
|
||||
for k in visible_ph.keys():
|
||||
if k.isdigit():
|
||||
if f"φ{k}φ" not in translated and f"φ/{k}φ" not in translated:
|
||||
missing.append(k)
|
||||
if missing:
|
||||
print(f" 缺失: {missing}")
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 70)
|
||||
print(f"测试结果汇总")
|
||||
print("=" * 70)
|
||||
print(f" 成功: {success_count}/{len(test_entries)}")
|
||||
print(f" 失败: {fail_count}/{len(test_entries)}")
|
||||
|
||||
if fail_count > 0:
|
||||
print(f"\n失败案例详情:")
|
||||
for r in results:
|
||||
if not r['success']:
|
||||
print(f"\n [{r['index']}] {r['entry_id'][-50:]}")
|
||||
print(f" 原文: {r['original'][:60]}...")
|
||||
print(f" 译文: {r['translated'][:60]}...")
|
||||
print(f" 还原: {r['restored'][:60]}...")
|
||||
print(f" 占位符: {r['placeholders']}")
|
||||
|
||||
# Also test entries without visible placeholders (only _prefix/_suffix)
|
||||
prefix_suffix_only = [e for e in translated
|
||||
if e.placeholders
|
||||
and all(k.startswith('_') for k in e.placeholders.keys())]
|
||||
|
||||
if prefix_suffix_only:
|
||||
print(f"\n\n额外测试: 只有 _prefix/_suffix 的段落 ({len(prefix_suffix_only)} 个)")
|
||||
print("-" * 70)
|
||||
|
||||
ps_success = 0
|
||||
ps_fail = 0
|
||||
|
||||
for entry in prefix_suffix_only[:5]: # Test first 5
|
||||
restored, success = restorer.restore(entry.translated_text, entry.placeholders)
|
||||
if success:
|
||||
ps_success += 1
|
||||
status = "✅"
|
||||
else:
|
||||
ps_fail += 1
|
||||
status = "❌"
|
||||
|
||||
print(f" {status} {entry.entry_id[-40:]}")
|
||||
if '_prefix' in entry.placeholders:
|
||||
print(f" _prefix: {entry.placeholders['_prefix'][:30]}...")
|
||||
if '_suffix' in entry.placeholders:
|
||||
print(f" _suffix: {entry.placeholders['_suffix'][:30]}...")
|
||||
|
||||
print(f"\n 结果: {ps_success}/{min(5, len(prefix_suffix_only))} 成功")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="测试占位符回填")
|
||||
parser.add_argument("--chapter", "-c", type=int, required=True, help="章节编号")
|
||||
parser.add_argument("--limit", "-l", type=int, default=20, help="测试数量限制 (默认20)")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not MANIFEST_PATH.exists():
|
||||
print(f"错误: Manifest 不存在: {MANIFEST_PATH}")
|
||||
return
|
||||
|
||||
# Load manifest
|
||||
manager = ManifestManager(MANIFEST_PATH)
|
||||
manager.load()
|
||||
print(f"已加载 manifest: {len(manager.entries)} 条目")
|
||||
|
||||
# Get chapter
|
||||
chapter, entries = get_chapter_entries(manager, args.chapter)
|
||||
if not chapter:
|
||||
return
|
||||
|
||||
# Test backfill
|
||||
test_backfill(chapter, entries, args.limit)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user