feat: Release v0.10 - Modular Architecture & External Config

- Refactor codebase into src/ (preprocessing, translation, assembly) - Add pipeline/ scripts for individual stages - Externalize configuration to config/config.yaml - Fix Cover Image preservation - Update documentation and manuals
2026-01-31 22:49:44 +08:00
parent 9ef82393be
commit 7a93c52b42
306 changed files with 30313 additions and 1071 deletions
@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""
+最小测试脚本：测试 Manifest -> TextProcessor -> LLMClient 的数据流
+验证 text_with_placeholders 是否在传递过程中丢失或损坏
+"""
+import sys
+import json
+import asyncio
+from pathlib import Path
+from loguru import logger
+
+# 添加 src 到路径
+sys.path.insert(0, '.')
+
+from src.manifest_manager import ManifestManager, ManifestItem
+from src.text_processor import TextProcessor
+from src.llm_client import LLMClient
+# Mock config
+from src.utils import load_config
+
+# 配置日志
+logger.remove()
+logger.add(sys.stdout, level="DEBUG")
+
+# 1. 创建临时的 Manifest 文件，模拟包含问题的真实数据
+# 模拟一个带有换行符的占位符文本，这是我们怀疑的根源
+mock_manifest_data = {
+    "metadata": {"book_id": "test_book"},
+    "items": [
+        {
+            "global_id": "p_00006",
+            "source_file": "test.html",
+            "original_html": "<p>in the name of <span id='page_vi'></span> abundance</p>",
+            "clean_text": "in the name of abundance",
+            "text_hash": "hash1",
+            "tag": "p",
+            # 模拟包含换行符的情况 (FormatExtractor 之前的问题)
+            "text_with_placeholders": "in the name of \nφ1φ\n abundance", 
+            "placeholder_map": {"1": "<span id='page_vi'></span>"},
+            "paragraph_type": "BODY",
+            "status": "pending"
+        },
+        {
+            "global_id": "p_00058",
+            "source_file": "test.html",
+            "original_html": "<p>all-<span id='a536'></span>hands</p>",
+            "clean_text": "all-hands",
+            "text_hash": "hash2",
+            "tag": "p",
+            # 正常情况
+            "text_with_placeholders": "all-φ1φhands",
+            "placeholder_map": {"1": "<span id='a536'></span>"},
+            "paragraph_type": "BODY",
+            "status": "pending"
+        }
+    ]
+}
+
+manifest_path = Path("cache/test_manifest.json")
+manifest_path.parent.mkdir(parents=True, exist_ok=True)
+with open(manifest_path, 'w') as f:
+    json.dump(mock_manifest_data, f)
+
+print(f"=== Created Mock Manifest at {manifest_path} ===")
+
+async def run_test():
+    # 2. 加载 Manifest
+    manifest = ManifestManager(str(manifest_path))
+    manifest.load()
+    print(f"Loaded {len(manifest.get_items())} items")
+    
+    # 3. 创建 TextProcessor 和 Chunks
+    # Mock config for processor
+    processor = TextProcessor({"translation": {"chunk_size": 1000}})
+    
+    # 这一步会从 manifest 读取 item
+    chunks = processor.create_chunks_from_manifest(manifest, mode="chinese")
+    print(f"Created {len(chunks)} chunks")
+    
+    chunk = chunks[0]
+    print(f"Chunk 0 has {len(chunk)} items")
+    
+    # 4. 模拟 LLMClient 构建 Prompt
+    # 不需要真正的 API key，只需要测试 _build_prompt
+    # 添加 dummy key 和 rate_limits 防止初始化报错
+    client = LLMClient({
+        "providers": {}, 
+        "llm": {
+            "api_key": "dummy_key", 
+            "models": {"fast": "dummy_model", "smart": "dummy_model"},
+            "rate_limits": {"requests_per_minute": 60, "concurrent_requests": 2}
+        }
+    }) 
+    
+    # 强制重新加载 prompts (确保我们使用最新的代码逻辑)
+    # 注意：我们之前修了 _load_prompts，如果 prompts.json 不存在会报错
+    # 这里我们假设 config/prompts.json 存在
+    
+    print("\n=== 构建 Prompt (Mode: Chinese) ===")
+    prompt = client._build_prompt(chunk, mode="chinese")
+    
+    print("-" * 40)
+    print(prompt)
+    print("-" * 40)
+    
+    # 验证关键点
+    print("\n=== 验证结果 ===")
+    
+    # 检查 p_00006
+    # 注意：我们之前修了 FormatExtractor，但那是针对**新提取**的内容。
+    # 这里我们测试的是**从旧 Manifest 读取**的内容。
+    # ManifestManager 读取时并不会自动清理换行符！
+    # 所以如果旧 manifest 里有换行，这里应该能复现出带换行的 prompt。
+    
+    has_p00006 = "p_00006 [BODY] in the name of \nφ1φ\n abundance" in prompt
+    print(f"p_00006 has newlines (bad): {has_p00006}")
+    
+    has_p00006_clean = "p_00006 [BODY] in the name of φ1φ abundance" in prompt
+    print(f"p_00006 is clean (good): {has_p00006_clean}")
+    
+    has_p00058 = "p_00058 [BODY] all-φ1φhands" in prompt
+    print(f"p_00058 is correct: {has_p00058}")
+
+if __name__ == "__main__":
+    asyncio.run(run_test())