谭凯
7a93c52b42
- Refactor codebase into src/ (preprocessing, translation, assembly) - Add pipeline/ scripts for individual stages - Externalize configuration to config/config.yaml - Fix Cover Image preservation - Update documentation and manuals
126 lines
4.3 KiB
Python
126 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
最小测试脚本：测试 Manifest -> TextProcessor -> LLMClient 的数据流
|
|
验证 text_with_placeholders 是否在传递过程中丢失或损坏
|
|
"""
|
|
import sys
|
|
import json
|
|
import asyncio
|
|
from pathlib import Path
|
|
from loguru import logger
|
|
|
|
# 添加 src 到路径
|
|
sys.path.insert(0, '.')
|
|
|
|
from src.manifest_manager import ManifestManager, ManifestItem
|
|
from src.text_processor import TextProcessor
|
|
from src.llm_client import LLMClient
|
|
# Mock config
|
|
from src.utils import load_config
|
|
|
|
# 配置日志
|
|
logger.remove()
|
|
logger.add(sys.stdout, level="DEBUG")
|
|
|
|
# 1. 创建临时的 Manifest 文件，模拟包含问题的真实数据
|
|
# 模拟一个带有换行符的占位符文本，这是我们怀疑的根源
|
|
mock_manifest_data = {
|
|
"metadata": {"book_id": "test_book"},
|
|
"items": [
|
|
{
|
|
"global_id": "p_00006",
|
|
"source_file": "test.html",
|
|
"original_html": "<p>in the name of <span id='page_vi'></span> abundance</p>",
|
|
"clean_text": "in the name of abundance",
|
|
"text_hash": "hash1",
|
|
"tag": "p",
|
|
# 模拟包含换行符的情况 (FormatExtractor 之前的问题)
|
|
"text_with_placeholders": "in the name of \nφ1φ\n abundance",
|
|
"placeholder_map": {"1": "<span id='page_vi'></span>"},
|
|
"paragraph_type": "BODY",
|
|
"status": "pending"
|
|
},
|
|
{
|
|
"global_id": "p_00058",
|
|
"source_file": "test.html",
|
|
"original_html": "<p>all-<span id='a536'></span>hands</p>",
|
|
"clean_text": "all-hands",
|
|
"text_hash": "hash2",
|
|
"tag": "p",
|
|
# 正常情况
|
|
"text_with_placeholders": "all-φ1φhands",
|
|
"placeholder_map": {"1": "<span id='a536'></span>"},
|
|
"paragraph_type": "BODY",
|
|
"status": "pending"
|
|
}
|
|
]
|
|
}
|
|
|
|
manifest_path = Path("cache/test_manifest.json")
|
|
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(manifest_path, 'w') as f:
|
|
json.dump(mock_manifest_data, f)
|
|
|
|
print(f"=== Created Mock Manifest at {manifest_path} ===")
|
|
|
|
async def run_test():
|
|
# 2. 加载 Manifest
|
|
manifest = ManifestManager(str(manifest_path))
|
|
manifest.load()
|
|
print(f"Loaded {len(manifest.get_items())} items")
|
|
|
|
# 3. 创建 TextProcessor 和 Chunks
|
|
# Mock config for processor
|
|
processor = TextProcessor({"translation": {"chunk_size": 1000}})
|
|
|
|
# 这一步会从 manifest 读取 item
|
|
chunks = processor.create_chunks_from_manifest(manifest, mode="chinese")
|
|
print(f"Created {len(chunks)} chunks")
|
|
|
|
chunk = chunks[0]
|
|
print(f"Chunk 0 has {len(chunk)} items")
|
|
|
|
# 4. 模拟 LLMClient 构建 Prompt
|
|
# 不需要真正的 API key，只需要测试 _build_prompt
|
|
# 添加 dummy key 和 rate_limits 防止初始化报错
|
|
client = LLMClient({
|
|
"providers": {},
|
|
"llm": {
|
|
"api_key": "dummy_key",
|
|
"models": {"fast": "dummy_model", "smart": "dummy_model"},
|
|
"rate_limits": {"requests_per_minute": 60, "concurrent_requests": 2}
|
|
}
|
|
})
|
|
|
|
# 强制重新加载 prompts (确保我们使用最新的代码逻辑)
|
|
# 注意：我们之前修了 _load_prompts，如果 prompts.json 不存在会报错
|
|
# 这里我们假设 config/prompts.json 存在
|
|
|
|
print("\n=== 构建 Prompt (Mode: Chinese) ===")
|
|
prompt = client._build_prompt(chunk, mode="chinese")
|
|
|
|
print("-" * 40)
|
|
print(prompt)
|
|
print("-" * 40)
|
|
|
|
# 验证关键点
|
|
print("\n=== 验证结果 ===")
|
|
|
|
# 检查 p_00006
|
|
# 注意：我们之前修了 FormatExtractor，但那是针对**新提取**的内容。
|
|
# 这里我们测试的是**从旧 Manifest 读取**的内容。
|
|
# ManifestManager 读取时并不会自动清理换行符！
|
|
# 所以如果旧 manifest 里有换行，这里应该能复现出带换行的 prompt。
|
|
|
|
has_p00006 = "p_00006 [BODY] in the name of \nφ1φ\n abundance" in prompt
|
|
print(f"p_00006 has newlines (bad): {has_p00006}")
|
|
|
|
has_p00006_clean = "p_00006 [BODY] in the name of φ1φ abundance" in prompt
|
|
print(f"p_00006 is clean (good): {has_p00006_clean}")
|
|
|
|
has_p00058 = "p_00058 [BODY] all-φ1φhands" in prompt
|
|
print(f"p_00058 is correct: {has_p00058}")
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(run_test())
|