#!/usr/bin/env python3 """ 最小测试脚本:测试 Manifest -> TextProcessor -> LLMClient 的数据流 验证 text_with_placeholders 是否在传递过程中丢失或损坏 """ import sys import json import asyncio from pathlib import Path from loguru import logger # 添加 src 到路径 sys.path.insert(0, '.') from src.manifest_manager import ManifestManager, ManifestItem from src.text_processor import TextProcessor from src.llm_client import LLMClient # Mock config from src.utils import load_config # 配置日志 logger.remove() logger.add(sys.stdout, level="DEBUG") # 1. 创建临时的 Manifest 文件,模拟包含问题的真实数据 # 模拟一个带有换行符的占位符文本,这是我们怀疑的根源 mock_manifest_data = { "metadata": {"book_id": "test_book"}, "items": [ { "global_id": "p_00006", "source_file": "test.html", "original_html": "

in the name of abundance

", "clean_text": "in the name of abundance", "text_hash": "hash1", "tag": "p", # 模拟包含换行符的情况 (FormatExtractor 之前的问题) "text_with_placeholders": "in the name of \nφ1φ\n abundance", "placeholder_map": {"1": ""}, "paragraph_type": "BODY", "status": "pending" }, { "global_id": "p_00058", "source_file": "test.html", "original_html": "

all-hands

", "clean_text": "all-hands", "text_hash": "hash2", "tag": "p", # 正常情况 "text_with_placeholders": "all-φ1φhands", "placeholder_map": {"1": ""}, "paragraph_type": "BODY", "status": "pending" } ] } manifest_path = Path("cache/test_manifest.json") manifest_path.parent.mkdir(parents=True, exist_ok=True) with open(manifest_path, 'w') as f: json.dump(mock_manifest_data, f) print(f"=== Created Mock Manifest at {manifest_path} ===") async def run_test(): # 2. 加载 Manifest manifest = ManifestManager(str(manifest_path)) manifest.load() print(f"Loaded {len(manifest.get_items())} items") # 3. 创建 TextProcessor 和 Chunks # Mock config for processor processor = TextProcessor({"translation": {"chunk_size": 1000}}) # 这一步会从 manifest 读取 item chunks = processor.create_chunks_from_manifest(manifest, mode="chinese") print(f"Created {len(chunks)} chunks") chunk = chunks[0] print(f"Chunk 0 has {len(chunk)} items") # 4. 模拟 LLMClient 构建 Prompt # 不需要真正的 API key,只需要测试 _build_prompt # 添加 dummy key 和 rate_limits 防止初始化报错 client = LLMClient({ "providers": {}, "llm": { "api_key": "dummy_key", "models": {"fast": "dummy_model", "smart": "dummy_model"}, "rate_limits": {"requests_per_minute": 60, "concurrent_requests": 2} } }) # 强制重新加载 prompts (确保我们使用最新的代码逻辑) # 注意:我们之前修了 _load_prompts,如果 prompts.json 不存在会报错 # 这里我们假设 config/prompts.json 存在 print("\n=== 构建 Prompt (Mode: Chinese) ===") prompt = client._build_prompt(chunk, mode="chinese") print("-" * 40) print(prompt) print("-" * 40) # 验证关键点 print("\n=== 验证结果 ===") # 检查 p_00006 # 注意:我们之前修了 FormatExtractor,但那是针对**新提取**的内容。 # 这里我们测试的是**从旧 Manifest 读取**的内容。 # ManifestManager 读取时并不会自动清理换行符! # 所以如果旧 manifest 里有换行,这里应该能复现出带换行的 prompt。 has_p00006 = "p_00006 [BODY] in the name of \nφ1φ\n abundance" in prompt print(f"p_00006 has newlines (bad): {has_p00006}") has_p00006_clean = "p_00006 [BODY] in the name of φ1φ abundance" in prompt print(f"p_00006 is clean (good): {has_p00006_clean}") has_p00058 = "p_00058 [BODY] all-φ1φhands" in prompt print(f"p_00058 is correct: {has_p00058}") if __name__ == "__main__": asyncio.run(run_test())