#!/usr/bin/env python3 """ Debug script to show chunk content and test short ID strategy. """ import asyncio import json import os import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).parent.parent)) from dotenv import load_dotenv from src.manifest_manager import ManifestManager from src.llm_client import LLMClient from src.data_model import ManifestEntry load_dotenv() # Configuration MANIFEST_PATH = Path("cache/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)_manifest.json") API_KEY = os.getenv("V3_API_KEY") BASE_URL = "https://api.gpt.ge/v1" MODEL = "gemini-3-flash-preview" EXTRA_HEADERS = {"x-foo": "true"} def show_current_chunk_format(): """Show current chunk format (problematic).""" print("\n" + "="*60) print("当前 Chunk 格式 (有问题)") print("="*60) # Load manifest manager = ManifestManager(MANIFEST_PATH) manager.load() # Get sample entries with placeholders samples = [e for e in manager.entries if e.placeholders][:3] print("\n发送给 LLM 的格式 (当前):") print("-"*60) for item in samples: context = item.context or "BODY" print(f"{item.entry_id} [{context}] {item.original_text[:50]}...") print("\n问题分析:") print(" 1. entry_id 太长 (包含 UUID): 容易被 LLM 截断或修改") print(" 2. [BODY] context 没必要发送") print(" 3. 依赖 LLM 精确复制长 ID,不可靠") def show_proposed_chunk_format(): """Show proposed short ID chunk format.""" print("\n" + "="*60) print("建议的 Chunk 格式 (短 ID)") print("="*60) manager = ManifestManager(MANIFEST_PATH) manager.load() samples = [e for e in manager.entries if e.placeholders][:5] print("\n发送给 LLM 的格式 (建议):") print("-"*60) # Build with short IDs id_map = {} # short_id -> entry_id for i, item in enumerate(samples, 1): short_id = f"#{i}" id_map[short_id] = item.entry_id text = item.original_text[:60] print(f"{short_id}: {text}...") print("\n期望 LLM 返回的格式:") print("-"*60) print("#1: φ1φpenguinrandomhouse.com(保持不翻译)") print("#2: 该产品在欧盟的产品安全授权代表为 φ1φPenguin Random House Irelandφ/1φ...") print("#3: φ1φ献辞") print("#4: φ1φ题记") print("#5: φ1φ作者说明") print("\n优势:") print(" 1. 短 ID (#1, #2...) 不会被 LLM 弄乱") print(" 2. 去掉了无用的 context 标签") print(" 3. 解析更可靠:用正则 ^#(\\d+): 匹配") print(" 4. ID 映射表保留在代码中,用于还原") print("\n映射表 (代码内保留):") for short_id, full_id in id_map.items(): print(f" {short_id} -> {full_id[:50]}...") async def test_short_id_translation(): """Test translation with short ID format.""" print("\n" + "="*60) print("测试短 ID 翻译") print("="*60) manager = ManifestManager(MANIFEST_PATH) manager.load() # Get 5 entries with varied content samples = [e for e in manager.entries if len(e.original_text) > 20][:5] # Build prompt with short IDs id_map = {} lines = [] for i, item in enumerate(samples, 1): short_id = f"#{i}" id_map[short_id] = item.entry_id text = item.original_text.replace('\n', ' ').strip() lines.append(f"{short_id}: {text}") user_prompt = "\n".join(lines) print("\n发送给 LLM 的 Prompt:") print("-"*60) print(user_prompt) # Create client from openai import AsyncOpenAI client = AsyncOpenAI( base_url=BASE_URL, api_key=API_KEY, default_headers=EXTRA_HEADERS ) system_prompt = """You are a professional English to Chinese translator. Translate each line to Chinese. Keep the format: - Each line starts with #N: (keep this ID exactly) - Preserve any φXφ placeholders exactly as-is - Only output translations, no explanations Example input: #1: Hello world #2: φ1φClick hereφ/1φ to continue Example output: #1: 你好世界 #2: φ1φ点击这里φ/1φ 继续""" print("\n发送请求...") try: resp = await client.chat.completions.create( model=MODEL, messages=[ {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt} ], temperature=0.3, ) raw_response = resp.choices[0].message.content.strip() print("\nLLM 返回:") print("-"*60) print(raw_response) # Parse with short ID print("\n解析结果:") print("-"*60) import re results = {} for line in raw_response.split("\n"): line = line.strip() match = re.match(r'^#(\d+):\s*(.+)$', line) if match: short_id = f"#{match.group(1)}" translation = match.group(2) if short_id in id_map: full_id = id_map[short_id] results[full_id] = translation print(f" {short_id} -> {translation[:40]}...") print(f"\n成功解析: {len(results)}/{len(samples)}") finally: await client.close() async def main(): print("="*60) print("Chunk ID 策略分析与测试") print("="*60) if not MANIFEST_PATH.exists(): print(f"ERROR: Manifest not found at {MANIFEST_PATH}") return # 1. Show current format (problems) show_current_chunk_format() # 2. Show proposed format show_proposed_chunk_format() # 3. Test short ID translation if API_KEY: await test_short_id_translation() else: print("\n跳过测试 (V3_API_KEY 未设置)") print("\n" + "="*60) print("分析完成") print("="*60) if __name__ == "__main__": asyncio.run(main())