feat: Release v0.10 - Modular Architecture & External Config
- Refactor codebase into src/ (preprocessing, translation, assembly) - Add pipeline/ scripts for individual stages - Externalize configuration to config/config.yaml - Fix Cover Image preservation - Update documentation and manuals
This commit is contained in:
谭凯
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Debug script to show chunk content and test short ID strategy.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from src.manifest_manager import ManifestManager
|
||||
from src.llm_client import LLMClient
|
||||
from src.data_model import ManifestEntry
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Configuration
|
||||
MANIFEST_PATH = Path("cache/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)_manifest.json")
|
||||
API_KEY = os.getenv("V3_API_KEY")
|
||||
BASE_URL = "https://api.gpt.ge/v1"
|
||||
MODEL = "gemini-3-flash-preview"
|
||||
EXTRA_HEADERS = {"x-foo": "true"}
|
||||
|
||||
|
||||
def show_current_chunk_format():
|
||||
"""Show current chunk format (problematic)."""
|
||||
print("\n" + "="*60)
|
||||
print("当前 Chunk 格式 (有问题)")
|
||||
print("="*60)
|
||||
|
||||
# Load manifest
|
||||
manager = ManifestManager(MANIFEST_PATH)
|
||||
manager.load()
|
||||
|
||||
# Get sample entries with placeholders
|
||||
samples = [e for e in manager.entries if e.placeholders][:3]
|
||||
|
||||
print("\n发送给 LLM 的格式 (当前):")
|
||||
print("-"*60)
|
||||
for item in samples:
|
||||
context = item.context or "BODY"
|
||||
print(f"{item.entry_id} [{context}] {item.original_text[:50]}...")
|
||||
|
||||
print("\n问题分析:")
|
||||
print(" 1. entry_id 太长 (包含 UUID): 容易被 LLM 截断或修改")
|
||||
print(" 2. [BODY] context 没必要发送")
|
||||
print(" 3. 依赖 LLM 精确复制长 ID，不可靠")
|
||||
|
||||
|
||||
def show_proposed_chunk_format():
|
||||
"""Show proposed short ID chunk format."""
|
||||
print("\n" + "="*60)
|
||||
print("建议的 Chunk 格式 (短 ID)")
|
||||
print("="*60)
|
||||
|
||||
manager = ManifestManager(MANIFEST_PATH)
|
||||
manager.load()
|
||||
|
||||
samples = [e for e in manager.entries if e.placeholders][:5]
|
||||
|
||||
print("\n发送给 LLM 的格式 (建议):")
|
||||
print("-"*60)
|
||||
|
||||
# Build with short IDs
|
||||
id_map = {} # short_id -> entry_id
|
||||
for i, item in enumerate(samples, 1):
|
||||
short_id = f"#{i}"
|
||||
id_map[short_id] = item.entry_id
|
||||
text = item.original_text[:60]
|
||||
print(f"{short_id}: {text}...")
|
||||
|
||||
print("\n期望 LLM 返回的格式:")
|
||||
print("-"*60)
|
||||
print("#1: φ1φpenguinrandomhouse.com（保持不翻译）")
|
||||
print("#2: 该产品在欧盟的产品安全授权代表为 φ1φPenguin Random House Irelandφ/1φ...")
|
||||
print("#3: φ1φ献辞")
|
||||
print("#4: φ1φ题记")
|
||||
print("#5: φ1φ作者说明")
|
||||
|
||||
print("\n优势:")
|
||||
print(" 1. 短 ID (#1, #2...) 不会被 LLM 弄乱")
|
||||
print(" 2. 去掉了无用的 context 标签")
|
||||
print(" 3. 解析更可靠：用正则 ^#(\\d+): 匹配")
|
||||
print(" 4. ID 映射表保留在代码中，用于还原")
|
||||
|
||||
print("\n映射表 (代码内保留):")
|
||||
for short_id, full_id in id_map.items():
|
||||
print(f" {short_id} -> {full_id[:50]}...")
|
||||
|
||||
|
||||
async def test_short_id_translation():
|
||||
"""Test translation with short ID format."""
|
||||
print("\n" + "="*60)
|
||||
print("测试短 ID 翻译")
|
||||
print("="*60)
|
||||
|
||||
manager = ManifestManager(MANIFEST_PATH)
|
||||
manager.load()
|
||||
|
||||
# Get 5 entries with varied content
|
||||
samples = [e for e in manager.entries if len(e.original_text) > 20][:5]
|
||||
|
||||
# Build prompt with short IDs
|
||||
id_map = {}
|
||||
lines = []
|
||||
for i, item in enumerate(samples, 1):
|
||||
short_id = f"#{i}"
|
||||
id_map[short_id] = item.entry_id
|
||||
text = item.original_text.replace('\n', ' ').strip()
|
||||
lines.append(f"{short_id}: {text}")
|
||||
|
||||
user_prompt = "\n".join(lines)
|
||||
|
||||
print("\n发送给 LLM 的 Prompt:")
|
||||
print("-"*60)
|
||||
print(user_prompt)
|
||||
|
||||
# Create client
|
||||
from openai import AsyncOpenAI
|
||||
client = AsyncOpenAI(
|
||||
base_url=BASE_URL,
|
||||
api_key=API_KEY,
|
||||
default_headers=EXTRA_HEADERS
|
||||
)
|
||||
|
||||
system_prompt = """You are a professional English to Chinese translator.
|
||||
|
||||
Translate each line to Chinese. Keep the format:
|
||||
- Each line starts with #N: (keep this ID exactly)
|
||||
- Preserve any φXφ placeholders exactly as-is
|
||||
- Only output translations, no explanations
|
||||
|
||||
Example input:
|
||||
#1: Hello world
|
||||
#2: φ1φClick hereφ/1φ to continue
|
||||
|
||||
Example output:
|
||||
#1: 你好世界
|
||||
#2: φ1φ点击这里φ/1φ 继续"""
|
||||
|
||||
print("\n发送请求...")
|
||||
|
||||
try:
|
||||
resp = await client.chat.completions.create(
|
||||
model=MODEL,
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt}
|
||||
],
|
||||
temperature=0.3,
|
||||
)
|
||||
raw_response = resp.choices[0].message.content.strip()
|
||||
|
||||
print("\nLLM 返回:")
|
||||
print("-"*60)
|
||||
print(raw_response)
|
||||
|
||||
# Parse with short ID
|
||||
print("\n解析结果:")
|
||||
print("-"*60)
|
||||
import re
|
||||
results = {}
|
||||
for line in raw_response.split("\n"):
|
||||
line = line.strip()
|
||||
match = re.match(r'^#(\d+):\s*(.+)$', line)
|
||||
if match:
|
||||
short_id = f"#{match.group(1)}"
|
||||
translation = match.group(2)
|
||||
if short_id in id_map:
|
||||
full_id = id_map[short_id]
|
||||
results[full_id] = translation
|
||||
print(f" {short_id} -> {translation[:40]}...")
|
||||
|
||||
print(f"\n成功解析: {len(results)}/{len(samples)}")
|
||||
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
|
||||
async def main():
|
||||
print("="*60)
|
||||
print("Chunk ID 策略分析与测试")
|
||||
print("="*60)
|
||||
|
||||
if not MANIFEST_PATH.exists():
|
||||
print(f"ERROR: Manifest not found at {MANIFEST_PATH}")
|
||||
return
|
||||
|
||||
# 1. Show current format (problems)
|
||||
show_current_chunk_format()
|
||||
|
||||
# 2. Show proposed format
|
||||
show_proposed_chunk_format()
|
||||
|
||||
# 3. Test short ID translation
|
||||
if API_KEY:
|
||||
await test_short_id_translation()
|
||||
else:
|
||||
print("\n跳过测试 (V3_API_KEY 未设置)")
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("分析完成")
|
||||
print("="*60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user