feat: Release v0.10 - Modular Architecture & External Config
- Refactor codebase into src/ (preprocessing, translation, assembly) - Add pipeline/ scripts for individual stages - Externalize configuration to config/config.yaml - Fix Cover Image preservation - Update documentation and manuals
This commit is contained in:
谭凯
@@ -0,0 +1,51 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import traceback
|
||||
|
||||
# Add src to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.data_model import BookStructure, ManifestEntry
|
||||
from src.manifest_manager import ManifestManager
|
||||
from src.backfill_engine import BackfillEngine
|
||||
from src.bilingual_builder import BilingualBuilder
|
||||
from src.utils import setup_logger
|
||||
|
||||
logger = setup_logger("build_final")
|
||||
|
||||
def build_final():
|
||||
BOOK_NAME = "Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)"
|
||||
WORK_DIR = Path(f".work/{BOOK_NAME}")
|
||||
MANIFEST_PATH = WORK_DIR / "manifest.json"
|
||||
STRUCTURE_PATH = WORK_DIR / "book_structure.json"
|
||||
OUTPUT_EPUB = Path("output/final_verification.epub")
|
||||
INPUT_EPUB = Path(f"input/{BOOK_NAME}.epub")
|
||||
|
||||
if not MANIFEST_PATH.exists() or not STRUCTURE_PATH.exists():
|
||||
logger.error("Missing manifest or structure. Run translation first.")
|
||||
return
|
||||
|
||||
# 1. Load Data
|
||||
logger.info("Loading structure and manifest...")
|
||||
structure = BookStructure.load(STRUCTURE_PATH)
|
||||
manager = ManifestManager(MANIFEST_PATH)
|
||||
manager.load()
|
||||
|
||||
# 2. Backfill
|
||||
logger.info("Backfilling translations...")
|
||||
backfiller = BackfillEngine()
|
||||
structure = backfiller.backfill(structure, manager.entries, mode="bilingual")
|
||||
|
||||
# 3. Build
|
||||
logger.info(f"Building final EPUB to {OUTPUT_EPUB}...")
|
||||
builder = BilingualBuilder(WORK_DIR, original_epub_path=INPUT_EPUB)
|
||||
builder.build(structure, OUTPUT_EPUB)
|
||||
|
||||
logger.info("Build complete.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
build_final()
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print(f"CRITICAL ERROR: {e}")
|
||||
@@ -0,0 +1,102 @@
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
import asyncio
|
||||
import httpx
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
# Add project root to sys.path
|
||||
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
|
||||
from src.common.config import load_global_config
|
||||
|
||||
async def main():
|
||||
print("--- Environment Debug ---")
|
||||
load_dotenv()
|
||||
env_key = os.getenv("OPENAI_API_KEY")
|
||||
if env_key:
|
||||
print(f"OPENAI_API_KEY found in env: {env_key[:8]}...{env_key[-4:]}")
|
||||
else:
|
||||
print("OPENAI_API_KEY NOT found in env!")
|
||||
|
||||
print("\n--- Config Loader Debug ---")
|
||||
try:
|
||||
config = load_global_config()
|
||||
llm_conf = config.get("llm", {})
|
||||
conf_key = llm_conf.get("api_key")
|
||||
base_url = llm_conf.get("base_url")
|
||||
model = llm_conf.get("model")
|
||||
|
||||
print(f"Config Base URL: {base_url}")
|
||||
print(f"Config Model: {model}")
|
||||
if conf_key:
|
||||
print(f"Config API Key: {conf_key[:8]}...{conf_key[-4:]}")
|
||||
if env_key and conf_key == env_key:
|
||||
print("Config Key matches Env Key.")
|
||||
else:
|
||||
print("Config Key DOES NOT match Env Key!")
|
||||
else:
|
||||
print("Config API Key NOT found!")
|
||||
|
||||
print("\n--- API Connectivity Test ---")
|
||||
if not conf_key or not base_url:
|
||||
print("Missing params for test.")
|
||||
return
|
||||
|
||||
headers = {"Authorization": f"Bearer {conf_key}"}
|
||||
url = f"{base_url}/models"
|
||||
print(f"Requesting: {url}")
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
try:
|
||||
resp = await client.get(url, headers=headers, timeout=10)
|
||||
print(f"Status Code: {resp.status_code}")
|
||||
if resp.status_code == 200:
|
||||
print("Success! Models listed.")
|
||||
else:
|
||||
print(f"Failed. Response: {resp.text}")
|
||||
except Exception as e:
|
||||
print(f"Exception during request: {e}")
|
||||
|
||||
print("\n--- Chat Completion Test (Mimicking LLMClient) ---")
|
||||
|
||||
proxy_url = os.environ.get("http_proxy") or os.environ.get("https_proxy")
|
||||
print(f"Proxy detected: {proxy_url}")
|
||||
|
||||
http_client = httpx.AsyncClient(
|
||||
proxy=proxy_url,
|
||||
timeout=60.0,
|
||||
follow_redirects=True
|
||||
) if proxy_url else None
|
||||
|
||||
aclient = AsyncOpenAI(
|
||||
api_key=conf_key,
|
||||
base_url=base_url,
|
||||
http_client=http_client
|
||||
)
|
||||
|
||||
print(f"Model: {model}")
|
||||
system_prompt = "You are a senior publishing editor."
|
||||
user_prompt = "Analyze this text."
|
||||
|
||||
try:
|
||||
print("Sending request with System Prompt...")
|
||||
resp = await aclient.chat.completions.create(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt}
|
||||
],
|
||||
temperature=0.3,
|
||||
)
|
||||
print("Success!")
|
||||
print(f"Response: {resp.choices[0].message.content}")
|
||||
except Exception as e:
|
||||
print(f"Chat Completion failed: {type(e).__name__}: {e}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Config loading failed: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,36 @@
|
||||
import json
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
path = ".work/Gambling Man/book_structure.json"
|
||||
target_file = "e9781668070765/xhtml/ch08.xhtml"
|
||||
target_id = "uuid-100309c1-0643-4aa1-8ffb-1d2bfbd0e376"
|
||||
|
||||
try:
|
||||
with open(path, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
target_res = None
|
||||
for item_id, res in data['resources'].items():
|
||||
if "ch08.xhtml" in res.get('href', ''):
|
||||
print(f"Found resource with ID: {item_id}, href: {res.get('href')}")
|
||||
target_res = res
|
||||
break
|
||||
|
||||
if not target_res:
|
||||
print(f"Resource {target_file} not found by href search.")
|
||||
exit(1)
|
||||
|
||||
content = target_res['content']
|
||||
soup = BeautifulSoup(content, 'html.parser')
|
||||
element = soup.find(id=target_id)
|
||||
|
||||
if element:
|
||||
print(f"--- HTML for {target_id} ---")
|
||||
print(element.prettify())
|
||||
print("--- Raw ---")
|
||||
print(str(element))
|
||||
else:
|
||||
print("Element not found")
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Placeholder Backfill Test Script
|
||||
Tests placeholder restoration on translated entries.
|
||||
|
||||
Usage:
|
||||
python scripts/test_backfill.py --chapter 38
|
||||
python scripts/test_backfill.py --chapter 38 --limit 10
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.manifest_manager import ManifestManager
|
||||
from src.format_restorer import FormatRestorer
|
||||
|
||||
# Configuration - Use unified .work directory
|
||||
BOOK_NAME = "Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)"
|
||||
WORK_DIR = Path(f".work/{BOOK_NAME}")
|
||||
MANIFEST_PATH = WORK_DIR / "manifest.json"
|
||||
|
||||
|
||||
def build_toc_from_manifest(manager: ManifestManager) -> list:
|
||||
"""Build TOC from manifest entries."""
|
||||
files = {}
|
||||
for entry in manager.entries:
|
||||
fp = entry.file_path
|
||||
if fp not in files:
|
||||
files[fp] = {'count': 0, 'first_text': ''}
|
||||
files[fp]['count'] += 1
|
||||
if not files[fp]['first_text'] and entry.original_text:
|
||||
files[fp]['first_text'] = entry.original_text[:40].replace('\n', ' ')
|
||||
|
||||
toc = []
|
||||
for i, (fp, info) in enumerate(sorted(files.items()), 1):
|
||||
toc.append({
|
||||
'index': i,
|
||||
'href': fp,
|
||||
'title': info['first_text'] or f"File {i}",
|
||||
'paragraphs': info['count']
|
||||
})
|
||||
return toc
|
||||
|
||||
|
||||
def get_chapter_entries(manager: ManifestManager, chapter_index: int) -> tuple:
|
||||
"""Get entries for a specific chapter by index."""
|
||||
toc = build_toc_from_manifest(manager)
|
||||
|
||||
if chapter_index < 1 or chapter_index > len(toc):
|
||||
print(f"错误: 章节编号 {chapter_index} 无效 (范围: 1-{len(toc)})")
|
||||
return None, None
|
||||
|
||||
chapter = toc[chapter_index - 1]
|
||||
href = chapter['href']
|
||||
|
||||
entries = [e for e in manager.entries if e.file_path == href]
|
||||
return chapter, entries
|
||||
|
||||
|
||||
def test_backfill(chapter: dict, entries: list, limit: int = None):
|
||||
"""Test placeholder restoration for a chapter."""
|
||||
|
||||
print(f"\n" + "=" * 70)
|
||||
print(f"占位符回填测试 - 章节 #{chapter['index']}: {chapter['title'][:40]}...")
|
||||
print("=" * 70)
|
||||
|
||||
# Filter entries with translation and placeholders
|
||||
translated = [e for e in entries if e.translated_text]
|
||||
with_placeholders = [e for e in translated if e.placeholders and len(e.placeholders) > 0]
|
||||
|
||||
print(f"\n统计:")
|
||||
print(f" 总段落: {len(entries)}")
|
||||
print(f" 已翻译: {len(translated)}")
|
||||
print(f" 有占位符: {len(with_placeholders)}")
|
||||
|
||||
if not translated:
|
||||
print("\n⚠️ 该章节没有已翻译的内容!")
|
||||
return
|
||||
|
||||
# Test restoration
|
||||
restorer = FormatRestorer()
|
||||
|
||||
success_count = 0
|
||||
fail_count = 0
|
||||
results = []
|
||||
|
||||
test_entries = with_placeholders[:limit] if limit else with_placeholders
|
||||
|
||||
print(f"\n测试 {len(test_entries)} 个带占位符的段落:")
|
||||
print("-" * 70)
|
||||
|
||||
for i, entry in enumerate(test_entries, 1):
|
||||
original = entry.original_text
|
||||
translated = entry.translated_text
|
||||
placeholders = entry.placeholders
|
||||
|
||||
# Get non-internal placeholders
|
||||
visible_ph = {k: v for k, v in placeholders.items() if not k.startswith('_')}
|
||||
|
||||
# Perform restoration
|
||||
restored, success = restorer.restore(translated, placeholders)
|
||||
|
||||
if success:
|
||||
success_count += 1
|
||||
status = "✅"
|
||||
else:
|
||||
fail_count += 1
|
||||
status = "❌"
|
||||
|
||||
results.append({
|
||||
'index': i,
|
||||
'entry_id': entry.entry_id,
|
||||
'original': original,
|
||||
'translated': translated,
|
||||
'restored': restored,
|
||||
'placeholders': visible_ph,
|
||||
'success': success
|
||||
})
|
||||
|
||||
# Print summary
|
||||
print(f"\n[{i}] {status} {entry.entry_id[-40:]}")
|
||||
print(f" 占位符: {list(visible_ph.keys())}")
|
||||
print(f" 原文: {original[:50]}...")
|
||||
print(f" 译文: {translated[:50]}...")
|
||||
|
||||
if not success:
|
||||
print(f" 还原: {restored[:50]}...")
|
||||
# Show what placeholders are missing
|
||||
missing = []
|
||||
for k in visible_ph.keys():
|
||||
if k.isdigit():
|
||||
if f"φ{k}φ" not in translated and f"φ/{k}φ" not in translated:
|
||||
missing.append(k)
|
||||
if missing:
|
||||
print(f" 缺失: {missing}")
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 70)
|
||||
print(f"测试结果汇总")
|
||||
print("=" * 70)
|
||||
print(f" 成功: {success_count}/{len(test_entries)}")
|
||||
print(f" 失败: {fail_count}/{len(test_entries)}")
|
||||
|
||||
if fail_count > 0:
|
||||
print(f"\n失败案例详情:")
|
||||
for r in results:
|
||||
if not r['success']:
|
||||
print(f"\n [{r['index']}] {r['entry_id'][-50:]}")
|
||||
print(f" 原文: {r['original'][:60]}...")
|
||||
print(f" 译文: {r['translated'][:60]}...")
|
||||
print(f" 还原: {r['restored'][:60]}...")
|
||||
print(f" 占位符: {r['placeholders']}")
|
||||
|
||||
# Also test entries without visible placeholders (only _prefix/_suffix)
|
||||
prefix_suffix_only = [e for e in translated
|
||||
if e.placeholders
|
||||
and all(k.startswith('_') for k in e.placeholders.keys())]
|
||||
|
||||
if prefix_suffix_only:
|
||||
print(f"\n\n额外测试: 只有 _prefix/_suffix 的段落 ({len(prefix_suffix_only)} 个)")
|
||||
print("-" * 70)
|
||||
|
||||
ps_success = 0
|
||||
ps_fail = 0
|
||||
|
||||
for entry in prefix_suffix_only[:5]: # Test first 5
|
||||
restored, success = restorer.restore(entry.translated_text, entry.placeholders)
|
||||
if success:
|
||||
ps_success += 1
|
||||
status = "✅"
|
||||
else:
|
||||
ps_fail += 1
|
||||
status = "❌"
|
||||
|
||||
print(f" {status} {entry.entry_id[-40:]}")
|
||||
if '_prefix' in entry.placeholders:
|
||||
print(f" _prefix: {entry.placeholders['_prefix'][:30]}...")
|
||||
if '_suffix' in entry.placeholders:
|
||||
print(f" _suffix: {entry.placeholders['_suffix'][:30]}...")
|
||||
|
||||
print(f"\n 结果: {ps_success}/{min(5, len(prefix_suffix_only))} 成功")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="测试占位符回填")
|
||||
parser.add_argument("--chapter", "-c", type=int, required=True, help="章节编号")
|
||||
parser.add_argument("--limit", "-l", type=int, default=20, help="测试数量限制 (默认20)")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not MANIFEST_PATH.exists():
|
||||
print(f"错误: Manifest 不存在: {MANIFEST_PATH}")
|
||||
return
|
||||
|
||||
# Load manifest
|
||||
manager = ManifestManager(MANIFEST_PATH)
|
||||
manager.load()
|
||||
print(f"已加载 manifest: {len(manager.entries)} 条目")
|
||||
|
||||
# Get chapter
|
||||
chapter, entries = get_chapter_entries(manager, args.chapter)
|
||||
if not chapter:
|
||||
return
|
||||
|
||||
# Test backfill
|
||||
test_backfill(chapter, entries, args.limit)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,25 @@
|
||||
import asyncio
|
||||
import os
|
||||
import httpx
|
||||
|
||||
async def test_conn():
|
||||
print(f"HTTP_PROXY: {os.environ.get('http_proxy')}")
|
||||
print(f"HTTPS_PROXY: {os.environ.get('https_proxy')}")
|
||||
print(f"ALL_PROXY: {os.environ.get('all_proxy')}")
|
||||
|
||||
url = "https://api.gpt.ge/v1/models"
|
||||
headers = {"Authorization": f"Bearer {os.environ.get('V3_API_KEY')}"}
|
||||
|
||||
print(f"Connecting to {url}...")
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10, follow_redirects=True) as client:
|
||||
resp = await client.get(url, headers=headers)
|
||||
print(f"Status: {resp.status_code}")
|
||||
print(f"Headers: {resp.headers}")
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
asyncio.run(test_conn())
|
||||
@@ -0,0 +1,40 @@
|
||||
import asyncio
|
||||
import os
|
||||
import httpx
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
async def test_openai():
|
||||
proxy_url = os.environ.get("http_proxy")
|
||||
print(f"Using proxy: {proxy_url}")
|
||||
|
||||
http_client = httpx.AsyncClient(
|
||||
proxy=proxy_url,
|
||||
timeout=30.0,
|
||||
follow_redirects=True
|
||||
)
|
||||
|
||||
client = AsyncOpenAI(
|
||||
base_url="https://api.gpt.ge/v1",
|
||||
api_key=os.environ.get("V3_API_KEY"),
|
||||
http_client=http_client
|
||||
)
|
||||
|
||||
print("Sending request...")
|
||||
try:
|
||||
response = await client.chat.completions.create(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
max_tokens=5
|
||||
)
|
||||
print(f"Response: {response.choices[0].message.content}")
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
print(f"Error: {e}")
|
||||
finally:
|
||||
await http_client.aclose()
|
||||
|
||||
if __name__ == "__main__":
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
asyncio.run(test_openai())
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Debug script to show chunk content and test short ID strategy.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from src.manifest_manager import ManifestManager
|
||||
from src.llm_client import LLMClient
|
||||
from src.data_model import ManifestEntry
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Configuration
|
||||
MANIFEST_PATH = Path("cache/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)_manifest.json")
|
||||
API_KEY = os.getenv("V3_API_KEY")
|
||||
BASE_URL = "https://api.gpt.ge/v1"
|
||||
MODEL = "gemini-3-flash-preview"
|
||||
EXTRA_HEADERS = {"x-foo": "true"}
|
||||
|
||||
|
||||
def show_current_chunk_format():
|
||||
"""Show current chunk format (problematic)."""
|
||||
print("\n" + "="*60)
|
||||
print("当前 Chunk 格式 (有问题)")
|
||||
print("="*60)
|
||||
|
||||
# Load manifest
|
||||
manager = ManifestManager(MANIFEST_PATH)
|
||||
manager.load()
|
||||
|
||||
# Get sample entries with placeholders
|
||||
samples = [e for e in manager.entries if e.placeholders][:3]
|
||||
|
||||
print("\n发送给 LLM 的格式 (当前):")
|
||||
print("-"*60)
|
||||
for item in samples:
|
||||
context = item.context or "BODY"
|
||||
print(f"{item.entry_id} [{context}] {item.original_text[:50]}...")
|
||||
|
||||
print("\n问题分析:")
|
||||
print(" 1. entry_id 太长 (包含 UUID): 容易被 LLM 截断或修改")
|
||||
print(" 2. [BODY] context 没必要发送")
|
||||
print(" 3. 依赖 LLM 精确复制长 ID，不可靠")
|
||||
|
||||
|
||||
def show_proposed_chunk_format():
|
||||
"""Show proposed short ID chunk format."""
|
||||
print("\n" + "="*60)
|
||||
print("建议的 Chunk 格式 (短 ID)")
|
||||
print("="*60)
|
||||
|
||||
manager = ManifestManager(MANIFEST_PATH)
|
||||
manager.load()
|
||||
|
||||
samples = [e for e in manager.entries if e.placeholders][:5]
|
||||
|
||||
print("\n发送给 LLM 的格式 (建议):")
|
||||
print("-"*60)
|
||||
|
||||
# Build with short IDs
|
||||
id_map = {} # short_id -> entry_id
|
||||
for i, item in enumerate(samples, 1):
|
||||
short_id = f"#{i}"
|
||||
id_map[short_id] = item.entry_id
|
||||
text = item.original_text[:60]
|
||||
print(f"{short_id}: {text}...")
|
||||
|
||||
print("\n期望 LLM 返回的格式:")
|
||||
print("-"*60)
|
||||
print("#1: φ1φpenguinrandomhouse.com（保持不翻译）")
|
||||
print("#2: 该产品在欧盟的产品安全授权代表为 φ1φPenguin Random House Irelandφ/1φ...")
|
||||
print("#3: φ1φ献辞")
|
||||
print("#4: φ1φ题记")
|
||||
print("#5: φ1φ作者说明")
|
||||
|
||||
print("\n优势:")
|
||||
print(" 1. 短 ID (#1, #2...) 不会被 LLM 弄乱")
|
||||
print(" 2. 去掉了无用的 context 标签")
|
||||
print(" 3. 解析更可靠：用正则 ^#(\\d+): 匹配")
|
||||
print(" 4. ID 映射表保留在代码中，用于还原")
|
||||
|
||||
print("\n映射表 (代码内保留):")
|
||||
for short_id, full_id in id_map.items():
|
||||
print(f" {short_id} -> {full_id[:50]}...")
|
||||
|
||||
|
||||
async def test_short_id_translation():
|
||||
"""Test translation with short ID format."""
|
||||
print("\n" + "="*60)
|
||||
print("测试短 ID 翻译")
|
||||
print("="*60)
|
||||
|
||||
manager = ManifestManager(MANIFEST_PATH)
|
||||
manager.load()
|
||||
|
||||
# Get 5 entries with varied content
|
||||
samples = [e for e in manager.entries if len(e.original_text) > 20][:5]
|
||||
|
||||
# Build prompt with short IDs
|
||||
id_map = {}
|
||||
lines = []
|
||||
for i, item in enumerate(samples, 1):
|
||||
short_id = f"#{i}"
|
||||
id_map[short_id] = item.entry_id
|
||||
text = item.original_text.replace('\n', ' ').strip()
|
||||
lines.append(f"{short_id}: {text}")
|
||||
|
||||
user_prompt = "\n".join(lines)
|
||||
|
||||
print("\n发送给 LLM 的 Prompt:")
|
||||
print("-"*60)
|
||||
print(user_prompt)
|
||||
|
||||
# Create client
|
||||
from openai import AsyncOpenAI
|
||||
client = AsyncOpenAI(
|
||||
base_url=BASE_URL,
|
||||
api_key=API_KEY,
|
||||
default_headers=EXTRA_HEADERS
|
||||
)
|
||||
|
||||
system_prompt = """You are a professional English to Chinese translator.
|
||||
|
||||
Translate each line to Chinese. Keep the format:
|
||||
- Each line starts with #N: (keep this ID exactly)
|
||||
- Preserve any φXφ placeholders exactly as-is
|
||||
- Only output translations, no explanations
|
||||
|
||||
Example input:
|
||||
#1: Hello world
|
||||
#2: φ1φClick hereφ/1φ to continue
|
||||
|
||||
Example output:
|
||||
#1: 你好世界
|
||||
#2: φ1φ点击这里φ/1φ 继续"""
|
||||
|
||||
print("\n发送请求...")
|
||||
|
||||
try:
|
||||
resp = await client.chat.completions.create(
|
||||
model=MODEL,
|
||||
messages=[
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt}
|
||||
],
|
||||
temperature=0.3,
|
||||
)
|
||||
raw_response = resp.choices[0].message.content.strip()
|
||||
|
||||
print("\nLLM 返回:")
|
||||
print("-"*60)
|
||||
print(raw_response)
|
||||
|
||||
# Parse with short ID
|
||||
print("\n解析结果:")
|
||||
print("-"*60)
|
||||
import re
|
||||
results = {}
|
||||
for line in raw_response.split("\n"):
|
||||
line = line.strip()
|
||||
match = re.match(r'^#(\d+):\s*(.+)$', line)
|
||||
if match:
|
||||
short_id = f"#{match.group(1)}"
|
||||
translation = match.group(2)
|
||||
if short_id in id_map:
|
||||
full_id = id_map[short_id]
|
||||
results[full_id] = translation
|
||||
print(f" {short_id} -> {translation[:40]}...")
|
||||
|
||||
print(f"\n成功解析: {len(results)}/{len(samples)}")
|
||||
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
|
||||
async def main():
|
||||
print("="*60)
|
||||
print("Chunk ID 策略分析与测试")
|
||||
print("="*60)
|
||||
|
||||
if not MANIFEST_PATH.exists():
|
||||
print(f"ERROR: Manifest not found at {MANIFEST_PATH}")
|
||||
return
|
||||
|
||||
# 1. Show current format (problems)
|
||||
show_current_chunk_format()
|
||||
|
||||
# 2. Show proposed format
|
||||
show_proposed_chunk_format()
|
||||
|
||||
# 3. Test short ID translation
|
||||
if API_KEY:
|
||||
await test_short_id_translation()
|
||||
else:
|
||||
print("\n跳过测试 (V3_API_KEY 未设置)")
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("分析完成")
|
||||
print("="*60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,355 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Translation Pipeline Debug Script
|
||||
Shows visible results at each step of the translation process.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Add project root to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
from src.manifest_manager import ManifestManager
|
||||
from src.llm_client import LLMClient
|
||||
from src.book_profiler import BookProfiler
|
||||
from src.translator import Translator
|
||||
from src.format_restorer import FormatRestorer
|
||||
from src.data_model import ManifestEntry, BookProfile
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Configuration
|
||||
MANIFEST_PATH = Path("cache/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)_manifest.json")
|
||||
API_KEY = os.getenv("V3_API_KEY")
|
||||
BASE_URL = "https://api.gpt.ge/v1"
|
||||
MODEL = "gemini-3-flash-preview"
|
||||
EXTRA_HEADERS = {"x-foo": "true"}
|
||||
|
||||
# Chunk size configuration
|
||||
MAX_CHUNK_SIZE = 15 # Maximum entries per chunk
|
||||
|
||||
|
||||
def group_entries_by_file(entries: list) -> dict:
|
||||
"""Group manifest entries by their source file (chapter)."""
|
||||
grouped = {}
|
||||
for entry in entries:
|
||||
file_path = entry.file_path
|
||||
if file_path not in grouped:
|
||||
grouped[file_path] = []
|
||||
grouped[file_path].append(entry)
|
||||
return grouped
|
||||
|
||||
|
||||
def create_chapter_aware_chunks(entries: list, max_size: int = MAX_CHUNK_SIZE) -> list:
|
||||
"""
|
||||
Create chunks that respect chapter boundaries.
|
||||
Returns list of (file_path, chunk_entries) tuples.
|
||||
"""
|
||||
grouped = group_entries_by_file(entries)
|
||||
chunks = []
|
||||
|
||||
for file_path, file_entries in grouped.items():
|
||||
# Split this file's entries into chunks of max_size
|
||||
for i in range(0, len(file_entries), max_size):
|
||||
chunk = file_entries[i:i + max_size]
|
||||
chunks.append((file_path, chunk))
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
def get_file_type(file_path: str) -> str:
|
||||
"""Determine the type of content based on file name."""
|
||||
fname = file_path.lower()
|
||||
if any(k in fname for k in ['toc', 'contents', 'nav']):
|
||||
return 'toc'
|
||||
elif any(k in fname for k in ['title', 'cover']):
|
||||
return 'cover'
|
||||
elif any(k in fname for k in ['copyright', 'colophon']):
|
||||
return 'legal'
|
||||
elif any(k in fname for k in ['author', 'about']):
|
||||
return 'author_bio'
|
||||
elif any(k in fname for k in ['index', 'bibliography', 'endnote', 'footnote']):
|
||||
return 'reference'
|
||||
else:
|
||||
return 'body'
|
||||
|
||||
|
||||
async def step1_load_manifest():
|
||||
"""Step 1: Load manifest and show statistics."""
|
||||
print("\n" + "="*60)
|
||||
print("STEP 1: Loading Manifest")
|
||||
print("="*60)
|
||||
|
||||
manager = ManifestManager(MANIFEST_PATH)
|
||||
manager.load()
|
||||
|
||||
entries = manager.entries
|
||||
untranslated = [e for e in entries if not e.translated_text]
|
||||
|
||||
print(f" Total entries: {len(entries)}")
|
||||
print(f" Untranslated: {len(untranslated)}")
|
||||
|
||||
# Show grouping by file
|
||||
grouped = group_entries_by_file(entries)
|
||||
print(f" Unique files: {len(grouped)}")
|
||||
|
||||
# Show sample entry
|
||||
if entries:
|
||||
sample = entries[0]
|
||||
print(f"\n Sample entry:")
|
||||
print(f" ID: {sample.entry_id}")
|
||||
print(f" File: {sample.file_path}")
|
||||
print(f" Original: {sample.original_text[:80]}...")
|
||||
print(f" Placeholders: {sample.placeholders}")
|
||||
|
||||
return manager
|
||||
|
||||
|
||||
async def step2_profile_book(manager: ManifestManager, llm_client: LLMClient):
|
||||
"""Step 2: Generate book profile."""
|
||||
print("\n" + "="*60)
|
||||
print("STEP 2: Generating Book Profile")
|
||||
print("="*60)
|
||||
|
||||
profiler = BookProfiler(llm_client)
|
||||
profile = await profiler.analyze(manager.entries)
|
||||
|
||||
print(f" Title: {profile.title}")
|
||||
print(f" Author: {profile.author}")
|
||||
print(f" Genre: {profile.genre}")
|
||||
print(f" Keywords: {profile.keywords}")
|
||||
print(f" Style Guide: {profile.style_guide[:200]}..." if profile.style_guide else " Style Guide: (none)")
|
||||
|
||||
return profile
|
||||
|
||||
|
||||
async def step3_create_chunks(manager: ManifestManager):
|
||||
"""Step 3: Create chapter-aware chunks."""
|
||||
print("\n" + "="*60)
|
||||
print("STEP 3: Creating Chapter-Aware Chunks")
|
||||
print("="*60)
|
||||
|
||||
untranslated = [e for e in manager.entries if not e.translated_text]
|
||||
chunks = create_chapter_aware_chunks(untranslated)
|
||||
|
||||
print(f" Total chunks: {len(chunks)}")
|
||||
|
||||
# Show chunk distribution
|
||||
print(f"\n Chunk distribution by file type:")
|
||||
type_counts = {}
|
||||
for file_path, chunk_entries in chunks:
|
||||
ftype = get_file_type(file_path)
|
||||
type_counts[ftype] = type_counts.get(ftype, 0) + 1
|
||||
|
||||
for ftype, count in sorted(type_counts.items()):
|
||||
print(f" {ftype}: {count} chunks")
|
||||
|
||||
# Show first 3 chunks
|
||||
print(f"\n First 3 chunks:")
|
||||
for i, (file_path, chunk_entries) in enumerate(chunks[:3]):
|
||||
ftype = get_file_type(file_path)
|
||||
print(f" [{i}] {file_path} ({ftype}): {len(chunk_entries)} entries")
|
||||
if chunk_entries:
|
||||
print(f" First: {chunk_entries[0].original_text[:50]}...")
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
async def step4_translate_sample(chunks: list, llm_client: LLMClient, profile: BookProfile):
|
||||
"""Step 4: Translate a sample chunk and show results."""
|
||||
print("\n" + "="*60)
|
||||
print("STEP 4: Translating Sample Chunk")
|
||||
print("="*60)
|
||||
|
||||
if not chunks:
|
||||
print(" No chunks to translate!")
|
||||
return
|
||||
|
||||
# Pick a proper body chapter (skip first few files which are usually cover/copyright/toc)
|
||||
sample_chunk = None
|
||||
skip_prefixes = ['cM', 'c9', 'c18'] # Cover, title, contents pages
|
||||
for file_path, chunk_entries in chunks:
|
||||
# Skip non-body files and known cover/toc files
|
||||
ftype = get_file_type(file_path)
|
||||
fname = Path(file_path).stem
|
||||
if ftype == 'body' and fname not in skip_prefixes and len(chunk_entries) > 3:
|
||||
sample_chunk = (file_path, chunk_entries[:5]) # Limit to 5 entries for demo
|
||||
break
|
||||
|
||||
if not sample_chunk:
|
||||
# Fallback to any body chunk
|
||||
for file_path, chunk_entries in chunks:
|
||||
if get_file_type(file_path) == 'body':
|
||||
sample_chunk = (file_path, chunk_entries[:5])
|
||||
break
|
||||
|
||||
if not sample_chunk:
|
||||
sample_chunk = chunks[0]
|
||||
sample_chunk = (sample_chunk[0], sample_chunk[1][:5])
|
||||
|
||||
file_path, entries = sample_chunk
|
||||
ftype = get_file_type(file_path)
|
||||
|
||||
print(f" Selected chunk: {file_path} ({ftype})")
|
||||
print(f" Entries: {len(entries)}")
|
||||
|
||||
# Show entries before translation
|
||||
print(f"\n === Before Translation ===")
|
||||
for i, entry in enumerate(entries):
|
||||
print(f" [{i}] {entry.entry_id}")
|
||||
print(f" Original: {entry.original_text[:60]}...")
|
||||
if entry.placeholders:
|
||||
print(f" Placeholders: {list(entry.placeholders.keys())}")
|
||||
|
||||
# Translate
|
||||
print(f"\n Translating...")
|
||||
results = await llm_client.translate_chunk(
|
||||
entries,
|
||||
instruction=profile.style_guide,
|
||||
mode="bilingual"
|
||||
)
|
||||
|
||||
# Apply results and show
|
||||
print(f"\n === After Translation ===")
|
||||
restorer = FormatRestorer()
|
||||
for i, entry in enumerate(entries):
|
||||
if entry.entry_id in results:
|
||||
translated = results[entry.entry_id]
|
||||
entry.translated_text = translated
|
||||
|
||||
print(f" [{i}] {entry.entry_id}")
|
||||
print(f" Original: {entry.original_text[:50]}...")
|
||||
print(f" Translated: {translated[:50]}...")
|
||||
|
||||
# Restore format
|
||||
if entry.placeholders:
|
||||
restored, success = restorer.restore(translated, entry.placeholders)
|
||||
print(f" Restored OK: {success}")
|
||||
if not success:
|
||||
print(f" Restored: {restored[:50]}...")
|
||||
else:
|
||||
print(f" [{i}] MISSING: {entry.entry_id}")
|
||||
|
||||
return entries
|
||||
|
||||
|
||||
async def step5_test_placeholders(manager: ManifestManager, llm_client: LLMClient, profile: BookProfile):
|
||||
"""Step 5: Test placeholder handling with entries that have placeholders."""
|
||||
print("\n" + "="*60)
|
||||
print("STEP 5: Testing Placeholder Handling")
|
||||
print("="*60)
|
||||
|
||||
# Find entries with placeholders
|
||||
entries_with_ph = [e for e in manager.entries if e.placeholders and len(e.placeholders) > 1]
|
||||
|
||||
print(f" Entries with placeholders: {len(entries_with_ph)}")
|
||||
|
||||
if not entries_with_ph:
|
||||
print(" No entries with placeholders found!")
|
||||
return
|
||||
|
||||
# Pick 5 diverse samples
|
||||
samples = entries_with_ph[:5]
|
||||
|
||||
print(f"\n === Selected Samples ({len(samples)}) ===")
|
||||
for i, entry in enumerate(samples):
|
||||
ph_keys = [k for k in entry.placeholders.keys() if not k.startswith('_')]
|
||||
print(f" [{i}] {entry.entry_id}")
|
||||
print(f" Original: {entry.original_text[:60]}...")
|
||||
print(f" Placeholders: {ph_keys}")
|
||||
|
||||
# Translate
|
||||
print(f"\n Translating {len(samples)} entries with placeholders...")
|
||||
results = await llm_client.translate_chunk(
|
||||
samples,
|
||||
instruction=profile.style_guide,
|
||||
mode="bilingual"
|
||||
)
|
||||
|
||||
# Show results with restoration
|
||||
print(f"\n === Translation Results ===")
|
||||
restorer = FormatRestorer()
|
||||
success_count = 0
|
||||
|
||||
for i, entry in enumerate(samples):
|
||||
print(f"\n [{i}] {entry.entry_id}")
|
||||
print(f" Original: {entry.original_text[:50]}...")
|
||||
|
||||
if entry.entry_id in results:
|
||||
translated = results[entry.entry_id]
|
||||
print(f" Translated: {translated[:50]}...")
|
||||
|
||||
# Check if placeholders are preserved
|
||||
ph_keys = [k for k in entry.placeholders.keys() if not k.startswith('_')]
|
||||
preserved = all(f"φ{k}φ" in translated or f"φ/{k}φ" in translated for k in ph_keys if k.isdigit())
|
||||
print(f" PH Preserved: {preserved}")
|
||||
|
||||
# Restore format
|
||||
restored, success = restorer.restore(translated, entry.placeholders)
|
||||
print(f" Restore OK: {success}")
|
||||
|
||||
if success:
|
||||
success_count += 1
|
||||
else:
|
||||
print(f" Restored: {restored[:50]}...")
|
||||
else:
|
||||
print(f" MISSING from results!")
|
||||
|
||||
print(f"\n Summary: {success_count}/{len(samples)} restored successfully")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all steps."""
|
||||
print("="*60)
|
||||
print("TRANSLATION PIPELINE DEBUG")
|
||||
print("="*60)
|
||||
|
||||
if not API_KEY:
|
||||
print("ERROR: V3_API_KEY not found in .env")
|
||||
return
|
||||
|
||||
if not MANIFEST_PATH.exists():
|
||||
print(f"ERROR: Manifest not found at {MANIFEST_PATH}")
|
||||
print("Run the main pipeline first to generate the manifest.")
|
||||
return
|
||||
|
||||
# Initialize LLM client
|
||||
llm_client = LLMClient(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
model=MODEL,
|
||||
extra_headers=EXTRA_HEADERS
|
||||
)
|
||||
|
||||
try:
|
||||
# Step 1: Load manifest
|
||||
manager = await step1_load_manifest()
|
||||
|
||||
# Step 2: Profile book
|
||||
profile = await step2_profile_book(manager, llm_client)
|
||||
|
||||
# Step 3: Create chunks
|
||||
chunks = await step3_create_chunks(manager)
|
||||
|
||||
# Step 4: Translate sample (simple text)
|
||||
await step4_translate_sample(chunks, llm_client, profile)
|
||||
|
||||
# Step 5: Test placeholders
|
||||
await step5_test_placeholders(manager, llm_client, profile)
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("DEBUG COMPLETE")
|
||||
print("="*60)
|
||||
|
||||
finally:
|
||||
await llm_client.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,353 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Chapter Translation Test Script
|
||||
Translate a complete chapter to test the full pipeline.
|
||||
|
||||
Usage:
|
||||
python scripts/translate_chapter.py --show-toc # 显示章节目录
|
||||
python scripts/translate_chapter.py --chapter 5 # 翻译第5章
|
||||
python scripts/translate_chapter.py --chapter 5 --test # 测试模式，只翻译前2个chunk
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from src.manifest_manager import ManifestManager
|
||||
from src.llm_client import LLMClient
|
||||
from src.book_profiler import BookProfiler
|
||||
from src.format_restorer import FormatRestorer
|
||||
from src.data_model import ManifestEntry, BookStructure
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Configuration - Use unified .work directory
|
||||
BOOK_NAME = "Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)"
|
||||
WORK_DIR = Path(f".work/{BOOK_NAME}")
|
||||
MANIFEST_PATH = WORK_DIR / "manifest.json"
|
||||
STRUCTURE_PATH = WORK_DIR / "book_structure.json"
|
||||
CHUNK_DIR = WORK_DIR / "chunks"
|
||||
|
||||
API_KEY = os.getenv("V3_API_KEY")
|
||||
BASE_URL = "https://api.gpt.ge/v1"
|
||||
MODEL = "gpt-4o-mini" # "gemini-3-flash-preview"
|
||||
EXTRA_HEADERS = {"x-foo": "true"}
|
||||
|
||||
# Chunk config - around 5000 chars per chunk
|
||||
CHUNK_SIZE_CHARS = 5000
|
||||
|
||||
|
||||
def load_toc_from_structure() -> list:
|
||||
"""Load TOC from book structure for readable chapter names."""
|
||||
if not STRUCTURE_PATH.exists():
|
||||
return []
|
||||
|
||||
try:
|
||||
structure = BookStructure.load(STRUCTURE_PATH)
|
||||
# Build TOC from spine order with chapter titles
|
||||
toc = []
|
||||
for i, item_id in enumerate(structure.spine, 1):
|
||||
if item_id in structure.resources:
|
||||
resource = structure.resources[item_id]
|
||||
href = resource.href
|
||||
|
||||
# Try to extract title from content
|
||||
title = extract_title_from_html(resource.content) if resource.content else None
|
||||
|
||||
toc.append({
|
||||
'index': i,
|
||||
'item_id': item_id,
|
||||
'href': href,
|
||||
'title': title or f"Chapter {i}"
|
||||
})
|
||||
return toc
|
||||
except Exception as e:
|
||||
print(f"警告: 无法加载书籍结构: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def extract_title_from_html(html: str) -> str:
|
||||
"""Extract title from HTML content."""
|
||||
from bs4 import BeautifulSoup
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
# Try h1, h2, h3 in order
|
||||
for tag in ['h1', 'h2', 'h3']:
|
||||
elem = soup.find(tag)
|
||||
if elem:
|
||||
return elem.get_text().strip()[:50]
|
||||
|
||||
# Try first paragraph
|
||||
p = soup.find('p')
|
||||
if p:
|
||||
text = p.get_text().strip()[:50]
|
||||
if text:
|
||||
return text + "..."
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def build_toc_from_manifest(manager: ManifestManager) -> list:
|
||||
"""Build TOC from manifest entries."""
|
||||
files = {}
|
||||
for entry in manager.entries:
|
||||
fp = entry.file_path
|
||||
if fp not in files:
|
||||
files[fp] = {
|
||||
'count': 0,
|
||||
'first_text': '',
|
||||
'total_chars': 0
|
||||
}
|
||||
files[fp]['count'] += 1
|
||||
files[fp]['total_chars'] += len(entry.original_text)
|
||||
if not files[fp]['first_text'] and entry.original_text:
|
||||
files[fp]['first_text'] = entry.original_text[:40].replace('\n', ' ')
|
||||
|
||||
toc = []
|
||||
for i, (fp, info) in enumerate(sorted(files.items()), 1):
|
||||
toc.append({
|
||||
'index': i,
|
||||
'href': fp,
|
||||
'title': info['first_text'] or f"File {i}",
|
||||
'paragraphs': info['count'],
|
||||
'chars': info['total_chars']
|
||||
})
|
||||
return toc
|
||||
|
||||
|
||||
def show_toc(manager: ManifestManager):
|
||||
"""Display TOC with chapter numbers."""
|
||||
toc = build_toc_from_manifest(manager)
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("章节目录 (Table of Contents)")
|
||||
print("=" * 70)
|
||||
print(f"{'#':>3} | {'段落':>5} | {'字符':>6} | 章节标题")
|
||||
print("-" * 70)
|
||||
|
||||
for item in toc:
|
||||
title = item['title'][:45] if len(item['title']) > 45 else item['title']
|
||||
print(f"{item['index']:3d} | {item['paragraphs']:5d} | {item['chars']:6d} | {title}")
|
||||
|
||||
print("-" * 70)
|
||||
print(f"共 {len(toc)} 个章节")
|
||||
print("\n用法: python scripts/translate_chapter.py --chapter <编号>")
|
||||
print("示例: python scripts/translate_chapter.py --chapter 5")
|
||||
|
||||
|
||||
def get_chapter_entries(manager: ManifestManager, chapter_index: int) -> tuple:
|
||||
"""Get entries for a specific chapter by index."""
|
||||
toc = build_toc_from_manifest(manager)
|
||||
|
||||
if chapter_index < 1 or chapter_index > len(toc):
|
||||
print(f"错误: 章节编号 {chapter_index} 无效 (范围: 1-{len(toc)})")
|
||||
return None, None
|
||||
|
||||
chapter = toc[chapter_index - 1]
|
||||
href = chapter['href']
|
||||
|
||||
entries = [e for e in manager.entries if e.file_path == href]
|
||||
return chapter, entries
|
||||
|
||||
|
||||
def create_char_based_chunks(entries: list, chunk_size: int = CHUNK_SIZE_CHARS) -> list:
|
||||
"""
|
||||
Create chunks based on character count (~5000 chars each).
|
||||
Returns list of entry lists.
|
||||
"""
|
||||
chunks = []
|
||||
current_chunk = []
|
||||
current_size = 0
|
||||
|
||||
for entry in entries:
|
||||
text_len = len(entry.original_text)
|
||||
|
||||
# If adding this entry exceeds limit and we have content, start new chunk
|
||||
if current_size + text_len > chunk_size and current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
current_chunk = []
|
||||
current_size = 0
|
||||
|
||||
current_chunk.append(entry)
|
||||
current_size += text_len
|
||||
|
||||
if current_chunk:
|
||||
chunks.append(current_chunk)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
async def translate_chapter(chapter: dict, entries: list, manager: ManifestManager,
|
||||
llm_client: LLMClient, profile, test_mode: bool = False):
|
||||
"""Translate a complete chapter."""
|
||||
|
||||
print(f"\n开始翻译章节 #{chapter['index']}: {chapter['title'][:40]}...")
|
||||
print(f" 文件: {chapter['href']}")
|
||||
print(f" 总段落: {len(entries)}")
|
||||
|
||||
# Filter untranslated
|
||||
untranslated = [e for e in entries if not e.translated_text]
|
||||
print(f" 待翻译: {len(untranslated)}")
|
||||
|
||||
if not untranslated:
|
||||
print(" ✅ 该章节已全部翻译!")
|
||||
return
|
||||
|
||||
# Create character-based chunks
|
||||
chunks = create_char_based_chunks(untranslated)
|
||||
print(f" 分块: {len(chunks)} 个 Chunk (约{CHUNK_SIZE_CHARS}字符/块)")
|
||||
|
||||
if test_mode:
|
||||
print(" [测试模式] 只翻译前2个 Chunk")
|
||||
chunks = chunks[:2]
|
||||
|
||||
# Show chunk stats
|
||||
for i, chunk in enumerate(chunks, 1):
|
||||
total_chars = sum(len(e.original_text) for e in chunk)
|
||||
print(f" Chunk {i}: {len(chunk)} 段落, {total_chars} 字符")
|
||||
|
||||
# Translate
|
||||
restorer = FormatRestorer()
|
||||
total_success = 0
|
||||
total_failed = 0
|
||||
|
||||
for i, chunk in enumerate(chunks, 1):
|
||||
chunk_chars = sum(len(e.original_text) for e in chunk)
|
||||
print(f"\n 翻译 Chunk {i}/{len(chunks)} ({len(chunk)} 段, {chunk_chars} 字符)...")
|
||||
|
||||
try:
|
||||
results = await llm_client.translate_chunk(
|
||||
chunk,
|
||||
instruction=profile.style_guide if hasattr(profile, 'style_guide') else None,
|
||||
mode="bilingual"
|
||||
)
|
||||
|
||||
# Apply results
|
||||
chunk_success = 0
|
||||
chunk_failed = 0
|
||||
|
||||
for entry in chunk:
|
||||
if entry.entry_id in results:
|
||||
translated = results[entry.entry_id]
|
||||
entry.translated_text = translated
|
||||
|
||||
# Verify placeholder restoration
|
||||
if entry.placeholders:
|
||||
_, success = restorer.restore(translated, entry.placeholders)
|
||||
if success:
|
||||
chunk_success += 1
|
||||
else:
|
||||
chunk_failed += 1
|
||||
print(f" ⚠️ 占位符还原警告: {entry.entry_id[-30:]}")
|
||||
else:
|
||||
chunk_success += 1
|
||||
else:
|
||||
chunk_failed += 1
|
||||
print(f" ❌ 缺失: {entry.entry_id[-30:]}")
|
||||
|
||||
total_success += chunk_success
|
||||
total_failed += chunk_failed
|
||||
print(f" ✓ 成功: {chunk_success}, 失败: {chunk_failed}")
|
||||
|
||||
# Save after each chunk
|
||||
manager.save()
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ Chunk {i} 翻译失败: {e}")
|
||||
total_failed += len(chunk)
|
||||
|
||||
print(f"\n翻译完成:")
|
||||
print(f" ✅ 成功: {total_success}")
|
||||
print(f" ❌ 失败: {total_failed}")
|
||||
|
||||
# Show sample results
|
||||
print(f"\n翻译样例 (前3段):")
|
||||
print("-" * 60)
|
||||
translated_entries = [e for e in entries if e.translated_text][:3]
|
||||
for entry in translated_entries:
|
||||
orig = entry.original_text[:40].replace('\n', ' ')
|
||||
trans = entry.translated_text[:40].replace('\n', ' ') if entry.translated_text else "(无)"
|
||||
print(f" 原: {orig}...")
|
||||
print(f" 译: {trans}...")
|
||||
print()
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser(description="翻译指定章节")
|
||||
parser.add_argument("--show-toc", action="store_true", help="显示章节目录")
|
||||
parser.add_argument("--chapter", "-c", type=int, help="章节编号 (从1开始)")
|
||||
parser.add_argument("--test", "-t", action="store_true", help="测试模式 (只翻译前2个chunk)")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not MANIFEST_PATH.exists():
|
||||
print(f"错误: Manifest 不存在: {MANIFEST_PATH}")
|
||||
print("请先运行主管道生成 manifest。")
|
||||
return
|
||||
|
||||
# Load manifest
|
||||
manager = ManifestManager(MANIFEST_PATH)
|
||||
manager.load()
|
||||
print(f"已加载 manifest: {len(manager.entries)} 条目")
|
||||
|
||||
# Show TOC
|
||||
if args.show_toc or not args.chapter:
|
||||
show_toc(manager)
|
||||
return
|
||||
|
||||
if not API_KEY:
|
||||
print("错误: V3_API_KEY 未设置")
|
||||
return
|
||||
|
||||
# Get chapter entries
|
||||
chapter, entries = get_chapter_entries(manager, args.chapter)
|
||||
if not chapter:
|
||||
return
|
||||
|
||||
# Initialize LLM client
|
||||
from src.utils import ensure_directory
|
||||
ensure_directory(CHUNK_DIR)
|
||||
|
||||
llm_client = LLMClient(
|
||||
api_key=API_KEY,
|
||||
base_url=BASE_URL,
|
||||
model=MODEL,
|
||||
extra_headers=EXTRA_HEADERS,
|
||||
chunk_dir=CHUNK_DIR
|
||||
)
|
||||
|
||||
try:
|
||||
# Generate profile
|
||||
print("\n生成书籍 Profile... (Skipping for debug)")
|
||||
# profiler = BookProfiler(llm_client)
|
||||
# profile = await profiler.analyze(manager.entries)
|
||||
# print(f" 风格: {profile.style_guide[:80] if profile.style_guide else '(无)'}...")
|
||||
|
||||
class DummyProfile:
|
||||
style_guide = "Keep technical terms. Translate accurately."
|
||||
profile = DummyProfile()
|
||||
|
||||
# Translate chapter
|
||||
await translate_chapter(chapter, entries, manager, llm_client, profile, args.test)
|
||||
|
||||
print(f"\n✅ Manifest 已保存: {MANIFEST_PATH}")
|
||||
print(f"✅ Chunk 文件保存在: {CHUNK_DIR}")
|
||||
|
||||
finally:
|
||||
await llm_client.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("DEBUG: Script started execution")
|
||||
try:
|
||||
asyncio.run(main())
|
||||
print("DEBUG: Script finished execution")
|
||||
except Exception as e:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
print(f"CRITICAL ERROR: {e}")
|
||||
@@ -0,0 +1,47 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
import traceback
|
||||
|
||||
# Add src to path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from src.epub_cleaner import EpubCleaner
|
||||
from src.bilingual_builder import BilingualBuilder
|
||||
from src.utils import setup_logger
|
||||
from src.data_model import BookStructure
|
||||
|
||||
logger = setup_logger("verify_toc")
|
||||
|
||||
def verify_fix():
|
||||
INPUT_EPUB = Path("input/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao).epub")
|
||||
OUTPUT_EPUB = Path("output/verify_toc.epub")
|
||||
WORK_DIR = Path("tmp/verify_toc")
|
||||
|
||||
if not INPUT_EPUB.exists():
|
||||
logger.error(f"Input not found: {INPUT_EPUB}")
|
||||
return
|
||||
|
||||
# Clean
|
||||
logger.info("Step 1: Cleaning EPUB...")
|
||||
cleaner = EpubCleaner(INPUT_EPUB, WORK_DIR)
|
||||
json_path = cleaner.clean()
|
||||
|
||||
# Load structure
|
||||
with open(json_path, 'r') as f:
|
||||
structure = BookStructure.model_validate_json(f.read())
|
||||
|
||||
# Build
|
||||
logger.info("Step 2: Building EPUB with TOC preservation...")
|
||||
builder = BilingualBuilder(WORK_DIR, original_epub_path=INPUT_EPUB)
|
||||
builder.build(structure, OUTPUT_EPUB)
|
||||
|
||||
logger.info(f"Step 3: EPUB generated at {OUTPUT_EPUB}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("Starting verification script...")
|
||||
try:
|
||||
verify_fix()
|
||||
print("Verification script finished.")
|
||||
except Exception as e:
|
||||
traceback.print_exc()
|
||||
print(f"CRITICAL ERROR: {e}")
|
||||
Reference in New Issue
Block a user