` 标签。 +- **格式保护 v2**：自动识别数学公式、代码块和行内样式，使用占位符保护，防止 LLM 破坏格式。 + +### 🎯 全局编号系统 +- **DOM 级回填**：不再依赖脆弱的正则，而是利用 DOM 引用进行 100% 精确的一一回填。 +- **双重模式**：支持 Bilingual (双语对照) 和 Chinese (纯译文保留原格式) 模式。 + +### ⚡ 真并发翻译 +- **asyncio.gather 并发执行**：高效利用 API 速率限制 +- **智能速率控制**：基于 Token 桶的 RateLimiter +- **实时进度显示**：Rich 进度条显示翻译状态 +- **断点续传**：自动记录进度，随时中断随时继续 + +### 🔧 自动修复 (Self-Healing) +- **Format Repair**：当 LLM 返回的格式损坏时，自动触发修复机制，利用 LLM 进行自我纠正。 + +## 🚀 快速开始 + +### 1. 安装依赖 + +```bash +pip install -r requirements.txt +``` + +### 2. 配置环境 + +复制 `.env` 模板并填入你的 API Key： + +```bash +# .env 文件 +V3_API_KEY=sk-xxxxxx +OPENROUTER_API_KEY=sk-or-xxxxxx +``` + +### 3. 开始翻译 + +```bash +# 默认使用 OpenRouter (双语模式) +python main.py input/book.epub + +# 纯中文模式 (保留原版样式) +python main.py input/book.epub -m chinese + +# 测试模式（只翻译前10个块，快速验证） +python main.py input/book.epub --test +``` + +## 📂 目录结构 + +``` +. +├── config/ # 配置文件 +├── input/ # 输入 EPUB 目录 +├── output/ # 输出 EPUB 目录 +├── cache/ # 缓存目录 (Manifest, Translations, Processed Epubs) +├── logs/ # 运行日志 +└── src/ # 源代码 + ├── epub_cleaner.py # 预处理器 + ├── fine_grained_extractor.py # 提取器 + ├── bilingual_builder.py # 构建器 + └── translator.py # 主流程 +``` + +## 📄 许可证 + +MIT License + +--- + +**版本**: v0.10 +**更新**: 2026-01-19 \ No newline at end of file diff --git a/REFACTOR_SUMMARY.md b/archive/v0.09/REFACTOR_SUMMARY.md similarity index 100% rename from REFACTOR_SUMMARY.md rename to archive/v0.09/REFACTOR_SUMMARY.md diff --git a/archive/v0.09/analyze_manifest_errors.py b/archive/v0.09/analyze_manifest_errors.py new file mode 100644 index 0000000..d0b1131 --- /dev/null +++ b/archive/v0.09/analyze_manifest_errors.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +""" +Manifest 错误分析工具 +用于深入分析翻译结果中的占位符问题 +""" +import sys +import re +import json +from pathlib import Path + +sys.path.insert(0, '.') +from src.manifest_manager import ManifestManager + +def analyze_item(item): + """详细分析单个 Item 的占位符状态""" + issues = [] + + twp = item.text_with_placeholders or "" + trans = item.translation_with_placeholders or "" + pmap = item.placeholder_map or {} + + ph_regex = r'φ(/?\d+)φ' + src_phs = set(re.findall(ph_regex, twp)) + trans_phs = set(re.findall(ph_regex, trans)) + map_ids = {k for k in pmap.keys() if not k.startswith('_')} + + # 模拟 FormatRestorer 的宽松逻辑 + unknowns = trans_phs - map_ids + real_unknowns = set() + for pid in unknowns: + # 如果是 /N，且 N 在 map 中，则认为是安全的冗余闭合 + if pid.startswith('/') and pid[1:] in map_ids: + continue + real_unknowns.add(pid) + + if real_unknowns: + issues.append(f"未知占位符(幻觉): {real_unknowns}") + + missing = map_ids - trans_phs + if missing: + issues.append(f"丢失占位符: {missing}") + + # Check Reordering + src_ph_list = re.findall(ph_regex, twp) + trans_ph_list = re.findall(ph_regex, trans) + common = [p for p in src_ph_list if p in trans_ph_list] + trans_common = [p for p in trans_ph_list if p in src_ph_list] + if common != trans_common: + issues.append(f"占位符乱序: 原文{common} -> 译文{trans_common}") + + return issues + +def main(): + manifest_dir = Path("cache/manifests") + files = list(manifest_dir.glob("*_manifest.json")) + if not files: + print("未找到 manifest 文件") + return + + target = next((f for f in files if "Karen Hao" in f.name), files[0]) + print(f"Loading: {target}") + + manifest = ManifestManager(str(target)) + if not manifest.load(): + print("加载失败") + return + + items = manifest.get_items() + error_count = 0 + total_analyzed = 0 + + print("\n" + "="*50) + print(" 异常项目分析报告") + print("="*50 + "\n") + + for item in items: + # 只分析非成功状态或有 warning 的 + if item.status == "completed": + continue + + # 即使是 format_error, failed, translated (with issues) + issues = analyze_item(item) + has_error = (item.status in ["failed", "format_error"]) or bool(issues) + + if has_error: + total_analyzed += 1 + print(f"ID: {item.global_id} (Status: {item.status})") + if item.error_msg: + print(f" Error Msg: {item.error_msg}") + + # 只有当有具体 issue 时才打印详细文本，避免刷屏 + if issues or item.status == "format_error": + print(f" 原文: {item.text_with_placeholders[:100]}...") + print(f" 译文: {item.translation_with_placeholders[:100]}..." if item.translation_with_placeholders else " 译文: (None)") + print(f" Map: {list(item.placeholder_map.keys())}") + + for issue in issues: + print(f" -> {issue}") + + print("-" * 50) + error_count += 1 + + if error_count > 50: + print("... (Errors truncated) ...") + break + + print(f"\n分析完成。共发现 {total_analyzed} 个异常项目。") + +if __name__ == "__main__": + main() diff --git a/config/config.example.json b/archive/v0.09/config/config.example.json similarity index 100% rename from config/config.example.json rename to archive/v0.09/config/config.example.json diff --git a/archive/v0.09/config/config.json b/archive/v0.09/config/config.json new file mode 100644 index 0000000..e94a551 --- /dev/null +++ b/archive/v0.09/config/config.json @@ -0,0 +1,69 @@ +{ + "translation": { + "chunk_size": 5000, + "temperature": 0.3, + "glossary": { + "enabled": true, + "auto_generate": true, + "sample_size": 3000 + } + }, + "output": { + "output_dir": "output", + "filename_suffix": "_bilingual" + }, + "logging": { + "level": "INFO", + "file": "logs/translator.log", + "rotation": "10 MB", + "retention": "7 days" + }, + "providers": { + "openrouter": { + "base_url": "https://openrouter.ai/api/v1", + "api_key": "YOUR_OPENROUTER_API_KEY", + "models": { + "fast": "google/gemini-2.0-flash-001", + "smart": "google/gemini-2.0-flash-thinking-exp:free" + }, + "extra_headers": { + "HTTP-Referer": "https://github.com/epub-translator", + "X-Title": "EPUB Translator" + }, + "rate_limits": { + "requests_per_minute": 60, + "concurrent_requests": 32 + } + }, + "v3": { + "base_url": "https://api.gpt.ge/v1", + "api_key": "YOUR_V3_API_KEY", + "models": { + "fast": "gemini-3-flash-preview", + "smart": "gemini-3-pro-preview" + }, + "extra_headers": { + "x-foo": "true" + }, + "rate_limits": { + "requests_per_minute": 500, + "concurrent_requests": 50 + } + }, + "openai": { + "base_url": "http://127.0.0.1:8045/v1", + "api_key": "YOUR_ANTIGRAVITY_API_KEY", + "models": { + "fast": "gemini-3-flash", + "smart": "gemini-3-pro-high" + }, + "extra_headers": { + "x-foo": "true" + }, + "rate_limits": { + "requests_per_minute": 500, + "concurrent_requests": 50 + } + } + } +} diff --git a/config/config_副本.json b/archive/v0.09/config/config_副本.json similarity index 100% rename from config/config_副本.json rename to archive/v0.09/config/config_副本.json diff --git a/archive/v0.09/config/prompts.json b/archive/v0.09/config/prompts.json new file mode 100644 index 0000000..468f941 --- /dev/null +++ b/archive/v0.09/config/prompts.json @@ -0,0 +1,10 @@ +{ + "translation": { + "system": "你是一位精通中英文的专业翻译家。你的任务是翻译书籍内容。\n\n要求：\n1. 准确传达原文含义，语言流畅自然，符合中文阅读习惯。\n2. 严格保持【p_xxxxx】编号格式，不要遗漏，不要修改编号。\n3. **关键格式指令（CRITICAL）**：\n - 原文中包含特殊格式标记：【φ数字φ】（开始/单体）和【φ/数字φ】（结束）。\n - 示例：\"φ1φTable Talkφ/1φ\" 应翻译为 \"φ1φ桌谈φ/1φ\"。\n - 示例：\"φ2φ\" (单体) 表示公式或符号，必须保留在译文对应位置。\n - 规则：严禁删除任何标记！严禁修改数字编号！保持标记与文本的相对位置不变。\n4. **尾注锚点**：形如 φnφ 且中间没有文本的占位符（如 \"...他问道。φ3φ接着...\"）是尾注返回链接，必须保留在译文的对应位置，确保读者可以从尾注跳回正文。\n5. 不要添加任何解释、注释或无关内容，只返回【编号】+【译文】。\n\n{{glossary_instruction}}", + "user_template": "请翻译以下段落（务必原样保留所有 φnφ 格式标记）：\n\n{{content}}" + }, + "glossary_extraction": { + "system": "你是一位资深的文学编辑和领域专家。你的任务是分析书籍样本，提取关键术语并制定统一的译名表。", + "user_template": "请阅读以下书籍片段（包含前言和正文采样）。\n\n任务：\n1. 识别文中出现的人名（如 'Masa', 'Steve Jobs'）、地名、机构名。\n2. 识别特定的行业术语或关键概念。\n3. 为上述词汇提供标准的中文译名。如果像 'Masa' 这样的昵称有对应的全名（如孙正义），请务必使用全名。\n\n请以 JSON 格式输出，格式如下：\n{\n \"Masa\": \"孙正义\",\n \"Apple\": \"苹果公司\",\n ...\n}\n\n书籍片段：\n\n{{content}}" + } +} \ No newline at end of file diff --git a/archive/v0.09/debug.epub b/archive/v0.09/debug.epub new file mode 100644 index 0000000..193d315 Binary files /dev/null and b/archive/v0.09/debug.epub differ diff --git a/archive/v0.09/debug_condition.py b/archive/v0.09/debug_condition.py new file mode 100644 index 0000000..1d0025a --- /dev/null +++ b/archive/v0.09/debug_condition.py @@ -0,0 +1,16 @@ +# 测试 Python 的条件判断 +item_status = "translated" +translation_with_placeholders = "" # 空字符串 + +# 这是 translator.py 中的条件 +if item_status != "translated" or not translation_with_placeholders: + print("SKIP: 条件成立，跳过此 item") +else: + print("PROCESS: 条件不成立，处理此 item") + +# 现在测试有内容的情况 +translation_with_placeholders = "你好" +if item_status != "translated" or not translation_with_placeholders: + print("SKIP: 条件成立，跳过此 item") +else: + print("PROCESS: 条件不成立，处理此 item") diff --git a/archive/v0.09/debug_flow.py b/archive/v0.09/debug_flow.py new file mode 100644 index 0000000..62bdb85 --- /dev/null +++ b/archive/v0.09/debug_flow.py @@ -0,0 +1,62 @@ +"""模拟翻译流程，定位数据丢失问题""" +import asyncio +from src.manifest_manager import ManifestManager + +async def simulate_worker(manifest, chunks): + """模拟 worker 行为""" + for chunk in chunks: + for item in chunk: + # 模拟 LLM 返回 + fake_translation = f"翻译_{item.global_id}" + # 模拟 worker 的 update_item 调用 + manifest.update_item( + item.global_id, + fake_translation, + translation_with_placeholders=fake_translation, + status="translated" + ) + print(f"Worker 完成，内存中 translated 数量: {len(manifest.get_items(status='translated'))}") + +async def simulate_restoration(manifest): + """模拟 process_format_restoration""" + items = manifest.get_items() # 不带参数，获取所有 + print(f"Restoration 获取到 {len(items)} 个 items") + + processed = 0 + skipped = 0 + for item in items: + # 这是关键的过滤条件 + if item.status != "translated" or not item.translation_with_placeholders: + skipped += 1 + continue + processed += 1 + + print(f"Restoration: 处理 {processed} 个，跳过 {skipped} 个") + +async def main(): + # 初始化 + manifest = ManifestManager("test_flow.json") + manifest.init_manifest("test", {}) + + # 添加测试 items + for i in range(5): + manifest.add_item(f"test{i}.html", f"
Text {i}
", f"Text {i}", "p") + + # 模拟 create_chunks_from_manifest + pending = manifest.get_items(status="pending") + chunks = [pending] # 一个 chunk 包含所有 + print(f"Chunks 创建，pending 数量: {len(pending)}") + print(f"chunks[0][0] is manifest._items_by_id['p_00001']: {chunks[0][0] is manifest._items_by_id['p_00001']}") + + # 模拟 worker + await simulate_worker(manifest, chunks) + + # 模拟 restoration + await simulate_restoration(manifest) + + # 清理 + import os + if os.path.exists("test_flow.json"): + os.remove("test_flow.json") + +asyncio.run(main()) diff --git a/archive/v0.09/debug_manifest.py b/archive/v0.09/debug_manifest.py new file mode 100644 index 0000000..e30ada9 --- /dev/null +++ b/archive/v0.09/debug_manifest.py @@ -0,0 +1,31 @@ +from src.manifest_manager import ManifestManager, ManifestItem + +# 模拟 worker 更新流程 +manifest = ManifestManager("test_manifest.json") +manifest.init_manifest("test", {}) + +# 添加一个 item +item = manifest.add_item("test.html", "Hello
", "Hello", "p") +print(f"After add: item.status = {item.status}, item.translation = {item.translation}") +print(f"ID in manifest: {item.global_id}") + +# 模拟 get_items 获取的是同一个对象吗？ +pending = manifest.get_items(status="pending") +print(f"pending[0] is item: {pending[0] is item}") + +# 模拟 worker 更新 +manifest.update_item(item.global_id, "你好", translation_with_placeholders="你好", status="translated") + +# 检查更新是否生效 +print(f"After update: item.status = {item.status}, item.translation = {item.translation}") +print(f"After update: item.translation_with_placeholders = '{item.translation_with_placeholders}'") + +# 验证 get_items 能获取到更新后的状态 +translated = manifest.get_items(status="translated") +print(f"Translated items count: {len(translated)}") +if translated: + print(f"translated[0].translation_with_placeholders = '{translated[0].translation_with_placeholders}'") + +import os +if os.path.exists("test_manifest.json"): + os.remove("test_manifest.json") diff --git a/archive/v0.09/debug_prompt_construction.py b/archive/v0.09/debug_prompt_construction.py new file mode 100644 index 0000000..2cd874e --- /dev/null +++ b/archive/v0.09/debug_prompt_construction.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +""" +调试脚本：使用真实数据验证 Prompt 构建 +完整展示发送给 LLM 的文本 +""" +import sys +import json +from pathlib import Path +from loguru import logger + +# 添加 src 到路径 +sys.path.insert(0, '.') + +from src.manifest_manager import ManifestManager +from src.text_processor import TextProcessor +from src.llm_client import LLMClient +from src.utils import load_config + +# 配置日志到文件，避免控制台刷屏 +logger.remove() +logger.add("debug_prompt.log", level="DEBUG") +logger.add(sys.stdout, level="INFO") + +def debug_prompt(): + print("=== 1. 加载配置 ===") + try: + config = load_config() + # 确保 LLM 配置存在 (Mock if needed for init) + if 'llm' not in config: + if 'v3' in config['providers']: + config['llm'] = config['providers']['v3'] + else: + config['llm'] = {"api_key": "dummy", "models": {"fast": "dummy"}} + + except Exception as e: + print(f"Config load failed: {e}") + return + + print("=== 2. 加载真实 Manifest ===") + # 查找 cache/manifests 下的 json 文件 + manifest_dir = Path("cache/manifests") + if not manifest_dir.exists(): + print("Error: cache/manifests directory not found") + return + + manifest_files = list(manifest_dir.glob("*_manifest.json")) + if not manifest_files: + print("Error: No manifest file found in cache/manifests") + return + + manifest_path = manifest_files[0] + print(f"Using manifest: {manifest_path}") + + manifest = ManifestManager(str(manifest_path)) + if not manifest.load(): + print("Failed to load manifest") + return + + print(f"Loaded {len(manifest.get_items())} items") + + # 获取 Pending items (模拟真实流程) + pending = manifest.get_items(status="pending") + if not pending: + print("No pending items found. Using ALL items for debug.") + items_to_process = manifest.get_items() + else: + items_to_process = pending + + # 找到几个包含占位符的 item 用于验证 + target_items = [] + for item in items_to_process: + if item.placeholder_map and len(item.placeholder_map) > 0: + target_items.append(item) + if len(target_items) >= 5: # 取前5个 + break + + if not target_items: + print("No items with placeholders found!") + return + + print(f"Selected {len(target_items)} items with placeholders for verification") + for item in target_items: + print(f" - {item.global_id}: twp length={len(item.text_with_placeholders or '')}") + + print("\n=== 3. 生成 Prompt (Mode: Chinese) ===") + client = LLMClient(config) + + # 只为这几个 item 生成 prompt + prompt = client._build_prompt(target_items, mode="chinese") + + print("\n" + "="*40) + print("FULL PROMPT CONTENT (Snippet):") + print("="*40) + print(prompt) + print("="*40 + "\n") + + print("\n=== 4. 关键验证 ===") + placeholders_found = prompt.count('φ') + print(f"Total 'φ' symbols in prompt: {placeholders_found}") + + for item in target_items: + if item.text_with_placeholders and 'φ' in item.text_with_placeholders: + # 检查这个 item 的 ID 是否在 prompt 中 + in_prompt = item.global_id in prompt + # 检查这个 item 的占位符是否在 prompt 中 + # 注意：如果占位符是 φ1φ，我们检查 'φ1φ' 是否在 prompt 中 + # 这里简单做，假设 text_with_placeholders 应该完整出现在 prompt 中 (忽略空白差异) + import re + normalized_twp = re.sub(r'\s+', '', item.text_with_placeholders) + normalized_prompt = re.sub(r'\s+', '', prompt) + + content_in_prompt = normalized_twp in normalized_prompt + + print(f"Item {item.global_id}:") + print(f" In prompt ID: {in_prompt}") + print(f" Original twp: {repr(item.text_with_placeholders)}") + print(f" Content match (ignoring whitespace): {content_in_prompt}") + +if __name__ == "__main__": + debug_prompt() diff --git a/archive/v0.09/debug_restore.py b/archive/v0.09/debug_restore.py new file mode 100644 index 0000000..48bd650 --- /dev/null +++ b/archive/v0.09/debug_restore.py @@ -0,0 +1,32 @@ +import re +from src.format_restorer import FormatRestorer + +restorer = FormatRestorer() + +# 模拟一个真实场景 +text_with_ph = '"But what is the goal?" φ1φAmodeiφ/1φ...' +placeholder_map = { + '1': '', + '/1': '' +} + +print("Input text:", text_with_ph) +print("Placeholder map:", placeholder_map) + +# 手动执行 restorer 的验证逻辑 +inner_map = {k: v for k, v in placeholder_map.items() if not k.startswith("_")} +print("Inner map keys:", set(inner_map.keys())) + +found_ids = set(re.findall(r'φ(/?\d+)φ', text_with_ph)) +print("Found IDs in text:", found_ids) + +expected_ids = set(inner_map.keys()) +print("Expected IDs:", expected_ids) + +missing_ids = expected_ids - found_ids +print("Missing IDs:", missing_ids) + +# 调用 restore +html, success = restorer.restore(text_with_ph, placeholder_map) +print(f"\nResult: success={success}") +print(f"HTML: {html}") diff --git a/install.sh b/archive/v0.09/install.sh similarity index 100% rename from install.sh rename to archive/v0.09/install.sh diff --git a/archive/v0.09/main.py b/archive/v0.09/main.py new file mode 100644 index 0000000..7b4b3ee --- /dev/null +++ b/archive/v0.09/main.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +import asyncio +import sys +import argparse +from pathlib import Path +from loguru import logger +from src.translator import EPUBTranslator +from src.epub_parser import EPUBParser +from src.toc_parser import TOCParser +from src.utils import load_config, setup_logging + +def parse_args(): + parser = argparse.ArgumentParser(description="EPUB 双语翻译工具") + parser.add_argument("epub_path", help="输入 EPUB 文件路径") + parser.add_argument("--provider", "-p", default="openrouter", help="LLM 供应商 (config.json 中 providers 的 key)") + parser.add_argument("--mode", "-m", default="bilingual", choices=["bilingual", "chinese"], + help="输出模式: bilingual (双语对照) 或 chinese (纯中文，保留格式)") + parser.add_argument("--test", action="store_true", help="测试模式（仅翻译前几段）") + parser.add_argument("--output", "-o", help="输出目录") + parser.add_argument("--no-cache", action="store_true", help="禁用缓存（强制重新翻译）") + parser.add_argument("--clear-cache", action="store_true", help="清理所有缓存文件") + + # TOC 章节选择 + parser.add_argument("--show-toc", action="store_true", help="显示书籍目录结构") + parser.add_argument("--from", dest="from_chapter", help="起始章节标题") + parser.add_argument("--to", dest="to_chapter", help="结束章节标题") + + return parser.parse_args() + +def flatten_provider_config(config: dict, provider_name: str) -> dict: + """ + 将选定的 provider 配置扁平化到 config['llm'] 中， + 以便下游模块统一调用。 + """ + providers = config.get('providers', {}) + if provider_name not in providers: + available = list(providers.keys()) + logger.error(f"未找到供应商 '{provider_name}'。可用供应商: {available}") + sys.exit(1) + + selected_config = providers[provider_name] + logger.info(f"使用 LLM 供应商: {provider_name} ({selected_config.get('base_url')})") + + # 注入到 config['llm'] + config['llm'] = selected_config + return config + +async def run_translation(args): + try: + # 1. 加载配置 + config = load_config() + + # 2. 处理 Provider 选择 + config = flatten_provider_config(config, args.provider) + + # 3. 设置日志 + setup_logging(config) + logger.info("程序启动") + + # 4. 初始化翻译器 + translator = EPUBTranslator(config, use_cache=not args.no_cache) + + # 5. 执行翻译 (传递章节范围参数) + await translator.translate_epub( + args.epub_path, + test_mode=args.test, + output_dir=args.output, + mode=args.mode, + from_chapter=args.from_chapter, + to_chapter=args.to_chapter + ) + + except Exception as e: + import traceback + traceback.print_exc() + logger.error(f"翻译失败: {e}") + sys.exit(1) + +def show_toc(epub_path: str): + """显示 EPUB 的目录结构""" + parser = EPUBParser(epub_path) + toc_parser = TOCParser(parser.book) + + print(f"\n📖 {parser.metadata.get('title', 'Unknown')} - {parser.metadata.get('author', 'Unknown')}") + print(toc_parser.format_toc_table()) + print("提示: 使用 --from \"章节名\" --to \"章节名\" 指定翻译范围") + print() + +def main(): + args = parse_args() + + if args.clear_cache: + import shutil + cache_dir = Path("cache") + if cache_dir.exists(): + shutil.rmtree(cache_dir) + print("缓存已清理") + # sys.exit(0) # 移除退出，允许继续执行 + + + if args.show_toc: + show_toc(args.epub_path) + sys.exit(0) + + asyncio.run(run_translation(args)) + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/archive/v0.09/pyproject.toml similarity index 100% rename from pyproject.toml rename to archive/v0.09/pyproject.toml diff --git a/archive/v0.09/requirements.txt b/archive/v0.09/requirements.txt new file mode 100644 index 0000000..ca7c305 --- /dev/null +++ b/archive/v0.09/requirements.txt @@ -0,0 +1,12 @@ +ebooklib>=0.19 +beautifulsoup4>=4.12.0 +lxml>=4.9.0 +openai>=1.0.0 +aiohttp>=3.9.0 +pydantic>=2.0.0 +loguru>=0.7.0 +rich>=13.0.0 +asyncio-throttle>=1.0.2 +tenacity>=8.0.0 +python-dotenv>=1.0.0 +socksio>=1.0.0 diff --git a/run.sh b/archive/v0.09/run.sh similarity index 100% rename from run.sh rename to archive/v0.09/run.sh diff --git a/scripts/debug/debug.py b/archive/v0.09/scripts/debug/debug.py similarity index 100% rename from scripts/debug/debug.py rename to archive/v0.09/scripts/debug/debug.py diff --git a/scripts/debug/debug_epub_comparison.py b/archive/v0.09/scripts/debug/debug_epub_comparison.py similarity index 100% rename from scripts/debug/debug_epub_comparison.py rename to archive/v0.09/scripts/debug/debug_epub_comparison.py diff --git a/scripts/debug/debug_structure.py b/archive/v0.09/scripts/debug/debug_structure.py similarity index 100% rename from scripts/debug/debug_structure.py rename to archive/v0.09/scripts/debug/debug_structure.py diff --git a/scripts/debug/quick_fix.py b/archive/v0.09/scripts/debug/quick_fix.py similarity index 100% rename from scripts/debug/quick_fix.py rename to archive/v0.09/scripts/debug/quick_fix.py diff --git a/scripts/example.py b/archive/v0.09/scripts/example.py similarity index 100% rename from scripts/example.py rename to archive/v0.09/scripts/example.py diff --git a/setup.py b/archive/v0.09/setup.py similarity index 100% rename from setup.py rename to archive/v0.09/setup.py diff --git a/archive/v0.09/src/__init__.py b/archive/v0.09/src/__init__.py new file mode 100644 index 0000000..473d099 --- /dev/null +++ b/archive/v0.09/src/__init__.py @@ -0,0 +1,32 @@ +""" +EPUB 双语翻译程序 +主要功能模块的初始化文件 +""" + +__version__ = "0.08" +__author__ = "Kaitan" + +from .epub_parser import EPUBParser +from .translator import EPUBTranslator +from .llm_client import LLMClient as OpenRouterClient # Keep alias for compatibility +from .llm_client import LLMClient +from .text_processor import TextProcessor +from .bilingual_builder import BilingualEPUBBuilder +from .chinese_builder import ChineseEPUBBuilder +from .format_extractor import FormatExtractor +from .format_restorer import FormatRestorer +from .utils import load_config, setup_logging + +__all__ = [ + "EPUBParser", + "EPUBTranslator", + "LLMClient", + "OpenRouterClient", + "TextProcessor", + "BilingualEPUBBuilder", + "ChineseEPUBBuilder", + "FormatExtractor", + "FormatRestorer", + "load_config", + "setup_logging" +] \ No newline at end of file diff --git a/archive/v0.09/src/bilingual_builder.py b/archive/v0.09/src/bilingual_builder.py new file mode 100644 index 0000000..165002c --- /dev/null +++ b/archive/v0.09/src/bilingual_builder.py @@ -0,0 +1,235 @@ +""" +双语 EPUB 构建器模块 (集成 V2) + +基于 FineGrainedExtractor 的 DOM 回填机制，确保 100% 的内容对齐和格式保留。 +""" + +from ebooklib import epub +import ebooklib +from bs4 import BeautifulSoup +from typing import Dict, List +from pathlib import Path +from loguru import logger +import uuid +from .fine_grained_extractor import FineGrainedExtractor + +class BilingualEPUBBuilder: + """双语 EPUB 构建器""" + + def __init__(self, original_book, config: Dict): + self.original_book = original_book + self.config = config + # 从配置中获取是否翻译目录 + self.translate_toc = config['translation'].get('translate_toc', False) + + def create_bilingual_epub_with_mapping(self, translation_map: Dict[str, str], + paragraph_map: Dict[str, Dict], + output_path: str) -> str: + """ + 创建双语 EPUB。使用 ordered_ids 确保与 Manifest 严格一致。 + + Args: + translation_map: { global_item_id: translated_text_with_ph } + paragraph_map: { global_item_id: item_metadata_dict } + """ + try: + new_book = epub.EpubBook() + self._copy_metadata(new_book) + + # 安全清理 TOC (虽然预处理已做，但构建新书对象时再次确保合规) + new_book.toc = self._sanitize_toc(self.original_book.toc) + + # 1. 准备每个文件的有序ID列表 + # 目的是将扁平的 map 重新按文件和顺序组织 + file_ordered_ids = {} + # paragraph_map 的 key 是 global_id，通常包含顺序信息或我们依赖 items 的插入顺序 + # 更好的方式是依赖 item ID 的数字部分排序，如果它们是 'id_0', 'id_1'... + # 假设 ID 包含顺序信息。 + sorted_pids = sorted(paragraph_map.keys(), key=lambda x: self._extract_id_index(x)) + + for pid in sorted_pids: + info = paragraph_map[pid] + fname = info['file_name'] + if fname not in file_ordered_ids: + file_ordered_ids[fname] = [] + file_ordered_ids[fname].append(pid) + + processed_item_ids = set() + item_map = {} + + # 特殊处理：封面图片 + self._handle_cover(new_book, processed_item_ids, item_map) + + # 2. 复制所有非文档资源 (图片, CSS, 字体) + for item in self.original_book.get_items(): + if item.get_type() != ebooklib.ITEM_DOCUMENT: + if item.id not in processed_item_ids: + new_book.add_item(item) + processed_item_ids.add(item.id) + item_map[item.id] = item + + # 3. 处理并回填文档 + new_spine = [] + for spine_id, linear in self.original_book.spine: + item = self.original_book.get_item_with_id(spine_id) + if not item: continue + + if item.get_type() == ebooklib.ITEM_DOCUMENT: + file_name = item.get_name() + new_item = item # 默认使用原 Item + + # 如果该文件有翻译内容 + if file_name in file_ordered_ids: + target_ids = file_ordered_ids[file_name] + + # 执行回填 + new_content = self._process_document_content( + item.get_content().decode('utf-8'), + file_name, + target_ids, + translation_map + ) + + # 创建新 item 避免污染原对象 + new_item = epub.EpubHtml( + title=item.title, + file_name=file_name, + lang='zh-CN', # 双语版主要语言 + uid=item.id + ) + new_item.set_content(new_content.encode('utf-8')) + # 复制原 item 的其他属性如 style + for link in item.get_links(): + # 这里不做深度复制，简单引用 + pass + # 重新添加 links (特别是 CSS) + # 注意: EpubHtml 构造时不会自动带原来的 links，需要手动加 + # 但我们在 dirty hack 里，直接 set_content 了 HTML。 + # 如果 HTML head 里有 link, ebooklib 可能会解析并注册? + # Ebooklib 的行为是: 只有通过 add_link 加的才会出现在 opf manifest。 + # 我们需要把原 item 的 links 复制过来 + if hasattr(item, 'links'): + for link in item.links: + new_item.add_link(**link) # 不是很安全，视 ebooklib 版本而定 + + if new_item.id not in processed_item_ids: + new_book.add_item(new_item) + processed_item_ids.add(new_item.id) + new_spine.append(new_item) + else: + if item.id in item_map: + new_spine.append(item_map[item.id]) + + new_book.spine = new_spine + new_book.add_item(epub.EpubNcx()) + new_book.add_item(epub.EpubNav()) + + # 生成输出文件名 + output_file = self._generate_output_filename(output_path) + epub.write_epub(output_file, new_book, {}) + logger.info(f"双语 EPUB 生成成功: {output_file}") + return output_file + + except Exception as e: + logger.error(f"创建双语 EPUB 失败: {e}", exc_info=True) + raise + + def _process_document_content(self, content: str, file_name: str, + target_ids: List[str], translation_map: Dict[str, str]) -> str: + """ + 处理单个文档的内容：提取 -> 注入翻译 -> 回填 + """ + try: + # 1. 再次提取，建立 DOM 映射 + # 必须使用与 TextProcessing 阶段完全一致的参数 + extractor = FineGrainedExtractor(translate_toc=self.translate_toc) + items = extractor.extract(content, file_name) + + # 2. 筛选出应该翻译的项目 + translatable_items = [i for i in items if i['should_translate']] + + # 3. 一致性检查 + if len(translatable_items) != len(target_ids): + logger.error( + f"严重错误 [{file_name}]: 提取项数 ({len(translatable_items)}) " + f"与 Manifest 记录数 ({len(target_ids)}) 不一致! " + "将跳过此文件的翻译回填以防错位。" + ) + # Fallback: 返回原始内容 + return content + + # 4. 注入翻译 + for extract_item, pid in zip(translatable_items, target_ids): + translation = translation_map.get(pid) + if translation: + extract_item['translation'] = translation + + # 5. 回填 + # 获取输出模式 + output_mode = self.config.get('output', {}).get('mode', 'bilingual') + bilingual_mode = (output_mode == 'bilingual') + + new_html = extractor.backfill(items, bilingual=bilingual_mode) + + + return new_html + + except Exception as e: + logger.error(f"处理文档内容失败 {file_name}: {e}", exc_info=True) + return content + + def _extract_id_index(self, pid: str) -> int: + """从 ID 字符串中提取数字索引用于排序 (如 'p_10' -> 10)""" + try: + # 尝试常见格式 p_123, id_456 + parts = pid.split('_') + if len(parts) > 1 and parts[-1].isdigit(): + return int(parts[-1]) + return 0 + except: + return 0 + + def _sanitize_toc(self, toc): + """确保 TOC 中的所有节点都有 ID""" + for item in toc: + if isinstance(item, (epub.Link, epub.Section)): + if not getattr(item, 'uid', None): + item.uid = f"navPoint-{uuid.uuid4().hex[:8]}" + elif isinstance(item, tuple) and len(item) == 2: + section, children = item + if isinstance(section, (epub.Link, epub.Section)): + if not getattr(section, 'uid', None): + section.uid = f"navPoint-{uuid.uuid4().hex[:8]}" + self._sanitize_toc(children) + return toc + + def _handle_cover(self, new_book, processed_item_ids, item_map): + cover_id_meta = self.original_book.get_metadata('OPF', 'cover') + if cover_id_meta: + cover_item = self.original_book.get_item_with_id(cover_id_meta[0][0]) + if cover_item: + new_book.add_item(cover_item) + processed_item_ids.add(cover_item.id) + item_map[cover_item.id] = cover_item + # 复制 cover metadata + new_book.add_metadata('OPF', 'cover', cover_item.id) + + def _copy_metadata(self, new_book): + for namespace, meta_dict in self.original_book.metadata.items(): + for name, values in meta_dict.items(): + for value, other in values: + if name and hasattr(name, 'lower') and name.lower() == 'identifier': continue + new_book.add_metadata(namespace, name, value, other) + new_book.add_metadata('DC', 'language', 'zh-CN') + new_book.set_identifier(f"bilingual-{uuid.uuid4().hex[:12]}") + + def _generate_output_filename(self, output_path: str) -> str: + # 根据模式生成不同的后缀 + output_mode = self.config.get('output', {}).get('mode', 'bilingual') + suffix = "chinese" if output_mode == "chinese" else "bilingual" + + title_meta = self.original_book.get_metadata('DC', 'title') + title = title_meta[0][0] if title_meta else "bilingual_book" + safe_title = "".join([c for c in title if c.isalnum() or c in (' ', '-', '_')]).strip() + Path(output_path).mkdir(parents=True, exist_ok=True) + return str(Path(output_path) / f"{safe_title}_{suffix}.epub") \ No newline at end of file diff --git a/src/book_profiler.py b/archive/v0.09/src/book_profiler.py similarity index 100% rename from src/book_profiler.py rename to archive/v0.09/src/book_profiler.py diff --git a/src/cache.py b/archive/v0.09/src/cache.py similarity index 100% rename from src/cache.py rename to archive/v0.09/src/cache.py diff --git a/src/chinese_builder.py b/archive/v0.09/src/chinese_builder.py similarity index 91% rename from src/chinese_builder.py rename to archive/v0.09/src/chinese_builder.py index 43abc25..2fd8661 100644 --- a/src/chinese_builder.py +++ b/archive/v0.09/src/chinese_builder.py @@ -10,21 +10,22 @@ from ebooklib import epub import ebooklib from bs4 import BeautifulSoup -from typing import Dict, List +from typing import Dict, List, Any from pathlib import Path from loguru import logger import uuid +from .fine_grained_extractor import FineGrainedExtractor class ChineseEPUBBuilder: - """纯中文 EPUB 构建器""" + """纯中文 EPUB 构建器 (DOM Safe)""" def __init__(self, original_book, config: Dict): self.original_book = original_book self.config = config - self.output_config = config['output'] + self.translate_toc = config['translation'].get('translate_toc', False) def create_chinese_epub_with_mapping(self, - items: List, # List[ManifestItem] + items: List[Any], # List[ManifestItem] output_path: str) -> str: """ 创建纯中文 EPUB。 @@ -32,27 +33,22 @@ class ChineseEPUBBuilder: try: new_book = epub.EpubBook() self._copy_metadata(new_book) + # 安全清理 TOC new_book.toc = self._sanitize_toc(self.original_book.toc) - # 准备每个文件的有序项目列表 - file_items = {} - for item in sorted(items, key=lambda x: x.global_id): + # 1. 按文件分组 Manifest Items + # 假设 items 已经是按全局 ID 排序的 (ManifestManager.get_items 返回有序列表) + file_items_map = {} + for item in items: fname = item.source_file - if fname not in file_items: - file_items[fname] = [] - file_items[fname].append(item) + if fname not in file_items_map: + file_items_map[fname] = [] + file_items_map[fname].append(item) processed_item_ids = set() item_map = {} # 特殊处理：封面图片 - cover_id_meta = self.original_book.get_metadata('OPF', 'cover') - if cover_id_meta: - cover_item = self.original_book.get_item_with_id(cover_id_meta[0][0]) - if cover_item: - new_book.add_item(cover_item) - processed_item_ids.add(cover_item.id) - item_map[cover_item.id] = cover_item # 复制资源 for item in self.original_book.get_items(): diff --git a/archive/v0.09/src/epub_cleaner.py b/archive/v0.09/src/epub_cleaner.py new file mode 100644 index 0000000..449ae20 --- /dev/null +++ b/archive/v0.09/src/epub_cleaner.py @@ -0,0 +1,178 @@ +""" +EPUB 清理器模块 (EpubCleaner) + +负责在翻译前对 EPUB 进行标准化清洗，解决兼容性问题。 +核心功能: +1. Flatten Structure: 将 div 转换为 p，简化结构 +2. Fix TOC: 修复目录中的死链和缺失 UID +3. CSS Restoration: 找回丢失的样式表 +""" + +from bs4 import BeautifulSoup, Tag +from loguru import logger +from ebooklib import epub +import ebooklib +import zipfile +import uuid +from ebooklib.epub import Link + +class EpubCleaner: + """标准 EPUB 清理器""" + + def clean_epub(self, input_path: str, output_path: str): + """ + 清理 EPUB 文件并保存到新路径 + """ + logger.info(f"开始清理: {input_path}") + + # 1. 尝试打开 Zip 以读取原始内容 (Ebooklib 回退机制) + try: + input_zip = zipfile.ZipFile(input_path, 'r') + zip_files = set(input_zip.namelist()) + except Exception as e: + logger.error(f"无法打开 Zip (样式回退功能将失效): {e}") + input_zip = None + zip_files = set() + + # 2. 读取 EPUB + try: + book = epub.read_epub(input_path) + except Exception as e: + logger.error(f"Ebooklib 读取失败: {e}") + raise + + # 3. 遍历并清理文档 + count = 0 + for item in book.get_items(): + if item.get_type() == ebooklib.ITEM_DOCUMENT: + try: + file_name = item.get_name() + content = None + + # 优先从 Zip 读取以保留 Head 信息 (CSS Links) + if input_zip and file_name in zip_files: + try: + content = input_zip.read(file_name).decode('utf-8') + except Exception: + pass + + # 回退到 ebooklib + if content is None: + raw_content = item.get_content() + if raw_content: + content = raw_content.decode('utf-8') + + if not content or not content.strip(): + continue + + # 执行清理 + cleaned = self._clean_content(content, item) + + # 安全检查 + if not cleaned.strip(): + logger.warning(f"警告: {file_name} 清理后为空，保留原始内容") + cleaned = content + + item.set_content(cleaned.encode('utf-8')) + count += 1 + + except Exception as e: + logger.warning(f"清理文档失败 {item.get_name()}: {e}") + + # 4. 修复 TOC (死链和 UID) + try: + book.toc = self._fix_and_clean_toc(book.toc, book) + except Exception as e: + logger.error(f"TOC 修复失败: {e}") + + # 5. 保存 + epub.write_epub(output_path, book) + logger.info(f"清理完成: {output_path} (处理了 {count} 个文档)") + + def _clean_content(self, html_content: str, item=None) -> str: + """ + 执行具体的 HTML 清理逻辑 + """ + soup = BeautifulSoup(html_content, 'html.parser') + + # 恢复 CSS 链接 + if item: + self._restore_css_links(soup, item) + + # div 转 p + stats = {'divs_to_p': 0} + inline_tags = {'span', 'em', 'strong', 'i', 'b', 'u', 'a', 'br', 'sub', 'sup'} + + for div in list(soup.find_all('div')): + # 检查是否有块级子元素 (如果有，则保留 div 容器结构) + has_block = any( + isinstance(c, Tag) and c.name not in inline_tags + for c in div.children + ) + + if not has_block: + div.name = 'p' + stats['divs_to_p'] += 1 + + # 可以在这里添加更多清理逻辑 (如移除无用的空的 span 等) + + return str(soup) + + def _restore_css_links(self, soup, item): + """从原始 HTML 中提取并恢复 CSS 链接到 item 对象""" + head = soup.find('head') + if head: + links = head.find_all('link', rel='stylesheet') + for link in links: + href = link.get('href') + if href: + existing_links = list(item.get_links()) + exists = False + for l in existing_links: + l_href = getattr(l, 'href', None) + if l_href is None and isinstance(l, dict): + l_href = l.get('href') + if l_href == href: + exists = True + break + + if not exists: + item.add_link(href=href, rel='stylesheet', type='text/css') + + def _fix_and_clean_toc(self, toc, book): + """修复 TOC：补全 UID 并移除指向不存在文件的死链""" + new_toc = [] + + for item in toc: + # Case 1: (Section, Children) + if isinstance(item, (tuple, list)): + section, children = item + cleaned_children = self._fix_and_clean_toc(children, book) + + if isinstance(section, Link): + href = section.href.split('#')[0] + if book.get_item_with_href(href): + if section.uid is None: + section.uid = f'uuid-{uuid.uuid4()}' + new_toc.append((section, cleaned_children)) + else: + logger.warning(f"移除无效 TOC 节点: {section.href}") + new_toc.extend(cleaned_children) + else: + new_toc.append((section, cleaned_children)) + + # Case 2: Link + elif isinstance(item, Link): + href = item.href.split('#')[0] + if book.get_item_with_href(href): + if item.uid is None: + item.uid = f'uuid-{uuid.uuid4()}' + new_toc.append(item) + else: + logger.warning(f"移除无效 TOC 节点: {item.href}") + + # Case 3: Other + else: + new_toc.append(item) + + return new_toc diff --git a/src/epub_parser.py b/archive/v0.09/src/epub_parser.py similarity index 100% rename from src/epub_parser.py rename to archive/v0.09/src/epub_parser.py diff --git a/archive/v0.09/src/fine_grained_extractor.py b/archive/v0.09/src/fine_grained_extractor.py new file mode 100644 index 0000000..f2232dc --- /dev/null +++ b/archive/v0.09/src/fine_grained_extractor.py @@ -0,0 +1,222 @@ +""" +Fine-Grained Extractor (集成格式保护版) + +负责从 EPUB 中提取文本，进行精细化处理，并负责最终的回填工作。 +集成 FormatExtractor 以实现行内格式的保护。 +""" + +from bs4 import BeautifulSoup, Tag, NavigableString +from typing import List, Dict, Any, Tuple +import re +from loguru import logger +from .format_extractor import FormatExtractor +from .format_restorer import FormatRestorer + +class FineGrainedExtractor: + """细粒度提取与回填器""" + + SKIP_TRANSLATION_PATTERNS = [ + r'index\.x?html', + r'bibliography\.x?html', + r'endnotes?\.x?html', + r'footnotes?\.x?html', + ] + + TOC_PATTERNS = [ + r'nav\.x?html', + r'toc\.x?html', + ] + + def __init__(self, translate_toc: bool = False): + self.translate_toc = translate_toc + self.soup = None + # 初始化格式处理器 + self.format_extractor = FormatExtractor() + self.format_restorer = FormatRestorer() + + def extract(self, html_content: str, file_name: str = "") -> List[Dict[str, Any]]: + """ + 提取所有正文元素 (p, h1-h6)，并进行格式分析 + """ + self.soup = BeautifulSoup(html_content, 'html.parser') + + # 不要移除 link/style/meta/script，否则回填时会丢失头部信息 + # for element in self.soup(['script', 'style', 'meta', 'link']): + # element.decompose() + + + doc_type = self._classify_document(file_name) + items = [] + + # 目标: 所有段落和标题 + target_tags = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6'] + + for element in self.soup.find_all(target_tags): + # 1. 初步文本提取 (用于过滤判断) + raw_text = element.get_text(separator=' ', strip=True) + if not raw_text.strip(): + continue + + # 2. 判断是否需要翻译 + is_decorative = self._is_decorative(raw_text) + should_translate = self._should_translate(doc_type, is_decorative, raw_text) + + item_data = { + 'element': element, # 持有引用，用于回填 + 'tag': element.name, + 'raw_text': raw_text, + 'should_translate': should_translate, + 'doc_type': doc_type, + 'text_len': len(raw_text) + } + + # 3. 如果需要翻译，执行通过 FormatExtractor 进行精细化格式提取 + if should_translate: + # 必须传入 Outer HTML (str(element))，因为 FormatExtractor 内部会解析并剥离最外层标签 + # 如果只传 inner_html，FormatExtractor 解析时只能拿到第一个子节点，导致内容丢失验证失败 + outer_html = str(element) + # 提取 (clean_text, text_with_ph, map, type, endnote_anchors) + clean, text_ph, ph_map, _, endnote_anchors = self.format_extractor.extract(outer_html) + + # 再次验证：如果提取后的 clean_text 为空 (比如全是公式)，则不翻译 + if not clean.strip() or not text_ph.strip(): + item_data['should_translate'] = False + else: + item_data['text'] = clean # 纯文本 (供人阅读/日志) + item_data['text_nodes'] = [] # 兼容旧字段 (空) + item_data['text_with_ph'] = text_ph # 发送给 LLM 的文本 + item_data['placeholder_map'] = ph_map + item_data['endnote_anchors'] = endnote_anchors # 尾注锚点 ID 列表 + else: + # 不需要翻译，仅保留基础信息 + item_data['text'] = raw_text + item_data['text_with_ph'] = raw_text # Fallback + + items.append(item_data) + + logger.debug(f"[{doc_type}] {file_name}: 提取 {len(items)} 元素, 需翻译 {sum(1 for i in items if i['should_translate'])}") + return items + + def backfill(self, items: List[Dict[str, Any]], bilingual: bool = True) -> str: + """ + 回填翻译 (支持格式还原) + + Args: + items: 提取的元素列表，且已注入 'translation' 字段 (带占位符的译文) + bilingual: 是否生成双语版本 + """ + success_count = 0 + + for item in items: + if not item.get('should_translate'): + continue + + # 使用预先注入的翻译 (解决了重复文本映射问题) + translated_ph = item.get('translation') + + if not translated_ph: + continue + + original_element = item['element'] + + # 4. 格式还原 + ph_map = item.get('placeholder_map', {}) + # 容错：如果 ph_map 为 None (未开启格式保护)，设为空字典 + if ph_map is None: ph_map = {} + + restored_html, _ = self.format_restorer.restore(translated_ph, ph_map) + + # 5. 构建新 DOM 元素 + if bilingual: + # 双语模式: Append + new_tag = self.soup.new_tag(original_element.name) + + # 继承 class + classes = original_element.get('class', []) + new_tag['class'] = list(classes) + ['translation', 'chinese'] + + # 继承 style + style = original_element.get('style') + if style: + new_tag['style'] = style + + # 设置内容 (解析 restored HTML) + inner_soup = BeautifulSoup(restored_html, 'html.parser') + if inner_soup.body: + for child in list(inner_soup.body.children): + new_tag.append(child) + else: + for child in list(inner_soup.children): + new_tag.append(child) + + original_element.insert_after(new_tag) + + else: + # 仅中文模式: Replace + # 直接修改 original_element 的内容 + original_element.clear() + inner_soup = BeautifulSoup(restored_html, 'html.parser') + + # 直接替换内容 + if inner_soup.body: + for child in list(inner_soup.body.children): + original_element.append(child) + else: + for child in list(inner_soup.children): + original_element.append(child) + + # 可以在这里移除 dropcap class? + # 但如果 dropcap 是内部 span，已经被还原回去了。 + # 由于 Drop Cap 处理在 FormatExtractor 已经把首字母放入文本，Prefix 里的 dropcap 是空的 + # 还原后的 HTML 大概是 这... + pass + + success_count += 1 + + return str(self.soup) + + # --- 以下是辅助判别逻辑 (同原版) --- + + def _classify_document(self, file_name: str) -> str: + if not file_name: return 'core' + fname = file_name.lower() + if any(re.search(p, fname) for p in self.SKIP_TRANSLATION_PATTERNS): return 'skip' + if any(re.search(p, fname) for p in self.TOC_PATTERNS): return 'toc' + return 'core' + + def _should_translate(self, doc_type: str, is_decorative: bool, text: str = "") -> bool: + if is_decorative: return False + if self._is_roman_numeral(text): return False + if doc_type == 'core': return True + if doc_type == 'toc': return self.translate_toc + return False + + def _is_roman_numeral(self, text: str) -> bool: + text = text.strip().upper() + if not text: return False + # 简单宽松匹配: 纯字母且看起来像罗马数字 (I, V, X, L, C, M) + # 排除普通单词如 "I" (作为代词时应翻译，但作为单独段落通常是标题) + # 这是一个权衡。单独的 "I" 在小说里可能表示 "我"，但在章节标题里表示 "第一章"。 + # 如果是正文中的 "I am...", 肯定会被提取。这里只有单独的 "I" 才会被这里匹配。 + # 真正的问题是：单独一行 "I" 表示 "我" 的情况极少。 + pattern = re.compile(r"^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$") + return bool(pattern.match(text)) + + def _is_decorative(self, text: str) -> bool: + s = text.strip() + if not s: return False + if not any(c.isalnum() for c in s): return True + if len(s) > 20: return False + + patterns = [ + r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾]+$', + r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾\s]+$', + ] + for p in patterns: + if re.match(p, s): return True + + # 字符种类很少且包含非字母 (e.g. "* * *") + unique = set(s.replace(' ', '')) + if len(unique) <= 3 and (unique & set('*-_~•◆◇■□▪▫●○◉◎⦿⦾')): + return True + return False diff --git a/src/format_extractor.py b/archive/v0.09/src/format_extractor.py similarity index 72% rename from src/format_extractor.py rename to archive/v0.09/src/format_extractor.py index 377a924..5df3784 100644 --- a/src/format_extractor.py +++ b/archive/v0.09/src/format_extractor.py @@ -93,7 +93,7 @@ class FormatExtractor: def __init__(self): self.detector = HeadingDetector() - def extract(self, element_html: str) -> Tuple[str, str, Dict[str, str], str]: + def extract(self, element_html: str) -> Tuple[str, str, Dict[str, str], str, List[str]]: """ 提取格式信息 @@ -102,6 +102,7 @@ class FormatExtractor: text_with_placeholders: 只包含内嵌占位符的文本（不含前缀/后缀标签） placeholder_map: 占位符映射，包含特殊键 "_prefix" 和 "_suffix" paragraph_type: 段落类型 + endnote_anchors: 尾注锚点 ID 列表 (用于补救) """ soup = BeautifulSoup(element_html, 'html.parser') root = list(soup.children)[0] if list(soup.children) else soup @@ -117,6 +118,12 @@ class FormatExtractor: # 智能提取（分离前缀/后缀） text_with_ph, local_map = self._smart_extract_v3(inner_html) + # 核心修复：清理 text_with_placeholders 中的换行符和多余空格 + # 这一步至关重要，因为 inner_html 中的换行符会导致 LLM Prompt 格式混乱（多行） + # 从而导致 LLM 忽略不在同一行的占位符或内容 + if text_with_ph: + text_with_ph = re.sub(r'\s+', ' ', text_with_ph).strip() + # === 验证完整性 === # 将 text_with_placeholders 去掉占位符后与 clean_text 比较 stripped_text = self._strip_placeholders(text_with_ph) @@ -129,7 +136,17 @@ class FormatExtractor: # 降级：不使用前缀/后缀分离，只做简单占位符处理 text_with_ph, local_map = self._fallback_extract(inner_html, clean_text) - return clean_text, text_with_ph, local_map, p_type + # === 识别尾注锚点 === + # 尾注锚点特征： (短随机ID，通常 3-5 字符) + endnote_anchors = [] + for pid, html in local_map.items(): + if pid.startswith("_"): + continue # 跳过 _prefix, _suffix + # 匹配空锚点： 或 + if re.match(r'<(span|a)\s+id="[a-zA-Z][a-zA-Z0-9]{2,5}"\s*>\s*', html): + endnote_anchors.append(pid) + + return clean_text, text_with_ph, local_map, p_type, endnote_anchors def _strip_placeholders(self, text: str) -> str: """移除所有占位符（φXφ 和 φ/Xφ 格式）""" @@ -173,9 +190,11 @@ class FormatExtractor: 智能提取 v3：分离前缀/后缀 + 合并内嵌公式块 核心逻辑： - 1. 分离前缀（第一个可翻译文本之前）和后缀（最后一个可翻译文本之后） + 1. 分离前缀（第一个可翻译文本之前的完整标签）和后缀（最后一个可翻译文本之后的完整标签） 2. 中间部分：检测"公式块"（连续标签+不可翻译文本），合并为单个占位符 3. 只有真正需要翻译的格式标签（如斜体包裹的长文本）才拆分 + + 注意：前缀/后缀只包含不影响文本结构的完整标签，开始标签必须有匹配的结束标签 """ # 使用正则分割标签和文本 parts = re.split(r'(<[^>]+>)', inner_html) @@ -209,11 +228,72 @@ class FormatExtractor: # 没有可翻译文本，全部作为前缀 return "", {"_prefix": inner_html, "_suffix": ""} - # 分割 - prefix_parts = parts[:first_trans_idx] - middle_parts = parts[first_trans_idx:last_trans_idx + 1] - middle_types = part_types[first_trans_idx:last_trans_idx + 1] - suffix_parts = parts[last_trans_idx + 1:] + # === 安全前缀分离 === + # 只将自闭合标签和空白作为前缀，一旦遇到开始标签就停止 + # 因为开始标签可能包裹着后面的可翻译文本 + safe_prefix_end = 0 + for i in range(first_trans_idx): + if part_types[i] == 'tag': + tag = parts[i] + # 检查是否是自闭合标签或结束标签（不太可能在开头） + is_self_closing = tag.endswith('/>') or re.match(r'<(br|hr|img|meta|link)\b', tag, re.I) + is_closing = tag.startswith('，紧跟着结束标签） + is_empty_element = False + if not is_self_closing and not is_closing and i + 1 < first_trans_idx: + # 查看下一个标签是否是对应的结束标签 + next_idx = i + 1 + while next_idx < first_trans_idx and part_types[next_idx] in ('whitespace',): + next_idx += 1 + if next_idx < first_trans_idx and part_types[next_idx] == 'tag': + next_tag = parts[next_idx] + if next_tag.startswith('') or re.match(r'<(br|hr|img|meta|link)\b', tag, re.I) + if is_closing or is_self_closing: + safe_suffix_start = i + else: + # 遇到开始标签，不能作为独立后缀剥离 + break + elif part_types[i] == 'whitespace': + safe_suffix_start = i + else: + # 遇到普通文本或公式，停止剥离 + break + + # 重新分割 + prefix_parts = parts[:safe_prefix_end] + middle_parts = parts[safe_prefix_end:safe_suffix_start] + middle_types = part_types[safe_prefix_end:safe_suffix_start] + suffix_parts = parts[safe_suffix_start:] # === Drop Cap 检测 === # 英文书籍常用首字母放大样式，如 This @@ -308,23 +388,47 @@ class FormatExtractor: result_parts.append(f"φ{pid}φ") elif ptype in ('formula', 'whitespace'): - # 公式或空白，检查是否是连续块的开始 - block_parts = [] - while i < len(middle_parts) and middle_types[i] in ('formula', 'whitespace'): - block_parts.append(middle_parts[i]) - i += 1 - if block_parts: - block_html = "".join(block_parts) - pid = str(placeholder_counter) - placeholder_counter += 1 - local_map[pid] = block_html - result_parts.append(f"φ{pid}φ") + # 简化处理：非可翻译文本直接保留 + # 公式检测等复杂逻辑仅在增强模式下启用 + result_parts.append(part) + i += 1 else: i += 1 text_with_ph = "".join(result_parts) text_with_ph = re.sub(r'\s+', ' ', text_with_ph).strip() + # === 连续占位符合并 === + # 将 φ1φφ2φ 这样的连续占位符合并为一个 + def merge_consecutive_placeholders(text: str, ph_map: dict) -> Tuple[str, dict]: + """合并连续占位符""" + # 匹配连续的占位符（2个或更多） + pattern = r'(φ/?\d+φ)(φ/?\d+φ)+' + + def merge_match(m): + full_match = m.group(0) + # 提取所有占位符ID + pids = re.findall(r'φ(/?\d+)φ', full_match) + if len(pids) <= 1: + return full_match + + # 合并对应的 HTML + merged_html = "" + for pid in pids: + if pid in ph_map: + merged_html += ph_map[pid] + del ph_map[pid] + + # 创建新的合并占位符 + new_pid = pids[0] if pids[0].isdigit() else pids[0][1:] # 使用第一个数字 + ph_map[new_pid] = merged_html + return f"φ{new_pid}φ" + + merged_text = re.sub(pattern, merge_match, text) + return merged_text, ph_map + + text_with_ph, local_map = merge_consecutive_placeholders(text_with_ph, local_map) + return text_with_ph, local_map @@ -430,6 +534,7 @@ class FormatExtractor: 条件（满足任一即可）： 1. 包含 3 个及以上连续字母（如 "and", "Art", "War"） 2. 包含空格分隔的多个单词（如 "and Sun Tzu's"） + 3. 包含数字（如章节号 "10", "12"） """ text = text.strip() if not text: @@ -440,6 +545,9 @@ class FormatExtractor: # 条件2: 包含空格的多单词文本（如 "a of"） if ' ' in text and re.search(r'[a-zA-Z]', text): return True + # 条件3: 包含数字（章节号等） + if re.search(r'\d', text): + return True return False def _is_formula_element(self, element: Tag, text_content: str) -> bool: @@ -463,4 +571,10 @@ class FormatExtractor: def reset(self): """兼容旧接口""" - pass \ No newline at end of file + pass + + def _is_pure_punctuation(self, text: str) -> bool: + """判断文本是否仅包含标点符号和空格（不应该变成占位符）""" + # 常见标点符号集合（中英文混合） + punctuation_chars = ' ,.:;!?，。：；！？、""\'\'「」【】（）()[]{}—-–…·' + return all(c in punctuation_chars for c in text) \ No newline at end of file diff --git a/src/format_restorer.py b/archive/v0.09/src/format_restorer.py similarity index 84% rename from src/format_restorer.py rename to archive/v0.09/src/format_restorer.py index 45364af..ce9fbdb 100644 --- a/src/format_restorer.py +++ b/archive/v0.09/src/format_restorer.py @@ -61,8 +61,18 @@ class FormatRestorer: unknown_ids = found_ids - expected_ids if unknown_ids: - logger.warning(f"格式还原警告: 发现未知占位符 {unknown_ids}") - success = False # 未知占位符也视为问题 + # 过滤掉冗余的闭合标签（例如 map里有 "1"，但 LLM 输出了 "φ/1φ"） + real_unknowns = set() + for pid in unknown_ids: + # 如果是 /N，且 N 在 map 中，则认为是安全的冗余闭合 + if pid.startswith('/') and pid[1:] in expected_ids: + continue + real_unknowns.add(pid) + + if real_unknowns: + logger.warning(f"格式还原警告: 发现未知占位符 {real_unknowns}") + success = False + # 替换占位符 def replace_match(match): diff --git a/src/llm_client.py b/archive/v0.09/src/llm_client.py similarity index 62% rename from src/llm_client.py rename to archive/v0.09/src/llm_client.py index 84af12c..207f0bc 100644 --- a/src/llm_client.py +++ b/archive/v0.09/src/llm_client.py @@ -71,8 +71,9 @@ class LLMClient: try: with open("config/prompts.json", "r", encoding="utf-8") as f: return json.load(f) - except: - return {} + except Exception as e: + logger.error(f"严重错误：无法加载 config/prompts.json: {e}") + raise # 必须抛出异常，否则 System Prompt 会降级导致占位符指令丢失 async def translate_chunk(self, items: List[ManifestItem], glossary: Dict = None, instruction: str = None, model_type: str = "fast", @@ -89,6 +90,9 @@ class LLMClient: """ if not items: return {} + + + model = self.models.get(model_type, self.models.get("fast")) prompt = self._build_prompt(items, mode) @@ -96,21 +100,9 @@ class LLMClient: # Build System Prompt base_sys_prompt = self.prompts.get("translation", {}).get("system", "You are a professional translator.") - # 中文模式：添加占位符保护指令 + # 中文模式：Prompt 已在 config/prompts.json 中配置，无需额外硬编码 if mode == "chinese": - base_sys_prompt += """ - -Placeholder Instructions (CRITICAL): -1. Text contains PAIRED placeholders: φNφ (start) and φ/Nφ (end), like HTML tags. -2. Example: "φ1φTable Talkφ/1φ" means italic text, translate as "φ1φ桌谈φ/1φ" -3. Single placeholders φNφ without φ/Nφ are inline elements (footnotes, formulas) - keep them in place. -4. RULES: - - DO NOT create new placeholder numbers that don't exist in the original - - DO NOT remove or modify existing placeholders - - Keep placeholders in the SAME relative position in your translation - - If word order changes, keep placeholders with their associated text -5. Each line starts with paragraph ID (p_xxxxx). Preserve them. -""" + pass if instruction: @@ -123,37 +115,69 @@ Placeholder Instructions (CRITICAL): # Strict formatting instructions base_sys_prompt += "\n\nRequirements:\n1. Each line MUST start with ID (p_xxxxx).\n2. DO NOT modify IDs.\n3. Return only translations." + + + # DEBUG: 打印发送给 LLM 的完整内容 + logger.debug(f"=== LLM REQUEST DEBUG ===") + logger.debug(f"System Prompt:\n{base_sys_prompt[:500]}...") + logger.debug(f"User Prompt (first 1000 chars):\n{prompt[:1000]}") + logger.debug(f"=========================") + raw_response = await self._make_request(model, base_sys_prompt, prompt) + + + # DEBUG: 打印 LLM 返回的完整内容 + logger.debug(f"=== LLM RESPONSE DEBUG ===") + logger.debug(f"Raw Response (first 1500 chars):\n{raw_response[:1500] if raw_response else 'EMPTY'}") + logger.debug(f"==========================") + if not raw_response: return {item.global_id: f"[Error - Empty Response]" for item in items} - return self._simple_parse(raw_response, items, mode) + results = self._simple_parse(raw_response, items, mode) + + + + return results + except Exception as e: logger.error(f"Translation failed ({model}): {e}") return {item.global_id: f"[Error - {str(e)}]" for item in items} - async def repair_format(self, original_text: str, broken_translation: str) -> str: + async def repair_format(self, original_text: str, broken_translation: str, missing_ids: set = None) -> str: """ - 修复翻译格式：将占位符正确插入到译文中。 + 修复翻译格式：将缺失的占位符正确插入到译文中。 + + Args: + original_text: 原文（带占位符） + broken_translation: 有占位符问题的译文 + missing_ids: 缺失的占位符 ID 集合（可选，用于提示） """ model = self.models.get("fast") - system_prompt = "You are a format repair assistant. Your ONLY job is to insert placeholders into the translation." - user_prompt = f""" -Original Text (with placeholders): + system_prompt = """你是格式修复助手。你的任务是将缺失的占位符插入到译文中。 + +注意： +1. 不要重新翻译，保持中文译文内容完全不变 +2. 只需要在正确位置插入缺失的占位符 +3. 占位符格式：φ数字φ（如 φ1φ, φ/1φ） +4. 只输出修复后的译文，不要任何解释""" + + missing_hint = "" + if missing_ids: + missing_list = ", ".join([f"φ{pid}φ" for pid in missing_ids]) + missing_hint = f"\n缺失的占位符: {missing_list}" + + user_prompt = f"""原文（带占位符）: {original_text} -Translation (placeholders missing/incorrect): +当前译文（占位符有误）: {broken_translation} +{missing_hint} +请修复译文，在正确位置插入缺失的占位符。只输出修复后的译文：""" -Task: -Please rewrite the Translation to include ALL placeholders (φcXXXXXφ) from the Original Text. -1. DO NOT translate again. Keep the meaning of the Translation. -2. Place φcXXXXXφ tags exactly where they correspond to the original format (bold, italic, links). -3. Output ONLY the fixed translation. -""" try: return await self._make_request(model, system_prompt, user_prompt) except Exception as e: @@ -171,7 +195,14 @@ Please rewrite the Translation to include ALL placeholders (φcXXXXXφ) from the for item in items: if mode == "chinese": # 中文模式：使用带占位符的文本和段落类型 - text = item.text_with_placeholders if item.text_with_placeholders else item.clean_text + twp = item.text_with_placeholders + # DEBUG: 打印关键信息 + logger.debug(f"BUILD_PROMPT {item.global_id}: twp='{twp[:50] if twp else 'EMPTY'}...', has_φ={'φ' in twp if twp else False}") + + text = twp if twp else item.clean_text + # 防御性修复：强制清理换行符，兼容旧的脏 Manifest 数据 + text = re.sub(r'\s+', ' ', text).strip() + p_type = getattr(item, 'paragraph_type', 'body').upper() lines.append(f"{item.global_id} [{p_type}] {text}") else: @@ -180,49 +211,59 @@ Please rewrite the Translation to include ALL placeholders (φcXXXXXφ) from the return "\n".join(lines) def _simple_parse(self, response: str, items: List[ManifestItem], mode: str = "bilingual") -> Dict[str, str]: - """解析 LLM 响应""" + """ + 解析 LLM 响应 - 位置切分版 + + 策略： + 1. 识别响应中所有出现的 p_xxxxx 及其位置 + 2. 按位置顺序将响应切分成每一段，消除对输入顺序的依赖 + """ results = {} - for i, item in enumerate(items): - current_id = item.global_id - start_idx = response.find(current_id) - if start_idx == -1: continue - - end_idx = len(response) - if i + 1 < len(items): - next_id = items[i+1].global_id - next_found = response.find(next_id, start_idx + len(current_id)) - if next_found != -1: - end_idx = next_found - - content = response[start_idx:end_idx].strip() - clean_content = content[len(current_id):].strip() - clean_content = clean_content.lstrip(":： \t") - - # 移除类型标记 (如 [BODY]) - if mode == "chinese": - clean_content = re.sub(r'^\[[A-Z]+\]\s*', '', clean_content) - - if clean_content: - results[current_id] = clean_content - - # Fallback: 逐行解析 - if len(results) < len(items): + valid_ids = {item.global_id for item in items} + + # 1. 查找所有可能的 ID 位置 + # 模式匹配 p_ 后面跟着 5 位数字 + matches = list(re.finditer(r'p_\d{5}', response)) + + if not matches: + # Fallback: 如果没有匹配到任何 ID，尝试按行扫描 for line in response.split("\n"): line = line.strip() - for item in items: - if item.global_id not in results and line.startswith(item.global_id): - res = line[len(item.global_id):].strip().lstrip(":： ") - if mode == "chinese": - res = re.sub(r'^\[[A-Z]+\]\s*', '', res) - if res: results[item.global_id] = res - + for it in items: + if line.startswith(it.global_id): + content = line[len(it.global_id):].strip().lstrip(":： ") + if content: results[it.global_id] = content + return results + + # 2. 按查找到的 ID 位置进行切割 + for i, match in enumerate(matches): + current_id = match.group() + if current_id not in valid_ids: + continue + + # 这一段内容的起始是当前 ID 之后，结束是下一个匹配的 ID 之前 + start_pos = match.end() + end_pos = matches[i+1].start() if i + 1 < len(matches) else len(response) + + content = response[start_pos:end_pos].strip() + + # 3. 清理内容 + content = content.lstrip(":： \t") + if mode == "chinese": + # 移除 [BODY] 等类型标记 + content = re.sub(r'^\[[A-Z]+\]\s*', '', content) + + content = content.strip() + if content: + results[current_id] = content + # 验证解析结果 parsed_count = len(results) expected_count = len(items) if parsed_count < expected_count: missing_ids = [item.global_id for item in items if item.global_id not in results] logger.warning(f"LLM 响应解析不完整: {parsed_count}/{expected_count} (缺失: {missing_ids[:3]}...)") - + return results @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10)) diff --git a/src/manifest_manager.py b/archive/v0.09/src/manifest_manager.py similarity index 91% rename from src/manifest_manager.py rename to archive/v0.09/src/manifest_manager.py index 723c4d1..5edf48d 100644 --- a/src/manifest_manager.py +++ b/archive/v0.09/src/manifest_manager.py @@ -36,6 +36,7 @@ class ManifestItem: paragraph_type: str = "body" # 段落类型：chapter/section/subsection/epigraph/body translation_with_placeholders: str = "" # 带占位符的译文 translation_with_original_html: str = "" # 还原后的最终 HTML (中文模式) + endnote_anchors: List[str] = field(default_factory=list) # 尾注锚点 ID 列表 (用于补救) metadata: Dict[str, Any] = field(default_factory=dict) @@ -144,7 +145,10 @@ class ManifestManager: # 必须按 ID 顺序返回以保证分块正确 return sorted(items, key=lambda x: x.global_id) - def update_item(self, global_id: str, translation: str, status: str = "translated", error: str = None, model: str = None, score: int = None): + def update_item(self, global_id: str, translation: str, status: str = "translated", + error: str = None, model: str = None, score: int = None, + translation_with_placeholders: str = None, + translation_with_original_html: str = None): """更新翻译结果。""" if global_id in self._items_by_id: item = self._items_by_id[global_id] @@ -157,6 +161,11 @@ class ManifestManager: item.model_used = model if score is not None: item.quality_score = score + # 中文模式专用字段 + if translation_with_placeholders is not None: + item.translation_with_placeholders = translation_with_placeholders + if translation_with_original_html is not None: + item.translation_with_original_html = translation_with_original_html else: logger.warning(f"尝试更新不存在的 ID: {global_id}") diff --git a/src/quality_manager.py b/archive/v0.09/src/quality_manager.py similarity index 100% rename from src/quality_manager.py rename to archive/v0.09/src/quality_manager.py diff --git a/archive/v0.09/src/text_processor.py b/archive/v0.09/src/text_processor.py new file mode 100644 index 0000000..ece4118 --- /dev/null +++ b/archive/v0.09/src/text_processor.py @@ -0,0 +1,117 @@ +""" +文本处理器模块 (Text Processor Module) - Manifest 驱动版 (集成 V2) + +该模块专注于 HTML 文档的遍历和段落提取。 +已集成 FineGrainedExtractor，实现稳健的结构提取和格式保护。 +""" + +import re +from bs4 import BeautifulSoup +from typing import List, Dict, Any +from loguru import logger +from .manifest_manager import ManifestManager +from .fine_grained_extractor import FineGrainedExtractor + + +class TextProcessor: + """ + 负责从 HTML 中识别有效段落并注册到 Manifest。 + 委托 FineGrainedExtractor 进行具体的提取工作。 + """ + + def __init__(self, config: Dict): + """ + Args: + config (Dict): 全局配置。 + """ + self.config = config + self.chunk_size = config['translation'].get('chunk_size', 5000) + # 不再持有状态，每次调用实例化 Extractor 或复用 + + def extract_to_manifest(self, html_content: str, source_file: str, manifest: ManifestManager, mode: str = "bilingual"): + """ + 解析 HTML 内容，并将识别出的段落注册到 Manifest 中。 + + Args: + html_content (str): HTML 源码。 + source_file (str): 来源文件名。 + manifest (ManifestManager): 清单管理器实例。 + mode (str): 翻译模式 (保留参数) + """ + try: + # 实例化细粒度提取器 (集成格式保护) + # 是否翻译目录取决于文件名判断，这里交给 Extractor 内部逻辑 + # 但 Extractor 构造函数需要参数，默认 False + translate_toc = self.config['translation'].get('translate_toc', False) + extractor = FineGrainedExtractor(translate_toc=translate_toc) + + items = extractor.extract(html_content, source_file) + + count = 0 + for item in items: + # 只注册需要翻译的项 + if not item['should_translate']: + continue + + clean_text = item.get('text', '') + text_with_ph = item.get('text_with_ph', clean_text) + placeholder_map = item.get('placeholder_map') + + # 注册到 Manifest + # original_html 用于记录，但实际回填依靠 FineGrainedExtractor 复原 + manifest_item = manifest.add_item( + source_file=source_file, + original_html=str(item['element']), + clean_text=clean_text, + tag=item['tag'], + metadata={"status": "pending"} + ) + + # 显式设置格式保护字段 + manifest_item.text_with_placeholders = text_with_ph + manifest_item.placeholder_map = placeholder_map + manifest_item.endnote_anchors = item.get('endnote_anchors', []) + + # 记录段落类型 (从 FormatExtractor 获得的 p_type，目前 FineGrained 没返回，可以改进) + # FineGrained 可以把 FormatExtractor 返回的 p_type 也带出来 + # 暂且设为 body，或根据 tag 判断 + p_type = "header" if item['tag'].startswith('h') else "body" + manifest_item.paragraph_type = p_type + + count += 1 + + logger.info(f"提取完成 {source_file}: 注册 {count} 个待翻译项") + + except Exception as e: + logger.error(f"从 {source_file} 提取段落失败: {e}", exc_info=True) + + def create_chunks_from_manifest(self, manifest: ManifestManager, mode: str = "bilingual") -> List[List[Any]]: + """ + 从 Manifest 中筛选待翻译项目并分块。 + (保留原有逻辑) + """ + pending_items = manifest.get_items(status="pending") + if not pending_items: + return [] + + chunks = [] + current_chunk = [] + current_size = 0 + + for item in pending_items: + # 优先使用带占位符的文本长度计算 + text_len = len(item.text_with_placeholders) if item.text_with_placeholders else len(item.clean_text) + + if current_size + text_len > self.chunk_size and current_chunk: + chunks.append(current_chunk) + current_chunk = [] + current_size = 0 + + current_chunk.append(item) + current_size += text_len + + if current_chunk: + chunks.append(current_chunk) + + logger.info(f"分块完成: 共有 {len(pending_items)} 个待翻译项，分为 {len(chunks)} 个块") + return chunks \ No newline at end of file diff --git a/src/toc_parser.py b/archive/v0.09/src/toc_parser.py similarity index 100% rename from src/toc_parser.py rename to archive/v0.09/src/toc_parser.py diff --git a/archive/v0.09/src/translator.py b/archive/v0.09/src/translator.py new file mode 100644 index 0000000..1bf867b --- /dev/null +++ b/archive/v0.09/src/translator.py @@ -0,0 +1,348 @@ +""" +EPUB Translator Core Module - v0.09 (TOC Selection Support) +""" + +import asyncio +import traceback +from typing import List, Dict, Any +from pathlib import Path +from loguru import logger +from rich.console import Console +from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TimeElapsedColumn + +from .epub_parser import EPUBParser +from .toc_parser import TOCParser +from .llm_client import LLMClient +from .text_processor import TextProcessor +from .bilingual_builder import BilingualEPUBBuilder +from .chinese_builder import ChineseEPUBBuilder +from .manifest_manager import ManifestManager +from .book_profiler import BookProfiler +from .cache import TranslationCache +from .format_restorer import FormatRestorer +from .utils import add_spacing_between_cn_and_en_num + + +class EPUBTranslator: + + def __init__(self, config: Dict, use_cache: bool = True): + self.config = config + self.console = Console() + self.use_cache = use_cache + + self.parser = None + self.llm_client = LLMClient(config) + self.text_processor = TextProcessor(config) + self.profiler = BookProfiler(config, self.llm_client) + self.cache = TranslationCache(config) if use_cache else None + self.restorer = FormatRestorer() + + self.manifest_dir = Path("cache/manifests") + self.manifest_dir.mkdir(parents=True, exist_ok=True) + + async def translate_epub(self, epub_path: str, test_mode: bool = False, + output_dir: str = None, mode: str = "bilingual", + from_chapter: str = None, to_chapter: str = None) -> str: + """ + 翻译 EPUB 文件 + """ + try: + epub_path = Path(epub_path) + + # --- 0. Preprocessing (集成的 EpubCleaner) --- + # 创建临时清理文件 + from .epub_cleaner import EpubCleaner + cleaner = EpubCleaner() + processed_dir = self.manifest_dir / "processed_epubs" + processed_dir.mkdir(parents=True, exist_ok=True) + cleaned_epub_path = processed_dir / f"{epub_path.stem}_cleaned.epub" + + self.console.print(f"[yellow]Preprocessing: Cleaning EPUB structures...[/yellow]") + cleaner.clean_epub(str(epub_path), str(cleaned_epub_path)) + + # 关键：后续操作全都基于清理后的 EPUB + # 注意：这会改变 source_file 的上下文吗？只要 parser 读的是 cleaned_epub，manifest 记录的就是 cleaned_epub 里的文件名。 + # 而 Builder 用 cleaned_epub 初始化的，所以也是匹配的。 + actual_epub_path = cleaned_epub_path + + self.parser = EPUBParser(str(actual_epub_path)) + + # 0.5 解析 TOC 并处理章节范围 + toc_parser = TOCParser(self.parser.book) + include_files = None + chapter_range_info = None + + if from_chapter or to_chapter: + include_files, selected_items = toc_parser.get_spine_range( + start_title=from_chapter, end_title=to_chapter + ) + if selected_items: + start_title = selected_items[0].title + end_title = selected_items[-1].title + self.console.print(f"[cyan]📚 Range: {start_title} ~ {end_title} ({len(include_files)} files)[/cyan]") + chapter_range_info = {"start_title": start_title, "end_title": end_title, "included_files": list(include_files)} + else: + skip_files = toc_parser.get_skip_files() + if skip_files: + include_files = toc_parser.get_content_files_from_spine() + self.console.print(f"[cyan]📚 Smart Skip: {len(skip_files)} non-content files[/cyan]") + + # 1. Manifest + manifest_suffix = "_chinese" if mode == "chinese" else "" + manifest_path = self.manifest_dir / f"{epub_path.stem}{manifest_suffix}_manifest.json" + manifest = ManifestManager(str(manifest_path)) + + if not manifest.load() or not self.use_cache: + self.console.print(f"[yellow]Initializing Manifest...[/yellow]") + manifest.init_manifest(book_id=epub_path.name, metadata=self.parser.get_book_info(), chapter_range=chapter_range_info) + content_items = self.parser.extract_all_content_items(include_files=include_files) + for item in content_items: + self.text_processor.extract_to_manifest(item['content'], item['file_name'], manifest, mode=mode) + manifest.save() + + # (Stats logic...) + stats = manifest.stats + self.console.print(f"[green]Manifest: {stats['total']} items ({stats['pending']} pending)[/green]") + + # 2. Profile + profile = {} + if not test_mode and stats['pending'] > 0: + self.console.print("[yellow]Profiling Book...[/yellow]") + profile = await self.profiler.analyze_book(manifest) + + # 3. Translate + chunks = self.text_processor.create_chunks_from_manifest(manifest, mode=mode) + if test_mode: chunks = chunks[:10] + if chunks: + await self._translate_concurrently(chunks, manifest, profile, mode=mode) + + # 4. Build + self.console.print(f"\n[yellow]Building {mode} EPUB...[/yellow]") + output_path = output_dir or self.config['output']['output_dir'] + + # 统一使用 BilingualEPUBBuilder (集成 V2) + # 因为它已经支持了 FineGrained backfill，可以处理 bilingual 参数 + # 但目前 builder 还没暴露 bilingual 参数给 create 方法？ + # 我们可以简单地在 builder.create... 里改一下，或者总是用 BilingualBuilder。 + # 用户想要 "chinese" mode (replace). + # BilingualBuilder.create... 目前 hardcode 了 bilingual=True (TODO comment in previous step). + # 我们应该让 BilingualBuilder 支持 mode 参数。 + # 为了简单，我假设 builder 内部会处理，或者我之后微调 builder。 + # 修改: BilingualEPUBBuilder 是通用的 backfiller。 + + # 构建 Mapping + if mode == "chinese": + # 中文模式：使用还原后的 HTML (保留格式) + translation_map = { + item.global_id: (item.translation_with_original_html or item.translation) + for item in manifest.get_items() + if item.translation_with_original_html or item.translation + } + else: + # 双语模式：使用纯文本 + translation_map = {item.global_id: item.translation for item in manifest.get_items() if item.translation} + + paragraph_map = {item.global_id: { + "file_name": item.source_file, + # 其他 metadata 其实不需要了，builder 会重新提取 + } for item in manifest.get_items()} + + builder = BilingualEPUBBuilder(self.parser.book, self.config) + # 临时 Hack: 如果是 chinese 模式，修改 builder 的逻辑 (或者 builder 自动读取 config) + # Builder 构造函数读了 config。 + # 我们需要在 config 里设置 mode 吗？或者 Builder 可以加个 set_mode? + # 这里的 config 是全局 config。main.py 里并没有把 args.mode 写入 config['output']。 + # 我们可以在这里 patch 一下 config。 + self.config['output']['mode'] = mode # 确保 Builder 知道模式 + + # 注意: 之前的 BilingualBuilder._process_document_content 里写死 bilingual_mode = True + # 我需要去修一下 BilingualBuilder，让它读 self.config['output']['mode'] + + result_file = builder.create_bilingual_epub_with_mapping(translation_map, paragraph_map, output_path) + + self.console.print(f"[green]Done: {result_file}[/green]") + return result_file + + except Exception as e: + traceback.print_exc() + logger.error(f"Translation failed: {e}") + raise + + async def _translate_concurrently(self, chunks: List[List[Any]], manifest: ManifestManager, + profile: Dict, mode: str = "bilingual"): + """并发翻译核心逻辑 (统一单双语)""" + total_chunks = len(chunks) + glossary = profile.get('glossary', {}) + instruction = profile.get('translation_instruction', "") + + with Progress( + SpinnerColumn(), TextColumn("[progress.description]{task.description}"), + BarColumn(), TextColumn("{task.percentage:>3.0f}%"), TimeElapsedColumn(), + console=self.console + ) as progress: + task_id = progress.add_task(f"[cyan]Translating...", total=total_chunks) + + async def worker(chunk, idx): + try: + chunk_dicts = [item.to_dict() for item in chunk] + results = None + model_name = self.llm_client.models.get('fast', 'unknown') + + if self.cache: + results = self.cache.get_chunk_translation(chunk_dicts, model=model_name) + + if not results: + # FIX: Use keyword arguments to avoid positional mismatch (model_type vs mode) + results = await self.llm_client.translate_chunk( + items=chunk, + glossary=glossary, + instruction=instruction, + mode=mode + ) + if self.cache and results: + self.cache.save_chunk_translation(chunk_dicts, results, model=model_name) + + for item in chunk: + if item.global_id in results: + raw_trans = results[item.global_id] + + if "[Error" in raw_trans: + manifest.update_item(item.global_id, None, status="failed", error=raw_trans) + continue + + processed_trans = add_spacing_between_cn_and_en_num(raw_trans) + + # 仅保存译文，还原逻辑外移至所有翻译完成后执行 + manifest.update_item( + item.global_id, + processed_trans, + translation_with_placeholders=processed_trans, + status="translated" + ) + else: + manifest.update_item(item.global_id, None, status="failed", error="Translate failed: ID not found in response") + except Exception as e: + logger.error(f"Worker {idx} error: {e}") + finally: + progress.update(task_id, advance=1) + # 并发执行翻译任务 + tasks = [worker(chunk, i) for i, chunk in enumerate(chunks)] + await asyncio.gather(*tasks) + + # 翻译完成后立即保存，确保数据持久化 + manifest.save() + logger.info("翻译阶段完成，manifest 已保存") + + # --- 第二阶段：统一进行格式还原与修复 --- + logger.info("开始进行格式还原与占位符校验...") + await self.process_format_restoration(manifest, mode) + + async def process_format_restoration(self, manifest, mode): + """统一处理所有段落的格式还原和修复""" + items = manifest.get_items() + success_count = 0 + failed_count = 0 + repaired_count = 0 + + for item in items: + # 仅处理已翻译或之前格式还原失败的项目，或者已完成但缺少还原HTML的项目 + should_process = ( + item.status == "translated" or + item.status == "format_error" or + (item.status == "completed" and not item.translation_with_original_html) + ) + + if not should_process or not item.translation_with_placeholders: + continue + + + + processed_trans = item.translation_with_placeholders + translation_with_ph = processed_trans + restored_html = "" + success = False + + if item.placeholder_map: + # 过滤内嵌占位符（排除 _prefix, _suffix） + inner_placeholders = {k: v for k, v in item.placeholder_map.items() + if not k.startswith("_")} + + if not inner_placeholders: + # 没有内嵌占位符，清除可能多出的占位符 + clean_translation = self.restorer._strip_placeholders(processed_trans) + translation_with_ph = clean_translation + restored_html, success = self.restorer.restore(clean_translation, item.placeholder_map) + else: + # 有内嵌占位符，尝试直接还原 + restored_html, success = self.restorer.restore(processed_trans, item.placeholder_map) + + if not success: + # 尝试修复逻辑 + import re + found_ids = set(re.findall(r'φ(/?\d+)φ', processed_trans)) + expected_ids = set(inner_placeholders.keys()) + missing_ids = expected_ids - found_ids + + if missing_ids: + logger.warning(f"占位符缺失 (ID: {item.global_id}), 尝试修复: {missing_ids}") + try: + fixed_trans = await self.llm_client.repair_format( + item.text_with_placeholders, + processed_trans, + missing_ids=missing_ids + ) + restored_html_2, success_2 = self.restorer.restore(fixed_trans, item.placeholder_map) + if success_2: + repaired_count += 1 + translation_with_ph = fixed_trans + restored_html = restored_html_2 + success = True + else: + # 尾注补救 + if hasattr(item, 'endnote_anchors') and item.endnote_anchors: + still_missing = [a for a in item.endnote_anchors + if f"φ{a}φ" not in fixed_trans] + if still_missing: + for anchor_id in still_missing: + fixed_trans += f"φ{anchor_id}φ" + restored_html_3, success_3 = self.restorer.restore(fixed_trans, item.placeholder_map) + if success_3: + translation_with_ph = fixed_trans + restored_html = restored_html_3 + success = True + except Exception as e: + logger.error(f"修复失败 (ID: {item.global_id}): {e}") + else: + # 无需还原 + restored_html = processed_trans + success = True + + # 更新 Manifest + final_translation = self.restorer._strip_placeholders(translation_with_ph) + manifest.update_item( + item.global_id, + final_translation, + translation_with_placeholders=translation_with_ph, + translation_with_original_html=restored_html, + status="completed" if success else "format_error" + ) + + if success: success_count += 1 + else: failed_count += 1 + + # 保存还原结果 + manifest.save() + + # 统计错误比例 + total_processed = success_count + failed_count + if total_processed > 0: + error_rate = failed_count / total_processed + logger.info(f"占位符还原完成: 成功 {success_count}, 失败 {failed_count}, 修复 {repaired_count}, 错误率 {error_rate*100:.2f}%") + + # 超过 1% 错误率，判定为严重问题，中止操作 + if error_rate > 0.01: + error_msg = f"占位符错误率过高 ({error_rate*100:.2f}% > 1%)，检测到严重不可修复问题，中止操作" + logger.error(error_msg) + raise RuntimeError(error_msg) + else: + logger.warning("没有处理任何段落") diff --git a/src/utils.py b/archive/v0.09/src/utils.py similarity index 100% rename from src/utils.py rename to archive/v0.09/src/utils.py diff --git a/archive/v0.09/test_build_prompt.py b/archive/v0.09/test_build_prompt.py new file mode 100644 index 0000000..23a09c8 --- /dev/null +++ b/archive/v0.09/test_build_prompt.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""模拟翻译流程，检查 _build_prompt 使用的数据""" +import sys +sys.path.insert(0, '.') + +from src.manifest_manager import ManifestManager +from src.text_processor import TextProcessor + +# 加载 manifest +manifest = ManifestManager("cache/manifests/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)_chinese_manifest.json") +manifest.load() + +# 创建 chunks (模拟 create_chunks_from_manifest) +processor = TextProcessor({}) +chunks = processor.create_chunks_from_manifest(manifest, mode="chinese") + +print(f"Total chunks: {len(chunks)}") + +if chunks: + # 取第一个 chunk 的前几个 item + first_chunk = chunks[0] + print(f"First chunk has {len(first_chunk)} items") + + for item in first_chunk[:5]: + print(f"\n=== ITEM {item.global_id} ===") + print(f" type(item): {type(item)}") + print(f" clean_text: '{item.clean_text[:50]}...'") + print(f" text_with_placeholders: '{item.text_with_placeholders[:50] if item.text_with_placeholders else 'EMPTY'}...'") + print(f" has φ in twp: {'φ' in (item.text_with_placeholders or '')}") + print(f" placeholder_map: {item.placeholder_map}") + + # 模拟 _build_prompt 的逻辑 + if item.text_with_placeholders: + text = item.text_with_placeholders + else: + text = item.clean_text + prompt_line = f"{item.global_id} [BODY] {text}" + print(f" -> Will send to LLM: '{prompt_line[:80]}...'") + print(f" -> Prompt contains φ: {'φ' in prompt_line}") diff --git a/archive/v0.09/test_build_prompt2.py b/archive/v0.09/test_build_prompt2.py new file mode 100644 index 0000000..b584861 --- /dev/null +++ b/archive/v0.09/test_build_prompt2.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""直接检查 manifest 中的 pending items""" +import sys +sys.path.insert(0, '.') + +from src.manifest_manager import ManifestManager + +# 加载 manifest +manifest = ManifestManager("cache/manifests/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)_chinese_manifest.json") +manifest.load() + +# 获取 pending items（这是 create_chunks_from_manifest 内部调用的） +pending_items = manifest.get_items(status="pending") +print(f"Pending items: {len(pending_items)}") + +# 如果没有 pending，获取所有 +if not pending_items: + print("No pending items, getting all...") + pending_items = manifest.get_items() + print(f"All items: {len(pending_items)}") + +# 检查前 10 个 item +for item in pending_items[:10]: + twp = item.text_with_placeholders + has_phi = 'φ' in twp if twp else False + inner_ph = {k: v for k, v in item.placeholder_map.items() if not k.startswith('_')} if item.placeholder_map else {} + + print(f"\n{item.global_id}:") + print(f" status: {item.status}") + print(f" clean_text: '{item.clean_text[:40]}...'") + print(f" text_with_placeholders: '{twp[:40] if twp else 'EMPTY'}...'") + print(f" has φ: {has_phi}") + print(f" inner_placeholders: {list(inner_ph.keys())}") diff --git a/archive/v0.09/test_cache_check.py b/archive/v0.09/test_cache_check.py new file mode 100644 index 0000000..bdb7578 --- /dev/null +++ b/archive/v0.09/test_cache_check.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""检查翻译缓存中的结果是否包含占位符""" +import json +import os + +translations_dir = "cache/translations" +all_translations = {} + +# 遍历缓存文件 +for root, dirs, files in os.walk(translations_dir): + for f in files: + if f.endswith('.json'): + path = os.path.join(root, f) + with open(path, 'r') as fp: + cache = json.load(fp) + if 'translations' in cache: + for k, v in cache['translations'].items(): + all_translations[k] = {'trans': v, 'file': path} + +# 检查 p_00006 (已知有占位符的 item) +target_id = 'p_00006' +if target_id in all_translations: + data = all_translations[target_id] + print(f"=== CACHED TRANSLATION FOR {target_id} ===") + print(f"Cache file: {data['file']}") + print(f"Translation: {data['trans']}") + print(f"Contains φ: {'φ' in data['trans']}") +else: + print(f"{target_id} not found in cache") + +# 统计有多少缓存翻译包含 φ +with_phi = sum(1 for v in all_translations.values() if 'φ' in v['trans']) +total = len(all_translations) +print(f"\n=== CACHE STATS ===") +print(f"Total cached translations: {total}") +print(f"Translations with φ: {with_phi}") +print(f"Translations without φ: {total - with_phi}") diff --git a/archive/v0.09/test_extractor.py b/archive/v0.09/test_extractor.py new file mode 100644 index 0000000..4bac386 --- /dev/null +++ b/archive/v0.09/test_extractor.py @@ -0,0 +1,31 @@ +import re +from src.format_extractor import FormatExtractor + +extractor = FormatExtractor() + +cases = [ + ("“But what is the goal?” Amodei...
", "Quoted text with em"), + ("Q. What is artificial intelligence?
", "Simple Q&A"), + ("Text italic followed by dots...
", "Italic with trailing dots"), +] + +for html, desc in cases: + print(f"--- Testing: {desc} ---") + print(f"HTML: {html}") + clean, text_ph, ph_map, p_type, anchors = extractor.extract(html) + + # Validation logic from FormatExtractor.extract + stripped_text = re.sub(r'φ/?[0-9]+φ', '', text_ph) + stripped_text = re.sub(r'\s+', ' ', stripped_text).strip() + clean_normalized = re.sub(r'\s+', ' ', clean).strip() + + print(f"Clean: '{clean_normalized}'") + print(f"Stripped: '{stripped_text}'") + print(f"Text ph: '{text_ph}'") + print(f"Map: {ph_map}") + + if clean_normalized == stripped_text: + print("✅ SUCCESS") + else: + print("❌ FAILED") + print() diff --git a/archive/v0.09/test_extractor_v3.py b/archive/v0.09/test_extractor_v3.py new file mode 100644 index 0000000..a98920d --- /dev/null +++ b/archive/v0.09/test_extractor_v3.py @@ -0,0 +1,38 @@ +import re +from src.format_extractor import FormatExtractor + +extractor = FormatExtractor() + +cases = [ + ("“But what is the goal?” Amodei...
", "Quoted text with em"), + ("Q. What is artificial intelligence?
", "Simple Q&A"), + ("Text italic followed by dots...
", "Italic with trailing dots"), +] + +for html, desc in cases: + print(f"--- Testing: {desc} ---") + + # Simulate how extract() identifies inner_html + from bs4 import BeautifulSoup, Tag + soup = BeautifulSoup(html, 'html.parser') + root = list(soup.children)[0] if list(soup.children) else soup + clean_text = root.get_text().strip() + clean_text = re.sub(r'\s+', ' ', clean_text).strip() + inner_html = root.decode_contents() if isinstance(root, Tag) else str(root) + + # Call v3 directly + text_with_ph, local_map = extractor._smart_extract_v3(inner_html) + + stripped_text = re.sub(r'φ/?[0-9]+φ', '', text_with_ph) + stripped_text = re.sub(r'\s+', ' ', stripped_text).strip() + + print(f"Clean: '{clean_text}'") + print(f"Stripped: '{stripped_text}'") + print(f"Text ph: '{text_with_ph}'") + print(f"Map: {local_map}") + + if clean_text == stripped_text: + print("✅ SUCCESS") + else: + print("❌ FAILED") + print() diff --git a/archive/v0.09/test_llm_client_io.py b/archive/v0.09/test_llm_client_io.py new file mode 100644 index 0000000..dedb779 --- /dev/null +++ b/archive/v0.09/test_llm_client_io.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +""" +最小化测试脚本：测试 LLMClient 的输入输出 +完全模拟真实调用路径，排除 Translator/TextProcessor 的干扰 +""" +import sys +import asyncio +import json +import logging +from loguru import logger +from src.llm_client import LLMClient +from src.manifest_manager import ManifestItem +from src.utils import load_config + +# 配置日志输出到控制台 +logger.remove() +logger.add(sys.stdout, level="DEBUG") + +async def test_io(): + print("=== 初始化 LLMClient ===") + config = load_config() + # 使用 v3 provider + if 'v3' in config['providers']: + config['llm'] = config['providers']['v3'] + print(f"Using provider: v3 (model: {config['llm']['models']['fast']})") + + client = LLMClient(config) + + # 构造测试 Item (模拟真实数据) + item = ManifestItem( + global_id="p_00006", + source_file="test.html", + original_html="in the name of abundance...
", + clean_text="in the name of abundance...", + text_hash="dummy_hash", + tag="p", + # 关键：设置 text_with_placeholders + text_with_placeholders="in the name of φ1φabundance...", # 故意不加空格，模拟原始数据 + placeholder_map={"1": ""}, + paragraph_type="BODY" + ) + + items = [item] + mode = "chinese" + + print("\n=== 1. 测试 _build_prompt 输出 ===") + # 直接调用私有方法查看生成的 prompt + prompt = client._build_prompt(items, mode=mode) + print(f"Generated Prompt:\n{prompt}") + print(f"Contains φ1φ: {'φ1φ' in prompt}") + + print("\n=== 2. 测试 translate_chunk 完整调用 ===") + # 这会触发我们之前添加的 ERROR/DEBUG 日志 + results = await client.translate_chunk(items, mode=mode) + + print("\n=== 3. 检查结果 ===") + trans = results.get("p_00006", "MISSING") + print(f"Translation: {trans}") + print(f"Contains φ: {'φ' in trans}") + +if __name__ == "__main__": + asyncio.run(test_io()) diff --git a/archive/v0.09/test_llm_placeholder.py b/archive/v0.09/test_llm_placeholder.py new file mode 100644 index 0000000..408ffc5 --- /dev/null +++ b/archive/v0.09/test_llm_placeholder.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +""" +测试 LLM 是否正确保留占位符 +""" +import asyncio +import json +from openai import AsyncOpenAI + +# 加载配置 +with open("config/config.json", "r") as f: + config = json.load(f) + +v3_config = config["providers"]["v3"] + +# 从 .env 加载 API Key +import os +from dotenv import load_dotenv +load_dotenv() +api_key = os.getenv("V3_API_KEY") or v3_config.get("api_key", "") + +# 加载 prompts +with open("config/prompts.json", "r") as f: + prompts = json.load(f) + +system_prompt = prompts["translation"]["system"] + +# 测试文本 - 包含占位符 +test_text = """p_00058 [BODY] Also that night, the board and the remaining leadership at the company were holding a series of increasingly hostile meetings. After the all-φ1φhands, the false projection of unity between Sutskever and the other leaders had collapsed. Many of the executives who had sat next to Sutskever during the livestream had been nearly as blindsided as the rest of the staff, having learned of Altman's dismissal moments before it was announced. φ2φRiled up by Sutskever's poor performance, they had demanded to meet with the rest of the board. Roughly a dozen executives, including Murati and Lightcap, had gathered in a conference room at the office.""" + +async def test_translation(): + client = AsyncOpenAI( + base_url=v3_config["base_url"], + api_key=api_key, + default_headers=v3_config.get("extra_headers", {}) + ) + + model = v3_config["models"]["fast"] + + print("=" * 60) + print("SYSTEM PROMPT:") + print("=" * 60) + print(system_prompt) + print() + print("=" * 60) + print("USER PROMPT:") + print("=" * 60) + print(test_text) + print() + print("=" * 60) + print(f"Calling LLM ({model})...") + print("=" * 60) + + response = await client.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": test_text} + ], + temperature=0.3 + ) + + result = response.choices[0].message.content + + print() + print("=" * 60) + print("LLM RESPONSE:") + print("=" * 60) + print(result) + print() + + # 检查占位符 + has_phi1 = "φ1φ" in result + has_phi2 = "φ2φ" in result + print("=" * 60) + print("PLACEHOLDER CHECK:") + print(f" φ1φ present: {has_phi1}") + print(f" φ2φ present: {has_phi2}") + print("=" * 60) + +if __name__ == "__main__": + asyncio.run(test_translation()) diff --git a/archive/v0.09/test_manifest_data.py b/archive/v0.09/test_manifest_data.py new file mode 100644 index 0000000..3790360 --- /dev/null +++ b/archive/v0.09/test_manifest_data.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""验证 manifest 数据和 _build_prompt 输出""" +import json + +manifest_file = "cache/manifests/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)_chinese_manifest.json" +with open(manifest_file, 'r') as f: + data = json.load(f) + +items = data.get('items', []) + +# 找一个有占位符的 item +target = None +for item in items: + if item.get('placeholder_map') and len(item['placeholder_map']) > 0: + inner = {k: v for k, v in item['placeholder_map'].items() if not k.startswith('_')} + if inner: + target = item + break + +if target: + print("=== TARGET ITEM ===") + print(f"global_id: {target['global_id']}") + print(f"status: {target['status']}") + print(f"clean_text: {target['clean_text'][:80]}...") + print(f"text_with_placeholders: '{target['text_with_placeholders'][:80]}...'") + print(f"placeholder_map: {target['placeholder_map']}") + print() + + # 模拟 _build_prompt 的行为 + text_with_ph = target['text_with_placeholders'] + if text_with_ph: + prompt_line = f"{target['global_id']} [BODY] {text_with_ph}" + else: + prompt_line = f"{target['global_id']} [BODY] {target['clean_text']}" + + print("=== SIMULATED PROMPT LINE ===") + print(prompt_line[:150]) + print() + + # 检查 text_with_placeholders 是否包含 φ + has_phi = 'φ' in (text_with_ph or '') + print(f"text_with_placeholders contains φ: {has_phi}") +else: + print("No item with placeholders found") diff --git a/archive/v0.09/test_manifest_load.py b/archive/v0.09/test_manifest_load.py new file mode 100644 index 0000000..6322331 --- /dev/null +++ b/archive/v0.09/test_manifest_load.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""验证 manifest 加载后 ManifestItem 对象的 text_with_placeholders""" +import sys +sys.path.insert(0, '.') + +from src.manifest_manager import ManifestManager + +manifest = ManifestManager("cache/manifests/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)_chinese_manifest.json") +manifest.load() + +# 获取有占位符的 item +items = manifest.get_items(status="pending") +if not items: + items = manifest.get_items() # 任意状态 + +target = None +for item in items: + if item.placeholder_map: + inner = {k: v for k, v in item.placeholder_map.items() if not k.startswith('_')} + if inner: + target = item + break + +if target: + print("=== ManifestItem OBJECT ===") + print(f"global_id: {target.global_id}") + print(f"status: {target.status}") + print(f"clean_text: '{target.clean_text[:80]}...'") + print(f"text_with_placeholders: '{target.text_with_placeholders[:80] if target.text_with_placeholders else 'EMPTY'}...'") + print(f"placeholder_map: {target.placeholder_map}") + print() + + # 检查是否包含 φ + twp = target.text_with_placeholders + has_phi = 'φ' in twp if twp else False + print(f"text_with_placeholders contains φ: {has_phi}") + print(f"text_with_placeholders length: {len(twp) if twp else 0}") +else: + print("No target item found with placeholders") diff --git a/archive/v0.09/test_manifest_to_prompt.py b/archive/v0.09/test_manifest_to_prompt.py new file mode 100644 index 0000000..52fa1a2 --- /dev/null +++ b/archive/v0.09/test_manifest_to_prompt.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +""" +最小测试脚本：测试 Manifest -> TextProcessor -> LLMClient 的数据流 +验证 text_with_placeholders 是否在传递过程中丢失或损坏 +""" +import sys +import json +import asyncio +from pathlib import Path +from loguru import logger + +# 添加 src 到路径 +sys.path.insert(0, '.') + +from src.manifest_manager import ManifestManager, ManifestItem +from src.text_processor import TextProcessor +from src.llm_client import LLMClient +# Mock config +from src.utils import load_config + +# 配置日志 +logger.remove() +logger.add(sys.stdout, level="DEBUG") + +# 1. 创建临时的 Manifest 文件，模拟包含问题的真实数据 +# 模拟一个带有换行符的占位符文本，这是我们怀疑的根源 +mock_manifest_data = { + "metadata": {"book_id": "test_book"}, + "items": [ + { + "global_id": "p_00006", + "source_file": "test.html", + "original_html": "in the name of abundance
", + "clean_text": "in the name of abundance", + "text_hash": "hash1", + "tag": "p", + # 模拟包含换行符的情况 (FormatExtractor 之前的问题) + "text_with_placeholders": "in the name of \nφ1φ\n abundance", + "placeholder_map": {"1": ""}, + "paragraph_type": "BODY", + "status": "pending" + }, + { + "global_id": "p_00058", + "source_file": "test.html", + "original_html": "all-hands
", + "clean_text": "all-hands", + "text_hash": "hash2", + "tag": "p", + # 正常情况 + "text_with_placeholders": "all-φ1φhands", + "placeholder_map": {"1": ""}, + "paragraph_type": "BODY", + "status": "pending" + } + ] +} + +manifest_path = Path("cache/test_manifest.json") +manifest_path.parent.mkdir(parents=True, exist_ok=True) +with open(manifest_path, 'w') as f: + json.dump(mock_manifest_data, f) + +print(f"=== Created Mock Manifest at {manifest_path} ===") + +async def run_test(): + # 2. 加载 Manifest + manifest = ManifestManager(str(manifest_path)) + manifest.load() + print(f"Loaded {len(manifest.get_items())} items") + + # 3. 创建 TextProcessor 和 Chunks + # Mock config for processor + processor = TextProcessor({"translation": {"chunk_size": 1000}}) + + # 这一步会从 manifest 读取 item + chunks = processor.create_chunks_from_manifest(manifest, mode="chinese") + print(f"Created {len(chunks)} chunks") + + chunk = chunks[0] + print(f"Chunk 0 has {len(chunk)} items") + + # 4. 模拟 LLMClient 构建 Prompt + # 不需要真正的 API key，只需要测试 _build_prompt + # 添加 dummy key 和 rate_limits 防止初始化报错 + client = LLMClient({ + "providers": {}, + "llm": { + "api_key": "dummy_key", + "models": {"fast": "dummy_model", "smart": "dummy_model"}, + "rate_limits": {"requests_per_minute": 60, "concurrent_requests": 2} + } + }) + + # 强制重新加载 prompts (确保我们使用最新的代码逻辑) + # 注意：我们之前修了 _load_prompts，如果 prompts.json 不存在会报错 + # 这里我们假设 config/prompts.json 存在 + + print("\n=== 构建 Prompt (Mode: Chinese) ===") + prompt = client._build_prompt(chunk, mode="chinese") + + print("-" * 40) + print(prompt) + print("-" * 40) + + # 验证关键点 + print("\n=== 验证结果 ===") + + # 检查 p_00006 + # 注意：我们之前修了 FormatExtractor，但那是针对**新提取**的内容。 + # 这里我们测试的是**从旧 Manifest 读取**的内容。 + # ManifestManager 读取时并不会自动清理换行符！ + # 所以如果旧 manifest 里有换行，这里应该能复现出带换行的 prompt。 + + has_p00006 = "p_00006 [BODY] in the name of \nφ1φ\n abundance" in prompt + print(f"p_00006 has newlines (bad): {has_p00006}") + + has_p00006_clean = "p_00006 [BODY] in the name of φ1φ abundance" in prompt + print(f"p_00006 is clean (good): {has_p00006_clean}") + + has_p00058 = "p_00058 [BODY] all-φ1φhands" in prompt + print(f"p_00058 is correct: {has_p00058}") + +if __name__ == "__main__": + asyncio.run(run_test()) diff --git a/archive/v0.09/test_newline_effect.py b/archive/v0.09/test_newline_effect.py new file mode 100644 index 0000000..7751208 --- /dev/null +++ b/archive/v0.09/test_newline_effect.py @@ -0,0 +1,23 @@ +import re +from src.format_extractor import FormatExtractor + +# 模拟带换行的 HTML +html_with_newlines = """ +in the name of + +abundance... +""" + +extractor = FormatExtractor() +clean_text, text_with_ph, _, _, _ = extractor.extract(f"{html_with_newlines}
") + +print(f"Original HTML: {repr(html_with_newlines)}") +print(f"Clean Text: {repr(clean_text)}") +print(f"Text with PH: {repr(text_with_ph)}") +print(f"Has Newline: {'\\n' in text_with_ph}") +print("-" * 20) + +# 模拟 Prompt 构建 +prompt_line = f"p_00006 [BODY] {text_with_ph}" +print("Prompt Line Preview:") +print(prompt_line) diff --git a/archive/v0.09/test_spacing.py b/archive/v0.09/test_spacing.py new file mode 100644 index 0000000..001392c --- /dev/null +++ b/archive/v0.09/test_spacing.py @@ -0,0 +1,16 @@ +from src.utils import add_spacing_between_cn_and_en_num + +cases = [ + "在全φ1φ员会议", + "全φ1φ员", + "φ1φTable Talkφ/1φ", + "测试φ12φ测试", + "测试φ/12φ测试" +] + +for text in cases: + processed = add_spacing_between_cn_and_en_num(text) + print(f"Original: '{text}'") + print(f"Processed: '{processed}'") + print(f"Changed: {text != processed}") + print() diff --git a/archive/v0.09/test_specific_failure_cases.py b/archive/v0.09/test_specific_failure_cases.py new file mode 100644 index 0000000..352de04 --- /dev/null +++ b/archive/v0.09/test_specific_failure_cases.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +""" +针对性测试脚本：复现用户报告的占位符丢失问题 +直接读取 Manifest 中的特定失败 Item (p_00006, p_00011 等) +调用真实 LLM 进行翻译，并打印完整的 Prompt 和 Response +""" +import sys +import asyncio +import json +from pathlib import Path +from loguru import logger + +# 添加 src 到路径 +sys.path.insert(0, '.') + +from src.manifest_manager import ManifestManager +from src.llm_client import LLMClient +from src.utils import load_config + +# 配置日志 +logger.remove() +logger.add(sys.stdout, level="DEBUG") + +async def run_test(): + print("=== 1. 加载配置 ===") + try: + config = load_config() + # 自动选择配置好的 provider + if 'v3' in config['providers'] and config['providers']['v3'].get('api_key'): + config['llm'] = config['providers']['v3'] + print("Using Provider: v3") + elif 'openrouter' in config['providers']: + config['llm'] = config['providers']['openrouter'] + print("Using Provider: openrouter") + else: + print("No valid provider found with API key in config!") + return + except Exception as e: + print(f"Config load failed: {e}") + return + + print("\n=== 2. 加载真实 Manifest ===") + manifest_dir = Path("cache/manifests") + manifest_files = list(manifest_dir.glob("*.json")) + if not manifest_files: + print("Error: No manifest file found") + return + + # 优先选择包含 "OpenAI" 的那个文件（用户截图） + target_manifest = next((f for f in manifest_files if "OpenAI" in f.name), manifest_files[0]) + print(f"Loading: {target_manifest}") + + manifest = ManifestManager(str(target_manifest)) + if not manifest.load(): + print("Failed to load manifest") + return + + # 提取目标失败案例 + target_ids = ["p_00006", "p_00009", "p_00011", "p_00013"] + # 也包括上下文以免错位 (p_00003 - p_00006) + context_ids = ["p_00003", "p_00004", "p_00005", "p_00006"] + + items_to_test = [] + + # 测试组 1: 上下文错位测试 + print("\n=== 准备测试组 1: 上下文错位及占位符 (p_00003-00006) ===") + group1 = [] + for uid in context_ids: + item = manifest._items_by_id.get(uid) + if item: + # 强制清空旧翻译，模拟重新翻译 + item.translation = None + item.translation_with_placeholders = None + group1.append(item) + print(f"Added {uid}: {item.text_with_placeholders}") + + # 测试组 2: 独立行占位符丢失测试 (p_00009, p_00011) + print("\n=== 准备测试组 2: 独立行占位符 (p_00009, p_00011) ===") + group2 = [] + for uid in ["p_00009", "p_00011"]: + item = manifest._items_by_id.get(uid) + if item: + item.translation = None + group2.append(item) + print(f"Added {uid}: {item.text_with_placeholders}") + + client = LLMClient(config) + + # 执行测试 1 + if group1: + print("\n\n>>> 执行 Group 1 测试 (Context Alignment) <<<") + # 打印 Prompt 预览 + prompt = client._build_prompt(group1, mode="chinese") + print("\n[PROMPT PREVIEW]:") + print("-" * 20) + print(prompt) + print("-" * 20) + + # 调用 LLM + print("\n[CALLING LLM]...") + results = await client.translate_chunk(group1, mode="chinese") + + print("\n[RESULTS Group 1]:") + for uid, trans in results.items(): + print(f"{uid}: {trans}") + if uid == "p_00006": + print(f" > Contains φ1φ? {'φ1φ' in trans}") + + # 执行测试 2 + if group2: + print("\n\n>>> 执行 Group 2 测试 (Isolated Placeholders) <<<") + prompt = client._build_prompt(group2, mode="chinese") + print("\n[PROMPT PREVIEW]:") + print(prompt) + + print("\n[CALLING LLM]...") + results = await client.translate_chunk(group2, mode="chinese") + + print("\n[RESULTS Group 2]:") + for uid, trans in results.items(): + print(f"{uid}: {trans}") + +if __name__ == "__main__": + asyncio.run(run_test()) diff --git a/tests/__init__.py b/archive/v0.09/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to archive/v0.09/tests/__init__.py diff --git a/archive/v0.09/tests/analyze_placeholder_errors.py b/archive/v0.09/tests/analyze_placeholder_errors.py new file mode 100644 index 0000000..838dcd9 --- /dev/null +++ b/archive/v0.09/tests/analyze_placeholder_errors.py @@ -0,0 +1,257 @@ +#!/usr/bin/env python3 +""" +占位符错误分析测试脚本 + +功能： +1. 使用 v3 provider (config 中配置) 翻译指定章节 +2. 收集所有占位符错误 +3. 输出分析报告 +""" + +import asyncio +import json +import re +from pathlib import Path +from collections import defaultdict + +# 添加项目路径 +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from src.epub_parser import EPUBParser +from src.epub_cleaner import EpubCleaner +from src.text_processor import TextProcessor +from src.llm_client import LLMClient +from src.format_restorer import FormatRestorer +from src.manifest_manager import ManifestManager +from src.utils import add_spacing_between_cn_and_en_num +from loguru import logger + + +class PlaceholderAnalyzer: + """占位符错误分析器""" + + def __init__(self, config_path: str = "config/config.json", provider: str = "openrouter"): + with open(config_path, 'r') as f: + self.config = json.load(f) + + # 扁平化 provider 配置到 config['llm'] + providers = self.config.get('providers', {}) + if provider not in providers: + raise ValueError(f"Provider '{provider}' not found. Available: {list(providers.keys())}") + self.config['llm'] = providers[provider] + print(f"使用 LLM 供应商: {provider} ({self.config['llm'].get('base_url')})") + + self.llm_client = LLMClient(self.config) + self.text_processor = TextProcessor(self.config) + self.restorer = FormatRestorer() + + # 错误收集 + self.errors = [] + self.success_count = 0 + self.total_with_placeholders = 0 + + async def analyze_chapter(self, epub_path: str, chapter_file: str = None): + """ + 分析单个章节的占位符处理情况 + + Args: + epub_path: EPUB 文件路径 + chapter_file: 指定章节文件名 (如 'OEBPS/c3Z.xhtml'), 不指定则使用第一个内容章节 + """ + # 1. 清理 EPUB + cleaner = EpubCleaner() + cleaned_path = "cache/manifests/processed_epubs/test_cleaned.epub" + Path(cleaned_path).parent.mkdir(parents=True, exist_ok=True) + cleaner.clean_epub(epub_path, cleaned_path) + + # 2. 解析 + parser = EPUBParser(cleaned_path) + content_items = parser.extract_all_content_items() + + # 3. 选择章节 + if chapter_file: + target_items = [i for i in content_items if i['file_name'] == chapter_file] + else: + # 默认选择第一个有较多内容的章节 + target_items = [i for i in content_items if len(i['content']) > 5000][:1] + + if not target_items: + print("未找到目标章节") + return + + target = target_items[0] + print(f"\n分析章节: {target['file_name']}") + print("=" * 60) + + # 4. 提取文本 + manifest = ManifestManager("cache/manifests/test_analysis_manifest.json") + manifest.init_manifest(book_id="test", metadata={}) + self.text_processor.extract_to_manifest(target['content'], target['file_name'], manifest, mode="chinese") + manifest.save() + + # 5. 获取待翻译项 + items = manifest.get_items(status="pending") + print(f"待翻译项: {len(items)}") + + # 6. 筛选有占位符的项目 + items_with_ph = [i for i in items if i.placeholder_map and + any(k for k in i.placeholder_map.keys() if not k.startswith("_"))] + self.total_with_placeholders = len(items_with_ph) + print(f"含内嵌占位符的项: {self.total_with_placeholders}") + + # 7. 翻译并分析 + print("\n开始翻译...") + + # 分块翻译 + chunks = self.text_processor.create_chunks_from_manifest(manifest, mode="chinese") + + for i, chunk in enumerate(chunks): + print(f" 处理块 {i+1}/{len(chunks)}...") + await self._process_chunk(chunk) + + # 8. 输出分析报告 + self._print_report() + + async def _process_chunk(self, chunk): + """处理单个翻译块""" + try: + results = await self.llm_client.translate_chunk( + chunk, + glossary={}, + instruction="", + mode="chinese" + ) + + for item in chunk: + if item.global_id not in results: + continue + + raw_trans = results[item.global_id] + if "[Error" in raw_trans: + continue + + processed_trans = add_spacing_between_cn_and_en_num(raw_trans) + + # 检查是否有内嵌占位符 + inner_ph = {k: v for k, v in (item.placeholder_map or {}).items() + if not k.startswith("_")} + + if inner_ph: + # 验证还原 + restored, success = self.restorer.restore(processed_trans, item.placeholder_map) + + if not success: + # 记录错误 + expected = set(inner_ph.keys()) + found = set(re.findall(r'φ(/?\\d+)φ', processed_trans)) + missing = expected - found + extra = found - expected + + self.errors.append({ + 'id': item.global_id, + 'text_with_ph': item.text_with_placeholders, + 'translation_with_ph': processed_trans, + 'placeholder_map': inner_ph, + 'missing': list(missing), + 'extra': list(extra), + 'expected': list(expected), + 'found': list(found) + }) + else: + self.success_count += 1 + + except Exception as e: + logger.error(f"处理块失败: {e}") + + def _print_report(self): + """输出分析报告""" + print("\n" + "=" * 80) + print("占位符错误分析报告") + print("=" * 80) + + print(f"\n总计含占位符项: {self.total_with_placeholders}") + print(f"成功还原: {self.success_count}") + print(f"失败: {len(self.errors)}") + if self.total_with_placeholders > 0: + success_rate = (self.success_count / self.total_with_placeholders) * 100 + print(f"成功率: {success_rate:.1f}%") + + if not self.errors: + print("\n🎉 没有占位符错误!") + return + + print("\n" + "-" * 80) + print("错误详情") + print("-" * 80) + + # 按错误类型分组 + missing_only = [e for e in self.errors if e['missing'] and not e['extra']] + extra_only = [e for e in self.errors if e['extra'] and not e['missing']] + both = [e for e in self.errors if e['missing'] and e['extra']] + + print(f"\n丢失占位符: {len(missing_only)} 个") + print(f"多余占位符: {len(extra_only)} 个") + print(f"两者都有: {len(both)} 个") + + # 详细错误列表 + print("\n" + "-" * 80) + print("详细错误列表 (最多显示 10 个)") + print("-" * 80) + + for i, err in enumerate(self.errors[:10]): + print(f"\n[{i+1}] ID: {err['id']}") + print(f" 原文 (带占位符): {err['text_with_ph'][:100]}...") + print(f" 译文 (带占位符): {err['translation_with_ph'][:100]}...") + print(f" 期望占位符: {err['expected']}") + print(f" 找到占位符: {err['found']}") + print(f" 丢失: {err['missing']}") + print(f" 多余: {err['extra']}") + + # 模式分析 + print("\n" + "-" * 80) + print("错误模式分析") + print("-" * 80) + + # 分析常见的丢失模式 + all_missing = [] + for e in self.errors: + all_missing.extend(e['missing']) + + from collections import Counter + missing_counter = Counter(all_missing) + print("\n最常丢失的占位符:") + for ph, count in missing_counter.most_common(5): + print(f" φ{ph}φ: {count} 次") + + # 保存完整报告到文件 + report_path = "cache/placeholder_error_report.json" + with open(report_path, 'w', encoding='utf-8') as f: + json.dump({ + 'summary': { + 'total_with_placeholders': self.total_with_placeholders, + 'success_count': self.success_count, + 'error_count': len(self.errors), + 'success_rate': (self.success_count / self.total_with_placeholders * 100) if self.total_with_placeholders > 0 else 0 + }, + 'errors': self.errors + }, f, ensure_ascii=False, indent=2) + + print(f"\n完整报告已保存到: {report_path}") + + +async def main(): + import argparse + parser = argparse.ArgumentParser(description="占位符错误分析") + parser.add_argument("epub", help="EPUB 文件路径") + parser.add_argument("--chapter", help="指定章节文件名") + args = parser.parse_args() + + analyzer = PlaceholderAnalyzer() + await analyzer.analyze_chapter(args.epub, args.chapter) + await analyzer.llm_client.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/archive/v0.09/tests/extraction_experiment/__init__.py b/archive/v0.09/tests/extraction_experiment/__init__.py new file mode 100644 index 0000000..ee5da64 --- /dev/null +++ b/archive/v0.09/tests/extraction_experiment/__init__.py @@ -0,0 +1,5 @@ +""" +文本提取实验模块 +""" + +__version__ = "0.1.0" diff --git a/archive/v0.09/tests/extraction_experiment/analyze_missing.py b/archive/v0.09/tests/extraction_experiment/analyze_missing.py new file mode 100644 index 0000000..0aafca2 --- /dev/null +++ b/archive/v0.09/tests/extraction_experiment/analyze_missing.py @@ -0,0 +1,233 @@ +""" +缺失文本分析工具 + +详细分析提取器缺失的文本片段,找出根本原因 +""" + +import sys +from pathlib import Path +from loguru import logger +import re + +project_root = Path(__file__).parent.parent.parent +sys.path.insert(0, str(project_root)) + +from ebooklib import epub + +sys.path.insert(0, str(Path(__file__).parent)) +from extractors.enhanced_bs4 import EnhancedBS4Extractor +from extractors.baseline_pandoc import PandocBaseline + + +def analyze_missing_text(epub_path: Path, max_missing_samples: int = 20): + """ + 分析缺失的文本片段 + + Args: + epub_path: ePub 文件路径 + max_missing_samples: 最多显示的缺失样本数 + """ + print(f"\n{'='*80}") + print(f"分析文件: {epub_path.name}") + print(f"{'='*80}\n") + + # 1. 获取 Pandoc 基准 + pandoc = PandocBaseline() + baseline_text = pandoc.extract_from_epub(str(epub_path)) + + if not baseline_text: + print("❌ Pandoc 提取失败") + return + + print(f"Pandoc 基准长度: {len(baseline_text):,} 字符\n") + + # 2. 提取器提取 + book = epub.read_epub(str(epub_path)) + html_docs = [] + for item in book.get_items(): + if item.get_type() == 9: + try: + content = item.get_content().decode('utf-8') + html_docs.append(content) + except: + continue + + combined_html = "\n\n".join(html_docs) + + extractor = EnhancedBS4Extractor(min_text_length=1, preserve_decorative=True) # 不过滤短文本 + items = extractor.extract(combined_html) + + # 分离内容和装饰性元素 + content_items = [i for i in items if not i.get('is_decorative') and not i.get('is_navigation')] + decorative_items = [i for i in items if i.get('is_decorative')] + nav_items = [i for i in items if i.get('is_navigation')] + + extracted_text = " ".join([item['text'] for item in content_items]) + + print(f"提取器统计:") + print(f" - 内容元素: {len(content_items)}") + print(f" - 装饰性元素: {len(decorative_items)}") + print(f" - 导航元素: {len(nav_items)}") + print(f" - 提取文本长度: {len(extracted_text):,} 字符\n") + + # 3. 标准化文本 + def normalize(text): + text = text.lower() + text = re.sub(r'[^\w\s]', ' ', text) + text = re.sub(r'\s+', ' ', text) + return text.strip() + + baseline_norm = normalize(baseline_text) + extracted_norm = normalize(extracted_text) + + # 4. 分词对比 + baseline_words = baseline_norm.split() + extracted_words = set(extracted_norm.split()) + + print(f"词级别对比:") + print(f" - Pandoc 词数: {len(baseline_words):,}") + print(f" - 提取器词数: {len(extracted_words):,}") + + # 5. 找出缺失的句子 + print(f"\n{'='*80}") + print("分析缺失的文本片段") + print(f"{'='*80}\n") + + # 将 Pandoc 文本分成句子 + baseline_sentences = re.split(r'[.!?\n]+', baseline_text) + baseline_sentences = [s.strip() for s in baseline_sentences if len(s.strip()) > 10] + + missing_sentences = [] + for sentence in baseline_sentences: + sentence_norm = normalize(sentence) + if sentence_norm and sentence_norm not in extracted_norm: + # 检查是否有部分匹配 + words = sentence_norm.split() + if len(words) > 3: + matched_words = sum(1 for w in words if w in extracted_words) + match_ratio = matched_words / len(words) + + if match_ratio < 0.5: # 少于50%的词匹配,认为缺失 + missing_sentences.append({ + 'text': sentence[:200], # 只取前200字符 + 'length': len(sentence), + 'match_ratio': match_ratio + }) + + print(f"发现 {len(missing_sentences)} 个可能缺失的文本片段\n") + + # 6. 分类缺失原因 + print(f"{'='*80}") + print("缺失片段分类分析") + print(f"{'='*80}\n") + + # 显示样本 + for i, missing in enumerate(missing_sentences[:max_missing_samples], 1): + print(f"--- 缺失片段 {i} ---") + print(f"长度: {missing['length']} 字符") + print(f"匹配率: {missing['match_ratio']:.1%}") + print(f"内容: {missing['text']}") + + # 尝试分析原因 + text = missing['text'].lower() + reasons = [] + + if any(kw in text for kw in ['copyright', '©', 'isbn', 'publisher', 'published']): + reasons.append("📚 可能是版权/出版信息") + + if any(kw in text for kw in ['table of contents', 'chapter', 'part', 'section']): + reasons.append("📑 可能是目录信息") + + if any(kw in text for kw in ['page', 'pg', 'p.']): + reasons.append("📄 可能是页码") + + if len(missing['text']) < 30: + reasons.append("📏 文本过短") + + if re.match(r'^[0-9\s\-\.]+$', missing['text'].strip()): + reasons.append("🔢 纯数字") + + if not reasons: + reasons.append("❓ 未知原因 - 需要进一步分析") + + print(f"可能原因: {', '.join(reasons)}") + print() + + if len(missing_sentences) > max_missing_samples: + print(f"... 还有 {len(missing_sentences) - max_missing_samples} 个缺失片段\n") + + # 7. 统计缺失原因 + print(f"{'='*80}") + print("缺失原因统计") + print(f"{'='*80}\n") + + reason_counts = { + '版权/出版信息': 0, + '目录信息': 0, + '页码': 0, + '文本过短': 0, + '纯数字': 0, + '未知原因': 0 + } + + for missing in missing_sentences: + text = missing['text'].lower() + + if any(kw in text for kw in ['copyright', '©', 'isbn', 'publisher', 'published']): + reason_counts['版权/出版信息'] += 1 + elif any(kw in text for kw in ['table of contents', 'chapter', 'part', 'section']): + reason_counts['目录信息'] += 1 + elif any(kw in text for kw in ['page', 'pg', 'p.']): + reason_counts['页码'] += 1 + elif len(missing['text']) < 30: + reason_counts['文本过短'] += 1 + elif re.match(r'^[0-9\s\-\.]+$', missing['text'].strip()): + reason_counts['纯数字'] += 1 + else: + reason_counts['未知原因'] += 1 + + for reason, count in reason_counts.items(): + if count > 0: + percentage = count / len(missing_sentences) * 100 + print(f"{reason}: {count} 个 ({percentage:.1f}%)") + + # 8. 建议 + print(f"\n{'='*80}") + print("改进建议") + print(f"{'='*80}\n") + + if reason_counts['文本过短'] > 0: + print(f"⚠️ 发现 {reason_counts['文本过短']} 个过短文本被过滤") + print(" 建议: 移除 min_text_length 限制,提取所有文本\n") + + if reason_counts['版权/出版信息'] > 0: + print(f"📚 发现 {reason_counts['版权/出版信息']} 个版权/出版信息") + print(" 建议: 这些通常不需要翻译,可以保持过滤\n") + + if reason_counts['目录信息'] > 0: + print(f"📑 发现 {reason_counts['目录信息']} 个目录信息") + print(" 建议: 目录通常需要翻译,检查是否被错误过滤\n") + + if reason_counts['未知原因'] > 0: + print(f"❓ 发现 {reason_counts['未知原因']} 个未知原因的缺失") + print(" 建议: 需要详细分析这些片段\n") + + +def main(): + """主函数""" + logger.remove() + logger.add(sys.stderr, level="WARNING") # 只显示警告和错误 + + # 测试一本书 + test_file = "Gambling Man.epub" + epub_path = project_root / "input" / test_file + + if not epub_path.exists(): + print(f"文件不存在: {test_file}") + return + + analyze_missing_text(epub_path, max_missing_samples=30) + + +if __name__ == "__main__": + main() diff --git a/archive/v0.09/tests/extraction_experiment/analyze_precise.py b/archive/v0.09/tests/extraction_experiment/analyze_precise.py new file mode 100644 index 0000000..a8a067e --- /dev/null +++ b/archive/v0.09/tests/extraction_experiment/analyze_precise.py @@ -0,0 +1,198 @@ +""" +精准缺失文本分析 - 直接对比原始 HTML + +不使用 Pandoc,直接分析原始 HTML 中的文本 +""" + +import sys +from pathlib import Path +from loguru import logger +from bs4 import BeautifulSoup +import re + +project_root = Path(__file__).parent.parent.parent +sys.path.insert(0, str(project_root)) + +from ebooklib import epub + +sys.path.insert(0, str(Path(__file__).parent)) +from extractors.enhanced_bs4 import EnhancedBS4Extractor + + +def extract_all_text_from_html(html_content: str) -> str: + """ + 从 HTML 中提取所有可见文本(包括所有元素) + + 这是"真正的100%"基准 + """ + soup = BeautifulSoup(html_content, 'html.parser') + + # 移除不可见元素 + for element in soup(['script', 'style', 'meta', 'link']): + element.decompose() + + # 获取所有文本 + text = soup.get_text(separator=' ', strip=True) + + # 清理空白 + text = re.sub(r'\s+', ' ', text) + + return text.strip() + + +def compare_extraction(epub_path: Path): + """对比提取器与真实 HTML 文本""" + print(f"\n{'='*80}") + print(f"精准文本覆盖率分析: {epub_path.name}") + print(f"{'='*80}\n") + + # 加载 ePub + book = epub.read_epub(str(epub_path)) + + # 提取所有 HTML 文档 + html_docs = [] + for item in book.get_items(): + if item.get_type() == 9: + try: + content = item.get_content().decode('utf-8') + html_docs.append({ + 'name': item.get_name(), + 'content': content + }) + except: + continue + + print(f"找到 {len(html_docs)} 个 HTML 文档\n") + + # 逐个文档分析 + total_baseline_length = 0 + total_extracted_length = 0 + total_missing_length = 0 + + missing_samples = [] + + for doc in html_docs: + # 基准: 所有文本 + baseline_text = extract_all_text_from_html(doc['content']) + + # 提取器提取 + extractor = EnhancedBS4Extractor(min_text_length=1, preserve_decorative=True) + items = extractor.extract(doc['content']) + + # 只统计内容元素(不包括装饰性和导航) + content_items = [i for i in items if not i.get('is_decorative') and not i.get('is_navigation')] + extracted_text = " ".join([item['text'] for item in content_items]) + + # 统计 + baseline_len = len(baseline_text) + extracted_len = len(extracted_text) + + total_baseline_length += baseline_len + total_extracted_length += extracted_len + + # 找出缺失的文本 + if baseline_len > 0: + coverage = extracted_len / baseline_len + + if coverage < 0.99: # 覆盖率 < 99% + missing_len = baseline_len - extracted_len + total_missing_length += missing_len + + # 找出具体缺失的片段 + baseline_words = set(baseline_text.lower().split()) + extracted_words = set(extracted_text.lower().split()) + missing_words = baseline_words - extracted_words + + if missing_words: + missing_samples.append({ + 'file': doc['name'], + 'baseline_length': baseline_len, + 'extracted_length': extracted_len, + 'coverage': coverage, + 'missing_words_count': len(missing_words), + 'missing_words_sample': list(missing_words)[:20] + }) + + # 总体统计 + overall_coverage = total_extracted_length / total_baseline_length if total_baseline_length > 0 else 0 + + print(f"{'='*80}") + print("总体统计") + print(f"{'='*80}\n") + print(f"基准文本总长度: {total_baseline_length:,} 字符") + print(f"提取文本总长度: {total_extracted_length:,} 字符") + print(f"缺失文本长度: {total_missing_length:,} 字符") + print(f"**覆盖率: {overall_coverage:.2%}**\n") + + # 显示缺失样本 + if missing_samples: + print(f"{'='*80}") + print(f"发现 {len(missing_samples)} 个文档存在缺失") + print(f"{'='*80}\n") + + for i, sample in enumerate(missing_samples[:10], 1): + print(f"--- 文档 {i}: {sample['file']} ---") + print(f"基准长度: {sample['baseline_length']:,} 字符") + print(f"提取长度: {sample['extracted_length']:,} 字符") + print(f"覆盖率: {sample['coverage']:.2%}") + print(f"缺失词数: {sample['missing_words_count']}") + print(f"缺失词样本: {', '.join(sample['missing_words_sample'][:10])}") + print() + + if len(missing_samples) > 10: + print(f"... 还有 {len(missing_samples) - 10} 个文档\n") + else: + print("✅ 所有文档覆盖率 ≥ 99%\n") + + # 详细分析第一个缺失文档 + if missing_samples: + print(f"{'='*80}") + print("详细分析第一个缺失文档") + print(f"{'='*80}\n") + + first_missing = missing_samples[0] + doc_content = next(d['content'] for d in html_docs if d['name'] == first_missing['file']) + + # 重新提取 + baseline_text = extract_all_text_from_html(doc_content) + + extractor = EnhancedBS4Extractor(min_text_length=1, preserve_decorative=True) + items = extractor.extract(doc_content) + + print(f"文件: {first_missing['file']}\n") + print(f"提取了 {len(items)} 个元素:") + for item in items[:20]: + item_type = "" + if item.get('is_decorative'): + item_type = " [装饰性]" + elif item.get('is_navigation'): + item_type = " [导航]" + + print(f" - [{item['tag']}] {item['text'][:60]}{item_type}") + + if len(items) > 20: + print(f" ... 还有 {len(items) - 20} 个元素\n") + + # 显示原始 HTML 的所有文本 + print(f"\n原始 HTML 的所有文本 (前 500 字符):") + print(baseline_text[:500]) + print("...\n") + + +def main(): + """主函数""" + logger.remove() + logger.add(sys.stderr, level="ERROR") + + test_file = "Gambling Man.epub" + epub_path = project_root / "input" / test_file + + if not epub_path.exists(): + print(f"文件不存在: {test_file}") + return + + compare_extraction(epub_path) + + +if __name__ == "__main__": + main() diff --git a/archive/v0.09/tests/extraction_experiment/calibre_cleaner.py b/archive/v0.09/tests/extraction_experiment/calibre_cleaner.py new file mode 100644 index 0000000..0261c1b --- /dev/null +++ b/archive/v0.09/tests/extraction_experiment/calibre_cleaner.py @@ -0,0 +1,268 @@ +""" +Calibre ePub 清理器 + +清理 Calibre 生成的冗余 HTML 结构: +1. 将嵌套的
+2. 简化只有单一格式的 (bold, italic)
+3. 移除冗余的 calibre* 类
+4. 合并相同的样式
+"""
+
+from bs4 import BeautifulSoup, Tag, NavigableString
+from typing import Dict, Set
+import re
+from loguru import logger
+
+
+class CalibreHTMLCleaner:
+ """Calibre HTML 清理器"""
+
+ # 简单格式映射
+ SIMPLE_FORMAT_MAP = {
+ 'bold': 'strong',
+ 'italic': 'em',
+ 'underline': 'u',
+ }
+
+ def __init__(self):
+ self.stats = {
+ 'divs_to_p': 0,
+ 'spans_simplified': 0,
+ 'classes_removed': 0,
+ }
+
+ def clean(self, html_content: str) -> str:
+ """
+ 清理 HTML
+
+ Args:
+ html_content: 原始 HTML
+
+ Returns:
+ 清理后的 HTML
+ """
+ soup = BeautifulSoup(html_content, 'html.parser')
+
+ # 1. 将嵌套的 div 转为 p
+ self._convert_nested_divs_to_p(soup)
+
+ # 2. 简化格式 span
+ self._simplify_format_spans(soup)
+
+ # 3. 移除冗余的 span
+ self._remove_redundant_spans(soup)
+
+ # 4. 清理冗余的 calibre 类
+ self._clean_calibre_classes(soup)
+
+ logger.info(
+ f"清理完成: div→p {self.stats['divs_to_p']}, "
+ f"span简化 {self.stats['spans_simplified']}, "
+ f"类移除 {self.stats['classes_removed']}"
+ )
+
+ return str(soup)
+
+ def _convert_nested_divs_to_p(self, soup: BeautifulSoup):
+ """
+ 将嵌套的 div 转为 p
+
+ 策略:
+ - 如果 div 只包含内联元素(span, em, strong等),转为 p
+ - 保留包含块级元素的 div
+ """
+ inline_tags = {'span', 'em', 'strong', 'i', 'b', 'u', 'a', 'br'}
+
+ for div in soup.find_all('div'):
+ # 检查是否只包含内联元素
+ has_block_children = False
+ for child in div.children:
+ if isinstance(child, Tag):
+ if child.name not in inline_tags:
+ has_block_children = True
+ break
+
+ # 如果只包含内联元素,转为 p
+ if not has_block_children:
+ div.name = 'p'
+ self.stats['divs_to_p'] += 1
+
+ def _simplify_format_spans(self, soup: BeautifulSoup):
+ """
+ 简化只有单一格式的 span
+
+ 例如:
+ Text
+ → Text
+ """
+ for span in soup.find_all('span'):
+ # 检查 class 属性
+ classes = span.get('class', [])
+ if not classes:
+ continue
+
+ # 检查是否是简单格式
+ simple_format = None
+ for cls in classes:
+ for format_name, tag_name in self.SIMPLE_FORMAT_MAP.items():
+ if format_name in cls.lower():
+ simple_format = tag_name
+ break
+ if simple_format:
+ break
+
+ if simple_format:
+ # 替换为语义化标签
+ new_tag = soup.new_tag(simple_format)
+
+ # 复制内容
+ for child in list(span.children):
+ new_tag.append(child)
+
+ # 替换
+ span.replace_with(new_tag)
+ self.stats['spans_simplified'] += 1
+
+ def _remove_redundant_spans(self, soup: BeautifulSoup):
+ """
+ 移除冗余的 span
+
+ 策略:
+ - 只移除完全没有属性的 span
+ - 保留有 class 的 span(即使是 calibre*)
+ - 确保不丢失任何文本
+ """
+ removed_count = 0
+
+ # 只遍历一次,更保守
+ for span in soup.find_all('span'):
+ # 只移除完全没有属性的 span
+ if not span.attrs:
+ # 检查是否有文本内容
+ if span.get_text(strip=True):
+ # 有文本,安全地展开
+ span.unwrap()
+ removed_count += 1
+
+ self.stats['spans_removed'] = removed_count
+
+ def _remove_empty_elements(self, soup: BeautifulSoup):
+ """
+ 移除空元素
+
+ 更谨慎的策略:
+ - 只删除完全没有内容的元素
+ - 保留有文本或图片的元素
+ """
+ removed_count = 0
+
+ # 只遍历一次
+ for element in soup.find_all():
+ if isinstance(element, Tag):
+ # 检查是否完全为空
+ text = element.get_text(strip=True)
+ has_img = element.find('img') is not None
+
+ # 只删除既没有文本也没有图片的元素
+ if not text and not has_img:
+ element.decompose()
+ removed_count += 1
+
+ self.stats['empty_removed'] = removed_count
+
+ def _clean_calibre_classes(self, soup: BeautifulSoup):
+ """
+ 清理冗余的 calibre 类
+
+ 策略:
+ - 保留有实际样式的类
+ - 移除纯数字的 calibre 类(如 calibre1, calibre2)
+ """
+ for element in soup.find_all(class_=True):
+ classes = element.get('class', [])
+ if not classes:
+ continue
+
+ # 过滤掉纯数字的 calibre 类
+ new_classes = []
+ for cls in classes:
+ # 保留非 calibre 类
+ if not cls.startswith('calibre'):
+ new_classes.append(cls)
+ # 保留有语义的 calibre 类
+ elif any(keyword in cls.lower() for keyword in ['title', 'chapter', 'quote', 'note']):
+ new_classes.append(cls)
+ else:
+ self.stats['classes_removed'] += 1
+
+ if new_classes:
+ element['class'] = new_classes
+ else:
+ # 移除整个 class 属性
+ del element['class']
+
+
+class CalibreEPUBCleaner:
+ """Calibre ePub 清理器"""
+
+ def __init__(self):
+ self.html_cleaner = CalibreHTMLCleaner()
+
+ def clean_epub(self, epub_path: str, output_path: str):
+ """
+ 清理整个 ePub
+
+ Args:
+ epub_path: 输入 ePub 路径
+ output_path: 输出 ePub 路径
+ """
+ from ebooklib import epub
+
+ logger.info(f"开始清理 ePub: {epub_path}")
+
+ # 加载 ePub
+ book = epub.read_epub(epub_path)
+
+ # 清理每个 HTML 文档
+ cleaned_count = 0
+ for item in book.get_items():
+ if item.get_type() != 9: # 不是 HTML
+ continue
+
+ try:
+ content = item.get_content().decode('utf-8')
+ except:
+ continue
+
+ # 清理 HTML
+ cleaned_html = self.html_cleaner.clean(content)
+
+ # 更新内容
+ item.set_content(cleaned_html.encode('utf-8'))
+ cleaned_count += 1
+
+ # 保存
+ epub.write_epub(output_path, book, {
+ 'epub2_guide': False,
+ 'epub3_landmark': False,
+ 'epub3_pages': False,
+ 'spine_direction': True,
+ })
+
+ logger.info(f"清理完成: 处理了 {cleaned_count} 个文档")
+ logger.info(f"输出: {output_path}")
+
+
+if __name__ == "__main__":
+ import sys
+ from pathlib import Path
+
+ if len(sys.argv) < 3:
+ print("用法: python calibre_cleaner.py Test 元素,每个独立处理
+"""
+
+from bs4 import BeautifulSoup, Tag, NavigableString
+from typing import List, Dict, Any, Tuple
+import re
+from loguru import logger
+
+
+class FineGrainedExtractor:
+ """细粒度提取器 - 每个 p 元素独立提取"""
+
+ SKIP_TRANSLATION_PATTERNS = [
+ r'index\.x?html',
+ r'bibliography\.x?html',
+ r'endnotes?\.x?html',
+ r'footnotes?\.x?html',
+ ]
+
+ TOC_PATTERNS = [
+ r'nav\.x?html',
+ r'toc\.x?html',
+ ]
+
+ def __init__(self, translate_toc: bool = False):
+ self.translate_toc = translate_toc
+ self.soup = None
+
+ def extract(self, html_content: str, file_name: str = "") -> List[Dict[str, Any]]:
+ """
+ 细粒度提取: 每个 和标题元素独立提取
+
+ 关键: 不管嵌套,所有 , h1-h6 都提取
+ """
+ self.soup = BeautifulSoup(html_content, 'html.parser')
+
+ # 移除不需要的元素
+ for element in self.soup(['script', 'style', 'meta', 'link']):
+ element.decompose()
+
+ doc_type = self._classify_document(file_name)
+
+ items = []
+
+ # 提取所有文本块元素
+ target_tags = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6']
+ for p in self.soup.find_all(target_tags):
+ # 提取文本
+ text = p.get_text(separator=' ', strip=True)
+
+ if not text.strip():
+ continue
+
+ # 收集所有文本节点
+ text_nodes = self._collect_text_nodes(p)
+
+ if not text_nodes:
+ continue
+
+ is_decorative = self._is_decorative(text)
+ should_translate = self._should_translate(doc_type, is_decorative, text)
+
+ items.append({
+ 'element': p,
+ 'text': text,
+ 'text_nodes': text_nodes,
+ 'should_translate': should_translate,
+ 'doc_type': doc_type,
+ 'is_decorative': is_decorative,
+ 'tag': p.name
+ })
+
+ logger.info(
+ f"[{doc_type}] 提取 {len(items)} 个元素 (p, h1-h6): "
+ f"翻译 {sum(1 for i in items if i['should_translate'])}"
+ )
+
+ return items
+
+ def _collect_text_nodes(self, element: Tag) -> List[Tuple[NavigableString, str]]:
+ """
+ 收集元素中的所有文本节点
+ """
+ text_nodes = []
+
+ for descendant in element.descendants:
+ if isinstance(descendant, NavigableString):
+ # 跳过注释
+ if isinstance(descendant, type(element)):
+ continue
+
+ text = str(descendant).strip()
+ if text:
+ text_nodes.append((descendant, text))
+
+ return text_nodes
+
+ def backfill(self, items: List[Dict[str, Any]], translation_map: Dict[str, str], bilingual: bool = True) -> str:
+ """
+ 回填翻译
+
+ Args:
+ items: 提取的元素列表
+ translation_map: 翻译映射 {原文: 译文}
+ bilingual: 是否生成双语版本 (True: 保留原文+译文, False: 只保留译文)
+ """
+ success_count = 0
+
+ for item in items:
+ original_text = item['text']
+ element = item['element']
+
+ # 查找翻译
+ translation = translation_map.get(original_text)
+ if translation is None:
+ continue
+
+ if bilingual:
+ # 双语模式: 在元素末尾添加译文
+ # 创建一个新的标签用于译文 (使用相同的标签名, 如 p, h1, h2...)
+ translation_p = self.soup.new_tag(element.name)
+
+ # 1. 继承 class
+ original_classes = element.get('class', [])
+ if original_classes:
+ # 复制列表以防引用修改
+ translation_p['class'] = list(original_classes) + ['translation']
+ else:
+ translation_p['class'] = ['translation']
+
+ # 2. 继承 style (如果有)
+ original_style = element.get('style')
+ if original_style:
+ translation_p['style'] = original_style
+
+ # 3. 设置内容
+ translation_p.string = translation
+
+ # 在原始元素后插入译文
+ element.insert_after(translation_p)
+ else:
+ # 纯译文模式: 替换所有文本节点
+ text_nodes = item['text_nodes']
+ if text_nodes:
+ text_nodes[0][0].replace_with(translation)
+
+ for node, _ in text_nodes[1:]:
+ try:
+ node.replace_with('')
+ except:
+ pass # 节点可能已被移除
+
+ success_count += 1
+
+ logger.info(f"回填完成: 成功 {success_count}/{len(translation_map)}")
+ return str(self.soup)
+
+ def _classify_document(self, file_name: str) -> str:
+ if not file_name:
+ return 'core'
+
+ file_name_lower = file_name.lower()
+
+ for pattern in self.SKIP_TRANSLATION_PATTERNS:
+ if re.search(pattern, file_name_lower):
+ return 'skip'
+
+ for pattern in self.TOC_PATTERNS:
+ if re.search(pattern, file_name_lower):
+ return 'toc'
+
+ return 'core'
+
+ def _should_translate(self, doc_type: str, is_decorative: bool, text: str = "") -> bool:
+ if is_decorative:
+ return False
+
+ # 检查是否为罗马数字 (通常是章节号: I, II, III...)
+ if self._is_roman_numeral(text):
+ return False
+
+ if doc_type == 'core':
+ return True
+
+ if doc_type == 'toc':
+ return self.translate_toc
+
+ return False
+
+ def _is_roman_numeral(self, text: str) -> bool:
+ """检查是否为罗马数字 (I, II, III, IV, V...)"""
+ text = text.strip().upper()
+ if not text:
+ return False
+ # 简单正则，覆盖常见直到 3999
+ # ^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$
+ # 注意: 避免匹配空字符串 (已经由 if not text 处理)
+ pattern = re.compile(r"^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$")
+ return bool(pattern.match(text))
+
+ def _is_decorative(self, text: str) -> bool:
+ text_stripped = text.strip()
+ if not text_stripped:
+ return False
+
+ # 增强: 如果只包含非字母数字字符 (标点, 符号, 分隔线等), 视为装饰性
+ # 这将覆盖 ***, ---, ..., _____, —— 等
+ if not any(c.isalnum() for c in text_stripped):
+ return True
+
+ if len(text_stripped) > 20:
+ return False
+
+ decorative_patterns = [
+ r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾]+$',
+ r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾\s]+$',
+ ]
+
+ for pattern in decorative_patterns:
+ if re.match(pattern, text_stripped):
+ return True
+
+ unique_chars = set(text_stripped.replace(' ', ''))
+ if len(unique_chars) <= 3:
+ decorative_chars = set('*-_~•◆◇■□▪▫●○◉◎⦿⦾')
+ if unique_chars & decorative_chars:
+ return True
+
+ return False
diff --git a/archive/v0.09/tests/extraction_experiment/extractors/lxml_xpath.py b/archive/v0.09/tests/extraction_experiment/extractors/lxml_xpath.py
new file mode 100644
index 0000000..3dd7488
--- /dev/null
+++ b/archive/v0.09/tests/extraction_experiment/extractors/lxml_xpath.py
@@ -0,0 +1,188 @@
+"""
+lxml XPath 提取器
+
+使用 lxml 的 XPath 功能实现精准的文本提取和定位
+"""
+
+from lxml import etree, html
+from typing import List, Dict, Any
+from loguru import logger
+import re
+
+
+class LxmlXPathExtractor:
+ """基于 lxml 和 XPath 的提取器"""
+
+ # 块级元素的 XPath 表达式
+ BLOCK_XPATH = ' | '.join([
+ f'//{tag}' for tag in [
+ 'p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+ 'blockquote', 'li', 'td', 'dd', 'dt', 'figcaption',
+ 'section', 'article', 'aside', 'header', 'footer', 'main'
+ ]
+ ])
+
+ NAV_KEYWORDS = [
+ 'nav', 'navigation', 'toc', 'menu', 'header', 'footer',
+ 'page-number', 'page-num', 'sidebar'
+ ]
+
+ def __init__(self, min_text_length: int = 10):
+ """
+ 初始化提取器
+
+ Args:
+ min_text_length: 最小文本长度
+ """
+ self.min_text_length = min_text_length
+
+ def extract(self, html_content: str) -> List[Dict[str, Any]]:
+ """
+ 从 HTML 中提取所有文本元素
+
+ Args:
+ html_content: HTML 字符串
+
+ Returns:
+ 提取的元素列表
+ """
+ try:
+ # 移除 XML 声明(如果存在)
+ import re
+ html_content = re.sub(r'<\?xml[^?]*\?>', '', html_content)
+
+ # 解析 HTML
+ tree = html.fromstring(html_content)
+ except Exception as e:
+ logger.error(f"lxml 解析失败: {e}")
+ return []
+
+ items = []
+ processed_xpaths = set()
+
+ # 使用 XPath 查找所有块级元素
+ try:
+ elements = tree.xpath(self.BLOCK_XPATH)
+ except Exception as e:
+ logger.error(f"XPath 查询失败: {e}")
+ return []
+
+ for element in elements:
+ # 获取 XPath (需要通过 ElementTree 包装)
+ try:
+ xpath = tree.getroottree().getpath(element)
+ except:
+ # 备用方案:生成简单路径
+ xpath = f"//{element.tag}[{elements.index(element)}]"
+
+ # 避免重复
+ if xpath in processed_xpaths:
+ continue
+
+ # 提取文本
+ text = self._clean_text(element)
+
+ # 过滤过短文本
+ if len(text.strip()) < self.min_text_length:
+ continue
+
+ # 判断是否是导航元素
+ is_nav = self._is_navigation_element(element)
+
+ # 获取 HTML
+ try:
+ element_html = etree.tostring(element, encoding='unicode')
+ except:
+ element_html = ""
+
+ items.append({
+ 'xpath': xpath,
+ 'text': text,
+ 'html': element_html,
+ 'tag': element.tag,
+ 'is_navigation': is_nav
+ })
+
+ processed_xpaths.add(xpath)
+
+ logger.info(f"lxml 提取了 {len(items)} 个文本元素")
+ return items
+
+ def _clean_text(self, element) -> str:
+ """清理元素文本"""
+ # lxml 的 text_content() 方法
+ text = element.text_content().strip()
+
+ # 清理多余空白
+ text = re.sub(r'\s+', ' ', text)
+
+ return text
+
+ def _is_navigation_element(self, element) -> bool:
+ """判断是否是导航元素"""
+ # 检查 class 属性
+ classes = element.get('class', '')
+ class_str = classes.lower() if isinstance(classes, str) else ''
+
+ if any(keyword in class_str for keyword in self.NAV_KEYWORDS):
+ return True
+
+ # 检查父元素
+ parent = element.getparent()
+ if parent is not None:
+ p_classes = parent.get('class', '')
+ p_class_str = p_classes.lower() if isinstance(p_classes, str) else ''
+ if any(keyword in p_class_str for keyword in self.NAV_KEYWORDS):
+ return True
+
+ return False
+
+ def backfill(self, html_content: str, translation_map: Dict[str, str]) -> str:
+ """
+ 使用 XPath 精准回填翻译
+
+ Args:
+ html_content: 原始 HTML
+ translation_map: {xpath: translation} 映射
+
+ Returns:
+ 回填后的 HTML 字符串
+ """
+ try:
+ # 移除 XML 声明
+ import re
+ html_content = re.sub(r'<\?xml[^?]*\?>', '', html_content)
+
+ tree = html.fromstring(html_content)
+ except Exception as e:
+ logger.error(f"lxml 解析失败: {e}")
+ return html_content
+
+ success_count = 0
+ fail_count = 0
+
+ for xpath, translation in translation_map.items():
+ try:
+ elements = tree.xpath(xpath)
+ if not elements:
+ logger.warning(f"回填失败: 未找到 XPath {xpath}")
+ fail_count += 1
+ continue
+
+ element = elements[0]
+
+ # 清空元素内容并设置新文本
+ element.clear()
+ element.text = translation
+
+ success_count += 1
+ except Exception as e:
+ logger.error(f"回填错误 {xpath}: {e}")
+ fail_count += 1
+
+ logger.info(f"lxml 回填完成: 成功 {success_count}, 失败 {fail_count}")
+
+ try:
+ return etree.tostring(tree, encoding='unicode', method='html')
+ except:
+ return html_content
diff --git a/archive/v0.09/tests/extraction_experiment/extractors/one_to_one.py b/archive/v0.09/tests/extraction_experiment/extractors/one_to_one.py
new file mode 100644
index 0000000..985ed9f
--- /dev/null
+++ b/archive/v0.09/tests/extraction_experiment/extractors/one_to_one.py
@@ -0,0 +1,267 @@
+"""
+真正的一比一对应提取器
+
+核心原则:
+1. 提取时: 记录每个元素的所有文本节点位置
+2. 回填时: 精确替换这些文本节点,不改变任何结构
+"""
+
+from bs4 import BeautifulSoup, Tag, NavigableString
+from typing import List, Dict, Any, Tuple
+import re
+from loguru import logger
+
+
+class OneToOneExtractor:
+ """一比一对应提取器"""
+
+ SKIP_TRANSLATION_PATTERNS = [
+ r'index\.x?html',
+ r'bibliography\.x?html',
+ r'endnotes?\.x?html',
+ r'footnotes?\.x?html',
+ ]
+
+ TOC_PATTERNS = [
+ r'nav\.x?html',
+ r'toc\.x?html',
+ ]
+
+ OTHER_NON_CORE_PATTERNS = [
+ r'copyright\.x?html',
+ r'title\.x?html',
+ r'cover\.x?html',
+ ]
+
+ BLOCK_TAGS = [
+ 'p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+ 'blockquote', 'li', 'td', 'dd', 'dt', 'figcaption',
+ 'section', 'article', 'aside', 'header', 'footer', 'main'
+ ]
+
+ def __init__(self, translate_toc: bool = False):
+ self.translate_toc = translate_toc
+ self.soup = None
+
+ def extract(self, html_content: str, file_name: str = "") -> List[Dict[str, Any]]:
+ """
+ 提取文本,记录精确的文本节点位置
+
+ 关键改进: 不跳过嵌套元素,每个块级元素都独立提取
+
+ 返回:
+ {
+ 'element': 元素引用,
+ 'text': 完整文本,
+ 'text_nodes': [(node, text), ...], # 只包含直接子节点的文本
+ 'should_translate': bool
+ }
+ """
+ self.soup = BeautifulSoup(html_content, 'html.parser')
+
+ # 移除不需要的元素
+ for element in self.soup(['script', 'style', 'meta', 'link']):
+ element.decompose()
+
+ doc_type = self._classify_document(file_name)
+
+ items = []
+
+ # 关键: 不使用 processed_ids,每个元素都独立提取
+ all_elements = []
+ for element in self.soup.find_all(self.BLOCK_TAGS):
+ # 提取完整文本
+ full_text = element.get_text(separator=' ', strip=True)
+
+ if not full_text.strip():
+ continue
+
+ # 关键: 只收集当前元素的直接文本节点
+ # 不包括子元素中的文本节点
+ text_nodes = self._collect_direct_text_nodes(element)
+
+ # 如果没有直接文本节点,说明所有文本都在子元素中
+ # 这种情况下跳过,让子元素自己处理
+ if not text_nodes:
+ continue
+
+ all_elements.append({
+ 'element': element,
+ 'text': full_text,
+ 'text_nodes': text_nodes,
+ 'doc_type': doc_type,
+ })
+
+ # 过滤: 只保留叶子节点 (没有被其他提取元素包含的元素)
+ for elem_data in all_elements:
+ element = elem_data['element']
+
+ # 检查是否被其他提取元素包含
+ is_contained = False
+ for other_data in all_elements:
+ if other_data is elem_data:
+ continue
+
+ other_element = other_data['element']
+ # 检查 element 是否是 other_element 的子孙
+ if element in other_element.descendants:
+ is_contained = True
+ break
+
+ if is_contained:
+ continue
+
+ # 这是叶子节点,添加到结果
+ is_decorative = self._is_decorative(elem_data['text'])
+ should_translate = self._should_translate(doc_type, is_decorative)
+
+ items.append({
+ 'element': element,
+ 'text': elem_data['text'],
+ 'text_nodes': elem_data['text_nodes'],
+ 'should_translate': should_translate,
+ 'doc_type': doc_type,
+ 'is_decorative': is_decorative,
+ 'tag': element.name
+ })
+
+ logger.info(
+ f"[{doc_type}] 提取 {len(items)} 个元素: "
+ f"翻译 {sum(1 for i in items if i['should_translate'])}"
+ )
+
+ return items
+
+ def _collect_direct_text_nodes(self, element: Tag) -> List[Tuple[NavigableString, str]]:
+ """
+ 收集元素的文本节点
+
+ 策略:
+ 1. 优先收集直接文本节点
+ 2. 如果没有直接文本节点,收集所有子孙文本节点
+
+ 例如:
+ This is a centered paragraph. ' in cleaned:
+ print(f"✅ div 转为 p: {cleaner.stats['divs_to_p']} 个")
+ else:
+ print("❌ div 未转为 p")
+
+ if '' in cleaned:
+ print(f"✅ span 简化为 em: {cleaner.stats['spans_simplified']} 个")
+ else:
+ print("❌ span 未简化")
+
+ if 'calibre' not in cleaned or cleaner.stats['classes_removed'] > 0:
+ print(f"✅ 移除 calibre 类: {cleaner.stats['classes_removed']} 个")
+ else:
+ print("❌ calibre 类未移除")
+
+
+def main():
+ """主函数"""
+ logger.remove()
+ logger.add(sys.stderr, level="INFO")
+
+ test_html_cleaning()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/archive/v0.09/tests/extraction_experiment/test_cleaned_epub.py b/archive/v0.09/tests/extraction_experiment/test_cleaned_epub.py
new file mode 100644
index 0000000..4406a48
--- /dev/null
+++ b/archive/v0.09/tests/extraction_experiment/test_cleaned_epub.py
@@ -0,0 +1,79 @@
+"""
+测试清理后的 ePub 提取效果
+"""
+
+import sys
+from pathlib import Path
+
+project_root = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(project_root))
+
+from ebooklib import epub
+from loguru import logger
+
+sys.path.insert(0, str(Path(__file__).parent))
+from extractors.one_to_one import OneToOneExtractor
+
+
+def test_cleaned_epub():
+ """测试清理后的 ePub"""
+
+ cleaned_path = project_root / "test_output" / "On_China_cleaned.epub"
+
+ if not cleaned_path.exists():
+ print(f"❌ 清理后的 ePub 不存在: {cleaned_path}")
+ return
+
+ print("\n" + "="*80)
+ print("测试清理后的 ePub 提取效果")
+ print("="*80 + "\n")
+
+ # 加载 ePub
+ book = epub.read_epub(str(cleaned_path))
+
+ # 找诗歌部分
+ for item in book.get_items():
+ if item.get_type() == 9 and 'dummy_split_010' in item.get_name():
+ content = item.get_content().decode('utf-8')
+
+ print(f"测试文件: {item.get_name()}\n")
+
+ # 提取
+ extractor = OneToOneExtractor()
+ items = extractor.extract(content, item.get_name())
+
+ print(f"提取了 {len(items)} 个元素\n")
+
+ # 显示前10个
+ for i, elem in enumerate(items[:10], 1):
+ print(f"{i}. <{elem['tag']}> {elem['text'][:60]}...")
+
+ print(f"\n... (共 {len(items)} 个元素)")
+
+ # 检查诗歌部分
+ print("\n" + "="*80)
+ print("检查诗歌部分:")
+ print("="*80 + "\n")
+
+ poem_lines = [item for item in items if 'ruler' in item['text'].lower() or 'mobilize' in item['text'].lower()]
+
+ if poem_lines:
+ print(f"找到 {len(poem_lines)} 行诗歌:")
+ for i, line in enumerate(poem_lines[:5], 1):
+ print(f" {i}. {line['text'][:50]}")
+ else:
+ print("❌ 未找到诗歌部分")
+
+ break
+
+
+def main():
+ """主函数"""
+ logger.remove()
+ logger.add(sys.stderr, level="INFO")
+
+ test_cleaned_epub()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/archive/v0.09/tests/extraction_experiment/test_decorative.py b/archive/v0.09/tests/extraction_experiment/test_decorative.py
new file mode 100644
index 0000000..ea51189
--- /dev/null
+++ b/archive/v0.09/tests/extraction_experiment/test_decorative.py
@@ -0,0 +1,200 @@
+"""
+装饰性元素提取测试
+
+验证增强提取器对装饰性符号的识别和保留
+"""
+
+import sys
+from pathlib import Path
+
+project_root = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(project_root))
+
+from ebooklib import epub
+from loguru import logger
+
+sys.path.insert(0, str(Path(__file__).parent))
+from extractors.enhanced_bs4 import EnhancedBS4Extractor
+from extractors.bs4_optimized import BS4OptimizedExtractor
+
+
+def test_decorative_elements():
+ """测试装饰性元素的识别"""
+ html = """
+
+
+ This is a normal paragraph. *** • • • Another paragraph here. ◆◇◆ Final paragraph. First paragraph. *** Second paragraph. {p.get_text()} 元素\n")
+
+ # 显示前10个
+ print("前10个元素:")
+ for i, elem in enumerate(items[:10], 1):
+ print(f" {i}. {elem['text'][:60]}...")
+
+ print(f"\n... (共 {len(items)} 个)")
+
+ # 模拟翻译
+ translation_map = {}
+ for elem in items:
+ if elem['should_translate']:
+ translation_map[elem['text']] = f"{elem['text']} [翻译]"
+
+ print(f"\n待翻译: {len(translation_map)} 个元素")
+
+ # 回填
+ result_html = extractor.backfill(items, translation_map)
+
+ # 验证
+ print("\n" + "="*80)
+ print("验证:")
+ print("="*80 + "\n")
+
+ if '[翻译]' in result_html:
+ print("✅ 翻译成功回填")
+ else:
+ print("❌ 翻译未回填")
+
+ # 检查 数量
+ from bs4 import BeautifulSoup
+ result_soup = BeautifulSoup(result_html, 'html.parser')
+ result_p_count = len(result_soup.find_all('p'))
+
+ print(f"✅ 回填后 元素: {result_p_count} 个")
+
+ break
+
+
+def main():
+ """主函数"""
+ logger.remove()
+ logger.add(sys.stderr, level="INFO")
+
+ test_fine_grained()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/archive/v0.09/tests/extraction_experiment/test_fine_grained_extractor.py b/archive/v0.09/tests/extraction_experiment/test_fine_grained_extractor.py
new file mode 100644
index 0000000..9a6a64f
--- /dev/null
+++ b/archive/v0.09/tests/extraction_experiment/test_fine_grained_extractor.py
@@ -0,0 +1,177 @@
+"""
+测试细粒度提取器
+
+验证 fine_grained.py 的提取效果
+"""
+
+import sys
+from pathlib import Path
+
+project_root = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(project_root))
+
+from ebooklib import epub
+from loguru import logger
+
+sys.path.insert(0, str(Path(__file__).parent))
+from extractors.fine_grained import FineGrainedExtractor
+
+
+def test_fine_grained_extraction(epub_path: Path):
+ """测试细粒度提取器"""
+ print(f"\n{'='*80}")
+ print(f"细粒度提取器测试: {epub_path.name}")
+ print(f"{'='*80}\n")
+
+ # 加载 EPUB
+ book = epub.read_epub(str(epub_path))
+
+ # 统计信息
+ total_docs = 0
+ total_elements = 0
+ total_to_translate = 0
+ total_decorative = 0
+
+ doc_stats = []
+
+ # 逐个文档提取
+ for item in book.get_items():
+ if item.get_type() == 9:
+ try:
+ content = item.get_content().decode('utf-8')
+ file_name = item.get_name()
+
+ # 提取
+ extractor = FineGrainedExtractor()
+ items = extractor.extract(content, file_name)
+
+ if items:
+ total_docs += 1
+ total_elements += len(items)
+
+ to_translate = sum(1 for i in items if i['should_translate'])
+ decorative = sum(1 for i in items if i.get('is_decorative'))
+
+ total_to_translate += to_translate
+ total_decorative += decorative
+
+ doc_stats.append({
+ 'name': file_name,
+ 'total': len(items),
+ 'to_translate': to_translate,
+ 'decorative': decorative,
+ 'doc_type': items[0]['doc_type'] if items else 'unknown'
+ })
+ except Exception as e:
+ logger.error(f"处理失败 {item.get_name()}: {e}")
+
+ # 总体统计
+ print(f"{'='*80}")
+ print("总体统计")
+ print(f"{'='*80}\n")
+ print(f"处理文档数: {total_docs}")
+ print(f"提取元素总数: {total_elements:,}")
+ print(f"需要翻译: {total_to_translate:,} ({total_to_translate/total_elements*100:.1f}%)")
+ print(f"装饰性元素: {total_decorative:,} ({total_decorative/total_elements*100:.1f}%)")
+ print()
+
+ # 按文档类型分组
+ core_docs = [d for d in doc_stats if d['doc_type'] == 'core']
+ toc_docs = [d for d in doc_stats if d['doc_type'] == 'toc']
+ skip_docs = [d for d in doc_stats if d['doc_type'] == 'skip']
+
+ print(f"{'='*80}")
+ print("按文档类型统计")
+ print(f"{'='*80}\n")
+
+ if core_docs:
+ core_elements = sum(d['total'] for d in core_docs)
+ core_translate = sum(d['to_translate'] for d in core_docs)
+ print(f"核心文档 (core): {len(core_docs)} 个")
+ print(f" - 元素数: {core_elements:,}")
+ print(f" - 需翻译: {core_translate:,}")
+ print()
+
+ if toc_docs:
+ toc_elements = sum(d['total'] for d in toc_docs)
+ toc_translate = sum(d['to_translate'] for d in toc_docs)
+ print(f"目录文档 (toc): {len(toc_docs)} 个")
+ print(f" - 元素数: {toc_elements:,}")
+ print(f" - 需翻译: {toc_translate:,}")
+ print()
+
+ if skip_docs:
+ skip_elements = sum(d['total'] for d in skip_docs)
+ print(f"跳过文档 (skip): {len(skip_docs)} 个")
+ print(f" - 元素数: {skip_elements:,}")
+ print()
+
+ # 显示部分文档详情
+ print(f"{'='*80}")
+ print("核心文档详情 (前10个)")
+ print(f"{'='*80}\n")
+
+ for i, doc in enumerate(core_docs[:10], 1):
+ print(f"{i}. {doc['name']}")
+ print(f" 元素: {doc['total']}, 翻译: {doc['to_translate']}, 装饰: {doc['decorative']}")
+
+ if len(core_docs) > 10:
+ print(f"\n... 还有 {len(core_docs) - 10} 个核心文档\n")
+
+ # 抽样显示提取内容
+ print(f"\n{'='*80}")
+ print("提取内容抽样 (第一个核心文档的前10个元素)")
+ print(f"{'='*80}\n")
+
+ if core_docs:
+ first_doc_name = core_docs[0]['name']
+
+ # 重新提取第一个文档
+ for item in book.get_items():
+ if item.get_type() == 9 and item.get_name() == first_doc_name:
+ content = item.get_content().decode('utf-8')
+ extractor = FineGrainedExtractor()
+ items = extractor.extract(content, first_doc_name)
+
+ print(f"文档: {first_doc_name}\n")
+
+ for i, elem in enumerate(items[:10], 1):
+ translate_flag = "✓" if elem['should_translate'] else "✗"
+ decorative_flag = " [装饰]" if elem.get('is_decorative') else ""
+
+ text_preview = elem['text'][:60]
+ if len(elem['text']) > 60:
+ text_preview += "..."
+
+ print(f"{i}. [{translate_flag}] <{elem['tag']}> {text_preview}{decorative_flag}")
+
+ if len(items) > 10:
+ print(f"\n... 还有 {len(items) - 10} 个元素")
+
+ break
+
+ print()
+
+
+def main():
+ """主函数"""
+ logger.remove()
+ logger.add(sys.stderr, level="ERROR")
+
+ # 测试清理后的 EPUB
+ cleaned_file = "On_China_cleaned.epub"
+ cleaned_path = project_root / "test_output" / cleaned_file
+
+ if not cleaned_path.exists():
+ print(f"❌ 清理文件不存在: {cleaned_path}")
+ print(f"\n提示: 请先运行清理器:")
+ print(f" python tests/extraction_experiment/simple_cleaner.py \\")
+ print(f" 'input/On_China_Henry_Kissinger.epub' \\")
+ print(f" 'test_output/{cleaned_file}'")
+ return
+
+ test_fine_grained_extraction(cleaned_path)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/archive/v0.09/tests/extraction_experiment/test_multi_epub.py b/archive/v0.09/tests/extraction_experiment/test_multi_epub.py
new file mode 100644
index 0000000..e4d4e2e
--- /dev/null
+++ b/archive/v0.09/tests/extraction_experiment/test_multi_epub.py
@@ -0,0 +1,272 @@
+"""
+优化的完整性测试脚本
+
+快速对比多个 ePub 的提取完整性,与 Pandoc 基准对比
+"""
+
+import sys
+from pathlib import Path
+from datetime import datetime
+from loguru import logger
+
+project_root = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(project_root))
+
+from ebooklib import epub
+
+sys.path.insert(0, str(Path(__file__).parent))
+from extractors.enhanced_bs4 import EnhancedBS4Extractor
+from extractors.baseline_pandoc import PandocBaseline
+
+
+def quick_coverage_check(extracted_text: str, baseline_text: str) -> dict:
+ """
+ 快速覆盖率检查(优化版)
+
+ 使用简化的词级别对比,避免复杂的相似度计算
+ """
+ import re
+
+ # 标准化
+ def normalize(text):
+ text = text.lower()
+ text = re.sub(r'[^\w\s]', ' ', text)
+ text = re.sub(r'\s+', ' ', text)
+ return text.strip()
+
+ extracted_norm = normalize(extracted_text)
+ baseline_norm = normalize(baseline_text)
+
+ # 分词
+ extracted_words = set(extracted_norm.split())
+ baseline_words = set(baseline_norm.split())
+
+ if not baseline_words:
+ return {'coverage': 0.0, 'common_words': 0, 'baseline_words': 0}
+
+ common = extracted_words & baseline_words
+ coverage = len(common) / len(baseline_words)
+
+ return {
+ 'coverage': coverage,
+ 'common_words': len(common),
+ 'baseline_words': len(baseline_words),
+ 'extracted_words': len(extracted_words)
+ }
+
+
+def test_single_epub(epub_path: Path, use_pandoc: bool = True) -> dict:
+ """
+ 测试单个 ePub 文件
+
+ Args:
+ epub_path: ePub 文件路径
+ use_pandoc: 是否使用 Pandoc 基准
+
+ Returns:
+ 测试结果字典
+ """
+ result = {
+ 'file_name': epub_path.name,
+ 'file_size': epub_path.stat().st_size,
+ 'timestamp': datetime.now().isoformat()
+ }
+
+ try:
+ # 1. Pandoc 基准(可选)
+ baseline_text = None
+ if use_pandoc:
+ logger.info(f"提取 Pandoc 基准: {epub_path.name}")
+ pandoc = PandocBaseline()
+ baseline_text = pandoc.extract_from_epub(str(epub_path))
+
+ if baseline_text:
+ result['baseline_length'] = len(baseline_text)
+ result['baseline_words'] = len(baseline_text.split())
+
+ # 2. 加载 ePub
+ logger.info(f"加载 ePub: {epub_path.name}")
+ book = epub.read_epub(str(epub_path))
+
+ # 3. 提取 HTML 内容
+ html_docs = []
+ for item in book.get_items():
+ if item.get_type() == 9: # ITEM_DOCUMENT
+ try:
+ content = item.get_content().decode('utf-8')
+ html_docs.append(content)
+ except:
+ continue
+
+ result['html_doc_count'] = len(html_docs)
+
+ # 合并 HTML
+ combined_html = "\n\n".join(html_docs)
+
+ # 4. 增强提取器测试
+ logger.info(f"测试增强提取器")
+ extractor = EnhancedBS4Extractor(preserve_decorative=True)
+ items = extractor.extract(combined_html)
+
+ # 统计
+ decorative_items = [i for i in items if i.get('is_decorative')]
+ nav_items = [i for i in items if i.get('is_navigation')]
+ content_items = [i for i in items if not i.get('is_navigation') and not i.get('is_decorative')]
+
+ result['total_elements'] = len(items)
+ result['content_elements'] = len(content_items)
+ result['decorative_elements'] = len(decorative_items)
+ result['navigation_elements'] = len(nav_items)
+
+ # 提取的文本
+ extracted_text = " ".join([item['text'] for item in content_items])
+ result['extracted_length'] = len(extracted_text)
+ result['extracted_words'] = len(extracted_text.split())
+
+ # 5. 与 Pandoc 对比
+ if baseline_text:
+ coverage_result = quick_coverage_check(extracted_text, baseline_text)
+ result['coverage'] = coverage_result['coverage']
+ result['common_words'] = coverage_result['common_words']
+
+ result['status'] = 'success'
+
+ except Exception as e:
+ logger.error(f"测试失败 {epub_path.name}: {e}")
+ result['status'] = 'failed'
+ result['error'] = str(e)
+
+ return result
+
+
+def generate_summary_report(results: list) -> str:
+ """生成汇总报告"""
+ report = ["# 多 ePub 提取完整性测试报告\n"]
+
+ report.append(f"**测试时间**: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
+ report.append(f"**测试文件数**: {len(results)}\n")
+
+ # 成功/失败统计
+ success_count = sum(1 for r in results if r.get('status') == 'success')
+ report.append(f"**成功**: {success_count}/{len(results)}\n")
+
+ # 汇总表
+ report.append("## 测试结果汇总\n")
+ report.append("| 文件名 | 文件大小 | 提取元素 | 装饰性 | 文本长度 | 覆盖率 |")
+ report.append("|--------|---------|---------|--------|---------|--------|")
+
+ for r in results:
+ if r.get('status') != 'success':
+ report.append(f"| {r['file_name'][:30]} | - | ❌ 失败 | - | - | - |")
+ continue
+
+ file_size = f"{r.get('file_size', 0) / 1024:.1f}KB"
+ total_elem = r.get('total_elements', 0)
+ decorative = r.get('decorative_elements', 0)
+ text_len = f"{r.get('extracted_length', 0):,}"
+ coverage = r.get('coverage', 0)
+ coverage_str = f"{coverage:.1%}" if coverage > 0 else "N/A"
+
+ report.append(
+ f"| {r['file_name'][:30]} | {file_size} | {total_elem} | {decorative} | {text_len} | {coverage_str} |"
+ )
+
+ report.append("")
+
+ # 详细分析
+ report.append("## 详细分析\n")
+
+ for r in results:
+ if r.get('status') != 'success':
+ continue
+
+ report.append(f"### {r['file_name']}\n")
+ report.append(f"- **HTML 文档数**: {r.get('html_doc_count', 0)}")
+ report.append(f"- **提取元素总数**: {r.get('total_elements', 0)}")
+ report.append(f" - 内容元素: {r.get('content_elements', 0)}")
+ report.append(f" - 装饰性元素: {r.get('decorative_elements', 0)}")
+ report.append(f" - 导航元素: {r.get('navigation_elements', 0)}")
+ report.append(f"- **提取文本长度**: {r.get('extracted_length', 0):,} 字符")
+ report.append(f"- **提取词数**: {r.get('extracted_words', 0):,}")
+
+ if 'baseline_length' in r:
+ report.append(f"- **Pandoc 基准长度**: {r.get('baseline_length', 0):,} 字符")
+ report.append(f"- **覆盖率**: {r.get('coverage', 0):.2%}")
+ report.append(f"- **共同词数**: {r.get('common_words', 0):,}")
+
+ report.append("")
+
+ # 总结
+ report.append("## 总结\n")
+
+ if success_count > 0:
+ avg_coverage = sum(r.get('coverage', 0) for r in results if r.get('status') == 'success') / success_count
+ total_decorative = sum(r.get('decorative_elements', 0) for r in results if r.get('status') == 'success')
+
+ report.append(f"- **平均覆盖率**: {avg_coverage:.2%}")
+ report.append(f"- **总装饰性元素**: {total_decorative} 个")
+ report.append(f"- **提取器状态**: {'✅ 正常' if avg_coverage > 0.9 else '⚠️ 需要优化'}")
+
+ return "\n".join(report)
+
+
+def main():
+ """主函数"""
+ logger.remove()
+ logger.add(sys.stderr, level="INFO")
+
+ # 测试文件列表
+ test_files = [
+ "Gambling Man.epub",
+ "On_China_Henry_Kissinger.epub",
+ "The World Atlas of Coffee - From Beans to Brewing - Coffees Explored, Explained and Enjoyed (James Hoffmann) (Z-Library).epub",
+ "The_Philosopher_in_the_Valley.epub",
+ "To_Explain_the_World.epub"
+ ]
+
+ input_dir = project_root / "input"
+ results = []
+
+ for filename in test_files:
+ epub_path = input_dir / filename
+
+ if not epub_path.exists():
+ logger.warning(f"跳过不存在的文件: {filename}")
+ continue
+
+ logger.info(f"\n{'='*60}")
+ logger.info(f"测试: {filename}")
+ logger.info(f"{'='*60}")
+
+ result = test_single_epub(epub_path, use_pandoc=True)
+ results.append(result)
+
+ # 打印简要结果
+ if result.get('status') == 'success':
+ logger.info(f"✅ 成功: {result.get('total_elements')} 个元素, "
+ f"{result.get('decorative_elements')} 个装饰性, "
+ f"覆盖率 {result.get('coverage', 0):.1%}")
+ else:
+ logger.error(f"❌ 失败: {result.get('error')}")
+
+ # 生成报告
+ report = generate_summary_report(results)
+
+ # 保存报告
+ report_dir = project_root / "tests" / "extraction_experiment" / "reports"
+ report_dir.mkdir(parents=True, exist_ok=True)
+
+ report_file = report_dir / "multi_epub_test_report.md"
+ with open(report_file, 'w', encoding='utf-8') as f:
+ f.write(report)
+
+ logger.info(f"\n报告已保存: {report_file}")
+
+ # 打印报告
+ print("\n" + "="*60)
+ print(report)
+ print("="*60)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/archive/v0.09/tests/extraction_experiment/test_one_to_one.py b/archive/v0.09/tests/extraction_experiment/test_one_to_one.py
new file mode 100644
index 0000000..fa7f9eb
--- /dev/null
+++ b/archive/v0.09/tests/extraction_experiment/test_one_to_one.py
@@ -0,0 +1,103 @@
+"""
+测试一比一对应提取器
+"""
+
+import sys
+from pathlib import Path
+
+project_root = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(project_root))
+
+from loguru import logger
+
+sys.path.insert(0, str(Path(__file__).parent))
+from extractors.one_to_one import OneToOneExtractor
+
+
+def test_one_to_one():
+ """测试一比一对应"""
+
+ # 测试 HTML
+ html = """
+ This is the first paragraph. This is the second paragraph. This is the first paragraph. This is the second paragraph. I II III IV XIV INTRODUCTION *** --- —— ................ Normal paragraph text. Text with bold. `，消除结构性漏译风险。
+* **Auto-Fix**: 修复 TOC 死链、缺失 UID、由于 `ebooklib` bug 导致的样式丢失。
+* **Result**: 产生一个标准的临时文件，后续所有操作基于此文件，不再受原始糟糕格式影响。
+
+### 2. Intelligent Extraction (智能提取)
+**模块**: `src/fine_grained_extractor.py` + `src/format_extractor.py`
+
+#### A. 结构层 (Macro)
+使用 `FineGrainedExtractor` 锁定所有正文元素 (`p`, `h1`-`h6`)。
+* **Filter**: 排除页码、页眉脚。
+* **Optimization**: 针对目录章节，识别 **罗马数字 (I, II)**、**单独数字 (1, 2)**、**修饰符 (***)**，这些内容**不送翻译**，直接在回填时保留原文，以维持原书排版美感。
+
+#### B. 内容层 (Micro)
+对每个提取的段落调用 `FormatExtractor`：
+* **Inline Style**: 将 ``, `` 转为配对占位符 `φ1φ...φ/1φ`。
+* **Formula Protection**: 识别 $E=mc^2$ 等数学公式，保护为不可变占位符。
+* **Drop Cap Handling**:
+ - 原始: `The`
+ - 提取给 LLM: "The" (完整单词，无格式干扰)
+ - 记录: Prefix 包含 Drop Cap 样式。
+
+### 3. Manifest Management (清单管理)
+**模块**: `src/manifest_manager.py`
+Manifest 是系统的**核心状态中心 (Source of Truth)**。
+* **作用**: 解耦提取和翻译。提取器只管往 Manifest 填数据，翻译器只管从 Manifest 取数据。
+* **Persistence**: 支持中断续传，翻译进度实时保存。
+
+### 4. Translation with Profiling (翻译)
+**模块**: `src/book_profiler.py` & `src/translator.py`
+* **Profiling**: 在翻译前，抽取部分文本分析书籍的类型（技术、小说、诗歌）、核心术语和语言风格，生成 `System Prompt`。
+* **Translation**: 这是纯文本层面的转换，LLM 处理的是带有 `φ` 占位符的文本。
+
+### 5. Robust Restoration (健壮还原)
+**模块**: `src/format_restorer.py`
+负责将 LLM 返回的文本还原为 HTML。
+* **Drop Cap Logic**:
+ - **原文回填**: 需要 Prefix `T`。
+ - **译文回填**: **丢弃** Drop Cap Prefix。中文不需要首字母下沉，否则会出现 "T这本书..." 的怪诞结果。
+* **Error Handling**:
+ - **Missing Placeholders**: 如果 LLM 丢了 `φ1φ`，自动在末尾补全或报错重试。
+ - **Hallucinated Placeholders**: 移除 LLM 臆造的不存在 ID。
+
+### 6. Backfill Strategy (回填策略)
+**模块**: `src/fine_grained_extractor.py` (backfill method)
+支持多种模式，且**严格遵循一对一 (One-to-One) 映射**，绝不依赖顺序，而是依赖元素的内存引用或唯一 ID。
+
+* **Mode A: Bilingual (双语)**
+ - 保留原文 DOM。
+ - 在原文后 `append` 一个新元素 ` 译文 ",
+ "_suffix": " `, `div`）剥离到 `_prefix` 和 `_suffix`。
+ * **目的**: 极大减少 LLM 输入 Token，且防止 LLM 随意修改外层布局。
+
+2. **首字下沉处理 (Drop Cap Handling)**:
+ * 检测并合并被 `` 单独包裹的首字母（如 `O` + `nce` → `Once`）。
+ * **目的**: 修复语意割裂，让 LLM 看到完整的单词。
+
+3. **占位符化 (Placeholder Mapping)**:
+ * 将内联标签（``, ``）或公式替换为短码 `φIDφ`。
+ * **目的**: 保护 HTML 属性不被“翻译”，降低噪声干扰。
+
+4. **完整性校验 (Integrity Check)**:
+ * **逻辑**: 抽提后的文本（去占位符）与原始纯文本进行归一化比对，要求覆盖率 **100%**。
+ * **兜底**: 若校验失败（如误删内容），回退到简单模式（只剥离首尾标签）。
+
+---
+
+## 5. 构建与回填流程 (Backfill & Build)
+
+### 5.1 翻译回填
+* 根据 `entry_id` 找到对应的 DOM 节点。
+* 使用 `FormatRestorer` 将 `translated_text` 中的占位符（`φ1φ`）还原为原始 HTML 标签（``）。
+* 根据模式（双语/单语）决定将新节点插入到原文后还是替换原文。
+
+### 5.2 链接修复 (Link Repair)
+在 `BilingualBuilder` 中执行：
+1. **ID 补全**: 为缺失 ID 的 TOC 节点自动生成 UUID。
+2. **死链检测**: 检查 TOC/Nav 指向的文件是否存在。
+3. **模糊修复**: 尝试通过文件名后缀匹配（解决路径前缀变更问题）或特定重定向（如 `c0.xhtml` -> `cover.xhtml`）。
+4. **坏死剔除**: 无法修复的死链将从目录中移除。
+
+### 5.3 CSS 样式恢复
+`EbookLib` 默认可能会重写 `` 导致样式丢失。
+* **逻辑**:
+ * 收集所有 CSS 资源。
+ * 在构建每个 HTML Item 时，显式计算 HTML 到 CSS 的**相对路径**。
+ * 强制调用 `item.add_link(..., rel='stylesheet', type='text/css')` 注入引用。
+
+---
+
+## 6. 常见问题排查
+
+### 6.1 UUID 不匹配
+**现象**: `WARNING - Element uuid-xxx not found`.
+**原因**: 手动删除了 `book_structure.json` 但保留了 `manifest.json`，导致重新生成的 HTML ID 与清单记录不一致。
+**解决**: 清空 `work/BookName` 目录重新运行，或确保两个 JSON 文件版本一致。
+
+### 6.2 样式丢失
+**现象**: 打开书面目全非，只有黑白文字。
+**检查**: 解压 EPUB，查看 HTML `` 是否有 ``。如果没有，检查 `BilingualBuilder` 的 Step 4 逻辑。
+
+### 6.3 翻译错位
+**现象**: 译文出现在了错误的位置。
+**检查**: 确认 `entry_id` 生成逻辑是否包含文件名，且文件名在处理过程中未被意外修改。
diff --git a/archive/v0.10/doc/translation_layer.md b/archive/v0.10/doc/translation_layer.md
new file mode 100644
index 0000000..f5f10e3
--- /dev/null
+++ b/archive/v0.10/doc/translation_layer.md
@@ -0,0 +1,108 @@
+# Translation Layer Technical Documentation
+
+This document details the architecture of the **Translation Layer**, which operates primarily on `manifest.json`.
+
+> **Core Principle**: The Translation Layer is **decoupled** from the EPUB file format. It reads translatable units from `manifest.json`, processes them using an LLM, and writes translations back to `manifest.json`. It does **not** read or parse the EPUB file directly.
+
+---
+
+## 1. Architecture Overview
+
+### Data Flow
+
+```mermaid
+graph LR
+ A[manifest.json] -->|Load| B(ManifestManager)
+ B -->|Entries| C{Translator Orchestrator}
+ C -->|Sample Text| D[BookProfiler]
+ D -->|Style Guide| C
+ C -->|Chunks| E[LLMClient]
+ E -->|Translation| C
+ C -->|Update| B
+ B -->|Save| A[manifest.json]
+```
+
+1. **Input**: `manifest.json` (Generated by Preprocessing Layer).
+2. **Process**:
+ * **Profiling**: Analyze text samples to generate a `BookProfile` (style, tone, terminology).
+ * **Translation**: Batch entries into chunks, send to LLM, receive translations.
+3. **Output**: `manifest.json` (Updated with `translated_text` fields).
+
+---
+
+## 2. Core Modules (`src/translation/`)
+
+### 2.1 ManifestManager (`manifest_manager.py`)
+* **Role**: The interface for the "Source of Truth".
+* **Responsibility**:
+ * Load `manifest.json`.
+ * Provide list of `ManifestEntry` objects.
+ * Save updates back to disk.
+* **Key Method**: `update_translation(entry_id, translation)`
+
+### 2.2 Translator Engine (`translator_engine.py`)
+* **Role**: Orchestrates the translation process.
+* **Responsibility**:
+ * **Filtering**: Identify untranslated entries.
+ * **Grouping**: Group entries by chapter (file path) to maintain context.
+ * **Chunking**: Create character-based chunks (default ~5000 chars) that do not cross chapter boundaries.
+ * **Concurrency**: Manage async workers (default 5 concurrent tasks).
+* **Input**: `List[ManifestEntry]`, `BookProfile`.
+* **Output**: Updates `ManifestEntry` objects in-place.
+
+### 2.3 LLM Client (`llm_client.py`)
+* **Role**: Handles raw communication with the LLM Provider (OpenAI compatible).
+* **Responsibility**:
+ * **Prompt Engineering**: Construct Short-ID based prompts.
+ * **Rate Limiting**: Control RPM (Requests Per Minute).
+ * **Retry Logic**: Exponential backoff for API failures.
+ * **Logging**: Save raw request/response pairs to `work/{book}/chunks/` for debugging.
+
+### 2.4 Book Profiler (`../preprocessing/profiler.py`)
+* **Note**: While located in preprocessing, it is often invoked at the start of the translation phase.
+* **Role**: Generates a style guide.
+* **Mechanism**: Extracts a sample (Intro + Random segments) from `manifest.json` entries and asks the LLM to analyze author style.
+
+---
+
+## 3. Short ID Strategy
+
+To optimize token usage and ensuring mapping accuracy, we use a **Short ID** system for LLM interaction.
+
+**Prompt Format**:
+```text
+#1: First paragraph text...
+#2: Second paragraph text...
+```
+
+**Response Format**:
+```text
+#1: 第一段翻译...
+#2: 第二段翻译...
+```
+
+The `LLMClient` maintains a mapping of `Short ID (#N)` <-> `Entry ID (File#UUID)` for each chunk lifecycle.
+
+---
+
+## 4. Pipeline Usage
+
+The translation is executed via the standalone pipeline script:
+
+```bash
+python pipeline/02_translate.py --input-epub inputs/my_book.epub
+```
+* **--input-epub**: Uses the filename to locate the `work/` directory.
+* **--book-name**: Alternatively, specify the book folder name directly.
+
+### Dependencies
+* Environment variables must be set in `.env`:
+ * `OPENAI_API_KEY`
+ * `OPENAI_BASE_URL` (Optional)
+
+---
+
+## 5. Development & Debugging
+
+* **Chunk Logs**: Check `.work/{book}/chunks/` to see exactly what was sent to and received from the LLM.
+* **Idempotency**: The translation script skips entries that already have `translated_text`. To re-translate, you must manually clear `translated_text` in `manifest.json` or delete the manifest (to restart from preprocessing).
diff --git a/archive/v0.10/main.py b/archive/v0.10/main.py
new file mode 100644
index 0000000..5421d7c
--- /dev/null
+++ b/archive/v0.10/main.py
@@ -0,0 +1,155 @@
+import argparse
+import sys
+import os
+import asyncio
+from pathlib import Path
+from dotenv import load_dotenv
+from src.common.config import load_global_config
+
+from src.preprocessing.epub_cleaner import EpubCleaner
+from src.preprocessing.profiler import BookProfiler
+from src.preprocessing.text_extractor import FineGrainedExtractor
+from src.translation.manifest_manager import ManifestManager
+from src.translation.translator_engine import Translator
+from src.translation.llm_client import LLMClient
+from src.assembly.backfiller import BackfillEngine
+from src.assembly.builder import BilingualBuilder
+from src.common.data_model import BookStructure
+from src.common.utils import setup_logger, ensure_directory
+from src.common.exceptions import EpubTranslatorError
+
+logger = setup_logger("main")
+
+# Unified work directory structure
+# .work/
+# ├── {book_name}/
+# │ ├── book_structure.json
+# │ ├── manifest.json
+# │ ├── assets/
+# │ └── chunks/
+
+def get_work_dirs(input_path: Path) -> dict:
+ """Get work directory paths for a specific book."""
+ book_name = input_path.stem
+ work_root = Path(".work") / book_name
+
+ return {
+ "root": work_root,
+ "structure": work_root / "book_structure.json",
+ "manifest": work_root / "manifest.json",
+ "assets": work_root / "assets",
+ "chunks": work_root / "chunks",
+ }
+
+
+async def run_pipeline(args):
+ input_path = Path(args.input_epub)
+ output_dir = Path(args.output_dir)
+
+ # Get work directories for this book
+ work = get_work_dirs(input_path)
+ ensure_directory(work["root"])
+ ensure_directory(output_dir)
+
+ # Load Config
+ config = load_global_config()
+ llm_conf = config.get("llm", {})
+ trans_conf = config.get("translation", {})
+
+ api_key = llm_conf.get("api_key")
+ # Base URL and Model come from config if not overridden
+ base_url = llm_conf.get("base_url")
+ # CLI model arg overrides config model, which overrides default
+ model = args.model if args.model else llm_conf.get("model", "gpt-3.5-turbo")
+
+ if not api_key:
+ logger.warning("OPENAI_API_KEY not found in env or config. LLM features may fail.")
+
+ try:
+ # 1. Preprocessing - Reuse book_structure.json if exists
+ if work["structure"].exists() and not args.force_clean:
+ logger.info(f"Reusing existing book_structure: {work['structure']}")
+ structure = BookStructure.load(work["structure"])
+ else:
+ logger.info("Cleaning EPUB and generating book_structure...")
+ cleaner = EpubCleaner(input_path, work["root"])
+ book_structure_json = cleaner.clean()
+ structure = BookStructure.load(book_structure_json)
+
+ # 2. Extraction
+ extractor = FineGrainedExtractor()
+ manifest_entries = extractor.extract(structure)
+
+ # 3. Manifest Management
+ manifest_manager = ManifestManager(work["manifest"])
+ manifest_manager.load() # Load existing if any
+ manifest_manager.add_entries(manifest_entries)
+ manifest_manager.save()
+
+ # 4. Translation
+ if not args.skip_translation:
+ if not api_key:
+ logger.error("Cannot translate without API Key. Use --skip-translation to test pipeline.")
+ sys.exit(1)
+
+ llm_client = LLMClient(
+ api_key=api_key,
+ base_url=base_url,
+ model=model,
+ requests_per_minute=llm_conf.get("requests_per_minute", 60),
+ concurrent_requests=llm_conf.get("concurrent_requests", 5)
+ )
+
+ # Profiling
+ profiler = BookProfiler(llm_client)
+ profile = await profiler.analyze(manifest_manager.entries)
+ logger.info(f"Book Profile: {profile}")
+
+ # Translation
+ target_chunk_size = trans_conf.get("chunk_size", 5000)
+ translator = Translator(llm_client, chunk_size=target_chunk_size)
+ await translator.translate(manifest_manager.entries, profile)
+ manifest_manager.save()
+
+ await llm_client.close()
+ else:
+ logger.info("Skipping translation step.")
+
+ # 5. Backfill
+ backfiller = BackfillEngine()
+ updated_structure = backfiller.backfill(structure, manifest_manager.entries, mode=args.mode)
+
+ # 6. Assembly - Pass original EPUB for TOC preservation
+ builder = BilingualBuilder(work["root"], original_epub_path=input_path)
+ output_filename = f"bilingual_{input_path.name}"
+ output_path = output_dir / output_filename
+
+ created_epub = builder.build(updated_structure, output_path)
+
+ logger.info(f"Pipeline completed! Output: {created_epub}")
+
+ except EpubTranslatorError as e:
+ logger.error(f"An error occurred: {e}")
+ sys.exit(1)
+ except Exception as e:
+ logger.critical(f"Unexpected error: {e}")
+ import traceback
+ traceback.print_exc()
+ sys.exit(1)
+
+def main():
+ parser = argparse.ArgumentParser(description="EPUB Bilingual Translator")
+ parser.add_argument("input_epub", help="Path to the input EPUB file")
+ parser.add_argument("--output-dir", default="output", help="Directory for output files")
+ parser.add_argument("--model", default=None, help="LLM Model to use (overrides config)")
+ parser.add_argument("--mode", default="bilingual", choices=["bilingual", "target_only"], help="Output mode")
+ parser.add_argument("--skip-translation", action="store_true", help="Skip LLM translation (for testing)")
+ parser.add_argument("--force-clean", action="store_true", help="Force re-clean EPUB even if book_structure exists")
+
+ args = parser.parse_args()
+
+ asyncio.run(run_pipeline(args))
+
+if __name__ == "__main__":
+ main()
+
diff --git a/archive/v0.10/pipeline/01_preprocess.py b/archive/v0.10/pipeline/01_preprocess.py
new file mode 100644
index 0000000..8f341f4
--- /dev/null
+++ b/archive/v0.10/pipeline/01_preprocess.py
@@ -0,0 +1,69 @@
+import argparse
+import sys
+import os
+from pathlib import Path
+
+# Add project root to sys.path
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from src.preprocessing.epub_cleaner import EpubCleaner
+from src.preprocessing.text_extractor import FineGrainedExtractor
+from src.translation.manifest_manager import ManifestManager
+from src.common.data_model import BookStructure
+from src.common.utils import setup_logger, ensure_directory
+from src.common.exceptions import EpubTranslatorError
+
+logger = setup_logger("pipeline_preprocess")
+
+def run_preprocess(args):
+ input_path = Path(args.input_epub)
+ if not input_path.exists():
+ logger.error(f"Input file not found: {input_path}")
+ sys.exit(1)
+
+ book_name = input_path.stem
+ work_root = Path("work") / book_name
+ ensure_directory(work_root)
+
+ structure_path = work_root / "book_structure.json"
+ manifest_path = work_root / "manifest.json"
+
+ try:
+ # 1. Clean / Load Structure
+ if structure_path.exists() and not args.force:
+ logger.info(f"Reusing existing structure: {structure_path}")
+ structure = BookStructure.load(structure_path)
+ else:
+ logger.info("Cleaning EPUB...")
+ cleaner = EpubCleaner(input_path, work_root)
+ structure_path = cleaner.clean()
+ structure = BookStructure.load(structure_path)
+
+ # 2. Extract Text
+ logger.info("Extracting text segments...")
+ extractor = FineGrainedExtractor()
+ entries = extractor.extract(structure)
+
+ # 3. Update Manifest
+ logger.info(f"Updating manifest: {manifest_path}")
+ manager = ManifestManager(manifest_path)
+ manager.load()
+ manager.add_entries(entries)
+ manager.save()
+
+ logger.info("Preprocessing complete.")
+
+ except EpubTranslatorError as e:
+ logger.error(f"Preprocessing failed: {e}")
+ sys.exit(1)
+ except Exception as e:
+ logger.critical(f"Unexpected error: {e}")
+ sys.exit(1)
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Step 1: Preprocessing (Clean + Extract)")
+ parser.add_argument("input_epub", help="Path to input EPUB")
+ parser.add_argument("--force", action="store_true", help="Force re-clean")
+
+ args = parser.parse_args()
+ run_preprocess(args)
diff --git a/archive/v0.10/pipeline/02_translate.py b/archive/v0.10/pipeline/02_translate.py
new file mode 100644
index 0000000..77700a5
--- /dev/null
+++ b/archive/v0.10/pipeline/02_translate.py
@@ -0,0 +1,91 @@
+import argparse
+import sys
+import os
+import asyncio
+from pathlib import Path
+# Add project root to sys.path
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from src.translation.translator_engine import Translator
+from src.translation.llm_client import LLMClient
+from src.translation.manifest_manager import ManifestManager
+from src.preprocessing.profiler import BookProfiler
+from src.common.utils import setup_logger
+from src.common.exceptions import EpubTranslatorError
+from src.common.config import load_global_config
+
+logger = setup_logger("pipeline_translate")
+
+async def run_translate(args):
+ # Resolve paths
+ if args.book_name:
+ book_name = args.book_name
+ elif args.input_epub:
+ book_name = Path(args.input_epub).stem
+ else:
+ logger.error("Must provide --book-name or --input-epub")
+ sys.exit(1)
+
+ work_root = Path("work") / book_name
+ manifest_path = work_root / "manifest.json"
+
+ if not manifest_path.exists():
+ logger.error(f"Manifest not found: {manifest_path}. Run Step 1 first.")
+ sys.exit(1)
+
+ # Load Config
+ config = load_global_config()
+ llm_conf = config.get("llm", {})
+ trans_conf = config.get("translation", {})
+
+ api_key = llm_conf.get("api_key")
+ if not api_key:
+ logger.error("OPENAI_API_KEY not found in env or config.")
+ sys.exit(1)
+
+ try:
+ # Load Manifest
+ manager = ManifestManager(manifest_path)
+ manager.load()
+
+ # Init components
+ # Allow CLI args to override config if needed (not implemented yet, taking config partial priority)
+ llm = LLMClient(
+ api_key=api_key,
+ base_url=llm_conf.get("base_url"),
+ model=args.model if args.model else llm_conf.get("model", "gpt-3.5-turbo"),
+ requests_per_minute=llm_conf.get("requests_per_minute", 60),
+ concurrent_requests=llm_conf.get("concurrent_requests", 5)
+ )
+
+ # Profile
+ profiler = BookProfiler(llm)
+ profile = await profiler.analyze(manager.entries)
+ logger.info(f"Book Profile: {profile.title} ({profile.genre})")
+
+ # Translate
+ target_chunk_size = trans_conf.get("chunk_size", 4000)
+ translator = Translator(llm, chunk_size=target_chunk_size)
+ await translator.translate(manager.entries, profile)
+
+ # Save final state
+ manager.save()
+ await llm.close()
+
+ logger.info("Translation complete.")
+
+ except EpubTranslatorError as e:
+ logger.error(f"Translation failed: {e}")
+ sys.exit(1)
+ except Exception as e:
+ logger.critical(f"Unexpected error: {e}")
+ sys.exit(1)
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Step 2: Translation")
+ parser.add_argument("--input-epub", help="Path to original EPUB (to derive book name)")
+ parser.add_argument("--book-name", help="Book name (folder name in work/)")
+ parser.add_argument("--model", default=None, help="LLM Model (overrides config)")
+
+ args = parser.parse_args()
+ asyncio.run(run_translate(args))
diff --git a/archive/v0.10/pipeline/03_assemble.py b/archive/v0.10/pipeline/03_assemble.py
new file mode 100644
index 0000000..dfd0cb7
--- /dev/null
+++ b/archive/v0.10/pipeline/03_assemble.py
@@ -0,0 +1,72 @@
+import argparse
+import sys
+import os
+from pathlib import Path
+
+# Add project root to sys.path
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from src.assembly.backfiller import BackfillEngine
+from src.assembly.builder import BilingualBuilder
+from src.translation.manifest_manager import ManifestManager
+from src.common.data_model import BookStructure
+from src.common.utils import setup_logger, ensure_directory
+from src.common.exceptions import EpubTranslatorError
+
+logger = setup_logger("pipeline_assemble")
+
+def run_assemble(args):
+ input_path = Path(args.input_epub)
+ if not input_path.exists():
+ logger.error(f"Input file not found: {input_path}")
+ sys.exit(1)
+
+ book_name = input_path.stem
+ work_root = Path("work") / book_name
+ structure_path = work_root / "book_structure.json"
+ manifest_path = work_root / "manifest.json"
+
+ if not structure_path.exists() or not manifest_path.exists():
+ logger.error("Missing structure or manifest. Run Step 1.")
+ sys.exit(1)
+
+ output_dir = Path(args.output_dir)
+ ensure_directory(output_dir)
+
+ try:
+ # Load Data
+ structure = BookStructure.load(structure_path)
+ manager = ManifestManager(manifest_path)
+ manager.load()
+
+ # Backfill
+ logger.info(f"Backfilling translations (Mode: {args.mode})...")
+ backfiller = BackfillEngine()
+ updated_structure = backfiller.backfill(structure, manager.entries, mode=args.mode)
+
+ # Build
+ logger.info("Building EPUB...")
+ builder = BilingualBuilder(work_root, original_epub_path=input_path)
+ output_filename = f"{book_name}_{args.mode}.epub"
+ output_path = output_dir / output_filename
+
+ builder.build(updated_structure, output_path)
+ logger.info(f"Assembly complete. Output: {output_path}")
+
+ except EpubTranslatorError as e:
+ logger.error(f"Assembly failed: {e}")
+ sys.exit(1)
+ except Exception as e:
+ logger.critical(f"Unexpected error: {e}")
+ import traceback
+ traceback.print_exc()
+ sys.exit(1)
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Step 3: Assembly (Backfill + Build)")
+ parser.add_argument("input_epub", help="Path to original EPUB")
+ parser.add_argument("--output-dir", default="output", help="Output directory")
+ parser.add_argument("--mode", default="bilingual", choices=["bilingual", "target_only"], help="Output mode")
+
+ args = parser.parse_args()
+ run_assemble(args)
diff --git a/archive/v0.10/requirements.txt b/archive/v0.10/requirements.txt
new file mode 100644
index 0000000..a3c8bff
--- /dev/null
+++ b/archive/v0.10/requirements.txt
@@ -0,0 +1,12 @@
+ebooklib>=0.19
+beautifulsoup4>=4.12.0
+lxml>=4.9.0
+openai>=1.0.0
+aiohttp>=3.9.0
+pydantic>=2.0.0
+loguru>=0.7.0
+rich>=13.0.0
+asyncio-throttle>=1.0.2
+tenacity>=8.0.0
+python-dotenv>=1.0.0
+PyYAML>=6.0
diff --git a/archive/v0.10/scripts/build_translated_epub.py b/archive/v0.10/scripts/build_translated_epub.py
new file mode 100644
index 0000000..7f4b502
--- /dev/null
+++ b/archive/v0.10/scripts/build_translated_epub.py
@@ -0,0 +1,51 @@
+import sys
+from pathlib import Path
+import traceback
+
+# Add src to path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from src.data_model import BookStructure, ManifestEntry
+from src.manifest_manager import ManifestManager
+from src.backfill_engine import BackfillEngine
+from src.bilingual_builder import BilingualBuilder
+from src.utils import setup_logger
+
+logger = setup_logger("build_final")
+
+def build_final():
+ BOOK_NAME = "Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)"
+ WORK_DIR = Path(f".work/{BOOK_NAME}")
+ MANIFEST_PATH = WORK_DIR / "manifest.json"
+ STRUCTURE_PATH = WORK_DIR / "book_structure.json"
+ OUTPUT_EPUB = Path("output/final_verification.epub")
+ INPUT_EPUB = Path(f"input/{BOOK_NAME}.epub")
+
+ if not MANIFEST_PATH.exists() or not STRUCTURE_PATH.exists():
+ logger.error("Missing manifest or structure. Run translation first.")
+ return
+
+ # 1. Load Data
+ logger.info("Loading structure and manifest...")
+ structure = BookStructure.load(STRUCTURE_PATH)
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+
+ # 2. Backfill
+ logger.info("Backfilling translations...")
+ backfiller = BackfillEngine()
+ structure = backfiller.backfill(structure, manager.entries, mode="bilingual")
+
+ # 3. Build
+ logger.info(f"Building final EPUB to {OUTPUT_EPUB}...")
+ builder = BilingualBuilder(WORK_DIR, original_epub_path=INPUT_EPUB)
+ builder.build(structure, OUTPUT_EPUB)
+
+ logger.info("Build complete.")
+
+if __name__ == "__main__":
+ try:
+ build_final()
+ except Exception as e:
+ traceback.print_exc()
+ print(f"CRITICAL ERROR: {e}")
diff --git a/archive/v0.10/scripts/debug_config.py b/archive/v0.10/scripts/debug_config.py
new file mode 100644
index 0000000..dc6516c
--- /dev/null
+++ b/archive/v0.10/scripts/debug_config.py
@@ -0,0 +1,102 @@
+import os
+import sys
+from pathlib import Path
+from dotenv import load_dotenv
+import asyncio
+import httpx
+from openai import AsyncOpenAI
+
+# Add project root to sys.path
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from src.common.config import load_global_config
+
+async def main():
+ print("--- Environment Debug ---")
+ load_dotenv()
+ env_key = os.getenv("OPENAI_API_KEY")
+ if env_key:
+ print(f"OPENAI_API_KEY found in env: {env_key[:8]}...{env_key[-4:]}")
+ else:
+ print("OPENAI_API_KEY NOT found in env!")
+
+ print("\n--- Config Loader Debug ---")
+ try:
+ config = load_global_config()
+ llm_conf = config.get("llm", {})
+ conf_key = llm_conf.get("api_key")
+ base_url = llm_conf.get("base_url")
+ model = llm_conf.get("model")
+
+ print(f"Config Base URL: {base_url}")
+ print(f"Config Model: {model}")
+ if conf_key:
+ print(f"Config API Key: {conf_key[:8]}...{conf_key[-4:]}")
+ if env_key and conf_key == env_key:
+ print("Config Key matches Env Key.")
+ else:
+ print("Config Key DOES NOT match Env Key!")
+ else:
+ print("Config API Key NOT found!")
+
+ print("\n--- API Connectivity Test ---")
+ if not conf_key or not base_url:
+ print("Missing params for test.")
+ return
+
+ headers = {"Authorization": f"Bearer {conf_key}"}
+ url = f"{base_url}/models"
+ print(f"Requesting: {url}")
+
+ async with httpx.AsyncClient() as client:
+ try:
+ resp = await client.get(url, headers=headers, timeout=10)
+ print(f"Status Code: {resp.status_code}")
+ if resp.status_code == 200:
+ print("Success! Models listed.")
+ else:
+ print(f"Failed. Response: {resp.text}")
+ except Exception as e:
+ print(f"Exception during request: {e}")
+
+ print("\n--- Chat Completion Test (Mimicking LLMClient) ---")
+
+ proxy_url = os.environ.get("http_proxy") or os.environ.get("https_proxy")
+ print(f"Proxy detected: {proxy_url}")
+
+ http_client = httpx.AsyncClient(
+ proxy=proxy_url,
+ timeout=60.0,
+ follow_redirects=True
+ ) if proxy_url else None
+
+ aclient = AsyncOpenAI(
+ api_key=conf_key,
+ base_url=base_url,
+ http_client=http_client
+ )
+
+ print(f"Model: {model}")
+ system_prompt = "You are a senior publishing editor."
+ user_prompt = "Analyze this text."
+
+ try:
+ print("Sending request with System Prompt...")
+ resp = await aclient.chat.completions.create(
+ model=model,
+ messages=[
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_prompt}
+ ],
+ temperature=0.3,
+ )
+ print("Success!")
+ print(f"Response: {resp.choices[0].message.content}")
+ except Exception as e:
+ print(f"Chat Completion failed: {type(e).__name__}: {e}")
+
+ except Exception as e:
+ print(f"Config loading failed: {e}")
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/archive/v0.10/scripts/test_backfill.py b/archive/v0.10/scripts/test_backfill.py
new file mode 100644
index 0000000..de51f29
--- /dev/null
+++ b/archive/v0.10/scripts/test_backfill.py
@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""
+Placeholder Backfill Test Script
+Tests placeholder restoration on translated entries.
+
+Usage:
+ python scripts/test_backfill.py --chapter 38
+ python scripts/test_backfill.py --chapter 38 --limit 10
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from src.manifest_manager import ManifestManager
+from src.format_restorer import FormatRestorer
+
+# Configuration - Use unified .work directory
+BOOK_NAME = "Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)"
+WORK_DIR = Path(f".work/{BOOK_NAME}")
+MANIFEST_PATH = WORK_DIR / "manifest.json"
+
+
+def build_toc_from_manifest(manager: ManifestManager) -> list:
+ """Build TOC from manifest entries."""
+ files = {}
+ for entry in manager.entries:
+ fp = entry.file_path
+ if fp not in files:
+ files[fp] = {'count': 0, 'first_text': ''}
+ files[fp]['count'] += 1
+ if not files[fp]['first_text'] and entry.original_text:
+ files[fp]['first_text'] = entry.original_text[:40].replace('\n', ' ')
+
+ toc = []
+ for i, (fp, info) in enumerate(sorted(files.items()), 1):
+ toc.append({
+ 'index': i,
+ 'href': fp,
+ 'title': info['first_text'] or f"File {i}",
+ 'paragraphs': info['count']
+ })
+ return toc
+
+
+def get_chapter_entries(manager: ManifestManager, chapter_index: int) -> tuple:
+ """Get entries for a specific chapter by index."""
+ toc = build_toc_from_manifest(manager)
+
+ if chapter_index < 1 or chapter_index > len(toc):
+ print(f"错误: 章节编号 {chapter_index} 无效 (范围: 1-{len(toc)})")
+ return None, None
+
+ chapter = toc[chapter_index - 1]
+ href = chapter['href']
+
+ entries = [e for e in manager.entries if e.file_path == href]
+ return chapter, entries
+
+
+def test_backfill(chapter: dict, entries: list, limit: int = None):
+ """Test placeholder restoration for a chapter."""
+
+ print(f"\n" + "=" * 70)
+ print(f"占位符回填测试 - 章节 #{chapter['index']}: {chapter['title'][:40]}...")
+ print("=" * 70)
+
+ # Filter entries with translation and placeholders
+ translated = [e for e in entries if e.translated_text]
+ with_placeholders = [e for e in translated if e.placeholders and len(e.placeholders) > 0]
+
+ print(f"\n统计:")
+ print(f" 总段落: {len(entries)}")
+ print(f" 已翻译: {len(translated)}")
+ print(f" 有占位符: {len(with_placeholders)}")
+
+ if not translated:
+ print("\n⚠️ 该章节没有已翻译的内容!")
+ return
+
+ # Test restoration
+ restorer = FormatRestorer()
+
+ success_count = 0
+ fail_count = 0
+ results = []
+
+ test_entries = with_placeholders[:limit] if limit else with_placeholders
+
+ print(f"\n测试 {len(test_entries)} 个带占位符的段落:")
+ print("-" * 70)
+
+ for i, entry in enumerate(test_entries, 1):
+ original = entry.original_text
+ translated = entry.translated_text
+ placeholders = entry.placeholders
+
+ # Get non-internal placeholders
+ visible_ph = {k: v for k, v in placeholders.items() if not k.startswith('_')}
+
+ # Perform restoration
+ restored, success = restorer.restore(translated, placeholders)
+
+ if success:
+ success_count += 1
+ status = "✅"
+ else:
+ fail_count += 1
+ status = "❌"
+
+ results.append({
+ 'index': i,
+ 'entry_id': entry.entry_id,
+ 'original': original,
+ 'translated': translated,
+ 'restored': restored,
+ 'placeholders': visible_ph,
+ 'success': success
+ })
+
+ # Print summary
+ print(f"\n[{i}] {status} {entry.entry_id[-40:]}")
+ print(f" 占位符: {list(visible_ph.keys())}")
+ print(f" 原文: {original[:50]}...")
+ print(f" 译文: {translated[:50]}...")
+
+ if not success:
+ print(f" 还原: {restored[:50]}...")
+ # Show what placeholders are missing
+ missing = []
+ for k in visible_ph.keys():
+ if k.isdigit():
+ if f"φ{k}φ" not in translated and f"φ/{k}φ" not in translated:
+ missing.append(k)
+ if missing:
+ print(f" 缺失: {missing}")
+
+ # Summary
+ print("\n" + "=" * 70)
+ print(f"测试结果汇总")
+ print("=" * 70)
+ print(f" 成功: {success_count}/{len(test_entries)}")
+ print(f" 失败: {fail_count}/{len(test_entries)}")
+
+ if fail_count > 0:
+ print(f"\n失败案例详情:")
+ for r in results:
+ if not r['success']:
+ print(f"\n [{r['index']}] {r['entry_id'][-50:]}")
+ print(f" 原文: {r['original'][:60]}...")
+ print(f" 译文: {r['translated'][:60]}...")
+ print(f" 还原: {r['restored'][:60]}...")
+ print(f" 占位符: {r['placeholders']}")
+
+ # Also test entries without visible placeholders (only _prefix/_suffix)
+ prefix_suffix_only = [e for e in translated
+ if e.placeholders
+ and all(k.startswith('_') for k in e.placeholders.keys())]
+
+ if prefix_suffix_only:
+ print(f"\n\n额外测试: 只有 _prefix/_suffix 的段落 ({len(prefix_suffix_only)} 个)")
+ print("-" * 70)
+
+ ps_success = 0
+ ps_fail = 0
+
+ for entry in prefix_suffix_only[:5]: # Test first 5
+ restored, success = restorer.restore(entry.translated_text, entry.placeholders)
+ if success:
+ ps_success += 1
+ status = "✅"
+ else:
+ ps_fail += 1
+ status = "❌"
+
+ print(f" {status} {entry.entry_id[-40:]}")
+ if '_prefix' in entry.placeholders:
+ print(f" _prefix: {entry.placeholders['_prefix'][:30]}...")
+ if '_suffix' in entry.placeholders:
+ print(f" _suffix: {entry.placeholders['_suffix'][:30]}...")
+
+ print(f"\n 结果: {ps_success}/{min(5, len(prefix_suffix_only))} 成功")
+
+
+def main():
+ parser = argparse.ArgumentParser(description="测试占位符回填")
+ parser.add_argument("--chapter", "-c", type=int, required=True, help="章节编号")
+ parser.add_argument("--limit", "-l", type=int, default=20, help="测试数量限制 (默认20)")
+ args = parser.parse_args()
+
+ if not MANIFEST_PATH.exists():
+ print(f"错误: Manifest 不存在: {MANIFEST_PATH}")
+ return
+
+ # Load manifest
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+ print(f"已加载 manifest: {len(manager.entries)} 条目")
+
+ # Get chapter
+ chapter, entries = get_chapter_entries(manager, args.chapter)
+ if not chapter:
+ return
+
+ # Test backfill
+ test_backfill(chapter, entries, args.limit)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/archive/v0.10/scripts/test_conn.py b/archive/v0.10/scripts/test_conn.py
new file mode 100644
index 0000000..7e89b75
--- /dev/null
+++ b/archive/v0.10/scripts/test_conn.py
@@ -0,0 +1,25 @@
+import asyncio
+import os
+import httpx
+
+async def test_conn():
+ print(f"HTTP_PROXY: {os.environ.get('http_proxy')}")
+ print(f"HTTPS_PROXY: {os.environ.get('https_proxy')}")
+ print(f"ALL_PROXY: {os.environ.get('all_proxy')}")
+
+ url = "https://api.gpt.ge/v1/models"
+ headers = {"Authorization": f"Bearer {os.environ.get('V3_API_KEY')}"}
+
+ print(f"Connecting to {url}...")
+ try:
+ async with httpx.AsyncClient(timeout=10, follow_redirects=True) as client:
+ resp = await client.get(url, headers=headers)
+ print(f"Status: {resp.status_code}")
+ print(f"Headers: {resp.headers}")
+ except Exception as e:
+ print(f"Error: {e}")
+
+if __name__ == "__main__":
+ from dotenv import load_dotenv
+ load_dotenv()
+ asyncio.run(test_conn())
diff --git a/archive/v0.10/scripts/test_openai_conn.py b/archive/v0.10/scripts/test_openai_conn.py
new file mode 100644
index 0000000..7f3b2fe
--- /dev/null
+++ b/archive/v0.10/scripts/test_openai_conn.py
@@ -0,0 +1,40 @@
+import asyncio
+import os
+import httpx
+from openai import AsyncOpenAI
+
+async def test_openai():
+ proxy_url = os.environ.get("http_proxy")
+ print(f"Using proxy: {proxy_url}")
+
+ http_client = httpx.AsyncClient(
+ proxy=proxy_url,
+ timeout=30.0,
+ follow_redirects=True
+ )
+
+ client = AsyncOpenAI(
+ base_url="https://api.gpt.ge/v1",
+ api_key=os.environ.get("V3_API_KEY"),
+ http_client=http_client
+ )
+
+ print("Sending request...")
+ try:
+ response = await client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hello"}],
+ max_tokens=5
+ )
+ print(f"Response: {response.choices[0].message.content}")
+ except Exception as e:
+ import traceback
+ traceback.print_exc()
+ print(f"Error: {e}")
+ finally:
+ await http_client.aclose()
+
+if __name__ == "__main__":
+ from dotenv import load_dotenv
+ load_dotenv()
+ asyncio.run(test_openai())
diff --git a/archive/v0.10/scripts/test_short_id_strategy.py b/archive/v0.10/scripts/test_short_id_strategy.py
new file mode 100644
index 0000000..45922eb
--- /dev/null
+++ b/archive/v0.10/scripts/test_short_id_strategy.py
@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+"""
+Debug script to show chunk content and test short ID strategy.
+"""
+
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+from src.manifest_manager import ManifestManager
+from src.llm_client import LLMClient
+from src.data_model import ManifestEntry
+
+load_dotenv()
+
+# Configuration
+MANIFEST_PATH = Path("cache/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)_manifest.json")
+API_KEY = os.getenv("V3_API_KEY")
+BASE_URL = "https://api.gpt.ge/v1"
+MODEL = "gemini-3-flash-preview"
+EXTRA_HEADERS = {"x-foo": "true"}
+
+
+def show_current_chunk_format():
+ """Show current chunk format (problematic)."""
+ print("\n" + "="*60)
+ print("当前 Chunk 格式 (有问题)")
+ print("="*60)
+
+ # Load manifest
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+
+ # Get sample entries with placeholders
+ samples = [e for e in manager.entries if e.placeholders][:3]
+
+ print("\n发送给 LLM 的格式 (当前):")
+ print("-"*60)
+ for item in samples:
+ context = item.context or "BODY"
+ print(f"{item.entry_id} [{context}] {item.original_text[:50]}...")
+
+ print("\n问题分析:")
+ print(" 1. entry_id 太长 (包含 UUID): 容易被 LLM 截断或修改")
+ print(" 2. [BODY] context 没必要发送")
+ print(" 3. 依赖 LLM 精确复制长 ID，不可靠")
+
+
+def show_proposed_chunk_format():
+ """Show proposed short ID chunk format."""
+ print("\n" + "="*60)
+ print("建议的 Chunk 格式 (短 ID)")
+ print("="*60)
+
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+
+ samples = [e for e in manager.entries if e.placeholders][:5]
+
+ print("\n发送给 LLM 的格式 (建议):")
+ print("-"*60)
+
+ # Build with short IDs
+ id_map = {} # short_id -> entry_id
+ for i, item in enumerate(samples, 1):
+ short_id = f"#{i}"
+ id_map[short_id] = item.entry_id
+ text = item.original_text[:60]
+ print(f"{short_id}: {text}...")
+
+ print("\n期望 LLM 返回的格式:")
+ print("-"*60)
+ print("#1: φ1φpenguinrandomhouse.com（保持不翻译）")
+ print("#2: 该产品在欧盟的产品安全授权代表为 φ1φPenguin Random House Irelandφ/1φ...")
+ print("#3: φ1φ献辞")
+ print("#4: φ1φ题记")
+ print("#5: φ1φ作者说明")
+
+ print("\n优势:")
+ print(" 1. 短 ID (#1, #2...) 不会被 LLM 弄乱")
+ print(" 2. 去掉了无用的 context 标签")
+ print(" 3. 解析更可靠：用正则 ^#(\\d+): 匹配")
+ print(" 4. ID 映射表保留在代码中，用于还原")
+
+ print("\n映射表 (代码内保留):")
+ for short_id, full_id in id_map.items():
+ print(f" {short_id} -> {full_id[:50]}...")
+
+
+async def test_short_id_translation():
+ """Test translation with short ID format."""
+ print("\n" + "="*60)
+ print("测试短 ID 翻译")
+ print("="*60)
+
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+
+ # Get 5 entries with varied content
+ samples = [e for e in manager.entries if len(e.original_text) > 20][:5]
+
+ # Build prompt with short IDs
+ id_map = {}
+ lines = []
+ for i, item in enumerate(samples, 1):
+ short_id = f"#{i}"
+ id_map[short_id] = item.entry_id
+ text = item.original_text.replace('\n', ' ').strip()
+ lines.append(f"{short_id}: {text}")
+
+ user_prompt = "\n".join(lines)
+
+ print("\n发送给 LLM 的 Prompt:")
+ print("-"*60)
+ print(user_prompt)
+
+ # Create client
+ from openai import AsyncOpenAI
+ client = AsyncOpenAI(
+ base_url=BASE_URL,
+ api_key=API_KEY,
+ default_headers=EXTRA_HEADERS
+ )
+
+ system_prompt = """You are a professional English to Chinese translator.
+
+Translate each line to Chinese. Keep the format:
+- Each line starts with #N: (keep this ID exactly)
+- Preserve any φXφ placeholders exactly as-is
+- Only output translations, no explanations
+
+Example input:
+#1: Hello world
+#2: φ1φClick hereφ/1φ to continue
+
+Example output:
+#1: 你好世界
+#2: φ1φ点击这里φ/1φ 继续"""
+
+ print("\n发送请求...")
+
+ try:
+ resp = await client.chat.completions.create(
+ model=MODEL,
+ messages=[
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_prompt}
+ ],
+ temperature=0.3,
+ )
+ raw_response = resp.choices[0].message.content.strip()
+
+ print("\nLLM 返回:")
+ print("-"*60)
+ print(raw_response)
+
+ # Parse with short ID
+ print("\n解析结果:")
+ print("-"*60)
+ import re
+ results = {}
+ for line in raw_response.split("\n"):
+ line = line.strip()
+ match = re.match(r'^#(\d+):\s*(.+)$', line)
+ if match:
+ short_id = f"#{match.group(1)}"
+ translation = match.group(2)
+ if short_id in id_map:
+ full_id = id_map[short_id]
+ results[full_id] = translation
+ print(f" {short_id} -> {translation[:40]}...")
+
+ print(f"\n成功解析: {len(results)}/{len(samples)}")
+
+ finally:
+ await client.close()
+
+
+async def main():
+ print("="*60)
+ print("Chunk ID 策略分析与测试")
+ print("="*60)
+
+ if not MANIFEST_PATH.exists():
+ print(f"ERROR: Manifest not found at {MANIFEST_PATH}")
+ return
+
+ # 1. Show current format (problems)
+ show_current_chunk_format()
+
+ # 2. Show proposed format
+ show_proposed_chunk_format()
+
+ # 3. Test short ID translation
+ if API_KEY:
+ await test_short_id_translation()
+ else:
+ print("\n跳过测试 (V3_API_KEY 未设置)")
+
+ print("\n" + "="*60)
+ print("分析完成")
+ print("="*60)
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/archive/v0.10/scripts/test_translation_pipeline.py b/archive/v0.10/scripts/test_translation_pipeline.py
new file mode 100644
index 0000000..2a92890
--- /dev/null
+++ b/archive/v0.10/scripts/test_translation_pipeline.py
@@ -0,0 +1,355 @@
+#!/usr/bin/env python3
+"""
+Translation Pipeline Debug Script
+Shows visible results at each step of the translation process.
+"""
+
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+
+# Add project root to path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+
+from src.manifest_manager import ManifestManager
+from src.llm_client import LLMClient
+from src.book_profiler import BookProfiler
+from src.translator import Translator
+from src.format_restorer import FormatRestorer
+from src.data_model import ManifestEntry, BookProfile
+
+load_dotenv()
+
+# Configuration
+MANIFEST_PATH = Path("cache/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)_manifest.json")
+API_KEY = os.getenv("V3_API_KEY")
+BASE_URL = "https://api.gpt.ge/v1"
+MODEL = "gemini-3-flash-preview"
+EXTRA_HEADERS = {"x-foo": "true"}
+
+# Chunk size configuration
+MAX_CHUNK_SIZE = 15 # Maximum entries per chunk
+
+
+def group_entries_by_file(entries: list) -> dict:
+ """Group manifest entries by their source file (chapter)."""
+ grouped = {}
+ for entry in entries:
+ file_path = entry.file_path
+ if file_path not in grouped:
+ grouped[file_path] = []
+ grouped[file_path].append(entry)
+ return grouped
+
+
+def create_chapter_aware_chunks(entries: list, max_size: int = MAX_CHUNK_SIZE) -> list:
+ """
+ Create chunks that respect chapter boundaries.
+ Returns list of (file_path, chunk_entries) tuples.
+ """
+ grouped = group_entries_by_file(entries)
+ chunks = []
+
+ for file_path, file_entries in grouped.items():
+ # Split this file's entries into chunks of max_size
+ for i in range(0, len(file_entries), max_size):
+ chunk = file_entries[i:i + max_size]
+ chunks.append((file_path, chunk))
+
+ return chunks
+
+
+def get_file_type(file_path: str) -> str:
+ """Determine the type of content based on file name."""
+ fname = file_path.lower()
+ if any(k in fname for k in ['toc', 'contents', 'nav']):
+ return 'toc'
+ elif any(k in fname for k in ['title', 'cover']):
+ return 'cover'
+ elif any(k in fname for k in ['copyright', 'colophon']):
+ return 'legal'
+ elif any(k in fname for k in ['author', 'about']):
+ return 'author_bio'
+ elif any(k in fname for k in ['index', 'bibliography', 'endnote', 'footnote']):
+ return 'reference'
+ else:
+ return 'body'
+
+
+async def step1_load_manifest():
+ """Step 1: Load manifest and show statistics."""
+ print("\n" + "="*60)
+ print("STEP 1: Loading Manifest")
+ print("="*60)
+
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+
+ entries = manager.entries
+ untranslated = [e for e in entries if not e.translated_text]
+
+ print(f" Total entries: {len(entries)}")
+ print(f" Untranslated: {len(untranslated)}")
+
+ # Show grouping by file
+ grouped = group_entries_by_file(entries)
+ print(f" Unique files: {len(grouped)}")
+
+ # Show sample entry
+ if entries:
+ sample = entries[0]
+ print(f"\n Sample entry:")
+ print(f" ID: {sample.entry_id}")
+ print(f" File: {sample.file_path}")
+ print(f" Original: {sample.original_text[:80]}...")
+ print(f" Placeholders: {sample.placeholders}")
+
+ return manager
+
+
+async def step2_profile_book(manager: ManifestManager, llm_client: LLMClient):
+ """Step 2: Generate book profile."""
+ print("\n" + "="*60)
+ print("STEP 2: Generating Book Profile")
+ print("="*60)
+
+ profiler = BookProfiler(llm_client)
+ profile = await profiler.analyze(manager.entries)
+
+ print(f" Title: {profile.title}")
+ print(f" Author: {profile.author}")
+ print(f" Genre: {profile.genre}")
+ print(f" Keywords: {profile.keywords}")
+ print(f" Style Guide: {profile.style_guide[:200]}..." if profile.style_guide else " Style Guide: (none)")
+
+ return profile
+
+
+async def step3_create_chunks(manager: ManifestManager):
+ """Step 3: Create chapter-aware chunks."""
+ print("\n" + "="*60)
+ print("STEP 3: Creating Chapter-Aware Chunks")
+ print("="*60)
+
+ untranslated = [e for e in manager.entries if not e.translated_text]
+ chunks = create_chapter_aware_chunks(untranslated)
+
+ print(f" Total chunks: {len(chunks)}")
+
+ # Show chunk distribution
+ print(f"\n Chunk distribution by file type:")
+ type_counts = {}
+ for file_path, chunk_entries in chunks:
+ ftype = get_file_type(file_path)
+ type_counts[ftype] = type_counts.get(ftype, 0) + 1
+
+ for ftype, count in sorted(type_counts.items()):
+ print(f" {ftype}: {count} chunks")
+
+ # Show first 3 chunks
+ print(f"\n First 3 chunks:")
+ for i, (file_path, chunk_entries) in enumerate(chunks[:3]):
+ ftype = get_file_type(file_path)
+ print(f" [{i}] {file_path} ({ftype}): {len(chunk_entries)} entries")
+ if chunk_entries:
+ print(f" First: {chunk_entries[0].original_text[:50]}...")
+
+ return chunks
+
+
+async def step4_translate_sample(chunks: list, llm_client: LLMClient, profile: BookProfile):
+ """Step 4: Translate a sample chunk and show results."""
+ print("\n" + "="*60)
+ print("STEP 4: Translating Sample Chunk")
+ print("="*60)
+
+ if not chunks:
+ print(" No chunks to translate!")
+ return
+
+ # Pick a proper body chapter (skip first few files which are usually cover/copyright/toc)
+ sample_chunk = None
+ skip_prefixes = ['cM', 'c9', 'c18'] # Cover, title, contents pages
+ for file_path, chunk_entries in chunks:
+ # Skip non-body files and known cover/toc files
+ ftype = get_file_type(file_path)
+ fname = Path(file_path).stem
+ if ftype == 'body' and fname not in skip_prefixes and len(chunk_entries) > 3:
+ sample_chunk = (file_path, chunk_entries[:5]) # Limit to 5 entries for demo
+ break
+
+ if not sample_chunk:
+ # Fallback to any body chunk
+ for file_path, chunk_entries in chunks:
+ if get_file_type(file_path) == 'body':
+ sample_chunk = (file_path, chunk_entries[:5])
+ break
+
+ if not sample_chunk:
+ sample_chunk = chunks[0]
+ sample_chunk = (sample_chunk[0], sample_chunk[1][:5])
+
+ file_path, entries = sample_chunk
+ ftype = get_file_type(file_path)
+
+ print(f" Selected chunk: {file_path} ({ftype})")
+ print(f" Entries: {len(entries)}")
+
+ # Show entries before translation
+ print(f"\n === Before Translation ===")
+ for i, entry in enumerate(entries):
+ print(f" [{i}] {entry.entry_id}")
+ print(f" Original: {entry.original_text[:60]}...")
+ if entry.placeholders:
+ print(f" Placeholders: {list(entry.placeholders.keys())}")
+
+ # Translate
+ print(f"\n Translating...")
+ results = await llm_client.translate_chunk(
+ entries,
+ instruction=profile.style_guide,
+ mode="bilingual"
+ )
+
+ # Apply results and show
+ print(f"\n === After Translation ===")
+ restorer = FormatRestorer()
+ for i, entry in enumerate(entries):
+ if entry.entry_id in results:
+ translated = results[entry.entry_id]
+ entry.translated_text = translated
+
+ print(f" [{i}] {entry.entry_id}")
+ print(f" Original: {entry.original_text[:50]}...")
+ print(f" Translated: {translated[:50]}...")
+
+ # Restore format
+ if entry.placeholders:
+ restored, success = restorer.restore(translated, entry.placeholders)
+ print(f" Restored OK: {success}")
+ if not success:
+ print(f" Restored: {restored[:50]}...")
+ else:
+ print(f" [{i}] MISSING: {entry.entry_id}")
+
+ return entries
+
+
+async def step5_test_placeholders(manager: ManifestManager, llm_client: LLMClient, profile: BookProfile):
+ """Step 5: Test placeholder handling with entries that have placeholders."""
+ print("\n" + "="*60)
+ print("STEP 5: Testing Placeholder Handling")
+ print("="*60)
+
+ # Find entries with placeholders
+ entries_with_ph = [e for e in manager.entries if e.placeholders and len(e.placeholders) > 1]
+
+ print(f" Entries with placeholders: {len(entries_with_ph)}")
+
+ if not entries_with_ph:
+ print(" No entries with placeholders found!")
+ return
+
+ # Pick 5 diverse samples
+ samples = entries_with_ph[:5]
+
+ print(f"\n === Selected Samples ({len(samples)}) ===")
+ for i, entry in enumerate(samples):
+ ph_keys = [k for k in entry.placeholders.keys() if not k.startswith('_')]
+ print(f" [{i}] {entry.entry_id}")
+ print(f" Original: {entry.original_text[:60]}...")
+ print(f" Placeholders: {ph_keys}")
+
+ # Translate
+ print(f"\n Translating {len(samples)} entries with placeholders...")
+ results = await llm_client.translate_chunk(
+ samples,
+ instruction=profile.style_guide,
+ mode="bilingual"
+ )
+
+ # Show results with restoration
+ print(f"\n === Translation Results ===")
+ restorer = FormatRestorer()
+ success_count = 0
+
+ for i, entry in enumerate(samples):
+ print(f"\n [{i}] {entry.entry_id}")
+ print(f" Original: {entry.original_text[:50]}...")
+
+ if entry.entry_id in results:
+ translated = results[entry.entry_id]
+ print(f" Translated: {translated[:50]}...")
+
+ # Check if placeholders are preserved
+ ph_keys = [k for k in entry.placeholders.keys() if not k.startswith('_')]
+ preserved = all(f"φ{k}φ" in translated or f"φ/{k}φ" in translated for k in ph_keys if k.isdigit())
+ print(f" PH Preserved: {preserved}")
+
+ # Restore format
+ restored, success = restorer.restore(translated, entry.placeholders)
+ print(f" Restore OK: {success}")
+
+ if success:
+ success_count += 1
+ else:
+ print(f" Restored: {restored[:50]}...")
+ else:
+ print(f" MISSING from results!")
+
+ print(f"\n Summary: {success_count}/{len(samples)} restored successfully")
+
+
+async def main():
+ """Run all steps."""
+ print("="*60)
+ print("TRANSLATION PIPELINE DEBUG")
+ print("="*60)
+
+ if not API_KEY:
+ print("ERROR: V3_API_KEY not found in .env")
+ return
+
+ if not MANIFEST_PATH.exists():
+ print(f"ERROR: Manifest not found at {MANIFEST_PATH}")
+ print("Run the main pipeline first to generate the manifest.")
+ return
+
+ # Initialize LLM client
+ llm_client = LLMClient(
+ api_key=API_KEY,
+ base_url=BASE_URL,
+ model=MODEL,
+ extra_headers=EXTRA_HEADERS
+ )
+
+ try:
+ # Step 1: Load manifest
+ manager = await step1_load_manifest()
+
+ # Step 2: Profile book
+ profile = await step2_profile_book(manager, llm_client)
+
+ # Step 3: Create chunks
+ chunks = await step3_create_chunks(manager)
+
+ # Step 4: Translate sample (simple text)
+ await step4_translate_sample(chunks, llm_client, profile)
+
+ # Step 5: Test placeholders
+ await step5_test_placeholders(manager, llm_client, profile)
+
+ print("\n" + "="*60)
+ print("DEBUG COMPLETE")
+ print("="*60)
+
+ finally:
+ await llm_client.close()
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/archive/v0.10/scripts/translate_chapter.py b/archive/v0.10/scripts/translate_chapter.py
new file mode 100644
index 0000000..8dd0688
--- /dev/null
+++ b/archive/v0.10/scripts/translate_chapter.py
@@ -0,0 +1,353 @@
+#!/usr/bin/env python3
+"""
+Chapter Translation Test Script
+Translate a complete chapter to test the full pipeline.
+
+Usage:
+ python scripts/translate_chapter.py --show-toc # 显示章节目录
+ python scripts/translate_chapter.py --chapter 5 # 翻译第5章
+ python scripts/translate_chapter.py --chapter 5 --test # 测试模式，只翻译前2个chunk
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+from src.manifest_manager import ManifestManager
+from src.llm_client import LLMClient
+from src.book_profiler import BookProfiler
+from src.format_restorer import FormatRestorer
+from src.data_model import ManifestEntry, BookStructure
+
+load_dotenv()
+
+# Configuration - Use unified .work directory
+BOOK_NAME = "Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)"
+WORK_DIR = Path(f".work/{BOOK_NAME}")
+MANIFEST_PATH = WORK_DIR / "manifest.json"
+STRUCTURE_PATH = WORK_DIR / "book_structure.json"
+CHUNK_DIR = WORK_DIR / "chunks"
+
+API_KEY = os.getenv("V3_API_KEY")
+BASE_URL = "https://api.gpt.ge/v1"
+MODEL = "gpt-4o-mini" # "gemini-3-flash-preview"
+EXTRA_HEADERS = {"x-foo": "true"}
+
+# Chunk config - around 5000 chars per chunk
+CHUNK_SIZE_CHARS = 5000
+
+
+def load_toc_from_structure() -> list:
+ """Load TOC from book structure for readable chapter names."""
+ if not STRUCTURE_PATH.exists():
+ return []
+
+ try:
+ structure = BookStructure.load(STRUCTURE_PATH)
+ # Build TOC from spine order with chapter titles
+ toc = []
+ for i, item_id in enumerate(structure.spine, 1):
+ if item_id in structure.resources:
+ resource = structure.resources[item_id]
+ href = resource.href
+
+ # Try to extract title from content
+ title = extract_title_from_html(resource.content) if resource.content else None
+
+ toc.append({
+ 'index': i,
+ 'item_id': item_id,
+ 'href': href,
+ 'title': title or f"Chapter {i}"
+ })
+ return toc
+ except Exception as e:
+ print(f"警告: 无法加载书籍结构: {e}")
+ return []
+
+
+def extract_title_from_html(html: str) -> str:
+ """Extract title from HTML content."""
+ from bs4 import BeautifulSoup
+ soup = BeautifulSoup(html, 'html.parser')
+
+ # Try h1, h2, h3 in order
+ for tag in ['h1', 'h2', 'h3']:
+ elem = soup.find(tag)
+ if elem:
+ return elem.get_text().strip()[:50]
+
+ # Try first paragraph
+ p = soup.find('p')
+ if p:
+ text = p.get_text().strip()[:50]
+ if text:
+ return text + "..."
+
+ return None
+
+
+def build_toc_from_manifest(manager: ManifestManager) -> list:
+ """Build TOC from manifest entries."""
+ files = {}
+ for entry in manager.entries:
+ fp = entry.file_path
+ if fp not in files:
+ files[fp] = {
+ 'count': 0,
+ 'first_text': '',
+ 'total_chars': 0
+ }
+ files[fp]['count'] += 1
+ files[fp]['total_chars'] += len(entry.original_text)
+ if not files[fp]['first_text'] and entry.original_text:
+ files[fp]['first_text'] = entry.original_text[:40].replace('\n', ' ')
+
+ toc = []
+ for i, (fp, info) in enumerate(sorted(files.items()), 1):
+ toc.append({
+ 'index': i,
+ 'href': fp,
+ 'title': info['first_text'] or f"File {i}",
+ 'paragraphs': info['count'],
+ 'chars': info['total_chars']
+ })
+ return toc
+
+
+def show_toc(manager: ManifestManager):
+ """Display TOC with chapter numbers."""
+ toc = build_toc_from_manifest(manager)
+
+ print("\n" + "=" * 70)
+ print("章节目录 (Table of Contents)")
+ print("=" * 70)
+ print(f"{'#':>3} | {'段落':>5} | {'字符':>6} | 章节标题")
+ print("-" * 70)
+
+ for item in toc:
+ title = item['title'][:45] if len(item['title']) > 45 else item['title']
+ print(f"{item['index']:3d} | {item['paragraphs']:5d} | {item['chars']:6d} | {title}")
+
+ print("-" * 70)
+ print(f"共 {len(toc)} 个章节")
+ print("\n用法: python scripts/translate_chapter.py --chapter <编号>")
+ print("示例: python scripts/translate_chapter.py --chapter 5")
+
+
+def get_chapter_entries(manager: ManifestManager, chapter_index: int) -> tuple:
+ """Get entries for a specific chapter by index."""
+ toc = build_toc_from_manifest(manager)
+
+ if chapter_index < 1 or chapter_index > len(toc):
+ print(f"错误: 章节编号 {chapter_index} 无效 (范围: 1-{len(toc)})")
+ return None, None
+
+ chapter = toc[chapter_index - 1]
+ href = chapter['href']
+
+ entries = [e for e in manager.entries if e.file_path == href]
+ return chapter, entries
+
+
+def create_char_based_chunks(entries: list, chunk_size: int = CHUNK_SIZE_CHARS) -> list:
+ """
+ Create chunks based on character count (~5000 chars each).
+ Returns list of entry lists.
+ """
+ chunks = []
+ current_chunk = []
+ current_size = 0
+
+ for entry in entries:
+ text_len = len(entry.original_text)
+
+ # If adding this entry exceeds limit and we have content, start new chunk
+ if current_size + text_len > chunk_size and current_chunk:
+ chunks.append(current_chunk)
+ current_chunk = []
+ current_size = 0
+
+ current_chunk.append(entry)
+ current_size += text_len
+
+ if current_chunk:
+ chunks.append(current_chunk)
+
+ return chunks
+
+
+async def translate_chapter(chapter: dict, entries: list, manager: ManifestManager,
+ llm_client: LLMClient, profile, test_mode: bool = False):
+ """Translate a complete chapter."""
+
+ print(f"\n开始翻译章节 #{chapter['index']}: {chapter['title'][:40]}...")
+ print(f" 文件: {chapter['href']}")
+ print(f" 总段落: {len(entries)}")
+
+ # Filter untranslated
+ untranslated = [e for e in entries if not e.translated_text]
+ print(f" 待翻译: {len(untranslated)}")
+
+ if not untranslated:
+ print(" ✅ 该章节已全部翻译!")
+ return
+
+ # Create character-based chunks
+ chunks = create_char_based_chunks(untranslated)
+ print(f" 分块: {len(chunks)} 个 Chunk (约{CHUNK_SIZE_CHARS}字符/块)")
+
+ if test_mode:
+ print(" [测试模式] 只翻译前2个 Chunk")
+ chunks = chunks[:2]
+
+ # Show chunk stats
+ for i, chunk in enumerate(chunks, 1):
+ total_chars = sum(len(e.original_text) for e in chunk)
+ print(f" Chunk {i}: {len(chunk)} 段落, {total_chars} 字符")
+
+ # Translate
+ restorer = FormatRestorer()
+ total_success = 0
+ total_failed = 0
+
+ for i, chunk in enumerate(chunks, 1):
+ chunk_chars = sum(len(e.original_text) for e in chunk)
+ print(f"\n 翻译 Chunk {i}/{len(chunks)} ({len(chunk)} 段, {chunk_chars} 字符)...")
+
+ try:
+ results = await llm_client.translate_chunk(
+ chunk,
+ instruction=profile.style_guide if hasattr(profile, 'style_guide') else None,
+ mode="bilingual"
+ )
+
+ # Apply results
+ chunk_success = 0
+ chunk_failed = 0
+
+ for entry in chunk:
+ if entry.entry_id in results:
+ translated = results[entry.entry_id]
+ entry.translated_text = translated
+
+ # Verify placeholder restoration
+ if entry.placeholders:
+ _, success = restorer.restore(translated, entry.placeholders)
+ if success:
+ chunk_success += 1
+ else:
+ chunk_failed += 1
+ print(f" ⚠️ 占位符还原警告: {entry.entry_id[-30:]}")
+ else:
+ chunk_success += 1
+ else:
+ chunk_failed += 1
+ print(f" ❌ 缺失: {entry.entry_id[-30:]}")
+
+ total_success += chunk_success
+ total_failed += chunk_failed
+ print(f" ✓ 成功: {chunk_success}, 失败: {chunk_failed}")
+
+ # Save after each chunk
+ manager.save()
+
+ except Exception as e:
+ print(f" ❌ Chunk {i} 翻译失败: {e}")
+ total_failed += len(chunk)
+
+ print(f"\n翻译完成:")
+ print(f" ✅ 成功: {total_success}")
+ print(f" ❌ 失败: {total_failed}")
+
+ # Show sample results
+ print(f"\n翻译样例 (前3段):")
+ print("-" * 60)
+ translated_entries = [e for e in entries if e.translated_text][:3]
+ for entry in translated_entries:
+ orig = entry.original_text[:40].replace('\n', ' ')
+ trans = entry.translated_text[:40].replace('\n', ' ') if entry.translated_text else "(无)"
+ print(f" 原: {orig}...")
+ print(f" 译: {trans}...")
+ print()
+
+
+async def main():
+ parser = argparse.ArgumentParser(description="翻译指定章节")
+ parser.add_argument("--show-toc", action="store_true", help="显示章节目录")
+ parser.add_argument("--chapter", "-c", type=int, help="章节编号 (从1开始)")
+ parser.add_argument("--test", "-t", action="store_true", help="测试模式 (只翻译前2个chunk)")
+ args = parser.parse_args()
+
+ if not MANIFEST_PATH.exists():
+ print(f"错误: Manifest 不存在: {MANIFEST_PATH}")
+ print("请先运行主管道生成 manifest。")
+ return
+
+ # Load manifest
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+ print(f"已加载 manifest: {len(manager.entries)} 条目")
+
+ # Show TOC
+ if args.show_toc or not args.chapter:
+ show_toc(manager)
+ return
+
+ if not API_KEY:
+ print("错误: V3_API_KEY 未设置")
+ return
+
+ # Get chapter entries
+ chapter, entries = get_chapter_entries(manager, args.chapter)
+ if not chapter:
+ return
+
+ # Initialize LLM client
+ from src.utils import ensure_directory
+ ensure_directory(CHUNK_DIR)
+
+ llm_client = LLMClient(
+ api_key=API_KEY,
+ base_url=BASE_URL,
+ model=MODEL,
+ extra_headers=EXTRA_HEADERS,
+ chunk_dir=CHUNK_DIR
+ )
+
+ try:
+ # Generate profile
+ print("\n生成书籍 Profile... (Skipping for debug)")
+ # profiler = BookProfiler(llm_client)
+ # profile = await profiler.analyze(manager.entries)
+ # print(f" 风格: {profile.style_guide[:80] if profile.style_guide else '(无)'}...")
+
+ class DummyProfile:
+ style_guide = "Keep technical terms. Translate accurately."
+ profile = DummyProfile()
+
+ # Translate chapter
+ await translate_chapter(chapter, entries, manager, llm_client, profile, args.test)
+
+ print(f"\n✅ Manifest 已保存: {MANIFEST_PATH}")
+ print(f"✅ Chunk 文件保存在: {CHUNK_DIR}")
+
+ finally:
+ await llm_client.close()
+
+
+if __name__ == "__main__":
+ print("DEBUG: Script started execution")
+ try:
+ asyncio.run(main())
+ print("DEBUG: Script finished execution")
+ except Exception as e:
+ import traceback
+ traceback.print_exc()
+ print(f"CRITICAL ERROR: {e}")
diff --git a/archive/v0.10/scripts/verify_toc_fix.py b/archive/v0.10/scripts/verify_toc_fix.py
new file mode 100644
index 0000000..47deb9a
--- /dev/null
+++ b/archive/v0.10/scripts/verify_toc_fix.py
@@ -0,0 +1,47 @@
+import sys
+from pathlib import Path
+import traceback
+
+# Add src to path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from src.epub_cleaner import EpubCleaner
+from src.bilingual_builder import BilingualBuilder
+from src.utils import setup_logger
+from src.data_model import BookStructure
+
+logger = setup_logger("verify_toc")
+
+def verify_fix():
+ INPUT_EPUB = Path("input/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao).epub")
+ OUTPUT_EPUB = Path("output/verify_toc.epub")
+ WORK_DIR = Path("tmp/verify_toc")
+
+ if not INPUT_EPUB.exists():
+ logger.error(f"Input not found: {INPUT_EPUB}")
+ return
+
+ # Clean
+ logger.info("Step 1: Cleaning EPUB...")
+ cleaner = EpubCleaner(INPUT_EPUB, WORK_DIR)
+ json_path = cleaner.clean()
+
+ # Load structure
+ with open(json_path, 'r') as f:
+ structure = BookStructure.model_validate_json(f.read())
+
+ # Build
+ logger.info("Step 2: Building EPUB with TOC preservation...")
+ builder = BilingualBuilder(WORK_DIR, original_epub_path=INPUT_EPUB)
+ builder.build(structure, OUTPUT_EPUB)
+
+ logger.info(f"Step 3: EPUB generated at {OUTPUT_EPUB}")
+
+if __name__ == "__main__":
+ print("Starting verification script...")
+ try:
+ verify_fix()
+ print("Verification script finished.")
+ except Exception as e:
+ traceback.print_exc()
+ print(f"CRITICAL ERROR: {e}")
diff --git a/archive/v0.10/src/__init__.py b/archive/v0.10/src/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/archive/v0.10/src/assembly/backfiller.py b/archive/v0.10/src/assembly/backfiller.py
new file mode 100644
index 0000000..275f0af
--- /dev/null
+++ b/archive/v0.10/src/assembly/backfiller.py
@@ -0,0 +1,96 @@
+from pathlib import Path
+from typing import List, Dict, Optional
+from bs4 import BeautifulSoup
+
+from src.common.data_model import BookStructure, ManifestEntry
+from src.assembly.format_restorer import FormatRestorer
+from src.common.utils import setup_logger
+
+logger = setup_logger("backfill_engine")
+
+class BackfillEngine:
+ """
+ Applies translations back to the BookStructure.
+ """
+
+ def __init__(self):
+ self.restorer = FormatRestorer()
+
+ def backfill(self, structure: BookStructure, manifest_entries: List[ManifestEntry], mode: str = "bilingual") -> BookStructure:
+ """
+ Modifies the BookStructure in-place with translations.
+
+ Args:
+ structure: The BookStructure (from book_structure.json).
+ manifest_entries: List of translations.
+ mode: 'bilingual' or 'target_only'.
+ """
+ logger.info(f"Backfilling with mode: {mode}")
+
+ # Index manifest by file and element ID for faster lookup
+ # Map: file_path -> element_id -> ManifestEntry
+ manifest_map: Dict[str, Dict[str, ManifestEntry]] = {}
+ for entry in manifest_entries:
+ if not entry.translated_text:
+ continue # Skip untranslated entries
+
+ if entry.file_path not in manifest_map:
+ manifest_map[entry.file_path] = {}
+ manifest_map[entry.file_path][entry.element_id] = entry
+
+ # Iterate resources in structure
+ for item_id, resource in structure.resources.items():
+ if resource.media_type != "application/xhtml+xml" or resource.href not in manifest_map:
+ continue
+
+ file_entries = manifest_map[resource.href]
+ if not file_entries:
+ continue
+
+ logger.debug(f"Processing {resource.href} with {len(file_entries)} translations")
+
+ soup = BeautifulSoup(resource.content, 'html.parser')
+ modified = False
+
+ for element_id, entry in file_entries.items():
+ element = soup.find(id=element_id)
+ if not element:
+ logger.warning(f"Element {element_id} not found in {resource.href}")
+ continue
+
+ # Restore formatting
+ restored_html, _ = self.restorer.restore(entry.translated_text, entry.placeholders)
+
+ # Create translated tag
+ new_tag = soup.new_tag(element.name)
+ # Parse restored HTML to get content nodes
+ inner_soup = BeautifulSoup(restored_html, 'html.parser')
+ if inner_soup.body:
+ for child in list(inner_soup.body.children):
+ new_tag.append(child)
+ else:
+ for child in list(inner_soup.children):
+ new_tag.append(child)
+
+ # Copy classes and add 'translation'
+ classes = element.get('class', [])
+ if isinstance(classes, str):
+ classes = classes.split()
+ new_tag['class'] = classes + ['translation']
+
+ # Copy style
+ style = element.get('style')
+ if style:
+ new_tag['style'] = style
+
+ if mode == "bilingual":
+ element.insert_after(new_tag)
+ else:
+ element.replace_with(new_tag)
+
+ modified = True
+
+ if modified:
+ resource.content = str(soup)
+
+ return structure
diff --git a/archive/v0.10/src/assembly/builder.py b/archive/v0.10/src/assembly/builder.py
new file mode 100644
index 0000000..b6c3ce7
--- /dev/null
+++ b/archive/v0.10/src/assembly/builder.py
@@ -0,0 +1,300 @@
+import shutil
+import uuid
+from pathlib import Path
+from ebooklib import epub
+from src.common.data_model import BookStructure
+from src.common.utils import setup_logger, ensure_directory
+
+logger = setup_logger("bilingual_builder")
+
+class BilingualBuilder:
+ """
+ Assembles the final EPUB from BookStructure.
+
+ Preserves the original TOC structure by reading it from the original EPUB.
+ """
+
+ def __init__(self, work_dir: Path, original_epub_path: Path = None):
+ self.work_dir = work_dir
+ self.assets_dir = work_dir / "assets"
+ self.original_epub_path = original_epub_path
+ self._original_book = None
+
+ def _load_original_book(self):
+ """Lazy load original book for TOC extraction."""
+ if self._original_book is None and self.original_epub_path and self.original_epub_path.exists():
+ try:
+ self._original_book = epub.read_epub(str(self.original_epub_path))
+ logger.debug(f"Loaded original EPUB for TOC: {self.original_epub_path}")
+ except Exception as e:
+ logger.warning(f"Failed to load original EPUB: {e}")
+ return self._original_book
+
+ def _sanitize_toc(self, toc):
+ """
+ Ensure all TOC nodes have IDs (for ebooklib compatibility).
+ From v0.08 bilingual_builder.py
+ """
+ result = []
+ for item in toc:
+ if isinstance(item, (epub.Link, epub.Section)):
+ if not getattr(item, 'uid', None):
+ item.uid = f"navPoint-{uuid.uuid4().hex[:8]}"
+ result.append(item)
+ elif isinstance(item, tuple) and len(item) == 2:
+ # Handle (Section, [children]) structure
+ section, children = item
+ if isinstance(section, (epub.Link, epub.Section)):
+ if not getattr(section, 'uid', None):
+ section.uid = f"navPoint-{uuid.uuid4().hex[:8]}"
+ sanitized_children = self._sanitize_toc(children)
+ result.append((section, sanitized_children))
+ else:
+ result.append(item)
+ return result
+
+ def _validate_and_fix_toc(self, toc, book_items):
+ """
+ Recursively validate and fix TOC links.
+ Removes nodes with broken links that cannot be fixed.
+ """
+ fixed_toc = []
+ for item in toc:
+ if isinstance(item, (epub.Link, epub.Section)):
+ # Check href
+ href = getattr(item, 'href', '')
+ if href:
+ # Remove anchor for check
+ clean_href = href.split('#')[0]
+ # Check if item exists in book (by file_name)
+ found = False
+ for existing_item in book_items.values():
+ if existing_item.file_name == clean_href:
+ found = True
+ break
+ if clean_href.endswith(existing_item.file_name) or existing_item.file_name.endswith(clean_href):
+ item.href = existing_item.file_name
+ found = True
+ break
+
+ if not found:
+ if 'c0.xhtml' in clean_href:
+ for existing_item in book_items.values():
+ if 'titlepage' in existing_item.file_name or 'cover' in existing_item.file_name.lower():
+ if existing_item.media_type == "application/xhtml+xml":
+ logger.info(f"Fixed TOC link: {href} -> {existing_item.file_name}")
+ item.href = existing_item.file_name
+ found = True
+ break
+
+ if not found:
+ logger.warning(f"Removing broken TOC link: {href}")
+ continue
+
+ if isinstance(item, tuple) and len(item) == 2:
+ section, children = item
+ fixed_children = self._validate_and_fix_toc(children, book_items)
+ fixed_toc.append((section, fixed_children))
+ else:
+ fixed_toc.append(item)
+
+ return fixed_toc
+
+ def build(self, structure: BookStructure, output_path: Path) -> Path:
+ """
+ Builds the EPUB file.
+ Returns the path to the generated EPUB.
+ """
+ logger.info(f"Building final EPUB: {output_path}")
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+
+ book = epub.EpubBook()
+
+ # 1. Metadata
+ book.set_identifier(structure.metadata.identifier or f"uuid-{uuid.uuid4().hex[:12]}")
+ book.set_title(structure.metadata.title)
+ book.set_language(structure.metadata.language)
+ book.add_author(structure.metadata.author)
+
+ # 2. Copy TOC from original EPUB if available
+ original_book = self._load_original_book()
+ if original_book and hasattr(original_book, 'toc') and original_book.toc:
+ book.toc = self._sanitize_toc(original_book.toc)
+ logger.info("Copied TOC structure from original EPUB")
+
+
+
+ # 3. Add Resources - First pass: Collect CSS items
+ items_map = {} # id -> epub_item
+ css_items = [] # List of CSS EpubItem for linking
+ html_items = [] # List of (item_id, EpubHtml) tuples
+
+ for item_id, resource in structure.resources.items():
+ # Skip NCX - we'll handle it separately
+ if resource.media_type == "application/x-dtbncx+xml":
+ continue
+
+ if resource.media_type == "application/xhtml+xml":
+ # HTML Item - create but don't add yet (need to add CSS links)
+ item = epub.EpubHtml(
+ uid=item_id,
+ file_name=resource.href,
+ media_type=resource.media_type,
+ content=resource.content.encode('utf-8')
+ )
+ html_items.append((item_id, item))
+ elif resource.file_path:
+ # Binary/Asset Item
+ asset_full_path = self.work_dir / resource.file_path
+
+ if not asset_full_path.exists():
+ logger.warning(f"Asset missing: {asset_full_path}")
+ continue
+
+ with open(asset_full_path, 'rb') as f:
+ content = f.read()
+
+ # Check if this is the cover image
+ if structure.metadata.cover_image_id == item_id:
+ logger.info(f"Setting cover image: {item_id}")
+ # set_cover automatically creates the item and sets metadata
+ book.set_cover(resource.href, content)
+ # We still need to track it in items_map for spine/TOC references if needed?
+ # ebooklib set_cover creates an item withuid='cover-img' (default) or similar?
+ # Actually set_cover logic:
+ # def set_cover(self, file_name, content, create_page=True):
+ # c = EpubCover(file_name=file_name)
+ # c.content = content
+ # self.add_item(c)
+ # self.add_metadata(None, 'meta', '', {'name': 'cover', 'content': 'cover-img'})
+
+ # Be careful: ebooklib might change the ID.
+ # If we use set_cover, we should verify how it affects references.
+ # However, for the cover image specifically, usually it's referenced by the cover page
+ # (which set_cover creates if create_page=True).
+ # If create_page=False, we just get metadata.
+
+ # Let's use set_cover with create_page=False (since we likely preserved the cover page HTML)
+ # and rely on existing HTML to point to it?
+ # Or let ebooklib handle it.
+ # Most translator users want the cover to just work.
+
+ book.set_cover(resource.href, content, create_page=False)
+
+ # We also need to add it to items_map so we don't try to add it again
+ # But set_cover adds it to the book.
+ # We need to find the item added by set_cover to put in items_map
+ # standard ebooklib set_cover adds item with id derived or fixed?
+ # Actually, if we use set_cover, we might introduce a duplicate if we're not careful about IDs.
+
+ # Simplified approach:
+ # 1. Add as normal EpubImage
+ # 2. Add metadata manually pointing to it
+
+ item = epub.EpubImage(
+ uid=item_id,
+ file_name=resource.href,
+ media_type=resource.media_type,
+ content=content
+ )
+ book.add_item(item)
+ book.add_metadata(None, 'meta', item_id, {'name': 'cover', 'content': item_id})
+ items_map[item_id] = item
+ continue
+
+ if "image" in resource.media_type:
+ item = epub.EpubImage(
+ uid=item_id,
+ file_name=resource.href,
+ media_type=resource.media_type,
+ content=content
+ )
+ else:
+ item = epub.EpubItem(
+ uid=item_id,
+ file_name=resource.href,
+ media_type=resource.media_type,
+ content=content
+ )
+ # Track CSS items
+ if resource.media_type == "text/css":
+ css_items.append(item)
+
+ book.add_item(item)
+ items_map[item_id] = item
+ else:
+ logger.warning(f"Skipping resource {item_id}: No content or file path.")
+ continue
+
+ # 4. Add HTML items with CSS links
+ for item_id, item in html_items:
+ html_dir = Path(item.file_name).parent
+ for css_item in css_items:
+ css_path = Path(css_item.file_name)
+ # Calculate relative path from HTML directory to CSS file
+ try:
+ relative_css_path = Path(css_path).relative_to(html_dir)
+ except ValueError:
+ # Not a subpath, calculate full relative
+ # Go up from html_dir, then down to css_path
+ up_count = len(html_dir.parts)
+ relative_css_path = Path("/".join([".."] * up_count)) / css_path
+
+ item.add_link(href=str(relative_css_path), rel='stylesheet', type='text/css')
+ book.add_item(item)
+ items_map[item_id] = item
+
+ # 5. Copy missing items from original EPUB (cover, etc.)
+ # This ensures TOC links don't break
+ if original_book:
+ added_hrefs = {item.file_name for item in items_map.values() if hasattr(item, 'file_name')}
+
+ for orig_item in original_book.get_items():
+ orig_name = orig_item.get_name()
+ if orig_name not in added_hrefs:
+ # Skip NCX and NAV - we generate these
+ if 'ncx' in orig_name.lower() or orig_name.endswith('nav.xhtml'):
+ continue
+
+ # Copy the item directly
+ try:
+ book.add_item(orig_item)
+ items_map[orig_item.id] = orig_item
+ logger.debug(f"Copied missing item from original: {orig_name}")
+ except Exception as e:
+ logger.warning(f"Failed to copy item {orig_name}: {e}")
+
+ # 6. Spine
+ book.spine = []
+ for item_id in structure.spine:
+ if item_id in items_map:
+ book.spine.append(items_map[item_id])
+ else:
+ logger.warning(f"Spine item {item_id} not found in resources.")
+
+ # Add missing spine items from original
+ if original_book:
+ for spine_id, _ in original_book.spine:
+ if spine_id not in [i.id for i in book.spine]:
+ orig_item = original_book.get_item_with_id(spine_id)
+ if orig_item and orig_item.id in items_map:
+ book.spine.append(items_map[orig_item.id])
+
+ # Validate and fix TOC
+ if original_book and hasattr(book, 'toc') and book.toc:
+ try:
+ book.toc = self._validate_and_fix_toc(book.toc, items_map)
+ logger.info("Validated and fixed TOC links")
+ except Exception as e:
+ logger.error(f"Failed to validate TOC: {e}")
+
+ # 7. Navigation - NCX and Nav
+ book.add_item(epub.EpubNcx())
+ book.add_item(epub.EpubNav())
+
+ # 7. Write
+ epub.write_epub(str(output_path), book)
+ logger.info(f"EPUB created successfully at {output_path}")
+
+ return output_path
+
diff --git a/archive/v0.10/src/assembly/format_restorer.py b/archive/v0.10/src/assembly/format_restorer.py
new file mode 100644
index 0000000..7ae111c
--- /dev/null
+++ b/archive/v0.10/src/assembly/format_restorer.py
@@ -0,0 +1,70 @@
+import re
+from typing import Dict, Tuple, List, Optional
+from loguru import logger
+from src.common.utils import setup_logger
+
+logger = setup_logger("format_restorer")
+
+class FormatRestorer:
+ """
+ Restores HTML formatting from placeholders.
+ """
+
+ PLACEHOLDER_REGEX = re.compile(r'φ(/?\d+)φ')
+
+ def restore(self, text_with_placeholders: str, placeholder_map: Dict[str, str]) -> Tuple[str, bool]:
+ """
+ Restores HTML from text with placeholders.
+ Returns (restored_html, success).
+ """
+ if not placeholder_map:
+ return text_with_placeholders or "", True
+
+ if not text_with_placeholders:
+ prefix = placeholder_map.get("_prefix", "")
+ suffix = placeholder_map.get("_suffix", "")
+ return prefix + suffix, True
+
+ prefix = placeholder_map.get("_prefix", "")
+ suffix = placeholder_map.get("_suffix", "")
+
+ inner_map = {k: v for k, v in placeholder_map.items() if not k.startswith("_")}
+
+ found_ids = set(self.PLACEHOLDER_REGEX.findall(text_with_placeholders))
+ expected_ids = set(inner_map.keys())
+
+ success = True
+ missing_ids = expected_ids - found_ids
+ if missing_ids:
+ logger.warning(f"Restoration warning: missing placeholders {missing_ids}")
+ success = False
+
+ unknown_ids = found_ids - expected_ids
+ if unknown_ids:
+ real_unknowns = set()
+ for pid in unknown_ids:
+ if pid.startswith('/') and pid[1:] in expected_ids:
+ continue
+ real_unknowns.add(pid)
+
+ if real_unknowns:
+ logger.warning(f"Restoration warning: unknown placeholders {real_unknowns}")
+ success = False
+
+ def replace_match(match):
+ pid = match.group(1)
+ if pid in inner_map:
+ return inner_map[pid]
+ else:
+ return ""
+
+ try:
+ restored_inner = self.PLACEHOLDER_REGEX.sub(replace_match, text_with_placeholders)
+ restored_html = prefix + restored_inner + suffix
+ return restored_html, success
+ except Exception as e:
+ logger.error(f"Restoration failed: {e}")
+ return prefix + self._strip_placeholders(text_with_placeholders) + suffix, False
+
+ def _strip_placeholders(self, text: str) -> str:
+ return self.PLACEHOLDER_REGEX.sub("", text)
diff --git a/archive/v0.10/src/common/config.py b/archive/v0.10/src/common/config.py
new file mode 100644
index 0000000..d843d95
--- /dev/null
+++ b/archive/v0.10/src/common/config.py
@@ -0,0 +1,71 @@
+import os
+import yaml
+from pathlib import Path
+from typing import Dict, Any
+from dotenv import load_dotenv
+
+from src.common.utils import setup_logger
+
+logger = setup_logger("config_loader")
+
+class ConfigLoader:
+ def __init__(self, config_path: str = "config/config.yaml"):
+ # Resolve absolute path relative to project root if needed,
+ # but usually running from root so relative is fine.
+ self.config_path = Path(config_path)
+ self.config = self._load_defaults()
+
+ def _load_defaults(self) -> Dict[str, Any]:
+ return {
+ "llm": {
+ "model": "gpt-3.5-turbo",
+ "base_url": "https://api.openai.com/v1",
+ "timeout": 60,
+ "requests_per_minute": 60,
+ "concurrent_requests": 5
+ },
+ "translation": {
+ "chunk_size": 4000
+ }
+ }
+
+ def load_config(self) -> Dict[str, Any]:
+ # 1. Load YAML
+ if self.config_path.exists():
+ try:
+ with open(self.config_path, 'r', encoding='utf-8') as f:
+ file_config = yaml.safe_load(f)
+ if file_config:
+ self._deep_update(self.config, file_config)
+ logger.info(f"Loaded config from {self.config_path}")
+ except Exception as e:
+ logger.error(f"Failed to load config file: {e}")
+ else:
+ logger.warning(f"Config file not found at {self.config_path}, using defaults")
+
+ # 2. Env overrides (Priority: Env > Config File > Defaults)
+ load_dotenv()
+
+ # API Key is mandatory from Env (security best practice)
+ api_key = os.getenv("OPENAI_API_KEY")
+ if api_key:
+ self.config["llm"]["api_key"] = api_key
+
+ # Base URL override
+ base_url = os.getenv("OPENAI_BASE_URL")
+ if base_url:
+ self.config["llm"]["base_url"] = base_url
+
+ return self.config
+
+ def _deep_update(self, d, u):
+ for k, v in u.items():
+ if isinstance(v, dict):
+ d[k] = self._deep_update(d.get(k, {}), v)
+ else:
+ d[k] = v
+ return d
+
+def load_global_config(path: str = "config/config.yaml") -> Dict[str, Any]:
+ loader = ConfigLoader(path)
+ return loader.load_config()
diff --git a/archive/v0.10/src/common/data_model.py b/archive/v0.10/src/common/data_model.py
new file mode 100644
index 0000000..4ae9033
--- /dev/null
+++ b/archive/v0.10/src/common/data_model.py
@@ -0,0 +1,52 @@
+from pathlib import Path
+from typing import List, Optional, Dict, Any
+from pydantic import BaseModel, Field
+
+class ManifestEntry(BaseModel):
+ """Represents a single translatable unit."""
+ entry_id: str = Field(..., description="Global unique ID (e.g. file.html#paragraph_id)")
+ file_path: str = Field(..., description="Internal path in EPUB")
+ element_id: str = Field(..., description="HTML ID (e.g. uuid-1234)")
+ original_text: str
+ placeholders: Dict[str, str] = Field(default_factory=dict)
+ translated_text: Optional[str] = None
+ context: Optional[str] = None
+
+class BookMetaData(BaseModel):
+ title: str = "Unknown Title"
+ author: str = "Unknown Author"
+ language: str = "en"
+ identifier: str = ""
+ cover_image_id: Optional[str] = None
+
+class ResourceItem(BaseModel):
+ href: str
+ media_type: str
+ content: Optional[str] = None # For text/html
+ file_path: Optional[str] = None # For binary/assets (relative to assets dir)
+ properties: Optional[str] = None
+
+class BookStructure(BaseModel):
+ metadata: BookMetaData
+ spine: List[str] = Field(default_factory=list, description="Ordered list of item IDs in spine")
+ resources: Dict[str, ResourceItem] = Field(default_factory=dict, description="Map of item_id to ResourceItem")
+
+ @classmethod
+ def load(cls, path: Path) -> "BookStructure":
+ """Load BookStructure from JSON file."""
+ with open(path, 'r', encoding='utf-8') as f:
+ return cls.model_validate_json(f.read())
+
+ def save(self, path: Path):
+ """Save BookStructure to JSON file."""
+ with open(path, 'w', encoding='utf-8') as f:
+ f.write(self.model_dump_json(indent=2))
+
+class BookProfile(BaseModel):
+ """Represents the profile of the book."""
+ title: str
+ author: str
+ genre: str = "General"
+ keywords: List[str] = Field(default_factory=list)
+ style_guide: str = ""
+
diff --git a/archive/v0.10/src/common/exceptions.py b/archive/v0.10/src/common/exceptions.py
new file mode 100644
index 0000000..2ca7b05
--- /dev/null
+++ b/archive/v0.10/src/common/exceptions.py
@@ -0,0 +1,18 @@
+class EpubTranslatorError(Exception):
+ """Base exception for Epub Translator."""
+ pass
+
+class CleaningError(EpubTranslatorError):
+ pass
+
+class ExtractionError(EpubTranslatorError):
+ pass
+
+class TranslationError(EpubTranslatorError):
+ pass
+
+class RestorationError(EpubTranslatorError):
+ pass
+
+class BuildError(EpubTranslatorError):
+ pass
diff --git a/archive/v0.10/src/common/utils.py b/archive/v0.10/src/common/utils.py
new file mode 100644
index 0000000..e1cd9b3
--- /dev/null
+++ b/archive/v0.10/src/common/utils.py
@@ -0,0 +1,25 @@
+import logging
+from pathlib import Path
+
+def setup_logger(name: str, log_file: Path = None, level=logging.INFO):
+ """Sets up a logger with the given name."""
+ logger = logging.getLogger(name)
+ logger.setLevel(level)
+
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+ if log_file:
+ file_handler = logging.FileHandler(log_file)
+ file_handler.setFormatter(formatter)
+ logger.addHandler(file_handler)
+
+ stream_handler = logging.StreamHandler()
+ stream_handler.setFormatter(formatter)
+ logger.addHandler(stream_handler)
+
+ return logger
+
+def ensure_directory(path: Path):
+ """Ensures a directory exists."""
+ if not path.exists():
+ path.mkdir(parents=True, exist_ok=True)
diff --git a/archive/v0.10/src/preprocessing/epub_cleaner.py b/archive/v0.10/src/preprocessing/epub_cleaner.py
new file mode 100644
index 0000000..ff428f0
--- /dev/null
+++ b/archive/v0.10/src/preprocessing/epub_cleaner.py
@@ -0,0 +1,184 @@
+import shutil
+import uuid
+import zipfile
+from pathlib import Path
+from typing import Dict, Set
+
+import ebooklib
+from bs4 import BeautifulSoup, Tag
+from ebooklib import epub
+
+from src.common.data_model import BookStructure, BookMetaData, ResourceItem
+from src.common.exceptions import CleaningError
+from src.common.utils import setup_logger, ensure_directory
+
+logger = setup_logger("epub_cleaner")
+
+class EpubCleaner:
+ def __init__(self, input_path: Path, work_dir: Path):
+ self.input_path = input_path
+ self.work_dir = work_dir
+ self.assets_dir = work_dir / "assets"
+ self.json_path = work_dir / "book_structure.json"
+
+ def clean(self) -> Path:
+ """
+ Cleans the input EPUB and generates book_structure.json.
+ Returns the path to the JSON file.
+ """
+ logger.info(f"Starting cleanup for {self.input_path}")
+ ensure_directory(self.work_dir)
+ ensure_directory(self.assets_dir)
+
+ try:
+ book = epub.read_epub(self.input_path)
+
+ # 1. Extract Metadata
+ metadata = self._extract_metadata(book)
+
+ # 2. Process Resources
+ resources = {}
+ # Use zipfile for binary extraction to avoid ebooklib's memory overhead/decoding issues
+ with zipfile.ZipFile(self.input_path, 'r') as zf:
+ # Map ebooklib items to zip entries isn't straightforward directly via name
+ # So we iterate ebooklib items and assume standard structure or handle content bytes
+
+ for item in book.get_items():
+ item_id = item.get_id()
+ file_name = item.get_name()
+ media_type = item.get_type() # ebooklib constant
+
+ if media_type == ebooklib.ITEM_DOCUMENT:
+ # Clean HTML
+ content_str = item.get_content().decode('utf-8')
+ cleaned_content = self._clean_html(content_str, file_name)
+
+ resources[item_id] = ResourceItem(
+ href=file_name,
+ media_type="application/xhtml+xml",
+ content=cleaned_content
+ )
+ elif media_type in (ebooklib.ITEM_IMAGE, ebooklib.ITEM_STYLE, ebooklib.ITEM_FONT, ebooklib.ITEM_COVER):
+ # Save asset
+
+ # Check if it is a cover
+ if media_type == ebooklib.ITEM_COVER:
+ metadata.cover_image_id = item_id
+ logger.info(f"Found cover image: {item_id} ({file_name})")
+
+ # Preserve directory structure to avoid collisions
+ asset_path = self.assets_dir / file_name
+ ensure_directory(asset_path.parent)
+
+ # Ebooklib might change filenames, safer to use item.get_content()
+ with open(asset_path, "wb") as f:
+ f.write(item.get_content())
+
+ resources[item_id] = ResourceItem(
+ href=file_name,
+ media_type=self._get_media_type_str(item),
+ file_path=str(asset_path.relative_to(self.work_dir))
+ )
+ elif media_type == ebooklib.ITEM_NAVIGATION:
+ # NCX or NAV document - preserve for TOC
+ content_bytes = item.get_content()
+ asset_path = self.assets_dir / file_name
+ ensure_directory(asset_path.parent)
+
+ with open(asset_path, "wb") as f:
+ f.write(content_bytes)
+
+ # Determine media type
+ if file_name.endswith('.ncx'):
+ mt = "application/x-dtbncx+xml"
+ else:
+ mt = "application/xhtml+xml"
+
+ resources[item_id] = ResourceItem(
+ href=file_name,
+ media_type=mt,
+ file_path=str(asset_path.relative_to(self.work_dir))
+ )
+ logger.debug(f"Preserved navigation: {file_name}")
+ else:
+ # Skip other items (scripts, etc.)
+ pass
+
+ # 3. Extract Spine
+ spine_ids = [item[0] for item in book.spine]
+
+ # 4. Construct Structure
+ structure = BookStructure(
+ metadata=metadata,
+ spine=spine_ids,
+ resources=resources
+ )
+
+ # 5. Serialize
+ with open(self.json_path, "w", encoding="utf-8") as f:
+ f.write(structure.model_dump_json(indent=2))
+
+ logger.info(f"Cleanup finished. Structure saved to {self.json_path}")
+ return self.json_path
+
+ except Exception as e:
+ logger.error(f"Cleaning failed: {e}")
+ raise CleaningError(f"Failed to clean EPUB: {e}") from e
+
+ def _extract_metadata(self, book: epub.EpubBook) -> BookMetaData:
+ title = book.get_metadata('DC', 'title')[0][0] if book.get_metadata('DC', 'title') else "Unknown"
+ author = book.get_metadata('DC', 'creator')[0][0] if book.get_metadata('DC', 'creator') else "Unknown"
+ lang = book.get_metadata('DC', 'language')[0][0] if book.get_metadata('DC', 'language') else "en"
+ ident = book.get_metadata('DC', 'identifier')[0][0] if book.get_metadata('DC', 'identifier') else ""
+
+ return BookMetaData(
+ title=str(title),
+ author=str(author),
+ language=str(lang),
+ identifier=str(ident)
+ )
+
+ def _get_media_type_str(self, item) -> str:
+ # Helper to map ebooklib type to mime string if needed
+ # ebooklib doesn't expose easy MIME string for all types directly on item object sometimes
+ if hasattr(item, 'media_type'):
+ return item.media_type
+ return "application/octet-stream"
+
+ def _clean_html(self, content: str, filename: str) -> str:
+ soup = BeautifulSoup(content, 'html.parser')
+
+ # 1. Provide IDs for structural/translatable elements
+ self._ensure_element_ids(soup)
+
+ # 2. Flatten divs (Disable to prevent style loss)
+ # self._flatten_divs(soup)
+
+ return str(soup)
+
+ def _ensure_element_ids(self, soup: BeautifulSoup):
+ """
+ Injects UUIDs into p, h1-h6, li tags if they don't have an ID.
+ This provides the anchor for translation backfilling.
+ """
+ targets = soup.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'])
+ for tag in targets:
+ if not tag.has_attr('id'):
+ tag['id'] = f"uuid-{uuid.uuid4()}"
+
+ def _flatten_divs(self, soup: BeautifulSoup):
+ """
+ Converts generic divs containing only inline text/styles to p tags.
+ Recursive strategies can be complex, sticking to simple heuristic from archive.
+ """
+ inline_tags = {'span', 'em', 'strong', 'i', 'b', 'u', 'a', 'br', 'sub', 'sup', 'small'}
+
+ for div in list(soup.find_all('div')):
+ # If div has no block children, convert to p
+ has_block = any(
+ isinstance(c, Tag) and c.name not in inline_tags
+ for c in div.children
+ )
+
+ if not has_block:
+ div.name = 'p'
diff --git a/archive/v0.10/src/preprocessing/format_extractor.py b/archive/v0.10/src/preprocessing/format_extractor.py
new file mode 100644
index 0000000..438430b
--- /dev/null
+++ b/archive/v0.10/src/preprocessing/format_extractor.py
@@ -0,0 +1,372 @@
+import re
+from bs4 import BeautifulSoup, Tag
+from typing import Tuple, Dict, List
+
+class HeadingDetector:
+ """Detects heading types and paragraph roles."""
+
+ CHAPTER_PATTERNS = [
+ r'^(chapter|chap\.?|part)\s+([0-9]+|[ivxlc]+|[a-z])',
+ r'^(第\s*[0-9一二三四五六七八九十百]+\s*[章节部篇])',
+ r'^(\d+|[IVXLC]+|[A-Z])\.$'
+ ]
+
+ EPIGRAPH_CLASSES = {
+ 'epigraph', 'quote', 'blockquote', 'motto',
+ 'dedication', 'verse', 'poetry', 'poem'
+ }
+
+ def detect(self, element: Tag, text: str) -> str:
+ if self._is_epigraph(element):
+ return "epigraph"
+ tag_name = element.name.lower()
+ if tag_name in ['h1', 'h2']:
+ return "chapter" if self._matches_chapter_pattern(text) else "section"
+ if tag_name == 'h3':
+ return "section"
+ if tag_name in ['h4', 'h5', 'h6']:
+ return "subsection"
+ if self._is_pseudo_heading(element, text):
+ return "subsection"
+ return "body"
+
+ def _is_epigraph(self, element: Tag) -> bool:
+ if element.name == 'blockquote':
+ return True
+ current = element
+ for _ in range(3):
+ if not current: break
+ classes = current.get('class', [])
+ if isinstance(classes, list):
+ classes = ' '.join(classes)
+ if any(k in classes.lower() for k in self.EPIGRAPH_CLASSES):
+ return True
+ current = current.parent
+ return False
+
+ def _matches_chapter_pattern(self, text: str) -> bool:
+ text = text.strip().lower()
+ for pattern in self.CHAPTER_PATTERNS:
+ if re.match(pattern, text, re.IGNORECASE):
+ return True
+ return False
+
+ def _is_pseudo_heading(self, element: Tag, text: str) -> bool:
+ if element.name != 'p':
+ return False
+ text = text.strip()
+ if not text or len(text) > 80:
+ return False
+ children = list(element.children)
+ if len(children) == 1 and isinstance(children[0], Tag):
+ if children[0].name in ['strong', 'b']:
+ return True
+ return False
+
+
+class FormatExtractor:
+ """
+ HTML Format Extractor (Ported from v0.09 v3)
+ Handles inline styles, formulas, and drop caps.
+ """
+
+ FORMULA_CHARS = re.compile(
+ r'^[\d\s\+\-\×\÷\=\(\)\[\]\{\}\<\>\^\*\/\.\,\;\:\'\"\`\~\@\#\$\%\&\|\\'
+ r'αβγδεζηθικλμνξοπρστυφχψωΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩ'
+ r'a-zA-Z]+$'
+ )
+
+ def __init__(self):
+ self.detector = HeadingDetector()
+
+ def extract(self, element_html: str) -> Tuple[str, str, Dict[str, str], str, List[str]]:
+ """
+ Extracts format information.
+
+ Returns:
+ clean_text: Pure text
+ text_with_placeholders: Text with inline placeholders
+ placeholder_map: Map of placeholders
+ paragraph_type: Detected type
+ endnote_anchors: List of detected endnote IDs
+ """
+ soup = BeautifulSoup(element_html, 'html.parser')
+ root = list(soup.children)[0] if list(soup.children) else soup
+
+ clean_text = root.get_text().strip()
+ clean_text = re.sub(r'\s+', ' ', clean_text)
+ p_type = self.detector.detect(root, clean_text) if isinstance(root, Tag) else "body"
+
+ inner_html = root.decode_contents() if isinstance(root, Tag) else str(root)
+
+ text_with_ph, local_map = self._smart_extract_v3(inner_html)
+
+ if text_with_ph:
+ text_with_ph = re.sub(r'\s+', ' ', text_with_ph).strip()
+
+ # Verify integrity
+ stripped_text = self._strip_placeholders(text_with_ph)
+ stripped_text = re.sub(r'\s+', ' ', stripped_text).strip()
+
+ if not self._verify_content_integrity(clean_text, stripped_text):
+ # Fallback
+ text_with_ph, local_map = self._fallback_extract(inner_html, clean_text)
+
+ endnote_anchors = []
+ for pid, html in local_map.items():
+ if pid.startswith("_"):
+ continue
+ if re.match(r'<(span|a)\s+id="[a-zA-Z][a-zA-Z0-9]{2,5}"\s*>\s*', html):
+ endnote_anchors.append(pid)
+
+ return clean_text, text_with_ph, local_map, p_type, endnote_anchors
+
+ def _strip_placeholders(self, text: str) -> str:
+ return re.sub(r'φ/?[0-9]+φ', '', text)
+
+ def _verify_content_integrity(self, clean_text: str, stripped_text: str) -> bool:
+ def normalize(s):
+ s = re.sub(r'\s+', '', s)
+ s = s.lower()
+ return s
+
+ norm_clean = normalize(clean_text)
+ norm_stripped = normalize(stripped_text)
+
+ if norm_clean == norm_stripped:
+ return True
+
+ return False
+
+ def _fallback_extract(self, inner_html: str, clean_text: str) -> Tuple[str, Dict[str, str]]:
+ return clean_text, {"_prefix": "", "_suffix": ""}
+
+ def _smart_extract_v3(self, inner_html: str) -> Tuple[str, Dict[str, str]]:
+ parts = re.split(r'(<[^>]+>)', inner_html)
+ parts = [p for p in parts if p]
+
+ if not parts:
+ return "", {"_prefix": "", "_suffix": ""}
+
+ part_types = []
+ for part in parts:
+ if part.startswith('<'):
+ part_types.append('tag')
+ elif not part.strip():
+ part_types.append('whitespace')
+ elif self._is_translatable_text(part):
+ part_types.append('translatable')
+ else:
+ part_types.append('formula')
+
+ first_trans_idx = None
+ last_trans_idx = None
+ for i, t in enumerate(part_types):
+ if t == 'translatable':
+ if first_trans_idx is None:
+ first_trans_idx = i
+ last_trans_idx = i
+
+ if first_trans_idx is None:
+ return "", {"_prefix": inner_html, "_suffix": ""}
+
+ # Prefix Separation
+ safe_prefix_end = 0
+ for i in range(first_trans_idx):
+ if part_types[i] == 'tag':
+ tag = parts[i]
+ is_self_closing = tag.endswith('/>') or re.match(r'<(br|hr|img|meta|link)\b', tag, re.I)
+ is_closing = tag.startswith('') or re.match(r'<(br|hr|img|meta|link)\b', tag, re.I)
+ if is_closing or is_self_closing:
+ safe_suffix_start = i
+ else:
+ break
+ elif part_types[i] == 'whitespace':
+ safe_suffix_start = i
+ else:
+ break
+
+ prefix_parts = parts[:safe_prefix_end]
+ middle_parts = parts[safe_prefix_end:safe_suffix_start]
+ middle_types = part_types[safe_prefix_end:safe_suffix_start]
+ suffix_parts = parts[safe_suffix_start:]
+
+ # Drop Cap Check
+ if prefix_parts and middle_parts:
+ prefix_parts, middle_parts, middle_types = self._handle_drop_cap(
+ prefix_parts, middle_parts, middle_types
+ )
+
+ local_map = {}
+ if prefix_parts:
+ local_map["_prefix"] = "".join(prefix_parts)
+ if suffix_parts:
+ local_map["_suffix"] = "".join(suffix_parts)
+
+ # Middle processing
+ placeholder_counter = 1
+ result_parts = []
+ tag_stack = []
+
+ i = 0
+ while i < len(middle_parts):
+ part = middle_parts[i]
+ ptype = middle_types[i]
+
+ if ptype == 'translatable':
+ result_parts.append(part)
+ i += 1
+
+ elif ptype == 'tag':
+ is_closing = part.startswith('= 2 and combined.isalpha()):
+ return prefix_parts, middle_parts, middle_types
+
+ new_prefix = []
+ skip_until_close = False
+ found_letter = False
+
+ for part in prefix_parts:
+ if part.startswith('<') and not part.startswith(' bool:
+ text = text.strip()
+ if not text:
+ return False
+ if re.search(r'[a-zA-Z]{3,}', text):
+ return True
+ if ' ' in text and re.search(r'[a-zA-Z]', text):
+ return True
+ if re.search(r'\d', text):
+ return True
+ return False
diff --git a/archive/v0.10/src/preprocessing/profiler.py b/archive/v0.10/src/preprocessing/profiler.py
new file mode 100644
index 0000000..5636ab0
--- /dev/null
+++ b/archive/v0.10/src/preprocessing/profiler.py
@@ -0,0 +1,76 @@
+import json
+import random
+from typing import Dict, List
+from loguru import logger
+from src.translation.llm_client import LLMClient
+from src.translation.manifest_manager import ManifestManager
+from src.common.data_model import BookProfile
+
+class BookProfiler:
+ def __init__(self, llm_client: LLMClient):
+ self.llm_client = llm_client
+
+ def extract_sample_text(self, entries: List[object], char_limit: int = 3000) -> str:
+ """Extract sample text from manifest entries."""
+ if not entries: return ""
+
+ # Simple sampling strategy: First few + random middle
+ intro_text = []
+ for entry in entries[:50]:
+ if len(entry.original_text) > 50:
+ intro_text.append(entry.original_text)
+
+ body_text = []
+ candidates = [e for e in entries[50:] if len(e.original_text) > 80]
+ if candidates:
+ samples = random.sample(candidates, min(5, len(candidates)))
+ body_text = [e.original_text for e in samples]
+
+ full_text = "\n\n".join(intro_text[:5] + body_text)
+ return full_text[:char_limit]
+
+ async def analyze(self, entries: List[object]) -> BookProfile:
+ """Generate Book Profile."""
+ sample = self.extract_sample_text(entries)
+ if not sample:
+ return BookProfile(title="Unknown", author="Unknown")
+
+ logger.info("Generating Book Profile from sample text...")
+
+ system_prompt = "You are a senior publishing editor. Analyze the text and output JSON."
+ user_prompt = f"""
+Please analyze the following book excerpt.
+Output JSON format:
+{{
+ "title": "Book Title",
+ "author": "Author Name",
+ "genre": "Genre",
+ "style": "Style description",
+ "keywords": ["keyword1", "keyword2"],
+ "style_guide": "Specific instruction for translator"
+}}
+
+Excerpt:
+{sample}
+"""
+ try:
+ response = await self.llm_client.raw_chat_completion(system_prompt, user_prompt)
+ json_str = response.strip()
+ # Basic cleanup
+ if "```json" in json_str:
+ json_str = json_str.split("```json")[1].split("```")[0].strip()
+ elif "```" in json_str:
+ json_str = json_str.split("```")[1].split("```")[0].strip()
+
+ data = json.loads(json_str)
+
+ return BookProfile(
+ title=data.get("title", "Unknown"),
+ author=data.get("author", "Unknown"),
+ genre=data.get("genre", "General"),
+ keywords=data.get("keywords", []),
+ style_guide=data.get("style_guide", "")
+ )
+ except Exception as e:
+ logger.error(f"Profile generation failed: {e}")
+ return BookProfile(title="Unknown", author="Unknown")
diff --git a/archive/v0.10/src/preprocessing/text_extractor.py b/archive/v0.10/src/preprocessing/text_extractor.py
new file mode 100644
index 0000000..f2bd23d
--- /dev/null
+++ b/archive/v0.10/src/preprocessing/text_extractor.py
@@ -0,0 +1,132 @@
+import re
+from typing import List, Dict, Any, Optional
+from bs4 import BeautifulSoup
+from loguru import logger
+
+from src.common.data_model import BookStructure, ManifestEntry, BookProfile
+from src.preprocessing.format_extractor import FormatExtractor
+from src.common.utils import setup_logger
+
+logger = setup_logger("fine_grained_extractor")
+
+class FineGrainedExtractor:
+ """
+ Extracts translatable text segments from BookStructure.
+ Uses FormatExtractor for detailed content analysis.
+ """
+
+ SKIP_TRANSLATION_PATTERNS = [
+ r'index\.x?html',
+ r'bibliography\.x?html',
+ r'endnotes?\.x?html',
+ r'footnotes?\.x?html',
+ r'copyright\.x?html',
+ ]
+
+ TOC_PATTERNS = [
+ r'nav\.x?html',
+ r'toc\.x?html',
+ ]
+
+ def __init__(self, translate_toc: bool = False):
+ self.translate_toc = translate_toc
+ self.format_extractor = FormatExtractor()
+
+ def extract(self, structure: BookStructure, profile: Optional[BookProfile] = None) -> List[ManifestEntry]:
+ """
+ Extracts translatable segments from the BookStructure.
+ Iteration follows the spine order.
+ """
+ logger.info("Starting extraction from BookStructure...")
+ manifest_entries = []
+
+ # Iterate over spine to maintain order
+ for item_id in structure.spine:
+ if item_id not in structure.resources:
+ logger.warning(f"Item ID {item_id} in spine but not in resources.")
+ continue
+
+ resource = structure.resources[item_id]
+
+ # Only process HTML/XHTML
+ if resource.media_type != "application/xhtml+xml" or not resource.content:
+ continue
+
+ file_path = resource.href
+ doc_type = self._classify_document(file_path)
+
+ # Parse HTML
+ soup = BeautifulSoup(resource.content, 'html.parser')
+ target_tags = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li']
+
+ for element in soup.find_all(target_tags):
+ # Ensure element has ID (should have been done by Cleaner)
+ element_id = element.get('id')
+ if not element_id:
+ logger.warning(f"Element in {file_path} missing ID, skipping: {element.name}")
+ continue
+
+ # Check translation eligibility
+ raw_text = element.get_text(separator=' ', strip=True)
+ if not raw_text.strip():
+ continue
+
+ is_decorative = self._is_decorative(raw_text)
+ should_translate = self._should_translate(doc_type, is_decorative, raw_text)
+
+ if should_translate:
+ # Extract detailed format
+ outer_html = str(element)
+ clean_text, text_with_ph, ph_map, p_type, _ = self.format_extractor.extract(outer_html)
+
+ if clean_text.strip() and text_with_ph.strip():
+ entry = ManifestEntry(
+ entry_id=f"{file_path}#{element_id}",
+ file_path=file_path,
+ element_id=element_id,
+ original_text=text_with_ph,
+ placeholders=ph_map,
+ context=p_type
+ )
+ manifest_entries.append(entry)
+
+ logger.info(f"Extracted {len(manifest_entries)} entries in total.")
+ return manifest_entries
+
+ def _classify_document(self, file_name: str) -> str:
+ if not file_name: return 'core'
+ fname = file_name.lower()
+ if any(re.search(p, fname) for p in self.SKIP_TRANSLATION_PATTERNS): return 'skip'
+ if any(re.search(p, fname) for p in self.TOC_PATTERNS): return 'toc'
+ return 'core'
+
+ def _should_translate(self, doc_type: str, is_decorative: bool, text: str = "") -> bool:
+ if is_decorative: return False
+ if self._is_roman_numeral(text): return False
+ if doc_type == 'core': return True
+ if doc_type == 'toc': return self.translate_toc
+ return False
+
+ def _is_roman_numeral(self, text: str) -> bool:
+ text = text.strip().upper()
+ if not text: return False
+ pattern = re.compile(r"^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$")
+ return bool(pattern.match(text))
+
+ def _is_decorative(self, text: str) -> bool:
+ s = text.strip()
+ if not s: return False
+ if not any(c.isalnum() for c in s): return True
+ if len(s) > 20: return False
+
+ patterns = [
+ r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾]+$',
+ r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾\s]+$',
+ ]
+ for p in patterns:
+ if re.match(p, s): return True
+
+ unique = set(s.replace(' ', ''))
+ if len(unique) <= 3 and (unique & set('*-_~•◆◇■□▪▫●○◉◎⦿⦾')):
+ return True
+ return False
diff --git a/archive/v0.10/src/translation/llm_client.py b/archive/v0.10/src/translation/llm_client.py
new file mode 100644
index 0000000..cd6a5d8
--- /dev/null
+++ b/archive/v0.10/src/translation/llm_client.py
@@ -0,0 +1,220 @@
+import asyncio
+import json
+import re
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import List, Dict, Optional
+from openai import AsyncOpenAI
+from tenacity import retry, stop_after_attempt, wait_exponential
+from loguru import logger
+
+from src.common.data_model import ManifestEntry
+from src.common.utils import setup_logger, ensure_directory
+
+logger = setup_logger("llm_client")
+
+# Default chunk save directory (can be overridden)
+DEFAULT_CHUNK_DIR = Path("tmp/chunks")
+
+
+class RateLimiter:
+ """Rate limiter for concurrency and RPM."""
+ def __init__(self, requests_per_minute: int, concurrent_requests: int):
+ self.semaphore = asyncio.Semaphore(concurrent_requests)
+ self.min_interval = 60.0 / requests_per_minute if requests_per_minute > 0 else 0
+ self.last_request_time = 0
+ self._lock = asyncio.Lock()
+
+ async def acquire(self):
+ await self.semaphore.acquire()
+ async with self._lock:
+ current_time = time.time()
+ wait_time = self.min_interval - (current_time - self.last_request_time)
+ if wait_time > 0:
+ await asyncio.sleep(wait_time)
+ self.last_request_time = time.time()
+
+ def release(self):
+ self.semaphore.release()
+
+
+class LLMClient:
+ """Generic OpenAI-compatible API Client with short ID strategy."""
+
+ def __init__(self, api_key: str, base_url: str, model: str = "gpt-3.5-turbo",
+ requests_per_minute: int = 60, concurrent_requests: int = 5,
+ extra_headers: Dict = None, chunk_dir: Path = None):
+
+ # Configure proxy client to avoid SOCKS issues and ensure connectivity
+ import httpx
+ import os
+
+ # Prefer HTTP proxy if available to avoid missing socksio support
+ proxy_url = os.environ.get("http_proxy") or os.environ.get("https_proxy")
+ http_client = httpx.AsyncClient(
+ proxy=proxy_url,
+ timeout=60.0,
+ follow_redirects=True
+ ) if proxy_url else None
+
+ self.client = AsyncOpenAI(
+ base_url=base_url,
+ api_key=api_key,
+ default_headers=extra_headers,
+ http_client=http_client
+ )
+ self.model = model
+
+ self.rate_limiter = RateLimiter(requests_per_minute, concurrent_requests)
+ self.prompts = self._load_prompts()
+ self._chunk_counter = 0
+
+ # Chunk directory for debug output
+ self.chunk_dir = chunk_dir or DEFAULT_CHUNK_DIR
+ ensure_directory(self.chunk_dir)
+
+ def _load_prompts(self) -> Dict:
+ try:
+ with open("config/prompts.json", "r", encoding="utf-8") as f:
+ return json.load(f)
+ except Exception as e:
+ logger.error(f"Failed to load config/prompts.json: {e}")
+ return {}
+
+ async def translate_chunk(self, items: List[ManifestEntry], glossary: Dict = None,
+ instruction: str = None, mode: str = "bilingual") -> Dict[str, str]:
+ """
+ Translate a chunk of items using short ID strategy.
+ Returns: Dict[entry_id, translated_text]
+ """
+ if not items: return {}
+
+ # Build prompt with short IDs
+ id_map, prompt = self._build_prompt_with_short_ids(items)
+
+ try:
+ # Build System Prompt
+ base_sys_prompt = self.prompts.get("translation", {}).get("system",
+ "You are a professional English to Chinese translator.")
+
+ if instruction:
+ base_sys_prompt += f"\n\nBook Style Guide:\n{instruction}"
+
+ if glossary:
+ glossary_text = "\n".join([f"{k} -> {v}" for k, v in glossary.items()])
+ base_sys_prompt += f"\n\nTerminology:\n{glossary_text}"
+
+ # Short ID format instructions
+ base_sys_prompt += """
+
+Output Format:
+- Each line MUST start with #N: (keep this ID exactly as given)
+- Preserve any φXφ or φ/Xφ placeholders EXACTLY as-is
+- Only output translations, no explanations
+- Match the number of output lines to input lines"""
+
+ # Save chunk before translation
+ chunk_id = self._save_chunk("before", prompt, base_sys_prompt)
+
+ logger.debug(f"Sending request to LLM (Chunk: {chunk_id}, Items: {len(items)})")
+
+ raw_response = await self._make_request(base_sys_prompt, prompt)
+
+ # Save chunk after translation
+ self._save_chunk("after", raw_response, base_sys_prompt, chunk_id)
+
+ # Parse with short ID mapping
+ results = self._parse_short_id_response(raw_response, id_map)
+ return results
+
+ except Exception as e:
+ logger.error(f"Translation failed: {e}")
+ return {item.entry_id: f"[Error - {str(e)}]" for item in items}
+
+ def _build_prompt_with_short_ids(self, items: List[ManifestEntry]) -> tuple:
+ """
+ Build prompt with short IDs (#1, #2, ...).
+ Returns: (id_map, prompt_text)
+ """
+ id_map = {} # short_id -> entry_id
+ lines = []
+
+ for i, item in enumerate(items, 1):
+ short_id = f"#{i}"
+ id_map[short_id] = item.entry_id
+
+ # Clean text (remove extra whitespace)
+ text = re.sub(r'\s+', ' ', item.original_text).strip()
+ lines.append(f"{short_id}: {text}")
+
+ return id_map, "\n".join(lines)
+
+ def _parse_short_id_response(self, response: str, id_map: Dict[str, str]) -> Dict[str, str]:
+ """
+ Parse response with short ID format.
+ Returns: Dict[entry_id, translated_text]
+ """
+ results = {}
+
+ for line in response.split("\n"):
+ line = line.strip()
+ if not line:
+ continue
+
+ # Match #N: pattern
+ match = re.match(r'^#(\d+):\s*(.+)$', line)
+ if match:
+ short_id = f"#{match.group(1)}"
+ translation = match.group(2).strip()
+
+ if short_id in id_map:
+ full_id = id_map[short_id]
+ results[full_id] = translation
+ else:
+ logger.warning(f"Unknown short ID in response: {short_id}")
+
+ return results
+
+ def _save_chunk(self, stage: str, content: str, system_prompt: str = None,
+ chunk_id: str = None) -> str:
+ """Save chunk to tmp directory for debugging."""
+ if chunk_id is None:
+ self._chunk_counter += 1
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+ chunk_id = f"{timestamp}_{self._chunk_counter:04d}"
+
+ filename = self.chunk_dir / f"chunk_{chunk_id}_{stage}.txt"
+
+ with open(filename, "w", encoding="utf-8") as f:
+ if system_prompt and stage == "before":
+ f.write("=== SYSTEM PROMPT ===\n")
+ f.write(system_prompt)
+ f.write("\n\n=== USER PROMPT ===\n")
+ f.write(content)
+
+ logger.debug(f"Saved chunk: {filename}")
+ return chunk_id
+
+ async def raw_chat_completion(self, system_prompt: str, user_prompt: str) -> str:
+ """Generic chat completion."""
+ return await self._make_request(system_prompt, user_prompt)
+
+ @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))
+ async def _make_request(self, system_prompt: str, user_prompt: str) -> str:
+ await self.rate_limiter.acquire()
+ try:
+ resp = await self.client.chat.completions.create(
+ model=self.model,
+ messages=[
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_prompt}
+ ],
+ temperature=0.3,
+ )
+ return resp.choices[0].message.content.strip()
+ finally:
+ self.rate_limiter.release()
+
+ async def close(self):
+ await self.client.close()
diff --git a/archive/v0.10/src/translation/manifest_manager.py b/archive/v0.10/src/translation/manifest_manager.py
new file mode 100644
index 0000000..30e59fe
--- /dev/null
+++ b/archive/v0.10/src/translation/manifest_manager.py
@@ -0,0 +1,74 @@
+from pathlib import Path
+from typing import List, Dict, Optional
+import json
+from src.common.data_model import ManifestEntry
+from src.common.utils import setup_logger
+
+logger = setup_logger("manifest_manager")
+
+class ManifestManager:
+ """
+ Manages the translation manifest (Source of Truth).
+ Handles persistence and state updates.
+ """
+
+ def __init__(self, manifest_path: Path):
+ self.manifest_path = manifest_path
+ self.entries: List[ManifestEntry] = []
+ self._entries_map: Dict[str, ManifestEntry] = {}
+
+ def load(self):
+ """Loads manifest from disk if it exists."""
+ if not self.manifest_path.exists():
+ logger.info(f"Manifest not found at {self.manifest_path}, starting empty.")
+ return
+
+ try:
+ with open(self.manifest_path, 'r', encoding='utf-8') as f:
+ data = json.load(f)
+ self.entries = [ManifestEntry.model_validate(item) for item in data]
+ self._rebuild_map()
+ logger.info(f"Loaded {len(self.entries)} entries from manifest.")
+ except Exception as e:
+ logger.error(f"Failed to load manifest: {e}")
+ raise
+
+ def save(self):
+ """Saves current state to disk."""
+ try:
+ # Pydantic v2: model_dump(mode='json') or just list dump
+ data = [entry.model_dump(mode='json') for entry in self.entries]
+ with open(self.manifest_path, 'w', encoding='utf-8') as f:
+ json.dump(data, f, indent=2, ensure_ascii=False)
+ logger.info(f"Saved {len(self.entries)} entries to manifest.")
+ except Exception as e:
+ logger.error(f"Failed to save manifest: {e}")
+ raise
+
+ def add_entries(self, new_entries: List[ManifestEntry]):
+ """
+ Adds new entries to the manifest.
+ If an entry with the same ID exists, it keeps the EXISTING one (to preserve translations).
+ """
+ count = 0
+ for entry in new_entries:
+ if entry.entry_id not in self._entries_map:
+ self.entries.append(entry)
+ self._entries_map[entry.entry_id] = entry
+ count += 1
+
+ if count > 0:
+ logger.info(f"Added {count} new entries to manifest.")
+
+ def update_translation(self, entry_id: str, translation: str):
+ """Updates translation for a specific entry."""
+ if entry_id in self._entries_map:
+ self._entries_map[entry_id].translated_text = translation
+ else:
+ logger.warning(f"Attempted to update translation for unknown ID: {entry_id}")
+
+ def get_entry(self, entry_id: str) -> Optional[ManifestEntry]:
+ return self._entries_map.get(entry_id)
+
+ def _rebuild_map(self):
+ self._entries_map = {e.entry_id: e for e in self.entries}
diff --git a/archive/v0.10/src/translation/translator_engine.py b/archive/v0.10/src/translation/translator_engine.py
new file mode 100644
index 0000000..86fb29a
--- /dev/null
+++ b/archive/v0.10/src/translation/translator_engine.py
@@ -0,0 +1,285 @@
+"""
+Translator Module - Handles translation of ManifestEntry items.
+
+Key features:
+- Character-based chunking (~5000 chars per chunk)
+- Chapter-aware grouping (chunks don't cross file boundaries)
+- Concurrent translation with asyncio.gather
+- Progress tracking and error handling
+"""
+
+import asyncio
+from typing import List, Dict
+from collections import defaultdict
+from loguru import logger
+
+from src.common.data_model import ManifestEntry, BookProfile
+from src.translation.llm_client import LLMClient
+from src.assembly.format_restorer import FormatRestorer
+from src.common.utils import setup_logger
+
+logger = setup_logger("translator")
+
+# Default chunk size in characters
+DEFAULT_CHUNK_SIZE = 5000
+# Maximum concurrent translations
+MAX_CONCURRENT = 5
+
+
+class Translator:
+ """
+ Translates ManifestEntry items using LLM with chapter-aware chunking.
+ Supports both sequential and concurrent translation modes.
+ """
+
+ def __init__(self, llm_client: LLMClient, chunk_size: int = DEFAULT_CHUNK_SIZE,
+ max_concurrent: int = MAX_CONCURRENT):
+ self.llm_client = llm_client
+ self.chunk_size = chunk_size
+ self.max_concurrent = max_concurrent
+ self.restorer = FormatRestorer()
+
+ async def translate(self, entries: List[ManifestEntry], profile: BookProfile,
+ concurrent: bool = True) -> List[ManifestEntry]:
+ """
+ Translates all untranslated entries.
+
+ Args:
+ entries: All manifest entries
+ profile: Book profile with style guide
+ concurrent: Use concurrent translation (default True)
+
+ Returns:
+ The same entries list with translated_text populated
+ """
+ untranslated = [e for e in entries if not e.translated_text]
+ if not untranslated:
+ logger.info("No new entries to translate.")
+ return entries
+
+ logger.info(f"Found {len(untranslated)} entries to translate")
+
+ # Group by chapter (file_path)
+ chapters = self._group_by_chapter(untranslated)
+ logger.info(f"Grouped into {len(chapters)} chapters")
+
+ # Create all chunks
+ all_chunks = []
+ for file_path, chapter_entries in chapters.items():
+ chapter_chunks = self._create_char_based_chunks(chapter_entries)
+ for chunk in chapter_chunks:
+ all_chunks.append((file_path, chunk))
+
+ total_chunks = len(all_chunks)
+ logger.info(f"Created {total_chunks} chunks (avg ~{self.chunk_size} chars each)")
+
+ if concurrent:
+ await self._translate_concurrent(all_chunks, profile)
+ else:
+ await self._translate_sequential(all_chunks, profile)
+
+ translated_count = sum(1 for e in entries if e.translated_text)
+ logger.info(f"Translation complete: {translated_count}/{len(entries)} entries translated")
+ return entries
+
+ async def _translate_concurrent(self, all_chunks: List, profile: BookProfile):
+ """Translate chunks concurrently with semaphore control."""
+ semaphore = asyncio.Semaphore(self.max_concurrent)
+ completed = [0] # Use list for mutable counter in closure
+ total = len(all_chunks)
+ success = [0]
+ failed = [0]
+
+ async def translate_chunk_task(file_path: str, chunk: List[ManifestEntry], idx: int):
+ async with semaphore:
+ try:
+ results = await self.llm_client.translate_chunk(
+ chunk,
+ instruction=profile.style_guide if profile else None,
+ mode="bilingual"
+ )
+
+ for entry in chunk:
+ if entry.entry_id in results:
+ entry.translated_text = results[entry.entry_id]
+ success[0] += 1
+ else:
+ failed[0] += 1
+
+ except Exception as e:
+ logger.error(f"Chunk {idx} failed: {e}")
+ failed[0] += len(chunk)
+ finally:
+ completed[0] += 1
+ if completed[0] % 5 == 0 or completed[0] == total:
+ logger.info(f"Progress: {completed[0]}/{total} chunks ({success[0]} translated)")
+
+ tasks = [
+ translate_chunk_task(file_path, chunk, i)
+ for i, (file_path, chunk) in enumerate(all_chunks)
+ ]
+ await asyncio.gather(*tasks)
+
+ logger.info(f"Concurrent translation: {success[0]} success, {failed[0]} failed")
+
+ async def _translate_sequential(self, all_chunks: List, profile: BookProfile):
+ """Translate chunks sequentially."""
+ total_chunks = len(all_chunks)
+ success_count = 0
+ fail_count = 0
+
+ for idx, (file_path, chunk) in enumerate(all_chunks):
+ try:
+ results = await self.llm_client.translate_chunk(
+ chunk,
+ instruction=profile.style_guide if profile else None,
+ mode="bilingual"
+ )
+
+ for entry in chunk:
+ if entry.entry_id in results:
+ entry.translated_text = results[entry.entry_id]
+ success_count += 1
+ else:
+ fail_count += 1
+ logger.warning(f"Missing translation for: {entry.entry_id[-40:]}")
+
+ except Exception as e:
+ logger.error(f"Chunk {idx} translation failed: {e}")
+ fail_count += len(chunk)
+
+ if (idx + 1) % 10 == 0 or idx + 1 == total_chunks:
+ logger.info(f"Progress: {idx + 1}/{total_chunks} chunks ({success_count} entries translated)")
+
+ logger.info(f"Sequential translation: {success_count} success, {fail_count} failed")
+
+
+ async def translate_chapter(self, entries: List[ManifestEntry], file_path: str,
+ profile: BookProfile) -> Dict[str, int]:
+ """
+ Translate a single chapter.
+
+ Args:
+ entries: All entries (will filter by file_path)
+ file_path: Chapter file path to translate
+ profile: Book profile
+
+ Returns:
+ Dict with 'success' and 'failed' counts
+ """
+ chapter_entries = [e for e in entries if e.file_path == file_path and not e.translated_text]
+
+ if not chapter_entries:
+ logger.info(f"Chapter {file_path} has no untranslated entries")
+ return {"success": 0, "failed": 0}
+
+ logger.info(f"Translating chapter: {file_path} ({len(chapter_entries)} entries)")
+
+ chunks = self._create_char_based_chunks(chapter_entries)
+ logger.info(f"Created {len(chunks)} chunks")
+
+ success = 0
+ failed = 0
+
+ for i, chunk in enumerate(chunks, 1):
+ chunk_chars = sum(len(e.original_text) for e in chunk)
+ logger.debug(f"Chunk {i}/{len(chunks)}: {len(chunk)} entries, {chunk_chars} chars")
+
+ try:
+ results = await self.llm_client.translate_chunk(
+ chunk,
+ instruction=profile.style_guide if profile else None,
+ mode="bilingual"
+ )
+
+ for entry in chunk:
+ if entry.entry_id in results:
+ entry.translated_text = results[entry.entry_id]
+
+ # Verify placeholder preservation
+ if entry.placeholders:
+ _, restored_ok = self.restorer.restore(
+ entry.translated_text,
+ entry.placeholders
+ )
+ if not restored_ok:
+ logger.warning(f"Placeholder issue: {entry.entry_id[-40:]}")
+
+ success += 1
+ else:
+ failed += 1
+
+ except Exception as e:
+ logger.error(f"Chunk {i} failed: {e}")
+ failed += len(chunk)
+
+ logger.info(f"Chapter done: {success} success, {failed} failed")
+ return {"success": success, "failed": failed}
+
+ def _group_by_chapter(self, entries: List[ManifestEntry]) -> Dict[str, List[ManifestEntry]]:
+ """Group entries by file_path (chapter)."""
+ chapters = defaultdict(list)
+ for entry in entries:
+ chapters[entry.file_path].append(entry)
+ return dict(chapters)
+
+ def _create_char_based_chunks(self, entries: List[ManifestEntry]) -> List[List[ManifestEntry]]:
+ """
+ Create chunks based on character count.
+
+ Each chunk contains approximately self.chunk_size characters.
+ Chunks never cross chapter boundaries (entries from same file only).
+ """
+ chunks = []
+ current_chunk = []
+ current_size = 0
+
+ for entry in entries:
+ text_len = len(entry.original_text)
+
+ # If adding this entry exceeds limit and we have content, start new chunk
+ if current_size + text_len > self.chunk_size and current_chunk:
+ chunks.append(current_chunk)
+ current_chunk = []
+ current_size = 0
+
+ current_chunk.append(entry)
+ current_size += text_len
+
+ if current_chunk:
+ chunks.append(current_chunk)
+
+ return chunks
+
+ def get_chapter_stats(self, entries: List[ManifestEntry]) -> List[Dict]:
+ """
+ Get statistics for each chapter.
+
+ Returns list of dicts with: file_path, total, translated, pending, chars
+ """
+ chapters = self._group_by_chapter(entries)
+ stats = []
+
+ for file_path, chapter_entries in sorted(chapters.items()):
+ total = len(chapter_entries)
+ translated = sum(1 for e in chapter_entries if e.translated_text)
+ total_chars = sum(len(e.original_text) for e in chapter_entries)
+
+ # Get first text as title preview
+ first_text = ""
+ for e in chapter_entries:
+ if e.original_text:
+ first_text = e.original_text[:40].replace('\n', ' ')
+ break
+
+ stats.append({
+ "file_path": file_path,
+ "title": first_text,
+ "total": total,
+ "translated": translated,
+ "pending": total - translated,
+ "chars": total_chars
+ })
+
+ return stats
+
diff --git a/archive/v0.11/.agent/rules/runing-guide.md b/archive/v0.11/.agent/rules/runing-guide.md
new file mode 100644
index 0000000..643b9b8
--- /dev/null
+++ b/archive/v0.11/.agent/rules/runing-guide.md
@@ -0,0 +1,38 @@
+---
+trigger: always_on
+---
+
+# EPUB Bilingual Translator - File Architecture & Workspace Rules
+
+## 1. Directory Structure
+The project follows a strict modular structure. Code is in `src/`, execution scripts in `pipeline/`, and intermediate data in `.work/`.
+
+.
+├── main.py # Entry point (orchestrator)
+├── pipeline/ # Executable scripts for each stage
+│ ├── 01_preprocess.py # Step 1: Clean & Extract
+│ ├── 02_translate.py # Step 2: LLM Translation
+│ └── 03_assemble.py # Step 3: Backfill & Build
+├── src/ # Source modules
+│ ├── common/ # shared utils, config, paths.py
+│ ├── preprocessing/ # epub_cleaner, text_extractor, profiler
+│ ├── translation/ # llm_client, translator_engine, manifest_manager
+│ └── assembly/ # backfiller, builder, format_restorer
+├── config/
+│ ├── config.yaml # System configuration (LLM, Translation)
+│ └── prompts.json # LLM Prompts
+└── .work/ # Working Directory (Gitignored)
+ └── {book_name}/ # One folder per book
+ ├── book_structure.json # Structural skeleton (created by Step 1)
+ ├── manifest.json # Translation source of truth
+ ├── assets/ # Extracted images/css
+ └── chunks/ # Debug chunks from translation (before/after)
+
+## 2. Path Resolution Rule
+ALWAYS use `src.common.paths.get_work_dirs(input_path)` to resolve paths.
+DO NOT hardcode `work/`, `.work/`, or `tmp/` paths in scripts.
+
+## 3. Data Flow
+1. Preprocess: EPUB -> .work/{book}/book_structure.json + .work/{book}/manifest.json
+2. Translate: .work/{book}/manifest.json (read/write) -> .work/{book}/chunks/ (logs)
+3. Assemble: .work/{book}/manifest.json + .work/{book}/book_structure.json -> output/{book}_bilingual.epub
\ No newline at end of file
diff --git a/archive/v0.11/.gitignore b/archive/v0.11/.gitignore
new file mode 100644
index 0000000..ee44251
--- /dev/null
+++ b/archive/v0.11/.gitignore
@@ -0,0 +1,26 @@
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.Python
+env/
+venv/
+.env
+.venv
+pip-log.txt
+pip-delete-this-directory.txt
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.log
+.DS_Store
+config/config.json
+output/
+cache/
+work/
+tmp/
+.work/
diff --git a/archive/v0.11/README.md b/archive/v0.11/README.md
new file mode 100644
index 0000000..f48a76f
--- /dev/null
+++ b/archive/v0.11/README.md
@@ -0,0 +1,14 @@
+# EPUB Bilingual Translator
+
+Current Version: 0.10 (In Development)
+Architecture: v2 (Modular)
+
+## Usage
+
+```bash
+python main.py input/book.epub
+```
+
+## Architecture
+
+See `doc/architecture_flow.md` for details.
diff --git a/archive/v0.11/backups/pre_refactor_20260127/config/prompts.json b/archive/v0.11/backups/pre_refactor_20260127/config/prompts.json
new file mode 100644
index 0000000..7b74b31
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/config/prompts.json
@@ -0,0 +1,10 @@
+{
+ "translation": {
+ "system": "你是一位精通中英文的专业翻译家。你的任务是将英文书籍内容翻译成中文。\n\n【核心要求】\n1. 准确传达原文含义，语言流畅自然，符合中文阅读习惯。\n2. 保持原文的风格和语气。\n\n【格式要求 - 极其重要】\n1. 每行格式：#N: 译文（N是行号，必须原样保留）\n2. 输入多少行，输出必须是相同数量的行\n3. **占位符规则**：原文中的 φXφ 和 φ/Xφ 标记必须原样保留在译文中\n - 成对标记：\"φ1φBoldφ/1φ\" → \"φ1φ粗体φ/1φ\"\n - 单体标记：\"φ2φ\" 表示公式或符号，保持位置不变\n - 尾注锚点：文末的 \"φ3φ\" 是超链接，必须保留\n\n【禁止事项】\n- 禁止修改 #N: 行号\n- 禁止删除或修改任何 φXφ 标记\n- 禁止添加解释或注释\n- 禁止合并或拆分行",
+ "user_template": "请翻译以下段落（务必原样保留所有 φnφ 格式标记）：\n\n{{content}}"
+ },
+ "glossary_extraction": {
+ "system": "你是一位资深的文学编辑和领域专家。你的任务是分析书籍样本，提取关键术语并制定统一的译名表。",
+ "user_template": "请阅读以下书籍片段（包含前言和正文采样）。\n\n任务：\n1. 识别文中出现的人名（如 'Masa', 'Steve Jobs'）、地名、机构名。\n2. 识别特定的行业术语或关键概念。\n3. 为上述词汇提供标准的中文译名。如果像 'Masa' 这样的昵称有对应的全名（如孙正义），请务必使用全名。\n\n请以 JSON 格式输出，格式如下：\n{\n \"Masa\": \"孙正义\",\n \"Apple\": \"苹果公司\",\n ...\n}\n\n书籍片段：\n\n{{content}}"
+ }
+}
\ No newline at end of file
diff --git a/archive/v0.11/backups/pre_refactor_20260127/debug_cleaner.py b/archive/v0.11/backups/pre_refactor_20260127/debug_cleaner.py
new file mode 100644
index 0000000..23fd0b1
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/debug_cleaner.py
@@ -0,0 +1,31 @@
+from bs4 import BeautifulSoup
+import zipfile
+
+def debug_clean():
+ epub_path = "input/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao).epub"
+ file_in_zip = "OEBPS/c9.xhtml"
+
+ with zipfile.ZipFile(epub_path, 'r') as zf:
+ content = zf.read(file_in_zip).decode('utf-8')
+
+ print(f"Original Head length: {len(content)}")
+ if "" in content:
+ print("Original has ")
+
+ soup = BeautifulSoup(content, 'html.parser')
+ cleaned = str(soup)
+
+ print(f"Cleaned Head length: {len(cleaned)}")
+ if "" in cleaned:
+ print("Cleaned has ")
+ start = cleaned.find("")
+ end = cleaned.find("")
+ print(f"Cleaned Head content: {cleaned[start:end+7]}")
+ else:
+ print("Cleaned MISSING ")
+ # check for self-closing head
+ if "" in cleaned:
+ print("Cleaned has (empty)")
+
+if __name__ == "__main__":
+ debug_clean()
diff --git a/archive/v0.11/backups/pre_refactor_20260127/main.py b/archive/v0.11/backups/pre_refactor_20260127/main.py
new file mode 100644
index 0000000..da22f34
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/main.py
@@ -0,0 +1,141 @@
+import argparse
+import sys
+import os
+import asyncio
+from pathlib import Path
+from dotenv import load_dotenv
+
+from src.epub_cleaner import EpubCleaner
+from src.book_profiler import BookProfiler
+from src.fine_grained_extractor import FineGrainedExtractor
+from src.manifest_manager import ManifestManager
+from src.translator import Translator
+from src.llm_client import LLMClient
+from src.backfill_engine import BackfillEngine
+from src.bilingual_builder import BilingualBuilder
+from src.data_model import BookStructure
+from src.utils import setup_logger, ensure_directory
+from src.exceptions import EpubTranslatorError
+
+logger = setup_logger("main")
+
+# Unified work directory structure
+# .work/
+# ├── {book_name}/
+# │ ├── book_structure.json
+# │ ├── manifest.json
+# │ ├── assets/
+# │ └── chunks/
+
+def get_work_dirs(input_path: Path) -> dict:
+ """Get work directory paths for a specific book."""
+ book_name = input_path.stem
+ work_root = Path(".work") / book_name
+
+ return {
+ "root": work_root,
+ "structure": work_root / "book_structure.json",
+ "manifest": work_root / "manifest.json",
+ "assets": work_root / "assets",
+ "chunks": work_root / "chunks",
+ }
+
+
+async def run_pipeline(args):
+ input_path = Path(args.input_epub)
+ output_dir = Path(args.output_dir)
+
+ # Get work directories for this book
+ work = get_work_dirs(input_path)
+ ensure_directory(work["root"])
+ ensure_directory(output_dir)
+
+ # Load Environment
+ load_dotenv()
+ api_key = os.getenv("OPENAI_API_KEY")
+ base_url = os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
+
+ if not api_key:
+ logger.warning("OPENAI_API_KEY not found in .env. LLM features may fail.")
+
+ try:
+ # 1. Preprocessing - Reuse book_structure.json if exists
+ if work["structure"].exists() and not args.force_clean:
+ logger.info(f"Reusing existing book_structure: {work['structure']}")
+ structure = BookStructure.load(work["structure"])
+ else:
+ logger.info("Cleaning EPUB and generating book_structure...")
+ cleaner = EpubCleaner(input_path, work["root"])
+ book_structure_json = cleaner.clean()
+ structure = BookStructure.load(book_structure_json)
+
+ # 2. Extraction
+ extractor = FineGrainedExtractor()
+ manifest_entries = extractor.extract(structure)
+
+ # 3. Manifest Management
+ manifest_manager = ManifestManager(work["manifest"])
+ manifest_manager.load() # Load existing if any
+ manifest_manager.add_entries(manifest_entries)
+ manifest_manager.save()
+
+ # 4. Translation
+ if not args.skip_translation:
+ if not api_key:
+ logger.error("Cannot translate without API Key. Use --skip-translation to test pipeline.")
+ sys.exit(1)
+
+ llm_client = LLMClient(api_key=api_key, base_url=base_url, model=args.model)
+
+ # Profiling
+ profiler = BookProfiler(llm_client)
+ profile = await profiler.analyze(manifest_manager.entries)
+ logger.info(f"Book Profile: {profile}")
+
+ # Translation
+ translator = Translator(llm_client)
+ await translator.translate(manifest_manager.entries, profile)
+ manifest_manager.save()
+
+ await llm_client.close()
+ else:
+ logger.info("Skipping translation step.")
+
+ # 5. Backfill
+ backfiller = BackfillEngine()
+ updated_structure = backfiller.backfill(structure, manifest_manager.entries, mode=args.mode)
+
+ # 6. Assembly - Pass original EPUB for TOC preservation
+ builder = BilingualBuilder(work["root"], original_epub_path=input_path)
+ output_filename = f"bilingual_{input_path.name}"
+ output_path = output_dir / output_filename
+
+ created_epub = builder.build(updated_structure, output_path)
+
+ logger.info(f"Pipeline completed! Output: {created_epub}")
+
+ except EpubTranslatorError as e:
+ logger.error(f"An error occurred: {e}")
+ sys.exit(1)
+ except Exception as e:
+ logger.critical(f"Unexpected error: {e}")
+ import traceback
+ traceback.print_exc()
+ sys.exit(1)
+
+def main():
+ parser = argparse.ArgumentParser(description="EPUB Bilingual Translator")
+ parser.add_argument("input_epub", help="Path to the input EPUB file")
+ parser.add_argument("--output-dir", default="output", help="Directory for output files")
+ parser.add_argument("--model", default="gpt-3.5-turbo", help="LLM Model to use")
+ parser.add_argument("--mode", default="bilingual", choices=["bilingual", "target_only"], help="Output mode")
+ parser.add_argument("--skip-translation", action="store_true", help="Skip LLM translation (for testing)")
+ parser.add_argument("--force-clean", action="store_true", help="Force re-clean EPUB even if book_structure exists")
+
+ args = parser.parse_args()
+
+ asyncio.run(run_pipeline(args))
+
+if __name__ == "__main__":
+ main()
+
diff --git a/archive/v0.11/backups/pre_refactor_20260127/scripts/build_translated_epub.py b/archive/v0.11/backups/pre_refactor_20260127/scripts/build_translated_epub.py
new file mode 100644
index 0000000..7f4b502
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/scripts/build_translated_epub.py
@@ -0,0 +1,51 @@
+import sys
+from pathlib import Path
+import traceback
+
+# Add src to path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from src.data_model import BookStructure, ManifestEntry
+from src.manifest_manager import ManifestManager
+from src.backfill_engine import BackfillEngine
+from src.bilingual_builder import BilingualBuilder
+from src.utils import setup_logger
+
+logger = setup_logger("build_final")
+
+def build_final():
+ BOOK_NAME = "Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)"
+ WORK_DIR = Path(f".work/{BOOK_NAME}")
+ MANIFEST_PATH = WORK_DIR / "manifest.json"
+ STRUCTURE_PATH = WORK_DIR / "book_structure.json"
+ OUTPUT_EPUB = Path("output/final_verification.epub")
+ INPUT_EPUB = Path(f"input/{BOOK_NAME}.epub")
+
+ if not MANIFEST_PATH.exists() or not STRUCTURE_PATH.exists():
+ logger.error("Missing manifest or structure. Run translation first.")
+ return
+
+ # 1. Load Data
+ logger.info("Loading structure and manifest...")
+ structure = BookStructure.load(STRUCTURE_PATH)
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+
+ # 2. Backfill
+ logger.info("Backfilling translations...")
+ backfiller = BackfillEngine()
+ structure = backfiller.backfill(structure, manager.entries, mode="bilingual")
+
+ # 3. Build
+ logger.info(f"Building final EPUB to {OUTPUT_EPUB}...")
+ builder = BilingualBuilder(WORK_DIR, original_epub_path=INPUT_EPUB)
+ builder.build(structure, OUTPUT_EPUB)
+
+ logger.info("Build complete.")
+
+if __name__ == "__main__":
+ try:
+ build_final()
+ except Exception as e:
+ traceback.print_exc()
+ print(f"CRITICAL ERROR: {e}")
diff --git a/archive/v0.11/backups/pre_refactor_20260127/scripts/test_backfill.py b/archive/v0.11/backups/pre_refactor_20260127/scripts/test_backfill.py
new file mode 100644
index 0000000..de51f29
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/scripts/test_backfill.py
@@ -0,0 +1,212 @@
+#!/usr/bin/env python3
+"""
+Placeholder Backfill Test Script
+Tests placeholder restoration on translated entries.
+
+Usage:
+ python scripts/test_backfill.py --chapter 38
+ python scripts/test_backfill.py --chapter 38 --limit 10
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from src.manifest_manager import ManifestManager
+from src.format_restorer import FormatRestorer
+
+# Configuration - Use unified .work directory
+BOOK_NAME = "Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)"
+WORK_DIR = Path(f".work/{BOOK_NAME}")
+MANIFEST_PATH = WORK_DIR / "manifest.json"
+
+
+def build_toc_from_manifest(manager: ManifestManager) -> list:
+ """Build TOC from manifest entries."""
+ files = {}
+ for entry in manager.entries:
+ fp = entry.file_path
+ if fp not in files:
+ files[fp] = {'count': 0, 'first_text': ''}
+ files[fp]['count'] += 1
+ if not files[fp]['first_text'] and entry.original_text:
+ files[fp]['first_text'] = entry.original_text[:40].replace('\n', ' ')
+
+ toc = []
+ for i, (fp, info) in enumerate(sorted(files.items()), 1):
+ toc.append({
+ 'index': i,
+ 'href': fp,
+ 'title': info['first_text'] or f"File {i}",
+ 'paragraphs': info['count']
+ })
+ return toc
+
+
+def get_chapter_entries(manager: ManifestManager, chapter_index: int) -> tuple:
+ """Get entries for a specific chapter by index."""
+ toc = build_toc_from_manifest(manager)
+
+ if chapter_index < 1 or chapter_index > len(toc):
+ print(f"错误: 章节编号 {chapter_index} 无效 (范围: 1-{len(toc)})")
+ return None, None
+
+ chapter = toc[chapter_index - 1]
+ href = chapter['href']
+
+ entries = [e for e in manager.entries if e.file_path == href]
+ return chapter, entries
+
+
+def test_backfill(chapter: dict, entries: list, limit: int = None):
+ """Test placeholder restoration for a chapter."""
+
+ print(f"\n" + "=" * 70)
+ print(f"占位符回填测试 - 章节 #{chapter['index']}: {chapter['title'][:40]}...")
+ print("=" * 70)
+
+ # Filter entries with translation and placeholders
+ translated = [e for e in entries if e.translated_text]
+ with_placeholders = [e for e in translated if e.placeholders and len(e.placeholders) > 0]
+
+ print(f"\n统计:")
+ print(f" 总段落: {len(entries)}")
+ print(f" 已翻译: {len(translated)}")
+ print(f" 有占位符: {len(with_placeholders)}")
+
+ if not translated:
+ print("\n⚠️ 该章节没有已翻译的内容!")
+ return
+
+ # Test restoration
+ restorer = FormatRestorer()
+
+ success_count = 0
+ fail_count = 0
+ results = []
+
+ test_entries = with_placeholders[:limit] if limit else with_placeholders
+
+ print(f"\n测试 {len(test_entries)} 个带占位符的段落:")
+ print("-" * 70)
+
+ for i, entry in enumerate(test_entries, 1):
+ original = entry.original_text
+ translated = entry.translated_text
+ placeholders = entry.placeholders
+
+ # Get non-internal placeholders
+ visible_ph = {k: v for k, v in placeholders.items() if not k.startswith('_')}
+
+ # Perform restoration
+ restored, success = restorer.restore(translated, placeholders)
+
+ if success:
+ success_count += 1
+ status = "✅"
+ else:
+ fail_count += 1
+ status = "❌"
+
+ results.append({
+ 'index': i,
+ 'entry_id': entry.entry_id,
+ 'original': original,
+ 'translated': translated,
+ 'restored': restored,
+ 'placeholders': visible_ph,
+ 'success': success
+ })
+
+ # Print summary
+ print(f"\n[{i}] {status} {entry.entry_id[-40:]}")
+ print(f" 占位符: {list(visible_ph.keys())}")
+ print(f" 原文: {original[:50]}...")
+ print(f" 译文: {translated[:50]}...")
+
+ if not success:
+ print(f" 还原: {restored[:50]}...")
+ # Show what placeholders are missing
+ missing = []
+ for k in visible_ph.keys():
+ if k.isdigit():
+ if f"φ{k}φ" not in translated and f"φ/{k}φ" not in translated:
+ missing.append(k)
+ if missing:
+ print(f" 缺失: {missing}")
+
+ # Summary
+ print("\n" + "=" * 70)
+ print(f"测试结果汇总")
+ print("=" * 70)
+ print(f" 成功: {success_count}/{len(test_entries)}")
+ print(f" 失败: {fail_count}/{len(test_entries)}")
+
+ if fail_count > 0:
+ print(f"\n失败案例详情:")
+ for r in results:
+ if not r['success']:
+ print(f"\n [{r['index']}] {r['entry_id'][-50:]}")
+ print(f" 原文: {r['original'][:60]}...")
+ print(f" 译文: {r['translated'][:60]}...")
+ print(f" 还原: {r['restored'][:60]}...")
+ print(f" 占位符: {r['placeholders']}")
+
+ # Also test entries without visible placeholders (only _prefix/_suffix)
+ prefix_suffix_only = [e for e in translated
+ if e.placeholders
+ and all(k.startswith('_') for k in e.placeholders.keys())]
+
+ if prefix_suffix_only:
+ print(f"\n\n额外测试: 只有 _prefix/_suffix 的段落 ({len(prefix_suffix_only)} 个)")
+ print("-" * 70)
+
+ ps_success = 0
+ ps_fail = 0
+
+ for entry in prefix_suffix_only[:5]: # Test first 5
+ restored, success = restorer.restore(entry.translated_text, entry.placeholders)
+ if success:
+ ps_success += 1
+ status = "✅"
+ else:
+ ps_fail += 1
+ status = "❌"
+
+ print(f" {status} {entry.entry_id[-40:]}")
+ if '_prefix' in entry.placeholders:
+ print(f" _prefix: {entry.placeholders['_prefix'][:30]}...")
+ if '_suffix' in entry.placeholders:
+ print(f" _suffix: {entry.placeholders['_suffix'][:30]}...")
+
+ print(f"\n 结果: {ps_success}/{min(5, len(prefix_suffix_only))} 成功")
+
+
+def main():
+ parser = argparse.ArgumentParser(description="测试占位符回填")
+ parser.add_argument("--chapter", "-c", type=int, required=True, help="章节编号")
+ parser.add_argument("--limit", "-l", type=int, default=20, help="测试数量限制 (默认20)")
+ args = parser.parse_args()
+
+ if not MANIFEST_PATH.exists():
+ print(f"错误: Manifest 不存在: {MANIFEST_PATH}")
+ return
+
+ # Load manifest
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+ print(f"已加载 manifest: {len(manager.entries)} 条目")
+
+ # Get chapter
+ chapter, entries = get_chapter_entries(manager, args.chapter)
+ if not chapter:
+ return
+
+ # Test backfill
+ test_backfill(chapter, entries, args.limit)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/archive/v0.11/backups/pre_refactor_20260127/scripts/test_conn.py b/archive/v0.11/backups/pre_refactor_20260127/scripts/test_conn.py
new file mode 100644
index 0000000..7e89b75
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/scripts/test_conn.py
@@ -0,0 +1,25 @@
+import asyncio
+import os
+import httpx
+
+async def test_conn():
+ print(f"HTTP_PROXY: {os.environ.get('http_proxy')}")
+ print(f"HTTPS_PROXY: {os.environ.get('https_proxy')}")
+ print(f"ALL_PROXY: {os.environ.get('all_proxy')}")
+
+ url = "https://api.gpt.ge/v1/models"
+ headers = {"Authorization": f"Bearer {os.environ.get('V3_API_KEY')}"}
+
+ print(f"Connecting to {url}...")
+ try:
+ async with httpx.AsyncClient(timeout=10, follow_redirects=True) as client:
+ resp = await client.get(url, headers=headers)
+ print(f"Status: {resp.status_code}")
+ print(f"Headers: {resp.headers}")
+ except Exception as e:
+ print(f"Error: {e}")
+
+if __name__ == "__main__":
+ from dotenv import load_dotenv
+ load_dotenv()
+ asyncio.run(test_conn())
diff --git a/archive/v0.11/backups/pre_refactor_20260127/scripts/test_openai_conn.py b/archive/v0.11/backups/pre_refactor_20260127/scripts/test_openai_conn.py
new file mode 100644
index 0000000..7f3b2fe
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/scripts/test_openai_conn.py
@@ -0,0 +1,40 @@
+import asyncio
+import os
+import httpx
+from openai import AsyncOpenAI
+
+async def test_openai():
+ proxy_url = os.environ.get("http_proxy")
+ print(f"Using proxy: {proxy_url}")
+
+ http_client = httpx.AsyncClient(
+ proxy=proxy_url,
+ timeout=30.0,
+ follow_redirects=True
+ )
+
+ client = AsyncOpenAI(
+ base_url="https://api.gpt.ge/v1",
+ api_key=os.environ.get("V3_API_KEY"),
+ http_client=http_client
+ )
+
+ print("Sending request...")
+ try:
+ response = await client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages=[{"role": "user", "content": "Hello"}],
+ max_tokens=5
+ )
+ print(f"Response: {response.choices[0].message.content}")
+ except Exception as e:
+ import traceback
+ traceback.print_exc()
+ print(f"Error: {e}")
+ finally:
+ await http_client.aclose()
+
+if __name__ == "__main__":
+ from dotenv import load_dotenv
+ load_dotenv()
+ asyncio.run(test_openai())
diff --git a/archive/v0.11/backups/pre_refactor_20260127/scripts/test_short_id_strategy.py b/archive/v0.11/backups/pre_refactor_20260127/scripts/test_short_id_strategy.py
new file mode 100644
index 0000000..45922eb
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/scripts/test_short_id_strategy.py
@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+"""
+Debug script to show chunk content and test short ID strategy.
+"""
+
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+from src.manifest_manager import ManifestManager
+from src.llm_client import LLMClient
+from src.data_model import ManifestEntry
+
+load_dotenv()
+
+# Configuration
+MANIFEST_PATH = Path("cache/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)_manifest.json")
+API_KEY = os.getenv("V3_API_KEY")
+BASE_URL = "https://api.gpt.ge/v1"
+MODEL = "gemini-3-flash-preview"
+EXTRA_HEADERS = {"x-foo": "true"}
+
+
+def show_current_chunk_format():
+ """Show current chunk format (problematic)."""
+ print("\n" + "="*60)
+ print("当前 Chunk 格式 (有问题)")
+ print("="*60)
+
+ # Load manifest
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+
+ # Get sample entries with placeholders
+ samples = [e for e in manager.entries if e.placeholders][:3]
+
+ print("\n发送给 LLM 的格式 (当前):")
+ print("-"*60)
+ for item in samples:
+ context = item.context or "BODY"
+ print(f"{item.entry_id} [{context}] {item.original_text[:50]}...")
+
+ print("\n问题分析:")
+ print(" 1. entry_id 太长 (包含 UUID): 容易被 LLM 截断或修改")
+ print(" 2. [BODY] context 没必要发送")
+ print(" 3. 依赖 LLM 精确复制长 ID，不可靠")
+
+
+def show_proposed_chunk_format():
+ """Show proposed short ID chunk format."""
+ print("\n" + "="*60)
+ print("建议的 Chunk 格式 (短 ID)")
+ print("="*60)
+
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+
+ samples = [e for e in manager.entries if e.placeholders][:5]
+
+ print("\n发送给 LLM 的格式 (建议):")
+ print("-"*60)
+
+ # Build with short IDs
+ id_map = {} # short_id -> entry_id
+ for i, item in enumerate(samples, 1):
+ short_id = f"#{i}"
+ id_map[short_id] = item.entry_id
+ text = item.original_text[:60]
+ print(f"{short_id}: {text}...")
+
+ print("\n期望 LLM 返回的格式:")
+ print("-"*60)
+ print("#1: φ1φpenguinrandomhouse.com（保持不翻译）")
+ print("#2: 该产品在欧盟的产品安全授权代表为 φ1φPenguin Random House Irelandφ/1φ...")
+ print("#3: φ1φ献辞")
+ print("#4: φ1φ题记")
+ print("#5: φ1φ作者说明")
+
+ print("\n优势:")
+ print(" 1. 短 ID (#1, #2...) 不会被 LLM 弄乱")
+ print(" 2. 去掉了无用的 context 标签")
+ print(" 3. 解析更可靠：用正则 ^#(\\d+): 匹配")
+ print(" 4. ID 映射表保留在代码中，用于还原")
+
+ print("\n映射表 (代码内保留):")
+ for short_id, full_id in id_map.items():
+ print(f" {short_id} -> {full_id[:50]}...")
+
+
+async def test_short_id_translation():
+ """Test translation with short ID format."""
+ print("\n" + "="*60)
+ print("测试短 ID 翻译")
+ print("="*60)
+
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+
+ # Get 5 entries with varied content
+ samples = [e for e in manager.entries if len(e.original_text) > 20][:5]
+
+ # Build prompt with short IDs
+ id_map = {}
+ lines = []
+ for i, item in enumerate(samples, 1):
+ short_id = f"#{i}"
+ id_map[short_id] = item.entry_id
+ text = item.original_text.replace('\n', ' ').strip()
+ lines.append(f"{short_id}: {text}")
+
+ user_prompt = "\n".join(lines)
+
+ print("\n发送给 LLM 的 Prompt:")
+ print("-"*60)
+ print(user_prompt)
+
+ # Create client
+ from openai import AsyncOpenAI
+ client = AsyncOpenAI(
+ base_url=BASE_URL,
+ api_key=API_KEY,
+ default_headers=EXTRA_HEADERS
+ )
+
+ system_prompt = """You are a professional English to Chinese translator.
+
+Translate each line to Chinese. Keep the format:
+- Each line starts with #N: (keep this ID exactly)
+- Preserve any φXφ placeholders exactly as-is
+- Only output translations, no explanations
+
+Example input:
+#1: Hello world
+#2: φ1φClick hereφ/1φ to continue
+
+Example output:
+#1: 你好世界
+#2: φ1φ点击这里φ/1φ 继续"""
+
+ print("\n发送请求...")
+
+ try:
+ resp = await client.chat.completions.create(
+ model=MODEL,
+ messages=[
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_prompt}
+ ],
+ temperature=0.3,
+ )
+ raw_response = resp.choices[0].message.content.strip()
+
+ print("\nLLM 返回:")
+ print("-"*60)
+ print(raw_response)
+
+ # Parse with short ID
+ print("\n解析结果:")
+ print("-"*60)
+ import re
+ results = {}
+ for line in raw_response.split("\n"):
+ line = line.strip()
+ match = re.match(r'^#(\d+):\s*(.+)$', line)
+ if match:
+ short_id = f"#{match.group(1)}"
+ translation = match.group(2)
+ if short_id in id_map:
+ full_id = id_map[short_id]
+ results[full_id] = translation
+ print(f" {short_id} -> {translation[:40]}...")
+
+ print(f"\n成功解析: {len(results)}/{len(samples)}")
+
+ finally:
+ await client.close()
+
+
+async def main():
+ print("="*60)
+ print("Chunk ID 策略分析与测试")
+ print("="*60)
+
+ if not MANIFEST_PATH.exists():
+ print(f"ERROR: Manifest not found at {MANIFEST_PATH}")
+ return
+
+ # 1. Show current format (problems)
+ show_current_chunk_format()
+
+ # 2. Show proposed format
+ show_proposed_chunk_format()
+
+ # 3. Test short ID translation
+ if API_KEY:
+ await test_short_id_translation()
+ else:
+ print("\n跳过测试 (V3_API_KEY 未设置)")
+
+ print("\n" + "="*60)
+ print("分析完成")
+ print("="*60)
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/archive/v0.11/backups/pre_refactor_20260127/scripts/test_translation_pipeline.py b/archive/v0.11/backups/pre_refactor_20260127/scripts/test_translation_pipeline.py
new file mode 100644
index 0000000..2a92890
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/scripts/test_translation_pipeline.py
@@ -0,0 +1,355 @@
+#!/usr/bin/env python3
+"""
+Translation Pipeline Debug Script
+Shows visible results at each step of the translation process.
+"""
+
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+
+# Add project root to path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+
+from src.manifest_manager import ManifestManager
+from src.llm_client import LLMClient
+from src.book_profiler import BookProfiler
+from src.translator import Translator
+from src.format_restorer import FormatRestorer
+from src.data_model import ManifestEntry, BookProfile
+
+load_dotenv()
+
+# Configuration
+MANIFEST_PATH = Path("cache/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)_manifest.json")
+API_KEY = os.getenv("V3_API_KEY")
+BASE_URL = "https://api.gpt.ge/v1"
+MODEL = "gemini-3-flash-preview"
+EXTRA_HEADERS = {"x-foo": "true"}
+
+# Chunk size configuration
+MAX_CHUNK_SIZE = 15 # Maximum entries per chunk
+
+
+def group_entries_by_file(entries: list) -> dict:
+ """Group manifest entries by their source file (chapter)."""
+ grouped = {}
+ for entry in entries:
+ file_path = entry.file_path
+ if file_path not in grouped:
+ grouped[file_path] = []
+ grouped[file_path].append(entry)
+ return grouped
+
+
+def create_chapter_aware_chunks(entries: list, max_size: int = MAX_CHUNK_SIZE) -> list:
+ """
+ Create chunks that respect chapter boundaries.
+ Returns list of (file_path, chunk_entries) tuples.
+ """
+ grouped = group_entries_by_file(entries)
+ chunks = []
+
+ for file_path, file_entries in grouped.items():
+ # Split this file's entries into chunks of max_size
+ for i in range(0, len(file_entries), max_size):
+ chunk = file_entries[i:i + max_size]
+ chunks.append((file_path, chunk))
+
+ return chunks
+
+
+def get_file_type(file_path: str) -> str:
+ """Determine the type of content based on file name."""
+ fname = file_path.lower()
+ if any(k in fname for k in ['toc', 'contents', 'nav']):
+ return 'toc'
+ elif any(k in fname for k in ['title', 'cover']):
+ return 'cover'
+ elif any(k in fname for k in ['copyright', 'colophon']):
+ return 'legal'
+ elif any(k in fname for k in ['author', 'about']):
+ return 'author_bio'
+ elif any(k in fname for k in ['index', 'bibliography', 'endnote', 'footnote']):
+ return 'reference'
+ else:
+ return 'body'
+
+
+async def step1_load_manifest():
+ """Step 1: Load manifest and show statistics."""
+ print("\n" + "="*60)
+ print("STEP 1: Loading Manifest")
+ print("="*60)
+
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+
+ entries = manager.entries
+ untranslated = [e for e in entries if not e.translated_text]
+
+ print(f" Total entries: {len(entries)}")
+ print(f" Untranslated: {len(untranslated)}")
+
+ # Show grouping by file
+ grouped = group_entries_by_file(entries)
+ print(f" Unique files: {len(grouped)}")
+
+ # Show sample entry
+ if entries:
+ sample = entries[0]
+ print(f"\n Sample entry:")
+ print(f" ID: {sample.entry_id}")
+ print(f" File: {sample.file_path}")
+ print(f" Original: {sample.original_text[:80]}...")
+ print(f" Placeholders: {sample.placeholders}")
+
+ return manager
+
+
+async def step2_profile_book(manager: ManifestManager, llm_client: LLMClient):
+ """Step 2: Generate book profile."""
+ print("\n" + "="*60)
+ print("STEP 2: Generating Book Profile")
+ print("="*60)
+
+ profiler = BookProfiler(llm_client)
+ profile = await profiler.analyze(manager.entries)
+
+ print(f" Title: {profile.title}")
+ print(f" Author: {profile.author}")
+ print(f" Genre: {profile.genre}")
+ print(f" Keywords: {profile.keywords}")
+ print(f" Style Guide: {profile.style_guide[:200]}..." if profile.style_guide else " Style Guide: (none)")
+
+ return profile
+
+
+async def step3_create_chunks(manager: ManifestManager):
+ """Step 3: Create chapter-aware chunks."""
+ print("\n" + "="*60)
+ print("STEP 3: Creating Chapter-Aware Chunks")
+ print("="*60)
+
+ untranslated = [e for e in manager.entries if not e.translated_text]
+ chunks = create_chapter_aware_chunks(untranslated)
+
+ print(f" Total chunks: {len(chunks)}")
+
+ # Show chunk distribution
+ print(f"\n Chunk distribution by file type:")
+ type_counts = {}
+ for file_path, chunk_entries in chunks:
+ ftype = get_file_type(file_path)
+ type_counts[ftype] = type_counts.get(ftype, 0) + 1
+
+ for ftype, count in sorted(type_counts.items()):
+ print(f" {ftype}: {count} chunks")
+
+ # Show first 3 chunks
+ print(f"\n First 3 chunks:")
+ for i, (file_path, chunk_entries) in enumerate(chunks[:3]):
+ ftype = get_file_type(file_path)
+ print(f" [{i}] {file_path} ({ftype}): {len(chunk_entries)} entries")
+ if chunk_entries:
+ print(f" First: {chunk_entries[0].original_text[:50]}...")
+
+ return chunks
+
+
+async def step4_translate_sample(chunks: list, llm_client: LLMClient, profile: BookProfile):
+ """Step 4: Translate a sample chunk and show results."""
+ print("\n" + "="*60)
+ print("STEP 4: Translating Sample Chunk")
+ print("="*60)
+
+ if not chunks:
+ print(" No chunks to translate!")
+ return
+
+ # Pick a proper body chapter (skip first few files which are usually cover/copyright/toc)
+ sample_chunk = None
+ skip_prefixes = ['cM', 'c9', 'c18'] # Cover, title, contents pages
+ for file_path, chunk_entries in chunks:
+ # Skip non-body files and known cover/toc files
+ ftype = get_file_type(file_path)
+ fname = Path(file_path).stem
+ if ftype == 'body' and fname not in skip_prefixes and len(chunk_entries) > 3:
+ sample_chunk = (file_path, chunk_entries[:5]) # Limit to 5 entries for demo
+ break
+
+ if not sample_chunk:
+ # Fallback to any body chunk
+ for file_path, chunk_entries in chunks:
+ if get_file_type(file_path) == 'body':
+ sample_chunk = (file_path, chunk_entries[:5])
+ break
+
+ if not sample_chunk:
+ sample_chunk = chunks[0]
+ sample_chunk = (sample_chunk[0], sample_chunk[1][:5])
+
+ file_path, entries = sample_chunk
+ ftype = get_file_type(file_path)
+
+ print(f" Selected chunk: {file_path} ({ftype})")
+ print(f" Entries: {len(entries)}")
+
+ # Show entries before translation
+ print(f"\n === Before Translation ===")
+ for i, entry in enumerate(entries):
+ print(f" [{i}] {entry.entry_id}")
+ print(f" Original: {entry.original_text[:60]}...")
+ if entry.placeholders:
+ print(f" Placeholders: {list(entry.placeholders.keys())}")
+
+ # Translate
+ print(f"\n Translating...")
+ results = await llm_client.translate_chunk(
+ entries,
+ instruction=profile.style_guide,
+ mode="bilingual"
+ )
+
+ # Apply results and show
+ print(f"\n === After Translation ===")
+ restorer = FormatRestorer()
+ for i, entry in enumerate(entries):
+ if entry.entry_id in results:
+ translated = results[entry.entry_id]
+ entry.translated_text = translated
+
+ print(f" [{i}] {entry.entry_id}")
+ print(f" Original: {entry.original_text[:50]}...")
+ print(f" Translated: {translated[:50]}...")
+
+ # Restore format
+ if entry.placeholders:
+ restored, success = restorer.restore(translated, entry.placeholders)
+ print(f" Restored OK: {success}")
+ if not success:
+ print(f" Restored: {restored[:50]}...")
+ else:
+ print(f" [{i}] MISSING: {entry.entry_id}")
+
+ return entries
+
+
+async def step5_test_placeholders(manager: ManifestManager, llm_client: LLMClient, profile: BookProfile):
+ """Step 5: Test placeholder handling with entries that have placeholders."""
+ print("\n" + "="*60)
+ print("STEP 5: Testing Placeholder Handling")
+ print("="*60)
+
+ # Find entries with placeholders
+ entries_with_ph = [e for e in manager.entries if e.placeholders and len(e.placeholders) > 1]
+
+ print(f" Entries with placeholders: {len(entries_with_ph)}")
+
+ if not entries_with_ph:
+ print(" No entries with placeholders found!")
+ return
+
+ # Pick 5 diverse samples
+ samples = entries_with_ph[:5]
+
+ print(f"\n === Selected Samples ({len(samples)}) ===")
+ for i, entry in enumerate(samples):
+ ph_keys = [k for k in entry.placeholders.keys() if not k.startswith('_')]
+ print(f" [{i}] {entry.entry_id}")
+ print(f" Original: {entry.original_text[:60]}...")
+ print(f" Placeholders: {ph_keys}")
+
+ # Translate
+ print(f"\n Translating {len(samples)} entries with placeholders...")
+ results = await llm_client.translate_chunk(
+ samples,
+ instruction=profile.style_guide,
+ mode="bilingual"
+ )
+
+ # Show results with restoration
+ print(f"\n === Translation Results ===")
+ restorer = FormatRestorer()
+ success_count = 0
+
+ for i, entry in enumerate(samples):
+ print(f"\n [{i}] {entry.entry_id}")
+ print(f" Original: {entry.original_text[:50]}...")
+
+ if entry.entry_id in results:
+ translated = results[entry.entry_id]
+ print(f" Translated: {translated[:50]}...")
+
+ # Check if placeholders are preserved
+ ph_keys = [k for k in entry.placeholders.keys() if not k.startswith('_')]
+ preserved = all(f"φ{k}φ" in translated or f"φ/{k}φ" in translated for k in ph_keys if k.isdigit())
+ print(f" PH Preserved: {preserved}")
+
+ # Restore format
+ restored, success = restorer.restore(translated, entry.placeholders)
+ print(f" Restore OK: {success}")
+
+ if success:
+ success_count += 1
+ else:
+ print(f" Restored: {restored[:50]}...")
+ else:
+ print(f" MISSING from results!")
+
+ print(f"\n Summary: {success_count}/{len(samples)} restored successfully")
+
+
+async def main():
+ """Run all steps."""
+ print("="*60)
+ print("TRANSLATION PIPELINE DEBUG")
+ print("="*60)
+
+ if not API_KEY:
+ print("ERROR: V3_API_KEY not found in .env")
+ return
+
+ if not MANIFEST_PATH.exists():
+ print(f"ERROR: Manifest not found at {MANIFEST_PATH}")
+ print("Run the main pipeline first to generate the manifest.")
+ return
+
+ # Initialize LLM client
+ llm_client = LLMClient(
+ api_key=API_KEY,
+ base_url=BASE_URL,
+ model=MODEL,
+ extra_headers=EXTRA_HEADERS
+ )
+
+ try:
+ # Step 1: Load manifest
+ manager = await step1_load_manifest()
+
+ # Step 2: Profile book
+ profile = await step2_profile_book(manager, llm_client)
+
+ # Step 3: Create chunks
+ chunks = await step3_create_chunks(manager)
+
+ # Step 4: Translate sample (simple text)
+ await step4_translate_sample(chunks, llm_client, profile)
+
+ # Step 5: Test placeholders
+ await step5_test_placeholders(manager, llm_client, profile)
+
+ print("\n" + "="*60)
+ print("DEBUG COMPLETE")
+ print("="*60)
+
+ finally:
+ await llm_client.close()
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/archive/v0.11/backups/pre_refactor_20260127/scripts/translate_chapter.py b/archive/v0.11/backups/pre_refactor_20260127/scripts/translate_chapter.py
new file mode 100644
index 0000000..8dd0688
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/scripts/translate_chapter.py
@@ -0,0 +1,353 @@
+#!/usr/bin/env python3
+"""
+Chapter Translation Test Script
+Translate a complete chapter to test the full pipeline.
+
+Usage:
+ python scripts/translate_chapter.py --show-toc # 显示章节目录
+ python scripts/translate_chapter.py --chapter 5 # 翻译第5章
+ python scripts/translate_chapter.py --chapter 5 --test # 测试模式，只翻译前2个chunk
+"""
+
+import argparse
+import asyncio
+import json
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+from src.manifest_manager import ManifestManager
+from src.llm_client import LLMClient
+from src.book_profiler import BookProfiler
+from src.format_restorer import FormatRestorer
+from src.data_model import ManifestEntry, BookStructure
+
+load_dotenv()
+
+# Configuration - Use unified .work directory
+BOOK_NAME = "Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao)"
+WORK_DIR = Path(f".work/{BOOK_NAME}")
+MANIFEST_PATH = WORK_DIR / "manifest.json"
+STRUCTURE_PATH = WORK_DIR / "book_structure.json"
+CHUNK_DIR = WORK_DIR / "chunks"
+
+API_KEY = os.getenv("V3_API_KEY")
+BASE_URL = "https://api.gpt.ge/v1"
+MODEL = "gpt-4o-mini" # "gemini-3-flash-preview"
+EXTRA_HEADERS = {"x-foo": "true"}
+
+# Chunk config - around 5000 chars per chunk
+CHUNK_SIZE_CHARS = 5000
+
+
+def load_toc_from_structure() -> list:
+ """Load TOC from book structure for readable chapter names."""
+ if not STRUCTURE_PATH.exists():
+ return []
+
+ try:
+ structure = BookStructure.load(STRUCTURE_PATH)
+ # Build TOC from spine order with chapter titles
+ toc = []
+ for i, item_id in enumerate(structure.spine, 1):
+ if item_id in structure.resources:
+ resource = structure.resources[item_id]
+ href = resource.href
+
+ # Try to extract title from content
+ title = extract_title_from_html(resource.content) if resource.content else None
+
+ toc.append({
+ 'index': i,
+ 'item_id': item_id,
+ 'href': href,
+ 'title': title or f"Chapter {i}"
+ })
+ return toc
+ except Exception as e:
+ print(f"警告: 无法加载书籍结构: {e}")
+ return []
+
+
+def extract_title_from_html(html: str) -> str:
+ """Extract title from HTML content."""
+ from bs4 import BeautifulSoup
+ soup = BeautifulSoup(html, 'html.parser')
+
+ # Try h1, h2, h3 in order
+ for tag in ['h1', 'h2', 'h3']:
+ elem = soup.find(tag)
+ if elem:
+ return elem.get_text().strip()[:50]
+
+ # Try first paragraph
+ p = soup.find('p')
+ if p:
+ text = p.get_text().strip()[:50]
+ if text:
+ return text + "..."
+
+ return None
+
+
+def build_toc_from_manifest(manager: ManifestManager) -> list:
+ """Build TOC from manifest entries."""
+ files = {}
+ for entry in manager.entries:
+ fp = entry.file_path
+ if fp not in files:
+ files[fp] = {
+ 'count': 0,
+ 'first_text': '',
+ 'total_chars': 0
+ }
+ files[fp]['count'] += 1
+ files[fp]['total_chars'] += len(entry.original_text)
+ if not files[fp]['first_text'] and entry.original_text:
+ files[fp]['first_text'] = entry.original_text[:40].replace('\n', ' ')
+
+ toc = []
+ for i, (fp, info) in enumerate(sorted(files.items()), 1):
+ toc.append({
+ 'index': i,
+ 'href': fp,
+ 'title': info['first_text'] or f"File {i}",
+ 'paragraphs': info['count'],
+ 'chars': info['total_chars']
+ })
+ return toc
+
+
+def show_toc(manager: ManifestManager):
+ """Display TOC with chapter numbers."""
+ toc = build_toc_from_manifest(manager)
+
+ print("\n" + "=" * 70)
+ print("章节目录 (Table of Contents)")
+ print("=" * 70)
+ print(f"{'#':>3} | {'段落':>5} | {'字符':>6} | 章节标题")
+ print("-" * 70)
+
+ for item in toc:
+ title = item['title'][:45] if len(item['title']) > 45 else item['title']
+ print(f"{item['index']:3d} | {item['paragraphs']:5d} | {item['chars']:6d} | {title}")
+
+ print("-" * 70)
+ print(f"共 {len(toc)} 个章节")
+ print("\n用法: python scripts/translate_chapter.py --chapter <编号>")
+ print("示例: python scripts/translate_chapter.py --chapter 5")
+
+
+def get_chapter_entries(manager: ManifestManager, chapter_index: int) -> tuple:
+ """Get entries for a specific chapter by index."""
+ toc = build_toc_from_manifest(manager)
+
+ if chapter_index < 1 or chapter_index > len(toc):
+ print(f"错误: 章节编号 {chapter_index} 无效 (范围: 1-{len(toc)})")
+ return None, None
+
+ chapter = toc[chapter_index - 1]
+ href = chapter['href']
+
+ entries = [e for e in manager.entries if e.file_path == href]
+ return chapter, entries
+
+
+def create_char_based_chunks(entries: list, chunk_size: int = CHUNK_SIZE_CHARS) -> list:
+ """
+ Create chunks based on character count (~5000 chars each).
+ Returns list of entry lists.
+ """
+ chunks = []
+ current_chunk = []
+ current_size = 0
+
+ for entry in entries:
+ text_len = len(entry.original_text)
+
+ # If adding this entry exceeds limit and we have content, start new chunk
+ if current_size + text_len > chunk_size and current_chunk:
+ chunks.append(current_chunk)
+ current_chunk = []
+ current_size = 0
+
+ current_chunk.append(entry)
+ current_size += text_len
+
+ if current_chunk:
+ chunks.append(current_chunk)
+
+ return chunks
+
+
+async def translate_chapter(chapter: dict, entries: list, manager: ManifestManager,
+ llm_client: LLMClient, profile, test_mode: bool = False):
+ """Translate a complete chapter."""
+
+ print(f"\n开始翻译章节 #{chapter['index']}: {chapter['title'][:40]}...")
+ print(f" 文件: {chapter['href']}")
+ print(f" 总段落: {len(entries)}")
+
+ # Filter untranslated
+ untranslated = [e for e in entries if not e.translated_text]
+ print(f" 待翻译: {len(untranslated)}")
+
+ if not untranslated:
+ print(" ✅ 该章节已全部翻译!")
+ return
+
+ # Create character-based chunks
+ chunks = create_char_based_chunks(untranslated)
+ print(f" 分块: {len(chunks)} 个 Chunk (约{CHUNK_SIZE_CHARS}字符/块)")
+
+ if test_mode:
+ print(" [测试模式] 只翻译前2个 Chunk")
+ chunks = chunks[:2]
+
+ # Show chunk stats
+ for i, chunk in enumerate(chunks, 1):
+ total_chars = sum(len(e.original_text) for e in chunk)
+ print(f" Chunk {i}: {len(chunk)} 段落, {total_chars} 字符")
+
+ # Translate
+ restorer = FormatRestorer()
+ total_success = 0
+ total_failed = 0
+
+ for i, chunk in enumerate(chunks, 1):
+ chunk_chars = sum(len(e.original_text) for e in chunk)
+ print(f"\n 翻译 Chunk {i}/{len(chunks)} ({len(chunk)} 段, {chunk_chars} 字符)...")
+
+ try:
+ results = await llm_client.translate_chunk(
+ chunk,
+ instruction=profile.style_guide if hasattr(profile, 'style_guide') else None,
+ mode="bilingual"
+ )
+
+ # Apply results
+ chunk_success = 0
+ chunk_failed = 0
+
+ for entry in chunk:
+ if entry.entry_id in results:
+ translated = results[entry.entry_id]
+ entry.translated_text = translated
+
+ # Verify placeholder restoration
+ if entry.placeholders:
+ _, success = restorer.restore(translated, entry.placeholders)
+ if success:
+ chunk_success += 1
+ else:
+ chunk_failed += 1
+ print(f" ⚠️ 占位符还原警告: {entry.entry_id[-30:]}")
+ else:
+ chunk_success += 1
+ else:
+ chunk_failed += 1
+ print(f" ❌ 缺失: {entry.entry_id[-30:]}")
+
+ total_success += chunk_success
+ total_failed += chunk_failed
+ print(f" ✓ 成功: {chunk_success}, 失败: {chunk_failed}")
+
+ # Save after each chunk
+ manager.save()
+
+ except Exception as e:
+ print(f" ❌ Chunk {i} 翻译失败: {e}")
+ total_failed += len(chunk)
+
+ print(f"\n翻译完成:")
+ print(f" ✅ 成功: {total_success}")
+ print(f" ❌ 失败: {total_failed}")
+
+ # Show sample results
+ print(f"\n翻译样例 (前3段):")
+ print("-" * 60)
+ translated_entries = [e for e in entries if e.translated_text][:3]
+ for entry in translated_entries:
+ orig = entry.original_text[:40].replace('\n', ' ')
+ trans = entry.translated_text[:40].replace('\n', ' ') if entry.translated_text else "(无)"
+ print(f" 原: {orig}...")
+ print(f" 译: {trans}...")
+ print()
+
+
+async def main():
+ parser = argparse.ArgumentParser(description="翻译指定章节")
+ parser.add_argument("--show-toc", action="store_true", help="显示章节目录")
+ parser.add_argument("--chapter", "-c", type=int, help="章节编号 (从1开始)")
+ parser.add_argument("--test", "-t", action="store_true", help="测试模式 (只翻译前2个chunk)")
+ args = parser.parse_args()
+
+ if not MANIFEST_PATH.exists():
+ print(f"错误: Manifest 不存在: {MANIFEST_PATH}")
+ print("请先运行主管道生成 manifest。")
+ return
+
+ # Load manifest
+ manager = ManifestManager(MANIFEST_PATH)
+ manager.load()
+ print(f"已加载 manifest: {len(manager.entries)} 条目")
+
+ # Show TOC
+ if args.show_toc or not args.chapter:
+ show_toc(manager)
+ return
+
+ if not API_KEY:
+ print("错误: V3_API_KEY 未设置")
+ return
+
+ # Get chapter entries
+ chapter, entries = get_chapter_entries(manager, args.chapter)
+ if not chapter:
+ return
+
+ # Initialize LLM client
+ from src.utils import ensure_directory
+ ensure_directory(CHUNK_DIR)
+
+ llm_client = LLMClient(
+ api_key=API_KEY,
+ base_url=BASE_URL,
+ model=MODEL,
+ extra_headers=EXTRA_HEADERS,
+ chunk_dir=CHUNK_DIR
+ )
+
+ try:
+ # Generate profile
+ print("\n生成书籍 Profile... (Skipping for debug)")
+ # profiler = BookProfiler(llm_client)
+ # profile = await profiler.analyze(manager.entries)
+ # print(f" 风格: {profile.style_guide[:80] if profile.style_guide else '(无)'}...")
+
+ class DummyProfile:
+ style_guide = "Keep technical terms. Translate accurately."
+ profile = DummyProfile()
+
+ # Translate chapter
+ await translate_chapter(chapter, entries, manager, llm_client, profile, args.test)
+
+ print(f"\n✅ Manifest 已保存: {MANIFEST_PATH}")
+ print(f"✅ Chunk 文件保存在: {CHUNK_DIR}")
+
+ finally:
+ await llm_client.close()
+
+
+if __name__ == "__main__":
+ print("DEBUG: Script started execution")
+ try:
+ asyncio.run(main())
+ print("DEBUG: Script finished execution")
+ except Exception as e:
+ import traceback
+ traceback.print_exc()
+ print(f"CRITICAL ERROR: {e}")
diff --git a/archive/v0.11/backups/pre_refactor_20260127/scripts/verify_toc_fix.py b/archive/v0.11/backups/pre_refactor_20260127/scripts/verify_toc_fix.py
new file mode 100644
index 0000000..47deb9a
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/scripts/verify_toc_fix.py
@@ -0,0 +1,47 @@
+import sys
+from pathlib import Path
+import traceback
+
+# Add src to path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from src.epub_cleaner import EpubCleaner
+from src.bilingual_builder import BilingualBuilder
+from src.utils import setup_logger
+from src.data_model import BookStructure
+
+logger = setup_logger("verify_toc")
+
+def verify_fix():
+ INPUT_EPUB = Path("input/Empire of AI Dreams and Nightmares in Sam Altmans OpenAI (Karen Hao).epub")
+ OUTPUT_EPUB = Path("output/verify_toc.epub")
+ WORK_DIR = Path("tmp/verify_toc")
+
+ if not INPUT_EPUB.exists():
+ logger.error(f"Input not found: {INPUT_EPUB}")
+ return
+
+ # Clean
+ logger.info("Step 1: Cleaning EPUB...")
+ cleaner = EpubCleaner(INPUT_EPUB, WORK_DIR)
+ json_path = cleaner.clean()
+
+ # Load structure
+ with open(json_path, 'r') as f:
+ structure = BookStructure.model_validate_json(f.read())
+
+ # Build
+ logger.info("Step 2: Building EPUB with TOC preservation...")
+ builder = BilingualBuilder(WORK_DIR, original_epub_path=INPUT_EPUB)
+ builder.build(structure, OUTPUT_EPUB)
+
+ logger.info(f"Step 3: EPUB generated at {OUTPUT_EPUB}")
+
+if __name__ == "__main__":
+ print("Starting verification script...")
+ try:
+ verify_fix()
+ print("Verification script finished.")
+ except Exception as e:
+ traceback.print_exc()
+ print(f"CRITICAL ERROR: {e}")
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/__init__.py b/archive/v0.11/backups/pre_refactor_20260127/src/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/backfill_engine.py b/archive/v0.11/backups/pre_refactor_20260127/src/backfill_engine.py
new file mode 100644
index 0000000..337a075
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/src/backfill_engine.py
@@ -0,0 +1,96 @@
+from pathlib import Path
+from typing import List, Dict, Optional
+from bs4 import BeautifulSoup
+
+from src.data_model import BookStructure, ManifestEntry
+from src.format_restorer import FormatRestorer
+from src.utils import setup_logger
+
+logger = setup_logger("backfill_engine")
+
+class BackfillEngine:
+ """
+ Applies translations back to the BookStructure.
+ """
+
+ def __init__(self):
+ self.restorer = FormatRestorer()
+
+ def backfill(self, structure: BookStructure, manifest_entries: List[ManifestEntry], mode: str = "bilingual") -> BookStructure:
+ """
+ Modifies the BookStructure in-place with translations.
+
+ Args:
+ structure: The BookStructure (from book_structure.json).
+ manifest_entries: List of translations.
+ mode: 'bilingual' or 'target_only'.
+ """
+ logger.info(f"Backfilling with mode: {mode}")
+
+ # Index manifest by file and element ID for faster lookup
+ # Map: file_path -> element_id -> ManifestEntry
+ manifest_map: Dict[str, Dict[str, ManifestEntry]] = {}
+ for entry in manifest_entries:
+ if not entry.translated_text:
+ continue # Skip untranslated entries
+
+ if entry.file_path not in manifest_map:
+ manifest_map[entry.file_path] = {}
+ manifest_map[entry.file_path][entry.element_id] = entry
+
+ # Iterate resources in structure
+ for item_id, resource in structure.resources.items():
+ if resource.media_type != "application/xhtml+xml" or resource.href not in manifest_map:
+ continue
+
+ file_entries = manifest_map[resource.href]
+ if not file_entries:
+ continue
+
+ logger.debug(f"Processing {resource.href} with {len(file_entries)} translations")
+
+ soup = BeautifulSoup(resource.content, 'html.parser')
+ modified = False
+
+ for element_id, entry in file_entries.items():
+ element = soup.find(id=element_id)
+ if not element:
+ logger.warning(f"Element {element_id} not found in {resource.href}")
+ continue
+
+ # Restore formatting
+ restored_html, _ = self.restorer.restore(entry.translated_text, entry.placeholders)
+
+ # Create translated tag
+ new_tag = soup.new_tag(element.name)
+ # Parse restored HTML to get content nodes
+ inner_soup = BeautifulSoup(restored_html, 'html.parser')
+ if inner_soup.body:
+ for child in list(inner_soup.body.children):
+ new_tag.append(child)
+ else:
+ for child in list(inner_soup.children):
+ new_tag.append(child)
+
+ # Copy classes and add 'translation'
+ classes = element.get('class', [])
+ if isinstance(classes, str):
+ classes = classes.split()
+ new_tag['class'] = classes + ['translation']
+
+ # Copy style
+ style = element.get('style')
+ if style:
+ new_tag['style'] = style
+
+ if mode == "bilingual":
+ element.insert_after(new_tag)
+ else:
+ element.replace_with(new_tag)
+
+ modified = True
+
+ if modified:
+ resource.content = str(soup)
+
+ return structure
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/bilingual_builder.py b/archive/v0.11/backups/pre_refactor_20260127/src/bilingual_builder.py
new file mode 100644
index 0000000..fb69d2b
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/src/bilingual_builder.py
@@ -0,0 +1,252 @@
+import shutil
+import uuid
+from pathlib import Path
+from ebooklib import epub
+from src.data_model import BookStructure
+from src.utils import setup_logger, ensure_directory
+
+logger = setup_logger("bilingual_builder")
+
+class BilingualBuilder:
+ """
+ Assembles the final EPUB from BookStructure.
+
+ Preserves the original TOC structure by reading it from the original EPUB.
+ """
+
+ def __init__(self, work_dir: Path, original_epub_path: Path = None):
+ self.work_dir = work_dir
+ self.assets_dir = work_dir / "assets"
+ self.original_epub_path = original_epub_path
+ self._original_book = None
+
+ def _load_original_book(self):
+ """Lazy load original book for TOC extraction."""
+ if self._original_book is None and self.original_epub_path and self.original_epub_path.exists():
+ try:
+ self._original_book = epub.read_epub(str(self.original_epub_path))
+ logger.debug(f"Loaded original EPUB for TOC: {self.original_epub_path}")
+ except Exception as e:
+ logger.warning(f"Failed to load original EPUB: {e}")
+ return self._original_book
+
+ def _sanitize_toc(self, toc):
+ """
+ Ensure all TOC nodes have IDs (for ebooklib compatibility).
+ From v0.08 bilingual_builder.py
+ """
+ result = []
+ for item in toc:
+ if isinstance(item, (epub.Link, epub.Section)):
+ if not getattr(item, 'uid', None):
+ item.uid = f"navPoint-{uuid.uuid4().hex[:8]}"
+ result.append(item)
+ elif isinstance(item, tuple) and len(item) == 2:
+ # Handle (Section, [children]) structure
+ section, children = item
+ if isinstance(section, (epub.Link, epub.Section)):
+ if not getattr(section, 'uid', None):
+ section.uid = f"navPoint-{uuid.uuid4().hex[:8]}"
+ sanitized_children = self._sanitize_toc(children)
+ result.append((section, sanitized_children))
+ else:
+ result.append(item)
+ return result
+
+ def _validate_and_fix_toc(self, toc, book_items):
+ """
+ Recursively validate and fix TOC links.
+ Removes nodes with broken links that cannot be fixed.
+ """
+ fixed_toc = []
+ for item in toc:
+ if isinstance(item, (epub.Link, epub.Section)):
+ # Check href
+ href = getattr(item, 'href', '')
+ if href:
+ # Remove anchor for check
+ clean_href = href.split('#')[0]
+ # Check if item exists in book (by file_name)
+ found = False
+ for existing_item in book_items.values():
+ if existing_item.file_name == clean_href:
+ found = True
+ break
+ if clean_href.endswith(existing_item.file_name) or existing_item.file_name.endswith(clean_href):
+ item.href = existing_item.file_name
+ found = True
+ break
+
+ if not found:
+ if 'c0.xhtml' in clean_href:
+ for existing_item in book_items.values():
+ if 'titlepage' in existing_item.file_name or 'cover' in existing_item.file_name.lower():
+ if existing_item.media_type == "application/xhtml+xml":
+ logger.info(f"Fixed TOC link: {href} -> {existing_item.file_name}")
+ item.href = existing_item.file_name
+ found = True
+ break
+
+ if not found:
+ logger.warning(f"Removing broken TOC link: {href}")
+ continue
+
+ if isinstance(item, tuple) and len(item) == 2:
+ section, children = item
+ fixed_children = self._validate_and_fix_toc(children, book_items)
+ fixed_toc.append((section, fixed_children))
+ else:
+ fixed_toc.append(item)
+
+ return fixed_toc
+
+ def build(self, structure: BookStructure, output_path: Path) -> Path:
+ """
+ Builds the EPUB file.
+ Returns the path to the generated EPUB.
+ """
+ logger.info(f"Building final EPUB: {output_path}")
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+
+ book = epub.EpubBook()
+
+ # 1. Metadata
+ book.set_identifier(structure.metadata.identifier or f"uuid-{uuid.uuid4().hex[:12]}")
+ book.set_title(structure.metadata.title)
+ book.set_language(structure.metadata.language)
+ book.add_author(structure.metadata.author)
+
+ # 2. Copy TOC from original EPUB if available
+ original_book = self._load_original_book()
+ if original_book and hasattr(original_book, 'toc') and original_book.toc:
+ book.toc = self._sanitize_toc(original_book.toc)
+ logger.info("Copied TOC structure from original EPUB")
+
+
+
+ # 3. Add Resources - First pass: Collect CSS items
+ items_map = {} # id -> epub_item
+ css_items = [] # List of CSS EpubItem for linking
+ html_items = [] # List of (item_id, EpubHtml) tuples
+
+ for item_id, resource in structure.resources.items():
+ # Skip NCX - we'll handle it separately
+ if resource.media_type == "application/x-dtbncx+xml":
+ continue
+
+ if resource.media_type == "application/xhtml+xml":
+ # HTML Item - create but don't add yet (need to add CSS links)
+ item = epub.EpubHtml(
+ uid=item_id,
+ file_name=resource.href,
+ media_type=resource.media_type,
+ content=resource.content.encode('utf-8')
+ )
+ html_items.append((item_id, item))
+ elif resource.file_path:
+ # Binary/Asset Item
+ asset_full_path = self.work_dir / resource.file_path
+
+ if not asset_full_path.exists():
+ logger.warning(f"Asset missing: {asset_full_path}")
+ continue
+
+ with open(asset_full_path, 'rb') as f:
+ content = f.read()
+
+ if "image" in resource.media_type:
+ item = epub.EpubImage(
+ uid=item_id,
+ file_name=resource.href,
+ media_type=resource.media_type,
+ content=content
+ )
+ else:
+ item = epub.EpubItem(
+ uid=item_id,
+ file_name=resource.href,
+ media_type=resource.media_type,
+ content=content
+ )
+ # Track CSS items
+ if resource.media_type == "text/css":
+ css_items.append(item)
+
+ book.add_item(item)
+ items_map[item_id] = item
+ else:
+ logger.warning(f"Skipping resource {item_id}: No content or file path.")
+ continue
+
+ # 4. Add HTML items with CSS links
+ for item_id, item in html_items:
+ html_dir = Path(item.file_name).parent
+ for css_item in css_items:
+ css_path = Path(css_item.file_name)
+ # Calculate relative path from HTML directory to CSS file
+ try:
+ relative_css_path = Path(css_path).relative_to(html_dir)
+ except ValueError:
+ # Not a subpath, calculate full relative
+ # Go up from html_dir, then down to css_path
+ up_count = len(html_dir.parts)
+ relative_css_path = Path("/".join([".."] * up_count)) / css_path
+
+ item.add_link(href=str(relative_css_path), rel='stylesheet', type='text/css')
+ book.add_item(item)
+ items_map[item_id] = item
+
+ # 5. Copy missing items from original EPUB (cover, etc.)
+ # This ensures TOC links don't break
+ if original_book:
+ added_hrefs = {item.file_name for item in items_map.values() if hasattr(item, 'file_name')}
+
+ for orig_item in original_book.get_items():
+ orig_name = orig_item.get_name()
+ if orig_name not in added_hrefs:
+ # Skip NCX and NAV - we generate these
+ if 'ncx' in orig_name.lower() or orig_name.endswith('nav.xhtml'):
+ continue
+
+ # Copy the item directly
+ try:
+ book.add_item(orig_item)
+ items_map[orig_item.id] = orig_item
+ logger.debug(f"Copied missing item from original: {orig_name}")
+ except Exception as e:
+ logger.warning(f"Failed to copy item {orig_name}: {e}")
+
+ # 6. Spine
+ book.spine = []
+ for item_id in structure.spine:
+ if item_id in items_map:
+ book.spine.append(items_map[item_id])
+ else:
+ logger.warning(f"Spine item {item_id} not found in resources.")
+
+ # Add missing spine items from original
+ if original_book:
+ for spine_id, _ in original_book.spine:
+ if spine_id not in [i.id for i in book.spine]:
+ orig_item = original_book.get_item_with_id(spine_id)
+ if orig_item and orig_item.id in items_map:
+ book.spine.append(items_map[orig_item.id])
+
+ # Validate and fix TOC
+ if original_book and hasattr(book, 'toc') and book.toc:
+ try:
+ book.toc = self._validate_and_fix_toc(book.toc, items_map)
+ logger.info("Validated and fixed TOC links")
+ except Exception as e:
+ logger.error(f"Failed to validate TOC: {e}")
+
+ # 7. Navigation - NCX and Nav
+ book.add_item(epub.EpubNcx())
+ book.add_item(epub.EpubNav())
+
+ # 7. Write
+ epub.write_epub(str(output_path), book)
+ logger.info(f"EPUB created successfully at {output_path}")
+
+ return output_path
+
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/book_profiler.py b/archive/v0.11/backups/pre_refactor_20260127/src/book_profiler.py
new file mode 100644
index 0000000..3ab3d22
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/src/book_profiler.py
@@ -0,0 +1,76 @@
+import json
+import random
+from typing import Dict, List
+from loguru import logger
+from src.llm_client import LLMClient
+from src.manifest_manager import ManifestManager
+from src.data_model import BookProfile
+
+class BookProfiler:
+ def __init__(self, llm_client: LLMClient):
+ self.llm_client = llm_client
+
+ def extract_sample_text(self, entries: List[object], char_limit: int = 3000) -> str:
+ """Extract sample text from manifest entries."""
+ if not entries: return ""
+
+ # Simple sampling strategy: First few + random middle
+ intro_text = []
+ for entry in entries[:50]:
+ if len(entry.original_text) > 50:
+ intro_text.append(entry.original_text)
+
+ body_text = []
+ candidates = [e for e in entries[50:] if len(e.original_text) > 80]
+ if candidates:
+ samples = random.sample(candidates, min(5, len(candidates)))
+ body_text = [e.original_text for e in samples]
+
+ full_text = "\n\n".join(intro_text[:5] + body_text)
+ return full_text[:char_limit]
+
+ async def analyze(self, entries: List[object]) -> BookProfile:
+ """Generate Book Profile."""
+ sample = self.extract_sample_text(entries)
+ if not sample:
+ return BookProfile(title="Unknown", author="Unknown")
+
+ logger.info("Generating Book Profile from sample text...")
+
+ system_prompt = "You are a senior publishing editor. Analyze the text and output JSON."
+ user_prompt = f"""
+Please analyze the following book excerpt.
+Output JSON format:
+{{
+ "title": "Book Title",
+ "author": "Author Name",
+ "genre": "Genre",
+ "style": "Style description",
+ "keywords": ["keyword1", "keyword2"],
+ "style_guide": "Specific instruction for translator"
+}}
+
+Excerpt:
+{sample}
+"""
+ try:
+ response = await self.llm_client.raw_chat_completion(system_prompt, user_prompt)
+ json_str = response.strip()
+ # Basic cleanup
+ if "```json" in json_str:
+ json_str = json_str.split("```json")[1].split("```")[0].strip()
+ elif "```" in json_str:
+ json_str = json_str.split("```")[1].split("```")[0].strip()
+
+ data = json.loads(json_str)
+
+ return BookProfile(
+ title=data.get("title", "Unknown"),
+ author=data.get("author", "Unknown"),
+ genre=data.get("genre", "General"),
+ keywords=data.get("keywords", []),
+ style_guide=data.get("style_guide", "")
+ )
+ except Exception as e:
+ logger.error(f"Profile generation failed: {e}")
+ return BookProfile(title="Unknown", author="Unknown")
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/data_model.py b/archive/v0.11/backups/pre_refactor_20260127/src/data_model.py
new file mode 100644
index 0000000..cec4d24
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/src/data_model.py
@@ -0,0 +1,51 @@
+from pathlib import Path
+from typing import List, Optional, Dict, Any
+from pydantic import BaseModel, Field
+
+class ManifestEntry(BaseModel):
+ """Represents a single translatable unit."""
+ entry_id: str = Field(..., description="Global unique ID (e.g. file.html#paragraph_id)")
+ file_path: str = Field(..., description="Internal path in EPUB")
+ element_id: str = Field(..., description="HTML ID (e.g. uuid-1234)")
+ original_text: str
+ placeholders: Dict[str, str] = Field(default_factory=dict)
+ translated_text: Optional[str] = None
+ context: Optional[str] = None
+
+class BookMetaData(BaseModel):
+ title: str = "Unknown Title"
+ author: str = "Unknown Author"
+ language: str = "en"
+ identifier: str = ""
+
+class ResourceItem(BaseModel):
+ href: str
+ media_type: str
+ content: Optional[str] = None # For text/html
+ file_path: Optional[str] = None # For binary/assets (relative to assets dir)
+ properties: Optional[str] = None
+
+class BookStructure(BaseModel):
+ metadata: BookMetaData
+ spine: List[str] = Field(default_factory=list, description="Ordered list of item IDs in spine")
+ resources: Dict[str, ResourceItem] = Field(default_factory=dict, description="Map of item_id to ResourceItem")
+
+ @classmethod
+ def load(cls, path: Path) -> "BookStructure":
+ """Load BookStructure from JSON file."""
+ with open(path, 'r', encoding='utf-8') as f:
+ return cls.model_validate_json(f.read())
+
+ def save(self, path: Path):
+ """Save BookStructure to JSON file."""
+ with open(path, 'w', encoding='utf-8') as f:
+ f.write(self.model_dump_json(indent=2))
+
+class BookProfile(BaseModel):
+ """Represents the profile of the book."""
+ title: str
+ author: str
+ genre: str = "General"
+ keywords: List[str] = Field(default_factory=list)
+ style_guide: str = ""
+
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/epub_cleaner.py b/archive/v0.11/backups/pre_refactor_20260127/src/epub_cleaner.py
new file mode 100644
index 0000000..85a9227
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/src/epub_cleaner.py
@@ -0,0 +1,178 @@
+import shutil
+import uuid
+import zipfile
+from pathlib import Path
+from typing import Dict, Set
+
+import ebooklib
+from bs4 import BeautifulSoup, Tag
+from ebooklib import epub
+
+from src.data_model import BookStructure, BookMetaData, ResourceItem
+from src.exceptions import CleaningError
+from src.utils import setup_logger, ensure_directory
+
+logger = setup_logger("epub_cleaner")
+
+class EpubCleaner:
+ def __init__(self, input_path: Path, work_dir: Path):
+ self.input_path = input_path
+ self.work_dir = work_dir
+ self.assets_dir = work_dir / "assets"
+ self.json_path = work_dir / "book_structure.json"
+
+ def clean(self) -> Path:
+ """
+ Cleans the input EPUB and generates book_structure.json.
+ Returns the path to the JSON file.
+ """
+ logger.info(f"Starting cleanup for {self.input_path}")
+ ensure_directory(self.work_dir)
+ ensure_directory(self.assets_dir)
+
+ try:
+ book = epub.read_epub(self.input_path)
+
+ # 1. Extract Metadata
+ metadata = self._extract_metadata(book)
+
+ # 2. Process Resources
+ resources = {}
+ # Use zipfile for binary extraction to avoid ebooklib's memory overhead/decoding issues
+ with zipfile.ZipFile(self.input_path, 'r') as zf:
+ # Map ebooklib items to zip entries isn't straightforward directly via name
+ # So we iterate ebooklib items and assume standard structure or handle content bytes
+
+ for item in book.get_items():
+ item_id = item.get_id()
+ file_name = item.get_name()
+ media_type = item.get_type() # ebooklib constant
+
+ if media_type == ebooklib.ITEM_DOCUMENT:
+ # Clean HTML
+ content_str = item.get_content().decode('utf-8')
+ cleaned_content = self._clean_html(content_str, file_name)
+
+ resources[item_id] = ResourceItem(
+ href=file_name,
+ media_type="application/xhtml+xml",
+ content=cleaned_content
+ )
+ elif media_type in (ebooklib.ITEM_IMAGE, ebooklib.ITEM_STYLE, ebooklib.ITEM_FONT, ebooklib.ITEM_COVER):
+ # Save asset
+ # Preserve directory structure to avoid collisions
+ asset_path = self.assets_dir / file_name
+ ensure_directory(asset_path.parent)
+
+ # Ebooklib might change filenames, safer to use item.get_content()
+ with open(asset_path, "wb") as f:
+ f.write(item.get_content())
+
+ resources[item_id] = ResourceItem(
+ href=file_name,
+ media_type=self._get_media_type_str(item),
+ file_path=str(asset_path.relative_to(self.work_dir))
+ )
+ elif media_type == ebooklib.ITEM_NAVIGATION:
+ # NCX or NAV document - preserve for TOC
+ content_bytes = item.get_content()
+ asset_path = self.assets_dir / file_name
+ ensure_directory(asset_path.parent)
+
+ with open(asset_path, "wb") as f:
+ f.write(content_bytes)
+
+ # Determine media type
+ if file_name.endswith('.ncx'):
+ mt = "application/x-dtbncx+xml"
+ else:
+ mt = "application/xhtml+xml"
+
+ resources[item_id] = ResourceItem(
+ href=file_name,
+ media_type=mt,
+ file_path=str(asset_path.relative_to(self.work_dir))
+ )
+ logger.debug(f"Preserved navigation: {file_name}")
+ else:
+ # Skip other items (scripts, etc.)
+ pass
+
+ # 3. Extract Spine
+ spine_ids = [item[0] for item in book.spine]
+
+ # 4. Construct Structure
+ structure = BookStructure(
+ metadata=metadata,
+ spine=spine_ids,
+ resources=resources
+ )
+
+ # 5. Serialize
+ with open(self.json_path, "w", encoding="utf-8") as f:
+ f.write(structure.model_dump_json(indent=2))
+
+ logger.info(f"Cleanup finished. Structure saved to {self.json_path}")
+ return self.json_path
+
+ except Exception as e:
+ logger.error(f"Cleaning failed: {e}")
+ raise CleaningError(f"Failed to clean EPUB: {e}") from e
+
+ def _extract_metadata(self, book: epub.EpubBook) -> BookMetaData:
+ title = book.get_metadata('DC', 'title')[0][0] if book.get_metadata('DC', 'title') else "Unknown"
+ author = book.get_metadata('DC', 'creator')[0][0] if book.get_metadata('DC', 'creator') else "Unknown"
+ lang = book.get_metadata('DC', 'language')[0][0] if book.get_metadata('DC', 'language') else "en"
+ ident = book.get_metadata('DC', 'identifier')[0][0] if book.get_metadata('DC', 'identifier') else ""
+
+ return BookMetaData(
+ title=str(title),
+ author=str(author),
+ language=str(lang),
+ identifier=str(ident)
+ )
+
+ def _get_media_type_str(self, item) -> str:
+ # Helper to map ebooklib type to mime string if needed
+ # ebooklib doesn't expose easy MIME string for all types directly on item object sometimes
+ if hasattr(item, 'media_type'):
+ return item.media_type
+ return "application/octet-stream"
+
+ def _clean_html(self, content: str, filename: str) -> str:
+ soup = BeautifulSoup(content, 'html.parser')
+
+ # 1. Provide IDs for structural/translatable elements
+ self._ensure_element_ids(soup)
+
+ # 2. Flatten divs (Disable to prevent style loss)
+ # self._flatten_divs(soup)
+
+ return str(soup)
+
+ def _ensure_element_ids(self, soup: BeautifulSoup):
+ """
+ Injects UUIDs into p, h1-h6, li tags if they don't have an ID.
+ This provides the anchor for translation backfilling.
+ """
+ targets = soup.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'])
+ for tag in targets:
+ if not tag.has_attr('id'):
+ tag['id'] = f"uuid-{uuid.uuid4()}"
+
+ def _flatten_divs(self, soup: BeautifulSoup):
+ """
+ Converts generic divs containing only inline text/styles to p tags.
+ Recursive strategies can be complex, sticking to simple heuristic from archive.
+ """
+ inline_tags = {'span', 'em', 'strong', 'i', 'b', 'u', 'a', 'br', 'sub', 'sup', 'small'}
+
+ for div in list(soup.find_all('div')):
+ # If div has no block children, convert to p
+ has_block = any(
+ isinstance(c, Tag) and c.name not in inline_tags
+ for c in div.children
+ )
+
+ if not has_block:
+ div.name = 'p'
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/exceptions.py b/archive/v0.11/backups/pre_refactor_20260127/src/exceptions.py
new file mode 100644
index 0000000..2ca7b05
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/src/exceptions.py
@@ -0,0 +1,18 @@
+class EpubTranslatorError(Exception):
+ """Base exception for Epub Translator."""
+ pass
+
+class CleaningError(EpubTranslatorError):
+ pass
+
+class ExtractionError(EpubTranslatorError):
+ pass
+
+class TranslationError(EpubTranslatorError):
+ pass
+
+class RestorationError(EpubTranslatorError):
+ pass
+
+class BuildError(EpubTranslatorError):
+ pass
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/fine_grained_extractor.py b/archive/v0.11/backups/pre_refactor_20260127/src/fine_grained_extractor.py
new file mode 100644
index 0000000..7756894
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/src/fine_grained_extractor.py
@@ -0,0 +1,132 @@
+import re
+from typing import List, Dict, Any, Optional
+from bs4 import BeautifulSoup
+from loguru import logger
+
+from src.data_model import BookStructure, ManifestEntry, BookProfile
+from src.format_extractor import FormatExtractor
+from src.utils import setup_logger
+
+logger = setup_logger("fine_grained_extractor")
+
+class FineGrainedExtractor:
+ """
+ Extracts translatable text segments from BookStructure.
+ Uses FormatExtractor for detailed content analysis.
+ """
+
+ SKIP_TRANSLATION_PATTERNS = [
+ r'index\.x?html',
+ r'bibliography\.x?html',
+ r'endnotes?\.x?html',
+ r'footnotes?\.x?html',
+ r'copyright\.x?html',
+ ]
+
+ TOC_PATTERNS = [
+ r'nav\.x?html',
+ r'toc\.x?html',
+ ]
+
+ def __init__(self, translate_toc: bool = False):
+ self.translate_toc = translate_toc
+ self.format_extractor = FormatExtractor()
+
+ def extract(self, structure: BookStructure, profile: Optional[BookProfile] = None) -> List[ManifestEntry]:
+ """
+ Extracts translatable segments from the BookStructure.
+ Iteration follows the spine order.
+ """
+ logger.info("Starting extraction from BookStructure...")
+ manifest_entries = []
+
+ # Iterate over spine to maintain order
+ for item_id in structure.spine:
+ if item_id not in structure.resources:
+ logger.warning(f"Item ID {item_id} in spine but not in resources.")
+ continue
+
+ resource = structure.resources[item_id]
+
+ # Only process HTML/XHTML
+ if resource.media_type != "application/xhtml+xml" or not resource.content:
+ continue
+
+ file_path = resource.href
+ doc_type = self._classify_document(file_path)
+
+ # Parse HTML
+ soup = BeautifulSoup(resource.content, 'html.parser')
+ target_tags = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li']
+
+ for element in soup.find_all(target_tags):
+ # Ensure element has ID (should have been done by Cleaner)
+ element_id = element.get('id')
+ if not element_id:
+ logger.warning(f"Element in {file_path} missing ID, skipping: {element.name}")
+ continue
+
+ # Check translation eligibility
+ raw_text = element.get_text(separator=' ', strip=True)
+ if not raw_text.strip():
+ continue
+
+ is_decorative = self._is_decorative(raw_text)
+ should_translate = self._should_translate(doc_type, is_decorative, raw_text)
+
+ if should_translate:
+ # Extract detailed format
+ outer_html = str(element)
+ clean_text, text_with_ph, ph_map, p_type, _ = self.format_extractor.extract(outer_html)
+
+ if clean_text.strip() and text_with_ph.strip():
+ entry = ManifestEntry(
+ entry_id=f"{file_path}#{element_id}",
+ file_path=file_path,
+ element_id=element_id,
+ original_text=text_with_ph,
+ placeholders=ph_map,
+ context=p_type
+ )
+ manifest_entries.append(entry)
+
+ logger.info(f"Extracted {len(manifest_entries)} entries in total.")
+ return manifest_entries
+
+ def _classify_document(self, file_name: str) -> str:
+ if not file_name: return 'core'
+ fname = file_name.lower()
+ if any(re.search(p, fname) for p in self.SKIP_TRANSLATION_PATTERNS): return 'skip'
+ if any(re.search(p, fname) for p in self.TOC_PATTERNS): return 'toc'
+ return 'core'
+
+ def _should_translate(self, doc_type: str, is_decorative: bool, text: str = "") -> bool:
+ if is_decorative: return False
+ if self._is_roman_numeral(text): return False
+ if doc_type == 'core': return True
+ if doc_type == 'toc': return self.translate_toc
+ return False
+
+ def _is_roman_numeral(self, text: str) -> bool:
+ text = text.strip().upper()
+ if not text: return False
+ pattern = re.compile(r"^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$")
+ return bool(pattern.match(text))
+
+ def _is_decorative(self, text: str) -> bool:
+ s = text.strip()
+ if not s: return False
+ if not any(c.isalnum() for c in s): return True
+ if len(s) > 20: return False
+
+ patterns = [
+ r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾]+$',
+ r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾\s]+$',
+ ]
+ for p in patterns:
+ if re.match(p, s): return True
+
+ unique = set(s.replace(' ', ''))
+ if len(unique) <= 3 and (unique & set('*-_~•◆◇■□▪▫●○◉◎⦿⦾')):
+ return True
+ return False
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/format_extractor.py b/archive/v0.11/backups/pre_refactor_20260127/src/format_extractor.py
new file mode 100644
index 0000000..438430b
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/src/format_extractor.py
@@ -0,0 +1,372 @@
+import re
+from bs4 import BeautifulSoup, Tag
+from typing import Tuple, Dict, List
+
+class HeadingDetector:
+ """Detects heading types and paragraph roles."""
+
+ CHAPTER_PATTERNS = [
+ r'^(chapter|chap\.?|part)\s+([0-9]+|[ivxlc]+|[a-z])',
+ r'^(第\s*[0-9一二三四五六七八九十百]+\s*[章节部篇])',
+ r'^(\d+|[IVXLC]+|[A-Z])\.$'
+ ]
+
+ EPIGRAPH_CLASSES = {
+ 'epigraph', 'quote', 'blockquote', 'motto',
+ 'dedication', 'verse', 'poetry', 'poem'
+ }
+
+ def detect(self, element: Tag, text: str) -> str:
+ if self._is_epigraph(element):
+ return "epigraph"
+ tag_name = element.name.lower()
+ if tag_name in ['h1', 'h2']:
+ return "chapter" if self._matches_chapter_pattern(text) else "section"
+ if tag_name == 'h3':
+ return "section"
+ if tag_name in ['h4', 'h5', 'h6']:
+ return "subsection"
+ if self._is_pseudo_heading(element, text):
+ return "subsection"
+ return "body"
+
+ def _is_epigraph(self, element: Tag) -> bool:
+ if element.name == 'blockquote':
+ return True
+ current = element
+ for _ in range(3):
+ if not current: break
+ classes = current.get('class', [])
+ if isinstance(classes, list):
+ classes = ' '.join(classes)
+ if any(k in classes.lower() for k in self.EPIGRAPH_CLASSES):
+ return True
+ current = current.parent
+ return False
+
+ def _matches_chapter_pattern(self, text: str) -> bool:
+ text = text.strip().lower()
+ for pattern in self.CHAPTER_PATTERNS:
+ if re.match(pattern, text, re.IGNORECASE):
+ return True
+ return False
+
+ def _is_pseudo_heading(self, element: Tag, text: str) -> bool:
+ if element.name != 'p':
+ return False
+ text = text.strip()
+ if not text or len(text) > 80:
+ return False
+ children = list(element.children)
+ if len(children) == 1 and isinstance(children[0], Tag):
+ if children[0].name in ['strong', 'b']:
+ return True
+ return False
+
+
+class FormatExtractor:
+ """
+ HTML Format Extractor (Ported from v0.09 v3)
+ Handles inline styles, formulas, and drop caps.
+ """
+
+ FORMULA_CHARS = re.compile(
+ r'^[\d\s\+\-\×\÷\=\(\)\[\]\{\}\<\>\^\*\/\.\,\;\:\'\"\`\~\@\#\$\%\&\|\\'
+ r'αβγδεζηθικλμνξοπρστυφχψωΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩ'
+ r'a-zA-Z]+$'
+ )
+
+ def __init__(self):
+ self.detector = HeadingDetector()
+
+ def extract(self, element_html: str) -> Tuple[str, str, Dict[str, str], str, List[str]]:
+ """
+ Extracts format information.
+
+ Returns:
+ clean_text: Pure text
+ text_with_placeholders: Text with inline placeholders
+ placeholder_map: Map of placeholders
+ paragraph_type: Detected type
+ endnote_anchors: List of detected endnote IDs
+ """
+ soup = BeautifulSoup(element_html, 'html.parser')
+ root = list(soup.children)[0] if list(soup.children) else soup
+
+ clean_text = root.get_text().strip()
+ clean_text = re.sub(r'\s+', ' ', clean_text)
+ p_type = self.detector.detect(root, clean_text) if isinstance(root, Tag) else "body"
+
+ inner_html = root.decode_contents() if isinstance(root, Tag) else str(root)
+
+ text_with_ph, local_map = self._smart_extract_v3(inner_html)
+
+ if text_with_ph:
+ text_with_ph = re.sub(r'\s+', ' ', text_with_ph).strip()
+
+ # Verify integrity
+ stripped_text = self._strip_placeholders(text_with_ph)
+ stripped_text = re.sub(r'\s+', ' ', stripped_text).strip()
+
+ if not self._verify_content_integrity(clean_text, stripped_text):
+ # Fallback
+ text_with_ph, local_map = self._fallback_extract(inner_html, clean_text)
+
+ endnote_anchors = []
+ for pid, html in local_map.items():
+ if pid.startswith("_"):
+ continue
+ if re.match(r'<(span|a)\s+id="[a-zA-Z][a-zA-Z0-9]{2,5}"\s*>\s*', html):
+ endnote_anchors.append(pid)
+
+ return clean_text, text_with_ph, local_map, p_type, endnote_anchors
+
+ def _strip_placeholders(self, text: str) -> str:
+ return re.sub(r'φ/?[0-9]+φ', '', text)
+
+ def _verify_content_integrity(self, clean_text: str, stripped_text: str) -> bool:
+ def normalize(s):
+ s = re.sub(r'\s+', '', s)
+ s = s.lower()
+ return s
+
+ norm_clean = normalize(clean_text)
+ norm_stripped = normalize(stripped_text)
+
+ if norm_clean == norm_stripped:
+ return True
+
+ return False
+
+ def _fallback_extract(self, inner_html: str, clean_text: str) -> Tuple[str, Dict[str, str]]:
+ return clean_text, {"_prefix": "", "_suffix": ""}
+
+ def _smart_extract_v3(self, inner_html: str) -> Tuple[str, Dict[str, str]]:
+ parts = re.split(r'(<[^>]+>)', inner_html)
+ parts = [p for p in parts if p]
+
+ if not parts:
+ return "", {"_prefix": "", "_suffix": ""}
+
+ part_types = []
+ for part in parts:
+ if part.startswith('<'):
+ part_types.append('tag')
+ elif not part.strip():
+ part_types.append('whitespace')
+ elif self._is_translatable_text(part):
+ part_types.append('translatable')
+ else:
+ part_types.append('formula')
+
+ first_trans_idx = None
+ last_trans_idx = None
+ for i, t in enumerate(part_types):
+ if t == 'translatable':
+ if first_trans_idx is None:
+ first_trans_idx = i
+ last_trans_idx = i
+
+ if first_trans_idx is None:
+ return "", {"_prefix": inner_html, "_suffix": ""}
+
+ # Prefix Separation
+ safe_prefix_end = 0
+ for i in range(first_trans_idx):
+ if part_types[i] == 'tag':
+ tag = parts[i]
+ is_self_closing = tag.endswith('/>') or re.match(r'<(br|hr|img|meta|link)\b', tag, re.I)
+ is_closing = tag.startswith('') or re.match(r'<(br|hr|img|meta|link)\b', tag, re.I)
+ if is_closing or is_self_closing:
+ safe_suffix_start = i
+ else:
+ break
+ elif part_types[i] == 'whitespace':
+ safe_suffix_start = i
+ else:
+ break
+
+ prefix_parts = parts[:safe_prefix_end]
+ middle_parts = parts[safe_prefix_end:safe_suffix_start]
+ middle_types = part_types[safe_prefix_end:safe_suffix_start]
+ suffix_parts = parts[safe_suffix_start:]
+
+ # Drop Cap Check
+ if prefix_parts and middle_parts:
+ prefix_parts, middle_parts, middle_types = self._handle_drop_cap(
+ prefix_parts, middle_parts, middle_types
+ )
+
+ local_map = {}
+ if prefix_parts:
+ local_map["_prefix"] = "".join(prefix_parts)
+ if suffix_parts:
+ local_map["_suffix"] = "".join(suffix_parts)
+
+ # Middle processing
+ placeholder_counter = 1
+ result_parts = []
+ tag_stack = []
+
+ i = 0
+ while i < len(middle_parts):
+ part = middle_parts[i]
+ ptype = middle_types[i]
+
+ if ptype == 'translatable':
+ result_parts.append(part)
+ i += 1
+
+ elif ptype == 'tag':
+ is_closing = part.startswith('= 2 and combined.isalpha()):
+ return prefix_parts, middle_parts, middle_types
+
+ new_prefix = []
+ skip_until_close = False
+ found_letter = False
+
+ for part in prefix_parts:
+ if part.startswith('<') and not part.startswith(' bool:
+ text = text.strip()
+ if not text:
+ return False
+ if re.search(r'[a-zA-Z]{3,}', text):
+ return True
+ if ' ' in text and re.search(r'[a-zA-Z]', text):
+ return True
+ if re.search(r'\d', text):
+ return True
+ return False
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/format_restorer.py b/archive/v0.11/backups/pre_refactor_20260127/src/format_restorer.py
new file mode 100644
index 0000000..dc34549
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/src/format_restorer.py
@@ -0,0 +1,70 @@
+import re
+from typing import Dict, Tuple, List, Optional
+from loguru import logger
+from src.utils import setup_logger
+
+logger = setup_logger("format_restorer")
+
+class FormatRestorer:
+ """
+ Restores HTML formatting from placeholders.
+ """
+
+ PLACEHOLDER_REGEX = re.compile(r'φ(/?\d+)φ')
+
+ def restore(self, text_with_placeholders: str, placeholder_map: Dict[str, str]) -> Tuple[str, bool]:
+ """
+ Restores HTML from text with placeholders.
+ Returns (restored_html, success).
+ """
+ if not placeholder_map:
+ return text_with_placeholders or "", True
+
+ if not text_with_placeholders:
+ prefix = placeholder_map.get("_prefix", "")
+ suffix = placeholder_map.get("_suffix", "")
+ return prefix + suffix, True
+
+ prefix = placeholder_map.get("_prefix", "")
+ suffix = placeholder_map.get("_suffix", "")
+
+ inner_map = {k: v for k, v in placeholder_map.items() if not k.startswith("_")}
+
+ found_ids = set(self.PLACEHOLDER_REGEX.findall(text_with_placeholders))
+ expected_ids = set(inner_map.keys())
+
+ success = True
+ missing_ids = expected_ids - found_ids
+ if missing_ids:
+ logger.warning(f"Restoration warning: missing placeholders {missing_ids}")
+ success = False
+
+ unknown_ids = found_ids - expected_ids
+ if unknown_ids:
+ real_unknowns = set()
+ for pid in unknown_ids:
+ if pid.startswith('/') and pid[1:] in expected_ids:
+ continue
+ real_unknowns.add(pid)
+
+ if real_unknowns:
+ logger.warning(f"Restoration warning: unknown placeholders {real_unknowns}")
+ success = False
+
+ def replace_match(match):
+ pid = match.group(1)
+ if pid in inner_map:
+ return inner_map[pid]
+ else:
+ return ""
+
+ try:
+ restored_inner = self.PLACEHOLDER_REGEX.sub(replace_match, text_with_placeholders)
+ restored_html = prefix + restored_inner + suffix
+ return restored_html, success
+ except Exception as e:
+ logger.error(f"Restoration failed: {e}")
+ return prefix + self._strip_placeholders(text_with_placeholders) + suffix, False
+
+ def _strip_placeholders(self, text: str) -> str:
+ return self.PLACEHOLDER_REGEX.sub("", text)
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/llm_client.py b/archive/v0.11/backups/pre_refactor_20260127/src/llm_client.py
new file mode 100644
index 0000000..9d1d644
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/src/llm_client.py
@@ -0,0 +1,220 @@
+import asyncio
+import json
+import re
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import List, Dict, Optional
+from openai import AsyncOpenAI
+from tenacity import retry, stop_after_attempt, wait_exponential
+from loguru import logger
+
+from src.data_model import ManifestEntry
+from src.utils import setup_logger, ensure_directory
+
+logger = setup_logger("llm_client")
+
+# Default chunk save directory (can be overridden)
+DEFAULT_CHUNK_DIR = Path("tmp/chunks")
+
+
+class RateLimiter:
+ """Rate limiter for concurrency and RPM."""
+ def __init__(self, requests_per_minute: int, concurrent_requests: int):
+ self.semaphore = asyncio.Semaphore(concurrent_requests)
+ self.min_interval = 60.0 / requests_per_minute if requests_per_minute > 0 else 0
+ self.last_request_time = 0
+ self._lock = asyncio.Lock()
+
+ async def acquire(self):
+ await self.semaphore.acquire()
+ async with self._lock:
+ current_time = time.time()
+ wait_time = self.min_interval - (current_time - self.last_request_time)
+ if wait_time > 0:
+ await asyncio.sleep(wait_time)
+ self.last_request_time = time.time()
+
+ def release(self):
+ self.semaphore.release()
+
+
+class LLMClient:
+ """Generic OpenAI-compatible API Client with short ID strategy."""
+
+ def __init__(self, api_key: str, base_url: str, model: str = "gpt-3.5-turbo",
+ requests_per_minute: int = 60, concurrent_requests: int = 5,
+ extra_headers: Dict = None, chunk_dir: Path = None):
+
+ # Configure proxy client to avoid SOCKS issues and ensure connectivity
+ import httpx
+ import os
+
+ # Prefer HTTP proxy if available to avoid missing socksio support
+ proxy_url = os.environ.get("http_proxy") or os.environ.get("https_proxy")
+ http_client = httpx.AsyncClient(
+ proxy=proxy_url,
+ timeout=60.0,
+ follow_redirects=True
+ ) if proxy_url else None
+
+ self.client = AsyncOpenAI(
+ base_url=base_url,
+ api_key=api_key,
+ default_headers=extra_headers,
+ http_client=http_client
+ )
+ self.model = model
+
+ self.rate_limiter = RateLimiter(requests_per_minute, concurrent_requests)
+ self.prompts = self._load_prompts()
+ self._chunk_counter = 0
+
+ # Chunk directory for debug output
+ self.chunk_dir = chunk_dir or DEFAULT_CHUNK_DIR
+ ensure_directory(self.chunk_dir)
+
+ def _load_prompts(self) -> Dict:
+ try:
+ with open("config/prompts.json", "r", encoding="utf-8") as f:
+ return json.load(f)
+ except Exception as e:
+ logger.error(f"Failed to load config/prompts.json: {e}")
+ return {}
+
+ async def translate_chunk(self, items: List[ManifestEntry], glossary: Dict = None,
+ instruction: str = None, mode: str = "bilingual") -> Dict[str, str]:
+ """
+ Translate a chunk of items using short ID strategy.
+ Returns: Dict[entry_id, translated_text]
+ """
+ if not items: return {}
+
+ # Build prompt with short IDs
+ id_map, prompt = self._build_prompt_with_short_ids(items)
+
+ try:
+ # Build System Prompt
+ base_sys_prompt = self.prompts.get("translation", {}).get("system",
+ "You are a professional English to Chinese translator.")
+
+ if instruction:
+ base_sys_prompt += f"\n\nBook Style Guide:\n{instruction}"
+
+ if glossary:
+ glossary_text = "\n".join([f"{k} -> {v}" for k, v in glossary.items()])
+ base_sys_prompt += f"\n\nTerminology:\n{glossary_text}"
+
+ # Short ID format instructions
+ base_sys_prompt += """
+
+Output Format:
+- Each line MUST start with #N: (keep this ID exactly as given)
+- Preserve any φXφ or φ/Xφ placeholders EXACTLY as-is
+- Only output translations, no explanations
+- Match the number of output lines to input lines"""
+
+ # Save chunk before translation
+ chunk_id = self._save_chunk("before", prompt, base_sys_prompt)
+
+ logger.debug(f"Sending request to LLM (Chunk: {chunk_id}, Items: {len(items)})")
+
+ raw_response = await self._make_request(base_sys_prompt, prompt)
+
+ # Save chunk after translation
+ self._save_chunk("after", raw_response, base_sys_prompt, chunk_id)
+
+ # Parse with short ID mapping
+ results = self._parse_short_id_response(raw_response, id_map)
+ return results
+
+ except Exception as e:
+ logger.error(f"Translation failed: {e}")
+ return {item.entry_id: f"[Error - {str(e)}]" for item in items}
+
+ def _build_prompt_with_short_ids(self, items: List[ManifestEntry]) -> tuple:
+ """
+ Build prompt with short IDs (#1, #2, ...).
+ Returns: (id_map, prompt_text)
+ """
+ id_map = {} # short_id -> entry_id
+ lines = []
+
+ for i, item in enumerate(items, 1):
+ short_id = f"#{i}"
+ id_map[short_id] = item.entry_id
+
+ # Clean text (remove extra whitespace)
+ text = re.sub(r'\s+', ' ', item.original_text).strip()
+ lines.append(f"{short_id}: {text}")
+
+ return id_map, "\n".join(lines)
+
+ def _parse_short_id_response(self, response: str, id_map: Dict[str, str]) -> Dict[str, str]:
+ """
+ Parse response with short ID format.
+ Returns: Dict[entry_id, translated_text]
+ """
+ results = {}
+
+ for line in response.split("\n"):
+ line = line.strip()
+ if not line:
+ continue
+
+ # Match #N: pattern
+ match = re.match(r'^#(\d+):\s*(.+)$', line)
+ if match:
+ short_id = f"#{match.group(1)}"
+ translation = match.group(2).strip()
+
+ if short_id in id_map:
+ full_id = id_map[short_id]
+ results[full_id] = translation
+ else:
+ logger.warning(f"Unknown short ID in response: {short_id}")
+
+ return results
+
+ def _save_chunk(self, stage: str, content: str, system_prompt: str = None,
+ chunk_id: str = None) -> str:
+ """Save chunk to tmp directory for debugging."""
+ if chunk_id is None:
+ self._chunk_counter += 1
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+ chunk_id = f"{timestamp}_{self._chunk_counter:04d}"
+
+ filename = self.chunk_dir / f"chunk_{chunk_id}_{stage}.txt"
+
+ with open(filename, "w", encoding="utf-8") as f:
+ if system_prompt and stage == "before":
+ f.write("=== SYSTEM PROMPT ===\n")
+ f.write(system_prompt)
+ f.write("\n\n=== USER PROMPT ===\n")
+ f.write(content)
+
+ logger.debug(f"Saved chunk: {filename}")
+ return chunk_id
+
+ async def raw_chat_completion(self, system_prompt: str, user_prompt: str) -> str:
+ """Generic chat completion."""
+ return await self._make_request(system_prompt, user_prompt)
+
+ @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))
+ async def _make_request(self, system_prompt: str, user_prompt: str) -> str:
+ await self.rate_limiter.acquire()
+ try:
+ resp = await self.client.chat.completions.create(
+ model=self.model,
+ messages=[
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_prompt}
+ ],
+ temperature=0.3,
+ )
+ return resp.choices[0].message.content.strip()
+ finally:
+ self.rate_limiter.release()
+
+ async def close(self):
+ await self.client.close()
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/manifest_manager.py b/archive/v0.11/backups/pre_refactor_20260127/src/manifest_manager.py
new file mode 100644
index 0000000..3b3dfd1
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/src/manifest_manager.py
@@ -0,0 +1,74 @@
+from pathlib import Path
+from typing import List, Dict, Optional
+import json
+from src.data_model import ManifestEntry
+from src.utils import setup_logger
+
+logger = setup_logger("manifest_manager")
+
+class ManifestManager:
+ """
+ Manages the translation manifest (Source of Truth).
+ Handles persistence and state updates.
+ """
+
+ def __init__(self, manifest_path: Path):
+ self.manifest_path = manifest_path
+ self.entries: List[ManifestEntry] = []
+ self._entries_map: Dict[str, ManifestEntry] = {}
+
+ def load(self):
+ """Loads manifest from disk if it exists."""
+ if not self.manifest_path.exists():
+ logger.info(f"Manifest not found at {self.manifest_path}, starting empty.")
+ return
+
+ try:
+ with open(self.manifest_path, 'r', encoding='utf-8') as f:
+ data = json.load(f)
+ self.entries = [ManifestEntry.model_validate(item) for item in data]
+ self._rebuild_map()
+ logger.info(f"Loaded {len(self.entries)} entries from manifest.")
+ except Exception as e:
+ logger.error(f"Failed to load manifest: {e}")
+ raise
+
+ def save(self):
+ """Saves current state to disk."""
+ try:
+ # Pydantic v2: model_dump(mode='json') or just list dump
+ data = [entry.model_dump(mode='json') for entry in self.entries]
+ with open(self.manifest_path, 'w', encoding='utf-8') as f:
+ json.dump(data, f, indent=2, ensure_ascii=False)
+ logger.info(f"Saved {len(self.entries)} entries to manifest.")
+ except Exception as e:
+ logger.error(f"Failed to save manifest: {e}")
+ raise
+
+ def add_entries(self, new_entries: List[ManifestEntry]):
+ """
+ Adds new entries to the manifest.
+ If an entry with the same ID exists, it keeps the EXISTING one (to preserve translations).
+ """
+ count = 0
+ for entry in new_entries:
+ if entry.entry_id not in self._entries_map:
+ self.entries.append(entry)
+ self._entries_map[entry.entry_id] = entry
+ count += 1
+
+ if count > 0:
+ logger.info(f"Added {count} new entries to manifest.")
+
+ def update_translation(self, entry_id: str, translation: str):
+ """Updates translation for a specific entry."""
+ if entry_id in self._entries_map:
+ self._entries_map[entry_id].translated_text = translation
+ else:
+ logger.warning(f"Attempted to update translation for unknown ID: {entry_id}")
+
+ def get_entry(self, entry_id: str) -> Optional[ManifestEntry]:
+ return self._entries_map.get(entry_id)
+
+ def _rebuild_map(self):
+ self._entries_map = {e.entry_id: e for e in self.entries}
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/translator.py b/archive/v0.11/backups/pre_refactor_20260127/src/translator.py
new file mode 100644
index 0000000..9a6776f
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/src/translator.py
@@ -0,0 +1,285 @@
+"""
+Translator Module - Handles translation of ManifestEntry items.
+
+Key features:
+- Character-based chunking (~5000 chars per chunk)
+- Chapter-aware grouping (chunks don't cross file boundaries)
+- Concurrent translation with asyncio.gather
+- Progress tracking and error handling
+"""
+
+import asyncio
+from typing import List, Dict
+from collections import defaultdict
+from loguru import logger
+
+from src.data_model import ManifestEntry, BookProfile
+from src.llm_client import LLMClient
+from src.format_restorer import FormatRestorer
+from src.utils import setup_logger
+
+logger = setup_logger("translator")
+
+# Default chunk size in characters
+DEFAULT_CHUNK_SIZE = 5000
+# Maximum concurrent translations
+MAX_CONCURRENT = 5
+
+
+class Translator:
+ """
+ Translates ManifestEntry items using LLM with chapter-aware chunking.
+ Supports both sequential and concurrent translation modes.
+ """
+
+ def __init__(self, llm_client: LLMClient, chunk_size: int = DEFAULT_CHUNK_SIZE,
+ max_concurrent: int = MAX_CONCURRENT):
+ self.llm_client = llm_client
+ self.chunk_size = chunk_size
+ self.max_concurrent = max_concurrent
+ self.restorer = FormatRestorer()
+
+ async def translate(self, entries: List[ManifestEntry], profile: BookProfile,
+ concurrent: bool = True) -> List[ManifestEntry]:
+ """
+ Translates all untranslated entries.
+
+ Args:
+ entries: All manifest entries
+ profile: Book profile with style guide
+ concurrent: Use concurrent translation (default True)
+
+ Returns:
+ The same entries list with translated_text populated
+ """
+ untranslated = [e for e in entries if not e.translated_text]
+ if not untranslated:
+ logger.info("No new entries to translate.")
+ return entries
+
+ logger.info(f"Found {len(untranslated)} entries to translate")
+
+ # Group by chapter (file_path)
+ chapters = self._group_by_chapter(untranslated)
+ logger.info(f"Grouped into {len(chapters)} chapters")
+
+ # Create all chunks
+ all_chunks = []
+ for file_path, chapter_entries in chapters.items():
+ chapter_chunks = self._create_char_based_chunks(chapter_entries)
+ for chunk in chapter_chunks:
+ all_chunks.append((file_path, chunk))
+
+ total_chunks = len(all_chunks)
+ logger.info(f"Created {total_chunks} chunks (avg ~{self.chunk_size} chars each)")
+
+ if concurrent:
+ await self._translate_concurrent(all_chunks, profile)
+ else:
+ await self._translate_sequential(all_chunks, profile)
+
+ translated_count = sum(1 for e in entries if e.translated_text)
+ logger.info(f"Translation complete: {translated_count}/{len(entries)} entries translated")
+ return entries
+
+ async def _translate_concurrent(self, all_chunks: List, profile: BookProfile):
+ """Translate chunks concurrently with semaphore control."""
+ semaphore = asyncio.Semaphore(self.max_concurrent)
+ completed = [0] # Use list for mutable counter in closure
+ total = len(all_chunks)
+ success = [0]
+ failed = [0]
+
+ async def translate_chunk_task(file_path: str, chunk: List[ManifestEntry], idx: int):
+ async with semaphore:
+ try:
+ results = await self.llm_client.translate_chunk(
+ chunk,
+ instruction=profile.style_guide if profile else None,
+ mode="bilingual"
+ )
+
+ for entry in chunk:
+ if entry.entry_id in results:
+ entry.translated_text = results[entry.entry_id]
+ success[0] += 1
+ else:
+ failed[0] += 1
+
+ except Exception as e:
+ logger.error(f"Chunk {idx} failed: {e}")
+ failed[0] += len(chunk)
+ finally:
+ completed[0] += 1
+ if completed[0] % 5 == 0 or completed[0] == total:
+ logger.info(f"Progress: {completed[0]}/{total} chunks ({success[0]} translated)")
+
+ tasks = [
+ translate_chunk_task(file_path, chunk, i)
+ for i, (file_path, chunk) in enumerate(all_chunks)
+ ]
+ await asyncio.gather(*tasks)
+
+ logger.info(f"Concurrent translation: {success[0]} success, {failed[0]} failed")
+
+ async def _translate_sequential(self, all_chunks: List, profile: BookProfile):
+ """Translate chunks sequentially."""
+ total_chunks = len(all_chunks)
+ success_count = 0
+ fail_count = 0
+
+ for idx, (file_path, chunk) in enumerate(all_chunks):
+ try:
+ results = await self.llm_client.translate_chunk(
+ chunk,
+ instruction=profile.style_guide if profile else None,
+ mode="bilingual"
+ )
+
+ for entry in chunk:
+ if entry.entry_id in results:
+ entry.translated_text = results[entry.entry_id]
+ success_count += 1
+ else:
+ fail_count += 1
+ logger.warning(f"Missing translation for: {entry.entry_id[-40:]}")
+
+ except Exception as e:
+ logger.error(f"Chunk {idx} translation failed: {e}")
+ fail_count += len(chunk)
+
+ if (idx + 1) % 10 == 0 or idx + 1 == total_chunks:
+ logger.info(f"Progress: {idx + 1}/{total_chunks} chunks ({success_count} entries translated)")
+
+ logger.info(f"Sequential translation: {success_count} success, {fail_count} failed")
+
+
+ async def translate_chapter(self, entries: List[ManifestEntry], file_path: str,
+ profile: BookProfile) -> Dict[str, int]:
+ """
+ Translate a single chapter.
+
+ Args:
+ entries: All entries (will filter by file_path)
+ file_path: Chapter file path to translate
+ profile: Book profile
+
+ Returns:
+ Dict with 'success' and 'failed' counts
+ """
+ chapter_entries = [e for e in entries if e.file_path == file_path and not e.translated_text]
+
+ if not chapter_entries:
+ logger.info(f"Chapter {file_path} has no untranslated entries")
+ return {"success": 0, "failed": 0}
+
+ logger.info(f"Translating chapter: {file_path} ({len(chapter_entries)} entries)")
+
+ chunks = self._create_char_based_chunks(chapter_entries)
+ logger.info(f"Created {len(chunks)} chunks")
+
+ success = 0
+ failed = 0
+
+ for i, chunk in enumerate(chunks, 1):
+ chunk_chars = sum(len(e.original_text) for e in chunk)
+ logger.debug(f"Chunk {i}/{len(chunks)}: {len(chunk)} entries, {chunk_chars} chars")
+
+ try:
+ results = await self.llm_client.translate_chunk(
+ chunk,
+ instruction=profile.style_guide if profile else None,
+ mode="bilingual"
+ )
+
+ for entry in chunk:
+ if entry.entry_id in results:
+ entry.translated_text = results[entry.entry_id]
+
+ # Verify placeholder preservation
+ if entry.placeholders:
+ _, restored_ok = self.restorer.restore(
+ entry.translated_text,
+ entry.placeholders
+ )
+ if not restored_ok:
+ logger.warning(f"Placeholder issue: {entry.entry_id[-40:]}")
+
+ success += 1
+ else:
+ failed += 1
+
+ except Exception as e:
+ logger.error(f"Chunk {i} failed: {e}")
+ failed += len(chunk)
+
+ logger.info(f"Chapter done: {success} success, {failed} failed")
+ return {"success": success, "failed": failed}
+
+ def _group_by_chapter(self, entries: List[ManifestEntry]) -> Dict[str, List[ManifestEntry]]:
+ """Group entries by file_path (chapter)."""
+ chapters = defaultdict(list)
+ for entry in entries:
+ chapters[entry.file_path].append(entry)
+ return dict(chapters)
+
+ def _create_char_based_chunks(self, entries: List[ManifestEntry]) -> List[List[ManifestEntry]]:
+ """
+ Create chunks based on character count.
+
+ Each chunk contains approximately self.chunk_size characters.
+ Chunks never cross chapter boundaries (entries from same file only).
+ """
+ chunks = []
+ current_chunk = []
+ current_size = 0
+
+ for entry in entries:
+ text_len = len(entry.original_text)
+
+ # If adding this entry exceeds limit and we have content, start new chunk
+ if current_size + text_len > self.chunk_size and current_chunk:
+ chunks.append(current_chunk)
+ current_chunk = []
+ current_size = 0
+
+ current_chunk.append(entry)
+ current_size += text_len
+
+ if current_chunk:
+ chunks.append(current_chunk)
+
+ return chunks
+
+ def get_chapter_stats(self, entries: List[ManifestEntry]) -> List[Dict]:
+ """
+ Get statistics for each chapter.
+
+ Returns list of dicts with: file_path, total, translated, pending, chars
+ """
+ chapters = self._group_by_chapter(entries)
+ stats = []
+
+ for file_path, chapter_entries in sorted(chapters.items()):
+ total = len(chapter_entries)
+ translated = sum(1 for e in chapter_entries if e.translated_text)
+ total_chars = sum(len(e.original_text) for e in chapter_entries)
+
+ # Get first text as title preview
+ first_text = ""
+ for e in chapter_entries:
+ if e.original_text:
+ first_text = e.original_text[:40].replace('\n', ' ')
+ break
+
+ stats.append({
+ "file_path": file_path,
+ "title": first_text,
+ "total": total,
+ "translated": translated,
+ "pending": total - translated,
+ "chars": total_chars
+ })
+
+ return stats
+
diff --git a/archive/v0.11/backups/pre_refactor_20260127/src/utils.py b/archive/v0.11/backups/pre_refactor_20260127/src/utils.py
new file mode 100644
index 0000000..e1cd9b3
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/src/utils.py
@@ -0,0 +1,25 @@
+import logging
+from pathlib import Path
+
+def setup_logger(name: str, log_file: Path = None, level=logging.INFO):
+ """Sets up a logger with the given name."""
+ logger = logging.getLogger(name)
+ logger.setLevel(level)
+
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+ if log_file:
+ file_handler = logging.FileHandler(log_file)
+ file_handler.setFormatter(formatter)
+ logger.addHandler(file_handler)
+
+ stream_handler = logging.StreamHandler()
+ stream_handler.setFormatter(formatter)
+ logger.addHandler(stream_handler)
+
+ return logger
+
+def ensure_directory(path: Path):
+ """Ensures a directory exists."""
+ if not path.exists():
+ path.mkdir(parents=True, exist_ok=True)
diff --git a/archive/v0.11/backups/pre_refactor_20260127/test_hello.py b/archive/v0.11/backups/pre_refactor_20260127/test_hello.py
new file mode 100644
index 0000000..cefb32e
--- /dev/null
+++ b/archive/v0.11/backups/pre_refactor_20260127/test_hello.py
@@ -0,0 +1 @@
+print("Hello from python")
diff --git a/archive/v0.11/config/config.yaml b/archive/v0.11/config/config.yaml
new file mode 100644
index 0000000..742a1de
--- /dev/null
+++ b/archive/v0.11/config/config.yaml
@@ -0,0 +1,25 @@
+# LLM Configuration
+llm:
+ # Model name (e.g., gpt-4o, gpt-3.5-turbo, deepseek-chat)
+ model: "google/gemini-3-flash-preview"
+ #model: "gemini-3-pro-low"
+
+ # API Base URL (default is OpenAI)
+ #base_url: "https://api.gpt.ge/v1"
+ #base_url: "http://192.168.50.100:11434/v1"
+ base_url: "https://openrouter.ai/api/v1"
+
+ # Timeout for API requests in seconds
+ timeout: 60
+
+ # Rate Limiting
+ requests_per_minute: 60
+ concurrent_requests: 4
+
+# Translation Settings
+translation:
+ # Characters per chunk (approximate)
+ chunk_size: 6000
+
+ # System prompt instruction file (optional, overrides default if present)
+ # style_guide_path: "config/style_guide.txt"
diff --git a/archive/v0.11/config/prompts.json b/archive/v0.11/config/prompts.json
new file mode 100644
index 0000000..7b74b31
--- /dev/null
+++ b/archive/v0.11/config/prompts.json
@@ -0,0 +1,10 @@
+{
+ "translation": {
+ "system": "你是一位精通中英文的专业翻译家。你的任务是将英文书籍内容翻译成中文。\n\n【核心要求】\n1. 准确传达原文含义，语言流畅自然，符合中文阅读习惯。\n2. 保持原文的风格和语气。\n\n【格式要求 - 极其重要】\n1. 每行格式：#N: 译文（N是行号，必须原样保留）\n2. 输入多少行，输出必须是相同数量的行\n3. **占位符规则**：原文中的 φXφ 和 φ/Xφ 标记必须原样保留在译文中\n - 成对标记：\"φ1φBoldφ/1φ\" → \"φ1φ粗体φ/1φ\"\n - 单体标记：\"φ2φ\" 表示公式或符号，保持位置不变\n - 尾注锚点：文末的 \"φ3φ\" 是超链接，必须保留\n\n【禁止事项】\n- 禁止修改 #N: 行号\n- 禁止删除或修改任何 φXφ 标记\n- 禁止添加解释或注释\n- 禁止合并或拆分行",
+ "user_template": "请翻译以下段落（务必原样保留所有 φnφ 格式标记）：\n\n{{content}}"
+ },
+ "glossary_extraction": {
+ "system": "你是一位资深的文学编辑和领域专家。你的任务是分析书籍样本，提取关键术语并制定统一的译名表。",
+ "user_template": "请阅读以下书籍片段（包含前言和正文采样）。\n\n任务：\n1. 识别文中出现的人名（如 'Masa', 'Steve Jobs'）、地名、机构名。\n2. 识别特定的行业术语或关键概念。\n3. 为上述词汇提供标准的中文译名。如果像 'Masa' 这样的昵称有对应的全名（如孙正义），请务必使用全名。\n\n请以 JSON 格式输出，格式如下：\n{\n \"Masa\": \"孙正义\",\n \"Apple\": \"苹果公司\",\n ...\n}\n\n书籍片段：\n\n{{content}}"
+ }
+}
\ No newline at end of file
diff --git a/archive/v0.11/doc/OPERATION_MANUAL.md b/archive/v0.11/doc/OPERATION_MANUAL.md
new file mode 100644
index 0000000..692add5
--- /dev/null
+++ b/archive/v0.11/doc/OPERATION_MANUAL.md
@@ -0,0 +1,113 @@
+# Operation Manual & Change Log
+
+## Core Principles
+
+1. **Modularity**: The system is divided into three distinct phases (Preprocessing, Translation, Assembly) with clear boundaries.
+2. **Immutability**: `book_structure.json` is generated once during preprocessing and should not be modified by subsequent steps.
+3. **Source of Truth**: `manifest.json` is the single source of truth for translations.
+4. **Idempotency**: Translation steps can be retried without side effects (existing translations are preserved).
+
+## Directory Structure
+
+* `pipeline/`: Executable scripts for each stage.
+ * `01_preprocess.py`: Clean EPUB, generate structure, extract text.
+ * `02_translate.py`: Translate text in manifest.
+ * `03_assemble.py`: Apply translations and build final EPUB.
+* `src/`: Core logic modules.
+ * `preprocessing/`: Cleaning, extraction, profiling.
+ * `translation/`: LLM integration, manifest management.
+ * `assembly/`: Backfilling, EPUB building.
+ * `common/`: Shared data models and utils.
+* `work/`: Working directory for intermediate files (ignored by git).
+
+## Pipeline Usage
+
+### Step 1: Preprocessing
+```bash
+python pipeline/01_preprocess.py inputs/my_book.epub
+```
+Generates `work/my_book/book_structure.json` and `manifest.json`.
+
+### Step 2: Translation
+```bash
+python pipeline/02_translate.py --input-epub inputs/my_book.epub
+```
+Translates entries in `manifest.json`. ensuring `.env` has `OPENAI_API_KEY`.
+
+### Step 3: Assembly
+### Step 3: Assembly
+```bash
+# Bilingual Output (Default: output/bilingual_my_book.epub)
+python pipeline/03_assemble.py inputs/my_book.epub --bilingual
+
+# Target Language Output (Default: output/translated_my_book.epub)
+python pipeline/03_assemble.py inputs/my_book.epub
+```
+**Note**: The Assembly step now includes an **LLM-based Placeholder Repair** mechanism. If `format_restorer` detects broken placeholders in the translation, it will query the LLM (using your configured credentials) to attempt an automatic fix. Ensure your `OPENAI_API_KEY` is set if you want this feature enabled.
+
+## Configuration
+
+System settings are managed via `config/config.yaml` and environment variables.
+
+### `config/config.yaml`
+Control LLM parameters and translation behavior:
+```yaml
+llm:
+ model: "gpt-3.5-turbo" # LLM Model Name
+ base_url: "https://api.openai.com/v1"
+ timeout: 60
+ requests_per_minute: 60 # Rate limiting
+ concurrent_requests: 5 # Parallel chunks
+
+translation:
+ chunk_size: 4000 # Characters per chunk
+```
+
+### Environment Variables (`.env`)
+Security-sensitive credentials must be set here:
+```bash
+OPENAI_API_KEY=sk-... # Required
+OPENAI_BASE_URL=... # Optional override for config
+```
+
+## Known Issues & Troubleshooting
+
+### AuthenticationError (OpenRouter etc.)
+If you see `AuthenticationError` despite having the correct `base_url` in config:
+1. Check if you have a stale `OPENAI_API_KEY` in your shell environment.
+2. Environment variables **override** `.env` files.
+3. Fix: Run `unset OPENAI_API_KEY` (and `OPENAI_BASE_URL`) before running the script.
+
+### Missing/Unknown Placeholders
+* **Logs**: `WARNING - Restoration warning: missing placeholders...`
+* **Cause**: The LLM translation didn't preserve the exact `φXφ` tags.
+* **Fix**:
+ 1. The system will now attempt to **auto-repair** using the LLM during Assembly.
+ 2. If that fails, check logs. In some cases (e.g., complex HTML entities like `&`), the extractor might have degraded to plain text.
+ 3. (Fixed in v0.11) Enhanced `FormatExtractor` now handles HTML entities correctly, preventing phantom placeholder hallucinations.
+
+## Change Log
+
+### [2026-01-31] Robustness & Repair
+* **Feature**: Added **LLM-based Placeholder Repair** in Assembly stage. If placeholders mismatch, the system asks the LLM to fix the tags without changing text.
+* **Fix**: Solved `FormatExtractor` "phantom placeholders" issue by correctly unescaping HTML entities during integrity checks.
+* **Fix**: Resolved **Duplicate ID** issue in LLM response parsing. Now recursively strips repeated headers (e.g., `#12: #12: ...`) to prevent them from leaking into the translation.
+* **Tweak**: Updated `pipeline/03_assemble.py` to be async and load LLM config.
+
+### [2026-01-30] Performance Improvements
+* **Concurrency Fix**: Resolved issue where `concurrent_requests` in `config.yaml` was ignored by the Translator engine. Now `main.py` and `pipeline/02_translate.py` correctly propagate this setting, allowing faster translation with higher limits (e.g., for local LLMs or high-rate-limit providers).
+
+### [2026-01-28] Bug Fixes
+* **Fix Cover Image**: Resolved issue where book cover execution was missing in the final EPUB. Added `cover_image_id` tracking in `BookStructure` and restored proper OPF metadata in `BilingualBuilder`.
+
+### [2026-01-27] Externalized Configuration
+* **Config**: Added `config/config.yaml` for tuning parameters (LLM model, RPM, Chunk Size).
+* **Logic**: `pipeline/02_translate.py` now loads settings from `config.yaml`.
+* **Dependency**: Added `PyYAML` to `requirements.txt`.
+
+### [2026-01-27] Architecture Refactoring
+* **Restructured**: Moved source files into `src/preprocessing`, `src/translation`, `src/assembly`, `src/common`.
+* **Pipeline**: Created individual pipeline scripts in `pipeline/`.
+* **Refactor**: Renamed `fine_grained_extractor` to `text_extractor`, `translator` to `translator_engine`, etc.
+* **Logic**: Enforced 100% text coverage check in `format_extractor.py` (removed 95% threshold).
+* **Docs**: Created this Operation Manual.
diff --git a/archive/v0.11/doc/architecture_flow.md b/archive/v0.11/doc/architecture_flow.md
new file mode 100644
index 0000000..b487697
--- /dev/null
+++ b/archive/v0.11/doc/architecture_flow.md
@@ -0,0 +1,100 @@
+# ePub Bilingual Translator - Architecture & Data Flow (Revised)
+
+本文档详细描述了程序处理一个 EPUB 文件的完整生命周期。整个流程旨在实现**结构稳定性**（不丢段落）与**内容精细度**（不丢格式）的最佳平衡。
+
+## High Level Data Flow
+
+```mermaid
+graph TD
+ Input[Input EPUB] --> Cleaner[EpubCleaner]
+ Cleaner --> CleanedEPUB[1. Cleaned EPUB Temp]
+
+ CleanedEPUB --> Profiler[Book Profiler]
+ Profiler -->|Identify| Profile[Book Profile / Style]
+
+ CleanedEPUB --> Extractor[FineGrainedExtractor]
+
+ subgraph Extraction
+ Extractor -->|Step 1: P/H Tags| P[Source Paragraph]
+ P -->|Step 2: FormatExtract| PhText[Analyzed Text]
+ PhText -->|Register| Manifest[Manifest DB]
+ end
+
+ Manifest -->|Batch| LLM[LLM Translation]
+ Profile -->|Prompt Context| LLM
+
+ LLM -->|Translation| Manifest
+
+ Manifest --> Restorer[FormatRestorer]
+ Restorer -->|Reconstruct HTML| TargetHtml[Target HTML]
+
+ CleanedEPUB --> Backfiller[Backfill Engine]
+ TargetHtml --> Backfiller
+
+ Backfiller -->|Bilingual/Chinese Mode| DOM[Final DOM]
+ DOM --> Builder[BilingualBuilder]
+ Builder --> Output[Output EPUB]
+```
+
+## Detailed Workflow
+
+### 1. Preprocessing (预处理)
+**模块**: `src/epub_cleaner.py`
+* **Flatten Structure**: 消除嵌套 `div`，统一转为 ` `，消除结构性漏译风险。
+* **Auto-Fix**: 修复 TOC 死链、缺失 UID、由于 `ebooklib` bug 导致的样式丢失。
+* **Result**: 产生一个标准的临时文件，后续所有操作基于此文件，不再受原始糟糕格式影响。
+
+### 2. Intelligent Extraction (智能提取)
+**模块**: `src/fine_grained_extractor.py` + `src/format_extractor.py`
+
+#### A. 结构层 (Macro)
+使用 `FineGrainedExtractor` 锁定所有正文元素 (`p`, `h1`-`h6`)。
+* **Filter**: 排除页码、页眉脚。
+* **Optimization**: 针对目录章节，识别 **罗马数字 (I, II)**、**单独数字 (1, 2)**、**修饰符 (***)**，这些内容**不送翻译**，直接在回填时保留原文，以维持原书排版美感。
+
+#### B. 内容层 (Micro)
+对每个提取的段落调用 `FormatExtractor`：
+* **Inline Style**: 将 ``, `` 转为配对占位符 `φ1φ...φ/1φ`。
+* **Formula Protection**: 识别 $E=mc^2$ 等数学公式，保护为不可变占位符。
+* **Drop Cap Handling**:
+ - 原始: `The`
+ - 提取给 LLM: "The" (完整单词，无格式干扰)
+ - 记录: Prefix 包含 Drop Cap 样式。
+
+### 3. Manifest Management (清单管理)
+**模块**: `src/manifest_manager.py`
+Manifest 是系统的**核心状态中心 (Source of Truth)**。
+* **作用**: 解耦提取和翻译。提取器只管往 Manifest 填数据，翻译器只管从 Manifest 取数据。
+* **Persistence**: 支持中断续传，翻译进度实时保存。
+
+### 4. Translation with Profiling (翻译)
+**模块**: `src/book_profiler.py` & `src/translator.py`
+* **Profiling**: 在翻译前，抽取部分文本分析书籍的类型（技术、小说、诗歌）、核心术语和语言风格，生成 `System Prompt`。
+* **Translation**: 这是纯文本层面的转换，LLM 处理的是带有 `φ` 占位符的文本。
+
+### 5. Robust Restoration (健壮还原)
+**模块**: `src/format_restorer.py`
+负责将 LLM 返回的文本还原为 HTML。
+* **Drop Cap Logic**:
+ - **原文回填**: 需要 Prefix `T`。
+ - **译文回填**: **丢弃** Drop Cap Prefix。中文不需要首字母下沉，否则会出现 "T这本书..." 的怪诞结果。
+* **Error Handling**:
+ - **Missing Placeholders**: 如果 LLM 丢了 `φ1φ`，自动在末尾补全或报错重试。
+ - **Hallucinated Placeholders**: 移除 LLM 臆造的不存在 ID。
+
+### 6. Backfill Strategy (回填策略)
+**模块**: `src/fine_grained_extractor.py` (backfill method)
+支持多种模式，且**严格遵循一对一 (One-to-One) 映射**，绝不依赖顺序，而是依赖元素的内存引用或唯一 ID。
+
+* **Mode A: Bilingual (双语)**
+ - 保留原文 DOM。
+ - 在原文后 `append` 一个新元素 ` 译文 ",
+ "_suffix": "
等纯装饰性标签
+ if self.preserve_decorative:
+ for hr in soup.find_all('hr'):
+ elem_id = id(hr)
+ if elem_id not in processed_ids:
+ path = self.path_utils.get_dom_path(hr)
+ items.append({
+ 'path': path,
+ 'element': hr,
+ 'text': '---', # 用文本表示水平线
+ 'html': str(hr),
+ 'tag': 'hr',
+ 'is_navigation': False,
+ 'is_decorative': True
+ })
+ processed_ids.add(elem_id)
+
+ logger.info(f"提取了 {len(items)} 个文本元素 (包含 {sum(1 for i in items if i.get('is_decorative'))} 个装饰性元素)")
+ return items
+
+ def _is_decorative_element(self, element: Tag, text: str) -> bool:
+ """
+ 判断元素是否是装饰性元素
+
+ 装饰性元素的特征:
+ 1. 只包含符号(如 ***, ---, •••)
+ 2. 文本很短但包含特殊 Unicode 符号
+ 3. 有特定的 class (如 'separator', 'divider')
+ """
+ # 检查 class
+ classes = element.get('class', [])
+ class_str = ' '.join(classes).lower() if isinstance(classes, list) else str(classes).lower()
+
+ decorative_classes = ['separator', 'divider', 'ornament', 'decoration', 'break']
+ if any(dc in class_str for dc in decorative_classes):
+ return True
+
+ # 检查文本是否匹配装饰性模式
+ text_stripped = text.strip()
+ if not text_stripped:
+ return False
+
+ for pattern in self.DECORATIVE_PATTERNS:
+ if re.match(pattern, text_stripped):
+ return True
+
+ # 检查是否只包含少量重复字符
+ if len(text_stripped) <= 20:
+ unique_chars = set(text_stripped.replace(' ', ''))
+ if len(unique_chars) <= 3: # 只有1-3种不同字符
+ # 检查是否是常见装饰符号
+ decorative_chars = set('*-_~•◆◇■□▪▫●○◉◎⦿⦾')
+ if unique_chars & decorative_chars:
+ return True
+
+ return False
+
+ def _is_contained_in_processed(self, element: Tag, processed_ids: Set[int]) -> bool:
+ """检查元素是否被已处理的父元素包含"""
+ for parent in element.parents:
+ if isinstance(parent, Tag) and id(parent) in processed_ids:
+ return True
+ return False
+
+ def _clean_text(self, element: Tag) -> str:
+ """
+ 清理元素文本
+
+ 注意: 对于装饰性元素,保留原始符号
+ """
+ # 创建副本
+ element_copy = BeautifulSoup(str(element), 'html.parser').find(element.name)
+ if not element_copy:
+ return ""
+
+ # 移除脚注引用(但保留装饰性符号)
+ for tag in element_copy.find_all(['sup', 'sub']):
+ tag.decompose()
+
+ footnote_patterns = re.compile(r'footnote|endnote|reference|note|super|sub', re.I)
+ for tag in element_copy.find_all(['a', 'span', 'div'], class_=footnote_patterns):
+ tag.decompose()
+
+ # 移除仅包含数字的 span
+ for tag in element_copy.find_all('span'):
+ if re.match(r'^(\[\d+\]|\(\d+\)|\d+)$', tag.get_text().strip()):
+ tag.decompose()
+
+ text = element_copy.get_text().strip()
+
+ # 清理残留引用标识
+ text = re.sub(r'(\.|。|,|，)\s*(\[\d+\]|\d+)(?=\s|$)', r'\1', text)
+
+ # 对于非装饰性文本,压缩空白
+ # 对于装饰性文本,保留原样
+ if not self._is_decorative_element(element, text):
+ text = re.sub(r'\s+', ' ', text)
+
+ return text
+
+ def _is_navigation_element(self, element: Tag) -> bool:
+ """判断是否是导航元素"""
+ classes = element.get('class', [])
+ class_str = ' '.join(classes).lower() if isinstance(classes, list) else str(classes).lower()
+
+ if any(keyword in class_str for keyword in self.NAV_KEYWORDS):
+ return True
+
+ parent = element.parent
+ if parent and isinstance(parent, Tag):
+ p_classes = parent.get('class', [])
+ p_class_str = ' '.join(p_classes).lower() if isinstance(p_classes, list) else str(p_classes).lower()
+ if any(keyword in p_class_str for keyword in self.NAV_KEYWORDS):
+ return True
+
+ return False
+
+ def backfill(self, html_content: str, translation_map: Dict[str, str]) -> str:
+ """
+ 使用 DOM 路径精准回填翻译
+
+ 对于装饰性元素,保持原样不翻译
+ """
+ soup = BeautifulSoup(html_content, 'html.parser')
+
+ success_count = 0
+ fail_count = 0
+ decorative_kept = 0
+
+ for path, translation in translation_map.items():
+ element = self.path_utils.find_by_path(soup, path)
+
+ if element is None:
+ logger.warning(f"回填失败: 未找到路径 {path}")
+ fail_count += 1
+ continue
+
+ # 检查是否是装饰性元素
+ original_text = element.get_text().strip()
+ if self._is_decorative_element(element, original_text):
+ # 装饰性元素保持原样
+ decorative_kept += 1
+ continue
+
+ # 创建新元素
+ new_tag = soup.new_tag(element.name)
+ new_tag.string = translation
+
+ # 复制属性
+ for attr, value in element.attrs.items():
+ new_tag[attr] = value
+
+ # 替换
+ element.replace_with(new_tag)
+ success_count += 1
+
+ logger.info(f"回填完成: 成功 {success_count}, 失败 {fail_count}, 装饰性元素保留 {decorative_kept}")
+ return str(soup)
diff --git a/archive/v0.09/tests/extraction_experiment/extractors/final_extractor.py b/archive/v0.09/tests/extraction_experiment/extractors/final_extractor.py
new file mode 100644
index 0000000..675ba0d
--- /dev/null
+++ b/archive/v0.09/tests/extraction_experiment/extractors/final_extractor.py
@@ -0,0 +1,319 @@
+"""
+最终版智能提取器
+
+明确的翻译策略:
+1. 正文: 100% 翻译
+2. 目录: 全翻译或全不翻译 (根据配置)
+3. 索引/参考文献/尾注: 明确不翻译
+4. 装饰性元素: 不翻译
+"""
+
+from bs4 import BeautifulSoup, Tag
+from typing import List, Dict, Any, Set
+import re
+from loguru import logger
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from dom_path_utils import DOMPathUtils
+
+
+class FinalExtractor:
+ """最终版智能提取器"""
+
+ # 明确不翻译的文档
+ SKIP_TRANSLATION_PATTERNS = [
+ r'index\.x?html', # 索引
+ r'bibliography\.x?html', # 参考文献
+ r'endnotes?\.x?html', # 尾注
+ r'footnotes?\.x?html', # 脚注
+ ]
+
+ # 目录文档 (可配置是否翻译)
+ TOC_PATTERNS = [
+ r'nav\.x?html',
+ r'toc\.x?html',
+ ]
+
+ # 其他非核心文档 (通常不翻译)
+ OTHER_NON_CORE_PATTERNS = [
+ r'copyright\.x?html',
+ r'title\.x?html',
+ r'cover\.x?html',
+ ]
+
+ BLOCK_TAGS = [
+ 'p', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+ 'blockquote', 'li', 'td', 'dd', 'dt', 'figcaption',
+ 'section', 'article', 'aside', 'header', 'footer', 'main'
+ ]
+
+ DECORATIVE_PATTERNS = [
+ r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾]+$',
+ r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾\s]+$',
+ r'^[\u2022-\u2027\u2030-\u205E]+$',
+ ]
+
+ def __init__(self, translate_toc: bool = False, preserve_decorative: bool = True):
+ """
+ 初始化提取器
+
+ Args:
+ translate_toc: 是否翻译目录 (默认不翻译)
+ preserve_decorative: 是否保留装饰性元素
+ """
+ self.translate_toc = translate_toc
+ self.preserve_decorative = preserve_decorative
+ self.path_utils = DOMPathUtils()
+
+ def extract(self, html_content: str, file_name: str = "") -> List[Dict[str, Any]]:
+ """
+ 提取文本元素
+
+ 每个 DOM 节点作为一个独立单位,不合并
+ """
+ soup = BeautifulSoup(html_content, 'html.parser')
+
+ # 移除不需要的元素
+ for element in soup(['script', 'style', 'meta', 'link']):
+ element.decompose()
+
+ # 判断文档类型
+ doc_type = self._classify_document(file_name)
+
+ items = []
+ processed_ids = set()
+
+ # 遍历所有块级元素
+ for element in soup.find_all(self.BLOCK_TAGS):
+ elem_id = id(element)
+
+ if elem_id in processed_ids:
+ continue
+
+ if self._is_contained_in_processed(element, processed_ids):
+ continue
+
+ # 提取文本 (100% 完整)
+ text = element.get_text(separator=' ', strip=True)
+
+ if not text.strip():
+ continue
+
+ # 判断是否装饰性
+ is_decorative = self._is_decorative_element(element, text)
+
+ # 生成路径
+ path = self.path_utils.get_dom_path(element)
+
+ # 决定是否翻译
+ should_translate = self._should_translate(doc_type, is_decorative)
+
+ items.append({
+ 'path': path,
+ 'element': element,
+ 'text': text,
+ 'html': str(element),
+ 'tag': element.name,
+ 'is_decorative': is_decorative,
+ 'doc_type': doc_type,
+ 'should_translate': should_translate,
+ 'file_name': file_name
+ })
+
+ processed_ids.add(elem_id)
+
+ # 添加
+ if self.preserve_decorative:
+ for hr in soup.find_all('hr'):
+ elem_id = id(hr)
+ if elem_id not in processed_ids:
+ path = self.path_utils.get_dom_path(hr)
+ items.append({
+ 'path': path,
+ 'element': hr,
+ 'text': '---',
+ 'html': str(hr),
+ 'tag': 'hr',
+ 'is_decorative': True,
+ 'doc_type': doc_type,
+ 'should_translate': False,
+ 'file_name': file_name
+ })
+ processed_ids.add(elem_id)
+
+ # 统计
+ translate_count = sum(1 for i in items if i['should_translate'])
+ skip_count = sum(1 for i in items if not i['should_translate'] and not i['is_decorative'])
+ decorative_count = sum(1 for i in items if i['is_decorative'])
+
+ logger.info(
+ f"[{doc_type}] 提取 {len(items)} 个元素: "
+ f"翻译 {translate_count}, 跳过 {skip_count}, 装饰 {decorative_count}"
+ )
+
+ return items
+
+ def _classify_document(self, file_name: str) -> str:
+ """
+ 分类文档类型
+
+ Returns:
+ 'core' - 核心正文
+ 'toc' - 目录
+ 'skip' - 明确跳过 (索引/参考文献/尾注)
+ 'other' - 其他非核心
+ """
+ if not file_name:
+ return 'core'
+
+ file_name_lower = file_name.lower()
+
+ # 检查是否是明确跳过的
+ for pattern in self.SKIP_TRANSLATION_PATTERNS:
+ if re.search(pattern, file_name_lower):
+ return 'skip'
+
+ # 检查是否是目录
+ for pattern in self.TOC_PATTERNS:
+ if re.search(pattern, file_name_lower):
+ return 'toc'
+
+ # 检查其他非核心
+ for pattern in self.OTHER_NON_CORE_PATTERNS:
+ if re.search(pattern, file_name_lower):
+ return 'other'
+
+ return 'core'
+
+ def _should_translate(self, doc_type: str, is_decorative: bool) -> bool:
+ """
+ 决定是否翻译
+
+ 规则:
+ 1. 装饰性: 不翻译
+ 2. core: 翻译
+ 3. toc: 根据配置
+ 4. skip: 不翻译
+ 5. other: 不翻译
+ """
+ if is_decorative:
+ return False
+
+ if doc_type == 'core':
+ return True
+
+ if doc_type == 'toc':
+ return self.translate_toc
+
+ # skip 和 other 都不翻译
+ return False
+
+ def _is_contained_in_processed(self, element: Tag, processed_ids: Set[int]) -> bool:
+ """检查元素是否被已处理的父元素包含"""
+ for parent in element.parents:
+ if isinstance(parent, Tag) and id(parent) in processed_ids:
+ return True
+ return False
+
+ def _is_decorative_element(self, element: Tag, text: str) -> bool:
+ """判断是否是装饰性元素"""
+ classes = element.get('class', [])
+ class_str = ' '.join(classes).lower() if isinstance(classes, list) else str(classes).lower()
+
+ decorative_classes = ['separator', 'divider', 'ornament', 'decoration', 'break']
+ if any(dc in class_str for dc in decorative_classes):
+ return True
+
+ text_stripped = text.strip()
+ if not text_stripped:
+ return False
+
+ for pattern in self.DECORATIVE_PATTERNS:
+ if re.match(pattern, text_stripped):
+ return True
+
+ if len(text_stripped) <= 20:
+ unique_chars = set(text_stripped.replace(' ', ''))
+ if len(unique_chars) <= 3:
+ decorative_chars = set('*-_~•◆◇■□▪▫●○◉◎⦿⦾')
+ if unique_chars & decorative_chars:
+ return True
+
+ return False
+
+ def backfill(self, html_content: str, translation_map: Dict[str, str]) -> str:
+ """
+ 精准回填翻译
+
+ 保留原始 HTML 结构和样式,只替换文本内容
+ """
+ soup = BeautifulSoup(html_content, 'html.parser')
+
+ success_count = 0
+ fail_count = 0
+
+ for path, translation in translation_map.items():
+ element = self.path_utils.find_by_path(soup, path)
+
+ if element is None:
+ logger.warning(f"回填失败: 未找到路径 {path}")
+ fail_count += 1
+ continue
+
+ # 保留原始元素结构,只替换文本节点
+ self._replace_text_nodes(element, translation)
+ success_count += 1
+
+ logger.info(f"回填完成: 成功 {success_count}, 失败 {fail_count}")
+ return str(soup)
+
+ def _replace_text_nodes(self, element: Tag, new_text: str):
+ """
+ 智能替换元素中的文本节点,完全保留 HTML 结构
+
+ 策略:
+ 1. 如果元素只包含纯文本(无子标签),直接替换
+ 2. 如果元素包含子标签,递归查找并替换所有文本节点
+ 3. 保留所有属性、class、style 等
+ """
+ from bs4 import NavigableString
+
+ # 检查是否有子标签
+ child_tags = [child for child in element.children if isinstance(child, Tag)]
+
+ if not child_tags:
+ # 只有文本节点,直接替换
+ element.clear()
+ element.string = new_text
+ else:
+ # 有子标签,需要智能处理
+ # 策略: 找到所有文本节点,用新文本替换
+ self._replace_all_text_nodes(element, new_text)
+
+ def _replace_all_text_nodes(self, element: Tag, new_text: str):
+ """
+ 递归替换元素中的所有文本节点
+
+ 保留所有子元素和属性,只替换文本内容
+ """
+ from bs4 import NavigableString
+
+ # 收集所有文本节点
+ text_nodes = []
+ for child in element.descendants:
+ if isinstance(child, NavigableString) and not isinstance(child, (type(None),)):
+ # 跳过空白文本
+ if child.strip():
+ text_nodes.append(child)
+
+ if not text_nodes:
+ # 没有文本节点,直接设置
+ element.string = new_text
+ return
+
+ # 简化策略: 清空所有内容,保留结构,设置新文本
+ # 这会丢失内部格式,但保留外层容器的所有属性
+ element.clear()
+ element.string = new_text
diff --git a/archive/v0.09/tests/extraction_experiment/extractors/fine_grained.py b/archive/v0.09/tests/extraction_experiment/extractors/fine_grained.py
new file mode 100644
index 0000000..aaaf4a9
--- /dev/null
+++ b/archive/v0.09/tests/extraction_experiment/extractors/fine_grained.py
@@ -0,0 +1,232 @@
+"""
+细粒度提取器
+
+策略: 提取所有
等装饰性标签
+ if self.preserve_decorative:
+ for hr in soup.find_all('hr'):
+ elem_id = id(hr)
+ if elem_id not in processed_ids:
+ path = self.path_utils.get_dom_path(hr)
+ items.append({
+ 'path': path,
+ 'element': hr,
+ 'text': '---',
+ 'html': str(hr),
+ 'tag': 'hr',
+ 'is_decorative': True,
+ 'is_core': is_core_document,
+ 'should_translate': False,
+ 'file_name': file_name
+ })
+ processed_ids.add(elem_id)
+
+ # 统计
+ core_count = sum(1 for i in items if i['is_core'])
+ translate_count = sum(1 for i in items if i['should_translate'])
+ decorative_count = sum(1 for i in items if i['is_decorative'])
+
+ logger.info(
+ f"提取了 {len(items)} 个元素 "
+ f"(核心: {core_count}, 需翻译: {translate_count}, 装饰性: {decorative_count})"
+ )
+
+ return items
+
+ def _is_core_document(self, file_name: str) -> bool:
+ """
+ 判断是否是核心文档(正文)
+
+ 非核心文档包括: 目录、索引、参考文献、版权页等
+ """
+ if not file_name:
+ return True # 默认认为是核心文档
+
+ file_name_lower = file_name.lower()
+
+ for pattern in self.NON_CORE_PATTERNS:
+ if re.search(pattern, file_name_lower):
+ return False
+
+ return True
+
+ def _should_translate(self, element: Tag, text: str,
+ is_core_document: bool, is_decorative: bool) -> bool:
+ """
+ 决定元素是否应该翻译
+
+ 规则:
+ 1. 装饰性元素: 不翻译
+ 2. 核心文档: 全部翻译
+ 3. 非核心文档: 根据复杂度决定
+ """
+ # 装饰性元素不翻译
+ if is_decorative:
+ return False
+
+ # 核心文档全部翻译
+ if is_core_document:
+ return True
+
+ # 非核心文档: 检查复杂度
+ complexity = self._calculate_complexity(element, text)
+
+ # 复杂度阈值: 如果太复杂,不翻译
+ if complexity > 0.5:
+ logger.debug(f"非核心元素复杂度过高 ({complexity:.2f}), 跳过翻译: {text[:50]}")
+ return False
+
+ return True
+
+ def _calculate_complexity(self, element: Tag, text: str) -> float:
+ """
+ 计算元素的复杂度
+
+ 复杂度指标:
+ - 嵌套深度
+ - 链接数量
+ - 数字比例
+ - 特殊字符比例
+
+ Returns:
+ 0.0 - 1.0, 越高越复杂
+ """
+ complexity_score = 0.0
+
+ # 1. 嵌套深度 (最大贡献 0.3)
+ depth = len(list(element.parents))
+ complexity_score += min(depth / 20, 0.3)
+
+ # 2. 链接数量 (最大贡献 0.3)
+ links = element.find_all('a')
+ if links:
+ link_ratio = len(links) / max(len(text.split()), 1)
+ complexity_score += min(link_ratio, 0.3)
+
+ # 3. 数字比例 (最大贡献 0.2)
+ digits = sum(c.isdigit() for c in text)
+ if text:
+ digit_ratio = digits / len(text)
+ complexity_score += min(digit_ratio * 2, 0.2)
+
+ # 4. 特殊字符比例 (最大贡献 0.2)
+ special_chars = sum(1 for c in text if not c.isalnum() and not c.isspace())
+ if text:
+ special_ratio = special_chars / len(text)
+ complexity_score += min(special_ratio * 2, 0.2)
+
+ return min(complexity_score, 1.0)
+
+ def _is_contained_in_processed(self, element: Tag, processed_ids: Set[int]) -> bool:
+ """检查元素是否被已处理的父元素包含"""
+ for parent in element.parents:
+ if isinstance(parent, Tag) and id(parent) in processed_ids:
+ return True
+ return False
+
+ def _extract_text(self, element: Tag) -> str:
+ """
+ 提取元素文本 - 100% 完整提取,不过滤任何内容
+
+ 注意: 这里不做任何清理,保证 100% 提取
+ """
+ return element.get_text(separator=' ', strip=True)
+
+ def _is_decorative_element(self, element: Tag, text: str) -> bool:
+ """判断是否是装饰性元素"""
+ # 检查 class
+ classes = element.get('class', [])
+ class_str = ' '.join(classes).lower() if isinstance(classes, list) else str(classes).lower()
+
+ decorative_classes = ['separator', 'divider', 'ornament', 'decoration', 'break']
+ if any(dc in class_str for dc in decorative_classes):
+ return True
+
+ # 检查文本模式
+ text_stripped = text.strip()
+ if not text_stripped:
+ return False
+
+ for pattern in self.DECORATIVE_PATTERNS:
+ if re.match(pattern, text_stripped):
+ return True
+
+ # 检查重复字符
+ if len(text_stripped) <= 20:
+ unique_chars = set(text_stripped.replace(' ', ''))
+ if len(unique_chars) <= 3:
+ decorative_chars = set('*-_~•◆◇■□▪▫●○◉◎⦿⦾')
+ if unique_chars & decorative_chars:
+ return True
+
+ return False
+
+ def backfill(self, html_content: str, translation_map: Dict[str, str]) -> str:
+ """
+ 精准回填翻译
+
+ 只回填 should_translate=True 的元素
+ """
+ soup = BeautifulSoup(html_content, 'html.parser')
+
+ success_count = 0
+ skip_count = 0
+ fail_count = 0
+
+ for path, translation in translation_map.items():
+ element = self.path_utils.find_by_path(soup, path)
+
+ if element is None:
+ logger.warning(f"回填失败: 未找到路径 {path}")
+ fail_count += 1
+ continue
+
+ # 检查是否应该翻译
+ # (这个信息应该在 translation_map 的构建阶段就过滤了)
+
+ # 创建新元素
+ new_tag = soup.new_tag(element.name)
+ new_tag.string = translation
+
+ # 复制属性
+ for attr, value in element.attrs.items():
+ new_tag[attr] = value
+
+ # 替换
+ element.replace_with(new_tag)
+ success_count += 1
+
+ logger.info(f"回填完成: 成功 {success_count}, 跳过 {skip_count}, 失败 {fail_count}")
+ return str(soup)
diff --git a/archive/v0.09/tests/extraction_experiment/reports/multi_epub_test_report.md b/archive/v0.09/tests/extraction_experiment/reports/multi_epub_test_report.md
new file mode 100644
index 0000000..773da85
--- /dev/null
+++ b/archive/v0.09/tests/extraction_experiment/reports/multi_epub_test_report.md
@@ -0,0 +1,90 @@
+# 多 ePub 提取完整性测试报告
+
+**测试时间**: 2026-01-19 12:26:59
+
+**测试文件数**: 5
+
+**成功**: 5/5
+
+## 测试结果汇总
+
+| 文件名 | 文件大小 | 提取元素 | 装饰性 | 文本长度 | 覆盖率 |
+|--------|---------|---------|--------|---------|--------|
+| Gambling Man.epub | 3428.3KB | 60 | 0 | 723,609 | 77.7% |
+| On_China_Henry_Kissinger.epub | 924.5KB | 3602 | 32 | 1,141,726 | 87.6% |
+| The World Atlas of Coffee - Fr | 20406.0KB | 1621 | 340 | 353,627 | 74.6% |
+| The_Philosopher_in_the_Valley. | 4687.7KB | 54 | 15 | 524,578 | 93.9% |
+| To_Explain_the_World.epub | 1756.4KB | 3564 | 105 | 782,608 | 87.9% |
+
+## 详细分析
+
+### Gambling Man.epub
+
+- **HTML 文档数**: 47
+- **提取元素总数**: 60
+ - 内容元素: 51
+ - 装饰性元素: 0
+ - 导航元素: 9
+- **提取文本长度**: 723,609 字符
+- **提取词数**: 118,115
+- **Pandoc 基准长度**: 889,384 字符
+- **覆盖率**: 77.70%
+- **共同词数**: 11,607
+
+### On_China_Henry_Kissinger.epub
+
+- **HTML 文档数**: 144
+- **提取元素总数**: 3602
+ - 内容元素: 3570
+ - 装饰性元素: 32
+ - 导航元素: 0
+- **提取文本长度**: 1,141,726 字符
+- **提取词数**: 181,503
+- **Pandoc 基准长度**: 1,509,848 字符
+- **覆盖率**: 87.57%
+- **共同词数**: 12,834
+
+### The World Atlas of Coffee - From Beans to Brewing - Coffees Explored, Explained and Enjoyed (James Hoffmann) (Z-Library).epub
+
+- **HTML 文档数**: 98
+- **提取元素总数**: 1621
+ - 内容元素: 1254
+ - 装饰性元素: 340
+ - 导航元素: 27
+- **提取文本长度**: 353,627 字符
+- **提取词数**: 59,885
+- **Pandoc 基准长度**: 560,850 字符
+- **覆盖率**: 74.62%
+- **共同词数**: 5,885
+
+### The_Philosopher_in_the_Valley.epub
+
+- **HTML 文档数**: 22
+- **提取元素总数**: 54
+ - 内容元素: 24
+ - 装饰性元素: 15
+ - 导航元素: 15
+- **提取文本长度**: 524,578 字符
+- **提取词数**: 86,566
+- **Pandoc 基准长度**: 550,137 字符
+- **覆盖率**: 93.95%
+- **共同词数**: 9,760
+
+### To_Explain_the_World.epub
+
+- **HTML 文档数**: 105
+- **提取元素总数**: 3564
+ - 内容元素: 3459
+ - 装饰性元素: 105
+ - 导航元素: 0
+- **提取文本长度**: 782,608 字符
+- **提取词数**: 133,415
+- **Pandoc 基准长度**: 950,720 字符
+- **覆盖率**: 87.90%
+- **共同词数**: 9,237
+
+## 总结
+
+- **平均覆盖率**: 84.35%
+- **总装饰性元素**: 492 个
+- **提取器状态**: ⚠️ 需要优化
\ No newline at end of file
diff --git a/archive/v0.09/tests/extraction_experiment/run_batch.py b/archive/v0.09/tests/extraction_experiment/run_batch.py
new file mode 100644
index 0000000..5259898
--- /dev/null
+++ b/archive/v0.09/tests/extraction_experiment/run_batch.py
@@ -0,0 +1,71 @@
+"""
+批量运行测试: 对 input 目录下的所有 EPUB 执行清理和生成双语版本
+"""
+
+import sys
+from pathlib import Path
+import os
+import time
+from loguru import logger
+
+# 配置路径
+project_root = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(project_root))
+sys.path.insert(0, str(Path(__file__).parent))
+
+# 导入功能模块
+from simple_cleaner import clean_epub
+from test_end_to_end import create_bilingual_epub
+
+def run_batch():
+ input_dir = project_root / "input"
+ output_dir = project_root / "test_output"
+ output_dir.mkdir(exist_ok=True)
+
+ epubs = list(input_dir.glob("*.epub"))
+ epubs.sort() # 按文件名排序
+
+ print(f"\n{'='*80}")
+ print(f"批量测试开始: 共 {len(epubs)} 个文件")
+ print(f"{'='*80}\n")
+
+ success_count = 0
+
+ for i, epub_path in enumerate(epubs, 1):
+ print(f"[{i}/{len(epubs)}] 📖 处理: {epub_path.name}")
+
+ cleaned_path = output_dir / f"{epub_path.stem}_cleaned.epub"
+ bilingual_path = output_dir / f"{epub_path.stem}_bilingual.epub"
+
+ try:
+ # 1. 清理
+ print(" ➤ 正在清理...")
+ start = time.time()
+ # 捕获日志或只允许 ERROR? 暂时保持默认
+ clean_epub(str(epub_path), str(cleaned_path))
+ print(f" ✓ 清理完成用时: {time.time() - start:.2f}s")
+
+ # 2. 生成双语
+ print(" ➤ 正在生成双语版本...")
+ start = time.time()
+ create_bilingual_epub(cleaned_path, bilingual_path)
+ print(f" ✓ 生成完成用时: {time.time() - start:.2f}s")
+
+ print(f" ✅ 成功! 输出: {bilingual_path.name}\n")
+ success_count += 1
+
+ except Exception as e:
+ print(f" ❌ 处理失败: {e}\n")
+ # 不中断后续任务
+ continue
+
+ print(f"{'='*80}")
+ print(f"批量测试结束: 成功 {success_count}/{len(epubs)}")
+ print(f"{'='*80}\n")
+
+if __name__ == "__main__":
+ # 配置 logger 只显示 WARNING 以上，以免刷屏
+ logger.remove()
+ logger.add(sys.stderr, level="WARNING")
+
+ run_batch()
diff --git a/archive/v0.09/tests/extraction_experiment/simple_cleaner.py b/archive/v0.09/tests/extraction_experiment/simple_cleaner.py
new file mode 100644
index 0000000..d44c866
--- /dev/null
+++ b/archive/v0.09/tests/extraction_experiment/simple_cleaner.py
@@ -0,0 +1,205 @@
+"""
+简化版 Calibre 清理器 - 避免复杂操作
+
+只做最基本的清理:
+1. div 转 p
+2. 移除 calibre 类
+"""
+
+from bs4 import BeautifulSoup, Tag
+from loguru import logger
+
+
+class SimpleCleaner:
+ """简化版清理器"""
+
+ def clean(self, html_content: str, item=None) -> str:
+ """
+ 清理 HTML
+
+ Args:
+ html_content: HTML 内容
+ item: EpubItem 对象(可选), 用于注册 links
+ """
+ soup = BeautifulSoup(html_content, 'html.parser')
+
+ # 0. 提取并保留 CSS 链接 (解决 ebooklib 丢失 link 的问题)
+ if item:
+ head = soup.find('head')
+ if head:
+ # 提取 link
+ links = head.find_all('link', rel='stylesheet')
+ for link in links:
+ href = link.get('href')
+ if href:
+ # 检查是否已存在(避免重复)
+ existing_links = list(item.get_links())
+
+ exists = False
+ for l in existing_links:
+ l_href = getattr(l, 'href', None)
+ if l_href is None and isinstance(l, dict):
+ l_href = l.get('href')
+
+ if l_href == href:
+ exists = True
+ break
+
+ if not exists:
+ logger.debug(f"恢复 CSS 链接: {href}")
+ item.add_link(href=href, rel='stylesheet', type='text/css')
+
+ stats = {'divs_to_p': 0, 'classes_removed': 0}
+
+ # 1. div 转 p (只转换没有块级子元素的)
+ inline_tags = {'span', 'em', 'strong', 'i', 'b', 'u', 'a', 'br'}
+
+ divs = list(soup.find_all('div')) # 先收集所有div
+ for div in divs:
+ has_block = any(
+ isinstance(c, Tag) and c.name not in inline_tags
+ for c in div.children
+ )
+
+ if not has_block:
+ div.name = 'p'
+ stats['divs_to_p'] += 1
+
+ # 2. 清理 calibre 类 - 暂时禁用，以保留样式
+ # elements = list(soup.find_all(class_=True)) # 先收集
+ # ... (保留原注释代码)
+
+ logger.info(f"清理完成: div→p {stats['divs_to_p']}, 类移除 {stats['classes_removed']}")
+
+ return str(soup)
+
+
+def clean_epub(input_path: str, output_path: str):
+ """清理 ePub"""
+ from ebooklib import epub
+ import zipfile
+
+ logger.info(f"开始清理: {input_path}")
+
+ # 打开 zip 以读取原始内容
+ try:
+ input_zip = zipfile.ZipFile(input_path, 'r')
+ zip_files = set(input_zip.namelist())
+ except Exception as e:
+ logger.error(f"无法打开 Zip: {e}")
+ input_zip = None
+ zip_files = set()
+
+ book = epub.read_epub(input_path)
+ cleaner = SimpleCleaner()
+
+ count = 0
+ for item in book.get_items():
+ if item.get_type() == 9:
+ try:
+ file_name = item.get_name()
+ content = None
+
+ # 优先从 Zip 读取以保留 Head 信息
+ if input_zip and file_name in zip_files:
+ try:
+ content = input_zip.read(file_name).decode('utf-8')
+ except Exception as e:
+ logger.warning(f"Zip 读取失败 {file_name}: {e}")
+
+ # 回退到 ebooklib
+ if content is None:
+ raw_content = item.get_content()
+ if raw_content:
+ content = raw_content.decode('utf-8')
+
+ # 检查原始内容
+ if not content or not content.strip():
+ logger.warning(f"跳过空文档: {item.get_name()}")
+ continue
+
+ # 清理并提取信息
+ # 注意: 我们需要传入 item 以便 cleaner 可以注册 links
+ cleaned = cleaner.clean(content, item)
+
+ # 检查清理后内容
+ if not cleaned.strip():
+ logger.error(f"⚠️ 清理后内容为空: {item.get_name()} (原始长度: {len(content)})")
+ # 如果清理变为空，保留原始内容
+ cleaned = content
+
+ item.set_content(cleaned.encode('utf-8'))
+ count += 1
+
+ if 'titlepage' in item.get_name():
+ logger.info(f"Titlepage 处理完成: {len(cleaned)} chars")
+
+ except Exception as e:
+ logger.warning(f"清理失败 {item.get_name()}: {e}")
+
+
+ # 修复 TOC：补全 UID 并移除指向不存在文件的死链
+ def fix_and_clean_toc(toc, book):
+ new_toc = []
+ import uuid
+ from ebooklib.epub import Link
+
+ for item in toc:
+ # Case 1: (Section, Children) 元组
+ if isinstance(item, (tuple, list)):
+ section, children = item
+ # 递归清理子节点
+ cleaned_children = fix_and_clean_toc(children, book)
+
+ # 检查 Section 节点
+ if isinstance(section, Link):
+ href = section.href.split('#')[0]
+ # 有效性检查：目标文件必须在 manifest 中存在
+ if book.get_item_with_href(href):
+ if section.uid is None:
+ section.uid = f'uuid-{uuid.uuid4()}'
+ new_toc.append((section, cleaned_children))
+ else:
+ logger.warning(f"移除无效 TOC 节点 (目标缺失): {section.title} -> {section.href}")
+ # 如果父节点无效，这里选择提升子节点，还是丢弃？
+ # 策略：如果父节点都无效了，就把子节点提升上来（如果子节点有效）
+ new_toc.extend(cleaned_children)
+ else:
+ # 如果 Section 不是 Link (罕见)，保留
+ new_toc.append((section, cleaned_children))
+
+ # Case 2: 单个 Link 节点
+ elif isinstance(item, Link):
+ href = item.href.split('#')[0]
+ if book.get_item_with_href(href):
+ if item.uid is None:
+ item.uid = f'uuid-{uuid.uuid4()}'
+ new_toc.append(item)
+ else:
+ logger.warning(f"移除无效 TOC 节点 (目标缺失): {item.title} -> {item.href}")
+
+ # Case 3: 其他 (如自定义 dict 等? 一般不会)
+ else:
+ new_toc.append(item)
+
+ return new_toc
+
+ try:
+ book.toc = fix_and_clean_toc(book.toc, book)
+ except Exception as e:
+ logger.warning(f"修复 TOC 失败: {e}")
+ import traceback
+ logger.warning(traceback.format_exc())
+
+ epub.write_epub(output_path, book)
+ logger.info(f"完成: 处理了 {count} 个文档")
+
+
+if __name__ == "__main__":
+ import sys
+
+ if len(sys.argv) < 3:
+ print("用法: python simple_cleaner.py A famous quote here.
+Chapter One
+
+"""
+
+ print("\n" + "="*80)
+ print("简单 HTML 骨架保留测试")
+ print("="*80 + "\n")
+
+ # 提取
+ extractor = BS4SkeletonExtractor()
+ items = extractor.extract(html)
+
+ print(f"提取了 {len(items)} 个元素:\n")
+
+ for i, item in enumerate(items, 1):
+ print(f"{i}. [{item['tag']}] {item['text'][:50]}")
+
+ # 模拟翻译
+ translation_map = {}
+ for item in items:
+ if item['should_translate']:
+ translation_map[item['text']] = f"{item['text']} [翻译]"
+
+ print(f"\n待翻译: {len(translation_map)} 个元素\n")
+
+ # 回填
+ result_html = extractor.backfill(items, translation_map)
+
+ print("="*80)
+ print("回填后的 HTML:")
+ print("="*80 + "\n")
+ print(result_html)
+
+ # 验证
+ print("\n" + "="*80)
+ print("验证结果:")
+ print("="*80 + "\n")
+
+ checks = [
+ ('class="calibre3"', 'class 属性'),
+ ('class="text-center"', 'class 属性'),
+ ('style="font-size: 18px; color: blue;"', 'style 属性'),
+ ('style="margin-left: 40px;"', 'style 属性'),
+ ('id="chapter1"', 'id 属性'),
+ ('', '内部格式标签'),
+ ('', '嵌套标签'),
+ ]
+
+ for pattern, name in checks:
+ if pattern in result_html:
+ print(f"✅ {name} 保留: {pattern}")
+ else:
+ print(f"❌ {name} 丢失: {pattern}")
+
+
+def test_real_epub():
+ """测试真实 ePub"""
+ epub_path = project_root / "input" / "On_China_Henry_Kissinger.epub"
+
+ if not epub_path.exists():
+ print(f"\n跳过真实 ePub 测试: 文件不存在")
+ return
+
+ print("\n" + "="*80)
+ print("真实 ePub 骨架保留测试")
+ print("="*80 + "\n")
+
+ book = epub.read_epub(str(epub_path))
+
+ # 找第一个内容文档
+ for item in book.get_items():
+ if item.get_type() == 9 and 'dummy_split_002' in item.get_name():
+ content = item.get_content().decode('utf-8')
+
+ print(f"测试文件: {item.get_name()}\n")
+
+ # 统计原始 HTML 的属性
+ original_classes = len(re.findall(r'class="[^"]*"', content))
+ original_styles = len(re.findall(r'style="[^"]*"', content))
+ original_ids = len(re.findall(r'id="[^"]*"', content))
+
+ print(f"原始 HTML 统计:")
+ print(f" - class 属性: {original_classes} 个")
+ print(f" - style 属性: {original_styles} 个")
+ print(f" - id 属性: {original_ids} 个\n")
+
+ # 提取
+ extractor = BS4SkeletonExtractor()
+ items = extractor.extract(content, item.get_name())
+
+ print(f"提取了 {len(items)} 个元素\n")
+
+ # 显示前 3 个
+ for i, elem in enumerate(items[:3], 1):
+ print(f"元素 {i}:")
+ print(f" 标签: <{elem['tag']}>")
+ print(f" 文本: {elem['text'][:60]}...")
+ print()
+
+ # 模拟翻译
+ translation_map = {}
+ for elem in items:
+ if elem['should_translate']:
+ translation_map[elem['text']] = f"{elem['text']} [翻译]"
+
+ print(f"待翻译: {len(translation_map)} 个元素\n")
+
+ # 回填
+ result_html = extractor.backfill(items, translation_map)
+
+ # 统计回填后的属性
+ result_classes = len(re.findall(r'class="[^"]*"', result_html))
+ result_styles = len(re.findall(r'style="[^"]*"', result_html))
+ result_ids = len(re.findall(r'id="[^"]*"', result_html))
+
+ print("="*80)
+ print("回填后 HTML 统计:")
+ print("="*80 + "\n")
+ print(f" - class 属性: {result_classes} 个")
+ print(f" - style 属性: {result_styles} 个")
+ print(f" - id 属性: {result_ids} 个\n")
+
+ # 验证
+ print("="*80)
+ print("验证结果:")
+ print("="*80 + "\n")
+
+ if original_classes == result_classes:
+ print(f"✅ 所有 class 属性保留 ({original_classes} 个)")
+ else:
+ print(f"❌ class 属性丢失: {original_classes} → {result_classes}")
+
+ if original_styles == result_styles:
+ print(f"✅ 所有 style 属性保留 ({original_styles} 个)")
+ else:
+ print(f"❌ style 属性丢失: {original_styles} → {result_styles}")
+
+ if original_ids == result_ids:
+ print(f"✅ 所有 id 属性保留 ({original_ids} 个)")
+ else:
+ print(f"❌ id 属性丢失: {original_ids} → {result_ids}")
+
+ # 检查翻译是否成功
+ if '[翻译]' in result_html:
+ print(f"✅ 翻译成功回填")
+ else:
+ print(f"❌ 翻译未回填")
+
+ break
+
+
+def main():
+ """主函数"""
+ logger.remove()
+ logger.add(sys.stderr, level="INFO")
+
+ # 测试 1: 简单 HTML
+ test_simple_html()
+
+ # 测试 2: 真实 ePub
+ test_real_epub()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/archive/v0.09/tests/extraction_experiment/test_calibre_cleaner.py b/archive/v0.09/tests/extraction_experiment/test_calibre_cleaner.py
new file mode 100644
index 0000000..68f7f5d
--- /dev/null
+++ b/archive/v0.09/tests/extraction_experiment/test_calibre_cleaner.py
@@ -0,0 +1,89 @@
+"""
+测试 Calibre 清理器
+"""
+
+import sys
+from pathlib import Path
+
+project_root = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(project_root))
+
+from loguru import logger
+from calibre_cleaner import CalibreHTMLCleaner
+
+
+def test_html_cleaning():
+ """测试 HTML 清理"""
+
+ # 测试用例: Calibre 生成的屎山代码
+ html = """
+ Chapter 1
+
+
+ Section 1.1
+ Chapter 1
+ A quote here.
+
+
+ """
+
+ print("\n" + "="*60)
+ print("测试 1: 简单 HTML 提取")
+ print("="*60)
+
+ # BS4 提取器
+ print("\n--- BS4 优化方案 ---")
+ bs4_extractor = BS4OptimizedExtractor(min_text_length=5)
+ bs4_items = bs4_extractor.extract(html)
+
+ print(f"提取元素数: {len(bs4_items)}")
+ for i, item in enumerate(bs4_items, 1):
+ nav_flag = " [导航]" if item['is_navigation'] else ""
+ print(f"{i}. [{item['tag']}] {item['text'][:50]}{nav_flag}")
+ print(f" 路径: {item['path']}")
+
+ # lxml 提取器
+ print("\n--- lxml XPath 方案 ---")
+ lxml_extractor = LxmlXPathExtractor(min_text_length=5)
+ lxml_items = lxml_extractor.extract(html)
+
+ print(f"提取元素数: {len(lxml_items)}")
+ for i, item in enumerate(lxml_items, 1):
+ nav_flag = " [导航]" if item['is_navigation'] else ""
+ print(f"{i}. [{item['tag']}] {item['text'][:50]}{nav_flag}")
+ print(f" XPath: {item['xpath']}")
+
+ return bs4_items, lxml_items
+
+
+def test_backfill(html, items, extractor, method_name):
+ """测试回填功能"""
+ print(f"\n--- {method_name} 回填测试 ---")
+
+ # 创建模拟翻译
+ translation_map = {}
+ for i, item in enumerate(items):
+ if not item.get('is_navigation', False):
+ path = item.get('path') or item.get('xpath')
+ translation_map[path] = f"TRANSLATED_{i:02d}"
+
+ print(f"待回填: {len(translation_map)} 个元素")
+
+ # 执行回填
+ backfilled_html = extractor.backfill(html, translation_map)
+
+ # 验证
+ soup = BeautifulSoup(backfilled_html, 'html.parser')
+ found_count = 0
+ for path, translation in translation_map.items():
+ if translation in soup.get_text():
+ found_count += 1
+
+ accuracy = found_count / len(translation_map) if translation_map else 0
+ print(f"回填准确性: {accuracy:.2%} ({found_count}/{len(translation_map)})")
+
+ return accuracy
+
+
+def test_epub_extraction(epub_path):
+ """测试真实 ePub 文件的提取"""
+ print("\n" + "="*60)
+ print(f"测试 2: ePub 文件提取 - {Path(epub_path).name}")
+ print("="*60)
+
+ # 加载 ePub
+ book = epub.read_epub(epub_path)
+
+ # 提取第一个 HTML 文档
+ html_content = None
+ for item in book.get_items():
+ if item.get_type() == 9: # ITEM_DOCUMENT
+ try:
+ html_content = item.get_content().decode('utf-8')
+ print(f"\n测试文件: {item.get_name()}")
+ break
+ except:
+ continue
+
+ if not html_content:
+ print("未找到 HTML 内容")
+ return
+
+ # BS4 提取
+ print("\n--- BS4 优化方案 ---")
+ bs4_extractor = BS4OptimizedExtractor()
+ bs4_items = bs4_extractor.extract(html_content)
+ print(f"提取元素数: {len(bs4_items)}")
+ print(f"总文本长度: {sum(len(item['text']) for item in bs4_items):,} 字符")
+
+ # 显示前5个元素
+ print("\n前 5 个元素:")
+ for i, item in enumerate(bs4_items[:5], 1):
+ print(f"{i}. [{item['tag']}] {item['text'][:80]}...")
+
+ # lxml 提取
+ print("\n--- lxml XPath 方案 ---")
+ lxml_extractor = LxmlXPathExtractor()
+ lxml_items = lxml_extractor.extract(html_content)
+ print(f"提取元素数: {len(lxml_items)}")
+ print(f"总文本长度: {sum(len(item['text']) for item in lxml_items):,} 字符")
+
+ # 显示前5个元素
+ print("\n前 5 个元素:")
+ for i, item in enumerate(lxml_items[:5], 1):
+ print(f"{i}. [{item['tag']}] {item['text'][:80]}...")
+
+ # 回填测试
+ print("\n" + "-"*60)
+ print("回填测试")
+ print("-"*60)
+
+ bs4_accuracy = test_backfill(html_content, bs4_items, bs4_extractor, "BS4")
+ lxml_accuracy = test_backfill(html_content, lxml_items, lxml_extractor, "lxml")
+
+ # 对比
+ print("\n" + "="*60)
+ print("对比总结")
+ print("="*60)
+ print(f"{'方案':<15} {'元素数':<10} {'回填准确性':<15}")
+ print("-"*60)
+ print(f"{'BS4 优化':<15} {len(bs4_items):<10} {bs4_accuracy:>13.2%}")
+ print(f"{'lxml XPath':<15} {len(lxml_items):<10} {lxml_accuracy:>13.2%}")
+
+
+def main():
+ """主函数"""
+ # 配置日志
+ logger.remove()
+ logger.add(sys.stderr, level="INFO")
+
+ # 测试 1: 简单 HTML
+ bs4_items, lxml_items = test_simple_html()
+
+ # 简单 HTML 回填测试
+ html = """
+
+
+ Chapter 1
+ Chapter 1: The Beginning
+ Section 1.1
+ The Detail
+