Feat: v0.12 Pipeline Separation, Spacing Fix, and Idempotent Restoration

archive/v0.12/src/assembly/backfiller.py

@@ -0,0 +1,107 @@
+from pathlib import Path
+from typing import List, Dict, Optional
+from bs4 import BeautifulSoup, NavigableString
+
+from src.common.data_model import BookStructure, ManifestEntry
+from src.common.utils import setup_logger
+
+logger = setup_logger("backfill_engine")
+
+class BackfillEngine:
+    """
+    Applies translations back to the BookStructure.
+    Now simplified to only inject 'translated_html' or fallback to text.
+    No format restoration logic here.
+    """
+    
+    def __init__(self):
+        # No more Restorer or LLM Client
+        pass
+        
+    async def backfill(self, structure: BookStructure, manifest_entries: List[ManifestEntry], mode: str = "bilingual") -> BookStructure:
+        """
+        Modifies the BookStructure in-place with translations.
+        
+        Args:
+            structure: The BookStructure (from book_structure.json).
+            manifest_entries: List of translations.
+            mode: 'bilingual' or 'target_only'.
+        """
+        logger.info(f"Backfilling with mode: {mode}")
+        
+        # Index manifest
+        manifest_map: Dict[str, Dict[str, ManifestEntry]] = {}
+        for entry in manifest_entries:
+            if not entry.translated_text:
+                continue 
+                
+            if entry.file_path not in manifest_map:
+                manifest_map[entry.file_path] = {}
+            manifest_map[entry.file_path][entry.element_id] = entry
+            
+        # Iterate resources
+        for item_id, resource in structure.resources.items():
+            if resource.media_type != "application/xhtml+xml" or resource.href not in manifest_map:
+                continue
+                
+            file_entries = manifest_map[resource.href]
+            if not file_entries:
+                continue
+            
+            logger.debug(f"Processing {resource.href} with {len(file_entries)} translations")
+            
+            soup = BeautifulSoup(resource.content, 'html.parser')
+            modified = False
+            
+            for element_id, entry in file_entries.items():
+                element = soup.find(id=element_id)
+                if not element:
+                    logger.warning(f"Element {element_id} not found in {resource.href}")
+                    continue
+                
+                # Determine content to inject
+                # Prefer translated_html (rich text), fallback to translated_text (plain text)
+                html_content = entry.translated_html
+                plain_text = entry.translated_text
+                
+                # Create translated tag
+                new_tag = soup.new_tag(element.name)
+                
+                if html_content:
+                    # Parse HTML fragment
+                    # Wrap in div to handle multiple top-level nodes
+                    inner_soup = BeautifulSoup(f"<div>{html_content}</div>", 'html.parser')
+                    container = inner_soup.find('div')
+                    if container:
+                        for child in list(container.children):
+                            new_tag.append(child)
+                    else:
+                        new_tag.string = plain_text
+                else:
+                    # Fallback to plain text
+                    new_tag.string = plain_text
+
+                # Copy attributes
+                # Copy classes and add 'translation'
+                classes = element.get('class', [])
+                if isinstance(classes, str):
+                    classes = classes.split()
+                new_tag['class'] = classes + ['translation']
+                
+                # Copy style
+                style = element.get('style')
+                if style:
+                    new_tag['style'] = style
+                
+                # Injection Strategy
+                if mode == "bilingual":
+                    element.insert_after(new_tag)
+                else:
+                    element.replace_with(new_tag)
+                
+                modified = True
+                
+            if modified:
+                resource.content = str(soup)
+                
+        return structure

archive/v0.12/src/assembly/builder.py

@@ -0,0 +1,289 @@
+import shutil
+import uuid
+from pathlib import Path
+from ebooklib import epub
+from src.common.data_model import BookStructure
+from src.common.utils import setup_logger, ensure_directory
+
+logger = setup_logger("bilingual_builder")
+
+class BilingualBuilder:
+    """
+    Assembles the final EPUB from BookStructure.
+    
+    Preserves the original TOC structure by reading it from the original EPUB.
+    """
+    
+    def __init__(self, work_dir: Path, original_epub_path: Path = None):
+        self.work_dir = work_dir
+        self.assets_dir = work_dir / "assets"
+        self.original_epub_path = original_epub_path
+        self._original_book = None
+        
+    def _load_original_book(self):
+        """Lazy load original book for TOC extraction."""
+        if self._original_book is None and self.original_epub_path and self.original_epub_path.exists():
+            try:
+                self._original_book = epub.read_epub(str(self.original_epub_path))
+                logger.debug(f"Loaded original EPUB for TOC: {self.original_epub_path}")
+            except Exception as e:
+                logger.warning(f"Failed to load original EPUB: {e}")
+        return self._original_book
+    
+    def _sanitize_toc(self, toc):
+        """
+        Ensure all TOC nodes have IDs (for ebooklib compatibility).
+        From v0.08 bilingual_builder.py
+        """
+        result = []
+        for item in toc:
+            if isinstance(item, (epub.Link, epub.Section)):
+                if not getattr(item, 'uid', None):
+                    item.uid = f"navPoint-{uuid.uuid4().hex[:8]}"
+                result.append(item)
+            elif isinstance(item, tuple) and len(item) == 2:
+                # Handle (Section, [children]) structure
+                section, children = item
+                if isinstance(section, (epub.Link, epub.Section)):
+                    if not getattr(section, 'uid', None):
+                        section.uid = f"navPoint-{uuid.uuid4().hex[:8]}"
+                sanitized_children = self._sanitize_toc(children)
+                result.append((section, sanitized_children))
+            else:
+                result.append(item)
+        return result
+        
+    def _validate_and_fix_toc(self, toc, book_items):
+        """
+        Recursively validate and fix TOC links.
+        Removes nodes with broken links that cannot be fixed.
+        """
+        fixed_toc = []
+        for item in toc:
+            if isinstance(item, (epub.Link, epub.Section)):
+                # Check href
+                href = getattr(item, 'href', '')
+                if href:
+                    # Remove anchor for check
+                    clean_href = href.split('#')[0]
+                    # Check if item exists in book (by file_name)
+                    found = False
+                    for existing_item in book_items.values():
+                        if existing_item.file_name == clean_href:
+                            found = True
+                            break
+                        if clean_href.endswith(existing_item.file_name) or existing_item.file_name.endswith(clean_href):
+                             item.href = existing_item.file_name
+                             found = True
+                             break
+                    
+                    if not found:
+                        if 'c0.xhtml' in clean_href:
+                            for existing_item in book_items.values():
+                                if 'titlepage' in existing_item.file_name or 'cover' in existing_item.file_name.lower():
+                                    if existing_item.media_type == "application/xhtml+xml":
+                                        logger.info(f"Fixed TOC link: {href} -> {existing_item.file_name}")
+                                        item.href = existing_item.file_name
+                                        found = True
+                                        break
+                        
+                        if not found:
+                             logger.warning(f"Removing broken TOC link: {href}")
+                             continue 
+            
+            if isinstance(item, tuple) and len(item) == 2:
+                section, children = item
+                fixed_children = self._validate_and_fix_toc(children, book_items)
+                fixed_toc.append((section, fixed_children))
+            else:
+                 fixed_toc.append(item)
+                 
+        return fixed_toc
+
+    def build(self, structure: BookStructure, output_path: Path) -> Path:
+        """
+        Builds the EPUB file.
+        Returns the path to the generated EPUB.
+        """
+        logger.info(f"Building final EPUB: {output_path}")
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+        
+        book = epub.EpubBook()
+        
+        # 1. Metadata
+        book.set_identifier(structure.metadata.identifier or f"uuid-{uuid.uuid4().hex[:12]}")
+        book.set_title(structure.metadata.title)
+        book.set_language(structure.metadata.language)
+        book.add_author(structure.metadata.author)
+        
+        # 2. Copy TOC from original EPUB if available
+        original_book = self._load_original_book()
+        if original_book and hasattr(original_book, 'toc') and original_book.toc:
+            book.toc = self._sanitize_toc(original_book.toc)
+            logger.info("Copied TOC structure from original EPUB")
+            
+
+        
+        # 3. Add Resources - First pass: Collect CSS items
+        items_map = {} # id -> epub_item
+        css_items = [] # List of CSS EpubItem for linking
+        html_items = [] # List of (item_id, EpubHtml) tuples
+        
+        for item_id, resource in structure.resources.items():
+            # Skip NCX - we'll handle it separately
+            if resource.media_type == "application/x-dtbncx+xml":
+                continue
+                
+            if resource.media_type == "application/xhtml+xml":
+                # HTML Item - create but don't add yet (need to add CSS links)
+                item = epub.EpubHtml(
+                    uid=item_id,
+                    file_name=resource.href,
+                    media_type=resource.media_type,
+                    content=resource.content.encode('utf-8')
+                )
+                html_items.append((item_id, item))
+            elif resource.file_path:
+                # Binary/Asset Item
+                asset_full_path = self.work_dir / resource.file_path
+                
+                if not asset_full_path.exists():
+                     logger.warning(f"Asset missing: {asset_full_path}")
+                     continue
+                     
+                with open(asset_full_path, 'rb') as f:
+                    content = f.read()
+                    
+                # Check if this is the cover image
+                if structure.metadata.cover_image_id == item_id:
+                    logger.info(f"Setting cover image: {item_id}")
+                    # Manually add cover metadata to avoid "set_cover" creating a duplicate item with hardcoded ID
+                    # 1. Add item normally (as EpubImage)
+                    item = epub.EpubImage(
+                        uid=item_id,
+                        file_name=resource.href,
+                        media_type=resource.media_type,
+                        content=content
+                    )
+                    book.add_item(item)
+                    items_map[item_id] = item
+                    
+                    # 2. Add metadata pointing to it
+                    book.add_metadata(None, 'meta', item_id, {'name': 'cover', 'content': item_id})
+                    continue
+
+                if "image" in resource.media_type:
+                     item = epub.EpubImage(
+                        uid=item_id,
+                        file_name=resource.href,
+                        media_type=resource.media_type,
+                        content=content
+                     )
+                else:
+                    item = epub.EpubItem(
+                        uid=item_id,
+                        file_name=resource.href,
+                        media_type=resource.media_type,
+                        content=content
+                    )
+                    # Track CSS items
+                    if resource.media_type == "text/css":
+                        css_items.append(item)
+                        
+                book.add_item(item)
+                items_map[item_id] = item
+            else:
+                logger.warning(f"Skipping resource {item_id}: No content or file path.")
+                continue
+        
+        # 4. Add HTML items with CSS links
+        for item_id, item in html_items:
+            html_dir = Path(item.file_name).parent
+            for css_item in css_items:
+                css_path = Path(css_item.file_name)
+                # Calculate relative path from HTML directory to CSS file
+                try:
+                    relative_css_path = Path(css_path).relative_to(html_dir)
+                except ValueError:
+                    # Not a subpath, calculate full relative
+                    # Go up from html_dir, then down to css_path
+                    up_count = len(html_dir.parts)
+                    relative_css_path = Path("/".join([".."] * up_count)) / css_path
+                
+                item.add_link(href=str(relative_css_path), rel='stylesheet', type='text/css')
+            book.add_item(item)
+            items_map[item_id] = item
+
+        # 5. Copy missing items from original EPUB (cover, etc.)
+        # This ensures TOC links don't break
+        if original_book:
+            added_hrefs = {item.file_name for item in items_map.values() if hasattr(item, 'file_name')}
+            
+            for orig_item in original_book.get_items():
+                orig_name = orig_item.get_name()
+                
+                # Check for existence (exact or suffix overlap to handle EPUB/OEBPS prefixes)
+                is_duplicate = False
+                if orig_name in added_hrefs:
+                    is_duplicate = True
+                else:
+                    for added in added_hrefs:
+                        if added.endswith(orig_name) or orig_name.endswith(added):
+                            is_duplicate = True
+                            break
+                            
+                if is_duplicate:
+                    continue
+
+                # Skip NCX and NAV - we generate these
+                if 'ncx' in orig_name.lower() or orig_name.endswith('nav.xhtml'):
+                    continue
+                
+                # Double check if it's an image we likely already processed
+                if orig_item.media_type and orig_item.media_type.startswith("image/"):
+                     # If we didn't match via filename, verify if we missed it or if it's truly new
+                     # But suffix check usually catches it.
+                     pass
+
+                # Copy the item directly
+                try:
+                    book.add_item(orig_item)
+                    items_map[orig_item.id] = orig_item
+                    logger.debug(f"Copied missing item from original: {orig_name}")
+                except Exception as e:
+                    logger.warning(f"Failed to copy item {orig_name}: {e}")
+
+        # 6. Spine
+        book.spine = []
+        for item_id in structure.spine:
+             if item_id in items_map:
+                 book.spine.append(items_map[item_id])
+             else:
+                 logger.warning(f"Spine item {item_id} not found in resources.")
+        
+        # Add missing spine items from original
+        if original_book:
+            for spine_id, _ in original_book.spine:
+                if spine_id not in [i.id for i in book.spine]:
+                    orig_item = original_book.get_item_with_id(spine_id)
+                    if orig_item and orig_item.id in items_map:
+                        book.spine.append(items_map[orig_item.id])
+                 
+        # Validate and fix TOC
+        if original_book and hasattr(book, 'toc') and book.toc:
+             try:
+                 book.toc = self._validate_and_fix_toc(book.toc, items_map)
+                 logger.info("Validated and fixed TOC links")
+             except Exception as e:
+                 logger.error(f"Failed to validate TOC: {e}")
+
+        # 7. Navigation - NCX and Nav
+        book.add_item(epub.EpubNcx())
+        book.add_item(epub.EpubNav())
+        
+        # 7. Write
+        epub.write_epub(str(output_path), book)
+        logger.info(f"EPUB created successfully at {output_path}")
+        
+        return output_path
+

archive/v0.12/src/assembly/format_restorer.py

@@ -0,0 +1,72 @@
+import re
+from typing import Dict, Tuple, List, Optional
+from loguru import logger
+from src.common.utils import setup_logger
+
+logger = setup_logger("format_restorer")
+
+class FormatRestorer:
+    """
+    Restores HTML formatting from placeholders.
+    """
+    
+    PLACEHOLDER_REGEX = re.compile(r'φ(/?\d+)φ')
+
+    def restore(self, text_with_placeholders: str, placeholder_map: Dict[str, str], context_id: str = None) -> Tuple[str, bool]:
+        """
+        Restores HTML from text with placeholders.
+        Returns (restored_html, success).
+        """
+        if not placeholder_map:
+            return text_with_placeholders or "", True
+            
+        if not text_with_placeholders:
+            prefix = placeholder_map.get("_prefix", "")
+            suffix = placeholder_map.get("_suffix", "")
+            return prefix + suffix, True
+
+        prefix = placeholder_map.get("_prefix", "")
+        suffix = placeholder_map.get("_suffix", "")
+        
+        inner_map = {k: v for k, v in placeholder_map.items() if not k.startswith("_")}
+        
+        found_ids = set(self.PLACEHOLDER_REGEX.findall(text_with_placeholders))
+        expected_ids = set(inner_map.keys())
+        
+        success = True
+        context_msg = f" [ID: ...{context_id[-6:]}]" if context_id else ""
+        
+        missing_ids = expected_ids - found_ids
+        if missing_ids:
+            logger.warning(f"Restoration warning: missing placeholders {missing_ids}{context_msg}")
+            success = False
+            
+        unknown_ids = found_ids - expected_ids
+        if unknown_ids:
+            real_unknowns = set()
+            for pid in unknown_ids:
+                if pid.startswith('/') and pid[1:] in expected_ids:
+                    continue
+                real_unknowns.add(pid)
+                
+            if real_unknowns:
+                logger.warning(f"Restoration warning: unknown placeholders {real_unknowns}{context_msg}")
+                success = False
+
+        def replace_match(match):
+            pid = match.group(1)
+            if pid in inner_map:
+                return inner_map[pid]
+            else:
+                return ""
+
+        try:
+            restored_inner = self.PLACEHOLDER_REGEX.sub(replace_match, text_with_placeholders)
+            restored_html = prefix + restored_inner + suffix
+            return restored_html, success
+        except Exception as e:
+            logger.error(f"Restoration failed: {e}")
+            return prefix + self._strip_placeholders(text_with_placeholders) + suffix, False
+
+    def _strip_placeholders(self, text: str) -> str:
+        return self.PLACEHOLDER_REGEX.sub("", text)

archive/v0.12/src/assembly/restoration_engine.py

@@ -0,0 +1,106 @@
+import asyncio
+from typing import List, Optional
+from loguru import logger
+
+from src.common.data_model import ManifestEntry
+from src.assembly.format_restorer import FormatRestorer
+from src.translation.llm_client import LLMClient
+from src.common.utils import add_spacing
+
+class RestorationEngine:
+    """
+    Handles post-translation format restoration:
+    1. Spacing optimization (Pangu)
+    2. HTML tag restoration (FormatRestorer)
+    3. LLM-based repair (if restoration fails)
+    """
+    
+    def __init__(self, llm_client: Optional[LLMClient] = None):
+        self.restorer = FormatRestorer()
+        self.llm_client = llm_client
+        
+    async def restore_entries(self, entries: List[ManifestEntry], force: bool = False) -> int:
+        """
+        Process a list of entries, updating their translated_html field.
+        Returns the number of successfully restored entries.
+        """
+        success_count = 0
+        
+        for entry in entries:
+            if not entry.translated_text:
+                continue
+                
+            # Idempotency check: Skip if already restored (unless forced)
+            if entry.translated_html and not force:
+                success_count += 1
+                continue
+                
+            # 1. Spacing Fix
+            spaced_text = add_spacing(entry.translated_text)
+            
+            # 2. Format Restoration
+            restored_html, success = self.restorer.restore(
+                spaced_text,
+                entry.placeholders,
+                context_id=entry.entry_id
+            )
+            
+            # 3. LLM Repair (if needed and available)
+            if not success and self.llm_client:
+                logger.debug(f"Attempting LLM repair for {entry.entry_id}...")
+                repaired_text = await self._repair_placeholders(entry, spaced_text)
+                if repaired_text:
+                    # Retry with repaired text
+                    repaired_html, repaired_success = self.restorer.restore(
+                        repaired_text,
+                        entry.placeholders,
+                        context_id=f"{entry.entry_id}-REPAIR"
+                    )
+                    if repaired_success:
+                         logger.info(f"LLM Repair successful for {entry.entry_id}")
+                         restored_html = repaired_html
+                         # We do NOT update translated_text here to preserve original LLM output? 
+                         # Actually user might want the spaced and repaired text as the "text".
+                         # But let's keep translated_text as raw-ish, and translated_html as final.
+                         # Update: To ensure consistency, maybe we should update translated_text?
+                         # The implementation plan says "Save result to ManifestEntry.translated_html".
+                         # It implicitly leaves translated_text alone or updates it?
+                         # Let's keep translated_text as is (except maybe spacing? no, keep it raw).
+                         # But wait, if we repair key text, next time we run, we might want to use the repaired text?
+                         # For now, only populate translated_html.
+                         success = True
+                    else:
+                         logger.warning(f"LLM Repair failed validation for {entry.entry_id}")
+            
+            # 4. Save Result
+            # Even if validation failed, we often get a "best effort" restored_html from restorer (fallback).
+            # The restorer usually returns *something*. 
+            # If completely failed (e.g. mismatch), restorer might return raw text or partial?
+            # FormatRestorer.restore returns (restored_str, success_bool).
+            # Even if success=False, restored_str is produced (often just stripping unused placeholders or keeping them raw).
+            
+            entry.translated_html = restored_html
+            if success:
+                success_count += 1
+                
+        return success_count
+
+    async def _repair_placeholders(self, entry: ManifestEntry, current_text: str) -> Optional[str]:
+        """Ask LLM to fix placeholders."""
+        try:
+            system_prompt = "You are a translation repair assistant."
+            user_prompt = f"""
+The following translation has incorrect placeholders. 
+Please fix the placeholders in the Translated text so they match the Original text structure EXACTLY.
+Do NOT change the Chinese translation content, only fix the φXφ tags.
+
+Original: {entry.original_text}
+Translated (Broken): {current_text}
+
+Output ONLY the fixed Translated text.
+"""
+            repaired = await self.llm_client.raw_chat_completion(system_prompt, user_prompt)
+            return repaired.strip()
+        except Exception as e:
+            logger.error(f"LLM Repair error: {e}")
+            return None

archive/v0.12/src/common/config.py

@@ -0,0 +1,71 @@
+import os
+import yaml
+from pathlib import Path
+from typing import Dict, Any
+from dotenv import load_dotenv
+
+from src.common.utils import setup_logger
+
+logger = setup_logger("config_loader")
+
+class ConfigLoader:
+    def __init__(self, config_path: str = "config/config.yaml"):
+        # Resolve absolute path relative to project root if needed, 
+        # but usually running from root so relative is fine.
+        self.config_path = Path(config_path)
+        self.config = self._load_defaults()
+        
+    def _load_defaults(self) -> Dict[str, Any]:
+        return {
+            "llm": {
+                "model": "gpt-3.5-turbo",
+                "base_url": "https://api.openai.com/v1",
+                "timeout": 60,
+                "requests_per_minute": 60,
+                "concurrent_requests": 5
+            },
+            "translation": {
+                "chunk_size": 4000
+            }
+        }
+
+    def load_config(self) -> Dict[str, Any]:
+        # 1. Load YAML
+        if self.config_path.exists():
+            try:
+                with open(self.config_path, 'r', encoding='utf-8') as f:
+                    file_config = yaml.safe_load(f)
+                    if file_config:
+                        self._deep_update(self.config, file_config)
+                logger.info(f"Loaded config from {self.config_path}")
+            except Exception as e:
+                logger.error(f"Failed to load config file: {e}")
+        else:
+            logger.warning(f"Config file not found at {self.config_path}, using defaults")
+        
+        # 2. Env overrides (Priority: Env > Config File > Defaults)
+        load_dotenv()
+        
+        # API Key is mandatory from Env (security best practice)
+        api_key = os.getenv("OPENAI_API_KEY")
+        if api_key:
+            self.config["llm"]["api_key"] = api_key
+            
+        # Base URL override
+        base_url = os.getenv("OPENAI_BASE_URL")
+        if base_url:
+             self.config["llm"]["base_url"] = base_url
+             
+        return self.config
+
+    def _deep_update(self, d, u):
+        for k, v in u.items():
+            if isinstance(v, dict):
+                d[k] = self._deep_update(d.get(k, {}), v)
+            else:
+                d[k] = v
+        return d
+
+def load_global_config(path: str = "config/config.yaml") -> Dict[str, Any]:
+    loader = ConfigLoader(path)
+    return loader.load_config()

archive/v0.12/src/common/data_model.py

@@ -0,0 +1,53 @@
+from pathlib import Path
+from typing import List, Optional, Dict, Any
+from pydantic import BaseModel, Field
+
+class ManifestEntry(BaseModel):
+    """Represents a single translatable unit."""
+    entry_id: str = Field(..., description="Global unique ID (e.g. file.html#paragraph_id)")
+    file_path: str = Field(..., description="Internal path in EPUB")
+    element_id: str = Field(..., description="HTML ID (e.g. uuid-1234)")
+    original_text: str
+    placeholders: Dict[str, str] = Field(default_factory=dict)
+    translated_text: Optional[str] = None
+    translated_html: Optional[str] = Field(None, description="Final HTML with restored tags and formatting")
+    context: Optional[str] = None
+
+class BookMetaData(BaseModel):
+    title: str = "Unknown Title"
+    author: str = "Unknown Author"
+    language: str = "en"
+    identifier: str = ""
+    cover_image_id: Optional[str] = None
+
+class ResourceItem(BaseModel):
+    href: str
+    media_type: str
+    content: Optional[str] = None  # For text/html
+    file_path: Optional[str] = None # For binary/assets (relative to assets dir)
+    properties: Optional[str] = None
+
+class BookStructure(BaseModel):
+    metadata: BookMetaData
+    spine: List[str] = Field(default_factory=list, description="Ordered list of item IDs in spine")
+    resources: Dict[str, ResourceItem] = Field(default_factory=dict, description="Map of item_id to ResourceItem")
+    
+    @classmethod
+    def load(cls, path: Path) -> "BookStructure":
+        """Load BookStructure from JSON file."""
+        with open(path, 'r', encoding='utf-8') as f:
+            return cls.model_validate_json(f.read())
+    
+    def save(self, path: Path):
+        """Save BookStructure to JSON file."""
+        with open(path, 'w', encoding='utf-8') as f:
+            f.write(self.model_dump_json(indent=2))
+
+class BookProfile(BaseModel):
+    """Represents the profile of the book."""
+    title: str
+    author: str
+    genre: str = "General"
+    keywords: List[str] = Field(default_factory=list)
+    style_guide: str = ""
+

archive/v0.12/src/common/exceptions.py

@@ -0,0 +1,18 @@
+class EpubTranslatorError(Exception):
+    """Base exception for Epub Translator."""
+    pass
+
+class CleaningError(EpubTranslatorError):
+    pass
+
+class ExtractionError(EpubTranslatorError):
+    pass
+
+class TranslationError(EpubTranslatorError):
+    pass
+
+class RestorationError(EpubTranslatorError):
+    pass
+
+class BuildError(EpubTranslatorError):
+    pass

archive/v0.12/src/common/paths.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+
+def get_work_dirs(input_path: Path) -> dict:
+    """
+    Get unified work directory paths for a specific book.
+    
+    Structure:
+    .work/
+      ├── {book_name}/
+      │   ├── book_structure.json
+      │   ├── manifest.json
+      │   ├── assets/
+      │   └── chunks/
+    """
+    book_name = input_path.stem
+    work_root = Path(".work") / book_name
+    
+    return {
+        "root": work_root,
+        "structure": work_root / "book_structure.json",
+        "manifest": work_root / "manifest.json",
+        "assets": work_root / "assets",
+        "chunks": work_root / "chunks",
+    }

archive/v0.12/src/common/utils.py

@@ -0,0 +1,43 @@
+import logging
+from pathlib import Path
+
+def setup_logger(name: str, log_file: Path = None, level=logging.INFO):
+    """Sets up a logger with the given name."""
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+    
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    
+    if log_file:
+        file_handler = logging.FileHandler(log_file)
+        file_handler.setFormatter(formatter)
+        logger.addHandler(file_handler)
+        
+    stream_handler = logging.StreamHandler()
+    stream_handler.setFormatter(formatter)
+    logger.addHandler(stream_handler)
+    
+    return logger
+
+def ensure_directory(path: Path):
+    """Ensures a directory exists."""
+    if not path.exists():
+        path.mkdir(parents=True, exist_ok=True)
+
+def add_spacing(text: str) -> str:
+    """
+    Add space between CJK and English/Number/Symbol characters.
+    Simplified version of pangu.js logic.
+    """
+    import re
+    if not text:
+        return text
+        
+    # CJK followed by non-CJK
+    text = re.sub(r'([\u4e00-\u9fa5])([a-zA-Z0-9])', r'\1 \2', text)
+    # Non-CJK followed by CJK
+    text = re.sub(r'([a-zA-Z0-9])([\u4e00-\u9fa5])', r'\1 \2', text)
+    
+    # Optional: Handle symbols like quote against CJK?
+    # For now, stick to user request: "中英文数字混排" (Chinese-English-Numbers)
+    return text

archive/v0.12/src/preprocessing/epub_cleaner.py

@@ -0,0 +1,184 @@
+import shutil
+import uuid
+import zipfile
+from pathlib import Path
+from typing import Dict, Set
+
+import ebooklib
+from bs4 import BeautifulSoup, Tag
+from ebooklib import epub
+
+from src.common.data_model import BookStructure, BookMetaData, ResourceItem
+from src.common.exceptions import CleaningError
+from src.common.utils import setup_logger, ensure_directory
+
+logger = setup_logger("epub_cleaner")
+
+class EpubCleaner:
+    def __init__(self, input_path: Path, work_dir: Path):
+        self.input_path = input_path
+        self.work_dir = work_dir
+        self.assets_dir = work_dir / "assets"
+        self.json_path = work_dir / "book_structure.json"
+        
+    def clean(self) -> Path:
+        """
+        Cleans the input EPUB and generates book_structure.json.
+        Returns the path to the JSON file.
+        """
+        logger.info(f"Starting cleanup for {self.input_path}")
+        ensure_directory(self.work_dir)
+        ensure_directory(self.assets_dir)
+
+        try:
+            book = epub.read_epub(self.input_path)
+            
+            # 1. Extract Metadata
+            metadata = self._extract_metadata(book)
+            
+            # 2. Process Resources
+            resources = {}
+            # Use zipfile for binary extraction to avoid ebooklib's memory overhead/decoding issues
+            with zipfile.ZipFile(self.input_path, 'r') as zf:
+                # Map ebooklib items to zip entries isn't straightforward directly via name
+                # So we iterate ebooklib items and assume standard structure or handle content bytes
+                
+                for item in book.get_items():
+                    item_id = item.get_id()
+                    file_name = item.get_name()
+                    media_type = item.get_type() # ebooklib constant
+                    
+                    if media_type == ebooklib.ITEM_DOCUMENT:
+                        # Clean HTML
+                        content_str = item.get_content().decode('utf-8')
+                        cleaned_content = self._clean_html(content_str, file_name)
+                        
+                        resources[item_id] = ResourceItem(
+                            href=file_name,
+                            media_type="application/xhtml+xml",
+                            content=cleaned_content
+                        )
+                    elif media_type in (ebooklib.ITEM_IMAGE, ebooklib.ITEM_STYLE, ebooklib.ITEM_FONT, ebooklib.ITEM_COVER):
+                        # Save asset
+                        
+                        # Check if it is a cover
+                        if media_type == ebooklib.ITEM_COVER:
+                            metadata.cover_image_id = item_id
+                            logger.info(f"Found cover image: {item_id} ({file_name})")
+                            
+                        # Preserve directory structure to avoid collisions
+                        asset_path = self.assets_dir / file_name
+                        ensure_directory(asset_path.parent)
+                        
+                        # Ebooklib might change filenames, safer to use item.get_content()
+                        with open(asset_path, "wb") as f:
+                            f.write(item.get_content())
+                            
+                        resources[item_id] = ResourceItem(
+                            href=file_name,
+                            media_type=self._get_media_type_str(item),
+                            file_path=str(asset_path.relative_to(self.work_dir))
+                        )
+                    elif media_type == ebooklib.ITEM_NAVIGATION:
+                        # NCX or NAV document - preserve for TOC
+                        content_bytes = item.get_content()
+                        asset_path = self.assets_dir / file_name
+                        ensure_directory(asset_path.parent)
+                        
+                        with open(asset_path, "wb") as f:
+                            f.write(content_bytes)
+                        
+                        # Determine media type
+                        if file_name.endswith('.ncx'):
+                            mt = "application/x-dtbncx+xml"
+                        else:
+                            mt = "application/xhtml+xml"
+                        
+                        resources[item_id] = ResourceItem(
+                            href=file_name,
+                            media_type=mt,
+                            file_path=str(asset_path.relative_to(self.work_dir))
+                        )
+                        logger.debug(f"Preserved navigation: {file_name}")
+                    else:
+                        # Skip other items (scripts, etc.)
+                        pass
+
+            # 3. Extract Spine
+            spine_ids = [item[0] for item in book.spine]
+
+            # 4. Construct Structure
+            structure = BookStructure(
+                metadata=metadata,
+                spine=spine_ids,
+                resources=resources
+            )
+
+            # 5. Serialize
+            with open(self.json_path, "w", encoding="utf-8") as f:
+                f.write(structure.model_dump_json(indent=2))
+            
+            logger.info(f"Cleanup finished. Structure saved to {self.json_path}")
+            return self.json_path
+
+        except Exception as e:
+            logger.error(f"Cleaning failed: {e}")
+            raise CleaningError(f"Failed to clean EPUB: {e}") from e
+
+    def _extract_metadata(self, book: epub.EpubBook) -> BookMetaData:
+        title = book.get_metadata('DC', 'title')[0][0] if book.get_metadata('DC', 'title') else "Unknown"
+        author = book.get_metadata('DC', 'creator')[0][0] if book.get_metadata('DC', 'creator') else "Unknown"
+        lang = book.get_metadata('DC', 'language')[0][0] if book.get_metadata('DC', 'language') else "en"
+        ident = book.get_metadata('DC', 'identifier')[0][0] if book.get_metadata('DC', 'identifier') else ""
+        
+        return BookMetaData(
+            title=str(title),
+            author=str(author),
+            language=str(lang),
+            identifier=str(ident)
+        )
+
+    def _get_media_type_str(self, item) -> str:
+        # Helper to map ebooklib type to mime string if needed
+        # ebooklib doesn't expose easy MIME string for all types directly on item object sometimes
+        if hasattr(item, 'media_type'):
+            return item.media_type
+        return "application/octet-stream"
+
+    def _clean_html(self, content: str, filename: str) -> str:
+        soup = BeautifulSoup(content, 'html.parser')
+        
+        # 1. Provide IDs for structural/translatable elements
+        self._ensure_element_ids(soup)
+        
+        # 2. Flatten divs (Disable to prevent style loss)
+        # self._flatten_divs(soup)
+        
+        return str(soup)
+
+    def _ensure_element_ids(self, soup: BeautifulSoup):
+        """
+        Injects UUIDs into p, h1-h6, li tags if they don't have an ID.
+        This provides the anchor for translation backfilling.
+        """
+        targets = soup.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'])
+        for tag in targets:
+            if not tag.has_attr('id'):
+                tag['id'] = f"uuid-{uuid.uuid4()}"
+    
+    def _flatten_divs(self, soup: BeautifulSoup):
+        """
+        Converts generic divs containing only inline text/styles to p tags.
+        Recursive strategies can be complex, sticking to simple heuristic from archive.
+        """
+        inline_tags = {'span', 'em', 'strong', 'i', 'b', 'u', 'a', 'br', 'sub', 'sup', 'small'}
+        
+        for div in list(soup.find_all('div')):
+            # If div has no block children, convert to p
+            has_block = any(
+                isinstance(c, Tag) and c.name not in inline_tags 
+                for c in div.children
+            )
+            
+            if not has_block:
+                div.name = 'p'

archive/v0.12/src/preprocessing/format_extractor.py

@@ -0,0 +1,375 @@
+import re
+import html
+from bs4 import BeautifulSoup, Tag
+from typing import Tuple, Dict, List
+
+class HeadingDetector:
+    """Detects heading types and paragraph roles."""
+    
+    CHAPTER_PATTERNS = [
+        r'^(chapter|chap\.?|part)\s+([0-9]+|[ivxlc]+|[a-z])',
+        r'^(第\s*[0-9一二三四五六七八九十百]+\s*[章节部篇])',
+        r'^(\d+|[IVXLC]+|[A-Z])\.$'
+    ]
+    
+    EPIGRAPH_CLASSES = {
+        'epigraph', 'quote', 'blockquote', 'motto', 
+        'dedication', 'verse', 'poetry', 'poem'
+    }
+
+    def detect(self, element: Tag, text: str) -> str:
+        if self._is_epigraph(element):
+            return "epigraph"
+        tag_name = element.name.lower()
+        if tag_name in ['h1', 'h2']:
+            return "chapter" if self._matches_chapter_pattern(text) else "section"
+        if tag_name == 'h3':
+            return "section"
+        if tag_name in ['h4', 'h5', 'h6']:
+            return "subsection"
+        if self._is_pseudo_heading(element, text):
+            return "subsection"
+        return "body"
+
+    def _is_epigraph(self, element: Tag) -> bool:
+        if element.name == 'blockquote':
+            return True
+        current = element
+        for _ in range(3):
+            if not current: break
+            classes = current.get('class', [])
+            if isinstance(classes, list):
+                classes = ' '.join(classes)
+            if any(k in classes.lower() for k in self.EPIGRAPH_CLASSES):
+                return True
+            current = current.parent
+        return False
+
+    def _matches_chapter_pattern(self, text: str) -> bool:
+        text = text.strip().lower()
+        for pattern in self.CHAPTER_PATTERNS:
+            if re.match(pattern, text, re.IGNORECASE):
+                return True
+        return False
+
+    def _is_pseudo_heading(self, element: Tag, text: str) -> bool:
+        if element.name != 'p':
+            return False
+        text = text.strip()
+        if not text or len(text) > 80:
+            return False
+        children = list(element.children)
+        if len(children) == 1 and isinstance(children[0], Tag):
+            if children[0].name in ['strong', 'b']:
+                return True
+        return False
+
+
+class FormatExtractor:
+    """
+    HTML Format Extractor (Ported from v0.09 v3)
+    Handles inline styles, formulas, and drop caps.
+    """
+
+    FORMULA_CHARS = re.compile(
+        r'^[\d\s\+\-\×\÷\=\(\)\[\]\{\}\<\>\^\*\/\.\,\;\:\'\"\`\~\@\#\$\%\&\|\\'
+        r'αβγδεζηθικλμνξοπρστυφχψωΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩ'
+        r'a-zA-Z]+$'
+    )
+
+    def __init__(self):
+        self.detector = HeadingDetector()
+
+    def extract(self, element_html: str) -> Tuple[str, str, Dict[str, str], str, List[str]]:
+        """
+        Extracts format information.
+        
+        Returns:
+            clean_text: Pure text
+            text_with_placeholders: Text with inline placeholders
+            placeholder_map: Map of placeholders
+            paragraph_type: Detected type
+            endnote_anchors: List of detected endnote IDs
+        """
+        soup = BeautifulSoup(element_html, 'html.parser')
+        root = list(soup.children)[0] if list(soup.children) else soup
+        
+        clean_text = root.get_text().strip()
+        clean_text = re.sub(r'\s+', ' ', clean_text)
+        p_type = self.detector.detect(root, clean_text) if isinstance(root, Tag) else "body"
+        
+        inner_html = root.decode_contents() if isinstance(root, Tag) else str(root)
+        
+        text_with_ph, local_map = self._smart_extract_v3(inner_html)
+        
+        if text_with_ph:
+            text_with_ph = re.sub(r'\s+', ' ', text_with_ph).strip()
+        
+        # Verify integrity
+        stripped_text = self._strip_placeholders(text_with_ph)
+        stripped_text = re.sub(r'\s+', ' ', stripped_text).strip()
+        
+        if not self._verify_content_integrity(clean_text, stripped_text):
+            # Fallback
+            text_with_ph, local_map = self._fallback_extract(inner_html, clean_text)
+        
+        endnote_anchors = []
+        for pid, html in local_map.items():
+            if pid.startswith("_"):
+                continue
+            if re.match(r'<(span|a)\s+id="[a-zA-Z][a-zA-Z0-9]{2,5}"\s*>\s*</\1>', html):
+                endnote_anchors.append(pid)
+        
+        return clean_text, text_with_ph, local_map, p_type, endnote_anchors
+    
+    def _strip_placeholders(self, text: str) -> str:
+        return re.sub(r'φ/?[0-9]+φ', '', text)
+    
+    def _verify_content_integrity(self, clean_text: str, stripped_text: str) -> bool:
+        def normalize(s):
+            # Unescape HTML entities first (e.g. &amp; -> &)
+            s = html.unescape(s)
+            s = re.sub(r'\s+', '', s)
+            s = s.lower()
+            return s
+        
+        norm_clean = normalize(clean_text)
+        norm_stripped = normalize(stripped_text)
+        
+        if norm_clean == norm_stripped:
+            return True
+            
+        return False
+    
+    def _fallback_extract(self, inner_html: str, clean_text: str) -> Tuple[str, Dict[str, str]]:
+        return clean_text, {"_prefix": "", "_suffix": ""}
+
+    def _smart_extract_v3(self, inner_html: str) -> Tuple[str, Dict[str, str]]:
+        parts = re.split(r'(<[^>]+>)', inner_html)
+        parts = [p for p in parts if p]
+        
+        if not parts:
+            return "", {"_prefix": "", "_suffix": ""}
+        
+        part_types = []
+        for part in parts:
+            if part.startswith('<'):
+                part_types.append('tag')
+            elif not part.strip():
+                part_types.append('whitespace')
+            elif self._is_translatable_text(part):
+                part_types.append('translatable')
+            else:
+                part_types.append('formula')
+        
+        first_trans_idx = None
+        last_trans_idx = None
+        for i, t in enumerate(part_types):
+            if t == 'translatable':
+                if first_trans_idx is None:
+                    first_trans_idx = i
+                last_trans_idx = i
+        
+        if first_trans_idx is None:
+            return "", {"_prefix": inner_html, "_suffix": ""}
+        
+        # Prefix Separation
+        safe_prefix_end = 0
+        for i in range(first_trans_idx):
+            if part_types[i] == 'tag':
+                tag = parts[i]
+                is_self_closing = tag.endswith('/>') or re.match(r'<(br|hr|img|meta|link)\b', tag, re.I)
+                is_closing = tag.startswith('</')
+                if is_self_closing or is_closing:
+                    safe_prefix_end = i + 1
+                else:
+                    break
+            elif part_types[i] == 'whitespace':
+                safe_prefix_end = i + 1
+            else:
+                break
+        
+        # Suffix Separation
+        safe_suffix_start = len(parts)
+        for i in range(len(parts) - 1, last_trans_idx, -1):
+            if part_types[i] == 'tag':
+                tag = parts[i]
+                is_closing = tag.startswith('</')
+                is_self_closing = tag.endswith('/>') or re.match(r'<(br|hr|img|meta|link)\b', tag, re.I)
+                if is_closing or is_self_closing:
+                    safe_suffix_start = i
+                else:
+                    break
+            elif part_types[i] == 'whitespace':
+                safe_suffix_start = i
+            else:
+                break
+        
+        prefix_parts = parts[:safe_prefix_end]
+        middle_parts = parts[safe_prefix_end:safe_suffix_start]
+        middle_types = part_types[safe_prefix_end:safe_suffix_start]
+        suffix_parts = parts[safe_suffix_start:]
+        
+        # Drop Cap Check
+        if prefix_parts and middle_parts:
+            prefix_parts, middle_parts, middle_types = self._handle_drop_cap(
+                prefix_parts, middle_parts, middle_types
+            )
+        
+        local_map = {}
+        if prefix_parts:
+            local_map["_prefix"] = "".join(prefix_parts)
+        if suffix_parts:
+            local_map["_suffix"] = "".join(suffix_parts)
+        
+        # Middle processing
+        placeholder_counter = 1
+        result_parts = []
+        tag_stack = []
+        
+        i = 0
+        while i < len(middle_parts):
+            part = middle_parts[i]
+            ptype = middle_types[i]
+            
+            if ptype == 'translatable':
+                result_parts.append(part)
+                i += 1
+                
+            elif ptype == 'tag':
+                is_closing = part.startswith('</')
+                if is_closing:
+                    if tag_stack:
+                        open_id, open_tag = tag_stack.pop()
+                        local_map[f"/{open_id}"] = part
+                        result_parts.append(f"φ/{open_id}φ")
+                    else:
+                        pid = str(placeholder_counter)
+                        placeholder_counter += 1
+                        local_map[pid] = part
+                        result_parts.append(f"φ{pid}φ")
+                    i += 1
+                else:
+                    has_translatable_after = False
+                    for j in range(i + 1, len(middle_parts)):
+                        if middle_types[j] == 'translatable':
+                            has_translatable_after = True
+                            break
+                        elif middle_types[j] == 'tag' and middle_parts[j].startswith('</'):
+                            break
+                    
+                    if has_translatable_after:
+                        pid = str(placeholder_counter)
+                        placeholder_counter += 1
+                        local_map[pid] = part
+                        result_parts.append(f"φ{pid}φ")
+                        tag_stack.append((pid, part))
+                        i += 1
+                    else:
+                        block_parts = []
+                        while i < len(middle_parts) and middle_types[i] != 'translatable':
+                            block_parts.append(middle_parts[i])
+                            i += 1
+                        if block_parts:
+                            block_html = "".join(block_parts)
+                            pid = str(placeholder_counter)
+                            placeholder_counter += 1
+                            local_map[pid] = block_html
+                            result_parts.append(f"φ{pid}φ")
+            else:
+                result_parts.append(part)
+                i += 1
+        
+        text_with_ph = "".join(result_parts)
+        text_with_ph = re.sub(r'\s+', ' ', text_with_ph).strip()
+        
+        # Merge consecutive placeholders
+        def merge_match(m):
+            full_match = m.group(0)
+            pids = re.findall(r'φ(/?\d+)φ', full_match)
+            if len(pids) <= 1:
+                return full_match
+            
+            merged_html = ""
+            for pid in pids:
+                if pid in local_map:
+                    merged_html += local_map[pid]
+                    del local_map[pid]
+            
+            new_pid = pids[0] if pids[0].isdigit() else pids[0][1:]
+            local_map[new_pid] = merged_html
+            return f"φ{new_pid}φ"
+        
+        text_with_ph = re.sub(r'(φ/?\d+φ)(φ/?\d+φ)+', merge_match, text_with_ph)
+        
+        return text_with_ph, local_map
+
+    def _handle_drop_cap(self, prefix_parts: List[str], middle_parts: List[str], middle_types: List[str]):
+        if not prefix_parts or not middle_parts:
+            return prefix_parts, middle_parts, middle_types
+        
+        prefix_text = ""
+        for part in prefix_parts:
+            if not part.startswith('<'):
+                prefix_text = part.strip()
+        
+        if not prefix_text or len(prefix_text) != 1 or not prefix_text.isupper():
+            return prefix_parts, middle_parts, middle_types
+        
+        first_middle_text = ""
+        first_middle_idx = -1
+        for i, (part, ptype) in enumerate(zip(middle_parts, middle_types)):
+            if ptype == 'translatable':
+                first_middle_text = part.strip()
+                first_middle_idx = i
+                break
+        
+        if not first_middle_text:
+            return prefix_parts, middle_parts, middle_types
+        
+        is_drop_cap = False
+        first_char = first_middle_text[0] if first_middle_text else ''
+        if first_char.islower() or first_char.isupper():
+            is_drop_cap = True
+        
+        combined = prefix_text + first_middle_text.split()[0] if first_middle_text else ""
+        if not (len(combined) >= 2 and combined.isalpha()):
+            return prefix_parts, middle_parts, middle_types
+            
+        new_prefix = []
+        skip_until_close = False
+        found_letter = False
+        
+        for part in prefix_parts:
+            if part.startswith('<') and not part.startswith('</'):
+                skip_until_close = True
+            elif part.startswith('</'):
+                if skip_until_close:
+                    skip_until_close = False
+                    continue
+                new_prefix.append(part)
+            elif part.strip() == prefix_text:
+                found_letter = True
+                continue
+            else:
+                if not skip_until_close:
+                    new_prefix.append(part)
+        
+        if found_letter:
+            middle_parts = middle_parts.copy()
+            middle_parts[first_middle_idx] = prefix_text + middle_parts[first_middle_idx]
+            prefix_parts = new_prefix
+            
+        return prefix_parts, middle_parts, middle_types
+
+    def _is_translatable_text(self, text: str) -> bool:
+        text = text.strip()
+        if not text:
+            return False
+        if re.search(r'[a-zA-Z]{3,}', text):
+            return True
+        if ' ' in text and re.search(r'[a-zA-Z]', text):
+            return True
+        if re.search(r'\d', text):
+            return True
+        return False

archive/v0.12/src/preprocessing/profiler.py

@@ -0,0 +1,83 @@
+import json
+import random
+from typing import Dict, List
+from loguru import logger
+from src.translation.llm_client import LLMClient
+from src.translation.manifest_manager import ManifestManager
+from src.common.data_model import BookProfile
+
+class BookProfiler:
+    def __init__(self, llm_client: LLMClient):
+        self.llm_client = llm_client
+
+    def extract_sample_text(self, entries: List[object], char_limit: int = 3000) -> str:
+        """Extract sample text from manifest entries."""
+        if not entries: return ""
+        
+        # Simple sampling strategy: First few + random middle
+        intro_text = []
+        for entry in entries[:50]:
+            if len(entry.original_text) > 50:
+                intro_text.append(entry.original_text)
+        
+        body_text = []
+        candidates = [e for e in entries[50:] if len(e.original_text) > 80]
+        if candidates:
+            samples = random.sample(candidates, min(5, len(candidates)))
+            body_text = [e.original_text for e in samples]
+            
+        # Join text
+        full_text = "\n\n".join(intro_text[:5] + body_text)
+        
+        # Strip placeholders to stop Profiler from seeing "garbage"
+        # Matches φ1φ, φ/1φ, etc.
+        import re
+        clean_text = re.sub(r'φ.*?φ', '', full_text)
+        
+        return clean_text[:char_limit]
+
+    async def analyze(self, entries: List[object]) -> BookProfile:
+        """Generate Book Profile."""
+        sample = self.extract_sample_text(entries)
+        if not sample: 
+            return BookProfile(title="Unknown", author="Unknown")
+        
+        logger.info("Generating Book Profile from sample text...")
+        
+        system_prompt = "You are a senior publishing editor. Analyze the text and output JSON."
+        user_prompt = f"""
+Please analyze the following book excerpt.
+Output JSON format:
+{{
+    "title": "Book Title",
+    "author": "Author Name",
+    "genre": "Genre",
+    "style": "Style description",
+    "keywords": ["keyword1", "keyword2"],
+    "style_guide": "Specific instruction for translator"
+}}
+
+Excerpt:
+{sample}
+"""
+        try:
+            response = await self.llm_client.raw_chat_completion(system_prompt, user_prompt)
+            json_str = response.strip()
+            # Basic cleanup
+            if "```json" in json_str:
+                json_str = json_str.split("```json")[1].split("```")[0].strip()
+            elif "```" in json_str:
+                json_str = json_str.split("```")[1].split("```")[0].strip()
+            
+            data = json.loads(json_str)
+            
+            return BookProfile(
+                title=data.get("title", "Unknown"),
+                author=data.get("author", "Unknown"),
+                genre=data.get("genre", "General"),
+                keywords=data.get("keywords", []),
+                style_guide=data.get("style_guide", "")
+            )
+        except Exception as e:
+            logger.error(f"Profile generation failed: {e}")
+            return BookProfile(title="Unknown", author="Unknown")

archive/v0.12/src/preprocessing/text_extractor.py

@@ -0,0 +1,132 @@
+import re
+from typing import List, Dict, Any, Optional
+from bs4 import BeautifulSoup
+from loguru import logger
+
+from src.common.data_model import BookStructure, ManifestEntry, BookProfile
+from src.preprocessing.format_extractor import FormatExtractor
+from src.common.utils import setup_logger
+
+logger = setup_logger("fine_grained_extractor")
+
+class FineGrainedExtractor:
+    """
+    Extracts translatable text segments from BookStructure.
+    Uses FormatExtractor for detailed content analysis.
+    """
+    
+    SKIP_TRANSLATION_PATTERNS = [
+        r'index\.x?html',
+        r'bibliography\.x?html',
+        r'endnotes?\.x?html',
+        r'footnotes?\.x?html',
+        r'copyright\.x?html',
+    ]
+    
+    TOC_PATTERNS = [
+        r'nav\.x?html',
+        r'toc\.x?html',
+    ]
+
+    def __init__(self, translate_toc: bool = False):
+        self.translate_toc = translate_toc
+        self.format_extractor = FormatExtractor()
+
+    def extract(self, structure: BookStructure, profile: Optional[BookProfile] = None) -> List[ManifestEntry]:
+        """
+        Extracts translatable segments from the BookStructure.
+        Iteration follows the spine order.
+        """
+        logger.info("Starting extraction from BookStructure...")
+        manifest_entries = []
+        
+        # Iterate over spine to maintain order
+        for item_id in structure.spine:
+            if item_id not in structure.resources:
+                logger.warning(f"Item ID {item_id} in spine but not in resources.")
+                continue
+            
+            resource = structure.resources[item_id]
+            
+            # Only process HTML/XHTML
+            if resource.media_type != "application/xhtml+xml" or not resource.content:
+                continue
+            
+            file_path = resource.href
+            doc_type = self._classify_document(file_path)
+            
+            # Parse HTML
+            soup = BeautifulSoup(resource.content, 'html.parser')
+            target_tags = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li']
+            
+            for element in soup.find_all(target_tags):
+                # Ensure element has ID (should have been done by Cleaner)
+                element_id = element.get('id')
+                if not element_id:
+                    logger.warning(f"Element in {file_path} missing ID, skipping: {element.name}")
+                    continue
+                
+                # Check translation eligibility
+                raw_text = element.get_text(separator=' ', strip=True)
+                if not raw_text.strip():
+                    continue
+                
+                is_decorative = self._is_decorative(raw_text)
+                should_translate = self._should_translate(doc_type, is_decorative, raw_text)
+                
+                if should_translate:
+                    # Extract detailed format
+                    outer_html = str(element)
+                    clean_text, text_with_ph, ph_map, p_type, _ = self.format_extractor.extract(outer_html)
+                    
+                    if clean_text.strip() and text_with_ph.strip():
+                        entry = ManifestEntry(
+                            entry_id=f"{file_path}#{element_id}",
+                            file_path=file_path,
+                            element_id=element_id,
+                            original_text=text_with_ph,
+                            placeholders=ph_map,
+                            context=p_type
+                        )
+                        manifest_entries.append(entry)
+        
+        logger.info(f"Extracted {len(manifest_entries)} entries in total.")
+        return manifest_entries
+
+    def _classify_document(self, file_name: str) -> str:
+        if not file_name: return 'core'
+        fname = file_name.lower()
+        if any(re.search(p, fname) for p in self.SKIP_TRANSLATION_PATTERNS): return 'skip'
+        if any(re.search(p, fname) for p in self.TOC_PATTERNS): return 'toc'
+        return 'core'
+
+    def _should_translate(self, doc_type: str, is_decorative: bool, text: str = "") -> bool:
+        if is_decorative: return False
+        if self._is_roman_numeral(text): return False
+        if doc_type == 'core': return True
+        if doc_type == 'toc': return self.translate_toc
+        return False
+
+    def _is_roman_numeral(self, text: str) -> bool:
+        text = text.strip().upper()
+        if not text: return False
+        pattern = re.compile(r"^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$")
+        return bool(pattern.match(text))
+
+    def _is_decorative(self, text: str) -> bool:
+        s = text.strip()
+        if not s: return False
+        if not any(c.isalnum() for c in s): return True
+        if len(s) > 20: return False
+        
+        patterns = [
+            r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾]+$',
+            r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾\s]+$',
+        ]
+        for p in patterns:
+            if re.match(p, s): return True
+            
+        unique = set(s.replace(' ', ''))
+        if len(unique) <= 3 and (unique & set('*-_~•◆◇■□▪▫●○◉◎⦿⦾')):
+            return True
+        return False

archive/v0.12/src/translation/llm_client.py

@@ -0,0 +1,231 @@
+import asyncio
+import json
+import re
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import List, Dict, Optional
+from openai import AsyncOpenAI
+from tenacity import retry, stop_after_attempt, wait_exponential
+from loguru import logger
+
+from src.common.data_model import ManifestEntry
+from src.common.utils import setup_logger, ensure_directory
+
+logger = setup_logger("llm_client")
+
+# Default chunk save directory (can be overridden)
+DEFAULT_CHUNK_DIR = Path("tmp/chunks")
+
+
+class RateLimiter:
+    """Rate limiter for concurrency and RPM."""
+    def __init__(self, requests_per_minute: int, concurrent_requests: int):
+        self.semaphore = asyncio.Semaphore(concurrent_requests)
+        self.min_interval = 60.0 / requests_per_minute if requests_per_minute > 0 else 0
+        self.last_request_time = 0
+        self._lock = asyncio.Lock()
+
+    async def acquire(self):
+        await self.semaphore.acquire()
+        async with self._lock:
+            current_time = time.time()
+            wait_time = self.min_interval - (current_time - self.last_request_time)
+            if wait_time > 0:
+                await asyncio.sleep(wait_time)
+            self.last_request_time = time.time()
+
+    def release(self):
+        self.semaphore.release()
+
+
+class LLMClient:
+    """Generic OpenAI-compatible API Client with short ID strategy."""
+    
+    def __init__(self, api_key: str, base_url: str, model: str = "gpt-3.5-turbo",
+                 requests_per_minute: int = 60, concurrent_requests: int = 5,
+                 extra_headers: Dict = None, chunk_dir: Path = None):
+        
+        # Configure proxy client to avoid SOCKS issues and ensure connectivity
+        import httpx
+        import os
+        
+        # Prefer HTTP proxy if available to avoid missing socksio support
+        proxy_url = os.environ.get("http_proxy") or os.environ.get("https_proxy")
+        http_client = httpx.AsyncClient(
+            proxy=proxy_url,
+            timeout=60.0,
+            follow_redirects=True
+        ) if proxy_url else None
+
+        self.client = AsyncOpenAI(
+            base_url=base_url,
+            api_key=api_key,
+            default_headers=extra_headers,
+            http_client=http_client
+        )
+        self.model = model
+        
+        self.rate_limiter = RateLimiter(requests_per_minute, concurrent_requests)
+        self.prompts = self._load_prompts()
+        self._chunk_counter = 0
+        
+        # Chunk directory for debug output
+        self.chunk_dir = chunk_dir or DEFAULT_CHUNK_DIR
+        ensure_directory(self.chunk_dir)
+        
+    def _load_prompts(self) -> Dict:
+        try:
+            with open("config/prompts.json", "r", encoding="utf-8") as f:
+                return json.load(f)
+        except Exception as e:
+            logger.error(f"Failed to load config/prompts.json: {e}")
+            return {}
+
+    async def translate_chunk(self, items: List[ManifestEntry], glossary: Dict = None, 
+                              instruction: str = None, mode: str = "bilingual") -> Dict[str, str]:
+        """
+        Translate a chunk of items using short ID strategy.
+        Returns: Dict[entry_id, translated_text]
+        """
+        if not items: return {}
+        
+        # Build prompt with short IDs
+        id_map, prompt = self._build_prompt_with_short_ids(items)
+        
+        try:
+            # Build System Prompt
+            base_sys_prompt = self.prompts.get("translation", {}).get("system", 
+                "You are a professional English to Chinese translator.")
+            
+            if instruction:
+                base_sys_prompt += f"\n\nBook Style Guide:\n{instruction}"
+            
+            if glossary:
+                glossary_text = "\n".join([f"{k} -> {v}" for k, v in glossary.items()])
+                base_sys_prompt += f"\n\nTerminology:\n{glossary_text}"
+            
+            # Short ID format instructions
+            base_sys_prompt += """
+
+Output Format:
+- Each line MUST start with #N: (keep this ID exactly as given)
+- Preserve any φXφ or φ/Xφ placeholders EXACTLY as-is
+- Only output translations, no explanations
+- Match the number of output lines to input lines"""
+
+            # Save chunk before translation
+            chunk_id = self._save_chunk("before", prompt, base_sys_prompt)
+            
+            logger.debug(f"Sending request to LLM (Chunk: {chunk_id}, Items: {len(items)})")
+            
+            raw_response = await self._make_request(base_sys_prompt, prompt)
+            
+            # Save chunk after translation
+            self._save_chunk("after", raw_response, base_sys_prompt, chunk_id)
+            
+            # Parse with short ID mapping
+            results = self._parse_short_id_response(raw_response, id_map)
+            return results
+            
+        except Exception as e:
+            logger.error(f"Translation failed: {e}")
+            return {item.entry_id: f"[Error - {str(e)}]" for item in items}
+
+    def _build_prompt_with_short_ids(self, items: List[ManifestEntry]) -> tuple:
+        """
+        Build prompt with short IDs (#1, #2, ...).
+        Returns: (id_map, prompt_text)
+        """
+        id_map = {}  # short_id -> entry_id
+        lines = []
+        
+        for i, item in enumerate(items, 1):
+            short_id = f"#{i}"
+            id_map[short_id] = item.entry_id
+            
+            # Clean text (remove extra whitespace)
+            text = re.sub(r'\s+', ' ', item.original_text).strip()
+            lines.append(f"{short_id}: {text}")
+        
+        return id_map, "\n".join(lines)
+
+    def _parse_short_id_response(self, response: str, id_map: Dict[str, str]) -> Dict[str, str]:
+        """
+        Parse response with short ID format.
+        Returns: Dict[entry_id, translated_text]
+        """
+        results = {}
+        
+        for line in response.split("\n"):
+            line = line.strip()
+            if not line:
+                continue
+                
+            # Match #N: pattern
+            # Handles both ASCII colon (:) and full-width colon (：)
+            match = re.match(r'^#(\d+)[:：]\s*(.+)$', line)
+            if match:
+                short_id = f"#{match.group(1)}"
+                content = match.group(2).strip()
+                
+                # Recursively strip repeated IDs (e.g. "#12: #12: Text")
+                while True:
+                    sub_match = re.match(r'^#(\d+)[:：]\s*(.+)$', content)
+                    if sub_match:
+                        # Check if the inner ID matches the outer ID, or just strip it anyway
+                        # Usually LLM repeats the same ID.
+                        content = sub_match.group(2).strip()
+                    else:
+                        break
+                
+                if short_id in id_map:
+                    full_id = id_map[short_id]
+                    results[full_id] = content
+                else:
+                    logger.warning(f"Unknown short ID in response: {short_id}")
+        
+        return results
+
+    def _save_chunk(self, stage: str, content: str, system_prompt: str = None, 
+                    chunk_id: str = None) -> str:
+        """Save chunk to tmp directory for debugging."""
+        if chunk_id is None:
+            self._chunk_counter += 1
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            chunk_id = f"{timestamp}_{self._chunk_counter:04d}"
+        
+        filename = self.chunk_dir / f"chunk_{chunk_id}_{stage}.txt"
+        
+        with open(filename, "w", encoding="utf-8") as f:
+            if system_prompt and stage == "before":
+                f.write("=== SYSTEM PROMPT ===\n")
+                f.write(system_prompt)
+                f.write("\n\n=== USER PROMPT ===\n")
+            f.write(content)
+        
+        logger.debug(f"Saved chunk: {filename}")
+        return chunk_id
+
+    async def raw_chat_completion(self, system_prompt: str, user_prompt: str) -> str:
+        """Generic chat completion."""
+        return await self._make_request(system_prompt, user_prompt)
+
+    @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=2, max=10))
+    async def _make_request(self, system_prompt: str, user_prompt: str) -> str:
+        await self.rate_limiter.acquire()
+        try:
+            resp = await self.client.chat.completions.create(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": user_prompt}
+                ],
+                temperature=0.3,
+            )
+            return resp.choices[0].message.content.strip()
+        finally:
+            self.rate_limiter.release()
+
+    async def close(self):
+        await self.client.close()

archive/v0.12/src/translation/manifest_manager.py

@@ -0,0 +1,74 @@
+from pathlib import Path
+from typing import List, Dict, Optional
+import json
+from src.common.data_model import ManifestEntry
+from src.common.utils import setup_logger
+
+logger = setup_logger("manifest_manager")
+
+class ManifestManager:
+    """
+    Manages the translation manifest (Source of Truth).
+    Handles persistence and state updates.
+    """
+    
+    def __init__(self, manifest_path: Path):
+        self.manifest_path = manifest_path
+        self.entries: List[ManifestEntry] = []
+        self._entries_map: Dict[str, ManifestEntry] = {}
+        
+    def load(self):
+        """Loads manifest from disk if it exists."""
+        if not self.manifest_path.exists():
+            logger.info(f"Manifest not found at {self.manifest_path}, starting empty.")
+            return
+
+        try:
+            with open(self.manifest_path, 'r', encoding='utf-8') as f:
+                data = json.load(f)
+                self.entries = [ManifestEntry.model_validate(item) for item in data]
+                self._rebuild_map()
+            logger.info(f"Loaded {len(self.entries)} entries from manifest.")
+        except Exception as e:
+            logger.error(f"Failed to load manifest: {e}")
+            raise
+
+    def save(self):
+        """Saves current state to disk."""
+        try:
+            # Pydantic v2: model_dump(mode='json') or just list dump
+            data = [entry.model_dump(mode='json') for entry in self.entries]
+            with open(self.manifest_path, 'w', encoding='utf-8') as f:
+                json.dump(data, f, indent=2, ensure_ascii=False)
+            logger.info(f"Saved {len(self.entries)} entries to manifest.")
+        except Exception as e:
+            logger.error(f"Failed to save manifest: {e}")
+            raise
+
+    def add_entries(self, new_entries: List[ManifestEntry]):
+        """
+        Adds new entries to the manifest.
+        If an entry with the same ID exists, it keeps the EXISTING one (to preserve translations).
+        """
+        count = 0
+        for entry in new_entries:
+            if entry.entry_id not in self._entries_map:
+                self.entries.append(entry)
+                self._entries_map[entry.entry_id] = entry
+                count += 1
+        
+        if count > 0:
+            logger.info(f"Added {count} new entries to manifest.")
+            
+    def update_translation(self, entry_id: str, translation: str):
+        """Updates translation for a specific entry."""
+        if entry_id in self._entries_map:
+            self._entries_map[entry_id].translated_text = translation
+        else:
+            logger.warning(f"Attempted to update translation for unknown ID: {entry_id}")
+
+    def get_entry(self, entry_id: str) -> Optional[ManifestEntry]:
+        return self._entries_map.get(entry_id)
+
+    def _rebuild_map(self):
+        self._entries_map = {e.entry_id: e for e in self.entries}

archive/v0.12/src/translation/translator_engine.py

@@ -0,0 +1,286 @@
+"""
+Translator Module - Handles translation of ManifestEntry items.
+
+Key features:
+- Character-based chunking (~5000 chars per chunk)
+- Chapter-aware grouping (chunks don't cross file boundaries)
+- Concurrent translation with asyncio.gather
+- Progress tracking and error handling
+"""
+
+import asyncio
+from typing import List, Dict
+from collections import defaultdict
+from loguru import logger
+
+from src.common.data_model import ManifestEntry, BookProfile
+from src.translation.llm_client import LLMClient
+from src.assembly.format_restorer import FormatRestorer
+from src.common.utils import setup_logger
+
+logger = setup_logger("translator")
+
+# Default chunk size in characters
+DEFAULT_CHUNK_SIZE = 5000
+# Maximum concurrent translations
+MAX_CONCURRENT = 5
+
+
+class Translator:
+    """
+    Translates ManifestEntry items using LLM with chapter-aware chunking.
+    Supports both sequential and concurrent translation modes.
+    """
+    
+    def __init__(self, llm_client: LLMClient, chunk_size: int = DEFAULT_CHUNK_SIZE, 
+                 max_concurrent: int = MAX_CONCURRENT):
+        self.llm_client = llm_client
+        self.chunk_size = chunk_size
+        self.max_concurrent = max_concurrent
+        self.restorer = FormatRestorer()
+        
+    async def translate(self, entries: List[ManifestEntry], profile: BookProfile, 
+                       concurrent: bool = True) -> List[ManifestEntry]:
+        """
+        Translates all untranslated entries.
+        
+        Args:
+            entries: All manifest entries
+            profile: Book profile with style guide
+            concurrent: Use concurrent translation (default True)
+            
+        Returns:
+            The same entries list with translated_text populated
+        """
+        untranslated = [e for e in entries if not e.translated_text]
+        if not untranslated:
+            logger.info("No new entries to translate.")
+            return entries
+            
+        logger.info(f"Found {len(untranslated)} entries to translate")
+        
+        # Group by chapter (file_path)
+        chapters = self._group_by_chapter(untranslated)
+        logger.info(f"Grouped into {len(chapters)} chapters")
+        
+        # Create all chunks
+        all_chunks = []
+        for file_path, chapter_entries in chapters.items():
+            chapter_chunks = self._create_char_based_chunks(chapter_entries)
+            for chunk in chapter_chunks:
+                all_chunks.append((file_path, chunk))
+        
+        total_chunks = len(all_chunks)
+        logger.info(f"Created {total_chunks} chunks (avg ~{self.chunk_size} chars each)")
+        
+        if concurrent:
+            await self._translate_concurrent(all_chunks, profile)
+        else:
+            await self._translate_sequential(all_chunks, profile)
+        
+        translated_count = sum(1 for e in entries if e.translated_text)
+        logger.info(f"Translation complete: {translated_count}/{len(entries)} entries translated")
+        return entries
+    
+    async def _translate_concurrent(self, all_chunks: List, profile: BookProfile):
+        """Translate chunks concurrently with semaphore control."""
+        semaphore = asyncio.Semaphore(self.max_concurrent)
+        completed = [0]  # Use list for mutable counter in closure
+        total = len(all_chunks)
+        success = [0]
+        failed = [0]
+        
+        async def translate_chunk_task(file_path: str, chunk: List[ManifestEntry], idx: int):
+            async with semaphore:
+                try:
+                    results = await self.llm_client.translate_chunk(
+                        chunk,
+                        instruction=profile.style_guide if profile else None,
+                        mode="bilingual"
+                    )
+                    
+                    for entry in chunk:
+                        if entry.entry_id in results:
+                            entry.translated_text = results[entry.entry_id]
+                            success[0] += 1
+                        else:
+                            failed[0] += 1
+                            
+                except Exception as e:
+                    logger.error(f"Chunk {idx} failed: {e}")
+                    failed[0] += len(chunk)
+                finally:
+                    completed[0] += 1
+                    if completed[0] % 5 == 0 or completed[0] == total:
+                        logger.info(f"Progress: {completed[0]}/{total} chunks ({success[0]} translated)")
+        
+        tasks = [
+            translate_chunk_task(file_path, chunk, i) 
+            for i, (file_path, chunk) in enumerate(all_chunks)
+        ]
+        await asyncio.gather(*tasks)
+        
+        logger.info(f"Concurrent translation: {success[0]} success, {failed[0]} failed")
+    
+    async def _translate_sequential(self, all_chunks: List, profile: BookProfile):
+        """Translate chunks sequentially."""
+        total_chunks = len(all_chunks)
+        success_count = 0
+        fail_count = 0
+        
+        for idx, (file_path, chunk) in enumerate(all_chunks):
+            try:
+                results = await self.llm_client.translate_chunk(
+                    chunk,
+                    instruction=profile.style_guide if profile else None,
+                    mode="bilingual"
+                )
+                
+                for entry in chunk:
+                    if entry.entry_id in results:
+                        entry.translated_text = results[entry.entry_id]
+                        success_count += 1
+                    else:
+                        fail_count += 1
+                        logger.warning(f"Missing translation for: {entry.entry_id[-40:]}")
+                        
+            except Exception as e:
+                logger.error(f"Chunk {idx} translation failed: {e}")
+                fail_count += len(chunk)
+            
+            if (idx + 1) % 10 == 0 or idx + 1 == total_chunks:
+                logger.info(f"Progress: {idx + 1}/{total_chunks} chunks ({success_count} entries translated)")
+        
+        logger.info(f"Sequential translation: {success_count} success, {fail_count} failed")
+
+    
+    async def translate_chapter(self, entries: List[ManifestEntry], file_path: str, 
+                                 profile: BookProfile) -> Dict[str, int]:
+        """
+        Translate a single chapter.
+        
+        Args:
+            entries: All entries (will filter by file_path)
+            file_path: Chapter file path to translate
+            profile: Book profile
+            
+        Returns:
+            Dict with 'success' and 'failed' counts
+        """
+        chapter_entries = [e for e in entries if e.file_path == file_path and not e.translated_text]
+        
+        if not chapter_entries:
+            logger.info(f"Chapter {file_path} has no untranslated entries")
+            return {"success": 0, "failed": 0}
+        
+        logger.info(f"Translating chapter: {file_path} ({len(chapter_entries)} entries)")
+        
+        chunks = self._create_char_based_chunks(chapter_entries)
+        logger.info(f"Created {len(chunks)} chunks")
+        
+        success = 0
+        failed = 0
+        
+        for i, chunk in enumerate(chunks, 1):
+            chunk_chars = sum(len(e.original_text) for e in chunk)
+            logger.debug(f"Chunk {i}/{len(chunks)}: {len(chunk)} entries, {chunk_chars} chars")
+            
+            try:
+                results = await self.llm_client.translate_chunk(
+                    chunk,
+                    instruction=profile.style_guide if profile else None,
+                    mode="bilingual"
+                )
+                
+                for entry in chunk:
+                    if entry.entry_id in results:
+                        entry.translated_text = results[entry.entry_id]
+                        
+                        # Verify placeholder preservation
+                        if entry.placeholders:
+                            _, restored_ok = self.restorer.restore(
+                                entry.translated_text, 
+                                entry.placeholders,
+                                context_id=entry.entry_id
+                            )
+                            if not restored_ok:
+                                logger.warning(f"Placeholder issue: {entry.entry_id[-40:]}")
+                        
+                        success += 1
+                    else:
+                        failed += 1
+                        
+            except Exception as e:
+                logger.error(f"Chunk {i} failed: {e}")
+                failed += len(chunk)
+        
+        logger.info(f"Chapter done: {success} success, {failed} failed")
+        return {"success": success, "failed": failed}
+    
+    def _group_by_chapter(self, entries: List[ManifestEntry]) -> Dict[str, List[ManifestEntry]]:
+        """Group entries by file_path (chapter)."""
+        chapters = defaultdict(list)
+        for entry in entries:
+            chapters[entry.file_path].append(entry)
+        return dict(chapters)
+    
+    def _create_char_based_chunks(self, entries: List[ManifestEntry]) -> List[List[ManifestEntry]]:
+        """
+        Create chunks based on character count.
+        
+        Each chunk contains approximately self.chunk_size characters.
+        Chunks never cross chapter boundaries (entries from same file only).
+        """
+        chunks = []
+        current_chunk = []
+        current_size = 0
+        
+        for entry in entries:
+            text_len = len(entry.original_text)
+            
+            # If adding this entry exceeds limit and we have content, start new chunk
+            if current_size + text_len > self.chunk_size and current_chunk:
+                chunks.append(current_chunk)
+                current_chunk = []
+                current_size = 0
+            
+            current_chunk.append(entry)
+            current_size += text_len
+        
+        if current_chunk:
+            chunks.append(current_chunk)
+        
+        return chunks
+    
+    def get_chapter_stats(self, entries: List[ManifestEntry]) -> List[Dict]:
+        """
+        Get statistics for each chapter.
+        
+        Returns list of dicts with: file_path, total, translated, pending, chars
+        """
+        chapters = self._group_by_chapter(entries)
+        stats = []
+        
+        for file_path, chapter_entries in sorted(chapters.items()):
+            total = len(chapter_entries)
+            translated = sum(1 for e in chapter_entries if e.translated_text)
+            total_chars = sum(len(e.original_text) for e in chapter_entries)
+            
+            # Get first text as title preview
+            first_text = ""
+            for e in chapter_entries:
+                if e.original_text:
+                    first_text = e.original_text[:40].replace('\n', ' ')
+                    break
+            
+            stats.append({
+                "file_path": file_path,
+                "title": first_text,
+                "total": total,
+                "translated": translated,
+                "pending": total - translated,
+                "chars": total_chars
+            })
+        
+        return stats
+