Synchronize chapter and navigation translations with Chinese prose indentation

src/assembly/backfiller.py

@@ -1,5 +1,7 @@
 import copy
+import re
 from src.common.html import parse_html
+from src.preprocessing.navigation import normalize_heading_html, sync_navigation
 from src.assembly.format_restorer import FormatRestorer
 from src.common.utils import setup_logger, add_spacing
 
@@ -10,6 +12,7 @@ class BackfillEngine:
     async def backfill(self, structure, manifest_entries, mode='bilingual', allow_partial=False):
         if mode not in {'bilingual', 'target_only'}:
             raise ValueError(f'Unknown output mode: {mode}')
+        titles = {r["entry_id"]: r for r in structure.navigation_titles}
         by_file = {}
         failures = []
         applied = 0
@@ -21,9 +24,16 @@ class BackfillEngine:
             restored, ok = FormatRestorer().restore(add_spacing(entry.translated_text), entry.placeholders,
                                              original_text=entry.original_text)
             if not ok:
+                entry.translated_html = None
+                entry.validation_errors = ["invalid restored XHTML"]
                 failures.append(entry.entry_id)
                 continue
             entry.translated_html = restored
+            record = titles.get(entry.entry_id)
+            if record:
+                entry.translated_html=normalize_heading_html(record,restored)
+            if entry.file_path=='__navigation__':
+                continue
             by_file.setdefault(entry.file_path, []).append(entry)
         if failures and not allow_partial:
             raise ValueError(f"Backfill rejected {len(failures)} unvalidated entries: {failures[:3]}")
@@ -49,6 +59,21 @@ class BackfillEngine:
                 new_tag['xml:lang'] = 'zh-CN'
                 classes = list(new_tag.get('class', []))
                 new_tag['class'] = classes + ['translation']
+                heading = entry.context and (entry.context.startswith('navigation-heading:') or entry.context in {'chapter','section','subsection'})
+                special = re.search(r'caption|footnote|endnote|epigraph|verse|poem|center|attribution|dedication', ' '.join(element.get('class', [])), re.I)
+                prose = ((tag_name in {'p','div'} or (tag_name=='span' and element.get('data-translation-unit')))
+                         and not heading and entry.context != 'epigraph'
+                         and not special and not re.search(r'text-align\s*:\s*(?:center|right)', element.get('style',''),re.I)
+                         and not element.find_parent(['blockquote', 'figure', 'aside'])
+                         and not element.find_parent(attrs={'epub:type': re.compile(r'footnote|endnote')}))
+                # Chinese body paragraphs use two em even when the publisher's
+                # opening-paragraph class sets text-indent:0.
+                new_tag['data-translation-role'] = 'body' if prose else 'heading' if heading else 'other'
+                if prose:
+                    # Publisher inline !important rules otherwise outrank CSS.
+                    style = re.sub(r'(?:^|;)\s*text-indent\s*:[^;]*', '', new_tag.get('style',''),flags=re.I).strip('; ')
+                    if style: new_tag['style'] = style
+                    else: new_tag.attrs.pop('style', None)
                 applied += 1
                 fragment = parse_html(f'<div>{entry.translated_html}</div>').body.div
                 for node in list(fragment.contents):
@@ -63,6 +88,8 @@ class BackfillEngine:
                 else:
                     element.replace_with(new_tag)
             resource.content = str(soup)
+            resource.translation_modified = True
+        sync_navigation(structure, manifest_entries, mode)
         if mode == 'target_only' and applied and not allow_partial:
             structure.metadata.language = 'zh-CN'
         return structure

src/assembly/bilingual_style.css

@@ -1,4 +1,4 @@
-/* Preserve the book's alignment, indent and inline font choices. */
+/* Preserve publisher alignment and inline fonts; apply Chinese prose indentation. */
 .translation {
     line-height: 1.6;
     clear: both;
@@ -17,3 +17,8 @@ p.translation::first-letter {
     color: inherit !important;
     float: none !important;
 }
+/* Prose gets two Chinese character widths, including chapter-opening prose.
+   Headings, captions, quotations and note markers keep their own layout. */
+.translation[data-translation-role="body"] {
+    text-indent: 2em !important;
+}

src/assembly/package_writer.py

@@ -37,15 +37,20 @@ def write_preserving_package(original, structure, output):
         etree.SubElement(manifest, f'{{{OPF}}}item', id=css_id, href=css_href, **{'media-type': 'text/css'})
         replacements[css_archive_path] = (Path(__file__).parent/'bilingual_style.css').read_bytes()
         for item_id, resource in structure.resources.items():
-            if resource.media_type != 'application/xhtml+xml' or not resource.content:
+            if resource.media_type not in {'application/xhtml+xml','application/x-dtbncx+xml'} or not resource.content:
                 continue
-            if 'nav' in (items.get(item_id).get('properties', '') if item_id in items else '').split() and 'translation-for' not in resource.content and 'class="translation"' not in resource.content:
+            if 'nav' in (items.get(item_id).get('properties', '') if item_id in items else '').split() and not resource.translation_modified and 'translation-for' not in resource.content and 'class="translation"' not in resource.content:
                 continue
             if item_id not in items:
                 raise ValueError(f'Unknown package item: {item_id}')
             archive_path = posixpath.normpath(posixpath.join(base, unquote(items[item_id].get('href'))))
             if archive_path not in source.namelist():
                 raise ValueError(f'Missing source resource: {archive_path}')
+            if resource.media_type=='application/x-dtbncx+xml':
+                if resource.translation_modified:
+                    etree.fromstring(resource.content.encode(),etree.XMLParser(resolve_entities=False,no_network=True))
+                    replacements[archive_path]=resource.content.encode()
+                continue
             soup = parse_html(resource.content)
             soup.html['xmlns'] = 'http://www.w3.org/1999/xhtml'
             soup.html['xmlns:epub'] = 'http://www.idpf.org/2007/ops'

src/common/data_model.py

@@ -28,11 +28,13 @@ class ResourceItem(BaseModel):
     content: Optional[str] = None  # For text/html
     file_path: Optional[str] = None # For binary/assets (relative to assets dir)
     properties: Optional[str] = None
+    translation_modified: bool = False
 
 class BookStructure(BaseModel):
     source_sha256: Optional[str] = None
-    preprocessing_version: int = 2
+    preprocessing_version: int = 3
     metadata: BookMetaData
+    navigation_titles: List[Dict[str, Any]] = Field(default_factory=list)
     spine: List[str] = Field(default_factory=list, description="Ordered list of item IDs in spine")
     resources: Dict[str, ResourceItem] = Field(default_factory=dict, description="Map of item_id to ResourceItem")
     

src/preprocessing/navigation.py

@@ -0,0 +1,296 @@
+"""One title translation per destination, shared by headings, NAV, NCX and TOC pages.
+
+Original href/src, anchors and hierarchy are never rebuilt from translated text.
+Uncertain heading matches are recorded rather than replacing arbitrary prose.
+"""
+import hashlib
+import posixpath
+import re
+import zipfile
+from urllib.parse import unquote, urlsplit
+from bs4 import NavigableString
+from lxml import etree
+from src.common.html import parse_html
+from src.common.data_model import ManifestEntry
+from src.preprocessing.format_extractor import FormatExtractor
+from src.assembly.package_writer import package_path
+
+HEADINGS = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']
+WORDS = {word: i for i, word in enumerate('zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen twenty'.split())}
+NUMBER = r'(?:\d+|[ivxlcdm]+|' + '|'.join(WORDS) + ')'
+
+
+def destination(document, href):
+    parts = urlsplit(href)
+    if parts.scheme or parts.netloc:
+        return None
+    path = (posixpath.normpath(posixpath.join(posixpath.dirname(document), unquote(parts.path)))
+            if parts.path else document)
+    return path + ('#' + unquote(parts.fragment) if parts.fragment else '')
+
+
+def plain(text):
+    return re.sub(r'\s+', ' ', text).strip()
+
+
+def is_toc(soup, href, properties=''):
+    return ('nav' in properties.split() or soup.find(attrs={'role': 'doc-toc'}) is not None
+            or any('toc' in str(t.get('epub:type', '')).split() for t in soup.find_all('nav'))
+            or re.search(r'(?:^|[/_\-\d])(toc|contents|nav)(?:[._\-]|$)', href, re.I) is not None
+            or any(plain(h.get_text()).casefold() in {'contents', 'table of contents', '目录'}
+                   for h in soup.find_all(HEADINGS)))
+
+
+def title_identity(label):
+    label = plain(label)
+    label, count = re.subn(r'^(?:chapter|chap\.?|part|book|section)\s+' + NUMBER + r'\b\s*[:.|\-–—]?\s*', '', label, count=1, flags=re.I)
+    if not count:
+        label = re.sub(r'^' + NUMBER + r'\b\s*[.:|\-–—]?\s*', '', label, count=1, flags=re.I)
+    return re.sub(r'[^\w]', '', label.casefold())
+
+
+def number_info(label):
+    match = re.match(r'^(chapter|chap\.?|part|book)\s+(' + NUMBER + r')\b', plain(label), re.I)
+    if match:
+        role = 'chapter' if match[1].lower().startswith('chap') else 'part'
+        number = match[2]
+    else:
+        match = re.match(r'^(\d+)\b', plain(label))
+        if not match: return None, None
+        role, number = 'chapter', match[1]
+    if number.isdigit():
+        value = int(number)
+    elif number.casefold() in WORDS:
+        value = WORDS[number.casefold()]
+    else:
+        values = {'I':1,'V':5,'X':10,'L':50,'C':100,'D':500,'M':1000}
+        value, previous = 0, 0
+        for char in reversed(number.upper()):
+            current=values[char]; value += -current if current < previous else current; previous=max(previous,current)
+    return role, value
+
+
+def chinese_number(number):
+    if number < 0 or number >= 10000:
+        return str(number)
+    if number == 0:
+        return '零'
+    digits='零一二三四五六七八九'; result=''; zero=False
+    for divisor,unit in [(1000,'千'),(100,'百'),(10,'十'),(1,'')]:
+        digit,number=divmod(number,divisor)
+        if digit:
+            if zero: result+='零'
+            if not (digit==1 and divisor==10 and not result): result+=digits[digit]
+            result+=unit;zero=False
+        elif result and number:
+            zero=True
+    return result
+
+
+def ordinal(record):
+    if record.get('number') is None:
+        return None
+    return '第'+chinese_number(record['number'])+('章' if record['role']=='chapter' else '部')
+
+
+def _headings(soup, target, label):
+    fragment = target.partition('#')[2]
+    anchor = soup.find(id=fragment) if fragment else None
+    if fragment and anchor is None:
+        return [], 'missing anchor'
+    identity = title_identity(label)
+    # Only inspect the opening leaf blocks or blocks at the explicit anchor.
+    # An exact match across adjacent blocks also recognizes Calibre-converted
+    # number/title/subtitle paragraphs without guessing from font sizes.
+    blocks = soup.find_all(HEADINGS + ['p', 'div'])
+    blocks = [t for t in blocks if not t.find(HEADINGS + ['p','div','section','blockquote','li'])]
+    if anchor:
+        nearby = [anchor] + anchor.find_all_next(HEADINGS + ['p','div'], limit=16)
+        block_ids = {id(t) for t in blocks}
+        blocks = [t for t in nearby if id(t) in block_ids]
+    blocks = [t for t in blocks if plain(t.get_text())][:12]
+    matches = []
+    for start in range(min(len(blocks), 6)):
+        for end in range(start+1, min(start+4, len(blocks))+1):
+            group = blocks[start:end]
+            if any(len(plain(t.get_text())) > 300 for t in group): break
+            combined = ' '.join(t.get_text(' ', strip=True) for t in group)
+            if title_identity(combined) == identity and identity:
+                matches.append(group)
+    # A number leaf and the subtitle alone can both match after stripping a
+    # numeric prefix. Prefer the complete adjacent title group.
+    if matches:
+        longest = max(len(g) for g in matches)
+        matches = [g for g in matches if len(g) == longest]
+        if len(matches) == 1: return matches[0], None
+        return [], 'ambiguous heading'
+    # An explicit heading anchor is authoritative even if the publisher uses
+    # an abbreviated TOC label; a prose anchor never receives this fallback.
+    if anchor and anchor.name in HEADINGS: return [anchor], None
+    return [], 'no matching heading'
+
+
+def prepare_navigation(original, structure, entries):
+    """Enrich the manifest with canonical heading entries, never translate TOC duplicates."""
+    soups={r.href:parse_html(r.content) for r in structure.resources.values()
+           if r.media_type=='application/xhtml+xml' and r.content}
+    resources={posixpath.normpath(unquote(r.href)):r for r in structure.resources.values()}
+    labels={}
+    def collect(document, href, label):
+        target=destination(document,href)
+        if target and plain(label):
+            record=labels.setdefault(target,{'target':target,'source_label':plain(label),'source_variants':[]})
+            if plain(label) not in record['source_variants']: record['source_variants'].append(plain(label))
+    # EPUB3 NAV and explicit contents pages take priority over legacy NCX names.
+    toc_files=set()
+    for href,soup in soups.items():
+        if not is_toc(soup,href,resources[posixpath.normpath(unquote(href))].properties or ''):
+            continue
+        toc_files.add(href)
+        toc_containers=[n for n in soup.find_all('nav') if 'toc' in str(n.get('epub:type','')).split() or n.get('role')=='doc-toc']
+        containers=toc_containers or [soup.body]
+        for container in containers:
+            for a in container.find_all('a',href=True): collect(href,a['href'],a.get_text(' ',strip=True))
+    with zipfile.ZipFile(original) as source:
+        base=posixpath.dirname(package_path(source))
+        for resource in structure.resources.values():
+            if resource.media_type!='application/x-dtbncx+xml': continue
+            raw=source.read(posixpath.normpath(posixpath.join(base,unquote(resource.href))))
+            doc=etree.fromstring(raw,etree.XMLParser(resolve_entities=False,no_network=True))
+            for point in doc.xpath('//*[local-name()="navMap"]//*[local-name()="navPoint"]'):
+                content=point.xpath('./*[local-name()="content"]/@src')
+                text=point.xpath('./*[local-name()="navLabel"]/*[local-name()="text"]')
+                if content and text: collect(resource.href,content[0],''.join(text[0].itertext()))
+            # NCX stays XML; it is not passed through an HTML parser.
+            resource.content=etree.tostring(doc,encoding='unicode')
+    # Ordinary manifest extraction must not independently translate visible
+    # TOC text. Those links are synchronized by target after title translation.
+    entries[:]=[e for e in entries if e.file_path not in toc_files]
+    by_id={e.entry_id:e for e in entries}; records=[]
+    for record in labels.values():
+        path=record['target'].partition('#')[0]; resource=resources.get(path)
+        role,number=number_info(record['source_label']);record.update(role=role,number=number)
+        headings=[];reason='missing document'
+        if resource and resource.href in soups:
+            headings,reason=_headings(soups[resource.href],record['target'],record['source_label'])
+        if headings:
+            ids=[]
+            for index,heading in enumerate(headings):
+                if not heading.get('id'):
+                    heading['id']='title-'+hashlib.sha256((record['target']+str(index)).encode()).hexdigest()[:16]
+                entry_id=f'{resource.href}#{heading["id"]}'
+                ids.append(entry_id)
+                if entry_id not in by_id:
+                    _,text,mapping,_,_=FormatExtractor().extract(str(heading))
+                    by_id[entry_id]=ManifestEntry(entry_id=entry_id,file_path=resource.href,element_id=heading['id'],
+                                                 original_text=text,placeholders=mapping)
+                    entries.append(by_id[entry_id])
+                by_id[entry_id].context='navigation-heading: '+record['source_label']
+            record.update(entry_id=ids[0],entry_ids=ids,heading_id=headings[0]['id'],file_path=resource.href)
+        else:
+            entry_id='navigation:'+record['target']
+            record.update(entry_id=entry_id,heading_id=None,file_path=path,binding_note=reason)
+            by_id[entry_id]=ManifestEntry(entry_id=entry_id,file_path='__navigation__',element_id=entry_id,
+                                         original_text=record['source_label'],context='navigation-label: '+record['source_label'])
+            entries.append(by_id[entry_id])
+        records.append(record)
+    # Contents headings are not link labels but still need the same heading policy.
+    for href in toc_files:
+        for h in soups[href].find_all(HEADINGS):
+            if plain(h.get_text()).casefold() not in {'contents','table of contents','目录'}: continue
+            if not h.get('id'): h['id']='translation-contents-heading'
+            entry_id=f'{href}#{h["id"]}'
+            if entry_id in by_id: continue
+            _,text,mapping,_,_=FormatExtractor().extract(str(h))
+            entry=ManifestEntry(entry_id=entry_id,file_path=href,element_id=h['id'],original_text=text,
+                                placeholders=mapping,context='navigation-heading: Contents')
+            entries.append(entry);by_id[entry_id]=entry
+    for href,soup in soups.items(): resources[posixpath.normpath(unquote(href))].content=str(soup)
+    structure.navigation_titles=records
+    return entries
+
+
+def normalize_heading_html(record, html):
+    """Convert separate Arabic/Roman chapter-number leaves without changing tags."""
+    prefix=ordinal(record)
+    if not prefix: return html
+    soup=parse_html('<div>'+html+'</div>');root=soup.body.div
+    nodes=[n for n in root.find_all(string=True) if plain(str(n))]
+    if not nodes: return html
+    node=nodes[0];text=plain(str(node));compact=re.sub(r'\s+', '', text).strip('：:')
+    # Some publishers put the digit in its own styled span: 第<span>1</span>部.
+    if (compact=='第' and len(nodes)>=3 and re.fullmatch(r'\d+|[IVXLCDM]+',plain(str(nodes[1])),re.I)
+            and re.fullmatch(r'[章部篇][：:]?',plain(str(nodes[2])))):
+        nodes[1].replace_with(chinese_number(record['number']))
+        nodes[2].replace_with('章' if record['role']=='chapter' else '部')
+    elif re.fullmatch(r'(?:\d+|[IVXLCDM]+|第[零一二三四五六七八九十百千万\d]+[章部篇])',compact,re.I):
+        node.replace_with(prefix)
+        if len(nodes)>1 and plain(str(nodes[1])) in {':','：'}: nodes[1].replace_with('')
+    elif re.match(r'^第[零一二三四五六七八九十百千万\d]+[章部篇]',compact):
+        node.replace_with(re.sub(r'^第[零一二三四五六七八九十百千万\d]+[章部篇]',prefix,compact))
+    return root.decode_contents()
+
+
+def navigation_labels(structure, entries):
+    by_id={e.entry_id:e for e in entries}; translated={}
+    for record in structure.navigation_titles:
+        units=[by_id.get(key) for key in record.get('entry_ids',[record['entry_id']])]
+        if any(not e or not e.translated_html or e.validation_errors for e in units): continue
+        units[0].translated_html=normalize_heading_html(record,units[0].translated_html)
+        label=plain(' '.join(parse_html(e.translated_html).get_text(' ',strip=True) for e in units))
+        prefix=ordinal(record)
+        # Standardize punctuation only for the navigation label; the rich
+        # heading can keep its number and subtitle on separate lines.
+        if prefix:
+            label=re.sub(r'^第\s*[零一二三四五六七八九十百千万\d]+\s*[章部篇]',prefix,label)
+        if prefix and label.startswith(prefix):
+            tail=label[len(prefix):].strip(' ：:.-–—')
+            label=prefix+('：'+tail if tail else '')
+        translated[record['target']]=label
+    return translated
+
+
+def sync_navigation(structure, entries, mode):
+    translated=navigation_labels(structure,entries)
+    if not translated: return
+    records={r['target']:r for r in structure.navigation_titles}
+    def display(target):
+        target_text=translated.get(target)
+        if not target_text: return None
+        source=records[target]['source_label']
+        return source+' / '+target_text if mode=='bilingual' else target_text
+    for resource in structure.resources.values():
+        if not resource.content: continue
+        modified=False
+        if resource.media_type=='application/x-dtbncx+xml':
+            doc=etree.fromstring(resource.content.encode(),etree.XMLParser(resolve_entities=False,no_network=True))
+            for point in doc.xpath('//*[local-name()="navMap"]//*[local-name()="navPoint"]'):
+                refs=point.xpath('./*[local-name()="content"]/@src')
+                labels=point.xpath('./*[local-name()="navLabel"]/*[local-name()="text"]')
+                text=display(destination(resource.href,refs[0])) if refs else None
+                if labels and text:
+                    labels[0].text=text;modified=True
+            if modified: resource.content=etree.tostring(doc,encoding='unicode')
+        elif resource.media_type=='application/xhtml+xml':
+            soup=parse_html(resource.content)
+            own = next((display(r['target']) for r in structure.navigation_titles
+                        if r['file_path'] == resource.href and r.get('heading_id') and display(r['target'])), None)
+            if own and soup.head:
+                if not soup.head.title: soup.head.append(soup.new_tag('title'))
+                soup.head.title.string=own; modified=True
+            if not is_toc(soup,resource.href,resource.properties or ''):
+                if modified:
+                    resource.content=str(soup); resource.translation_modified=True
+                continue
+            for link in soup.find_all('a',href=True):
+                text=display(destination(resource.href,link['href']))
+                if not text: continue
+                # Preserve the link node, its attributes and styled descendants.
+                nodes=[n for n in link.find_all(string=True) if plain(str(n))]
+                if nodes:
+                    nodes[0].replace_with(text)
+                    for n in nodes[1:]: n.replace_with('')
+                else: link.append(NavigableString(text))
+                link['data-navigation-translated']='true';modified=True
+            if modified: resource.content=str(soup)
+        if modified: resource.translation_modified=True

src/translation/llm_client.py

@@ -144,6 +144,7 @@ Output Format:
         """
         id_map = {}  # short_id -> entry_id
         lines = []
+        title_notes = []
         
         for i, item in enumerate(items, 1):
             short_id = f"#{i}"
@@ -151,9 +152,18 @@ Output Format:
             
             # Clean text (remove extra whitespace)
             text = re.sub(r'\s+', ' ', item.original_text).strip()
+            if item.context and item.context.startswith('navigation-'):
+                title_notes.append(f"{short_id}: {item.context}")
             lines.append(f"{short_id}: {text}")
         
-        return id_map, "\n".join(lines)
+        if title_notes:
+            instructions = ("Title context (instructions only; do not copy into translations):\n"
+                            "Translate Chapter N as 第N章, Part N as 第N部, and translate the title itself. "
+                            "Keep number/subtitle styling tokens in their original regions.\n"
+                            + "\n".join(title_notes) + "\n\nSource units:\n")
+        else:
+            instructions = ""
+        return id_map, instructions + "\n".join(lines)
 
     def _parse_short_id_response(self, response: str, id_map: Dict[str, str]) -> Dict[str, str]:
         """

src/translation/manifest_manager.py

@@ -59,6 +59,7 @@ class ManifestManager:
         for entry in new_entries:
             previous = self._entries_map.get(entry.entry_id)
             if previous and previous.original_text == entry.original_text and previous.placeholders == entry.placeholders:
+                previous.context = entry.context
                 reconciled.append(previous)
             else:
                 reconciled.append(entry)

src/translation/translator_engine.py

@@ -26,7 +26,7 @@ class Translator:
         self.restorer = FormatRestorer()
 
     def _fingerprint(self, entry, profile):
-        data = [entry.original_text, entry.placeholders, self.glossary.fingerprint(),
+        data = [entry.original_text, entry.placeholders, entry.context, self.glossary.fingerprint(),
                 getattr(self.llm_client, 'model', ''), profile.style_guide if profile else '']
         return hashlib.sha256(json.dumps(data, sort_keys=True, ensure_ascii=False).encode()).hexdigest()