From 3e2093dba88d37d01b51afa6e188260daee79819 Mon Sep 17 00:00:00 2001 From: Kai Tan Date: Sat, 10 Oct 2026 00:42:37 +0800 Subject: [PATCH] Fix edition font UUID rebinding and Protocols translation layout --- CHANGELOG.md | 9 ++ doc/QA_PROTOCOLS_2026-10-10.md | 57 +++++++ src/assembly/bilingual_style.css | 5 + src/assembly/format_restorer.py | 7 + src/edition/fonts.py | 246 +++++++++++++++++++++++++++++++ src/edition/localize.py | 12 ++ src/preprocessing/navigation.py | 14 +- src/qa/headless.py | 13 +- tests/test_edition.py | 40 +++++ tests/test_fonts.py | 103 +++++++++++++ tests/test_navigation.py | 12 ++ tests/test_regressions.py | 8 + 12 files changed, 522 insertions(+), 4 deletions(-) create mode 100644 doc/QA_PROTOCOLS_2026-10-10.md create mode 100644 src/edition/fonts.py create mode 100644 tests/test_fonts.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 93d6457..e32a0d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## 2026-10-10 — Protocols AZW3 edition and embedded fonts + +- Bind graphical chapter headings to navigation using image alt text while preserving source markup. +- Preserve publisher heading and note spacing; apply translation separation to prose and bilingual insertions. +- Rebind standard IDPF/Adobe font obfuscation when creating independent edition UUIDs, preserving original font bytes and auditing decoded content. +- Serve decoded embedded fonts in headless QA; cover legacy Calibre font MIME types and local CSS/style/preload references. +- Reject empty translated spans that delete styled protocol numbers. +- Complete Protocols in both EPUB modes with Chinese covers, source-bound metadata, resource preservation and documented sample QA. + ## 2026-10-08 — nested chapter anchors and Highest Exam regression - Bind TOC anchors nested inside headings to their enclosing title; exclude punctuation-only ornaments from heading groups. diff --git a/doc/QA_PROTOCOLS_2026-10-10.md b/doc/QA_PROTOCOLS_2026-10-10.md new file mode 100644 index 0000000..a938d52 --- /dev/null +++ b/doc/QA_PROTOCOLS_2026-10-10.md @@ -0,0 +1,57 @@ +# Protocols — AZW3 转 EPUB、Luna 翻译与抽样 QA + +本次按更新后的 `skills/epub-translate/SKILL.md` 完成纯中文与中英对照两版，版本 `luna-r1`。项目译名为《实践方案：人体操作手册》；没有确认正式中文出版书名。正文由 `gpt-6-luna` 子 agent 按完整 Manifest 执行，无外部翻译 API。封面使用 Codex 内置 imagegen 编辑原封面，工具未公开精确模型 ID，记录为 `model: null`。 + +## 输入与范围 + +- 原始 AZW3 SHA256：`70c07109484301970f8ba9df7a503b11a56d768102883a2a6f02fc68c7daba27`。 +- Calibre 转换后冻结 EPUB SHA256：`5c0f743c8afa9c0a24044c3dfde267d05c0141674aae3c7b4398d7a05be9fefa`。 +- AZW3 解包与 EPUB 的正文空白词元计数均为 228,905，计数差为零。这是转换完整度证据，不能替代结构或逐句语义审校。 +- 完整提取 5,009 单元，69 个大段上下文批次；术语表 94 条。范围包含前言、声明、全部正文、致谢、作者介绍、参考文献、索引和版权页。 +- 原书 84 个 XHTML 文档、64 个目录项。书内注释与印刷页码／索引链接分别审计，不能把全部内部链接称为脚注。 + +## 此次修复 + +1. 章节标题中使用图片表示 `Protocols`，标题匹配原本忽略图片 alt。现以副本中的 alt 参与目录绑定，原图片和 DOM 保留；实际章节标题译文成为目录共享来源，减少14个重复提取的目录单元。 +2. 通用翻译 margin 覆盖出版方章节标题顶部留白。现仅对正文和双语插入单元增加间距，保留中文替换标题及注释的出版方间距。 +3. 换 AI 版本 UUID 会破坏 EPUB 标准字体混淆。新增 IDPF/Adobe 字体处理：保留原字体字节，克隆并重新绑定活跃字体，改写 manifest、encryption.xml 和本地 CSS／inline／font preload 引用。7个字体的解码结果独立与 AZW3 原始字体逐字节核对；headless 在内存中解码字体再提供给浏览器。 +4. 转换源有两个空 Calibre 页分隔 div 与章节标题重 ID；重建后的最终输出不再出现这些重复 ID，原章节标题锚点和链接目标仍有效。 +5. 视觉抽查及全书开头强调标记位置筛查修正两处短提示粗体移位、两处学习时间段的粗体范围、一处脚注上标延伸到解释句；修正翻译结果中的标记位置，再完整回填。长度比例筛查未发现其他长范围错移，但它不能替代语义检查。 +6. 最终英文残留检查修正索引标题和57个遗漏索引词、6个注释章名；品牌、URL、文献标题及刊名按原作保留。 +7. 图形编号抽查发现个人成长方案3／4的嵌套标记内数字漏掉，恢复原数字；加入校验，禁止数字标记范围被翻译为空，允许非空的本地化数字。 + +## 版本整理与验收边界 + +两版使用不同颜色徽标、独立 UUID、模式及修订号一致的 metadata；作者保留原拼写 Andrew D. Huberman，封面保留 with Jessica Wapner 的原作贡献者署名。原作 ISBN／UUID 记录为 source，不用作 AI 译本出版身份。中文译本日期为生成日期，原出版社／日期留在版本来源记录。 + +原 AZW3 的15个图片／字体资产全部按 SHA 保留在最终 EPUB；转换 EPUB 的14个原二进制资源原路径原字节不变。额外保留转换时未使用的小封面缩略图和原始解码字体，使用新封面、新字体克隆作为活跃资源。 + +独立 Luna 语义抽查23个长段（15个运动、营养、压力、光照、学习样本；8个睡眠与引言样本），并由主 agent 核对补充剂剂量、警告、否定与不确定性、列表、标题和书后文字。不是全书逐句人工校对。 + +最终 headless 对照抽样包含封面、引言、图形章节标题、睡眠补充剂列表、饮酒段落、压力管理、WOOP、表达性写作、参考注释和索引；另查看180×270书库缩略图。浏览器禁用书内脚本和外部请求。原始自动报告中的 visual/semantic 状态保留 pending，实际人工／agent 审核证据另写 review-record.json。Chromium 连续 XHTML 的截图不代表所有阅读器的分页。 + +原图形章节词 `Protocols`、整页英文图片扉页、商标图片内文字保留英文；它们不属于文本翻译提取范围。参考文献部分保留可追溯的原文文献标题／刊名。索引保持原作顺序和印刷页码链接，未重排为中文拼音索引。 + +## 本地证据 + +工作目录：`.work/luna-protocols/`（不提交书籍或译文到仓库）。 + +- `conversion-source.json`、`conversion-review.json`：AZW3 转换与来源。 +- `manifest.json`、`book_structure.json`、`agent-run.json`、`results/`：全书范围、源摘要与翻译结果。 +- `semantic-review-luna.json`、`semantic-review-sleep-luna.json`：独立语义抽查。 +- `edition/edition.json`、各模式 `review.json`：书名来源、封面提供方及图片 SHA。 +- `asset-font-verification.json`：最终资源 SHA 与字体解码验证。 +- `qa-中文版/`、`qa-中英对照版/`：最终结构报告与对照截图。 +- `qa-thumb-final-中文版/`、`qa-thumb-final-中英对照版/`：实际 EPUB 缩略图。 +- `review-record.json`：最终输出 SHA、实际已看截图和抽样范围。 + +书目核对依据：[原作出版社页面](https://www.simonandschuster.com/books/Protocols/Andrew-D-Huberman/9781668032152)、[作者关于翻译版本的 FAQ](https://www.hubermanlab.com/faq/will-protocols-be-available-in-languages-other-than-english)。前者用于原作核对，后者不构成具体中文书名的权威依据。字体处理依据：[W3C EPUB 3.3 字体混淆规范](https://www.w3.org/TR/epub-33/#sec-font-obfuscation)、[Calibre 字体处理实现](https://github.com/kovidgoyal/calibre/blob/master/src/calibre/ebooks/oeb/polish/container.py)。 + +## 最终验证结果 + +两版均为5,009/5,009完整回填、64个目录项、零新增结构问题、零新增自动布局问题。原书406个书内注释引用保留；双语版同时保留原文和译文引用，所以为812个。每版抽样18次内部目标点击（原文9次、译文9次），其中原文／译文各4次分类为书内注释且返回链接成功；其余5次各为非注释目标，不能混称脚注。 + +每版生成13个文档、48张截图，最终记录明确列出实际已查看图片与SHA；另有前几轮原文／译文对照证据。独立语义抽查23个长段，原自动报告没有改写成全书视觉／语义通过。测试结果：89 passed，2 skipped；实际 Chromium EPUB 复验另行完成。 + +- Protocols_中文版_luna-r1.epub：`86ce3cf735443bc33d6e92093d275e61d6b568b15ebe8228949a46f9203bfef2`。 +- Protocols_中英对照版_luna-r1.epub：`7995e864d5044a7637d8a5f14a9e782936f7df22bdb7b0bfbd5e78f0a34a12e0`。 diff --git a/src/assembly/bilingual_style.css b/src/assembly/bilingual_style.css index e8932e7..8e2b2c7 100644 --- a/src/assembly/bilingual_style.css +++ b/src/assembly/bilingual_style.css @@ -2,6 +2,11 @@ .translation { line-height: 1.6; clear: both; +} +/* Preserve publisher spacing on replacement headings and notes. Only prose + and a bilingual insertion need the additional translation separation. */ +.translation[data-translation-role="body"], +.translation[data-translation-for] { margin-top: 0.5em; } /* Units wrapped around loose text must remain visible before nested blocks. */ diff --git a/src/assembly/format_restorer.py b/src/assembly/format_restorer.py index d47324f..bb52144 100644 --- a/src/assembly/format_restorer.py +++ b/src/assembly/format_restorer.py @@ -32,6 +32,13 @@ class FormatRestorer: stack.append(pid) if stack: issues.append('unclosed placeholder') + if original_text is not None: + # Token counts alone cannot detect deletion of a styled chapter or + # protocol number. Accept localized numerals, but never an empty span. + for match in re.finditer(r'φ(\d+)φ([0-9]+)φ/\1φ', original_text): + target = re.search(r'φ' + match[1] + r'φ(.*?)φ/' + match[1] + r'φ', text, re.S) + if target and not self.PLACEHOLDER_REGEX.sub('', target[1]).strip(): + issues.append('empty numeric placeholder span: ' + match[1]) return issues def restore(self, text_with_placeholders, placeholder_map, context_id=None, original_text=None): diff --git a/src/edition/fonts.py b/src/edition/fonts.py new file mode 100644 index 0000000..6627a50 --- /dev/null +++ b/src/edition/fonts.py @@ -0,0 +1,246 @@ +"""Safe decoding and identifier rebinding for standard EPUB font obfuscation.""" +from __future__ import annotations + +import hashlib +import posixpath +import re +import uuid +import zipfile +from urllib.parse import unquote, urlsplit, quote + +from lxml import etree + +OPF = 'http://www.idpf.org/2007/opf' +DC = 'http://purl.org/dc/elements/1.1/' +IDPF = 'http://www.idpf.org/2008/embedding' +ADOBE = 'http://ns.adobe.com/pdf/enc#RC' +ENCRYPTION = 'http://www.w3.org/2001/04/xmlenc#' +PARSER = lambda: etree.XMLParser(resolve_entities=False, no_network=True) +_MAGICS = (b'\x00\x01\x00\x00', b'OTTO', b'true', b'typ1', b'ttcf', b'wOFF', b'wOF2') +_FONT_MEDIA_TYPES = { + 'application/font-sfnt', 'application/vnd.ms-opentype', + 'application/x-font-truetype', 'application/x-font-ttf', + 'application/x-font-opentype', 'application/x-font-otf', + 'application/x-font-woff', 'application/font-woff', + 'application/font-woff2', 'font-woff', 'font-woff2', +} + + +def primary_identifier(root): + """Return the package identifier designated by unique-identifier.""" + uid = root.get('unique-identifier') + matches = root.xpath('//*[local-name()="identifier" and @id=$uid]', uid=uid) + if not uid or len(matches) != 1 or not (matches[0].text or '').strip(): + raise ValueError('Missing or ambiguous OPF primary identifier') + return matches[0].text + + +def _key(identifier, algorithm): + if algorithm == IDPF: + return hashlib.sha1(re.sub(r'[ \t\r\n]+', '', identifier).encode('utf-8')).digest() + if algorithm == ADOBE: + value = identifier.strip() + if value.lower().startswith('urn:uuid:'): + value = value[9:] + try: + return uuid.UUID(value.strip('{}')).bytes + except (ValueError, AttributeError) as exc: + raise ValueError('Adobe font obfuscation requires a UUID primary identifier') from exc + raise ValueError(f'Unsupported EPUB font obfuscation algorithm: {algorithm}') + + +def _safe_archive_path(base, uri): + parts = urlsplit(uri) + if parts.scheme or parts.netloc or parts.query or parts.fragment or not parts.path: + raise ValueError(f'Unsafe encrypted font URI: {uri}') + raw = unquote(parts.path) + if raw.startswith('/') or '\\' in raw or '\x00' in raw: + raise ValueError(f'Unsafe encrypted font path: {uri}') + path = posixpath.normpath(posixpath.join(posixpath.dirname(base), raw)) + if path in {'.', '..'} or path.startswith('../'): + raise ValueError(f'Encrypted font escapes EPUB root: {uri}') + return path + + +def _obfuscated(z, opf_path): + try: + root = etree.fromstring(z.read('META-INF/encryption.xml'), PARSER()) + except KeyError: + return [] + found = [] + for node in root.xpath('//*[local-name()="EncryptedData"]'): + methods = node.xpath('.//*[local-name()="EncryptionMethod"]/@Algorithm') + refs = node.xpath('.//*[local-name()="CipherReference"]/@URI') + if not methods and not refs: + continue + if len(methods) != 1 or len(refs) != 1: + raise ValueError('Malformed EPUB encryption.xml font entry') + algorithm = methods[0] + if algorithm not in {IDPF, ADOBE}: + raise ValueError(f'Unsupported EPUB encryption algorithm: {algorithm}') + # CipherReference URIs are rooted at the OCF container root. + path = _safe_archive_path('encryption.xml', refs[0]) + if path in {x[0] for x in found}: + raise ValueError(f'Duplicate encrypted resource: {path}') + # The OPF must declare encrypted resources as fonts, not arbitrary payloads. + package = etree.fromstring(z.read(opf_path), PARSER()) + items = package.xpath('//*[local-name()="manifest"]/*[local-name()="item"]') + item = next((it for it in items if _safe_archive_path(opf_path, it.get('href', '')) == path), None) + media_type = item.get('media-type', '').lower() if item is not None else '' + if item is None or not (media_type.startswith('font/') or media_type in _FONT_MEDIA_TYPES): + raise ValueError(f'Encrypted resource is not a manifest font: {path}') + found.append((path, algorithm)) + return found + + +def _xor_font(data, key, algorithm): + length = 1040 if algorithm == IDPF else 1024 + n = min(length, len(data)) + return bytes(data[i] ^ key[i % len(key)] for i in range(n)) + data[n:] + + +def _font_magic(data): + return data.startswith(_MAGICS) + + +def decoded_fonts(z, opf_path, root): + """Return decoded bytes for all standard IDPF/Adobe-obfuscated manifest fonts.""" + identifier = primary_identifier(root) + result = {} + for path, algorithm in _obfuscated(z, opf_path): + try: + data = z.read(path) + except KeyError as exc: + raise ValueError(f'Encrypted font is missing from EPUB: {path}') from exc + decoded = _xor_font(data, _key(identifier, algorithm), algorithm) + if not _font_magic(decoded): + raise ValueError(f'Decoded resource does not have a recognized font signature: {path}') + result[path] = decoded + return result + + +def _href_for_path(base, target): + return quote(posixpath.relpath(target, posixpath.dirname(base)), safe='/._-') + + +def _mapped_ref(value, document_path, path_map): + """Map local resource references only; external/data URLs stay untouched.""" + parts = urlsplit(value.strip()) + if parts.scheme or parts.netloc or parts.query or parts.fragment or not parts.path: + return None + try: + target = _safe_archive_path(document_path, parts.path) + except ValueError: + return None + replacement = path_map.get(target) + return _href_for_path(document_path, replacement) if replacement else None + + +def _rewrite_css_urls(text, document_path, path_map): + pattern = re.compile(r'(?Purl\(\s*)(?P[\"\']?)(?P[^\"\')]*?)(?P=quote)(?P\s*\))', re.I) + def substitute(match): + new_url = _mapped_ref(match.group('url'), document_path, path_map) + if new_url is None: + return match.group(0) + return match.group('prefix') + match.group('quote') + new_url + match.group('quote') + match.group('suffix') + return pattern.sub(substitute, text) + + +def rebind_fonts(z, opf_path, root, old_identifier, new_identifier, additions, replacements): + """Clone decoded fonts under edition-* paths, preserving the source font bytes.""" + primary = primary_identifier(root) + if primary not in {old_identifier, new_identifier}: + raise ValueError('Provided identifiers do not match OPF primary identifier') + # localize() may already have installed the new identifier on the in-memory + # OPF; decoding must still use the source identifier that protected the bytes. + if primary != old_identifier: + uid = root.get('unique-identifier') + node = root.xpath('//*[local-name()="identifier" and @id=$uid]', uid=uid)[0] + node.text = old_identifier + try: + fonts = decoded_fonts(z, opf_path, root) + finally: + node.text = new_identifier + else: + fonts = decoded_fonts(z, opf_path, root) + if not fonts: + return [] + manifest = root.find(f'{{{OPF}}}manifest') + if manifest is None: + raise ValueError('OPF manifest missing while rebinding fonts') + ids = {node.get('id') for node in manifest} + records = [] + path_map = {} + for original_path, decoded in fonts.items(): + algorithm = next(a for p, a in _obfuscated(z, opf_path) if p == original_path) + item = next((it for it in manifest if _safe_archive_path(opf_path, it.get('href', '')) == original_path), None) + if item is None: + raise ValueError(f'Encrypted font is missing from OPF manifest: {original_path}') + basename = posixpath.basename(original_path) + active_path = posixpath.join(posixpath.dirname(original_path), 'edition-' + basename) + if active_path in z.namelist() or active_path in additions: + raise ValueError(f'Edition font clone path already exists: {active_path}') + active_data = _xor_font(decoded, _key(new_identifier, algorithm), algorithm) + if not _font_magic(_xor_font(active_data, _key(new_identifier, algorithm), algorithm)): + raise ValueError(f'Rebound font failed signature check: {active_path}') + old_href = item.get('href') + new_href = _href_for_path(opf_path, active_path) + path_map[original_path] = active_path + item.set('href', new_href) + old_id = item.get('id') or 'font' + preserved_id = old_id + '-source' + while preserved_id in ids: + preserved_id += '-new' + ids.add(preserved_id) + source_href = _href_for_path(opf_path, original_path) + etree.SubElement(manifest, f'{{{OPF}}}item', id=preserved_id, href=source_href, **{'media-type': 'application/octet-stream'}) + additions[active_path] = active_data + records.append({'original_path': original_path, 'active_path': active_path, + 'original_sha256': hashlib.sha256(z.read(original_path)).hexdigest(), + 'decoded_sha256': hashlib.sha256(decoded).hexdigest(), 'algorithm': algorithm}) + + # Retarget encryption.xml to cloned active resources. It is written via replacements. + enc_path = 'META-INF/encryption.xml' + enc = etree.fromstring(replacements.get(enc_path, z.read(enc_path)), PARSER()) + for node in enc.xpath('//*[local-name()="CipherReference"]'): + uri = node.get('URI', '') + target = _safe_archive_path('encryption.xml', uri) + if target in path_map: + node.set('URI', path_map[target]) + replacements[enc_path] = etree.tostring(enc, encoding='utf-8', xml_declaration=True) + + # Retarget CSS url(...) and inline XHTML style declarations using pending replacements. + for name in z.namelist(): + lower = name.lower() + if lower.endswith('.css'): + raw = replacements.get(name, z.read(name)) + text = raw.decode('utf-8') + text = _rewrite_css_urls(text, name, path_map) + replacements[name] = text.encode('utf-8') + elif lower.endswith(('.xhtml', '.html', '.htm')): + raw = replacements.get(name, z.read(name)) + try: + doc = etree.fromstring(raw, PARSER()) + except etree.XMLSyntaxError: + continue + changed = False + for node in doc.iter(): + if not isinstance(node.tag, str): + continue + style = node.get('style') + if style: + updated = _rewrite_css_urls(style, name, path_map) + if updated != style: + node.set('style', updated); changed = True + if etree.QName(node).localname.lower() == 'link' and 'preload' in node.get('rel', '').lower().split() and node.get('as', '').lower() == 'font': + href = node.get('href') + updated = _mapped_ref(href, name, path_map) if href else None + if updated: + node.set('href', updated); changed = True + if etree.QName(node).localname.lower() == 'style' and node.text: + updated = _rewrite_css_urls(node.text, name, path_map) + if updated != node.text: + node.text = updated; changed = True + if changed: + replacements[name] = etree.tostring(doc, encoding='utf-8', xml_declaration=True) + return records diff --git a/src/edition/localize.py b/src/edition/localize.py index 8e7c243..b412db7 100644 --- a/src/edition/localize.py +++ b/src/edition/localize.py @@ -9,6 +9,7 @@ from pathlib import Path from urllib.parse import unquote, urlsplit, quote from lxml import etree from src.assembly.package_writer import package_path, OPF, DC +from src.edition.fonts import primary_identifier, rebind_fonts, decoded_fonts LABELS = {'target_only': '中文版', 'bilingual': '中英对照版'} PREFIX = 'https://kais.cloud/epub-translator/metadata/' @@ -144,6 +145,7 @@ def localize(epub, output, config, mode, cover, review): verify_binding(root, cfg) image, mime, suffix, reviewed = reviewed_cover(cover, review, cfg, mode) metadata = root.find(f'{{{OPF}}}metadata'); manifest = root.find(f'{{{OPF}}}manifest') + old_identifier = primary_identifier(root) original = {'titles': dc_values(root, 'title'), 'authors': dc_values(root, 'creator'), 'identifiers': dc_values(root, 'identifier'), 'languages': dc_values(root, 'language'), 'publishers': dc_values(root, 'publisher'), 'dates': dc_values(root, 'date')} @@ -276,6 +278,7 @@ def localize(epub, output, config, mode, cover, review): record['cover_bindings'].append({'file':page_path, 'target':image_path}) etree.SubElement(manifest, f'{{{OPF}}}item', id=page_id, href=page_href, **{'media-type': 'application/xhtml+xml'}) root.find(f'{{{OPF}}}spine').insert(0, etree.Element(f'{{{OPF}}}itemref', idref=page_id, linear='yes')) + record['font_bindings'] = rebind_fonts(z, opf_path, root, old_identifier, identifier, additions, replacements) if root.get('version', '').startswith('3'): prefix = root.get('prefix', '') if 'translation:' in prefix: raise ValueError('Source already defines translation prefix') @@ -315,6 +318,15 @@ def audit_edition(epub): with zipfile.ZipFile(epub) as z: if record['cover_path'] not in z.namelist() or sha(z.read(record['cover_path'])) != record['cover_sha256']: issues.append('edition_cover_hash_mismatch') + try: + fonts = decoded_fonts(z, opf_path, root) + for binding in record.get('font_bindings', []): + if binding['original_path'] not in z.namelist() or sha(z.read(binding['original_path'])) != binding['original_sha256']: + issues.append('edition_original_font_changed') + if binding['active_path'] not in fonts or sha(fonts[binding['active_path']]) != binding['decoded_sha256']: + issues.append('edition_font_binding_mismatch') + except (ValueError, KeyError): + issues.append('edition_font_binding_invalid') for binding in record['cover_bindings']: path = binding['file'] if path not in z.namelist(): diff --git a/src/preprocessing/navigation.py b/src/preprocessing/navigation.py index 9e2b37a..889d258 100644 --- a/src/preprocessing/navigation.py +++ b/src/preprocessing/navigation.py @@ -93,6 +93,14 @@ def ordinal(record): return '第'+chinese_number(record['number'])+('章' if record['role']=='chapter' else '部') +def _heading_text(tag): + """Use image alternative text to match a publisher's graphical title.""" + copy = parse_html(str(tag)) + for image in copy.find_all('img'): + image.replace_with(' ' + image.get('alt', '') + ' ') + return copy.get_text(' ', strip=True) + + def _headings(soup, target, label): fragment = target.partition('#')[2] anchor = soup.find(id=fragment) if fragment else None @@ -114,13 +122,13 @@ def _headings(soup, target, label): nearby = [anchor] + anchor.find_all_next(HEADINGS + ['p','div'], limit=16) block_ids = {id(t) for t in blocks} blocks = [t for t in nearby if id(t) in block_ids] - blocks = [t for t in blocks if any(c.isalnum() for c in plain(t.get_text()))][:12] + blocks = [t for t in blocks if any(c.isalnum() for c in _heading_text(t))][:12] matches = [] for start in range(min(len(blocks), 6)): for end in range(start+1, min(start+4, len(blocks))+1): group = blocks[start:end] - if any(len(plain(t.get_text())) > 300 for t in group): break - combined = ' '.join(t.get_text(' ', strip=True) for t in group) + if any(len(_heading_text(t)) > 300 for t in group): break + combined = ' '.join(_heading_text(t) for t in group) number_only = (not identity and len(group) == 1 and group[0].name in HEADINGS and number_info(label)[0] is not None and number_info(combined) == number_info(label)) diff --git a/src/qa/headless.py b/src/qa/headless.py index 341b58f..97217c3 100644 --- a/src/qa/headless.py +++ b/src/qa/headless.py @@ -16,6 +16,7 @@ from urllib.parse import quote, unquote, urlsplit from lxml import etree from src.common.html import parse_html from src.assembly.package_writer import package_path +from src.edition.fonts import decoded_fonts from src.assembly.format_restorer import FormatRestorer from src.common.data_model import ManifestEntry from src.qa.epub_audit import audit_epub, compare_audits @@ -266,6 +267,15 @@ def audit_pair(source, translated, manifest=None, structure=None): def archive_server(paths): """Serve allowlisted ZIP members on loopback, without extracting executable book data.""" archives = {key: zipfile.ZipFile(path) for key, path in paths.items()} + fonts = {} + try: + for key, archive in archives.items(): + opf_path = package_path(archive) + root = etree.fromstring(archive.read(opf_path), etree.XMLParser(resolve_entities=False, no_network=True)) + fonts[key] = decoded_fonts(archive, opf_path, root) + except Exception: + for archive in archives.values(): archive.close() + raise class Handler(BaseHTTPRequestHandler): def log_message(self, *args): pass @@ -274,7 +284,8 @@ def archive_server(paths): archive = archives.get(prefix) if not archive or name not in archive.namelist() or name.endswith('/'): self.send_error(404); return - data = archive.read(name) + data = fonts[prefix].get(name) + if data is None: data = archive.read(name) mime = 'text/html' if name.lower().endswith(('.xhtml', '.html', '.htm')) else mimetypes.guess_type(name)[0] or 'application/octet-stream' self.send_response(200) self.send_header('Content-Type', mime + ('; charset=utf-8' if mime.startswith('text/') else '')) diff --git a/tests/test_edition.py b/tests/test_edition.py index 1979c75..8ad8689 100644 --- a/tests/test_edition.py +++ b/tests/test_edition.py @@ -142,3 +142,43 @@ def test_headless_can_render_a_new_cover_without_a_source_counterpart(tmp_path): assert result['sampled_documents']==[cover_page] assert len(result['images'])==1 and result['images'][0]['book']=='translated' assert (evidence/result['images'][0]['image']).exists() + + +def test_new_edition_uuid_keeps_obfuscated_font_readable_and_audited(tmp_path): + from src.edition.fonts import decoded_fonts + import uuid + source,cfg,image,review,_ = fixture(tmp_path) + old_id = 'urn:uuid:' + str(uuid.uuid4()) + plain = b'\x00\x01\x00\x00' + bytes(range(256)) * 8 + key = uuid.UUID(old_id.removeprefix('urn:uuid:')).bytes + encoded = bytes(byte ^ key[i % 16] if i < 1024 else byte for i,byte in enumerate(plain)) + font_source = tmp_path/'fonts.epub' + with zipfile.ZipFile(source) as z, zipfile.ZipFile(font_source,'w') as target: + for info in z.infolist(): + data = z.read(info.filename) + if info.filename == 'EPUB/package.opf': + data = data.replace(b'id="old-id">9780000000001', b'id="old-id">'+old_id.encode()) + data = data.replace(b'', b'') + target.writestr(info,data) + target.writestr('EPUB/fonts/face.ttf',encoded) + target.writestr('EPUB/style.css', '@font-face {font-family: Face; src:url("fonts/face.ttf")}') + target.writestr('META-INF/encryption.xml', '') + config=json.loads(cfg.read_text());config['source']['identifier']=old_id;cfg.write_text(json.dumps(config)) + out=tmp_path/'new-fonts.epub';record=localize(font_source,out,cfg,'target_only',image,review) + assert record['identifier'] != old_id + assert audit_edition(out)['issues'] == [] + assert audit_pair(font_source,out)[0]['new_issues'] == [] + with zipfile.ZipFile(out) as z: + opf,root=read_package(out) + assert list(decoded_fonts(z,opf,root).values()) == [plain] + assert z.read('EPUB/fonts/face.ttf') == encoded + from src.qa.headless import archive_server + from urllib.request import urlopen + with archive_server({'edition': out}) as url: + with urlopen(url + '/edition/' + record['font_bindings'][0]['active_path']) as response: + assert response.read() == plain + damaged=tmp_path/'damaged-fonts.epub' + active=record['font_bindings'][0]['active_path'] + with zipfile.ZipFile(out) as z,zipfile.ZipFile(damaged,'w') as target: + for info in z.infolist(): target.writestr(info,b'broken' if info.filename==active else z.read(info.filename)) + assert any('font_binding' in issue for issue in audit_edition(damaged)['issues']) diff --git a/tests/test_fonts.py b/tests/test_fonts.py new file mode 100644 index 0000000..e18d4ab --- /dev/null +++ b/tests/test_fonts.py @@ -0,0 +1,103 @@ +import hashlib +import uuid +import zipfile + +import pytest +from lxml import etree + +from src.edition.fonts import ADOBE, IDPF, _key, decoded_fonts, primary_identifier, rebind_fonts + +OPF = 'http://www.idpf.org/2007/opf' + + +def key(identifier, algorithm): + if algorithm == IDPF: + return hashlib.sha1(''.join(identifier.split()).encode()).digest() + return uuid.UUID(identifier.removeprefix('urn:uuid:')).bytes + + +def xor(data, identifier, algorithm): + k = key(identifier, algorithm); n = 1040 if algorithm == IDPF else 1024 + return bytes(b ^ k[i % len(k)] for i, b in enumerate(data[:n])) + data[n:] + + +def fixture(tmp_path, algorithm): + old = 'urn:uuid:' + str(uuid.UUID('12345678-1234-5678-1234-567812345678')) + new = 'urn:uuid:' + str(uuid.UUID('87654321-4321-8765-4321-876543218765')) + plain = b'\x00\x01\x00\x00' + bytes(range(256)) * 20 + encrypted = xor(plain, old, algorithm) + opf = f'''{old}''' + enc = f'''''' + path = tmp_path / 'book.epub' + with zipfile.ZipFile(path, 'w') as z: + z.writestr('EPUB/package.opf', opf) + z.writestr('META-INF/encryption.xml', enc) + z.writestr('EPUB/fonts/a.ttf', encrypted) + z.writestr('EPUB/styles/main.css', b'@font-face {src:url("./../fonts/%61.ttf")} /* url(https://example.org/fonts/a.ttf) */') + z.writestr('EPUB/text/ch.xhtml', '

x

') + root = etree.fromstring(opf.encode()) + return path, root, old, new, plain, encrypted + + +@pytest.mark.parametrize('algorithm', [IDPF, ADOBE]) +def test_decode_and_rebind_preserves_original_and_rekeys_css_urls(tmp_path, algorithm): + path, root, old, new, plain, encrypted = fixture(tmp_path, algorithm) + with zipfile.ZipFile(path) as z: + assert primary_identifier(root) == old + assert decoded_fonts(z, 'EPUB/package.opf', root) == {'EPUB/fonts/a.ttf': plain} + additions, replacements = {}, {} + records = rebind_fonts(z, 'EPUB/package.opf', root, old, new, additions, replacements) + assert len(records) == 1 and records[0]['algorithm'] == algorithm + assert records[0]['original_sha256'] == hashlib.sha256(encrypted).hexdigest() + assert records[0]['decoded_sha256'] == hashlib.sha256(plain).hexdigest() + assert additions['EPUB/fonts/edition-a.ttf'] == xor(plain, new, algorithm) + assert z.read('EPUB/fonts/a.ttf') == encrypted + assert root.xpath('//*[local-name()="item" and @id="font"]/@href') == ['fonts/edition-a.ttf'] + assert root.xpath('//*[local-name()="item" and @id="font-source"]/@media-type') == ['application/octet-stream'] + assert b'../fonts/edition-a.ttf' in replacements['EPUB/styles/main.css'] + assert b'../fonts/edition-a.ttf' in replacements['EPUB/text/ch.xhtml'] + assert b'https://example.org/fonts/a.ttf' in replacements['EPUB/styles/main.css'] + assert b'edition-a.ttf' in replacements['EPUB/text/ch.xhtml'] + enc = etree.fromstring(replacements['META-INF/encryption.xml']) + assert enc.xpath('//*[local-name()="CipherReference"]/@URI') == ['EPUB/fonts/edition-a.ttf'] + # The replacement payload decodes under the new primary identifier. + root.xpath('//*[local-name()="identifier" and @id="uid"]')[0].text = new + replacements['EPUB/package.opf'] = etree.tostring(root) + class ZipView: + def read(self, name): + if name in replacements: return replacements[name] + return additions[name] if name in additions else z.read(name) + def namelist(self): return z.namelist() + list(additions) + view = ZipView() + decoded = decoded_fonts(view, 'EPUB/package.opf', root) + assert decoded['EPUB/fonts/edition-a.ttf'] == plain + + +def test_rejects_unknown_algorithm_and_non_font_payload(tmp_path): + path, root, old, new, plain, encrypted = fixture(tmp_path, 'urn:unknown') + with zipfile.ZipFile(path) as z, pytest.raises(ValueError, match='Unsupported EPUB encryption algorithm'): + decoded_fonts(z, 'EPUB/package.opf', root) + path, root, old, new, plain, encrypted = fixture(tmp_path, IDPF) + with zipfile.ZipFile(path, 'w') as z: + z.writestr('EPUB/package.opf', f'''{old}''') + z.writestr('META-INF/encryption.xml', f'''''') + z.writestr('EPUB/fonts/a.ttf', xor(b'not-a-font'+bytes(1200), old, IDPF)) + root = etree.fromstring(f'''{old}'''.encode()) + with zipfile.ZipFile(path) as z, pytest.raises(ValueError, match='recognized font signature'): + decoded_fonts(z, 'EPUB/package.opf', root) + + +def test_rejects_unsafe_cipher_path(tmp_path): + path, root, old, new, plain, encrypted = fixture(tmp_path, IDPF) + with zipfile.ZipFile(path, 'w') as z: + z.writestr('EPUB/package.opf', f'''{old}''') + z.writestr('META-INF/encryption.xml', f'''''') + root = etree.fromstring(f'''{old}'''.encode()) + with zipfile.ZipFile(path) as z, pytest.raises(ValueError, match='escapes EPUB root'): + decoded_fonts(z, 'EPUB/package.opf', root) + + +def test_idpf_whitespace_removal_is_limited_to_xml_whitespace(): + assert _key('ab c\td\re\nf', IDPF) == hashlib.sha1(b'abcdef').digest() + # Unicode whitespace is not part of XML's four whitespace code points. + assert _key('ab\u00a0c', IDPF) == hashlib.sha1('ab\u00a0c'.encode()).digest() diff --git a/tests/test_navigation.py b/tests/test_navigation.py index a51b97b..5eebfb3 100644 --- a/tests/test_navigation.py +++ b/tests/test_navigation.py @@ -315,3 +315,15 @@ def test_nested_toc_anchor_binds_heading_and_normalizes_visible_ordinal(): # A nested anchor in ordinary prose must still not become a title. assert not _headings(parse_html('

A sentence.

'), 'chapter.xhtml#prose', 'Chapter 1. Obsession')[0] + + +def test_graphical_title_binds_using_alt_without_mutating_image(): + from src.common.html import parse_html + from src.preprocessing.navigation import _headings + soup = parse_html('

CHAPTER 1' + 'Protocols.' + 'for SLEEP

') + before = str(soup) + headings, reason = _headings(soup, 'ch.xhtml#ch', 'Chapter 1: Protocols for Sleep') + assert reason is None and [h['id'] for h in headings] == ['heading'] + assert str(soup) == before diff --git a/tests/test_regressions.py b/tests/test_regressions.py index 2eb0272..f1b13bc 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -381,3 +381,11 @@ def test_quota_failure_preserves_completed_translation_and_checkpoint(): assert entries[0].translated_text == '已完成。' assert not any(e.translated_text for e in entries[1:]) assert snapshots and snapshots[-1][0] == '已完成。' + + +def test_styled_protocol_number_cannot_be_deleted(): + e = entry('

Protocol 3 Title

') + missing = e.original_text.replace('3φ/', 'φ/') + assert FormatRestorer().restore(missing, e.placeholders, original_text=e.original_text) == ('', False) + localized = e.original_text.replace('3φ/', '三φ/') + assert FormatRestorer().restore(localized, e.placeholders, original_text=e.original_text)[1]