From cad96d59c8e082bf7ccbb78311b60d0f569aad0c Mon Sep 17 00:00:00 2001 From: Kai Tan Date: Wed, 28 Oct 2026 23:28:40 +0800 Subject: [PATCH] Add lightweight standalone PDF to EPUB import and source-page QA --- CHANGELOG.md | 9 ++ README.md | 15 +- doc/OCR_EPUB_ROADMAP.md | 2 +- doc/PDF_IMPORT.md | 90 ++++++++++++ doc/QA_PDF_IMPORT_2026-10-28.md | 60 ++++++++ requirements-pdf.txt | 8 + scripts/pdf_qa.py | 67 +++++++++ scripts/pdf_to_epub.py | 30 ++++ src/pdf_import/__init__.py | 1 + src/pdf_import/converter.py | 253 ++++++++++++++++++++++++++++++++ src/pdf_import/layout.py | 78 ++++++++++ src/pdf_import/ocr.py | 92 ++++++++++++ src/pdf_import/vision.swift | 35 +++++ tests/test_pdf_import.py | 145 ++++++++++++++++++ 14 files changed, 883 insertions(+), 2 deletions(-) create mode 100644 doc/PDF_IMPORT.md create mode 100644 doc/QA_PDF_IMPORT_2026-10-28.md create mode 100644 requirements-pdf.txt create mode 100644 scripts/pdf_qa.py create mode 100644 scripts/pdf_to_epub.py create mode 100644 src/pdf_import/__init__.py create mode 100644 src/pdf_import/converter.py create mode 100644 src/pdf_import/layout.py create mode 100644 src/pdf_import/ocr.py create mode 100644 src/pdf_import/vision.swift create mode 100644 tests/test_pdf_import.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b1f774e..67b47a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## 2026-10-28 — lightweight standalone PDF import preview + +- Add PDFium/Pillow PDF import with native text, macOS system Vision OCR and an optional Tesseract adapter; no translation or heavyweight model dependency required. +- Support Chinese reflow directly and English EPUB export for the existing translation pipeline, without automatically starting paid translation. +- Cache source-bound page recognition, retain coordinates and raw OCR, reconstruct single-column paragraphs across pages, and preserve bookmark hierarchy and source-page anchors. +- Preserve selected illustration/title/back-cover pages, export digital PDF image regions, and apply source-bound exact OCR corrections without rewriting raw checkpoints. +- Add PDF/source-anchor desktop and narrow-screen QA with actual NAV clicks; validate all 476 pages and 87 bookmark targets on the user-provided sample. +- Keep output explicitly marked as an OCR reflow draft; complex layouts, scanned-page figure segmentation and footnote reconstruction remain outside this release. + ## 2026-10-10 — Highest Exam edition metadata - Preserve OPF metadata XML comments while replacing edition values and their refinements. diff --git a/README.md b/README.md index 54fefe5..13473cc 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ Mac 沙箱中 Calibre 的 Qt 渲染可能需要允许进程运行。QA 使用项 无法可靠绑定的目录条目仅翻译标签，不改写猜测的章节正文；具体条目记录在 `book_structure.json` 的 `navigation_titles[].binding_note`。这类条目需要检查章节标题是否一致，不能视为全书标题同步已经通过。索引、版权、注释全文仍遵守原有跳过策略；导航中这些页面的名称可以翻译。 -后续扫描 PDF/OCR 大版本的图文、结构与目录方案见 [路线图](doc/OCR_EPUB_ROADMAP.md)。 +轻量 PDF/OCR 首版见 [使用说明](doc/PDF_IMPORT.md)；后续困难版式见 [路线图](doc/OCR_EPUB_ROADMAP.md)。 ### 中文封面与书库 metadata（v0.14） @@ -149,3 +149,16 @@ Mac 沙箱中 Calibre 的 Qt 渲染可能需要允许进程运行。QA 使用项 路径相对于 JSON 文件；常规译本保留，另生成 `edition_*.epub`。缺少已审封面会报错，不自动使用另一个版本的图片。基础封装不新增依赖、无网络请求；生图和书目检索由 skill 编排。 2026-10-08 接入核实：本机 Codex 使用 ChatGPT 登录，缓存目录没有可选择的 image 模型。官方模型目录提供 `gpt-image-2.5-sunburst`／`gpt-image-2.5-flare`，但[ChatGPT OAuth API 路径暂不支持图片生成](https://developers.openai.com/siwc/token-sharing-open-source/preview-limitations)。本轮实际使用 Codex 内置 imagegen 成功生成两版；该工具不公开模型 ID／选择参数，因此记录 `model: null`，不能宣称使用 GPT-Image-2.5。后续运行先检查当时的官方支持情况和工具能力；不要读取或转用私有 OAuth token 绕过限制，也不自动改用付费 API。 + +### 轻量 PDF → EPUB（独立模块） + +新增 `scripts/pdf_to_epub.py`，仅安装 `requirements-pdf.txt` 即可独立使用。中文 PDF 直接重排；英文 PDF 先导出 EPUB，再进入现有翻译流程，转换阶段不调用翻译 API。支持文本层、macOS 系统 Vision OCR、可选 Tesseract、逐页恢复、原书签目录和来源锚点。首版限定横排单栏；样本已跑通476页全书，输出仍标为 OCR 重排初稿。 + +```bash +.venv/bin/pip install -r requirements-pdf.txt +.venv/bin/python scripts/pdf_to_epub.py input/book.pdf \ + --output output/pdf/book.epub --work .work/pdf-book \ + --language zh-CN --title '书名' --author '作者' +``` + +独立环境、英文翻译衔接、图片页保留和视觉QA见 [PDF导入使用说明](doc/PDF_IMPORT.md)。 diff --git a/doc/OCR_EPUB_ROADMAP.md b/doc/OCR_EPUB_ROADMAP.md index 568ef39..56e961d 100644 --- a/doc/OCR_EPUB_ROADMAP.md +++ b/doc/OCR_EPUB_ROADMAP.md @@ -1,6 +1,6 @@ # 扫描 PDF → EPUB：后续大版本 -这是独立的大版本方向，本轮不实现。目标是把旧中文书的扫描 PDF 转成可重排、可搜索、可点击目录的 EPUB，同时保留原书中有阅读意义的结构与图文关系。 +2026-10-28 已开始独立模块首版，见 [PDF_IMPORT.md](PDF_IMPORT.md)。首版优先轻量方案和横排单栏；以下困难版式仍属后续范围。目标是把旧中文书的扫描 PDF 转成可重排、可搜索、可点击目录的 EPUB，同时保留原书中有阅读意义的结构与图文关系。 ## 输入与评测样本 diff --git a/doc/PDF_IMPORT.md b/doc/PDF_IMPORT.md new file mode 100644 index 0000000..ca8549b --- /dev/null +++ b/doc/PDF_IMPORT.md @@ -0,0 +1,90 @@ +# 轻量 PDF → EPUB 独立模块 + +首版只处理横排单栏、普通段落和简单图片。中文 PDF 转为中文重排 EPUB；英文 PDF 转为英文 EPUB，然后由现有翻译程序处理。PDF 模块不导入翻译客户端，不读取 API Key，不发起翻译请求。 + +## 安装和后端 + +只使用 PDF 模块时，可单独建环境，不安装原项目的翻译 API 依赖： + +```bash +python3 -m venv .work/pdf-env +.work/pdf-env/bin/pip install -r requirements-pdf.txt +``` + +当前项目环境也可以直接安装该文件。相对已有项目，新增 PDF 相关依赖仅 `pypdfium2`、`Pillow`；独立安装还包含 EPUB 封装／结构审计所需的 EbookLib、lxml、BeautifulSoup、html5lib。没有 Docling、MinerU、Torch 或额外模型权重。 + +- `--ocr auto`：正常文本 PDF 使用文本层；全页扫描图片重新 OCR，即使已有隐藏 OCR 层也不直接相信旧结果。Mac 默认使用系统 Apple Vision；其他系统使用可选 Tesseract。 +- `--ocr native`：只用已有文本层。扫描页会记录警告，不能把已有 OCR 质量视为已验证。 +- `--ocr vision`：强制系统 Vision，需要 macOS 和 Apple command line tools 的 `swiftc`。辅助程序在工作目录编译并复用，本地处理图片。Codex 沙箱内 Vision 如报 nilError，应允许本地 OCR 进程在沙箱外执行；不是 API 认证错误。 +- `--ocr tesseract`：强制可选 Tesseract CLI；需自行安装程序以及 `eng`／`chi_sim` 语言数据。适配器已实现，本轮机器没有安装 Tesseract，未进行真实 Tesseract 识别验收。 + +官方依据：[PDFium Python API](https://pypdfium2.readthedocs.io/en/stable/python_api.html)、[Apple Vision](https://developer.apple.com/documentation/vision/recognizing-text-in-images)、[Tesseract TSV](https://tesseract-ocr.github.io/tessdoc/Command-Line-Usage.html)。PDFium 在本模块中串行调用，避免跨线程调用原生库。 + +## 中文 PDF 直接重排 + +```bash +.work/pdf-env/bin/python scripts/pdf_to_epub.py input/chinese.pdf \ + --output output/pdf/chinese.epub --work .work/pdf-chinese \ + --language zh-CN --title '书名' --author '作者' --cover-page 1 +``` + +中文正文使用两格缩进；保留原语言，不做繁简转换或自由改写。输出 metadata 明确标注“PDF重排初稿”，不误标为 AI 翻译版。 + +## 英文 PDF → EPUB → 翻译 + +```bash +.work/pdf-env/bin/python scripts/pdf_to_epub.py input/english.pdf \ + --output output/pdf/english.epub --work .work/pdf-english \ + --language en --title 'Original Title' --author 'Original Author' + +# 在已有翻译环境中执行，沿用它的模型、凭据和配置： +.venv/bin/python main.py output/pdf/english.epub --complete-book --bilingual +``` + +PDF 转换完成只打印可选的后续翻译命令，不执行。英文原生文本 PDF 已通过真实 PDF 夹具转换，并验证可由现有 EpubCleaner／FineGrainedExtractor 完整提取；没有调用模型翻译此夹具。 + +## 小样、图片与恢复 + +- `--pages 1-20,30`：按1起算、闭区间选页；未覆盖全书时，书名和报告明确标注部分样例。 +- `--cover-page 1`：显式指定封面，不默认把所有 PDF 的第一页当封面。 +- `--image-pages 2,13,476`：手动保留插图、扉页、原目录或封底整页原貌。少量文字／空白页也会保留为原页图片并列入报告，不声称所有页均已文字重排。 +- `--dpi 180`：扫描渲染分辨率，支持96–400。 +- 同一工作目录一次运行一个任务。再次运行相同命令会复用完成的逐页 OCR；源 SHA、引擎、OCR 实现／系统版本、语言或 DPI 变化时不复用不匹配结果。另一份源文件必须使用新目录。 + +工作目录保留 `source.json`、`pages/` 原始识别／坐标／置信度、`renders/` 原页图、`book.json` 段落与书签结构、`report.json`。OCR 失败保留已完成页面，禁止把仅完成部分的处理结果写成全书 EPUB。 + +EPUB 按原书签组织章节和目录层级；没有书签时使用识别出的简单章节标题，或明确的来源页导航。每个原页都有 `pdf-page-N` 来源锚点；跨页仅合并明显连续的正文，保留原页追踪。页眉删除记录和原 OCR 行仍在中间结果中。 + +普通文本 PDF 内的独立图片按页面区域渲染嵌入；扫描页中的混合图文首版不做自动精细分割，需要 `--image-pages` 保留原页。双栏、竖排、公式、复杂表格、脚注双向链接重建不在本轮范围，不作为已实现功能。 + +## QA + +可选安装现有 `requirements-qa.txt` 和 Chromium 后： + +```bash +PLAYWRIGHT_BROWSERS_PATH="$PWD/.work/playwright" \ +.venv/bin/python scripts/pdf_qa.py output/pdf/chinese.epub \ + --work .work/pdf-chinese --pages 1,4,16 --out .work/pdf-chinese/qa +``` + +生成原页／EPUB 900px／390px 对照，实际点击生成的 NAV 目录链接，并检查目标及来源锚点。HTML 报告和 JSON 分开保留自动结果与视觉待审状态。结构审计通过不等于 OCR 字词准确；Vision 置信分数不是字错率，也不能据此给全书准确率。 + +EPUB 在临时文件中构建并通过结构审计后才替换输出。没有依赖 Calibre 或桌面阅读器；Chromium 连续 XHTML 不能代表所有阅读器分页。 + +## 可追溯的局部校正 + +`--corrections corrections.json` 支持查看原扫描后确认的逐行校正： + +```json +{ + "source_sha256": "当前PDF的完整SHA256", + "edits": [{ + "line_id": "p0101-l0011", + "expected": "原始OCR整行原文", + "replacement": "核对原扫描后的整行文字", + "evidence": "核对依据" + }] +} +``` + +源 SHA、行 ID 或 expected 不匹配就停止；重复条目和空替换也拒绝。原始 `pages/*.json` 不改写，校正写在派生 `book.json` 和报告里，可在重建时重复应用。识别内容变化会改变输出版本标识。此功能用于有证据的文字校对，不替代全书审读。 diff --git a/doc/QA_PDF_IMPORT_2026-10-28.md b/doc/QA_PDF_IMPORT_2026-10-28.md new file mode 100644 index 0000000..640f42b --- /dev/null +++ b/doc/QA_PDF_IMPORT_2026-10-28.md @@ -0,0 +1,60 @@ +# 轻量 PDF 导入首版与样本验收 + +按本轮收窄的范围开发：不引入 Docling 或大型模型，不实现困难版式。新增独立 `src/pdf_import/`、`scripts/pdf_to_epub.py` 和可选 `scripts/pdf_qa.py`。中文直接重排为 EPUB；英文先生成英文 EPUB，再进入已有翻译程序。转换阶段不调用翻译 API。 + +## 已实现与依赖 + +新增 PDF 依赖为 PDFium Python 包与 Pillow。独立环境只装 `requirements-pdf.txt`，包含 EPUB 导出／审计的轻量基础库；实际验证未加载 OpenAI SDK、Torch、Transformers、Docling 或 Paddle。macOS 使用系统 Vision；Swift 辅助进程常驻批量识别，未额外下载 OCR 模型权重。Tesseract 适配器可选，本机未安装程序，未做真实 Tesseract 验收。 + +原生文本提取与扫描页识别分流，扫描页即使已有隐藏 OCR 层也重新识别。页面缓存按源 SHA、页号、实际引擎／实现与系统版本、语言、DPI 和 PDFium 版本匹配。失败只保留已完成页面；未完成全书时不写出冒充全书的结果。部分选页输出有明确样例标签。 + +根据来源坐标重建普通段落，修复首轮发现的页眉混入正文与连续缩进段落被逐行拆开的情况；明显的跨页连续正文合并，原页锚点保留。书签层级生成 NAV／NCX。保留指定整页图，数字 PDF 的独立图片按页面区域渲染嵌入。构建临时 EPUB，结构审计通过后才替换最终输出。 + +## 样本 + +用户指定下载目录中的《中国的世界高原 上》，沈卫平。476页，无密码；抽查为横排单栏扫描，已有 ABBYY 文本层，但可见漏字，不能当作可靠原文。 + +源 SHA256：`aebc9f263ab0599f6d11c7db6c5e89e1316f2a950f2faa9ddbc72a94fe3da335`。 + +```bash +.venv/bin/python scripts/pdf_to_epub.py \ + '/Users/kaitan/Downloads/中国的世界高原 上 (沈卫平) (z-library.sk, 1lib.sk, z-lib.sk).pdf' \ + --output 'output/pdf/中国的世界高原_上_重排初稿.epub' \ + --work .work/pdf-highland --language zh-CN \ + --title '中国的世界高原（上）' --author 沈卫平 \ + --cover-page 1 --image-pages 2,13,476 \ + --corrections .work/pdf-highland/corrections.json +``` + +476/476页处理，476个来源页锚点均存在。459页为重排文字，17页为封面、扉页、空白／短文字页等原页图片；报告逐项列明。第2、13、476页为显式指定保留原貌的扉页、纸本目录和带签名二维码的封底；其余少量空白／短文字页自动保留原页图，不称为可搜索的重排文字。 + +原87个书签对应87个有效目录目标，层级为10个一级、77个二级。浏览器实际点击87个 NAV 链接，目标均到达。509行页眉／页码候选从派生正文移除，原识别行和删除记录仍保存。 + +## 对照 QA 与修复 + +以原页序1、4、16、101、248、476生成原扫描／900px／390px对照，检查封面、普通段落、跨页衔接、引文和封底保留。自动结构审计零问题，抽样自动布局检查零问题。实际查看的图片及 SHA 写入 `.work/pdf-highland/review-record.json`。 + +原页对照确认第101页两行中的三个字形识别错误，使用带源 SHA、稳定行 ID、expected 原文和 replacement 的校正文件修正；原始逐页 OCR JSON 不变。校正不匹配或重复时拒绝应用，不做全书无证据替换。 + +OCR 准确率仍待全文校对，未计算人工真值字错率。Vision 的低置信提示不是准确率百分比。输出 metadata 明确标为“PDF重排初稿”；没有宣称所有文字或每个阅读器分页已验收。 + +## 独立与英文路径验证 + +在新环境 `.work/pdf-light-env` 仅安装 `requirements-pdf.txt`，实际将两页英文原生文本 PDF 夹具转换为 EPUB，并打印后续翻译命令，没有加载翻译 SDK 或发起翻译。另用现有 EpubCleaner 和 FineGrainedExtractor 验证生成英文 EPUB 可进入现有提取链路。后续翻译在原项目 `.venv` 中执行，沿用其模型与凭据选择。 + +测试：98 passed，2 skipped。新增覆盖页范围、页眉误识别、英文提取衔接、扫描页覆盖旧文本层、缓存复用／失效、失败恢复与禁止部分全书输出、连续缩进段落及源绑定校正。 + +## 范围限制 + +- 首版针对横排单栏普通书；没有实现双栏、竖排正文、公式、复杂表格或脚注双向链接重建。 +- 扫描页混合插图不自动精细切分；可指定整页保留。并非所有字体、粗体及页面坐标都被重建。 +- 来源锚点使用PDF文件页序，不冒充纸本页码。纸本目录页保留原图，交互目录由实际 PDF 书签生成。 +- headless 为连续 XHTML，不代表 Calibre、Apple Books 等所有阅读器的分页。全书文字校对没有完成。 + +## 本地证据 + +`.work/pdf-highland/`：`source.json`、原始 `pages/`、`renders/`、`book.json`、`corrections.json`、`report.json`、`qa-final/review.html`、`qa-final/report.json` 和 SHA 绑定的 `review-record.json`。原PDF不修改，书籍和识别文字不提交仓库。 + +输出：`output/pdf/中国的世界高原_上_重排初稿.epub`。 + +输出 SHA256：`9e2bece683420803d59292aa49fb44ee6e1cd002331364af7380ba0a5971b71d`。 diff --git a/requirements-pdf.txt b/requirements-pdf.txt new file mode 100644 index 0000000..f8a0640 --- /dev/null +++ b/requirements-pdf.txt @@ -0,0 +1,8 @@ +# Optional PDF conversion only; no model weights or translation SDK required. +pypdfium2>=5,<6 +Pillow>=10,<13 +# Minimal EPUB export/audit dependencies (the translation API stack is optional). +ebooklib>=0.19,<1 +lxml>=4.9 +beautifulsoup4>=4.12 +html5lib>=1.1 diff --git a/scripts/pdf_qa.py b/scripts/pdf_qa.py new file mode 100644 index 0000000..8ce85b2 --- /dev/null +++ b/scripts/pdf_qa.py @@ -0,0 +1,67 @@ +"""Compare cached source PDF pages and actual EPUB source anchors locally.""" +import argparse +import hashlib +import json +import shutil +import sys +from pathlib import Path +from urllib.parse import quote +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('epub',type=Path);parser.add_argument('--work',type=Path,required=True) + parser.add_argument('--pages',default='1,4,16');parser.add_argument('--out',type=Path,required=True) + args=parser.parse_args() + from src.qa.headless import archive_server,book_inventory,LAYOUT_JS + from src.qa.epub_audit import audit_epub + from playwright.sync_api import sync_playwright + data=json.loads((args.work/'book.json').read_text()) + pages={p['number']:p for p in data['pages']};selected=[int(p) for p in args.pages.split(',')] + if any(p not in pages for p in selected):parser.error('Requested page not present in the conversion IR') + args.out.mkdir(parents=True,exist_ok=True) + inventory=book_inventory(args.epub);evidence=[];clicks=[];sections=[] + with archive_server({'book':args.epub}) as base,sync_playwright() as pw: + browser=pw.chromium.launch(headless=True) + try: + context=browser.new_context(java_script_enabled=False,service_workers='block') + context.route('**/*',lambda route:route.continue_() if route.request.url.startswith(base+'/') else route.abort()) + tab=context.new_page() + for number in selected: + anchor=f'pdf-page-{number}' + document=next((name for name,soup in inventory['documents'].items() if soup.find(id=anchor)),None) + if not document:raise ValueError(f'EPUB lost source anchor {anchor}') + source_name=f'pdf-{number:04d}.jpg';shutil.copy2(pages[number]['render'],args.out/source_name) + rendered=[] + for width in [900,390]: + tab.set_viewport_size({'width':width,'height':1100}) + tab.goto(base+'/book/'+quote(document)+'#'+anchor,wait_until='networkidle') + tab.evaluate('() => document.fonts.ready') + tab.evaluate('(id) => {const e=document.getElementById(id); scrollTo(0,Math.max(0,e.getBoundingClientRect().top+scrollY-140));}',anchor) + name=f'epub-{number:04d}-{width}.png';tab.screenshot(path=str(args.out/name));rendered.append(name) + evidence.append({'source_page':number,'document':document,'viewport_width':width,'image':name,'layout_findings':tab.evaluate(LAYOUT_JS)}) + sections.append(f'

PDF 原页序 {number}

原扫描页EPUB实际排版窄屏排版
') + # Real browser navigation for every generated TOC target, including fragments. + for entry in inventory['toc']: + target=entry['target'];name,_,fragment=target.partition('#') + nav=entry['file'] + nav_soup=inventory['documents'].get(nav) + if nav_soup is not None: + from src.qa.headless import destination + anchors=nav_soup.find_all('a',href=True) + link_index=next(i for i,a in enumerate(anchors) if destination(nav,a['href'])==target) + tab.goto(base+'/book/'+quote(nav),wait_until='domcontentloaded') + tab.locator('a[href]').nth(link_index).click(timeout=5000) + else: + tab.goto(base+'/book/'+quote(name)+(('#'+quote(fragment)) if fragment else ''),wait_until='domcontentloaded') + reached=(tab.url.split('#')[0]==base+'/book/'+quote(name)) and (not fragment or tab.evaluate('(id) => !!document.getElementById(id)',fragment)) + clicks.append({'target':target,'destination_reached':reached}) + finally:browser.close() + report={'source_sha256':data['source_sha256'],'epub_sha256':hashlib.sha256(args.epub.read_bytes()).hexdigest(),'epub_audit':audit_epub(args.epub),'evidence':evidence,'toc_navigation_checks':clicks,'visual_review':'pending','ocr_accuracy_review':'pending','limits':'Chromium continuous XHTML, not all reader pagination; source anchor may be inside a continued paragraph.'} + (args.out/'report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n') + (args.out/'review.html').write_text('PDF → EPUB QA

PDF → EPUB 对照

左：原扫描；中：EPUB；右：窄屏。截图生成不等于视觉或OCR验收。

'+''.join(sections)) + print(f'PDF source comparisons: {len(selected)}; TOC target navigation: {len(clicks)}') + return int(bool(report['epub_audit']['issues']) or any(not c['destination_reached'] for c in clicks) or any(e['layout_findings'] for e in evidence)) + +if __name__=='__main__':sys.exit(main()) diff --git a/scripts/pdf_to_epub.py b/scripts/pdf_to_epub.py new file mode 100644 index 0000000..503f6af --- /dev/null +++ b/scripts/pdf_to_epub.py @@ -0,0 +1,30 @@ +"""Independent lightweight PDF import. English output can enter the EPUB translator.""" +import argparse +import json +import shlex +import sys +from pathlib import Path +sys.path.insert(0,str(Path(__file__).resolve().parents[1])) +from src.pdf_import.converter import convert + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('source',type=Path);parser.add_argument('--output',required=True,type=Path) + parser.add_argument('--work',required=True,type=Path);parser.add_argument('--language',choices=['zh-CN','en'],required=True) + parser.add_argument('--title');parser.add_argument('--author');parser.add_argument('--pages',help='1-based inclusive selection, e.g. 1-30,40; output is explicitly partial') + parser.add_argument('--ocr',choices=['auto','native','vision','tesseract'],default='auto') + parser.add_argument('--dpi',type=int,default=180);parser.add_argument('--cover-page',type=int);parser.add_argument('--image-pages',help='Preserve chosen illustration/title/back-cover pages as images, e.g. 2,13,476') + parser.add_argument('--corrections',type=Path,help='Source SHA-bound, exact OCR line corrections JSON') + args=parser.parse_args() + try: + report=convert(args.source,args.output,args.work,args.language,args.title,args.author,args.ocr,args.pages,args.dpi,args.cover_page,image_pages=args.image_pages,corrections=args.corrections) + except (ValueError,RuntimeError,ImportError,OSError) as exc: + parser.exit(1,f'PDF import failed: {exc}\n') + print(json.dumps({k:report[k] for k in ['output','partial','processed_pages','source_pages','engines']},ensure_ascii=False)) + if args.language=='en': + translation_python=Path(__file__).resolve().parents[1]/'.venv/bin/python' + print('Optional next step in the translation environment (not executed): '+shlex.join([str(translation_python) if translation_python.exists() else 'python',str(Path(__file__).resolve().parents[1]/'main.py'),str(args.output.resolve()),'--complete-book','--bilingual'])) + return 0 + +if __name__=='__main__':sys.exit(main()) diff --git a/src/pdf_import/__init__.py b/src/pdf_import/__init__.py new file mode 100644 index 0000000..65b79cd --- /dev/null +++ b/src/pdf_import/__init__.py @@ -0,0 +1 @@ +"""Optional lightweight PDF import, independent of the translation pipeline.""" diff --git a/src/pdf_import/converter.py b/src/pdf_import/converter.py new file mode 100644 index 0000000..d170936 --- /dev/null +++ b/src/pdf_import/converter.py @@ -0,0 +1,253 @@ +"""PDF -> source-traceable, reflowable EPUB. No translation calls or heavy models.""" +import hashlib +import html +import json +import platform +import re +import sys +import uuid +from pathlib import Path +from ebooklib import epub +from .layout import paragraphs, repeated_furniture, join_text + +SCHEMA = 1 + + +def digest(path): + h = hashlib.sha256() + with Path(path).open('rb') as f: + for chunk in iter(lambda:f.read(1024*1024), b''): h.update(chunk) + return h.hexdigest() + + +def save_json(path, data): + path = Path(path); path.parent.mkdir(parents=True, exist_ok=True) + temp = path.with_suffix('.tmp'); temp.write_text(json.dumps(data, ensure_ascii=False, indent=2)+'\n'); temp.replace(path) + + +def page_selection(spec, count): + if not spec: return list(range(count)) + selected = set() + for item in spec.split(','): + if not re.fullmatch(r'\d+(?:-\d+)?', item): raise ValueError('Pages must look like 1-20,25 (1-based inclusive)') + pair = [int(x) for x in item.split('-')]; lo, hi = pair[0], pair[-1] + if not 1 <= lo <= hi <= count: raise ValueError('Page range is outside the source PDF') + selected.update(range(lo-1, hi)) + return sorted(selected) + + +def native_lines(page): + text = page.get_textpage() + try: + lines=[]; width,height=page.get_size() + for i in range(text.count_rects()): + left,bottom,right,top=text.get_rect(i) + value=text.get_text_bounded(left,bottom,right,top).strip() + if value: + lines.append({'text':value,'bbox':[left/width,1-top/height,right/width,1-bottom/height], 'confidence':None}) + return lines + finally: text.close() + + +def image_boxes(page): + import pypdfium2.raw as raw + w,h=page.get_size(); boxes=[] + for obj in page.get_objects(filter=[raw.FPDF_PAGEOBJ_IMAGE]): + left,bottom,right,top=obj.get_bounds() + boxes.append([max(0,left/w), max(0,1-top/h), min(1,right/w), min(1,1-bottom/h)]) + return boxes + + +def validate_lines(lines): + if not isinstance(lines,list): raise ValueError('OCR lines must be an array') + for line in lines: + box=line.get('bbox',[]) + if not isinstance(line.get('text'),str) or len(box)!=4 or not all(isinstance(v,(int,float)) and 0<=v<=1 for v in box) or box[0]>=box[2] or box[1]>=box[3]: + raise ValueError('Invalid OCR text or normalized top-left bounding box') + + +def apply_corrections(pages, correction, source_sha): + """Source-bound exact edits; never overwrite the raw OCR checkpoint.""" + if correction.get('source_sha256')!=source_sha or not isinstance(correction.get('edits'),list): + raise ValueError('Corrections must be bound to this PDF SHA and contain an edits array') + lines={line['id']:line for page in pages for line in page['lines']};applied=[];seen=set() + for edit in correction['edits']: + key=edit.get('line_id');line=lines.get(key) + if key in seen or line is None or line['text']!=edit.get('expected') or not isinstance(edit.get('replacement'),str) or not edit['replacement'].strip(): + raise ValueError('Stale, duplicate or invalid correction: '+str(key)) + line['text']=edit['replacement'];seen.add(key);applied.append(edit) + return applied + + +def convert(source, output, work, language='zh-CN', title=None, author=None, + engine='auto', pages=None, dpi=180, cover_page=None, ocr_timeout=120, image_pages=None, corrections=None): + import pypdfium2 as pdfium + source,output,work=Path(source),Path(output),Path(work) + if not source.is_file(): raise ValueError('PDF source does not exist') + if source.resolve()==output.resolve(): raise ValueError('Output must not overwrite source') + if output.suffix.lower()!='.epub': raise ValueError('Output must have .epub suffix') + if not 96<=dpi<=400: raise ValueError('DPI must be between 96 and 400') + work.mkdir(parents=True,exist_ok=True); source_sha=digest(source) + source_identity=work/'source.json' + if source_identity.exists() and json.loads(source_identity.read_text())['sha256']!=source_sha: + raise ValueError('Work directory belongs to another source; use a new directory') + save_json(source_identity,{'path':str(source.resolve()),'sha256':source_sha}) + document=pdfium.PdfDocument(source); ocr=None + try: + selected=page_selection(pages,len(document)); partial=len(selected)!=len(document) + preserved_images=set(i+1 for i in page_selection(image_pages,len(document))) if image_pages else set() + if cover_page is not None and not 1<=cover_page<=len(document): raise ValueError('Cover page outside source PDF') + backend=('vision' if platform.system()=='Darwin' else 'tesseract') if engine=='auto' else engine + outlines=[] + for bookmark in document.get_toc(): + dest=bookmark.get_dest() + if dest and dest.get_index() in selected: + outlines.append({'title':bookmark.get_title(),'page':dest.get_index()+1,'level':bookmark.level}) + results=[]; warnings=[] + for index in selected: + page=document[index] + try: + boxes=image_boxes(page) + scan=any((b[2]-b[0])*(b[3]-b[1])>.65 for b in boxes) + native=native_lines(page) if engine=='native' or (engine=='auto' and not scan) else [] + text_size=sum(len(l['text']) for l in native) + use_ocr=engine not in {'native','auto'} or (engine=='auto' and (scan or (text_size==0 and bool(boxes)))) + is_cover=index+1==cover_page + is_image=is_cover or index+1 in preserved_images + if use_ocr and not is_image and ocr is None: + from .ocr import VisionOCR,TesseractOCR + ocr=(VisionOCR if backend=='vision' else TesseractOCR)(work,language,ocr_timeout) + used='cover-image' if is_cover else 'page-image' if is_image else backend if use_ocr else 'native' + stamp={'source':source_sha,'page':index+1,'engine':used,'ocr_identity':ocr.identity if use_ocr and not is_image else None,'language':language,'dpi':dpi,'pdfium':str(pdfium.PYPDFIUM_INFO),'schema':SCHEMA} + cache=work/'pages'/f'{index+1:04d}.json' + cached=json.loads(cache.read_text()) if cache.exists() else None + render=work/'renders'/f'{index+1:04d}.jpg';render.parent.mkdir(exist_ok=True) + if cached and cached.get('stamp')==stamp and render.exists(): + result=cached + else: + bitmap=page.render(scale=dpi/72) + try: bitmap.to_pil().convert('RGB').save(render,quality=88) + finally: bitmap.close() + lines=[] if is_image else ocr.recognize(render) if use_ocr else native + validate_lines(lines) + for n,line in enumerate(lines): line['id']=f'p{index+1:04d}-l{n+1:04d}' + result={'number':index+1,'width':page.get_width(),'height':page.get_height(),'engine':used,'scan':scan,'lines':lines,'image_boxes':boxes,'render':str(render.resolve()),'stamp':stamp} + save_json(cache,result) + results.append(result) + print(f'PDF page {index+1}/{len(document)}: {used}, {len(result["lines"])} lines',file=sys.stderr,flush=True) + finally: page.close() + applied=apply_corrections(results,json.loads(Path(corrections).read_text()),source_sha) if corrections else [] + repeated=repeated_furniture(results) + for page in results: + blocks,removed=paragraphs(page,language,repeated) + page['blocks']=blocks;page['removed_furniture']=removed + count=sum(len(l['text']) for l in page['lines']) + page['image_fallback']=page['engine'] in {'cover-image','page-image'} or count<30 + if page['image_fallback'] and page['engine'] not in {'cover-image','page-image'}: + warnings.append({'page':page['number'],'kind':'page_image_fallback','detail':'Blank/illustrated/short-text page retained as an image; inspect source, not claimed as reflowed text.'}) + removed_ids={l['id'] for l in removed} + low=[l['id'] for l in page['lines'] if l['id'] not in removed_ids and l.get('confidence') is not None and l['confidence']<.6] + if low: warnings.append({'page':page['number'],'kind':'low_ocr_confidence','line_ids':low}) + if engine=='native' and page['scan']: warnings.append({'page':page['number'],'kind':'existing_scan_text_layer','detail':'Existing OCR text layer reused without verification.'}) + book={'schema':SCHEMA,'source_sha256':source_sha,'source':str(source.resolve()),'source_pages':len(document),'selected_pages':[i+1 for i in selected],'partial':partial,'language':language,'title':title or source.stem,'author':author or 'Unknown','outlines':outlines,'image_pages':sorted(preserved_images),'cover_page':cover_page,'pages':results,'warnings':warnings,'applied_corrections':applied,'scope':'horizontal single-column; no automatic footnote/table reconstruction'} + save_json(work/'book.json',book) + export_epub(book,output,work,cover_page) + from src.qa.epub_audit import audit_epub + audit=audit_epub(output) + report={'source_sha256':source_sha,'output_sha256':digest(output),'output':str(output.resolve()),'partial':partial,'processed_pages':len(results),'source_pages':len(document),'image_only_pages':[p['number'] for p in results if p['image_fallback']],'removed_furniture_lines':sum(len(p['removed_furniture']) for p in results),'engines':{name:sum(p['engine']==name for p in results) for name in set(p['engine'] for p in results)},'warnings':warnings,'applied_corrections':applied,'epub_audit':audit,'visual_review':'pending','ocr_accuracy_review':'pending','translation':'not requested; English EPUB can be passed to main.py separately'} + save_json(work/'report.json',report) + if audit['issues']: raise ValueError('EPUB structural audit failed; inspect report.json') + return report + finally: + if ocr: ocr.close() + document.close() + + +STYLE='''body {margin: 5%; line-height:1.65;} p {margin:.5em 0;} html[lang^="zh"] p {text-indent:2em;} h1,h2 {text-align:center;page-break-before:always;} img {max-width:100%;height:auto;} figure {margin:1em 0;text-align:center;} figcaption {font-size:.8em;} .source-page {font-size:.7em;color:#777;text-indent:0;} .image-page {text-align:center;text-indent:0;} .image-page img {max-height:90vh;width:auto;object-fit:contain;}''' + + +def export_epub(data, output, work, cover_page): + from PIL import Image + import io + book=epub.EpubBook(); identity={k:data[k] for k in ['source_sha256','language','title','author','selected_pages']} + identity['content_sha256']=hashlib.sha256(json.dumps([[p['number'],p['image_fallback'],[(b['kind'],b['text']) for b in p['blocks']]] for p in data['pages']],ensure_ascii=False).encode()).hexdigest() + book.set_identifier('urn:uuid:'+str(uuid.uuid5(uuid.NAMESPACE_URL,json.dumps(identity,sort_keys=True)))) + label='【部分样例】' if data['partial'] else '' + book.set_title(label+data['title']+'【PDF重排初稿】');book.set_language(data['language']);book.add_author(data['author']) + book.add_metadata('DC','source','urn:sha256:'+data['source_sha256']) + book.add_metadata('DC','description','PDF结构重建初稿；OCR未逐字校对。'+('仅含选定页，非完整书。' if data['partial'] else '')) + css=epub.EpubItem(uid='style',file_name='styles/book.css',media_type='text/css',content=STYLE.encode());book.add_item(css) + chapters=[]; toc_records=[]; body=[]; current=None; previous=None; previous_page=None + def finish(): + if current is not None: + current.content=(''+''.join(body)+'').encode() + current.add_link(href='../styles/book.css',rel='stylesheet',type='text/css') + book.add_item(current);chapters.append(current) + for page in data['pages']: + number=page['number']; outlines=[o for o in data['outlines'] if o['page']==number] + labels=[o['title'] for o in outlines] + if current is None or labels or page['image_fallback'] or previous_page is None or number!=previous_page+1: + finish();body=[];previous=None + current=epub.EpubHtml(title=' / '.join(labels) or f'PDF {number}',file_name=f'chapters/p{number:04d}.xhtml',lang=data['language']) + for outline in outlines: + toc_records.append((epub.Link(current.file_name+f'#pdf-page-{number}',outline['title'],f'nav-{len(toc_records)+1}'),outline['level'])) + marker=f'' + if not page['image_fallback']: + for heading in labels: body.append('

'+html.escape(heading)+'

') + if page['image_fallback']: + body.append(marker) + image=Path(page['render']).read_bytes();name=f'images/page-{number:04d}.jpg' + book.add_item(epub.EpubItem(uid=f'image-{number}',file_name=name,media_type='image/jpeg',content=image)) + body.append(f'
PDF第{number}页原貌
') + if number==cover_page: book.set_cover('images/cover.jpg',image,create_page=False) + previous=None + else: + entries=[dict(b) for b in page['blocks']] + for n,box in enumerate(page['image_boxes']): + area=(box[2]-box[0])*(box[3]-box[1]) + if .02PDF第{number}页图片');previous=None;continue + plain=re.sub(r'\s','',block['text']) + if any(plain==re.sub(r'\s','',label) or (index==0 and plain in {'序','序言','前言','自序','引言','后记','目录'} and re.sub(r'\s','',label).startswith(plain)) for label in labels):continue + # Merge only obvious prose continuation across consecutive source pages. + continuation=marker_pending and previous and not labels and previous_page==number-1 and previous['kind']=='paragraph' and block['kind']=='paragraph' and not block['indented'] and not re.search(r'[。！？.!?:：;；”’"]$',previous['text']) and body[-1].endswith('

') + if continuation: + extra=join_text(previous['text'],block['text'],data['language'])[len(previous['text']):] + body[-1]=body[-1][:-4]+marker+f''+html.escape(extra)+'

' + marker_pending=False;previous={**block,'text':join_text(previous['text'],block['text'],data['language'])};continue + if marker_pending:body.append(marker);marker_pending=False + tag='h2' if block['kind']=='heading' else 'p' + body.append(f'<{tag} id="{block["id"]}">{html.escape(block["text"])}') + if block['kind']=='heading' and not data['outlines']: + toc_records.append((epub.Link(current.file_name+'#'+block['id'],block['text'],f'nav-{len(toc_records)+1}'),0)) + previous=block + if marker_pending:body.append(marker) + previous_page=number + if page['image_fallback']: + finish();current=None;body=[];previous=None + finish() + if not toc_records:toc_records=[(epub.Link(c.file_name,c.title,f'nav-{i+1}'),0) for i,c in enumerate(chapters)] + roots=[];stack=[] + for link,level in toc_records: + node={'link':link,'children':[]} + while stack and stack[-1][0]>=level:stack.pop() + (stack[-1][1]['children'] if stack else roots).append(node);stack.append((level,node)) + def flatten(nodes):return [(node['link'],flatten(node['children'])) if node['children'] else node['link'] for node in nodes] + book.toc=flatten(roots);book.add_item(epub.EpubNcx());nav=epub.EpubNav();nav.title='目录' if data['language'].startswith('zh') else 'Contents';book.add_item(nav);book.spine=['nav']+chapters + output=Path(output);output.parent.mkdir(parents=True,exist_ok=True);temp=output.with_suffix('.tmp.epub') + try: + epub.write_epub(str(temp),book,{'epub3_pages':False}) + from src.qa.epub_audit import audit_epub + if audit_epub(temp)['issues']:raise ValueError('Generated EPUB failed structural audit; existing output retained') + temp.replace(output) + finally:temp.unlink(missing_ok=True) diff --git a/src/pdf_import/layout.py b/src/pdf_import/layout.py new file mode 100644 index 0000000..7d4ab1b --- /dev/null +++ b/src/pdf_import/layout.py @@ -0,0 +1,78 @@ +"""Conservative reconstruction for horizontal, single-column prose only.""" +import re +import statistics +from collections import Counter +from difflib import SequenceMatcher + +CJK = r'\u3400-\u9fff' + + +def clean_text(text, language): + text = re.sub(r'\s+', ' ', text).strip() + if language.startswith('zh'): + text = re.sub(r'(?<=['+CJK+r']) +(?=['+CJK+r'])', '', text) + return text + + +def join_text(left, right, language): + if not left: return right + if not right: return left + if language.startswith('zh'): + space = ' ' if left[-1].isascii() and left[-1].isalnum() and right[0].isascii() and right[0].isalnum() else '' + else: + space = '' if left.endswith('-') else ' ' + return left + space + right + + +def furniture_key(text): + return re.sub(r'[\W\d_]+', '', text).casefold() + + +def repeated_furniture(pages): + counts = Counter() + for page in pages: + counts.update({furniture_key(line['text']) for line in page['lines'] if line['bbox'][3] < .10 or line['bbox'][1] > .94}) + return {key for key, count in counts.items() if key and count >= 3} + + +def paragraphs(page, language, repeated): + lines = sorted(page['lines'], key=lambda line: (round(line['bbox'][1], 3), line['bbox'][0])) + kept, removed = [], [] + for line in lines: + text = clean_text(line['text'], language) + if not text: continue + edge = line['bbox'][3] < .10 or line['bbox'][1] > .94 + folio = len(text)<45 and bool(re.search(r'(?:^\d{1,4}\s*[|I丨!！/]|[|I丨!！/]\s*\d{1,4}$)', text) or re.match(r'^\d{3}\s+[^\d]', text)) + key=furniture_key(text) + similar_header=len(key)>=4 and any(len(other)>=4 and SequenceMatcher(None,key,other).ratio()>=.72 for other in repeated) + folio |= bool(re.search(r'\d{3}$',text)) + if edge and (folio or key in repeated or similar_header or re.fullmatch(r'[\W\d_]+', text)): + removed.append(line); continue + kept.append({**line, 'text': text}) + if not kept: return [], removed + heights = [l['bbox'][3]-l['bbox'][1] for l in kept] + height = statistics.median(heights) + lefts = sorted(l['bbox'][0] for l in kept) + baseline = lefts[max(0, len(lefts)//5)] + right = statistics.median(l['bbox'][2] for l in kept) + blocks = [] + for line_index, line in enumerate(kept): + box = line['bbox']; text = line['text'] + # This is a heuristic heading proposal, not a claim of semantic certainty. + heading = bool(re.fullmatch(r'第[一二三四五六七八九十百零〇两\d]+[章节部篇卷].{0,30}', text) or re.fullmatch(r'(?i)(chapter|part)\s+[\divxlc]+(?:\b.*)?', text)) + heading |= len(text) <= 10 and box[0] > .28 and box[2] < .75 and (bool(re.fullmatch(r'\d{1,3}', text)) or heights[line_index] > height*1.35) + indented = box[0] > baseline + max(.015, height*.7) + previous = blocks[-1] if blocks else None + indent_start=indented and (line_index==0 or box[0]>kept[line_index-1]['bbox'][0]+max(.009,height*.3)) + split = heading or previous is None or previous['kind']=='heading' or indent_start + if previous: + gap = box[1]-previous['bbox'][3] + split |= gap > height*.9 or previous['bbox'][2] < right-.10 + if split: + blocks.append({'kind':'heading' if heading else 'paragraph', 'text':text, 'bbox':list(box), 'indented':indented, 'source_page':page['number'], 'line_ids':[line['id']]}) + else: + previous['text'] = join_text(previous['text'], text, language) + previous['bbox'] = [min(previous['bbox'][0],box[0]), previous['bbox'][1], max(previous['bbox'][2],box[2]), box[3]] + previous['line_ids'].append(line['id']) + for index, block in enumerate(blocks): block['id'] = f'p{page["number"]:04d}-b{index+1:04d}' + return blocks, removed diff --git a/src/pdf_import/ocr.py b/src/pdf_import/ocr.py new file mode 100644 index 0000000..af9101b --- /dev/null +++ b/src/pdf_import/ocr.py @@ -0,0 +1,92 @@ +"""Small OCR adapters. Vision uses macOS frameworks; Tesseract is an optional CLI.""" +import csv +import hashlib +import io +import json +import platform +import selectors +import shutil +import subprocess +from pathlib import Path + + +class VisionOCR: + name = 'vision' + + def __init__(self, work, language, timeout=120): + if platform.system() != 'Darwin': + raise RuntimeError('Vision OCR requires macOS; select tesseract or native instead') + compiler = shutil.which('swiftc') + if not compiler: + raise RuntimeError('Vision OCR requires Apple command line tools (swiftc)') + source = Path(__file__).with_name('vision.swift') + # Include OS version in cache identity: framework OCR can change between releases. + self.identity = hashlib.sha256(source.read_bytes() + platform.mac_ver()[0].encode()).hexdigest() + cache = Path(work)/'vision-runtime'; cache.mkdir(parents=True, exist_ok=True) + binary = cache/('vision-' + self.identity[:16]) + if not binary.exists(): + module_cache = cache/'module-cache'; module_cache.mkdir(exist_ok=True) + built = subprocess.run([compiler, '-O', '-module-cache-path', str(module_cache), str(source), '-o', str(binary)], capture_output=True, text=True, timeout=180) + if built.returncode: + raise RuntimeError('Vision helper compilation failed: ' + built.stderr[-3000:]) + self.languages = ['zh-Hans', 'en-US'] if language.startswith('zh') else ['en-US'] + self.timeout = timeout + self.log = (cache/'runtime.log').open('a') + self.process = subprocess.Popen([str(binary)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=self.log, text=True, bufsize=1) + + def recognize(self, image): + self.process.stdin.write(json.dumps({'image': str(Path(image).resolve()), 'languages': self.languages})+'\n') + self.process.stdin.flush() + with selectors.DefaultSelector() as selector: + selector.register(self.process.stdout, selectors.EVENT_READ) + if not selector.select(self.timeout): + self.process.kill() + raise RuntimeError('Vision OCR timed out; completed page cache retained') + raw = self.process.stdout.readline() + if not raw: + raise RuntimeError('Vision OCR stopped; inspect vision-runtime/runtime.log') + result = json.loads(raw) + if result.get('error'): + raise RuntimeError('Vision OCR failed: ' + result['error']) + return result['lines'] + + def close(self): + if self.process.poll() is None: + self.process.stdin.close() + try: self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill(); self.process.wait() + self.log.close() + + +class TesseractOCR: + name = 'tesseract' + + def __init__(self, work, language, timeout=120): + self.binary = shutil.which('tesseract') + if not self.binary: + raise RuntimeError('Install tesseract and language data, or use Vision on macOS') + self.language = 'chi_sim+eng' if language.startswith('zh') else 'eng' + self.timeout = timeout + version = subprocess.run([self.binary, '--version'], capture_output=True, text=True, check=True) + self.identity = version.stdout.splitlines()[0] + ':' + self.language + + def recognize(self, image): + from PIL import Image + with Image.open(image) as picture: width, height = picture.size + output = subprocess.run([self.binary, str(image), 'stdout', '-l', self.language, 'tsv'], capture_output=True, text=True, check=True, timeout=self.timeout) + groups = {} + for word in csv.DictReader(io.StringIO(output.stdout), delimiter='\t', quoting=csv.QUOTE_NONE): + if not (word.get('text') or '').strip(): continue + key = (word['block_num'], word['par_num'], word['line_num']) + groups.setdefault(key, []).append(word) + lines = [] + for words in groups.values(): + left = min(int(w['left']) for w in words); top = min(int(w['top']) for w in words) + right = max(int(w['left'])+int(w['width']) for w in words) + bottom = max(int(w['top'])+int(w['height']) for w in words) + text = (' ' if self.language == 'eng' else '').join(w['text'] for w in words) + lines.append({'text': text, 'bbox': [left/width, top/height, right/width, bottom/height], 'confidence': min(float(w['conf'])/100 for w in words)}) + return lines + + def close(self): pass diff --git a/src/pdf_import/vision.swift b/src/pdf_import/vision.swift new file mode 100644 index 0000000..401f1fa --- /dev/null +++ b/src/pdf_import/vision.swift @@ -0,0 +1,35 @@ +import Foundation +import Vision +import ImageIO +import Darwin + +struct Input: Decodable { let image: String; let languages: [String] } +struct Line: Encodable { let text: String; let bbox: [Double]; let confidence: Float } +struct Output: Encodable { let lines: [Line]; let engine: String; let error: String? } +let decoder = JSONDecoder() +let encoder = JSONEncoder() +while let line = readLine() { + autoreleasepool { + let result: Output + do { + let job = try decoder.decode(Input.self, from: Data(line.utf8)) + let request = VNRecognizeTextRequest() + request.recognitionLevel = .accurate + request.recognitionLanguages = job.languages + request.usesLanguageCorrection = true + let handler = VNImageRequestHandler(url: URL(fileURLWithPath: job.image), options: [:]) + try handler.perform([request]) + let lines = (request.results ?? []).compactMap { observation -> Line? in + guard let candidate = observation.topCandidates(1).first else { return nil } + let r = observation.boundingBox + return Line(text: candidate.string, bbox: [r.minX, 1-r.maxY, r.maxX, 1-r.minY].map(Double.init), confidence: candidate.confidence) + } + result = Output(lines: lines, engine: "apple-vision", error: nil) + } catch { + result = Output(lines: [], engine: "apple-vision", error: String(describing: error)) + } + if let data = try? encoder.encode(result), let output = String(data: data, encoding: .utf8) { + print(output); fflush(stdout) + } + } +} diff --git a/tests/test_pdf_import.py b/tests/test_pdf_import.py new file mode 100644 index 0000000..fc0282d --- /dev/null +++ b/tests/test_pdf_import.py @@ -0,0 +1,145 @@ +"""Lightweight import: selection, provenance, OCR routing, reflow and EPUB links.""" +import json +import zipfile +from pathlib import Path +import pytest +from src.pdf_import.converter import convert, page_selection, validate_lines +from src.pdf_import.layout import paragraphs, repeated_furniture, join_text +from src.qa.epub_audit import audit_epub + + +@pytest.fixture +def native_pdf(tmp_path): + pytest.importorskip('pypdfium2') + # Minimal native-text PDF fixture; no external PDF authoring dependency. + def write(name='input.pdf', image=False): + streams=[b'BT /F1 18 Tf 60 740 Td (Chapter 1: Start) Tj 0 -40 Td /F1 12 Tf (A paragraph with enough native English text to use without OCR.) Tj ET',b'BT /F1 12 Tf 60 740 Td (This is the second page with more native English prose.) Tj ET'] + if image:streams=[b'q 600 0 0 800 0 0 cm /Im1 Do Q '+s for s in streams] + resources=b'<< /Font << /F1 7 0 R >>'+(b' /XObject << /Im1 8 0 R >>' if image else b'')+b' >>' + objects=[b'<< /Type /Catalog /Pages 2 0 R >>',b'<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>'] + objects.extend(b'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Resources '+resources+b' /Contents '+str(n).encode()+b' 0 R >>' for n in [5,6]) + objects.extend(b'<< /Length '+str(len(s)).encode()+b' >>\nstream\n'+s+b'\nendstream' for s in streams) + objects.append(b'<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>') + if image:objects.append(b'<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceRGB /BitsPerComponent 8 /Length 3 >>\nstream\n\xff\xff\xff\nendstream') + raw=b'%PDF-1.4\n';offsets=[0] + for i,obj in enumerate(objects):offsets.append(len(raw));raw+=f'{i+1} 0 obj\n'.encode()+obj+b'\nendobj\n' + xref=len(raw);raw+=f'xref\n0 {len(objects)+1}\n0000000000 65535 f \n'.encode() + raw+=b''.join(f'{n:010d} 00000 n \n'.encode() for n in offsets[1:]) + raw+=f'trailer\n<< /Size {len(objects)+1} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF'.encode() + path=tmp_path/name;path.write_bytes(raw);return path + return write + + +def test_page_ranges_do_not_silently_drop_pages(): + assert page_selection(None,3)==[0,1,2] + assert page_selection('3,1-2,2',3)==[0,1,2] + for value in ['0','3-2','1-4','-1','1,','abc']: + with pytest.raises(ValueError):page_selection(value,3) + + +def test_folio_ocr_variants_are_removed_but_body_years_survive(): + lines=[{'id':'header','text':'002 | 中国的世界系','bbox':[.15,.04,.4,.06]}, {'id':'body','text':'2019年发生的事。','bbox':[.15,.12,.8,.14]}] + blocks,removed=paragraphs({'number':5,'lines':lines},'zh-CN',set()) + assert [b['text'] for b in blocks]==['2019年发生的事。'] + assert removed[0]['id']=='header' + assert join_text('跨页','继续。','zh-CN')=='跨页继续。' + assert join_text('Some','words','en')=='Some words' + + +def test_native_english_import_is_independent_and_complete(native_pdf,tmp_path): + source=native_pdf();out=tmp_path/'book.epub';work=tmp_path/'work' + report=convert(source,out,work,language='en',engine='native') + assert report['engines']=={'native':2} and not report['partial'] + assert audit_epub(out)['issues']==[] + with zipfile.ZipFile(out) as z: + chapters=[n for n in z.namelist() if '/chapters/' in n] + assert len(chapters)==1 # Consecutive pages flow inside one reading document. + text=z.read(chapters[0]).decode() + assert 'native English' in text and 'second page' in text + assert 'pdf-page-1' in text and 'pdf-page-2' in text + assert '../styles/book.css' in text and 'lang="en"' in text + from src.preprocessing.epub_cleaner import EpubCleaner + from src.preprocessing.text_extractor import FineGrainedExtractor + from src.common.data_model import BookStructure + structure=BookStructure.load(EpubCleaner(out,tmp_path/'translation-input').clean()) + units=FineGrainedExtractor(complete_book=True).extract(structure) + assert any('native English' in unit.original_text for unit in units) + ir=json.loads((work/'book.json').read_text()) + assert len(ir['pages'])==2 and ir['pages'][0]['lines'][0]['bbox'] + partial=convert(source,tmp_path/'partial.epub',work,language='en',engine='native',pages='2') + assert partial['partial'] and partial['processed_pages']==1 + with zipfile.ZipFile(tmp_path/'partial.epub') as z: + opf=z.read('EPUB/content.opf').decode();assert '部分样例' in opf + with pytest.raises(ValueError,match='another source'): + convert(native_pdf('another.pdf',image=True),tmp_path/'other.epub',work,engine='native') + + +def test_scanned_text_layer_is_replaced_and_ocr_cache_is_reused(native_pdf,tmp_path,monkeypatch): + from src.pdf_import import ocr + import src.pdf_import.converter as converter + called=[] + class FakeOCR: + identity='fixture-v1' + def __init__(self,*args):pass + def recognize(self,image): + called.append(image) + return [{'text':'这是来自扫描图像的新识别结果，必须保留完整正文内容，不能使用已有错误文本层。','bbox':[.1,.2,.9,.25],'confidence':.9}] + def close(self):pass + monkeypatch.setattr(ocr,'VisionOCR',FakeOCR) + monkeypatch.setattr(ocr,'TesseractOCR',FakeOCR) + source=native_pdf(image=True);out=tmp_path/'scan.epub';work=tmp_path/'work' + convert(source,out,work,language='zh-CN',engine='auto') + assert len(called)==2 + convert(source,out,work,language='zh-CN',engine='auto') + assert len(called)==2 + with zipfile.ZipFile(out) as z: + body=b''.join(z.read(n) for n in z.namelist() if '/chapters/' in n).decode() + assert '新识别结果' in body and 'native English' not in body + assert 'lang="zh-CN"' in body + FakeOCR.identity='fixture-v2' + convert(source,out,work,language='zh-CN',engine='auto') + assert len(called)==4 + + +def test_invalid_ocr_geometry_fails_closed(): + with pytest.raises(ValueError):validate_lines([{'text':'bad','bbox':[.8,.2,.1,.4]}]) + with pytest.raises(ValueError):validate_lines([{'text':'bad','bbox':[0,0,float('nan'),1]}]) + + +def test_ocr_failure_leaves_page_checkpoint_and_no_partial_final(native_pdf,tmp_path,monkeypatch): + from src.pdf_import import ocr + called=[] + class FakeOCR: + identity='resume-fixture' + fail=True + def __init__(self,*args):pass + def recognize(self,image): + called.append(Path(image).name) + if Path(image).stem=='0002' and self.fail:raise RuntimeError('fixture OCR failure') + return [{'text':'A sufficiently long recognized paragraph to remain searchable in the EPUB.', 'bbox':[.1,.2,.9,.3], 'confidence':1}] + def close(self):pass + monkeypatch.setattr(ocr,'VisionOCR',FakeOCR);monkeypatch.setattr(ocr,'TesseractOCR',FakeOCR) + source=native_pdf(image=True);out=tmp_path/'output.epub';work=tmp_path/'work' + with pytest.raises(RuntimeError,match='fixture OCR failure'):convert(source,out,work,language='en') + assert not out.exists() and (work/'pages/0001.json').exists() and not (work/'pages/0002.json').exists() + FakeOCR.fail=False + convert(source,out,work,language='en') + assert called==['0001.jpg','0002.jpg','0002.jpg'] + assert audit_epub(out)['issues']==[] + + +def test_consistently_indented_quote_lines_stay_one_paragraph(): + lines=[{'id':'a','text':'这是正文段落。','bbox':[.1,.11,.9,.13]}, {'id':'b','text':'一段尚未结束的引文','bbox':[.2,.16,.8,.18]}, {'id':'c','text':'继续在相同缩进位置。','bbox':[.2,.19,.8,.21]}] + blocks,_=paragraphs({'number':1,'lines':lines},'zh-CN',set()) + assert len(blocks)==2 + assert blocks[1]['text']=='一段尚未结束的引文继续在相同缩进位置。' + + +def test_corrections_require_source_and_exact_expected_text(): + from src.pdf_import.converter import apply_corrections + pages=[{'lines':[{'id':'p0001-l0001','text':'牲富'}]}] + patch={'source_sha256':'source','edits':[{'line_id':'p0001-l0001','expected':'牲富','replacement':'牲畜'}]} + with pytest.raises(ValueError):apply_corrections(pages,patch,'different-source') + applied=apply_corrections(pages,patch,'source') + assert pages[0]['lines'][0]['text']=='牲畜' and len(applied)==1 + with pytest.raises(ValueError,match='Stale'):apply_corrections(pages,patch,'source')