Add FireRed OCR comparison and preserve attribution layout

src/pdf_import/converter.py

@@ -80,7 +80,7 @@ def apply_corrections(pages, correction, source_sha):
 
 
 def convert(source, output, work, language='zh-CN', title=None, author=None,
-            engine='auto', pages=None, dpi=180, cover_page=None, ocr_timeout=120, image_pages=None, corrections=None):
+            engine='auto', pages=None, dpi=180, cover_page=None, ocr_timeout=120, image_pages=None, corrections=None, ocr_url=None, ocr_model='firered-ocr', ocr_prompt=None, geometry_work=None):
     import pypdfium2 as pdfium
     source,output,work=Path(source),Path(output),Path(work)
     if not source.is_file(): raise ValueError('PDF source does not exist')
@@ -97,6 +97,9 @@ def convert(source, output, work, language='zh-CN', title=None, author=None,
         selected=page_selection(pages,len(document)); partial=len(selected)!=len(document)
         preserved_images=set(i+1 for i in page_selection(image_pages,len(document))) if image_pages else set()
         if cover_page is not None and not 1<=cover_page<=len(document): raise ValueError('Cover page outside source PDF')
+        geometry_dir=Path(geometry_work) if geometry_work else None
+        if geometry_dir and json.loads((geometry_dir/'source.json').read_text())['sha256']!=source_sha:
+            raise ValueError('Reference geometry belongs to another PDF')
         backend=('vision' if platform.system()=='Darwin' else 'tesseract') if engine=='auto' else engine
         outlines=[]
         for bookmark in document.get_toc():
@@ -115,8 +118,12 @@ def convert(source, output, work, language='zh-CN', title=None, author=None,
                 is_cover=index+1==cover_page
                 is_image=is_cover or index+1 in preserved_images
                 if use_ocr and not is_image and ocr is None:
-                    from .ocr import VisionOCR,TesseractOCR
-                    ocr=(VisionOCR if backend=='vision' else TesseractOCR)(work,language,ocr_timeout)
+                    if backend=='firered':
+                        from .firered import FireRedOCR
+                        ocr=FireRedOCR(work,language,ocr_timeout,base_url=ocr_url,model=ocr_model,prompt_path=ocr_prompt)
+                    else:
+                        from .ocr import VisionOCR,TesseractOCR
+                        ocr=(VisionOCR if backend=='vision' else TesseractOCR)(work,language,ocr_timeout)
                 used='cover-image' if is_cover else 'page-image' if is_image else backend if use_ocr else 'native'
                 stamp={'source':source_sha,'page':index+1,'engine':used,'ocr_identity':ocr.identity if use_ocr and not is_image else None,'language':language,'dpi':dpi,'pdfium':str(pdfium.PYPDFIUM_INFO),'schema':SCHEMA}
                 cache=work/'pages'/f'{index+1:04d}.json'
@@ -128,10 +135,12 @@ def convert(source, output, work, language='zh-CN', title=None, author=None,
                     bitmap=page.render(scale=dpi/72)
                     try: bitmap.to_pil().convert('RGB').save(render,quality=88)
                     finally: bitmap.close()
-                    lines=[] if is_image else ocr.recognize(render) if use_ocr else native
+                    recognition=[] if is_image else ocr.recognize(render) if use_ocr else native
+                    extra=recognition if isinstance(recognition,dict) else {}
+                    lines=extra.get('lines',[]) if isinstance(recognition,dict) else recognition
                     validate_lines(lines)
                     for n,line in enumerate(lines): line['id']=f'p{index+1:04d}-l{n+1:04d}'
-                    result={'number':index+1,'width':page.get_width(),'height':page.get_height(),'engine':used,'scan':scan,'lines':lines,'image_boxes':boxes,'render':str(render.resolve()),'stamp':stamp}
+                    result={'number':index+1,'width':page.get_width(),'height':page.get_height(),'engine':used,'scan':scan,'lines':lines,'image_boxes':boxes,'render':str(render.resolve()),'stamp':stamp,**extra}
                     save_json(cache,result)
                 results.append(result)
                 print(f'PDF page {index+1}/{len(document)}: {used}, {len(result["lines"])} lines',file=sys.stderr,flush=True)
@@ -139,9 +148,16 @@ def convert(source, output, work, language='zh-CN', title=None, author=None,
         applied=apply_corrections(results,json.loads(Path(corrections).read_text()),source_sha) if corrections else []
         repeated=repeated_furniture(results)
         for page in results:
-            blocks,removed=paragraphs(page,language,repeated)
+            if 'markdown' in page:
+                from .markdown_layout import markdown_blocks
+                geometry=json.loads((geometry_dir/'pages'/f'{page["number"]:04d}.json').read_text()) if geometry_dir else None
+                if geometry and (geometry['number']!=page['number'] or geometry['stamp']['source']!=source_sha):
+                    raise ValueError('Reference page geometry provenance mismatch')
+                blocks=markdown_blocks(page,language,geometry);removed=[]
+            else:
+                blocks,removed=paragraphs(page,language,repeated)
             page['blocks']=blocks;page['removed_furniture']=removed
-            count=sum(len(l['text']) for l in page['lines'])
+            count=sum(len(b['text']) for b in blocks) if 'markdown' in page else sum(len(l['text']) for l in page['lines'])
             page['image_fallback']=page['engine'] in {'cover-image','page-image'} or count<30
             if page['image_fallback'] and page['engine'] not in {'cover-image','page-image'}:
                 warnings.append({'page':page['number'],'kind':'page_image_fallback','detail':'Blank/illustrated/short-text page retained as an image; inspect source, not claimed as reflowed text.'})
@@ -149,7 +165,7 @@ def convert(source, output, work, language='zh-CN', title=None, author=None,
             low=[l['id'] for l in page['lines'] if l['id'] not in removed_ids and l.get('confidence') is not None and l['confidence']<.6]
             if low: warnings.append({'page':page['number'],'kind':'low_ocr_confidence','line_ids':low})
             if engine=='native' and page['scan']: warnings.append({'page':page['number'],'kind':'existing_scan_text_layer','detail':'Existing OCR text layer reused without verification.'})
-        book={'schema':SCHEMA,'source_sha256':source_sha,'source':str(source.resolve()),'source_pages':len(document),'selected_pages':[i+1 for i in selected],'partial':partial,'language':language,'title':title or source.stem,'author':author or 'Unknown','outlines':outlines,'image_pages':sorted(preserved_images),'cover_page':cover_page,'pages':results,'warnings':warnings,'applied_corrections':applied,'scope':'horizontal single-column; no automatic footnote/table reconstruction'}
+        book={'schema':SCHEMA,'source_sha256':source_sha,'source':str(source.resolve()),'source_pages':len(document),'selected_pages':[i+1 for i in selected],'partial':partial,'language':language,'title':title or source.stem,'author':author or 'Unknown','outlines':outlines,'image_pages':sorted(preserved_images),'cover_page':cover_page,'geometry_work':str(geometry_dir.resolve()) if geometry_dir else None,'pages':results,'warnings':warnings,'applied_corrections':applied,'scope':'horizontal single-column; no automatic footnote/table reconstruction'}
         save_json(work/'book.json',book)
         export_epub(book,output,work,cover_page)
         from src.qa.epub_audit import audit_epub
@@ -163,7 +179,7 @@ def convert(source, output, work, language='zh-CN', title=None, author=None,
         document.close()
 
 
-STYLE='''body {margin: 5%; line-height:1.65;} p {margin:.5em 0;} html[lang^="zh"] p {text-indent:2em;} h1,h2 {text-align:center;page-break-before:always;} img {max-width:100%;height:auto;} figure {margin:1em 0;text-align:center;} figcaption {font-size:.8em;} .source-page {font-size:.7em;color:#777;text-indent:0;} .image-page {text-align:center;text-indent:0;} .image-page img {max-height:90vh;width:auto;object-fit:contain;}'''
+STYLE='''body {margin: 5%; line-height:1.65;} p {margin:.5em 0;} html[lang^="zh"] p {text-indent:2em;} h1,h2 {text-align:center;page-break-before:always;} img {max-width:100%;height:auto;} figure {margin:1em 0;text-align:center;} figcaption {font-size:.8em;} .source-page {font-size:.7em;color:#777;text-indent:0;} .attribution {text-align:right !important;text-indent:0 !important;margin:.4em 0;} .image-page {text-align:center;text-indent:0;} .image-page img {max-height:90vh;width:auto;object-fit:contain;}'''
 
 
 def export_epub(data, output, work, cover_page):
@@ -206,7 +222,7 @@ def export_epub(data, output, work, cover_page):
             for n,box in enumerate(page['image_boxes']):
                 area=(box[2]-box[0])*(box[3]-box[1])
                 if .02<area<.65: entries.append({'kind':'image','bbox':box,'image_index':n})
-            entries.sort(key=lambda b:(b['bbox'][1],b['bbox'][0]))
+            if all(b.get('bbox') for b in entries):entries.sort(key=lambda b:(b['bbox'][1],b['bbox'][0]))
             marker_pending=True
             for index,block in enumerate(entries):
                 if block['kind']=='image':
@@ -220,14 +236,15 @@ def export_epub(data, output, work, cover_page):
                 plain=re.sub(r'\s','',block['text'])
                 if any(plain==re.sub(r'\s','',label) or (index==0 and plain in {'序','序言','前言','自序','引言','后记','目录'} and re.sub(r'\s','',label).startswith(plain)) for label in labels):continue
                 # Merge only obvious prose continuation across consecutive source pages.
-                continuation=marker_pending and previous and not labels and previous_page==number-1 and previous['kind']=='paragraph' and block['kind']=='paragraph' and not block['indented'] and not re.search(r'[。！？.!?:：;；”’"]$',previous['text']) and body[-1].endswith('</p>')
+                continuation=marker_pending and previous and not labels and previous_page==number-1 and previous['kind']=='paragraph' and block['kind']=='paragraph' and previous.get('role')!='attribution' and block.get('role')!='attribution' and not block['indented'] and not re.search(r'[。！？.!?:：;；”’"]$',previous['text']) and body[-1].endswith('</p>')
                 if continuation:
                     extra=join_text(previous['text'],block['text'],data['language'])[len(previous['text']):]
                     body[-1]=body[-1][:-4]+marker+f'<span id="{block["id"]}"></span>'+html.escape(extra)+'</p>'
                     marker_pending=False;previous={**block,'text':join_text(previous['text'],block['text'],data['language'])};continue
                 if marker_pending:body.append(marker);marker_pending=False
                 tag='h2' if block['kind']=='heading' else 'p'
-                body.append(f'<{tag} id="{block["id"]}">{html.escape(block["text"])}</{tag}>')
+                attributes=' class="attribution"' if block.get('alignment')=='right' else ''
+                body.append(f'<{tag} id="{block["id"]}"{attributes}>{html.escape(block["text"])}</{tag}>')
                 if block['kind']=='heading' and not data['outlines']:
                     toc_records.append((epub.Link(current.file_name+'#'+block['id'],block['text'],f'nav-{len(toc_records)+1}'),0))
                 previous=block