67 lines
5.1 KiB
Python
67 lines
5.1 KiB
Python
"""Same-page OCR differences, plus optional manually verified snippet CER (not full-book accuracy)."""
|
|
import argparse
|
|
import html
|
|
import json
|
|
import re
|
|
import sys
|
|
import unicodedata
|
|
from pathlib import Path
|
|
from difflib import SequenceMatcher
|
|
sys.path.insert(0,str(Path(__file__).resolve().parents[1]))
|
|
from src.pdf_import.converter import save_json
|
|
|
|
|
|
def normalized(text):
|
|
return ''.join(c for c in unicodedata.normalize('NFKC',text) if c.isalnum())
|
|
|
|
|
|
def snippet_errors(reference,hypothesis):
|
|
"""Levenshtein alignment to the best substring on the SAME source page."""
|
|
ref,hyp=normalized(reference),normalized(hypothesis)
|
|
if not ref:raise ValueError('Empty ground truth')
|
|
matrix=[list(range(0,len(hyp)+1))]
|
|
matrix[0]=[0]*(len(hyp)+1) # Free page prefix/suffix outside the verified snippet.
|
|
for i,c in enumerate(ref,1):
|
|
row=[i]
|
|
for j,d in enumerate(hyp,1):row.append(min(matrix[i-1][j]+1,row[j-1]+1,matrix[i-1][j-1]+(c!=d)))
|
|
matrix.append(row)
|
|
end=min(range(len(hyp)+1),key=lambda j:matrix[-1][j]);i,j=len(ref),end;counts={'substitutions':0,'deletions':0,'insertions':0}
|
|
while i:
|
|
if j and matrix[i][j]==matrix[i-1][j-1]+(ref[i-1]!=hyp[j-1]):
|
|
if ref[i-1]!=hyp[j-1]:counts['substitutions']+=1
|
|
i-=1;j-=1
|
|
elif matrix[i][j]==matrix[i-1][j]+1:counts['deletions']+=1;i-=1
|
|
else:counts['insertions']+=1;j-=1
|
|
return {'reference_characters':len(ref),'errors':sum(counts.values()),**counts,'cer':sum(counts.values())/len(ref),'matched_normalized_text':hyp[j:end]}
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--vision-work',type=Path,required=True);parser.add_argument('--firered-work',type=Path,required=True);parser.add_argument('--ground-truth',type=Path);parser.add_argument('--out',type=Path,required=True);args=parser.parse_args()
|
|
source=json.loads((args.vision_work/'source.json').read_text())['sha256'];pages=[];sections=[];texts={}
|
|
args.out.mkdir(parents=True,exist_ok=True)
|
|
import os
|
|
for n in range(1,51):
|
|
vp=json.loads((args.vision_work/'pages'/f'{n:04d}.json').read_text());fp=json.loads((args.firered_work/'comparison-pages'/f'{n:04d}.json').read_text())
|
|
if vp['stamp']['source']!=source or vp['number']!=n or fp['page']!=n:raise ValueError('Page/source provenance mismatch')
|
|
v='\n'.join(line['text'] for line in vp['lines']);f=fp['result']['markdown'];texts[n]=(v,f)
|
|
pages.append({'page':n,'vision_reference_engine':vp['engine'],'text_comparison_eligible':vp['engine']=='vision','vision_characters':len(normalized(v)),'firered_characters':len(normalized(f)),'agreement_ratio_not_accuracy':SequenceMatcher(None,normalized(v),normalized(f),autojunk=False).ratio(),'finish_reason':fp['result']['finish_reason'],'seconds':fp['result']['elapsed_seconds'],'firered_image_size':fp['result']['image_size']})
|
|
image=os.path.relpath(args.vision_work/'renders'/f'{n:04d}.jpg',args.out)
|
|
sections.append(f'<section id="p{n}"><h2>PDF 原页序 {n}</h2><div><img loading="lazy" src="{html.escape(image)}"/><pre><b>Vision 原始结果</b>\n{html.escape(v)}</pre><pre><b>FireRed 原始 Markdown</b>\n{html.escape(f)}</pre></div></section>')
|
|
samples=[]
|
|
if args.ground_truth:
|
|
gt=json.loads(args.ground_truth.read_text())
|
|
if gt['source_sha256']!=source:raise ValueError('Ground truth belongs to another PDF')
|
|
for sample in gt['samples']:
|
|
n=sample['page'];v,f=texts[n]
|
|
samples.append({'page':n,'scope':sample['scope'],'vision':snippet_errors(sample['text'],v),'firered':snippet_errors(sample['text'],f)})
|
|
summary={}
|
|
for engine in ['vision','firered']:
|
|
count=sum(s[engine]['reference_characters'] for s in samples);errors=sum(s[engine]['errors'] for s in samples)
|
|
summary[engine]={'sample_reference_characters':count,'sample_errors':errors,'sample_cer':errors/count if count else None}
|
|
report={'source_sha256':source,'page_range':[1,50],'pages':pages,'verified_snippets':samples,'summary':summary,'limits':['Agreement ratio is disagreement between OCR engines, not accuracy.','CER covers only manually transcribed diagnostic snippets, not all 50 pages or the book.','Whitespace and punctuation excluded using Unicode NFKC alphanumeric normalization.','FireRed input is capped at 1600px to fit this server context; reference Vision uses existing 180dpi cache.']}
|
|
save_json(args.out/'comparison.json',report)
|
|
(args.out/'index.html').write_text('<!doctype html><meta charset="utf-8"><title>Vision / FireRed 前50页对照</title><style>body{font:16px sans-serif;margin:24px}section{margin:3em 0}section>div{display:grid;grid-template-columns:1fr 1fr 1fr;gap:16px}img{width:100%;align-self:start}pre{white-space:pre-wrap;overflow-wrap:anywhere;line-height:1.7}</style><h1>原扫描 / Vision / FireRed：前50页</h1><p>原始识别结果未作人工文字校正。差异不等于错误；量化指标仅针对人工核对片段。Vision原缓存中的封面、扉页及原目录页为保留原图模式，未OCR，不把它们的零字符作为漏识。</p>'+''.join(sections))
|
|
print(json.dumps(summary,ensure_ascii=False))
|
|
|
|
if __name__=='__main__':main()
|