66 lines
3.0 KiB
Python
66 lines
3.0 KiB
Python
"""One HTML5 repair policy shared by extraction and backfilling."""
|
|
import re
|
|
from bs4 import BeautifulSoup, NavigableString, Tag
|
|
|
|
BLOCK_TAGS = {'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'div',
|
|
'section', 'article', 'blockquote', 'table', 'tr', 'td', 'th',
|
|
'ul', 'ol', 'dl', 'dt', 'dd', 'figure', 'figcaption', 'pre'}
|
|
TARGET_TAGS = {'p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'td', 'th',
|
|
'dt', 'dd', 'figcaption', 'blockquote', 'div', 'section', 'article'}
|
|
SKIP_TAGS = {'script', 'style', 'pre', 'code', 'math', 'svg', 'textarea'}
|
|
|
|
|
|
def parse_html(content):
|
|
return BeautifulSoup(re.sub(r'^\s*<\?xml[^>]*\?>', '', content), 'html5lib')
|
|
|
|
|
|
def normalize_navigation_markup(soup):
|
|
"""Remove only our generated wrappers around nested NAV labels.
|
|
|
|
Reader TOC parsers expect li > a + ol. A generated li > span > a
|
|
retains a valid href but is interpreted as an unlinked section by Calibre.
|
|
This also upgrades source structures cached by earlier versions.
|
|
"""
|
|
for wrapper in list(soup.find_all('span', attrs={'data-translation-unit': 'true'})):
|
|
parent = wrapper.parent
|
|
if (parent and parent.name == 'li' and wrapper.find_parent('nav')
|
|
and parent.find('ol', recursive=False)
|
|
and wrapper.find('a', href=True, recursive=False)):
|
|
wrapper.unwrap()
|
|
|
|
|
|
def translation_targets(soup):
|
|
"""Choose disjoint leaf blocks; wrap mixed direct text without renaming containers.
|
|
|
|
A li containing p must never be extracted twice. Text before a nested list or
|
|
inside an otherwise bare body still needs its own unit.
|
|
"""
|
|
normalize_navigation_markup(soup)
|
|
for container in list(soup.find_all(list(TARGET_TAGS | {'body'}))):
|
|
if container.find_parent(list(SKIP_TAGS)) or container.find_parent('nav'):
|
|
continue
|
|
has_blocks = container.find(list(BLOCK_TAGS)) is not None
|
|
if container.name != 'body' and not has_blocks:
|
|
continue
|
|
run = []
|
|
def flush():
|
|
if run and any(isinstance(x, NavigableString) and x.strip() or
|
|
isinstance(x, Tag) and x.get_text(strip=True) for x in run):
|
|
wrapper = soup.new_tag('span')
|
|
wrapper['data-translation-unit'] = 'true'
|
|
run[0].insert_before(wrapper)
|
|
for node in run:
|
|
wrapper.append(node.extract())
|
|
run.clear()
|
|
for child in list(container.children):
|
|
if isinstance(child, Tag) and (child.name in BLOCK_TAGS | SKIP_TAGS or child.get('data-translation-unit') == 'true' or child.find(list(BLOCK_TAGS))):
|
|
flush()
|
|
else:
|
|
run.append(child)
|
|
flush()
|
|
return [tag for tag in soup.find_all(True)
|
|
if (tag.get('data-translation-unit') == 'true' or
|
|
tag.name in TARGET_TAGS and not tag.find(list(BLOCK_TAGS)))
|
|
and not tag.find_parent(list(SKIP_TAGS))
|
|
and not tag.find_parent(attrs={'data-translation-unit': 'true'})]
|