Release v0.13: Typography V3 (Safe mode) & Endnote Extraction Fix
This commit is contained in:
谭凯
@@ -0,0 +1,184 @@
|
||||
import shutil
|
||||
import uuid
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Dict, Set
|
||||
|
||||
import ebooklib
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
from ebooklib import epub
|
||||
|
||||
from src.common.data_model import BookStructure, BookMetaData, ResourceItem
|
||||
from src.common.exceptions import CleaningError
|
||||
from src.common.utils import setup_logger, ensure_directory
|
||||
|
||||
logger = setup_logger("epub_cleaner")
|
||||
|
||||
class EpubCleaner:
|
||||
def __init__(self, input_path: Path, work_dir: Path):
|
||||
self.input_path = input_path
|
||||
self.work_dir = work_dir
|
||||
self.assets_dir = work_dir / "assets"
|
||||
self.json_path = work_dir / "book_structure.json"
|
||||
|
||||
def clean(self) -> Path:
|
||||
"""
|
||||
Cleans the input EPUB and generates book_structure.json.
|
||||
Returns the path to the JSON file.
|
||||
"""
|
||||
logger.info(f"Starting cleanup for {self.input_path}")
|
||||
ensure_directory(self.work_dir)
|
||||
ensure_directory(self.assets_dir)
|
||||
|
||||
try:
|
||||
book = epub.read_epub(self.input_path)
|
||||
|
||||
# 1. Extract Metadata
|
||||
metadata = self._extract_metadata(book)
|
||||
|
||||
# 2. Process Resources
|
||||
resources = {}
|
||||
# Use zipfile for binary extraction to avoid ebooklib's memory overhead/decoding issues
|
||||
with zipfile.ZipFile(self.input_path, 'r') as zf:
|
||||
# Map ebooklib items to zip entries isn't straightforward directly via name
|
||||
# So we iterate ebooklib items and assume standard structure or handle content bytes
|
||||
|
||||
for item in book.get_items():
|
||||
item_id = item.get_id()
|
||||
file_name = item.get_name()
|
||||
media_type = item.get_type() # ebooklib constant
|
||||
|
||||
if media_type == ebooklib.ITEM_DOCUMENT:
|
||||
# Clean HTML
|
||||
content_str = item.get_content().decode('utf-8')
|
||||
cleaned_content = self._clean_html(content_str, file_name)
|
||||
|
||||
resources[item_id] = ResourceItem(
|
||||
href=file_name,
|
||||
media_type="application/xhtml+xml",
|
||||
content=cleaned_content
|
||||
)
|
||||
elif media_type in (ebooklib.ITEM_IMAGE, ebooklib.ITEM_STYLE, ebooklib.ITEM_FONT, ebooklib.ITEM_COVER):
|
||||
# Save asset
|
||||
|
||||
# Check if it is a cover
|
||||
if media_type == ebooklib.ITEM_COVER:
|
||||
metadata.cover_image_id = item_id
|
||||
logger.info(f"Found cover image: {item_id} ({file_name})")
|
||||
|
||||
# Preserve directory structure to avoid collisions
|
||||
asset_path = self.assets_dir / file_name
|
||||
ensure_directory(asset_path.parent)
|
||||
|
||||
# Ebooklib might change filenames, safer to use item.get_content()
|
||||
with open(asset_path, "wb") as f:
|
||||
f.write(item.get_content())
|
||||
|
||||
resources[item_id] = ResourceItem(
|
||||
href=file_name,
|
||||
media_type=self._get_media_type_str(item),
|
||||
file_path=str(asset_path.relative_to(self.work_dir))
|
||||
)
|
||||
elif media_type == ebooklib.ITEM_NAVIGATION:
|
||||
# NCX or NAV document - preserve for TOC
|
||||
content_bytes = item.get_content()
|
||||
asset_path = self.assets_dir / file_name
|
||||
ensure_directory(asset_path.parent)
|
||||
|
||||
with open(asset_path, "wb") as f:
|
||||
f.write(content_bytes)
|
||||
|
||||
# Determine media type
|
||||
if file_name.endswith('.ncx'):
|
||||
mt = "application/x-dtbncx+xml"
|
||||
else:
|
||||
mt = "application/xhtml+xml"
|
||||
|
||||
resources[item_id] = ResourceItem(
|
||||
href=file_name,
|
||||
media_type=mt,
|
||||
file_path=str(asset_path.relative_to(self.work_dir))
|
||||
)
|
||||
logger.debug(f"Preserved navigation: {file_name}")
|
||||
else:
|
||||
# Skip other items (scripts, etc.)
|
||||
pass
|
||||
|
||||
# 3. Extract Spine
|
||||
spine_ids = [item[0] for item in book.spine]
|
||||
|
||||
# 4. Construct Structure
|
||||
structure = BookStructure(
|
||||
metadata=metadata,
|
||||
spine=spine_ids,
|
||||
resources=resources
|
||||
)
|
||||
|
||||
# 5. Serialize
|
||||
with open(self.json_path, "w", encoding="utf-8") as f:
|
||||
f.write(structure.model_dump_json(indent=2))
|
||||
|
||||
logger.info(f"Cleanup finished. Structure saved to {self.json_path}")
|
||||
return self.json_path
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Cleaning failed: {e}")
|
||||
raise CleaningError(f"Failed to clean EPUB: {e}") from e
|
||||
|
||||
def _extract_metadata(self, book: epub.EpubBook) -> BookMetaData:
|
||||
title = book.get_metadata('DC', 'title')[0][0] if book.get_metadata('DC', 'title') else "Unknown"
|
||||
author = book.get_metadata('DC', 'creator')[0][0] if book.get_metadata('DC', 'creator') else "Unknown"
|
||||
lang = book.get_metadata('DC', 'language')[0][0] if book.get_metadata('DC', 'language') else "en"
|
||||
ident = book.get_metadata('DC', 'identifier')[0][0] if book.get_metadata('DC', 'identifier') else ""
|
||||
|
||||
return BookMetaData(
|
||||
title=str(title),
|
||||
author=str(author),
|
||||
language=str(lang),
|
||||
identifier=str(ident)
|
||||
)
|
||||
|
||||
def _get_media_type_str(self, item) -> str:
|
||||
# Helper to map ebooklib type to mime string if needed
|
||||
# ebooklib doesn't expose easy MIME string for all types directly on item object sometimes
|
||||
if hasattr(item, 'media_type'):
|
||||
return item.media_type
|
||||
return "application/octet-stream"
|
||||
|
||||
def _clean_html(self, content: str, filename: str) -> str:
|
||||
soup = BeautifulSoup(content, 'html.parser')
|
||||
|
||||
# 1. Provide IDs for structural/translatable elements
|
||||
self._ensure_element_ids(soup)
|
||||
|
||||
# 2. Flatten divs (Disable to prevent style loss)
|
||||
# self._flatten_divs(soup)
|
||||
|
||||
return str(soup)
|
||||
|
||||
def _ensure_element_ids(self, soup: BeautifulSoup):
|
||||
"""
|
||||
Injects UUIDs into p, h1-h6, li tags if they don't have an ID.
|
||||
This provides the anchor for translation backfilling.
|
||||
"""
|
||||
targets = soup.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li'])
|
||||
for tag in targets:
|
||||
if not tag.has_attr('id'):
|
||||
tag['id'] = f"uuid-{uuid.uuid4()}"
|
||||
|
||||
def _flatten_divs(self, soup: BeautifulSoup):
|
||||
"""
|
||||
Converts generic divs containing only inline text/styles to p tags.
|
||||
Recursive strategies can be complex, sticking to simple heuristic from archive.
|
||||
"""
|
||||
inline_tags = {'span', 'em', 'strong', 'i', 'b', 'u', 'a', 'br', 'sub', 'sup', 'small'}
|
||||
|
||||
for div in list(soup.find_all('div')):
|
||||
# If div has no block children, convert to p
|
||||
has_block = any(
|
||||
isinstance(c, Tag) and c.name not in inline_tags
|
||||
for c in div.children
|
||||
)
|
||||
|
||||
if not has_block:
|
||||
div.name = 'p'
|
||||
@@ -0,0 +1,406 @@
|
||||
import re
|
||||
import html
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
from typing import Tuple, Dict, List
|
||||
|
||||
class HeadingDetector:
|
||||
"""Detects heading types and paragraph roles."""
|
||||
|
||||
CHAPTER_PATTERNS = [
|
||||
r'^(chapter|chap\.?|part)\s+([0-9]+|[ivxlc]+|[a-z])',
|
||||
r'^(第\s*[0-9一二三四五六七八九十百]+\s*[章节部篇])',
|
||||
r'^(\d+|[IVXLC]+|[A-Z])\.$'
|
||||
]
|
||||
|
||||
EPIGRAPH_CLASSES = {
|
||||
'epigraph', 'quote', 'blockquote', 'motto',
|
||||
'dedication', 'verse', 'poetry', 'poem'
|
||||
}
|
||||
|
||||
def detect(self, element: Tag, text: str) -> str:
|
||||
if self._is_epigraph(element):
|
||||
return "epigraph"
|
||||
tag_name = element.name.lower()
|
||||
if tag_name in ['h1', 'h2']:
|
||||
return "chapter" if self._matches_chapter_pattern(text) else "section"
|
||||
if tag_name == 'h3':
|
||||
return "section"
|
||||
if tag_name in ['h4', 'h5', 'h6']:
|
||||
return "subsection"
|
||||
if self._is_pseudo_heading(element, text):
|
||||
return "subsection"
|
||||
return "body"
|
||||
|
||||
def _is_epigraph(self, element: Tag) -> bool:
|
||||
if element.name == 'blockquote':
|
||||
return True
|
||||
current = element
|
||||
for _ in range(3):
|
||||
if not current: break
|
||||
classes = current.get('class', [])
|
||||
if isinstance(classes, list):
|
||||
classes = ' '.join(classes)
|
||||
if any(k in classes.lower() for k in self.EPIGRAPH_CLASSES):
|
||||
return True
|
||||
current = current.parent
|
||||
return False
|
||||
|
||||
def _matches_chapter_pattern(self, text: str) -> bool:
|
||||
text = text.strip().lower()
|
||||
for pattern in self.CHAPTER_PATTERNS:
|
||||
if re.match(pattern, text, re.IGNORECASE):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _is_pseudo_heading(self, element: Tag, text: str) -> bool:
|
||||
if element.name != 'p':
|
||||
return False
|
||||
text = text.strip()
|
||||
if not text or len(text) > 80:
|
||||
return False
|
||||
children = list(element.children)
|
||||
if len(children) == 1 and isinstance(children[0], Tag):
|
||||
if children[0].name in ['strong', 'b']:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class FormatExtractor:
|
||||
"""
|
||||
HTML Format Extractor (Ported from v0.09 v3)
|
||||
Handles inline styles, formulas, and drop caps.
|
||||
"""
|
||||
|
||||
FORMULA_CHARS = re.compile(
|
||||
r'^[\d\s\+\-\×\÷\=\(\)\[\]\{\}\<\>\^\*\/\.\,\;\:\'\"\`\~\@\#\$\%\&\|\\'
|
||||
r'αβγδεζηθικλμνξοπρστυφχψωΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩ'
|
||||
r'a-zA-Z]+$'
|
||||
)
|
||||
|
||||
def __init__(self):
|
||||
self.detector = HeadingDetector()
|
||||
|
||||
def extract(self, element_html: str) -> Tuple[str, str, Dict[str, str], str, List[str]]:
|
||||
"""
|
||||
Extracts format information.
|
||||
|
||||
Returns:
|
||||
clean_text: Pure text
|
||||
text_with_placeholders: Text with inline placeholders
|
||||
placeholder_map: Map of placeholders
|
||||
paragraph_type: Detected type
|
||||
endnote_anchors: List of detected endnote IDs
|
||||
"""
|
||||
soup = BeautifulSoup(element_html, 'html.parser')
|
||||
root = list(soup.children)[0] if list(soup.children) else soup
|
||||
|
||||
clean_text = root.get_text().strip()
|
||||
clean_text = re.sub(r'\s+', ' ', clean_text)
|
||||
p_type = self.detector.detect(root, clean_text) if isinstance(root, Tag) else "body"
|
||||
|
||||
inner_html = root.decode_contents() if isinstance(root, Tag) else str(root)
|
||||
|
||||
text_with_ph, local_map = self._smart_extract_v3(inner_html)
|
||||
|
||||
if text_with_ph:
|
||||
text_with_ph = re.sub(r'\s+', ' ', text_with_ph).strip()
|
||||
|
||||
# Verify integrity
|
||||
# FIX: Instead of stripping placeholders blindly (which removes content like [28] if grouped),
|
||||
# we must 'hydrate' them: replace placeholder with the text content of its mapping.
|
||||
hydrated_text = self._hydrate_placeholders(text_with_ph, local_map)
|
||||
hydrated_text = re.sub(r'\s+', ' ', hydrated_text).strip()
|
||||
|
||||
if not self._verify_content_integrity(clean_text, hydrated_text):
|
||||
# Fallback
|
||||
text_with_ph, local_map = self._fallback_extract(inner_html, clean_text)
|
||||
|
||||
endnote_anchors = []
|
||||
for pid, html_content in local_map.items():
|
||||
if pid.startswith("_"):
|
||||
continue
|
||||
if re.match(r'<(span|a)\s+id="[a-zA-Z][a-zA-Z0-9]{2,5}"\s*>\s*</\1>', html_content):
|
||||
endnote_anchors.append(pid)
|
||||
|
||||
return clean_text, text_with_ph, local_map, p_type, endnote_anchors
|
||||
|
||||
def _hydrate_placeholders(self, text: str, ph_map: Dict[str, str]) -> str:
|
||||
"""
|
||||
Replaces placeholders with the text content of their HTML values.
|
||||
Used for integrity checking.
|
||||
"""
|
||||
def replace_ph(match):
|
||||
ph = match.group(0)
|
||||
pid = match.group(1)
|
||||
# Handle closing tags /pid -> ignore or handle?
|
||||
# Placeholders format: φ1φ or φ/1φ.
|
||||
# Map keys: '1', '/1'.
|
||||
|
||||
clean_pid = pid
|
||||
if pid.startswith('/'):
|
||||
# Closing tag placeholder usually maps to </span>. Text is empty.
|
||||
return ""
|
||||
|
||||
if clean_pid in ph_map:
|
||||
content = ph_map[clean_pid]
|
||||
# Extract text from the HTML content
|
||||
# e.g. '<a ...>[28]</a>' -> '[28]'
|
||||
# e.g. '<span class="bold">' -> ''
|
||||
return BeautifulSoup(content, 'html.parser').get_text()
|
||||
return ""
|
||||
|
||||
return re.sub(r'φ(/?\d+)φ', replace_ph, text)
|
||||
|
||||
def _verify_content_integrity(self, clean_text: str, stripped_text: str) -> bool:
|
||||
def normalize(s):
|
||||
# Unescape HTML entities first (e.g. & -> &)
|
||||
s = html.unescape(s)
|
||||
s = re.sub(r'\s+', '', s)
|
||||
s = s.lower()
|
||||
return s
|
||||
|
||||
norm_clean = normalize(clean_text)
|
||||
norm_stripped = normalize(stripped_text)
|
||||
|
||||
if norm_clean == norm_stripped:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _fallback_extract(self, inner_html: str, clean_text: str) -> Tuple[str, Dict[str, str]]:
|
||||
return clean_text, {"_prefix": "", "_suffix": ""}
|
||||
|
||||
def _smart_extract_v3(self, inner_html: str) -> Tuple[str, Dict[str, str]]:
|
||||
parts = re.split(r'(<[^>]+>)', inner_html)
|
||||
parts = [p for p in parts if p]
|
||||
|
||||
if not parts:
|
||||
return "", {"_prefix": "", "_suffix": ""}
|
||||
|
||||
part_types = []
|
||||
for part in parts:
|
||||
if part.startswith('<'):
|
||||
part_types.append('tag')
|
||||
elif not part.strip():
|
||||
part_types.append('whitespace')
|
||||
elif self._is_translatable_text(part):
|
||||
part_types.append('translatable')
|
||||
else:
|
||||
part_types.append('formula')
|
||||
|
||||
first_trans_idx = None
|
||||
last_trans_idx = None
|
||||
for i, t in enumerate(part_types):
|
||||
if t == 'translatable':
|
||||
if first_trans_idx is None:
|
||||
first_trans_idx = i
|
||||
last_trans_idx = i
|
||||
|
||||
if first_trans_idx is None:
|
||||
return "", {"_prefix": inner_html, "_suffix": ""}
|
||||
|
||||
# Prefix Separation
|
||||
safe_prefix_end = 0
|
||||
for i in range(first_trans_idx):
|
||||
if part_types[i] == 'tag':
|
||||
tag = parts[i]
|
||||
is_self_closing = tag.endswith('/>') or re.match(r'<(br|hr|img|meta|link)\b', tag, re.I)
|
||||
is_closing = tag.startswith('</')
|
||||
if is_self_closing or is_closing:
|
||||
safe_prefix_end = i + 1
|
||||
else:
|
||||
break
|
||||
elif part_types[i] == 'whitespace':
|
||||
safe_prefix_end = i + 1
|
||||
else:
|
||||
break
|
||||
|
||||
# Suffix Separation
|
||||
safe_suffix_start = len(parts)
|
||||
for i in range(len(parts) - 1, last_trans_idx, -1):
|
||||
if part_types[i] == 'tag':
|
||||
tag = parts[i]
|
||||
is_closing = tag.startswith('</')
|
||||
is_self_closing = tag.endswith('/>') or re.match(r'<(br|hr|img|meta|link)\b', tag, re.I)
|
||||
if is_closing or is_self_closing:
|
||||
safe_suffix_start = i
|
||||
else:
|
||||
break
|
||||
elif part_types[i] == 'whitespace':
|
||||
safe_suffix_start = i
|
||||
else:
|
||||
break
|
||||
|
||||
prefix_parts = parts[:safe_prefix_end]
|
||||
middle_parts = parts[safe_prefix_end:safe_suffix_start]
|
||||
middle_types = part_types[safe_prefix_end:safe_suffix_start]
|
||||
suffix_parts = parts[safe_suffix_start:]
|
||||
|
||||
# Drop Cap Check
|
||||
if prefix_parts and middle_parts:
|
||||
prefix_parts, middle_parts, middle_types = self._handle_drop_cap(
|
||||
prefix_parts, middle_parts, middle_types
|
||||
)
|
||||
|
||||
local_map = {}
|
||||
if prefix_parts:
|
||||
local_map["_prefix"] = "".join(prefix_parts)
|
||||
if suffix_parts:
|
||||
local_map["_suffix"] = "".join(suffix_parts)
|
||||
|
||||
# Middle processing
|
||||
placeholder_counter = 1
|
||||
result_parts = []
|
||||
tag_stack = []
|
||||
|
||||
i = 0
|
||||
while i < len(middle_parts):
|
||||
part = middle_parts[i]
|
||||
ptype = middle_types[i]
|
||||
|
||||
if ptype == 'translatable':
|
||||
result_parts.append(part)
|
||||
i += 1
|
||||
|
||||
elif ptype == 'tag':
|
||||
is_closing = part.startswith('</')
|
||||
if is_closing:
|
||||
if tag_stack:
|
||||
open_id, open_tag = tag_stack.pop()
|
||||
local_map[f"/{open_id}"] = part
|
||||
result_parts.append(f"φ/{open_id}φ")
|
||||
else:
|
||||
pid = str(placeholder_counter)
|
||||
placeholder_counter += 1
|
||||
local_map[pid] = part
|
||||
result_parts.append(f"φ{pid}φ")
|
||||
i += 1
|
||||
else:
|
||||
has_translatable_after = False
|
||||
for j in range(i + 1, len(middle_parts)):
|
||||
if middle_types[j] == 'translatable':
|
||||
has_translatable_after = True
|
||||
break
|
||||
elif middle_types[j] == 'tag' and middle_parts[j].startswith('</'):
|
||||
break
|
||||
|
||||
if has_translatable_after:
|
||||
pid = str(placeholder_counter)
|
||||
placeholder_counter += 1
|
||||
local_map[pid] = part
|
||||
result_parts.append(f"φ{pid}φ")
|
||||
tag_stack.append((pid, part))
|
||||
i += 1
|
||||
else:
|
||||
block_parts = []
|
||||
while i < len(middle_parts) and middle_types[i] != 'translatable':
|
||||
block_parts.append(middle_parts[i])
|
||||
i += 1
|
||||
if block_parts:
|
||||
block_html = "".join(block_parts)
|
||||
pid = str(placeholder_counter)
|
||||
placeholder_counter += 1
|
||||
local_map[pid] = block_html
|
||||
result_parts.append(f"φ{pid}φ")
|
||||
else:
|
||||
result_parts.append(part)
|
||||
i += 1
|
||||
|
||||
text_with_ph = "".join(result_parts)
|
||||
text_with_ph = re.sub(r'\s+', ' ', text_with_ph).strip()
|
||||
|
||||
# Merge consecutive placeholders
|
||||
def merge_match(m):
|
||||
full_match = m.group(0)
|
||||
pids = re.findall(r'φ(/?\d+)φ', full_match)
|
||||
if len(pids) <= 1:
|
||||
return full_match
|
||||
|
||||
merged_html = ""
|
||||
for pid in pids:
|
||||
if pid in local_map:
|
||||
merged_html += local_map[pid]
|
||||
del local_map[pid]
|
||||
|
||||
new_pid = pids[0] if pids[0].isdigit() else pids[0][1:]
|
||||
local_map[new_pid] = merged_html
|
||||
return f"φ{new_pid}φ"
|
||||
|
||||
text_with_ph = re.sub(r'(φ/?\d+φ)(φ/?\d+φ)+', merge_match, text_with_ph)
|
||||
|
||||
return text_with_ph, local_map
|
||||
|
||||
def _handle_drop_cap(self, prefix_parts: List[str], middle_parts: List[str], middle_types: List[str]):
|
||||
if not prefix_parts or not middle_parts:
|
||||
return prefix_parts, middle_parts, middle_types
|
||||
|
||||
prefix_text = ""
|
||||
for part in prefix_parts:
|
||||
if not part.startswith('<'):
|
||||
prefix_text = part.strip()
|
||||
|
||||
if not prefix_text or len(prefix_text) != 1 or not prefix_text.isupper():
|
||||
return prefix_parts, middle_parts, middle_types
|
||||
|
||||
first_middle_text = ""
|
||||
first_middle_idx = -1
|
||||
for i, (part, ptype) in enumerate(zip(middle_parts, middle_types)):
|
||||
if ptype == 'translatable':
|
||||
first_middle_text = part.strip()
|
||||
first_middle_idx = i
|
||||
break
|
||||
|
||||
if not first_middle_text:
|
||||
return prefix_parts, middle_parts, middle_types
|
||||
|
||||
is_drop_cap = False
|
||||
first_char = first_middle_text[0] if first_middle_text else ''
|
||||
if first_char.islower() or first_char.isupper():
|
||||
is_drop_cap = True
|
||||
|
||||
combined = prefix_text + first_middle_text.split()[0] if first_middle_text else ""
|
||||
if not (len(combined) >= 2 and combined.isalpha()):
|
||||
return prefix_parts, middle_parts, middle_types
|
||||
|
||||
new_prefix = []
|
||||
skip_until_close = False
|
||||
found_letter = False
|
||||
|
||||
for part in prefix_parts:
|
||||
if part.startswith('<') and not part.startswith('</'):
|
||||
skip_until_close = True
|
||||
elif part.startswith('</'):
|
||||
if skip_until_close:
|
||||
skip_until_close = False
|
||||
continue
|
||||
new_prefix.append(part)
|
||||
elif part.strip() == prefix_text:
|
||||
found_letter = True
|
||||
continue
|
||||
else:
|
||||
if not skip_until_close:
|
||||
new_prefix.append(part)
|
||||
|
||||
if found_letter:
|
||||
middle_parts = middle_parts.copy()
|
||||
middle_parts[first_middle_idx] = prefix_text + middle_parts[first_middle_idx]
|
||||
prefix_parts = new_prefix
|
||||
|
||||
return prefix_parts, middle_parts, middle_types
|
||||
|
||||
def _is_translatable_text(self, text: str) -> bool:
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return False
|
||||
|
||||
# Treat citations like [28] or [1] as non-translatable (to be grouped into placeholders)
|
||||
if re.match(r'^\[\d+\]$', text):
|
||||
return False
|
||||
|
||||
if re.search(r'[a-zA-Z]{3,}', text):
|
||||
return True
|
||||
if ' ' in text and re.search(r'[a-zA-Z]', text):
|
||||
return True
|
||||
if re.search(r'\d', text):
|
||||
return True
|
||||
return False
|
||||
@@ -0,0 +1,83 @@
|
||||
import json
|
||||
import random
|
||||
from typing import Dict, List
|
||||
from loguru import logger
|
||||
from src.translation.llm_client import LLMClient
|
||||
from src.translation.manifest_manager import ManifestManager
|
||||
from src.common.data_model import BookProfile
|
||||
|
||||
class BookProfiler:
|
||||
def __init__(self, llm_client: LLMClient):
|
||||
self.llm_client = llm_client
|
||||
|
||||
def extract_sample_text(self, entries: List[object], char_limit: int = 3000) -> str:
|
||||
"""Extract sample text from manifest entries."""
|
||||
if not entries: return ""
|
||||
|
||||
# Simple sampling strategy: First few + random middle
|
||||
intro_text = []
|
||||
for entry in entries[:50]:
|
||||
if len(entry.original_text) > 50:
|
||||
intro_text.append(entry.original_text)
|
||||
|
||||
body_text = []
|
||||
candidates = [e for e in entries[50:] if len(e.original_text) > 80]
|
||||
if candidates:
|
||||
samples = random.sample(candidates, min(5, len(candidates)))
|
||||
body_text = [e.original_text for e in samples]
|
||||
|
||||
# Join text
|
||||
full_text = "\n\n".join(intro_text[:5] + body_text)
|
||||
|
||||
# Strip placeholders to stop Profiler from seeing "garbage"
|
||||
# Matches φ1φ, φ/1φ, etc.
|
||||
import re
|
||||
clean_text = re.sub(r'φ.*?φ', '', full_text)
|
||||
|
||||
return clean_text[:char_limit]
|
||||
|
||||
async def analyze(self, entries: List[object]) -> BookProfile:
|
||||
"""Generate Book Profile."""
|
||||
sample = self.extract_sample_text(entries)
|
||||
if not sample:
|
||||
return BookProfile(title="Unknown", author="Unknown")
|
||||
|
||||
logger.info("Generating Book Profile from sample text...")
|
||||
|
||||
system_prompt = "You are a senior publishing editor. Analyze the text and output JSON."
|
||||
user_prompt = f"""
|
||||
Please analyze the following book excerpt.
|
||||
Output JSON format:
|
||||
{{
|
||||
"title": "Book Title",
|
||||
"author": "Author Name",
|
||||
"genre": "Genre",
|
||||
"style": "Style description",
|
||||
"keywords": ["keyword1", "keyword2"],
|
||||
"style_guide": "Specific instruction for translator"
|
||||
}}
|
||||
|
||||
Excerpt:
|
||||
{sample}
|
||||
"""
|
||||
try:
|
||||
response = await self.llm_client.raw_chat_completion(system_prompt, user_prompt)
|
||||
json_str = response.strip()
|
||||
# Basic cleanup
|
||||
if "```json" in json_str:
|
||||
json_str = json_str.split("```json")[1].split("```")[0].strip()
|
||||
elif "```" in json_str:
|
||||
json_str = json_str.split("```")[1].split("```")[0].strip()
|
||||
|
||||
data = json.loads(json_str)
|
||||
|
||||
return BookProfile(
|
||||
title=data.get("title", "Unknown"),
|
||||
author=data.get("author", "Unknown"),
|
||||
genre=data.get("genre", "General"),
|
||||
keywords=data.get("keywords", []),
|
||||
style_guide=data.get("style_guide", "")
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Profile generation failed: {e}")
|
||||
return BookProfile(title="Unknown", author="Unknown")
|
||||
@@ -0,0 +1,132 @@
|
||||
import re
|
||||
from typing import List, Dict, Any, Optional
|
||||
from bs4 import BeautifulSoup
|
||||
from loguru import logger
|
||||
|
||||
from src.common.data_model import BookStructure, ManifestEntry, BookProfile
|
||||
from src.preprocessing.format_extractor import FormatExtractor
|
||||
from src.common.utils import setup_logger
|
||||
|
||||
logger = setup_logger("fine_grained_extractor")
|
||||
|
||||
class FineGrainedExtractor:
|
||||
"""
|
||||
Extracts translatable text segments from BookStructure.
|
||||
Uses FormatExtractor for detailed content analysis.
|
||||
"""
|
||||
|
||||
SKIP_TRANSLATION_PATTERNS = [
|
||||
r'index\.x?html',
|
||||
r'bibliography\.x?html',
|
||||
r'endnotes?\.x?html',
|
||||
r'footnotes?\.x?html',
|
||||
r'copyright\.x?html',
|
||||
]
|
||||
|
||||
TOC_PATTERNS = [
|
||||
r'nav\.x?html',
|
||||
r'toc\.x?html',
|
||||
]
|
||||
|
||||
def __init__(self, translate_toc: bool = False):
|
||||
self.translate_toc = translate_toc
|
||||
self.format_extractor = FormatExtractor()
|
||||
|
||||
def extract(self, structure: BookStructure, profile: Optional[BookProfile] = None) -> List[ManifestEntry]:
|
||||
"""
|
||||
Extracts translatable segments from the BookStructure.
|
||||
Iteration follows the spine order.
|
||||
"""
|
||||
logger.info("Starting extraction from BookStructure...")
|
||||
manifest_entries = []
|
||||
|
||||
# Iterate over spine to maintain order
|
||||
for item_id in structure.spine:
|
||||
if item_id not in structure.resources:
|
||||
logger.warning(f"Item ID {item_id} in spine but not in resources.")
|
||||
continue
|
||||
|
||||
resource = structure.resources[item_id]
|
||||
|
||||
# Only process HTML/XHTML
|
||||
if resource.media_type != "application/xhtml+xml" or not resource.content:
|
||||
continue
|
||||
|
||||
file_path = resource.href
|
||||
doc_type = self._classify_document(file_path)
|
||||
|
||||
# Parse HTML
|
||||
soup = BeautifulSoup(resource.content, 'html.parser')
|
||||
target_tags = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li']
|
||||
|
||||
for element in soup.find_all(target_tags):
|
||||
# Ensure element has ID (should have been done by Cleaner)
|
||||
element_id = element.get('id')
|
||||
if not element_id:
|
||||
logger.warning(f"Element in {file_path} missing ID, skipping: {element.name}")
|
||||
continue
|
||||
|
||||
# Check translation eligibility
|
||||
raw_text = element.get_text(separator=' ', strip=True)
|
||||
if not raw_text.strip():
|
||||
continue
|
||||
|
||||
is_decorative = self._is_decorative(raw_text)
|
||||
should_translate = self._should_translate(doc_type, is_decorative, raw_text)
|
||||
|
||||
if should_translate:
|
||||
# Extract detailed format
|
||||
outer_html = str(element)
|
||||
clean_text, text_with_ph, ph_map, p_type, _ = self.format_extractor.extract(outer_html)
|
||||
|
||||
if clean_text.strip() and text_with_ph.strip():
|
||||
entry = ManifestEntry(
|
||||
entry_id=f"{file_path}#{element_id}",
|
||||
file_path=file_path,
|
||||
element_id=element_id,
|
||||
original_text=text_with_ph,
|
||||
placeholders=ph_map,
|
||||
context=p_type
|
||||
)
|
||||
manifest_entries.append(entry)
|
||||
|
||||
logger.info(f"Extracted {len(manifest_entries)} entries in total.")
|
||||
return manifest_entries
|
||||
|
||||
def _classify_document(self, file_name: str) -> str:
|
||||
if not file_name: return 'core'
|
||||
fname = file_name.lower()
|
||||
if any(re.search(p, fname) for p in self.SKIP_TRANSLATION_PATTERNS): return 'skip'
|
||||
if any(re.search(p, fname) for p in self.TOC_PATTERNS): return 'toc'
|
||||
return 'core'
|
||||
|
||||
def _should_translate(self, doc_type: str, is_decorative: bool, text: str = "") -> bool:
|
||||
if is_decorative: return False
|
||||
if self._is_roman_numeral(text): return False
|
||||
if doc_type == 'core': return True
|
||||
if doc_type == 'toc': return self.translate_toc
|
||||
return False
|
||||
|
||||
def _is_roman_numeral(self, text: str) -> bool:
|
||||
text = text.strip().upper()
|
||||
if not text: return False
|
||||
pattern = re.compile(r"^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$")
|
||||
return bool(pattern.match(text))
|
||||
|
||||
def _is_decorative(self, text: str) -> bool:
|
||||
s = text.strip()
|
||||
if not s: return False
|
||||
if not any(c.isalnum() for c in s): return True
|
||||
if len(s) > 20: return False
|
||||
|
||||
patterns = [
|
||||
r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾]+$',
|
||||
r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾\s]+$',
|
||||
]
|
||||
for p in patterns:
|
||||
if re.match(p, s): return True
|
||||
|
||||
unique = set(s.replace(' ', ''))
|
||||
if len(unique) <= 3 and (unique & set('*-_~•◆◇■□▪▫●○◉◎⦿⦾')):
|
||||
return True
|
||||
return False
|
||||
Reference in New Issue
Block a user