139 lines
5.4 KiB
Python
139 lines
5.4 KiB
Python
import re
|
|
from typing import List, Dict, Any, Optional
|
|
from bs4 import BeautifulSoup
|
|
from loguru import logger
|
|
|
|
from src.common.html import parse_html, translation_targets
|
|
from src.common.data_model import BookStructure, ManifestEntry, BookProfile
|
|
from src.preprocessing.format_extractor import FormatExtractor
|
|
from src.common.utils import setup_logger
|
|
|
|
logger = setup_logger("fine_grained_extractor")
|
|
|
|
class FineGrainedExtractor:
|
|
"""
|
|
Extracts translatable text segments from BookStructure.
|
|
Uses FormatExtractor for detailed content analysis.
|
|
"""
|
|
|
|
SKIP_TRANSLATION_PATTERNS = [
|
|
r'index\.x?html',
|
|
r'bibliography\.x?html',
|
|
r'endnotes?\.x?html',
|
|
r'footnotes?\.x?html',
|
|
r'copyright\.x?html',
|
|
]
|
|
|
|
TOC_PATTERNS = [
|
|
r'nav\.x?html',
|
|
r'toc\.x?html',
|
|
]
|
|
|
|
def __init__(self, translate_toc: bool = False, complete_book: bool = False):
|
|
self.translate_toc = translate_toc
|
|
self.complete_book = complete_book
|
|
self.format_extractor = FormatExtractor()
|
|
|
|
def extract(self, structure: BookStructure, profile: Optional[BookProfile] = None) -> List[ManifestEntry]:
|
|
"""
|
|
Extracts translatable segments from the BookStructure.
|
|
Iteration follows the spine order.
|
|
"""
|
|
logger.info("Starting extraction from BookStructure...")
|
|
manifest_entries = []
|
|
|
|
# Iterate over spine to maintain order
|
|
item_ids = list(structure.spine)
|
|
if self.complete_book:
|
|
item_ids.extend(item_id for item_id in structure.resources if item_id not in item_ids)
|
|
for item_id in item_ids:
|
|
if item_id not in structure.resources:
|
|
logger.warning(f"Item ID {item_id} in spine but not in resources.")
|
|
continue
|
|
|
|
resource = structure.resources[item_id]
|
|
|
|
# Only process HTML/XHTML
|
|
if resource.media_type != "application/xhtml+xml" or not resource.content:
|
|
continue
|
|
|
|
file_path = resource.href
|
|
doc_type = self._classify_document(file_path)
|
|
|
|
# Parse HTML
|
|
soup = parse_html(resource.content)
|
|
target_tags = ['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li']
|
|
|
|
for element in translation_targets(soup):
|
|
# Ensure element has ID (should have been done by Cleaner)
|
|
element_id = element.get('id')
|
|
if not element_id:
|
|
logger.warning(f"Element in {file_path} missing ID, skipping: {element.name}")
|
|
continue
|
|
|
|
# Check translation eligibility
|
|
raw_text = element.get_text(separator=' ', strip=True)
|
|
if not raw_text.strip():
|
|
continue
|
|
|
|
is_decorative = self._is_decorative(raw_text)
|
|
should_translate = self._should_translate(doc_type, is_decorative, raw_text)
|
|
|
|
if should_translate:
|
|
# Extract detailed format
|
|
outer_html = str(element)
|
|
clean_text, text_with_ph, ph_map, p_type, _ = self.format_extractor.extract(outer_html)
|
|
|
|
if clean_text.strip() and text_with_ph.strip():
|
|
entry = ManifestEntry(
|
|
entry_id=f"{file_path}#{element_id}",
|
|
file_path=file_path,
|
|
element_id=element_id,
|
|
original_text=text_with_ph,
|
|
placeholders=ph_map,
|
|
context=p_type
|
|
)
|
|
manifest_entries.append(entry)
|
|
|
|
logger.info(f"Extracted {len(manifest_entries)} entries in total.")
|
|
return manifest_entries
|
|
|
|
def _classify_document(self, file_name: str) -> str:
|
|
if not file_name: return 'core'
|
|
fname = file_name.lower()
|
|
if any(re.search(p, fname) for p in self.TOC_PATTERNS): return 'toc'
|
|
if any(re.search(p, fname) for p in self.SKIP_TRANSLATION_PATTERNS):
|
|
return 'core' if self.complete_book else 'skip'
|
|
return 'core'
|
|
|
|
def _should_translate(self, doc_type: str, is_decorative: bool, text: str = "") -> bool:
|
|
if is_decorative: return False
|
|
if self._is_roman_numeral(text): return False
|
|
if doc_type == 'core': return True
|
|
if doc_type == 'toc': return self.translate_toc
|
|
return False
|
|
|
|
def _is_roman_numeral(self, text: str) -> bool:
|
|
text = text.strip().upper()
|
|
if not text: return False
|
|
pattern = re.compile(r"^M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})$")
|
|
return bool(pattern.match(text))
|
|
|
|
def _is_decorative(self, text: str) -> bool:
|
|
s = text.strip()
|
|
if not s: return False
|
|
if not any(c.isalnum() for c in s): return True
|
|
if len(s) > 20: return False
|
|
|
|
patterns = [
|
|
r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾]+$',
|
|
r'^[*\-_~•◆◇■□▪▫●○◉◎⦿⦾\s]+$',
|
|
]
|
|
for p in patterns:
|
|
if re.match(p, s): return True
|
|
|
|
unique = set(s.replace(' ', ''))
|
|
if len(unique) <= 3 and (unique & set('*-_~•◆◇■□▪▫●○◉◎⦿⦾')):
|
|
return True
|
|
return False
|