Constrain automatic aliases and cache terminology matchers
This commit is contained in:
@@ -7,6 +7,7 @@
|
||||
- Bind number-only Part headings such as House of Huawei's “Part I” and normalize “第一部分” to “第一部”.
|
||||
- Keep different entities with shared short aliases (Sun, Fuji, Noda) separate and persistently exclude ambiguous aliases from automatic terminology constraints.
|
||||
- Match single Latin named tokens with case sensitivity (WHO/who, Jobs/jobs, Arm/arm) and version translation fingerprints when matching policy changes.
|
||||
- Restrict automatic single-name aliases to evidenced nicknames and acronyms; exclude month names and nationality words, and cache compiled alias patterns for whole-book validation.
|
||||
- Filter ordinary-word aliases and implicit surname collisions in automatic glossaries; do not force context-dependent single-word senses. Explicit user glossary constraints remain authoritative.
|
||||
|
||||
## 2026-10-08 — clickable Part navigation
|
||||
|
||||
@@ -75,3 +75,5 @@ OCR、扫描页图文提取、版面中间结构与可点击目录的后续大
|
||||
全文运行发现 WHO/who、Jobs/jobs、Arm/arm 被忽略大小写规则混淆,会对普通代词、就业或分部描述强制套用机构/人物译名。单个带大写字母的 Latin 名称标记现按大小写匹配,完整短语和普通术语继续兼容大小写;匹配策略加入缓存指纹版本。累计 54 项测试通过。两书共 17 个疑似受旧规则影响的单元被重新加入翻译队列,其余已验证缓存迁移到新规则。
|
||||
|
||||
自动表进一步收紧:Father/Dad 不作为 Otosan 广告角色的全局别名;carry 不作为 carried interest 的全局别名;WHO 等同时是普通词的短标记不强制套用机构名,即使正文为装饰性全大写。没有被模型显式列为别名的 Sun Yafang 也能揭示 Sun Microsystems 的 Sun 歧义。普通小写单词的不同词义不强制使用一种译法;用户手工表不套用自动筛选规则。累计 55 项测试通过,过滤记录及被重新翻译的单元保存在本地 QA 目录。
|
||||
|
||||
自动短别名仅保留有明确昵称说明的称呼和缩写,完整名称继续校验;月份、国籍词和缺乏消歧依据的单个姓名不作为全书硬约束。Masa 的显式昵称说明仍保留。术语正则表达式缓存避免大量术语反复编译。累计 56 项测试通过。
|
||||
|
||||
@@ -4,13 +4,19 @@ import json
|
||||
import re
|
||||
import copy
|
||||
from pathlib import Path
|
||||
from functools import lru_cache
|
||||
from src.common.utils import setup_logger
|
||||
|
||||
logger = setup_logger('glossary')
|
||||
COMMON_ALIASES = {'who', 'us', 'it', 'he', 'she', 'we', 'you', 'they', 'me', 'them',
|
||||
'our', 'my', 'the', 'a', 'an', 'in', 'on', 'at', 'to', 'of', 'or',
|
||||
'dad', 'father', 'mother', 'mom', 'son', 'brother', 'sister',
|
||||
'family', 'face', 'carry'}
|
||||
'family', 'face', 'carry', 'arm',
|
||||
'january', 'february', 'march', 'april', 'may', 'june', 'july',
|
||||
'august', 'september', 'october', 'november', 'december',
|
||||
'chinese', 'japanese', 'korean', 'russian', 'english', 'french',
|
||||
'german', 'british', 'american', 'indian', 'african', 'asian',
|
||||
'european', 'turkish', 'italian'}
|
||||
|
||||
|
||||
def sanitize_automatic_terms(terms, ambiguous_aliases):
|
||||
@@ -23,6 +29,11 @@ def sanitize_automatic_terms(terms, ambiguous_aliases):
|
||||
keep = []
|
||||
for alias in term.get('aliases', []):
|
||||
unsafe = alias.casefold() in COMMON_ALIASES or re.fullmatch(r'[a-z]+', alias)
|
||||
if re.fullmatch(r'[A-Za-z][A-Za-z.-]*', alias) and alias != term['source']:
|
||||
acronym = bool(re.fullmatch(r'[A-Z][A-Z.-]+', alias))
|
||||
nickname = 'nickname' in term.get('notes', '').casefold() and alias in term.get('notes', '')
|
||||
if not acronym and not nickname:
|
||||
unsafe = True
|
||||
# A surname shared implicitly by another full name is ambiguous,
|
||||
# even when the model did not explicitly propose it as an alias.
|
||||
for other in terms:
|
||||
@@ -48,13 +59,18 @@ def source_digest(entries):
|
||||
|
||||
|
||||
def matches(source, text):
|
||||
return alias_pattern(source).search(text) is not None
|
||||
|
||||
|
||||
@lru_cache(maxsize=8192)
|
||||
def alias_pattern(source):
|
||||
# Word boundaries for Latin aliases prevent Son matching Sony / grandson.
|
||||
escaped = re.escape(source)
|
||||
if source and source[0].isascii() and source[0].isalnum():
|
||||
escaped = r'(?<!\w)' + escaped
|
||||
if source and source[-1].isascii() and source[-1].isalnum():
|
||||
escaped += r'(?!\w)'
|
||||
return re.search(escaped, text, matching_flags(source)) is not None
|
||||
return re.compile(escaped, matching_flags(source))
|
||||
|
||||
|
||||
def matching_flags(alias):
|
||||
@@ -94,12 +110,7 @@ class BookGlossary:
|
||||
excluded = [m.span() for context in term.get('exclude_contexts', [])
|
||||
for m in re.finditer(re.escape(context), text, re.IGNORECASE)]
|
||||
for alias in [term['source']] + term.get('aliases', []):
|
||||
pattern = re.escape(alias)
|
||||
if alias[0].isascii() and alias[0].isalnum():
|
||||
pattern = r'(?<!\w)' + pattern
|
||||
if alias[-1].isascii() and alias[-1].isalnum():
|
||||
pattern += r'(?!\w)'
|
||||
for match in re.finditer(pattern, text, matching_flags(alias)):
|
||||
for match in alias_pattern(alias).finditer(text):
|
||||
start, end = match.span()
|
||||
if any(start < right and end > left for left, right in excluded):
|
||||
continue
|
||||
|
||||
@@ -116,6 +116,18 @@ def test_automatic_glossary_does_not_force_generic_senses_or_implicit_surnames()
|
||||
assert not BookGlossary([{'source': 'Fortune', 'target': '《财富》'}]).validate('Fortune', '据《财富》报道。')
|
||||
|
||||
|
||||
def test_auto_aliases_require_evidence_but_explicit_nicknames_remain():
|
||||
from src.translation.glossary import sanitize_automatic_terms
|
||||
terms, _ = sanitize_automatic_terms([
|
||||
{'source': 'Theresa May', 'target': '特蕾莎·梅', 'aliases': ['May']},
|
||||
{'source': 'University of California, Berkeley', 'target': '加州大学伯克利分校', 'aliases': ['Berkeley', 'UC Berkeley']},
|
||||
{'source': 'Masayoshi Son', 'target': '孙正义', 'aliases': ['Masa'], 'notes': 'Masa is his nickname.'}], set())
|
||||
glossary = BookGlossary(terms)
|
||||
assert not glossary.validate('In May he stayed at the Berkeley hotel.', '五月他住在伯克利酒店。')
|
||||
assert glossary.validate('Masa studied at UC Berkeley.', '他去上学。')
|
||||
assert glossary.validate('Theresa May visited.', '五月来访。')
|
||||
|
||||
|
||||
@pytest.mark.parametrize('html', [
|
||||
'<p>Hello <em><b>world</b></em>.</p>',
|
||||
'<p>The angle φ = 10, and φ1φ is literal mathematical text.</p>',
|
||||
|
||||
Reference in New Issue
Block a user