Constrain automatic aliases and cache terminology matchers

This commit is contained in:
Kai Tan
2026-10-08 09:21:47 +08:00
parent 2c3feb37c9
commit 4f124a0335
4 changed files with 34 additions and 8 deletions
+1
View File
@@ -7,6 +7,7 @@
- Bind number-only Part headings such as House of Huawei's “Part I” and normalize “第一部分” to “第一部”.
- Keep different entities with shared short aliases (Sun, Fuji, Noda) separate and persistently exclude ambiguous aliases from automatic terminology constraints.
- Match single Latin named tokens with case sensitivity (WHO/who, Jobs/jobs, Arm/arm) and version translation fingerprints when matching policy changes.
- Restrict automatic single-name aliases to evidenced nicknames and acronyms; exclude month names and nationality words, and cache compiled alias patterns for whole-book validation.
- Filter ordinary-word aliases and implicit surname collisions in automatic glossaries; do not force context-dependent single-word senses. Explicit user glossary constraints remain authoritative.
## 2026-10-08 — clickable Part navigation
+2
View File
@@ -75,3 +75,5 @@ OCR、扫描页图文提取、版面中间结构与可点击目录的后续大
全文运行发现 WHO/who、Jobs/jobs、Arm/arm 被忽略大小写规则混淆,会对普通代词、就业或分部描述强制套用机构/人物译名。单个带大写字母的 Latin 名称标记现按大小写匹配,完整短语和普通术语继续兼容大小写;匹配策略加入缓存指纹版本。累计 54 项测试通过。两书共 17 个疑似受旧规则影响的单元被重新加入翻译队列,其余已验证缓存迁移到新规则。
自动表进一步收紧:Father/Dad 不作为 Otosan 广告角色的全局别名;carry 不作为 carried interest 的全局别名;WHO 等同时是普通词的短标记不强制套用机构名,即使正文为装饰性全大写。没有被模型显式列为别名的 Sun Yafang 也能揭示 Sun Microsystems 的 Sun 歧义。普通小写单词的不同词义不强制使用一种译法;用户手工表不套用自动筛选规则。累计 55 项测试通过,过滤记录及被重新翻译的单元保存在本地 QA 目录。
自动短别名仅保留有明确昵称说明的称呼和缩写,完整名称继续校验;月份、国籍词和缺乏消歧依据的单个姓名不作为全书硬约束。Masa 的显式昵称说明仍保留。术语正则表达式缓存避免大量术语反复编译。累计 56 项测试通过。
+19 -8
View File
@@ -4,13 +4,19 @@ import json
import re
import copy
from pathlib import Path
from functools import lru_cache
from src.common.utils import setup_logger
logger = setup_logger('glossary')
COMMON_ALIASES = {'who', 'us', 'it', 'he', 'she', 'we', 'you', 'they', 'me', 'them',
'our', 'my', 'the', 'a', 'an', 'in', 'on', 'at', 'to', 'of', 'or',
'dad', 'father', 'mother', 'mom', 'son', 'brother', 'sister',
'family', 'face', 'carry'}
'family', 'face', 'carry', 'arm',
'january', 'february', 'march', 'april', 'may', 'june', 'july',
'august', 'september', 'october', 'november', 'december',
'chinese', 'japanese', 'korean', 'russian', 'english', 'french',
'german', 'british', 'american', 'indian', 'african', 'asian',
'european', 'turkish', 'italian'}
def sanitize_automatic_terms(terms, ambiguous_aliases):
@@ -23,6 +29,11 @@ def sanitize_automatic_terms(terms, ambiguous_aliases):
keep = []
for alias in term.get('aliases', []):
unsafe = alias.casefold() in COMMON_ALIASES or re.fullmatch(r'[a-z]+', alias)
if re.fullmatch(r'[A-Za-z][A-Za-z.-]*', alias) and alias != term['source']:
acronym = bool(re.fullmatch(r'[A-Z][A-Z.-]+', alias))
nickname = 'nickname' in term.get('notes', '').casefold() and alias in term.get('notes', '')
if not acronym and not nickname:
unsafe = True
# A surname shared implicitly by another full name is ambiguous,
# even when the model did not explicitly propose it as an alias.
for other in terms:
@@ -48,13 +59,18 @@ def source_digest(entries):
def matches(source, text):
return alias_pattern(source).search(text) is not None
@lru_cache(maxsize=8192)
def alias_pattern(source):
# Word boundaries for Latin aliases prevent Son matching Sony / grandson.
escaped = re.escape(source)
if source and source[0].isascii() and source[0].isalnum():
escaped = r'(?<!\w)' + escaped
if source and source[-1].isascii() and source[-1].isalnum():
escaped += r'(?!\w)'
return re.search(escaped, text, matching_flags(source)) is not None
return re.compile(escaped, matching_flags(source))
def matching_flags(alias):
@@ -94,12 +110,7 @@ class BookGlossary:
excluded = [m.span() for context in term.get('exclude_contexts', [])
for m in re.finditer(re.escape(context), text, re.IGNORECASE)]
for alias in [term['source']] + term.get('aliases', []):
pattern = re.escape(alias)
if alias[0].isascii() and alias[0].isalnum():
pattern = r'(?<!\w)' + pattern
if alias[-1].isascii() and alias[-1].isalnum():
pattern += r'(?!\w)'
for match in re.finditer(pattern, text, matching_flags(alias)):
for match in alias_pattern(alias).finditer(text):
start, end = match.span()
if any(start < right and end > left for left, right in excluded):
continue
+12
View File
@@ -116,6 +116,18 @@ def test_automatic_glossary_does_not_force_generic_senses_or_implicit_surnames()
assert not BookGlossary([{'source': 'Fortune', 'target': '《财富》'}]).validate('Fortune', '据《财富》报道。')
def test_auto_aliases_require_evidence_but_explicit_nicknames_remain():
from src.translation.glossary import sanitize_automatic_terms
terms, _ = sanitize_automatic_terms([
{'source': 'Theresa May', 'target': '特蕾莎·梅', 'aliases': ['May']},
{'source': 'University of California, Berkeley', 'target': '加州大学伯克利分校', 'aliases': ['Berkeley', 'UC Berkeley']},
{'source': 'Masayoshi Son', 'target': '孙正义', 'aliases': ['Masa'], 'notes': 'Masa is his nickname.'}], set())
glossary = BookGlossary(terms)
assert not glossary.validate('In May he stayed at the Berkeley hotel.', '五月他住在伯克利酒店。')
assert glossary.validate('Masa studied at UC Berkeley.', '他去上学。')
assert glossary.validate('Theresa May visited.', '五月来访。')
@pytest.mark.parametrize('html', [
'<p>Hello <em><b>world</b></em>.</p>',
'<p>The angle φ = 10, and φ1φ is literal mathematical text.</p>',