diff --git a/CHANGELOG.md b/CHANGELOG.md index 174ff91..ca5a9fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Bind number-only Part headings such as House of Huawei's “Part I” and normalize “第一部分” to “第一部”. - Keep different entities with shared short aliases (Sun, Fuji, Noda) separate and persistently exclude ambiguous aliases from automatic terminology constraints. - Match single Latin named tokens with case sensitivity (WHO/who, Jobs/jobs, Arm/arm) and version translation fingerprints when matching policy changes. +- Restrict automatic single-name aliases to evidenced nicknames and acronyms; exclude month names and nationality words, and cache compiled alias patterns for whole-book validation. - Filter ordinary-word aliases and implicit surname collisions in automatic glossaries; do not force context-dependent single-word senses. Explicit user glossary constraints remain authoritative. ## 2026-10-08 — clickable Part navigation diff --git a/doc/QA_2026-10-08.md b/doc/QA_2026-10-08.md index b423470..4a10b91 100644 --- a/doc/QA_2026-10-08.md +++ b/doc/QA_2026-10-08.md @@ -75,3 +75,5 @@ OCR、扫描页图文提取、版面中间结构与可点击目录的后续大 全文运行发现 WHO/who、Jobs/jobs、Arm/arm 被忽略大小写规则混淆,会对普通代词、就业或分部描述强制套用机构/人物译名。单个带大写字母的 Latin 名称标记现按大小写匹配,完整短语和普通术语继续兼容大小写;匹配策略加入缓存指纹版本。累计 54 项测试通过。两书共 17 个疑似受旧规则影响的单元被重新加入翻译队列,其余已验证缓存迁移到新规则。 自动表进一步收紧:Father/Dad 不作为 Otosan 广告角色的全局别名;carry 不作为 carried interest 的全局别名;WHO 等同时是普通词的短标记不强制套用机构名,即使正文为装饰性全大写。没有被模型显式列为别名的 Sun Yafang 也能揭示 Sun Microsystems 的 Sun 歧义。普通小写单词的不同词义不强制使用一种译法;用户手工表不套用自动筛选规则。累计 55 项测试通过,过滤记录及被重新翻译的单元保存在本地 QA 目录。 + +自动短别名仅保留有明确昵称说明的称呼和缩写,完整名称继续校验;月份、国籍词和缺乏消歧依据的单个姓名不作为全书硬约束。Masa 的显式昵称说明仍保留。术语正则表达式缓存避免大量术语反复编译。累计 56 项测试通过。 diff --git a/src/translation/glossary.py b/src/translation/glossary.py index ae4cb24..20e37d3 100644 --- a/src/translation/glossary.py +++ b/src/translation/glossary.py @@ -4,13 +4,19 @@ import json import re import copy from pathlib import Path +from functools import lru_cache from src.common.utils import setup_logger logger = setup_logger('glossary') COMMON_ALIASES = {'who', 'us', 'it', 'he', 'she', 'we', 'you', 'they', 'me', 'them', 'our', 'my', 'the', 'a', 'an', 'in', 'on', 'at', 'to', 'of', 'or', 'dad', 'father', 'mother', 'mom', 'son', 'brother', 'sister', - 'family', 'face', 'carry'} + 'family', 'face', 'carry', 'arm', + 'january', 'february', 'march', 'april', 'may', 'june', 'july', + 'august', 'september', 'october', 'november', 'december', + 'chinese', 'japanese', 'korean', 'russian', 'english', 'french', + 'german', 'british', 'american', 'indian', 'african', 'asian', + 'european', 'turkish', 'italian'} def sanitize_automatic_terms(terms, ambiguous_aliases): @@ -23,6 +29,11 @@ def sanitize_automatic_terms(terms, ambiguous_aliases): keep = [] for alias in term.get('aliases', []): unsafe = alias.casefold() in COMMON_ALIASES or re.fullmatch(r'[a-z]+', alias) + if re.fullmatch(r'[A-Za-z][A-Za-z.-]*', alias) and alias != term['source']: + acronym = bool(re.fullmatch(r'[A-Z][A-Z.-]+', alias)) + nickname = 'nickname' in term.get('notes', '').casefold() and alias in term.get('notes', '') + if not acronym and not nickname: + unsafe = True # A surname shared implicitly by another full name is ambiguous, # even when the model did not explicitly propose it as an alias. for other in terms: @@ -48,13 +59,18 @@ def source_digest(entries): def matches(source, text): + return alias_pattern(source).search(text) is not None + + +@lru_cache(maxsize=8192) +def alias_pattern(source): # Word boundaries for Latin aliases prevent Son matching Sony / grandson. escaped = re.escape(source) if source and source[0].isascii() and source[0].isalnum(): escaped = r'(? left for left, right in excluded): continue diff --git a/tests/test_regressions.py b/tests/test_regressions.py index a2ef386..f1d667e 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -116,6 +116,18 @@ def test_automatic_glossary_does_not_force_generic_senses_or_implicit_surnames() assert not BookGlossary([{'source': 'Fortune', 'target': '《财富》'}]).validate('Fortune', '据《财富》报道。') +def test_auto_aliases_require_evidence_but_explicit_nicknames_remain(): + from src.translation.glossary import sanitize_automatic_terms + terms, _ = sanitize_automatic_terms([ + {'source': 'Theresa May', 'target': '特蕾莎·梅', 'aliases': ['May']}, + {'source': 'University of California, Berkeley', 'target': '加州大学伯克利分校', 'aliases': ['Berkeley', 'UC Berkeley']}, + {'source': 'Masayoshi Son', 'target': '孙正义', 'aliases': ['Masa'], 'notes': 'Masa is his nickname.'}], set()) + glossary = BookGlossary(terms) + assert not glossary.validate('In May he stayed at the Berkeley hotel.', '五月他住在伯克利酒店。') + assert glossary.validate('Masa studied at UC Berkeley.', '他去上学。') + assert glossary.validate('Theresa May visited.', '五月来访。') + + @pytest.mark.parametrize('html', [ '
Hello world.
', 'The angle φ = 10, and φ1φ is literal mathematical text.
',