Filter unsafe automatic terminology constraints before book translation
This commit is contained in:
Kai Tan
@@ -7,6 +7,7 @@
|
||||
- Bind number-only Part headings such as House of Huawei's “Part I” and normalize “第一部分” to “第一部”.
|
||||
- Keep different entities with shared short aliases (Sun, Fuji, Noda) separate and persistently exclude ambiguous aliases from automatic terminology constraints.
|
||||
- Match single Latin named tokens with case sensitivity (WHO/who, Jobs/jobs, Arm/arm) and version translation fingerprints when matching policy changes.
|
||||
- Filter ordinary-word aliases and implicit surname collisions in automatic glossaries; do not force context-dependent single-word senses. Explicit user glossary constraints remain authoritative.
|
||||
|
||||
## 2026-10-08 — clickable Part navigation
|
||||
|
||||
|
||||
@@ -73,3 +73,5 @@ OCR、扫描页图文提取、版面中间结构与可点击目录的后续大
|
||||
术语复核发现，Sun Microsystems／David Sun、Fuji Bank／Fuji TV、Kazuo Noda／Yoshihiko Noda 分别共享 Sun、Fuji、Noda 短别名。现保留各完整名称及其译名，同时从双方删除歧义短别名，并持久化禁止再次自动加入这些别名。裸姓氏留给模型按上下文消歧，明确全名仍做术语校验。累计 52 项测试通过。House of Huawei 新绑定去掉了 3 个重复的独立 Part 导航标签，完整清单为 2,837 单元；缓存迁移核对了旧源摘要及已扫描批次文字完全相同后保留扫描进度。
|
||||
|
||||
全文运行发现 WHO/who、Jobs/jobs、Arm/arm 被忽略大小写规则混淆，会对普通代词、就业或分部描述强制套用机构／人物译名。单个带大写字母的 Latin 名称标记现按大小写匹配，完整短语和普通术语继续兼容大小写；匹配策略加入缓存指纹版本。累计 54 项测试通过。两书共 17 个疑似受旧规则影响的单元被重新加入翻译队列，其余已验证缓存迁移到新规则。
|
||||
|
||||
自动表进一步收紧：Father/Dad 不作为 Otosan 广告角色的全局别名；carry 不作为 carried interest 的全局别名；WHO 等同时是普通词的短标记不强制套用机构名，即使正文为装饰性全大写。没有被模型显式列为别名的 Sun Yafang 也能揭示 Sun Microsystems 的 Sun 歧义。普通小写单词的不同词义不强制使用一种译法；用户手工表不套用自动筛选规则。累计 55 项测试通过，过滤记录及被重新翻译的单元保存在本地 QA 目录。
|
||||
|
||||
@@ -7,6 +7,39 @@ from pathlib import Path
|
||||
from src.common.utils import setup_logger
|
||||
|
||||
logger = setup_logger('glossary')
|
||||
COMMON_ALIASES = {'who', 'us', 'it', 'he', 'she', 'we', 'you', 'they', 'me', 'them',
|
||||
'our', 'my', 'the', 'a', 'an', 'in', 'on', 'at', 'to', 'of', 'or',
|
||||
'dad', 'father', 'mother', 'mom', 'son', 'brother', 'sister',
|
||||
'family', 'face', 'carry'}
|
||||
|
||||
|
||||
def sanitize_automatic_terms(terms, ambiguous_aliases):
|
||||
"""Only automatic terminology is filtered; explicit user glossaries stay authoritative."""
|
||||
terms = copy.deepcopy(terms)
|
||||
excluded = []
|
||||
for term in terms:
|
||||
if re.fullmatch(r'[a-z]+', term['source']) or term['source'].casefold() in COMMON_ALIASES:
|
||||
term['enforce'] = False
|
||||
keep = []
|
||||
for alias in term.get('aliases', []):
|
||||
unsafe = alias.casefold() in COMMON_ALIASES or re.fullmatch(r'[a-z]+', alias)
|
||||
# A surname shared implicitly by another full name is ambiguous,
|
||||
# even when the model did not explicitly propose it as an alias.
|
||||
for other in terms:
|
||||
words = other['source'].split()
|
||||
if (other['target'] != term['target'] and 2 <= len(words) <= 4
|
||||
and all(re.fullmatch(r'[A-Z][A-Za-z.-]*', w) for w in words)
|
||||
and alias.casefold() in {words[0].casefold(), words[-1].casefold()}):
|
||||
ambiguous_aliases.add(alias.casefold())
|
||||
if unsafe or alias.casefold() in ambiguous_aliases:
|
||||
excluded.append({'source': term['source'], 'alias': alias})
|
||||
else:
|
||||
keep.append(alias)
|
||||
term['aliases'] = keep
|
||||
# A later entity may reveal ambiguity in an earlier alias.
|
||||
for term in terms:
|
||||
term['aliases'] = [a for a in term['aliases'] if a.casefold() not in ambiguous_aliases]
|
||||
return terms, excluded
|
||||
|
||||
|
||||
def source_digest(entries):
|
||||
@@ -86,6 +119,8 @@ class BookGlossary:
|
||||
result = {}
|
||||
for _, _, index, alias in self._active_terms(text):
|
||||
term = self.terms[index]
|
||||
if not term.get('enforce', True):
|
||||
continue
|
||||
note = term.get('notes', '')
|
||||
if ' ' not in alias and alias.isascii():
|
||||
note += '; Apply only to this named entity, not a family/clan/surname expression.'
|
||||
@@ -97,7 +132,10 @@ class BookGlossary:
|
||||
errors = []
|
||||
for index in {x[2] for x in self._active_terms(source)}:
|
||||
term = self.terms[index]
|
||||
if term['target'] not in translation:
|
||||
if not term.get('enforce', True):
|
||||
continue
|
||||
target = term['target'].strip('“”‘’"\'《》「」')
|
||||
if target not in translation:
|
||||
errors.append(f"required term: {term['source']} -> {term['target']}")
|
||||
for forbidden in term.get('forbidden_targets', []):
|
||||
if forbidden in translation:
|
||||
@@ -158,7 +196,16 @@ async def prepare_glossary(entries, llm, path, override=None, auto=True, chunk_c
|
||||
data = json.loads(path.read_text())
|
||||
if data.get('source_digest') == digest:
|
||||
if data.get('scan_complete', True):
|
||||
return BookGlossary(data['terms'])
|
||||
ambiguous_aliases = set(data.get('ambiguous_aliases', []))
|
||||
filtered, excluded = sanitize_automatic_terms(data['terms'], ambiguous_aliases)
|
||||
if filtered != data['terms']:
|
||||
data['terms'] = filtered
|
||||
data['excluded_automatic_aliases'] = data.get('excluded_automatic_aliases', []) + excluded
|
||||
data['ambiguous_aliases'] = sorted(ambiguous_aliases)
|
||||
temp = path.with_suffix('.tmp')
|
||||
temp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
temp.replace(path)
|
||||
return BookGlossary(filtered)
|
||||
if data.get('chunk_chars') == chunk_chars and data.get('model') == getattr(llm, 'model', ''):
|
||||
terms, conflicts, start = data['terms'], data.get('conflicts', []), data.get('next_chunk', 0)
|
||||
ambiguous_aliases = set(data.get('ambiguous_aliases', []))
|
||||
@@ -195,4 +242,11 @@ async def prepare_glossary(entries, llm, path, override=None, auto=True, chunk_c
|
||||
ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
temp.replace(path)
|
||||
logger.info('Terminology scan %s/%s: %s terms, %s conflicts recorded', index + 1, len(chunks), len(terms), len(conflicts))
|
||||
terms, excluded = sanitize_automatic_terms(terms, ambiguous_aliases)
|
||||
if chunks:
|
||||
data = json.loads(path.read_text())
|
||||
data.update(terms=terms, ambiguous_aliases=sorted(ambiguous_aliases), excluded_automatic_aliases=excluded)
|
||||
temp = path.with_suffix('.tmp')
|
||||
temp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
temp.replace(path)
|
||||
return BookGlossary(terms)
|
||||
|
||||
@@ -97,6 +97,25 @@ def test_name_aliases_do_not_match_common_lowercase_words():
|
||||
assert set(glossary.relevant([entry('<p>Jobs invested in ARM.</p>')])) == {'Jobs', 'ARM'}
|
||||
|
||||
|
||||
def test_automatic_glossary_does_not_force_generic_senses_or_implicit_surnames():
|
||||
from src.translation.glossary import sanitize_automatic_terms
|
||||
terms, excluded = sanitize_automatic_terms([
|
||||
{'source': 'Otosan', 'target': '爸爸', 'aliases': ['Father']},
|
||||
{'source': 'face', 'target': '面子'},
|
||||
{'source': 'carried interest', 'target': '绩效分成', 'aliases': ['carry']},
|
||||
{'source': 'World Health Organization', 'target': '世界卫生组织', 'aliases': ['WHO']},
|
||||
{'source': 'Sun Microsystems', 'target': '太阳微系统公司', 'aliases': ['Sun']},
|
||||
{'source': 'Sun Yafang', 'target': '孙亚芳'}], set())
|
||||
assert excluded
|
||||
glossary = BookGlossary(terms)
|
||||
assert not glossary.validate('Father had a smile on his face and could carry a bag.', '父亲面带微笑，可以提包。')
|
||||
assert not glossary.validate('WHO AM I?', '我是谁？')
|
||||
assert not glossary.validate('Sun Yafang visited. Sun gave a talk.', '孙亚芳到访并发表演讲。')
|
||||
assert glossary.validate('carried interest', '费用')
|
||||
assert glossary.validate('Otosan', '广告')
|
||||
assert not BookGlossary([{'source': 'Fortune', 'target': '《财富》'}]).validate('Fortune', '据《财富》报道。')
|
||||
|
||||
|
||||
@pytest.mark.parametrize('html', [
|
||||
'<p>Hello <em><b>world</b></em>.</p>',
|
||||
'<p>The angle φ = 10, and φ1φ is literal mathematical text.</p>',
|
||||
|
||||
Reference in New Issue
Block a user