Avoid matching named aliases against ordinary lowercase words
This commit is contained in:
@@ -6,6 +6,7 @@
|
||||
- Checkpoint whole-book terminology scans and record conflicting model proposals while retaining established translations.
|
||||
- Bind number-only Part headings such as House of Huawei's “Part I” and normalize “第一部分” to “第一部”.
|
||||
- Keep different entities with shared short aliases (Sun, Fuji, Noda) separate and persistently exclude ambiguous aliases from automatic terminology constraints.
|
||||
- Match single Latin named tokens with case sensitivity (WHO/who, Jobs/jobs, Arm/arm) and version translation fingerprints when matching policy changes.
|
||||
|
||||
## 2026-10-08 — clickable Part navigation
|
||||
|
||||
|
||||
@@ -71,3 +71,5 @@ OCR、扫描页图文提取、版面中间结构与可点击目录的后续大
|
||||
追加验证:两本书完整模式共 6,782 个单元,原文格式往返失败 0、新增结构问题 0。《House of Huawei》的 Part I/II/III 只有编号,没有副标题,原绑定算法去掉编号后得到空标题,导致未绑定;现允许明确的 h1–h6 纯编号标题匹配,同文字的普通段落仍不匹配。目录和正文复用同一译文,并将“第一部分”规范为“第一部”。累计 51 项测试通过。
|
||||
|
||||
术语复核发现,Sun Microsystems/David Sun、Fuji Bank/Fuji TV、Kazuo Noda/Yoshihiko Noda 分别共享 Sun、Fuji、Noda 短别名。现保留各完整名称及其译名,同时从双方删除歧义短别名,并持久化禁止再次自动加入这些别名。裸姓氏留给模型按上下文消歧,明确全名仍做术语校验。累计 52 项测试通过。House of Huawei 新绑定去掉了 3 个重复的独立 Part 导航标签,完整清单为 2,837 单元;缓存迁移核对了旧源摘要及已扫描批次文字完全相同后保留扫描进度。
|
||||
|
||||
全文运行发现 WHO/who、Jobs/jobs、Arm/arm 被忽略大小写规则混淆,会对普通代词、就业或分部描述强制套用机构/人物译名。单个带大写字母的 Latin 名称标记现按大小写匹配,完整短语和普通术语继续兼容大小写;匹配策略加入缓存指纹版本。累计 54 项测试通过。两书共 17 个疑似受旧规则影响的单元被重新加入翻译队列,其余已验证缓存迁移到新规则。
|
||||
|
||||
@@ -21,7 +21,14 @@ def matches(source, text):
|
||||
escaped = r'(?<!\w)' + escaped
|
||||
if source and source[-1].isascii() and source[-1].isalnum():
|
||||
escaped += r'(?!\w)'
|
||||
return re.search(escaped, text, re.IGNORECASE) is not None
|
||||
return re.search(escaped, text, matching_flags(source)) is not None
|
||||
|
||||
|
||||
def matching_flags(alias):
|
||||
# WHO/who, US/us, Jobs/jobs and Arm/arm are different words. Single Latin
|
||||
# named tokens keep their case; full phrases retain tolerant matching.
|
||||
named_token = alias.isascii() and not re.search(r'\s', alias) and any(c.isupper() for c in alias)
|
||||
return 0 if named_token else re.IGNORECASE
|
||||
|
||||
|
||||
class BookGlossary:
|
||||
@@ -59,7 +66,7 @@ class BookGlossary:
|
||||
pattern = r'(?<!\w)' + pattern
|
||||
if alias[-1].isascii() and alias[-1].isalnum():
|
||||
pattern += r'(?!\w)'
|
||||
for match in re.finditer(pattern, text, re.IGNORECASE):
|
||||
for match in re.finditer(pattern, text, matching_flags(alias)):
|
||||
start, end = match.span()
|
||||
if any(start < right and end > left for left, right in excluded):
|
||||
continue
|
||||
@@ -98,7 +105,7 @@ class BookGlossary:
|
||||
return errors
|
||||
|
||||
def fingerprint(self):
|
||||
return hashlib.sha256(json.dumps(self.terms, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
return hashlib.sha256(json.dumps(['case-sensitive-named-tokens-v3', self.terms], sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
|
||||
|
||||
def merge_proposal(terms, proposal, conflicts, ambiguous_aliases):
|
||||
|
||||
@@ -78,6 +78,25 @@ def test_shared_surname_is_removed_from_both_distinct_entities():
|
||||
assert glossary.validate('Sun Microsystems supplied computers.', '孙先生提供了电脑。')
|
||||
|
||||
|
||||
def test_acronyms_do_not_match_ordinary_pronouns():
|
||||
glossary = BookGlossary([
|
||||
{'source': 'World Health Organization', 'target': '世界卫生组织', 'aliases': ['WHO']},
|
||||
{'source': 'United States', 'target': '美国', 'aliases': ['US']},
|
||||
{'source': 'information technology', 'target': '信息技术', 'aliases': ['IT']}])
|
||||
assert not glossary.validate('The people who helped us did it.', '帮助我们的人做了这件事。')
|
||||
assert not glossary.relevant([entry('<p>Who helped us do it?</p>')])
|
||||
assert glossary.validate('WHO met the US IT team.', '他们会见了团队。')
|
||||
assert matches('WHO', 'The WHO report') and not matches('WHO', 'people who helped')
|
||||
|
||||
|
||||
def test_name_aliases_do_not_match_common_lowercase_words():
|
||||
glossary = BookGlossary([{'source': 'Steve Jobs', 'target': '乔布斯', 'aliases': ['Jobs']},
|
||||
{'source': 'Arm Holdings', 'target': '安谋', 'aliases': ['Arm', 'ARM']}])
|
||||
assert not glossary.validate('More jobs in its US arm.', '其美国分部增加就业。')
|
||||
assert glossary.validate('Jobs bought Arm.', '他收购了这家公司。')
|
||||
assert set(glossary.relevant([entry('<p>Jobs invested in ARM.</p>')])) == {'Jobs', 'ARM'}
|
||||
|
||||
|
||||
@pytest.mark.parametrize('html', [
|
||||
'<p>Hello <em><b>world</b></em>.</p>',
|
||||
'<p>The angle φ = 10, and φ1φ is literal mathematical text.</p>',
|
||||
|
||||
Reference in New Issue
Block a user