Disambiguate shared glossary aliases without discarding full entity names

This commit is contained in:
Kai Tan
2026-10-08 08:42:17 +08:00
parent 0e4aec82d8
commit dd5a99dc23
4 changed files with 60 additions and 22 deletions
+1
View File
@@ -5,6 +5,7 @@
- Add explicit configuration/credential file options and a Zenmux Sonnet 5.5 configuration without credentials.
- Checkpoint whole-book terminology scans and record conflicting model proposals while retaining established translations.
- Bind number-only Part headings such as House of Huawei's “Part I” and normalize “第一部分” to “第一部”.
- Keep different entities with shared short aliases (Sun, Fuji, Noda) separate and persistently exclude ambiguous aliases from automatic terminology constraints.
## 2026-10-08 — clickable Part navigation
+2
View File
@@ -69,3 +69,5 @@ OCR、扫描页图文提取、版面中间结构与可点击目录的后续大
真实 Sonnet 5.5 全书运行中发现,单批术语返回同一别名的不同译名会中止全书。现在自动扫描逐条校验建议,保留已确定译名并记录冲突,每批原子保存结果及进度。手工术语表的冲突仍严格报错。两本书的实际运行结果将在完成后单独记录;本节不代表已完成全书译文验收。
追加验证:两本书完整模式共 6,782 个单元,原文格式往返失败 0、新增结构问题 0。《House of Huawei》的 Part I/II/III 只有编号,没有副标题,原绑定算法去掉编号后得到空标题,导致未绑定;现允许明确的 h1–h6 纯编号标题匹配,同文字的普通段落仍不匹配。目录和正文复用同一译文,并将“第一部分”规范为“第一部”。累计 51 项测试通过。
术语复核发现,Sun Microsystems/David Sun、Fuji Bank/Fuji TV、Kazuo Noda/Yoshihiko Noda 分别共享 Sun、Fuji、Noda 短别名。现保留各完整名称及其译名,同时从双方删除歧义短别名,并持久化禁止再次自动加入这些别名。裸姓氏留给模型按上下文消歧,明确全名仍做术语校验。累计 52 项测试通过。House of Huawei 新绑定去掉了 3 个重复的独立 Part 导航标签,完整清单为 2,837 单元;缓存迁移核对了旧源摘要及已扫描批次文字完全相同后保留扫描进度。
+43 -22
View File
@@ -101,11 +101,50 @@ class BookGlossary:
return hashlib.sha256(json.dumps(self.terms, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
def merge_proposal(terms, proposal, conflicts, ambiguous_aliases):
"""Keep distinct full names; never force a shared surname onto one entity."""
try:
term = copy.deepcopy(BookGlossary([proposal]).terms[0])
except (ValueError, AttributeError) as exc:
conflicts.append({'proposal': proposal, 'reason': str(exc)})
return terms
if term['source'].casefold() in ambiguous_aliases:
conflicts.append({'proposal': term, 'reason': 'ambiguous source name'})
return terms
term['aliases'] = [a for a in term.get('aliases', []) if a.casefold() not in ambiguous_aliases]
candidate = copy.deepcopy(terms)
existing = next((t for t in candidate if term['source'].casefold() in
{a.casefold() for a in [t['source']] + t.get('aliases', [])}), None)
if existing and existing['target'] != term['target']:
conflicts.append({'existing': copy.deepcopy(existing), 'proposal': term, 'reason': 'canonical translation conflict'})
return terms
for other in candidate:
if other is existing or other['target'] == term['target']:
continue
shared = ({a.casefold() for a in term['aliases']} &
{a.casefold() for a in [other['source']] + other.get('aliases', [])})
for alias in shared:
conflicts.append({'existing': copy.deepcopy(other), 'proposal': copy.deepcopy(term),
'reason': 'shared alias between distinct entities', 'alias': alias})
term['aliases'] = [a for a in term['aliases'] if a.casefold() != alias]
if alias != other['source'].casefold():
ambiguous_aliases.add(alias)
for stored in candidate:
stored['aliases'] = [a for a in stored.get('aliases', []) if a.casefold() != alias]
if existing:
existing['aliases'] = sorted(set(existing.get('aliases', []) + term['aliases']) - {existing['source']})
else:
candidate.append(term)
BookGlossary(candidate)
return candidate
async def prepare_glossary(entries, llm, path, override=None, auto=True, chunk_chars=24000):
"""Scan the whole book before translation; never race glossary updates."""
path = Path(path)
digest = source_digest(entries)
terms, conflicts, start = [], [], 0
ambiguous_aliases = set()
if override:
return BookGlossary.load(override)
if path.exists():
@@ -115,6 +154,7 @@ async def prepare_glossary(entries, llm, path, override=None, auto=True, chunk_c
return BookGlossary(data['terms'])
if data.get('chunk_chars') == chunk_chars and data.get('model') == getattr(llm, 'model', ''):
terms, conflicts, start = data['terms'], data.get('conflicts', []), data.get('next_chunk', 0)
ambiguous_aliases = set(data.get('ambiguous_aliases', []))
if not auto:
return BookGlossary()
chunks, current, size = [], [], 0
@@ -139,31 +179,12 @@ async def prepare_glossary(entries, llm, path, override=None, auto=True, chunk_c
# Validate proposals individually: a contradictory model suggestion must
# not discard the already frozen terms or stop the entire book scan.
for proposal in json.loads(response)['terms']:
try:
term = BookGlossary([proposal]).terms[0]
except (ValueError, AttributeError) as exc:
conflicts.append({'proposal': proposal, 'reason': str(exc)})
continue
aliases = {a.casefold() for a in [term['source']] + term.get('aliases', [])}
existing = next((t for t in terms if aliases & {a.casefold() for a in [t['source']] + t.get('aliases', [])}), None)
if existing and existing['target'] != term['target']:
conflicts.append({'existing': existing, 'proposal': term})
continue
candidate = copy.deepcopy(terms)
if existing:
candidate[terms.index(existing)]['aliases'] = sorted(set(existing.get('aliases', []) + [term['source']] + term.get('aliases', [])) - {existing['source']})
else:
candidate.append(term)
try:
BookGlossary(candidate)
except ValueError as exc:
conflicts.append({'proposal': term, 'reason': str(exc)})
continue
terms = candidate
terms = merge_proposal(terms, proposal, conflicts, ambiguous_aliases)
temp = path.with_suffix('.tmp')
temp.write_text(json.dumps({'source_digest': digest, 'terms': terms, 'conflicts': conflicts,
'scan_complete': index + 1 == len(chunks), 'next_chunk': index + 1,
'chunk_chars': chunk_chars, 'model': getattr(llm, 'model', '')},
'chunk_chars': chunk_chars, 'model': getattr(llm, 'model', ''),
'ambiguous_aliases': sorted(ambiguous_aliases)},
ensure_ascii=False, indent=2), encoding='utf-8')
temp.replace(path)
logger.info('Terminology scan %s/%s: %s terms, %s conflicts recorded', index + 1, len(chunks), len(terms), len(conflicts))
+14
View File
@@ -64,6 +64,20 @@ def test_zenmux_credentials_override_unrelated_openai_key(tmp_path, monkeypatch)
assert load_global_config(config)['llm']['api_key'] == 'test-zenmux-key'
def test_shared_surname_is_removed_from_both_distinct_entities():
from src.translation.glossary import merge_proposal
terms = [{'source': 'Sun Microsystems', 'target': '太阳微系统公司', 'aliases': ['Sun']}]
conflicts, ambiguous = [], set()
terms = merge_proposal(terms, {'source': 'David Sun', 'target': '孙大卫', 'aliases': ['Sun']}, conflicts, ambiguous)
assert len(terms) == 2 and ambiguous == {'sun'}
assert all(not t['aliases'] for t in terms)
terms = merge_proposal(terms, {'source': 'Sun Microsystems', 'target': '太阳微系统公司', 'aliases': ['Sun']}, conflicts, ambiguous)
glossary = BookGlossary(terms)
assert not glossary.validate('David Sun founded Kingston.', '孙大卫创办了金士顿。')
assert not glossary.validate('Sun made an investment.', '孙先生作了投资。')
assert glossary.validate('Sun Microsystems supplied computers.', '孙先生提供了电脑。')
@pytest.mark.parametrize('html', [
'<p>Hello <em><b>world</b></em>.</p>',
'<p>The angle φ = 10, and φ1φ is literal mathematical text.</p>',