Filter unsafe automatic terminology constraints before book translation

src/translation/glossary.py

@@ -7,6 +7,39 @@ from pathlib import Path
 from src.common.utils import setup_logger
 
 logger = setup_logger('glossary')
+COMMON_ALIASES = {'who', 'us', 'it', 'he', 'she', 'we', 'you', 'they', 'me', 'them',
+                  'our', 'my', 'the', 'a', 'an', 'in', 'on', 'at', 'to', 'of', 'or',
+                  'dad', 'father', 'mother', 'mom', 'son', 'brother', 'sister',
+                  'family', 'face', 'carry'}
+
+
+def sanitize_automatic_terms(terms, ambiguous_aliases):
+    """Only automatic terminology is filtered; explicit user glossaries stay authoritative."""
+    terms = copy.deepcopy(terms)
+    excluded = []
+    for term in terms:
+        if re.fullmatch(r'[a-z]+', term['source']) or term['source'].casefold() in COMMON_ALIASES:
+            term['enforce'] = False
+        keep = []
+        for alias in term.get('aliases', []):
+            unsafe = alias.casefold() in COMMON_ALIASES or re.fullmatch(r'[a-z]+', alias)
+            # A surname shared implicitly by another full name is ambiguous,
+            # even when the model did not explicitly propose it as an alias.
+            for other in terms:
+                words = other['source'].split()
+                if (other['target'] != term['target'] and 2 <= len(words) <= 4
+                        and all(re.fullmatch(r'[A-Z][A-Za-z.-]*', w) for w in words)
+                        and alias.casefold() in {words[0].casefold(), words[-1].casefold()}):
+                    ambiguous_aliases.add(alias.casefold())
+            if unsafe or alias.casefold() in ambiguous_aliases:
+                excluded.append({'source': term['source'], 'alias': alias})
+            else:
+                keep.append(alias)
+        term['aliases'] = keep
+    # A later entity may reveal ambiguity in an earlier alias.
+    for term in terms:
+        term['aliases'] = [a for a in term['aliases'] if a.casefold() not in ambiguous_aliases]
+    return terms, excluded
 
 
 def source_digest(entries):
@@ -86,6 +119,8 @@ class BookGlossary:
         result = {}
         for _, _, index, alias in self._active_terms(text):
             term = self.terms[index]
+            if not term.get('enforce', True):
+                continue
             note = term.get('notes', '')
             if ' ' not in alias and alias.isascii():
                 note += '; Apply only to this named entity, not a family/clan/surname expression.'
@@ -97,7 +132,10 @@ class BookGlossary:
         errors = []
         for index in {x[2] for x in self._active_terms(source)}:
             term = self.terms[index]
-            if term['target'] not in translation:
+            if not term.get('enforce', True):
+                continue
+            target = term['target'].strip('“”‘’"\'《》「」')
+            if target not in translation:
                 errors.append(f"required term: {term['source']} -> {term['target']}")
             for forbidden in term.get('forbidden_targets', []):
                 if forbidden in translation:
@@ -158,7 +196,16 @@ async def prepare_glossary(entries, llm, path, override=None, auto=True, chunk_c
         data = json.loads(path.read_text())
         if data.get('source_digest') == digest:
             if data.get('scan_complete', True):
-                return BookGlossary(data['terms'])
+                ambiguous_aliases = set(data.get('ambiguous_aliases', []))
+                filtered, excluded = sanitize_automatic_terms(data['terms'], ambiguous_aliases)
+                if filtered != data['terms']:
+                    data['terms'] = filtered
+                    data['excluded_automatic_aliases'] = data.get('excluded_automatic_aliases', []) + excluded
+                    data['ambiguous_aliases'] = sorted(ambiguous_aliases)
+                    temp = path.with_suffix('.tmp')
+                    temp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
+                    temp.replace(path)
+                return BookGlossary(filtered)
             if data.get('chunk_chars') == chunk_chars and data.get('model') == getattr(llm, 'model', ''):
                 terms, conflicts, start = data['terms'], data.get('conflicts', []), data.get('next_chunk', 0)
                 ambiguous_aliases = set(data.get('ambiguous_aliases', []))
@@ -195,4 +242,11 @@ async def prepare_glossary(entries, llm, path, override=None, auto=True, chunk_c
                                   ensure_ascii=False, indent=2), encoding='utf-8')
         temp.replace(path)
         logger.info('Terminology scan %s/%s: %s terms, %s conflicts recorded', index + 1, len(chunks), len(terms), len(conflicts))
+    terms, excluded = sanitize_automatic_terms(terms, ambiguous_aliases)
+    if chunks:
+        data = json.loads(path.read_text())
+        data.update(terms=terms, ambiguous_aliases=sorted(ambiguous_aliases), excluded_automatic_aliases=excluded)
+        temp = path.with_suffix('.tmp')
+        temp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
+        temp.replace(path)
     return BookGlossary(terms)