959 lines
43 KiB
Python
959 lines
43 KiB
Python
#!/usr/bin/env python3
|
||
"""S1 — entity/term candidate flagger v3. Local, no LLM cost.
|
||
|
||
Inherits the correct design of the production v2 script
|
||
(`OBSIDIAN/bin/obsidian-asr-phonetic-correct.py`): phonetics raise a flag,
|
||
*sentence context decides*, so this stays a high-recall flagger that never
|
||
rewrites anything. It emits candidates and lets the S2 cleaning model judge each
|
||
one against the sentence it sits in — the step that correctly rejects
|
||
主要(adverb)→朱岩 while accepting 中药("找大涛")→王宗耀.
|
||
|
||
Three changes over v2, each fixing a measured problem:
|
||
|
||
1. **Speed.** v2 took 31s per transcript: it re-scanned all 349 minutes notes on
|
||
every run and did a pure-Python O(n·m) syllable DP against every candidate.
|
||
v3 caches the corpus scan on disk (keyed by directory mtimes) and pre-buckets
|
||
candidates by (length, first-syllable-initial), so the DP only runs on
|
||
plausible pairs. Target <5s.
|
||
|
||
2. **Precision.** v2 offered 「分钟」→冯昆仑 and 「很远」→恒元生物 — ordinary words
|
||
displaced by entities. v3 consults the term-bank allowlist (1,644 high-frequency
|
||
general words from THUOCL): a span that IS a common word is demoted, not
|
||
dropped, and carries an explicit `common_word: true` warning so the model
|
||
defaults to keeping it.
|
||
|
||
3. **Coverage.** v2 only searched vault entities. v3 adds the domain term bank
|
||
(CDMO/CMC jargon, 239 Chinese INN drug names) and INN suffix detection, so it
|
||
flags 批记录 mis-hearings (披露/P图) and drug names never seen in the vault.
|
||
|
||
Output is JSON: candidates grouped per chunk index, ready to inject into S2.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import math
|
||
import os
|
||
import re
|
||
import sys
|
||
import time
|
||
import urllib.request
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
|
||
from .config import CFG
|
||
|
||
def _vault() -> Path:
|
||
return CFG.vault
|
||
|
||
|
||
def _bank() -> Path:
|
||
return CFG.termbank_dir
|
||
|
||
|
||
def _cache() -> Path:
|
||
return CFG.cache_dir / "flagger"
|
||
|
||
HAN = re.compile(r"[\u4e00-\u9fff]+")
|
||
HAN_FULL = re.compile(r"^[\u4e00-\u9fff]+$")
|
||
SPEAKER_TAG = re.compile(r"^\s*[【\[]([^\]】]+)[】\]]\s*")
|
||
SENT_SPLIT = re.compile(r"[。！？!?…\n；;]+")
|
||
INITIALS = ["zh", "ch", "sh", "b", "p", "m", "f", "d", "t", "n", "l", "g", "k",
|
||
"h", "j", "q", "x", "r", "z", "c", "s", "y", "w"]
|
||
FOLD = {"zh": "z", "ch": "c", "sh": "s"}
|
||
|
||
try:
|
||
from pypinyin import Style, lazy_pinyin
|
||
except ImportError: # pragma: no cover - fall back to the vendored copy
|
||
sys.path.insert(0, str(CFG.obsidian / "bin" / "_vendor"))
|
||
from pypinyin import Style, lazy_pinyin
|
||
|
||
|
||
# ------------------------------------------------------------------ phonetics
|
||
def norm_syllable(s: str) -> tuple[str, str]:
|
||
init, fin = "", s
|
||
for cand in INITIALS:
|
||
if s.startswith(cand):
|
||
init, fin = cand, s[len(cand):]
|
||
break
|
||
init = FOLD.get(init, init)
|
||
if fin.endswith("ng"):
|
||
fin = fin[:-1]
|
||
# Input-method style equivalences beyond the v2 set: these are the confusions
|
||
# a Mandarin speaker (and therefore an ASR acoustic model) actually makes.
|
||
fin = fin.replace("uang", "uan").replace("iang", "ian")
|
||
if fin in ("en", "eng"):
|
||
fin = "en"
|
||
if fin in ("in", "ing"):
|
||
fin = "in"
|
||
return init, fin
|
||
|
||
|
||
def to_syls(text: str) -> list[tuple[str, str]]:
|
||
bare = lazy_pinyin(text)
|
||
toned = lazy_pinyin(text, style=Style.TONE3, neutral_tone_with_five=True)
|
||
out = []
|
||
for b, t in zip(bare, toned):
|
||
if not re.match(r"[a-z]", b):
|
||
continue
|
||
out.append((b, t[len(b):] if t.startswith(b) else ""))
|
||
return out
|
||
|
||
|
||
_PERSONAL_SYL: dict[tuple[tuple[str, str], tuple[str, str]], int] | None = None
|
||
|
||
|
||
def personal_syllable_confusions() -> dict:
|
||
"""Learn directional heard->intended syllable errors from confirmed fixes."""
|
||
global _PERSONAL_SYL
|
||
if _PERSONAL_SYL is not None:
|
||
return _PERSONAL_SYL
|
||
out: dict[tuple[tuple[str, str], tuple[str, str]], int] = {}
|
||
try:
|
||
from . import lexicon
|
||
for heard, targets in lexicon.confusion_map().items():
|
||
hs = to_syls(heard)
|
||
for intended, n, confirmed in targets:
|
||
if not confirmed:
|
||
continue
|
||
ts = to_syls(intended)
|
||
if len(hs) != len(ts):
|
||
continue
|
||
for h, t in zip(hs, ts):
|
||
hk, tk = norm_syllable(h[0]), norm_syllable(t[0])
|
||
if hk == tk:
|
||
continue
|
||
out[(hk, tk)] = out.get((hk, tk), 0) + max(1, n)
|
||
except Exception:
|
||
pass
|
||
_PERSONAL_SYL = out
|
||
return out
|
||
|
||
|
||
def syl_sim(a, b, *, personal: bool = False) -> float:
|
||
(ap, at), (bp, bt) = a, b
|
||
ai, af = norm_syllable(ap)
|
||
bi, bf = norm_syllable(bp)
|
||
if (ai, af) == (bi, bf):
|
||
base = 1.0
|
||
else:
|
||
base = 0.0
|
||
if ai == bi:
|
||
base += 0.5
|
||
elif {ai, bi} <= {"n", "l"} or {ai, bi} <= {"f", "h"}:
|
||
base += 0.35 # n/l and f/h merge in many southern accents
|
||
if af == bf:
|
||
base += 0.5
|
||
elif af and bf and af[0] == bf[0]:
|
||
base += 0.2
|
||
learned_n = (personal_syllable_confusions().get(((ai, af), (bi, bf)), 0)
|
||
if personal else 0)
|
||
if learned_n:
|
||
# Direction matters: this is how the ASR wrote the sound -> what Kai
|
||
# confirmed. One confirmation is strong evidence, repetition raises it
|
||
# gradually, and graph/in-play gating still limits which entities are active.
|
||
return max(base, min(0.98, 0.89 + 0.03 * math.log1p(learned_n)))
|
||
return base * (1.0 if at == bt and at else 0.94)
|
||
|
||
|
||
def seq_score(a, b, *, personal: bool = False) -> float:
|
||
n, m = len(a), len(b)
|
||
if not n or not m:
|
||
return 0.0
|
||
dp = [[0.0] * (m + 1) for _ in range(n + 1)]
|
||
for i in range(n + 1):
|
||
dp[i][0] = i
|
||
for j in range(m + 1):
|
||
dp[0][j] = j
|
||
for i in range(1, n + 1):
|
||
ai = a[i - 1]
|
||
for j in range(1, m + 1):
|
||
dp[i][j] = min(dp[i - 1][j - 1] +
|
||
(1 - syl_sim(ai, b[j - 1], personal=personal)),
|
||
dp[i - 1][j] + 1, dp[i][j - 1] + 1)
|
||
return 1 - dp[n][m] / max(n, m)
|
||
|
||
|
||
# ------------------------------------------------------------ candidate pool
|
||
@dataclass
|
||
class Cand:
|
||
form: str # the surface form to match phonetically
|
||
canonical: str # what to correct to
|
||
kind: str # 人名 / 公司 / 工艺术语 / 药品通用名 / …
|
||
source: str
|
||
weight: float = 0.0 # in-play priority
|
||
syls: list = field(default_factory=list)
|
||
rule: str = "full" # how `form` was derived from `canonical` (see RULE_MIN)
|
||
owner: str = "" # affiliation (entity.company) used for disambiguation
|
||
role: str = ""
|
||
anchor: int = 0 # honorific forms: +1 = leading particle, -1 = trailing
|
||
|
||
|
||
# How close a span must sound to a candidate form, per derivation rule. These are
|
||
# not arbitrary: a rule that carries extra structural evidence can afford a lower
|
||
# phonetic bar, and a rule with little structure needs a higher one.
|
||
RULE_MIN = {
|
||
"full": 0.86, # whole-name homophone, the v3 behaviour
|
||
"alias": 1.0, # exact card/learned alias — an identity binding, not a guess
|
||
"org-short": 0.84, # 迈威(生物): the dropped tail is itself evidence
|
||
"given": 0.88, # 光辉←李光辉: short and riskier, so stricter
|
||
"honorific": 0.82, # X总/老X: the title is a strong anchor, only X must match
|
||
"one-off": 0.85, # same length, one syllable substituted
|
||
"consistency": 0.80, # another spelling of a target already established in-doc
|
||
"observed": 0.90, # fuzzy neighbour of a confirmed historical ASR error
|
||
}
|
||
|
||
ORG_TAIL = ("生物医药", "生物科技", "有限公司", "股份公司", "生物", "医药", "制药",
|
||
"药业", "基金", "资本", "投资", "科技", "集团", "股份", "公司", "银行",
|
||
"证券", "医院", "研究院", "实验室")
|
||
HONOR_PREFIX = ("老", "小")
|
||
HONOR_SUFFIX = ("总", "博", "哥", "姐", "老师", "董") # 博=博士: 张博→张成海, 许博→许俊彦
|
||
|
||
|
||
def org_head(name: str) -> str:
|
||
"""迈威生物→迈威, 宝山基金→宝山. The form people actually say out loud."""
|
||
for t in ORG_TAIL:
|
||
if name.endswith(t) and len(name) - len(t) >= 2:
|
||
return name[:len(name) - len(t)]
|
||
return ""
|
||
|
||
|
||
def variants(canonical: str, kind: str) -> list[tuple[str, str, int]]:
|
||
"""Spoken surface forms of an entity, as (form, rule).
|
||
|
||
This is the fix for the dominant failure mode measured on 2026-07-21: the
|
||
matcher only compared whole strings of near-equal length, so every colloquial
|
||
short form was unreachable — 麦威 could not find 迈威生物, 光辉 could not find
|
||
李光辉, and 老万 was actively discarded because it matched an alias *exactly*.
|
||
"""
|
||
out: list[tuple[str, str, int]] = []
|
||
if kind in ("公司", "机构"):
|
||
h = org_head(canonical)
|
||
if 2 <= len(h) <= 5:
|
||
out.append((h, "org-short", 0))
|
||
return out
|
||
if kind != "人名" or not (2 <= len(canonical) <= 4):
|
||
return out
|
||
surname, given = canonical[0], canonical[1:]
|
||
if len(given) >= 2:
|
||
out.append((given, "given", 0)) # 光辉 ← 李光辉
|
||
for p in HONOR_PREFIX:
|
||
out.append((p + surname, "honorific", +1)) # 老万 / 小万
|
||
for suf in HONOR_SUFFIX:
|
||
out.append((surname + suf, "honorific", -1)) # 万总 / 万博
|
||
if len(given) == 1:
|
||
out.append((given + suf, "honorific", -1)) # 洵总 ← 邹洵
|
||
return out
|
||
|
||
|
||
def _dir_sig(paths) -> str:
|
||
h = hashlib.sha256()
|
||
for p in paths:
|
||
p = Path(p)
|
||
if p.exists():
|
||
if p.is_file():
|
||
st = p.stat()
|
||
h.update(f"{p}:{st.st_mtime_ns}:{st.st_size}".encode())
|
||
continue
|
||
files = sorted(p.glob("*.md"))
|
||
h.update(f"{p}:{len(files)}".encode())
|
||
for f in files:
|
||
st = f.stat()
|
||
h.update(f"{f.name}:{st.st_mtime_ns}:{st.st_size}".encode())
|
||
return h.hexdigest()[:16]
|
||
|
||
|
||
def load_pool(use_termbank=True) -> dict[str, Cand]:
|
||
"""Vault entities (cards + glossary) ∪ optional domain term bank."""
|
||
_cache().mkdir(parents=True, exist_ok=True)
|
||
if CFG.use_lexicon:
|
||
try:
|
||
from . import lexicon
|
||
lexicon.sync_from_vault()
|
||
except Exception:
|
||
pass
|
||
sig = _dir_sig([_vault() / "People", _vault() / "Companies", _vault() / "minutes",
|
||
CFG.lexicon_db])
|
||
cpath = _cache() / f"pool6-{sig}-{int(use_termbank)}.json"
|
||
if cpath.exists():
|
||
d = json.loads(cpath.read_text())
|
||
return {k: Cand(**v) for k, v in d.items()}
|
||
|
||
pool: dict[str, Cand] = {}
|
||
|
||
def add(form, canonical, kind, source, rule="full", owner="", role="", anchor=0):
|
||
"""Register a matchable form.
|
||
|
||
Keyed by (form, rule, canonical) rather than form alone: 迈威 is both an
|
||
exact card alias of 迈威生物 *and* its org-short form. Keeping only the
|
||
alias entry made 麦威 unmatchable, because the alias rule matches exactly
|
||
and never phonetically. Both entries must coexist.
|
||
"""
|
||
form = (form or "").strip()
|
||
if not (2 <= len(form) <= 12) or not HAN_FULL.match(form):
|
||
return
|
||
key = f"{form}\x00{rule}\x00{canonical}"
|
||
if key in pool:
|
||
return
|
||
pool[key] = Cand(form, canonical, kind, source, syls=to_syls(form),
|
||
rule=rule, owner=owner, role=role, anchor=anchor)
|
||
|
||
# Entities and their aliases come from the SQLite lexicon, which mirrors the
|
||
# vault cards (read-only) and additionally holds anything Kai has confirmed.
|
||
ents: dict = {}
|
||
if CFG.use_lexicon:
|
||
try:
|
||
from . import lexicon
|
||
ents = lexicon.entities()
|
||
for name, e in ents.items():
|
||
add(name, name, e.kind, "lexicon-entity", "full", e.company, e.role)
|
||
for al, targets in lexicon.alias_map().items():
|
||
# Every target, not just the first. The glossary deliberately keeps
|
||
# one-to-many rows (许博→许俊彦/许恒皓, 徐总→徐劲前/邹洵) because they
|
||
# are context-dependent; collapsing them to one silently discarded
|
||
# the correct reading.
|
||
for canon, kind, origin, _conf in targets:
|
||
e = ents.get(canon)
|
||
add(al, canon, kind, f"alias-{origin}", "alias",
|
||
e.company if e else "", e.role if e else "")
|
||
# Exact confirmed errors are already aliases. This second form is
|
||
# deliberately gated on in-play and allows a nearby decoder variant
|
||
# (张台海) to recall the same canonical person learned from
|
||
# 张泰海→张成海 without enabling global fuzzy matching.
|
||
for heard, targets in lexicon.confusion_map().items():
|
||
for canon, _n, confirmed in targets:
|
||
if not confirmed:
|
||
continue
|
||
e = ents.get(canon)
|
||
add(heard, canon, e.kind if e else lexicon.guess_kind(canon),
|
||
"observed-correction", "observed",
|
||
e.company if e else "", e.role if e else "")
|
||
except Exception:
|
||
ents = {}
|
||
if not ents: # lexicon disabled/broken
|
||
for sub, kind in (("People", "人名"), ("Companies", "公司")):
|
||
d = _vault() / sub
|
||
if not d.exists():
|
||
continue
|
||
for f in d.glob("*.md"):
|
||
add(f.stem, f.stem, kind, f"card-{sub.lower()}")
|
||
|
||
# Spoken short forms. Generated for every entity; only the in-play ones are
|
||
# activated at match time, so this costs nothing in precision.
|
||
for c in list(pool.values()):
|
||
if c.rule != "full" or c.kind not in ("人名", "公司"):
|
||
continue
|
||
for form, rule, anchor in variants(c.canonical, c.kind):
|
||
add(form, c.canonical, c.kind, "variant", rule, c.owner, c.role, anchor)
|
||
|
||
if use_termbank:
|
||
tb = _bank() / "terms.json"
|
||
if tb.exists():
|
||
for term, e in json.loads(tb.read_text()).items():
|
||
# Only the on-domain, distinctive slices are worth phonetic search;
|
||
# 27k THUOCL general words would flood the candidate space with the
|
||
# very false positives v3 is trying to eliminate.
|
||
if e["kind"] in ("工艺术语", "药品通用名", "机构", "已确认纠错目标"):
|
||
add(term, term, e["kind"], e["sources"][0])
|
||
|
||
cpath.write_text(json.dumps({k: v.__dict__ for k, v in pool.items()},
|
||
ensure_ascii=False), encoding="utf-8")
|
||
return pool
|
||
|
||
|
||
def load_allowlist() -> set[str]:
|
||
p = _bank() / "allowlist.txt"
|
||
return set(p.read_text().split()) if p.exists() else set()
|
||
|
||
|
||
# ------------------------------------------------------------------- in-play
|
||
# A wikilink in a minutes note points at other notes as often as at entities.
|
||
# Unfiltered, 25 of the 70 in-play slots on 2026-07-21 were note titles, bare dates
|
||
# and `_glossary` — burning candidate space and prompt budget on non-entities.
|
||
NOT_ENTITY = re.compile(r"^\d{4}-\d{2}|^_|\.md$|[|/#]|^\d+$")
|
||
|
||
|
||
def _is_entity_name(s: str) -> bool:
|
||
s = s.strip()
|
||
return bool(s) and 2 <= len(s) <= 12 and not NOT_ENTITY.search(s)
|
||
|
||
|
||
def _minutes_entities(text: str) -> set[str]:
|
||
out = set()
|
||
for key in ("people", "companies"):
|
||
m = re.search(rf"^{key}:[ \t]*(.*)$", text, re.M)
|
||
if m:
|
||
out |= {x for x in re.findall(r"[^,\[\]\s，、]+", m.group(1))}
|
||
out |= {w.strip() for w in re.findall(r"\[\[([^\]|#]+)", text)}
|
||
return {o for o in out if _is_entity_name(o)}
|
||
|
||
|
||
def _corpus() -> list[tuple[str, set[str], str]]:
|
||
_cache().mkdir(parents=True, exist_ok=True)
|
||
sig = _dir_sig([_vault() / "minutes"])
|
||
cpath = _cache() / f"corpus-{sig}.json"
|
||
if cpath.exists():
|
||
return [(a, set(b), c) for a, b, c in json.loads(cpath.read_text())]
|
||
out = []
|
||
for f in sorted((_vault() / "minutes").glob("*.md")):
|
||
txt = f.read_text(errors="ignore")
|
||
m = re.search(r"^date:\s*(\d{4}-\d{2}-\d{2})", txt, re.M)
|
||
out.append((f.name, _minutes_entities(txt), m.group(1) if m else ""))
|
||
cpath.write_text(json.dumps([(a, sorted(b), c) for a, b, c in out],
|
||
ensure_ascii=False), encoding="utf-8")
|
||
return out
|
||
|
||
|
||
def hindsight_recall(seed: list[str], extra_query: str = "") -> tuple[set[str], list[str]]:
|
||
"""Semantic neighbourhood from Hindsight. Returns (entity names, memory texts).
|
||
|
||
The memory texts matter as much as the names: injected into S2/S4 they give the
|
||
model the *relationship* ("吴楠 is 刘冀's colleague at 跨境资本") that lets it
|
||
accept or reject a phonetic candidate on business grounds, not just sound.
|
||
"""
|
||
if not seed and not extra_query:
|
||
return set(), []
|
||
query = (" ".join(sorted(seed)) + " " + extra_query).strip() or "会议 相关人物 公司"
|
||
body = json.dumps({"query": query, "max_tokens": 1500}).encode()
|
||
url = f"{CFG.hindsight_api}/v1/default/banks/{CFG.hindsight_bank}/memories/recall"
|
||
try:
|
||
req = urllib.request.Request(url, data=body,
|
||
headers={"Content-Type": "application/json"})
|
||
with urllib.request.urlopen(req, timeout=CFG.hindsight_timeout) as r:
|
||
d = json.loads(r.read())
|
||
except Exception:
|
||
return set(), []
|
||
names = set(d.get("entities", {}).keys())
|
||
texts = []
|
||
for r_ in d.get("results", []):
|
||
names.update(r_.get("entities", []))
|
||
if r_.get("text"):
|
||
texts.append(r_["text"])
|
||
return ({n for n in names if 2 <= len(n) <= 8 and HAN_FULL.match(n)}, texts)
|
||
|
||
|
||
def _super_nodes(participants: set[str], corpus) -> set[str]:
|
||
if len(participants) < 2:
|
||
return set()
|
||
freq = {p: sum(1 for _, e, _ in corpus if p in e) for p in participants}
|
||
fmin = max(1, min(freq.values()))
|
||
hubs = {p for p in participants if freq[p] >= 20 and freq[p] >= 3 * fmin}
|
||
return hubs if len(hubs) < len(participants) else set()
|
||
|
||
|
||
def compute_in_play(participants: set[str], recent_days=90, use_hindsight=True,
|
||
extra_seed: set[str] | None = None):
|
||
corpus = _corpus()
|
||
hubs = _super_nodes(participants, corpus)
|
||
seed = ((participants - hubs) or participants) | (extra_seed or set())
|
||
dates = [d for _, _, d in corpus if d]
|
||
ref = max(dates) if dates else ""
|
||
weight = {p: 999.0 for p in participants}
|
||
for _, ents, d in corpus:
|
||
if not (ents & seed):
|
||
continue
|
||
if ref and d and recent_days:
|
||
try:
|
||
from datetime import date
|
||
a = date.fromisoformat(d)
|
||
b = date.fromisoformat(ref)
|
||
if (b - a).days > recent_days:
|
||
continue
|
||
except Exception:
|
||
pass
|
||
for e in ents:
|
||
weight[e] = weight.get(e, 0) + 1
|
||
try:
|
||
from . import graph
|
||
graph.sync()
|
||
for entity, score in graph.context_weights(seed).items():
|
||
weight[entity] = weight.get(entity, 0.0) + score
|
||
except Exception:
|
||
pass
|
||
in_play = set(sorted(weight, key=lambda e: -weight[e])[:60])
|
||
hs_names, hs_texts = hindsight_recall(sorted(seed)) if use_hindsight else (set(), [])
|
||
return in_play | hs_names, {"hubs": sorted(hubs), "hindsight_names": sorted(hs_names),
|
||
"hindsight_texts": hs_texts, "weight": weight}
|
||
|
||
|
||
# ----------------------------------------------------------------- flagging
|
||
def sentences_of(text: str):
|
||
for raw in text.splitlines():
|
||
line = raw.strip()
|
||
if not line or line.startswith(("#", "<!--", "---")):
|
||
continue
|
||
spk = ""
|
||
m = SPEAKER_TAG.match(line)
|
||
if m:
|
||
spk = m.group(1)
|
||
line = SPEAKER_TAG.sub("", line)
|
||
for piece in SENT_SPLIT.split(line):
|
||
piece = piece.strip()
|
||
if piece:
|
||
yield spk, piece
|
||
|
||
|
||
def phon_key(syls: list[tuple[str, str]]) -> str:
|
||
"""Collapse a word to an accent-folded consonant/vowel skeleton.
|
||
|
||
This is the input-method trick: 中药 and 宗耀 both become "zonyao", so an exact
|
||
dict lookup finds the homophone class instantly and only genuinely ambiguous
|
||
pairs need the expensive alignment. Tones are dropped here and used later as a
|
||
tiebreak, mirroring how a listener hears a name they半-recognize.
|
||
"""
|
||
out = []
|
||
for bare, _ in syls:
|
||
i, f = norm_syllable(bare)
|
||
out.append(i + f)
|
||
return "".join(out)
|
||
|
||
|
||
def flag_text(text: str, pool: dict[str, Cand], in_play: set[str],
|
||
allowlist: set[str], min_score=0.86, use_termbank_global=True,
|
||
entity_weights: dict[str, float] | None = None):
|
||
"""Find phonetic near-matches. Entity candidates are gated on being in-play;
|
||
domain terms (工艺术语/药品通用名) are always searched, since a drug or a
|
||
批记录 can appear in a meeting with no prior vault history.
|
||
|
||
Performance note: the v2 approach (score every span against every candidate
|
||
with a Python syllable DP) is O(spans x candidates x syllables^2) and took
|
||
>400s once the term bank widened the candidate pool. v3 first buckets
|
||
candidates by phonetic key and length, uses rapidfuzz (C) to shortlist keys,
|
||
and only then runs the fine-grained tonal alignment on survivors.
|
||
"""
|
||
from rapidfuzz import process as rf_process
|
||
|
||
# An in-play name with no candidate entry can never be matched. 宝山基金 has no
|
||
# Companies card — it reached in-play only through a minutes filename — so
|
||
# 保山 (a perfect homophone of its short form) had nothing to match against.
|
||
# Materialise those names, including their spoken short forms.
|
||
pool = dict(pool)
|
||
# Skip names already reachable as *any* form, not just as a canonical. 迈威 is
|
||
# the alias/short form of 迈威生物 and also appears in in-play as a bare name;
|
||
# materialising it again produced two competing candidates for one company.
|
||
known = {c.canonical for c in pool.values()} | {c.form for c in pool.values()}
|
||
for name in sorted(in_play):
|
||
if name in known or not _is_entity_name(name) or not HAN_FULL.match(name):
|
||
continue
|
||
is_org = name.endswith(ORG_TAIL)
|
||
kind = "公司" if is_org else "人名"
|
||
if not is_org and not (2 <= len(name) <= 4):
|
||
continue
|
||
# Variants are only derived for names whose type is *known*. An in-play name
|
||
# with no card is a guess already; 三金 (short for 桂林三金) was being read as
|
||
# a surname and generating 老三, which then matched 老尚. Only an explicit
|
||
# organisation suffix is trustworthy enough to expand here.
|
||
forms = [(name, "full", 0)] + (variants(name, kind) if is_org else [])
|
||
for form, rule, anchor in forms:
|
||
if not (2 <= len(form) <= 12 and HAN_FULL.match(form)):
|
||
continue
|
||
pool.setdefault(f"{form}\x00{rule}\x00{name}",
|
||
Cand(form, name, kind, "in-play", syls=to_syls(form),
|
||
rule=rule, anchor=anchor))
|
||
|
||
active: list[Cand] = []
|
||
for c in pool.values():
|
||
if c.rule == "alias":
|
||
# Curated identity bindings (card `aliases:` + imported glossary). These
|
||
# are hand-confirmed and matched exactly, so gating them on in-play
|
||
# would only lose recall — this is what silently dropped 老万→万亚坤.
|
||
active.append(c)
|
||
elif c.kind in ("人名", "公司", "项目", "已确认目标"):
|
||
if c.canonical in in_play or c.form in in_play:
|
||
active.append(c)
|
||
elif use_termbank_global:
|
||
active.append(c)
|
||
|
||
# key -> candidates sharing that phonetic skeleton
|
||
by_key: dict[str, list[Cand]] = {}
|
||
for c in active:
|
||
by_key.setdefault(phon_key(c.syls), []).append(c)
|
||
keys = list(by_key)
|
||
lens = {len(c.form) for c in active} or {2, 3}
|
||
widths = sorted({w for L in lens for w in (max(2, L - 1), L, L + 1)})
|
||
|
||
# Collect every distinct span once; scoring a span is independent of where it
|
||
# occurs, so we score uniques and then attribute hits back to sentences.
|
||
span_positions: dict[str, tuple[str, str]] = {}
|
||
for spk, sent in sentences_of(text):
|
||
for run in HAN.findall(sent):
|
||
for w in widths:
|
||
for i in range(len(run) - w + 1):
|
||
span_positions.setdefault(run[i:i + w], (spk, sent))
|
||
|
||
span_keys: dict[str, tuple[str, list]] = {}
|
||
for span in span_positions:
|
||
syls = to_syls(span)
|
||
if syls:
|
||
span_keys[span] = (phon_key(syls), syls)
|
||
|
||
findings: dict[tuple, dict] = {}
|
||
|
||
def consider(span, cand, syls):
|
||
# Already the canonical name: nothing to say.
|
||
if span == cand.canonical:
|
||
return
|
||
rule = cand.rule
|
||
if span == cand.form:
|
||
# Exact hit on a non-canonical form. Previously this returned early —
|
||
# which is precisely why 老万 (an exact card alias of 万亚坤) and 迈威
|
||
# produced no output at all and the drafting model had to guess.
|
||
sc = 1.0
|
||
else:
|
||
if abs(len(span) - len(cand.form)) > 1:
|
||
return
|
||
if rule == "alias":
|
||
return # curated aliases match exactly or not at all
|
||
if rule == "honorific":
|
||
# 老X / X总 / X博. The particle carries no identity information, so
|
||
# it must match *literally*; allowing it to drift turned this rule
|
||
# into a generic 2-syllable fuzzy match and produced 想谈→谭凯,
|
||
# 理解→李光辉, 就动→刘冀 — 36 junk flags on one transcript.
|
||
if len(span) != len(cand.form):
|
||
return
|
||
if cand.anchor > 0:
|
||
if span[0] != cand.form[0]:
|
||
return
|
||
a, b = span[1:], cand.form[1:]
|
||
elif cand.anchor < 0:
|
||
if span[-1] != cand.form[-1]:
|
||
return
|
||
a, b = span[:-1], cand.form[:-1]
|
||
else:
|
||
return
|
||
sc = seq_score(
|
||
to_syls(a), to_syls(b),
|
||
personal=(cand.kind in ("人名", "公司", "项目")
|
||
and cand.canonical in in_play))
|
||
if sc < 0.86:
|
||
return
|
||
else:
|
||
sc = seq_score(
|
||
syls, cand.syls,
|
||
personal=(cand.kind in ("人名", "公司", "项目")
|
||
and cand.canonical in in_play))
|
||
if sc < RULE_MIN.get(rule, min_score):
|
||
return
|
||
spk, sent = span_positions[span]
|
||
key = (span, cand.canonical)
|
||
prev = findings.get(key)
|
||
# On a tie, keep the curated alias rather than the phonetic match: both score
|
||
# 1.0 for 万金→万津, and losing the alias provenance downgraded a confirmed
|
||
# mapping back into a guess.
|
||
better = prev is None or sc > prev["score"] or (
|
||
sc == prev["score"] and rule == "alias" and prev.get("rule") != "alias")
|
||
if better:
|
||
findings[key] = {
|
||
"span": span, "target": cand.canonical, "kind": cand.kind,
|
||
"score": round(sc, 3), "via": cand.form, "speaker": spk,
|
||
"sentence": sent, "common_word": span in allowlist,
|
||
"source": cand.source, "rule": rule, "owner": cand.owner,
|
||
"role": cand.role,
|
||
}
|
||
|
||
# 1. exact skeleton collisions — the high-confidence homophone class
|
||
for span, (k, syls) in span_keys.items():
|
||
for cand in by_key.get(k, ()):
|
||
consider(span, cand, syls)
|
||
|
||
# 2. near-miss skeletons via rapidfuzz shortlist (handles one-syllable slips
|
||
# like 李小雨/李晓宇 that the folded key alone does not merge)
|
||
uniq_keys = sorted({k for k, _ in span_keys.values()})
|
||
matches = rf_process.cdist(uniq_keys, keys, workers=-1, score_cutoff=78)
|
||
shortlist: dict[str, list[str]] = {}
|
||
for row_i, row in enumerate(matches):
|
||
hits = [keys[j] for j, v in enumerate(row) if v]
|
||
if hits:
|
||
shortlist[uniq_keys[row_i]] = hits
|
||
for span, (k, syls) in span_keys.items():
|
||
for tk in shortlist.get(k, ()):
|
||
if tk == k:
|
||
continue
|
||
for cand in by_key[tk]:
|
||
consider(span, cand, syls)
|
||
|
||
# 3. single-character substitution on equal-length names. 张泰海/张成海 scores
|
||
# 0.82 phonetically (tai vs cheng is a poor match) yet is obviously the same
|
||
# person: the surname and the tail are identical and only one syllable moved.
|
||
# Anchoring on shared characters rather than sound catches the whole class.
|
||
by_len: dict[int, list[Cand]] = {}
|
||
for c in active:
|
||
# Canonical names only. Aliases can be arbitrary phrases, and running
|
||
# single-character substitution against one produced 我老板/跟老板/那老板/
|
||
# 的老板 → 邹洵 (18 junk flags from the alias 林老板 alone).
|
||
if c.rule == "full" and c.kind in ("人名", "公司"):
|
||
by_len.setdefault(len(c.form), []).append(c)
|
||
for span, (k, syls) in span_keys.items():
|
||
if len(span) < 3:
|
||
continue
|
||
for cand in by_len.get(len(span), ()):
|
||
same = sum(1 for a, b in zip(span, cand.form) if a == b)
|
||
if same != len(span) - 1 or span == cand.form:
|
||
continue
|
||
# The head must hold. ASR drifts inside a name (张成海→张泰海); a name
|
||
# whose first character changed is a different name.
|
||
if span[0] != cand.form[0]:
|
||
continue
|
||
spk, sent = span_positions[span]
|
||
key = (span, cand.canonical)
|
||
if key not in findings:
|
||
findings[key] = {
|
||
"span": span, "target": cand.canonical, "kind": cand.kind,
|
||
"score": RULE_MIN["one-off"], "via": cand.form, "speaker": spk,
|
||
"sentence": sent, "common_word": span in allowlist,
|
||
"source": cand.source, "rule": "one-off", "owner": cand.owner,
|
||
"role": cand.role,
|
||
}
|
||
# Group alternatives for the same span so the model sees a real choice set,
|
||
# which is the whole point: "这里听到 X，可能是 A 或 B，按句判".
|
||
# 4. intra-document consistency. Once a target is firmly established somewhere
|
||
# in the transcript, its *other* spellings in the same transcript can be
|
||
# accepted at a lower bar — the document itself is the evidence. This is the
|
||
# 万金/万津/万杰 pattern: one ASR name drifts across several spellings, and
|
||
# only the strongest of them clears the standalone threshold.
|
||
firm: dict[str, set[str]] = {}
|
||
firm_alias: set[str] = set() # targets established by a confirmed alias
|
||
entity_weights = entity_weights or {}
|
||
for f in findings.values():
|
||
if f["rule"] == "alias" and str(f.get("source", "")).startswith("alias-"):
|
||
firm_alias.add(f["target"])
|
||
if f["rule"] == "alias" or f["score"] >= 0.95:
|
||
# Every established surface form is a comparison anchor, the canonical
|
||
# name included. Keeping only the first one made 万杰 unreachable because
|
||
# the anchor happened to be 老万 rather than 万津.
|
||
firm.setdefault(f["target"], {f["target"]}).add(f["via"])
|
||
# Shortlist by phonetic skeleton before running the DP, same trick as pass 2:
|
||
# brute-forcing every span against every anchor pushed S1 from 4.7s to 24.5s on
|
||
# a 27k transcript.
|
||
anchor_keys: dict[str, list[tuple[str, str, list]]] = {}
|
||
for target, vias in firm.items():
|
||
for v in vias:
|
||
anchor_keys.setdefault(phon_key(to_syls(v)), []).append((target, v, to_syls(v)))
|
||
if anchor_keys:
|
||
akeys = list(anchor_keys)
|
||
cm = rf_process.cdist(uniq_keys, akeys, workers=-1, score_cutoff=65)
|
||
near: dict[str, list[str]] = {}
|
||
for i, row in enumerate(cm):
|
||
hits = [akeys[j] for j, v in enumerate(row) if v]
|
||
if hits:
|
||
near[uniq_keys[i]] = hits
|
||
else:
|
||
near = {}
|
||
plan: dict[str, set[str]] = {}
|
||
for span, (k, syls) in span_keys.items():
|
||
for ak in near.get(k, ()):
|
||
for target, _v, _vs in anchor_keys[ak]:
|
||
plan.setdefault(target, set()).add(span)
|
||
|
||
for target, vias in firm.items():
|
||
anchors = [(v, to_syls(v)) for v in vias]
|
||
for span in plan.get(target, ()):
|
||
k, syls = span_keys[span]
|
||
if span == target or (span, target) in findings or span in allowlist:
|
||
continue
|
||
if len(span) < 2:
|
||
continue
|
||
best = max(((seq_score(syls, vs), v) for v, vs in anchors
|
||
if len(v) == len(span)), default=(0.0, ""))
|
||
sc, via = best
|
||
if not via or sc < 0.80:
|
||
continue
|
||
# Below near-homophone level, demand a shared head. A 2-syllable span at
|
||
# 0.82 means roughly one syllable matched, which let 总总/全中→邹洵 and
|
||
# 接商→宝山基金 through. Either the sound is nearly identical (军实/君实,
|
||
# 1.0) or the first character is stable (万杰/万津, 白发/白帆).
|
||
if sc < 0.90 and span[0] != via[0]:
|
||
continue
|
||
spk, sent = span_positions[span]
|
||
src = next((c for c in active if c.canonical == target), None)
|
||
findings[(span, target)] = {
|
||
"span": span, "target": target, "kind": src.kind if src else "人名",
|
||
"score": round(sc, 3), "via": via, "speaker": spk, "sentence": sent,
|
||
"common_word": False,
|
||
"source": "consistency-alias" if target in firm_alias else "consistency",
|
||
"rule": "consistency", "owner": src.owner if src else "",
|
||
"role": src.role if src else "",
|
||
}
|
||
|
||
# Which organisations are actually being discussed. An honorific like 老万 has
|
||
# one candidate per matching surname; the affiliation is what decides between
|
||
# them, and the vault already records it (万亚坤.company = 洛启生物, which is
|
||
# in play here). This is the cross-referencing step that was missing.
|
||
mentioned_orgs = set()
|
||
for c in active:
|
||
if c.kind == "公司" and (c.canonical in text or (c.form and c.form in text)):
|
||
mentioned_orgs.add(c.canonical)
|
||
|
||
grouped: dict[str, dict] = {}
|
||
for f in findings.values():
|
||
g = grouped.setdefault(f["span"], {
|
||
"span": f["span"], "common_word": f["common_word"],
|
||
"sentence": f["sentence"], "speaker": f["speaker"], "candidates": [],
|
||
})
|
||
owner = f.get("owner") or ""
|
||
support = []
|
||
if owner and owner in mentioned_orgs:
|
||
support.append(f"{owner}(本次在谈)")
|
||
elif owner and owner in in_play:
|
||
support.append(f"{owner}(近期相关)")
|
||
if f.get("role"):
|
||
support.append(f["role"])
|
||
recent = float(entity_weights.get(f["target"], 0.0))
|
||
context_boost = min(1.5, math.log1p(max(0.0, recent)) / 2.5)
|
||
affiliation_boost = 2.0 if (owner and owner in mentioned_orgs) else 0.0
|
||
boost = affiliation_boost + context_boost
|
||
if recent >= 0.5:
|
||
support.append(f"近期权重{recent:.1f}")
|
||
g["candidates"].append({"target": f["target"], "kind": f["kind"],
|
||
"score": f["score"], "via": f["via"],
|
||
"source": f["source"], "rule": f.get("rule", "full"),
|
||
"owner": owner, "support": "、".join(support),
|
||
"boost": round(boost, 3),
|
||
"context_weight": round(recent, 3)})
|
||
for g in grouped.values():
|
||
# Affiliation evidence outranks raw phonetic score: a same-surname person
|
||
# from an unrelated company is a worse guess than a slightly weaker match
|
||
# from the company under discussion.
|
||
g["candidates"].sort(
|
||
key=lambda c: (-(c["score"] + 0.20 * c["boost"]), -c["score"]))
|
||
# A single, unambiguous, human-confirmed mapping is not a suggestion. Kai's
|
||
# reaction to a hedged draft was "肯定是指 李光辉 啊" — treating these as
|
||
# optional is what makes the output feel useless. Ambiguous ones (徐总 has
|
||
# three possible people) deliberately stay a choice set.
|
||
c0 = g["candidates"][0]
|
||
src0 = str(c0.get("source", ""))
|
||
settled = (c0["rule"] == "alias"
|
||
and src0.startswith(("alias-card", "alias-learned", "alias-glossary")))
|
||
# A perfect homophone of an entity already pinned down by a confirmed alias,
|
||
# in the same recording, is not a separate judgement call: 老善 and 老尚 are
|
||
# one word the ASR spelled two ways, and 老尚 is confirmed to be 单永强.
|
||
settled = settled or (c0["rule"] == "consistency"
|
||
and src0 == "consistency-alias" and c0["score"] >= 0.97)
|
||
# Curated evidence outranks generated guesses. 老万 is an explicit card alias
|
||
# of 万亚坤; the rival candidates are only honorific forms auto-derived from
|
||
# other 万/王-surnamed people, which is far weaker. But two *curated* targets
|
||
# (徐总 → 徐劲前 or 邹洵) is real ambiguity and must stay a choice.
|
||
n_alias = sum(1 for c in g["candidates"]
|
||
if c["rule"] == "alias"
|
||
and str(c.get("source", "")).startswith("alias-"))
|
||
g["settled"] = bool(settled and n_alias <= 1
|
||
and (len(g["candidates"]) == 1 or n_alias == 1))
|
||
return sorted(grouped.values(),
|
||
key=lambda g: (g["common_word"], -g["candidates"][0]["boost"],
|
||
-g["candidates"][0]["score"]))
|
||
|
||
|
||
def analyze(text: str, *, use_hindsight=True, use_termbank=True, min_score=None,
|
||
extra_in_play: set[str] | None = None):
|
||
"""Flag candidates.
|
||
|
||
`extra_in_play` carries entity names recovered by the S1c keyword recall. It
|
||
matters most for backends that emit no speaker labels: with no participants to
|
||
seed from, the co-occurrence graph produces almost nothing, and vault entities
|
||
could never enter the candidate space at all.
|
||
"""
|
||
t0 = time.time()
|
||
min_score = CFG.flag_min_score if min_score is None else min_score
|
||
pool = load_pool(use_termbank=use_termbank)
|
||
allow = load_allowlist()
|
||
speakers = {s for s, _ in sentences_of(text) if s and HAN_FULL.match(s)}
|
||
extra = {n for n in (extra_in_play or set()) if _is_entity_name(n)}
|
||
in_play, ctx = compute_in_play(speakers, use_hindsight=use_hindsight,
|
||
extra_seed=extra)
|
||
in_play |= extra
|
||
flags = flag_text(text, pool, in_play, allow, min_score=min_score,
|
||
use_termbank_global=use_termbank,
|
||
entity_weights=ctx.get("weight", {}))
|
||
return {
|
||
"elapsed": round(time.time() - t0, 2),
|
||
"pool_size": len(pool),
|
||
"speakers": sorted(speakers),
|
||
"in_play": sorted(in_play),
|
||
"hubs": ctx["hubs"],
|
||
"hindsight_names": ctx["hindsight_names"],
|
||
"hindsight_texts": ctx["hindsight_texts"],
|
||
"flags": flags,
|
||
}
|
||
|
||
|
||
RULE_LABEL = {
|
||
"alias": "已确认别名",
|
||
"full": "整名近音",
|
||
"org-short": "机构简称",
|
||
"given": "去姓称名",
|
||
"honorific": "称谓",
|
||
"one-off": "一字之差",
|
||
"consistency": "同篇另一写法",
|
||
"observed": "历史误识别近音",
|
||
}
|
||
|
||
|
||
def render_for_prompt(flags: list[dict], limit=60) -> str:
|
||
"""The block injected into an S2 chunk prompt.
|
||
|
||
Each candidate carries *why* it is a candidate (derivation rule) and *what
|
||
supports it* (the affiliation the vault records, and whether that organisation
|
||
is being discussed in this very recording). Without that, the model can only
|
||
compare sounds, and it hedged on 老万 even though the vault says 万亚坤 is
|
||
洛启生物's 实控人 and 洛启 is the subject of the call.
|
||
"""
|
||
lines = []
|
||
must, maybe = [], []
|
||
for g in flags[:limit]:
|
||
parts = []
|
||
for c in g["candidates"][:3]:
|
||
tag = RULE_LABEL.get(c.get("rule", "full"), "")
|
||
sup = f"，{c['support']}" if c.get("support") else ""
|
||
parts.append(f"{c['target']}（{c['kind']}·{tag}{sup}）")
|
||
cands = "、".join(parts)
|
||
if not g["common_word"]:
|
||
warn = ""
|
||
elif g["candidates"][0].get("rule") == "alias":
|
||
# Both a common word and a confirmed alias (光辉, 公司→工时). The mapping
|
||
# is real, so the instruction is to check usage rather than default to
|
||
# keeping the original.
|
||
warn = "【此词也是常用词:先判断句中是否确实在指人/机构,是则用全名,否则保留原词】"
|
||
else:
|
||
warn = "【注意:此词本身是常用词,除非句子读不通否则保留原词】"
|
||
sent = re.sub(r"\s+", "", g["sentence"])
|
||
if len(sent) > 70:
|
||
idx = sent.find(g["span"])
|
||
lo = max(0, idx - 26)
|
||
sent = ("…" if lo else "") + sent[lo:idx + len(g["span"]) + 26] + "…"
|
||
if g.get("settled") and not g["common_word"]:
|
||
must.append(f"- 「{g['span']}」= {g['candidates'][0]['target']}"
|
||
f"｜原句: {sent}")
|
||
else:
|
||
maybe.append(f"- 「{g['span']}」→ 候选: {cands}{warn}｜原句: {sent}")
|
||
out = []
|
||
if must:
|
||
out.append("【A. 已确认映射：Kai 已核对过，出现即按右侧替换，不要标疑似】\n"
|
||
+ "\n".join(must))
|
||
if maybe:
|
||
out.append("【B. 待判断候选：按所在句子自行裁决】\n" + "\n".join(maybe))
|
||
return "\n\n".join(out)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("transcripts", nargs="+")
|
||
ap.add_argument("--no-hindsight", action="store_true")
|
||
ap.add_argument("--no-termbank", action="store_true")
|
||
ap.add_argument("--min-score", type=float, default=0.86)
|
||
ap.add_argument("--json", action="store_true")
|
||
a = ap.parse_args()
|
||
for p in a.transcripts:
|
||
text = Path(p).read_text(errors="ignore")
|
||
r = analyze(text, use_hindsight=not a.no_hindsight,
|
||
use_termbank=not a.no_termbank, min_score=a.min_score)
|
||
if a.json:
|
||
print(json.dumps(r, ensure_ascii=False, indent=2))
|
||
continue
|
||
print("=" * 78)
|
||
print(f"{Path(p).parent.name} {r['elapsed']}s pool={r['pool_size']} "
|
||
f"in_play={len(r['in_play'])} hs={len(r['hindsight_names'])} "
|
||
f"flags={len(r['flags'])}")
|
||
print(f"speakers: {'、'.join(r['speakers']) or '(none)'} hubs: {'、'.join(r['hubs'])}")
|
||
print(render_for_prompt(r["flags"], 25))
|