845 lines
31 KiB
Python
845 lines
31 KiB
Python
#!/usr/bin/env python3
|
||
"""Deterministic entity / alias / correction lexicon (SQLite).
|
||
|
||
Design rule, decided 2026-07-25:
|
||
|
||
**Obsidian is a read-only source of truth.** Cards supply canonical names and
|
||
the `aliases:` / `company:` / `role:` frontmatter. This program pulls from them
|
||
and never writes back — no `_glossary.md` edits, no card mutation.
|
||
|
||
Everything the pipeline *learns* (a confirmed nickname like 老尚=单永强, a
|
||
recurring ASR misrecognition like 张泰海→张成海) accumulates here instead. Same
|
||
separation as the voiceprint registry: an independent structured store that
|
||
grows with use, next to the vault rather than inside it.
|
||
|
||
Why SQLite and not Hindsight: alias resolution must be an exact, repeatable
|
||
key→value lookup. Hindsight stores LLM-extracted memories and recalls them
|
||
semantically — excellent for *relationships* ("万亚坤 is 洛启's 实控人", which is
|
||
what disambiguates 老万), useless as a guarantee that 老尚 resolves every single
|
||
time. The two are complementary and both are used: lexicon decides identity,
|
||
Hindsight supplies the context that ranks competing identities.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
import sqlite3
|
||
import time
|
||
from dataclasses import dataclass
|
||
from pathlib import Path
|
||
|
||
from .config import CFG
|
||
|
||
SCHEMA = """
|
||
CREATE TABLE IF NOT EXISTS entity (
|
||
canonical TEXT PRIMARY KEY,
|
||
kind TEXT NOT NULL, -- 人名 / 公司
|
||
company TEXT DEFAULT '', -- affiliation, for honorific disambiguation
|
||
role TEXT DEFAULT '',
|
||
related TEXT DEFAULT '', -- related_people, comma-joined
|
||
card TEXT DEFAULT '',
|
||
synced_at REAL
|
||
);
|
||
CREATE TABLE IF NOT EXISTS alias (
|
||
alias TEXT NOT NULL,
|
||
canonical TEXT NOT NULL,
|
||
kind TEXT NOT NULL,
|
||
origin TEXT NOT NULL, -- card | learned | manual | derived
|
||
confirmed INTEGER DEFAULT 0, -- 1 = Kai said so explicitly
|
||
hits INTEGER DEFAULT 0,
|
||
created_at REAL,
|
||
note TEXT DEFAULT '',
|
||
PRIMARY KEY (alias, canonical)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS alias_by_alias ON alias(alias);
|
||
CREATE TABLE IF NOT EXISTS correction (
|
||
surface TEXT NOT NULL, -- what the ASR wrote
|
||
canonical TEXT NOT NULL, -- what it should be
|
||
n INTEGER DEFAULT 1, -- times confirmed
|
||
confirmed INTEGER DEFAULT 0,
|
||
last_seen REAL,
|
||
note TEXT DEFAULT '',
|
||
PRIMARY KEY (surface, canonical)
|
||
);
|
||
CREATE TABLE IF NOT EXISTS relation (
|
||
subject TEXT NOT NULL,
|
||
predicate TEXT NOT NULL,
|
||
object TEXT NOT NULL,
|
||
confidence REAL DEFAULT 0.5,
|
||
source TEXT DEFAULT '',
|
||
first_seen REAL,
|
||
last_seen REAL,
|
||
active INTEGER DEFAULT 1,
|
||
PRIMARY KEY (subject, predicate, object, source)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS relation_by_subject ON relation(subject, active, last_seen);
|
||
CREATE INDEX IF NOT EXISTS relation_by_object ON relation(object, active, last_seen);
|
||
CREATE TABLE IF NOT EXISTS interaction (
|
||
entity TEXT NOT NULL,
|
||
occurred_at REAL NOT NULL,
|
||
kind TEXT DEFAULT 'mention',
|
||
source TEXT DEFAULT '',
|
||
weight REAL DEFAULT 1.0,
|
||
PRIMARY KEY (entity, occurred_at, kind, source)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS interaction_recent ON interaction(entity, occurred_at DESC);
|
||
CREATE TABLE IF NOT EXISTS entity_mention (
|
||
entity TEXT NOT NULL,
|
||
kind TEXT NOT NULL, -- 人名 / 公司
|
||
source TEXT NOT NULL, -- one formal minute; one count per source
|
||
occurred_at REAL NOT NULL,
|
||
PRIMARY KEY (entity, kind, source)
|
||
);
|
||
CREATE INDEX IF NOT EXISTS entity_mention_count ON entity_mention(entity,kind);
|
||
CREATE TABLE IF NOT EXISTS observation (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
surface TEXT NOT NULL,
|
||
canonical TEXT DEFAULT '',
|
||
source TEXT DEFAULT '',
|
||
context TEXT DEFAULT '',
|
||
confidence REAL DEFAULT 0.0,
|
||
confirmed INTEGER DEFAULT 0,
|
||
created_at REAL NOT NULL
|
||
);
|
||
CREATE INDEX IF NOT EXISTS observation_surface ON observation(surface, created_at DESC);
|
||
CREATE TABLE IF NOT EXISTS phonetic_confusion (
|
||
heard TEXT NOT NULL,
|
||
intended TEXT NOT NULL,
|
||
n INTEGER DEFAULT 1,
|
||
confirmed INTEGER DEFAULT 0,
|
||
last_seen REAL,
|
||
note TEXT DEFAULT '',
|
||
PRIMARY KEY (heard, intended)
|
||
);
|
||
CREATE TABLE IF NOT EXISTS review_question (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
recording TEXT NOT NULL,
|
||
question TEXT NOT NULL,
|
||
payload TEXT DEFAULT '{}',
|
||
status TEXT DEFAULT 'pending',
|
||
answer TEXT DEFAULT '',
|
||
created_at REAL NOT NULL,
|
||
answered_at REAL
|
||
);
|
||
CREATE INDEX IF NOT EXISTS review_pending ON review_question(status, created_at);
|
||
CREATE TABLE IF NOT EXISTS source_document (
|
||
source TEXT PRIMARY KEY,
|
||
signature TEXT NOT NULL,
|
||
processed_at REAL NOT NULL,
|
||
entity_count INTEGER DEFAULT 0,
|
||
relation_count INTEGER DEFAULT 0
|
||
);
|
||
CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT);
|
||
"""
|
||
|
||
HAN_FULL = re.compile(r"^[\u4e00-\u9fff]+$")
|
||
# Anything that looks like a note title or a date is not an entity. This is what
|
||
# polluted in_play: 25 of 70 slots were minutes filenames, dates and _glossary.
|
||
JUNK = re.compile(r"^\d{4}-\d{2}|^_|\.md$|[|/#]")
|
||
|
||
|
||
ORG_SUFFIX = ("生物", "医药", "制药", "药业", "基金", "资本", "投资", "科技", "集团",
|
||
"股份", "公司", "银行", "证券", "医院", "研究院", "实验室", "中心")
|
||
|
||
|
||
def guess_kind(name: str) -> str:
|
||
"""Entity type for a name with no card. Only used for display labelling."""
|
||
return "公司" if name.endswith(ORG_SUFFIX) else "人名"
|
||
|
||
|
||
@dataclass
|
||
class Entity:
|
||
canonical: str
|
||
kind: str
|
||
company: str = ""
|
||
role: str = ""
|
||
related: tuple = ()
|
||
|
||
|
||
def connect() -> sqlite3.Connection:
|
||
CFG.state_dir.mkdir(parents=True, exist_ok=True)
|
||
con = sqlite3.connect(CFG.lexicon_db, timeout=30)
|
||
con.execute("PRAGMA journal_mode=WAL")
|
||
con.row_factory = sqlite3.Row
|
||
con.executescript(SCHEMA)
|
||
# One-time compatibility migration for the 158 corrections learned before the
|
||
# personal phonetic-confusion channel existed.
|
||
con.execute(
|
||
"INSERT OR IGNORE INTO phonetic_confusion(heard,intended,n,confirmed,"
|
||
"last_seen,note) SELECT surface,canonical,n,confirmed,last_seen,note "
|
||
"FROM correction")
|
||
con.commit()
|
||
return con
|
||
|
||
|
||
# --------------------------------------------------------------- vault pull
|
||
def _fm(text: str, key: str) -> str:
|
||
# `[ \t]*` not `\s*`: \s matches newlines, so a blank `role:` field used to
|
||
# swallow the next line and report `status: active` as the role.
|
||
m = re.search(rf"^{key}:[ \t]*(.*)$", text, re.M)
|
||
return m.group(1).strip().strip("\"'") if m else ""
|
||
|
||
|
||
def _list_field(raw: str) -> list[str]:
|
||
return [x for x in re.findall(r"[^,\[\]\s，、\"']+", raw or "") if x]
|
||
|
||
|
||
def vault_signature() -> str:
|
||
import hashlib
|
||
h = hashlib.sha256()
|
||
if CFG.glossary.exists():
|
||
st = CFG.glossary.stat()
|
||
h.update(f"gl:{st.st_mtime_ns}:{st.st_size}".encode())
|
||
for sub in ("People", "Companies", "Projects"):
|
||
d = CFG.vault / sub
|
||
if not d.exists():
|
||
continue
|
||
for f in sorted(d.glob("*.md")):
|
||
st = f.stat()
|
||
h.update(f"{f.name}:{st.st_mtime_ns}:{st.st_size}".encode())
|
||
return h.hexdigest()[:16]
|
||
|
||
|
||
GL_MAP = re.compile(r"^(?:>\s*)?[-*]\s*(.+?)\s*(?:→|⇒|➔|->)\s*(.+)$")
|
||
GL_ENT = re.compile(r"^[-*]\s*([\u4e00-\u9fff]{2,6})\s*[—–]\s*(.+)$")
|
||
_PAREN = re.compile(r"[（(][^）)]*[）)]")
|
||
|
||
|
||
def _clean_form(s: str) -> str:
|
||
return re.sub(r"\*\*|\[\[|\]\]|[`\"'「」“”]", "", s).strip()
|
||
|
||
|
||
def _split_forms(s: str) -> list[str]:
|
||
"""`胡律 / 胡丽 / 胡博士（跨境资本语境）` -> [胡律, 胡丽, 胡博士]."""
|
||
s = _PAREN.sub("", s)
|
||
return [_clean_form(x) for x in re.split(r"[/、,，]|\s+or\s+", s) if _clean_form(x)]
|
||
|
||
|
||
def _split_targets(rhs: str) -> list[tuple[str, str]]:
|
||
"""Right-hand side -> [(canonical, context_hint)].
|
||
|
||
Handles the one-to-many context-dependent rows the glossary explicitly flags,
|
||
e.g. `许博 → **许俊彦**(白帆生物语境) / **许恒皓**(愚公生物语境)`, which must stay
|
||
ambiguous: the pipeline offers both and the sentence decides.
|
||
"""
|
||
parts = re.split(r"\s*/\s*(?=\*\*)", rhs) if rhs.count("**") > 1 else [rhs]
|
||
out = []
|
||
for it in parts:
|
||
ctx = "；".join(x.strip() for x in re.findall(r"[（(]([^）)]*)[）)]", it))
|
||
name = _clean_form(_PAREN.sub("", it)).split("；")[0].split(";")[0].strip()
|
||
name = name.split("，")[0].strip()
|
||
if name:
|
||
out.append((name, ctx))
|
||
return out
|
||
|
||
|
||
def _import_glossary(con: sqlite3.Connection, now: float) -> int:
|
||
"""Read `minutes/_glossary.md` into the lexicon. The file is never modified.
|
||
|
||
It carries ~130 hand-confirmed mappings, several of which are deliberately
|
||
one-to-many. Importing them means the pipeline stops re-deriving what Kai
|
||
already settled, while the vault keeps a single human-editable copy.
|
||
"""
|
||
gl = CFG.glossary
|
||
if not gl.exists():
|
||
return 0
|
||
n = 0
|
||
|
||
def put(alias: str, canonical: str, ctx: str, origin: str):
|
||
nonlocal n
|
||
if not (HAN_FULL.match(alias) and HAN_FULL.match(canonical)):
|
||
return
|
||
if alias == canonical or not (2 <= len(alias) <= 8) or not (2 <= len(canonical) <= 8):
|
||
return
|
||
r = con.execute("SELECT kind FROM entity WHERE canonical=?", (canonical,)).fetchone()
|
||
kind = r["kind"] if r else guess_kind(canonical)
|
||
con.execute(
|
||
"INSERT OR IGNORE INTO alias(alias,canonical,kind,origin,confirmed,"
|
||
"created_at,note) VALUES(?,?,?,?,1,?,?)",
|
||
(alias, canonical, kind, origin, now, ctx))
|
||
con.execute(
|
||
"INSERT OR IGNORE INTO correction(surface,canonical,n,confirmed,last_seen,note)"
|
||
" VALUES(?,?,1,1,?,?)", (alias, canonical, now, ctx))
|
||
n += 1
|
||
|
||
for raw in gl.read_text(errors="ignore").splitlines():
|
||
line = raw.strip()
|
||
if not line or line.startswith("#"):
|
||
continue
|
||
m = GL_MAP.match(line)
|
||
if m:
|
||
for tgt, ctx in _split_targets(m.group(2)):
|
||
for form in _split_forms(m.group(1)):
|
||
put(form, tgt, ctx, "glossary")
|
||
continue
|
||
# `- 万亚坤 — 洛启生物实控人（老万；原音 万博/王博/「平台总代理坤」）`
|
||
m = GL_ENT.match(line)
|
||
if m:
|
||
canon, desc = m.group(1), m.group(2)
|
||
for grp in re.findall(r"[（(]([^）)]*)[）)]", desc):
|
||
for seg in re.split(r"[；;]", grp):
|
||
seg = re.sub(r"^\s*(原音|又称|简称|别名)\s*", "", seg.strip())
|
||
for form in _split_forms(seg):
|
||
put(form, canon, "", "glossary")
|
||
return n
|
||
|
||
|
||
def sync_from_vault(force: bool = False) -> dict:
|
||
"""Pull entities + card aliases from Obsidian. Read-only on the vault side.
|
||
|
||
Learned rows are preserved: only `origin='card'` aliases are refreshed, so a
|
||
nickname Kai confirmed once is never lost when a card is edited.
|
||
"""
|
||
con = connect()
|
||
sig = vault_signature()
|
||
cur = con.execute("SELECT v FROM meta WHERE k='vault_sig'").fetchone()
|
||
if not force and cur and cur["v"] == sig:
|
||
con.close()
|
||
return {"skipped": True, "sig": sig}
|
||
|
||
now = time.time()
|
||
n_ent = n_alias = 0
|
||
con.execute("DELETE FROM alias WHERE origin IN ('card','glossary')")
|
||
con.execute("DELETE FROM entity WHERE card <> ''")
|
||
con.execute("DELETE FROM relation WHERE source LIKE 'card:%'")
|
||
for sub, kind in (("People", "人名"), ("Companies", "公司"), ("Projects", "项目")):
|
||
d = CFG.vault / sub
|
||
if not d.exists():
|
||
continue
|
||
for f in sorted(d.glob("*.md")):
|
||
name = f.stem
|
||
if JUNK.search(name):
|
||
continue
|
||
if kind != "项目" and not HAN_FULL.match(name):
|
||
continue
|
||
if kind == "项目" and not (2 <= len(name) <= 80):
|
||
continue
|
||
txt = f.read_text(errors="ignore")
|
||
head = txt[:1200]
|
||
company = _fm(head, "company")
|
||
related = _list_field(_fm(head, "related_people"))
|
||
con.execute(
|
||
"INSERT OR REPLACE INTO entity(canonical,kind,company,role,related,"
|
||
"card,synced_at) VALUES(?,?,?,?,?,?,?)",
|
||
(name, kind, company, _fm(head, "role"),
|
||
",".join(related), str(f), now))
|
||
if company:
|
||
con.execute(
|
||
"INSERT OR REPLACE INTO relation(subject,predicate,object,"
|
||
"confidence,source,first_seen,last_seen) VALUES(?,?,?,?,?,?,?)",
|
||
(name, "任职于", company, 1.0, f"card:{f}", now, now))
|
||
for other in related:
|
||
con.execute(
|
||
"INSERT OR REPLACE INTO relation(subject,predicate,object,"
|
||
"confidence,source,first_seen,last_seen) VALUES(?,?,?,?,?,?,?)",
|
||
(name, "相关", other, 0.9, f"card:{f}", now, now))
|
||
n_ent += 1
|
||
for a in _list_field(_fm(head, "aliases")):
|
||
if a == name or not HAN_FULL.match(a) or len(a) < 2:
|
||
continue
|
||
con.execute(
|
||
"INSERT OR IGNORE INTO alias(alias,canonical,kind,origin,confirmed,"
|
||
"created_at) VALUES(?,?,?,'card',1,?)", (a, name, kind, now))
|
||
n_alias += 1
|
||
# The vault glossary is *imported*, never edited. It holds ~130 corrections Kai
|
||
# already confirmed by hand; they become exact-match bindings here so the
|
||
# pipeline stops re-deriving them, and the file itself stays untouched.
|
||
n_gl = _import_glossary(con, now)
|
||
|
||
con.execute("INSERT OR REPLACE INTO meta(k,v) VALUES('vault_sig',?)", (sig,))
|
||
con.execute("INSERT OR REPLACE INTO meta(k,v) VALUES('synced_at',?)", (str(now),))
|
||
con.commit()
|
||
con.close()
|
||
return {"skipped": False, "entities": n_ent, "card_aliases": n_alias,
|
||
"glossary": n_gl, "sig": sig}
|
||
|
||
|
||
# ------------------------------------------------------------------ reading
|
||
def entities() -> dict[str, Entity]:
|
||
con = connect()
|
||
out = {r["canonical"]: Entity(
|
||
r["canonical"], r["kind"], r["company"] or "", r["role"] or "",
|
||
tuple(x for x in (r["related"] or "").split(",") if x))
|
||
for r in con.execute("SELECT * FROM entity")}
|
||
con.close()
|
||
return out
|
||
|
||
|
||
def alias_map() -> dict[str, list[tuple[str, str, str, int]]]:
|
||
"""alias -> [(canonical, kind, origin, confirmed)]. May be ambiguous."""
|
||
con = connect()
|
||
out: dict[str, list] = {}
|
||
for r in con.execute("SELECT * FROM alias ORDER BY confirmed DESC, hits DESC"):
|
||
out.setdefault(r["alias"], []).append(
|
||
(r["canonical"], r["kind"], r["origin"], r["confirmed"]))
|
||
con.close()
|
||
return out
|
||
|
||
|
||
def corrections() -> dict[str, list[tuple[str, int]]]:
|
||
con = connect()
|
||
out: dict[str, list] = {}
|
||
for r in con.execute("SELECT * FROM correction ORDER BY n DESC"):
|
||
out.setdefault(r["surface"], []).append((r["canonical"], r["n"]))
|
||
con.close()
|
||
return out
|
||
|
||
|
||
def confusion_map() -> dict[str, list[tuple[str, int, int]]]:
|
||
"""Observed ASR surface -> canonical, count, confirmation.
|
||
|
||
Older databases predate `phonetic_confusion`, so confirmed correction rows are
|
||
included as a compatibility source without a destructive migration.
|
||
"""
|
||
con = connect()
|
||
out: dict[str, list[tuple[str, int, int]]] = {}
|
||
for r in con.execute(
|
||
"SELECT surface AS heard,canonical AS intended,n,confirmed "
|
||
"FROM correction UNION ALL "
|
||
"SELECT heard,intended,n,confirmed FROM phonetic_confusion"):
|
||
item = (r["intended"], r["n"], r["confirmed"])
|
||
if item not in out.setdefault(r["heard"], []):
|
||
out[r["heard"]].append(item)
|
||
con.close()
|
||
return out
|
||
|
||
|
||
# ------------------------------------------------------------------ writing
|
||
def learn_alias(alias: str, canonical: str, *, kind: str = "", origin: str = "learned",
|
||
confirmed: int = 1, note: str = "") -> None:
|
||
con = connect()
|
||
if not kind:
|
||
r = con.execute("SELECT kind FROM entity WHERE canonical=?", (canonical,)).fetchone()
|
||
kind = r["kind"] if r else guess_kind(canonical)
|
||
con.execute(
|
||
"INSERT INTO alias(alias,canonical,kind,origin,confirmed,created_at,note)"
|
||
" VALUES(?,?,?,?,?,?,?) ON CONFLICT(alias,canonical) DO UPDATE SET"
|
||
" confirmed=max(confirmed,excluded.confirmed), origin=excluded.origin,"
|
||
" note=excluded.note",
|
||
(alias, canonical, kind, origin, confirmed, time.time(), note))
|
||
con.commit()
|
||
con.close()
|
||
|
||
|
||
def learn_correction(surface: str, canonical: str, *, confirmed: int = 1,
|
||
note: str = "") -> None:
|
||
con = connect()
|
||
con.execute(
|
||
"INSERT INTO correction(surface,canonical,n,confirmed,last_seen,note)"
|
||
" VALUES(?,?,1,?,?,?) ON CONFLICT(surface,canonical) DO UPDATE SET"
|
||
" n=n+1, confirmed=max(confirmed,excluded.confirmed), last_seen=excluded.last_seen",
|
||
(surface, canonical, confirmed, time.time(), note))
|
||
con.execute(
|
||
"INSERT INTO phonetic_confusion(heard,intended,n,confirmed,last_seen,note)"
|
||
" VALUES(?,?,1,?,?,?) ON CONFLICT(heard,intended) DO UPDATE SET"
|
||
" n=n+1,confirmed=max(confirmed,excluded.confirmed),"
|
||
" last_seen=excluded.last_seen,note=excluded.note",
|
||
(surface, canonical, confirmed, time.time(), note))
|
||
con.commit()
|
||
con.close()
|
||
|
||
|
||
def record_observation(surface: str, canonical: str = "", *, source: str = "",
|
||
context: str = "", confidence: float = 0.0,
|
||
confirmed: int = 0) -> int:
|
||
con = connect()
|
||
cur = con.execute(
|
||
"INSERT INTO observation(surface,canonical,source,context,confidence,"
|
||
"confirmed,created_at) VALUES(?,?,?,?,?,?,?)",
|
||
(surface, canonical, source, context[:1000], confidence, confirmed, time.time()))
|
||
con.commit()
|
||
row_id = int(cur.lastrowid)
|
||
con.close()
|
||
return row_id
|
||
|
||
|
||
def upsert_relation(subject: str, predicate: str, object_: str, *,
|
||
confidence: float = 0.5, source: str = "") -> None:
|
||
now = time.time()
|
||
con = connect()
|
||
con.execute(
|
||
"INSERT INTO relation(subject,predicate,object,confidence,source,first_seen,"
|
||
"last_seen) VALUES(?,?,?,?,?,?,?) ON CONFLICT(subject,predicate,object,source)"
|
||
" DO UPDATE SET confidence=max(confidence,excluded.confidence),"
|
||
" last_seen=excluded.last_seen,active=1",
|
||
(subject, predicate, object_, confidence, source, now, now))
|
||
con.commit()
|
||
con.close()
|
||
|
||
|
||
def record_interaction(entity: str, *, occurred_at: float | None = None,
|
||
kind: str = "mention", source: str = "",
|
||
weight: float = 1.0) -> None:
|
||
con = connect()
|
||
con.execute(
|
||
"INSERT OR IGNORE INTO interaction(entity,occurred_at,kind,source,weight)"
|
||
" VALUES(?,?,?,?,?)",
|
||
(entity, occurred_at or time.time(), kind, source, weight))
|
||
con.commit()
|
||
con.close()
|
||
|
||
|
||
def relationship_card(names: set[str], *, limit: int = 30) -> list[str]:
|
||
"""Return compact, recent relation facts for other applications and ASR."""
|
||
if not names:
|
||
return []
|
||
con = connect()
|
||
marks = ",".join("?" for _ in names)
|
||
rows = con.execute(
|
||
f"SELECT subject,predicate,object,max(confidence) confidence,"
|
||
f"max(last_seen) last_seen,count(*) support FROM relation "
|
||
f"WHERE active=1 AND (subject IN ({marks}) OR object IN ({marks})) "
|
||
"GROUP BY subject,predicate,object "
|
||
"ORDER BY confidence DESC,last_seen DESC,support DESC LIMIT ?",
|
||
(*names, *names, limit)).fetchall()
|
||
con.close()
|
||
return [f"{r['subject']} {r['predicate']} {r['object']}"
|
||
f"（置信度 {r['confidence']:.2f}，支持 {r['support']} 次）" for r in rows]
|
||
|
||
|
||
def bump_hits(pairs) -> None:
|
||
if not pairs:
|
||
return
|
||
con = connect()
|
||
con.executemany("UPDATE alias SET hits=hits+1 WHERE alias=? AND canonical=?",
|
||
list(pairs))
|
||
con.commit()
|
||
con.close()
|
||
|
||
|
||
def stats() -> dict:
|
||
con = connect()
|
||
q = lambda s: con.execute(s).fetchone()[0]
|
||
d = {
|
||
"entities": q("SELECT count(*) FROM entity"),
|
||
"people": q("SELECT count(*) FROM entity WHERE kind='人名'"),
|
||
"companies": q("SELECT count(*) FROM entity WHERE kind='公司'"),
|
||
"with_company": q("SELECT count(*) FROM entity WHERE company<>''"),
|
||
"with_related": q("SELECT count(*) FROM entity WHERE related<>''"),
|
||
"aliases": q("SELECT count(*) FROM alias"),
|
||
"aliases_card": q("SELECT count(*) FROM alias WHERE origin='card'"),
|
||
"aliases_learned": q("SELECT count(*) FROM alias WHERE origin<>'card'"),
|
||
"corrections": q("SELECT count(*) FROM correction"),
|
||
"db": str(CFG.lexicon_db),
|
||
}
|
||
r = con.execute("SELECT v FROM meta WHERE k='synced_at'").fetchone()
|
||
d["synced_at"] = r["v"] if r else ""
|
||
con.close()
|
||
return d
|
||
|
||
|
||
# ------------------------------------------------------------------ Web UI CRUD
|
||
def list_aliases(q: str = "", origin_filter: str = "", limit: int = 200, offset: int = 0) -> list[dict]:
|
||
con = connect()
|
||
sql = "SELECT * FROM alias WHERE 1=1"
|
||
args: list = []
|
||
if q.strip():
|
||
pat = f"%{q.strip()}%"
|
||
sql += " AND (alias LIKE ? OR canonical LIKE ? OR note LIKE ?)"
|
||
args.extend([pat, pat, pat])
|
||
if origin_filter == "learned":
|
||
sql += " AND origin <> 'card'"
|
||
elif origin_filter == "card":
|
||
sql += " AND origin = 'card'"
|
||
sql += " ORDER BY confirmed DESC, created_at DESC, alias ASC LIMIT ? OFFSET ?"
|
||
args.extend((max(1, min(limit, 500)), max(0, offset)))
|
||
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
|
||
con.close()
|
||
return rows
|
||
|
||
|
||
def count_aliases(q: str = "", origin_filter: str = "") -> int:
|
||
con = connect()
|
||
sql = "SELECT count(*) FROM alias WHERE 1=1"
|
||
args: list = []
|
||
if q.strip():
|
||
pat = f"%{q.strip()}%"
|
||
sql += " AND (alias LIKE ? OR canonical LIKE ? OR note LIKE ?)"
|
||
args.extend([pat, pat, pat])
|
||
if origin_filter == "learned":
|
||
sql += " AND origin <> 'card'"
|
||
elif origin_filter == "card":
|
||
sql += " AND origin = 'card'"
|
||
n = int(con.execute(sql, args).fetchone()[0])
|
||
con.close()
|
||
return n
|
||
|
||
|
||
def update_alias(
|
||
old_alias: str,
|
||
old_canonical: str,
|
||
new_alias: str,
|
||
new_canonical: str,
|
||
*,
|
||
kind: str = "",
|
||
confirmed: int = 1,
|
||
note: str = "",
|
||
) -> bool:
|
||
con = connect()
|
||
row = con.execute(
|
||
"SELECT * FROM alias WHERE alias=? AND canonical=?",
|
||
(old_alias, old_canonical),
|
||
).fetchone()
|
||
origin = row["origin"] if row else "manual"
|
||
hits = row["hits"] if row else 0
|
||
created_at = row["created_at"] if row else time.time()
|
||
if not kind:
|
||
er = con.execute("SELECT kind FROM entity WHERE canonical=?", (new_canonical,)).fetchone()
|
||
kind = er["kind"] if er else (row["kind"] if row else guess_kind(new_canonical))
|
||
con.execute(
|
||
"DELETE FROM alias WHERE alias=? AND canonical=?",
|
||
(old_alias, old_canonical),
|
||
)
|
||
con.execute(
|
||
"INSERT OR REPLACE INTO alias(alias,canonical,kind,origin,confirmed,hits,created_at,note) "
|
||
"VALUES(?,?,?,?,?,?,?,?)",
|
||
(new_alias, new_canonical, kind, origin, int(bool(confirmed)), hits, created_at, note),
|
||
)
|
||
con.commit()
|
||
con.close()
|
||
return True
|
||
|
||
|
||
def delete_alias(alias: str, canonical: str) -> bool:
|
||
con = connect()
|
||
cur = con.execute("DELETE FROM alias WHERE alias=? AND canonical=?", (alias, canonical))
|
||
con.commit()
|
||
deleted = cur.rowcount > 0
|
||
con.close()
|
||
return deleted
|
||
|
||
|
||
def list_corrections(q: str = "", limit: int = 200, offset: int = 0) -> list[dict]:
|
||
con = connect()
|
||
sql = "SELECT * FROM correction WHERE 1=1"
|
||
args: list = []
|
||
if q.strip():
|
||
pat = f"%{q.strip()}%"
|
||
sql += " AND (surface LIKE ? OR canonical LIKE ? OR note LIKE ?)"
|
||
args.extend([pat, pat, pat])
|
||
sql += " ORDER BY confirmed DESC, n DESC, last_seen DESC LIMIT ? OFFSET ?"
|
||
args.extend((max(1, min(limit, 500)), max(0, offset)))
|
||
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
|
||
con.close()
|
||
return rows
|
||
|
||
|
||
def count_corrections(q: str = "") -> int:
|
||
con = connect()
|
||
sql = "SELECT count(*) FROM correction WHERE 1=1"
|
||
args: list = []
|
||
if q.strip():
|
||
pat = f"%{q.strip()}%"
|
||
sql += " AND (surface LIKE ? OR canonical LIKE ? OR note LIKE ?)"
|
||
args.extend([pat, pat, pat])
|
||
n = int(con.execute(sql, args).fetchone()[0])
|
||
con.close()
|
||
return n
|
||
|
||
|
||
def update_correction(
|
||
old_surface: str,
|
||
old_canonical: str,
|
||
new_surface: str,
|
||
new_canonical: str,
|
||
*,
|
||
confirmed: int = 1,
|
||
note: str = "",
|
||
) -> bool:
|
||
con = connect()
|
||
row = con.execute(
|
||
"SELECT * FROM correction WHERE surface=? AND canonical=?",
|
||
(old_surface, old_canonical),
|
||
).fetchone()
|
||
n = row["n"] if row else 1
|
||
now = time.time()
|
||
con.execute("DELETE FROM correction WHERE surface=? AND canonical=?", (old_surface, old_canonical))
|
||
con.execute(
|
||
"DELETE FROM phonetic_confusion WHERE heard=? AND intended=?",
|
||
(old_surface, old_canonical),
|
||
)
|
||
con.execute(
|
||
"INSERT OR REPLACE INTO correction(surface,canonical,n,confirmed,last_seen,note) "
|
||
"VALUES(?,?,?,?,?,?)",
|
||
(new_surface, new_canonical, n, int(bool(confirmed)), now, note),
|
||
)
|
||
con.execute(
|
||
"INSERT OR REPLACE INTO phonetic_confusion(heard,intended,n,confirmed,last_seen,note) "
|
||
"VALUES(?,?,?,?,?,?)",
|
||
(new_surface, new_canonical, n, int(bool(confirmed)), now, note),
|
||
)
|
||
con.commit()
|
||
con.close()
|
||
return True
|
||
|
||
|
||
def delete_correction(surface: str, canonical: str) -> bool:
|
||
con = connect()
|
||
cur = con.execute("DELETE FROM correction WHERE surface=? AND canonical=?", (surface, canonical))
|
||
con.execute("DELETE FROM phonetic_confusion WHERE heard=? AND intended=?", (surface, canonical))
|
||
con.commit()
|
||
deleted = cur.rowcount > 0
|
||
con.close()
|
||
return deleted
|
||
|
||
|
||
def list_entities(q: str = "", kind: str = "", limit: int = 200, offset: int = 0) -> list[dict]:
|
||
con = connect()
|
||
sql = "SELECT * FROM entity WHERE 1=1"
|
||
args: list = []
|
||
if q.strip():
|
||
pat = f"%{q.strip()}%"
|
||
sql += " AND (canonical LIKE ? OR company LIKE ? OR role LIKE ? OR related LIKE ?)"
|
||
args.extend([pat, pat, pat, pat])
|
||
if kind.strip():
|
||
sql += " AND kind=?"
|
||
args.append(kind.strip())
|
||
sql += " ORDER BY (CASE WHEN card='' THEN 0 ELSE 1 END), canonical ASC LIMIT ? OFFSET ?"
|
||
args.extend((max(1, min(limit, 500)), max(0, offset)))
|
||
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
|
||
con.close()
|
||
return rows
|
||
|
||
|
||
def count_entities(q: str = "", kind: str = "") -> int:
|
||
con = connect()
|
||
sql = "SELECT count(*) FROM entity WHERE 1=1"
|
||
args: list = []
|
||
if q.strip():
|
||
pat = f"%{q.strip()}%"
|
||
sql += " AND (canonical LIKE ? OR company LIKE ? OR role LIKE ? OR related LIKE ?)"
|
||
args.extend([pat, pat, pat, pat])
|
||
if kind.strip():
|
||
sql += " AND kind=?"
|
||
args.append(kind.strip())
|
||
n = int(con.execute(sql, args).fetchone()[0])
|
||
con.close()
|
||
return n
|
||
|
||
|
||
def upsert_entity(
|
||
canonical: str,
|
||
kind: str = "人名",
|
||
*,
|
||
company: str = "",
|
||
role: str = "",
|
||
related: str = "",
|
||
card: str = "",
|
||
) -> None:
|
||
con = connect()
|
||
con.execute(
|
||
"INSERT OR REPLACE INTO entity(canonical,kind,company,role,related,card,synced_at) "
|
||
"VALUES(?,?,?,?,?,?,?)",
|
||
(canonical, kind or guess_kind(canonical), company, role, related, card, time.time()),
|
||
)
|
||
con.commit()
|
||
con.close()
|
||
|
||
|
||
def update_entity(
|
||
old_canonical: str,
|
||
new_canonical: str,
|
||
kind: str,
|
||
*,
|
||
company: str = "",
|
||
role: str = "",
|
||
related: str = "",
|
||
) -> bool:
|
||
con = connect()
|
||
row = con.execute("SELECT * FROM entity WHERE canonical=?", (old_canonical,)).fetchone()
|
||
card = row["card"] if row else ""
|
||
con.execute("DELETE FROM entity WHERE canonical=?", (old_canonical,))
|
||
con.execute(
|
||
"INSERT OR REPLACE INTO entity(canonical,kind,company,role,related,card,synced_at) "
|
||
"VALUES(?,?,?,?,?,?,?)",
|
||
(new_canonical, kind or guess_kind(new_canonical), company, role, related, card, time.time()),
|
||
)
|
||
if old_canonical != new_canonical:
|
||
con.execute("UPDATE alias SET canonical=? WHERE canonical=?", (new_canonical, old_canonical))
|
||
con.execute("UPDATE correction SET canonical=? WHERE canonical=?", (new_canonical, old_canonical))
|
||
con.commit()
|
||
con.close()
|
||
return True
|
||
|
||
|
||
def delete_entity(canonical: str) -> bool:
|
||
con = connect()
|
||
cur = con.execute("DELETE FROM entity WHERE canonical=?", (canonical,))
|
||
con.commit()
|
||
deleted = cur.rowcount > 0
|
||
con.close()
|
||
return deleted
|
||
|
||
|
||
def list_relations(q: str = "", limit: int = 200, offset: int = 0) -> list[dict]:
|
||
con = connect()
|
||
sql = "SELECT * FROM relation WHERE 1=1"
|
||
args: list = []
|
||
if q.strip():
|
||
pat = f"%{q.strip()}%"
|
||
sql += " AND (subject LIKE ? OR predicate LIKE ? OR object LIKE ? OR source LIKE ?)"
|
||
args.extend([pat, pat, pat, pat])
|
||
sql += " ORDER BY active DESC, confidence DESC, last_seen DESC LIMIT ? OFFSET ?"
|
||
args.extend((max(1, min(limit, 500)), max(0, offset)))
|
||
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
|
||
con.close()
|
||
return rows
|
||
|
||
|
||
def count_relations(q: str = "") -> int:
|
||
con = connect()
|
||
sql = "SELECT count(*) FROM relation WHERE 1=1"
|
||
args: list = []
|
||
if q.strip():
|
||
pat = f"%{q.strip()}%"
|
||
sql += " AND (subject LIKE ? OR predicate LIKE ? OR object LIKE ? OR source LIKE ?)"
|
||
args.extend([pat, pat, pat, pat])
|
||
n = int(con.execute(sql, args).fetchone()[0])
|
||
con.close()
|
||
return n
|
||
|
||
|
||
def update_relation(
|
||
old_subject: str,
|
||
old_predicate: str,
|
||
old_object: str,
|
||
old_source: str,
|
||
subject: str,
|
||
predicate: str,
|
||
object_: str,
|
||
*,
|
||
confidence: float = 0.8,
|
||
active: int = 1,
|
||
) -> bool:
|
||
con = connect()
|
||
row = con.execute(
|
||
"SELECT * FROM relation WHERE subject=? AND predicate=? AND object=? AND source=?",
|
||
(old_subject, old_predicate, old_object, old_source),
|
||
).fetchone()
|
||
first_seen = row["first_seen"] if row else time.time()
|
||
source = old_source or "manual"
|
||
con.execute(
|
||
"DELETE FROM relation WHERE subject=? AND predicate=? AND object=? AND source=?",
|
||
(old_subject, old_predicate, old_object, old_source),
|
||
)
|
||
con.execute(
|
||
"INSERT OR REPLACE INTO relation(subject,predicate,object,confidence,source,first_seen,last_seen,active) "
|
||
"VALUES(?,?,?,?,?,?,?,?)",
|
||
(subject, predicate, object_, float(confidence), source, first_seen, time.time(), int(bool(active))),
|
||
)
|
||
con.commit()
|
||
con.close()
|
||
return True
|
||
|
||
|
||
def delete_relation(subject: str, predicate: str, object_: str, source: str) -> bool:
|
||
con = connect()
|
||
cur = con.execute(
|
||
"DELETE FROM relation WHERE subject=? AND predicate=? AND object=? AND source=?",
|
||
(subject, predicate, object_, source),
|
||
)
|
||
con.commit()
|
||
deleted = cur.rowcount > 0
|
||
con.close()
|
||
return deleted
|
||
|