218 lines
8.4 KiB
Python
218 lines
8.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Incremental entity/relation/recency index over curated Obsidian minutes."""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import math
|
|
import re
|
|
import time
|
|
from datetime import date, datetime
|
|
from pathlib import Path
|
|
|
|
from . import lexicon
|
|
from .config import CFG
|
|
|
|
FM_KEYS = ("people", "companies", "projects", "project")
|
|
LINK = re.compile(r"\[\[([^\]|#]+)")
|
|
DATE = re.compile(r"^date:[ \t]*(\d{4}-\d{2}-\d{2})", re.M)
|
|
SENTENCE = re.compile(r"(?<=[。！？；\n])")
|
|
PREDICATES = (
|
|
"投资", "任职", "负责", "参与", "合作", "对接", "收购", "并购",
|
|
"创立", "加入", "管理", "支持", "服务", "委托", "出资", "控股",
|
|
)
|
|
|
|
|
|
def _list_field(text: str, key: str) -> set[str]:
|
|
m = re.search(rf"^{key}:[ \t]*(.*)$", text, re.M)
|
|
if not m:
|
|
return set()
|
|
return {x.strip() for x in re.findall(r"[^,\[\]\s，、]+", m.group(1))
|
|
if x.strip()}
|
|
|
|
|
|
def _timestamp(text: str, path: Path) -> float:
|
|
m = DATE.search(text)
|
|
if m:
|
|
try:
|
|
return datetime.combine(date.fromisoformat(m.group(1)),
|
|
datetime.min.time()).timestamp()
|
|
except ValueError:
|
|
pass
|
|
m = re.search(r"(\d{4}-\d{2}-\d{2})", path.name)
|
|
if m:
|
|
try:
|
|
return datetime.combine(date.fromisoformat(m.group(1)),
|
|
datetime.min.time()).timestamp()
|
|
except ValueError:
|
|
pass
|
|
return path.stat().st_mtime
|
|
|
|
|
|
def _signature(path: Path) -> str:
|
|
st = path.stat()
|
|
return hashlib.sha256(
|
|
f"{path}:{st.st_mtime_ns}:{st.st_size}".encode()).hexdigest()[:20]
|
|
|
|
|
|
def _minutes_files() -> list[Path]:
|
|
root = CFG.vault / "minutes"
|
|
if not root.exists():
|
|
return []
|
|
return sorted(p for p in root.rglob("*.md")
|
|
if "_drafts" not in p.parts and not p.name.startswith("_"))
|
|
|
|
|
|
def _ordered(a: str, b: str, kinds: dict[str, str]) -> tuple[str, str]:
|
|
rank = {"人名": 0, "公司": 1, "项目": 2}
|
|
ka, kb = rank.get(kinds.get(a, ""), 9), rank.get(kinds.get(b, ""), 9)
|
|
return (a, b) if (ka, a) <= (kb, b) else (b, a)
|
|
|
|
|
|
def sync(force: bool = False) -> dict:
|
|
"""Index changed minutes and remove facts belonging to deleted notes."""
|
|
lexicon.sync_from_vault()
|
|
con = lexicon.connect()
|
|
kinds = {r["canonical"]: r["kind"]
|
|
for r in con.execute("SELECT canonical,kind FROM entity")}
|
|
known = set(kinds)
|
|
files = _minutes_files()
|
|
active_sources = {f"minute:{p.relative_to(CFG.vault)}" for p in files}
|
|
|
|
old_sources = {r["source"] for r in con.execute(
|
|
"SELECT source FROM source_document WHERE source LIKE 'minute:%'")}
|
|
removed = old_sources - active_sources
|
|
for source in removed:
|
|
con.execute("DELETE FROM relation WHERE source=?", (source,))
|
|
con.execute("DELETE FROM interaction WHERE source=?", (source,))
|
|
con.execute("DELETE FROM entity_mention WHERE source=?", (source,))
|
|
con.execute("DELETE FROM source_document WHERE source=?", (source,))
|
|
|
|
changed = skipped = entities_n = relations_n = 0
|
|
now = time.time()
|
|
for p in files:
|
|
source = f"minute:{p.relative_to(CFG.vault)}"
|
|
sig = _signature(p)
|
|
old = con.execute("SELECT signature FROM source_document WHERE source=?",
|
|
(source,)).fetchone()
|
|
if not force and old and old["signature"] == sig:
|
|
skipped += 1
|
|
continue
|
|
text = p.read_text(errors="ignore")
|
|
typed = {
|
|
"人名": _list_field(text, "people"),
|
|
"公司": _list_field(text, "companies"),
|
|
}
|
|
con.execute("DELETE FROM entity_mention WHERE source=?", (source,))
|
|
for kind, names in typed.items():
|
|
for name in names:
|
|
con.execute(
|
|
"INSERT OR REPLACE INTO entity_mention(entity,kind,source,"
|
|
"occurred_at) VALUES(?,?,?,?)",
|
|
(name, kind, source, _timestamp(text, p)))
|
|
entities = set()
|
|
for key in FM_KEYS:
|
|
entities |= _list_field(text, key)
|
|
entities |= {x.strip() for x in LINK.findall(text)}
|
|
entities &= known
|
|
occurred = _timestamp(text, p)
|
|
|
|
con.execute("DELETE FROM relation WHERE source=?", (source,))
|
|
con.execute("DELETE FROM interaction WHERE source=?", (source,))
|
|
for entity in entities:
|
|
con.execute(
|
|
"INSERT OR REPLACE INTO interaction(entity,occurred_at,kind,source,"
|
|
"weight) VALUES(?,?,?,?,?)",
|
|
(entity, occurred, "meeting", source, 1.0))
|
|
|
|
nrel = 0
|
|
ordered = sorted(entities)
|
|
for i, a in enumerate(ordered):
|
|
for b in ordered[i + 1:]:
|
|
subject, object_ = _ordered(a, b, kinds)
|
|
con.execute(
|
|
"INSERT OR REPLACE INTO relation(subject,predicate,object,"
|
|
"confidence,source,first_seen,last_seen) VALUES(?,?,?,?,?,?,?)",
|
|
(subject, "同会", object_, 0.35, source, occurred, occurred))
|
|
nrel += 1
|
|
|
|
# A predicate is promoted only when two known linked entities occur in the
|
|
# same sentence and the relationship word is explicit.
|
|
for sentence in SENTENCE.split(text):
|
|
linked = [x.strip() for x in LINK.findall(sentence) if x.strip() in known]
|
|
if len(linked) < 2:
|
|
continue
|
|
predicate = next((x for x in PREDICATES if x in sentence), "")
|
|
if not predicate:
|
|
continue
|
|
for a, b in zip(linked, linked[1:]):
|
|
if a == b:
|
|
continue
|
|
con.execute(
|
|
"INSERT OR REPLACE INTO relation(subject,predicate,object,"
|
|
"confidence,source,first_seen,last_seen) VALUES(?,?,?,?,?,?,?)",
|
|
(a, predicate, b, 0.75, source, occurred, occurred))
|
|
nrel += 1
|
|
|
|
con.execute(
|
|
"INSERT OR REPLACE INTO source_document(source,signature,processed_at,"
|
|
"entity_count,relation_count) VALUES(?,?,?,?,?)",
|
|
(source, sig, now, len(entities), nrel))
|
|
changed += 1
|
|
entities_n += len(entities)
|
|
relations_n += nrel
|
|
con.commit()
|
|
con.close()
|
|
return {"files": len(files), "changed": changed, "skipped": skipped,
|
|
"removed": len(removed), "entities_seen": entities_n,
|
|
"relations_written": relations_n}
|
|
|
|
|
|
def context_weights(seeds: set[str], *, half_life_days: float = 45.0,
|
|
limit: int = 100) -> dict[str, float]:
|
|
"""Score graph neighbours by relation strength, recency, and frequency."""
|
|
if not seeds:
|
|
return {}
|
|
con = lexicon.connect()
|
|
marks = ",".join("?" for _ in seeds)
|
|
rows = con.execute(
|
|
f"SELECT subject,predicate,object,max(confidence) confidence,"
|
|
f"max(last_seen) last_seen,count(*) support FROM relation WHERE active=1 "
|
|
f"AND (subject IN ({marks}) OR object IN ({marks})) "
|
|
"GROUP BY subject,predicate,object",
|
|
(*seeds, *seeds)).fetchall()
|
|
now = time.time()
|
|
out: dict[str, float] = {s: 12.0 for s in seeds}
|
|
for r in rows:
|
|
other = r["object"] if r["subject"] in seeds else r["subject"]
|
|
age = max(0.0, (now - (r["last_seen"] or now)) / 86400)
|
|
decay = math.exp(-math.log(2) * age / half_life_days)
|
|
support = 1.0 + math.log1p(r["support"])
|
|
out[other] = out.get(other, 0.0) + r["confidence"] * decay * support
|
|
|
|
# Direct contact frequency is useful even when a relation has only one edge.
|
|
if out:
|
|
names = list(out)
|
|
marks2 = ",".join("?" for _ in names)
|
|
interactions = con.execute(
|
|
f"SELECT entity,count(*) n,max(occurred_at) latest FROM interaction "
|
|
f"WHERE entity IN ({marks2}) GROUP BY entity", names).fetchall()
|
|
for r in interactions:
|
|
age = max(0.0, (now - r["latest"]) / 86400)
|
|
decay = math.exp(-math.log(2) * age / half_life_days)
|
|
out[r["entity"]] += decay * math.log1p(r["n"])
|
|
con.close()
|
|
return dict(sorted(out.items(), key=lambda x: -x[1])[:limit])
|
|
|
|
|
|
def stats() -> dict:
|
|
con = lexicon.connect()
|
|
q = lambda sql: con.execute(sql).fetchone()[0]
|
|
out = {
|
|
"documents": q("SELECT count(*) FROM source_document"),
|
|
"relations": q("SELECT count(*) FROM relation"),
|
|
"interactions": q("SELECT count(*) FROM interaction"),
|
|
"projects": q("SELECT count(*) FROM entity WHERE kind='项目'"),
|
|
}
|
|
con.close()
|
|
return out
|