#!/usr/bin/env python3 """Incremental entity/relation/recency index over curated Obsidian minutes.""" from __future__ import annotations import hashlib import math import re import time from datetime import date, datetime from pathlib import Path from . import lexicon from .config import CFG FM_KEYS = ("people", "companies", "projects", "project") LINK = re.compile(r"\[\[([^\]|#]+)") DATE = re.compile(r"^date:[ \t]*(\d{4}-\d{2}-\d{2})", re.M) SENTENCE = re.compile(r"(?<=[。!?;\n])") PREDICATES = ( "投资", "任职", "负责", "参与", "合作", "对接", "收购", "并购", "创立", "加入", "管理", "支持", "服务", "委托", "出资", "控股", ) def _list_field(text: str, key: str) -> set[str]: m = re.search(rf"^{key}:[ \t]*(.*)$", text, re.M) if not m: return set() return {x.strip() for x in re.findall(r"[^,\[\]\s,、]+", m.group(1)) if x.strip()} def _timestamp(text: str, path: Path) -> float: m = DATE.search(text) if m: try: return datetime.combine(date.fromisoformat(m.group(1)), datetime.min.time()).timestamp() except ValueError: pass m = re.search(r"(\d{4}-\d{2}-\d{2})", path.name) if m: try: return datetime.combine(date.fromisoformat(m.group(1)), datetime.min.time()).timestamp() except ValueError: pass return path.stat().st_mtime def _signature(path: Path) -> str: st = path.stat() return hashlib.sha256( f"{path}:{st.st_mtime_ns}:{st.st_size}".encode()).hexdigest()[:20] def _minutes_files() -> list[Path]: root = CFG.vault / "minutes" if not root.exists(): return [] return sorted(p for p in root.rglob("*.md") if "_drafts" not in p.parts and not p.name.startswith("_")) def _ordered(a: str, b: str, kinds: dict[str, str]) -> tuple[str, str]: rank = {"人名": 0, "公司": 1, "项目": 2} ka, kb = rank.get(kinds.get(a, ""), 9), rank.get(kinds.get(b, ""), 9) return (a, b) if (ka, a) <= (kb, b) else (b, a) def sync(force: bool = False) -> dict: """Index changed minutes and remove facts belonging to deleted notes.""" lexicon.sync_from_vault() con = lexicon.connect() kinds = {r["canonical"]: r["kind"] for r in con.execute("SELECT canonical,kind FROM entity")} known = set(kinds) files = _minutes_files() active_sources = {f"minute:{p.relative_to(CFG.vault)}" for p in files} old_sources = {r["source"] for r in con.execute( "SELECT source FROM source_document WHERE source LIKE 'minute:%'")} removed = old_sources - active_sources for source in removed: con.execute("DELETE FROM relation WHERE source=?", (source,)) con.execute("DELETE FROM interaction WHERE source=?", (source,)) con.execute("DELETE FROM entity_mention WHERE source=?", (source,)) con.execute("DELETE FROM source_document WHERE source=?", (source,)) changed = skipped = entities_n = relations_n = 0 now = time.time() for p in files: source = f"minute:{p.relative_to(CFG.vault)}" sig = _signature(p) old = con.execute("SELECT signature FROM source_document WHERE source=?", (source,)).fetchone() if not force and old and old["signature"] == sig: skipped += 1 continue text = p.read_text(errors="ignore") typed = { "人名": _list_field(text, "people"), "公司": _list_field(text, "companies"), } con.execute("DELETE FROM entity_mention WHERE source=?", (source,)) for kind, names in typed.items(): for name in names: con.execute( "INSERT OR REPLACE INTO entity_mention(entity,kind,source," "occurred_at) VALUES(?,?,?,?)", (name, kind, source, _timestamp(text, p))) entities = set() for key in FM_KEYS: entities |= _list_field(text, key) entities |= {x.strip() for x in LINK.findall(text)} entities &= known occurred = _timestamp(text, p) con.execute("DELETE FROM relation WHERE source=?", (source,)) con.execute("DELETE FROM interaction WHERE source=?", (source,)) for entity in entities: con.execute( "INSERT OR REPLACE INTO interaction(entity,occurred_at,kind,source," "weight) VALUES(?,?,?,?,?)", (entity, occurred, "meeting", source, 1.0)) nrel = 0 ordered = sorted(entities) for i, a in enumerate(ordered): for b in ordered[i + 1:]: subject, object_ = _ordered(a, b, kinds) con.execute( "INSERT OR REPLACE INTO relation(subject,predicate,object," "confidence,source,first_seen,last_seen) VALUES(?,?,?,?,?,?,?)", (subject, "同会", object_, 0.35, source, occurred, occurred)) nrel += 1 # A predicate is promoted only when two known linked entities occur in the # same sentence and the relationship word is explicit. for sentence in SENTENCE.split(text): linked = [x.strip() for x in LINK.findall(sentence) if x.strip() in known] if len(linked) < 2: continue predicate = next((x for x in PREDICATES if x in sentence), "") if not predicate: continue for a, b in zip(linked, linked[1:]): if a == b: continue con.execute( "INSERT OR REPLACE INTO relation(subject,predicate,object," "confidence,source,first_seen,last_seen) VALUES(?,?,?,?,?,?,?)", (a, predicate, b, 0.75, source, occurred, occurred)) nrel += 1 con.execute( "INSERT OR REPLACE INTO source_document(source,signature,processed_at," "entity_count,relation_count) VALUES(?,?,?,?,?)", (source, sig, now, len(entities), nrel)) changed += 1 entities_n += len(entities) relations_n += nrel con.commit() con.close() return {"files": len(files), "changed": changed, "skipped": skipped, "removed": len(removed), "entities_seen": entities_n, "relations_written": relations_n} def context_weights(seeds: set[str], *, half_life_days: float = 45.0, limit: int = 100) -> dict[str, float]: """Score graph neighbours by relation strength, recency, and frequency.""" if not seeds: return {} con = lexicon.connect() marks = ",".join("?" for _ in seeds) rows = con.execute( f"SELECT subject,predicate,object,max(confidence) confidence," f"max(last_seen) last_seen,count(*) support FROM relation WHERE active=1 " f"AND (subject IN ({marks}) OR object IN ({marks})) " "GROUP BY subject,predicate,object", (*seeds, *seeds)).fetchall() now = time.time() out: dict[str, float] = {s: 12.0 for s in seeds} for r in rows: other = r["object"] if r["subject"] in seeds else r["subject"] age = max(0.0, (now - (r["last_seen"] or now)) / 86400) decay = math.exp(-math.log(2) * age / half_life_days) support = 1.0 + math.log1p(r["support"]) out[other] = out.get(other, 0.0) + r["confidence"] * decay * support # Direct contact frequency is useful even when a relation has only one edge. if out: names = list(out) marks2 = ",".join("?" for _ in names) interactions = con.execute( f"SELECT entity,count(*) n,max(occurred_at) latest FROM interaction " f"WHERE entity IN ({marks2}) GROUP BY entity", names).fetchall() for r in interactions: age = max(0.0, (now - r["latest"]) / 86400) decay = math.exp(-math.log(2) * age / half_life_days) out[r["entity"]] += decay * math.log1p(r["n"]) con.close() return dict(sorted(out.items(), key=lambda x: -x[1])[:limit]) def stats() -> dict: con = lexicon.connect() q = lambda sql: con.execute(sql).fetchone()[0] out = { "documents": q("SELECT count(*) FROM source_document"), "relations": q("SELECT count(*) FROM relation"), "interactions": q("SELECT count(*) FROM interaction"), "projects": q("SELECT count(*) FROM entity WHERE kind='项目'"), } con.close() return out