deploy standalone ASR pipeline on tools-pve
This commit is contained in:
Codex
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Incremental entity/relation/recency index over curated Obsidian minutes."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
import re
|
||||
import time
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from . import lexicon
|
||||
from .config import CFG
|
||||
|
||||
FM_KEYS = ("people", "companies", "projects", "project")
|
||||
LINK = re.compile(r"\[\[([^\]|#]+)")
|
||||
DATE = re.compile(r"^date:[ \t]*(\d{4}-\d{2}-\d{2})", re.M)
|
||||
SENTENCE = re.compile(r"(?<=[。！？；\n])")
|
||||
PREDICATES = (
|
||||
"投资", "任职", "负责", "参与", "合作", "对接", "收购", "并购",
|
||||
"创立", "加入", "管理", "支持", "服务", "委托", "出资", "控股",
|
||||
)
|
||||
|
||||
|
||||
def _list_field(text: str, key: str) -> set[str]:
|
||||
m = re.search(rf"^{key}:[ \t]*(.*)$", text, re.M)
|
||||
if not m:
|
||||
return set()
|
||||
return {x.strip() for x in re.findall(r"[^,\[\]\s，、]+", m.group(1))
|
||||
if x.strip()}
|
||||
|
||||
|
||||
def _timestamp(text: str, path: Path) -> float:
|
||||
m = DATE.search(text)
|
||||
if m:
|
||||
try:
|
||||
return datetime.combine(date.fromisoformat(m.group(1)),
|
||||
datetime.min.time()).timestamp()
|
||||
except ValueError:
|
||||
pass
|
||||
m = re.search(r"(\d{4}-\d{2}-\d{2})", path.name)
|
||||
if m:
|
||||
try:
|
||||
return datetime.combine(date.fromisoformat(m.group(1)),
|
||||
datetime.min.time()).timestamp()
|
||||
except ValueError:
|
||||
pass
|
||||
return path.stat().st_mtime
|
||||
|
||||
|
||||
def _signature(path: Path) -> str:
|
||||
st = path.stat()
|
||||
return hashlib.sha256(
|
||||
f"{path}:{st.st_mtime_ns}:{st.st_size}".encode()).hexdigest()[:20]
|
||||
|
||||
|
||||
def _minutes_files() -> list[Path]:
|
||||
root = CFG.vault / "minutes"
|
||||
if not root.exists():
|
||||
return []
|
||||
return sorted(p for p in root.rglob("*.md")
|
||||
if "_drafts" not in p.parts and not p.name.startswith("_"))
|
||||
|
||||
|
||||
def _ordered(a: str, b: str, kinds: dict[str, str]) -> tuple[str, str]:
|
||||
rank = {"人名": 0, "公司": 1, "项目": 2}
|
||||
ka, kb = rank.get(kinds.get(a, ""), 9), rank.get(kinds.get(b, ""), 9)
|
||||
return (a, b) if (ka, a) <= (kb, b) else (b, a)
|
||||
|
||||
|
||||
def sync(force: bool = False) -> dict:
|
||||
"""Index changed minutes and remove facts belonging to deleted notes."""
|
||||
lexicon.sync_from_vault()
|
||||
con = lexicon.connect()
|
||||
kinds = {r["canonical"]: r["kind"]
|
||||
for r in con.execute("SELECT canonical,kind FROM entity")}
|
||||
known = set(kinds)
|
||||
files = _minutes_files()
|
||||
active_sources = {f"minute:{p.relative_to(CFG.vault)}" for p in files}
|
||||
|
||||
old_sources = {r["source"] for r in con.execute(
|
||||
"SELECT source FROM source_document WHERE source LIKE 'minute:%'")}
|
||||
removed = old_sources - active_sources
|
||||
for source in removed:
|
||||
con.execute("DELETE FROM relation WHERE source=?", (source,))
|
||||
con.execute("DELETE FROM interaction WHERE source=?", (source,))
|
||||
con.execute("DELETE FROM entity_mention WHERE source=?", (source,))
|
||||
con.execute("DELETE FROM source_document WHERE source=?", (source,))
|
||||
|
||||
changed = skipped = entities_n = relations_n = 0
|
||||
now = time.time()
|
||||
for p in files:
|
||||
source = f"minute:{p.relative_to(CFG.vault)}"
|
||||
sig = _signature(p)
|
||||
old = con.execute("SELECT signature FROM source_document WHERE source=?",
|
||||
(source,)).fetchone()
|
||||
if not force and old and old["signature"] == sig:
|
||||
skipped += 1
|
||||
continue
|
||||
text = p.read_text(errors="ignore")
|
||||
typed = {
|
||||
"人名": _list_field(text, "people"),
|
||||
"公司": _list_field(text, "companies"),
|
||||
}
|
||||
con.execute("DELETE FROM entity_mention WHERE source=?", (source,))
|
||||
for kind, names in typed.items():
|
||||
for name in names:
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO entity_mention(entity,kind,source,"
|
||||
"occurred_at) VALUES(?,?,?,?)",
|
||||
(name, kind, source, _timestamp(text, p)))
|
||||
entities = set()
|
||||
for key in FM_KEYS:
|
||||
entities |= _list_field(text, key)
|
||||
entities |= {x.strip() for x in LINK.findall(text)}
|
||||
entities &= known
|
||||
occurred = _timestamp(text, p)
|
||||
|
||||
con.execute("DELETE FROM relation WHERE source=?", (source,))
|
||||
con.execute("DELETE FROM interaction WHERE source=?", (source,))
|
||||
for entity in entities:
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO interaction(entity,occurred_at,kind,source,"
|
||||
"weight) VALUES(?,?,?,?,?)",
|
||||
(entity, occurred, "meeting", source, 1.0))
|
||||
|
||||
nrel = 0
|
||||
ordered = sorted(entities)
|
||||
for i, a in enumerate(ordered):
|
||||
for b in ordered[i + 1:]:
|
||||
subject, object_ = _ordered(a, b, kinds)
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO relation(subject,predicate,object,"
|
||||
"confidence,source,first_seen,last_seen) VALUES(?,?,?,?,?,?,?)",
|
||||
(subject, "同会", object_, 0.35, source, occurred, occurred))
|
||||
nrel += 1
|
||||
|
||||
# A predicate is promoted only when two known linked entities occur in the
|
||||
# same sentence and the relationship word is explicit.
|
||||
for sentence in SENTENCE.split(text):
|
||||
linked = [x.strip() for x in LINK.findall(sentence) if x.strip() in known]
|
||||
if len(linked) < 2:
|
||||
continue
|
||||
predicate = next((x for x in PREDICATES if x in sentence), "")
|
||||
if not predicate:
|
||||
continue
|
||||
for a, b in zip(linked, linked[1:]):
|
||||
if a == b:
|
||||
continue
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO relation(subject,predicate,object,"
|
||||
"confidence,source,first_seen,last_seen) VALUES(?,?,?,?,?,?,?)",
|
||||
(a, predicate, b, 0.75, source, occurred, occurred))
|
||||
nrel += 1
|
||||
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO source_document(source,signature,processed_at,"
|
||||
"entity_count,relation_count) VALUES(?,?,?,?,?)",
|
||||
(source, sig, now, len(entities), nrel))
|
||||
changed += 1
|
||||
entities_n += len(entities)
|
||||
relations_n += nrel
|
||||
con.commit()
|
||||
con.close()
|
||||
return {"files": len(files), "changed": changed, "skipped": skipped,
|
||||
"removed": len(removed), "entities_seen": entities_n,
|
||||
"relations_written": relations_n}
|
||||
|
||||
|
||||
def context_weights(seeds: set[str], *, half_life_days: float = 45.0,
|
||||
limit: int = 100) -> dict[str, float]:
|
||||
"""Score graph neighbours by relation strength, recency, and frequency."""
|
||||
if not seeds:
|
||||
return {}
|
||||
con = lexicon.connect()
|
||||
marks = ",".join("?" for _ in seeds)
|
||||
rows = con.execute(
|
||||
f"SELECT subject,predicate,object,max(confidence) confidence,"
|
||||
f"max(last_seen) last_seen,count(*) support FROM relation WHERE active=1 "
|
||||
f"AND (subject IN ({marks}) OR object IN ({marks})) "
|
||||
"GROUP BY subject,predicate,object",
|
||||
(*seeds, *seeds)).fetchall()
|
||||
now = time.time()
|
||||
out: dict[str, float] = {s: 12.0 for s in seeds}
|
||||
for r in rows:
|
||||
other = r["object"] if r["subject"] in seeds else r["subject"]
|
||||
age = max(0.0, (now - (r["last_seen"] or now)) / 86400)
|
||||
decay = math.exp(-math.log(2) * age / half_life_days)
|
||||
support = 1.0 + math.log1p(r["support"])
|
||||
out[other] = out.get(other, 0.0) + r["confidence"] * decay * support
|
||||
|
||||
# Direct contact frequency is useful even when a relation has only one edge.
|
||||
if out:
|
||||
names = list(out)
|
||||
marks2 = ",".join("?" for _ in names)
|
||||
interactions = con.execute(
|
||||
f"SELECT entity,count(*) n,max(occurred_at) latest FROM interaction "
|
||||
f"WHERE entity IN ({marks2}) GROUP BY entity", names).fetchall()
|
||||
for r in interactions:
|
||||
age = max(0.0, (now - r["latest"]) / 86400)
|
||||
decay = math.exp(-math.log(2) * age / half_life_days)
|
||||
out[r["entity"]] += decay * math.log1p(r["n"])
|
||||
con.close()
|
||||
return dict(sorted(out.items(), key=lambda x: -x[1])[:limit])
|
||||
|
||||
|
||||
def stats() -> dict:
|
||||
con = lexicon.connect()
|
||||
q = lambda sql: con.execute(sql).fetchone()[0]
|
||||
out = {
|
||||
"documents": q("SELECT count(*) FROM source_document"),
|
||||
"relations": q("SELECT count(*) FROM relation"),
|
||||
"interactions": q("SELECT count(*) FROM interaction"),
|
||||
"projects": q("SELECT count(*) FROM entity WHERE kind='项目'"),
|
||||
}
|
||||
con.close()
|
||||
return out
|
||||
Reference in New Issue
Block a user