48 lines
1.7 KiB
Python
48 lines
1.7 KiB
Python
#!/usr/bin/env python3
|
||
"""Compact recalled prose into small, entity-centred context cards."""
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
from collections import defaultdict
|
||
|
||
SPLIT = re.compile(r"(?<=[。！？；])|\s*\|\s*")
|
||
META = re.compile(r"^(When|Involving|来源|时间)\s*[:：]\s*", re.I)
|
||
|
||
|
||
def _key(text: str) -> str:
|
||
text = META.sub("", text)
|
||
return re.sub(r"[\s，。；、,:：|（）()“”\"'\-]+", "", text).lower()
|
||
|
||
|
||
def compact(names: list[str], texts: list[str], *, per_entity: int = 3,
|
||
total: int = 12) -> str:
|
||
"""Deduplicate Hindsight recall before it reaches any model prompt.
|
||
|
||
Repeated memories often restate the same relationship with progressively more
|
||
prose. Keep one normalized fact per subject and cap every entity so a single
|
||
well-known person cannot consume the whole context window.
|
||
"""
|
||
buckets: dict[str, list[str]] = defaultdict(list)
|
||
seen: set[str] = set()
|
||
known = sorted(set(names), key=len, reverse=True)
|
||
for raw in texts:
|
||
for fact in SPLIT.split(re.sub(r"\s+", " ", raw).strip()):
|
||
fact = fact.strip(" -|")
|
||
if len(fact) < 8:
|
||
continue
|
||
k = _key(fact)
|
||
if not k or any(k == old or k in old or old in k for old in seen):
|
||
continue
|
||
seen.add(k)
|
||
owner = next((n for n in known if n in fact), "相关背景")
|
||
if len(buckets[owner]) < per_entity:
|
||
buckets[owner].append(fact[:220])
|
||
|
||
lines = []
|
||
for owner, facts in buckets.items():
|
||
lines.append(f"{owner}：")
|
||
lines.extend(f"- {f}" for f in facts)
|
||
if sum(1 for x in lines if x.startswith("- ")) >= total:
|
||
break
|
||
return "\n".join(lines)
|