feat(asr-pipeline): merge PR #1 webauth/CSRF/locking hardening and web_password fallback
This commit is contained in:
@@ -44,14 +44,16 @@ from pathlib import Path
|
||||
|
||||
from .config import CFG
|
||||
|
||||
LAB = CFG.root
|
||||
BANK = CFG.termbank_dir
|
||||
CACHE = CFG.cache_dir / "flagger"
|
||||
VAULT = CFG.vault
|
||||
def _vault() -> Path:
|
||||
return CFG.vault
|
||||
|
||||
HS_API = CFG.hindsight_api
|
||||
HS_BANK = CFG.hindsight_bank
|
||||
HS_TIMEOUT = CFG.hindsight_timeout
|
||||
|
||||
def _bank() -> Path:
|
||||
return CFG.termbank_dir
|
||||
|
||||
|
||||
def _cache() -> Path:
|
||||
return CFG.cache_dir / "flagger"
|
||||
|
||||
HAN = re.compile(r"[\u4e00-\u9fff]+")
|
||||
HAN_FULL = re.compile(r"^[\u4e00-\u9fff]+$")
|
||||
@@ -264,16 +266,16 @@ def _dir_sig(paths) -> str:
|
||||
|
||||
def load_pool(use_termbank=True) -> dict[str, Cand]:
|
||||
"""Vault entities (cards + glossary) ∪ optional domain term bank."""
|
||||
CACHE.mkdir(parents=True, exist_ok=True)
|
||||
_cache().mkdir(parents=True, exist_ok=True)
|
||||
if CFG.use_lexicon:
|
||||
try:
|
||||
from . import lexicon
|
||||
lexicon.sync_from_vault()
|
||||
except Exception:
|
||||
pass
|
||||
sig = _dir_sig([VAULT / "People", VAULT / "Companies", VAULT / "minutes",
|
||||
sig = _dir_sig([_vault() / "People", _vault() / "Companies", _vault() / "minutes",
|
||||
CFG.lexicon_db])
|
||||
cpath = CACHE / f"pool6-{sig}-{int(use_termbank)}.json"
|
||||
cpath = _cache() / f"pool6-{sig}-{int(use_termbank)}.json"
|
||||
if cpath.exists():
|
||||
d = json.loads(cpath.read_text())
|
||||
return {k: Cand(**v) for k, v in d.items()}
|
||||
@@ -331,7 +333,7 @@ def load_pool(use_termbank=True) -> dict[str, Cand]:
|
||||
ents = {}
|
||||
if not ents: # lexicon disabled/broken
|
||||
for sub, kind in (("People", "人名"), ("Companies", "公司")):
|
||||
d = VAULT / sub
|
||||
d = _vault() / sub
|
||||
if not d.exists():
|
||||
continue
|
||||
for f in d.glob("*.md"):
|
||||
@@ -346,7 +348,7 @@ def load_pool(use_termbank=True) -> dict[str, Cand]:
|
||||
add(form, c.canonical, c.kind, "variant", rule, c.owner, c.role, anchor)
|
||||
|
||||
if use_termbank:
|
||||
tb = BANK / "terms.json"
|
||||
tb = _bank() / "terms.json"
|
||||
if tb.exists():
|
||||
for term, e in json.loads(tb.read_text()).items():
|
||||
# Only the on-domain, distinctive slices are worth phonetic search;
|
||||
@@ -361,7 +363,7 @@ def load_pool(use_termbank=True) -> dict[str, Cand]:
|
||||
|
||||
|
||||
def load_allowlist() -> set[str]:
|
||||
p = BANK / "allowlist.txt"
|
||||
p = _bank() / "allowlist.txt"
|
||||
return set(p.read_text().split()) if p.exists() else set()
|
||||
|
||||
|
||||
@@ -388,13 +390,13 @@ def _minutes_entities(text: str) -> set[str]:
|
||||
|
||||
|
||||
def _corpus() -> list[tuple[str, set[str], str]]:
|
||||
CACHE.mkdir(parents=True, exist_ok=True)
|
||||
sig = _dir_sig([VAULT / "minutes"])
|
||||
cpath = CACHE / f"corpus-{sig}.json"
|
||||
_cache().mkdir(parents=True, exist_ok=True)
|
||||
sig = _dir_sig([_vault() / "minutes"])
|
||||
cpath = _cache() / f"corpus-{sig}.json"
|
||||
if cpath.exists():
|
||||
return [(a, set(b), c) for a, b, c in json.loads(cpath.read_text())]
|
||||
out = []
|
||||
for f in sorted((VAULT / "minutes").glob("*.md")):
|
||||
for f in sorted((_vault() / "minutes").glob("*.md")):
|
||||
txt = f.read_text(errors="ignore")
|
||||
m = re.search(r"^date:\s*(\d{4}-\d{2}-\d{2})", txt, re.M)
|
||||
out.append((f.name, _minutes_entities(txt), m.group(1) if m else ""))
|
||||
@@ -414,11 +416,11 @@ def hindsight_recall(seed: list[str], extra_query: str = "") -> tuple[set[str],
|
||||
return set(), []
|
||||
query = (" ".join(sorted(seed)) + " " + extra_query).strip() or "会议 相关人物 公司"
|
||||
body = json.dumps({"query": query, "max_tokens": 1500}).encode()
|
||||
url = f"{HS_API}/v1/default/banks/{HS_BANK}/memories/recall"
|
||||
url = f"{CFG.hindsight_api}/v1/default/banks/{CFG.hindsight_bank}/memories/recall"
|
||||
try:
|
||||
req = urllib.request.Request(url, data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=HS_TIMEOUT) as r:
|
||||
with urllib.request.urlopen(req, timeout=CFG.hindsight_timeout) as r:
|
||||
d = json.loads(r.read())
|
||||
except Exception:
|
||||
return set(), []
|
||||
|
||||
Reference in New Issue
Block a user