feat(asr-pipeline): merge PR #1 webauth/CSRF/locking hardening and web_password fallback
This commit is contained in:
@@ -153,7 +153,7 @@ def bootstrap_default_settings() -> None:
|
||||
"hindsight_bank": _env("HINDSIGHT_BANK", "personal-context"),
|
||||
"use_hindsight": "1" if _env("ASR_HINDSIGHT", "1") != "0" else "0",
|
||||
"use_termbank": "1" if _env("ASR_TERMBANK", "1") != "0" else "0",
|
||||
"git_sync": "1" if _env("ASR_GIT_SYNC", "0") != "0" else "0",
|
||||
"git_sync": "0" if _env("ASR_GIT_SYNC", "0") == "0" else "1",
|
||||
}
|
||||
con = sqlite3.connect(db_path, timeout=15)
|
||||
try:
|
||||
@@ -320,7 +320,7 @@ class Config:
|
||||
)
|
||||
git_sync = _SettingAttr(
|
||||
"git_sync",
|
||||
lambda: _env("ASR_GIT_SYNC", "1"),
|
||||
lambda: _env("ASR_GIT_SYNC", "0"),
|
||||
lambda s: str(s).lower() not in ("0", "false", "no", "off"),
|
||||
)
|
||||
|
||||
@@ -399,7 +399,7 @@ class Config:
|
||||
|
||||
# ---- behaviour ------------------------------------------------------
|
||||
notify: bool = field(
|
||||
default_factory=lambda: _env("ASR_NOTIFY", "1") != "0")
|
||||
default_factory=lambda: _env("ASR_NOTIFY", "0") != "0")
|
||||
cache_llm: bool = field(
|
||||
default_factory=lambda: _env("ASR_CACHE", "0") != "0")
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
"""Persistent mobile control plane for recording intake and human review."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
@@ -14,7 +16,7 @@ from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
from . import review
|
||||
from . import review, webauth
|
||||
from .config import CFG
|
||||
from .log import log
|
||||
|
||||
@@ -166,6 +168,7 @@ def _shell(title: str, body: str, script: str = "") -> bytes:
|
||||
return f"""<!doctype html>
|
||||
<html lang="zh-CN"><head>
|
||||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta name="csrf" content="{html.escape(webauth.csrf_token(), quote=True)}">
|
||||
<meta name="color-scheme" content="light"><meta http-equiv="Cache-Control" content="no-store">
|
||||
<title>{html.escape(title)}</title>
|
||||
<style>
|
||||
@@ -236,6 +239,7 @@ document.querySelector('#upload').addEventListener('submit',e=>{
|
||||
const x=new XMLHttpRequest(), b=document.querySelector('#send'), p=document.querySelector('#progress'), out=document.querySelector('#result');
|
||||
b.disabled=true;p.hidden=false;out.textContent='正在上传…';
|
||||
x.open('POST','/api/upload');x.setRequestHeader('Content-Type','application/octet-stream');
|
||||
x.setRequestHeader('X-CSRF-Token', document.querySelector('meta[name=csrf]')?.content || '');
|
||||
x.setRequestHeader('X-Filename',encodeURIComponent(f.name));
|
||||
x.setRequestHeader('X-Title',encodeURIComponent(document.querySelector('#title').value));
|
||||
x.setRequestHeader('X-Context',encodeURIComponent(document.querySelector('#context').value));
|
||||
@@ -293,6 +297,7 @@ def _review_page(action: str, job: dict | None = None) -> bytes:
|
||||
body = f"""<section class="panel"><div class="meta">{html.escape(title)} · 剩余 {len(rows)} 个 · 问题 #{row['id']}</div>
|
||||
<h1>{html.escape(row['question'])}</h1>{source_html}{draft_html}
|
||||
<form method="post" action="{html.escape(action, quote=True)}">
|
||||
<input type="hidden" name="csrf" value="{html.escape(webauth.csrf_token(), quote=True)}">
|
||||
<input type="hidden" name="id" value="{row['id']}">{buttons}
|
||||
<div class="custom"><input name="custom" maxlength="120" autocomplete="off"
|
||||
placeholder="{html.escape(placeholder, quote=True)}"><button type="submit">提交</button></div>
|
||||
@@ -342,12 +347,43 @@ class Handler(BaseHTTPRequestHandler):
|
||||
return True, _job(match.group(1))
|
||||
return False, None
|
||||
|
||||
def _browser_ok(self) -> bool:
|
||||
if webauth.allow_anon():
|
||||
return True
|
||||
if not webauth.password():
|
||||
return False
|
||||
return webauth.cookie_matches(self.headers.get("Cookie", ""))
|
||||
|
||||
def _login_page(self, err: str = "") -> bytes:
|
||||
if not webauth.password() and not webauth.allow_anon():
|
||||
err = err or "未配置 ASR_WEB_PASSWORD,控制面已关闭。"
|
||||
token = html.escape(webauth.csrf_token(), quote=True)
|
||||
alert = f"<p>{html.escape(err)}</p>" if err else ""
|
||||
body = (
|
||||
f'<section class="panel"><h1>访问验证</h1>{alert}'
|
||||
f'<form method="post" action="/login">'
|
||||
f'<input type="hidden" name="csrf" value="{token}">'
|
||||
f'<label>访问口令</label><input name="password" type="password" required>'
|
||||
f'<div class="actions"><button type="submit">登录</button></div></form></section>'
|
||||
)
|
||||
return _shell("登录", body)
|
||||
|
||||
def do_GET(self):
|
||||
path = urllib.parse.urlsplit(self.path).path
|
||||
if path == "/":
|
||||
return self._send(_home())
|
||||
if path == "/health":
|
||||
return self._json({"ok": True, "pending": len(review.pending(100))})
|
||||
if path in {"/login", "/logout"}:
|
||||
if path == "/logout":
|
||||
self.send_response(HTTPStatus.SEE_OTHER)
|
||||
self.send_header("Location", "/login")
|
||||
self.send_header("Set-Cookie", "asr_session=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0")
|
||||
self.end_headers()
|
||||
return
|
||||
return self._send(self._login_page())
|
||||
if not self._browser_ok():
|
||||
return self._send(self._login_page(), HTTPStatus.UNAUTHORIZED)
|
||||
if path == "/":
|
||||
return self._send(_home())
|
||||
if path == "/api/jobs":
|
||||
jobs = _jobs()
|
||||
for j in jobs:
|
||||
@@ -372,9 +408,9 @@ class Handler(BaseHTTPRequestHandler):
|
||||
|
||||
def _upload(self):
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
length = int(self.headers.get("Content-Length", "0") or 0)
|
||||
except ValueError:
|
||||
length = 0
|
||||
return self._json({"error": "bad content-length"}, HTTPStatus.BAD_REQUEST)
|
||||
if length <= 0 or length > MAX_UPLOAD:
|
||||
return self._json({"error": "invalid upload size"},
|
||||
HTTPStatus.REQUEST_ENTITY_TOO_LARGE)
|
||||
@@ -408,7 +444,7 @@ class Handler(BaseHTTPRequestHandler):
|
||||
while remaining:
|
||||
data = self.rfile.read(min(CHUNK, remaining))
|
||||
if not data:
|
||||
raise EOFError("upload ended early")
|
||||
raise EOFError("incomplete upload")
|
||||
f.write(data)
|
||||
remaining -= len(data)
|
||||
f.flush()
|
||||
@@ -426,16 +462,28 @@ class Handler(BaseHTTPRequestHandler):
|
||||
update_job(job_id, "queued", "录音已落盘,等待 watcher 接收")
|
||||
return self._json({"ok": True, "job_id": job_id,
|
||||
"stored_name": stored}, HTTPStatus.CREATED)
|
||||
except EOFError:
|
||||
part.unlink(missing_ok=True)
|
||||
update_job(job_id, "failed", "上传未完成")
|
||||
return self._json({"error": "incomplete upload"}, HTTPStatus.BAD_REQUEST)
|
||||
except Exception as e:
|
||||
part.unlink(missing_ok=True)
|
||||
update_job(job_id, "failed", f"上传失败:{e}")
|
||||
return self._json({"error": str(e)}, HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
update_job(job_id, "failed", "上传失败")
|
||||
log(f"upload failed: {e}")
|
||||
return self._json({"error": "upload failed"}, HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
|
||||
def _answer(self):
|
||||
try:
|
||||
length = min(int(self.headers.get("Content-Length", "0")), 4096)
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0") or 0)
|
||||
except ValueError as exc:
|
||||
raise ValueError("bad content-length") from exc
|
||||
if length < 0 or length > 4096:
|
||||
raise ValueError("bad content-length")
|
||||
form = urllib.parse.parse_qs(
|
||||
self.rfile.read(length).decode("utf-8"))
|
||||
if not webauth.csrf_matches((form.get("csrf") or [""])[0]):
|
||||
raise ValueError("csrf")
|
||||
qid = int(form.get("id", ["0"])[0])
|
||||
answer = (form.get("custom", [""])[0].strip()
|
||||
or form.get("answer", [""])[0].strip())
|
||||
@@ -460,7 +508,31 @@ class Handler(BaseHTTPRequestHandler):
|
||||
|
||||
def do_POST(self):
|
||||
path = urllib.parse.urlsplit(self.path).path
|
||||
if path == "/login":
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0") or 0)
|
||||
except ValueError:
|
||||
return self._json({"error": "bad content-length"}, HTTPStatus.BAD_REQUEST)
|
||||
if length < 0:
|
||||
return self._json({"error": "bad content-length"}, HTTPStatus.BAD_REQUEST)
|
||||
length = min(length, 4096)
|
||||
form = urllib.parse.parse_qs(self.rfile.read(length).decode("utf-8", "ignore"))
|
||||
pw = form.get("password", [""])[0]
|
||||
if webauth.password() and hmac.compare_digest(pw, webauth.password()):
|
||||
self.send_response(HTTPStatus.SEE_OTHER)
|
||||
self.send_header("Location", "/")
|
||||
self.send_header(
|
||||
"Set-Cookie",
|
||||
f"asr_session={webauth.session_token()}; Path=/; HttpOnly; SameSite=Strict; Max-Age=86400",
|
||||
)
|
||||
self.end_headers()
|
||||
return
|
||||
return self._send(self._login_page("口令错误或未配置 ASR_WEB_PASSWORD"), HTTPStatus.UNAUTHORIZED)
|
||||
if not self._browser_ok():
|
||||
return self._json({"error": "unauthorized"}, HTTPStatus.UNAUTHORIZED)
|
||||
if path in ("/api/upload", "/hook/intake"):
|
||||
if not webauth.csrf_matches(self.headers.get("X-CSRF-Token", "")):
|
||||
return self._json({"error": "csrf"}, HTTPStatus.FORBIDDEN)
|
||||
return self._upload()
|
||||
if self._review_job()[0]:
|
||||
return self._answer()
|
||||
|
||||
@@ -44,14 +44,16 @@ from pathlib import Path
|
||||
|
||||
from .config import CFG
|
||||
|
||||
LAB = CFG.root
|
||||
BANK = CFG.termbank_dir
|
||||
CACHE = CFG.cache_dir / "flagger"
|
||||
VAULT = CFG.vault
|
||||
def _vault() -> Path:
|
||||
return CFG.vault
|
||||
|
||||
HS_API = CFG.hindsight_api
|
||||
HS_BANK = CFG.hindsight_bank
|
||||
HS_TIMEOUT = CFG.hindsight_timeout
|
||||
|
||||
def _bank() -> Path:
|
||||
return CFG.termbank_dir
|
||||
|
||||
|
||||
def _cache() -> Path:
|
||||
return CFG.cache_dir / "flagger"
|
||||
|
||||
HAN = re.compile(r"[\u4e00-\u9fff]+")
|
||||
HAN_FULL = re.compile(r"^[\u4e00-\u9fff]+$")
|
||||
@@ -264,16 +266,16 @@ def _dir_sig(paths) -> str:
|
||||
|
||||
def load_pool(use_termbank=True) -> dict[str, Cand]:
|
||||
"""Vault entities (cards + glossary) ∪ optional domain term bank."""
|
||||
CACHE.mkdir(parents=True, exist_ok=True)
|
||||
_cache().mkdir(parents=True, exist_ok=True)
|
||||
if CFG.use_lexicon:
|
||||
try:
|
||||
from . import lexicon
|
||||
lexicon.sync_from_vault()
|
||||
except Exception:
|
||||
pass
|
||||
sig = _dir_sig([VAULT / "People", VAULT / "Companies", VAULT / "minutes",
|
||||
sig = _dir_sig([_vault() / "People", _vault() / "Companies", _vault() / "minutes",
|
||||
CFG.lexicon_db])
|
||||
cpath = CACHE / f"pool6-{sig}-{int(use_termbank)}.json"
|
||||
cpath = _cache() / f"pool6-{sig}-{int(use_termbank)}.json"
|
||||
if cpath.exists():
|
||||
d = json.loads(cpath.read_text())
|
||||
return {k: Cand(**v) for k, v in d.items()}
|
||||
@@ -331,7 +333,7 @@ def load_pool(use_termbank=True) -> dict[str, Cand]:
|
||||
ents = {}
|
||||
if not ents: # lexicon disabled/broken
|
||||
for sub, kind in (("People", "人名"), ("Companies", "公司")):
|
||||
d = VAULT / sub
|
||||
d = _vault() / sub
|
||||
if not d.exists():
|
||||
continue
|
||||
for f in d.glob("*.md"):
|
||||
@@ -346,7 +348,7 @@ def load_pool(use_termbank=True) -> dict[str, Cand]:
|
||||
add(form, c.canonical, c.kind, "variant", rule, c.owner, c.role, anchor)
|
||||
|
||||
if use_termbank:
|
||||
tb = BANK / "terms.json"
|
||||
tb = _bank() / "terms.json"
|
||||
if tb.exists():
|
||||
for term, e in json.loads(tb.read_text()).items():
|
||||
# Only the on-domain, distinctive slices are worth phonetic search;
|
||||
@@ -361,7 +363,7 @@ def load_pool(use_termbank=True) -> dict[str, Cand]:
|
||||
|
||||
|
||||
def load_allowlist() -> set[str]:
|
||||
p = BANK / "allowlist.txt"
|
||||
p = _bank() / "allowlist.txt"
|
||||
return set(p.read_text().split()) if p.exists() else set()
|
||||
|
||||
|
||||
@@ -388,13 +390,13 @@ def _minutes_entities(text: str) -> set[str]:
|
||||
|
||||
|
||||
def _corpus() -> list[tuple[str, set[str], str]]:
|
||||
CACHE.mkdir(parents=True, exist_ok=True)
|
||||
sig = _dir_sig([VAULT / "minutes"])
|
||||
cpath = CACHE / f"corpus-{sig}.json"
|
||||
_cache().mkdir(parents=True, exist_ok=True)
|
||||
sig = _dir_sig([_vault() / "minutes"])
|
||||
cpath = _cache() / f"corpus-{sig}.json"
|
||||
if cpath.exists():
|
||||
return [(a, set(b), c) for a, b, c in json.loads(cpath.read_text())]
|
||||
out = []
|
||||
for f in sorted((VAULT / "minutes").glob("*.md")):
|
||||
for f in sorted((_vault() / "minutes").glob("*.md")):
|
||||
txt = f.read_text(errors="ignore")
|
||||
m = re.search(r"^date:\s*(\d{4}-\d{2}-\d{2})", txt, re.M)
|
||||
out.append((f.name, _minutes_entities(txt), m.group(1) if m else ""))
|
||||
@@ -414,11 +416,11 @@ def hindsight_recall(seed: list[str], extra_query: str = "") -> tuple[set[str],
|
||||
return set(), []
|
||||
query = (" ".join(sorted(seed)) + " " + extra_query).strip() or "会议 相关人物 公司"
|
||||
body = json.dumps({"query": query, "max_tokens": 1500}).encode()
|
||||
url = f"{HS_API}/v1/default/banks/{HS_BANK}/memories/recall"
|
||||
url = f"{CFG.hindsight_api}/v1/default/banks/{CFG.hindsight_bank}/memories/recall"
|
||||
try:
|
||||
req = urllib.request.Request(url, data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=HS_TIMEOUT) as r:
|
||||
with urllib.request.urlopen(req, timeout=CFG.hindsight_timeout) as r:
|
||||
d = json.loads(r.read())
|
||||
except Exception:
|
||||
return set(), []
|
||||
|
||||
@@ -157,7 +157,8 @@ class Entity:
|
||||
|
||||
def connect() -> sqlite3.Connection:
|
||||
CFG.state_dir.mkdir(parents=True, exist_ok=True)
|
||||
con = sqlite3.connect(CFG.lexicon_db)
|
||||
con = sqlite3.connect(CFG.lexicon_db, timeout=30)
|
||||
con.execute("PRAGMA journal_mode=WAL")
|
||||
con.row_factory = sqlite3.Row
|
||||
con.executescript(SCHEMA)
|
||||
# One-time compatibility migration for the 158 corrections learned before the
|
||||
@@ -527,7 +528,7 @@ def stats() -> dict:
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ Web UI CRUD
|
||||
def list_aliases(q: str = "", origin_filter: str = "", limit: int = 200) -> list[dict]:
|
||||
def list_aliases(q: str = "", origin_filter: str = "", limit: int = 200, offset: int = 0) -> list[dict]:
|
||||
con = connect()
|
||||
sql = "SELECT * FROM alias WHERE 1=1"
|
||||
args: list = []
|
||||
@@ -539,13 +540,30 @@ def list_aliases(q: str = "", origin_filter: str = "", limit: int = 200) -> list
|
||||
sql += " AND origin <> 'card'"
|
||||
elif origin_filter == "card":
|
||||
sql += " AND origin = 'card'"
|
||||
sql += " ORDER BY confirmed DESC, created_at DESC, alias ASC LIMIT ?"
|
||||
args.append(limit)
|
||||
sql += " ORDER BY confirmed DESC, created_at DESC, alias ASC LIMIT ? OFFSET ?"
|
||||
args.extend((max(1, min(limit, 500)), max(0, offset)))
|
||||
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
|
||||
con.close()
|
||||
return rows
|
||||
|
||||
|
||||
def count_aliases(q: str = "", origin_filter: str = "") -> int:
|
||||
con = connect()
|
||||
sql = "SELECT count(*) FROM alias WHERE 1=1"
|
||||
args: list = []
|
||||
if q.strip():
|
||||
pat = f"%{q.strip()}%"
|
||||
sql += " AND (alias LIKE ? OR canonical LIKE ? OR note LIKE ?)"
|
||||
args.extend([pat, pat, pat])
|
||||
if origin_filter == "learned":
|
||||
sql += " AND origin <> 'card'"
|
||||
elif origin_filter == "card":
|
||||
sql += " AND origin = 'card'"
|
||||
n = int(con.execute(sql, args).fetchone()[0])
|
||||
con.close()
|
||||
return n
|
||||
|
||||
|
||||
def update_alias(
|
||||
old_alias: str,
|
||||
old_canonical: str,
|
||||
@@ -590,7 +608,7 @@ def delete_alias(alias: str, canonical: str) -> bool:
|
||||
return deleted
|
||||
|
||||
|
||||
def list_corrections(q: str = "", limit: int = 200) -> list[dict]:
|
||||
def list_corrections(q: str = "", limit: int = 200, offset: int = 0) -> list[dict]:
|
||||
con = connect()
|
||||
sql = "SELECT * FROM correction WHERE 1=1"
|
||||
args: list = []
|
||||
@@ -598,13 +616,26 @@ def list_corrections(q: str = "", limit: int = 200) -> list[dict]:
|
||||
pat = f"%{q.strip()}%"
|
||||
sql += " AND (surface LIKE ? OR canonical LIKE ? OR note LIKE ?)"
|
||||
args.extend([pat, pat, pat])
|
||||
sql += " ORDER BY confirmed DESC, n DESC, last_seen DESC LIMIT ?"
|
||||
args.append(limit)
|
||||
sql += " ORDER BY confirmed DESC, n DESC, last_seen DESC LIMIT ? OFFSET ?"
|
||||
args.extend((max(1, min(limit, 500)), max(0, offset)))
|
||||
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
|
||||
con.close()
|
||||
return rows
|
||||
|
||||
|
||||
def count_corrections(q: str = "") -> int:
|
||||
con = connect()
|
||||
sql = "SELECT count(*) FROM correction WHERE 1=1"
|
||||
args: list = []
|
||||
if q.strip():
|
||||
pat = f"%{q.strip()}%"
|
||||
sql += " AND (surface LIKE ? OR canonical LIKE ? OR note LIKE ?)"
|
||||
args.extend([pat, pat, pat])
|
||||
n = int(con.execute(sql, args).fetchone()[0])
|
||||
con.close()
|
||||
return n
|
||||
|
||||
|
||||
def update_correction(
|
||||
old_surface: str,
|
||||
old_canonical: str,
|
||||
@@ -651,7 +682,7 @@ def delete_correction(surface: str, canonical: str) -> bool:
|
||||
return deleted
|
||||
|
||||
|
||||
def list_entities(q: str = "", kind: str = "", limit: int = 200) -> list[dict]:
|
||||
def list_entities(q: str = "", kind: str = "", limit: int = 200, offset: int = 0) -> list[dict]:
|
||||
con = connect()
|
||||
sql = "SELECT * FROM entity WHERE 1=1"
|
||||
args: list = []
|
||||
@@ -662,13 +693,29 @@ def list_entities(q: str = "", kind: str = "", limit: int = 200) -> list[dict]:
|
||||
if kind.strip():
|
||||
sql += " AND kind=?"
|
||||
args.append(kind.strip())
|
||||
sql += " ORDER BY (CASE WHEN card='' THEN 0 ELSE 1 END), canonical ASC LIMIT ?"
|
||||
args.append(limit)
|
||||
sql += " ORDER BY (CASE WHEN card='' THEN 0 ELSE 1 END), canonical ASC LIMIT ? OFFSET ?"
|
||||
args.extend((max(1, min(limit, 500)), max(0, offset)))
|
||||
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
|
||||
con.close()
|
||||
return rows
|
||||
|
||||
|
||||
def count_entities(q: str = "", kind: str = "") -> int:
|
||||
con = connect()
|
||||
sql = "SELECT count(*) FROM entity WHERE 1=1"
|
||||
args: list = []
|
||||
if q.strip():
|
||||
pat = f"%{q.strip()}%"
|
||||
sql += " AND (canonical LIKE ? OR company LIKE ? OR role LIKE ? OR related LIKE ?)"
|
||||
args.extend([pat, pat, pat, pat])
|
||||
if kind.strip():
|
||||
sql += " AND kind=?"
|
||||
args.append(kind.strip())
|
||||
n = int(con.execute(sql, args).fetchone()[0])
|
||||
con.close()
|
||||
return n
|
||||
|
||||
|
||||
def upsert_entity(
|
||||
canonical: str,
|
||||
kind: str = "人名",
|
||||
@@ -723,7 +770,7 @@ def delete_entity(canonical: str) -> bool:
|
||||
return deleted
|
||||
|
||||
|
||||
def list_relations(q: str = "", limit: int = 200) -> list[dict]:
|
||||
def list_relations(q: str = "", limit: int = 200, offset: int = 0) -> list[dict]:
|
||||
con = connect()
|
||||
sql = "SELECT * FROM relation WHERE 1=1"
|
||||
args: list = []
|
||||
@@ -731,13 +778,26 @@ def list_relations(q: str = "", limit: int = 200) -> list[dict]:
|
||||
pat = f"%{q.strip()}%"
|
||||
sql += " AND (subject LIKE ? OR predicate LIKE ? OR object LIKE ? OR source LIKE ?)"
|
||||
args.extend([pat, pat, pat, pat])
|
||||
sql += " ORDER BY active DESC, confidence DESC, last_seen DESC LIMIT ?"
|
||||
args.append(limit)
|
||||
sql += " ORDER BY active DESC, confidence DESC, last_seen DESC LIMIT ? OFFSET ?"
|
||||
args.extend((max(1, min(limit, 500)), max(0, offset)))
|
||||
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
|
||||
con.close()
|
||||
return rows
|
||||
|
||||
|
||||
def count_relations(q: str = "") -> int:
|
||||
con = connect()
|
||||
sql = "SELECT count(*) FROM relation WHERE 1=1"
|
||||
args: list = []
|
||||
if q.strip():
|
||||
pat = f"%{q.strip()}%"
|
||||
sql += " AND (subject LIKE ? OR predicate LIKE ? OR object LIKE ? OR source LIKE ?)"
|
||||
args.extend([pat, pat, pat, pat])
|
||||
n = int(con.execute(sql, args).fetchone()[0])
|
||||
con.close()
|
||||
return n
|
||||
|
||||
|
||||
def update_relation(
|
||||
old_subject: str,
|
||||
old_predicate: str,
|
||||
|
||||
@@ -49,10 +49,8 @@ _key: list[str] = []
|
||||
|
||||
|
||||
def _auth() -> str:
|
||||
with _lock:
|
||||
if not _key:
|
||||
_key.append(CFG.api_key())
|
||||
return _key[0]
|
||||
# Read on every call so a key saved from /settings is used without a restart.
|
||||
return CFG.api_key()
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
"""One machine-wide lock for processes that consume the recording inbox."""
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def acquire_recording_lock(path: Path):
|
||||
"""Return an open file holding an exclusive lock, or exit if another consumer has it.
|
||||
|
||||
The handle must stay referenced for the life of the process.
|
||||
"""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = path.open("a+")
|
||||
try:
|
||||
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
handle.close()
|
||||
raise SystemExit(f"another recording consumer holds {path}")
|
||||
handle.seek(0)
|
||||
handle.truncate()
|
||||
handle.write(str(os.getpid()))
|
||||
handle.flush()
|
||||
return handle
|
||||
@@ -93,7 +93,7 @@ KEYWORD_PROMPT = """从下面的会议转写摘选中,提取用于检索背景
|
||||
# (「谭凯(白帆生物总经理)」、「此项目负责人为 [[冯昆仑]](背景确认)」) even when the
|
||||
# recording never said them. The facts were often correct, but a meeting note that
|
||||
# cannot distinguish "said on the call" from "known from notes" is unsafe to finalize.
|
||||
DRAFT_PROMPT = """你是会议录音整理助手,正在根据**带来源编号的事实卡片**产出「粗整理草稿」。
|
||||
DRAFT_PROMPT = """你是会议录音整理助手,正在根据下方整理要点写出会议草稿。
|
||||
|
||||
严格遵循:
|
||||
1. 只产出粗整理草稿,**不是**正式纪要。
|
||||
@@ -133,8 +133,11 @@ DRAFT_PROMPT = """你是会议录音整理助手,正在根据**带来源编号
|
||||
(例如「徐总(疑似 [[徐劲前]] 或 [[邹洵]])」),而不是笼统写「身份不明」。
|
||||
13. 判定简称指向谁时使用背景里的任职信息,这属于**辨识**而非越界断言,
|
||||
不需要加〔背景〕;但如果你要额外陈述这个人的职务、公司归属本身,那就要加〔背景〕。
|
||||
14. 对匿名说话人,除非事实卡片已经给出有证据的候选,否则只写“身份待确认”;
|
||||
14. 对匿名说话人,除非整理要点已经给出有证据的候选,否则只写“身份待确认”;
|
||||
不要从背景中枚举可能的人名让用户猜。
|
||||
15. 正文和标题不要出现「卡片」「提示词」「系统指令」「粗整理」这些内部说法。
|
||||
记录里没有的信息写「记录未写明」,不要写「卡片未提供」或「卡片提及」。
|
||||
标题用业务名称,不要带「草稿」二字。
|
||||
|
||||
{speaker_note}
|
||||
|
||||
@@ -153,7 +156,7 @@ companies: [有把握的公司…]
|
||||
transcript: {transcript}
|
||||
{extra_fm}---
|
||||
|
||||
事实卡片:
|
||||
整理要点:
|
||||
{body}
|
||||
|
||||
只输出 markdown 草稿全文,不要任何额外说明。"""
|
||||
|
||||
@@ -409,28 +409,45 @@ def probe_duration_seconds(audio: Path) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _encode_multipart(
|
||||
def _multipart_framing(
|
||||
fields: dict[str, str],
|
||||
file_field: str,
|
||||
file_path: Path,
|
||||
) -> tuple[bytes, str]:
|
||||
) -> tuple[bytes, bytes, str, int]:
|
||||
"""Build multipart head/tail without reading the audio into memory."""
|
||||
boundary = f"----ASRBoundary{uuid.uuid4().hex}"
|
||||
body = bytearray()
|
||||
safe_name = file_path.name.replace('"', "").replace("\r", "").replace("\n", "") or "audio"
|
||||
head = bytearray()
|
||||
for name, val in fields.items():
|
||||
body.extend(f"--{boundary}\r\n".encode())
|
||||
body.extend(
|
||||
f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode())
|
||||
body.extend(str(val).encode("utf-8"))
|
||||
body.extend(b"\r\n")
|
||||
mime = mimetypes.guess_type(file_path.name)[0] or "application/octet-stream"
|
||||
body.extend(f"--{boundary}\r\n".encode())
|
||||
body.extend(
|
||||
head.extend(f"--{boundary}\r\n".encode())
|
||||
head.extend(f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode())
|
||||
head.extend(str(val).encode("utf-8"))
|
||||
head.extend(b"\r\n")
|
||||
mime = mimetypes.guess_type(safe_name)[0] or "application/octet-stream"
|
||||
head.extend(f"--{boundary}\r\n".encode())
|
||||
head.extend(
|
||||
f'Content-Disposition: form-data; name="{file_field}"; '
|
||||
f'filename="{file_path.name}"\r\n'.encode())
|
||||
body.extend(f"Content-Type: {mime}\r\n\r\n".encode())
|
||||
body.extend(file_path.read_bytes())
|
||||
body.extend(f"\r\n--{boundary}--\r\n".encode())
|
||||
return bytes(body), f"multipart/form-data; boundary={boundary}"
|
||||
f'filename="{safe_name}"\r\n'.encode())
|
||||
head.extend(f"Content-Type: {mime}\r\n\r\n".encode())
|
||||
tail = f"\r\n--{boundary}--\r\n".encode()
|
||||
total = len(head) + file_path.stat().st_size + len(tail)
|
||||
return bytes(head), tail, f"multipart/form-data; boundary={boundary}", total
|
||||
|
||||
|
||||
def _open_http(url: str, timeout: int) -> tuple[http.client.HTTPConnection, str]:
|
||||
parsed = urllib.parse.urlsplit(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise RuntimeError(f"unsupported ASR URL: {url}")
|
||||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||
if parsed.scheme == "https":
|
||||
conn: http.client.HTTPConnection = http.client.HTTPSConnection(
|
||||
parsed.hostname, port, timeout=timeout)
|
||||
else:
|
||||
conn = http.client.HTTPConnection(parsed.hostname, port, timeout=timeout)
|
||||
path = parsed.path or "/"
|
||||
if parsed.query:
|
||||
path = f"{path}?{parsed.query}"
|
||||
return conn, path
|
||||
|
||||
|
||||
def _http_multipart(
|
||||
@@ -443,29 +460,48 @@ def _http_multipart(
|
||||
timeout: int = 900,
|
||||
retries: int = 3,
|
||||
) -> dict[str, Any]:
|
||||
data, content_type = _encode_multipart(fields, file_field, file_path)
|
||||
req_headers = {"Content-Type": content_type, **(headers or {})}
|
||||
"""POST audio as multipart, streaming the file in 1 MiB chunks."""
|
||||
head, tail, content_type, total = _multipart_framing(fields, file_field, file_path)
|
||||
last_exc: Exception | None = None
|
||||
raw = ""
|
||||
for attempt in range(1, max(1, retries) + 1):
|
||||
req = urllib.request.Request(url, data=data, headers=req_headers, method="POST")
|
||||
conn, path = _open_http(url, timeout)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read().decode("utf-8", errors="ignore")
|
||||
break
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", errors="ignore")[:300]
|
||||
if exc.code in (502, 503, 504) and attempt < retries:
|
||||
last_exc = exc
|
||||
conn.putrequest("POST", path)
|
||||
conn.putheader("Content-Type", content_type)
|
||||
conn.putheader("Content-Length", str(total))
|
||||
for key, value in (headers or {}).items():
|
||||
conn.putheader(key, value)
|
||||
conn.endheaders()
|
||||
conn.send(head)
|
||||
with file_path.open("rb") as stream:
|
||||
while True:
|
||||
chunk = stream.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
conn.send(chunk)
|
||||
conn.send(tail)
|
||||
resp = conn.getresponse()
|
||||
raw = resp.read().decode("utf-8", errors="ignore")
|
||||
status = resp.status
|
||||
if status in (502, 503, 504) and attempt < retries:
|
||||
last_exc = RuntimeError(f"ASR HTTP {status}")
|
||||
time.sleep(2 * attempt)
|
||||
continue
|
||||
raise RuntimeError(f"ASR HTTP {exc.code}: {detail}") from exc
|
||||
except (urllib.error.URLError, http.client.HTTPException, ConnectionError, TimeoutError, OSError) as exc:
|
||||
if status >= 400:
|
||||
raise RuntimeError(f"ASR HTTP {status}: {raw[:300]}")
|
||||
break
|
||||
except RuntimeError:
|
||||
raise
|
||||
except (http.client.HTTPException, ConnectionError, TimeoutError, OSError) as exc:
|
||||
last_exc = exc
|
||||
if attempt < retries:
|
||||
log(f"multipart upload transient error (attempt {attempt}/{retries}): {exc}")
|
||||
time.sleep(2 * attempt)
|
||||
continue
|
||||
raise RuntimeError(f"ASR upload connection error after {retries} attempts: {exc}") from exc
|
||||
finally:
|
||||
conn.close()
|
||||
else:
|
||||
raise RuntimeError(f"ASR upload failed: {last_exc}")
|
||||
try:
|
||||
|
||||
@@ -337,6 +337,18 @@ def normalize_chunks(chunks, cleaned_parts: list[str], flags: list[dict],
|
||||
}
|
||||
|
||||
|
||||
def scrub_internal_wording(text: str) -> str:
|
||||
"""Keep pipeline jargon out of text a person will read."""
|
||||
for src, dst in (
|
||||
("卡片未提供", "记录未写明"),
|
||||
("卡片提及", "录音中提到"),
|
||||
("事实卡片", "整理要点"),
|
||||
("粗整理草稿", "整理稿"),
|
||||
):
|
||||
text = text.replace(src, dst)
|
||||
return text
|
||||
|
||||
|
||||
def extract_evidence(parts: list[str], usage: Usage) -> str:
|
||||
jobs = [prompts.EVIDENCE_PROMPT.format(chunk_id=i, body=body)
|
||||
for i, body in enumerate(parts)]
|
||||
@@ -347,7 +359,7 @@ def extract_evidence(parts: list[str], usage: Usage) -> str:
|
||||
max_tokens=1400, usage=usage)
|
||||
except PartialFailure as pf:
|
||||
outs = pf.outputs
|
||||
return "\n".join(x for x in outs if x.strip())
|
||||
return scrub_internal_wording("\n".join(x for x in outs if x.strip()))
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ S4
|
||||
@@ -386,7 +398,7 @@ def write_draft(cleaned: str, meta: dict, context: str, date: str,
|
||||
speaker_note=prompts.SPEAKER_NOTE.get(backend, ""),
|
||||
context=ctx, date=date, source_detail=detail,
|
||||
transcript=transcript_path, extra_fm=extra, body=cleaned)
|
||||
return chat(CFG.draft_model, prompt, max_tokens=12000, usage=usage)
|
||||
return scrub_internal_wording(chat(CFG.draft_model, prompt, max_tokens=12000, usage=usage))
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ driver
|
||||
|
||||
@@ -323,6 +323,7 @@ def answer(question_id: int, value: str, *, allow_update: bool = False) -> dict:
|
||||
return result
|
||||
payload = json.loads(row["payload"] or "{}")
|
||||
recording = row["recording"]
|
||||
previous = str(row["answer"] or "")
|
||||
con.execute(
|
||||
"UPDATE review_question SET status='answered',answer=?,answered_at=? WHERE id=?",
|
||||
(value, time.time(), question_id))
|
||||
@@ -343,9 +344,14 @@ def answer(question_id: int, value: str, *, allow_update: bool = False) -> dict:
|
||||
confidence=1.0, confirmed=1)
|
||||
saved = False
|
||||
voiceprint_enrolled = False
|
||||
if payload.get("kind") == "voiceprint":
|
||||
voiceprint_enrolled = _confirm_voiceprint(
|
||||
payload.get("draft_path", ""), payload.get("speaker_id", ""), value)
|
||||
if payload.get("kind") == "voiceprint" and not (allow_update and value == previous):
|
||||
try:
|
||||
voiceprint_enrolled = _confirm_voiceprint(
|
||||
payload.get("draft_path", ""), payload.get("speaker_id", ""), value)
|
||||
except Exception as exc:
|
||||
from .log import log
|
||||
log(f"voiceprint enroll failed after answer saved: {exc}", level="WARN")
|
||||
voiceprint_enrolled = False
|
||||
if payload.get("draft_path"):
|
||||
try:
|
||||
saved = _persist_draft_feedback(recording, payload["draft_path"])
|
||||
@@ -401,17 +407,32 @@ def reopen_question(question_id: int) -> bool:
|
||||
return ok
|
||||
|
||||
|
||||
def _question_matches(row, transcript_path: str, draft_path: str, job_id: str) -> bool:
|
||||
payload = json.loads(row["payload"] or "{}")
|
||||
return bool(
|
||||
(transcript_path and row["recording"] == transcript_path)
|
||||
or (draft_path and payload.get("draft_path") == draft_path)
|
||||
or (job_id and payload.get("job_id") == job_id)
|
||||
)
|
||||
|
||||
|
||||
def delete_for_job(transcript_path: str = "", draft_path: str = "", job_id: str = "") -> int:
|
||||
return _delete_questions(transcript_path, draft_path, job_id, pending_only=False)
|
||||
|
||||
|
||||
def delete_pending_for_job(transcript_path: str = "", draft_path: str = "", job_id: str = "") -> int:
|
||||
"""Drop unanswered questions. Answered rows stay so a failed refine can be retried."""
|
||||
return _delete_questions(transcript_path, draft_path, job_id, pending_only=True)
|
||||
|
||||
|
||||
def _delete_questions(transcript_path: str, draft_path: str, job_id: str, *, pending_only: bool) -> int:
|
||||
con = lexicon.connect()
|
||||
rows = con.execute("SELECT id, recording, payload FROM review_question").fetchall()
|
||||
rows = con.execute("SELECT id, recording, payload, status FROM review_question").fetchall()
|
||||
to_delete: list[int] = []
|
||||
for r in rows:
|
||||
payload = json.loads(r["payload"] or "{}")
|
||||
if (
|
||||
(transcript_path and r["recording"] == transcript_path)
|
||||
or (draft_path and payload.get("draft_path") == draft_path)
|
||||
or (job_id and payload.get("job_id") == job_id)
|
||||
):
|
||||
if pending_only and r["status"] != "pending":
|
||||
continue
|
||||
if _question_matches(r, transcript_path, draft_path, job_id):
|
||||
to_delete.append(int(r["id"]))
|
||||
if to_delete:
|
||||
marks = ",".join("?" for _ in to_delete)
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Browser session helpers shared by the new workbench and the legacy control UI.
|
||||
|
||||
An empty ``ASR_WEB_PASSWORD`` closes the UI. Set ``ASR_WEB_ALLOW_ANON=1`` only
|
||||
for a local process that is not reachable from the network.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
from http import cookies
|
||||
|
||||
|
||||
from .config import get_setting
|
||||
|
||||
|
||||
def allow_anon() -> bool:
|
||||
return os.getenv("ASR_WEB_ALLOW_ANON", "").strip().lower() in {
|
||||
"1", "true", "yes", "on",
|
||||
}
|
||||
|
||||
|
||||
def password() -> str:
|
||||
env_pw = os.getenv("ASR_WEB_PASSWORD")
|
||||
if env_pw not in (None, ""):
|
||||
return env_pw
|
||||
return get_setting("web_password", "")
|
||||
|
||||
|
||||
def _secret() -> bytes:
|
||||
return (password() or os.getenv("ASR_AGENT_TOKEN", "") or "asr-web-unset").encode("utf-8")
|
||||
|
||||
|
||||
def session_token() -> str:
|
||||
return hmac.new(_secret(), b"asr-web-session-v1", hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
def csrf_token() -> str:
|
||||
return hmac.new(_secret(), b"asr-web-csrf-v1", hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
def cookie_matches(header: str) -> bool:
|
||||
if not password():
|
||||
return False
|
||||
jar = cookies.SimpleCookie()
|
||||
try:
|
||||
jar.load(header or "")
|
||||
except Exception:
|
||||
return False
|
||||
morsel = jar.get("asr_session")
|
||||
return bool(morsel and hmac.compare_digest(morsel.value, session_token()))
|
||||
|
||||
|
||||
def csrf_matches(supplied: str) -> bool:
|
||||
if not supplied:
|
||||
return False
|
||||
return hmac.compare_digest(supplied, csrf_token())
|
||||
Reference in New Issue
Block a user