feat(asr-pipeline): merge PR #1 webauth/CSRF/locking hardening and web_password fallback
Build image / build (push) Canceled after 0s
Build ASR Pipeline / image (push) Canceled after 0s

This commit is contained in:
Codex
2026-11-02 06:37:33 -08:00
parent 9debd1c47d
commit 3c70861463
44 changed files with 4354 additions and 333 deletions
@@ -153,7 +153,7 @@ def bootstrap_default_settings() -> None:
"hindsight_bank": _env("HINDSIGHT_BANK", "personal-context"),
"use_hindsight": "1" if _env("ASR_HINDSIGHT", "1") != "0" else "0",
"use_termbank": "1" if _env("ASR_TERMBANK", "1") != "0" else "0",
"git_sync": "1" if _env("ASR_GIT_SYNC", "0") != "0" else "0",
"git_sync": "0" if _env("ASR_GIT_SYNC", "0") == "0" else "1",
}
con = sqlite3.connect(db_path, timeout=15)
try:
@@ -320,7 +320,7 @@ class Config:
)
git_sync = _SettingAttr(
"git_sync",
lambda: _env("ASR_GIT_SYNC", "1"),
lambda: _env("ASR_GIT_SYNC", "0"),
lambda s: str(s).lower() not in ("0", "false", "no", "off"),
)
@@ -399,7 +399,7 @@ class Config:
# ---- behaviour ------------------------------------------------------
notify: bool = field(
default_factory=lambda: _env("ASR_NOTIFY", "1") != "0")
default_factory=lambda: _env("ASR_NOTIFY", "0") != "0")
cache_llm: bool = field(
default_factory=lambda: _env("ASR_CACHE", "0") != "0")
@@ -2,6 +2,8 @@
"""Persistent mobile control plane for recording intake and human review."""
from __future__ import annotations
import hashlib
import hmac
import html
import json
import os
@@ -14,7 +16,7 @@ from http import HTTPStatus
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from . import review
from . import review, webauth
from .config import CFG
from .log import log
@@ -166,6 +168,7 @@ def _shell(title: str, body: str, script: str = "") -> bytes:
return f"""<!doctype html>
<html lang="zh-CN"><head>
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="csrf" content="{html.escape(webauth.csrf_token(), quote=True)}">
<meta name="color-scheme" content="light"><meta http-equiv="Cache-Control" content="no-store">
<title>{html.escape(title)}</title>
<style>
@@ -236,6 +239,7 @@ document.querySelector('#upload').addEventListener('submit',e=>{
const x=new XMLHttpRequest(), b=document.querySelector('#send'), p=document.querySelector('#progress'), out=document.querySelector('#result');
b.disabled=true;p.hidden=false;out.textContent='正在上传…';
x.open('POST','/api/upload');x.setRequestHeader('Content-Type','application/octet-stream');
x.setRequestHeader('X-CSRF-Token', document.querySelector('meta[name=csrf]')?.content || '');
x.setRequestHeader('X-Filename',encodeURIComponent(f.name));
x.setRequestHeader('X-Title',encodeURIComponent(document.querySelector('#title').value));
x.setRequestHeader('X-Context',encodeURIComponent(document.querySelector('#context').value));
@@ -293,6 +297,7 @@ def _review_page(action: str, job: dict | None = None) -> bytes:
body = f"""<section class="panel"><div class="meta">{html.escape(title)} · 剩余 {len(rows)} 个 · 问题 #{row['id']}</div>
<h1>{html.escape(row['question'])}</h1>{source_html}{draft_html}
<form method="post" action="{html.escape(action, quote=True)}">
<input type="hidden" name="csrf" value="{html.escape(webauth.csrf_token(), quote=True)}">
<input type="hidden" name="id" value="{row['id']}">{buttons}
<div class="custom"><input name="custom" maxlength="120" autocomplete="off"
placeholder="{html.escape(placeholder, quote=True)}"><button type="submit">提交</button></div>
@@ -342,12 +347,43 @@ class Handler(BaseHTTPRequestHandler):
return True, _job(match.group(1))
return False, None
def _browser_ok(self) -> bool:
if webauth.allow_anon():
return True
if not webauth.password():
return False
return webauth.cookie_matches(self.headers.get("Cookie", ""))
def _login_page(self, err: str = "") -> bytes:
if not webauth.password() and not webauth.allow_anon():
err = err or "未配置 ASR_WEB_PASSWORD,控制面已关闭。"
token = html.escape(webauth.csrf_token(), quote=True)
alert = f"<p>{html.escape(err)}</p>" if err else ""
body = (
f'<section class="panel"><h1>访问验证</h1>{alert}'
f'<form method="post" action="/login">'
f'<input type="hidden" name="csrf" value="{token}">'
f'<label>访问口令</label><input name="password" type="password" required>'
f'<div class="actions"><button type="submit">登录</button></div></form></section>'
)
return _shell("登录", body)
def do_GET(self):
path = urllib.parse.urlsplit(self.path).path
if path == "/":
return self._send(_home())
if path == "/health":
return self._json({"ok": True, "pending": len(review.pending(100))})
if path in {"/login", "/logout"}:
if path == "/logout":
self.send_response(HTTPStatus.SEE_OTHER)
self.send_header("Location", "/login")
self.send_header("Set-Cookie", "asr_session=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0")
self.end_headers()
return
return self._send(self._login_page())
if not self._browser_ok():
return self._send(self._login_page(), HTTPStatus.UNAUTHORIZED)
if path == "/":
return self._send(_home())
if path == "/api/jobs":
jobs = _jobs()
for j in jobs:
@@ -372,9 +408,9 @@ class Handler(BaseHTTPRequestHandler):
def _upload(self):
try:
length = int(self.headers.get("Content-Length", "0"))
length = int(self.headers.get("Content-Length", "0") or 0)
except ValueError:
length = 0
return self._json({"error": "bad content-length"}, HTTPStatus.BAD_REQUEST)
if length <= 0 or length > MAX_UPLOAD:
return self._json({"error": "invalid upload size"},
HTTPStatus.REQUEST_ENTITY_TOO_LARGE)
@@ -408,7 +444,7 @@ class Handler(BaseHTTPRequestHandler):
while remaining:
data = self.rfile.read(min(CHUNK, remaining))
if not data:
raise EOFError("upload ended early")
raise EOFError("incomplete upload")
f.write(data)
remaining -= len(data)
f.flush()
@@ -426,16 +462,28 @@ class Handler(BaseHTTPRequestHandler):
update_job(job_id, "queued", "录音已落盘,等待 watcher 接收")
return self._json({"ok": True, "job_id": job_id,
"stored_name": stored}, HTTPStatus.CREATED)
except EOFError:
part.unlink(missing_ok=True)
update_job(job_id, "failed", "上传未完成")
return self._json({"error": "incomplete upload"}, HTTPStatus.BAD_REQUEST)
except Exception as e:
part.unlink(missing_ok=True)
update_job(job_id, "failed", f"上传失败:{e}")
return self._json({"error": str(e)}, HTTPStatus.INTERNAL_SERVER_ERROR)
update_job(job_id, "failed", "上传失败")
log(f"upload failed: {e}")
return self._json({"error": "upload failed"}, HTTPStatus.INTERNAL_SERVER_ERROR)
def _answer(self):
try:
length = min(int(self.headers.get("Content-Length", "0")), 4096)
try:
length = int(self.headers.get("Content-Length", "0") or 0)
except ValueError as exc:
raise ValueError("bad content-length") from exc
if length < 0 or length > 4096:
raise ValueError("bad content-length")
form = urllib.parse.parse_qs(
self.rfile.read(length).decode("utf-8"))
if not webauth.csrf_matches((form.get("csrf") or [""])[0]):
raise ValueError("csrf")
qid = int(form.get("id", ["0"])[0])
answer = (form.get("custom", [""])[0].strip()
or form.get("answer", [""])[0].strip())
@@ -460,7 +508,31 @@ class Handler(BaseHTTPRequestHandler):
def do_POST(self):
path = urllib.parse.urlsplit(self.path).path
if path == "/login":
try:
length = int(self.headers.get("Content-Length", "0") or 0)
except ValueError:
return self._json({"error": "bad content-length"}, HTTPStatus.BAD_REQUEST)
if length < 0:
return self._json({"error": "bad content-length"}, HTTPStatus.BAD_REQUEST)
length = min(length, 4096)
form = urllib.parse.parse_qs(self.rfile.read(length).decode("utf-8", "ignore"))
pw = form.get("password", [""])[0]
if webauth.password() and hmac.compare_digest(pw, webauth.password()):
self.send_response(HTTPStatus.SEE_OTHER)
self.send_header("Location", "/")
self.send_header(
"Set-Cookie",
f"asr_session={webauth.session_token()}; Path=/; HttpOnly; SameSite=Strict; Max-Age=86400",
)
self.end_headers()
return
return self._send(self._login_page("口令错误或未配置 ASR_WEB_PASSWORD"), HTTPStatus.UNAUTHORIZED)
if not self._browser_ok():
return self._json({"error": "unauthorized"}, HTTPStatus.UNAUTHORIZED)
if path in ("/api/upload", "/hook/intake"):
if not webauth.csrf_matches(self.headers.get("X-CSRF-Token", "")):
return self._json({"error": "csrf"}, HTTPStatus.FORBIDDEN)
return self._upload()
if self._review_job()[0]:
return self._answer()
@@ -44,14 +44,16 @@ from pathlib import Path
from .config import CFG
LAB = CFG.root
BANK = CFG.termbank_dir
CACHE = CFG.cache_dir / "flagger"
VAULT = CFG.vault
def _vault() -> Path:
return CFG.vault
HS_API = CFG.hindsight_api
HS_BANK = CFG.hindsight_bank
HS_TIMEOUT = CFG.hindsight_timeout
def _bank() -> Path:
return CFG.termbank_dir
def _cache() -> Path:
return CFG.cache_dir / "flagger"
HAN = re.compile(r"[\u4e00-\u9fff]+")
HAN_FULL = re.compile(r"^[\u4e00-\u9fff]+$")
@@ -264,16 +266,16 @@ def _dir_sig(paths) -> str:
def load_pool(use_termbank=True) -> dict[str, Cand]:
"""Vault entities (cards + glossary) ∪ optional domain term bank."""
CACHE.mkdir(parents=True, exist_ok=True)
_cache().mkdir(parents=True, exist_ok=True)
if CFG.use_lexicon:
try:
from . import lexicon
lexicon.sync_from_vault()
except Exception:
pass
sig = _dir_sig([VAULT / "People", VAULT / "Companies", VAULT / "minutes",
sig = _dir_sig([_vault() / "People", _vault() / "Companies", _vault() / "minutes",
CFG.lexicon_db])
cpath = CACHE / f"pool6-{sig}-{int(use_termbank)}.json"
cpath = _cache() / f"pool6-{sig}-{int(use_termbank)}.json"
if cpath.exists():
d = json.loads(cpath.read_text())
return {k: Cand(**v) for k, v in d.items()}
@@ -331,7 +333,7 @@ def load_pool(use_termbank=True) -> dict[str, Cand]:
ents = {}
if not ents: # lexicon disabled/broken
for sub, kind in (("People", "人名"), ("Companies", "公司")):
d = VAULT / sub
d = _vault() / sub
if not d.exists():
continue
for f in d.glob("*.md"):
@@ -346,7 +348,7 @@ def load_pool(use_termbank=True) -> dict[str, Cand]:
add(form, c.canonical, c.kind, "variant", rule, c.owner, c.role, anchor)
if use_termbank:
tb = BANK / "terms.json"
tb = _bank() / "terms.json"
if tb.exists():
for term, e in json.loads(tb.read_text()).items():
# Only the on-domain, distinctive slices are worth phonetic search;
@@ -361,7 +363,7 @@ def load_pool(use_termbank=True) -> dict[str, Cand]:
def load_allowlist() -> set[str]:
p = BANK / "allowlist.txt"
p = _bank() / "allowlist.txt"
return set(p.read_text().split()) if p.exists() else set()
@@ -388,13 +390,13 @@ def _minutes_entities(text: str) -> set[str]:
def _corpus() -> list[tuple[str, set[str], str]]:
CACHE.mkdir(parents=True, exist_ok=True)
sig = _dir_sig([VAULT / "minutes"])
cpath = CACHE / f"corpus-{sig}.json"
_cache().mkdir(parents=True, exist_ok=True)
sig = _dir_sig([_vault() / "minutes"])
cpath = _cache() / f"corpus-{sig}.json"
if cpath.exists():
return [(a, set(b), c) for a, b, c in json.loads(cpath.read_text())]
out = []
for f in sorted((VAULT / "minutes").glob("*.md")):
for f in sorted((_vault() / "minutes").glob("*.md")):
txt = f.read_text(errors="ignore")
m = re.search(r"^date:\s*(\d{4}-\d{2}-\d{2})", txt, re.M)
out.append((f.name, _minutes_entities(txt), m.group(1) if m else ""))
@@ -414,11 +416,11 @@ def hindsight_recall(seed: list[str], extra_query: str = "") -> tuple[set[str],
return set(), []
query = (" ".join(sorted(seed)) + " " + extra_query).strip() or "会议 相关人物 公司"
body = json.dumps({"query": query, "max_tokens": 1500}).encode()
url = f"{HS_API}/v1/default/banks/{HS_BANK}/memories/recall"
url = f"{CFG.hindsight_api}/v1/default/banks/{CFG.hindsight_bank}/memories/recall"
try:
req = urllib.request.Request(url, data=body,
headers={"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=HS_TIMEOUT) as r:
with urllib.request.urlopen(req, timeout=CFG.hindsight_timeout) as r:
d = json.loads(r.read())
except Exception:
return set(), []
@@ -157,7 +157,8 @@ class Entity:
def connect() -> sqlite3.Connection:
CFG.state_dir.mkdir(parents=True, exist_ok=True)
con = sqlite3.connect(CFG.lexicon_db)
con = sqlite3.connect(CFG.lexicon_db, timeout=30)
con.execute("PRAGMA journal_mode=WAL")
con.row_factory = sqlite3.Row
con.executescript(SCHEMA)
# One-time compatibility migration for the 158 corrections learned before the
@@ -527,7 +528,7 @@ def stats() -> dict:
# ------------------------------------------------------------------ Web UI CRUD
def list_aliases(q: str = "", origin_filter: str = "", limit: int = 200) -> list[dict]:
def list_aliases(q: str = "", origin_filter: str = "", limit: int = 200, offset: int = 0) -> list[dict]:
con = connect()
sql = "SELECT * FROM alias WHERE 1=1"
args: list = []
@@ -539,13 +540,30 @@ def list_aliases(q: str = "", origin_filter: str = "", limit: int = 200) -> list
sql += " AND origin <> 'card'"
elif origin_filter == "card":
sql += " AND origin = 'card'"
sql += " ORDER BY confirmed DESC, created_at DESC, alias ASC LIMIT ?"
args.append(limit)
sql += " ORDER BY confirmed DESC, created_at DESC, alias ASC LIMIT ? OFFSET ?"
args.extend((max(1, min(limit, 500)), max(0, offset)))
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
con.close()
return rows
def count_aliases(q: str = "", origin_filter: str = "") -> int:
con = connect()
sql = "SELECT count(*) FROM alias WHERE 1=1"
args: list = []
if q.strip():
pat = f"%{q.strip()}%"
sql += " AND (alias LIKE ? OR canonical LIKE ? OR note LIKE ?)"
args.extend([pat, pat, pat])
if origin_filter == "learned":
sql += " AND origin <> 'card'"
elif origin_filter == "card":
sql += " AND origin = 'card'"
n = int(con.execute(sql, args).fetchone()[0])
con.close()
return n
def update_alias(
old_alias: str,
old_canonical: str,
@@ -590,7 +608,7 @@ def delete_alias(alias: str, canonical: str) -> bool:
return deleted
def list_corrections(q: str = "", limit: int = 200) -> list[dict]:
def list_corrections(q: str = "", limit: int = 200, offset: int = 0) -> list[dict]:
con = connect()
sql = "SELECT * FROM correction WHERE 1=1"
args: list = []
@@ -598,13 +616,26 @@ def list_corrections(q: str = "", limit: int = 200) -> list[dict]:
pat = f"%{q.strip()}%"
sql += " AND (surface LIKE ? OR canonical LIKE ? OR note LIKE ?)"
args.extend([pat, pat, pat])
sql += " ORDER BY confirmed DESC, n DESC, last_seen DESC LIMIT ?"
args.append(limit)
sql += " ORDER BY confirmed DESC, n DESC, last_seen DESC LIMIT ? OFFSET ?"
args.extend((max(1, min(limit, 500)), max(0, offset)))
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
con.close()
return rows
def count_corrections(q: str = "") -> int:
con = connect()
sql = "SELECT count(*) FROM correction WHERE 1=1"
args: list = []
if q.strip():
pat = f"%{q.strip()}%"
sql += " AND (surface LIKE ? OR canonical LIKE ? OR note LIKE ?)"
args.extend([pat, pat, pat])
n = int(con.execute(sql, args).fetchone()[0])
con.close()
return n
def update_correction(
old_surface: str,
old_canonical: str,
@@ -651,7 +682,7 @@ def delete_correction(surface: str, canonical: str) -> bool:
return deleted
def list_entities(q: str = "", kind: str = "", limit: int = 200) -> list[dict]:
def list_entities(q: str = "", kind: str = "", limit: int = 200, offset: int = 0) -> list[dict]:
con = connect()
sql = "SELECT * FROM entity WHERE 1=1"
args: list = []
@@ -662,13 +693,29 @@ def list_entities(q: str = "", kind: str = "", limit: int = 200) -> list[dict]:
if kind.strip():
sql += " AND kind=?"
args.append(kind.strip())
sql += " ORDER BY (CASE WHEN card='' THEN 0 ELSE 1 END), canonical ASC LIMIT ?"
args.append(limit)
sql += " ORDER BY (CASE WHEN card='' THEN 0 ELSE 1 END), canonical ASC LIMIT ? OFFSET ?"
args.extend((max(1, min(limit, 500)), max(0, offset)))
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
con.close()
return rows
def count_entities(q: str = "", kind: str = "") -> int:
con = connect()
sql = "SELECT count(*) FROM entity WHERE 1=1"
args: list = []
if q.strip():
pat = f"%{q.strip()}%"
sql += " AND (canonical LIKE ? OR company LIKE ? OR role LIKE ? OR related LIKE ?)"
args.extend([pat, pat, pat, pat])
if kind.strip():
sql += " AND kind=?"
args.append(kind.strip())
n = int(con.execute(sql, args).fetchone()[0])
con.close()
return n
def upsert_entity(
canonical: str,
kind: str = "人名",
@@ -723,7 +770,7 @@ def delete_entity(canonical: str) -> bool:
return deleted
def list_relations(q: str = "", limit: int = 200) -> list[dict]:
def list_relations(q: str = "", limit: int = 200, offset: int = 0) -> list[dict]:
con = connect()
sql = "SELECT * FROM relation WHERE 1=1"
args: list = []
@@ -731,13 +778,26 @@ def list_relations(q: str = "", limit: int = 200) -> list[dict]:
pat = f"%{q.strip()}%"
sql += " AND (subject LIKE ? OR predicate LIKE ? OR object LIKE ? OR source LIKE ?)"
args.extend([pat, pat, pat, pat])
sql += " ORDER BY active DESC, confidence DESC, last_seen DESC LIMIT ?"
args.append(limit)
sql += " ORDER BY active DESC, confidence DESC, last_seen DESC LIMIT ? OFFSET ?"
args.extend((max(1, min(limit, 500)), max(0, offset)))
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
con.close()
return rows
def count_relations(q: str = "") -> int:
con = connect()
sql = "SELECT count(*) FROM relation WHERE 1=1"
args: list = []
if q.strip():
pat = f"%{q.strip()}%"
sql += " AND (subject LIKE ? OR predicate LIKE ? OR object LIKE ? OR source LIKE ?)"
args.extend([pat, pat, pat, pat])
n = int(con.execute(sql, args).fetchone()[0])
con.close()
return n
def update_relation(
old_subject: str,
old_predicate: str,
@@ -49,10 +49,8 @@ _key: list[str] = []
def _auth() -> str:
with _lock:
if not _key:
_key.append(CFG.api_key())
return _key[0]
# Read on every call so a key saved from /settings is used without a restart.
return CFG.api_key()
@dataclass
@@ -0,0 +1,25 @@
"""One machine-wide lock for processes that consume the recording inbox."""
from __future__ import annotations
import fcntl
import os
from pathlib import Path
def acquire_recording_lock(path: Path):
"""Return an open file holding an exclusive lock, or exit if another consumer has it.
The handle must stay referenced for the life of the process.
"""
path.parent.mkdir(parents=True, exist_ok=True)
handle = path.open("a+")
try:
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except OSError:
handle.close()
raise SystemExit(f"another recording consumer holds {path}")
handle.seek(0)
handle.truncate()
handle.write(str(os.getpid()))
handle.flush()
return handle
@@ -93,7 +93,7 @@ KEYWORD_PROMPT = """从下面的会议转写摘选中,提取用于检索背景
# (「谭凯(白帆生物总经理)」、「此项目负责人为 [[冯昆仑]](背景确认)」) even when the
# recording never said them. The facts were often correct, but a meeting note that
# cannot distinguish "said on the call" from "known from notes" is unsafe to finalize.
DRAFT_PROMPT = """你是会议录音整理助手,正在根据**带来源编号的事实卡片**产出「粗整理草稿」。
DRAFT_PROMPT = """你是会议录音整理助手,正在根据下方整理要点写出会议草稿。
严格遵循:
1. 只产出粗整理草稿,**不是**正式纪要。
@@ -133,8 +133,11 @@ DRAFT_PROMPT = """你是会议录音整理助手,正在根据**带来源编号
(例如「徐总(疑似 [[徐劲前]] 或 [[邹洵]])」),而不是笼统写「身份不明」。
13. 判定简称指向谁时使用背景里的任职信息,这属于**辨识**而非越界断言,
不需要加〔背景〕;但如果你要额外陈述这个人的职务、公司归属本身,那就要加〔背景〕。
14. 对匿名说话人,除非事实卡片已经给出有证据的候选,否则只写“身份待确认”;
14. 对匿名说话人,除非整理要点已经给出有证据的候选,否则只写“身份待确认”;
不要从背景中枚举可能的人名让用户猜。
15. 正文和标题不要出现「卡片」「提示词」「系统指令」「粗整理」这些内部说法。
记录里没有的信息写「记录未写明」,不要写「卡片未提供」或「卡片提及」。
标题用业务名称,不要带「草稿」二字。
{speaker_note}
@@ -153,7 +156,7 @@ companies: [有把握的公司…]
transcript: {transcript}
{extra_fm}---
事实卡片:
整理要点:
{body}
只输出 markdown 草稿全文,不要任何额外说明。"""
@@ -409,28 +409,45 @@ def probe_duration_seconds(audio: Path) -> int:
return 0
def _encode_multipart(
def _multipart_framing(
fields: dict[str, str],
file_field: str,
file_path: Path,
) -> tuple[bytes, str]:
) -> tuple[bytes, bytes, str, int]:
"""Build multipart head/tail without reading the audio into memory."""
boundary = f"----ASRBoundary{uuid.uuid4().hex}"
body = bytearray()
safe_name = file_path.name.replace('"', "").replace("\r", "").replace("\n", "") or "audio"
head = bytearray()
for name, val in fields.items():
body.extend(f"--{boundary}\r\n".encode())
body.extend(
f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode())
body.extend(str(val).encode("utf-8"))
body.extend(b"\r\n")
mime = mimetypes.guess_type(file_path.name)[0] or "application/octet-stream"
body.extend(f"--{boundary}\r\n".encode())
body.extend(
head.extend(f"--{boundary}\r\n".encode())
head.extend(f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode())
head.extend(str(val).encode("utf-8"))
head.extend(b"\r\n")
mime = mimetypes.guess_type(safe_name)[0] or "application/octet-stream"
head.extend(f"--{boundary}\r\n".encode())
head.extend(
f'Content-Disposition: form-data; name="{file_field}"; '
f'filename="{file_path.name}"\r\n'.encode())
body.extend(f"Content-Type: {mime}\r\n\r\n".encode())
body.extend(file_path.read_bytes())
body.extend(f"\r\n--{boundary}--\r\n".encode())
return bytes(body), f"multipart/form-data; boundary={boundary}"
f'filename="{safe_name}"\r\n'.encode())
head.extend(f"Content-Type: {mime}\r\n\r\n".encode())
tail = f"\r\n--{boundary}--\r\n".encode()
total = len(head) + file_path.stat().st_size + len(tail)
return bytes(head), tail, f"multipart/form-data; boundary={boundary}", total
def _open_http(url: str, timeout: int) -> tuple[http.client.HTTPConnection, str]:
parsed = urllib.parse.urlsplit(url)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise RuntimeError(f"unsupported ASR URL: {url}")
port = parsed.port or (443 if parsed.scheme == "https" else 80)
if parsed.scheme == "https":
conn: http.client.HTTPConnection = http.client.HTTPSConnection(
parsed.hostname, port, timeout=timeout)
else:
conn = http.client.HTTPConnection(parsed.hostname, port, timeout=timeout)
path = parsed.path or "/"
if parsed.query:
path = f"{path}?{parsed.query}"
return conn, path
def _http_multipart(
@@ -443,29 +460,48 @@ def _http_multipart(
timeout: int = 900,
retries: int = 3,
) -> dict[str, Any]:
data, content_type = _encode_multipart(fields, file_field, file_path)
req_headers = {"Content-Type": content_type, **(headers or {})}
"""POST audio as multipart, streaming the file in 1 MiB chunks."""
head, tail, content_type, total = _multipart_framing(fields, file_field, file_path)
last_exc: Exception | None = None
raw = ""
for attempt in range(1, max(1, retries) + 1):
req = urllib.request.Request(url, data=data, headers=req_headers, method="POST")
conn, path = _open_http(url, timeout)
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read().decode("utf-8", errors="ignore")
break
except urllib.error.HTTPError as exc:
detail = exc.read().decode("utf-8", errors="ignore")[:300]
if exc.code in (502, 503, 504) and attempt < retries:
last_exc = exc
conn.putrequest("POST", path)
conn.putheader("Content-Type", content_type)
conn.putheader("Content-Length", str(total))
for key, value in (headers or {}).items():
conn.putheader(key, value)
conn.endheaders()
conn.send(head)
with file_path.open("rb") as stream:
while True:
chunk = stream.read(1024 * 1024)
if not chunk:
break
conn.send(chunk)
conn.send(tail)
resp = conn.getresponse()
raw = resp.read().decode("utf-8", errors="ignore")
status = resp.status
if status in (502, 503, 504) and attempt < retries:
last_exc = RuntimeError(f"ASR HTTP {status}")
time.sleep(2 * attempt)
continue
raise RuntimeError(f"ASR HTTP {exc.code}: {detail}") from exc
except (urllib.error.URLError, http.client.HTTPException, ConnectionError, TimeoutError, OSError) as exc:
if status >= 400:
raise RuntimeError(f"ASR HTTP {status}: {raw[:300]}")
break
except RuntimeError:
raise
except (http.client.HTTPException, ConnectionError, TimeoutError, OSError) as exc:
last_exc = exc
if attempt < retries:
log(f"multipart upload transient error (attempt {attempt}/{retries}): {exc}")
time.sleep(2 * attempt)
continue
raise RuntimeError(f"ASR upload connection error after {retries} attempts: {exc}") from exc
finally:
conn.close()
else:
raise RuntimeError(f"ASR upload failed: {last_exc}")
try:
@@ -337,6 +337,18 @@ def normalize_chunks(chunks, cleaned_parts: list[str], flags: list[dict],
}
def scrub_internal_wording(text: str) -> str:
"""Keep pipeline jargon out of text a person will read."""
for src, dst in (
("卡片未提供", "记录未写明"),
("卡片提及", "录音中提到"),
("事实卡片", "整理要点"),
("粗整理草稿", "整理稿"),
):
text = text.replace(src, dst)
return text
def extract_evidence(parts: list[str], usage: Usage) -> str:
jobs = [prompts.EVIDENCE_PROMPT.format(chunk_id=i, body=body)
for i, body in enumerate(parts)]
@@ -347,7 +359,7 @@ def extract_evidence(parts: list[str], usage: Usage) -> str:
max_tokens=1400, usage=usage)
except PartialFailure as pf:
outs = pf.outputs
return "\n".join(x for x in outs if x.strip())
return scrub_internal_wording("\n".join(x for x in outs if x.strip()))
# ------------------------------------------------------------------ S4
@@ -386,7 +398,7 @@ def write_draft(cleaned: str, meta: dict, context: str, date: str,
speaker_note=prompts.SPEAKER_NOTE.get(backend, ""),
context=ctx, date=date, source_detail=detail,
transcript=transcript_path, extra_fm=extra, body=cleaned)
return chat(CFG.draft_model, prompt, max_tokens=12000, usage=usage)
return scrub_internal_wording(chat(CFG.draft_model, prompt, max_tokens=12000, usage=usage))
# ------------------------------------------------------------------ driver
@@ -323,6 +323,7 @@ def answer(question_id: int, value: str, *, allow_update: bool = False) -> dict:
return result
payload = json.loads(row["payload"] or "{}")
recording = row["recording"]
previous = str(row["answer"] or "")
con.execute(
"UPDATE review_question SET status='answered',answer=?,answered_at=? WHERE id=?",
(value, time.time(), question_id))
@@ -343,9 +344,14 @@ def answer(question_id: int, value: str, *, allow_update: bool = False) -> dict:
confidence=1.0, confirmed=1)
saved = False
voiceprint_enrolled = False
if payload.get("kind") == "voiceprint":
voiceprint_enrolled = _confirm_voiceprint(
payload.get("draft_path", ""), payload.get("speaker_id", ""), value)
if payload.get("kind") == "voiceprint" and not (allow_update and value == previous):
try:
voiceprint_enrolled = _confirm_voiceprint(
payload.get("draft_path", ""), payload.get("speaker_id", ""), value)
except Exception as exc:
from .log import log
log(f"voiceprint enroll failed after answer saved: {exc}", level="WARN")
voiceprint_enrolled = False
if payload.get("draft_path"):
try:
saved = _persist_draft_feedback(recording, payload["draft_path"])
@@ -401,17 +407,32 @@ def reopen_question(question_id: int) -> bool:
return ok
def _question_matches(row, transcript_path: str, draft_path: str, job_id: str) -> bool:
payload = json.loads(row["payload"] or "{}")
return bool(
(transcript_path and row["recording"] == transcript_path)
or (draft_path and payload.get("draft_path") == draft_path)
or (job_id and payload.get("job_id") == job_id)
)
def delete_for_job(transcript_path: str = "", draft_path: str = "", job_id: str = "") -> int:
return _delete_questions(transcript_path, draft_path, job_id, pending_only=False)
def delete_pending_for_job(transcript_path: str = "", draft_path: str = "", job_id: str = "") -> int:
"""Drop unanswered questions. Answered rows stay so a failed refine can be retried."""
return _delete_questions(transcript_path, draft_path, job_id, pending_only=True)
def _delete_questions(transcript_path: str, draft_path: str, job_id: str, *, pending_only: bool) -> int:
con = lexicon.connect()
rows = con.execute("SELECT id, recording, payload FROM review_question").fetchall()
rows = con.execute("SELECT id, recording, payload, status FROM review_question").fetchall()
to_delete: list[int] = []
for r in rows:
payload = json.loads(r["payload"] or "{}")
if (
(transcript_path and r["recording"] == transcript_path)
or (draft_path and payload.get("draft_path") == draft_path)
or (job_id and payload.get("job_id") == job_id)
):
if pending_only and r["status"] != "pending":
continue
if _question_matches(r, transcript_path, draft_path, job_id):
to_delete.append(int(r["id"]))
if to_delete:
marks = ",".join("?" for _ in to_delete)
@@ -0,0 +1,57 @@
"""Browser session helpers shared by the new workbench and the legacy control UI.
An empty ``ASR_WEB_PASSWORD`` closes the UI. Set ``ASR_WEB_ALLOW_ANON=1`` only
for a local process that is not reachable from the network.
"""
from __future__ import annotations
import hashlib
import hmac
import os
from http import cookies
from .config import get_setting
def allow_anon() -> bool:
return os.getenv("ASR_WEB_ALLOW_ANON", "").strip().lower() in {
"1", "true", "yes", "on",
}
def password() -> str:
env_pw = os.getenv("ASR_WEB_PASSWORD")
if env_pw not in (None, ""):
return env_pw
return get_setting("web_password", "")
def _secret() -> bytes:
return (password() or os.getenv("ASR_AGENT_TOKEN", "") or "asr-web-unset").encode("utf-8")
def session_token() -> str:
return hmac.new(_secret(), b"asr-web-session-v1", hashlib.sha256).hexdigest()
def csrf_token() -> str:
return hmac.new(_secret(), b"asr-web-csrf-v1", hashlib.sha256).hexdigest()
def cookie_matches(header: str) -> bool:
if not password():
return False
jar = cookies.SimpleCookie()
try:
jar.load(header or "")
except Exception:
return False
morsel = jar.get("asr_session")
return bool(morsel and hmac.compare_digest(morsel.value, session_token()))
def csrf_matches(supplied: str) -> bool:
if not supplied:
return False
return hmac.compare_digest(supplied, csrf_token())