444 lines
18 KiB
Python
444 lines
18 KiB
Python
#!/usr/bin/env python3
|
||
"""Persistent human-review queue with transport-independent storage."""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import subprocess
|
||
import time
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
from . import lexicon
|
||
from .config import CFG
|
||
|
||
FEEDBACK_START = "<!-- asr-web-review:start -->"
|
||
FEEDBACK_END = "<!-- asr-web-review:end -->"
|
||
CARD_THRESHOLD = 2
|
||
|
||
|
||
def enqueue(recording: str, question: str, payload: dict) -> int:
|
||
con = lexicon.connect()
|
||
old = con.execute(
|
||
"SELECT id FROM review_question WHERE recording=? AND question=? "
|
||
"AND status='pending'", (recording, question)).fetchone()
|
||
if old:
|
||
con.close()
|
||
return int(old["id"])
|
||
cur = con.execute(
|
||
"INSERT INTO review_question(recording,question,payload,created_at)"
|
||
" VALUES(?,?,?,?)",
|
||
(recording, question, json.dumps(payload, ensure_ascii=False), time.time()))
|
||
con.commit()
|
||
row_id = int(cur.lastrowid)
|
||
con.close()
|
||
return row_id
|
||
|
||
|
||
def pending(limit: int = 20) -> list[dict]:
|
||
con = lexicon.connect()
|
||
rows = con.execute(
|
||
"SELECT * FROM review_question WHERE status='pending' "
|
||
"ORDER BY created_at LIMIT ?", (limit,)).fetchall()
|
||
con.close()
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
def enqueue_draft(recording: str, draft_path: Path) -> int:
|
||
"""Queue the draft's explicit uncertainties plus one overall review item."""
|
||
path = draft_path.resolve()
|
||
try:
|
||
path.relative_to(CFG.drafts.resolve())
|
||
except ValueError:
|
||
raise ValueError(f"draft outside review root: {path}")
|
||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||
fm_match = re.match(r"(?s)^---\n(.*?)\n---", text)
|
||
fm = fm_match.group(1) if fm_match else ""
|
||
bg_match = re.search(
|
||
r"(?ms)^## 通话背景（待确认）\s*$\n(.*?)(?=^## |\Z)", text)
|
||
if bg_match:
|
||
background = re.sub(r"\s+", " ", bg_match.group(1)).strip()
|
||
else:
|
||
body = text[fm_match.end():] if fm_match else text
|
||
body = re.sub(r"(?m)^# .*$", "", body).strip()
|
||
background = re.sub(r"\s+", " ", body[:700]).strip()
|
||
count = 0
|
||
enqueue(recording,
|
||
"系统判断的通话背景是否基本准确？若偏差较大，请在下方写明正确背景以重跑草稿。",
|
||
{
|
||
"kind": "background_check", "draft_path": str(path),
|
||
"question_index": 0, "sentence": background,
|
||
"candidates": ["背景基本准确"],
|
||
})
|
||
count += 1
|
||
|
||
named = {}
|
||
in_speakers = False
|
||
for line in fm.splitlines():
|
||
if line.strip() == "voiceprint_speakers:":
|
||
in_speakers = True
|
||
continue
|
||
if in_speakers:
|
||
sm = re.match(r"^\s+(speaker_\d+):\s*(.+?)\s*$", line)
|
||
if sm:
|
||
named[sm.group(1)] = sm.group(2).strip()
|
||
continue
|
||
in_speakers = False
|
||
transcript_path = ""
|
||
tm = re.search(r"(?m)^transcript:\s*(.+?)\s*$", fm)
|
||
if tm:
|
||
transcript_path = tm.group(1).strip().strip("\"'")
|
||
transcript = ""
|
||
try:
|
||
transcript = Path(transcript_path).read_text(encoding="utf-8", errors="ignore")
|
||
except OSError:
|
||
pass
|
||
all_speakers = sorted(set(re.findall(r"(?m)^【(speaker_\d+)】", transcript)))
|
||
missing_speakers = [s for s in all_speakers if s not in named]
|
||
for speaker in missing_speakers:
|
||
excerpts = re.findall(
|
||
rf"(?m)^【{re.escape(speaker)}】(.+)$", transcript)
|
||
spoken_chars = sum(len(re.sub(r"\s+", "", x)) for x in excerpts)
|
||
if spoken_chars < 80 and max((len(x) for x in excerpts), default=0) < 50:
|
||
continue
|
||
excerpt = "\n\n".join(x.strip() for x in excerpts[:3])[:1200]
|
||
enqueue(recording, f"{speaker} 尚未注册声纹；你认识这位说话人吗？", {
|
||
"kind": "voiceprint", "speaker_id": speaker,
|
||
"draft_path": str(path), "question_index": count,
|
||
"sentence": excerpt or "没有足够发言摘录，请无法确认时直接跳过。",
|
||
"candidates": [],
|
||
})
|
||
count += 1
|
||
|
||
m = re.search(r"(?ms)^## 待你确认\s*$\n(.*?)(?=^## |\Z)", text)
|
||
questions = []
|
||
if m:
|
||
questions = [
|
||
re.sub(r"\s+", " ", x).strip()
|
||
for x in re.findall(r"(?m)^\d+\.\s+(.+)$", m.group(1))
|
||
]
|
||
for i, question in enumerate(questions, 1):
|
||
if any(speaker in question and "身份" in question
|
||
for speaker in missing_speakers):
|
||
continue
|
||
candidates = list(dict.fromkeys(re.findall(r"\[\[([^\]|#]+)", question)))
|
||
enqueue(recording, question, {
|
||
"kind": "draft_question", "draft_path": str(path),
|
||
"question_index": count, "sentence": question,
|
||
"candidates": candidates[:4],
|
||
})
|
||
count += 1
|
||
|
||
cards = {
|
||
"人名": {p.stem for p in (CFG.vault / "People").glob("*.md")
|
||
if not p.name.startswith("_")},
|
||
"公司": {p.stem for p in (CFG.vault / "Companies").glob("*.md")
|
||
if not p.name.startswith("_")},
|
||
}
|
||
con = lexicon.connect()
|
||
for kind, key in (("人名", "people"), ("公司", "companies")):
|
||
names = re.findall(r"[^,\[\]\s，、]+",
|
||
_frontmatter_value(text, key))
|
||
for name in names:
|
||
if name in cards[kind] or not name or name.startswith("speaker_"):
|
||
continue
|
||
history = con.execute(
|
||
"SELECT count(DISTINCT source) FROM entity_mention "
|
||
"WHERE entity=? AND kind=?", (name, kind)).fetchone()[0]
|
||
if history + 1 < CARD_THRESHOLD:
|
||
continue
|
||
label = "人物卡" if kind == "人名" else "公司卡"
|
||
enqueue(recording,
|
||
f"“{name}”计入本次后已在 {history + 1} 场正式记录中出现，"
|
||
f"是否建立{label}？", {
|
||
"kind": "entity_card", "entity": name,
|
||
"entity_kind": kind, "draft_path": str(path),
|
||
"question_index": count,
|
||
"sentence": "只建立保守存根；职位、重要性、合作阶段等判断字段不会自动猜。",
|
||
"candidates": [f"建立{label}", "暂不建立"],
|
||
})
|
||
count += 1
|
||
con.close()
|
||
enqueue(recording, "这份草稿整体是否可以作为后续定稿的基础？", {
|
||
"kind": "draft_overall", "draft_path": str(path),
|
||
"question_index": count,
|
||
"sentence": "可展开查看完整草稿；选择“还需修改”时，请在输入框说明修改方向。",
|
||
"candidates": ["草稿整体可用", "还需修改"],
|
||
})
|
||
return count + 1
|
||
|
||
|
||
def _frontmatter_value(text: str, key: str) -> str:
|
||
m = re.match(r"(?s)^---\n(.*?)\n---", text)
|
||
if not m:
|
||
return ""
|
||
km = re.search(rf"(?m)^{re.escape(key)}:\s*(.+?)\s*$", m.group(1))
|
||
return km.group(1).strip().strip("\"'") if km else ""
|
||
|
||
|
||
def _voiceprint_name(value: str) -> str:
|
||
if value in {"无法确认，保留原文", "不知道", "不认识"}:
|
||
return ""
|
||
name = re.split(r"[，,；;（(\s]", value.strip(), maxsplit=1)[0]
|
||
return name if 2 <= len(name) <= 12 and not name.startswith("speaker_") else ""
|
||
|
||
|
||
def _confirm_voiceprint(draft_path: str, speaker_id: str, value: str) -> bool:
|
||
"""Persist an explicit speaker identity and, when a voiceprint session exists,
|
||
enroll it against the acoustic voiceprint service."""
|
||
name = _voiceprint_name(value)
|
||
if not name:
|
||
return False
|
||
path = Path(draft_path).resolve()
|
||
try:
|
||
path.relative_to(CFG.drafts.resolve())
|
||
except ValueError:
|
||
return False
|
||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||
session_id = _frontmatter_value(text, "voiceprint_session")
|
||
fm = re.match(r"(?s)^---\n(.*?)\n---", text)
|
||
if not fm:
|
||
return False
|
||
block = fm.group(1)
|
||
speaker_line = rf"(?m)^(\s+{re.escape(speaker_id)}:)\s*.*$"
|
||
if re.search(speaker_line, block):
|
||
block = re.sub(speaker_line, rf"\1 {name}", block)
|
||
elif re.search(r"(?m)^voiceprint_speakers:\s*(?:\{\})?\s*$", block):
|
||
block = re.sub(
|
||
r"(?m)^voiceprint_speakers:\s*(?:\{\})?\s*$",
|
||
f"voiceprint_speakers:\n {speaker_id}: {name}", block, count=1)
|
||
elif re.search(r"(?m)^voiceprint_speakers:\s*$", block):
|
||
block = re.sub(
|
||
r"(?m)^(voiceprint_speakers:\s*)$",
|
||
rf"\1\n {speaker_id}: {name}", block, count=1)
|
||
else:
|
||
block += f"\nvoiceprint_speakers:\n {speaker_id}: {name}"
|
||
if session_id:
|
||
if re.search(r"(?m)^voiceprint_enroll:", block):
|
||
block = re.sub(r"(?m)^voiceprint_enroll:.*$",
|
||
"voiceprint_enroll: true", block)
|
||
else:
|
||
block += "\nvoiceprint_enroll: true"
|
||
text = text[:fm.start()] + "---\n" + block + "\n---" + text[fm.end():]
|
||
from . import vault
|
||
vault.write_draft_file(path, text)
|
||
|
||
if not session_id or not CFG.voiceprint_enroll_url:
|
||
return False
|
||
body = json.dumps({
|
||
"session_id": session_id, "speaker_id": int(speaker_id.split("_")[1]),
|
||
"name": name,
|
||
}).encode()
|
||
req = urllib.request.Request(
|
||
CFG.voiceprint_enroll_url, data=body,
|
||
headers={"Content-Type": "application/json"})
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=20) as response:
|
||
result = json.load(response)
|
||
return result.get("status") == "enrolled"
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def _persist_draft_feedback(recording: str, draft_path: str) -> bool:
|
||
"""Once a draft's web questions are complete, append an auditable answer block."""
|
||
path = Path(draft_path).resolve()
|
||
try:
|
||
path.relative_to(CFG.drafts.resolve())
|
||
except ValueError:
|
||
return False
|
||
con = lexicon.connect()
|
||
rows = con.execute(
|
||
"SELECT * FROM review_question WHERE recording=? ORDER BY created_at,id",
|
||
(recording,)).fetchall()
|
||
con.close()
|
||
selected = []
|
||
for row in rows:
|
||
payload = json.loads(row["payload"] or "{}")
|
||
if payload.get("draft_path") == str(path):
|
||
if row["status"] == "pending":
|
||
return False
|
||
selected.append((row["question"], row["answer"] or ""))
|
||
if not selected or not path.exists():
|
||
return False
|
||
lines = [FEEDBACK_START, "## Web 人工确认记录", ""]
|
||
for i, (question, answer_value) in enumerate(selected, 1):
|
||
lines.extend([f"{i}. **问题**：{question}", f" - **回答**：{answer_value}"])
|
||
lines.extend(["", FEEDBACK_END])
|
||
block = "\n".join(lines)
|
||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||
pattern = re.compile(
|
||
re.escape(FEEDBACK_START) + r".*?" + re.escape(FEEDBACK_END), re.S)
|
||
text = pattern.sub(block, text) if pattern.search(text) else text.rstrip() + "\n\n" + block + "\n"
|
||
from . import vault
|
||
vault.write_draft_file(path, text)
|
||
return True
|
||
|
||
|
||
def _request_revision(draft_path: str) -> bool:
|
||
queue_dir = os.environ.get("ASR_REVISION_QUEUE_DIR", "")
|
||
if queue_dir:
|
||
try:
|
||
path = Path(draft_path).resolve()
|
||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||
job = _frontmatter_value(text, "intake_job")
|
||
queue = Path(queue_dir)
|
||
queue.mkdir(parents=True, exist_ok=True)
|
||
target = queue / f"{path.stem}.json"
|
||
tmp = queue / f".{path.stem}.{os.getpid()}.tmp"
|
||
tmp.write_text(json.dumps({"draft": str(path), "job_id": job},
|
||
ensure_ascii=False), encoding="utf-8")
|
||
tmp.replace(target)
|
||
return True
|
||
except Exception:
|
||
return False
|
||
name = Path(draft_path).name
|
||
try:
|
||
escaped = subprocess.run(
|
||
["systemd-escape", name], capture_output=True, text=True,
|
||
timeout=5, check=True).stdout.strip()
|
||
result = subprocess.run(
|
||
["systemctl", "--user", "start", "--no-block",
|
||
f"asr-review-revise@{escaped}.service"],
|
||
capture_output=True, text=True, timeout=10)
|
||
return result.returncode == 0
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
def answer(question_id: int, value: str, *, allow_update: bool = False) -> dict:
|
||
con = lexicon.connect()
|
||
row = con.execute("SELECT * FROM review_question WHERE id=?",
|
||
(question_id,)).fetchone()
|
||
if not row:
|
||
con.close()
|
||
raise KeyError(f"review question {question_id} not found")
|
||
if row["status"] != "pending" and not allow_update:
|
||
result = {
|
||
"id": question_id, "answer": row["answer"], "learned": False,
|
||
"already_answered": True,
|
||
}
|
||
con.close()
|
||
return result
|
||
payload = json.loads(row["payload"] or "{}")
|
||
recording = row["recording"]
|
||
previous = str(row["answer"] or "")
|
||
con.execute(
|
||
"UPDATE review_question SET status='answered',answer=?,answered_at=? WHERE id=?",
|
||
(value, time.time(), question_id))
|
||
con.commit()
|
||
con.close()
|
||
|
||
# A positive answer to an entity choice becomes reusable deterministic memory.
|
||
source = payload.get("surface", "")
|
||
candidates = payload.get("candidates") or []
|
||
canonical = value if value in candidates else ""
|
||
if source and canonical:
|
||
lexicon.learn_alias(source, canonical, origin="review", confirmed=1,
|
||
note=f"review question {question_id}")
|
||
lexicon.learn_correction(source, canonical, confirmed=1,
|
||
note=f"review question {question_id}")
|
||
lexicon.record_observation(source, canonical, source=row["recording"],
|
||
context=payload.get("sentence", ""),
|
||
confidence=1.0, confirmed=1)
|
||
saved = False
|
||
voiceprint_enrolled = False
|
||
if payload.get("kind") == "voiceprint" and not (allow_update and value == previous):
|
||
try:
|
||
voiceprint_enrolled = _confirm_voiceprint(
|
||
payload.get("draft_path", ""), payload.get("speaker_id", ""), value)
|
||
except Exception as exc:
|
||
from .log import log
|
||
log(f"voiceprint enroll failed after answer saved: {exc}", level="WARN")
|
||
voiceprint_enrolled = False
|
||
if payload.get("draft_path"):
|
||
try:
|
||
saved = _persist_draft_feedback(recording, payload["draft_path"])
|
||
except Exception as e:
|
||
# The answer is already durable in SQLite. A transient git/filesystem
|
||
# failure must not turn a successful mobile submission into a retry.
|
||
from .log import log
|
||
log(f"review answer saved but draft feedback sync failed: {e}",
|
||
level="WARN")
|
||
saved = False
|
||
revision_started = saved and _request_revision(payload["draft_path"])
|
||
if revision_started:
|
||
try:
|
||
text = Path(payload["draft_path"]).read_text(
|
||
encoding="utf-8", errors="ignore")
|
||
job = _frontmatter_value(text, "intake_job")
|
||
if job:
|
||
from .control_web import update_job
|
||
if (payload.get("kind") == "publish_approval"
|
||
and value == "发布正式稿"):
|
||
update_job(job, "publishing", "发布已确认，正在写入正式纪要")
|
||
else:
|
||
update_job(job, "refining", "正在应用回答并生成定稿预览")
|
||
except Exception:
|
||
pass
|
||
return {
|
||
"id": question_id, "answer": value, "learned": bool(canonical),
|
||
"draft_feedback_saved": saved, "voiceprint_enrolled": voiceprint_enrolled,
|
||
"revision_started": revision_started,
|
||
}
|
||
|
||
|
||
def recent_answered(limit: int = 30) -> list[dict]:
|
||
con = lexicon.connect()
|
||
rows = con.execute(
|
||
"SELECT * FROM review_question WHERE status='answered' "
|
||
"ORDER BY answered_at DESC, id DESC LIMIT ?",
|
||
(limit,),
|
||
).fetchall()
|
||
con.close()
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
def reopen_question(question_id: int) -> bool:
|
||
con = lexicon.connect()
|
||
cur = con.execute(
|
||
"UPDATE review_question SET status='pending', answer='', answered_at=NULL WHERE id=?",
|
||
(question_id,),
|
||
)
|
||
con.commit()
|
||
ok = cur.rowcount > 0
|
||
con.close()
|
||
return ok
|
||
|
||
|
||
def _question_matches(row, transcript_path: str, draft_path: str, job_id: str) -> bool:
|
||
payload = json.loads(row["payload"] or "{}")
|
||
return bool(
|
||
(transcript_path and row["recording"] == transcript_path)
|
||
or (draft_path and payload.get("draft_path") == draft_path)
|
||
or (job_id and payload.get("job_id") == job_id)
|
||
)
|
||
|
||
|
||
def delete_for_job(transcript_path: str = "", draft_path: str = "", job_id: str = "") -> int:
|
||
return _delete_questions(transcript_path, draft_path, job_id, pending_only=False)
|
||
|
||
|
||
def delete_pending_for_job(transcript_path: str = "", draft_path: str = "", job_id: str = "") -> int:
|
||
"""Drop unanswered questions. Answered rows stay so a failed refine can be retried."""
|
||
return _delete_questions(transcript_path, draft_path, job_id, pending_only=True)
|
||
|
||
|
||
def _delete_questions(transcript_path: str, draft_path: str, job_id: str, *, pending_only: bool) -> int:
|
||
con = lexicon.connect()
|
||
rows = con.execute("SELECT id, recording, payload, status FROM review_question").fetchall()
|
||
to_delete: list[int] = []
|
||
for r in rows:
|
||
if pending_only and r["status"] != "pending":
|
||
continue
|
||
if _question_matches(r, transcript_path, draft_path, job_id):
|
||
to_delete.append(int(r["id"]))
|
||
if to_delete:
|
||
marks = ",".join("?" for _ in to_delete)
|
||
con.execute(f"DELETE FROM review_question WHERE id IN ({marks})", to_delete)
|
||
con.commit()
|
||
con.close()
|
||
return len(to_delete)
|
||
|