#!/usr/bin/env python3 """Persistent human-review queue with transport-independent storage.""" from __future__ import annotations import json import os import re import subprocess import time import urllib.request from pathlib import Path from . import lexicon from .config import CFG FEEDBACK_START = "" FEEDBACK_END = "" CARD_THRESHOLD = 2 def enqueue(recording: str, question: str, payload: dict) -> int: con = lexicon.connect() old = con.execute( "SELECT id FROM review_question WHERE recording=? AND question=? " "AND status='pending'", (recording, question)).fetchone() if old: con.close() return int(old["id"]) cur = con.execute( "INSERT INTO review_question(recording,question,payload,created_at)" " VALUES(?,?,?,?)", (recording, question, json.dumps(payload, ensure_ascii=False), time.time())) con.commit() row_id = int(cur.lastrowid) con.close() return row_id def pending(limit: int = 20) -> list[dict]: con = lexicon.connect() rows = con.execute( "SELECT * FROM review_question WHERE status='pending' " "ORDER BY created_at LIMIT ?", (limit,)).fetchall() con.close() return [dict(r) for r in rows] def enqueue_draft(recording: str, draft_path: Path) -> int: """Queue the draft's explicit uncertainties plus one overall review item.""" path = draft_path.resolve() try: path.relative_to(CFG.drafts.resolve()) except ValueError: raise ValueError(f"draft outside review root: {path}") text = path.read_text(encoding="utf-8", errors="ignore") fm_match = re.match(r"(?s)^---\n(.*?)\n---", text) fm = fm_match.group(1) if fm_match else "" bg_match = re.search( r"(?ms)^## 通话背景(待确认)\s*$\n(.*?)(?=^## |\Z)", text) if bg_match: background = re.sub(r"\s+", " ", bg_match.group(1)).strip() else: body = text[fm_match.end():] if fm_match else text body = re.sub(r"(?m)^# .*$", "", body).strip() background = re.sub(r"\s+", " ", body[:700]).strip() count = 0 enqueue(recording, "系统判断的通话背景是否基本准确?若偏差较大,请在下方写明正确背景以重跑草稿。", { "kind": "background_check", "draft_path": str(path), "question_index": 0, "sentence": background, "candidates": ["背景基本准确"], }) count += 1 named = {} in_speakers = False for line in fm.splitlines(): if line.strip() == "voiceprint_speakers:": in_speakers = True continue if in_speakers: sm = re.match(r"^\s+(speaker_\d+):\s*(.+?)\s*$", line) if sm: named[sm.group(1)] = sm.group(2).strip() continue in_speakers = False transcript_path = "" tm = re.search(r"(?m)^transcript:\s*(.+?)\s*$", fm) if tm: transcript_path = tm.group(1).strip().strip("\"'") transcript = "" try: transcript = Path(transcript_path).read_text(encoding="utf-8", errors="ignore") except OSError: pass all_speakers = sorted(set(re.findall(r"(?m)^【(speaker_\d+)】", transcript))) missing_speakers = [s for s in all_speakers if s not in named] for speaker in missing_speakers: excerpts = re.findall( rf"(?m)^【{re.escape(speaker)}】(.+)$", transcript) spoken_chars = sum(len(re.sub(r"\s+", "", x)) for x in excerpts) if spoken_chars < 80 and max((len(x) for x in excerpts), default=0) < 50: continue excerpt = "\n\n".join(x.strip() for x in excerpts[:3])[:1200] enqueue(recording, f"{speaker} 尚未注册声纹;你认识这位说话人吗?", { "kind": "voiceprint", "speaker_id": speaker, "draft_path": str(path), "question_index": count, "sentence": excerpt or "没有足够发言摘录,请无法确认时直接跳过。", "candidates": [], }) count += 1 m = re.search(r"(?ms)^## 待你确认\s*$\n(.*?)(?=^## |\Z)", text) questions = [] if m: questions = [ re.sub(r"\s+", " ", x).strip() for x in re.findall(r"(?m)^\d+\.\s+(.+)$", m.group(1)) ] for i, question in enumerate(questions, 1): if any(speaker in question and "身份" in question for speaker in missing_speakers): continue candidates = list(dict.fromkeys(re.findall(r"\[\[([^\]|#]+)", question))) enqueue(recording, question, { "kind": "draft_question", "draft_path": str(path), "question_index": count, "sentence": question, "candidates": candidates[:4], }) count += 1 cards = { "人名": {p.stem for p in (CFG.vault / "People").glob("*.md") if not p.name.startswith("_")}, "公司": {p.stem for p in (CFG.vault / "Companies").glob("*.md") if not p.name.startswith("_")}, } con = lexicon.connect() for kind, key in (("人名", "people"), ("公司", "companies")): names = re.findall(r"[^,\[\]\s,、]+", _frontmatter_value(text, key)) for name in names: if name in cards[kind] or not name or name.startswith("speaker_"): continue history = con.execute( "SELECT count(DISTINCT source) FROM entity_mention " "WHERE entity=? AND kind=?", (name, kind)).fetchone()[0] if history + 1 < CARD_THRESHOLD: continue label = "人物卡" if kind == "人名" else "公司卡" enqueue(recording, f"“{name}”计入本次后已在 {history + 1} 场正式记录中出现," f"是否建立{label}?", { "kind": "entity_card", "entity": name, "entity_kind": kind, "draft_path": str(path), "question_index": count, "sentence": "只建立保守存根;职位、重要性、合作阶段等判断字段不会自动猜。", "candidates": [f"建立{label}", "暂不建立"], }) count += 1 con.close() enqueue(recording, "这份草稿整体是否可以作为后续定稿的基础?", { "kind": "draft_overall", "draft_path": str(path), "question_index": count, "sentence": "可展开查看完整草稿;选择“还需修改”时,请在输入框说明修改方向。", "candidates": ["草稿整体可用", "还需修改"], }) return count + 1 def _frontmatter_value(text: str, key: str) -> str: m = re.match(r"(?s)^---\n(.*?)\n---", text) if not m: return "" km = re.search(rf"(?m)^{re.escape(key)}:\s*(.+?)\s*$", m.group(1)) return km.group(1).strip().strip("\"'") if km else "" def _voiceprint_name(value: str) -> str: if value in {"无法确认,保留原文", "不知道", "不认识"}: return "" name = re.split(r"[,,;;((\s]", value.strip(), maxsplit=1)[0] return name if 2 <= len(name) <= 12 and not name.startswith("speaker_") else "" def _confirm_voiceprint(draft_path: str, speaker_id: str, value: str) -> bool: """Persist an explicit speaker identity and, when a voiceprint session exists, enroll it against the acoustic voiceprint service.""" name = _voiceprint_name(value) if not name: return False path = Path(draft_path).resolve() try: path.relative_to(CFG.drafts.resolve()) except ValueError: return False text = path.read_text(encoding="utf-8", errors="ignore") session_id = _frontmatter_value(text, "voiceprint_session") fm = re.match(r"(?s)^---\n(.*?)\n---", text) if not fm: return False block = fm.group(1) speaker_line = rf"(?m)^(\s+{re.escape(speaker_id)}:)\s*.*$" if re.search(speaker_line, block): block = re.sub(speaker_line, rf"\1 {name}", block) elif re.search(r"(?m)^voiceprint_speakers:\s*(?:\{\})?\s*$", block): block = re.sub( r"(?m)^voiceprint_speakers:\s*(?:\{\})?\s*$", f"voiceprint_speakers:\n {speaker_id}: {name}", block, count=1) elif re.search(r"(?m)^voiceprint_speakers:\s*$", block): block = re.sub( r"(?m)^(voiceprint_speakers:\s*)$", rf"\1\n {speaker_id}: {name}", block, count=1) else: block += f"\nvoiceprint_speakers:\n {speaker_id}: {name}" if session_id: if re.search(r"(?m)^voiceprint_enroll:", block): block = re.sub(r"(?m)^voiceprint_enroll:.*$", "voiceprint_enroll: true", block) else: block += "\nvoiceprint_enroll: true" text = text[:fm.start()] + "---\n" + block + "\n---" + text[fm.end():] from . import vault vault.write_draft_file(path, text) if not session_id or not CFG.voiceprint_enroll_url: return False body = json.dumps({ "session_id": session_id, "speaker_id": int(speaker_id.split("_")[1]), "name": name, }).encode() req = urllib.request.Request( CFG.voiceprint_enroll_url, data=body, headers={"Content-Type": "application/json"}) try: with urllib.request.urlopen(req, timeout=20) as response: result = json.load(response) return result.get("status") == "enrolled" except Exception: return False def _persist_draft_feedback(recording: str, draft_path: str) -> bool: """Once a draft's web questions are complete, append an auditable answer block.""" path = Path(draft_path).resolve() try: path.relative_to(CFG.drafts.resolve()) except ValueError: return False con = lexicon.connect() rows = con.execute( "SELECT * FROM review_question WHERE recording=? ORDER BY created_at,id", (recording,)).fetchall() con.close() selected = [] for row in rows: payload = json.loads(row["payload"] or "{}") if payload.get("draft_path") == str(path): if row["status"] == "pending": return False selected.append((row["question"], row["answer"] or "")) if not selected or not path.exists(): return False lines = [FEEDBACK_START, "## Web 人工确认记录", ""] for i, (question, answer_value) in enumerate(selected, 1): lines.extend([f"{i}. **问题**:{question}", f" - **回答**:{answer_value}"]) lines.extend(["", FEEDBACK_END]) block = "\n".join(lines) text = path.read_text(encoding="utf-8", errors="ignore") pattern = re.compile( re.escape(FEEDBACK_START) + r".*?" + re.escape(FEEDBACK_END), re.S) text = pattern.sub(block, text) if pattern.search(text) else text.rstrip() + "\n\n" + block + "\n" from . import vault vault.write_draft_file(path, text) return True def _request_revision(draft_path: str) -> bool: queue_dir = os.environ.get("ASR_REVISION_QUEUE_DIR", "") if queue_dir: try: path = Path(draft_path).resolve() text = path.read_text(encoding="utf-8", errors="ignore") job = _frontmatter_value(text, "intake_job") queue = Path(queue_dir) queue.mkdir(parents=True, exist_ok=True) target = queue / f"{path.stem}.json" tmp = queue / f".{path.stem}.{os.getpid()}.tmp" tmp.write_text(json.dumps({"draft": str(path), "job_id": job}, ensure_ascii=False), encoding="utf-8") tmp.replace(target) return True except Exception: return False name = Path(draft_path).name try: escaped = subprocess.run( ["systemd-escape", name], capture_output=True, text=True, timeout=5, check=True).stdout.strip() result = subprocess.run( ["systemctl", "--user", "start", "--no-block", f"asr-review-revise@{escaped}.service"], capture_output=True, text=True, timeout=10) return result.returncode == 0 except Exception: return False def answer(question_id: int, value: str, *, allow_update: bool = False) -> dict: con = lexicon.connect() row = con.execute("SELECT * FROM review_question WHERE id=?", (question_id,)).fetchone() if not row: con.close() raise KeyError(f"review question {question_id} not found") if row["status"] != "pending" and not allow_update: result = { "id": question_id, "answer": row["answer"], "learned": False, "already_answered": True, } con.close() return result payload = json.loads(row["payload"] or "{}") recording = row["recording"] con.execute( "UPDATE review_question SET status='answered',answer=?,answered_at=? WHERE id=?", (value, time.time(), question_id)) con.commit() con.close() # A positive answer to an entity choice becomes reusable deterministic memory. source = payload.get("surface", "") candidates = payload.get("candidates") or [] canonical = value if value in candidates else "" if source and canonical: lexicon.learn_alias(source, canonical, origin="review", confirmed=1, note=f"review question {question_id}") lexicon.learn_correction(source, canonical, confirmed=1, note=f"review question {question_id}") lexicon.record_observation(source, canonical, source=row["recording"], context=payload.get("sentence", ""), confidence=1.0, confirmed=1) saved = False voiceprint_enrolled = False if payload.get("kind") == "voiceprint": voiceprint_enrolled = _confirm_voiceprint( payload.get("draft_path", ""), payload.get("speaker_id", ""), value) if payload.get("draft_path"): try: saved = _persist_draft_feedback(recording, payload["draft_path"]) except Exception as e: # The answer is already durable in SQLite. A transient git/filesystem # failure must not turn a successful mobile submission into a retry. from .log import log log(f"review answer saved but draft feedback sync failed: {e}", level="WARN") saved = False revision_started = saved and _request_revision(payload["draft_path"]) if revision_started: try: text = Path(payload["draft_path"]).read_text( encoding="utf-8", errors="ignore") job = _frontmatter_value(text, "intake_job") if job: from .control_web import update_job if (payload.get("kind") == "publish_approval" and value == "发布正式稿"): update_job(job, "publishing", "发布已确认,正在写入正式纪要") else: update_job(job, "refining", "正在应用回答并生成定稿预览") except Exception: pass return { "id": question_id, "answer": value, "learned": bool(canonical), "draft_feedback_saved": saved, "voiceprint_enrolled": voiceprint_enrolled, "revision_started": revision_started, } def recent_answered(limit: int = 30) -> list[dict]: con = lexicon.connect() rows = con.execute( "SELECT * FROM review_question WHERE status='answered' " "ORDER BY answered_at DESC, id DESC LIMIT ?", (limit,), ).fetchall() con.close() return [dict(r) for r in rows] def reopen_question(question_id: int) -> bool: con = lexicon.connect() cur = con.execute( "UPDATE review_question SET status='pending', answer='', answered_at=NULL WHERE id=?", (question_id,), ) con.commit() ok = cur.rowcount > 0 con.close() return ok def delete_for_job(transcript_path: str = "", draft_path: str = "", job_id: str = "") -> int: con = lexicon.connect() rows = con.execute("SELECT id, recording, payload FROM review_question").fetchall() to_delete: list[int] = [] for r in rows: payload = json.loads(r["payload"] or "{}") if ( (transcript_path and r["recording"] == transcript_path) or (draft_path and payload.get("draft_path") == draft_path) or (job_id and payload.get("job_id") == job_id) ): to_delete.append(int(r["id"])) if to_delete: marks = ",".join("?" for _ in to_delete) con.execute(f"DELETE FROM review_question WHERE id IN ({marks})", to_delete) con.commit() con.close() return len(to_delete)