deploy standalone ASR pipeline on tools-pve

2026-09-18 02:32:14 -07:00
parent f30f72b295
commit b23f00f41a
38 changed files with 182773 additions and 0 deletions
@@ -0,0 +1,372 @@
+#!/usr/bin/env python3
+"""Persistent human-review queue with transport-independent storage."""
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+import time
+import urllib.request
+from pathlib import Path
+
+from . import lexicon
+from .config import CFG
+
+FEEDBACK_START = "<!-- asr-web-review:start -->"
+FEEDBACK_END = "<!-- asr-web-review:end -->"
+CARD_THRESHOLD = 2
+
+
+def enqueue(recording: str, question: str, payload: dict) -> int:
+    con = lexicon.connect()
+    old = con.execute(
+        "SELECT id FROM review_question WHERE recording=? AND question=? "
+        "AND status='pending'", (recording, question)).fetchone()
+    if old:
+        con.close()
+        return int(old["id"])
+    cur = con.execute(
+        "INSERT INTO review_question(recording,question,payload,created_at)"
+        " VALUES(?,?,?,?)",
+        (recording, question, json.dumps(payload, ensure_ascii=False), time.time()))
+    con.commit()
+    row_id = int(cur.lastrowid)
+    con.close()
+    return row_id
+
+
+def pending(limit: int = 20) -> list[dict]:
+    con = lexicon.connect()
+    rows = con.execute(
+        "SELECT * FROM review_question WHERE status='pending' "
+        "ORDER BY created_at LIMIT ?", (limit,)).fetchall()
+    con.close()
+    return [dict(r) for r in rows]
+
+
+def enqueue_draft(recording: str, draft_path: Path) -> int:
+    """Queue the draft's explicit uncertainties plus one overall review item."""
+    path = draft_path.resolve()
+    try:
+        path.relative_to(CFG.drafts.resolve())
+    except ValueError:
+        raise ValueError(f"draft outside review root: {path}")
+    text = path.read_text(encoding="utf-8", errors="ignore")
+    fm_match = re.match(r"(?s)^---\n(.*?)\n---", text)
+    fm = fm_match.group(1) if fm_match else ""
+    bg_match = re.search(
+        r"(?ms)^## 通话背景（待确认）\s*$\n(.*?)(?=^## |\Z)", text)
+    if bg_match:
+        background = re.sub(r"\s+", " ", bg_match.group(1)).strip()
+    else:
+        body = text[fm_match.end():] if fm_match else text
+        body = re.sub(r"(?m)^# .*$", "", body).strip()
+        background = re.sub(r"\s+", " ", body[:700]).strip()
+    count = 0
+    enqueue(recording,
+            "系统判断的通话背景是否基本准确？若偏差较大，请在下方写明正确背景以重跑草稿。",
+            {
+        "kind": "background_check", "draft_path": str(path),
+        "question_index": 0, "sentence": background,
+        "candidates": ["背景基本准确"],
+    })
+    count += 1
+
+    named = {}
+    in_speakers = False
+    for line in fm.splitlines():
+        if line.strip() == "voiceprint_speakers:":
+            in_speakers = True
+            continue
+        if in_speakers:
+            sm = re.match(r"^\s+(speaker_\d+):\s*(.+?)\s*$", line)
+            if sm:
+                named[sm.group(1)] = sm.group(2).strip()
+                continue
+            in_speakers = False
+    transcript_path = ""
+    tm = re.search(r"(?m)^transcript:\s*(.+?)\s*$", fm)
+    if tm:
+        transcript_path = tm.group(1).strip().strip("\"'")
+    transcript = ""
+    try:
+        transcript = Path(transcript_path).read_text(encoding="utf-8", errors="ignore")
+    except OSError:
+        pass
+    all_speakers = sorted(set(re.findall(r"(?m)^【(speaker_\d+)】", transcript)))
+    missing_speakers = [s for s in all_speakers if s not in named]
+    for speaker in missing_speakers:
+        excerpts = re.findall(
+            rf"(?m)^【{re.escape(speaker)}】(.+)$", transcript)
+        spoken_chars = sum(len(re.sub(r"\s+", "", x)) for x in excerpts)
+        if spoken_chars < 80 and max((len(x) for x in excerpts), default=0) < 50:
+            continue
+        excerpt = "\n\n".join(x.strip() for x in excerpts[:3])[:1200]
+        enqueue(recording, f"{speaker} 尚未注册声纹；你认识这位说话人吗？", {
+            "kind": "voiceprint", "speaker_id": speaker,
+            "draft_path": str(path), "question_index": count,
+            "sentence": excerpt or "没有足够发言摘录，请无法确认时直接跳过。",
+            "candidates": [],
+        })
+        count += 1
+
+    m = re.search(r"(?ms)^## 待你确认\s*$\n(.*?)(?=^## |\Z)", text)
+    questions = []
+    if m:
+        questions = [
+            re.sub(r"\s+", " ", x).strip()
+            for x in re.findall(r"(?m)^\d+\.\s+(.+)$", m.group(1))
+        ]
+    for i, question in enumerate(questions, 1):
+        if any(speaker in question and "身份" in question
+               for speaker in missing_speakers):
+            continue
+        candidates = list(dict.fromkeys(re.findall(r"\[\[([^\]|#]+)", question)))
+        enqueue(recording, question, {
+            "kind": "draft_question", "draft_path": str(path),
+            "question_index": count, "sentence": question,
+            "candidates": candidates[:4],
+        })
+        count += 1
+
+    cards = {
+        "人名": {p.stem for p in (CFG.vault / "People").glob("*.md")
+                 if not p.name.startswith("_")},
+        "公司": {p.stem for p in (CFG.vault / "Companies").glob("*.md")
+                 if not p.name.startswith("_")},
+    }
+    con = lexicon.connect()
+    for kind, key in (("人名", "people"), ("公司", "companies")):
+        names = re.findall(r"[^,\[\]\s，、]+",
+                           _frontmatter_value(text, key))
+        for name in names:
+            if name in cards[kind] or not name or name.startswith("speaker_"):
+                continue
+            history = con.execute(
+                "SELECT count(DISTINCT source) FROM entity_mention "
+                "WHERE entity=? AND kind=?", (name, kind)).fetchone()[0]
+            if history + 1 < CARD_THRESHOLD:
+                continue
+            label = "人物卡" if kind == "人名" else "公司卡"
+            enqueue(recording,
+                    f"“{name}”计入本次后已在 {history + 1} 场正式记录中出现，"
+                    f"是否建立{label}？", {
+                        "kind": "entity_card", "entity": name,
+                        "entity_kind": kind, "draft_path": str(path),
+                        "question_index": count,
+                        "sentence": "只建立保守存根；职位、重要性、合作阶段等判断字段不会自动猜。",
+                        "candidates": [f"建立{label}", "暂不建立"],
+                    })
+            count += 1
+    con.close()
+    enqueue(recording, "这份草稿整体是否可以作为后续定稿的基础？", {
+        "kind": "draft_overall", "draft_path": str(path),
+        "question_index": count,
+        "sentence": "可展开查看完整草稿；选择“还需修改”时，请在输入框说明修改方向。",
+        "candidates": ["草稿整体可用", "还需修改"],
+    })
+    return count + 1
+
+
+def _frontmatter_value(text: str, key: str) -> str:
+    m = re.match(r"(?s)^---\n(.*?)\n---", text)
+    if not m:
+        return ""
+    km = re.search(rf"(?m)^{re.escape(key)}:\s*(.+?)\s*$", m.group(1))
+    return km.group(1).strip().strip("\"'") if km else ""
+
+
+def _voiceprint_name(value: str) -> str:
+    if value in {"无法确认，保留原文", "不知道", "不认识"}:
+        return ""
+    name = re.split(r"[，,；;（(\s]", value.strip(), maxsplit=1)[0]
+    return name if 2 <= len(name) <= 12 and not name.startswith("speaker_") else ""
+
+
+def _confirm_voiceprint(draft_path: str, speaker_id: str, value: str) -> bool:
+    """Persist an explicit speaker identity and immediately enroll its session."""
+    name = _voiceprint_name(value)
+    if not name:
+        return False
+    path = Path(draft_path).resolve()
+    try:
+        path.relative_to(CFG.drafts.resolve())
+    except ValueError:
+        return False
+    text = path.read_text(encoding="utf-8", errors="ignore")
+    session_id = _frontmatter_value(text, "voiceprint_session")
+    if not session_id:
+        return False
+    fm = re.match(r"(?s)^---\n(.*?)\n---", text)
+    if not fm:
+        return False
+    block = fm.group(1)
+    speaker_line = rf"(?m)^(\s+{re.escape(speaker_id)}:)\s*.*$"
+    if re.search(speaker_line, block):
+        block = re.sub(speaker_line, rf"\1 {name}", block)
+    elif re.search(r"(?m)^voiceprint_speakers:\s*$", block):
+        block = re.sub(
+            r"(?m)^(voiceprint_speakers:\s*)$",
+            rf"\1\n  {speaker_id}: {name}", block, count=1)
+    else:
+        block += f"\nvoiceprint_speakers:\n  {speaker_id}: {name}"
+    if re.search(r"(?m)^voiceprint_enroll:", block):
+        block = re.sub(r"(?m)^voiceprint_enroll:.*$",
+                       "voiceprint_enroll: true", block)
+    else:
+        block += "\nvoiceprint_enroll: true"
+    text = text[:fm.start()] + "---\n" + block + "\n---" + text[fm.end():]
+    from . import vault
+    vault.write_draft_file(path, text)
+
+    body = json.dumps({
+        "session_id": session_id, "speaker_id": int(speaker_id.split("_")[1]),
+        "name": name,
+    }).encode()
+    req = urllib.request.Request(
+        "http://192.168.50.100:10099/voiceprints/enroll", data=body,
+        headers={"Content-Type": "application/json"})
+    try:
+        with urllib.request.urlopen(req, timeout=20) as response:
+            result = json.load(response)
+        return result.get("status") == "enrolled"
+    except Exception:
+        return False
+
+
+def _persist_draft_feedback(recording: str, draft_path: str) -> bool:
+    """Once a draft's web questions are complete, append an auditable answer block."""
+    path = Path(draft_path).resolve()
+    try:
+        path.relative_to(CFG.drafts.resolve())
+    except ValueError:
+        return False
+    con = lexicon.connect()
+    rows = con.execute(
+        "SELECT * FROM review_question WHERE recording=? ORDER BY created_at,id",
+        (recording,)).fetchall()
+    con.close()
+    selected = []
+    for row in rows:
+        payload = json.loads(row["payload"] or "{}")
+        if payload.get("draft_path") == str(path):
+            if row["status"] == "pending":
+                return False
+            selected.append((row["question"], row["answer"] or ""))
+    if not selected or not path.exists():
+        return False
+    lines = [FEEDBACK_START, "## Web 人工确认记录", ""]
+    for i, (question, answer_value) in enumerate(selected, 1):
+        lines.extend([f"{i}. **问题**：{question}", f"   - **回答**：{answer_value}"])
+    lines.extend(["", FEEDBACK_END])
+    block = "\n".join(lines)
+    text = path.read_text(encoding="utf-8", errors="ignore")
+    pattern = re.compile(
+        re.escape(FEEDBACK_START) + r".*?" + re.escape(FEEDBACK_END), re.S)
+    text = pattern.sub(block, text) if pattern.search(text) else text.rstrip() + "\n\n" + block + "\n"
+    from . import vault
+    vault.write_draft_file(path, text)
+    return True
+
+
+def _request_revision(draft_path: str) -> bool:
+    queue_dir = os.environ.get("ASR_REVISION_QUEUE_DIR", "")
+    if queue_dir:
+        try:
+            path = Path(draft_path).resolve()
+            text = path.read_text(encoding="utf-8", errors="ignore")
+            job = _frontmatter_value(text, "intake_job")
+            queue = Path(queue_dir)
+            queue.mkdir(parents=True, exist_ok=True)
+            target = queue / f"{path.stem}.json"
+            tmp = queue / f".{path.stem}.{os.getpid()}.tmp"
+            tmp.write_text(json.dumps({"draft": str(path), "job_id": job},
+                                      ensure_ascii=False), encoding="utf-8")
+            tmp.replace(target)
+            return True
+        except Exception:
+            return False
+    name = Path(draft_path).name
+    try:
+        escaped = subprocess.run(
+            ["systemd-escape", name], capture_output=True, text=True,
+            timeout=5, check=True).stdout.strip()
+        result = subprocess.run(
+            ["systemctl", "--user", "start", "--no-block",
+             f"asr-review-revise@{escaped}.service"],
+            capture_output=True, text=True, timeout=10)
+        return result.returncode == 0
+    except Exception:
+        return False
+
+
+def answer(question_id: int, value: str) -> dict:
+    con = lexicon.connect()
+    row = con.execute("SELECT * FROM review_question WHERE id=?",
+                      (question_id,)).fetchone()
+    if not row:
+        con.close()
+        raise KeyError(f"review question {question_id} not found")
+    if row["status"] != "pending":
+        result = {
+            "id": question_id, "answer": row["answer"], "learned": False,
+            "already_answered": True,
+        }
+        con.close()
+        return result
+    payload = json.loads(row["payload"] or "{}")
+    recording = row["recording"]
+    con.execute(
+        "UPDATE review_question SET status='answered',answer=?,answered_at=? WHERE id=?",
+        (value, time.time(), question_id))
+    con.commit()
+    con.close()
+
+    # A positive answer to an entity choice becomes reusable deterministic memory.
+    source = payload.get("surface", "")
+    candidates = payload.get("candidates") or []
+    canonical = value if value in candidates else ""
+    if source and canonical:
+        lexicon.learn_alias(source, canonical, origin="review", confirmed=1,
+                            note=f"review question {question_id}")
+        lexicon.learn_correction(source, canonical, confirmed=1,
+                                 note=f"review question {question_id}")
+        lexicon.record_observation(source, canonical, source=row["recording"],
+                                   context=payload.get("sentence", ""),
+                                   confidence=1.0, confirmed=1)
+    saved = False
+    voiceprint_enrolled = False
+    if payload.get("kind") == "voiceprint":
+        voiceprint_enrolled = _confirm_voiceprint(
+            payload.get("draft_path", ""), payload.get("speaker_id", ""), value)
+    if payload.get("draft_path"):
+        try:
+            saved = _persist_draft_feedback(recording, payload["draft_path"])
+        except Exception as e:
+            # The answer is already durable in SQLite. A transient git/filesystem
+            # failure must not turn a successful mobile submission into a retry.
+            from .log import log
+            log(f"review answer saved but draft feedback sync failed: {e}",
+                level="WARN")
+            saved = False
+    revision_started = saved and _request_revision(payload["draft_path"])
+    if revision_started:
+        try:
+            text = Path(payload["draft_path"]).read_text(
+                encoding="utf-8", errors="ignore")
+            job = _frontmatter_value(text, "intake_job")
+            if job:
+                from .control_web import update_job
+                if (payload.get("kind") == "publish_approval"
+                        and value == "发布正式稿"):
+                    update_job(job, "publishing", "发布已确认，正在写入正式纪要")
+                else:
+                    update_job(job, "refining", "正在应用回答并生成定稿预览")
+        except Exception:
+            pass
+    return {
+        "id": question_id, "answer": value, "learned": bool(canonical),
+        "draft_feedback_saved": saved, "voiceprint_enrolled": voiceprint_enrolled,
+        "revision_started": revision_started,
+    }