deploy standalone ASR pipeline on tools-pve

servers/tools-pve/asr-pipeline/asrservice/__init__.py

@@ -0,0 +1 @@
+"""Standalone ASR service for web, Agent API, and SSH drop-folder intake."""

servers/tools-pve/asr-pipeline/asrservice/config.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+DATA = Path(os.getenv("ASR_DATA_DIR", "/data"))
+INBOX = DATA / "inbox"
+PROCESSING = DATA / "processing"
+PROCESSED = DATA / "processed"
+FAILED = DATA / "failed"
+TRANSCRIPTS = DATA / "transcripts"
+STATE = Path(os.getenv("ASR_STATE_DIR", "/state"))
+DRAFTS = DATA / "artifacts" / "drafts"
+RESULTS = DATA / "artifacts" / "results"
+LOGS = DATA / "logs"
+META = STATE / "uploads"
+REVISION_QUEUE = STATE / "revision-queue"
+DB = STATE / "intake.db"
+SUPPORTED = {".m4a", ".mp3", ".wav", ".mp4", ".aac", ".ogg", ".opus", ".webm", ".flac"}
+PUBLIC_URL = os.getenv("ASR_PUBLIC_URL", "http://192.168.50.49:8788").rstrip("/")
+AGENT_TOKEN = os.getenv("ASR_AGENT_TOKEN", "")
+DIARIZE_URL = os.getenv("ASR_DIARIZE_URL", "http://192.168.50.100:10099/asr")
+MAX_UPLOAD = int(os.getenv("ASR_UPLOAD_MAX_BYTES", str(4 * 1024**3)))
+POLL_SECONDS = max(1, int(os.getenv("ASR_POLL_SECONDS", "5")))
+STABLE_SECONDS = max(1, int(os.getenv("ASR_STABLE_SECONDS", "10")))
+MAX_ATTEMPTS = max(1, int(os.getenv("ASR_MAX_ATTEMPTS", "3")))
+
+
+def ensure_dirs() -> None:
+    for path in (INBOX, PROCESSING, PROCESSED, FAILED, TRANSCRIPTS, DRAFTS,
+                 RESULTS, STATE, LOGS, META, REVISION_QUEUE, DATA / "cache",
+                 DATA / "artifacts" / "raw", DATA / "vault" / "minutes",
+                 DATA / "vault" / "People", DATA / "vault" / "Companies"):
+        path.mkdir(parents=True, exist_ok=True)

servers/tools-pve/asr-pipeline/asrservice/db.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+import time
+from typing import Any
+
+from .config import DB, ensure_dirs
+
+STATUSES = {"uploading", "queued", "transcribing", "refining", "review",
+            "publishing", "complete", "failed"}
+
+
+def connect() -> sqlite3.Connection:
+    ensure_dirs()
+    con = sqlite3.connect(DB, timeout=30)
+    con.row_factory = sqlite3.Row
+    con.execute("PRAGMA journal_mode=WAL")
+    con.execute("PRAGMA foreign_keys=ON")
+    con.executescript("""
+      CREATE TABLE IF NOT EXISTS intake_job(
+        id TEXT PRIMARY KEY, original_name TEXT NOT NULL, stored_name TEXT NOT NULL,
+        title TEXT NOT NULL DEFAULT '', context TEXT NOT NULL DEFAULT '',
+        size INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL,
+        detail TEXT NOT NULL DEFAULT '', transcript TEXT NOT NULL DEFAULT '',
+        draft TEXT NOT NULL DEFAULT '', result TEXT NOT NULL DEFAULT '',
+        source TEXT NOT NULL DEFAULT 'web', sha256 TEXT NOT NULL DEFAULT '',
+        attempts INTEGER NOT NULL DEFAULT 0, error_code TEXT NOT NULL DEFAULT '',
+        created_at REAL NOT NULL, updated_at REAL NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS job_event(
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        job_id TEXT NOT NULL REFERENCES intake_job(id) ON DELETE CASCADE,
+        status TEXT NOT NULL, detail TEXT NOT NULL DEFAULT '', created_at REAL NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS ix_job_event_job ON job_event(job_id,id);
+    """)
+    cols = {r[1] for r in con.execute("PRAGMA table_info(intake_job)")}
+    additions = {
+        "draft": "TEXT NOT NULL DEFAULT ''", "result": "TEXT NOT NULL DEFAULT ''",
+        "source": "TEXT NOT NULL DEFAULT 'web'", "sha256": "TEXT NOT NULL DEFAULT ''",
+        "attempts": "INTEGER NOT NULL DEFAULT 0", "error_code": "TEXT NOT NULL DEFAULT ''",
+    }
+    for name, ddl in additions.items():
+        if name not in cols:
+            con.execute(f"ALTER TABLE intake_job ADD COLUMN {name} {ddl}")
+    con.commit()
+    return con
+
+
+def create_job(job_id: str, original_name: str, stored_name: str, *, title: str = "",
+               context: str = "", size: int = 0, source: str = "web") -> dict[str, Any]:
+    now = time.time()
+    with connect() as con:
+        con.execute("INSERT INTO intake_job(id,original_name,stored_name,title,context,size,"
+                    "status,source,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?)",
+                    (job_id, original_name, stored_name, title, context, size,
+                     "queued", source, now, now))
+        con.execute("INSERT INTO job_event(job_id,status,detail,created_at) VALUES(?,?,?,?)",
+                    (job_id, "queued", "录音已进入队列", now))
+    return get_job(job_id) or {}
+
+
+def update_job(job_id: str, status: str, detail: str = "", **fields: Any) -> bool:
+    if status not in STATUSES:
+        raise ValueError(f"invalid status: {status}")
+    allowed = {"transcript", "draft", "result", "sha256", "attempts", "error_code"}
+    changes = {k: v for k, v in fields.items() if k in allowed and v is not None}
+    sets = ["status=?", "updated_at=?", "detail=?"]
+    values: list[Any] = [status, time.time(), detail[:1000]]
+    for key, value in changes.items():
+        sets.append(f"{key}=?")
+        values.append(value)
+    values.append(job_id)
+    with connect() as con:
+        cur = con.execute(f"UPDATE intake_job SET {','.join(sets)} WHERE id=?", values)
+        if cur.rowcount:
+            con.execute("INSERT INTO job_event(job_id,status,detail,created_at) VALUES(?,?,?,?)",
+                        (job_id, status, detail[:1000], time.time()))
+    return bool(cur.rowcount)
+
+
+def get_job(job_id: str) -> dict[str, Any] | None:
+    with connect() as con:
+        row = con.execute("SELECT * FROM intake_job WHERE id=?", (job_id,)).fetchone()
+        return dict(row) if row else None
+
+
+def find_by_stored(stored_name: str) -> dict[str, Any] | None:
+    with connect() as con:
+        row = con.execute("SELECT * FROM intake_job WHERE stored_name=? ORDER BY created_at DESC LIMIT 1",
+                          (stored_name,)).fetchone()
+        return dict(row) if row else None
+
+
+def list_jobs(limit: int = 30, offset: int = 0, status: str = "") -> list[dict[str, Any]]:
+    limit = min(max(limit, 1), 200)
+    offset = max(offset, 0)
+    sql, args = "SELECT * FROM intake_job", []
+    if status:
+        sql += " WHERE status=?"
+        args.append(status)
+    sql += " ORDER BY created_at DESC LIMIT ? OFFSET ?"
+    args.extend((limit, offset))
+    with connect() as con:
+        return [dict(r) for r in con.execute(sql, args).fetchall()]
+
+
+def events(job_id: str) -> list[dict[str, Any]]:
+    with connect() as con:
+        return [dict(r) for r in con.execute(
+            "SELECT status,detail,created_at FROM job_event WHERE job_id=? ORDER BY id",
+            (job_id,)).fetchall()]
+
+
+def pending_recovery() -> list[dict[str, Any]]:
+    with connect() as con:
+        return [dict(r) for r in con.execute(
+            "SELECT * FROM intake_job WHERE status IN ('queued','transcribing','refining','publishing') "
+            "ORDER BY created_at").fetchall()]

servers/tools-pve/asr-pipeline/asrservice/healthcheck.py

@@ -0,0 +1,5 @@
+import time
+from .config import STATE
+
+heartbeat = STATE / "worker.heartbeat"
+raise SystemExit(0 if heartbeat.exists() and time.time() - heartbeat.stat().st_mtime < 90 else 1)

servers/tools-pve/asr-pipeline/asrservice/notify.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import os
+from urllib import parse, request
+
+
+def send(status: str, filename: str = "", detail: str = "") -> bool:
+    backend = os.getenv("ASR_NOTIFY_BACKEND", "none").strip().lower()
+    try:
+        if backend == "telegram":
+            token = os.getenv("TELEGRAM_BOT_TOKEN", "")
+            chat = os.getenv("TELEGRAM_CHAT_ID", "")
+            if not token or not chat:
+                return False
+            body = parse.urlencode({"chat_id": chat,
+                                    "text": "\n".join(x for x in (status, filename, detail) if x)}).encode()
+            req = request.Request(f"https://api.telegram.org/bot{token}/sendMessage",
+                                  data=body, method="POST")
+        elif backend == "hermes":
+            url = os.getenv("HERMES_WEBHOOK_URL", "")
+            secret = os.getenv("HERMES_WEBHOOK_SECRET", "")
+            if not url or not secret:
+                return False
+            body = json.dumps({"status": status, "filename": filename, "detail": detail},
+                              ensure_ascii=False).encode()
+            sig = hmac.new(secret.encode(), body, hashlib.sha256).hexdigest()
+            req = request.Request(url, data=body, method="POST",
+                                  headers={"Content-Type": "application/json",
+                                           "X-Webhook-Signature": sig})
+        else:
+            return False
+        with request.urlopen(req, timeout=10) as response:
+            return 200 <= response.status < 300
+    except Exception:
+        return False

servers/tools-pve/asr-pipeline/asrservice/web.py

@@ -0,0 +1,178 @@
+from __future__ import annotations
+
+import hashlib, hmac, html, json, os, re, time, urllib.parse, uuid
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+from asrpipe import review
+from . import db
+from .config import AGENT_TOKEN, DRAFTS, FAILED, INBOX, MAX_UPLOAD, PUBLIC_URL, RESULTS, STATE, SUPPORTED, TRANSCRIPTS, ensure_dirs
+
+LABELS = {"queued":"等待处理","transcribing":"正在转写","refining":"正在整理","review":"等待确认","publishing":"正在定稿","complete":"已完成","failed":"处理失败"}
+
+def safe(value, limit):
+    return re.sub(r"\s+", " ", re.sub(r"[\x00-\x1f\x7f]+", " ", value or "")).strip()[:limit]
+
+def stem(value):
+    return re.sub(r"[^\w\u3400-\u9fff.-]+", "-", safe(value, 80)).strip("-.") or "录音"
+
+def public(row, detailed=False):
+    out = {k: row.get(k) for k in ("id","original_name","title","size","status","detail","source","sha256","attempts","error_code","created_at","updated_at")}
+    out["status_label"] = LABELS.get(row["status"], row["status"])
+    out["web_url"] = f"{PUBLIC_URL}/jobs/{row['id']}/"
+    out["review_url"] = f"{PUBLIC_URL}/jobs/{row['id']}/review/"
+    out["artifacts"] = {name:f"/api/v1/jobs/{row['id']}/artifacts/{name}" for name in ("transcript","draft","result") if row.get(name)}
+    if detailed:
+        out["context"] = row.get("context", "")
+        out["events"] = db.events(row["id"])
+    return out
+
+def pending(job):
+    found=[]
+    for row in review.pending(1000):
+        payload=json.loads(row["payload"] or "{}")
+        if ((job.get("transcript") and row["recording"]==job["transcript"])
+                or (job.get("draft") and payload.get("draft_path")==job["draft"])):
+            found.append(row)
+    return found
+
+def shell(title, body, script=""):
+    return f'''<!doctype html><html lang=zh-CN><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>{html.escape(title)}</title><style>
+:root{{--bg:#f4f6f8;--card:#fff;--ink:#18212b;--muted:#647180;--line:#d9dfe6;--accent:#176b5b;--bad:#b33a2b}}@media(prefers-color-scheme:dark){{:root{{--bg:#11161b;--card:#192128;--ink:#eef3f5;--muted:#a8b3bd;--line:#34414b;--accent:#63c7ae}}}}*{{box-sizing:border-box}}body{{margin:0;background:var(--bg);color:var(--ink);font:16px/1.55 system-ui,sans-serif}}header{{background:var(--card);border-bottom:1px solid var(--line)}}header div,main{{max-width:900px;margin:auto;padding:16px}}header div,.row{{display:flex;justify-content:space-between;gap:12px}}a{{color:var(--accent);text-decoration:none}}nav a{{margin-left:16px}}.card{{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:18px;margin:15px 0}}.job{{padding:13px 0;border-top:1px solid var(--line)}}.job:first-child{{border:0}}.muted{{color:var(--muted);font-size:14px}}.badge{{font-weight:700;color:var(--accent)}}.failed{{color:var(--bad)}}input,textarea,button{{font:inherit}}input,textarea{{width:100%;padding:10px;border:1px solid var(--line);border-radius:7px;background:var(--card);color:var(--ink)}}label{{display:block;margin:12px 0 5px;font-weight:650}}button{{border:0;border-radius:7px;padding:11px 16px;background:var(--accent);color:white;font-weight:700}}button.alt{{background:#68747f}}progress{{width:100%}}pre{{white-space:pre-wrap;overflow-wrap:anywhere;max-height:55vh;overflow:auto}}.timeline{{border-left:2px solid var(--line);padding-left:16px}}.choice{{display:block;width:100%;margin:7px 0}}@media(max-width:520px){{.row{{display:block}}}}</style></head><body><header><div><strong>ASR 录音工作台</strong><nav><a href=/>任务</a><a href=/docs>Agent API</a></nav></div></header><main>{body}</main>{script}</body></html>'''.encode()
+
+def home():
+    body='''<section class=card><h1>上传录音</h1><p class=muted>也可通过 SSH 上传到 tools-pve:/srv/asr/inbox，完成后自动处理。</p><form id=up><label>录音文件</label><input id=file type=file required><label>标题</label><input id=title maxlength=80><label>已知背景</label><textarea id=context maxlength=2000></textarea><p><button id=send>上传并处理</button></p><progress id=bar max=100 hidden></progress><div id=msg class=muted></div></form></section><section class=card><div class=row><h2>最近任务</h2><span class=muted>自动刷新</span></div><div id=jobs>读取中…</div></section>'''
+    script='''<script>const E=s=>String(s??'').replace(/[&<>"']/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));async function load(){let d=await(await fetch('/ui/jobs')).json();jobs.innerHTML=d.jobs.length?d.jobs.map(j=>`<div class=job><div class=row><strong>${E(j.title||j.original_name)}</strong><span class="badge ${E(j.status)}">${E(j.status_label)}</span></div><div class=muted>${new Date(j.created_at*1000).toLocaleString()} · ${E(j.source)} · ${E(j.detail)}</div><a href="/jobs/${E(j.id)}/">查看任务</a></div>`).join(''):'暂无任务'}load();setInterval(load,8000);up.onsubmit=x=>{x.preventDefault();let f=file.files[0],q=new XMLHttpRequest();send.disabled=true;bar.hidden=false;msg.textContent='上传中…';q.open('POST','/ui/upload');q.setRequestHeader('X-Filename',encodeURIComponent(f.name));q.setRequestHeader('X-Title',encodeURIComponent(title.value));q.setRequestHeader('X-Context',encodeURIComponent(context.value));q.upload.onprogress=v=>{if(v.lengthComputable)bar.value=v.loaded/v.total*100};q.onload=()=>{send.disabled=false;bar.hidden=true;msg.textContent=q.status===201?'已进入队列':'上传失败';if(q.status===201){x.target.reset();load()}};q.send(f)};</script>'''
+    return shell("ASR 录音工作台",body,script)
+
+def job_page(job):
+    events=''.join(f"<p><strong>{html.escape(LABELS.get(e['status'],e['status']))}</strong><br><span class=muted>{time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(e['created_at']))} · {html.escape(e['detail'])}</span></p>" for e in db.events(job["id"]))
+    links=' · '.join(f'<a href="/jobs/{job["id"]}/artifact/{n}">{label}</a>' for n,label in (("transcript","转写"),("draft","草稿"),("result","正式稿")) if job.get(n))
+    retry=f'<form method=post action="/jobs/{job["id"]}/retry"><button>重新处理</button></form>' if job["status"]=="failed" else ""
+    review_link=f'<p><a href="/jobs/{job["id"]}/review/">继续人工确认</a></p>' if pending(job) else ""
+    return shell("任务详情",f"<section class=card><div class=row><h1>{html.escape(job.get('title') or job['original_name'])}</h1><span class='badge {job['status']}'>{html.escape(LABELS.get(job['status'],job['status']))}</span></div><p>{html.escape(job.get('detail',''))}</p><p>{links}</p>{review_link}{retry}</section><section class=card><h2>处理时间线</h2><div class=timeline>{events}</div></section>")
+
+def review_page(job):
+    rows=pending(job)
+    if not rows: return shell("人工确认","<section class=card><h1>当前没有待确认问题</h1><p><a href=../>返回任务</a></p></section>")
+    row=rows[0]; payload=json.loads(row["payload"] or "{}"); choices=payload.get("candidates") or []
+    buttons=''.join(f'<button class=choice name=answer value="{html.escape(str(v),quote=True)}">{html.escape(str(v))}</button>' for v in choices)+'<button class="choice alt" name=answer value="无法确认，保留原文">无法确认 / 保留原文</button>'
+    draft=""; path=Path(str(payload.get("draft_path") or ""))
+    try: path.resolve().relative_to(DRAFTS.resolve()); draft=path.read_text(encoding="utf-8")
+    except (OSError,ValueError): pass
+    return shell("人工确认",f"<section class=card><p class=muted>剩余 {len(rows)} 个问题</p><h1>{html.escape(row['question'])}</h1><p>{html.escape(str(payload.get('sentence') or ''))}</p><details><summary>查看草稿</summary><pre>{html.escape(draft)}</pre></details><form method=post><input type=hidden name=id value={row['id']}>{buttons}<label>自定义回答</label><input name=custom maxlength=120><p><button>提交</button></p></form></section>")
+
+class Handler(BaseHTTPRequestHandler):
+    server_version="ASRService/1"
+    def log_message(self,*_): pass
+    def send_data(self,data,status=200,ctype="text/html; charset=utf-8",disposition=""):
+        self.send_response(status); self.send_header("Content-Type",ctype); self.send_header("Cache-Control","no-store"); self.send_header("X-Content-Type-Options","nosniff"); self.send_header("Content-Security-Policy","default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'")
+        if disposition:self.send_header("Content-Disposition",disposition)
+        self.send_header("Content-Length",str(len(data)));self.end_headers();self.wfile.write(data)
+    def json(self,obj,status=200):self.send_data(json.dumps(obj,ensure_ascii=False).encode(),status,"application/json; charset=utf-8")
+    def auth(self):
+        value=self.headers.get("Authorization","")
+        return bool(AGENT_TOKEN and value.startswith("Bearer ") and hmac.compare_digest(value[7:],AGENT_TOKEN))
+    def need_auth(self):
+        if self.auth():return True
+        self.json({"error":"unauthorized"},401);return False
+    def header(self,name,limit):return safe(urllib.parse.unquote(self.headers.get(name,"")),limit)
+    def upload(self,source,protected):
+        if protected and not self.auth():return self.json({"error":"unauthorized"},401)
+        try:length=int(self.headers.get("Content-Length","0"))
+        except ValueError:length=0
+        name=self.header("X-Filename",240);suffix=Path(name).suffix.lower()
+        if length<=0 or length>MAX_UPLOAD:return self.json({"error":"invalid size"},413)
+        if suffix not in SUPPORTED:return self.json({"error":"unsupported type"},415)
+        jid=uuid.uuid4().hex[:24]; title=self.header("X-Title",80); stored=f"{time.strftime('%Y-%m-%d_%H_%M_%S')}-{stem(title or Path(name).stem)}-{jid[:8]}{suffix}"; staging=INBOX/".uploading";staging.mkdir(parents=True,exist_ok=True);part=staging/f"{jid}.part";remaining=length;digest=hashlib.sha256()
+        try:
+            with part.open("xb") as out:
+                while remaining:
+                    chunk=self.rfile.read(min(1024*1024,remaining))
+                    if not chunk:raise EOFError
+                    out.write(chunk);digest.update(chunk);remaining-=len(chunk)
+                out.flush();os.fsync(out.fileno())
+            job=db.create_job(jid,name,stored,title=title,context=self.header("X-Context",2000),size=length,source=source);db.update_job(jid,"queued","录音已落盘，等待 worker",sha256=digest.hexdigest());part.replace(INBOX/stored);self.json(public(job),201)
+        except Exception:part.unlink(missing_ok=True);self.json({"error":"upload failed"},500)
+    def artifact(self,job,name,api=False):
+        if not job:return self.json({"error":"not found"},404)
+        roots={"transcript":TRANSCRIPTS,"draft":DRAFTS,"result":RESULTS};path=Path(job.get(name) or "")
+        try:path.resolve().relative_to(roots[name].resolve());data=path.read_bytes()
+        except (OSError,ValueError):return self.json({"error":"not found"},404)
+        self.send_data(data,200,"text/markdown; charset=utf-8",f'inline; filename="{path.name}"')
+    def do_GET(self):
+        path=urllib.parse.urlsplit(self.path).path
+        if path in ("/health","/healthz"):return self.json({"ok":True,"service":"asr-web"})
+        if path=="/readyz":
+            try:db.connect().close();ok=(STATE/"worker.heartbeat").exists() and time.time()-(STATE/"worker.heartbeat").stat().st_mtime<90
+            except Exception:ok=False
+            return self.json({"ready":ok},200 if ok else 503)
+        if path=="/":return self.send_data(home())
+        if path=="/docs":return self.send_data(shell("Agent API","<section class=card><h1>Agent API</h1><p>使用 Bearer Token。POST /api/v1/jobs 的 body 是原始录音，设置 X-Filename、X-Title、X-Context；GET /api/v1/jobs 与 /api/v1/jobs/{id} 查询任务。</p></section>"))
+        if path=="/ui/jobs":return self.json({"jobs":[public(x) for x in db.list_jobs()]})
+        m=re.fullmatch(r"/jobs/([0-9a-f]{24})/",path)
+        if m:
+            job=db.get_job(m.group(1));return self.send_data(job_page(job)) if job else self.send_data(b"Not found",404,"text/plain")
+        m=re.fullmatch(r"/jobs/([0-9a-f]{24})/review/",path)
+        if m:
+            job=db.get_job(m.group(1));return self.send_data(review_page(job)) if job else self.send_data(b"Not found",404,"text/plain")
+        m=re.fullmatch(r"/jobs/([0-9a-f]{24})/artifact/(transcript|draft|result)",path)
+        if m:return self.artifact(db.get_job(m.group(1)),m.group(2))
+        if path=="/api/v1/jobs":
+            if not self.need_auth():return
+            q=urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query);return self.json({"jobs":[public(x) for x in db.list_jobs(int(q.get('limit',[30])[0]),int(q.get('offset',[0])[0]),q.get('status',[''])[0])]})
+        m=re.fullmatch(r"/api/v1/jobs/([0-9a-f]{24})",path)
+        if m:
+            if not self.need_auth():return
+            job=db.get_job(m.group(1));return self.json(public(job,True)) if job else self.json({"error":"not found"},404)
+        m=re.fullmatch(r"/api/v1/jobs/([0-9a-f]{24})/artifacts/(transcript|draft|result)",path)
+        if m:
+            if not self.need_auth():return
+            return self.artifact(db.get_job(m.group(1)),m.group(2),True)
+        m=re.fullmatch(r"/api/v1/jobs/([0-9a-f]{24})/reviews",path)
+        if m:
+            if not self.need_auth():return
+            job=db.get_job(m.group(1))
+            if not job:return self.json({"error":"not found"},404)
+            rows=[]
+            for row in pending(job):
+                payload=json.loads(row["payload"] or "{}")
+                rows.append({"id":row["id"],"question":row["question"],
+                             "kind":payload.get("kind",""),
+                             "sentence":payload.get("sentence",""),
+                             "candidates":payload.get("candidates") or []})
+            return self.json({"reviews":rows})
+        self.send_data(b"Not found",404,"text/plain")
+    def do_POST(self):
+        path=urllib.parse.urlsplit(self.path).path
+        if path=="/ui/upload":return self.upload("web",False)
+        if path=="/api/v1/jobs":return self.upload("agent",True)
+        m=re.fullmatch(r"/jobs/([0-9a-f]{24})/review/",path)
+        if m:
+            job=db.get_job(m.group(1));length=min(int(self.headers.get("Content-Length","0")),4096);form=urllib.parse.parse_qs(self.rfile.read(length).decode());qid=int(form.get("id",["0"])[0]);answer=(form.get("custom",[""])[0].strip() or form.get("answer",[""])[0].strip())
+            if not job or qid not in {x["id"] for x in pending(job)} or not answer:return self.send_data(b"Invalid answer",400,"text/plain")
+            review.answer(qid,answer[:120]);self.send_response(303);self.send_header("Location",path);self.end_headers();return
+        m=re.fullmatch(r"/(?:api/v1/)?jobs/([0-9a-f]{24})/retry",path)
+        if m:
+            api=path.startswith("/api/")
+            if api and not self.need_auth():return
+            job=db.get_job(m.group(1));source=next((p for p in FAILED.iterdir() if job and p.name==job["stored_name"]),None)
+            if not job or job["status"]!="failed" or not source:return self.json({"error":"cannot retry"},409)
+            source.replace(INBOX/source.name);db.update_job(job["id"],"queued","重新排队",error_code="")
+            if api:return self.json({"ok":True},202)
+            self.send_response(303);self.send_header("Location",f"/jobs/{job['id']}/");self.end_headers();return
+        m=re.fullmatch(r"/api/v1/jobs/([0-9a-f]{24})/reviews/(\d+)",path)
+        if m:
+            if not self.need_auth():return
+            job=db.get_job(m.group(1));length=min(int(self.headers.get("Content-Length","0")),4096)
+            try:answer=str(json.loads(self.rfile.read(length)).get("answer") or "").strip()
+            except (ValueError,AttributeError):answer=""
+            qid=int(m.group(2))
+            if not job or qid not in {x["id"] for x in pending(job)} or not answer:
+                return self.json({"error":"invalid review answer"},400)
+            return self.json(review.answer(qid,answer[:120]))
+        self.send_data(b"Not found",404,"text/plain")
+
+def main():
+    ensure_dirs();db.connect().close();server=ThreadingHTTPServer(("0.0.0.0",8788),Handler);server.daemon_threads=True;server.serve_forever()
+if __name__=="__main__":main()

servers/tools-pve/asr-pipeline/asrservice/worker.py

@@ -0,0 +1,205 @@
+from __future__ import annotations
+
+import fcntl
+import hashlib
+import json
+import os
+import shutil
+import sqlite3
+import subprocess
+import time
+import uuid
+from pathlib import Path
+
+from asrpipe import refine, review, revise, vault
+from . import db
+from .config import (DIARIZE_URL, DRAFTS, FAILED, INBOX, MAX_ATTEMPTS, META,
+                     POLL_SECONDS, PROCESSED, PROCESSING, PUBLIC_URL,
+                     REVISION_QUEUE, STABLE_SECONDS, STATE, SUPPORTED, TRANSCRIPTS, LOGS,
+                     ensure_dirs)
+from .notify import send
+
+
+def log(message: str) -> None:
+    ensure_dirs()
+    with (LOGS / "worker.log").open("a", encoding="utf-8") as stream:
+        stream.write(time.strftime("[%Y-%m-%d %H:%M:%S] ") + message + "\n")
+
+
+def sha256(path: Path) -> str:
+    h = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def transcript_from_response(payload: dict) -> str:
+    segments = payload.get("segments")
+    if not isinstance(segments, list) or not segments:
+        raise RuntimeError("ASR response contains no segments")
+    vp = payload.get("voiceprint") or {}
+    sid = str(vp.get("session_id") or "")
+    matched = [f"{m.get('speaker_local')}={m.get('person_name')}" for m in vp.get("matches", [])
+               if m.get("status") == "matched"]
+    speakers = {str(s.get("speaker")) for s in segments if s.get("speaker") is not None}
+    lines = [f"<!-- voiceprint_session: {sid} clusters: {len(speakers)} -->",
+             "# 转写(Qwen3-ASR 1.7B + CAM++ 分角色 + 声纹)"
+             + (" | 声纹命名: " + ", ".join(matched) if matched else ""), ""]
+    turns: list[list[str]] = []
+    for segment in segments:
+        speaker, text = str(segment.get("speaker", "?")), str(segment.get("text") or "").strip()
+        if not text:
+            continue
+        if turns and turns[-1][0] == speaker:
+            turns[-1][1] += text
+        else:
+            turns.append([speaker, text])
+    lines.extend(f"【{speaker}】{text}\n" for speaker, text in turns)
+    return "\n".join(lines).strip() + "\n"
+
+
+def transcribe(audio: Path, job_id: str) -> Path:
+    response_path = TRANSCRIPTS / f"{job_id}.asr.json"
+    result = subprocess.run([
+        "curl", "-sS", "--fail-with-body", "--max-time", "900",
+        "-F", f"file=@{audio}", "-F", "language=Chinese",
+        "-F", "save_voiceprint_session=true", DIARIZE_URL,
+    ], capture_output=True, timeout=930)
+    if result.returncode:
+        raise RuntimeError(f"ASR HTTP failed ({result.returncode})")
+    response_path.write_bytes(result.stdout)
+    try:
+        payload = json.loads(result.stdout)
+    except ValueError as exc:
+        raise RuntimeError("ASR returned invalid JSON") from exc
+    transcript = TRANSCRIPTS / f"{job_id}.txt"
+    transcript.write_text(transcript_from_response(payload), encoding="utf-8")
+    return transcript
+
+
+def process_audio(source: Path, job: dict) -> None:
+    job_id = job["id"]
+    attempts = int(job.get("attempts") or 0) + 1
+    digest = sha256(source)
+    db.update_job(job_id, "transcribing", "正在进行分角色转写", sha256=digest,
+                  attempts=attempts, error_code="")
+    transcript = transcribe(source, job_id)
+    intake = {"job_id": job_id, "title": job.get("title", ""),
+              "context": job.get("context", ""), "source": job.get("source", "")}
+    raw = transcript.read_text(encoding="utf-8")
+    raw = f"<!-- intake: {json.dumps(intake, ensure_ascii=False)} -->\n" + raw
+    transcript.write_text(raw, encoding="utf-8")
+    db.update_job(job_id, "refining", "转写完成，正在生成整理稿", transcript=str(transcript))
+    result = refine.refine(raw, date=time.strftime("%Y-%m-%d"),
+                           transcript_path=str(transcript), queue_review=True)
+    draft = DRAFTS / f"{job_id}.md"
+    vault.write_draft_file(draft, result.draft)
+    questions = review.enqueue_draft(str(transcript), draft)
+    db.update_job(job_id, "review", f"整理稿已生成，{questions} 个问题待确认",
+                  transcript=str(transcript), draft=str(draft))
+    target = PROCESSED / source.name
+    if target.exists():
+        target = PROCESSED / f"{source.stem}-{job_id[:8]}{source.suffix}"
+    shutil.move(str(source), target)
+    send("ASR 草稿待确认", job.get("title") or job["original_name"],
+         f"{PUBLIC_URL}/jobs/{job_id}/review/")
+
+
+def claim(path: Path) -> tuple[Path, dict]:
+    job = db.find_by_stored(path.name)
+    if not job:
+        job_id = uuid.uuid4().hex[:24]
+        job = db.create_job(job_id, path.name, path.name, title=path.stem,
+                            size=path.stat().st_size, source="ssh")
+    target = PROCESSING / path.name
+    if target.exists():
+        target = PROCESSING / f"{path.stem}-{job['id'][:8]}{path.suffix}"
+    path.replace(target)
+    return target, job
+
+
+def scan_inbox() -> None:
+    now = time.time()
+    for path in sorted(INBOX.iterdir()):
+        if not path.is_file() or path.suffix.lower() not in SUPPORTED:
+            continue
+        if now - path.stat().st_mtime < STABLE_SECONDS:
+            continue
+        source, job = claim(path)
+        try:
+            process_audio(source, job)
+        except Exception as exc:
+            attempts = int((db.get_job(job["id"]) or {}).get("attempts") or 0)
+            retry = attempts < MAX_ATTEMPTS
+            db.update_job(job["id"], "queued" if retry else "failed",
+                          "处理失败，等待重试" if retry else "处理失败",
+                          error_code=type(exc).__name__)
+            log(f"job={job['id']} failed type={type(exc).__name__}")
+            target = INBOX / source.name if retry else FAILED / source.name
+            if source.exists():
+                shutil.move(str(source), target)
+
+
+def process_revisions() -> None:
+    for marker in sorted(REVISION_QUEUE.glob("*.json")):
+        try:
+            payload = json.loads(marker.read_text(encoding="utf-8"))
+            draft = Path(payload["draft"])
+            job_id = str(payload.get("job_id") or "")
+            db.update_job(job_id, "publishing", "正在应用人工确认")
+            revise.revise(draft)
+            current = db.get_job(job_id) or {}
+            if current.get("status") == "complete":
+                send("ASR 正式记录已生成", current.get("title") or current.get("original_name", ""),
+                     f"{PUBLIC_URL}/jobs/{job_id}/")
+            elif current.get("status") != "review":
+                db.update_job(job_id, "review", "定稿预览已生成，等待发布确认", draft=str(draft))
+            marker.unlink(missing_ok=True)
+        except Exception as exc:
+            log(f"revision marker={marker.name} failed type={type(exc).__name__}")
+
+
+def recover() -> None:
+    for job in db.pending_recovery():
+        if job["status"] in {"transcribing", "refining"}:
+            candidate = next((p for p in PROCESSING.iterdir() if p.name == job["stored_name"]), None)
+            if candidate:
+                shutil.move(str(candidate), INBOX / candidate.name)
+                db.update_job(job["id"], "queued", "worker 重启后重新排队")
+
+
+def backup_databases() -> None:
+    # STATE is local disk; durable copies belong on the NFS data volume.
+    backup_dir = Path("/data/backups")
+    backup_dir.mkdir(parents=True, exist_ok=True)
+    for name in ("intake.db", "lexicon.db"):
+        source = STATE / name
+        if not source.exists():
+            continue
+        target = backup_dir / f"{name}.latest"
+        temp = backup_dir / f".{name}.{os.getpid()}.tmp"
+        with sqlite3.connect(source) as src, sqlite3.connect(temp) as dst:
+            src.backup(dst)
+        temp.replace(target)
+
+
+def main() -> None:
+    ensure_dirs()
+    lock = (STATE / "worker.lock").open("w")
+    fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    recover()
+    (STATE / "worker.heartbeat").write_text(str(time.time()))
+    last_backup = 0.0
+    while True:
+        scan_inbox()
+        process_revisions()
+        if time.time() - last_backup >= 3600:
+            backup_databases()
+            last_backup = time.time()
+        (STATE / "worker.heartbeat").write_text(str(time.time()))
+        time.sleep(POLL_SECONDS)
+
+
+if __name__ == "__main__":
+    main()