feat(asr-pipeline): merge PR #1 webauth/CSRF/locking hardening and web_password fallback

servers/tools-pve/asr-pipeline/asrservice/worker.py

@@ -1,18 +1,18 @@
 from __future__ import annotations
 
-import fcntl
 import hashlib
 import json
 import os
-import re
 import shutil
 import sqlite3
+import threading
 import time
 import uuid
 from pathlib import Path
 
 from asrpipe import graph, lexicon, providers, refine, review, revise, vault
 from asrpipe.config import CFG
+from asrpipe.locking import acquire_recording_lock
 from . import db
 from .config import (
     DATA, DRAFTS, FAILED, INBOX, LOGS, MAX_ATTEMPTS,
@@ -21,6 +21,8 @@ from .config import (
 )
 from .notify import send
 
+_RECORDING_LOCK = None
+
 
 def log(message: str) -> None:
     ensure_dirs()
@@ -70,6 +72,31 @@ def transcribe(
     return transcript, res
 
 
+def intake_header(payload: dict) -> str:
+    blob = json.dumps(payload, ensure_ascii=False).replace("--", "- -")
+    return f"<!-- intake: {blob} -->"
+
+
+def replace_intake(raw: str, header: str) -> str:
+    if raw.startswith("<!-- intake:"):
+        _first, _sep, rest = raw.partition("\n")
+        return header + ("\n" + rest if _sep else "")
+    return header + "\n" + raw
+
+
+def beat() -> None:
+    ensure_dirs()
+    (STATE / "worker.heartbeat").write_text(str(time.time()))
+
+
+def _beat_loop(stop: threading.Event) -> None:
+    while not stop.wait(15):
+        try:
+            beat()
+        except Exception:
+            pass
+
+
 def process_audio(source: Path, job: dict) -> None:
     job_id = job["id"]
     attempts = int(job.get("attempts") or 0) + 1
@@ -97,7 +124,7 @@ def process_audio(source: Path, job: dict) -> None:
         "asr_model": res.model,
     }
     raw = transcript.read_text(encoding="utf-8")
-    raw = f"<!-- intake: {json.dumps(intake, ensure_ascii=False)} -->\n" + raw
+    raw = replace_intake(raw, intake_header(intake))
     transcript.write_text(raw, encoding="utf-8")
     db.update_job(
         job_id, "refining", f"转写完成 ({res.provider}:{res.model})，正在生成整理稿",
@@ -137,25 +164,21 @@ def rerefine_job(job: dict) -> None:
         "asr_provider": job.get("asr_provider") or CFG.asr_provider,
         "asr_model": job.get("asr_model") or "",
     }
-    intake_line = f"<!-- intake: {json.dumps(intake, ensure_ascii=False)} -->"
-    if re.match(r"^<!--\s*intake:\s*\{.*?\}\s*-->", raw):
-        raw = re.sub(r"^<!--\s*intake:\s*\{.*?\}\s*-->", intake_line, raw, count=1)
-    else:
-        raw = intake_line + "\n" + raw
+    raw = replace_intake(raw, intake_header(intake))
     transcript.write_text(raw, encoding="utf-8")
 
     db.update_job(job_id, "refining", "正在基于现有转写重新生成精修草稿")
-    review.delete_for_job(
-        transcript_path=str(transcript),
-        draft_path=str(job.get("draft") or ""),
-        job_id=job_id,
-    )
     result = refine.refine(
         raw, date=time.strftime("%Y-%m-%d"),
         transcript_path=str(transcript), queue_review=True,
     )
     draft = DRAFTS / f"{job_id}.md"
     vault.write_draft_file(draft, result.draft)
+    review.delete_pending_for_job(
+        transcript_path=str(transcript),
+        draft_path=str(job.get("draft") or draft),
+        job_id=job_id,
+    )
     questions = review.enqueue_draft(str(transcript), draft)
     db.update_job(
         job_id, "review", f"整理稿已重新生成，{questions} 个问题待确认",
@@ -176,6 +199,9 @@ def claim(path: Path) -> tuple[Path, dict]:
     if target.exists():
         target = PROCESSING / f"{path.stem}-{job['id'][:8]}{path.suffix}"
     path.replace(target)
+    if target.name != job.get("stored_name"):
+        db.update_job(job["id"], job["status"], job.get("detail") or "已认领", stored_name=target.name)
+        job = db.get_job(job["id"]) or job
     return target, job
 
 
@@ -199,15 +225,54 @@ def scan_inbox() -> None:
                 error_code=type(exc).__name__,
             )
             log(f"job={job['id']} failed type={type(exc).__name__} retry={retry} err={exc}")
-            target = INBOX / source.name if retry else FAILED / source.name
-            if source.exists():
-                shutil.move(str(source), target)
+            try:
+                if source.exists():
+                    folder = INBOX if retry else FAILED
+                    target = folder / source.name
+                    if target.exists():
+                        target = folder / f"{source.stem}-{job['id'][:8]}{source.suffix}"
+                    shutil.move(str(source), target)
+                    if retry:
+                        old = time.time() - max(STABLE_SECONDS, 60)
+                        os.utime(target, (old, old))
+            except Exception as move_exc:
+                log(f"job={job['id']} audio left in processing after failure: {move_exc}")
+
+
+def _defer_revision(marker: Path, payload: dict, exc: BaseException) -> None:
+    attempts = int(payload.get("attempts") or 0) + 1
+    payload["attempts"] = attempts
+    payload["last_error"] = str(exc)[:300]
+    job_id = str(payload.get("job_id") or "")
+    if attempts >= MAX_ATTEMPTS:
+        marker.unlink(missing_ok=True)
+        if job_id:
+            db.update_job(job_id, "failed", f"修订失败：{exc}", error_code=type(exc).__name__)
+        log(f"revision marker={marker.name} gave up after {attempts}: {exc}")
+        return
+    tmp = marker.with_name(f".{marker.name}.{os.getpid()}.tmp")
+    tmp.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
+    tmp.replace(marker)
+    if job_id:
+        db.update_job(
+            job_id, "review",
+            f"修订失败，将重试 ({attempts}/{MAX_ATTEMPTS})：{exc}",
+            error_code=type(exc).__name__,
+        )
+    log(f"revision marker={marker.name} retry {attempts}/{MAX_ATTEMPTS}: {exc}")
 
 
 def process_revisions() -> None:
     for marker in sorted(REVISION_QUEUE.glob("*.json")):
+        if marker.name.startswith("."):
+            continue
         try:
             payload = json.loads(marker.read_text(encoding="utf-8"))
+        except Exception as exc:
+            log(f"revision marker={marker.name} unreadable: {exc}")
+            _defer_revision(marker, {}, exc)
+            continue
+        try:
             action = str(payload.get("action") or "revise")
             job_id = str(payload.get("job_id") or "")
             if action == "rerefine":
@@ -231,27 +296,45 @@ def process_revisions() -> None:
                 )
             marker.unlink(missing_ok=True)
         except Exception as exc:
-            log(f"revision marker={marker.name} failed type={type(exc).__name__} err={exc}")
-            try:
-                payload = json.loads(marker.read_text(encoding="utf-8"))
-                jid = str(payload.get("job_id") or "")
-                if jid:
-                    db.update_job(jid, "failed", f"精修/修订失败：{exc}", error_code=type(exc).__name__)
-            except Exception:
-                pass
-            marker.unlink(missing_ok=True)
+            _defer_revision(marker, payload, exc)
+
+
+def _park_audio(path: Path) -> Path:
+    dest = INBOX / path.name
+    if dest.exists():
+        dest = INBOX / f"{path.stem}-recovered{path.suffix}"
+    shutil.move(str(path), dest)
+    old = time.time() - max(STABLE_SECONDS, 60)
+    os.utime(dest, (old, old))
+    return dest
 
 
 def recover() -> None:
-    for job in db.pending_recovery():
-        if job["status"] in {"transcribing", "refining"}:
-            candidate = next(
-                (p for p in PROCESSING.iterdir() if p.name == job["stored_name"]),
-                None,
+    """Return audio left in processing/ to the inbox and requeue its job."""
+    if not PROCESSING.exists():
+        return
+    files = [p for p in PROCESSING.iterdir() if p.is_file()]
+    jobs = db.pending_recovery()
+    claimed: set[str] = set()
+    for path in files:
+        match = None
+        for job in jobs:
+            if job["id"] in claimed:
+                continue
+            stored = str(job.get("stored_name") or "")
+            if path.name == stored or (job["id"][:8] and job["id"][:8] in path.name):
+                match = job
+                break
+        dest = _park_audio(path)
+        if match:
+            claimed.add(match["id"])
+            db.update_job(
+                match["id"], "queued", "worker 重启后重新排队",
+                stored_name=dest.name, error_code="",
             )
-            if candidate:
-                shutil.move(str(candidate), INBOX / candidate.name)
-                db.update_job(job["id"], "queued", "worker 重启后重新排队")
+            log(f"recovered job={match['id']} audio={dest.name}")
+        else:
+            log(f"recovered orphan audio={dest.name}")
 
 
 def sync_vault_knowledge() -> None:
@@ -281,24 +364,36 @@ def run_loop() -> None:
     os.environ.setdefault("ASR_ROLE", "worker")
     os.environ.setdefault("ASR_REVISION_QUEUE_DIR", str(REVISION_QUEUE))
     ensure_dirs()
-    lock = (STATE / "worker.lock").open("w")
-    fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    global _RECORDING_LOCK
+    lock_path = Path(os.getenv("ASR_CONSUMER_LOCK", str(STATE / "consumer.lock")))
+    _RECORDING_LOCK = acquire_recording_lock(lock_path)
     recover()
     sync_vault_knowledge()
-    (STATE / "worker.heartbeat").write_text(str(time.time()))
+    beat()
+    stop_beat = threading.Event()
+    threading.Thread(target=_beat_loop, args=(stop_beat,), name="asr-heartbeat", daemon=True).start()
     last_backup = 0.0
     last_vault_sync = time.time()
     while True:
-        scan_inbox()
-        process_revisions()
+        try:
+            scan_inbox()
+        except Exception as exc:
+            log(f"scan_inbox failed: {exc}")
+        try:
+            process_revisions()
+        except Exception as exc:
+            log(f"process_revisions failed: {exc}")
         now = time.time()
         if now - last_vault_sync >= 900:
             sync_vault_knowledge()
             last_vault_sync = now
         if now - last_backup >= 3600:
-            backup_databases()
+            try:
+                backup_databases()
+            except Exception as exc:
+                log(f"backup failed: {exc}")
             last_backup = now
-        (STATE / "worker.heartbeat").write_text(str(now))
+        beat()
         time.sleep(POLL_SECONDS)