309 lines
11 KiB
Python
309 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sqlite3
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
from asrpipe import graph, lexicon, providers, refine, review, revise, vault
|
|
from asrpipe.config import CFG
|
|
from . import db
|
|
from .config import (
|
|
DATA, DRAFTS, FAILED, INBOX, LOGS, MAX_ATTEMPTS,
|
|
POLL_SECONDS, PROCESSED, PROCESSING, PUBLIC_URL, REVISION_QUEUE,
|
|
STABLE_SECONDS, STATE, SUPPORTED, TRANSCRIPTS, ensure_dirs,
|
|
)
|
|
from .notify import send
|
|
|
|
|
|
def log(message: str) -> None:
|
|
ensure_dirs()
|
|
with (LOGS / "worker.log").open("a", encoding="utf-8") as stream:
|
|
stream.write(time.strftime("[%Y-%m-%d %H:%M:%S] ") + message + "\n")
|
|
|
|
|
|
def sha256(path: Path) -> str:
|
|
h = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def transcript_from_response(payload: dict) -> str:
|
|
"""Backward-compatible helper for qwen3cam JSON responses."""
|
|
return providers.transcript_from_qwen3cam_response(payload)
|
|
|
|
|
|
def transcribe(
|
|
audio: Path,
|
|
job_id: str,
|
|
*,
|
|
provider: str = "",
|
|
model: str = "",
|
|
title: str = "",
|
|
user_context: str = "",
|
|
) -> tuple[Path, providers.TranscribeResult]:
|
|
res = providers.transcribe_audio(
|
|
audio,
|
|
provider=provider or CFG.asr_provider,
|
|
model=model,
|
|
title=title,
|
|
user_context=user_context,
|
|
)
|
|
response_path = TRANSCRIPTS / f"{job_id}.asr.json"
|
|
try:
|
|
response_path.write_text(
|
|
json.dumps(res.raw_response, ensure_ascii=False, indent=2),
|
|
encoding="utf-8",
|
|
)
|
|
except Exception:
|
|
pass
|
|
transcript = TRANSCRIPTS / f"{job_id}.txt"
|
|
transcript.write_text(res.transcript_text, encoding="utf-8")
|
|
return transcript, res
|
|
|
|
|
|
def process_audio(source: Path, job: dict) -> None:
|
|
job_id = job["id"]
|
|
attempts = int(job.get("attempts") or 0) + 1
|
|
prov = providers.normalize_provider_name(job.get("asr_provider") or CFG.asr_provider)
|
|
model = str(job.get("asr_model") or "")
|
|
digest = sha256(source)
|
|
prov_label = providers.PROVIDERS.get(prov, {}).get("label", prov)
|
|
db.update_job(
|
|
job_id, "transcribing", f"正在转写 ({prov_label})",
|
|
sha256=digest, attempts=attempts, error_code="",
|
|
asr_provider=prov, asr_model=model,
|
|
)
|
|
transcript, res = transcribe(
|
|
source, job_id,
|
|
provider=prov, model=model,
|
|
title=str(job.get("title") or ""),
|
|
user_context=str(job.get("context") or ""),
|
|
)
|
|
intake = {
|
|
"job_id": job_id,
|
|
"title": job.get("title", ""),
|
|
"context": job.get("context", ""),
|
|
"source": job.get("source", ""),
|
|
"asr_provider": res.provider,
|
|
"asr_model": res.model,
|
|
}
|
|
raw = transcript.read_text(encoding="utf-8")
|
|
raw = f"<!-- intake: {json.dumps(intake, ensure_ascii=False)} -->\n" + raw
|
|
transcript.write_text(raw, encoding="utf-8")
|
|
db.update_job(
|
|
job_id, "refining", f"转写完成 ({res.provider}:{res.model})，正在生成整理稿",
|
|
transcript=str(transcript), asr_provider=res.provider, asr_model=res.model,
|
|
)
|
|
result = refine.refine(
|
|
raw, date=time.strftime("%Y-%m-%d"),
|
|
transcript_path=str(transcript), queue_review=True,
|
|
)
|
|
draft = DRAFTS / f"{job_id}.md"
|
|
vault.write_draft_file(draft, result.draft)
|
|
questions = review.enqueue_draft(str(transcript), draft)
|
|
db.update_job(
|
|
job_id, "review", f"整理稿已生成，{questions} 个问题待确认",
|
|
transcript=str(transcript), draft=str(draft),
|
|
)
|
|
target = PROCESSED / source.name
|
|
if target.exists():
|
|
target = PROCESSED / f"{source.stem}-{job_id[:8]}{source.suffix}"
|
|
shutil.move(str(source), target)
|
|
send("ASR 草稿待确认", job.get("title") or job["original_name"],
|
|
f"{PUBLIC_URL}/jobs/{job_id}/review/")
|
|
|
|
|
|
def rerefine_job(job: dict) -> None:
|
|
"""Re-run LLM refinement on an existing raw transcript without re-running ASR."""
|
|
job_id = job["id"]
|
|
transcript = Path(str(job.get("transcript") or ""))
|
|
if not transcript.is_file():
|
|
raise FileNotFoundError(f"transcript file missing for job {job_id}: {transcript}")
|
|
raw = transcript.read_text(encoding="utf-8", errors="ignore")
|
|
intake = {
|
|
"job_id": job_id,
|
|
"title": job.get("title", ""),
|
|
"context": job.get("context", ""),
|
|
"source": job.get("source", ""),
|
|
"asr_provider": job.get("asr_provider") or CFG.asr_provider,
|
|
"asr_model": job.get("asr_model") or "",
|
|
}
|
|
intake_line = f"<!-- intake: {json.dumps(intake, ensure_ascii=False)} -->"
|
|
if re.match(r"^<!--\s*intake:\s*\{.*?\}\s*-->", raw):
|
|
raw = re.sub(r"^<!--\s*intake:\s*\{.*?\}\s*-->", intake_line, raw, count=1)
|
|
else:
|
|
raw = intake_line + "\n" + raw
|
|
transcript.write_text(raw, encoding="utf-8")
|
|
|
|
db.update_job(job_id, "refining", "正在基于现有转写重新生成精修草稿")
|
|
review.delete_for_job(
|
|
transcript_path=str(transcript),
|
|
draft_path=str(job.get("draft") or ""),
|
|
job_id=job_id,
|
|
)
|
|
result = refine.refine(
|
|
raw, date=time.strftime("%Y-%m-%d"),
|
|
transcript_path=str(transcript), queue_review=True,
|
|
)
|
|
draft = DRAFTS / f"{job_id}.md"
|
|
vault.write_draft_file(draft, result.draft)
|
|
questions = review.enqueue_draft(str(transcript), draft)
|
|
db.update_job(
|
|
job_id, "review", f"整理稿已重新生成，{questions} 个问题待确认",
|
|
transcript=str(transcript), draft=str(draft),
|
|
)
|
|
|
|
|
|
def claim(path: Path) -> tuple[Path, dict]:
|
|
job = db.find_by_stored(path.name)
|
|
if not job:
|
|
job_id = uuid.uuid4().hex[:24]
|
|
job = db.create_job(
|
|
job_id, path.name, path.name, title=path.stem,
|
|
size=path.stat().st_size, source="ssh",
|
|
asr_provider=CFG.asr_provider,
|
|
)
|
|
target = PROCESSING / path.name
|
|
if target.exists():
|
|
target = PROCESSING / f"{path.stem}-{job['id'][:8]}{path.suffix}"
|
|
path.replace(target)
|
|
return target, job
|
|
|
|
|
|
def scan_inbox() -> None:
|
|
now = time.time()
|
|
for path in sorted(INBOX.iterdir()):
|
|
if not path.is_file() or path.suffix.lower() not in SUPPORTED:
|
|
continue
|
|
if now - path.stat().st_mtime < STABLE_SECONDS:
|
|
continue
|
|
source, job = claim(path)
|
|
try:
|
|
process_audio(source, job)
|
|
except Exception as exc:
|
|
attempts = int((db.get_job(job["id"]) or {}).get("attempts") or 0)
|
|
retry = attempts < MAX_ATTEMPTS
|
|
db.update_job(
|
|
job["id"], "queued" if retry else "failed",
|
|
f"处理失败，等待重试 ({exc})" if retry else f"处理失败：{exc}",
|
|
error_code=type(exc).__name__,
|
|
)
|
|
log(f"job={job['id']} failed type={type(exc).__name__} err={exc}")
|
|
target = INBOX / source.name if retry else FAILED / source.name
|
|
if source.exists():
|
|
shutil.move(str(source), target)
|
|
|
|
|
|
def process_revisions() -> None:
|
|
for marker in sorted(REVISION_QUEUE.glob("*.json")):
|
|
try:
|
|
payload = json.loads(marker.read_text(encoding="utf-8"))
|
|
action = str(payload.get("action") or "revise")
|
|
job_id = str(payload.get("job_id") or "")
|
|
if action == "rerefine":
|
|
job = db.get_job(job_id)
|
|
if job:
|
|
rerefine_job(job)
|
|
marker.unlink(missing_ok=True)
|
|
continue
|
|
draft = Path(payload["draft"])
|
|
db.update_job(job_id, "publishing", "正在应用人工确认")
|
|
revise.revise(draft)
|
|
current = db.get_job(job_id) or {}
|
|
if current.get("status") == "complete":
|
|
send("ASR 正式记录已生成",
|
|
current.get("title") or current.get("original_name", ""),
|
|
f"{PUBLIC_URL}/jobs/{job_id}/")
|
|
elif current.get("status") != "review":
|
|
db.update_job(
|
|
job_id, "review", "定稿预览已生成，等待发布确认",
|
|
draft=str(draft),
|
|
)
|
|
marker.unlink(missing_ok=True)
|
|
except Exception as exc:
|
|
log(f"revision marker={marker.name} failed type={type(exc).__name__} err={exc}")
|
|
try:
|
|
payload = json.loads(marker.read_text(encoding="utf-8"))
|
|
jid = str(payload.get("job_id") or "")
|
|
if jid:
|
|
db.update_job(jid, "failed", f"精修/修订失败：{exc}", error_code=type(exc).__name__)
|
|
except Exception:
|
|
pass
|
|
marker.unlink(missing_ok=True)
|
|
|
|
|
|
def recover() -> None:
|
|
for job in db.pending_recovery():
|
|
if job["status"] in {"transcribing", "refining"}:
|
|
candidate = next(
|
|
(p for p in PROCESSING.iterdir() if p.name == job["stored_name"]),
|
|
None,
|
|
)
|
|
if candidate:
|
|
shutil.move(str(candidate), INBOX / candidate.name)
|
|
db.update_job(job["id"], "queued", "worker 重启后重新排队")
|
|
|
|
|
|
def sync_vault_knowledge() -> None:
|
|
try:
|
|
lexicon.sync_from_vault()
|
|
graph.sync()
|
|
except Exception as exc:
|
|
log(f"vault knowledge sync skipped: {exc}")
|
|
|
|
|
|
def backup_databases() -> None:
|
|
backup_dir = DATA / "backups"
|
|
backup_dir.mkdir(parents=True, exist_ok=True)
|
|
for name in ("intake.db", "lexicon.db"):
|
|
source = STATE / name
|
|
if not source.exists():
|
|
continue
|
|
target = backup_dir / f"{name}.latest"
|
|
temp = backup_dir / f".{name}.{os.getpid()}.tmp"
|
|
with sqlite3.connect(source) as src, sqlite3.connect(temp) as dst:
|
|
src.backup(dst)
|
|
temp.replace(target)
|
|
|
|
|
|
def run_loop() -> None:
|
|
os.environ.setdefault("ASR_ROLE", "worker")
|
|
os.environ.setdefault("ASR_REVISION_QUEUE_DIR", str(REVISION_QUEUE))
|
|
ensure_dirs()
|
|
lock = (STATE / "worker.lock").open("w")
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
recover()
|
|
sync_vault_knowledge()
|
|
(STATE / "worker.heartbeat").write_text(str(time.time()))
|
|
last_backup = 0.0
|
|
last_vault_sync = time.time()
|
|
while True:
|
|
scan_inbox()
|
|
process_revisions()
|
|
now = time.time()
|
|
if now - last_vault_sync >= 900:
|
|
sync_vault_knowledge()
|
|
last_vault_sync = now
|
|
if now - last_backup >= 3600:
|
|
backup_databases()
|
|
last_backup = now
|
|
(STATE / "worker.heartbeat").write_text(str(now))
|
|
time.sleep(POLL_SECONDS)
|
|
|
|
|
|
def main() -> None:
|
|
run_loop()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|