deploy standalone ASR pipeline on tools-pve
This commit is contained in:
Codex
@@ -0,0 +1,382 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Standalone CLI. No Claude Code, no opencode, no agent runtime.
|
||||
|
||||
asr-refine run-all # process every pending transcript
|
||||
asr-refine run <file.txt> # process one transcript
|
||||
asr-refine dry-run <file.txt> # full pipeline, write nothing to the vault
|
||||
asr-refine doctor # check credentials, services, term bank
|
||||
asr-refine build-termbank # refresh the domain term bank (network)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import filecmp
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from .config import CFG
|
||||
from .log import log
|
||||
|
||||
_LOCK_HANDLE = None
|
||||
|
||||
|
||||
def _lock():
|
||||
"""Single instance. The systemd path unit can fire several times in quick
|
||||
succession when a batch of transcripts lands, and two concurrent runs would
|
||||
race on git and on the pending index."""
|
||||
d = Path(os.environ.get("XDG_RUNTIME_DIR", "/tmp"))
|
||||
global _LOCK_HANDLE
|
||||
f = (d / "asr-pipeline.lock").open("w")
|
||||
try:
|
||||
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
log("another run holds the lock; exiting")
|
||||
sys.exit(0)
|
||||
_LOCK_HANDLE = f
|
||||
return f
|
||||
|
||||
|
||||
def resolve_archive(slug: str, inbox_path: Path) -> str:
|
||||
"""Prefer the persistent transcript archive over the `.asr-inbox/` staging path.
|
||||
|
||||
The inbox copy is deleted on 定稿, so a note whose `transcript:` points there
|
||||
ends up dangling. `recordings/transcripts/<fid>-<YYYY-MM-DD_HH_MM_SS>.txt`
|
||||
survives, so map the slug's timestamp back onto it when possible.
|
||||
"""
|
||||
m = re.match(r"^(\d{4}-\d{2}-\d{2})-(\d{2})(\d{2})(\d{2})", slug)
|
||||
if m:
|
||||
d, hh, mm, ss = m.groups()
|
||||
want = f"{d}_{hh}_{mm}_{ss}.txt"
|
||||
arch = CFG.obsidian / "recordings" / "transcripts"
|
||||
if arch.is_dir():
|
||||
hits = sorted(arch.glob(f"*{want[:-4]}*.txt"))
|
||||
if hits:
|
||||
if len(hits) > 1:
|
||||
for hit in hits:
|
||||
try:
|
||||
if (hit.stat().st_size == inbox_path.stat().st_size
|
||||
and filecmp.cmp(hit, inbox_path,
|
||||
shallow=False)):
|
||||
return str(hit)
|
||||
except OSError:
|
||||
continue
|
||||
return str(hits[0])
|
||||
return str(inbox_path)
|
||||
|
||||
|
||||
def process(path: Path, *, dry_run=False) -> int:
|
||||
from . import refine as R
|
||||
from . import vault
|
||||
|
||||
slug = path.stem
|
||||
if not dry_run and vault.is_done(slug):
|
||||
log(f"skip (already processed): {path}")
|
||||
return 0
|
||||
raw = path.read_text(errors="ignore")
|
||||
if not raw.strip():
|
||||
log(f"skip (empty): {path}", level="WARN")
|
||||
return 0
|
||||
intake_job = ""
|
||||
m_intake = re.search(r"(?m)^<!-- intake:\s*(\{.*\})\s*-->$", raw)
|
||||
if m_intake:
|
||||
try:
|
||||
intake_job = str(json.loads(m_intake.group(1)).get("job_id") or "")
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
date, title = vault.slug_parts(slug)
|
||||
dest = vault.draft_path(date, title) if not dry_run else Path("/dev/null")
|
||||
transcript_ref = resolve_archive(slug, path)
|
||||
if not dry_run:
|
||||
existing = vault.find_draft_for_transcript(transcript_ref)
|
||||
if existing:
|
||||
log(f"recover interrupted run: {existing}")
|
||||
from . import review, review_web
|
||||
review.enqueue_draft(transcript_ref, existing)
|
||||
if intake_job:
|
||||
try:
|
||||
from .control_web import update_job
|
||||
update_job(intake_job, "review",
|
||||
"已恢复整理稿，等待确认", draft=str(existing))
|
||||
except Exception as e:
|
||||
log(f"intake recovery status failed: {e}", level="WARN")
|
||||
if not review_web.request_start(intake_job):
|
||||
vault.notify("ASR Web 审核启动失败",
|
||||
f"{existing.name} 已恢复，但 Web 服务启动失败。")
|
||||
return 1
|
||||
vault.mark_done(slug)
|
||||
return 0
|
||||
log(f"refining {path.name} ({len(raw)} chars) -> "
|
||||
f"{dest if not dry_run else '(dry-run)'} "
|
||||
f"clean={CFG.clean_model} draft={CFG.draft_model}")
|
||||
|
||||
t0 = time.time()
|
||||
try:
|
||||
res = R.refine(raw, date=date, transcript_path=transcript_ref,
|
||||
queue_review=not dry_run)
|
||||
except Exception as e:
|
||||
log(f"FAILED {path.name}: {e} (left unprocessed for retry)", level="ERROR")
|
||||
return 1
|
||||
|
||||
f = res.stats["fidelity"]
|
||||
log(f"done {path.name} in {time.time()-t0:.0f}s | "
|
||||
f"chunks={res.stats['n_chunks']} flags={res.stats['n_flags']} "
|
||||
f"hs={res.stats['n_hindsight']} "
|
||||
f"gate_fail={res.stats['gate_failed']} retried={res.stats['retried']} "
|
||||
f"fallback={res.stats['fallback']} | "
|
||||
f"ratio={f['char_ratio']} num={f['number_recall']} "
|
||||
f"latin={f['latin_recall']} term={f['term_recall']} | "
|
||||
f"${res.usage['cost_usd']} calls={res.usage['calls']}")
|
||||
|
||||
if dry_run:
|
||||
out = CFG.root / "dry-run" / slug
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
(out / "cleaned.txt").write_text(res.cleaned, encoding="utf-8")
|
||||
if res.evidence:
|
||||
(out / "evidence.md").write_text(res.evidence, encoding="utf-8")
|
||||
(out / "draft.md").write_text(res.draft, encoding="utf-8")
|
||||
(out / "report.json").write_text(json.dumps(
|
||||
{"stats": res.stats, "usage": res.usage, "timings": res.timings},
|
||||
ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(f"dry-run output -> {out}")
|
||||
return 0
|
||||
|
||||
vault.write_draft_file(dest, res.draft)
|
||||
from . import review, review_web
|
||||
draft_questions = review.enqueue_draft(transcript_ref, dest)
|
||||
if intake_job:
|
||||
try:
|
||||
from .control_web import update_job
|
||||
update_job(intake_job, "review", "整理稿已生成，等待确认",
|
||||
draft=str(dest))
|
||||
except Exception as e:
|
||||
log(f"intake status update failed: {e}", level="WARN")
|
||||
if not review_web.request_start(intake_job):
|
||||
vault.notify("ASR Web 审核启动失败",
|
||||
f"{date} {title} 的私有粗稿已保留；"
|
||||
f"{draft_questions} 个问题可用 "
|
||||
"asr-refine review pending 查看")
|
||||
return 1
|
||||
vault.mark_done(slug)
|
||||
log(f"staged private draft {dest} | questions={draft_questions}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_run(a):
|
||||
_lock()
|
||||
return process(Path(a.file).expanduser(), dry_run=False)
|
||||
|
||||
|
||||
def cmd_dry_run(a):
|
||||
return process(Path(a.file).expanduser(), dry_run=True)
|
||||
|
||||
|
||||
def cmd_run_all(a):
|
||||
_lock()
|
||||
from . import vault
|
||||
todo = vault.pending_transcripts()
|
||||
if not todo:
|
||||
log(f"no pending transcripts in {CFG.inbox}")
|
||||
return 0
|
||||
log(f"{len(todo)} pending transcript(s)")
|
||||
rc = 0
|
||||
for p in todo:
|
||||
rc |= process(p)
|
||||
return rc
|
||||
|
||||
|
||||
def cmd_doctor(a):
|
||||
from . import fidelity
|
||||
ok = True
|
||||
|
||||
def chk(name, good, detail=""):
|
||||
nonlocal ok
|
||||
ok &= bool(good)
|
||||
print(f" [{'OK ' if good else 'FAIL'}] {name}" + (f" {detail}" if detail else ""))
|
||||
|
||||
print("paths")
|
||||
chk("vault", CFG.vault.is_dir(), str(CFG.vault))
|
||||
chk("inbox", CFG.inbox.is_dir(), str(CFG.inbox))
|
||||
chk("drafts dir writable", os.access(CFG.drafts.parent, os.W_OK), str(CFG.drafts))
|
||||
chk("glossary", CFG.glossary.exists(), str(CFG.glossary))
|
||||
chk("notifier", CFG.notifier.exists(), str(CFG.notifier))
|
||||
|
||||
print("credentials")
|
||||
try:
|
||||
k = CFG.api_key()
|
||||
chk("api key", len(k) > 20, f"len={len(k)} source={'env' if os.environ.get('ANTHROPIC_AUTH_TOKEN') else CFG.env_file.name}")
|
||||
except Exception as e:
|
||||
chk("api key", False, str(e))
|
||||
|
||||
print("lexicon")
|
||||
try:
|
||||
from . import lexicon
|
||||
lexicon.sync_from_vault()
|
||||
st = lexicon.stats()
|
||||
chk("entities", st["entities"] > 50,
|
||||
f"{st['entities']} ({st['with_company']} 有归属, {st['with_related']} 有关系)")
|
||||
chk("aliases", st["aliases"] > 50,
|
||||
f"{st['aliases']} = {st['aliases_card']} 卡片 + {st['aliases_learned']} 学习/术语表")
|
||||
except Exception as e:
|
||||
chk("lexicon", False, str(e))
|
||||
|
||||
print("term bank")
|
||||
n = len(fidelity.domain_terms())
|
||||
chk("terms.json", n > 1000, f"{n} terms")
|
||||
chk("allowlist", (CFG.termbank_dir / "allowlist.txt").exists(),
|
||||
f"{len((CFG.termbank_dir/'allowlist.txt').read_text().split()) if (CFG.termbank_dir/'allowlist.txt').exists() else 0} words")
|
||||
|
||||
print("services")
|
||||
import urllib.request
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
f"{CFG.hindsight_api}/health", timeout=6) as r:
|
||||
chk("hindsight", r.status == 200, CFG.hindsight_api)
|
||||
except Exception as e:
|
||||
chk("hindsight (optional)", False, f"{CFG.hindsight_api}: {e}")
|
||||
try:
|
||||
from .llm import chat
|
||||
t0 = time.time()
|
||||
out = chat(CFG.clean_model, "回一个字:好", max_tokens=20)
|
||||
chk(f"clean model {CFG.clean_model}", bool(out), f"{time.time()-t0:.1f}s {out[:8]!r}")
|
||||
t0 = time.time()
|
||||
out = chat(CFG.draft_model, "回一个字:好", max_tokens=20)
|
||||
chk(f"draft model {CFG.draft_model}", bool(out), f"{time.time()-t0:.1f}s {out[:8]!r}")
|
||||
except Exception as e:
|
||||
chk("model endpoint", False, str(e))
|
||||
|
||||
print("pending")
|
||||
from . import vault
|
||||
todo = vault.pending_transcripts()
|
||||
print(f" {len(todo)} transcript(s) awaiting processing")
|
||||
for p in todo[:5]:
|
||||
print(f" {p.name}")
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def cmd_build_termbank(a):
|
||||
from . import termbank
|
||||
termbank.build()
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_lexicon(a):
|
||||
"""Entity/alias lexicon. Obsidian stays read-only; learned bindings live here."""
|
||||
from . import lexicon
|
||||
act = a.action
|
||||
if act == "sync":
|
||||
print(lexicon.sync_from_vault(force=True))
|
||||
elif act == "stats":
|
||||
for k, v in lexicon.stats().items():
|
||||
print(f" {k:18s} {v}")
|
||||
elif act == "learn":
|
||||
if not (a.alias and a.canonical):
|
||||
print("usage: lexicon learn <别名> <规范名> [--note …]", file=sys.stderr)
|
||||
return 2
|
||||
lexicon.learn_alias(a.alias, a.canonical, note=a.note or "")
|
||||
lexicon.learn_correction(a.alias, a.canonical, note=a.note or "")
|
||||
print(f"learned: {a.alias} -> {a.canonical}"
|
||||
+ (f" ({a.note})" if a.note else ""))
|
||||
elif act == "show":
|
||||
am = lexicon.alias_map()
|
||||
key = a.alias
|
||||
if key:
|
||||
print(f"{key} -> {am.get(key) or 'unknown'}")
|
||||
else:
|
||||
learned = [(k, v) for k, v in am.items()
|
||||
if any(o not in ("card",) for _, _, o, _ in v)]
|
||||
print(f"{len(am)} aliases total, {len(learned)} beyond the cards")
|
||||
for k, v in sorted(learned)[:40]:
|
||||
print(f" {k:10s} -> {'、'.join(c for c, _, _, _ in v)}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_review(a):
|
||||
from . import review
|
||||
if a.action == "serve":
|
||||
from . import review_web
|
||||
return review_web.serve()
|
||||
if a.action == "state":
|
||||
from .review_web import STATE
|
||||
if not STATE.exists():
|
||||
print("inactive")
|
||||
return 1
|
||||
print(STATE.read_text(encoding="utf-8"))
|
||||
return 0
|
||||
if a.action == "pending":
|
||||
rows = review.pending(a.limit)
|
||||
for r in rows:
|
||||
payload = json.loads(r["payload"] or "{}")
|
||||
print(f"#{r['id']} [{r['recording']}] {r['question']}")
|
||||
if payload.get("sentence"):
|
||||
print(f" {payload['sentence'][:180]}")
|
||||
return 0
|
||||
if a.id is None or not a.answer:
|
||||
print("usage: asr-refine review answer <id> <answer>", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
result = review.answer(a.id, a.answer)
|
||||
except KeyError as e:
|
||||
print(str(e), file=sys.stderr)
|
||||
return 2
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_graph(a):
|
||||
from . import graph
|
||||
result = graph.sync(force=a.force) if a.action == "sync" else graph.stats()
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_revise(a):
|
||||
from . import revise
|
||||
path = (CFG.drafts / a.draft).resolve()
|
||||
try:
|
||||
path.relative_to(CFG.drafts.resolve())
|
||||
except ValueError:
|
||||
print("draft outside review root", file=sys.stderr)
|
||||
return 2
|
||||
return 0 if revise.revise(path) else 1
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(prog="asr-refine",
|
||||
description="Standalone ASR transcript -> Obsidian draft")
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
p = sub.add_parser("run"); p.add_argument("file"); p.set_defaults(fn=cmd_run)
|
||||
p = sub.add_parser("dry-run"); p.add_argument("file"); p.set_defaults(fn=cmd_dry_run)
|
||||
sub.add_parser("run-all").set_defaults(fn=cmd_run_all)
|
||||
sub.add_parser("doctor").set_defaults(fn=cmd_doctor)
|
||||
sub.add_parser("build-termbank").set_defaults(fn=cmd_build_termbank)
|
||||
p = sub.add_parser("lexicon", help="entity/alias lexicon (SQLite; vault is read-only)")
|
||||
p.add_argument("action", choices=["sync", "stats", "learn", "show"])
|
||||
p.add_argument("alias", nargs="?")
|
||||
p.add_argument("canonical", nargs="?")
|
||||
p.add_argument("--note", default="")
|
||||
p.set_defaults(fn=cmd_lexicon)
|
||||
p = sub.add_parser("review", help="human review queue (transport independent)")
|
||||
p.add_argument("action", choices=["pending", "answer", "serve", "state"])
|
||||
p.add_argument("id", nargs="?", type=int)
|
||||
p.add_argument("answer", nargs="?")
|
||||
p.add_argument("--limit", type=int, default=20)
|
||||
p.set_defaults(fn=cmd_review)
|
||||
p = sub.add_parser("graph", help="incremental entity/relation/recency graph")
|
||||
p.add_argument("action", choices=["sync", "stats"])
|
||||
p.add_argument("--force", action="store_true")
|
||||
p.set_defaults(fn=cmd_graph)
|
||||
p = sub.add_parser("revise", help="publish a Web-confirmed private draft")
|
||||
p.add_argument("draft", help="draft filename under minutes/_drafts")
|
||||
p.set_defaults(fn=cmd_revise)
|
||||
a = ap.parse_args(argv)
|
||||
return a.fn(a)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user