feat(asr-pipeline): merge PR #1 webauth/CSRF/locking hardening and web_password fallback
This commit is contained in:
Codex
@@ -0,0 +1,362 @@
|
||||
#!/usr/bin/env python3
|
||||
import fcntl
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT = Path(__file__).resolve().parents[2]
|
||||
BASE = Path(os.environ.get("MEETING_INBOX_BASE", str(PROJECT / "recordings")))
|
||||
INBOX = BASE / "inbox"
|
||||
PROCESSING = BASE / "processing"
|
||||
PROCESSED = BASE / "processed"
|
||||
FAILED = BASE / "failed"
|
||||
TRANSCRIPTS = BASE / "transcripts"
|
||||
STATE = BASE / "state"
|
||||
LOGS = BASE / "logs"
|
||||
ASR = Path(os.environ.get("MEETING_ASR_SCRIPT", str(PROJECT / "recordings" / "bin" / "asr-transcribe.sh")))
|
||||
SEEN_FILE = STATE / "seen.json"
|
||||
UPLOAD_META = STATE / "uploads"
|
||||
CONTROL_DB = Path(os.environ.get(
|
||||
"ASR_CONTROL_DB",
|
||||
str(Path(os.environ.get("ASR_STATE_DIR", str(PROJECT / "state"))) / "intake.db")))
|
||||
LOG = LOGS / "meeting-watch.log"
|
||||
POLL_SECONDS = int(os.environ.get("MEETING_WATCH_POLL_SECONDS", "30"))
|
||||
STABLE_AGE_SECONDS = int(os.environ.get("MEETING_WATCH_STABLE_AGE_SECONDS", "60"))
|
||||
WEBHOOK_URL = os.environ.get("MEETING_WEBHOOK_URL", "http://127.0.0.1:8644/webhooks/meeting-recording")
|
||||
WEBHOOK_ROUTE = os.environ.get("MEETING_WEBHOOK_ROUTE", "meeting-recording")
|
||||
WEBHOOK_SECRET = os.environ.get("MEETING_WEBHOOK_SECRET", "")
|
||||
STATUS_WEBHOOK_URL = os.environ.get("MEETING_STATUS_WEBHOOK_URL", "http://127.0.0.1:8644/webhooks/meeting-recording-status")
|
||||
STATUS_WEBHOOK_ROUTE = os.environ.get("MEETING_STATUS_WEBHOOK_ROUTE", "meeting-recording-status")
|
||||
SUBSCRIPTIONS = Path.home() / ".hermes" / "webhook_subscriptions.json"
|
||||
SUPPORTED = {".m4a", ".mp3", ".wav", ".mp4", ".aac", ".ogg", ".opus", ".webm", ".flac"}
|
||||
|
||||
|
||||
def log(msg):
|
||||
LOGS.mkdir(parents=True, exist_ok=True)
|
||||
with LOG.open("a", encoding="utf-8") as f:
|
||||
f.write(time.strftime("[%Y-%m-%d %H:%M:%S] ") + msg + "\n")
|
||||
|
||||
|
||||
def load_seen():
|
||||
try:
|
||||
return json.loads(SEEN_FILE.read_text())
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def save_seen(seen):
|
||||
STATE.mkdir(parents=True, exist_ok=True)
|
||||
tmp = SEEN_FILE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(seen, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
tmp.replace(SEEN_FILE)
|
||||
|
||||
|
||||
def upload_meta(filename):
|
||||
try:
|
||||
return json.loads((UPLOAD_META / f"{filename}.json").read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def update_job(meta, status, detail="", transcript=""):
|
||||
job_id = str(meta.get("job_id") or "")
|
||||
if not job_id or not CONTROL_DB.exists():
|
||||
return
|
||||
try:
|
||||
con = sqlite3.connect(CONTROL_DB, timeout=15)
|
||||
fields, vals = ["status=?", "updated_at=?"], [status, time.time()]
|
||||
if detail:
|
||||
fields.append("detail=?"); vals.append(detail[:1000])
|
||||
if transcript:
|
||||
fields.append("transcript=?"); vals.append(transcript)
|
||||
vals.append(job_id)
|
||||
con.execute(f"UPDATE intake_job SET {','.join(fields)} WHERE id=?", vals)
|
||||
con.commit()
|
||||
con.close()
|
||||
except Exception as exc:
|
||||
log(f"CONTROL_STATUS_FAILED job={job_id} status={status} error={exc}")
|
||||
|
||||
|
||||
def ensure_job(path, fid):
|
||||
"""Create a visible control-plane task for files dropped outside Web upload."""
|
||||
job_id = f"fs-{fid}"
|
||||
now = time.time()
|
||||
try:
|
||||
CONTROL_DB.parent.mkdir(parents=True, exist_ok=True)
|
||||
con = sqlite3.connect(CONTROL_DB, timeout=15)
|
||||
con.execute("""
|
||||
CREATE TABLE IF NOT EXISTS intake_job(
|
||||
id TEXT PRIMARY KEY, original_name TEXT NOT NULL,
|
||||
stored_name TEXT NOT NULL, title TEXT NOT NULL DEFAULT '',
|
||||
context TEXT NOT NULL DEFAULT '', size INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL, detail TEXT NOT NULL DEFAULT '',
|
||||
transcript TEXT NOT NULL DEFAULT '', created_at REAL NOT NULL,
|
||||
updated_at REAL NOT NULL, draft TEXT NOT NULL DEFAULT '',
|
||||
result TEXT NOT NULL DEFAULT ''
|
||||
)
|
||||
""")
|
||||
columns = {row[1] for row in con.execute("PRAGMA table_info(intake_job)")}
|
||||
for name in ("draft", "result"):
|
||||
if name not in columns:
|
||||
con.execute(
|
||||
f"ALTER TABLE intake_job ADD COLUMN {name} "
|
||||
"TEXT NOT NULL DEFAULT ''")
|
||||
con.execute(
|
||||
"INSERT OR IGNORE INTO intake_job("
|
||||
"id,original_name,stored_name,title,context,size,status,detail,"
|
||||
"transcript,created_at,updated_at,draft,result"
|
||||
") VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
(job_id, path.name, path.name, path.stem, "", path.stat().st_size,
|
||||
"queued", "监控目录已接收", "", now, now, "", ""))
|
||||
con.commit()
|
||||
con.close()
|
||||
return {
|
||||
"job_id": job_id, "title": path.stem, "context": "",
|
||||
"original_name": path.name, "stored_name": path.name,
|
||||
}
|
||||
except Exception as exc:
|
||||
log(f"CONTROL_JOB_CREATE_FAILED file={path} error={exc}")
|
||||
return {}
|
||||
|
||||
|
||||
def duration_seconds(path):
|
||||
try:
|
||||
out = subprocess.check_output([
|
||||
"ffprobe", "-v", "error", "-show_entries", "format=duration",
|
||||
"-of", "default=nk=1:nw=1", str(path)
|
||||
], text=True, timeout=30).strip()
|
||||
return int(float(out) + 0.999) if out else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def file_id(path):
|
||||
st = path.stat()
|
||||
h = hashlib.sha256()
|
||||
h.update(str(path.name).encode())
|
||||
h.update(str(st.st_size).encode())
|
||||
h.update(str(int(st.st_mtime)).encode())
|
||||
if st.st_size <= 256 * 1024 * 1024:
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()[:24]
|
||||
|
||||
|
||||
def stable(path):
|
||||
try:
|
||||
st1 = path.stat()
|
||||
if time.time() - st1.st_mtime < STABLE_AGE_SECONDS:
|
||||
return False
|
||||
time.sleep(2)
|
||||
st2 = path.stat()
|
||||
return st1.st_size == st2.st_size and int(st1.st_mtime) == int(st2.st_mtime)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def unique_path(directory, name):
|
||||
p = directory / name
|
||||
if not p.exists():
|
||||
return p
|
||||
stem, suffix = p.stem, p.suffix
|
||||
for i in range(1, 10000):
|
||||
q = directory / f"{stem}-{i}{suffix}"
|
||||
if not q.exists():
|
||||
return q
|
||||
raise RuntimeError(f"cannot allocate unique path for {name}")
|
||||
|
||||
|
||||
def webhook_secret(route=WEBHOOK_ROUTE):
|
||||
if route == WEBHOOK_ROUTE and WEBHOOK_SECRET:
|
||||
return WEBHOOK_SECRET
|
||||
try:
|
||||
data = json.loads(SUBSCRIPTIONS.read_text(encoding="utf-8"))
|
||||
return str(data.get(route, {}).get("secret", ""))
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def post_webhook(payload, url=WEBHOOK_URL, route=WEBHOOK_ROUTE, label="WEBHOOK", timeout=30):
|
||||
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||
headers = {"Content-Type": "application/json"}
|
||||
secret = webhook_secret(route)
|
||||
if secret and secret != "INSECURE_NO_AUTH":
|
||||
headers["X-Webhook-Signature"] = hmac.new(secret.encode(), data, hashlib.sha256).hexdigest()
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
headers=headers,
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
body = resp.read(1000).decode("utf-8", errors="ignore")
|
||||
log(f"{label} status={resp.status} body={body[:300]!r}")
|
||||
return resp.status
|
||||
|
||||
|
||||
def post_status(status, filename, detail=""):
|
||||
payload = {"status": status, "filename": filename, "detail": detail}
|
||||
try:
|
||||
return post_webhook(
|
||||
payload,
|
||||
url=STATUS_WEBHOOK_URL,
|
||||
route=STATUS_WEBHOOK_ROUTE,
|
||||
label="STATUS_WEBHOOK",
|
||||
timeout=10,
|
||||
)
|
||||
except Exception as exc:
|
||||
log(f"STATUS_WEBHOOK_FAILED status={status} file={filename} error={exc}")
|
||||
return None
|
||||
|
||||
|
||||
import re
|
||||
|
||||
_CONSUMER_LOCK = None
|
||||
|
||||
ASR_INBOX = Path(os.environ.get("ASR_VAULT", str(Path.home() / "obsidian-vault"))) / ".asr-inbox"
|
||||
|
||||
|
||||
def bridge_to_asr_inbox(transcript, stem, title=""):
|
||||
"""Atomically drop the finished transcript into the Obsidian .asr-inbox so the
|
||||
Claude refine pipeline picks it up. Slug carries date+time for the draft name."""
|
||||
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})[_-](\d{2})[_-](\d{2})[_-](\d{2})", stem)
|
||||
if m:
|
||||
y, mo, d, hh, mm, ss = m.groups()
|
||||
tail = stem[m.end():].strip("_- ")
|
||||
safe_tail = re.sub(
|
||||
r"[^\w一-鿿-]+", "_", title or tail).strip("_-")
|
||||
safe_tail = re.sub(r"-[0-9a-f]{8}$", "", safe_tail)
|
||||
label = safe_tail or "录音"
|
||||
slug = f"{y}-{mo}-{d}-{hh}{mm}{ss}-{label}"
|
||||
else:
|
||||
safe = re.sub(r"[^\w一-鿿-]+", "_", stem).strip("_") or "录音"
|
||||
slug = f"{time.strftime('%Y-%m-%d-%H%M%S')}-{safe}"
|
||||
incoming = ASR_INBOX / ".incoming"
|
||||
incoming.mkdir(parents=True, exist_ok=True)
|
||||
(ASR_INBOX / ".state").mkdir(parents=True, exist_ok=True)
|
||||
# Collision guard: never overwrite an unprocessed transcript.
|
||||
dest = ASR_INBOX / f"{slug}.txt"
|
||||
n = 2
|
||||
while dest.exists():
|
||||
dest = ASR_INBOX / f"{slug}-{n}.txt"
|
||||
n += 1
|
||||
tmp = incoming / dest.name
|
||||
shutil.copyfile(str(transcript), str(tmp))
|
||||
tmp.replace(dest) # atomic within the vault filesystem -> triggers the path unit
|
||||
return dest
|
||||
|
||||
|
||||
def process(path):
|
||||
seen = load_seen()
|
||||
fid = file_id(path)
|
||||
intake = upload_meta(path.name) or ensure_job(path, fid)
|
||||
if fid in seen:
|
||||
log(f"SKIP already seen file={path} id={fid}")
|
||||
return
|
||||
PROCESSING.mkdir(parents=True, exist_ok=True)
|
||||
TRANSCRIPTS.mkdir(parents=True, exist_ok=True)
|
||||
PROCESSED.mkdir(parents=True, exist_ok=True)
|
||||
FAILED.mkdir(parents=True, exist_ok=True)
|
||||
proc = unique_path(PROCESSING, path.name)
|
||||
try:
|
||||
shutil.move(str(path), str(proc))
|
||||
except FileNotFoundError:
|
||||
log(f"SKIP disappeared before claim file={path}")
|
||||
return
|
||||
transcript = TRANSCRIPTS / f"{fid}-{proc.stem}.txt"
|
||||
dur = duration_seconds(proc)
|
||||
log(f"PROCESS file={proc} id={fid} duration={dur}")
|
||||
update_job(intake, "transcribing", f"录音时长约 {dur}s" if dur else "正在转写")
|
||||
post_status("开始转录", proc.name, f"时长约 {dur}s；完成后会继续发送粗整理。")
|
||||
try:
|
||||
subprocess.run([str(ASR), str(proc), str(transcript)], check=True, timeout=None)
|
||||
if intake:
|
||||
original = transcript.read_text(encoding="utf-8", errors="ignore")
|
||||
marker = json.dumps({
|
||||
"job_id": intake.get("job_id", ""),
|
||||
"title": intake.get("title", ""),
|
||||
"context": intake.get("context", ""),
|
||||
}, ensure_ascii=False)
|
||||
transcript.write_text(f"<!-- intake: {marker} -->\n" + original,
|
||||
encoding="utf-8")
|
||||
done = unique_path(PROCESSED, proc.name)
|
||||
shutil.move(str(proc), str(done))
|
||||
post_status("转录完成，正在生成粗整理", done.name, f"转录文件：{transcript}")
|
||||
# Refinement is handled by the Claude Code pipeline (not the Hermes agent
|
||||
# webhook). Bridge the finished transcript into the Obsidian .asr-inbox so
|
||||
# obsidian-asr-refine (systemd path unit) produces a rough-outline draft.
|
||||
bridged = bridge_to_asr_inbox(
|
||||
transcript, proc.stem, str(intake.get("title") or ""))
|
||||
update_job(intake, "refining", "转写完成，正在生成整理稿",
|
||||
transcript=str(transcript))
|
||||
log(f"BRIDGED transcript -> {bridged}")
|
||||
seen[fid] = {"filename": done.name, "audio_path": str(done), "transcript_path": str(transcript), "ts": int(time.time())}
|
||||
save_seen(seen)
|
||||
log(f"DONE file={done} transcript={transcript}")
|
||||
except Exception as exc:
|
||||
fail = unique_path(FAILED, proc.name)
|
||||
if proc.exists():
|
||||
shutil.move(str(proc), str(fail))
|
||||
err = FAILED / f"{fail.name}.error.txt"
|
||||
err.write_text(str(exc), encoding="utf-8")
|
||||
post_status("处理失败", fail.name, f"错误：{exc}")
|
||||
update_job(intake, "failed", f"处理失败：{exc}")
|
||||
log(f"FAILED file={path} error={exc}")
|
||||
|
||||
|
||||
def once():
|
||||
for p in sorted(INBOX.iterdir() if INBOX.exists() else []):
|
||||
if not p.is_file() or p.suffix.lower() not in SUPPORTED:
|
||||
continue
|
||||
if stable(p):
|
||||
process(p)
|
||||
|
||||
|
||||
def hold_consumer_lock():
|
||||
"""Share ASR_STATE_DIR/consumer.lock with asrservice.worker. Exit if it is taken."""
|
||||
path = Path(os.environ.get("ASR_CONSUMER_LOCK", str(CONTROL_DB.parent / "consumer.lock")))
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = path.open("a+")
|
||||
try:
|
||||
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
handle.close()
|
||||
log(f"EXIT another recording consumer holds {path}")
|
||||
raise SystemExit(1)
|
||||
handle.seek(0)
|
||||
handle.truncate()
|
||||
handle.write(str(os.getpid()))
|
||||
handle.flush()
|
||||
return handle
|
||||
|
||||
|
||||
def main():
|
||||
global _CONSUMER_LOCK
|
||||
_CONSUMER_LOCK = hold_consumer_lock()
|
||||
for d in [INBOX, PROCESSING, PROCESSED, FAILED, TRANSCRIPTS, STATE, LOGS]:
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
if not ASR.exists():
|
||||
raise SystemExit(f"missing ASR script: {ASR}")
|
||||
log("START watcher")
|
||||
if "--once" in sys.argv:
|
||||
once()
|
||||
return
|
||||
while True:
|
||||
try:
|
||||
once()
|
||||
except Exception as exc:
|
||||
log(f"LOOP_ERROR {exc}")
|
||||
time.sleep(POLL_SECONDS)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user