363 lines
13 KiB
Python
Executable File
363 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import fcntl
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import os
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
PROJECT = Path(__file__).resolve().parents[2]
|
|
BASE = Path(os.environ.get("MEETING_INBOX_BASE", str(PROJECT / "recordings")))
|
|
INBOX = BASE / "inbox"
|
|
PROCESSING = BASE / "processing"
|
|
PROCESSED = BASE / "processed"
|
|
FAILED = BASE / "failed"
|
|
TRANSCRIPTS = BASE / "transcripts"
|
|
STATE = BASE / "state"
|
|
LOGS = BASE / "logs"
|
|
ASR = Path(os.environ.get("MEETING_ASR_SCRIPT", str(PROJECT / "recordings" / "bin" / "asr-transcribe.sh")))
|
|
SEEN_FILE = STATE / "seen.json"
|
|
UPLOAD_META = STATE / "uploads"
|
|
CONTROL_DB = Path(os.environ.get(
|
|
"ASR_CONTROL_DB",
|
|
str(Path(os.environ.get("ASR_STATE_DIR", str(PROJECT / "state"))) / "intake.db")))
|
|
LOG = LOGS / "meeting-watch.log"
|
|
POLL_SECONDS = int(os.environ.get("MEETING_WATCH_POLL_SECONDS", "30"))
|
|
STABLE_AGE_SECONDS = int(os.environ.get("MEETING_WATCH_STABLE_AGE_SECONDS", "60"))
|
|
WEBHOOK_URL = os.environ.get("MEETING_WEBHOOK_URL", "http://127.0.0.1:8644/webhooks/meeting-recording")
|
|
WEBHOOK_ROUTE = os.environ.get("MEETING_WEBHOOK_ROUTE", "meeting-recording")
|
|
WEBHOOK_SECRET = os.environ.get("MEETING_WEBHOOK_SECRET", "")
|
|
STATUS_WEBHOOK_URL = os.environ.get("MEETING_STATUS_WEBHOOK_URL", "http://127.0.0.1:8644/webhooks/meeting-recording-status")
|
|
STATUS_WEBHOOK_ROUTE = os.environ.get("MEETING_STATUS_WEBHOOK_ROUTE", "meeting-recording-status")
|
|
SUBSCRIPTIONS = Path.home() / ".hermes" / "webhook_subscriptions.json"
|
|
SUPPORTED = {".m4a", ".mp3", ".wav", ".mp4", ".aac", ".ogg", ".opus", ".webm", ".flac"}
|
|
|
|
|
|
def log(msg):
|
|
LOGS.mkdir(parents=True, exist_ok=True)
|
|
with LOG.open("a", encoding="utf-8") as f:
|
|
f.write(time.strftime("[%Y-%m-%d %H:%M:%S] ") + msg + "\n")
|
|
|
|
|
|
def load_seen():
|
|
try:
|
|
return json.loads(SEEN_FILE.read_text())
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def save_seen(seen):
|
|
STATE.mkdir(parents=True, exist_ok=True)
|
|
tmp = SEEN_FILE.with_suffix(".tmp")
|
|
tmp.write_text(json.dumps(seen, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
tmp.replace(SEEN_FILE)
|
|
|
|
|
|
def upload_meta(filename):
|
|
try:
|
|
return json.loads((UPLOAD_META / f"{filename}.json").read_text(encoding="utf-8"))
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def update_job(meta, status, detail="", transcript=""):
|
|
job_id = str(meta.get("job_id") or "")
|
|
if not job_id or not CONTROL_DB.exists():
|
|
return
|
|
try:
|
|
con = sqlite3.connect(CONTROL_DB, timeout=15)
|
|
fields, vals = ["status=?", "updated_at=?"], [status, time.time()]
|
|
if detail:
|
|
fields.append("detail=?"); vals.append(detail[:1000])
|
|
if transcript:
|
|
fields.append("transcript=?"); vals.append(transcript)
|
|
vals.append(job_id)
|
|
con.execute(f"UPDATE intake_job SET {','.join(fields)} WHERE id=?", vals)
|
|
con.commit()
|
|
con.close()
|
|
except Exception as exc:
|
|
log(f"CONTROL_STATUS_FAILED job={job_id} status={status} error={exc}")
|
|
|
|
|
|
def ensure_job(path, fid):
|
|
"""Create a visible control-plane task for files dropped outside Web upload."""
|
|
job_id = f"fs-{fid}"
|
|
now = time.time()
|
|
try:
|
|
CONTROL_DB.parent.mkdir(parents=True, exist_ok=True)
|
|
con = sqlite3.connect(CONTROL_DB, timeout=15)
|
|
con.execute("""
|
|
CREATE TABLE IF NOT EXISTS intake_job(
|
|
id TEXT PRIMARY KEY, original_name TEXT NOT NULL,
|
|
stored_name TEXT NOT NULL, title TEXT NOT NULL DEFAULT '',
|
|
context TEXT NOT NULL DEFAULT '', size INTEGER NOT NULL DEFAULT 0,
|
|
status TEXT NOT NULL, detail TEXT NOT NULL DEFAULT '',
|
|
transcript TEXT NOT NULL DEFAULT '', created_at REAL NOT NULL,
|
|
updated_at REAL NOT NULL, draft TEXT NOT NULL DEFAULT '',
|
|
result TEXT NOT NULL DEFAULT ''
|
|
)
|
|
""")
|
|
columns = {row[1] for row in con.execute("PRAGMA table_info(intake_job)")}
|
|
for name in ("draft", "result"):
|
|
if name not in columns:
|
|
con.execute(
|
|
f"ALTER TABLE intake_job ADD COLUMN {name} "
|
|
"TEXT NOT NULL DEFAULT ''")
|
|
con.execute(
|
|
"INSERT OR IGNORE INTO intake_job("
|
|
"id,original_name,stored_name,title,context,size,status,detail,"
|
|
"transcript,created_at,updated_at,draft,result"
|
|
") VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
|
(job_id, path.name, path.name, path.stem, "", path.stat().st_size,
|
|
"queued", "监控目录已接收", "", now, now, "", ""))
|
|
con.commit()
|
|
con.close()
|
|
return {
|
|
"job_id": job_id, "title": path.stem, "context": "",
|
|
"original_name": path.name, "stored_name": path.name,
|
|
}
|
|
except Exception as exc:
|
|
log(f"CONTROL_JOB_CREATE_FAILED file={path} error={exc}")
|
|
return {}
|
|
|
|
|
|
def duration_seconds(path):
|
|
try:
|
|
out = subprocess.check_output([
|
|
"ffprobe", "-v", "error", "-show_entries", "format=duration",
|
|
"-of", "default=nk=1:nw=1", str(path)
|
|
], text=True, timeout=30).strip()
|
|
return int(float(out) + 0.999) if out else None
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def file_id(path):
|
|
st = path.stat()
|
|
h = hashlib.sha256()
|
|
h.update(str(path.name).encode())
|
|
h.update(str(st.st_size).encode())
|
|
h.update(str(int(st.st_mtime)).encode())
|
|
if st.st_size <= 256 * 1024 * 1024:
|
|
with path.open("rb") as f:
|
|
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()[:24]
|
|
|
|
|
|
def stable(path):
|
|
try:
|
|
st1 = path.stat()
|
|
if time.time() - st1.st_mtime < STABLE_AGE_SECONDS:
|
|
return False
|
|
time.sleep(2)
|
|
st2 = path.stat()
|
|
return st1.st_size == st2.st_size and int(st1.st_mtime) == int(st2.st_mtime)
|
|
except FileNotFoundError:
|
|
return False
|
|
|
|
|
|
def unique_path(directory, name):
|
|
p = directory / name
|
|
if not p.exists():
|
|
return p
|
|
stem, suffix = p.stem, p.suffix
|
|
for i in range(1, 10000):
|
|
q = directory / f"{stem}-{i}{suffix}"
|
|
if not q.exists():
|
|
return q
|
|
raise RuntimeError(f"cannot allocate unique path for {name}")
|
|
|
|
|
|
def webhook_secret(route=WEBHOOK_ROUTE):
|
|
if route == WEBHOOK_ROUTE and WEBHOOK_SECRET:
|
|
return WEBHOOK_SECRET
|
|
try:
|
|
data = json.loads(SUBSCRIPTIONS.read_text(encoding="utf-8"))
|
|
return str(data.get(route, {}).get("secret", ""))
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def post_webhook(payload, url=WEBHOOK_URL, route=WEBHOOK_ROUTE, label="WEBHOOK", timeout=30):
|
|
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
headers = {"Content-Type": "application/json"}
|
|
secret = webhook_secret(route)
|
|
if secret and secret != "INSECURE_NO_AUTH":
|
|
headers["X-Webhook-Signature"] = hmac.new(secret.encode(), data, hashlib.sha256).hexdigest()
|
|
req = urllib.request.Request(
|
|
url,
|
|
data=data,
|
|
headers=headers,
|
|
method="POST",
|
|
)
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
body = resp.read(1000).decode("utf-8", errors="ignore")
|
|
log(f"{label} status={resp.status} body={body[:300]!r}")
|
|
return resp.status
|
|
|
|
|
|
def post_status(status, filename, detail=""):
|
|
payload = {"status": status, "filename": filename, "detail": detail}
|
|
try:
|
|
return post_webhook(
|
|
payload,
|
|
url=STATUS_WEBHOOK_URL,
|
|
route=STATUS_WEBHOOK_ROUTE,
|
|
label="STATUS_WEBHOOK",
|
|
timeout=10,
|
|
)
|
|
except Exception as exc:
|
|
log(f"STATUS_WEBHOOK_FAILED status={status} file={filename} error={exc}")
|
|
return None
|
|
|
|
|
|
import re
|
|
|
|
_CONSUMER_LOCK = None
|
|
|
|
ASR_INBOX = Path(os.environ.get("ASR_VAULT", str(Path.home() / "obsidian-vault"))) / ".asr-inbox"
|
|
|
|
|
|
def bridge_to_asr_inbox(transcript, stem, title=""):
|
|
"""Atomically drop the finished transcript into the Obsidian .asr-inbox so the
|
|
Claude refine pipeline picks it up. Slug carries date+time for the draft name."""
|
|
m = re.match(r"^(\d{4})-(\d{2})-(\d{2})[_-](\d{2})[_-](\d{2})[_-](\d{2})", stem)
|
|
if m:
|
|
y, mo, d, hh, mm, ss = m.groups()
|
|
tail = stem[m.end():].strip("_- ")
|
|
safe_tail = re.sub(
|
|
r"[^\w一-鿿-]+", "_", title or tail).strip("_-")
|
|
safe_tail = re.sub(r"-[0-9a-f]{8}$", "", safe_tail)
|
|
label = safe_tail or "录音"
|
|
slug = f"{y}-{mo}-{d}-{hh}{mm}{ss}-{label}"
|
|
else:
|
|
safe = re.sub(r"[^\w一-鿿-]+", "_", stem).strip("_") or "录音"
|
|
slug = f"{time.strftime('%Y-%m-%d-%H%M%S')}-{safe}"
|
|
incoming = ASR_INBOX / ".incoming"
|
|
incoming.mkdir(parents=True, exist_ok=True)
|
|
(ASR_INBOX / ".state").mkdir(parents=True, exist_ok=True)
|
|
# Collision guard: never overwrite an unprocessed transcript.
|
|
dest = ASR_INBOX / f"{slug}.txt"
|
|
n = 2
|
|
while dest.exists():
|
|
dest = ASR_INBOX / f"{slug}-{n}.txt"
|
|
n += 1
|
|
tmp = incoming / dest.name
|
|
shutil.copyfile(str(transcript), str(tmp))
|
|
tmp.replace(dest) # atomic within the vault filesystem -> triggers the path unit
|
|
return dest
|
|
|
|
|
|
def process(path):
|
|
seen = load_seen()
|
|
fid = file_id(path)
|
|
intake = upload_meta(path.name) or ensure_job(path, fid)
|
|
if fid in seen:
|
|
log(f"SKIP already seen file={path} id={fid}")
|
|
return
|
|
PROCESSING.mkdir(parents=True, exist_ok=True)
|
|
TRANSCRIPTS.mkdir(parents=True, exist_ok=True)
|
|
PROCESSED.mkdir(parents=True, exist_ok=True)
|
|
FAILED.mkdir(parents=True, exist_ok=True)
|
|
proc = unique_path(PROCESSING, path.name)
|
|
try:
|
|
shutil.move(str(path), str(proc))
|
|
except FileNotFoundError:
|
|
log(f"SKIP disappeared before claim file={path}")
|
|
return
|
|
transcript = TRANSCRIPTS / f"{fid}-{proc.stem}.txt"
|
|
dur = duration_seconds(proc)
|
|
log(f"PROCESS file={proc} id={fid} duration={dur}")
|
|
update_job(intake, "transcribing", f"录音时长约 {dur}s" if dur else "正在转写")
|
|
post_status("开始转录", proc.name, f"时长约 {dur}s；完成后会继续发送粗整理。")
|
|
try:
|
|
subprocess.run([str(ASR), str(proc), str(transcript)], check=True, timeout=None)
|
|
if intake:
|
|
original = transcript.read_text(encoding="utf-8", errors="ignore")
|
|
marker = json.dumps({
|
|
"job_id": intake.get("job_id", ""),
|
|
"title": intake.get("title", ""),
|
|
"context": intake.get("context", ""),
|
|
}, ensure_ascii=False)
|
|
transcript.write_text(f"<!-- intake: {marker} -->\n" + original,
|
|
encoding="utf-8")
|
|
done = unique_path(PROCESSED, proc.name)
|
|
shutil.move(str(proc), str(done))
|
|
post_status("转录完成，正在生成粗整理", done.name, f"转录文件：{transcript}")
|
|
# Refinement is handled by the Claude Code pipeline (not the Hermes agent
|
|
# webhook). Bridge the finished transcript into the Obsidian .asr-inbox so
|
|
# obsidian-asr-refine (systemd path unit) produces a rough-outline draft.
|
|
bridged = bridge_to_asr_inbox(
|
|
transcript, proc.stem, str(intake.get("title") or ""))
|
|
update_job(intake, "refining", "转写完成，正在生成整理稿",
|
|
transcript=str(transcript))
|
|
log(f"BRIDGED transcript -> {bridged}")
|
|
seen[fid] = {"filename": done.name, "audio_path": str(done), "transcript_path": str(transcript), "ts": int(time.time())}
|
|
save_seen(seen)
|
|
log(f"DONE file={done} transcript={transcript}")
|
|
except Exception as exc:
|
|
fail = unique_path(FAILED, proc.name)
|
|
if proc.exists():
|
|
shutil.move(str(proc), str(fail))
|
|
err = FAILED / f"{fail.name}.error.txt"
|
|
err.write_text(str(exc), encoding="utf-8")
|
|
post_status("处理失败", fail.name, f"错误：{exc}")
|
|
update_job(intake, "failed", f"处理失败：{exc}")
|
|
log(f"FAILED file={path} error={exc}")
|
|
|
|
|
|
def once():
|
|
for p in sorted(INBOX.iterdir() if INBOX.exists() else []):
|
|
if not p.is_file() or p.suffix.lower() not in SUPPORTED:
|
|
continue
|
|
if stable(p):
|
|
process(p)
|
|
|
|
|
|
def hold_consumer_lock():
|
|
"""Share ASR_STATE_DIR/consumer.lock with asrservice.worker. Exit if it is taken."""
|
|
path = Path(os.environ.get("ASR_CONSUMER_LOCK", str(CONTROL_DB.parent / "consumer.lock")))
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
handle = path.open("a+")
|
|
try:
|
|
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError:
|
|
handle.close()
|
|
log(f"EXIT another recording consumer holds {path}")
|
|
raise SystemExit(1)
|
|
handle.seek(0)
|
|
handle.truncate()
|
|
handle.write(str(os.getpid()))
|
|
handle.flush()
|
|
return handle
|
|
|
|
|
|
def main():
|
|
global _CONSUMER_LOCK
|
|
_CONSUMER_LOCK = hold_consumer_lock()
|
|
for d in [INBOX, PROCESSING, PROCESSED, FAILED, TRANSCRIPTS, STATE, LOGS]:
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
if not ASR.exists():
|
|
raise SystemExit(f"missing ASR script: {ASR}")
|
|
log("START watcher")
|
|
if "--once" in sys.argv:
|
|
once()
|
|
return
|
|
while True:
|
|
try:
|
|
once()
|
|
except Exception as exc:
|
|
log(f"LOOP_ERROR {exc}")
|
|
time.sleep(POLL_SECONDS)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|