deploy standalone ASR pipeline on tools-pve
This commit is contained in:
Codex
@@ -0,0 +1,7 @@
|
||||
.git
|
||||
.env
|
||||
tests
|
||||
README.md
|
||||
__pycache__
|
||||
*.pyc
|
||||
seed
|
||||
@@ -0,0 +1,11 @@
|
||||
ZENMUX_API_KEY=
|
||||
ASR_AGENT_TOKEN=
|
||||
ASR_PUBLIC_URL=http://192.168.50.49:8788
|
||||
ASR_DIARIZE_URL=http://192.168.50.100:10099/asr
|
||||
HINDSIGHT_API=http://192.168.50.47:8888
|
||||
HINDSIGHT_BANK=personal-context
|
||||
ASR_NOTIFY_BACKEND=none
|
||||
HERMES_WEBHOOK_URL=
|
||||
HERMES_WEBHOOK_SECRET=
|
||||
TELEGRAM_BOT_TOKEN=
|
||||
TELEGRAM_CHAT_ID=
|
||||
@@ -0,0 +1,4 @@
|
||||
.env
|
||||
seed/*.db
|
||||
__pycache__/
|
||||
*.pyc
|
||||
@@ -0,0 +1,34 @@
|
||||
FROM python:3.13-slim-bookworm@sha256:ed86c82274b3c69b52fb5820f358f0bd7df0b603332063cb5c6e32bd220c3e6e
|
||||
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl tini \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY asrpipe/ ./asrpipe/
|
||||
COPY asrservice/ ./asrservice/
|
||||
COPY termbank/ ./termbank/
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONPATH=/app \
|
||||
ASR_STATE_DIR=/state \
|
||||
ASR_VAULT=/data/vault \
|
||||
ASR_TERMBANK_DIR=/app/termbank \
|
||||
ASR_CACHE_DIR=/data/cache \
|
||||
ASR_LOG=/data/logs/asr-pipeline.log \
|
||||
ASR_RECORDINGS_ROOT=/data \
|
||||
ASR_DRAFTS_DIR=/data/artifacts/drafts \
|
||||
ASR_RESULTS_DIR=/data/artifacts/results \
|
||||
ASR_REVISION_QUEUE_DIR=/state/revision-queue \
|
||||
ASR_GIT_SYNC=0 \
|
||||
ASR_NOTIFY=0 \
|
||||
ASR_REVIEW_HOST=0.0.0.0 \
|
||||
ASR_REVIEW_PORT=8788
|
||||
|
||||
EXPOSE 8788
|
||||
ENTRYPOINT ["/usr/bin/tini", "--"]
|
||||
CMD ["python", "-m", "asrservice.web"]
|
||||
@@ -0,0 +1,51 @@
|
||||
# ASR Pipeline on tools-pve
|
||||
|
||||
独立的录音转写、精修、人工审核和导出服务。WebUI 与 Agent API 共用 `192.168.50.49:8788`；worker 调用 unRaid 上的 Qwen3-ASR/CAM++ 服务以及 ZenMux 模型 API。
|
||||
|
||||
## 数据边界
|
||||
|
||||
- `/mnt/truenas/personal/ASR-Pipeline` → 容器 `/data`：原始录音、转写、ASR 原始响应、草稿、正式稿、日志和 SQLite 在线备份。
|
||||
- `/opt/asr-pipeline/state` → 容器 `/state`：活动 SQLite、worker heartbeat、revision queue。SQLite WAL 不放在 NFS。
|
||||
- `/srv/asr/inbox`：指向 TrueNAS inbox 的宿主机链接，供 `scp`/`sftp` 上传。
|
||||
- `.env`：仅存在 Arcane Project / tools-pve，权限 `0600`，不进入 Git。
|
||||
|
||||
SSH 大文件建议先上传为 `.part`，完成后在同一目录原子改名为受支持扩展名：
|
||||
|
||||
```bash
|
||||
scp call.m4a tools@192.168.50.49:/srv/asr/inbox/call.m4a.part
|
||||
ssh tools@192.168.50.49 'mv /srv/asr/inbox/call.m4a.part /srv/asr/inbox/call.m4a'
|
||||
```
|
||||
|
||||
## Agent API
|
||||
|
||||
所有 `/api/v1/*` 路由使用 `Authorization: Bearer $ASR_AGENT_TOKEN`。
|
||||
|
||||
```bash
|
||||
curl -X POST http://192.168.50.49:8788/api/v1/jobs \
|
||||
-H "Authorization: Bearer $ASR_AGENT_TOKEN" \
|
||||
-H 'X-Filename: call.m4a' -H 'X-Title: 项目通话' \
|
||||
-H 'X-Context: 参与人和项目背景' --data-binary @call.m4a
|
||||
|
||||
curl -H "Authorization: Bearer $ASR_AGENT_TOKEN" \
|
||||
http://192.168.50.49:8788/api/v1/jobs/JOB_ID
|
||||
```
|
||||
|
||||
路由：
|
||||
|
||||
- `POST /api/v1/jobs`：上传并创建异步任务。
|
||||
- `GET /api/v1/jobs?limit=&offset=&status=`：分页查询。
|
||||
- `GET /api/v1/jobs/{id}`：任务详情、事件时间线与产物链接。
|
||||
- `GET /api/v1/jobs/{id}/artifacts/{transcript|draft|result}`：读取产物。
|
||||
- `GET /api/v1/jobs/{id}/reviews`：待人工确认项。
|
||||
- `POST /api/v1/jobs/{id}/reviews/{question_id}`：JSON `{"answer":"..."}`。
|
||||
- `POST /api/v1/jobs/{id}/retry`：重试失败任务。
|
||||
|
||||
`/healthz` 仅表示 Web 进程存活；`/readyz` 同时检查数据库与 worker heartbeat。
|
||||
|
||||
## 通知
|
||||
|
||||
`ASR_NOTIFY_BACKEND=none|telegram|hermes`。Telegram 需要 `TELEGRAM_BOT_TOKEN` 与 `TELEGRAM_CHAT_ID`；Hermes 需要一个明确存在的签名 webhook URL 与共享密钥。发送失败只影响提醒，不改变任务结果。
|
||||
|
||||
## 运维
|
||||
|
||||
项目由 Arcane Git Sync 管理，自动同步、自动 pull 和自动 redeploy 均关闭。Compose 变更先在 Git 中验证，再对 tools-pve 的准确 Sync 手动同步。活动数据库每小时通过 SQLite backup API 写到 `/data/backups/*.latest`。
|
||||
@@ -0,0 +1,382 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Standalone CLI. No Claude Code, no opencode, no agent runtime.
|
||||
|
||||
asr-refine run-all # process every pending transcript
|
||||
asr-refine run <file.txt> # process one transcript
|
||||
asr-refine dry-run <file.txt> # full pipeline, write nothing to the vault
|
||||
asr-refine doctor # check credentials, services, term bank
|
||||
asr-refine build-termbank # refresh the domain term bank (network)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import filecmp
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from .config import CFG
|
||||
from .log import log
|
||||
|
||||
_LOCK_HANDLE = None
|
||||
|
||||
|
||||
def _lock():
|
||||
"""Single instance. The systemd path unit can fire several times in quick
|
||||
succession when a batch of transcripts lands, and two concurrent runs would
|
||||
race on git and on the pending index."""
|
||||
d = Path(os.environ.get("XDG_RUNTIME_DIR", "/tmp"))
|
||||
global _LOCK_HANDLE
|
||||
f = (d / "asr-pipeline.lock").open("w")
|
||||
try:
|
||||
fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
log("another run holds the lock; exiting")
|
||||
sys.exit(0)
|
||||
_LOCK_HANDLE = f
|
||||
return f
|
||||
|
||||
|
||||
def resolve_archive(slug: str, inbox_path: Path) -> str:
|
||||
"""Prefer the persistent transcript archive over the `.asr-inbox/` staging path.
|
||||
|
||||
The inbox copy is deleted on 定稿, so a note whose `transcript:` points there
|
||||
ends up dangling. `recordings/transcripts/<fid>-<YYYY-MM-DD_HH_MM_SS>.txt`
|
||||
survives, so map the slug's timestamp back onto it when possible.
|
||||
"""
|
||||
m = re.match(r"^(\d{4}-\d{2}-\d{2})-(\d{2})(\d{2})(\d{2})", slug)
|
||||
if m:
|
||||
d, hh, mm, ss = m.groups()
|
||||
want = f"{d}_{hh}_{mm}_{ss}.txt"
|
||||
arch = CFG.obsidian / "recordings" / "transcripts"
|
||||
if arch.is_dir():
|
||||
hits = sorted(arch.glob(f"*{want[:-4]}*.txt"))
|
||||
if hits:
|
||||
if len(hits) > 1:
|
||||
for hit in hits:
|
||||
try:
|
||||
if (hit.stat().st_size == inbox_path.stat().st_size
|
||||
and filecmp.cmp(hit, inbox_path,
|
||||
shallow=False)):
|
||||
return str(hit)
|
||||
except OSError:
|
||||
continue
|
||||
return str(hits[0])
|
||||
return str(inbox_path)
|
||||
|
||||
|
||||
def process(path: Path, *, dry_run=False) -> int:
|
||||
from . import refine as R
|
||||
from . import vault
|
||||
|
||||
slug = path.stem
|
||||
if not dry_run and vault.is_done(slug):
|
||||
log(f"skip (already processed): {path}")
|
||||
return 0
|
||||
raw = path.read_text(errors="ignore")
|
||||
if not raw.strip():
|
||||
log(f"skip (empty): {path}", level="WARN")
|
||||
return 0
|
||||
intake_job = ""
|
||||
m_intake = re.search(r"(?m)^<!-- intake:\s*(\{.*\})\s*-->$", raw)
|
||||
if m_intake:
|
||||
try:
|
||||
intake_job = str(json.loads(m_intake.group(1)).get("job_id") or "")
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
date, title = vault.slug_parts(slug)
|
||||
dest = vault.draft_path(date, title) if not dry_run else Path("/dev/null")
|
||||
transcript_ref = resolve_archive(slug, path)
|
||||
if not dry_run:
|
||||
existing = vault.find_draft_for_transcript(transcript_ref)
|
||||
if existing:
|
||||
log(f"recover interrupted run: {existing}")
|
||||
from . import review, review_web
|
||||
review.enqueue_draft(transcript_ref, existing)
|
||||
if intake_job:
|
||||
try:
|
||||
from .control_web import update_job
|
||||
update_job(intake_job, "review",
|
||||
"已恢复整理稿，等待确认", draft=str(existing))
|
||||
except Exception as e:
|
||||
log(f"intake recovery status failed: {e}", level="WARN")
|
||||
if not review_web.request_start(intake_job):
|
||||
vault.notify("ASR Web 审核启动失败",
|
||||
f"{existing.name} 已恢复，但 Web 服务启动失败。")
|
||||
return 1
|
||||
vault.mark_done(slug)
|
||||
return 0
|
||||
log(f"refining {path.name} ({len(raw)} chars) -> "
|
||||
f"{dest if not dry_run else '(dry-run)'} "
|
||||
f"clean={CFG.clean_model} draft={CFG.draft_model}")
|
||||
|
||||
t0 = time.time()
|
||||
try:
|
||||
res = R.refine(raw, date=date, transcript_path=transcript_ref,
|
||||
queue_review=not dry_run)
|
||||
except Exception as e:
|
||||
log(f"FAILED {path.name}: {e} (left unprocessed for retry)", level="ERROR")
|
||||
return 1
|
||||
|
||||
f = res.stats["fidelity"]
|
||||
log(f"done {path.name} in {time.time()-t0:.0f}s | "
|
||||
f"chunks={res.stats['n_chunks']} flags={res.stats['n_flags']} "
|
||||
f"hs={res.stats['n_hindsight']} "
|
||||
f"gate_fail={res.stats['gate_failed']} retried={res.stats['retried']} "
|
||||
f"fallback={res.stats['fallback']} | "
|
||||
f"ratio={f['char_ratio']} num={f['number_recall']} "
|
||||
f"latin={f['latin_recall']} term={f['term_recall']} | "
|
||||
f"${res.usage['cost_usd']} calls={res.usage['calls']}")
|
||||
|
||||
if dry_run:
|
||||
out = CFG.root / "dry-run" / slug
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
(out / "cleaned.txt").write_text(res.cleaned, encoding="utf-8")
|
||||
if res.evidence:
|
||||
(out / "evidence.md").write_text(res.evidence, encoding="utf-8")
|
||||
(out / "draft.md").write_text(res.draft, encoding="utf-8")
|
||||
(out / "report.json").write_text(json.dumps(
|
||||
{"stats": res.stats, "usage": res.usage, "timings": res.timings},
|
||||
ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(f"dry-run output -> {out}")
|
||||
return 0
|
||||
|
||||
vault.write_draft_file(dest, res.draft)
|
||||
from . import review, review_web
|
||||
draft_questions = review.enqueue_draft(transcript_ref, dest)
|
||||
if intake_job:
|
||||
try:
|
||||
from .control_web import update_job
|
||||
update_job(intake_job, "review", "整理稿已生成，等待确认",
|
||||
draft=str(dest))
|
||||
except Exception as e:
|
||||
log(f"intake status update failed: {e}", level="WARN")
|
||||
if not review_web.request_start(intake_job):
|
||||
vault.notify("ASR Web 审核启动失败",
|
||||
f"{date} {title} 的私有粗稿已保留；"
|
||||
f"{draft_questions} 个问题可用 "
|
||||
"asr-refine review pending 查看")
|
||||
return 1
|
||||
vault.mark_done(slug)
|
||||
log(f"staged private draft {dest} | questions={draft_questions}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_run(a):
|
||||
_lock()
|
||||
return process(Path(a.file).expanduser(), dry_run=False)
|
||||
|
||||
|
||||
def cmd_dry_run(a):
|
||||
return process(Path(a.file).expanduser(), dry_run=True)
|
||||
|
||||
|
||||
def cmd_run_all(a):
|
||||
_lock()
|
||||
from . import vault
|
||||
todo = vault.pending_transcripts()
|
||||
if not todo:
|
||||
log(f"no pending transcripts in {CFG.inbox}")
|
||||
return 0
|
||||
log(f"{len(todo)} pending transcript(s)")
|
||||
rc = 0
|
||||
for p in todo:
|
||||
rc |= process(p)
|
||||
return rc
|
||||
|
||||
|
||||
def cmd_doctor(a):
|
||||
from . import fidelity
|
||||
ok = True
|
||||
|
||||
def chk(name, good, detail=""):
|
||||
nonlocal ok
|
||||
ok &= bool(good)
|
||||
print(f" [{'OK ' if good else 'FAIL'}] {name}" + (f" {detail}" if detail else ""))
|
||||
|
||||
print("paths")
|
||||
chk("vault", CFG.vault.is_dir(), str(CFG.vault))
|
||||
chk("inbox", CFG.inbox.is_dir(), str(CFG.inbox))
|
||||
chk("drafts dir writable", os.access(CFG.drafts.parent, os.W_OK), str(CFG.drafts))
|
||||
chk("glossary", CFG.glossary.exists(), str(CFG.glossary))
|
||||
chk("notifier", CFG.notifier.exists(), str(CFG.notifier))
|
||||
|
||||
print("credentials")
|
||||
try:
|
||||
k = CFG.api_key()
|
||||
chk("api key", len(k) > 20, f"len={len(k)} source={'env' if os.environ.get('ANTHROPIC_AUTH_TOKEN') else CFG.env_file.name}")
|
||||
except Exception as e:
|
||||
chk("api key", False, str(e))
|
||||
|
||||
print("lexicon")
|
||||
try:
|
||||
from . import lexicon
|
||||
lexicon.sync_from_vault()
|
||||
st = lexicon.stats()
|
||||
chk("entities", st["entities"] > 50,
|
||||
f"{st['entities']} ({st['with_company']} 有归属, {st['with_related']} 有关系)")
|
||||
chk("aliases", st["aliases"] > 50,
|
||||
f"{st['aliases']} = {st['aliases_card']} 卡片 + {st['aliases_learned']} 学习/术语表")
|
||||
except Exception as e:
|
||||
chk("lexicon", False, str(e))
|
||||
|
||||
print("term bank")
|
||||
n = len(fidelity.domain_terms())
|
||||
chk("terms.json", n > 1000, f"{n} terms")
|
||||
chk("allowlist", (CFG.termbank_dir / "allowlist.txt").exists(),
|
||||
f"{len((CFG.termbank_dir/'allowlist.txt').read_text().split()) if (CFG.termbank_dir/'allowlist.txt').exists() else 0} words")
|
||||
|
||||
print("services")
|
||||
import urllib.request
|
||||
try:
|
||||
with urllib.request.urlopen(
|
||||
f"{CFG.hindsight_api}/health", timeout=6) as r:
|
||||
chk("hindsight", r.status == 200, CFG.hindsight_api)
|
||||
except Exception as e:
|
||||
chk("hindsight (optional)", False, f"{CFG.hindsight_api}: {e}")
|
||||
try:
|
||||
from .llm import chat
|
||||
t0 = time.time()
|
||||
out = chat(CFG.clean_model, "回一个字:好", max_tokens=20)
|
||||
chk(f"clean model {CFG.clean_model}", bool(out), f"{time.time()-t0:.1f}s {out[:8]!r}")
|
||||
t0 = time.time()
|
||||
out = chat(CFG.draft_model, "回一个字:好", max_tokens=20)
|
||||
chk(f"draft model {CFG.draft_model}", bool(out), f"{time.time()-t0:.1f}s {out[:8]!r}")
|
||||
except Exception as e:
|
||||
chk("model endpoint", False, str(e))
|
||||
|
||||
print("pending")
|
||||
from . import vault
|
||||
todo = vault.pending_transcripts()
|
||||
print(f" {len(todo)} transcript(s) awaiting processing")
|
||||
for p in todo[:5]:
|
||||
print(f" {p.name}")
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def cmd_build_termbank(a):
|
||||
from . import termbank
|
||||
termbank.build()
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_lexicon(a):
|
||||
"""Entity/alias lexicon. Obsidian stays read-only; learned bindings live here."""
|
||||
from . import lexicon
|
||||
act = a.action
|
||||
if act == "sync":
|
||||
print(lexicon.sync_from_vault(force=True))
|
||||
elif act == "stats":
|
||||
for k, v in lexicon.stats().items():
|
||||
print(f" {k:18s} {v}")
|
||||
elif act == "learn":
|
||||
if not (a.alias and a.canonical):
|
||||
print("usage: lexicon learn <别名> <规范名> [--note …]", file=sys.stderr)
|
||||
return 2
|
||||
lexicon.learn_alias(a.alias, a.canonical, note=a.note or "")
|
||||
lexicon.learn_correction(a.alias, a.canonical, note=a.note or "")
|
||||
print(f"learned: {a.alias} -> {a.canonical}"
|
||||
+ (f" ({a.note})" if a.note else ""))
|
||||
elif act == "show":
|
||||
am = lexicon.alias_map()
|
||||
key = a.alias
|
||||
if key:
|
||||
print(f"{key} -> {am.get(key) or 'unknown'}")
|
||||
else:
|
||||
learned = [(k, v) for k, v in am.items()
|
||||
if any(o not in ("card",) for _, _, o, _ in v)]
|
||||
print(f"{len(am)} aliases total, {len(learned)} beyond the cards")
|
||||
for k, v in sorted(learned)[:40]:
|
||||
print(f" {k:10s} -> {'、'.join(c for c, _, _, _ in v)}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_review(a):
|
||||
from . import review
|
||||
if a.action == "serve":
|
||||
from . import review_web
|
||||
return review_web.serve()
|
||||
if a.action == "state":
|
||||
from .review_web import STATE
|
||||
if not STATE.exists():
|
||||
print("inactive")
|
||||
return 1
|
||||
print(STATE.read_text(encoding="utf-8"))
|
||||
return 0
|
||||
if a.action == "pending":
|
||||
rows = review.pending(a.limit)
|
||||
for r in rows:
|
||||
payload = json.loads(r["payload"] or "{}")
|
||||
print(f"#{r['id']} [{r['recording']}] {r['question']}")
|
||||
if payload.get("sentence"):
|
||||
print(f" {payload['sentence'][:180]}")
|
||||
return 0
|
||||
if a.id is None or not a.answer:
|
||||
print("usage: asr-refine review answer <id> <answer>", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
result = review.answer(a.id, a.answer)
|
||||
except KeyError as e:
|
||||
print(str(e), file=sys.stderr)
|
||||
return 2
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_graph(a):
|
||||
from . import graph
|
||||
result = graph.sync(force=a.force) if a.action == "sync" else graph.stats()
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_revise(a):
|
||||
from . import revise
|
||||
path = (CFG.drafts / a.draft).resolve()
|
||||
try:
|
||||
path.relative_to(CFG.drafts.resolve())
|
||||
except ValueError:
|
||||
print("draft outside review root", file=sys.stderr)
|
||||
return 2
|
||||
return 0 if revise.revise(path) else 1
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(prog="asr-refine",
|
||||
description="Standalone ASR transcript -> Obsidian draft")
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
p = sub.add_parser("run"); p.add_argument("file"); p.set_defaults(fn=cmd_run)
|
||||
p = sub.add_parser("dry-run"); p.add_argument("file"); p.set_defaults(fn=cmd_dry_run)
|
||||
sub.add_parser("run-all").set_defaults(fn=cmd_run_all)
|
||||
sub.add_parser("doctor").set_defaults(fn=cmd_doctor)
|
||||
sub.add_parser("build-termbank").set_defaults(fn=cmd_build_termbank)
|
||||
p = sub.add_parser("lexicon", help="entity/alias lexicon (SQLite; vault is read-only)")
|
||||
p.add_argument("action", choices=["sync", "stats", "learn", "show"])
|
||||
p.add_argument("alias", nargs="?")
|
||||
p.add_argument("canonical", nargs="?")
|
||||
p.add_argument("--note", default="")
|
||||
p.set_defaults(fn=cmd_lexicon)
|
||||
p = sub.add_parser("review", help="human review queue (transport independent)")
|
||||
p.add_argument("action", choices=["pending", "answer", "serve", "state"])
|
||||
p.add_argument("id", nargs="?", type=int)
|
||||
p.add_argument("answer", nargs="?")
|
||||
p.add_argument("--limit", type=int, default=20)
|
||||
p.set_defaults(fn=cmd_review)
|
||||
p = sub.add_parser("graph", help="incremental entity/relation/recency graph")
|
||||
p.add_argument("action", choices=["sync", "stats"])
|
||||
p.add_argument("--force", action="store_true")
|
||||
p.set_defaults(fn=cmd_graph)
|
||||
p = sub.add_parser("revise", help="publish a Web-confirmed private draft")
|
||||
p.add_argument("draft", help="draft filename under minutes/_drafts")
|
||||
p.set_defaults(fn=cmd_revise)
|
||||
a = ap.parse_args(argv)
|
||||
return a.fn(a)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Configuration for the standalone ASR post-processing pipeline.
|
||||
|
||||
This program deliberately has **no dependency on Claude Code, opencode, or any
|
||||
agent CLI**. It talks to the model provider over plain HTTPS and does its own file
|
||||
and git handling, so it can run from systemd, cron, or a shell with nothing but
|
||||
python3 and a network route.
|
||||
|
||||
Everything is overridable by environment variable so the systemd unit stays the
|
||||
single place where deployment-specific values live. Credentials are read from the
|
||||
same 600-perm env file the legacy unit uses, so token rotation remains a one-file
|
||||
change.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
PKG = Path(__file__).resolve().parent
|
||||
ROOT = PKG.parent # OBSIDIAN/asr-pipeline
|
||||
OBSIDIAN = ROOT.parent # OBSIDIAN project dir
|
||||
|
||||
|
||||
def _env(name: str, default: str) -> str:
|
||||
v = os.environ.get(name)
|
||||
return v if v not in (None, "") else default
|
||||
|
||||
|
||||
def _envf(name: str, default: float) -> float:
|
||||
try:
|
||||
return float(os.environ["ASR_" + name])
|
||||
except (KeyError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _envi(name: str, default: int) -> int:
|
||||
try:
|
||||
return int(os.environ["ASR_" + name])
|
||||
except (KeyError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
# ---- paths ----------------------------------------------------------
|
||||
vault: Path = field(default_factory=lambda: Path(
|
||||
_env("ASR_VAULT", str(Path.home() / "obsidian-vault"))))
|
||||
obsidian: Path = OBSIDIAN
|
||||
root: Path = ROOT
|
||||
|
||||
# ---- models ---------------------------------------------------------
|
||||
# Cleaning: measured against claude-haiku-4.5 on a 118k-char English-mixed call,
|
||||
# gemini-3.5-flash-lite tripped the fidelity gate 19/87 times vs haiku's 78/87
|
||||
# (haiku's failures were 69x "over-compression" — it behaves like a summarizer
|
||||
# and deleted terms such as AUTOIMMUNE/ADCS/CD), while being 2x faster and
|
||||
# 2.9x cheaper. Hence flash-lite is the default cleaner.
|
||||
clean_model: str = _env("ASR_CLEAN_MODEL", "google/gemini-3.5-flash-lite")
|
||||
# Drafting: one call, needs judgement and structure. sonnet-5 replaces the
|
||||
# legacy headless opus-4-8.
|
||||
draft_model: str = _env("ASR_DRAFT_MODEL", "anthropic/claude-sonnet-5")
|
||||
keyword_model: str = _env("ASR_KEYWORD_MODEL", "google/gemini-3.5-flash-lite")
|
||||
|
||||
endpoint: str = _env("ASR_ENDPOINT", "https://zenmux.ai/api/v1/chat/completions")
|
||||
# ZenMux returned HTTP 429 at ~12 concurrent requests and was stable at 6.
|
||||
concurrency: int = _envi("CONCURRENCY", 6)
|
||||
http_timeout: float = _envf("HTTP_TIMEOUT", 420.0)
|
||||
max_retries: int = _envi("MAX_RETRIES", 5)
|
||||
|
||||
# ---- chunking -------------------------------------------------------
|
||||
chunk_budget: int = _envi("CHUNK_BUDGET", 1800)
|
||||
chunk_overlap: int = _envi("CHUNK_OVERLAP", 200)
|
||||
collapse_noise: bool = _env("ASR_COLLAPSE_NOISE", "1") != "0"
|
||||
|
||||
# ---- fidelity gate --------------------------------------------------
|
||||
gate_enabled: bool = _env("ASR_GATE", "1") != "0"
|
||||
gate_min_ratio: float = _envf("GATE_MIN_RATIO", 0.55)
|
||||
gate_max_ratio: float = _envf("GATE_MAX_RATIO", 1.35)
|
||||
|
||||
# ---- knowledge sources ----------------------------------------------
|
||||
use_termbank: bool = _env("ASR_TERMBANK", "1") != "0"
|
||||
use_hindsight: bool = _env("ASR_HINDSIGHT", "1") != "0"
|
||||
use_keyword_recall: bool = _env("ASR_KEYWORD_RECALL", "1") != "0"
|
||||
use_lexicon: bool = _env("ASR_LEXICON", "1") != "0"
|
||||
use_normalize: bool = _env("ASR_NORMALIZE", "1") != "0"
|
||||
use_evidence: bool = _env("ASR_EVIDENCE", "1") != "0"
|
||||
hindsight_api: str = _env("HINDSIGHT_API", "http://192.168.50.47:8888")
|
||||
hindsight_bank: str = _env("HINDSIGHT_BANK", "personal-context")
|
||||
hindsight_timeout: float = _envf("HINDSIGHT_TIMEOUT", 12.0)
|
||||
flag_min_score: float = _envf("FLAG_MIN_SCORE", 0.86)
|
||||
review_min_score: float = _envf("REVIEW_MIN_SCORE", 0.82)
|
||||
review_max_score: float = _envf("REVIEW_MAX_SCORE", 0.96)
|
||||
review_host: str = _env("ASR_REVIEW_HOST", "0.0.0.0")
|
||||
review_public_host: str = _env("ASR_REVIEW_PUBLIC_HOST", "192.168.50.145")
|
||||
review_port: int = _envi("REVIEW_PORT", 8765)
|
||||
review_idle_seconds: int = _envi("REVIEW_IDLE_SECONDS", 2700)
|
||||
review_max_seconds: int = _envi("REVIEW_MAX_SECONDS", 7200)
|
||||
|
||||
# ---- behaviour ------------------------------------------------------
|
||||
git_sync: bool = _env("ASR_GIT_SYNC", "1") != "0"
|
||||
notify: bool = _env("ASR_NOTIFY", "1") != "0"
|
||||
cache_llm: bool = _env("ASR_CACHE", "0") != "0" # off in production
|
||||
|
||||
@property
|
||||
def inbox(self) -> Path:
|
||||
return self.vault / ".asr-inbox"
|
||||
|
||||
@property
|
||||
def state(self) -> Path:
|
||||
return self.inbox / ".state"
|
||||
|
||||
@property
|
||||
def drafts(self) -> Path:
|
||||
"""Private rough drafts. They are never part of the Obsidian repository."""
|
||||
return Path(_env("ASR_DRAFTS_DIR", str(self.state_dir / "drafts")))
|
||||
|
||||
@property
|
||||
def minutes(self) -> Path:
|
||||
return Path(_env("ASR_RESULTS_DIR", str(self.vault / "minutes")))
|
||||
|
||||
@property
|
||||
def pending_index(self) -> Path:
|
||||
# Legacy compatibility only; the Web queue replaced the Vault draft index.
|
||||
return self.vault / "minutes" / "_drafts" / "_待确认.md"
|
||||
|
||||
@property
|
||||
def glossary(self) -> Path:
|
||||
return self.vault / "minutes" / "_glossary.md"
|
||||
|
||||
@property
|
||||
def termbank_dir(self) -> Path:
|
||||
return Path(_env("ASR_TERMBANK_DIR", str(self.root / "termbank")))
|
||||
|
||||
@property
|
||||
def cache_dir(self) -> Path:
|
||||
return Path(_env("ASR_CACHE_DIR", str(self.root / "cache")))
|
||||
|
||||
@property
|
||||
def state_dir(self) -> Path:
|
||||
"""Program-owned persistent state. Never inside the vault."""
|
||||
return Path(_env("ASR_STATE_DIR", str(self.root / "state")))
|
||||
|
||||
@property
|
||||
def lexicon_db(self) -> Path:
|
||||
return Path(_env("ASR_LEXICON_DB", str(self.state_dir / "lexicon.db")))
|
||||
|
||||
@property
|
||||
def log_file(self) -> Path:
|
||||
return Path(_env("ASR_LOG", str(self.obsidian / "logs" / "asr-pipeline.log")))
|
||||
|
||||
@property
|
||||
def notifier(self) -> Path:
|
||||
return self.obsidian / "bin" / "meeting-status-notify.py"
|
||||
|
||||
@property
|
||||
def env_file(self) -> Path:
|
||||
"""600-perm credentials file, shared with the legacy systemd unit."""
|
||||
return Path(_env("ASR_ENV_FILE", str(self.obsidian / "bin" / ".asr-refine.env")))
|
||||
|
||||
def api_key(self) -> str:
|
||||
for k in ("ZENMUX_API_KEY", "ANTHROPIC_AUTH_TOKEN"):
|
||||
v = os.environ.get(k)
|
||||
if v:
|
||||
return v
|
||||
p = self.env_file
|
||||
if p.exists():
|
||||
for line in p.read_text(errors="ignore").splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("ANTHROPIC_AUTH_TOKEN="):
|
||||
val = line.split("=", 1)[1].strip().strip("'\"")
|
||||
if val:
|
||||
return val
|
||||
raise RuntimeError(
|
||||
"no API key: set ZENMUX_API_KEY/ANTHROPIC_AUTH_TOKEN or provide "
|
||||
f"{p} with ANTHROPIC_AUTH_TOKEN=…")
|
||||
|
||||
|
||||
CFG = Config()
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compact recalled prose into small, entity-centred context cards."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
SPLIT = re.compile(r"(?<=[。！？；])|\s*\|\s*")
|
||||
META = re.compile(r"^(When|Involving|来源|时间)\s*[:：]\s*", re.I)
|
||||
|
||||
|
||||
def _key(text: str) -> str:
|
||||
text = META.sub("", text)
|
||||
return re.sub(r"[\s，。；、,:：|（）()“”\"'\-]+", "", text).lower()
|
||||
|
||||
|
||||
def compact(names: list[str], texts: list[str], *, per_entity: int = 3,
|
||||
total: int = 12) -> str:
|
||||
"""Deduplicate Hindsight recall before it reaches any model prompt.
|
||||
|
||||
Repeated memories often restate the same relationship with progressively more
|
||||
prose. Keep one normalized fact per subject and cap every entity so a single
|
||||
well-known person cannot consume the whole context window.
|
||||
"""
|
||||
buckets: dict[str, list[str]] = defaultdict(list)
|
||||
seen: set[str] = set()
|
||||
known = sorted(set(names), key=len, reverse=True)
|
||||
for raw in texts:
|
||||
for fact in SPLIT.split(re.sub(r"\s+", " ", raw).strip()):
|
||||
fact = fact.strip(" -|")
|
||||
if len(fact) < 8:
|
||||
continue
|
||||
k = _key(fact)
|
||||
if not k or any(k == old or k in old or old in k for old in seen):
|
||||
continue
|
||||
seen.add(k)
|
||||
owner = next((n for n in known if n in fact), "相关背景")
|
||||
if len(buckets[owner]) < per_entity:
|
||||
buckets[owner].append(fact[:220])
|
||||
|
||||
lines = []
|
||||
for owner, facts in buckets.items():
|
||||
lines.append(f"{owner}：")
|
||||
lines.extend(f"- {f}" for f in facts)
|
||||
if sum(1 for x in lines if x.startswith("- ")) >= total:
|
||||
break
|
||||
return "\n".join(lines)
|
||||
@@ -0,0 +1,489 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Persistent mobile control plane for recording intake and human review."""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import sqlite3
|
||||
import time
|
||||
import urllib.parse
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
from . import review
|
||||
from .config import CFG
|
||||
from .log import log
|
||||
|
||||
STATE = CFG.state_dir / "review-web.json"
|
||||
DB = CFG.state_dir / "intake.db"
|
||||
RECORDINGS = Path(os.environ.get(
|
||||
"ASR_RECORDINGS_ROOT", str(CFG.obsidian / "recordings")))
|
||||
INBOX = RECORDINGS / "inbox"
|
||||
META = RECORDINGS / "state" / "uploads"
|
||||
MAX_UPLOAD = int(os.environ.get("ASR_UPLOAD_MAX_BYTES", str(4 * 1024**3)))
|
||||
CHUNK = 1024 * 1024
|
||||
SUPPORTED = {".m4a", ".mp3", ".wav", ".mp4", ".aac", ".ogg",
|
||||
".opus", ".webm", ".flac"}
|
||||
|
||||
STATUS_LABEL = {
|
||||
"uploading": "上传中",
|
||||
"queued": "等待转写",
|
||||
"transcribing": "正在转写",
|
||||
"refining": "正在整理",
|
||||
"review": "等待确认",
|
||||
"publishing": "正在发布",
|
||||
"complete": "已完成",
|
||||
"failed": "处理失败",
|
||||
}
|
||||
|
||||
|
||||
def _connect() -> sqlite3.Connection:
|
||||
CFG.state_dir.mkdir(parents=True, exist_ok=True)
|
||||
con = sqlite3.connect(DB, timeout=20)
|
||||
con.row_factory = sqlite3.Row
|
||||
con.execute("PRAGMA journal_mode=WAL")
|
||||
con.execute("""
|
||||
CREATE TABLE IF NOT EXISTS intake_job(
|
||||
id TEXT PRIMARY KEY,
|
||||
original_name TEXT NOT NULL,
|
||||
stored_name TEXT NOT NULL,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
context TEXT NOT NULL DEFAULT '',
|
||||
size INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
transcript TEXT NOT NULL DEFAULT '',
|
||||
created_at REAL NOT NULL,
|
||||
updated_at REAL NOT NULL
|
||||
)
|
||||
""")
|
||||
columns = {row["name"] for row in con.execute("PRAGMA table_info(intake_job)")}
|
||||
for name in ("draft", "result"):
|
||||
if name not in columns:
|
||||
con.execute(
|
||||
f"ALTER TABLE intake_job ADD COLUMN {name} TEXT NOT NULL DEFAULT ''")
|
||||
return con
|
||||
|
||||
|
||||
def update_job(job_id: str, status: str, detail: str = "",
|
||||
transcript: str = "", draft: str = "",
|
||||
result: str = "") -> bool:
|
||||
if os.environ.get("ASR_ROLE"):
|
||||
from asrservice.db import update_job as service_update_job
|
||||
return service_update_job(job_id, status, detail, transcript=transcript,
|
||||
draft=draft, result=result)
|
||||
if not job_id:
|
||||
return False
|
||||
con = _connect()
|
||||
fields = ["status=?", "updated_at=?"]
|
||||
values: list[object] = [status, time.time()]
|
||||
if detail:
|
||||
fields.append("detail=?")
|
||||
values.append(detail[:1000])
|
||||
if transcript:
|
||||
fields.append("transcript=?")
|
||||
values.append(transcript)
|
||||
if draft:
|
||||
fields.append("draft=?")
|
||||
values.append(draft)
|
||||
if result:
|
||||
fields.append("result=?")
|
||||
values.append(result)
|
||||
values.append(job_id)
|
||||
cur = con.execute(
|
||||
f"UPDATE intake_job SET {','.join(fields)} WHERE id=?", values)
|
||||
con.commit()
|
||||
con.close()
|
||||
return cur.rowcount > 0
|
||||
|
||||
|
||||
def _jobs(limit: int = 30) -> list[dict]:
|
||||
con = _connect()
|
||||
rows = con.execute(
|
||||
"SELECT * FROM intake_job ORDER BY created_at DESC LIMIT ?",
|
||||
(limit,)).fetchall()
|
||||
con.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def _job(job_id: str) -> dict | None:
|
||||
con = _connect()
|
||||
row = con.execute("SELECT * FROM intake_job WHERE id=?", (job_id,)).fetchone()
|
||||
con.close()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def _review_url(job_id: str = "") -> str:
|
||||
base = f"http://{CFG.review_public_host}:{CFG.review_port}"
|
||||
return f"{base}/jobs/{job_id}/review/" if job_id else f"{base}/review/"
|
||||
|
||||
|
||||
def _pending_for_job(job: dict, limit: int = 200) -> list[dict]:
|
||||
transcript = str(job.get("transcript") or "")
|
||||
draft = str(job.get("draft") or "")
|
||||
rows = []
|
||||
for row in review.pending(limit):
|
||||
payload = json.loads(row["payload"] or "{}")
|
||||
if ((transcript and row["recording"] == transcript)
|
||||
or (draft and payload.get("draft_path") == draft)
|
||||
or payload.get("job_id") == job["id"]):
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
|
||||
def _safe_text(value: str, limit: int) -> str:
|
||||
value = re.sub(r"[\x00-\x1f\x7f]+", " ", value or "")
|
||||
return re.sub(r"\s+", " ", value).strip()[:limit]
|
||||
|
||||
|
||||
def _safe_stem(value: str) -> str:
|
||||
value = _safe_text(value, 80)
|
||||
value = re.sub(r"[^\w\u3400-\u9fff.-]+", "-", value, flags=re.UNICODE)
|
||||
return value.strip("-.") or "录音"
|
||||
|
||||
|
||||
def _write_state() -> None:
|
||||
CFG.state_dir.mkdir(parents=True, exist_ok=True)
|
||||
data = {
|
||||
"pid": os.getpid(),
|
||||
"url": _review_url(),
|
||||
"home": f"http://{CFG.review_public_host}:{CFG.review_port}/",
|
||||
"port": CFG.review_port,
|
||||
"started_at": time.time(),
|
||||
}
|
||||
tmp = STATE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
os.chmod(tmp, 0o600)
|
||||
tmp.replace(STATE)
|
||||
|
||||
|
||||
def _shell(title: str, body: str, script: str = "") -> bytes:
|
||||
return f"""<!doctype html>
|
||||
<html lang="zh-CN"><head>
|
||||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta name="color-scheme" content="light"><meta http-equiv="Cache-Control" content="no-store">
|
||||
<title>{html.escape(title)}</title>
|
||||
<style>
|
||||
:root{{--ink:#17202a;--muted:#637083;--line:#d9dee5;--paper:#f5f7f8;
|
||||
--accent:#176b5b;--danger:#b33a2b;--white:#fff;--warn:#986b12}}
|
||||
*{{box-sizing:border-box}} body{{margin:0;background:var(--paper);color:var(--ink);
|
||||
font-family:system-ui,-apple-system,"PingFang SC","Microsoft YaHei",sans-serif;
|
||||
font-size:16px;line-height:1.55;letter-spacing:0}}
|
||||
header{{background:var(--white);border-bottom:1px solid var(--line);padding:15px 18px}}
|
||||
header .inner{{max-width:760px;margin:auto;display:flex;align-items:center;justify-content:space-between}}
|
||||
header strong{{font-size:18px}} nav a{{color:var(--accent);text-decoration:none;margin-left:16px}}
|
||||
main{{max-width:760px;margin:0 auto;padding:18px 14px 44px}}
|
||||
.panel{{background:var(--white);border:1px solid var(--line);border-radius:8px;padding:18px;
|
||||
margin-bottom:14px}} h1{{font-size:22px;margin:0 0 8px}} h2{{font-size:18px;margin:0 0 12px}}
|
||||
.muted,.meta{{color:var(--muted);font-size:14px}} label{{display:block;font-weight:650;margin:12px 0 5px}}
|
||||
input,textarea{{width:100%;border:1px solid #98a4b0;border-radius:6px;background:white;
|
||||
padding:11px 12px;font:inherit}} textarea{{min-height:92px;resize:vertical}}
|
||||
button{{min-height:46px;border:0;border-radius:6px;padding:10px 15px;background:var(--accent);
|
||||
color:white;font:inherit;font-weight:650}} button:disabled{{opacity:.55}}
|
||||
button.secondary{{background:#e9edf1;color:var(--ink);border:1px solid var(--line)}}
|
||||
.actions{{display:flex;gap:10px;align-items:center;margin-top:14px}} .actions button{{flex:1}}
|
||||
progress{{width:100%;height:12px;margin-top:12px}} .job{{padding:12px 0;border-top:1px solid var(--line)}}
|
||||
.job:first-child{{border-top:0}} .jobtop{{display:flex;gap:10px;justify-content:space-between}}
|
||||
.joblink{{display:inline-block;margin-top:7px;color:var(--accent);font-weight:650;text-decoration:none}}
|
||||
.badge{{white-space:nowrap;font-size:13px;color:var(--accent);font-weight:700}}
|
||||
.badge.failed{{color:var(--danger)}} .badge.review{{color:var(--warn)}}
|
||||
.source{{margin:14px 0;padding:12px;border-left:4px solid #d4a72c;background:#fff9e8}}
|
||||
.choice{{width:100%;margin:5px 0}} .custom{{display:flex;gap:8px;margin-top:8px}}
|
||||
.custom input{{min-width:0;flex:1}} .custom button{{width:auto}}
|
||||
.draft{{margin:14px 0;border:1px solid var(--line);border-radius:6px;background:#fbfcfd}}
|
||||
.draft summary{{padding:12px 14px;font-weight:650;cursor:pointer}}
|
||||
.draft pre{{margin:0;padding:14px;border-top:1px solid var(--line);max-height:58vh;
|
||||
overflow:auto;white-space:pre-wrap;overflow-wrap:anywhere;font:14px/1.55 ui-monospace,monospace}}
|
||||
@media(max-width:480px){{main{{padding:12px 10px 28px}}.panel{{padding:15px}}
|
||||
.custom,.actions{{display:grid}}.custom button{{width:100%}}}}
|
||||
</style></head><body><header><div class="inner"><strong>录音工作台</strong>
|
||||
<nav><a href="/">上传</a><a href="/review/">确认</a></nav></div></header>
|
||||
<main>{body}</main>{script}</body></html>""".encode("utf-8")
|
||||
|
||||
|
||||
def _home() -> bytes:
|
||||
body = """<section class="panel"><h1>上传录音</h1>
|
||||
<div class="muted">支持 mp3、m4a、wav、mp4、aac、ogg、opus、webm、flac。上传完成后自动进入现有 ASR 流程。</div>
|
||||
<form id="upload">
|
||||
<label for="audio">录音文件</label><input id="audio" type="file" accept="audio/*,video/mp4" required>
|
||||
<label for="title">标题</label><input id="title" maxlength="80" placeholder="例如：洛启项目通话">
|
||||
<label for="context">已知背景（可选）</label><textarea id="context" maxlength="2000"
|
||||
placeholder="参与人、公司、项目、希望重点确认的事项。该信息会单独标记为用户提供，不会混入转写原文。"></textarea>
|
||||
<div class="actions"><button id="send" type="submit">上传并开始处理</button></div>
|
||||
<progress id="progress" max="100" value="0" hidden></progress>
|
||||
<div id="result" class="meta"></div></form></section>
|
||||
<section class="panel"><h2>最近任务</h2><div id="jobs"><div class="muted">正在读取…</div></div></section>"""
|
||||
script = """<script>
|
||||
const esc=s=>String(s??'').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));
|
||||
async function jobs(){
|
||||
try{const r=await fetch('/api/jobs',{cache:'no-store'}), d=await r.json();
|
||||
const box=document.querySelector('#jobs');
|
||||
box.innerHTML=d.jobs.length?d.jobs.map(j=>`<div class="job"><div class="jobtop">
|
||||
<strong>${esc(j.title||j.original_name)}</strong><span class="badge ${esc(j.status)}">${esc(j.status_label)}</span>
|
||||
</div><div class="meta">${new Date(j.created_at*1000).toLocaleString()} · ${esc(j.original_name)}
|
||||
${j.detail?'<br>'+esc(j.detail):''}</div>
|
||||
<a class="joblink" href="${esc(j.review_url)}">${j.status==='complete'?'查看任务':'查看 / 继续确认'}</a>
|
||||
</div>`).join(''):'<div class="muted">还没有任务</div>';
|
||||
}catch(e){document.querySelector('#jobs').textContent='任务状态暂时不可用';}}
|
||||
jobs(); setInterval(jobs,10000);
|
||||
document.querySelector('#upload').addEventListener('submit',e=>{
|
||||
e.preventDefault(); const f=document.querySelector('#audio').files[0]; if(!f)return;
|
||||
const x=new XMLHttpRequest(), b=document.querySelector('#send'), p=document.querySelector('#progress'), out=document.querySelector('#result');
|
||||
b.disabled=true;p.hidden=false;out.textContent='正在上传…';
|
||||
x.open('POST','/api/upload');x.setRequestHeader('Content-Type','application/octet-stream');
|
||||
x.setRequestHeader('X-Filename',encodeURIComponent(f.name));
|
||||
x.setRequestHeader('X-Title',encodeURIComponent(document.querySelector('#title').value));
|
||||
x.setRequestHeader('X-Context',encodeURIComponent(document.querySelector('#context').value));
|
||||
x.upload.onprogress=v=>{if(v.lengthComputable)p.value=Math.round(v.loaded/v.total*100)};
|
||||
x.onload=()=>{b.disabled=false;if(x.status===201){out.textContent='上传完成，已进入处理队列。';e.target.reset();p.value=0;p.hidden=true;jobs()}
|
||||
else out.textContent='上传失败：'+x.responseText};
|
||||
x.onerror=()=>{b.disabled=false;out.textContent='网络中断，请重试'};x.send(f);
|
||||
});
|
||||
</script>"""
|
||||
return _shell("录音工作台", body, script)
|
||||
|
||||
|
||||
def _review_page(action: str, job: dict | None = None) -> bytes:
|
||||
rows = _pending_for_job(job) if job else review.pending(100)
|
||||
if not rows:
|
||||
if job:
|
||||
label = STATUS_LABEL.get(job["status"], job["status"])
|
||||
detail = html.escape(str(job.get("detail") or ""))
|
||||
return _shell("任务状态",
|
||||
f'<section class="panel"><div class="meta">{html.escape(label)}</div>'
|
||||
f'<h1>{html.escape(str(job.get("title") or job["original_name"]))}</h1>'
|
||||
f'<p>{detail or "当前没有待确认问题。"}</p>'
|
||||
'<p class="muted">页面会随任务阶段更新，可以稍后刷新。</p></section>')
|
||||
return _shell("确认完成",
|
||||
'<section class="panel"><h1>当前没有待确认项目</h1>'
|
||||
'<p class="muted">新草稿生成后，Telegram 会再次发送这里的链接。</p></section>')
|
||||
row = rows[0]
|
||||
payload = json.loads(row["payload"] or "{}")
|
||||
choices = payload.get("candidates") or []
|
||||
buttons = "".join(
|
||||
f'<button class="choice" type="submit" name="answer" '
|
||||
f'value="{html.escape(str(v), quote=True)}">{html.escape(str(v))}</button>'
|
||||
for v in choices)
|
||||
buttons += ('<button class="choice secondary" type="submit" name="answer" '
|
||||
'value="无法确认，保留原文">都不是 / 无法确认</button>')
|
||||
source = html.escape(str(payload.get("sentence", "")))
|
||||
source_html = f'<div class="source">{source}</div>' if source else ""
|
||||
draft_html = ""
|
||||
if payload.get("draft_path"):
|
||||
try:
|
||||
path = Path(payload["draft_path"]).resolve()
|
||||
path.relative_to(CFG.drafts.resolve())
|
||||
draft = path.read_text(encoding="utf-8", errors="ignore")
|
||||
draft_html = ('<details class="draft"><summary>查看完整草稿</summary><pre>'
|
||||
+ html.escape(draft) + "</pre></details>")
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
placeholder = ("只输入姓名，例如：高瑶"
|
||||
if payload.get("kind") == "voiceprint"
|
||||
else ("背景不准时，写明正确背景并提交重跑"
|
||||
if payload.get("kind") == "background_check"
|
||||
else "输入正确名称或补充说明"))
|
||||
title = (str(job.get("title") or job["original_name"])
|
||||
if job else "ASR 快速确认")
|
||||
body = f"""<section class="panel"><div class="meta">{html.escape(title)} · 剩余 {len(rows)} 个 · 问题 #{row['id']}</div>
|
||||
<h1>{html.escape(row['question'])}</h1>{source_html}{draft_html}
|
||||
<form method="post" action="{html.escape(action, quote=True)}">
|
||||
<input type="hidden" name="id" value="{row['id']}">{buttons}
|
||||
<div class="custom"><input name="custom" maxlength="120" autocomplete="off"
|
||||
placeholder="{html.escape(placeholder, quote=True)}"><button type="submit">提交</button></div>
|
||||
</form></section>"""
|
||||
return _shell("ASR 快速确认", body)
|
||||
|
||||
|
||||
class Server(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
allow_reuse_address = True
|
||||
|
||||
def __init__(self, addr, handler):
|
||||
super().__init__(addr, handler)
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "ASRControl/1"
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
status = args[1] if len(args) > 1 else "?"
|
||||
log(f"control web: {self.command} {urllib.parse.urlsplit(self.path).path} status={status}")
|
||||
|
||||
def _headers(self, status=HTTPStatus.OK,
|
||||
content_type="text/html; charset=utf-8"):
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Cache-Control", "no-store, no-cache, must-revalidate")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
|
||||
def _send(self, data: bytes, status=HTTPStatus.OK,
|
||||
content_type="text/html; charset=utf-8"):
|
||||
self._headers(status, content_type)
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def _json(self, obj: object, status=HTTPStatus.OK):
|
||||
self._send(json.dumps(obj, ensure_ascii=False).encode("utf-8"),
|
||||
status, "application/json; charset=utf-8")
|
||||
|
||||
def _review_job(self) -> tuple[bool, dict | None]:
|
||||
path = urllib.parse.urlsplit(self.path).path
|
||||
if path == "/review/":
|
||||
return True, None
|
||||
match = re.fullmatch(r"/jobs/([0-9a-z-]{8,64})/review/", path)
|
||||
if match:
|
||||
return True, _job(match.group(1))
|
||||
return False, None
|
||||
|
||||
def do_GET(self):
|
||||
path = urllib.parse.urlsplit(self.path).path
|
||||
if path == "/":
|
||||
return self._send(_home())
|
||||
if path == "/health":
|
||||
return self._json({"ok": True, "pending": len(review.pending(100))})
|
||||
if path == "/api/jobs":
|
||||
jobs = _jobs()
|
||||
for j in jobs:
|
||||
j["status_label"] = STATUS_LABEL.get(j["status"], j["status"])
|
||||
j["review_url"] = f"/jobs/{j['id']}/review/"
|
||||
j["has_draft"] = bool(j.get("draft"))
|
||||
j.pop("context", None)
|
||||
j.pop("transcript", None)
|
||||
j.pop("draft", None)
|
||||
j.pop("result", None)
|
||||
return self._json({"jobs": jobs})
|
||||
is_review, job = self._review_job()
|
||||
if is_review:
|
||||
if path != "/review/" and not job:
|
||||
return self._send(b"Task not found", HTTPStatus.NOT_FOUND,
|
||||
"text/plain")
|
||||
return self._send(_review_page(path, job))
|
||||
return self._send(b"Not found", HTTPStatus.NOT_FOUND, "text/plain")
|
||||
|
||||
def _header(self, name: str, limit: int) -> str:
|
||||
return _safe_text(urllib.parse.unquote(self.headers.get(name, "")), limit)
|
||||
|
||||
def _upload(self):
|
||||
try:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
except ValueError:
|
||||
length = 0
|
||||
if length <= 0 or length > MAX_UPLOAD:
|
||||
return self._json({"error": "invalid upload size"},
|
||||
HTTPStatus.REQUEST_ENTITY_TOO_LARGE)
|
||||
original = self._header("X-Filename", 240)
|
||||
title = self._header("X-Title", 80)
|
||||
context = self._header("X-Context", 2000)
|
||||
suffix = Path(original).suffix.lower()
|
||||
if suffix not in SUPPORTED:
|
||||
return self._json({"error": f"unsupported file type: {suffix or '(none)'}"},
|
||||
HTTPStatus.UNSUPPORTED_MEDIA_TYPE)
|
||||
job_id = secrets.token_hex(12)
|
||||
stem = _safe_stem(title or Path(original).stem)
|
||||
stored = f"{time.strftime('%Y-%m-%d_%H_%M_%S')}-{stem}-{job_id[:8]}{suffix}"
|
||||
INBOX.mkdir(parents=True, exist_ok=True)
|
||||
staging = INBOX / ".uploading"
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
target = INBOX / stored
|
||||
part = staging / f"{job_id}.part"
|
||||
con = _connect()
|
||||
now = time.time()
|
||||
con.execute(
|
||||
"INSERT INTO intake_job(id,original_name,stored_name,title,context,size,"
|
||||
"status,created_at,updated_at) VALUES(?,?,?,?,?,?,?, ?,?)",
|
||||
(job_id, original, stored, title, context, length,
|
||||
"uploading", now, now))
|
||||
con.commit()
|
||||
con.close()
|
||||
remaining = length
|
||||
try:
|
||||
with part.open("xb") as f:
|
||||
while remaining:
|
||||
data = self.rfile.read(min(CHUNK, remaining))
|
||||
if not data:
|
||||
raise EOFError("upload ended early")
|
||||
f.write(data)
|
||||
remaining -= len(data)
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
META.mkdir(parents=True, exist_ok=True)
|
||||
meta = {
|
||||
"job_id": job_id, "title": title, "context": context,
|
||||
"original_name": original, "stored_name": stored,
|
||||
}
|
||||
meta_tmp = META / f".{stored}.tmp"
|
||||
meta_tmp.write_text(json.dumps(meta, ensure_ascii=False),
|
||||
encoding="utf-8")
|
||||
meta_tmp.replace(META / f"{stored}.json")
|
||||
part.replace(target)
|
||||
update_job(job_id, "queued", "录音已落盘，等待 watcher 接收")
|
||||
return self._json({"ok": True, "job_id": job_id,
|
||||
"stored_name": stored}, HTTPStatus.CREATED)
|
||||
except Exception as e:
|
||||
part.unlink(missing_ok=True)
|
||||
update_job(job_id, "failed", f"上传失败：{e}")
|
||||
return self._json({"error": str(e)}, HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
|
||||
def _answer(self):
|
||||
try:
|
||||
length = min(int(self.headers.get("Content-Length", "0")), 4096)
|
||||
form = urllib.parse.parse_qs(
|
||||
self.rfile.read(length).decode("utf-8"))
|
||||
qid = int(form.get("id", ["0"])[0])
|
||||
answer = (form.get("custom", [""])[0].strip()
|
||||
or form.get("answer", [""])[0].strip())
|
||||
if not qid or not answer or len(answer) > 120:
|
||||
raise ValueError("invalid answer")
|
||||
is_review, job = self._review_job()
|
||||
if not is_review:
|
||||
raise ValueError("invalid review path")
|
||||
allowed = (_pending_for_job(job, 1000) if job
|
||||
else review.pending(1000))
|
||||
if qid not in {row["id"] for row in allowed}:
|
||||
raise ValueError("问题不属于当前任务或已处理")
|
||||
review.answer(qid, answer)
|
||||
self.send_response(HTTPStatus.SEE_OTHER)
|
||||
self.send_header("Location", urllib.parse.urlsplit(self.path).path)
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
except Exception as e:
|
||||
self._send(_shell("提交失败",
|
||||
f'<section class="panel"><h1>提交失败</h1><p>{html.escape(str(e))}</p></section>'),
|
||||
HTTPStatus.BAD_REQUEST)
|
||||
|
||||
def do_POST(self):
|
||||
path = urllib.parse.urlsplit(self.path).path
|
||||
if path in ("/api/upload", "/hook/intake"):
|
||||
return self._upload()
|
||||
if self._review_job()[0]:
|
||||
return self._answer()
|
||||
return self._send(b"Not found", HTTPStatus.NOT_FOUND, "text/plain")
|
||||
|
||||
|
||||
def serve() -> int:
|
||||
try:
|
||||
server = Server((CFG.review_host, CFG.review_port), Handler)
|
||||
except OSError as e:
|
||||
STATE.unlink(missing_ok=True)
|
||||
log(f"control web bind failed on {CFG.review_host}:{CFG.review_port}: {e}",
|
||||
level="ERROR")
|
||||
return 1
|
||||
_write_state()
|
||||
log(f"control web listening on {CFG.review_public_host}:{CFG.review_port}")
|
||||
try:
|
||||
server.serve_forever(poll_interval=0.5)
|
||||
finally:
|
||||
server.server_close()
|
||||
STATE.unlink(missing_ok=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(serve())
|
||||
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env python3
|
||||
"""S3 — deterministic fidelity gate. No LLM, no cost.
|
||||
|
||||
Motivation (measured 2026-07-25): asking gemini-3.5-flash-lite to "去口语化，禁止
|
||||
省略" on 900–2500 char chunks silently dropped 17/49 distinct numeric tokens and
|
||||
136 distinct English tokens from a 118k-char BD call — including `Kras`, `IBD`,
|
||||
`EDC`, `IRT` and the figure `4.5` in "raised about 4.5 million". Output/input char
|
||||
ratio was 0.41–0.47 regardless of chunk size, i.e. the model summarizes when told
|
||||
to rewrite, and prompt wording alone does not stop it.
|
||||
|
||||
So fidelity cannot be asserted, it must be *checked*. This module compares a
|
||||
cleaned chunk against its source over token classes that must survive a purely
|
||||
stylistic rewrite:
|
||||
|
||||
numbers every digit run, as a multiset (口语 "四幺八二" survives as Han and is
|
||||
not expected here, but arabic figures must)
|
||||
latin acronyms / gene names / product codes ≥2 chars
|
||||
speakers the 【…】 turn tags
|
||||
han_terms domain terms from the term bank that appeared in the source
|
||||
|
||||
`check()` returns a verdict; the pipeline retries a failing chunk with a stricter
|
||||
prompt and, on a second failure, keeps the original text and flags it. That makes
|
||||
information loss a *visible, bounded* event instead of a silent one.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from .config import CFG
|
||||
|
||||
BANK = CFG.termbank_dir
|
||||
|
||||
NUM = re.compile(r"\d+(?:[.,]\d+)?%?")
|
||||
# S0's noise-collapse marker is synthetic metadata, not transcript content. Its
|
||||
# repeat count is a number ("[…重复噪声已折叠：beep×3696…]"), and scoring it as content
|
||||
# made one fixture report number_recall 0.00 off a single token that the cleaner had
|
||||
# merely reworded. Strip markers from both sides before any comparison.
|
||||
COLLAPSE_MARKER = re.compile(r"\[…重复噪声已折叠：.*?…\]")
|
||||
LATIN = re.compile(r"[A-Za-z][A-Za-z0-9\-]{1,}")
|
||||
SPEAKER = re.compile(r"【([^】]+)】")
|
||||
# Filler words a legitimate 去口语化 pass is expected to delete. Used only for the
|
||||
# expected-compression estimate, never for the pass/fail decision.
|
||||
FILLERS = ["嗯", "呃", "啊", "哦", "那个", "这个", "就是", "然后", "其实", "对对",
|
||||
"哈哈", "唉", "诶", "喔", "嘛", "吧", "呀", "咯", "哎"]
|
||||
|
||||
_STOP_LATIN = {
|
||||
"the", "and", "for", "you", "that", "this", "with", "have", "was",
|
||||
"are", "not", "but", "they", "our", "can", "all", "will", "would",
|
||||
"yeah", "okay", "ok", "uh", "um", "so", "we", "it", "is", "to",
|
||||
"of", "in", "on", "at", "be", "do", "if", "or", "as", "an", "a",
|
||||
"i", "he", "she", "his", "her", "them", "there", "then", "just",
|
||||
"very", "know", "think", "like", "get", "got", "one", "also",
|
||||
"from", "what", "when", "who", "how", "yes", "no", "right",
|
||||
# --- interjections and ordinary verbs the ASR emits in caps -------------
|
||||
# Measured cost of omitting these: on a 118k-char English-mixed call the gate
|
||||
# tripped on lost tokens "AH", "OH", "PAY", "BECAUSE", "EVEN", "WHEREAS" and
|
||||
# forced 53 retries for gemini-3.5-flash-lite, ~40% of them for interjections a
|
||||
# cleaner is *supposed* to delete. Treating them as content made the gate
|
||||
# expensive without making it safer.
|
||||
"ah", "oh", "eh", "hm", "hmm", "mm", "mhm", "huh", "wow", "hey", "hi",
|
||||
"well", "actually", "really", "maybe", "sure", "fine", "good", "great",
|
||||
"because", "even", "whereas", "however", "though", "although", "since",
|
||||
"while", "before", "after", "about", "into", "over", "under", "than",
|
||||
"such", "some", "any", "more", "most", "much", "many", "other", "same",
|
||||
"only", "still", "already", "always", "never", "again", "here", "where",
|
||||
"why", "which", "whose", "whom", "these", "those", "their", "your", "my",
|
||||
"me", "us", "him", "was", "were", "been", "being", "does", "did", "done",
|
||||
"has", "had", "having", "should", "could", "must", "may", "might", "shall",
|
||||
"want", "need", "make", "made", "take", "took", "give", "gave", "come",
|
||||
"came", "went", "said", "say", "says", "tell", "told", "see", "saw",
|
||||
"look", "mean", "means", "means", "let", "put", "keep", "kept", "find",
|
||||
"found", "feel", "felt", "seem", "seems", "sorry", "thanks", "thank",
|
||||
"please", "bye", "abc", "pay", "everybody", "everything", "something",
|
||||
"anything", "nothing", "everyone", "someone", "anyone", "nobody",
|
||||
"first", "second", "third", "last", "next", "both", "each", "every",
|
||||
"little", "big", "small", "long", "short", "high", "low", "new", "old",
|
||||
"time", "year", "day", "way", "thing", "things", "people", "person",
|
||||
"lot", "lots", "bit", "kind", "sort", "part", "point", "case", "fact",
|
||||
"course", "example", "problem", "question", "answer", "reason", "result",
|
||||
}
|
||||
|
||||
|
||||
def _load_terms() -> set[str]:
|
||||
p = BANK / "terms.json"
|
||||
if not p.exists():
|
||||
return set()
|
||||
return set(json.loads(p.read_text()).keys())
|
||||
|
||||
|
||||
_TERMS: set[str] | None = None
|
||||
|
||||
|
||||
def domain_terms() -> set[str]:
|
||||
global _TERMS
|
||||
if _TERMS is None:
|
||||
_TERMS = _load_terms()
|
||||
return _TERMS
|
||||
|
||||
|
||||
def _latin_tokens(text: str) -> Counter:
|
||||
"""Meaningful Latin tokens: acronyms, gene/product codes, CamelCase. Ordinary
|
||||
English function words are excluded — a cleaner legitimately rewrites
|
||||
"Yeah, so we, we did that" and must not be punished for it."""
|
||||
out = Counter()
|
||||
for t in LATIN.findall(text):
|
||||
low = t.lower()
|
||||
if low in _STOP_LATIN or len(t) < 2:
|
||||
continue
|
||||
# Keep: all-caps (EDC/IRT/GMP), mixed case with an inner capital (Kras is
|
||||
# lowercase-tail but capitalized; KRAS is caps), anything with a digit or
|
||||
# hyphen (BF4182, CD3-x), and capitalized proper nouns.
|
||||
if t.isupper() or any(c.isdigit() for c in t) or "-" in t or t[0].isupper():
|
||||
out[t.upper()] += 1
|
||||
return out
|
||||
|
||||
|
||||
def _han_terms(text: str, vocab: set[str]) -> Counter:
|
||||
out = Counter()
|
||||
for t in vocab:
|
||||
if len(t) >= 3 and t in text: # ≥3 chars: shorter ones collide constantly
|
||||
out[t] += text.count(t)
|
||||
return out
|
||||
|
||||
|
||||
@dataclass
|
||||
class Verdict:
|
||||
ok: bool
|
||||
ratio: float
|
||||
lost_numbers: list[str] = field(default_factory=list)
|
||||
lost_latin: list[str] = field(default_factory=list)
|
||||
lost_speakers: list[str] = field(default_factory=list)
|
||||
lost_terms: list[str] = field(default_factory=list)
|
||||
reasons: list[str] = field(default_factory=list)
|
||||
|
||||
def brief(self) -> str:
|
||||
if self.ok:
|
||||
return f"ok ratio={self.ratio:.2f}"
|
||||
return f"FAIL ratio={self.ratio:.2f} " + "; ".join(self.reasons)
|
||||
|
||||
def as_dict(self):
|
||||
return {
|
||||
"ok": self.ok, "ratio": round(self.ratio, 3),
|
||||
"lost_numbers": self.lost_numbers, "lost_latin": self.lost_latin,
|
||||
"lost_speakers": self.lost_speakers, "lost_terms": self.lost_terms,
|
||||
"reasons": self.reasons,
|
||||
}
|
||||
|
||||
|
||||
def check(src: str, out: str, *, min_ratio: float = 0.55,
|
||||
max_ratio: float = 1.35, vocab: set[str] | None = None,
|
||||
allow_lost_latin: int = 0) -> Verdict:
|
||||
"""Compare a cleaned chunk against its source.
|
||||
|
||||
min_ratio 0.55 is deliberately generous: real 去口语化 on filler-heavy Chinese
|
||||
speech legitimately removes 25–40% of characters. Anything below 0.55 is not
|
||||
cleaning, it is summarizing.
|
||||
"""
|
||||
vocab = domain_terms() if vocab is None else vocab
|
||||
src = COLLAPSE_MARKER.sub("", src)
|
||||
out = COLLAPSE_MARKER.sub("", out)
|
||||
ratio = len(out) / max(1, len(src))
|
||||
v = Verdict(ok=True, ratio=ratio)
|
||||
|
||||
if ratio < min_ratio:
|
||||
v.ok = False
|
||||
v.reasons.append(f"过度压缩(ratio {ratio:.2f}<{min_ratio})")
|
||||
if ratio > max_ratio:
|
||||
v.ok = False
|
||||
v.reasons.append(f"异常膨胀(ratio {ratio:.2f}>{max_ratio})")
|
||||
|
||||
sn, on = Counter(NUM.findall(src)), Counter(NUM.findall(out))
|
||||
lost_n = sorted((sn - on).elements())
|
||||
if lost_n:
|
||||
v.lost_numbers = lost_n
|
||||
v.ok = False
|
||||
v.reasons.append(f"丢数字{len(lost_n)}个:{lost_n[:6]}")
|
||||
|
||||
sl, ol = _latin_tokens(src), _latin_tokens(out)
|
||||
lost_l = sorted((sl - ol).elements())
|
||||
if len(lost_l) > allow_lost_latin:
|
||||
v.lost_latin = lost_l
|
||||
v.ok = False
|
||||
v.reasons.append(f"丢英文术语{len(lost_l)}个:{lost_l[:6]}")
|
||||
|
||||
ss, os_ = Counter(SPEAKER.findall(src)), Counter(SPEAKER.findall(out))
|
||||
lost_s = sorted(set((ss - os_).elements()))
|
||||
if lost_s:
|
||||
v.lost_speakers = lost_s
|
||||
v.ok = False
|
||||
v.reasons.append(f"丢说话人标签:{lost_s[:5]}")
|
||||
|
||||
if vocab:
|
||||
st, ot = _han_terms(src, vocab), _han_terms(out, vocab)
|
||||
lost_t = sorted(set((st - ot).elements()))
|
||||
if lost_t:
|
||||
v.lost_terms = lost_t
|
||||
v.ok = False
|
||||
v.reasons.append(f"丢领域术语{len(lost_t)}个:{lost_t[:6]}")
|
||||
|
||||
return v
|
||||
|
||||
|
||||
def expected_compression(src: str) -> float:
|
||||
"""Rough share of characters attributable to filler, for interpreting ratios."""
|
||||
n = sum(src.count(f) * len(f) for f in FILLERS)
|
||||
return n / max(1, len(src))
|
||||
|
||||
|
||||
def score_document(src: str, out: str, vocab: set[str] | None = None) -> dict:
|
||||
"""Document-level metrics for the横评 report (not a gate)."""
|
||||
vocab = domain_terms() if vocab is None else vocab
|
||||
src = COLLAPSE_MARKER.sub("", src)
|
||||
out = COLLAPSE_MARKER.sub("", out)
|
||||
sn, on = Counter(NUM.findall(src)), Counter(NUM.findall(out))
|
||||
sl, ol = _latin_tokens(src), _latin_tokens(out)
|
||||
st, ot = _han_terms(src, vocab), _han_terms(out, vocab)
|
||||
ss, os_ = set(SPEAKER.findall(src)), set(SPEAKER.findall(out))
|
||||
|
||||
def recall(a: Counter, b: Counter) -> float:
|
||||
tot = sum(a.values())
|
||||
if not tot:
|
||||
return 1.0
|
||||
kept = sum(min(c, b[k]) for k, c in a.items())
|
||||
return kept / tot
|
||||
|
||||
return {
|
||||
"char_ratio": round(len(out) / max(1, len(src)), 3),
|
||||
"filler_share": round(expected_compression(src), 3),
|
||||
"number_recall": round(recall(sn, on), 3),
|
||||
"latin_recall": round(recall(sl, ol), 3),
|
||||
"term_recall": round(recall(st, ot), 3),
|
||||
"speaker_recall": round(len(ss & os_) / len(ss), 3) if ss else 1.0,
|
||||
"src_numbers": sum(sn.values()), "src_latin": sum(sl.values()),
|
||||
"src_terms": sum(st.values()), "src_speakers": len(ss),
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
src = Path(sys.argv[1]).read_text(errors="ignore")
|
||||
out = Path(sys.argv[2]).read_text(errors="ignore")
|
||||
print(json.dumps(score_document(src, out), ensure_ascii=False, indent=2))
|
||||
print(check(src, out).brief())
|
||||
@@ -0,0 +1,956 @@
|
||||
#!/usr/bin/env python3
|
||||
"""S1 — entity/term candidate flagger v3. Local, no LLM cost.
|
||||
|
||||
Inherits the correct design of the production v2 script
|
||||
(`OBSIDIAN/bin/obsidian-asr-phonetic-correct.py`): phonetics raise a flag,
|
||||
*sentence context decides*, so this stays a high-recall flagger that never
|
||||
rewrites anything. It emits candidates and lets the S2 cleaning model judge each
|
||||
one against the sentence it sits in — the step that correctly rejects
|
||||
主要(adverb)→朱岩 while accepting 中药("找大涛")→王宗耀.
|
||||
|
||||
Three changes over v2, each fixing a measured problem:
|
||||
|
||||
1. **Speed.** v2 took 31s per transcript: it re-scanned all 349 minutes notes on
|
||||
every run and did a pure-Python O(n·m) syllable DP against every candidate.
|
||||
v3 caches the corpus scan on disk (keyed by directory mtimes) and pre-buckets
|
||||
candidates by (length, first-syllable-initial), so the DP only runs on
|
||||
plausible pairs. Target <5s.
|
||||
|
||||
2. **Precision.** v2 offered 「分钟」→冯昆仑 and 「很远」→恒元生物 — ordinary words
|
||||
displaced by entities. v3 consults the term-bank allowlist (1,644 high-frequency
|
||||
general words from THUOCL): a span that IS a common word is demoted, not
|
||||
dropped, and carries an explicit `common_word: true` warning so the model
|
||||
defaults to keeping it.
|
||||
|
||||
3. **Coverage.** v2 only searched vault entities. v3 adds the domain term bank
|
||||
(CDMO/CMC jargon, 239 Chinese INN drug names) and INN suffix detection, so it
|
||||
flags 批记录 mis-hearings (披露/P图) and drug names never seen in the vault.
|
||||
|
||||
Output is JSON: candidates grouped per chunk index, ready to inject into S2.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from .config import CFG
|
||||
|
||||
LAB = CFG.root
|
||||
BANK = CFG.termbank_dir
|
||||
CACHE = CFG.cache_dir / "flagger"
|
||||
VAULT = CFG.vault
|
||||
|
||||
HS_API = CFG.hindsight_api
|
||||
HS_BANK = CFG.hindsight_bank
|
||||
HS_TIMEOUT = CFG.hindsight_timeout
|
||||
|
||||
HAN = re.compile(r"[\u4e00-\u9fff]+")
|
||||
HAN_FULL = re.compile(r"^[\u4e00-\u9fff]+$")
|
||||
SPEAKER_TAG = re.compile(r"^\s*[【\[]([^\]】]+)[】\]]\s*")
|
||||
SENT_SPLIT = re.compile(r"[。！？!?…\n；;]+")
|
||||
INITIALS = ["zh", "ch", "sh", "b", "p", "m", "f", "d", "t", "n", "l", "g", "k",
|
||||
"h", "j", "q", "x", "r", "z", "c", "s", "y", "w"]
|
||||
FOLD = {"zh": "z", "ch": "c", "sh": "s"}
|
||||
|
||||
try:
|
||||
from pypinyin import Style, lazy_pinyin
|
||||
except ImportError: # pragma: no cover - fall back to the vendored copy
|
||||
sys.path.insert(0, str(CFG.obsidian / "bin" / "_vendor"))
|
||||
from pypinyin import Style, lazy_pinyin
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ phonetics
|
||||
def norm_syllable(s: str) -> tuple[str, str]:
|
||||
init, fin = "", s
|
||||
for cand in INITIALS:
|
||||
if s.startswith(cand):
|
||||
init, fin = cand, s[len(cand):]
|
||||
break
|
||||
init = FOLD.get(init, init)
|
||||
if fin.endswith("ng"):
|
||||
fin = fin[:-1]
|
||||
# Input-method style equivalences beyond the v2 set: these are the confusions
|
||||
# a Mandarin speaker (and therefore an ASR acoustic model) actually makes.
|
||||
fin = fin.replace("uang", "uan").replace("iang", "ian")
|
||||
if fin in ("en", "eng"):
|
||||
fin = "en"
|
||||
if fin in ("in", "ing"):
|
||||
fin = "in"
|
||||
return init, fin
|
||||
|
||||
|
||||
def to_syls(text: str) -> list[tuple[str, str]]:
|
||||
bare = lazy_pinyin(text)
|
||||
toned = lazy_pinyin(text, style=Style.TONE3, neutral_tone_with_five=True)
|
||||
out = []
|
||||
for b, t in zip(bare, toned):
|
||||
if not re.match(r"[a-z]", b):
|
||||
continue
|
||||
out.append((b, t[len(b):] if t.startswith(b) else ""))
|
||||
return out
|
||||
|
||||
|
||||
_PERSONAL_SYL: dict[tuple[tuple[str, str], tuple[str, str]], int] | None = None
|
||||
|
||||
|
||||
def personal_syllable_confusions() -> dict:
|
||||
"""Learn directional heard->intended syllable errors from confirmed fixes."""
|
||||
global _PERSONAL_SYL
|
||||
if _PERSONAL_SYL is not None:
|
||||
return _PERSONAL_SYL
|
||||
out: dict[tuple[tuple[str, str], tuple[str, str]], int] = {}
|
||||
try:
|
||||
from . import lexicon
|
||||
for heard, targets in lexicon.confusion_map().items():
|
||||
hs = to_syls(heard)
|
||||
for intended, n, confirmed in targets:
|
||||
if not confirmed:
|
||||
continue
|
||||
ts = to_syls(intended)
|
||||
if len(hs) != len(ts):
|
||||
continue
|
||||
for h, t in zip(hs, ts):
|
||||
hk, tk = norm_syllable(h[0]), norm_syllable(t[0])
|
||||
if hk == tk:
|
||||
continue
|
||||
out[(hk, tk)] = out.get((hk, tk), 0) + max(1, n)
|
||||
except Exception:
|
||||
pass
|
||||
_PERSONAL_SYL = out
|
||||
return out
|
||||
|
||||
|
||||
def syl_sim(a, b, *, personal: bool = False) -> float:
|
||||
(ap, at), (bp, bt) = a, b
|
||||
ai, af = norm_syllable(ap)
|
||||
bi, bf = norm_syllable(bp)
|
||||
if (ai, af) == (bi, bf):
|
||||
base = 1.0
|
||||
else:
|
||||
base = 0.0
|
||||
if ai == bi:
|
||||
base += 0.5
|
||||
elif {ai, bi} <= {"n", "l"} or {ai, bi} <= {"f", "h"}:
|
||||
base += 0.35 # n/l and f/h merge in many southern accents
|
||||
if af == bf:
|
||||
base += 0.5
|
||||
elif af and bf and af[0] == bf[0]:
|
||||
base += 0.2
|
||||
learned_n = (personal_syllable_confusions().get(((ai, af), (bi, bf)), 0)
|
||||
if personal else 0)
|
||||
if learned_n:
|
||||
# Direction matters: this is how the ASR wrote the sound -> what Kai
|
||||
# confirmed. One confirmation is strong evidence, repetition raises it
|
||||
# gradually, and graph/in-play gating still limits which entities are active.
|
||||
return max(base, min(0.98, 0.89 + 0.03 * math.log1p(learned_n)))
|
||||
return base * (1.0 if at == bt and at else 0.94)
|
||||
|
||||
|
||||
def seq_score(a, b, *, personal: bool = False) -> float:
|
||||
n, m = len(a), len(b)
|
||||
if not n or not m:
|
||||
return 0.0
|
||||
dp = [[0.0] * (m + 1) for _ in range(n + 1)]
|
||||
for i in range(n + 1):
|
||||
dp[i][0] = i
|
||||
for j in range(m + 1):
|
||||
dp[0][j] = j
|
||||
for i in range(1, n + 1):
|
||||
ai = a[i - 1]
|
||||
for j in range(1, m + 1):
|
||||
dp[i][j] = min(dp[i - 1][j - 1] +
|
||||
(1 - syl_sim(ai, b[j - 1], personal=personal)),
|
||||
dp[i - 1][j] + 1, dp[i][j - 1] + 1)
|
||||
return 1 - dp[n][m] / max(n, m)
|
||||
|
||||
|
||||
# ------------------------------------------------------------ candidate pool
|
||||
@dataclass
|
||||
class Cand:
|
||||
form: str # the surface form to match phonetically
|
||||
canonical: str # what to correct to
|
||||
kind: str # 人名 / 公司 / 工艺术语 / 药品通用名 / …
|
||||
source: str
|
||||
weight: float = 0.0 # in-play priority
|
||||
syls: list = field(default_factory=list)
|
||||
rule: str = "full" # how `form` was derived from `canonical` (see RULE_MIN)
|
||||
owner: str = "" # affiliation (entity.company) used for disambiguation
|
||||
role: str = ""
|
||||
anchor: int = 0 # honorific forms: +1 = leading particle, -1 = trailing
|
||||
|
||||
|
||||
# How close a span must sound to a candidate form, per derivation rule. These are
|
||||
# not arbitrary: a rule that carries extra structural evidence can afford a lower
|
||||
# phonetic bar, and a rule with little structure needs a higher one.
|
||||
RULE_MIN = {
|
||||
"full": 0.86, # whole-name homophone, the v3 behaviour
|
||||
"alias": 1.0, # exact card/learned alias — an identity binding, not a guess
|
||||
"org-short": 0.84, # 迈威(生物): the dropped tail is itself evidence
|
||||
"given": 0.88, # 光辉←李光辉: short and riskier, so stricter
|
||||
"honorific": 0.82, # X总/老X: the title is a strong anchor, only X must match
|
||||
"one-off": 0.85, # same length, one syllable substituted
|
||||
"consistency": 0.80, # another spelling of a target already established in-doc
|
||||
"observed": 0.90, # fuzzy neighbour of a confirmed historical ASR error
|
||||
}
|
||||
|
||||
ORG_TAIL = ("生物医药", "生物科技", "有限公司", "股份公司", "生物", "医药", "制药",
|
||||
"药业", "基金", "资本", "投资", "科技", "集团", "股份", "公司", "银行",
|
||||
"证券", "医院", "研究院", "实验室")
|
||||
HONOR_PREFIX = ("老", "小")
|
||||
HONOR_SUFFIX = ("总", "博", "哥", "姐", "老师", "董") # 博=博士: 张博→张成海, 许博→许俊彦
|
||||
|
||||
|
||||
def org_head(name: str) -> str:
|
||||
"""迈威生物→迈威, 宝山基金→宝山. The form people actually say out loud."""
|
||||
for t in ORG_TAIL:
|
||||
if name.endswith(t) and len(name) - len(t) >= 2:
|
||||
return name[:len(name) - len(t)]
|
||||
return ""
|
||||
|
||||
|
||||
def variants(canonical: str, kind: str) -> list[tuple[str, str, int]]:
|
||||
"""Spoken surface forms of an entity, as (form, rule).
|
||||
|
||||
This is the fix for the dominant failure mode measured on 2026-07-21: the
|
||||
matcher only compared whole strings of near-equal length, so every colloquial
|
||||
short form was unreachable — 麦威 could not find 迈威生物, 光辉 could not find
|
||||
李光辉, and 老万 was actively discarded because it matched an alias *exactly*.
|
||||
"""
|
||||
out: list[tuple[str, str, int]] = []
|
||||
if kind in ("公司", "机构"):
|
||||
h = org_head(canonical)
|
||||
if 2 <= len(h) <= 5:
|
||||
out.append((h, "org-short", 0))
|
||||
return out
|
||||
if kind != "人名" or not (2 <= len(canonical) <= 4):
|
||||
return out
|
||||
surname, given = canonical[0], canonical[1:]
|
||||
if len(given) >= 2:
|
||||
out.append((given, "given", 0)) # 光辉 ← 李光辉
|
||||
for p in HONOR_PREFIX:
|
||||
out.append((p + surname, "honorific", +1)) # 老万 / 小万
|
||||
for suf in HONOR_SUFFIX:
|
||||
out.append((surname + suf, "honorific", -1)) # 万总 / 万博
|
||||
if len(given) == 1:
|
||||
out.append((given + suf, "honorific", -1)) # 洵总 ← 邹洵
|
||||
return out
|
||||
|
||||
|
||||
def _dir_sig(paths) -> str:
|
||||
h = hashlib.sha256()
|
||||
for p in paths:
|
||||
p = Path(p)
|
||||
if p.exists():
|
||||
if p.is_file():
|
||||
st = p.stat()
|
||||
h.update(f"{p}:{st.st_mtime_ns}:{st.st_size}".encode())
|
||||
continue
|
||||
files = sorted(p.glob("*.md"))
|
||||
h.update(f"{p}:{len(files)}".encode())
|
||||
for f in files:
|
||||
st = f.stat()
|
||||
h.update(f"{f.name}:{st.st_mtime_ns}:{st.st_size}".encode())
|
||||
return h.hexdigest()[:16]
|
||||
|
||||
|
||||
def load_pool(use_termbank=True) -> dict[str, Cand]:
|
||||
"""Vault entities (cards + glossary) ∪ optional domain term bank."""
|
||||
CACHE.mkdir(parents=True, exist_ok=True)
|
||||
if CFG.use_lexicon:
|
||||
try:
|
||||
from . import lexicon
|
||||
lexicon.sync_from_vault()
|
||||
except Exception:
|
||||
pass
|
||||
sig = _dir_sig([VAULT / "People", VAULT / "Companies", VAULT / "minutes",
|
||||
CFG.lexicon_db])
|
||||
cpath = CACHE / f"pool6-{sig}-{int(use_termbank)}.json"
|
||||
if cpath.exists():
|
||||
d = json.loads(cpath.read_text())
|
||||
return {k: Cand(**v) for k, v in d.items()}
|
||||
|
||||
pool: dict[str, Cand] = {}
|
||||
|
||||
def add(form, canonical, kind, source, rule="full", owner="", role="", anchor=0):
|
||||
"""Register a matchable form.
|
||||
|
||||
Keyed by (form, rule, canonical) rather than form alone: 迈威 is both an
|
||||
exact card alias of 迈威生物 *and* its org-short form. Keeping only the
|
||||
alias entry made 麦威 unmatchable, because the alias rule matches exactly
|
||||
and never phonetically. Both entries must coexist.
|
||||
"""
|
||||
form = (form or "").strip()
|
||||
if not (2 <= len(form) <= 12) or not HAN_FULL.match(form):
|
||||
return
|
||||
key = f"{form}\x00{rule}\x00{canonical}"
|
||||
if key in pool:
|
||||
return
|
||||
pool[key] = Cand(form, canonical, kind, source, syls=to_syls(form),
|
||||
rule=rule, owner=owner, role=role, anchor=anchor)
|
||||
|
||||
# Entities and their aliases come from the SQLite lexicon, which mirrors the
|
||||
# vault cards (read-only) and additionally holds anything Kai has confirmed.
|
||||
ents: dict = {}
|
||||
if CFG.use_lexicon:
|
||||
try:
|
||||
from . import lexicon
|
||||
ents = lexicon.entities()
|
||||
for name, e in ents.items():
|
||||
add(name, name, e.kind, "lexicon-entity", "full", e.company, e.role)
|
||||
for al, targets in lexicon.alias_map().items():
|
||||
# Every target, not just the first. The glossary deliberately keeps
|
||||
# one-to-many rows (许博→许俊彦/许恒皓, 徐总→徐劲前/邹洵) because they
|
||||
# are context-dependent; collapsing them to one silently discarded
|
||||
# the correct reading.
|
||||
for canon, kind, origin, _conf in targets:
|
||||
e = ents.get(canon)
|
||||
add(al, canon, kind, f"alias-{origin}", "alias",
|
||||
e.company if e else "", e.role if e else "")
|
||||
# Exact confirmed errors are already aliases. This second form is
|
||||
# deliberately gated on in-play and allows a nearby decoder variant
|
||||
# (张台海) to recall the same canonical person learned from
|
||||
# 张泰海→张成海 without enabling global fuzzy matching.
|
||||
for heard, targets in lexicon.confusion_map().items():
|
||||
for canon, _n, confirmed in targets:
|
||||
if not confirmed:
|
||||
continue
|
||||
e = ents.get(canon)
|
||||
add(heard, canon, e.kind if e else lexicon.guess_kind(canon),
|
||||
"observed-correction", "observed",
|
||||
e.company if e else "", e.role if e else "")
|
||||
except Exception:
|
||||
ents = {}
|
||||
if not ents: # lexicon disabled/broken
|
||||
for sub, kind in (("People", "人名"), ("Companies", "公司")):
|
||||
d = VAULT / sub
|
||||
if not d.exists():
|
||||
continue
|
||||
for f in d.glob("*.md"):
|
||||
add(f.stem, f.stem, kind, f"card-{sub.lower()}")
|
||||
|
||||
# Spoken short forms. Generated for every entity; only the in-play ones are
|
||||
# activated at match time, so this costs nothing in precision.
|
||||
for c in list(pool.values()):
|
||||
if c.rule != "full" or c.kind not in ("人名", "公司"):
|
||||
continue
|
||||
for form, rule, anchor in variants(c.canonical, c.kind):
|
||||
add(form, c.canonical, c.kind, "variant", rule, c.owner, c.role, anchor)
|
||||
|
||||
if use_termbank:
|
||||
tb = BANK / "terms.json"
|
||||
if tb.exists():
|
||||
for term, e in json.loads(tb.read_text()).items():
|
||||
# Only the on-domain, distinctive slices are worth phonetic search;
|
||||
# 27k THUOCL general words would flood the candidate space with the
|
||||
# very false positives v3 is trying to eliminate.
|
||||
if e["kind"] in ("工艺术语", "药品通用名", "机构", "已确认纠错目标"):
|
||||
add(term, term, e["kind"], e["sources"][0])
|
||||
|
||||
cpath.write_text(json.dumps({k: v.__dict__ for k, v in pool.items()},
|
||||
ensure_ascii=False), encoding="utf-8")
|
||||
return pool
|
||||
|
||||
|
||||
def load_allowlist() -> set[str]:
|
||||
p = BANK / "allowlist.txt"
|
||||
return set(p.read_text().split()) if p.exists() else set()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- in-play
|
||||
# A wikilink in a minutes note points at other notes as often as at entities.
|
||||
# Unfiltered, 25 of the 70 in-play slots on 2026-07-21 were note titles, bare dates
|
||||
# and `_glossary` — burning candidate space and prompt budget on non-entities.
|
||||
NOT_ENTITY = re.compile(r"^\d{4}-\d{2}|^_|\.md$|[|/#]|^\d+$")
|
||||
|
||||
|
||||
def _is_entity_name(s: str) -> bool:
|
||||
s = s.strip()
|
||||
return bool(s) and 2 <= len(s) <= 12 and not NOT_ENTITY.search(s)
|
||||
|
||||
|
||||
def _minutes_entities(text: str) -> set[str]:
|
||||
out = set()
|
||||
for key in ("people", "companies"):
|
||||
m = re.search(rf"^{key}:[ \t]*(.*)$", text, re.M)
|
||||
if m:
|
||||
out |= {x for x in re.findall(r"[^,\[\]\s，、]+", m.group(1))}
|
||||
out |= {w.strip() for w in re.findall(r"\[\[([^\]|#]+)", text)}
|
||||
return {o for o in out if _is_entity_name(o)}
|
||||
|
||||
|
||||
def _corpus() -> list[tuple[str, set[str], str]]:
|
||||
CACHE.mkdir(parents=True, exist_ok=True)
|
||||
sig = _dir_sig([VAULT / "minutes"])
|
||||
cpath = CACHE / f"corpus-{sig}.json"
|
||||
if cpath.exists():
|
||||
return [(a, set(b), c) for a, b, c in json.loads(cpath.read_text())]
|
||||
out = []
|
||||
for f in sorted((VAULT / "minutes").glob("*.md")):
|
||||
txt = f.read_text(errors="ignore")
|
||||
m = re.search(r"^date:\s*(\d{4}-\d{2}-\d{2})", txt, re.M)
|
||||
out.append((f.name, _minutes_entities(txt), m.group(1) if m else ""))
|
||||
cpath.write_text(json.dumps([(a, sorted(b), c) for a, b, c in out],
|
||||
ensure_ascii=False), encoding="utf-8")
|
||||
return out
|
||||
|
||||
|
||||
def hindsight_recall(seed: list[str], extra_query: str = "") -> tuple[set[str], list[str]]:
|
||||
"""Semantic neighbourhood from Hindsight. Returns (entity names, memory texts).
|
||||
|
||||
The memory texts matter as much as the names: injected into S2/S4 they give the
|
||||
model the *relationship* ("吴楠 is 刘冀's colleague at 跨境资本") that lets it
|
||||
accept or reject a phonetic candidate on business grounds, not just sound.
|
||||
"""
|
||||
if not seed and not extra_query:
|
||||
return set(), []
|
||||
query = (" ".join(sorted(seed)) + " " + extra_query).strip() or "会议 相关人物 公司"
|
||||
body = json.dumps({"query": query, "max_tokens": 1500}).encode()
|
||||
url = f"{HS_API}/v1/default/banks/{HS_BANK}/memories/recall"
|
||||
try:
|
||||
req = urllib.request.Request(url, data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=HS_TIMEOUT) as r:
|
||||
d = json.loads(r.read())
|
||||
except Exception:
|
||||
return set(), []
|
||||
names = set(d.get("entities", {}).keys())
|
||||
texts = []
|
||||
for r_ in d.get("results", []):
|
||||
names.update(r_.get("entities", []))
|
||||
if r_.get("text"):
|
||||
texts.append(r_["text"])
|
||||
return ({n for n in names if 2 <= len(n) <= 8 and HAN_FULL.match(n)}, texts)
|
||||
|
||||
|
||||
def _super_nodes(participants: set[str], corpus) -> set[str]:
|
||||
if len(participants) < 2:
|
||||
return set()
|
||||
freq = {p: sum(1 for _, e, _ in corpus if p in e) for p in participants}
|
||||
fmin = max(1, min(freq.values()))
|
||||
hubs = {p for p in participants if freq[p] >= 20 and freq[p] >= 3 * fmin}
|
||||
return hubs if len(hubs) < len(participants) else set()
|
||||
|
||||
|
||||
def compute_in_play(participants: set[str], recent_days=90, use_hindsight=True,
|
||||
extra_seed: set[str] | None = None):
|
||||
corpus = _corpus()
|
||||
hubs = _super_nodes(participants, corpus)
|
||||
seed = ((participants - hubs) or participants) | (extra_seed or set())
|
||||
dates = [d for _, _, d in corpus if d]
|
||||
ref = max(dates) if dates else ""
|
||||
weight = {p: 999.0 for p in participants}
|
||||
for _, ents, d in corpus:
|
||||
if not (ents & seed):
|
||||
continue
|
||||
if ref and d and recent_days:
|
||||
try:
|
||||
from datetime import date
|
||||
a = date.fromisoformat(d)
|
||||
b = date.fromisoformat(ref)
|
||||
if (b - a).days > recent_days:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
for e in ents:
|
||||
weight[e] = weight.get(e, 0) + 1
|
||||
try:
|
||||
from . import graph
|
||||
graph.sync()
|
||||
for entity, score in graph.context_weights(seed).items():
|
||||
weight[entity] = weight.get(entity, 0.0) + score
|
||||
except Exception:
|
||||
pass
|
||||
in_play = set(sorted(weight, key=lambda e: -weight[e])[:60])
|
||||
hs_names, hs_texts = hindsight_recall(sorted(seed)) if use_hindsight else (set(), [])
|
||||
return in_play | hs_names, {"hubs": sorted(hubs), "hindsight_names": sorted(hs_names),
|
||||
"hindsight_texts": hs_texts, "weight": weight}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- flagging
|
||||
def sentences_of(text: str):
|
||||
for raw in text.splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith(("#", "<!--", "---")):
|
||||
continue
|
||||
spk = ""
|
||||
m = SPEAKER_TAG.match(line)
|
||||
if m:
|
||||
spk = m.group(1)
|
||||
line = SPEAKER_TAG.sub("", line)
|
||||
for piece in SENT_SPLIT.split(line):
|
||||
piece = piece.strip()
|
||||
if piece:
|
||||
yield spk, piece
|
||||
|
||||
|
||||
def phon_key(syls: list[tuple[str, str]]) -> str:
|
||||
"""Collapse a word to an accent-folded consonant/vowel skeleton.
|
||||
|
||||
This is the input-method trick: 中药 and 宗耀 both become "zonyao", so an exact
|
||||
dict lookup finds the homophone class instantly and only genuinely ambiguous
|
||||
pairs need the expensive alignment. Tones are dropped here and used later as a
|
||||
tiebreak, mirroring how a listener hears a name they半-recognize.
|
||||
"""
|
||||
out = []
|
||||
for bare, _ in syls:
|
||||
i, f = norm_syllable(bare)
|
||||
out.append(i + f)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def flag_text(text: str, pool: dict[str, Cand], in_play: set[str],
|
||||
allowlist: set[str], min_score=0.86, use_termbank_global=True,
|
||||
entity_weights: dict[str, float] | None = None):
|
||||
"""Find phonetic near-matches. Entity candidates are gated on being in-play;
|
||||
domain terms (工艺术语/药品通用名) are always searched, since a drug or a
|
||||
批记录 can appear in a meeting with no prior vault history.
|
||||
|
||||
Performance note: the v2 approach (score every span against every candidate
|
||||
with a Python syllable DP) is O(spans x candidates x syllables^2) and took
|
||||
>400s once the term bank widened the candidate pool. v3 first buckets
|
||||
candidates by phonetic key and length, uses rapidfuzz (C) to shortlist keys,
|
||||
and only then runs the fine-grained tonal alignment on survivors.
|
||||
"""
|
||||
from rapidfuzz import process as rf_process
|
||||
|
||||
# An in-play name with no candidate entry can never be matched. 宝山基金 has no
|
||||
# Companies card — it reached in-play only through a minutes filename — so
|
||||
# 保山 (a perfect homophone of its short form) had nothing to match against.
|
||||
# Materialise those names, including their spoken short forms.
|
||||
pool = dict(pool)
|
||||
# Skip names already reachable as *any* form, not just as a canonical. 迈威 is
|
||||
# the alias/short form of 迈威生物 and also appears in in-play as a bare name;
|
||||
# materialising it again produced two competing candidates for one company.
|
||||
known = {c.canonical for c in pool.values()} | {c.form for c in pool.values()}
|
||||
for name in sorted(in_play):
|
||||
if name in known or not _is_entity_name(name) or not HAN_FULL.match(name):
|
||||
continue
|
||||
is_org = name.endswith(ORG_TAIL)
|
||||
kind = "公司" if is_org else "人名"
|
||||
if not is_org and not (2 <= len(name) <= 4):
|
||||
continue
|
||||
# Variants are only derived for names whose type is *known*. An in-play name
|
||||
# with no card is a guess already; 三金 (short for 桂林三金) was being read as
|
||||
# a surname and generating 老三, which then matched 老尚. Only an explicit
|
||||
# organisation suffix is trustworthy enough to expand here.
|
||||
forms = [(name, "full", 0)] + (variants(name, kind) if is_org else [])
|
||||
for form, rule, anchor in forms:
|
||||
if not (2 <= len(form) <= 12 and HAN_FULL.match(form)):
|
||||
continue
|
||||
pool.setdefault(f"{form}\x00{rule}\x00{name}",
|
||||
Cand(form, name, kind, "in-play", syls=to_syls(form),
|
||||
rule=rule, anchor=anchor))
|
||||
|
||||
active: list[Cand] = []
|
||||
for c in pool.values():
|
||||
if c.rule == "alias":
|
||||
# Curated identity bindings (card `aliases:` + imported glossary). These
|
||||
# are hand-confirmed and matched exactly, so gating them on in-play
|
||||
# would only lose recall — this is what silently dropped 老万→万亚坤.
|
||||
active.append(c)
|
||||
elif c.kind in ("人名", "公司", "项目", "已确认目标"):
|
||||
if c.canonical in in_play or c.form in in_play:
|
||||
active.append(c)
|
||||
elif use_termbank_global:
|
||||
active.append(c)
|
||||
|
||||
# key -> candidates sharing that phonetic skeleton
|
||||
by_key: dict[str, list[Cand]] = {}
|
||||
for c in active:
|
||||
by_key.setdefault(phon_key(c.syls), []).append(c)
|
||||
keys = list(by_key)
|
||||
lens = {len(c.form) for c in active} or {2, 3}
|
||||
widths = sorted({w for L in lens for w in (max(2, L - 1), L, L + 1)})
|
||||
|
||||
# Collect every distinct span once; scoring a span is independent of where it
|
||||
# occurs, so we score uniques and then attribute hits back to sentences.
|
||||
span_positions: dict[str, tuple[str, str]] = {}
|
||||
for spk, sent in sentences_of(text):
|
||||
for run in HAN.findall(sent):
|
||||
for w in widths:
|
||||
for i in range(len(run) - w + 1):
|
||||
span_positions.setdefault(run[i:i + w], (spk, sent))
|
||||
|
||||
span_keys: dict[str, tuple[str, list]] = {}
|
||||
for span in span_positions:
|
||||
syls = to_syls(span)
|
||||
if syls:
|
||||
span_keys[span] = (phon_key(syls), syls)
|
||||
|
||||
findings: dict[tuple, dict] = {}
|
||||
|
||||
def consider(span, cand, syls):
|
||||
# Already the canonical name: nothing to say.
|
||||
if span == cand.canonical:
|
||||
return
|
||||
rule = cand.rule
|
||||
if span == cand.form:
|
||||
# Exact hit on a non-canonical form. Previously this returned early —
|
||||
# which is precisely why 老万 (an exact card alias of 万亚坤) and 迈威
|
||||
# produced no output at all and the drafting model had to guess.
|
||||
sc = 1.0
|
||||
else:
|
||||
if abs(len(span) - len(cand.form)) > 1:
|
||||
return
|
||||
if rule == "alias":
|
||||
return # curated aliases match exactly or not at all
|
||||
if rule == "honorific":
|
||||
# 老X / X总 / X博. The particle carries no identity information, so
|
||||
# it must match *literally*; allowing it to drift turned this rule
|
||||
# into a generic 2-syllable fuzzy match and produced 想谈→谭凯,
|
||||
# 理解→李光辉, 就动→刘冀 — 36 junk flags on one transcript.
|
||||
if len(span) != len(cand.form):
|
||||
return
|
||||
if cand.anchor > 0:
|
||||
if span[0] != cand.form[0]:
|
||||
return
|
||||
a, b = span[1:], cand.form[1:]
|
||||
elif cand.anchor < 0:
|
||||
if span[-1] != cand.form[-1]:
|
||||
return
|
||||
a, b = span[:-1], cand.form[:-1]
|
||||
else:
|
||||
return
|
||||
sc = seq_score(
|
||||
to_syls(a), to_syls(b),
|
||||
personal=(cand.kind in ("人名", "公司", "项目")
|
||||
and cand.canonical in in_play))
|
||||
if sc < 0.86:
|
||||
return
|
||||
else:
|
||||
sc = seq_score(
|
||||
syls, cand.syls,
|
||||
personal=(cand.kind in ("人名", "公司", "项目")
|
||||
and cand.canonical in in_play))
|
||||
if sc < RULE_MIN.get(rule, min_score):
|
||||
return
|
||||
spk, sent = span_positions[span]
|
||||
key = (span, cand.canonical)
|
||||
prev = findings.get(key)
|
||||
# On a tie, keep the curated alias rather than the phonetic match: both score
|
||||
# 1.0 for 万金→万津, and losing the alias provenance downgraded a confirmed
|
||||
# mapping back into a guess.
|
||||
better = prev is None or sc > prev["score"] or (
|
||||
sc == prev["score"] and rule == "alias" and prev.get("rule") != "alias")
|
||||
if better:
|
||||
findings[key] = {
|
||||
"span": span, "target": cand.canonical, "kind": cand.kind,
|
||||
"score": round(sc, 3), "via": cand.form, "speaker": spk,
|
||||
"sentence": sent, "common_word": span in allowlist,
|
||||
"source": cand.source, "rule": rule, "owner": cand.owner,
|
||||
"role": cand.role,
|
||||
}
|
||||
|
||||
# 1. exact skeleton collisions — the high-confidence homophone class
|
||||
for span, (k, syls) in span_keys.items():
|
||||
for cand in by_key.get(k, ()):
|
||||
consider(span, cand, syls)
|
||||
|
||||
# 2. near-miss skeletons via rapidfuzz shortlist (handles one-syllable slips
|
||||
# like 李小雨/李晓宇 that the folded key alone does not merge)
|
||||
uniq_keys = sorted({k for k, _ in span_keys.values()})
|
||||
matches = rf_process.cdist(uniq_keys, keys, workers=-1, score_cutoff=78)
|
||||
shortlist: dict[str, list[str]] = {}
|
||||
for row_i, row in enumerate(matches):
|
||||
hits = [keys[j] for j, v in enumerate(row) if v]
|
||||
if hits:
|
||||
shortlist[uniq_keys[row_i]] = hits
|
||||
for span, (k, syls) in span_keys.items():
|
||||
for tk in shortlist.get(k, ()):
|
||||
if tk == k:
|
||||
continue
|
||||
for cand in by_key[tk]:
|
||||
consider(span, cand, syls)
|
||||
|
||||
# 3. single-character substitution on equal-length names. 张泰海/张成海 scores
|
||||
# 0.82 phonetically (tai vs cheng is a poor match) yet is obviously the same
|
||||
# person: the surname and the tail are identical and only one syllable moved.
|
||||
# Anchoring on shared characters rather than sound catches the whole class.
|
||||
by_len: dict[int, list[Cand]] = {}
|
||||
for c in active:
|
||||
# Canonical names only. Aliases can be arbitrary phrases, and running
|
||||
# single-character substitution against one produced 我老板/跟老板/那老板/
|
||||
# 的老板 → 邹洵 (18 junk flags from the alias 林老板 alone).
|
||||
if c.rule == "full" and c.kind in ("人名", "公司"):
|
||||
by_len.setdefault(len(c.form), []).append(c)
|
||||
for span, (k, syls) in span_keys.items():
|
||||
if len(span) < 3:
|
||||
continue
|
||||
for cand in by_len.get(len(span), ()):
|
||||
same = sum(1 for a, b in zip(span, cand.form) if a == b)
|
||||
if same != len(span) - 1 or span == cand.form:
|
||||
continue
|
||||
# The head must hold. ASR drifts inside a name (张成海→张泰海); a name
|
||||
# whose first character changed is a different name.
|
||||
if span[0] != cand.form[0]:
|
||||
continue
|
||||
spk, sent = span_positions[span]
|
||||
key = (span, cand.canonical)
|
||||
if key not in findings:
|
||||
findings[key] = {
|
||||
"span": span, "target": cand.canonical, "kind": cand.kind,
|
||||
"score": RULE_MIN["one-off"], "via": cand.form, "speaker": spk,
|
||||
"sentence": sent, "common_word": span in allowlist,
|
||||
"source": cand.source, "rule": "one-off", "owner": cand.owner,
|
||||
"role": cand.role,
|
||||
}
|
||||
# Group alternatives for the same span so the model sees a real choice set,
|
||||
# which is the whole point: "这里听到 X，可能是 A 或 B，按句判".
|
||||
# 4. intra-document consistency. Once a target is firmly established somewhere
|
||||
# in the transcript, its *other* spellings in the same transcript can be
|
||||
# accepted at a lower bar — the document itself is the evidence. This is the
|
||||
# 万金/万津/万杰 pattern: one ASR name drifts across several spellings, and
|
||||
# only the strongest of them clears the standalone threshold.
|
||||
firm: dict[str, set[str]] = {}
|
||||
firm_alias: set[str] = set() # targets established by a confirmed alias
|
||||
entity_weights = entity_weights or {}
|
||||
for f in findings.values():
|
||||
if f["rule"] == "alias" and str(f.get("source", "")).startswith("alias-"):
|
||||
firm_alias.add(f["target"])
|
||||
if f["rule"] == "alias" or f["score"] >= 0.95:
|
||||
# Every established surface form is a comparison anchor, the canonical
|
||||
# name included. Keeping only the first one made 万杰 unreachable because
|
||||
# the anchor happened to be 老万 rather than 万津.
|
||||
firm.setdefault(f["target"], {f["target"]}).add(f["via"])
|
||||
# Shortlist by phonetic skeleton before running the DP, same trick as pass 2:
|
||||
# brute-forcing every span against every anchor pushed S1 from 4.7s to 24.5s on
|
||||
# a 27k transcript.
|
||||
anchor_keys: dict[str, list[tuple[str, str, list]]] = {}
|
||||
for target, vias in firm.items():
|
||||
for v in vias:
|
||||
anchor_keys.setdefault(phon_key(to_syls(v)), []).append((target, v, to_syls(v)))
|
||||
if anchor_keys:
|
||||
akeys = list(anchor_keys)
|
||||
cm = rf_process.cdist(uniq_keys, akeys, workers=-1, score_cutoff=65)
|
||||
near: dict[str, list[str]] = {}
|
||||
for i, row in enumerate(cm):
|
||||
hits = [akeys[j] for j, v in enumerate(row) if v]
|
||||
if hits:
|
||||
near[uniq_keys[i]] = hits
|
||||
else:
|
||||
near = {}
|
||||
plan: dict[str, set[str]] = {}
|
||||
for span, (k, syls) in span_keys.items():
|
||||
for ak in near.get(k, ()):
|
||||
for target, _v, _vs in anchor_keys[ak]:
|
||||
plan.setdefault(target, set()).add(span)
|
||||
|
||||
for target, vias in firm.items():
|
||||
anchors = [(v, to_syls(v)) for v in vias]
|
||||
for span in plan.get(target, ()):
|
||||
k, syls = span_keys[span]
|
||||
if span == target or (span, target) in findings or span in allowlist:
|
||||
continue
|
||||
if len(span) < 2:
|
||||
continue
|
||||
best = max(((seq_score(syls, vs), v) for v, vs in anchors
|
||||
if len(v) == len(span)), default=(0.0, ""))
|
||||
sc, via = best
|
||||
if not via or sc < 0.80:
|
||||
continue
|
||||
# Below near-homophone level, demand a shared head. A 2-syllable span at
|
||||
# 0.82 means roughly one syllable matched, which let 总总/全中→邹洵 and
|
||||
# 接商→宝山基金 through. Either the sound is nearly identical (军实/君实,
|
||||
# 1.0) or the first character is stable (万杰/万津, 白发/白帆).
|
||||
if sc < 0.90 and span[0] != via[0]:
|
||||
continue
|
||||
spk, sent = span_positions[span]
|
||||
src = next((c for c in active if c.canonical == target), None)
|
||||
findings[(span, target)] = {
|
||||
"span": span, "target": target, "kind": src.kind if src else "人名",
|
||||
"score": round(sc, 3), "via": via, "speaker": spk, "sentence": sent,
|
||||
"common_word": False,
|
||||
"source": "consistency-alias" if target in firm_alias else "consistency",
|
||||
"rule": "consistency", "owner": src.owner if src else "",
|
||||
"role": src.role if src else "",
|
||||
}
|
||||
|
||||
# Which organisations are actually being discussed. An honorific like 老万 has
|
||||
# one candidate per matching surname; the affiliation is what decides between
|
||||
# them, and the vault already records it (万亚坤.company = 洛启生物, which is
|
||||
# in play here). This is the cross-referencing step that was missing.
|
||||
mentioned_orgs = set()
|
||||
for c in active:
|
||||
if c.kind == "公司" and (c.canonical in text or (c.form and c.form in text)):
|
||||
mentioned_orgs.add(c.canonical)
|
||||
|
||||
grouped: dict[str, dict] = {}
|
||||
for f in findings.values():
|
||||
g = grouped.setdefault(f["span"], {
|
||||
"span": f["span"], "common_word": f["common_word"],
|
||||
"sentence": f["sentence"], "speaker": f["speaker"], "candidates": [],
|
||||
})
|
||||
owner = f.get("owner") or ""
|
||||
support = []
|
||||
if owner and owner in mentioned_orgs:
|
||||
support.append(f"{owner}(本次在谈)")
|
||||
elif owner and owner in in_play:
|
||||
support.append(f"{owner}(近期相关)")
|
||||
if f.get("role"):
|
||||
support.append(f["role"])
|
||||
recent = float(entity_weights.get(f["target"], 0.0))
|
||||
context_boost = min(1.5, math.log1p(max(0.0, recent)) / 2.5)
|
||||
affiliation_boost = 2.0 if (owner and owner in mentioned_orgs) else 0.0
|
||||
boost = affiliation_boost + context_boost
|
||||
if recent >= 0.5:
|
||||
support.append(f"近期权重{recent:.1f}")
|
||||
g["candidates"].append({"target": f["target"], "kind": f["kind"],
|
||||
"score": f["score"], "via": f["via"],
|
||||
"source": f["source"], "rule": f.get("rule", "full"),
|
||||
"owner": owner, "support": "、".join(support),
|
||||
"boost": round(boost, 3),
|
||||
"context_weight": round(recent, 3)})
|
||||
for g in grouped.values():
|
||||
# Affiliation evidence outranks raw phonetic score: a same-surname person
|
||||
# from an unrelated company is a worse guess than a slightly weaker match
|
||||
# from the company under discussion.
|
||||
g["candidates"].sort(
|
||||
key=lambda c: (-(c["score"] + 0.20 * c["boost"]), -c["score"]))
|
||||
# A single, unambiguous, human-confirmed mapping is not a suggestion. Kai's
|
||||
# reaction to a hedged draft was "肯定是指 李光辉 啊" — treating these as
|
||||
# optional is what makes the output feel useless. Ambiguous ones (徐总 has
|
||||
# three possible people) deliberately stay a choice set.
|
||||
c0 = g["candidates"][0]
|
||||
src0 = str(c0.get("source", ""))
|
||||
settled = (c0["rule"] == "alias"
|
||||
and src0.startswith(("alias-card", "alias-learned", "alias-glossary")))
|
||||
# A perfect homophone of an entity already pinned down by a confirmed alias,
|
||||
# in the same recording, is not a separate judgement call: 老善 and 老尚 are
|
||||
# one word the ASR spelled two ways, and 老尚 is confirmed to be 单永强.
|
||||
settled = settled or (c0["rule"] == "consistency"
|
||||
and src0 == "consistency-alias" and c0["score"] >= 0.97)
|
||||
# Curated evidence outranks generated guesses. 老万 is an explicit card alias
|
||||
# of 万亚坤; the rival candidates are only honorific forms auto-derived from
|
||||
# other 万/王-surnamed people, which is far weaker. But two *curated* targets
|
||||
# (徐总 → 徐劲前 or 邹洵) is real ambiguity and must stay a choice.
|
||||
n_alias = sum(1 for c in g["candidates"]
|
||||
if c["rule"] == "alias"
|
||||
and str(c.get("source", "")).startswith("alias-"))
|
||||
g["settled"] = bool(settled and n_alias <= 1
|
||||
and (len(g["candidates"]) == 1 or n_alias == 1))
|
||||
return sorted(grouped.values(),
|
||||
key=lambda g: (g["common_word"], -g["candidates"][0]["boost"],
|
||||
-g["candidates"][0]["score"]))
|
||||
|
||||
|
||||
def analyze(text: str, *, use_hindsight=True, use_termbank=True, min_score=None,
|
||||
extra_in_play: set[str] | None = None):
|
||||
"""Flag candidates.
|
||||
|
||||
`extra_in_play` carries entity names recovered by the S1c keyword recall. It
|
||||
matters most for backends that emit no speaker labels: with no participants to
|
||||
seed from, the co-occurrence graph produces almost nothing, and vault entities
|
||||
could never enter the candidate space at all.
|
||||
"""
|
||||
t0 = time.time()
|
||||
min_score = CFG.flag_min_score if min_score is None else min_score
|
||||
pool = load_pool(use_termbank=use_termbank)
|
||||
allow = load_allowlist()
|
||||
speakers = {s for s, _ in sentences_of(text) if s and HAN_FULL.match(s)}
|
||||
extra = {n for n in (extra_in_play or set()) if _is_entity_name(n)}
|
||||
in_play, ctx = compute_in_play(speakers, use_hindsight=use_hindsight,
|
||||
extra_seed=extra)
|
||||
in_play |= extra
|
||||
flags = flag_text(text, pool, in_play, allow, min_score=min_score,
|
||||
use_termbank_global=use_termbank,
|
||||
entity_weights=ctx.get("weight", {}))
|
||||
return {
|
||||
"elapsed": round(time.time() - t0, 2),
|
||||
"pool_size": len(pool),
|
||||
"speakers": sorted(speakers),
|
||||
"in_play": sorted(in_play),
|
||||
"hubs": ctx["hubs"],
|
||||
"hindsight_names": ctx["hindsight_names"],
|
||||
"hindsight_texts": ctx["hindsight_texts"],
|
||||
"flags": flags,
|
||||
}
|
||||
|
||||
|
||||
RULE_LABEL = {
|
||||
"alias": "已确认别名",
|
||||
"full": "整名近音",
|
||||
"org-short": "机构简称",
|
||||
"given": "去姓称名",
|
||||
"honorific": "称谓",
|
||||
"one-off": "一字之差",
|
||||
"consistency": "同篇另一写法",
|
||||
"observed": "历史误识别近音",
|
||||
}
|
||||
|
||||
|
||||
def render_for_prompt(flags: list[dict], limit=60) -> str:
|
||||
"""The block injected into an S2 chunk prompt.
|
||||
|
||||
Each candidate carries *why* it is a candidate (derivation rule) and *what
|
||||
supports it* (the affiliation the vault records, and whether that organisation
|
||||
is being discussed in this very recording). Without that, the model can only
|
||||
compare sounds, and it hedged on 老万 even though the vault says 万亚坤 is
|
||||
洛启生物's 实控人 and 洛启 is the subject of the call.
|
||||
"""
|
||||
lines = []
|
||||
must, maybe = [], []
|
||||
for g in flags[:limit]:
|
||||
parts = []
|
||||
for c in g["candidates"][:3]:
|
||||
tag = RULE_LABEL.get(c.get("rule", "full"), "")
|
||||
sup = f"，{c['support']}" if c.get("support") else ""
|
||||
parts.append(f"{c['target']}（{c['kind']}·{tag}{sup}）")
|
||||
cands = "、".join(parts)
|
||||
if not g["common_word"]:
|
||||
warn = ""
|
||||
elif g["candidates"][0].get("rule") == "alias":
|
||||
# Both a common word and a confirmed alias (光辉, 公司→工时). The mapping
|
||||
# is real, so the instruction is to check usage rather than default to
|
||||
# keeping the original.
|
||||
warn = "【此词也是常用词:先判断句中是否确实在指人/机构,是则用全名,否则保留原词】"
|
||||
else:
|
||||
warn = "【注意:此词本身是常用词,除非句子读不通否则保留原词】"
|
||||
sent = re.sub(r"\s+", "", g["sentence"])
|
||||
if len(sent) > 70:
|
||||
idx = sent.find(g["span"])
|
||||
lo = max(0, idx - 26)
|
||||
sent = ("…" if lo else "") + sent[lo:idx + len(g["span"]) + 26] + "…"
|
||||
if g.get("settled") and not g["common_word"]:
|
||||
must.append(f"- 「{g['span']}」= {g['candidates'][0]['target']}"
|
||||
f"｜原句: {sent}")
|
||||
else:
|
||||
maybe.append(f"- 「{g['span']}」→ 候选: {cands}{warn}｜原句: {sent}")
|
||||
out = []
|
||||
if must:
|
||||
out.append("【A. 已确认映射：Kai 已核对过，出现即按右侧替换，不要标疑似】\n"
|
||||
+ "\n".join(must))
|
||||
if maybe:
|
||||
out.append("【B. 待判断候选：按所在句子自行裁决】\n" + "\n".join(maybe))
|
||||
return "\n\n".join(out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("transcripts", nargs="+")
|
||||
ap.add_argument("--no-hindsight", action="store_true")
|
||||
ap.add_argument("--no-termbank", action="store_true")
|
||||
ap.add_argument("--min-score", type=float, default=0.86)
|
||||
ap.add_argument("--json", action="store_true")
|
||||
a = ap.parse_args()
|
||||
for p in a.transcripts:
|
||||
text = Path(p).read_text(errors="ignore")
|
||||
r = analyze(text, use_hindsight=not a.no_hindsight,
|
||||
use_termbank=not a.no_termbank, min_score=a.min_score)
|
||||
if a.json:
|
||||
print(json.dumps(r, ensure_ascii=False, indent=2))
|
||||
continue
|
||||
print("=" * 78)
|
||||
print(f"{Path(p).parent.name} {r['elapsed']}s pool={r['pool_size']} "
|
||||
f"in_play={len(r['in_play'])} hs={len(r['hindsight_names'])} "
|
||||
f"flags={len(r['flags'])}")
|
||||
print(f"speakers: {'、'.join(r['speakers']) or '(none)'} hubs: {'、'.join(r['hubs'])}")
|
||||
print(render_for_prompt(r["flags"], 25))
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Incremental entity/relation/recency index over curated Obsidian minutes."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
import re
|
||||
import time
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from . import lexicon
|
||||
from .config import CFG
|
||||
|
||||
FM_KEYS = ("people", "companies", "projects", "project")
|
||||
LINK = re.compile(r"\[\[([^\]|#]+)")
|
||||
DATE = re.compile(r"^date:[ \t]*(\d{4}-\d{2}-\d{2})", re.M)
|
||||
SENTENCE = re.compile(r"(?<=[。！？；\n])")
|
||||
PREDICATES = (
|
||||
"投资", "任职", "负责", "参与", "合作", "对接", "收购", "并购",
|
||||
"创立", "加入", "管理", "支持", "服务", "委托", "出资", "控股",
|
||||
)
|
||||
|
||||
|
||||
def _list_field(text: str, key: str) -> set[str]:
|
||||
m = re.search(rf"^{key}:[ \t]*(.*)$", text, re.M)
|
||||
if not m:
|
||||
return set()
|
||||
return {x.strip() for x in re.findall(r"[^,\[\]\s，、]+", m.group(1))
|
||||
if x.strip()}
|
||||
|
||||
|
||||
def _timestamp(text: str, path: Path) -> float:
|
||||
m = DATE.search(text)
|
||||
if m:
|
||||
try:
|
||||
return datetime.combine(date.fromisoformat(m.group(1)),
|
||||
datetime.min.time()).timestamp()
|
||||
except ValueError:
|
||||
pass
|
||||
m = re.search(r"(\d{4}-\d{2}-\d{2})", path.name)
|
||||
if m:
|
||||
try:
|
||||
return datetime.combine(date.fromisoformat(m.group(1)),
|
||||
datetime.min.time()).timestamp()
|
||||
except ValueError:
|
||||
pass
|
||||
return path.stat().st_mtime
|
||||
|
||||
|
||||
def _signature(path: Path) -> str:
|
||||
st = path.stat()
|
||||
return hashlib.sha256(
|
||||
f"{path}:{st.st_mtime_ns}:{st.st_size}".encode()).hexdigest()[:20]
|
||||
|
||||
|
||||
def _minutes_files() -> list[Path]:
|
||||
root = CFG.vault / "minutes"
|
||||
if not root.exists():
|
||||
return []
|
||||
return sorted(p for p in root.rglob("*.md")
|
||||
if "_drafts" not in p.parts and not p.name.startswith("_"))
|
||||
|
||||
|
||||
def _ordered(a: str, b: str, kinds: dict[str, str]) -> tuple[str, str]:
|
||||
rank = {"人名": 0, "公司": 1, "项目": 2}
|
||||
ka, kb = rank.get(kinds.get(a, ""), 9), rank.get(kinds.get(b, ""), 9)
|
||||
return (a, b) if (ka, a) <= (kb, b) else (b, a)
|
||||
|
||||
|
||||
def sync(force: bool = False) -> dict:
|
||||
"""Index changed minutes and remove facts belonging to deleted notes."""
|
||||
lexicon.sync_from_vault()
|
||||
con = lexicon.connect()
|
||||
kinds = {r["canonical"]: r["kind"]
|
||||
for r in con.execute("SELECT canonical,kind FROM entity")}
|
||||
known = set(kinds)
|
||||
files = _minutes_files()
|
||||
active_sources = {f"minute:{p.relative_to(CFG.vault)}" for p in files}
|
||||
|
||||
old_sources = {r["source"] for r in con.execute(
|
||||
"SELECT source FROM source_document WHERE source LIKE 'minute:%'")}
|
||||
removed = old_sources - active_sources
|
||||
for source in removed:
|
||||
con.execute("DELETE FROM relation WHERE source=?", (source,))
|
||||
con.execute("DELETE FROM interaction WHERE source=?", (source,))
|
||||
con.execute("DELETE FROM entity_mention WHERE source=?", (source,))
|
||||
con.execute("DELETE FROM source_document WHERE source=?", (source,))
|
||||
|
||||
changed = skipped = entities_n = relations_n = 0
|
||||
now = time.time()
|
||||
for p in files:
|
||||
source = f"minute:{p.relative_to(CFG.vault)}"
|
||||
sig = _signature(p)
|
||||
old = con.execute("SELECT signature FROM source_document WHERE source=?",
|
||||
(source,)).fetchone()
|
||||
if not force and old and old["signature"] == sig:
|
||||
skipped += 1
|
||||
continue
|
||||
text = p.read_text(errors="ignore")
|
||||
typed = {
|
||||
"人名": _list_field(text, "people"),
|
||||
"公司": _list_field(text, "companies"),
|
||||
}
|
||||
con.execute("DELETE FROM entity_mention WHERE source=?", (source,))
|
||||
for kind, names in typed.items():
|
||||
for name in names:
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO entity_mention(entity,kind,source,"
|
||||
"occurred_at) VALUES(?,?,?,?)",
|
||||
(name, kind, source, _timestamp(text, p)))
|
||||
entities = set()
|
||||
for key in FM_KEYS:
|
||||
entities |= _list_field(text, key)
|
||||
entities |= {x.strip() for x in LINK.findall(text)}
|
||||
entities &= known
|
||||
occurred = _timestamp(text, p)
|
||||
|
||||
con.execute("DELETE FROM relation WHERE source=?", (source,))
|
||||
con.execute("DELETE FROM interaction WHERE source=?", (source,))
|
||||
for entity in entities:
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO interaction(entity,occurred_at,kind,source,"
|
||||
"weight) VALUES(?,?,?,?,?)",
|
||||
(entity, occurred, "meeting", source, 1.0))
|
||||
|
||||
nrel = 0
|
||||
ordered = sorted(entities)
|
||||
for i, a in enumerate(ordered):
|
||||
for b in ordered[i + 1:]:
|
||||
subject, object_ = _ordered(a, b, kinds)
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO relation(subject,predicate,object,"
|
||||
"confidence,source,first_seen,last_seen) VALUES(?,?,?,?,?,?,?)",
|
||||
(subject, "同会", object_, 0.35, source, occurred, occurred))
|
||||
nrel += 1
|
||||
|
||||
# A predicate is promoted only when two known linked entities occur in the
|
||||
# same sentence and the relationship word is explicit.
|
||||
for sentence in SENTENCE.split(text):
|
||||
linked = [x.strip() for x in LINK.findall(sentence) if x.strip() in known]
|
||||
if len(linked) < 2:
|
||||
continue
|
||||
predicate = next((x for x in PREDICATES if x in sentence), "")
|
||||
if not predicate:
|
||||
continue
|
||||
for a, b in zip(linked, linked[1:]):
|
||||
if a == b:
|
||||
continue
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO relation(subject,predicate,object,"
|
||||
"confidence,source,first_seen,last_seen) VALUES(?,?,?,?,?,?,?)",
|
||||
(a, predicate, b, 0.75, source, occurred, occurred))
|
||||
nrel += 1
|
||||
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO source_document(source,signature,processed_at,"
|
||||
"entity_count,relation_count) VALUES(?,?,?,?,?)",
|
||||
(source, sig, now, len(entities), nrel))
|
||||
changed += 1
|
||||
entities_n += len(entities)
|
||||
relations_n += nrel
|
||||
con.commit()
|
||||
con.close()
|
||||
return {"files": len(files), "changed": changed, "skipped": skipped,
|
||||
"removed": len(removed), "entities_seen": entities_n,
|
||||
"relations_written": relations_n}
|
||||
|
||||
|
||||
def context_weights(seeds: set[str], *, half_life_days: float = 45.0,
|
||||
limit: int = 100) -> dict[str, float]:
|
||||
"""Score graph neighbours by relation strength, recency, and frequency."""
|
||||
if not seeds:
|
||||
return {}
|
||||
con = lexicon.connect()
|
||||
marks = ",".join("?" for _ in seeds)
|
||||
rows = con.execute(
|
||||
f"SELECT subject,predicate,object,max(confidence) confidence,"
|
||||
f"max(last_seen) last_seen,count(*) support FROM relation WHERE active=1 "
|
||||
f"AND (subject IN ({marks}) OR object IN ({marks})) "
|
||||
"GROUP BY subject,predicate,object",
|
||||
(*seeds, *seeds)).fetchall()
|
||||
now = time.time()
|
||||
out: dict[str, float] = {s: 12.0 for s in seeds}
|
||||
for r in rows:
|
||||
other = r["object"] if r["subject"] in seeds else r["subject"]
|
||||
age = max(0.0, (now - (r["last_seen"] or now)) / 86400)
|
||||
decay = math.exp(-math.log(2) * age / half_life_days)
|
||||
support = 1.0 + math.log1p(r["support"])
|
||||
out[other] = out.get(other, 0.0) + r["confidence"] * decay * support
|
||||
|
||||
# Direct contact frequency is useful even when a relation has only one edge.
|
||||
if out:
|
||||
names = list(out)
|
||||
marks2 = ",".join("?" for _ in names)
|
||||
interactions = con.execute(
|
||||
f"SELECT entity,count(*) n,max(occurred_at) latest FROM interaction "
|
||||
f"WHERE entity IN ({marks2}) GROUP BY entity", names).fetchall()
|
||||
for r in interactions:
|
||||
age = max(0.0, (now - r["latest"]) / 86400)
|
||||
decay = math.exp(-math.log(2) * age / half_life_days)
|
||||
out[r["entity"]] += decay * math.log1p(r["n"])
|
||||
con.close()
|
||||
return dict(sorted(out.items(), key=lambda x: -x[1])[:limit])
|
||||
|
||||
|
||||
def stats() -> dict:
|
||||
con = lexicon.connect()
|
||||
q = lambda sql: con.execute(sql).fetchone()[0]
|
||||
out = {
|
||||
"documents": q("SELECT count(*) FROM source_document"),
|
||||
"relations": q("SELECT count(*) FROM relation"),
|
||||
"interactions": q("SELECT count(*) FROM interaction"),
|
||||
"projects": q("SELECT count(*) FROM entity WHERE kind='项目'"),
|
||||
}
|
||||
con.close()
|
||||
return out
|
||||
@@ -0,0 +1,526 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic entity / alias / correction lexicon (SQLite).
|
||||
|
||||
Design rule, decided 2026-07-25:
|
||||
|
||||
**Obsidian is a read-only source of truth.** Cards supply canonical names and
|
||||
the `aliases:` / `company:` / `role:` frontmatter. This program pulls from them
|
||||
and never writes back — no `_glossary.md` edits, no card mutation.
|
||||
|
||||
Everything the pipeline *learns* (a confirmed nickname like 老尚=单永强, a
|
||||
recurring ASR misrecognition like 张泰海→张成海) accumulates here instead. Same
|
||||
separation as the voiceprint registry: an independent structured store that
|
||||
grows with use, next to the vault rather than inside it.
|
||||
|
||||
Why SQLite and not Hindsight: alias resolution must be an exact, repeatable
|
||||
key→value lookup. Hindsight stores LLM-extracted memories and recalls them
|
||||
semantically — excellent for *relationships* ("万亚坤 is 洛启's 实控人", which is
|
||||
what disambiguates 老万), useless as a guarantee that 老尚 resolves every single
|
||||
time. The two are complementary and both are used: lexicon decides identity,
|
||||
Hindsight supplies the context that ranks competing identities.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sqlite3
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from .config import CFG
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS entity (
|
||||
canonical TEXT PRIMARY KEY,
|
||||
kind TEXT NOT NULL, -- 人名 / 公司
|
||||
company TEXT DEFAULT '', -- affiliation, for honorific disambiguation
|
||||
role TEXT DEFAULT '',
|
||||
related TEXT DEFAULT '', -- related_people, comma-joined
|
||||
card TEXT DEFAULT '',
|
||||
synced_at REAL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS alias (
|
||||
alias TEXT NOT NULL,
|
||||
canonical TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
origin TEXT NOT NULL, -- card | learned | manual | derived
|
||||
confirmed INTEGER DEFAULT 0, -- 1 = Kai said so explicitly
|
||||
hits INTEGER DEFAULT 0,
|
||||
created_at REAL,
|
||||
note TEXT DEFAULT '',
|
||||
PRIMARY KEY (alias, canonical)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS alias_by_alias ON alias(alias);
|
||||
CREATE TABLE IF NOT EXISTS correction (
|
||||
surface TEXT NOT NULL, -- what the ASR wrote
|
||||
canonical TEXT NOT NULL, -- what it should be
|
||||
n INTEGER DEFAULT 1, -- times confirmed
|
||||
confirmed INTEGER DEFAULT 0,
|
||||
last_seen REAL,
|
||||
note TEXT DEFAULT '',
|
||||
PRIMARY KEY (surface, canonical)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS relation (
|
||||
subject TEXT NOT NULL,
|
||||
predicate TEXT NOT NULL,
|
||||
object TEXT NOT NULL,
|
||||
confidence REAL DEFAULT 0.5,
|
||||
source TEXT DEFAULT '',
|
||||
first_seen REAL,
|
||||
last_seen REAL,
|
||||
active INTEGER DEFAULT 1,
|
||||
PRIMARY KEY (subject, predicate, object, source)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS relation_by_subject ON relation(subject, active, last_seen);
|
||||
CREATE INDEX IF NOT EXISTS relation_by_object ON relation(object, active, last_seen);
|
||||
CREATE TABLE IF NOT EXISTS interaction (
|
||||
entity TEXT NOT NULL,
|
||||
occurred_at REAL NOT NULL,
|
||||
kind TEXT DEFAULT 'mention',
|
||||
source TEXT DEFAULT '',
|
||||
weight REAL DEFAULT 1.0,
|
||||
PRIMARY KEY (entity, occurred_at, kind, source)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS interaction_recent ON interaction(entity, occurred_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS entity_mention (
|
||||
entity TEXT NOT NULL,
|
||||
kind TEXT NOT NULL, -- 人名 / 公司
|
||||
source TEXT NOT NULL, -- one formal minute; one count per source
|
||||
occurred_at REAL NOT NULL,
|
||||
PRIMARY KEY (entity, kind, source)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS entity_mention_count ON entity_mention(entity,kind);
|
||||
CREATE TABLE IF NOT EXISTS observation (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
surface TEXT NOT NULL,
|
||||
canonical TEXT DEFAULT '',
|
||||
source TEXT DEFAULT '',
|
||||
context TEXT DEFAULT '',
|
||||
confidence REAL DEFAULT 0.0,
|
||||
confirmed INTEGER DEFAULT 0,
|
||||
created_at REAL NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS observation_surface ON observation(surface, created_at DESC);
|
||||
CREATE TABLE IF NOT EXISTS phonetic_confusion (
|
||||
heard TEXT NOT NULL,
|
||||
intended TEXT NOT NULL,
|
||||
n INTEGER DEFAULT 1,
|
||||
confirmed INTEGER DEFAULT 0,
|
||||
last_seen REAL,
|
||||
note TEXT DEFAULT '',
|
||||
PRIMARY KEY (heard, intended)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS review_question (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
recording TEXT NOT NULL,
|
||||
question TEXT NOT NULL,
|
||||
payload TEXT DEFAULT '{}',
|
||||
status TEXT DEFAULT 'pending',
|
||||
answer TEXT DEFAULT '',
|
||||
created_at REAL NOT NULL,
|
||||
answered_at REAL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS review_pending ON review_question(status, created_at);
|
||||
CREATE TABLE IF NOT EXISTS source_document (
|
||||
source TEXT PRIMARY KEY,
|
||||
signature TEXT NOT NULL,
|
||||
processed_at REAL NOT NULL,
|
||||
entity_count INTEGER DEFAULT 0,
|
||||
relation_count INTEGER DEFAULT 0
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS meta (k TEXT PRIMARY KEY, v TEXT);
|
||||
"""
|
||||
|
||||
HAN_FULL = re.compile(r"^[\u4e00-\u9fff]+$")
|
||||
# Anything that looks like a note title or a date is not an entity. This is what
|
||||
# polluted in_play: 25 of 70 slots were minutes filenames, dates and _glossary.
|
||||
JUNK = re.compile(r"^\d{4}-\d{2}|^_|\.md$|[|/#]")
|
||||
|
||||
|
||||
ORG_SUFFIX = ("生物", "医药", "制药", "药业", "基金", "资本", "投资", "科技", "集团",
|
||||
"股份", "公司", "银行", "证券", "医院", "研究院", "实验室", "中心")
|
||||
|
||||
|
||||
def guess_kind(name: str) -> str:
|
||||
"""Entity type for a name with no card. Only used for display labelling."""
|
||||
return "公司" if name.endswith(ORG_SUFFIX) else "人名"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Entity:
|
||||
canonical: str
|
||||
kind: str
|
||||
company: str = ""
|
||||
role: str = ""
|
||||
related: tuple = ()
|
||||
|
||||
|
||||
def connect() -> sqlite3.Connection:
|
||||
CFG.state_dir.mkdir(parents=True, exist_ok=True)
|
||||
con = sqlite3.connect(CFG.lexicon_db)
|
||||
con.row_factory = sqlite3.Row
|
||||
con.executescript(SCHEMA)
|
||||
# One-time compatibility migration for the 158 corrections learned before the
|
||||
# personal phonetic-confusion channel existed.
|
||||
con.execute(
|
||||
"INSERT OR IGNORE INTO phonetic_confusion(heard,intended,n,confirmed,"
|
||||
"last_seen,note) SELECT surface,canonical,n,confirmed,last_seen,note "
|
||||
"FROM correction")
|
||||
con.commit()
|
||||
return con
|
||||
|
||||
|
||||
# --------------------------------------------------------------- vault pull
|
||||
def _fm(text: str, key: str) -> str:
|
||||
# `[ \t]*` not `\s*`: \s matches newlines, so a blank `role:` field used to
|
||||
# swallow the next line and report `status: active` as the role.
|
||||
m = re.search(rf"^{key}:[ \t]*(.*)$", text, re.M)
|
||||
return m.group(1).strip().strip("\"'") if m else ""
|
||||
|
||||
|
||||
def _list_field(raw: str) -> list[str]:
|
||||
return [x for x in re.findall(r"[^,\[\]\s，、\"']+", raw or "") if x]
|
||||
|
||||
|
||||
def vault_signature() -> str:
|
||||
import hashlib
|
||||
h = hashlib.sha256()
|
||||
if CFG.glossary.exists():
|
||||
st = CFG.glossary.stat()
|
||||
h.update(f"gl:{st.st_mtime_ns}:{st.st_size}".encode())
|
||||
for sub in ("People", "Companies", "Projects"):
|
||||
d = CFG.vault / sub
|
||||
if not d.exists():
|
||||
continue
|
||||
for f in sorted(d.glob("*.md")):
|
||||
st = f.stat()
|
||||
h.update(f"{f.name}:{st.st_mtime_ns}:{st.st_size}".encode())
|
||||
return h.hexdigest()[:16]
|
||||
|
||||
|
||||
GL_MAP = re.compile(r"^(?:>\s*)?[-*]\s*(.+?)\s*(?:→|⇒|➔|->)\s*(.+)$")
|
||||
GL_ENT = re.compile(r"^[-*]\s*([\u4e00-\u9fff]{2,6})\s*[—–]\s*(.+)$")
|
||||
_PAREN = re.compile(r"[（(][^）)]*[）)]")
|
||||
|
||||
|
||||
def _clean_form(s: str) -> str:
|
||||
return re.sub(r"\*\*|\[\[|\]\]|[`\"'「」“”]", "", s).strip()
|
||||
|
||||
|
||||
def _split_forms(s: str) -> list[str]:
|
||||
"""`胡律 / 胡丽 / 胡博士（跨境资本语境）` -> [胡律, 胡丽, 胡博士]."""
|
||||
s = _PAREN.sub("", s)
|
||||
return [_clean_form(x) for x in re.split(r"[/、,，]|\s+or\s+", s) if _clean_form(x)]
|
||||
|
||||
|
||||
def _split_targets(rhs: str) -> list[tuple[str, str]]:
|
||||
"""Right-hand side -> [(canonical, context_hint)].
|
||||
|
||||
Handles the one-to-many context-dependent rows the glossary explicitly flags,
|
||||
e.g. `许博 → **许俊彦**(白帆生物语境) / **许恒皓**(愚公生物语境)`, which must stay
|
||||
ambiguous: the pipeline offers both and the sentence decides.
|
||||
"""
|
||||
parts = re.split(r"\s*/\s*(?=\*\*)", rhs) if rhs.count("**") > 1 else [rhs]
|
||||
out = []
|
||||
for it in parts:
|
||||
ctx = "；".join(x.strip() for x in re.findall(r"[（(]([^）)]*)[）)]", it))
|
||||
name = _clean_form(_PAREN.sub("", it)).split("；")[0].split(";")[0].strip()
|
||||
name = name.split("，")[0].strip()
|
||||
if name:
|
||||
out.append((name, ctx))
|
||||
return out
|
||||
|
||||
|
||||
def _import_glossary(con: sqlite3.Connection, now: float) -> int:
|
||||
"""Read `minutes/_glossary.md` into the lexicon. The file is never modified.
|
||||
|
||||
It carries ~130 hand-confirmed mappings, several of which are deliberately
|
||||
one-to-many. Importing them means the pipeline stops re-deriving what Kai
|
||||
already settled, while the vault keeps a single human-editable copy.
|
||||
"""
|
||||
gl = CFG.glossary
|
||||
if not gl.exists():
|
||||
return 0
|
||||
n = 0
|
||||
|
||||
def put(alias: str, canonical: str, ctx: str, origin: str):
|
||||
nonlocal n
|
||||
if not (HAN_FULL.match(alias) and HAN_FULL.match(canonical)):
|
||||
return
|
||||
if alias == canonical or not (2 <= len(alias) <= 8) or not (2 <= len(canonical) <= 8):
|
||||
return
|
||||
r = con.execute("SELECT kind FROM entity WHERE canonical=?", (canonical,)).fetchone()
|
||||
kind = r["kind"] if r else guess_kind(canonical)
|
||||
con.execute(
|
||||
"INSERT OR IGNORE INTO alias(alias,canonical,kind,origin,confirmed,"
|
||||
"created_at,note) VALUES(?,?,?,?,1,?,?)",
|
||||
(alias, canonical, kind, origin, now, ctx))
|
||||
con.execute(
|
||||
"INSERT OR IGNORE INTO correction(surface,canonical,n,confirmed,last_seen,note)"
|
||||
" VALUES(?,?,1,1,?,?)", (alias, canonical, now, ctx))
|
||||
n += 1
|
||||
|
||||
for raw in gl.read_text(errors="ignore").splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
m = GL_MAP.match(line)
|
||||
if m:
|
||||
for tgt, ctx in _split_targets(m.group(2)):
|
||||
for form in _split_forms(m.group(1)):
|
||||
put(form, tgt, ctx, "glossary")
|
||||
continue
|
||||
# `- 万亚坤 — 洛启生物实控人（老万；原音 万博/王博/「平台总代理坤」）`
|
||||
m = GL_ENT.match(line)
|
||||
if m:
|
||||
canon, desc = m.group(1), m.group(2)
|
||||
for grp in re.findall(r"[（(]([^）)]*)[）)]", desc):
|
||||
for seg in re.split(r"[；;]", grp):
|
||||
seg = re.sub(r"^\s*(原音|又称|简称|别名)\s*", "", seg.strip())
|
||||
for form in _split_forms(seg):
|
||||
put(form, canon, "", "glossary")
|
||||
return n
|
||||
|
||||
|
||||
def sync_from_vault(force: bool = False) -> dict:
|
||||
"""Pull entities + card aliases from Obsidian. Read-only on the vault side.
|
||||
|
||||
Learned rows are preserved: only `origin='card'` aliases are refreshed, so a
|
||||
nickname Kai confirmed once is never lost when a card is edited.
|
||||
"""
|
||||
con = connect()
|
||||
sig = vault_signature()
|
||||
cur = con.execute("SELECT v FROM meta WHERE k='vault_sig'").fetchone()
|
||||
if not force and cur and cur["v"] == sig:
|
||||
con.close()
|
||||
return {"skipped": True, "sig": sig}
|
||||
|
||||
now = time.time()
|
||||
n_ent = n_alias = 0
|
||||
con.execute("DELETE FROM alias WHERE origin IN ('card','glossary')")
|
||||
con.execute("DELETE FROM entity")
|
||||
con.execute("DELETE FROM relation WHERE source LIKE 'card:%'")
|
||||
for sub, kind in (("People", "人名"), ("Companies", "公司"), ("Projects", "项目")):
|
||||
d = CFG.vault / sub
|
||||
if not d.exists():
|
||||
continue
|
||||
for f in sorted(d.glob("*.md")):
|
||||
name = f.stem
|
||||
if JUNK.search(name):
|
||||
continue
|
||||
if kind != "项目" and not HAN_FULL.match(name):
|
||||
continue
|
||||
if kind == "项目" and not (2 <= len(name) <= 80):
|
||||
continue
|
||||
txt = f.read_text(errors="ignore")
|
||||
head = txt[:1200]
|
||||
company = _fm(head, "company")
|
||||
related = _list_field(_fm(head, "related_people"))
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO entity(canonical,kind,company,role,related,"
|
||||
"card,synced_at) VALUES(?,?,?,?,?,?,?)",
|
||||
(name, kind, company, _fm(head, "role"),
|
||||
",".join(related), str(f), now))
|
||||
if company:
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO relation(subject,predicate,object,"
|
||||
"confidence,source,first_seen,last_seen) VALUES(?,?,?,?,?,?,?)",
|
||||
(name, "任职于", company, 1.0, f"card:{f}", now, now))
|
||||
for other in related:
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO relation(subject,predicate,object,"
|
||||
"confidence,source,first_seen,last_seen) VALUES(?,?,?,?,?,?,?)",
|
||||
(name, "相关", other, 0.9, f"card:{f}", now, now))
|
||||
n_ent += 1
|
||||
for a in _list_field(_fm(head, "aliases")):
|
||||
if a == name or not HAN_FULL.match(a) or len(a) < 2:
|
||||
continue
|
||||
con.execute(
|
||||
"INSERT OR IGNORE INTO alias(alias,canonical,kind,origin,confirmed,"
|
||||
"created_at) VALUES(?,?,?,'card',1,?)", (a, name, kind, now))
|
||||
n_alias += 1
|
||||
# The vault glossary is *imported*, never edited. It holds ~130 corrections Kai
|
||||
# already confirmed by hand; they become exact-match bindings here so the
|
||||
# pipeline stops re-deriving them, and the file itself stays untouched.
|
||||
n_gl = _import_glossary(con, now)
|
||||
|
||||
con.execute("INSERT OR REPLACE INTO meta(k,v) VALUES('vault_sig',?)", (sig,))
|
||||
con.execute("INSERT OR REPLACE INTO meta(k,v) VALUES('synced_at',?)", (str(now),))
|
||||
con.commit()
|
||||
con.close()
|
||||
return {"skipped": False, "entities": n_ent, "card_aliases": n_alias,
|
||||
"glossary": n_gl, "sig": sig}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ reading
|
||||
def entities() -> dict[str, Entity]:
|
||||
con = connect()
|
||||
out = {r["canonical"]: Entity(
|
||||
r["canonical"], r["kind"], r["company"] or "", r["role"] or "",
|
||||
tuple(x for x in (r["related"] or "").split(",") if x))
|
||||
for r in con.execute("SELECT * FROM entity")}
|
||||
con.close()
|
||||
return out
|
||||
|
||||
|
||||
def alias_map() -> dict[str, list[tuple[str, str, str, int]]]:
|
||||
"""alias -> [(canonical, kind, origin, confirmed)]. May be ambiguous."""
|
||||
con = connect()
|
||||
out: dict[str, list] = {}
|
||||
for r in con.execute("SELECT * FROM alias ORDER BY confirmed DESC, hits DESC"):
|
||||
out.setdefault(r["alias"], []).append(
|
||||
(r["canonical"], r["kind"], r["origin"], r["confirmed"]))
|
||||
con.close()
|
||||
return out
|
||||
|
||||
|
||||
def corrections() -> dict[str, list[tuple[str, int]]]:
|
||||
con = connect()
|
||||
out: dict[str, list] = {}
|
||||
for r in con.execute("SELECT * FROM correction ORDER BY n DESC"):
|
||||
out.setdefault(r["surface"], []).append((r["canonical"], r["n"]))
|
||||
con.close()
|
||||
return out
|
||||
|
||||
|
||||
def confusion_map() -> dict[str, list[tuple[str, int, int]]]:
|
||||
"""Observed ASR surface -> canonical, count, confirmation.
|
||||
|
||||
Older databases predate `phonetic_confusion`, so confirmed correction rows are
|
||||
included as a compatibility source without a destructive migration.
|
||||
"""
|
||||
con = connect()
|
||||
out: dict[str, list[tuple[str, int, int]]] = {}
|
||||
for r in con.execute(
|
||||
"SELECT surface AS heard,canonical AS intended,n,confirmed "
|
||||
"FROM correction UNION ALL "
|
||||
"SELECT heard,intended,n,confirmed FROM phonetic_confusion"):
|
||||
item = (r["intended"], r["n"], r["confirmed"])
|
||||
if item not in out.setdefault(r["heard"], []):
|
||||
out[r["heard"]].append(item)
|
||||
con.close()
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ writing
|
||||
def learn_alias(alias: str, canonical: str, *, kind: str = "", origin: str = "learned",
|
||||
confirmed: int = 1, note: str = "") -> None:
|
||||
con = connect()
|
||||
if not kind:
|
||||
r = con.execute("SELECT kind FROM entity WHERE canonical=?", (canonical,)).fetchone()
|
||||
kind = r["kind"] if r else guess_kind(canonical)
|
||||
con.execute(
|
||||
"INSERT INTO alias(alias,canonical,kind,origin,confirmed,created_at,note)"
|
||||
" VALUES(?,?,?,?,?,?,?) ON CONFLICT(alias,canonical) DO UPDATE SET"
|
||||
" confirmed=max(confirmed,excluded.confirmed), origin=excluded.origin,"
|
||||
" note=excluded.note",
|
||||
(alias, canonical, kind, origin, confirmed, time.time(), note))
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
|
||||
def learn_correction(surface: str, canonical: str, *, confirmed: int = 1,
|
||||
note: str = "") -> None:
|
||||
con = connect()
|
||||
con.execute(
|
||||
"INSERT INTO correction(surface,canonical,n,confirmed,last_seen,note)"
|
||||
" VALUES(?,?,1,?,?,?) ON CONFLICT(surface,canonical) DO UPDATE SET"
|
||||
" n=n+1, confirmed=max(confirmed,excluded.confirmed), last_seen=excluded.last_seen",
|
||||
(surface, canonical, confirmed, time.time(), note))
|
||||
con.execute(
|
||||
"INSERT INTO phonetic_confusion(heard,intended,n,confirmed,last_seen,note)"
|
||||
" VALUES(?,?,1,?,?,?) ON CONFLICT(heard,intended) DO UPDATE SET"
|
||||
" n=n+1,confirmed=max(confirmed,excluded.confirmed),"
|
||||
" last_seen=excluded.last_seen,note=excluded.note",
|
||||
(surface, canonical, confirmed, time.time(), note))
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
|
||||
def record_observation(surface: str, canonical: str = "", *, source: str = "",
|
||||
context: str = "", confidence: float = 0.0,
|
||||
confirmed: int = 0) -> int:
|
||||
con = connect()
|
||||
cur = con.execute(
|
||||
"INSERT INTO observation(surface,canonical,source,context,confidence,"
|
||||
"confirmed,created_at) VALUES(?,?,?,?,?,?,?)",
|
||||
(surface, canonical, source, context[:1000], confidence, confirmed, time.time()))
|
||||
con.commit()
|
||||
row_id = int(cur.lastrowid)
|
||||
con.close()
|
||||
return row_id
|
||||
|
||||
|
||||
def upsert_relation(subject: str, predicate: str, object_: str, *,
|
||||
confidence: float = 0.5, source: str = "") -> None:
|
||||
now = time.time()
|
||||
con = connect()
|
||||
con.execute(
|
||||
"INSERT INTO relation(subject,predicate,object,confidence,source,first_seen,"
|
||||
"last_seen) VALUES(?,?,?,?,?,?,?) ON CONFLICT(subject,predicate,object,source)"
|
||||
" DO UPDATE SET confidence=max(confidence,excluded.confidence),"
|
||||
" last_seen=excluded.last_seen,active=1",
|
||||
(subject, predicate, object_, confidence, source, now, now))
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
|
||||
def record_interaction(entity: str, *, occurred_at: float | None = None,
|
||||
kind: str = "mention", source: str = "",
|
||||
weight: float = 1.0) -> None:
|
||||
con = connect()
|
||||
con.execute(
|
||||
"INSERT OR IGNORE INTO interaction(entity,occurred_at,kind,source,weight)"
|
||||
" VALUES(?,?,?,?,?)",
|
||||
(entity, occurred_at or time.time(), kind, source, weight))
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
|
||||
def relationship_card(names: set[str], *, limit: int = 30) -> list[str]:
|
||||
"""Return compact, recent relation facts for other applications and ASR."""
|
||||
if not names:
|
||||
return []
|
||||
con = connect()
|
||||
marks = ",".join("?" for _ in names)
|
||||
rows = con.execute(
|
||||
f"SELECT subject,predicate,object,max(confidence) confidence,"
|
||||
f"max(last_seen) last_seen,count(*) support FROM relation "
|
||||
f"WHERE active=1 AND (subject IN ({marks}) OR object IN ({marks})) "
|
||||
"GROUP BY subject,predicate,object "
|
||||
"ORDER BY confidence DESC,last_seen DESC,support DESC LIMIT ?",
|
||||
(*names, *names, limit)).fetchall()
|
||||
con.close()
|
||||
return [f"{r['subject']} {r['predicate']} {r['object']}"
|
||||
f"（置信度 {r['confidence']:.2f}，支持 {r['support']} 次）" for r in rows]
|
||||
|
||||
|
||||
def bump_hits(pairs) -> None:
|
||||
if not pairs:
|
||||
return
|
||||
con = connect()
|
||||
con.executemany("UPDATE alias SET hits=hits+1 WHERE alias=? AND canonical=?",
|
||||
list(pairs))
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
|
||||
def stats() -> dict:
|
||||
con = connect()
|
||||
q = lambda s: con.execute(s).fetchone()[0]
|
||||
d = {
|
||||
"entities": q("SELECT count(*) FROM entity"),
|
||||
"people": q("SELECT count(*) FROM entity WHERE kind='人名'"),
|
||||
"companies": q("SELECT count(*) FROM entity WHERE kind='公司'"),
|
||||
"with_company": q("SELECT count(*) FROM entity WHERE company<>''"),
|
||||
"with_related": q("SELECT count(*) FROM entity WHERE related<>''"),
|
||||
"aliases": q("SELECT count(*) FROM alias"),
|
||||
"aliases_card": q("SELECT count(*) FROM alias WHERE origin='card'"),
|
||||
"aliases_learned": q("SELECT count(*) FROM alias WHERE origin<>'card'"),
|
||||
"corrections": q("SELECT count(*) FROM correction"),
|
||||
"db": str(CFG.lexicon_db),
|
||||
}
|
||||
r = con.execute("SELECT v FROM meta WHERE k='synced_at'").fetchone()
|
||||
d["synced_at"] = r["v"] if r else ""
|
||||
con.close()
|
||||
return d
|
||||
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env python3
|
||||
"""HTTPS client for the model provider. No agent CLI, no SDK.
|
||||
|
||||
Deliberately built on urllib so the program has zero third-party HTTP
|
||||
dependencies: it must keep working inside a bare systemd unit years from now.
|
||||
|
||||
Behaviours encoded from measurement rather than guesswork:
|
||||
* concurrency 6 (HTTP 429 appears at ~12)
|
||||
* HTTP 402 "Payment Required" is transient on ZenMux — credit seems reserved
|
||||
against max_tokens, so a burst of large-budget calls can trip it and then
|
||||
succeed moments later; it gets a much longer backoff than a rate limit
|
||||
* `temperature` is rejected outright by claude-sonnet-5 / opus-5 but accepted by
|
||||
haiku-4.5, sonnet-4.6, gemini and qwen, so it cannot be inferred from the
|
||||
vendor prefix; the client self-heals on the 400 and remembers
|
||||
* an empty completion (reasoning models burning the whole budget on hidden
|
||||
thoughts) is retried rather than silently propagated as ""
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .config import CFG
|
||||
from .log import log
|
||||
|
||||
PRICES = {
|
||||
"google/gemini-3.5-flash-lite": (0.3, 2.5),
|
||||
"google/gemini-3.5-flash": (1.5, 9.0),
|
||||
"anthropic/claude-haiku-4.5": (1.0, 5.0),
|
||||
"anthropic/claude-sonnet-5": (2.0, 10.0),
|
||||
"anthropic/claude-sonnet-4.6": (3.0, 15.0),
|
||||
"anthropic/claude-opus-5": (5.0, 25.0),
|
||||
"qwen/qwen3.7-plus": (0.1373076, 0.5492304),
|
||||
}
|
||||
|
||||
_sem = threading.BoundedSemaphore(CFG.concurrency)
|
||||
_NO_TEMPERATURE: set[str] = {"anthropic/claude-sonnet-5", "anthropic/claude-opus-5"}
|
||||
_lock = threading.Lock()
|
||||
_key: list[str] = []
|
||||
|
||||
|
||||
def _auth() -> str:
|
||||
with _lock:
|
||||
if not _key:
|
||||
_key.append(CFG.api_key())
|
||||
return _key[0]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Usage:
|
||||
calls: int = 0
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
retries: int = 0
|
||||
failures: int = 0
|
||||
cost_usd: float = 0.0
|
||||
latencies: list[float] = field(default_factory=list)
|
||||
_lk: threading.Lock = field(default_factory=threading.Lock, repr=False)
|
||||
|
||||
def add(self, model, pt, ct, latency, retries=0):
|
||||
with self._lk:
|
||||
self.calls += 1
|
||||
self.prompt_tokens += pt
|
||||
self.completion_tokens += ct
|
||||
self.retries += retries
|
||||
if latency:
|
||||
self.latencies.append(latency)
|
||||
pin, pout = PRICES.get(model, (0.0, 0.0))
|
||||
self.cost_usd += pt / 1e6 * pin + ct / 1e6 * pout
|
||||
|
||||
def summary(self) -> dict:
|
||||
lat = sorted(self.latencies)
|
||||
return {
|
||||
"calls": self.calls,
|
||||
"prompt_tokens": self.prompt_tokens,
|
||||
"completion_tokens": self.completion_tokens,
|
||||
"cost_usd": round(self.cost_usd, 4),
|
||||
"retries": self.retries,
|
||||
"failures": self.failures,
|
||||
"latency_avg": round(sum(lat) / len(lat), 2) if lat else 0,
|
||||
"latency_max": round(lat[-1], 2) if lat else 0,
|
||||
}
|
||||
|
||||
|
||||
def _cache_path(model: str, payload: str):
|
||||
h = hashlib.sha256(f"{model}\x00{payload}".encode()).hexdigest()[:32]
|
||||
d = CFG.cache_dir / "llm" / model.replace("/", "_")
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d / f"{h}.json"
|
||||
|
||||
|
||||
def chat(model: str, prompt: str, *, system: str | None = None,
|
||||
max_tokens: int = 8000, temperature: float = 0.0,
|
||||
usage: Usage | None = None) -> str:
|
||||
msgs = ([{"role": "system", "content": system}] if system else []) + \
|
||||
[{"role": "user", "content": prompt}]
|
||||
|
||||
def payload_of() -> str:
|
||||
body = {"model": model, "messages": msgs, "max_tokens": max_tokens}
|
||||
if model not in _NO_TEMPERATURE:
|
||||
body["temperature"] = temperature
|
||||
return json.dumps(body, ensure_ascii=False, sort_keys=True)
|
||||
|
||||
payload = payload_of()
|
||||
cpath = _cache_path(model, payload) if CFG.cache_llm else None
|
||||
if cpath is not None and cpath.exists():
|
||||
try:
|
||||
c = json.loads(cpath.read_text())
|
||||
if usage:
|
||||
usage.add(model, 0, 0, 0)
|
||||
return c["text"]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
retries = 0
|
||||
last = None
|
||||
for attempt in range(CFG.max_retries):
|
||||
with _sem:
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
CFG.endpoint, data=payload.encode(),
|
||||
headers={"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {_auth()}"})
|
||||
t0 = time.time()
|
||||
with urllib.request.urlopen(req, timeout=CFG.http_timeout) as r:
|
||||
d = json.loads(r.read())
|
||||
latency = time.time() - t0
|
||||
except urllib.error.HTTPError as e:
|
||||
detail = ""
|
||||
try:
|
||||
detail = e.read().decode("utf-8", "ignore")
|
||||
except Exception:
|
||||
pass
|
||||
last = f"HTTP{e.code}: {detail[:200]}"
|
||||
if e.code == 400 and "temperature" in detail:
|
||||
with _lock:
|
||||
_NO_TEMPERATURE.add(model)
|
||||
payload = payload_of()
|
||||
if CFG.cache_llm:
|
||||
cpath = _cache_path(model, payload)
|
||||
retries += 1
|
||||
continue
|
||||
if e.code not in (402, 408, 429, 500, 502, 503, 504):
|
||||
if usage:
|
||||
usage.failures += 1
|
||||
raise RuntimeError(f"{model} {last}") from e
|
||||
retries += 1
|
||||
delay = (min(20 * (attempt + 1), 120) if e.code == 402
|
||||
else min(2 ** attempt + random.uniform(0, 1.5), 45))
|
||||
log(f"llm retry {attempt+1}/{CFG.max_retries} {model} {last} "
|
||||
f"sleep {delay:.0f}s", level="WARN")
|
||||
time.sleep(delay)
|
||||
continue
|
||||
except Exception as e: # network / SSL / timeout
|
||||
last = repr(e)
|
||||
retries += 1
|
||||
time.sleep(min(2 ** attempt + random.uniform(0, 1.5), 45))
|
||||
continue
|
||||
|
||||
text = (d["choices"][0]["message"].get("content") or "").strip()
|
||||
u = d.get("usage", {})
|
||||
pt, ct = u.get("prompt_tokens", 0), u.get("completion_tokens", 0)
|
||||
if not text:
|
||||
last = "empty completion"
|
||||
retries += 1
|
||||
time.sleep(1.5 * (attempt + 1))
|
||||
continue
|
||||
if cpath is not None:
|
||||
cpath.write_text(json.dumps({"text": text}, ensure_ascii=False))
|
||||
if usage:
|
||||
usage.add(model, pt, ct, latency, retries=retries)
|
||||
return text
|
||||
|
||||
if usage:
|
||||
usage.failures += 1
|
||||
raise RuntimeError(f"{model} failed after {CFG.max_retries} attempts: {last}")
|
||||
|
||||
|
||||
def chat_many(model: str, prompts: list[str], **kw) -> list[str]:
|
||||
"""Parallel map preserving order. Concurrency is bounded by the module
|
||||
semaphore, so handing it an 87-chunk transcript is safe."""
|
||||
import concurrent.futures as cf
|
||||
|
||||
out: list[str] = [""] * len(prompts)
|
||||
errors: list[tuple[int, BaseException]] = []
|
||||
with cf.ThreadPoolExecutor(max_workers=CFG.concurrency) as ex:
|
||||
futs = {ex.submit(chat, model, p, **kw): i for i, p in enumerate(prompts)}
|
||||
for f in cf.as_completed(futs):
|
||||
i = futs[f]
|
||||
try:
|
||||
out[i] = f.result()
|
||||
except BaseException as e: # noqa: BLE001 - recorded, decided by caller
|
||||
errors.append((i, e))
|
||||
if errors:
|
||||
# Partial failure must not silently yield empty chunks: the caller falls
|
||||
# back to the original text for these indices, which the gate then flags.
|
||||
for i, e in errors:
|
||||
log(f"chunk {i} failed: {e}", level="ERROR")
|
||||
raise PartialFailure(out, [i for i, _ in errors])
|
||||
return out
|
||||
|
||||
|
||||
class PartialFailure(Exception):
|
||||
def __init__(self, outputs: list[str], failed: list[int]):
|
||||
super().__init__(f"{len(failed)} of {len(outputs)} chunks failed")
|
||||
self.outputs = outputs
|
||||
self.failed = failed
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Logging to both stderr and the shared log file.
|
||||
|
||||
Kept trivial and dependency-free on purpose: systemd captures stderr, while the
|
||||
file gives the same `[timestamp] message` shape the legacy shell pipeline wrote,
|
||||
so existing habits (`tail -f OBSIDIAN/logs/…`) keep working.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from .config import CFG
|
||||
|
||||
_lk = threading.Lock()
|
||||
|
||||
|
||||
def log(msg: str, *, level: str = "INFO") -> None:
|
||||
line = f"[{time.strftime('%F %T %z')}] {level}: {msg}"
|
||||
with _lk:
|
||||
print(line, file=sys.stderr, flush=True)
|
||||
try:
|
||||
CFG.log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with CFG.log_file.open("a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
except Exception:
|
||||
pass # never let logging break the pipeline
|
||||
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Conservative Chinese-number classification and normalization."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import asdict, dataclass
|
||||
|
||||
DIGIT = {
|
||||
"零": 0, "〇": 0, "一": 1, "幺": 1, "二": 2, "两": 2, "三": 3,
|
||||
"四": 4, "五": 5, "六": 6, "七": 7, "八": 8, "九": 9,
|
||||
}
|
||||
SMALL = {"十": 10, "百": 100, "千": 1000}
|
||||
BIG = {"万": 10_000, "亿": 100_000_000}
|
||||
CN = "零〇一二两三四五六七八九十百千万亿幺"
|
||||
CN_RE = rf"[{CN}]+"
|
||||
PURE_DIGITS = set(DIGIT)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Decision:
|
||||
raw: str
|
||||
normalized: str
|
||||
kind: str
|
||||
confidence: float
|
||||
start: int
|
||||
end: int
|
||||
context: str
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def chinese_to_int(raw: str) -> int | None:
|
||||
raw = raw.replace("两", "二")
|
||||
if not raw or any(c not in DIGIT and c not in SMALL and c not in BIG for c in raw):
|
||||
return None
|
||||
if set(raw) <= PURE_DIGITS:
|
||||
return int("".join(str(DIGIT[c]) for c in raw))
|
||||
# "两万五" is conventionally ambiguous between 20,005 and 25,000.
|
||||
if any(c in BIG for c in raw) and raw[-1] in DIGIT:
|
||||
return None
|
||||
total = section = number = 0
|
||||
for c in raw:
|
||||
if c in DIGIT:
|
||||
number = DIGIT[c]
|
||||
elif c in SMALL:
|
||||
section += (number or 1) * SMALL[c]
|
||||
number = 0
|
||||
else:
|
||||
section += number
|
||||
total += (section or 1) * BIG[c]
|
||||
section = number = 0
|
||||
return total + section + number
|
||||
|
||||
|
||||
def _ctx(text: str, start: int, end: int, width: int = 18) -> str:
|
||||
return text[max(0, start - width):min(len(text), end + width)]
|
||||
|
||||
|
||||
def _add(found: list[Decision], text: str, start: int, end: int,
|
||||
normalized: str, kind: str, confidence: float):
|
||||
raw = text[start:end]
|
||||
if raw == normalized:
|
||||
return
|
||||
found.append(Decision(raw, normalized, kind, confidence, start, end,
|
||||
_ctx(text, start, end)))
|
||||
|
||||
|
||||
def _overlaps(a: Decision, accepted: list[Decision]) -> bool:
|
||||
return any(a.start < b.end and b.start < a.end for b in accepted)
|
||||
|
||||
|
||||
def _money_text(raw: str, currency: str) -> str | None:
|
||||
if raw and raw[-1] in BIG:
|
||||
base = chinese_to_int(raw[:-1])
|
||||
return f"{base}{raw[-1]}{currency}" if base is not None else None
|
||||
value = chinese_to_int(raw)
|
||||
return f"{value}{currency}" if value is not None else None
|
||||
|
||||
|
||||
def normalize(text: str) -> tuple[str, list[dict]]:
|
||||
"""Normalize only values whose semantic type is visible in nearby syntax."""
|
||||
found: list[Decision] = []
|
||||
|
||||
# Percentages.
|
||||
for m in re.finditer(rf"百分之(?P<n>{CN_RE})", text):
|
||||
value = chinese_to_int(m.group("n"))
|
||||
if value is not None:
|
||||
_add(found, text, m.start(), m.end(), f"{value}%", "percentage", 0.99)
|
||||
|
||||
# Full dates and standalone day-of-month references.
|
||||
for m in re.finditer(
|
||||
rf"(?:(?P<y>{CN_RE})年)?(?P<mo>{CN_RE})月(?P<d>{CN_RE})(?P<u>日|号)", text):
|
||||
y = chinese_to_int(m.group("y")) if m.group("y") else None
|
||||
mo, d = chinese_to_int(m.group("mo")), chinese_to_int(m.group("d"))
|
||||
if mo is not None and d is not None and 1 <= mo <= 12 and 1 <= d <= 31:
|
||||
norm = f"{y}年" if y is not None else ""
|
||||
norm += f"{mo}月{d}{m.group('u')}"
|
||||
_add(found, text, m.start(), m.end(), norm, "date", 0.98)
|
||||
for m in re.finditer(rf"(?P<n>{CN_RE})(?P<u>日|号)", text):
|
||||
value = chinese_to_int(m.group("n"))
|
||||
if value is not None and 1 <= value <= 31:
|
||||
_add(found, text, m.start(), m.end(), f"{value}{m.group('u')}",
|
||||
"day", 0.93)
|
||||
|
||||
# Money: preserve the stated currency and avoid silently inventing one.
|
||||
for m in re.finditer(
|
||||
rf"(?P<n>{CN_RE})(?P<u>元|块钱|人民币|美元|美金|港币)", text):
|
||||
norm = _money_text(m.group("n"), m.group("u"))
|
||||
if norm is not None:
|
||||
_add(found, text, m.start(), m.end(), norm,
|
||||
"money", 0.98)
|
||||
for m in re.finditer(rf"(?P<n>[一二两三四五六七八九十百千万]+)个(?P<u>亿|万)", text):
|
||||
value = chinese_to_int(m.group("n"))
|
||||
if value is not None and re.search(r"投|融|估值|收入|销售|利润|资金|金额",
|
||||
_ctx(text, m.start(), m.end())):
|
||||
_add(found, text, m.start(), m.end(), f"{value}{m.group('u')}元",
|
||||
"money", 0.90)
|
||||
for m in re.finditer(
|
||||
rf"(?P<n>[一二两三四五六七八九十百千]+)(?P<u>万|亿)", text):
|
||||
context = _ctx(text, m.start(), m.end())
|
||||
value = chinese_to_int(m.group("n"))
|
||||
if value is not None and re.search(
|
||||
r"投|融|估值|收入|销售|利润|资金|金额|打款|退出|回购|报价", context):
|
||||
_add(found, text, m.start(), m.end(),
|
||||
f"{value}{m.group('u')}元", "money", 0.91)
|
||||
|
||||
# Quantities with an explicit unit.
|
||||
units = "个月|小时|分钟|公斤|毫克|毫升|人|家|份|批|次|台|套|件|天|年|吨|克|升|例|名"
|
||||
for m in re.finditer(rf"(?P<n>{CN_RE})(?P<u>{units})", text):
|
||||
value = chinese_to_int(m.group("n"))
|
||||
if value is not None:
|
||||
_add(found, text, m.start(), m.end(), f"{value}{m.group('u')}",
|
||||
"quantity", 0.96)
|
||||
|
||||
# Codes are spoken digit-by-digit. 幺 is a particularly strong signal.
|
||||
for m in re.finditer(
|
||||
rf"(?<![{CN}])(?P<n>[零〇一二两三四五六七八九幺]{{2,}})(?![{CN}])",
|
||||
text):
|
||||
raw = m.group("n")
|
||||
context = _ctx(text, m.start(), m.end())
|
||||
code_context = ("幺" in raw or
|
||||
re.search(r"项目|编号|代号|批号|工单|报告|BF|DF|CF|MFG",
|
||||
context, re.I))
|
||||
if code_context:
|
||||
value = "".join(str(DIGIT[c]) for c in raw)
|
||||
_add(found, text, m.start(), m.end(), value, "code", 0.97)
|
||||
|
||||
# Highest-confidence, longest non-overlapping decisions win.
|
||||
accepted: list[Decision] = []
|
||||
for d in sorted(found, key=lambda x: (-x.confidence, -(x.end - x.start), x.start)):
|
||||
if not _overlaps(d, accepted):
|
||||
accepted.append(d)
|
||||
|
||||
out = text
|
||||
for d in sorted(accepted, key=lambda x: x.start, reverse=True):
|
||||
out = out[:d.start] + d.normalized + out[d.end:]
|
||||
|
||||
# Preserve unresolved numeric speech as an explicit structured observation.
|
||||
uncertain: list[Decision] = []
|
||||
occupied = accepted[:]
|
||||
for m in re.finditer(CN_RE, text):
|
||||
if len(m.group()) < 2:
|
||||
continue
|
||||
d = Decision(m.group(), m.group(), "unknown", 0.30, m.start(), m.end(),
|
||||
_ctx(text, m.start(), m.end()))
|
||||
if not _overlaps(d, occupied):
|
||||
uncertain.append(d)
|
||||
all_decisions = sorted(accepted + uncertain, key=lambda x: x.start)
|
||||
return out, [d.as_dict() for d in all_decisions]
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prompt templates for the staged pipeline.
|
||||
|
||||
Kept in one module so wording changes are reviewable in isolation — prompt text is
|
||||
the most-edited and least-typechecked part of the system.
|
||||
"""
|
||||
|
||||
CLEAN_SYS = (
|
||||
"你只做会议转写的去口语化和断句，不做事实判断、术语纠错或总结。"
|
||||
)
|
||||
|
||||
CLEAN_RULES = """任务：对给定片段做「去口语化」。
|
||||
|
||||
只做三件事：
|
||||
1. 删除无意义语气词、结巴和紧邻的机械重复。
|
||||
2. 补标点，把明显破碎的口语整理成可读句子。
|
||||
3. 保留原有说话人分段。
|
||||
|
||||
不要总结，不要纠正人名、术语或数字，不要增加解释。听不懂的词原样保留。
|
||||
数字、英文、项目代号、【说话人】和 `[…重复噪声已折叠：…]` 必须保留。
|
||||
只输出清洗后的正文。"""
|
||||
|
||||
NORMALIZE_SYS = (
|
||||
"你是会议转写的实体、术语和数字规范化裁决器。只处理给出的候选，不做去口语化或总结。"
|
||||
)
|
||||
|
||||
NORMALIZE_RULES = """任务：在清洗文本中裁决候选词并作最小修改。
|
||||
|
||||
- A 组映射已经由程序执行，不要撤销。
|
||||
- B 组只有在整句语义、人物/公司/项目关系和近期语境共同支持时才替换。
|
||||
- 多个候选无法区分时保留原词，紧跟 `[?候选A/候选B]`。
|
||||
- 中文数字仅在类型和单位明确时转成阿拉伯数字；项目编号逐字转换，如“四幺八二”→“4182”。
|
||||
- 数值、单位或身份不确定时保留原文并加 `[?…]`，不得猜测。
|
||||
- 不增加候选清单和背景资料中没有的事实。
|
||||
|
||||
只输出规范化后的正文，保留全部【说话人】标签。"""
|
||||
|
||||
EVIDENCE_SYS = "你从一段会议转写中提取可追溯的事实卡片，不撰写纪要。"
|
||||
|
||||
EVIDENCE_PROMPT = """从下面片段提取值得进入会议草稿的关键信息。
|
||||
|
||||
保留：结论、决定、行动项、负责人、关键数字、争议、未决问题、重要关系和判断依据。
|
||||
省略：寒暄、重复表态、无意义细节。
|
||||
不得补充背景事实，不确定的主体或数字必须标“待确认”。
|
||||
每条都带片段编号 `{chunk_id}`，格式：
|
||||
- [片段 {chunk_id}] 类型｜主体｜事实或线索｜置信度:明确/较可信/待确认
|
||||
|
||||
片段：
|
||||
{body}"""
|
||||
|
||||
JUDGE_RULES = """附加任务：术语/人名校正裁决。
|
||||
|
||||
下面给出脚本按「发音相近 + 本场语境」自动圈出的**候选**（只是嫌疑，未必对）。
|
||||
每个候选都带来源说明，括号里的含义：
|
||||
- `已确认别名`：Kai 已经确认过的固定叫法或已知误识别映射。**证据最强**，若句子读得通就应当替换。
|
||||
- `机构简称`／`去姓称名`／`称谓`：口语里的简称形式（如「迈威」指迈威生物、「光辉」指李光辉、
|
||||
「老万」「万总」指姓万的人）。**这类是正常口语，不是错字**——判定为同一实体后直接用全名。
|
||||
- `一字之差`／`同篇另一写法`：同一实体在本篇中的其他写法，通常是转写漂移，应统一。
|
||||
- `X(本次在谈)`：该候选所属公司正是本次录音讨论的对象，是**强支持证据**。
|
||||
- `X(近期相关)`：该公司近期与本人有往来，是弱支持证据。
|
||||
|
||||
候选分两组：
|
||||
- **A 组「已确认映射」**：Kai 已人工核对的固定对应。只要句中该词确实指人/机构，**必须替换**，
|
||||
不要保留原写法、不要加「疑似」、不要标 `[?…]`。这类是 ASR 的固定误识别或口语简称，
|
||||
不是需要你判断的疑点。
|
||||
- **B 组「待判断候选」**：只是嫌疑，按下面规则裁决。
|
||||
|
||||
逐条按**它所在的整句**判断：
|
||||
- 如果原词在句中读得通（是常用词、副词、动宾结构），**保留原词**，不要改。
|
||||
- 候选带「已确认别名」或带「(本次在谈)」支持、且句中指的确实是人/机构时，**应当替换为全名**，
|
||||
不要因为"转写没明说"而回避——简称就是口语常态。
|
||||
- 同一候选有多个可能人物时（例如「徐总」可能是三个人），**不要硬选**：保留原词并标注
|
||||
`[?候选A/候选B]`，把判断留给后续环节。
|
||||
- 其余拿不准的：保留原词，并在该词后紧跟标注 `[?候选词]`（例如 `元烨[?原液]`）。
|
||||
- 标注了「此词本身是常用词」的，默认保留原词，除非句子完全读不通。
|
||||
|
||||
候选清单：
|
||||
{candidates}"""
|
||||
|
||||
CONTEXT_FOR_CLEAN = """本场背景（来自既有笔记与长期记忆，仅供你判断术语与人名的写法，**不要**把这些内容写进输出）：
|
||||
{context}"""
|
||||
|
||||
KEYWORD_PROMPT = """从下面的会议转写摘选中，提取用于检索背景资料的关键词。
|
||||
只要专有名词：人名、公司名、项目/产品代号、机构、专业术语。
|
||||
转写有识别错误，可以给出你认为正确的写法。
|
||||
输出一行，用空格分隔，最多 25 个词，不要任何其他内容。
|
||||
|
||||
转写摘选：
|
||||
{sample}"""
|
||||
|
||||
# The provenance rule below exists because of a measured failure: with background
|
||||
# context injected, drafts asserted context-derived facts as transcript facts
|
||||
# (「谭凯（白帆生物总经理）」、「此项目负责人为 [[冯昆仑]]（背景确认）」) even when the
|
||||
# recording never said them. The facts were often correct, but a meeting note that
|
||||
# cannot distinguish "said on the call" from "known from notes" is unsafe to finalize.
|
||||
DRAFT_PROMPT = """你是会议录音整理助手，正在根据**带来源编号的事实卡片**产出「粗整理草稿」。
|
||||
|
||||
严格遵循：
|
||||
1. 只产出粗整理草稿，**不是**正式纪要。
|
||||
2. 清楚区分三类信息：明确事实 / 讨论中的想法 / 推测·高不确定。
|
||||
3. **出处标注（重要）**：本草稿的每一条断言都必须能追溯到来源。
|
||||
- 来自本次转写的内容：正常写。
|
||||
- 来自下方「本场背景」而转写中**没有**说到的内容（例如某人的职务、公司归属、
|
||||
项目历史）：必须加后缀 `〔背景〕`，并且**不得**写成「明确事实」。
|
||||
- 你自己的推断：加后缀 `〔推断〕` 或写「疑似」。
|
||||
- 绝不把背景知识或推断伪装成转写内容。转写里没说的职务、公司名、金额、日期，
|
||||
一律不能不加标注地断言。
|
||||
4. 用 [[双链]] 标注有把握的人物/公司/项目；不确定的用纯文本 + 「疑似」。
|
||||
5. 转写中形如 `词[?候选]` 的标注表示清洗阶段无法确定的词，请结合全文语境判断，
|
||||
并在「待确认」中列出。
|
||||
6. 形如 `[…重复噪声已折叠：…]` 的标记表示该处录音是长段非语音噪声（保持音/静默），
|
||||
不要当成内容；如影响理解可在「待确认」提一句。
|
||||
7. 末尾必须有一节 `## 待你确认`，列出需要人工复核的关键不确定点（名词、说话人归属、
|
||||
日期、金额、结构）。
|
||||
7a. 正文标题后、议题正文前必须先写 `## 通话背景（待确认）`，用 2-4 句话概括：
|
||||
通话类型、参与方、正在处理的事项、各方关系。只写本次转写能支持的内容；推断必须
|
||||
明标「疑似」。这段会先交给用户判断背景是否靠谱，以及是否值得携带正确背景重跑。
|
||||
|
||||
**结构要求（按议题归拢，不要平铺）**：
|
||||
8. 正文按**议题**分节（一、二、三…），一个议题一节。每节内部用这个顺序：
|
||||
先一句「结论/当前状态」，再列支撑事实，最后写「待办/下一步」（如果有）。
|
||||
同一议题的事实必须收在同一节，不要因为在录音里出现的先后不同就拆成多条平行 bullet。
|
||||
9. 事实合并：同一件事的多次提及、同一金额/时间的重复确认、同一人的多轮补充，
|
||||
**合并成一条**并保留关键数字与限定条件，不要逐句罗列。
|
||||
单节内的 bullet 控制在 7 条以内；超出说明该拆成子议题或该合并。
|
||||
10. 每条 bullet 只讲一件事，避免「A，同时 B，另外 C」的堆叠长句。
|
||||
|
||||
**关于人名与简称（重要）**：
|
||||
11. 转写里的简称、称谓、老X/X总/X博就是口语常态，不是缺陷。若上下文足以确定指谁
|
||||
（例如正在谈某公司、而背景里该公司只有一位对应角色的人），**直接写全名并加 [[双链]]**，
|
||||
不要写成「具体身份不确定，需人工核实」。这类过度回避会让草稿失去价值。
|
||||
12. 只有在**真的存在多个候选且无法区分**时才标「疑似」，并把候选一起列出
|
||||
（例如「徐总（疑似 [[徐劲前]] 或 [[邹洵]]）」），而不是笼统写「身份不明」。
|
||||
13. 判定简称指向谁时使用背景里的任职信息，这属于**辨识**而非越界断言，
|
||||
不需要加〔背景〕；但如果你要额外陈述这个人的职务、公司归属本身，那就要加〔背景〕。
|
||||
14. 对匿名说话人，除非事实卡片已经给出有证据的候选，否则只写“身份待确认”；
|
||||
不要从背景中枚举可能的人名让用户猜。
|
||||
|
||||
{speaker_note}
|
||||
|
||||
{context}
|
||||
|
||||
frontmatter 严格用下列字段：
|
||||
---
|
||||
date: {date}
|
||||
type: minutes
|
||||
tags: [会议纪要, 草稿, 待确认]
|
||||
source: 录音转录
|
||||
source_detail: {source_detail}
|
||||
status: 草稿待确认
|
||||
people: [有把握的人名…]
|
||||
companies: [有把握的公司…]
|
||||
transcript: {transcript}
|
||||
{extra_fm}---
|
||||
|
||||
事实卡片：
|
||||
{body}
|
||||
|
||||
只输出 markdown 草稿全文，不要任何额外说明。"""
|
||||
|
||||
SPEAKER_NOTE = {
|
||||
"qwen3cam": (
|
||||
"转写来自 Qwen3-ASR + CAM++ 分角色 + 声纹：\n"
|
||||
"- 中文真名的说话人是**声纹已确认**的，直接用，不要写「疑似」。\n"
|
||||
"- `speaker_N` 是声纹未命名的说话人，按内容/口吻推断是谁并标「疑似」。\n"
|
||||
"- 不同 speaker_N 是不同真人（CAM++ 已分），但可能过度聚类（同一人被拆成多个 N），"
|
||||
"按内容合并同一真人。\n"
|
||||
"- frontmatter 的 people 优先填**声纹命名的真人**。"),
|
||||
"vibevoice": "转写带 S# 说话人标签，仅作软提示，块间编号不保证一致。",
|
||||
"qwen3-plain": "转写**无说话人信息**，据内容推断发言归属，不要臆造 speaker 标签。",
|
||||
}
|
||||
@@ -0,0 +1,468 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Stage orchestration: transcript text -> rough-outline draft.
|
||||
|
||||
S0 segment collapse ASR degeneration, split into speaker-aware chunks
|
||||
S1 flag phonetic + term-bank candidates, Hindsight in-play recall
|
||||
S1c keyword LLM keyword extraction -> Hindsight recall (restores context
|
||||
for backends that emit no speaker tags)
|
||||
S2 clean per-chunk 去口语化 + per-sentence candidate adjudication
|
||||
S3 gate deterministic fidelity check, strict retry, fallback to source
|
||||
S4 draft one strong-model call, structure + provenance labelling only
|
||||
|
||||
Design invariant: **no stage may silently lose content**. S3 is what makes that
|
||||
true; without it, per-chunk cleaning dropped up to 22% of English terms and 11% of
|
||||
numbers on the evaluation set, and no amount of prompt wording fixed it.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from . import context as context_compactor
|
||||
from . import fidelity, flagger, numbers as number_norm, prompts, segment
|
||||
from .config import CFG
|
||||
from .llm import PartialFailure, Usage, chat, chat_many
|
||||
from .log import log
|
||||
|
||||
SOURCE_DETAIL = {
|
||||
"qwen3cam": "Qwen3-ASR 1.7B + CAM++ 分角色 + 声纹自动命名",
|
||||
"vibevoice": "VibeVoice 分角色",
|
||||
"qwen3-plain": "Qwen3-ASR 1.7B",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Result:
|
||||
draft: str
|
||||
cleaned: str
|
||||
meta: dict
|
||||
usage: dict
|
||||
timings: dict
|
||||
stats: dict
|
||||
flags: list = field(default_factory=list)
|
||||
context: str = ""
|
||||
evidence: str = ""
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ S1c
|
||||
def keyword_recall(raw: str, usage: Usage) -> tuple[set[str], list[str], str]:
|
||||
"""LLM keyword extraction, then Hindsight recall on those keywords.
|
||||
|
||||
Needed because the phonetic flagger seeds its in-play entity set from speaker
|
||||
tags: on `qwen3-plain` transcripts there are none, and measurement showed
|
||||
in_play=0 for every such fixture — no vault person or company could ever be
|
||||
offered as a candidate. Reading names out of the text restores that signal and
|
||||
was the single biggest quality lever (term-correction recall 0.21 -> 0.40).
|
||||
"""
|
||||
head, mid = raw[:3000], raw[len(raw) // 2: len(raw) // 2 + 3000]
|
||||
try:
|
||||
out = chat(CFG.keyword_model,
|
||||
prompts.KEYWORD_PROMPT.format(sample=head + "\n…\n" + mid),
|
||||
max_tokens=300, usage=usage)
|
||||
except Exception as e:
|
||||
log(f"keyword recall failed, continuing without it: {e}", level="WARN")
|
||||
return set(), [], ""
|
||||
words = [w.strip("、,，。;；:：·") for w in re.split(r"[\s，、]+", out) if w.strip()]
|
||||
words = [w for w in words if 2 <= len(w) <= 12][:25]
|
||||
names, texts = flagger.hindsight_recall(words, "会议 背景 相关人物 公司 项目")
|
||||
return names, texts, " ".join(words)
|
||||
|
||||
|
||||
def build_context(fa: dict, extra_texts: list[str], limit: int = 6) -> str:
|
||||
names = fa.get("hindsight_names") or []
|
||||
texts = (fa.get("hindsight_texts") or []) + (extra_texts or [])
|
||||
try:
|
||||
from . import lexicon
|
||||
texts += lexicon.relationship_card(set(names), limit=12)
|
||||
except Exception:
|
||||
pass
|
||||
compacted = context_compactor.compact(names, texts, per_entity=2,
|
||||
total=max(6, limit * 2))
|
||||
head = "相关实体：" + "、".join(names[:30]) if names else ""
|
||||
return "\n".join(x for x in (head, compacted) if x)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ S2 + S3
|
||||
def clean_chunks(chunks, usage: Usage) -> tuple[list[str], list[dict], dict]:
|
||||
vocab = fidelity.domain_terms() if CFG.use_termbank else set()
|
||||
stats = {"skipped_trivial": 0, "retried": 0, "fallback": 0, "gate_failed": 0}
|
||||
|
||||
def trivial(c) -> bool:
|
||||
stripped = re.sub(r"\[…重复噪声已折叠：.*?…\]|[\s【】]", "", c.body)
|
||||
return len(stripped) < 40
|
||||
|
||||
prompt_list, target_idx = [], []
|
||||
for c in chunks:
|
||||
if trivial(c):
|
||||
continue
|
||||
parts = [prompts.CLEAN_RULES]
|
||||
if c.overlap_prefix:
|
||||
parts.append("上一片段结尾（只读上下文，**不要**重复输出这部分）：\n"
|
||||
+ c.overlap_prefix)
|
||||
parts.append("待清洗片段：\n" + c.body)
|
||||
prompt_list.append("\n\n".join(parts))
|
||||
target_idx.append(c.idx)
|
||||
stats["skipped_trivial"] = len(chunks) - len(prompt_list)
|
||||
|
||||
max_tok = min(16000, 1200 + 3 * max((c.chars for c in chunks), default=600))
|
||||
outs: list[str] = []
|
||||
if prompt_list:
|
||||
try:
|
||||
outs = chat_many(CFG.clean_model, prompt_list,
|
||||
system=prompts.CLEAN_SYS, max_tokens=max_tok, usage=usage)
|
||||
except PartialFailure as pf:
|
||||
# Keep what succeeded; failed indices fall back to source text below.
|
||||
outs = pf.outputs
|
||||
log(f"{len(pf.failed)} chunk(s) unrecoverable, using source text",
|
||||
level="WARN")
|
||||
|
||||
by_idx = dict(zip(target_idx, outs))
|
||||
prompt_by_idx = dict(zip(target_idx, prompt_list))
|
||||
|
||||
finals, verdicts = [], []
|
||||
for c in chunks:
|
||||
out = by_idx.get(c.idx) or c.body
|
||||
p = prompt_by_idx.get(c.idx)
|
||||
v = fidelity.check(c.body, out, vocab=vocab,
|
||||
min_ratio=CFG.gate_min_ratio, max_ratio=CFG.gate_max_ratio)
|
||||
if CFG.gate_enabled and not v.ok and p is not None:
|
||||
stats["retried"] += 1
|
||||
strict = (p + "\n\n**上一次输出不合格**：" + v.brief()
|
||||
+ "\n请重做。逐句改写，不要合并、不要省略、不要总结；"
|
||||
"数字、英文、说话人标签必须保留；听不懂的词原样保留。")
|
||||
try:
|
||||
out2 = chat(CFG.clean_model, strict, system=prompts.CLEAN_SYS,
|
||||
max_tokens=min(16000, 1500 + 3 * c.chars), usage=usage)
|
||||
v2 = fidelity.check(c.body, out2, vocab=vocab,
|
||||
min_ratio=CFG.gate_min_ratio,
|
||||
max_ratio=CFG.gate_max_ratio)
|
||||
if v2.ok or v2.ratio > v.ratio:
|
||||
out, v = out2, v2
|
||||
except Exception as e:
|
||||
log(f"strict retry failed for chunk {c.idx}: {e}", level="WARN")
|
||||
if not v.ok:
|
||||
# A visibly unclean chunk beats one with facts deleted.
|
||||
out = c.body
|
||||
v.reasons.append("两次失败→保留原文")
|
||||
stats["fallback"] += 1
|
||||
if not v.ok:
|
||||
stats["gate_failed"] += 1
|
||||
finals.append(out)
|
||||
verdicts.append(v.as_dict())
|
||||
return finals, verdicts, stats
|
||||
|
||||
|
||||
def _apply_settled(text: str, flags: list[dict]) -> tuple[str, list[dict]]:
|
||||
"""Apply unambiguous human-confirmed mappings without asking an LLM."""
|
||||
chosen: dict[str, str] = {}
|
||||
for g in flags:
|
||||
if not g.get("settled") or g.get("common_word") or not g.get("candidates"):
|
||||
continue
|
||||
chosen[g["span"]] = g["candidates"][0]["target"]
|
||||
markers: dict[str, str] = {}
|
||||
out = text
|
||||
for i, (surface, target) in enumerate(
|
||||
sorted(chosen.items(), key=lambda x: len(x[0]), reverse=True)):
|
||||
if surface not in out or surface == target:
|
||||
continue
|
||||
marker = f"\uFFF0ASR{i}\uFFF1"
|
||||
out = out.replace(surface, marker)
|
||||
markers[marker] = target
|
||||
for marker, target in markers.items():
|
||||
out = out.replace(marker, target)
|
||||
decisions = [{"surface": s, "target": t, "mode": "deterministic"}
|
||||
for s, t in chosen.items() if s in text]
|
||||
return out, decisions
|
||||
|
||||
|
||||
def _common_lexical_context(span: str, sentence: str,
|
||||
allowlist: set[str]) -> bool:
|
||||
"""Reject a candidate that cuts through an ordinary Chinese word."""
|
||||
if span in {"很好"}:
|
||||
return True
|
||||
for match in re.finditer(re.escape(span), sentence):
|
||||
start, end = match.span()
|
||||
lo, hi = max(0, start - 4), min(len(sentence), end + 4)
|
||||
for left in range(lo, end):
|
||||
for right in range(max(left + 2, start + 1),
|
||||
min(hi, left + 6) + 1):
|
||||
if right <= start or left >= end:
|
||||
continue
|
||||
word = sentence[left:right]
|
||||
if (word in allowlist
|
||||
and re.fullmatch(r"[\u3400-\u9fff]+", word)
|
||||
and (left < start or right > end)):
|
||||
return True
|
||||
if (len(span) == 2 and span[-1] in "总哥姐"
|
||||
and start > 0
|
||||
and re.fullmatch(r"[\u3400-\u9fff]", sentence[start - 1])
|
||||
and sentence[start - 1] not in "问跟和找请让给叫说"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _queue_ambiguous(recording: str, groups: list[dict],
|
||||
normalized: str = "") -> int:
|
||||
"""Queue only ambiguities the context adjudicator explicitly left unresolved."""
|
||||
try:
|
||||
from . import review
|
||||
except Exception:
|
||||
return 0
|
||||
markers = re.findall(r"\[\?([^\]]+)\]", normalized)
|
||||
if not markers:
|
||||
return 0
|
||||
allowlist = flagger.load_allowlist()
|
||||
n = 0
|
||||
for g in groups:
|
||||
cs = g.get("candidates") or []
|
||||
if g.get("settled") or g.get("common_word") or not cs:
|
||||
continue
|
||||
if _common_lexical_context(
|
||||
g["span"], str(g.get("sentence", "")), allowlist):
|
||||
continue
|
||||
choices = [c["target"] for c in cs[:3]]
|
||||
if not any(g["span"] in marker
|
||||
or any(choice in marker for choice in choices)
|
||||
for marker in markers):
|
||||
continue
|
||||
top = cs[0]
|
||||
score = float(top.get("score", 0))
|
||||
if not (CFG.review_min_score <= score <= CFG.review_max_score):
|
||||
continue
|
||||
rule = top.get("rule", "")
|
||||
source = str(top.get("source", ""))
|
||||
supported = bool(top.get("boost") or top.get("support"))
|
||||
curated = rule == "alias" and source.startswith(
|
||||
("alias-card", "alias-learned", "alias-glossary", "alias-review"))
|
||||
consistent = rule == "consistency" and source == "consistency-alias"
|
||||
contextual_short = (rule in {"honorific", "org-short", "given"}
|
||||
and supported and score >= 0.90)
|
||||
via = str(top.get("via", ""))
|
||||
observed_neighbour = (
|
||||
rule == "observed" and supported and score >= 0.95
|
||||
and len(g["span"]) >= 3 and via
|
||||
and (g["span"][0] == via[0] or score >= 0.99))
|
||||
# A broad phonetic channel is useful for recall but is not, by itself,
|
||||
# sufficient reason to interrupt a human. "也说→叶硕" and "们七→范梦奇"
|
||||
# are representative false positives from that channel.
|
||||
if not (curated or consistent or contextual_short or observed_neighbour):
|
||||
continue
|
||||
if len(choices) == 1:
|
||||
q = f"“{g['span']}”是不是指“{choices[0]}”？"
|
||||
else:
|
||||
q = f"“{g['span']}”更可能指哪一个：{' / '.join(choices)}？"
|
||||
review.enqueue(recording, q, {
|
||||
"surface": g["span"], "candidates": choices,
|
||||
"sentence": g.get("sentence", ""), "kind": top.get("kind", "")})
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def normalize_chunks(chunks, cleaned_parts: list[str], flags: list[dict],
|
||||
context: str, recording: str,
|
||||
usage: Usage, *,
|
||||
queue_review: bool = True) -> tuple[list[str], list[dict], dict]:
|
||||
"""S3: deterministic confirmed mappings, then narrow LLM adjudication."""
|
||||
prompts_by_idx: dict[int, str] = {}
|
||||
base_by_idx: dict[int, str] = {}
|
||||
decisions: list[dict] = []
|
||||
number_decisions: list[dict] = []
|
||||
unresolved_by_idx: dict[int, list[dict]] = {}
|
||||
queued = 0
|
||||
for c, cleaned in zip(chunks, cleaned_parts):
|
||||
cf = [g for g in flags if g["span"] in c.body]
|
||||
base, done = _apply_settled(cleaned, cf)
|
||||
decisions.extend({**d, "chunk": c.idx} for d in done)
|
||||
base, nums = number_norm.normalize(base)
|
||||
number_decisions.extend({**d, "chunk": c.idx} for d in nums)
|
||||
unresolved = [g for g in cf if not g.get("settled")]
|
||||
unresolved_by_idx[c.idx] = unresolved
|
||||
uncertain_numbers = [d for d in nums if d["confidence"] < 0.8]
|
||||
base_by_idx[c.idx] = base
|
||||
if not unresolved and not uncertain_numbers:
|
||||
continue
|
||||
parts = [prompts.NORMALIZE_RULES]
|
||||
if unresolved:
|
||||
parts.append(prompts.JUDGE_RULES.format(
|
||||
candidates=flagger.render_for_prompt(unresolved, 20)))
|
||||
if context:
|
||||
parts.append(prompts.CONTEXT_FOR_CLEAN.format(context=context))
|
||||
if uncertain_numbers:
|
||||
lines = "\n".join(
|
||||
f"- 原值「{d['raw']}」｜上下文：{d['context']}"
|
||||
for d in uncertain_numbers[:12])
|
||||
parts.append("以下中文数字类型不明确。不能可靠确定时原样保留并标待确认：\n"
|
||||
+ lines)
|
||||
parts.append("待规范化文本：\n" + base)
|
||||
prompts_by_idx[c.idx] = "\n\n".join(parts)
|
||||
|
||||
idxs = list(prompts_by_idx)
|
||||
outs = []
|
||||
if idxs:
|
||||
try:
|
||||
outs = chat_many(CFG.clean_model, [prompts_by_idx[i] for i in idxs],
|
||||
system=prompts.NORMALIZE_SYS, max_tokens=6000, usage=usage)
|
||||
except PartialFailure as pf:
|
||||
outs = pf.outputs
|
||||
normalized = dict(zip(idxs, outs))
|
||||
finals = []
|
||||
fallback = 0
|
||||
for c in chunks:
|
||||
base = base_by_idx[c.idx]
|
||||
out = normalized.get(c.idx) or base
|
||||
ratio = len(out) / max(1, len(base))
|
||||
src_speakers = set(fidelity.SPEAKER.findall(base))
|
||||
out_speakers = set(fidelity.SPEAKER.findall(out))
|
||||
if not (0.75 <= ratio <= 1.25) or not src_speakers.issubset(out_speakers):
|
||||
out = base
|
||||
fallback += 1
|
||||
elif queue_review:
|
||||
queued += _queue_ambiguous(
|
||||
recording, unresolved_by_idx.get(c.idx, []), out)
|
||||
finals.append(out)
|
||||
return finals, decisions, {
|
||||
"normalize_calls": len(idxs), "normalize_fallback": fallback,
|
||||
"review_queued": queued, "deterministic_corrections": len(decisions),
|
||||
"number_decisions": number_decisions,
|
||||
"numbers_normalized": sum(
|
||||
1 for d in number_decisions if d["raw"] != d["normalized"]),
|
||||
"numbers_uncertain": sum(
|
||||
1 for d in number_decisions if d["confidence"] < 0.8),
|
||||
}
|
||||
|
||||
|
||||
def extract_evidence(parts: list[str], usage: Usage) -> str:
|
||||
jobs = [prompts.EVIDENCE_PROMPT.format(chunk_id=i, body=body)
|
||||
for i, body in enumerate(parts)]
|
||||
if not jobs:
|
||||
return ""
|
||||
try:
|
||||
outs = chat_many(CFG.clean_model, jobs, system=prompts.EVIDENCE_SYS,
|
||||
max_tokens=1400, usage=usage)
|
||||
except PartialFailure as pf:
|
||||
outs = pf.outputs
|
||||
return "\n".join(x for x in outs if x.strip())
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ S4
|
||||
def write_draft(cleaned: str, meta: dict, context: str, date: str,
|
||||
transcript_path: str, usage: Usage) -> str:
|
||||
extra = ""
|
||||
intake = meta.get("intake") or {}
|
||||
if intake.get("job_id"):
|
||||
extra += f"intake_job: {intake['job_id']}\n"
|
||||
if meta.get("voiceprint_session"):
|
||||
extra += f"voiceprint_session: {meta['voiceprint_session']}\n"
|
||||
named = meta.get("voiceprint_named") or {}
|
||||
if named:
|
||||
extra += "voiceprint_speakers:\n"
|
||||
for k, v in named.items():
|
||||
extra += f" {k}: {v}\n"
|
||||
extra += ("# 说明：voiceprint_speakers 只写声纹已确认命名的；你据内容推断的匿名"
|
||||
"说话人不要当确认名写进来（会误导回注册），留正文「疑似」即可。\n")
|
||||
user_context = str(intake.get("context") or "").strip()
|
||||
if user_context:
|
||||
supplied = (
|
||||
"用户在上传录音时明确提供的背景（可用于辨识本场主题和实体；"
|
||||
"不能把其中未在录音出现的内容冒充为本场发言）：\n"
|
||||
+ user_context)
|
||||
context = supplied + ("\n\n" + context if context else "")
|
||||
ctx = prompts.CONTEXT_FOR_CLEAN.format(context=context) if context else ""
|
||||
if ctx:
|
||||
ctx = "本场背景（来自既有笔记与长期记忆；引用其中内容时必须按上面第 3 条加 〔背景〕）：\n" \
|
||||
+ context
|
||||
detail = SOURCE_DETAIL.get(meta.get("backend", ""), "ASR")
|
||||
prompt = prompts.DRAFT_PROMPT.format(
|
||||
speaker_note=prompts.SPEAKER_NOTE.get(meta.get("backend", ""), ""),
|
||||
context=ctx, date=date, source_detail=detail,
|
||||
transcript=transcript_path, extra_fm=extra, body=cleaned)
|
||||
return chat(CFG.draft_model, prompt, max_tokens=12000, usage=usage)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ driver
|
||||
def refine(raw: str, *, date: str, transcript_path: str,
|
||||
queue_review: bool = True) -> Result:
|
||||
usage = Usage()
|
||||
timings: dict = {}
|
||||
t_all = time.time()
|
||||
|
||||
t = time.time()
|
||||
chunks, meta = segment.chunk(raw, CFG.chunk_budget, CFG.chunk_overlap,
|
||||
collapse_noise=CFG.collapse_noise)
|
||||
timings["s0_segment"] = round(time.time() - t, 2)
|
||||
if meta.get("degenerate_chars"):
|
||||
ev = meta["degenerate_events"]
|
||||
log(f"S0 collapsed {meta['degenerate_chars']} chars of ASR degeneration "
|
||||
f"({len(ev)} run(s), e.g. {ev[0]['unit']!r}x{ev[0]['repeats']})")
|
||||
|
||||
# S1c runs *before* S1 so its entity names can seed the candidate space. Doing
|
||||
# it the other way round meant the recalled names only ever reached the prompt
|
||||
# as background prose, never as phonetic candidates — which is why transcripts
|
||||
# without speaker labels produced almost no flags.
|
||||
t = time.time()
|
||||
kw, extra_texts, names = "", [], set()
|
||||
if CFG.use_keyword_recall and CFG.use_hindsight:
|
||||
names, extra_texts, kw = keyword_recall(raw, usage)
|
||||
timings["s1c_recall"] = round(time.time() - t, 2)
|
||||
|
||||
t = time.time()
|
||||
fa = {"flags": [], "hindsight_names": [], "hindsight_texts": [], "in_play": []}
|
||||
if CFG.use_termbank or CFG.use_hindsight:
|
||||
try:
|
||||
fa = flagger.analyze(raw, use_hindsight=CFG.use_hindsight,
|
||||
use_termbank=CFG.use_termbank,
|
||||
extra_in_play=names)
|
||||
except Exception as e:
|
||||
log(f"flagger failed, continuing without candidates: {e}", level="WARN")
|
||||
if names:
|
||||
fa["hindsight_names"] = sorted(set(fa.get("hindsight_names", [])) | names)
|
||||
timings["s1_flag"] = round(time.time() - t, 2)
|
||||
|
||||
context = build_context(fa, extra_texts) if CFG.use_hindsight else ""
|
||||
|
||||
t = time.time()
|
||||
cleaned_parts, verdicts, stats = clean_chunks(chunks, usage)
|
||||
cleaned = "\n\n".join(cleaned_parts)
|
||||
timings["s2_clean"] = round(time.time() - t, 2)
|
||||
|
||||
t = time.time()
|
||||
normalized_parts = cleaned_parts
|
||||
if CFG.use_normalize:
|
||||
normalized_parts, decisions, nstats = normalize_chunks(
|
||||
chunks, cleaned_parts, fa["flags"], context, transcript_path, usage,
|
||||
queue_review=queue_review)
|
||||
stats.update(nstats)
|
||||
stats["normalization_decisions"] = decisions
|
||||
normalized = "\n\n".join(normalized_parts)
|
||||
timings["s3_normalize"] = round(time.time() - t, 2)
|
||||
|
||||
t = time.time()
|
||||
evidence = extract_evidence(normalized_parts, usage) if CFG.use_evidence else normalized
|
||||
timings["s4_evidence"] = round(time.time() - t, 2)
|
||||
|
||||
t = time.time()
|
||||
draft = write_draft(evidence, meta, context, date, transcript_path, usage)
|
||||
timings["s4_draft"] = round(time.time() - t, 2)
|
||||
|
||||
src_body = "\n\n".join(c.body for c in chunks)
|
||||
stats.update({
|
||||
"raw_chars": len(raw),
|
||||
"source_chars": len(src_body),
|
||||
"cleaned_chars": len(cleaned),
|
||||
"normalized_chars": len(normalized),
|
||||
"evidence_chars": len(evidence),
|
||||
"draft_chars": len(draft),
|
||||
"n_chunks": len(chunks),
|
||||
"n_turns": meta.get("n_turns", 0),
|
||||
"n_flags": len(fa["flags"]),
|
||||
"n_hindsight": len(fa.get("hindsight_names", [])),
|
||||
"keywords": kw,
|
||||
"degenerate_chars": meta.get("degenerate_chars", 0),
|
||||
"fidelity": fidelity.score_document(src_body, cleaned),
|
||||
"wall_s": round(time.time() - t_all, 2),
|
||||
})
|
||||
return Result(draft=draft, cleaned=normalized, meta=meta, usage=usage.summary(),
|
||||
timings=timings, stats=stats, flags=fa["flags"], context=context,
|
||||
evidence=evidence)
|
||||
@@ -0,0 +1,372 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Persistent human-review queue with transport-independent storage."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from . import lexicon
|
||||
from .config import CFG
|
||||
|
||||
FEEDBACK_START = "<!-- asr-web-review:start -->"
|
||||
FEEDBACK_END = "<!-- asr-web-review:end -->"
|
||||
CARD_THRESHOLD = 2
|
||||
|
||||
|
||||
def enqueue(recording: str, question: str, payload: dict) -> int:
|
||||
con = lexicon.connect()
|
||||
old = con.execute(
|
||||
"SELECT id FROM review_question WHERE recording=? AND question=? "
|
||||
"AND status='pending'", (recording, question)).fetchone()
|
||||
if old:
|
||||
con.close()
|
||||
return int(old["id"])
|
||||
cur = con.execute(
|
||||
"INSERT INTO review_question(recording,question,payload,created_at)"
|
||||
" VALUES(?,?,?,?)",
|
||||
(recording, question, json.dumps(payload, ensure_ascii=False), time.time()))
|
||||
con.commit()
|
||||
row_id = int(cur.lastrowid)
|
||||
con.close()
|
||||
return row_id
|
||||
|
||||
|
||||
def pending(limit: int = 20) -> list[dict]:
|
||||
con = lexicon.connect()
|
||||
rows = con.execute(
|
||||
"SELECT * FROM review_question WHERE status='pending' "
|
||||
"ORDER BY created_at LIMIT ?", (limit,)).fetchall()
|
||||
con.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def enqueue_draft(recording: str, draft_path: Path) -> int:
|
||||
"""Queue the draft's explicit uncertainties plus one overall review item."""
|
||||
path = draft_path.resolve()
|
||||
try:
|
||||
path.relative_to(CFG.drafts.resolve())
|
||||
except ValueError:
|
||||
raise ValueError(f"draft outside review root: {path}")
|
||||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||||
fm_match = re.match(r"(?s)^---\n(.*?)\n---", text)
|
||||
fm = fm_match.group(1) if fm_match else ""
|
||||
bg_match = re.search(
|
||||
r"(?ms)^## 通话背景（待确认）\s*$\n(.*?)(?=^## |\Z)", text)
|
||||
if bg_match:
|
||||
background = re.sub(r"\s+", " ", bg_match.group(1)).strip()
|
||||
else:
|
||||
body = text[fm_match.end():] if fm_match else text
|
||||
body = re.sub(r"(?m)^# .*$", "", body).strip()
|
||||
background = re.sub(r"\s+", " ", body[:700]).strip()
|
||||
count = 0
|
||||
enqueue(recording,
|
||||
"系统判断的通话背景是否基本准确？若偏差较大，请在下方写明正确背景以重跑草稿。",
|
||||
{
|
||||
"kind": "background_check", "draft_path": str(path),
|
||||
"question_index": 0, "sentence": background,
|
||||
"candidates": ["背景基本准确"],
|
||||
})
|
||||
count += 1
|
||||
|
||||
named = {}
|
||||
in_speakers = False
|
||||
for line in fm.splitlines():
|
||||
if line.strip() == "voiceprint_speakers:":
|
||||
in_speakers = True
|
||||
continue
|
||||
if in_speakers:
|
||||
sm = re.match(r"^\s+(speaker_\d+):\s*(.+?)\s*$", line)
|
||||
if sm:
|
||||
named[sm.group(1)] = sm.group(2).strip()
|
||||
continue
|
||||
in_speakers = False
|
||||
transcript_path = ""
|
||||
tm = re.search(r"(?m)^transcript:\s*(.+?)\s*$", fm)
|
||||
if tm:
|
||||
transcript_path = tm.group(1).strip().strip("\"'")
|
||||
transcript = ""
|
||||
try:
|
||||
transcript = Path(transcript_path).read_text(encoding="utf-8", errors="ignore")
|
||||
except OSError:
|
||||
pass
|
||||
all_speakers = sorted(set(re.findall(r"(?m)^【(speaker_\d+)】", transcript)))
|
||||
missing_speakers = [s for s in all_speakers if s not in named]
|
||||
for speaker in missing_speakers:
|
||||
excerpts = re.findall(
|
||||
rf"(?m)^【{re.escape(speaker)}】(.+)$", transcript)
|
||||
spoken_chars = sum(len(re.sub(r"\s+", "", x)) for x in excerpts)
|
||||
if spoken_chars < 80 and max((len(x) for x in excerpts), default=0) < 50:
|
||||
continue
|
||||
excerpt = "\n\n".join(x.strip() for x in excerpts[:3])[:1200]
|
||||
enqueue(recording, f"{speaker} 尚未注册声纹；你认识这位说话人吗？", {
|
||||
"kind": "voiceprint", "speaker_id": speaker,
|
||||
"draft_path": str(path), "question_index": count,
|
||||
"sentence": excerpt or "没有足够发言摘录，请无法确认时直接跳过。",
|
||||
"candidates": [],
|
||||
})
|
||||
count += 1
|
||||
|
||||
m = re.search(r"(?ms)^## 待你确认\s*$\n(.*?)(?=^## |\Z)", text)
|
||||
questions = []
|
||||
if m:
|
||||
questions = [
|
||||
re.sub(r"\s+", " ", x).strip()
|
||||
for x in re.findall(r"(?m)^\d+\.\s+(.+)$", m.group(1))
|
||||
]
|
||||
for i, question in enumerate(questions, 1):
|
||||
if any(speaker in question and "身份" in question
|
||||
for speaker in missing_speakers):
|
||||
continue
|
||||
candidates = list(dict.fromkeys(re.findall(r"\[\[([^\]|#]+)", question)))
|
||||
enqueue(recording, question, {
|
||||
"kind": "draft_question", "draft_path": str(path),
|
||||
"question_index": count, "sentence": question,
|
||||
"candidates": candidates[:4],
|
||||
})
|
||||
count += 1
|
||||
|
||||
cards = {
|
||||
"人名": {p.stem for p in (CFG.vault / "People").glob("*.md")
|
||||
if not p.name.startswith("_")},
|
||||
"公司": {p.stem for p in (CFG.vault / "Companies").glob("*.md")
|
||||
if not p.name.startswith("_")},
|
||||
}
|
||||
con = lexicon.connect()
|
||||
for kind, key in (("人名", "people"), ("公司", "companies")):
|
||||
names = re.findall(r"[^,\[\]\s，、]+",
|
||||
_frontmatter_value(text, key))
|
||||
for name in names:
|
||||
if name in cards[kind] or not name or name.startswith("speaker_"):
|
||||
continue
|
||||
history = con.execute(
|
||||
"SELECT count(DISTINCT source) FROM entity_mention "
|
||||
"WHERE entity=? AND kind=?", (name, kind)).fetchone()[0]
|
||||
if history + 1 < CARD_THRESHOLD:
|
||||
continue
|
||||
label = "人物卡" if kind == "人名" else "公司卡"
|
||||
enqueue(recording,
|
||||
f"“{name}”计入本次后已在 {history + 1} 场正式记录中出现，"
|
||||
f"是否建立{label}？", {
|
||||
"kind": "entity_card", "entity": name,
|
||||
"entity_kind": kind, "draft_path": str(path),
|
||||
"question_index": count,
|
||||
"sentence": "只建立保守存根；职位、重要性、合作阶段等判断字段不会自动猜。",
|
||||
"candidates": [f"建立{label}", "暂不建立"],
|
||||
})
|
||||
count += 1
|
||||
con.close()
|
||||
enqueue(recording, "这份草稿整体是否可以作为后续定稿的基础？", {
|
||||
"kind": "draft_overall", "draft_path": str(path),
|
||||
"question_index": count,
|
||||
"sentence": "可展开查看完整草稿；选择“还需修改”时，请在输入框说明修改方向。",
|
||||
"candidates": ["草稿整体可用", "还需修改"],
|
||||
})
|
||||
return count + 1
|
||||
|
||||
|
||||
def _frontmatter_value(text: str, key: str) -> str:
|
||||
m = re.match(r"(?s)^---\n(.*?)\n---", text)
|
||||
if not m:
|
||||
return ""
|
||||
km = re.search(rf"(?m)^{re.escape(key)}:\s*(.+?)\s*$", m.group(1))
|
||||
return km.group(1).strip().strip("\"'") if km else ""
|
||||
|
||||
|
||||
def _voiceprint_name(value: str) -> str:
|
||||
if value in {"无法确认，保留原文", "不知道", "不认识"}:
|
||||
return ""
|
||||
name = re.split(r"[，,；;（(\s]", value.strip(), maxsplit=1)[0]
|
||||
return name if 2 <= len(name) <= 12 and not name.startswith("speaker_") else ""
|
||||
|
||||
|
||||
def _confirm_voiceprint(draft_path: str, speaker_id: str, value: str) -> bool:
|
||||
"""Persist an explicit speaker identity and immediately enroll its session."""
|
||||
name = _voiceprint_name(value)
|
||||
if not name:
|
||||
return False
|
||||
path = Path(draft_path).resolve()
|
||||
try:
|
||||
path.relative_to(CFG.drafts.resolve())
|
||||
except ValueError:
|
||||
return False
|
||||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||||
session_id = _frontmatter_value(text, "voiceprint_session")
|
||||
if not session_id:
|
||||
return False
|
||||
fm = re.match(r"(?s)^---\n(.*?)\n---", text)
|
||||
if not fm:
|
||||
return False
|
||||
block = fm.group(1)
|
||||
speaker_line = rf"(?m)^(\s+{re.escape(speaker_id)}:)\s*.*$"
|
||||
if re.search(speaker_line, block):
|
||||
block = re.sub(speaker_line, rf"\1 {name}", block)
|
||||
elif re.search(r"(?m)^voiceprint_speakers:\s*$", block):
|
||||
block = re.sub(
|
||||
r"(?m)^(voiceprint_speakers:\s*)$",
|
||||
rf"\1\n {speaker_id}: {name}", block, count=1)
|
||||
else:
|
||||
block += f"\nvoiceprint_speakers:\n {speaker_id}: {name}"
|
||||
if re.search(r"(?m)^voiceprint_enroll:", block):
|
||||
block = re.sub(r"(?m)^voiceprint_enroll:.*$",
|
||||
"voiceprint_enroll: true", block)
|
||||
else:
|
||||
block += "\nvoiceprint_enroll: true"
|
||||
text = text[:fm.start()] + "---\n" + block + "\n---" + text[fm.end():]
|
||||
from . import vault
|
||||
vault.write_draft_file(path, text)
|
||||
|
||||
body = json.dumps({
|
||||
"session_id": session_id, "speaker_id": int(speaker_id.split("_")[1]),
|
||||
"name": name,
|
||||
}).encode()
|
||||
req = urllib.request.Request(
|
||||
"http://192.168.50.100:10099/voiceprints/enroll", data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=20) as response:
|
||||
result = json.load(response)
|
||||
return result.get("status") == "enrolled"
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _persist_draft_feedback(recording: str, draft_path: str) -> bool:
|
||||
"""Once a draft's web questions are complete, append an auditable answer block."""
|
||||
path = Path(draft_path).resolve()
|
||||
try:
|
||||
path.relative_to(CFG.drafts.resolve())
|
||||
except ValueError:
|
||||
return False
|
||||
con = lexicon.connect()
|
||||
rows = con.execute(
|
||||
"SELECT * FROM review_question WHERE recording=? ORDER BY created_at,id",
|
||||
(recording,)).fetchall()
|
||||
con.close()
|
||||
selected = []
|
||||
for row in rows:
|
||||
payload = json.loads(row["payload"] or "{}")
|
||||
if payload.get("draft_path") == str(path):
|
||||
if row["status"] == "pending":
|
||||
return False
|
||||
selected.append((row["question"], row["answer"] or ""))
|
||||
if not selected or not path.exists():
|
||||
return False
|
||||
lines = [FEEDBACK_START, "## Web 人工确认记录", ""]
|
||||
for i, (question, answer_value) in enumerate(selected, 1):
|
||||
lines.extend([f"{i}. **问题**：{question}", f" - **回答**：{answer_value}"])
|
||||
lines.extend(["", FEEDBACK_END])
|
||||
block = "\n".join(lines)
|
||||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||||
pattern = re.compile(
|
||||
re.escape(FEEDBACK_START) + r".*?" + re.escape(FEEDBACK_END), re.S)
|
||||
text = pattern.sub(block, text) if pattern.search(text) else text.rstrip() + "\n\n" + block + "\n"
|
||||
from . import vault
|
||||
vault.write_draft_file(path, text)
|
||||
return True
|
||||
|
||||
|
||||
def _request_revision(draft_path: str) -> bool:
|
||||
queue_dir = os.environ.get("ASR_REVISION_QUEUE_DIR", "")
|
||||
if queue_dir:
|
||||
try:
|
||||
path = Path(draft_path).resolve()
|
||||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||||
job = _frontmatter_value(text, "intake_job")
|
||||
queue = Path(queue_dir)
|
||||
queue.mkdir(parents=True, exist_ok=True)
|
||||
target = queue / f"{path.stem}.json"
|
||||
tmp = queue / f".{path.stem}.{os.getpid()}.tmp"
|
||||
tmp.write_text(json.dumps({"draft": str(path), "job_id": job},
|
||||
ensure_ascii=False), encoding="utf-8")
|
||||
tmp.replace(target)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
name = Path(draft_path).name
|
||||
try:
|
||||
escaped = subprocess.run(
|
||||
["systemd-escape", name], capture_output=True, text=True,
|
||||
timeout=5, check=True).stdout.strip()
|
||||
result = subprocess.run(
|
||||
["systemctl", "--user", "start", "--no-block",
|
||||
f"asr-review-revise@{escaped}.service"],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
return result.returncode == 0
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def answer(question_id: int, value: str) -> dict:
|
||||
con = lexicon.connect()
|
||||
row = con.execute("SELECT * FROM review_question WHERE id=?",
|
||||
(question_id,)).fetchone()
|
||||
if not row:
|
||||
con.close()
|
||||
raise KeyError(f"review question {question_id} not found")
|
||||
if row["status"] != "pending":
|
||||
result = {
|
||||
"id": question_id, "answer": row["answer"], "learned": False,
|
||||
"already_answered": True,
|
||||
}
|
||||
con.close()
|
||||
return result
|
||||
payload = json.loads(row["payload"] or "{}")
|
||||
recording = row["recording"]
|
||||
con.execute(
|
||||
"UPDATE review_question SET status='answered',answer=?,answered_at=? WHERE id=?",
|
||||
(value, time.time(), question_id))
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
# A positive answer to an entity choice becomes reusable deterministic memory.
|
||||
source = payload.get("surface", "")
|
||||
candidates = payload.get("candidates") or []
|
||||
canonical = value if value in candidates else ""
|
||||
if source and canonical:
|
||||
lexicon.learn_alias(source, canonical, origin="review", confirmed=1,
|
||||
note=f"review question {question_id}")
|
||||
lexicon.learn_correction(source, canonical, confirmed=1,
|
||||
note=f"review question {question_id}")
|
||||
lexicon.record_observation(source, canonical, source=row["recording"],
|
||||
context=payload.get("sentence", ""),
|
||||
confidence=1.0, confirmed=1)
|
||||
saved = False
|
||||
voiceprint_enrolled = False
|
||||
if payload.get("kind") == "voiceprint":
|
||||
voiceprint_enrolled = _confirm_voiceprint(
|
||||
payload.get("draft_path", ""), payload.get("speaker_id", ""), value)
|
||||
if payload.get("draft_path"):
|
||||
try:
|
||||
saved = _persist_draft_feedback(recording, payload["draft_path"])
|
||||
except Exception as e:
|
||||
# The answer is already durable in SQLite. A transient git/filesystem
|
||||
# failure must not turn a successful mobile submission into a retry.
|
||||
from .log import log
|
||||
log(f"review answer saved but draft feedback sync failed: {e}",
|
||||
level="WARN")
|
||||
saved = False
|
||||
revision_started = saved and _request_revision(payload["draft_path"])
|
||||
if revision_started:
|
||||
try:
|
||||
text = Path(payload["draft_path"]).read_text(
|
||||
encoding="utf-8", errors="ignore")
|
||||
job = _frontmatter_value(text, "intake_job")
|
||||
if job:
|
||||
from .control_web import update_job
|
||||
if (payload.get("kind") == "publish_approval"
|
||||
and value == "发布正式稿"):
|
||||
update_job(job, "publishing", "发布已确认，正在写入正式纪要")
|
||||
else:
|
||||
update_job(job, "refining", "正在应用回答并生成定稿预览")
|
||||
except Exception:
|
||||
pass
|
||||
return {
|
||||
"id": question_id, "answer": value, "learned": bool(canonical),
|
||||
"draft_feedback_saved": saved, "voiceprint_enrolled": voiceprint_enrolled,
|
||||
"revision_started": revision_started,
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Ephemeral, capability-URL mobile UI for the human review queue."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hmac
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
from . import review
|
||||
from .config import CFG
|
||||
from .log import log
|
||||
|
||||
STATE = CFG.state_dir / "review-web.json"
|
||||
UNIT = "asr-review-web.service"
|
||||
|
||||
|
||||
def _url(token: str) -> str:
|
||||
return f"http://{CFG.review_public_host}:{CFG.review_port}/r/{token}/"
|
||||
|
||||
|
||||
def request_start(job_id: str = "") -> bool:
|
||||
"""Ensure control service is running and send a task-scoped review URL."""
|
||||
try:
|
||||
r = subprocess.run(["systemctl", "--user", "start", UNIT],
|
||||
capture_output=True, text=True, timeout=15)
|
||||
if r.returncode:
|
||||
log(f"review web start failed: {r.stderr.strip()[:240]}", level="WARN")
|
||||
return False
|
||||
from . import control_web
|
||||
job = control_web._job(job_id) if job_id else None
|
||||
if job_id and not job:
|
||||
log(f"review notification missing job: {job_id}", level="WARN")
|
||||
return False
|
||||
url = control_web._review_url(job_id)
|
||||
rows = (control_web._pending_for_job(job, 1000)
|
||||
if job else review.pending(1000))
|
||||
from . import vault
|
||||
n = len(rows)
|
||||
title = str(job.get("title") or job["original_name"]) if job else "ASR"
|
||||
vault.notify("ASR 快速确认",
|
||||
f"{title}：{n} 个问题待确认\n{url}\n"
|
||||
f"录音上传入口：http://{CFG.review_public_host}:{CFG.review_port}/",
|
||||
redact_log=True)
|
||||
return True
|
||||
except Exception as e:
|
||||
log(f"review web start failed: {e}", level="WARN")
|
||||
return False
|
||||
|
||||
|
||||
def read_state() -> dict | None:
|
||||
try:
|
||||
return json.loads(STATE.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _write_state(token: str, started: float) -> None:
|
||||
CFG.state_dir.mkdir(parents=True, exist_ok=True)
|
||||
data = {
|
||||
"pid": os.getpid(), "url": _url(token), "port": CFG.review_port,
|
||||
"started_at": started, "expires_at": started + CFG.review_max_seconds,
|
||||
}
|
||||
tmp = STATE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
os.chmod(tmp, 0o600)
|
||||
tmp.replace(STATE)
|
||||
|
||||
|
||||
def _page(title: str, body: str) -> bytes:
|
||||
return f"""<!doctype html>
|
||||
<html lang="zh-CN"><head>
|
||||
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta name="color-scheme" content="light">
|
||||
<meta http-equiv="Cache-Control" content="no-store">
|
||||
<title>{html.escape(title)}</title>
|
||||
<style>
|
||||
:root{{--ink:#17202a;--muted:#637083;--line:#d9dee5;--paper:#f7f8fa;
|
||||
--accent:#176b5b;--accent2:#c84b31;--white:#fff}}
|
||||
*{{box-sizing:border-box}} body{{margin:0;background:var(--paper);color:var(--ink);
|
||||
font-family:system-ui,-apple-system,"PingFang SC","Microsoft YaHei",sans-serif;
|
||||
font-size:17px;line-height:1.55;letter-spacing:0}}
|
||||
header{{background:var(--white);border-bottom:1px solid var(--line);padding:16px 18px}}
|
||||
header strong{{font-size:18px}} header span{{display:block;color:var(--muted);font-size:14px}}
|
||||
main{{max-width:680px;margin:0 auto;padding:18px 14px 40px}}
|
||||
.question{{background:var(--white);border:1px solid var(--line);border-radius:8px;padding:18px}}
|
||||
h1{{font-size:21px;line-height:1.35;margin:0 0 12px}} .source{{margin:14px 0;
|
||||
padding:12px;border-left:4px solid #d4a72c;background:#fff9e8;color:#39424e}}
|
||||
form{{display:grid;gap:10px}} button{{width:100%;min-height:50px;border:0;border-radius:6px;
|
||||
padding:12px 14px;background:var(--accent);color:white;font:inherit;font-weight:650}}
|
||||
button.secondary{{background:#e9edf1;color:var(--ink);border:1px solid var(--line)}}
|
||||
input{{width:100%;min-height:50px;border:1px solid #9aa5b1;border-radius:6px;
|
||||
padding:10px 12px;font:inherit;background:white}} .custom{{display:flex;gap:8px}}
|
||||
.custom input{{min-width:0;flex:1}} .custom button{{width:auto;min-width:82px}}
|
||||
.meta{{color:var(--muted);font-size:14px;margin-bottom:10px}} .done{{text-align:center;
|
||||
background:var(--white);border:1px solid var(--line);border-radius:8px;padding:36px 18px}}
|
||||
.draft{{margin:14px 0;border:1px solid var(--line);border-radius:6px;background:#fbfcfd}}
|
||||
.draft summary{{padding:12px 14px;font-weight:650;cursor:pointer}}
|
||||
.draft pre{{margin:0;padding:14px;border-top:1px solid var(--line);max-height:58vh;
|
||||
overflow:auto;white-space:pre-wrap;overflow-wrap:anywhere;font:14px/1.55 ui-monospace,
|
||||
SFMono-Regular,Consolas,monospace}}
|
||||
.done h1{{color:var(--accent)}} @media(max-width:420px){{main{{padding:12px 10px 28px}}
|
||||
.question{{padding:15px}} .custom{{display:grid}} .custom button{{width:100%}}}}
|
||||
</style></head><body><header><strong>ASR 快速确认</strong>
|
||||
<span>确认结果会写入个人事实库，并用于后续录音</span></header>
|
||||
<main>{body}</main></body></html>""".encode("utf-8")
|
||||
|
||||
|
||||
class Server(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, addr, handler, token: str):
|
||||
super().__init__(addr, handler)
|
||||
self.token = token
|
||||
self.started_at = time.time()
|
||||
self.last_activity = self.started_at
|
||||
self.opened = False
|
||||
self.finish_at: float | None = None
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "ASRReview/1"
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
# BaseHTTPRequestHandler includes the full URL in its default log line.
|
||||
# The path is a bearer capability, so never persist it.
|
||||
status = args[1] if len(args) > 1 else "?"
|
||||
log(f"review web request: {self.command} status={status}")
|
||||
|
||||
def _headers(self, status=HTTPStatus.OK, content_type="text/html; charset=utf-8"):
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Cache-Control", "no-store, no-cache, must-revalidate")
|
||||
self.send_header("Pragma", "no-cache")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("X-Frame-Options", "DENY")
|
||||
self.send_header("Referrer-Policy", "no-referrer")
|
||||
self.send_header("Content-Security-Policy",
|
||||
"default-src 'none'; style-src 'unsafe-inline'; "
|
||||
"form-action 'self'; base-uri 'none'; frame-ancestors 'none'")
|
||||
|
||||
def _valid_path(self) -> bool:
|
||||
parts = urllib.parse.urlsplit(self.path)
|
||||
want = f"/r/{self.server.token}/"
|
||||
return hmac.compare_digest(parts.path, want)
|
||||
|
||||
def _send(self, data: bytes, status=HTTPStatus.OK):
|
||||
self._headers(status)
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/health":
|
||||
body = json.dumps({"ok": True, "pending": len(review.pending(100))}).encode()
|
||||
self._headers(content_type="application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
return
|
||||
if not self._valid_path():
|
||||
self._send(b"Not found", HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
self.server.last_activity = time.time()
|
||||
self.server.opened = True
|
||||
rows = review.pending(100)
|
||||
if not rows:
|
||||
if self.server.finish_at is None:
|
||||
self.server.finish_at = time.time() + 10
|
||||
self._send(_page("完成",
|
||||
'<section class="done"><h1>本轮确认完成</h1>'
|
||||
'<p>结果已经写入个人事实库，本页面即将自动关闭。</p></section>'))
|
||||
return
|
||||
row = rows[0]
|
||||
payload = json.loads(row["payload"] or "{}")
|
||||
choices = payload.get("candidates") or []
|
||||
buttons = []
|
||||
for value in choices:
|
||||
v = html.escape(str(value), quote=True)
|
||||
buttons.append(
|
||||
f'<button type="submit" name="answer" value="{v}">{v}</button>')
|
||||
buttons.append(
|
||||
'<button class="secondary" type="submit" name="answer" '
|
||||
'value="无法确认，保留原文">都不是 / 无法确认</button>')
|
||||
source = html.escape(str(payload.get("sentence", "")))
|
||||
source_html = f'<div class="source">{source}</div>' if source else ""
|
||||
draft_html = ""
|
||||
draft_path = payload.get("draft_path")
|
||||
if draft_path:
|
||||
try:
|
||||
path = Path(draft_path).resolve()
|
||||
path.relative_to(CFG.drafts.resolve())
|
||||
draft = path.read_text(encoding="utf-8", errors="ignore")
|
||||
draft_html = (
|
||||
'<details class="draft"><summary>查看完整草稿</summary><pre>'
|
||||
+ html.escape(draft) + "</pre></details>")
|
||||
except (OSError, ValueError):
|
||||
draft_html = ""
|
||||
placeholder = ("只输入姓名，例如：高瑶"
|
||||
if payload.get("kind") == "voiceprint"
|
||||
else ("背景不准时，在这里写明正确背景并提交重跑"
|
||||
if payload.get("kind") == "background_check"
|
||||
else "输入正确名称或补充说明"))
|
||||
body = f"""<section class="question">
|
||||
<div class="meta">剩余 {len(rows)} 个 · 问题 #{row['id']}</div>
|
||||
<h1>{html.escape(row['question'])}</h1>{source_html}{draft_html}
|
||||
<form method="post" action="/r/{self.server.token}/">
|
||||
<input type="hidden" name="id" value="{row['id']}">
|
||||
{''.join(buttons)}
|
||||
<div class="custom"><input name="custom" maxlength="120" autocomplete="off"
|
||||
placeholder="{html.escape(placeholder, quote=True)}"><button type="submit">提交</button></div>
|
||||
</form></section>"""
|
||||
self._send(_page("ASR 快速确认", body))
|
||||
|
||||
def do_POST(self):
|
||||
if not self._valid_path():
|
||||
self._send(b"Not found", HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
try:
|
||||
length = min(int(self.headers.get("Content-Length", "0")), 4096)
|
||||
form = urllib.parse.parse_qs(self.rfile.read(length).decode("utf-8"))
|
||||
qid = int(form.get("id", ["0"])[0])
|
||||
custom = form.get("custom", [""])[0].strip()
|
||||
answer = custom or form.get("answer", [""])[0].strip()
|
||||
if not qid or not answer or len(answer) > 120:
|
||||
raise ValueError("invalid answer")
|
||||
pending_ids = {r["id"] for r in review.pending(1000)}
|
||||
if qid not in pending_ids:
|
||||
raise ValueError("问题已处理或不存在")
|
||||
review.answer(qid, answer)
|
||||
self.server.last_activity = time.time()
|
||||
self.send_response(HTTPStatus.SEE_OTHER)
|
||||
self.send_header("Location", f"/r/{self.server.token}/")
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
except Exception as e:
|
||||
self._send(_page("提交失败",
|
||||
f'<section class="done"><h1>提交失败</h1><p>{html.escape(str(e))}</p></section>'),
|
||||
HTTPStatus.BAD_REQUEST)
|
||||
|
||||
|
||||
def serve() -> int:
|
||||
rows = review.pending(100)
|
||||
if not rows:
|
||||
log("review web: no pending questions")
|
||||
return 0
|
||||
token = secrets.token_urlsafe(32)
|
||||
started = time.time()
|
||||
try:
|
||||
server = Server((CFG.review_host, CFG.review_port), Handler, token)
|
||||
except OSError as e:
|
||||
STATE.unlink(missing_ok=True)
|
||||
log(f"review web bind failed on {CFG.review_host}:{CFG.review_port}: {e}",
|
||||
level="ERROR")
|
||||
return 1
|
||||
_write_state(token, started)
|
||||
server.timeout = 1
|
||||
url = _url(token)
|
||||
log(f"review web listening on {CFG.review_public_host}:{CFG.review_port} "
|
||||
f"({len(rows)} question(s))")
|
||||
try:
|
||||
from . import vault
|
||||
vault.notify("ASR 快速确认",
|
||||
f"{len(rows)} 个问题待确认：{url}\n"
|
||||
f"链接最多保留 {CFG.review_max_seconds // 60} 分钟，完成后自动销毁。",
|
||||
redact_log=True)
|
||||
except Exception as e:
|
||||
log(f"review web notification failed: {e}", level="WARN")
|
||||
try:
|
||||
while True:
|
||||
server.handle_request()
|
||||
now = time.time()
|
||||
if server.finish_at and now >= server.finish_at:
|
||||
# A new recording can enqueue work during the 10-second grace
|
||||
# period while systemd still considers this unit active.
|
||||
if review.pending(1):
|
||||
server.finish_at = None
|
||||
log("review web received new work during shutdown grace")
|
||||
else:
|
||||
break
|
||||
if now - started >= CFG.review_max_seconds:
|
||||
log("review web reached maximum lifetime")
|
||||
break
|
||||
if server.opened and now - server.last_activity >= CFG.review_idle_seconds:
|
||||
log("review web idle timeout")
|
||||
break
|
||||
finally:
|
||||
server.server_close()
|
||||
try:
|
||||
STATE.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(serve())
|
||||
@@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare a final preview, then publish it only after explicit Web approval."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import graph, lexicon, vault
|
||||
from .config import CFG
|
||||
from .llm import chat
|
||||
from .log import log
|
||||
from .review import FEEDBACK_END, FEEDBACK_START
|
||||
|
||||
PROMPT = """请根据文末「Web 人工确认记录」将粗稿整理成可直接进入 Obsidian 的正式记录。
|
||||
|
||||
严格要求：
|
||||
1. 只使用原稿、人工回答及原稿已标明的背景，不补充外部事实。
|
||||
2. 将确认的人名、关系、背景和术语真正改入正文；人工回答有“应该”等保留不确定性。
|
||||
3. 删除已解决的「待你确认」以及 Web 审核记录，不写 ASR 修正、speaker 归属等过程说明。
|
||||
4. YAML 必须符合：
|
||||
- type: minutes
|
||||
- tags: [会议纪要]
|
||||
- 不得有 status: 草稿待确认
|
||||
- 保留 date/source/source_detail/transcript/voiceprint_session/
|
||||
voiceprint_speakers/voiceprint_enroll/intake_job
|
||||
- people/companies 使用裸名列表。
|
||||
5. H1 使用简短、有辨识度的业务标题，不要写“粗整理草稿”或“会议纪要”。
|
||||
6. H1 后先写：
|
||||
> [!summary] Summary
|
||||
> 结论式概述。
|
||||
然后按需写 `[!important] 决策`、Tasks 格式待办、`[!question] Open Questions`。
|
||||
7. 正文按业务主线组织，中文全角标点；有把握的实体用 [[双链]]。
|
||||
8. 不完整或仍存疑的内容可留 Open Questions，绝不硬猜。
|
||||
9. 输出完整 Markdown 文件，不要解释。
|
||||
|
||||
待发布粗稿：
|
||||
{draft}
|
||||
"""
|
||||
|
||||
|
||||
def _field(text: str, key: str) -> str:
|
||||
m = re.search(rf"(?m)^{re.escape(key)}:\s*(.+?)\s*$", text)
|
||||
return m.group(1).strip().strip("\"'") if m else ""
|
||||
|
||||
|
||||
def _safe_title(markdown: str) -> str:
|
||||
m = re.search(r"(?m)^#\s+(.+?)\s*$", markdown)
|
||||
title = m.group(1).strip() if m else "通话记录"
|
||||
title = re.sub(r"[\[\]<>:\"/\\|?*#]", "", title)
|
||||
title = re.sub(r"\s+", "", title).strip(". ")
|
||||
return title[:72] or "通话记录"
|
||||
|
||||
|
||||
def _formal_path(markdown: str) -> Path:
|
||||
date = _field(markdown, "date") or time.strftime("%Y-%m-%d")
|
||||
base = CFG.minutes / f"{date}-{_safe_title(markdown)}.md"
|
||||
transcript = _field(markdown, "transcript")
|
||||
if CFG.minutes.exists() and transcript:
|
||||
for path in CFG.minutes.glob("*.md"):
|
||||
try:
|
||||
if _field(path.read_text(errors="ignore"), "transcript") == transcript:
|
||||
return path
|
||||
except OSError:
|
||||
continue
|
||||
if not base.exists():
|
||||
return base
|
||||
n = 2
|
||||
while base.with_name(f"{base.stem}-{n}.md").exists():
|
||||
n += 1
|
||||
return base.with_name(f"{base.stem}-{n}.md")
|
||||
|
||||
|
||||
def _answered_card_requests(draft_path: Path) -> list[tuple[str, str]]:
|
||||
con = lexicon.connect()
|
||||
rows = con.execute(
|
||||
"SELECT payload,answer,status FROM review_question ORDER BY id").fetchall()
|
||||
con.close()
|
||||
out = []
|
||||
for row in rows:
|
||||
payload = json.loads(row["payload"] or "{}")
|
||||
if (payload.get("draft_path") != str(draft_path)
|
||||
or payload.get("kind") != "entity_card"
|
||||
or row["status"] != "answered"
|
||||
or not str(row["answer"]).startswith("建立")):
|
||||
continue
|
||||
out.append((payload["entity"], payload["entity_kind"]))
|
||||
return out
|
||||
|
||||
|
||||
def _create_card(name: str, kind: str, minute: Path, date: str) -> Path | None:
|
||||
if not name or "/" in name or name.startswith("_"):
|
||||
return None
|
||||
if kind == "人名":
|
||||
path = CFG.vault / "People" / f"{name}.md"
|
||||
if path.exists():
|
||||
return None
|
||||
body = f"""---
|
||||
type: person
|
||||
name: "{name}"
|
||||
aliases: []
|
||||
company:
|
||||
role:
|
||||
status: active
|
||||
importance:
|
||||
owner: Cooper
|
||||
tags: []
|
||||
last_contact: '{date}'
|
||||
mentions: 1
|
||||
first_contact: '{date}'
|
||||
next_followup:
|
||||
location:
|
||||
channels: []
|
||||
related_projects: []
|
||||
related_companies: []
|
||||
source: 会议提及
|
||||
created: {time.strftime("%Y-%m-%d")}
|
||||
---
|
||||
|
||||
# {name}
|
||||
|
||||
> [!summary] Summary
|
||||
> 在 [[{minute.stem}]] 中被确认；详细身份与关系待后续补充。
|
||||
|
||||
## 相关会议
|
||||
|
||||
- [[{minute.stem}]]
|
||||
"""
|
||||
else:
|
||||
path = CFG.vault / "Companies" / f"{name}.md"
|
||||
if path.exists():
|
||||
return None
|
||||
body = f"""---
|
||||
type: company
|
||||
name: "{name}"
|
||||
aliases: []
|
||||
industry:
|
||||
stage:
|
||||
relationship:
|
||||
key_people: []
|
||||
related_projects: []
|
||||
related_companies: []
|
||||
tags: []
|
||||
source: 会议提及
|
||||
created: {time.strftime("%Y-%m-%d")}
|
||||
last_contact: '{date}'
|
||||
mentions: 1
|
||||
first_contact: '{date}'
|
||||
---
|
||||
|
||||
# {name}
|
||||
|
||||
> [!summary] Summary
|
||||
> 在 [[{minute.stem}]] 中被确认；公司信息与关系阶段待后续补充。
|
||||
|
||||
## 相关会议
|
||||
|
||||
- [[{minute.stem}]]
|
||||
"""
|
||||
vault.write_draft_file(path, body)
|
||||
return path
|
||||
|
||||
|
||||
def _latest_publish_answer(path: Path) -> str:
|
||||
con = lexicon.connect()
|
||||
rows = con.execute(
|
||||
"SELECT payload,answer,status FROM review_question ORDER BY id DESC"
|
||||
).fetchall()
|
||||
con.close()
|
||||
for row in rows:
|
||||
payload = json.loads(row["payload"] or "{}")
|
||||
if (payload.get("draft_path") == str(path)
|
||||
and payload.get("kind") == "publish_approval"
|
||||
and row["status"] == "answered"):
|
||||
return str(row["answer"] or "")
|
||||
return ""
|
||||
|
||||
|
||||
def _validate_formal(out: str, original: str) -> None:
|
||||
if not out.startswith("---\n") or "type: minutes" not in out:
|
||||
raise RuntimeError("formal output failed frontmatter validation")
|
||||
transcript = _field(original, "transcript")
|
||||
if transcript and _field(out, "transcript") != transcript:
|
||||
raise RuntimeError("formal output changed transcript provenance")
|
||||
if "[!summary]" not in out:
|
||||
raise RuntimeError("formal output missing Summary callout")
|
||||
if not (0.35 <= len(out) / max(1, len(original)) <= 1.35):
|
||||
raise RuntimeError("formal output length outside safety range")
|
||||
|
||||
|
||||
def _as_preview(out: str) -> str:
|
||||
fm = re.match(r"(?s)^---\n(.*?)\n---", out)
|
||||
if not fm:
|
||||
raise RuntimeError("formal preview missing frontmatter")
|
||||
block = re.sub(r"(?m)^status:.*\n?", "", fm.group(1))
|
||||
block += "\nstatus: 待发布确认"
|
||||
return "---\n" + block + "\n---" + out[fm.end():]
|
||||
|
||||
|
||||
def prepare(path: Path) -> bool:
|
||||
path = path.resolve()
|
||||
path.relative_to(CFG.drafts.resolve())
|
||||
original = path.read_text(encoding="utf-8", errors="ignore")
|
||||
intake_job = _field(original, "intake_job")
|
||||
if FEEDBACK_START not in original or FEEDBACK_END not in original:
|
||||
log(f"publish skipped without completed feedback: {path}", level="WARN")
|
||||
return False
|
||||
out = chat(CFG.draft_model, PROMPT.format(draft=original), max_tokens=14000)
|
||||
out = re.sub(r"^```(?:markdown)?\s*|\s*```$", "", out.strip(),
|
||||
flags=re.I | re.S).strip() + "\n"
|
||||
if "status: 草稿待确认" in out or FEEDBACK_START in out:
|
||||
raise RuntimeError("formal output retained draft-only metadata")
|
||||
_validate_formal(out, original)
|
||||
vault.write_draft_file(path, _as_preview(out))
|
||||
from . import review, review_web
|
||||
review.enqueue(
|
||||
_field(original, "transcript") or path.name,
|
||||
"这是应用第一轮回答后的定稿预览。确认发布到 Obsidian 吗？",
|
||||
{
|
||||
"kind": "publish_approval", "draft_path": str(path),
|
||||
"sentence": "请展开查看完整定稿；选择退回时，在输入框写明修改意见。",
|
||||
"candidates": ["发布正式稿", "退回修改"],
|
||||
})
|
||||
if intake_job:
|
||||
try:
|
||||
from .control_web import update_job
|
||||
update_job(intake_job, "review", "定稿预览已生成，等待发布确认",
|
||||
draft=str(path))
|
||||
except Exception as e:
|
||||
log(f"preview intake status failed: {e}", level="WARN")
|
||||
review_web.request_start(intake_job)
|
||||
log(f"prepared final preview {path}")
|
||||
return True
|
||||
|
||||
|
||||
def _finalize_preview(text: str) -> str:
|
||||
text = re.sub(
|
||||
re.escape(FEEDBACK_START) + r".*?" + re.escape(FEEDBACK_END),
|
||||
"", text, flags=re.S)
|
||||
text = re.sub(r"(?m)^status:\s*待发布确认\s*\n?", "", text)
|
||||
return text.rstrip() + "\n"
|
||||
|
||||
|
||||
def publish(path: Path) -> bool:
|
||||
path = path.resolve()
|
||||
path.relative_to(CFG.drafts.resolve())
|
||||
original = path.read_text(encoding="utf-8", errors="ignore")
|
||||
if _latest_publish_answer(path) != "发布正式稿":
|
||||
log(f"publish skipped without explicit approval: {path}", level="WARN")
|
||||
return False
|
||||
out = _finalize_preview(original)
|
||||
_validate_formal(out, original)
|
||||
intake_job = _field(original, "intake_job")
|
||||
|
||||
destination = _formal_path(out)
|
||||
vault.write_draft_file(destination, out)
|
||||
date = _field(out, "date") or time.strftime("%Y-%m-%d")
|
||||
touched = [destination]
|
||||
for name, kind in _answered_card_requests(path):
|
||||
card = _create_card(name, kind, destination, date)
|
||||
if card:
|
||||
touched.append(card)
|
||||
state = vault.sync(touched, f"minutes: {destination.stem}（Web 确认后正式发布）")
|
||||
if state == "error":
|
||||
raise RuntimeError("formal minute commit failed")
|
||||
|
||||
graph.sync()
|
||||
archive = CFG.state_dir / "published"
|
||||
archive.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(path), archive / path.name)
|
||||
vault.notify("ASR 正式记录已发布",
|
||||
f"{destination.name} 已应用 Web 回答并发布（同步：{state}）。")
|
||||
log(f"published {destination} | cards={len(touched)-1} | git={state}")
|
||||
if intake_job:
|
||||
try:
|
||||
from .control_web import update_job
|
||||
update_job(intake_job, "complete", f"正式纪要：{destination.name}",
|
||||
result=str(destination))
|
||||
except Exception as e:
|
||||
log(f"intake completion status failed: {e}", level="WARN")
|
||||
return True
|
||||
|
||||
|
||||
def revise(path: Path) -> bool:
|
||||
path = path.resolve()
|
||||
if _latest_publish_answer(path) == "发布正式稿":
|
||||
return publish(path)
|
||||
return prepare(path)
|
||||
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python3
|
||||
"""S0 — deterministic pre-processing and chunking. No LLM, no cost.
|
||||
|
||||
Handles the three transcript shapes the pipeline actually produces:
|
||||
qwen3cam `【谭凯】…` / `【speaker_3】…` turns, blank-line separated, plus a
|
||||
`<!-- voiceprint_session: … -->` header line.
|
||||
qwen3-plain `--- 第N段 [00:00 - 09:00] ---` segment banners then a wall of text
|
||||
with no speaker information at all.
|
||||
vibevoice `[mm:ss] S0: …` lines, with `[复读截断]` / `·` noise artifacts.
|
||||
|
||||
Chunking rule: never split inside a speaker turn if avoidable, because the
|
||||
cleaning model needs the whole turn to tell a filler word from a real answer.
|
||||
Turns longer than the budget are split on sentence boundaries. Each chunk carries
|
||||
`overlap_prefix` — the tail of the previous chunk, given to the model as
|
||||
read-only context so it can resolve a pronoun or a name that started earlier
|
||||
without being asked to re-emit it.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
VOICEPRINT_HDR = re.compile(r"^<!--\s*voiceprint_session:")
|
||||
SEGMENT_BANNER = re.compile(r"^---\s*第\d+段.*---\s*$")
|
||||
TURN_TAG = re.compile(r"^【([^】]+)】\s*(.*)$", re.S)
|
||||
VIBE_TAG = re.compile(r"^\[(\d{1,2}:\d{2})\]\s*(S\d+)\s*[:：]\s*(.*)$", re.S)
|
||||
# Must include ASCII terminators: the qwen3-plain backend emits English stretches
|
||||
# ("...so that's why they wanted us to set up company.") with no CJK punctuation at
|
||||
# all, and a CJK-only rule left single 15k-char "sentences" that blew the budget.
|
||||
SENT_END = re.compile(r"(?<=[。！？；;?!])(?![”’\"'）)])")
|
||||
# Last-resort split points when a run has no terminator for thousands of chars.
|
||||
SOFT_BREAK = re.compile(r"(?<=[，,、：:])")
|
||||
NOISE = re.compile(r"\[复读截断\]|^[·\s]+$")
|
||||
|
||||
# ASR decoders loop when fed non-speech audio (hold music, a phone beep, dead air).
|
||||
# Measured on the fixture set: one 24k-char call transcript is 77% "beep beep beep…"
|
||||
# (3,698 repetitions) and a 58k-char meeting has a 3.7k-char run of "呃，". Left in
|
||||
# place this garbage consumed 11 of 17 chunks, i.e. ~65% of the LLM spend on that
|
||||
# recording, and provoked the cleaning model into emitting 26k chars of noise per
|
||||
# chunk. Collapsing it is free, deterministic, and strictly beneficial.
|
||||
DEGENERATE = re.compile(
|
||||
r"((?:[A-Za-z]{1,12}|[\u4e00-\u9fff]{1,4})[ ，,、。!?！？]{0,2})\1{7,}")
|
||||
DEGEN_MIN_SPAN = 120 # only collapse runs long enough to be a real artifact
|
||||
DEGEN_KEEP = 2 # keep a couple of instances so the context stays readable
|
||||
|
||||
|
||||
def collapse_degenerate(text: str) -> tuple[str, list[dict]]:
|
||||
"""Replace long ASR repetition loops with a short, explicit marker.
|
||||
|
||||
Legitimate speech repetition (对对对, 好的好的) is short and must survive, so the
|
||||
rule requires both a high repeat count and a span over DEGEN_MIN_SPAN chars.
|
||||
The marker stays in the text rather than deleting silently: a recording with
|
||||
four minutes of hold music is a fact worth surfacing in the draft.
|
||||
"""
|
||||
events: list[dict] = []
|
||||
|
||||
def sub(m):
|
||||
span, unit = m.group(0), m.group(1)
|
||||
if len(span) < DEGEN_MIN_SPAN:
|
||||
return span
|
||||
n = len(span) // max(1, len(unit))
|
||||
events.append({"unit": unit, "repeats": n, "chars": len(span)})
|
||||
return unit * DEGEN_KEEP + f"[…重复噪声已折叠：{unit.strip()}×{n}…]"
|
||||
|
||||
return DEGENERATE.sub(sub, text), events
|
||||
|
||||
|
||||
@dataclass
|
||||
class Turn:
|
||||
speaker: str # "" when the backend gives no speaker
|
||||
text: str
|
||||
idx: int
|
||||
ts: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Chunk:
|
||||
idx: int
|
||||
turns: list[Turn]
|
||||
overlap_prefix: str = ""
|
||||
meta: dict = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def body(self) -> str:
|
||||
out = []
|
||||
for t in self.turns:
|
||||
out.append(f"【{t.speaker}】{t.text}" if t.speaker else t.text)
|
||||
return "\n\n".join(out)
|
||||
|
||||
@property
|
||||
def chars(self) -> int:
|
||||
return len(self.body)
|
||||
|
||||
|
||||
def detect_backend(raw: str) -> str:
|
||||
head = raw[:500]
|
||||
if "CAM++" in head or TURN_TAG.match(raw.strip().splitlines()[-1] if raw.strip() else ""):
|
||||
return "qwen3cam"
|
||||
if re.search(r"^\[\d{1,2}:\d{2}\]\s*S\d+\s*[:：]", raw, re.M):
|
||||
return "vibevoice"
|
||||
if SEGMENT_BANNER.search(raw):
|
||||
return "qwen3-plain"
|
||||
return "qwen3cam" if "【" in raw else "qwen3-plain"
|
||||
|
||||
|
||||
def parse(raw: str) -> tuple[list[Turn], dict]:
|
||||
"""Split a transcript into turns plus header metadata."""
|
||||
meta: dict = {
|
||||
"voiceprint_session": "", "voiceprint_named": {}, "banners": [],
|
||||
"intake": {},
|
||||
}
|
||||
lines = raw.splitlines()
|
||||
body_lines = []
|
||||
for ln in lines:
|
||||
if ln.startswith("<!-- intake:") and ln.endswith("-->"):
|
||||
try:
|
||||
meta["intake"] = json.loads(
|
||||
ln[len("<!-- intake:"):-len("-->")].strip())
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
continue
|
||||
if VOICEPRINT_HDR.match(ln):
|
||||
m = re.search(r"voiceprint_session:\s*(\S+)", ln)
|
||||
if m:
|
||||
meta["voiceprint_session"] = m.group(1)
|
||||
m = re.search(r"clusters:\s*(\d+)", ln)
|
||||
if m:
|
||||
meta["clusters"] = int(m.group(1))
|
||||
continue
|
||||
if ln.startswith("# 转写"):
|
||||
for pair in re.findall(r"(speaker_\d+)=([^,\s]+)", ln):
|
||||
meta["voiceprint_named"][pair[0]] = pair[1]
|
||||
continue
|
||||
if SEGMENT_BANNER.match(ln):
|
||||
meta["banners"].append(ln.strip())
|
||||
continue
|
||||
body_lines.append(ln)
|
||||
|
||||
backend = detect_backend(raw)
|
||||
meta["backend"] = backend
|
||||
turns: list[Turn] = []
|
||||
|
||||
if backend == "vibevoice":
|
||||
for ln in body_lines:
|
||||
m = VIBE_TAG.match(ln.strip())
|
||||
if m:
|
||||
txt = NOISE.sub("", m.group(3)).strip()
|
||||
if txt:
|
||||
turns.append(Turn(m.group(2), txt, len(turns), m.group(1)))
|
||||
return turns, meta
|
||||
|
||||
blocks = [b.strip() for b in re.split(r"\n\s*\n", "\n".join(body_lines)) if b.strip()]
|
||||
for b in blocks:
|
||||
m = TURN_TAG.match(b)
|
||||
if m:
|
||||
txt = NOISE.sub("", m.group(2)).strip()
|
||||
if txt:
|
||||
turns.append(Turn(m.group(1), txt, len(turns)))
|
||||
else:
|
||||
txt = NOISE.sub("", b).strip()
|
||||
if txt:
|
||||
turns.append(Turn("", txt, len(turns)))
|
||||
return turns, meta
|
||||
|
||||
|
||||
def _pack(pieces: list[str], budget: int) -> list[str]:
|
||||
out, cur = [], ""
|
||||
for s in pieces:
|
||||
if len(cur) + len(s) > budget and cur:
|
||||
out.append(cur)
|
||||
cur = s
|
||||
else:
|
||||
cur += s
|
||||
if cur:
|
||||
out.append(cur)
|
||||
return out
|
||||
|
||||
|
||||
def _split_long(text: str, budget: int) -> list[str]:
|
||||
"""Break an over-long single turn, escalating through split granularities.
|
||||
|
||||
A hard character cut is the final fallback. It is ugly, but an oversized chunk
|
||||
is worse: it is exactly the condition that makes the cleaning model start
|
||||
summarizing instead of rewriting, which is the failure mode this whole
|
||||
pipeline exists to prevent.
|
||||
"""
|
||||
out = _pack([s for s in SENT_END.split(text) if s], budget)
|
||||
if all(len(p) <= budget for p in out):
|
||||
return out or [text]
|
||||
stage2: list[str] = []
|
||||
for p in out:
|
||||
if len(p) <= budget:
|
||||
stage2.append(p)
|
||||
else:
|
||||
stage2 += _pack([s for s in SOFT_BREAK.split(p) if s], budget)
|
||||
final: list[str] = []
|
||||
for p in stage2:
|
||||
if len(p) <= budget:
|
||||
final.append(p)
|
||||
else:
|
||||
final += [p[i:i + budget] for i in range(0, len(p), budget)]
|
||||
return final or [text]
|
||||
|
||||
|
||||
def chunk(raw: str, budget: int = 1800, overlap_chars: int = 200,
|
||||
collapse_noise: bool = True) -> tuple[list[Chunk], dict]:
|
||||
events: list[dict] = []
|
||||
if collapse_noise:
|
||||
raw, events = collapse_degenerate(raw)
|
||||
turns, meta = parse(raw)
|
||||
meta["degenerate_events"] = events
|
||||
meta["degenerate_chars"] = sum(e["chars"] for e in events)
|
||||
chunks: list[Chunk] = []
|
||||
cur: list[Turn] = []
|
||||
size = 0
|
||||
|
||||
def flush():
|
||||
nonlocal cur, size
|
||||
if cur:
|
||||
chunks.append(Chunk(len(chunks), cur, meta={}))
|
||||
cur, size = [], 0
|
||||
|
||||
for t in turns:
|
||||
if len(t.text) > budget:
|
||||
flush()
|
||||
for i, piece in enumerate(_split_long(t.text, budget)):
|
||||
chunks.append(Chunk(len(chunks), [Turn(t.speaker, piece, t.idx, t.ts)],
|
||||
meta={"split_turn": True, "part": i}))
|
||||
continue
|
||||
if size + len(t.text) > budget and cur:
|
||||
flush()
|
||||
cur.append(t)
|
||||
size += len(t.text)
|
||||
flush()
|
||||
|
||||
if overlap_chars:
|
||||
for i in range(1, len(chunks)):
|
||||
prev = chunks[i - 1].body
|
||||
chunks[i].overlap_prefix = prev[-overlap_chars:]
|
||||
meta["n_turns"] = len(turns)
|
||||
meta["n_chunks"] = len(chunks)
|
||||
meta["speakers"] = sorted({t.speaker for t in turns if t.speaker})
|
||||
return chunks, meta
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
for p in sys.argv[1:]:
|
||||
raw = Path(p).read_text(errors="ignore")
|
||||
for budget in (900, 1800, 3600):
|
||||
cs, meta = chunk(raw, budget)
|
||||
sizes = [c.chars for c in cs]
|
||||
print(f"{Path(p).name[:40]:<42} budget={budget:<5} chunks={len(cs):<4} "
|
||||
f"turns={meta['n_turns']:<5} spk={len(meta['speakers']):<3} "
|
||||
f"backend={meta['backend']:<12} "
|
||||
f"size min/avg/max={min(sizes)}/{sum(sizes)//len(sizes)}/{max(sizes)}")
|
||||
@@ -0,0 +1,293 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the domain term bank used to seed ASR misrecognition candidates.
|
||||
|
||||
Why not ChEMBL / DrugBank directly (evaluated 2026-07-25):
|
||||
ChEMBL's 4,225 approved molecules and DrugBank's dumps are **English** chemical
|
||||
names (PRAZOSIN, OFLOXACIN). Kai's transcripts are Chinese speech, so a Chinese
|
||||
ASR never emits "OFLOXACIN" — it emits 氧氟沙星. An English chemical dictionary
|
||||
therefore contributes ~nothing to phonetic correction of Chinese audio, while
|
||||
costing a multi-GB download. Skipped deliberately.
|
||||
|
||||
What actually pays off, in descending order of value:
|
||||
1. **INN stem generation** — the WHO INN system gives Chinese biologics highly
|
||||
regular suffixes (单抗/珠单抗/替尼/西普/他滨...). Any Han n-gram ending in one
|
||||
of those stems is almost certainly a drug name, which lets us *recognize* drug
|
||||
mentions we have never seen, not just ones in a list.
|
||||
2. **zhwiki intitle: search** — 237 hits for 单抗, 49 for 替尼, all real INN
|
||||
Chinese generic names (帕博利珠单抗, 奥希替尼...). Small, free, exactly on-domain.
|
||||
3. **Curated CDMO/antibody/CMC jargon** — the vocabulary that actually dominates
|
||||
Kai's meetings (原液/下游/批记录/中试/放行/一次性反应器/GMP 证...). ASR mangles
|
||||
these constantly (批记录→披露/P图) and no public corpus contains them as a set.
|
||||
4. **THUOCL medical/IT/caijing** — general high-frequency words. Weak on their own
|
||||
(医院/检查) but useful as a "this is a real word, do not 'correct' it" allowlist,
|
||||
which is how we suppress the 主要→朱岩 class of false positives.
|
||||
|
||||
Output: termbank/terms.json { term: {sources: [...], kind: str} }
|
||||
termbank/allowlist.txt common words that must never be rewritten
|
||||
|
||||
python3 -m lab.termbank build # download + assemble (network)
|
||||
python3 -m lab.termbank stats
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from .config import CFG
|
||||
|
||||
LAB = CFG.root
|
||||
BANK = CFG.termbank_dir
|
||||
UA = {"User-Agent": "asr-lab/1.0 (personal research; contact: local)"}
|
||||
|
||||
# ---------------------------------------------------------------- INN stems
|
||||
# WHO INN suffixes, Chinese renderings. A Han string ending in one of these is
|
||||
# a drug/biologic with very high probability.
|
||||
INN_SUFFIXES = [
|
||||
"单抗", "珠单抗", "尤单抗", "利珠单抗", "妥珠单抗", "西单抗", "木单抗", "库单抗",
|
||||
"替尼", "布替尼", "沙替尼", "래替尼", "洛替尼", "非替尼",
|
||||
"西普", "普肽", "格鲁肽", "鲁肽", "那肽",
|
||||
"他滨", "他赛", "紫杉醇", "沙星", "霉素", "环素", "西林", "唑胺", "磺胺",
|
||||
"干扰素", "白介素", "促红素", "生长素",
|
||||
"酶", "疫苗", "佐剂",
|
||||
]
|
||||
# Suffix -> what kind of thing it names, for the prompt hint.
|
||||
INN_KIND = {"单抗": "抗体药", "替尼": "小分子抑制剂", "肽": "多肽药", "疫苗": "疫苗"}
|
||||
|
||||
# --------------------------------------------------- curated CDMO / CMC jargon
|
||||
# Hand-built from Kai's own minutes + glossary. These are the terms that actually
|
||||
# break: each line is the CORRECT form; the flagger finds phonetic neighbours of
|
||||
# them in the transcript. Grouped only for readability.
|
||||
CDMO_TERMS = """
|
||||
原液 制剂 下游 上游 中试 放行 灌装 冻干 除菌过滤 层析 亲和层析 离子交换 疏水层析
|
||||
超滤 透析 病毒灭活 病毒去除 纳米过滤 收率 纯度 电荷异构体 糖型 聚体 碎片
|
||||
批记录 批生产记录 批分析记录 工艺验证 清洁验证 培养基模拟灌装 无菌工艺模拟
|
||||
偏差 变更 CAPA 超标 超趋势 纠正预防措施 质量回顾 年度质量回顾
|
||||
稳定性 加速稳定性 长期稳定性 强制降解 影响因素试验
|
||||
细胞库 主细胞库 工作细胞库 细胞株 稳定池 单克隆 克隆筛选 传代 传代稳定性
|
||||
反应器 生物反应器 一次性反应器 摇瓶 摇床 补料 流加 灌流 收获 澄清
|
||||
质谱 肽图 N端测序 效价 生物活性 亲和力 抗体偶联 载药量 游离药物
|
||||
工艺表征 工艺性能确认 关键质量属性 关键工艺参数 设计空间 控制策略
|
||||
技术转移 工艺开发 分析方法开发 方法验证 方法转移 对照品 标准品
|
||||
临床前 毒理 非GLP 药代 药效 首次人体试验 剂量爬坡 剂量探索
|
||||
申报 临床试验申请 新药上市申请 药品注册 药品上市许可持有人 受托生产
|
||||
现场检查 迎检 迎审 缺陷项 主要缺陷 严重缺陷 一般缺陷 整改
|
||||
生产许可证 药品生产许可证 B证 C证 D证 委托生产 受托方 委托方
|
||||
洁净区 洁净级别 压差 环境监测 沉降菌 浮游菌 表面微生物 尘埃粒子
|
||||
计算机化系统 数据完整性 审计追踪 电子签名 权限管理 系统验证
|
||||
双抗 双特异性抗体 三特异性抗体 纳米抗体 单域抗体 融合蛋白 重组蛋白
|
||||
抗体偶联药物 细胞治疗 基因治疗 慢病毒 腺病毒 质粒 转染 瞬转 稳转
|
||||
产能 产能利用率 排产 排期 交付 交付周期 项目里程碑 阶段付款
|
||||
报价 服务包 里程碑款 技术服务费 居间费 保密协议 主服务协议 工作说明书
|
||||
"""
|
||||
|
||||
# Company / platform names common in Kai's domain that generic corpora lack.
|
||||
DOMAIN_ORGS = """
|
||||
药明生物 药明康德 药明合联 康龙化成 泰格医药 昭衍新药 金斯瑞 义翘神州
|
||||
百奥赛图 和铂医药 信达生物 君实生物 恒瑞医药 复宏汉霖 三生国健 神州细胞
|
||||
迈威生物 洛启生物 麦济生物 维眸生物 白帆生物 鼎康生物 恒元生物 愚公生物
|
||||
奥浦迈 澳斯康 东富龙 楚天科技 森松 赛多利斯 思拓凡 默克 丹纳赫 颇尔
|
||||
药审中心 国家药监局 药品审评中心 核查中心 中检院
|
||||
"""
|
||||
|
||||
THUOCL = {
|
||||
"medical": "https://raw.githubusercontent.com/thunlp/THUOCL/master/data/THUOCL_medical.txt",
|
||||
"it": "https://raw.githubusercontent.com/thunlp/THUOCL/master/data/THUOCL_IT.txt",
|
||||
"caijing": "https://raw.githubusercontent.com/thunlp/THUOCL/master/data/THUOCL_caijing.txt",
|
||||
}
|
||||
|
||||
# The allowlist's job is to stop entity candidates from displacing ordinary words
|
||||
# (主要→朱岩, 分钟→冯昆仑, 具体→聚体). THUOCL's domain lists cannot do this: they
|
||||
# contain 医院/检查 but not 具体/自己/硬件. jieba's dictionary is a general-purpose
|
||||
# frequency table over 330k Han words, and measurement showed freq>=500 covers the
|
||||
# entire observed false-positive class (具体 自己 硬件 不了 主要 时间 分钟) while
|
||||
# still excluding the真 targets we must keep flagging (元烨 药名 中式 姚明).
|
||||
JIEBA_DICT = "https://raw.githubusercontent.com/fxsjy/jieba/master/jieba/dict.txt"
|
||||
JIEBA_MIN_FREQ = 500
|
||||
|
||||
HAN = re.compile(r"^[\u4e00-\u9fff]+$")
|
||||
|
||||
|
||||
_last_wiki = [0.0]
|
||||
|
||||
|
||||
def fetch(url: str, timeout=60, throttle=0.0) -> str:
|
||||
# zhwiki returns 429 after a handful of rapid api.php calls; pace them and
|
||||
# retry once on 429 so a term-bank build is reproducible rather than
|
||||
# silently losing whole drug-name families.
|
||||
if throttle:
|
||||
import time
|
||||
wait = throttle - (time.time() - _last_wiki[0])
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
_last_wiki[0] = time.time()
|
||||
req = urllib.request.Request(url, headers=UA)
|
||||
for attempt in range(4):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.read().decode("utf-8", "ignore")
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code != 429 or attempt == 3:
|
||||
raise
|
||||
import time
|
||||
time.sleep(6 * (attempt + 1))
|
||||
raise RuntimeError("unreachable")
|
||||
|
||||
|
||||
def wiki_intitle(keyword: str, limit=500) -> list[str]:
|
||||
"""zhwiki titles containing `keyword`. INN generic names live here."""
|
||||
out, offset = [], 0
|
||||
while offset < limit:
|
||||
q = urllib.parse.urlencode({
|
||||
"action": "query", "list": "search",
|
||||
"srsearch": f"intitle:{keyword}", "srlimit": 50, "sroffset": offset,
|
||||
"srnamespace": 0, "format": "json",
|
||||
})
|
||||
try:
|
||||
d = json.loads(fetch(f"https://zh.wikipedia.org/w/api.php?{q}", 40, throttle=1.5))
|
||||
except Exception as e:
|
||||
print(f" wiki {keyword} @{offset} failed: {e}", file=sys.stderr)
|
||||
break
|
||||
hits = d.get("query", {}).get("search", [])
|
||||
if not hits:
|
||||
break
|
||||
out += [h["title"] for h in hits]
|
||||
if "continue" not in d:
|
||||
break
|
||||
offset += 50
|
||||
return out
|
||||
|
||||
|
||||
def build():
|
||||
BANK.mkdir(parents=True, exist_ok=True)
|
||||
terms: dict[str, dict] = {}
|
||||
|
||||
def add(t: str, source: str, kind: str):
|
||||
t = t.strip()
|
||||
if not (2 <= len(t) <= 20):
|
||||
return
|
||||
e = terms.setdefault(t, {"sources": [], "kind": kind})
|
||||
if source not in e["sources"]:
|
||||
e["sources"].append(source)
|
||||
|
||||
print("[1/4] curated CDMO/CMC jargon")
|
||||
for t in CDMO_TERMS.split():
|
||||
add(t, "cdmo", "工艺术语")
|
||||
for t in DOMAIN_ORGS.split():
|
||||
add(t, "org", "机构")
|
||||
print(f" {len(terms)} terms")
|
||||
|
||||
print("[2/4] zhwiki INN generic drug names")
|
||||
n0 = len(terms)
|
||||
for kw in ("单抗", "替尼", "西普", "格鲁肽", "干扰素", "他滨", "疫苗", "沙星"):
|
||||
titles = wiki_intitle(kw)
|
||||
good = 0
|
||||
for t in titles:
|
||||
# Keep only names that genuinely END with a drug stem; the search also
|
||||
# returns 朱塞佩·塔替尼 (a violinist) and 塔替尼广场.
|
||||
if HAN.match(t) and any(t.endswith(s) for s in INN_SUFFIXES):
|
||||
add(t, "inn-wiki", "药品通用名")
|
||||
good += 1
|
||||
print(f" {kw}: {len(titles)} hits -> {good} kept")
|
||||
print(f" +{len(terms) - n0} drug names")
|
||||
|
||||
print("[3/5] jieba general-frequency dictionary -> allowlist")
|
||||
allow: set[str] = set()
|
||||
try:
|
||||
raw = fetch(JIEBA_DICT, 120)
|
||||
n = 0
|
||||
for line in raw.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) < 2 or not parts[1].isdigit():
|
||||
continue
|
||||
w, f = parts[0], int(parts[1])
|
||||
if HAN.match(w) and 2 <= len(w) <= 4:
|
||||
n += 1
|
||||
if f >= JIEBA_MIN_FREQ:
|
||||
allow.add(w)
|
||||
print(f" {n} han words scanned, {len(allow)} with freq>={JIEBA_MIN_FREQ}")
|
||||
except Exception as e:
|
||||
print(f" jieba dict FAILED: {e}", file=sys.stderr)
|
||||
|
||||
print("[4/5] THUOCL domain wordlists (domain vocabulary)")
|
||||
for name, url in THUOCL.items():
|
||||
try:
|
||||
raw = fetch(url, 90)
|
||||
except Exception as e:
|
||||
print(f" {name}: FAILED {e}", file=sys.stderr)
|
||||
continue
|
||||
words = []
|
||||
for line in raw.splitlines():
|
||||
parts = line.split()
|
||||
if not parts:
|
||||
continue
|
||||
w = parts[0]
|
||||
freq = int(parts[1]) if len(parts) > 1 and parts[1].isdigit() else 0
|
||||
if HAN.match(w) and 2 <= len(w) <= 8:
|
||||
words.append((w, freq))
|
||||
# Medical/IT words are domain vocabulary. They deliberately do NOT feed
|
||||
# the allowlist: THUOCL frequency is corpus-relative (医院 ranks huge) and
|
||||
# using it as a "this is a common word" signal both over-blocks real terms
|
||||
# and under-covers the general words that actually cause false positives.
|
||||
if name in ("medical", "it"):
|
||||
for w, f in words:
|
||||
if 3 <= len(w) <= 8 and f < 10000:
|
||||
add(w, f"thuocl-{name}", "领域词")
|
||||
print(f" {name}: {len(words)} words")
|
||||
|
||||
print("[5/5] vault glossary + entity cards")
|
||||
n0 = len(terms)
|
||||
gl = CFG.glossary
|
||||
if gl.exists():
|
||||
for line in gl.read_text(errors="ignore").splitlines():
|
||||
m = re.match(r"^-\s*(.+?)\s*[→⇒➔]\s*(.+)$", line.strip())
|
||||
if m:
|
||||
right = re.sub(r"（.*?）|\[\[|\]\]", "", m.group(2))
|
||||
right = right.split("；")[0].split("/")[0].strip()
|
||||
if right:
|
||||
add(right, "glossary", "已确认纠错目标")
|
||||
else:
|
||||
m2 = re.match(r"^-\s*([^\s—:：]+)\s*[—:：]", line.strip())
|
||||
if m2:
|
||||
add(re.sub(r"\[\[|\]\]", "", m2.group(1)), "glossary", "术语表")
|
||||
for sub, kind in (("People", "人名"), ("Companies", "公司")):
|
||||
d = CFG.vault / sub
|
||||
for f in d.glob("*.md") if d.exists() else []:
|
||||
add(f.stem, f"card-{sub.lower()}", kind)
|
||||
print(f" +{len(terms) - n0}")
|
||||
|
||||
# An allowlisted common word must not also be offered as a correction target.
|
||||
for w in allow:
|
||||
if w in terms and terms[w]["sources"] == ["thuocl-medical"]:
|
||||
del terms[w]
|
||||
|
||||
(BANK / "terms.json").write_text(
|
||||
json.dumps(terms, ensure_ascii=False, indent=0), encoding="utf-8")
|
||||
(BANK / "allowlist.txt").write_text(
|
||||
"\n".join(sorted(allow)), encoding="utf-8")
|
||||
(BANK / "inn_suffixes.json").write_text(
|
||||
json.dumps({"suffixes": INN_SUFFIXES, "kinds": INN_KIND}, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
print(f"\nwrote {len(terms)} terms, {len(allow)} allowlist -> {BANK}")
|
||||
|
||||
|
||||
def stats():
|
||||
terms = json.loads((BANK / "terms.json").read_text())
|
||||
from collections import Counter
|
||||
c = Counter(s for e in terms.values() for s in e["sources"])
|
||||
k = Counter(e["kind"] for e in terms.values())
|
||||
print(f"total {len(terms)} terms")
|
||||
print("by source:", dict(c))
|
||||
print("by kind: ", dict(k))
|
||||
allow = (BANK / "allowlist.txt").read_text().split()
|
||||
print(f"allowlist {len(allow)}: {allow[:15]}")
|
||||
drugs = [t for t, e in terms.items() if e["kind"] == "药品通用名"]
|
||||
print(f"drug names {len(drugs)}: {drugs[:12]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
(build if len(sys.argv) < 2 or sys.argv[1] == "build" else stats)()
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Vault side effects: draft file, pending index, git sync, notification.
|
||||
|
||||
Ported from the legacy shell implementation with its hard-won details preserved:
|
||||
* a same date+title never overwrites an existing draft (suffix -2, -3, …)
|
||||
* the pending index row is created BEFORE staging, so one commit carries both
|
||||
* the 5-minute `obsidian-sync.sh` cron may have already staged the files; that is
|
||||
tolerated rather than treated as an error
|
||||
* a failed push is reported as "local only", never as success
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from .config import CFG
|
||||
from .log import log
|
||||
|
||||
PENDING_HEADER = """---
|
||||
type: index
|
||||
title: 待确认草稿
|
||||
note: 自动维护，确认定稿后请把对应行删除
|
||||
---
|
||||
|
||||
# 待确认草稿 (ASR 自动精修)
|
||||
|
||||
| 日期 | 标题 | 草稿 |
|
||||
|---|---|---|
|
||||
"""
|
||||
|
||||
|
||||
def slug_parts(slug: str) -> tuple[str, str]:
|
||||
"""`2026-07-21-125915录音` -> ('2026-07-21', '125915录音')."""
|
||||
m = re.match(r"^(\d{4}-\d{2}-\d{2})[-_](.+)$", slug)
|
||||
if m:
|
||||
return m.group(1), m.group(2)
|
||||
import time
|
||||
return time.strftime("%Y-%m-%d"), slug
|
||||
|
||||
|
||||
def draft_path(date: str, title: str) -> Path:
|
||||
CFG.drafts.mkdir(parents=True, exist_ok=True)
|
||||
p = CFG.drafts / f"{date}-{title}.md"
|
||||
if not p.exists():
|
||||
return p
|
||||
n = 2
|
||||
while (CFG.drafts / f"{date}-{title}-{n}.md").exists():
|
||||
n += 1
|
||||
return CFG.drafts / f"{date}-{title}-{n}.md"
|
||||
|
||||
|
||||
def write_draft_file(path: Path, body: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
if not body.endswith("\n"):
|
||||
body += "\n"
|
||||
with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=path.parent,
|
||||
prefix=f".{path.name}.", delete=False) as f:
|
||||
f.write(body)
|
||||
tmp = Path(f.name)
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def find_draft_for_transcript(transcript: str) -> Path | None:
|
||||
"""Recover an interrupted run without paying for or creating a second draft."""
|
||||
if not CFG.drafts.exists():
|
||||
return None
|
||||
needle = f"transcript: {transcript}"
|
||||
for p in CFG.drafts.glob("*.md"):
|
||||
try:
|
||||
if needle in p.read_text(errors="ignore")[:2000]:
|
||||
return p
|
||||
except OSError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def update_pending(date: str, title: str, path: Path) -> None:
|
||||
p = CFG.pending_index
|
||||
if not p.exists():
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(PENDING_HEADER, encoding="utf-8")
|
||||
base = path.stem
|
||||
text = p.read_text(errors="ignore")
|
||||
if f"[[{base}]]" in text:
|
||||
return
|
||||
text += f"| {date} | {title} | [[{base}]] |\n"
|
||||
write_draft_file(p, text)
|
||||
|
||||
|
||||
def _git(*args, check=False) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["git", *args], cwd=CFG.vault, capture_output=True,
|
||||
text=True, timeout=180, check=check)
|
||||
|
||||
|
||||
def sync(paths: list[Path], message: str) -> str:
|
||||
"""Commit + push. Returns 'pushed' | 'local' | 'noop' | 'disabled' | 'error'."""
|
||||
if not CFG.git_sync:
|
||||
return "disabled"
|
||||
try:
|
||||
rel = [str(p.relative_to(CFG.vault)) for p in paths if p.exists()]
|
||||
if not rel:
|
||||
return "noop"
|
||||
_git("add", "--", *rel)
|
||||
if _git("diff", "--cached", "--quiet").returncode == 0:
|
||||
log("nothing staged (likely already committed by the auto-sync cron)")
|
||||
return "noop"
|
||||
# Commit only our paths. The 5-minute vault sync may already have unrelated
|
||||
# files staged; absorbing those into this transaction makes rollback and
|
||||
# diagnosis needlessly difficult.
|
||||
c = _git("commit", "-q", "--only", "-m", message, "--", *rel)
|
||||
if c.returncode != 0:
|
||||
log(f"git commit failed: {c.stderr.strip()[:200]}", level="ERROR")
|
||||
return "error"
|
||||
p = _git("push", "-q")
|
||||
if p.returncode != 0:
|
||||
log(f"committed but push failed (local only): {p.stderr.strip()[:200]}",
|
||||
level="WARN")
|
||||
return "local"
|
||||
return "pushed"
|
||||
except Exception as e:
|
||||
log(f"git sync error: {e}", level="ERROR")
|
||||
return "error"
|
||||
|
||||
|
||||
def notify(title: str, body: str, *, redact_log: bool = False) -> None:
|
||||
if not CFG.notify:
|
||||
return
|
||||
n = CFG.notifier
|
||||
if not n.exists():
|
||||
log(f"NOTIFY (no notifier): {title} — {body}")
|
||||
return
|
||||
try:
|
||||
r = subprocess.run(["python3", str(n), "草稿已生成，待确认", title, body],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
detail = "[sensitive body redacted]" if redact_log else body
|
||||
log(f"NOTIFY: {title} — {detail} ({r.stdout.strip()[:80]})")
|
||||
except Exception as e:
|
||||
log(f"notify failed: {e}", level="WARN")
|
||||
|
||||
|
||||
def mark_done(slug: str) -> Path:
|
||||
CFG.state.mkdir(parents=True, exist_ok=True)
|
||||
m = CFG.state / f"{slug}.done"
|
||||
m.touch()
|
||||
return m
|
||||
|
||||
|
||||
def is_done(slug: str) -> bool:
|
||||
return (CFG.state / f"{slug}.done").exists()
|
||||
|
||||
|
||||
def pending_transcripts() -> list[Path]:
|
||||
if not CFG.inbox.exists():
|
||||
return []
|
||||
return sorted(p for p in CFG.inbox.glob("*.txt") if not is_done(p.stem))
|
||||
@@ -0,0 +1 @@
|
||||
"""Standalone ASR service for web, Agent API, and SSH drop-folder intake."""
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
DATA = Path(os.getenv("ASR_DATA_DIR", "/data"))
|
||||
INBOX = DATA / "inbox"
|
||||
PROCESSING = DATA / "processing"
|
||||
PROCESSED = DATA / "processed"
|
||||
FAILED = DATA / "failed"
|
||||
TRANSCRIPTS = DATA / "transcripts"
|
||||
STATE = Path(os.getenv("ASR_STATE_DIR", "/state"))
|
||||
DRAFTS = DATA / "artifacts" / "drafts"
|
||||
RESULTS = DATA / "artifacts" / "results"
|
||||
LOGS = DATA / "logs"
|
||||
META = STATE / "uploads"
|
||||
REVISION_QUEUE = STATE / "revision-queue"
|
||||
DB = STATE / "intake.db"
|
||||
SUPPORTED = {".m4a", ".mp3", ".wav", ".mp4", ".aac", ".ogg", ".opus", ".webm", ".flac"}
|
||||
PUBLIC_URL = os.getenv("ASR_PUBLIC_URL", "http://192.168.50.49:8788").rstrip("/")
|
||||
AGENT_TOKEN = os.getenv("ASR_AGENT_TOKEN", "")
|
||||
DIARIZE_URL = os.getenv("ASR_DIARIZE_URL", "http://192.168.50.100:10099/asr")
|
||||
MAX_UPLOAD = int(os.getenv("ASR_UPLOAD_MAX_BYTES", str(4 * 1024**3)))
|
||||
POLL_SECONDS = max(1, int(os.getenv("ASR_POLL_SECONDS", "5")))
|
||||
STABLE_SECONDS = max(1, int(os.getenv("ASR_STABLE_SECONDS", "10")))
|
||||
MAX_ATTEMPTS = max(1, int(os.getenv("ASR_MAX_ATTEMPTS", "3")))
|
||||
|
||||
|
||||
def ensure_dirs() -> None:
|
||||
for path in (INBOX, PROCESSING, PROCESSED, FAILED, TRANSCRIPTS, DRAFTS,
|
||||
RESULTS, STATE, LOGS, META, REVISION_QUEUE, DATA / "cache",
|
||||
DATA / "artifacts" / "raw", DATA / "vault" / "minutes",
|
||||
DATA / "vault" / "People", DATA / "vault" / "Companies"):
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
@@ -0,0 +1,120 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from .config import DB, ensure_dirs
|
||||
|
||||
STATUSES = {"uploading", "queued", "transcribing", "refining", "review",
|
||||
"publishing", "complete", "failed"}
|
||||
|
||||
|
||||
def connect() -> sqlite3.Connection:
|
||||
ensure_dirs()
|
||||
con = sqlite3.connect(DB, timeout=30)
|
||||
con.row_factory = sqlite3.Row
|
||||
con.execute("PRAGMA journal_mode=WAL")
|
||||
con.execute("PRAGMA foreign_keys=ON")
|
||||
con.executescript("""
|
||||
CREATE TABLE IF NOT EXISTS intake_job(
|
||||
id TEXT PRIMARY KEY, original_name TEXT NOT NULL, stored_name TEXT NOT NULL,
|
||||
title TEXT NOT NULL DEFAULT '', context TEXT NOT NULL DEFAULT '',
|
||||
size INTEGER NOT NULL DEFAULT 0, status TEXT NOT NULL,
|
||||
detail TEXT NOT NULL DEFAULT '', transcript TEXT NOT NULL DEFAULT '',
|
||||
draft TEXT NOT NULL DEFAULT '', result TEXT NOT NULL DEFAULT '',
|
||||
source TEXT NOT NULL DEFAULT 'web', sha256 TEXT NOT NULL DEFAULT '',
|
||||
attempts INTEGER NOT NULL DEFAULT 0, error_code TEXT NOT NULL DEFAULT '',
|
||||
created_at REAL NOT NULL, updated_at REAL NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS job_event(
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
job_id TEXT NOT NULL REFERENCES intake_job(id) ON DELETE CASCADE,
|
||||
status TEXT NOT NULL, detail TEXT NOT NULL DEFAULT '', created_at REAL NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ix_job_event_job ON job_event(job_id,id);
|
||||
""")
|
||||
cols = {r[1] for r in con.execute("PRAGMA table_info(intake_job)")}
|
||||
additions = {
|
||||
"draft": "TEXT NOT NULL DEFAULT ''", "result": "TEXT NOT NULL DEFAULT ''",
|
||||
"source": "TEXT NOT NULL DEFAULT 'web'", "sha256": "TEXT NOT NULL DEFAULT ''",
|
||||
"attempts": "INTEGER NOT NULL DEFAULT 0", "error_code": "TEXT NOT NULL DEFAULT ''",
|
||||
}
|
||||
for name, ddl in additions.items():
|
||||
if name not in cols:
|
||||
con.execute(f"ALTER TABLE intake_job ADD COLUMN {name} {ddl}")
|
||||
con.commit()
|
||||
return con
|
||||
|
||||
|
||||
def create_job(job_id: str, original_name: str, stored_name: str, *, title: str = "",
|
||||
context: str = "", size: int = 0, source: str = "web") -> dict[str, Any]:
|
||||
now = time.time()
|
||||
with connect() as con:
|
||||
con.execute("INSERT INTO intake_job(id,original_name,stored_name,title,context,size,"
|
||||
"status,source,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?)",
|
||||
(job_id, original_name, stored_name, title, context, size,
|
||||
"queued", source, now, now))
|
||||
con.execute("INSERT INTO job_event(job_id,status,detail,created_at) VALUES(?,?,?,?)",
|
||||
(job_id, "queued", "录音已进入队列", now))
|
||||
return get_job(job_id) or {}
|
||||
|
||||
|
||||
def update_job(job_id: str, status: str, detail: str = "", **fields: Any) -> bool:
|
||||
if status not in STATUSES:
|
||||
raise ValueError(f"invalid status: {status}")
|
||||
allowed = {"transcript", "draft", "result", "sha256", "attempts", "error_code"}
|
||||
changes = {k: v for k, v in fields.items() if k in allowed and v is not None}
|
||||
sets = ["status=?", "updated_at=?", "detail=?"]
|
||||
values: list[Any] = [status, time.time(), detail[:1000]]
|
||||
for key, value in changes.items():
|
||||
sets.append(f"{key}=?")
|
||||
values.append(value)
|
||||
values.append(job_id)
|
||||
with connect() as con:
|
||||
cur = con.execute(f"UPDATE intake_job SET {','.join(sets)} WHERE id=?", values)
|
||||
if cur.rowcount:
|
||||
con.execute("INSERT INTO job_event(job_id,status,detail,created_at) VALUES(?,?,?,?)",
|
||||
(job_id, status, detail[:1000], time.time()))
|
||||
return bool(cur.rowcount)
|
||||
|
||||
|
||||
def get_job(job_id: str) -> dict[str, Any] | None:
|
||||
with connect() as con:
|
||||
row = con.execute("SELECT * FROM intake_job WHERE id=?", (job_id,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def find_by_stored(stored_name: str) -> dict[str, Any] | None:
|
||||
with connect() as con:
|
||||
row = con.execute("SELECT * FROM intake_job WHERE stored_name=? ORDER BY created_at DESC LIMIT 1",
|
||||
(stored_name,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def list_jobs(limit: int = 30, offset: int = 0, status: str = "") -> list[dict[str, Any]]:
|
||||
limit = min(max(limit, 1), 200)
|
||||
offset = max(offset, 0)
|
||||
sql, args = "SELECT * FROM intake_job", []
|
||||
if status:
|
||||
sql += " WHERE status=?"
|
||||
args.append(status)
|
||||
sql += " ORDER BY created_at DESC LIMIT ? OFFSET ?"
|
||||
args.extend((limit, offset))
|
||||
with connect() as con:
|
||||
return [dict(r) for r in con.execute(sql, args).fetchall()]
|
||||
|
||||
|
||||
def events(job_id: str) -> list[dict[str, Any]]:
|
||||
with connect() as con:
|
||||
return [dict(r) for r in con.execute(
|
||||
"SELECT status,detail,created_at FROM job_event WHERE job_id=? ORDER BY id",
|
||||
(job_id,)).fetchall()]
|
||||
|
||||
|
||||
def pending_recovery() -> list[dict[str, Any]]:
|
||||
with connect() as con:
|
||||
return [dict(r) for r in con.execute(
|
||||
"SELECT * FROM intake_job WHERE status IN ('queued','transcribing','refining','publishing') "
|
||||
"ORDER BY created_at").fetchall()]
|
||||
@@ -0,0 +1,5 @@
|
||||
import time
|
||||
from .config import STATE
|
||||
|
||||
heartbeat = STATE / "worker.heartbeat"
|
||||
raise SystemExit(0 if heartbeat.exists() and time.time() - heartbeat.stat().st_mtime < 90 else 1)
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
from urllib import parse, request
|
||||
|
||||
|
||||
def send(status: str, filename: str = "", detail: str = "") -> bool:
|
||||
backend = os.getenv("ASR_NOTIFY_BACKEND", "none").strip().lower()
|
||||
try:
|
||||
if backend == "telegram":
|
||||
token = os.getenv("TELEGRAM_BOT_TOKEN", "")
|
||||
chat = os.getenv("TELEGRAM_CHAT_ID", "")
|
||||
if not token or not chat:
|
||||
return False
|
||||
body = parse.urlencode({"chat_id": chat,
|
||||
"text": "\n".join(x for x in (status, filename, detail) if x)}).encode()
|
||||
req = request.Request(f"https://api.telegram.org/bot{token}/sendMessage",
|
||||
data=body, method="POST")
|
||||
elif backend == "hermes":
|
||||
url = os.getenv("HERMES_WEBHOOK_URL", "")
|
||||
secret = os.getenv("HERMES_WEBHOOK_SECRET", "")
|
||||
if not url or not secret:
|
||||
return False
|
||||
body = json.dumps({"status": status, "filename": filename, "detail": detail},
|
||||
ensure_ascii=False).encode()
|
||||
sig = hmac.new(secret.encode(), body, hashlib.sha256).hexdigest()
|
||||
req = request.Request(url, data=body, method="POST",
|
||||
headers={"Content-Type": "application/json",
|
||||
"X-Webhook-Signature": sig})
|
||||
else:
|
||||
return False
|
||||
with request.urlopen(req, timeout=10) as response:
|
||||
return 200 <= response.status < 300
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,178 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib, hmac, html, json, os, re, time, urllib.parse, uuid
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
from asrpipe import review
|
||||
from . import db
|
||||
from .config import AGENT_TOKEN, DRAFTS, FAILED, INBOX, MAX_UPLOAD, PUBLIC_URL, RESULTS, STATE, SUPPORTED, TRANSCRIPTS, ensure_dirs
|
||||
|
||||
LABELS = {"queued":"等待处理","transcribing":"正在转写","refining":"正在整理","review":"等待确认","publishing":"正在定稿","complete":"已完成","failed":"处理失败"}
|
||||
|
||||
def safe(value, limit):
|
||||
return re.sub(r"\s+", " ", re.sub(r"[\x00-\x1f\x7f]+", " ", value or "")).strip()[:limit]
|
||||
|
||||
def stem(value):
|
||||
return re.sub(r"[^\w\u3400-\u9fff.-]+", "-", safe(value, 80)).strip("-.") or "录音"
|
||||
|
||||
def public(row, detailed=False):
|
||||
out = {k: row.get(k) for k in ("id","original_name","title","size","status","detail","source","sha256","attempts","error_code","created_at","updated_at")}
|
||||
out["status_label"] = LABELS.get(row["status"], row["status"])
|
||||
out["web_url"] = f"{PUBLIC_URL}/jobs/{row['id']}/"
|
||||
out["review_url"] = f"{PUBLIC_URL}/jobs/{row['id']}/review/"
|
||||
out["artifacts"] = {name:f"/api/v1/jobs/{row['id']}/artifacts/{name}" for name in ("transcript","draft","result") if row.get(name)}
|
||||
if detailed:
|
||||
out["context"] = row.get("context", "")
|
||||
out["events"] = db.events(row["id"])
|
||||
return out
|
||||
|
||||
def pending(job):
|
||||
found=[]
|
||||
for row in review.pending(1000):
|
||||
payload=json.loads(row["payload"] or "{}")
|
||||
if ((job.get("transcript") and row["recording"]==job["transcript"])
|
||||
or (job.get("draft") and payload.get("draft_path")==job["draft"])):
|
||||
found.append(row)
|
||||
return found
|
||||
|
||||
def shell(title, body, script=""):
|
||||
return f'''<!doctype html><html lang=zh-CN><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>{html.escape(title)}</title><style>
|
||||
:root{{--bg:#f4f6f8;--card:#fff;--ink:#18212b;--muted:#647180;--line:#d9dfe6;--accent:#176b5b;--bad:#b33a2b}}@media(prefers-color-scheme:dark){{:root{{--bg:#11161b;--card:#192128;--ink:#eef3f5;--muted:#a8b3bd;--line:#34414b;--accent:#63c7ae}}}}*{{box-sizing:border-box}}body{{margin:0;background:var(--bg);color:var(--ink);font:16px/1.55 system-ui,sans-serif}}header{{background:var(--card);border-bottom:1px solid var(--line)}}header div,main{{max-width:900px;margin:auto;padding:16px}}header div,.row{{display:flex;justify-content:space-between;gap:12px}}a{{color:var(--accent);text-decoration:none}}nav a{{margin-left:16px}}.card{{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:18px;margin:15px 0}}.job{{padding:13px 0;border-top:1px solid var(--line)}}.job:first-child{{border:0}}.muted{{color:var(--muted);font-size:14px}}.badge{{font-weight:700;color:var(--accent)}}.failed{{color:var(--bad)}}input,textarea,button{{font:inherit}}input,textarea{{width:100%;padding:10px;border:1px solid var(--line);border-radius:7px;background:var(--card);color:var(--ink)}}label{{display:block;margin:12px 0 5px;font-weight:650}}button{{border:0;border-radius:7px;padding:11px 16px;background:var(--accent);color:white;font-weight:700}}button.alt{{background:#68747f}}progress{{width:100%}}pre{{white-space:pre-wrap;overflow-wrap:anywhere;max-height:55vh;overflow:auto}}.timeline{{border-left:2px solid var(--line);padding-left:16px}}.choice{{display:block;width:100%;margin:7px 0}}@media(max-width:520px){{.row{{display:block}}}}</style></head><body><header><div><strong>ASR 录音工作台</strong><nav><a href=/>任务</a><a href=/docs>Agent API</a></nav></div></header><main>{body}</main>{script}</body></html>'''.encode()
|
||||
|
||||
def home():
|
||||
body='''<section class=card><h1>上传录音</h1><p class=muted>也可通过 SSH 上传到 tools-pve:/srv/asr/inbox，完成后自动处理。</p><form id=up><label>录音文件</label><input id=file type=file required><label>标题</label><input id=title maxlength=80><label>已知背景</label><textarea id=context maxlength=2000></textarea><p><button id=send>上传并处理</button></p><progress id=bar max=100 hidden></progress><div id=msg class=muted></div></form></section><section class=card><div class=row><h2>最近任务</h2><span class=muted>自动刷新</span></div><div id=jobs>读取中…</div></section>'''
|
||||
script='''<script>const E=s=>String(s??'').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]));async function load(){let d=await(await fetch('/ui/jobs')).json();jobs.innerHTML=d.jobs.length?d.jobs.map(j=>`<div class=job><div class=row><strong>${E(j.title||j.original_name)}</strong><span class="badge ${E(j.status)}">${E(j.status_label)}</span></div><div class=muted>${new Date(j.created_at*1000).toLocaleString()} · ${E(j.source)} · ${E(j.detail)}</div><a href="/jobs/${E(j.id)}/">查看任务</a></div>`).join(''):'暂无任务'}load();setInterval(load,8000);up.onsubmit=x=>{x.preventDefault();let f=file.files[0],q=new XMLHttpRequest();send.disabled=true;bar.hidden=false;msg.textContent='上传中…';q.open('POST','/ui/upload');q.setRequestHeader('X-Filename',encodeURIComponent(f.name));q.setRequestHeader('X-Title',encodeURIComponent(title.value));q.setRequestHeader('X-Context',encodeURIComponent(context.value));q.upload.onprogress=v=>{if(v.lengthComputable)bar.value=v.loaded/v.total*100};q.onload=()=>{send.disabled=false;bar.hidden=true;msg.textContent=q.status===201?'已进入队列':'上传失败';if(q.status===201){x.target.reset();load()}};q.send(f)};</script>'''
|
||||
return shell("ASR 录音工作台",body,script)
|
||||
|
||||
def job_page(job):
|
||||
events=''.join(f"<p><strong>{html.escape(LABELS.get(e['status'],e['status']))}</strong><br><span class=muted>{time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(e['created_at']))} · {html.escape(e['detail'])}</span></p>" for e in db.events(job["id"]))
|
||||
links=' · '.join(f'<a href="/jobs/{job["id"]}/artifact/{n}">{label}</a>' for n,label in (("transcript","转写"),("draft","草稿"),("result","正式稿")) if job.get(n))
|
||||
retry=f'<form method=post action="/jobs/{job["id"]}/retry"><button>重新处理</button></form>' if job["status"]=="failed" else ""
|
||||
review_link=f'<p><a href="/jobs/{job["id"]}/review/">继续人工确认</a></p>' if pending(job) else ""
|
||||
return shell("任务详情",f"<section class=card><div class=row><h1>{html.escape(job.get('title') or job['original_name'])}</h1><span class='badge {job['status']}'>{html.escape(LABELS.get(job['status'],job['status']))}</span></div><p>{html.escape(job.get('detail',''))}</p><p>{links}</p>{review_link}{retry}</section><section class=card><h2>处理时间线</h2><div class=timeline>{events}</div></section>")
|
||||
|
||||
def review_page(job):
|
||||
rows=pending(job)
|
||||
if not rows: return shell("人工确认","<section class=card><h1>当前没有待确认问题</h1><p><a href=../>返回任务</a></p></section>")
|
||||
row=rows[0]; payload=json.loads(row["payload"] or "{}"); choices=payload.get("candidates") or []
|
||||
buttons=''.join(f'<button class=choice name=answer value="{html.escape(str(v),quote=True)}">{html.escape(str(v))}</button>' for v in choices)+'<button class="choice alt" name=answer value="无法确认，保留原文">无法确认 / 保留原文</button>'
|
||||
draft=""; path=Path(str(payload.get("draft_path") or ""))
|
||||
try: path.resolve().relative_to(DRAFTS.resolve()); draft=path.read_text(encoding="utf-8")
|
||||
except (OSError,ValueError): pass
|
||||
return shell("人工确认",f"<section class=card><p class=muted>剩余 {len(rows)} 个问题</p><h1>{html.escape(row['question'])}</h1><p>{html.escape(str(payload.get('sentence') or ''))}</p><details><summary>查看草稿</summary><pre>{html.escape(draft)}</pre></details><form method=post><input type=hidden name=id value={row['id']}>{buttons}<label>自定义回答</label><input name=custom maxlength=120><p><button>提交</button></p></form></section>")
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version="ASRService/1"
|
||||
def log_message(self,*_): pass
|
||||
def send_data(self,data,status=200,ctype="text/html; charset=utf-8",disposition=""):
|
||||
self.send_response(status); self.send_header("Content-Type",ctype); self.send_header("Cache-Control","no-store"); self.send_header("X-Content-Type-Options","nosniff"); self.send_header("Content-Security-Policy","default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'")
|
||||
if disposition:self.send_header("Content-Disposition",disposition)
|
||||
self.send_header("Content-Length",str(len(data)));self.end_headers();self.wfile.write(data)
|
||||
def json(self,obj,status=200):self.send_data(json.dumps(obj,ensure_ascii=False).encode(),status,"application/json; charset=utf-8")
|
||||
def auth(self):
|
||||
value=self.headers.get("Authorization","")
|
||||
return bool(AGENT_TOKEN and value.startswith("Bearer ") and hmac.compare_digest(value[7:],AGENT_TOKEN))
|
||||
def need_auth(self):
|
||||
if self.auth():return True
|
||||
self.json({"error":"unauthorized"},401);return False
|
||||
def header(self,name,limit):return safe(urllib.parse.unquote(self.headers.get(name,"")),limit)
|
||||
def upload(self,source,protected):
|
||||
if protected and not self.auth():return self.json({"error":"unauthorized"},401)
|
||||
try:length=int(self.headers.get("Content-Length","0"))
|
||||
except ValueError:length=0
|
||||
name=self.header("X-Filename",240);suffix=Path(name).suffix.lower()
|
||||
if length<=0 or length>MAX_UPLOAD:return self.json({"error":"invalid size"},413)
|
||||
if suffix not in SUPPORTED:return self.json({"error":"unsupported type"},415)
|
||||
jid=uuid.uuid4().hex[:24]; title=self.header("X-Title",80); stored=f"{time.strftime('%Y-%m-%d_%H_%M_%S')}-{stem(title or Path(name).stem)}-{jid[:8]}{suffix}"; staging=INBOX/".uploading";staging.mkdir(parents=True,exist_ok=True);part=staging/f"{jid}.part";remaining=length;digest=hashlib.sha256()
|
||||
try:
|
||||
with part.open("xb") as out:
|
||||
while remaining:
|
||||
chunk=self.rfile.read(min(1024*1024,remaining))
|
||||
if not chunk:raise EOFError
|
||||
out.write(chunk);digest.update(chunk);remaining-=len(chunk)
|
||||
out.flush();os.fsync(out.fileno())
|
||||
job=db.create_job(jid,name,stored,title=title,context=self.header("X-Context",2000),size=length,source=source);db.update_job(jid,"queued","录音已落盘，等待 worker",sha256=digest.hexdigest());part.replace(INBOX/stored);self.json(public(job),201)
|
||||
except Exception:part.unlink(missing_ok=True);self.json({"error":"upload failed"},500)
|
||||
def artifact(self,job,name,api=False):
|
||||
if not job:return self.json({"error":"not found"},404)
|
||||
roots={"transcript":TRANSCRIPTS,"draft":DRAFTS,"result":RESULTS};path=Path(job.get(name) or "")
|
||||
try:path.resolve().relative_to(roots[name].resolve());data=path.read_bytes()
|
||||
except (OSError,ValueError):return self.json({"error":"not found"},404)
|
||||
self.send_data(data,200,"text/markdown; charset=utf-8",f'inline; filename="{path.name}"')
|
||||
def do_GET(self):
|
||||
path=urllib.parse.urlsplit(self.path).path
|
||||
if path in ("/health","/healthz"):return self.json({"ok":True,"service":"asr-web"})
|
||||
if path=="/readyz":
|
||||
try:db.connect().close();ok=(STATE/"worker.heartbeat").exists() and time.time()-(STATE/"worker.heartbeat").stat().st_mtime<90
|
||||
except Exception:ok=False
|
||||
return self.json({"ready":ok},200 if ok else 503)
|
||||
if path=="/":return self.send_data(home())
|
||||
if path=="/docs":return self.send_data(shell("Agent API","<section class=card><h1>Agent API</h1><p>使用 Bearer Token。POST /api/v1/jobs 的 body 是原始录音，设置 X-Filename、X-Title、X-Context；GET /api/v1/jobs 与 /api/v1/jobs/{id} 查询任务。</p></section>"))
|
||||
if path=="/ui/jobs":return self.json({"jobs":[public(x) for x in db.list_jobs()]})
|
||||
m=re.fullmatch(r"/jobs/([0-9a-f]{24})/",path)
|
||||
if m:
|
||||
job=db.get_job(m.group(1));return self.send_data(job_page(job)) if job else self.send_data(b"Not found",404,"text/plain")
|
||||
m=re.fullmatch(r"/jobs/([0-9a-f]{24})/review/",path)
|
||||
if m:
|
||||
job=db.get_job(m.group(1));return self.send_data(review_page(job)) if job else self.send_data(b"Not found",404,"text/plain")
|
||||
m=re.fullmatch(r"/jobs/([0-9a-f]{24})/artifact/(transcript|draft|result)",path)
|
||||
if m:return self.artifact(db.get_job(m.group(1)),m.group(2))
|
||||
if path=="/api/v1/jobs":
|
||||
if not self.need_auth():return
|
||||
q=urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query);return self.json({"jobs":[public(x) for x in db.list_jobs(int(q.get('limit',[30])[0]),int(q.get('offset',[0])[0]),q.get('status',[''])[0])]})
|
||||
m=re.fullmatch(r"/api/v1/jobs/([0-9a-f]{24})",path)
|
||||
if m:
|
||||
if not self.need_auth():return
|
||||
job=db.get_job(m.group(1));return self.json(public(job,True)) if job else self.json({"error":"not found"},404)
|
||||
m=re.fullmatch(r"/api/v1/jobs/([0-9a-f]{24})/artifacts/(transcript|draft|result)",path)
|
||||
if m:
|
||||
if not self.need_auth():return
|
||||
return self.artifact(db.get_job(m.group(1)),m.group(2),True)
|
||||
m=re.fullmatch(r"/api/v1/jobs/([0-9a-f]{24})/reviews",path)
|
||||
if m:
|
||||
if not self.need_auth():return
|
||||
job=db.get_job(m.group(1))
|
||||
if not job:return self.json({"error":"not found"},404)
|
||||
rows=[]
|
||||
for row in pending(job):
|
||||
payload=json.loads(row["payload"] or "{}")
|
||||
rows.append({"id":row["id"],"question":row["question"],
|
||||
"kind":payload.get("kind",""),
|
||||
"sentence":payload.get("sentence",""),
|
||||
"candidates":payload.get("candidates") or []})
|
||||
return self.json({"reviews":rows})
|
||||
self.send_data(b"Not found",404,"text/plain")
|
||||
def do_POST(self):
|
||||
path=urllib.parse.urlsplit(self.path).path
|
||||
if path=="/ui/upload":return self.upload("web",False)
|
||||
if path=="/api/v1/jobs":return self.upload("agent",True)
|
||||
m=re.fullmatch(r"/jobs/([0-9a-f]{24})/review/",path)
|
||||
if m:
|
||||
job=db.get_job(m.group(1));length=min(int(self.headers.get("Content-Length","0")),4096);form=urllib.parse.parse_qs(self.rfile.read(length).decode());qid=int(form.get("id",["0"])[0]);answer=(form.get("custom",[""])[0].strip() or form.get("answer",[""])[0].strip())
|
||||
if not job or qid not in {x["id"] for x in pending(job)} or not answer:return self.send_data(b"Invalid answer",400,"text/plain")
|
||||
review.answer(qid,answer[:120]);self.send_response(303);self.send_header("Location",path);self.end_headers();return
|
||||
m=re.fullmatch(r"/(?:api/v1/)?jobs/([0-9a-f]{24})/retry",path)
|
||||
if m:
|
||||
api=path.startswith("/api/")
|
||||
if api and not self.need_auth():return
|
||||
job=db.get_job(m.group(1));source=next((p for p in FAILED.iterdir() if job and p.name==job["stored_name"]),None)
|
||||
if not job or job["status"]!="failed" or not source:return self.json({"error":"cannot retry"},409)
|
||||
source.replace(INBOX/source.name);db.update_job(job["id"],"queued","重新排队",error_code="")
|
||||
if api:return self.json({"ok":True},202)
|
||||
self.send_response(303);self.send_header("Location",f"/jobs/{job['id']}/");self.end_headers();return
|
||||
m=re.fullmatch(r"/api/v1/jobs/([0-9a-f]{24})/reviews/(\d+)",path)
|
||||
if m:
|
||||
if not self.need_auth():return
|
||||
job=db.get_job(m.group(1));length=min(int(self.headers.get("Content-Length","0")),4096)
|
||||
try:answer=str(json.loads(self.rfile.read(length)).get("answer") or "").strip()
|
||||
except (ValueError,AttributeError):answer=""
|
||||
qid=int(m.group(2))
|
||||
if not job or qid not in {x["id"] for x in pending(job)} or not answer:
|
||||
return self.json({"error":"invalid review answer"},400)
|
||||
return self.json(review.answer(qid,answer[:120]))
|
||||
self.send_data(b"Not found",404,"text/plain")
|
||||
|
||||
def main():
|
||||
ensure_dirs();db.connect().close();server=ThreadingHTTPServer(("0.0.0.0",8788),Handler);server.daemon_threads=True;server.serve_forever()
|
||||
if __name__=="__main__":main()
|
||||
@@ -0,0 +1,205 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from asrpipe import refine, review, revise, vault
|
||||
from . import db
|
||||
from .config import (DIARIZE_URL, DRAFTS, FAILED, INBOX, MAX_ATTEMPTS, META,
|
||||
POLL_SECONDS, PROCESSED, PROCESSING, PUBLIC_URL,
|
||||
REVISION_QUEUE, STABLE_SECONDS, STATE, SUPPORTED, TRANSCRIPTS, LOGS,
|
||||
ensure_dirs)
|
||||
from .notify import send
|
||||
|
||||
|
||||
def log(message: str) -> None:
|
||||
ensure_dirs()
|
||||
with (LOGS / "worker.log").open("a", encoding="utf-8") as stream:
|
||||
stream.write(time.strftime("[%Y-%m-%d %H:%M:%S] ") + message + "\n")
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def transcript_from_response(payload: dict) -> str:
|
||||
segments = payload.get("segments")
|
||||
if not isinstance(segments, list) or not segments:
|
||||
raise RuntimeError("ASR response contains no segments")
|
||||
vp = payload.get("voiceprint") or {}
|
||||
sid = str(vp.get("session_id") or "")
|
||||
matched = [f"{m.get('speaker_local')}={m.get('person_name')}" for m in vp.get("matches", [])
|
||||
if m.get("status") == "matched"]
|
||||
speakers = {str(s.get("speaker")) for s in segments if s.get("speaker") is not None}
|
||||
lines = [f"<!-- voiceprint_session: {sid} clusters: {len(speakers)} -->",
|
||||
"# 转写(Qwen3-ASR 1.7B + CAM++ 分角色 + 声纹)"
|
||||
+ (" | 声纹命名: " + ", ".join(matched) if matched else ""), ""]
|
||||
turns: list[list[str]] = []
|
||||
for segment in segments:
|
||||
speaker, text = str(segment.get("speaker", "?")), str(segment.get("text") or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
if turns and turns[-1][0] == speaker:
|
||||
turns[-1][1] += text
|
||||
else:
|
||||
turns.append([speaker, text])
|
||||
lines.extend(f"【{speaker}】{text}\n" for speaker, text in turns)
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def transcribe(audio: Path, job_id: str) -> Path:
|
||||
response_path = TRANSCRIPTS / f"{job_id}.asr.json"
|
||||
result = subprocess.run([
|
||||
"curl", "-sS", "--fail-with-body", "--max-time", "900",
|
||||
"-F", f"file=@{audio}", "-F", "language=Chinese",
|
||||
"-F", "save_voiceprint_session=true", DIARIZE_URL,
|
||||
], capture_output=True, timeout=930)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f"ASR HTTP failed ({result.returncode})")
|
||||
response_path.write_bytes(result.stdout)
|
||||
try:
|
||||
payload = json.loads(result.stdout)
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("ASR returned invalid JSON") from exc
|
||||
transcript = TRANSCRIPTS / f"{job_id}.txt"
|
||||
transcript.write_text(transcript_from_response(payload), encoding="utf-8")
|
||||
return transcript
|
||||
|
||||
|
||||
def process_audio(source: Path, job: dict) -> None:
|
||||
job_id = job["id"]
|
||||
attempts = int(job.get("attempts") or 0) + 1
|
||||
digest = sha256(source)
|
||||
db.update_job(job_id, "transcribing", "正在进行分角色转写", sha256=digest,
|
||||
attempts=attempts, error_code="")
|
||||
transcript = transcribe(source, job_id)
|
||||
intake = {"job_id": job_id, "title": job.get("title", ""),
|
||||
"context": job.get("context", ""), "source": job.get("source", "")}
|
||||
raw = transcript.read_text(encoding="utf-8")
|
||||
raw = f"<!-- intake: {json.dumps(intake, ensure_ascii=False)} -->\n" + raw
|
||||
transcript.write_text(raw, encoding="utf-8")
|
||||
db.update_job(job_id, "refining", "转写完成，正在生成整理稿", transcript=str(transcript))
|
||||
result = refine.refine(raw, date=time.strftime("%Y-%m-%d"),
|
||||
transcript_path=str(transcript), queue_review=True)
|
||||
draft = DRAFTS / f"{job_id}.md"
|
||||
vault.write_draft_file(draft, result.draft)
|
||||
questions = review.enqueue_draft(str(transcript), draft)
|
||||
db.update_job(job_id, "review", f"整理稿已生成，{questions} 个问题待确认",
|
||||
transcript=str(transcript), draft=str(draft))
|
||||
target = PROCESSED / source.name
|
||||
if target.exists():
|
||||
target = PROCESSED / f"{source.stem}-{job_id[:8]}{source.suffix}"
|
||||
shutil.move(str(source), target)
|
||||
send("ASR 草稿待确认", job.get("title") or job["original_name"],
|
||||
f"{PUBLIC_URL}/jobs/{job_id}/review/")
|
||||
|
||||
|
||||
def claim(path: Path) -> tuple[Path, dict]:
|
||||
job = db.find_by_stored(path.name)
|
||||
if not job:
|
||||
job_id = uuid.uuid4().hex[:24]
|
||||
job = db.create_job(job_id, path.name, path.name, title=path.stem,
|
||||
size=path.stat().st_size, source="ssh")
|
||||
target = PROCESSING / path.name
|
||||
if target.exists():
|
||||
target = PROCESSING / f"{path.stem}-{job['id'][:8]}{path.suffix}"
|
||||
path.replace(target)
|
||||
return target, job
|
||||
|
||||
|
||||
def scan_inbox() -> None:
|
||||
now = time.time()
|
||||
for path in sorted(INBOX.iterdir()):
|
||||
if not path.is_file() or path.suffix.lower() not in SUPPORTED:
|
||||
continue
|
||||
if now - path.stat().st_mtime < STABLE_SECONDS:
|
||||
continue
|
||||
source, job = claim(path)
|
||||
try:
|
||||
process_audio(source, job)
|
||||
except Exception as exc:
|
||||
attempts = int((db.get_job(job["id"]) or {}).get("attempts") or 0)
|
||||
retry = attempts < MAX_ATTEMPTS
|
||||
db.update_job(job["id"], "queued" if retry else "failed",
|
||||
"处理失败，等待重试" if retry else "处理失败",
|
||||
error_code=type(exc).__name__)
|
||||
log(f"job={job['id']} failed type={type(exc).__name__}")
|
||||
target = INBOX / source.name if retry else FAILED / source.name
|
||||
if source.exists():
|
||||
shutil.move(str(source), target)
|
||||
|
||||
|
||||
def process_revisions() -> None:
|
||||
for marker in sorted(REVISION_QUEUE.glob("*.json")):
|
||||
try:
|
||||
payload = json.loads(marker.read_text(encoding="utf-8"))
|
||||
draft = Path(payload["draft"])
|
||||
job_id = str(payload.get("job_id") or "")
|
||||
db.update_job(job_id, "publishing", "正在应用人工确认")
|
||||
revise.revise(draft)
|
||||
current = db.get_job(job_id) or {}
|
||||
if current.get("status") == "complete":
|
||||
send("ASR 正式记录已生成", current.get("title") or current.get("original_name", ""),
|
||||
f"{PUBLIC_URL}/jobs/{job_id}/")
|
||||
elif current.get("status") != "review":
|
||||
db.update_job(job_id, "review", "定稿预览已生成，等待发布确认", draft=str(draft))
|
||||
marker.unlink(missing_ok=True)
|
||||
except Exception as exc:
|
||||
log(f"revision marker={marker.name} failed type={type(exc).__name__}")
|
||||
|
||||
|
||||
def recover() -> None:
|
||||
for job in db.pending_recovery():
|
||||
if job["status"] in {"transcribing", "refining"}:
|
||||
candidate = next((p for p in PROCESSING.iterdir() if p.name == job["stored_name"]), None)
|
||||
if candidate:
|
||||
shutil.move(str(candidate), INBOX / candidate.name)
|
||||
db.update_job(job["id"], "queued", "worker 重启后重新排队")
|
||||
|
||||
|
||||
def backup_databases() -> None:
|
||||
# STATE is local disk; durable copies belong on the NFS data volume.
|
||||
backup_dir = Path("/data/backups")
|
||||
backup_dir.mkdir(parents=True, exist_ok=True)
|
||||
for name in ("intake.db", "lexicon.db"):
|
||||
source = STATE / name
|
||||
if not source.exists():
|
||||
continue
|
||||
target = backup_dir / f"{name}.latest"
|
||||
temp = backup_dir / f".{name}.{os.getpid()}.tmp"
|
||||
with sqlite3.connect(source) as src, sqlite3.connect(temp) as dst:
|
||||
src.backup(dst)
|
||||
temp.replace(target)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ensure_dirs()
|
||||
lock = (STATE / "worker.lock").open("w")
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
recover()
|
||||
(STATE / "worker.heartbeat").write_text(str(time.time()))
|
||||
last_backup = 0.0
|
||||
while True:
|
||||
scan_inbox()
|
||||
process_revisions()
|
||||
if time.time() - last_backup >= 3600:
|
||||
backup_databases()
|
||||
last_backup = time.time()
|
||||
(STATE / "worker.heartbeat").write_text(str(time.time()))
|
||||
time.sleep(POLL_SECONDS)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,50 @@
|
||||
services:
|
||||
asr-web:
|
||||
image: tools-asr-pipeline:2026-09-18
|
||||
pull_policy: never
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: tools-asr-web
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
env_file:
|
||||
- /opt/asr-pipeline/.env
|
||||
environment:
|
||||
ASR_ROLE: web
|
||||
ports:
|
||||
- "192.168.50.49:8788:8788"
|
||||
volumes:
|
||||
- /mnt/truenas/personal/ASR-Pipeline:/data
|
||||
- /opt/asr-pipeline/state:/state
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8788/health"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 20s
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
asr-worker:
|
||||
image: tools-asr-pipeline:2026-09-18
|
||||
pull_policy: never
|
||||
container_name: tools-asr-worker
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
env_file:
|
||||
- /opt/asr-pipeline/.env
|
||||
environment:
|
||||
ASR_ROLE: worker
|
||||
command: ["python", "-m", "asrservice.worker"]
|
||||
volumes:
|
||||
- /mnt/truenas/personal/ASR-Pipeline:/data
|
||||
- /opt/asr-pipeline/state:/state
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-m", "asrservice.healthcheck"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 20s
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
@@ -0,0 +1,3 @@
|
||||
pypinyin==0.55.0
|
||||
rapidfuzz==3.14.1
|
||||
numpy==2.3.3
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"suffixes": [
|
||||
"单抗",
|
||||
"珠单抗",
|
||||
"尤单抗",
|
||||
"利珠单抗",
|
||||
"妥珠单抗",
|
||||
"西单抗",
|
||||
"木单抗",
|
||||
"库单抗",
|
||||
"替尼",
|
||||
"布替尼",
|
||||
"沙替尼",
|
||||
"래替尼",
|
||||
"洛替尼",
|
||||
"非替尼",
|
||||
"西普",
|
||||
"普肽",
|
||||
"格鲁肽",
|
||||
"鲁肽",
|
||||
"那肽",
|
||||
"他滨",
|
||||
"他赛",
|
||||
"紫杉醇",
|
||||
"沙星",
|
||||
"霉素",
|
||||
"环素",
|
||||
"西林",
|
||||
"唑胺",
|
||||
"磺胺",
|
||||
"干扰素",
|
||||
"白介素",
|
||||
"促红素",
|
||||
"生长素",
|
||||
"酶",
|
||||
"疫苗",
|
||||
"佐剂"
|
||||
],
|
||||
"kinds": {
|
||||
"单抗": "抗体药",
|
||||
"替尼": "小分子抑制剂",
|
||||
"肽": "多肽药",
|
||||
"疫苗": "疫苗"
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ServiceTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tmp = tempfile.TemporaryDirectory()
|
||||
os.environ["ASR_DATA_DIR"] = str(Path(cls.tmp.name) / "data")
|
||||
os.environ["ASR_STATE_DIR"] = str(Path(cls.tmp.name) / "state")
|
||||
from asrservice import config, db, worker
|
||||
cls.config, cls.db, cls.worker = config, db, worker
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.tmp.cleanup()
|
||||
|
||||
def test_job_events_follow_updates(self):
|
||||
row = self.db.create_job("a" * 24, "x.m4a", "x.m4a", source="ssh")
|
||||
self.assertEqual(row["status"], "queued")
|
||||
self.db.update_job(row["id"], "transcribing", "started", attempts=1)
|
||||
current = self.db.get_job(row["id"])
|
||||
self.assertEqual(current["attempts"], 1)
|
||||
self.assertEqual([e["status"] for e in self.db.events(row["id"])],
|
||||
["queued", "transcribing"])
|
||||
|
||||
def test_transcript_contract(self):
|
||||
text = self.worker.transcript_from_response({
|
||||
"segments": [
|
||||
{"speaker": "speaker_0", "text": "你好。"},
|
||||
{"speaker": "speaker_0", "text": "继续。"},
|
||||
{"speaker": "speaker_1", "text": "收到。"},
|
||||
],
|
||||
"voiceprint": {"session_id": "s1", "matches": []},
|
||||
})
|
||||
self.assertIn("voiceprint_session: s1", text)
|
||||
self.assertIn("【speaker_0】你好。继续。", text)
|
||||
self.assertEqual(text.count("【speaker_0】"), 1)
|
||||
|
||||
def test_transcript_rejects_empty_segments(self):
|
||||
with self.assertRaises(RuntimeError):
|
||||
self.worker.transcript_from_response({"segments": []})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user