feat(tools-pve/asr-pipeline): SQLite model config, task delete/rerefine, multi-recording review, lexicon CRUD
Build image / build (push) Canceled after 0s
Build ASR Pipeline / image (push) Canceled after 0s

This commit is contained in:
Codex
2026-11-01 23:14:31 -08:00
parent 2367c5eb4e
commit be589ff5e0
7 changed files with 2064 additions and 340 deletions
+301 -66
View File
@@ -1,22 +1,31 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Configuration for the standalone ASR post-processing pipeline and web service. """Configuration for the standalone ASR post-processing pipeline and web service.
This program has **no dependency on Claude Code, opencode, or any agent CLI**. Model endpoints, provider selections, and API keys are stored in SQLite
It talks to LiteLLM / model providers over plain HTTPS and does its own file (`app_setting` table in `ASR_STATE_DIR/intake.db`) so they can be managed
and git handling, so it can run inside Docker (via Arcane), systemd, or a shell. dynamically from the Web UI without restarting containers. Environment variables
serve as initial bootstrap defaults when a setting is not yet in SQLite.
Everything is overridable by environment variable.
""" """
from __future__ import annotations from __future__ import annotations
import os import os
import sqlite3
import threading
import time
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
from typing import Any, Callable
PKG = Path(__file__).resolve().parent PKG = Path(__file__).resolve().parent
ROOT = PKG.parent ROOT = PKG.parent
OBSIDIAN = Path(os.environ.get("ASR_WORKSPACE", str(ROOT))) OBSIDIAN = Path(os.environ.get("ASR_WORKSPACE", str(ROOT)))
_CACHE_LOCK = threading.Lock()
_SETTINGS_CACHE: dict[str, str] = {}
_SETTINGS_TS: float = 0.0
_SETTINGS_DB_PATH: Path | None = None
_CACHE_TTL = 1.0
def _env(name: str, default: str) -> str: def _env(name: str, default: str) -> str:
v = os.environ.get(name) v = os.environ.get(name)
@@ -37,6 +46,165 @@ def _envi(name: str, default: int) -> int:
return default return default
def _state_db_path() -> Path:
state_dir = Path(_env("ASR_STATE_DIR", str(ROOT / "state")))
return state_dir / "intake.db"
def _ensure_settings_table(con: sqlite3.Connection) -> None:
con.execute("""
CREATE TABLE IF NOT EXISTS app_setting(
key TEXT PRIMARY KEY,
value TEXT NOT NULL DEFAULT '',
updated_at REAL NOT NULL
)
""")
def load_db_settings(force: bool = False) -> dict[str, str]:
"""Read `app_setting` rows from `intake.db` with a 1s in-memory cache."""
global _SETTINGS_CACHE, _SETTINGS_TS, _SETTINGS_DB_PATH
db_path = _state_db_path()
now = time.time()
with _CACHE_LOCK:
if (
not force
and _SETTINGS_DB_PATH == db_path
and (now - _SETTINGS_TS) < _CACHE_TTL
):
return dict(_SETTINGS_CACHE)
if not db_path.exists():
_SETTINGS_CACHE = {}
_SETTINGS_TS = now
_SETTINGS_DB_PATH = db_path
return {}
try:
con = sqlite3.connect(db_path, timeout=5)
con.row_factory = sqlite3.Row
_ensure_settings_table(con)
rows = con.execute("SELECT key, value FROM app_setting").fetchall()
con.close()
_SETTINGS_CACHE = {str(r["key"]): str(r["value"]) for r in rows}
_SETTINGS_TS = now
_SETTINGS_DB_PATH = db_path
return dict(_SETTINGS_CACHE)
except Exception:
return dict(_SETTINGS_CACHE)
def get_setting(key: str, default: str = "") -> str:
data = load_db_settings()
val = data.get(key)
return val if val not in (None, "") else default
def set_settings(updates: dict[str, Any]) -> None:
"""Persist multiple settings into `intake.db` and invalidate cache."""
global _SETTINGS_TS
db_path = _state_db_path()
db_path.parent.mkdir(parents=True, exist_ok=True)
now = time.time()
con = sqlite3.connect(db_path, timeout=15)
try:
con.execute("PRAGMA journal_mode=WAL")
_ensure_settings_table(con)
for k, v in updates.items():
if v is None:
continue
con.execute(
"INSERT INTO app_setting(key, value, updated_at) VALUES(?,?,?) "
"ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at",
(str(k), str(v), now),
)
con.commit()
finally:
con.close()
with _CACHE_LOCK:
_SETTINGS_TS = 0.0
if "CFG" in globals():
for k in updates:
CFG.__dict__.pop(f"_override_{k}", None)
load_db_settings(force=True)
def bootstrap_default_settings() -> None:
"""Seed `app_setting` from environment variables for keys not yet stored."""
db_path = _state_db_path()
db_path.parent.mkdir(parents=True, exist_ok=True)
now = time.time()
defaults = {
"asr_provider": _env("ASR_PROVIDER", _env("MEETING_ASR_BACKEND", "qwen3cam")).lower(),
"diarize_url": _env("ASR_DIARIZE_URL", _env("MEETING_DIARIZE_URL", "http://192.168.50.100:10099/asr")),
"voiceprint_enroll_url": _env("ASR_VOICEPRINT_ENROLL_URL", "http://192.168.50.100:10099/voiceprints/enroll"),
"litellm_base_url": _env("LITELLM_BASE_URL", _env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")).rstrip("/"),
"litellm_api_key": _env("LITELLM_API_KEY", _env("ASR_LLM_API_KEY", _env("OPENAI_API_KEY", ""))),
"clean_model": _env("ASR_CLEAN_MODEL", "aistudio/gemini-flash-lite-latest"),
"keyword_model": _env("ASR_KEYWORD_MODEL", "aistudio/gemini-flash-lite-latest"),
"draft_model": _env("ASR_DRAFT_MODEL", "codex/gpt-sol-latest"),
"gemini_asr_model": _env("ASR_GEMINI_MODEL", "aistudio/gemini-3.5-transcribe"),
"openai_asr_model": _env("ASR_OPENAI_MODEL", "openai/gpt-4o-transcribe-diarize"),
"soniox_api_base": _env("SONIOX_API_BASE", "https://api.soniox.com/v1").rstrip("/"),
"soniox_api_key": _env("SONIOX_API_KEY", ""),
"soniox_model": _env("SONIOX_MODEL", "stt-async-v5"),
"grok_api_base": _env("XAI_API_BASE", "https://api.x.ai/v1").rstrip("/"),
"grok_api_key": _env("XAI_API_KEY", _env("GROK_API_KEY", "")),
"grok_asr_model": _env("ASR_GROK_MODEL", "grok-voice-transcribe-2.0"),
"hindsight_api": _env("HINDSIGHT_API", "http://192.168.50.47:8888").rstrip("/"),
"hindsight_bank": _env("HINDSIGHT_BANK", "personal-context"),
"use_hindsight": "1" if _env("ASR_HINDSIGHT", "1") != "0" else "0",
"use_termbank": "1" if _env("ASR_TERMBANK", "1") != "0" else "0",
"git_sync": "1" if _env("ASR_GIT_SYNC", "0") != "0" else "0",
}
con = sqlite3.connect(db_path, timeout=15)
try:
con.execute("PRAGMA journal_mode=WAL")
_ensure_settings_table(con)
for k, v in defaults.items():
con.execute(
"INSERT OR IGNORE INTO app_setting(key, value, updated_at) VALUES(?,?,?)",
(k, str(v), now),
)
# If key was seeded empty earlier but env now provides a non-empty secret, populate it
if v and k.endswith("_api_key"):
con.execute(
"UPDATE app_setting SET value=?, updated_at=? WHERE key=? AND value=''",
(str(v), now, k),
)
con.commit()
finally:
con.close()
load_db_settings(force=True)
class _SettingAttr:
"""Descriptor that resolves from instance override -> SQLite `app_setting` -> env fallback."""
def __init__(
self,
key: str,
fallback: Callable[[], str],
transform: Callable[[str], Any] | None = None,
) -> None:
self.key = key
self.fallback = fallback
self.transform = transform
self.override_attr = f"_override_{key}"
def __get__(self, instance: Any, owner: Any = None) -> Any:
if instance is None:
return self
if self.override_attr in instance.__dict__:
return instance.__dict__[self.override_attr]
raw = get_setting(self.key, self.fallback())
return self.transform(raw) if self.transform else raw
def __set__(self, instance: Any, value: Any) -> None:
instance.__dict__[self.override_attr] = value
def __delete__(self, instance: Any) -> None:
instance.__dict__.pop(self.override_attr, None)
@dataclass @dataclass
class Config: class Config:
# ---- paths ---------------------------------------------------------- # ---- paths ----------------------------------------------------------
@@ -46,61 +214,130 @@ class Config:
_env("ASR_WORKSPACE", str(ROOT)))) _env("ASR_WORKSPACE", str(ROOT))))
root: Path = ROOT root: Path = ROOT
# ---- LiteLLM / LLM routing (ZenMux deprecated) ---------------------- # ---- SQLite-backed dynamic model & provider settings ----------------
litellm_base_url: str = field(default_factory=lambda: _env( litellm_base_url = _SettingAttr(
"LITELLM_BASE_URL", "litellm_base_url",
_env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")).rstrip("/")) lambda: _env("LITELLM_BASE_URL", _env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")),
lambda s: s.rstrip("/"),
)
clean_model = _SettingAttr(
"clean_model",
lambda: _env("ASR_CLEAN_MODEL", "aistudio/gemini-flash-lite-latest"),
)
draft_model = _SettingAttr(
"draft_model",
lambda: _env("ASR_DRAFT_MODEL", "codex/gpt-sol-latest"),
)
keyword_model = _SettingAttr(
"keyword_model",
lambda: _env("ASR_KEYWORD_MODEL", "aistudio/gemini-flash-lite-latest"),
)
clean_model: str = field(default_factory=lambda: _env( asr_provider = _SettingAttr(
"ASR_CLEAN_MODEL", "aistudio/gemini-flash-lite-latest")) "asr_provider",
draft_model: str = field(default_factory=lambda: _env( lambda: _env("ASR_PROVIDER", _env("MEETING_ASR_BACKEND", "qwen3cam")),
"ASR_DRAFT_MODEL", "codex/gpt-sol-latest")) lambda s: s.strip().lower() or "qwen3cam",
keyword_model: str = field(default_factory=lambda: _env( )
"ASR_KEYWORD_MODEL", "aistudio/gemini-flash-lite-latest")) diarize_url = _SettingAttr(
"diarize_url",
lambda: _env("ASR_DIARIZE_URL", _env("MEETING_DIARIZE_URL", "http://192.168.50.100:10099/asr")),
)
voiceprint_enroll_url = _SettingAttr(
"voiceprint_enroll_url",
lambda: _env("ASR_VOICEPRINT_ENROLL_URL", "http://192.168.50.100:10099/voiceprints/enroll"),
)
endpoint: str = field(default_factory=lambda: _env( soniox_api_key = _SettingAttr(
"ASR_ENDPOINT", "soniox_api_key",
_env("LITELLM_BASE_URL", lambda: _env("SONIOX_API_KEY", ""),
_env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")).rstrip("/") )
+ "/chat/completions")) soniox_api_base = _SettingAttr(
transcriptions_endpoint: str = field(default_factory=lambda: _env( "soniox_api_base",
"ASR_TRANSCRIPTIONS_ENDPOINT", lambda: _env("SONIOX_API_BASE", "https://api.soniox.com/v1"),
_env("LITELLM_BASE_URL", lambda s: s.rstrip("/"),
_env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")).rstrip("/") )
+ "/audio/transcriptions")) soniox_model = _SettingAttr(
"soniox_model",
lambda: _env("SONIOX_MODEL", "stt-async-v5"),
)
gemini_asr_model = _SettingAttr(
"gemini_asr_model",
lambda: _env("ASR_GEMINI_MODEL", "aistudio/gemini-3.5-transcribe"),
)
openai_asr_model = _SettingAttr(
"openai_asr_model",
lambda: _env("ASR_OPENAI_MODEL", "openai/gpt-4o-transcribe-diarize"),
)
grok_api_key = _SettingAttr(
"grok_api_key",
lambda: _env("XAI_API_KEY", _env("GROK_API_KEY", "")),
)
grok_api_base = _SettingAttr(
"grok_api_base",
lambda: _env("XAI_API_BASE", "https://api.x.ai/v1"),
lambda s: s.rstrip("/"),
)
grok_asr_model = _SettingAttr(
"grok_asr_model",
lambda: _env("ASR_GROK_MODEL", "grok-voice-transcribe-2.0"),
)
hindsight_api = _SettingAttr(
"hindsight_api",
lambda: _env("HINDSIGHT_API", "http://192.168.50.47:8888"),
lambda s: s.rstrip("/"),
)
hindsight_bank = _SettingAttr(
"hindsight_bank",
lambda: _env("HINDSIGHT_BANK", "personal-context"),
)
use_hindsight = _SettingAttr(
"use_hindsight",
lambda: _env("ASR_HINDSIGHT", "1"),
lambda s: str(s).lower() not in ("0", "false", "no", "off"),
)
use_termbank = _SettingAttr(
"use_termbank",
lambda: _env("ASR_TERMBANK", "1"),
lambda s: str(s).lower() not in ("0", "false", "no", "off"),
)
git_sync = _SettingAttr(
"git_sync",
lambda: _env("ASR_GIT_SYNC", "1"),
lambda s: str(s).lower() not in ("0", "false", "no", "off"),
)
@property
def endpoint(self) -> str:
if "_override_endpoint" in self.__dict__:
return self.__dict__["_override_endpoint"]
explicit = os.environ.get("ASR_ENDPOINT")
if explicit:
return explicit
return f"{self.litellm_base_url}/chat/completions"
@endpoint.setter
def endpoint(self, value: str) -> None:
self.__dict__["_override_endpoint"] = value
@property
def transcriptions_endpoint(self) -> str:
if "_override_transcriptions_endpoint" in self.__dict__:
return self.__dict__["_override_transcriptions_endpoint"]
explicit = os.environ.get("ASR_TRANSCRIPTIONS_ENDPOINT")
if explicit:
return explicit
return f"{self.litellm_base_url}/audio/transcriptions"
@transcriptions_endpoint.setter
def transcriptions_endpoint(self, value: str) -> None:
self.__dict__["_override_transcriptions_endpoint"] = value
concurrency: int = field(default_factory=lambda: _envi("CONCURRENCY", 6)) concurrency: int = field(default_factory=lambda: _envi("CONCURRENCY", 6))
http_timeout: float = field(default_factory=lambda: _envf("HTTP_TIMEOUT", 420.0)) http_timeout: float = field(default_factory=lambda: _envf("HTTP_TIMEOUT", 420.0))
max_retries: int = field(default_factory=lambda: _envi("MAX_RETRIES", 5)) max_retries: int = field(default_factory=lambda: _envi("MAX_RETRIES", 5))
# ---- ASR providers --------------------------------------------------
asr_provider: str = field(default_factory=lambda: _env(
"ASR_PROVIDER", _env("MEETING_ASR_BACKEND", "qwen3cam")).lower())
diarize_url: str = field(default_factory=lambda: _env(
"ASR_DIARIZE_URL",
_env("MEETING_DIARIZE_URL", "http://192.168.50.100:10099/asr")))
voiceprint_enroll_url: str = field(default_factory=lambda: _env(
"ASR_VOICEPRINT_ENROLL_URL",
"http://192.168.50.100:10099/voiceprints/enroll"))
soniox_api_key: str = field(default_factory=lambda: _env("SONIOX_API_KEY", ""))
soniox_api_base: str = field(default_factory=lambda: _env(
"SONIOX_API_BASE", "https://api.soniox.com/v1").rstrip("/"))
soniox_model: str = field(default_factory=lambda: _env(
"SONIOX_MODEL", "stt-async-v5"))
gemini_asr_model: str = field(default_factory=lambda: _env(
"ASR_GEMINI_MODEL", "aistudio/gemini-3.5-transcribe"))
openai_asr_model: str = field(default_factory=lambda: _env(
"ASR_OPENAI_MODEL", "openai/gpt-4o-transcribe-diarize"))
grok_api_key: str = field(default_factory=lambda: _env(
"XAI_API_KEY", _env("GROK_API_KEY", "")))
grok_api_base: str = field(default_factory=lambda: _env(
"XAI_API_BASE", "https://api.x.ai/v1").rstrip("/"))
grok_asr_model: str = field(default_factory=lambda: _env(
"ASR_GROK_MODEL", "grok-voice-transcribe-2.0"))
segment_seconds: int = field(default_factory=lambda: _envi( segment_seconds: int = field(default_factory=lambda: _envi(
"SEGMENT_SECONDS", int(_env("MEETING_ASR_SEGMENT_SECONDS", "540")))) "SEGMENT_SECONDS", int(_env("MEETING_ASR_SEGMENT_SECONDS", "540"))))
@@ -119,10 +356,6 @@ class Config:
default_factory=lambda: _envf("GATE_MAX_RATIO", 1.35)) default_factory=lambda: _envf("GATE_MAX_RATIO", 1.35))
# ---- knowledge sources ---------------------------------------------- # ---- knowledge sources ----------------------------------------------
use_termbank: bool = field(
default_factory=lambda: _env("ASR_TERMBANK", "1") != "0")
use_hindsight: bool = field(
default_factory=lambda: _env("ASR_HINDSIGHT", "1") != "0")
use_keyword_recall: bool = field( use_keyword_recall: bool = field(
default_factory=lambda: _env("ASR_KEYWORD_RECALL", "1") != "0") default_factory=lambda: _env("ASR_KEYWORD_RECALL", "1") != "0")
use_lexicon: bool = field( use_lexicon: bool = field(
@@ -131,10 +364,6 @@ class Config:
default_factory=lambda: _env("ASR_NORMALIZE", "1") != "0") default_factory=lambda: _env("ASR_NORMALIZE", "1") != "0")
use_evidence: bool = field( use_evidence: bool = field(
default_factory=lambda: _env("ASR_EVIDENCE", "1") != "0") default_factory=lambda: _env("ASR_EVIDENCE", "1") != "0")
hindsight_api: str = field(default_factory=lambda: _env(
"HINDSIGHT_API", "http://192.168.50.47:8888"))
hindsight_bank: str = field(default_factory=lambda: _env(
"HINDSIGHT_BANK", "personal-context"))
hindsight_timeout: float = field( hindsight_timeout: float = field(
default_factory=lambda: _envf("HINDSIGHT_TIMEOUT", 12.0)) default_factory=lambda: _envf("HINDSIGHT_TIMEOUT", 12.0))
flag_min_score: float = field( flag_min_score: float = field(
@@ -154,13 +383,17 @@ class Config:
default_factory=lambda: _envi("REVIEW_MAX_SECONDS", 7200)) default_factory=lambda: _envi("REVIEW_MAX_SECONDS", 7200))
# ---- behaviour ------------------------------------------------------ # ---- behaviour ------------------------------------------------------
git_sync: bool = field(
default_factory=lambda: _env("ASR_GIT_SYNC", "1") != "0")
notify: bool = field( notify: bool = field(
default_factory=lambda: _env("ASR_NOTIFY", "1") != "0") default_factory=lambda: _env("ASR_NOTIFY", "1") != "0")
cache_llm: bool = field( cache_llm: bool = field(
default_factory=lambda: _env("ASR_CACHE", "0") != "0") default_factory=lambda: _env("ASR_CACHE", "0") != "0")
def __post_init__(self) -> None:
# Clear dataclass-populated default descriptors so __get__ is invoked dynamically
for k in list(self.__dict__.keys()):
if isinstance(getattr(type(self), k, None), _SettingAttr):
del self.__dict__[k]
@property @property
def inbox(self) -> Path: def inbox(self) -> Path:
return self.vault / ".asr-inbox" return self.vault / ".asr-inbox"
@@ -217,6 +450,9 @@ class Config:
return Path(_env("ASR_ENV_FILE", str(ROOT / ".env"))) return Path(_env("ASR_ENV_FILE", str(ROOT / ".env")))
def api_key(self) -> str: def api_key(self) -> str:
db_key = get_setting("litellm_api_key", "")
if db_key:
return db_key
for k in ("LITELLM_API_KEY", "ASR_LLM_API_KEY", "OPENAI_API_KEY", for k in ("LITELLM_API_KEY", "ASR_LLM_API_KEY", "OPENAI_API_KEY",
"ANTHROPIC_AUTH_TOKEN"): "ANTHROPIC_AUTH_TOKEN"):
v = os.environ.get(k) v = os.environ.get(k)
@@ -233,8 +469,7 @@ class Config:
if val: if val:
return val return val
raise RuntimeError( raise RuntimeError(
"no API key: set LITELLM_API_KEY / ASR_LLM_API_KEY / OPENAI_API_KEY " "no API key: configure LiteLLM API Key in /settings or set LITELLM_API_KEY")
f"or provide {p}")
CFG = Config() CFG = Config()
@@ -298,7 +298,7 @@ def sync_from_vault(force: bool = False) -> dict:
now = time.time() now = time.time()
n_ent = n_alias = 0 n_ent = n_alias = 0
con.execute("DELETE FROM alias WHERE origin IN ('card','glossary')") con.execute("DELETE FROM alias WHERE origin IN ('card','glossary')")
con.execute("DELETE FROM entity") con.execute("DELETE FROM entity WHERE card <> ''")
con.execute("DELETE FROM relation WHERE source LIKE 'card:%'") con.execute("DELETE FROM relation WHERE source LIKE 'card:%'")
for sub, kind in (("People", "人名"), ("Companies", "公司"), ("Projects", "项目")): for sub, kind in (("People", "人名"), ("Companies", "公司"), ("Projects", "项目")):
d = CFG.vault / sub d = CFG.vault / sub
@@ -524,3 +524,261 @@ def stats() -> dict:
d["synced_at"] = r["v"] if r else "" d["synced_at"] = r["v"] if r else ""
con.close() con.close()
return d return d
# ------------------------------------------------------------------ Web UI CRUD
def list_aliases(q: str = "", origin_filter: str = "", limit: int = 200) -> list[dict]:
con = connect()
sql = "SELECT * FROM alias WHERE 1=1"
args: list = []
if q.strip():
pat = f"%{q.strip()}%"
sql += " AND (alias LIKE ? OR canonical LIKE ? OR note LIKE ?)"
args.extend([pat, pat, pat])
if origin_filter == "learned":
sql += " AND origin <> 'card'"
elif origin_filter == "card":
sql += " AND origin = 'card'"
sql += " ORDER BY confirmed DESC, created_at DESC, alias ASC LIMIT ?"
args.append(limit)
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
con.close()
return rows
def update_alias(
old_alias: str,
old_canonical: str,
new_alias: str,
new_canonical: str,
*,
kind: str = "",
confirmed: int = 1,
note: str = "",
) -> bool:
con = connect()
row = con.execute(
"SELECT * FROM alias WHERE alias=? AND canonical=?",
(old_alias, old_canonical),
).fetchone()
origin = row["origin"] if row else "manual"
hits = row["hits"] if row else 0
created_at = row["created_at"] if row else time.time()
if not kind:
er = con.execute("SELECT kind FROM entity WHERE canonical=?", (new_canonical,)).fetchone()
kind = er["kind"] if er else (row["kind"] if row else guess_kind(new_canonical))
con.execute(
"DELETE FROM alias WHERE alias=? AND canonical=?",
(old_alias, old_canonical),
)
con.execute(
"INSERT OR REPLACE INTO alias(alias,canonical,kind,origin,confirmed,hits,created_at,note) "
"VALUES(?,?,?,?,?,?,?,?)",
(new_alias, new_canonical, kind, origin, int(bool(confirmed)), hits, created_at, note),
)
con.commit()
con.close()
return True
def delete_alias(alias: str, canonical: str) -> bool:
con = connect()
cur = con.execute("DELETE FROM alias WHERE alias=? AND canonical=?", (alias, canonical))
con.commit()
deleted = cur.rowcount > 0
con.close()
return deleted
def list_corrections(q: str = "", limit: int = 200) -> list[dict]:
con = connect()
sql = "SELECT * FROM correction WHERE 1=1"
args: list = []
if q.strip():
pat = f"%{q.strip()}%"
sql += " AND (surface LIKE ? OR canonical LIKE ? OR note LIKE ?)"
args.extend([pat, pat, pat])
sql += " ORDER BY confirmed DESC, n DESC, last_seen DESC LIMIT ?"
args.append(limit)
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
con.close()
return rows
def update_correction(
old_surface: str,
old_canonical: str,
new_surface: str,
new_canonical: str,
*,
confirmed: int = 1,
note: str = "",
) -> bool:
con = connect()
row = con.execute(
"SELECT * FROM correction WHERE surface=? AND canonical=?",
(old_surface, old_canonical),
).fetchone()
n = row["n"] if row else 1
now = time.time()
con.execute("DELETE FROM correction WHERE surface=? AND canonical=?", (old_surface, old_canonical))
con.execute(
"DELETE FROM phonetic_confusion WHERE heard=? AND intended=?",
(old_surface, old_canonical),
)
con.execute(
"INSERT OR REPLACE INTO correction(surface,canonical,n,confirmed,last_seen,note) "
"VALUES(?,?,?,?,?,?)",
(new_surface, new_canonical, n, int(bool(confirmed)), now, note),
)
con.execute(
"INSERT OR REPLACE INTO phonetic_confusion(heard,intended,n,confirmed,last_seen,note) "
"VALUES(?,?,?,?,?,?)",
(new_surface, new_canonical, n, int(bool(confirmed)), now, note),
)
con.commit()
con.close()
return True
def delete_correction(surface: str, canonical: str) -> bool:
con = connect()
cur = con.execute("DELETE FROM correction WHERE surface=? AND canonical=?", (surface, canonical))
con.execute("DELETE FROM phonetic_confusion WHERE heard=? AND intended=?", (surface, canonical))
con.commit()
deleted = cur.rowcount > 0
con.close()
return deleted
def list_entities(q: str = "", kind: str = "", limit: int = 200) -> list[dict]:
con = connect()
sql = "SELECT * FROM entity WHERE 1=1"
args: list = []
if q.strip():
pat = f"%{q.strip()}%"
sql += " AND (canonical LIKE ? OR company LIKE ? OR role LIKE ? OR related LIKE ?)"
args.extend([pat, pat, pat, pat])
if kind.strip():
sql += " AND kind=?"
args.append(kind.strip())
sql += " ORDER BY (CASE WHEN card='' THEN 0 ELSE 1 END), canonical ASC LIMIT ?"
args.append(limit)
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
con.close()
return rows
def upsert_entity(
canonical: str,
kind: str = "人名",
*,
company: str = "",
role: str = "",
related: str = "",
card: str = "",
) -> None:
con = connect()
con.execute(
"INSERT OR REPLACE INTO entity(canonical,kind,company,role,related,card,synced_at) "
"VALUES(?,?,?,?,?,?,?)",
(canonical, kind or guess_kind(canonical), company, role, related, card, time.time()),
)
con.commit()
con.close()
def update_entity(
old_canonical: str,
new_canonical: str,
kind: str,
*,
company: str = "",
role: str = "",
related: str = "",
) -> bool:
con = connect()
row = con.execute("SELECT * FROM entity WHERE canonical=?", (old_canonical,)).fetchone()
card = row["card"] if row else ""
con.execute("DELETE FROM entity WHERE canonical=?", (old_canonical,))
con.execute(
"INSERT OR REPLACE INTO entity(canonical,kind,company,role,related,card,synced_at) "
"VALUES(?,?,?,?,?,?,?)",
(new_canonical, kind or guess_kind(new_canonical), company, role, related, card, time.time()),
)
if old_canonical != new_canonical:
con.execute("UPDATE alias SET canonical=? WHERE canonical=?", (new_canonical, old_canonical))
con.execute("UPDATE correction SET canonical=? WHERE canonical=?", (new_canonical, old_canonical))
con.commit()
con.close()
return True
def delete_entity(canonical: str) -> bool:
con = connect()
cur = con.execute("DELETE FROM entity WHERE canonical=?", (canonical,))
con.commit()
deleted = cur.rowcount > 0
con.close()
return deleted
def list_relations(q: str = "", limit: int = 200) -> list[dict]:
con = connect()
sql = "SELECT * FROM relation WHERE 1=1"
args: list = []
if q.strip():
pat = f"%{q.strip()}%"
sql += " AND (subject LIKE ? OR predicate LIKE ? OR object LIKE ? OR source LIKE ?)"
args.extend([pat, pat, pat, pat])
sql += " ORDER BY active DESC, confidence DESC, last_seen DESC LIMIT ?"
args.append(limit)
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
con.close()
return rows
def update_relation(
old_subject: str,
old_predicate: str,
old_object: str,
old_source: str,
subject: str,
predicate: str,
object_: str,
*,
confidence: float = 0.8,
active: int = 1,
) -> bool:
con = connect()
row = con.execute(
"SELECT * FROM relation WHERE subject=? AND predicate=? AND object=? AND source=?",
(old_subject, old_predicate, old_object, old_source),
).fetchone()
first_seen = row["first_seen"] if row else time.time()
source = old_source or "manual"
con.execute(
"DELETE FROM relation WHERE subject=? AND predicate=? AND object=? AND source=?",
(old_subject, old_predicate, old_object, old_source),
)
con.execute(
"INSERT OR REPLACE INTO relation(subject,predicate,object,confidence,source,first_seen,last_seen,active) "
"VALUES(?,?,?,?,?,?,?,?)",
(subject, predicate, object_, float(confidence), source, first_seen, time.time(), int(bool(active))),
)
con.commit()
con.close()
return True
def delete_relation(subject: str, predicate: str, object_: str, source: str) -> bool:
con = connect()
cur = con.execute(
"DELETE FROM relation WHERE subject=? AND predicate=? AND object=? AND source=?",
(subject, predicate, object_, source),
)
con.commit()
deleted = cur.rowcount > 0
con.close()
return deleted
@@ -307,14 +307,14 @@ def _request_revision(draft_path: str) -> bool:
return False return False
def answer(question_id: int, value: str) -> dict: def answer(question_id: int, value: str, *, allow_update: bool = False) -> dict:
con = lexicon.connect() con = lexicon.connect()
row = con.execute("SELECT * FROM review_question WHERE id=?", row = con.execute("SELECT * FROM review_question WHERE id=?",
(question_id,)).fetchone() (question_id,)).fetchone()
if not row: if not row:
con.close() con.close()
raise KeyError(f"review question {question_id} not found") raise KeyError(f"review question {question_id} not found")
if row["status"] != "pending": if row["status"] != "pending" and not allow_update:
result = { result = {
"id": question_id, "answer": row["answer"], "learned": False, "id": question_id, "answer": row["answer"], "learned": False,
"already_answered": True, "already_answered": True,
@@ -376,3 +376,47 @@ def answer(question_id: int, value: str) -> dict:
"draft_feedback_saved": saved, "voiceprint_enrolled": voiceprint_enrolled, "draft_feedback_saved": saved, "voiceprint_enrolled": voiceprint_enrolled,
"revision_started": revision_started, "revision_started": revision_started,
} }
def recent_answered(limit: int = 30) -> list[dict]:
con = lexicon.connect()
rows = con.execute(
"SELECT * FROM review_question WHERE status='answered' "
"ORDER BY answered_at DESC, id DESC LIMIT ?",
(limit,),
).fetchall()
con.close()
return [dict(r) for r in rows]
def reopen_question(question_id: int) -> bool:
con = lexicon.connect()
cur = con.execute(
"UPDATE review_question SET status='pending', answer='', answered_at=NULL WHERE id=?",
(question_id,),
)
con.commit()
ok = cur.rowcount > 0
con.close()
return ok
def delete_for_job(transcript_path: str = "", draft_path: str = "", job_id: str = "") -> int:
con = lexicon.connect()
rows = con.execute("SELECT id, recording, payload FROM review_question").fetchall()
to_delete: list[int] = []
for r in rows:
payload = json.loads(r["payload"] or "{}")
if (
(transcript_path and r["recording"] == transcript_path)
or (draft_path and payload.get("draft_path") == draft_path)
or (job_id and payload.get("job_id") == job_id)
):
to_delete.append(int(r["id"]))
if to_delete:
marks = ",".join("?" for _ in to_delete)
con.execute(f"DELETE FROM review_question WHERE id IN ({marks})", to_delete)
con.commit()
con.close()
return len(to_delete)
+104 -7
View File
@@ -4,15 +4,19 @@ import sqlite3
import time import time
from typing import Any from typing import Any
from .config import DB, DEFAULT_PROVIDER, ensure_dirs from asrpipe.config import CFG, bootstrap_default_settings, get_setting, load_db_settings, set_settings
from .config import DB, ensure_dirs
STATUSES = { STATUSES = {
"uploading", "queued", "transcribing", "refining", "review", "uploading", "queued", "transcribing", "refining", "review",
"publishing", "complete", "failed", "publishing", "complete", "failed",
} }
_BOOTSTRAPPED_DB: str = ""
def connect() -> sqlite3.Connection: def connect() -> sqlite3.Connection:
global _BOOTSTRAPPED_DB
ensure_dirs() ensure_dirs()
con = sqlite3.connect(DB, timeout=30) con = sqlite3.connect(DB, timeout=30)
con.row_factory = sqlite3.Row con.row_factory = sqlite3.Row
@@ -37,8 +41,14 @@ def connect() -> sqlite3.Connection:
status TEXT NOT NULL, detail TEXT NOT NULL DEFAULT '', created_at REAL NOT NULL status TEXT NOT NULL, detail TEXT NOT NULL DEFAULT '', created_at REAL NOT NULL
); );
CREATE INDEX IF NOT EXISTS ix_job_event_job ON job_event(job_id,id); CREATE INDEX IF NOT EXISTS ix_job_event_job ON job_event(job_id,id);
CREATE TABLE IF NOT EXISTS app_setting(
key TEXT PRIMARY KEY,
value TEXT NOT NULL DEFAULT '',
updated_at REAL NOT NULL
);
""") """)
cols = {r[1] for r in con.execute("PRAGMA table_info(intake_job)")} cols = {r[1] for r in con.execute("PRAGMA table_info(intake_job)")}
default_prov = CFG.asr_provider or "qwen3cam"
additions = { additions = {
"draft": "TEXT NOT NULL DEFAULT ''", "draft": "TEXT NOT NULL DEFAULT ''",
"result": "TEXT NOT NULL DEFAULT ''", "result": "TEXT NOT NULL DEFAULT ''",
@@ -46,13 +56,19 @@ def connect() -> sqlite3.Connection:
"sha256": "TEXT NOT NULL DEFAULT ''", "sha256": "TEXT NOT NULL DEFAULT ''",
"attempts": "INTEGER NOT NULL DEFAULT 0", "attempts": "INTEGER NOT NULL DEFAULT 0",
"error_code": "TEXT NOT NULL DEFAULT ''", "error_code": "TEXT NOT NULL DEFAULT ''",
"asr_provider": f"TEXT NOT NULL DEFAULT '{DEFAULT_PROVIDER}'", "asr_provider": f"TEXT NOT NULL DEFAULT '{default_prov}'",
"asr_model": "TEXT NOT NULL DEFAULT ''", "asr_model": "TEXT NOT NULL DEFAULT ''",
} }
for name, ddl in additions.items(): for name, ddl in additions.items():
if name not in cols: if name not in cols:
con.execute(f"ALTER TABLE intake_job ADD COLUMN {name} {ddl}") con.execute(f"ALTER TABLE intake_job ADD COLUMN {name} {ddl}")
con.commit() con.commit()
if _BOOTSTRAPPED_DB != str(DB):
_BOOTSTRAPPED_DB = str(DB)
try:
bootstrap_default_settings()
except Exception:
pass
return con return con
@@ -69,7 +85,7 @@ def create_job(
asr_model: str = "", asr_model: str = "",
) -> dict[str, Any]: ) -> dict[str, Any]:
now = time.time() now = time.time()
prov = (asr_provider or DEFAULT_PROVIDER or "qwen3cam").strip().lower() prov = (asr_provider or CFG.asr_provider or "qwen3cam").strip().lower()
with connect() as con: with connect() as con:
con.execute( con.execute(
"INSERT INTO intake_job(id,original_name,stored_name,title,context,size," "INSERT INTO intake_job(id,original_name,stored_name,title,context,size,"
@@ -88,7 +104,7 @@ def update_job(job_id: str, status: str, detail: str = "", **fields: Any) -> boo
raise ValueError(f"invalid status: {status}") raise ValueError(f"invalid status: {status}")
allowed = { allowed = {
"transcript", "draft", "result", "sha256", "attempts", "transcript", "draft", "result", "sha256", "attempts",
"error_code", "asr_provider", "asr_model", "error_code", "asr_provider", "asr_model", "title", "context",
} }
changes = {k: v for k, v in fields.items() if k in allowed and v is not None} changes = {k: v for k, v in fields.items() if k in allowed and v is not None}
sets = ["status=?", "updated_at=?", "detail=?"] sets = ["status=?", "updated_at=?", "detail=?"]
@@ -106,6 +122,44 @@ def update_job(job_id: str, status: str, detail: str = "", **fields: Any) -> boo
return bool(cur.rowcount) return bool(cur.rowcount)
def update_job_meta(
job_id: str,
*,
title: str | None = None,
context: str | None = None,
asr_provider: str | None = None,
asr_model: str | None = None,
) -> bool:
sets = ["updated_at=?"]
values: list[Any] = [time.time()]
if title is not None:
sets.append("title=?")
values.append(title[:120])
if context is not None:
sets.append("context=?")
values.append(context[:2000])
if asr_provider is not None:
sets.append("asr_provider=?")
values.append(asr_provider[:40])
if asr_model is not None:
sets.append("asr_model=?")
values.append(asr_model[:80])
values.append(job_id)
with connect() as con:
cur = con.execute(f"UPDATE intake_job SET {','.join(sets)} WHERE id=?", values)
return bool(cur.rowcount)
def delete_job(job_id: str) -> dict[str, Any] | None:
job = get_job(job_id)
if not job:
return None
with connect() as con:
con.execute("DELETE FROM job_event WHERE job_id=?", (job_id,))
con.execute("DELETE FROM intake_job WHERE id=?", (job_id,))
return job
def get_job(job_id: str) -> dict[str, Any] | None: def get_job(job_id: str) -> dict[str, Any] | None:
with connect() as con: with connect() as con:
row = con.execute("SELECT * FROM intake_job WHERE id=?", (job_id,)).fetchone() row = con.execute("SELECT * FROM intake_job WHERE id=?", (job_id,)).fetchone()
@@ -120,13 +174,27 @@ def find_by_stored(stored_name: str) -> dict[str, Any] | None:
return dict(row) if row else None return dict(row) if row else None
def list_jobs(limit: int = 30, offset: int = 0, status: str = "") -> list[dict[str, Any]]: def list_jobs(
limit: int = 30,
offset: int = 0,
status: str = "",
q: str = "",
provider: str = "",
) -> list[dict[str, Any]]:
limit = min(max(limit, 1), 200) limit = min(max(limit, 1), 200)
offset = max(offset, 0) offset = max(offset, 0)
sql, args = "SELECT * FROM intake_job", [] sql = "SELECT * FROM intake_job WHERE 1=1"
args: list[Any] = []
if status: if status:
sql += " WHERE status=?" sql += " AND status=?"
args.append(status) args.append(status)
if provider:
sql += " AND asr_provider=?"
args.append(provider)
if q.strip():
pat = f"%{q.strip()}%"
sql += " AND (title LIKE ? OR original_name LIKE ? OR context LIKE ? OR detail LIKE ?)"
args.extend([pat, pat, pat, pat])
sql += " ORDER BY created_at DESC LIMIT ? OFFSET ?" sql += " ORDER BY created_at DESC LIMIT ? OFFSET ?"
args.extend((limit, offset)) args.extend((limit, offset))
with connect() as con: with connect() as con:
@@ -145,3 +213,32 @@ def pending_recovery() -> list[dict[str, Any]]:
return [dict(r) for r in con.execute( return [dict(r) for r in con.execute(
"SELECT * FROM intake_job WHERE status IN ('queued','transcribing','refining','publishing') " "SELECT * FROM intake_job WHERE status IN ('queued','transcribing','refining','publishing') "
"ORDER BY created_at").fetchall()] "ORDER BY created_at").fetchall()]
def get_all_settings() -> dict[str, str]:
connect().close()
return load_db_settings(force=True)
def save_settings(updates: dict[str, Any]) -> dict[str, str]:
connect().close()
set_settings(updates)
return load_db_settings(force=True)
__all__ = [
"STATUSES",
"connect",
"create_job",
"update_job",
"update_job_meta",
"delete_job",
"get_job",
"find_by_stored",
"list_jobs",
"events",
"pending_recovery",
"get_all_settings",
"save_settings",
"get_setting",
]
File diff suppressed because it is too large Load Diff
@@ -4,6 +4,7 @@ import fcntl
import hashlib import hashlib
import json import json
import os import os
import re
import shutil import shutil
import sqlite3 import sqlite3
import time import time
@@ -11,9 +12,10 @@ import uuid
from pathlib import Path from pathlib import Path
from asrpipe import graph, lexicon, providers, refine, review, revise, vault from asrpipe import graph, lexicon, providers, refine, review, revise, vault
from asrpipe.config import CFG
from . import db from . import db
from .config import ( from .config import (
DATA, DEFAULT_PROVIDER, DRAFTS, FAILED, INBOX, LOGS, MAX_ATTEMPTS, DATA, DRAFTS, FAILED, INBOX, LOGS, MAX_ATTEMPTS,
POLL_SECONDS, PROCESSED, PROCESSING, PUBLIC_URL, REVISION_QUEUE, POLL_SECONDS, PROCESSED, PROCESSING, PUBLIC_URL, REVISION_QUEUE,
STABLE_SECONDS, STATE, SUPPORTED, TRANSCRIPTS, ensure_dirs, STABLE_SECONDS, STATE, SUPPORTED, TRANSCRIPTS, ensure_dirs,
) )
@@ -50,7 +52,7 @@ def transcribe(
) -> tuple[Path, providers.TranscribeResult]: ) -> tuple[Path, providers.TranscribeResult]:
res = providers.transcribe_audio( res = providers.transcribe_audio(
audio, audio,
provider=provider or DEFAULT_PROVIDER, provider=provider or CFG.asr_provider,
model=model, model=model,
title=title, title=title,
user_context=user_context, user_context=user_context,
@@ -71,7 +73,7 @@ def transcribe(
def process_audio(source: Path, job: dict) -> None: def process_audio(source: Path, job: dict) -> None:
job_id = job["id"] job_id = job["id"]
attempts = int(job.get("attempts") or 0) + 1 attempts = int(job.get("attempts") or 0) + 1
prov = providers.normalize_provider_name(job.get("asr_provider") or DEFAULT_PROVIDER) prov = providers.normalize_provider_name(job.get("asr_provider") or CFG.asr_provider)
model = str(job.get("asr_model") or "") model = str(job.get("asr_model") or "")
digest = sha256(source) digest = sha256(source)
prov_label = providers.PROVIDERS.get(prov, {}).get("label", prov) prov_label = providers.PROVIDERS.get(prov, {}).get("label", prov)
@@ -120,6 +122,47 @@ def process_audio(source: Path, job: dict) -> None:
f"{PUBLIC_URL}/jobs/{job_id}/review/") f"{PUBLIC_URL}/jobs/{job_id}/review/")
def rerefine_job(job: dict) -> None:
"""Re-run LLM refinement on an existing raw transcript without re-running ASR."""
job_id = job["id"]
transcript = Path(str(job.get("transcript") or ""))
if not transcript.is_file():
raise FileNotFoundError(f"transcript file missing for job {job_id}: {transcript}")
raw = transcript.read_text(encoding="utf-8", errors="ignore")
intake = {
"job_id": job_id,
"title": job.get("title", ""),
"context": job.get("context", ""),
"source": job.get("source", ""),
"asr_provider": job.get("asr_provider") or CFG.asr_provider,
"asr_model": job.get("asr_model") or "",
}
intake_line = f"<!-- intake: {json.dumps(intake, ensure_ascii=False)} -->"
if re.match(r"^<!--\s*intake:\s*\{.*?\}\s*-->", raw):
raw = re.sub(r"^<!--\s*intake:\s*\{.*?\}\s*-->", intake_line, raw, count=1)
else:
raw = intake_line + "\n" + raw
transcript.write_text(raw, encoding="utf-8")
db.update_job(job_id, "refining", "正在基于现有转写重新生成精修草稿")
review.delete_for_job(
transcript_path=str(transcript),
draft_path=str(job.get("draft") or ""),
job_id=job_id,
)
result = refine.refine(
raw, date=time.strftime("%Y-%m-%d"),
transcript_path=str(transcript), queue_review=True,
)
draft = DRAFTS / f"{job_id}.md"
vault.write_draft_file(draft, result.draft)
questions = review.enqueue_draft(str(transcript), draft)
db.update_job(
job_id, "review", f"整理稿已重新生成,{questions} 个问题待确认",
transcript=str(transcript), draft=str(draft),
)
def claim(path: Path) -> tuple[Path, dict]: def claim(path: Path) -> tuple[Path, dict]:
job = db.find_by_stored(path.name) job = db.find_by_stored(path.name)
if not job: if not job:
@@ -127,7 +170,7 @@ def claim(path: Path) -> tuple[Path, dict]:
job = db.create_job( job = db.create_job(
job_id, path.name, path.name, title=path.stem, job_id, path.name, path.name, title=path.stem,
size=path.stat().st_size, source="ssh", size=path.stat().st_size, source="ssh",
asr_provider=DEFAULT_PROVIDER, asr_provider=CFG.asr_provider,
) )
target = PROCESSING / path.name target = PROCESSING / path.name
if target.exists(): if target.exists():
@@ -164,8 +207,15 @@ def process_revisions() -> None:
for marker in sorted(REVISION_QUEUE.glob("*.json")): for marker in sorted(REVISION_QUEUE.glob("*.json")):
try: try:
payload = json.loads(marker.read_text(encoding="utf-8")) payload = json.loads(marker.read_text(encoding="utf-8"))
draft = Path(payload["draft"]) action = str(payload.get("action") or "revise")
job_id = str(payload.get("job_id") or "") job_id = str(payload.get("job_id") or "")
if action == "rerefine":
job = db.get_job(job_id)
if job:
rerefine_job(job)
marker.unlink(missing_ok=True)
continue
draft = Path(payload["draft"])
db.update_job(job_id, "publishing", "正在应用人工确认") db.update_job(job_id, "publishing", "正在应用人工确认")
revise.revise(draft) revise.revise(draft)
current = db.get_job(job_id) or {} current = db.get_job(job_id) or {}
@@ -181,6 +231,14 @@ def process_revisions() -> None:
marker.unlink(missing_ok=True) marker.unlink(missing_ok=True)
except Exception as exc: except Exception as exc:
log(f"revision marker={marker.name} failed type={type(exc).__name__} err={exc}") log(f"revision marker={marker.name} failed type={type(exc).__name__} err={exc}")
try:
payload = json.loads(marker.read_text(encoding="utf-8"))
jid = str(payload.get("job_id") or "")
if jid:
db.update_job(jid, "failed", f"精修/修订失败:{exc}", error_code=type(exc).__name__)
except Exception:
pass
marker.unlink(missing_ok=True)
def recover() -> None: def recover() -> None:
@@ -5,6 +5,7 @@ import os
import tempfile import tempfile
import threading import threading
import unittest import unittest
import urllib.parse
import urllib.request import urllib.request
from http.server import ThreadingHTTPServer from http.server import ThreadingHTTPServer
from pathlib import Path from pathlib import Path
@@ -20,6 +21,7 @@ os.environ["ASR_NOTIFY"] = "0"
os.environ["ASR_GIT_SYNC"] = "0" os.environ["ASR_GIT_SYNC"] = "0"
from asrpipe import lexicon, providers, review, segment # noqa: E402 from asrpipe import lexicon, providers, review, segment # noqa: E402
from asrpipe.config import CFG # noqa: E402
from asrservice import config as svc_cfg, db as svc_db, web as svc_web # noqa: E402 from asrservice import config as svc_cfg, db as svc_db, web as svc_web # noqa: E402
@@ -33,6 +35,28 @@ class ProviderAndServiceTests(unittest.TestCase):
def tearDown(self): def tearDown(self):
self.tmp.cleanup() self.tmp.cleanup()
def test_sqlite_settings_override_config_dynamically(self):
svc_db.save_settings({
"asr_provider": "soniox",
"litellm_base_url": "https://api.opfo.xyz/v1",
"litellm_api_key": "sk-sqlite-dynamic-key",
"clean_model": "aistudio/gemini-flash-latest",
"draft_model": "openai/gpt-sol-latest",
"soniox_api_key": "soniox-db-secret",
"use_hindsight": "0",
})
self.assertEqual(CFG.asr_provider, "soniox")
self.assertEqual(CFG.litellm_base_url, "https://api.opfo.xyz/v1")
self.assertEqual(CFG.endpoint, "https://api.opfo.xyz/v1/chat/completions")
self.assertEqual(CFG.transcriptions_endpoint, "https://api.opfo.xyz/v1/audio/transcriptions")
self.assertEqual(CFG.api_key(), "sk-sqlite-dynamic-key")
self.assertEqual(CFG.clean_model, "aistudio/gemini-flash-latest")
self.assertEqual(CFG.draft_model, "openai/gpt-sol-latest")
self.assertEqual(CFG.soniox_api_key, "soniox-db-secret")
self.assertFalse(CFG.use_hindsight)
# Restore default for subsequent tests
svc_db.save_settings({"asr_provider": "qwen3cam", "use_hindsight": "1"})
def test_soniox_token_diarization_and_segment_parse(self): def test_soniox_token_diarization_and_segment_parse(self):
payload = { payload = {
"tokens": [ "tokens": [
@@ -135,20 +159,76 @@ class ProviderAndServiceTests(unittest.TestCase):
updated = draft.read_text(encoding="utf-8") updated = draft.read_text(encoding="utf-8")
self.assertIn("speaker_1: 张三", updated) self.assertIn("speaker_1: 张三", updated)
def test_db_and_web_endpoints(self): def test_lexicon_crud_operations(self):
job_id = "0123456789abcdef01234567" # Entity CRUD
job = svc_db.create_job( lexicon.upsert_entity("测试生物", "公司", company="测试集团", role="子公司", related="李四")
job_id=job_id, ents = lexicon.list_entities(q="测试生物")
original_name="meeting.m4a", self.assertEqual(len(ents), 1)
stored_name=f"20261102-{job_id}.m4a", self.assertEqual(ents[0]["company"], "测试集团")
title="架构周会",
context="验证 Web API", lexicon.update_entity("测试生物", "测试生物科技", "公司", company="新集团", role="总部", related="李四,王五")
ents2 = lexicon.list_entities(q="测试生物科技")
self.assertEqual(len(ents2), 1)
self.assertEqual(ents2[0]["company"], "新集团")
self.assertTrue(lexicon.delete_entity("测试生物科技"))
self.assertEqual(len(lexicon.list_entities(q="测试生物科技")), 0)
# Alias & Correction CRUD
lexicon.learn_alias("老测试", "测试人", kind="人名", origin="manual", note="初始")
lexicon.update_alias("老测试", "测试人", "小测试", "测试人", kind="人名", confirmed=1, note="已改")
aliases = lexicon.list_aliases(q="小测试")
self.assertEqual(len(aliases), 1)
self.assertEqual(aliases[0]["note"], "已改")
self.assertTrue(lexicon.delete_alias("小测试", "测试人"))
lexicon.learn_correction("误识词", "正词", note="c1")
lexicon.update_correction("误识词", "正词", "新误识词", "正词", confirmed=1, note="c2")
cors = lexicon.list_corrections(q="新误识词")
self.assertEqual(len(cors), 1)
self.assertEqual(cors[0]["note"], "c2")
self.assertTrue(lexicon.delete_correction("新误识词", "正词"))
# Relation CRUD
lexicon.upsert_relation("张三", "任职于", "洛启生物", confidence=0.8, source="manual")
lexicon.update_relation("张三", "任职于", "洛启生物", "manual", "张三", "实控人", "洛启生物", confidence=0.95, active=1)
rels = lexicon.list_relations(q="张三")
self.assertTrue(any(r["predicate"] == "实控人" for r in rels))
self.assertTrue(lexicon.delete_relation("张三", "实控人", "洛启生物", "manual"))
def test_db_web_endpoints_multi_review_and_job_delete(self):
job1_id = "0123456789abcdef01234567"
job2_id = "fedcba987654321001234567"
t1 = svc_cfg.TRANSCRIPTS / f"{job1_id}.txt"
t2 = svc_cfg.TRANSCRIPTS / f"{job2_id}.txt"
t1.write_text("【speaker_0】录音一内容\n", encoding="utf-8")
t2.write_text("【speaker_0】录音二内容\n", encoding="utf-8")
svc_db.create_job(
job_id=job1_id,
original_name="meeting1.m4a",
stored_name=f"20261102-{job1_id}.m4a",
title="第一场尽调会",
context="验证多录音确认",
size=2048, size=2048,
asr_provider="soniox", asr_provider="soniox",
asr_model="stt-async-v5", asr_model="stt-async-v5",
) )
self.assertEqual(job["asr_provider"], "soniox") svc_db.update_job(job1_id, "review", "待确认", transcript=str(t1))
self.assertEqual(job["asr_model"], "stt-async-v5")
svc_db.create_job(
job_id=job2_id,
original_name="meeting2.m4a",
stored_name=f"20261102-{job2_id}.m4a",
title="第二场投决会",
context="验证批量操作",
size=4096,
asr_provider="qwen3cam",
)
svc_db.update_job(job2_id, "review", "待确认", transcript=str(t2))
q1 = review.enqueue(str(t1), "第一场问题A?", {"kind": "draft_question", "candidates": ["选项A1", "选项A2"]})
q2 = review.enqueue(str(t1), "第一场问题B?", {"kind": "draft_question", "candidates": ["选项B1"]})
q3 = review.enqueue(str(t2), "第二场问题C?", {"kind": "draft_question", "candidates": ["选项C1"]})
svc_web.AGENT_TOKEN = "test-agent-token" svc_web.AGENT_TOKEN = "test-agent-token"
server = ThreadingHTTPServer(("127.0.0.1", 0), svc_web.Handler) server = ThreadingHTTPServer(("127.0.0.1", 0), svc_web.Handler)
@@ -164,6 +244,17 @@ class ProviderAndServiceTests(unittest.TestCase):
with urllib.request.urlopen(req, timeout=5) as resp: with urllib.request.urlopen(req, timeout=5) as resp:
return resp.status, resp.read().decode("utf-8") return resp.status, resp.read().decode("utf-8")
def post_form(path: str, data: dict[str, str]) -> tuple[int, str]:
encoded = urllib.parse.urlencode(data).encode("utf-8")
req = urllib.request.Request(
f"http://127.0.0.1:{port}{path}",
data=encoded,
headers={"Content-Type": "application/x-www-form-urlencoded"},
method="POST",
)
with urllib.request.urlopen(req, timeout=5) as resp:
return resp.status, resp.read().decode("utf-8")
status, body = fetch("/health") status, body = fetch("/health")
self.assertEqual(status, 200) self.assertEqual(status, 200)
self.assertEqual(json.loads(body)["ok"], True) self.assertEqual(json.loads(body)["ok"], True)
@@ -173,22 +264,57 @@ class ProviderAndServiceTests(unittest.TestCase):
headers={"Authorization": "Bearer test-agent-token"}, headers={"Authorization": "Bearer test-agent-token"},
) )
self.assertEqual(status, 200) self.assertEqual(status, 200)
prov_data = json.loads(body) prov_ids = [p["id"] for p in json.loads(body)["providers"]]
prov_ids = [p["id"] for p in prov_data["providers"]]
self.assertEqual(prov_ids, ["qwen3cam", "soniox", "gemini", "openai", "grok"]) self.assertEqual(prov_ids, ["qwen3cam", "soniox", "gemini", "openai", "grok"])
status, body = fetch(f"/jobs/{job_id}/") # Multi-recording review page shows both recordings grouped
status, body = fetch("/review/")
self.assertEqual(status, 200) self.assertEqual(status, 200)
self.assertIn("架构周会", body) self.assertIn("第一场尽调会", body)
self.assertIn("soniox", body) self.assertIn("第二场投决会", body)
self.assertIn("第一场问题A?", body)
self.assertIn("第二场问题C?", body)
status, body = fetch("/lexicon") # Batch submit both questions for recording 1
status, body = post_form(f"/jobs/{job1_id}/review/", {
"question_ids": f"{q1},{q2}",
f"preset_{q1}": "选项A1",
f"custom_{q2}": "自定义回答B",
"batch_submit": "1",
})
self.assertEqual(status, 200) self.assertEqual(status, 200)
self.assertIn("词库", body) self.assertEqual(len(svc_web.pending(svc_db.get_job(job1_id))), 0)
self.assertEqual(len(svc_web.pending(svc_db.get_job(job2_id))), 1)
status, body = fetch("/settings") # Edit job metadata
status, body = post_form(f"/jobs/{job1_id}/meta", {
"title": "第一场尽调会(已改名)",
"context": "更新后的背景",
"provider": "openai",
"model": "openai/gpt-4o-transcribe-diarize",
"next_action": "save",
})
self.assertEqual(status, 200) self.assertEqual(status, 200)
self.assertIn("LiteLLM", body) self.assertIn("第一场尽调会(已改名)", body)
# Save settings via Web UI POST
status, body = post_form("/settings", {
"litellm_base_url": "http://192.168.50.49:4000/v1",
"clean_model": "aistudio/gemini-flash-lite-latest",
"keyword_model": "aistudio/gemini-flash-lite-latest",
"draft_model": "codex/gpt-sol-latest",
"asr_provider": "soniox",
"use_hindsight": "1",
})
self.assertEqual(status, 200)
self.assertEqual(CFG.asr_provider, "soniox")
# Delete job 2 and verify its pending review question is also cleaned up
status, _ = post_form(f"/jobs/{job2_id}/delete", {"delete_files": "1"})
self.assertEqual(status, 200)
self.assertIsNone(svc_db.get_job(job2_id))
self.assertFalse(t2.exists())
self.assertNotIn(q3, [r["id"] for r in review.pending(200)])
finally: finally:
server.shutdown() server.shutdown()
server.server_close() server.server_close()