feat(tools-pve/asr-pipeline): SQLite model config, task delete/rerefine, multi-recording review, lexicon CRUD
This commit is contained in:
@@ -1,22 +1,31 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Configuration for the standalone ASR post-processing pipeline and web service.
|
"""Configuration for the standalone ASR post-processing pipeline and web service.
|
||||||
|
|
||||||
This program has **no dependency on Claude Code, opencode, or any agent CLI**.
|
Model endpoints, provider selections, and API keys are stored in SQLite
|
||||||
It talks to LiteLLM / model providers over plain HTTPS and does its own file
|
(`app_setting` table in `ASR_STATE_DIR/intake.db`) so they can be managed
|
||||||
and git handling, so it can run inside Docker (via Arcane), systemd, or a shell.
|
dynamically from the Web UI without restarting containers. Environment variables
|
||||||
|
serve as initial bootstrap defaults when a setting is not yet in SQLite.
|
||||||
Everything is overridable by environment variable.
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any, Callable
|
||||||
|
|
||||||
PKG = Path(__file__).resolve().parent
|
PKG = Path(__file__).resolve().parent
|
||||||
ROOT = PKG.parent
|
ROOT = PKG.parent
|
||||||
OBSIDIAN = Path(os.environ.get("ASR_WORKSPACE", str(ROOT)))
|
OBSIDIAN = Path(os.environ.get("ASR_WORKSPACE", str(ROOT)))
|
||||||
|
|
||||||
|
_CACHE_LOCK = threading.Lock()
|
||||||
|
_SETTINGS_CACHE: dict[str, str] = {}
|
||||||
|
_SETTINGS_TS: float = 0.0
|
||||||
|
_SETTINGS_DB_PATH: Path | None = None
|
||||||
|
_CACHE_TTL = 1.0
|
||||||
|
|
||||||
|
|
||||||
def _env(name: str, default: str) -> str:
|
def _env(name: str, default: str) -> str:
|
||||||
v = os.environ.get(name)
|
v = os.environ.get(name)
|
||||||
@@ -37,6 +46,165 @@ def _envi(name: str, default: int) -> int:
|
|||||||
return default
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def _state_db_path() -> Path:
|
||||||
|
state_dir = Path(_env("ASR_STATE_DIR", str(ROOT / "state")))
|
||||||
|
return state_dir / "intake.db"
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_settings_table(con: sqlite3.Connection) -> None:
|
||||||
|
con.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS app_setting(
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL DEFAULT '',
|
||||||
|
updated_at REAL NOT NULL
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
|
||||||
|
|
||||||
|
def load_db_settings(force: bool = False) -> dict[str, str]:
|
||||||
|
"""Read `app_setting` rows from `intake.db` with a 1s in-memory cache."""
|
||||||
|
global _SETTINGS_CACHE, _SETTINGS_TS, _SETTINGS_DB_PATH
|
||||||
|
db_path = _state_db_path()
|
||||||
|
now = time.time()
|
||||||
|
with _CACHE_LOCK:
|
||||||
|
if (
|
||||||
|
not force
|
||||||
|
and _SETTINGS_DB_PATH == db_path
|
||||||
|
and (now - _SETTINGS_TS) < _CACHE_TTL
|
||||||
|
):
|
||||||
|
return dict(_SETTINGS_CACHE)
|
||||||
|
if not db_path.exists():
|
||||||
|
_SETTINGS_CACHE = {}
|
||||||
|
_SETTINGS_TS = now
|
||||||
|
_SETTINGS_DB_PATH = db_path
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
con = sqlite3.connect(db_path, timeout=5)
|
||||||
|
con.row_factory = sqlite3.Row
|
||||||
|
_ensure_settings_table(con)
|
||||||
|
rows = con.execute("SELECT key, value FROM app_setting").fetchall()
|
||||||
|
con.close()
|
||||||
|
_SETTINGS_CACHE = {str(r["key"]): str(r["value"]) for r in rows}
|
||||||
|
_SETTINGS_TS = now
|
||||||
|
_SETTINGS_DB_PATH = db_path
|
||||||
|
return dict(_SETTINGS_CACHE)
|
||||||
|
except Exception:
|
||||||
|
return dict(_SETTINGS_CACHE)
|
||||||
|
|
||||||
|
|
||||||
|
def get_setting(key: str, default: str = "") -> str:
|
||||||
|
data = load_db_settings()
|
||||||
|
val = data.get(key)
|
||||||
|
return val if val not in (None, "") else default
|
||||||
|
|
||||||
|
|
||||||
|
def set_settings(updates: dict[str, Any]) -> None:
|
||||||
|
"""Persist multiple settings into `intake.db` and invalidate cache."""
|
||||||
|
global _SETTINGS_TS
|
||||||
|
db_path = _state_db_path()
|
||||||
|
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
now = time.time()
|
||||||
|
con = sqlite3.connect(db_path, timeout=15)
|
||||||
|
try:
|
||||||
|
con.execute("PRAGMA journal_mode=WAL")
|
||||||
|
_ensure_settings_table(con)
|
||||||
|
for k, v in updates.items():
|
||||||
|
if v is None:
|
||||||
|
continue
|
||||||
|
con.execute(
|
||||||
|
"INSERT INTO app_setting(key, value, updated_at) VALUES(?,?,?) "
|
||||||
|
"ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at",
|
||||||
|
(str(k), str(v), now),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
finally:
|
||||||
|
con.close()
|
||||||
|
with _CACHE_LOCK:
|
||||||
|
_SETTINGS_TS = 0.0
|
||||||
|
if "CFG" in globals():
|
||||||
|
for k in updates:
|
||||||
|
CFG.__dict__.pop(f"_override_{k}", None)
|
||||||
|
load_db_settings(force=True)
|
||||||
|
|
||||||
|
|
||||||
|
def bootstrap_default_settings() -> None:
|
||||||
|
"""Seed `app_setting` from environment variables for keys not yet stored."""
|
||||||
|
db_path = _state_db_path()
|
||||||
|
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
now = time.time()
|
||||||
|
defaults = {
|
||||||
|
"asr_provider": _env("ASR_PROVIDER", _env("MEETING_ASR_BACKEND", "qwen3cam")).lower(),
|
||||||
|
"diarize_url": _env("ASR_DIARIZE_URL", _env("MEETING_DIARIZE_URL", "http://192.168.50.100:10099/asr")),
|
||||||
|
"voiceprint_enroll_url": _env("ASR_VOICEPRINT_ENROLL_URL", "http://192.168.50.100:10099/voiceprints/enroll"),
|
||||||
|
"litellm_base_url": _env("LITELLM_BASE_URL", _env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")).rstrip("/"),
|
||||||
|
"litellm_api_key": _env("LITELLM_API_KEY", _env("ASR_LLM_API_KEY", _env("OPENAI_API_KEY", ""))),
|
||||||
|
"clean_model": _env("ASR_CLEAN_MODEL", "aistudio/gemini-flash-lite-latest"),
|
||||||
|
"keyword_model": _env("ASR_KEYWORD_MODEL", "aistudio/gemini-flash-lite-latest"),
|
||||||
|
"draft_model": _env("ASR_DRAFT_MODEL", "codex/gpt-sol-latest"),
|
||||||
|
"gemini_asr_model": _env("ASR_GEMINI_MODEL", "aistudio/gemini-3.5-transcribe"),
|
||||||
|
"openai_asr_model": _env("ASR_OPENAI_MODEL", "openai/gpt-4o-transcribe-diarize"),
|
||||||
|
"soniox_api_base": _env("SONIOX_API_BASE", "https://api.soniox.com/v1").rstrip("/"),
|
||||||
|
"soniox_api_key": _env("SONIOX_API_KEY", ""),
|
||||||
|
"soniox_model": _env("SONIOX_MODEL", "stt-async-v5"),
|
||||||
|
"grok_api_base": _env("XAI_API_BASE", "https://api.x.ai/v1").rstrip("/"),
|
||||||
|
"grok_api_key": _env("XAI_API_KEY", _env("GROK_API_KEY", "")),
|
||||||
|
"grok_asr_model": _env("ASR_GROK_MODEL", "grok-voice-transcribe-2.0"),
|
||||||
|
"hindsight_api": _env("HINDSIGHT_API", "http://192.168.50.47:8888").rstrip("/"),
|
||||||
|
"hindsight_bank": _env("HINDSIGHT_BANK", "personal-context"),
|
||||||
|
"use_hindsight": "1" if _env("ASR_HINDSIGHT", "1") != "0" else "0",
|
||||||
|
"use_termbank": "1" if _env("ASR_TERMBANK", "1") != "0" else "0",
|
||||||
|
"git_sync": "1" if _env("ASR_GIT_SYNC", "0") != "0" else "0",
|
||||||
|
}
|
||||||
|
con = sqlite3.connect(db_path, timeout=15)
|
||||||
|
try:
|
||||||
|
con.execute("PRAGMA journal_mode=WAL")
|
||||||
|
_ensure_settings_table(con)
|
||||||
|
for k, v in defaults.items():
|
||||||
|
con.execute(
|
||||||
|
"INSERT OR IGNORE INTO app_setting(key, value, updated_at) VALUES(?,?,?)",
|
||||||
|
(k, str(v), now),
|
||||||
|
)
|
||||||
|
# If key was seeded empty earlier but env now provides a non-empty secret, populate it
|
||||||
|
if v and k.endswith("_api_key"):
|
||||||
|
con.execute(
|
||||||
|
"UPDATE app_setting SET value=?, updated_at=? WHERE key=? AND value=''",
|
||||||
|
(str(v), now, k),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
finally:
|
||||||
|
con.close()
|
||||||
|
load_db_settings(force=True)
|
||||||
|
|
||||||
|
|
||||||
|
class _SettingAttr:
|
||||||
|
"""Descriptor that resolves from instance override -> SQLite `app_setting` -> env fallback."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
key: str,
|
||||||
|
fallback: Callable[[], str],
|
||||||
|
transform: Callable[[str], Any] | None = None,
|
||||||
|
) -> None:
|
||||||
|
self.key = key
|
||||||
|
self.fallback = fallback
|
||||||
|
self.transform = transform
|
||||||
|
self.override_attr = f"_override_{key}"
|
||||||
|
|
||||||
|
def __get__(self, instance: Any, owner: Any = None) -> Any:
|
||||||
|
if instance is None:
|
||||||
|
return self
|
||||||
|
if self.override_attr in instance.__dict__:
|
||||||
|
return instance.__dict__[self.override_attr]
|
||||||
|
raw = get_setting(self.key, self.fallback())
|
||||||
|
return self.transform(raw) if self.transform else raw
|
||||||
|
|
||||||
|
def __set__(self, instance: Any, value: Any) -> None:
|
||||||
|
instance.__dict__[self.override_attr] = value
|
||||||
|
|
||||||
|
def __delete__(self, instance: Any) -> None:
|
||||||
|
instance.__dict__.pop(self.override_attr, None)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Config:
|
class Config:
|
||||||
# ---- paths ----------------------------------------------------------
|
# ---- paths ----------------------------------------------------------
|
||||||
@@ -46,61 +214,130 @@ class Config:
|
|||||||
_env("ASR_WORKSPACE", str(ROOT))))
|
_env("ASR_WORKSPACE", str(ROOT))))
|
||||||
root: Path = ROOT
|
root: Path = ROOT
|
||||||
|
|
||||||
# ---- LiteLLM / LLM routing (ZenMux deprecated) ----------------------
|
# ---- SQLite-backed dynamic model & provider settings ----------------
|
||||||
litellm_base_url: str = field(default_factory=lambda: _env(
|
litellm_base_url = _SettingAttr(
|
||||||
"LITELLM_BASE_URL",
|
"litellm_base_url",
|
||||||
_env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")).rstrip("/"))
|
lambda: _env("LITELLM_BASE_URL", _env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")),
|
||||||
|
lambda s: s.rstrip("/"),
|
||||||
|
)
|
||||||
|
clean_model = _SettingAttr(
|
||||||
|
"clean_model",
|
||||||
|
lambda: _env("ASR_CLEAN_MODEL", "aistudio/gemini-flash-lite-latest"),
|
||||||
|
)
|
||||||
|
draft_model = _SettingAttr(
|
||||||
|
"draft_model",
|
||||||
|
lambda: _env("ASR_DRAFT_MODEL", "codex/gpt-sol-latest"),
|
||||||
|
)
|
||||||
|
keyword_model = _SettingAttr(
|
||||||
|
"keyword_model",
|
||||||
|
lambda: _env("ASR_KEYWORD_MODEL", "aistudio/gemini-flash-lite-latest"),
|
||||||
|
)
|
||||||
|
|
||||||
clean_model: str = field(default_factory=lambda: _env(
|
asr_provider = _SettingAttr(
|
||||||
"ASR_CLEAN_MODEL", "aistudio/gemini-flash-lite-latest"))
|
"asr_provider",
|
||||||
draft_model: str = field(default_factory=lambda: _env(
|
lambda: _env("ASR_PROVIDER", _env("MEETING_ASR_BACKEND", "qwen3cam")),
|
||||||
"ASR_DRAFT_MODEL", "codex/gpt-sol-latest"))
|
lambda s: s.strip().lower() or "qwen3cam",
|
||||||
keyword_model: str = field(default_factory=lambda: _env(
|
)
|
||||||
"ASR_KEYWORD_MODEL", "aistudio/gemini-flash-lite-latest"))
|
diarize_url = _SettingAttr(
|
||||||
|
"diarize_url",
|
||||||
|
lambda: _env("ASR_DIARIZE_URL", _env("MEETING_DIARIZE_URL", "http://192.168.50.100:10099/asr")),
|
||||||
|
)
|
||||||
|
voiceprint_enroll_url = _SettingAttr(
|
||||||
|
"voiceprint_enroll_url",
|
||||||
|
lambda: _env("ASR_VOICEPRINT_ENROLL_URL", "http://192.168.50.100:10099/voiceprints/enroll"),
|
||||||
|
)
|
||||||
|
|
||||||
endpoint: str = field(default_factory=lambda: _env(
|
soniox_api_key = _SettingAttr(
|
||||||
"ASR_ENDPOINT",
|
"soniox_api_key",
|
||||||
_env("LITELLM_BASE_URL",
|
lambda: _env("SONIOX_API_KEY", ""),
|
||||||
_env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")).rstrip("/")
|
)
|
||||||
+ "/chat/completions"))
|
soniox_api_base = _SettingAttr(
|
||||||
transcriptions_endpoint: str = field(default_factory=lambda: _env(
|
"soniox_api_base",
|
||||||
"ASR_TRANSCRIPTIONS_ENDPOINT",
|
lambda: _env("SONIOX_API_BASE", "https://api.soniox.com/v1"),
|
||||||
_env("LITELLM_BASE_URL",
|
lambda s: s.rstrip("/"),
|
||||||
_env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")).rstrip("/")
|
)
|
||||||
+ "/audio/transcriptions"))
|
soniox_model = _SettingAttr(
|
||||||
|
"soniox_model",
|
||||||
|
lambda: _env("SONIOX_MODEL", "stt-async-v5"),
|
||||||
|
)
|
||||||
|
|
||||||
|
gemini_asr_model = _SettingAttr(
|
||||||
|
"gemini_asr_model",
|
||||||
|
lambda: _env("ASR_GEMINI_MODEL", "aistudio/gemini-3.5-transcribe"),
|
||||||
|
)
|
||||||
|
openai_asr_model = _SettingAttr(
|
||||||
|
"openai_asr_model",
|
||||||
|
lambda: _env("ASR_OPENAI_MODEL", "openai/gpt-4o-transcribe-diarize"),
|
||||||
|
)
|
||||||
|
|
||||||
|
grok_api_key = _SettingAttr(
|
||||||
|
"grok_api_key",
|
||||||
|
lambda: _env("XAI_API_KEY", _env("GROK_API_KEY", "")),
|
||||||
|
)
|
||||||
|
grok_api_base = _SettingAttr(
|
||||||
|
"grok_api_base",
|
||||||
|
lambda: _env("XAI_API_BASE", "https://api.x.ai/v1"),
|
||||||
|
lambda s: s.rstrip("/"),
|
||||||
|
)
|
||||||
|
grok_asr_model = _SettingAttr(
|
||||||
|
"grok_asr_model",
|
||||||
|
lambda: _env("ASR_GROK_MODEL", "grok-voice-transcribe-2.0"),
|
||||||
|
)
|
||||||
|
|
||||||
|
hindsight_api = _SettingAttr(
|
||||||
|
"hindsight_api",
|
||||||
|
lambda: _env("HINDSIGHT_API", "http://192.168.50.47:8888"),
|
||||||
|
lambda s: s.rstrip("/"),
|
||||||
|
)
|
||||||
|
hindsight_bank = _SettingAttr(
|
||||||
|
"hindsight_bank",
|
||||||
|
lambda: _env("HINDSIGHT_BANK", "personal-context"),
|
||||||
|
)
|
||||||
|
use_hindsight = _SettingAttr(
|
||||||
|
"use_hindsight",
|
||||||
|
lambda: _env("ASR_HINDSIGHT", "1"),
|
||||||
|
lambda s: str(s).lower() not in ("0", "false", "no", "off"),
|
||||||
|
)
|
||||||
|
use_termbank = _SettingAttr(
|
||||||
|
"use_termbank",
|
||||||
|
lambda: _env("ASR_TERMBANK", "1"),
|
||||||
|
lambda s: str(s).lower() not in ("0", "false", "no", "off"),
|
||||||
|
)
|
||||||
|
git_sync = _SettingAttr(
|
||||||
|
"git_sync",
|
||||||
|
lambda: _env("ASR_GIT_SYNC", "1"),
|
||||||
|
lambda s: str(s).lower() not in ("0", "false", "no", "off"),
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def endpoint(self) -> str:
|
||||||
|
if "_override_endpoint" in self.__dict__:
|
||||||
|
return self.__dict__["_override_endpoint"]
|
||||||
|
explicit = os.environ.get("ASR_ENDPOINT")
|
||||||
|
if explicit:
|
||||||
|
return explicit
|
||||||
|
return f"{self.litellm_base_url}/chat/completions"
|
||||||
|
|
||||||
|
@endpoint.setter
|
||||||
|
def endpoint(self, value: str) -> None:
|
||||||
|
self.__dict__["_override_endpoint"] = value
|
||||||
|
|
||||||
|
@property
|
||||||
|
def transcriptions_endpoint(self) -> str:
|
||||||
|
if "_override_transcriptions_endpoint" in self.__dict__:
|
||||||
|
return self.__dict__["_override_transcriptions_endpoint"]
|
||||||
|
explicit = os.environ.get("ASR_TRANSCRIPTIONS_ENDPOINT")
|
||||||
|
if explicit:
|
||||||
|
return explicit
|
||||||
|
return f"{self.litellm_base_url}/audio/transcriptions"
|
||||||
|
|
||||||
|
@transcriptions_endpoint.setter
|
||||||
|
def transcriptions_endpoint(self, value: str) -> None:
|
||||||
|
self.__dict__["_override_transcriptions_endpoint"] = value
|
||||||
|
|
||||||
concurrency: int = field(default_factory=lambda: _envi("CONCURRENCY", 6))
|
concurrency: int = field(default_factory=lambda: _envi("CONCURRENCY", 6))
|
||||||
http_timeout: float = field(default_factory=lambda: _envf("HTTP_TIMEOUT", 420.0))
|
http_timeout: float = field(default_factory=lambda: _envf("HTTP_TIMEOUT", 420.0))
|
||||||
max_retries: int = field(default_factory=lambda: _envi("MAX_RETRIES", 5))
|
max_retries: int = field(default_factory=lambda: _envi("MAX_RETRIES", 5))
|
||||||
|
|
||||||
# ---- ASR providers --------------------------------------------------
|
|
||||||
asr_provider: str = field(default_factory=lambda: _env(
|
|
||||||
"ASR_PROVIDER", _env("MEETING_ASR_BACKEND", "qwen3cam")).lower())
|
|
||||||
diarize_url: str = field(default_factory=lambda: _env(
|
|
||||||
"ASR_DIARIZE_URL",
|
|
||||||
_env("MEETING_DIARIZE_URL", "http://192.168.50.100:10099/asr")))
|
|
||||||
voiceprint_enroll_url: str = field(default_factory=lambda: _env(
|
|
||||||
"ASR_VOICEPRINT_ENROLL_URL",
|
|
||||||
"http://192.168.50.100:10099/voiceprints/enroll"))
|
|
||||||
|
|
||||||
soniox_api_key: str = field(default_factory=lambda: _env("SONIOX_API_KEY", ""))
|
|
||||||
soniox_api_base: str = field(default_factory=lambda: _env(
|
|
||||||
"SONIOX_API_BASE", "https://api.soniox.com/v1").rstrip("/"))
|
|
||||||
soniox_model: str = field(default_factory=lambda: _env(
|
|
||||||
"SONIOX_MODEL", "stt-async-v5"))
|
|
||||||
|
|
||||||
gemini_asr_model: str = field(default_factory=lambda: _env(
|
|
||||||
"ASR_GEMINI_MODEL", "aistudio/gemini-3.5-transcribe"))
|
|
||||||
openai_asr_model: str = field(default_factory=lambda: _env(
|
|
||||||
"ASR_OPENAI_MODEL", "openai/gpt-4o-transcribe-diarize"))
|
|
||||||
|
|
||||||
grok_api_key: str = field(default_factory=lambda: _env(
|
|
||||||
"XAI_API_KEY", _env("GROK_API_KEY", "")))
|
|
||||||
grok_api_base: str = field(default_factory=lambda: _env(
|
|
||||||
"XAI_API_BASE", "https://api.x.ai/v1").rstrip("/"))
|
|
||||||
grok_asr_model: str = field(default_factory=lambda: _env(
|
|
||||||
"ASR_GROK_MODEL", "grok-voice-transcribe-2.0"))
|
|
||||||
|
|
||||||
segment_seconds: int = field(default_factory=lambda: _envi(
|
segment_seconds: int = field(default_factory=lambda: _envi(
|
||||||
"SEGMENT_SECONDS", int(_env("MEETING_ASR_SEGMENT_SECONDS", "540"))))
|
"SEGMENT_SECONDS", int(_env("MEETING_ASR_SEGMENT_SECONDS", "540"))))
|
||||||
|
|
||||||
@@ -119,10 +356,6 @@ class Config:
|
|||||||
default_factory=lambda: _envf("GATE_MAX_RATIO", 1.35))
|
default_factory=lambda: _envf("GATE_MAX_RATIO", 1.35))
|
||||||
|
|
||||||
# ---- knowledge sources ----------------------------------------------
|
# ---- knowledge sources ----------------------------------------------
|
||||||
use_termbank: bool = field(
|
|
||||||
default_factory=lambda: _env("ASR_TERMBANK", "1") != "0")
|
|
||||||
use_hindsight: bool = field(
|
|
||||||
default_factory=lambda: _env("ASR_HINDSIGHT", "1") != "0")
|
|
||||||
use_keyword_recall: bool = field(
|
use_keyword_recall: bool = field(
|
||||||
default_factory=lambda: _env("ASR_KEYWORD_RECALL", "1") != "0")
|
default_factory=lambda: _env("ASR_KEYWORD_RECALL", "1") != "0")
|
||||||
use_lexicon: bool = field(
|
use_lexicon: bool = field(
|
||||||
@@ -131,10 +364,6 @@ class Config:
|
|||||||
default_factory=lambda: _env("ASR_NORMALIZE", "1") != "0")
|
default_factory=lambda: _env("ASR_NORMALIZE", "1") != "0")
|
||||||
use_evidence: bool = field(
|
use_evidence: bool = field(
|
||||||
default_factory=lambda: _env("ASR_EVIDENCE", "1") != "0")
|
default_factory=lambda: _env("ASR_EVIDENCE", "1") != "0")
|
||||||
hindsight_api: str = field(default_factory=lambda: _env(
|
|
||||||
"HINDSIGHT_API", "http://192.168.50.47:8888"))
|
|
||||||
hindsight_bank: str = field(default_factory=lambda: _env(
|
|
||||||
"HINDSIGHT_BANK", "personal-context"))
|
|
||||||
hindsight_timeout: float = field(
|
hindsight_timeout: float = field(
|
||||||
default_factory=lambda: _envf("HINDSIGHT_TIMEOUT", 12.0))
|
default_factory=lambda: _envf("HINDSIGHT_TIMEOUT", 12.0))
|
||||||
flag_min_score: float = field(
|
flag_min_score: float = field(
|
||||||
@@ -154,13 +383,17 @@ class Config:
|
|||||||
default_factory=lambda: _envi("REVIEW_MAX_SECONDS", 7200))
|
default_factory=lambda: _envi("REVIEW_MAX_SECONDS", 7200))
|
||||||
|
|
||||||
# ---- behaviour ------------------------------------------------------
|
# ---- behaviour ------------------------------------------------------
|
||||||
git_sync: bool = field(
|
|
||||||
default_factory=lambda: _env("ASR_GIT_SYNC", "1") != "0")
|
|
||||||
notify: bool = field(
|
notify: bool = field(
|
||||||
default_factory=lambda: _env("ASR_NOTIFY", "1") != "0")
|
default_factory=lambda: _env("ASR_NOTIFY", "1") != "0")
|
||||||
cache_llm: bool = field(
|
cache_llm: bool = field(
|
||||||
default_factory=lambda: _env("ASR_CACHE", "0") != "0")
|
default_factory=lambda: _env("ASR_CACHE", "0") != "0")
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
# Clear dataclass-populated default descriptors so __get__ is invoked dynamically
|
||||||
|
for k in list(self.__dict__.keys()):
|
||||||
|
if isinstance(getattr(type(self), k, None), _SettingAttr):
|
||||||
|
del self.__dict__[k]
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def inbox(self) -> Path:
|
def inbox(self) -> Path:
|
||||||
return self.vault / ".asr-inbox"
|
return self.vault / ".asr-inbox"
|
||||||
@@ -217,6 +450,9 @@ class Config:
|
|||||||
return Path(_env("ASR_ENV_FILE", str(ROOT / ".env")))
|
return Path(_env("ASR_ENV_FILE", str(ROOT / ".env")))
|
||||||
|
|
||||||
def api_key(self) -> str:
|
def api_key(self) -> str:
|
||||||
|
db_key = get_setting("litellm_api_key", "")
|
||||||
|
if db_key:
|
||||||
|
return db_key
|
||||||
for k in ("LITELLM_API_KEY", "ASR_LLM_API_KEY", "OPENAI_API_KEY",
|
for k in ("LITELLM_API_KEY", "ASR_LLM_API_KEY", "OPENAI_API_KEY",
|
||||||
"ANTHROPIC_AUTH_TOKEN"):
|
"ANTHROPIC_AUTH_TOKEN"):
|
||||||
v = os.environ.get(k)
|
v = os.environ.get(k)
|
||||||
@@ -233,8 +469,7 @@ class Config:
|
|||||||
if val:
|
if val:
|
||||||
return val
|
return val
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"no API key: set LITELLM_API_KEY / ASR_LLM_API_KEY / OPENAI_API_KEY "
|
"no API key: configure LiteLLM API Key in /settings or set LITELLM_API_KEY")
|
||||||
f"or provide {p}")
|
|
||||||
|
|
||||||
|
|
||||||
CFG = Config()
|
CFG = Config()
|
||||||
|
|||||||
@@ -298,7 +298,7 @@ def sync_from_vault(force: bool = False) -> dict:
|
|||||||
now = time.time()
|
now = time.time()
|
||||||
n_ent = n_alias = 0
|
n_ent = n_alias = 0
|
||||||
con.execute("DELETE FROM alias WHERE origin IN ('card','glossary')")
|
con.execute("DELETE FROM alias WHERE origin IN ('card','glossary')")
|
||||||
con.execute("DELETE FROM entity")
|
con.execute("DELETE FROM entity WHERE card <> ''")
|
||||||
con.execute("DELETE FROM relation WHERE source LIKE 'card:%'")
|
con.execute("DELETE FROM relation WHERE source LIKE 'card:%'")
|
||||||
for sub, kind in (("People", "人名"), ("Companies", "公司"), ("Projects", "项目")):
|
for sub, kind in (("People", "人名"), ("Companies", "公司"), ("Projects", "项目")):
|
||||||
d = CFG.vault / sub
|
d = CFG.vault / sub
|
||||||
@@ -524,3 +524,261 @@ def stats() -> dict:
|
|||||||
d["synced_at"] = r["v"] if r else ""
|
d["synced_at"] = r["v"] if r else ""
|
||||||
con.close()
|
con.close()
|
||||||
return d
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ Web UI CRUD
|
||||||
|
def list_aliases(q: str = "", origin_filter: str = "", limit: int = 200) -> list[dict]:
|
||||||
|
con = connect()
|
||||||
|
sql = "SELECT * FROM alias WHERE 1=1"
|
||||||
|
args: list = []
|
||||||
|
if q.strip():
|
||||||
|
pat = f"%{q.strip()}%"
|
||||||
|
sql += " AND (alias LIKE ? OR canonical LIKE ? OR note LIKE ?)"
|
||||||
|
args.extend([pat, pat, pat])
|
||||||
|
if origin_filter == "learned":
|
||||||
|
sql += " AND origin <> 'card'"
|
||||||
|
elif origin_filter == "card":
|
||||||
|
sql += " AND origin = 'card'"
|
||||||
|
sql += " ORDER BY confirmed DESC, created_at DESC, alias ASC LIMIT ?"
|
||||||
|
args.append(limit)
|
||||||
|
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
|
||||||
|
con.close()
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def update_alias(
|
||||||
|
old_alias: str,
|
||||||
|
old_canonical: str,
|
||||||
|
new_alias: str,
|
||||||
|
new_canonical: str,
|
||||||
|
*,
|
||||||
|
kind: str = "",
|
||||||
|
confirmed: int = 1,
|
||||||
|
note: str = "",
|
||||||
|
) -> bool:
|
||||||
|
con = connect()
|
||||||
|
row = con.execute(
|
||||||
|
"SELECT * FROM alias WHERE alias=? AND canonical=?",
|
||||||
|
(old_alias, old_canonical),
|
||||||
|
).fetchone()
|
||||||
|
origin = row["origin"] if row else "manual"
|
||||||
|
hits = row["hits"] if row else 0
|
||||||
|
created_at = row["created_at"] if row else time.time()
|
||||||
|
if not kind:
|
||||||
|
er = con.execute("SELECT kind FROM entity WHERE canonical=?", (new_canonical,)).fetchone()
|
||||||
|
kind = er["kind"] if er else (row["kind"] if row else guess_kind(new_canonical))
|
||||||
|
con.execute(
|
||||||
|
"DELETE FROM alias WHERE alias=? AND canonical=?",
|
||||||
|
(old_alias, old_canonical),
|
||||||
|
)
|
||||||
|
con.execute(
|
||||||
|
"INSERT OR REPLACE INTO alias(alias,canonical,kind,origin,confirmed,hits,created_at,note) "
|
||||||
|
"VALUES(?,?,?,?,?,?,?,?)",
|
||||||
|
(new_alias, new_canonical, kind, origin, int(bool(confirmed)), hits, created_at, note),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def delete_alias(alias: str, canonical: str) -> bool:
|
||||||
|
con = connect()
|
||||||
|
cur = con.execute("DELETE FROM alias WHERE alias=? AND canonical=?", (alias, canonical))
|
||||||
|
con.commit()
|
||||||
|
deleted = cur.rowcount > 0
|
||||||
|
con.close()
|
||||||
|
return deleted
|
||||||
|
|
||||||
|
|
||||||
|
def list_corrections(q: str = "", limit: int = 200) -> list[dict]:
|
||||||
|
con = connect()
|
||||||
|
sql = "SELECT * FROM correction WHERE 1=1"
|
||||||
|
args: list = []
|
||||||
|
if q.strip():
|
||||||
|
pat = f"%{q.strip()}%"
|
||||||
|
sql += " AND (surface LIKE ? OR canonical LIKE ? OR note LIKE ?)"
|
||||||
|
args.extend([pat, pat, pat])
|
||||||
|
sql += " ORDER BY confirmed DESC, n DESC, last_seen DESC LIMIT ?"
|
||||||
|
args.append(limit)
|
||||||
|
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
|
||||||
|
con.close()
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def update_correction(
|
||||||
|
old_surface: str,
|
||||||
|
old_canonical: str,
|
||||||
|
new_surface: str,
|
||||||
|
new_canonical: str,
|
||||||
|
*,
|
||||||
|
confirmed: int = 1,
|
||||||
|
note: str = "",
|
||||||
|
) -> bool:
|
||||||
|
con = connect()
|
||||||
|
row = con.execute(
|
||||||
|
"SELECT * FROM correction WHERE surface=? AND canonical=?",
|
||||||
|
(old_surface, old_canonical),
|
||||||
|
).fetchone()
|
||||||
|
n = row["n"] if row else 1
|
||||||
|
now = time.time()
|
||||||
|
con.execute("DELETE FROM correction WHERE surface=? AND canonical=?", (old_surface, old_canonical))
|
||||||
|
con.execute(
|
||||||
|
"DELETE FROM phonetic_confusion WHERE heard=? AND intended=?",
|
||||||
|
(old_surface, old_canonical),
|
||||||
|
)
|
||||||
|
con.execute(
|
||||||
|
"INSERT OR REPLACE INTO correction(surface,canonical,n,confirmed,last_seen,note) "
|
||||||
|
"VALUES(?,?,?,?,?,?)",
|
||||||
|
(new_surface, new_canonical, n, int(bool(confirmed)), now, note),
|
||||||
|
)
|
||||||
|
con.execute(
|
||||||
|
"INSERT OR REPLACE INTO phonetic_confusion(heard,intended,n,confirmed,last_seen,note) "
|
||||||
|
"VALUES(?,?,?,?,?,?)",
|
||||||
|
(new_surface, new_canonical, n, int(bool(confirmed)), now, note),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def delete_correction(surface: str, canonical: str) -> bool:
|
||||||
|
con = connect()
|
||||||
|
cur = con.execute("DELETE FROM correction WHERE surface=? AND canonical=?", (surface, canonical))
|
||||||
|
con.execute("DELETE FROM phonetic_confusion WHERE heard=? AND intended=?", (surface, canonical))
|
||||||
|
con.commit()
|
||||||
|
deleted = cur.rowcount > 0
|
||||||
|
con.close()
|
||||||
|
return deleted
|
||||||
|
|
||||||
|
|
||||||
|
def list_entities(q: str = "", kind: str = "", limit: int = 200) -> list[dict]:
|
||||||
|
con = connect()
|
||||||
|
sql = "SELECT * FROM entity WHERE 1=1"
|
||||||
|
args: list = []
|
||||||
|
if q.strip():
|
||||||
|
pat = f"%{q.strip()}%"
|
||||||
|
sql += " AND (canonical LIKE ? OR company LIKE ? OR role LIKE ? OR related LIKE ?)"
|
||||||
|
args.extend([pat, pat, pat, pat])
|
||||||
|
if kind.strip():
|
||||||
|
sql += " AND kind=?"
|
||||||
|
args.append(kind.strip())
|
||||||
|
sql += " ORDER BY (CASE WHEN card='' THEN 0 ELSE 1 END), canonical ASC LIMIT ?"
|
||||||
|
args.append(limit)
|
||||||
|
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
|
||||||
|
con.close()
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_entity(
|
||||||
|
canonical: str,
|
||||||
|
kind: str = "人名",
|
||||||
|
*,
|
||||||
|
company: str = "",
|
||||||
|
role: str = "",
|
||||||
|
related: str = "",
|
||||||
|
card: str = "",
|
||||||
|
) -> None:
|
||||||
|
con = connect()
|
||||||
|
con.execute(
|
||||||
|
"INSERT OR REPLACE INTO entity(canonical,kind,company,role,related,card,synced_at) "
|
||||||
|
"VALUES(?,?,?,?,?,?,?)",
|
||||||
|
(canonical, kind or guess_kind(canonical), company, role, related, card, time.time()),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
|
||||||
|
|
||||||
|
def update_entity(
|
||||||
|
old_canonical: str,
|
||||||
|
new_canonical: str,
|
||||||
|
kind: str,
|
||||||
|
*,
|
||||||
|
company: str = "",
|
||||||
|
role: str = "",
|
||||||
|
related: str = "",
|
||||||
|
) -> bool:
|
||||||
|
con = connect()
|
||||||
|
row = con.execute("SELECT * FROM entity WHERE canonical=?", (old_canonical,)).fetchone()
|
||||||
|
card = row["card"] if row else ""
|
||||||
|
con.execute("DELETE FROM entity WHERE canonical=?", (old_canonical,))
|
||||||
|
con.execute(
|
||||||
|
"INSERT OR REPLACE INTO entity(canonical,kind,company,role,related,card,synced_at) "
|
||||||
|
"VALUES(?,?,?,?,?,?,?)",
|
||||||
|
(new_canonical, kind or guess_kind(new_canonical), company, role, related, card, time.time()),
|
||||||
|
)
|
||||||
|
if old_canonical != new_canonical:
|
||||||
|
con.execute("UPDATE alias SET canonical=? WHERE canonical=?", (new_canonical, old_canonical))
|
||||||
|
con.execute("UPDATE correction SET canonical=? WHERE canonical=?", (new_canonical, old_canonical))
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def delete_entity(canonical: str) -> bool:
|
||||||
|
con = connect()
|
||||||
|
cur = con.execute("DELETE FROM entity WHERE canonical=?", (canonical,))
|
||||||
|
con.commit()
|
||||||
|
deleted = cur.rowcount > 0
|
||||||
|
con.close()
|
||||||
|
return deleted
|
||||||
|
|
||||||
|
|
||||||
|
def list_relations(q: str = "", limit: int = 200) -> list[dict]:
|
||||||
|
con = connect()
|
||||||
|
sql = "SELECT * FROM relation WHERE 1=1"
|
||||||
|
args: list = []
|
||||||
|
if q.strip():
|
||||||
|
pat = f"%{q.strip()}%"
|
||||||
|
sql += " AND (subject LIKE ? OR predicate LIKE ? OR object LIKE ? OR source LIKE ?)"
|
||||||
|
args.extend([pat, pat, pat, pat])
|
||||||
|
sql += " ORDER BY active DESC, confidence DESC, last_seen DESC LIMIT ?"
|
||||||
|
args.append(limit)
|
||||||
|
rows = [dict(r) for r in con.execute(sql, args).fetchall()]
|
||||||
|
con.close()
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def update_relation(
|
||||||
|
old_subject: str,
|
||||||
|
old_predicate: str,
|
||||||
|
old_object: str,
|
||||||
|
old_source: str,
|
||||||
|
subject: str,
|
||||||
|
predicate: str,
|
||||||
|
object_: str,
|
||||||
|
*,
|
||||||
|
confidence: float = 0.8,
|
||||||
|
active: int = 1,
|
||||||
|
) -> bool:
|
||||||
|
con = connect()
|
||||||
|
row = con.execute(
|
||||||
|
"SELECT * FROM relation WHERE subject=? AND predicate=? AND object=? AND source=?",
|
||||||
|
(old_subject, old_predicate, old_object, old_source),
|
||||||
|
).fetchone()
|
||||||
|
first_seen = row["first_seen"] if row else time.time()
|
||||||
|
source = old_source or "manual"
|
||||||
|
con.execute(
|
||||||
|
"DELETE FROM relation WHERE subject=? AND predicate=? AND object=? AND source=?",
|
||||||
|
(old_subject, old_predicate, old_object, old_source),
|
||||||
|
)
|
||||||
|
con.execute(
|
||||||
|
"INSERT OR REPLACE INTO relation(subject,predicate,object,confidence,source,first_seen,last_seen,active) "
|
||||||
|
"VALUES(?,?,?,?,?,?,?,?)",
|
||||||
|
(subject, predicate, object_, float(confidence), source, first_seen, time.time(), int(bool(active))),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def delete_relation(subject: str, predicate: str, object_: str, source: str) -> bool:
|
||||||
|
con = connect()
|
||||||
|
cur = con.execute(
|
||||||
|
"DELETE FROM relation WHERE subject=? AND predicate=? AND object=? AND source=?",
|
||||||
|
(subject, predicate, object_, source),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
deleted = cur.rowcount > 0
|
||||||
|
con.close()
|
||||||
|
return deleted
|
||||||
|
|
||||||
|
|||||||
@@ -307,14 +307,14 @@ def _request_revision(draft_path: str) -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def answer(question_id: int, value: str) -> dict:
|
def answer(question_id: int, value: str, *, allow_update: bool = False) -> dict:
|
||||||
con = lexicon.connect()
|
con = lexicon.connect()
|
||||||
row = con.execute("SELECT * FROM review_question WHERE id=?",
|
row = con.execute("SELECT * FROM review_question WHERE id=?",
|
||||||
(question_id,)).fetchone()
|
(question_id,)).fetchone()
|
||||||
if not row:
|
if not row:
|
||||||
con.close()
|
con.close()
|
||||||
raise KeyError(f"review question {question_id} not found")
|
raise KeyError(f"review question {question_id} not found")
|
||||||
if row["status"] != "pending":
|
if row["status"] != "pending" and not allow_update:
|
||||||
result = {
|
result = {
|
||||||
"id": question_id, "answer": row["answer"], "learned": False,
|
"id": question_id, "answer": row["answer"], "learned": False,
|
||||||
"already_answered": True,
|
"already_answered": True,
|
||||||
@@ -376,3 +376,47 @@ def answer(question_id: int, value: str) -> dict:
|
|||||||
"draft_feedback_saved": saved, "voiceprint_enrolled": voiceprint_enrolled,
|
"draft_feedback_saved": saved, "voiceprint_enrolled": voiceprint_enrolled,
|
||||||
"revision_started": revision_started,
|
"revision_started": revision_started,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def recent_answered(limit: int = 30) -> list[dict]:
|
||||||
|
con = lexicon.connect()
|
||||||
|
rows = con.execute(
|
||||||
|
"SELECT * FROM review_question WHERE status='answered' "
|
||||||
|
"ORDER BY answered_at DESC, id DESC LIMIT ?",
|
||||||
|
(limit,),
|
||||||
|
).fetchall()
|
||||||
|
con.close()
|
||||||
|
return [dict(r) for r in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def reopen_question(question_id: int) -> bool:
|
||||||
|
con = lexicon.connect()
|
||||||
|
cur = con.execute(
|
||||||
|
"UPDATE review_question SET status='pending', answer='', answered_at=NULL WHERE id=?",
|
||||||
|
(question_id,),
|
||||||
|
)
|
||||||
|
con.commit()
|
||||||
|
ok = cur.rowcount > 0
|
||||||
|
con.close()
|
||||||
|
return ok
|
||||||
|
|
||||||
|
|
||||||
|
def delete_for_job(transcript_path: str = "", draft_path: str = "", job_id: str = "") -> int:
|
||||||
|
con = lexicon.connect()
|
||||||
|
rows = con.execute("SELECT id, recording, payload FROM review_question").fetchall()
|
||||||
|
to_delete: list[int] = []
|
||||||
|
for r in rows:
|
||||||
|
payload = json.loads(r["payload"] or "{}")
|
||||||
|
if (
|
||||||
|
(transcript_path and r["recording"] == transcript_path)
|
||||||
|
or (draft_path and payload.get("draft_path") == draft_path)
|
||||||
|
or (job_id and payload.get("job_id") == job_id)
|
||||||
|
):
|
||||||
|
to_delete.append(int(r["id"]))
|
||||||
|
if to_delete:
|
||||||
|
marks = ",".join("?" for _ in to_delete)
|
||||||
|
con.execute(f"DELETE FROM review_question WHERE id IN ({marks})", to_delete)
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
return len(to_delete)
|
||||||
|
|
||||||
|
|||||||
@@ -4,15 +4,19 @@ import sqlite3
|
|||||||
import time
|
import time
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from .config import DB, DEFAULT_PROVIDER, ensure_dirs
|
from asrpipe.config import CFG, bootstrap_default_settings, get_setting, load_db_settings, set_settings
|
||||||
|
from .config import DB, ensure_dirs
|
||||||
|
|
||||||
STATUSES = {
|
STATUSES = {
|
||||||
"uploading", "queued", "transcribing", "refining", "review",
|
"uploading", "queued", "transcribing", "refining", "review",
|
||||||
"publishing", "complete", "failed",
|
"publishing", "complete", "failed",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_BOOTSTRAPPED_DB: str = ""
|
||||||
|
|
||||||
|
|
||||||
def connect() -> sqlite3.Connection:
|
def connect() -> sqlite3.Connection:
|
||||||
|
global _BOOTSTRAPPED_DB
|
||||||
ensure_dirs()
|
ensure_dirs()
|
||||||
con = sqlite3.connect(DB, timeout=30)
|
con = sqlite3.connect(DB, timeout=30)
|
||||||
con.row_factory = sqlite3.Row
|
con.row_factory = sqlite3.Row
|
||||||
@@ -37,8 +41,14 @@ def connect() -> sqlite3.Connection:
|
|||||||
status TEXT NOT NULL, detail TEXT NOT NULL DEFAULT '', created_at REAL NOT NULL
|
status TEXT NOT NULL, detail TEXT NOT NULL DEFAULT '', created_at REAL NOT NULL
|
||||||
);
|
);
|
||||||
CREATE INDEX IF NOT EXISTS ix_job_event_job ON job_event(job_id,id);
|
CREATE INDEX IF NOT EXISTS ix_job_event_job ON job_event(job_id,id);
|
||||||
|
CREATE TABLE IF NOT EXISTS app_setting(
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL DEFAULT '',
|
||||||
|
updated_at REAL NOT NULL
|
||||||
|
);
|
||||||
""")
|
""")
|
||||||
cols = {r[1] for r in con.execute("PRAGMA table_info(intake_job)")}
|
cols = {r[1] for r in con.execute("PRAGMA table_info(intake_job)")}
|
||||||
|
default_prov = CFG.asr_provider or "qwen3cam"
|
||||||
additions = {
|
additions = {
|
||||||
"draft": "TEXT NOT NULL DEFAULT ''",
|
"draft": "TEXT NOT NULL DEFAULT ''",
|
||||||
"result": "TEXT NOT NULL DEFAULT ''",
|
"result": "TEXT NOT NULL DEFAULT ''",
|
||||||
@@ -46,13 +56,19 @@ def connect() -> sqlite3.Connection:
|
|||||||
"sha256": "TEXT NOT NULL DEFAULT ''",
|
"sha256": "TEXT NOT NULL DEFAULT ''",
|
||||||
"attempts": "INTEGER NOT NULL DEFAULT 0",
|
"attempts": "INTEGER NOT NULL DEFAULT 0",
|
||||||
"error_code": "TEXT NOT NULL DEFAULT ''",
|
"error_code": "TEXT NOT NULL DEFAULT ''",
|
||||||
"asr_provider": f"TEXT NOT NULL DEFAULT '{DEFAULT_PROVIDER}'",
|
"asr_provider": f"TEXT NOT NULL DEFAULT '{default_prov}'",
|
||||||
"asr_model": "TEXT NOT NULL DEFAULT ''",
|
"asr_model": "TEXT NOT NULL DEFAULT ''",
|
||||||
}
|
}
|
||||||
for name, ddl in additions.items():
|
for name, ddl in additions.items():
|
||||||
if name not in cols:
|
if name not in cols:
|
||||||
con.execute(f"ALTER TABLE intake_job ADD COLUMN {name} {ddl}")
|
con.execute(f"ALTER TABLE intake_job ADD COLUMN {name} {ddl}")
|
||||||
con.commit()
|
con.commit()
|
||||||
|
if _BOOTSTRAPPED_DB != str(DB):
|
||||||
|
_BOOTSTRAPPED_DB = str(DB)
|
||||||
|
try:
|
||||||
|
bootstrap_default_settings()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
return con
|
return con
|
||||||
|
|
||||||
|
|
||||||
@@ -69,7 +85,7 @@ def create_job(
|
|||||||
asr_model: str = "",
|
asr_model: str = "",
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
now = time.time()
|
now = time.time()
|
||||||
prov = (asr_provider or DEFAULT_PROVIDER or "qwen3cam").strip().lower()
|
prov = (asr_provider or CFG.asr_provider or "qwen3cam").strip().lower()
|
||||||
with connect() as con:
|
with connect() as con:
|
||||||
con.execute(
|
con.execute(
|
||||||
"INSERT INTO intake_job(id,original_name,stored_name,title,context,size,"
|
"INSERT INTO intake_job(id,original_name,stored_name,title,context,size,"
|
||||||
@@ -88,7 +104,7 @@ def update_job(job_id: str, status: str, detail: str = "", **fields: Any) -> boo
|
|||||||
raise ValueError(f"invalid status: {status}")
|
raise ValueError(f"invalid status: {status}")
|
||||||
allowed = {
|
allowed = {
|
||||||
"transcript", "draft", "result", "sha256", "attempts",
|
"transcript", "draft", "result", "sha256", "attempts",
|
||||||
"error_code", "asr_provider", "asr_model",
|
"error_code", "asr_provider", "asr_model", "title", "context",
|
||||||
}
|
}
|
||||||
changes = {k: v for k, v in fields.items() if k in allowed and v is not None}
|
changes = {k: v for k, v in fields.items() if k in allowed and v is not None}
|
||||||
sets = ["status=?", "updated_at=?", "detail=?"]
|
sets = ["status=?", "updated_at=?", "detail=?"]
|
||||||
@@ -106,6 +122,44 @@ def update_job(job_id: str, status: str, detail: str = "", **fields: Any) -> boo
|
|||||||
return bool(cur.rowcount)
|
return bool(cur.rowcount)
|
||||||
|
|
||||||
|
|
||||||
|
def update_job_meta(
|
||||||
|
job_id: str,
|
||||||
|
*,
|
||||||
|
title: str | None = None,
|
||||||
|
context: str | None = None,
|
||||||
|
asr_provider: str | None = None,
|
||||||
|
asr_model: str | None = None,
|
||||||
|
) -> bool:
|
||||||
|
sets = ["updated_at=?"]
|
||||||
|
values: list[Any] = [time.time()]
|
||||||
|
if title is not None:
|
||||||
|
sets.append("title=?")
|
||||||
|
values.append(title[:120])
|
||||||
|
if context is not None:
|
||||||
|
sets.append("context=?")
|
||||||
|
values.append(context[:2000])
|
||||||
|
if asr_provider is not None:
|
||||||
|
sets.append("asr_provider=?")
|
||||||
|
values.append(asr_provider[:40])
|
||||||
|
if asr_model is not None:
|
||||||
|
sets.append("asr_model=?")
|
||||||
|
values.append(asr_model[:80])
|
||||||
|
values.append(job_id)
|
||||||
|
with connect() as con:
|
||||||
|
cur = con.execute(f"UPDATE intake_job SET {','.join(sets)} WHERE id=?", values)
|
||||||
|
return bool(cur.rowcount)
|
||||||
|
|
||||||
|
|
||||||
|
def delete_job(job_id: str) -> dict[str, Any] | None:
|
||||||
|
job = get_job(job_id)
|
||||||
|
if not job:
|
||||||
|
return None
|
||||||
|
with connect() as con:
|
||||||
|
con.execute("DELETE FROM job_event WHERE job_id=?", (job_id,))
|
||||||
|
con.execute("DELETE FROM intake_job WHERE id=?", (job_id,))
|
||||||
|
return job
|
||||||
|
|
||||||
|
|
||||||
def get_job(job_id: str) -> dict[str, Any] | None:
|
def get_job(job_id: str) -> dict[str, Any] | None:
|
||||||
with connect() as con:
|
with connect() as con:
|
||||||
row = con.execute("SELECT * FROM intake_job WHERE id=?", (job_id,)).fetchone()
|
row = con.execute("SELECT * FROM intake_job WHERE id=?", (job_id,)).fetchone()
|
||||||
@@ -120,13 +174,27 @@ def find_by_stored(stored_name: str) -> dict[str, Any] | None:
|
|||||||
return dict(row) if row else None
|
return dict(row) if row else None
|
||||||
|
|
||||||
|
|
||||||
def list_jobs(limit: int = 30, offset: int = 0, status: str = "") -> list[dict[str, Any]]:
|
def list_jobs(
|
||||||
|
limit: int = 30,
|
||||||
|
offset: int = 0,
|
||||||
|
status: str = "",
|
||||||
|
q: str = "",
|
||||||
|
provider: str = "",
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
limit = min(max(limit, 1), 200)
|
limit = min(max(limit, 1), 200)
|
||||||
offset = max(offset, 0)
|
offset = max(offset, 0)
|
||||||
sql, args = "SELECT * FROM intake_job", []
|
sql = "SELECT * FROM intake_job WHERE 1=1"
|
||||||
|
args: list[Any] = []
|
||||||
if status:
|
if status:
|
||||||
sql += " WHERE status=?"
|
sql += " AND status=?"
|
||||||
args.append(status)
|
args.append(status)
|
||||||
|
if provider:
|
||||||
|
sql += " AND asr_provider=?"
|
||||||
|
args.append(provider)
|
||||||
|
if q.strip():
|
||||||
|
pat = f"%{q.strip()}%"
|
||||||
|
sql += " AND (title LIKE ? OR original_name LIKE ? OR context LIKE ? OR detail LIKE ?)"
|
||||||
|
args.extend([pat, pat, pat, pat])
|
||||||
sql += " ORDER BY created_at DESC LIMIT ? OFFSET ?"
|
sql += " ORDER BY created_at DESC LIMIT ? OFFSET ?"
|
||||||
args.extend((limit, offset))
|
args.extend((limit, offset))
|
||||||
with connect() as con:
|
with connect() as con:
|
||||||
@@ -145,3 +213,32 @@ def pending_recovery() -> list[dict[str, Any]]:
|
|||||||
return [dict(r) for r in con.execute(
|
return [dict(r) for r in con.execute(
|
||||||
"SELECT * FROM intake_job WHERE status IN ('queued','transcribing','refining','publishing') "
|
"SELECT * FROM intake_job WHERE status IN ('queued','transcribing','refining','publishing') "
|
||||||
"ORDER BY created_at").fetchall()]
|
"ORDER BY created_at").fetchall()]
|
||||||
|
|
||||||
|
|
||||||
|
def get_all_settings() -> dict[str, str]:
|
||||||
|
connect().close()
|
||||||
|
return load_db_settings(force=True)
|
||||||
|
|
||||||
|
|
||||||
|
def save_settings(updates: dict[str, Any]) -> dict[str, str]:
|
||||||
|
connect().close()
|
||||||
|
set_settings(updates)
|
||||||
|
return load_db_settings(force=True)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"STATUSES",
|
||||||
|
"connect",
|
||||||
|
"create_job",
|
||||||
|
"update_job",
|
||||||
|
"update_job_meta",
|
||||||
|
"delete_job",
|
||||||
|
"get_job",
|
||||||
|
"find_by_stored",
|
||||||
|
"list_jobs",
|
||||||
|
"events",
|
||||||
|
"pending_recovery",
|
||||||
|
"get_all_settings",
|
||||||
|
"save_settings",
|
||||||
|
"get_setting",
|
||||||
|
]
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -4,6 +4,7 @@ import fcntl
|
|||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import time
|
import time
|
||||||
@@ -11,9 +12,10 @@ import uuid
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from asrpipe import graph, lexicon, providers, refine, review, revise, vault
|
from asrpipe import graph, lexicon, providers, refine, review, revise, vault
|
||||||
|
from asrpipe.config import CFG
|
||||||
from . import db
|
from . import db
|
||||||
from .config import (
|
from .config import (
|
||||||
DATA, DEFAULT_PROVIDER, DRAFTS, FAILED, INBOX, LOGS, MAX_ATTEMPTS,
|
DATA, DRAFTS, FAILED, INBOX, LOGS, MAX_ATTEMPTS,
|
||||||
POLL_SECONDS, PROCESSED, PROCESSING, PUBLIC_URL, REVISION_QUEUE,
|
POLL_SECONDS, PROCESSED, PROCESSING, PUBLIC_URL, REVISION_QUEUE,
|
||||||
STABLE_SECONDS, STATE, SUPPORTED, TRANSCRIPTS, ensure_dirs,
|
STABLE_SECONDS, STATE, SUPPORTED, TRANSCRIPTS, ensure_dirs,
|
||||||
)
|
)
|
||||||
@@ -50,7 +52,7 @@ def transcribe(
|
|||||||
) -> tuple[Path, providers.TranscribeResult]:
|
) -> tuple[Path, providers.TranscribeResult]:
|
||||||
res = providers.transcribe_audio(
|
res = providers.transcribe_audio(
|
||||||
audio,
|
audio,
|
||||||
provider=provider or DEFAULT_PROVIDER,
|
provider=provider or CFG.asr_provider,
|
||||||
model=model,
|
model=model,
|
||||||
title=title,
|
title=title,
|
||||||
user_context=user_context,
|
user_context=user_context,
|
||||||
@@ -71,7 +73,7 @@ def transcribe(
|
|||||||
def process_audio(source: Path, job: dict) -> None:
|
def process_audio(source: Path, job: dict) -> None:
|
||||||
job_id = job["id"]
|
job_id = job["id"]
|
||||||
attempts = int(job.get("attempts") or 0) + 1
|
attempts = int(job.get("attempts") or 0) + 1
|
||||||
prov = providers.normalize_provider_name(job.get("asr_provider") or DEFAULT_PROVIDER)
|
prov = providers.normalize_provider_name(job.get("asr_provider") or CFG.asr_provider)
|
||||||
model = str(job.get("asr_model") or "")
|
model = str(job.get("asr_model") or "")
|
||||||
digest = sha256(source)
|
digest = sha256(source)
|
||||||
prov_label = providers.PROVIDERS.get(prov, {}).get("label", prov)
|
prov_label = providers.PROVIDERS.get(prov, {}).get("label", prov)
|
||||||
@@ -120,6 +122,47 @@ def process_audio(source: Path, job: dict) -> None:
|
|||||||
f"{PUBLIC_URL}/jobs/{job_id}/review/")
|
f"{PUBLIC_URL}/jobs/{job_id}/review/")
|
||||||
|
|
||||||
|
|
||||||
|
def rerefine_job(job: dict) -> None:
|
||||||
|
"""Re-run LLM refinement on an existing raw transcript without re-running ASR."""
|
||||||
|
job_id = job["id"]
|
||||||
|
transcript = Path(str(job.get("transcript") or ""))
|
||||||
|
if not transcript.is_file():
|
||||||
|
raise FileNotFoundError(f"transcript file missing for job {job_id}: {transcript}")
|
||||||
|
raw = transcript.read_text(encoding="utf-8", errors="ignore")
|
||||||
|
intake = {
|
||||||
|
"job_id": job_id,
|
||||||
|
"title": job.get("title", ""),
|
||||||
|
"context": job.get("context", ""),
|
||||||
|
"source": job.get("source", ""),
|
||||||
|
"asr_provider": job.get("asr_provider") or CFG.asr_provider,
|
||||||
|
"asr_model": job.get("asr_model") or "",
|
||||||
|
}
|
||||||
|
intake_line = f"<!-- intake: {json.dumps(intake, ensure_ascii=False)} -->"
|
||||||
|
if re.match(r"^<!--\s*intake:\s*\{.*?\}\s*-->", raw):
|
||||||
|
raw = re.sub(r"^<!--\s*intake:\s*\{.*?\}\s*-->", intake_line, raw, count=1)
|
||||||
|
else:
|
||||||
|
raw = intake_line + "\n" + raw
|
||||||
|
transcript.write_text(raw, encoding="utf-8")
|
||||||
|
|
||||||
|
db.update_job(job_id, "refining", "正在基于现有转写重新生成精修草稿")
|
||||||
|
review.delete_for_job(
|
||||||
|
transcript_path=str(transcript),
|
||||||
|
draft_path=str(job.get("draft") or ""),
|
||||||
|
job_id=job_id,
|
||||||
|
)
|
||||||
|
result = refine.refine(
|
||||||
|
raw, date=time.strftime("%Y-%m-%d"),
|
||||||
|
transcript_path=str(transcript), queue_review=True,
|
||||||
|
)
|
||||||
|
draft = DRAFTS / f"{job_id}.md"
|
||||||
|
vault.write_draft_file(draft, result.draft)
|
||||||
|
questions = review.enqueue_draft(str(transcript), draft)
|
||||||
|
db.update_job(
|
||||||
|
job_id, "review", f"整理稿已重新生成,{questions} 个问题待确认",
|
||||||
|
transcript=str(transcript), draft=str(draft),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def claim(path: Path) -> tuple[Path, dict]:
|
def claim(path: Path) -> tuple[Path, dict]:
|
||||||
job = db.find_by_stored(path.name)
|
job = db.find_by_stored(path.name)
|
||||||
if not job:
|
if not job:
|
||||||
@@ -127,7 +170,7 @@ def claim(path: Path) -> tuple[Path, dict]:
|
|||||||
job = db.create_job(
|
job = db.create_job(
|
||||||
job_id, path.name, path.name, title=path.stem,
|
job_id, path.name, path.name, title=path.stem,
|
||||||
size=path.stat().st_size, source="ssh",
|
size=path.stat().st_size, source="ssh",
|
||||||
asr_provider=DEFAULT_PROVIDER,
|
asr_provider=CFG.asr_provider,
|
||||||
)
|
)
|
||||||
target = PROCESSING / path.name
|
target = PROCESSING / path.name
|
||||||
if target.exists():
|
if target.exists():
|
||||||
@@ -164,8 +207,15 @@ def process_revisions() -> None:
|
|||||||
for marker in sorted(REVISION_QUEUE.glob("*.json")):
|
for marker in sorted(REVISION_QUEUE.glob("*.json")):
|
||||||
try:
|
try:
|
||||||
payload = json.loads(marker.read_text(encoding="utf-8"))
|
payload = json.loads(marker.read_text(encoding="utf-8"))
|
||||||
draft = Path(payload["draft"])
|
action = str(payload.get("action") or "revise")
|
||||||
job_id = str(payload.get("job_id") or "")
|
job_id = str(payload.get("job_id") or "")
|
||||||
|
if action == "rerefine":
|
||||||
|
job = db.get_job(job_id)
|
||||||
|
if job:
|
||||||
|
rerefine_job(job)
|
||||||
|
marker.unlink(missing_ok=True)
|
||||||
|
continue
|
||||||
|
draft = Path(payload["draft"])
|
||||||
db.update_job(job_id, "publishing", "正在应用人工确认")
|
db.update_job(job_id, "publishing", "正在应用人工确认")
|
||||||
revise.revise(draft)
|
revise.revise(draft)
|
||||||
current = db.get_job(job_id) or {}
|
current = db.get_job(job_id) or {}
|
||||||
@@ -181,6 +231,14 @@ def process_revisions() -> None:
|
|||||||
marker.unlink(missing_ok=True)
|
marker.unlink(missing_ok=True)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log(f"revision marker={marker.name} failed type={type(exc).__name__} err={exc}")
|
log(f"revision marker={marker.name} failed type={type(exc).__name__} err={exc}")
|
||||||
|
try:
|
||||||
|
payload = json.loads(marker.read_text(encoding="utf-8"))
|
||||||
|
jid = str(payload.get("job_id") or "")
|
||||||
|
if jid:
|
||||||
|
db.update_job(jid, "failed", f"精修/修订失败:{exc}", error_code=type(exc).__name__)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
marker.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
def recover() -> None:
|
def recover() -> None:
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import os
|
|||||||
import tempfile
|
import tempfile
|
||||||
import threading
|
import threading
|
||||||
import unittest
|
import unittest
|
||||||
|
import urllib.parse
|
||||||
import urllib.request
|
import urllib.request
|
||||||
from http.server import ThreadingHTTPServer
|
from http.server import ThreadingHTTPServer
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -20,6 +21,7 @@ os.environ["ASR_NOTIFY"] = "0"
|
|||||||
os.environ["ASR_GIT_SYNC"] = "0"
|
os.environ["ASR_GIT_SYNC"] = "0"
|
||||||
|
|
||||||
from asrpipe import lexicon, providers, review, segment # noqa: E402
|
from asrpipe import lexicon, providers, review, segment # noqa: E402
|
||||||
|
from asrpipe.config import CFG # noqa: E402
|
||||||
from asrservice import config as svc_cfg, db as svc_db, web as svc_web # noqa: E402
|
from asrservice import config as svc_cfg, db as svc_db, web as svc_web # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
@@ -33,6 +35,28 @@ class ProviderAndServiceTests(unittest.TestCase):
|
|||||||
def tearDown(self):
|
def tearDown(self):
|
||||||
self.tmp.cleanup()
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_sqlite_settings_override_config_dynamically(self):
|
||||||
|
svc_db.save_settings({
|
||||||
|
"asr_provider": "soniox",
|
||||||
|
"litellm_base_url": "https://api.opfo.xyz/v1",
|
||||||
|
"litellm_api_key": "sk-sqlite-dynamic-key",
|
||||||
|
"clean_model": "aistudio/gemini-flash-latest",
|
||||||
|
"draft_model": "openai/gpt-sol-latest",
|
||||||
|
"soniox_api_key": "soniox-db-secret",
|
||||||
|
"use_hindsight": "0",
|
||||||
|
})
|
||||||
|
self.assertEqual(CFG.asr_provider, "soniox")
|
||||||
|
self.assertEqual(CFG.litellm_base_url, "https://api.opfo.xyz/v1")
|
||||||
|
self.assertEqual(CFG.endpoint, "https://api.opfo.xyz/v1/chat/completions")
|
||||||
|
self.assertEqual(CFG.transcriptions_endpoint, "https://api.opfo.xyz/v1/audio/transcriptions")
|
||||||
|
self.assertEqual(CFG.api_key(), "sk-sqlite-dynamic-key")
|
||||||
|
self.assertEqual(CFG.clean_model, "aistudio/gemini-flash-latest")
|
||||||
|
self.assertEqual(CFG.draft_model, "openai/gpt-sol-latest")
|
||||||
|
self.assertEqual(CFG.soniox_api_key, "soniox-db-secret")
|
||||||
|
self.assertFalse(CFG.use_hindsight)
|
||||||
|
# Restore default for subsequent tests
|
||||||
|
svc_db.save_settings({"asr_provider": "qwen3cam", "use_hindsight": "1"})
|
||||||
|
|
||||||
def test_soniox_token_diarization_and_segment_parse(self):
|
def test_soniox_token_diarization_and_segment_parse(self):
|
||||||
payload = {
|
payload = {
|
||||||
"tokens": [
|
"tokens": [
|
||||||
@@ -135,20 +159,76 @@ class ProviderAndServiceTests(unittest.TestCase):
|
|||||||
updated = draft.read_text(encoding="utf-8")
|
updated = draft.read_text(encoding="utf-8")
|
||||||
self.assertIn("speaker_1: 张三", updated)
|
self.assertIn("speaker_1: 张三", updated)
|
||||||
|
|
||||||
def test_db_and_web_endpoints(self):
|
def test_lexicon_crud_operations(self):
|
||||||
job_id = "0123456789abcdef01234567"
|
# Entity CRUD
|
||||||
job = svc_db.create_job(
|
lexicon.upsert_entity("测试生物", "公司", company="测试集团", role="子公司", related="李四")
|
||||||
job_id=job_id,
|
ents = lexicon.list_entities(q="测试生物")
|
||||||
original_name="meeting.m4a",
|
self.assertEqual(len(ents), 1)
|
||||||
stored_name=f"20261102-{job_id}.m4a",
|
self.assertEqual(ents[0]["company"], "测试集团")
|
||||||
title="架构周会",
|
|
||||||
context="验证 Web API",
|
lexicon.update_entity("测试生物", "测试生物科技", "公司", company="新集团", role="总部", related="李四,王五")
|
||||||
|
ents2 = lexicon.list_entities(q="测试生物科技")
|
||||||
|
self.assertEqual(len(ents2), 1)
|
||||||
|
self.assertEqual(ents2[0]["company"], "新集团")
|
||||||
|
self.assertTrue(lexicon.delete_entity("测试生物科技"))
|
||||||
|
self.assertEqual(len(lexicon.list_entities(q="测试生物科技")), 0)
|
||||||
|
|
||||||
|
# Alias & Correction CRUD
|
||||||
|
lexicon.learn_alias("老测试", "测试人", kind="人名", origin="manual", note="初始")
|
||||||
|
lexicon.update_alias("老测试", "测试人", "小测试", "测试人", kind="人名", confirmed=1, note="已改")
|
||||||
|
aliases = lexicon.list_aliases(q="小测试")
|
||||||
|
self.assertEqual(len(aliases), 1)
|
||||||
|
self.assertEqual(aliases[0]["note"], "已改")
|
||||||
|
self.assertTrue(lexicon.delete_alias("小测试", "测试人"))
|
||||||
|
|
||||||
|
lexicon.learn_correction("误识词", "正词", note="c1")
|
||||||
|
lexicon.update_correction("误识词", "正词", "新误识词", "正词", confirmed=1, note="c2")
|
||||||
|
cors = lexicon.list_corrections(q="新误识词")
|
||||||
|
self.assertEqual(len(cors), 1)
|
||||||
|
self.assertEqual(cors[0]["note"], "c2")
|
||||||
|
self.assertTrue(lexicon.delete_correction("新误识词", "正词"))
|
||||||
|
|
||||||
|
# Relation CRUD
|
||||||
|
lexicon.upsert_relation("张三", "任职于", "洛启生物", confidence=0.8, source="manual")
|
||||||
|
lexicon.update_relation("张三", "任职于", "洛启生物", "manual", "张三", "实控人", "洛启生物", confidence=0.95, active=1)
|
||||||
|
rels = lexicon.list_relations(q="张三")
|
||||||
|
self.assertTrue(any(r["predicate"] == "实控人" for r in rels))
|
||||||
|
self.assertTrue(lexicon.delete_relation("张三", "实控人", "洛启生物", "manual"))
|
||||||
|
|
||||||
|
def test_db_web_endpoints_multi_review_and_job_delete(self):
|
||||||
|
job1_id = "0123456789abcdef01234567"
|
||||||
|
job2_id = "fedcba987654321001234567"
|
||||||
|
t1 = svc_cfg.TRANSCRIPTS / f"{job1_id}.txt"
|
||||||
|
t2 = svc_cfg.TRANSCRIPTS / f"{job2_id}.txt"
|
||||||
|
t1.write_text("【speaker_0】录音一内容\n", encoding="utf-8")
|
||||||
|
t2.write_text("【speaker_0】录音二内容\n", encoding="utf-8")
|
||||||
|
|
||||||
|
svc_db.create_job(
|
||||||
|
job_id=job1_id,
|
||||||
|
original_name="meeting1.m4a",
|
||||||
|
stored_name=f"20261102-{job1_id}.m4a",
|
||||||
|
title="第一场尽调会",
|
||||||
|
context="验证多录音确认",
|
||||||
size=2048,
|
size=2048,
|
||||||
asr_provider="soniox",
|
asr_provider="soniox",
|
||||||
asr_model="stt-async-v5",
|
asr_model="stt-async-v5",
|
||||||
)
|
)
|
||||||
self.assertEqual(job["asr_provider"], "soniox")
|
svc_db.update_job(job1_id, "review", "待确认", transcript=str(t1))
|
||||||
self.assertEqual(job["asr_model"], "stt-async-v5")
|
|
||||||
|
svc_db.create_job(
|
||||||
|
job_id=job2_id,
|
||||||
|
original_name="meeting2.m4a",
|
||||||
|
stored_name=f"20261102-{job2_id}.m4a",
|
||||||
|
title="第二场投决会",
|
||||||
|
context="验证批量操作",
|
||||||
|
size=4096,
|
||||||
|
asr_provider="qwen3cam",
|
||||||
|
)
|
||||||
|
svc_db.update_job(job2_id, "review", "待确认", transcript=str(t2))
|
||||||
|
|
||||||
|
q1 = review.enqueue(str(t1), "第一场问题A?", {"kind": "draft_question", "candidates": ["选项A1", "选项A2"]})
|
||||||
|
q2 = review.enqueue(str(t1), "第一场问题B?", {"kind": "draft_question", "candidates": ["选项B1"]})
|
||||||
|
q3 = review.enqueue(str(t2), "第二场问题C?", {"kind": "draft_question", "candidates": ["选项C1"]})
|
||||||
|
|
||||||
svc_web.AGENT_TOKEN = "test-agent-token"
|
svc_web.AGENT_TOKEN = "test-agent-token"
|
||||||
server = ThreadingHTTPServer(("127.0.0.1", 0), svc_web.Handler)
|
server = ThreadingHTTPServer(("127.0.0.1", 0), svc_web.Handler)
|
||||||
@@ -164,6 +244,17 @@ class ProviderAndServiceTests(unittest.TestCase):
|
|||||||
with urllib.request.urlopen(req, timeout=5) as resp:
|
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||||
return resp.status, resp.read().decode("utf-8")
|
return resp.status, resp.read().decode("utf-8")
|
||||||
|
|
||||||
|
def post_form(path: str, data: dict[str, str]) -> tuple[int, str]:
|
||||||
|
encoded = urllib.parse.urlencode(data).encode("utf-8")
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"http://127.0.0.1:{port}{path}",
|
||||||
|
data=encoded,
|
||||||
|
headers={"Content-Type": "application/x-www-form-urlencoded"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(req, timeout=5) as resp:
|
||||||
|
return resp.status, resp.read().decode("utf-8")
|
||||||
|
|
||||||
status, body = fetch("/health")
|
status, body = fetch("/health")
|
||||||
self.assertEqual(status, 200)
|
self.assertEqual(status, 200)
|
||||||
self.assertEqual(json.loads(body)["ok"], True)
|
self.assertEqual(json.loads(body)["ok"], True)
|
||||||
@@ -173,22 +264,57 @@ class ProviderAndServiceTests(unittest.TestCase):
|
|||||||
headers={"Authorization": "Bearer test-agent-token"},
|
headers={"Authorization": "Bearer test-agent-token"},
|
||||||
)
|
)
|
||||||
self.assertEqual(status, 200)
|
self.assertEqual(status, 200)
|
||||||
prov_data = json.loads(body)
|
prov_ids = [p["id"] for p in json.loads(body)["providers"]]
|
||||||
prov_ids = [p["id"] for p in prov_data["providers"]]
|
|
||||||
self.assertEqual(prov_ids, ["qwen3cam", "soniox", "gemini", "openai", "grok"])
|
self.assertEqual(prov_ids, ["qwen3cam", "soniox", "gemini", "openai", "grok"])
|
||||||
|
|
||||||
status, body = fetch(f"/jobs/{job_id}/")
|
# Multi-recording review page shows both recordings grouped
|
||||||
|
status, body = fetch("/review/")
|
||||||
self.assertEqual(status, 200)
|
self.assertEqual(status, 200)
|
||||||
self.assertIn("架构周会", body)
|
self.assertIn("第一场尽调会", body)
|
||||||
self.assertIn("soniox", body)
|
self.assertIn("第二场投决会", body)
|
||||||
|
self.assertIn("第一场问题A?", body)
|
||||||
|
self.assertIn("第二场问题C?", body)
|
||||||
|
|
||||||
status, body = fetch("/lexicon")
|
# Batch submit both questions for recording 1
|
||||||
|
status, body = post_form(f"/jobs/{job1_id}/review/", {
|
||||||
|
"question_ids": f"{q1},{q2}",
|
||||||
|
f"preset_{q1}": "选项A1",
|
||||||
|
f"custom_{q2}": "自定义回答B",
|
||||||
|
"batch_submit": "1",
|
||||||
|
})
|
||||||
self.assertEqual(status, 200)
|
self.assertEqual(status, 200)
|
||||||
self.assertIn("词库", body)
|
self.assertEqual(len(svc_web.pending(svc_db.get_job(job1_id))), 0)
|
||||||
|
self.assertEqual(len(svc_web.pending(svc_db.get_job(job2_id))), 1)
|
||||||
|
|
||||||
status, body = fetch("/settings")
|
# Edit job metadata
|
||||||
|
status, body = post_form(f"/jobs/{job1_id}/meta", {
|
||||||
|
"title": "第一场尽调会(已改名)",
|
||||||
|
"context": "更新后的背景",
|
||||||
|
"provider": "openai",
|
||||||
|
"model": "openai/gpt-4o-transcribe-diarize",
|
||||||
|
"next_action": "save",
|
||||||
|
})
|
||||||
self.assertEqual(status, 200)
|
self.assertEqual(status, 200)
|
||||||
self.assertIn("LiteLLM", body)
|
self.assertIn("第一场尽调会(已改名)", body)
|
||||||
|
|
||||||
|
# Save settings via Web UI POST
|
||||||
|
status, body = post_form("/settings", {
|
||||||
|
"litellm_base_url": "http://192.168.50.49:4000/v1",
|
||||||
|
"clean_model": "aistudio/gemini-flash-lite-latest",
|
||||||
|
"keyword_model": "aistudio/gemini-flash-lite-latest",
|
||||||
|
"draft_model": "codex/gpt-sol-latest",
|
||||||
|
"asr_provider": "soniox",
|
||||||
|
"use_hindsight": "1",
|
||||||
|
})
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
self.assertEqual(CFG.asr_provider, "soniox")
|
||||||
|
|
||||||
|
# Delete job 2 and verify its pending review question is also cleaned up
|
||||||
|
status, _ = post_form(f"/jobs/{job2_id}/delete", {"delete_files": "1"})
|
||||||
|
self.assertEqual(status, 200)
|
||||||
|
self.assertIsNone(svc_db.get_job(job2_id))
|
||||||
|
self.assertFalse(t2.exists())
|
||||||
|
self.assertNotIn(q3, [r["id"] for r in review.pending(200)])
|
||||||
finally:
|
finally:
|
||||||
server.shutdown()
|
server.shutdown()
|
||||||
server.server_close()
|
server.server_close()
|
||||||
|
|||||||
Reference in New Issue
Block a user