491 lines
18 KiB
Python
491 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Configuration for the standalone ASR post-processing pipeline and web service.
|
|
|
|
Model endpoints, provider selections, and API keys are stored in SQLite
|
|
(`app_setting` table in `ASR_STATE_DIR/intake.db`) so they can be managed
|
|
dynamically from the Web UI without restarting containers. Environment variables
|
|
serve as initial bootstrap defaults when a setting is not yet in SQLite.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
PKG = Path(__file__).resolve().parent
|
|
ROOT = PKG.parent
|
|
OBSIDIAN = Path(os.environ.get("ASR_WORKSPACE", str(ROOT)))
|
|
|
|
_CACHE_LOCK = threading.Lock()
|
|
_SETTINGS_CACHE: dict[str, str] = {}
|
|
_SETTINGS_TS: float = 0.0
|
|
_SETTINGS_DB_PATH: Path | None = None
|
|
_CACHE_TTL = 1.0
|
|
|
|
|
|
def _env(name: str, default: str) -> str:
|
|
v = os.environ.get(name)
|
|
return v if v not in (None, "") else default
|
|
|
|
|
|
def _envf(name: str, default: float) -> float:
|
|
try:
|
|
return float(os.environ["ASR_" + name])
|
|
except (KeyError, ValueError):
|
|
return default
|
|
|
|
|
|
def _envi(name: str, default: int) -> int:
|
|
try:
|
|
return int(os.environ["ASR_" + name])
|
|
except (KeyError, ValueError):
|
|
return default
|
|
|
|
|
|
def _state_db_path() -> Path:
|
|
state_dir = Path(_env("ASR_STATE_DIR", str(ROOT / "state")))
|
|
return state_dir / "intake.db"
|
|
|
|
|
|
def _ensure_settings_table(con: sqlite3.Connection) -> None:
|
|
con.execute("""
|
|
CREATE TABLE IF NOT EXISTS app_setting(
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT NOT NULL DEFAULT '',
|
|
updated_at REAL NOT NULL
|
|
)
|
|
""")
|
|
|
|
|
|
def load_db_settings(force: bool = False) -> dict[str, str]:
|
|
"""Read `app_setting` rows from `intake.db` with a 1s in-memory cache."""
|
|
global _SETTINGS_CACHE, _SETTINGS_TS, _SETTINGS_DB_PATH
|
|
db_path = _state_db_path()
|
|
now = time.time()
|
|
with _CACHE_LOCK:
|
|
if (
|
|
not force
|
|
and _SETTINGS_DB_PATH == db_path
|
|
and (now - _SETTINGS_TS) < _CACHE_TTL
|
|
):
|
|
return dict(_SETTINGS_CACHE)
|
|
if not db_path.exists():
|
|
_SETTINGS_CACHE = {}
|
|
_SETTINGS_TS = now
|
|
_SETTINGS_DB_PATH = db_path
|
|
return {}
|
|
try:
|
|
con = sqlite3.connect(db_path, timeout=5)
|
|
con.row_factory = sqlite3.Row
|
|
_ensure_settings_table(con)
|
|
rows = con.execute("SELECT key, value FROM app_setting").fetchall()
|
|
con.close()
|
|
_SETTINGS_CACHE = {str(r["key"]): str(r["value"]) for r in rows}
|
|
_SETTINGS_TS = now
|
|
_SETTINGS_DB_PATH = db_path
|
|
return dict(_SETTINGS_CACHE)
|
|
except Exception:
|
|
return dict(_SETTINGS_CACHE)
|
|
|
|
|
|
def get_setting(key: str, default: str = "") -> str:
|
|
data = load_db_settings()
|
|
val = data.get(key)
|
|
return val if val not in (None, "") else default
|
|
|
|
|
|
def set_settings(updates: dict[str, Any]) -> None:
|
|
"""Persist multiple settings into `intake.db` and invalidate cache."""
|
|
global _SETTINGS_TS
|
|
db_path = _state_db_path()
|
|
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
now = time.time()
|
|
con = sqlite3.connect(db_path, timeout=15)
|
|
try:
|
|
con.execute("PRAGMA journal_mode=WAL")
|
|
_ensure_settings_table(con)
|
|
for k, v in updates.items():
|
|
if v is None:
|
|
continue
|
|
con.execute(
|
|
"INSERT INTO app_setting(key, value, updated_at) VALUES(?,?,?) "
|
|
"ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at",
|
|
(str(k), str(v), now),
|
|
)
|
|
con.commit()
|
|
finally:
|
|
con.close()
|
|
with _CACHE_LOCK:
|
|
_SETTINGS_TS = 0.0
|
|
if "CFG" in globals():
|
|
for k in updates:
|
|
CFG.__dict__.pop(f"_override_{k}", None)
|
|
load_db_settings(force=True)
|
|
|
|
|
|
def bootstrap_default_settings() -> None:
|
|
"""Seed `app_setting` from environment variables for keys not yet stored."""
|
|
db_path = _state_db_path()
|
|
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
now = time.time()
|
|
defaults = {
|
|
"asr_provider": _env("ASR_PROVIDER", _env("MEETING_ASR_BACKEND", "qwen3cam")).lower(),
|
|
"diarize_url": _env("ASR_DIARIZE_URL", _env("MEETING_DIARIZE_URL", "http://192.168.50.100:10099/asr")),
|
|
"voiceprint_enroll_url": _env("ASR_VOICEPRINT_ENROLL_URL", "http://192.168.50.100:10099/voiceprints/enroll"),
|
|
"litellm_base_url": _env("LITELLM_BASE_URL", _env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")).rstrip("/"),
|
|
"litellm_api_key": _env("LITELLM_API_KEY", _env("ASR_LLM_API_KEY", _env("OPENAI_API_KEY", ""))),
|
|
"clean_model": _env("ASR_CLEAN_MODEL", "aistudio/gemini-flash-lite-latest"),
|
|
"keyword_model": _env("ASR_KEYWORD_MODEL", "aistudio/gemini-flash-lite-latest"),
|
|
"draft_model": _env("ASR_DRAFT_MODEL", "codex/gpt-sol-latest"),
|
|
"gemini_asr_model": _env("ASR_GEMINI_MODEL", "aistudio/gemini-3.5-transcribe"),
|
|
"openai_asr_model": _env("ASR_OPENAI_MODEL", "openai/gpt-4o-transcribe-diarize"),
|
|
"soniox_api_base": _env("SONIOX_API_BASE", "https://api.soniox.com/v1").rstrip("/"),
|
|
"soniox_api_key": _env("SONIOX_API_KEY", ""),
|
|
"soniox_model": _env("SONIOX_MODEL", "stt-async-v5"),
|
|
"grok_api_base": _env("XAI_API_BASE", "https://api.x.ai/v1").rstrip("/"),
|
|
"grok_api_key": _env("XAI_API_KEY", _env("GROK_API_KEY", "")),
|
|
"grok_asr_model": _env("ASR_GROK_MODEL", "grok-voice-transcribe-2.0"),
|
|
"hindsight_api": _env("HINDSIGHT_API", "http://192.168.50.47:8888").rstrip("/"),
|
|
"hindsight_bank": _env("HINDSIGHT_BANK", "personal-context"),
|
|
"use_hindsight": "1" if _env("ASR_HINDSIGHT", "1") != "0" else "0",
|
|
"use_termbank": "1" if _env("ASR_TERMBANK", "1") != "0" else "0",
|
|
"git_sync": "0" if _env("ASR_GIT_SYNC", "0") == "0" else "1",
|
|
}
|
|
con = sqlite3.connect(db_path, timeout=15)
|
|
try:
|
|
con.execute("PRAGMA journal_mode=WAL")
|
|
_ensure_settings_table(con)
|
|
for k, v in defaults.items():
|
|
con.execute(
|
|
"INSERT OR IGNORE INTO app_setting(key, value, updated_at) VALUES(?,?,?)",
|
|
(k, str(v), now),
|
|
)
|
|
# If key was seeded empty earlier but env now provides a non-empty secret, populate it
|
|
if v and k.endswith("_api_key"):
|
|
con.execute(
|
|
"UPDATE app_setting SET value=?, updated_at=? WHERE key=? AND value=''",
|
|
(str(v), now, k),
|
|
)
|
|
con.commit()
|
|
finally:
|
|
con.close()
|
|
load_db_settings(force=True)
|
|
|
|
|
|
class _SettingAttr:
|
|
"""Descriptor that resolves from instance override -> SQLite `app_setting` -> env fallback."""
|
|
|
|
def __init__(
|
|
self,
|
|
key: str,
|
|
fallback: Callable[[], str],
|
|
transform: Callable[[str], Any] | None = None,
|
|
) -> None:
|
|
self.key = key
|
|
self.fallback = fallback
|
|
self.transform = transform
|
|
self.override_attr = f"_override_{key}"
|
|
|
|
def __get__(self, instance: Any, owner: Any = None) -> Any:
|
|
if instance is None:
|
|
return self
|
|
if self.override_attr in instance.__dict__:
|
|
return instance.__dict__[self.override_attr]
|
|
raw = get_setting(self.key, self.fallback())
|
|
return self.transform(raw) if self.transform else raw
|
|
|
|
def __set__(self, instance: Any, value: Any) -> None:
|
|
instance.__dict__[self.override_attr] = value
|
|
|
|
def __delete__(self, instance: Any) -> None:
|
|
instance.__dict__.pop(self.override_attr, None)
|
|
|
|
|
|
def _default_vault_path() -> Path:
|
|
state_vault = _state_db_path().parent / "vault"
|
|
if (state_vault / ".git").exists():
|
|
return state_vault
|
|
env_vault = _env("ASR_VAULT", "")
|
|
if env_vault:
|
|
p = Path(env_vault)
|
|
if (p / ".git").exists() or not state_vault.exists():
|
|
return p
|
|
return Path.home() / "obsidian-vault"
|
|
|
|
|
|
@dataclass
|
|
class Config:
|
|
# ---- paths ----------------------------------------------------------
|
|
vault = _SettingAttr(
|
|
"vault_dir",
|
|
lambda: str(_default_vault_path()),
|
|
lambda s: Path(s),
|
|
)
|
|
obsidian: Path = field(default_factory=lambda: Path(
|
|
_env("ASR_WORKSPACE", str(ROOT))))
|
|
root: Path = ROOT
|
|
|
|
# ---- SQLite-backed dynamic model & provider settings ----------------
|
|
litellm_base_url = _SettingAttr(
|
|
"litellm_base_url",
|
|
lambda: _env("LITELLM_BASE_URL", _env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")),
|
|
lambda s: s.rstrip("/"),
|
|
)
|
|
clean_model = _SettingAttr(
|
|
"clean_model",
|
|
lambda: _env("ASR_CLEAN_MODEL", "aistudio/gemini-flash-lite-latest"),
|
|
)
|
|
draft_model = _SettingAttr(
|
|
"draft_model",
|
|
lambda: _env("ASR_DRAFT_MODEL", "codex/gpt-sol-latest"),
|
|
)
|
|
keyword_model = _SettingAttr(
|
|
"keyword_model",
|
|
lambda: _env("ASR_KEYWORD_MODEL", "aistudio/gemini-flash-lite-latest"),
|
|
)
|
|
|
|
asr_provider = _SettingAttr(
|
|
"asr_provider",
|
|
lambda: _env("ASR_PROVIDER", _env("MEETING_ASR_BACKEND", "qwen3cam")),
|
|
lambda s: s.strip().lower() or "qwen3cam",
|
|
)
|
|
diarize_url = _SettingAttr(
|
|
"diarize_url",
|
|
lambda: _env("ASR_DIARIZE_URL", _env("MEETING_DIARIZE_URL", "http://192.168.50.100:10099/asr")),
|
|
)
|
|
voiceprint_enroll_url = _SettingAttr(
|
|
"voiceprint_enroll_url",
|
|
lambda: _env("ASR_VOICEPRINT_ENROLL_URL", "http://192.168.50.100:10099/voiceprints/enroll"),
|
|
)
|
|
|
|
soniox_api_key = _SettingAttr(
|
|
"soniox_api_key",
|
|
lambda: _env("SONIOX_API_KEY", ""),
|
|
)
|
|
soniox_api_base = _SettingAttr(
|
|
"soniox_api_base",
|
|
lambda: _env("SONIOX_API_BASE", "https://api.soniox.com/v1"),
|
|
lambda s: s.rstrip("/"),
|
|
)
|
|
soniox_model = _SettingAttr(
|
|
"soniox_model",
|
|
lambda: _env("SONIOX_MODEL", "stt-async-v5"),
|
|
)
|
|
|
|
gemini_asr_model = _SettingAttr(
|
|
"gemini_asr_model",
|
|
lambda: _env("ASR_GEMINI_MODEL", "aistudio/gemini-3.5-transcribe"),
|
|
)
|
|
openai_asr_model = _SettingAttr(
|
|
"openai_asr_model",
|
|
lambda: _env("ASR_OPENAI_MODEL", "openai/gpt-4o-transcribe-diarize"),
|
|
)
|
|
|
|
grok_api_key = _SettingAttr(
|
|
"grok_api_key",
|
|
lambda: _env("XAI_API_KEY", _env("GROK_API_KEY", "")),
|
|
)
|
|
grok_api_base = _SettingAttr(
|
|
"grok_api_base",
|
|
lambda: _env("XAI_API_BASE", "https://api.x.ai/v1"),
|
|
lambda s: s.rstrip("/"),
|
|
)
|
|
grok_asr_model = _SettingAttr(
|
|
"grok_asr_model",
|
|
lambda: _env("ASR_GROK_MODEL", "grok-voice-transcribe-2.0"),
|
|
)
|
|
|
|
hindsight_api = _SettingAttr(
|
|
"hindsight_api",
|
|
lambda: _env("HINDSIGHT_API", "http://192.168.50.47:8888"),
|
|
lambda s: s.rstrip("/"),
|
|
)
|
|
hindsight_bank = _SettingAttr(
|
|
"hindsight_bank",
|
|
lambda: _env("HINDSIGHT_BANK", "personal-context"),
|
|
)
|
|
use_hindsight = _SettingAttr(
|
|
"use_hindsight",
|
|
lambda: _env("ASR_HINDSIGHT", "1"),
|
|
lambda s: str(s).lower() not in ("0", "false", "no", "off"),
|
|
)
|
|
use_termbank = _SettingAttr(
|
|
"use_termbank",
|
|
lambda: _env("ASR_TERMBANK", "1"),
|
|
lambda s: str(s).lower() not in ("0", "false", "no", "off"),
|
|
)
|
|
git_sync = _SettingAttr(
|
|
"git_sync",
|
|
lambda: _env("ASR_GIT_SYNC", "0"),
|
|
lambda s: str(s).lower() not in ("0", "false", "no", "off"),
|
|
)
|
|
|
|
@property
|
|
def endpoint(self) -> str:
|
|
if "_override_endpoint" in self.__dict__:
|
|
return self.__dict__["_override_endpoint"]
|
|
explicit = os.environ.get("ASR_ENDPOINT")
|
|
if explicit:
|
|
return explicit
|
|
return f"{self.litellm_base_url}/chat/completions"
|
|
|
|
@endpoint.setter
|
|
def endpoint(self, value: str) -> None:
|
|
self.__dict__["_override_endpoint"] = value
|
|
|
|
@property
|
|
def transcriptions_endpoint(self) -> str:
|
|
if "_override_transcriptions_endpoint" in self.__dict__:
|
|
return self.__dict__["_override_transcriptions_endpoint"]
|
|
explicit = os.environ.get("ASR_TRANSCRIPTIONS_ENDPOINT")
|
|
if explicit:
|
|
return explicit
|
|
return f"{self.litellm_base_url}/audio/transcriptions"
|
|
|
|
@transcriptions_endpoint.setter
|
|
def transcriptions_endpoint(self, value: str) -> None:
|
|
self.__dict__["_override_transcriptions_endpoint"] = value
|
|
|
|
concurrency: int = field(default_factory=lambda: _envi("CONCURRENCY", 6))
|
|
http_timeout: float = field(default_factory=lambda: _envf("HTTP_TIMEOUT", 420.0))
|
|
max_retries: int = field(default_factory=lambda: _envi("MAX_RETRIES", 5))
|
|
segment_seconds: int = field(default_factory=lambda: _envi(
|
|
"SEGMENT_SECONDS", int(_env("MEETING_ASR_SEGMENT_SECONDS", "540"))))
|
|
|
|
# ---- chunking -------------------------------------------------------
|
|
chunk_budget: int = field(default_factory=lambda: _envi("CHUNK_BUDGET", 1800))
|
|
chunk_overlap: int = field(default_factory=lambda: _envi("CHUNK_OVERLAP", 200))
|
|
collapse_noise: bool = field(
|
|
default_factory=lambda: _env("ASR_COLLAPSE_NOISE", "1") != "0")
|
|
|
|
# ---- fidelity gate --------------------------------------------------
|
|
gate_enabled: bool = field(
|
|
default_factory=lambda: _env("ASR_GATE", "1") != "0")
|
|
gate_min_ratio: float = field(
|
|
default_factory=lambda: _envf("GATE_MIN_RATIO", 0.55))
|
|
gate_max_ratio: float = field(
|
|
default_factory=lambda: _envf("GATE_MAX_RATIO", 1.35))
|
|
|
|
# ---- knowledge sources ----------------------------------------------
|
|
use_keyword_recall: bool = field(
|
|
default_factory=lambda: _env("ASR_KEYWORD_RECALL", "1") != "0")
|
|
use_lexicon: bool = field(
|
|
default_factory=lambda: _env("ASR_LEXICON", "1") != "0")
|
|
use_normalize: bool = field(
|
|
default_factory=lambda: _env("ASR_NORMALIZE", "1") != "0")
|
|
use_evidence: bool = field(
|
|
default_factory=lambda: _env("ASR_EVIDENCE", "1") != "0")
|
|
hindsight_timeout: float = field(
|
|
default_factory=lambda: _envf("HINDSIGHT_TIMEOUT", 12.0))
|
|
flag_min_score: float = field(
|
|
default_factory=lambda: _envf("FLAG_MIN_SCORE", 0.86))
|
|
review_min_score: float = field(
|
|
default_factory=lambda: _envf("REVIEW_MIN_SCORE", 0.82))
|
|
review_max_score: float = field(
|
|
default_factory=lambda: _envf("REVIEW_MAX_SCORE", 0.96))
|
|
review_host: str = field(
|
|
default_factory=lambda: _env("ASR_REVIEW_HOST", "0.0.0.0"))
|
|
review_public_host: str = field(
|
|
default_factory=lambda: _env("ASR_REVIEW_PUBLIC_HOST", "192.168.50.145"))
|
|
review_port: int = field(default_factory=lambda: _envi("REVIEW_PORT", 8765))
|
|
review_idle_seconds: int = field(
|
|
default_factory=lambda: _envi("REVIEW_IDLE_SECONDS", 2700))
|
|
review_max_seconds: int = field(
|
|
default_factory=lambda: _envi("REVIEW_MAX_SECONDS", 7200))
|
|
|
|
# ---- behaviour ------------------------------------------------------
|
|
notify: bool = field(
|
|
default_factory=lambda: _env("ASR_NOTIFY", "0") != "0")
|
|
cache_llm: bool = field(
|
|
default_factory=lambda: _env("ASR_CACHE", "0") != "0")
|
|
|
|
def __post_init__(self) -> None:
|
|
# Clear dataclass-populated default descriptors so __get__ is invoked dynamically
|
|
for k in list(self.__dict__.keys()):
|
|
if isinstance(getattr(type(self), k, None), _SettingAttr):
|
|
del self.__dict__[k]
|
|
|
|
@property
|
|
def inbox(self) -> Path:
|
|
return self.vault / ".asr-inbox"
|
|
|
|
@property
|
|
def state(self) -> Path:
|
|
return self.inbox / ".state"
|
|
|
|
@property
|
|
def drafts(self) -> Path:
|
|
"""Private rough drafts. They are never part of the Obsidian repository."""
|
|
return Path(_env("ASR_DRAFTS_DIR", str(self.state_dir / "drafts")))
|
|
|
|
@property
|
|
def minutes(self) -> Path:
|
|
return Path(_env("ASR_RESULTS_DIR", str(self.vault / "minutes")))
|
|
|
|
@property
|
|
def pending_index(self) -> Path:
|
|
return self.vault / "minutes" / "_drafts" / "_待确认.md"
|
|
|
|
@property
|
|
def glossary(self) -> Path:
|
|
return self.vault / "minutes" / "_glossary.md"
|
|
|
|
@property
|
|
def termbank_dir(self) -> Path:
|
|
return Path(_env("ASR_TERMBANK_DIR", str(self.root / "termbank")))
|
|
|
|
@property
|
|
def cache_dir(self) -> Path:
|
|
return Path(_env("ASR_CACHE_DIR", str(self.root / "cache")))
|
|
|
|
@property
|
|
def state_dir(self) -> Path:
|
|
"""Program-owned persistent state. Never inside the vault."""
|
|
return Path(_env("ASR_STATE_DIR", str(self.root / "state")))
|
|
|
|
@property
|
|
def lexicon_db(self) -> Path:
|
|
return Path(_env("ASR_LEXICON_DB", str(self.state_dir / "lexicon.db")))
|
|
|
|
@property
|
|
def log_file(self) -> Path:
|
|
return Path(_env("ASR_LOG", str(self.obsidian / "logs" / "asr-pipeline.log")))
|
|
|
|
@property
|
|
def notifier(self) -> Path:
|
|
return Path(_env("ASR_NOTIFIER", str(ROOT / "scripts" / "meeting-status-notify.py")))
|
|
|
|
@property
|
|
def env_file(self) -> Path:
|
|
"""600-perm credentials file."""
|
|
return Path(_env("ASR_ENV_FILE", str(ROOT / ".env")))
|
|
|
|
def api_key(self) -> str:
|
|
db_key = get_setting("litellm_api_key", "")
|
|
if db_key:
|
|
return db_key
|
|
for k in ("LITELLM_API_KEY", "ASR_LLM_API_KEY", "OPENAI_API_KEY",
|
|
"ANTHROPIC_AUTH_TOKEN"):
|
|
v = os.environ.get(k)
|
|
if v:
|
|
return v
|
|
p = self.env_file
|
|
if p.exists():
|
|
for line in p.read_text(errors="ignore").splitlines():
|
|
line = line.strip()
|
|
for prefix in ("LITELLM_API_KEY=", "ASR_LLM_API_KEY=",
|
|
"OPENAI_API_KEY=", "ANTHROPIC_AUTH_TOKEN="):
|
|
if line.startswith(prefix):
|
|
val = line.split("=", 1)[1].strip().strip("'\"")
|
|
if val:
|
|
return val
|
|
raise RuntimeError(
|
|
"no API key: configure LiteLLM API Key in /settings or set LITELLM_API_KEY")
|
|
|
|
|
|
CFG = Config()
|