feat(tools-pve/asr-pipeline): SQLite model config, task delete/rerefine, multi-recording review, lexicon CRUD

2026-11-01 23:14:31 -08:00
parent 2367c5eb4e
commit be589ff5e0
7 changed files with 2064 additions and 340 deletions
@@ -1,22 +1,31 @@
 #!/usr/bin/env python3
 """Configuration for the standalone ASR post-processing pipeline and web service.

-This program has **no dependency on Claude Code, opencode, or any agent CLI**.
-It talks to LiteLLM / model providers over plain HTTPS and does its own file
-and git handling, so it can run inside Docker (via Arcane), systemd, or a shell.
-
-Everything is overridable by environment variable.
+Model endpoints, provider selections, and API keys are stored in SQLite
+(`app_setting` table in `ASR_STATE_DIR/intake.db`) so they can be managed
+dynamically from the Web UI without restarting containers. Environment variables
+serve as initial bootstrap defaults when a setting is not yet in SQLite.
 """
 from __future__ import annotations

 import os
+import sqlite3
+import threading
+import time
 from dataclasses import dataclass, field
 from pathlib import Path
+from typing import Any, Callable

 PKG = Path(__file__).resolve().parent
 ROOT = PKG.parent
 OBSIDIAN = Path(os.environ.get("ASR_WORKSPACE", str(ROOT)))

+_CACHE_LOCK = threading.Lock()
+_SETTINGS_CACHE: dict[str, str] = {}
+_SETTINGS_TS: float = 0.0
+_SETTINGS_DB_PATH: Path | None = None
+_CACHE_TTL = 1.0
+

 def _env(name: str, default: str) -> str:
    v = os.environ.get(name)
@@ -37,6 +46,165 @@ def _envi(name: str, default: int) -> int:
        return default


+def _state_db_path() -> Path:
+    state_dir = Path(_env("ASR_STATE_DIR", str(ROOT / "state")))
+    return state_dir / "intake.db"
+
+
+def _ensure_settings_table(con: sqlite3.Connection) -> None:
+    con.execute("""
+      CREATE TABLE IF NOT EXISTS app_setting(
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL DEFAULT '',
+        updated_at REAL NOT NULL
+      )
+    """)
+
+
+def load_db_settings(force: bool = False) -> dict[str, str]:
+    """Read `app_setting` rows from `intake.db` with a 1s in-memory cache."""
+    global _SETTINGS_CACHE, _SETTINGS_TS, _SETTINGS_DB_PATH
+    db_path = _state_db_path()
+    now = time.time()
+    with _CACHE_LOCK:
+        if (
+            not force
+            and _SETTINGS_DB_PATH == db_path
+            and (now - _SETTINGS_TS) < _CACHE_TTL
+        ):
+            return dict(_SETTINGS_CACHE)
+        if not db_path.exists():
+            _SETTINGS_CACHE = {}
+            _SETTINGS_TS = now
+            _SETTINGS_DB_PATH = db_path
+            return {}
+        try:
+            con = sqlite3.connect(db_path, timeout=5)
+            con.row_factory = sqlite3.Row
+            _ensure_settings_table(con)
+            rows = con.execute("SELECT key, value FROM app_setting").fetchall()
+            con.close()
+            _SETTINGS_CACHE = {str(r["key"]): str(r["value"]) for r in rows}
+            _SETTINGS_TS = now
+            _SETTINGS_DB_PATH = db_path
+            return dict(_SETTINGS_CACHE)
+        except Exception:
+            return dict(_SETTINGS_CACHE)
+
+
+def get_setting(key: str, default: str = "") -> str:
+    data = load_db_settings()
+    val = data.get(key)
+    return val if val not in (None, "") else default
+
+
+def set_settings(updates: dict[str, Any]) -> None:
+    """Persist multiple settings into `intake.db` and invalidate cache."""
+    global _SETTINGS_TS
+    db_path = _state_db_path()
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    now = time.time()
+    con = sqlite3.connect(db_path, timeout=15)
+    try:
+        con.execute("PRAGMA journal_mode=WAL")
+        _ensure_settings_table(con)
+        for k, v in updates.items():
+            if v is None:
+                continue
+            con.execute(
+                "INSERT INTO app_setting(key, value, updated_at) VALUES(?,?,?) "
+                "ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at",
+                (str(k), str(v), now),
+            )
+        con.commit()
+    finally:
+        con.close()
+    with _CACHE_LOCK:
+        _SETTINGS_TS = 0.0
+    if "CFG" in globals():
+        for k in updates:
+            CFG.__dict__.pop(f"_override_{k}", None)
+    load_db_settings(force=True)
+
+
+def bootstrap_default_settings() -> None:
+    """Seed `app_setting` from environment variables for keys not yet stored."""
+    db_path = _state_db_path()
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+    now = time.time()
+    defaults = {
+        "asr_provider": _env("ASR_PROVIDER", _env("MEETING_ASR_BACKEND", "qwen3cam")).lower(),
+        "diarize_url": _env("ASR_DIARIZE_URL", _env("MEETING_DIARIZE_URL", "http://192.168.50.100:10099/asr")),
+        "voiceprint_enroll_url": _env("ASR_VOICEPRINT_ENROLL_URL", "http://192.168.50.100:10099/voiceprints/enroll"),
+        "litellm_base_url": _env("LITELLM_BASE_URL", _env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")).rstrip("/"),
+        "litellm_api_key": _env("LITELLM_API_KEY", _env("ASR_LLM_API_KEY", _env("OPENAI_API_KEY", ""))),
+        "clean_model": _env("ASR_CLEAN_MODEL", "aistudio/gemini-flash-lite-latest"),
+        "keyword_model": _env("ASR_KEYWORD_MODEL", "aistudio/gemini-flash-lite-latest"),
+        "draft_model": _env("ASR_DRAFT_MODEL", "codex/gpt-sol-latest"),
+        "gemini_asr_model": _env("ASR_GEMINI_MODEL", "aistudio/gemini-3.5-transcribe"),
+        "openai_asr_model": _env("ASR_OPENAI_MODEL", "openai/gpt-4o-transcribe-diarize"),
+        "soniox_api_base": _env("SONIOX_API_BASE", "https://api.soniox.com/v1").rstrip("/"),
+        "soniox_api_key": _env("SONIOX_API_KEY", ""),
+        "soniox_model": _env("SONIOX_MODEL", "stt-async-v5"),
+        "grok_api_base": _env("XAI_API_BASE", "https://api.x.ai/v1").rstrip("/"),
+        "grok_api_key": _env("XAI_API_KEY", _env("GROK_API_KEY", "")),
+        "grok_asr_model": _env("ASR_GROK_MODEL", "grok-voice-transcribe-2.0"),
+        "hindsight_api": _env("HINDSIGHT_API", "http://192.168.50.47:8888").rstrip("/"),
+        "hindsight_bank": _env("HINDSIGHT_BANK", "personal-context"),
+        "use_hindsight": "1" if _env("ASR_HINDSIGHT", "1") != "0" else "0",
+        "use_termbank": "1" if _env("ASR_TERMBANK", "1") != "0" else "0",
+        "git_sync": "1" if _env("ASR_GIT_SYNC", "0") != "0" else "0",
+    }
+    con = sqlite3.connect(db_path, timeout=15)
+    try:
+        con.execute("PRAGMA journal_mode=WAL")
+        _ensure_settings_table(con)
+        for k, v in defaults.items():
+            con.execute(
+                "INSERT OR IGNORE INTO app_setting(key, value, updated_at) VALUES(?,?,?)",
+                (k, str(v), now),
+            )
+            # If key was seeded empty earlier but env now provides a non-empty secret, populate it
+            if v and k.endswith("_api_key"):
+                con.execute(
+                    "UPDATE app_setting SET value=?, updated_at=? WHERE key=? AND value=''",
+                    (str(v), now, k),
+                )
+        con.commit()
+    finally:
+        con.close()
+    load_db_settings(force=True)
+
+
+class _SettingAttr:
+    """Descriptor that resolves from instance override -> SQLite `app_setting` -> env fallback."""
+
+    def __init__(
+        self,
+        key: str,
+        fallback: Callable[[], str],
+        transform: Callable[[str], Any] | None = None,
+    ) -> None:
+        self.key = key
+        self.fallback = fallback
+        self.transform = transform
+        self.override_attr = f"_override_{key}"
+
+    def __get__(self, instance: Any, owner: Any = None) -> Any:
+        if instance is None:
+            return self
+        if self.override_attr in instance.__dict__:
+            return instance.__dict__[self.override_attr]
+        raw = get_setting(self.key, self.fallback())
+        return self.transform(raw) if self.transform else raw
+
+    def __set__(self, instance: Any, value: Any) -> None:
+        instance.__dict__[self.override_attr] = value
+
+    def __delete__(self, instance: Any) -> None:
+        instance.__dict__.pop(self.override_attr, None)
+
+
@dataclass
 class Config:
    # ---- paths ----------------------------------------------------------
@@ -46,61 +214,130 @@ class Config:
        _env("ASR_WORKSPACE", str(ROOT))))
    root: Path = ROOT

-    # ---- LiteLLM / LLM routing (ZenMux deprecated) ----------------------
-    litellm_base_url: str = field(default_factory=lambda: _env(
-        "LITELLM_BASE_URL",
-        _env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")).rstrip("/"))
+    # ---- SQLite-backed dynamic model & provider settings ----------------
+    litellm_base_url = _SettingAttr(
+        "litellm_base_url",
+        lambda: _env("LITELLM_BASE_URL", _env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")),
+        lambda s: s.rstrip("/"),
+    )
+    clean_model = _SettingAttr(
+        "clean_model",
+        lambda: _env("ASR_CLEAN_MODEL", "aistudio/gemini-flash-lite-latest"),
+    )
+    draft_model = _SettingAttr(
+        "draft_model",
+        lambda: _env("ASR_DRAFT_MODEL", "codex/gpt-sol-latest"),
+    )
+    keyword_model = _SettingAttr(
+        "keyword_model",
+        lambda: _env("ASR_KEYWORD_MODEL", "aistudio/gemini-flash-lite-latest"),
+    )

-    clean_model: str = field(default_factory=lambda: _env(
-        "ASR_CLEAN_MODEL", "aistudio/gemini-flash-lite-latest"))
-    draft_model: str = field(default_factory=lambda: _env(
-        "ASR_DRAFT_MODEL", "codex/gpt-sol-latest"))
-    keyword_model: str = field(default_factory=lambda: _env(
-        "ASR_KEYWORD_MODEL", "aistudio/gemini-flash-lite-latest"))
+    asr_provider = _SettingAttr(
+        "asr_provider",
+        lambda: _env("ASR_PROVIDER", _env("MEETING_ASR_BACKEND", "qwen3cam")),
+        lambda s: s.strip().lower() or "qwen3cam",
+    )
+    diarize_url = _SettingAttr(
+        "diarize_url",
+        lambda: _env("ASR_DIARIZE_URL", _env("MEETING_DIARIZE_URL", "http://192.168.50.100:10099/asr")),
+    )
+    voiceprint_enroll_url = _SettingAttr(
+        "voiceprint_enroll_url",
+        lambda: _env("ASR_VOICEPRINT_ENROLL_URL", "http://192.168.50.100:10099/voiceprints/enroll"),
+    )

-    endpoint: str = field(default_factory=lambda: _env(
-        "ASR_ENDPOINT",
-        _env("LITELLM_BASE_URL",
-             _env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")).rstrip("/")
-        + "/chat/completions"))
-    transcriptions_endpoint: str = field(default_factory=lambda: _env(
-        "ASR_TRANSCRIPTIONS_ENDPOINT",
-        _env("LITELLM_BASE_URL",
-             _env("ASR_LLM_BASE_URL", "http://192.168.50.49:4000/v1")).rstrip("/")
-        + "/audio/transcriptions"))
+    soniox_api_key = _SettingAttr(
+        "soniox_api_key",
+        lambda: _env("SONIOX_API_KEY", ""),
+    )
+    soniox_api_base = _SettingAttr(
+        "soniox_api_base",
+        lambda: _env("SONIOX_API_BASE", "https://api.soniox.com/v1"),
+        lambda s: s.rstrip("/"),
+    )
+    soniox_model = _SettingAttr(
+        "soniox_model",
+        lambda: _env("SONIOX_MODEL", "stt-async-v5"),
+    )
+
+    gemini_asr_model = _SettingAttr(
+        "gemini_asr_model",
+        lambda: _env("ASR_GEMINI_MODEL", "aistudio/gemini-3.5-transcribe"),
+    )
+    openai_asr_model = _SettingAttr(
+        "openai_asr_model",
+        lambda: _env("ASR_OPENAI_MODEL", "openai/gpt-4o-transcribe-diarize"),
+    )
+
+    grok_api_key = _SettingAttr(
+        "grok_api_key",
+        lambda: _env("XAI_API_KEY", _env("GROK_API_KEY", "")),
+    )
+    grok_api_base = _SettingAttr(
+        "grok_api_base",
+        lambda: _env("XAI_API_BASE", "https://api.x.ai/v1"),
+        lambda s: s.rstrip("/"),
+    )
+    grok_asr_model = _SettingAttr(
+        "grok_asr_model",
+        lambda: _env("ASR_GROK_MODEL", "grok-voice-transcribe-2.0"),
+    )
+
+    hindsight_api = _SettingAttr(
+        "hindsight_api",
+        lambda: _env("HINDSIGHT_API", "http://192.168.50.47:8888"),
+        lambda s: s.rstrip("/"),
+    )
+    hindsight_bank = _SettingAttr(
+        "hindsight_bank",
+        lambda: _env("HINDSIGHT_BANK", "personal-context"),
+    )
+    use_hindsight = _SettingAttr(
+        "use_hindsight",
+        lambda: _env("ASR_HINDSIGHT", "1"),
+        lambda s: str(s).lower() not in ("0", "false", "no", "off"),
+    )
+    use_termbank = _SettingAttr(
+        "use_termbank",
+        lambda: _env("ASR_TERMBANK", "1"),
+        lambda s: str(s).lower() not in ("0", "false", "no", "off"),
+    )
+    git_sync = _SettingAttr(
+        "git_sync",
+        lambda: _env("ASR_GIT_SYNC", "1"),
+        lambda s: str(s).lower() not in ("0", "false", "no", "off"),
+    )
+
+    @property
+    def endpoint(self) -> str:
+        if "_override_endpoint" in self.__dict__:
+            return self.__dict__["_override_endpoint"]
+        explicit = os.environ.get("ASR_ENDPOINT")
+        if explicit:
+            return explicit
+        return f"{self.litellm_base_url}/chat/completions"
+
+    @endpoint.setter
+    def endpoint(self, value: str) -> None:
+        self.__dict__["_override_endpoint"] = value
+
+    @property
+    def transcriptions_endpoint(self) -> str:
+        if "_override_transcriptions_endpoint" in self.__dict__:
+            return self.__dict__["_override_transcriptions_endpoint"]
+        explicit = os.environ.get("ASR_TRANSCRIPTIONS_ENDPOINT")
+        if explicit:
+            return explicit
+        return f"{self.litellm_base_url}/audio/transcriptions"
+
+    @transcriptions_endpoint.setter
+    def transcriptions_endpoint(self, value: str) -> None:
+        self.__dict__["_override_transcriptions_endpoint"] = value

    concurrency: int = field(default_factory=lambda: _envi("CONCURRENCY", 6))
    http_timeout: float = field(default_factory=lambda: _envf("HTTP_TIMEOUT", 420.0))
    max_retries: int = field(default_factory=lambda: _envi("MAX_RETRIES", 5))
-
-    # ---- ASR providers --------------------------------------------------
-    asr_provider: str = field(default_factory=lambda: _env(
-        "ASR_PROVIDER", _env("MEETING_ASR_BACKEND", "qwen3cam")).lower())
-    diarize_url: str = field(default_factory=lambda: _env(
-        "ASR_DIARIZE_URL",
-        _env("MEETING_DIARIZE_URL", "http://192.168.50.100:10099/asr")))
-    voiceprint_enroll_url: str = field(default_factory=lambda: _env(
-        "ASR_VOICEPRINT_ENROLL_URL",
-        "http://192.168.50.100:10099/voiceprints/enroll"))
-
-    soniox_api_key: str = field(default_factory=lambda: _env("SONIOX_API_KEY", ""))
-    soniox_api_base: str = field(default_factory=lambda: _env(
-        "SONIOX_API_BASE", "https://api.soniox.com/v1").rstrip("/"))
-    soniox_model: str = field(default_factory=lambda: _env(
-        "SONIOX_MODEL", "stt-async-v5"))
-
-    gemini_asr_model: str = field(default_factory=lambda: _env(
-        "ASR_GEMINI_MODEL", "aistudio/gemini-3.5-transcribe"))
-    openai_asr_model: str = field(default_factory=lambda: _env(
-        "ASR_OPENAI_MODEL", "openai/gpt-4o-transcribe-diarize"))
-
-    grok_api_key: str = field(default_factory=lambda: _env(
-        "XAI_API_KEY", _env("GROK_API_KEY", "")))
-    grok_api_base: str = field(default_factory=lambda: _env(
-        "XAI_API_BASE", "https://api.x.ai/v1").rstrip("/"))
-    grok_asr_model: str = field(default_factory=lambda: _env(
-        "ASR_GROK_MODEL", "grok-voice-transcribe-2.0"))
-
    segment_seconds: int = field(default_factory=lambda: _envi(
        "SEGMENT_SECONDS", int(_env("MEETING_ASR_SEGMENT_SECONDS", "540"))))

@@ -119,10 +356,6 @@ class Config:
        default_factory=lambda: _envf("GATE_MAX_RATIO", 1.35))

    # ---- knowledge sources ----------------------------------------------
-    use_termbank: bool = field(
-        default_factory=lambda: _env("ASR_TERMBANK", "1") != "0")
-    use_hindsight: bool = field(
-        default_factory=lambda: _env("ASR_HINDSIGHT", "1") != "0")
    use_keyword_recall: bool = field(
        default_factory=lambda: _env("ASR_KEYWORD_RECALL", "1") != "0")
    use_lexicon: bool = field(
@@ -131,10 +364,6 @@ class Config:
        default_factory=lambda: _env("ASR_NORMALIZE", "1") != "0")
    use_evidence: bool = field(
        default_factory=lambda: _env("ASR_EVIDENCE", "1") != "0")
-    hindsight_api: str = field(default_factory=lambda: _env(
-        "HINDSIGHT_API", "http://192.168.50.47:8888"))
-    hindsight_bank: str = field(default_factory=lambda: _env(
-        "HINDSIGHT_BANK", "personal-context"))
    hindsight_timeout: float = field(
        default_factory=lambda: _envf("HINDSIGHT_TIMEOUT", 12.0))
    flag_min_score: float = field(
@@ -154,13 +383,17 @@ class Config:
        default_factory=lambda: _envi("REVIEW_MAX_SECONDS", 7200))

    # ---- behaviour ------------------------------------------------------
-    git_sync: bool = field(
-        default_factory=lambda: _env("ASR_GIT_SYNC", "1") != "0")
    notify: bool = field(
        default_factory=lambda: _env("ASR_NOTIFY", "1") != "0")
    cache_llm: bool = field(
        default_factory=lambda: _env("ASR_CACHE", "0") != "0")

+    def __post_init__(self) -> None:
+        # Clear dataclass-populated default descriptors so __get__ is invoked dynamically
+        for k in list(self.__dict__.keys()):
+            if isinstance(getattr(type(self), k, None), _SettingAttr):
+                del self.__dict__[k]
+
    @property
    def inbox(self) -> Path:
        return self.vault / ".asr-inbox"
@@ -217,6 +450,9 @@ class Config:
        return Path(_env("ASR_ENV_FILE", str(ROOT / ".env")))

    def api_key(self) -> str:
+        db_key = get_setting("litellm_api_key", "")
+        if db_key:
+            return db_key
        for k in ("LITELLM_API_KEY", "ASR_LLM_API_KEY", "OPENAI_API_KEY",
                  "ANTHROPIC_AUTH_TOKEN"):
            v = os.environ.get(k)
@@ -233,8 +469,7 @@ class Config:
                        if val:
                            return val
        raise RuntimeError(
-            "no API key: set LITELLM_API_KEY / ASR_LLM_API_KEY / OPENAI_API_KEY "
-            f"or provide {p}")
+            "no API key: configure LiteLLM API Key in /settings or set LITELLM_API_KEY")


 CFG = Config()