Kai
88b06d782f
The curator Python backend (package, tests, systemd units, config templates, scripts) now lives under scenarios/curator/backend and is the single source of truth; the live checkout at the workspace path is a runtime copy. Exported from the app repo's tracked tree via git archive (no history, .pi/venv/caches excluded). 149 unit tests pass from the new location. profile.toml backend is now repo-relative (scenarios/curator/backend); verify-generated.sh resolves a relative backend against REPO_ROOT. verify-no-secrets ASSIGN heuristic now requires value entropy so vendored kwargs like token=extraction_token no longer false-positive. README documents the backend/ layout and the operator-owned app rollout step.
201 lines
8.0 KiB
Python
201 lines
8.0 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from html.parser import HTMLParser
|
|
import json
|
|
import urllib.parse
|
|
import urllib.request
|
|
from typing import Any
|
|
|
|
from .config import Settings
|
|
from .db import Database
|
|
|
|
|
|
class _DuckDuckGoParser(HTMLParser):
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.results: list[dict[str, str]] = []
|
|
self._kind = ""
|
|
self._href = ""
|
|
self._text: list[str] = []
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
values = dict(attrs)
|
|
classes = set((values.get("class") or "").split())
|
|
if tag == "a" and "result__a" in classes:
|
|
self._kind = "title"
|
|
self._href = values.get("href") or ""
|
|
self._text = []
|
|
elif tag in {"a", "div"} and "result__snippet" in classes:
|
|
self._kind = "snippet"
|
|
self._text = []
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
if self._kind:
|
|
self._text.append(data)
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
if not self._kind or tag not in {"a", "div"}:
|
|
return
|
|
text = " ".join("".join(self._text).split())
|
|
if self._kind == "title" and text:
|
|
self.results.append({"title": text, "href": self._href, "snippet": ""})
|
|
elif self._kind == "snippet" and text and self.results:
|
|
self.results[-1]["snippet"] = text
|
|
self._kind = ""
|
|
self._href = ""
|
|
self._text = []
|
|
|
|
|
|
class WebSearchProvider:
|
|
"""Search public web evidence through Tavily with a DuckDuckGo fallback."""
|
|
|
|
def __init__(self, settings: Settings, database: Database):
|
|
self.settings = settings
|
|
self.database = database
|
|
|
|
@staticmethod
|
|
def _cache_key(plan: dict[str, Any]) -> str:
|
|
value = json.dumps(
|
|
{
|
|
"title": plan.get("title") or "",
|
|
"creator": plan.get("creator") or "",
|
|
"aliases": plan.get("aliases") or [],
|
|
},
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
)
|
|
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
|
|
def _tavily(self, query: str, max_results: int | None = None) -> list[dict[str, Any]]:
|
|
if not self.settings.tavily_api_key:
|
|
raise RuntimeError("未配置 CURATOR_TAVILY_API_KEY")
|
|
payload = json.dumps({
|
|
"api_key": self.settings.tavily_api_key,
|
|
"query": query,
|
|
"search_depth": "advanced",
|
|
"topic": "general",
|
|
"max_results": max(
|
|
1,
|
|
min(max_results or self.settings.book_web_review_max_results, 10),
|
|
),
|
|
"include_answer": False,
|
|
"include_raw_content": False,
|
|
}).encode("utf-8")
|
|
request = urllib.request.Request(
|
|
self.settings.tavily_search_url,
|
|
data=payload,
|
|
headers={"Content-Type": "application/json", "User-Agent": "Curator/0.3"},
|
|
method="POST",
|
|
)
|
|
with urllib.request.urlopen(request, timeout=30) as response:
|
|
value = json.load(response)
|
|
evidence: list[dict[str, Any]] = []
|
|
seen: set[str] = set()
|
|
for raw in value.get("results") or []:
|
|
url = str(raw.get("url") or "").strip()
|
|
title = str(raw.get("title") or "").strip()
|
|
if not url or not title or url in seen:
|
|
continue
|
|
host = urllib.parse.urlparse(url).hostname or ""
|
|
if host.endswith(("zlib.li", "singlelogin.re")):
|
|
continue
|
|
seen.add(url)
|
|
evidence.append({
|
|
"provider": "tavily",
|
|
"title": title,
|
|
"url": url,
|
|
"domain": host.removeprefix("www."),
|
|
"snippet": str(raw.get("content") or "").strip()[:1600],
|
|
"relevance": raw.get("score"),
|
|
"published_at": str(raw.get("published_date") or ""),
|
|
})
|
|
return evidence
|
|
|
|
def _duckduckgo(self, query: str, max_results: int | None = None) -> list[dict[str, Any]]:
|
|
url = "https://html.duckduckgo.com/html/?" + urllib.parse.urlencode({"q": query})
|
|
request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 Curator/0.3"})
|
|
with urllib.request.urlopen(request, timeout=30) as response:
|
|
parser = _DuckDuckGoParser()
|
|
parser.feed(response.read().decode("utf-8", errors="replace"))
|
|
evidence: list[dict[str, Any]] = []
|
|
blocked = ("z-library.", "zlib.", "singlelogin.", "scribd.com")
|
|
for raw in parser.results:
|
|
href = raw.get("href") or ""
|
|
parsed_href = urllib.parse.urlparse(href if href.startswith("http") else "https:" + href)
|
|
target = urllib.parse.parse_qs(parsed_href.query).get("uddg", [href])[0]
|
|
host = (urllib.parse.urlparse(target).hostname or "").removeprefix("www.")
|
|
if not target.startswith(("http://", "https://")) or any(part in host for part in blocked):
|
|
continue
|
|
evidence.append({
|
|
"provider": "duckduckgo",
|
|
"title": raw.get("title") or "",
|
|
"url": target,
|
|
"domain": host,
|
|
"snippet": (raw.get("snippet") or "")[:1600],
|
|
"relevance": None,
|
|
"published_at": "",
|
|
})
|
|
if len(evidence) >= max(
|
|
1,
|
|
min(max_results or self.settings.book_web_review_max_results, 10),
|
|
):
|
|
break
|
|
return evidence
|
|
|
|
def search(self, query: str, max_results: int = 5) -> dict[str, Any]:
|
|
"""Return bounded general-purpose search results without exposing backend errors."""
|
|
query = query.strip()
|
|
limit = max(1, min(max_results, 8))
|
|
if not query:
|
|
return {"provider": "none", "results": []}
|
|
if self.settings.tavily_api_key:
|
|
try:
|
|
found = self._tavily(query, limit)
|
|
except Exception:
|
|
found = []
|
|
if found:
|
|
return {"provider": "tavily", "results": found[:limit]}
|
|
try:
|
|
found = self._duckduckgo(query, limit)
|
|
except Exception:
|
|
found = []
|
|
return {
|
|
"provider": "duckduckgo" if found else "none",
|
|
"results": found[:limit],
|
|
}
|
|
|
|
|
|
class BookWebReviewProvider(WebSearchProvider):
|
|
"""Discover attributed web evidence for a book; interpretation stays with the LLM."""
|
|
|
|
def lookup(self, plan: dict[str, Any]) -> dict[str, Any]:
|
|
title = str(plan.get("title") or "").strip()
|
|
creator = str(plan.get("creator") or "").strip()
|
|
result: dict[str, Any] = {"evidence": [], "errors": [], "providers_checked": []}
|
|
if not title:
|
|
result["errors"].append("book-web-reviews: 缺少书名")
|
|
return result
|
|
cache_key = self._cache_key(plan)
|
|
cached = None if plan.get("refresh") else self.database.cache_get("book-web-reviews", cache_key)
|
|
if cached is not None:
|
|
cached["cache"] = {"hit": True, "ttl_seconds": self.settings.review_cache_ttl_seconds}
|
|
return cached
|
|
query = " ".join(value for value in (f'\"{title}\"', creator, "书评 评价 review 值得读") if value)
|
|
if self.settings.tavily_api_key:
|
|
result["providers_checked"].append("tavily")
|
|
try:
|
|
result["evidence"] = self._tavily(query)
|
|
except Exception as exc:
|
|
result["errors"].append(f"tavily: {exc}")
|
|
if not result["evidence"]:
|
|
result["providers_checked"].append("duckduckgo")
|
|
try:
|
|
result["evidence"] = self._duckduckgo(query)
|
|
except Exception as exc:
|
|
result["errors"].append(f"duckduckgo: {exc}")
|
|
if result["evidence"]:
|
|
self.database.cache_put("book-web-reviews", cache_key, result, self.settings.review_cache_ttl_seconds)
|
|
result["cache"] = {"hit": False, "ttl_seconds": self.settings.review_cache_ttl_seconds}
|
|
return result
|