feat(curator): vendor the application backend as the scenario's tracked source
The curator Python backend (package, tests, systemd units, config templates, scripts) now lives under scenarios/curator/backend and is the single source of truth; the live checkout at the workspace path is a runtime copy. Exported from the app repo's tracked tree via git archive (no history, .pi/venv/caches excluded). 149 unit tests pass from the new location. profile.toml backend is now repo-relative (scenarios/curator/backend); verify-generated.sh resolves a relative backend against REPO_ROOT. verify-no-secrets ASSIGN heuristic now requires value entropy so vendored kwargs like token=extraction_token no longer false-positive. README documents the backend/ layout and the operator-owned app rollout step.
This commit is contained in:
Kai
@@ -0,0 +1,200 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from html.parser import HTMLParser
|
||||
import json
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
|
||||
|
||||
class _DuckDuckGoParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.results: list[dict[str, str]] = []
|
||||
self._kind = ""
|
||||
self._href = ""
|
||||
self._text: list[str] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
values = dict(attrs)
|
||||
classes = set((values.get("class") or "").split())
|
||||
if tag == "a" and "result__a" in classes:
|
||||
self._kind = "title"
|
||||
self._href = values.get("href") or ""
|
||||
self._text = []
|
||||
elif tag in {"a", "div"} and "result__snippet" in classes:
|
||||
self._kind = "snippet"
|
||||
self._text = []
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._kind:
|
||||
self._text.append(data)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if not self._kind or tag not in {"a", "div"}:
|
||||
return
|
||||
text = " ".join("".join(self._text).split())
|
||||
if self._kind == "title" and text:
|
||||
self.results.append({"title": text, "href": self._href, "snippet": ""})
|
||||
elif self._kind == "snippet" and text and self.results:
|
||||
self.results[-1]["snippet"] = text
|
||||
self._kind = ""
|
||||
self._href = ""
|
||||
self._text = []
|
||||
|
||||
|
||||
class WebSearchProvider:
|
||||
"""Search public web evidence through Tavily with a DuckDuckGo fallback."""
|
||||
|
||||
def __init__(self, settings: Settings, database: Database):
|
||||
self.settings = settings
|
||||
self.database = database
|
||||
|
||||
@staticmethod
|
||||
def _cache_key(plan: dict[str, Any]) -> str:
|
||||
value = json.dumps(
|
||||
{
|
||||
"title": plan.get("title") or "",
|
||||
"creator": plan.get("creator") or "",
|
||||
"aliases": plan.get("aliases") or [],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
def _tavily(self, query: str, max_results: int | None = None) -> list[dict[str, Any]]:
|
||||
if not self.settings.tavily_api_key:
|
||||
raise RuntimeError("未配置 CURATOR_TAVILY_API_KEY")
|
||||
payload = json.dumps({
|
||||
"api_key": self.settings.tavily_api_key,
|
||||
"query": query,
|
||||
"search_depth": "advanced",
|
||||
"topic": "general",
|
||||
"max_results": max(
|
||||
1,
|
||||
min(max_results or self.settings.book_web_review_max_results, 10),
|
||||
),
|
||||
"include_answer": False,
|
||||
"include_raw_content": False,
|
||||
}).encode("utf-8")
|
||||
request = urllib.request.Request(
|
||||
self.settings.tavily_search_url,
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json", "User-Agent": "Curator/0.3"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
value = json.load(response)
|
||||
evidence: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for raw in value.get("results") or []:
|
||||
url = str(raw.get("url") or "").strip()
|
||||
title = str(raw.get("title") or "").strip()
|
||||
if not url or not title or url in seen:
|
||||
continue
|
||||
host = urllib.parse.urlparse(url).hostname or ""
|
||||
if host.endswith(("zlib.li", "singlelogin.re")):
|
||||
continue
|
||||
seen.add(url)
|
||||
evidence.append({
|
||||
"provider": "tavily",
|
||||
"title": title,
|
||||
"url": url,
|
||||
"domain": host.removeprefix("www."),
|
||||
"snippet": str(raw.get("content") or "").strip()[:1600],
|
||||
"relevance": raw.get("score"),
|
||||
"published_at": str(raw.get("published_date") or ""),
|
||||
})
|
||||
return evidence
|
||||
|
||||
def _duckduckgo(self, query: str, max_results: int | None = None) -> list[dict[str, Any]]:
|
||||
url = "https://html.duckduckgo.com/html/?" + urllib.parse.urlencode({"q": query})
|
||||
request = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 Curator/0.3"})
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
parser = _DuckDuckGoParser()
|
||||
parser.feed(response.read().decode("utf-8", errors="replace"))
|
||||
evidence: list[dict[str, Any]] = []
|
||||
blocked = ("z-library.", "zlib.", "singlelogin.", "scribd.com")
|
||||
for raw in parser.results:
|
||||
href = raw.get("href") or ""
|
||||
parsed_href = urllib.parse.urlparse(href if href.startswith("http") else "https:" + href)
|
||||
target = urllib.parse.parse_qs(parsed_href.query).get("uddg", [href])[0]
|
||||
host = (urllib.parse.urlparse(target).hostname or "").removeprefix("www.")
|
||||
if not target.startswith(("http://", "https://")) or any(part in host for part in blocked):
|
||||
continue
|
||||
evidence.append({
|
||||
"provider": "duckduckgo",
|
||||
"title": raw.get("title") or "",
|
||||
"url": target,
|
||||
"domain": host,
|
||||
"snippet": (raw.get("snippet") or "")[:1600],
|
||||
"relevance": None,
|
||||
"published_at": "",
|
||||
})
|
||||
if len(evidence) >= max(
|
||||
1,
|
||||
min(max_results or self.settings.book_web_review_max_results, 10),
|
||||
):
|
||||
break
|
||||
return evidence
|
||||
|
||||
def search(self, query: str, max_results: int = 5) -> dict[str, Any]:
|
||||
"""Return bounded general-purpose search results without exposing backend errors."""
|
||||
query = query.strip()
|
||||
limit = max(1, min(max_results, 8))
|
||||
if not query:
|
||||
return {"provider": "none", "results": []}
|
||||
if self.settings.tavily_api_key:
|
||||
try:
|
||||
found = self._tavily(query, limit)
|
||||
except Exception:
|
||||
found = []
|
||||
if found:
|
||||
return {"provider": "tavily", "results": found[:limit]}
|
||||
try:
|
||||
found = self._duckduckgo(query, limit)
|
||||
except Exception:
|
||||
found = []
|
||||
return {
|
||||
"provider": "duckduckgo" if found else "none",
|
||||
"results": found[:limit],
|
||||
}
|
||||
|
||||
|
||||
class BookWebReviewProvider(WebSearchProvider):
|
||||
"""Discover attributed web evidence for a book; interpretation stays with the LLM."""
|
||||
|
||||
def lookup(self, plan: dict[str, Any]) -> dict[str, Any]:
|
||||
title = str(plan.get("title") or "").strip()
|
||||
creator = str(plan.get("creator") or "").strip()
|
||||
result: dict[str, Any] = {"evidence": [], "errors": [], "providers_checked": []}
|
||||
if not title:
|
||||
result["errors"].append("book-web-reviews: 缺少书名")
|
||||
return result
|
||||
cache_key = self._cache_key(plan)
|
||||
cached = None if plan.get("refresh") else self.database.cache_get("book-web-reviews", cache_key)
|
||||
if cached is not None:
|
||||
cached["cache"] = {"hit": True, "ttl_seconds": self.settings.review_cache_ttl_seconds}
|
||||
return cached
|
||||
query = " ".join(value for value in (f'\"{title}\"', creator, "书评 评价 review 值得读") if value)
|
||||
if self.settings.tavily_api_key:
|
||||
result["providers_checked"].append("tavily")
|
||||
try:
|
||||
result["evidence"] = self._tavily(query)
|
||||
except Exception as exc:
|
||||
result["errors"].append(f"tavily: {exc}")
|
||||
if not result["evidence"]:
|
||||
result["providers_checked"].append("duckduckgo")
|
||||
try:
|
||||
result["evidence"] = self._duckduckgo(query)
|
||||
except Exception as exc:
|
||||
result["errors"].append(f"duckduckgo: {exc}")
|
||||
if result["evidence"]:
|
||||
self.database.cache_put("book-web-reviews", cache_key, result, self.settings.review_cache_ttl_seconds)
|
||||
result["cache"] = {"hit": False, "ttl_seconds": self.settings.review_cache_ttl_seconds}
|
||||
return result
|
||||
Reference in New Issue
Block a user