fix(curator): query Douban only by Chinese title or ISBN, never a Latin title

Douban's English-language book records are sparse and low quality. BookPageProvider now picks a CJK title from the title or aliases for Douban (and still allows a precise ISBN), and skips Douban entirely when only a Latin title is available; Goodreads continues to cover Latin titles. Aliases now reach the page search so a translated book's Chinese title is used. New unit test covers the four cases; ISBN cross-title matching is preserved.
This commit is contained in:
Kai
2026-08-30 21:03:00 -07:00
parent 53ffa2fd7f
commit a337ea978f
3 changed files with 53 additions and 5 deletions
@@ -10,6 +10,8 @@ from typing import Any
from .media_catalog import title_key from .media_catalog import title_key
_CJK = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]")
class _SearchParser(HTMLParser): class _SearchParser(HTMLParser):
def __init__(self) -> None: def __init__(self) -> None:
@@ -89,10 +91,27 @@ class BookPageProvider:
"goodreads.com": "goodreads", "goodreads.com": "goodreads",
} }
def search(self, title: str, author: str = "", isbn: str = "", limit: int = 6) -> list[dict[str, Any]]: def search(self, title: str, author: str = "", isbn: str = "", limit: int = 6,
terms = " ".join(value for value in (f'"{title}"' if title else "", author, isbn) if value) aliases: list[str] | None = None) -> list[dict[str, Any]]:
candidates = [title, *(aliases or [])]
douban_title = next((c.strip() for c in candidates if c and _CJK.search(c)), "")
goodreads_terms = " ".join(value for value in (f'"{title}"' if title else "", author, isbn) if value)
urls: list[str] = [] urls: list[str] = []
for provider in ("douban", "goodreads"): for provider in ("douban", "goodreads"):
if provider == "douban":
# Douban's English-language records are sparse and low quality, so it
# is queried only with a Chinese title (or a precise ISBN), never with
# a Latin title. Without either, Goodreads covers the book alone.
parts = [f'"{douban_title}"'] if douban_title else []
if author and _CJK.search(author):
parts.append(author)
if isbn:
parts.append(isbn)
if not parts:
continue
terms = " ".join(parts)
else:
terms = goodreads_terms
for url in self._search_site(provider, terms): for url in self._search_site(provider, terms):
if url not in urls: if url not in urls:
urls.append(url) urls.append(url)
@@ -32,8 +32,8 @@ class BookReviewProvider:
) )
return hashlib.sha256(value.encode("utf-8")).hexdigest() return hashlib.sha256(value.encode("utf-8")).hexdigest()
def _public_pages(self, title: str, author: str, isbn: str) -> list[dict[str, Any]]: def _public_pages(self, title: str, author: str, isbn: str, aliases: list[str]) -> list[dict[str, Any]]:
return BookPageProvider().search(title, author, isbn, 8) return BookPageProvider().search(title, author, isbn, 8, aliases=aliases)
@staticmethod @staticmethod
def _text_key(value: str) -> str: def _text_key(value: str) -> str:
@@ -95,7 +95,7 @@ class BookReviewProvider:
for name, loader in (("douban-goodreads-pages", self._public_pages),): for name, loader in (("douban-goodreads-pages", self._public_pages),):
result["providers_checked"].append(name) result["providers_checked"].append(name)
try: try:
result["results"].extend(self._matched_results(loader(title, author, isbn), title, aliases, author, isbn)) result["results"].extend(self._matched_results(loader(title, author, isbn, aliases), title, aliases, author, isbn))
except Exception as exc: except Exception as exc:
result["errors"].append(f"{name}: {exc}") result["errors"].append(f"{name}: {exc}")
if result["results"]: if result["results"]:
@@ -368,6 +368,35 @@ class CuratorTest(unittest.TestCase):
"非洲运转之道", "乔·斯塔威尔", "978-1-84668-407-9", "非洲运转之道", "乔·斯塔威尔", "978-1-84668-407-9",
)) ))
def test_douban_is_queried_only_with_a_chinese_title(self) -> None:
provider = BookPageProvider()
calls: list[tuple[str, str]] = []
provider._search_site = lambda prov, terms: calls.append((prov, terms)) or [] # type: ignore[method-assign]
# English-only title, no ISBN, no alias: Douban is skipped, Goodreads runs.
calls.clear()
provider.search("Clean Code", "Robert Martin")
self.assertEqual([p for p, _ in calls], ["goodreads"])
# A Chinese alias routes Douban to the Chinese title, not the English one.
calls.clear()
provider.search("Sapiens", "Yuval Noah Harari", aliases=["人类简史"])
douban = [t for p, t in calls if p == "douban"]
self.assertEqual(len(douban), 1)
self.assertIn("人类简史", douban[0])
self.assertNotIn("Sapiens", douban[0])
# A precise ISBN still reaches Douban even without a Chinese title.
calls.clear()
provider.search("How Africa Works", "", isbn="9781846684079")
douban = [t for p, t in calls if p == "douban"]
self.assertEqual(len(douban), 1)
self.assertIn("9781846684079", douban[0])
self.assertNotIn("How Africa Works", douban[0])
# A Chinese title is used directly (no regression).
calls.clear()
provider.search("三体", "刘慈欣")
douban = [t for p, t in calls if p == "douban"]
self.assertEqual(len(douban), 1)
self.assertIn("三体", douban[0])
def test_book_web_review_falls_back_and_caches_attributed_evidence(self) -> None: def test_book_web_review_falls_back_and_caches_attributed_evidence(self) -> None:
provider = BookWebReviewProvider(self.settings, self.database) provider = BookWebReviewProvider(self.settings, self.database)
calls: list[str] = [] calls: list[str] = []