fix(curator): query Douban only by Chinese title or ISBN, never a Latin title
Douban's English-language book records are sparse and low quality. BookPageProvider now picks a CJK title from the title or aliases for Douban (and still allows a precise ISBN), and skips Douban entirely when only a Latin title is available; Goodreads continues to cover Latin titles. Aliases now reach the page search so a translated book's Chinese title is used. New unit test covers the four cases; ISBN cross-title matching is preserved.
This commit is contained in:
@@ -10,6 +10,8 @@ from typing import Any
|
||||
|
||||
from .media_catalog import title_key
|
||||
|
||||
_CJK = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]")
|
||||
|
||||
|
||||
class _SearchParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
@@ -89,10 +91,27 @@ class BookPageProvider:
|
||||
"goodreads.com": "goodreads",
|
||||
}
|
||||
|
||||
def search(self, title: str, author: str = "", isbn: str = "", limit: int = 6) -> list[dict[str, Any]]:
|
||||
terms = " ".join(value for value in (f'"{title}"' if title else "", author, isbn) if value)
|
||||
def search(self, title: str, author: str = "", isbn: str = "", limit: int = 6,
|
||||
aliases: list[str] | None = None) -> list[dict[str, Any]]:
|
||||
candidates = [title, *(aliases or [])]
|
||||
douban_title = next((c.strip() for c in candidates if c and _CJK.search(c)), "")
|
||||
goodreads_terms = " ".join(value for value in (f'"{title}"' if title else "", author, isbn) if value)
|
||||
urls: list[str] = []
|
||||
for provider in ("douban", "goodreads"):
|
||||
if provider == "douban":
|
||||
# Douban's English-language records are sparse and low quality, so it
|
||||
# is queried only with a Chinese title (or a precise ISBN), never with
|
||||
# a Latin title. Without either, Goodreads covers the book alone.
|
||||
parts = [f'"{douban_title}"'] if douban_title else []
|
||||
if author and _CJK.search(author):
|
||||
parts.append(author)
|
||||
if isbn:
|
||||
parts.append(isbn)
|
||||
if not parts:
|
||||
continue
|
||||
terms = " ".join(parts)
|
||||
else:
|
||||
terms = goodreads_terms
|
||||
for url in self._search_site(provider, terms):
|
||||
if url not in urls:
|
||||
urls.append(url)
|
||||
|
||||
@@ -32,8 +32,8 @@ class BookReviewProvider:
|
||||
)
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||
|
||||
def _public_pages(self, title: str, author: str, isbn: str) -> list[dict[str, Any]]:
|
||||
return BookPageProvider().search(title, author, isbn, 8)
|
||||
def _public_pages(self, title: str, author: str, isbn: str, aliases: list[str]) -> list[dict[str, Any]]:
|
||||
return BookPageProvider().search(title, author, isbn, 8, aliases=aliases)
|
||||
|
||||
@staticmethod
|
||||
def _text_key(value: str) -> str:
|
||||
@@ -95,7 +95,7 @@ class BookReviewProvider:
|
||||
for name, loader in (("douban-goodreads-pages", self._public_pages),):
|
||||
result["providers_checked"].append(name)
|
||||
try:
|
||||
result["results"].extend(self._matched_results(loader(title, author, isbn), title, aliases, author, isbn))
|
||||
result["results"].extend(self._matched_results(loader(title, author, isbn, aliases), title, aliases, author, isbn))
|
||||
except Exception as exc:
|
||||
result["errors"].append(f"{name}: {exc}")
|
||||
if result["results"]:
|
||||
|
||||
@@ -368,6 +368,35 @@ class CuratorTest(unittest.TestCase):
|
||||
"非洲运转之道", "乔·斯塔威尔", "978-1-84668-407-9",
|
||||
))
|
||||
|
||||
def test_douban_is_queried_only_with_a_chinese_title(self) -> None:
|
||||
provider = BookPageProvider()
|
||||
calls: list[tuple[str, str]] = []
|
||||
provider._search_site = lambda prov, terms: calls.append((prov, terms)) or [] # type: ignore[method-assign]
|
||||
# English-only title, no ISBN, no alias: Douban is skipped, Goodreads runs.
|
||||
calls.clear()
|
||||
provider.search("Clean Code", "Robert Martin")
|
||||
self.assertEqual([p for p, _ in calls], ["goodreads"])
|
||||
# A Chinese alias routes Douban to the Chinese title, not the English one.
|
||||
calls.clear()
|
||||
provider.search("Sapiens", "Yuval Noah Harari", aliases=["人类简史"])
|
||||
douban = [t for p, t in calls if p == "douban"]
|
||||
self.assertEqual(len(douban), 1)
|
||||
self.assertIn("人类简史", douban[0])
|
||||
self.assertNotIn("Sapiens", douban[0])
|
||||
# A precise ISBN still reaches Douban even without a Chinese title.
|
||||
calls.clear()
|
||||
provider.search("How Africa Works", "", isbn="9781846684079")
|
||||
douban = [t for p, t in calls if p == "douban"]
|
||||
self.assertEqual(len(douban), 1)
|
||||
self.assertIn("9781846684079", douban[0])
|
||||
self.assertNotIn("How Africa Works", douban[0])
|
||||
# A Chinese title is used directly (no regression).
|
||||
calls.clear()
|
||||
provider.search("三体", "刘慈欣")
|
||||
douban = [t for p, t in calls if p == "douban"]
|
||||
self.assertEqual(len(douban), 1)
|
||||
self.assertIn("三体", douban[0])
|
||||
|
||||
def test_book_web_review_falls_back_and_caches_attributed_evidence(self) -> None:
|
||||
provider = BookWebReviewProvider(self.settings, self.database)
|
||||
calls: list[str] = []
|
||||
|
||||
Reference in New Issue
Block a user