fix(curator): query Douban only by Chinese title or ISBN, never a Latin title
Douban's English-language book records are sparse and low quality. BookPageProvider now picks a CJK title from the title or aliases for Douban (and still allows a precise ISBN), and skips Douban entirely when only a Latin title is available; Goodreads continues to cover Latin titles. Aliases now reach the page search so a translated book's Chinese title is used. New unit test covers the four cases; ISBN cross-title matching is preserved.
This commit is contained in:
@@ -10,6 +10,8 @@ from typing import Any
|
|||||||
|
|
||||||
from .media_catalog import title_key
|
from .media_catalog import title_key
|
||||||
|
|
||||||
|
_CJK = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]")
|
||||||
|
|
||||||
|
|
||||||
class _SearchParser(HTMLParser):
|
class _SearchParser(HTMLParser):
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
@@ -89,10 +91,27 @@ class BookPageProvider:
|
|||||||
"goodreads.com": "goodreads",
|
"goodreads.com": "goodreads",
|
||||||
}
|
}
|
||||||
|
|
||||||
def search(self, title: str, author: str = "", isbn: str = "", limit: int = 6) -> list[dict[str, Any]]:
|
def search(self, title: str, author: str = "", isbn: str = "", limit: int = 6,
|
||||||
terms = " ".join(value for value in (f'"{title}"' if title else "", author, isbn) if value)
|
aliases: list[str] | None = None) -> list[dict[str, Any]]:
|
||||||
|
candidates = [title, *(aliases or [])]
|
||||||
|
douban_title = next((c.strip() for c in candidates if c and _CJK.search(c)), "")
|
||||||
|
goodreads_terms = " ".join(value for value in (f'"{title}"' if title else "", author, isbn) if value)
|
||||||
urls: list[str] = []
|
urls: list[str] = []
|
||||||
for provider in ("douban", "goodreads"):
|
for provider in ("douban", "goodreads"):
|
||||||
|
if provider == "douban":
|
||||||
|
# Douban's English-language records are sparse and low quality, so it
|
||||||
|
# is queried only with a Chinese title (or a precise ISBN), never with
|
||||||
|
# a Latin title. Without either, Goodreads covers the book alone.
|
||||||
|
parts = [f'"{douban_title}"'] if douban_title else []
|
||||||
|
if author and _CJK.search(author):
|
||||||
|
parts.append(author)
|
||||||
|
if isbn:
|
||||||
|
parts.append(isbn)
|
||||||
|
if not parts:
|
||||||
|
continue
|
||||||
|
terms = " ".join(parts)
|
||||||
|
else:
|
||||||
|
terms = goodreads_terms
|
||||||
for url in self._search_site(provider, terms):
|
for url in self._search_site(provider, terms):
|
||||||
if url not in urls:
|
if url not in urls:
|
||||||
urls.append(url)
|
urls.append(url)
|
||||||
|
|||||||
@@ -32,8 +32,8 @@ class BookReviewProvider:
|
|||||||
)
|
)
|
||||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
def _public_pages(self, title: str, author: str, isbn: str) -> list[dict[str, Any]]:
|
def _public_pages(self, title: str, author: str, isbn: str, aliases: list[str]) -> list[dict[str, Any]]:
|
||||||
return BookPageProvider().search(title, author, isbn, 8)
|
return BookPageProvider().search(title, author, isbn, 8, aliases=aliases)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _text_key(value: str) -> str:
|
def _text_key(value: str) -> str:
|
||||||
@@ -95,7 +95,7 @@ class BookReviewProvider:
|
|||||||
for name, loader in (("douban-goodreads-pages", self._public_pages),):
|
for name, loader in (("douban-goodreads-pages", self._public_pages),):
|
||||||
result["providers_checked"].append(name)
|
result["providers_checked"].append(name)
|
||||||
try:
|
try:
|
||||||
result["results"].extend(self._matched_results(loader(title, author, isbn), title, aliases, author, isbn))
|
result["results"].extend(self._matched_results(loader(title, author, isbn, aliases), title, aliases, author, isbn))
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
result["errors"].append(f"{name}: {exc}")
|
result["errors"].append(f"{name}: {exc}")
|
||||||
if result["results"]:
|
if result["results"]:
|
||||||
|
|||||||
@@ -368,6 +368,35 @@ class CuratorTest(unittest.TestCase):
|
|||||||
"非洲运转之道", "乔·斯塔威尔", "978-1-84668-407-9",
|
"非洲运转之道", "乔·斯塔威尔", "978-1-84668-407-9",
|
||||||
))
|
))
|
||||||
|
|
||||||
|
def test_douban_is_queried_only_with_a_chinese_title(self) -> None:
|
||||||
|
provider = BookPageProvider()
|
||||||
|
calls: list[tuple[str, str]] = []
|
||||||
|
provider._search_site = lambda prov, terms: calls.append((prov, terms)) or [] # type: ignore[method-assign]
|
||||||
|
# English-only title, no ISBN, no alias: Douban is skipped, Goodreads runs.
|
||||||
|
calls.clear()
|
||||||
|
provider.search("Clean Code", "Robert Martin")
|
||||||
|
self.assertEqual([p for p, _ in calls], ["goodreads"])
|
||||||
|
# A Chinese alias routes Douban to the Chinese title, not the English one.
|
||||||
|
calls.clear()
|
||||||
|
provider.search("Sapiens", "Yuval Noah Harari", aliases=["人类简史"])
|
||||||
|
douban = [t for p, t in calls if p == "douban"]
|
||||||
|
self.assertEqual(len(douban), 1)
|
||||||
|
self.assertIn("人类简史", douban[0])
|
||||||
|
self.assertNotIn("Sapiens", douban[0])
|
||||||
|
# A precise ISBN still reaches Douban even without a Chinese title.
|
||||||
|
calls.clear()
|
||||||
|
provider.search("How Africa Works", "", isbn="9781846684079")
|
||||||
|
douban = [t for p, t in calls if p == "douban"]
|
||||||
|
self.assertEqual(len(douban), 1)
|
||||||
|
self.assertIn("9781846684079", douban[0])
|
||||||
|
self.assertNotIn("How Africa Works", douban[0])
|
||||||
|
# A Chinese title is used directly (no regression).
|
||||||
|
calls.clear()
|
||||||
|
provider.search("三体", "刘慈欣")
|
||||||
|
douban = [t for p, t in calls if p == "douban"]
|
||||||
|
self.assertEqual(len(douban), 1)
|
||||||
|
self.assertIn("三体", douban[0])
|
||||||
|
|
||||||
def test_book_web_review_falls_back_and_caches_attributed_evidence(self) -> None:
|
def test_book_web_review_falls_back_and_caches_attributed_evidence(self) -> None:
|
||||||
provider = BookWebReviewProvider(self.settings, self.database)
|
provider = BookWebReviewProvider(self.settings, self.database)
|
||||||
calls: list[str] = []
|
calls: list[str] = []
|
||||||
|
|||||||
Reference in New Issue
Block a user