"""图书元数据刮削模块。 数据源策略: - 中文书:豆瓣网页爬取 + Goodreads 补评分 - 英文书:Goodreads 为主力 - 兜底:Google Books API / Open Library ISBN API """ import json import re import time import random try: import requests as _requests _HAS_REQUESTS = True except ImportError: import urllib.request _HAS_REQUESTS = False # ── 常量 ──────────────────────────────────────────────────────────── _UA = ( "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/123.0.0.0 Safari/537.36" ) DOUBAN_ISBN_URL = "https://book.douban.com/isbn/{isbn}" DOUBAN_SUBJECT_URL = "https://book.douban.com/subject/{sid}/" GOODREADS_BOOK_URL = "https://www.goodreads.com/book/show/{book_id}" GOODREADS_SEARCH_URL = "https://www.goodreads.com/search?q={query}" GOOGLE_BOOKS_URL = "https://www.googleapis.com/books/v1/volumes?q=isbn:{isbn}" OPEN_LIBRARY_URL = ( "https://openlibrary.org/api/books?bibkeys=ISBN:{isbn}&format=json&jscmd=data" ) OPEN_LIBRARY_COVER = "https://covers.openlibrary.org/b/isbn/{isbn}-L.jpg" # ── 工具函数 ───────────────────────────────────────────────────────── def _get(url, timeout=15): """带 UA 的 HTTP GET,返回响应文本或 None。优先使用 requests 库。""" headers = {"User-Agent": _UA} if _HAS_REQUESTS: try: resp = _requests.get(url, headers=headers, timeout=timeout, allow_redirects=True) if resp.status_code == 200: return resp.text return None except Exception: return None else: import urllib.request req = urllib.request.Request(url, headers=headers) try: with urllib.request.urlopen(req, timeout=timeout) as resp: return resp.read().decode("utf-8", errors="replace") except Exception: return None def _get_with_playwright(url, timeout=30000): """用 Playwright 无头浏览器获取页面 HTML(用于豆瓣等反爬严格的站点)。""" try: from playwright.sync_api import sync_playwright except ImportError: print("⚠️ 需要安装 Playwright:pip install playwright && playwright install") return None html = None # 多种启动策略:系统 Chrome → Playwright Chromium → 不同 headless 模式 launch_options = [ {"channel": "chrome", "headless": True}, # 系统 Chrome {"channel": "chromium", "headless": True}, # 系统 Chromium {"headless": True, # Playwright 自带 "args": ["--disable-blink-features=AutomationControlled"]}, {"headless": False, # 非 headless(最后手段) "args": ["--headless=new"]}, # Chrome 新 headless 模式 ] try: with sync_playwright() as p: browser = None for opts in launch_options: try: browser = p.chromium.launch(**opts) break except Exception: continue if not browser: print("⚠️ 无法启动浏览器。请确保已安装 Chrome 或运行:playwright install") return None context = browser.new_context( user_agent=_UA, viewport={"width": 1280, "height": 800}, locale="zh-CN", ) page = context.new_page() page.goto(url, wait_until="domcontentloaded", timeout=timeout) # 等待核心内容出现 try: page.wait_for_selector("#info", timeout=8000) except Exception: pass # 即使超时也继续,页面可能已部分加载 html = page.content() browser.close() except Exception as e: print(f"⚠️ Playwright 访问失败:{e}") return None return html def _sleep(): """礼貌延时,避免被封。""" time.sleep(random.uniform(1.0, 2.5)) def _is_isbn(s): digits = re.sub(r"[-\s]", "", s) return re.fullmatch(r"\d{10}|\d{13}", digits) is not None def _clean_isbn(s): return re.sub(r"[-\s]", "", s) def _is_chinese_isbn(isbn): """以 978-7 / 7 开头的 ISBN 通常是中文出版物。""" d = _clean_isbn(isbn) return d.startswith("9787") or d.startswith("7") # ── 豆瓣刮削 ───────────────────────────────────────────────────────── def _parse_douban_html(html): """解析豆瓣图书详情页 HTML,返回 dict。需要 beautifulsoup4。""" try: from bs4 import BeautifulSoup except ImportError: raise RuntimeError("请先安装 beautifulsoup4:pip install beautifulsoup4") soup = BeautifulSoup(html, "html.parser") result = {} # 书名 title_tag = soup.find("h1") if title_tag: span = title_tag.find("span") result["title"] = (span or title_tag).get_text(strip=True) # 封面 cover_tag = soup.find("a", attrs={"class": "nbg"}) if cover_tag: img = cover_tag.find("img") if img: result["cover_url"] = img.get("src", "") # 豆瓣评分 rating_tag = soup.find("strong", attrs={"class": "ll rating_num"}) if rating_tag: txt = rating_tag.get_text(strip=True) try: result["douban_score"] = float(txt) except ValueError: pass # 作者、出版社等在 #info 块 info = soup.find("div", id="info") if info: # 方法1:通过 标签精确提取 for span in info.find_all("span", class_="pl"): label = span.get_text(strip=True).rstrip(":: ") # 获取同级后续文本和链接 values = [] for sib in span.next_siblings: if hasattr(sib, 'name'): if sib.name == 'br': break if sib.name == 'span' and 'pl' in (sib.get('class') or []): break if sib.name == 'a': values.append(sib.get_text(strip=True)) elif sib.name == 'span': values.append(sib.get_text(strip=True)) else: t = str(sib).strip().strip("::/·, ") if t: values.append(t) value = " ".join(v for v in values if v).strip(" /::") if not value: continue if "作者" in label: # 去掉 [国籍] 标注 author_clean = re.sub(r"\[.*?\]", "", value).strip(" /") # 清理多余空格 author_clean = re.sub(r"\s+", " ", author_clean).strip() if author_clean: result["author"] = author_clean elif "译者" in label: result["translator"] = re.sub(r"\s+", " ", value).strip(" /") elif "出版社" in label: result["publisher"] = value elif "出版年" in label: result["pub_date"] = value elif "ISBN" in label.upper(): result["isbn"] = value # 方法2:兜底 — 如果 span.pl 没有拿到,用纯文本 if not result.get("author"): text = info.get_text(separator="\n") for line in text.splitlines(): line = line.strip() if "作者" in line: parts = re.split(r"[::]", line, 1) if len(parts) == 2 and parts[1].strip(): author = re.sub(r"\[.*?\]", "", parts[1]).strip(" /") if author: result["author"] = author break # 标签 tag_links = soup.select("a.tag") if tag_links: tags = [a.get_text(strip=True) for a in tag_links[:6]] result["tags"] = ",".join(tags) return result if result.get("title") else None def fetch_douban(identifier: str): """ 从豆瓣刮削图书信息。 identifier 可以是: - ISBN(10位或13位) - 豆瓣图书链接(含 subject/id) - 豆瓣 subject id(纯数字) """ identifier = identifier.strip() if _is_isbn(identifier): url = DOUBAN_ISBN_URL.format(isbn=_clean_isbn(identifier)) elif "douban.com/subject/" in identifier: url = identifier.split("?")[0].rstrip("/") + "/" elif re.fullmatch(r"\d+", identifier): url = DOUBAN_SUBJECT_URL.format(sid=identifier) else: return None _sleep() html = _get_with_playwright(url) if not html or "豆瓣" not in html: # Playwright 失败时尝试普通 requests(偶尔可能成功) html = _get(url) if not html or "豆瓣" not in html: return None data = _parse_douban_html(html) return data # ── Goodreads 刮削 ──────────────────────────────────────────────────── def _parse_goodreads_html(html): """解析 Goodreads 图书详情页 HTML,返回 dict。""" try: from bs4 import BeautifulSoup except ImportError: raise RuntimeError("请先安装 beautifulsoup4:pip install beautifulsoup4") soup = BeautifulSoup(html, "html.parser") result = {} # 书名 —

title_tag = soup.find("h1", attrs={"data-testid": "bookTitle"}) if not title_tag: title_tag = soup.find("h1") if title_tag: result["title"] = title_tag.get_text(strip=True) # 作者 — author_tags = soup.select("span.ContributorLink__name") if author_tags: authors = [] for a in author_tags: name = a.get_text(strip=True) # 排除角色标注如 "(Primary Contributor)" if name and "Contributor" not in name: authors.append(name) if authors: result["author"] = " / ".join(dict.fromkeys(authors)) # 去重保序 # 评分 — 在页面文本中匹配 "X.XX" 紧跟 ratings 数字 # 格式通常是 "3.8015,305 ratings" 或类似 rating_match = re.search( r'"ratingValue"\s*:\s*([\d.]+)', html ) if rating_match: try: result["goodreads_score"] = float(rating_match.group(1)) except ValueError: pass if not result.get("goodreads_score"): # 备选:从可见文本中提取 rating_div = soup.find("div", attrs={"class": re.compile(r"RatingStatistics__rating")}) if rating_div: txt = rating_div.get_text(strip=True) try: result["goodreads_score"] = float(txt) except ValueError: pass if not result.get("goodreads_score"): # 再备选:页面全文正则("4.18" 后跟 "ratings") fallback = re.search(r'(\d\.\d{1,2})\s*[\d,]+\s*rating', html) if fallback: try: result["goodreads_score"] = float(fallback.group(1)) except ValueError: pass # 分类标签 — genre links genre_links = soup.select("span.BookPageMetadataSection__genreButton a") if not genre_links: genre_links = soup.select("a[href*='/genres/']") if genre_links: tags = [] for a in genre_links: tag = a.get_text(strip=True) if tag and tag not in tags and len(tags) < 6: tags.append(tag) if tags: result["tags"] = ",".join(tags) # 出版信息 — 在页面文本中查找 pub_match = re.search( r'(?:First published|Published)\s+(\w+\s+\d{1,2},\s*\d{4}|\w+\s+\d{4}|\d{4})', html ) if pub_match: result["pub_date"] = pub_match.group(1) # 封面 cover_tag = soup.find("img", attrs={"class": re.compile(r"ResponsiveImage")}) if cover_tag: src = cover_tag.get("src", "") if src and "nophoto" not in src: result["cover_url"] = src return result if result.get("title") else None def fetch_goodreads(identifier: str): """ 从 Goodreads 刮削图书信息。 identifier 可以是: - Goodreads 链接(含 /book/show/) - ISBN(会用搜索页查找) """ identifier = identifier.strip() if "goodreads.com/book/show/" in identifier: url = identifier.split("?")[0].split("&")[0] elif _is_isbn(identifier): # 先通过搜索页找到书籍链接 search_url = GOODREADS_SEARCH_URL.format(query=_clean_isbn(identifier)) _sleep() search_html = _get(search_url) if not search_html: return None # 从搜索结果中提取第一个 /book/show/ 链接 match = re.search(r'/book/show/(\d+[^"\s\']*)', search_html) if not match: return None url = f"https://www.goodreads.com/book/show/{match.group(1)}" else: return None _sleep() html = _get(url) if not html or "goodreads" not in html.lower(): return None return _parse_goodreads_html(html) def fetch_goodreads_score(isbn: str): """只获取 Goodreads 评分(用于补全其他来源的数据)。""" data = fetch_goodreads(isbn) if data and data.get("goodreads_score"): return data["goodreads_score"] return None # ── Google Books API ────────────────────────────────────────────────── def fetch_google_books(isbn: str): """通过 Google Books API 获取图书信息(无需 key)。""" isbn = _clean_isbn(isbn) url = GOOGLE_BOOKS_URL.format(isbn=isbn) html = _get(url) if not html: return None try: data = json.loads(html) except json.JSONDecodeError: return None if data.get("totalItems", 0) == 0: return None item = data["items"][0] vol = item.get("volumeInfo", {}) result = {"title": vol.get("title", "")} authors = vol.get("authors", []) if authors: result["author"] = " / ".join(authors) if vol.get("publisher"): result["publisher"] = vol["publisher"] if vol.get("publishedDate"): result["pub_date"] = vol["publishedDate"][:7] # YYYY-MM # 封面 images = vol.get("imageLinks", {}) cover = images.get("thumbnail") or images.get("smallThumbnail") if cover: # 换成更大图 result["cover_url"] = cover.replace("zoom=1", "zoom=3") # 分类标签 cats = vol.get("categories", []) if cats: result["tags"] = ",".join(cats[:4]) # 评分(Google Books 有 averageRating 字段) if vol.get("averageRating"): result["goodreads_score"] = float(vol["averageRating"]) return result if result.get("title") else None # ── Open Library 兜底 ───────────────────────────────────────────────── def fetch_open_library(isbn: str): """Open Library ISBN API 兜底。""" isbn = _clean_isbn(isbn) url = OPEN_LIBRARY_URL.format(isbn=isbn) html = _get(url) if not html: return None try: data = json.loads(html) except json.JSONDecodeError: return None key = f"ISBN:{isbn}" if key not in data: return None book = data[key] result = {"title": book.get("title", "")} authors = book.get("authors", []) if authors: result["author"] = " / ".join(a.get("name", "") for a in authors) pubs = book.get("publishers", []) if pubs: result["publisher"] = pubs[0].get("name", "") if book.get("publish_date"): result["pub_date"] = book["publish_date"] # 封面:直接用 covers API(即使信息里没有封面也能试) cover_url = f"https://covers.openlibrary.org/b/isbn/{isbn}-L.jpg" result["cover_url"] = cover_url subjects = book.get("subjects", []) if subjects: tags = [s.get("name", "") for s in subjects[:4]] result["tags"] = ",".join(t for t in tags if t) return result if result.get("title") else None # ── 主入口 ──────────────────────────────────────────────────────────── def fetch_by_isbn(isbn: str): """ 根据 ISBN 自动选择数据源。 中文 ISBN → 豆瓣 + Goodreads 补 GR 评分。 英文 ISBN → Goodreads 为主力 → Google Books 兜底。 """ if _is_chinese_isbn(isbn): result = fetch_douban(isbn) if result: # 尝试补全 Goodreads 评分 if not result.get("goodreads_score"): gr_score = fetch_goodreads_score(isbn) if gr_score: result["goodreads_score"] = gr_score return result, "豆瓣" result = fetch_open_library(isbn) if result: return result, "Open Library" else: # 英文书:Goodreads 为主力 result = fetch_goodreads(isbn) if result: return result, "Goodreads" # 兜底 result = fetch_google_books(isbn) if result: return result, "Google Books" result = fetch_open_library(isbn) if result: return result, "Open Library" return None, None def preview(data: dict, source: str): """格式化打印刮削结果预览。""" lines = [f"\n📖 《{data.get('title', '未知')}》 [来源: {source}]"] if data.get("author"): lines.append(f" 作者:{data['author']}") if data.get("translator"): lines.append(f" 译者:{data['translator']}") if data.get("publisher") or data.get("pub_date"): pub = " · ".join(filter(None, [data.get("publisher"), data.get("pub_date")])) lines.append(f" 出版:{pub}") if data.get("douban_score"): lines.append(f" 豆瓣:{data['douban_score']}") if data.get("goodreads_score"): lines.append(f" GR:{data['goodreads_score']}") if data.get("tags"): lines.append(f" 标签:{data['tags']}") if data.get("cover_url"): lines.append(f" 封面:{data['cover_url']}") return "\n".join(lines)