feat: add book scraping (Douban via Playwright + Goodreads + Google Books)

- scrape.py: Douban scraping with Playwright (multi-strategy browser launch)
- scrape.py: Goodreads scraping with requests + BeautifulSoup
- scrape.py: Google Books API + Open Library as fallback
- bookshelf.py: add 'fetch' subcommand (ISBN / Douban URL / GR URL)
- Fix Douban #info parsing using span.pl tags
- Add requirements.txt (requests, beautifulsoup4, playwright)
This commit is contained in:
kai
2026-03-25 14:20:12 +08:00
parent e87be58ba3
commit 8c8095d9ac
4 changed files with 635 additions and 0 deletions
+76
View File
@@ -5,6 +5,7 @@ import argparse
import sys
import db
import web
import scrape
STATUS_CHOICES = ["to-read", "reading", "read"]
@@ -78,6 +79,72 @@ def cmd_delete(args):
print(f"🗑️ 已删除:《{book['title']}》(ID: {args.id})")
def cmd_fetch(args):
"""刮削图书元数据,可选择直接写入数据库。"""
identifier = args.identifier
# 选择刮削策略
if "douban.com" in identifier:
print(f"🔍 正在从豆瓣刮削:{identifier}")
data = scrape.fetch_douban(identifier)
source = "豆瓣"
elif "goodreads.com" in identifier:
print(f"🔍 正在从 Goodreads 刮削:{identifier}")
data = scrape.fetch_goodreads(identifier)
source = "Goodreads"
elif scrape._is_isbn(identifier):
print(f"🔍 正在刮削 ISBN{identifier}")
data, source = scrape.fetch_by_isbn(identifier)
else:
print("❌ 请提供有效的 ISBN(10/13位)、豆瓣链接或 Goodreads 链接。")
sys.exit(1)
if not data:
print("❌ 刮削失败,未能获取到图书信息。")
print(" 提示:豆瓣偶尔会拒绝请求(418/403),请稍后重试。")
sys.exit(1)
# 打印预览
print(scrape.preview(data, source))
print()
if args.update_id:
# 更新已有记录
db.init_db()
book = db.get_book(args.update_id)
if not book:
print(f"❌ 未找到 ID 为 {args.update_id} 的书目。")
sys.exit(1)
# 只覆盖刮削到的字段,保留用户已设置的字段
updates = {k: v for k, v in data.items() if v}
if args.format:
updates["format"] = args.format
if args.status:
updates["status"] = args.status
db.update_book(args.update_id, **updates)
print(f"✅ 已更新:《{book['title']}》→《{data.get('title', book['title'])}》(ID: {args.update_id})")
elif args.add:
# 写入新记录
db.init_db()
book_id = db.add_book(
data.get("title", ""),
author=data.get("author"),
translator=data.get("translator"),
publisher=data.get("publisher"),
pub_date=data.get("pub_date"),
cover_url=data.get("cover_url"),
format=args.format or "paper",
status=args.status or "to-read",
douban_score=data.get("douban_score"),
goodreads_score=data.get("goodreads_score"),
tags=data.get("tags", ""),
)
print(f"✅ 已添加:《{data.get('title')}》(ID: {book_id})")
else:
print("💡 加 --add 选项可直接写入数据库,或用 --update-id <ID> 更新已有记录。")
def cmd_build(args):
db.init_db()
output_path = web.build()
@@ -142,6 +209,15 @@ def main():
p_del.add_argument("id", type=int, help="书目 ID")
p_del.set_defaults(func=cmd_delete)
# --- fetch ---
p_fetch = sub.add_parser("fetch", help="从豆瓣/Google Books 刮削书目信息")
p_fetch.add_argument("identifier", help="ISBN10/13位)或豆瓣图书链接")
p_fetch.add_argument("--add", action="store_true", help="刮削后直接写入数据库")
p_fetch.add_argument("--update-id", type=int, metavar="ID", help="刮削后更新指定 ID 的书目")
p_fetch.add_argument("--format", choices=FORMAT_CHOICES, help="书籍格式(入库时使用)")
p_fetch.add_argument("--status", choices=STATUS_CHOICES, help="阅读状态(入库时使用)")
p_fetch.set_defaults(func=cmd_fetch)
# --- build ---
p_build = sub.add_parser("build", help="生成展示网页")
p_build.set_defaults(func=cmd_build)