kai/zlib-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

release: cut v0.01 with working downloads and unstable proxy rotation

Browse Source
This commit is contained in:
Kai Tan
2026-04-01 00:03:53 +08:00
parent 97c48995b1
commit 97fff0b30e
23 changed files with 250 additions and 600 deletions
Download Patch File Download Diff File Expand all files Collapse all files
zlib_dl.py
+247 -68
View File
@@ -198,6 +198,39 @@ class ZLibSession:
return {"https": self.proxy_url, "http": self.proxy_url}
return None
def _debug_log_exit_ip(self, stage: str):
"""在 verbose 模式下打印当前会话出口 IP，便于核对代理是否一致"""
if not self.verbose:
return
proxies = self._get_proxies()
probe_targets = [
("http://httpbin.org/ip", lambda data: data.get("origin", "unknown")),
("http://ip-api.com/json", lambda data: data.get("query", "unknown")),
("https://api.ipify.org?format=json", lambda data: data.get("ip", "unknown")),
]
proxy_desc = self.proxy_url if self.proxy_url else "直连"
last_error = None
for probe_url, parser in probe_targets:
try:
resp = self.session.get(
probe_url,
impersonate="chrome",
proxies=proxies,
timeout=15,
)
ip = parser(resp.json())
print(
f" 🌐 [{stage}] 当前出口 IP: {ip} (proxy={proxy_desc}, probe={probe_url})",
file=sys.stderr,
)
return
except Exception as e:
last_error = e
print(f" 🌐 [{stage}] 当前出口 IP 获取失败: {last_error}", file=sys.stderr)
def _request(self, url: str, **kwargs) -> cffi_requests.Response:
"""发起请求，自动处理 PoW 挑战"""
proxies = self._get_proxies()
@@ -231,6 +264,7 @@ class ZLibSession:
def search(self, query: str, max_results: int = 50) -> List[BookResult]:
"""搜索图书，返回结果列表"""
self._debug_log_exit_ip("search")
url = f"{self.BASE_URL}/s/?q={query}"
resp = self._request(url)
@@ -271,8 +305,9 @@ class ZLibSession:
def _request_download(self, url: str, **kwargs) -> cffi_requests.Response:
"""发起下载请求，自动处理 PoW 挑战（支持 allow_redirects）"""
proxies = self._get_proxies()
timeout = kwargs.pop("timeout", (20, 90))
resp = self.session.get(
url, impersonate="chrome", proxies=proxies, timeout=300,
url, impersonate="chrome", proxies=proxies, timeout=timeout,
allow_redirects=True, **kwargs
)
@@ -303,7 +338,7 @@ class ZLibSession:
# 无论什么页面，PoW 是在哪个域名的哪个具体 URL 上发生的，就全盘用带着 Cookie 的新状态重新向该域名发出最初的请求
resp = self.session.get(
resp.url, impersonate="chrome", proxies=proxies, timeout=300,
resp.url, impersonate="chrome", proxies=proxies, timeout=timeout,
allow_redirects=True, **kwargs
)
@@ -327,6 +362,7 @@ class ZLibSession:
if detail_url:
if self.verbose:
print(f" 🔗 正在预热书籍详情页以注册当前代理指纹和获取新 Session...", file=sys.stderr)
self._debug_log_exit_ip("warmup")
try:
dt_resp = self._request_download(detail_url, headers={"Referer": self.BASE_URL + "/"})
import re
@@ -348,7 +384,13 @@ class ZLibSession:
return False, f"预热书籍详情页或解析出错: {e}"
try:
resp = self._request_download(url, headers={"Referer": detail_url} if detail_url else {})
self._debug_log_exit_ip("download")
resp = self._request_download(
url,
headers={"Referer": detail_url} if detail_url else {},
stream=True,
timeout=(20, 120),
)
except Exception as e:
return False, f"下载请求失败: {e}"
@@ -376,12 +418,67 @@ class ZLibSession:
else:
return False, "下载失败，服务端依然返回了 HTML 重定向网页！"
# 写入文件
# 写入文件（流式下载：只要持续有数据就继续）
os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
with open(output_path, "wb") as f:
f.write(resp.content)
tmp_path = output_path + ".part"
total_bytes = 0
return True, f"下载完成: {len(resp.content)} bytes"
try:
content_length = resp.headers.get("Content-Length", "").strip()
expected_total = int(content_length) if content_length.isdigit() else 0
if self.verbose:
if expected_total > 0:
print(
f" 📦 服务器文件大小: {expected_total / (1024 * 1024):.2f} MB",
file=sys.stderr,
)
else:
print(" 📦 服务器未返回文件大小（Content-Length）", file=sys.stderr)
started_at = time.time()
last_log_at = started_at
with open(tmp_path, "wb") as f:
for chunk in resp.iter_content(chunk_size=262144):
if not chunk:
continue
f.write(chunk)
total_bytes += len(chunk)
now = time.time()
if self.verbose and now - last_log_at >= 3:
elapsed = max(0.001, now - started_at)
speed = total_bytes / elapsed
if expected_total > 0:
pct = (total_bytes / expected_total) * 100
print(
f" ⏬ 已下载 {total_bytes / (1024 * 1024):.2f} / {expected_total / (1024 * 1024):.2f} MB ({pct:.1f}%), 速度 {speed / (1024 * 1024):.2f} MB/s",
file=sys.stderr,
)
else:
print(
f" ⏬ 已下载 {total_bytes / (1024 * 1024):.2f} MB, 速度 {speed / (1024 * 1024):.2f} MB/s",
file=sys.stderr,
)
last_log_at = now
os.replace(tmp_path, output_path)
if expected_total > 0:
return True, f"下载完成: {total_bytes} bytes / 期望 {expected_total} bytes"
return True, f"下载完成: {total_bytes} bytes"
except Exception as e:
try:
if os.path.exists(tmp_path):
os.remove(tmp_path)
except Exception:
pass
return False, f"下载流中断: {e}"
finally:
try:
resp.close()
except Exception:
pass
# ============================================================
@@ -589,6 +686,12 @@ proxies:
port: {proxy.port}
cipher: "{proxy.method}"
password: "{proxy.password}"{plugin_yaml}{interface_yaml}
proxy-groups:
- name: GLOBAL
type: select
proxies:
- "{proxy.name}"
"""
with open(cfg_path, 'w', encoding='utf-8') as f:
f.write(yaml_config)
@@ -679,6 +782,18 @@ proxies:
proxy = self.proxies[self.current_index]
return proxy.downloads_count >= self.MAX_DOWNLOADS_PER_PROXY
def has_proxy_pool(self) -> bool:
"""是否配置了可轮换的代理池"""
return len(self.proxies) > 0
def ensure_active(self) -> bool:
"""确保当前有活跃代理（有代理池时）"""
if not self.proxies:
return True
if 0 <= self.current_index < len(self.proxies):
return True
return self.rotate()
def rotate(self) -> bool:
"""
轮换到下一个代理。
@@ -991,16 +1106,53 @@ def sanitize_filename(name: str) -> str:
return name.strip()[:200]
def _should_rotate_proxy(message: str) -> bool:
"""根据错误信息判断是否应该切换代理重试"""
msg_lower = message.lower()
return (
"限" in message
or "limit" in msg_lower
or "429" in message
or "503" in message
or "504" in message
or "403" in message
or "502" in message
or "timeout" in msg_lower
or "请求失败" in message
or "html" in msg_lower
)
def _create_session_for_current_proxy(
proxy_mgr: ProxyManager,
explicit_proxy: Optional[str],
verbose: bool,
) -> ZLibSession:
"""根据当前代理状态创建新的 ZLibSession"""
proxy_url = explicit_proxy if explicit_proxy else proxy_mgr.get_current_proxy_url()
return ZLibSession(proxy_url=proxy_url, verbose=verbose)
def cmd_download(args):
"""执行下载命令"""
explicit_proxy = getattr(args, "proxy", None)
proxy_file = getattr(args, "proxy_file", None)
if explicit_proxy and proxy_file:
print("错误: --proxy 与 --proxy-file 不能同时使用", file=sys.stderr)
sys.exit(1)
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
# 代理管理器
proxy_mgr = ProxyManager(
proxy_file=getattr(args, "proxy_file", None), verbose=args.verbose
proxy_file=proxy_file, verbose=args.verbose
)
if proxy_mgr.has_proxy_pool() and not proxy_mgr.ensure_active():
print("错误: 代理池不可用，无法启动任何代理", file=sys.stderr)
sys.exit(1)
download_results: List[DownloadResult] = []
try:
@@ -1014,11 +1166,13 @@ def cmd_download(args):
for dl_path in dl_paths:
result = _download_one(
dl_path=dl_path,
page_url="",
title=title,
author=author,
fmt=fmt or "",
output_dir=output_dir,
proxy_mgr=proxy_mgr,
explicit_proxy=explicit_proxy,
verbose=args.verbose,
)
download_results.append(result)
@@ -1037,8 +1191,11 @@ def cmd_download(args):
print(f" 🔍 搜索: {query_str}", file=sys.stderr)
# 搜索
proxy_url = proxy_mgr.get_current_proxy_url()
session = ZLibSession(proxy_url=proxy_url, verbose=args.verbose)
session = _create_session_for_current_proxy(
proxy_mgr=proxy_mgr,
explicit_proxy=explicit_proxy,
verbose=args.verbose,
)
results = session.search(query_str)
if not results:
@@ -1066,6 +1223,7 @@ def cmd_download(args):
fmt=book.format,
output_dir=output_dir,
proxy_mgr=proxy_mgr,
explicit_proxy=explicit_proxy,
verbose=args.verbose,
)
download_results.append(result)
@@ -1082,6 +1240,7 @@ def cmd_download(args):
fmt=book.format,
output_dir=output_dir,
proxy_mgr=proxy_mgr,
explicit_proxy=explicit_proxy,
verbose=args.verbose,
)
download_results.append(result)
@@ -1120,6 +1279,7 @@ def _download_one(
fmt: str,
output_dir: Path,
proxy_mgr: ProxyManager,
explicit_proxy: Optional[str],
verbose: bool,
) -> DownloadResult:
"""下载单个文件，带代理轮询逻辑"""
@@ -1147,8 +1307,77 @@ def _download_one(
message="文件已存在，跳过",
)
# 检查代理轮换
if proxy_mgr.should_rotate():
uses_proxy_pool = proxy_mgr.has_proxy_pool()
# 进入下载前，若当前代理已达配额，先轮换
if uses_proxy_pool and proxy_mgr.should_rotate() and not proxy_mgr.rotate():
return DownloadResult(
status="error",
title=title,
author=author,
format=fmt,
filesize="",
filepath="",
message="所有代理已用完，且当前代理下载次数已达上限",
)
while True:
if uses_proxy_pool and not proxy_mgr.ensure_active():
return DownloadResult(
status="error",
title=title,
author=author,
format=fmt,
filesize="",
filepath="",
message="所有代理已用完",
)
if uses_proxy_pool:
proxy_name = proxy_mgr.get_current_proxy_name()
elif explicit_proxy:
proxy_name = explicit_proxy
else:
proxy_name = "直连"
if verbose:
print(f" ⬇️ 下载: {filename} (代理: {proxy_name})", file=sys.stderr)
# 每次下载尝试都创建全新 Session；切代理后会自动带新出口重建会话
session = _create_session_for_current_proxy(
proxy_mgr=proxy_mgr,
explicit_proxy=explicit_proxy,
verbose=verbose,
)
success, message = session.download_file(dl_path, page_url, output_path)
if success:
if uses_proxy_pool:
proxy_mgr.record_download()
filesize = os.path.getsize(output_path)
size_str = _format_size(filesize)
return DownloadResult(
status="success",
title=title,
author=author,
format=fmt,
filesize=size_str,
filepath=output_path,
message=f"下载成功 ({size_str})",
proxy_used=proxy_name,
)
if not uses_proxy_pool or not _should_rotate_proxy(message):
return DownloadResult(
status="error",
title=title,
author=author,
format=fmt,
filesize="",
filepath="",
message=message,
)
if not proxy_mgr.rotate():
return DownloadResult(
status="error",
@@ -1157,64 +1386,14 @@ def _download_one(
format=fmt,
filesize="",
filepath="",
message="所有代理已用完，且当前代理下载次数已达上限",
message=f"{message}；且所有代理已用完",
)
proxy_url = proxy_mgr.get_current_proxy_url()
proxy_name = proxy_mgr.get_current_proxy_name()
if verbose:
print(
f" ⬇️ 下载: {filename} (代理: {proxy_name})", file=sys.stderr
)
# 完全创建新 Session，废弃之前的旧 Session 以获取全新的 CDN 下载授权
session = ZLibSession(proxy_url=proxy_url, verbose=verbose)
success, message = session.download_file(dl_path, page_url, output_path)
if success:
proxy_mgr.record_download()
filesize = os.path.getsize(output_path)
size_str = format_size(filesize)
return DownloadResult(
status="success",
title=title,
author=author,
format=fmt,
filesize=size_str,
filepath=output_path,
message=f"下载成功 ({size_str})",
proxy_used=proxy_name,
)
else:
# 如果是限制错误（达到配额）或遇到 503/403 等代理被屏蔽导致的无限验证循环，尝试切换代理重试
msg_lower = message.lower()
should_retry = (
"限" in message or "limit" in msg_lower or
"503" in message or "403" in message or "502" in message or
"timeout" in msg_lower or "请求失败" in message or "html" in msg_lower
)
if should_retry:
if proxy_mgr.rotate():
if verbose:
print(
f" 🔄 下载受阻 ({message})，切换代理重试: {proxy_mgr.get_current_proxy_name()}",
file=sys.stderr,
)
return _download_one(
dl_path, page_url, title, author, fmt, output_dir, proxy_mgr, verbose
)
return DownloadResult(
status="error",
title=title,
author=author,
format=fmt,
filesize="",
filepath="",
message=message,
)
if verbose:
print(
f" 🔄 下载受阻 ({message})，切换代理重试: {proxy_mgr.get_current_proxy_name()}",
file=sys.stderr,
)
def _format_size(size_bytes: int) -> str:
"""格式化文件大小"""