kai/zlib-dl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

release: cut v0.02 with Surge API proxy rotation

Browse Source
This commit is contained in:
Kai Tan
2026-04-01 00:31:51 +08:00
parent 97fff0b30e
commit 99b3e1caff
3 changed files with 270 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -1,3 +1,4 @@
downloads/
__pycache__/
*.pyc
.env
README.md
+25 -1
View File
@@ -7,7 +7,7 @@
- ✅ **自动绕过反爬保护**：自动解决网站的 SHA1 工作量证明（PoW）挑战
- ✅ **智能搜索匹配**：支持书名、作者、ISBN 等多维度匹配
- ✅ **格式优先级**：自动选择 epub + pdf 版本下载
- ✅ **代理轮询**：支持 Shadowsocks（Surge 格式）代理列表，每 IP 5 本后自动切换
- ✅ **代理轮询**：支持 Shadowsocks（Surge 格式）代理列表，或直接通过 Surge `select` 组切换出口
- ✅ **AI 友好**：JSON 输出，结构化搜索结果，适合 AI Agent 调用
## 安装
@@ -23,6 +23,8 @@ pip install curl_cffi beautifulsoup4 lxml
脚本内置自动代理引擎配置。如果使用 Shadowsocks 代理列表（如 Surge 格式），脚本**会在首次运行时自动将对应平台架构的 `mihomo` 引擎打包进执行环境中**，完全脱离 Host 依赖，无需手动通过 brew 安装任何插件。
如果你已经在 macOS 上使用 Surge，并且将 zlib 域名路由到一个专门的 `select` 组，也可以直接使用 Surge HTTP API 轮换该组，无需再让脚本自行启动 `mihomo`。
## 使用方法
### 搜索图书
@@ -55,6 +57,9 @@ python3 zlib_dl.py download --title "Project Hail Mary" --author "Andy Weir" -o
# 使用代理列表
python3 zlib_dl.py download --title "Project Hail Mary" --author "Andy Weir" --proxy-file proxies.txt
# 使用 Surge 的 Z-Library 策略组轮换出口（推荐先配置 .env）
python3 zlib_dl.py download --title "Project Hail Mary" --author "Andy Weir"
```
### 详细模式
@@ -91,6 +96,25 @@ http://127.0.0.1:8080
- 达到上限后自动停止当前代理进程，启动下一个
- SS 代理会自动启动 `mihomo` 进程创建本地隧道，无需宿主机环境准备
### Surge 策略组模式
- 为 Surge 打开 HTTP API，例如：`http-api = <key>@127.0.0.1:6171`
- 将 zlib 相关域名规则指向一个 `select` 组，例如 `Z-Library`
- 脚本会默认通过 `http://127.0.0.1:6152` 访问 Surge 本地 HTTP 代理
- 当命中 IP 限额或下载失败时，脚本会调用 Surge API，把 `Z-Library` 切到下一个策略
在项目根目录创建 `.env`：
```env
SURGE_API_URL=http://127.0.0.1:6171
SURGE_API_KEY=6f8d2c1a9b4e7f35c0d2a8e1b7f94c63
SURGE_GROUP=Z-Library
SURGE_HTTP_PROXY=http://127.0.0.1:6152
SURGE_SOCKS_PROXY=socks5://127.0.0.1:6153
```
CLI 参数优先级高于 `.env`；如果不传 `--surge-api`，但 `.env` 里存在 `SURGE_API_URL`，下载命令会自动进入 Surge 模式。
## AI Agent 集成
### JSON 搜索结果结构
zlib_dl.py
+241 -8
View File
@@ -24,6 +24,8 @@ import sys
import time
import unicodedata
import urllib.request
import urllib.parse
import urllib.error
from dataclasses import dataclass, field, asdict
from pathlib import Path
from typing import Optional, List, Dict, Any, Tuple
@@ -139,6 +141,14 @@ class ProxyConfig:
_process: Any = field(default=None, repr=False)
@dataclass
class SurgeConfig:
api_url: str
api_key: str
group_name: str
proxy_url: str
# ============================================================
# PoW 挑战解决器
# ============================================================
@@ -178,6 +188,26 @@ class ChallengeSolver:
return None
def load_dotenv(dotenv_path: str = ".env"):
"""加载本地 .env 文件，不覆盖已存在的环境变量"""
if not os.path.exists(dotenv_path):
return
try:
with open(dotenv_path, "r", encoding="utf-8") as f:
for raw_line in f:
line = raw_line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
key = key.strip()
value = value.strip().strip('"').strip("'")
if key and key not in os.environ:
os.environ[key] = value
except Exception:
pass
# ============================================================
# ZLib 会话管理
# ============================================================
@@ -222,14 +252,14 @@ class ZLibSession:
)
ip = parser(resp.json())
print(
f" 🌐 [{stage}] 当前出口 IP: {ip} (proxy={proxy_desc}, probe={probe_url})",
f" 🌐 [{stage}] 调试探针出口 IP: {ip} (proxy={proxy_desc}, probe={probe_url})",
file=sys.stderr,
)
return
except Exception as e:
last_error = e
print(f" 🌐 [{stage}] 当前出口 IP 获取失败: {last_error}", file=sys.stderr)
print(f" 🌐 [{stage}] 调试探针出口 IP 获取失败: {last_error}", file=sys.stderr)
def _request(self, url: str, **kwargs) -> cffi_requests.Response:
"""发起请求，自动处理 PoW 挑战"""
@@ -485,6 +515,173 @@ class ZLibSession:
# 代理管理器
# ============================================================
class SurgeRuntime:
"""通过 Surge HTTP API 轮换指定策略组"""
MAX_DOWNLOADS_PER_PROXY = 5
def __init__(self, config: SurgeConfig, verbose: bool = False):
self.config = config
self.verbose = verbose
self.policy_names: List[str] = []
self.current_index = -1
self.downloads_count = 0
def _api_get(self, path: str, params: Optional[Dict[str, str]] = None) -> Any:
url = urllib.parse.urljoin(self.config.api_url.rstrip("/") + "/", path.lstrip("/"))
if params:
url = f"{url}?{urllib.parse.urlencode(params)}"
req = urllib.request.Request(url, headers={"X-Key": self.config.api_key, "Accept": "application/json"})
with urllib.request.urlopen(req, timeout=10) as resp:
return json.loads(resp.read().decode("utf-8"))
def _api_post(self, path: str, payload: Dict[str, Any]) -> Any:
url = urllib.parse.urljoin(self.config.api_url.rstrip("/") + "/", path.lstrip("/"))
data = json.dumps(payload).encode("utf-8")
req = urllib.request.Request(
url,
data=data,
method="POST",
headers={
"X-Key": self.config.api_key,
"Accept": "application/json",
"Content-Type": "application/json",
},
)
with urllib.request.urlopen(req, timeout=10) as resp:
return json.loads(resp.read().decode("utf-8"))
def _extract_group_policies(self, payload: Any) -> List[str]:
def _coerce_names(items: Any) -> List[str]:
names: List[str] = []
if isinstance(items, list):
for item in items:
if isinstance(item, str):
names.append(item)
elif isinstance(item, dict) and item.get("name"):
names.append(str(item["name"]))
return names
if isinstance(payload, dict):
direct = payload.get(self.config.group_name)
names = _coerce_names(direct)
if names:
return names
groups = payload.get("policy_groups")
if isinstance(groups, list):
for group in groups:
if isinstance(group, dict) and group.get("name") == self.config.group_name:
names = _coerce_names(group.get("policies") or group.get("options") or [])
if names:
return names
groups = payload.get("groups")
if isinstance(groups, dict):
group = groups.get(self.config.group_name)
if isinstance(group, dict):
names = _coerce_names(group.get("policies") or group.get("options") or [])
if names:
return names
names = _coerce_names(group)
if names:
return names
raise RuntimeError(f"无法从 Surge API 响应中解析策略组 {self.config.group_name}")
def _load_policies(self) -> List[str]:
data = self._api_get("/v1/policy_groups")
policies = self._extract_group_policies(data)
if not policies:
raise RuntimeError(f"策略组 {self.config.group_name} 中没有可用代理")
self.policy_names = policies
return policies
def _sync_current_index(self):
current = self.get_current_proxy_name(refresh=True)
if current in self.policy_names:
self.current_index = self.policy_names.index(current)
def has_proxy_pool(self) -> bool:
return True
def ensure_active(self) -> bool:
try:
if not self.policy_names:
self._load_policies()
self._sync_current_index()
return self.current_index >= 0
except Exception as e:
print(f"代理错误: 初始化 Surge 策略组失败: {e}", file=sys.stderr)
return False
def get_current_proxy_url(self) -> Optional[str]:
return self.config.proxy_url
def get_current_proxy_name(self, refresh: bool = True) -> str:
try:
data = self._api_get("/v1/policy_groups/select", {"group_name": self.config.group_name}) if refresh else None
if data is not None:
policy = data.get("policy")
if isinstance(policy, str) and policy:
if policy in self.policy_names:
self.current_index = self.policy_names.index(policy)
return policy
except Exception:
pass
if 0 <= self.current_index < len(self.policy_names):
return self.policy_names[self.current_index]
return self.config.group_name
def record_download(self):
self.downloads_count += 1
def should_rotate(self) -> bool:
return self.downloads_count >= self.MAX_DOWNLOADS_PER_PROXY
def rotate(self) -> bool:
try:
if not self.policy_names:
self._load_policies()
self._sync_current_index()
if not self.policy_names:
return False
next_index = self.current_index + 1
if next_index >= len(self.policy_names):
if self.verbose:
print(" ⚠️ Surge 策略组中的代理已用完", file=sys.stderr)
return False
next_policy = self.policy_names[next_index]
self._api_post(
"/v1/policy_groups/select",
{"group_name": self.config.group_name, "policy": next_policy},
)
time.sleep(1.0)
self.current_index = next_index
self.downloads_count = 0
if self.verbose:
print(f" 📡 Surge 已切换策略组 {self.config.group_name} -> {next_policy}", file=sys.stderr)
return True
except Exception as e:
print(f"代理错误: 切换 Surge 策略组失败: {e}", file=sys.stderr)
return False
def remaining_proxies(self) -> int:
if not self.policy_names:
return 0
return max(0, len(self.policy_names) - self.current_index - 1)
def remaining_downloads(self) -> int:
return max(0, self.MAX_DOWNLOADS_PER_PROXY - self.downloads_count)
def cleanup(self):
return None
class ProxyManager:
"""管理代理列表和轮询"""
@@ -1124,7 +1321,7 @@ def _should_rotate_proxy(message: str) -> bool:
def _create_session_for_current_proxy(
proxy_mgr: ProxyManager,
proxy_mgr: Any,
explicit_proxy: Optional[str],
verbose: bool,
) -> ZLibSession:
@@ -1137,20 +1334,47 @@ def cmd_download(args):
"""执行下载命令"""
explicit_proxy = getattr(args, "proxy", None)
proxy_file = getattr(args, "proxy_file", None)
if explicit_proxy and proxy_file:
print("错误: --proxy 与 --proxy-file 不能同时使用", file=sys.stderr)
surge_key = getattr(args, "surge_key", None) or os.environ.get("SURGE_API_KEY")
surge_api = getattr(args, "surge_api", None) or os.environ.get("SURGE_API_URL")
use_surge = bool(surge_api)
if len([item for item in [explicit_proxy, proxy_file, surge_api] if item]) > 1:
print("错误: --proxy / --proxy-file / --surge-api 只能选择一种模式", file=sys.stderr)
sys.exit(1)
if use_surge and not surge_key:
print("错误: Surge 模式需要 --surge-key 或环境变量 SURGE_API_KEY", file=sys.stderr)
sys.exit(1)
output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
# 代理管理器
if use_surge:
surge_key_str = str(surge_key)
surge_proxy_url = (
getattr(args, "surge_http_proxy", None)
or os.environ.get("SURGE_HTTP_PROXY")
or getattr(args, "surge_socks_proxy", None)
or os.environ.get("SURGE_SOCKS_PROXY")
or "http://127.0.0.1:6152"
)
proxy_mgr = SurgeRuntime(
config=SurgeConfig(
api_url=surge_api,
api_key=surge_key_str,
group_name=getattr(args, "surge_group", None) or os.environ.get("SURGE_GROUP", "Z-Library"),
proxy_url=surge_proxy_url,
),
verbose=args.verbose,
)
explicit_proxy = None
else:
proxy_mgr = ProxyManager(
proxy_file=proxy_file, verbose=args.verbose
)
if proxy_mgr.has_proxy_pool() and not proxy_mgr.ensure_active():
print("错误: 代理池不可用，无法启动任何代理", file=sys.stderr)
print("错误: 代理不可用，无法初始化当前代理运行时", file=sys.stderr)
sys.exit(1)
download_results: List[DownloadResult] = []
@@ -1278,7 +1502,7 @@ def _download_one(
author: str,
fmt: str,
output_dir: Path,
proxy_mgr: ProxyManager,
proxy_mgr: Any,
explicit_proxy: Optional[str],
verbose: bool,
) -> DownloadResult:
@@ -1432,6 +1656,9 @@ def main():
# 使用代理下载
%(prog)s download --title "Project Hail Mary" --author "Andy Weir" --proxy-file proxies.txt
# 使用 Surge 策略组轮换下载
%(prog)s download --title "Project Hail Mary" --author "Andy Weir" --surge-api http://127.0.0.1:6171
""",
)
@@ -1464,7 +1691,13 @@ def main():
dl_p.add_argument("--output-dir", "-o", default="./downloads", help="下载目录 (默认 ./downloads)")
dl_p.add_argument("--proxy-file", "-p", default=None, help="代理列表文件")
dl_p.add_argument("--proxy", default=None, help="单个代理 URL (socks5://host:port)")
dl_p.add_argument("--surge-api", default=None, help="Surge HTTP API 地址 (如 http://127.0.0.1:6171)")
dl_p.add_argument("--surge-key", default=None, help="Surge HTTP API Key，也可用环境变量 SURGE_API_KEY")
dl_p.add_argument("--surge-group", default=None, help="Surge select 组名 (默认读取 .env 或 Z-Library)")
dl_p.add_argument("--surge-http-proxy", default=None, help="Surge 本地 HTTP 代理 (默认读取 .env 或 http://127.0.0.1:6152)")
dl_p.add_argument("--surge-socks-proxy", default=None, help="Surge 本地 SOCKS5 代理 (如 socks5://127.0.0.1:6153)")
load_dotenv()
args = parser.parse_args()
if not args.command: