release: cut v0.02 with Surge API proxy rotation

This commit is contained in:
Kai Tan
2026-04-01 00:31:51 +08:00
parent 97fff0b30e
commit 3f7a6bed0d
3 changed files with 270 additions and 12 deletions
+1
View File
@@ -1,3 +1,4 @@
downloads/ downloads/
__pycache__/ __pycache__/
*.pyc *.pyc
.env
+25 -1
View File
@@ -7,7 +7,7 @@
-**自动绕过反爬保护**:自动解决网站的 SHA1 工作量证明(PoW)挑战 -**自动绕过反爬保护**:自动解决网站的 SHA1 工作量证明(PoW)挑战
-**智能搜索匹配**:支持书名、作者、ISBN 等多维度匹配 -**智能搜索匹配**:支持书名、作者、ISBN 等多维度匹配
-**格式优先级**:自动选择 epub + pdf 版本下载 -**格式优先级**:自动选择 epub + pdf 版本下载
-**代理轮询**:支持 ShadowsocksSurge 格式)代理列表,每 IP 5 本后自动切换 -**代理轮询**:支持 ShadowsocksSurge 格式)代理列表,或直接通过 Surge `select` 组切换出口
-**AI 友好**:JSON 输出,结构化搜索结果,适合 AI Agent 调用 -**AI 友好**:JSON 输出,结构化搜索结果,适合 AI Agent 调用
## 安装 ## 安装
@@ -23,6 +23,8 @@ pip install curl_cffi beautifulsoup4 lxml
脚本内置自动代理引擎配置。如果使用 Shadowsocks 代理列表(如 Surge 格式),脚本**会在首次运行时自动将对应平台架构的 `mihomo` 引擎打包进执行环境中**,完全脱离 Host 依赖,无需手动通过 brew 安装任何插件。 脚本内置自动代理引擎配置。如果使用 Shadowsocks 代理列表(如 Surge 格式),脚本**会在首次运行时自动将对应平台架构的 `mihomo` 引擎打包进执行环境中**,完全脱离 Host 依赖,无需手动通过 brew 安装任何插件。
如果你已经在 macOS 上使用 Surge,并且将 zlib 域名路由到一个专门的 `select` 组,也可以直接使用 Surge HTTP API 轮换该组,无需再让脚本自行启动 `mihomo`
## 使用方法 ## 使用方法
### 搜索图书 ### 搜索图书
@@ -55,6 +57,9 @@ python3 zlib_dl.py download --title "Project Hail Mary" --author "Andy Weir" -o
# 使用代理列表 # 使用代理列表
python3 zlib_dl.py download --title "Project Hail Mary" --author "Andy Weir" --proxy-file proxies.txt python3 zlib_dl.py download --title "Project Hail Mary" --author "Andy Weir" --proxy-file proxies.txt
# 使用 Surge 的 Z-Library 策略组轮换出口(推荐先配置 .env)
python3 zlib_dl.py download --title "Project Hail Mary" --author "Andy Weir"
``` ```
### 详细模式 ### 详细模式
@@ -91,6 +96,25 @@ http://127.0.0.1:8080
- 达到上限后自动停止当前代理进程,启动下一个 - 达到上限后自动停止当前代理进程,启动下一个
- SS 代理会自动启动 `mihomo` 进程创建本地隧道,无需宿主机环境准备 - SS 代理会自动启动 `mihomo` 进程创建本地隧道,无需宿主机环境准备
### Surge 策略组模式
- 为 Surge 打开 HTTP API,例如:`http-api = <key>@127.0.0.1:6171`
- 将 zlib 相关域名规则指向一个 `select` 组,例如 `Z-Library`
- 脚本会默认通过 `http://127.0.0.1:6152` 访问 Surge 本地 HTTP 代理
- 当命中 IP 限额或下载失败时,脚本会调用 Surge API,把 `Z-Library` 切到下一个策略
在项目根目录创建 `.env`
```env
SURGE_API_URL=http://127.0.0.1:6171
SURGE_API_KEY=6f8d2c1a9b4e7f35c0d2a8e1b7f94c63
SURGE_GROUP=Z-Library
SURGE_HTTP_PROXY=http://127.0.0.1:6152
SURGE_SOCKS_PROXY=socks5://127.0.0.1:6153
```
CLI 参数优先级高于 `.env`;如果不传 `--surge-api`,但 `.env` 里存在 `SURGE_API_URL`,下载命令会自动进入 Surge 模式。
## AI Agent 集成 ## AI Agent 集成
### JSON 搜索结果结构 ### JSON 搜索结果结构
+241 -8
View File
@@ -24,6 +24,8 @@ import sys
import time import time
import unicodedata import unicodedata
import urllib.request import urllib.request
import urllib.parse
import urllib.error
from dataclasses import dataclass, field, asdict from dataclasses import dataclass, field, asdict
from pathlib import Path from pathlib import Path
from typing import Optional, List, Dict, Any, Tuple from typing import Optional, List, Dict, Any, Tuple
@@ -139,6 +141,14 @@ class ProxyConfig:
_process: Any = field(default=None, repr=False) _process: Any = field(default=None, repr=False)
@dataclass
class SurgeConfig:
api_url: str
api_key: str
group_name: str
proxy_url: str
# ============================================================ # ============================================================
# PoW 挑战解决器 # PoW 挑战解决器
# ============================================================ # ============================================================
@@ -178,6 +188,26 @@ class ChallengeSolver:
return None return None
def load_dotenv(dotenv_path: str = ".env"):
"""加载本地 .env 文件,不覆盖已存在的环境变量"""
if not os.path.exists(dotenv_path):
return
try:
with open(dotenv_path, "r", encoding="utf-8") as f:
for raw_line in f:
line = raw_line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, value = line.split("=", 1)
key = key.strip()
value = value.strip().strip('"').strip("'")
if key and key not in os.environ:
os.environ[key] = value
except Exception:
pass
# ============================================================ # ============================================================
# ZLib 会话管理 # ZLib 会话管理
# ============================================================ # ============================================================
@@ -222,14 +252,14 @@ class ZLibSession:
) )
ip = parser(resp.json()) ip = parser(resp.json())
print( print(
f" 🌐 [{stage}] 当前出口 IP: {ip} (proxy={proxy_desc}, probe={probe_url})", f" 🌐 [{stage}] 调试探针出口 IP: {ip} (proxy={proxy_desc}, probe={probe_url})",
file=sys.stderr, file=sys.stderr,
) )
return return
except Exception as e: except Exception as e:
last_error = e last_error = e
print(f" 🌐 [{stage}] 当前出口 IP 获取失败: {last_error}", file=sys.stderr) print(f" 🌐 [{stage}] 调试探针出口 IP 获取失败: {last_error}", file=sys.stderr)
def _request(self, url: str, **kwargs) -> cffi_requests.Response: def _request(self, url: str, **kwargs) -> cffi_requests.Response:
"""发起请求,自动处理 PoW 挑战""" """发起请求,自动处理 PoW 挑战"""
@@ -485,6 +515,173 @@ class ZLibSession:
# 代理管理器 # 代理管理器
# ============================================================ # ============================================================
class SurgeRuntime:
"""通过 Surge HTTP API 轮换指定策略组"""
MAX_DOWNLOADS_PER_PROXY = 5
def __init__(self, config: SurgeConfig, verbose: bool = False):
self.config = config
self.verbose = verbose
self.policy_names: List[str] = []
self.current_index = -1
self.downloads_count = 0
def _api_get(self, path: str, params: Optional[Dict[str, str]] = None) -> Any:
url = urllib.parse.urljoin(self.config.api_url.rstrip("/") + "/", path.lstrip("/"))
if params:
url = f"{url}?{urllib.parse.urlencode(params)}"
req = urllib.request.Request(url, headers={"X-Key": self.config.api_key, "Accept": "application/json"})
with urllib.request.urlopen(req, timeout=10) as resp:
return json.loads(resp.read().decode("utf-8"))
def _api_post(self, path: str, payload: Dict[str, Any]) -> Any:
url = urllib.parse.urljoin(self.config.api_url.rstrip("/") + "/", path.lstrip("/"))
data = json.dumps(payload).encode("utf-8")
req = urllib.request.Request(
url,
data=data,
method="POST",
headers={
"X-Key": self.config.api_key,
"Accept": "application/json",
"Content-Type": "application/json",
},
)
with urllib.request.urlopen(req, timeout=10) as resp:
return json.loads(resp.read().decode("utf-8"))
def _extract_group_policies(self, payload: Any) -> List[str]:
def _coerce_names(items: Any) -> List[str]:
names: List[str] = []
if isinstance(items, list):
for item in items:
if isinstance(item, str):
names.append(item)
elif isinstance(item, dict) and item.get("name"):
names.append(str(item["name"]))
return names
if isinstance(payload, dict):
direct = payload.get(self.config.group_name)
names = _coerce_names(direct)
if names:
return names
groups = payload.get("policy_groups")
if isinstance(groups, list):
for group in groups:
if isinstance(group, dict) and group.get("name") == self.config.group_name:
names = _coerce_names(group.get("policies") or group.get("options") or [])
if names:
return names
groups = payload.get("groups")
if isinstance(groups, dict):
group = groups.get(self.config.group_name)
if isinstance(group, dict):
names = _coerce_names(group.get("policies") or group.get("options") or [])
if names:
return names
names = _coerce_names(group)
if names:
return names
raise RuntimeError(f"无法从 Surge API 响应中解析策略组 {self.config.group_name}")
def _load_policies(self) -> List[str]:
data = self._api_get("/v1/policy_groups")
policies = self._extract_group_policies(data)
if not policies:
raise RuntimeError(f"策略组 {self.config.group_name} 中没有可用代理")
self.policy_names = policies
return policies
def _sync_current_index(self):
current = self.get_current_proxy_name(refresh=True)
if current in self.policy_names:
self.current_index = self.policy_names.index(current)
def has_proxy_pool(self) -> bool:
return True
def ensure_active(self) -> bool:
try:
if not self.policy_names:
self._load_policies()
self._sync_current_index()
return self.current_index >= 0
except Exception as e:
print(f"代理错误: 初始化 Surge 策略组失败: {e}", file=sys.stderr)
return False
def get_current_proxy_url(self) -> Optional[str]:
return self.config.proxy_url
def get_current_proxy_name(self, refresh: bool = True) -> str:
try:
data = self._api_get("/v1/policy_groups/select", {"group_name": self.config.group_name}) if refresh else None
if data is not None:
policy = data.get("policy")
if isinstance(policy, str) and policy:
if policy in self.policy_names:
self.current_index = self.policy_names.index(policy)
return policy
except Exception:
pass
if 0 <= self.current_index < len(self.policy_names):
return self.policy_names[self.current_index]
return self.config.group_name
def record_download(self):
self.downloads_count += 1
def should_rotate(self) -> bool:
return self.downloads_count >= self.MAX_DOWNLOADS_PER_PROXY
def rotate(self) -> bool:
try:
if not self.policy_names:
self._load_policies()
self._sync_current_index()
if not self.policy_names:
return False
next_index = self.current_index + 1
if next_index >= len(self.policy_names):
if self.verbose:
print(" ⚠️ Surge 策略组中的代理已用完", file=sys.stderr)
return False
next_policy = self.policy_names[next_index]
self._api_post(
"/v1/policy_groups/select",
{"group_name": self.config.group_name, "policy": next_policy},
)
time.sleep(1.0)
self.current_index = next_index
self.downloads_count = 0
if self.verbose:
print(f" 📡 Surge 已切换策略组 {self.config.group_name} -> {next_policy}", file=sys.stderr)
return True
except Exception as e:
print(f"代理错误: 切换 Surge 策略组失败: {e}", file=sys.stderr)
return False
def remaining_proxies(self) -> int:
if not self.policy_names:
return 0
return max(0, len(self.policy_names) - self.current_index - 1)
def remaining_downloads(self) -> int:
return max(0, self.MAX_DOWNLOADS_PER_PROXY - self.downloads_count)
def cleanup(self):
return None
class ProxyManager: class ProxyManager:
"""管理代理列表和轮询""" """管理代理列表和轮询"""
@@ -1124,7 +1321,7 @@ def _should_rotate_proxy(message: str) -> bool:
def _create_session_for_current_proxy( def _create_session_for_current_proxy(
proxy_mgr: ProxyManager, proxy_mgr: Any,
explicit_proxy: Optional[str], explicit_proxy: Optional[str],
verbose: bool, verbose: bool,
) -> ZLibSession: ) -> ZLibSession:
@@ -1137,20 +1334,47 @@ def cmd_download(args):
"""执行下载命令""" """执行下载命令"""
explicit_proxy = getattr(args, "proxy", None) explicit_proxy = getattr(args, "proxy", None)
proxy_file = getattr(args, "proxy_file", None) proxy_file = getattr(args, "proxy_file", None)
if explicit_proxy and proxy_file: surge_key = getattr(args, "surge_key", None) or os.environ.get("SURGE_API_KEY")
print("错误: --proxy 与 --proxy-file 不能同时使用", file=sys.stderr) surge_api = getattr(args, "surge_api", None) or os.environ.get("SURGE_API_URL")
use_surge = bool(surge_api)
if len([item for item in [explicit_proxy, proxy_file, surge_api] if item]) > 1:
print("错误: --proxy / --proxy-file / --surge-api 只能选择一种模式", file=sys.stderr)
sys.exit(1)
if use_surge and not surge_key:
print("错误: Surge 模式需要 --surge-key 或环境变量 SURGE_API_KEY", file=sys.stderr)
sys.exit(1) sys.exit(1)
output_dir = Path(args.output_dir) output_dir = Path(args.output_dir)
output_dir.mkdir(parents=True, exist_ok=True) output_dir.mkdir(parents=True, exist_ok=True)
# 代理管理器 if use_surge:
surge_key_str = str(surge_key)
surge_proxy_url = (
getattr(args, "surge_http_proxy", None)
or os.environ.get("SURGE_HTTP_PROXY")
or getattr(args, "surge_socks_proxy", None)
or os.environ.get("SURGE_SOCKS_PROXY")
or "http://127.0.0.1:6152"
)
proxy_mgr = SurgeRuntime(
config=SurgeConfig(
api_url=surge_api,
api_key=surge_key_str,
group_name=getattr(args, "surge_group", None) or os.environ.get("SURGE_GROUP", "Z-Library"),
proxy_url=surge_proxy_url,
),
verbose=args.verbose,
)
explicit_proxy = None
else:
proxy_mgr = ProxyManager( proxy_mgr = ProxyManager(
proxy_file=proxy_file, verbose=args.verbose proxy_file=proxy_file, verbose=args.verbose
) )
if proxy_mgr.has_proxy_pool() and not proxy_mgr.ensure_active(): if proxy_mgr.has_proxy_pool() and not proxy_mgr.ensure_active():
print("错误: 代理不可用,无法启动任何代理", file=sys.stderr) print("错误: 代理不可用,无法初始化当前代理运行时", file=sys.stderr)
sys.exit(1) sys.exit(1)
download_results: List[DownloadResult] = [] download_results: List[DownloadResult] = []
@@ -1278,7 +1502,7 @@ def _download_one(
author: str, author: str,
fmt: str, fmt: str,
output_dir: Path, output_dir: Path,
proxy_mgr: ProxyManager, proxy_mgr: Any,
explicit_proxy: Optional[str], explicit_proxy: Optional[str],
verbose: bool, verbose: bool,
) -> DownloadResult: ) -> DownloadResult:
@@ -1432,6 +1656,9 @@ def main():
# 使用代理下载 # 使用代理下载
%(prog)s download --title "Project Hail Mary" --author "Andy Weir" --proxy-file proxies.txt %(prog)s download --title "Project Hail Mary" --author "Andy Weir" --proxy-file proxies.txt
# 使用 Surge 策略组轮换下载
%(prog)s download --title "Project Hail Mary" --author "Andy Weir" --surge-api http://127.0.0.1:6171
""", """,
) )
@@ -1464,7 +1691,13 @@ def main():
dl_p.add_argument("--output-dir", "-o", default="./downloads", help="下载目录 (默认 ./downloads)") dl_p.add_argument("--output-dir", "-o", default="./downloads", help="下载目录 (默认 ./downloads)")
dl_p.add_argument("--proxy-file", "-p", default=None, help="代理列表文件") dl_p.add_argument("--proxy-file", "-p", default=None, help="代理列表文件")
dl_p.add_argument("--proxy", default=None, help="单个代理 URL (socks5://host:port)") dl_p.add_argument("--proxy", default=None, help="单个代理 URL (socks5://host:port)")
dl_p.add_argument("--surge-api", default=None, help="Surge HTTP API 地址 (如 http://127.0.0.1:6171)")
dl_p.add_argument("--surge-key", default=None, help="Surge HTTP API Key,也可用环境变量 SURGE_API_KEY")
dl_p.add_argument("--surge-group", default=None, help="Surge select 组名 (默认读取 .env 或 Z-Library)")
dl_p.add_argument("--surge-http-proxy", default=None, help="Surge 本地 HTTP 代理 (默认读取 .env 或 http://127.0.0.1:6152)")
dl_p.add_argument("--surge-socks-proxy", default=None, help="Surge 本地 SOCKS5 代理 (如 socks5://127.0.0.1:6153)")
load_dotenv()
args = parser.parse_args() args = parser.parse_args()
if not args.command: if not args.command: