deploy WeChat fulltext service to tools-pve

This commit is contained in:
Codex
2026-09-17 19:46:27 -07:00
parent fa57ab7f3b
commit bead2bb3ea
8 changed files with 1346 additions and 0 deletions
@@ -0,0 +1,677 @@
#!/usr/bin/env node
import { chromium } from 'playwright';
import { mkdir, writeFile } from 'node:fs/promises';
import { homedir } from 'node:os';
import { basename, dirname, join, resolve } from 'node:path';
import { pathToFileURL } from 'node:url';
import TurndownService from 'turndown';
import turndownPluginGfm from 'turndown-plugin-gfm';
const { gfm } = turndownPluginGfm;
const DEFAULTS = {
format: 'all',
pdfMode: 'continuous',
outputDir: process.cwd(),
profile: join(homedir(), '.cache', 'wechat-article-exporter', 'profile'),
selector: '',
timeout: 90_000,
imageTimeout: 45_000,
manualWait: 300_000,
headed: false,
allowMissingImages: false,
keepRemoteLinks: false,
};
function usage(exitCode = 0) {
const command = basename(process.argv[1] || 'wechat-export.mjs');
console.log(`
用法:
node ${command} <微信文章 URL> [选项]
选项:
-o, --output-dir <目录> 输出目录(默认: 当前目录)
--format <all|html|pdf|markdown> 输出类型(默认: all)
--pdf-mode <continuous|a4> 单张长页 PDF,或标准 A4 分页(默认: continuous)
--selector <CSS> 手工指定正文元素
--profile <目录> 持久浏览器资料目录(保存验证状态/Cookie)
--headed 显示浏览器,可人工完成微信验证
--manual-wait <秒> headed 模式等待人工验证的时间(默认: 300)
--timeout <秒> 页面加载超时(默认: 90)
--image-timeout <秒> 等待图片加载的总时间(默认: 45)
--allow-missing-images 即使图片不完整也输出(默认会失败关闭)
--keep-remote-links 不移除正文中的远程超链接
-h, --help 显示帮助
示例:
node ${command} 'https://mp.weixin.qq.com/s/...' -o ./out
node ${command} 'https://mp.weixin.qq.com/s/...' --headed --pdf-mode a4
`);
process.exit(exitCode);
}
function parseArgs(argv) {
const options = { ...DEFAULTS };
let url = '';
const valueFor = (arg, index) => {
const value = argv[index + 1];
if (!value || value.startsWith('--')) throw new Error(`${arg} 缺少参数`);
return value;
};
for (let i = 0; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === '-h' || arg === '--help') usage(0);
else if (arg === '-o' || arg === '--output-dir') options.outputDir = valueFor(arg, i++);
else if (arg === '--format') options.format = valueFor(arg, i++);
else if (arg === '--pdf-mode') options.pdfMode = valueFor(arg, i++);
else if (arg === '--selector') options.selector = valueFor(arg, i++);
else if (arg === '--profile') options.profile = valueFor(arg, i++);
else if (arg === '--timeout') options.timeout = Number(valueFor(arg, i++)) * 1000;
else if (arg === '--image-timeout') options.imageTimeout = Number(valueFor(arg, i++)) * 1000;
else if (arg === '--manual-wait') options.manualWait = Number(valueFor(arg, i++)) * 1000;
else if (arg === '--headed') options.headed = true;
else if (arg === '--allow-missing-images') options.allowMissingImages = true;
else if (arg === '--keep-remote-links') options.keepRemoteLinks = true;
else if (arg.startsWith('-')) throw new Error(`未知选项: ${arg}`);
else if (!url) url = arg;
else throw new Error(`多余参数: ${arg}`);
}
if (!url) usage(1);
if (!['all', 'html', 'pdf', 'markdown'].includes(options.format)) throw new Error('--format 必须是 all、html、pdf 或 markdown');
if (!['continuous', 'a4'].includes(options.pdfMode)) throw new Error('--pdf-mode 必须是 continuous 或 a4');
for (const key of ['timeout', 'imageTimeout', 'manualWait']) {
if (!Number.isFinite(options[key]) || options[key] <= 0) throw new Error(`${key} 必须是正数`);
}
options.outputDir = resolve(options.outputDir);
options.profile = resolve(options.profile);
return { url, options };
}
function safeFilename(value) {
const cleaned = String(value || 'wechat-article')
.normalize('NFKC')
.replace(/[\\/:*?"<>|\u0000-\u001f]/g, '-')
.replace(/\s+/g, ' ')
.replace(/[. ]+$/g, '')
.trim();
return (cleaned || 'wechat-article').slice(0, 100);
}
function htmlEscape(value) {
return String(value ?? '')
.replaceAll('&', '&amp;')
.replaceAll('<', '&lt;')
.replaceAll('>', '&gt;')
.replaceAll('"', '&quot;')
.replaceAll("'", '&#39;');
}
function inferMime(buffer, declared = '') {
const type = declared.split(';')[0].trim().toLowerCase();
if (type.startsWith('image/')) return type;
if (buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return 'image/png';
if (buffer.subarray(0, 3).equals(Buffer.from([0xff, 0xd8, 0xff]))) return 'image/jpeg';
if (buffer.subarray(0, 6).toString('ascii') === 'GIF87a' || buffer.subarray(0, 6).toString('ascii') === 'GIF89a') return 'image/gif';
if (buffer.subarray(0, 4).toString('ascii') === 'RIFF' && buffer.subarray(8, 12).toString('ascii') === 'WEBP') return 'image/webp';
const prefix = buffer.subarray(0, 300).toString('utf8').trimStart().toLowerCase();
if (prefix.startsWith('<svg') || (prefix.startsWith('<?xml') && prefix.includes('<svg'))) return 'image/svg+xml';
return '';
}
function cleanErrorMessage(error) {
return String(error?.message || error || '未知错误').replace(/\u001b\[[0-9;]*m/g, '');
}
function isBlockedPage(url, text) {
return /wappoc_appmsgcaptcha|captcha|verify/i.test(url)
|| /环境异常|完成验证|访问过于频繁|拖动下方滑块|安全验证/.test(text);
}
async function chooseArticleSelector(page, explicitSelector) {
const candidates = explicitSelector
? [explicitSelector]
: ['#js_content', '.rich_media_content', 'article', '[data-role="content"]', 'main'];
for (const selector of candidates) {
const candidate = page.locator(selector).first();
if (await candidate.count() && await candidate.isVisible().catch(() => false)) {
const size = await candidate.evaluate((element) => ({
text: (element.innerText || '').trim().length,
images: element.querySelectorAll('img,svg image,canvas').length,
}));
if (size.text >= 80 || size.images > 0) return selector;
}
}
return '';
}
async function waitForArticle(page, options) {
let selector = await chooseArticleSelector(page, options.selector);
if (selector) return selector;
const bodyText = await page.locator('body').innerText({ timeout: 5_000 }).catch(() => '');
const blocked = isBlockedPage(page.url(), bodyText);
if (!options.headed) {
const reason = blocked ? '微信要求安全验证' : '没有识别到正文';
throw new Error(`${reason}。请用 --headed 和相同 --profile 重试,完成验证后脚本会继续。`);
}
console.error(blocked
? `微信要求验证;请在浏览器中完成操作。最多等待 ${options.manualWait / 1000} 秒…`
: `尚未识别到正文;请在浏览器中完成加载。最多等待 ${options.manualWait / 1000} 秒…`);
const deadline = Date.now() + options.manualWait;
while (Date.now() < deadline) {
await page.waitForTimeout(1000);
selector = await chooseArticleSelector(page, options.selector);
if (selector) return selector;
}
throw new Error('等待人工验证/正文加载超时');
}
async function activateLazyImages(page, selector, imageTimeout) {
await page.locator(selector).evaluate((root) => {
for (const image of root.querySelectorAll('img')) {
const lazy = image.getAttribute('data-src')
|| image.getAttribute('data-original')
|| image.getAttribute('data-backsrc');
if (lazy && image.getAttribute('src') !== lazy) {
image.setAttribute('src', lazy);
}
image.loading = 'eager';
image.decoding = 'sync';
}
for (const source of root.querySelectorAll('source')) {
const lazy = source.getAttribute('data-srcset');
if (lazy) source.setAttribute('srcset', lazy);
}
});
let previousHeight = 0;
let stablePasses = 0;
for (let pass = 0; pass < 3 && stablePasses < 2; pass += 1) {
const height = await page.evaluate(() => document.documentElement.scrollHeight);
for (let y = 0; y < height; y += 600) {
await page.evaluate((position) => window.scrollTo(0, position), y);
await page.waitForTimeout(80);
}
await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight));
await page.waitForTimeout(600);
const newHeight = await page.evaluate(() => document.documentElement.scrollHeight);
stablePasses = Math.abs(newHeight - previousHeight) < 10 ? stablePasses + 1 : 0;
previousHeight = newHeight;
}
const deadline = Date.now() + imageTimeout;
while (Date.now() < deadline) {
const status = await page.locator(selector).evaluate((root) => {
const images = [...root.querySelectorAll('img')].filter((image) => {
const src = image.currentSrc || image.src || image.dataset.src;
return src && !src.startsWith('data:');
});
return {
total: images.length,
pending: images.filter((image) => !image.complete).length,
};
});
if (status.pending === 0) break;
await page.waitForTimeout(250);
}
await page.evaluate(() => window.scrollTo(0, 0));
}
async function snapshotArticle(page, selector, keepRemoteLinks) {
return page.locator(selector).evaluate((root, keepLinks) => {
const absolute = (value) => {
if (!value) return '';
if (/^(data|blob):/i.test(value)) return value;
try { return new URL(value, document.baseURI).href; } catch { return value; }
};
const clone = root.cloneNode(true);
const originals = [root, ...root.querySelectorAll('*')];
const clones = [clone, ...clone.querySelectorAll('*')];
const resources = [];
let resourceId = 0;
const register = (element, kind, url) => {
const resolved = absolute(url);
if (!resolved) return '';
const id = `wxasset-${resourceId++}`;
resources.push({ id, kind, url: resolved });
element.setAttribute(`data-wxasset-${kind}`, id);
return id;
};
for (let i = 0; i < originals.length; i += 1) {
const original = originals[i];
const copy = clones[i];
if (!copy) continue;
if (original instanceof HTMLCanvasElement) {
try {
const image = document.createElement('img');
image.src = original.toDataURL('image/png');
image.alt = original.getAttribute('aria-label') || 'canvas';
register(image, 'img', image.src);
image.removeAttribute('src');
copy.replaceWith(image);
} catch {
register(copy, 'canvas', `blob:tainted-canvas-${resourceId}`);
}
continue;
}
if (original instanceof HTMLImageElement) {
const url = original.currentSrc || original.src || original.dataset.src
|| original.getAttribute('data-original') || original.getAttribute('data-backsrc');
register(copy, 'img', url);
copy.setAttribute('alt', original.alt || '');
copy.removeAttribute('src');
copy.removeAttribute('srcset');
copy.removeAttribute('data-src');
copy.removeAttribute('data-original');
copy.removeAttribute('data-backsrc');
copy.removeAttribute('data-srcset');
copy.removeAttribute('loading');
} else if (original instanceof SVGImageElement) {
register(copy, 'svg-image', original.href?.baseVal || original.getAttribute('href'));
copy.removeAttribute('href');
copy.removeAttribute('xlink:href');
}
const background = getComputedStyle(original).backgroundImage;
if (background && background !== 'none') {
const match = background.match(/^url\(["']?(.*?)["']?\)$/);
if (match) {
const id = register(copy, 'background', match[1]);
if (id) copy.style.backgroundImage = `url("__${id}__")`;
}
}
}
for (const bad of clone.querySelectorAll('script,noscript,link,meta,form,button,source')) bad.remove();
for (const frame of clone.querySelectorAll('iframe,video')) {
const source = frame.getAttribute('data-src') || frame.getAttribute('src') || '';
const note = document.createElement('p');
note.className = 'wx-export-media-note';
note.textContent = source ? `原文中的媒体内容:${absolute(source)}` : '原文中的媒体内容(请回到原文查看)';
frame.replaceWith(note);
}
for (const anchor of clone.querySelectorAll('a')) {
const href = absolute(anchor.getAttribute('href'));
if (keepLinks && /^https?:/i.test(href)) {
anchor.setAttribute('href', href);
anchor.setAttribute('rel', 'noopener noreferrer');
} else {
anchor.removeAttribute('href');
anchor.removeAttribute('target');
}
}
clone.removeAttribute('style');
clone.classList.add('wx-export-content');
const text = (selectors) => {
for (const candidate of selectors) {
const element = document.querySelector(candidate);
const value = element?.textContent?.trim();
if (value) return value;
}
return '';
};
const title = text(['#activity-name', '.rich_media_title', 'h1']) || document.title || '微信文章';
const author = text(['#js_name', '.rich_media_meta_nickname', '[property="og:article:author"]']);
const published = text(['#publish_time', '.rich_media_meta_text']) || '';
const canonical = document.querySelector('link[rel="canonical"]')?.href || location.href;
return { title, author, published, canonical, html: clone.outerHTML, resources };
}, keepRemoteLinks);
}
function dataUriToAsset(url) {
const match = url.match(/^data:([^;,]+)?(;base64)?,(.*)$/s);
if (!match) throw new Error('无效的 data URI');
const mime = match[1] || 'text/plain';
const buffer = match[2] ? Buffer.from(match[3], 'base64') : Buffer.from(decodeURIComponent(match[3]));
if (!mime.startsWith('image/')) throw new Error(`data URI 不是图片: ${mime}`);
return { mime, buffer, source: 'inline' };
}
async function fetchAssets(context, resources, responseCache, referer) {
const uniqueUrls = [...new Set(resources.map((item) => item.url))];
const results = new Map();
let cursor = 0;
const fetchOne = async (url) => {
if (url.startsWith('data:')) return dataUriToAsset(url);
if (url.startsWith('blob:')) throw new Error('无法离线保存 blob 图片');
const cached = await responseCache.get(url)?.catch(() => null);
if (cached?.buffer?.length) {
const mime = inferMime(cached.buffer, cached.contentType);
if (mime) return { mime, buffer: cached.buffer, source: 'browser-cache' };
}
let lastError;
for (let attempt = 1; attempt <= 3; attempt += 1) {
try {
const response = await context.request.get(url, {
headers: {
Referer: referer,
Accept: 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
},
timeout: 30_000,
failOnStatusCode: false,
});
if (!response.ok()) throw new Error(`HTTP ${response.status()}`);
const buffer = await response.body();
const mime = inferMime(buffer, response.headers()['content-type'] || '');
if (!mime) throw new Error(`响应不是图片 (${response.headers()['content-type'] || 'unknown'})`);
return { mime, buffer, source: `request-${attempt}` };
} catch (error) {
lastError = error;
await new Promise((done) => setTimeout(done, 350 * (2 ** (attempt - 1))));
}
}
throw lastError;
};
async function worker() {
while (cursor < uniqueUrls.length) {
const url = uniqueUrls[cursor++];
try {
results.set(url, { ok: true, ...(await fetchOne(url)) });
} catch (error) {
results.set(url, { ok: false, error: cleanErrorMessage(error) });
}
}
}
await Promise.all(Array.from({ length: Math.min(6, uniqueUrls.length || 1) }, worker));
return results;
}
function embedAssets(snapshot, assets) {
let html = snapshot.html;
const failures = [];
const records = [];
for (const resource of snapshot.resources) {
const asset = assets.get(resource.url);
const marker = `data-wxasset-${resource.kind}="${resource.id}"`;
if (!asset?.ok) {
failures.push({ ...resource, error: asset?.error || '未抓取' });
records.push({ ...resource, ok: false, error: asset?.error || '未抓取' });
if (resource.kind === 'img') html = html.replace(marker, `src="${htmlEscape(resource.url)}"`);
else if (resource.kind === 'svg-image') html = html.replace(marker, `href="${htmlEscape(resource.url)}"`);
else if (resource.kind === 'background') {
html = html.replace(`__${resource.id}__`, resource.url.replaceAll('"', '%22'));
html = html.replace(marker, '');
}
continue;
}
const dataUri = `data:${asset.mime};base64,${asset.buffer.toString('base64')}`;
const bytes = asset.buffer.length;
records.push({ ...resource, ok: true, mime: asset.mime, bytes, source: asset.source });
if (resource.kind === 'img') {
html = html.replace(marker, `src="${dataUri}"`);
} else if (resource.kind === 'svg-image') {
html = html.replace(marker, `href="${dataUri}"`);
} else if (resource.kind === 'background') {
html = html.replace(`__${resource.id}__`, dataUri);
html = html.replace(marker, '');
}
}
html = html.replaceAll(/\sdata-wxasset-[\w-]+="[^"]+"/g, '');
return { html, failures, records };
}
function standaloneHtml(snapshot, articleHtml) {
const metadata = [snapshot.author, snapshot.published].filter(Boolean).map(htmlEscape).join(' ');
return `<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="generator" content="wechat-article-exporter">
<title>${htmlEscape(snapshot.title)}</title>
<style>
:root { color-scheme: light; }
* { box-sizing: border-box; }
html { background: #e9e9e9; }
body { margin: 0; color: #222; font-family: -apple-system,BlinkMacSystemFont,"Segoe UI","PingFang SC","Hiragino Sans GB","Microsoft YaHei",Arial,sans-serif; }
.wx-export-page { width: 210mm; min-height: 297mm; margin: 12mm auto; padding: 16mm 18mm 20mm; background: #fff; box-shadow: 0 2mm 8mm rgba(0,0,0,.12); }
.wx-export-header { margin: 0 0 10mm; padding-bottom: 6mm; border-bottom: .25mm solid #ddd; }
.wx-export-title { margin: 0 0 4mm; font-size: 25px; line-height: 1.35; font-weight: 700; overflow-wrap: anywhere; }
.wx-export-meta { color: #777; font-size: 14px; line-height: 1.6; }
.wx-export-source { margin-top: 2mm; color: #888; font-size: 11px; overflow-wrap: anywhere; }
.wx-export-source a { color: inherit; }
.wx-export-content { width: 100% !important; max-width: 100% !important; visibility: visible !important; font-size: 16px; line-height: 1.8; overflow-wrap: anywhere; }
.wx-export-content img, .wx-export-content svg, .wx-export-content canvas { max-width: 100% !important; height: auto !important; }
.wx-export-content table { max-width: 100%; border-collapse: collapse; }
.wx-export-content pre { white-space: pre-wrap; overflow-wrap: anywhere; }
.wx-export-content iframe { max-width: 100%; }
.wx-export-media-note { padding: 3mm; color: #666; background: #f5f5f5; font-size: 13px; overflow-wrap: anywhere; }
img, figure, table, blockquote, pre { break-inside: avoid; page-break-inside: avoid; }
@media (max-width: 210mm) {
html { background: #fff; }
.wx-export-page { width: 100%; min-height: 0; margin: 0; padding: 8vw 6vw 12vw; box-shadow: none; }
}
@media print {
@page { size: A4; margin: 0; }
html, body { width: 210mm; background: #fff; }
.wx-export-page { width: 210mm; min-height: 0; margin: 0; padding: 16mm 18mm 20mm; box-shadow: none; }
}
</style>
</head>
<body>
<main class="wx-export-page">
<header class="wx-export-header">
<h1 class="wx-export-title">${htmlEscape(snapshot.title)}</h1>
${metadata ? `<div class="wx-export-meta">${metadata}</div>` : ''}
<div class="wx-export-source">来源:<a href="${htmlEscape(snapshot.canonical)}">${htmlEscape(snapshot.canonical)}</a></div>
</header>
${articleHtml}
</main>
</body>
</html>`;
}
function mimeExtension(mime) {
const extensions = {
'image/jpeg': 'jpg',
'image/png': 'png',
'image/gif': 'gif',
'image/webp': 'webp',
'image/svg+xml': 'svg',
'image/avif': 'avif',
};
return extensions[mime] || 'bin';
}
function yamlString(value) {
return JSON.stringify(String(value || ''));
}
async function writeMarkdown(snapshot, assets, outputDir, base) {
const assetDirectoryName = `${base}.assets`;
const assetDirectory = join(outputDir, assetDirectoryName);
await mkdir(assetDirectory, { recursive: true });
const filenames = new Map();
let assetNumber = 0;
for (const resource of snapshot.resources) {
const asset = assets.get(resource.url);
if (!asset?.ok || filenames.has(resource.url)) continue;
assetNumber += 1;
const filename = `image-${String(assetNumber).padStart(3, '0')}.${mimeExtension(asset.mime)}`;
await writeFile(join(assetDirectory, filename), asset.buffer);
filenames.set(resource.url, `${assetDirectoryName}/${filename}`);
}
let sourceHtml = snapshot.html;
for (const resource of snapshot.resources) {
const marker = `data-wxasset-${resource.kind}="${resource.id}"`;
const localPath = filenames.get(resource.url);
if (resource.kind === 'img') {
sourceHtml = sourceHtml.replace(marker, localPath ? `src="${htmlEscape(localPath)}"` : '');
} else if (resource.kind === 'svg-image') {
sourceHtml = sourceHtml.replace(marker, localPath ? `href="${htmlEscape(localPath)}"` : '');
} else if (resource.kind === 'background') {
sourceHtml = sourceHtml.replace(`__${resource.id}__`, localPath || '');
sourceHtml = sourceHtml.replace(marker, '');
}
}
sourceHtml = sourceHtml.replaceAll(/\sdata-wxasset-[\w-]+="[^"]+"/g, '');
const turndown = new TurndownService({
headingStyle: 'atx',
bulletListMarker: '-',
codeBlockStyle: 'fenced',
emDelimiter: '*',
strongDelimiter: '**',
});
turndown.use(gfm);
turndown.addRule('wechatMediaNote', {
filter: (node) => node.nodeType === 1 && node.classList?.contains('wx-export-media-note'),
replacement: (content) => `\n\n> ${content.trim()}\n\n`,
});
const body = turndown.turndown(sourceHtml).replace(/\n{3,}/g, '\n\n').trim();
const markdown = `---
title: ${yamlString(snapshot.title)}
author: ${yamlString(snapshot.author)}
published: ${yamlString(snapshot.published)}
source: ${yamlString(snapshot.canonical)}
archived_at: ${yamlString(new Date().toISOString())}
---
# ${snapshot.title}
${body}
`;
const markdownPath = join(outputDir, `${base}.md`);
await writeFile(markdownPath, markdown);
return markdownPath;
}
async function writePdf(browserContext, htmlPath, pdfPath, mode) {
const page = await browserContext.newPage();
await page.goto(pathToFileURL(htmlPath).href, { waitUntil: 'load' });
await page.emulateMedia({ media: 'print' });
await page.evaluate(() => document.fonts?.ready);
if (mode === 'a4') {
await page.pdf({ path: pdfPath, format: 'A4', printBackground: true, margin: { top: 0, right: 0, bottom: 0, left: 0 } });
} else {
const heightPx = await page.evaluate(() => Math.max(
document.body.scrollHeight,
document.documentElement.scrollHeight,
document.querySelector('.wx-export-page')?.scrollHeight || 0,
));
const heightMm = Math.ceil((heightPx * 25.4) / 96) + 2;
if (heightMm > 5000) {
throw new Error(`正文约 ${heightMm}mm,超过 Chromium 单张 PDF 的安全高度 5000mm;请使用 --pdf-mode a4,单页 HTML 已保留`);
}
await page.pdf({
path: pdfPath,
width: '210mm',
height: `${heightMm}mm`,
printBackground: true,
margin: { top: 0, right: 0, bottom: 0, left: 0 },
preferCSSPageSize: false,
});
}
await page.close();
}
async function main() {
const { url, options } = parseArgs(process.argv.slice(2));
await mkdir(options.outputDir, { recursive: true });
await mkdir(options.profile, { recursive: true });
const responseCache = new Map();
const context = await chromium.launchPersistentContext(options.profile, {
headless: !options.headed,
viewport: { width: 1280, height: 900 },
locale: 'zh-CN',
timezoneId: 'Asia/Shanghai',
serviceWorkers: 'block',
userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
});
let exitCode = 0;
try {
const page = context.pages()[0] || await context.newPage();
page.setDefaultTimeout(options.timeout);
page.on('response', (response) => {
if (response.request().resourceType() !== 'image' || !response.ok()) return;
const promise = response.body()
.then((buffer) => ({ buffer, contentType: response.headers()['content-type'] || '' }))
.catch(() => null);
responseCache.set(response.url(), promise);
});
console.error(`打开:${url}`);
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: options.timeout });
const selector = await waitForArticle(page, options);
console.error(`正文:${selector};触发懒加载图片…`);
await activateLazyImages(page, selector, options.imageTimeout);
const snapshot = await snapshotArticle(page, selector, options.keepRemoteLinks);
console.error(`发现 ${snapshot.resources.length} 个图片引用,正在抓取并内嵌…`);
const assets = await fetchAssets(context, snapshot.resources, responseCache, page.url());
const embedded = embedAssets(snapshot, assets);
const base = safeFilename(snapshot.title);
const manifestPath = join(options.outputDir, `${base}.assets.json`);
const manifest = {
sourceUrl: url,
finalUrl: page.url(),
title: snapshot.title,
articleSelector: selector,
generatedAt: new Date().toISOString(),
discoveredReferences: snapshot.resources.length,
uniqueResources: assets.size,
embeddedReferences: embedded.records.filter((item) => item.ok).length,
failedReferences: embedded.failures.length,
complete: embedded.failures.length === 0,
resources: embedded.records,
};
await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
if (embedded.failures.length && !options.allowMissingImages) {
const summary = embedded.failures.slice(0, 5).map((item) => `- ${item.url}: ${item.error}`).join('\n');
throw new Error(`有 ${embedded.failures.length} 个图片引用抓取失败;已写完整性报告,未生成不完整成品。\n${summary}\n报告:${manifestPath}`);
}
if (options.format === 'markdown' || options.format === 'all') {
const markdownPath = await writeMarkdown(snapshot, assets, options.outputDir, base);
console.error(`Markdown:${markdownPath}`);
}
const htmlPath = join(options.outputDir, `${base}.html`);
const html = standaloneHtml(snapshot, embedded.html);
await writeFile(htmlPath, html);
console.error(`HTML:${htmlPath}`);
if (options.format === 'pdf' || options.format === 'all') {
const pdfPath = join(options.outputDir, `${base}.${options.pdfMode === 'continuous' ? 'continuous' : 'a4'}.pdf`);
await writePdf(context, htmlPath, pdfPath, options.pdfMode);
console.error(`PDF:${pdfPath}`);
}
if (options.format === 'pdf') {
// PDF 以离线 HTML 为中间源;保留它有助于审计图片完整性和重新排版。
console.error('提示:保留了生成 PDF 所用的自包含 HTML 中间文件。');
}
console.error(`完整性:${manifest.complete ? '通过' : '允许缺图'}(${manifest.embeddedReferences}/${manifest.discoveredReferences} 个引用已内嵌)`);
} catch (error) {
exitCode = 2;
console.error(`失败:${cleanErrorMessage(error)}`);
} finally {
await context.close();
}
process.exitCode = exitCode;
}
if (pathToFileURL(process.argv[1]).href === import.meta.url) {
main().catch((error) => {
console.error(`失败:${cleanErrorMessage(error.stack || error)}`);
process.exitCode = 2;
});
}
export { htmlEscape, inferMime, safeFilename };