#!/usr/bin/env node import { chromium } from 'playwright'; import { mkdir, writeFile } from 'node:fs/promises'; import { homedir } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import TurndownService from 'turndown'; import turndownPluginGfm from 'turndown-plugin-gfm'; const { gfm } = turndownPluginGfm; const DEFAULTS = { format: 'all', pdfMode: 'continuous', outputDir: process.cwd(), profile: join(homedir(), '.cache', 'wechat-article-exporter', 'profile'), selector: '', timeout: 90_000, imageTimeout: 45_000, manualWait: 300_000, headed: false, allowMissingImages: false, keepRemoteLinks: false, }; function usage(exitCode = 0) { const command = basename(process.argv[1] || 'wechat-export.mjs'); console.log(` 用法: node ${command} <微信文章 URL> [选项] 选项: -o, --output-dir <目录> 输出目录(默认: 当前目录) --format 输出类型(默认: all) --pdf-mode 单张长页 PDF,或标准 A4 分页(默认: continuous) --selector 手工指定正文元素 --profile <目录> 持久浏览器资料目录(保存验证状态/Cookie) --headed 显示浏览器,可人工完成微信验证 --manual-wait <秒> headed 模式等待人工验证的时间(默认: 300) --timeout <秒> 页面加载超时(默认: 90) --image-timeout <秒> 等待图片加载的总时间(默认: 45) --allow-missing-images 即使图片不完整也输出(默认会失败关闭) --keep-remote-links 不移除正文中的远程超链接 -h, --help 显示帮助 示例: node ${command} 'https://mp.weixin.qq.com/s/...' -o ./out node ${command} 'https://mp.weixin.qq.com/s/...' --headed --pdf-mode a4 `); process.exit(exitCode); } function parseArgs(argv) { const options = { ...DEFAULTS }; let url = ''; const valueFor = (arg, index) => { const value = argv[index + 1]; if (!value || value.startsWith('--')) throw new Error(`${arg} 缺少参数`); return value; }; for (let i = 0; i < argv.length; i += 1) { const arg = argv[i]; if (arg === '-h' || arg === '--help') usage(0); else if (arg === '-o' || arg === '--output-dir') options.outputDir = valueFor(arg, i++); else if (arg === '--format') options.format = valueFor(arg, i++); else if (arg === '--pdf-mode') options.pdfMode = valueFor(arg, i++); else if (arg === '--selector') options.selector = valueFor(arg, i++); else if (arg === '--profile') options.profile = valueFor(arg, i++); else if (arg === '--timeout') options.timeout = Number(valueFor(arg, i++)) * 1000; else if (arg === '--image-timeout') options.imageTimeout = Number(valueFor(arg, i++)) * 1000; else if (arg === '--manual-wait') options.manualWait = Number(valueFor(arg, i++)) * 1000; else if (arg === '--headed') options.headed = true; else if (arg === '--allow-missing-images') options.allowMissingImages = true; else if (arg === '--keep-remote-links') options.keepRemoteLinks = true; else if (arg.startsWith('-')) throw new Error(`未知选项: ${arg}`); else if (!url) url = arg; else throw new Error(`多余参数: ${arg}`); } if (!url) usage(1); if (!['all', 'html', 'pdf', 'markdown'].includes(options.format)) throw new Error('--format 必须是 all、html、pdf 或 markdown'); if (!['continuous', 'a4'].includes(options.pdfMode)) throw new Error('--pdf-mode 必须是 continuous 或 a4'); for (const key of ['timeout', 'imageTimeout', 'manualWait']) { if (!Number.isFinite(options[key]) || options[key] <= 0) throw new Error(`${key} 必须是正数`); } options.outputDir = resolve(options.outputDir); options.profile = resolve(options.profile); return { url, options }; } function safeFilename(value) { const cleaned = String(value || 'wechat-article') .normalize('NFKC') .replace(/[\\/:*?"<>|\u0000-\u001f]/g, '-') .replace(/\s+/g, ' ') .replace(/[. ]+$/g, '') .trim(); return (cleaned || 'wechat-article').slice(0, 100); } function htmlEscape(value) { return String(value ?? '') .replaceAll('&', '&') .replaceAll('<', '<') .replaceAll('>', '>') .replaceAll('"', '"') .replaceAll("'", '''); } function inferMime(buffer, declared = '') { const type = declared.split(';')[0].trim().toLowerCase(); if (type.startsWith('image/')) return type; if (buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return 'image/png'; if (buffer.subarray(0, 3).equals(Buffer.from([0xff, 0xd8, 0xff]))) return 'image/jpeg'; if (buffer.subarray(0, 6).toString('ascii') === 'GIF87a' || buffer.subarray(0, 6).toString('ascii') === 'GIF89a') return 'image/gif'; if (buffer.subarray(0, 4).toString('ascii') === 'RIFF' && buffer.subarray(8, 12).toString('ascii') === 'WEBP') return 'image/webp'; const prefix = buffer.subarray(0, 300).toString('utf8').trimStart().toLowerCase(); if (prefix.startsWith(' false)) { const size = await candidate.evaluate((element) => ({ text: (element.innerText || '').trim().length, images: element.querySelectorAll('img,svg image,canvas').length, })); if (size.text >= 80 || size.images > 0) return selector; } } return ''; } async function waitForArticle(page, options) { let selector = await chooseArticleSelector(page, options.selector); if (selector) return selector; const bodyText = await page.locator('body').innerText({ timeout: 5_000 }).catch(() => ''); const blocked = isBlockedPage(page.url(), bodyText); if (!options.headed) { const reason = blocked ? '微信要求安全验证' : '没有识别到正文'; throw new Error(`${reason}。请用 --headed 和相同 --profile 重试,完成验证后脚本会继续。`); } console.error(blocked ? `微信要求验证;请在浏览器中完成操作。最多等待 ${options.manualWait / 1000} 秒…` : `尚未识别到正文;请在浏览器中完成加载。最多等待 ${options.manualWait / 1000} 秒…`); const deadline = Date.now() + options.manualWait; while (Date.now() < deadline) { await page.waitForTimeout(1000); selector = await chooseArticleSelector(page, options.selector); if (selector) return selector; } throw new Error('等待人工验证/正文加载超时'); } async function activateLazyImages(page, selector, imageTimeout) { await page.locator(selector).evaluate((root) => { for (const image of root.querySelectorAll('img')) { const lazy = image.getAttribute('data-src') || image.getAttribute('data-original') || image.getAttribute('data-backsrc'); if (lazy && image.getAttribute('src') !== lazy) { image.setAttribute('src', lazy); } image.loading = 'eager'; image.decoding = 'sync'; } for (const source of root.querySelectorAll('source')) { const lazy = source.getAttribute('data-srcset'); if (lazy) source.setAttribute('srcset', lazy); } }); let previousHeight = 0; let stablePasses = 0; for (let pass = 0; pass < 3 && stablePasses < 2; pass += 1) { const height = await page.evaluate(() => document.documentElement.scrollHeight); for (let y = 0; y < height; y += 600) { await page.evaluate((position) => window.scrollTo(0, position), y); await page.waitForTimeout(80); } await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight)); await page.waitForTimeout(600); const newHeight = await page.evaluate(() => document.documentElement.scrollHeight); stablePasses = Math.abs(newHeight - previousHeight) < 10 ? stablePasses + 1 : 0; previousHeight = newHeight; } const deadline = Date.now() + imageTimeout; while (Date.now() < deadline) { const status = await page.locator(selector).evaluate((root) => { const images = [...root.querySelectorAll('img')].filter((image) => { const src = image.currentSrc || image.src || image.dataset.src; return src && !src.startsWith('data:'); }); return { total: images.length, pending: images.filter((image) => !image.complete).length, }; }); if (status.pending === 0) break; await page.waitForTimeout(250); } await page.evaluate(() => window.scrollTo(0, 0)); } async function snapshotArticle(page, selector, keepRemoteLinks) { return page.locator(selector).evaluate((root, keepLinks) => { const absolute = (value) => { if (!value) return ''; if (/^(data|blob):/i.test(value)) return value; try { return new URL(value, document.baseURI).href; } catch { return value; } }; const clone = root.cloneNode(true); const originals = [root, ...root.querySelectorAll('*')]; const clones = [clone, ...clone.querySelectorAll('*')]; const resources = []; let resourceId = 0; const register = (element, kind, url) => { const resolved = absolute(url); if (!resolved) return ''; const id = `wxasset-${resourceId++}`; resources.push({ id, kind, url: resolved }); element.setAttribute(`data-wxasset-${kind}`, id); return id; }; for (let i = 0; i < originals.length; i += 1) { const original = originals[i]; const copy = clones[i]; if (!copy) continue; if (original instanceof HTMLCanvasElement) { try { const image = document.createElement('img'); image.src = original.toDataURL('image/png'); image.alt = original.getAttribute('aria-label') || 'canvas'; register(image, 'img', image.src); image.removeAttribute('src'); copy.replaceWith(image); } catch { register(copy, 'canvas', `blob:tainted-canvas-${resourceId}`); } continue; } if (original instanceof HTMLImageElement) { const url = original.currentSrc || original.src || original.dataset.src || original.getAttribute('data-original') || original.getAttribute('data-backsrc'); register(copy, 'img', url); copy.setAttribute('alt', original.alt || ''); copy.removeAttribute('src'); copy.removeAttribute('srcset'); copy.removeAttribute('data-src'); copy.removeAttribute('data-original'); copy.removeAttribute('data-backsrc'); copy.removeAttribute('data-srcset'); copy.removeAttribute('loading'); } else if (original instanceof SVGImageElement) { register(copy, 'svg-image', original.href?.baseVal || original.getAttribute('href')); copy.removeAttribute('href'); copy.removeAttribute('xlink:href'); } const background = getComputedStyle(original).backgroundImage; if (background && background !== 'none') { const match = background.match(/^url\(["']?(.*?)["']?\)$/); if (match) { const id = register(copy, 'background', match[1]); if (id) copy.style.backgroundImage = `url("__${id}__")`; } } } for (const bad of clone.querySelectorAll('script,noscript,link,meta,form,button,source')) bad.remove(); for (const frame of clone.querySelectorAll('iframe,video')) { const source = frame.getAttribute('data-src') || frame.getAttribute('src') || ''; const note = document.createElement('p'); note.className = 'wx-export-media-note'; note.textContent = source ? `原文中的媒体内容:${absolute(source)}` : '原文中的媒体内容(请回到原文查看)'; frame.replaceWith(note); } for (const anchor of clone.querySelectorAll('a')) { const href = absolute(anchor.getAttribute('href')); if (keepLinks && /^https?:/i.test(href)) { anchor.setAttribute('href', href); anchor.setAttribute('rel', 'noopener noreferrer'); } else { anchor.removeAttribute('href'); anchor.removeAttribute('target'); } } clone.removeAttribute('style'); clone.classList.add('wx-export-content'); const text = (selectors) => { for (const candidate of selectors) { const element = document.querySelector(candidate); const value = element?.textContent?.trim(); if (value) return value; } return ''; }; const title = text(['#activity-name', '.rich_media_title', 'h1']) || document.title || '微信文章'; const author = text(['#js_name', '.rich_media_meta_nickname', '[property="og:article:author"]']); const published = text(['#publish_time', '.rich_media_meta_text']) || ''; const canonical = document.querySelector('link[rel="canonical"]')?.href || location.href; return { title, author, published, canonical, html: clone.outerHTML, resources }; }, keepRemoteLinks); } function dataUriToAsset(url) { const match = url.match(/^data:([^;,]+)?(;base64)?,(.*)$/s); if (!match) throw new Error('无效的 data URI'); const mime = match[1] || 'text/plain'; const buffer = match[2] ? Buffer.from(match[3], 'base64') : Buffer.from(decodeURIComponent(match[3])); if (!mime.startsWith('image/')) throw new Error(`data URI 不是图片: ${mime}`); return { mime, buffer, source: 'inline' }; } async function fetchAssets(context, resources, responseCache, referer) { const uniqueUrls = [...new Set(resources.map((item) => item.url))]; const results = new Map(); let cursor = 0; const fetchOne = async (url) => { if (url.startsWith('data:')) return dataUriToAsset(url); if (url.startsWith('blob:')) throw new Error('无法离线保存 blob 图片'); const cached = await responseCache.get(url)?.catch(() => null); if (cached?.buffer?.length) { const mime = inferMime(cached.buffer, cached.contentType); if (mime) return { mime, buffer: cached.buffer, source: 'browser-cache' }; } let lastError; for (let attempt = 1; attempt <= 3; attempt += 1) { try { const response = await context.request.get(url, { headers: { Referer: referer, Accept: 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8', }, timeout: 30_000, failOnStatusCode: false, }); if (!response.ok()) throw new Error(`HTTP ${response.status()}`); const buffer = await response.body(); const mime = inferMime(buffer, response.headers()['content-type'] || ''); if (!mime) throw new Error(`响应不是图片 (${response.headers()['content-type'] || 'unknown'})`); return { mime, buffer, source: `request-${attempt}` }; } catch (error) { lastError = error; await new Promise((done) => setTimeout(done, 350 * (2 ** (attempt - 1)))); } } throw lastError; }; async function worker() { while (cursor < uniqueUrls.length) { const url = uniqueUrls[cursor++]; try { results.set(url, { ok: true, ...(await fetchOne(url)) }); } catch (error) { results.set(url, { ok: false, error: cleanErrorMessage(error) }); } } } await Promise.all(Array.from({ length: Math.min(6, uniqueUrls.length || 1) }, worker)); return results; } function embedAssets(snapshot, assets) { let html = snapshot.html; const failures = []; const records = []; for (const resource of snapshot.resources) { const asset = assets.get(resource.url); const marker = `data-wxasset-${resource.kind}="${resource.id}"`; if (!asset?.ok) { failures.push({ ...resource, error: asset?.error || '未抓取' }); records.push({ ...resource, ok: false, error: asset?.error || '未抓取' }); if (resource.kind === 'img') html = html.replace(marker, `src="${htmlEscape(resource.url)}"`); else if (resource.kind === 'svg-image') html = html.replace(marker, `href="${htmlEscape(resource.url)}"`); else if (resource.kind === 'background') { html = html.replace(`__${resource.id}__`, resource.url.replaceAll('"', '%22')); html = html.replace(marker, ''); } continue; } const dataUri = `data:${asset.mime};base64,${asset.buffer.toString('base64')}`; const bytes = asset.buffer.length; records.push({ ...resource, ok: true, mime: asset.mime, bytes, source: asset.source }); if (resource.kind === 'img') { html = html.replace(marker, `src="${dataUri}"`); } else if (resource.kind === 'svg-image') { html = html.replace(marker, `href="${dataUri}"`); } else if (resource.kind === 'background') { html = html.replace(`__${resource.id}__`, dataUri); html = html.replace(marker, ''); } } html = html.replaceAll(/\sdata-wxasset-[\w-]+="[^"]+"/g, ''); return { html, failures, records }; } function standaloneHtml(snapshot, articleHtml) { const metadata = [snapshot.author, snapshot.published].filter(Boolean).map(htmlEscape).join(' '); return ` ${htmlEscape(snapshot.title)}

${htmlEscape(snapshot.title)}

${metadata ? `
${metadata}
` : ''}
${articleHtml}
`; } function mimeExtension(mime) { const extensions = { 'image/jpeg': 'jpg', 'image/png': 'png', 'image/gif': 'gif', 'image/webp': 'webp', 'image/svg+xml': 'svg', 'image/avif': 'avif', }; return extensions[mime] || 'bin'; } function yamlString(value) { return JSON.stringify(String(value || '')); } async function writeMarkdown(snapshot, assets, outputDir, base) { const assetDirectoryName = `${base}.assets`; const assetDirectory = join(outputDir, assetDirectoryName); await mkdir(assetDirectory, { recursive: true }); const filenames = new Map(); let assetNumber = 0; for (const resource of snapshot.resources) { const asset = assets.get(resource.url); if (!asset?.ok || filenames.has(resource.url)) continue; assetNumber += 1; const filename = `image-${String(assetNumber).padStart(3, '0')}.${mimeExtension(asset.mime)}`; await writeFile(join(assetDirectory, filename), asset.buffer); filenames.set(resource.url, `${assetDirectoryName}/${filename}`); } let sourceHtml = snapshot.html; for (const resource of snapshot.resources) { const marker = `data-wxasset-${resource.kind}="${resource.id}"`; const localPath = filenames.get(resource.url); if (resource.kind === 'img') { sourceHtml = sourceHtml.replace(marker, localPath ? `src="${htmlEscape(localPath)}"` : ''); } else if (resource.kind === 'svg-image') { sourceHtml = sourceHtml.replace(marker, localPath ? `href="${htmlEscape(localPath)}"` : ''); } else if (resource.kind === 'background') { sourceHtml = sourceHtml.replace(`__${resource.id}__`, localPath || ''); sourceHtml = sourceHtml.replace(marker, ''); } } sourceHtml = sourceHtml.replaceAll(/\sdata-wxasset-[\w-]+="[^"]+"/g, ''); const turndown = new TurndownService({ headingStyle: 'atx', bulletListMarker: '-', codeBlockStyle: 'fenced', emDelimiter: '*', strongDelimiter: '**', }); turndown.use(gfm); turndown.addRule('wechatMediaNote', { filter: (node) => node.nodeType === 1 && node.classList?.contains('wx-export-media-note'), replacement: (content) => `\n\n> ${content.trim()}\n\n`, }); const body = turndown.turndown(sourceHtml).replace(/\n{3,}/g, '\n\n').trim(); const markdown = `--- title: ${yamlString(snapshot.title)} author: ${yamlString(snapshot.author)} published: ${yamlString(snapshot.published)} source: ${yamlString(snapshot.canonical)} archived_at: ${yamlString(new Date().toISOString())} --- # ${snapshot.title} ${body} `; const markdownPath = join(outputDir, `${base}.md`); await writeFile(markdownPath, markdown); return markdownPath; } async function writePdf(browserContext, htmlPath, pdfPath, mode) { const page = await browserContext.newPage(); await page.goto(pathToFileURL(htmlPath).href, { waitUntil: 'load' }); await page.emulateMedia({ media: 'print' }); await page.evaluate(() => document.fonts?.ready); if (mode === 'a4') { await page.pdf({ path: pdfPath, format: 'A4', printBackground: true, margin: { top: 0, right: 0, bottom: 0, left: 0 } }); } else { const heightPx = await page.evaluate(() => Math.max( document.body.scrollHeight, document.documentElement.scrollHeight, document.querySelector('.wx-export-page')?.scrollHeight || 0, )); const heightMm = Math.ceil((heightPx * 25.4) / 96) + 2; if (heightMm > 5000) { throw new Error(`正文约 ${heightMm}mm,超过 Chromium 单张 PDF 的安全高度 5000mm;请使用 --pdf-mode a4,单页 HTML 已保留`); } await page.pdf({ path: pdfPath, width: '210mm', height: `${heightMm}mm`, printBackground: true, margin: { top: 0, right: 0, bottom: 0, left: 0 }, preferCSSPageSize: false, }); } await page.close(); } async function main() { const { url, options } = parseArgs(process.argv.slice(2)); await mkdir(options.outputDir, { recursive: true }); await mkdir(options.profile, { recursive: true }); const responseCache = new Map(); const context = await chromium.launchPersistentContext(options.profile, { headless: !options.headed, viewport: { width: 1280, height: 900 }, locale: 'zh-CN', timezoneId: 'Asia/Shanghai', serviceWorkers: 'block', userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', }); let exitCode = 0; try { const page = context.pages()[0] || await context.newPage(); page.setDefaultTimeout(options.timeout); page.on('response', (response) => { if (response.request().resourceType() !== 'image' || !response.ok()) return; const promise = response.body() .then((buffer) => ({ buffer, contentType: response.headers()['content-type'] || '' })) .catch(() => null); responseCache.set(response.url(), promise); }); console.error(`打开:${url}`); await page.goto(url, { waitUntil: 'domcontentloaded', timeout: options.timeout }); const selector = await waitForArticle(page, options); console.error(`正文:${selector};触发懒加载图片…`); await activateLazyImages(page, selector, options.imageTimeout); const snapshot = await snapshotArticle(page, selector, options.keepRemoteLinks); console.error(`发现 ${snapshot.resources.length} 个图片引用,正在抓取并内嵌…`); const assets = await fetchAssets(context, snapshot.resources, responseCache, page.url()); const embedded = embedAssets(snapshot, assets); const base = safeFilename(snapshot.title); const manifestPath = join(options.outputDir, `${base}.assets.json`); const manifest = { sourceUrl: url, finalUrl: page.url(), title: snapshot.title, articleSelector: selector, generatedAt: new Date().toISOString(), discoveredReferences: snapshot.resources.length, uniqueResources: assets.size, embeddedReferences: embedded.records.filter((item) => item.ok).length, failedReferences: embedded.failures.length, complete: embedded.failures.length === 0, resources: embedded.records, }; await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); if (embedded.failures.length && !options.allowMissingImages) { const summary = embedded.failures.slice(0, 5).map((item) => `- ${item.url}: ${item.error}`).join('\n'); throw new Error(`有 ${embedded.failures.length} 个图片引用抓取失败;已写完整性报告,未生成不完整成品。\n${summary}\n报告:${manifestPath}`); } if (options.format === 'markdown' || options.format === 'all') { const markdownPath = await writeMarkdown(snapshot, assets, options.outputDir, base); console.error(`Markdown:${markdownPath}`); } const htmlPath = join(options.outputDir, `${base}.html`); const html = standaloneHtml(snapshot, embedded.html); await writeFile(htmlPath, html); console.error(`HTML:${htmlPath}`); if (options.format === 'pdf' || options.format === 'all') { const pdfPath = join(options.outputDir, `${base}.${options.pdfMode === 'continuous' ? 'continuous' : 'a4'}.pdf`); await writePdf(context, htmlPath, pdfPath, options.pdfMode); console.error(`PDF:${pdfPath}`); } if (options.format === 'pdf') { // PDF 以离线 HTML 为中间源;保留它有助于审计图片完整性和重新排版。 console.error('提示:保留了生成 PDF 所用的自包含 HTML 中间文件。'); } console.error(`完整性:${manifest.complete ? '通过' : '允许缺图'}(${manifest.embeddedReferences}/${manifest.discoveredReferences} 个引用已内嵌)`); } catch (error) { exitCode = 2; console.error(`失败:${cleanErrorMessage(error)}`); } finally { await context.close(); } process.exitCode = exitCode; } if (pathToFileURL(process.argv[1]).href === import.meta.url) { main().catch((error) => { console.error(`失败:${cleanErrorMessage(error.stack || error)}`); process.exitCode = 2; }); } export { htmlEscape, inferMime, safeFilename };