deploy WeChat fulltext service to tools-pve
This commit is contained in:
Codex
@@ -0,0 +1,27 @@
|
||||
FROM mcr.microsoft.com/playwright:v1.58.2-noble
|
||||
|
||||
USER root
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
fonts-noto-cjk novnc openbox websockify x11vnc xvfb \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
COPY app/package.json app/package-lock.json ./
|
||||
RUN npm ci --omit=dev
|
||||
COPY --chown=pwuser:pwuser app/ ./
|
||||
COPY --chown=pwuser:pwuser entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh \
|
||||
&& chmod -R a+rX /app \
|
||||
&& mkdir -p /data \
|
||||
&& chown -R pwuser:pwuser /data /app
|
||||
|
||||
USER pwuser
|
||||
ENV DISPLAY=:99 \
|
||||
HOME=/home/pwuser \
|
||||
PORT=8091 \
|
||||
DATA_DIR=/data
|
||||
|
||||
EXPOSE 8091 6080
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# WeChat Fulltext
|
||||
|
||||
Internal LAN-only service on `tools-pve` that accepts a WeChat public-account
|
||||
article URL and returns archived Markdown, HTML, PDF, and an image manifest.
|
||||
|
||||
- API: `http://192.168.50.49:8091`
|
||||
- Manual browser verification: `http://192.168.50.49:8092/vnc.html`
|
||||
- WeChat upstream login: `http://192.168.50.49:8093/login.html`
|
||||
- State: `/opt/wechat-fulltext/data/`
|
||||
|
||||
The upstream image is pinned to the immutable amd64 digest for v1.7.0. The
|
||||
locally built exporter image is versioned and uses `pull_policy: never`.
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
{
|
||||
"name": "wechat-article-exporter",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "wechat-article-exporter",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"playwright": "1.58.2",
|
||||
"turndown": "7.2.0",
|
||||
"turndown-plugin-gfm": "1.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/@mixmark-io/domino": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz",
|
||||
"integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==",
|
||||
"license": "BSD-2-Clause"
|
||||
},
|
||||
"node_modules/fsevents": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
|
||||
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright": {
|
||||
"version": "1.58.2",
|
||||
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz",
|
||||
"integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright-core": "1.58.2"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "2.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright-core": {
|
||||
"version": "1.58.2",
|
||||
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz",
|
||||
"integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==",
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"playwright-core": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/turndown": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.0.tgz",
|
||||
"integrity": "sha512-eCZGBN4nNNqM9Owkv9HAtWRYfLA4h909E/WGAWWBpmB275ehNhZyk87/Tpvjbp0jjNl9XwCsbe6bm6CqFsgD+A==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@mixmark-io/domino": "^2.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/turndown-plugin-gfm": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz",
|
||||
"integrity": "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==",
|
||||
"license": "MIT"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "wechat-article-exporter",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"description": "Export a WeChat article to clean Markdown, self-contained HTML, and PDF.",
|
||||
"scripts": {
|
||||
"export": "node wechat-export.mjs",
|
||||
"test": "node test/smoke-test.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"playwright": "1.58.2",
|
||||
"turndown": "7.2.0",
|
||||
"turndown-plugin-gfm": "1.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,453 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises';
|
||||
import { createServer } from 'node:http';
|
||||
import { basename, dirname, join, normalize, resolve, sep } from 'node:path';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const exporter = join(here, 'wechat-export.mjs');
|
||||
const port = Number(process.env.PORT || 8091);
|
||||
const dataDirectory = resolve(process.env.DATA_DIR || '/data');
|
||||
const articlesDirectory = join(dataDirectory, 'articles');
|
||||
const profileDirectory = join(dataDirectory, 'profile');
|
||||
const publicBaseUrl = (process.env.PUBLIC_BASE_URL || `http://127.0.0.1:${port}`).replace(/\/$/, '');
|
||||
const verificationUrl = process.env.VERIFICATION_URL || '';
|
||||
const defaultProvider = process.env.DEFAULT_PROVIDER || 'mptext';
|
||||
const mptextEndpoint = process.env.MPTEXT_ENDPOINT || 'https://down.mptext.top/api/public/v1/download';
|
||||
const directBrowserEnabled = process.env.DIRECT_BROWSER_ENABLED === 'true';
|
||||
const upstreamTimeout = Number(process.env.UPSTREAM_TIMEOUT_SECONDS || 60) * 1000;
|
||||
const mptextMaxAttempts = Math.max(1, Number(process.env.MPTEXT_MAX_ATTEMPTS || 2));
|
||||
const mptextRetryDelays = String(process.env.MPTEXT_RETRY_DELAYS_SECONDS || '5')
|
||||
.split(',')
|
||||
.map((value) => Math.max(0, Number(value.trim()) || 0));
|
||||
const secondaryUpstreamEndpoint = process.env.SECONDARY_UPSTREAM_ENDPOINT || '';
|
||||
const secondaryUpstreamPublicUrl = process.env.SECONDARY_UPSTREAM_PUBLIC_URL || '';
|
||||
const secondaryUpstreamTimeout = Number(process.env.SECONDARY_UPSTREAM_TIMEOUT_SECONDS || 120) * 1000;
|
||||
const jobs = new Map();
|
||||
let queue = Promise.resolve();
|
||||
|
||||
function sendJson(response, status, payload) {
|
||||
const body = `${JSON.stringify(payload, null, 2)}\n`;
|
||||
response.writeHead(status, {
|
||||
'content-type': 'application/json; charset=utf-8',
|
||||
'content-length': Buffer.byteLength(body),
|
||||
'cache-control': 'no-store',
|
||||
});
|
||||
response.end(body);
|
||||
}
|
||||
|
||||
function normalizeWechatUrl(value) {
|
||||
const url = new URL(String(value || ''));
|
||||
if (url.protocol !== 'https:' || url.hostname !== 'mp.weixin.qq.com') {
|
||||
throw new Error('Only https://mp.weixin.qq.com article URLs are accepted');
|
||||
}
|
||||
if (!url.pathname.startsWith('/s')) throw new Error('URL is not a WeChat article link');
|
||||
url.hash = '';
|
||||
return url.href;
|
||||
}
|
||||
|
||||
function articleId(url) {
|
||||
return createHash('sha256').update(url).digest('hex').slice(0, 20);
|
||||
}
|
||||
|
||||
function statusUrl(id) {
|
||||
return `${publicBaseUrl}/v1/articles/${id}`;
|
||||
}
|
||||
|
||||
async function readJson(path) {
|
||||
return JSON.parse(await readFile(path, 'utf8'));
|
||||
}
|
||||
|
||||
async function cachedResult(id) {
|
||||
try {
|
||||
const result = await readJson(join(articlesDirectory, id, 'result.json'));
|
||||
return result.status === 'completed' ? result : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function publicFileUrl(id, relativePath) {
|
||||
return `${publicBaseUrl}/v1/articles/${id}/files/${relativePath.split('/').map(encodeURIComponent).join('/')}`;
|
||||
}
|
||||
|
||||
async function collectResult(job) {
|
||||
const entries = await readdir(job.directory, { withFileTypes: true });
|
||||
const files = entries.filter((entry) => entry.isFile()).map((entry) => entry.name);
|
||||
const markdown = files.find((name) => name.endsWith('.md'));
|
||||
const pdf = files.find((name) => name.endsWith('.a4.pdf')) || files.find((name) => name.endsWith('.pdf'));
|
||||
const html = files.find((name) => name.endsWith('.html'));
|
||||
const manifest = files.find((name) => name.endsWith('.assets.json'));
|
||||
if (!markdown || !pdf || !html || !manifest) throw new Error('Exporter completed without all required artifacts');
|
||||
const assetManifest = await readJson(join(job.directory, manifest));
|
||||
assetManifest.sourceUrl = job.url;
|
||||
assetManifest.acquisitionProvider = job.acquisition_provider || job.provider;
|
||||
await writeFile(join(job.directory, manifest), `${JSON.stringify(assetManifest, null, 2)}\n`);
|
||||
return {
|
||||
id: job.id,
|
||||
status: 'completed',
|
||||
cached: false,
|
||||
source_url: job.url,
|
||||
acquisition_provider: job.acquisition_provider || job.provider,
|
||||
title: assetManifest.title,
|
||||
complete: assetManifest.complete,
|
||||
generated_at: new Date().toISOString(),
|
||||
markdown_url: publicFileUrl(job.id, markdown),
|
||||
pdf_url: publicFileUrl(job.id, pdf),
|
||||
html_url: publicFileUrl(job.id, html),
|
||||
manifest_url: publicFileUrl(job.id, manifest),
|
||||
status_url: statusUrl(job.id),
|
||||
};
|
||||
}
|
||||
|
||||
function htmlAttribute(value) {
|
||||
return String(value).replaceAll('&', '&').replaceAll('"', '"').replaceAll('<', '<');
|
||||
}
|
||||
|
||||
function htmlText(value) {
|
||||
return htmlAttribute(value).replaceAll("'", ''').replaceAll('>', '>');
|
||||
}
|
||||
|
||||
function sleep(milliseconds) {
|
||||
return new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds));
|
||||
}
|
||||
|
||||
function containsArticleBody(html) {
|
||||
return /<(?:div|section)[^>]+id=["']js_content["']/i.test(html);
|
||||
}
|
||||
|
||||
async function saveUpstreamAttempt(job, attempt, response, html, error = '') {
|
||||
const prefix = `_upstream-attempt-${String(attempt).padStart(2, '0')}`;
|
||||
await writeFile(join(job.directory, `${prefix}.htm`), html);
|
||||
await writeFile(join(job.directory, `${prefix}.json`), `${JSON.stringify({
|
||||
attempt,
|
||||
fetched_at: new Date().toISOString(),
|
||||
endpoint: new URL(mptextEndpoint).origin,
|
||||
http_status: response?.status || 0,
|
||||
content_type: response?.headers?.get('content-type') || '',
|
||||
bytes: Buffer.byteLength(html),
|
||||
contains_article_body: containsArticleBody(html),
|
||||
error,
|
||||
}, null, 2)}\n`);
|
||||
}
|
||||
|
||||
async function fetchMptextHtml(job) {
|
||||
let lastStatus = 0;
|
||||
let lastBytes = 0;
|
||||
let lastError = '';
|
||||
for (let attempt = 1; attempt <= mptextMaxAttempts; attempt += 1) {
|
||||
const endpoint = new URL(mptextEndpoint);
|
||||
endpoint.searchParams.set('url', job.url);
|
||||
endpoint.searchParams.set('format', 'html');
|
||||
let response;
|
||||
let html = '';
|
||||
try {
|
||||
response = await fetch(endpoint, {
|
||||
headers: { Accept: 'text/html,application/xhtml+xml' },
|
||||
signal: AbortSignal.timeout(upstreamTimeout),
|
||||
});
|
||||
html = await response.text();
|
||||
lastStatus = response.status;
|
||||
lastBytes = Buffer.byteLength(html);
|
||||
lastError = response.ok && containsArticleBody(html)
|
||||
? ''
|
||||
: response.ok
|
||||
? 'response does not contain #js_content'
|
||||
: `HTTP ${response.status}`;
|
||||
await saveUpstreamAttempt(job, attempt, response, html, lastError);
|
||||
job.upstream_attempts = attempt;
|
||||
if (response.ok && containsArticleBody(html)) {
|
||||
const metadata = `<base href="${htmlAttribute(job.url)}"><link rel="canonical" href="${htmlAttribute(job.url)}">`;
|
||||
const archivedHtml = /<head(?:\s[^>]*)?>/i.test(html)
|
||||
? html.replace(/<head(?:\s[^>]*)?>/i, (head) => `${head}\n${metadata}`)
|
||||
: `${metadata}\n${html}`;
|
||||
const sourcePath = join(job.directory, '_upstream-source.htm');
|
||||
await writeFile(sourcePath, archivedHtml);
|
||||
job.acquisition_provider = 'mptext';
|
||||
return pathToFileURL(sourcePath).href;
|
||||
}
|
||||
} catch (error) {
|
||||
lastError = String(error?.message || error);
|
||||
await saveUpstreamAttempt(job, attempt, response, html, lastError);
|
||||
job.upstream_attempts = attempt;
|
||||
}
|
||||
if (attempt < mptextMaxAttempts) {
|
||||
const delaySeconds = mptextRetryDelays[Math.min(attempt - 1, mptextRetryDelays.length - 1)] || 0;
|
||||
job.retrying_in_seconds = delaySeconds;
|
||||
await sleep(delaySeconds * 1000);
|
||||
delete job.retrying_in_seconds;
|
||||
}
|
||||
}
|
||||
if (secondaryUpstreamEndpoint) {
|
||||
try {
|
||||
return await fetchSecondaryUpstreamHtml(job);
|
||||
} catch (error) {
|
||||
const secondaryError = String(error?.message || error);
|
||||
job.secondary_upstream_error = secondaryError;
|
||||
if (/未登录|登录已失效|login|credential|auth/i.test(secondaryError)) {
|
||||
job.secondary_login_url = secondaryUpstreamPublicUrl || undefined;
|
||||
}
|
||||
throw new Error(
|
||||
`mptext did not return an article after ${mptextMaxAttempts} attempts; `
|
||||
+ `secondary upstream failed: ${secondaryError}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`mptext did not return an article after ${mptextMaxAttempts} attempts `
|
||||
+ `(last HTTP ${lastStatus || 'network error'}, ${lastBytes} bytes): ${lastError}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchSecondaryUpstreamHtml(job) {
|
||||
let response;
|
||||
let raw = '';
|
||||
try {
|
||||
response = await fetch(secondaryUpstreamEndpoint, {
|
||||
method: 'POST',
|
||||
headers: { Accept: 'application/json', 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ url: job.url }),
|
||||
signal: AbortSignal.timeout(secondaryUpstreamTimeout),
|
||||
});
|
||||
raw = await response.text();
|
||||
} catch (error) {
|
||||
await writeFile(join(job.directory, '_secondary-upstream.json'), `${JSON.stringify({
|
||||
fetched_at: new Date().toISOString(),
|
||||
endpoint: secondaryUpstreamEndpoint,
|
||||
http_status: response?.status || 0,
|
||||
error: String(error?.message || error),
|
||||
}, null, 2)}\n`);
|
||||
throw error;
|
||||
}
|
||||
|
||||
let payload;
|
||||
try {
|
||||
payload = JSON.parse(raw);
|
||||
} catch {
|
||||
throw new Error(`HTTP ${response.status}: secondary upstream returned invalid JSON`);
|
||||
} finally {
|
||||
await writeFile(join(job.directory, '_secondary-upstream-response.json'), raw);
|
||||
}
|
||||
|
||||
if (!response.ok || !payload?.success) {
|
||||
throw new Error(payload?.error || `HTTP ${response.status}`);
|
||||
}
|
||||
const data = payload.data || {};
|
||||
if (!data.content || typeof data.content !== 'string') {
|
||||
throw new Error('secondary upstream response does not contain article HTML');
|
||||
}
|
||||
|
||||
const title = data.title || '微信文章';
|
||||
const author = data.author || '';
|
||||
const published = data.publish_time_str || '';
|
||||
const archivedHtml = `<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<base href="${htmlAttribute(job.url)}">
|
||||
<link rel="canonical" href="${htmlAttribute(job.url)}">
|
||||
<meta property="og:title" content="${htmlAttribute(title)}">
|
||||
<title>${htmlText(title)}</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1 id="activity-name">${htmlText(title)}</h1>
|
||||
<div id="js_name">${htmlText(author)}</div>
|
||||
<time id="publish_time">${htmlText(published)}</time>
|
||||
<div id="js_content" class="rich_media_content">${data.content}</div>
|
||||
</body>
|
||||
</html>`;
|
||||
const sourcePath = join(job.directory, '_upstream-source.htm');
|
||||
await writeFile(sourcePath, archivedHtml);
|
||||
job.acquisition_provider = 'wechat-download-api';
|
||||
return pathToFileURL(sourcePath).href;
|
||||
}
|
||||
|
||||
async function runJob(job) {
|
||||
job.status = 'running';
|
||||
job.started_at = new Date().toISOString();
|
||||
await rm(job.directory, { recursive: true, force: true });
|
||||
await mkdir(job.directory, { recursive: true });
|
||||
await mkdir(profileDirectory, { recursive: true });
|
||||
|
||||
let inputUrl = job.url;
|
||||
if (job.provider === 'mptext') inputUrl = await fetchMptextHtml(job);
|
||||
else if (job.provider !== 'browser') throw new Error(`Unsupported acquisition provider: ${job.provider}`);
|
||||
|
||||
const args = [
|
||||
exporter,
|
||||
inputUrl,
|
||||
'--output-dir', job.directory,
|
||||
'--profile', profileDirectory,
|
||||
'--format', 'all',
|
||||
'--pdf-mode', 'a4',
|
||||
];
|
||||
if (job.provider === 'browser') {
|
||||
args.push('--headed', '--manual-wait', String(Number(process.env.MANUAL_WAIT_SECONDS || 600)));
|
||||
} else {
|
||||
args.push('--selector', '#js_content');
|
||||
}
|
||||
const child = spawn(process.execPath, args, { cwd: here, env: process.env });
|
||||
job.pid = child.pid;
|
||||
let log = '';
|
||||
const capture = (chunk) => {
|
||||
const text = chunk.toString();
|
||||
log = `${log}${text}`.slice(-64 * 1024);
|
||||
if (/微信要求(?:安全)?验证|完成验证|人工验证/.test(text)) {
|
||||
job.status = 'verification_required';
|
||||
job.verification_url = verificationUrl || undefined;
|
||||
}
|
||||
};
|
||||
child.stdout.on('data', capture);
|
||||
child.stderr.on('data', capture);
|
||||
const exitCode = await new Promise((done, reject) => {
|
||||
child.once('error', reject);
|
||||
child.once('close', done);
|
||||
});
|
||||
if (exitCode !== 0) throw new Error(log.trim().split('\n').slice(-8).join('\n') || `Exporter exited ${exitCode}`);
|
||||
const result = await collectResult(job);
|
||||
await writeFile(join(job.directory, 'result.json'), `${JSON.stringify(result, null, 2)}\n`);
|
||||
Object.assign(job, result);
|
||||
}
|
||||
|
||||
function enqueue(job) {
|
||||
jobs.set(job.id, job);
|
||||
queue = queue.then(async () => {
|
||||
try {
|
||||
await runJob(job);
|
||||
} catch (error) {
|
||||
job.status = 'failed';
|
||||
job.error = String(error?.message || error);
|
||||
job.finished_at = new Date().toISOString();
|
||||
try {
|
||||
await writeFile(join(job.directory, 'failure.json'), `${JSON.stringify({
|
||||
id: job.id,
|
||||
status: job.status,
|
||||
source_url: job.url,
|
||||
provider: job.provider,
|
||||
created_at: job.created_at,
|
||||
started_at: job.started_at,
|
||||
finished_at: job.finished_at,
|
||||
upstream_attempts: job.upstream_attempts || 0,
|
||||
secondary_upstream_error: job.secondary_upstream_error,
|
||||
secondary_login_url: job.secondary_login_url,
|
||||
error: job.error,
|
||||
}, null, 2)}\n`);
|
||||
} catch {
|
||||
// The in-memory status remains available even if diagnostic persistence fails.
|
||||
}
|
||||
console.error(`[wechat-article-service] job ${job.id} failed: ${job.error}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function bodyJson(request) {
|
||||
const chunks = [];
|
||||
let bytes = 0;
|
||||
for await (const chunk of request) {
|
||||
bytes += chunk.length;
|
||||
if (bytes > 16 * 1024) throw new Error('Request body too large');
|
||||
chunks.push(chunk);
|
||||
}
|
||||
return JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}');
|
||||
}
|
||||
|
||||
function contentType(path) {
|
||||
if (path.endsWith('.md')) return 'text/markdown; charset=utf-8';
|
||||
if (path.endsWith('.pdf')) return 'application/pdf';
|
||||
if (path.endsWith('.html')) return 'text/html; charset=utf-8';
|
||||
if (path.endsWith('.json')) return 'application/json; charset=utf-8';
|
||||
if (path.endsWith('.png')) return 'image/png';
|
||||
if (path.endsWith('.jpg') || path.endsWith('.jpeg')) return 'image/jpeg';
|
||||
if (path.endsWith('.gif')) return 'image/gif';
|
||||
if (path.endsWith('.webp')) return 'image/webp';
|
||||
if (path.endsWith('.svg')) return 'image/svg+xml';
|
||||
if (path.endsWith('.avif')) return 'image/avif';
|
||||
return 'application/octet-stream';
|
||||
}
|
||||
|
||||
async function serveFile(response, id, requestedPath) {
|
||||
const root = resolve(articlesDirectory, id);
|
||||
const target = resolve(root, normalize(requestedPath));
|
||||
if (!target.startsWith(`${root}${sep}`)) return sendJson(response, 400, { error: 'Invalid path' });
|
||||
const info = await stat(target);
|
||||
if (!info.isFile()) return sendJson(response, 404, { error: 'Not found' });
|
||||
response.writeHead(200, {
|
||||
'content-type': contentType(target),
|
||||
'content-length': info.size,
|
||||
'cache-control': 'private, max-age=86400',
|
||||
});
|
||||
createReadStream(target).pipe(response);
|
||||
}
|
||||
|
||||
await Promise.all([mkdir(articlesDirectory, { recursive: true }), mkdir(profileDirectory, { recursive: true })]);
|
||||
|
||||
const server = createServer(async (request, response) => {
|
||||
try {
|
||||
const requestUrl = new URL(request.url, `http://${request.headers.host || 'localhost'}`);
|
||||
if (request.method === 'GET' && requestUrl.pathname === '/health') {
|
||||
return sendJson(response, 200, { status: 'ok', queued_or_running: [...jobs.values()].filter((job) => !['completed', 'failed'].includes(job.status)).length });
|
||||
}
|
||||
if (request.method === 'GET' && requestUrl.pathname === '/') {
|
||||
return sendJson(response, 200, {
|
||||
service: 'wechat-article-service',
|
||||
default_provider: defaultProvider,
|
||||
direct_browser_enabled: directBrowserEnabled,
|
||||
mptext_max_attempts: mptextMaxAttempts,
|
||||
mptext_retry_delays_seconds: mptextRetryDelays,
|
||||
secondary_upstream_enabled: Boolean(secondaryUpstreamEndpoint),
|
||||
secondary_upstream_login_url: secondaryUpstreamPublicUrl || undefined,
|
||||
create: 'POST /v1/articles with JSON {"url":"https://mp.weixin.qq.com/s/...","force":false,"provider":"mptext"}',
|
||||
status: 'GET /v1/articles/{id}',
|
||||
});
|
||||
}
|
||||
if (request.method === 'POST' && requestUrl.pathname === '/v1/articles') {
|
||||
const payload = await bodyJson(request);
|
||||
const url = normalizeWechatUrl(payload.url);
|
||||
const provider = String(payload.provider || defaultProvider);
|
||||
if (!['mptext', 'browser'].includes(provider)) throw new Error('provider must be mptext or browser');
|
||||
if (provider === 'browser' && !directBrowserEnabled) {
|
||||
throw new Error('Direct browser acquisition is disabled to avoid triggering WeChat risk control');
|
||||
}
|
||||
const id = articleId(url);
|
||||
const existing = await cachedResult(id);
|
||||
if (existing && !payload.force) return sendJson(response, 200, { ...existing, cached: true });
|
||||
const active = jobs.get(id);
|
||||
if (active && !['completed', 'failed'].includes(active.status)) return sendJson(response, 202, active);
|
||||
const job = {
|
||||
id,
|
||||
status: 'queued',
|
||||
source_url: url,
|
||||
provider,
|
||||
url,
|
||||
directory: join(articlesDirectory, id),
|
||||
created_at: new Date().toISOString(),
|
||||
status_url: statusUrl(id),
|
||||
};
|
||||
enqueue(job);
|
||||
return sendJson(response, 202, job);
|
||||
}
|
||||
const statusMatch = requestUrl.pathname.match(/^\/v1\/articles\/([a-f0-9]{20})$/);
|
||||
if (request.method === 'GET' && statusMatch) {
|
||||
const id = statusMatch[1];
|
||||
const active = jobs.get(id);
|
||||
if (active) return sendJson(response, 200, active);
|
||||
const existing = await cachedResult(id);
|
||||
return existing ? sendJson(response, 200, existing) : sendJson(response, 404, { error: 'Article job not found' });
|
||||
}
|
||||
const fileMatch = requestUrl.pathname.match(/^\/v1\/articles\/([a-f0-9]{20})\/files\/(.+)$/);
|
||||
if (request.method === 'GET' && fileMatch) {
|
||||
return await serveFile(response, fileMatch[1], decodeURIComponent(fileMatch[2]));
|
||||
}
|
||||
return sendJson(response, 404, { error: 'Not found' });
|
||||
} catch (error) {
|
||||
return sendJson(response, 400, { error: String(error?.message || error) });
|
||||
}
|
||||
});
|
||||
|
||||
server.listen(port, '0.0.0.0', () => {
|
||||
console.log(`wechat-article-service listening on 0.0.0.0:${port}`);
|
||||
});
|
||||
@@ -0,0 +1,677 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { chromium } from 'playwright';
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { homedir } from 'node:os';
|
||||
import { basename, dirname, join, resolve } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import TurndownService from 'turndown';
|
||||
import turndownPluginGfm from 'turndown-plugin-gfm';
|
||||
|
||||
const { gfm } = turndownPluginGfm;
|
||||
|
||||
const DEFAULTS = {
|
||||
format: 'all',
|
||||
pdfMode: 'continuous',
|
||||
outputDir: process.cwd(),
|
||||
profile: join(homedir(), '.cache', 'wechat-article-exporter', 'profile'),
|
||||
selector: '',
|
||||
timeout: 90_000,
|
||||
imageTimeout: 45_000,
|
||||
manualWait: 300_000,
|
||||
headed: false,
|
||||
allowMissingImages: false,
|
||||
keepRemoteLinks: false,
|
||||
};
|
||||
|
||||
function usage(exitCode = 0) {
|
||||
const command = basename(process.argv[1] || 'wechat-export.mjs');
|
||||
console.log(`
|
||||
用法:
|
||||
node ${command} <微信文章 URL> [选项]
|
||||
|
||||
选项:
|
||||
-o, --output-dir <目录> 输出目录（默认: 当前目录）
|
||||
--format <all|html|pdf|markdown> 输出类型（默认: all）
|
||||
--pdf-mode <continuous|a4> 单张长页 PDF，或标准 A4 分页（默认: continuous）
|
||||
--selector <CSS> 手工指定正文元素
|
||||
--profile <目录> 持久浏览器资料目录（保存验证状态/Cookie）
|
||||
--headed 显示浏览器，可人工完成微信验证
|
||||
--manual-wait <秒> headed 模式等待人工验证的时间（默认: 300）
|
||||
--timeout <秒> 页面加载超时（默认: 90）
|
||||
--image-timeout <秒> 等待图片加载的总时间（默认: 45）
|
||||
--allow-missing-images 即使图片不完整也输出（默认会失败关闭）
|
||||
--keep-remote-links 不移除正文中的远程超链接
|
||||
-h, --help 显示帮助
|
||||
|
||||
示例:
|
||||
node ${command} 'https://mp.weixin.qq.com/s/...' -o ./out
|
||||
node ${command} 'https://mp.weixin.qq.com/s/...' --headed --pdf-mode a4
|
||||
`);
|
||||
process.exit(exitCode);
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = { ...DEFAULTS };
|
||||
let url = '';
|
||||
const valueFor = (arg, index) => {
|
||||
const value = argv[index + 1];
|
||||
if (!value || value.startsWith('--')) throw new Error(`${arg} 缺少参数`);
|
||||
return value;
|
||||
};
|
||||
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const arg = argv[i];
|
||||
if (arg === '-h' || arg === '--help') usage(0);
|
||||
else if (arg === '-o' || arg === '--output-dir') options.outputDir = valueFor(arg, i++);
|
||||
else if (arg === '--format') options.format = valueFor(arg, i++);
|
||||
else if (arg === '--pdf-mode') options.pdfMode = valueFor(arg, i++);
|
||||
else if (arg === '--selector') options.selector = valueFor(arg, i++);
|
||||
else if (arg === '--profile') options.profile = valueFor(arg, i++);
|
||||
else if (arg === '--timeout') options.timeout = Number(valueFor(arg, i++)) * 1000;
|
||||
else if (arg === '--image-timeout') options.imageTimeout = Number(valueFor(arg, i++)) * 1000;
|
||||
else if (arg === '--manual-wait') options.manualWait = Number(valueFor(arg, i++)) * 1000;
|
||||
else if (arg === '--headed') options.headed = true;
|
||||
else if (arg === '--allow-missing-images') options.allowMissingImages = true;
|
||||
else if (arg === '--keep-remote-links') options.keepRemoteLinks = true;
|
||||
else if (arg.startsWith('-')) throw new Error(`未知选项: ${arg}`);
|
||||
else if (!url) url = arg;
|
||||
else throw new Error(`多余参数: ${arg}`);
|
||||
}
|
||||
|
||||
if (!url) usage(1);
|
||||
if (!['all', 'html', 'pdf', 'markdown'].includes(options.format)) throw new Error('--format 必须是 all、html、pdf 或 markdown');
|
||||
if (!['continuous', 'a4'].includes(options.pdfMode)) throw new Error('--pdf-mode 必须是 continuous 或 a4');
|
||||
for (const key of ['timeout', 'imageTimeout', 'manualWait']) {
|
||||
if (!Number.isFinite(options[key]) || options[key] <= 0) throw new Error(`${key} 必须是正数`);
|
||||
}
|
||||
options.outputDir = resolve(options.outputDir);
|
||||
options.profile = resolve(options.profile);
|
||||
return { url, options };
|
||||
}
|
||||
|
||||
function safeFilename(value) {
|
||||
const cleaned = String(value || 'wechat-article')
|
||||
.normalize('NFKC')
|
||||
.replace(/[\\/:*?"<>|\u0000-\u001f]/g, '-')
|
||||
.replace(/\s+/g, ' ')
|
||||
.replace(/[. ]+$/g, '')
|
||||
.trim();
|
||||
return (cleaned || 'wechat-article').slice(0, 100);
|
||||
}
|
||||
|
||||
function htmlEscape(value) {
|
||||
return String(value ?? '')
|
||||
.replaceAll('&', '&')
|
||||
.replaceAll('<', '<')
|
||||
.replaceAll('>', '>')
|
||||
.replaceAll('"', '"')
|
||||
.replaceAll("'", ''');
|
||||
}
|
||||
|
||||
function inferMime(buffer, declared = '') {
|
||||
const type = declared.split(';')[0].trim().toLowerCase();
|
||||
if (type.startsWith('image/')) return type;
|
||||
if (buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return 'image/png';
|
||||
if (buffer.subarray(0, 3).equals(Buffer.from([0xff, 0xd8, 0xff]))) return 'image/jpeg';
|
||||
if (buffer.subarray(0, 6).toString('ascii') === 'GIF87a' || buffer.subarray(0, 6).toString('ascii') === 'GIF89a') return 'image/gif';
|
||||
if (buffer.subarray(0, 4).toString('ascii') === 'RIFF' && buffer.subarray(8, 12).toString('ascii') === 'WEBP') return 'image/webp';
|
||||
const prefix = buffer.subarray(0, 300).toString('utf8').trimStart().toLowerCase();
|
||||
if (prefix.startsWith('<svg') || (prefix.startsWith('<?xml') && prefix.includes('<svg'))) return 'image/svg+xml';
|
||||
return '';
|
||||
}
|
||||
|
||||
function cleanErrorMessage(error) {
|
||||
return String(error?.message || error || '未知错误').replace(/\u001b\[[0-9;]*m/g, '');
|
||||
}
|
||||
|
||||
function isBlockedPage(url, text) {
|
||||
return /wappoc_appmsgcaptcha|captcha|verify/i.test(url)
|
||||
|| /环境异常|完成验证|访问过于频繁|拖动下方滑块|安全验证/.test(text);
|
||||
}
|
||||
|
||||
async function chooseArticleSelector(page, explicitSelector) {
|
||||
const candidates = explicitSelector
|
||||
? [explicitSelector]
|
||||
: ['#js_content', '.rich_media_content', 'article', '[data-role="content"]', 'main'];
|
||||
for (const selector of candidates) {
|
||||
const candidate = page.locator(selector).first();
|
||||
if (await candidate.count() && await candidate.isVisible().catch(() => false)) {
|
||||
const size = await candidate.evaluate((element) => ({
|
||||
text: (element.innerText || '').trim().length,
|
||||
images: element.querySelectorAll('img,svg image,canvas').length,
|
||||
}));
|
||||
if (size.text >= 80 || size.images > 0) return selector;
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
async function waitForArticle(page, options) {
|
||||
let selector = await chooseArticleSelector(page, options.selector);
|
||||
if (selector) return selector;
|
||||
|
||||
const bodyText = await page.locator('body').innerText({ timeout: 5_000 }).catch(() => '');
|
||||
const blocked = isBlockedPage(page.url(), bodyText);
|
||||
if (!options.headed) {
|
||||
const reason = blocked ? '微信要求安全验证' : '没有识别到正文';
|
||||
throw new Error(`${reason}。请用 --headed 和相同 --profile 重试，完成验证后脚本会继续。`);
|
||||
}
|
||||
|
||||
console.error(blocked
|
||||
? `微信要求验证；请在浏览器中完成操作。最多等待 ${options.manualWait / 1000} 秒…`
|
||||
: `尚未识别到正文；请在浏览器中完成加载。最多等待 ${options.manualWait / 1000} 秒…`);
|
||||
const deadline = Date.now() + options.manualWait;
|
||||
while (Date.now() < deadline) {
|
||||
await page.waitForTimeout(1000);
|
||||
selector = await chooseArticleSelector(page, options.selector);
|
||||
if (selector) return selector;
|
||||
}
|
||||
throw new Error('等待人工验证/正文加载超时');
|
||||
}
|
||||
|
||||
async function activateLazyImages(page, selector, imageTimeout) {
|
||||
await page.locator(selector).evaluate((root) => {
|
||||
for (const image of root.querySelectorAll('img')) {
|
||||
const lazy = image.getAttribute('data-src')
|
||||
|| image.getAttribute('data-original')
|
||||
|| image.getAttribute('data-backsrc');
|
||||
if (lazy && image.getAttribute('src') !== lazy) {
|
||||
image.setAttribute('src', lazy);
|
||||
}
|
||||
image.loading = 'eager';
|
||||
image.decoding = 'sync';
|
||||
}
|
||||
for (const source of root.querySelectorAll('source')) {
|
||||
const lazy = source.getAttribute('data-srcset');
|
||||
if (lazy) source.setAttribute('srcset', lazy);
|
||||
}
|
||||
});
|
||||
|
||||
let previousHeight = 0;
|
||||
let stablePasses = 0;
|
||||
for (let pass = 0; pass < 3 && stablePasses < 2; pass += 1) {
|
||||
const height = await page.evaluate(() => document.documentElement.scrollHeight);
|
||||
for (let y = 0; y < height; y += 600) {
|
||||
await page.evaluate((position) => window.scrollTo(0, position), y);
|
||||
await page.waitForTimeout(80);
|
||||
}
|
||||
await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight));
|
||||
await page.waitForTimeout(600);
|
||||
const newHeight = await page.evaluate(() => document.documentElement.scrollHeight);
|
||||
stablePasses = Math.abs(newHeight - previousHeight) < 10 ? stablePasses + 1 : 0;
|
||||
previousHeight = newHeight;
|
||||
}
|
||||
|
||||
const deadline = Date.now() + imageTimeout;
|
||||
while (Date.now() < deadline) {
|
||||
const status = await page.locator(selector).evaluate((root) => {
|
||||
const images = [...root.querySelectorAll('img')].filter((image) => {
|
||||
const src = image.currentSrc || image.src || image.dataset.src;
|
||||
return src && !src.startsWith('data:');
|
||||
});
|
||||
return {
|
||||
total: images.length,
|
||||
pending: images.filter((image) => !image.complete).length,
|
||||
};
|
||||
});
|
||||
if (status.pending === 0) break;
|
||||
await page.waitForTimeout(250);
|
||||
}
|
||||
await page.evaluate(() => window.scrollTo(0, 0));
|
||||
}
|
||||
|
||||
async function snapshotArticle(page, selector, keepRemoteLinks) {
|
||||
return page.locator(selector).evaluate((root, keepLinks) => {
|
||||
const absolute = (value) => {
|
||||
if (!value) return '';
|
||||
if (/^(data|blob):/i.test(value)) return value;
|
||||
try { return new URL(value, document.baseURI).href; } catch { return value; }
|
||||
};
|
||||
const clone = root.cloneNode(true);
|
||||
const originals = [root, ...root.querySelectorAll('*')];
|
||||
const clones = [clone, ...clone.querySelectorAll('*')];
|
||||
const resources = [];
|
||||
let resourceId = 0;
|
||||
|
||||
const register = (element, kind, url) => {
|
||||
const resolved = absolute(url);
|
||||
if (!resolved) return '';
|
||||
const id = `wxasset-${resourceId++}`;
|
||||
resources.push({ id, kind, url: resolved });
|
||||
element.setAttribute(`data-wxasset-${kind}`, id);
|
||||
return id;
|
||||
};
|
||||
|
||||
for (let i = 0; i < originals.length; i += 1) {
|
||||
const original = originals[i];
|
||||
const copy = clones[i];
|
||||
if (!copy) continue;
|
||||
if (original instanceof HTMLCanvasElement) {
|
||||
try {
|
||||
const image = document.createElement('img');
|
||||
image.src = original.toDataURL('image/png');
|
||||
image.alt = original.getAttribute('aria-label') || 'canvas';
|
||||
register(image, 'img', image.src);
|
||||
image.removeAttribute('src');
|
||||
copy.replaceWith(image);
|
||||
} catch {
|
||||
register(copy, 'canvas', `blob:tainted-canvas-${resourceId}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (original instanceof HTMLImageElement) {
|
||||
const url = original.currentSrc || original.src || original.dataset.src
|
||||
|| original.getAttribute('data-original') || original.getAttribute('data-backsrc');
|
||||
register(copy, 'img', url);
|
||||
copy.setAttribute('alt', original.alt || '');
|
||||
copy.removeAttribute('src');
|
||||
copy.removeAttribute('srcset');
|
||||
copy.removeAttribute('data-src');
|
||||
copy.removeAttribute('data-original');
|
||||
copy.removeAttribute('data-backsrc');
|
||||
copy.removeAttribute('data-srcset');
|
||||
copy.removeAttribute('loading');
|
||||
} else if (original instanceof SVGImageElement) {
|
||||
register(copy, 'svg-image', original.href?.baseVal || original.getAttribute('href'));
|
||||
copy.removeAttribute('href');
|
||||
copy.removeAttribute('xlink:href');
|
||||
}
|
||||
|
||||
const background = getComputedStyle(original).backgroundImage;
|
||||
if (background && background !== 'none') {
|
||||
const match = background.match(/^url\(["']?(.*?)["']?\)$/);
|
||||
if (match) {
|
||||
const id = register(copy, 'background', match[1]);
|
||||
if (id) copy.style.backgroundImage = `url("__${id}__")`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const bad of clone.querySelectorAll('script,noscript,link,meta,form,button,source')) bad.remove();
|
||||
for (const frame of clone.querySelectorAll('iframe,video')) {
|
||||
const source = frame.getAttribute('data-src') || frame.getAttribute('src') || '';
|
||||
const note = document.createElement('p');
|
||||
note.className = 'wx-export-media-note';
|
||||
note.textContent = source ? `原文中的媒体内容：${absolute(source)}` : '原文中的媒体内容（请回到原文查看）';
|
||||
frame.replaceWith(note);
|
||||
}
|
||||
for (const anchor of clone.querySelectorAll('a')) {
|
||||
const href = absolute(anchor.getAttribute('href'));
|
||||
if (keepLinks && /^https?:/i.test(href)) {
|
||||
anchor.setAttribute('href', href);
|
||||
anchor.setAttribute('rel', 'noopener noreferrer');
|
||||
} else {
|
||||
anchor.removeAttribute('href');
|
||||
anchor.removeAttribute('target');
|
||||
}
|
||||
}
|
||||
clone.removeAttribute('style');
|
||||
clone.classList.add('wx-export-content');
|
||||
|
||||
const text = (selectors) => {
|
||||
for (const candidate of selectors) {
|
||||
const element = document.querySelector(candidate);
|
||||
const value = element?.textContent?.trim();
|
||||
if (value) return value;
|
||||
}
|
||||
return '';
|
||||
};
|
||||
const title = text(['#activity-name', '.rich_media_title', 'h1']) || document.title || '微信文章';
|
||||
const author = text(['#js_name', '.rich_media_meta_nickname', '[property="og:article:author"]']);
|
||||
const published = text(['#publish_time', '.rich_media_meta_text']) || '';
|
||||
const canonical = document.querySelector('link[rel="canonical"]')?.href || location.href;
|
||||
return { title, author, published, canonical, html: clone.outerHTML, resources };
|
||||
}, keepRemoteLinks);
|
||||
}
|
||||
|
||||
function dataUriToAsset(url) {
|
||||
const match = url.match(/^data:([^;,]+)?(;base64)?,(.*)$/s);
|
||||
if (!match) throw new Error('无效的 data URI');
|
||||
const mime = match[1] || 'text/plain';
|
||||
const buffer = match[2] ? Buffer.from(match[3], 'base64') : Buffer.from(decodeURIComponent(match[3]));
|
||||
if (!mime.startsWith('image/')) throw new Error(`data URI 不是图片: ${mime}`);
|
||||
return { mime, buffer, source: 'inline' };
|
||||
}
|
||||
|
||||
async function fetchAssets(context, resources, responseCache, referer) {
|
||||
const uniqueUrls = [...new Set(resources.map((item) => item.url))];
|
||||
const results = new Map();
|
||||
let cursor = 0;
|
||||
|
||||
const fetchOne = async (url) => {
|
||||
if (url.startsWith('data:')) return dataUriToAsset(url);
|
||||
if (url.startsWith('blob:')) throw new Error('无法离线保存 blob 图片');
|
||||
|
||||
const cached = await responseCache.get(url)?.catch(() => null);
|
||||
if (cached?.buffer?.length) {
|
||||
const mime = inferMime(cached.buffer, cached.contentType);
|
||||
if (mime) return { mime, buffer: cached.buffer, source: 'browser-cache' };
|
||||
}
|
||||
|
||||
let lastError;
|
||||
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
||||
try {
|
||||
const response = await context.request.get(url, {
|
||||
headers: {
|
||||
Referer: referer,
|
||||
Accept: 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
|
||||
},
|
||||
timeout: 30_000,
|
||||
failOnStatusCode: false,
|
||||
});
|
||||
if (!response.ok()) throw new Error(`HTTP ${response.status()}`);
|
||||
const buffer = await response.body();
|
||||
const mime = inferMime(buffer, response.headers()['content-type'] || '');
|
||||
if (!mime) throw new Error(`响应不是图片 (${response.headers()['content-type'] || 'unknown'})`);
|
||||
return { mime, buffer, source: `request-${attempt}` };
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
await new Promise((done) => setTimeout(done, 350 * (2 ** (attempt - 1))));
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
};
|
||||
|
||||
async function worker() {
|
||||
while (cursor < uniqueUrls.length) {
|
||||
const url = uniqueUrls[cursor++];
|
||||
try {
|
||||
results.set(url, { ok: true, ...(await fetchOne(url)) });
|
||||
} catch (error) {
|
||||
results.set(url, { ok: false, error: cleanErrorMessage(error) });
|
||||
}
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: Math.min(6, uniqueUrls.length || 1) }, worker));
|
||||
return results;
|
||||
}
|
||||
|
||||
function embedAssets(snapshot, assets) {
|
||||
let html = snapshot.html;
|
||||
const failures = [];
|
||||
const records = [];
|
||||
for (const resource of snapshot.resources) {
|
||||
const asset = assets.get(resource.url);
|
||||
const marker = `data-wxasset-${resource.kind}="${resource.id}"`;
|
||||
if (!asset?.ok) {
|
||||
failures.push({ ...resource, error: asset?.error || '未抓取' });
|
||||
records.push({ ...resource, ok: false, error: asset?.error || '未抓取' });
|
||||
if (resource.kind === 'img') html = html.replace(marker, `src="${htmlEscape(resource.url)}"`);
|
||||
else if (resource.kind === 'svg-image') html = html.replace(marker, `href="${htmlEscape(resource.url)}"`);
|
||||
else if (resource.kind === 'background') {
|
||||
html = html.replace(`__${resource.id}__`, resource.url.replaceAll('"', '%22'));
|
||||
html = html.replace(marker, '');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const dataUri = `data:${asset.mime};base64,${asset.buffer.toString('base64')}`;
|
||||
const bytes = asset.buffer.length;
|
||||
records.push({ ...resource, ok: true, mime: asset.mime, bytes, source: asset.source });
|
||||
if (resource.kind === 'img') {
|
||||
html = html.replace(marker, `src="${dataUri}"`);
|
||||
} else if (resource.kind === 'svg-image') {
|
||||
html = html.replace(marker, `href="${dataUri}"`);
|
||||
} else if (resource.kind === 'background') {
|
||||
html = html.replace(`__${resource.id}__`, dataUri);
|
||||
html = html.replace(marker, '');
|
||||
}
|
||||
}
|
||||
html = html.replaceAll(/\sdata-wxasset-[\w-]+="[^"]+"/g, '');
|
||||
return { html, failures, records };
|
||||
}
|
||||
|
||||
function standaloneHtml(snapshot, articleHtml) {
|
||||
const metadata = [snapshot.author, snapshot.published].filter(Boolean).map(htmlEscape).join(' ');
|
||||
return `<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta name="generator" content="wechat-article-exporter">
|
||||
<title>${htmlEscape(snapshot.title)}</title>
|
||||
<style>
|
||||
:root { color-scheme: light; }
|
||||
* { box-sizing: border-box; }
|
||||
html { background: #e9e9e9; }
|
||||
body { margin: 0; color: #222; font-family: -apple-system,BlinkMacSystemFont,"Segoe UI","PingFang SC","Hiragino Sans GB","Microsoft YaHei",Arial,sans-serif; }
|
||||
.wx-export-page { width: 210mm; min-height: 297mm; margin: 12mm auto; padding: 16mm 18mm 20mm; background: #fff; box-shadow: 0 2mm 8mm rgba(0,0,0,.12); }
|
||||
.wx-export-header { margin: 0 0 10mm; padding-bottom: 6mm; border-bottom: .25mm solid #ddd; }
|
||||
.wx-export-title { margin: 0 0 4mm; font-size: 25px; line-height: 1.35; font-weight: 700; overflow-wrap: anywhere; }
|
||||
.wx-export-meta { color: #777; font-size: 14px; line-height: 1.6; }
|
||||
.wx-export-source { margin-top: 2mm; color: #888; font-size: 11px; overflow-wrap: anywhere; }
|
||||
.wx-export-source a { color: inherit; }
|
||||
.wx-export-content { width: 100% !important; max-width: 100% !important; visibility: visible !important; font-size: 16px; line-height: 1.8; overflow-wrap: anywhere; }
|
||||
.wx-export-content img, .wx-export-content svg, .wx-export-content canvas { max-width: 100% !important; height: auto !important; }
|
||||
.wx-export-content table { max-width: 100%; border-collapse: collapse; }
|
||||
.wx-export-content pre { white-space: pre-wrap; overflow-wrap: anywhere; }
|
||||
.wx-export-content iframe { max-width: 100%; }
|
||||
.wx-export-media-note { padding: 3mm; color: #666; background: #f5f5f5; font-size: 13px; overflow-wrap: anywhere; }
|
||||
img, figure, table, blockquote, pre { break-inside: avoid; page-break-inside: avoid; }
|
||||
@media (max-width: 210mm) {
|
||||
html { background: #fff; }
|
||||
.wx-export-page { width: 100%; min-height: 0; margin: 0; padding: 8vw 6vw 12vw; box-shadow: none; }
|
||||
}
|
||||
@media print {
|
||||
@page { size: A4; margin: 0; }
|
||||
html, body { width: 210mm; background: #fff; }
|
||||
.wx-export-page { width: 210mm; min-height: 0; margin: 0; padding: 16mm 18mm 20mm; box-shadow: none; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main class="wx-export-page">
|
||||
<header class="wx-export-header">
|
||||
<h1 class="wx-export-title">${htmlEscape(snapshot.title)}</h1>
|
||||
${metadata ? `<div class="wx-export-meta">${metadata}</div>` : ''}
|
||||
<div class="wx-export-source">来源：<a href="${htmlEscape(snapshot.canonical)}">${htmlEscape(snapshot.canonical)}</a></div>
|
||||
</header>
|
||||
${articleHtml}
|
||||
</main>
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
|
||||
function mimeExtension(mime) {
|
||||
const extensions = {
|
||||
'image/jpeg': 'jpg',
|
||||
'image/png': 'png',
|
||||
'image/gif': 'gif',
|
||||
'image/webp': 'webp',
|
||||
'image/svg+xml': 'svg',
|
||||
'image/avif': 'avif',
|
||||
};
|
||||
return extensions[mime] || 'bin';
|
||||
}
|
||||
|
||||
function yamlString(value) {
|
||||
return JSON.stringify(String(value || ''));
|
||||
}
|
||||
|
||||
async function writeMarkdown(snapshot, assets, outputDir, base) {
|
||||
const assetDirectoryName = `${base}.assets`;
|
||||
const assetDirectory = join(outputDir, assetDirectoryName);
|
||||
await mkdir(assetDirectory, { recursive: true });
|
||||
|
||||
const filenames = new Map();
|
||||
let assetNumber = 0;
|
||||
for (const resource of snapshot.resources) {
|
||||
const asset = assets.get(resource.url);
|
||||
if (!asset?.ok || filenames.has(resource.url)) continue;
|
||||
assetNumber += 1;
|
||||
const filename = `image-${String(assetNumber).padStart(3, '0')}.${mimeExtension(asset.mime)}`;
|
||||
await writeFile(join(assetDirectory, filename), asset.buffer);
|
||||
filenames.set(resource.url, `${assetDirectoryName}/${filename}`);
|
||||
}
|
||||
|
||||
let sourceHtml = snapshot.html;
|
||||
for (const resource of snapshot.resources) {
|
||||
const marker = `data-wxasset-${resource.kind}="${resource.id}"`;
|
||||
const localPath = filenames.get(resource.url);
|
||||
if (resource.kind === 'img') {
|
||||
sourceHtml = sourceHtml.replace(marker, localPath ? `src="${htmlEscape(localPath)}"` : '');
|
||||
} else if (resource.kind === 'svg-image') {
|
||||
sourceHtml = sourceHtml.replace(marker, localPath ? `href="${htmlEscape(localPath)}"` : '');
|
||||
} else if (resource.kind === 'background') {
|
||||
sourceHtml = sourceHtml.replace(`__${resource.id}__`, localPath || '');
|
||||
sourceHtml = sourceHtml.replace(marker, '');
|
||||
}
|
||||
}
|
||||
sourceHtml = sourceHtml.replaceAll(/\sdata-wxasset-[\w-]+="[^"]+"/g, '');
|
||||
|
||||
const turndown = new TurndownService({
|
||||
headingStyle: 'atx',
|
||||
bulletListMarker: '-',
|
||||
codeBlockStyle: 'fenced',
|
||||
emDelimiter: '*',
|
||||
strongDelimiter: '**',
|
||||
});
|
||||
turndown.use(gfm);
|
||||
turndown.addRule('wechatMediaNote', {
|
||||
filter: (node) => node.nodeType === 1 && node.classList?.contains('wx-export-media-note'),
|
||||
replacement: (content) => `\n\n> ${content.trim()}\n\n`,
|
||||
});
|
||||
const body = turndown.turndown(sourceHtml).replace(/\n{3,}/g, '\n\n').trim();
|
||||
const markdown = `---
|
||||
title: ${yamlString(snapshot.title)}
|
||||
author: ${yamlString(snapshot.author)}
|
||||
published: ${yamlString(snapshot.published)}
|
||||
source: ${yamlString(snapshot.canonical)}
|
||||
archived_at: ${yamlString(new Date().toISOString())}
|
||||
---
|
||||
|
||||
# ${snapshot.title}
|
||||
|
||||
${body}
|
||||
`;
|
||||
const markdownPath = join(outputDir, `${base}.md`);
|
||||
await writeFile(markdownPath, markdown);
|
||||
return markdownPath;
|
||||
}
|
||||
|
||||
async function writePdf(browserContext, htmlPath, pdfPath, mode) {
|
||||
const page = await browserContext.newPage();
|
||||
await page.goto(pathToFileURL(htmlPath).href, { waitUntil: 'load' });
|
||||
await page.emulateMedia({ media: 'print' });
|
||||
await page.evaluate(() => document.fonts?.ready);
|
||||
|
||||
if (mode === 'a4') {
|
||||
await page.pdf({ path: pdfPath, format: 'A4', printBackground: true, margin: { top: 0, right: 0, bottom: 0, left: 0 } });
|
||||
} else {
|
||||
const heightPx = await page.evaluate(() => Math.max(
|
||||
document.body.scrollHeight,
|
||||
document.documentElement.scrollHeight,
|
||||
document.querySelector('.wx-export-page')?.scrollHeight || 0,
|
||||
));
|
||||
const heightMm = Math.ceil((heightPx * 25.4) / 96) + 2;
|
||||
if (heightMm > 5000) {
|
||||
throw new Error(`正文约 ${heightMm}mm，超过 Chromium 单张 PDF 的安全高度 5000mm；请使用 --pdf-mode a4，单页 HTML 已保留`);
|
||||
}
|
||||
await page.pdf({
|
||||
path: pdfPath,
|
||||
width: '210mm',
|
||||
height: `${heightMm}mm`,
|
||||
printBackground: true,
|
||||
margin: { top: 0, right: 0, bottom: 0, left: 0 },
|
||||
preferCSSPageSize: false,
|
||||
});
|
||||
}
|
||||
await page.close();
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { url, options } = parseArgs(process.argv.slice(2));
|
||||
await mkdir(options.outputDir, { recursive: true });
|
||||
await mkdir(options.profile, { recursive: true });
|
||||
|
||||
const responseCache = new Map();
|
||||
const context = await chromium.launchPersistentContext(options.profile, {
|
||||
headless: !options.headed,
|
||||
viewport: { width: 1280, height: 900 },
|
||||
locale: 'zh-CN',
|
||||
timezoneId: 'Asia/Shanghai',
|
||||
serviceWorkers: 'block',
|
||||
userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
|
||||
});
|
||||
|
||||
let exitCode = 0;
|
||||
try {
|
||||
const page = context.pages()[0] || await context.newPage();
|
||||
page.setDefaultTimeout(options.timeout);
|
||||
page.on('response', (response) => {
|
||||
if (response.request().resourceType() !== 'image' || !response.ok()) return;
|
||||
const promise = response.body()
|
||||
.then((buffer) => ({ buffer, contentType: response.headers()['content-type'] || '' }))
|
||||
.catch(() => null);
|
||||
responseCache.set(response.url(), promise);
|
||||
});
|
||||
|
||||
console.error(`打开：${url}`);
|
||||
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: options.timeout });
|
||||
const selector = await waitForArticle(page, options);
|
||||
console.error(`正文：${selector}；触发懒加载图片…`);
|
||||
await activateLazyImages(page, selector, options.imageTimeout);
|
||||
const snapshot = await snapshotArticle(page, selector, options.keepRemoteLinks);
|
||||
console.error(`发现 ${snapshot.resources.length} 个图片引用，正在抓取并内嵌…`);
|
||||
|
||||
const assets = await fetchAssets(context, snapshot.resources, responseCache, page.url());
|
||||
const embedded = embedAssets(snapshot, assets);
|
||||
const base = safeFilename(snapshot.title);
|
||||
const manifestPath = join(options.outputDir, `${base}.assets.json`);
|
||||
const manifest = {
|
||||
sourceUrl: url,
|
||||
finalUrl: page.url(),
|
||||
title: snapshot.title,
|
||||
articleSelector: selector,
|
||||
generatedAt: new Date().toISOString(),
|
||||
discoveredReferences: snapshot.resources.length,
|
||||
uniqueResources: assets.size,
|
||||
embeddedReferences: embedded.records.filter((item) => item.ok).length,
|
||||
failedReferences: embedded.failures.length,
|
||||
complete: embedded.failures.length === 0,
|
||||
resources: embedded.records,
|
||||
};
|
||||
await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
|
||||
|
||||
if (embedded.failures.length && !options.allowMissingImages) {
|
||||
const summary = embedded.failures.slice(0, 5).map((item) => `- ${item.url}: ${item.error}`).join('\n');
|
||||
throw new Error(`有 ${embedded.failures.length} 个图片引用抓取失败；已写完整性报告，未生成不完整成品。\n${summary}\n报告：${manifestPath}`);
|
||||
}
|
||||
|
||||
if (options.format === 'markdown' || options.format === 'all') {
|
||||
const markdownPath = await writeMarkdown(snapshot, assets, options.outputDir, base);
|
||||
console.error(`Markdown：${markdownPath}`);
|
||||
}
|
||||
|
||||
const htmlPath = join(options.outputDir, `${base}.html`);
|
||||
const html = standaloneHtml(snapshot, embedded.html);
|
||||
await writeFile(htmlPath, html);
|
||||
console.error(`HTML：${htmlPath}`);
|
||||
|
||||
if (options.format === 'pdf' || options.format === 'all') {
|
||||
const pdfPath = join(options.outputDir, `${base}.${options.pdfMode === 'continuous' ? 'continuous' : 'a4'}.pdf`);
|
||||
await writePdf(context, htmlPath, pdfPath, options.pdfMode);
|
||||
console.error(`PDF：${pdfPath}`);
|
||||
}
|
||||
if (options.format === 'pdf') {
|
||||
// PDF 以离线 HTML 为中间源；保留它有助于审计图片完整性和重新排版。
|
||||
console.error('提示：保留了生成 PDF 所用的自包含 HTML 中间文件。');
|
||||
}
|
||||
console.error(`完整性：${manifest.complete ? '通过' : '允许缺图'}（${manifest.embeddedReferences}/${manifest.discoveredReferences} 个引用已内嵌）`);
|
||||
} catch (error) {
|
||||
exitCode = 2;
|
||||
console.error(`失败：${cleanErrorMessage(error)}`);
|
||||
} finally {
|
||||
await context.close();
|
||||
}
|
||||
process.exitCode = exitCode;
|
||||
}
|
||||
|
||||
if (pathToFileURL(process.argv[1]).href === import.meta.url) {
|
||||
main().catch((error) => {
|
||||
console.error(`失败：${cleanErrorMessage(error.stack || error)}`);
|
||||
process.exitCode = 2;
|
||||
});
|
||||
}
|
||||
|
||||
export { htmlEscape, inferMime, safeFilename };
|
||||
@@ -0,0 +1,59 @@
|
||||
services:
|
||||
wechat-fulltext:
|
||||
image: wechat-fulltext:2026-09-18
|
||||
pull_policy: never
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: tools-wechat-fulltext
|
||||
user: "1000:1000"
|
||||
restart: unless-stopped
|
||||
shm_size: 1gb
|
||||
environment:
|
||||
HOME: /data/home
|
||||
PUBLIC_BASE_URL: http://192.168.50.49:8091
|
||||
VERIFICATION_URL: http://192.168.50.49:8092/vnc.html?autoconnect=true&resize=scale
|
||||
MANUAL_WAIT_SECONDS: "3600"
|
||||
DEFAULT_PROVIDER: mptext
|
||||
DIRECT_BROWSER_ENABLED: "false"
|
||||
UPSTREAM_TIMEOUT_SECONDS: "20"
|
||||
MPTEXT_MAX_ATTEMPTS: "2"
|
||||
MPTEXT_RETRY_DELAYS_SECONDS: "5"
|
||||
SECONDARY_UPSTREAM_ENDPOINT: http://wechat-download-upstream:5000/api/article
|
||||
SECONDARY_UPSTREAM_PUBLIC_URL: http://192.168.50.49:8093/login.html
|
||||
SECONDARY_UPSTREAM_TIMEOUT_SECONDS: "120"
|
||||
ports:
|
||||
- "192.168.50.49:8091:8091"
|
||||
- "192.168.50.49:8092:6080"
|
||||
volumes:
|
||||
- /opt/wechat-fulltext/data/article-service:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:8091/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"]
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 20s
|
||||
|
||||
wechat-download-upstream:
|
||||
image: tmwgsicp/wechat-download-api@sha256:6977f00af2334af4a8dc9ed19d0a82a5214fd4eca4d1c724f7d14b1299cc66ec
|
||||
container_name: tools-wechat-download-upstream
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
TZ: America/Los_Angeles
|
||||
SITE_URL: http://192.168.50.49:8093
|
||||
RATE_LIMIT_GLOBAL: "20"
|
||||
RATE_LIMIT_PER_IP: "20"
|
||||
RATE_LIMIT_ARTICLE_INTERVAL: "1"
|
||||
RSS_FETCH_FULL_CONTENT: "false"
|
||||
ENABLE_MCP: "0"
|
||||
ports:
|
||||
- "192.168.50.49:8093:5000"
|
||||
volumes:
|
||||
- /opt/wechat-fulltext/data/download-api:/app/data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-sf", "http://127.0.0.1:5000/api/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p "$HOME"
|
||||
rm -f /tmp/.X99-lock /tmp/.X11-unix/X99
|
||||
Xvfb :99 -screen 0 1440x1000x24 -ac +extension RANDR >/tmp/xvfb.log 2>&1 &
|
||||
sleep 1
|
||||
openbox >/tmp/openbox.log 2>&1 &
|
||||
x11vnc -display :99 -forever -shared -nopw -rfbport 5900 >/tmp/x11vnc.log 2>&1 &
|
||||
websockify --web=/usr/share/novnc 6080 localhost:5900 >/tmp/novnc.log 2>&1 &
|
||||
|
||||
exec node /app/service.mjs
|
||||
|
||||
Reference in New Issue
Block a user