454 lines
18 KiB
JavaScript
454 lines
18 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { createHash } from 'node:crypto';
|
|
import { spawn } from 'node:child_process';
|
|
import { createReadStream } from 'node:fs';
|
|
import { mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises';
|
|
import { createServer } from 'node:http';
|
|
import { basename, dirname, join, normalize, resolve, sep } from 'node:path';
|
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
|
|
const here = dirname(fileURLToPath(import.meta.url));
|
|
const exporter = join(here, 'wechat-export.mjs');
|
|
const port = Number(process.env.PORT || 8091);
|
|
const dataDirectory = resolve(process.env.DATA_DIR || '/data');
|
|
const articlesDirectory = join(dataDirectory, 'articles');
|
|
const profileDirectory = join(dataDirectory, 'profile');
|
|
const publicBaseUrl = (process.env.PUBLIC_BASE_URL || `http://127.0.0.1:${port}`).replace(/\/$/, '');
|
|
const verificationUrl = process.env.VERIFICATION_URL || '';
|
|
const defaultProvider = process.env.DEFAULT_PROVIDER || 'mptext';
|
|
const mptextEndpoint = process.env.MPTEXT_ENDPOINT || 'https://down.mptext.top/api/public/v1/download';
|
|
const directBrowserEnabled = process.env.DIRECT_BROWSER_ENABLED === 'true';
|
|
const upstreamTimeout = Number(process.env.UPSTREAM_TIMEOUT_SECONDS || 60) * 1000;
|
|
const mptextMaxAttempts = Math.max(1, Number(process.env.MPTEXT_MAX_ATTEMPTS || 2));
|
|
const mptextRetryDelays = String(process.env.MPTEXT_RETRY_DELAYS_SECONDS || '5')
|
|
.split(',')
|
|
.map((value) => Math.max(0, Number(value.trim()) || 0));
|
|
const secondaryUpstreamEndpoint = process.env.SECONDARY_UPSTREAM_ENDPOINT || '';
|
|
const secondaryUpstreamPublicUrl = process.env.SECONDARY_UPSTREAM_PUBLIC_URL || '';
|
|
const secondaryUpstreamTimeout = Number(process.env.SECONDARY_UPSTREAM_TIMEOUT_SECONDS || 120) * 1000;
|
|
const jobs = new Map();
|
|
let queue = Promise.resolve();
|
|
|
|
function sendJson(response, status, payload) {
|
|
const body = `${JSON.stringify(payload, null, 2)}\n`;
|
|
response.writeHead(status, {
|
|
'content-type': 'application/json; charset=utf-8',
|
|
'content-length': Buffer.byteLength(body),
|
|
'cache-control': 'no-store',
|
|
});
|
|
response.end(body);
|
|
}
|
|
|
|
function normalizeWechatUrl(value) {
|
|
const url = new URL(String(value || ''));
|
|
if (url.protocol !== 'https:' || url.hostname !== 'mp.weixin.qq.com') {
|
|
throw new Error('Only https://mp.weixin.qq.com article URLs are accepted');
|
|
}
|
|
if (!url.pathname.startsWith('/s')) throw new Error('URL is not a WeChat article link');
|
|
url.hash = '';
|
|
return url.href;
|
|
}
|
|
|
|
function articleId(url) {
|
|
return createHash('sha256').update(url).digest('hex').slice(0, 20);
|
|
}
|
|
|
|
function statusUrl(id) {
|
|
return `${publicBaseUrl}/v1/articles/${id}`;
|
|
}
|
|
|
|
async function readJson(path) {
|
|
return JSON.parse(await readFile(path, 'utf8'));
|
|
}
|
|
|
|
async function cachedResult(id) {
|
|
try {
|
|
const result = await readJson(join(articlesDirectory, id, 'result.json'));
|
|
return result.status === 'completed' ? result : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function publicFileUrl(id, relativePath) {
|
|
return `${publicBaseUrl}/v1/articles/${id}/files/${relativePath.split('/').map(encodeURIComponent).join('/')}`;
|
|
}
|
|
|
|
async function collectResult(job) {
|
|
const entries = await readdir(job.directory, { withFileTypes: true });
|
|
const files = entries.filter((entry) => entry.isFile()).map((entry) => entry.name);
|
|
const markdown = files.find((name) => name.endsWith('.md'));
|
|
const pdf = files.find((name) => name.endsWith('.a4.pdf')) || files.find((name) => name.endsWith('.pdf'));
|
|
const html = files.find((name) => name.endsWith('.html'));
|
|
const manifest = files.find((name) => name.endsWith('.assets.json'));
|
|
if (!markdown || !pdf || !html || !manifest) throw new Error('Exporter completed without all required artifacts');
|
|
const assetManifest = await readJson(join(job.directory, manifest));
|
|
assetManifest.sourceUrl = job.url;
|
|
assetManifest.acquisitionProvider = job.acquisition_provider || job.provider;
|
|
await writeFile(join(job.directory, manifest), `${JSON.stringify(assetManifest, null, 2)}\n`);
|
|
return {
|
|
id: job.id,
|
|
status: 'completed',
|
|
cached: false,
|
|
source_url: job.url,
|
|
acquisition_provider: job.acquisition_provider || job.provider,
|
|
title: assetManifest.title,
|
|
complete: assetManifest.complete,
|
|
generated_at: new Date().toISOString(),
|
|
markdown_url: publicFileUrl(job.id, markdown),
|
|
pdf_url: publicFileUrl(job.id, pdf),
|
|
html_url: publicFileUrl(job.id, html),
|
|
manifest_url: publicFileUrl(job.id, manifest),
|
|
status_url: statusUrl(job.id),
|
|
};
|
|
}
|
|
|
|
function htmlAttribute(value) {
|
|
return String(value).replaceAll('&', '&').replaceAll('"', '"').replaceAll('<', '<');
|
|
}
|
|
|
|
function htmlText(value) {
|
|
return htmlAttribute(value).replaceAll("'", ''').replaceAll('>', '>');
|
|
}
|
|
|
|
function sleep(milliseconds) {
|
|
return new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds));
|
|
}
|
|
|
|
function containsArticleBody(html) {
|
|
return /<(?:div|section)[^>]+id=["']js_content["']/i.test(html);
|
|
}
|
|
|
|
async function saveUpstreamAttempt(job, attempt, response, html, error = '') {
|
|
const prefix = `_upstream-attempt-${String(attempt).padStart(2, '0')}`;
|
|
await writeFile(join(job.directory, `${prefix}.htm`), html);
|
|
await writeFile(join(job.directory, `${prefix}.json`), `${JSON.stringify({
|
|
attempt,
|
|
fetched_at: new Date().toISOString(),
|
|
endpoint: new URL(mptextEndpoint).origin,
|
|
http_status: response?.status || 0,
|
|
content_type: response?.headers?.get('content-type') || '',
|
|
bytes: Buffer.byteLength(html),
|
|
contains_article_body: containsArticleBody(html),
|
|
error,
|
|
}, null, 2)}\n`);
|
|
}
|
|
|
|
async function fetchMptextHtml(job) {
|
|
let lastStatus = 0;
|
|
let lastBytes = 0;
|
|
let lastError = '';
|
|
for (let attempt = 1; attempt <= mptextMaxAttempts; attempt += 1) {
|
|
const endpoint = new URL(mptextEndpoint);
|
|
endpoint.searchParams.set('url', job.url);
|
|
endpoint.searchParams.set('format', 'html');
|
|
let response;
|
|
let html = '';
|
|
try {
|
|
response = await fetch(endpoint, {
|
|
headers: { Accept: 'text/html,application/xhtml+xml' },
|
|
signal: AbortSignal.timeout(upstreamTimeout),
|
|
});
|
|
html = await response.text();
|
|
lastStatus = response.status;
|
|
lastBytes = Buffer.byteLength(html);
|
|
lastError = response.ok && containsArticleBody(html)
|
|
? ''
|
|
: response.ok
|
|
? 'response does not contain #js_content'
|
|
: `HTTP ${response.status}`;
|
|
await saveUpstreamAttempt(job, attempt, response, html, lastError);
|
|
job.upstream_attempts = attempt;
|
|
if (response.ok && containsArticleBody(html)) {
|
|
const metadata = `<base href="${htmlAttribute(job.url)}"><link rel="canonical" href="${htmlAttribute(job.url)}">`;
|
|
const archivedHtml = /<head(?:\s[^>]*)?>/i.test(html)
|
|
? html.replace(/<head(?:\s[^>]*)?>/i, (head) => `${head}\n${metadata}`)
|
|
: `${metadata}\n${html}`;
|
|
const sourcePath = join(job.directory, '_upstream-source.htm');
|
|
await writeFile(sourcePath, archivedHtml);
|
|
job.acquisition_provider = 'mptext';
|
|
return pathToFileURL(sourcePath).href;
|
|
}
|
|
} catch (error) {
|
|
lastError = String(error?.message || error);
|
|
await saveUpstreamAttempt(job, attempt, response, html, lastError);
|
|
job.upstream_attempts = attempt;
|
|
}
|
|
if (attempt < mptextMaxAttempts) {
|
|
const delaySeconds = mptextRetryDelays[Math.min(attempt - 1, mptextRetryDelays.length - 1)] || 0;
|
|
job.retrying_in_seconds = delaySeconds;
|
|
await sleep(delaySeconds * 1000);
|
|
delete job.retrying_in_seconds;
|
|
}
|
|
}
|
|
if (secondaryUpstreamEndpoint) {
|
|
try {
|
|
return await fetchSecondaryUpstreamHtml(job);
|
|
} catch (error) {
|
|
const secondaryError = String(error?.message || error);
|
|
job.secondary_upstream_error = secondaryError;
|
|
if (/未登录|登录已失效|login|credential|auth/i.test(secondaryError)) {
|
|
job.secondary_login_url = secondaryUpstreamPublicUrl || undefined;
|
|
}
|
|
throw new Error(
|
|
`mptext did not return an article after ${mptextMaxAttempts} attempts; `
|
|
+ `secondary upstream failed: ${secondaryError}`,
|
|
);
|
|
}
|
|
}
|
|
throw new Error(
|
|
`mptext did not return an article after ${mptextMaxAttempts} attempts `
|
|
+ `(last HTTP ${lastStatus || 'network error'}, ${lastBytes} bytes): ${lastError}`,
|
|
);
|
|
}
|
|
|
|
async function fetchSecondaryUpstreamHtml(job) {
|
|
let response;
|
|
let raw = '';
|
|
try {
|
|
response = await fetch(secondaryUpstreamEndpoint, {
|
|
method: 'POST',
|
|
headers: { Accept: 'application/json', 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ url: job.url }),
|
|
signal: AbortSignal.timeout(secondaryUpstreamTimeout),
|
|
});
|
|
raw = await response.text();
|
|
} catch (error) {
|
|
await writeFile(join(job.directory, '_secondary-upstream.json'), `${JSON.stringify({
|
|
fetched_at: new Date().toISOString(),
|
|
endpoint: secondaryUpstreamEndpoint,
|
|
http_status: response?.status || 0,
|
|
error: String(error?.message || error),
|
|
}, null, 2)}\n`);
|
|
throw error;
|
|
}
|
|
|
|
let payload;
|
|
try {
|
|
payload = JSON.parse(raw);
|
|
} catch {
|
|
throw new Error(`HTTP ${response.status}: secondary upstream returned invalid JSON`);
|
|
} finally {
|
|
await writeFile(join(job.directory, '_secondary-upstream-response.json'), raw);
|
|
}
|
|
|
|
if (!response.ok || !payload?.success) {
|
|
throw new Error(payload?.error || `HTTP ${response.status}`);
|
|
}
|
|
const data = payload.data || {};
|
|
if (!data.content || typeof data.content !== 'string') {
|
|
throw new Error('secondary upstream response does not contain article HTML');
|
|
}
|
|
|
|
const title = data.title || '微信文章';
|
|
const author = data.author || '';
|
|
const published = data.publish_time_str || '';
|
|
const archivedHtml = `<!doctype html>
|
|
<html lang="zh-CN">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<base href="${htmlAttribute(job.url)}">
|
|
<link rel="canonical" href="${htmlAttribute(job.url)}">
|
|
<meta property="og:title" content="${htmlAttribute(title)}">
|
|
<title>${htmlText(title)}</title>
|
|
</head>
|
|
<body>
|
|
<h1 id="activity-name">${htmlText(title)}</h1>
|
|
<div id="js_name">${htmlText(author)}</div>
|
|
<time id="publish_time">${htmlText(published)}</time>
|
|
<div id="js_content" class="rich_media_content">${data.content}</div>
|
|
</body>
|
|
</html>`;
|
|
const sourcePath = join(job.directory, '_upstream-source.htm');
|
|
await writeFile(sourcePath, archivedHtml);
|
|
job.acquisition_provider = 'wechat-download-api';
|
|
return pathToFileURL(sourcePath).href;
|
|
}
|
|
|
|
async function runJob(job) {
|
|
job.status = 'running';
|
|
job.started_at = new Date().toISOString();
|
|
await rm(job.directory, { recursive: true, force: true });
|
|
await mkdir(job.directory, { recursive: true });
|
|
await mkdir(profileDirectory, { recursive: true });
|
|
|
|
let inputUrl = job.url;
|
|
if (job.provider === 'mptext') inputUrl = await fetchMptextHtml(job);
|
|
else if (job.provider !== 'browser') throw new Error(`Unsupported acquisition provider: ${job.provider}`);
|
|
|
|
const args = [
|
|
exporter,
|
|
inputUrl,
|
|
'--output-dir', job.directory,
|
|
'--profile', profileDirectory,
|
|
'--format', 'all',
|
|
'--pdf-mode', 'a4',
|
|
];
|
|
if (job.provider === 'browser') {
|
|
args.push('--headed', '--manual-wait', String(Number(process.env.MANUAL_WAIT_SECONDS || 600)));
|
|
} else {
|
|
args.push('--selector', '#js_content');
|
|
}
|
|
const child = spawn(process.execPath, args, { cwd: here, env: process.env });
|
|
job.pid = child.pid;
|
|
let log = '';
|
|
const capture = (chunk) => {
|
|
const text = chunk.toString();
|
|
log = `${log}${text}`.slice(-64 * 1024);
|
|
if (/微信要求(?:安全)?验证|完成验证|人工验证/.test(text)) {
|
|
job.status = 'verification_required';
|
|
job.verification_url = verificationUrl || undefined;
|
|
}
|
|
};
|
|
child.stdout.on('data', capture);
|
|
child.stderr.on('data', capture);
|
|
const exitCode = await new Promise((done, reject) => {
|
|
child.once('error', reject);
|
|
child.once('close', done);
|
|
});
|
|
if (exitCode !== 0) throw new Error(log.trim().split('\n').slice(-8).join('\n') || `Exporter exited ${exitCode}`);
|
|
const result = await collectResult(job);
|
|
await writeFile(join(job.directory, 'result.json'), `${JSON.stringify(result, null, 2)}\n`);
|
|
Object.assign(job, result);
|
|
}
|
|
|
|
function enqueue(job) {
|
|
jobs.set(job.id, job);
|
|
queue = queue.then(async () => {
|
|
try {
|
|
await runJob(job);
|
|
} catch (error) {
|
|
job.status = 'failed';
|
|
job.error = String(error?.message || error);
|
|
job.finished_at = new Date().toISOString();
|
|
try {
|
|
await writeFile(join(job.directory, 'failure.json'), `${JSON.stringify({
|
|
id: job.id,
|
|
status: job.status,
|
|
source_url: job.url,
|
|
provider: job.provider,
|
|
created_at: job.created_at,
|
|
started_at: job.started_at,
|
|
finished_at: job.finished_at,
|
|
upstream_attempts: job.upstream_attempts || 0,
|
|
secondary_upstream_error: job.secondary_upstream_error,
|
|
secondary_login_url: job.secondary_login_url,
|
|
error: job.error,
|
|
}, null, 2)}\n`);
|
|
} catch {
|
|
// The in-memory status remains available even if diagnostic persistence fails.
|
|
}
|
|
console.error(`[wechat-article-service] job ${job.id} failed: ${job.error}`);
|
|
}
|
|
});
|
|
}
|
|
|
|
async function bodyJson(request) {
|
|
const chunks = [];
|
|
let bytes = 0;
|
|
for await (const chunk of request) {
|
|
bytes += chunk.length;
|
|
if (bytes > 16 * 1024) throw new Error('Request body too large');
|
|
chunks.push(chunk);
|
|
}
|
|
return JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}');
|
|
}
|
|
|
|
function contentType(path) {
|
|
if (path.endsWith('.md')) return 'text/markdown; charset=utf-8';
|
|
if (path.endsWith('.pdf')) return 'application/pdf';
|
|
if (path.endsWith('.html')) return 'text/html; charset=utf-8';
|
|
if (path.endsWith('.json')) return 'application/json; charset=utf-8';
|
|
if (path.endsWith('.png')) return 'image/png';
|
|
if (path.endsWith('.jpg') || path.endsWith('.jpeg')) return 'image/jpeg';
|
|
if (path.endsWith('.gif')) return 'image/gif';
|
|
if (path.endsWith('.webp')) return 'image/webp';
|
|
if (path.endsWith('.svg')) return 'image/svg+xml';
|
|
if (path.endsWith('.avif')) return 'image/avif';
|
|
return 'application/octet-stream';
|
|
}
|
|
|
|
async function serveFile(response, id, requestedPath) {
|
|
const root = resolve(articlesDirectory, id);
|
|
const target = resolve(root, normalize(requestedPath));
|
|
if (!target.startsWith(`${root}${sep}`)) return sendJson(response, 400, { error: 'Invalid path' });
|
|
const info = await stat(target);
|
|
if (!info.isFile()) return sendJson(response, 404, { error: 'Not found' });
|
|
response.writeHead(200, {
|
|
'content-type': contentType(target),
|
|
'content-length': info.size,
|
|
'cache-control': 'private, max-age=86400',
|
|
});
|
|
createReadStream(target).pipe(response);
|
|
}
|
|
|
|
await Promise.all([mkdir(articlesDirectory, { recursive: true }), mkdir(profileDirectory, { recursive: true })]);
|
|
|
|
const server = createServer(async (request, response) => {
|
|
try {
|
|
const requestUrl = new URL(request.url, `http://${request.headers.host || 'localhost'}`);
|
|
if (request.method === 'GET' && requestUrl.pathname === '/health') {
|
|
return sendJson(response, 200, { status: 'ok', queued_or_running: [...jobs.values()].filter((job) => !['completed', 'failed'].includes(job.status)).length });
|
|
}
|
|
if (request.method === 'GET' && requestUrl.pathname === '/') {
|
|
return sendJson(response, 200, {
|
|
service: 'wechat-article-service',
|
|
default_provider: defaultProvider,
|
|
direct_browser_enabled: directBrowserEnabled,
|
|
mptext_max_attempts: mptextMaxAttempts,
|
|
mptext_retry_delays_seconds: mptextRetryDelays,
|
|
secondary_upstream_enabled: Boolean(secondaryUpstreamEndpoint),
|
|
secondary_upstream_login_url: secondaryUpstreamPublicUrl || undefined,
|
|
create: 'POST /v1/articles with JSON {"url":"https://mp.weixin.qq.com/s/...","force":false,"provider":"mptext"}',
|
|
status: 'GET /v1/articles/{id}',
|
|
});
|
|
}
|
|
if (request.method === 'POST' && requestUrl.pathname === '/v1/articles') {
|
|
const payload = await bodyJson(request);
|
|
const url = normalizeWechatUrl(payload.url);
|
|
const provider = String(payload.provider || defaultProvider);
|
|
if (!['mptext', 'browser'].includes(provider)) throw new Error('provider must be mptext or browser');
|
|
if (provider === 'browser' && !directBrowserEnabled) {
|
|
throw new Error('Direct browser acquisition is disabled to avoid triggering WeChat risk control');
|
|
}
|
|
const id = articleId(url);
|
|
const existing = await cachedResult(id);
|
|
if (existing && !payload.force) return sendJson(response, 200, { ...existing, cached: true });
|
|
const active = jobs.get(id);
|
|
if (active && !['completed', 'failed'].includes(active.status)) return sendJson(response, 202, active);
|
|
const job = {
|
|
id,
|
|
status: 'queued',
|
|
source_url: url,
|
|
provider,
|
|
url,
|
|
directory: join(articlesDirectory, id),
|
|
created_at: new Date().toISOString(),
|
|
status_url: statusUrl(id),
|
|
};
|
|
enqueue(job);
|
|
return sendJson(response, 202, job);
|
|
}
|
|
const statusMatch = requestUrl.pathname.match(/^\/v1\/articles\/([a-f0-9]{20})$/);
|
|
if (request.method === 'GET' && statusMatch) {
|
|
const id = statusMatch[1];
|
|
const active = jobs.get(id);
|
|
if (active) return sendJson(response, 200, active);
|
|
const existing = await cachedResult(id);
|
|
return existing ? sendJson(response, 200, existing) : sendJson(response, 404, { error: 'Article job not found' });
|
|
}
|
|
const fileMatch = requestUrl.pathname.match(/^\/v1\/articles\/([a-f0-9]{20})\/files\/(.+)$/);
|
|
if (request.method === 'GET' && fileMatch) {
|
|
return await serveFile(response, fileMatch[1], decodeURIComponent(fileMatch[2]));
|
|
}
|
|
return sendJson(response, 404, { error: 'Not found' });
|
|
} catch (error) {
|
|
return sendJson(response, 400, { error: String(error?.message || error) });
|
|
}
|
|
});
|
|
|
|
server.listen(port, '0.0.0.0', () => {
|
|
console.log(`wechat-article-service listening on 0.0.0.0:${port}`);
|
|
});
|