diff --git a/servers/tools-pve/wechat-fulltext/Dockerfile b/servers/tools-pve/wechat-fulltext/Dockerfile new file mode 100644 index 0000000..19ffab2 --- /dev/null +++ b/servers/tools-pve/wechat-fulltext/Dockerfile @@ -0,0 +1,27 @@ +FROM mcr.microsoft.com/playwright:v1.58.2-noble + +USER root +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + fonts-noto-cjk novnc openbox websockify x11vnc xvfb \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY app/package.json app/package-lock.json ./ +RUN npm ci --omit=dev +COPY --chown=pwuser:pwuser app/ ./ +COPY --chown=pwuser:pwuser entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh \ + && chmod -R a+rX /app \ + && mkdir -p /data \ + && chown -R pwuser:pwuser /data /app + +USER pwuser +ENV DISPLAY=:99 \ + HOME=/home/pwuser \ + PORT=8091 \ + DATA_DIR=/data + +EXPOSE 8091 6080 +ENTRYPOINT ["/entrypoint.sh"] + diff --git a/servers/tools-pve/wechat-fulltext/README.md b/servers/tools-pve/wechat-fulltext/README.md new file mode 100644 index 0000000..0889dbd --- /dev/null +++ b/servers/tools-pve/wechat-fulltext/README.md @@ -0,0 +1,13 @@ +# WeChat Fulltext + +Internal LAN-only service on `tools-pve` that accepts a WeChat public-account +article URL and returns archived Markdown, HTML, PDF, and an image manifest. + +- API: `http://192.168.50.49:8091` +- Manual browser verification: `http://192.168.50.49:8092/vnc.html` +- WeChat upstream login: `http://192.168.50.49:8093/login.html` +- State: `/opt/wechat-fulltext/data/` + +The upstream image is pinned to the immutable amd64 digest for v1.7.0. The +locally built exporter image is versioned and uses `pull_policy: never`. + diff --git a/servers/tools-pve/wechat-fulltext/app/package-lock.json b/servers/tools-pve/wechat-fulltext/app/package-lock.json new file mode 100644 index 0000000..7bb02a8 --- /dev/null +++ b/servers/tools-pve/wechat-fulltext/app/package-lock.json @@ -0,0 +1,85 @@ +{ + "name": "wechat-article-exporter", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "wechat-article-exporter", + "version": "1.0.0", + "dependencies": { + "playwright": "1.58.2", + "turndown": "7.2.0", + "turndown-plugin-gfm": "1.0.2" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@mixmark-io/domino": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", + "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==", + "license": "BSD-2-Clause" + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", + "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.58.2" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.58.2", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", + "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/turndown": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.0.tgz", + "integrity": "sha512-eCZGBN4nNNqM9Owkv9HAtWRYfLA4h909E/WGAWWBpmB275ehNhZyk87/Tpvjbp0jjNl9XwCsbe6bm6CqFsgD+A==", + "license": "MIT", + "dependencies": { + "@mixmark-io/domino": "^2.2.0" + } + }, + "node_modules/turndown-plugin-gfm": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/turndown-plugin-gfm/-/turndown-plugin-gfm-1.0.2.tgz", + "integrity": "sha512-vwz9tfvF7XN/jE0dGoBei3FXWuvll78ohzCZQuOb+ZjWrs3a0XhQVomJEb2Qh4VHTPNRO4GPZh0V7VRbiWwkRg==", + "license": "MIT" + } + } +} diff --git a/servers/tools-pve/wechat-fulltext/app/package.json b/servers/tools-pve/wechat-fulltext/app/package.json new file mode 100644 index 0000000..3a68585 --- /dev/null +++ b/servers/tools-pve/wechat-fulltext/app/package.json @@ -0,0 +1,19 @@ +{ + "name": "wechat-article-exporter", + "version": "1.0.0", + "private": true, + "type": "module", + "description": "Export a WeChat article to clean Markdown, self-contained HTML, and PDF.", + "scripts": { + "export": "node wechat-export.mjs", + "test": "node test/smoke-test.mjs" + }, + "dependencies": { + "playwright": "1.58.2", + "turndown": "7.2.0", + "turndown-plugin-gfm": "1.0.2" + }, + "engines": { + "node": ">=20" + } +} diff --git a/servers/tools-pve/wechat-fulltext/app/service.mjs b/servers/tools-pve/wechat-fulltext/app/service.mjs new file mode 100644 index 0000000..e14339a --- /dev/null +++ b/servers/tools-pve/wechat-fulltext/app/service.mjs @@ -0,0 +1,453 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { createReadStream } from 'node:fs'; +import { mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { basename, dirname, join, normalize, resolve, sep } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const here = dirname(fileURLToPath(import.meta.url)); +const exporter = join(here, 'wechat-export.mjs'); +const port = Number(process.env.PORT || 8091); +const dataDirectory = resolve(process.env.DATA_DIR || '/data'); +const articlesDirectory = join(dataDirectory, 'articles'); +const profileDirectory = join(dataDirectory, 'profile'); +const publicBaseUrl = (process.env.PUBLIC_BASE_URL || `http://127.0.0.1:${port}`).replace(/\/$/, ''); +const verificationUrl = process.env.VERIFICATION_URL || ''; +const defaultProvider = process.env.DEFAULT_PROVIDER || 'mptext'; +const mptextEndpoint = process.env.MPTEXT_ENDPOINT || 'https://down.mptext.top/api/public/v1/download'; +const directBrowserEnabled = process.env.DIRECT_BROWSER_ENABLED === 'true'; +const upstreamTimeout = Number(process.env.UPSTREAM_TIMEOUT_SECONDS || 60) * 1000; +const mptextMaxAttempts = Math.max(1, Number(process.env.MPTEXT_MAX_ATTEMPTS || 2)); +const mptextRetryDelays = String(process.env.MPTEXT_RETRY_DELAYS_SECONDS || '5') + .split(',') + .map((value) => Math.max(0, Number(value.trim()) || 0)); +const secondaryUpstreamEndpoint = process.env.SECONDARY_UPSTREAM_ENDPOINT || ''; +const secondaryUpstreamPublicUrl = process.env.SECONDARY_UPSTREAM_PUBLIC_URL || ''; +const secondaryUpstreamTimeout = Number(process.env.SECONDARY_UPSTREAM_TIMEOUT_SECONDS || 120) * 1000; +const jobs = new Map(); +let queue = Promise.resolve(); + +function sendJson(response, status, payload) { + const body = `${JSON.stringify(payload, null, 2)}\n`; + response.writeHead(status, { + 'content-type': 'application/json; charset=utf-8', + 'content-length': Buffer.byteLength(body), + 'cache-control': 'no-store', + }); + response.end(body); +} + +function normalizeWechatUrl(value) { + const url = new URL(String(value || '')); + if (url.protocol !== 'https:' || url.hostname !== 'mp.weixin.qq.com') { + throw new Error('Only https://mp.weixin.qq.com article URLs are accepted'); + } + if (!url.pathname.startsWith('/s')) throw new Error('URL is not a WeChat article link'); + url.hash = ''; + return url.href; +} + +function articleId(url) { + return createHash('sha256').update(url).digest('hex').slice(0, 20); +} + +function statusUrl(id) { + return `${publicBaseUrl}/v1/articles/${id}`; +} + +async function readJson(path) { + return JSON.parse(await readFile(path, 'utf8')); +} + +async function cachedResult(id) { + try { + const result = await readJson(join(articlesDirectory, id, 'result.json')); + return result.status === 'completed' ? result : null; + } catch { + return null; + } +} + +function publicFileUrl(id, relativePath) { + return `${publicBaseUrl}/v1/articles/${id}/files/${relativePath.split('/').map(encodeURIComponent).join('/')}`; +} + +async function collectResult(job) { + const entries = await readdir(job.directory, { withFileTypes: true }); + const files = entries.filter((entry) => entry.isFile()).map((entry) => entry.name); + const markdown = files.find((name) => name.endsWith('.md')); + const pdf = files.find((name) => name.endsWith('.a4.pdf')) || files.find((name) => name.endsWith('.pdf')); + const html = files.find((name) => name.endsWith('.html')); + const manifest = files.find((name) => name.endsWith('.assets.json')); + if (!markdown || !pdf || !html || !manifest) throw new Error('Exporter completed without all required artifacts'); + const assetManifest = await readJson(join(job.directory, manifest)); + assetManifest.sourceUrl = job.url; + assetManifest.acquisitionProvider = job.acquisition_provider || job.provider; + await writeFile(join(job.directory, manifest), `${JSON.stringify(assetManifest, null, 2)}\n`); + return { + id: job.id, + status: 'completed', + cached: false, + source_url: job.url, + acquisition_provider: job.acquisition_provider || job.provider, + title: assetManifest.title, + complete: assetManifest.complete, + generated_at: new Date().toISOString(), + markdown_url: publicFileUrl(job.id, markdown), + pdf_url: publicFileUrl(job.id, pdf), + html_url: publicFileUrl(job.id, html), + manifest_url: publicFileUrl(job.id, manifest), + status_url: statusUrl(job.id), + }; +} + +function htmlAttribute(value) { + return String(value).replaceAll('&', '&').replaceAll('"', '"').replaceAll('<', '<'); +} + +function htmlText(value) { + return htmlAttribute(value).replaceAll("'", ''').replaceAll('>', '>'); +} + +function sleep(milliseconds) { + return new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)); +} + +function containsArticleBody(html) { + return /<(?:div|section)[^>]+id=["']js_content["']/i.test(html); +} + +async function saveUpstreamAttempt(job, attempt, response, html, error = '') { + const prefix = `_upstream-attempt-${String(attempt).padStart(2, '0')}`; + await writeFile(join(job.directory, `${prefix}.htm`), html); + await writeFile(join(job.directory, `${prefix}.json`), `${JSON.stringify({ + attempt, + fetched_at: new Date().toISOString(), + endpoint: new URL(mptextEndpoint).origin, + http_status: response?.status || 0, + content_type: response?.headers?.get('content-type') || '', + bytes: Buffer.byteLength(html), + contains_article_body: containsArticleBody(html), + error, + }, null, 2)}\n`); +} + +async function fetchMptextHtml(job) { + let lastStatus = 0; + let lastBytes = 0; + let lastError = ''; + for (let attempt = 1; attempt <= mptextMaxAttempts; attempt += 1) { + const endpoint = new URL(mptextEndpoint); + endpoint.searchParams.set('url', job.url); + endpoint.searchParams.set('format', 'html'); + let response; + let html = ''; + try { + response = await fetch(endpoint, { + headers: { Accept: 'text/html,application/xhtml+xml' }, + signal: AbortSignal.timeout(upstreamTimeout), + }); + html = await response.text(); + lastStatus = response.status; + lastBytes = Buffer.byteLength(html); + lastError = response.ok && containsArticleBody(html) + ? '' + : response.ok + ? 'response does not contain #js_content' + : `HTTP ${response.status}`; + await saveUpstreamAttempt(job, attempt, response, html, lastError); + job.upstream_attempts = attempt; + if (response.ok && containsArticleBody(html)) { + const metadata = ``; + const archivedHtml = /]*)?>/i.test(html) + ? html.replace(/]*)?>/i, (head) => `${head}\n${metadata}`) + : `${metadata}\n${html}`; + const sourcePath = join(job.directory, '_upstream-source.htm'); + await writeFile(sourcePath, archivedHtml); + job.acquisition_provider = 'mptext'; + return pathToFileURL(sourcePath).href; + } + } catch (error) { + lastError = String(error?.message || error); + await saveUpstreamAttempt(job, attempt, response, html, lastError); + job.upstream_attempts = attempt; + } + if (attempt < mptextMaxAttempts) { + const delaySeconds = mptextRetryDelays[Math.min(attempt - 1, mptextRetryDelays.length - 1)] || 0; + job.retrying_in_seconds = delaySeconds; + await sleep(delaySeconds * 1000); + delete job.retrying_in_seconds; + } + } + if (secondaryUpstreamEndpoint) { + try { + return await fetchSecondaryUpstreamHtml(job); + } catch (error) { + const secondaryError = String(error?.message || error); + job.secondary_upstream_error = secondaryError; + if (/未登录|登录已失效|login|credential|auth/i.test(secondaryError)) { + job.secondary_login_url = secondaryUpstreamPublicUrl || undefined; + } + throw new Error( + `mptext did not return an article after ${mptextMaxAttempts} attempts; ` + + `secondary upstream failed: ${secondaryError}`, + ); + } + } + throw new Error( + `mptext did not return an article after ${mptextMaxAttempts} attempts ` + + `(last HTTP ${lastStatus || 'network error'}, ${lastBytes} bytes): ${lastError}`, + ); +} + +async function fetchSecondaryUpstreamHtml(job) { + let response; + let raw = ''; + try { + response = await fetch(secondaryUpstreamEndpoint, { + method: 'POST', + headers: { Accept: 'application/json', 'Content-Type': 'application/json' }, + body: JSON.stringify({ url: job.url }), + signal: AbortSignal.timeout(secondaryUpstreamTimeout), + }); + raw = await response.text(); + } catch (error) { + await writeFile(join(job.directory, '_secondary-upstream.json'), `${JSON.stringify({ + fetched_at: new Date().toISOString(), + endpoint: secondaryUpstreamEndpoint, + http_status: response?.status || 0, + error: String(error?.message || error), + }, null, 2)}\n`); + throw error; + } + + let payload; + try { + payload = JSON.parse(raw); + } catch { + throw new Error(`HTTP ${response.status}: secondary upstream returned invalid JSON`); + } finally { + await writeFile(join(job.directory, '_secondary-upstream-response.json'), raw); + } + + if (!response.ok || !payload?.success) { + throw new Error(payload?.error || `HTTP ${response.status}`); + } + const data = payload.data || {}; + if (!data.content || typeof data.content !== 'string') { + throw new Error('secondary upstream response does not contain article HTML'); + } + + const title = data.title || '微信文章'; + const author = data.author || ''; + const published = data.publish_time_str || ''; + const archivedHtml = ` + + + + + + + ${htmlText(title)} + + +

${htmlText(title)}

+
${htmlText(author)}
+ +
${data.content}
+ +`; + const sourcePath = join(job.directory, '_upstream-source.htm'); + await writeFile(sourcePath, archivedHtml); + job.acquisition_provider = 'wechat-download-api'; + return pathToFileURL(sourcePath).href; +} + +async function runJob(job) { + job.status = 'running'; + job.started_at = new Date().toISOString(); + await rm(job.directory, { recursive: true, force: true }); + await mkdir(job.directory, { recursive: true }); + await mkdir(profileDirectory, { recursive: true }); + + let inputUrl = job.url; + if (job.provider === 'mptext') inputUrl = await fetchMptextHtml(job); + else if (job.provider !== 'browser') throw new Error(`Unsupported acquisition provider: ${job.provider}`); + + const args = [ + exporter, + inputUrl, + '--output-dir', job.directory, + '--profile', profileDirectory, + '--format', 'all', + '--pdf-mode', 'a4', + ]; + if (job.provider === 'browser') { + args.push('--headed', '--manual-wait', String(Number(process.env.MANUAL_WAIT_SECONDS || 600))); + } else { + args.push('--selector', '#js_content'); + } + const child = spawn(process.execPath, args, { cwd: here, env: process.env }); + job.pid = child.pid; + let log = ''; + const capture = (chunk) => { + const text = chunk.toString(); + log = `${log}${text}`.slice(-64 * 1024); + if (/微信要求(?:安全)?验证|完成验证|人工验证/.test(text)) { + job.status = 'verification_required'; + job.verification_url = verificationUrl || undefined; + } + }; + child.stdout.on('data', capture); + child.stderr.on('data', capture); + const exitCode = await new Promise((done, reject) => { + child.once('error', reject); + child.once('close', done); + }); + if (exitCode !== 0) throw new Error(log.trim().split('\n').slice(-8).join('\n') || `Exporter exited ${exitCode}`); + const result = await collectResult(job); + await writeFile(join(job.directory, 'result.json'), `${JSON.stringify(result, null, 2)}\n`); + Object.assign(job, result); +} + +function enqueue(job) { + jobs.set(job.id, job); + queue = queue.then(async () => { + try { + await runJob(job); + } catch (error) { + job.status = 'failed'; + job.error = String(error?.message || error); + job.finished_at = new Date().toISOString(); + try { + await writeFile(join(job.directory, 'failure.json'), `${JSON.stringify({ + id: job.id, + status: job.status, + source_url: job.url, + provider: job.provider, + created_at: job.created_at, + started_at: job.started_at, + finished_at: job.finished_at, + upstream_attempts: job.upstream_attempts || 0, + secondary_upstream_error: job.secondary_upstream_error, + secondary_login_url: job.secondary_login_url, + error: job.error, + }, null, 2)}\n`); + } catch { + // The in-memory status remains available even if diagnostic persistence fails. + } + console.error(`[wechat-article-service] job ${job.id} failed: ${job.error}`); + } + }); +} + +async function bodyJson(request) { + const chunks = []; + let bytes = 0; + for await (const chunk of request) { + bytes += chunk.length; + if (bytes > 16 * 1024) throw new Error('Request body too large'); + chunks.push(chunk); + } + return JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}'); +} + +function contentType(path) { + if (path.endsWith('.md')) return 'text/markdown; charset=utf-8'; + if (path.endsWith('.pdf')) return 'application/pdf'; + if (path.endsWith('.html')) return 'text/html; charset=utf-8'; + if (path.endsWith('.json')) return 'application/json; charset=utf-8'; + if (path.endsWith('.png')) return 'image/png'; + if (path.endsWith('.jpg') || path.endsWith('.jpeg')) return 'image/jpeg'; + if (path.endsWith('.gif')) return 'image/gif'; + if (path.endsWith('.webp')) return 'image/webp'; + if (path.endsWith('.svg')) return 'image/svg+xml'; + if (path.endsWith('.avif')) return 'image/avif'; + return 'application/octet-stream'; +} + +async function serveFile(response, id, requestedPath) { + const root = resolve(articlesDirectory, id); + const target = resolve(root, normalize(requestedPath)); + if (!target.startsWith(`${root}${sep}`)) return sendJson(response, 400, { error: 'Invalid path' }); + const info = await stat(target); + if (!info.isFile()) return sendJson(response, 404, { error: 'Not found' }); + response.writeHead(200, { + 'content-type': contentType(target), + 'content-length': info.size, + 'cache-control': 'private, max-age=86400', + }); + createReadStream(target).pipe(response); +} + +await Promise.all([mkdir(articlesDirectory, { recursive: true }), mkdir(profileDirectory, { recursive: true })]); + +const server = createServer(async (request, response) => { + try { + const requestUrl = new URL(request.url, `http://${request.headers.host || 'localhost'}`); + if (request.method === 'GET' && requestUrl.pathname === '/health') { + return sendJson(response, 200, { status: 'ok', queued_or_running: [...jobs.values()].filter((job) => !['completed', 'failed'].includes(job.status)).length }); + } + if (request.method === 'GET' && requestUrl.pathname === '/') { + return sendJson(response, 200, { + service: 'wechat-article-service', + default_provider: defaultProvider, + direct_browser_enabled: directBrowserEnabled, + mptext_max_attempts: mptextMaxAttempts, + mptext_retry_delays_seconds: mptextRetryDelays, + secondary_upstream_enabled: Boolean(secondaryUpstreamEndpoint), + secondary_upstream_login_url: secondaryUpstreamPublicUrl || undefined, + create: 'POST /v1/articles with JSON {"url":"https://mp.weixin.qq.com/s/...","force":false,"provider":"mptext"}', + status: 'GET /v1/articles/{id}', + }); + } + if (request.method === 'POST' && requestUrl.pathname === '/v1/articles') { + const payload = await bodyJson(request); + const url = normalizeWechatUrl(payload.url); + const provider = String(payload.provider || defaultProvider); + if (!['mptext', 'browser'].includes(provider)) throw new Error('provider must be mptext or browser'); + if (provider === 'browser' && !directBrowserEnabled) { + throw new Error('Direct browser acquisition is disabled to avoid triggering WeChat risk control'); + } + const id = articleId(url); + const existing = await cachedResult(id); + if (existing && !payload.force) return sendJson(response, 200, { ...existing, cached: true }); + const active = jobs.get(id); + if (active && !['completed', 'failed'].includes(active.status)) return sendJson(response, 202, active); + const job = { + id, + status: 'queued', + source_url: url, + provider, + url, + directory: join(articlesDirectory, id), + created_at: new Date().toISOString(), + status_url: statusUrl(id), + }; + enqueue(job); + return sendJson(response, 202, job); + } + const statusMatch = requestUrl.pathname.match(/^\/v1\/articles\/([a-f0-9]{20})$/); + if (request.method === 'GET' && statusMatch) { + const id = statusMatch[1]; + const active = jobs.get(id); + if (active) return sendJson(response, 200, active); + const existing = await cachedResult(id); + return existing ? sendJson(response, 200, existing) : sendJson(response, 404, { error: 'Article job not found' }); + } + const fileMatch = requestUrl.pathname.match(/^\/v1\/articles\/([a-f0-9]{20})\/files\/(.+)$/); + if (request.method === 'GET' && fileMatch) { + return await serveFile(response, fileMatch[1], decodeURIComponent(fileMatch[2])); + } + return sendJson(response, 404, { error: 'Not found' }); + } catch (error) { + return sendJson(response, 400, { error: String(error?.message || error) }); + } +}); + +server.listen(port, '0.0.0.0', () => { + console.log(`wechat-article-service listening on 0.0.0.0:${port}`); +}); diff --git a/servers/tools-pve/wechat-fulltext/app/wechat-export.mjs b/servers/tools-pve/wechat-fulltext/app/wechat-export.mjs new file mode 100644 index 0000000..cc7ca6a --- /dev/null +++ b/servers/tools-pve/wechat-fulltext/app/wechat-export.mjs @@ -0,0 +1,677 @@ +#!/usr/bin/env node + +import { chromium } from 'playwright'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { basename, dirname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import TurndownService from 'turndown'; +import turndownPluginGfm from 'turndown-plugin-gfm'; + +const { gfm } = turndownPluginGfm; + +const DEFAULTS = { + format: 'all', + pdfMode: 'continuous', + outputDir: process.cwd(), + profile: join(homedir(), '.cache', 'wechat-article-exporter', 'profile'), + selector: '', + timeout: 90_000, + imageTimeout: 45_000, + manualWait: 300_000, + headed: false, + allowMissingImages: false, + keepRemoteLinks: false, +}; + +function usage(exitCode = 0) { + const command = basename(process.argv[1] || 'wechat-export.mjs'); + console.log(` +用法: + node ${command} <微信文章 URL> [选项] + +选项: + -o, --output-dir <目录> 输出目录（默认: 当前目录） + --format 输出类型（默认: all） + --pdf-mode 单张长页 PDF，或标准 A4 分页（默认: continuous） + --selector 手工指定正文元素 + --profile <目录> 持久浏览器资料目录（保存验证状态/Cookie） + --headed 显示浏览器，可人工完成微信验证 + --manual-wait <秒> headed 模式等待人工验证的时间（默认: 300） + --timeout <秒> 页面加载超时（默认: 90） + --image-timeout <秒> 等待图片加载的总时间（默认: 45） + --allow-missing-images 即使图片不完整也输出（默认会失败关闭） + --keep-remote-links 不移除正文中的远程超链接 + -h, --help 显示帮助 + +示例: + node ${command} 'https://mp.weixin.qq.com/s/...' -o ./out + node ${command} 'https://mp.weixin.qq.com/s/...' --headed --pdf-mode a4 +`); + process.exit(exitCode); +} + +function parseArgs(argv) { + const options = { ...DEFAULTS }; + let url = ''; + const valueFor = (arg, index) => { + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`${arg} 缺少参数`); + return value; + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '-h' || arg === '--help') usage(0); + else if (arg === '-o' || arg === '--output-dir') options.outputDir = valueFor(arg, i++); + else if (arg === '--format') options.format = valueFor(arg, i++); + else if (arg === '--pdf-mode') options.pdfMode = valueFor(arg, i++); + else if (arg === '--selector') options.selector = valueFor(arg, i++); + else if (arg === '--profile') options.profile = valueFor(arg, i++); + else if (arg === '--timeout') options.timeout = Number(valueFor(arg, i++)) * 1000; + else if (arg === '--image-timeout') options.imageTimeout = Number(valueFor(arg, i++)) * 1000; + else if (arg === '--manual-wait') options.manualWait = Number(valueFor(arg, i++)) * 1000; + else if (arg === '--headed') options.headed = true; + else if (arg === '--allow-missing-images') options.allowMissingImages = true; + else if (arg === '--keep-remote-links') options.keepRemoteLinks = true; + else if (arg.startsWith('-')) throw new Error(`未知选项: ${arg}`); + else if (!url) url = arg; + else throw new Error(`多余参数: ${arg}`); + } + + if (!url) usage(1); + if (!['all', 'html', 'pdf', 'markdown'].includes(options.format)) throw new Error('--format 必须是 all、html、pdf 或 markdown'); + if (!['continuous', 'a4'].includes(options.pdfMode)) throw new Error('--pdf-mode 必须是 continuous 或 a4'); + for (const key of ['timeout', 'imageTimeout', 'manualWait']) { + if (!Number.isFinite(options[key]) || options[key] <= 0) throw new Error(`${key} 必须是正数`); + } + options.outputDir = resolve(options.outputDir); + options.profile = resolve(options.profile); + return { url, options }; +} + +function safeFilename(value) { + const cleaned = String(value || 'wechat-article') + .normalize('NFKC') + .replace(/[\\/:*?"<>|\u0000-\u001f]/g, '-') + .replace(/\s+/g, ' ') + .replace(/[. ]+$/g, '') + .trim(); + return (cleaned || 'wechat-article').slice(0, 100); +} + +function htmlEscape(value) { + return String(value ?? '') + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} + +function inferMime(buffer, declared = '') { + const type = declared.split(';')[0].trim().toLowerCase(); + if (type.startsWith('image/')) return type; + if (buffer.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return 'image/png'; + if (buffer.subarray(0, 3).equals(Buffer.from([0xff, 0xd8, 0xff]))) return 'image/jpeg'; + if (buffer.subarray(0, 6).toString('ascii') === 'GIF87a' || buffer.subarray(0, 6).toString('ascii') === 'GIF89a') return 'image/gif'; + if (buffer.subarray(0, 4).toString('ascii') === 'RIFF' && buffer.subarray(8, 12).toString('ascii') === 'WEBP') return 'image/webp'; + const prefix = buffer.subarray(0, 300).toString('utf8').trimStart().toLowerCase(); + if (prefix.startsWith(' false)) { + const size = await candidate.evaluate((element) => ({ + text: (element.innerText || '').trim().length, + images: element.querySelectorAll('img,svg image,canvas').length, + })); + if (size.text >= 80 || size.images > 0) return selector; + } + } + return ''; +} + +async function waitForArticle(page, options) { + let selector = await chooseArticleSelector(page, options.selector); + if (selector) return selector; + + const bodyText = await page.locator('body').innerText({ timeout: 5_000 }).catch(() => ''); + const blocked = isBlockedPage(page.url(), bodyText); + if (!options.headed) { + const reason = blocked ? '微信要求安全验证' : '没有识别到正文'; + throw new Error(`${reason}。请用 --headed 和相同 --profile 重试，完成验证后脚本会继续。`); + } + + console.error(blocked + ? `微信要求验证；请在浏览器中完成操作。最多等待 ${options.manualWait / 1000} 秒…` + : `尚未识别到正文；请在浏览器中完成加载。最多等待 ${options.manualWait / 1000} 秒…`); + const deadline = Date.now() + options.manualWait; + while (Date.now() < deadline) { + await page.waitForTimeout(1000); + selector = await chooseArticleSelector(page, options.selector); + if (selector) return selector; + } + throw new Error('等待人工验证/正文加载超时'); +} + +async function activateLazyImages(page, selector, imageTimeout) { + await page.locator(selector).evaluate((root) => { + for (const image of root.querySelectorAll('img')) { + const lazy = image.getAttribute('data-src') + || image.getAttribute('data-original') + || image.getAttribute('data-backsrc'); + if (lazy && image.getAttribute('src') !== lazy) { + image.setAttribute('src', lazy); + } + image.loading = 'eager'; + image.decoding = 'sync'; + } + for (const source of root.querySelectorAll('source')) { + const lazy = source.getAttribute('data-srcset'); + if (lazy) source.setAttribute('srcset', lazy); + } + }); + + let previousHeight = 0; + let stablePasses = 0; + for (let pass = 0; pass < 3 && stablePasses < 2; pass += 1) { + const height = await page.evaluate(() => document.documentElement.scrollHeight); + for (let y = 0; y < height; y += 600) { + await page.evaluate((position) => window.scrollTo(0, position), y); + await page.waitForTimeout(80); + } + await page.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight)); + await page.waitForTimeout(600); + const newHeight = await page.evaluate(() => document.documentElement.scrollHeight); + stablePasses = Math.abs(newHeight - previousHeight) < 10 ? stablePasses + 1 : 0; + previousHeight = newHeight; + } + + const deadline = Date.now() + imageTimeout; + while (Date.now() < deadline) { + const status = await page.locator(selector).evaluate((root) => { + const images = [...root.querySelectorAll('img')].filter((image) => { + const src = image.currentSrc || image.src || image.dataset.src; + return src && !src.startsWith('data:'); + }); + return { + total: images.length, + pending: images.filter((image) => !image.complete).length, + }; + }); + if (status.pending === 0) break; + await page.waitForTimeout(250); + } + await page.evaluate(() => window.scrollTo(0, 0)); +} + +async function snapshotArticle(page, selector, keepRemoteLinks) { + return page.locator(selector).evaluate((root, keepLinks) => { + const absolute = (value) => { + if (!value) return ''; + if (/^(data|blob):/i.test(value)) return value; + try { return new URL(value, document.baseURI).href; } catch { return value; } + }; + const clone = root.cloneNode(true); + const originals = [root, ...root.querySelectorAll('*')]; + const clones = [clone, ...clone.querySelectorAll('*')]; + const resources = []; + let resourceId = 0; + + const register = (element, kind, url) => { + const resolved = absolute(url); + if (!resolved) return ''; + const id = `wxasset-${resourceId++}`; + resources.push({ id, kind, url: resolved }); + element.setAttribute(`data-wxasset-${kind}`, id); + return id; + }; + + for (let i = 0; i < originals.length; i += 1) { + const original = originals[i]; + const copy = clones[i]; + if (!copy) continue; + if (original instanceof HTMLCanvasElement) { + try { + const image = document.createElement('img'); + image.src = original.toDataURL('image/png'); + image.alt = original.getAttribute('aria-label') || 'canvas'; + register(image, 'img', image.src); + image.removeAttribute('src'); + copy.replaceWith(image); + } catch { + register(copy, 'canvas', `blob:tainted-canvas-${resourceId}`); + } + continue; + } + if (original instanceof HTMLImageElement) { + const url = original.currentSrc || original.src || original.dataset.src + || original.getAttribute('data-original') || original.getAttribute('data-backsrc'); + register(copy, 'img', url); + copy.setAttribute('alt', original.alt || ''); + copy.removeAttribute('src'); + copy.removeAttribute('srcset'); + copy.removeAttribute('data-src'); + copy.removeAttribute('data-original'); + copy.removeAttribute('data-backsrc'); + copy.removeAttribute('data-srcset'); + copy.removeAttribute('loading'); + } else if (original instanceof SVGImageElement) { + register(copy, 'svg-image', original.href?.baseVal || original.getAttribute('href')); + copy.removeAttribute('href'); + copy.removeAttribute('xlink:href'); + } + + const background = getComputedStyle(original).backgroundImage; + if (background && background !== 'none') { + const match = background.match(/^url\(["']?(.*?)["']?\)$/); + if (match) { + const id = register(copy, 'background', match[1]); + if (id) copy.style.backgroundImage = `url("__${id}__")`; + } + } + } + + for (const bad of clone.querySelectorAll('script,noscript,link,meta,form,button,source')) bad.remove(); + for (const frame of clone.querySelectorAll('iframe,video')) { + const source = frame.getAttribute('data-src') || frame.getAttribute('src') || ''; + const note = document.createElement('p'); + note.className = 'wx-export-media-note'; + note.textContent = source ? `原文中的媒体内容：${absolute(source)}` : '原文中的媒体内容（请回到原文查看）'; + frame.replaceWith(note); + } + for (const anchor of clone.querySelectorAll('a')) { + const href = absolute(anchor.getAttribute('href')); + if (keepLinks && /^https?:/i.test(href)) { + anchor.setAttribute('href', href); + anchor.setAttribute('rel', 'noopener noreferrer'); + } else { + anchor.removeAttribute('href'); + anchor.removeAttribute('target'); + } + } + clone.removeAttribute('style'); + clone.classList.add('wx-export-content'); + + const text = (selectors) => { + for (const candidate of selectors) { + const element = document.querySelector(candidate); + const value = element?.textContent?.trim(); + if (value) return value; + } + return ''; + }; + const title = text(['#activity-name', '.rich_media_title', 'h1']) || document.title || '微信文章'; + const author = text(['#js_name', '.rich_media_meta_nickname', '[property="og:article:author"]']); + const published = text(['#publish_time', '.rich_media_meta_text']) || ''; + const canonical = document.querySelector('link[rel="canonical"]')?.href || location.href; + return { title, author, published, canonical, html: clone.outerHTML, resources }; + }, keepRemoteLinks); +} + +function dataUriToAsset(url) { + const match = url.match(/^data:([^;,]+)?(;base64)?,(.*)$/s); + if (!match) throw new Error('无效的 data URI'); + const mime = match[1] || 'text/plain'; + const buffer = match[2] ? Buffer.from(match[3], 'base64') : Buffer.from(decodeURIComponent(match[3])); + if (!mime.startsWith('image/')) throw new Error(`data URI 不是图片: ${mime}`); + return { mime, buffer, source: 'inline' }; +} + +async function fetchAssets(context, resources, responseCache, referer) { + const uniqueUrls = [...new Set(resources.map((item) => item.url))]; + const results = new Map(); + let cursor = 0; + + const fetchOne = async (url) => { + if (url.startsWith('data:')) return dataUriToAsset(url); + if (url.startsWith('blob:')) throw new Error('无法离线保存 blob 图片'); + + const cached = await responseCache.get(url)?.catch(() => null); + if (cached?.buffer?.length) { + const mime = inferMime(cached.buffer, cached.contentType); + if (mime) return { mime, buffer: cached.buffer, source: 'browser-cache' }; + } + + let lastError; + for (let attempt = 1; attempt <= 3; attempt += 1) { + try { + const response = await context.request.get(url, { + headers: { + Referer: referer, + Accept: 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8', + }, + timeout: 30_000, + failOnStatusCode: false, + }); + if (!response.ok()) throw new Error(`HTTP ${response.status()}`); + const buffer = await response.body(); + const mime = inferMime(buffer, response.headers()['content-type'] || ''); + if (!mime) throw new Error(`响应不是图片 (${response.headers()['content-type'] || 'unknown'})`); + return { mime, buffer, source: `request-${attempt}` }; + } catch (error) { + lastError = error; + await new Promise((done) => setTimeout(done, 350 * (2 ** (attempt - 1)))); + } + } + throw lastError; + }; + + async function worker() { + while (cursor < uniqueUrls.length) { + const url = uniqueUrls[cursor++]; + try { + results.set(url, { ok: true, ...(await fetchOne(url)) }); + } catch (error) { + results.set(url, { ok: false, error: cleanErrorMessage(error) }); + } + } + } + await Promise.all(Array.from({ length: Math.min(6, uniqueUrls.length || 1) }, worker)); + return results; +} + +function embedAssets(snapshot, assets) { + let html = snapshot.html; + const failures = []; + const records = []; + for (const resource of snapshot.resources) { + const asset = assets.get(resource.url); + const marker = `data-wxasset-${resource.kind}="${resource.id}"`; + if (!asset?.ok) { + failures.push({ ...resource, error: asset?.error || '未抓取' }); + records.push({ ...resource, ok: false, error: asset?.error || '未抓取' }); + if (resource.kind === 'img') html = html.replace(marker, `src="${htmlEscape(resource.url)}"`); + else if (resource.kind === 'svg-image') html = html.replace(marker, `href="${htmlEscape(resource.url)}"`); + else if (resource.kind === 'background') { + html = html.replace(`__${resource.id}__`, resource.url.replaceAll('"', '%22')); + html = html.replace(marker, ''); + } + continue; + } + const dataUri = `data:${asset.mime};base64,${asset.buffer.toString('base64')}`; + const bytes = asset.buffer.length; + records.push({ ...resource, ok: true, mime: asset.mime, bytes, source: asset.source }); + if (resource.kind === 'img') { + html = html.replace(marker, `src="${dataUri}"`); + } else if (resource.kind === 'svg-image') { + html = html.replace(marker, `href="${dataUri}"`); + } else if (resource.kind === 'background') { + html = html.replace(`__${resource.id}__`, dataUri); + html = html.replace(marker, ''); + } + } + html = html.replaceAll(/\sdata-wxasset-[\w-]+="[^"]+"/g, ''); + return { html, failures, records }; +} + +function standaloneHtml(snapshot, articleHtml) { + const metadata = [snapshot.author, snapshot.published].filter(Boolean).map(htmlEscape).join('　'); + return ` + + + + + + ${htmlEscape(snapshot.title)} + + + +
+
+

${htmlEscape(snapshot.title)}

+ ${metadata ? `
${metadata}
` : ''} +
来源：${htmlEscape(snapshot.canonical)}
+
+ ${articleHtml} +
+ +`; +} + +function mimeExtension(mime) { + const extensions = { + 'image/jpeg': 'jpg', + 'image/png': 'png', + 'image/gif': 'gif', + 'image/webp': 'webp', + 'image/svg+xml': 'svg', + 'image/avif': 'avif', + }; + return extensions[mime] || 'bin'; +} + +function yamlString(value) { + return JSON.stringify(String(value || '')); +} + +async function writeMarkdown(snapshot, assets, outputDir, base) { + const assetDirectoryName = `${base}.assets`; + const assetDirectory = join(outputDir, assetDirectoryName); + await mkdir(assetDirectory, { recursive: true }); + + const filenames = new Map(); + let assetNumber = 0; + for (const resource of snapshot.resources) { + const asset = assets.get(resource.url); + if (!asset?.ok || filenames.has(resource.url)) continue; + assetNumber += 1; + const filename = `image-${String(assetNumber).padStart(3, '0')}.${mimeExtension(asset.mime)}`; + await writeFile(join(assetDirectory, filename), asset.buffer); + filenames.set(resource.url, `${assetDirectoryName}/${filename}`); + } + + let sourceHtml = snapshot.html; + for (const resource of snapshot.resources) { + const marker = `data-wxasset-${resource.kind}="${resource.id}"`; + const localPath = filenames.get(resource.url); + if (resource.kind === 'img') { + sourceHtml = sourceHtml.replace(marker, localPath ? `src="${htmlEscape(localPath)}"` : ''); + } else if (resource.kind === 'svg-image') { + sourceHtml = sourceHtml.replace(marker, localPath ? `href="${htmlEscape(localPath)}"` : ''); + } else if (resource.kind === 'background') { + sourceHtml = sourceHtml.replace(`__${resource.id}__`, localPath || ''); + sourceHtml = sourceHtml.replace(marker, ''); + } + } + sourceHtml = sourceHtml.replaceAll(/\sdata-wxasset-[\w-]+="[^"]+"/g, ''); + + const turndown = new TurndownService({ + headingStyle: 'atx', + bulletListMarker: '-', + codeBlockStyle: 'fenced', + emDelimiter: '*', + strongDelimiter: '**', + }); + turndown.use(gfm); + turndown.addRule('wechatMediaNote', { + filter: (node) => node.nodeType === 1 && node.classList?.contains('wx-export-media-note'), + replacement: (content) => `\n\n> ${content.trim()}\n\n`, + }); + const body = turndown.turndown(sourceHtml).replace(/\n{3,}/g, '\n\n').trim(); + const markdown = `--- +title: ${yamlString(snapshot.title)} +author: ${yamlString(snapshot.author)} +published: ${yamlString(snapshot.published)} +source: ${yamlString(snapshot.canonical)} +archived_at: ${yamlString(new Date().toISOString())} +--- + +# ${snapshot.title} + +${body} +`; + const markdownPath = join(outputDir, `${base}.md`); + await writeFile(markdownPath, markdown); + return markdownPath; +} + +async function writePdf(browserContext, htmlPath, pdfPath, mode) { + const page = await browserContext.newPage(); + await page.goto(pathToFileURL(htmlPath).href, { waitUntil: 'load' }); + await page.emulateMedia({ media: 'print' }); + await page.evaluate(() => document.fonts?.ready); + + if (mode === 'a4') { + await page.pdf({ path: pdfPath, format: 'A4', printBackground: true, margin: { top: 0, right: 0, bottom: 0, left: 0 } }); + } else { + const heightPx = await page.evaluate(() => Math.max( + document.body.scrollHeight, + document.documentElement.scrollHeight, + document.querySelector('.wx-export-page')?.scrollHeight || 0, + )); + const heightMm = Math.ceil((heightPx * 25.4) / 96) + 2; + if (heightMm > 5000) { + throw new Error(`正文约 ${heightMm}mm，超过 Chromium 单张 PDF 的安全高度 5000mm；请使用 --pdf-mode a4，单页 HTML 已保留`); + } + await page.pdf({ + path: pdfPath, + width: '210mm', + height: `${heightMm}mm`, + printBackground: true, + margin: { top: 0, right: 0, bottom: 0, left: 0 }, + preferCSSPageSize: false, + }); + } + await page.close(); +} + +async function main() { + const { url, options } = parseArgs(process.argv.slice(2)); + await mkdir(options.outputDir, { recursive: true }); + await mkdir(options.profile, { recursive: true }); + + const responseCache = new Map(); + const context = await chromium.launchPersistentContext(options.profile, { + headless: !options.headed, + viewport: { width: 1280, height: 900 }, + locale: 'zh-CN', + timezoneId: 'Asia/Shanghai', + serviceWorkers: 'block', + userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36', + }); + + let exitCode = 0; + try { + const page = context.pages()[0] || await context.newPage(); + page.setDefaultTimeout(options.timeout); + page.on('response', (response) => { + if (response.request().resourceType() !== 'image' || !response.ok()) return; + const promise = response.body() + .then((buffer) => ({ buffer, contentType: response.headers()['content-type'] || '' })) + .catch(() => null); + responseCache.set(response.url(), promise); + }); + + console.error(`打开：${url}`); + await page.goto(url, { waitUntil: 'domcontentloaded', timeout: options.timeout }); + const selector = await waitForArticle(page, options); + console.error(`正文：${selector}；触发懒加载图片…`); + await activateLazyImages(page, selector, options.imageTimeout); + const snapshot = await snapshotArticle(page, selector, options.keepRemoteLinks); + console.error(`发现 ${snapshot.resources.length} 个图片引用，正在抓取并内嵌…`); + + const assets = await fetchAssets(context, snapshot.resources, responseCache, page.url()); + const embedded = embedAssets(snapshot, assets); + const base = safeFilename(snapshot.title); + const manifestPath = join(options.outputDir, `${base}.assets.json`); + const manifest = { + sourceUrl: url, + finalUrl: page.url(), + title: snapshot.title, + articleSelector: selector, + generatedAt: new Date().toISOString(), + discoveredReferences: snapshot.resources.length, + uniqueResources: assets.size, + embeddedReferences: embedded.records.filter((item) => item.ok).length, + failedReferences: embedded.failures.length, + complete: embedded.failures.length === 0, + resources: embedded.records, + }; + await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + + if (embedded.failures.length && !options.allowMissingImages) { + const summary = embedded.failures.slice(0, 5).map((item) => `- ${item.url}: ${item.error}`).join('\n'); + throw new Error(`有 ${embedded.failures.length} 个图片引用抓取失败；已写完整性报告，未生成不完整成品。\n${summary}\n报告：${manifestPath}`); + } + + if (options.format === 'markdown' || options.format === 'all') { + const markdownPath = await writeMarkdown(snapshot, assets, options.outputDir, base); + console.error(`Markdown：${markdownPath}`); + } + + const htmlPath = join(options.outputDir, `${base}.html`); + const html = standaloneHtml(snapshot, embedded.html); + await writeFile(htmlPath, html); + console.error(`HTML：${htmlPath}`); + + if (options.format === 'pdf' || options.format === 'all') { + const pdfPath = join(options.outputDir, `${base}.${options.pdfMode === 'continuous' ? 'continuous' : 'a4'}.pdf`); + await writePdf(context, htmlPath, pdfPath, options.pdfMode); + console.error(`PDF：${pdfPath}`); + } + if (options.format === 'pdf') { + // PDF 以离线 HTML 为中间源；保留它有助于审计图片完整性和重新排版。 + console.error('提示：保留了生成 PDF 所用的自包含 HTML 中间文件。'); + } + console.error(`完整性：${manifest.complete ? '通过' : '允许缺图'}（${manifest.embeddedReferences}/${manifest.discoveredReferences} 个引用已内嵌）`); + } catch (error) { + exitCode = 2; + console.error(`失败：${cleanErrorMessage(error)}`); + } finally { + await context.close(); + } + process.exitCode = exitCode; +} + +if (pathToFileURL(process.argv[1]).href === import.meta.url) { + main().catch((error) => { + console.error(`失败：${cleanErrorMessage(error.stack || error)}`); + process.exitCode = 2; + }); +} + +export { htmlEscape, inferMime, safeFilename }; diff --git a/servers/tools-pve/wechat-fulltext/compose.yaml b/servers/tools-pve/wechat-fulltext/compose.yaml new file mode 100644 index 0000000..88d2926 --- /dev/null +++ b/servers/tools-pve/wechat-fulltext/compose.yaml @@ -0,0 +1,59 @@ +services: + wechat-fulltext: + image: wechat-fulltext:2026-09-18 + pull_policy: never + build: + context: . + dockerfile: Dockerfile + container_name: tools-wechat-fulltext + user: "1000:1000" + restart: unless-stopped + shm_size: 1gb + environment: + HOME: /data/home + PUBLIC_BASE_URL: http://192.168.50.49:8091 + VERIFICATION_URL: http://192.168.50.49:8092/vnc.html?autoconnect=true&resize=scale + MANUAL_WAIT_SECONDS: "3600" + DEFAULT_PROVIDER: mptext + DIRECT_BROWSER_ENABLED: "false" + UPSTREAM_TIMEOUT_SECONDS: "20" + MPTEXT_MAX_ATTEMPTS: "2" + MPTEXT_RETRY_DELAYS_SECONDS: "5" + SECONDARY_UPSTREAM_ENDPOINT: http://wechat-download-upstream:5000/api/article + SECONDARY_UPSTREAM_PUBLIC_URL: http://192.168.50.49:8093/login.html + SECONDARY_UPSTREAM_TIMEOUT_SECONDS: "120" + ports: + - "192.168.50.49:8091:8091" + - "192.168.50.49:8092:6080" + volumes: + - /opt/wechat-fulltext/data/article-service:/data + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:8091/health').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s + + wechat-download-upstream: + image: tmwgsicp/wechat-download-api@sha256:6977f00af2334af4a8dc9ed19d0a82a5214fd4eca4d1c724f7d14b1299cc66ec + container_name: tools-wechat-download-upstream + restart: unless-stopped + environment: + TZ: America/Los_Angeles + SITE_URL: http://192.168.50.49:8093 + RATE_LIMIT_GLOBAL: "20" + RATE_LIMIT_PER_IP: "20" + RATE_LIMIT_ARTICLE_INTERVAL: "1" + RSS_FETCH_FULL_CONTENT: "false" + ENABLE_MCP: "0" + ports: + - "192.168.50.49:8093:5000" + volumes: + - /opt/wechat-fulltext/data/download-api:/app/data + healthcheck: + test: ["CMD", "curl", "-sf", "http://127.0.0.1:5000/api/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 15s + diff --git a/servers/tools-pve/wechat-fulltext/entrypoint.sh b/servers/tools-pve/wechat-fulltext/entrypoint.sh new file mode 100755 index 0000000..3300290 --- /dev/null +++ b/servers/tools-pve/wechat-fulltext/entrypoint.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +mkdir -p "$HOME" +rm -f /tmp/.X99-lock /tmp/.X11-unix/X99 +Xvfb :99 -screen 0 1440x1000x24 -ac +extension RANDR >/tmp/xvfb.log 2>&1 & +sleep 1 +openbox >/tmp/openbox.log 2>&1 & +x11vnc -display :99 -forever -shared -nopw -rfbport 5900 >/tmp/x11vnc.log 2>&1 & +websockify --web=/usr/share/novnc 6080 localhost:5900 >/tmp/novnc.log 2>&1 & + +exec node /app/service.mjs +