From cac16a68e71a4873765cd29d3db55544600da4b5 Mon Sep 17 00:00:00 2001 From: Digidai Date: Wed, 6 May 2026 19:06:09 +0800 Subject: [PATCH] Fix WeChat empty conversion caching --- src/__tests__/adapters-behavior.test.ts | 21 +++++ src/__tests__/cache-edge.test.ts | 25 ++++++ src/__tests__/index-mocked-branches.test.ts | 57 ++++++++++++ src/browser/adapters/wechat.ts | 96 ++++++++++++++++++++- src/cache/index.ts | 3 + src/handlers/convert.ts | 85 +++++++++++++++++- 6 files changed, 283 insertions(+), 4 deletions(-) diff --git a/src/__tests__/adapters-behavior.test.ts b/src/__tests__/adapters-behavior.test.ts index a913698..ebbdda5 100644 --- a/src/__tests__/adapters-behavior.test.ts +++ b/src/__tests__/adapters-behavior.test.ts @@ -116,6 +116,27 @@ describe("adapter behavior", () => { expect(processed).toContain("plain article"); }); + it("wechat postProcess promotes js_content into a focused article document", () => { + const html = `Fallback +

Article Title

+ Author Name +
+

real article body

+ +
+
outside noise
+ `; + + const processed = wechatAdapter.postProcess!(html); + + expect(processed).toContain('data-adapter="wechat"'); + expect(processed).toContain("

Article Title

"); + expect(processed).toContain("作者: Author Name"); + expect(processed).toContain("real article body"); + expect(processed).toContain('src="https://mmbiz.qpic.cn/mmbiz_png/a/640"'); + expect(processed).not.toContain("outside noise"); + }); + it("keeps feishu adapter as no-op extraction", async () => { expect(feishuAdapter.alwaysBrowser).toBe(true); expect(await feishuAdapter.extract({} as any, new Map())).toBeNull(); diff --git a/src/__tests__/cache-edge.test.ts b/src/__tests__/cache-edge.test.ts index 37dac33..033b308 100644 --- a/src/__tests__/cache-edge.test.ts +++ b/src/__tests__/cache-edge.test.ts @@ -74,6 +74,31 @@ describe("cache edge behavior", () => { expect(result).toBeNull(); }); + it("returns null for empty cached content", async () => { + const { env, mocks } = createCacheEnv(); + mocks.kvGet.mockResolvedValueOnce(JSON.stringify({ + content: " ", + method: "readability+turndown", + title: "", + })); + + const result = await getCached(env, "https://example.com/empty-cache", "markdown"); + + expect(result).toBeNull(); + }); + + it("does not store empty cache content", async () => { + const { env, mocks } = createCacheEnv(); + + await setCache(env, "https://example.com/empty-write", "markdown", { + content: "", + method: "readability+turndown", + title: "", + }); + + expect(mocks.kvPut).not.toHaveBeenCalled(); + }); + it("does not use hot cache when ttl is zero", async () => { const { env, mocks } = createCacheEnv(); const url = "https://example.com/no-hot-cache"; diff --git a/src/__tests__/index-mocked-branches.test.ts b/src/__tests__/index-mocked-branches.test.ts index 8fd9efc..7788df5 100644 --- a/src/__tests__/index-mocked-branches.test.ts +++ b/src/__tests__/index-mocked-branches.test.ts @@ -34,6 +34,9 @@ const mocked = vi.hoisted(() => ({ fetchViaProxy: vi.fn(), fetchViaProxyPool: vi.fn(), }, + browserUse: { + fetchViaBrowserUse: vi.fn(), + }, })); vi.mock("cloudflare:sockets", () => ({ @@ -77,6 +80,10 @@ vi.mock("../proxy", () => ({ fetchViaProxyPool: mocked.proxy.fetchViaProxyPool, })); +vi.mock("../browser-use", () => ({ + fetchViaBrowserUse: mocked.browserUse.fetchViaBrowserUse, +})); + import worker from "../index"; import { createProxyRetrySignal } from "../browser/proxy-retry"; @@ -145,6 +152,7 @@ beforeEach(() => { attempts: 1, errors: [], }); + mocked.browserUse.fetchViaBrowserUse.mockResolvedValue(null); }); describe("index mocked branch coverage", () => { @@ -208,6 +216,55 @@ describe("index mocked branch coverage", () => { expect(payload.message).toContain("configure PROXY_URL"); }); + it("does not cache empty output when anonymous always-browser fallbacks return no content", async () => { + mocked.browser.alwaysNeedsBrowser.mockReturnValue(true); + mocked.converter.htmlToMarkdown.mockImplementation((html: string) => ({ + markdown: html.trim() ? "# md body" : "", + title: "", + contentHtml: html, + })); + + const req = new Request("https://md.example.com/https://example.com/needs-browser?raw=true", { + headers: { Accept: "application/json" }, + }); + const res = await worker.fetch(req, createMockEnv().env, mockCtx()); + const payload = await res.json() as { error?: string; message?: string }; + + expect(res.status).toBe(502); + expect(payload.error).toBe("Fetch Failed"); + expect(payload.message).toContain("requires browser or proxy access"); + expect(mocked.cache.setCache).not.toHaveBeenCalled(); + }); + + it("uses static WeChat fetch for anonymous users when remote browser and proxy produce no content", async () => { + mocked.browser.alwaysNeedsBrowser.mockReturnValue(true); + const fetchMock = vi.fn().mockResolvedValue( + new Response("
wechat article
", { + status: 200, + headers: { "Content-Type": "text/html; charset=utf-8" }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + mocked.converter.htmlToMarkdown.mockImplementation((html: string) => ({ + markdown: html.includes("wechat article") ? "# wechat article" : "", + title: "wechat", + contentHtml: "
wechat article
", + })); + + const req = new Request("https://md.example.com/https://mp.weixin.qq.com/s/abc?raw=true", { + headers: { Accept: "text/markdown" }, + }); + const res = await worker.fetch(req, createMockEnv().env, mockCtx()); + + expect(res.status).toBe(200); + expect(await res.text()).toContain("proxied:# wechat article"); + expect(fetchMock).toHaveBeenCalled(); + const init = fetchMock.mock.calls[0]?.[1] as RequestInit; + expect((init.headers as Record)["User-Agent"]).toContain("MicroMessenger"); + expect((init.headers as Record).Referer).toBe("https://mp.weixin.qq.com/"); + expect(res.headers.get("X-Markdown-Fallbacks")).toContain("wechat_static_fallback"); + }); + it("retries through proxy and succeeds after PROXY_RETRY signal", async () => { mocked.browser.alwaysNeedsBrowser.mockReturnValue(true); mocked.browser.fetchWithBrowser.mockRejectedValueOnce( diff --git a/src/browser/adapters/wechat.ts b/src/browser/adapters/wechat.ts index 9b96a26..0969fdf 100644 --- a/src/browser/adapters/wechat.ts +++ b/src/browser/adapters/wechat.ts @@ -2,6 +2,64 @@ import type { SiteAdapter, ExtractResult } from "../../types"; import { WECHAT_UA } from "../../config"; import { STEALTH_SCRIPT } from "../stealth"; import { createProxyRetrySignal } from "../proxy-retry"; +import { parseHTML } from "linkedom"; + +function escapeHtml(value: string): string { + return value.replace(/[&<>"']/g, (ch) => { + switch (ch) { + case "&": return "&"; + case "<": return "<"; + case ">": return ">"; + case '"': return """; + default: return "'"; + } + }); +} + +function normalizeText(value: string | null | undefined): string { + return (value || "").replace(/\s+/g, " ").trim(); +} + +function extractWechatArticleHtml(html: string): string | null { + if (!html.includes("js_content")) return null; + + try { + const { document } = parseHTML(html); + const content = document.querySelector("#js_content") as any; + if (!content) return null; + + content.querySelectorAll?.("img[data-src]").forEach((img: any) => { + const real = img.getAttribute("data-src"); + if (real) img.setAttribute("src", real); + }); + + const title = normalizeText( + (document.querySelector("#activity-name") as any)?.textContent || + (document.querySelector("meta[property='og:title']") as any)?.getAttribute?.("content") || + document.title, + ); + const author = normalizeText((document.querySelector("#js_name") as any)?.textContent); + const publishTime = normalizeText( + (document.querySelector("[data-wechat-meta='publish_time']") as any)?.textContent, + ); + + const metaParts: string[] = []; + if (author) metaParts.push(`

作者: ${escapeHtml(author)}

`); + if (publishTime) metaParts.push(`

${escapeHtml(publishTime)}

`); + + return [ + "", + `${escapeHtml(title)}`, + "
", + title ? `

${escapeHtml(title)}

` : "", + ...metaParts, + content.outerHTML || content.innerHTML || "", + "
", + ].join(""); + } catch { + return null; + } +} export const wechatAdapter: SiteAdapter = { match(url: string): boolean { @@ -97,6 +155,42 @@ export const wechatAdapter: SiteAdapter = { }); })() `); + + const articleHtml = await page.evaluate(` + (function() { + function esc(s) { + return String(s || '').replace(/[&<>"']/g, function(ch) { + return ({'&':'&','<':'<','>':'>','"':'"',"'":'''})[ch]; + }); + } + function txt(sel) { + var el = document.querySelector(sel); + return el ? (el.textContent || '').replace(/\\s+/g, ' ').trim() : ''; + } + var content = document.getElementById('js_content'); + if (!content) return ''; + content.querySelectorAll('img[data-src]').forEach(function(img) { + var real = img.getAttribute('data-src'); + if (real) img.setAttribute('src', real); + }); + var title = txt('#activity-name') || document.title || ''; + var author = txt('#js_name'); + var publish = txt('[data-wechat-meta="publish_time"]'); + var parts = [ + '' + esc(title) + '
' + ]; + if (title) parts.push('

' + esc(title) + '

'); + if (author) parts.push('

作者: ' + esc(author) + '

'); + if (publish) parts.push('

' + esc(publish) + '

'); + parts.push(content.outerHTML || content.innerHTML || ''); + parts.push('
'); + return parts.join(''); + })() + `); + if (articleHtml && articleHtml.length > 200) { + return { html: articleHtml }; + } + const html = await page.content(); return { html }; }, @@ -150,6 +244,6 @@ export const wechatAdapter: SiteAdapter = { }, ); - return result; + return extractWechatArticleHtml(result) || result; }, }; diff --git a/src/cache/index.ts b/src/cache/index.ts index a8e99de..f9a431b 100644 --- a/src/cache/index.ts +++ b/src/cache/index.ts @@ -126,6 +126,7 @@ function parseCachedPayload(raw: string): CachedPayload | null { if ( !parsed || typeof parsed.content !== "string" || + parsed.content.trim().length === 0 || typeof parsed.method !== "string" || typeof parsed.title !== "string" ) { @@ -311,6 +312,8 @@ export async function setCache( engine?: string, ): Promise { try { + if (!data.content.trim()) return; + const key = await cacheKey(url, format, selector, engine); const effectiveTtl = ttl ?? getTtlForUrl(url); diff --git a/src/handlers/convert.ts b/src/handlers/convert.ts index 9066387..24dcc05 100644 --- a/src/handlers/convert.ts +++ b/src/handlers/convert.ts @@ -199,6 +199,7 @@ export function asFetchConvertError(error: unknown): ConvertError { export function isLikelyChallengeHtml(body: string): boolean { const lower = body.toLowerCase(); return ( + isWechatVerificationHtml(body) || (lower.includes("passport.weibo") || lower.includes("qrcode_login") || lower.includes("login_type")) || @@ -213,6 +214,14 @@ export function isLikelyChallengeHtml(body: string): boolean { ); } +export function isWechatVerificationHtml(body: string): boolean { + return ( + body.includes("wappoc_appmsgcaptcha") || + body.includes("当前环境异常") || + body.includes("完成验证后即可继续访问") + ); +} + // ─── CF REST API helpers ───────────────────────────────────── export function getCfRestConfig(env: Env): CfRestConfig | null { @@ -499,7 +508,9 @@ async function tryFetchAndParse( } // 早期浏览器路径 — 对总是需要浏览器的站点跳过多余的静态获取 - if (!finalHtml && alwaysNeedsBrowser(targetUrl) && browserAllowed) { + const requiresBrowser = alwaysNeedsBrowser(targetUrl); + + if (!finalHtml && requiresBrowser && browserAllowed) { const result = await tryBrowserRendering( targetUrl, env, host, fallbacks, abortSignal, progress, ); @@ -508,7 +519,7 @@ async function tryFetchAndParse( method = "browser+readability+turndown"; browserRendered = true; } - } else if (!finalHtml && alwaysNeedsBrowser(targetUrl) && !browserAllowed) { + } else if (!finalHtml && requiresBrowser && !browserAllowed) { // Anonymous 用户无 browser 权限 — 先走 Browser Use 远程浏览器，再降级住宅代理 if (env.BROWSER_USE_API_KEY) { throwIfAborted(abortSignal); @@ -529,6 +540,36 @@ async function tryFetchAndParse( method = "proxy+readability+turndown"; } } + + // WeChat often serves usable article HTML to a MicroMessenger UA. Use this + // as a low-cost fallback for anonymous users before failing the request. + if (!finalHtml && targetUrl.includes("mp.weixin.qq.com")) { + const staticResult = await tryStaticFetch( + targetUrl, env, host, format, selector, forceBrowser, noCache, engine, + fallbacks, browserRendered, paywallDetected, sourceContentType, + resolvedUrl, method, progress, abortSignal, browserAllowed, + ); + if (staticResult.earlyReturn) { + return staticResult.earlyReturn; + } + finalHtml = staticResult.finalHtml; + method = staticResult.method; + resolvedUrl = staticResult.resolvedUrl; + browserRendered = staticResult.browserRendered; + paywallDetected = staticResult.paywallDetected; + sourceContentType = staticResult.sourceContentType; + if (finalHtml) { + fallbacks.add("wechat_static_fallback"); + } + } + + if (!finalHtml) { + throw new ConvertError( + "Fetch Failed", + "This URL requires browser or proxy access, but no content was returned by the available fallback methods.", + 502, + ); + } } else if (!finalHtml) { // 3. 静态获取 const staticResult = await tryStaticFetch( @@ -549,6 +590,21 @@ async function tryFetchAndParse( // 7. 去除