Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions src/__tests__/adapters-behavior.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,27 @@ describe("adapter behavior", () => {
expect(processed).toContain("plain article");
});

it("wechat postProcess promotes js_content into a focused article document", () => {
const html = `<html><head><title>Fallback</title></head><body>
<h1 id="activity-name">Article Title</h1>
<span id="js_name">Author Name</span>
<div id="js_content">
<p>real article body</p>
<img data-src="https://mmbiz.qpic.cn/mmbiz_png/a/640" />
</div>
<div class="share-widget">outside noise</div>
</body></html>`;

const processed = wechatAdapter.postProcess!(html);

expect(processed).toContain('data-adapter="wechat"');
expect(processed).toContain("<h1>Article Title</h1>");
expect(processed).toContain("作者: Author Name");
expect(processed).toContain("real article body");
expect(processed).toContain('src="https://mmbiz.qpic.cn/mmbiz_png/a/640"');
expect(processed).not.toContain("outside noise");
});

it("keeps feishu adapter as no-op extraction", async () => {
expect(feishuAdapter.alwaysBrowser).toBe(true);
expect(await feishuAdapter.extract({} as any, new Map())).toBeNull();
Expand Down
25 changes: 25 additions & 0 deletions src/__tests__/cache-edge.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,31 @@ describe("cache edge behavior", () => {
expect(result).toBeNull();
});

it("returns null for empty cached content", async () => {
const { env, mocks } = createCacheEnv();
mocks.kvGet.mockResolvedValueOnce(JSON.stringify({
content: " ",
method: "readability+turndown",
title: "",
}));

const result = await getCached(env, "https://example.com/empty-cache", "markdown");

expect(result).toBeNull();
});

it("does not store empty cache content", async () => {
const { env, mocks } = createCacheEnv();

await setCache(env, "https://example.com/empty-write", "markdown", {
content: "",
method: "readability+turndown",
title: "",
});

expect(mocks.kvPut).not.toHaveBeenCalled();
});

it("does not use hot cache when ttl is zero", async () => {
const { env, mocks } = createCacheEnv();
const url = "https://example.com/no-hot-cache";
Expand Down
57 changes: 57 additions & 0 deletions src/__tests__/index-mocked-branches.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ const mocked = vi.hoisted(() => ({
fetchViaProxy: vi.fn(),
fetchViaProxyPool: vi.fn(),
},
browserUse: {
fetchViaBrowserUse: vi.fn(),
},
}));

vi.mock("cloudflare:sockets", () => ({
Expand Down Expand Up @@ -77,6 +80,10 @@ vi.mock("../proxy", () => ({
fetchViaProxyPool: mocked.proxy.fetchViaProxyPool,
}));

vi.mock("../browser-use", () => ({
fetchViaBrowserUse: mocked.browserUse.fetchViaBrowserUse,
}));

import worker from "../index";
import { createProxyRetrySignal } from "../browser/proxy-retry";

Expand Down Expand Up @@ -145,6 +152,7 @@ beforeEach(() => {
attempts: 1,
errors: [],
});
mocked.browserUse.fetchViaBrowserUse.mockResolvedValue(null);
});

describe("index mocked branch coverage", () => {
Expand Down Expand Up @@ -208,6 +216,55 @@ describe("index mocked branch coverage", () => {
expect(payload.message).toContain("configure PROXY_URL");
});

it("does not cache empty output when anonymous always-browser fallbacks return no content", async () => {
mocked.browser.alwaysNeedsBrowser.mockReturnValue(true);
mocked.converter.htmlToMarkdown.mockImplementation((html: string) => ({
markdown: html.trim() ? "# md body" : "",
title: "",
contentHtml: html,
}));

const req = new Request("https://md.example.com/https://example.com/needs-browser?raw=true", {
headers: { Accept: "application/json" },
});
const res = await worker.fetch(req, createMockEnv().env, mockCtx());
const payload = await res.json() as { error?: string; message?: string };

expect(res.status).toBe(502);
expect(payload.error).toBe("Fetch Failed");
expect(payload.message).toContain("requires browser or proxy access");
expect(mocked.cache.setCache).not.toHaveBeenCalled();
});

it("uses static WeChat fetch for anonymous users when remote browser and proxy produce no content", async () => {
mocked.browser.alwaysNeedsBrowser.mockReturnValue(true);
const fetchMock = vi.fn().mockResolvedValue(
new Response("<html><body><div id=\"js_content\">wechat article</div></body></html>", {
status: 200,
headers: { "Content-Type": "text/html; charset=utf-8" },
}),
);
vi.stubGlobal("fetch", fetchMock);
mocked.converter.htmlToMarkdown.mockImplementation((html: string) => ({
markdown: html.includes("wechat article") ? "# wechat article" : "",
title: "wechat",
contentHtml: "<article>wechat article</article>",
}));

const req = new Request("https://md.example.com/https://mp.weixin.qq.com/s/abc?raw=true", {
headers: { Accept: "text/markdown" },
});
const res = await worker.fetch(req, createMockEnv().env, mockCtx());

expect(res.status).toBe(200);
expect(await res.text()).toContain("proxied:# wechat article");
expect(fetchMock).toHaveBeenCalled();
const init = fetchMock.mock.calls[0]?.[1] as RequestInit;
expect((init.headers as Record<string, string>)["User-Agent"]).toContain("MicroMessenger");
expect((init.headers as Record<string, string>).Referer).toBe("https://mp.weixin.qq.com/");
expect(res.headers.get("X-Markdown-Fallbacks")).toContain("wechat_static_fallback");
});

it("retries through proxy and succeeds after PROXY_RETRY signal", async () => {
mocked.browser.alwaysNeedsBrowser.mockReturnValue(true);
mocked.browser.fetchWithBrowser.mockRejectedValueOnce(
Expand Down
96 changes: 95 additions & 1 deletion src/browser/adapters/wechat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,64 @@ import type { SiteAdapter, ExtractResult } from "../../types";
import { WECHAT_UA } from "../../config";
import { STEALTH_SCRIPT } from "../stealth";
import { createProxyRetrySignal } from "../proxy-retry";
import { parseHTML } from "linkedom";

function escapeHtml(value: string): string {
return value.replace(/[&<>"']/g, (ch) => {
switch (ch) {
case "&": return "&amp;";
case "<": return "&lt;";
case ">": return "&gt;";
case '"': return "&quot;";
default: return "&#039;";
}
});
}

function normalizeText(value: string | null | undefined): string {
return (value || "").replace(/\s+/g, " ").trim();
}

function extractWechatArticleHtml(html: string): string | null {
if (!html.includes("js_content")) return null;

try {
const { document } = parseHTML(html);
const content = document.querySelector("#js_content") as any;
if (!content) return null;

content.querySelectorAll?.("img[data-src]").forEach((img: any) => {
const real = img.getAttribute("data-src");
if (real) img.setAttribute("src", real);
});

const title = normalizeText(
(document.querySelector("#activity-name") as any)?.textContent ||
(document.querySelector("meta[property='og:title']") as any)?.getAttribute?.("content") ||
document.title,
);
const author = normalizeText((document.querySelector("#js_name") as any)?.textContent);
const publishTime = normalizeText(
(document.querySelector("[data-wechat-meta='publish_time']") as any)?.textContent,
);

const metaParts: string[] = [];
if (author) metaParts.push(`<p data-wechat-meta="author">作者: ${escapeHtml(author)}</p>`);
if (publishTime) metaParts.push(`<p data-wechat-meta="publish_time">${escapeHtml(publishTime)}</p>`);

return [
"<html><head>",
`<title>${escapeHtml(title)}</title>`,
"</head><body><article data-adapter=\"wechat\">",
title ? `<h1>${escapeHtml(title)}</h1>` : "",
...metaParts,
content.outerHTML || content.innerHTML || "",
"</article></body></html>",
].join("");
} catch {
return null;
}
}

export const wechatAdapter: SiteAdapter = {
match(url: string): boolean {
Expand Down Expand Up @@ -97,6 +155,42 @@ export const wechatAdapter: SiteAdapter = {
});
})()
`);

const articleHtml = await page.evaluate(`
(function() {
function esc(s) {
return String(s || '').replace(/[&<>"']/g, function(ch) {
return ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#039;'})[ch];
});
}
function txt(sel) {
var el = document.querySelector(sel);
return el ? (el.textContent || '').replace(/\\s+/g, ' ').trim() : '';
}
var content = document.getElementById('js_content');
if (!content) return '';
content.querySelectorAll('img[data-src]').forEach(function(img) {
var real = img.getAttribute('data-src');
if (real) img.setAttribute('src', real);
});
var title = txt('#activity-name') || document.title || '';
var author = txt('#js_name');
var publish = txt('[data-wechat-meta="publish_time"]');
var parts = [
'<html><head><title>' + esc(title) + '</title></head><body><article data-adapter="wechat">'
];
if (title) parts.push('<h1>' + esc(title) + '</h1>');
if (author) parts.push('<p data-wechat-meta="author">作者: ' + esc(author) + '</p>');
if (publish) parts.push('<p data-wechat-meta="publish_time">' + esc(publish) + '</p>');
parts.push(content.outerHTML || content.innerHTML || '');
parts.push('</article></body></html>');
return parts.join('');
})()
`);
if (articleHtml && articleHtml.length > 200) {
return { html: articleHtml };
}

const html = await page.content();
return { html };
},
Expand Down Expand Up @@ -150,6 +244,6 @@ export const wechatAdapter: SiteAdapter = {
},
);

return result;
return extractWechatArticleHtml(result) || result;
},
};
3 changes: 3 additions & 0 deletions src/cache/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ function parseCachedPayload(raw: string): CachedPayload | null {
if (
!parsed ||
typeof parsed.content !== "string" ||
parsed.content.trim().length === 0 ||
typeof parsed.method !== "string" ||
typeof parsed.title !== "string"
) {
Expand Down Expand Up @@ -311,6 +312,8 @@ export async function setCache(
engine?: string,
): Promise<void> {
try {
if (!data.content.trim()) return;

const key = await cacheKey(url, format, selector, engine);
const effectiveTtl = ttl ?? getTtlForUrl(url);

Expand Down
Loading
Loading