From f0b1f1a866babf18800b97d7e52ca0865dd611db Mon Sep 17 00:00:00 2001 From: neil Date: Wed, 2 Sep 2026 23:13:53 +0800 Subject: [PATCH 1/2] feat(fetch): unify navigation mode across adapters --- CONTEXT.md | 13 +++--- README.md | 30 +++++++++---- docs/adr/0001-containerized-rest-api.md | 2 +- docs/http-service.md | 26 ++++++------ packages/dsh-web/README.md | 16 +++++-- packages/dsh-web/src/client.ts | 11 +++-- packages/dsh-web/src/tools.ts | 37 +++++++++------- packages/dsh-web/test/artifact.test.ts | 4 +- packages/dsh-web/test/client.test.ts | 9 +++- packages/dsh-web/test/tools.test.ts | 24 +++++++++++ packages/pi-web/src/tool.ts | 56 +++++++++++++++---------- packages/pi-web/test/extension.test.ts | 48 +++++++++++++++++++-- packages/pi-web/test/packed-smoke.mjs | 4 +- packages/web-core/src/fetch.ts | 29 ++++++++----- packages/web-core/src/index.ts | 2 + packages/web-core/src/markdown.ts | 53 +++++++++++++++++------ packages/web-core/test/fetch.test.ts | 42 ++++++++++++++----- packages/web-core/test/markdown.test.ts | 50 ++++++++++++++++++---- packages/web/src/http.ts | 42 +++++++++++++++---- packages/web/src/mcp.ts | 37 ++++++++++------ packages/web/src/program.ts | 34 +++++++++++++-- packages/web/src/runner.ts | 13 ++++++ packages/web/test/http.test.ts | 55 +++++++++++++++++++++++- packages/web/test/mcp.test.ts | 36 ++++++++++++++++ packages/web/test/packed-smoke.mjs | 7 ++-- packages/web/test/program.test.ts | 53 +++++++++++++++++++++-- 26 files changed, 583 insertions(+), 150 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 5cd9d1c..e724cee 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -39,10 +39,13 @@ _Avoid_: Backend-specific renderer labels, automatic fallback **Page Navigation**: The shared page-reading module owns its fixed 5,000-character policy. A long, unsectioned request with navigable headings returns a navigation tree; a -headingless long document uses the normal bounded response. `full: true` -returns complete Markdown, and a tree's `section_id` retrieves one section. -`full: true` and `section_id` are mutually exclusive. -_Avoid_: Caller-selected tree thresholds, public `tree` controls +headingless long document uses the normal bounded response. The request +`mode` is `auto` by default and may be `full`, `tree`, or `section`; `auto` is +input-only because results report the actual mode. `mode: "full"` returns +complete Markdown, `mode: "tree"` forces the heading tree, and +`mode: "section"` with a returned `section_id` retrieves one section. +`section_id` is valid exactly for `mode: "section"`. +_Avoid_: Caller-selected tree thresholds, legacy navigation booleans **Release Contract**: The versioned public distribution of Guion Web: its npm packages, GHCR container image, and the generated `openapi.yaml` attached to the matching GitHub Release. @@ -53,6 +56,6 @@ The Hono-based `/v1` JSON API shipped by `web serve` and the GHCR image. It uses server-local credentials, Bridge Route, and optional DeepSeek provider configuration; clients do not select providers or submit a generic Bridge command. Its page-reading routes use -the same `render: "http" | "browser"`, `full`, and `section_id` contract; the +the same `render: "http" | "browser"`, `mode`, and `section_id` contract; the browser executable name appears only in operator setup. _Avoid_: Remote MCP, public service diff --git a/README.md b/README.md index 71bce84..8986838 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ are generated into `openapi.yaml` from the same route definitions: | Route | Request | Purpose | | ------------ | --------------------------------------------------------- | --------------------------------------------------------------- | | `/v1/search` | `{ "query": "..." }` | Server-selected search: Bridge→Exa by default, or DeepSeek only | -| `/v1/fetch` | `{ "url", "section_id?", "full?", "render?", "waitMs?" }` | Fetch Markdown | +| `/v1/fetch` | `{ "url", "mode?", "section_id?", "render?", "waitMs?" }` | Fetch Markdown | | `/v1/links` | `{ "url", "limit?", "render?", "waitMs?" }` | List page HTTP(S) links | The complete human-readable contract is in the [HTTP service reference](docs/http-service.md). @@ -121,7 +121,9 @@ web search --provider exa -- "Node AbortSignal" web search --provider deepseek -- "Node AbortSignal" web search --provider kepos-bridge -- "Node AbortSignal" web fetch https://example.com/article -web fetch https://example.com/article --section introduction +web fetch https://example.com/article --mode section --section introduction +web fetch https://example.com/article --mode tree +web fetch https://example.com/article --mode full web links https://example.com/article --limit 50 web docs resolve react web docs fetch /facebook/react --topic hooks --tokens 2000 @@ -129,12 +131,16 @@ web sgraph --count 10 -- "repo:^github\\.com/nodejs/node$ AbortSignal" ``` Use `--` before a search or Sourcegraph query that begins with a hyphen. `fetch` -supports `--full` and `--section`; long extracted documents with navigable -headings automatically return a heading tree so a later request can retrieve a -stable `section_id`. A headingless long document uses the normal bounded -response. `--full` returns the complete extracted Markdown, and `--full` cannot -be combined with `--section`. `links` lists up to 100 unique HTTP(S) anchors from the original -page DOM. +supports `--mode auto|full|tree|section`; omitted mode means `auto`. `--section` +is required only with `--mode section`, and is rejected with every other mode. +Long extracted documents with navigable headings automatically return a heading +tree so a later request can retrieve a stable `section_id`. `mode: "auto"` is +request-only; Fetch results report the actual `mode` as `"full"`, `"tree"`, or +`"section"`. A headingless long document uses the normal bounded response. +`mode: "full"` returns the complete extracted Markdown, while `mode: "tree"` +always returns the heading-tree representation, including the explicit +no-headings result. `links` lists up to 100 unique HTTP(S) anchors from the +original page DOM. ## MCP @@ -153,6 +159,8 @@ The server exposes six read-only tools: `search`, `fetch`, `links`, `docs_resolv messages; diagnostics go to stderr. For a client-rendered page, explicitly call `fetch` or `links` with `render: "browser"` and an integer `waitMs`; this optional retry requires a host-installed executable and never happens automatically. +The `fetch` tool uses `mode: "auto" | "full" | "tree" | "section"`; pass a +returned `section_id` only with `mode: "section"`. `auto` is request-only. ## Pi @@ -167,6 +175,9 @@ the bundled core in-process. Pi and TypeBox are peer dependencies supplied by the host; no CLI executable or MCP configuration is required. `web_fetch` uses HTTP rendering by default and can explicitly use `render: "browser"` with an integer `waitMs` when its host provides that optional executable. +Its navigation input is `mode: "auto" | "full" | "tree" | "section"`; +`section_id` is required exactly for `mode: "section"`, and `auto` is +request-only because results report the resolved mode. `web_links` uses the same explicit rendering contract and lists HTTP(S) anchors from the original page DOM. @@ -196,6 +207,9 @@ also run in-process. The host DSH packages and React are peers supplied by DSH. `web_fetch` uses HTTP rendering by default and can explicitly use `render: "browser"` with an integer `waitMs` on a host that supplies the optional executable. +Its navigation input uses the same `mode` and `section_id` contract as the +other adapters; `auto` is request-only and Fetch results report `full`, `tree`, +or `section`. `web_links` uses the same explicit rendering contract and lists HTTP(S) anchors from the original page DOM. diff --git a/docs/adr/0001-containerized-rest-api.md b/docs/adr/0001-containerized-rest-api.md index 01f7480..c15605f 100644 --- a/docs/adr/0001-containerized-rest-api.md +++ b/docs/adr/0001-containerized-rest-api.md @@ -4,4 +4,4 @@ Guion Web will add a self-hosted, single-user HTTP service in a portable contain ## Consequences -The service's provider credentials, Bridge Route, and optional `WEB_SEARCH_PROVIDER=deepseek` selection are server-local configuration; clients cannot choose a provider or supply a Bridge Route per request. With no provider selection, search tries Kepos Bridge and uses Exa only when the Bridge is operationally unavailable; it does not fall back for cancellation, malformed client input, or an empty result set. A failed Bridge attempt is retried through Exa exactly once, and a successful empty Bridge response is returned unchanged. When DeepSeek is selected, the service calls only DeepSeek and never falls back. DeepSeek performs one auxiliary model call internally and returns the same normalized result contract; its Messages/tool protocol is not exposed to HTTP callers. Typed Bridge operations are intentionally not exposed: Exa has no equivalent official weather, sports, or time API, and its premium finance integration is not contract-compatible. Fetch and Links use `render: "http"` by default or explicit `render: "browser"` with a required `waitMs`; the operator-installed executable remains an implementation detail. The shared page-reading module owns automatic navigation trees, complete `full` extraction, and `section_id` continuation. `openapi.yaml` is generated from the release build rather than manually maintained or independently versioned; the standalone [HTTP service reference](../http-service.md) is its human-readable companion. Public or multi-tenant deployment hardening is deliberately deferred in `.scratch/defered/public-http-service-security.md`. +The service's provider credentials, Bridge Route, and optional `WEB_SEARCH_PROVIDER=deepseek` selection are server-local configuration; clients cannot choose a provider or supply a Bridge Route per request. With no provider selection, search tries Kepos Bridge and uses Exa only when the Bridge is operationally unavailable; it does not fall back for cancellation, malformed client input, or an empty result set. A failed Bridge attempt is retried through Exa exactly once, and a successful empty Bridge response is returned unchanged. When DeepSeek is selected, the service calls only DeepSeek and never falls back. DeepSeek performs one auxiliary model call internally and returns the same normalized result contract; its Messages/tool protocol is not exposed to HTTP callers. Typed Bridge operations are intentionally not exposed: Exa has no equivalent official weather, sports, or time API, and its premium finance integration is not contract-compatible. Fetch and Links use `render: "http"` by default or explicit `render: "browser"` with a required `waitMs`; the operator-installed executable remains an implementation detail. Fetch navigation uses `mode: "auto"` by default, with explicit `"full"`, `"tree"`, or `"section"` modes; `section_id` is valid only for `"section"`. `openapi.yaml` is generated from the release build rather than manually maintained or independently versioned; the standalone [HTTP service reference](../http-service.md) is its human-readable companion. Public or multi-tenant deployment hardening is deliberately deferred in `.scratch/defered/public-http-service-security.md`. diff --git a/docs/http-service.md b/docs/http-service.md index 4735d66..41c247a 100644 --- a/docs/http-service.md +++ b/docs/http-service.md @@ -55,7 +55,7 @@ Request: { "url": "https://example.test/article", "render": "http", - "full": false, + "mode": "section", "section_id": "7i" } ``` @@ -63,9 +63,10 @@ Request: `url` must be an absolute `http:` or `https:` URL. `render` is optional and must be exactly `"http"` or `"browser"`; omission selects `"http"`. `waitMs` is an integer from 0 through 30,000, required with `render: "browser"` and -forbidden with `render: "http"` (or when `render` is omitted). `full` is an -optional boolean. `section_id` is an optional non-empty string returned in a -navigation tree. `full: true` and `section_id` cannot be sent together. +forbidden with `render: "http"` (or when `render` is omitted). `mode` is +optional and defaults to `"auto"`; it must be one of `"auto"`, `"full"`, +`"tree"`, or `"section"`. `section_id` is required exactly when `mode` is +`"section"` and is rejected for every other mode. HTTP rendering fetches the page with Node HTTP, linkedom, and Defuddle. Browser rendering invokes the operator-installed `agent-browser` executable @@ -73,14 +74,15 @@ through the isolated renderer implementation; the executable name is not a public request value. Browser rendering is never selected automatically, and the service does not fall back between renderers. -The shared module owns the 5,000-character automatic-tree policy. A non-full, -unsectioned document longer than that threshold with navigable headings returns -`mode: "tree"` with stable section IDs. Use one of those IDs in a subsequent -request to retrieve a section. A long document without headings uses the normal -bounded `mode: "full"` response because it has no section to navigate. The -`full: true` option returns the complete extracted Markdown without the Core -content limit. A short result uses `mode: "full"`; a section request uses -`mode: "section"`. +The shared module owns the 5,000-character automatic-tree policy. An `"auto"` +request for an unsectioned document longer than that threshold with navigable +headings returns `mode: "tree"` with stable section IDs. Use one of those IDs +in a subsequent request with `mode: "section"` to retrieve a section. A long +document without headings uses the normal bounded `mode: "full"` response +because it has no section to navigate. `mode: "full"` returns the complete +extracted Markdown without the Core content limit. `mode: "tree"` always +returns the heading-tree representation, including the explicit no-headings +result. A short `"auto"` request uses `mode: "full"`; `"auto"` is input-only. Response `200`: diff --git a/packages/dsh-web/README.md b/packages/dsh-web/README.md index 95c97c2..239eecc 100644 --- a/packages/dsh-web/README.md +++ b/packages/dsh-web/README.md @@ -48,9 +48,19 @@ It uses the same HTTP default and explicit `render: "browser"` / required `waitMs` contract as `web_fetch`. Long `web_fetch` documents with navigable headings return a navigation tree -automatically. A headingless long document uses the normal bounded response. -Use `full: true` for complete Markdown or pass a returned `section_id` to -continue with one section; those fields are mutually exclusive. +automatically when `mode: "auto"` (the default). A headingless long document +uses the normal bounded response. Set `mode: "full"` for complete Markdown, +`mode: "tree"` to force the heading tree, or `mode: "section"` with a returned +`section_id` to continue with one section. `section_id` is required exactly for +`mode: "section"`; `"auto"` is request-only because results report the actual +`mode` as `"full"`, `"tree"`, or `"section"`. + +For example, request a tree and then continue with one returned section: + +```json +{ "url": "https://example.test/article", "mode": "tree" } +{ "url": "https://example.test/article", "mode": "section", "section_id": "7i" } +``` Rendered requests are bounded and constrained to the requested hostname, `*.` (the target and its subdomains), and this fixed common diff --git a/packages/dsh-web/src/client.ts b/packages/dsh-web/src/client.ts index 80a91dd..b5cb0f4 100644 --- a/packages/dsh-web/src/client.ts +++ b/packages/dsh-web/src/client.ts @@ -312,10 +312,15 @@ function formatWait(waitMs: number): string { } function fetchResultMode(args: Record): string { - if (typeof args.section_id === "string" && args.section_id !== "") + if ( + args.mode === "section" && + typeof args.section_id === "string" && + args.section_id !== "" + ) return `Section: ${args.section_id}`; - if (args.full === true) return "Full document"; - return "Adaptive document"; + if (args.mode === "full") return "Full document"; + if (args.mode === "tree") return "Heading tree"; + return "Automatic navigation"; } function excerpt(output: string): string { diff --git a/packages/dsh-web/src/tools.ts b/packages/dsh-web/src/tools.ts index 2c5acc5..a9f1bd3 100644 --- a/packages/dsh-web/src/tools.ts +++ b/packages/dsh-web/src/tools.ts @@ -1,6 +1,7 @@ import { createWebOperations, DEFAULT_LINK_LIMIT, + FETCH_MODES, normalizeDocsToolInput, formatSize, MAX_LINK_LIMIT, @@ -10,6 +11,7 @@ import { type DocsResolveResult, type DocsToolInput, type FetchInput, + type FetchMode, type FetchResult, type LinksInput, type LinksResult, @@ -42,13 +44,15 @@ const fetchParameters = { required: true, description: "HTTP or HTTPS URL to fetch", }, - section_id: { + mode: { type: "string", - description: "Optional heading section ID to return", + enum: [...FETCH_MODES], + default: "auto", + description: "Navigation mode: auto (default), full, tree, or section", }, - full: { - type: "boolean", - description: "Return full content without automatic tree mode", + section_id: { + type: "string", + description: "Heading section ID; required only with mode section", }, render: { type: "string", @@ -212,7 +216,7 @@ const fetchOutput = { type: "text" as const, text: boundedToolText( value.content, - "Use web_fetch with full: true or a returned section_id to navigate the document.", + 'Use web_fetch with mode: "full" for the complete document, or mode: "section" with a returned section_id to navigate the document.', ), }, ], @@ -382,26 +386,31 @@ function normalizeFetch(input: unknown): FetchInput { if (!isRecord(input)) throw new Error("web_fetch input must be an object"); rejectUnknownFields( input, - ["url", "section_id", "full", "render", "waitMs"], + ["url", "mode", "section_id", "render", "waitMs"], "web_fetch", ); const url = requireString(input, "url"); + const mode = input.mode; + if (mode !== undefined && !FETCH_MODES.includes(mode as FetchMode)) + throw new Error('mode must be one of "auto", "full", "tree", or "section"'); + const selectedMode = mode as FetchMode | undefined; const sectionID = input.section_id; if ( sectionID !== undefined && (typeof sectionID !== "string" || sectionID.trim().length === 0) ) throw new Error("section_id must be a non-empty string"); - const full = input.full; - if (full !== undefined && typeof full !== "boolean") - throw new Error("full must be a boolean"); - if (full === true && sectionID !== undefined) - throw new Error("full and section_id cannot be used together"); + if (selectedMode === "section") { + if (sectionID === undefined) + throw new Error('section_id is required when mode is "section"'); + } else if (sectionID !== undefined) { + throw new Error('section_id is only valid with mode "section"'); + } const renderOptions = validateRenderOptions(input); return { url, + ...(selectedMode === undefined ? {} : { mode: selectedMode }), ...(sectionID === undefined ? {} : { section_id: sectionID }), - ...(full === undefined ? {} : { full }), ...renderOptions, }; } @@ -489,7 +498,7 @@ function webFetchTool( defineTool({ name: "web_fetch", description: - "Use HTTP rendering for static, SSR, and pre-rendered pages. For client-rendered or SPA pages, set render: browser with required waitMs when the host provides browser capability; there is no automatic fallback.", + "Use HTTP rendering for static, SSR, and pre-rendered pages. mode selects auto, full, tree, or section navigation; mode section requires section_id. For client-rendered or SPA pages, set render: browser with required waitMs when the host provides browser capability; there is no automatic fallback.", parameters: fetchParameters, output: fetchOutput, isConcurrencySafe: () => true, diff --git a/packages/dsh-web/test/artifact.test.ts b/packages/dsh-web/test/artifact.test.ts index 7f6c481..b950308 100644 --- a/packages/dsh-web/test/artifact.test.ts +++ b/packages/dsh-web/test/artifact.test.ts @@ -293,7 +293,7 @@ describe("DSH 0.1.2-alpha.3 packed package contract", () => { throw new Error("packed DSH artifact did not register web_links"); process.env.PATH = `${browser.bin}:${originalPath ?? ""}`; const direct = await fetchTool.execute( - { url: "https://93.184.216.34/direct", full: true }, + { url: "https://93.184.216.34/direct", mode: "full" }, { signal: new AbortController().signal }, ); expect(direct.content).toBe("Packed DSH browserless fixture.\n"); @@ -313,7 +313,7 @@ describe("DSH 0.1.2-alpha.3 packed package contract", () => { url: "https://93.184.216.34/rendered", render: "browser", waitMs: 0, - full: true, + mode: "full", }, { signal: new AbortController().signal }, ); diff --git a/packages/dsh-web/test/client.test.ts b/packages/dsh-web/test/client.test.ts index b7d3cf2..c17eea7 100644 --- a/packages/dsh-web/test/client.test.ts +++ b/packages/dsh-web/test/client.test.ts @@ -56,6 +56,7 @@ describe("DSH settings client credential surface", () => { fetchDetails({ render: "browser", waitMs: 2_000, + mode: "section", section_id: "installation", }), ).toEqual([ @@ -65,12 +66,16 @@ describe("DSH settings client credential surface", () => { ]); expect(fetchDetails({})).toEqual([ { label: "Backend", value: "HTTP rendered" }, - { label: "Result", value: "Adaptive document" }, + { label: "Result", value: "Automatic navigation" }, ]); - expect(fetchDetails({ full: true })).toEqual([ + expect(fetchDetails({ mode: "full" })).toEqual([ { label: "Backend", value: "HTTP rendered" }, { label: "Result", value: "Full document" }, ]); + expect(fetchDetails({ mode: "tree" })).toEqual([ + { label: "Backend", value: "HTTP rendered" }, + { label: "Result", value: "Heading tree" }, + ]); }); it("shadows the host fetch view and registers dedicated views for links and docs", () => { diff --git a/packages/dsh-web/test/tools.test.ts b/packages/dsh-web/test/tools.test.ts index 4171a61..5831e21 100644 --- a/packages/dsh-web/test/tools.test.ts +++ b/packages/dsh-web/test/tools.test.ts @@ -86,6 +86,12 @@ describe("DSH direct web tools", () => { "http", "browser", ]); + expect((definitions[0]!.parameters as any).properties.mode.enum).toEqual([ + "auto", + "full", + "tree", + "section", + ]); expect((definitions[0]!.parameters as any).properties.waitMs.type).toBe( "integer", ); @@ -148,6 +154,7 @@ describe("DSH direct web tools", () => { fetch!, { url: "https://example.test", + mode: "section", section_id: "install", render: "browser", waitMs: 2000, @@ -191,6 +198,7 @@ describe("DSH direct web tools", () => { kind: "fetch", input: { url: "https://example.test", + mode: "section", section_id: "install", render: "browser", waitMs: 2000, @@ -237,6 +245,22 @@ describe("DSH direct web tools", () => { it("preserves shared fetch validation, cancellation, and structured renderer failures", async () => { const fetch = createWebToolDefinitions(dependencies())[0]!; + await expect( + call(fetch, { url: "https://example.test", mode: "section" }), + ).rejects.toThrow('section_id is required when mode is "section"'); + await expect( + call(fetch, { + url: "https://example.test", + mode: "full", + section_id: "intro", + }), + ).rejects.toThrow('section_id is only valid with mode "section"'); + await expect( + call(fetch, { url: "https://example.test", full: true }), + ).rejects.toThrow(/does not accept field full/); + await expect( + call(fetch, { url: "https://example.test", mode: "invalid" }), + ).rejects.toThrow(/mode.*auto.*full.*tree.*section/); await expect( call(fetch, { url: "https://example.test", diff --git a/packages/pi-web/src/tool.ts b/packages/pi-web/src/tool.ts index 4e29082..aa26868 100644 --- a/packages/pi-web/src/tool.ts +++ b/packages/pi-web/src/tool.ts @@ -3,9 +3,11 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { createWebOperations, DEFAULT_LINK_LIMIT, + FETCH_MODES, MAX_LINK_LIMIT, normalizeDocsToolInput, type DocsToolInput, + type FetchMode, type LinksResult, type SearchProvider, type WebCredentials, @@ -36,18 +38,19 @@ export const webSearchSchema = Type.Object( const fetchNavigationProperties = { url: Type.String({ description: "HTTP or HTTPS URL to fetch" }), + mode: Type.Optional( + StringEnum(FETCH_MODES, { + default: "auto", + description: "Navigation mode: auto (default), full, tree, or section", + }), + ), section_id: Type.Optional( Type.String({ - description: "Optional heading section ID to return", + description: "Heading section ID; required only with mode section", minLength: 1, pattern: "\\S", }), ), - full: Type.Optional( - Type.Boolean({ - description: "Return full content without automatic tree mode", - }), - ), }; const webFetchBaseSchema = Type.Union([ @@ -82,14 +85,18 @@ export const webFetchSchema = { ...webFetchBaseSchema, allOf: [ { - not: { - type: "object", - required: ["full", "section_id"], - properties: { - full: { const: true }, - section_id: { type: "string" }, + oneOf: [ + { + properties: { + mode: { enum: ["auto", "full", "tree"] }, + }, + not: { required: ["section_id"] }, }, - }, + { + properties: { mode: { const: "section" } }, + required: ["mode", "section_id"], + }, + ], }, ], } as typeof webFetchBaseSchema; @@ -205,7 +212,7 @@ const SEARCH_PROMPT_GUIDELINES = [ "Use web_search to search the web for current facts.", ]; const FETCH_PROMPT_GUIDELINES = [ - "Use web_fetch to read a web page; long documents with navigable headings return a navigation tree automatically, so follow up with full: true or a returned section_id. Headingless long documents use the normal bounded response.", + 'Use web_fetch to read a web page; mode: "auto" is the default and long documents with navigable headings return a navigation tree. Follow up with mode: "section" and a returned section_id, or mode: "full" to read everything. Headingless long documents use the normal bounded response.', 'web_fetch uses HTTP fetching by default; set render: "http" explicitly when desired.', 'For a client-rendered or SPA page, or after javascript_rendering_may_be_required, retry explicitly with render: "browser" and waitMs: 2000 only when the host provides browser capability. Increase waitMs explicitly or abandon an incomplete page; there is no automatic fallback.', "Never send waitMs with HTTP rendering. The browser is a host capability, not a package dependency.", @@ -272,24 +279,29 @@ function requireQueries(input: unknown): string[] { function normalizeFetch(input: unknown): WebFetchInput { if (!isRecord(input)) throw new Error("web_fetch input must be an object"); for (const field of Object.keys(input)) { - if (!["url", "section_id", "full", "render", "waitMs"].includes(field)) + if (!["url", "mode", "section_id", "render", "waitMs"].includes(field)) throw new Error(`web_fetch input does not accept field ${field}`); } const url = requireString(input, "url"); const renderOptions = validateRenderOptions(input); - if (input.full !== undefined && typeof input.full !== "boolean") - throw new Error("full must be a boolean"); + const mode = input.mode; + if (mode !== undefined && !FETCH_MODES.includes(mode as FetchMode)) + throw new Error('mode must be one of "auto", "full", "tree", or "section"'); if (input.section_id !== undefined) { if (typeof input.section_id !== "string" || input.section_id.trim() === "") throw new Error("section_id must be a non-empty string"); } - if (input.full === true && input.section_id !== undefined) - throw new Error("full and section_id cannot be used together"); + if (mode === "section") { + if (input.section_id === undefined) + throw new Error('section_id is required when mode is "section"'); + } else if (input.section_id !== undefined) { + throw new Error('section_id is only valid with mode "section"'); + } const typed = input as unknown as WebFetchInput; const navigation = { url, + ...(typed.mode === undefined ? {} : { mode: typed.mode }), ...(typed.section_id === undefined ? {} : { section_id: typed.section_id }), - ...(typed.full === undefined ? {} : { full: typed.full }), }; return { ...navigation, ...renderOptions }; } @@ -436,14 +448,14 @@ export function webFetchTool(dependencies: WebToolDependencies = {}) { name: "web_fetch", label: "Web fetch", description: - "Fetch and read an HTTP or HTTPS web page as Markdown, with HTTP rendering by default or explicit browser rendering for client-rendered pages. Browser rendering requires waitMs 0 through 30000. Long documents with navigable headings return a navigation tree unless full or section_id is requested; headingless long documents use the normal bounded response. Text output is limited to 2,000 lines or 50KB; truncated output is saved to a temporary file.", + "Fetch and read an HTTP or HTTPS web page as Markdown, with HTTP rendering by default or explicit browser rendering for client-rendered pages. Browser rendering requires waitMs 0 through 30000. mode selects auto, full, tree, or section navigation; mode section requires section_id. Text output is limited to 2,000 lines or 50KB; truncated output is saved to a temporary file.", promptSnippet: "Fetch a web page with web_fetch", promptGuidelines: FETCH_PROMPT_GUIDELINES, parameters: webFetchSchema, execute: async (params, signal) => { const data = await operations.fetch(normalizeFetch(params), signal); return modelTextResult(data, data.content, { - hint: "Use web_fetch with full: true or a returned section_id to navigate the document.", + hint: 'Use web_fetch with mode: "full" for the complete document, or mode: "section" with a returned section_id to navigate the document.', }); }, }); diff --git a/packages/pi-web/test/extension.test.ts b/packages/pi-web/test/extension.test.ts index 99add5f..e9118b3 100644 --- a/packages/pi-web/test/extension.test.ts +++ b/packages/pi-web/test/extension.test.ts @@ -91,10 +91,25 @@ describe("pi-web extension", () => { ).rejects.toThrow(/does not accept query/); }); - it("enforces the render/wait contract and forwards explicit browser retries", async () => { + it("enforces the render/wait and navigation contracts", async () => { expect(Value.Check(webFetchSchema, { url: "https://fixture.test" })).toBe( true, ); + for (const mode of ["auto", "full", "tree"] as const) { + expect( + Value.Check(webFetchSchema, { + url: "https://fixture.test", + mode, + }), + ).toBe(true); + } + expect( + Value.Check(webFetchSchema, { + url: "https://fixture.test", + mode: "section", + section_id: "intro", + }), + ).toBe(true); expect( Value.Check(webFetchSchema, { url: "https://fixture.test", @@ -138,13 +153,36 @@ describe("pi-web extension", () => { expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - full: true, + mode: "section", + }), + ).toBe(false); + expect( + Value.Check(webFetchSchema, { + url: "https://fixture.test", + mode: "full", section_id: "intro", }), ).toBe(false); + expect( + Value.Check(webFetchSchema, { + url: "https://fixture.test", + full: true, + }), + ).toBe(false); + expect( + Value.Check(webFetchSchema, { + url: "https://fixture.test", + mode: "invalid", + }), + ).toBe(false); const fetch = vi.fn( - async (input: { url: string; render?: string; waitMs?: number }) => ({ + async (input: { + url: string; + mode?: string; + render?: string; + waitMs?: number; + }) => ({ url: input.url, mode: "full" as const, content: "Rendered Markdown", @@ -153,12 +191,14 @@ describe("pi-web extension", () => { const tool = webFetchTool({ operations: operations({ fetch }) }); const result = await call(tool, { url: "https://fixture.test", + mode: "full", render: "browser", waitMs: 1250, }); expect(fetch).toHaveBeenCalledWith( { url: "https://fixture.test", + mode: "full", render: "browser", waitMs: 1250, }, @@ -503,7 +543,7 @@ describe("pi-web extension", () => { expect(details.url).toBe("https://fixture.test"); expect(details.truncation.truncated).toBe(true); expect(result.content[0]?.text).toContain( - "Use web_fetch with full: true or a returned section_id to navigate the document.", + 'Use web_fetch with mode: "full" for the complete document, or mode: "section" with a returned section_id to navigate the document.', ); expect(await readFile(details.fullOutputPath, "utf8")).toBe(content); } finally { diff --git a/packages/pi-web/test/packed-smoke.mjs b/packages/pi-web/test/packed-smoke.mjs index f5d366f..d54e2f6 100644 --- a/packages/pi-web/test/packed-smoke.mjs +++ b/packages/pi-web/test/packed-smoke.mjs @@ -192,7 +192,7 @@ if (command === "open" && args.some((value) => value.includes("/blocked"))) { throw new Error("packed extension did not register web_fetch"); const direct = await fetchTool.execute("test", { url: "https://93.184.216.34/direct", - full: true, + mode: "full", }); if (!direct.content[0]?.text.includes("Browserless output.")) throw new Error("packed extension did not execute browserless fetch"); @@ -217,7 +217,7 @@ if (command === "open" && args.some((value) => value.includes("/blocked"))) { url: "https://93.184.216.34/rendered", render: "browser", waitMs: 0, - full: true, + mode: "full", }); if ( !rendered.content[0]?.text.includes( diff --git a/packages/web-core/src/fetch.ts b/packages/web-core/src/fetch.ts index c22c825..0f32fe9 100644 --- a/packages/web-core/src/fetch.ts +++ b/packages/web-core/src/fetch.ts @@ -9,7 +9,7 @@ import { spawn } from "node:child_process"; import { Defuddle } from "defuddle/node"; import { parseHTML } from "linkedom"; -import { renderMarkdown } from "./markdown.js"; +import { FETCH_MODES, renderMarkdown, type FetchMode } from "./markdown.js"; import { boundedRequest, isOperationAborted, @@ -45,8 +45,8 @@ export const RENDER_CDN_ALLOWLIST = [ export type FetchInput = { url: string; + mode?: FetchMode; section_id?: string; - full?: boolean; render?: "http" | "browser"; waitMs?: number; }; @@ -126,15 +126,15 @@ export async function fetchWebPage( validateFetchFields(input); const url = validateURL(input.url); const render = validateRenderInput(input); - validateNavigationInput(input); + const mode = validateNavigationInput(input); const content = render === "browser" ? await renderPage(url, input.waitMs!, callerSignal, options) : await fetchCached(url, callerSignal, options); throwIfAborted(callerSignal); const rendered = renderMarkdown(content, { + mode, section_id: input.section_id, - full: input.full === true, }); return { url, mode: rendered.mode, content: rendered.content }; } @@ -182,11 +182,9 @@ function validateRenderInput( function validateFetchFields(input: FetchInput): void { validateKnownFields( input, - ["url", "section_id", "full", "render", "waitMs"], + ["url", "mode", "section_id", "render", "waitMs"], "fetch", ); - if (typeof input.full !== "undefined" && typeof input.full !== "boolean") - throw new Error("full must be a boolean"); if (typeof input.section_id !== "undefined") { if (typeof input.section_id !== "string" || input.section_id.trim() === "") throw new Error("section_id must be a non-empty string"); @@ -209,10 +207,19 @@ function validateKnownFields( } function validateNavigationInput( - input: Pick, -): void { - if (input.full === true && input.section_id !== undefined) - throw new Error("full and section_id cannot be used together"); + input: Pick, +): FetchMode { + const mode = input.mode ?? "auto"; + if (!FETCH_MODES.includes(mode as FetchMode)) + throw new Error('mode must be one of "auto", "full", "tree", or "section"'); + if (mode === "section") { + if (input.section_id === undefined) + throw new Error('section_id is required when mode is "section"'); + return mode; + } + if (input.section_id !== undefined) + throw new Error('section_id is only valid with mode "section"'); + return mode; } function validateLinkLimit(limit: number | undefined): number { diff --git a/packages/web-core/src/index.ts b/packages/web-core/src/index.ts index c3de6d8..f910076 100644 --- a/packages/web-core/src/index.ts +++ b/packages/web-core/src/index.ts @@ -60,6 +60,8 @@ export { export { renderMarkdown, truncateContent, + FETCH_MODES, + type FetchMode, type MarkdownResult, type MarkdownNavigationOptions, } from "./markdown.js"; diff --git a/packages/web-core/src/markdown.ts b/packages/web-core/src/markdown.ts index ccea28c..e1e1fa3 100644 --- a/packages/web-core/src/markdown.ts +++ b/packages/web-core/src/markdown.ts @@ -18,9 +18,12 @@ export type MarkdownResult = { mode: "full" | "tree" | "section"; }; +export const FETCH_MODES = ["auto", "full", "tree", "section"] as const; +export type FetchMode = (typeof FETCH_MODES)[number]; + export type MarkdownNavigationOptions = { + mode?: FetchMode; section_id?: string; - full?: boolean; }; const DEFAULT_TREE_THRESHOLD = 5000; @@ -35,30 +38,56 @@ export function renderMarkdown( source: string, options: MarkdownNavigationOptions = {}, ): MarkdownResult { + validateNavigationOptions(options); const headings = assignIds(parseHeadings(source)); + const mode = options.mode ?? "auto"; const section = options.section_id?.trim(); - if (options.full === true && section !== undefined) - throw new Error("full and section_id cannot be used together"); - if (section) { + if (mode === "section") { + if (!section) + throw new Error('section_id is required when mode is "section"'); return { content: extractSection(source, headings, section), mode: "section", }; } + if (section !== undefined) + throw new Error('section_id is only valid with mode "section"'); + + if (mode === "tree") return { content: renderTree(source, headings), mode }; + + if (mode === "full") + return { + content: source, + mode: "full", + }; const charCount = Array.from(source).length; - if ( - !options.full && - charCount > DEFAULT_TREE_THRESHOLD && - headings.length > 0 - ) + if (charCount > DEFAULT_TREE_THRESHOLD && headings.length > 0) return { content: renderTree(source, headings), mode: "tree" }; return { - content: options.full ? source : truncateContent(source), + content: truncateContent(source), mode: "full", }; } +function validateNavigationOptions(options: object): void { + for (const field of Object.keys(options)) { + if (field !== "mode" && field !== "section_id") + throw new Error(`navigation options do not accept field ${field}`); + } + + const mode = (options as { mode?: unknown }).mode; + if (mode !== undefined && !FETCH_MODES.includes(mode as FetchMode)) + throw new Error('mode must be one of "auto", "full", "tree", or "section"'); + + const section = (options as { section_id?: unknown }).section_id; + if ( + section !== undefined && + (typeof section !== "string" || section.trim() === "") + ) + throw new Error("section_id must be a non-empty string"); +} + export function truncateContent(content: string): string { const chars = Array.from(content); if (chars.length <= MAX_CONTENT_CHARS) return content; @@ -194,7 +223,7 @@ function renderTree(source: string, headings: Heading[]): string { `└── [${heading.id}] ${"#".repeat(heading.level)} ${heading.text} (${formatNumber(sectionCharCount(source, headings, index))} chars)\n`, ) .join(""); - return `${header}${tree}\nUse section_id to read a section, or full: true to read everything.\n`; + return `${header}${tree}\nUse mode: "section" with section_id to read a section, or mode: "full" to read everything.\n`; } const nodes = bodyHeadings.map((heading) => { @@ -223,7 +252,7 @@ function renderTree(source: string, headings: Heading[]): string { hasMore.delete(depth); }); - return `${header}${tree}\nUse section_id to read a section, or full: true to read everything.\n`; + return `${header}${tree}\nUse mode: "section" with section_id to read a section, or mode: "full" to read everything.\n`; } function sectionCharCount( diff --git a/packages/web-core/test/fetch.test.ts b/packages/web-core/test/fetch.test.ts index fb7ce64..0754b60 100644 --- a/packages/web-core/test/fetch.test.ts +++ b/packages/web-core/test/fetch.test.ts @@ -78,7 +78,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { "invalid URL", ); await expect( - fetchPage({ url: `${url}/redirect`, full: true }), + fetchPage({ url: `${url}/redirect`, mode: "full" }), ).resolves.toMatchObject({ content: "redirected text", mode: "full", @@ -138,11 +138,11 @@ describe.sequential("browserless fetch migrated from Organon", () => { try { await withTempCache(async (fetchPage) => { await expect( - fetchPage({ url: `${url}/text`, full: true }), + fetchPage({ url: `${url}/text`, mode: "full" }), ).resolves.toMatchObject({ content: "package main\n\nfunc main() {}\n", }); - await expect(fetchPage({ url, full: true })).resolves.toEqual({ + await expect(fetchPage({ url, mode: "full" })).resolves.toEqual({ url, mode: "full", content: "Extracted text.\n", @@ -369,7 +369,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { url: "https://render.test/page", render: "browser", waitMs: 0, - full: true, + mode: "full", }, undefined, { cacheDirectory, fetch: directFetch, resolveHost }, @@ -754,7 +754,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { }); try { await withTempCache(async (fetchPage) => { - const result = await fetchPage({ url, full: true }); + const result = await fetchPage({ url, mode: "full" }); expect(result.content).toContain("Initial static content."); expect(result.content).not.toContain("SPA_MARKER_RENDERED"); }); @@ -780,7 +780,12 @@ describe.sequential("browserless fetch migrated from Organon", () => { mode: "tree", }); await expect( - fetchWebPage({ ...input, section_id: "7i" }, undefined, { cache }), + fetchWebPage({ ...input, mode: "tree" }, undefined, { cache }), + ).resolves.toMatchObject({ mode: "tree" }); + await expect( + fetchWebPage({ ...input, mode: "section", section_id: "7i" }, undefined, { + cache, + }), ).resolves.toMatchObject({ mode: "section", content: "## Install\nInstall content.\n", @@ -793,7 +798,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { write: vi.fn(async () => {}), }; const result = await fetchWebPage( - { url: "https://navigation.test/complete", full: true }, + { url: "https://navigation.test/complete", mode: "full" }, undefined, { cache: completeCache }, ); @@ -807,24 +812,41 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebPage( { url: input.url, - full: true, + mode: "full", section_id: "7i", } as never, undefined, { cache }, ), - ).rejects.toThrow("full and section_id cannot be used together"); + ).rejects.toThrow('section_id is only valid with mode "section"'); await expect( fetchWebPage({ url: input.url, tree: true } as never, undefined, { cache, }), ).rejects.toThrow("does not accept field tree"); + await expect( + fetchWebPage({ ...input, mode: "section" } as never, undefined, { + cache, + }), + ).rejects.toThrow('section_id is required when mode is "section"'); + await expect( + fetchWebPage({ ...input, mode: "auto", section_id: "7i" }, undefined, { + cache, + }), + ).rejects.toThrow('section_id is only valid with mode "section"'); + await expect( + fetchWebPage({ ...input, mode: "invalid" } as never, undefined, { + cache, + }), + ).rejects.toThrow( + 'mode must be one of "auto", "full", "tree", or "section"', + ); await expect( fetchWebPage({ url: input.url, tree_threshold: 1 } as never, undefined, { cache, }), ).rejects.toThrow("does not accept field tree_threshold"); - expect(cache.read).toHaveBeenCalledTimes(2); + expect(cache.read).toHaveBeenCalledTimes(3); }); }); diff --git a/packages/web-core/test/markdown.test.ts b/packages/web-core/test/markdown.test.ts index d41c8bd..d4fb7c1 100644 --- a/packages/web-core/test/markdown.test.ts +++ b/packages/web-core/test/markdown.test.ts @@ -46,13 +46,15 @@ describe("Markdown navigation", () => { /\[([0-9A-Za-z]{2,3})\] ## Quote emphasis/, )?.[1]; expect(quoteID).toBeDefined(); - expect(renderMarkdown(source, { section_id: quoteID })).toEqual({ + expect( + renderMarkdown(source, { mode: "section", section_id: quoteID }), + ).toEqual({ mode: "section", content: "> ## Quote *emphasis*\n\n- ### List heading\n", }); }); - it("uses the fixed automatic tree policy and full/section navigation", () => { + it("uses the fixed automatic tree policy and explicit navigation modes", () => { const source = "# Test page\n\n## Install\nInstall content.\n\n### Details\nDetails content.\n\n## Next\nNext content.\n"; const tree = renderMarkdown(`${source}${"x".repeat(5001)}`); @@ -60,20 +62,22 @@ describe("Markdown navigation", () => { expect(tree.content).toContain("[7i] ## Install"); expect(tree.content).toContain("[eD] ### Details"); - expect(renderMarkdown(source, { section_id: "7i" })).toEqual({ + expect( + renderMarkdown(source, { mode: "section", section_id: "7i" }), + ).toEqual({ mode: "section", content: "## Install\nInstall content.\n\n### Details\nDetails content.\n", }); - expect(() => renderMarkdown(source, { section_id: "missing" })).toThrow( - 'section "missing" not found', - ); + expect(() => + renderMarkdown(source, { mode: "section", section_id: "missing" }), + ).toThrow('section "missing" not found'); expect(renderMarkdown("plain content")).toEqual({ content: "plain content", mode: "full", }); const complete = "# H\n\n" + "x".repeat(30_001); - expect(renderMarkdown(complete, { full: true })).toEqual({ + expect(renderMarkdown(complete, { mode: "full" })).toEqual({ content: complete, mode: "full", }); @@ -88,7 +92,9 @@ describe("Markdown navigation", () => { expect(tree.mode).toBe("tree"); expect(sectionID).toBeDefined(); - expect(renderMarkdown(source, { section_id: sectionID })).toEqual({ + expect( + renderMarkdown(source, { mode: "section", section_id: sectionID }), + ).toEqual({ content: source, mode: "section", }); @@ -102,9 +108,35 @@ describe("Markdown navigation", () => { }); const complete = "x".repeat(30_001); - expect(renderMarkdown(complete, { full: true })).toEqual({ + expect(renderMarkdown(complete, { mode: "full" })).toEqual({ content: complete, mode: "full", }); }); + + it("supports explicit tree mode, including documents without headings", () => { + expect(renderMarkdown("short content", { mode: "tree" })).toEqual({ + content: "(no headings)\n", + mode: "tree", + }); + expect( + renderMarkdown("# Heading\n\ncontent\n", { mode: "tree" }).mode, + ).toBe("tree"); + }); + + it("requires section mode and rejects incompatible navigation fields", () => { + expect(() => renderMarkdown("# Heading\n")).not.toThrow(); + expect(() => renderMarkdown("# Heading\n", { mode: "section" })).toThrow( + 'section_id is required when mode is "section"', + ); + expect(() => + renderMarkdown("# Heading\n", { mode: "auto", section_id: "x" }), + ).toThrow('section_id is only valid with mode "section"'); + expect(() => + renderMarkdown("# Heading\n", { mode: "full", section_id: "x" }), + ).toThrow('section_id is only valid with mode "section"'); + expect(() => + renderMarkdown("# Heading\n", { mode: "invalid" as never }), + ).toThrow('mode must be one of "auto", "full", "tree", or "section"'); + }); }); diff --git a/packages/web/src/http.ts b/packages/web/src/http.ts index 462aff7..c8d3959 100644 --- a/packages/web/src/http.ts +++ b/packages/web/src/http.ts @@ -1,6 +1,7 @@ import { DEFAULT_LINK_LIMIT, DEFAULT_KEPOS_BRIDGE_ENDPOINT, + FETCH_MODES, FetchCapabilityError, isOperationAborted, isRequestTimeout, @@ -106,6 +107,10 @@ const HttpUrlSchema = z const FetchRequestSchema = z .object({ url: HttpUrlSchema, + mode: z.enum(FETCH_MODES).default("auto").openapi({ + description: + 'Navigation mode: "auto" (default), "full", "tree", or "section".', + }), section_id: z .string() .min(1) @@ -114,16 +119,39 @@ const FetchRequestSchema = z "section_id must be a non-empty string", ) .optional(), - full: z.boolean().default(false), render: z.enum(["http", "browser"]).default("http"), waitMs: z.number().int().min(0).max(30_000).optional(), }) .strict() - .refine( - (input) => !(input.full && input.section_id !== undefined), - "full and section_id cannot be used together", - ) - .openapi("FetchRequest"); + .superRefine((input, context) => { + if (input.mode === "section" && input.section_id === undefined) { + context.addIssue({ + code: "custom", + message: 'section_id is required when mode is "section"', + path: ["section_id"], + }); + } else if (input.mode !== "section" && input.section_id !== undefined) { + context.addIssue({ + code: "custom", + message: 'section_id is only valid with mode "section"', + path: ["section_id"], + }); + } + }) + .openapi("FetchRequest", { + oneOf: [ + { + properties: { + mode: { enum: ["auto", "full", "tree"] }, + }, + not: { required: ["section_id"] }, + }, + { + properties: { mode: { const: "section" } }, + required: ["mode", "section_id"], + }, + ], + }); const LinksRequestSchema = z .object({ @@ -179,7 +207,7 @@ const fetchRoute = createRoute({ operationId: "fetch", summary: "Fetch a web page", description: - "Fetch through HTTP by default; browser rendering requires render=browser and waitMs.", + 'Fetch through HTTP by default with auto navigation; use mode "full", "tree", or "section" for explicit navigation. Browser rendering requires render=browser and waitMs.', request: jsonRequest(FetchRequestSchema), responses: { 200: jsonResponse(FetchResponseSchema, "Fetched page."), diff --git a/packages/web/src/mcp.ts b/packages/web/src/mcp.ts index 23f5526..1afcd72 100644 --- a/packages/web/src/mcp.ts +++ b/packages/web/src/mcp.ts @@ -9,9 +9,11 @@ import { Command } from "commander"; import { DEFAULT_LINK_LIMIT, + FETCH_MODES, FetchCapabilityError, MAX_LINK_LIMIT, RENDER_REPORT_URL, + type FetchMode, type FetchErrorDetails, type WebCredentials, type WebOperations, @@ -20,8 +22,8 @@ import { type SearchToolInput = { query: string }; type FetchToolInput = { url: string; + mode?: FetchMode; section_id?: string; - full?: boolean; render?: "http" | "browser"; waitMs?: number; }; @@ -61,14 +63,17 @@ const fetchInputSchema = schema({ additionalProperties: false, properties: { url: { type: "string", description: "HTTP or HTTPS URL to fetch" }, + mode: { + type: "string", + enum: [...FETCH_MODES], + default: "auto", + description: "navigation mode: auto (default), full, tree, or section", + }, section_id: { type: "string", minLength: 1, - description: "optional heading section ID to return", - }, - full: { - type: "boolean", - description: "return full content without automatic tree mode", + pattern: "\\S", + description: "heading section ID; required only with mode section", }, render: { type: "string", @@ -96,10 +101,18 @@ const fetchInputSchema = schema({ ], allOf: [ { - not: { - required: ["section_id", "full"], - properties: { full: { const: true } }, - }, + oneOf: [ + { + properties: { + mode: { enum: FETCH_MODES.filter((mode) => mode !== "section") }, + }, + not: { required: ["section_id"] }, + }, + { + properties: { mode: { const: "section" } }, + required: ["mode", "section_id"], + }, + ], }, ], }); @@ -337,14 +350,14 @@ export function createMcpServer(dependencies: McpDependencies): McpServer { fetchInputSchema, fetchOutputSchema, ), - async ({ url, section_id, full, render, waitMs }, context) => + async ({ url, mode, section_id, render, waitMs }, context) => runTool( () => dependencies.operations.fetch( { url, + ...(mode !== undefined ? { mode } : {}), ...(section_id !== undefined ? { section_id } : {}), - ...(full !== undefined ? { full } : {}), ...(render !== undefined ? { render } : {}), ...(waitMs !== undefined ? { waitMs } : {}), }, diff --git a/packages/web/src/program.ts b/packages/web/src/program.ts index 6007c88..18b0b15 100644 --- a/packages/web/src/program.ts +++ b/packages/web/src/program.ts @@ -5,8 +5,10 @@ import { parseHttpPort, startHttpServer } from "./serve.js"; import { DEFAULT_HTTP_HOST, DEFAULT_HTTP_PORT } from "./serve.js"; import { + FETCH_MODES, formatSearchResults, type DocsLibrary, + type FetchMode, type LinksResult, type WebCredentials, type WebOperations, @@ -243,8 +245,12 @@ function createFetchCommand(dependencies: ProgramDependencies): Command { "Required post-load wait for --render browser (0-30000)", Number, ) + .option( + "--mode ", + "Navigation mode: auto (default), full, tree, or section", + parseFetchMode, + ) .option("-s, --section ", "Read one heading section") - .option("--full", "Return the complete extracted Markdown") .option("--json", "Output the structured result as JSON") .action( async ( @@ -252,19 +258,20 @@ function createFetchCommand(dependencies: ProgramDependencies): Command { options: { render?: "http" | "browser"; wait?: number; + mode?: FetchMode; section?: string; - full?: boolean; json?: boolean; }, ) => { + validateCliNavigation(options.mode, options.section); const input = { url, ...(options.render !== undefined ? { render: options.render } : {}), ...(options.wait !== undefined ? { waitMs: options.wait } : {}), + ...(options.mode !== undefined ? { mode: options.mode } : {}), ...(options.section !== undefined ? { section_id: options.section } : {}), - ...(options.full !== undefined ? { full: options.full } : {}), }; const result = await dependencies.operations.fetch(input); if (options.json) { @@ -276,6 +283,27 @@ function createFetchCommand(dependencies: ProgramDependencies): Command { ); } +function parseFetchMode(value: string): FetchMode { + if (!FETCH_MODES.includes(value as FetchMode)) + throw new Error("--mode must be one of auto, full, tree, or section"); + return value as FetchMode; +} + +function validateCliNavigation( + mode: FetchMode | undefined, + section: string | undefined, +): void { + if (section !== undefined && section.trim() === "") + throw new Error("--section must be a non-empty string"); + if (mode === "section") { + if (section === undefined) + throw new Error('--section is required when --mode is "section"'); + return; + } + if (section !== undefined) + throw new Error('--section requires --mode "section"'); +} + function createLinksCommand(dependencies: ProgramDependencies): Command { const writeOut = dependencies.writeOut ?? ((text: string) => process.stdout.write(text)); diff --git a/packages/web/src/runner.ts b/packages/web/src/runner.ts index bf45587..3777844 100644 --- a/packages/web/src/runner.ts +++ b/packages/web/src/runner.ts @@ -22,16 +22,29 @@ export async function runCli( output: CliOutput, ): Promise { const program = createProgram({ ...dependencies, writeOut: output.stdout }); + configureExitOverride(program); program.configureOutput({ writeOut: output.stdout, writeErr: output.stderr }); try { await program.parseAsync(argv); return 0; } catch (error) { + if ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "commander.helpDisplayed" + ) + return 0; output.stderr(formatCliError(error)); return 1; } } +function configureExitOverride(command: import("commander").Command): void { + command.exitOverride(); + for (const child of command.commands) configureExitOverride(child); +} + function formatCliError(error: unknown): string { if (!(error instanceof FetchCapabilityError)) return `${error instanceof Error ? error.message : "web search failed"}\n`; diff --git a/packages/web/test/http.test.ts b/packages/web/test/http.test.ts index e442bc6..a810568 100644 --- a/packages/web/test/http.test.ts +++ b/packages/web/test/http.test.ts @@ -227,7 +227,7 @@ describe("personal HTTP service", () => { expect(ops.fetch).toHaveBeenCalledWith( expect.objectContaining({ url: "https://example.test", - full: false, + mode: "auto", render: "browser", waitMs: 0, }), @@ -235,6 +235,49 @@ describe("personal HTTP service", () => { ); }); + it("forwards every navigation mode and rejects incompatible section IDs", async () => { + const ops = operations({ + fetch: vi.fn(async (input) => ({ + url: input.url, + mode: input.mode === "tree" ? "tree" : "full", + content: input.mode === "tree" ? "tree" : "page", + })) as WebOperations["fetch"], + }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + + for (const mode of ["auto", "full", "tree"] as const) { + const result = await json(app, "/v1/fetch", { + url: "https://example.test", + mode, + }); + expect(result.response.status).toBe(200); + } + const section = await json(app, "/v1/fetch", { + url: "https://example.test", + mode: "section", + section_id: "intro", + }); + expect(section.response.status).toBe(200); + expect(ops.fetch).toHaveBeenNthCalledWith( + 4, + expect.objectContaining({ mode: "section", section_id: "intro" }), + expect.any(AbortSignal), + ); + + for (const body of [ + { url: "https://example.test", mode: "section" }, + { url: "https://example.test", mode: "auto", section_id: "intro" }, + { url: "https://example.test", mode: "full", section_id: "intro" }, + { url: "https://example.test", mode: "tree", section_id: "intro" }, + { url: "https://example.test", section_id: "intro" }, + { url: "https://example.test", mode: "invalid" }, + { url: "https://example.test", full: true }, + ]) { + expect((await json(app, "/v1/fetch", body)).response.status).toBe(400); + } + expect(ops.fetch).toHaveBeenCalledTimes(4); + }); + it("forwards links with the same explicit rendered-fetch contract", async () => { const ops = operations({ links: vi.fn(async (input) => ({ @@ -293,7 +336,7 @@ describe("personal HTTP service", () => { ( await json(app, "/v1/fetch", { url: "https://example.test", - full: true, + mode: "full", section_id: "intro", }) ).response.status, @@ -357,6 +400,14 @@ describe("personal HTTP service", () => { .enum, ).toEqual(["Exa", "DeepSeek", "Kepos Bridge"]); const fetchRequest = (document.components?.schemas as any).FetchRequest; + expect(fetchRequest.properties.mode.enum).toEqual([ + "auto", + "full", + "tree", + "section", + ]); + expect(fetchRequest.properties.mode.default).toBe("auto"); + expect(fetchRequest.properties.full).toBeUndefined(); expect(fetchRequest.properties.render.enum).toEqual(["http", "browser"]); expect(fetchRequest.properties.tree).toBeUndefined(); expect(fetchRequest.properties.tree_threshold).toBeUndefined(); diff --git a/packages/web/test/mcp.test.ts b/packages/web/test/mcp.test.ts index dbfad30..9a45267 100644 --- a/packages/web/test/mcp.test.ts +++ b/packages/web/test/mcp.test.ts @@ -250,6 +250,11 @@ describe("web stdio MCP adapter", () => { enum: ["http", "browser"], default: "http", }); + expect(fetchProperties.mode).toMatchObject({ + enum: ["auto", "full", "tree", "section"], + default: "auto", + }); + expect(fetchProperties.full).toBeUndefined(); expect(fetchProperties.waitMs).toMatchObject({ type: "integer", minimum: 0, @@ -288,6 +293,8 @@ describe("web stdio MCP adapter", () => { name: "fetch", arguments: { url: "https://example.test/page", + mode: "section", + section_id: "intro", render: "browser", waitMs: 125, }, @@ -341,6 +348,8 @@ describe("web stdio MCP adapter", () => { expect(operations.fetch).toHaveBeenCalledWith( { url: "https://example.test/page", + mode: "section", + section_id: "intro", render: "browser", waitMs: 125, }, @@ -406,6 +415,33 @@ describe("web stdio MCP adapter", () => { expect(operations.fetch).not.toHaveBeenCalled(); }); + it("validates navigation modes before invoking the core", async () => { + const { client, operations } = await connect(); + await client.listTools(); + for (const arguments_ of [ + { url: "https://example.test/page", mode: "section" }, + { url: "https://example.test/page", mode: "auto", section_id: "intro" }, + { url: "https://example.test/page", mode: "full", section_id: "intro" }, + { url: "https://example.test/page", mode: "tree", section_id: "intro" }, + { url: "https://example.test/page", section_id: "intro" }, + { url: "https://example.test/page", mode: "invalid" }, + { url: "https://example.test/page", full: true }, + ]) { + const result = await client.callTool({ + name: "fetch", + arguments: arguments_, + }); + expect(result.isError).toBe(true); + expect(result.content).toMatchObject([ + { + type: "text", + text: expect.stringContaining("Input validation error"), + }, + ]); + } + expect(operations.fetch).not.toHaveBeenCalled(); + }); + it("keeps fetch capability details structured and recovers after a renderer failure", async () => { const operations = webService(); vi.mocked(operations.fetch) diff --git a/packages/web/test/packed-smoke.mjs b/packages/web/test/packed-smoke.mjs index 512ed41..786f1f1 100644 --- a/packages/web/test/packed-smoke.mjs +++ b/packages/web/test/packed-smoke.mjs @@ -174,7 +174,7 @@ else console.log(JSON.stringify({ success: true, data: {} })); const mcpFetch = await client.callTool({ name: "fetch", - arguments: { url: `http://127.0.0.1:${port}/page`, full: true }, + arguments: { url: `http://127.0.0.1:${port}/page`, mode: "full" }, }); if ( mcpFetch.isError || @@ -198,7 +198,7 @@ else console.log(JSON.stringify({ success: true, data: {} })); const result = await execFileAsync( binary, - ["fetch", `http://127.0.0.1:${port}/page`, "--full", "--json"], + ["fetch", `http://127.0.0.1:${port}/page`, "--mode", "full", "--json"], { cwd: root }, ); const fetched = JSON.parse(result.stdout); @@ -231,7 +231,8 @@ else console.log(JSON.stringify({ success: true, data: {} })); "https://93.184.216.34/rendered", "--render=browser", "--wait=0", - "--full", + "--mode", + "full", "--json", ], { cwd: root, env: fakeEnvironment }, diff --git a/packages/web/test/program.test.ts b/packages/web/test/program.test.ts index 4cae55a..d713a76 100644 --- a/packages/web/test/program.test.ts +++ b/packages/web/test/program.test.ts @@ -205,12 +205,13 @@ describe("web search Commander adapter", () => { it("passes fetch navigation flags and writes human Markdown", async () => { const { program, operations, output } = setup(); await program.parseAsync( - ["fetch", "https://example.test/page", "-s", "7i"], + ["fetch", "https://example.test/page", "--mode", "section", "-s", "7i"], { from: "user" }, ); expect(operations.fetch).toHaveBeenCalledWith({ url: "https://example.test/page", + mode: "section", section_id: "7i", }); expect(output()).toEqual({ stdout: "# Fixture page\n", stderr: "" }); @@ -233,13 +234,13 @@ describe("web search Commander adapter", () => { it("writes fetch JSON as exactly one document", async () => { const { program, operations, output } = setup(); await program.parseAsync( - ["fetch", "https://example.test/page", "--full", "--json"], + ["fetch", "https://example.test/page", "--mode", "full", "--json"], { from: "user" }, ); expect(operations.fetch).toHaveBeenCalledWith({ url: "https://example.test/page", - full: true, + mode: "full", }); expect(JSON.parse(output().stdout)).toEqual( await operations.fetch.mock.results[0]!.value, @@ -247,6 +248,52 @@ describe("web search Commander adapter", () => { expect(output().stdout.endsWith("\n")).toBe(true); }); + it("rejects invalid navigation mode and section combinations before fetching", async () => { + for (const argv of [ + ["fetch", "https://example.test/page", "--mode", "invalid"], + ["fetch", "https://example.test/page", "--mode", "section"], + [ + "fetch", + "https://example.test/page", + "--mode", + "full", + "--section", + "intro", + ], + ["fetch", "https://example.test/page", "--section", "intro"], + ["fetch", "https://example.test/page", "--full"], + ]) { + const { program, operations } = setup(); + program.exitOverride(); + await expect( + program.parseAsync(argv, { from: "user" }), + ).rejects.toThrow(); + expect(operations.fetch).not.toHaveBeenCalled(); + } + }); + + it("returns a nonzero result for removed CLI flags without invoking fetch", async () => { + const operations = setup().operations; + let stdout = ""; + let stderr = ""; + const exitCode = await runCli( + ["node", "web", "fetch", "https://example.test/page", "--full"], + { operations, credentials: () => ({}) }, + { + stdout: (text) => { + stdout += text; + }, + stderr: (text) => { + stderr += text; + }, + }, + ); + expect(exitCode).toBe(1); + expect(stdout).toBe(""); + expect(stderr).toContain("unknown option '--full'"); + expect(operations.fetch).not.toHaveBeenCalled(); + }); + it("forwards link discovery options and supports concise and JSON output", async () => { const { program, operations, output } = setup(); await program.parseAsync( From 3ef2cfa3664365e55b94088704c4821461514307 Mon Sep 17 00:00:00 2001 From: neil Date: Wed, 2 Sep 2026 23:48:36 +0800 Subject: [PATCH 2/2] fix(fetch): align navigation contract across adapters --- CONTEXT.md | 13 ++--- README.md | 44 ++++++++++------- docs/adr/0001-containerized-rest-api.md | 2 +- docs/http-service.md | 24 +++++---- packages/dsh-web/README.md | 13 +++-- packages/dsh-web/src/client.ts | 2 +- packages/dsh-web/src/tools.ts | 26 +++++----- packages/dsh-web/test/client.test.ts | 6 ++- packages/dsh-web/test/tools.test.ts | 40 +++++++++++---- packages/pi-web/src/tool.ts | 32 ++++++------ packages/pi-web/test/extension.test.ts | 49 ++++++++++++++++-- packages/pi-web/test/packed-smoke.mjs | 5 +- packages/web-core/src/fetch.ts | 21 ++++---- packages/web-core/src/markdown.ts | 34 ++++++++----- packages/web-core/test/fetch.test.ts | 28 +++++++---- packages/web-core/test/markdown.test.ts | 66 ++++++++++++++++--------- packages/web/src/http.ts | 30 +++++------ packages/web/src/mcp.ts | 24 +++++---- packages/web/src/program.ts | 13 ++--- packages/web/src/runner.ts | 19 ++++--- packages/web/test/http.test.ts | 57 +++++++++++++-------- packages/web/test/mcp.test.ts | 43 +++++++++++++--- packages/web/test/packed-smoke.mjs | 2 + packages/web/test/program.test.ts | 58 +++++++++++++++------- 24 files changed, 419 insertions(+), 232 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index e724cee..c50bab2 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -39,12 +39,13 @@ _Avoid_: Backend-specific renderer labels, automatic fallback **Page Navigation**: The shared page-reading module owns its fixed 5,000-character policy. A long, unsectioned request with navigable headings returns a navigation tree; a -headingless long document uses the normal bounded response. The request -`mode` is `auto` by default and may be `full`, `tree`, or `section`; `auto` is -input-only because results report the actual mode. `mode: "full"` returns -complete Markdown, `mode: "tree"` forces the heading tree, and -`mode: "section"` with a returned `section_id` retrieves one section. -`section_id` is valid exactly for `mode: "section"`. +headingless long document uses the normal bounded automatic response. The +request `mode` is `auto` by default and may be `full` or `tree`; `section_id` +is allowed with omitted or `auto` mode and retrieves one section. Full and tree +reject `section_id`, and input mode `section` is removed. Ordinary automatic +document results report `mode: "auto"`; full, tree, and section results report +their corresponding modes. Every Fetch result includes `truncated`, which is +true only when automatic content was cut by the content-length limit. _Avoid_: Caller-selected tree thresholds, legacy navigation booleans **Release Contract**: diff --git a/README.md b/README.md index 8986838..ff9cb0b 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,8 @@ web search --provider exa -- "Node AbortSignal" web search --provider deepseek -- "Node AbortSignal" web search --provider kepos-bridge -- "Node AbortSignal" web fetch https://example.com/article -web fetch https://example.com/article --mode section --section introduction +web fetch https://example.com/article --section introduction +web fetch https://example.com/article --mode auto --section introduction web fetch https://example.com/article --mode tree web fetch https://example.com/article --mode full web links https://example.com/article --limit 50 @@ -131,16 +132,18 @@ web sgraph --count 10 -- "repo:^github\\.com/nodejs/node$ AbortSignal" ``` Use `--` before a search or Sourcegraph query that begins with a hyphen. `fetch` -supports `--mode auto|full|tree|section`; omitted mode means `auto`. `--section` -is required only with `--mode section`, and is rejected with every other mode. -Long extracted documents with navigable headings automatically return a heading -tree so a later request can retrieve a stable `section_id`. `mode: "auto"` is -request-only; Fetch results report the actual `mode` as `"full"`, `"tree"`, or -`"section"`. A headingless long document uses the normal bounded response. -`mode: "full"` returns the complete extracted Markdown, while `mode: "tree"` -always returns the heading-tree representation, including the explicit -no-headings result. `links` lists up to 100 unique HTTP(S) anchors from the -original page DOM. +supports `--mode auto|full|tree`; omitted mode means `auto`. `--section` may be +used with omitted mode or `--mode auto` to retrieve a section, and is rejected +with `--mode full` or `--mode tree`. Long extracted documents with navigable +headings automatically return a heading tree so a later request can retrieve a +stable `section_id`. Ordinary automatic document results report `mode: "auto"`; +heading-tree, explicit full-document, and section results report `"tree"`, +`"full"`, and `"section"` respectively. A headingless long document uses the +normal bounded automatic response. `truncated` is true only when that response +is cut by the content-length limit. `mode: "full"` returns the complete +extracted Markdown, while `mode: "tree"` always returns the heading-tree +representation, including the explicit no-headings result. `links` lists up to +100 unique HTTP(S) anchors from the original page DOM. ## MCP @@ -159,8 +162,11 @@ The server exposes six read-only tools: `search`, `fetch`, `links`, `docs_resolv messages; diagnostics go to stderr. For a client-rendered page, explicitly call `fetch` or `links` with `render: "browser"` and an integer `waitMs`; this optional retry requires a host-installed executable and never happens automatically. -The `fetch` tool uses `mode: "auto" | "full" | "tree" | "section"`; pass a -returned `section_id` only with `mode: "section"`. `auto` is request-only. +The `fetch` tool accepts input `mode: "auto" | "full" | "tree"` (default +`"auto"`). Pass a returned `section_id` with omitted mode or `mode: "auto"` +to retrieve that section; `mode: "full"` and `mode: "tree"` reject +`section_id`. Results include `mode: "auto" | "full" | "tree" | "section"` +and `truncated`, which is true only when content was cut by the length limit. ## Pi @@ -175,9 +181,10 @@ the bundled core in-process. Pi and TypeBox are peer dependencies supplied by the host; no CLI executable or MCP configuration is required. `web_fetch` uses HTTP rendering by default and can explicitly use `render: "browser"` with an integer `waitMs` when its host provides that optional executable. -Its navigation input is `mode: "auto" | "full" | "tree" | "section"`; -`section_id` is required exactly for `mode: "section"`, and `auto` is -request-only because results report the resolved mode. +Its navigation input is `mode: "auto" | "full" | "tree"` (default `"auto"`); +`section_id` with omitted/`"auto"` mode retrieves a section, while full/tree +reject it. Results report `mode: "auto" | "full" | "tree" | "section"` and a +`truncated` flag that only indicates content cut by the length limit. `web_links` uses the same explicit rendering contract and lists HTTP(S) anchors from the original page DOM. @@ -208,8 +215,9 @@ also run in-process. The host DSH packages and React are peers supplied by DSH. `render: "browser"` with an integer `waitMs` on a host that supplies the optional executable. Its navigation input uses the same `mode` and `section_id` contract as the -other adapters; `auto` is request-only and Fetch results report `full`, `tree`, -or `section`. +other adapters: input mode is `auto|full|tree` (default `auto`), and omitted or +`auto` mode plus `section_id` retrieves a section. Results report +`auto|full|tree|section` and `truncated`. `web_links` uses the same explicit rendering contract and lists HTTP(S) anchors from the original page DOM. diff --git a/docs/adr/0001-containerized-rest-api.md b/docs/adr/0001-containerized-rest-api.md index c15605f..fd9effa 100644 --- a/docs/adr/0001-containerized-rest-api.md +++ b/docs/adr/0001-containerized-rest-api.md @@ -4,4 +4,4 @@ Guion Web will add a self-hosted, single-user HTTP service in a portable contain ## Consequences -The service's provider credentials, Bridge Route, and optional `WEB_SEARCH_PROVIDER=deepseek` selection are server-local configuration; clients cannot choose a provider or supply a Bridge Route per request. With no provider selection, search tries Kepos Bridge and uses Exa only when the Bridge is operationally unavailable; it does not fall back for cancellation, malformed client input, or an empty result set. A failed Bridge attempt is retried through Exa exactly once, and a successful empty Bridge response is returned unchanged. When DeepSeek is selected, the service calls only DeepSeek and never falls back. DeepSeek performs one auxiliary model call internally and returns the same normalized result contract; its Messages/tool protocol is not exposed to HTTP callers. Typed Bridge operations are intentionally not exposed: Exa has no equivalent official weather, sports, or time API, and its premium finance integration is not contract-compatible. Fetch and Links use `render: "http"` by default or explicit `render: "browser"` with a required `waitMs`; the operator-installed executable remains an implementation detail. Fetch navigation uses `mode: "auto"` by default, with explicit `"full"`, `"tree"`, or `"section"` modes; `section_id` is valid only for `"section"`. `openapi.yaml` is generated from the release build rather than manually maintained or independently versioned; the standalone [HTTP service reference](../http-service.md) is its human-readable companion. Public or multi-tenant deployment hardening is deliberately deferred in `.scratch/defered/public-http-service-security.md`. +The service's provider credentials, Bridge Route, and optional `WEB_SEARCH_PROVIDER=deepseek` selection are server-local configuration; clients cannot choose a provider or supply a Bridge Route per request. With no provider selection, search tries Kepos Bridge and uses Exa only when the Bridge is operationally unavailable; it does not fall back for cancellation, malformed client input, or an empty result set. A failed Bridge attempt is retried through Exa exactly once, and a successful empty Bridge response is returned unchanged. When DeepSeek is selected, the service calls only DeepSeek and never falls back. DeepSeek performs one auxiliary model call internally and returns the same normalized result contract; its Messages/tool protocol is not exposed to HTTP callers. Typed Bridge operations are intentionally not exposed: Exa has no equivalent official weather, sports, or time API, and its premium finance integration is not contract-compatible. Fetch and Links use `render: "http"` by default or explicit `render: "browser"` with a required `waitMs`; the operator-installed executable remains an implementation detail. Fetch navigation input uses `mode: "auto"` by default, with explicit `"full"` or `"tree"` modes; a non-empty `section_id` with omitted or `"auto"` mode retrieves a section, while full/tree reject it. Results report `"auto"`, `"full"`, `"tree"`, or `"section"` as appropriate and include a `truncated` flag for content cut by the Core limit. `openapi.yaml` is generated from the release build rather than manually maintained or independently versioned; the standalone [HTTP service reference](../http-service.md) is its human-readable companion. Public or multi-tenant deployment hardening is deliberately deferred in `.scratch/defered/public-http-service-security.md`. diff --git a/docs/http-service.md b/docs/http-service.md index 41c247a..0eaadb2 100644 --- a/docs/http-service.md +++ b/docs/http-service.md @@ -55,7 +55,6 @@ Request: { "url": "https://example.test/article", "render": "http", - "mode": "section", "section_id": "7i" } ``` @@ -65,8 +64,9 @@ must be exactly `"http"` or `"browser"`; omission selects `"http"`. `waitMs` is an integer from 0 through 30,000, required with `render: "browser"` and forbidden with `render: "http"` (or when `render` is omitted). `mode` is optional and defaults to `"auto"`; it must be one of `"auto"`, `"full"`, -`"tree"`, or `"section"`. `section_id` is required exactly when `mode` is -`"section"` and is rejected for every other mode. +or `"tree"`. A non-empty `section_id` may be supplied with omitted mode or +`mode: "auto"` to retrieve that section; it is rejected with `"full"` or +`"tree"`. HTTP rendering fetches the page with Node HTTP, linkedom, and Defuddle. Browser rendering invokes the operator-installed `agent-browser` executable @@ -77,12 +77,15 @@ the service does not fall back between renderers. The shared module owns the 5,000-character automatic-tree policy. An `"auto"` request for an unsectioned document longer than that threshold with navigable headings returns `mode: "tree"` with stable section IDs. Use one of those IDs -in a subsequent request with `mode: "section"` to retrieve a section. A long -document without headings uses the normal bounded `mode: "full"` response -because it has no section to navigate. `mode: "full"` returns the complete -extracted Markdown without the Core content limit. `mode: "tree"` always -returns the heading-tree representation, including the explicit no-headings -result. A short `"auto"` request uses `mode: "full"`; `"auto"` is input-only. +in a subsequent request with omitted mode or `mode: "auto"` to retrieve a +section. A short automatic response and a headingless long document report +`mode: "auto"`; the latter remains bounded by the Core content limit. +`mode: "full"` returns the complete extracted Markdown without that limit and +reports `mode: "full"`. `mode: "tree"` always returns the heading-tree +representation, including the explicit no-headings result, and reports +`mode: "tree"`. Section requests report `mode: "section"`. Every response +includes `truncated`, which is true only when ordinary automatic content was +cut by the Core content-length limit. Response `200`: @@ -90,7 +93,8 @@ Response `200`: { "url": "https://example.test/article", "mode": "full", - "content": "# Article\n...\n" + "content": "# Article\n...\n", + "truncated": false } ``` diff --git a/packages/dsh-web/README.md b/packages/dsh-web/README.md index 239eecc..61340d9 100644 --- a/packages/dsh-web/README.md +++ b/packages/dsh-web/README.md @@ -50,16 +50,19 @@ It uses the same HTTP default and explicit `render: "browser"` / required Long `web_fetch` documents with navigable headings return a navigation tree automatically when `mode: "auto"` (the default). A headingless long document uses the normal bounded response. Set `mode: "full"` for complete Markdown, -`mode: "tree"` to force the heading tree, or `mode: "section"` with a returned -`section_id` to continue with one section. `section_id` is required exactly for -`mode: "section"`; `"auto"` is request-only because results report the actual -`mode` as `"full"`, `"tree"`, or `"section"`. +`mode: "tree"` to force the heading tree, or supply a returned `section_id` +with omitted mode or `mode: "auto"` to continue with one section. Input mode +`"section"` is not supported, and `section_id` is rejected with `"full"` or +`"tree"`. Ordinary automatic document results report `mode: "auto"`; tree, +full, and section results report `"tree"`, `"full"`, and `"section"`. +Every result includes `truncated`, which is true only when automatic content +was cut by the content-length limit. For example, request a tree and then continue with one returned section: ```json { "url": "https://example.test/article", "mode": "tree" } -{ "url": "https://example.test/article", "mode": "section", "section_id": "7i" } +{ "url": "https://example.test/article", "section_id": "7i" } ``` Rendered requests are bounded and constrained to the requested hostname, diff --git a/packages/dsh-web/src/client.ts b/packages/dsh-web/src/client.ts index b5cb0f4..4344c35 100644 --- a/packages/dsh-web/src/client.ts +++ b/packages/dsh-web/src/client.ts @@ -313,7 +313,7 @@ function formatWait(waitMs: number): string { function fetchResultMode(args: Record): string { if ( - args.mode === "section" && + (args.mode === undefined || args.mode === "auto") && typeof args.section_id === "string" && args.section_id !== "" ) diff --git a/packages/dsh-web/src/tools.ts b/packages/dsh-web/src/tools.ts index a9f1bd3..d22e11a 100644 --- a/packages/dsh-web/src/tools.ts +++ b/packages/dsh-web/src/tools.ts @@ -48,11 +48,12 @@ const fetchParameters = { type: "string", enum: [...FETCH_MODES], default: "auto", - description: "Navigation mode: auto (default), full, tree, or section", + description: "Navigation mode: auto (default), full, or tree", }, section_id: { type: "string", - description: "Heading section ID; required only with mode section", + description: + "Heading section ID; retrieves a section with omitted/auto mode", }, render: { type: "string", @@ -205,10 +206,11 @@ const fetchOutput = { url: { type: "string", required: true }, mode: { type: "string", - enum: ["full", "tree", "section"], + enum: ["auto", "full", "tree", "section"], required: true, }, content: { type: "string", required: true }, + truncated: { type: "boolean", required: true }, }, } as const, render: (_args: unknown, value: FetchResult) => [ @@ -216,7 +218,7 @@ const fetchOutput = { type: "text" as const, text: boundedToolText( value.content, - 'Use web_fetch with mode: "full" for the complete document, or mode: "section" with a returned section_id to navigate the document.', + 'Use web_fetch with mode: "full" for the complete document, or section_id with the default/auto mode to navigate to a section.', ), }, ], @@ -392,7 +394,7 @@ function normalizeFetch(input: unknown): FetchInput { const url = requireString(input, "url"); const mode = input.mode; if (mode !== undefined && !FETCH_MODES.includes(mode as FetchMode)) - throw new Error('mode must be one of "auto", "full", "tree", or "section"'); + throw new Error('mode must be one of "auto", "full", or "tree"'); const selectedMode = mode as FetchMode | undefined; const sectionID = input.section_id; if ( @@ -400,12 +402,12 @@ function normalizeFetch(input: unknown): FetchInput { (typeof sectionID !== "string" || sectionID.trim().length === 0) ) throw new Error("section_id must be a non-empty string"); - if (selectedMode === "section") { - if (sectionID === undefined) - throw new Error('section_id is required when mode is "section"'); - } else if (sectionID !== undefined) { - throw new Error('section_id is only valid with mode "section"'); - } + if ( + sectionID !== undefined && + selectedMode !== undefined && + selectedMode !== "auto" + ) + throw new Error('section_id is only valid with mode "auto"'); const renderOptions = validateRenderOptions(input); return { url, @@ -498,7 +500,7 @@ function webFetchTool( defineTool({ name: "web_fetch", description: - "Use HTTP rendering for static, SSR, and pre-rendered pages. mode selects auto, full, tree, or section navigation; mode section requires section_id. For client-rendered or SPA pages, set render: browser with required waitMs when the host provides browser capability; there is no automatic fallback.", + "Use HTTP rendering for static, SSR, and pre-rendered pages. mode selects auto, full, or tree navigation; section_id with omitted/auto mode retrieves a section. For client-rendered or SPA pages, set render: browser with required waitMs when the host provides browser capability; there is no automatic fallback.", parameters: fetchParameters, output: fetchOutput, isConcurrencySafe: () => true, diff --git a/packages/dsh-web/test/client.test.ts b/packages/dsh-web/test/client.test.ts index c17eea7..a0c287c 100644 --- a/packages/dsh-web/test/client.test.ts +++ b/packages/dsh-web/test/client.test.ts @@ -56,7 +56,7 @@ describe("DSH settings client credential surface", () => { fetchDetails({ render: "browser", waitMs: 2_000, - mode: "section", + mode: "auto", section_id: "installation", }), ).toEqual([ @@ -64,6 +64,10 @@ describe("DSH settings client credential surface", () => { { label: "Wait", value: "2 s" }, { label: "Result", value: "Section: installation" }, ]); + expect(fetchDetails({ section_id: "installation" })).toEqual([ + { label: "Backend", value: "HTTP rendered" }, + { label: "Result", value: "Section: installation" }, + ]); expect(fetchDetails({})).toEqual([ { label: "Backend", value: "HTTP rendered" }, { label: "Result", value: "Automatic navigation" }, diff --git a/packages/dsh-web/test/tools.test.ts b/packages/dsh-web/test/tools.test.ts index 5831e21..d5e0b71 100644 --- a/packages/dsh-web/test/tools.test.ts +++ b/packages/dsh-web/test/tools.test.ts @@ -90,7 +90,6 @@ describe("DSH direct web tools", () => { "auto", "full", "tree", - "section", ]); expect((definitions[0]!.parameters as any).properties.waitMs.type).toBe( "integer", @@ -119,7 +118,12 @@ describe("DSH direct web tools", () => { operations: operations({ fetch: async (input, abortSignal) => { calls.push({ kind: "fetch", input, abortSignal }); - return { url: input.url, mode: "section", content: "selected" }; + return { + url: input.url, + mode: "section", + content: "selected", + truncated: false, + }; }, links: async (input, abortSignal) => { calls.push({ kind: "links", input, abortSignal }); @@ -154,14 +158,21 @@ describe("DSH direct web tools", () => { fetch!, { url: "https://example.test", - mode: "section", + mode: "auto", section_id: "install", render: "browser", waitMs: 2000, }, controller.signal, ), - ).resolves.toMatchObject({ mode: "section" }); + ).resolves.toMatchObject({ mode: "section", truncated: false }); + await expect( + call( + fetch!, + { url: "https://example.test", section_id: "install" }, + controller.signal, + ), + ).resolves.toMatchObject({ mode: "section", truncated: false }); await expect( call( links!, @@ -198,13 +209,21 @@ describe("DSH direct web tools", () => { kind: "fetch", input: { url: "https://example.test", - mode: "section", + mode: "auto", section_id: "install", render: "browser", waitMs: 2000, }, abortSignal: controller.signal, }, + { + kind: "fetch", + input: { + url: "https://example.test", + section_id: "install", + }, + abortSignal: controller.signal, + }, { kind: "links", input: { @@ -245,22 +264,22 @@ describe("DSH direct web tools", () => { it("preserves shared fetch validation, cancellation, and structured renderer failures", async () => { const fetch = createWebToolDefinitions(dependencies())[0]!; - await expect( - call(fetch, { url: "https://example.test", mode: "section" }), - ).rejects.toThrow('section_id is required when mode is "section"'); await expect( call(fetch, { url: "https://example.test", mode: "full", section_id: "intro", }), - ).rejects.toThrow('section_id is only valid with mode "section"'); + ).rejects.toThrow('section_id is only valid with mode "auto"'); await expect( call(fetch, { url: "https://example.test", full: true }), ).rejects.toThrow(/does not accept field full/); await expect( call(fetch, { url: "https://example.test", mode: "invalid" }), - ).rejects.toThrow(/mode.*auto.*full.*tree.*section/); + ).rejects.toThrow(/mode.*auto.*full.*tree/); + await expect( + call(fetch, { url: "https://example.test", mode: "section" }), + ).rejects.toThrow(/mode.*auto.*full.*tree/); await expect( call(fetch, { url: "https://example.test", @@ -610,6 +629,7 @@ describe("DSH direct web tools", () => { url: input.url, mode: "full" as const, content: "fixture", + truncated: false, })); const [fetchTool] = createWebToolDefinitions( dependencies({ operations: operations({ fetch }) }), diff --git a/packages/pi-web/src/tool.ts b/packages/pi-web/src/tool.ts index aa26868..7c2cd1d 100644 --- a/packages/pi-web/src/tool.ts +++ b/packages/pi-web/src/tool.ts @@ -41,12 +41,13 @@ const fetchNavigationProperties = { mode: Type.Optional( StringEnum(FETCH_MODES, { default: "auto", - description: "Navigation mode: auto (default), full, tree, or section", + description: "Navigation mode: auto (default), full, or tree", }), ), section_id: Type.Optional( Type.String({ - description: "Heading section ID; required only with mode section", + description: + "Heading section ID; retrieves a section with omitted/auto mode", minLength: 1, pattern: "\\S", }), @@ -87,14 +88,16 @@ export const webFetchSchema = { { oneOf: [ { - properties: { - mode: { enum: ["auto", "full", "tree"] }, - }, + required: ["mode"], + properties: { mode: { const: "auto" } }, + }, + { + required: ["mode"], + properties: { mode: { enum: ["full", "tree"] } }, not: { required: ["section_id"] }, }, { - properties: { mode: { const: "section" } }, - required: ["mode", "section_id"], + not: { required: ["mode"] }, }, ], }, @@ -212,7 +215,7 @@ const SEARCH_PROMPT_GUIDELINES = [ "Use web_search to search the web for current facts.", ]; const FETCH_PROMPT_GUIDELINES = [ - 'Use web_fetch to read a web page; mode: "auto" is the default and long documents with navigable headings return a navigation tree. Follow up with mode: "section" and a returned section_id, or mode: "full" to read everything. Headingless long documents use the normal bounded response.', + 'Use web_fetch to read a web page; mode: "auto" is the default and long documents with navigable headings return a navigation tree. Follow up with section_id using omitted/auto mode to read a section, or use mode: "full" to read everything. Headingless long documents use the normal bounded response.', 'web_fetch uses HTTP fetching by default; set render: "http" explicitly when desired.', 'For a client-rendered or SPA page, or after javascript_rendering_may_be_required, retry explicitly with render: "browser" and waitMs: 2000 only when the host provides browser capability. Increase waitMs explicitly or abandon an incomplete page; there is no automatic fallback.', "Never send waitMs with HTTP rendering. The browser is a host capability, not a package dependency.", @@ -286,16 +289,13 @@ function normalizeFetch(input: unknown): WebFetchInput { const renderOptions = validateRenderOptions(input); const mode = input.mode; if (mode !== undefined && !FETCH_MODES.includes(mode as FetchMode)) - throw new Error('mode must be one of "auto", "full", "tree", or "section"'); + throw new Error('mode must be one of "auto", "full", or "tree"'); if (input.section_id !== undefined) { if (typeof input.section_id !== "string" || input.section_id.trim() === "") throw new Error("section_id must be a non-empty string"); } - if (mode === "section") { - if (input.section_id === undefined) - throw new Error('section_id is required when mode is "section"'); - } else if (input.section_id !== undefined) { - throw new Error('section_id is only valid with mode "section"'); + if ((mode === "full" || mode === "tree") && input.section_id !== undefined) { + throw new Error('section_id is only valid with mode "auto"'); } const typed = input as unknown as WebFetchInput; const navigation = { @@ -448,14 +448,14 @@ export function webFetchTool(dependencies: WebToolDependencies = {}) { name: "web_fetch", label: "Web fetch", description: - "Fetch and read an HTTP or HTTPS web page as Markdown, with HTTP rendering by default or explicit browser rendering for client-rendered pages. Browser rendering requires waitMs 0 through 30000. mode selects auto, full, tree, or section navigation; mode section requires section_id. Text output is limited to 2,000 lines or 50KB; truncated output is saved to a temporary file.", + "Fetch and read an HTTP or HTTPS web page as Markdown, with HTTP rendering by default or explicit browser rendering for client-rendered pages. Browser rendering requires waitMs 0 through 30000. mode selects auto, full, or tree navigation; section_id with omitted/auto mode retrieves a section. Text output is limited to 2,000 lines or 50KB; truncated output is saved to a temporary file.", promptSnippet: "Fetch a web page with web_fetch", promptGuidelines: FETCH_PROMPT_GUIDELINES, parameters: webFetchSchema, execute: async (params, signal) => { const data = await operations.fetch(normalizeFetch(params), signal); return modelTextResult(data, data.content, { - hint: 'Use web_fetch with mode: "full" for the complete document, or mode: "section" with a returned section_id to navigate the document.', + hint: 'Use web_fetch with mode: "full" for the complete document, or section_id with omitted/auto mode to navigate to a section.', }); }, }); diff --git a/packages/pi-web/test/extension.test.ts b/packages/pi-web/test/extension.test.ts index e9118b3..f27296f 100644 --- a/packages/pi-web/test/extension.test.ts +++ b/packages/pi-web/test/extension.test.ts @@ -106,7 +106,13 @@ describe("pi-web extension", () => { expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - mode: "section", + section_id: "intro", + }), + ).toBe(true); + expect( + Value.Check(webFetchSchema, { + url: "https://fixture.test", + mode: "auto", section_id: "intro", }), ).toBe(true); @@ -153,13 +159,14 @@ describe("pi-web extension", () => { expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - mode: "section", + mode: "full", + section_id: "intro", }), ).toBe(false); expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - mode: "full", + mode: "tree", section_id: "intro", }), ).toBe(false); @@ -175,6 +182,12 @@ describe("pi-web extension", () => { mode: "invalid", }), ).toBe(false); + expect( + Value.Check(webFetchSchema, { + url: "https://fixture.test", + mode: "section", + }), + ).toBe(false); const fetch = vi.fn( async (input: { @@ -186,6 +199,7 @@ describe("pi-web extension", () => { url: input.url, mode: "full" as const, content: "Rendered Markdown", + truncated: false, }), ); const tool = webFetchTool({ operations: operations({ fetch }) }); @@ -205,6 +219,27 @@ describe("pi-web extension", () => { undefined, ); expect(result.content[0]?.text).toBe("Rendered Markdown"); + expect(result.details).toMatchObject({ mode: "full", truncated: false }); + + await call(tool, { + url: "https://fixture.test", + section_id: "intro", + }); + expect(fetch).toHaveBeenNthCalledWith( + 2, + { url: "https://fixture.test", section_id: "intro" }, + undefined, + ); + await call(tool, { + url: "https://fixture.test", + mode: "auto", + section_id: "intro", + }); + expect(fetch).toHaveBeenNthCalledWith( + 3, + { url: "https://fixture.test", mode: "auto", section_id: "intro" }, + undefined, + ); await expect( call(tool, { @@ -219,7 +254,10 @@ describe("pi-web extension", () => { waitMs: 0, }), ).rejects.toThrow("waitMs is only valid"); - expect(fetch).toHaveBeenCalledTimes(1); + await expect( + call(tool, { url: "https://fixture.test", mode: "section" }), + ).rejects.toThrow(/mode.*auto.*full.*tree/); + expect(fetch).toHaveBeenCalledTimes(3); expect(Value.Check(webLinksSchema, { url: "https://fixture.test" })).toBe( true, @@ -529,6 +567,7 @@ describe("pi-web extension", () => { url: "https://fixture.test", mode: "full", content, + truncated: false, }), }), }); @@ -543,7 +582,7 @@ describe("pi-web extension", () => { expect(details.url).toBe("https://fixture.test"); expect(details.truncation.truncated).toBe(true); expect(result.content[0]?.text).toContain( - 'Use web_fetch with mode: "full" for the complete document, or mode: "section" with a returned section_id to navigate the document.', + 'Use web_fetch with mode: "full" for the complete document, or section_id with omitted/auto mode to navigate to a section.', ); expect(await readFile(details.fullOutputPath, "utf8")).toBe(content); } finally { diff --git a/packages/pi-web/test/packed-smoke.mjs b/packages/pi-web/test/packed-smoke.mjs index d54e2f6..6bc27be 100644 --- a/packages/pi-web/test/packed-smoke.mjs +++ b/packages/pi-web/test/packed-smoke.mjs @@ -194,7 +194,10 @@ if (command === "open" && args.some((value) => value.includes("/blocked"))) { url: "https://93.184.216.34/direct", mode: "full", }); - if (!direct.content[0]?.text.includes("Browserless output.")) + if ( + !direct.content[0]?.text.includes("Browserless output.") || + direct.details?.truncated !== false + ) throw new Error("packed extension did not execute browserless fetch"); try { await readFile(fakeLog, "utf8"); diff --git a/packages/web-core/src/fetch.ts b/packages/web-core/src/fetch.ts index 0f32fe9..f5ad4f5 100644 --- a/packages/web-core/src/fetch.ts +++ b/packages/web-core/src/fetch.ts @@ -53,8 +53,9 @@ export type FetchInput = { export type FetchResult = { url: string; - mode: "full" | "tree" | "section"; + mode: "auto" | "full" | "tree" | "section"; content: string; + truncated: boolean; }; export type LinksInput = { @@ -136,7 +137,12 @@ export async function fetchWebPage( mode, section_id: input.section_id, }); - return { url, mode: rendered.mode, content: rendered.content }; + return { + url, + mode: rendered.mode, + content: rendered.content, + truncated: rendered.truncated, + }; } /** Lists HTTP(S) links from the original or browser-rendered page DOM. */ @@ -211,14 +217,9 @@ function validateNavigationInput( ): FetchMode { const mode = input.mode ?? "auto"; if (!FETCH_MODES.includes(mode as FetchMode)) - throw new Error('mode must be one of "auto", "full", "tree", or "section"'); - if (mode === "section") { - if (input.section_id === undefined) - throw new Error('section_id is required when mode is "section"'); - return mode; - } - if (input.section_id !== undefined) - throw new Error('section_id is only valid with mode "section"'); + throw new Error('mode must be one of "auto", "full", or "tree"'); + if (input.section_id !== undefined && mode !== "auto") + throw new Error('section_id is only valid with mode "auto"'); return mode; } diff --git a/packages/web-core/src/markdown.ts b/packages/web-core/src/markdown.ts index e1e1fa3..8ea9a1e 100644 --- a/packages/web-core/src/markdown.ts +++ b/packages/web-core/src/markdown.ts @@ -15,10 +15,11 @@ type Heading = { export type MarkdownResult = { content: string; - mode: "full" | "tree" | "section"; + mode: "auto" | "full" | "tree" | "section"; + truncated: boolean; }; -export const FETCH_MODES = ["auto", "full", "tree", "section"] as const; +export const FETCH_MODES = ["auto", "full", "tree"] as const; export type FetchMode = (typeof FETCH_MODES)[number]; export type MarkdownNavigationOptions = { @@ -42,31 +43,38 @@ export function renderMarkdown( const headings = assignIds(parseHeadings(source)); const mode = options.mode ?? "auto"; const section = options.section_id?.trim(); - if (mode === "section") { - if (!section) - throw new Error('section_id is required when mode is "section"'); + if (section !== undefined) { + if (mode !== "auto") + throw new Error('section_id is only valid with mode "auto"'); return { content: extractSection(source, headings, section), mode: "section", + truncated: false, }; } - if (section !== undefined) - throw new Error('section_id is only valid with mode "section"'); - if (mode === "tree") return { content: renderTree(source, headings), mode }; + if (mode === "tree") + return { content: renderTree(source, headings), mode, truncated: false }; if (mode === "full") return { content: source, mode: "full", + truncated: false, }; const charCount = Array.from(source).length; if (charCount > DEFAULT_TREE_THRESHOLD && headings.length > 0) - return { content: renderTree(source, headings), mode: "tree" }; + return { + content: renderTree(source, headings), + mode: "tree", + truncated: false, + }; + const truncated = charCount > MAX_CONTENT_CHARS; return { content: truncateContent(source), - mode: "full", + mode: "auto", + truncated, }; } @@ -78,7 +86,7 @@ function validateNavigationOptions(options: object): void { const mode = (options as { mode?: unknown }).mode; if (mode !== undefined && !FETCH_MODES.includes(mode as FetchMode)) - throw new Error('mode must be one of "auto", "full", "tree", or "section"'); + throw new Error('mode must be one of "auto", "full", or "tree"'); const section = (options as { section_id?: unknown }).section_id; if ( @@ -223,7 +231,7 @@ function renderTree(source: string, headings: Heading[]): string { `└── [${heading.id}] ${"#".repeat(heading.level)} ${heading.text} (${formatNumber(sectionCharCount(source, headings, index))} chars)\n`, ) .join(""); - return `${header}${tree}\nUse mode: "section" with section_id to read a section, or mode: "full" to read everything.\n`; + return `${header}${tree}\nUse section_id with the default or mode: "auto" to read a section, or mode: "full" to read everything.\n`; } const nodes = bodyHeadings.map((heading) => { @@ -252,7 +260,7 @@ function renderTree(source: string, headings: Heading[]): string { hasMore.delete(depth); }); - return `${header}${tree}\nUse mode: "section" with section_id to read a section, or mode: "full" to read everything.\n`; + return `${header}${tree}\nUse section_id with the default or mode: "auto" to read a section, or mode: "full" to read everything.\n`; } function sectionCharCount( diff --git a/packages/web-core/test/fetch.test.ts b/packages/web-core/test/fetch.test.ts index 0754b60..7b0b41e 100644 --- a/packages/web-core/test/fetch.test.ts +++ b/packages/web-core/test/fetch.test.ts @@ -146,6 +146,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { url, mode: "full", content: "Extracted text.\n", + truncated: false, }); }); } finally { @@ -783,12 +784,20 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebPage({ ...input, mode: "tree" }, undefined, { cache }), ).resolves.toMatchObject({ mode: "tree" }); await expect( - fetchWebPage({ ...input, mode: "section", section_id: "7i" }, undefined, { + fetchWebPage({ ...input, section_id: "7i" }, undefined, { cache }), + ).resolves.toMatchObject({ + mode: "section", + content: "## Install\nInstall content.\n", + truncated: false, + }); + await expect( + fetchWebPage({ ...input, mode: "auto", section_id: "7i" }, undefined, { cache, }), ).resolves.toMatchObject({ mode: "section", content: "## Install\nInstall content.\n", + truncated: false, }); const complete = "# Complete\n\n" + "x".repeat(30_001); @@ -806,6 +815,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { url: "https://navigation.test/complete", mode: "full", content: complete, + truncated: false, }); await expect( @@ -818,35 +828,33 @@ describe.sequential("browserless fetch migrated from Organon", () => { undefined, { cache }, ), - ).rejects.toThrow('section_id is only valid with mode "section"'); + ).rejects.toThrow('section_id is only valid with mode "auto"'); await expect( fetchWebPage({ url: input.url, tree: true } as never, undefined, { cache, }), ).rejects.toThrow("does not accept field tree"); await expect( - fetchWebPage({ ...input, mode: "section" } as never, undefined, { + fetchWebPage({ ...input, mode: "tree", section_id: "7i" }, undefined, { cache, }), - ).rejects.toThrow('section_id is required when mode is "section"'); + ).rejects.toThrow('section_id is only valid with mode "auto"'); await expect( - fetchWebPage({ ...input, mode: "auto", section_id: "7i" }, undefined, { + fetchWebPage({ ...input, mode: "section" } as never, undefined, { cache, }), - ).rejects.toThrow('section_id is only valid with mode "section"'); + ).rejects.toThrow('mode must be one of "auto", "full", or "tree"'); await expect( fetchWebPage({ ...input, mode: "invalid" } as never, undefined, { cache, }), - ).rejects.toThrow( - 'mode must be one of "auto", "full", "tree", or "section"', - ); + ).rejects.toThrow('mode must be one of "auto", "full", or "tree"'); await expect( fetchWebPage({ url: input.url, tree_threshold: 1 } as never, undefined, { cache, }), ).rejects.toThrow("does not accept field tree_threshold"); - expect(cache.read).toHaveBeenCalledTimes(3); + expect(cache.read).toHaveBeenCalledTimes(4); }); }); diff --git a/packages/web-core/test/markdown.test.ts b/packages/web-core/test/markdown.test.ts index d4fb7c1..1661d14 100644 --- a/packages/web-core/test/markdown.test.ts +++ b/packages/web-core/test/markdown.test.ts @@ -46,11 +46,10 @@ describe("Markdown navigation", () => { /\[([0-9A-Za-z]{2,3})\] ## Quote emphasis/, )?.[1]; expect(quoteID).toBeDefined(); - expect( - renderMarkdown(source, { mode: "section", section_id: quoteID }), - ).toEqual({ + expect(renderMarkdown(source, { section_id: quoteID })).toEqual({ mode: "section", content: "> ## Quote *emphasis*\n\n- ### List heading\n", + truncated: false, }); }); @@ -62,25 +61,32 @@ describe("Markdown navigation", () => { expect(tree.content).toContain("[7i] ## Install"); expect(tree.content).toContain("[eD] ### Details"); - expect( - renderMarkdown(source, { mode: "section", section_id: "7i" }), - ).toEqual({ + expect(renderMarkdown(source, { section_id: "7i" })).toEqual({ mode: "section", content: "## Install\nInstall content.\n\n### Details\nDetails content.\n", + truncated: false, }); - expect(() => - renderMarkdown(source, { mode: "section", section_id: "missing" }), - ).toThrow('section "missing" not found'); + expect(() => renderMarkdown(source, { section_id: "missing" })).toThrow( + 'section "missing" not found', + ); expect(renderMarkdown("plain content")).toEqual({ content: "plain content", - mode: "full", + mode: "auto", + truncated: false, }); const complete = "# H\n\n" + "x".repeat(30_001); expect(renderMarkdown(complete, { mode: "full" })).toEqual({ content: complete, mode: "full", + truncated: false, }); + const bounded = renderMarkdown("x".repeat(30_001)); + expect(bounded.mode).toBe("auto"); + expect(bounded.truncated).toBe(true); + expect(bounded.content).toContain( + "[content truncated at 30000 characters]", + ); }); it("lists and retrieves an H1-only long document section", () => { @@ -92,25 +98,26 @@ describe("Markdown navigation", () => { expect(tree.mode).toBe("tree"); expect(sectionID).toBeDefined(); - expect( - renderMarkdown(source, { mode: "section", section_id: sectionID }), - ).toEqual({ + expect(renderMarkdown(source, { section_id: sectionID })).toEqual({ content: source, mode: "section", + truncated: false, }); }); - it("keeps headingless long documents bounded and selectable only by headings", () => { + it("keeps headingless long documents in auto mode and bounded when necessary", () => { const source = "x".repeat(5001); expect(renderMarkdown(source)).toEqual({ content: source, - mode: "full", + mode: "auto", + truncated: false, }); const complete = "x".repeat(30_001); expect(renderMarkdown(complete, { mode: "full" })).toEqual({ content: complete, mode: "full", + truncated: false, }); }); @@ -118,25 +125,36 @@ describe("Markdown navigation", () => { expect(renderMarkdown("short content", { mode: "tree" })).toEqual({ content: "(no headings)\n", mode: "tree", + truncated: false, }); expect( renderMarkdown("# Heading\n\ncontent\n", { mode: "tree" }).mode, ).toBe("tree"); }); - it("requires section mode and rejects incompatible navigation fields", () => { - expect(() => renderMarkdown("# Heading\n")).not.toThrow(); - expect(() => renderMarkdown("# Heading\n", { mode: "section" })).toThrow( - 'section_id is required when mode is "section"', - ); + it("allows auto section continuation and rejects incompatible navigation fields", () => { + const source = "# Heading\n\n## Install\nInstall content.\n"; + expect(renderMarkdown(source, { section_id: "7i" })).toEqual({ + mode: "section", + content: "## Install\nInstall content.\n", + truncated: false, + }); + expect(renderMarkdown(source, { mode: "auto", section_id: "7i" })).toEqual({ + mode: "section", + content: "## Install\nInstall content.\n", + truncated: false, + }); expect(() => - renderMarkdown("# Heading\n", { mode: "auto", section_id: "x" }), - ).toThrow('section_id is only valid with mode "section"'); + renderMarkdown("# Heading\n", { mode: "section" as never }), + ).toThrow('mode must be one of "auto", "full", or "tree"'); expect(() => renderMarkdown("# Heading\n", { mode: "full", section_id: "x" }), - ).toThrow('section_id is only valid with mode "section"'); + ).toThrow('section_id is only valid with mode "auto"'); + expect(() => + renderMarkdown("# Heading\n", { mode: "tree", section_id: "x" }), + ).toThrow('section_id is only valid with mode "auto"'); expect(() => renderMarkdown("# Heading\n", { mode: "invalid" as never }), - ).toThrow('mode must be one of "auto", "full", "tree", or "section"'); + ).toThrow('mode must be one of "auto", "full", or "tree"'); }); }); diff --git a/packages/web/src/http.ts b/packages/web/src/http.ts index c8d3959..bde5415 100644 --- a/packages/web/src/http.ts +++ b/packages/web/src/http.ts @@ -74,8 +74,9 @@ const SearchResponseSchema = z const FetchResponseSchema = z .object({ url: z.string(), - mode: z.enum(["full", "tree", "section"]), + mode: z.enum(["auto", "full", "tree", "section"]), content: z.string(), + truncated: z.boolean(), }) .strict() .openapi("FetchResponse"); @@ -108,8 +109,7 @@ const FetchRequestSchema = z .object({ url: HttpUrlSchema, mode: z.enum(FETCH_MODES).default("auto").openapi({ - description: - 'Navigation mode: "auto" (default), "full", "tree", or "section".', + description: 'Navigation mode: "auto" (default), "full", or "tree".', }), section_id: z .string() @@ -124,16 +124,10 @@ const FetchRequestSchema = z }) .strict() .superRefine((input, context) => { - if (input.mode === "section" && input.section_id === undefined) { - context.addIssue({ - code: "custom", - message: 'section_id is required when mode is "section"', - path: ["section_id"], - }); - } else if (input.mode !== "section" && input.section_id !== undefined) { + if (input.mode !== "auto" && input.section_id !== undefined) { context.addIssue({ code: "custom", - message: 'section_id is only valid with mode "section"', + message: 'section_id is only valid with mode "auto"', path: ["section_id"], }); } @@ -141,14 +135,16 @@ const FetchRequestSchema = z .openapi("FetchRequest", { oneOf: [ { - properties: { - mode: { enum: ["auto", "full", "tree"] }, - }, + required: ["mode"], + properties: { mode: { const: "auto" } }, + }, + { + required: ["mode"], + properties: { mode: { enum: ["full", "tree"] } }, not: { required: ["section_id"] }, }, { - properties: { mode: { const: "section" } }, - required: ["mode", "section_id"], + not: { required: ["mode"] }, }, ], }); @@ -207,7 +203,7 @@ const fetchRoute = createRoute({ operationId: "fetch", summary: "Fetch a web page", description: - 'Fetch through HTTP by default with auto navigation; use mode "full", "tree", or "section" for explicit navigation. Browser rendering requires render=browser and waitMs.', + 'Fetch through HTTP by default with auto navigation; use mode "full" or "tree" for explicit navigation. Supply section_id with omitted mode or mode "auto" to retrieve a section. Browser rendering requires render=browser and waitMs.', request: jsonRequest(FetchRequestSchema), responses: { 200: jsonResponse(FetchResponseSchema, "Fetched page."), diff --git a/packages/web/src/mcp.ts b/packages/web/src/mcp.ts index 1afcd72..76d8794 100644 --- a/packages/web/src/mcp.ts +++ b/packages/web/src/mcp.ts @@ -67,13 +67,14 @@ const fetchInputSchema = schema({ type: "string", enum: [...FETCH_MODES], default: "auto", - description: "navigation mode: auto (default), full, tree, or section", + description: "navigation mode: auto (default), full, or tree", }, section_id: { type: "string", minLength: 1, pattern: "\\S", - description: "heading section ID; required only with mode section", + description: + "heading section ID; retrieves a section with omitted/auto mode", }, render: { type: "string", @@ -103,14 +104,16 @@ const fetchInputSchema = schema({ { oneOf: [ { - properties: { - mode: { enum: FETCH_MODES.filter((mode) => mode !== "section") }, - }, + required: ["mode"], + properties: { mode: { const: "auto" } }, + }, + { + required: ["mode"], + properties: { mode: { enum: ["full", "tree"] } }, not: { required: ["section_id"] }, }, { - properties: { mode: { const: "section" } }, - required: ["mode", "section_id"], + not: { required: ["mode"] }, }, ], }, @@ -226,10 +229,11 @@ const fetchOutputSchema = schema({ type: "object", properties: { url: { type: "string" }, - mode: { type: "string", enum: ["full", "tree", "section"] }, + mode: { type: "string", enum: ["auto", "full", "tree", "section"] }, content: { type: "string" }, + truncated: { type: "boolean" }, }, - required: ["url", "mode", "content"], + required: ["url", "mode", "content", "truncated"], }); const linksOutputSchema = schema({ type: "object", @@ -346,7 +350,7 @@ export function createMcpServer(dependencies: McpDependencies): McpServer { "fetch", toolConfig( "Fetch a web page", - "Use the default HTTP renderer for static, SSR, and pre-rendered pages. For client-rendered or SPA pages, set render: browser with required waitMs when the host provides browser capability; there is no automatic fallback.", + "Use the default HTTP renderer for static, SSR, and pre-rendered pages. Input mode selects auto, full, or tree navigation; section_id with omitted/auto mode retrieves a section. For client-rendered or SPA pages, set render: browser with required waitMs when the host provides browser capability; there is no automatic fallback.", fetchInputSchema, fetchOutputSchema, ), diff --git a/packages/web/src/program.ts b/packages/web/src/program.ts index 18b0b15..9ea2653 100644 --- a/packages/web/src/program.ts +++ b/packages/web/src/program.ts @@ -247,7 +247,7 @@ function createFetchCommand(dependencies: ProgramDependencies): Command { ) .option( "--mode ", - "Navigation mode: auto (default), full, tree, or section", + "Navigation mode: auto (default), full, or tree", parseFetchMode, ) .option("-s, --section ", "Read one heading section") @@ -285,7 +285,7 @@ function createFetchCommand(dependencies: ProgramDependencies): Command { function parseFetchMode(value: string): FetchMode { if (!FETCH_MODES.includes(value as FetchMode)) - throw new Error("--mode must be one of auto, full, tree, or section"); + throw new Error("--mode must be one of auto, full, or tree"); return value as FetchMode; } @@ -295,13 +295,8 @@ function validateCliNavigation( ): void { if (section !== undefined && section.trim() === "") throw new Error("--section must be a non-empty string"); - if (mode === "section") { - if (section === undefined) - throw new Error('--section is required when --mode is "section"'); - return; - } - if (section !== undefined) - throw new Error('--section requires --mode "section"'); + if (section !== undefined && mode !== undefined && mode !== "auto") + throw new Error('--section is only valid with --mode "auto"'); } function createLinksCommand(dependencies: ProgramDependencies): Command { diff --git a/packages/web/src/runner.ts b/packages/web/src/runner.ts index 3777844..e9a0053 100644 --- a/packages/web/src/runner.ts +++ b/packages/web/src/runner.ts @@ -22,8 +22,15 @@ export async function runCli( output: CliOutput, ): Promise { const program = createProgram({ ...dependencies, writeOut: output.stdout }); - configureExitOverride(program); + const fetchCommand = program.commands.find( + (command) => command.name() === "fetch", + ); + fetchCommand?.exitOverride(); program.configureOutput({ writeOut: output.stdout, writeErr: output.stderr }); + fetchCommand?.configureOutput({ + writeOut: output.stdout, + writeErr: output.stderr, + }); try { await program.parseAsync(argv); return 0; @@ -32,19 +39,15 @@ export async function runCli( typeof error === "object" && error !== null && "code" in error && - error.code === "commander.helpDisplayed" + typeof error.code === "string" && + error.code.startsWith("commander.") ) - return 0; + return error.code === "commander.helpDisplayed" ? 0 : 1; output.stderr(formatCliError(error)); return 1; } } -function configureExitOverride(command: import("commander").Command): void { - command.exitOverride(); - for (const child of command.commands) configureExitOverride(child); -} - function formatCliError(error: unknown): string { if (!(error instanceof FetchCapabilityError)) return `${error instanceof Error ? error.message : "web search failed"}\n`; diff --git a/packages/web/test/http.test.ts b/packages/web/test/http.test.ts index a810568..c52c1da 100644 --- a/packages/web/test/http.test.ts +++ b/packages/web/test/http.test.ts @@ -17,6 +17,7 @@ function operations(overrides: Partial = {}): WebOperations { url: input.url, mode: "full" as const, content: "page", + truncated: false, })), links: vi.fn(async (input) => ({ url: input.url, @@ -212,6 +213,7 @@ describe("personal HTTP service", () => { url: input.url, mode: "tree" as const, content: "markdown", + truncated: false, })), }); const app = createHttpApp({ ...dependencies(), operations: ops }); @@ -223,7 +225,7 @@ describe("personal HTTP service", () => { }); expect(result.response.status).toBe(200); - expect(result.body).toMatchObject({ mode: "tree" }); + expect(result.body).toMatchObject({ mode: "tree", truncated: false }); expect(ops.fetch).toHaveBeenCalledWith( expect.objectContaining({ url: "https://example.test", @@ -239,43 +241,55 @@ describe("personal HTTP service", () => { const ops = operations({ fetch: vi.fn(async (input) => ({ url: input.url, - mode: input.mode === "tree" ? "tree" : "full", + mode: + input.section_id !== undefined + ? "section" + : input.mode === "tree" + ? "tree" + : input.mode === "auto" + ? "auto" + : "full", content: input.mode === "tree" ? "tree" : "page", + truncated: false, })) as WebOperations["fetch"], }); const app = createHttpApp({ ...dependencies(), operations: ops }); - for (const mode of ["auto", "full", "tree"] as const) { - const result = await json(app, "/v1/fetch", { + for (const body of [ + { url: "https://example.test", mode: "auto" }, + { url: "https://example.test", mode: "full" }, + { url: "https://example.test", mode: "tree" }, + { url: "https://example.test", - mode, - }); + mode: "auto", + section_id: "intro", + }, + { url: "https://example.test", section_id: "intro" }, + ]) { + const result = await json(app, "/v1/fetch", body); expect(result.response.status).toBe(200); } - const section = await json(app, "/v1/fetch", { - url: "https://example.test", - mode: "section", - section_id: "intro", - }); - expect(section.response.status).toBe(200); expect(ops.fetch).toHaveBeenNthCalledWith( 4, - expect.objectContaining({ mode: "section", section_id: "intro" }), + expect.objectContaining({ mode: "auto", section_id: "intro" }), + expect.any(AbortSignal), + ); + expect(ops.fetch).toHaveBeenNthCalledWith( + 5, + expect.objectContaining({ mode: "auto", section_id: "intro" }), expect.any(AbortSignal), ); for (const body of [ - { url: "https://example.test", mode: "section" }, - { url: "https://example.test", mode: "auto", section_id: "intro" }, { url: "https://example.test", mode: "full", section_id: "intro" }, { url: "https://example.test", mode: "tree", section_id: "intro" }, - { url: "https://example.test", section_id: "intro" }, + { url: "https://example.test", mode: "section" }, { url: "https://example.test", mode: "invalid" }, { url: "https://example.test", full: true }, ]) { expect((await json(app, "/v1/fetch", body)).response.status).toBe(400); } - expect(ops.fetch).toHaveBeenCalledTimes(4); + expect(ops.fetch).toHaveBeenCalledTimes(5); }); it("forwards links with the same explicit rendered-fetch contract", async () => { @@ -400,14 +414,17 @@ describe("personal HTTP service", () => { .enum, ).toEqual(["Exa", "DeepSeek", "Kepos Bridge"]); const fetchRequest = (document.components?.schemas as any).FetchRequest; - expect(fetchRequest.properties.mode.enum).toEqual([ + expect(fetchRequest.properties.mode.enum).toEqual(["auto", "full", "tree"]); + expect(fetchRequest.properties.mode.default).toBe("auto"); + expect(fetchRequest.properties.full).toBeUndefined(); + const fetchResponse = (document.components?.schemas as any).FetchResponse; + expect(fetchResponse.properties.mode.enum).toEqual([ "auto", "full", "tree", "section", ]); - expect(fetchRequest.properties.mode.default).toBe("auto"); - expect(fetchRequest.properties.full).toBeUndefined(); + expect(fetchResponse.properties.truncated.type).toBe("boolean"); expect(fetchRequest.properties.render.enum).toEqual(["http", "browser"]); expect(fetchRequest.properties.tree).toBeUndefined(); expect(fetchRequest.properties.tree_threshold).toBeUndefined(); diff --git a/packages/web/test/mcp.test.ts b/packages/web/test/mcp.test.ts index 9a45267..6e13f34 100644 --- a/packages/web/test/mcp.test.ts +++ b/packages/web/test/mcp.test.ts @@ -30,6 +30,7 @@ function webService(): WebOperations { url: "https://example.test/page", mode: "tree" as const, content: "# Page", + truncated: false, })), links: vi.fn(async (input) => ({ url: input.url, @@ -162,6 +163,7 @@ describe("web stdio MCP adapter", () => { url: "https://example.test/page", mode: "full", content: "Rendered through stdio", + truncated: false, }); const connection = await connectStdio(operations); @@ -246,15 +248,21 @@ describe("web stdio MCP adapter", () => { string, unknown >; + const fetchOutputProperties = byName.fetch!.outputSchema! + .properties as Record; expect(fetchProperties.render).toMatchObject({ enum: ["http", "browser"], default: "http", }); expect(fetchProperties.mode).toMatchObject({ - enum: ["auto", "full", "tree", "section"], + enum: ["auto", "full", "tree"], default: "auto", }); expect(fetchProperties.full).toBeUndefined(); + expect(fetchOutputProperties.mode).toMatchObject({ + enum: ["auto", "full", "tree", "section"], + }); + expect(fetchOutputProperties.truncated).toEqual({ type: "boolean" }); expect(fetchProperties.waitMs).toMatchObject({ type: "integer", minimum: 0, @@ -293,7 +301,7 @@ describe("web stdio MCP adapter", () => { name: "fetch", arguments: { url: "https://example.test/page", - mode: "section", + mode: "auto", section_id: "intro", render: "browser", waitMs: 125, @@ -348,7 +356,7 @@ describe("web stdio MCP adapter", () => { expect(operations.fetch).toHaveBeenCalledWith( { url: "https://example.test/page", - mode: "section", + mode: "auto", section_id: "intro", render: "browser", waitMs: 125, @@ -419,11 +427,33 @@ describe("web stdio MCP adapter", () => { const { client, operations } = await connect(); await client.listTools(); for (const arguments_ of [ - { url: "https://example.test/page", mode: "section" }, + { url: "https://example.test/page", section_id: "intro" }, + { + url: "https://example.test/page", + mode: "auto", + section_id: "intro", + }, + ]) { + const result = await client.callTool({ + name: "fetch", + arguments: arguments_, + }); + expect(result.isError).not.toBe(true); + } + expect(operations.fetch).toHaveBeenNthCalledWith( + 1, + { url: "https://example.test/page", section_id: "intro" }, + expect.any(AbortSignal), + ); + expect(operations.fetch).toHaveBeenNthCalledWith( + 2, { url: "https://example.test/page", mode: "auto", section_id: "intro" }, + expect.any(AbortSignal), + ); + for (const arguments_ of [ { url: "https://example.test/page", mode: "full", section_id: "intro" }, { url: "https://example.test/page", mode: "tree", section_id: "intro" }, - { url: "https://example.test/page", section_id: "intro" }, + { url: "https://example.test/page", mode: "section" }, { url: "https://example.test/page", mode: "invalid" }, { url: "https://example.test/page", full: true }, ]) { @@ -439,7 +469,7 @@ describe("web stdio MCP adapter", () => { }, ]); } - expect(operations.fetch).not.toHaveBeenCalled(); + expect(operations.fetch).toHaveBeenCalledTimes(2); }); it("keeps fetch capability details structured and recovers after a renderer failure", async () => { @@ -456,6 +486,7 @@ describe("web stdio MCP adapter", () => { url: "https://example.test/page", mode: "full", content: "Rendered page", + truncated: false, }); const { client } = await connect(operations); diff --git a/packages/web/test/packed-smoke.mjs b/packages/web/test/packed-smoke.mjs index 786f1f1..7472a4d 100644 --- a/packages/web/test/packed-smoke.mjs +++ b/packages/web/test/packed-smoke.mjs @@ -179,6 +179,7 @@ else console.log(JSON.stringify({ success: true, data: {} })); if ( mcpFetch.isError || mcpFetch.structuredContent?.mode !== "full" || + mcpFetch.structuredContent?.truncated !== false || mcpFetch.structuredContent?.content !== "Packed fetch fixture.\n" ) { throw new Error("packed MCP stdio could not fetch the local fixture"); @@ -204,6 +205,7 @@ else console.log(JSON.stringify({ success: true, data: {} })); const fetched = JSON.parse(result.stdout); if ( fetched.mode !== "full" || + fetched.truncated !== false || fetched.content !== "Packed fetch fixture.\n" ) { throw new Error("installed web CLI could not fetch the local fixture"); diff --git a/packages/web/test/program.test.ts b/packages/web/test/program.test.ts index d713a76..af91005 100644 --- a/packages/web/test/program.test.ts +++ b/packages/web/test/program.test.ts @@ -24,6 +24,7 @@ function setup() { url: "https://example.test/page", mode: "full" as const, content: "# Fixture page\n", + truncated: false, })), links: vi.fn(async (input: { url: string }) => ({ url: input.url, @@ -205,18 +206,31 @@ describe("web search Commander adapter", () => { it("passes fetch navigation flags and writes human Markdown", async () => { const { program, operations, output } = setup(); await program.parseAsync( - ["fetch", "https://example.test/page", "--mode", "section", "-s", "7i"], + ["fetch", "https://example.test/page", "--mode", "auto", "-s", "7i"], { from: "user" }, ); expect(operations.fetch).toHaveBeenCalledWith({ url: "https://example.test/page", - mode: "section", + mode: "auto", section_id: "7i", }); expect(output()).toEqual({ stdout: "# Fixture page\n", stderr: "" }); }); + it("allows a section flag with the default automatic mode", async () => { + const { program, operations } = setup(); + await program.parseAsync( + ["fetch", "https://example.test/page", "--section", "7i"], + { from: "user" }, + ); + + expect(operations.fetch).toHaveBeenCalledWith({ + url: "https://example.test/page", + section_id: "7i", + }); + }); + it("forwards explicit browser rendering options", async () => { const { program, operations } = setup(); await program.parseAsync( @@ -260,8 +274,6 @@ describe("web search Commander adapter", () => { "--section", "intro", ], - ["fetch", "https://example.test/page", "--section", "intro"], - ["fetch", "https://example.test/page", "--full"], ]) { const { program, operations } = setup(); program.exitOverride(); @@ -276,22 +288,30 @@ describe("web search Commander adapter", () => { const operations = setup().operations; let stdout = ""; let stderr = ""; - const exitCode = await runCli( - ["node", "web", "fetch", "https://example.test/page", "--full"], - { operations, credentials: () => ({}) }, - { - stdout: (text) => { - stdout += text; - }, - stderr: (text) => { - stderr += text; + const processStderrWrite = vi + .spyOn(process.stderr, "write") + .mockReturnValue(true); + try { + const exitCode = await runCli( + ["node", "web", "fetch", "https://example.test/page", "--full"], + { operations, credentials: () => ({}) }, + { + stdout: (text) => { + stdout += text; + }, + stderr: (text) => { + stderr += text; + }, }, - }, - ); - expect(exitCode).toBe(1); - expect(stdout).toBe(""); - expect(stderr).toContain("unknown option '--full'"); - expect(operations.fetch).not.toHaveBeenCalled(); + ); + expect(exitCode).toBe(1); + expect(stdout).toBe(""); + expect(stderr).toBe("error: unknown option '--full'\n"); + expect(processStderrWrite).not.toHaveBeenCalled(); + expect(operations.fetch).not.toHaveBeenCalled(); + } finally { + processStderrWrite.mockRestore(); + } }); it("forwards link discovery options and supports concise and JSON output", async () => {