From 3316d5bb5e14e81d89dbdd84239d16b5de007b10 Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 13:34:31 +0800 Subject: [PATCH 01/14] feat(http): add personal research service --- .dockerignore | 8 + .github/workflows/release.yaml | 47 +- .../defered/public-http-service-security.md | 16 + CONTEXT.md | 39 + Dockerfile | 30 + README.md | 82 +- docs/adr/0001-containerized-rest-api.md | 7 + packages/web-core/src/index.ts | 9 +- packages/web-core/test/search.test.ts | 12 + packages/web/package.json | 18 +- packages/web/src/generate-openapi.ts | 24 + packages/web/src/http.ts | 829 ++++++++++++++++++ packages/web/src/program.ts | 33 + packages/web/src/serve.ts | 38 + packages/web/test/http.test.ts | 373 ++++++++ packages/web/tsup.config.ts | 7 +- pnpm-lock.yaml | 74 ++ 17 files changed, 1631 insertions(+), 15 deletions(-) create mode 100644 .dockerignore create mode 100644 .scratch/defered/public-http-service-security.md create mode 100644 CONTEXT.md create mode 100644 Dockerfile create mode 100644 docs/adr/0001-containerized-rest-api.md create mode 100644 packages/web/src/generate-openapi.ts create mode 100644 packages/web/src/http.ts create mode 100644 packages/web/src/serve.ts create mode 100644 packages/web/test/http.test.ts diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9af656f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git +.scratch +node_modules +**/node_modules +**/dist +coverage +*.tgz +.env* diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e72065b..172eb0b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -63,6 +63,7 @@ jobs: path: | packages/web/package.json packages/web/dist + packages/web/dist/openapi.yaml packages/pi-web/package.json packages/pi-web/dist packages/dsh-web/package.json @@ -161,8 +162,41 @@ jobs: working-directory: packages/${{ matrix.directory }} run: npm publish --no-git-checks --access public --provenance --tag "${{ steps.version.outputs.dist-tag }}" + container: + needs: preflight + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup Node + uses: actions/setup-node@v7 + with: + node-version: 24 + + - name: Synchronize image package version + run: node scripts/sync-version.mjs "${GITHUB_REF_NAME#v}" + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Build and publish immutable release image + env: + IMAGE_NAME: ghcr.io/${{ github.repository }} + run: | + image="${IMAGE_NAME,,}:${GITHUB_REF_NAME}" + docker build --tag "$image" . + docker push "$image" + github-release: - needs: publish + needs: [publish, container] runs-on: ubuntu-latest permissions: contents: write @@ -171,3 +205,14 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: gh release create "$GITHUB_REF_NAME" --generate-notes --repo "$GITHUB_REPOSITORY" + + - name: Download generated OpenAPI artifact + uses: actions/download-artifact@v5 + with: + name: release-dist + path: packages + + - name: Attach generated OpenAPI contract + env: + GH_TOKEN: ${{ github.token }} + run: gh release upload "$GITHUB_REF_NAME" packages/web/dist/openapi.yaml#openapi.yaml --repo "$GITHUB_REPOSITORY" diff --git a/.scratch/defered/public-http-service-security.md b/.scratch/defered/public-http-service-security.md new file mode 100644 index 0000000..9ebcf91 --- /dev/null +++ b/.scratch/defered/public-http-service-security.md @@ -0,0 +1,16 @@ +# Public HTTP service hardening + +Before Guion Web's container is presented as a public or multi-tenant service, define and implement the security boundary for outbound requests and rendered browsing. The current first release is a Personal Web Service only. + +Required decisions and work: + +- Block private, loopback, link-local, and otherwise non-public network targets across redirects and DNS resolution, including DNS rebinding. +- Run `agent-browser` in an isolated browser/process and restrict its network egress consistently with direct fetch. +- Set resource and concurrency limits appropriate for a remotely reachable renderer. +- Define the authentication and tenant boundary when the service stops being personal. + +This deferred work does not change the v1 search policy: the Personal Web +Service tries the server-configured Bridge Route first, preserves successful +empty results, and falls back exactly once to the server-configured Exa +credential only for a non-cancellation Bridge failure. Typed Bridge Data +Operations report Bridge failure directly. diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..20cf6f1 --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,39 @@ +# Guion Web + +Guion Web provides read-only research operations through local integrations and a self-hosted HTTP service. This glossary keeps the interfaces and their deployment boundary distinct. + +## Language + +**Personal Web Service**: +A self-hosted, single-trust-boundary deployment of Guion Web for its operator and their agents. It is not a public or multi-tenant hosted service. +_Avoid_: Public service, SaaS + +**Research Operation**: +One of Guion Web's read-only capabilities: Search, Fetch, Links, Documentation Resolve or Fetch, or Public-source Search. It is independent of the transport that exposes it. +_Avoid_: Tool, endpoint + +**Search**: +The general web-retrieval Research Operation. In the Personal Web Service, it uses Kepos Bridge first and transparently falls back to Exa only when the Bridge is unavailable; its response identifies the provider that supplied results. +_Avoid_: Bridge search, provider-selected search + +**Bridge Data Operation**: +A typed, Bridge-only lookup for weather, sports, finance, or time. It has no Exa fallback because Exa search is not an equivalent result source. +_Avoid_: Generic Bridge command, special search + +**Bridge Route**: +The server-local URL used by the Kepos Bridge search provider. The service operator configures it; API callers never supply it. +_Avoid_: Bridge URL parameter + +**Rendered Fetch**: +A fetch performed with the host-installed `agent-browser` browser runtime, explicitly selected instead of direct HTTP fetching. +_Avoid_: Browser fetch, automatic fallback + +**Release Contract**: +The versioned public distribution of Guion Web: its npm packages, GHCR container image, and the generated `openapi.yaml` attached to the matching GitHub Release. +_Avoid_: Checked-in OpenAPI file, independently versioned schema + +**HTTP Service**: +The Hono-based `/v1` JSON API shipped by `web serve` and the GHCR image. It +uses server-local credentials and Bridge Route configuration; clients do not +select providers or submit a generic Bridge command. +_Avoid_: Remote MCP, public service diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..271da47 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +# Guion Web's supported HTTP-service image. The build stage bundles the web +# executable; the runtime stage adds the optional rendered-fetch capability. +FROM node:24-bookworm-slim AS build + +WORKDIR /workspace +RUN corepack enable +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY packages/web-core/package.json packages/web-core/package.json +COPY packages/web/package.json packages/web/package.json +COPY packages/pi-web/package.json packages/pi-web/package.json +COPY packages/dsh-web/package.json packages/dsh-web/package.json +RUN pnpm install --frozen-lockfile +COPY . . +RUN pnpm --filter @guionai/web run build + +FROM node:24-bookworm + +ENV NODE_ENV=production +WORKDIR /app + +# Rendering is deliberately explicit at the HTTP contract. The executable and +# browser runtime are image capabilities, while credentials remain env-only. +ARG AGENT_BROWSER_VERSION=0.36.0 +RUN npm install --global agent-browser@${AGENT_BROWSER_VERSION} \ + && agent-browser install + +COPY --from=build /workspace/packages/web/dist ./dist + +EXPOSE 8787 +ENTRYPOINT ["node", "/app/dist/cli.js", "serve"] diff --git a/README.md b/README.md index a34fec1..bc900f2 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,15 @@ Guion Web is a Node.js web research toolkit. It provides Exa, Brave, or a managed Kepos Bridge search endpoint, Context7 library documentation lookup, Sourcegraph public code search, page-link -discovery, and two page-fetch backends through a CLI, stdio MCP server, Pi extension, and DeepSeek -Harness (DSH) integration: direct HTML-to-Markdown extraction and explicit -`agent-browser` rendering for client-rendered pages on supported hosts. +discovery, and two page-fetch backends through a CLI, stdio MCP server, personal +HTTP service, Pi extension, and DeepSeek Harness (DSH) integration: direct +HTML-to-Markdown extraction and explicit `agent-browser` rendering for +client-rendered pages on supported hosts. ## Install and configure -Node.js 20 or later is required. `@guionai/web` intentionally exposes only -its `web` executable and stdio MCP server; it does not provide a root +Node.js 20 or later is required. `@guionai/web` exposes its `web` executable, +stdio MCP server, and personal HTTP service; it does not provide a root JavaScript or TypeScript SDK. Use the Pi or DSH packages for those host integrations. @@ -27,18 +28,79 @@ default route unless it runs in DSH, whose live settings card can override the route. Context7 works anonymously when its key is absent. +The HTTP service always requires a non-empty `EXA_API_KEY`: it tries the +server-local Kepos Bridge route first and retries Exa once when Bridge fails. +HTTP clients cannot select a provider, pass credentials, or override the Bridge +route per request. Set `KEPOS_BRIDGE_ENDPOINT` to replace the default route +(`http://codex-bridge.localhost:17480/codex/web-search`); it must be a complete +HTTP(S) URL without credentials, query, or fragment. + ```bash export EXA_API_KEY="..." # or export BRAVE_API_KEY="..." # optional, for authenticated Context7 requests export CONTEXT7_API_KEY="..." +# optional complete Bridge route for `web serve` +export KEPOS_BRIDGE_ENDPOINT="http://127.0.0.1:8787/codex/web-search" ``` Do not put credentials in command arguments or commit them. The CLI reads these environment variables directly; it does not load a dotenv file or an older application configuration path. +## Personal HTTP service + +Run the service with the server-local environment above: + +```bash +web serve --host 0.0.0.0 --port 8787 +# or use the published image +docker run --rm -p 8787:8787 \ + -e EXA_API_KEY="$EXA_API_KEY" \ + -e KEPOS_BRIDGE_ENDPOINT="http://host.docker.internal:17480/codex/web-search" \ + ghcr.io/guionai/web:v0.1.0 +``` + +Every operation is a versioned JSON `POST` route. Request and response schemas +are generated into `openapi.yaml` from the same route definitions: + +| Route | Request | Purpose | +| ------------------- | ------------------------------------------------------------------------------------- | -------------------------------------------------------- | +| `/v1/search` | `{ "query": "..." }` | Kepos Bridge search with one Exa retry on Bridge failure | +| `/v1/fetch` | `{ "url", "tree?", "section_id?", "full?", "tree_threshold?", "render?", "waitMs?" }` | Fetch Markdown | +| `/v1/links` | `{ "url", "limit?", "render?", "waitMs?" }` | List page HTTP(S) links | +| `/v1/docs/resolve` | `{ "query" }` | Resolve a Context7 library | +| `/v1/docs/fetch` | `{ "library_id", "topic?", "tokens?" }` | Fetch Context7 documentation | +| `/v1/source-search` | `{ "query", "count?", "context?", "timeout?" }` | Search public source through Sourcegraph | +| `/v1/weather` | `{ "location", "start?", "duration?" }` | Typed Bridge weather lookup | +| `/v1/sports` | `{ "fn", "league", ... }` | Typed Bridge schedule or standings lookup | +| `/v1/finance` | `{ "ticker", "type", "market?" }` | Typed Bridge quote or index lookup | +| `/v1/time` | `{ "utc_offset" }` | Typed Bridge time lookup | + +Weather, sports, finance, and time are separate Bridge-only operations; they do +not fall back to Exa and there is no generic Bridge-command route. Search keeps +a successful empty Bridge result, retries Exa exactly once for a non-cancellation +Bridge failure, and reports the provider in its response. Invalid JSON bodies, +unknown fields, and invalid typed values are rejected before an upstream call. +Upstream failures are bounded JSON errors and never include credentials or raw +provider response bodies. +Error responses use a stable `{ "code", "message", "details"? }` JSON shape; +upstream failures use 502 (or 504 for an upstream timeout), while client +cancellation is reported as 499. + +Fetch and Links use direct HTTP fetching when `render` is omitted (or set to +`"fetch"`). Rendered fetching is explicit and requires both +`render: "agent-browser"` and an integer `waitMs` from 0 through 30,000; direct +fetch never silently switches backends. The container installs `agent-browser` +and its browser runtime, while credentials and Bridge configuration remain +server-local environment variables. + +This is a Personal Web Service: a single-trust-boundary deployment for its +operator and agents. It is not hardened for public or multi-tenant exposure; +SSRF/egress isolation, browser sandboxing, quotas, and authentication remain +deferred in `.scratch/defered/public-http-service-security.md`. + ## CLI `web` has human-readable output by default. Add `--json` for exactly one JSON @@ -220,9 +282,13 @@ before any publication begins. Three independent, non-fail-fast protected `npm` Environment matrix cells then publish one package each through npm Trusted Publishing with provenance. The synchronized version selects npm's `latest` tag for stable SemVer and `beta` for -a prerelease. After all three cells succeed, the workflow creates the GitHub -release with generated notes and source archives. It publishes no binaries or -platform archives. +a prerelease. A matching immutable-tagged image is published to +`ghcr.io/guionai/web:` with the `web serve` entrypoint and the +`agent-browser` runtime. After all three npm cells and the image job succeed, +the workflow creates the GitHub release with generated notes, source archives, +and the build-generated `openapi.yaml` asset. The asset is generated from the +same Hono route schemas as the image and package; it is not checked in or +versioned independently. It publishes no binaries or platform archives. If publication partially fails, use GitHub Actions **Re-run failed jobs**. Never use **Re-run all jobs**: npm versions are immutable, so the jobs that already diff --git a/docs/adr/0001-containerized-rest-api.md b/docs/adr/0001-containerized-rest-api.md new file mode 100644 index 0000000..d9f0a32 --- /dev/null +++ b/docs/adr/0001-containerized-rest-api.md @@ -0,0 +1,7 @@ +# Expose research operations through a containerized REST API + +Guion Web will add a self-hosted, single-user HTTP service in a portable container. It will expose every existing read-only Research Operation plus typed Bridge Data Operations for weather, sports, finance, and time as REST routes defined with Hono and `@hono/zod-openapi`. Each tagged release will publish the generated OpenAPI 3.1 document as a downloadable `openapi.yaml` GitHub Release asset, alongside the matching npm packages and GHCR image. The current stdio MCP server remains supported, but remote Streamable HTTP MCP is outside this first service slice. + +## Consequences + +The service's provider credentials and Bridge Route are server-local configuration; clients cannot choose a provider or supply a Bridge Route per request. Search tries Kepos Bridge and uses Exa only when the Bridge is operationally unavailable; it does not fall back for cancellation, malformed client input, or an empty result set. A failed Bridge attempt is retried through Exa exactly once, and a successful empty Bridge response is returned unchanged. Weather, sports, finance, and time remain separate typed Bridge-only routes rather than a generic command passthrough, and never fall back to Exa. `agent-browser` is available as the explicit Rendered Fetch backend in the image. `openapi.yaml` is generated from the release build rather than manually maintained or independently versioned. Public or multi-tenant deployment hardening is deliberately deferred in `.scratch/defered/public-http-service-security.md`. diff --git a/packages/web-core/src/index.ts b/packages/web-core/src/index.ts index da84422..0dfcff6 100644 --- a/packages/web-core/src/index.ts +++ b/packages/web-core/src/index.ts @@ -37,6 +37,7 @@ export { isOperationAborted, isRequestTimeout, isResponseBodyLimit, + throwIfAborted, } from "./request.js"; export { @@ -133,6 +134,8 @@ export type SearchInput = { provider?: string; credentials: SearchCredentials; maxResults?: number; + /** Permit an empty successful Kepos result for the HTTP service contract. */ + allowEmptyKeposResults?: boolean; /** Complete route used by the Kepos Bridge provider. */ keposBridgeEndpoint?: string; signal?: AbortSignal; @@ -355,7 +358,11 @@ async function searchKeposBridge(input: SearchInput): Promise { timeoutMs: input.timeoutMs ?? undefined, }); const usable = (response.results ?? []).filter(isUsableTextResult); - if (usable.length === 0) + const returnedResults = response.results ?? []; + if ( + usable.length === 0 && + (!input.allowEmptyKeposResults || returnedResults.length > 0) + ) throw new Error("Kepos Bridge search returned no usable text results"); const limited = input.maxResults === undefined diff --git a/packages/web-core/test/search.test.ts b/packages/web-core/test/search.test.ts index 3abbe29..8b7b266 100644 --- a/packages/web-core/test/search.test.ts +++ b/packages/web-core/test/search.test.ts @@ -199,6 +199,18 @@ describe("search providers migrated from Organon fixtures", () => { ).rejects.toThrow(/Kepos Bridge provider/); }); + it("can preserve an explicitly empty successful Bridge result for HTTP callers", async () => { + await expect( + search({ + query: "empty", + provider: "kepos-bridge", + allowEmptyKeposResults: true, + credentials: {}, + fetch: async () => response({ output: "no matches", results: [] }), + }), + ).resolves.toEqual({ provider: "Kepos Bridge", results: [] }); + }); + it("does not expose remote error bodies or credentials", async () => { await expect( search({ diff --git a/packages/web/package.json b/packages/web/package.json index d8c999c..980463c 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -1,7 +1,7 @@ { "name": "@guionai/web", "version": "0.1.0", - "description": "Web research CLI and stdio MCP server with direct and optional agent-browser page-fetch backends.", + "description": "Web research CLI, stdio MCP server, and personal HTTP service with direct and optional agent-browser page-fetch backends.", "homepage": "https://github.com/guionai/web#readme", "bugs": { "url": "https://github.com/guionai/web/issues" @@ -27,13 +27,23 @@ "node": ">=20" }, "dependencies": { - "commander": "12.1.0" + "@hono/node-server": "2.1.1", + "@hono/zod-openapi": "1.6.1", + "commander": "12.1.0", + "hono": "4.13.5", + "yaml": "2.9.0", + "zod": "4.4.3" }, "bundledDependencies": [ - "commander" + "@hono/node-server", + "@hono/zod-openapi", + "commander", + "hono", + "yaml", + "zod" ], "scripts": { - "build": "tsup", + "build": "tsup && node dist/generate-openapi.js", "typecheck": "tsc --noEmit -p ../../tsconfig.json", "test:pack": "node test/packed-smoke.mjs" }, diff --git a/packages/web/src/generate-openapi.ts b/packages/web/src/generate-openapi.ts new file mode 100644 index 0000000..f90997b --- /dev/null +++ b/packages/web/src/generate-openapi.ts @@ -0,0 +1,24 @@ +#!/usr/bin/env node +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { stringify } from "yaml"; + +import { createHttpOpenAPIDocument } from "./http.js"; + +const packageDirectory = dirname(fileURLToPath(import.meta.url)); +const packageManifestPath = resolve(packageDirectory, "../package.json"); +const outputPath = process.argv[2] + ? resolve(process.argv[2]) + : join(packageDirectory, "openapi.yaml"); + +const manifest = JSON.parse(await readFile(packageManifestPath, "utf8")) as { + version?: unknown; +}; +if (typeof manifest.version !== "string" || manifest.version.length === 0) + throw new Error("package version is required to generate OpenAPI"); + +const document = createHttpOpenAPIDocument(manifest.version); +await mkdir(dirname(outputPath), { recursive: true }); +await writeFile(outputPath, stringify(document), "utf8"); +process.stdout.write(`generated ${outputPath}\n`); diff --git a/packages/web/src/http.ts b/packages/web/src/http.ts new file mode 100644 index 0000000..8521be9 --- /dev/null +++ b/packages/web/src/http.ts @@ -0,0 +1,829 @@ +import { + DEFAULT_LINK_LIMIT, + DEFAULT_KEPOS_BRIDGE_ENDPOINT, + FetchCapabilityError, + isOperationAborted, + isRequestTimeout, + MAX_LINK_LIMIT, + RENDER_REPORT_URL, + throwIfAborted, + validateKeposBridgeEndpoint, + type KeposBridgeResponse, + type SearchResponse, + type WebCredentials, + type WebOperations, +} from "@guionai/web-core"; +import * as webCoreModule from "@guionai/web-core"; +import { HTTPException } from "hono/http-exception"; +import { + createRoute, + OpenAPIHono, + z, + type RouteConfig, +} from "@hono/zod-openapi"; + +import { credentialsFromEnvironment } from "./runtime.js"; + +const DEFAULT_SOURCE_COUNT = 10; +const DEFAULT_SOURCE_CONTEXT = 10; +const DEFAULT_SOURCE_TIMEOUT = 0; + +/** Dependencies and server-local configuration for the personal HTTP service. */ +export type HttpServiceDependencies = { + operations?: WebOperations; + credentials?: WebCredentials | (() => WebCredentials); + keposBridgeEndpoint?: string; + /** Injectable environment for startup/configuration tests. */ + environment?: NodeJS.ProcessEnv; + /** Used by build-time OpenAPI generation to skip runtime credential checks. */ + validateStartup?: boolean; +}; + +export type HttpServiceState = { + operations: WebOperations; + credentials: WebCredentials; + keposBridgeEndpoint: string; +}; + +export type HttpError = { + code: string; + message: string; + details?: Record; +}; + +const ErrorSchema = z + .object({ + code: z.string().openapi({ example: "upstream_error" }), + message: z.string().openapi({ example: "search failed" }), + details: z.record(z.string(), z.unknown()).optional(), + }) + .strict() + .openapi("HttpError"); + +const SearchResultSchema = z + .object({ + title: z.string(), + link: z.string(), + snippet: z.string(), + position: z.number().int(), + }) + .strict() + .openapi("SearchResult"); + +const SearchResponseSchema = z + .object({ + provider: z.enum(["Exa", "Brave", "Kepos Bridge"]), + results: z.array(SearchResultSchema), + }) + .strict() + .openapi("SearchResponse"); + +const FetchResponseSchema = z + .object({ + url: z.string(), + mode: z.enum(["full", "tree", "section"]), + content: z.string(), + }) + .strict() + .openapi("FetchResponse"); + +const LinkSchema = z + .object({ text: z.string(), url: z.string() }) + .strict() + .openapi("PageLink"); + +const LinksResponseSchema = z + .object({ + url: z.string(), + links: z.array(LinkSchema), + truncated: z.boolean(), + }) + .strict() + .openapi("LinksResponse"); + +const DocsLibrarySchema = z + .object({ + id: z.string(), + title: z.string(), + description: z.string(), + trust_score: z.number(), + total_snippets: z.number().int(), + versions: z.array(z.string()).optional(), + }) + .strict() + .openapi("DocsLibrary"); + +const DocsResolveResponseSchema = z + .object({ query: z.string(), libraries: z.array(DocsLibrarySchema) }) + .strict() + .openapi("DocsResolveResponse"); + +const DocsFetchResponseSchema = z + .object({ + library_id: z.string(), + topic: z.string().optional(), + content: z.string(), + }) + .strict() + .openapi("DocsFetchResponse"); + +const SourceSearchResponseSchema = z + .object({ content: z.string() }) + .strict() + .openapi("SourceSearchResponse"); + +const KeposResponseSchema = z + .object({ output: z.string(), results: z.array(z.unknown()).optional() }) + .strict() + .openapi("KeposResponse"); + +const SearchRequestSchema = z + .object({ query: z.string().min(1) }) + .strict() + .openapi("SearchRequest"); + +const HttpUrlSchema = z + .string() + .min(1) + .refine(isHttpURL, "url must be an absolute HTTP(S) URL"); + +const FetchRequestSchema = z + .object({ + url: HttpUrlSchema, + tree: z.boolean().default(false), + section_id: z.string().optional(), + full: z.boolean().default(false), + tree_threshold: z.number().int().default(5000), + render: z.enum(["fetch", "agent-browser"]).default("fetch"), + waitMs: z.number().int().min(0).max(30_000).optional(), + }) + .strict() + .openapi("FetchRequest"); + +const LinksRequestSchema = z + .object({ + url: HttpUrlSchema, + limit: z + .number() + .int() + .min(1) + .max(MAX_LINK_LIMIT) + .default(DEFAULT_LINK_LIMIT), + render: z.enum(["fetch", "agent-browser"]).default("fetch"), + waitMs: z.number().int().min(0).max(30_000).optional(), + }) + .strict() + .openapi("LinksRequest"); + +const DocsResolveRequestSchema = z + .object({ query: z.string().min(1) }) + .strict() + .openapi("DocsResolveRequest"); + +const DocsFetchRequestSchema = z + .object({ + library_id: z.string().min(1), + topic: z.string().optional(), + tokens: z.number().int().min(0).default(0), + }) + .strict() + .openapi("DocsFetchRequest"); + +const SourceSearchRequestSchema = z + .object({ + query: z.string().min(1), + count: z.number().int().default(DEFAULT_SOURCE_COUNT), + context: z.number().int().default(DEFAULT_SOURCE_CONTEXT), + timeout: z.number().int().min(0).default(DEFAULT_SOURCE_TIMEOUT), + }) + .strict() + .openapi("SourceSearchRequest"); + +const DateSchema = z.string().regex(/^\d{4}-\d{2}-\d{2}$/); +const NonBlankStringSchema = z + .string() + .refine((value) => value.trim().length > 0, "must be a non-blank string"); + +const WeatherRequestSchema = z + .object({ + location: NonBlankStringSchema, + start: DateSchema.optional(), + duration: z.number().int().positive().safe().optional(), + }) + .strict() + .openapi("WeatherRequest"); + +const SportsRequestSchema = z + .object({ + fn: z.enum(["schedule", "standings"]), + league: z.enum([ + "nba", + "wnba", + "nfl", + "nhl", + "mlb", + "epl", + "ncaamb", + "ncaawb", + "ipl", + ]), + team: NonBlankStringSchema.optional(), + opponent: NonBlankStringSchema.optional(), + date_from: DateSchema.optional(), + date_to: DateSchema.optional(), + num_games: z.number().int().positive().safe().optional(), + locale: NonBlankStringSchema.optional(), + }) + .strict() + .openapi("SportsRequest"); + +const FinanceRequestSchema = z + .object({ + ticker: NonBlankStringSchema, + type: z.enum(["equity", "fund", "crypto", "index"]), + market: NonBlankStringSchema.optional(), + }) + .strict() + .openapi("FinanceRequest"); + +const TimeRequestSchema = z + .object({ + utc_offset: z.string().regex(/^[+-](?:[01]\d|2[0-3]):[0-5]\d$/), + }) + .strict() + .openapi("TimeRequest"); + +const commonResponses = { + 400: { + content: { "application/json": { schema: ErrorSchema } }, + description: "The request body is invalid.", + }, + 499: { + content: { "application/json": { schema: ErrorSchema } }, + description: "The client cancelled the request.", + }, + 502: { + content: { "application/json": { schema: ErrorSchema } }, + description: "The upstream operation failed.", + }, + 504: { + content: { "application/json": { schema: ErrorSchema } }, + description: "The upstream operation timed out.", + }, +} as const; + +const searchRoute = createRoute({ + method: "post", + path: "/v1/search", + operationId: "search", + summary: "Search the web", + description: + "Search through the server-local Kepos Bridge, retrying once through Exa when Bridge is unavailable.", + request: jsonRequest(SearchRequestSchema), + responses: { + 200: jsonResponse(SearchResponseSchema, "Search results."), + ...commonResponses, + }, +}); + +const fetchRoute = createRoute({ + method: "post", + path: "/v1/fetch", + operationId: "fetch", + summary: "Fetch a web page", + description: + "Fetch direct HTML by default; agent-browser rendering requires render=agent-browser and waitMs.", + request: jsonRequest(FetchRequestSchema), + responses: { + 200: jsonResponse(FetchResponseSchema, "Fetched page."), + ...commonResponses, + }, +}); + +const linksRoute = createRoute({ + method: "post", + path: "/v1/links", + operationId: "links", + summary: "List page links", + description: + "List HTTP(S) anchors using direct fetch by default or explicit agent-browser rendering.", + request: jsonRequest(LinksRequestSchema), + responses: { + 200: jsonResponse(LinksResponseSchema, "Page links."), + ...commonResponses, + }, +}); + +const docsResolveRoute = createRoute({ + method: "post", + path: "/v1/docs/resolve", + operationId: "docsResolve", + summary: "Resolve documentation", + request: jsonRequest(DocsResolveRequestSchema), + responses: { + 200: jsonResponse(DocsResolveResponseSchema, "Resolved libraries."), + ...commonResponses, + }, +}); + +const docsFetchRoute = createRoute({ + method: "post", + path: "/v1/docs/fetch", + operationId: "docsFetch", + summary: "Fetch documentation", + request: jsonRequest(DocsFetchRequestSchema), + responses: { + 200: jsonResponse(DocsFetchResponseSchema, "Documentation content."), + ...commonResponses, + }, +}); + +const sourceSearchRoute = createRoute({ + method: "post", + path: "/v1/source-search", + operationId: "sourceSearch", + summary: "Search public source code", + request: jsonRequest(SourceSearchRequestSchema), + responses: { + 200: jsonResponse(SourceSearchResponseSchema, "Sourcegraph results."), + ...commonResponses, + }, +}); + +const weatherRoute = createRoute({ + method: "post", + path: "/v1/weather", + operationId: "weather", + summary: "Look up weather", + request: jsonRequest(WeatherRequestSchema), + responses: { + 200: jsonResponse(KeposResponseSchema, "Weather data."), + ...commonResponses, + }, +}); + +const sportsRoute = createRoute({ + method: "post", + path: "/v1/sports", + operationId: "sports", + summary: "Look up sports", + request: jsonRequest(SportsRequestSchema), + responses: { + 200: jsonResponse(KeposResponseSchema, "Sports data."), + ...commonResponses, + }, +}); + +const financeRoute = createRoute({ + method: "post", + path: "/v1/finance", + operationId: "finance", + summary: "Look up finance", + request: jsonRequest(FinanceRequestSchema), + responses: { + 200: jsonResponse(KeposResponseSchema, "Finance data."), + ...commonResponses, + }, +}); + +const timeRoute = createRoute({ + method: "post", + path: "/v1/time", + operationId: "time", + summary: "Look up time", + request: jsonRequest(TimeRequestSchema), + responses: { + 200: jsonResponse(KeposResponseSchema, "Time data."), + ...commonResponses, + }, +}); + +/** Creates the in-process Hono application used by `web serve` and tests. */ +export function createHttpApp( + dependencies: HttpServiceDependencies = {}, +): OpenAPIHono { + const state = resolveHttpServiceState(dependencies); + const app = new OpenAPIHono({ + defaultHook: (result, context) => { + if (result.success) return; + return context.json( + errorBody("invalid_request", "Request validation failed"), + 400, + ); + }, + }); + + app.openapi(searchRoute, async (context) => { + const input = context.req.valid("json"); + try { + const result = await searchWithFallback( + state, + input.query, + context.req.raw.signal, + ); + return context.json(parseResponse(SearchResponseSchema, result), 200); + } catch (error) { + return failureResponse(context, error, "search") as never; + } + }); + + app.openapi(fetchRoute, async (context) => { + const input = context.req.valid("json"); + const renderError = validateRenderFields(input.render, input.waitMs); + if (renderError) return context.json(renderError, 400) as never; + try { + const result = await state.operations.fetch( + input, + context.req.raw.signal, + ); + throwIfAborted(context.req.raw.signal); + return context.json(parseResponse(FetchResponseSchema, result), 200); + } catch (error) { + return failureResponse(context, error, "fetch") as never; + } + }); + + app.openapi(linksRoute, async (context) => { + const input = context.req.valid("json"); + const renderError = validateRenderFields(input.render, input.waitMs); + if (renderError) return context.json(renderError, 400) as never; + try { + const result = await state.operations.links( + { ...input, limit: input.limit ?? DEFAULT_LINK_LIMIT }, + context.req.raw.signal, + ); + throwIfAborted(context.req.raw.signal); + return context.json(parseResponse(LinksResponseSchema, result), 200); + } catch (error) { + return failureResponse(context, error, "links") as never; + } + }); + + app.openapi(docsResolveRoute, async (context) => { + const input = context.req.valid("json"); + try { + const result = await state.operations.docsResolve({ + query: input.query, + credentials: state.credentials, + signal: context.req.raw.signal, + }); + throwIfAborted(context.req.raw.signal); + return context.json( + parseResponse(DocsResolveResponseSchema, result), + 200, + ); + } catch (error) { + return failureResponse(context, error, "docs resolve") as never; + } + }); + + app.openapi(docsFetchRoute, async (context) => { + const input = context.req.valid("json"); + try { + const result = await state.operations.docsFetch({ + ...input, + credentials: state.credentials, + signal: context.req.raw.signal, + }); + throwIfAborted(context.req.raw.signal); + return context.json(parseResponse(DocsFetchResponseSchema, result), 200); + } catch (error) { + return failureResponse(context, error, "docs fetch") as never; + } + }); + + app.openapi(sourceSearchRoute, async (context) => { + const input = context.req.valid("json"); + try { + const result = await state.operations.sgraphSearch({ + query: input.query, + count: input.count ?? DEFAULT_SOURCE_COUNT, + context: input.context ?? DEFAULT_SOURCE_CONTEXT, + timeout: input.timeout ?? DEFAULT_SOURCE_TIMEOUT, + signal: context.req.raw.signal, + }); + throwIfAborted(context.req.raw.signal); + return context.json( + parseResponse(SourceSearchResponseSchema, result), + 200, + ); + } catch (error) { + return failureResponse(context, error, "source search") as never; + } + }); + + registerKeposRoute(app, weatherRoute, "weather", "weather", state); + registerKeposRoute(app, sportsRoute, "sports", "sports", state); + registerKeposRoute(app, financeRoute, "finance", "finance", state); + registerKeposRoute(app, timeRoute, "time", "time", state); + + app.onError((error, context) => failureResponse(context, error, "request")); + app.notFound((context) => + context.json(errorBody("not_found", "Route not found"), 404), + ); + return app; +} + +/** Alias retained for callers that refer to the service as an HTTP app. */ +export const createHttpService = createHttpApp; + +/** Builds the OpenAPI 3.1 document from the registered route schemas. */ +export function createHttpOpenAPIDocument(version = "0.1.0") { + const app = createHttpApp({ + credentials: { exaApiKey: "build-placeholder" }, + keposBridgeEndpoint: DEFAULT_KEPOS_BRIDGE_ENDPOINT, + validateStartup: false, + }); + return app.getOpenAPI31Document({ + openapi: "3.1.0", + info: { + title: "Guion Web Personal HTTP Service", + version, + description: + "Read-only Guion Web research operations and typed Kepos Bridge data operations.", + }, + }); +} + +export function resolveHttpServiceState( + dependencies: HttpServiceDependencies = {}, +): HttpServiceState { + const environment = dependencies.environment ?? process.env; + const credentials = resolveCredentials(dependencies.credentials, environment); + if (dependencies.validateStartup !== false) { + if ( + typeof credentials.exaApiKey !== "string" || + credentials.exaApiKey.trim().length === 0 + ) { + throw new Error( + "EXA_API_KEY is required and must be non-empty for HTTP service", + ); + } + } + const endpoint = + dependencies.keposBridgeEndpoint ?? + environment.KEPOS_BRIDGE_ENDPOINT ?? + DEFAULT_KEPOS_BRIDGE_ENDPOINT; + return { + operations: dependencies.operations ?? webCoreModule.createWebOperations(), + credentials, + keposBridgeEndpoint: validateKeposBridgeEndpoint(endpoint), + }; +} + +function resolveCredentials( + value: HttpServiceDependencies["credentials"], + environment: NodeJS.ProcessEnv, +): WebCredentials { + const credentials = + typeof value === "function" + ? value() + : (value ?? credentialsFromEnvironment(environment)); + return { ...credentials }; +} + +function jsonRequest(schema: T) { + return { + body: { + required: true, + content: { "application/json": { schema } }, + }, + } as const; +} + +function jsonResponse(schema: T, description: string) { + return { + content: { "application/json": { schema } }, + description, + } as const; +} + +function isHttpURL(value: string): boolean { + try { + const url = new URL(value); + return url.protocol === "http:" || url.protocol === "https:"; + } catch { + return false; + } +} + +function validateRenderFields( + render: "fetch" | "agent-browser" | undefined, + waitMs: number | undefined, +): HttpError | undefined { + const selected = render ?? "fetch"; + if (selected === "fetch" && waitMs !== undefined) + return errorBody( + "invalid_request", + "waitMs is only valid with render agent-browser", + ); + if (selected === "agent-browser" && waitMs === undefined) + return errorBody( + "invalid_request", + "waitMs is required with render agent-browser", + ); + return undefined; +} + +async function searchWithFallback( + state: HttpServiceState, + query: string, + signal: AbortSignal, +): Promise { + throwIfAborted(signal); + try { + const result = await state.operations.search({ + query, + provider: "kepos-bridge", + credentials: state.credentials, + keposBridgeEndpoint: state.keposBridgeEndpoint, + allowEmptyKeposResults: true, + signal, + }); + throwIfAborted(signal); + return parseResponse(SearchResponseSchema, result); + } catch (bridgeError) { + if (isCancellation(bridgeError, signal)) throw bridgeError; + throwIfAborted(signal); + try { + const result = await state.operations.search({ + query, + provider: "exa", + credentials: state.credentials, + signal, + }); + throwIfAborted(signal); + return parseResponse(SearchResponseSchema, result); + } catch (exaError) { + if (isCancellation(exaError, signal)) throw exaError; + if (isRequestTimeout(exaError)) throw exaError; + throw new Error("Search upstream providers failed"); + } + } +} + +function registerKeposRoute< + R extends RouteConfig, + Input extends Record, +>( + app: OpenAPIHono, + route: R, + command: "weather" | "sports" | "finance" | "time", + operationName: string, + state: HttpServiceState, +): void { + (app as any).openapi(route, async (context: any) => { + const input = context.req.valid("json") as Input; + try { + const result = await state.operations.keposBridge({ + endpoint: state.keposBridgeEndpoint, + commands: { [command]: [input] }, + signal: context.req.raw.signal, + }); + throwIfAborted(context.req.raw.signal); + const normalized = normalizeKeposResponse(result); + return context.json(normalized, 200); + } catch (error) { + return failureResponse(context, error, operationName) as never; + } + }); +} + +function isKeposResponse(value: unknown): value is KeposBridgeResponse { + return ( + !!value && + typeof value === "object" && + !Array.isArray(value) && + typeof (value as { output?: unknown }).output === "string" && + ((value as { results?: unknown }).results === undefined || + Array.isArray((value as { results?: unknown }).results)) + ); +} + +function normalizeKeposResponse(value: unknown): KeposBridgeResponse { + if (!isKeposResponse(value)) throw new Error("malformed Bridge response"); + return { + output: value.output, + ...(value.results === undefined ? {} : { results: value.results }), + }; +} + +function parseResponse(schema: z.ZodType, value: unknown): T { + const parsed = schema.safeParse(value); + if (!parsed.success) throw new Error("malformed operation response"); + return parsed.data; +} + +function failureResponse( + context: { json: (body: HttpError, status: number) => Response }, + error: unknown, + operation: string, +): Response { + const mapped = classifyFailure(error, operation); + return context.json(mapped.body, mapped.status); +} + +function classifyFailure( + error: unknown, + operation: string, +): { status: 400 | 499 | 502 | 504; body: HttpError } { + if ( + error instanceof HTTPException && + error.status >= 400 && + error.status < 500 + ) + return { + status: 400, + body: errorBody("invalid_request", "Request body is invalid"), + }; + if (error instanceof SyntaxError) + return { + status: 400, + body: errorBody("invalid_request", "Request body is not valid JSON"), + }; + if (error instanceof FetchCapabilityError) { + return { + status: 502, + body: { + code: error.code, + message: `${operation} requires an explicit capability retry`, + details: safeFetchDetails(error.details), + }, + }; + } + if (isOperationAborted(error) || isAbortError(error)) + return { + status: 499, + body: errorBody("request_cancelled", "Request cancelled"), + }; + if (isRequestTimeout(error)) + return { + status: 504, + body: errorBody("upstream_timeout", `${operation} timed out`), + }; + return { + status: 502, + body: errorBody("upstream_error", `${operation} failed`), + }; +} + +function errorBody(code: string, message: string): HttpError { + return { code, message }; +} + +function isCancellation(error: unknown, signal: AbortSignal): boolean { + return signal.aborted || isOperationAborted(error) || isAbortError(error); +} + +function isAbortError(error: unknown): boolean { + return ( + (error instanceof DOMException && error.name === "AbortError") || + (error instanceof Error && + (error.name === "AbortError" || error.message === "Operation aborted")) + ); +} + +function safeFetchDetails( + details: FetchCapabilityError["details"], +): Record | undefined { + const safe: Record = {}; + if (typeof details.retryableWithRender === "boolean") + safe.retryableWithRender = details.retryableWithRender; + if (typeof details.retryable === "boolean") + safe.retryable = details.retryable; + if ( + details.suggestedArguments?.render === "agent-browser" && + details.suggestedArguments.waitMs === 2000 + ) { + safe.suggestedArguments = { render: "agent-browser", waitMs: 2000 }; + } + if (details.reportUrl === RENDER_REPORT_URL) + safe.reportUrl = details.reportUrl; + if ( + typeof details.blockedHostname === "string" && + /^[a-z0-9.-]+$/i.test(details.blockedHostname) && + details.blockedHostname.length <= 253 + ) { + safe.blockedHostname = details.blockedHostname.toLowerCase(); + } + return Object.keys(safe).length > 0 ? safe : undefined; +} + +export { + DocsFetchResponseSchema, + DocsLibrarySchema, + DocsResolveResponseSchema, + ErrorSchema, + FetchRequestSchema, + FetchResponseSchema, + FinanceRequestSchema, + KeposResponseSchema, + LinksRequestSchema, + LinksResponseSchema, + SearchRequestSchema, + SearchResponseSchema, + SourceSearchRequestSchema, + SportsRequestSchema, + TimeRequestSchema, + WeatherRequestSchema, +}; diff --git a/packages/web/src/program.ts b/packages/web/src/program.ts index aa8ca79..7dc6fba 100644 --- a/packages/web/src/program.ts +++ b/packages/web/src/program.ts @@ -1,6 +1,8 @@ import { Command } from "commander"; import { createMcpCommand } from "./mcp.js"; +import { parseHttpPort, startHttpServer } from "./serve.js"; +import { DEFAULT_HTTP_HOST, DEFAULT_HTTP_PORT } from "./serve.js"; import { formatSearchResults, @@ -28,10 +30,41 @@ export function createProgram(dependencies: ProgramDependencies): Command { .addCommand(createLinksCommand(dependencies)) .addCommand(createDocsCommand(dependencies)) .addCommand(createSGraphCommand(dependencies)) + .addCommand(createServeCommand(dependencies)) .addCommand(createMcpCommand(dependencies)); return program; } +function createServeCommand(dependencies: ProgramDependencies): Command { + const writeOut = + dependencies.writeOut ?? ((text: string) => process.stdout.write(text)); + return new Command("serve") + .description("Serve all web research operations over HTTP") + .option("--host ", "HTTP listen hostname", DEFAULT_HTTP_HOST) + .option( + "--port ", + "HTTP listen port", + parseHttpPort, + DEFAULT_HTTP_PORT, + ) + .action((options: { host: string; port: number }) => { + startHttpServer( + { + operations: dependencies.operations, + credentials: dependencies.credentials, + }, + { + hostname: options.host, + port: options.port, + onListening: ({ hostname, port }) => + writeOut( + `Guion Web HTTP service listening on ${hostname}:${port}\n`, + ), + }, + ); + }); +} + function createSearchCommand(dependencies: ProgramDependencies): Command { const writeOut = dependencies.writeOut ?? ((text: string) => process.stdout.write(text)); diff --git a/packages/web/src/serve.ts b/packages/web/src/serve.ts new file mode 100644 index 0000000..976a527 --- /dev/null +++ b/packages/web/src/serve.ts @@ -0,0 +1,38 @@ +import { serve, type ServerType } from "@hono/node-server"; + +import { createHttpApp, type HttpServiceDependencies } from "./http.js"; + +export const DEFAULT_HTTP_HOST = "0.0.0.0"; +export const DEFAULT_HTTP_PORT = 8787; + +export type HttpServerOptions = { + hostname?: string; + port?: number; + onListening?: (address: { hostname: string; port: number }) => void; +}; + +/** Starts the Hono HTTP service on a Node HTTP server. */ +export function startHttpServer( + dependencies: HttpServiceDependencies = {}, + options: HttpServerOptions = {}, +): ServerType { + const app = createHttpApp(dependencies); + const hostname = options.hostname ?? DEFAULT_HTTP_HOST; + const port = options.port ?? DEFAULT_HTTP_PORT; + if (!Number.isInteger(port) || port < 1 || port > 65_535) + throw new Error("HTTP port must be an integer from 1 through 65535"); + return serve({ fetch: app.fetch, hostname, port }, (address) => + options.onListening?.({ + hostname: + typeof address.address === "string" ? address.address : hostname, + port: address.port, + }), + ); +} + +export function parseHttpPort(value: string): number { + const port = Number(value); + if (!Number.isInteger(port) || port < 1 || port > 65_535) + throw new Error("HTTP port must be an integer from 1 through 65535"); + return port; +} diff --git a/packages/web/test/http.test.ts b/packages/web/test/http.test.ts new file mode 100644 index 0000000..8ce47f7 --- /dev/null +++ b/packages/web/test/http.test.ts @@ -0,0 +1,373 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + createHttpApp, + createHttpOpenAPIDocument, + type HttpServiceDependencies, +} from "../src/http.js"; +import { RequestTimeoutError, type WebOperations } from "@guionai/web-core"; + +function operations(overrides: Partial = {}): WebOperations { + return { + search: vi.fn(async () => ({ + provider: "Kepos Bridge" as const, + results: [], + })), + fetch: vi.fn(async (input) => ({ + url: input.url, + mode: "full" as const, + content: "page", + })), + links: vi.fn(async (input) => ({ + url: input.url, + links: [], + truncated: false, + })), + docsResolve: vi.fn(async (input) => ({ + query: input.query, + libraries: [], + })), + docsFetch: vi.fn(async (input) => ({ + library_id: input.library_id, + content: "docs", + })), + sgraphSearch: vi.fn(async () => ({ content: "source" })), + keposBridge: vi.fn(async () => ({ output: "bridge", results: [] })), + ...overrides, + }; +} + +function dependencies( + overrides: Partial = {}, +): HttpServiceDependencies { + return { + operations: operations(), + credentials: { exaApiKey: "exa-secret", context7ApiKey: "ctx-secret" }, + keposBridgeEndpoint: "http://bridge.test/route", + ...overrides, + }; +} + +async function json( + app: ReturnType, + path: string, + body: unknown, +): Promise<{ response: Response; body: any }> { + const response = await app.request(path, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + return { response, body: await response.json() }; +} + +describe("personal HTTP service", () => { + it("returns an empty successful Bridge search without retrying Exa", async () => { + const ops = operations(); + const app = createHttpApp({ + ...dependencies(), + operations: ops, + }); + const result = await json(app, "/v1/search", { query: "empty" }); + expect(result.response.status).toBe(200); + expect(result.body).toEqual({ provider: "Kepos Bridge", results: [] }); + expect(ops.search).toHaveBeenCalledTimes(1); + expect(ops.search).toHaveBeenCalledWith( + expect.objectContaining({ + query: "empty", + provider: "kepos-bridge", + keposBridgeEndpoint: "http://bridge.test/route", + allowEmptyKeposResults: true, + }), + ); + }); + + it("retries Exa exactly once after a Bridge failure", async () => { + const ops = operations({ + search: vi + .fn() + .mockRejectedValueOnce(new Error("bridge unavailable")) + .mockResolvedValueOnce({ + provider: "Exa", + results: [ + { + title: "result", + link: "https://example.test", + snippet: "snippet", + position: 1, + }, + ], + }), + }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + const result = await json(app, "/v1/search", { query: "fallback" }); + expect(result.response.status).toBe(200); + expect(result.body.provider).toBe("Exa"); + expect(ops.search).toHaveBeenCalledTimes(2); + expect(ops.search).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ provider: "exa", query: "fallback" }), + ); + }); + + it("treats a non-empty Bridge payload with no usable links as a failure", async () => { + const ops = operations({ + search: vi + .fn() + .mockRejectedValueOnce(new Error("bridge invalid")) + .mockResolvedValueOnce({ provider: "Exa", results: [] }), + }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + const result = await json(app, "/v1/search", { query: "invalid" }); + expect(result.response.status).toBe(200); + expect(result.body).toEqual({ provider: "Exa", results: [] }); + expect(ops.search).toHaveBeenCalledTimes(2); + }); + + it("does not retry a cancelled Bridge request", async () => { + const controller = new AbortController(); + const ops = operations({ + search: vi.fn(async ({ signal }) => { + controller.abort(); + throw new Error("Operation aborted"); + }), + }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + const response = await app.request("/v1/search", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ query: "cancel" }), + signal: controller.signal, + }); + expect(response.status).toBe(499); + expect(ops.search).toHaveBeenCalledTimes(1); + }); + + it("maps an Exa timeout after Bridge failure to a documented timeout", async () => { + const ops = operations({ + search: vi + .fn() + .mockRejectedValueOnce(new Error("bridge unavailable")) + .mockRejectedValueOnce(new RequestTimeoutError("exa timed out", true)), + }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + const result = await json(app, "/v1/search", { query: "timeout" }); + expect(result.response.status).toBe(504); + expect(result.body).toEqual({ + code: "upstream_timeout", + message: "search timed out", + }); + expect(ops.search).toHaveBeenCalledTimes(2); + }); + + it("keeps Bridge data operations typed, separate, and Bridge-only", async () => { + const ops = operations({ + keposBridge: vi.fn(async () => ({ + output: "bridge", + results: [{ future: true }], + future_field: "ignored", + })), + }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + const weather = await json(app, "/v1/weather", { + location: "Country, Area, City", + start: "2026-09-01", + duration: 3, + }); + expect(weather.response.status).toBe(200); + expect(weather.body).toEqual({ + output: "bridge", + results: [{ future: true }], + }); + await json(app, "/v1/sports", { fn: "schedule", league: "nba" }); + await json(app, "/v1/finance", { ticker: "BTC", type: "crypto" }); + await json(app, "/v1/time", { utc_offset: "+08:00" }); + expect(ops.keposBridge).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + endpoint: "http://bridge.test/route", + commands: { + weather: [ + { + location: "Country, Area, City", + start: "2026-09-01", + duration: 3, + }, + ], + }, + }), + ); + expect(ops.keposBridge).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + commands: { sports: [{ fn: "schedule", league: "nba" }] }, + }), + ); + expect(ops.keposBridge).toHaveBeenNthCalledWith( + 3, + expect.objectContaining({ + commands: { finance: [{ ticker: "BTC", type: "crypto" }] }, + }), + ); + expect(ops.keposBridge).toHaveBeenNthCalledWith( + 4, + expect.objectContaining({ + commands: { time: [{ utc_offset: "+08:00" }] }, + }), + ); + expect(ops.search).not.toHaveBeenCalled(); + }); + + it("forwards every research operation with server credentials and defaults", async () => { + const ops = operations({ + fetch: vi.fn(async (input) => ({ + url: input.url, + mode: "tree" as const, + content: "markdown", + })), + links: vi.fn(async (input) => ({ + url: input.url, + links: [{ text: "docs", url: "https://example.test/docs" }], + truncated: false, + })), + docsResolve: vi.fn(async (input) => ({ + query: input.query, + libraries: [ + { + id: "/acme/docs", + title: "Docs", + description: "fixture", + trust_score: 9, + total_snippets: 2, + }, + ], + })), + docsFetch: vi.fn(async (input) => ({ + library_id: input.library_id, + content: "documentation", + })), + sgraphSearch: vi.fn(async () => ({ content: "source results" })), + }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + expect( + ( + await json(app, "/v1/fetch", { + url: "https://example.test", + render: "agent-browser", + waitMs: 0, + }) + ).body, + ).toMatchObject({ mode: "tree" }); + expect( + (await json(app, "/v1/links", { url: "https://example.test" })).body, + ).toMatchObject({ links: [{ text: "docs" }] }); + expect( + (await json(app, "/v1/docs/resolve", { query: "acme" })).body, + ).toMatchObject({ libraries: [{ id: "/acme/docs" }] }); + expect( + ( + await json(app, "/v1/docs/fetch", { + library_id: "/acme/docs", + topic: "install", + }) + ).body, + ).toMatchObject({ content: "documentation" }); + expect( + (await json(app, "/v1/source-search", { query: "repo:acme" })).body, + ).toEqual({ content: "source results" }); + expect(ops.fetch).toHaveBeenCalledWith( + expect.objectContaining({ + url: "https://example.test", + tree: false, + full: false, + tree_threshold: 5000, + render: "agent-browser", + waitMs: 0, + }), + expect.any(AbortSignal), + ); + expect(ops.links).toHaveBeenCalledWith( + { url: "https://example.test", limit: 100, render: "fetch" }, + expect.any(AbortSignal), + ); + expect(ops.docsResolve).toHaveBeenCalledWith( + expect.objectContaining({ + query: "acme", + credentials: { exaApiKey: "exa-secret", context7ApiKey: "ctx-secret" }, + }), + ); + expect(ops.docsFetch).toHaveBeenCalledWith( + expect.objectContaining({ + library_id: "/acme/docs", + topic: "install", + tokens: 0, + }), + ); + expect(ops.sgraphSearch).toHaveBeenCalledWith( + expect.objectContaining({ + query: "repo:acme", + count: 10, + context: 10, + timeout: 0, + }), + ); + }); + + it("rejects invalid request bodies before invoking operations", async () => { + const ops = operations(); + const app = createHttpApp({ ...dependencies(), operations: ops }); + const invalid = await json(app, "/v1/weather", { + location: "", + command: "search_query", + }); + expect(invalid.response.status).toBe(400); + expect(invalid.body.code).toBe("invalid_request"); + expect(ops.keposBridge).not.toHaveBeenCalled(); + + const render = await json(app, "/v1/fetch", { + url: "https://example.test", + waitMs: 100, + }); + expect(render.response.status).toBe(400); + expect(ops.fetch).not.toHaveBeenCalled(); + + const malformed = await app.request("/v1/search", { + method: "POST", + headers: { "content-type": "application/json" }, + body: "not-json", + }); + expect(malformed.status).toBe(400); + expect(await malformed.json()).toMatchObject({ code: "invalid_request" }); + }); + + it("exposes all typed routes in a generated OpenAPI 3.1 document", () => { + const document = createHttpOpenAPIDocument("1.2.3"); + expect(document.openapi).toBe("3.1.0"); + expect(document.info.version).toBe("1.2.3"); + expect(Object.keys(document.paths ?? {}).sort()).toEqual([ + "/v1/docs/fetch", + "/v1/docs/resolve", + "/v1/fetch", + "/v1/finance", + "/v1/links", + "/v1/search", + "/v1/source-search", + "/v1/sports", + "/v1/time", + "/v1/weather", + ]); + expect( + Object.keys(document.paths ?? {}).some((path) => path.includes("bridge")), + ).toBe(false); + }); + + it("requires an Exa key and validates the server-local Bridge endpoint", () => { + expect(() => createHttpApp({ credentials: {} })).toThrow("EXA_API_KEY"); + expect(() => + createHttpApp({ + credentials: { exaApiKey: "key" }, + keposBridgeEndpoint: "https://user:pass@example.test/route", + }), + ).toThrow("Kepos Bridge endpoint"); + }); +}); diff --git a/packages/web/tsup.config.ts b/packages/web/tsup.config.ts index c066be8..6fbddd9 100644 --- a/packages/web/tsup.config.ts +++ b/packages/web/tsup.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "tsup"; export default defineConfig({ - entry: { cli: "src/cli.ts" }, + entry: { cli: "src/cli.ts", "generate-openapi": "src/generate-openapi.ts" }, format: ["esm"], platform: "node", target: "node20", @@ -16,6 +16,11 @@ export default defineConfig({ "defuddle", "linkedom", "markdown-it", + "@hono/node-server", + "@hono/zod-openapi", + "hono", + "zod", + "yaml", ], banner: { js: 'import { createRequire } from "node:module"; const require = createRequire(import.meta.url);', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 888db7f..a3dfcf8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -122,9 +122,24 @@ importers: packages/web: dependencies: + '@hono/node-server': + specifier: 2.1.1 + version: 2.1.1(hono@4.13.5) + '@hono/zod-openapi': + specifier: 1.6.1 + version: 1.6.1(hono@4.13.5)(zod@4.4.3) commander: specifier: 12.1.0 version: 12.1.0 + hono: + specifier: 4.13.5 + version: 4.13.5 + yaml: + specifier: 2.9.0 + version: 2.9.0 + zod: + specifier: 4.4.3 + version: 4.4.3 devDependencies: '@modelcontextprotocol/server': specifier: 2.0.0 @@ -153,6 +168,11 @@ packages: zod: optional: true + '@asteasolutions/zod-to-openapi@9.1.0': + resolution: {integrity: sha512-pLMeRgRYS7/vZIgAAkOe2P6+XGTifR1MT9pqDoxZ11EmcJJ+DPmdPqLN8ZZF4U8R9KHCCQCS9c2wtuE8wSrfkw==} + peerDependencies: + zod: ^4.0.0 + '@aws-crypto/sha256-browser@5.2.0': resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} @@ -809,6 +829,25 @@ packages: '@modelcontextprotocol/sdk': optional: true + '@hono/node-server@2.1.1': + resolution: {integrity: sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==} + engines: {node: '>=20'} + peerDependencies: + hono: ^4 + + '@hono/zod-openapi@1.6.1': + resolution: {integrity: sha512-z1xS3FZxl4bBkU3kMIsQsLtsqxPTha5XZCsDEPMSZ6+3RRBQI7KED1GeoDpq2WueXCacsNIjcB0MtCnEHY7ahQ==} + engines: {node: '>=16.0.0'} + peerDependencies: + hono: '>=4.10.0' + zod: ^4.0.0 + + '@hono/zod-validator@0.9.1': + resolution: {integrity: sha512-iiv6w0qrIc0arfvCtUqBWsvl4fXjzaTcQcCJTTtCnkawF9HHGE+KjEl0ox3gQJ6rKZEgE3mLExlQCF72M+mNuw==} + peerDependencies: + hono: '>=4.11.2' + zod: ^3.25.0 || ^4.0.0 + '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -1549,6 +1588,10 @@ packages: highlight.js@10.7.3: resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} + hono@4.13.5: + resolution: {integrity: sha512-O6+/eCYRkzzzy0rPWwKLiGBR1nFuUPZynnwjxN1MBA62NNqbT0wQEzQyK2gSO5yDIDB336sXQleAhOHrzlYyKw==} + engines: {node: '>=16.9.0'} + hosted-git-info@9.0.3: resolution: {integrity: sha512-Hc+ghLoSt6QaYZUv0WBiIvmMDZuZZ7oaDvdH8MbfOO4lOsxdXLEvuC6ePoGs9H1X9oCLyq6+NVN0MKqD+ydxyg==} engines: {node: ^20.17.0 || >=22.9.0} @@ -1939,6 +1982,9 @@ packages: zod: optional: true + openapi3-ts@4.6.1: + resolution: {integrity: sha512-XW9MOldkhoICNeXVzzmXzmOW5G73ppOEGmh7fLCqHjgfdEYCGGN+00MlVCeUZgovjjfC56j9tvtDt1zGabNjjA==} + p-retry@4.6.2: resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} engines: {node: '>=8'} @@ -2378,6 +2424,11 @@ snapshots: optionalDependencies: zod: 4.4.3 + '@asteasolutions/zod-to-openapi@9.1.0(zod@4.4.3)': + dependencies: + openapi3-ts: 4.6.1 + zod: 4.4.3 + '@aws-crypto/sha256-browser@5.2.0': dependencies: '@aws-crypto/sha256-js': 5.2.0 @@ -3095,6 +3146,23 @@ snapshots: - supports-color - utf-8-validate + '@hono/node-server@2.1.1(hono@4.13.5)': + dependencies: + hono: 4.13.5 + + '@hono/zod-openapi@1.6.1(hono@4.13.5)(zod@4.4.3)': + dependencies: + '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) + '@hono/zod-validator': 0.9.1(hono@4.13.5)(zod@4.4.3) + hono: 4.13.5 + openapi3-ts: 4.6.1 + zod: 4.4.3 + + '@hono/zod-validator@0.9.1(hono@4.13.5)(zod@4.4.3)': + dependencies: + hono: 4.13.5 + zod: 4.4.3 + '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -3817,6 +3885,8 @@ snapshots: highlight.js@10.7.3: {} + hono@4.13.5: {} + hosted-git-info@9.0.3: dependencies: lru-cache: 11.5.2 @@ -4371,6 +4441,10 @@ snapshots: ws: 8.21.3 zod: 4.4.3 + openapi3-ts@4.6.1: + dependencies: + yaml: 2.9.0 + p-retry@4.6.2: dependencies: '@types/retry': 0.12.0 From ae5e2ead774b74c510d73a67ea69beea020cfbfb Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 13:35:46 +0800 Subject: [PATCH 02/14] chore(http): add implementation report --- .../http-web-service/implementation-report.md | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 .scratch/http-web-service/implementation-report.md diff --git a/.scratch/http-web-service/implementation-report.md b/.scratch/http-web-service/implementation-report.md new file mode 100644 index 0000000..5e22441 --- /dev/null +++ b/.scratch/http-web-service/implementation-report.md @@ -0,0 +1,61 @@ +# Guion Web HTTP service implementation report + +## Scope + +- Repository: `guionai/web` +- Branch: `http-web-service` +- Fixed point: `ebad44eb55f8a406b855318cb8a4b4cd04633227` +- Implementation commit: `3316d5bb5e14e81d89dbdd84239d16b5de007b10` (`feat(http): add personal research service`) +- Delivery boundary: the complete HTTP-service spec and tickets 01, 02, and 03; code review and deployment were excluded. + +The implementation was completed in dependency order: the server-local Bridge-first Search and typed Bridge Data Operations, the remaining Research Operations and container guide, and then the release/OpenAPI/GHCR contract. + +## Ticket outcomes + +### 01 — Serve Search and Bridge Data over HTTP + +- Added `web serve` with Hono and `@hono/zod-openapi` route schemas. +- Added `/v1/search`, `/v1/weather`, `/v1/sports`, `/v1/finance`, and `/v1/time`. +- HTTP Search sends server-local Kepos Bridge configuration first, preserves a successful empty result, retries Exa exactly once for a non-cancellation Bridge failure, and does not retry cancellation. +- Startup requires a non-empty `EXA_API_KEY`; `KEPOS_BRIDGE_ENDPOINT` is validated and otherwise uses the core default. Context7 remains optional. +- Typed Data Operations accept only their documented fields, call only their corresponding Bridge command, and have no generic command or Exa fallback. +- Invalid bodies, malformed JSON, capability failures, timeouts, cancellations, and upstream failures map to bounded documented JSON error responses without credentials or raw upstream bodies. + +### 02 — Complete the Research API and Container Guide + +- Added `/v1/fetch`, `/v1/links`, `/v1/docs/resolve`, `/v1/docs/fetch`, and `/v1/source-search` with request and response schemas. +- Fetch and Links default to direct fetching; rendered fetching requires explicit `render: "agent-browser"` and an integer `waitMs` from 0 through 30,000. +- Added the Node 24 container image, installing the pinned `agent-browser` runtime and starting `web serve` on port 8787. Credentials and Bridge configuration remain environment-only. +- Updated `README.md`, `CONTEXT.md`, ADR 0001, and `.scratch/defered/public-http-service-security.md` to describe the personal-service boundary and deferred public hardening. + +### 03 — Publish the Versioned HTTP Service Contract + +- The web build now generates `packages/web/dist/openapi.yaml` from the registered route schemas with the package version. +- The release preflight uploads the generated contract; the release workflow publishes a release-tagged GHCR image with `packages: write` and creates the GitHub Release only after npm and image jobs succeed. +- The matching generated `openapi.yaml` is attached to the GitHub Release and is not checked in or independently versioned. + +## Verification + +All checks below completed successfully against the implementation commit: + +- `pnpm format:check` — all files matched Prettier. +- `pnpm typecheck` — TypeScript completed with no errors. +- `pnpm build` — all workspace packages built; `packages/web/dist/openapi.yaml` generated. +- Generated OpenAPI parse check — OpenAPI `3.1.0`, package version `0.1.0`, exactly the 10 documented `/v1` paths, and no generic Bridge route. +- `pnpm test` — 18 test files and 109 tests passed. The existing DSH source-map warning was non-fatal. +- `pnpm test:release` — release version synchronization fixtures passed. +- `pnpm test:pack` — packed-installation/host-loading smoke tests passed for `@guionai/web`, `@guionai/pi-web`, and `@guionai/dsh-web`. +- Focused in-process HTTP tests use injected operation fakes and test-owned credentials; they cover validation, exact fallback/cancellation behavior, typed Bridge command shapes, response normalization, startup configuration, and OpenAPI paths without provider or browser network calls. + +The local Docker daemon was unavailable for a Docker build check (`/Users/neil/.orbstack/run/docker.sock` did not exist). The Dockerfile and workflow were reviewed statically; image deployment remains outside this task's boundary. + +## Changed paths and size + +Against the fixed point, excluding generated `dist` output (including `openapi.yaml`) and `pnpm-lock.yaml`: + +- Product code: 932 additions, 1 deletion (`packages/web/src`, `packages/web-core/src`). +- Tests: 385 additions (`packages/web/test`, `packages/web-core/test`). +- Configuration and documentation: 240 additions, 14 deletions (Docker, release workflow, manifests, README, CONTEXT, ADR, and deferred note). +- Total: 1,557 additions and 15 deletions (net 1,542 lines). + +The total is within the spec estimate of 1,230–1,900 lines. The focused HTTP suite is 385 lines, just below the 400-line test estimate because route behavior is consolidated in one in-process fixture file rather than duplicated across transport tests. From 73405d85f047e35354bb340c90faf8374b23a2ba Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 13:45:26 +0800 Subject: [PATCH 03/14] fix(http): close service review gaps --- packages/web/src/generate-openapi.ts | 33 ++++++++----- packages/web/src/http.ts | 5 +- packages/web/test/http.test.ts | 46 ++++++++++++++++++ packages/web/test/openapi-artifact.test.ts | 54 ++++++++++++++++++++++ 4 files changed, 122 insertions(+), 16 deletions(-) create mode 100644 packages/web/test/openapi-artifact.test.ts diff --git a/packages/web/src/generate-openapi.ts b/packages/web/src/generate-openapi.ts index f90997b..6afd90b 100644 --- a/packages/web/src/generate-openapi.ts +++ b/packages/web/src/generate-openapi.ts @@ -8,17 +8,26 @@ import { createHttpOpenAPIDocument } from "./http.js"; const packageDirectory = dirname(fileURLToPath(import.meta.url)); const packageManifestPath = resolve(packageDirectory, "../package.json"); -const outputPath = process.argv[2] - ? resolve(process.argv[2]) - : join(packageDirectory, "openapi.yaml"); -const manifest = JSON.parse(await readFile(packageManifestPath, "utf8")) as { - version?: unknown; -}; -if (typeof manifest.version !== "string" || manifest.version.length === 0) - throw new Error("package version is required to generate OpenAPI"); +/** Serializes the release OpenAPI contract to a YAML file. */ +export async function generateOpenAPI(output?: string): Promise { + const outputPath = output + ? resolve(output) + : join(packageDirectory, "openapi.yaml"); + const manifest = JSON.parse(await readFile(packageManifestPath, "utf8")) as { + version?: unknown; + }; + if (typeof manifest.version !== "string" || manifest.version.length === 0) + throw new Error("package version is required to generate OpenAPI"); -const document = createHttpOpenAPIDocument(manifest.version); -await mkdir(dirname(outputPath), { recursive: true }); -await writeFile(outputPath, stringify(document), "utf8"); -process.stdout.write(`generated ${outputPath}\n`); + const document = createHttpOpenAPIDocument(manifest.version); + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, stringify(document), "utf8"); + return outputPath; +} + +const invokedPath = process.argv[1]; +if (invokedPath && resolve(invokedPath) === fileURLToPath(import.meta.url)) { + const outputPath = await generateOpenAPI(process.argv[2]); + process.stdout.write(`generated ${outputPath}\n`); +} diff --git a/packages/web/src/http.ts b/packages/web/src/http.ts index 8521be9..a05abf7 100644 --- a/packages/web/src/http.ts +++ b/packages/web/src/http.ts @@ -72,7 +72,7 @@ const SearchResultSchema = z const SearchResponseSchema = z .object({ - provider: z.enum(["Exa", "Brave", "Kepos Bridge"]), + provider: z.enum(["Exa", "Kepos Bridge"]), results: z.array(SearchResultSchema), }) .strict() @@ -524,9 +524,6 @@ export function createHttpApp( return app; } -/** Alias retained for callers that refer to the service as an HTTP app. */ -export const createHttpService = createHttpApp; - /** Builds the OpenAPI 3.1 document from the registered route schemas. */ export function createHttpOpenAPIDocument(version = "0.1.0") { const app = createHttpApp({ diff --git a/packages/web/test/http.test.ts b/packages/web/test/http.test.ts index 8ce47f7..259d631 100644 --- a/packages/web/test/http.test.ts +++ b/packages/web/test/http.test.ts @@ -340,6 +340,48 @@ describe("personal HTTP service", () => { expect(await malformed.json()).toMatchObject({ code: "invalid_request" }); }); + it.each([ + { + path: "/v1/links", + body: { url: "not-an-http-url" }, + }, + { + path: "/v1/docs/resolve", + body: { query: "" }, + }, + { + path: "/v1/docs/fetch", + body: { library_id: "", tokens: -1 }, + }, + { + path: "/v1/source-search", + body: { query: "", count: 10 }, + }, + { + path: "/v1/sports", + body: { fn: "schedule", league: "not-a-league" }, + }, + { + path: "/v1/finance", + body: { ticker: "", type: "equity" }, + }, + { + path: "/v1/time", + body: { utc_offset: "UTC" }, + }, + ])( + "rejects invalid $path bodies before any operation call", + async ({ path, body }) => { + const ops = operations(); + const app = createHttpApp({ ...dependencies(), operations: ops }); + const result = await json(app, path, body); + expect(result.response.status).toBe(400); + expect(result.body).toMatchObject({ code: "invalid_request" }); + for (const operation of Object.values(ops)) + expect(operation).not.toHaveBeenCalled(); + }, + ); + it("exposes all typed routes in a generated OpenAPI 3.1 document", () => { const document = createHttpOpenAPIDocument("1.2.3"); expect(document.openapi).toBe("3.1.0"); @@ -359,6 +401,10 @@ describe("personal HTTP service", () => { expect( Object.keys(document.paths ?? {}).some((path) => path.includes("bridge")), ).toBe(false); + expect( + (document.components?.schemas as any).SearchResponse.properties.provider + .enum, + ).toEqual(["Exa", "Kepos Bridge"]); }); it("requires an Exa key and validates the server-local Bridge endpoint", () => { diff --git a/packages/web/test/openapi-artifact.test.ts b/packages/web/test/openapi-artifact.test.ts new file mode 100644 index 0000000..6392df3 --- /dev/null +++ b/packages/web/test/openapi-artifact.test.ts @@ -0,0 +1,54 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { parse } from "yaml"; +import { describe, expect, it } from "vitest"; + +import { generateOpenAPI } from "../src/generate-openapi.js"; + +const packageRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); +const expectedPaths = [ + "/v1/docs/fetch", + "/v1/docs/resolve", + "/v1/fetch", + "/v1/finance", + "/v1/links", + "/v1/search", + "/v1/source-search", + "/v1/sports", + "/v1/time", + "/v1/weather", +]; + +describe("release OpenAPI artifact", () => { + it("serializes and parses the versioned contract from a test-owned file", async () => { + const artifactRoot = mkdtempSync( + join(tmpdir(), "guionai-web-openapi-artifact-"), + ); + try { + const outputPath = join(artifactRoot, "openapi.yaml"); + await generateOpenAPI(outputPath); + const document = parse(readFileSync(outputPath, "utf8")) as { + openapi?: unknown; + info?: { version?: unknown }; + paths?: Record; + }; + const manifest = JSON.parse( + readFileSync(join(packageRoot, "package.json"), "utf8"), + ) as { version: string }; + + expect(document.openapi).toBe("3.1.0"); + expect(document.info?.version).toBe(manifest.version); + expect(Object.keys(document.paths ?? {}).sort()).toEqual(expectedPaths); + expect( + Object.keys(document.paths ?? {}).some((path) => + path.includes("bridge"), + ), + ).toBe(false); + } finally { + rmSync(artifactRoot, { recursive: true, force: true }); + } + }); +}); From 36f465b5ed6b173368ae350fe21f328d3dacbf67 Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 13:46:16 +0800 Subject: [PATCH 04/14] chore(http): update review report --- .../http-web-service/implementation-report.md | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.scratch/http-web-service/implementation-report.md b/.scratch/http-web-service/implementation-report.md index 5e22441..d6282dd 100644 --- a/.scratch/http-web-service/implementation-report.md +++ b/.scratch/http-web-service/implementation-report.md @@ -6,10 +6,18 @@ - Branch: `http-web-service` - Fixed point: `ebad44eb55f8a406b855318cb8a4b4cd04633227` - Implementation commit: `3316d5bb5e14e81d89dbdd84239d16b5de007b10` (`feat(http): add personal research service`) +- Review-fix commit: `73405d85f047e35354bb340c90faf8374b23a2ba` (`fix(http): close service review gaps`) - Delivery boundary: the complete HTTP-service spec and tickets 01, 02, and 03; code review and deployment were excluded. The implementation was completed in dependency order: the server-local Bridge-first Search and typed Bridge Data Operations, the remaining Research Operations and container guide, and then the release/OpenAPI/GHCR contract. +## Review fixes + +- Removed the unused `createHttpService` alias and compatibility comment; the canonical `createHttpApp` export is the only HTTP-app constructor. +- Added a test-owned temporary-file artifact test that calls the YAML generator, reads and parses `openapi.yaml`, checks package-version parity, all ten `/v1` paths, and the absence of a generic Bridge route. +- Added table-driven invalid-body cases for Links, Docs Resolve, Docs Fetch, Source Search, Sports, Finance, and Time. Every case asserts HTTP 400, `invalid_request`, and no operation invocation. +- Narrowed the HTTP Search response/OpenAPI provider enum to `Exa` and `Kepos Bridge`; provider-selection behavior for CLI, MCP, Pi, and DSH remains unchanged. + ## Ticket outcomes ### 01 — Serve Search and Bridge Data over HTTP @@ -36,13 +44,13 @@ The implementation was completed in dependency order: the server-local Bridge-fi ## Verification -All checks below completed successfully against the implementation commit: +All checks below completed successfully against the review-fix commit: - `pnpm format:check` — all files matched Prettier. - `pnpm typecheck` — TypeScript completed with no errors. - `pnpm build` — all workspace packages built; `packages/web/dist/openapi.yaml` generated. - Generated OpenAPI parse check — OpenAPI `3.1.0`, package version `0.1.0`, exactly the 10 documented `/v1` paths, and no generic Bridge route. -- `pnpm test` — 18 test files and 109 tests passed. The existing DSH source-map warning was non-fatal. +- `pnpm test` — 19 test files and 117 tests passed. The existing DSH source-map warning was non-fatal. - `pnpm test:release` — release version synchronization fixtures passed. - `pnpm test:pack` — packed-installation/host-loading smoke tests passed for `@guionai/web`, `@guionai/pi-web`, and `@guionai/dsh-web`. - Focused in-process HTTP tests use injected operation fakes and test-owned credentials; they cover validation, exact fallback/cancellation behavior, typed Bridge command shapes, response normalization, startup configuration, and OpenAPI paths without provider or browser network calls. @@ -51,11 +59,11 @@ The local Docker daemon was unavailable for a Docker build check (`/Users/neil/. ## Changed paths and size -Against the fixed point, excluding generated `dist` output (including `openapi.yaml`) and `pnpm-lock.yaml`: +Against the fixed point, excluding generated `dist` output (including `openapi.yaml`), `pnpm-lock.yaml`, and this report: -- Product code: 932 additions, 1 deletion (`packages/web/src`, `packages/web-core/src`). -- Tests: 385 additions (`packages/web/test`, `packages/web-core/test`). +- Product code: 938 additions, 1 deletion (`packages/web/src`, `packages/web-core/src`). +- Tests: 485 additions (`packages/web/test`, `packages/web-core/test`). - Configuration and documentation: 240 additions, 14 deletions (Docker, release workflow, manifests, README, CONTEXT, ADR, and deferred note). -- Total: 1,557 additions and 15 deletions (net 1,542 lines). +- Total: 1,663 additions and 15 deletions (net 1,648 lines). -The total is within the spec estimate of 1,230–1,900 lines. The focused HTTP suite is 385 lines, just below the 400-line test estimate because route behavior is consolidated in one in-process fixture file rather than duplicated across transport tests. +The total is within the spec estimate of 1,230–1,900 lines. The review adds focused file-artifact and table-driven validation coverage while keeping all fixtures in process or test-owned temporary directories. From 7ab78e72806fccb2850c4702eb722d6cebc2a76e Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 14:23:50 +0800 Subject: [PATCH 05/14] fix(http): support ARM container rendering --- Dockerfile | 10 ++++++++-- README.md | 5 +++-- packages/web-core/src/fetch.ts | 3 +++ packages/web-core/test/fetch.test.ts | 8 ++++++++ 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 271da47..a78f982 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,13 +16,19 @@ RUN pnpm --filter @guionai/web run build FROM node:24-bookworm ENV NODE_ENV=production +# Chrome for Testing has no Linux ARM64 distribution. Debian's Chromium works +# on both released container architectures, and agent-browser documents this +# executable override for containers. +ENV AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium WORKDIR /app # Rendering is deliberately explicit at the HTTP contract. The executable and # browser runtime are image capabilities, while credentials remain env-only. ARG AGENT_BROWSER_VERSION=0.36.0 -RUN npm install --global agent-browser@${AGENT_BROWSER_VERSION} \ - && agent-browser install +RUN apt-get update \ + && apt-get install --yes --no-install-recommends chromium \ + && rm -rf /var/lib/apt/lists/* \ + && npm install --global agent-browser@${AGENT_BROWSER_VERSION} COPY --from=build /workspace/packages/web/dist ./dist diff --git a/README.md b/README.md index bc900f2..191b800 100644 --- a/README.md +++ b/README.md @@ -93,8 +93,9 @@ Fetch and Links use direct HTTP fetching when `render` is omitted (or set to `"fetch"`). Rendered fetching is explicit and requires both `render: "agent-browser"` and an integer `waitMs` from 0 through 30,000; direct fetch never silently switches backends. The container installs `agent-browser` -and its browser runtime, while credentials and Bridge configuration remain -server-local environment variables. +with Debian Chromium (including Linux ARM64, where Chrome for Testing has no +build), while credentials and Bridge configuration remain server-local +environment variables. This is a Personal Web Service: a single-trust-boundary deployment for its operator and agents. It is not hardened for public or multi-tenant exposure; diff --git a/packages/web-core/src/fetch.ts b/packages/web-core/src/fetch.ts index b76c88f..492d87e 100644 --- a/packages/web-core/src/fetch.ts +++ b/packages/web-core/src/fetch.ts @@ -641,6 +641,9 @@ function rendererEnvironment( TEMP: workDirectory, AGENT_BROWSER_CONFIG: configPath, }; + if (process.env.AGENT_BROWSER_EXECUTABLE_PATH !== undefined) + environment.AGENT_BROWSER_EXECUTABLE_PATH = + process.env.AGENT_BROWSER_EXECUTABLE_PATH; for (const name of [ "HTTP_PROXY", "HTTPS_PROXY", diff --git a/packages/web-core/test/fetch.test.ts b/packages/web-core/test/fetch.test.ts index bbdaec5..48a6100 100644 --- a/packages/web-core/test/fetch.test.ts +++ b/packages/web-core/test/fetch.test.ts @@ -338,6 +338,8 @@ describe.sequential("browserless fetch migrated from Organon", () => { const cacheDirectory = mkdtempSync( join(tmpdir(), "guionai-web-render-cache-"), ); + const previousExecutable = process.env.AGENT_BROWSER_EXECUTABLE_PATH; + process.env.AGENT_BROWSER_EXECUTABLE_PATH = "/usr/bin/chromium"; const resolveHost = async () => ["93.184.216.34"]; const directFetch = vi.fn( async () => @@ -405,6 +407,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { expect(open.home).toBeUndefined(); expect(open.config).toBe("{}\n"); expect(open.profile).toBeUndefined(); + expect(open.executable).toBe("/usr/bin/chromium"); await fetchWebPage( { @@ -417,6 +420,9 @@ describe.sequential("browserless fetch migrated from Organon", () => { ); expect(readFakeLog(logPath)).toHaveLength(6); } finally { + if (previousExecutable === undefined) + delete process.env.AGENT_BROWSER_EXECUTABLE_PATH; + else process.env.AGENT_BROWSER_EXECUTABLE_PATH = previousExecutable; rmSync(cacheDirectory, { recursive: true, force: true }); } }); @@ -786,6 +792,7 @@ appendFileSync(${JSON.stringify(logPath)}, JSON.stringify({ home: process.env.HOME, config: readFileSync(process.env.AGENT_BROWSER_CONFIG, "utf8"), profile: process.env.AGENT_BROWSER_PROFILE, + executable: process.env.AGENT_BROWSER_EXECUTABLE_PATH, }) + "\\n"); if (command === "open" && args.includes("https://ignore-term.test/page")) { process.on("SIGTERM", () => {}); @@ -819,6 +826,7 @@ type FakeCommand = { home?: string; config: string; profile?: string; + executable?: string; }; function readFakeLog(path: string): FakeCommand[] { From bbfba75292028f81b0c8d0a9a06963dd025cbdce Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 14:47:06 +0800 Subject: [PATCH 06/14] refactor(http): expose search and fetch only --- README.md | 34 +- docs/adr/0001-containerized-rest-api.md | 4 +- packages/web/src/http.ts | 386 +-------------------- packages/web/test/http.test.ts | 292 ++++------------ packages/web/test/openapi-artifact.test.ts | 13 +- 5 files changed, 78 insertions(+), 651 deletions(-) diff --git a/README.md b/README.md index 191b800..6a1caa8 100644 --- a/README.md +++ b/README.md @@ -62,35 +62,27 @@ docker run --rm -p 8787:8787 \ ghcr.io/guionai/web:v0.1.0 ``` -Every operation is a versioned JSON `POST` route. Request and response schemas +Every HTTP operation is a versioned JSON `POST` route. Request and response schemas are generated into `openapi.yaml` from the same route definitions: -| Route | Request | Purpose | -| ------------------- | ------------------------------------------------------------------------------------- | -------------------------------------------------------- | -| `/v1/search` | `{ "query": "..." }` | Kepos Bridge search with one Exa retry on Bridge failure | -| `/v1/fetch` | `{ "url", "tree?", "section_id?", "full?", "tree_threshold?", "render?", "waitMs?" }` | Fetch Markdown | -| `/v1/links` | `{ "url", "limit?", "render?", "waitMs?" }` | List page HTTP(S) links | -| `/v1/docs/resolve` | `{ "query" }` | Resolve a Context7 library | -| `/v1/docs/fetch` | `{ "library_id", "topic?", "tokens?" }` | Fetch Context7 documentation | -| `/v1/source-search` | `{ "query", "count?", "context?", "timeout?" }` | Search public source through Sourcegraph | -| `/v1/weather` | `{ "location", "start?", "duration?" }` | Typed Bridge weather lookup | -| `/v1/sports` | `{ "fn", "league", ... }` | Typed Bridge schedule or standings lookup | -| `/v1/finance` | `{ "ticker", "type", "market?" }` | Typed Bridge quote or index lookup | -| `/v1/time` | `{ "utc_offset" }` | Typed Bridge time lookup | - -Weather, sports, finance, and time are separate Bridge-only operations; they do -not fall back to Exa and there is no generic Bridge-command route. Search keeps -a successful empty Bridge result, retries Exa exactly once for a non-cancellation -Bridge failure, and reports the provider in its response. Invalid JSON bodies, -unknown fields, and invalid typed values are rejected before an upstream call. +| Route | Request | Purpose | +| ------------ | ------------------------------------------------------------------------------------- | -------------------------------------------------------- | +| `/v1/search` | `{ "query": "..." }` | Kepos Bridge search with one Exa retry on Bridge failure | +| `/v1/fetch` | `{ "url", "tree?", "section_id?", "full?", "tree_threshold?", "render?", "waitMs?" }` | Fetch Markdown | + +Search keeps a successful empty Bridge result, retries Exa exactly once for a +non-cancellation Bridge failure, and reports the provider in its response. +Weather, sports, finance, and time are not exposed because the configured +providers do not offer contract-equivalent official typed data APIs. Invalid JSON +bodies, unknown fields, and invalid typed values are rejected before an upstream call. Upstream failures are bounded JSON errors and never include credentials or raw provider response bodies. Error responses use a stable `{ "code", "message", "details"? }` JSON shape; upstream failures use 502 (or 504 for an upstream timeout), while client cancellation is reported as 499. -Fetch and Links use direct HTTP fetching when `render` is omitted (or set to -`"fetch"`). Rendered fetching is explicit and requires both +Fetch uses direct HTTP fetching when `render` is omitted (or set to `"fetch"`). +Rendered fetching is explicit and requires both `render: "agent-browser"` and an integer `waitMs` from 0 through 30,000; direct fetch never silently switches backends. The container installs `agent-browser` with Debian Chromium (including Linux ARM64, where Chrome for Testing has no diff --git a/docs/adr/0001-containerized-rest-api.md b/docs/adr/0001-containerized-rest-api.md index d9f0a32..9203130 100644 --- a/docs/adr/0001-containerized-rest-api.md +++ b/docs/adr/0001-containerized-rest-api.md @@ -1,7 +1,7 @@ # Expose research operations through a containerized REST API -Guion Web will add a self-hosted, single-user HTTP service in a portable container. It will expose every existing read-only Research Operation plus typed Bridge Data Operations for weather, sports, finance, and time as REST routes defined with Hono and `@hono/zod-openapi`. Each tagged release will publish the generated OpenAPI 3.1 document as a downloadable `openapi.yaml` GitHub Release asset, alongside the matching npm packages and GHCR image. The current stdio MCP server remains supported, but remote Streamable HTTP MCP is outside this first service slice. +Guion Web will add a self-hosted, single-user HTTP service in a portable container. It will expose only provider-neutral web search and page fetch as REST routes defined with Hono and `@hono/zod-openapi`. Each tagged release will publish the generated OpenAPI 3.1 document as a downloadable `openapi.yaml` GitHub Release asset, alongside the matching npm packages and GHCR image. The current stdio MCP server remains supported, but remote Streamable HTTP MCP is outside this first service slice. ## Consequences -The service's provider credentials and Bridge Route are server-local configuration; clients cannot choose a provider or supply a Bridge Route per request. Search tries Kepos Bridge and uses Exa only when the Bridge is operationally unavailable; it does not fall back for cancellation, malformed client input, or an empty result set. A failed Bridge attempt is retried through Exa exactly once, and a successful empty Bridge response is returned unchanged. Weather, sports, finance, and time remain separate typed Bridge-only routes rather than a generic command passthrough, and never fall back to Exa. `agent-browser` is available as the explicit Rendered Fetch backend in the image. `openapi.yaml` is generated from the release build rather than manually maintained or independently versioned. Public or multi-tenant deployment hardening is deliberately deferred in `.scratch/defered/public-http-service-security.md`. +The service's provider credentials and Bridge Route are server-local configuration; clients cannot choose a provider or supply a Bridge Route per request. Search tries Kepos Bridge and uses Exa only when the Bridge is operationally unavailable; it does not fall back for cancellation, malformed client input, or an empty result set. A failed Bridge attempt is retried through Exa exactly once, and a successful empty Bridge response is returned unchanged. Typed Bridge operations are intentionally not exposed: Exa has no equivalent official weather, sports, or time API, and its premium finance integration is not contract-compatible. `agent-browser` is available as the explicit Rendered Fetch backend in the image. `openapi.yaml` is generated from the release build rather than manually maintained or independently versioned. Public or multi-tenant deployment hardening is deliberately deferred in `.scratch/defered/public-http-service-security.md`. diff --git a/packages/web/src/http.ts b/packages/web/src/http.ts index a05abf7..70d1db2 100644 --- a/packages/web/src/http.ts +++ b/packages/web/src/http.ts @@ -1,33 +1,21 @@ import { - DEFAULT_LINK_LIMIT, DEFAULT_KEPOS_BRIDGE_ENDPOINT, FetchCapabilityError, isOperationAborted, isRequestTimeout, - MAX_LINK_LIMIT, RENDER_REPORT_URL, throwIfAborted, validateKeposBridgeEndpoint, - type KeposBridgeResponse, type SearchResponse, type WebCredentials, type WebOperations, } from "@guionai/web-core"; import * as webCoreModule from "@guionai/web-core"; import { HTTPException } from "hono/http-exception"; -import { - createRoute, - OpenAPIHono, - z, - type RouteConfig, -} from "@hono/zod-openapi"; +import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi"; import { credentialsFromEnvironment } from "./runtime.js"; -const DEFAULT_SOURCE_COUNT = 10; -const DEFAULT_SOURCE_CONTEXT = 10; -const DEFAULT_SOURCE_TIMEOUT = 0; - /** Dependencies and server-local configuration for the personal HTTP service. */ export type HttpServiceDependencies = { operations?: WebOperations; @@ -87,56 +75,6 @@ const FetchResponseSchema = z .strict() .openapi("FetchResponse"); -const LinkSchema = z - .object({ text: z.string(), url: z.string() }) - .strict() - .openapi("PageLink"); - -const LinksResponseSchema = z - .object({ - url: z.string(), - links: z.array(LinkSchema), - truncated: z.boolean(), - }) - .strict() - .openapi("LinksResponse"); - -const DocsLibrarySchema = z - .object({ - id: z.string(), - title: z.string(), - description: z.string(), - trust_score: z.number(), - total_snippets: z.number().int(), - versions: z.array(z.string()).optional(), - }) - .strict() - .openapi("DocsLibrary"); - -const DocsResolveResponseSchema = z - .object({ query: z.string(), libraries: z.array(DocsLibrarySchema) }) - .strict() - .openapi("DocsResolveResponse"); - -const DocsFetchResponseSchema = z - .object({ - library_id: z.string(), - topic: z.string().optional(), - content: z.string(), - }) - .strict() - .openapi("DocsFetchResponse"); - -const SourceSearchResponseSchema = z - .object({ content: z.string() }) - .strict() - .openapi("SourceSearchResponse"); - -const KeposResponseSchema = z - .object({ output: z.string(), results: z.array(z.unknown()).optional() }) - .strict() - .openapi("KeposResponse"); - const SearchRequestSchema = z .object({ query: z.string().min(1) }) .strict() @@ -160,99 +98,6 @@ const FetchRequestSchema = z .strict() .openapi("FetchRequest"); -const LinksRequestSchema = z - .object({ - url: HttpUrlSchema, - limit: z - .number() - .int() - .min(1) - .max(MAX_LINK_LIMIT) - .default(DEFAULT_LINK_LIMIT), - render: z.enum(["fetch", "agent-browser"]).default("fetch"), - waitMs: z.number().int().min(0).max(30_000).optional(), - }) - .strict() - .openapi("LinksRequest"); - -const DocsResolveRequestSchema = z - .object({ query: z.string().min(1) }) - .strict() - .openapi("DocsResolveRequest"); - -const DocsFetchRequestSchema = z - .object({ - library_id: z.string().min(1), - topic: z.string().optional(), - tokens: z.number().int().min(0).default(0), - }) - .strict() - .openapi("DocsFetchRequest"); - -const SourceSearchRequestSchema = z - .object({ - query: z.string().min(1), - count: z.number().int().default(DEFAULT_SOURCE_COUNT), - context: z.number().int().default(DEFAULT_SOURCE_CONTEXT), - timeout: z.number().int().min(0).default(DEFAULT_SOURCE_TIMEOUT), - }) - .strict() - .openapi("SourceSearchRequest"); - -const DateSchema = z.string().regex(/^\d{4}-\d{2}-\d{2}$/); -const NonBlankStringSchema = z - .string() - .refine((value) => value.trim().length > 0, "must be a non-blank string"); - -const WeatherRequestSchema = z - .object({ - location: NonBlankStringSchema, - start: DateSchema.optional(), - duration: z.number().int().positive().safe().optional(), - }) - .strict() - .openapi("WeatherRequest"); - -const SportsRequestSchema = z - .object({ - fn: z.enum(["schedule", "standings"]), - league: z.enum([ - "nba", - "wnba", - "nfl", - "nhl", - "mlb", - "epl", - "ncaamb", - "ncaawb", - "ipl", - ]), - team: NonBlankStringSchema.optional(), - opponent: NonBlankStringSchema.optional(), - date_from: DateSchema.optional(), - date_to: DateSchema.optional(), - num_games: z.number().int().positive().safe().optional(), - locale: NonBlankStringSchema.optional(), - }) - .strict() - .openapi("SportsRequest"); - -const FinanceRequestSchema = z - .object({ - ticker: NonBlankStringSchema, - type: z.enum(["equity", "fund", "crypto", "index"]), - market: NonBlankStringSchema.optional(), - }) - .strict() - .openapi("FinanceRequest"); - -const TimeRequestSchema = z - .object({ - utc_offset: z.string().regex(/^[+-](?:[01]\d|2[0-3]):[0-5]\d$/), - }) - .strict() - .openapi("TimeRequest"); - const commonResponses = { 400: { content: { "application/json": { schema: ErrorSchema } }, @@ -300,104 +145,6 @@ const fetchRoute = createRoute({ }, }); -const linksRoute = createRoute({ - method: "post", - path: "/v1/links", - operationId: "links", - summary: "List page links", - description: - "List HTTP(S) anchors using direct fetch by default or explicit agent-browser rendering.", - request: jsonRequest(LinksRequestSchema), - responses: { - 200: jsonResponse(LinksResponseSchema, "Page links."), - ...commonResponses, - }, -}); - -const docsResolveRoute = createRoute({ - method: "post", - path: "/v1/docs/resolve", - operationId: "docsResolve", - summary: "Resolve documentation", - request: jsonRequest(DocsResolveRequestSchema), - responses: { - 200: jsonResponse(DocsResolveResponseSchema, "Resolved libraries."), - ...commonResponses, - }, -}); - -const docsFetchRoute = createRoute({ - method: "post", - path: "/v1/docs/fetch", - operationId: "docsFetch", - summary: "Fetch documentation", - request: jsonRequest(DocsFetchRequestSchema), - responses: { - 200: jsonResponse(DocsFetchResponseSchema, "Documentation content."), - ...commonResponses, - }, -}); - -const sourceSearchRoute = createRoute({ - method: "post", - path: "/v1/source-search", - operationId: "sourceSearch", - summary: "Search public source code", - request: jsonRequest(SourceSearchRequestSchema), - responses: { - 200: jsonResponse(SourceSearchResponseSchema, "Sourcegraph results."), - ...commonResponses, - }, -}); - -const weatherRoute = createRoute({ - method: "post", - path: "/v1/weather", - operationId: "weather", - summary: "Look up weather", - request: jsonRequest(WeatherRequestSchema), - responses: { - 200: jsonResponse(KeposResponseSchema, "Weather data."), - ...commonResponses, - }, -}); - -const sportsRoute = createRoute({ - method: "post", - path: "/v1/sports", - operationId: "sports", - summary: "Look up sports", - request: jsonRequest(SportsRequestSchema), - responses: { - 200: jsonResponse(KeposResponseSchema, "Sports data."), - ...commonResponses, - }, -}); - -const financeRoute = createRoute({ - method: "post", - path: "/v1/finance", - operationId: "finance", - summary: "Look up finance", - request: jsonRequest(FinanceRequestSchema), - responses: { - 200: jsonResponse(KeposResponseSchema, "Finance data."), - ...commonResponses, - }, -}); - -const timeRoute = createRoute({ - method: "post", - path: "/v1/time", - operationId: "time", - summary: "Look up time", - request: jsonRequest(TimeRequestSchema), - responses: { - 200: jsonResponse(KeposResponseSchema, "Time data."), - ...commonResponses, - }, -}); - /** Creates the in-process Hono application used by `web serve` and tests. */ export function createHttpApp( dependencies: HttpServiceDependencies = {}, @@ -443,80 +190,6 @@ export function createHttpApp( } }); - app.openapi(linksRoute, async (context) => { - const input = context.req.valid("json"); - const renderError = validateRenderFields(input.render, input.waitMs); - if (renderError) return context.json(renderError, 400) as never; - try { - const result = await state.operations.links( - { ...input, limit: input.limit ?? DEFAULT_LINK_LIMIT }, - context.req.raw.signal, - ); - throwIfAborted(context.req.raw.signal); - return context.json(parseResponse(LinksResponseSchema, result), 200); - } catch (error) { - return failureResponse(context, error, "links") as never; - } - }); - - app.openapi(docsResolveRoute, async (context) => { - const input = context.req.valid("json"); - try { - const result = await state.operations.docsResolve({ - query: input.query, - credentials: state.credentials, - signal: context.req.raw.signal, - }); - throwIfAborted(context.req.raw.signal); - return context.json( - parseResponse(DocsResolveResponseSchema, result), - 200, - ); - } catch (error) { - return failureResponse(context, error, "docs resolve") as never; - } - }); - - app.openapi(docsFetchRoute, async (context) => { - const input = context.req.valid("json"); - try { - const result = await state.operations.docsFetch({ - ...input, - credentials: state.credentials, - signal: context.req.raw.signal, - }); - throwIfAborted(context.req.raw.signal); - return context.json(parseResponse(DocsFetchResponseSchema, result), 200); - } catch (error) { - return failureResponse(context, error, "docs fetch") as never; - } - }); - - app.openapi(sourceSearchRoute, async (context) => { - const input = context.req.valid("json"); - try { - const result = await state.operations.sgraphSearch({ - query: input.query, - count: input.count ?? DEFAULT_SOURCE_COUNT, - context: input.context ?? DEFAULT_SOURCE_CONTEXT, - timeout: input.timeout ?? DEFAULT_SOURCE_TIMEOUT, - signal: context.req.raw.signal, - }); - throwIfAborted(context.req.raw.signal); - return context.json( - parseResponse(SourceSearchResponseSchema, result), - 200, - ); - } catch (error) { - return failureResponse(context, error, "source search") as never; - } - }); - - registerKeposRoute(app, weatherRoute, "weather", "weather", state); - registerKeposRoute(app, sportsRoute, "sports", "sports", state); - registerKeposRoute(app, financeRoute, "finance", "finance", state); - registerKeposRoute(app, timeRoute, "time", "time", state); - app.onError((error, context) => failureResponse(context, error, "request")); app.notFound((context) => context.json(errorBody("not_found", "Route not found"), 404), @@ -659,52 +332,6 @@ async function searchWithFallback( } } -function registerKeposRoute< - R extends RouteConfig, - Input extends Record, ->( - app: OpenAPIHono, - route: R, - command: "weather" | "sports" | "finance" | "time", - operationName: string, - state: HttpServiceState, -): void { - (app as any).openapi(route, async (context: any) => { - const input = context.req.valid("json") as Input; - try { - const result = await state.operations.keposBridge({ - endpoint: state.keposBridgeEndpoint, - commands: { [command]: [input] }, - signal: context.req.raw.signal, - }); - throwIfAborted(context.req.raw.signal); - const normalized = normalizeKeposResponse(result); - return context.json(normalized, 200); - } catch (error) { - return failureResponse(context, error, operationName) as never; - } - }); -} - -function isKeposResponse(value: unknown): value is KeposBridgeResponse { - return ( - !!value && - typeof value === "object" && - !Array.isArray(value) && - typeof (value as { output?: unknown }).output === "string" && - ((value as { results?: unknown }).results === undefined || - Array.isArray((value as { results?: unknown }).results)) - ); -} - -function normalizeKeposResponse(value: unknown): KeposBridgeResponse { - if (!isKeposResponse(value)) throw new Error("malformed Bridge response"); - return { - output: value.output, - ...(value.results === undefined ? {} : { results: value.results }), - }; -} - function parseResponse(schema: z.ZodType, value: unknown): T { const parsed = schema.safeParse(value); if (!parsed.success) throw new Error("malformed operation response"); @@ -807,20 +434,9 @@ function safeFetchDetails( } export { - DocsFetchResponseSchema, - DocsLibrarySchema, - DocsResolveResponseSchema, ErrorSchema, FetchRequestSchema, FetchResponseSchema, - FinanceRequestSchema, - KeposResponseSchema, - LinksRequestSchema, - LinksResponseSchema, SearchRequestSchema, SearchResponseSchema, - SourceSearchRequestSchema, - SportsRequestSchema, - TimeRequestSchema, - WeatherRequestSchema, }; diff --git a/packages/web/test/http.test.ts b/packages/web/test/http.test.ts index 259d631..f6af57e 100644 --- a/packages/web/test/http.test.ts +++ b/packages/web/test/http.test.ts @@ -42,7 +42,7 @@ function dependencies( ): HttpServiceDependencies { return { operations: operations(), - credentials: { exaApiKey: "exa-secret", context7ApiKey: "ctx-secret" }, + credentials: { exaApiKey: "exa-secret" }, keposBridgeEndpoint: "http://bridge.test/route", ...overrides, }; @@ -52,7 +52,7 @@ async function json( app: ReturnType, path: string, body: unknown, -): Promise<{ response: Response; body: any }> { +): Promise<{ response: Response; body: unknown }> { const response = await app.request(path, { method: "POST", headers: { "content-type": "application/json" }, @@ -64,11 +64,10 @@ async function json( describe("personal HTTP service", () => { it("returns an empty successful Bridge search without retrying Exa", async () => { const ops = operations(); - const app = createHttpApp({ - ...dependencies(), - operations: ops, - }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + const result = await json(app, "/v1/search", { query: "empty" }); + expect(result.response.status).toBe(200); expect(result.body).toEqual({ provider: "Kepos Bridge", results: [] }); expect(ops.search).toHaveBeenCalledTimes(1); @@ -87,22 +86,14 @@ describe("personal HTTP service", () => { search: vi .fn() .mockRejectedValueOnce(new Error("bridge unavailable")) - .mockResolvedValueOnce({ - provider: "Exa", - results: [ - { - title: "result", - link: "https://example.test", - snippet: "snippet", - position: 1, - }, - ], - }), + .mockResolvedValueOnce({ provider: "Exa", results: [] }), }); const app = createHttpApp({ ...dependencies(), operations: ops }); + const result = await json(app, "/v1/search", { query: "fallback" }); + expect(result.response.status).toBe(200); - expect(result.body.provider).toBe("Exa"); + expect(result.body).toEqual({ provider: "Exa", results: [] }); expect(ops.search).toHaveBeenCalledTimes(2); expect(ops.search).toHaveBeenNthCalledWith( 2, @@ -110,35 +101,23 @@ describe("personal HTTP service", () => { ); }); - it("treats a non-empty Bridge payload with no usable links as a failure", async () => { - const ops = operations({ - search: vi - .fn() - .mockRejectedValueOnce(new Error("bridge invalid")) - .mockResolvedValueOnce({ provider: "Exa", results: [] }), - }); - const app = createHttpApp({ ...dependencies(), operations: ops }); - const result = await json(app, "/v1/search", { query: "invalid" }); - expect(result.response.status).toBe(200); - expect(result.body).toEqual({ provider: "Exa", results: [] }); - expect(ops.search).toHaveBeenCalledTimes(2); - }); - it("does not retry a cancelled Bridge request", async () => { const controller = new AbortController(); const ops = operations({ - search: vi.fn(async ({ signal }) => { + search: vi.fn(async () => { controller.abort(); throw new Error("Operation aborted"); }), }); const app = createHttpApp({ ...dependencies(), operations: ops }); + const response = await app.request("/v1/search", { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ query: "cancel" }), signal: controller.signal, }); + expect(response.status).toBe(499); expect(ops.search).toHaveBeenCalledTimes(1); }); @@ -151,130 +130,34 @@ describe("personal HTTP service", () => { .mockRejectedValueOnce(new RequestTimeoutError("exa timed out", true)), }); const app = createHttpApp({ ...dependencies(), operations: ops }); + const result = await json(app, "/v1/search", { query: "timeout" }); + expect(result.response.status).toBe(504); expect(result.body).toEqual({ code: "upstream_timeout", message: "search timed out", }); - expect(ops.search).toHaveBeenCalledTimes(2); - }); - - it("keeps Bridge data operations typed, separate, and Bridge-only", async () => { - const ops = operations({ - keposBridge: vi.fn(async () => ({ - output: "bridge", - results: [{ future: true }], - future_field: "ignored", - })), - }); - const app = createHttpApp({ ...dependencies(), operations: ops }); - const weather = await json(app, "/v1/weather", { - location: "Country, Area, City", - start: "2026-09-01", - duration: 3, - }); - expect(weather.response.status).toBe(200); - expect(weather.body).toEqual({ - output: "bridge", - results: [{ future: true }], - }); - await json(app, "/v1/sports", { fn: "schedule", league: "nba" }); - await json(app, "/v1/finance", { ticker: "BTC", type: "crypto" }); - await json(app, "/v1/time", { utc_offset: "+08:00" }); - expect(ops.keposBridge).toHaveBeenNthCalledWith( - 1, - expect.objectContaining({ - endpoint: "http://bridge.test/route", - commands: { - weather: [ - { - location: "Country, Area, City", - start: "2026-09-01", - duration: 3, - }, - ], - }, - }), - ); - expect(ops.keposBridge).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ - commands: { sports: [{ fn: "schedule", league: "nba" }] }, - }), - ); - expect(ops.keposBridge).toHaveBeenNthCalledWith( - 3, - expect.objectContaining({ - commands: { finance: [{ ticker: "BTC", type: "crypto" }] }, - }), - ); - expect(ops.keposBridge).toHaveBeenNthCalledWith( - 4, - expect.objectContaining({ - commands: { time: [{ utc_offset: "+08:00" }] }, - }), - ); - expect(ops.search).not.toHaveBeenCalled(); }); - it("forwards every research operation with server credentials and defaults", async () => { + it("forwards fetch with its explicit rendered-fetch contract", async () => { const ops = operations({ fetch: vi.fn(async (input) => ({ url: input.url, mode: "tree" as const, content: "markdown", })), - links: vi.fn(async (input) => ({ - url: input.url, - links: [{ text: "docs", url: "https://example.test/docs" }], - truncated: false, - })), - docsResolve: vi.fn(async (input) => ({ - query: input.query, - libraries: [ - { - id: "/acme/docs", - title: "Docs", - description: "fixture", - trust_score: 9, - total_snippets: 2, - }, - ], - })), - docsFetch: vi.fn(async (input) => ({ - library_id: input.library_id, - content: "documentation", - })), - sgraphSearch: vi.fn(async () => ({ content: "source results" })), }); const app = createHttpApp({ ...dependencies(), operations: ops }); - expect( - ( - await json(app, "/v1/fetch", { - url: "https://example.test", - render: "agent-browser", - waitMs: 0, - }) - ).body, - ).toMatchObject({ mode: "tree" }); - expect( - (await json(app, "/v1/links", { url: "https://example.test" })).body, - ).toMatchObject({ links: [{ text: "docs" }] }); - expect( - (await json(app, "/v1/docs/resolve", { query: "acme" })).body, - ).toMatchObject({ libraries: [{ id: "/acme/docs" }] }); - expect( - ( - await json(app, "/v1/docs/fetch", { - library_id: "/acme/docs", - topic: "install", - }) - ).body, - ).toMatchObject({ content: "documentation" }); - expect( - (await json(app, "/v1/source-search", { query: "repo:acme" })).body, - ).toEqual({ content: "source results" }); + + const result = await json(app, "/v1/fetch", { + url: "https://example.test", + render: "agent-browser", + waitMs: 0, + }); + + expect(result.response.status).toBe(200); + expect(result.body).toMatchObject({ mode: "tree" }); expect(ops.fetch).toHaveBeenCalledWith( expect.objectContaining({ url: "https://example.test", @@ -286,49 +169,24 @@ describe("personal HTTP service", () => { }), expect.any(AbortSignal), ); - expect(ops.links).toHaveBeenCalledWith( - { url: "https://example.test", limit: 100, render: "fetch" }, - expect.any(AbortSignal), - ); - expect(ops.docsResolve).toHaveBeenCalledWith( - expect.objectContaining({ - query: "acme", - credentials: { exaApiKey: "exa-secret", context7ApiKey: "ctx-secret" }, - }), - ); - expect(ops.docsFetch).toHaveBeenCalledWith( - expect.objectContaining({ - library_id: "/acme/docs", - topic: "install", - tokens: 0, - }), - ); - expect(ops.sgraphSearch).toHaveBeenCalledWith( - expect.objectContaining({ - query: "repo:acme", - count: 10, - context: 10, - timeout: 0, - }), - ); }); - it("rejects invalid request bodies before invoking operations", async () => { + it("rejects invalid search and fetch requests before an operation", async () => { const ops = operations(); const app = createHttpApp({ ...dependencies(), operations: ops }); - const invalid = await json(app, "/v1/weather", { - location: "", - command: "search_query", - }); - expect(invalid.response.status).toBe(400); - expect(invalid.body.code).toBe("invalid_request"); - expect(ops.keposBridge).not.toHaveBeenCalled(); - const render = await json(app, "/v1/fetch", { - url: "https://example.test", - waitMs: 100, - }); - expect(render.response.status).toBe(400); + expect((await json(app, "/v1/search", { query: "" })).response.status).toBe( + 400, + ); + expect( + ( + await json(app, "/v1/fetch", { + url: "https://example.test", + waitMs: 100, + }) + ).response.status, + ).toBe(400); + expect(ops.search).not.toHaveBeenCalled(); expect(ops.fetch).not.toHaveBeenCalled(); const malformed = await app.request("/v1/search", { @@ -341,66 +199,38 @@ describe("personal HTTP service", () => { }); it.each([ - { - path: "/v1/links", - body: { url: "not-an-http-url" }, - }, - { - path: "/v1/docs/resolve", - body: { query: "" }, - }, - { - path: "/v1/docs/fetch", - body: { library_id: "", tokens: -1 }, - }, - { - path: "/v1/source-search", - body: { query: "", count: 10 }, - }, - { - path: "/v1/sports", - body: { fn: "schedule", league: "not-a-league" }, - }, - { - path: "/v1/finance", - body: { ticker: "", type: "equity" }, - }, - { - path: "/v1/time", - body: { utc_offset: "UTC" }, - }, - ])( - "rejects invalid $path bodies before any operation call", - async ({ path, body }) => { - const ops = operations(); - const app = createHttpApp({ ...dependencies(), operations: ops }); - const result = await json(app, path, body); - expect(result.response.status).toBe(400); - expect(result.body).toMatchObject({ code: "invalid_request" }); - for (const operation of Object.values(ops)) - expect(operation).not.toHaveBeenCalled(); - }, - ); + "/v1/links", + "/v1/docs/resolve", + "/v1/docs/fetch", + "/v1/source-search", + "/v1/weather", + "/v1/sports", + "/v1/finance", + "/v1/time", + ])("does not expose %s", async (path) => { + const ops = operations(); + const app = createHttpApp({ ...dependencies(), operations: ops }); + + const result = await json(app, path, {}); - it("exposes all typed routes in a generated OpenAPI 3.1 document", () => { + expect(result.response.status).toBe(404); + expect(result.body).toEqual({ + code: "not_found", + message: "Route not found", + }); + for (const operation of Object.values(ops)) + expect(operation).not.toHaveBeenCalled(); + }); + + it("exposes only search and fetch in a generated OpenAPI 3.1 document", () => { const document = createHttpOpenAPIDocument("1.2.3"); + expect(document.openapi).toBe("3.1.0"); expect(document.info.version).toBe("1.2.3"); expect(Object.keys(document.paths ?? {}).sort()).toEqual([ - "/v1/docs/fetch", - "/v1/docs/resolve", "/v1/fetch", - "/v1/finance", - "/v1/links", "/v1/search", - "/v1/source-search", - "/v1/sports", - "/v1/time", - "/v1/weather", ]); - expect( - Object.keys(document.paths ?? {}).some((path) => path.includes("bridge")), - ).toBe(false); expect( (document.components?.schemas as any).SearchResponse.properties.provider .enum, diff --git a/packages/web/test/openapi-artifact.test.ts b/packages/web/test/openapi-artifact.test.ts index 6392df3..ed7bc3e 100644 --- a/packages/web/test/openapi-artifact.test.ts +++ b/packages/web/test/openapi-artifact.test.ts @@ -9,18 +9,7 @@ import { describe, expect, it } from "vitest"; import { generateOpenAPI } from "../src/generate-openapi.js"; const packageRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); -const expectedPaths = [ - "/v1/docs/fetch", - "/v1/docs/resolve", - "/v1/fetch", - "/v1/finance", - "/v1/links", - "/v1/search", - "/v1/source-search", - "/v1/sports", - "/v1/time", - "/v1/weather", -]; +const expectedPaths = ["/v1/fetch", "/v1/search"]; describe("release OpenAPI artifact", () => { it("serializes and parses the versioned contract from a test-owned file", async () => { From 9fff0766001f5ebcad91045c34958af72841da09 Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 14:49:29 +0800 Subject: [PATCH 07/14] feat(http): restore link discovery route --- README.md | 5 +- docs/adr/0001-containerized-rest-api.md | 2 +- packages/web/src/http.ts | 65 +++++++++++++++++++++- packages/web/test/http.test.ts | 44 ++++++++++++++- packages/web/test/openapi-artifact.test.ts | 2 +- 5 files changed, 111 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 6a1caa8..adb758d 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ are generated into `openapi.yaml` from the same route definitions: | ------------ | ------------------------------------------------------------------------------------- | -------------------------------------------------------- | | `/v1/search` | `{ "query": "..." }` | Kepos Bridge search with one Exa retry on Bridge failure | | `/v1/fetch` | `{ "url", "tree?", "section_id?", "full?", "tree_threshold?", "render?", "waitMs?" }` | Fetch Markdown | +| `/v1/links` | `{ "url", "limit?", "render?", "waitMs?" }` | List page HTTP(S) links | Search keeps a successful empty Bridge result, retries Exa exactly once for a non-cancellation Bridge failure, and reports the provider in its response. @@ -81,8 +82,8 @@ Error responses use a stable `{ "code", "message", "details"? }` JSON shape; upstream failures use 502 (or 504 for an upstream timeout), while client cancellation is reported as 499. -Fetch uses direct HTTP fetching when `render` is omitted (or set to `"fetch"`). -Rendered fetching is explicit and requires both +Fetch and Links use direct HTTP fetching when `render` is omitted (or set to +`"fetch"`). Rendered fetching is explicit and requires both `render: "agent-browser"` and an integer `waitMs` from 0 through 30,000; direct fetch never silently switches backends. The container installs `agent-browser` with Debian Chromium (including Linux ARM64, where Chrome for Testing has no diff --git a/docs/adr/0001-containerized-rest-api.md b/docs/adr/0001-containerized-rest-api.md index 9203130..7a4ee66 100644 --- a/docs/adr/0001-containerized-rest-api.md +++ b/docs/adr/0001-containerized-rest-api.md @@ -1,6 +1,6 @@ # Expose research operations through a containerized REST API -Guion Web will add a self-hosted, single-user HTTP service in a portable container. It will expose only provider-neutral web search and page fetch as REST routes defined with Hono and `@hono/zod-openapi`. Each tagged release will publish the generated OpenAPI 3.1 document as a downloadable `openapi.yaml` GitHub Release asset, alongside the matching npm packages and GHCR image. The current stdio MCP server remains supported, but remote Streamable HTTP MCP is outside this first service slice. +Guion Web will add a self-hosted, single-user HTTP service in a portable container. It will expose only provider-neutral web search, page fetch, and link discovery as REST routes defined with Hono and `@hono/zod-openapi`. Each tagged release will publish the generated OpenAPI 3.1 document as a downloadable `openapi.yaml` GitHub Release asset, alongside the matching npm packages and GHCR image. The current stdio MCP server remains supported, but remote Streamable HTTP MCP is outside this first service slice. ## Consequences diff --git a/packages/web/src/http.ts b/packages/web/src/http.ts index 70d1db2..e6466b0 100644 --- a/packages/web/src/http.ts +++ b/packages/web/src/http.ts @@ -1,8 +1,10 @@ import { + DEFAULT_LINK_LIMIT, DEFAULT_KEPOS_BRIDGE_ENDPOINT, FetchCapabilityError, isOperationAborted, isRequestTimeout, + MAX_LINK_LIMIT, RENDER_REPORT_URL, throwIfAborted, validateKeposBridgeEndpoint, @@ -75,6 +77,20 @@ const FetchResponseSchema = z .strict() .openapi("FetchResponse"); +const LinkSchema = z + .object({ text: z.string(), url: z.string() }) + .strict() + .openapi("PageLink"); + +const LinksResponseSchema = z + .object({ + url: z.string(), + links: z.array(LinkSchema), + truncated: z.boolean(), + }) + .strict() + .openapi("LinksResponse"); + const SearchRequestSchema = z .object({ query: z.string().min(1) }) .strict() @@ -98,6 +114,21 @@ const FetchRequestSchema = z .strict() .openapi("FetchRequest"); +const LinksRequestSchema = z + .object({ + url: HttpUrlSchema, + limit: z + .number() + .int() + .min(1) + .max(MAX_LINK_LIMIT) + .default(DEFAULT_LINK_LIMIT), + render: z.enum(["fetch", "agent-browser"]).default("fetch"), + waitMs: z.number().int().min(0).max(30_000).optional(), + }) + .strict() + .openapi("LinksRequest"); + const commonResponses = { 400: { content: { "application/json": { schema: ErrorSchema } }, @@ -145,6 +176,20 @@ const fetchRoute = createRoute({ }, }); +const linksRoute = createRoute({ + method: "post", + path: "/v1/links", + operationId: "links", + summary: "List page links", + description: + "List HTTP(S) anchors using direct fetch by default or explicit agent-browser rendering.", + request: jsonRequest(LinksRequestSchema), + responses: { + 200: jsonResponse(LinksResponseSchema, "Page links."), + ...commonResponses, + }, +}); + /** Creates the in-process Hono application used by `web serve` and tests. */ export function createHttpApp( dependencies: HttpServiceDependencies = {}, @@ -190,6 +235,22 @@ export function createHttpApp( } }); + app.openapi(linksRoute, async (context) => { + const input = context.req.valid("json"); + const renderError = validateRenderFields(input.render, input.waitMs); + if (renderError) return context.json(renderError, 400) as never; + try { + const result = await state.operations.links( + { ...input, limit: input.limit ?? DEFAULT_LINK_LIMIT }, + context.req.raw.signal, + ); + throwIfAborted(context.req.raw.signal); + return context.json(parseResponse(LinksResponseSchema, result), 200); + } catch (error) { + return failureResponse(context, error, "links") as never; + } + }); + app.onError((error, context) => failureResponse(context, error, "request")); app.notFound((context) => context.json(errorBody("not_found", "Route not found"), 404), @@ -210,7 +271,7 @@ export function createHttpOpenAPIDocument(version = "0.1.0") { title: "Guion Web Personal HTTP Service", version, description: - "Read-only Guion Web research operations and typed Kepos Bridge data operations.", + "Provider-neutral Guion Web search, page fetch, and link discovery.", }, }); } @@ -437,6 +498,8 @@ export { ErrorSchema, FetchRequestSchema, FetchResponseSchema, + LinksRequestSchema, + LinksResponseSchema, SearchRequestSchema, SearchResponseSchema, }; diff --git a/packages/web/test/http.test.ts b/packages/web/test/http.test.ts index f6af57e..fcbebf5 100644 --- a/packages/web/test/http.test.ts +++ b/packages/web/test/http.test.ts @@ -171,6 +171,37 @@ describe("personal HTTP service", () => { ); }); + it("forwards links with the same explicit rendered-fetch contract", async () => { + const ops = operations({ + links: vi.fn(async (input) => ({ + url: input.url, + links: [{ text: "Docs", url: "https://example.test/docs" }], + truncated: false, + })), + }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + + const result = await json(app, "/v1/links", { + url: "https://example.test", + render: "agent-browser", + waitMs: 0, + }); + + expect(result.response.status).toBe(200); + expect(result.body).toMatchObject({ + links: [{ text: "Docs", url: "https://example.test/docs" }], + }); + expect(ops.links).toHaveBeenCalledWith( + { + url: "https://example.test", + limit: 100, + render: "agent-browser", + waitMs: 0, + }, + expect.any(AbortSignal), + ); + }); + it("rejects invalid search and fetch requests before an operation", async () => { const ops = operations(); const app = createHttpApp({ ...dependencies(), operations: ops }); @@ -186,8 +217,17 @@ describe("personal HTTP service", () => { }) ).response.status, ).toBe(400); + expect( + ( + await json(app, "/v1/links", { + url: "https://example.test", + waitMs: 100, + }) + ).response.status, + ).toBe(400); expect(ops.search).not.toHaveBeenCalled(); expect(ops.fetch).not.toHaveBeenCalled(); + expect(ops.links).not.toHaveBeenCalled(); const malformed = await app.request("/v1/search", { method: "POST", @@ -199,7 +239,6 @@ describe("personal HTTP service", () => { }); it.each([ - "/v1/links", "/v1/docs/resolve", "/v1/docs/fetch", "/v1/source-search", @@ -222,13 +261,14 @@ describe("personal HTTP service", () => { expect(operation).not.toHaveBeenCalled(); }); - it("exposes only search and fetch in a generated OpenAPI 3.1 document", () => { + it("exposes search, fetch, and links in a generated OpenAPI 3.1 document", () => { const document = createHttpOpenAPIDocument("1.2.3"); expect(document.openapi).toBe("3.1.0"); expect(document.info.version).toBe("1.2.3"); expect(Object.keys(document.paths ?? {}).sort()).toEqual([ "/v1/fetch", + "/v1/links", "/v1/search", ]); expect( diff --git a/packages/web/test/openapi-artifact.test.ts b/packages/web/test/openapi-artifact.test.ts index ed7bc3e..c37cf52 100644 --- a/packages/web/test/openapi-artifact.test.ts +++ b/packages/web/test/openapi-artifact.test.ts @@ -9,7 +9,7 @@ import { describe, expect, it } from "vitest"; import { generateOpenAPI } from "../src/generate-openapi.js"; const packageRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); -const expectedPaths = ["/v1/fetch", "/v1/search"]; +const expectedPaths = ["/v1/fetch", "/v1/links", "/v1/search"]; describe("release OpenAPI artifact", () => { it("serializes and parses the versioned contract from a test-owned file", async () => { From bb4900a5dba9b5d71fdd64363ce860953b2dc098 Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 15:37:33 +0800 Subject: [PATCH 08/14] feat(http): unify page-reading contract --- README.md | 81 +++++++------ docs/adr/0001-containerized-rest-api.md | 2 +- docs/http-service.md | 154 ++++++++++++++++++++++++ packages/dsh-web/README.md | 16 ++- packages/dsh-web/package.json | 2 +- packages/dsh-web/src/client.ts | 5 +- packages/dsh-web/src/tools.ts | 99 +++++++++------ packages/dsh-web/test/artifact.test.ts | 4 +- packages/dsh-web/test/client.test.ts | 10 +- packages/dsh-web/test/tools.test.ts | 39 +++--- packages/pi-web/package.json | 2 +- packages/pi-web/src/tool.ts | 124 +++++++++++-------- packages/pi-web/test/extension.test.ts | 45 +++---- packages/pi-web/test/packed-smoke.mjs | 4 +- packages/web-core/src/fetch.ts | 90 ++++++++++---- packages/web-core/src/index.ts | 1 + packages/web-core/src/markdown.ts | 32 ++--- packages/web-core/test/fetch.test.ts | 98 ++++++++++++--- packages/web-core/test/markdown.test.ts | 49 +++----- packages/web/package.json | 2 +- packages/web/src/http.ts | 39 +++--- packages/web/src/mcp.ts | 68 +++++------ packages/web/src/program.ts | 40 +++--- packages/web/src/runner.ts | 2 +- packages/web/test/http.test.ts | 31 ++++- packages/web/test/mcp.test.ts | 48 +++----- packages/web/test/packed-smoke.mjs | 4 +- packages/web/test/program.test.ts | 37 ++---- 28 files changed, 703 insertions(+), 425 deletions(-) create mode 100644 docs/http-service.md diff --git a/README.md b/README.md index adb758d..f7105ee 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@ Guion Web is a Node.js web research toolkit. It provides Exa, Brave, or a managed Kepos Bridge search endpoint, Context7 library documentation lookup, Sourcegraph public code search, page-link -discovery, and two page-fetch backends through a CLI, stdio MCP server, personal -HTTP service, Pi extension, and DeepSeek Harness (DSH) integration: direct -HTML-to-Markdown extraction and explicit `agent-browser` rendering for -client-rendered pages on supported hosts. +discovery, and two page-rendering modes through a CLI, stdio MCP server, personal +HTTP service, Pi extension, and DeepSeek Harness (DSH) integration: HTTP +HTML-to-Markdown extraction and explicit browser rendering for client-rendered +pages on supported hosts. ## Install and configure @@ -65,11 +65,13 @@ docker run --rm -p 8787:8787 \ Every HTTP operation is a versioned JSON `POST` route. Request and response schemas are generated into `openapi.yaml` from the same route definitions: -| Route | Request | Purpose | -| ------------ | ------------------------------------------------------------------------------------- | -------------------------------------------------------- | -| `/v1/search` | `{ "query": "..." }` | Kepos Bridge search with one Exa retry on Bridge failure | -| `/v1/fetch` | `{ "url", "tree?", "section_id?", "full?", "tree_threshold?", "render?", "waitMs?" }` | Fetch Markdown | -| `/v1/links` | `{ "url", "limit?", "render?", "waitMs?" }` | List page HTTP(S) links | +| Route | Request | Purpose | +| ------------ | --------------------------------------------------------- | -------------------------------------------------------- | +| `/v1/search` | `{ "query": "..." }` | Kepos Bridge search with one Exa retry on Bridge failure | +| `/v1/fetch` | `{ "url", "section_id?", "full?", "render?", "waitMs?" }` | Fetch Markdown | +| `/v1/links` | `{ "url", "limit?", "render?", "waitMs?" }` | List page HTTP(S) links | + +The complete human-readable contract is in the [HTTP service reference](docs/http-service.md). Search keeps a successful empty Bridge result, retries Exa exactly once for a non-cancellation Bridge failure, and reports the provider in its response. @@ -82,13 +84,13 @@ Error responses use a stable `{ "code", "message", "details"? }` JSON shape; upstream failures use 502 (or 504 for an upstream timeout), while client cancellation is reported as 499. -Fetch and Links use direct HTTP fetching when `render` is omitted (or set to -`"fetch"`). Rendered fetching is explicit and requires both -`render: "agent-browser"` and an integer `waitMs` from 0 through 30,000; direct -fetch never silently switches backends. The container installs `agent-browser` -with Debian Chromium (including Linux ARM64, where Chrome for Testing has no -build), while credentials and Bridge configuration remain server-local -environment variables. +Fetch and Links use HTTP rendering when `render` is omitted (or set to +`"http"`). Browser rendering is explicit and requires both `render: "browser"` +and an integer `waitMs` from 0 through 30,000; HTTP rendering never silently +switches backends. The container installs the `agent-browser` executable with +Debian Chromium (including Linux ARM64, where Chrome for Testing has no build), +while credentials and Bridge configuration remain server-local environment +variables. This is a Personal Web Service: a single-trust-boundary deployment for its operator and agents. It is not hardened for public or multi-tenant exposure; @@ -103,7 +105,7 @@ document on stdout, which is useful for automation. ```bash web search --provider exa -- "Node AbortSignal" web search --provider kepos-bridge -- "Node AbortSignal" -web fetch https://example.com/article --tree +web fetch https://example.com/article web fetch https://example.com/article --section introduction web links https://example.com/article --limit 50 web docs resolve react @@ -112,9 +114,11 @@ web sgraph --count 10 -- "repo:^github\\.com/nodejs/node$ AbortSignal" ``` Use `--` before a search or Sourcegraph query that begins with a hyphen. `fetch` -supports `--full`, `--tree`, and `--section`; long extracted documents default to -a heading tree so a later request can retrieve a stable section ID. `links` lists -up to 100 unique HTTP(S) anchors from the original page DOM. +supports `--full` and `--section`; long extracted documents automatically return +a heading tree so a later request can retrieve a stable `section_id`. `--full` +returns the complete extracted Markdown, and `--full` cannot be combined with +`--section`. `links` lists up to 100 unique HTTP(S) anchors from the original +page DOM. ## MCP @@ -130,7 +134,7 @@ web mcp --provider kepos-bridge The server exposes six read-only tools: `search`, `fetch`, `links`, `docs_resolve`, `docs_fetch`, and `source_search`. Its stdout is reserved for MCP protocol messages; diagnostics go to stderr. For a client-rendered page, explicitly call -`fetch` or `links` with `render: "agent-browser"` and an integer `waitMs`; this optional +`fetch` or `links` with `render: "browser"` and an integer `waitMs`; this optional retry requires a host-installed executable and never happens automatically. ## Pi @@ -144,7 +148,7 @@ pi install npm:@guionai/pi-web It registers `web_search`, `web_fetch`, `web_links`, `web_docs`, and `web_source_search` and calls the bundled core in-process. Pi and TypeBox are peer dependencies supplied by the host; no CLI executable or MCP configuration is required. `web_fetch` uses -direct fetch by default and can explicitly use `render: "agent-browser"` with +HTTP rendering by default and can explicitly use `render: "browser"` with an integer `waitMs` when its host provides that optional executable. `web_links` uses the same explicit rendering contract and lists HTTP(S) anchors from the original page DOM. @@ -169,37 +173,36 @@ credentials. Selecting Kepos Bridge additionally exposes `web_weather`, `web_sports`, `web_finance`, and `web_time`; these tools are removed when another provider is selected. Fetch, link discovery, documentation, and Sourcegraph tools also run in-process. The host DSH packages and React are peers supplied by DSH. -`web_fetch` uses direct fetch by default and can explicitly use -`render: "agent-browser"` with an integer `waitMs` on a host that supplies the +`web_fetch` uses HTTP rendering by default and can explicitly use +`render: "browser"` with an integer `waitMs` on a host that supplies the optional executable. `web_links` uses the same explicit rendering contract and lists HTTP(S) anchors from the original page DOM. -## Page-fetch backends +## Page-rendering modes -`web fetch` has two backends. `fetch` (the default) uses Node `fetch`, -`linkedom`, and Defuddle for direct HTML-to-Markdown extraction from static, -SSR, and pre-rendered pages. `agent-browser` renders client-side pages through -a separately installed host executable. Direct fetch is used by default; choose -agent-browser explicitly when needed. The implementation never falls back -automatically: +`web fetch` has two renderers. `http` (the default) uses Node `fetch`, `linkedom`, +and Defuddle for HTML-to-Markdown extraction from static, SSR, and pre-rendered +pages. `browser` renders client-side pages through the separately installed +host browser capability. HTTP rendering is used by default; choose browser +explicitly when needed. The implementation never falls back automatically: ```bash -web fetch https://example.com/app --render=agent-browser --wait=2000 +web fetch https://example.com/app --render=browser --wait=2000 # If it is still incomplete, retry explicitly with more time, or abandon it: -web fetch https://example.com/app --render=agent-browser --wait=10000 +web fetch https://example.com/app --render=browser --wait=10000 ``` -`web links` uses the same direct or explicit browser-rendered source, but parses +`web links` uses the same HTTP or explicit browser-rendered source, but parses the original DOM rather than Defuddle output so navigation and other links outside the readable article remain discoverable. It returns only HTTP(S) `a[href]` destinations, deduplicated and capped at 100 by default. -`--wait` is mandatory with `--render=agent-browser`, including `--wait=0`, and -accepts only an integer from 0 through 30,000 milliseconds. Direct `fetch` -or `links` requests must not provide `--wait`. The same `render: "agent-browser"` and required +`--wait` is mandatory with `--render=browser`, including `--wait=0`, and +accepts only an integer from 0 through 30,000 milliseconds. HTTP `fetch` +or `links` requests must not provide `--wait`. The same `render: "browser"` and required `waitMs` fields are available on the MCP `fetch`/`links`, Pi `web_fetch`/`web_links`, and DSH -`web_fetch`/`web_links` tools. A direct-fetch failure may return the structured +`web_fetch`/`web_links` tools. An HTTP-rendering failure may return the structured `javascript_rendering_may_be_required` hint with the 2,000 ms suggestion; the agent decides whether to retry with a longer wait or abandon the page. @@ -215,7 +218,7 @@ agent-browser install `agent-browser install` manages its own browser runtime; Guion packages never run it, bundle it, or reuse browser credentials. A compatible executable must be directly runnable from `PATH` without a shell. The renderer is supported on -macOS and Linux hosts. Direct fetch remains available, and the three npm +macOS and Linux hosts. HTTP rendering remains available, and the three npm packages remain installable when `agent-browser` is absent. A rendered session is fresh and non-persistent. Before launch, the target must diff --git a/docs/adr/0001-containerized-rest-api.md b/docs/adr/0001-containerized-rest-api.md index 7a4ee66..f23e4e8 100644 --- a/docs/adr/0001-containerized-rest-api.md +++ b/docs/adr/0001-containerized-rest-api.md @@ -4,4 +4,4 @@ Guion Web will add a self-hosted, single-user HTTP service in a portable contain ## Consequences -The service's provider credentials and Bridge Route are server-local configuration; clients cannot choose a provider or supply a Bridge Route per request. Search tries Kepos Bridge and uses Exa only when the Bridge is operationally unavailable; it does not fall back for cancellation, malformed client input, or an empty result set. A failed Bridge attempt is retried through Exa exactly once, and a successful empty Bridge response is returned unchanged. Typed Bridge operations are intentionally not exposed: Exa has no equivalent official weather, sports, or time API, and its premium finance integration is not contract-compatible. `agent-browser` is available as the explicit Rendered Fetch backend in the image. `openapi.yaml` is generated from the release build rather than manually maintained or independently versioned. Public or multi-tenant deployment hardening is deliberately deferred in `.scratch/defered/public-http-service-security.md`. +The service's provider credentials and Bridge Route are server-local configuration; clients cannot choose a provider or supply a Bridge Route per request. Search tries Kepos Bridge and uses Exa only when the Bridge is operationally unavailable; it does not fall back for cancellation, malformed client input, or an empty result set. A failed Bridge attempt is retried through Exa exactly once, and a successful empty Bridge response is returned unchanged. Typed Bridge operations are intentionally not exposed: Exa has no equivalent official weather, sports, or time API, and its premium finance integration is not contract-compatible. Fetch and Links use `render: "http"` by default or explicit `render: "browser"` with a required `waitMs`; the operator-installed executable remains an implementation detail. The shared page-reading module owns automatic navigation trees, complete `full` extraction, and `section_id` continuation. `openapi.yaml` is generated from the release build rather than manually maintained or independently versioned; the standalone [HTTP service reference](../http-service.md) is its human-readable companion. Public or multi-tenant deployment hardening is deliberately deferred in `.scratch/defered/public-http-service-security.md`. diff --git a/docs/http-service.md b/docs/http-service.md new file mode 100644 index 0000000..3d6dc69 --- /dev/null +++ b/docs/http-service.md @@ -0,0 +1,154 @@ +# Guion Web personal HTTP service + +The personal service is a single-trust-boundary JSON API. It exposes exactly +three `POST` routes: `/v1/search`, `/v1/fetch`, and `/v1/links`. It is intended +for an operator and trusted agents, not for public or multi-tenant traffic. + +## Calling the service + +Send `Content-Type: application/json` and a JSON request body. Successful +responses are JSON and preserve the shared Guion page-reading vocabulary. +Unknown fields, malformed JSON, missing required fields, and values outside the +constraints below are rejected before an operation is called. + +### `POST /v1/search` + +Request: + +```json +{ "query": "AbortSignal" } +``` + +`query` is a non-empty string. The service chooses providers server-side; a +caller cannot select a provider, supply credentials, or override the Bridge +route. Search tries the server-local Kepos Bridge first. A successful empty +Bridge result is returned as-is. If Bridge fails for a non-cancellation reason, +the service retries Exa once. The service requires a non-empty `EXA_API_KEY` +at startup because Exa is the fallback provider. + +Response `200`: + +```json +{ + "provider": "Kepos Bridge", + "results": [ + { + "title": "...", + "link": "https://example.test", + "snippet": "...", + "position": 1 + } + ] +} +``` + +`provider` is `"Kepos Bridge"` or `"Exa"`; each result has string `title`, +`link`, and `snippet` fields plus an integer `position`. + +### `POST /v1/fetch` + +Request: + +```json +{ + "url": "https://example.test/article", + "render": "http", + "full": false, + "section_id": "7i" +} +``` + +`url` must be an absolute `http:` or `https:` URL. `render` is optional and +must be exactly `"http"` or `"browser"`; omission selects `"http"`. `waitMs` +is an integer from 0 through 30,000, required with `render: "browser"` and +forbidden with `render: "http"` (or when `render` is omitted). `full` is an +optional boolean. `section_id` is an optional non-empty string returned in a +navigation tree. `full: true` and `section_id` cannot be sent together. + +HTTP rendering fetches the page with Node HTTP, linkedom, and Defuddle. +Browser rendering invokes the operator-installed `agent-browser` executable +through the isolated renderer implementation; the executable name is not a +public request value. Browser rendering is never selected automatically, and +the service does not fall back between renderers. + +The shared module owns the 5,000-character automatic-tree policy. A non-full, +unsectioned document longer than that threshold returns `mode: "tree"` with +stable section IDs. Use one of those IDs in a subsequent request to retrieve a +section. `full: true` returns the complete extracted Markdown without the Core +content limit. A short or unsectioned result uses `mode: "full"`; a section +request uses `mode: "section"`. + +Response `200`: + +```json +{ + "url": "https://example.test/article", + "mode": "full", + "content": "# Article\n...\n" +} +``` + +### `POST /v1/links` + +Request: + +```json +{ + "url": "https://example.test/article", + "limit": 50, + "render": "browser", + "waitMs": 0 +} +``` + +`url`, `render`, and `waitMs` follow the Fetch rules. `limit` is optional, +defaults to 100, and must be an integer from 1 through 100. Links are read from +the source DOM rather than Defuddle's article body, deduplicated, and limited +to HTTP(S) `a[href]` destinations. + +Response `200`: + +```json +{ + "url": "https://example.test/article", + "links": [{ "text": "Guide", "url": "https://example.test/guide" }], + "truncated": false +} +``` + +## Errors + +Every error is a bounded JSON object with this shape: + +```json +{ "code": "invalid_request", "message": "Request validation failed" } +``` + +`details` is optional and contains only safe, documented fields. The service +uses these status codes: + +| Status | Meaning | +| -----: | ------------------------------------------------------------------------ | +| `400` | Invalid JSON, unknown fields, invalid values, or renderer/wait mismatch. | +| `499` | The client cancelled the request. | +| `502` | An upstream provider, page fetch, renderer, or response contract failed. | +| `504` | An upstream operation timed out. | +| `404` | The path is not one of the three retained routes. | + +A browserless Fetch failure may use `javascript_rendering_may_be_required` and +include `details: { "retryableWithRender": true, "suggestedArguments": { +"render": "browser", "waitMs": 2000 } }`. A renderer allowlist failure uses +`render_domain_not_allowed` with `retryable: false`, the report URL, and a +validated blocked hostname when available. Increasing `waitMs` does not fix an +allowlist failure. Error responses never include credentials or raw upstream +response bodies. + +## Configuration and OpenAPI + +Credentials and the optional `KEPOS_BRIDGE_ENDPOINT` are server-local +environment variables. They are not accepted in request bodies. The route +schemas in [`packages/web/src/http.ts`](../packages/web/src/http.ts) are the +machine-readable source of truth. The build generates a version-matched +OpenAPI 3.1 artifact at `packages/web/dist/openapi.yaml`; releases attach that +file as the downloadable `openapi.yaml` asset. This document is its +human-readable companion and is not an independently versioned schema. diff --git a/packages/dsh-web/README.md b/packages/dsh-web/README.md index b357f8b..ea83de8 100644 --- a/packages/dsh-web/README.md +++ b/packages/dsh-web/README.md @@ -24,11 +24,11 @@ strings, and fragments are rejected and its path is used exactly as entered. The published package is a dual host/browser bundle. Its host and client artifacts, profile patch, and exact DSH `0.1.2-alpha.3` peer contract are included -in the npm package. Search, direct page fetch, page-link discovery, optional agent-browser rendering, +in the npm package. Search, HTTP page rendering, page-link discovery, optional browser rendering, Context7 documentation, and Sourcegraph all run in-process through the bundled -Guion Web core. `web_fetch` has two page-fetch backends: direct fetch (the -default) and explicit `render: "agent-browser"` with required `waitMs` (an -integer from 0 through 30,000) for client-rendered pages through a host-installed +Guion Web core. `web_fetch` has two page-rendering modes: HTTP (the default) and +explicit `render: "browser"` with required `waitMs` (an integer from 0 through +30,000) for client-rendered pages through a host-installed `agent-browser` executable. To enable that optional capability, install [agent-browser](https://github.com/vercel-labs/agent-browser) separately with @@ -40,9 +40,13 @@ state or credentials. `web_links` lists up to 100 unique HTTP(S) anchors from the original page DOM, so it includes navigation and other links that readable-content extraction drops. -It uses the same direct default and explicit `render: "agent-browser"` / required +It uses the same HTTP default and explicit `render: "browser"` / required `waitMs` contract as `web_fetch`. +Long `web_fetch` documents return a navigation tree automatically. Use +`full: true` for complete Markdown or pass a returned `section_id` to continue +with one section; those fields are mutually exclusive. + Rendered requests are bounded and constrained to the requested hostname, `*.` (the target and its subdomains), and this fixed common CDN list: `cdn.jsdelivr.net`, `unpkg.com`, `cdnjs.cloudflare.com`, @@ -50,7 +54,7 @@ CDN list: `cdn.jsdelivr.net`, `unpkg.com`, `cdnjs.cloudflare.com`, The caller cannot widen the list. A redirect, API, frame, worker, socket, or other dependency on an unknown domain fails closed as `render_domain_not_allowed`; increasing `waitMs` will not help. For example, -retry a shell with `render: "agent-browser", waitMs: 2000`, then explicitly +retry a shell with `render: "browser", waitMs: 2000`, then explicitly retry with a longer wait such as `waitMs: 10000` or abandon the page. Report a likely missing first-party or common-CDN domain at https://github.com/guionai/web/issues/new, including the page URL and blocked diff --git a/packages/dsh-web/package.json b/packages/dsh-web/package.json index 5741972..34d4c71 100644 --- a/packages/dsh-web/package.json +++ b/packages/dsh-web/package.json @@ -1,7 +1,7 @@ { "name": "@guionai/dsh-web", "version": "0.1.0", - "description": "DeepSeek Harness 0.1.2-alpha.3 provider for Exa, Brave, and Kepos Bridge web research with direct and optional agent-browser page-fetch backends.", + "description": "DeepSeek Harness 0.1.2-alpha.3 provider for Exa, Brave, and Kepos Bridge web research with HTTP and optional browser page-rendering modes.", "homepage": "https://github.com/guionai/web#readme", "bugs": { "url": "https://github.com/guionai/web/issues" diff --git a/packages/dsh-web/src/client.ts b/packages/dsh-web/src/client.ts index f12941f..d39d111 100644 --- a/packages/dsh-web/src/client.ts +++ b/packages/dsh-web/src/client.ts @@ -290,11 +290,11 @@ function toolSummary( export function fetchDetails( args: Record, ): Array<{ label: string; value: string }> { - const browserRendered = args.render === "agent-browser"; + const browserRendered = args.render === "browser"; const details = [ { label: "Backend", - value: browserRendered ? "Browser rendered" : "Direct fetch", + value: browserRendered ? "Browser rendered" : "HTTP rendered", }, ]; if (browserRendered && typeof args.waitMs === "number") { @@ -313,7 +313,6 @@ function formatWait(waitMs: number): string { function fetchResultMode(args: Record): string { if (typeof args.section_id === "string" && args.section_id !== "") return `Section: ${args.section_id}`; - if (args.tree === true) return "Heading tree"; if (args.full === true) return "Full document"; return "Adaptive document"; } diff --git a/packages/dsh-web/src/tools.ts b/packages/dsh-web/src/tools.ts index ea48aae..92d9aee 100644 --- a/packages/dsh-web/src/tools.ts +++ b/packages/dsh-web/src/tools.ts @@ -9,6 +9,7 @@ import { type DocsFetchResult, type DocsResolveResult, type DocsToolInput, + type FetchInput, type FetchResult, type LinksInput, type LinksResult, @@ -26,8 +27,6 @@ import { defineTool, type ToolDefinition } from "@deepseek-ai/dsh-tools"; import { CONTEXT7_CREDENTIAL_REF } from "./contract.js"; -const DEFAULT_TREE_THRESHOLD = 5000; - export interface WebToolDependencies { credentials: { resolve(ref: CredentialRef): Promise; @@ -43,7 +42,6 @@ const fetchParameters = { required: true, description: "HTTP or HTTPS URL to fetch", }, - tree: { type: "boolean", description: "Show the page heading tree" }, section_id: { type: "string", description: "Optional heading section ID to return", @@ -52,20 +50,14 @@ const fetchParameters = { type: "boolean", description: "Return full content without automatic tree mode", }, - tree_threshold: { - type: "integer", - default: DEFAULT_TREE_THRESHOLD, - description: "Automatic tree threshold; defaults to 5000", - }, render: { type: "string", - enum: ["fetch", "agent-browser"], - description: "Page-fetch backend; defaults to direct fetch", + enum: ["http", "browser"], + description: "Page renderer; defaults to HTTP", }, waitMs: { type: "integer", - description: - "Required post-load wait for agent-browser rendering (0-30000)", + description: "Required post-load wait for browser rendering (0-30000)", }, } as const; @@ -82,13 +74,12 @@ const linksParameters = { }, render: { type: "string", - enum: ["fetch", "agent-browser"], - description: "Page-fetch backend; defaults to direct fetch", + enum: ["http", "browser"], + description: "Page renderer; defaults to HTTP", }, waitMs: { type: "integer", - description: - "Required post-load wait for agent-browser rendering (0-30000)", + description: "Required post-load wait for browser rendering (0-30000)", }, } as const; @@ -221,7 +212,7 @@ const fetchOutput = { type: "text" as const, text: boundedToolText( value.content, - "Use web_fetch with tree or section_id to navigate the document.", + "Use web_fetch with full: true or a returned section_id to navigate the document.", ), }, ], @@ -380,14 +371,14 @@ function normalizeLinks(input: unknown): LinksInput { const render = input.render; const waitMs = input.waitMs; - if (render !== undefined && render !== "fetch" && render !== "agent-browser") - throw new Error('render must be "fetch" or "agent-browser"'); - if (render !== "agent-browser") { + if (render !== undefined && render !== "http" && render !== "browser") + throw new Error('render must be "http" or "browser"'); + if (render !== "browser") { if (waitMs !== undefined) - throw new Error("waitMs is only valid with render agent-browser"); + throw new Error("waitMs is only valid with render browser"); } else { if (waitMs === undefined) - throw new Error("waitMs is required with render agent-browser"); + throw new Error("waitMs is required with render browser"); if ( typeof waitMs !== "number" || !Number.isInteger(waitMs) || @@ -405,6 +396,52 @@ function normalizeLinks(input: unknown): LinksInput { }; } +function normalizeFetch(input: unknown): FetchInput { + if (!isRecord(input)) throw new Error("web_fetch input must be an object"); + rejectUnknownFields( + input, + ["url", "section_id", "full", "render", "waitMs"], + "web_fetch", + ); + const url = requireString(input, "url"); + const sectionID = input.section_id; + if ( + sectionID !== undefined && + (typeof sectionID !== "string" || sectionID.trim().length === 0) + ) + throw new Error("section_id must be a non-empty string"); + const full = input.full; + if (full !== undefined && typeof full !== "boolean") + throw new Error("full must be a boolean"); + if (full === true && sectionID !== undefined) + throw new Error("full and section_id cannot be used together"); + const render = input.render; + if (render !== undefined && render !== "http" && render !== "browser") + throw new Error('render must be "http" or "browser"'); + const waitMs = input.waitMs; + if (render !== "browser") { + if (waitMs !== undefined) + throw new Error("waitMs is only valid with render browser"); + } else { + if (waitMs === undefined) + throw new Error("waitMs is required with render browser"); + if ( + typeof waitMs !== "number" || + !Number.isInteger(waitMs) || + waitMs < 0 || + waitMs > 30_000 + ) + throw new Error("waitMs must be an integer from 0 through 30000"); + } + return { + url, + ...(sectionID === undefined ? {} : { section_id: sectionID }), + ...(full === undefined ? {} : { full }), + ...(render === undefined ? {} : { render }), + ...(waitMs === undefined ? {} : { waitMs }), + }; +} + function normalizeDocs(input: unknown): DocsToolInput { if (!isRecord(input)) throw new Error("web_docs input must be an object"); return normalizeDocsToolInput(input); @@ -458,24 +495,12 @@ function webFetchTool( defineTool({ name: "web_fetch", description: - "Use direct fetch for static, SSR, and pre-rendered pages. For client-rendered or SPA pages, set render: agent-browser with required waitMs on a host that has agent-browser installed; there is no automatic fallback.", + "Use HTTP rendering for static, SSR, and pre-rendered pages. For client-rendered or SPA pages, set render: browser with required waitMs when the host provides browser capability; there is no automatic fallback.", parameters: fetchParameters, output: fetchOutput, isConcurrencySafe: () => true, async execute(args, exec) { - rejectUnknownFields(args, Object.keys(fetchParameters), "web_fetch"); - return operations.fetch( - { - url: requireString(args, "url"), - tree: args.tree, - section_id: args.section_id, - full: args.full, - tree_threshold: args.tree_threshold, - render: args.render, - waitMs: args.waitMs, - }, - exec.signal, - ); + return operations.fetch(normalizeFetch(args), exec.signal); }, }), ); @@ -489,7 +514,7 @@ function webLinksTool( defineTool({ name: "web_links", description: - "List HTTP(S) links from a page. Use direct fetch for static pages, or explicit agent-browser rendering with waitMs for client-rendered pages.", + "List HTTP(S) links from a page. Use HTTP rendering for static pages, or explicit browser rendering with waitMs for client-rendered pages.", parameters: linksParameters, output: linksOutput, isConcurrencySafe: () => true, diff --git a/packages/dsh-web/test/artifact.test.ts b/packages/dsh-web/test/artifact.test.ts index 2b8bb2a..7f6c481 100644 --- a/packages/dsh-web/test/artifact.test.ts +++ b/packages/dsh-web/test/artifact.test.ts @@ -311,7 +311,7 @@ describe("DSH 0.1.2-alpha.3 packed package contract", () => { const rendered = await fetchTool.execute( { url: "https://93.184.216.34/rendered", - render: "agent-browser", + render: "browser", waitMs: 0, full: true, }, @@ -331,7 +331,7 @@ describe("DSH 0.1.2-alpha.3 packed package contract", () => { fetchTool.execute( { url: "https://93.184.216.34/blocked", - render: "agent-browser", + render: "browser", waitMs: 0, }, { signal: new AbortController().signal }, diff --git a/packages/dsh-web/test/client.test.ts b/packages/dsh-web/test/client.test.ts index 68b60b8..dc76969 100644 --- a/packages/dsh-web/test/client.test.ts +++ b/packages/dsh-web/test/client.test.ts @@ -53,7 +53,7 @@ describe("DSH settings client credential surface", () => { it("labels the fetch backend, wait, and retrieval mode from its request", () => { expect( fetchDetails({ - render: "agent-browser", + render: "browser", waitMs: 2_000, section_id: "installation", }), @@ -62,12 +62,12 @@ describe("DSH settings client credential surface", () => { { label: "Wait", value: "2 s" }, { label: "Result", value: "Section: installation" }, ]); - expect(fetchDetails({ tree: true })).toEqual([ - { label: "Backend", value: "Direct fetch" }, - { label: "Result", value: "Heading tree" }, + expect(fetchDetails({})).toEqual([ + { label: "Backend", value: "HTTP rendered" }, + { label: "Result", value: "Adaptive document" }, ]); expect(fetchDetails({ full: true })).toEqual([ - { label: "Backend", value: "Direct fetch" }, + { label: "Backend", value: "HTTP rendered" }, { label: "Result", value: "Full document" }, ]); }); diff --git a/packages/dsh-web/test/tools.test.ts b/packages/dsh-web/test/tools.test.ts index 91c30b3..4171a61 100644 --- a/packages/dsh-web/test/tools.test.ts +++ b/packages/dsh-web/test/tools.test.ts @@ -70,7 +70,7 @@ describe("DSH direct web tools", () => { definitions[0]!.isConcurrencySafe?.({ url: "https://example.test" }), definitions[1]!.isConcurrencySafe?.({ url: "https://example.test", - render: "agent-browser", + render: "browser", waitMs: 0, }), definitions[2]!.isConcurrencySafe?.({ @@ -83,8 +83,8 @@ describe("DSH direct web tools", () => { false, ); expect((definitions[0]!.parameters as any).properties.render.enum).toEqual([ - "fetch", - "agent-browser", + "http", + "browser", ]); expect((definitions[0]!.parameters as any).properties.waitMs.type).toBe( "integer", @@ -93,8 +93,8 @@ describe("DSH direct web tools", () => { 100, ); expect((definitions[1]!.parameters as any).properties.render.enum).toEqual([ - "fetch", - "agent-browser", + "http", + "browser", ]); expect((definitions[2]!.parameters as any).properties.action.enum).toEqual([ "resolve", @@ -149,7 +149,7 @@ describe("DSH direct web tools", () => { { url: "https://example.test", section_id: "install", - render: "agent-browser", + render: "browser", waitMs: 2000, }, controller.signal, @@ -161,7 +161,7 @@ describe("DSH direct web tools", () => { { url: "https://example.test", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 2000, }, controller.signal, @@ -191,11 +191,8 @@ describe("DSH direct web tools", () => { kind: "fetch", input: { url: "https://example.test", - tree: undefined, section_id: "install", - full: undefined, - tree_threshold: undefined, - render: "agent-browser", + render: "browser", waitMs: 2000, }, abortSignal: controller.signal, @@ -205,7 +202,7 @@ describe("DSH direct web tools", () => { input: { url: "https://example.test", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 2000, }, abortSignal: controller.signal, @@ -243,7 +240,7 @@ describe("DSH direct web tools", () => { await expect( call(fetch, { url: "https://example.test", - render: "agent-browser", + render: "browser", }), ).rejects.toThrow("waitMs is required"); await expect( @@ -255,7 +252,7 @@ describe("DSH direct web tools", () => { await expect( call(fetch, { url: "https://example.test", - render: "agent-browser", + render: "browser", waitMs: 30_001, }), ).rejects.toThrow("waitMs must be an integer"); @@ -264,7 +261,7 @@ describe("DSH direct web tools", () => { "javascript_rendering_may_be_required", { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }, ); const allowlist = new FetchCapabilityError("render_domain_not_allowed", { @@ -293,7 +290,7 @@ describe("DSH direct web tools", () => { )[0]!; const pending = call( cancel, - { url: "https://example.test", render: "fetch" }, + { url: "https://example.test", render: "http" }, controller.signal, ); controller.abort(); @@ -312,13 +309,13 @@ describe("DSH direct web tools", () => { await expect( call(retryDefinition, { url: "https://example.test", - render: "fetch", + render: "http", }), ).rejects.toMatchObject({ code: "javascript_rendering_may_be_required", details: { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }, }); @@ -334,7 +331,7 @@ describe("DSH direct web tools", () => { await expect( call(allowlistDefinition, { url: "https://example.test", - render: "agent-browser", + render: "browser", waitMs: 0, }), ).rejects.toMatchObject({ @@ -396,13 +393,13 @@ describe("DSH direct web tools", () => { await expect( call(linksTool!, { url: "https://example.test", - render: "agent-browser", + render: "browser", }), ).rejects.toThrow("waitMs is required"); await expect( call(linksTool!, { url: "https://example.test", - render: "agent-browser", + render: "browser", waitMs: -1, }), ).rejects.toThrow("waitMs must be an integer from 0 through 30000"); diff --git a/packages/pi-web/package.json b/packages/pi-web/package.json index 20eed4d..72bc9d0 100644 --- a/packages/pi-web/package.json +++ b/packages/pi-web/package.json @@ -1,7 +1,7 @@ { "name": "@guionai/pi-web", "version": "0.1.0", - "description": "Pi extension for web research with direct and optional agent-browser page-fetch backends.", + "description": "Pi extension for web research with HTTP and optional browser page-rendering modes.", "homepage": "https://github.com/guionai/web#readme", "bugs": { "url": "https://github.com/guionai/web/issues" diff --git a/packages/pi-web/src/tool.ts b/packages/pi-web/src/tool.ts index e0a1690..b5a33da 100644 --- a/packages/pi-web/src/tool.ts +++ b/packages/pi-web/src/tool.ts @@ -16,8 +16,6 @@ import { Type, type Static, type TSchema } from "typebox"; import { modelTextResult } from "./model-text.js"; -const DEFAULT_TREE_THRESHOLD = 5000; - export const webSearchSchema = Type.Object( { queries: Type.Array( @@ -38,32 +36,27 @@ export const webSearchSchema = Type.Object( const fetchNavigationProperties = { url: Type.String({ description: "HTTP or HTTPS URL to fetch" }), - tree: Type.Optional( - Type.Boolean({ description: "Show the page heading tree" }), - ), section_id: Type.Optional( - Type.String({ description: "Optional heading section ID to return" }), + Type.String({ + description: "Optional heading section ID to return", + minLength: 1, + pattern: "\\S", + }), ), full: Type.Optional( Type.Boolean({ description: "Return full content without automatic tree mode", }), ), - tree_threshold: Type.Optional( - Type.Integer({ - description: `Automatic tree threshold; defaults to ${DEFAULT_TREE_THRESHOLD}`, - default: DEFAULT_TREE_THRESHOLD, - }), - ), }; -export const webFetchSchema = Type.Union([ +const webFetchBaseSchema = Type.Union([ Type.Object( { ...fetchNavigationProperties, render: Type.Optional( - StringEnum(["fetch"] as const, { - description: "Use direct HTTP fetching (the default)", + StringEnum(["http"] as const, { + description: "Use HTTP fetching (the default)", }), ), }, @@ -72,8 +65,8 @@ export const webFetchSchema = Type.Union([ Type.Object( { ...fetchNavigationProperties, - render: StringEnum(["agent-browser"] as const, { - description: "Render the page through the host-installed agent-browser", + render: StringEnum(["browser"] as const, { + description: "Render the page through the host-installed browser", }), waitMs: Type.Integer({ description: "Additional post-load wait in milliseconds", @@ -85,6 +78,22 @@ export const webFetchSchema = Type.Union([ ), ]); +export const webFetchSchema = { + ...webFetchBaseSchema, + allOf: [ + { + not: { + type: "object", + required: ["full", "section_id"], + properties: { + full: { const: true }, + section_id: { type: "string" }, + }, + }, + }, + ], +} as typeof webFetchBaseSchema; + const linksProperties = { url: Type.String({ description: "HTTP or HTTPS URL to inspect" }), limit: Type.Optional( @@ -102,8 +111,8 @@ export const webLinksSchema = Type.Union([ { ...linksProperties, render: Type.Optional( - StringEnum(["fetch"] as const, { - description: "Use direct HTTP fetching (the default)", + StringEnum(["http"] as const, { + description: "Use HTTP fetching (the default)", }), ), }, @@ -112,8 +121,8 @@ export const webLinksSchema = Type.Union([ Type.Object( { ...linksProperties, - render: StringEnum(["agent-browser"] as const, { - description: "Render the page through the host-installed agent-browser", + render: StringEnum(["browser"] as const, { + description: "Render the page through the host-installed browser", }), waitMs: Type.Integer({ description: "Additional post-load wait in milliseconds", @@ -196,15 +205,15 @@ const SEARCH_PROMPT_GUIDELINES = [ "Use web_search to search the web for current facts.", ]; const FETCH_PROMPT_GUIDELINES = [ - "Use web_fetch to read a web page; large pages are truncated with a continuation notice, so follow up with tree or section_id to navigate.", - 'web_fetch has two backends: omit render or use render: "fetch" for direct HTML-to-Markdown (the default for static, SSR, and pre-rendered pages).', - 'For a client-rendered or SPA page, or after javascript_rendering_may_be_required, retry explicitly with render: "agent-browser" and waitMs: 2000 only when the host has agent-browser installed. Increase waitMs explicitly or abandon an incomplete page; there is no automatic fallback.', - "Never send waitMs with direct fetch. agent-browser is a host capability, not a package dependency.", + "Use web_fetch to read a web page; long documents return a navigation tree automatically, so follow up with full: true or a returned section_id.", + 'web_fetch uses HTTP fetching by default; set render: "http" explicitly when desired.', + 'For a client-rendered or SPA page, or after javascript_rendering_may_be_required, retry explicitly with render: "browser" and waitMs: 2000 only when the host provides browser capability. Increase waitMs explicitly or abandon an incomplete page; there is no automatic fallback.', + "Never send waitMs with HTTP rendering. The browser is a host capability, not a package dependency.", ]; const LINKS_PROMPT_GUIDELINES = [ "Use web_links to discover HTTP(S) destinations from a page, including navigation and links outside the readable article body.", - 'Use direct fetch by default. For a client-rendered or SPA page, explicitly use render: "agent-browser" with waitMs; there is no automatic fallback.', - "Never send waitMs with direct fetch. agent-browser is a host capability, not a package dependency.", + 'Use HTTP rendering by default. For a client-rendered or SPA page, explicitly use render: "browser" with waitMs; there is no automatic fallback.', + "Never send waitMs with HTTP rendering. The browser is a host capability, not a package dependency.", ]; const DOCS_PROMPT_GUIDELINES = [ "Use web_docs with action resolve, then action fetch, to read library documentation instead of fetching documentation sites page by page.", @@ -261,17 +270,21 @@ function requireQueries(input: unknown): string[] { function normalizeFetch(input: unknown): WebFetchInput { if (!isRecord(input)) throw new Error("web_fetch input must be an object"); + for (const field of Object.keys(input)) { + if (!["url", "section_id", "full", "render", "waitMs"].includes(field)) + throw new Error(`web_fetch input does not accept field ${field}`); + } const url = requireString(input, "url"); const render = input.render; const waitMs = input.waitMs; - if (render !== undefined && render !== "fetch" && render !== "agent-browser") - throw new Error('render must be "fetch" or "agent-browser"'); - if (render !== "agent-browser") { + if (render !== undefined && render !== "http" && render !== "browser") + throw new Error('render must be "http" or "browser"'); + if (render !== "browser") { if (waitMs !== undefined) - throw new Error("waitMs is only valid with render agent-browser"); + throw new Error("waitMs is only valid with render browser"); } else { if (waitMs === undefined) - throw new Error("waitMs is required with render agent-browser"); + throw new Error("waitMs is required with render browser"); if ( typeof waitMs !== "number" || !Number.isInteger(waitMs) || @@ -280,34 +293,44 @@ function normalizeFetch(input: unknown): WebFetchInput { ) throw new Error("waitMs must be an integer from 0 through 30000"); } + if (input.full !== undefined && typeof input.full !== "boolean") + throw new Error("full must be a boolean"); + if (input.section_id !== undefined) { + if (typeof input.section_id !== "string" || input.section_id.trim() === "") + throw new Error("section_id must be a non-empty string"); + } + if (input.full === true && input.section_id !== undefined) + throw new Error("full and section_id cannot be used together"); const typed = input as unknown as WebFetchInput; const navigation = { url, - tree: typed.tree, - section_id: typed.section_id, - full: typed.full, - tree_threshold: typed.tree_threshold, + ...(typed.section_id === undefined ? {} : { section_id: typed.section_id }), + ...(typed.full === undefined ? {} : { full: typed.full }), }; - if (render === "agent-browser") + if (render === "browser") return { ...navigation, render, waitMs: waitMs as number }; - if (render === "fetch") return { ...navigation, render }; + if (render === "http") return { ...navigation, render }; return navigation; } function normalizeLinks(input: unknown): WebLinksInput { if (!isRecord(input)) throw new Error("web_links input must be an object"); + for (const field of Object.keys(input)) { + if (!["url", "limit", "render", "waitMs"].includes(field)) + throw new Error(`web_links input does not accept field ${field}`); + } const url = requireString(input, "url"); const render = input.render; const waitMs = input.waitMs; const limit = input.limit; - if (render !== undefined && render !== "fetch" && render !== "agent-browser") - throw new Error('render must be "fetch" or "agent-browser"'); - if (render !== "agent-browser") { + if (render !== undefined && render !== "http" && render !== "browser") + throw new Error('render must be "http" or "browser"'); + if (render !== "browser") { if (waitMs !== undefined) - throw new Error("waitMs is only valid with render agent-browser"); + throw new Error("waitMs is only valid with render browser"); } else { if (waitMs === undefined) - throw new Error("waitMs is required with render agent-browser"); + throw new Error("waitMs is required with render browser"); if ( typeof waitMs !== "number" || !Number.isInteger(waitMs) || @@ -327,10 +350,13 @@ function normalizeLinks(input: unknown): WebLinksInput { `limit must be an integer from 1 through ${MAX_LINK_LIMIT}`, ); const typed = input as unknown as WebLinksInput; - const result = { url, limit: typed.limit }; - if (render === "agent-browser") + const result = { + url, + ...(typed.limit === undefined ? {} : { limit: typed.limit }), + }; + if (render === "browser") return { ...result, render, waitMs: waitMs as number }; - if (render === "fetch") return { ...result, render }; + if (render === "http") return { ...result, render }; return result; } @@ -419,14 +445,14 @@ export function webFetchTool(dependencies: WebToolDependencies = {}) { name: "web_fetch", label: "Web fetch", description: - "Fetch and read an HTTP or HTTPS web page as Markdown, with direct fetch or explicit agent-browser rendering for client-rendered pages. Rendered fetch requires waitMs 0 through 30000. Text output is limited to 2,000 lines or 50KB; truncated output is saved to a temporary file.", + "Fetch and read an HTTP or HTTPS web page as Markdown, with HTTP rendering by default or explicit browser rendering for client-rendered pages. Browser rendering requires waitMs 0 through 30000. Long documents return a navigation tree unless full or section_id is requested. Text output is limited to 2,000 lines or 50KB; truncated output is saved to a temporary file.", promptSnippet: "Fetch a web page with web_fetch", promptGuidelines: FETCH_PROMPT_GUIDELINES, parameters: webFetchSchema, execute: async (params, signal) => { const data = await operations.fetch(normalizeFetch(params), signal); return modelTextResult(data, data.content, { - hint: "Use web_fetch with tree or section_id to navigate the document.", + hint: "Use web_fetch with full: true or a returned section_id to navigate the document.", }); }, }); @@ -438,7 +464,7 @@ export function webLinksTool(dependencies: WebToolDependencies = {}) { name: "web_links", label: "Web links", description: - "List HTTP(S) links from a web page, with direct fetch or explicit agent-browser rendering for client-rendered pages.", + "List HTTP(S) links from a web page, with HTTP rendering by default or explicit browser rendering for client-rendered pages.", promptSnippet: "List links from a web page with web_links", promptGuidelines: LINKS_PROMPT_GUIDELINES, parameters: webLinksSchema, diff --git a/packages/pi-web/test/extension.test.ts b/packages/pi-web/test/extension.test.ts index 1c42dd3..b12c332 100644 --- a/packages/pi-web/test/extension.test.ts +++ b/packages/pi-web/test/extension.test.ts @@ -98,43 +98,50 @@ describe("pi-web extension", () => { expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - render: "fetch", + render: "http", }), ).toBe(true); expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", waitMs: 0, }), ).toBe(true); expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", waitMs: 30_000, }), ).toBe(true); expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", }), ).toBe(false); expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - render: "fetch", + render: "http", waitMs: 0, }), ).toBe(false); expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", waitMs: 30_001, }), ).toBe(false); + expect( + Value.Check(webFetchSchema, { + url: "https://fixture.test", + full: true, + section_id: "intro", + }), + ).toBe(false); const fetch = vi.fn( async (input: { url: string; render?: string; waitMs?: number }) => ({ @@ -146,17 +153,13 @@ describe("pi-web extension", () => { const tool = webFetchTool({ operations: operations({ fetch }) }); const result = await call(tool, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", waitMs: 1250, }); expect(fetch).toHaveBeenCalledWith( { url: "https://fixture.test", - tree: undefined, - section_id: undefined, - full: undefined, - tree_threshold: undefined, - render: "agent-browser", + render: "browser", waitMs: 1250, }, undefined, @@ -166,13 +169,13 @@ describe("pi-web extension", () => { await expect( call(tool, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", }), ).rejects.toThrow("waitMs is required"); await expect( call(tool, { url: "https://fixture.test", - render: "fetch", + render: "http", waitMs: 0, }), ).rejects.toThrow("waitMs is only valid"); @@ -184,7 +187,7 @@ describe("pi-web extension", () => { expect( Value.Check(webLinksSchema, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", waitMs: 0, limit: 25, }), @@ -192,7 +195,7 @@ describe("pi-web extension", () => { expect( Value.Check(webLinksSchema, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", }), ).toBe(false); expect( @@ -211,14 +214,14 @@ describe("pi-web extension", () => { const linksResult = await call(linksTool, { url: "https://fixture.test", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 1250, }); expect(links).toHaveBeenCalledWith( { url: "https://fixture.test", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 1250, }, undefined, @@ -233,7 +236,7 @@ describe("pi-web extension", () => { code: "javascript_rendering_may_be_required", details: { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }, }, ); @@ -266,7 +269,7 @@ describe("pi-web extension", () => { await expect( call(blockedTool, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", waitMs: 0, }), ).rejects.toBe(blocked); @@ -469,7 +472,7 @@ describe("pi-web extension", () => { expect(details.url).toBe("https://fixture.test"); expect(details.truncation.truncated).toBe(true); expect(result.content[0]?.text).toContain( - "Use web_fetch with tree or section_id to navigate the document.", + "Use web_fetch with full: true or a returned section_id to navigate the document.", ); expect(await readFile(details.fullOutputPath, "utf8")).toBe(content); } finally { diff --git a/packages/pi-web/test/packed-smoke.mjs b/packages/pi-web/test/packed-smoke.mjs index 9aba947..f5d366f 100644 --- a/packages/pi-web/test/packed-smoke.mjs +++ b/packages/pi-web/test/packed-smoke.mjs @@ -215,7 +215,7 @@ if (command === "open" && args.some((value) => value.includes("/blocked"))) { const rendered = await fetchTool.execute("test", { url: "https://93.184.216.34/rendered", - render: "agent-browser", + render: "browser", waitMs: 0, full: true, }); @@ -248,7 +248,7 @@ if (command === "open" && args.some((value) => value.includes("/blocked"))) { try { await fetchTool.execute("test", { url: "https://93.184.216.34/blocked", - render: "agent-browser", + render: "browser", waitMs: 0, }); throw new Error("blocked rendered fetch unexpectedly succeeded"); diff --git a/packages/web-core/src/fetch.ts b/packages/web-core/src/fetch.ts index 492d87e..c22c825 100644 --- a/packages/web-core/src/fetch.ts +++ b/packages/web-core/src/fetch.ts @@ -9,7 +9,7 @@ import { spawn } from "node:child_process"; import { Defuddle } from "defuddle/node"; import { parseHTML } from "linkedom"; -import { renderMarkdown, truncateContent } from "./markdown.js"; +import { renderMarkdown } from "./markdown.js"; import { boundedRequest, isOperationAborted, @@ -45,11 +45,9 @@ export const RENDER_CDN_ALLOWLIST = [ export type FetchInput = { url: string; - tree?: boolean; section_id?: string; full?: boolean; - tree_threshold?: number; - render?: "fetch" | "agent-browser"; + render?: "http" | "browser"; waitMs?: number; }; @@ -62,7 +60,7 @@ export type FetchResult = { export type LinksInput = { url: string; limit?: number; - render?: "fetch" | "agent-browser"; + render?: "http" | "browser"; waitMs?: number; }; @@ -79,7 +77,7 @@ export type LinksResult = { export type FetchErrorDetails = { retryableWithRender?: boolean; - suggestedArguments?: { render: "agent-browser"; waitMs: 2000 }; + suggestedArguments?: { render: "browser"; waitMs: 2000 }; retryable?: boolean; reportUrl?: string; blockedHostname?: string; @@ -125,20 +123,19 @@ export async function fetchWebPage( callerSignal?: AbortSignal, options?: FetchOptions, ): Promise { + validateFetchFields(input); const url = validateURL(input.url); const render = validateRenderInput(input); + validateNavigationInput(input); const content = - render === "agent-browser" + render === "browser" ? await renderPage(url, input.waitMs!, callerSignal, options) : await fetchCached(url, callerSignal, options); throwIfAborted(callerSignal); - const rendered = renderMarkdown( - content, - input.tree ?? false, - input.section_id, - input.full ?? false, - input.tree_threshold, - ); + const rendered = renderMarkdown(content, { + section_id: input.section_id, + full: input.full === true, + }); return { url, mode: rendered.mode, content: rendered.content }; } @@ -148,11 +145,12 @@ export async function fetchWebLinks( callerSignal?: AbortSignal, options?: FetchOptions, ): Promise { + validateLinksFields(input); const url = validateURL(input.url); const render = validateRenderInput(input); const limit = validateLinkLimit(input.limit); const source = - render === "agent-browser" + render === "browser" ? await renderPageHTML(url, input.waitMs!, callerSignal, options) : await fetchPageHTML(url, callerSignal, options); throwIfAborted(callerSignal); @@ -161,17 +159,17 @@ export async function fetchWebLinks( function validateRenderInput( input: Pick, -): "fetch" | "agent-browser" { - const render = input.render ?? "fetch"; - if (render !== "fetch" && render !== "agent-browser") - throw new Error('render must be "fetch" or "agent-browser"'); - if (render === "fetch") { +): "http" | "browser" { + const render = input.render ?? "http"; + if (render !== "http" && render !== "browser") + throw new Error('render must be "http" or "browser"'); + if (render === "http") { if (input.waitMs !== undefined) - throw new Error("waitMs is only valid with render agent-browser"); + throw new Error("waitMs is only valid with render browser"); return render; } if (input.waitMs === undefined) - throw new Error("waitMs is required with render agent-browser"); + throw new Error("waitMs is required with render browser"); if ( !Number.isInteger(input.waitMs) || input.waitMs < 0 || @@ -181,6 +179,42 @@ function validateRenderInput( return render; } +function validateFetchFields(input: FetchInput): void { + validateKnownFields( + input, + ["url", "section_id", "full", "render", "waitMs"], + "fetch", + ); + if (typeof input.full !== "undefined" && typeof input.full !== "boolean") + throw new Error("full must be a boolean"); + if (typeof input.section_id !== "undefined") { + if (typeof input.section_id !== "string" || input.section_id.trim() === "") + throw new Error("section_id must be a non-empty string"); + } +} + +function validateLinksFields(input: LinksInput): void { + validateKnownFields(input, ["url", "limit", "render", "waitMs"], "links"); +} + +function validateKnownFields( + input: object, + fields: readonly string[], + operation: string, +): void { + for (const field of Object.keys(input)) { + if (!fields.includes(field)) + throw new Error(`${operation} input does not accept field ${field}`); + } +} + +function validateNavigationInput( + input: Pick, +): void { + if (input.full === true && input.section_id !== undefined) + throw new Error("full and section_id cannot be used together"); +} + function validateLinkLimit(limit: number | undefined): number { if (limit === undefined) return DEFAULT_LINK_LIMIT; if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LINK_LIMIT) @@ -220,7 +254,7 @@ async function fetchLocal( try { const response = await downloadPage(url, callerSignal, options); if (response.contentType !== "" && response.contentType !== "text/html") - return truncateContent(new TextDecoder().decode(response.body)); + return new TextDecoder().decode(response.body); return extractHTML( new TextDecoder().decode(response.body), url, @@ -576,12 +610,12 @@ async function extractHTML( if (suggestRender) { throw new FetchCapabilityError("javascript_rendering_may_be_required", { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }); } throw new Error("no content could be extracted"); } - return truncateContent(content.endsWith("\n") ? content : content + "\n"); + return ensureTrailingNewline(content); } catch (error) { if (error instanceof FetchCapabilityError) throw error; throw new Error(`defuddle parse failed: ${errorMessage(error)}`); @@ -1102,6 +1136,10 @@ function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } +function ensureTrailingNewline(content: string): string { + return content.endsWith("\n") ? content : `${content}\n`; +} + function defaultCacheDir(): string { const home = process.env.HOME?.trim() || homedir(); if (process.platform === "win32") { @@ -1151,7 +1189,7 @@ class DailyCache implements FetchCache { }); } catch { if (signal?.aborted) throwIfAborted(signal); - // Cache failures must not make a successful direct fetch fail. + // Cache failures must not make a successful HTTP fetch fail. } } diff --git a/packages/web-core/src/index.ts b/packages/web-core/src/index.ts index 0dfcff6..0da03b6 100644 --- a/packages/web-core/src/index.ts +++ b/packages/web-core/src/index.ts @@ -61,6 +61,7 @@ export { renderMarkdown, truncateContent, type MarkdownResult, + type MarkdownNavigationOptions, } from "./markdown.js"; export { DEFAULT_MAX_BYTES, diff --git a/packages/web-core/src/markdown.ts b/packages/web-core/src/markdown.ts index 21d4345..75496cc 100644 --- a/packages/web-core/src/markdown.ts +++ b/packages/web-core/src/markdown.ts @@ -18,6 +18,11 @@ export type MarkdownResult = { mode: "full" | "tree" | "section"; }; +export type MarkdownNavigationOptions = { + section_id?: string; + full?: boolean; +}; + const DEFAULT_TREE_THRESHOLD = 5000; const MAX_CONTENT_CHARS = 30_000; const BASE62 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; @@ -28,13 +33,12 @@ const markdown = new MarkdownIt(); export function renderMarkdown( source: string, - showTree: boolean, - sectionId: string | undefined, - full: boolean, - treeThreshold: number | undefined, + options: MarkdownNavigationOptions = {}, ): MarkdownResult { const headings = assignIds(parseHeadings(source)); - const section = sectionId?.trim(); + const section = options.section_id?.trim(); + if (options.full === true && section !== undefined) + throw new Error("full and section_id cannot be used together"); if (section) { return { content: extractSection(source, headings, section), @@ -42,15 +46,13 @@ export function renderMarkdown( }; } - const threshold = - treeThreshold && treeThreshold > 0 ? treeThreshold : DEFAULT_TREE_THRESHOLD; const charCount = Array.from(source).length; - if (showTree || (!full && charCount > threshold)) { - if (headings.length > 0) { - return { content: renderTree(source, headings), mode: "tree" }; - } - } - return { content: truncateContent(source), mode: "full" }; + if (!options.full && charCount > DEFAULT_TREE_THRESHOLD) + return { content: renderTree(source, headings), mode: "tree" }; + return { + content: options.full ? source : truncateContent(source), + mode: "full", + }; } export function truncateContent(content: string): string { @@ -182,7 +184,7 @@ function renderTree(source: string, headings: Heading[]): string { 99, ); if (bodyHeadings.length === 0) { - return `${header}(empty)\n\nUse -s to read a section, or --full to read everything.\n`; + return `${header}(empty)\n\nUse section_id to read a section, or full: true to read everything.\n`; } const nodes = bodyHeadings.map((heading) => { @@ -211,7 +213,7 @@ function renderTree(source: string, headings: Heading[]): string { hasMore.delete(depth); }); - return `${header}${tree}\nUse -s to read a section, or --full to read everything.\n`; + return `${header}${tree}\nUse section_id to read a section, or full: true to read everything.\n`; } function sectionCharCount( diff --git a/packages/web-core/test/fetch.test.ts b/packages/web-core/test/fetch.test.ts index 48a6100..fb7ce64 100644 --- a/packages/web-core/test/fetch.test.ts +++ b/packages/web-core/test/fetch.test.ts @@ -213,7 +213,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { code: "javascript_rendering_may_be_required", details: { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }, }); await expect(fetchPage({ url })).rejects.toThrow( @@ -350,7 +350,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { try { await expect( fetchWebPage( - { url: "https://render.test/page", render: "fetch" }, + { url: "https://render.test/page", render: "http" }, undefined, { cacheDirectory, fetch: directFetch, resolveHost }, ), @@ -367,7 +367,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebPage( { url: "https://render.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, full: true, }, @@ -412,7 +412,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { await fetchWebPage( { url: "https://render.test/page", - render: "agent-browser", + render: "browser", waitMs: 1, }, undefined, @@ -435,7 +435,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebLinks( { url: "https://render.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -468,7 +468,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebLinks( { url: "https://render.test/start", - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -498,7 +498,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { const pending = fetchWebPage( { url: "https://render.test/page", - render: "agent-browser", + render: "browser", waitMs: 30_000, }, controller.signal, @@ -520,7 +520,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { const pending = fetchWebPage( { url: "https://ignore-term.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, controller.signal, @@ -546,7 +546,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { const failure = await fetchWebPage( { url: `https://user:${secret}@render.test/page`, - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -571,7 +571,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { await expect( fetchWebPage({ url: `https://[${address}]/page`, - render: "agent-browser", + render: "browser", waitMs: 0, }), ).rejects.toThrow("private or reserved"); @@ -583,7 +583,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebPage( { url: "https://resolved.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -604,7 +604,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebPage( { url: "https://render.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -629,7 +629,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebPage( { url: "https://blocked.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -661,7 +661,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { await expect( fetchWebPage({ url: "https://render.test/page", - render: "agent-browser", + render: "browser", }), ).rejects.toThrow("waitMs is required"); await expect( @@ -670,7 +670,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { await expect( fetchWebPage({ url: "https://render.test/page", - render: "agent-browser", + render: "browser", waitMs: 30_001, }), ).rejects.toThrow("waitMs must be an integer"); @@ -680,7 +680,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { await expect( fetchWebPage({ url: "http://127.0.0.1/private", - render: "agent-browser", + render: "browser", waitMs: 0, }), ).rejects.toThrow("private or reserved"); @@ -690,7 +690,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebPage( { url: "https://blocked.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -762,6 +762,70 @@ describe.sequential("browserless fetch migrated from Organon", () => { await close(server); } }); + + it("owns automatic tree navigation, complete full extraction, and input validation", async () => { + const source = + "# Page\n\n## Install\nInstall content.\n\n## Next\nNext content.\n" + + "x".repeat(5_001); + const cache = { + prepare: vi.fn(async () => {}), + read: vi.fn(async () => source), + write: vi.fn(async () => {}), + }; + const input = { url: "https://navigation.test/page" }; + + await expect( + fetchWebPage(input, undefined, { cache }), + ).resolves.toMatchObject({ + mode: "tree", + }); + await expect( + fetchWebPage({ ...input, section_id: "7i" }, undefined, { cache }), + ).resolves.toMatchObject({ + mode: "section", + content: "## Install\nInstall content.\n", + }); + + const complete = "# Complete\n\n" + "x".repeat(30_001); + const completeCache = { + prepare: vi.fn(async () => {}), + read: vi.fn(async () => complete), + write: vi.fn(async () => {}), + }; + const result = await fetchWebPage( + { url: "https://navigation.test/complete", full: true }, + undefined, + { cache: completeCache }, + ); + expect(result).toEqual({ + url: "https://navigation.test/complete", + mode: "full", + content: complete, + }); + + await expect( + fetchWebPage( + { + url: input.url, + full: true, + section_id: "7i", + } as never, + undefined, + { cache }, + ), + ).rejects.toThrow("full and section_id cannot be used together"); + await expect( + fetchWebPage({ url: input.url, tree: true } as never, undefined, { + cache, + }), + ).rejects.toThrow("does not accept field tree"); + await expect( + fetchWebPage({ url: input.url, tree_threshold: 1 } as never, undefined, { + cache, + }), + ).rejects.toThrow("does not accept field tree_threshold"); + expect(cache.read).toHaveBeenCalledTimes(2); + }); }); async function withFakeAgentBrowser( diff --git a/packages/web-core/test/markdown.test.ts b/packages/web-core/test/markdown.test.ts index d2e12bc..e2e042c 100644 --- a/packages/web-core/test/markdown.test.ts +++ b/packages/web-core/test/markdown.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { renderMarkdown } from "../src/index.js"; -describe("Markdown navigation migrated from Organon", () => { +describe("Markdown navigation", () => { it("uses token labels and source maps for nested, inline, and Setext headings", () => { const source = [ "# Héading **bold** `code` [link](https://example.com)", @@ -28,8 +28,7 @@ describe("Markdown navigation migrated from Organon", () => { "after", "", ].join("\n"); - - const tree = renderMarkdown(source, true, undefined, false, 5000); + const tree = renderMarkdown(`${source}\n${"x".repeat(5001)}`); expect(tree.mode).toBe("tree"); expect(tree.content).toContain("# Héading bold code link"); expect(tree.content).toContain("## Quote emphasis"); @@ -47,53 +46,41 @@ describe("Markdown navigation migrated from Organon", () => { /\[([0-9A-Za-z]{2,3})\] ## Quote emphasis/, )?.[1]; expect(quoteID).toBeDefined(); - expect(renderMarkdown(source, false, quoteID, false, 5000)).toEqual({ + expect(renderMarkdown(source, { section_id: quoteID })).toEqual({ mode: "section", content: "> ## Quote *emphasis*\n\n- ### List heading\n", }); }); - it("preserves opaque IDs and full, tree, section, threshold, and truncation modes", () => { + it("uses the fixed automatic tree policy and full/section navigation", () => { const source = "# Test page\n\n## Install\nInstall content.\n\n### Details\nDetails content.\n\n## Next\nNext content.\n"; - const tree = renderMarkdown(source, true, undefined, false, 5000); - expect(tree).toMatchObject({ mode: "tree" }); + const tree = renderMarkdown(`${source}${"x".repeat(5001)}`); + expect(tree.mode).toBe("tree"); expect(tree.content).toContain("[7i] ## Install"); expect(tree.content).toContain("[eD] ### Details"); - expect(renderMarkdown(source, false, "7i", false, 5000)).toEqual({ + expect(renderMarkdown(source, { section_id: "7i" })).toEqual({ mode: "section", content: "## Install\nInstall content.\n\n### Details\nDetails content.\n", }); - expect(() => renderMarkdown(source, false, "missing", false, 5000)).toThrow( + expect(() => renderMarkdown(source, { section_id: "missing" })).toThrow( 'section "missing" not found', ); - expect(renderMarkdown("plain content", true, undefined, false, 1)).toEqual({ + expect(renderMarkdown("plain content")).toEqual({ content: "plain content", mode: "full", }); - expect( - renderMarkdown( - "# H\n\n" + "x".repeat(6000), - false, - undefined, - false, - 5000, - ).mode, - ).toBe("tree"); - expect( - renderMarkdown("# H\n\n" + "x".repeat(6000), false, undefined, true, 1) - .mode, - ).toBe("full"); + expect(renderMarkdown("x".repeat(5001))).toEqual({ + content: "(no headings)\n", + mode: "tree", + }); - const unicode = "😀".repeat(30_001); - const bounded = renderMarkdown(unicode, false, undefined, false, 50_000); - expect( - Array.from(bounded.content.split("\n[content truncated")[0]!).length, - ).toBe(30_000); - expect(bounded.content).toContain( - "[content truncated at 30000 characters]", - ); + const complete = "# H\n\n" + "x".repeat(30_001); + expect(renderMarkdown(complete, { full: true })).toEqual({ + content: complete, + mode: "full", + }); }); }); diff --git a/packages/web/package.json b/packages/web/package.json index 980463c..f3a458d 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -1,7 +1,7 @@ { "name": "@guionai/web", "version": "0.1.0", - "description": "Web research CLI, stdio MCP server, and personal HTTP service with direct and optional agent-browser page-fetch backends.", + "description": "Web research CLI, stdio MCP server, and personal HTTP service with HTTP and optional browser page-rendering modes.", "homepage": "https://github.com/guionai/web#readme", "bugs": { "url": "https://github.com/guionai/web/issues" diff --git a/packages/web/src/http.ts b/packages/web/src/http.ts index e6466b0..422a357 100644 --- a/packages/web/src/http.ts +++ b/packages/web/src/http.ts @@ -104,14 +104,23 @@ const HttpUrlSchema = z const FetchRequestSchema = z .object({ url: HttpUrlSchema, - tree: z.boolean().default(false), - section_id: z.string().optional(), + section_id: z + .string() + .min(1) + .refine( + (value) => value.trim().length > 0, + "section_id must be a non-empty string", + ) + .optional(), full: z.boolean().default(false), - tree_threshold: z.number().int().default(5000), - render: z.enum(["fetch", "agent-browser"]).default("fetch"), + render: z.enum(["http", "browser"]).default("http"), waitMs: z.number().int().min(0).max(30_000).optional(), }) .strict() + .refine( + (input) => !(input.full && input.section_id !== undefined), + "full and section_id cannot be used together", + ) .openapi("FetchRequest"); const LinksRequestSchema = z @@ -123,7 +132,7 @@ const LinksRequestSchema = z .min(1) .max(MAX_LINK_LIMIT) .default(DEFAULT_LINK_LIMIT), - render: z.enum(["fetch", "agent-browser"]).default("fetch"), + render: z.enum(["http", "browser"]).default("http"), waitMs: z.number().int().min(0).max(30_000).optional(), }) .strict() @@ -168,7 +177,7 @@ const fetchRoute = createRoute({ operationId: "fetch", summary: "Fetch a web page", description: - "Fetch direct HTML by default; agent-browser rendering requires render=agent-browser and waitMs.", + "Fetch through HTTP by default; browser rendering requires render=browser and waitMs.", request: jsonRequest(FetchRequestSchema), responses: { 200: jsonResponse(FetchResponseSchema, "Fetched page."), @@ -182,7 +191,7 @@ const linksRoute = createRoute({ operationId: "links", summary: "List page links", description: - "List HTTP(S) anchors using direct fetch by default or explicit agent-browser rendering.", + "List HTTP(S) anchors using HTTP rendering by default or explicit browser rendering.", request: jsonRequest(LinksRequestSchema), responses: { 200: jsonResponse(LinksResponseSchema, "Page links."), @@ -339,19 +348,19 @@ function isHttpURL(value: string): boolean { } function validateRenderFields( - render: "fetch" | "agent-browser" | undefined, + render: "http" | "browser" | undefined, waitMs: number | undefined, ): HttpError | undefined { - const selected = render ?? "fetch"; - if (selected === "fetch" && waitMs !== undefined) + const selected = render ?? "http"; + if (selected === "http" && waitMs !== undefined) return errorBody( "invalid_request", - "waitMs is only valid with render agent-browser", + "waitMs is only valid with render browser", ); - if (selected === "agent-browser" && waitMs === undefined) + if (selected === "browser" && waitMs === undefined) return errorBody( "invalid_request", - "waitMs is required with render agent-browser", + "waitMs is required with render browser", ); return undefined; } @@ -477,10 +486,10 @@ function safeFetchDetails( if (typeof details.retryable === "boolean") safe.retryable = details.retryable; if ( - details.suggestedArguments?.render === "agent-browser" && + details.suggestedArguments?.render === "browser" && details.suggestedArguments.waitMs === 2000 ) { - safe.suggestedArguments = { render: "agent-browser", waitMs: 2000 }; + safe.suggestedArguments = { render: "browser", waitMs: 2000 }; } if (details.reportUrl === RENDER_REPORT_URL) safe.reportUrl = details.reportUrl; diff --git a/packages/web/src/mcp.ts b/packages/web/src/mcp.ts index 36e5c7b..e3e0f08 100644 --- a/packages/web/src/mcp.ts +++ b/packages/web/src/mcp.ts @@ -17,22 +17,18 @@ import { type WebOperations, } from "@guionai/web-core"; -const DEFAULT_FETCH_TREE_THRESHOLD = 5000; - type SearchToolInput = { query: string }; type FetchToolInput = { url: string; - tree?: boolean; section_id?: string; full?: boolean; - tree_threshold?: number; - render?: "fetch" | "agent-browser"; + render?: "http" | "browser"; waitMs?: number; }; type LinksToolInput = { url: string; limit?: number; - render?: "fetch" | "agent-browser"; + render?: "http" | "browser"; waitMs?: number; }; type DocsResolveToolInput = { query: string }; @@ -65,45 +61,47 @@ const fetchInputSchema = schema({ additionalProperties: false, properties: { url: { type: "string", description: "HTTP or HTTPS URL to fetch" }, - tree: { type: "boolean", description: "show the page heading tree" }, section_id: { type: "string", + minLength: 1, description: "optional heading section ID to return", }, full: { type: "boolean", description: "return full content without automatic tree mode", }, - tree_threshold: { - type: "integer", - description: "automatic tree threshold; defaults to 5000", - default: DEFAULT_FETCH_TREE_THRESHOLD, - }, render: { type: "string", - enum: ["fetch", "agent-browser"], - default: "fetch", - description: "optional page-fetch backend; direct fetch is the default", + enum: ["http", "browser"], + default: "http", + description: "optional page renderer; HTTP fetching is the default", }, waitMs: { type: "integer", minimum: 0, maximum: 30_000, - description: - "required post-load wait for agent-browser rendering (0-30000)", + description: "required post-load wait for browser rendering (0-30000)", }, }, required: ["url"], oneOf: [ { - properties: { render: { enum: ["fetch"] } }, + properties: { render: { enum: ["http"] } }, not: { required: ["waitMs"] }, }, { - properties: { render: { const: "agent-browser" } }, + properties: { render: { const: "browser" } }, required: ["render", "waitMs"], }, ], + allOf: [ + { + not: { + required: ["section_id", "full"], + properties: { full: { const: true } }, + }, + }, + ], }); const linksInputSchema = schema({ type: "object", @@ -119,26 +117,25 @@ const linksInputSchema = schema({ }, render: { type: "string", - enum: ["fetch", "agent-browser"], - default: "fetch", - description: "optional page-fetch backend; direct fetch is the default", + enum: ["http", "browser"], + default: "http", + description: "optional page renderer; HTTP fetching is the default", }, waitMs: { type: "integer", minimum: 0, maximum: 30_000, - description: - "required post-load wait for agent-browser rendering (0-30000)", + description: "required post-load wait for browser rendering (0-30000)", }, }, required: ["url"], oneOf: [ { - properties: { render: { enum: ["fetch"] } }, + properties: { render: { enum: ["http"] } }, not: { required: ["waitMs"] }, }, { - properties: { render: { const: "agent-browser" } }, + properties: { render: { const: "browser" } }, required: ["render", "waitMs"], }, ], @@ -309,7 +306,7 @@ export function createMcpServer(dependencies: McpDependencies): McpServer { "links", toolConfig( "List page links", - "List HTTP or HTTPS links from a static page or explicit agent-browser rendering.", + "List HTTP or HTTPS links from an HTTP or browser-rendered page.", linksInputSchema, linksOutputSchema, ), @@ -333,23 +330,18 @@ export function createMcpServer(dependencies: McpDependencies): McpServer { "fetch", toolConfig( "Fetch a web page", - "Use direct fetch (omit render or set render: fetch) for static, SSR, and pre-rendered pages. For client-rendered or SPA pages, set render: agent-browser with required waitMs on a host that has agent-browser installed; there is no automatic fallback.", + "Use the default HTTP renderer for static, SSR, and pre-rendered pages. For client-rendered or SPA pages, set render: browser with required waitMs when the host provides browser capability; there is no automatic fallback.", fetchInputSchema, fetchOutputSchema, ), - async ( - { url, tree, section_id, full, tree_threshold, render, waitMs }, - context, - ) => + async ({ url, section_id, full, render, waitMs }, context) => runTool( () => dependencies.operations.fetch( { url, - tree: tree ?? false, - section_id, - full: full ?? false, - tree_threshold: tree_threshold ?? DEFAULT_FETCH_TREE_THRESHOLD, + ...(section_id !== undefined ? { section_id } : {}), + ...(full !== undefined ? { full } : {}), ...(render !== undefined ? { render } : {}), ...(waitMs !== undefined ? { waitMs } : {}), }, @@ -506,10 +498,10 @@ function safeFetchErrorDetails(details: FetchErrorDetails): FetchErrorDetails { if (typeof details.retryable === "boolean") safe.retryable = details.retryable; if ( - details.suggestedArguments?.render === "agent-browser" && + details.suggestedArguments?.render === "browser" && details.suggestedArguments.waitMs === 2000 ) { - safe.suggestedArguments = { render: "agent-browser", waitMs: 2000 }; + safe.suggestedArguments = { render: "browser", waitMs: 2000 }; } if (details.reportUrl === RENDER_REPORT_URL) safe.reportUrl = details.reportUrl; diff --git a/packages/web/src/program.ts b/packages/web/src/program.ts index 7dc6fba..f72773b 100644 --- a/packages/web/src/program.ts +++ b/packages/web/src/program.ts @@ -237,45 +237,36 @@ function createFetchCommand(dependencies: ProgramDependencies): Command { return new Command("fetch") .description("Fetch a static, SSR, or pre-rendered page as Markdown") .argument("", "HTTP or HTTPS URL") - .option("--tree", "Show the heading tree") - .option( - "--render ", - "Rendering backend: fetch (default) or agent-browser", - ) + .option("--render ", "Rendering mode: http (default) or browser") .option( "--wait ", - "Required post-load wait for --render agent-browser (0-30000)", + "Required post-load wait for --render browser (0-30000)", Number, ) .option("-s, --section ", "Read one heading section") - .option( - "--full", - "Return full content instead of automatic tree navigation", - ) - .option("--tree-threshold ", "Auto-tree threshold", Number) + .option("--full", "Return the complete extracted Markdown") .option("--json", "Output the structured result as JSON") .action( async ( url: string, options: { - tree?: boolean; - render?: "fetch" | "agent-browser"; + render?: "http" | "browser"; wait?: number; section?: string; full?: boolean; - treeThreshold?: number; json?: boolean; }, ) => { - const result = await dependencies.operations.fetch({ + const input = { url, - tree: options.tree, ...(options.render !== undefined ? { render: options.render } : {}), ...(options.wait !== undefined ? { waitMs: options.wait } : {}), - section_id: options.section, - full: options.full, - tree_threshold: options.treeThreshold, - }); + ...(options.section !== undefined + ? { section_id: options.section } + : {}), + ...(options.full !== undefined ? { full: options.full } : {}), + }; + const result = await dependencies.operations.fetch(input); if (options.json) { writeOut(JSON.stringify(result) + "\n"); return; @@ -292,13 +283,10 @@ function createLinksCommand(dependencies: ProgramDependencies): Command { .description("List HTTP(S) links from a web page") .argument("", "HTTP or HTTPS URL") .option("--limit ", "Maximum links to return (1-100)", Number) - .option( - "--render ", - "Rendering backend: fetch (default) or agent-browser", - ) + .option("--render ", "Rendering mode: http (default) or browser") .option( "--wait ", - "Required post-load wait for --render agent-browser (0-30000)", + "Required post-load wait for --render browser (0-30000)", Number, ) .option("--json", "Output the structured result as JSON") @@ -307,7 +295,7 @@ function createLinksCommand(dependencies: ProgramDependencies): Command { url: string, options: { limit?: number; - render?: "fetch" | "agent-browser"; + render?: "http" | "browser"; wait?: number; json?: boolean; }, diff --git a/packages/web/src/runner.ts b/packages/web/src/runner.ts index c5fdd4b..bf45587 100644 --- a/packages/web/src/runner.ts +++ b/packages/web/src/runner.ts @@ -38,7 +38,7 @@ function formatCliError(error: unknown): string { if (error.code === "javascript_rendering_may_be_required") { return ( "javascript_rendering_may_be_required: content may require JavaScript rendering\n" + - "Retry: web fetch --render=agent-browser --wait=2000\n" + "Retry: web fetch --render=browser --wait=2000\n" ); } if (error.code === "render_domain_not_allowed") { diff --git a/packages/web/test/http.test.ts b/packages/web/test/http.test.ts index fcbebf5..f0ca31a 100644 --- a/packages/web/test/http.test.ts +++ b/packages/web/test/http.test.ts @@ -152,7 +152,7 @@ describe("personal HTTP service", () => { const result = await json(app, "/v1/fetch", { url: "https://example.test", - render: "agent-browser", + render: "browser", waitMs: 0, }); @@ -161,10 +161,8 @@ describe("personal HTTP service", () => { expect(ops.fetch).toHaveBeenCalledWith( expect.objectContaining({ url: "https://example.test", - tree: false, full: false, - tree_threshold: 5000, - render: "agent-browser", + render: "browser", waitMs: 0, }), expect.any(AbortSignal), @@ -183,7 +181,7 @@ describe("personal HTTP service", () => { const result = await json(app, "/v1/links", { url: "https://example.test", - render: "agent-browser", + render: "browser", waitMs: 0, }); @@ -195,7 +193,7 @@ describe("personal HTTP service", () => { { url: "https://example.test", limit: 100, - render: "agent-browser", + render: "browser", waitMs: 0, }, expect.any(AbortSignal), @@ -225,6 +223,23 @@ describe("personal HTTP service", () => { }) ).response.status, ).toBe(400); + expect( + ( + await json(app, "/v1/fetch", { + url: "https://example.test", + full: true, + section_id: "intro", + }) + ).response.status, + ).toBe(400); + expect( + ( + await json(app, "/v1/fetch", { + url: "https://example.test", + tree: true, + }) + ).response.status, + ).toBe(400); expect(ops.search).not.toHaveBeenCalled(); expect(ops.fetch).not.toHaveBeenCalled(); expect(ops.links).not.toHaveBeenCalled(); @@ -275,6 +290,10 @@ describe("personal HTTP service", () => { (document.components?.schemas as any).SearchResponse.properties.provider .enum, ).toEqual(["Exa", "Kepos Bridge"]); + const fetchRequest = (document.components?.schemas as any).FetchRequest; + expect(fetchRequest.properties.render.enum).toEqual(["http", "browser"]); + expect(fetchRequest.properties.tree).toBeUndefined(); + expect(fetchRequest.properties.tree_threshold).toBeUndefined(); }); it("requires an Exa key and validates the server-local Bridge endpoint", () => { diff --git a/packages/web/test/mcp.test.ts b/packages/web/test/mcp.test.ts index 561ccfd..5bf6636 100644 --- a/packages/web/test/mcp.test.ts +++ b/packages/web/test/mcp.test.ts @@ -155,7 +155,7 @@ describe("web stdio MCP adapter", () => { .mockRejectedValueOnce( new FetchCapabilityError("javascript_rendering_may_be_required", { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }), ) .mockResolvedValueOnce({ @@ -174,7 +174,7 @@ describe("web stdio MCP adapter", () => { name: "fetch", arguments: { url: "https://example.test/page", - render: "agent-browser", + render: "browser", waitMs: 2000, }, }); @@ -185,7 +185,7 @@ describe("web stdio MCP adapter", () => { code: "javascript_rendering_may_be_required", details: { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }, }, }); @@ -196,11 +196,7 @@ describe("web stdio MCP adapter", () => { 2, { url: "https://example.test/page", - tree: false, - full: false, - section_id: undefined, - tree_threshold: 5000, - render: "agent-browser", + render: "browser", waitMs: 2000, }, expect.any(AbortSignal), @@ -250,10 +246,9 @@ describe("web stdio MCP adapter", () => { string, unknown >; - expect(fetchProperties.tree_threshold).toMatchObject({ default: 5000 }); expect(fetchProperties.render).toMatchObject({ - enum: ["fetch", "agent-browser"], - default: "fetch", + enum: ["http", "browser"], + default: "http", }); expect(fetchProperties.waitMs).toMatchObject({ type: "integer", @@ -267,8 +262,8 @@ describe("web stdio MCP adapter", () => { maximum: 100, }); expect(linksProperties.render).toMatchObject({ - enum: ["fetch", "agent-browser"], - default: "fetch", + enum: ["http", "browser"], + default: "http", }); expect(docsFetchProperties.tokens).toMatchObject({ default: 0 }); expect(sgraphProperties).toMatchObject({ @@ -293,8 +288,7 @@ describe("web stdio MCP adapter", () => { name: "fetch", arguments: { url: "https://example.test/page", - tree: true, - render: "agent-browser", + render: "browser", waitMs: 125, }, }); @@ -303,7 +297,7 @@ describe("web stdio MCP adapter", () => { arguments: { url: "https://example.test/page", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 125, }, }); @@ -347,11 +341,7 @@ describe("web stdio MCP adapter", () => { expect(operations.fetch).toHaveBeenCalledWith( { url: "https://example.test/page", - tree: true, - full: false, - section_id: undefined, - tree_threshold: 5000, - render: "agent-browser", + render: "browser", waitMs: 125, }, expect.any(AbortSignal), @@ -360,7 +350,7 @@ describe("web stdio MCP adapter", () => { { url: "https://example.test/page", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 125, }, expect.any(AbortSignal), @@ -387,17 +377,17 @@ describe("web stdio MCP adapter", () => { await client.listTools(); for (const arguments_ of [ - { url: "https://example.test/page", render: "agent-browser" }, - { url: "https://example.test/page", render: "fetch", waitMs: 0 }, - { url: "https://example.test/page", render: "agent-browser", waitMs: -1 }, + { url: "https://example.test/page", render: "browser" }, + { url: "https://example.test/page", render: "http", waitMs: 0 }, + { url: "https://example.test/page", render: "browser", waitMs: -1 }, { url: "https://example.test/page", - render: "agent-browser", + render: "browser", waitMs: 30_001, }, { url: "https://example.test/page", - render: "agent-browser", + render: "browser", waitMs: 1.5, }, ]) { @@ -437,7 +427,7 @@ describe("web stdio MCP adapter", () => { name: "fetch", arguments: { url: "https://example.test/page", - render: "agent-browser", + render: "browser", waitMs: 2000, }, }); @@ -549,7 +539,7 @@ describe("web stdio MCP adapter", () => { name: "fetch", arguments: { url: "https://example.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, }, diff --git a/packages/web/test/packed-smoke.mjs b/packages/web/test/packed-smoke.mjs index 9fd18f6..512ed41 100644 --- a/packages/web/test/packed-smoke.mjs +++ b/packages/web/test/packed-smoke.mjs @@ -229,7 +229,7 @@ else console.log(JSON.stringify({ success: true, data: {} })); [ "fetch", "https://93.184.216.34/rendered", - "--render=agent-browser", + "--render=browser", "--wait=0", "--full", "--json", @@ -261,7 +261,7 @@ else console.log(JSON.stringify({ success: true, data: {} })); [ "fetch", "https://93.184.216.34/blocked", - "--render=agent-browser", + "--render=browser", "--wait=0", ], { cwd: root, env: fakeEnvironment }, diff --git a/packages/web/test/program.test.ts b/packages/web/test/program.test.ts index 71062b9..e061736 100644 --- a/packages/web/test/program.test.ts +++ b/packages/web/test/program.test.ts @@ -184,24 +184,13 @@ describe("web search Commander adapter", () => { it("passes fetch navigation flags and writes human Markdown", async () => { const { program, operations, output } = setup(); await program.parseAsync( - [ - "fetch", - "https://example.test/page", - "--tree", - "-s", - "7i", - "--tree-threshold", - "9000", - ], + ["fetch", "https://example.test/page", "-s", "7i"], { from: "user" }, ); expect(operations.fetch).toHaveBeenCalledWith({ url: "https://example.test/page", - tree: true, section_id: "7i", - full: undefined, - tree_threshold: 9000, }); expect(output()).toEqual({ stdout: "# Fixture page\n", stderr: "" }); }); @@ -209,23 +198,14 @@ describe("web search Commander adapter", () => { it("forwards explicit browser rendering options", async () => { const { program, operations } = setup(); await program.parseAsync( - [ - "fetch", - "https://example.test/page", - "--render=agent-browser", - "--wait=0", - ], + ["fetch", "https://example.test/page", "--render=browser", "--wait=0"], { from: "user" }, ); expect(operations.fetch).toHaveBeenCalledWith({ url: "https://example.test/page", - tree: undefined, - render: "agent-browser", + render: "browser", waitMs: 0, - section_id: undefined, - full: undefined, - tree_threshold: undefined, }); }); @@ -238,10 +218,7 @@ describe("web search Commander adapter", () => { expect(operations.fetch).toHaveBeenCalledWith({ url: "https://example.test/page", - tree: undefined, - section_id: undefined, full: true, - tree_threshold: undefined, }); expect(JSON.parse(output().stdout)).toEqual( await operations.fetch.mock.results[0]!.value, @@ -257,7 +234,7 @@ describe("web search Commander adapter", () => { "https://example.test/page", "--limit", "25", - "--render=agent-browser", + "--render=browser", "--wait=0", ], { from: "user" }, @@ -266,7 +243,7 @@ describe("web search Commander adapter", () => { expect(operations.links).toHaveBeenCalledWith({ url: "https://example.test/page", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 0, }); expect(output()).toEqual({ @@ -289,7 +266,7 @@ describe("web search Commander adapter", () => { fetch: vi.fn(async () => { throw new FetchCapabilityError("javascript_rendering_may_be_required", { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }); }), links: vi.fn(), @@ -317,7 +294,7 @@ describe("web search Commander adapter", () => { expect(stdout).toBe(""); expect(stderr).toBe( "javascript_rendering_may_be_required: content may require JavaScript rendering\n" + - "Retry: web fetch --render=agent-browser --wait=2000\n", + "Retry: web fetch --render=browser --wait=2000\n", ); }); From def658c3ed1e5e33f8b23cfc02a88f5a81768b89 Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 15:39:05 +0800 Subject: [PATCH 09/14] chore(http): record implementation report --- .../http-web-service/implementation-report.md | 128 +++++++++++------- 1 file changed, 77 insertions(+), 51 deletions(-) diff --git a/.scratch/http-web-service/implementation-report.md b/.scratch/http-web-service/implementation-report.md index d6282dd..d9760a7 100644 --- a/.scratch/http-web-service/implementation-report.md +++ b/.scratch/http-web-service/implementation-report.md @@ -1,69 +1,95 @@ -# Guion Web HTTP service implementation report +# Guion Web page-reading contract implementation report ## Scope - Repository: `guionai/web` - Branch: `http-web-service` -- Fixed point: `ebad44eb55f8a406b855318cb8a4b4cd04633227` -- Implementation commit: `3316d5bb5e14e81d89dbdd84239d16b5de007b10` (`feat(http): add personal research service`) -- Review-fix commit: `73405d85f047e35354bb340c90faf8374b23a2ba` (`fix(http): close service review gaps`) -- Delivery boundary: the complete HTTP-service spec and tickets 01, 02, and 03; code review and deployment were excluded. +- Fixed point: `9fff0766001f5ebcad91045c34958af72841da09` +- Implementation commit: `bb4900a5dba9b5d71fdd64363ce860953b2dc098` (`feat(http): unify page-reading contract`) +- Delivery boundary: the complete `http-web-service` spec and tickets 01 and 02. + Code review and deployment were excluded. -The implementation was completed in dependency order: the server-local Bridge-first Search and typed Bridge Data Operations, the remaining Research Operations and container guide, and then the release/OpenAPI/GHCR contract. - -## Review fixes - -- Removed the unused `createHttpService` alias and compatibility comment; the canonical `createHttpApp` export is the only HTTP-app constructor. -- Added a test-owned temporary-file artifact test that calls the YAML generator, reads and parses `openapi.yaml`, checks package-version parity, all ten `/v1` paths, and the absence of a generic Bridge route. -- Added table-driven invalid-body cases for Links, Docs Resolve, Docs Fetch, Source Search, Sports, Finance, and Time. Every case asserts HTTP 400, `invalid_request`, and no operation invocation. -- Narrowed the HTTP Search response/OpenAPI provider enum to `Exa` and `Kepos Bridge`; provider-selection behavior for CLI, MCP, Pi, and DSH remains unchanged. +The tickets were implemented in dependency order: the shared Core page-reading +seam first, then every adapter, HTTP/OpenAPI contract, tests, and documentation. ## Ticket outcomes -### 01 — Serve Search and Bridge Data over HTTP - -- Added `web serve` with Hono and `@hono/zod-openapi` route schemas. -- Added `/v1/search`, `/v1/weather`, `/v1/sports`, `/v1/finance`, and `/v1/time`. -- HTTP Search sends server-local Kepos Bridge configuration first, preserves a successful empty result, retries Exa exactly once for a non-cancellation Bridge failure, and does not retry cancellation. -- Startup requires a non-empty `EXA_API_KEY`; `KEPOS_BRIDGE_ENDPOINT` is validated and otherwise uses the core default. Context7 remains optional. -- Typed Data Operations accept only their documented fields, call only their corresponding Bridge command, and have no generic command or Exa fallback. -- Invalid bodies, malformed JSON, capability failures, timeouts, cancellations, and upstream failures map to bounded documented JSON error responses without credentials or raw upstream bodies. - -### 02 — Complete the Research API and Container Guide - -- Added `/v1/fetch`, `/v1/links`, `/v1/docs/resolve`, `/v1/docs/fetch`, and `/v1/source-search` with request and response schemas. -- Fetch and Links default to direct fetching; rendered fetching requires explicit `render: "agent-browser"` and an integer `waitMs` from 0 through 30,000. -- Added the Node 24 container image, installing the pinned `agent-browser` runtime and starting `web serve` on port 8787. Credentials and Bridge configuration remain environment-only. -- Updated `README.md`, `CONTEXT.md`, ADR 0001, and `.scratch/defered/public-http-service-security.md` to describe the personal-service boundary and deferred public hardening. - -### 03 — Publish the Versioned HTTP Service Contract - -- The web build now generates `packages/web/dist/openapi.yaml` from the registered route schemas with the package version. -- The release preflight uploads the generated contract; the release workflow publishes a release-tagged GHCR image with `packages: write` and creates the GitHub Release only after npm and image jobs succeed. -- The matching generated `openapi.yaml` is attached to the GitHub Release and is not checked in or independently versioned. +### 01 — Unify the page-reading module interface + +- Core Fetch and Links now accept only `render: "http" | "browser"`. + HTTP is the default; browser requires `waitMs` from 0 through 30,000, while + HTTP forbids `waitMs`. Capability errors suggest `render: "browser"` with + `waitMs: 2000`. +- Public `tree` and `tree_threshold` inputs were removed. The module owns the + fixed 5,000-character policy: non-full, unsectioned long content returns a + navigation tree, `full: true` returns complete extracted Markdown without the + Core 30,000-character truncation, and `section_id` retrieves one tree section. +- `full: true` with `section_id` is rejected. Core validation also rejects + unknown legacy fields and invalid navigation values. +- Core tests use injected cache, HTTP, and browser seams; no live provider or + browser state is required. + +### 02 — Align adapters, HTTP, and reference documentation + +- CLI flags/help, stdio MCP schemas, Pi TypeBox schemas/prompts, DSH tool + definitions, and HTTP schemas now expose the same renderer and navigation + vocabulary without compatibility aliases. +- HTTP remains exactly `/v1/search`, `/v1/fetch`, and `/v1/links`. Search keeps + server-selected Kepos Bridge-first behavior with one Exa retry; Fetch and + Links enforce the shared renderer rules. +- The build generates the version-matched OpenAPI 3.1 artifact at + `packages/web/dist/openapi.yaml`; the release artifact test parses a + test-owned generated file and asserts exactly the three retained routes and + the unified request schemas. +- Added [`docs/http-service.md`](../../docs/http-service.md), a standalone + human-readable reference covering requests, responses, validation, provider + behavior, renderer behavior, errors, and the OpenAPI release asset. README + and ADR 0001 link to and describe the same contract. Operator setup may still + name the installed browser executable. ## Verification -All checks below completed successfully against the review-fix commit: +All checks completed successfully against the implementation commit: -- `pnpm format:check` — all files matched Prettier. -- `pnpm typecheck` — TypeScript completed with no errors. -- `pnpm build` — all workspace packages built; `packages/web/dist/openapi.yaml` generated. -- Generated OpenAPI parse check — OpenAPI `3.1.0`, package version `0.1.0`, exactly the 10 documented `/v1` paths, and no generic Bridge route. -- `pnpm test` — 19 test files and 117 tests passed. The existing DSH source-map warning was non-fatal. +- `pnpm format:check` and explicit Prettier checks for the ADR, DSH README, and + standalone HTTP reference. +- `pnpm typecheck`. +- `pnpm test` — 19 test files and 117 tests passed. The existing DSH source-map + warning was non-fatal. +- `pnpm build` — all workspace packages built and generated + `packages/web/dist/openapi.yaml`. - `pnpm test:release` — release version synchronization fixtures passed. -- `pnpm test:pack` — packed-installation/host-loading smoke tests passed for `@guionai/web`, `@guionai/pi-web`, and `@guionai/dsh-web`. -- Focused in-process HTTP tests use injected operation fakes and test-owned credentials; they cover validation, exact fallback/cancellation behavior, typed Bridge command shapes, response normalization, startup configuration, and OpenAPI paths without provider or browser network calls. +- `pnpm test:pack` — packed-installation/host-loading checks passed for + `@guionai/web`, `@guionai/pi-web`, and `@guionai/dsh-web`. +- Docker validation with the supported image: `docker build --tag +guionai-web-http-smoke .` passed; a disposable container returned HTTP 400 + for invalid Fetch input and HTTP 200 for a real browser-rendered + `https://example.com` Fetch with `full: true`. -The local Docker daemon was unavailable for a Docker build check (`/Users/neil/.orbstack/run/docker.sock` did not exist). The Dockerfile and workflow were reviewed statically; image deployment remains outside this task's boundary. +No live credentials were used. The Docker smoke used a disposable container +and a non-secret placeholder startup key; it was stopped after verification. ## Changed paths and size -Against the fixed point, excluding generated `dist` output (including `openapi.yaml`), `pnpm-lock.yaml`, and this report: - -- Product code: 938 additions, 1 deletion (`packages/web/src`, `packages/web-core/src`). -- Tests: 485 additions (`packages/web/test`, `packages/web-core/test`). -- Configuration and documentation: 240 additions, 14 deletions (Docker, release workflow, manifests, README, CONTEXT, ADR, and deferred note). -- Total: 1,663 additions and 15 deletions (net 1,648 lines). - -The total is within the spec estimate of 1,230–1,900 lines. The review adds focused file-artifact and table-driven validation coverage while keeping all fixtures in process or test-owned temporary directories. +Against the fixed point, excluding generated `dist` output, lockfiles, and +this report: + +- Product code: 290 additions and 210 deletions (500 changed lines) across + Core, CLI, MCP, Pi, DSH, and HTTP. +- Tests: 203 additions and 194 deletions (397 changed lines), including Core + navigation/renderer seams, adapter forwarding/validation, and OpenAPI + artifact parsing. +- Documentation/configuration: 210 additions and 49 deletions (259 changed + lines), including the 154-line standalone HTTP reference and aligned README, + ADR, package metadata, and DSH guide. +- Total: 703 additions and 425 deletions (1,128 changed lines), within the + spec estimate of 780–1,400 total changed lines. Product code is modestly + above its 250–450 estimate because each owned adapter now performs explicit + input normalization and legacy-field rejection at its boundary. + +## Remaining concerns + +- Public or multi-tenant HTTP hardening remains out of scope as documented by + ADR 0001 and the existing deferred security note. +- Code review and deployment were intentionally not run for this implementation + task. From 90de2fa9d77a6dc2ac8c34284f469e1d722e3d12 Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 15:56:47 +0800 Subject: [PATCH 10/14] fix(http): repair navigation and renderer validation --- CONTEXT.md | 20 +++++-- packages/dsh-web/src/tools.ts | 70 ++++++++++------------ packages/pi-web/src/tool.ts | 78 +++++++++++-------------- packages/web-core/src/markdown.ts | 8 ++- packages/web-core/test/markdown.test.ts | 15 +++++ 5 files changed, 104 insertions(+), 87 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 20cf6f1..848c39b 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -24,9 +24,19 @@ _Avoid_: Generic Bridge command, special search The server-local URL used by the Kepos Bridge search provider. The service operator configures it; API callers never supply it. _Avoid_: Bridge URL parameter -**Rendered Fetch**: -A fetch performed with the host-installed `agent-browser` browser runtime, explicitly selected instead of direct HTTP fetching. -_Avoid_: Browser fetch, automatic fallback +**Page Rendering**: +Fetch and Links use `render: "http"` by default or explicit `render: "browser"`. +Browser rendering requires `waitMs` from 0 through 30,000 and is never selected +automatically. The operator-installed `agent-browser` runtime is an implementation +and setup detail, not an adapter-facing request value. +_Avoid_: Backend-specific renderer labels, automatic fallback + +**Page Navigation**: +The shared page-reading module owns its fixed 5,000-character policy. A long, +unsectioned request returns a navigation tree; `full: true` returns complete +Markdown, and a tree's `section_id` retrieves one section. `full: true` and +`section_id` are mutually exclusive. +_Avoid_: Caller-selected tree thresholds, public `tree` controls **Release Contract**: The versioned public distribution of Guion Web: its npm packages, GHCR container image, and the generated `openapi.yaml` attached to the matching GitHub Release. @@ -35,5 +45,7 @@ _Avoid_: Checked-in OpenAPI file, independently versioned schema **HTTP Service**: The Hono-based `/v1` JSON API shipped by `web serve` and the GHCR image. It uses server-local credentials and Bridge Route configuration; clients do not -select providers or submit a generic Bridge command. +select providers or submit a generic Bridge command. Its page-reading routes use +the same `render: "http" | "browser"`, `full`, and `section_id` contract; the +browser executable name appears only in operator setup. _Avoid_: Remote MCP, public service diff --git a/packages/dsh-web/src/tools.ts b/packages/dsh-web/src/tools.ts index 92d9aee..2c5acc5 100644 --- a/packages/dsh-web/src/tools.ts +++ b/packages/dsh-web/src/tools.ts @@ -369,30 +369,12 @@ function normalizeLinks(input: unknown): LinksInput { `limit must be an integer from 1 through ${MAX_LINK_LIMIT}`, ); - const render = input.render; - const waitMs = input.waitMs; - if (render !== undefined && render !== "http" && render !== "browser") - throw new Error('render must be "http" or "browser"'); - if (render !== "browser") { - if (waitMs !== undefined) - throw new Error("waitMs is only valid with render browser"); - } else { - if (waitMs === undefined) - throw new Error("waitMs is required with render browser"); - if ( - typeof waitMs !== "number" || - !Number.isInteger(waitMs) || - waitMs < 0 || - waitMs > 30_000 - ) - throw new Error("waitMs must be an integer from 0 through 30000"); - } + const renderOptions = validateRenderOptions(input); return { url, ...(limit !== undefined ? { limit } : {}), - ...(render !== undefined ? { render } : {}), - ...(waitMs !== undefined ? { waitMs } : {}), + ...renderOptions, }; } @@ -415,33 +397,45 @@ function normalizeFetch(input: unknown): FetchInput { throw new Error("full must be a boolean"); if (full === true && sectionID !== undefined) throw new Error("full and section_id cannot be used together"); + const renderOptions = validateRenderOptions(input); + return { + url, + ...(sectionID === undefined ? {} : { section_id: sectionID }), + ...(full === undefined ? {} : { full }), + ...renderOptions, + }; +} + +function validateRenderOptions(input: { + render?: unknown; + waitMs?: unknown; +}): RenderOptions { const render = input.render; + const waitMs = input.waitMs; if (render !== undefined && render !== "http" && render !== "browser") throw new Error('render must be "http" or "browser"'); - const waitMs = input.waitMs; if (render !== "browser") { if (waitMs !== undefined) throw new Error("waitMs is only valid with render browser"); - } else { - if (waitMs === undefined) - throw new Error("waitMs is required with render browser"); - if ( - typeof waitMs !== "number" || - !Number.isInteger(waitMs) || - waitMs < 0 || - waitMs > 30_000 - ) - throw new Error("waitMs must be an integer from 0 through 30000"); + return render === undefined ? {} : { render }; } - return { - url, - ...(sectionID === undefined ? {} : { section_id: sectionID }), - ...(full === undefined ? {} : { full }), - ...(render === undefined ? {} : { render }), - ...(waitMs === undefined ? {} : { waitMs }), - }; + if (waitMs === undefined) + throw new Error("waitMs is required with render browser"); + if ( + typeof waitMs !== "number" || + !Number.isInteger(waitMs) || + waitMs < 0 || + waitMs > 30_000 + ) + throw new Error("waitMs must be an integer from 0 through 30000"); + return { render, waitMs }; } +type RenderOptions = + | { render?: undefined; waitMs?: undefined } + | { render: "http"; waitMs?: undefined } + | { render: "browser"; waitMs: number }; + function normalizeDocs(input: unknown): DocsToolInput { if (!isRecord(input)) throw new Error("web_docs input must be an object"); return normalizeDocsToolInput(input); diff --git a/packages/pi-web/src/tool.ts b/packages/pi-web/src/tool.ts index b5a33da..61508b0 100644 --- a/packages/pi-web/src/tool.ts +++ b/packages/pi-web/src/tool.ts @@ -275,24 +275,7 @@ function normalizeFetch(input: unknown): WebFetchInput { throw new Error(`web_fetch input does not accept field ${field}`); } const url = requireString(input, "url"); - const render = input.render; - const waitMs = input.waitMs; - if (render !== undefined && render !== "http" && render !== "browser") - throw new Error('render must be "http" or "browser"'); - if (render !== "browser") { - if (waitMs !== undefined) - throw new Error("waitMs is only valid with render browser"); - } else { - if (waitMs === undefined) - throw new Error("waitMs is required with render browser"); - if ( - typeof waitMs !== "number" || - !Number.isInteger(waitMs) || - waitMs < 0 || - waitMs > 30_000 - ) - throw new Error("waitMs must be an integer from 0 through 30000"); - } + const renderOptions = validateRenderOptions(input); if (input.full !== undefined && typeof input.full !== "boolean") throw new Error("full must be a boolean"); if (input.section_id !== undefined) { @@ -307,10 +290,7 @@ function normalizeFetch(input: unknown): WebFetchInput { ...(typed.section_id === undefined ? {} : { section_id: typed.section_id }), ...(typed.full === undefined ? {} : { full: typed.full }), }; - if (render === "browser") - return { ...navigation, render, waitMs: waitMs as number }; - if (render === "http") return { ...navigation, render }; - return navigation; + return { ...navigation, ...renderOptions }; } function normalizeLinks(input: unknown): WebLinksInput { @@ -320,25 +300,8 @@ function normalizeLinks(input: unknown): WebLinksInput { throw new Error(`web_links input does not accept field ${field}`); } const url = requireString(input, "url"); - const render = input.render; - const waitMs = input.waitMs; + const renderOptions = validateRenderOptions(input); const limit = input.limit; - if (render !== undefined && render !== "http" && render !== "browser") - throw new Error('render must be "http" or "browser"'); - if (render !== "browser") { - if (waitMs !== undefined) - throw new Error("waitMs is only valid with render browser"); - } else { - if (waitMs === undefined) - throw new Error("waitMs is required with render browser"); - if ( - typeof waitMs !== "number" || - !Number.isInteger(waitMs) || - waitMs < 0 || - waitMs > 30_000 - ) - throw new Error("waitMs must be an integer from 0 through 30000"); - } if ( limit !== undefined && (typeof limit !== "number" || @@ -354,12 +317,39 @@ function normalizeLinks(input: unknown): WebLinksInput { url, ...(typed.limit === undefined ? {} : { limit: typed.limit }), }; - if (render === "browser") - return { ...result, render, waitMs: waitMs as number }; - if (render === "http") return { ...result, render }; - return result; + return { ...result, ...renderOptions }; } +function validateRenderOptions(input: { + render?: unknown; + waitMs?: unknown; +}): RenderOptions { + const render = input.render; + const waitMs = input.waitMs; + if (render !== undefined && render !== "http" && render !== "browser") + throw new Error('render must be "http" or "browser"'); + if (render !== "browser") { + if (waitMs !== undefined) + throw new Error("waitMs is only valid with render browser"); + return render === undefined ? {} : { render }; + } + if (waitMs === undefined) + throw new Error("waitMs is required with render browser"); + if ( + typeof waitMs !== "number" || + !Number.isInteger(waitMs) || + waitMs < 0 || + waitMs > 30_000 + ) + throw new Error("waitMs must be an integer from 0 through 30000"); + return { render, waitMs }; +} + +type RenderOptions = + | { render?: undefined; waitMs?: undefined } + | { render: "http"; waitMs?: undefined } + | { render: "browser"; waitMs: number }; + function mergeSearchResults(responses: SearchResponse[]): SearchResponse { const results: SearchResponse["results"] = []; for (let resultIndex = 0; ; resultIndex += 1) { diff --git a/packages/web-core/src/markdown.ts b/packages/web-core/src/markdown.ts index 75496cc..7f3028a 100644 --- a/packages/web-core/src/markdown.ts +++ b/packages/web-core/src/markdown.ts @@ -184,7 +184,13 @@ function renderTree(source: string, headings: Heading[]): string { 99, ); if (bodyHeadings.length === 0) { - return `${header}(empty)\n\nUse section_id to read a section, or full: true to read everything.\n`; + const tree = headings + .map( + (heading, index) => + `└── [${heading.id}] ${"#".repeat(heading.level)} ${heading.text} (${formatNumber(sectionCharCount(source, headings, index))} chars)\n`, + ) + .join(""); + return `${header}${tree}\nUse section_id to read a section, or full: true to read everything.\n`; } const nodes = bodyHeadings.map((heading) => { diff --git a/packages/web-core/test/markdown.test.ts b/packages/web-core/test/markdown.test.ts index e2e042c..a8c5bd5 100644 --- a/packages/web-core/test/markdown.test.ts +++ b/packages/web-core/test/markdown.test.ts @@ -83,4 +83,19 @@ describe("Markdown navigation", () => { mode: "full", }); }); + + it("lists and retrieves an H1-only long document section", () => { + const source = `# Only title\n\n${"x".repeat(5001)}\n`; + const tree = renderMarkdown(source); + const sectionID = tree.content.match( + /\[([0-9A-Za-z]{2,3})\] # Only title/, + )?.[1]; + + expect(tree.mode).toBe("tree"); + expect(sectionID).toBeDefined(); + expect(renderMarkdown(source, { section_id: sectionID })).toEqual({ + content: source, + mode: "section", + }); + }); }); From a461206ee94494aa8091c2769ec2e5387c6f3b1c Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 15:59:34 +0800 Subject: [PATCH 11/14] chore(http): update implementation report --- .../http-web-service/implementation-report.md | 45 ++++++++++++++++--- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/.scratch/http-web-service/implementation-report.md b/.scratch/http-web-service/implementation-report.md index d9760a7..3ff1fab 100644 --- a/.scratch/http-web-service/implementation-report.md +++ b/.scratch/http-web-service/implementation-report.md @@ -5,7 +5,7 @@ - Repository: `guionai/web` - Branch: `http-web-service` - Fixed point: `9fff0766001f5ebcad91045c34958af72841da09` -- Implementation commit: `bb4900a5dba9b5d71fdd64363ce860953b2dc098` (`feat(http): unify page-reading contract`) +- Implementation commits: `bb4900a5dba9b5d71fdd64363ce860953b2dc098` (`feat(http): unify page-reading contract`) and `90de2fa9d77a6dc2ac8c34284f469e1d722e3d12` (`fix(http): repair navigation and renderer validation`) - Delivery boundary: the complete `http-web-service` spec and tickets 01 and 02. Code review and deployment were excluded. @@ -28,6 +28,9 @@ seam first, then every adapter, HTTP/OpenAPI contract, tests, and documentation. unknown legacy fields and invalid navigation values. - Core tests use injected cache, HTTP, and browser seams; no live provider or browser state is required. +- H1-only long documents now list their H1 as a selectable tree node with a + stable `section_id`; a Core behavior test retrieves the emitted ID and checks + the complete section content. ### 02 — Align adapters, HTTP, and reference documentation @@ -46,6 +49,22 @@ seam first, then every adapter, HTTP/OpenAPI contract, tests, and documentation. behavior, renderer behavior, errors, and the OpenAPI release asset. README and ADR 0001 link to and describe the same contract. Operator setup may still name the installed browser executable. +- CONTEXT.md now uses the public `render: "http" | "browser"` vocabulary for + glossary and HTTP guidance; `agent-browser` appears only as an operator + implementation/setup detail. Pi and DSH each consolidate their repeated + renderer/wait checks in one local helper while preserving validation order, + messages, and forwarding behavior. + +## Review-fix batch + +The post-review repairs were applied together in `90de2fa`: + +- Core tree output for an H1-only long document emits its deterministic heading + ID, and the behavior test uses that ID to retrieve the section. +- CONTEXT.md describes the public HTTP/browser renderer contract and reserves + the `agent-browser` name for operator implementation/setup wording. +- Pi and DSH each own a local `validateRenderOptions` helper; no + cross-adapter abstraction was added. ## Verification @@ -69,23 +88,37 @@ guionai-web-http-smoke .` passed; a disposable container returned HTTP 400 No live credentials were used. The Docker smoke used a disposable container and a non-secret placeholder startup key; it was stopped after verification. +The review-fix commit was additionally verified with: + +- `pnpm typecheck`. +- `pnpm format:check` and explicit Prettier checks for the changed source files, + CONTEXT.md, and the HTTP reference. +- `pnpm test` — 19 test files and 118 tests passed. The existing DSH source-map + warning was non-fatal. +- `pnpm build` — all workspace packages built and generated + `packages/web/dist/openapi.yaml`. +- `pnpm test:release` — release version synchronization fixtures passed. +- `pnpm test:pack` — packed-installation/host-loading checks passed for + `@guionai/web`, `@guionai/pi-web`, and `@guionai/dsh-web`. + ## Changed paths and size Against the fixed point, excluding generated `dist` output, lockfiles, and this report: -- Product code: 290 additions and 210 deletions (500 changed lines) across +- Product code: 324 additions and 254 deletions (578 changed lines) across Core, CLI, MCP, Pi, DSH, and HTTP. -- Tests: 203 additions and 194 deletions (397 changed lines), including Core +- Tests: 218 additions and 166 deletions (384 changed lines), including Core navigation/renderer seams, adapter forwarding/validation, and OpenAPI artifact parsing. -- Documentation/configuration: 210 additions and 49 deletions (259 changed +- Documentation/configuration: 226 additions and 53 deletions (279 changed lines), including the 154-line standalone HTTP reference and aligned README, ADR, package metadata, and DSH guide. -- Total: 703 additions and 425 deletions (1,128 changed lines), within the +- Total: 768 additions and 473 deletions (1,241 changed lines), within the spec estimate of 780–1,400 total changed lines. Product code is modestly above its 250–450 estimate because each owned adapter now performs explicit - input normalization and legacy-field rejection at its boundary. + input normalization and legacy-field rejection at its boundary; the review + repair remains a small local delta. ## Remaining concerns From 19b63287601c5c7472cb85cf0dae1080513b212d Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 16:03:00 +0800 Subject: [PATCH 12/14] fix(core): guard automatic tree without headings --- CONTEXT.md | 7 ++++--- docs/http-service.md | 12 +++++++----- packages/web-core/src/markdown.ts | 6 +++++- packages/web-core/test/markdown.test.ts | 19 ++++++++++++++----- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 848c39b..e7fdedd 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -33,9 +33,10 @@ _Avoid_: Backend-specific renderer labels, automatic fallback **Page Navigation**: The shared page-reading module owns its fixed 5,000-character policy. A long, -unsectioned request returns a navigation tree; `full: true` returns complete -Markdown, and a tree's `section_id` retrieves one section. `full: true` and -`section_id` are mutually exclusive. +unsectioned request with navigable headings returns a navigation tree; a +headingless long document uses the normal bounded response. `full: true` +returns complete Markdown, and a tree's `section_id` retrieves one section. +`full: true` and `section_id` are mutually exclusive. _Avoid_: Caller-selected tree thresholds, public `tree` controls **Release Contract**: diff --git a/docs/http-service.md b/docs/http-service.md index 3d6dc69..0eb78de 100644 --- a/docs/http-service.md +++ b/docs/http-service.md @@ -72,11 +72,13 @@ public request value. Browser rendering is never selected automatically, and the service does not fall back between renderers. The shared module owns the 5,000-character automatic-tree policy. A non-full, -unsectioned document longer than that threshold returns `mode: "tree"` with -stable section IDs. Use one of those IDs in a subsequent request to retrieve a -section. `full: true` returns the complete extracted Markdown without the Core -content limit. A short or unsectioned result uses `mode: "full"`; a section -request uses `mode: "section"`. +unsectioned document longer than that threshold with navigable headings returns +`mode: "tree"` with stable section IDs. Use one of those IDs in a subsequent +request to retrieve a section. A long document without headings uses the normal +bounded `mode: "full"` response because it has no section to navigate. The +`full: true` option returns the complete extracted Markdown without the Core +content limit. A short result uses `mode: "full"`; a section request uses +`mode: "section"`. Response `200`: diff --git a/packages/web-core/src/markdown.ts b/packages/web-core/src/markdown.ts index 7f3028a..ccea28c 100644 --- a/packages/web-core/src/markdown.ts +++ b/packages/web-core/src/markdown.ts @@ -47,7 +47,11 @@ export function renderMarkdown( } const charCount = Array.from(source).length; - if (!options.full && charCount > DEFAULT_TREE_THRESHOLD) + if ( + !options.full && + charCount > DEFAULT_TREE_THRESHOLD && + headings.length > 0 + ) return { content: renderTree(source, headings), mode: "tree" }; return { content: options.full ? source : truncateContent(source), diff --git a/packages/web-core/test/markdown.test.ts b/packages/web-core/test/markdown.test.ts index a8c5bd5..d41c8bd 100644 --- a/packages/web-core/test/markdown.test.ts +++ b/packages/web-core/test/markdown.test.ts @@ -72,11 +72,6 @@ describe("Markdown navigation", () => { content: "plain content", mode: "full", }); - expect(renderMarkdown("x".repeat(5001))).toEqual({ - content: "(no headings)\n", - mode: "tree", - }); - const complete = "# H\n\n" + "x".repeat(30_001); expect(renderMarkdown(complete, { full: true })).toEqual({ content: complete, @@ -98,4 +93,18 @@ describe("Markdown navigation", () => { mode: "section", }); }); + + it("keeps headingless long documents bounded and selectable only by headings", () => { + const source = "x".repeat(5001); + expect(renderMarkdown(source)).toEqual({ + content: source, + mode: "full", + }); + + const complete = "x".repeat(30_001); + expect(renderMarkdown(complete, { full: true })).toEqual({ + content: complete, + mode: "full", + }); + }); }); From c15a50a473800c628fbb22b3d05419d71c4bcdf5 Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 16:04:28 +0800 Subject: [PATCH 13/14] chore(core): record heading guard repair --- .../http-web-service/implementation-report.md | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/.scratch/http-web-service/implementation-report.md b/.scratch/http-web-service/implementation-report.md index 3ff1fab..364ea8e 100644 --- a/.scratch/http-web-service/implementation-report.md +++ b/.scratch/http-web-service/implementation-report.md @@ -5,7 +5,7 @@ - Repository: `guionai/web` - Branch: `http-web-service` - Fixed point: `9fff0766001f5ebcad91045c34958af72841da09` -- Implementation commits: `bb4900a5dba9b5d71fdd64363ce860953b2dc098` (`feat(http): unify page-reading contract`) and `90de2fa9d77a6dc2ac8c34284f469e1d722e3d12` (`fix(http): repair navigation and renderer validation`) +- Implementation commits: `bb4900a5dba9b5d71fdd64363ce860953b2dc098` (`feat(http): unify page-reading contract`), `90de2fa9d77a6dc2ac8c34284f469e1d722e3d12` (`fix(http): repair navigation and renderer validation`), and `19b63287601c5c7472cb85cf0dae1080513b212d` (`fix(core): guard automatic tree without headings`) - Delivery boundary: the complete `http-web-service` spec and tickets 01 and 02. Code review and deployment were excluded. @@ -21,9 +21,11 @@ seam first, then every adapter, HTTP/OpenAPI contract, tests, and documentation. HTTP forbids `waitMs`. Capability errors suggest `render: "browser"` with `waitMs: 2000`. - Public `tree` and `tree_threshold` inputs were removed. The module owns the - fixed 5,000-character policy: non-full, unsectioned long content returns a - navigation tree, `full: true` returns complete extracted Markdown without the - Core 30,000-character truncation, and `section_id` retrieves one tree section. + fixed 5,000-character policy: non-full, unsectioned long content with + navigable headings returns a navigation tree, while headingless long content + uses the normal bounded response. `full: true` returns complete extracted + Markdown without the Core 30,000-character truncation, and `section_id` + retrieves one tree section. - `full: true` with `section_id` is rejected. Core validation also rejects unknown legacy fields and invalid navigation values. - Core tests use injected cache, HTTP, and browser seams; no live provider or @@ -57,7 +59,8 @@ seam first, then every adapter, HTTP/OpenAPI contract, tests, and documentation. ## Review-fix batch -The post-review repairs were applied together in `90de2fa`: +The post-review repairs were applied in the scoped commits `90de2fa` and +`19b6328`: - Core tree output for an H1-only long document emits its deterministic heading ID, and the behavior test uses that ID to retrieve the section. @@ -65,6 +68,9 @@ The post-review repairs were applied together in `90de2fa`: the `agent-browser` name for operator implementation/setup wording. - Pi and DSH each own a local `validateRenderOptions` helper; no cross-adapter abstraction was added. +- Core now enters automatic tree mode only when headings exist. A headingless + long document stays in bounded `mode: "full"`, while `full: true` still + returns the complete content; the Core behavior test covers both paths. ## Verification @@ -88,6 +94,13 @@ guionai-web-http-smoke .` passed; a disposable container returned HTTP 400 No live credentials were used. The Docker smoke used a disposable container and a non-secret placeholder startup key; it was stopped after verification. +The final Core edge repair was additionally verified with: + +- `pnpm exec vitest run packages/web-core/test/markdown.test.ts` — 4 tests + passed. +- `pnpm typecheck`, `pnpm format:check`, and explicit Prettier checks for the + changed Core source/test files and HTTP/CONTEXT documentation. + The review-fix commit was additionally verified with: - `pnpm typecheck`. @@ -106,15 +119,15 @@ The review-fix commit was additionally verified with: Against the fixed point, excluding generated `dist` output, lockfiles, and this report: -- Product code: 324 additions and 254 deletions (578 changed lines) across +- Product code: 328 additions and 254 deletions (582 changed lines) across Core, CLI, MCP, Pi, DSH, and HTTP. -- Tests: 218 additions and 166 deletions (384 changed lines), including Core +- Tests: 227 additions and 166 deletions (393 changed lines), including Core navigation/renderer seams, adapter forwarding/validation, and OpenAPI artifact parsing. -- Documentation/configuration: 226 additions and 53 deletions (279 changed +- Documentation/configuration: 229 additions and 53 deletions (282 changed lines), including the 154-line standalone HTTP reference and aligned README, ADR, package metadata, and DSH guide. -- Total: 768 additions and 473 deletions (1,241 changed lines), within the +- Total: 784 additions and 473 deletions (1,257 changed lines), within the spec estimate of 780–1,400 total changed lines. Product code is modestly above its 250–450 estimate because each owned adapter now performs explicit input normalization and legacy-field rejection at its boundary; the review From 6bd1a8ccc3e53a2c33c06829ae2e43039b7520a7 Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 2 Sep 2026 16:29:07 +0800 Subject: [PATCH 14/14] docs(page-reading): align headingless navigation guidance --- README.md | 9 +++++---- packages/dsh-web/README.md | 7 ++++--- packages/pi-web/src/tool.ts | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f7105ee..c43369f 100644 --- a/README.md +++ b/README.md @@ -114,10 +114,11 @@ web sgraph --count 10 -- "repo:^github\\.com/nodejs/node$ AbortSignal" ``` Use `--` before a search or Sourcegraph query that begins with a hyphen. `fetch` -supports `--full` and `--section`; long extracted documents automatically return -a heading tree so a later request can retrieve a stable `section_id`. `--full` -returns the complete extracted Markdown, and `--full` cannot be combined with -`--section`. `links` lists up to 100 unique HTTP(S) anchors from the original +supports `--full` and `--section`; long extracted documents with navigable +headings automatically return a heading tree so a later request can retrieve a +stable `section_id`. A headingless long document uses the normal bounded +response. `--full` returns the complete extracted Markdown, and `--full` cannot +be combined with `--section`. `links` lists up to 100 unique HTTP(S) anchors from the original page DOM. ## MCP diff --git a/packages/dsh-web/README.md b/packages/dsh-web/README.md index ea83de8..e870ee1 100644 --- a/packages/dsh-web/README.md +++ b/packages/dsh-web/README.md @@ -43,9 +43,10 @@ so it includes navigation and other links that readable-content extraction drops It uses the same HTTP default and explicit `render: "browser"` / required `waitMs` contract as `web_fetch`. -Long `web_fetch` documents return a navigation tree automatically. Use -`full: true` for complete Markdown or pass a returned `section_id` to continue -with one section; those fields are mutually exclusive. +Long `web_fetch` documents with navigable headings return a navigation tree +automatically. A headingless long document uses the normal bounded response. +Use `full: true` for complete Markdown or pass a returned `section_id` to +continue with one section; those fields are mutually exclusive. Rendered requests are bounded and constrained to the requested hostname, `*.` (the target and its subdomains), and this fixed common diff --git a/packages/pi-web/src/tool.ts b/packages/pi-web/src/tool.ts index 61508b0..3e25e00 100644 --- a/packages/pi-web/src/tool.ts +++ b/packages/pi-web/src/tool.ts @@ -205,7 +205,7 @@ const SEARCH_PROMPT_GUIDELINES = [ "Use web_search to search the web for current facts.", ]; const FETCH_PROMPT_GUIDELINES = [ - "Use web_fetch to read a web page; long documents return a navigation tree automatically, so follow up with full: true or a returned section_id.", + "Use web_fetch to read a web page; long documents with navigable headings return a navigation tree automatically, so follow up with full: true or a returned section_id. Headingless long documents use the normal bounded response.", 'web_fetch uses HTTP fetching by default; set render: "http" explicitly when desired.', 'For a client-rendered or SPA page, or after javascript_rendering_may_be_required, retry explicitly with render: "browser" and waitMs: 2000 only when the host provides browser capability. Increase waitMs explicitly or abandon an incomplete page; there is no automatic fallback.', "Never send waitMs with HTTP rendering. The browser is a host capability, not a package dependency.", @@ -435,7 +435,7 @@ export function webFetchTool(dependencies: WebToolDependencies = {}) { name: "web_fetch", label: "Web fetch", description: - "Fetch and read an HTTP or HTTPS web page as Markdown, with HTTP rendering by default or explicit browser rendering for client-rendered pages. Browser rendering requires waitMs 0 through 30000. Long documents return a navigation tree unless full or section_id is requested. Text output is limited to 2,000 lines or 50KB; truncated output is saved to a temporary file.", + "Fetch and read an HTTP or HTTPS web page as Markdown, with HTTP rendering by default or explicit browser rendering for client-rendered pages. Browser rendering requires waitMs 0 through 30000. Long documents with navigable headings return a navigation tree unless full or section_id is requested; headingless long documents use the normal bounded response. Text output is limited to 2,000 lines or 50KB; truncated output is saved to a temporary file.", promptSnippet: "Fetch a web page with web_fetch", promptGuidelines: FETCH_PROMPT_GUIDELINES, parameters: webFetchSchema,