diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9af656f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +.git +.scratch +node_modules +**/node_modules +**/dist +coverage +*.tgz +.env* diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e72065b..172eb0b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -63,6 +63,7 @@ jobs: path: | packages/web/package.json packages/web/dist + packages/web/dist/openapi.yaml packages/pi-web/package.json packages/pi-web/dist packages/dsh-web/package.json @@ -161,8 +162,41 @@ jobs: working-directory: packages/${{ matrix.directory }} run: npm publish --no-git-checks --access public --provenance --tag "${{ steps.version.outputs.dist-tag }}" + container: + needs: preflight + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup Node + uses: actions/setup-node@v7 + with: + node-version: 24 + + - name: Synchronize image package version + run: node scripts/sync-version.mjs "${GITHUB_REF_NAME#v}" + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Build and publish immutable release image + env: + IMAGE_NAME: ghcr.io/${{ github.repository }} + run: | + image="${IMAGE_NAME,,}:${GITHUB_REF_NAME}" + docker build --tag "$image" . + docker push "$image" + github-release: - needs: publish + needs: [publish, container] runs-on: ubuntu-latest permissions: contents: write @@ -171,3 +205,14 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: gh release create "$GITHUB_REF_NAME" --generate-notes --repo "$GITHUB_REPOSITORY" + + - name: Download generated OpenAPI artifact + uses: actions/download-artifact@v5 + with: + name: release-dist + path: packages + + - name: Attach generated OpenAPI contract + env: + GH_TOKEN: ${{ github.token }} + run: gh release upload "$GITHUB_REF_NAME" packages/web/dist/openapi.yaml#openapi.yaml --repo "$GITHUB_REPOSITORY" diff --git a/.scratch/defered/public-http-service-security.md b/.scratch/defered/public-http-service-security.md new file mode 100644 index 0000000..9ebcf91 --- /dev/null +++ b/.scratch/defered/public-http-service-security.md @@ -0,0 +1,16 @@ +# Public HTTP service hardening + +Before Guion Web's container is presented as a public or multi-tenant service, define and implement the security boundary for outbound requests and rendered browsing. The current first release is a Personal Web Service only. + +Required decisions and work: + +- Block private, loopback, link-local, and otherwise non-public network targets across redirects and DNS resolution, including DNS rebinding. +- Run `agent-browser` in an isolated browser/process and restrict its network egress consistently with direct fetch. +- Set resource and concurrency limits appropriate for a remotely reachable renderer. +- Define the authentication and tenant boundary when the service stops being personal. + +This deferred work does not change the v1 search policy: the Personal Web +Service tries the server-configured Bridge Route first, preserves successful +empty results, and falls back exactly once to the server-configured Exa +credential only for a non-cancellation Bridge failure. Typed Bridge Data +Operations report Bridge failure directly. diff --git a/.scratch/http-web-service/implementation-report.md b/.scratch/http-web-service/implementation-report.md new file mode 100644 index 0000000..364ea8e --- /dev/null +++ b/.scratch/http-web-service/implementation-report.md @@ -0,0 +1,141 @@ +# Guion Web page-reading contract implementation report + +## Scope + +- Repository: `guionai/web` +- Branch: `http-web-service` +- Fixed point: `9fff0766001f5ebcad91045c34958af72841da09` +- Implementation commits: `bb4900a5dba9b5d71fdd64363ce860953b2dc098` (`feat(http): unify page-reading contract`), `90de2fa9d77a6dc2ac8c34284f469e1d722e3d12` (`fix(http): repair navigation and renderer validation`), and `19b63287601c5c7472cb85cf0dae1080513b212d` (`fix(core): guard automatic tree without headings`) +- Delivery boundary: the complete `http-web-service` spec and tickets 01 and 02. + Code review and deployment were excluded. + +The tickets were implemented in dependency order: the shared Core page-reading +seam first, then every adapter, HTTP/OpenAPI contract, tests, and documentation. + +## Ticket outcomes + +### 01 — Unify the page-reading module interface + +- Core Fetch and Links now accept only `render: "http" | "browser"`. + HTTP is the default; browser requires `waitMs` from 0 through 30,000, while + HTTP forbids `waitMs`. Capability errors suggest `render: "browser"` with + `waitMs: 2000`. +- Public `tree` and `tree_threshold` inputs were removed. The module owns the + fixed 5,000-character policy: non-full, unsectioned long content with + navigable headings returns a navigation tree, while headingless long content + uses the normal bounded response. `full: true` returns complete extracted + Markdown without the Core 30,000-character truncation, and `section_id` + retrieves one tree section. +- `full: true` with `section_id` is rejected. Core validation also rejects + unknown legacy fields and invalid navigation values. +- Core tests use injected cache, HTTP, and browser seams; no live provider or + browser state is required. +- H1-only long documents now list their H1 as a selectable tree node with a + stable `section_id`; a Core behavior test retrieves the emitted ID and checks + the complete section content. + +### 02 — Align adapters, HTTP, and reference documentation + +- CLI flags/help, stdio MCP schemas, Pi TypeBox schemas/prompts, DSH tool + definitions, and HTTP schemas now expose the same renderer and navigation + vocabulary without compatibility aliases. +- HTTP remains exactly `/v1/search`, `/v1/fetch`, and `/v1/links`. Search keeps + server-selected Kepos Bridge-first behavior with one Exa retry; Fetch and + Links enforce the shared renderer rules. +- The build generates the version-matched OpenAPI 3.1 artifact at + `packages/web/dist/openapi.yaml`; the release artifact test parses a + test-owned generated file and asserts exactly the three retained routes and + the unified request schemas. +- Added [`docs/http-service.md`](../../docs/http-service.md), a standalone + human-readable reference covering requests, responses, validation, provider + behavior, renderer behavior, errors, and the OpenAPI release asset. README + and ADR 0001 link to and describe the same contract. Operator setup may still + name the installed browser executable. +- CONTEXT.md now uses the public `render: "http" | "browser"` vocabulary for + glossary and HTTP guidance; `agent-browser` appears only as an operator + implementation/setup detail. Pi and DSH each consolidate their repeated + renderer/wait checks in one local helper while preserving validation order, + messages, and forwarding behavior. + +## Review-fix batch + +The post-review repairs were applied in the scoped commits `90de2fa` and +`19b6328`: + +- Core tree output for an H1-only long document emits its deterministic heading + ID, and the behavior test uses that ID to retrieve the section. +- CONTEXT.md describes the public HTTP/browser renderer contract and reserves + the `agent-browser` name for operator implementation/setup wording. +- Pi and DSH each own a local `validateRenderOptions` helper; no + cross-adapter abstraction was added. +- Core now enters automatic tree mode only when headings exist. A headingless + long document stays in bounded `mode: "full"`, while `full: true` still + returns the complete content; the Core behavior test covers both paths. + +## Verification + +All checks completed successfully against the implementation commit: + +- `pnpm format:check` and explicit Prettier checks for the ADR, DSH README, and + standalone HTTP reference. +- `pnpm typecheck`. +- `pnpm test` — 19 test files and 117 tests passed. The existing DSH source-map + warning was non-fatal. +- `pnpm build` — all workspace packages built and generated + `packages/web/dist/openapi.yaml`. +- `pnpm test:release` — release version synchronization fixtures passed. +- `pnpm test:pack` — packed-installation/host-loading checks passed for + `@guionai/web`, `@guionai/pi-web`, and `@guionai/dsh-web`. +- Docker validation with the supported image: `docker build --tag +guionai-web-http-smoke .` passed; a disposable container returned HTTP 400 + for invalid Fetch input and HTTP 200 for a real browser-rendered + `https://example.com` Fetch with `full: true`. + +No live credentials were used. The Docker smoke used a disposable container +and a non-secret placeholder startup key; it was stopped after verification. + +The final Core edge repair was additionally verified with: + +- `pnpm exec vitest run packages/web-core/test/markdown.test.ts` — 4 tests + passed. +- `pnpm typecheck`, `pnpm format:check`, and explicit Prettier checks for the + changed Core source/test files and HTTP/CONTEXT documentation. + +The review-fix commit was additionally verified with: + +- `pnpm typecheck`. +- `pnpm format:check` and explicit Prettier checks for the changed source files, + CONTEXT.md, and the HTTP reference. +- `pnpm test` — 19 test files and 118 tests passed. The existing DSH source-map + warning was non-fatal. +- `pnpm build` — all workspace packages built and generated + `packages/web/dist/openapi.yaml`. +- `pnpm test:release` — release version synchronization fixtures passed. +- `pnpm test:pack` — packed-installation/host-loading checks passed for + `@guionai/web`, `@guionai/pi-web`, and `@guionai/dsh-web`. + +## Changed paths and size + +Against the fixed point, excluding generated `dist` output, lockfiles, and +this report: + +- Product code: 328 additions and 254 deletions (582 changed lines) across + Core, CLI, MCP, Pi, DSH, and HTTP. +- Tests: 227 additions and 166 deletions (393 changed lines), including Core + navigation/renderer seams, adapter forwarding/validation, and OpenAPI + artifact parsing. +- Documentation/configuration: 229 additions and 53 deletions (282 changed + lines), including the 154-line standalone HTTP reference and aligned README, + ADR, package metadata, and DSH guide. +- Total: 784 additions and 473 deletions (1,257 changed lines), within the + spec estimate of 780–1,400 total changed lines. Product code is modestly + above its 250–450 estimate because each owned adapter now performs explicit + input normalization and legacy-field rejection at its boundary; the review + repair remains a small local delta. + +## Remaining concerns + +- Public or multi-tenant HTTP hardening remains out of scope as documented by + ADR 0001 and the existing deferred security note. +- Code review and deployment were intentionally not run for this implementation + task. diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..e7fdedd --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,52 @@ +# Guion Web + +Guion Web provides read-only research operations through local integrations and a self-hosted HTTP service. This glossary keeps the interfaces and their deployment boundary distinct. + +## Language + +**Personal Web Service**: +A self-hosted, single-trust-boundary deployment of Guion Web for its operator and their agents. It is not a public or multi-tenant hosted service. +_Avoid_: Public service, SaaS + +**Research Operation**: +One of Guion Web's read-only capabilities: Search, Fetch, Links, Documentation Resolve or Fetch, or Public-source Search. It is independent of the transport that exposes it. +_Avoid_: Tool, endpoint + +**Search**: +The general web-retrieval Research Operation. In the Personal Web Service, it uses Kepos Bridge first and transparently falls back to Exa only when the Bridge is unavailable; its response identifies the provider that supplied results. +_Avoid_: Bridge search, provider-selected search + +**Bridge Data Operation**: +A typed, Bridge-only lookup for weather, sports, finance, or time. It has no Exa fallback because Exa search is not an equivalent result source. +_Avoid_: Generic Bridge command, special search + +**Bridge Route**: +The server-local URL used by the Kepos Bridge search provider. The service operator configures it; API callers never supply it. +_Avoid_: Bridge URL parameter + +**Page Rendering**: +Fetch and Links use `render: "http"` by default or explicit `render: "browser"`. +Browser rendering requires `waitMs` from 0 through 30,000 and is never selected +automatically. The operator-installed `agent-browser` runtime is an implementation +and setup detail, not an adapter-facing request value. +_Avoid_: Backend-specific renderer labels, automatic fallback + +**Page Navigation**: +The shared page-reading module owns its fixed 5,000-character policy. A long, +unsectioned request with navigable headings returns a navigation tree; a +headingless long document uses the normal bounded response. `full: true` +returns complete Markdown, and a tree's `section_id` retrieves one section. +`full: true` and `section_id` are mutually exclusive. +_Avoid_: Caller-selected tree thresholds, public `tree` controls + +**Release Contract**: +The versioned public distribution of Guion Web: its npm packages, GHCR container image, and the generated `openapi.yaml` attached to the matching GitHub Release. +_Avoid_: Checked-in OpenAPI file, independently versioned schema + +**HTTP Service**: +The Hono-based `/v1` JSON API shipped by `web serve` and the GHCR image. It +uses server-local credentials and Bridge Route configuration; clients do not +select providers or submit a generic Bridge command. Its page-reading routes use +the same `render: "http" | "browser"`, `full`, and `section_id` contract; the +browser executable name appears only in operator setup. +_Avoid_: Remote MCP, public service diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a78f982 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# Guion Web's supported HTTP-service image. The build stage bundles the web +# executable; the runtime stage adds the optional rendered-fetch capability. +FROM node:24-bookworm-slim AS build + +WORKDIR /workspace +RUN corepack enable +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY packages/web-core/package.json packages/web-core/package.json +COPY packages/web/package.json packages/web/package.json +COPY packages/pi-web/package.json packages/pi-web/package.json +COPY packages/dsh-web/package.json packages/dsh-web/package.json +RUN pnpm install --frozen-lockfile +COPY . . +RUN pnpm --filter @guionai/web run build + +FROM node:24-bookworm + +ENV NODE_ENV=production +# Chrome for Testing has no Linux ARM64 distribution. Debian's Chromium works +# on both released container architectures, and agent-browser documents this +# executable override for containers. +ENV AGENT_BROWSER_EXECUTABLE_PATH=/usr/bin/chromium +WORKDIR /app + +# Rendering is deliberately explicit at the HTTP contract. The executable and +# browser runtime are image capabilities, while credentials remain env-only. +ARG AGENT_BROWSER_VERSION=0.36.0 +RUN apt-get update \ + && apt-get install --yes --no-install-recommends chromium \ + && rm -rf /var/lib/apt/lists/* \ + && npm install --global agent-browser@${AGENT_BROWSER_VERSION} + +COPY --from=build /workspace/packages/web/dist ./dist + +EXPOSE 8787 +ENTRYPOINT ["node", "/app/dist/cli.js", "serve"] diff --git a/README.md b/README.md index a34fec1..c43369f 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,15 @@ Guion Web is a Node.js web research toolkit. It provides Exa, Brave, or a managed Kepos Bridge search endpoint, Context7 library documentation lookup, Sourcegraph public code search, page-link -discovery, and two page-fetch backends through a CLI, stdio MCP server, Pi extension, and DeepSeek -Harness (DSH) integration: direct HTML-to-Markdown extraction and explicit -`agent-browser` rendering for client-rendered pages on supported hosts. +discovery, and two page-rendering modes through a CLI, stdio MCP server, personal +HTTP service, Pi extension, and DeepSeek Harness (DSH) integration: HTTP +HTML-to-Markdown extraction and explicit browser rendering for client-rendered +pages on supported hosts. ## Install and configure -Node.js 20 or later is required. `@guionai/web` intentionally exposes only -its `web` executable and stdio MCP server; it does not provide a root +Node.js 20 or later is required. `@guionai/web` exposes its `web` executable, +stdio MCP server, and personal HTTP service; it does not provide a root JavaScript or TypeScript SDK. Use the Pi or DSH packages for those host integrations. @@ -27,18 +28,75 @@ default route unless it runs in DSH, whose live settings card can override the route. Context7 works anonymously when its key is absent. +The HTTP service always requires a non-empty `EXA_API_KEY`: it tries the +server-local Kepos Bridge route first and retries Exa once when Bridge fails. +HTTP clients cannot select a provider, pass credentials, or override the Bridge +route per request. Set `KEPOS_BRIDGE_ENDPOINT` to replace the default route +(`http://codex-bridge.localhost:17480/codex/web-search`); it must be a complete +HTTP(S) URL without credentials, query, or fragment. + ```bash export EXA_API_KEY="..." # or export BRAVE_API_KEY="..." # optional, for authenticated Context7 requests export CONTEXT7_API_KEY="..." +# optional complete Bridge route for `web serve` +export KEPOS_BRIDGE_ENDPOINT="http://127.0.0.1:8787/codex/web-search" ``` Do not put credentials in command arguments or commit them. The CLI reads these environment variables directly; it does not load a dotenv file or an older application configuration path. +## Personal HTTP service + +Run the service with the server-local environment above: + +```bash +web serve --host 0.0.0.0 --port 8787 +# or use the published image +docker run --rm -p 8787:8787 \ + -e EXA_API_KEY="$EXA_API_KEY" \ + -e KEPOS_BRIDGE_ENDPOINT="http://host.docker.internal:17480/codex/web-search" \ + ghcr.io/guionai/web:v0.1.0 +``` + +Every HTTP operation is a versioned JSON `POST` route. Request and response schemas +are generated into `openapi.yaml` from the same route definitions: + +| Route | Request | Purpose | +| ------------ | --------------------------------------------------------- | -------------------------------------------------------- | +| `/v1/search` | `{ "query": "..." }` | Kepos Bridge search with one Exa retry on Bridge failure | +| `/v1/fetch` | `{ "url", "section_id?", "full?", "render?", "waitMs?" }` | Fetch Markdown | +| `/v1/links` | `{ "url", "limit?", "render?", "waitMs?" }` | List page HTTP(S) links | + +The complete human-readable contract is in the [HTTP service reference](docs/http-service.md). + +Search keeps a successful empty Bridge result, retries Exa exactly once for a +non-cancellation Bridge failure, and reports the provider in its response. +Weather, sports, finance, and time are not exposed because the configured +providers do not offer contract-equivalent official typed data APIs. Invalid JSON +bodies, unknown fields, and invalid typed values are rejected before an upstream call. +Upstream failures are bounded JSON errors and never include credentials or raw +provider response bodies. +Error responses use a stable `{ "code", "message", "details"? }` JSON shape; +upstream failures use 502 (or 504 for an upstream timeout), while client +cancellation is reported as 499. + +Fetch and Links use HTTP rendering when `render` is omitted (or set to +`"http"`). Browser rendering is explicit and requires both `render: "browser"` +and an integer `waitMs` from 0 through 30,000; HTTP rendering never silently +switches backends. The container installs the `agent-browser` executable with +Debian Chromium (including Linux ARM64, where Chrome for Testing has no build), +while credentials and Bridge configuration remain server-local environment +variables. + +This is a Personal Web Service: a single-trust-boundary deployment for its +operator and agents. It is not hardened for public or multi-tenant exposure; +SSRF/egress isolation, browser sandboxing, quotas, and authentication remain +deferred in `.scratch/defered/public-http-service-security.md`. + ## CLI `web` has human-readable output by default. Add `--json` for exactly one JSON @@ -47,7 +105,7 @@ document on stdout, which is useful for automation. ```bash web search --provider exa -- "Node AbortSignal" web search --provider kepos-bridge -- "Node AbortSignal" -web fetch https://example.com/article --tree +web fetch https://example.com/article web fetch https://example.com/article --section introduction web links https://example.com/article --limit 50 web docs resolve react @@ -56,9 +114,12 @@ web sgraph --count 10 -- "repo:^github\\.com/nodejs/node$ AbortSignal" ``` Use `--` before a search or Sourcegraph query that begins with a hyphen. `fetch` -supports `--full`, `--tree`, and `--section`; long extracted documents default to -a heading tree so a later request can retrieve a stable section ID. `links` lists -up to 100 unique HTTP(S) anchors from the original page DOM. +supports `--full` and `--section`; long extracted documents with navigable +headings automatically return a heading tree so a later request can retrieve a +stable `section_id`. A headingless long document uses the normal bounded +response. `--full` returns the complete extracted Markdown, and `--full` cannot +be combined with `--section`. `links` lists up to 100 unique HTTP(S) anchors from the original +page DOM. ## MCP @@ -74,7 +135,7 @@ web mcp --provider kepos-bridge The server exposes six read-only tools: `search`, `fetch`, `links`, `docs_resolve`, `docs_fetch`, and `source_search`. Its stdout is reserved for MCP protocol messages; diagnostics go to stderr. For a client-rendered page, explicitly call -`fetch` or `links` with `render: "agent-browser"` and an integer `waitMs`; this optional +`fetch` or `links` with `render: "browser"` and an integer `waitMs`; this optional retry requires a host-installed executable and never happens automatically. ## Pi @@ -88,7 +149,7 @@ pi install npm:@guionai/pi-web It registers `web_search`, `web_fetch`, `web_links`, `web_docs`, and `web_source_search` and calls the bundled core in-process. Pi and TypeBox are peer dependencies supplied by the host; no CLI executable or MCP configuration is required. `web_fetch` uses -direct fetch by default and can explicitly use `render: "agent-browser"` with +HTTP rendering by default and can explicitly use `render: "browser"` with an integer `waitMs` when its host provides that optional executable. `web_links` uses the same explicit rendering contract and lists HTTP(S) anchors from the original page DOM. @@ -113,37 +174,36 @@ credentials. Selecting Kepos Bridge additionally exposes `web_weather`, `web_sports`, `web_finance`, and `web_time`; these tools are removed when another provider is selected. Fetch, link discovery, documentation, and Sourcegraph tools also run in-process. The host DSH packages and React are peers supplied by DSH. -`web_fetch` uses direct fetch by default and can explicitly use -`render: "agent-browser"` with an integer `waitMs` on a host that supplies the +`web_fetch` uses HTTP rendering by default and can explicitly use +`render: "browser"` with an integer `waitMs` on a host that supplies the optional executable. `web_links` uses the same explicit rendering contract and lists HTTP(S) anchors from the original page DOM. -## Page-fetch backends +## Page-rendering modes -`web fetch` has two backends. `fetch` (the default) uses Node `fetch`, -`linkedom`, and Defuddle for direct HTML-to-Markdown extraction from static, -SSR, and pre-rendered pages. `agent-browser` renders client-side pages through -a separately installed host executable. Direct fetch is used by default; choose -agent-browser explicitly when needed. The implementation never falls back -automatically: +`web fetch` has two renderers. `http` (the default) uses Node `fetch`, `linkedom`, +and Defuddle for HTML-to-Markdown extraction from static, SSR, and pre-rendered +pages. `browser` renders client-side pages through the separately installed +host browser capability. HTTP rendering is used by default; choose browser +explicitly when needed. The implementation never falls back automatically: ```bash -web fetch https://example.com/app --render=agent-browser --wait=2000 +web fetch https://example.com/app --render=browser --wait=2000 # If it is still incomplete, retry explicitly with more time, or abandon it: -web fetch https://example.com/app --render=agent-browser --wait=10000 +web fetch https://example.com/app --render=browser --wait=10000 ``` -`web links` uses the same direct or explicit browser-rendered source, but parses +`web links` uses the same HTTP or explicit browser-rendered source, but parses the original DOM rather than Defuddle output so navigation and other links outside the readable article remain discoverable. It returns only HTTP(S) `a[href]` destinations, deduplicated and capped at 100 by default. -`--wait` is mandatory with `--render=agent-browser`, including `--wait=0`, and -accepts only an integer from 0 through 30,000 milliseconds. Direct `fetch` -or `links` requests must not provide `--wait`. The same `render: "agent-browser"` and required +`--wait` is mandatory with `--render=browser`, including `--wait=0`, and +accepts only an integer from 0 through 30,000 milliseconds. HTTP `fetch` +or `links` requests must not provide `--wait`. The same `render: "browser"` and required `waitMs` fields are available on the MCP `fetch`/`links`, Pi `web_fetch`/`web_links`, and DSH -`web_fetch`/`web_links` tools. A direct-fetch failure may return the structured +`web_fetch`/`web_links` tools. An HTTP-rendering failure may return the structured `javascript_rendering_may_be_required` hint with the 2,000 ms suggestion; the agent decides whether to retry with a longer wait or abandon the page. @@ -159,7 +219,7 @@ agent-browser install `agent-browser install` manages its own browser runtime; Guion packages never run it, bundle it, or reuse browser credentials. A compatible executable must be directly runnable from `PATH` without a shell. The renderer is supported on -macOS and Linux hosts. Direct fetch remains available, and the three npm +macOS and Linux hosts. HTTP rendering remains available, and the three npm packages remain installable when `agent-browser` is absent. A rendered session is fresh and non-persistent. Before launch, the target must @@ -220,9 +280,13 @@ before any publication begins. Three independent, non-fail-fast protected `npm` Environment matrix cells then publish one package each through npm Trusted Publishing with provenance. The synchronized version selects npm's `latest` tag for stable SemVer and `beta` for -a prerelease. After all three cells succeed, the workflow creates the GitHub -release with generated notes and source archives. It publishes no binaries or -platform archives. +a prerelease. A matching immutable-tagged image is published to +`ghcr.io/guionai/web:` with the `web serve` entrypoint and the +`agent-browser` runtime. After all three npm cells and the image job succeed, +the workflow creates the GitHub release with generated notes, source archives, +and the build-generated `openapi.yaml` asset. The asset is generated from the +same Hono route schemas as the image and package; it is not checked in or +versioned independently. It publishes no binaries or platform archives. If publication partially fails, use GitHub Actions **Re-run failed jobs**. Never use **Re-run all jobs**: npm versions are immutable, so the jobs that already diff --git a/docs/adr/0001-containerized-rest-api.md b/docs/adr/0001-containerized-rest-api.md new file mode 100644 index 0000000..f23e4e8 --- /dev/null +++ b/docs/adr/0001-containerized-rest-api.md @@ -0,0 +1,7 @@ +# Expose research operations through a containerized REST API + +Guion Web will add a self-hosted, single-user HTTP service in a portable container. It will expose only provider-neutral web search, page fetch, and link discovery as REST routes defined with Hono and `@hono/zod-openapi`. Each tagged release will publish the generated OpenAPI 3.1 document as a downloadable `openapi.yaml` GitHub Release asset, alongside the matching npm packages and GHCR image. The current stdio MCP server remains supported, but remote Streamable HTTP MCP is outside this first service slice. + +## Consequences + +The service's provider credentials and Bridge Route are server-local configuration; clients cannot choose a provider or supply a Bridge Route per request. Search tries Kepos Bridge and uses Exa only when the Bridge is operationally unavailable; it does not fall back for cancellation, malformed client input, or an empty result set. A failed Bridge attempt is retried through Exa exactly once, and a successful empty Bridge response is returned unchanged. Typed Bridge operations are intentionally not exposed: Exa has no equivalent official weather, sports, or time API, and its premium finance integration is not contract-compatible. Fetch and Links use `render: "http"` by default or explicit `render: "browser"` with a required `waitMs`; the operator-installed executable remains an implementation detail. The shared page-reading module owns automatic navigation trees, complete `full` extraction, and `section_id` continuation. `openapi.yaml` is generated from the release build rather than manually maintained or independently versioned; the standalone [HTTP service reference](../http-service.md) is its human-readable companion. Public or multi-tenant deployment hardening is deliberately deferred in `.scratch/defered/public-http-service-security.md`. diff --git a/docs/http-service.md b/docs/http-service.md new file mode 100644 index 0000000..0eb78de --- /dev/null +++ b/docs/http-service.md @@ -0,0 +1,156 @@ +# Guion Web personal HTTP service + +The personal service is a single-trust-boundary JSON API. It exposes exactly +three `POST` routes: `/v1/search`, `/v1/fetch`, and `/v1/links`. It is intended +for an operator and trusted agents, not for public or multi-tenant traffic. + +## Calling the service + +Send `Content-Type: application/json` and a JSON request body. Successful +responses are JSON and preserve the shared Guion page-reading vocabulary. +Unknown fields, malformed JSON, missing required fields, and values outside the +constraints below are rejected before an operation is called. + +### `POST /v1/search` + +Request: + +```json +{ "query": "AbortSignal" } +``` + +`query` is a non-empty string. The service chooses providers server-side; a +caller cannot select a provider, supply credentials, or override the Bridge +route. Search tries the server-local Kepos Bridge first. A successful empty +Bridge result is returned as-is. If Bridge fails for a non-cancellation reason, +the service retries Exa once. The service requires a non-empty `EXA_API_KEY` +at startup because Exa is the fallback provider. + +Response `200`: + +```json +{ + "provider": "Kepos Bridge", + "results": [ + { + "title": "...", + "link": "https://example.test", + "snippet": "...", + "position": 1 + } + ] +} +``` + +`provider` is `"Kepos Bridge"` or `"Exa"`; each result has string `title`, +`link`, and `snippet` fields plus an integer `position`. + +### `POST /v1/fetch` + +Request: + +```json +{ + "url": "https://example.test/article", + "render": "http", + "full": false, + "section_id": "7i" +} +``` + +`url` must be an absolute `http:` or `https:` URL. `render` is optional and +must be exactly `"http"` or `"browser"`; omission selects `"http"`. `waitMs` +is an integer from 0 through 30,000, required with `render: "browser"` and +forbidden with `render: "http"` (or when `render` is omitted). `full` is an +optional boolean. `section_id` is an optional non-empty string returned in a +navigation tree. `full: true` and `section_id` cannot be sent together. + +HTTP rendering fetches the page with Node HTTP, linkedom, and Defuddle. +Browser rendering invokes the operator-installed `agent-browser` executable +through the isolated renderer implementation; the executable name is not a +public request value. Browser rendering is never selected automatically, and +the service does not fall back between renderers. + +The shared module owns the 5,000-character automatic-tree policy. A non-full, +unsectioned document longer than that threshold with navigable headings returns +`mode: "tree"` with stable section IDs. Use one of those IDs in a subsequent +request to retrieve a section. A long document without headings uses the normal +bounded `mode: "full"` response because it has no section to navigate. The +`full: true` option returns the complete extracted Markdown without the Core +content limit. A short result uses `mode: "full"`; a section request uses +`mode: "section"`. + +Response `200`: + +```json +{ + "url": "https://example.test/article", + "mode": "full", + "content": "# Article\n...\n" +} +``` + +### `POST /v1/links` + +Request: + +```json +{ + "url": "https://example.test/article", + "limit": 50, + "render": "browser", + "waitMs": 0 +} +``` + +`url`, `render`, and `waitMs` follow the Fetch rules. `limit` is optional, +defaults to 100, and must be an integer from 1 through 100. Links are read from +the source DOM rather than Defuddle's article body, deduplicated, and limited +to HTTP(S) `a[href]` destinations. + +Response `200`: + +```json +{ + "url": "https://example.test/article", + "links": [{ "text": "Guide", "url": "https://example.test/guide" }], + "truncated": false +} +``` + +## Errors + +Every error is a bounded JSON object with this shape: + +```json +{ "code": "invalid_request", "message": "Request validation failed" } +``` + +`details` is optional and contains only safe, documented fields. The service +uses these status codes: + +| Status | Meaning | +| -----: | ------------------------------------------------------------------------ | +| `400` | Invalid JSON, unknown fields, invalid values, or renderer/wait mismatch. | +| `499` | The client cancelled the request. | +| `502` | An upstream provider, page fetch, renderer, or response contract failed. | +| `504` | An upstream operation timed out. | +| `404` | The path is not one of the three retained routes. | + +A browserless Fetch failure may use `javascript_rendering_may_be_required` and +include `details: { "retryableWithRender": true, "suggestedArguments": { +"render": "browser", "waitMs": 2000 } }`. A renderer allowlist failure uses +`render_domain_not_allowed` with `retryable: false`, the report URL, and a +validated blocked hostname when available. Increasing `waitMs` does not fix an +allowlist failure. Error responses never include credentials or raw upstream +response bodies. + +## Configuration and OpenAPI + +Credentials and the optional `KEPOS_BRIDGE_ENDPOINT` are server-local +environment variables. They are not accepted in request bodies. The route +schemas in [`packages/web/src/http.ts`](../packages/web/src/http.ts) are the +machine-readable source of truth. The build generates a version-matched +OpenAPI 3.1 artifact at `packages/web/dist/openapi.yaml`; releases attach that +file as the downloadable `openapi.yaml` asset. This document is its +human-readable companion and is not an independently versioned schema. diff --git a/packages/dsh-web/README.md b/packages/dsh-web/README.md index b357f8b..e870ee1 100644 --- a/packages/dsh-web/README.md +++ b/packages/dsh-web/README.md @@ -24,11 +24,11 @@ strings, and fragments are rejected and its path is used exactly as entered. The published package is a dual host/browser bundle. Its host and client artifacts, profile patch, and exact DSH `0.1.2-alpha.3` peer contract are included -in the npm package. Search, direct page fetch, page-link discovery, optional agent-browser rendering, +in the npm package. Search, HTTP page rendering, page-link discovery, optional browser rendering, Context7 documentation, and Sourcegraph all run in-process through the bundled -Guion Web core. `web_fetch` has two page-fetch backends: direct fetch (the -default) and explicit `render: "agent-browser"` with required `waitMs` (an -integer from 0 through 30,000) for client-rendered pages through a host-installed +Guion Web core. `web_fetch` has two page-rendering modes: HTTP (the default) and +explicit `render: "browser"` with required `waitMs` (an integer from 0 through +30,000) for client-rendered pages through a host-installed `agent-browser` executable. To enable that optional capability, install [agent-browser](https://github.com/vercel-labs/agent-browser) separately with @@ -40,9 +40,14 @@ state or credentials. `web_links` lists up to 100 unique HTTP(S) anchors from the original page DOM, so it includes navigation and other links that readable-content extraction drops. -It uses the same direct default and explicit `render: "agent-browser"` / required +It uses the same HTTP default and explicit `render: "browser"` / required `waitMs` contract as `web_fetch`. +Long `web_fetch` documents with navigable headings return a navigation tree +automatically. A headingless long document uses the normal bounded response. +Use `full: true` for complete Markdown or pass a returned `section_id` to +continue with one section; those fields are mutually exclusive. + Rendered requests are bounded and constrained to the requested hostname, `*.` (the target and its subdomains), and this fixed common CDN list: `cdn.jsdelivr.net`, `unpkg.com`, `cdnjs.cloudflare.com`, @@ -50,7 +55,7 @@ CDN list: `cdn.jsdelivr.net`, `unpkg.com`, `cdnjs.cloudflare.com`, The caller cannot widen the list. A redirect, API, frame, worker, socket, or other dependency on an unknown domain fails closed as `render_domain_not_allowed`; increasing `waitMs` will not help. For example, -retry a shell with `render: "agent-browser", waitMs: 2000`, then explicitly +retry a shell with `render: "browser", waitMs: 2000`, then explicitly retry with a longer wait such as `waitMs: 10000` or abandon the page. Report a likely missing first-party or common-CDN domain at https://github.com/guionai/web/issues/new, including the page URL and blocked diff --git a/packages/dsh-web/package.json b/packages/dsh-web/package.json index 5741972..34d4c71 100644 --- a/packages/dsh-web/package.json +++ b/packages/dsh-web/package.json @@ -1,7 +1,7 @@ { "name": "@guionai/dsh-web", "version": "0.1.0", - "description": "DeepSeek Harness 0.1.2-alpha.3 provider for Exa, Brave, and Kepos Bridge web research with direct and optional agent-browser page-fetch backends.", + "description": "DeepSeek Harness 0.1.2-alpha.3 provider for Exa, Brave, and Kepos Bridge web research with HTTP and optional browser page-rendering modes.", "homepage": "https://github.com/guionai/web#readme", "bugs": { "url": "https://github.com/guionai/web/issues" diff --git a/packages/dsh-web/src/client.ts b/packages/dsh-web/src/client.ts index f12941f..d39d111 100644 --- a/packages/dsh-web/src/client.ts +++ b/packages/dsh-web/src/client.ts @@ -290,11 +290,11 @@ function toolSummary( export function fetchDetails( args: Record, ): Array<{ label: string; value: string }> { - const browserRendered = args.render === "agent-browser"; + const browserRendered = args.render === "browser"; const details = [ { label: "Backend", - value: browserRendered ? "Browser rendered" : "Direct fetch", + value: browserRendered ? "Browser rendered" : "HTTP rendered", }, ]; if (browserRendered && typeof args.waitMs === "number") { @@ -313,7 +313,6 @@ function formatWait(waitMs: number): string { function fetchResultMode(args: Record): string { if (typeof args.section_id === "string" && args.section_id !== "") return `Section: ${args.section_id}`; - if (args.tree === true) return "Heading tree"; if (args.full === true) return "Full document"; return "Adaptive document"; } diff --git a/packages/dsh-web/src/tools.ts b/packages/dsh-web/src/tools.ts index ea48aae..2c5acc5 100644 --- a/packages/dsh-web/src/tools.ts +++ b/packages/dsh-web/src/tools.ts @@ -9,6 +9,7 @@ import { type DocsFetchResult, type DocsResolveResult, type DocsToolInput, + type FetchInput, type FetchResult, type LinksInput, type LinksResult, @@ -26,8 +27,6 @@ import { defineTool, type ToolDefinition } from "@deepseek-ai/dsh-tools"; import { CONTEXT7_CREDENTIAL_REF } from "./contract.js"; -const DEFAULT_TREE_THRESHOLD = 5000; - export interface WebToolDependencies { credentials: { resolve(ref: CredentialRef): Promise; @@ -43,7 +42,6 @@ const fetchParameters = { required: true, description: "HTTP or HTTPS URL to fetch", }, - tree: { type: "boolean", description: "Show the page heading tree" }, section_id: { type: "string", description: "Optional heading section ID to return", @@ -52,20 +50,14 @@ const fetchParameters = { type: "boolean", description: "Return full content without automatic tree mode", }, - tree_threshold: { - type: "integer", - default: DEFAULT_TREE_THRESHOLD, - description: "Automatic tree threshold; defaults to 5000", - }, render: { type: "string", - enum: ["fetch", "agent-browser"], - description: "Page-fetch backend; defaults to direct fetch", + enum: ["http", "browser"], + description: "Page renderer; defaults to HTTP", }, waitMs: { type: "integer", - description: - "Required post-load wait for agent-browser rendering (0-30000)", + description: "Required post-load wait for browser rendering (0-30000)", }, } as const; @@ -82,13 +74,12 @@ const linksParameters = { }, render: { type: "string", - enum: ["fetch", "agent-browser"], - description: "Page-fetch backend; defaults to direct fetch", + enum: ["http", "browser"], + description: "Page renderer; defaults to HTTP", }, waitMs: { type: "integer", - description: - "Required post-load wait for agent-browser rendering (0-30000)", + description: "Required post-load wait for browser rendering (0-30000)", }, } as const; @@ -221,7 +212,7 @@ const fetchOutput = { type: "text" as const, text: boundedToolText( value.content, - "Use web_fetch with tree or section_id to navigate the document.", + "Use web_fetch with full: true or a returned section_id to navigate the document.", ), }, ], @@ -378,33 +369,73 @@ function normalizeLinks(input: unknown): LinksInput { `limit must be an integer from 1 through ${MAX_LINK_LIMIT}`, ); - const render = input.render; - const waitMs = input.waitMs; - if (render !== undefined && render !== "fetch" && render !== "agent-browser") - throw new Error('render must be "fetch" or "agent-browser"'); - if (render !== "agent-browser") { - if (waitMs !== undefined) - throw new Error("waitMs is only valid with render agent-browser"); - } else { - if (waitMs === undefined) - throw new Error("waitMs is required with render agent-browser"); - if ( - typeof waitMs !== "number" || - !Number.isInteger(waitMs) || - waitMs < 0 || - waitMs > 30_000 - ) - throw new Error("waitMs must be an integer from 0 through 30000"); - } + const renderOptions = validateRenderOptions(input); return { url, ...(limit !== undefined ? { limit } : {}), - ...(render !== undefined ? { render } : {}), - ...(waitMs !== undefined ? { waitMs } : {}), + ...renderOptions, }; } +function normalizeFetch(input: unknown): FetchInput { + if (!isRecord(input)) throw new Error("web_fetch input must be an object"); + rejectUnknownFields( + input, + ["url", "section_id", "full", "render", "waitMs"], + "web_fetch", + ); + const url = requireString(input, "url"); + const sectionID = input.section_id; + if ( + sectionID !== undefined && + (typeof sectionID !== "string" || sectionID.trim().length === 0) + ) + throw new Error("section_id must be a non-empty string"); + const full = input.full; + if (full !== undefined && typeof full !== "boolean") + throw new Error("full must be a boolean"); + if (full === true && sectionID !== undefined) + throw new Error("full and section_id cannot be used together"); + const renderOptions = validateRenderOptions(input); + return { + url, + ...(sectionID === undefined ? {} : { section_id: sectionID }), + ...(full === undefined ? {} : { full }), + ...renderOptions, + }; +} + +function validateRenderOptions(input: { + render?: unknown; + waitMs?: unknown; +}): RenderOptions { + const render = input.render; + const waitMs = input.waitMs; + if (render !== undefined && render !== "http" && render !== "browser") + throw new Error('render must be "http" or "browser"'); + if (render !== "browser") { + if (waitMs !== undefined) + throw new Error("waitMs is only valid with render browser"); + return render === undefined ? {} : { render }; + } + if (waitMs === undefined) + throw new Error("waitMs is required with render browser"); + if ( + typeof waitMs !== "number" || + !Number.isInteger(waitMs) || + waitMs < 0 || + waitMs > 30_000 + ) + throw new Error("waitMs must be an integer from 0 through 30000"); + return { render, waitMs }; +} + +type RenderOptions = + | { render?: undefined; waitMs?: undefined } + | { render: "http"; waitMs?: undefined } + | { render: "browser"; waitMs: number }; + function normalizeDocs(input: unknown): DocsToolInput { if (!isRecord(input)) throw new Error("web_docs input must be an object"); return normalizeDocsToolInput(input); @@ -458,24 +489,12 @@ function webFetchTool( defineTool({ name: "web_fetch", description: - "Use direct fetch for static, SSR, and pre-rendered pages. For client-rendered or SPA pages, set render: agent-browser with required waitMs on a host that has agent-browser installed; there is no automatic fallback.", + "Use HTTP rendering for static, SSR, and pre-rendered pages. For client-rendered or SPA pages, set render: browser with required waitMs when the host provides browser capability; there is no automatic fallback.", parameters: fetchParameters, output: fetchOutput, isConcurrencySafe: () => true, async execute(args, exec) { - rejectUnknownFields(args, Object.keys(fetchParameters), "web_fetch"); - return operations.fetch( - { - url: requireString(args, "url"), - tree: args.tree, - section_id: args.section_id, - full: args.full, - tree_threshold: args.tree_threshold, - render: args.render, - waitMs: args.waitMs, - }, - exec.signal, - ); + return operations.fetch(normalizeFetch(args), exec.signal); }, }), ); @@ -489,7 +508,7 @@ function webLinksTool( defineTool({ name: "web_links", description: - "List HTTP(S) links from a page. Use direct fetch for static pages, or explicit agent-browser rendering with waitMs for client-rendered pages.", + "List HTTP(S) links from a page. Use HTTP rendering for static pages, or explicit browser rendering with waitMs for client-rendered pages.", parameters: linksParameters, output: linksOutput, isConcurrencySafe: () => true, diff --git a/packages/dsh-web/test/artifact.test.ts b/packages/dsh-web/test/artifact.test.ts index 2b8bb2a..7f6c481 100644 --- a/packages/dsh-web/test/artifact.test.ts +++ b/packages/dsh-web/test/artifact.test.ts @@ -311,7 +311,7 @@ describe("DSH 0.1.2-alpha.3 packed package contract", () => { const rendered = await fetchTool.execute( { url: "https://93.184.216.34/rendered", - render: "agent-browser", + render: "browser", waitMs: 0, full: true, }, @@ -331,7 +331,7 @@ describe("DSH 0.1.2-alpha.3 packed package contract", () => { fetchTool.execute( { url: "https://93.184.216.34/blocked", - render: "agent-browser", + render: "browser", waitMs: 0, }, { signal: new AbortController().signal }, diff --git a/packages/dsh-web/test/client.test.ts b/packages/dsh-web/test/client.test.ts index 68b60b8..dc76969 100644 --- a/packages/dsh-web/test/client.test.ts +++ b/packages/dsh-web/test/client.test.ts @@ -53,7 +53,7 @@ describe("DSH settings client credential surface", () => { it("labels the fetch backend, wait, and retrieval mode from its request", () => { expect( fetchDetails({ - render: "agent-browser", + render: "browser", waitMs: 2_000, section_id: "installation", }), @@ -62,12 +62,12 @@ describe("DSH settings client credential surface", () => { { label: "Wait", value: "2 s" }, { label: "Result", value: "Section: installation" }, ]); - expect(fetchDetails({ tree: true })).toEqual([ - { label: "Backend", value: "Direct fetch" }, - { label: "Result", value: "Heading tree" }, + expect(fetchDetails({})).toEqual([ + { label: "Backend", value: "HTTP rendered" }, + { label: "Result", value: "Adaptive document" }, ]); expect(fetchDetails({ full: true })).toEqual([ - { label: "Backend", value: "Direct fetch" }, + { label: "Backend", value: "HTTP rendered" }, { label: "Result", value: "Full document" }, ]); }); diff --git a/packages/dsh-web/test/tools.test.ts b/packages/dsh-web/test/tools.test.ts index 91c30b3..4171a61 100644 --- a/packages/dsh-web/test/tools.test.ts +++ b/packages/dsh-web/test/tools.test.ts @@ -70,7 +70,7 @@ describe("DSH direct web tools", () => { definitions[0]!.isConcurrencySafe?.({ url: "https://example.test" }), definitions[1]!.isConcurrencySafe?.({ url: "https://example.test", - render: "agent-browser", + render: "browser", waitMs: 0, }), definitions[2]!.isConcurrencySafe?.({ @@ -83,8 +83,8 @@ describe("DSH direct web tools", () => { false, ); expect((definitions[0]!.parameters as any).properties.render.enum).toEqual([ - "fetch", - "agent-browser", + "http", + "browser", ]); expect((definitions[0]!.parameters as any).properties.waitMs.type).toBe( "integer", @@ -93,8 +93,8 @@ describe("DSH direct web tools", () => { 100, ); expect((definitions[1]!.parameters as any).properties.render.enum).toEqual([ - "fetch", - "agent-browser", + "http", + "browser", ]); expect((definitions[2]!.parameters as any).properties.action.enum).toEqual([ "resolve", @@ -149,7 +149,7 @@ describe("DSH direct web tools", () => { { url: "https://example.test", section_id: "install", - render: "agent-browser", + render: "browser", waitMs: 2000, }, controller.signal, @@ -161,7 +161,7 @@ describe("DSH direct web tools", () => { { url: "https://example.test", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 2000, }, controller.signal, @@ -191,11 +191,8 @@ describe("DSH direct web tools", () => { kind: "fetch", input: { url: "https://example.test", - tree: undefined, section_id: "install", - full: undefined, - tree_threshold: undefined, - render: "agent-browser", + render: "browser", waitMs: 2000, }, abortSignal: controller.signal, @@ -205,7 +202,7 @@ describe("DSH direct web tools", () => { input: { url: "https://example.test", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 2000, }, abortSignal: controller.signal, @@ -243,7 +240,7 @@ describe("DSH direct web tools", () => { await expect( call(fetch, { url: "https://example.test", - render: "agent-browser", + render: "browser", }), ).rejects.toThrow("waitMs is required"); await expect( @@ -255,7 +252,7 @@ describe("DSH direct web tools", () => { await expect( call(fetch, { url: "https://example.test", - render: "agent-browser", + render: "browser", waitMs: 30_001, }), ).rejects.toThrow("waitMs must be an integer"); @@ -264,7 +261,7 @@ describe("DSH direct web tools", () => { "javascript_rendering_may_be_required", { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }, ); const allowlist = new FetchCapabilityError("render_domain_not_allowed", { @@ -293,7 +290,7 @@ describe("DSH direct web tools", () => { )[0]!; const pending = call( cancel, - { url: "https://example.test", render: "fetch" }, + { url: "https://example.test", render: "http" }, controller.signal, ); controller.abort(); @@ -312,13 +309,13 @@ describe("DSH direct web tools", () => { await expect( call(retryDefinition, { url: "https://example.test", - render: "fetch", + render: "http", }), ).rejects.toMatchObject({ code: "javascript_rendering_may_be_required", details: { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }, }); @@ -334,7 +331,7 @@ describe("DSH direct web tools", () => { await expect( call(allowlistDefinition, { url: "https://example.test", - render: "agent-browser", + render: "browser", waitMs: 0, }), ).rejects.toMatchObject({ @@ -396,13 +393,13 @@ describe("DSH direct web tools", () => { await expect( call(linksTool!, { url: "https://example.test", - render: "agent-browser", + render: "browser", }), ).rejects.toThrow("waitMs is required"); await expect( call(linksTool!, { url: "https://example.test", - render: "agent-browser", + render: "browser", waitMs: -1, }), ).rejects.toThrow("waitMs must be an integer from 0 through 30000"); diff --git a/packages/pi-web/package.json b/packages/pi-web/package.json index 20eed4d..72bc9d0 100644 --- a/packages/pi-web/package.json +++ b/packages/pi-web/package.json @@ -1,7 +1,7 @@ { "name": "@guionai/pi-web", "version": "0.1.0", - "description": "Pi extension for web research with direct and optional agent-browser page-fetch backends.", + "description": "Pi extension for web research with HTTP and optional browser page-rendering modes.", "homepage": "https://github.com/guionai/web#readme", "bugs": { "url": "https://github.com/guionai/web/issues" diff --git a/packages/pi-web/src/tool.ts b/packages/pi-web/src/tool.ts index e0a1690..3e25e00 100644 --- a/packages/pi-web/src/tool.ts +++ b/packages/pi-web/src/tool.ts @@ -16,8 +16,6 @@ import { Type, type Static, type TSchema } from "typebox"; import { modelTextResult } from "./model-text.js"; -const DEFAULT_TREE_THRESHOLD = 5000; - export const webSearchSchema = Type.Object( { queries: Type.Array( @@ -38,32 +36,27 @@ export const webSearchSchema = Type.Object( const fetchNavigationProperties = { url: Type.String({ description: "HTTP or HTTPS URL to fetch" }), - tree: Type.Optional( - Type.Boolean({ description: "Show the page heading tree" }), - ), section_id: Type.Optional( - Type.String({ description: "Optional heading section ID to return" }), + Type.String({ + description: "Optional heading section ID to return", + minLength: 1, + pattern: "\\S", + }), ), full: Type.Optional( Type.Boolean({ description: "Return full content without automatic tree mode", }), ), - tree_threshold: Type.Optional( - Type.Integer({ - description: `Automatic tree threshold; defaults to ${DEFAULT_TREE_THRESHOLD}`, - default: DEFAULT_TREE_THRESHOLD, - }), - ), }; -export const webFetchSchema = Type.Union([ +const webFetchBaseSchema = Type.Union([ Type.Object( { ...fetchNavigationProperties, render: Type.Optional( - StringEnum(["fetch"] as const, { - description: "Use direct HTTP fetching (the default)", + StringEnum(["http"] as const, { + description: "Use HTTP fetching (the default)", }), ), }, @@ -72,8 +65,8 @@ export const webFetchSchema = Type.Union([ Type.Object( { ...fetchNavigationProperties, - render: StringEnum(["agent-browser"] as const, { - description: "Render the page through the host-installed agent-browser", + render: StringEnum(["browser"] as const, { + description: "Render the page through the host-installed browser", }), waitMs: Type.Integer({ description: "Additional post-load wait in milliseconds", @@ -85,6 +78,22 @@ export const webFetchSchema = Type.Union([ ), ]); +export const webFetchSchema = { + ...webFetchBaseSchema, + allOf: [ + { + not: { + type: "object", + required: ["full", "section_id"], + properties: { + full: { const: true }, + section_id: { type: "string" }, + }, + }, + }, + ], +} as typeof webFetchBaseSchema; + const linksProperties = { url: Type.String({ description: "HTTP or HTTPS URL to inspect" }), limit: Type.Optional( @@ -102,8 +111,8 @@ export const webLinksSchema = Type.Union([ { ...linksProperties, render: Type.Optional( - StringEnum(["fetch"] as const, { - description: "Use direct HTTP fetching (the default)", + StringEnum(["http"] as const, { + description: "Use HTTP fetching (the default)", }), ), }, @@ -112,8 +121,8 @@ export const webLinksSchema = Type.Union([ Type.Object( { ...linksProperties, - render: StringEnum(["agent-browser"] as const, { - description: "Render the page through the host-installed agent-browser", + render: StringEnum(["browser"] as const, { + description: "Render the page through the host-installed browser", }), waitMs: Type.Integer({ description: "Additional post-load wait in milliseconds", @@ -196,15 +205,15 @@ const SEARCH_PROMPT_GUIDELINES = [ "Use web_search to search the web for current facts.", ]; const FETCH_PROMPT_GUIDELINES = [ - "Use web_fetch to read a web page; large pages are truncated with a continuation notice, so follow up with tree or section_id to navigate.", - 'web_fetch has two backends: omit render or use render: "fetch" for direct HTML-to-Markdown (the default for static, SSR, and pre-rendered pages).', - 'For a client-rendered or SPA page, or after javascript_rendering_may_be_required, retry explicitly with render: "agent-browser" and waitMs: 2000 only when the host has agent-browser installed. Increase waitMs explicitly or abandon an incomplete page; there is no automatic fallback.', - "Never send waitMs with direct fetch. agent-browser is a host capability, not a package dependency.", + "Use web_fetch to read a web page; long documents with navigable headings return a navigation tree automatically, so follow up with full: true or a returned section_id. Headingless long documents use the normal bounded response.", + 'web_fetch uses HTTP fetching by default; set render: "http" explicitly when desired.', + 'For a client-rendered or SPA page, or after javascript_rendering_may_be_required, retry explicitly with render: "browser" and waitMs: 2000 only when the host provides browser capability. Increase waitMs explicitly or abandon an incomplete page; there is no automatic fallback.', + "Never send waitMs with HTTP rendering. The browser is a host capability, not a package dependency.", ]; const LINKS_PROMPT_GUIDELINES = [ "Use web_links to discover HTTP(S) destinations from a page, including navigation and links outside the readable article body.", - 'Use direct fetch by default. For a client-rendered or SPA page, explicitly use render: "agent-browser" with waitMs; there is no automatic fallback.', - "Never send waitMs with direct fetch. agent-browser is a host capability, not a package dependency.", + 'Use HTTP rendering by default. For a client-rendered or SPA page, explicitly use render: "browser" with waitMs; there is no automatic fallback.', + "Never send waitMs with HTTP rendering. The browser is a host capability, not a package dependency.", ]; const DOCS_PROMPT_GUIDELINES = [ "Use web_docs with action resolve, then action fetch, to read library documentation instead of fetching documentation sites page by page.", @@ -261,61 +270,38 @@ function requireQueries(input: unknown): string[] { function normalizeFetch(input: unknown): WebFetchInput { if (!isRecord(input)) throw new Error("web_fetch input must be an object"); + for (const field of Object.keys(input)) { + if (!["url", "section_id", "full", "render", "waitMs"].includes(field)) + throw new Error(`web_fetch input does not accept field ${field}`); + } const url = requireString(input, "url"); - const render = input.render; - const waitMs = input.waitMs; - if (render !== undefined && render !== "fetch" && render !== "agent-browser") - throw new Error('render must be "fetch" or "agent-browser"'); - if (render !== "agent-browser") { - if (waitMs !== undefined) - throw new Error("waitMs is only valid with render agent-browser"); - } else { - if (waitMs === undefined) - throw new Error("waitMs is required with render agent-browser"); - if ( - typeof waitMs !== "number" || - !Number.isInteger(waitMs) || - waitMs < 0 || - waitMs > 30_000 - ) - throw new Error("waitMs must be an integer from 0 through 30000"); + const renderOptions = validateRenderOptions(input); + if (input.full !== undefined && typeof input.full !== "boolean") + throw new Error("full must be a boolean"); + if (input.section_id !== undefined) { + if (typeof input.section_id !== "string" || input.section_id.trim() === "") + throw new Error("section_id must be a non-empty string"); } + if (input.full === true && input.section_id !== undefined) + throw new Error("full and section_id cannot be used together"); const typed = input as unknown as WebFetchInput; const navigation = { url, - tree: typed.tree, - section_id: typed.section_id, - full: typed.full, - tree_threshold: typed.tree_threshold, + ...(typed.section_id === undefined ? {} : { section_id: typed.section_id }), + ...(typed.full === undefined ? {} : { full: typed.full }), }; - if (render === "agent-browser") - return { ...navigation, render, waitMs: waitMs as number }; - if (render === "fetch") return { ...navigation, render }; - return navigation; + return { ...navigation, ...renderOptions }; } function normalizeLinks(input: unknown): WebLinksInput { if (!isRecord(input)) throw new Error("web_links input must be an object"); + for (const field of Object.keys(input)) { + if (!["url", "limit", "render", "waitMs"].includes(field)) + throw new Error(`web_links input does not accept field ${field}`); + } const url = requireString(input, "url"); - const render = input.render; - const waitMs = input.waitMs; + const renderOptions = validateRenderOptions(input); const limit = input.limit; - if (render !== undefined && render !== "fetch" && render !== "agent-browser") - throw new Error('render must be "fetch" or "agent-browser"'); - if (render !== "agent-browser") { - if (waitMs !== undefined) - throw new Error("waitMs is only valid with render agent-browser"); - } else { - if (waitMs === undefined) - throw new Error("waitMs is required with render agent-browser"); - if ( - typeof waitMs !== "number" || - !Number.isInteger(waitMs) || - waitMs < 0 || - waitMs > 30_000 - ) - throw new Error("waitMs must be an integer from 0 through 30000"); - } if ( limit !== undefined && (typeof limit !== "number" || @@ -327,13 +313,43 @@ function normalizeLinks(input: unknown): WebLinksInput { `limit must be an integer from 1 through ${MAX_LINK_LIMIT}`, ); const typed = input as unknown as WebLinksInput; - const result = { url, limit: typed.limit }; - if (render === "agent-browser") - return { ...result, render, waitMs: waitMs as number }; - if (render === "fetch") return { ...result, render }; - return result; + const result = { + url, + ...(typed.limit === undefined ? {} : { limit: typed.limit }), + }; + return { ...result, ...renderOptions }; } +function validateRenderOptions(input: { + render?: unknown; + waitMs?: unknown; +}): RenderOptions { + const render = input.render; + const waitMs = input.waitMs; + if (render !== undefined && render !== "http" && render !== "browser") + throw new Error('render must be "http" or "browser"'); + if (render !== "browser") { + if (waitMs !== undefined) + throw new Error("waitMs is only valid with render browser"); + return render === undefined ? {} : { render }; + } + if (waitMs === undefined) + throw new Error("waitMs is required with render browser"); + if ( + typeof waitMs !== "number" || + !Number.isInteger(waitMs) || + waitMs < 0 || + waitMs > 30_000 + ) + throw new Error("waitMs must be an integer from 0 through 30000"); + return { render, waitMs }; +} + +type RenderOptions = + | { render?: undefined; waitMs?: undefined } + | { render: "http"; waitMs?: undefined } + | { render: "browser"; waitMs: number }; + function mergeSearchResults(responses: SearchResponse[]): SearchResponse { const results: SearchResponse["results"] = []; for (let resultIndex = 0; ; resultIndex += 1) { @@ -419,14 +435,14 @@ export function webFetchTool(dependencies: WebToolDependencies = {}) { name: "web_fetch", label: "Web fetch", description: - "Fetch and read an HTTP or HTTPS web page as Markdown, with direct fetch or explicit agent-browser rendering for client-rendered pages. Rendered fetch requires waitMs 0 through 30000. Text output is limited to 2,000 lines or 50KB; truncated output is saved to a temporary file.", + "Fetch and read an HTTP or HTTPS web page as Markdown, with HTTP rendering by default or explicit browser rendering for client-rendered pages. Browser rendering requires waitMs 0 through 30000. Long documents with navigable headings return a navigation tree unless full or section_id is requested; headingless long documents use the normal bounded response. Text output is limited to 2,000 lines or 50KB; truncated output is saved to a temporary file.", promptSnippet: "Fetch a web page with web_fetch", promptGuidelines: FETCH_PROMPT_GUIDELINES, parameters: webFetchSchema, execute: async (params, signal) => { const data = await operations.fetch(normalizeFetch(params), signal); return modelTextResult(data, data.content, { - hint: "Use web_fetch with tree or section_id to navigate the document.", + hint: "Use web_fetch with full: true or a returned section_id to navigate the document.", }); }, }); @@ -438,7 +454,7 @@ export function webLinksTool(dependencies: WebToolDependencies = {}) { name: "web_links", label: "Web links", description: - "List HTTP(S) links from a web page, with direct fetch or explicit agent-browser rendering for client-rendered pages.", + "List HTTP(S) links from a web page, with HTTP rendering by default or explicit browser rendering for client-rendered pages.", promptSnippet: "List links from a web page with web_links", promptGuidelines: LINKS_PROMPT_GUIDELINES, parameters: webLinksSchema, diff --git a/packages/pi-web/test/extension.test.ts b/packages/pi-web/test/extension.test.ts index 1c42dd3..b12c332 100644 --- a/packages/pi-web/test/extension.test.ts +++ b/packages/pi-web/test/extension.test.ts @@ -98,43 +98,50 @@ describe("pi-web extension", () => { expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - render: "fetch", + render: "http", }), ).toBe(true); expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", waitMs: 0, }), ).toBe(true); expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", waitMs: 30_000, }), ).toBe(true); expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", }), ).toBe(false); expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - render: "fetch", + render: "http", waitMs: 0, }), ).toBe(false); expect( Value.Check(webFetchSchema, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", waitMs: 30_001, }), ).toBe(false); + expect( + Value.Check(webFetchSchema, { + url: "https://fixture.test", + full: true, + section_id: "intro", + }), + ).toBe(false); const fetch = vi.fn( async (input: { url: string; render?: string; waitMs?: number }) => ({ @@ -146,17 +153,13 @@ describe("pi-web extension", () => { const tool = webFetchTool({ operations: operations({ fetch }) }); const result = await call(tool, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", waitMs: 1250, }); expect(fetch).toHaveBeenCalledWith( { url: "https://fixture.test", - tree: undefined, - section_id: undefined, - full: undefined, - tree_threshold: undefined, - render: "agent-browser", + render: "browser", waitMs: 1250, }, undefined, @@ -166,13 +169,13 @@ describe("pi-web extension", () => { await expect( call(tool, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", }), ).rejects.toThrow("waitMs is required"); await expect( call(tool, { url: "https://fixture.test", - render: "fetch", + render: "http", waitMs: 0, }), ).rejects.toThrow("waitMs is only valid"); @@ -184,7 +187,7 @@ describe("pi-web extension", () => { expect( Value.Check(webLinksSchema, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", waitMs: 0, limit: 25, }), @@ -192,7 +195,7 @@ describe("pi-web extension", () => { expect( Value.Check(webLinksSchema, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", }), ).toBe(false); expect( @@ -211,14 +214,14 @@ describe("pi-web extension", () => { const linksResult = await call(linksTool, { url: "https://fixture.test", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 1250, }); expect(links).toHaveBeenCalledWith( { url: "https://fixture.test", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 1250, }, undefined, @@ -233,7 +236,7 @@ describe("pi-web extension", () => { code: "javascript_rendering_may_be_required", details: { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }, }, ); @@ -266,7 +269,7 @@ describe("pi-web extension", () => { await expect( call(blockedTool, { url: "https://fixture.test", - render: "agent-browser", + render: "browser", waitMs: 0, }), ).rejects.toBe(blocked); @@ -469,7 +472,7 @@ describe("pi-web extension", () => { expect(details.url).toBe("https://fixture.test"); expect(details.truncation.truncated).toBe(true); expect(result.content[0]?.text).toContain( - "Use web_fetch with tree or section_id to navigate the document.", + "Use web_fetch with full: true or a returned section_id to navigate the document.", ); expect(await readFile(details.fullOutputPath, "utf8")).toBe(content); } finally { diff --git a/packages/pi-web/test/packed-smoke.mjs b/packages/pi-web/test/packed-smoke.mjs index 9aba947..f5d366f 100644 --- a/packages/pi-web/test/packed-smoke.mjs +++ b/packages/pi-web/test/packed-smoke.mjs @@ -215,7 +215,7 @@ if (command === "open" && args.some((value) => value.includes("/blocked"))) { const rendered = await fetchTool.execute("test", { url: "https://93.184.216.34/rendered", - render: "agent-browser", + render: "browser", waitMs: 0, full: true, }); @@ -248,7 +248,7 @@ if (command === "open" && args.some((value) => value.includes("/blocked"))) { try { await fetchTool.execute("test", { url: "https://93.184.216.34/blocked", - render: "agent-browser", + render: "browser", waitMs: 0, }); throw new Error("blocked rendered fetch unexpectedly succeeded"); diff --git a/packages/web-core/src/fetch.ts b/packages/web-core/src/fetch.ts index b76c88f..c22c825 100644 --- a/packages/web-core/src/fetch.ts +++ b/packages/web-core/src/fetch.ts @@ -9,7 +9,7 @@ import { spawn } from "node:child_process"; import { Defuddle } from "defuddle/node"; import { parseHTML } from "linkedom"; -import { renderMarkdown, truncateContent } from "./markdown.js"; +import { renderMarkdown } from "./markdown.js"; import { boundedRequest, isOperationAborted, @@ -45,11 +45,9 @@ export const RENDER_CDN_ALLOWLIST = [ export type FetchInput = { url: string; - tree?: boolean; section_id?: string; full?: boolean; - tree_threshold?: number; - render?: "fetch" | "agent-browser"; + render?: "http" | "browser"; waitMs?: number; }; @@ -62,7 +60,7 @@ export type FetchResult = { export type LinksInput = { url: string; limit?: number; - render?: "fetch" | "agent-browser"; + render?: "http" | "browser"; waitMs?: number; }; @@ -79,7 +77,7 @@ export type LinksResult = { export type FetchErrorDetails = { retryableWithRender?: boolean; - suggestedArguments?: { render: "agent-browser"; waitMs: 2000 }; + suggestedArguments?: { render: "browser"; waitMs: 2000 }; retryable?: boolean; reportUrl?: string; blockedHostname?: string; @@ -125,20 +123,19 @@ export async function fetchWebPage( callerSignal?: AbortSignal, options?: FetchOptions, ): Promise { + validateFetchFields(input); const url = validateURL(input.url); const render = validateRenderInput(input); + validateNavigationInput(input); const content = - render === "agent-browser" + render === "browser" ? await renderPage(url, input.waitMs!, callerSignal, options) : await fetchCached(url, callerSignal, options); throwIfAborted(callerSignal); - const rendered = renderMarkdown( - content, - input.tree ?? false, - input.section_id, - input.full ?? false, - input.tree_threshold, - ); + const rendered = renderMarkdown(content, { + section_id: input.section_id, + full: input.full === true, + }); return { url, mode: rendered.mode, content: rendered.content }; } @@ -148,11 +145,12 @@ export async function fetchWebLinks( callerSignal?: AbortSignal, options?: FetchOptions, ): Promise { + validateLinksFields(input); const url = validateURL(input.url); const render = validateRenderInput(input); const limit = validateLinkLimit(input.limit); const source = - render === "agent-browser" + render === "browser" ? await renderPageHTML(url, input.waitMs!, callerSignal, options) : await fetchPageHTML(url, callerSignal, options); throwIfAborted(callerSignal); @@ -161,17 +159,17 @@ export async function fetchWebLinks( function validateRenderInput( input: Pick, -): "fetch" | "agent-browser" { - const render = input.render ?? "fetch"; - if (render !== "fetch" && render !== "agent-browser") - throw new Error('render must be "fetch" or "agent-browser"'); - if (render === "fetch") { +): "http" | "browser" { + const render = input.render ?? "http"; + if (render !== "http" && render !== "browser") + throw new Error('render must be "http" or "browser"'); + if (render === "http") { if (input.waitMs !== undefined) - throw new Error("waitMs is only valid with render agent-browser"); + throw new Error("waitMs is only valid with render browser"); return render; } if (input.waitMs === undefined) - throw new Error("waitMs is required with render agent-browser"); + throw new Error("waitMs is required with render browser"); if ( !Number.isInteger(input.waitMs) || input.waitMs < 0 || @@ -181,6 +179,42 @@ function validateRenderInput( return render; } +function validateFetchFields(input: FetchInput): void { + validateKnownFields( + input, + ["url", "section_id", "full", "render", "waitMs"], + "fetch", + ); + if (typeof input.full !== "undefined" && typeof input.full !== "boolean") + throw new Error("full must be a boolean"); + if (typeof input.section_id !== "undefined") { + if (typeof input.section_id !== "string" || input.section_id.trim() === "") + throw new Error("section_id must be a non-empty string"); + } +} + +function validateLinksFields(input: LinksInput): void { + validateKnownFields(input, ["url", "limit", "render", "waitMs"], "links"); +} + +function validateKnownFields( + input: object, + fields: readonly string[], + operation: string, +): void { + for (const field of Object.keys(input)) { + if (!fields.includes(field)) + throw new Error(`${operation} input does not accept field ${field}`); + } +} + +function validateNavigationInput( + input: Pick, +): void { + if (input.full === true && input.section_id !== undefined) + throw new Error("full and section_id cannot be used together"); +} + function validateLinkLimit(limit: number | undefined): number { if (limit === undefined) return DEFAULT_LINK_LIMIT; if (!Number.isInteger(limit) || limit < 1 || limit > MAX_LINK_LIMIT) @@ -220,7 +254,7 @@ async function fetchLocal( try { const response = await downloadPage(url, callerSignal, options); if (response.contentType !== "" && response.contentType !== "text/html") - return truncateContent(new TextDecoder().decode(response.body)); + return new TextDecoder().decode(response.body); return extractHTML( new TextDecoder().decode(response.body), url, @@ -576,12 +610,12 @@ async function extractHTML( if (suggestRender) { throw new FetchCapabilityError("javascript_rendering_may_be_required", { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }); } throw new Error("no content could be extracted"); } - return truncateContent(content.endsWith("\n") ? content : content + "\n"); + return ensureTrailingNewline(content); } catch (error) { if (error instanceof FetchCapabilityError) throw error; throw new Error(`defuddle parse failed: ${errorMessage(error)}`); @@ -641,6 +675,9 @@ function rendererEnvironment( TEMP: workDirectory, AGENT_BROWSER_CONFIG: configPath, }; + if (process.env.AGENT_BROWSER_EXECUTABLE_PATH !== undefined) + environment.AGENT_BROWSER_EXECUTABLE_PATH = + process.env.AGENT_BROWSER_EXECUTABLE_PATH; for (const name of [ "HTTP_PROXY", "HTTPS_PROXY", @@ -1099,6 +1136,10 @@ function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } +function ensureTrailingNewline(content: string): string { + return content.endsWith("\n") ? content : `${content}\n`; +} + function defaultCacheDir(): string { const home = process.env.HOME?.trim() || homedir(); if (process.platform === "win32") { @@ -1148,7 +1189,7 @@ class DailyCache implements FetchCache { }); } catch { if (signal?.aborted) throwIfAborted(signal); - // Cache failures must not make a successful direct fetch fail. + // Cache failures must not make a successful HTTP fetch fail. } } diff --git a/packages/web-core/src/index.ts b/packages/web-core/src/index.ts index da84422..0da03b6 100644 --- a/packages/web-core/src/index.ts +++ b/packages/web-core/src/index.ts @@ -37,6 +37,7 @@ export { isOperationAborted, isRequestTimeout, isResponseBodyLimit, + throwIfAborted, } from "./request.js"; export { @@ -60,6 +61,7 @@ export { renderMarkdown, truncateContent, type MarkdownResult, + type MarkdownNavigationOptions, } from "./markdown.js"; export { DEFAULT_MAX_BYTES, @@ -133,6 +135,8 @@ export type SearchInput = { provider?: string; credentials: SearchCredentials; maxResults?: number; + /** Permit an empty successful Kepos result for the HTTP service contract. */ + allowEmptyKeposResults?: boolean; /** Complete route used by the Kepos Bridge provider. */ keposBridgeEndpoint?: string; signal?: AbortSignal; @@ -355,7 +359,11 @@ async function searchKeposBridge(input: SearchInput): Promise { timeoutMs: input.timeoutMs ?? undefined, }); const usable = (response.results ?? []).filter(isUsableTextResult); - if (usable.length === 0) + const returnedResults = response.results ?? []; + if ( + usable.length === 0 && + (!input.allowEmptyKeposResults || returnedResults.length > 0) + ) throw new Error("Kepos Bridge search returned no usable text results"); const limited = input.maxResults === undefined diff --git a/packages/web-core/src/markdown.ts b/packages/web-core/src/markdown.ts index 21d4345..ccea28c 100644 --- a/packages/web-core/src/markdown.ts +++ b/packages/web-core/src/markdown.ts @@ -18,6 +18,11 @@ export type MarkdownResult = { mode: "full" | "tree" | "section"; }; +export type MarkdownNavigationOptions = { + section_id?: string; + full?: boolean; +}; + const DEFAULT_TREE_THRESHOLD = 5000; const MAX_CONTENT_CHARS = 30_000; const BASE62 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; @@ -28,13 +33,12 @@ const markdown = new MarkdownIt(); export function renderMarkdown( source: string, - showTree: boolean, - sectionId: string | undefined, - full: boolean, - treeThreshold: number | undefined, + options: MarkdownNavigationOptions = {}, ): MarkdownResult { const headings = assignIds(parseHeadings(source)); - const section = sectionId?.trim(); + const section = options.section_id?.trim(); + if (options.full === true && section !== undefined) + throw new Error("full and section_id cannot be used together"); if (section) { return { content: extractSection(source, headings, section), @@ -42,15 +46,17 @@ export function renderMarkdown( }; } - const threshold = - treeThreshold && treeThreshold > 0 ? treeThreshold : DEFAULT_TREE_THRESHOLD; const charCount = Array.from(source).length; - if (showTree || (!full && charCount > threshold)) { - if (headings.length > 0) { - return { content: renderTree(source, headings), mode: "tree" }; - } - } - return { content: truncateContent(source), mode: "full" }; + if ( + !options.full && + charCount > DEFAULT_TREE_THRESHOLD && + headings.length > 0 + ) + return { content: renderTree(source, headings), mode: "tree" }; + return { + content: options.full ? source : truncateContent(source), + mode: "full", + }; } export function truncateContent(content: string): string { @@ -182,7 +188,13 @@ function renderTree(source: string, headings: Heading[]): string { 99, ); if (bodyHeadings.length === 0) { - return `${header}(empty)\n\nUse -s to read a section, or --full to read everything.\n`; + const tree = headings + .map( + (heading, index) => + `└── [${heading.id}] ${"#".repeat(heading.level)} ${heading.text} (${formatNumber(sectionCharCount(source, headings, index))} chars)\n`, + ) + .join(""); + return `${header}${tree}\nUse section_id to read a section, or full: true to read everything.\n`; } const nodes = bodyHeadings.map((heading) => { @@ -211,7 +223,7 @@ function renderTree(source: string, headings: Heading[]): string { hasMore.delete(depth); }); - return `${header}${tree}\nUse -s to read a section, or --full to read everything.\n`; + return `${header}${tree}\nUse section_id to read a section, or full: true to read everything.\n`; } function sectionCharCount( diff --git a/packages/web-core/test/fetch.test.ts b/packages/web-core/test/fetch.test.ts index bbdaec5..fb7ce64 100644 --- a/packages/web-core/test/fetch.test.ts +++ b/packages/web-core/test/fetch.test.ts @@ -213,7 +213,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { code: "javascript_rendering_may_be_required", details: { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }, }); await expect(fetchPage({ url })).rejects.toThrow( @@ -338,6 +338,8 @@ describe.sequential("browserless fetch migrated from Organon", () => { const cacheDirectory = mkdtempSync( join(tmpdir(), "guionai-web-render-cache-"), ); + const previousExecutable = process.env.AGENT_BROWSER_EXECUTABLE_PATH; + process.env.AGENT_BROWSER_EXECUTABLE_PATH = "/usr/bin/chromium"; const resolveHost = async () => ["93.184.216.34"]; const directFetch = vi.fn( async () => @@ -348,7 +350,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { try { await expect( fetchWebPage( - { url: "https://render.test/page", render: "fetch" }, + { url: "https://render.test/page", render: "http" }, undefined, { cacheDirectory, fetch: directFetch, resolveHost }, ), @@ -365,7 +367,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebPage( { url: "https://render.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, full: true, }, @@ -405,11 +407,12 @@ describe.sequential("browserless fetch migrated from Organon", () => { expect(open.home).toBeUndefined(); expect(open.config).toBe("{}\n"); expect(open.profile).toBeUndefined(); + expect(open.executable).toBe("/usr/bin/chromium"); await fetchWebPage( { url: "https://render.test/page", - render: "agent-browser", + render: "browser", waitMs: 1, }, undefined, @@ -417,6 +420,9 @@ describe.sequential("browserless fetch migrated from Organon", () => { ); expect(readFakeLog(logPath)).toHaveLength(6); } finally { + if (previousExecutable === undefined) + delete process.env.AGENT_BROWSER_EXECUTABLE_PATH; + else process.env.AGENT_BROWSER_EXECUTABLE_PATH = previousExecutable; rmSync(cacheDirectory, { recursive: true, force: true }); } }); @@ -429,7 +435,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebLinks( { url: "https://render.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -462,7 +468,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebLinks( { url: "https://render.test/start", - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -492,7 +498,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { const pending = fetchWebPage( { url: "https://render.test/page", - render: "agent-browser", + render: "browser", waitMs: 30_000, }, controller.signal, @@ -514,7 +520,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { const pending = fetchWebPage( { url: "https://ignore-term.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, controller.signal, @@ -540,7 +546,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { const failure = await fetchWebPage( { url: `https://user:${secret}@render.test/page`, - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -565,7 +571,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { await expect( fetchWebPage({ url: `https://[${address}]/page`, - render: "agent-browser", + render: "browser", waitMs: 0, }), ).rejects.toThrow("private or reserved"); @@ -577,7 +583,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebPage( { url: "https://resolved.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -598,7 +604,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebPage( { url: "https://render.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -623,7 +629,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebPage( { url: "https://blocked.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -655,7 +661,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { await expect( fetchWebPage({ url: "https://render.test/page", - render: "agent-browser", + render: "browser", }), ).rejects.toThrow("waitMs is required"); await expect( @@ -664,7 +670,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { await expect( fetchWebPage({ url: "https://render.test/page", - render: "agent-browser", + render: "browser", waitMs: 30_001, }), ).rejects.toThrow("waitMs must be an integer"); @@ -674,7 +680,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { await expect( fetchWebPage({ url: "http://127.0.0.1/private", - render: "agent-browser", + render: "browser", waitMs: 0, }), ).rejects.toThrow("private or reserved"); @@ -684,7 +690,7 @@ describe.sequential("browserless fetch migrated from Organon", () => { fetchWebPage( { url: "https://blocked.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, undefined, @@ -756,6 +762,70 @@ describe.sequential("browserless fetch migrated from Organon", () => { await close(server); } }); + + it("owns automatic tree navigation, complete full extraction, and input validation", async () => { + const source = + "# Page\n\n## Install\nInstall content.\n\n## Next\nNext content.\n" + + "x".repeat(5_001); + const cache = { + prepare: vi.fn(async () => {}), + read: vi.fn(async () => source), + write: vi.fn(async () => {}), + }; + const input = { url: "https://navigation.test/page" }; + + await expect( + fetchWebPage(input, undefined, { cache }), + ).resolves.toMatchObject({ + mode: "tree", + }); + await expect( + fetchWebPage({ ...input, section_id: "7i" }, undefined, { cache }), + ).resolves.toMatchObject({ + mode: "section", + content: "## Install\nInstall content.\n", + }); + + const complete = "# Complete\n\n" + "x".repeat(30_001); + const completeCache = { + prepare: vi.fn(async () => {}), + read: vi.fn(async () => complete), + write: vi.fn(async () => {}), + }; + const result = await fetchWebPage( + { url: "https://navigation.test/complete", full: true }, + undefined, + { cache: completeCache }, + ); + expect(result).toEqual({ + url: "https://navigation.test/complete", + mode: "full", + content: complete, + }); + + await expect( + fetchWebPage( + { + url: input.url, + full: true, + section_id: "7i", + } as never, + undefined, + { cache }, + ), + ).rejects.toThrow("full and section_id cannot be used together"); + await expect( + fetchWebPage({ url: input.url, tree: true } as never, undefined, { + cache, + }), + ).rejects.toThrow("does not accept field tree"); + await expect( + fetchWebPage({ url: input.url, tree_threshold: 1 } as never, undefined, { + cache, + }), + ).rejects.toThrow("does not accept field tree_threshold"); + expect(cache.read).toHaveBeenCalledTimes(2); + }); }); async function withFakeAgentBrowser( @@ -786,6 +856,7 @@ appendFileSync(${JSON.stringify(logPath)}, JSON.stringify({ home: process.env.HOME, config: readFileSync(process.env.AGENT_BROWSER_CONFIG, "utf8"), profile: process.env.AGENT_BROWSER_PROFILE, + executable: process.env.AGENT_BROWSER_EXECUTABLE_PATH, }) + "\\n"); if (command === "open" && args.includes("https://ignore-term.test/page")) { process.on("SIGTERM", () => {}); @@ -819,6 +890,7 @@ type FakeCommand = { home?: string; config: string; profile?: string; + executable?: string; }; function readFakeLog(path: string): FakeCommand[] { diff --git a/packages/web-core/test/markdown.test.ts b/packages/web-core/test/markdown.test.ts index d2e12bc..d41c8bd 100644 --- a/packages/web-core/test/markdown.test.ts +++ b/packages/web-core/test/markdown.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { renderMarkdown } from "../src/index.js"; -describe("Markdown navigation migrated from Organon", () => { +describe("Markdown navigation", () => { it("uses token labels and source maps for nested, inline, and Setext headings", () => { const source = [ "# Héading **bold** `code` [link](https://example.com)", @@ -28,8 +28,7 @@ describe("Markdown navigation migrated from Organon", () => { "after", "", ].join("\n"); - - const tree = renderMarkdown(source, true, undefined, false, 5000); + const tree = renderMarkdown(`${source}\n${"x".repeat(5001)}`); expect(tree.mode).toBe("tree"); expect(tree.content).toContain("# Héading bold code link"); expect(tree.content).toContain("## Quote emphasis"); @@ -47,53 +46,65 @@ describe("Markdown navigation migrated from Organon", () => { /\[([0-9A-Za-z]{2,3})\] ## Quote emphasis/, )?.[1]; expect(quoteID).toBeDefined(); - expect(renderMarkdown(source, false, quoteID, false, 5000)).toEqual({ + expect(renderMarkdown(source, { section_id: quoteID })).toEqual({ mode: "section", content: "> ## Quote *emphasis*\n\n- ### List heading\n", }); }); - it("preserves opaque IDs and full, tree, section, threshold, and truncation modes", () => { + it("uses the fixed automatic tree policy and full/section navigation", () => { const source = "# Test page\n\n## Install\nInstall content.\n\n### Details\nDetails content.\n\n## Next\nNext content.\n"; - const tree = renderMarkdown(source, true, undefined, false, 5000); - expect(tree).toMatchObject({ mode: "tree" }); + const tree = renderMarkdown(`${source}${"x".repeat(5001)}`); + expect(tree.mode).toBe("tree"); expect(tree.content).toContain("[7i] ## Install"); expect(tree.content).toContain("[eD] ### Details"); - expect(renderMarkdown(source, false, "7i", false, 5000)).toEqual({ + expect(renderMarkdown(source, { section_id: "7i" })).toEqual({ mode: "section", content: "## Install\nInstall content.\n\n### Details\nDetails content.\n", }); - expect(() => renderMarkdown(source, false, "missing", false, 5000)).toThrow( + expect(() => renderMarkdown(source, { section_id: "missing" })).toThrow( 'section "missing" not found', ); - expect(renderMarkdown("plain content", true, undefined, false, 1)).toEqual({ + expect(renderMarkdown("plain content")).toEqual({ content: "plain content", mode: "full", }); - expect( - renderMarkdown( - "# H\n\n" + "x".repeat(6000), - false, - undefined, - false, - 5000, - ).mode, - ).toBe("tree"); - expect( - renderMarkdown("# H\n\n" + "x".repeat(6000), false, undefined, true, 1) - .mode, - ).toBe("full"); + const complete = "# H\n\n" + "x".repeat(30_001); + expect(renderMarkdown(complete, { full: true })).toEqual({ + content: complete, + mode: "full", + }); + }); - const unicode = "😀".repeat(30_001); - const bounded = renderMarkdown(unicode, false, undefined, false, 50_000); - expect( - Array.from(bounded.content.split("\n[content truncated")[0]!).length, - ).toBe(30_000); - expect(bounded.content).toContain( - "[content truncated at 30000 characters]", - ); + it("lists and retrieves an H1-only long document section", () => { + const source = `# Only title\n\n${"x".repeat(5001)}\n`; + const tree = renderMarkdown(source); + const sectionID = tree.content.match( + /\[([0-9A-Za-z]{2,3})\] # Only title/, + )?.[1]; + + expect(tree.mode).toBe("tree"); + expect(sectionID).toBeDefined(); + expect(renderMarkdown(source, { section_id: sectionID })).toEqual({ + content: source, + mode: "section", + }); + }); + + it("keeps headingless long documents bounded and selectable only by headings", () => { + const source = "x".repeat(5001); + expect(renderMarkdown(source)).toEqual({ + content: source, + mode: "full", + }); + + const complete = "x".repeat(30_001); + expect(renderMarkdown(complete, { full: true })).toEqual({ + content: complete, + mode: "full", + }); }); }); diff --git a/packages/web-core/test/search.test.ts b/packages/web-core/test/search.test.ts index 3abbe29..8b7b266 100644 --- a/packages/web-core/test/search.test.ts +++ b/packages/web-core/test/search.test.ts @@ -199,6 +199,18 @@ describe("search providers migrated from Organon fixtures", () => { ).rejects.toThrow(/Kepos Bridge provider/); }); + it("can preserve an explicitly empty successful Bridge result for HTTP callers", async () => { + await expect( + search({ + query: "empty", + provider: "kepos-bridge", + allowEmptyKeposResults: true, + credentials: {}, + fetch: async () => response({ output: "no matches", results: [] }), + }), + ).resolves.toEqual({ provider: "Kepos Bridge", results: [] }); + }); + it("does not expose remote error bodies or credentials", async () => { await expect( search({ diff --git a/packages/web/package.json b/packages/web/package.json index d8c999c..f3a458d 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -1,7 +1,7 @@ { "name": "@guionai/web", "version": "0.1.0", - "description": "Web research CLI and stdio MCP server with direct and optional agent-browser page-fetch backends.", + "description": "Web research CLI, stdio MCP server, and personal HTTP service with HTTP and optional browser page-rendering modes.", "homepage": "https://github.com/guionai/web#readme", "bugs": { "url": "https://github.com/guionai/web/issues" @@ -27,13 +27,23 @@ "node": ">=20" }, "dependencies": { - "commander": "12.1.0" + "@hono/node-server": "2.1.1", + "@hono/zod-openapi": "1.6.1", + "commander": "12.1.0", + "hono": "4.13.5", + "yaml": "2.9.0", + "zod": "4.4.3" }, "bundledDependencies": [ - "commander" + "@hono/node-server", + "@hono/zod-openapi", + "commander", + "hono", + "yaml", + "zod" ], "scripts": { - "build": "tsup", + "build": "tsup && node dist/generate-openapi.js", "typecheck": "tsc --noEmit -p ../../tsconfig.json", "test:pack": "node test/packed-smoke.mjs" }, diff --git a/packages/web/src/generate-openapi.ts b/packages/web/src/generate-openapi.ts new file mode 100644 index 0000000..6afd90b --- /dev/null +++ b/packages/web/src/generate-openapi.ts @@ -0,0 +1,33 @@ +#!/usr/bin/env node +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { stringify } from "yaml"; + +import { createHttpOpenAPIDocument } from "./http.js"; + +const packageDirectory = dirname(fileURLToPath(import.meta.url)); +const packageManifestPath = resolve(packageDirectory, "../package.json"); + +/** Serializes the release OpenAPI contract to a YAML file. */ +export async function generateOpenAPI(output?: string): Promise { + const outputPath = output + ? resolve(output) + : join(packageDirectory, "openapi.yaml"); + const manifest = JSON.parse(await readFile(packageManifestPath, "utf8")) as { + version?: unknown; + }; + if (typeof manifest.version !== "string" || manifest.version.length === 0) + throw new Error("package version is required to generate OpenAPI"); + + const document = createHttpOpenAPIDocument(manifest.version); + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, stringify(document), "utf8"); + return outputPath; +} + +const invokedPath = process.argv[1]; +if (invokedPath && resolve(invokedPath) === fileURLToPath(import.meta.url)) { + const outputPath = await generateOpenAPI(process.argv[2]); + process.stdout.write(`generated ${outputPath}\n`); +} diff --git a/packages/web/src/http.ts b/packages/web/src/http.ts new file mode 100644 index 0000000..422a357 --- /dev/null +++ b/packages/web/src/http.ts @@ -0,0 +1,514 @@ +import { + DEFAULT_LINK_LIMIT, + DEFAULT_KEPOS_BRIDGE_ENDPOINT, + FetchCapabilityError, + isOperationAborted, + isRequestTimeout, + MAX_LINK_LIMIT, + RENDER_REPORT_URL, + throwIfAborted, + validateKeposBridgeEndpoint, + type SearchResponse, + type WebCredentials, + type WebOperations, +} from "@guionai/web-core"; +import * as webCoreModule from "@guionai/web-core"; +import { HTTPException } from "hono/http-exception"; +import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi"; + +import { credentialsFromEnvironment } from "./runtime.js"; + +/** Dependencies and server-local configuration for the personal HTTP service. */ +export type HttpServiceDependencies = { + operations?: WebOperations; + credentials?: WebCredentials | (() => WebCredentials); + keposBridgeEndpoint?: string; + /** Injectable environment for startup/configuration tests. */ + environment?: NodeJS.ProcessEnv; + /** Used by build-time OpenAPI generation to skip runtime credential checks. */ + validateStartup?: boolean; +}; + +export type HttpServiceState = { + operations: WebOperations; + credentials: WebCredentials; + keposBridgeEndpoint: string; +}; + +export type HttpError = { + code: string; + message: string; + details?: Record; +}; + +const ErrorSchema = z + .object({ + code: z.string().openapi({ example: "upstream_error" }), + message: z.string().openapi({ example: "search failed" }), + details: z.record(z.string(), z.unknown()).optional(), + }) + .strict() + .openapi("HttpError"); + +const SearchResultSchema = z + .object({ + title: z.string(), + link: z.string(), + snippet: z.string(), + position: z.number().int(), + }) + .strict() + .openapi("SearchResult"); + +const SearchResponseSchema = z + .object({ + provider: z.enum(["Exa", "Kepos Bridge"]), + results: z.array(SearchResultSchema), + }) + .strict() + .openapi("SearchResponse"); + +const FetchResponseSchema = z + .object({ + url: z.string(), + mode: z.enum(["full", "tree", "section"]), + content: z.string(), + }) + .strict() + .openapi("FetchResponse"); + +const LinkSchema = z + .object({ text: z.string(), url: z.string() }) + .strict() + .openapi("PageLink"); + +const LinksResponseSchema = z + .object({ + url: z.string(), + links: z.array(LinkSchema), + truncated: z.boolean(), + }) + .strict() + .openapi("LinksResponse"); + +const SearchRequestSchema = z + .object({ query: z.string().min(1) }) + .strict() + .openapi("SearchRequest"); + +const HttpUrlSchema = z + .string() + .min(1) + .refine(isHttpURL, "url must be an absolute HTTP(S) URL"); + +const FetchRequestSchema = z + .object({ + url: HttpUrlSchema, + section_id: z + .string() + .min(1) + .refine( + (value) => value.trim().length > 0, + "section_id must be a non-empty string", + ) + .optional(), + full: z.boolean().default(false), + render: z.enum(["http", "browser"]).default("http"), + waitMs: z.number().int().min(0).max(30_000).optional(), + }) + .strict() + .refine( + (input) => !(input.full && input.section_id !== undefined), + "full and section_id cannot be used together", + ) + .openapi("FetchRequest"); + +const LinksRequestSchema = z + .object({ + url: HttpUrlSchema, + limit: z + .number() + .int() + .min(1) + .max(MAX_LINK_LIMIT) + .default(DEFAULT_LINK_LIMIT), + render: z.enum(["http", "browser"]).default("http"), + waitMs: z.number().int().min(0).max(30_000).optional(), + }) + .strict() + .openapi("LinksRequest"); + +const commonResponses = { + 400: { + content: { "application/json": { schema: ErrorSchema } }, + description: "The request body is invalid.", + }, + 499: { + content: { "application/json": { schema: ErrorSchema } }, + description: "The client cancelled the request.", + }, + 502: { + content: { "application/json": { schema: ErrorSchema } }, + description: "The upstream operation failed.", + }, + 504: { + content: { "application/json": { schema: ErrorSchema } }, + description: "The upstream operation timed out.", + }, +} as const; + +const searchRoute = createRoute({ + method: "post", + path: "/v1/search", + operationId: "search", + summary: "Search the web", + description: + "Search through the server-local Kepos Bridge, retrying once through Exa when Bridge is unavailable.", + request: jsonRequest(SearchRequestSchema), + responses: { + 200: jsonResponse(SearchResponseSchema, "Search results."), + ...commonResponses, + }, +}); + +const fetchRoute = createRoute({ + method: "post", + path: "/v1/fetch", + operationId: "fetch", + summary: "Fetch a web page", + description: + "Fetch through HTTP by default; browser rendering requires render=browser and waitMs.", + request: jsonRequest(FetchRequestSchema), + responses: { + 200: jsonResponse(FetchResponseSchema, "Fetched page."), + ...commonResponses, + }, +}); + +const linksRoute = createRoute({ + method: "post", + path: "/v1/links", + operationId: "links", + summary: "List page links", + description: + "List HTTP(S) anchors using HTTP rendering by default or explicit browser rendering.", + request: jsonRequest(LinksRequestSchema), + responses: { + 200: jsonResponse(LinksResponseSchema, "Page links."), + ...commonResponses, + }, +}); + +/** Creates the in-process Hono application used by `web serve` and tests. */ +export function createHttpApp( + dependencies: HttpServiceDependencies = {}, +): OpenAPIHono { + const state = resolveHttpServiceState(dependencies); + const app = new OpenAPIHono({ + defaultHook: (result, context) => { + if (result.success) return; + return context.json( + errorBody("invalid_request", "Request validation failed"), + 400, + ); + }, + }); + + app.openapi(searchRoute, async (context) => { + const input = context.req.valid("json"); + try { + const result = await searchWithFallback( + state, + input.query, + context.req.raw.signal, + ); + return context.json(parseResponse(SearchResponseSchema, result), 200); + } catch (error) { + return failureResponse(context, error, "search") as never; + } + }); + + app.openapi(fetchRoute, async (context) => { + const input = context.req.valid("json"); + const renderError = validateRenderFields(input.render, input.waitMs); + if (renderError) return context.json(renderError, 400) as never; + try { + const result = await state.operations.fetch( + input, + context.req.raw.signal, + ); + throwIfAborted(context.req.raw.signal); + return context.json(parseResponse(FetchResponseSchema, result), 200); + } catch (error) { + return failureResponse(context, error, "fetch") as never; + } + }); + + app.openapi(linksRoute, async (context) => { + const input = context.req.valid("json"); + const renderError = validateRenderFields(input.render, input.waitMs); + if (renderError) return context.json(renderError, 400) as never; + try { + const result = await state.operations.links( + { ...input, limit: input.limit ?? DEFAULT_LINK_LIMIT }, + context.req.raw.signal, + ); + throwIfAborted(context.req.raw.signal); + return context.json(parseResponse(LinksResponseSchema, result), 200); + } catch (error) { + return failureResponse(context, error, "links") as never; + } + }); + + app.onError((error, context) => failureResponse(context, error, "request")); + app.notFound((context) => + context.json(errorBody("not_found", "Route not found"), 404), + ); + return app; +} + +/** Builds the OpenAPI 3.1 document from the registered route schemas. */ +export function createHttpOpenAPIDocument(version = "0.1.0") { + const app = createHttpApp({ + credentials: { exaApiKey: "build-placeholder" }, + keposBridgeEndpoint: DEFAULT_KEPOS_BRIDGE_ENDPOINT, + validateStartup: false, + }); + return app.getOpenAPI31Document({ + openapi: "3.1.0", + info: { + title: "Guion Web Personal HTTP Service", + version, + description: + "Provider-neutral Guion Web search, page fetch, and link discovery.", + }, + }); +} + +export function resolveHttpServiceState( + dependencies: HttpServiceDependencies = {}, +): HttpServiceState { + const environment = dependencies.environment ?? process.env; + const credentials = resolveCredentials(dependencies.credentials, environment); + if (dependencies.validateStartup !== false) { + if ( + typeof credentials.exaApiKey !== "string" || + credentials.exaApiKey.trim().length === 0 + ) { + throw new Error( + "EXA_API_KEY is required and must be non-empty for HTTP service", + ); + } + } + const endpoint = + dependencies.keposBridgeEndpoint ?? + environment.KEPOS_BRIDGE_ENDPOINT ?? + DEFAULT_KEPOS_BRIDGE_ENDPOINT; + return { + operations: dependencies.operations ?? webCoreModule.createWebOperations(), + credentials, + keposBridgeEndpoint: validateKeposBridgeEndpoint(endpoint), + }; +} + +function resolveCredentials( + value: HttpServiceDependencies["credentials"], + environment: NodeJS.ProcessEnv, +): WebCredentials { + const credentials = + typeof value === "function" + ? value() + : (value ?? credentialsFromEnvironment(environment)); + return { ...credentials }; +} + +function jsonRequest(schema: T) { + return { + body: { + required: true, + content: { "application/json": { schema } }, + }, + } as const; +} + +function jsonResponse(schema: T, description: string) { + return { + content: { "application/json": { schema } }, + description, + } as const; +} + +function isHttpURL(value: string): boolean { + try { + const url = new URL(value); + return url.protocol === "http:" || url.protocol === "https:"; + } catch { + return false; + } +} + +function validateRenderFields( + render: "http" | "browser" | undefined, + waitMs: number | undefined, +): HttpError | undefined { + const selected = render ?? "http"; + if (selected === "http" && waitMs !== undefined) + return errorBody( + "invalid_request", + "waitMs is only valid with render browser", + ); + if (selected === "browser" && waitMs === undefined) + return errorBody( + "invalid_request", + "waitMs is required with render browser", + ); + return undefined; +} + +async function searchWithFallback( + state: HttpServiceState, + query: string, + signal: AbortSignal, +): Promise { + throwIfAborted(signal); + try { + const result = await state.operations.search({ + query, + provider: "kepos-bridge", + credentials: state.credentials, + keposBridgeEndpoint: state.keposBridgeEndpoint, + allowEmptyKeposResults: true, + signal, + }); + throwIfAborted(signal); + return parseResponse(SearchResponseSchema, result); + } catch (bridgeError) { + if (isCancellation(bridgeError, signal)) throw bridgeError; + throwIfAborted(signal); + try { + const result = await state.operations.search({ + query, + provider: "exa", + credentials: state.credentials, + signal, + }); + throwIfAborted(signal); + return parseResponse(SearchResponseSchema, result); + } catch (exaError) { + if (isCancellation(exaError, signal)) throw exaError; + if (isRequestTimeout(exaError)) throw exaError; + throw new Error("Search upstream providers failed"); + } + } +} + +function parseResponse(schema: z.ZodType, value: unknown): T { + const parsed = schema.safeParse(value); + if (!parsed.success) throw new Error("malformed operation response"); + return parsed.data; +} + +function failureResponse( + context: { json: (body: HttpError, status: number) => Response }, + error: unknown, + operation: string, +): Response { + const mapped = classifyFailure(error, operation); + return context.json(mapped.body, mapped.status); +} + +function classifyFailure( + error: unknown, + operation: string, +): { status: 400 | 499 | 502 | 504; body: HttpError } { + if ( + error instanceof HTTPException && + error.status >= 400 && + error.status < 500 + ) + return { + status: 400, + body: errorBody("invalid_request", "Request body is invalid"), + }; + if (error instanceof SyntaxError) + return { + status: 400, + body: errorBody("invalid_request", "Request body is not valid JSON"), + }; + if (error instanceof FetchCapabilityError) { + return { + status: 502, + body: { + code: error.code, + message: `${operation} requires an explicit capability retry`, + details: safeFetchDetails(error.details), + }, + }; + } + if (isOperationAborted(error) || isAbortError(error)) + return { + status: 499, + body: errorBody("request_cancelled", "Request cancelled"), + }; + if (isRequestTimeout(error)) + return { + status: 504, + body: errorBody("upstream_timeout", `${operation} timed out`), + }; + return { + status: 502, + body: errorBody("upstream_error", `${operation} failed`), + }; +} + +function errorBody(code: string, message: string): HttpError { + return { code, message }; +} + +function isCancellation(error: unknown, signal: AbortSignal): boolean { + return signal.aborted || isOperationAborted(error) || isAbortError(error); +} + +function isAbortError(error: unknown): boolean { + return ( + (error instanceof DOMException && error.name === "AbortError") || + (error instanceof Error && + (error.name === "AbortError" || error.message === "Operation aborted")) + ); +} + +function safeFetchDetails( + details: FetchCapabilityError["details"], +): Record | undefined { + const safe: Record = {}; + if (typeof details.retryableWithRender === "boolean") + safe.retryableWithRender = details.retryableWithRender; + if (typeof details.retryable === "boolean") + safe.retryable = details.retryable; + if ( + details.suggestedArguments?.render === "browser" && + details.suggestedArguments.waitMs === 2000 + ) { + safe.suggestedArguments = { render: "browser", waitMs: 2000 }; + } + if (details.reportUrl === RENDER_REPORT_URL) + safe.reportUrl = details.reportUrl; + if ( + typeof details.blockedHostname === "string" && + /^[a-z0-9.-]+$/i.test(details.blockedHostname) && + details.blockedHostname.length <= 253 + ) { + safe.blockedHostname = details.blockedHostname.toLowerCase(); + } + return Object.keys(safe).length > 0 ? safe : undefined; +} + +export { + ErrorSchema, + FetchRequestSchema, + FetchResponseSchema, + LinksRequestSchema, + LinksResponseSchema, + SearchRequestSchema, + SearchResponseSchema, +}; diff --git a/packages/web/src/mcp.ts b/packages/web/src/mcp.ts index 36e5c7b..e3e0f08 100644 --- a/packages/web/src/mcp.ts +++ b/packages/web/src/mcp.ts @@ -17,22 +17,18 @@ import { type WebOperations, } from "@guionai/web-core"; -const DEFAULT_FETCH_TREE_THRESHOLD = 5000; - type SearchToolInput = { query: string }; type FetchToolInput = { url: string; - tree?: boolean; section_id?: string; full?: boolean; - tree_threshold?: number; - render?: "fetch" | "agent-browser"; + render?: "http" | "browser"; waitMs?: number; }; type LinksToolInput = { url: string; limit?: number; - render?: "fetch" | "agent-browser"; + render?: "http" | "browser"; waitMs?: number; }; type DocsResolveToolInput = { query: string }; @@ -65,45 +61,47 @@ const fetchInputSchema = schema({ additionalProperties: false, properties: { url: { type: "string", description: "HTTP or HTTPS URL to fetch" }, - tree: { type: "boolean", description: "show the page heading tree" }, section_id: { type: "string", + minLength: 1, description: "optional heading section ID to return", }, full: { type: "boolean", description: "return full content without automatic tree mode", }, - tree_threshold: { - type: "integer", - description: "automatic tree threshold; defaults to 5000", - default: DEFAULT_FETCH_TREE_THRESHOLD, - }, render: { type: "string", - enum: ["fetch", "agent-browser"], - default: "fetch", - description: "optional page-fetch backend; direct fetch is the default", + enum: ["http", "browser"], + default: "http", + description: "optional page renderer; HTTP fetching is the default", }, waitMs: { type: "integer", minimum: 0, maximum: 30_000, - description: - "required post-load wait for agent-browser rendering (0-30000)", + description: "required post-load wait for browser rendering (0-30000)", }, }, required: ["url"], oneOf: [ { - properties: { render: { enum: ["fetch"] } }, + properties: { render: { enum: ["http"] } }, not: { required: ["waitMs"] }, }, { - properties: { render: { const: "agent-browser" } }, + properties: { render: { const: "browser" } }, required: ["render", "waitMs"], }, ], + allOf: [ + { + not: { + required: ["section_id", "full"], + properties: { full: { const: true } }, + }, + }, + ], }); const linksInputSchema = schema({ type: "object", @@ -119,26 +117,25 @@ const linksInputSchema = schema({ }, render: { type: "string", - enum: ["fetch", "agent-browser"], - default: "fetch", - description: "optional page-fetch backend; direct fetch is the default", + enum: ["http", "browser"], + default: "http", + description: "optional page renderer; HTTP fetching is the default", }, waitMs: { type: "integer", minimum: 0, maximum: 30_000, - description: - "required post-load wait for agent-browser rendering (0-30000)", + description: "required post-load wait for browser rendering (0-30000)", }, }, required: ["url"], oneOf: [ { - properties: { render: { enum: ["fetch"] } }, + properties: { render: { enum: ["http"] } }, not: { required: ["waitMs"] }, }, { - properties: { render: { const: "agent-browser" } }, + properties: { render: { const: "browser" } }, required: ["render", "waitMs"], }, ], @@ -309,7 +306,7 @@ export function createMcpServer(dependencies: McpDependencies): McpServer { "links", toolConfig( "List page links", - "List HTTP or HTTPS links from a static page or explicit agent-browser rendering.", + "List HTTP or HTTPS links from an HTTP or browser-rendered page.", linksInputSchema, linksOutputSchema, ), @@ -333,23 +330,18 @@ export function createMcpServer(dependencies: McpDependencies): McpServer { "fetch", toolConfig( "Fetch a web page", - "Use direct fetch (omit render or set render: fetch) for static, SSR, and pre-rendered pages. For client-rendered or SPA pages, set render: agent-browser with required waitMs on a host that has agent-browser installed; there is no automatic fallback.", + "Use the default HTTP renderer for static, SSR, and pre-rendered pages. For client-rendered or SPA pages, set render: browser with required waitMs when the host provides browser capability; there is no automatic fallback.", fetchInputSchema, fetchOutputSchema, ), - async ( - { url, tree, section_id, full, tree_threshold, render, waitMs }, - context, - ) => + async ({ url, section_id, full, render, waitMs }, context) => runTool( () => dependencies.operations.fetch( { url, - tree: tree ?? false, - section_id, - full: full ?? false, - tree_threshold: tree_threshold ?? DEFAULT_FETCH_TREE_THRESHOLD, + ...(section_id !== undefined ? { section_id } : {}), + ...(full !== undefined ? { full } : {}), ...(render !== undefined ? { render } : {}), ...(waitMs !== undefined ? { waitMs } : {}), }, @@ -506,10 +498,10 @@ function safeFetchErrorDetails(details: FetchErrorDetails): FetchErrorDetails { if (typeof details.retryable === "boolean") safe.retryable = details.retryable; if ( - details.suggestedArguments?.render === "agent-browser" && + details.suggestedArguments?.render === "browser" && details.suggestedArguments.waitMs === 2000 ) { - safe.suggestedArguments = { render: "agent-browser", waitMs: 2000 }; + safe.suggestedArguments = { render: "browser", waitMs: 2000 }; } if (details.reportUrl === RENDER_REPORT_URL) safe.reportUrl = details.reportUrl; diff --git a/packages/web/src/program.ts b/packages/web/src/program.ts index aa8ca79..f72773b 100644 --- a/packages/web/src/program.ts +++ b/packages/web/src/program.ts @@ -1,6 +1,8 @@ import { Command } from "commander"; import { createMcpCommand } from "./mcp.js"; +import { parseHttpPort, startHttpServer } from "./serve.js"; +import { DEFAULT_HTTP_HOST, DEFAULT_HTTP_PORT } from "./serve.js"; import { formatSearchResults, @@ -28,10 +30,41 @@ export function createProgram(dependencies: ProgramDependencies): Command { .addCommand(createLinksCommand(dependencies)) .addCommand(createDocsCommand(dependencies)) .addCommand(createSGraphCommand(dependencies)) + .addCommand(createServeCommand(dependencies)) .addCommand(createMcpCommand(dependencies)); return program; } +function createServeCommand(dependencies: ProgramDependencies): Command { + const writeOut = + dependencies.writeOut ?? ((text: string) => process.stdout.write(text)); + return new Command("serve") + .description("Serve all web research operations over HTTP") + .option("--host ", "HTTP listen hostname", DEFAULT_HTTP_HOST) + .option( + "--port ", + "HTTP listen port", + parseHttpPort, + DEFAULT_HTTP_PORT, + ) + .action((options: { host: string; port: number }) => { + startHttpServer( + { + operations: dependencies.operations, + credentials: dependencies.credentials, + }, + { + hostname: options.host, + port: options.port, + onListening: ({ hostname, port }) => + writeOut( + `Guion Web HTTP service listening on ${hostname}:${port}\n`, + ), + }, + ); + }); +} + function createSearchCommand(dependencies: ProgramDependencies): Command { const writeOut = dependencies.writeOut ?? ((text: string) => process.stdout.write(text)); @@ -204,45 +237,36 @@ function createFetchCommand(dependencies: ProgramDependencies): Command { return new Command("fetch") .description("Fetch a static, SSR, or pre-rendered page as Markdown") .argument("", "HTTP or HTTPS URL") - .option("--tree", "Show the heading tree") - .option( - "--render ", - "Rendering backend: fetch (default) or agent-browser", - ) + .option("--render ", "Rendering mode: http (default) or browser") .option( "--wait ", - "Required post-load wait for --render agent-browser (0-30000)", + "Required post-load wait for --render browser (0-30000)", Number, ) .option("-s, --section ", "Read one heading section") - .option( - "--full", - "Return full content instead of automatic tree navigation", - ) - .option("--tree-threshold ", "Auto-tree threshold", Number) + .option("--full", "Return the complete extracted Markdown") .option("--json", "Output the structured result as JSON") .action( async ( url: string, options: { - tree?: boolean; - render?: "fetch" | "agent-browser"; + render?: "http" | "browser"; wait?: number; section?: string; full?: boolean; - treeThreshold?: number; json?: boolean; }, ) => { - const result = await dependencies.operations.fetch({ + const input = { url, - tree: options.tree, ...(options.render !== undefined ? { render: options.render } : {}), ...(options.wait !== undefined ? { waitMs: options.wait } : {}), - section_id: options.section, - full: options.full, - tree_threshold: options.treeThreshold, - }); + ...(options.section !== undefined + ? { section_id: options.section } + : {}), + ...(options.full !== undefined ? { full: options.full } : {}), + }; + const result = await dependencies.operations.fetch(input); if (options.json) { writeOut(JSON.stringify(result) + "\n"); return; @@ -259,13 +283,10 @@ function createLinksCommand(dependencies: ProgramDependencies): Command { .description("List HTTP(S) links from a web page") .argument("", "HTTP or HTTPS URL") .option("--limit ", "Maximum links to return (1-100)", Number) - .option( - "--render ", - "Rendering backend: fetch (default) or agent-browser", - ) + .option("--render ", "Rendering mode: http (default) or browser") .option( "--wait ", - "Required post-load wait for --render agent-browser (0-30000)", + "Required post-load wait for --render browser (0-30000)", Number, ) .option("--json", "Output the structured result as JSON") @@ -274,7 +295,7 @@ function createLinksCommand(dependencies: ProgramDependencies): Command { url: string, options: { limit?: number; - render?: "fetch" | "agent-browser"; + render?: "http" | "browser"; wait?: number; json?: boolean; }, diff --git a/packages/web/src/runner.ts b/packages/web/src/runner.ts index c5fdd4b..bf45587 100644 --- a/packages/web/src/runner.ts +++ b/packages/web/src/runner.ts @@ -38,7 +38,7 @@ function formatCliError(error: unknown): string { if (error.code === "javascript_rendering_may_be_required") { return ( "javascript_rendering_may_be_required: content may require JavaScript rendering\n" + - "Retry: web fetch --render=agent-browser --wait=2000\n" + "Retry: web fetch --render=browser --wait=2000\n" ); } if (error.code === "render_domain_not_allowed") { diff --git a/packages/web/src/serve.ts b/packages/web/src/serve.ts new file mode 100644 index 0000000..976a527 --- /dev/null +++ b/packages/web/src/serve.ts @@ -0,0 +1,38 @@ +import { serve, type ServerType } from "@hono/node-server"; + +import { createHttpApp, type HttpServiceDependencies } from "./http.js"; + +export const DEFAULT_HTTP_HOST = "0.0.0.0"; +export const DEFAULT_HTTP_PORT = 8787; + +export type HttpServerOptions = { + hostname?: string; + port?: number; + onListening?: (address: { hostname: string; port: number }) => void; +}; + +/** Starts the Hono HTTP service on a Node HTTP server. */ +export function startHttpServer( + dependencies: HttpServiceDependencies = {}, + options: HttpServerOptions = {}, +): ServerType { + const app = createHttpApp(dependencies); + const hostname = options.hostname ?? DEFAULT_HTTP_HOST; + const port = options.port ?? DEFAULT_HTTP_PORT; + if (!Number.isInteger(port) || port < 1 || port > 65_535) + throw new Error("HTTP port must be an integer from 1 through 65535"); + return serve({ fetch: app.fetch, hostname, port }, (address) => + options.onListening?.({ + hostname: + typeof address.address === "string" ? address.address : hostname, + port: address.port, + }), + ); +} + +export function parseHttpPort(value: string): number { + const port = Number(value); + if (!Number.isInteger(port) || port < 1 || port > 65_535) + throw new Error("HTTP port must be an integer from 1 through 65535"); + return port; +} diff --git a/packages/web/test/http.test.ts b/packages/web/test/http.test.ts new file mode 100644 index 0000000..f0ca31a --- /dev/null +++ b/packages/web/test/http.test.ts @@ -0,0 +1,308 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + createHttpApp, + createHttpOpenAPIDocument, + type HttpServiceDependencies, +} from "../src/http.js"; +import { RequestTimeoutError, type WebOperations } from "@guionai/web-core"; + +function operations(overrides: Partial = {}): WebOperations { + return { + search: vi.fn(async () => ({ + provider: "Kepos Bridge" as const, + results: [], + })), + fetch: vi.fn(async (input) => ({ + url: input.url, + mode: "full" as const, + content: "page", + })), + links: vi.fn(async (input) => ({ + url: input.url, + links: [], + truncated: false, + })), + docsResolve: vi.fn(async (input) => ({ + query: input.query, + libraries: [], + })), + docsFetch: vi.fn(async (input) => ({ + library_id: input.library_id, + content: "docs", + })), + sgraphSearch: vi.fn(async () => ({ content: "source" })), + keposBridge: vi.fn(async () => ({ output: "bridge", results: [] })), + ...overrides, + }; +} + +function dependencies( + overrides: Partial = {}, +): HttpServiceDependencies { + return { + operations: operations(), + credentials: { exaApiKey: "exa-secret" }, + keposBridgeEndpoint: "http://bridge.test/route", + ...overrides, + }; +} + +async function json( + app: ReturnType, + path: string, + body: unknown, +): Promise<{ response: Response; body: unknown }> { + const response = await app.request(path, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + return { response, body: await response.json() }; +} + +describe("personal HTTP service", () => { + it("returns an empty successful Bridge search without retrying Exa", async () => { + const ops = operations(); + const app = createHttpApp({ ...dependencies(), operations: ops }); + + const result = await json(app, "/v1/search", { query: "empty" }); + + expect(result.response.status).toBe(200); + expect(result.body).toEqual({ provider: "Kepos Bridge", results: [] }); + expect(ops.search).toHaveBeenCalledTimes(1); + expect(ops.search).toHaveBeenCalledWith( + expect.objectContaining({ + query: "empty", + provider: "kepos-bridge", + keposBridgeEndpoint: "http://bridge.test/route", + allowEmptyKeposResults: true, + }), + ); + }); + + it("retries Exa exactly once after a Bridge failure", async () => { + const ops = operations({ + search: vi + .fn() + .mockRejectedValueOnce(new Error("bridge unavailable")) + .mockResolvedValueOnce({ provider: "Exa", results: [] }), + }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + + const result = await json(app, "/v1/search", { query: "fallback" }); + + expect(result.response.status).toBe(200); + expect(result.body).toEqual({ provider: "Exa", results: [] }); + expect(ops.search).toHaveBeenCalledTimes(2); + expect(ops.search).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ provider: "exa", query: "fallback" }), + ); + }); + + it("does not retry a cancelled Bridge request", async () => { + const controller = new AbortController(); + const ops = operations({ + search: vi.fn(async () => { + controller.abort(); + throw new Error("Operation aborted"); + }), + }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + + const response = await app.request("/v1/search", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ query: "cancel" }), + signal: controller.signal, + }); + + expect(response.status).toBe(499); + expect(ops.search).toHaveBeenCalledTimes(1); + }); + + it("maps an Exa timeout after Bridge failure to a documented timeout", async () => { + const ops = operations({ + search: vi + .fn() + .mockRejectedValueOnce(new Error("bridge unavailable")) + .mockRejectedValueOnce(new RequestTimeoutError("exa timed out", true)), + }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + + const result = await json(app, "/v1/search", { query: "timeout" }); + + expect(result.response.status).toBe(504); + expect(result.body).toEqual({ + code: "upstream_timeout", + message: "search timed out", + }); + }); + + it("forwards fetch with its explicit rendered-fetch contract", async () => { + const ops = operations({ + fetch: vi.fn(async (input) => ({ + url: input.url, + mode: "tree" as const, + content: "markdown", + })), + }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + + const result = await json(app, "/v1/fetch", { + url: "https://example.test", + render: "browser", + waitMs: 0, + }); + + expect(result.response.status).toBe(200); + expect(result.body).toMatchObject({ mode: "tree" }); + expect(ops.fetch).toHaveBeenCalledWith( + expect.objectContaining({ + url: "https://example.test", + full: false, + render: "browser", + waitMs: 0, + }), + expect.any(AbortSignal), + ); + }); + + it("forwards links with the same explicit rendered-fetch contract", async () => { + const ops = operations({ + links: vi.fn(async (input) => ({ + url: input.url, + links: [{ text: "Docs", url: "https://example.test/docs" }], + truncated: false, + })), + }); + const app = createHttpApp({ ...dependencies(), operations: ops }); + + const result = await json(app, "/v1/links", { + url: "https://example.test", + render: "browser", + waitMs: 0, + }); + + expect(result.response.status).toBe(200); + expect(result.body).toMatchObject({ + links: [{ text: "Docs", url: "https://example.test/docs" }], + }); + expect(ops.links).toHaveBeenCalledWith( + { + url: "https://example.test", + limit: 100, + render: "browser", + waitMs: 0, + }, + expect.any(AbortSignal), + ); + }); + + it("rejects invalid search and fetch requests before an operation", async () => { + const ops = operations(); + const app = createHttpApp({ ...dependencies(), operations: ops }); + + expect((await json(app, "/v1/search", { query: "" })).response.status).toBe( + 400, + ); + expect( + ( + await json(app, "/v1/fetch", { + url: "https://example.test", + waitMs: 100, + }) + ).response.status, + ).toBe(400); + expect( + ( + await json(app, "/v1/links", { + url: "https://example.test", + waitMs: 100, + }) + ).response.status, + ).toBe(400); + expect( + ( + await json(app, "/v1/fetch", { + url: "https://example.test", + full: true, + section_id: "intro", + }) + ).response.status, + ).toBe(400); + expect( + ( + await json(app, "/v1/fetch", { + url: "https://example.test", + tree: true, + }) + ).response.status, + ).toBe(400); + expect(ops.search).not.toHaveBeenCalled(); + expect(ops.fetch).not.toHaveBeenCalled(); + expect(ops.links).not.toHaveBeenCalled(); + + const malformed = await app.request("/v1/search", { + method: "POST", + headers: { "content-type": "application/json" }, + body: "not-json", + }); + expect(malformed.status).toBe(400); + expect(await malformed.json()).toMatchObject({ code: "invalid_request" }); + }); + + it.each([ + "/v1/docs/resolve", + "/v1/docs/fetch", + "/v1/source-search", + "/v1/weather", + "/v1/sports", + "/v1/finance", + "/v1/time", + ])("does not expose %s", async (path) => { + const ops = operations(); + const app = createHttpApp({ ...dependencies(), operations: ops }); + + const result = await json(app, path, {}); + + expect(result.response.status).toBe(404); + expect(result.body).toEqual({ + code: "not_found", + message: "Route not found", + }); + for (const operation of Object.values(ops)) + expect(operation).not.toHaveBeenCalled(); + }); + + it("exposes search, fetch, and links in a generated OpenAPI 3.1 document", () => { + const document = createHttpOpenAPIDocument("1.2.3"); + + expect(document.openapi).toBe("3.1.0"); + expect(document.info.version).toBe("1.2.3"); + expect(Object.keys(document.paths ?? {}).sort()).toEqual([ + "/v1/fetch", + "/v1/links", + "/v1/search", + ]); + expect( + (document.components?.schemas as any).SearchResponse.properties.provider + .enum, + ).toEqual(["Exa", "Kepos Bridge"]); + const fetchRequest = (document.components?.schemas as any).FetchRequest; + expect(fetchRequest.properties.render.enum).toEqual(["http", "browser"]); + expect(fetchRequest.properties.tree).toBeUndefined(); + expect(fetchRequest.properties.tree_threshold).toBeUndefined(); + }); + + it("requires an Exa key and validates the server-local Bridge endpoint", () => { + expect(() => createHttpApp({ credentials: {} })).toThrow("EXA_API_KEY"); + expect(() => + createHttpApp({ + credentials: { exaApiKey: "key" }, + keposBridgeEndpoint: "https://user:pass@example.test/route", + }), + ).toThrow("Kepos Bridge endpoint"); + }); +}); diff --git a/packages/web/test/mcp.test.ts b/packages/web/test/mcp.test.ts index 561ccfd..5bf6636 100644 --- a/packages/web/test/mcp.test.ts +++ b/packages/web/test/mcp.test.ts @@ -155,7 +155,7 @@ describe("web stdio MCP adapter", () => { .mockRejectedValueOnce( new FetchCapabilityError("javascript_rendering_may_be_required", { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }), ) .mockResolvedValueOnce({ @@ -174,7 +174,7 @@ describe("web stdio MCP adapter", () => { name: "fetch", arguments: { url: "https://example.test/page", - render: "agent-browser", + render: "browser", waitMs: 2000, }, }); @@ -185,7 +185,7 @@ describe("web stdio MCP adapter", () => { code: "javascript_rendering_may_be_required", details: { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }, }, }); @@ -196,11 +196,7 @@ describe("web stdio MCP adapter", () => { 2, { url: "https://example.test/page", - tree: false, - full: false, - section_id: undefined, - tree_threshold: 5000, - render: "agent-browser", + render: "browser", waitMs: 2000, }, expect.any(AbortSignal), @@ -250,10 +246,9 @@ describe("web stdio MCP adapter", () => { string, unknown >; - expect(fetchProperties.tree_threshold).toMatchObject({ default: 5000 }); expect(fetchProperties.render).toMatchObject({ - enum: ["fetch", "agent-browser"], - default: "fetch", + enum: ["http", "browser"], + default: "http", }); expect(fetchProperties.waitMs).toMatchObject({ type: "integer", @@ -267,8 +262,8 @@ describe("web stdio MCP adapter", () => { maximum: 100, }); expect(linksProperties.render).toMatchObject({ - enum: ["fetch", "agent-browser"], - default: "fetch", + enum: ["http", "browser"], + default: "http", }); expect(docsFetchProperties.tokens).toMatchObject({ default: 0 }); expect(sgraphProperties).toMatchObject({ @@ -293,8 +288,7 @@ describe("web stdio MCP adapter", () => { name: "fetch", arguments: { url: "https://example.test/page", - tree: true, - render: "agent-browser", + render: "browser", waitMs: 125, }, }); @@ -303,7 +297,7 @@ describe("web stdio MCP adapter", () => { arguments: { url: "https://example.test/page", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 125, }, }); @@ -347,11 +341,7 @@ describe("web stdio MCP adapter", () => { expect(operations.fetch).toHaveBeenCalledWith( { url: "https://example.test/page", - tree: true, - full: false, - section_id: undefined, - tree_threshold: 5000, - render: "agent-browser", + render: "browser", waitMs: 125, }, expect.any(AbortSignal), @@ -360,7 +350,7 @@ describe("web stdio MCP adapter", () => { { url: "https://example.test/page", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 125, }, expect.any(AbortSignal), @@ -387,17 +377,17 @@ describe("web stdio MCP adapter", () => { await client.listTools(); for (const arguments_ of [ - { url: "https://example.test/page", render: "agent-browser" }, - { url: "https://example.test/page", render: "fetch", waitMs: 0 }, - { url: "https://example.test/page", render: "agent-browser", waitMs: -1 }, + { url: "https://example.test/page", render: "browser" }, + { url: "https://example.test/page", render: "http", waitMs: 0 }, + { url: "https://example.test/page", render: "browser", waitMs: -1 }, { url: "https://example.test/page", - render: "agent-browser", + render: "browser", waitMs: 30_001, }, { url: "https://example.test/page", - render: "agent-browser", + render: "browser", waitMs: 1.5, }, ]) { @@ -437,7 +427,7 @@ describe("web stdio MCP adapter", () => { name: "fetch", arguments: { url: "https://example.test/page", - render: "agent-browser", + render: "browser", waitMs: 2000, }, }); @@ -549,7 +539,7 @@ describe("web stdio MCP adapter", () => { name: "fetch", arguments: { url: "https://example.test/page", - render: "agent-browser", + render: "browser", waitMs: 0, }, }, diff --git a/packages/web/test/openapi-artifact.test.ts b/packages/web/test/openapi-artifact.test.ts new file mode 100644 index 0000000..c37cf52 --- /dev/null +++ b/packages/web/test/openapi-artifact.test.ts @@ -0,0 +1,43 @@ +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { parse } from "yaml"; +import { describe, expect, it } from "vitest"; + +import { generateOpenAPI } from "../src/generate-openapi.js"; + +const packageRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); +const expectedPaths = ["/v1/fetch", "/v1/links", "/v1/search"]; + +describe("release OpenAPI artifact", () => { + it("serializes and parses the versioned contract from a test-owned file", async () => { + const artifactRoot = mkdtempSync( + join(tmpdir(), "guionai-web-openapi-artifact-"), + ); + try { + const outputPath = join(artifactRoot, "openapi.yaml"); + await generateOpenAPI(outputPath); + const document = parse(readFileSync(outputPath, "utf8")) as { + openapi?: unknown; + info?: { version?: unknown }; + paths?: Record; + }; + const manifest = JSON.parse( + readFileSync(join(packageRoot, "package.json"), "utf8"), + ) as { version: string }; + + expect(document.openapi).toBe("3.1.0"); + expect(document.info?.version).toBe(manifest.version); + expect(Object.keys(document.paths ?? {}).sort()).toEqual(expectedPaths); + expect( + Object.keys(document.paths ?? {}).some((path) => + path.includes("bridge"), + ), + ).toBe(false); + } finally { + rmSync(artifactRoot, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/web/test/packed-smoke.mjs b/packages/web/test/packed-smoke.mjs index 9fd18f6..512ed41 100644 --- a/packages/web/test/packed-smoke.mjs +++ b/packages/web/test/packed-smoke.mjs @@ -229,7 +229,7 @@ else console.log(JSON.stringify({ success: true, data: {} })); [ "fetch", "https://93.184.216.34/rendered", - "--render=agent-browser", + "--render=browser", "--wait=0", "--full", "--json", @@ -261,7 +261,7 @@ else console.log(JSON.stringify({ success: true, data: {} })); [ "fetch", "https://93.184.216.34/blocked", - "--render=agent-browser", + "--render=browser", "--wait=0", ], { cwd: root, env: fakeEnvironment }, diff --git a/packages/web/test/program.test.ts b/packages/web/test/program.test.ts index 71062b9..e061736 100644 --- a/packages/web/test/program.test.ts +++ b/packages/web/test/program.test.ts @@ -184,24 +184,13 @@ describe("web search Commander adapter", () => { it("passes fetch navigation flags and writes human Markdown", async () => { const { program, operations, output } = setup(); await program.parseAsync( - [ - "fetch", - "https://example.test/page", - "--tree", - "-s", - "7i", - "--tree-threshold", - "9000", - ], + ["fetch", "https://example.test/page", "-s", "7i"], { from: "user" }, ); expect(operations.fetch).toHaveBeenCalledWith({ url: "https://example.test/page", - tree: true, section_id: "7i", - full: undefined, - tree_threshold: 9000, }); expect(output()).toEqual({ stdout: "# Fixture page\n", stderr: "" }); }); @@ -209,23 +198,14 @@ describe("web search Commander adapter", () => { it("forwards explicit browser rendering options", async () => { const { program, operations } = setup(); await program.parseAsync( - [ - "fetch", - "https://example.test/page", - "--render=agent-browser", - "--wait=0", - ], + ["fetch", "https://example.test/page", "--render=browser", "--wait=0"], { from: "user" }, ); expect(operations.fetch).toHaveBeenCalledWith({ url: "https://example.test/page", - tree: undefined, - render: "agent-browser", + render: "browser", waitMs: 0, - section_id: undefined, - full: undefined, - tree_threshold: undefined, }); }); @@ -238,10 +218,7 @@ describe("web search Commander adapter", () => { expect(operations.fetch).toHaveBeenCalledWith({ url: "https://example.test/page", - tree: undefined, - section_id: undefined, full: true, - tree_threshold: undefined, }); expect(JSON.parse(output().stdout)).toEqual( await operations.fetch.mock.results[0]!.value, @@ -257,7 +234,7 @@ describe("web search Commander adapter", () => { "https://example.test/page", "--limit", "25", - "--render=agent-browser", + "--render=browser", "--wait=0", ], { from: "user" }, @@ -266,7 +243,7 @@ describe("web search Commander adapter", () => { expect(operations.links).toHaveBeenCalledWith({ url: "https://example.test/page", limit: 25, - render: "agent-browser", + render: "browser", waitMs: 0, }); expect(output()).toEqual({ @@ -289,7 +266,7 @@ describe("web search Commander adapter", () => { fetch: vi.fn(async () => { throw new FetchCapabilityError("javascript_rendering_may_be_required", { retryableWithRender: true, - suggestedArguments: { render: "agent-browser", waitMs: 2000 }, + suggestedArguments: { render: "browser", waitMs: 2000 }, }); }), links: vi.fn(), @@ -317,7 +294,7 @@ describe("web search Commander adapter", () => { expect(stdout).toBe(""); expect(stderr).toBe( "javascript_rendering_may_be_required: content may require JavaScript rendering\n" + - "Retry: web fetch --render=agent-browser --wait=2000\n", + "Retry: web fetch --render=browser --wait=2000\n", ); }); diff --git a/packages/web/tsup.config.ts b/packages/web/tsup.config.ts index c066be8..6fbddd9 100644 --- a/packages/web/tsup.config.ts +++ b/packages/web/tsup.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "tsup"; export default defineConfig({ - entry: { cli: "src/cli.ts" }, + entry: { cli: "src/cli.ts", "generate-openapi": "src/generate-openapi.ts" }, format: ["esm"], platform: "node", target: "node20", @@ -16,6 +16,11 @@ export default defineConfig({ "defuddle", "linkedom", "markdown-it", + "@hono/node-server", + "@hono/zod-openapi", + "hono", + "zod", + "yaml", ], banner: { js: 'import { createRequire } from "node:module"; const require = createRequire(import.meta.url);', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 888db7f..a3dfcf8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -122,9 +122,24 @@ importers: packages/web: dependencies: + '@hono/node-server': + specifier: 2.1.1 + version: 2.1.1(hono@4.13.5) + '@hono/zod-openapi': + specifier: 1.6.1 + version: 1.6.1(hono@4.13.5)(zod@4.4.3) commander: specifier: 12.1.0 version: 12.1.0 + hono: + specifier: 4.13.5 + version: 4.13.5 + yaml: + specifier: 2.9.0 + version: 2.9.0 + zod: + specifier: 4.4.3 + version: 4.4.3 devDependencies: '@modelcontextprotocol/server': specifier: 2.0.0 @@ -153,6 +168,11 @@ packages: zod: optional: true + '@asteasolutions/zod-to-openapi@9.1.0': + resolution: {integrity: sha512-pLMeRgRYS7/vZIgAAkOe2P6+XGTifR1MT9pqDoxZ11EmcJJ+DPmdPqLN8ZZF4U8R9KHCCQCS9c2wtuE8wSrfkw==} + peerDependencies: + zod: ^4.0.0 + '@aws-crypto/sha256-browser@5.2.0': resolution: {integrity: sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==} @@ -809,6 +829,25 @@ packages: '@modelcontextprotocol/sdk': optional: true + '@hono/node-server@2.1.1': + resolution: {integrity: sha512-ELuehkj5VCBdgEw9zs+ivkKwyzzUCSQuE96YmiPvn1ECBoZCczbFXJLeEGMTYjphP6gydh4pHMqEYPVMYUVgQg==} + engines: {node: '>=20'} + peerDependencies: + hono: ^4 + + '@hono/zod-openapi@1.6.1': + resolution: {integrity: sha512-z1xS3FZxl4bBkU3kMIsQsLtsqxPTha5XZCsDEPMSZ6+3RRBQI7KED1GeoDpq2WueXCacsNIjcB0MtCnEHY7ahQ==} + engines: {node: '>=16.0.0'} + peerDependencies: + hono: '>=4.10.0' + zod: ^4.0.0 + + '@hono/zod-validator@0.9.1': + resolution: {integrity: sha512-iiv6w0qrIc0arfvCtUqBWsvl4fXjzaTcQcCJTTtCnkawF9HHGE+KjEl0ox3gQJ6rKZEgE3mLExlQCF72M+mNuw==} + peerDependencies: + hono: '>=4.11.2' + zod: ^3.25.0 || ^4.0.0 + '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -1549,6 +1588,10 @@ packages: highlight.js@10.7.3: resolution: {integrity: sha512-tzcUFauisWKNHaRkN4Wjl/ZA07gENAjFl3J/c480dprkGTg5EQstgaNFqBfUqCq54kZRIEcreTsAgF/m2quD7A==} + hono@4.13.5: + resolution: {integrity: sha512-O6+/eCYRkzzzy0rPWwKLiGBR1nFuUPZynnwjxN1MBA62NNqbT0wQEzQyK2gSO5yDIDB336sXQleAhOHrzlYyKw==} + engines: {node: '>=16.9.0'} + hosted-git-info@9.0.3: resolution: {integrity: sha512-Hc+ghLoSt6QaYZUv0WBiIvmMDZuZZ7oaDvdH8MbfOO4lOsxdXLEvuC6ePoGs9H1X9oCLyq6+NVN0MKqD+ydxyg==} engines: {node: ^20.17.0 || >=22.9.0} @@ -1939,6 +1982,9 @@ packages: zod: optional: true + openapi3-ts@4.6.1: + resolution: {integrity: sha512-XW9MOldkhoICNeXVzzmXzmOW5G73ppOEGmh7fLCqHjgfdEYCGGN+00MlVCeUZgovjjfC56j9tvtDt1zGabNjjA==} + p-retry@4.6.2: resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} engines: {node: '>=8'} @@ -2378,6 +2424,11 @@ snapshots: optionalDependencies: zod: 4.4.3 + '@asteasolutions/zod-to-openapi@9.1.0(zod@4.4.3)': + dependencies: + openapi3-ts: 4.6.1 + zod: 4.4.3 + '@aws-crypto/sha256-browser@5.2.0': dependencies: '@aws-crypto/sha256-js': 5.2.0 @@ -3095,6 +3146,23 @@ snapshots: - supports-color - utf-8-validate + '@hono/node-server@2.1.1(hono@4.13.5)': + dependencies: + hono: 4.13.5 + + '@hono/zod-openapi@1.6.1(hono@4.13.5)(zod@4.4.3)': + dependencies: + '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) + '@hono/zod-validator': 0.9.1(hono@4.13.5)(zod@4.4.3) + hono: 4.13.5 + openapi3-ts: 4.6.1 + zod: 4.4.3 + + '@hono/zod-validator@0.9.1(hono@4.13.5)(zod@4.4.3)': + dependencies: + hono: 4.13.5 + zod: 4.4.3 + '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -3817,6 +3885,8 @@ snapshots: highlight.js@10.7.3: {} + hono@4.13.5: {} + hosted-git-info@9.0.3: dependencies: lru-cache: 11.5.2 @@ -4371,6 +4441,10 @@ snapshots: ws: 8.21.3 zod: 4.4.3 + openapi3-ts@4.6.1: + dependencies: + yaml: 2.9.0 + p-retry@4.6.2: dependencies: '@types/retry': 0.12.0