-The Python SDK's
+client.map()returns thelinkslist directly, not the full envelope.The Python SDK's
client.map()returns thelinkslist directly, not the full +envelope. The discovered sitemaps come back on that list as.sitemaps.
Discover all URLs on a site without downloading page content.
urls = client.map("https://example.com", max_depth=2, use_sitemap=True)
-print(f"Found {len(urls)} URLs")const urls = await client.map("https://example.com", { maxDepth: 2, useSitemap: true });
-console.log(`Found ${urls.length} URLs`);curl -X POST https://api.fastcrw.com/v1/map \
-H "Authorization: Bearer $CRW_API_KEY" \
diff --git a/docs/sdk-reference/index.html b/docs/sdk-reference/index.html
index 466a59c2..aee5902b 100644
--- a/docs/sdk-reference/index.html
+++ b/docs/sdk-reference/index.html
@@ -542,9 +542,23 @@ map
Discover all reachable URLs on a site without scraping page content.
# Python
urls = client.map("https://example.com", max_depth=2, use_sitemap=True)
-print(urls) # ["https://example.com/about", ...]
+print(urls) # ["https://example.com/about", ...]
+print(urls.sitemaps) # ["https://example.com/sitemap.xml", ...]// TypeScript
-const urls = await crw.map("https://example.com", { maxDepth: 2, useSitemap: true });The sitemap URLs the engine read ride on the returned list as sitemaps. The
+list is empty when the site exposes no sitemap or useSitemap is off, and
+partial when discovery stops early on limit, timeout or the internal sitemap
+budget; in local (subprocess) mode the MCP layer caps it at the map limit, so a
+deep sitemap index reports fewer entries there than over HTTP. Iteration,
+indexing, len and JSON serialization are unchanged, so existing code keeps
+working; in Python
+the value is now a list subclass, so an exact type(x) is list check no longer
+matches. Because sitemaps is an attribute on the list rather than an element of
+it, it does not carry over to a new list built by a transform (slicing,
+sorted(), spread, structuredClone); read it off the returned value directly.
Search the web and optionally fetch page content for each result in the same call.
# Python
diff --git a/sdks/python/README.md b/sdks/python/README.md
index a0753735..76e14a5c 100644
--- a/sdks/python/README.md
+++ b/sdks/python/README.md
@@ -83,6 +83,7 @@ print(job["id"])
# Map all URLs on a site:
urls = client.map("https://example.com")
print(urls)
+print(urls.sitemaps) # sitemap URLs the engine read, [] if the site has none
```
### Search
diff --git a/sdks/python/src/crw/__init__.py b/sdks/python/src/crw/__init__.py
index 748f33a6..762ae90a 100644
--- a/sdks/python/src/crw/__init__.py
+++ b/sdks/python/src/crw/__init__.py
@@ -1,6 +1,6 @@
"""CRW Python SDK — scrape, crawl, and map any website."""
-from crw.client import CrwClient
+from crw.client import CrwClient, MapLinks
from crw.exceptions import (
CrwApiError,
CrwBinaryNotFoundError,
@@ -12,6 +12,7 @@
__all__ = [
"CrwClient",
+ "MapLinks",
"CrwError",
"CrwApiError",
"CrwBinaryNotFoundError",
diff --git a/sdks/python/src/crw/client.py b/sdks/python/src/crw/client.py
index 1aaca33b..1ae66fb8 100644
--- a/sdks/python/src/crw/client.py
+++ b/sdks/python/src/crw/client.py
@@ -7,6 +7,7 @@
import os
import subprocess
import time
+from collections.abc import Iterable
from typing import Any, cast
from urllib.parse import parse_qs, quote, urlencode, urlsplit
@@ -124,6 +125,77 @@ class SearchResults(list):
llm_usage: dict | None = None
+class MapLinks(list[str]):
+ """The discovered URLs, with the site's sitemap URLs hanging off them.
+
+ `sitemaps` rides BESIDE `links` in the engine's map response and returning
+ only `links` dropped it. Subclassing `list` keeps every existing caller
+ working (`for u in urls`, indexing, len, `== [...]`) while `urls.sitemaps`
+ becomes reachable.
+
+ Set per instance, never as a class default, so no two results share one list
+ and it is always a list even against an engine old enough not to send the
+ field. It does not survive a transform that builds a new list: slicing, `+`,
+ `sorted()`, `.copy()`, `list(x)`. `copy.copy`, `copy.deepcopy` and `pickle`
+ keep it.
+
+ In local (subprocess) mode the MCP layer caps both lists at its map limit,
+ so a site with a very deep sitemap index reports fewer entries there than
+ over HTTP.
+ """
+
+ def __init__(
+ self,
+ links: Iterable[str] = (),
+ sitemaps: Iterable[str] | None = None,
+ ) -> None:
+ super().__init__(links)
+ self.sitemaps: list[str] = list(sitemaps or [])
+
+
+# Variables that switch `crw-mcp` from embedded to proxy mode. Both are verified
+# against the shipped binary: `CRW_API_URL` is bound by its CLI, and
+# `CRW_CLIENT__API_URL` reaches the same setting through the config layer.
+_PROXY_MODE_ENV_VARS = ("CRW_API_URL", "CRW_CLIENT__API_URL")
+
+
+def _local_child_env() -> dict[str, str]:
+ """Environment for the `crw-mcp` subprocess in CRW_LOCAL mode.
+
+ The child inherits our environment, so either variable above left in the
+ shell silently turned "run the local engine" into "call the cloud": every
+ tool then answered with the REST envelope instead of the flat payload, and
+ the call was billed. CRW_LOCAL means local, so the child does not get them.
+
+ This cannot close every route: `crw-mcp` also reads `client.api_url` from
+ `~/.config/crw/config.toml`, and it has no flag to force embedded mode. That
+ is why `map()` still handles the envelope shape in `_map_payload`.
+
+ Windows needs no case folding here: `os.environ` upper-cases its keys there.
+ """
+ return {k: v for k, v in os.environ.items() if k not in _PROXY_MODE_ENV_VARS}
+
+
+def _map_payload(result: dict[str, Any]) -> dict[str, Any]:
+ """Pick `crw_map`'s payload out of whichever shape the local `crw-mcp` sent.
+
+ It answers flat (`{success, links, sitemaps}`) when it runs embedded and
+ with the REST envelope (`{success, data: {...}}`) when it proxies. It
+ proxies whenever `CRW_API_URL` is set, which the subprocess inherits from
+ us even though CRW_LOCAL mode never reads that variable itself. Without
+ this, `links` came back empty in that state.
+
+ Probe `data.links` rather than a bare `data`, so a flat response that grows
+ some unrelated top-level `data` later cannot make us unwrap into it, and a
+ gateway answering `{"data": null}` cannot raise. Same discriminator the
+ engine's own MCP bounds use.
+ """
+ envelope = result.get("data")
+ if isinstance(envelope, dict) and "links" in envelope:
+ return envelope
+ return result
+
+
class CrwClient:
"""CRW web scraper client.
@@ -265,21 +337,29 @@ def map(
max_depth: int = 2,
use_sitemap: bool = True,
**kwargs: Any,
- ) -> list[str]:
+ ) -> MapLinks:
"""Discover URLs on a website.
Returns:
- List of discovered URLs.
+ The discovered URLs as a list, carrying the site's sitemap URLs on
+ its ``sitemaps`` attribute.
"""
args: dict[str, Any] = {"url": url, "maxDepth": max_depth, "useSitemap": use_sitemap}
args.update(kwargs)
if self._api_url:
data = self._http_post("/v1/map", args)
- return data.get("links", [])
-
- result = self._tool_call("crw_map", args)
- return result.get("links", [])
+ else:
+ data = _map_payload(self._tool_call("crw_map", args))
+ # Guard the shapes so a self-hosted server or gateway answering
+ # `{"links": null}` yields an empty list in both SDKs, rather than
+ # raising here and returning [] in TypeScript.
+ links = data.get("links")
+ sitemaps = data.get("sitemaps")
+ return MapLinks(
+ links if isinstance(links, list) else [],
+ sitemaps if isinstance(sitemaps, list) else [],
+ )
def search(
self,
@@ -752,6 +832,7 @@ def _ensure_process(self) -> subprocess.Popen:
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
+ env=_local_child_env(),
)
return self._process
@@ -834,7 +915,12 @@ def _http_request(
raise CrwApiError(result.get("error", "API error"))
if raw:
return result
- return result.get("data", result)
+ # `.get("data", result)` only covers a MISSING key. A gateway in front
+ # of the engine can answer `{"success": true, "data": null}`, and that
+ # returned None, so the caller raised AttributeError instead of the
+ # empty result TypeScript gives for the same bytes.
+ data = result.get("data")
+ return result if data is None else data
def _http_multipart(self, path: str, body: bytes, content_type: str) -> dict:
import urllib.request
diff --git a/sdks/python/tests/test_client_integration.py b/sdks/python/tests/test_client_integration.py
index 2c1d25e2..d2a4df68 100644
--- a/sdks/python/tests/test_client_integration.py
+++ b/sdks/python/tests/test_client_integration.py
@@ -54,6 +54,15 @@ def test_map_real_site(self, cloud_client: CrwClient) -> None:
assert len(links) >= 1
assert all(isinstance(link, str) for link in links)
+ def test_map_exposes_sitemaps(self, cloud_client: CrwClient) -> None:
+ # Assert the value, not just the attribute: hasattr() cannot fail once
+ # the attribute exists, so it would pass on a build that never fills it.
+ # docs.fastcrw.com is built from this repo, so its sitemap is ours to
+ # keep serving; the marketing site's is not.
+ links = cloud_client.map("https://docs.fastcrw.com")
+ assert links.sitemaps, "engine reported no sitemaps for a site that has one"
+ assert all(s.startswith("http") for s in links.sitemaps)
+
@pytest.mark.integration
@pytest.mark.timeout(30)
diff --git a/sdks/python/tests/test_client_unit.py b/sdks/python/tests/test_client_unit.py
index 5d854909..92028e4a 100644
--- a/sdks/python/tests/test_client_unit.py
+++ b/sdks/python/tests/test_client_unit.py
@@ -7,7 +7,7 @@
import pytest
-from crw.client import CLOUD_API_URL, CrwClient
+from crw.client import CLOUD_API_URL, CrwClient, _local_child_env
from crw.exceptions import CrwApiError, CrwError, CrwExtractCancelledError, CrwTimeoutError
@@ -160,6 +160,22 @@ def test_crawl_raises_on_failure(self) -> None:
# ---------------------------------------------------------------------------
+@pytest.mark.unit
+class TestLocalChildEnv:
+ def test_crw_api_url_is_not_inherited(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ # crw-mcp binds CRW_API_URL itself and proxies to the cloud when it is
+ # set, so an inherited one turned CRW_LOCAL into a billed cloud call
+ # answering with the REST envelope instead of the flat payload.
+ monkeypatch.setenv("CRW_API_URL", "https://api.fastcrw.com")
+ monkeypatch.setenv("CRW_CLIENT__API_URL", "https://api.fastcrw.com")
+ monkeypatch.setenv("CRW_API_KEY", "crw_live_test")
+ env = _local_child_env()
+ assert "CRW_API_URL" not in env
+ assert "CRW_CLIENT__API_URL" not in env
+ # The key is harmless: an embedded child ignores it.
+ assert env["CRW_API_KEY"] == "crw_live_test"
+
+
@pytest.mark.unit
class TestMap:
def test_map_http_returns_links(self) -> None:
@@ -171,6 +187,118 @@ def test_map_http_returns_links(self) -> None:
assert result == ["https://example.com/a", "https://example.com/b"]
+ def test_map_http_exposes_sitemaps(self) -> None:
+ client = CrwClient(api_url="https://fastcrw.com/api", api_key="crw_live_test")
+ mock_response = {
+ "links": ["https://example.com/a"],
+ "sitemaps": ["https://example.com/sitemap.xml"],
+ }
+
+ with patch.object(client, "_http_post", return_value=mock_response):
+ result = client.map("https://example.com")
+
+ assert result.sitemaps == ["https://example.com/sitemap.xml"]
+
+ def test_map_missing_sitemaps_is_an_empty_list(self) -> None:
+ # An engine old enough not to send the key must not surface None, so
+ # len(result.sitemaps) is always safe.
+ client = CrwClient(api_url="https://fastcrw.com/api", api_key="crw_live_test")
+
+ with patch.object(client, "_http_post", return_value={"links": []}):
+ result = client.map("https://example.com")
+
+ assert result.sitemaps == []
+
+ def test_map_result_still_behaves_as_a_plain_list(self) -> None:
+ client = CrwClient(api_url="https://fastcrw.com/api", api_key="crw_live_test")
+ mock_response = {"links": ["https://example.com/a"], "sitemaps": ["https://s.xml"]}
+
+ with patch.object(client, "_http_post", return_value=mock_response):
+ result = client.map("https://example.com")
+
+ assert isinstance(result, list)
+ assert result == ["https://example.com/a"]
+ assert len(result) == 1
+ assert json.dumps(result) == json.dumps(["https://example.com/a"])
+
+ def test_map_malformed_body_yields_empty_lists(self) -> None:
+ # A self-hosted server or gateway can answer anything; neither field may
+ # raise out of the SDK nor turn a string into a list of characters.
+ client = CrwClient(api_url="https://fastcrw.com/api", api_key="crw_live_test")
+
+ with patch.object(client, "_http_post", return_value={"links": "none", "sitemaps": 3}):
+ result = client.map("https://example.com")
+
+ assert result == []
+ assert result.sitemaps == []
+
+ with patch.object(client, "_http_post", return_value={"links": None}):
+ assert client.map("https://example.com") == []
+
+ def test_map_local_flat_shape(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ # Embedded crw-mcp answers flat.
+ client = _local_client(monkeypatch)
+ mock_response = {
+ "success": True,
+ "links": ["https://example.com/a"],
+ "sitemaps": ["https://example.com/sitemap.xml"],
+ }
+
+ with patch.object(client, "_tool_call", return_value=mock_response):
+ result = client.map("https://example.com")
+
+ assert result == ["https://example.com/a"]
+ assert result.sitemaps == ["https://example.com/sitemap.xml"]
+
+ def test_map_local_proxy_envelope(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ # A crw-mcp that inherited CRW_API_URL proxies and answers with the REST
+ # envelope. Before the unwrap this returned [] and dropped sitemaps.
+ client = _local_client(monkeypatch)
+ mock_response = {
+ "success": True,
+ "data": {
+ "links": ["https://example.com/a"],
+ "sitemaps": ["https://example.com/sitemap.xml"],
+ },
+ "creditCost": 1,
+ }
+
+ with patch.object(client, "_tool_call", return_value=mock_response):
+ result = client.map("https://example.com")
+
+ assert result == ["https://example.com/a"]
+ assert result.sitemaps == ["https://example.com/sitemap.xml"]
+
+ def test_map_local_ignores_an_unrelated_top_level_data(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ # The probe is `data.links`, not a bare `data`: a flat response that
+ # grows some other top-level `data` must not be unwrapped into.
+ client = _local_client(monkeypatch)
+ mock_response = {
+ "success": True,
+ "links": ["https://example.com/a"],
+ "sitemaps": ["https://example.com/sitemap.xml"],
+ "data": {"note": 1},
+ }
+
+ with patch.object(client, "_tool_call", return_value=mock_response):
+ result = client.map("https://example.com")
+
+ assert result == ["https://example.com/a"]
+ assert result.sitemaps == ["https://example.com/sitemap.xml"]
+
+ def test_map_local_null_data_does_not_raise(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ # A third-party gateway may answer {"data": null}; unwrapping blindly
+ # would raise AttributeError here.
+ client = _local_client(monkeypatch)
+
+ with patch.object(client, "_tool_call", return_value={"success": True, "data": None}):
+ result = client.map("https://example.com")
+
+ assert result == []
+ assert result.sitemaps == []
+
# ---------------------------------------------------------------------------
# Search
diff --git a/sdks/typescript/README.md b/sdks/typescript/README.md
index ecb0acd4..93ff3695 100644
--- a/sdks/typescript/README.md
+++ b/sdks/typescript/README.md
@@ -49,7 +49,7 @@ CRW_LOCAL=1 node app.js
|---|---|---|
| `scrape(url, opts?)` | Scrape one URL | both |
| `crawl(url, opts?)` | Crawl a site (async, polled) | both |
-| `map(url, opts?)` | Discover URLs | both |
+| `map(url, opts?)` | Discover URLs (+ the site's `sitemaps`) | both |
| `search(query, opts?)` | Web search (+ optional scrape) | both¹ |
| `parseFile(bytes, opts?)` | PDF → markdown / structured JSON | both |
| `extract({urls, schema?})` | Structured LLM extraction | HTTP |
diff --git a/sdks/typescript/src/client.ts b/sdks/typescript/src/client.ts
index f04876e8..f8fff6cb 100644
--- a/sdks/typescript/src/client.ts
+++ b/sdks/typescript/src/client.ts
@@ -17,6 +17,7 @@ import type {
ExtractStatus,
Json,
MapOptions,
+ MapResult,
ParseFileOptions,
ParseResult,
ScrapeOptions,
@@ -53,6 +54,23 @@ function httpOnlyHint(name: string, reason: string): string {
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+/**
+ * Pick `crw_map`'s payload out of whichever shape the local `crw-mcp` returned.
+ * It answers flat (`{success, links, sitemaps}`) when it runs embedded and with
+ * the REST envelope (`{success, data: {…}}`) when it proxies. It proxies
+ * whenever `CRW_API_URL` is set, which the subprocess inherits from us even
+ * though CRW_LOCAL mode never reads that variable itself. Without this, `links`
+ * came back empty in that state.
+ *
+ * Probe `data.links` rather than a bare `data`, so a flat response that grows
+ * some unrelated top-level `data` later cannot make us unwrap into it. Same
+ * discriminator the engine's own MCP bounds use.
+ */
+function mapPayload(result: Json): Json {
+ const envelope = result.data as Json | null | undefined;
+ return envelope && typeof envelope === "object" && "links" in envelope ? envelope : result;
+}
+
/**
* Next page request, rebuilt from OUR base path + the cursor's skip/limit. The
* server's `next` is an absolute URL from its own public origin/prefix; reusing
@@ -132,15 +150,29 @@ export class CrwClient {
return this.pollLocalCrawl(jobId, pollInterval, timeout);
}
- async map(url: string, opts: MapOptions = {}): Promise {
+ async map(url: string, opts: MapOptions = {}): Promise {
const { maxDepth = 2, useSitemap = true, ...rest } = opts;
const args: Json = { url, maxDepth, useSitemap, ...rest };
- if (this.apiUrl) {
- const data = await this.httpPost("/v1/map", args);
- return (data.links as string[]) ?? [];
- }
- const result = await this.localTransport().toolCall("crw_map", args);
- return (result.links as string[]) ?? [];
+ const data = this.apiUrl
+ ? await this.httpPost("/v1/map", args)
+ : mapPayload(await this.localTransport().toolCall("crw_map", args));
+ // Guard the shapes, as search() does before its own attach: a self-hosted
+ // server or gateway answering `{"links": "none"}` would otherwise make
+ // Object.defineProperty throw, or smuggle a non-array out typed as string[].
+ const links = (Array.isArray(data.links) ? (data.links as string[]) : []) as MapResult;
+ // `sitemaps` rides BESIDE `links` in the engine's response and the old
+ // unwrap dropped it: a caller could not see which sitemaps a site exposes
+ // even though they paid for the discovery. Attach it the way search()
+ // attaches its answer siblings: non-enumerably, so `links.sitemaps` works
+ // while `for…of`, spread and JSON.stringify keep seeing exactly the array
+ // they saw before.
+ Object.defineProperty(links, "sitemaps", {
+ value: Array.isArray(data.sitemaps) ? (data.sitemaps as string[]) : [],
+ enumerable: false,
+ configurable: true,
+ writable: true,
+ });
+ return links;
}
/**
diff --git a/sdks/typescript/src/index.ts b/sdks/typescript/src/index.ts
index 66bc65eb..c25eb04e 100644
--- a/sdks/typescript/src/index.ts
+++ b/sdks/typescript/src/index.ts
@@ -34,6 +34,7 @@ export type {
BlockOutcome,
LlmUsage,
CrawlResult,
+ MapResult,
SearchResult,
SearchResultItem,
ImageResultItem,
diff --git a/sdks/typescript/src/local.ts b/sdks/typescript/src/local.ts
index 6474b3c8..47861e78 100644
--- a/sdks/typescript/src/local.ts
+++ b/sdks/typescript/src/local.ts
@@ -20,6 +20,34 @@ interface Pending {
reject: (e: Error) => void;
}
+/**
+ * Variables that switch `crw-mcp` from embedded to proxy mode. Both are
+ * verified against the shipped binary: `CRW_API_URL` is bound by its CLI, and
+ * `CRW_CLIENT__API_URL` reaches the same setting through the config layer.
+ */
+const PROXY_MODE_ENV_VARS = ["CRW_API_URL", "CRW_CLIENT__API_URL"];
+
+/**
+ * Environment for the `crw-mcp` subprocess in CRW_LOCAL mode.
+ *
+ * The child inherits our environment, so either variable above left in the
+ * shell silently turned "run the local engine" into "call the cloud": every
+ * tool then answered with the REST envelope instead of the flat payload, and
+ * the call was billed. CRW_LOCAL means local, so the child does not get them.
+ *
+ * This cannot close every route: `crw-mcp` also reads `client.api_url` from
+ * `~/.config/crw/config.toml`, and it has no flag to force embedded mode. That
+ * is why `map()` still handles the envelope shape in `mapPayload`.
+ */
+export function localChildEnv(): NodeJS.ProcessEnv {
+ // Windows environment names are case-insensitive, so the child would read a
+ // differently-spelled key that an exact-match filter left behind. POSIX names
+ // are case-sensitive, so there only the exact spellings matter.
+ const fold = (k: string) => (process.platform === "win32" ? k.toUpperCase() : k);
+ const drop = new Set(PROXY_MODE_ENV_VARS.map(fold));
+ return Object.fromEntries(Object.entries(process.env).filter(([k]) => !drop.has(fold(k))));
+}
+
export class LocalTransport {
private proc: McpProc | null = null;
private nextId = 0;
@@ -37,7 +65,7 @@ export class LocalTransport {
private ensureProcess(): McpProc {
if (this.proc && this.proc.exitCode === null) return this.proc;
const bin = this.resolveBinary();
- const proc = spawn(bin, [], { stdio: ["pipe", "pipe", "ignore"] });
+ const proc = spawn(bin, [], { stdio: ["pipe", "pipe", "ignore"], env: localChildEnv() });
proc.on("error", (err: NodeJS.ErrnoException) => {
const failure =
err.code === "ENOENT"
diff --git a/sdks/typescript/src/types.ts b/sdks/typescript/src/types.ts
index f53a32ff..b534c70b 100644
--- a/sdks/typescript/src/types.ts
+++ b/sdks/typescript/src/types.ts
@@ -534,6 +534,27 @@ export interface V2Document {
export type ScrapeResult = ScrapeDocument;
/** Crawl (`/v1/crawl`) returns the scraped document for each page, in crawl order. */
export type CrawlResult = ScrapeDocument[];
+/**
+ * Map (`/v1/map`) returns the discovered URLs, with the sitemap URLs the engine
+ * read attached as `sitemaps`, the same way {@link SearchAnswer} rides
+ * alongside search results. Assignable to `string[]`, so
+ * `const urls: string[] = await crw.map(u)` keeps compiling.
+ *
+ * `sitemaps` is always an array at runtime: the client normalizes a missing
+ * field to `[]`. It is typed optional only so a plain `string[]` still assigns
+ * to this type, which keeps a consumer's own test stub
+ * (`mockResolvedValue(["https://a"])`) compiling. Same shape as
+ * {@link SearchAnswer}. It is non-enumerable, so it does not survive
+ * `structuredClone` (and therefore an RSC, worker or `postMessage` boundary) or
+ * any transform that builds a new array (`slice`, `filter`, spread).
+ * `JSON.stringify` is unaffected, since an array serializes its indices either
+ * way. Read it off the returned array directly.
+ *
+ * In local (subprocess) mode the MCP layer caps both lists at its map limit, so
+ * a site with a very deep sitemap index reports fewer entries there than over
+ * HTTP.
+ */
+export type MapResult = string[] & { sitemaps?: string[] };
/** Parse (`/v2/parse`) returns a v2 document (markdown/json/metadata). */
export type ParseResult = V2Document;
/** Batch scrape (`/v2/batch/scrape`) returns one v2 document per URL. */
diff --git a/sdks/typescript/test/client.test.ts b/sdks/typescript/test/client.test.ts
index f3029b45..b46084eb 100644
--- a/sdks/typescript/test/client.test.ts
+++ b/sdks/typescript/test/client.test.ts
@@ -7,6 +7,7 @@ import {
CrwExtractCancelledError,
CrwTimeoutError,
} from "../dist/esm/index.js";
+import { localChildEnv } from "../dist/esm/local.js";
const origFetch = globalThis.fetch;
const origEnv = { ...process.env };
@@ -337,5 +338,136 @@ test("capabilities unwraps and uses GET /v1/capabilities", async () => {
assert.equal((caps as { version: string }).version, "0.14.0");
});
+test("map returns the links array with sitemaps attached", async () => {
+ const calls = mockFetch({
+ success: true,
+ data: {
+ links: ["https://example.com/a", "https://example.com/b"],
+ sitemaps: ["https://example.com/sitemap.xml"],
+ },
+ creditCost: 1,
+ });
+ const c = new CrwClient({ apiKey: "crw_live_test" });
+ const links = await c.map("https://example.com");
+ assert.equal(calls[0].url, `${CLOUD_API_URL}/v1/map`);
+ assert.deepEqual([...links], ["https://example.com/a", "https://example.com/b"]);
+ // The whole point of #557: the sitemaps the engine reported are reachable.
+ const sitemaps: string[] = links.sitemaps ?? [];
+ assert.deepEqual(sitemaps, ["https://example.com/sitemap.xml"]);
+});
+
+test("map normalizes a missing sitemaps field to an empty list", async () => {
+ // An engine old enough not to send the key must not surface `undefined`,
+ // so `links.sitemaps.length` is always safe.
+ mockFetch({ success: true, data: { links: ["https://example.com/a"] } });
+ const c = new CrwClient({ apiKey: "crw_live_test" });
+ const links = await c.map("https://example.com");
+ assert.deepEqual(links.sitemaps, []);
+ assert.equal(links.sitemaps.length, 0);
+});
+
+test("map result is still a plain array for existing callers", async () => {
+ mockFetch({
+ success: true,
+ data: { links: ["https://example.com/a"], sitemaps: ["https://example.com/sitemap.xml"] },
+ });
+ const c = new CrwClient({ apiKey: "crw_live_test" });
+ const links = await c.map("https://example.com");
+ assert.ok(Array.isArray(links));
+ assert.equal(links.length, 1);
+ assert.equal([...links].length, 1);
+ // `sitemaps` is non-enumerable, so nothing an existing caller does changes.
+ assert.equal(JSON.stringify(links), JSON.stringify(["https://example.com/a"]));
+ assert.deepEqual(Object.keys(links), ["0"]);
+});
+
+/**
+ * Drive local (subprocess) mode without a real crw-mcp: stand a stub in for the
+ * transport the client would otherwise spawn. Mirrors the Python suite's
+ * `patch.object(client, "_tool_call", ...)`.
+ */
+function localClientReturning(toolResult: unknown) {
+ process.env.CRW_LOCAL = "1";
+ const c = new CrwClient();
+ (c as unknown as { local: { toolCall: () => Promise; close: () => void } }).local = {
+ toolCall: async () => toolResult,
+ close: () => {},
+ };
+ return c;
+}
+
+test("map survives a malformed links/sitemaps body", async () => {
+ // A self-hosted server or gateway can answer anything; neither field may
+ // throw out of the SDK or escape typed as string[] when it is not one.
+ mockFetch({ success: true, data: { links: "none", sitemaps: 3 } });
+ const c = new CrwClient({ apiKey: "crw_live_test" });
+ const links = await c.map("https://example.com");
+ assert.ok(Array.isArray(links));
+ assert.deepEqual([...links], []);
+ assert.deepEqual(links.sitemaps, []);
+});
+
+test("map in local mode reads the flat embedded shape", async () => {
+ const c = localClientReturning({
+ success: true,
+ links: ["https://example.com/a"],
+ sitemaps: ["https://example.com/sitemap.xml"],
+ });
+ const links = await c.map("https://example.com");
+ assert.deepEqual([...links], ["https://example.com/a"]);
+ assert.deepEqual(links.sitemaps, ["https://example.com/sitemap.xml"]);
+});
+
+test("map in local mode unwraps the proxy REST envelope", async () => {
+ // A crw-mcp that inherited CRW_API_URL proxies and answers with the envelope.
+ // Without the unwrap this returned [] and dropped sitemaps entirely.
+ const c = localClientReturning({
+ success: true,
+ data: {
+ links: ["https://example.com/a"],
+ sitemaps: ["https://example.com/sitemap.xml"],
+ },
+ creditCost: 1,
+ });
+ const links = await c.map("https://example.com");
+ assert.deepEqual([...links], ["https://example.com/a"]);
+ assert.deepEqual(links.sitemaps, ["https://example.com/sitemap.xml"]);
+});
+
+test("map in local mode ignores an unrelated top-level data field", async () => {
+ // The probe is `data.links`, not a bare `data`: a flat response that grows
+ // some other top-level `data` must not be unwrapped into.
+ const c = localClientReturning({
+ success: true,
+ links: ["https://example.com/a"],
+ sitemaps: ["https://example.com/sitemap.xml"],
+ data: { note: 1 },
+ });
+ const links = await c.map("https://example.com");
+ assert.deepEqual([...links], ["https://example.com/a"]);
+ assert.deepEqual(links.sitemaps, ["https://example.com/sitemap.xml"]);
+});
+
+test("map in local mode does not unwrap a null data field", async () => {
+ const c = localClientReturning({ success: true, data: null });
+ const links = await c.map("https://example.com");
+ assert.deepEqual([...links], []);
+ assert.deepEqual(links.sitemaps, []);
+});
+
+test("the local subprocess does not inherit CRW_API_URL", () => {
+ // crw-mcp binds CRW_API_URL itself and proxies to the cloud when it is set,
+ // so an inherited one turned CRW_LOCAL into a billed cloud call answering
+ // with the REST envelope instead of the flat payload.
+ process.env.CRW_API_URL = "https://api.fastcrw.com";
+ process.env.CRW_CLIENT__API_URL = "https://api.fastcrw.com";
+ process.env.CRW_API_KEY = "crw_live_test";
+ const env = localChildEnv();
+ assert.equal(env.CRW_API_URL, undefined);
+ assert.equal(env.CRW_CLIENT__API_URL, undefined);
+ // The key is harmless: an embedded child ignores it.
+ assert.equal(env.CRW_API_KEY, "crw_live_test");
+});
+
// silence unused import lint in some configs
void CrwError;