fix(sdk): return the map sitemaps and keep local mode local

docs/docs/recipe-product-catalog.md

@@ -108,7 +108,8 @@ for u in urls[:3]:
 }
 ```
 
-> The Python SDK's `client.map()` returns the `links` list directly, not the full envelope.
+> The Python SDK's `client.map()` returns the `links` list directly, not the full
+> envelope. The discovered sitemaps come back on that list as `.sitemaps`.
 
 ---
 

docs/docs/sdk-examples.md

@@ -171,14 +171,14 @@ Discover all URLs on a site without downloading page content.
 
 ```python
 urls = client.map("https://example.com", max_depth=2, use_sitemap=True)
-print(f"Found {len(urls)} URLs")
+print(f"Found {len(urls)} URLs across {len(urls.sitemaps)} sitemaps")
 ```
 
 ### TypeScript
 
 ```ts
 const urls = await client.map("https://example.com", { maxDepth: 2, useSitemap: true });
-console.log(`Found ${urls.length} URLs`);
+console.log(`Found ${urls.length} URLs across ${(urls.sitemaps ?? []).length} sitemaps`);
 ```
 
 ### curl (fallback)

docs/docs/sdk-reference.md

@@ -211,14 +211,29 @@ Discover all reachable URLs on a site without scraping page content.
 ```python
 # Python
 urls = client.map("https://example.com", max_depth=2, use_sitemap=True)
-print(urls)  # ["https://example.com/about", ...]
+print(urls)            # ["https://example.com/about", ...]
+print(urls.sitemaps)   # ["https://example.com/sitemap.xml", ...]
 ```
 
 ```ts
 // TypeScript
 const urls = await crw.map("https://example.com", { maxDepth: 2, useSitemap: true });
+console.log(urls);            // ["https://example.com/about", ...]
+console.log(urls.sitemaps);   // ["https://example.com/sitemap.xml", ...]
 ```
 
+The sitemap URLs the engine read ride on the returned list as `sitemaps`. The
+list is empty when the site exposes no sitemap or `useSitemap` is off, and
+partial when discovery stops early on `limit`, `timeout` or the internal sitemap
+budget; in local (subprocess) mode the MCP layer caps it at the map limit, so a
+deep sitemap index reports fewer entries there than over HTTP. Iteration,
+indexing, `len` and JSON serialization are unchanged, so existing code keeps
+working; in Python
+the value is now a `list` subclass, so an exact `type(x) is list` check no longer
+matches. Because `sitemaps` is an attribute on the list rather than an element of
+it, it does not carry over to a new list built by a transform (slicing,
+`sorted()`, spread, `structuredClone`); read it off the returned value directly.
+
 ### search
 
 Search the web and optionally fetch page content for each result in the same call.

docs/recipe-product-catalog/index.html

@@ -360,7 +360,8 @@ <h2 id="step-1-map-the-category-page-to-product-urls">Step 1: Map the category p
   ]
 }</code></pre>
 <blockquote>
-<p>The Python SDK&#39;s <code>client.map()</code> returns the <code>links</code> list directly, not the full envelope.</p>
+<p>The Python SDK&#39;s <code>client.map()</code> returns the <code>links</code> list directly, not the full
+envelope. The discovered sitemaps come back on that list as <code>.sitemaps</code>.</p>
 </blockquote>
 <hr>
 <h2 id="step-2-define-the-product-schema">Step 2: Define the product schema</h2>

docs/sdk-examples/index.html

@@ -405,10 +405,10 @@ <h2 id="map">Map</h2>
 <p>Discover all URLs on a site without downloading page content.</p>
 <h3 id="python-3">Python</h3>
 <pre data-lang="python"><code class="language-python">urls = client.map(&quot;https://example.com&quot;, max_depth=2, use_sitemap=True)
-print(f&quot;Found {len(urls)} URLs&quot;)</code></pre>
+print(f&quot;Found {len(urls)} URLs across {len(urls.sitemaps)} sitemaps&quot;)</code></pre>
 <h3 id="typescript-3">TypeScript</h3>
 <pre data-lang="ts"><code class="language-ts">const urls = await client.map(&quot;https://example.com&quot;, { maxDepth: 2, useSitemap: true });
-console.log(`Found ${urls.length} URLs`);</code></pre>
+console.log(`Found ${urls.length} URLs across ${(urls.sitemaps ?? []).length} sitemaps`);</code></pre>
 <h3 id="curl-fallback-2">curl (fallback)</h3>
 <pre data-lang="bash"><code class="language-bash">curl -X POST https://api.fastcrw.com/v1/map \
   -H &quot;Authorization: Bearer $CRW_API_KEY&quot; \

docs/sdk-reference/index.html

@@ -542,9 +542,23 @@ <h3 id="map">map</h3>
 <p>Discover all reachable URLs on a site without scraping page content.</p>
 <pre data-lang="python"><code class="language-python"># Python
 urls = client.map(&quot;https://example.com&quot;, max_depth=2, use_sitemap=True)
-print(urls)  # [&quot;https://example.com/about&quot;, ...]</code></pre>
+print(urls)            # [&quot;https://example.com/about&quot;, ...]
+print(urls.sitemaps)   # [&quot;https://example.com/sitemap.xml&quot;, ...]</code></pre>
 <pre data-lang="ts"><code class="language-ts">// TypeScript
-const urls = await crw.map(&quot;https://example.com&quot;, { maxDepth: 2, useSitemap: true });</code></pre>
+const urls = await crw.map(&quot;https://example.com&quot;, { maxDepth: 2, useSitemap: true });
+console.log(urls);            // [&quot;https://example.com/about&quot;, ...]
+console.log(urls.sitemaps);   // [&quot;https://example.com/sitemap.xml&quot;, ...]</code></pre>
+<p>The sitemap URLs the engine read ride on the returned list as <code>sitemaps</code>. The
+list is empty when the site exposes no sitemap or <code>useSitemap</code> is off, and
+partial when discovery stops early on <code>limit</code>, <code>timeout</code> or the internal sitemap
+budget; in local (subprocess) mode the MCP layer caps it at the map limit, so a
+deep sitemap index reports fewer entries there than over HTTP. Iteration,
+indexing, <code>len</code> and JSON serialization are unchanged, so existing code keeps
+working; in Python
+the value is now a <code>list</code> subclass, so an exact <code>type(x) is list</code> check no longer
+matches. Because <code>sitemaps</code> is an attribute on the list rather than an element of
+it, it does not carry over to a new list built by a transform (slicing,
+<code>sorted()</code>, spread, <code>structuredClone</code>); read it off the returned value directly.</p>
 <h3 id="search">search</h3>
 <p>Search the web and optionally fetch page content for each result in the same call.</p>
 <pre data-lang="python"><code class="language-python"># Python

sdks/python/README.md

@@ -83,6 +83,7 @@ print(job["id"])
 # Map all URLs on a site:
 urls = client.map("https://example.com")
 print(urls)
+print(urls.sitemaps)  # sitemap URLs the engine read, [] if the site has none
 ```
 
 ### Search

sdks/python/src/crw/__init__.py

@@ -1,6 +1,6 @@
 """CRW Python SDK — scrape, crawl, and map any website."""
 
-from crw.client import CrwClient
+from crw.client import CrwClient, MapLinks
 from crw.exceptions import (
     CrwApiError,
     CrwBinaryNotFoundError,
@@ -12,6 +12,7 @@
 
 __all__ = [
     "CrwClient",
+    "MapLinks",
     "CrwError",
     "CrwApiError",
     "CrwBinaryNotFoundError",

sdks/python/src/crw/client.py

@@ -7,6 +7,7 @@
 import os
 import subprocess
 import time
+from collections.abc import Iterable
 from typing import Any, cast
 from urllib.parse import parse_qs, quote, urlencode, urlsplit
 
@@ -124,6 +125,77 @@ class SearchResults(list):
     llm_usage: dict | None = None
 
 
+class MapLinks(list[str]):
+    """The discovered URLs, with the site's sitemap URLs hanging off them.
+
+    `sitemaps` rides BESIDE `links` in the engine's map response and returning
+    only `links` dropped it. Subclassing `list` keeps every existing caller
+    working (`for u in urls`, indexing, len, `== [...]`) while `urls.sitemaps`
+    becomes reachable.
+
+    Set per instance, never as a class default, so no two results share one list
+    and it is always a list even against an engine old enough not to send the
+    field. It does not survive a transform that builds a new list: slicing, `+`,
+    `sorted()`, `.copy()`, `list(x)`. `copy.copy`, `copy.deepcopy` and `pickle`
+    keep it.
+
+    In local (subprocess) mode the MCP layer caps both lists at its map limit,
+    so a site with a very deep sitemap index reports fewer entries there than
+    over HTTP.
+    """
+
+    def __init__(
+        self,
+        links: Iterable[str] = (),
+        sitemaps: Iterable[str] | None = None,
+    ) -> None:
+        super().__init__(links)
+        self.sitemaps: list[str] = list(sitemaps or [])
+
+
+# Variables that switch `crw-mcp` from embedded to proxy mode. Both are verified
+# against the shipped binary: `CRW_API_URL` is bound by its CLI, and
+# `CRW_CLIENT__API_URL` reaches the same setting through the config layer.
+_PROXY_MODE_ENV_VARS = ("CRW_API_URL", "CRW_CLIENT__API_URL")
+
+
+def _local_child_env() -> dict[str, str]:
+    """Environment for the `crw-mcp` subprocess in CRW_LOCAL mode.
+
+    The child inherits our environment, so either variable above left in the
+    shell silently turned "run the local engine" into "call the cloud": every
+    tool then answered with the REST envelope instead of the flat payload, and
+    the call was billed. CRW_LOCAL means local, so the child does not get them.
+
+    This cannot close every route: `crw-mcp` also reads `client.api_url` from
+    `~/.config/crw/config.toml`, and it has no flag to force embedded mode. That
+    is why `map()` still handles the envelope shape in `_map_payload`.
+
+    Windows needs no case folding here: `os.environ` upper-cases its keys there.
+    """
+    return {k: v for k, v in os.environ.items() if k not in _PROXY_MODE_ENV_VARS}
+
+
+def _map_payload(result: dict[str, Any]) -> dict[str, Any]:
+    """Pick `crw_map`'s payload out of whichever shape the local `crw-mcp` sent.
+
+    It answers flat (`{success, links, sitemaps}`) when it runs embedded and
+    with the REST envelope (`{success, data: {...}}`) when it proxies. It
+    proxies whenever `CRW_API_URL` is set, which the subprocess inherits from
+    us even though CRW_LOCAL mode never reads that variable itself. Without
+    this, `links` came back empty in that state.
+
+    Probe `data.links` rather than a bare `data`, so a flat response that grows
+    some unrelated top-level `data` later cannot make us unwrap into it, and a
+    gateway answering `{"data": null}` cannot raise. Same discriminator the
+    engine's own MCP bounds use.
+    """
+    envelope = result.get("data")
+    if isinstance(envelope, dict) and "links" in envelope:
+        return envelope
+    return result
+
+
 class CrwClient:
     """CRW web scraper client.
 
@@ -265,21 +337,29 @@ def map(
         max_depth: int = 2,
         use_sitemap: bool = True,
         **kwargs: Any,
-    ) -> list[str]:
+    ) -> MapLinks:
         """Discover URLs on a website.
 
         Returns:
-            List of discovered URLs.
+            The discovered URLs as a list, carrying the site's sitemap URLs on
+            its ``sitemaps`` attribute.
         """
         args: dict[str, Any] = {"url": url, "maxDepth": max_depth, "useSitemap": use_sitemap}
         args.update(kwargs)
 
         if self._api_url:
             data = self._http_post("/v1/map", args)
-            return data.get("links", [])
-
-        result = self._tool_call("crw_map", args)
-        return result.get("links", [])
+        else:
+            data = _map_payload(self._tool_call("crw_map", args))
+        # Guard the shapes so a self-hosted server or gateway answering
+        # `{"links": null}` yields an empty list in both SDKs, rather than
+        # raising here and returning [] in TypeScript.
+        links = data.get("links")
+        sitemaps = data.get("sitemaps")
+        return MapLinks(
+            links if isinstance(links, list) else [],
+            sitemaps if isinstance(sitemaps, list) else [],
+        )
 
     def search(
         self,
@@ -752,6 +832,7 @@ def _ensure_process(self) -> subprocess.Popen:
                 stdout=subprocess.PIPE,
                 stderr=subprocess.DEVNULL,
                 text=True,
+                env=_local_child_env(),
             )
         return self._process
 
@@ -834,7 +915,12 @@ def _http_request(
             raise CrwApiError(result.get("error", "API error"))
         if raw:
             return result
-        return result.get("data", result)
+        # `.get("data", result)` only covers a MISSING key. A gateway in front
+        # of the engine can answer `{"success": true, "data": null}`, and that
+        # returned None, so the caller raised AttributeError instead of the
+        # empty result TypeScript gives for the same bytes.
+        data = result.get("data")
+        return result if data is None else data
 
     def _http_multipart(self, path: str, body: bytes, content_type: str) -> dict:
         import urllib.request

sdks/python/tests/test_client_integration.py

@@ -54,6 +54,15 @@ def test_map_real_site(self, cloud_client: CrwClient) -> None:
         assert len(links) >= 1
         assert all(isinstance(link, str) for link in links)
 
+    def test_map_exposes_sitemaps(self, cloud_client: CrwClient) -> None:
+        # Assert the value, not just the attribute: hasattr() cannot fail once
+        # the attribute exists, so it would pass on a build that never fills it.
+        # docs.fastcrw.com is built from this repo, so its sitemap is ours to
+        # keep serving; the marketing site's is not.
+        links = cloud_client.map("https://docs.fastcrw.com")
+        assert links.sitemaps, "engine reported no sitemaps for a site that has one"
+        assert all(s.startswith("http") for s in links.sitemaps)
+
 
 @pytest.mark.integration
 @pytest.mark.timeout(30)