From a5ebade28d23d701b3c9d7ce86d67969badb1ee9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 20 Aug 2026 13:21:13 +0000 Subject: [PATCH] docs(agent-quickstart): add per-language quickstarts for search, scrape, and interact Add canonical agent quickstart files for Node.js, Python, Rust, Java, and Elixir. Each file covers search, scrape, and interact endpoints with confirmed parameters, types, and examples sourced from the SDK code and v2 OpenAPI spec. Co-Authored-By: Claude Opus 4.6 (1M context) Claude-Session: https://claude.ai/code/session_015LrWqx4unfQtJG74egnXPK --- agent-quickstart/elixir.mdx | 189 ++++++++++++++++++++++++++++++ agent-quickstart/java.mdx | 221 ++++++++++++++++++++++++++++++++++++ agent-quickstart/node.mdx | 177 +++++++++++++++++++++++++++++ agent-quickstart/python.mdx | 175 ++++++++++++++++++++++++++++ agent-quickstart/rust.mdx | 218 +++++++++++++++++++++++++++++++++++ 5 files changed, 980 insertions(+) create mode 100644 agent-quickstart/elixir.mdx create mode 100644 agent-quickstart/java.mdx create mode 100644 agent-quickstart/node.mdx create mode 100644 agent-quickstart/python.mdx create mode 100644 agent-quickstart/rust.mdx diff --git a/agent-quickstart/elixir.mdx b/agent-quickstart/elixir.mdx new file mode 100644 index 000000000..297780888 --- /dev/null +++ b/agent-quickstart/elixir.mdx @@ -0,0 +1,189 @@ +--- +title: "Elixir Agent Quickstart" +description: "Canonical Firecrawl Elixir quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Elixir Agent Quickstart + +Canonical quickstart for external agents. Generated from SDK source (`firecrawl` hex package **1.9.1**) and the v2 OpenAPI spec. Function names are auto-generated from the OpenAPI spec and match the SDK exactly. + +## Install + +```elixir +# mix.exs +defp deps do + [ + {:firecrawl, "~> 1.9"} + ] +end +``` + +## Authenticate + +```elixir +# Option 1: Application config +config :firecrawl, api_key: "fc-your-api-key" + +# Option 2: Per-request in opts +Firecrawl.scrape_and_extract_from_url([url: "https://example.com"], api_key: "fc-...") +``` + +All functions accept `api_key` and `base_url` (default `"https://api.firecrawl.dev/v2"`) in the trailing `opts` keyword list. + +## When To Use What + +- `search_and_scrape`: use when you start with a query and need discovery. +- `scrape_and_extract_from_url`: use when you already have a URL and want page content. +- `interact_with_scrape_browser_session`: use when the page needs clicks, forms, or post-scrape browser actions. + +## Search + +### Why use it + +Use search to discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:`, for example `site:docs.firecrawl.dev crawl webhooks`. + +### Preferred SDK function + +`Firecrawl.search_and_scrape(params, opts \\ [])` → `{:ok, Req.Response.t()} | {:error, Exception.t()}` + +Bang variant: `Firecrawl.search_and_scrape!(params, opts)` raises on error. + +### Example + +```elixir +{:ok, response} = Firecrawl.search_and_scrape( + query: "site:docs.firecrawl.dev webhook retries", + limit: 5, + scrape_options: [formats: ["markdown"]] +) + +results = response.body +for hit <- results["web"] || [] do + IO.puts("#{hit["url"]}: #{String.slice(hit["markdown"] || "", 0..199)}") +end +``` + +### Parameters + +| Parameter | Type | JSON key | Description | +|---|---|---|---| +| `query` | `:string` (required) | `query` | Search query. Use `site:example.com` to limit to a domain. | +| `sources` | `{:list, :any}` | `sources` | Which sources: `"web"`, `"news"`, `"images"`. | +| `categories` | `{:list, :any}` | `categories` | Filter by category: `"github"`, `"research"`, `"pdf"`. | +| `include_domains` | `{:list, :string}` | `includeDomains` | Only include these domains. | +| `exclude_domains` | `{:list, :string}` | `excludeDomains` | Exclude these domains. | +| `limit` | `:integer` | `limit` | Max number of results. | +| `tbs` | `:string` | `tbs` | Time-based filter (e.g. `qdr:d`). | +| `location` | `:string` | `location` | Localized results. | +| `ignore_invalid_urls` | `:boolean` | `ignoreInvalidURLs` | Drop invalid URLs. | +| `timeout` | `:integer` | `timeout` | Timeout in milliseconds. | +| `highlights` | `:boolean` | `highlights` | Generate highlights. | +| `scrape_options` | `:keyword_list` | `scrapeOptions` | Scrape each result (see Scrape parameters). | + +## Scrape + +### Why use it + +Use scrape when you already have a URL and want structured content in one or more formats. + +### Preferred SDK function + +`Firecrawl.scrape_and_extract_from_url(params, opts \\ [])` → `{:ok, Req.Response.t()} | {:error, Exception.t()}` + +Bang variant: `Firecrawl.scrape_and_extract_from_url!(params, opts)` raises on error. + +### Example + +```elixir +{:ok, response} = Firecrawl.scrape_and_extract_from_url( + url: "https://docs.firecrawl.dev", + formats: ["markdown"], + only_main_content: true +) + +doc = response.body["data"] +IO.puts(doc["markdown"]) +``` + +### Parameters + +| Parameter | Type | JSON key | Description | +|---|---|---|---| +| `url` | `:string` (required) | `url` | Target URL. | +| `formats` | `{:list, :any}` | `formats` | Output formats: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, etc. | +| `headers` | `:any` | `headers` | Custom HTTP headers. | +| `include_tags` | `{:list, :string}` | `includeTags` | Only include these HTML tags. | +| `exclude_tags` | `{:list, :string}` | `excludeTags` | Exclude these HTML tags. | +| `only_main_content` | `:boolean` | `onlyMainContent` | Strip nav, footer, and boilerplate. | +| `timeout` | `:integer` | `timeout` | Timeout in milliseconds. | +| `wait_for` | `:integer` | `waitFor` | Wait for page to render (milliseconds). | +| `mobile` | `:boolean` | `mobile` | Use a mobile viewport. | +| `parsers` | `{:list, :any}` | `parsers` | File parsing controls. | +| `actions` | `{:list, :any}` | `actions` | Pre-scrape browser actions. | +| `location` | `:keyword_list` | `location` | `[country: ..., languages: [...]]` for geo-aware scraping. | +| `skip_tls_verification` | `:boolean` | `skipTlsVerification` | Skip TLS verification. | +| `remove_base64_images` | `:boolean` | `removeBase64Images` | Drop base64 images from markdown output. | +| `block_ads` | `:boolean` | `blockAds` | Block ads and cookie popups. | +| `proxy` | `:basic \| :enhanced \| :auto` | `proxy` | Proxy control. | +| `max_age` | `:integer` | `maxAge` | Use cached data up to this age (milliseconds). | +| `min_age` | `:integer` | `minAge` | Use cached data only if at least this old (milliseconds). | +| `store_in_cache` | `:boolean` | `storeInCache` | Cache the result. | +| `lockdown` | `:boolean` | `lockdown` | Serve from cache only. | +| `profile` | `:keyword_list` | `profile` | Persistent browser profile. | + +## Interact + +### Why use it + +Use `interact` to run code in the browser session tied to a scrape job. The Elixir SDK accepts `code` (required) and does not support `prompt` — use code-based interaction. + +### Preferred SDK function + +`Firecrawl.interact_with_scrape_browser_session(job_id, params, opts \\ [])` → `{:ok, Req.Response.t()} | {:error, Exception.t()}` + +Bang variant: `Firecrawl.interact_with_scrape_browser_session!(job_id, params, opts)` raises on error. + +### Example + +```elixir +{:ok, scrape_response} = Firecrawl.scrape_and_extract_from_url( + url: "https://example.com", + formats: ["markdown"] +) + +job_id = get_in(scrape_response.body, ["data", "metadata", "scrapeId"]) + +{:ok, result} = Firecrawl.interact_with_scrape_browser_session(job_id, + code: "console.log(await page.title());" +) + +IO.puts(result.body["stdout"]) +``` + +### Parameters + +| Parameter | Type | JSON key | Description | +|---|---|---|---| +| `job_id` | `String.t()` (path) | — | Scrape job ID from scrape response metadata. | +| `code` | `:string` (required) | `code` | Code to execute in the browser session. | +| `language` | `:python \| :node \| :bash` | `language` | Execution runtime. Default: `"node"` (server-side). | +| `timeout` | `:integer` | `timeout` | Execution timeout in seconds. | + +### Stop session + +`Firecrawl.stop_interactive_scrape_browser_session(job_id, opts \\ [])` → `{:ok, Req.Response.t()} | {:error, Exception.t()}` + +## Notes + +- The Elixir SDK is **auto-generated from the OpenAPI spec** (`generate.exs`). Function names are long and OpenAPI-derived — do not rename them. +- No deprecated aliases exist in the Elixir SDK. +- `interact` does not support `prompt` — only code-based interaction is available. +- The SDK uses `Req` under the hood. Extra `opts` are passed through to `Req`. +- The proxy parameter accepts atoms (`:basic`, `:enhanced`, `:auto`), not strings. +- Errors return `{:error, Firecrawl.Error.t()}` with `status` and `body` fields. + +## Source Of Truth + +- `firecrawl/apps/elixir-sdk/lib/firecrawl.ex` +- `firecrawl/apps/elixir-sdk/mix.exs` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/java.mdx b/agent-quickstart/java.mdx new file mode 100644 index 000000000..6bd8d686b --- /dev/null +++ b/agent-quickstart/java.mdx @@ -0,0 +1,221 @@ +--- +title: "Java Agent Quickstart" +description: "Canonical Firecrawl Java quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Java Agent Quickstart + +Canonical quickstart for external agents. Generated from SDK source (`firecrawl-java` **1.14.0**) and the v2 OpenAPI spec. Method names, parameters, and types match `FirecrawlClient`. + +## Install + +**Gradle (Kotlin DSL):** + +```kotlin +implementation("com.firecrawl:firecrawl-java:1.14.0") +``` + +**Maven:** + +```xml + + com.firecrawl + firecrawl-java + 1.14.0 + +``` + +## Authenticate + +```java +import com.firecrawl.client.FirecrawlClient; + +FirecrawlClient client = FirecrawlClient.builder() + .apiKey(System.getenv("FIRECRAWL_API_KEY")) + .build(); + +// Or read from env automatically: +FirecrawlClient client = FirecrawlClient.fromEnv(); +``` + +Builder options: `apiKey` (falls back to `FIRECRAWL_API_KEY` env var, then `firecrawl.apiKey` system property), `apiUrl` (default `"https://api.firecrawl.dev"`, falls back to `FIRECRAWL_API_URL`), `timeoutMs` (default `300000`), `maxRetries` (default `3`), `backoffFactor` (default `0.5`). + +## When To Use What + +- `search`: use when you start with a query and need discovery. +- `scrape`: use when you already have a URL and want page content. +- `interact`: use when the page needs clicks, forms, or post-scrape browser actions. + +## Search + +### Why use it + +Use search to discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:`, for example `site:docs.firecrawl.dev crawl webhooks`. + +### Preferred SDK method + +`client.search(query, options)` → `SearchData` + +### Example + +```java +import com.firecrawl.models.SearchOptions; +import com.firecrawl.models.ScrapeOptions; +import com.firecrawl.models.SearchData; +import java.util.List; + +SearchData results = client.search( + "site:docs.firecrawl.dev webhook retries", + SearchOptions.builder() + .limit(5) + .scrapeOptions(ScrapeOptions.builder() + .formats(List.of("markdown")) + .onlyMainContent(true) + .build()) + .build() +); + +if (results.getWeb() != null) { + for (var hit : results.getWeb()) { + System.out.println(hit); + } +} +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | `String` | Search query. Use `site:example.com` to limit to a domain. | +| `options.sources` | `List` | Which sources: `"web"`, `"news"`, `"images"`. | +| `options.categories` | `List` | Filter by category: `"github"`, `"research"`, `"pdf"`. | +| `options.includeDomains` | `List` | Only include these domains. | +| `options.excludeDomains` | `List` | Exclude these domains. | +| `options.limit` | `Integer` | Max number of results. | +| `options.tbs` | `String` | Time-based filter (e.g. `qdr:d`). | +| `options.location` | `String` | Localized results. | +| `options.ignoreInvalidURLs` | `Boolean` | Drop invalid URLs. | +| `options.timeout` | `Integer` | Timeout in milliseconds. | +| `options.highlights` | `Boolean` | Generate highlights. Defaults to true. | +| `options.scrapeOptions` | `ScrapeOptions` | Scrape each result (see Scrape parameters). | + +## Scrape + +### Why use it + +Use scrape when you already have a URL and want structured content in one or more formats. + +### Preferred SDK method + +`client.scrape(url, options)` → `Document` + +### Example + +```java +import com.firecrawl.models.ScrapeOptions; +import com.firecrawl.models.Document; +import java.util.List; + +Document doc = client.scrape( + "https://docs.firecrawl.dev", + ScrapeOptions.builder() + .formats(List.of("markdown")) + .onlyMainContent(true) + .build() +); + +System.out.println(doc.getMarkdown()); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `url` | `String` | Target URL. | +| `options.formats` | `List` | Output formats. Strings: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"json"`, `"audio"`, `"video"`. Maps for structured formats like JSON extraction. | +| `options.headers` | `Map` | Custom HTTP headers. | +| `options.includeTags` | `List` | Only include these HTML tags. | +| `options.excludeTags` | `List` | Exclude these HTML tags. | +| `options.onlyMainContent` | `Boolean` | Strip nav, footer, and boilerplate. | +| `options.timeout` | `Integer` | Timeout in milliseconds. | +| `options.waitFor` | `Integer` | Wait for page to render (milliseconds). | +| `options.mobile` | `Boolean` | Use a mobile viewport. | +| `options.parsers` | `List` | File parsing controls (e.g. `"pdf"` or PDF config map). | +| `options.actions` | `List>` | Pre-scrape browser actions. | +| `options.location` | `LocationConfig` | `{ country, languages }` for geo-aware scraping. | +| `options.skipTlsVerification` | `Boolean` | Skip TLS verification. | +| `options.removeBase64Images` | `Boolean` | Drop base64 images from markdown output. | +| `options.blockAds` | `Boolean` | Block ads and cookie popups. | +| `options.proxy` | `String` | Proxy control: `"basic"`, `"stealth"`, `"enhanced"`, `"auto"`. | +| `options.maxAge` | `Long` | Use cached data up to this age (milliseconds). | +| `options.storeInCache` | `Boolean` | Cache the result. | +| `options.lockdown` | `Boolean` | Serve from cache only, never make outbound request. | +| `options.redactPII` | `Boolean` | Redact PII from content. | +| `options.auditMetadata` | `AuditMetadata` | User attribution for SIEM logging. `username` is required. | + +## Interact + +### Why use it + +Use `interact` to run code in the browser session tied to a scrape job. The Java SDK accepts `code` (required) and does not support `prompt` — use code-based interaction. + +### Preferred SDK method + +`client.interact(jobId, code)` → `BrowserExecuteResponse` + +Overloads: `interact(jobId, code, language, timeout)` and `interact(jobId, code, language, timeout, origin)`. + +### Example + +```java +import com.firecrawl.models.Document; +import com.firecrawl.models.ScrapeOptions; +import com.firecrawl.models.BrowserExecuteResponse; +import java.util.List; + +Document doc = client.scrape("https://example.com", + ScrapeOptions.builder().formats(List.of("markdown")).build()); + +String jobId = (String) doc.getMetadata().get("scrapeId"); + +BrowserExecuteResponse result = client.interact(jobId, + "console.log(await page.title());"); + +System.out.println(result.getStdout()); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `jobId` | `String` | Scrape job ID from `document.getMetadata().get("scrapeId")`. | +| `code` | `String` | Code to execute in the browser session. | +| `language` | `String` | Execution runtime: `"python"`, `"node"`, `"bash"`. Default: `"node"`. | +| `timeout` | `Integer` | Execution timeout in seconds (1-300). Default: `30`. | +| `origin` | `String` | Optional origin label for telemetry. | + +### Stop session + +`client.stopInteractiveBrowser(jobId)` → `BrowserDeleteResponse` + +### Async variants + +All methods have `*Async` variants returning `CompletableFuture<...>`: +- `scrapeAsync(url, options)` +- `searchAsync(query, options)` +- `interactAsync(jobId, code, ...)` +- `stopInteractiveBrowserAsync(jobId)` + +## Notes + +- Java uses `camelCase` parameter names matching the API directly. +- `interact` does not support `prompt` — only code-based interaction is available. Use `code` with Playwright page commands. +- Deprecated aliases: `scrapeExecute` → `interact`; `deleteScrapeBrowser` → `stopInteractiveBrowser`. +- `FirecrawlClient.fromEnv()` reads `FIRECRAWL_API_KEY` from the environment. + +## Source Of Truth + +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java` +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/ScrapeOptions.java` +- `firecrawl/apps/java-sdk/src/main/java/com/firecrawl/models/SearchOptions.java` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/node.mdx b/agent-quickstart/node.mdx new file mode 100644 index 000000000..423b3d59d --- /dev/null +++ b/agent-quickstart/node.mdx @@ -0,0 +1,177 @@ +--- +title: "Node.js Agent Quickstart" +description: "Canonical Firecrawl Node.js quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Node.js Agent Quickstart + +Canonical quickstart for external agents. Generated from SDK source (`firecrawl` **4.34.0**) and the v2 OpenAPI spec. Method names, parameters, and types match the SDK public API. + +## Install + +```bash +npm install firecrawl +``` + +## Authenticate + +```ts +import { Firecrawl } from "firecrawl"; + +const client = new Firecrawl({ + apiKey: process.env.FIRECRAWL_API_KEY, +}); +``` + +Constructor options: `apiKey` (string, falls back to `FIRECRAWL_API_KEY` env var), `apiUrl` (string, falls back to `FIRECRAWL_API_URL` or `https://api.firecrawl.dev`), `timeoutMs` (number), `maxRetries` (number), `backoffFactor` (number). + +## When To Use What + +- `search`: use when you start with a query and need discovery. +- `scrape`: use when you already have a URL and want page content. +- `interact`: use when the page needs clicks, forms, or other browser actions after a scrape has created a session. + +## Search + +### Why use it + +Use search to discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:`, for example `site:docs.firecrawl.dev crawl webhooks`. + +### Preferred SDK method + +`client.search(query, options?)` → `Promise` + +### Example + +```ts +const results = await client.search("site:docs.firecrawl.dev webhook retries", { + limit: 5, + scrapeOptions: { + formats: ["markdown"], + onlyMainContent: true, + }, +}); + +for (const hit of results.web ?? []) { + console.log(hit.url, hit.markdown?.slice(0, 200)); +} +``` + +### Return value + +`SearchData` has optional arrays: `web`, `news`, `images`, `developer`. Each entry is either a lightweight result or a full `Document` when `scrapeOptions` hydrated the hit. Do **not** access `result.data` — it does not exist. + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | `string` | Search query. Use `site:example.com` to limit to a domain. | +| `options.sources` | `("web" \| "news" \| "images")[]` | Which sources to search. | +| `options.categories` | `("github" \| "research" \| "pdf" \| "developer")[]` | Filter by category. | +| `options.includeDomains` | `string[]` | Only include these domains. Cannot combine with `excludeDomains`. | +| `options.excludeDomains` | `string[]` | Exclude these domains. Cannot combine with `includeDomains`. | +| `options.limit` | `number` | Max number of results. | +| `options.tbs` | `string` | Time-based filter (e.g. `qdr:d` for past day, `qdr:w` for past week). | +| `options.location` | `string` | Localized results (e.g. `"San Francisco,California,United States"`). | +| `options.ignoreInvalidURLs` | `boolean` | Drop URLs that cannot be scraped. | +| `options.timeout` | `number` | Request timeout in milliseconds. | +| `options.highlights` | `boolean` | Generate query-relevant highlights. | +| `options.scrapeOptions` | `ScrapeOptions` | Scrape each search result (see Scrape parameters). | + +## Scrape + +### Why use it + +Use scrape when you already have a URL and want structured content in one or more formats. + +### Preferred SDK method + +`client.scrape(url, options?)` → `Promise` + +### Example + +```ts +const doc = await client.scrape("https://docs.firecrawl.dev", { + formats: ["markdown"], + onlyMainContent: true, +}); + +console.log(doc.markdown); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `url` | `string` | Target URL. | +| `options.formats` | `FormatOption[]` | Output formats. Strings: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"`, `"attributes"`, `"branding"`, `"audio"`, `"video"`. Objects: `{ type: "json", prompt?, schema? }`, `{ type: "question", question }`, `{ type: "highlights", query }`, `{ type: "screenshot", fullPage?, quality?, viewport? }`, `{ type: "changeTracking", modes, schema?, prompt?, tag? }`, `{ type: "attributes", selectors: [{ selector, attribute }] }`. | +| `options.headers` | `Record` | Custom HTTP headers. | +| `options.includeTags` | `string[]` | Only include these HTML tags. | +| `options.excludeTags` | `string[]` | Exclude these HTML tags. | +| `options.onlyMainContent` | `boolean` | Strip nav, footer, and boilerplate. | +| `options.timeout` | `number` | Timeout in milliseconds. | +| `options.waitFor` | `number` | Wait for page to render (milliseconds). | +| `options.mobile` | `boolean` | Use a mobile viewport. | +| `options.parsers` | `("pdf" \| { type: "pdf", mode?, maxPages? })[]` | File parsing controls. | +| `options.actions` | `ActionOption[]` | Pre-scrape browser actions: `wait`, `click`, `write`, `press`, `scroll`, `screenshot`, `scrape`, `executeJavascript`, `pdf`. | +| `options.location` | `{ country?, languages? }` | Geo or language-aware scraping. | +| `options.skipTlsVerification` | `boolean` | Skip TLS verification. | +| `options.removeBase64Images` | `boolean` | Drop base64 images from markdown output. | +| `options.fastMode` | `boolean` | Faster scrapes with reduced fidelity. | +| `options.blockAds` | `boolean` | Block ads and cookie popups. | +| `options.proxy` | `string` | Proxy control: `"basic"`, `"stealth"`, `"enhanced"`, `"auto"`, or a custom URL. | +| `options.maxAge` | `number` | Use cached data up to this age (milliseconds). | +| `options.minAge` | `number` | Use cached data only if at least this old (milliseconds). | +| `options.storeInCache` | `boolean` | Cache the result. | +| `options.profile` | `{ name, saveChanges? }` | Persistent browser profile shared across scrapes and interactions. | + +## Interact + +### Why use it + +Use `interact` for code or natural-language control of the browser session tied to a scrape job (via `metadata.scrapeId`). The SDK requires at least one of `code` or `prompt`. + +### Preferred SDK method + +`client.interact(jobId, args)` → `Promise` + +### Example + +```ts +const doc = await client.scrape("https://example.com", { formats: ["markdown"] }); +const jobId = doc.metadata?.scrapeId; +if (!jobId) throw new Error("Missing scrapeId from scrape response"); + +const result = await client.interact(jobId, { + prompt: "Click the pricing tab and summarize the plans.", +}); +console.log(result.output); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `jobId` | `string` | Scrape job ID from `document.metadata.scrapeId`. | +| `args.code` | `string` | Code to execute in the browser session (e.g. Playwright `page` usage). At least one of `code` or `prompt` required. | +| `args.prompt` | `string` | Natural-language instruction for the browser agent. At least one of `code` or `prompt` required. | +| `args.language` | `"python" \| "node" \| "bash"` | Execution runtime. Default: `"node"`. | +| `args.timeout` | `number` | Execution timeout in seconds. | + +### Stop session + +`client.stopInteraction(jobId)` → `Promise` + +## Notes + +- The default `Firecrawl` export is the v2 client; v1 is available via `client.v1`. +- Deprecated aliases: `scrapeExecute` → `interact`; `stopInteractiveBrowser` and `deleteScrapeBrowser` → `stopInteraction`; `scrapeUrl` → `scrape`. +- Zod schemas passed in `formats` (for `json` or `changeTracking`) are automatically converted to JSON Schema by the SDK. +- The package declares **Node.js >= 22** in `engines`. + +## Source Of Truth + +- `firecrawl/apps/js-sdk/firecrawl/src/index.ts` +- `firecrawl/apps/js-sdk/firecrawl/src/v2/client.ts` +- `firecrawl/apps/js-sdk/firecrawl/src/v2/types.ts` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/python.mdx b/agent-quickstart/python.mdx new file mode 100644 index 000000000..f06956804 --- /dev/null +++ b/agent-quickstart/python.mdx @@ -0,0 +1,175 @@ +--- +title: "Python Agent Quickstart" +description: "Canonical Firecrawl Python quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Python Agent Quickstart + +Canonical quickstart for external agents. Generated from SDK source (`firecrawl-py` **4.22.1**) and the v2 OpenAPI spec. Method names, parameters, and types match the v2 client in `firecrawl/v2/client.py`. + +## Install + +```bash +pip install firecrawl-py +``` + +## Authenticate + +```python +import os +from firecrawl import Firecrawl + +client = Firecrawl(api_key=os.environ.get("FIRECRAWL_API_KEY")) +``` + +Constructor parameters: `api_key` (str, falls back to `FIRECRAWL_API_KEY` env var), `api_url` (str, default `"https://api.firecrawl.dev"`), `timeout` (float), `max_retries` (int, default `3`), `backoff_factor` (float, default `0.5`). + +## When To Use What + +- `search`: use when you start with a query and need discovery. +- `scrape`: use when you already have a URL and want page content. +- `interact`: use when the page needs clicks, forms, or post-scrape browser actions. + +## Search + +### Why use it + +Use search to discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:`, for example `site:docs.firecrawl.dev crawl webhooks`. + +### Preferred SDK method + +`client.search(query, **options)` → `SearchData` + +### Example + +```python +results = client.search( + "site:docs.firecrawl.dev webhook retries", + limit=5, + scrape_options={"formats": ["markdown"], "only_main_content": True}, +) + +for hit in results.web or []: + print(hit.url, (hit.markdown or "")[:200]) +``` + +### Return value + +`SearchData` has optional lists: `web`, `news`, `images`, `developer`. Each entry is either a lightweight result or a full `Document` when `scrape_options` hydrated the hit. Do **not** access `result.data` — it does not exist. + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | `str` | Search query. Use `site:example.com` to limit to a domain. | +| `sources` | `list[str \| Source]` | Which sources to search: `"web"`, `"news"`, `"images"`. | +| `categories` | `list[str \| Category]` | Filter by category: `"github"`, `"research"`, `"pdf"`. | +| `include_domains` | `list[str]` | Only include these domains. Cannot combine with `exclude_domains`. | +| `exclude_domains` | `list[str]` | Exclude these domains. Cannot combine with `include_domains`. | +| `limit` | `int` | Max number of results. | +| `tbs` | `str` | Time-based filter (e.g. `qdr:d` for past day, `qdr:w` for past week). | +| `location` | `str` | Localized results. | +| `ignore_invalid_urls` | `bool` | Drop URLs that cannot be scraped. | +| `timeout` | `int` | Request timeout in milliseconds. | +| `highlights` | `bool` | Generate query-relevant highlights. | +| `scrape_options` | `ScrapeOptions` | Scrape each search result (see Scrape parameters). | + +## Scrape + +### Why use it + +Use scrape when you already have a URL and want structured content in one or more formats. + +### Preferred SDK method + +`client.scrape(url, **options)` → `Document` + +### Example + +```python +doc = client.scrape( + "https://docs.firecrawl.dev", + formats=["markdown"], + only_main_content=True, +) + +print(doc.markdown) +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `url` | `str` | Target URL. | +| `formats` | `list[FormatOption]` | Output formats. Strings: `"markdown"`, `"html"`, `"rawHtml"`, `"links"`, `"images"`, `"screenshot"`, `"summary"`, `"changeTracking"`, `"attributes"`, `"branding"`, `"audio"`, `"video"`. Dicts: `{"type": "json", "prompt": ..., "schema": ...}`, `{"type": "question", "question": ...}`, `{"type": "highlights", "query": ...}`, `{"type": "screenshot", "fullPage": ..., "quality": ..., "viewport": ...}`, `{"type": "changeTracking", "modes": [...], "schema": ..., "prompt": ..., "tag": ...}`, `{"type": "attributes", "selectors": [{"selector": ..., "attribute": ...}]}`. | +| `headers` | `dict[str, str]` | Custom HTTP headers. | +| `include_tags` | `list[str]` | Only include these HTML tags. | +| `exclude_tags` | `list[str]` | Exclude these HTML tags. | +| `only_main_content` | `bool` | Strip nav, footer, and boilerplate. | +| `timeout` | `int` | Timeout in milliseconds. | +| `wait_for` | `int` | Wait for page to render (milliseconds). | +| `mobile` | `bool` | Use a mobile viewport. | +| `parsers` | `list[str \| PDFParser]` | File parsing controls. `"pdf"` or `{"type": "pdf", "mode": ..., "max_pages": ...}`. | +| `actions` | `list[Action]` | Pre-scrape browser actions: `wait`, `click`, `write`, `press`, `scroll`, `screenshot`, `scrape`, `executeJavascript`, `pdf`. | +| `location` | `Location` | `{"country": ..., "languages": [...]}` for geo-aware scraping. | +| `skip_tls_verification` | `bool` | Skip TLS verification. | +| `remove_base64_images` | `bool` | Drop base64 images from markdown output. | +| `fast_mode` | `bool` | Faster scrapes with reduced fidelity. | +| `block_ads` | `bool` | Block ads and cookie popups. | +| `proxy` | `str` | Proxy control: `"basic"`, `"stealth"`, `"enhanced"`, `"auto"`, or a custom URL. | +| `max_age` | `int` | Use cached data up to this age (milliseconds). | +| `store_in_cache` | `bool` | Cache the result. | +| `profile` | `dict` | `{"name": ..., "save_changes": ...}` for persistent browser profile. | + +## Interact + +### Why use it + +Use `interact` for code or natural-language control of the browser session tied to a scrape job (via `metadata.scrape_id`). The SDK requires at least one of `code` or `prompt`. + +### Preferred SDK method + +`client.interact(job_id, code=None, *, prompt=None, language="node", timeout=None)` → `BrowserExecuteResponse` + +### Example + +```python +doc = client.scrape("https://example.com", formats=["markdown"]) +job_id = doc.metadata.get("scrapeId") if doc.metadata else None +if not job_id: + raise ValueError("Missing scrapeId from scrape response") + +result = client.interact( + job_id, + prompt="Click the pricing tab and summarize the plans.", +) +print(result.output) +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `job_id` | `str` | Scrape job ID from `document.metadata["scrapeId"]`. | +| `code` | `str` | Code to execute in the browser session. At least one of `code` or `prompt` required. | +| `prompt` | `str` | Natural-language instruction for the browser agent. At least one of `code` or `prompt` required. | +| `language` | `"python" \| "node" \| "bash"` | Execution runtime. Default: `"node"`. | +| `timeout` | `int` | Execution timeout in seconds. | + +### Stop session + +`client.stop_interaction(job_id)` → `BrowserDeleteResponse` + +## Notes + +- `Firecrawl` is the v2 client. `FirecrawlApp` is a deprecated alias for `Firecrawl`. +- `AsyncFirecrawl` (alias `AsyncFirecrawlApp`) provides async versions of all methods. +- Deprecated aliases: `scrape_execute` → `interact`; `stop_interactive_browser` and `delete_scrape_browser` → `stop_interaction`; `scrape_url` → `scrape`. +- Python uses `snake_case` parameter names. The SDK converts them to `camelCase` for the API. + +## Source Of Truth + +- `firecrawl/apps/python-sdk/firecrawl/client.py` +- `firecrawl/apps/python-sdk/firecrawl/v2/client.py` +- `firecrawl/apps/python-sdk/firecrawl/v2/types.py` +- `firecrawl-docs/api-reference/v2-openapi.json` diff --git a/agent-quickstart/rust.mdx b/agent-quickstart/rust.mdx new file mode 100644 index 000000000..497259672 --- /dev/null +++ b/agent-quickstart/rust.mdx @@ -0,0 +1,218 @@ +--- +title: "Rust Agent Quickstart" +description: "Canonical Firecrawl Rust quickstart for external agents using search, scrape, and interact." +--- + +# Firecrawl Rust Agent Quickstart + +Canonical quickstart for external agents. Generated from SDK source (`firecrawl` crate **2.14.0**) and the v2 OpenAPI spec. Method names, parameters, and types match the v2 client. + +## Install + +```toml +[dependencies] +firecrawl = "2" +``` + +## Authenticate + +```rust +use firecrawl::Client; + +// Cloud client +let client = Client::new("fc-your-api-key")?; + +// Self-hosted (API key optional for keyless free tier) +let client = Client::new_selfhosted("https://your-instance.com", Some("fc-..."))?; +``` + +## When To Use What + +- `search`: use when you start with a query and need discovery. +- `scrape`: use when you already have a URL and want page content. +- `interact`: use when the page needs clicks, forms, or post-scrape browser actions. + +## Search + +### Why use it + +Use search to discover relevant pages from a query, then pick URLs to scrape or interact with. Constrain results to a site with `site:`, for example `site:docs.firecrawl.dev crawl webhooks`. + +### Preferred SDK method + +`client.search(query, options)` → `Result` + +### Example + +```rust +use firecrawl::{Client, SearchOptions, ScrapeOptions, Format}; + +let client = Client::new(std::env::var("FIRECRAWL_API_KEY")?)?; + +let response = client.search( + "site:docs.firecrawl.dev webhook retries", + SearchOptions { + limit: Some(5), + scrape_options: Some(ScrapeOptions { + formats: Some(vec![Format::Markdown]), + ..Default::default() + }), + ..Default::default() + }, +).await?; + +if let Some(web_results) = &response.data.web { + for hit in web_results { + println!("{:?}", hit); + } +} +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `query` | `impl AsRef` | Search query. Use `site:example.com` to limit to a domain. | +| `options.limit` | `Option` | Max number of results. Default: 5, max: 20. | +| `options.sources` | `Option>` | Which sources: `Web`, `News`, `Images`. | +| `options.categories` | `Option>` | Filter by category: `Github`, `Research`, `Pdf`. | +| `options.include_domains` | `Option>` | Only include these domains. | +| `options.exclude_domains` | `Option>` | Exclude these domains. | +| `options.tbs` | `Option` | Time-based filter (e.g. `qdr:d`). | +| `options.location` | `Option` | Geographic location string. | +| `options.ignore_invalid_urls` | `Option` | Drop invalid URLs. | +| `options.timeout` | `Option` | Timeout in milliseconds. | +| `options.highlights` | `Option` | Generate highlights. Defaults to true. | +| `options.scrape_options` | `Option` | Scrape each result (see Scrape parameters). | + +## Scrape + +### Why use it + +Use scrape when you already have a URL and want structured content in one or more formats. + +### Preferred SDK method + +`client.scrape(url, options)` → `Result` + +### Example + +```rust +use firecrawl::{Client, ScrapeOptions, Format}; + +let client = Client::new(std::env::var("FIRECRAWL_API_KEY")?)?; + +let doc = client.scrape( + "https://docs.firecrawl.dev", + ScrapeOptions { + formats: Some(vec![Format::Markdown]), + only_main_content: Some(true), + ..Default::default() + }, +).await?; + +if let Some(md) = &doc.markdown { + println!("{}", md); +} +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `url` | `impl AsRef` | Target URL. | +| `options.formats` | `Option>` | Output formats: `Markdown`, `Html`, `RawHtml`, `Links`, `Images`, `Screenshot`, `Summary`, `ChangeTracking`, `Json`, `Attributes`, `Branding`, `Audio`, `Video`. | +| `options.headers` | `Option>` | Custom HTTP headers. | +| `options.include_tags` | `Option>` | Only include these HTML tags. | +| `options.exclude_tags` | `Option>` | Exclude these HTML tags. | +| `options.only_main_content` | `Option` | Strip nav, footer, and boilerplate. | +| `options.timeout` | `Option` | Timeout in milliseconds. | +| `options.wait_for` | `Option` | Wait for page to render (milliseconds). | +| `options.mobile` | `Option` | Use a mobile viewport. | +| `options.parsers` | `Option>` | File parsing controls. | +| `options.actions` | `Option>` | Pre-scrape browser actions. | +| `options.location` | `Option` | `{ country, languages }` for geo-aware scraping. | +| `options.skip_tls_verification` | `Option` | Skip TLS verification. | +| `options.remove_base64_images` | `Option` | Drop base64 images from markdown output. | +| `options.fast_mode` | `Option` | Faster scrapes with reduced fidelity. | +| `options.block_ads` | `Option` | Block ads and cookie popups. | +| `options.proxy` | `Option` | Proxy control: `Basic`, `Stealth`, `Enhanced`, `Auto`. | +| `options.max_age` | `Option` | Use cached data up to this age. | +| `options.min_age` | `Option` | Use cached data only if at least this old. | +| `options.store_in_cache` | `Option` | Cache the result. | +| `options.profile` | `Option` | Persistent browser profile. | +| `options.json_options` | `Option` | JSON extraction options (prompt, schema). | +| `options.screenshot_options` | `Option` | Screenshot options (full_page, quality, viewport). | +| `options.change_tracking_options` | `Option` | Change tracking options (modes, schema, prompt, tag). | +| `options.attribute_selectors` | `Option>` | Attribute extraction selectors. | + +### Convenience method + +`client.scrape_with_schema(url, schema, prompt)` → `Result` + +Calls `scrape()` with `formats: [Json]` and the provided JSON schema and optional prompt. Returns the extracted JSON directly. + +## Interact + +### Why use it + +Use `interact` for code or natural-language control of the browser session tied to a scrape job. The SDK requires at least one of `code` or `prompt`. + +### Preferred SDK method + +`client.interact(job_id, options)` → `Result` + +### Example + +```rust +use firecrawl::{Client, ScrapeOptions, ScrapeExecuteOptions, Format}; + +let client = Client::new(std::env::var("FIRECRAWL_API_KEY")?)?; + +let doc = client.scrape("https://example.com", ScrapeOptions { + formats: Some(vec![Format::Markdown]), + ..Default::default() +}).await?; + +let job_id = doc.metadata + .as_ref() + .and_then(|m| m.get("scrapeId")) + .and_then(|v| v.as_str()) + .expect("Missing scrapeId"); + +let result = client.interact(job_id, ScrapeExecuteOptions { + code: Some("console.log(await page.title());".to_string()), + ..Default::default() +}).await?; + +println!("{:?}", result.output); +``` + +### Parameters + +| Parameter | Type | Description | +|---|---|---| +| `job_id` | `impl AsRef` | Scrape job ID from document metadata. | +| `options.code` | `Option` | Code to execute in the browser session. At least one of `code` or `prompt` required. | +| `options.prompt` | `Option` | Natural-language instruction for the browser agent. At least one of `code` or `prompt` required. | +| `options.language` | `Option` | Execution runtime: `Python`, `Node`, `Bash`. Default: `Node`. | +| `options.timeout` | `Option` | Execution timeout in seconds. | + +### Stop session + +`client.stop_interaction(job_id)` → `Result` + +## Notes + +- Rust uses `snake_case` field names. The SDK serializes them to `camelCase` for the API. +- All options structs derive `Default`, so use `..Default::default()` for unset fields. +- Deprecated aliases: `scrape_execute` → `interact`; `stop_interactive_browser` and `delete_scrape_browser` → `stop_interaction`. +- The SDK auto-sets `origin` to `"rust-sdk@{version}"` if not provided. + +## Source Of Truth + +- `firecrawl/apps/rust-sdk/src/client.rs` +- `firecrawl/apps/rust-sdk/src/scrape.rs` +- `firecrawl/apps/rust-sdk/src/search.rs` +- `firecrawl/apps/rust-sdk/src/types.rs` +- `firecrawl-docs/api-reference/v2-openapi.json`