Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "firecrawl-cli",
"version": "1.16.0",
"version": "1.16.1",
"description": "Command-line interface for Firecrawl. Scrape, crawl, and extract data from any website directly from your terminal.",
"main": "dist/index.js",
"bin": {
Expand Down
42 changes: 42 additions & 0 deletions src/__tests__/commands/scrape.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,48 @@ describe('executeScrape', () => {
});
});

it('should include schema in json format when provided', async () => {
const mockResponse = { json: { title: 'Example Domain' } };
const schema = {
type: 'object',
properties: {
title: { type: 'string' },
},
required: ['title'],
};
mockClient.scrape.mockResolvedValue(mockResponse);

await executeScrape({
url: 'https://example.com',
formats: ['json'],
schema,
});

expect(mockClient.scrape).toHaveBeenCalledWith('https://example.com', {
formats: [{ type: 'json', schema }],
integration: 'cli',
});
});

it('should include actions and proxy when provided', async () => {
const mockResponse = { markdown: '# Test' };
const actions = [{ type: 'wait', milliseconds: 100 }];
mockClient.scrape.mockResolvedValue(mockResponse);

await executeScrape({
url: 'https://example.com',
actions,
proxy: 'basic',
});

expect(mockClient.scrape).toHaveBeenCalledWith('https://example.com', {
formats: ['markdown'],
integration: 'cli',
actions,
proxy: 'basic',
});
});

it('should not include location parameter when not provided', async () => {
const mockResponse = { markdown: '# Test' };
mockClient.scrape.mockResolvedValue(mockResponse);
Expand Down
84 changes: 71 additions & 13 deletions src/commands/agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
import type {
AgentOptions,
AgentResult,
AgentStatus,
AgentStatusResult,
} from '../types/agent';
import type { AgentWebhookConfig } from '@mendable/firecrawl-js';
import { getClient } from '../utils/client';
import { isJobId } from '../utils/job';
import { writeOutput } from '../utils/output';
Expand Down Expand Up @@ -62,6 +64,12 @@ function loadSchemaFromFile(filePath: string): Record<string, unknown> {
}
}

type AgentStatusFromApi = 'processing' | 'completed' | 'failed';

function normalizeAgentStatus(status: AgentStatusFromApi): AgentStatus {
return status as AgentStatus;
}

/**
* Execute agent status check (with optional wait/polling)
*/
Expand All @@ -75,11 +83,16 @@ async function checkAgentStatus(
if (!options.wait) {
try {
const status = await app.getAgentStatus(jobId);
const normalizedStatus = normalizeAgentStatus(
status.status as AgentStatusFromApi
);
const isCancelled = normalizedStatus === 'cancelled';

return {
success: status.success,
success: isCancelled ? true : status.success,
data: {
id: jobId,
status: status.status,
status: normalizedStatus,
data: status.data,
creditsUsed: status.creditsUsed,
expiresAt: status.expiresAt,
Expand Down Expand Up @@ -113,30 +126,35 @@ async function checkAgentStatus(
try {
// Check initial status
let agentStatus = await app.getAgentStatus(jobId);
spinner.update(`Agent ${agentStatus.status}... (Job ID: ${jobId})`);
const normalizedStatusInitial = normalizeAgentStatus(
agentStatus.status as AgentStatusFromApi
);
spinner.update(`Agent ${normalizedStatusInitial}... (Job ID: ${jobId})`);

while (true) {
if (agentStatus.status === 'completed') {
const currentNormalizedStatus = normalizeAgentStatus(agentStatus.status);

if (currentNormalizedStatus === 'completed') {
spinner.succeed('Agent completed');
return {
success: agentStatus.success,
data: {
id: jobId,
status: agentStatus.status,
status: currentNormalizedStatus,
data: agentStatus.data,
creditsUsed: agentStatus.creditsUsed,
expiresAt: agentStatus.expiresAt,
},
};
}

if (agentStatus.status === 'failed') {
if (currentNormalizedStatus === 'failed') {
spinner.fail('Agent failed');
return {
success: false,
data: {
id: jobId,
status: agentStatus.status,
status: currentNormalizedStatus,
data: agentStatus.data,
creditsUsed: agentStatus.creditsUsed,
expiresAt: agentStatus.expiresAt,
Expand All @@ -145,6 +163,20 @@ async function checkAgentStatus(
};
}

if (currentNormalizedStatus === 'cancelled') {
spinner.succeed('Agent cancelled');
return {
success: true,
data: {
id: jobId,
status: currentNormalizedStatus,
data: agentStatus.data,
creditsUsed: agentStatus.creditsUsed,
expiresAt: agentStatus.expiresAt,
},
};
}

// Check timeout
if (timeoutMs && Date.now() - startTime > timeoutMs) {
spinner.fail(`Timeout after ${options.timeout}s`);
Expand All @@ -156,7 +188,10 @@ async function checkAgentStatus(

await new Promise((resolve) => setTimeout(resolve, pollMs));
agentStatus = await app.getAgentStatus(jobId);
spinner.update(`Agent ${agentStatus.status}... (Job ID: ${jobId})`);
const loopNormalizedStatus = normalizeAgentStatus(
agentStatus.status as AgentStatusFromApi
);
spinner.update(`Agent ${loopNormalizedStatus}... (Job ID: ${jobId})`);
}
} catch (error) {
spinner.fail('Failed to check agent status');
Expand All @@ -177,7 +212,25 @@ export async function executeAgent(
): Promise<AgentResult | AgentStatusResult> {
try {
const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl });
const { prompt, status, wait, pollInterval, timeout } = options;
const { prompt, status, cancel, wait, pollInterval, timeout } = options;

if (cancel) {
const cancelled = await app.cancelAgent(prompt);
if (!cancelled) {
return {
success: false,
error: `Failed to cancel agent job ${prompt}`,
};
}

return {
success: true,
data: {
id: prompt,
status: 'cancelled',
},
};
}

// If status flag is set or input looks like a job ID, check status
if (status || isJobId(prompt)) {
Expand All @@ -201,6 +254,7 @@ export async function executeAgent(
maxCredits?: number;
pollInterval?: number;
timeout?: number;
webhook?: string | AgentWebhookConfig;
integration?: string;
} = {
prompt,
Expand All @@ -219,6 +273,9 @@ export async function executeAgent(
if (options.maxCredits !== undefined) {
agentParams.maxCredits = options.maxCredits;
}
if (options.webhook) {
agentParams.webhook = options.webhook;
}

// If wait mode, use polling with spinner
if (wait) {
Expand Down Expand Up @@ -259,30 +316,31 @@ export async function executeAgent(
await new Promise((resolve) => setTimeout(resolve, pollMs));

const agentStatus = await app.getAgentStatus(jobId);
const normalizedStatus = normalizeAgentStatus(agentStatus.status);

if (agentStatus.status === 'completed') {
if (normalizedStatus === 'completed') {
process.removeListener('SIGINT', handleInterrupt);
spinner.succeed('Agent completed');
return {
success: agentStatus.success,
data: {
id: jobId,
status: agentStatus.status,
status: normalizedStatus,
data: agentStatus.data,
creditsUsed: agentStatus.creditsUsed,
expiresAt: agentStatus.expiresAt,
},
};
}

if (agentStatus.status === 'failed') {
if (normalizedStatus === 'failed') {
process.removeListener('SIGINT', handleInterrupt);
spinner.fail('Agent failed');
return {
success: false,
data: {
id: jobId,
status: agentStatus.status,
status: normalizedStatus,
data: agentStatus.data,
creditsUsed: agentStatus.creditsUsed,
expiresAt: agentStatus.expiresAt,
Expand Down
37 changes: 36 additions & 1 deletion src/commands/crawl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,34 @@ export async function executeCrawl(
): Promise<CrawlResult | CrawlStatusResult> {
try {
const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl });
const { urlOrJobId, status, wait, pollInterval, timeout } = options;
const { urlOrJobId, status, wait, pollInterval, timeout, cancel } = options;

if (cancel) {
if (!isJobId(urlOrJobId)) {
return {
success: false,
error: 'Cancel mode requires a job ID',
};
}

const cancelled = await app.cancelCrawl(urlOrJobId);
if (!cancelled) {
return {
success: false,
error: `Failed to cancel crawl job ${urlOrJobId}`,
};
}

return {
success: true,
data: {
id: urlOrJobId,
status: 'cancelled',
total: 0,
completed: 0,
},
};
}

// If status flag is set or input looks like a job ID, check status
if (status || isJobId(urlOrJobId)) {
Expand Down Expand Up @@ -95,6 +122,14 @@ export async function executeCrawl(
crawlOptions.maxConcurrency = options.maxConcurrency;
}

if (options.scrapeOptions) {
crawlOptions.scrapeOptions = options.scrapeOptions;
}

if (options.webhook) {
crawlOptions.webhook = options.webhook;
}

// If wait mode, use the convenience crawl method with polling
if (wait) {
// Set polling options
Expand Down
19 changes: 17 additions & 2 deletions src/commands/scrape.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,20 @@ export async function executeScrape(
formats.push({ type: 'query', prompt: options.query } as any);
}

const resolvedFormats = formats.map((format) => {
if (format === 'json' && options.schema) {
return { type: 'json', schema: options.schema } as FormatOption;
}
return format;
});

// If no formats specified, default to markdown
if (formats.length === 0) {
formats.push('markdown');
resolvedFormats.push('markdown');
}

const scrapeParams: Record<string, unknown> = {
formats,
formats: resolvedFormats,
integration: 'cli',
};

Expand Down Expand Up @@ -118,6 +125,14 @@ export async function executeScrape(
scrapeParams.profile = options.profile;
}

if (options.actions) {
scrapeParams.actions = options.actions;
}

if (options.proxy) {
scrapeParams.proxy = options.proxy;
}

if (options.lockdown) {
scrapeParams.lockdown = true;
}
Expand Down
Loading
Loading