From b7c16d0df6d3fb38d6a870f65f2e71b1f294caa3 Mon Sep 17 00:00:00 2001 From: Developers Digest <124798203+developersdigest@users.noreply.github.com> Date: Wed, 15 Apr 2026 13:56:17 -0400 Subject: [PATCH 1/2] feat: vendor agent scaffolder into firecrawl-cli MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the `npx firecrawl-agent-cli` delegator with an in-process scaffolder vendored from firecrawl/firecrawl-agent. No separate npm package for the agent CLI — the root firecrawl-cli clones the public agent repo at runtime to fetch templates + agent-core. - Vendor under src/utils/agent-scaffold/ (manifest, scaffold, credentials, ui, create-flow). Upstream-tracked; keep in sync. - manifest.ts: strip the "bundled next to CLI" branch (doesn't apply here), always clone the public repo. Nested-manifest fallback handles the current `.internal/cli/agent-manifest.json` location. - Extract handleCreate from upstream init.ts; drop the commander wrapper since the root CLI owns the command surface. - create.ts now lazy-imports the scaffolder so non-create paths stay snappy. Smoke-tested end-to-end against a local agent repo via --from: next template scaffolds correctly with agent-core, .env.local, and expected file layout. --- src/commands/create.ts | 102 +--- src/utils/agent-scaffold/create-flow.ts | 600 ++++++++++++++++++++++++ src/utils/agent-scaffold/credentials.ts | 63 +++ src/utils/agent-scaffold/manifest.ts | 199 ++++++++ src/utils/agent-scaffold/scaffold.ts | 246 ++++++++++ src/utils/agent-scaffold/ui.ts | 41 ++ 6 files changed, 1175 insertions(+), 76 deletions(-) create mode 100644 src/utils/agent-scaffold/create-flow.ts create mode 100644 src/utils/agent-scaffold/credentials.ts create mode 100644 src/utils/agent-scaffold/manifest.ts create mode 100644 src/utils/agent-scaffold/scaffold.ts create mode 100644 src/utils/agent-scaffold/ui.ts diff --git a/src/commands/create.ts b/src/commands/create.ts index 0b753f332..2d820671a 100644 --- a/src/commands/create.ts +++ b/src/commands/create.ts @@ -1,47 +1,17 @@ /** * `firecrawl create` command — scaffolds Firecrawl starter projects. * - * Hidden from --help until `firecrawl-agent-cli` is published to npm. + * Hidden from --help until the flow is battle-tested in the wild. The + * scaffolder is vendored under `src/utils/agent-scaffold/` (mirrored from + * `firecrawl/firecrawl-agent`). At runtime it clones the public agent repo + * to get templates — no separate npm package for the agent CLI. + * * Once visible, the command tree will grow to include additional kinds * (scrape, browser, ai, app). For now, `agent` is the only kind. - * - * Implementation is a thin delegator: `firecrawl create agent ...` execs - * `npx -y firecrawl-agent-cli create ...` and passes all flags through. - * This avoids vendoring the scaffold code in the root CLI; the agent repo - * remains the single source of truth for templates and the manifest. */ import { Command } from 'commander'; -import { spawn } from 'child_process'; - -/** npm package name of the Firecrawl Agent CLI (bin: `firecrawl-agent`). */ -const AGENT_CLI_PACKAGE = 'firecrawl-agent-cli'; - -/** - * Execute `npx -y create ...` with inherited stdio so - * the agent CLI's interactive prompts render in the user's terminal. - * Resolves with the child exit code; callers forward it to `process.exit`. - */ -function runAgentCli(args: string[]): Promise { - const npx = process.platform === 'win32' ? 'npx.cmd' : 'npx'; - return new Promise((resolve) => { - const child = spawn(npx, ['-y', AGENT_CLI_PACKAGE, 'create', ...args], { - stdio: 'inherit', - env: process.env, - }); - child.on('exit', (code) => resolve(code ?? 1)); - child.on('error', (err) => { - console.error( - `\nFailed to launch ${AGENT_CLI_PACKAGE} via npx:`, - err.message - ); - console.error( - `\n Install it directly and retry: npm install -g ${AGENT_CLI_PACKAGE}\n` - ); - resolve(1); - }); - }); -} +import type { CreateOptions } from '../utils/agent-scaffold/create-flow'; function collect(val: string, acc: string[]): string[] { acc.push(val); @@ -49,8 +19,8 @@ function collect(val: string, acc: string[]): string[] { } /** - * Build the `agent` subcommand. Flag surface mirrors `firecrawl-agent create` - * exactly — anything the downstream CLI accepts is passed through verbatim. + * Build the `agent` subcommand. Flag surface mirrors the upstream agent CLI. + * The scaffold flow itself lives in `utils/agent-scaffold/create-flow.ts`. */ function createAgentSubcommand(): Command { return new Command('agent') @@ -88,48 +58,28 @@ function createAgentSubcommand(): Command { [] ) .option('--skip-install', 'Skip npm install') - .allowUnknownOption() // Forward future flags without requiring a CLI update .action( async ( projectName: string | undefined, - options: Record, - cmd: Command + options: CreateOptions & Record ) => { - const args: string[] = []; - if (projectName) args.push(projectName); - - // Pass through known options. Commander camelCases hyphenated flags, - // so we map back to the CLI-facing kebab-case form. - const flagMap: Array<[string, string]> = [ - ['template', '-t'], - ['provider', '--provider'], - ['model', '--model'], - ['subAgentProvider', '--sub-agent-provider'], - ['subAgentModel', '--sub-agent-model'], - ['from', '--from'], - ['apiKey', '--api-key'], - ]; - for (const [optKey, flag] of flagMap) { - const val = options[optKey]; - if (typeof val === 'string' && val.length > 0) args.push(flag, val); - } - - // --key is repeatable - const keys = options.key; - if (Array.isArray(keys)) { - for (const k of keys) { - if (typeof k === 'string' && k.length > 0) args.push('--key', k); - } - } - - if (options.skipInstall) args.push('--skip-install'); - - // Forward any unknown/forward-compatible options verbatim. - const passthrough = cmd.args.slice(projectName ? 1 : 0); - for (const extra of passthrough) args.push(extra); - - const code = await runAgentCli(args); - if (code !== 0) process.exit(code); + // Lazy-load the scaffolder so startup cost (inquirer, git) only hits + // users who actually invoke `create`. Keeps the rest of the CLI snappy. + const { handleCreate } = + await import('../utils/agent-scaffold/create-flow'); + await handleCreate(projectName, { + template: options.template, + provider: options.provider, + model: options.model, + subAgentProvider: options.subAgentProvider, + subAgentModel: options.subAgentModel, + from: options.from, + apiKey: options.apiKey, + key: Array.isArray(options.key) + ? (options.key as string[]) + : undefined, + skipInstall: options.skipInstall as boolean | undefined, + }); } ); } diff --git a/src/utils/agent-scaffold/create-flow.ts b/src/utils/agent-scaffold/create-flow.ts new file mode 100644 index 000000000..6ffaa4b59 --- /dev/null +++ b/src/utils/agent-scaffold/create-flow.ts @@ -0,0 +1,600 @@ +/** + * Vendored from firecrawl/firecrawl-agent:.internal/cli/src/commands/init.ts + * + * This is the interactive-to-non-interactive create flow. The original file + * also exposed a `createInitCommand` commander builder — we drop that here + * because the root CLI's `src/commands/create.ts` owns the command surface. + * `handleCreate` (renamed from `handleInit`) is what we export. + * + * Keep in sync with upstream when the create flow evolves. + */ + +import { select, password, input } from '@inquirer/prompts'; +import * as path from 'path'; +import { spawn } from 'child_process'; +import { + getTemplates, + getProviders, + getTemplate, + loadExternalManifest, +} from './manifest'; +import type { ProviderEntry, TemplateEntry } from './manifest'; +import { resolveFirecrawlApiKey } from './credentials'; +import { scaffoldProject } from './scaffold'; +import { + printBanner, + success, + warn, + info, + dim, + reset, + green, + bold, +} from './ui'; + +function resolveSpawnCommand(command: string): string { + if (process.platform === 'win32' && command === 'npm') { + return 'npm.cmd'; + } + return command; +} + +export interface CreateOptions { + template?: string; + provider?: string; + model?: string; + subAgentProvider?: string; + subAgentModel?: string; + from?: string; + apiKey?: string; + key?: string[]; + skipInstall?: boolean; +} + +function getSelectedProvider( + providers: ProviderEntry[], + providerId?: string +): ProviderEntry | undefined { + return providers.find((provider) => provider.id === providerId); +} + +function parseKeyFlags(keys: string[]): Record { + const map: Record = {}; + const providerEnvMap: Record = { + anthropic: 'ANTHROPIC_API_KEY', + openai: 'OPENAI_API_KEY', + google: 'GOOGLE_GENERATIVE_AI_API_KEY', + gateway: 'AI_GATEWAY_API_KEY', + }; + + for (const entry of keys) { + const eq = entry.indexOf('='); + if (eq === -1) continue; + const provider = entry.slice(0, eq).toLowerCase(); + const value = entry.slice(eq + 1); + const envVar = providerEnvMap[provider] ?? provider.toUpperCase(); + map[envVar] = value; + } + return map; +} + +/** + * Prompt the user to pick a provider+model pair from the available manifest. + * Used by both the initial orchestrator prompt and the review-loop "Change…" + * actions so changes stay in sync. + */ +async function promptProviderAndModel( + availableProviders: ProviderEntry[], + message: string +): Promise<{ provider: ProviderEntry; modelId: string }> { + const providerId = await select({ + message, + choices: availableProviders + .filter((p) => p.models && p.models.length > 0) + .map((provider) => ({ + name: `${provider.name} ${dim}${provider.models[0].name}${reset}`, + value: provider.id, + })), + }); + const provider = getSelectedProvider(availableProviders, providerId)!; + + let modelId: string; + if (provider.id === 'custom-openai') { + modelId = + (await input({ message: 'Model ID', default: 'gpt-4o' })).trim() || + 'gpt-4o'; + } else if (provider.models.length > 1) { + modelId = await select({ + message: 'Model', + choices: provider.models.map((m) => ({ name: m.name, value: m.id })), + }); + } else { + modelId = provider.models[0]?.id ?? 'gpt-4o'; + } + + return { provider, modelId }; +} + +export async function handleCreate( + rawName: string | undefined, + options: CreateOptions +): Promise { + printBanner(); + + // --- Project name: arg > interactive prompt > default --- + let projectName: string; + + if (rawName) { + projectName = rawName; + } else if (process.stdin.isTTY) { + projectName = + ( + await input({ + message: 'Project name', + default: 'my-firecrawl-agent', + }) + ).trim() || 'my-firecrawl-agent'; + } else { + projectName = 'my-firecrawl-agent'; + } + + // Load external manifest if --from is provided + if (options.from) { + try { + await loadExternalManifest(options.from); + success(`Loaded manifest from ${options.from}`); + } catch (err) { + warn(err instanceof Error ? err.message : String(err)); + process.exit(1); + } + console.log(''); + } + + const availableProviders = getProviders(); + + // --- Template selection (only interactive prompt if not passed via flag) --- + const templates = getTemplates(); + let template: TemplateEntry; + + if (options.template) { + const found = getTemplate(options.template); + if (!found) { + warn( + `Unknown template "${options.template}". Available: ${templates.map((t) => t.id).join(', ')}` + ); + process.exit(1); + } + template = found; + } else { + const templateId = await select({ + message: 'Template', + choices: templates.map((t) => ({ + name: `${t.name} ${dim}${t.description}${reset}`, + value: t.id, + })), + }); + template = getTemplate(templateId)!; + } + + let selectedProvider = getSelectedProvider( + availableProviders, + options.provider + ); + if (options.provider && !selectedProvider) { + warn( + `Unknown provider "${options.provider}". Available: ${availableProviders.map((p) => p.id).join(', ')}` + ); + process.exit(1); + } + + if (!selectedProvider) { + if (process.stdin.isTTY) { + const providerId = await select({ + message: 'Default model provider', + choices: availableProviders + .filter((p) => p.models && p.models.length > 0) + .map((provider) => ({ + name: `${provider.name} ${dim}${provider.models[0].name}${reset}`, + value: provider.id, + })), + }); + selectedProvider = getSelectedProvider(availableProviders, providerId)!; + } else { + selectedProvider = + getSelectedProvider(availableProviders, 'google') ?? + availableProviders[0]; + } + } + + // --- Orchestrator model selection --- + let selectedModelId: string; + + if (options.model) { + selectedModelId = options.model; + } else if (selectedProvider.id === 'custom-openai') { + selectedModelId = ( + await input({ + message: 'Model ID', + default: 'gpt-4o', + }) + ).trim(); + } else if (selectedProvider.models.length > 1 && process.stdin.isTTY) { + selectedModelId = await select({ + message: 'Default model', + choices: selectedProvider.models.map((m) => ({ + name: m.name, + value: m.id, + })), + }); + } else { + selectedModelId = selectedProvider.models[0]?.id ?? 'gpt-4o'; + } + + // --- Sub-agent model selection --- + let selectedSubProvider: ProviderEntry = selectedProvider; + let selectedSubModelId: string = selectedModelId; + + if (options.subAgentProvider || options.subAgentModel) { + const subProv = options.subAgentProvider + ? getSelectedProvider(availableProviders, options.subAgentProvider) + : selectedProvider; + if (options.subAgentProvider && !subProv) { + warn( + `Unknown sub-agent provider "${options.subAgentProvider}". Available: ${availableProviders.map((p) => p.id).join(', ')}` + ); + process.exit(1); + } + selectedSubProvider = subProv ?? selectedProvider; + selectedSubModelId = + options.subAgentModel ?? + (selectedSubProvider.id === selectedProvider.id + ? selectedModelId + : (selectedSubProvider.models[0]?.id ?? selectedModelId)); + } else if (process.stdin.isTTY && !options.model) { + const subChoice = await select({ + message: 'Sub-agent model', + choices: [ + { + name: `Same as orchestrator ${dim}(${selectedModelId})${reset}`, + value: '__same__', + }, + ...availableProviders.flatMap((p) => + p.models.map((m) => ({ + name: `${p.name} — ${m.name}`, + value: `${p.id}::${m.id}`, + })) + ), + ], + }); + if (subChoice !== '__same__') { + const [pId, mId] = subChoice.split('::'); + const p = getSelectedProvider(availableProviders, pId); + if (p) { + selectedSubProvider = p; + selectedSubModelId = mId; + } + } + } + + // --- Custom OpenAI endpoint --- + let customEndpoint: string | undefined; + if (selectedProvider.endpointEnvVar && process.stdin.isTTY) { + customEndpoint = + ( + await input({ + message: `Base URL ${dim}(OpenAI-compatible endpoint)${reset}`, + default: 'https://api.openai.com/v1', + }) + ).trim() || undefined; + } + + // --- Collect all env vars silently --- + const envVars: Record = {}; + const missing = new Set(); + + envVars.MODEL_PROVIDER = selectedProvider.id; + envVars.MODEL_ID = selectedModelId; + + if (selectedProvider.endpointEnvVar && customEndpoint) { + envVars[selectedProvider.endpointEnvVar] = customEndpoint; + } + + // Firecrawl API key: flag > env > credentials > prompt + if (options.apiKey) { + envVars.FIRECRAWL_API_KEY = options.apiKey; + } else { + const resolved = await resolveFirecrawlApiKey(); + if (resolved) { + envVars.FIRECRAWL_API_KEY = resolved.key; + } + } + + // Provider keys from --key flags + const flagKeys = parseKeyFlags(options.key ?? []); + Object.assign(envVars, flagKeys); + + // Auto-detect remaining provider keys from environment + for (const provider of availableProviders) { + if (!envVars[provider.envVar] && process.env[provider.envVar]) { + envVars[provider.envVar] = process.env[provider.envVar]!; + } + } + + // Prompt for missing required keys when running interactively + if (!envVars.FIRECRAWL_API_KEY) { + const key = await password({ + message: `Firecrawl API key ${dim}(https://firecrawl.dev/app/api-keys)${reset}`, + }); + if (key) { + envVars.FIRECRAWL_API_KEY = key; + } else { + missing.add('FIRECRAWL_API_KEY'); + } + } + + if (!envVars[selectedProvider.envVar] && process.stdin.isTTY) { + const key = await password({ + message: `${selectedProvider.name} API key ${dim}(${selectedProvider.hint})${reset}`, + }); + if (key) { + envVars[selectedProvider.envVar] = key; + } else { + missing.add(selectedProvider.envVar); + } + } + + if ( + selectedSubProvider.id !== selectedProvider.id && + !envVars[selectedSubProvider.envVar] && + process.stdin.isTTY + ) { + const key = await password({ + message: `${selectedSubProvider.name} API key ${dim}(${selectedSubProvider.hint})${reset}`, + }); + if (key) { + envVars[selectedSubProvider.envVar] = key; + } else { + missing.add(selectedSubProvider.envVar); + } + } + + for (const envVar of template.optionalEnvVars) { + if (!envVars[envVar]) missing.add(envVar); + } + + // --- Config summary review loop --- + const fullyFlagged = !!( + options.template && + (options.apiKey || envVars.FIRECRAWL_API_KEY) && + options.provider && + options.model + ); + if (process.stdin.isTTY && !fullyFlagged) { + let confirmed = false; + while (!confirmed) { + printConfigSummary({ + template, + orchProvider: selectedProvider, + orchModelId: selectedModelId, + subProvider: selectedSubProvider, + subModelId: selectedSubModelId, + envVars, + missing, + }); + const action = await select({ + message: 'Review', + choices: [ + { name: 'Continue', value: 'continue' }, + { name: 'Change orchestrator model', value: 'orch' }, + { name: 'Change sub-agent model', value: 'sub' }, + { name: 'Cancel', value: 'cancel' }, + ], + }); + if (action === 'cancel') { + info('Cancelled.'); + process.exit(0); + } + if (action === 'orch') { + const picked = await promptProviderAndModel( + availableProviders, + 'Orchestrator model' + ); + const prev = selectedProvider; + selectedProvider = picked.provider; + selectedModelId = picked.modelId; + envVars.MODEL_PROVIDER = selectedProvider.id; + envVars.MODEL_ID = selectedModelId; + if (prev.envVar !== selectedProvider.envVar) { + if (!envVars[selectedProvider.envVar]) { + const key = await password({ + message: `${selectedProvider.name} API key ${dim}(${selectedProvider.hint})${reset}`, + }); + if (key) envVars[selectedProvider.envVar] = key; + else missing.add(selectedProvider.envVar); + } + } + continue; + } + if (action === 'sub') { + const sameChoice = await select({ + message: 'Sub-agent model', + choices: [ + { + name: `Same as orchestrator ${dim}(${selectedModelId})${reset}`, + value: '__same__', + }, + { name: 'Different provider / model', value: '__different__' }, + ], + }); + if (sameChoice === '__same__') { + selectedSubProvider = selectedProvider; + selectedSubModelId = selectedModelId; + } else { + const picked = await promptProviderAndModel( + availableProviders, + 'Sub-agent model' + ); + selectedSubProvider = picked.provider; + selectedSubModelId = picked.modelId; + if ( + selectedSubProvider.id !== selectedProvider.id && + !envVars[selectedSubProvider.envVar] + ) { + const key = await password({ + message: `${selectedSubProvider.name} API key ${dim}(${selectedSubProvider.hint})${reset}`, + }); + if (key) envVars[selectedSubProvider.envVar] = key; + else missing.add(selectedSubProvider.envVar); + } + } + continue; + } + confirmed = true; + } + } + + // --- Scaffold --- + const projectDir = path.resolve(process.cwd(), projectName); + console.log(''); + info(`Creating a new Firecrawl Agent app in ${projectDir}`); + console.log(''); + + await scaffoldProject({ + projectDir, + template, + envVars, + selectedProvider: selectedProvider.id, + defaultModelId: envVars.MODEL_ID, + subAgentProvider: selectedSubProvider.id, + subAgentModelId: selectedSubModelId, + skipInstall: options.skipInstall, + }); + + // --- Summary --- + console.log(''); + console.log(` ${green}${bold}Ready!${reset} ${projectDir}`); + console.log(''); + + const detected = Object.keys(envVars).filter( + (k) => /_API_KEY$/.test(k) && envVars[k] + ); + if (detected.length > 0) { + info(`Keys: ${detected.join(', ')}`); + } + info(`Orchestrator: ${selectedProvider.name} (${envVars.MODEL_ID})`); + if ( + selectedSubProvider.id === selectedProvider.id && + selectedSubModelId === selectedModelId + ) { + info(`Sub-agent: same as orchestrator`); + } else { + info(`Sub-agent: ${selectedSubProvider.name} (${selectedSubModelId})`); + } + if (missing.size > 0) { + info( + `Missing: ${Array.from(missing).join(', ')} ${dim}(add to .env later)${reset}` + ); + } + console.log(''); + + // --- What next? --- + if (fullyFlagged || !process.stdin.isTTY) { + console.log(` cd ${projectName} && ${template.devCommand}`); + console.log(''); + return; + } + + const action = await select({ + message: 'Next', + choices: [ + { + name: `Start dev server ${dim}${template.devCommand}${reset}`, + value: 'dev', + }, + { name: 'Exit', value: 'exit' }, + ], + }); + + if (action === 'dev') { + console.log(''); + const [cmd, ...args] = template.devCommand.split(' '); + spawn(resolveSpawnCommand(cmd), args, { + cwd: projectDir, + stdio: 'inherit', + }); + } else { + console.log(` cd ${projectName} && ${template.devCommand}`); + console.log(''); + } +} + +function printConfigSummary(args: { + template: TemplateEntry; + orchProvider: ProviderEntry; + orchModelId: string; + subProvider: ProviderEntry; + subModelId: string; + envVars: Record; + missing: Set; +}): void { + const { + template, + orchProvider, + orchModelId, + subProvider, + subModelId, + envVars, + missing, + } = args; + const sameAsOrch = + subProvider.id === orchProvider.id && subModelId === orchModelId; + const mask = (v?: string) => (v ? v.slice(0, 5) + '••••' + v.slice(-4) : ''); + console.log(''); + console.log(` ${bold}Config${reset}`); + console.log(` ${dim}────────────────────────────────${reset}`); + console.log( + ` Orchestrator ${green}${orchProvider.name}${reset} ${dim}·${reset} ${orchModelId}` + ); + console.log( + ` Sub-agent ${green}${subProvider.name}${reset} ${dim}·${reset} ${subModelId}` + + (sameAsOrch ? ` ${dim}(same as orchestrator)${reset}` : '') + ); + console.log(` Template ${template.name}`); + const fcKey = envVars.FIRECRAWL_API_KEY; + console.log( + ` Firecrawl key ${fcKey ? `${mask(fcKey)} ${green}✓${reset}` : `${dim}(missing)${reset} !`}` + ); + const orchKey = envVars[orchProvider.envVar]; + console.log( + ` ${orchProvider.name} key` + + ' '.repeat(Math.max(1, 14 - orchProvider.name.length - 4)) + + (orchKey + ? `${mask(orchKey)} ${green}✓${reset}` + : `${dim}(missing)${reset} !`) + ); + if (!sameAsOrch) { + const subKey = envVars[subProvider.envVar]; + console.log( + ` ${subProvider.name} key (sub)` + + ' '.repeat(Math.max(1, 8 - subProvider.name.length)) + + (subKey + ? `${mask(subKey)} ${green}✓${reset}` + : `${dim}(missing)${reset} !`) + ); + } + if (missing.size > 0) { + console.log( + ` ${dim}Other missing: ${ + Array.from(missing) + .filter( + (m) => + m !== orchProvider.envVar && + m !== subProvider.envVar && + m !== 'FIRECRAWL_API_KEY' + ) + .join(', ') || '—' + }${reset}` + ); + } + console.log(''); +} diff --git a/src/utils/agent-scaffold/credentials.ts b/src/utils/agent-scaffold/credentials.ts new file mode 100644 index 000000000..38aee617e --- /dev/null +++ b/src/utils/agent-scaffold/credentials.ts @@ -0,0 +1,63 @@ +/** + * Vendored from firecrawl/firecrawl-agent:.internal/cli/src/utils/credentials.ts + * + * Resolves the Firecrawl API key from env or the shared `firecrawl-cli` config + * directory. Same path as the root CLI's own credentials store (that's the + * point — both CLIs read from the same place), so keeping this as a light + * standalone copy avoids coupling the scaffolder to the root CLI's richer + * credentials module. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +interface StoredCredentials { + apiKey?: string; + apiUrl?: string; +} + +function getConfigDir(): string { + const homeDir = os.homedir(); + switch (os.platform()) { + case 'darwin': + return path.join( + homeDir, + 'Library', + 'Application Support', + 'firecrawl-cli' + ); + case 'win32': + return path.join(homeDir, 'AppData', 'Roaming', 'firecrawl-cli'); + default: + return path.join(homeDir, '.config', 'firecrawl-cli'); + } +} + +export function loadFirecrawlCredentials(): StoredCredentials | null { + try { + const credPath = path.join(getConfigDir(), 'credentials.json'); + if (!fs.existsSync(credPath)) return null; + return JSON.parse(fs.readFileSync(credPath, 'utf-8')) as StoredCredentials; + } catch { + return null; + } +} + +export async function resolveFirecrawlApiKey(): Promise<{ + key: string; + source: 'env' | 'credentials' | 'prompt'; +} | null> { + // 1. Environment variable + if (process.env.FIRECRAWL_API_KEY) { + return { key: process.env.FIRECRAWL_API_KEY, source: 'env' }; + } + + // 2. Stored credentials from firecrawl-cli + const creds = loadFirecrawlCredentials(); + if (creds?.apiKey) { + return { key: creds.apiKey, source: 'credentials' }; + } + + return null; +} diff --git a/src/utils/agent-scaffold/manifest.ts b/src/utils/agent-scaffold/manifest.ts new file mode 100644 index 000000000..d77081810 --- /dev/null +++ b/src/utils/agent-scaffold/manifest.ts @@ -0,0 +1,199 @@ +/** + * Vendored from firecrawl/firecrawl-agent:.internal/cli/src/utils/manifest.ts + * + * Divergence from upstream: removed the "look for agent-manifest.json next to + * the bundled CLI" branch — when this code runs inside the root firecrawl-cli + * npm package there is no bundled manifest. `loadManifest()` always clones + * the public agent repo into a tmp dir and reads the manifest from there, + * which is exactly what `loadExternalManifestSync` already does. + * + * Keep in sync with upstream when the manifest schema evolves. + */ + +import * as path from 'path'; +import * as fs from 'fs'; +import { execSync } from 'child_process'; +import { info } from './ui'; + +export interface TemplateEntry { + id: string; + name: string; + description: string; + path: string; + requiredEnvVars: string[]; + optionalEnvVars: string[]; + envFile?: string; + devCommand: string; + deploy: string[]; +} + +export interface ModelEntry { + id: string; + name: string; +} + +export interface ProviderEntry { + id: string; + name: string; + envVar: string; + hint: string; + models: ModelEntry[]; + endpointEnvVar?: string; +} + +export interface Manifest { + version: number; + templates: TemplateEntry[]; + providers: ProviderEntry[]; +} + +let cached: Manifest | null = null; +let cachedSourceRoot: string | null = null; + +const DEFAULT_REMOTE = 'firecrawl/firecrawl-agent'; + +export function loadManifest(): Manifest { + if (cached) return cached; + const { manifest } = loadExternalManifestSync(DEFAULT_REMOTE); + return manifest; +} + +function cloneRepo(source: string, dest: string): void { + const ghToken = process.env.GITHUB_TOKEN || process.env.GH_TOKEN; + + // Try gh CLI first (handles private repos via gh auth) + try { + execSync(`gh repo clone ${source} "${dest}" -- --depth 1`, { + stdio: 'pipe', + }); + return; + } catch {} + + // Fall back to git clone with token if available + const cloneUrl = ghToken + ? `https://${ghToken}@github.com/${source}.git` + : `https://github.com/${source}.git`; + execSync(`git clone --depth 1 ${cloneUrl} "${dest}"`, { stdio: 'pipe' }); +} + +function loadExternalManifestSync(source: string): { + manifest: Manifest; + sourceRoot: string; +} { + const tmpDir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'fc-agent-')); + cloneRepo(source, tmpDir); + const manifestPath = path.join(tmpDir, 'agent-manifest.json'); + if (!fs.existsSync(manifestPath)) { + // Upstream currently ships the manifest at `.internal/cli/agent-manifest.json`. + // Fall back to that location so the scaffolder keeps working until the + // manifest is promoted to repo root. + const nestedPath = path.join( + tmpDir, + '.internal', + 'cli', + 'agent-manifest.json' + ); + if (fs.existsSync(nestedPath)) { + const manifest = JSON.parse( + fs.readFileSync(nestedPath, 'utf-8') + ) as Manifest; + cached = manifest; + cachedSourceRoot = tmpDir; + return { manifest, sourceRoot: tmpDir }; + } + throw new Error(`No agent-manifest.json found in ${source}`); + } + const manifest = JSON.parse( + fs.readFileSync(manifestPath, 'utf-8') + ) as Manifest; + cached = manifest; + cachedSourceRoot = tmpDir; + return { manifest, sourceRoot: tmpDir }; +} + +/** + * Load manifest from an external GitHub repo or local path. + * Returns the manifest and the root directory where templates live. + * + * Supported sources: + * - "user/repo" — clones from GitHub, reads agent-manifest.json + * - "/absolute/path" or "./relative" — reads from local directory + */ +export async function loadExternalManifest(source: string): Promise<{ + manifest: Manifest; + sourceRoot: string; +}> { + // Local path + if (source.startsWith('/') || source.startsWith('.')) { + const absPath = path.resolve(source); + const manifestPath = path.join(absPath, 'agent-manifest.json'); + if (fs.existsSync(manifestPath)) { + const manifest = JSON.parse( + fs.readFileSync(manifestPath, 'utf-8') + ) as Manifest; + cached = manifest; + cachedSourceRoot = absPath; + return { manifest, sourceRoot: absPath }; + } + // Same upstream fallback as the remote clone path: while the manifest + // lives under `.internal/cli/` rather than repo root, still accept + // local agent-repo checkouts. Source root stays at repo root so + // templates resolve correctly. + const nestedPath = path.join( + absPath, + '.internal', + 'cli', + 'agent-manifest.json' + ); + if (fs.existsSync(nestedPath)) { + const manifest = JSON.parse( + fs.readFileSync(nestedPath, 'utf-8') + ) as Manifest; + cached = manifest; + cachedSourceRoot = absPath; + return { manifest, sourceRoot: absPath }; + } + throw new Error(`No agent-manifest.json found in ${absPath}`); + } + + // GitHub repo (user/repo format) + const tmpDir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'fc-agent-')); + info(`Cloning ${source}...`); + try { + cloneRepo(source, tmpDir); + } catch { + throw new Error( + `Failed to clone ${source} - for private repos, install the gh CLI (gh auth login) or set GITHUB_TOKEN` + ); + } + + const manifestPath = path.join(tmpDir, 'agent-manifest.json'); + if (!fs.existsSync(manifestPath)) { + throw new Error(`No agent-manifest.json found in ${source}`); + } + + const manifest = JSON.parse( + fs.readFileSync(manifestPath, 'utf-8') + ) as Manifest; + cached = manifest; + cachedSourceRoot = tmpDir; + return { manifest, sourceRoot: tmpDir }; +} + +export function getSourceRoot(): string { + if (cachedSourceRoot) return cachedSourceRoot; + loadManifest(); + return cachedSourceRoot!; +} + +export function getTemplates(): TemplateEntry[] { + return (cached ?? loadManifest()).templates; +} + +export function getTemplate(id: string): TemplateEntry | undefined { + return getTemplates().find((t) => t.id === id); +} + +export function getProviders(): ProviderEntry[] { + return (cached ?? loadManifest()).providers; +} diff --git a/src/utils/agent-scaffold/scaffold.ts b/src/utils/agent-scaffold/scaffold.ts new file mode 100644 index 000000000..c63756bfd --- /dev/null +++ b/src/utils/agent-scaffold/scaffold.ts @@ -0,0 +1,246 @@ +/** + * Vendored from firecrawl/firecrawl-agent:.internal/cli/src/utils/scaffold.ts + * Copies templates from the cloned agent repo into the user's project dir, + * then merges agent-core deps and writes .env. No divergence from upstream + * — keep in sync if upstream changes scaffolding behavior. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { execSync } from 'child_process'; +import type { TemplateEntry } from './manifest'; +import { getSourceRoot } from './manifest'; +import { success, info, warn } from './ui'; + +const SKIP = new Set([ + 'node_modules', + '.next', + '.git', + '.DS_Store', + 'README.md', +]); + +/** Files to skip when copying agent-core (dev artifacts, build config) */ +const AGENT_CORE_SKIP = new Set([ + 'dist', + 'tsup.config.ts', + 'tsconfig.json', + 'package.json', +]); + +function copyDirRecursive( + src: string, + dest: string, + extraSkip?: (name: string) => boolean +): void { + if (!fs.existsSync(src)) return; + fs.mkdirSync(dest, { recursive: true }); + + for (const entry of fs.readdirSync(src, { withFileTypes: true })) { + if (SKIP.has(entry.name)) continue; + if (extraSkip?.(entry.name)) continue; + const srcPath = path.join(src, entry.name); + // Skip symlinks — agent-core is copied separately + try { + if (fs.lstatSync(srcPath).isSymbolicLink()) continue; + } catch { + /* proceed */ + } + const destPath = path.join(dest, entry.name); + + if (entry.isDirectory()) { + copyDirRecursive(srcPath, destPath, extraSkip); + } else { + fs.copyFileSync(srcPath, destPath); + } + } +} + +/** Clean up any leftover workspace references in scaffolded files. */ +function cleanupScaffold(dir: string): void { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory() && !SKIP.has(entry.name)) { + cleanupScaffold(fullPath); + } else if (entry.name === 'package.json') { + const pkg = JSON.parse(fs.readFileSync(fullPath, 'utf-8')); + if (pkg.dependencies?.['@firecrawl/agent-core']) { + delete pkg.dependencies['@firecrawl/agent-core']; + fs.writeFileSync( + fullPath, + JSON.stringify(pkg, null, 2) + '\n', + 'utf-8' + ); + } + } + } +} + +function generateEnvFile(keys: Record): string { + const lines: string[] = []; + for (const [envVar, value] of Object.entries(keys)) { + if (value) lines.push(`${envVar}=${value}`); + } + return lines.join('\n') + '\n'; +} + +export interface ScaffoldOptions { + projectDir: string; + template: TemplateEntry; + envVars: Record; + selectedProvider?: string; + defaultModelId?: string; + /** Sub-agent provider. Defaults to selectedProvider when omitted. */ + subAgentProvider?: string; + /** Sub-agent model id. Defaults to defaultModelId when omitted. */ + subAgentModelId?: string; + skipInstall?: boolean; +} + +function applyTemplateProviderDefaults( + projectDir: string, + templateId: string, + orchProvider: string | undefined, + orchModel: string | undefined, + subProvider: string | undefined, + subModel: string | undefined +): void { + if (!orchProvider || !orchModel || templateId !== 'next') return; + + const configPath = path.join(projectDir, 'app', '(agent)', '_config.ts'); + if (!fs.existsSync(configPath)) return; + + const effectiveSubProvider = subProvider ?? orchProvider; + const effectiveSubModel = subModel ?? orchModel; + + let content = fs.readFileSync(configPath, 'utf-8'); + content = content + .replace( + /^(\s*)orchestrator:\s*\{ provider: "[^"]+", model: "[^"]+" \} satisfies ModelRef,$/m, + `$1orchestrator: { provider: "${orchProvider}", model: "${orchModel}" } satisfies ModelRef,` + ) + .replace( + /^(\s*)subAgent:\s*\{ provider: "[^"]+", model: "[^"]+" \} satisfies ModelRef,$/m, + `$1subAgent: { provider: "${effectiveSubProvider}", model: "${effectiveSubModel}" } satisfies ModelRef,` + ) + .replace( + /^(\s*)background:\s*\{ provider: "[^"]+", model: "[^"]+" \} satisfies ModelRef,$/m, + `$1background: { provider: "${orchProvider}", model: "${orchModel}" } satisfies ModelRef,` + ); + + fs.writeFileSync(configPath, content, 'utf-8'); +} + +export async function scaffoldProject(opts: ScaffoldOptions): Promise { + const { + projectDir, + template, + envVars, + selectedProvider, + defaultModelId, + subAgentProvider, + subAgentModelId, + skipInstall, + } = opts; + const sourceRoot = getSourceRoot(); + + fs.mkdirSync(projectDir, { recursive: true }); + + // Copy agent-core (skip dev artifacts, test files, build output) + const agentCoreSrc = path.join(sourceRoot, 'agent-core'); + if (fs.existsSync(agentCoreSrc)) { + copyDirRecursive( + agentCoreSrc, + path.join(projectDir, 'agent-core'), + (name) => { + if (AGENT_CORE_SKIP.has(name)) return true; + if (name.endsWith('.test.ts')) return true; + return false; + } + ); + } + + // Copy template files + const templateSrc = path.join(sourceRoot, template.path); + for (const entry of fs.readdirSync(templateSrc, { withFileTypes: true })) { + if (SKIP.has(entry.name)) continue; + const srcPath = path.join(templateSrc, entry.name); + // Skip symlinks (agent-core is already copied above) + try { + if (fs.lstatSync(srcPath).isSymbolicLink()) continue; + } catch { + /* proceed */ + } + const destPath = path.join(projectDir, entry.name); + + if (entry.isDirectory()) { + copyDirRecursive(srcPath, destPath); + } else { + fs.copyFileSync(srcPath, destPath); + } + } + + // Clean up any leftover workspace references + cleanupScaffold(projectDir); + + // Merge agent-core deps into the project package.json + const agentCorePkgPath = path.join(sourceRoot, 'agent-core', 'package.json'); + const projectPkgPath = path.join(projectDir, 'package.json'); + if (fs.existsSync(agentCorePkgPath) && fs.existsSync(projectPkgPath)) { + const corePkg = JSON.parse(fs.readFileSync(agentCorePkgPath, 'utf-8')); + const projectPkg = JSON.parse(fs.readFileSync(projectPkgPath, 'utf-8')); + // Add agent-core runtime deps to the project + for (const [dep, version] of Object.entries(corePkg.dependencies ?? {})) { + if (!projectPkg.dependencies?.[dep]) { + projectPkg.dependencies = projectPkg.dependencies ?? {}; + projectPkg.dependencies[dep] = version; + } + } + // Add agent-core peer deps (optional providers) + for (const [dep, version] of Object.entries( + corePkg.peerDependencies ?? {} + )) { + if (!projectPkg.dependencies?.[dep]) { + projectPkg.dependencies = projectPkg.dependencies ?? {}; + projectPkg.dependencies[dep] = version; + } + } + fs.writeFileSync( + projectPkgPath, + JSON.stringify(projectPkg, null, 2) + '\n', + 'utf-8' + ); + } + + applyTemplateProviderDefaults( + projectDir, + template.id, + selectedProvider, + defaultModelId, + subAgentProvider, + subAgentModelId + ); + success(`${template.name} template scaffolded`); + + // Write .env file + const envFileName = template.envFile ?? '.env'; + const envPath = path.join(projectDir, envFileName); + fs.writeFileSync(envPath, generateEnvFile(envVars), 'utf-8'); + success(`Created ${envFileName}`); + + // Install dependencies + if (!skipInstall) { + info('Installing dependencies...'); + try { + execSync('npm install', { cwd: projectDir, stdio: 'pipe' }); + success('Dependencies installed'); + } catch (err) { + const stderr = + err instanceof Error && 'stderr' in err + ? (err as { stderr: Buffer }).stderr?.toString().trim() + : ''; + warn('npm install failed — run it manually in the project directory'); + if (stderr) console.error(`\n${stderr}\n`); + } + } +} diff --git a/src/utils/agent-scaffold/ui.ts b/src/utils/agent-scaffold/ui.ts new file mode 100644 index 000000000..96447599b --- /dev/null +++ b/src/utils/agent-scaffold/ui.ts @@ -0,0 +1,41 @@ +/** + * Vendored from firecrawl/firecrawl-agent:.internal/cli/src/utils/ui.ts + * Keep in sync if upstream changes — this file intentionally mirrors the + * agent repo's CLI output style so scaffold messages match docs verbatim. + */ + +export const orange = '\x1b[38;5;208m'; +export const reset = '\x1b[0m'; +export const dim = '\x1b[2m'; +export const bold = '\x1b[1m'; +export const green = '\x1b[32m'; +export const red = '\x1b[31m'; +export const cyan = '\x1b[36m'; +export const yellow = '\x1b[33m'; + +export function printBanner(): void { + console.log(''); + console.log(` ${orange}${bold}firecrawl-agent${reset}`); + console.log(` ${dim}AI-powered web research agent${reset}`); + console.log(''); +} + +export function step(n: number, label: string): void { + console.log(` ${dim}${n}.${reset} ${label}`); +} + +export function success(msg: string): void { + console.log(` ${green}✓${reset} ${msg}`); +} + +export function warn(msg: string): void { + console.log(` ${yellow}!${reset} ${msg}`); +} + +export function error(msg: string): void { + console.log(` ${red}✗${reset} ${msg}`); +} + +export function info(msg: string): void { + console.log(` ${dim}${msg}${reset}`); +} From 47af8daf5cd0ae65e626096ee4a4a880fa7890f5 Mon Sep 17 00:00:00 2001 From: Developers Digest <124798203+developersdigest@users.noreply.github.com> Date: Wed, 15 Apr 2026 13:58:19 -0400 Subject: [PATCH 2/2] chore: friendlier error when default clone fails --- src/utils/agent-scaffold/manifest.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/utils/agent-scaffold/manifest.ts b/src/utils/agent-scaffold/manifest.ts index d77081810..7df39f32f 100644 --- a/src/utils/agent-scaffold/manifest.ts +++ b/src/utils/agent-scaffold/manifest.ts @@ -81,7 +81,15 @@ function loadExternalManifestSync(source: string): { sourceRoot: string; } { const tmpDir = fs.mkdtempSync(path.join(require('os').tmpdir(), 'fc-agent-')); - cloneRepo(source, tmpDir); + try { + cloneRepo(source, tmpDir); + } catch { + // Default remote is public once launched; until then, gh auth covers + // insiders and everyone else sees this hint. + throw new Error( + `Could not clone ${source}. If the repo isn't public yet, authenticate with \`gh auth login\` or set GITHUB_TOKEN. Otherwise, check your network / git install.` + ); + } const manifestPath = path.join(tmpDir, 'agent-manifest.json'); if (!fs.existsSync(manifestPath)) { // Upstream currently ships the manifest at `.internal/cli/agent-manifest.json`.