diff --git a/e2e-harness/__tests__/e2e-result.test.ts b/e2e-harness/__tests__/e2e-result.test.ts index cb03ec971..f64e9a118 100644 --- a/e2e-harness/__tests__/e2e-result.test.ts +++ b/e2e-harness/__tests__/e2e-result.test.ts @@ -15,12 +15,14 @@ import { OutroKind, RunPhase } from '@lib/wizard-session'; import type { AskQuestion, WizardSession } from '@lib/wizard-session'; import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@lib/programs/warehouse-source/detect'; import { Overlay } from '@ui/tui/router'; +import { TASK_OUTCOMES_KEY } from '@lib/agent/runner/sequence/orchestrator/queue'; import { E2eRunRecorder, abortReasonFrom, buildE2eResult, detectedSourcesFrom, readReportFile, + taskOutcomesFrom, } from '../e2e-result'; import { DEFAULT_E2E_PROFILE, decideE2eAction } from '../e2e-profile'; import type { CiState } from '../wizard-ci-driver'; @@ -422,6 +424,9 @@ describe('buildE2eResult', () => { matchedSignal: 'found DATABASE_URL', }, ], + [TASK_OUTCOMES_KEY]: [ + { type: 'ai-observability', status: 'not needed', optional: true }, + ], }, outroData: null, }, @@ -447,6 +452,7 @@ describe('buildE2eResult', () => { 'runPhase', 'screenPath', 'skillsComplete', + 'taskOutcomes', 'tasks', 'unansweredAsks', ].sort(), @@ -463,6 +469,16 @@ describe('buildE2eResult', () => { ]); }); + it("reports the queue's terminal outcomes by type", () => { + expect(build().taskOutcomes).toEqual([ + { type: 'ai-observability', status: 'not needed', optional: true }, + ]); + }); + + it('has no outcomes for a linear run, which drains no queue', () => { + expect(taskOutcomesFrom({ frameworkContext: {} })).toEqual([]); + }); + it('reports the sources detection found', () => { expect(build().detectedSources).toEqual([ { diff --git a/e2e-harness/e2e-result.ts b/e2e-harness/e2e-result.ts index e34a52dfd..9dd819251 100644 --- a/e2e-harness/e2e-result.ts +++ b/e2e-harness/e2e-result.ts @@ -22,6 +22,10 @@ import fs from 'fs'; import path from 'path'; import { OutroKind, type WizardSession } from '@lib/wizard-session'; +import { + TASK_OUTCOMES_KEY, + type TaskOutcome, +} from '@lib/agent/runner/sequence/orchestrator/queue'; import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@lib/programs/warehouse-source/detect'; import type { DetectedSource } from '@lib/warehouse-sources/types'; import type { E2eDecisionReport } from './e2e-profile.js'; @@ -219,6 +223,20 @@ export function detectedSourcesFrom( return Array.isArray(raw) ? (raw as DetectedSource[]) : []; } +/** + * The drained queue's final outcomes the orchestrator wrote into + * frameworkContext. Unlike the UI `tasks` rows (label + display status, where + * done and failed both render `completed`), these carry the task `type` and + * the queue's real terminal status — the stable vocabulary e2e expectations + * assert on. Empty on linear runs, which have no queue. + */ +export function taskOutcomesFrom( + session: Pick, +): TaskOutcome[] { + const raw = session.frameworkContext[TASK_OUTCOMES_KEY]; + return Array.isArray(raw) ? (raw as TaskOutcome[]) : []; +} + /** The keys the result payload carried before the warehouse work. */ export interface E2eResultBase { runPhase: string; @@ -248,6 +266,11 @@ export function buildE2eResult(args: { refusedAsks: recorder.refusedAsks, notices: recorder.notices, tasks: tasks.map((t) => ({ label: t.label, status: t.status })), + taskOutcomes: taskOutcomesFrom(session).map((t) => ({ + type: t.type, + status: t.status, + optional: t.optional, + })), detectedSources: detectedSourcesFrom(session).map((s) => ({ kind: s.kind, label: s.label, diff --git a/src/lib/agent/__tests__/__snapshots__/commandments.test.ts.snap b/src/lib/agent/__tests__/__snapshots__/commandments.test.ts.snap index 9135f7aab..a1c395e11 100644 --- a/src/lib/agent/__tests__/__snapshots__/commandments.test.ts.snap +++ b/src/lib/agent/__tests__/__snapshots__/commandments.test.ts.snap @@ -93,7 +93,7 @@ Below are important guidance on the harness constraints you are bound to. Follow - If a \`bash\` command is blocked, do NOT retry it or a reworded variant — the fence is deterministic and will block it again. Change approach: inspect with \`read\`/\`grep\`, fix the \`edit\` and continue, or skip a step that is not essential. Retrying blocked commands only wastes turns. - If you get stuck on something outside your control — a package install that keeps failing, a command you are not permitted to run, or a fix outside the scope of this integration — do NOT spiral retrying it. Note it in the setup report for the user to resolve, and move on with the rest of the work. - A \`[YARA]\` block from the security scanner is on YOUR side — it caught a real problem in the edit you just tried (PII in a \`capture()\`, a hardcoded secret or host URL). Read the block reason, understand exactly what it flagged, and change the CODE to comply — e.g. a PII block means move that field off the event and onto the person via \`identify()\`/\`$set\`, keeping the event itself. Retrying the same edit will just block again, and dropping the step loses the instrumentation — so fix it to satisfy the scanner, then continue. -- Call \`load_skill_menu\` once to choose the skill, then \`install_skill\`. Do not call \`load_skill_menu\` again this session. +- Call \`load_skill_menu\` once per category needed by the workflow, then \`install_skill\` with the chosen ID. Reuse a category’s menu instead of loading it again. - Follow the skill's steps in order. Finish the SDK setup — install it, import it at the top of the module, and INITIALIZE it at the framework's entry point for every runtime the integration targets (typically both client and server) — BEFORE adding any event capture. A capture against an uninitialized SDK silently no-ops, so initialization comes first. If you're stuck and cannot install an SDK, add capture calls and add a clear note at the top of the integration report. Never guard a capture behind a runtime "if the SDK happens to be installed" check or a dynamic \`require\`; that ships an uninitialized SDK and no events fire. Do not jump ahead to the fix/revise step just to get a build passing. - Never write a PostHog URL or token as a literal in source (e.g. 'https://us.i.posthog.com') — it is blocked. Read them from environment variables (process.env.POSTHOG_HOST, os.environ['POSTHOG_HOST'], etc.). - To inspect or change a project's \`.env\` files, go straight to the wizard-tools MCP: \`check_env_keys\` to see which keys are present, \`set_env_values\` to write them. A plain \`read\`, \`edit\`, or \`write\` of any \`.env*\` file is blocked — reach for those tools first rather than discovering the block. @@ -192,7 +192,7 @@ Below are important guidance on the harness constraints you are bound to. Follow - If a \`bash\` command is blocked, do NOT retry it or a reworded variant — the fence is deterministic and will block it again. Change approach: inspect with \`read\`/\`grep\`, fix the \`edit\` and continue, or skip a step that is not essential. Retrying blocked commands only wastes turns. - If you get stuck on something outside your control — a package install that keeps failing, a command you are not permitted to run, or a fix outside the scope of this integration — do NOT spiral retrying it. Note it in the setup report for the user to resolve, and move on with the rest of the work. - A \`[YARA]\` block from the security scanner is on YOUR side — it caught a real problem in the edit you just tried (PII in a \`capture()\`, a hardcoded secret or host URL). Read the block reason, understand exactly what it flagged, and change the CODE to comply — e.g. a PII block means move that field off the event and onto the person via \`identify()\`/\`$set\`, keeping the event itself. Retrying the same edit will just block again, and dropping the step loses the instrumentation — so fix it to satisfy the scanner, then continue. -- Call \`load_skill_menu\` once to choose the skill, then \`install_skill\`. Do not call \`load_skill_menu\` again this session. +- Call \`load_skill_menu\` once per category needed by the workflow, then \`install_skill\` with the chosen ID. Reuse a category’s menu instead of loading it again. - Follow the skill's steps in order. Finish the SDK setup — install it, import it at the top of the module, and INITIALIZE it at the framework's entry point for every runtime the integration targets (typically both client and server) — BEFORE adding any event capture. A capture against an uninitialized SDK silently no-ops, so initialization comes first. If you're stuck and cannot install an SDK, add capture calls and add a clear note at the top of the integration report. Never guard a capture behind a runtime "if the SDK happens to be installed" check or a dynamic \`require\`; that ships an uninitialized SDK and no events fire. Do not jump ahead to the fix/revise step just to get a build passing. - Never write a PostHog URL or token as a literal in source (e.g. 'https://us.i.posthog.com') — it is blocked. Read them from environment variables (process.env.POSTHOG_HOST, os.environ['POSTHOG_HOST'], etc.). - To inspect or change a project's \`.env\` files, go straight to the wizard-tools MCP: \`check_env_keys\` to see which keys are present, \`set_env_values\` to write them. A plain \`read\`, \`edit\`, or \`write\` of any \`.env*\` file is blocked — reach for those tools first rather than discovering the block. diff --git a/src/lib/agent/__tests__/agent-prompt-loader.test.ts b/src/lib/agent/__tests__/agent-prompt-loader.test.ts index ab23e6a49..9bb529006 100644 --- a/src/lib/agent/__tests__/agent-prompt-loader.test.ts +++ b/src/lib/agent/__tests__/agent-prompt-loader.test.ts @@ -143,15 +143,17 @@ Connect the sources. it('marks the sink and the runner-seeded task from frontmatter', () => { const p = parseAgentPrompt( - '---\nsink: true\nrunnerSeeded: true\n---\nx', + '---\nsink: true\nrunnerSeeded: true\noptional: true\n---\nx', 't', ); expect(p.sink).toBe(true); expect(p.runnerSeeded).toBe(true); + expect(p.optional).toBe(true); const plain = parseAgentPrompt('---\nmodel: x\n---\nx', 't'); expect(plain.sink).toBe(false); expect(plain.runnerSeeded).toBe(false); + expect(plain.optional).toBe(false); }); it('defaults missing array fields to empty and models to undefined', () => { @@ -212,6 +214,7 @@ describe('buildRegistry', () => { seed: false, sink: false, runnerSeeded: false, + optional: false, skills: [], allowedTools: [], disallowedTools: [], @@ -243,15 +246,17 @@ describe('buildRegistry', () => { prompt({ type: 'plan', flow: 'f', seed: true }), prompt({ type: 'install', flow: 'f' }), prompt({ type: 'warehouse', flow: 'f', runnerSeeded: true }), + prompt({ type: 'logs', flow: 'f', optional: true }), prompt({ type: 'report', flow: 'f', sink: true }), ], 'f', ); // The type still runs — it is only the planner that cannot reach it. - expect(registry.types).toEqual(['install', 'warehouse', 'report']); - expect(registry.enqueueableTypes).toEqual(['install', 'report']); + expect(registry.types).toEqual(['install', 'warehouse', 'logs', 'report']); + expect(registry.enqueueableTypes).toEqual(['install', 'logs', 'report']); expect(registry.runnerSeededTypes).toEqual(['warehouse']); + expect(registry.optionalTypes).toEqual(['logs']); expect(registry.sinkTypes).toEqual(['report']); }); @@ -322,6 +327,7 @@ describe('resolveTask', () => { seed: false, sink: false, runnerSeeded: false, + optional: false, modelPi: 'openai/gpt-5.6-luna', effortPi: 'low', modelSdk: 'claude-haiku-4-5', diff --git a/src/lib/agent/agent-prompt-loader.ts b/src/lib/agent/agent-prompt-loader.ts index c69d8fc87..f9c58f93d 100644 --- a/src/lib/agent/agent-prompt-loader.ts +++ b/src/lib/agent/agent-prompt-loader.ts @@ -89,7 +89,7 @@ export function renderToolInventory(toolNames: readonly string[]): string { const TASK_BASICS = `You are one step in a larger PostHog workflow made of several tasks, run as a fresh agent with no memory of the other tasks beyond the context you are given. Other tasks — before and after you — own the rest of the work, so stay strictly on your own task: do not do a neighbouring step's job, redo what an upstream handoff already did, or reach beyond what you were asked. Do only your task, then report exactly once by calling complete_task with a structured handoff: what your goal was, what you did, and what the next agent should know. When you are given context from previous steps, trust it — those agents already did their work, so do not re-verify or re-read what their handoffs tell you. Build on it and move fast. Read a file before you edit it, so your own changes do not duplicate what is already there. Work only inside this project's own directory: never read, list, or search (find, ls, grep, glob) outside it — not the OS, not other projects, not global package caches. If your task seems to need something outside this directory, it does not — skip that part and say so in your handoff rather than hunting across the filesystem. If your task does not apply to this project — there is genuinely nothing for it to do — report it with status \`not needed\` and say why, rather than marking it done.`; -const SEED_BASICS = `You are the orchestrator. Plan the work and seed the queue with enqueue_task — each call returns an id you can pass as a dependency to a later task. Give each task a short label for the UI — the action in a few words, not file names, class names, or other specifics. The last task you queue, the one that reports on the run, must depend on every other task in the queue — directly, or through a task it already depends on. You are not a task yourself: do not call complete_task and do not edit the project.`; +const SEED_BASICS = `You are the orchestrator. Plan the work and seed the queue with enqueue_task — each call names the task kind in its \`type\` field (the tool has no \`task\` field) and returns an id you can pass as a dependency to a later task. Give each task a short label for the UI — the action in a few words, not file names, class names, or other specifics. The last task you queue, the one that reports on the run, must depend on every other task in the queue — directly, or through a task it already depends on. You are not a task yourself: do not call complete_task and do not edit the project.`; /** * Tasks the wizard queued before the planner ran. It has to see them: they are @@ -214,6 +214,8 @@ export interface AgentPrompt { * one an agent can reach — it can neither invent the task nor forget it. */ runnerSeeded: boolean; + /** Marks a supplementary task: terminal failure unblocks dependents and never fails the run. */ + optional: boolean; /** Per-harness model/effort; both profiles also require gateway admission and prompt compatibility. */ modelPi?: string; effortPi?: ThinkingLevel; @@ -258,6 +260,8 @@ export interface AgentRegistry { readonly sinkTypes: string[]; /** The types only the wizard queues, from what it detected before the run. */ readonly runnerSeededTypes: string[]; + /** The types whose terminal failure must not block dependents or fail the run. */ + readonly optionalTypes: string[]; /** The flow's planner, the one prompt marked `seed: true` in its frontmatter. */ readonly seed?: AgentPrompt; get(type: string): AgentPrompt | undefined; @@ -298,6 +302,7 @@ export function buildRegistry( enqueueableTypes: tasks.filter((p) => !p.runnerSeeded).map((p) => p.type), sinkTypes: tasks.filter((p) => p.sink).map((p) => p.type), runnerSeededTypes: tasks.filter((p) => p.runnerSeeded).map((p) => p.type), + optionalTypes: tasks.filter((p) => p.optional).map((p) => p.type), seed: inFlow.find((p) => p.seed), get: (type) => byType.get(type), }; @@ -391,6 +396,7 @@ export function parseAgentPrompt( seed: fields.seed === 'true', sink: fields.sink === 'true', runnerSeeded: fields.runnerSeeded === 'true', + optional: fields.optional === 'true', modelPi: str(fields.model_pi), effortPi: effort(fields.effort_pi, 'effort_pi'), modelSdk: str(fields.model_sdk), diff --git a/src/lib/agent/runner/harness/pi/orchestrator-tools.ts b/src/lib/agent/runner/harness/pi/orchestrator-tools.ts index 83ebbadc3..dfb81dd93 100644 --- a/src/lib/agent/runner/harness/pi/orchestrator-tools.ts +++ b/src/lib/agent/runner/harness/pi/orchestrator-tools.ts @@ -99,7 +99,7 @@ export function createPiOrchestratorTools( description: 'Add a task to the orchestrator queue. Use it to seed work and to enqueue follow-up work you discover. Keep tasks small and discrete.', promptSnippet: - 'enqueue_task(type, label, dependsOn, reason) — add a task to the queue; returns its id', + 'enqueue_task({type, label, dependsOn, reason}) — add a task to the queue; the task type goes in the "type" field, and it returns the task id', parameters: Type.Object({ type: Type.String({ description: `The task type. One of: ${ctx.validTypes.join(', ')}.`, diff --git a/src/lib/agent/runner/harness/pi/runtime-notes.ts b/src/lib/agent/runner/harness/pi/runtime-notes.ts index 0d9dabfa3..5350d72ab 100644 --- a/src/lib/agent/runner/harness/pi/runtime-notes.ts +++ b/src/lib/agent/runner/harness/pi/runtime-notes.ts @@ -69,7 +69,7 @@ const DONT_SPIRAL = '- If you get stuck on something outside your control — a package install that keeps failing, a command you are not permitted to run, or a fix outside the scope of this integration — do NOT spiral retrying it. Note it in the setup report for the user to resolve, and move on with the rest of the work.'; const SKILL_MENU = - '- Call `load_skill_menu` once to choose the skill, then `install_skill`. Do not call `load_skill_menu` again this session.'; + '- Call `load_skill_menu` once per category needed by the workflow, then `install_skill` with the chosen ID. Reuse a category’s menu instead of loading it again.'; const SKILL_STEPS = "- Follow the skill's steps in order. Finish the SDK setup — install it, import it at the top of the module, and INITIALIZE it at the framework's entry point for every runtime the integration targets (typically both client and server) — BEFORE adding any event capture. A capture against an uninitialized SDK silently no-ops, so initialization comes first. If you're stuck and cannot install an SDK, add capture calls and add a clear note at the top of the integration report. Never guard a capture behind a runtime \"if the SDK happens to be installed\" check or a dynamic `require`; that ships an uninitialized SDK and no events fire. Do not jump ahead to the fix/revise step just to get a build passing."; diff --git a/src/lib/agent/runner/sequence/orchestrator/__tests__/queue-tools.test.ts b/src/lib/agent/runner/sequence/orchestrator/__tests__/queue-tools.test.ts index 3ee223d9e..fb9f38311 100644 --- a/src/lib/agent/runner/sequence/orchestrator/__tests__/queue-tools.test.ts +++ b/src/lib/agent/runner/sequence/orchestrator/__tests__/queue-tools.test.ts @@ -132,6 +132,14 @@ describe('apply functions', () => { expect(r.task.enqueuedBy).toBe('orchestrator'); }); + it('stamps optional from the registry list, not the caller', () => { + ctx.optionalTypes = ['capture']; + const opt = applyEnqueue(ctx, { type: 'capture', reason: 'x' }); + const req = applyEnqueue(ctx, { type: 'install', reason: 'x' }); + expect(opt.ok && opt.task.optional).toBe(true); + expect(req.ok && !req.task.optional).toBe(true); + }); + it('attributes a follow-up enqueue to the running task', () => { const parent = store.enqueue({ type: 'init' }); ctx.currentTaskId = parent.id; diff --git a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts index 8778431ad..83ba0b6cf 100644 --- a/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts +++ b/src/lib/agent/runner/sequence/orchestrator/orchestrator-runner.ts @@ -58,8 +58,10 @@ import { QueueStore, QUEUE_DIR_NAME, SkipReason, + TASK_OUTCOMES_KEY, TaskStatus, type QueuedTask, + type TaskOutcome, } from './queue'; import { drainQueue, type RunTask } from './executor'; import { RunMetrics } from './run-metrics'; @@ -700,6 +702,8 @@ export async function runOrchestrator( // — a single planner edge is enough to pull the task back to the front of // the drain and put its prompt in front of the code work again. runnerSeededTypes: registry.runnerSeededTypes, + // Optionality comes from the task's frontmatter, never from the enqueue call. + optionalTypes: registry.optionalTypes, currentTaskId, }); @@ -1077,6 +1081,12 @@ export async function runOrchestrator( try { await drainQueue(store, runTask); } finally { + // The queue file is wiped below; the e2e harness reads outcomes from here. + session.frameworkContext[TASK_OUTCOMES_KEY] = store.list().map((t) => ({ + type: t.type, + status: t.status, + optional: t.optional === true, + })) satisfies TaskOutcome[]; try { if (referenceSkillId && referenceInstallPath) { promoteReferenceSkill( diff --git a/src/lib/agent/runner/sequence/orchestrator/queue-tools.ts b/src/lib/agent/runner/sequence/orchestrator/queue-tools.ts index 2e2955823..ad7546a30 100644 --- a/src/lib/agent/runner/sequence/orchestrator/queue-tools.ts +++ b/src/lib/agent/runner/sequence/orchestrator/queue-tools.ts @@ -86,6 +86,13 @@ export interface OrchestratorToolsContext { * {@link seededDepViolations}. */ runnerSeededTypes?: readonly string[]; + /** + * Types marked `optional: true` in their frontmatter. Enqueue stamps the flag + * from this list — the task's definition decides, never the enqueuing agent — + * so terminal failure of such a task unblocks dependents and never fails the + * run. + */ + optionalTypes?: readonly string[]; /** * The id of the task this tool server is bound to. Each task agent gets its * own wizard-tools server, so attribution holds when independent tasks run @@ -311,6 +318,7 @@ export function applyEnqueue( inputs: args.inputs ?? {}, dependsOn: args.dependsOn ?? [], model: args.model, + optional: (ctx.optionalTypes ?? []).includes(args.type) || undefined, enqueuedBy: ctx.currentTaskId ?? 'orchestrator', }); return { ok: true, task }; diff --git a/src/lib/agent/runner/sequence/orchestrator/queue.ts b/src/lib/agent/runner/sequence/orchestrator/queue.ts index 5896ef07e..08c66ccde 100644 --- a/src/lib/agent/runner/sequence/orchestrator/queue.ts +++ b/src/lib/agent/runner/sequence/orchestrator/queue.ts @@ -114,7 +114,7 @@ export interface QueuedTask { handoff?: TaskHandoff; /** 'orchestrator' for seeded tasks, or the id of the task that enqueued this one. */ enqueuedBy: string; - /** Wizard-seeded only: terminal failure unblocks dependents and never fails the run. */ + /** Terminal failure unblocks dependents and never fails the run. */ optional?: boolean; createdAt: string; startedAt?: string; @@ -132,6 +132,16 @@ export interface QueueFile { tasks: QueuedTask[]; } +/** Session frameworkContext key holding the drained queue's final outcomes — + * written by the runner before the cache wipe, read by the e2e harness. */ +export const TASK_OUTCOMES_KEY = 'orchestrator-task-outcomes'; + +export interface TaskOutcome { + type: string; + status: TaskStatus; + optional: boolean; +} + /** The structured handoff a task leaves for the next agent. */ export interface TaskHandoff { goals: string; diff --git a/src/lib/programs/__tests__/helpers/integration-prompt.no-jest.ts b/src/lib/programs/__tests__/helpers/integration-prompt.no-jest.ts new file mode 100644 index 000000000..72758ac55 --- /dev/null +++ b/src/lib/programs/__tests__/helpers/integration-prompt.no-jest.ts @@ -0,0 +1,61 @@ +/** + * Shared fixtures for tests that build the default integration's run + * definition and prompt (`warehouse-suggestion.test.ts`, + * `posthog-integration-prompt.test.ts`). + */ + +import { posthogIntegrationConfig } from '@lib/programs/posthog-integration/index'; +import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@lib/programs/warehouse-source/detect'; +import { buildSession, type WizardSession } from '@lib/wizard-session'; +import type { DetectedSource } from '@lib/warehouse-sources/types'; + +export const CREDENTIALS = { + accessToken: 'tok', + projectApiKey: 'phc_test', + projectId: '1', + host: { + apiHost: 'https://us.i.posthog.com', + appHost: 'https://us.posthog.com', + }, +}; + +const FRAMEWORK_CONFIG = { + metadata: { name: 'Next.js', docsUrl: 'https://posthog.com/docs' }, + environment: { getEnvVars: () => ({ POSTHOG_KEY: 'phc_test' }) }, + ui: { getOutroChanges: () => ['Added PostHog provider'] }, + detection: { + usesPackageJson: false, + getVersion: () => '15.0.0', + packageName: 'next', + packageDisplayName: 'Next.js', + }, + analytics: { getTags: () => ({}) }, + prompts: { projectTypeDetection: 'app router' }, +}; + +export function sessionWith(sources: DetectedSource[]): WizardSession { + const s = buildSession({ installDir: '/tmp/app' }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + s.frameworkConfig = FRAMEWORK_CONFIG as any; + if (sources.length > 0) { + s.frameworkContext[DETECTED_WAREHOUSE_SOURCES_KEY] = sources; + } + return s; +} + +export async function resolveRun(session: WizardSession) { + const { run } = posthogIntegrationConfig; + if (typeof run !== 'function') throw new Error('expected a run function'); + return run(session); +} + +export const promptFor = async (sources: DetectedSource[]) => { + const s = sessionWith(sources); + const runDef = await resolveRun(s); + return runDef.customPrompt!({ + projectId: 1, + projectApiKey: 'phc_test', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + host: CREDENTIALS.host as any, + }); +}; diff --git a/src/lib/programs/__tests__/posthog-integration-prompt.test.ts b/src/lib/programs/__tests__/posthog-integration-prompt.test.ts new file mode 100644 index 000000000..38b81818b --- /dev/null +++ b/src/lib/programs/__tests__/posthog-integration-prompt.test.ts @@ -0,0 +1,27 @@ +/** + * The default integration prompt's skill-workflow instructions. + * + * STEP 1 scopes the linear agent to exactly three skill categories: the + * framework skill first, then AI Observability and Logs when its workflow + * calls for them. These tests pin that scoping — the allowlist, the delegation + * to the skill's workflow, and the hard guard against every other category. + */ + +import { promptFor } from './helpers/integration-prompt.no-jest'; + +describe('default integration skill workflow', () => { + it('loads the framework category first and delegates observability to its workflow', async () => { + const prompt = await promptFor([]); + expect(prompt).toContain('category: "integration"'); + expect(prompt).toContain('AI Observability and Logs skills'); + expect(prompt).toContain('before verification and the setup report'); + }); + + it('forbids every category outside the three the run uses', async () => { + const prompt = await promptFor([]); + expect(prompt).toContain( + 'Do NOT load or install skills from any other category', + ); + expect(prompt).toContain('do not substitute `llm-analytics`'); + }); +}); diff --git a/src/lib/programs/__tests__/warehouse-suggestion.test.ts b/src/lib/programs/__tests__/warehouse-suggestion.test.ts index de7a39b9c..144b5f36b 100644 --- a/src/lib/programs/__tests__/warehouse-suggestion.test.ts +++ b/src/lib/programs/__tests__/warehouse-suggestion.test.ts @@ -13,11 +13,15 @@ * say the run already connected the sources. */ -import { posthogIntegrationConfig } from '@lib/programs/posthog-integration/index'; import { POSTHOG_INTEGRATION_PROGRAM } from '@lib/programs/posthog-integration/steps'; -import { DETECTED_WAREHOUSE_SOURCES_KEY } from '@lib/programs/warehouse-source/detect'; -import { buildSession, type WizardSession } from '@lib/wizard-session'; +import type { WizardSession } from '@lib/wizard-session'; import type { DetectedSource } from '@lib/warehouse-sources/types'; +import { + CREDENTIALS, + promptFor, + resolveRun, + sessionWith, +} from './helpers/integration-prompt.no-jest'; const POSTGRES: DetectedSource = { kind: 'postgres', @@ -33,46 +37,6 @@ const STRIPE: DetectedSource = { matchedSignal: 'stripe in package.json', }; -const CREDENTIALS = { - accessToken: 'tok', - projectApiKey: 'phc_test', - projectId: '1', - host: { - apiHost: 'https://us.i.posthog.com', - appHost: 'https://us.posthog.com', - }, -}; - -const FRAMEWORK_CONFIG = { - metadata: { name: 'Next.js', docsUrl: 'https://posthog.com/docs' }, - environment: { getEnvVars: () => ({ POSTHOG_KEY: 'phc_test' }) }, - ui: { getOutroChanges: () => ['Added PostHog provider'] }, - detection: { - usesPackageJson: false, - getVersion: () => '15.0.0', - packageName: 'next', - packageDisplayName: 'Next.js', - }, - analytics: { getTags: () => ({}) }, - prompts: { projectTypeDetection: 'app router' }, -}; - -function sessionWith(sources: DetectedSource[]): WizardSession { - const s = buildSession({ installDir: '/tmp/app' }); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - s.frameworkConfig = FRAMEWORK_CONFIG as any; - if (sources.length > 0) { - s.frameworkContext[DETECTED_WAREHOUSE_SOURCES_KEY] = sources; - } - return s; -} - -async function resolveRun(session: WizardSession) { - const { run } = posthogIntegrationConfig; - if (typeof run !== 'function') throw new Error('expected a run function'); - return run(session); -} - describe('outro suggestion', () => { it('gives every detected source its own pre-filled link', async () => { const s = sessionWith([POSTGRES, STRIPE]); @@ -133,17 +97,6 @@ describe('outro suggestion', () => { }); }); -const promptFor = async (sources: DetectedSource[]) => { - const s = sessionWith(sources); - const runDef = await resolveRun(s); - return runDef.customPrompt!({ - projectId: 1, - projectApiKey: 'phc_test', - // eslint-disable-next-line @typescript-eslint/no-explicit-any - host: CREDENTIALS.host as any, - }); -}; - describe('report instruction', () => { it('asks the agent to note the sources in the report checklist', async () => { const prompt = await promptFor([POSTGRES]); diff --git a/src/lib/programs/posthog-integration/index.ts b/src/lib/programs/posthog-integration/index.ts index c3ac0b913..7bb6c0a93 100644 --- a/src/lib/programs/posthog-integration/index.ts +++ b/src/lib/programs/posthog-integration/index.ts @@ -333,12 +333,12 @@ Project context: Instructions (follow these steps IN ORDER - do not skip or reorder): -STEP 1: Call load_skill_menu (from the wizard-tools MCP server) to see available skills. +STEP 1: Call load_skill_menu (from the wizard-tools MCP server) with category: "integration" to see available framework skills. If the tool fails, emit: ${ AgentSignals.ERROR_MCP_MISSING } Could not load skill menu and halt. - Choose a skill from the \`integration\` category that matches this project's framework. Do NOT pick skills from other categories (llm-analytics, error-tracking, feature-flags, omnibus, etc.) — those are handled separately. + Choose a skill from the \`integration\` category that matches this project's framework. Start with this framework skill; load the AI Observability and Logs skills when its workflow calls for them, before verification and the setup report. Both are included by default where applicable; the skills define applicability and how to report skipped work. These three categories — \`integration\`, \`ai-observability\`, \`logs\` — are the only ones this run uses. Do NOT load or install skills from any other category (llm-analytics, error-tracking, feature-flags, audit, etc.) — those are handled separately. In particular, \`ai-observability\` is the category for AI Observability; do not substitute \`llm-analytics\`. If no suitable integration skill is found, emit: ${ AgentSignals.ERROR_RESOURCE_MISSING } Could not find a suitable skill for this project.