From fe46d82aa405a78b7013f81c674afcf958e864c5 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 8 Aug 2026 14:39:32 -0700 Subject: [PATCH] Expand anonymous product telemetry to cover slash commands, skills, plugins, subagents, permissions, compaction, and crashes Adds allowlisted capture() call sites for slash_command, skill_used, plugin_loaded, plugin_used, subagent_start/end, permission_prompt, compaction, crash, and auth_failure, each with its own property allowlist and paired tests proving free text/paths/secrets never leak into the payload. No tool_use event added (that's PostHog AI spans' job). --- src/agent/use-skill.ts | 4 + src/index.ts | 7 + src/permission/gate.ts | 15 ++ src/plugins/loader.ts | 7 + src/plugins/register.ts | 3 + src/session/runtime-assembly.ts | 22 +- src/subagent/task-tool.ts | 19 ++ src/telemetry/index.ts | 52 ++++- src/tui/runner.ts | 4 + tests/unit/telemetry-product-events.test.ts | 226 ++++++++++++++++++++ 10 files changed, 354 insertions(+), 5 deletions(-) create mode 100644 tests/unit/telemetry-product-events.test.ts diff --git a/src/agent/use-skill.ts b/src/agent/use-skill.ts index 538aff3bd..c50d46145 100644 --- a/src/agent/use-skill.ts +++ b/src/agent/use-skill.ts @@ -4,6 +4,7 @@ import type { ToolDefinition } from "@intx/types/runtime"; import { type } from "arktype"; import { resolveSkillBody } from "../extensions/skills.js"; +import { getTelemetry } from "../telemetry/singleton.js"; // Lazy skill loading: the available skills are listed by name + description in // the system prompt, but their full instructions are pulled into context only @@ -34,6 +35,9 @@ export function createUseSkillTool(cwd: string, skillDirs: string[] = []): Agent if (name.length === 0) return "Error: use_skill requires a non-empty name."; const body = await resolveSkillBody(cwd, name, skillDirs); if (body === undefined) return `No skill named "${name}" is available.`; + // Only send the name once resolved against a real skill — never the + // raw, unvalidated model-supplied string. + getTelemetry().capture("skill_used", { skill_name: name }); return `Skill "${name}" — follow these instructions for this task:\n\n${body}`; }, }); diff --git a/src/index.ts b/src/index.ts index 35106d3d6..318dae3ef 100644 --- a/src/index.ts +++ b/src/index.ts @@ -154,6 +154,13 @@ export async function handleFatal(kind: CrashKind, error: unknown): Promise 0) { + getTelemetry().capture("plugin_loaded", { plugin_id: dataOnly.manifest.id, origin }); + } return mod; } return null; @@ -209,6 +213,9 @@ export async function loadPluginEntry( result.origin = origin; result.pluginPath = resolve(pluginDir); } + if (origin !== undefined && manifest !== null && manifest.id.length > 0) { + getTelemetry().capture("plugin_loaded", { plugin_id: manifest.id, origin }); + } return result; } catch (err) { // Route through the same sink as skill/load warnings so a diagnostics diff --git a/src/plugins/register.ts b/src/plugins/register.ts index de2f9420f..23912e74a 100644 --- a/src/plugins/register.ts +++ b/src/plugins/register.ts @@ -2,6 +2,7 @@ import type { PluginModule } from "./loader.js"; import type { PluginConfig } from "../config/settings.js"; import { registerCommandPlugin } from "../tui/commands/registry.js"; import { registerWorkflowPlugin } from "../workflows/index.js"; +import { getTelemetry } from "../telemetry/singleton.js"; export function isPluginEnabled(config: Record, id: string): boolean { return config[id]?.enabled === true; @@ -47,6 +48,7 @@ export function registerCommandPlugins( if (!isEnabledCommandPlugin(mod, config)) continue; registerCommandPlugin(mod.commandPlugin!); registered.push(mod.manifest!.id); + getTelemetry().capture("plugin_used", { plugin_id: mod.manifest!.id }); } return registered; } @@ -60,6 +62,7 @@ export function registerWorkflowPlugins( if (!isEnabledWorkflowPlugin(mod, config)) continue; registerWorkflowPlugin(mod.workflowPlugin!); registered.push(mod.manifest!.id); + getTelemetry().capture("plugin_used", { plugin_id: mod.manifest!.id }); } return registered; } \ No newline at end of file diff --git a/src/session/runtime-assembly.ts b/src/session/runtime-assembly.ts index 8dfa3e332..b7f5a92c0 100644 --- a/src/session/runtime-assembly.ts +++ b/src/session/runtime-assembly.ts @@ -40,6 +40,7 @@ import type { Approval, GrantScope } from "../permission/types.js"; import type { ReasoningEffort } from "../provider/reasoning-effort.js"; import type { SubAgentProvider } from "../subagent/index.js"; import { COMPACTOR_KEEP_RECENT_TURNS, createPruningCompactor } from "./compactor.js"; +import { getTelemetry } from "../telemetry/singleton.js"; // --------------------------------------------------------------------------- // 1. Sub-agent provider literal @@ -252,9 +253,28 @@ export type SessionPruningCompactorArgs = { export function createSessionPruningCompactor( args: SessionPruningCompactorArgs, ): Compactor { - return createPruningCompactor({ + const compactor = createPruningCompactor({ keepRecentTurns: COMPACTOR_KEEP_RECENT_TURNS, summaryMaxChars: SESSION_COMPACTOR_SUMMARY_MAX_CHARS, ...(args.compactionMode !== "pruning" ? { summarize: args.summarize } : {}), }); + return { + ...compactor, + async apply(turns, ctx) { + const turnsBefore = turns.length; + const startedAt = Date.now(); + const result = await compactor.apply(turns, ctx); + // "no compaction needed" is a distinct trigger from an actual prune — + // both are captured so the event reflects compactor activity, not just + // successful compactions. + const trigger = result.record.decisions.summarizedTurnCount !== undefined ? "pruning" : "no_op"; + getTelemetry().capture("compaction", { + trigger, + duration_ms: Date.now() - startedAt, + turns_before: turnsBefore, + turns_after: result.output.length, + }); + return result; + }, + }; } diff --git a/src/subagent/task-tool.ts b/src/subagent/task-tool.ts index 0a481f28d..d31454ea7 100644 --- a/src/subagent/task-tool.ts +++ b/src/subagent/task-tool.ts @@ -43,6 +43,8 @@ import { cleanupSubAgentWorktree, createSubAgentWorktree, WorktreeError } from " import { generateSessionId } from "../session/index.js"; import { end, start } from "../perf/index.js"; import { currentTurnId } from "../perf/reactor-spans.js"; +import { getSessionId } from "../telemetry/index.js"; +import { getTelemetry } from "../telemetry/singleton.js"; import { join } from "node:path"; import type { NestedDispatchDeps, @@ -484,6 +486,14 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool { ...(turnId !== null && turnId.length > 0 ? { turn_id: turnId } : {}), }, }); + const subagentStartedAt = Date.now(); + // agentLabel is either a resolved, known profile id or the fixed + // "worker" fallback — never raw free text. + getTelemetry().capture("subagent_start", { + agent_name: agentLabel, + parent_session_id: getSessionId(), + }); + let subagentStatus: "completed" | "cancelled" | "failed" = "completed"; try { if (deps.useWorktree === true) { const worktreePath = join(deps.getWorkdirBase(), "worktrees", generateSessionId()); @@ -562,6 +572,7 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool { turnBudgetStopAfterDispatches: TURN_BUDGET_STOP_AFTER_DISPATCHES, }; if (wasCancelled) { + subagentStatus = "cancelled"; if ( session !== undefined && deps.sessions?.get(session.id)?.status === "running" @@ -585,6 +596,7 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool { (session !== undefined && deps.sessions?.get(session.id)?.status === "cancelled") ) { + subagentStatus = "cancelled"; briefLedger.recordOutcome(fingerprint, "cancelled"); if ( session !== undefined && @@ -594,6 +606,7 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool { } return await finishWithWorktree(taskToolResult(call.id, cancelledSubAgentMessage(description))); } + subagentStatus = "failed"; // Run never produced a body — undo the admit so turn-budget retry budget // is not burned by auth/provider crashes. briefLedger.release(fingerprint); @@ -613,6 +626,12 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool { } finally { end(subagentSpanId); + getTelemetry().capture("subagent_end", { + agent_name: agentLabel, + parent_session_id: getSessionId(), + status: subagentStatus, + duration_ms: Date.now() - subagentStartedAt, + }); } }, }); diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index c1901a328..a6f44d522 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -24,12 +24,35 @@ const FLUSH_DEADLINE_MS = 500; export const TELEMETRY_NOTICE = "Anonymous usage telemetry is enabled (no prompts, code, or paths collected). Disable in /settings > Telemetry. Docs: docs/TELEMETRY.md"; -export type TelemetryEvent = "cli_start" | "session_end" | "inference_turn"; +export type TelemetryEvent = + | "cli_start" + | "session_end" + | "inference_turn" + | "slash_command" + | "skill_used" + | "plugin_loaded" + | "plugin_used" + | "subagent_start" + | "subagent_end" + | "permission_prompt" + | "compaction" + | "crash" + | "auth_failure"; + +// A per-process identifier, minted once and reused for every capture() call +// in this process (see CL-5727). It links events from the same run without +// ever carrying identity: it is not persisted and not derived from anything +// user-controlled. +const sessionId = crypto.randomUUID(); + +export function getSessionId(): string { + return sessionId; +} // Per-event property allowlist. Anything not listed here is stripped before // the payload leaves the process. Together with the fixed common properties -// capture() appends (service_version, os_type, os_arch, schema_version), -// this bounds everything telemetry can ever contain. +// capture() appends (service_version, os_type, os_arch, schema_version, +// session_id), this bounds everything telemetry can ever contain. const EVENT_PROPERTY_ALLOWLIST: Record = { cli_start: [], session_end: ["status", "turn_count", "duration_ms", "session_mode", "exit_reason"], @@ -43,8 +66,28 @@ const EVENT_PROPERTY_ALLOWLIST: Record = { "thinking_tokens", "duration_ms", ], + // command_name is the canonical registry name only (e.g. "feedback") — + // never the raw command line or any argument text typed after it. + slash_command: ["command_name"], + // skill_name is the resolved, known-skill identifier only, and only sent + // when resolution succeeded — never a filesystem path. + skill_used: ["skill_name"], + // plugin_id is the manifest-declared id, never the plugin's local path. + plugin_loaded: ["plugin_id", "origin"], + plugin_used: ["plugin_id"], + subagent_start: ["agent_name", "parent_session_id"], + subagent_end: ["agent_name", "parent_session_id", "status", "duration_ms"], + // decision and permission_kind are fixed enums — never the literal + // command, tool args, or file path being approved. + permission_prompt: ["decision", "permission_kind"], + compaction: ["trigger", "duration_ms", "turns_before", "turns_after"], + // error_class is a fixed/allowlisted code — never a raw error message. + crash: ["error_class"], + auth_failure: ["error_class"], }; +const KNOWN_EVENTS = new Set(Object.keys(EVENT_PROPERTY_ALLOWLIST) as TelemetryEvent[]); + const FALSY_ENV_FLAG_VALUES = new Set(["", "0", "false", "off", "no"]); // Trimmed so .env files and shell scripts that produce " 0" or "false\n" @@ -128,7 +171,7 @@ export function createTelemetry(options: CreateTelemetryOptions): Telemetry { function capture(event: TelemetryEvent, properties?: Record): void { if (!enabled) return; - if (!(event === "cli_start" || event === "session_end" || event === "inference_turn")) return; + if (!KNOWN_EVENTS.has(event)) return; const body = { api_key: apiKey, @@ -140,6 +183,7 @@ export function createTelemetry(options: CreateTelemetryOptions): Telemetry { os_type: process.platform, os_arch: process.arch, schema_version: 1, + session_id: sessionId, }, }; diff --git a/src/tui/runner.ts b/src/tui/runner.ts index 1a4d4ec6c..dba434005 100644 --- a/src/tui/runner.ts +++ b/src/tui/runner.ts @@ -1852,6 +1852,9 @@ export async function runTUI(initialConfig: Config): Promise { isCodexAuthError, isXaiAuthError, ); + if (kind === "codex_auth" || kind === "xai_auth") { + getTelemetry().capture("auth_failure", { error_class: kind }); + } if (!shouldSettleUiAfterSendFailure(kind)) return; recordRunError(err); systemRow(err instanceof Error ? err.message : String(err)); @@ -1961,6 +1964,7 @@ export async function runTUI(initialConfig: Config): Promise { systemRow(`Unknown command: ${name}`); return; } + getTelemetry().capture("slash_command", { command_name: command.name }); applyCommandResult(command.handler(args, commandContext)); }; diff --git a/tests/unit/telemetry-product-events.test.ts b/tests/unit/telemetry-product-events.test.ts new file mode 100644 index 000000000..d4a4cd808 --- /dev/null +++ b/tests/unit/telemetry-product-events.test.ts @@ -0,0 +1,226 @@ +import { test, expect } from "bun:test"; +import { createTelemetry, getSessionId } from "../../src/telemetry/index.js"; +import type { Settings } from "../../src/config/settings.js"; + +function settingsWith(installationId = "id"): Settings { + return { providers: {}, telemetry: { installationId } }; +} + +function captureHarness(): { impl: typeof fetch; calls: () => Array<{ event: string; properties: Record }> } { + const calls: Array<{ event: string; properties: Record }> = []; + const impl = ((_url: string, init: RequestInit) => { + calls.push(JSON.parse(init.body as string)); + return Promise.resolve(new Response("1", { status: 200 })); + }) as unknown as typeof fetch; + return { impl, calls: () => calls }; +} + +async function settle(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +// Serializes the full body (not just the allowlisted properties object) so a +// property smuggled in outside the allowlist mechanism (e.g. a typo'd key +// that happens to collide, or a future refactor) would still be caught. +function bodyContainsSubstring(body: unknown, substring: string): boolean { + return JSON.stringify(body).toLowerCase().includes(substring.toLowerCase()); +} + +test("getSessionId returns a stable id across calls in the same process", () => { + expect(getSessionId()).toBe(getSessionId()); + expect(typeof getSessionId()).toBe("string"); + expect(getSessionId().length).toBeGreaterThan(0); +}); + +test("slash_command only ever carries command_name, never raw args or free text", async () => { + const { impl, calls } = captureHarness(); + const telemetry = createTelemetry({ settings: settingsWith(), env: {}, fetchFn: impl, apiKey: "test-key" }); + + telemetry.capture("slash_command", { + command_name: "feedback", + args: "arg with/a/path and secret text", + raw_command_line: "/feedback arg with/a/path", + }); + await settle(); + + expect(calls().length).toBe(1); + const body = calls()[0]!; + expect(body.properties.command_name).toBe("feedback"); + expect(bodyContainsSubstring(body, "path")).toBe(false); + expect(bodyContainsSubstring(body, "secret text")).toBe(false); + expect(bodyContainsSubstring(body, "raw_command_line")).toBe(false); +}); + +test("skill_used only ever carries skill_name, never a filesystem path", async () => { + const { impl, calls } = captureHarness(); + const telemetry = createTelemetry({ settings: settingsWith(), env: {}, fetchFn: impl, apiKey: "test-key" }); + + telemetry.capture("skill_used", { + skill_name: "philosophy", + skill_path: "/Users/someone/.claude/skills/philosophy/SKILL.md", + }); + await settle(); + + const body = calls()[0]!; + expect(body.properties.skill_name).toBe("philosophy"); + expect(bodyContainsSubstring(body, "/users/")).toBe(false); + expect(bodyContainsSubstring(body, "skill_path")).toBe(false); +}); + +test("plugin_loaded carries plugin_id and origin only, never a plugin's local path", async () => { + const { impl, calls } = captureHarness(); + const telemetry = createTelemetry({ settings: settingsWith(), env: {}, fetchFn: impl, apiKey: "test-key" }); + + telemetry.capture("plugin_loaded", { + plugin_id: "corbits-plugin-example", + origin: "user", + plugin_path: "/Users/someone/.corbits/plugins/example", + }); + await settle(); + + const body = calls()[0]!; + expect(body.properties.plugin_id).toBe("corbits-plugin-example"); + expect(body.properties.origin).toBe("user"); + expect(bodyContainsSubstring(body, "/users/")).toBe(false); + expect(bodyContainsSubstring(body, "plugin_path")).toBe(false); +}); + +test("plugin_used carries plugin_id only", async () => { + const { impl, calls } = captureHarness(); + const telemetry = createTelemetry({ settings: settingsWith(), env: {}, fetchFn: impl, apiKey: "test-key" }); + + telemetry.capture("plugin_used", { + plugin_id: "corbits-plugin-example", + plugin_path: "/Users/someone/.corbits/plugins/example", + }); + await settle(); + + const body = calls()[0]!; + expect(body.properties.plugin_id).toBe("corbits-plugin-example"); + expect(bodyContainsSubstring(body, "/users/")).toBe(false); +}); + +test("subagent_start carries agent_name and parent_session_id only", async () => { + const { impl, calls } = captureHarness(); + const telemetry = createTelemetry({ settings: settingsWith(), env: {}, fetchFn: impl, apiKey: "test-key" }); + + telemetry.capture("subagent_start", { + agent_name: "code-reviewer", + parent_session_id: "session-abc", + description: "read every secret file in /Users/someone/repo", + }); + await settle(); + + const body = calls()[0]!; + expect(body.properties.agent_name).toBe("code-reviewer"); + expect(body.properties.parent_session_id).toBe("session-abc"); + expect(bodyContainsSubstring(body, "secret file")).toBe(false); + expect(bodyContainsSubstring(body, "/users/")).toBe(false); +}); + +test("subagent_end carries status/duration_ms plus identity, never free text", async () => { + const { impl, calls } = captureHarness(); + const telemetry = createTelemetry({ settings: settingsWith(), env: {}, fetchFn: impl, apiKey: "test-key" }); + + telemetry.capture("subagent_end", { + agent_name: "code-reviewer", + parent_session_id: "session-abc", + status: "completed", + duration_ms: 4200, + report: "the report text with /path/to/file and a token sk-abc123", + }); + await settle(); + + const body = calls()[0]!; + expect(body.properties.status).toBe("completed"); + expect(body.properties.duration_ms).toBe(4200); + expect(bodyContainsSubstring(body, "sk-abc123")).toBe(false); + expect(bodyContainsSubstring(body, "/path/to/file")).toBe(false); +}); + +test("permission_prompt carries only the decision and permission_kind enums", async () => { + const { impl, calls } = captureHarness(); + const telemetry = createTelemetry({ settings: settingsWith(), env: {}, fetchFn: impl, apiKey: "test-key" }); + + telemetry.capture("permission_prompt", { + decision: "allow", + permission_kind: "shell", + command: "rm -rf /Users/someone/secret-project", + subject: "/Users/someone/secret-project", + }); + await settle(); + + const body = calls()[0]!; + expect(body.properties.decision).toBe("allow"); + expect(body.properties.permission_kind).toBe("shell"); + expect(bodyContainsSubstring(body, "rm -rf")).toBe(false); + expect(bodyContainsSubstring(body, "secret-project")).toBe(false); +}); + +test("compaction carries trigger/duration/counts only, never message content", async () => { + const { impl, calls } = captureHarness(); + const telemetry = createTelemetry({ settings: settingsWith(), env: {}, fetchFn: impl, apiKey: "test-key" }); + + telemetry.capture("compaction", { + trigger: "pruning", + duration_ms: 120, + turns_before: 40, + turns_after: 12, + summary: "user asked about their password reset flow", + }); + await settle(); + + const body = calls()[0]!; + expect(body.properties.trigger).toBe("pruning"); + expect(body.properties.duration_ms).toBe(120); + expect(body.properties.turns_before).toBe(40); + expect(body.properties.turns_after).toBe(12); + expect(bodyContainsSubstring(body, "password reset")).toBe(false); +}); + +test("crash carries only an allowlisted error_class, never a raw message or stack", async () => { + const { impl, calls } = captureHarness(); + const telemetry = createTelemetry({ settings: settingsWith(), env: {}, fetchFn: impl, apiKey: "test-key" }); + + telemetry.capture("crash", { + error_class: "TypeError", + message: "Cannot read properties of undefined at /Users/someone/repo/src/index.ts:42", + stack: "at Object. (/Users/someone/repo/src/index.ts:42:10)", + }); + await settle(); + + const body = calls()[0]!; + expect(body.properties.error_class).toBe("TypeError"); + expect(bodyContainsSubstring(body, "/users/")).toBe(false); + expect(bodyContainsSubstring(body, "cannot read properties")).toBe(false); +}); + +test("auth_failure carries only an allowlisted error_class, never a raw message", async () => { + const { impl, calls } = captureHarness(); + const telemetry = createTelemetry({ settings: settingsWith(), env: {}, fetchFn: impl, apiKey: "test-key" }); + + telemetry.capture("auth_failure", { + error_class: "codex_auth", + message: "token expired for profile personal-account@example.com", + }); + await settle(); + + const body = calls()[0]!; + expect(body.properties.error_class).toBe("codex_auth"); + expect(bodyContainsSubstring(body, "personal-account")).toBe(false); + expect(bodyContainsSubstring(body, "token expired")).toBe(false); +}); + +test("every capture body carries the same session_id across events", async () => { + const { impl, calls } = captureHarness(); + const telemetry = createTelemetry({ settings: settingsWith(), env: {}, fetchFn: impl, apiKey: "test-key" }); + + telemetry.capture("slash_command", { command_name: "settings" }); + telemetry.capture("skill_used", { skill_name: "style" }); + await settle(); + + expect(calls().length).toBe(2); + const [first, second] = calls(); + expect(first!.properties.session_id).toBe(getSessionId()); + expect(second!.properties.session_id).toBe(getSessionId()); +});