From 7c1f2c4fb82f4461aca4ed767ba1f10402771df0 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 8 Aug 2026 14:39:58 -0700 Subject: [PATCH 1/2] Emit PostHog AI observability spans and generations per turn PostHog's LLM analytics views query the $ai_-prefixed properties and nothing else, so the documented names are not ours to choose: an unprefixed property still arrives on the event but is invisible to every trace, cost, and latency view. $ai_latency is a duration in seconds, while the runtime measures milliseconds throughout. Only ids, enums, and counts leave the process. A tool name and a provider error message are both free text that routinely embed a local path or a prompt excerpt, so each is classified into a fixed enum at the boundary and the original discarded. The trace is deliberately flat: PostHog accepts a trace id as a span's parent, and top-level tool calls are all the turn record exposes. --- docs/TELEMETRY.md | 56 ++++- src/session/run-sink.test.ts | 92 +++++++ src/session/run-sink.ts | 38 ++- src/telemetry/ai-observability.test.ts | 336 +++++++++++++++++++++++++ src/telemetry/ai-observability.ts | 177 +++++++++++++ src/telemetry/index.ts | 61 ++++- src/tui/runner.ts | 12 +- tests/unit/telemetry.test.ts | 96 +++++++ 8 files changed, 859 insertions(+), 9 deletions(-) create mode 100644 src/telemetry/ai-observability.test.ts create mode 100644 src/telemetry/ai-observability.ts diff --git a/docs/TELEMETRY.md b/docs/TELEMETRY.md index 42d3b8695..ee115adcf 100644 --- a/docs/TELEMETRY.md +++ b/docs/TELEMETRY.md @@ -13,6 +13,8 @@ Each event carries a small set of properties: | `cli_start` | Once per used session (see First-run disclosure) | (none beyond common properties) | | `session_end` | When a TUI session finishes | `status`, `turn_count`, `duration_ms`, `session_mode`, `exit_reason` | | `inference_turn` | Once per completed turn | `provider_id`, `model_id`, `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`, `thinking_tokens`, `duration_ms` | +| `$ai_generation` | Once per turn — on completion, and once for a turn that ends in an error instead | `$ai_trace_id`, `$ai_provider`, `$ai_model`, `$ai_input_tokens`, `$ai_output_tokens`, `$ai_latency`, `$ai_is_error`, `$ai_error`, `cache_read_tokens`, `cache_write_tokens`, `thinking_tokens` | +| `$ai_span` | Once per top-level tool call in a completed turn | `$ai_trace_id`, `$ai_span_id`, `$ai_parent_id`, `$ai_span_name`, `$ai_is_error` | | `slash_command` | A slash command is dispatched in the TUI | `command_name` | | `skill_used` | `use_skill` loads a skill that resolved | (none beyond common properties) | | `plugin_loaded` | A plugin is discovered and loaded at startup | `origin` | @@ -37,9 +39,10 @@ request IP; no location data is collected by the client. Every event is capped to an explicit property allowlist before it leaves the process — no other field can ever be attached, even by accident. -`provider_id` is the canonical provider kind resolved by the runtime (e.g. -`openai-compatible`), never the free-text name you gave the provider in -onboarding or settings. `model_id` is the model identifier exactly as +`provider_id` (and its AI-event equivalent `$ai_provider`) is the canonical +provider kind resolved by the runtime (e.g. `openai-compatible`), never the +free-text name you gave the provider in onboarding or settings. `model_id` +(equivalently `$ai_model`) is the model identifier exactly as configured — it is the one user-entered string that is sent, so do not put anything identifying in a model name. @@ -75,6 +78,53 @@ The mapping is `src/telemetry/classify.ts`, and the tests that feed each emission site a deliberately identifying name and assert it reaches no part of the payload are in `tests/unit/telemetry-product-events.test.ts`. +## AI observability events + +`$ai_generation` and `$ai_span` are the two PostHog AI observability events, +emitted from `src/telemetry/ai-observability.ts`. PostHog's LLM analytics +views query the `$ai_`-prefixed properties and nothing else, which is why +these names are not ours to choose. `$ai_latency` is a duration in **seconds** +as a float, per PostHog's schema — the runtime measures milliseconds and +converts. + +The trace is **flat**. Every turn gets one `$ai_trace_id` derived from the +runtime's session id and the turn index; the turn's `$ai_generation` and each +of its `$ai_span`s carry it, and every span's `$ai_parent_id` is that same +trace id rather than another span. PostHog documents `$ai_parent_id` as +accepting either a trace id or a span id, so this is a legal trace, and it is +all the runtime can honestly describe: the turn record only exposes top-level +tool calls. No `$ai_trace` event is emitted — PostHog synthesises the trace +from its children. + +`$ai_span_id` is the provider-generated opaque tool call id. It identifies +the call within the trace and carries nothing else. + +`$ai_span_name` is one of a fixed enum (`tool_call`, `subagent_call`). The raw +tool name is never sent: an MCP tool name embeds the server identifier it was +configured under, which can be a local path. + +`$ai_error` is likewise one of a fixed enum (`rate_limit`, `auth`, `timeout`, +`cancelled`, `inference_failed`). The provider's error message is classified +into one of these and then discarded — a raw message routinely embeds the +request URL, a prompt excerpt, or a file path. + +The cache and thinking token counts keep unprefixed names because PostHog does +not publish property names for them in its manual-capture schema; a guessed +`$ai_` name would land as an unread custom property either way. + +Stopping a turn mid-inference is reported, not silent: the runtime aborts the +in-flight call and classifies the resulting error as `cancelled`, so a stopped +turn produces the same errored `$ai_generation` as a failed one and is told +apart by `$ai_error`. A turn that never reaches inference at all — suspended +at an approval prompt and never resumed — emits nothing, because the runtime +raises no event for it. + +Exactly one `$ai_generation` is ever emitted per turn. A single give-up +usually surfaces twice at the event stream (the failed inference, then the +reactor terminating), and a turn that already reported completion is finished; +`src/session/run-sink.ts` latches on both so neither can double-count a turn +or append a phantom failure to a successful one. + ## What's never collected - Prompts, model output, or any conversation content diff --git a/src/session/run-sink.test.ts b/src/session/run-sink.test.ts index fbdaa16db..1e50f9b10 100644 --- a/src/session/run-sink.test.ts +++ b/src/session/run-sink.test.ts @@ -77,6 +77,98 @@ describe("createRunSink", () => { expect(runSink.getTokenUsage()).toEqual({ input: 1, output: 1, cacheRead: 0, cacheWrite: 0, thinking: 0 }); }); + test("reports the in-flight turn to onTurnFailed when a turn errors instead of completing", () => { + const failures: { turnIndex: number; error: string }[] = []; + const runSink = createRunSink({ + emitter: new EventEmitter(), + hookManager: stubHookManager([]), + onTurnFailed: (info) => failures.push(info), + }); + + runSink.sink(event("inference.start", {})); + runSink.sink(event("inference.error", { error: { message: "429 rate limit" } })); + + expect(failures).toEqual([{ turnIndex: 0, error: "429 rate limit" }]); + }); + + // Regression: one give-up reaches the sink twice — the director surfaces + // the failed inference, then the reactor terminates the run — and reporting + // both files two failed turns under a single turn's identity. + test("reports one failure per turn across both error paths, not one per error event", () => { + const failures: { turnIndex: number; error: string }[] = []; + const runSink = createRunSink({ + emitter: new EventEmitter(), + hookManager: stubHookManager([]), + onTurnFailed: (info) => failures.push(info), + }); + + runSink.sink(event("inference.start", {})); + runSink.sink(event("inference.error", { error: { message: "429 rate limit" } })); + runSink.sink(event("reactor.error", { error: "reactor gave up" })); + + expect(failures).toEqual([{ turnIndex: 0, error: "429 rate limit" }]); + }); + + test("reports no failure for a turn that already completed", () => { + const failures: { turnIndex: number; error: string }[] = []; + const completions: number[] = []; + const runSink = createRunSink({ + emitter: new EventEmitter(), + hookManager: stubHookManager([]), + onTurnComplete: (ctx) => completions.push(ctx.turnIndex), + onTurnFailed: (info) => failures.push(info), + }); + + runSink.sink(event("inference.start", {})); + runSink.sink(event("inference.done", { + turn: { role: "assistant", content: [], model: "test", timestamp: 0 }, + usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, thinking: 0 }, + source: { provider: "test", model: "test" }, + })); + runSink.sink(event("reactor.error", { error: "reactor gave up at shutdown" })); + + expect(completions).toEqual([0]); + expect(failures).toEqual([]); + }); + + // A retry re-enters inference.start under the same turn index, so a second + // report would land on the trace id the first one already claimed. + test("reports one failure for a turn that fails, retries, and fails again", () => { + const failures: { turnIndex: number; error: string }[] = []; + const runSink = createRunSink({ + emitter: new EventEmitter(), + hookManager: stubHookManager([]), + onTurnFailed: (info) => failures.push(info), + }); + + runSink.sink(event("inference.start", {})); + runSink.sink(event("inference.error", { error: { message: "500 upstream" } })); + runSink.sink(event("inference.start", {})); + runSink.sink(event("inference.error", { error: { message: "500 upstream again" } })); + + expect(failures).toEqual([{ turnIndex: 0, error: "500 upstream" }]); + }); + + test("still reports a failure after reset clears the latch", () => { + const failures: { turnIndex: number; error: string }[] = []; + const runSink = createRunSink({ + emitter: new EventEmitter(), + hookManager: stubHookManager([]), + onTurnFailed: (info) => failures.push(info), + }); + + runSink.sink(event("inference.start", {})); + runSink.sink(event("inference.error", { error: { message: "first session" } })); + runSink.reset(); + runSink.sink(event("inference.start", {})); + runSink.sink(event("inference.error", { error: { message: "second session" } })); + + expect(failures).toEqual([ + { turnIndex: 0, error: "first session" }, + { turnIndex: 0, error: "second session" }, + ]); + }); + test("seeds the turn count from a resumed session's prior turnsUsed", () => { const runSink = createRunSink({ emitter: new EventEmitter(), diff --git a/src/session/run-sink.ts b/src/session/run-sink.ts index d86f344b0..ced6c40d6 100644 --- a/src/session/run-sink.ts +++ b/src/session/run-sink.ts @@ -20,6 +20,12 @@ export type RunSinkArgs = { // turn actually ran against, so consumers report per-turn provider/model // even if the live selection changed mid-run. onTurnComplete?: (ctx: import("./hooks.js").TurnContext) => void; + // Fired at most once per turn, when that turn ends in an error instead of + // completing. onTurnComplete only ever sees turns that produced a full + // TurnContext, so a consumer relying on it alone goes silent exactly when a + // run goes wrong. The turn index is the collector's current count: the + // in-flight turn is the one that would have been recorded next. + onTurnFailed?: (info: { turnIndex: number; error: string }) => void; // Continues a resumed session's persisted run.json turn count instead of // restarting the collector at zero. initialTurnCount?: number; @@ -81,7 +87,8 @@ export function resolveExecRunStatus(args: { } export function createRunSink(args: RunSinkArgs): RunSink { - const { emitter, hookManager, onTurnComplete, initialTurnCount, onTurnBoundarySnapshot } = args; + const { emitter, hookManager, onTurnComplete, onTurnFailed, initialTurnCount, onTurnBoundarySnapshot } = + args; function hasConfiguredHooks(): boolean { return hookManager.getStatuses().length > 0; @@ -110,12 +117,36 @@ export function createRunSink(args: RunSinkArgs): RunSink { let runCompleted = false; let runError: string | undefined; let turnCollector = createCollector(initialTurnCount); + // True between `inference.start` and whichever event settles that turn. + // One give-up reaches this sink twice — the director surfaces the failed + // inference, then the reactor terminates the run — and a turn that already + // completed is finished, so a later shutdown error belongs to no turn at + // all. Both cases resolve to the same question: is there a turn in flight + // for this error to be about? + let turnInFlight = false; + // Retries re-enter `inference.start` without advancing the turn count, so + // a second failure on a retried turn would report the index a consumer + // already recorded a failure for. Consumers key per-turn identity off that + // index, which makes a repeat indistinguishable from a duplicate. + let failedTurnIndex: number | null = null; // Always-on local PerfTrace: not gated by lifecycle hooks. let perfObserver = createPerfReactorObserver(); + function reportTurnFailure(error: string): void { + if (!turnInFlight) return; + turnInFlight = false; + const turnIndex = turnCollector.getTurnCount(); + if (turnIndex === failedTurnIndex) return; + failedTurnIndex = turnIndex; + onTurnFailed?.({ turnIndex, error }); + } + const sink = (event: ReactorEmittedEvent): void => { turnCollector.observe(event); perfObserver.observe(event); + if (event.type === "inference.start") { + turnInFlight = true; + } if (event.type === "reactor.done") { runCompleted = true; // Terminal success clears any earlier transient inference error. @@ -125,16 +156,19 @@ export function createRunSink(args: RunSinkArgs): RunSink { // (ChatDirector retries timeout/retryable/aborted). Leaving the sticky error // would mark a recovered successful send as failed. if (onTurnBoundary(event)) { + turnInFlight = false; runError = undefined; onTurnBoundarySnapshot?.(); } if (event.type === "reactor.error") { const data = event.data as { error: string }; runError = data.error; + reportTurnFailure(data.error); } if (event.type === "inference.error") { const data = event.data as { error: { message: string } }; runError = data.error.message; + reportTurnFailure(data.error.message); } emitter.emit("event", event); }; @@ -151,6 +185,8 @@ export function createRunSink(args: RunSinkArgs): RunSink { reset: () => { runCompleted = false; runError = undefined; + turnInFlight = false; + failedTurnIndex = null; turnCollector = createCollector(); perfObserver.reset(); }, diff --git a/src/telemetry/ai-observability.test.ts b/src/telemetry/ai-observability.test.ts new file mode 100644 index 000000000..38d87e322 --- /dev/null +++ b/src/telemetry/ai-observability.test.ts @@ -0,0 +1,336 @@ +import { describe, expect, test } from "bun:test"; +import type { ToolCall, ToolResult } from "@intx/types/runtime"; +import type { Telemetry } from "./index.js"; +import type { TurnContext } from "../session/hooks.js"; +import { + classifyErrorKind, + classifySpanKind, + createTurnObserver, + emitAiObservability, + emitAiTurnFailure, + secondsFromMs, + turnTraceId, +} from "./ai-observability.js"; + +const SUBAGENT_TOOL_NAME = "task"; +const SESSION_ID = "0199-parent-session"; + +function fakeTelemetry(): { telemetry: Telemetry; captured: { event: string; properties: Record }[] } { + const captured: { event: string; properties: Record }[] = []; + const telemetry: Telemetry = { + enabled: true, + capture: (event, properties = {}) => { + captured.push({ event, properties }); + }, + flush: async () => {}, + discard: () => {}, + }; + return { telemetry, captured }; +} + +function fakeTurnContext(overrides: Partial = {}): TurnContext { + const toolCalls: ToolCall[] = [ + { + id: "call-1", + name: "read_file", + arguments: { path: "/Users/attacker/secret-project/plan.md" }, + }, + { + id: "call-2", + name: SUBAGENT_TOOL_NAME, + arguments: { description: "explore", prompt: "find the leaked API key XYZ-SECRET-123" }, + }, + ]; + const toolResults: ToolResult[] = [ + { callId: "call-1", content: "file contents: super secret prompt text" }, + { callId: "call-2", content: "sub-agent report containing prompt XYZ-SECRET-123", isError: true }, + ]; + return { + turnIndex: 3, + assistantTurn: { + role: "assistant", + content: [{ type: "text", text: "here is the plan: XYZ-SECRET-123" }], + model: "model-x", + timestamp: 0, + }, + toolCalls, + toolResults, + usage: { input: 10, output: 20, cacheRead: 1, cacheWrite: 2, thinking: 3 }, + source: { provider: "openai-compatible", model: "model-x" }, + durationMs: 4560, + ...overrides, + } as TurnContext; +} + +const emitOptions = { sessionId: SESSION_ID, subagentToolName: SUBAGENT_TOOL_NAME }; +const FAILED_TURN_SOURCE = { provider: "openai-compatible", model: "model-x" }; + +describe("secondsFromMs", () => { + test("converts milliseconds to fractional seconds, the unit PostHog documents", () => { + expect(secondsFromMs(361)).toBe(0.361); + expect(secondsFromMs(4560)).toBe(4.56); + expect(secondsFromMs(0)).toBe(0); + }); +}); + +describe("classifySpanKind", () => { + test("classifies the subagent tool as subagent_call", () => { + expect(classifySpanKind(SUBAGENT_TOOL_NAME, SUBAGENT_TOOL_NAME)).toBe("subagent_call"); + }); + + test("classifies every other tool as tool_call, regardless of name", () => { + expect(classifySpanKind("read_file", SUBAGENT_TOOL_NAME)).toBe("tool_call"); + expect(classifySpanKind("mcp__acme__fetch_secret", SUBAGENT_TOOL_NAME)).toBe("tool_call"); + }); +}); + +describe("classifyErrorKind", () => { + test("reduces provider messages to fixed reasons", () => { + expect(classifyErrorKind("HTTP 429 rate limit exceeded")).toBe("rate_limit"); + expect(classifyErrorKind("401 Unauthorized")).toBe("auth"); + expect(classifyErrorKind("request timed out after 60s")).toBe("timeout"); + expect(classifyErrorKind("The operation was aborted")).toBe("cancelled"); + expect(classifyErrorKind("upstream returned garbage")).toBe("inference_failed"); + }); + + // The message the runtime itself produces when the user stops a turn. + // docs/TELEMETRY.md states a stopped turn is reported as `cancelled`, and + // the enum member exists only because this path can produce it. + test("classifies the runtime's own abort message as cancelled", () => { + expect(classifyErrorKind("inference aborted")).toBe("cancelled"); + }); + + test("reads a status code only where one was written, not any matching digits", () => { + expect(classifyErrorKind("the model used 1401 tokens and stopped")).toBe("inference_failed"); + expect(classifyErrorKind("retry after 4290 ms")).toBe("inference_failed"); + expect(classifyErrorKind("HTTP 403 forbidden")).toBe("auth"); + }); + + test("reports an abort that followed a timeout as cancelled, not timeout", () => { + expect(classifyErrorKind("request aborted after timeout")).toBe("cancelled"); + }); + + test("still reports a timeout that was never aborted as timeout", () => { + expect(classifyErrorKind("inference call exceeded inactivity timeout (60000 ms)")).toBe("timeout"); + }); +}); + +describe("turnTraceId", () => { + test("derives from the runtime session id and turn index rather than inventing a random id", () => { + expect(turnTraceId("session-abc", 3)).toBe("session-abc:turn:3"); + expect(turnTraceId("session-abc", 3)).toBe(turnTraceId("session-abc", 3)); + }); +}); + +describe("createTurnObserver", () => { + // Regression: the trace id must be built from whatever session id is live + // at emission. A call site that captured it once would keep filing turns + // under the session the process started in, which is the bug asserting two + // different strings produce two different ids can never catch. + test("re-reads the session id per turn, so a new session starts a new trace", () => { + const { telemetry, captured } = fakeTelemetry(); + let sessionId = "session-one"; + const observer = createTurnObserver({ + telemetry: () => telemetry, + getSessionId: () => sessionId, + getSource: () => ({ provider: "openai-compatible", model: "model-x" }), + subagentToolName: SUBAGENT_TOOL_NAME, + }); + + observer.onTurnComplete(fakeTurnContext({ turnIndex: 0, toolCalls: [], toolResults: [] })); + sessionId = "session-two"; + observer.onTurnComplete(fakeTurnContext({ turnIndex: 0, toolCalls: [], toolResults: [] })); + + const traceIds = captured.map((c) => c.properties.$ai_trace_id); + expect(traceIds).toEqual(["session-one:turn:0", "session-two:turn:0"]); + }); + + test("re-reads the session id for a failed turn too", () => { + const { telemetry, captured } = fakeTelemetry(); + let sessionId = "session-one"; + const observer = createTurnObserver({ + telemetry: () => telemetry, + getSessionId: () => sessionId, + getSource: () => ({ provider: "openai-compatible", model: "model-x" }), + subagentToolName: SUBAGENT_TOOL_NAME, + }); + + observer.onTurnFailed({ turnIndex: 0, error: "boom" }); + sessionId = "session-two"; + observer.onTurnFailed({ turnIndex: 0, error: "boom" }); + + expect(captured.map((c) => c.properties.$ai_trace_id)).toEqual([ + "session-one:turn:0", + "session-two:turn:0", + ]); + }); + + test("attributes a failed turn to the source live at the moment it failed", () => { + const { telemetry, captured } = fakeTelemetry(); + let source = { provider: "openai-compatible", model: "model-x" }; + const observer = createTurnObserver({ + telemetry: () => telemetry, + getSessionId: () => SESSION_ID, + getSource: () => source, + subagentToolName: SUBAGENT_TOOL_NAME, + }); + + observer.onTurnFailed({ turnIndex: 0, error: "429 rate limit" }); + source = { provider: "codex", model: "model-y" }; + observer.onTurnFailed({ turnIndex: 1, error: "429 rate limit" }); + + expect(captured[0]?.properties.$ai_provider).toBe("openai-compatible"); + expect(captured[0]?.properties.$ai_model).toBe("model-x"); + expect(captured[1]?.properties.$ai_provider).toBe("codex"); + expect(captured[1]?.properties.$ai_model).toBe("model-y"); + }); +}); + +describe("emitAiObservability", () => { + test("emits one $ai_generation and one $ai_span per tool call", () => { + const { telemetry, captured } = fakeTelemetry(); + + emitAiObservability(telemetry, fakeTurnContext(), emitOptions); + + expect(captured.length).toBe(3); + expect(captured[0]?.event).toBe("$ai_generation"); + expect(captured[1]?.event).toBe("$ai_span"); + expect(captured[2]?.event).toBe("$ai_span"); + }); + + test("reports latency in seconds, not milliseconds", () => { + const { telemetry, captured } = fakeTelemetry(); + + emitAiObservability(telemetry, fakeTurnContext({ durationMs: 361 }), emitOptions); + + const generation = captured.find((c) => c.event === "$ai_generation"); + expect(generation?.properties.$ai_latency).toBe(0.361); + expect(generation?.properties.$ai_latency).not.toBe(361); + }); + + test("names every field PostHog's LLM analytics views actually query", () => { + const { telemetry, captured } = fakeTelemetry(); + + emitAiObservability(telemetry, fakeTurnContext(), emitOptions); + + const generation = captured.find((c) => c.event === "$ai_generation"); + expect(generation?.properties.$ai_provider).toBe("openai-compatible"); + expect(generation?.properties.$ai_model).toBe("model-x"); + expect(generation?.properties.$ai_input_tokens).toBe(10); + expect(generation?.properties.$ai_output_tokens).toBe(20); + expect(generation?.properties).not.toHaveProperty("provider_id"); + expect(generation?.properties).not.toHaveProperty("input_tokens"); + expect(generation?.properties).not.toHaveProperty("duration_ms"); + }); + + test("flat trace: spans parent onto the trace id, not onto each other", () => { + const { telemetry, captured } = fakeTelemetry(); + const ctx = fakeTurnContext(); + + emitAiObservability(telemetry, ctx, emitOptions); + + const traceId = turnTraceId(SESSION_ID, ctx.turnIndex); + const generation = captured.find((c) => c.event === "$ai_generation"); + const spans = captured.filter((c) => c.event === "$ai_span"); + + expect(generation?.properties.$ai_trace_id).toBe(traceId); + for (const span of spans) { + expect(span.properties.$ai_trace_id).toBe(traceId); + expect(span.properties.$ai_parent_id).toBe(traceId); + } + expect(spans[0]?.properties.$ai_span_id).toBe("call-1"); + expect(spans[1]?.properties.$ai_span_id).toBe("call-2"); + }); + + test("names the span by fixed enum, never the raw tool name", () => { + const { telemetry, captured } = fakeTelemetry(); + + emitAiObservability(telemetry, fakeTurnContext(), emitOptions); + + const spans = captured.filter((c) => c.event === "$ai_span"); + expect(spans[0]?.properties.$ai_span_name).toBe("tool_call"); + expect(spans[1]?.properties.$ai_span_name).toBe("subagent_call"); + for (const span of spans) { + expect(span.properties.$ai_span_name).not.toBe("read_file"); + expect(span.properties.$ai_span_name).not.toBe(SUBAGENT_TOOL_NAME); + } + }); + + test("propagates tool error state onto the span without the result content", () => { + const { telemetry, captured } = fakeTelemetry(); + + emitAiObservability(telemetry, fakeTurnContext(), emitOptions); + + const spans = captured.filter((c) => c.event === "$ai_span"); + expect(spans[0]?.properties.$ai_is_error).toBe(false); + expect(spans[1]?.properties.$ai_is_error).toBe(true); + }); + + test("never leaks prompt text, tool arguments, tool results, or file paths", () => { + const { telemetry, captured } = fakeTelemetry(); + + emitAiObservability(telemetry, fakeTurnContext(), emitOptions); + + const serialized = JSON.stringify(captured); + expect(serialized).not.toContain("secret-project"); + expect(serialized).not.toContain("plan.md"); + expect(serialized).not.toContain("XYZ-SECRET-123"); + expect(serialized).not.toContain("super secret prompt text"); + expect(serialized).not.toContain("find the leaked"); + expect(serialized).not.toContain("here is the plan"); + expect(serialized).not.toContain("/Users/attacker"); + }); +}); + +describe("emitAiTurnFailure", () => { + test("emits an errored $ai_generation for a turn that never completed", () => { + const { telemetry, captured } = fakeTelemetry(); + + emitAiTurnFailure(telemetry, { + sessionId: SESSION_ID, + turnIndex: 7, + source: FAILED_TURN_SOURCE, + error: "HTTP 429 rate limit exceeded", + }); + + expect(captured.length).toBe(1); + expect(captured[0]?.event).toBe("$ai_generation"); + expect(captured[0]?.properties.$ai_trace_id).toBe(turnTraceId(SESSION_ID, 7)); + expect(captured[0]?.properties.$ai_is_error).toBe(true); + expect(captured[0]?.properties.$ai_error).toBe("rate_limit"); + }); + + // Which model is failing is the first question failure data is asked, and + // a generation with no provider or model cannot answer it. + test("attributes the failure to a provider and model", () => { + const { telemetry, captured } = fakeTelemetry(); + + emitAiTurnFailure(telemetry, { + sessionId: SESSION_ID, + turnIndex: 7, + source: FAILED_TURN_SOURCE, + error: "HTTP 429 rate limit exceeded", + }); + + expect(captured[0]?.properties.$ai_provider).toBe("openai-compatible"); + expect(captured[0]?.properties.$ai_model).toBe("model-x"); + }); + + test("never leaks the raw provider error message", () => { + const { telemetry, captured } = fakeTelemetry(); + + emitAiTurnFailure(telemetry, { + sessionId: SESSION_ID, + turnIndex: 7, + source: FAILED_TURN_SOURCE, + error: + "429 rate limit on https://api.internal.acme.corp/v1/chat while reading /Users/attacker/secret-project/plan.md: XYZ-SECRET-123", + }); + + const serialized = JSON.stringify(captured); + expect(serialized).not.toContain("acme.corp"); + expect(serialized).not.toContain("/Users/attacker"); + expect(serialized).not.toContain("secret-project"); + expect(serialized).not.toContain("XYZ-SECRET-123"); + }); +}); diff --git a/src/telemetry/ai-observability.ts b/src/telemetry/ai-observability.ts new file mode 100644 index 000000000..3123f6fa6 --- /dev/null +++ b/src/telemetry/ai-observability.ts @@ -0,0 +1,177 @@ +// Emits PostHog AI observability events ($ai_generation, $ai_span) from a +// completed turn, in the same privacy mode as product telemetry: only ids, +// enums, and counts ever leave the process. TurnContext already carries tool +// call arguments and results for lifecycle hooks — this module reads only +// the scalar/id fields off it and never the content fields. + +import type { TurnContext } from "../session/hooks.js"; +import type { AiErrorKind, AiSpanKind, Telemetry } from "./index.js"; + +// PostHog reports latency in seconds as a float; the runtime measures every +// duration in milliseconds. Reporting milliseconds under the seconds-typed +// property inflates every latency by 1000x and still renders plausibly. +const MS_PER_SECOND = 1000; + +export function secondsFromMs(durationMs: number): number { + return durationMs / MS_PER_SECOND; +} + +// Scoped to the runtime's per-session id rather than the process-wide +// telemetry session id: sub-agents run in this same process, so a +// process-wide scope would give a parent's turn 3 and a sub-agent's turn 3 +// the same trace id and silently merge two unrelated traces. +export function turnTraceId(sessionId: string, turnIndex: number): string { + return `${sessionId}:turn:${turnIndex}`; +} + +// Maps a tool call to a fixed span kind. Takes the tool's canonical name +// (e.g. the subagent task tool's registered name) rather than reaching into +// subagent internals, so this module has no dependency on tool +// implementations beyond the one identifier it needs to classify. +export function classifySpanKind(toolName: string, subagentToolName: string): AiSpanKind { + return toolName === subagentToolName ? "subagent_call" : "tool_call"; +} + +// Word-bounded so a status code is only read where one was actually written. +// A bare substring match reads "used 1401 tokens" as an auth rejection and +// "retry after 4290ms" as a rate limit, which is a misclassification that +// looks entirely plausible in a dashboard. +const RATE_LIMIT_STATUS = /\b429\b/; +const AUTH_STATUS = /\b(?:401|403)\b/; + +// Reduces a provider error to a fixed reason. The message itself is never +// sent: it routinely embeds the request URL, the offending prompt, or a +// local file path. +export function classifyErrorKind(message: string): AiErrorKind { + const text = message.toLowerCase(); + if (text.includes("rate limit") || RATE_LIMIT_STATUS.test(text)) return "rate_limit"; + if (text.includes("unauthorized") || AUTH_STATUS.test(text)) return "auth"; + // Ahead of the timeout check: the runtime aborts the in-flight call when a + // total timeout fires, so its message names both, and what the user did — + // or had done to their turn — is the more useful of the two readings. + if (text.includes("abort") || text.includes("cancel")) return "cancelled"; + if (text.includes("timeout") || text.includes("timed out")) return "timeout"; + return "inference_failed"; +} + +export type EmitAiObservabilityOptions = { + // The runtime's per-session id, which scopes the trace id. + sessionId: string; + // Name of the tool that spawns a sub-agent, used to classify that call's + // span kind as "subagent_call" instead of the generic "tool_call". + subagentToolName: string; +}; + +// Called once per completed turn. Emits one $ai_generation for the model +// call, then one $ai_span per tool call in the turn, all sharing the turn's +// $ai_trace_id. The trace is flat by construction: every span's +// $ai_parent_id is the trace id, which PostHog accepts, and TurnContext only +// exposes top-level tool calls so there is no nesting to describe. PostHog +// synthesises the trace itself from these children, so no $ai_trace event is +// emitted. +export function emitAiObservability( + telemetry: Telemetry, + ctx: TurnContext, + options: EmitAiObservabilityOptions, +): void { + const traceId = turnTraceId(options.sessionId, ctx.turnIndex); + + telemetry.capture("$ai_generation", { + $ai_trace_id: traceId, + $ai_provider: ctx.source.provider, + $ai_model: ctx.source.model, + $ai_input_tokens: ctx.usage.input, + $ai_output_tokens: ctx.usage.output, + $ai_latency: secondsFromMs(ctx.durationMs), + $ai_is_error: false, + cache_read_tokens: ctx.usage.cacheRead, + cache_write_tokens: ctx.usage.cacheWrite, + thinking_tokens: ctx.usage.thinking, + }); + + const resultsByCallId = new Map(ctx.toolResults.map((result) => [result.callId, result])); + + for (const call of ctx.toolCalls) { + const result = resultsByCallId.get(call.id); + telemetry.capture("$ai_span", { + $ai_trace_id: traceId, + // The provider's own opaque call id, which is what makes it safe to + // send: it identifies the call within the trace and nothing else. + $ai_span_id: call.id, + $ai_parent_id: traceId, + $ai_span_name: classifySpanKind(call.name, options.subagentToolName), + $ai_is_error: result?.isError === true, + }); + } +} + +// The canonical provider kind and model id a turn ran against. Never the +// sourceId: that is the free-text label the user typed in onboarding or +// settings. +export type TurnSource = { + provider: string; + model: string; +}; + +export type EmitAiTurnFailureOptions = { + sessionId: string; + turnIndex: number; + // The failed turn has no TurnContext to read its source from, so the caller + // supplies it. Without these two the first question failure data is ever + // asked — which model is rate-limiting us — has no answer at all. + source: TurnSource; + // The raw provider message, classified here and never forwarded. + error: string; +}; + +// Called when a turn ends without ever completing, which is where +// observability earns its keep and where a completion-only emitter is +// silent. Emits the $ai_generation the turn never got to emit, marked as an +// error, with no token counts or latency because the turn produced none. +export function emitAiTurnFailure(telemetry: Telemetry, options: EmitAiTurnFailureOptions): void { + telemetry.capture("$ai_generation", { + $ai_trace_id: turnTraceId(options.sessionId, options.turnIndex), + $ai_provider: options.source.provider, + $ai_model: options.source.model, + $ai_is_error: true, + $ai_error: classifyErrorKind(options.error), + }); +} + +export type CreateTurnObserverOptions = { + // Read per emission because the TUI replaces the telemetry handle when the + // user toggles the setting mid-session. + telemetry: () => Telemetry; + // Also read per emission: starting a new session reassigns the runtime + // session id in this same process, and a trace id built from a captured + // one would file the new session's turns under the old session's traces. + getSessionId: () => string; + // The source the next inference will run against, which is the best + // available attribution for a turn that failed before producing one. + getSource: () => TurnSource; + subagentToolName: string; +}; + +// Binds the emitters to the live session and source, giving the run sink two +// plain callbacks and keeping the "read it now, do not capture it" rule in +// one place instead of at each call site. +export function createTurnObserver(options: CreateTurnObserverOptions): { + onTurnComplete: (ctx: TurnContext) => void; + onTurnFailed: (info: { turnIndex: number; error: string }) => void; +} { + return { + onTurnComplete: (ctx) => { + emitAiObservability(options.telemetry(), ctx, { + sessionId: options.getSessionId(), + subagentToolName: options.subagentToolName, + }); + }, + onTurnFailed: (info) => { + emitAiTurnFailure(options.telemetry(), { + sessionId: options.getSessionId(), + source: options.getSource(), + ...info, + }); + }, + }; +} diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index 084475267..820e1d785 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -46,6 +46,8 @@ export type TelemetryEvent = | "cli_start" | "session_end" | "inference_turn" + | "$ai_generation" + | "$ai_span" | "slash_command" | "skill_used" | "plugin_loaded" @@ -56,6 +58,21 @@ export type TelemetryEvent = | "crash" | "auth_failure"; +// Fixed enum of AI observability span names. The raw tool name is never sent +// as a property: an MCP tool name carries the server identifier it was +// configured under (`mcp____`), which can be a local path or +// otherwise identifying string. Callers map a tool call to one of these +// kinds before capturing "$ai_span". +export const AI_SPAN_KINDS = ["tool_call", "subagent_call"] as const; +export type AiSpanKind = (typeof AI_SPAN_KINDS)[number]; + +// Fixed enum of AI observability error reasons. A provider error message is +// free text that routinely carries request URLs, prompt excerpts, and file +// paths, so the message itself never leaves the process — callers classify +// it into one of these before capturing. +export const AI_ERROR_KINDS = ["rate_limit", "auth", "timeout", "cancelled", "inference_failed"] as const; +export type AiErrorKind = (typeof AI_ERROR_KINDS)[number]; + // One id per interactive process (TUI session or CLI invocation), generated // once at module load and reused by every createTelemetry() instance for the // life of the process — including across the toggle handler's re-creation on @@ -85,6 +102,39 @@ const EVENT_PROPERTY_ALLOWLIST: Record = { "thinking_tokens", "duration_ms", ], + // PostHog's LLM analytics views read the $ai_-prefixed properties and + // nothing else, so every field these two events exist to surface has to + // carry the documented name: an unprefixed property still arrives, but + // only as an ordinary custom property no trace, cost, or latency view + // will ever query. + // + // $ai_provider/$ai_model carry canonical runtime ids, never the free-text + // name a user gave a provider in onboarding or settings. $ai_latency is + // in seconds, per PostHog's schema. + // + // The cache and thinking token counts stay on our own names: PostHog + // documents cost inputs for them but does not publish the property names + // in the manual-capture schema, and guessing a name that lands as an + // unread custom property is worse than owning one we can read ourselves. + $ai_generation: [ + "$ai_trace_id", + "$ai_provider", + "$ai_model", + "$ai_input_tokens", + "$ai_output_tokens", + "$ai_latency", + "$ai_is_error", + "$ai_error", + "cache_read_tokens", + "cache_write_tokens", + "thinking_tokens", + ], + // The trace is flat: every span's $ai_parent_id is the turn's + // $ai_trace_id. PostHog documents $ai_parent_id as accepting a trace id or + // another span id, so a flat trace is legal and it is all the runtime can + // honestly describe — TurnContext only sees top-level tool calls. + // $ai_span_name is one of AI_SPAN_KINDS only, never the raw tool name. + $ai_span: ["$ai_trace_id", "$ai_span_id", "$ai_parent_id", "$ai_span_name", "$ai_is_error"], // Every identifier below is a first-party enum produced by // src/telemetry/classify.ts, not the name the user or author wrote. The // allowlist bounds which keys travel; the classifiers bound which values @@ -106,8 +156,6 @@ const EVENT_PROPERTY_ALLOWLIST: Record = { auth_failure: ["auth_provider"], }; -const KNOWN_EVENTS: ReadonlySet = new Set(Object.keys(EVENT_PROPERTY_ALLOWLIST)); - const FALSY_ENV_FLAG_VALUES = new Set(["", "0", "false", "off", "no"]); // Trimmed so .env files and shell scripts that produce " 0" or "false\n" @@ -151,7 +199,9 @@ function allowedProperties( const allowed = EVENT_PROPERTY_ALLOWLIST[event]; const result: Record = {}; for (const key of allowed) { - if (key in properties) result[key] = properties[key]; + // Own-property only: `in` would pick "constructor" or "toString" off + // Object.prototype and ship a function as a property value. + if (Object.hasOwn(properties, key)) result[key] = properties[key]; } return result; } @@ -263,7 +313,10 @@ export function createTelemetry(options: CreateTelemetryOptions): Telemetry { function capture(event: TelemetryEvent, properties?: Record): void { if (!enabled) return; - if (!KNOWN_EVENTS.has(event)) return; + // Own-property only: `in` walks Object.prototype, so capture("toString") + // or capture("constructor") would clear the guard this exists to be and + // hand allowedProperties a function where it expects an allowlist array. + if (!Object.hasOwn(EVENT_PROPERTY_ALLOWLIST, event)) return; queue.push({ event, diff --git a/src/tui/runner.ts b/src/tui/runner.ts index 947bb97e3..3a36c0bdc 100644 --- a/src/tui/runner.ts +++ b/src/tui/runner.ts @@ -93,6 +93,7 @@ import { } from "./commands/registry.js"; import { registerBuiltInCommands } from "./commands/built-in.js"; import type { PluginModule } from "../plugins/loader.js"; +import { createTurnObserver } from "../telemetry/ai-observability.js"; import { activateHeldTelemetry, telemetryFirstRunPending } from "../telemetry/first-run.js"; import { TELEMETRY_NOTICE } from "../telemetry/index.js"; import { classifyCommandName } from "../telemetry/classify.js"; @@ -116,7 +117,7 @@ import { detectLanguageServerAvailable } from "../agent/lsp-availability.js"; import { normalizeToolDefinitionsForProvider } from "../agent/tool-schema-normalize.js"; import { resolveSessionMode, type SessionMode } from "../config/session-mode.js"; import { promptSessionModeIfUnset } from "./session-mode-prompt.js"; -import { createSubAgentSessionStore, type SubAgentProvider } from "../subagent/index.js"; +import { createSubAgentSessionStore, taskToolDefinition, type SubAgentProvider } from "../subagent/index.js"; import type { InferenceSource, ToolDefinition, InboundMessage } from "@intx/types/runtime"; import { createSessionOperationQueue } from "./session-operation-queue.js"; import { setAgentSourceUnlessClosed } from "./agent-source-sync.js"; @@ -1329,6 +1330,13 @@ export async function runTUI(initialConfig: Config): Promise { }); }; + const turnObserver = createTurnObserver({ + telemetry: getTelemetry, + getSessionId: () => sessionId, + getSource: () => liveSource, + subagentToolName: taskToolDefinition.name, + }); + const runSink = createRunSink({ emitter, hookManager, @@ -1347,7 +1355,9 @@ export async function runTUI(initialConfig: Config): Promise { thinking_tokens: ctx.usage.thinking, duration_ms: ctx.durationMs, }); + turnObserver.onTurnComplete(ctx); }, + onTurnFailed: turnObserver.onTurnFailed, // persistRunSnapshot is defined below but not invoked until the stream // starts consuming events, well after this closure captures it. onTurnBoundarySnapshot: () => { diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index 7e0ae3344..6006aa274 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -178,6 +178,102 @@ test("capture strips properties not in inference_turn's allowlist", async () => expect(body.properties.prompt).toBeUndefined(); }); +test("capture strips properties not in $ai_generation's allowlist", async () => { + const { impl, events } = recordingFetch(); + const telemetry = createTelemetry({ + settings: settingsWith("id"), + env: {}, + fetchFn: impl, + apiKey: "test-key", + }); + telemetry.capture("$ai_generation", { + $ai_trace_id: "trace-1", + $ai_provider: "openai-compatible", + $ai_model: "model-x", + $ai_input_tokens: 10, + $ai_output_tokens: 20, + $ai_latency: 0.4, + $ai_is_error: false, + cache_read_tokens: 1, + cache_write_tokens: 2, + thinking_tokens: 3, + prompt: "should-not-appear", + completion: "should-not-appear", + }); + await telemetry.flush(); + expect(events().length).toBe(1); + const body = events()[0]!; + expect(body.event).toBe("$ai_generation"); + expect(body.properties.$ai_trace_id).toBe("trace-1"); + expect(body.properties.$ai_provider).toBe("openai-compatible"); + expect(body.properties.$ai_model).toBe("model-x"); + expect(body.properties.$ai_latency).toBe(0.4); + expect(body.properties.prompt).toBeUndefined(); + expect(body.properties.completion).toBeUndefined(); +}); + +test("capture strips properties not in $ai_span's allowlist, including raw tool name and args", async () => { + const { impl, events } = recordingFetch(); + const telemetry = createTelemetry({ + settings: settingsWith("id"), + env: {}, + fetchFn: impl, + apiKey: "test-key", + }); + telemetry.capture("$ai_span", { + $ai_trace_id: "trace-1", + $ai_span_id: "span-1", + $ai_parent_id: "trace-1", + $ai_span_name: "tool_call", + tool_name: "mcp__acme__fetch_secret", + tool_arguments: { path: "/etc/passwd" }, + tool_result: "should-not-appear", + }); + await telemetry.flush(); + expect(events().length).toBe(1); + const body = events()[0]!; + expect(body.event).toBe("$ai_span"); + expect(body.properties.$ai_trace_id).toBe("trace-1"); + expect(body.properties.$ai_span_id).toBe("span-1"); + expect(body.properties.$ai_parent_id).toBe("trace-1"); + expect(body.properties.$ai_span_name).toBe("tool_call"); + expect(body.properties.tool_name).toBeUndefined(); + expect(body.properties.tool_arguments).toBeUndefined(); + expect(body.properties.tool_result).toBeUndefined(); +}); + +test("capture transmits nothing for an event name inherited from Object.prototype", async () => { + const { impl, events } = recordingFetch(); + const telemetry = createTelemetry({ + settings: settingsWith("id"), + env: {}, + fetchFn: impl, + apiKey: "test-key", + }); + for (const name of ["toString", "constructor", "valueOf", "hasOwnProperty", "__proto__"]) { + telemetry.capture(name as Parameters[0], { leaked: "should-not-appear" }); + } + await telemetry.flush(); + expect(events().length).toBe(0); +}); + +test("capture ignores allowlisted property names inherited from a payload's prototype", async () => { + const { impl, events } = recordingFetch(); + const telemetry = createTelemetry({ + settings: settingsWith("id"), + env: {}, + fetchFn: impl, + apiKey: "test-key", + }); + const properties = Object.create({ $ai_trace_id: "inherited-should-not-appear" }) as Record; + properties.$ai_span_id = "span-1"; + telemetry.capture("$ai_span", properties); + await telemetry.flush(); + const body = events()[0]; + expect(body?.properties.$ai_span_id).toBe("span-1"); + expect(body?.properties.$ai_trace_id).toBeUndefined(); +}); + test("capture payload shape includes distinct_id and common props, with no client-side geoip flag", async () => { const { impl, bodies, events } = recordingFetch(); const telemetry = createTelemetry({ From d9b6a155ad7a61594e56ae6706220d5b096e4850 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sat, 8 Aug 2026 16:10:51 -0700 Subject: [PATCH 2/2] Retire the inference_turn event in favour of $ai_generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both events fired from the same hook with the same payload, so every turn was reported twice, and only $ai_generation reaches PostHog's LLM analytics views — inference_turn's numbers were readable in the raw event stream and nowhere else. This is a deliberate removal of the duplicate, not an oversight: nothing outside this repository consumed inference_turn. --- docs/TELEMETRY.md | 8 +++----- src/telemetry/ai-observability.ts | 3 +++ src/telemetry/index.ts | 11 ---------- src/tui/runner.ts | 17 +--------------- tests/unit/telemetry.test.ts | 34 ------------------------------- 5 files changed, 7 insertions(+), 66 deletions(-) diff --git a/docs/TELEMETRY.md b/docs/TELEMETRY.md index ee115adcf..548901057 100644 --- a/docs/TELEMETRY.md +++ b/docs/TELEMETRY.md @@ -12,7 +12,6 @@ Each event carries a small set of properties: |---|---|---| | `cli_start` | Once per used session (see First-run disclosure) | (none beyond common properties) | | `session_end` | When a TUI session finishes | `status`, `turn_count`, `duration_ms`, `session_mode`, `exit_reason` | -| `inference_turn` | Once per completed turn | `provider_id`, `model_id`, `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`, `thinking_tokens`, `duration_ms` | | `$ai_generation` | Once per turn — on completion, and once for a turn that ends in an error instead | `$ai_trace_id`, `$ai_provider`, `$ai_model`, `$ai_input_tokens`, `$ai_output_tokens`, `$ai_latency`, `$ai_is_error`, `$ai_error`, `cache_read_tokens`, `cache_write_tokens`, `thinking_tokens` | | `$ai_span` | Once per top-level tool call in a completed turn | `$ai_trace_id`, `$ai_span_id`, `$ai_parent_id`, `$ai_span_name`, `$ai_is_error` | | `slash_command` | A slash command is dispatched in the TUI | `command_name` | @@ -39,10 +38,9 @@ request IP; no location data is collected by the client. Every event is capped to an explicit property allowlist before it leaves the process — no other field can ever be attached, even by accident. -`provider_id` (and its AI-event equivalent `$ai_provider`) is the canonical -provider kind resolved by the runtime (e.g. `openai-compatible`), never the -free-text name you gave the provider in onboarding or settings. `model_id` -(equivalently `$ai_model`) is the model identifier exactly as +`$ai_provider` is the canonical provider kind resolved by the runtime (e.g. +`openai-compatible`), never the free-text name you gave the provider in +onboarding or settings. `$ai_model` is the model identifier exactly as configured — it is the one user-entered string that is sent, so do not put anything identifying in a model name. diff --git a/src/telemetry/ai-observability.ts b/src/telemetry/ai-observability.ts index 3123f6fa6..84882f463 100644 --- a/src/telemetry/ai-observability.ts +++ b/src/telemetry/ai-observability.ts @@ -78,6 +78,9 @@ export function emitAiObservability( telemetry.capture("$ai_generation", { $ai_trace_id: traceId, + // The canonical provider kind, never ctx.source.sourceId: sourceId is the + // user-typed label from onboarding/settings, and free text must not leave + // the process under the no-PII contract. $ai_provider: ctx.source.provider, $ai_model: ctx.source.model, $ai_input_tokens: ctx.usage.input, diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index 820e1d785..a947bc9ed 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -45,7 +45,6 @@ export const TELEMETRY_NOTICE = export type TelemetryEvent = | "cli_start" | "session_end" - | "inference_turn" | "$ai_generation" | "$ai_span" | "slash_command" @@ -92,16 +91,6 @@ export function getSessionId(): string { const EVENT_PROPERTY_ALLOWLIST: Record = { cli_start: [], session_end: ["status", "turn_count", "duration_ms", "session_mode", "exit_reason"], - inference_turn: [ - "provider_id", - "model_id", - "input_tokens", - "output_tokens", - "cache_read_tokens", - "cache_write_tokens", - "thinking_tokens", - "duration_ms", - ], // PostHog's LLM analytics views read the $ai_-prefixed properties and // nothing else, so every field these two events exist to surface has to // carry the documented name: an unprefixed property still arrives, but diff --git a/src/tui/runner.ts b/src/tui/runner.ts index 3a36c0bdc..54dbd300d 100644 --- a/src/tui/runner.ts +++ b/src/tui/runner.ts @@ -1341,22 +1341,7 @@ export async function runTUI(initialConfig: Config): Promise { emitter, hookManager, initialTurnCount: resumeSeed.turnsUsed, - onTurnComplete: (ctx) => { - // provider_id is the canonical provider kind, never ctx.source.sourceId: - // sourceId is the user-typed label from onboarding/settings, and free - // text must not leave the process under the no-PII contract. - getTelemetry().capture("inference_turn", { - provider_id: ctx.source.provider, - model_id: ctx.source.model, - input_tokens: ctx.usage.input, - output_tokens: ctx.usage.output, - cache_read_tokens: ctx.usage.cacheRead, - cache_write_tokens: ctx.usage.cacheWrite, - thinking_tokens: ctx.usage.thinking, - duration_ms: ctx.durationMs, - }); - turnObserver.onTurnComplete(ctx); - }, + onTurnComplete: turnObserver.onTurnComplete, onTurnFailed: turnObserver.onTurnFailed, // persistRunSnapshot is defined below but not invoked until the stream // starts consuming events, well after this closure captures it. diff --git a/tests/unit/telemetry.test.ts b/tests/unit/telemetry.test.ts index 6006aa274..42e833c24 100644 --- a/tests/unit/telemetry.test.ts +++ b/tests/unit/telemetry.test.ts @@ -144,40 +144,6 @@ test("capture strips properties not in the event's allowlist", async () => { expect(body.properties.secret_field).toBeUndefined(); }); -test("capture strips properties not in inference_turn's allowlist", async () => { - const { impl, events } = recordingFetch(); - const telemetry = createTelemetry({ - settings: settingsWith("id"), - env: {}, - fetchFn: impl, - apiKey: "test-key", - }); - telemetry.capture("inference_turn", { - provider_id: "anthropic", - model_id: "claude-x", - input_tokens: 10, - output_tokens: 20, - cache_read_tokens: 1, - cache_write_tokens: 2, - thinking_tokens: 3, - duration_ms: 400, - prompt: "should-not-appear", - }); - await telemetry.flush(); - expect(events().length).toBe(1); - const body = events()[0]; - expect(body.event).toBe("inference_turn"); - expect(body.properties.provider_id).toBe("anthropic"); - expect(body.properties.model_id).toBe("claude-x"); - expect(body.properties.input_tokens).toBe(10); - expect(body.properties.output_tokens).toBe(20); - expect(body.properties.cache_read_tokens).toBe(1); - expect(body.properties.cache_write_tokens).toBe(2); - expect(body.properties.thinking_tokens).toBe(3); - expect(body.properties.duration_ms).toBe(400); - expect(body.properties.prompt).toBeUndefined(); -}); - test("capture strips properties not in $ai_generation's allowlist", async () => { const { impl, events } = recordingFetch(); const telemetry = createTelemetry({