Emit PostHog AI observability spans and generations per turn

Adds $ai_generation (one per model turn, under a per-turn $ai_trace_id
derived from the process session id plus turn index) and $ai_span (one
per tool call, linked via $ai_parent_id, classified into a fixed
tool_call/subagent_call enum rather than the raw tool name). Reuses
the existing capture path, distinct_id, and session_id plumbing;
tool execution appears only as spans, never a separate product event.

Author: TheGreatAxios

src/telemetry/ai-observability.test.ts

@@ -0,0 +1,146 @@
+import { describe, expect, test } from "bun:test";
+import type { ToolCall, ToolResult } from "@intx/types/runtime";
+import type { Telemetry } from "./index.js";
+import type { TurnContext } from "../session/hooks.js";
+import { classifySpanKind, emitAiObservability, turnTraceId } from "./ai-observability.js";
+
+const SUBAGENT_TOOL_NAME = "task";
+
+function fakeTelemetry(): { telemetry: Telemetry; captured: { event: string; properties: Record<string, unknown> }[] } {
+  const captured: { event: string; properties: Record<string, unknown> }[] = [];
+  const telemetry: Telemetry = {
+    enabled: true,
+    capture: (event, properties = {}) => {
+      captured.push({ event, properties });
+    },
+    flush: async () => {},
+  };
+  return { telemetry, captured };
+}
+
+function fakeTurnContext(overrides: Partial<TurnContext> = {}): TurnContext {
+  const toolCalls: ToolCall[] = [
+    {
+      id: "call-1",
+      name: "read_file",
+      arguments: { path: "/Users/attacker/secret-project/plan.md" },
+    },
+    {
+      id: "call-2",
+      name: SUBAGENT_TOOL_NAME,
+      arguments: { description: "explore", prompt: "find the leaked API key XYZ-SECRET-123" },
+    },
+  ];
+  const toolResults: ToolResult[] = [
+    { callId: "call-1", content: "file contents: super secret prompt text" },
+    { callId: "call-2", content: "sub-agent report containing prompt XYZ-SECRET-123", isError: true },
+  ];
+  return {
+    turnIndex: 3,
+    assistantTurn: {
+      role: "assistant",
+      content: [{ type: "text", text: "here is the plan: XYZ-SECRET-123" }],
+      model: "claude-x",
+      timestamp: 0,
+    },
+    toolCalls,
+    toolResults,
+    usage: { input: 10, output: 20, cacheRead: 1, cacheWrite: 2, thinking: 3 },
+    source: { provider: "anthropic", model: "claude-x" },
+    durationMs: 456,
+    ...overrides,
+  } as TurnContext;
+}
+
+describe("classifySpanKind", () => {
+  test("classifies the subagent tool as subagent_call", () => {
+    expect(classifySpanKind(SUBAGENT_TOOL_NAME, SUBAGENT_TOOL_NAME)).toBe("subagent_call");
+  });
+
+  test("classifies every other tool as tool_call, regardless of name", () => {
+    expect(classifySpanKind("read_file", SUBAGENT_TOOL_NAME)).toBe("tool_call");
+    expect(classifySpanKind("mcp__acme__fetch_secret", SUBAGENT_TOOL_NAME)).toBe("tool_call");
+  });
+});
+
+describe("turnTraceId", () => {
+  test("derives from an explicit session id and turn index rather than inventing a random id", () => {
+    expect(turnTraceId(3, "session-abc")).toBe("session-abc:turn:3");
+    expect(turnTraceId(3, "session-abc")).toBe(turnTraceId(3, "session-abc"));
+  });
+});
+
+describe("emitAiObservability", () => {
+  test("emits one $ai_generation and one $ai_span per tool call", () => {
+    const { telemetry, captured } = fakeTelemetry();
+    const ctx = fakeTurnContext();
+
+    emitAiObservability(telemetry, ctx, { subagentToolName: SUBAGENT_TOOL_NAME });
+
+    expect(captured.length).toBe(3);
+    expect(captured[0]?.event).toBe("$ai_generation");
+    expect(captured[1]?.event).toBe("$ai_span");
+    expect(captured[2]?.event).toBe("$ai_span");
+  });
+
+  test("tree linkage: generation and spans share $ai_trace_id, spans carry $ai_parent_id", () => {
+    const { telemetry, captured } = fakeTelemetry();
+    const ctx = fakeTurnContext();
+
+    emitAiObservability(telemetry, ctx, { subagentToolName: SUBAGENT_TOOL_NAME });
+
+    const traceId = turnTraceId(ctx.turnIndex);
+    const generation = captured.find((c) => c.event === "$ai_generation")!;
+    const spans = captured.filter((c) => c.event === "$ai_span");
+
+    expect(generation.properties.$ai_trace_id).toBe(traceId);
+    for (const span of spans) {
+      expect(span.properties.$ai_trace_id).toBe(traceId);
+      expect(span.properties.$ai_parent_id).toBe(traceId);
+    }
+    expect(spans[0]?.properties.$ai_span_id).toBe("call-1");
+    expect(spans[1]?.properties.$ai_span_id).toBe("call-2");
+  });
+
+  test("classifies the tool-call span kind by fixed enum, never the raw tool name", () => {
+    const { telemetry, captured } = fakeTelemetry();
+    const ctx = fakeTurnContext();
+
+    emitAiObservability(telemetry, ctx, { subagentToolName: SUBAGENT_TOOL_NAME });
+
+    const spans = captured.filter((c) => c.event === "$ai_span");
+    expect(spans[0]?.properties.span_kind).toBe("tool_call");
+    expect(spans[1]?.properties.span_kind).toBe("subagent_call");
+    for (const span of spans) {
+      expect(span.properties.span_kind).not.toBe("read_file");
+      expect(span.properties.span_kind).not.toBe(SUBAGENT_TOOL_NAME);
+    }
+  });
+
+  test("propagates tool error status onto the span without the result content", () => {
+    const { telemetry, captured } = fakeTelemetry();
+    const ctx = fakeTurnContext();
+
+    emitAiObservability(telemetry, ctx, { subagentToolName: SUBAGENT_TOOL_NAME });
+
+    const spans = captured.filter((c) => c.event === "$ai_span");
+    expect(spans[0]?.properties.status).toBe("ok");
+    expect(spans[1]?.properties.status).toBe("error");
+  });
+
+  test("never leaks prompt text, tool arguments, tool results, or file paths", () => {
+    const { telemetry, captured } = fakeTelemetry();
+    const ctx = fakeTurnContext();
+
+    emitAiObservability(telemetry, ctx, { subagentToolName: SUBAGENT_TOOL_NAME });
+
+    const serialized = JSON.stringify(captured);
+    expect(serialized).not.toContain("secret-project");
+    expect(serialized).not.toContain("plan.md");
+    expect(serialized).not.toContain("XYZ-SECRET-123");
+    expect(serialized).not.toContain("super secret prompt text");
+    expect(serialized).not.toContain("find the leaked");
+    expect(serialized).not.toContain("here is the plan");
+    expect(serialized).not.toContain("/Users/attacker");
+  });
+});

src/telemetry/ai-observability.ts

@@ -0,0 +1,70 @@
+// Emits PostHog AI observability events ($ai_generation, $ai_span) from a
+// completed turn, in the same privacy mode as product telemetry: only ids,
+// enums, and counts ever leave the process. TurnContext already carries tool
+// call arguments and results for lifecycle hooks — this module reads only
+// the scalar/id fields off it and never the content fields.
+
+import type { TurnContext } from "../session/hooks.js";
+import { getSessionId, type AiSpanKind, type Telemetry } from "./index.js";
+
+// Turn ids are derived from the existing session id and the runtime's own
+// turn index rather than a freshly minted uuid per turn, so the trace id is
+// reproducible from data the runtime already tracks (per CL-5723: reuse an
+// existing turn id before inventing a new one).
+export function turnTraceId(turnIndex: number, sessionId: string = getSessionId()): string {
+  return `${sessionId}:turn:${turnIndex}`;
+}
+
+// Maps a tool call to a fixed span kind. Takes the tool's canonical name
+// (e.g. the subagent task tool's registered name) rather than reaching into
+// subagent internals, so this module has no dependency on tool
+// implementations beyond the one identifier it needs to classify.
+export function classifySpanKind(toolName: string, subagentToolName: string): AiSpanKind {
+  return toolName === subagentToolName ? "subagent_call" : "tool_call";
+}
+
+export type EmitAiObservabilityOptions = {
+  // Name of the tool that spawns a sub-agent, used to classify that call's
+  // span kind as "subagent_call" instead of the generic "tool_call".
+  subagentToolName: string;
+};
+
+// Called once per completed turn (the same onTurnComplete hook point
+// inference_turn already uses). Emits one $ai_generation for the model call,
+// then one $ai_span per tool call in the turn, all sharing the turn's
+// $ai_trace_id; each span's $ai_parent_id is the trace id since TurnContext
+// only tracks top-level tool calls (spans nested further, e.g. inside a
+// sub-agent's own turns, are that sub-agent process's own emission).
+export function emitAiObservability(
+  telemetry: Telemetry,
+  ctx: TurnContext,
+  options: EmitAiObservabilityOptions,
+): void {
+  const traceId = turnTraceId(ctx.turnIndex);
+
+  telemetry.capture("$ai_generation", {
+    $ai_trace_id: traceId,
+    provider_id: ctx.source.provider,
+    model_id: ctx.source.model,
+    input_tokens: ctx.usage.input,
+    output_tokens: ctx.usage.output,
+    cache_read_tokens: ctx.usage.cacheRead,
+    cache_write_tokens: ctx.usage.cacheWrite,
+    thinking_tokens: ctx.usage.thinking,
+    duration_ms: ctx.durationMs,
+    status: "ok",
+  });
+
+  const resultsByCallId = new Map(ctx.toolResults.map((result) => [result.callId, result]));
+
+  for (const call of ctx.toolCalls) {
+    const result = resultsByCallId.get(call.id);
+    telemetry.capture("$ai_span", {
+      $ai_trace_id: traceId,
+      $ai_span_id: call.id,
+      $ai_parent_id: traceId,
+      span_kind: classifySpanKind(call.name, options.subagentToolName),
+      status: result?.isError === true ? "error" : "ok",
+    });
+  }
+}

src/telemetry/index.ts

@@ -26,7 +26,15 @@ const FLUSH_DEADLINE_MS = 500;
 export const TELEMETRY_NOTICE =
   "Anonymous usage telemetry is enabled (no prompts, code, or paths collected). Disable in /settings > Telemetry. Docs: docs/TELEMETRY.md";
 
-export type TelemetryEvent = "cli_start" | "session_end" | "inference_turn";
+export type TelemetryEvent = "cli_start" | "session_end" | "inference_turn" | "$ai_generation" | "$ai_span";
+
+// Fixed enum of AI observability span kinds. The raw tool name is never sent
+// as a property: an MCP tool name carries the server identifier it was
+// configured under (`mcp__<server>__<tool>`), which can be a local path or
+// otherwise identifying string. Callers map a tool call to one of these
+// kinds before capturing "$ai_span".
+export const AI_SPAN_KINDS = ["tool_call", "subagent_call"] as const;
+export type AiSpanKind = (typeof AI_SPAN_KINDS)[number];
 
 // One id per interactive process (TUI session or CLI invocation), generated
 // once at module load and reused by every createTelemetry() instance for the
@@ -57,6 +65,25 @@ const EVENT_PROPERTY_ALLOWLIST: Record<TelemetryEvent, readonly string[]> = {
     "thinking_tokens",
     "duration_ms",
   ],
+  // $ai_trace_id is the PostHog AI observability field that groups every
+  // event from one turn into a trace. provider_id/model_id follow the same
+  // canonical-id contract as inference_turn (never free-text provider names).
+  $ai_generation: [
+    "$ai_trace_id",
+    "provider_id",
+    "model_id",
+    "input_tokens",
+    "output_tokens",
+    "cache_read_tokens",
+    "cache_write_tokens",
+    "thinking_tokens",
+    "duration_ms",
+    "status",
+  ],
+  // $ai_parent_id is either the turn's $ai_trace_id (top-level tool call) or
+  // another span's $ai_span_id (nested work). span_kind is one of
+  // AI_SPAN_KINDS only -- never the raw tool name.
+  $ai_span: ["$ai_trace_id", "$ai_span_id", "$ai_parent_id", "span_kind", "duration_ms", "status"],
 };
 
 const FALSY_ENV_FLAG_VALUES = new Set(["", "0", "false", "off", "no"]);
@@ -142,7 +169,7 @@ export function createTelemetry(options: CreateTelemetryOptions): Telemetry {
 
   function capture(event: TelemetryEvent, properties?: Record<string, unknown>): void {
     if (!enabled) return;
-    if (!(event === "cli_start" || event === "session_end" || event === "inference_turn")) return;
+    if (!(event in EVENT_PROPERTY_ALLOWLIST)) return;
 
     const body = {
       api_key: apiKey,

src/tui/runner.ts

@@ -93,6 +93,7 @@ import {
 } from "./commands/registry.js";
 import { registerBuiltInCommands } from "./commands/built-in.js";
 import type { PluginModule } from "../plugins/loader.js";
+import { emitAiObservability } from "../telemetry/ai-observability.js";
 import { activateHeldTelemetry, telemetryFirstRunPending } from "../telemetry/first-run.js";
 import { TELEMETRY_NOTICE } from "../telemetry/index.js";
 import { getTelemetry, setTelemetry } from "../telemetry/singleton.js";
@@ -115,7 +116,7 @@ import { detectLanguageServerAvailable } from "../agent/lsp-availability.js";
 import { normalizeToolDefinitionsForProvider } from "../agent/tool-schema-normalize.js";
 import { resolveSessionMode, type SessionMode } from "../config/session-mode.js";
 import { promptSessionModeIfUnset } from "./session-mode-prompt.js";
-import { createSubAgentSessionStore, type SubAgentProvider } from "../subagent/index.js";
+import { createSubAgentSessionStore, taskToolDefinition, type SubAgentProvider } from "../subagent/index.js";
 import type { InferenceSource, ToolDefinition, InboundMessage } from "@intx/types/runtime";
 import { createSessionOperationQueue } from "./session-operation-queue.js";
 import { setAgentSourceUnlessClosed } from "./agent-source-sync.js";
@@ -1416,6 +1417,7 @@ export async function runTUI(initialConfig: Config): Promise<number> {
         thinking_tokens: ctx.usage.thinking,
         duration_ms: ctx.durationMs,
       });
+      emitAiObservability(getTelemetry(), ctx, { subagentToolName: taskToolDefinition.name });
     },
     // persistRunSnapshot is defined below but not invoked until the stream
     // starts consuming events, well after this closure captures it.

tests/unit/telemetry.test.ts

@@ -172,6 +172,78 @@ test("capture strips properties not in inference_turn's allowlist", async () =>
   expect(body.properties.prompt).toBeUndefined();
 });
 
+test("capture strips properties not in $ai_generation's allowlist", async () => {
+  const calls: unknown[] = [];
+  const impl = ((_url: string, init: RequestInit) => {
+    calls.push(JSON.parse(init.body as string));
+    return Promise.resolve(new Response("1", { status: 200 }));
+  }) as unknown as typeof fetch;
+  const telemetry = createTelemetry({
+    settings: settingsWith("id"),
+    env: {},
+    fetchFn: impl,
+    apiKey: "test-key",
+  });
+  telemetry.capture("$ai_generation", {
+    $ai_trace_id: "trace-1",
+    provider_id: "anthropic",
+    model_id: "claude-x",
+    input_tokens: 10,
+    output_tokens: 20,
+    cache_read_tokens: 1,
+    cache_write_tokens: 2,
+    thinking_tokens: 3,
+    duration_ms: 400,
+    status: "ok",
+    prompt: "should-not-appear",
+    completion: "should-not-appear",
+  });
+  await new Promise((resolve) => setTimeout(resolve, 0));
+  expect(calls.length).toBe(1);
+  const body = calls[0] as { event: string; properties: Record<string, unknown> };
+  expect(body.event).toBe("$ai_generation");
+  expect(body.properties.$ai_trace_id).toBe("trace-1");
+  expect(body.properties.provider_id).toBe("anthropic");
+  expect(body.properties.duration_ms).toBe(400);
+  expect(body.properties.prompt).toBeUndefined();
+  expect(body.properties.completion).toBeUndefined();
+});
+
+test("capture strips properties not in $ai_span's allowlist, including raw tool name and args", async () => {
+  const calls: unknown[] = [];
+  const impl = ((_url: string, init: RequestInit) => {
+    calls.push(JSON.parse(init.body as string));
+    return Promise.resolve(new Response("1", { status: 200 }));
+  }) as unknown as typeof fetch;
+  const telemetry = createTelemetry({
+    settings: settingsWith("id"),
+    env: {},
+    fetchFn: impl,
+    apiKey: "test-key",
+  });
+  telemetry.capture("$ai_span", {
+    $ai_trace_id: "trace-1",
+    $ai_span_id: "span-1",
+    $ai_parent_id: "trace-1",
+    span_kind: "tool_call",
+    status: "ok",
+    tool_name: "mcp__acme__fetch_secret",
+    tool_arguments: { path: "/etc/passwd" },
+    tool_result: "should-not-appear",
+  });
+  await new Promise((resolve) => setTimeout(resolve, 0));
+  expect(calls.length).toBe(1);
+  const body = calls[0] as { event: string; properties: Record<string, unknown> };
+  expect(body.event).toBe("$ai_span");
+  expect(body.properties.$ai_trace_id).toBe("trace-1");
+  expect(body.properties.$ai_span_id).toBe("span-1");
+  expect(body.properties.$ai_parent_id).toBe("trace-1");
+  expect(body.properties.span_kind).toBe("tool_call");
+  expect(body.properties.tool_name).toBeUndefined();
+  expect(body.properties.tool_arguments).toBeUndefined();
+  expect(body.properties.tool_result).toBeUndefined();
+});
+
 test("capture payload shape includes distinct_id and common props, with no client-side geoip flag", async () => {
   const calls: unknown[] = [];
   const impl = ((_url: string, init: RequestInit) => {