Tag dispatch outcome records with model identity (CL-6968)

Outcome records in interventions.jsonl were written parent-side with no
provider/model/family tag, so per-model intervention counts had no
dispatch denominator to divide by. Tag each outcome with the provider/
model that actually served inference (preferring the last inference.done
source over the pre-dispatch resolved provider, since a mid-run failover
can diverge from it), and teach intervention-forensics.ts to report
interventions-per-dispatch per model as an explicit rate alongside the
existing raw counts.

Author: TheGreatAxios

scripts/intervention-forensics.ts

@@ -11,12 +11,20 @@
 //   edited  — stops that fired on a run which had already edited files.
 //   early   — stops that fired before half the turn budget was spent.
 //
-// Also aggregates outcome records (CL-6938): what each completed dispatch
-// actually produced (a salvage kind, or clean-complete), by kind. This is the
-// log's only outcome signal, letting a stop record be read alongside what the
+// Also aggregates outcome records: what each completed dispatch actually
+// produced (a salvage kind, or clean-complete), by kind. This is the log's
+// only outcome signal, letting a stop record be read alongside what the
 // dispatch it touched actually produced — it is still not gate-pass or
 // retry-success tracking.
 //
+// Outcome records are now tagged with the dispatched child's provider/model/
+// family (CL-6968), so they double as the per-model dispatch denominator:
+// interventions per model, divided by dispatches per model, is the one table
+// below that is an actual rate. Everything else in this script stays a raw
+// count — do not add another table that looks like a rate without a tracked
+// denominator behind it (that mistake shipped once already and had to be
+// stripped, see CL-6968).
+//
 // Run: bun run scripts/intervention-forensics.ts
 //
 // Prints only aggregate counts and the `detail` field's first token, never turn
@@ -88,6 +96,12 @@ findAll(root, INTERVENTION_FILE, files);
 
 const buckets = new Map<string, Bucket>();
 const outcomes = new Map<string, number>();
+// Dispatch counts per model (the outcome record's denominator, CL-6968) and
+// intervention counts per model (stop+nudge only — block/outcome are not
+// leaf signals), so a rate can be computed instead of a bare count.
+const dispatchesByModel = new Map<string, number>();
+const interventionsByModel = new Map<string, number>();
+let untaggedOutcomes = 0;
 let records = 0;
 let malformed = 0;
 
@@ -115,6 +129,12 @@ for (const file of files) {
     if (record.class === "outcome" && record.outcome !== undefined) {
       const kind = record.outcome.kind;
       outcomes.set(kind, (outcomes.get(kind) ?? 0) + 1);
+      if (record.model !== undefined) {
+        dispatchesByModel.set(record.model, (dispatchesByModel.get(record.model) ?? 0) + 1);
+      } else {
+        // Written before CL-6968 tagged outcome records with model identity.
+        untaggedOutcomes++;
+      }
       continue;
     }
     const key = `${record.class ?? "?"}/${record.id}`;
@@ -128,6 +148,9 @@ for (const file of files) {
     bucket.byFamily.set(family, (bucket.byFamily.get(family) ?? 0) + 1);
     const model = record.model ?? "unknown";
     bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + 1);
+    if (record.class === "stop" || record.class === "nudge") {
+      interventionsByModel.set(model, (interventionsByModel.get(model) ?? 0) + 1);
+    }
     if (record.measurement !== undefined) {
       bucket.values.push(record.measurement.value);
       if (record.measurement.threshold !== undefined) {
@@ -178,10 +201,10 @@ for (const [key, bucket] of rows) {
 
 // CL-6775: streamed degenerate-repetition aborts (mid-stream, not a turn-level
 // stop) get their own model breakdown — "repetition-<detector>" ids, one row
-// per model, so "which model loops most" reads off directly. This is a count,
-// not a rate normalized by dispatch volume: the log's outcome records (total
-// completed dispatches) are written from the parent side without a model tag,
-// so a per-model denominator is not yet tracked — see the PR description.
+// per model, so "which model loops most" reads off directly. Still a raw
+// count, not a rate — see the "interventions per dispatch by model" table
+// below for the rate version, computed from the same per-model dispatch
+// denominator (outcome records, CL-6968).
 const repetitionRows = rows.filter(([key]) => key.includes("/repetition-"));
 if (repetitionRows.length > 0) {
   const totalsByModel = new Map<string, number>();
@@ -209,6 +232,32 @@ console.log(
   "\nedited = stops on runs that had already edited files; early = stops before half the turn budget (context, not a false-positive rate).",
 );
 
+// The one real rate in this script: interventions per dispatch, per model.
+// dispatches = outcome records tagged with that model (CL-6968); interventions
+// = stop+nudge records for that model. Everything above this is a count.
+if (dispatchesByModel.size > 0 || interventionsByModel.size > 0) {
+  console.log("\ninterventions per dispatch by model (stop+nudge count / dispatch count = rate)");
+  const models = new Set([...dispatchesByModel.keys(), ...interventionsByModel.keys()]);
+  const modelRows = [...models]
+    .map((model) => {
+      const dispatches = dispatchesByModel.get(model) ?? 0;
+      const interventions = interventionsByModel.get(model) ?? 0;
+      const rate = dispatches > 0 ? (interventions / dispatches).toFixed(3) : "-";
+      return { model, dispatches, interventions, rate };
+    })
+    .sort((a, b) => b.interventions - a.interventions);
+  for (const { model, dispatches, interventions, rate } of modelRows) {
+    console.log(
+      `${model.padEnd(33)} interventions=${String(interventions).padStart(4)} dispatches=${String(dispatches).padStart(5)} rate=${rate}`,
+    );
+  }
+  if (untaggedOutcomes > 0) {
+    console.log(
+      `(${untaggedOutcomes} outcome record(s) predate model tagging (CL-6968) and are excluded from every dispatch count above.)`,
+    );
+  }
+}
+
 if (outcomes.size > 0) {
   console.log("\ndispatch outcomes");
   const outcomeRows = [...outcomes.entries()].sort((a, b) => b[1] - a[1]);

src/subagent/intervention-log.ts

@@ -96,7 +96,13 @@ export type InterventionContext = Pick<
 >;
 
 export type InterventionSink = (
-  event: Omit<InterventionRecord, "ts" | "role" | "provider" | "model" | "family" | "intent">,
+  event: Omit<InterventionRecord, "ts" | "role" | "provider" | "model" | "family" | "intent"> &
+    // Outcome records are written parent-side, one per completed dispatch, so
+    // provider/model/family are not fixed at sink construction like a leaf's
+    // context — they vary per call with the child that was actually dispatched.
+    // Omitting these keys (not passing them as undefined) leaves the sink's
+    // bound context untouched for callers that do have a fixed context.
+    Partial<Pick<InterventionRecord, "provider" | "model" | "family">>,
 ) => void;
 
 /** Sink that drops everything — the default, so logging is never required. */

src/subagent/task-tool.ts

@@ -43,6 +43,7 @@ import {
   TURN_BUDGET_STOP_AFTER_DISPATCHES,
 } from "./brief-dispatch.js";
 import { createInterventionLog, type InterventionSink } from "./intervention-log.js";
+import { detectModelFamily } from "./provider-family.js";
 import { isSubAgentCancelError } from "./dispose.js";
 import { cleanupSubAgentWorktree, createSubAgentWorktree, WorktreeError } from "./worktree.js";
 import { generateSessionId } from "../session/index.js";
@@ -257,10 +258,26 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool {
   };
   // Every completed dispatch gets an outcome record — the log otherwise
   // carries shape and run state but never what the run actually produced.
+  // Tagged with the dispatched child's provider/model/family (CL-6968) so
+  // per-model intervention counts finally have a denominator: the same
+  // provider/model this dispatch actually ran under, taken after profile/
+  // agent inference resolution — never a name the parent merely intended.
   let outcomeLog: InterventionSink | null = null;
-  const recordOutcome = (kind: string, dispatchCount: number): void => {
+  const recordOutcome = (
+    kind: string,
+    dispatchCount: number,
+    identity: { provider: string; model: string },
+  ): void => {
     outcomeLog ??= createInterventionLog(deps.getWorkdirBase(), { role: "parent" });
-    outcomeLog({ id: "dispatch-outcome", class: "outcome", outcome: { kind, dispatchCount } });
+    const family = detectModelFamily({ providerName: identity.provider, model: identity.model });
+    outcomeLog({
+      id: "dispatch-outcome",
+      class: "outcome",
+      outcome: { kind, dispatchCount },
+      provider: identity.provider,
+      model: identity.model,
+      family,
+    });
   };
   return tool({
     definition: taskToolDefinition,
@@ -608,6 +625,28 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool {
             }
           : deps.onEvent;
 
+      // A dispatch can fail over to a different configured provider/model
+      // mid-run (source priority list, `resolveInferenceWithPolicy` builds the
+      // primary; the reactor retries the next source on error) — so the
+      // provider/model this call resolved before dispatch is only what the
+      // parent *intended*. `inference.done` carries the source that actually
+      // served each cycle; track the last one seen so the outcome record can
+      // prefer it over the resolved-but-possibly-superseded `provider` value.
+      let lastCycleSource: { provider: string; model: string } | undefined;
+      const onEvent = (event: ReactorEmittedEvent): void => {
+        if (event.type === "inference.done") {
+          const source = (event.data as { source?: { provider?: string; model?: string } }).source;
+          if (
+            source !== undefined &&
+            typeof source.provider === "string" &&
+            typeof source.model === "string"
+          ) {
+            lastCycleSource = { provider: source.provider, model: source.model };
+          }
+        }
+        recordEvent?.(event);
+      };
+
       const sandbox: SubAgentSandboxDeps = {
         permissionGate: deps.permissionGate,
         ...(deps.inheritMcpTools !== undefined ? { inheritMcpTools: deps.inheritMcpTools } : {}),
@@ -732,7 +771,7 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool {
             ...(doNot.length > 0 ? { doNot } : {}),
             ...(reportFocus !== undefined && reportFocus.length > 0 ? { reportFocus } : {}),
             signal: childCtl.signal,
-            ...(recordEvent !== undefined ? { onEvent: recordEvent } : {}),
+            onEvent,
             ...(deps.onProgress !== undefined ? { onProgress: deps.onProgress } : {}),
             ...(capabilities !== undefined ? { capabilities } : {}),
             ...(systemPromptRole !== undefined ? { systemPromptRole } : {}),
@@ -749,7 +788,15 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool {
             (session !== undefined && deps.sessions?.get(session.id)?.status === "cancelled");
           const salvage = classifyBriefSalvage(result);
           briefLedger.recordOutcome(fingerprint, salvage);
-          recordOutcome(salvage ?? "clean-complete", dispatchCount);
+          // Prefer the last provider/model that actually served inference
+          // (captured off inference.done above) over the pre-dispatch
+          // `provider` this call resolved — a mid-run failover to a backup
+          // source means the two can diverge, and the outcome record should
+          // describe what the child ran under, not what the parent intended.
+          recordOutcome(salvage ?? "clean-complete", dispatchCount, {
+            provider: lastCycleSource?.provider ?? provider.providerName,
+            model: lastCycleSource?.model ?? provider.model,
+          });
           const hintOptions = {
             dispatchCount,
             turnBudgetStopAfterDispatches: TURN_BUDGET_STOP_AFTER_DISPATCHES,