diff --git a/scripts/intervention-forensics.ts b/scripts/intervention-forensics.ts index 7539974b..0e73e022 100644 --- a/scripts/intervention-forensics.ts +++ b/scripts/intervention-forensics.ts @@ -11,12 +11,20 @@ // edited — stops that fired on a run which had already edited files. // early — stops that fired before half the turn budget was spent. // -// Also aggregates outcome records (CL-6938): what each completed dispatch -// actually produced (a salvage kind, or clean-complete), by kind. This is the -// log's only outcome signal, letting a stop record be read alongside what the +// Also aggregates outcome records: what each completed dispatch actually +// produced (a salvage kind, or clean-complete), by kind. This is the log's +// only outcome signal, letting a stop record be read alongside what the // dispatch it touched actually produced — it is still not gate-pass or // retry-success tracking. // +// Outcome records are now tagged with the dispatched child's provider/model/ +// family (CL-6968), so they double as the per-model dispatch denominator: +// interventions per model, divided by dispatches per model, is the one table +// below that is an actual rate. Everything else in this script stays a raw +// count — do not add another table that looks like a rate without a tracked +// denominator behind it (that mistake shipped once already and had to be +// stripped, see CL-6968). +// // Run: bun run scripts/intervention-forensics.ts // // Prints only aggregate counts and the `detail` field's first token, never turn @@ -88,6 +96,12 @@ findAll(root, INTERVENTION_FILE, files); const buckets = new Map(); const outcomes = new Map(); +// Dispatch counts per model (the outcome record's denominator, CL-6968) and +// intervention counts per model (stop+nudge only — block/outcome are not +// leaf signals), so a rate can be computed instead of a bare count. +const dispatchesByModel = new Map(); +const interventionsByModel = new Map(); +let untaggedOutcomes = 0; let records = 0; let malformed = 0; @@ -115,6 +129,12 @@ for (const file of files) { if (record.class === "outcome" && record.outcome !== undefined) { const kind = record.outcome.kind; outcomes.set(kind, (outcomes.get(kind) ?? 0) + 1); + if (record.model !== undefined) { + dispatchesByModel.set(record.model, (dispatchesByModel.get(record.model) ?? 0) + 1); + } else { + // Written before CL-6968 tagged outcome records with model identity. + untaggedOutcomes++; + } continue; } const key = `${record.class ?? "?"}/${record.id}`; @@ -128,6 +148,9 @@ for (const file of files) { bucket.byFamily.set(family, (bucket.byFamily.get(family) ?? 0) + 1); const model = record.model ?? "unknown"; bucket.byModel.set(model, (bucket.byModel.get(model) ?? 0) + 1); + if (record.class === "stop" || record.class === "nudge") { + interventionsByModel.set(model, (interventionsByModel.get(model) ?? 0) + 1); + } if (record.measurement !== undefined) { bucket.values.push(record.measurement.value); if (record.measurement.threshold !== undefined) { @@ -178,10 +201,10 @@ for (const [key, bucket] of rows) { // CL-6775: streamed degenerate-repetition aborts (mid-stream, not a turn-level // stop) get their own model breakdown — "repetition-" ids, one row -// per model, so "which model loops most" reads off directly. This is a count, -// not a rate normalized by dispatch volume: the log's outcome records (total -// completed dispatches) are written from the parent side without a model tag, -// so a per-model denominator is not yet tracked — see the PR description. +// per model, so "which model loops most" reads off directly. Still a raw +// count, not a rate — see the "interventions per dispatch by model" table +// below for the rate version, computed from the same per-model dispatch +// denominator (outcome records, CL-6968). const repetitionRows = rows.filter(([key]) => key.includes("/repetition-")); if (repetitionRows.length > 0) { const totalsByModel = new Map(); @@ -209,6 +232,32 @@ console.log( "\nedited = stops on runs that had already edited files; early = stops before half the turn budget (context, not a false-positive rate).", ); +// The one real rate in this script: interventions per dispatch, per model. +// dispatches = outcome records tagged with that model (CL-6968); interventions +// = stop+nudge records for that model. Everything above this is a count. +if (dispatchesByModel.size > 0 || interventionsByModel.size > 0) { + console.log("\ninterventions per dispatch by model (stop+nudge count / dispatch count = rate)"); + const models = new Set([...dispatchesByModel.keys(), ...interventionsByModel.keys()]); + const modelRows = [...models] + .map((model) => { + const dispatches = dispatchesByModel.get(model) ?? 0; + const interventions = interventionsByModel.get(model) ?? 0; + const rate = dispatches > 0 ? (interventions / dispatches).toFixed(3) : "-"; + return { model, dispatches, interventions, rate }; + }) + .sort((a, b) => b.interventions - a.interventions); + for (const { model, dispatches, interventions, rate } of modelRows) { + console.log( + `${model.padEnd(33)} interventions=${String(interventions).padStart(4)} dispatches=${String(dispatches).padStart(5)} rate=${rate}`, + ); + } + if (untaggedOutcomes > 0) { + console.log( + `(${untaggedOutcomes} outcome record(s) predate model tagging (CL-6968) and are excluded from every dispatch count above.)`, + ); + } +} + if (outcomes.size > 0) { console.log("\ndispatch outcomes"); const outcomeRows = [...outcomes.entries()].sort((a, b) => b[1] - a[1]); diff --git a/src/subagent/intervention-log.ts b/src/subagent/intervention-log.ts index 67f29024..abf6ccd9 100644 --- a/src/subagent/intervention-log.ts +++ b/src/subagent/intervention-log.ts @@ -96,7 +96,13 @@ export type InterventionContext = Pick< >; export type InterventionSink = ( - event: Omit, + event: Omit & + // Outcome records are written parent-side, one per completed dispatch, so + // provider/model/family are not fixed at sink construction like a leaf's + // context — they vary per call with the child that was actually dispatched. + // Omitting these keys (not passing them as undefined) leaves the sink's + // bound context untouched for callers that do have a fixed context. + Partial>, ) => void; /** Sink that drops everything — the default, so logging is never required. */ diff --git a/src/subagent/task-tool.ts b/src/subagent/task-tool.ts index f2cc67a6..0981c540 100644 --- a/src/subagent/task-tool.ts +++ b/src/subagent/task-tool.ts @@ -43,6 +43,7 @@ import { TURN_BUDGET_STOP_AFTER_DISPATCHES, } from "./brief-dispatch.js"; import { createInterventionLog, type InterventionSink } from "./intervention-log.js"; +import { detectModelFamily } from "./provider-family.js"; import { isSubAgentCancelError } from "./dispose.js"; import { cleanupSubAgentWorktree, createSubAgentWorktree, WorktreeError } from "./worktree.js"; import { generateSessionId } from "../session/index.js"; @@ -257,10 +258,26 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool { }; // Every completed dispatch gets an outcome record — the log otherwise // carries shape and run state but never what the run actually produced. + // Tagged with the dispatched child's provider/model/family (CL-6968) so + // per-model intervention counts finally have a denominator: the same + // provider/model this dispatch actually ran under, taken after profile/ + // agent inference resolution — never a name the parent merely intended. let outcomeLog: InterventionSink | null = null; - const recordOutcome = (kind: string, dispatchCount: number): void => { + const recordOutcome = ( + kind: string, + dispatchCount: number, + identity: { provider: string; model: string }, + ): void => { outcomeLog ??= createInterventionLog(deps.getWorkdirBase(), { role: "parent" }); - outcomeLog({ id: "dispatch-outcome", class: "outcome", outcome: { kind, dispatchCount } }); + const family = detectModelFamily({ providerName: identity.provider, model: identity.model }); + outcomeLog({ + id: "dispatch-outcome", + class: "outcome", + outcome: { kind, dispatchCount }, + provider: identity.provider, + model: identity.model, + family, + }); }; return tool({ definition: taskToolDefinition, @@ -608,6 +625,28 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool { } : deps.onEvent; + // A dispatch can fail over to a different configured provider/model + // mid-run (source priority list, `resolveInferenceWithPolicy` builds the + // primary; the reactor retries the next source on error) — so the + // provider/model this call resolved before dispatch is only what the + // parent *intended*. `inference.done` carries the source that actually + // served each cycle; track the last one seen so the outcome record can + // prefer it over the resolved-but-possibly-superseded `provider` value. + let lastCycleSource: { provider: string; model: string } | undefined; + const onEvent = (event: ReactorEmittedEvent): void => { + if (event.type === "inference.done") { + const source = (event.data as { source?: { provider?: string; model?: string } }).source; + if ( + source !== undefined && + typeof source.provider === "string" && + typeof source.model === "string" + ) { + lastCycleSource = { provider: source.provider, model: source.model }; + } + } + recordEvent?.(event); + }; + const sandbox: SubAgentSandboxDeps = { permissionGate: deps.permissionGate, ...(deps.inheritMcpTools !== undefined ? { inheritMcpTools: deps.inheritMcpTools } : {}), @@ -732,7 +771,7 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool { ...(doNot.length > 0 ? { doNot } : {}), ...(reportFocus !== undefined && reportFocus.length > 0 ? { reportFocus } : {}), signal: childCtl.signal, - ...(recordEvent !== undefined ? { onEvent: recordEvent } : {}), + onEvent, ...(deps.onProgress !== undefined ? { onProgress: deps.onProgress } : {}), ...(capabilities !== undefined ? { capabilities } : {}), ...(systemPromptRole !== undefined ? { systemPromptRole } : {}), @@ -749,7 +788,15 @@ export function createTaskTool(deps: TaskToolDeps): AgentTool { (session !== undefined && deps.sessions?.get(session.id)?.status === "cancelled"); const salvage = classifyBriefSalvage(result); briefLedger.recordOutcome(fingerprint, salvage); - recordOutcome(salvage ?? "clean-complete", dispatchCount); + // Prefer the last provider/model that actually served inference + // (captured off inference.done above) over the pre-dispatch + // `provider` this call resolved — a mid-run failover to a backup + // source means the two can diverge, and the outcome record should + // describe what the child ran under, not what the parent intended. + recordOutcome(salvage ?? "clean-complete", dispatchCount, { + provider: lastCycleSource?.provider ?? provider.providerName, + model: lastCycleSource?.model ?? provider.model, + }); const hintOptions = { dispatchCount, turnBudgetStopAfterDispatches: TURN_BUDGET_STOP_AFTER_DISPATCHES,