diff --git a/packages/agent-core-v2/src/agent/llmRequester/llmRequester.ts b/packages/agent-core-v2/src/agent/llmRequester/llmRequester.ts index 87187621ad..f394eccbb1 100644 --- a/packages/agent-core-v2/src/agent/llmRequester/llmRequester.ts +++ b/packages/agent-core-v2/src/agent/llmRequester/llmRequester.ts @@ -1,6 +1,7 @@ import { createDecorator } from '#/_base/di/instantiation'; import type { FinishReason } from '#/app/llmProtocol/finishReason'; import type { Message, StreamedMessagePart } from '#/app/llmProtocol/message'; +import type { ThinkingEffort } from '#/app/llmProtocol/thinkingEffort'; import type { Tool } from '#/app/llmProtocol/tool'; import type { TokenUsage } from '#/app/llmProtocol/usage'; import type { LLMRequestTrace } from '#/app/llmProtocol/requestTrace'; @@ -64,9 +65,15 @@ export interface LLMRequestTask { readonly result: Promise; } +export interface PreparedTurnRequestConfig { + readonly thinkingEffort: ThinkingEffort; +} + export interface IAgentLLMRequesterService { readonly _serviceBrand: undefined; + prepareTurnConfig(turnId: number): PreparedTurnRequestConfig | undefined; + request( overrides?: LLMRequestOverrides, onPart?: LLMRequestPartHandler, diff --git a/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts b/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts index 6e17d74736..5ef72eb850 100644 --- a/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts +++ b/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts @@ -8,7 +8,10 @@ * completion-token budget, then drives a bounded request chain: one primary * `model.request(input, signal)` attempt plus projection rebuilds for request * structure or media compatibility; general retry policy remains in the - * loop's `stepRetry` plugin. + * loop's `stepRetry` plugin. When a model is configured, `prepareTurnConfig` + * snapshots the model, effective thinking effort, and system prompt at the turn + * boundary so loop telemetry and every request in that turn share one + * configuration. * Forwards streamed `part` events to the caller's `onPart` * handler, records `usage` through `IAgentUsageService`, resolves to an * `LLMRequestFinish` on the `finish` event, logs the request lifecycle @@ -77,6 +80,7 @@ import { type LLMRequestSource, type LLMRequestTask, type LLMStreamTiming, + type PreparedTurnRequestConfig, } from './llmRequester'; import type { LLMRequestTrace } from '#/app/llmProtocol/requestTrace'; import { @@ -152,6 +156,12 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService { @IEventBus private readonly eventBus: IEventBus, ) {} + prepareTurnConfig(turnId: number): PreparedTurnRequestConfig | undefined { + if (!this.profile.hasProvider()) return undefined; + const config = this.getOrCreateTurnConfig(turnId); + return { thinkingEffort: config.resolved.thinkingLevel }; + } + async request( overrides: LLMRequestOverrides = {}, onPart: LLMRequestPartHandler = noopOnPart, @@ -529,7 +539,10 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService { private resolveTurnConfig(source: LLMRequestSource | undefined): TurnRequestConfig | undefined { if (source?.type !== 'turn') return undefined; - const turnId = source.turnId; + return this.getOrCreateTurnConfig(source.turnId); + } + + private getOrCreateTurnConfig(turnId: number): TurnRequestConfig { for (const id of this.turnConfigs.keys()) { if (id < turnId) this.turnConfigs.delete(id); } diff --git a/packages/agent-core-v2/src/agent/loop/loopService.ts b/packages/agent-core-v2/src/agent/loop/loopService.ts index fce65bfecb..f088bdc92c 100644 --- a/packages/agent-core-v2/src/agent/loop/loopService.ts +++ b/packages/agent-core-v2/src/agent/loop/loopService.ts @@ -373,9 +373,17 @@ export class AgentLoopService extends Disposable implements IAgentLoopService { const telemetryContext = this.telemetryContext.get(); const turnTelemetry = this.telemetry.withContext(telemetryContext); const { mode, provider_type, protocol } = telemetryContext; + let thinkingEffort: string | undefined; let result: TurnResult | undefined; try { - const started: TurnStartedTelemetryEvent = { turn_id: turn.id, mode, provider_type, protocol }; + thinkingEffort = this.llmRequester.prepareTurnConfig(turn.id)?.thinkingEffort; + const started: TurnStartedTelemetryEvent = { + turn_id: turn.id, + mode, + provider_type, + protocol, + thinking_effort: thinkingEffort, + }; turnTelemetry.track2('turn_started', started); result = await this.run({ turnId: turn.id, @@ -411,6 +419,7 @@ export class AgentLoopService extends Disposable implements IAgentLoopService { interrupt_reason: interruptReasonFor(result), provider_type, protocol, + thinking_effort: thinkingEffort, trace_id: traceId, }; turnTelemetry.track2('turn_interrupted', interrupted); @@ -423,6 +432,7 @@ export class AgentLoopService extends Disposable implements IAgentLoopService { mode, provider_type, protocol, + thinking_effort: thinkingEffort, trace_id: traceId, }; turnTelemetry.track2('turn_ended', ended); diff --git a/packages/agent-core-v2/src/app/telemetry/agentTelemetryContext.ts b/packages/agent-core-v2/src/app/telemetry/agentTelemetryContext.ts index 7ba536f4ae..8dec84014e 100644 --- a/packages/agent-core-v2/src/app/telemetry/agentTelemetryContext.ts +++ b/packages/agent-core-v2/src/app/telemetry/agentTelemetryContext.ts @@ -4,8 +4,8 @@ * Agent-scoped ambient telemetry context: a per-agent property bag that domains * contribute to (the `plan` domain sets `mode`, the `profile` domain mirrors * the resolved model protocol into `provider_type` / `protocol`, the `loop` - * domain sets `turn_id` at turn start and keeps `trace_id` at the active - * turn's most recent request) and that turn-scoped + * domain sets `turn_id` at turn start and keeps `trace_id` at the active turn's + * most recent request) and that turn-scoped * telemetry snapshots at launch. Decouples turn telemetry from any * specific contributor so the turn domain does not need to know about plan or * profile. Bound at Agent scope. diff --git a/packages/agent-core-v2/src/app/telemetry/events.ts b/packages/agent-core-v2/src/app/telemetry/events.ts index 774fbf30b3..0323b4f451 100644 --- a/packages/agent-core-v2/src/app/telemetry/events.ts +++ b/packages/agent-core-v2/src/app/telemetry/events.ts @@ -47,6 +47,7 @@ export interface TurnStartedEvent { mode: 'agent' | 'plan'; provider_type?: string; protocol?: string; + thinking_effort?: string; } export interface TurnInterruptedEvent { @@ -56,6 +57,7 @@ export interface TurnInterruptedEvent { interrupt_reason: 'user_cancelled' | 'aborted' | 'max_steps' | 'error' | 'filtered' | 'blocked'; provider_type?: string; protocol?: string; + thinking_effort?: string; trace_id?: string; } @@ -66,6 +68,7 @@ export interface TurnEndedEvent { mode: 'agent' | 'plan'; provider_type?: string; protocol?: string; + thinking_effort?: string; trace_id?: string; } @@ -410,6 +413,7 @@ export const telemetryEventDefinitions = { mode: 'Agent mode the turn runs in', provider_type: 'Provider protocol type', protocol: 'Request protocol', + thinking_effort: 'Effective thinking effort the turn runs with', }, }), turn_interrupted: defineTelemetryEvent({ @@ -422,6 +426,7 @@ export const telemetryEventDefinitions = { interrupt_reason: 'Why the turn was interrupted', provider_type: 'Provider protocol type', protocol: 'Request protocol', + thinking_effort: 'Effective thinking effort the turn ran with', trace_id: 'Trace id of the most recent LLM request in this turn (the failed request when the turn errored); absent for non-Kimi protocols', }, @@ -436,6 +441,7 @@ export const telemetryEventDefinitions = { mode: 'Agent mode the turn ran in', provider_type: 'Provider protocol type', protocol: 'Request protocol', + thinking_effort: 'Effective thinking effort the turn ran with', trace_id: 'Trace id of the most recent LLM request in this turn; absent for non-Kimi protocols', }, diff --git a/packages/agent-core-v2/test/agent/loop/loop.test.ts b/packages/agent-core-v2/test/agent/loop/loop.test.ts index 1f4224b6f8..6dba3eb115 100644 --- a/packages/agent-core-v2/test/agent/loop/loop.test.ts +++ b/packages/agent-core-v2/test/agent/loop/loop.test.ts @@ -635,7 +635,7 @@ describe('turn telemetry', () => { expect(records).toContainEqual({ event: 'turn_started', - properties: { turn_id: 0, mode: 'agent', provider_type: 'kimi', protocol: 'kimi' }, + properties: { turn_id: 0, mode: 'agent', provider_type: 'kimi', protocol: 'kimi', thinking_effort: 'off' }, }); expect(records).toContainEqual({ event: 'turn_ended', @@ -646,6 +646,7 @@ describe('turn telemetry', () => { mode: 'agent', provider_type: 'kimi', protocol: 'kimi', + thinking_effort: 'off', }), }); expect(records.some((record) => record.event === 'turn_interrupted')).toBe(false); @@ -654,6 +655,50 @@ describe('turn telemetry', () => { } }); + it('keeps turn telemetry aligned with the request config across pre-step changes', async () => { + const records: TelemetryRecord[] = []; + const local = createTestAgent({ telemetry: recordingTelemetry(records) }); + try { + const localLoop = local.get(IAgentLoopService); + const localProfile = local.get(IAgentProfileService); + local.configure({ + modelCapabilities: { + image_in: false, + video_in: false, + audio_in: false, + thinking: true, + tool_use: true, + max_context_tokens: 1_000_000, + }, + }); + localProfile.update({ activeToolNames: [] }); + localProfile.setThinking('on'); + localLoop.hooks.onWillBeginStep.register('test-change-thinking', async (_ctx, next) => { + localProfile.setThinking('off'); + await next(); + }); + local.mockNextResponse({ type: 'text', text: 'hi' }); + + await local.rpc.prompt({ input: [{ type: 'text', text: 'Hello' }] }); + await local.untilTurnEnd(); + + const request = local.allEvents.find( + (event) => event.type === '[wire]' && event.event === 'llm.request', + ); + expect(request?.args).toMatchObject({ thinkingEffort: 'on' }); + expect(records).toContainEqual({ + event: 'turn_started', + properties: expect.objectContaining({ turn_id: 0, thinking_effort: 'on' }), + }); + expect(records).toContainEqual({ + event: 'turn_ended', + properties: expect.objectContaining({ turn_id: 0, thinking_effort: 'on' }), + }); + } finally { + await local.dispose(); + } + }); + it('attaches the latest request trace id to turn_ended', async () => { const records: TelemetryRecord[] = []; const local = createTestAgent({ telemetry: recordingTelemetry(records) }); @@ -926,6 +971,7 @@ function createTimingRequester(): IAgentLLMRequesterService { const requester: IAgentLLMRequesterService = { _serviceBrand: undefined, + prepareTurnConfig: () => ({ thinkingEffort: 'off' }), async request(_overrides, onPart = () => {}) { await onPart({ type: 'text', text: 'answer' }); return { diff --git a/packages/agent-core/src/agent/turn/index.ts b/packages/agent-core/src/agent/turn/index.ts index 3136aa6889..506f60ae0d 100644 --- a/packages/agent-core/src/agent/turn/index.ts +++ b/packages/agent-core/src/agent/turn/index.ts @@ -532,7 +532,7 @@ export class TurnFlow { const telemetryMode = this.telemetryMode(); this.telemetryModeByTurn.set(turnId, telemetryMode); this.currentStepByTurn.set(turnId, 0); - this.agent.telemetry.track('turn_started', { turn_id: turnId, mode: telemetryMode, ...this.requestProtocolProps() }); + this.agent.telemetry.track('turn_started', { turn_id: turnId, mode: telemetryMode, thinking_effort: this.agent.config.thinkingEffort, ...this.requestProtocolProps() }); this.agent.fullCompaction.resetForTurn(); this.agent.usage.beginTurn(); this.agent.emitEvent({ type: 'turn.started', turnId, origin }); @@ -651,6 +651,7 @@ export class TurnFlow { reason: ended.reason, duration_ms: ended.durationMs, mode: this.telemetryModeByTurn.get(turnId) ?? this.telemetryMode(), + thinking_effort: this.agent.config.thinkingEffort, ...this.requestProtocolProps(), trace_id: terminalTraceId, }); @@ -1178,6 +1179,7 @@ export class TurnFlow { this.agent.telemetry.track('turn_interrupted', { turn_id: turnId, mode: this.telemetryModeByTurn.get(turnId) ?? this.telemetryMode(), + thinking_effort: this.agent.config.thinkingEffort, at_step: atStep, interrupt_reason: interruptReason, ...this.requestProtocolProps(), diff --git a/packages/agent-core/test/agent/turn.test.ts b/packages/agent-core/test/agent/turn.test.ts index 3af3fb3069..e3218956b2 100644 --- a/packages/agent-core/test/agent/turn.test.ts +++ b/packages/agent-core/test/agent/turn.test.ts @@ -519,11 +519,11 @@ describe('Agent turn flow', () => { expect(records).toContainEqual({ event: 'turn_started', - properties: { turn_id: 0, mode: 'agent' }, + properties: { turn_id: 0, mode: 'agent', thinking_effort: 'off' }, }); expect(records).toContainEqual({ event: 'turn_interrupted', - properties: { turn_id: 0, mode: 'agent', at_step: 0, interrupt_reason: 'error' }, + properties: { turn_id: 0, mode: 'agent', thinking_effort: 'off', at_step: 0, interrupt_reason: 'error' }, }); }); @@ -649,7 +649,7 @@ describe('Agent turn flow', () => { const started = records.find((candidate) => candidate.event === 'turn_started'); expect(started).toEqual({ event: 'turn_started', - properties: expect.objectContaining({ mode: 'agent', provider_type: 'kimi', protocol: 'kimi' }), + properties: expect.objectContaining({ mode: 'agent', provider_type: 'kimi', protocol: 'kimi', thinking_effort: 'off' }), }); const ended = records.find((candidate) => candidate.event === 'turn_ended'); @@ -661,6 +661,7 @@ describe('Agent turn flow', () => { reason: 'completed', provider_type: 'kimi', protocol: 'kimi', + thinking_effort: 'off', duration_ms: expect.any(Number), }), });