From 78d815502bff3e16b2351eb8ff2e7cf765ec567d Mon Sep 17 00:00:00 2001 From: CookSleep Date: Sat, 12 Sep 2026 16:08:25 +0800 Subject: [PATCH 1/7] feat(gemini): support agentic video understanding --- packages/adapter-gemini/src/index.ts | 2 + .../src/locales/en-US.schema.yml | 1 + .../src/locales/zh-CN.schema.yml | 1 + packages/adapter-gemini/src/requester.ts | 33 +++---- packages/adapter-gemini/src/types.ts | 10 ++- packages/adapter-gemini/src/utils.ts | 88 ++++++++++++++----- 6 files changed, 93 insertions(+), 42 deletions(-) diff --git a/packages/adapter-gemini/src/index.ts b/packages/adapter-gemini/src/index.ts index f48ef5b97..54eac1901 100644 --- a/packages/adapter-gemini/src/index.ts +++ b/packages/adapter-gemini/src/index.ts @@ -53,6 +53,7 @@ export interface Config extends ChatLunaPlugin.Config { codeExecution: boolean urlContext: boolean imageGeneration: boolean + agenticVideoUnderstanding: boolean imageModelSearch: boolean thinkingBudget: number includeThoughts: boolean @@ -123,6 +124,7 @@ export const Config: Schema = Schema.intersect([ imageModelSearch: Schema.boolean().default(false), groundingContentDisplay: Schema.boolean().default(false), imageGeneration: Schema.boolean().default(false), + agenticVideoUnderstanding: Schema.boolean().default(false), useCamelCaseSystemInstruction: Schema.boolean().default(false), useCamelCaseMediaFields: Schema.boolean().default(false), nonStreaming: Schema.boolean().default(false) diff --git a/packages/adapter-gemini/src/locales/en-US.schema.yml b/packages/adapter-gemini/src/locales/en-US.schema.yml index 33ec53f29..f621d08e7 100644 --- a/packages/adapter-gemini/src/locales/en-US.schema.yml +++ b/packages/adapter-gemini/src/locales/en-US.schema.yml @@ -33,6 +33,7 @@ $inner: thinkingBudget: 'Thinking budget (-1-24576). (0: dynamic thinking) Higher: more tokens spent on thinking. Currently only supports `gemini-2.5` series models.' groundingContentDisplay: 'Enable display of search results' imageGeneration: 'Enable image generation (only for `gemini-*-image-*` and `gemini-2.5-flash-image-preview` model)' + agenticVideoUnderstanding: 'Enable agentic video understanding for video inputs. The model selectively explores video and audio in segments and can increase the sampling frame rate for fast-moving scenes as needed. This reduces long-video input costs and improves understanding of specific details, but may reduce global understanding of shorter videos and increase response latency. Currently supported by `gemini-3.5-flash-lite` `gemini-3.6-flash` `gemini-3.7-flash` and `gemini-3.8-flash`.' imageModelSearch: 'Enable search for image generation models. When enabled, supported image models will generate additional variants with a `-search` suffix (e.g. `gemini-3-pro-image-search`).' includeThoughts: 'Enable retrieval of model thoughts' codeExecution: 'Enable code execution tool' diff --git a/packages/adapter-gemini/src/locales/zh-CN.schema.yml b/packages/adapter-gemini/src/locales/zh-CN.schema.yml index 684b147aa..7dc56242a 100644 --- a/packages/adapter-gemini/src/locales/zh-CN.schema.yml +++ b/packages/adapter-gemini/src/locales/zh-CN.schema.yml @@ -31,6 +31,7 @@ $inner: temperature: '回复的随机性程度，数值越高，回复越随机（范围：0~2）。' googleSearch: '为模型启用谷歌搜索。' imageGeneration: '为模型启用图像生成。目前仅支持 `gemini-*-image-*` 和 `gemini-2.5-flash-image-preview` 模型。' + agenticVideoUnderstanding: '为视频输入启用 Agentic 视频理解功能。开启后，模型会选择性、可分片地浏览视频的图像、音频信息，并可根据需要，增加高速片段的采样帧率。此功能旨在降低长视频的输入成本，并提升模型对于部分细节的理解能力。但此功能可能降低模型对于较短视频的全局理解能力，并增加响应延迟。目前仅支持 `gemini-3.5-flash-lite` `gemini-3.6-flash` `gemini-3.7-flash` `gemini-3.8-flash`。' thinkingBudget: '思考预算，范围：(-1~24576)，设置的数值越大，思考时花费的 Token 越多,-1 为动态思考。目前仅支持 gemini 2.5 系列模型。' groundingContentDisplay: '是否显示谷歌搜索结果。' imageModelSearch: '为图片生成模型启用搜索功能。开启后，支持搜索的图片模型将额外生成带 `-search` 后缀的变体（如 `gemini-3-pro-image-search`）。' diff --git a/packages/adapter-gemini/src/requester.ts b/packages/adapter-gemini/src/requester.ts index b370640de..9ec493ea2 100644 --- a/packages/adapter-gemini/src/requester.ts +++ b/packages/adapter-gemini/src/requester.ts @@ -602,7 +602,11 @@ export class GeminiRequester if ( updatedContent || updatedToolCalling || - chunk['thoughtSignature'] != null + chunk['thoughtSignature'] != null || + chunk['toolCall'] != null || + chunk['toolResponse'] != null || + chunk['executableCode'] != null || + chunk['codeExecutionResult'] != null ) { const messageChunk = this._createMessageChunk( updatedContent, @@ -768,7 +772,14 @@ export class GeminiRequester }) const sig = chunk['thoughtSignature'] let thoughtData: Record | undefined - if (sig != null) { + if ( + chunk['toolCall'] != null || + chunk['toolResponse'] != null || + chunk['executableCode'] != null || + chunk['codeExecutionResult'] != null + ) { + thoughtData = { parts: [chunk] } + } else if (sig != null) { const id = functionCall?.id ?? chunk['functionCall']?.id if (id != null) { thoughtData = { @@ -777,23 +788,7 @@ export class GeminiRequester } } } else { - const part = { - thoughtSignature: sig, - toolCall: chunk['toolCall'], - toolResponse: chunk['toolResponse'], - executableCode: chunk['executableCode'], - codeExecutionResult: chunk['codeExecutionResult'] - } - const contextId = - chunk['toolCall']?.id ?? - chunk['toolResponse']?.id ?? - chunk['executableCode']?.id ?? - chunk['codeExecutionResult']?.id - - thoughtData = - contextId != null - ? { [contextId]: [part] } - : { parts: [part] } + thoughtData = { parts: [{ thoughtSignature: sig }] } } } diff --git a/packages/adapter-gemini/src/types.ts b/packages/adapter-gemini/src/types.ts index 40c5da189..10a3e3d9e 100644 --- a/packages/adapter-gemini/src/types.ts +++ b/packages/adapter-gemini/src/types.ts @@ -60,6 +60,7 @@ export type ChatInlineDataPart = { displayName?: string data?: string } + mediaProcessing?: 'AGENTIC' } export type ChatUploadDataPart = { @@ -67,6 +68,7 @@ export type ChatUploadDataPart = { mime_type: string data?: string } + media_processing?: 'AGENTIC' } export type ChatFunctionCallingPart = { @@ -82,7 +84,13 @@ export type ChatFunctionResponsePart = { functionResponse: { name: string response: Record - parts?: (ChatInlineDataPart | ChatUploadDataPart)[] + parts?: (( + | Pick + | Pick + ) & { + mediaProcessing?: never + media_processing?: never + })[] id?: string } } diff --git a/packages/adapter-gemini/src/utils.ts b/packages/adapter-gemini/src/utils.ts index 90456c475..bad81e808 100644 --- a/packages/adapter-gemini/src/utils.ts +++ b/packages/adapter-gemini/src/utils.ts @@ -41,6 +41,17 @@ export async function langchainMessageToGeminiMessage( plugin: ChatLunaPlugin, model?: string ): Promise { + const match = model?.match( + /gemini[-_\s]*(\d+)(?:[._](\d+))?[-_\s]*flash(?:[-_\s]*(lite))?/i + ) + // Accept provider prefixes/suffixes and future Flash versions. + const agentic = + plugin.config.agenticVideoUnderstanding && + match != null && + (Number(match[1]) > 3 || + (Number(match[1]) === 3 && + (Number(match[2]) >= 6 || + (Number(match[2]) === 5 && match[3] != null)))) const result: ChatCompletionResponseMessage[] = [] for (let i = 0; i < messages.length; i++) { const message = messages[i] @@ -60,7 +71,8 @@ export async function langchainMessageToGeminiMessage( await processFunctionMessage( plugin, msg, - plugin.config.useCamelCaseSystemInstruction + plugin.config.useCamelCaseSystemInstruction, + agentic ) ).parts ) @@ -76,7 +88,8 @@ export async function langchainMessageToGeminiMessage( await processFunctionMessage( plugin, message, - plugin.config.useCamelCaseSystemInstruction + plugin.config.useCamelCaseSystemInstruction, + agentic ) ) continue @@ -90,7 +103,11 @@ export async function langchainMessageToGeminiMessage( ? message.content.length > 0 ? [{ text: message.content }] : [] - : await processGeminiContentParts(plugin, message.content) + : await processGeminiContentParts( + plugin, + message.content, + agentic + ) item.parts = [...getContextParts(thoughtData), ...parts] @@ -176,17 +193,17 @@ function isContextPart(part: any): part is ChatPart { } function getContextParts(data: Record, id?: string) { - const parts = data['parts'] ?? [data, ...Object.values(data)] - const raw = id != null ? data[id] : parts + const raw = id != null ? data[id] : [data, ...Object.values(data)] if (raw == null) return [] - return (Array.isArray(raw) ? raw : [raw]).filter(isContextPart) + return (Array.isArray(raw) ? raw.flat() : [raw]).filter(isContextPart) } async function processFunctionMessage( plugin: ChatLunaPlugin, message: AIMessage | ToolMessage, - removeId: boolean + removeId: boolean, + agentic: boolean ): Promise { const thoughtData: Record = message.additional_kwargs['thought_data'] ?? {} @@ -194,11 +211,19 @@ async function processFunctionMessage( if (message['tool_calls']) { message = message as AIMessage const toolCalls = message.tool_calls - const parts: ChatPart[] = [] + const parts: ChatPart[] = getContextParts( + Object.fromEntries( + Object.entries(thoughtData).filter( + ([id]) => !toolCalls.some((call) => call.id === id) + ) + ) + ) for (const toolCall of toolCalls) { // tool context: replay context tied to this tool call first. - parts.push(...getContextParts(thoughtData, toolCall.id)) + if (toolCall.id != null) { + parts.push(...getContextParts(thoughtData, toolCall.id)) + } const functionCall: ChatFunctionCallingPart['functionCall'] = { name: toolCall.name, @@ -228,6 +253,7 @@ async function processFunctionMessage( } const finalMessage = message as ToolMessage + const media: ChatPart[] = [] const functionResponse: ChatFunctionResponsePart['functionResponse'] = { name: message.name, @@ -250,10 +276,21 @@ async function processFunctionMessage( (part) => isMessageContentImageUrl(part) || isGeminiFileLikeContent(part) - ) + ), + agentic ) - if (parts.length > 0) { - functionResponse.parts = parts + for (const part of parts) { + if (part['mediaProcessing'] || part['media_processing']) { + media.push(part) + } else if ('inlineData' in part) { + ;(functionResponse.parts ??= []).push({ + inlineData: part.inlineData + }) + } else if ('inline_data' in part) { + ;(functionResponse.parts ??= []).push({ + inline_data: part.inline_data + }) + } } } else { functionResponse.response = parseJsonArgs(message.content as string) @@ -268,7 +305,8 @@ async function processFunctionMessage( parts: [ { functionResponse - } + }, + ...media ] } } @@ -324,29 +362,34 @@ function isGeminiFileLikeContent( function createGeminiInlineDataPart( plugin: ChatLunaPlugin, data: string, - mimeType: string + mimeType: string, + agentic = false ) { if (plugin.config.useCamelCaseMediaFields) { return { - inlineData: { data, mimeType } + inlineData: { data, mimeType }, + mediaProcessing: agentic ? ('AGENTIC' as const) : undefined } } return { - inline_data: { data, mime_type: mimeType } + inline_data: { data, mime_type: mimeType }, + media_processing: agentic ? ('AGENTIC' as const) : undefined } } async function processGeminiFileLikeContent( plugin: ChatLunaPlugin, - part: GeminiFileLikeContent + part: GeminiFileLikeContent, + agentic: boolean ) { try { const { buffer, mimeType } = await fetchFileLikeUrl(plugin, part) return createGeminiInlineDataPart( plugin, buffer.toString('base64'), - mimeType + mimeType, + agentic && mimeType.startsWith('video/') ) } catch (e) { logger.warn(`Failed to fetch ${part.type}`, e) @@ -356,8 +399,9 @@ async function processGeminiFileLikeContent( async function processGeminiContentParts( plugin: ChatLunaPlugin, - content: MessageContentComplex[] -) { + content: MessageContentComplex[], + agentic: boolean +): Promise { const mappedParts = await Promise.all( content.map(async (part) => { if (isMessageContentText(part)) { @@ -367,9 +411,9 @@ async function processGeminiContentParts( return await processGeminiImageContent(plugin, part) } if (isGeminiFileLikeContent(part)) { - return await processGeminiFileLikeContent(plugin, part) + return await processGeminiFileLikeContent(plugin, part, agentic) } - return part as any + return part as unknown as ChatPart }) ) From 15558b516d72562aa76807e229d9f30914849d3b Mon Sep 17 00:00:00 2001 From: CookSleep Date: Sat, 12 Sep 2026 16:09:14 +0800 Subject: [PATCH 2/7] fix(gemini): skip replaying agentic media traces --- packages/adapter-gemini/src/requester.ts | 14 ++++++++------ packages/adapter-gemini/src/utils.ts | 6 ++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/adapter-gemini/src/requester.ts b/packages/adapter-gemini/src/requester.ts index 9ec493ea2..fa5adca9d 100644 --- a/packages/adapter-gemini/src/requester.ts +++ b/packages/adapter-gemini/src/requester.ts @@ -602,9 +602,9 @@ export class GeminiRequester if ( updatedContent || updatedToolCalling || - chunk['thoughtSignature'] != null || - chunk['toolCall'] != null || - chunk['toolResponse'] != null || + (chunk['thoughtSignature'] != null && + chunk['toolCall'] == null && + chunk['toolResponse'] == null) || chunk['executableCode'] != null || chunk['codeExecutionResult'] != null ) { @@ -773,13 +773,15 @@ export class GeminiRequester const sig = chunk['thoughtSignature'] let thoughtData: Record | undefined if ( - chunk['toolCall'] != null || - chunk['toolResponse'] != null || chunk['executableCode'] != null || chunk['codeExecutionResult'] != null ) { thoughtData = { parts: [chunk] } - } else if (sig != null) { + } else if ( + sig != null && + chunk['toolCall'] == null && + chunk['toolResponse'] == null + ) { const id = functionCall?.id ?? chunk['functionCall']?.id if (id != null) { thoughtData = { diff --git a/packages/adapter-gemini/src/utils.ts b/packages/adapter-gemini/src/utils.ts index bad81e808..7476ee72b 100644 --- a/packages/adapter-gemini/src/utils.ts +++ b/packages/adapter-gemini/src/utils.ts @@ -182,13 +182,11 @@ function parseJsonArgs(args: string): Record { } function isContextPart(part: any): part is ChatPart { + // Gemini rejects replayed Agentic media tool steps. return ( typeof part === 'object' && part != null && - (part['toolCall'] != null || - part['toolResponse'] != null || - part['executableCode'] != null || - part['codeExecutionResult'] != null) + (part['executableCode'] != null || part['codeExecutionResult'] != null) ) } From 4711b1cf97a18a357d53efb1cf13609dd9354b67 Mon Sep 17 00:00:00 2001 From: CookSleep Date: Sat, 12 Sep 2026 16:25:15 +0800 Subject: [PATCH 3/7] =?UTF-8?q?fix(gemini):=20=E4=BF=9D=E7=95=99=E9=9D=9E?= =?UTF-8?q?=E5=AA=92=E4=BD=93=E5=B7=A5=E5=85=B7=E8=BD=A8=E8=BF=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/adapter-gemini/src/requester.ts | 17 +++++++++-------- packages/adapter-gemini/src/utils.ts | 16 ++++++++++++++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/packages/adapter-gemini/src/requester.ts b/packages/adapter-gemini/src/requester.ts index fa5adca9d..1546496a3 100644 --- a/packages/adapter-gemini/src/requester.ts +++ b/packages/adapter-gemini/src/requester.ts @@ -39,6 +39,7 @@ import { createChatGenerationParams, getUsage, isChatResponse, + isMediaProcessingPart, partAsType, partAsTypeCheck, prepareModelConfig @@ -552,6 +553,8 @@ export class GeminiRequester let functionIndex = 0 for await (const chunk of iterable) { + if (isMediaProcessingPart(chunk)) continue + let parsedChunk: ChatUsageMetadataPart | undefined if ( (parsedChunk = partAsTypeCheck( @@ -602,9 +605,9 @@ export class GeminiRequester if ( updatedContent || updatedToolCalling || - (chunk['thoughtSignature'] != null && - chunk['toolCall'] == null && - chunk['toolResponse'] == null) || + chunk['thoughtSignature'] != null || + chunk['toolCall'] != null || + chunk['toolResponse'] != null || chunk['executableCode'] != null || chunk['codeExecutionResult'] != null ) { @@ -773,15 +776,13 @@ export class GeminiRequester const sig = chunk['thoughtSignature'] let thoughtData: Record | undefined if ( + chunk['toolCall'] != null || + chunk['toolResponse'] != null || chunk['executableCode'] != null || chunk['codeExecutionResult'] != null ) { thoughtData = { parts: [chunk] } - } else if ( - sig != null && - chunk['toolCall'] == null && - chunk['toolResponse'] == null - ) { + } else if (sig != null) { const id = functionCall?.id ?? chunk['functionCall']?.id if (id != null) { thoughtData = { diff --git a/packages/adapter-gemini/src/utils.ts b/packages/adapter-gemini/src/utils.ts index 7476ee72b..43d513333 100644 --- a/packages/adapter-gemini/src/utils.ts +++ b/packages/adapter-gemini/src/utils.ts @@ -181,12 +181,24 @@ function parseJsonArgs(args: string): Record { } } +export function isMediaProcessingPart(part: ChatPart): boolean { + const tool = part['toolCall'] ?? part['toolResponse'] + // Agentic media steps can omit toolType and cannot be replayed by Gemini. + return ( + tool != null && + (tool.toolType == null || tool.toolType === 'MEDIA_PROCESSING') + ) +} + function isContextPart(part: any): part is ChatPart { - // Gemini rejects replayed Agentic media tool steps. return ( typeof part === 'object' && part != null && - (part['executableCode'] != null || part['codeExecutionResult'] != null) + !isMediaProcessingPart(part) && + (part['toolCall'] != null || + part['toolResponse'] != null || + part['executableCode'] != null || + part['codeExecutionResult'] != null) ) } From 58a23428bd786ed707e334170cf185843f7a0869 Mon Sep 17 00:00:00 2001 From: CookSleep Date: Sat, 12 Sep 2026 16:31:56 +0800 Subject: [PATCH 4/7] =?UTF-8?q?refactor(gemini):=20=E9=99=8D=E4=BD=8E?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=E6=B6=88=E6=81=AF=E5=A4=84=E7=90=86=E5=A4=8D?= =?UTF-8?q?=E6=9D=82=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/adapter-gemini/src/utils.ts | 84 ++++++++++++++-------------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/packages/adapter-gemini/src/utils.ts b/packages/adapter-gemini/src/utils.ts index 43d513333..597aca320 100644 --- a/packages/adapter-gemini/src/utils.ts +++ b/packages/adapter-gemini/src/utils.ts @@ -209,57 +209,55 @@ function getContextParts(data: Record, id?: string) { return (Array.isArray(raw) ? raw.flat() : [raw]).filter(isContextPart) } -async function processFunctionMessage( - plugin: ChatLunaPlugin, - message: AIMessage | ToolMessage, - removeId: boolean, - agentic: boolean -): Promise { +function processFunctionCalls( + message: AIMessage, + removeId: boolean +): ChatCompletionResponseMessage { const thoughtData: Record = message.additional_kwargs['thought_data'] ?? {} - - if (message['tool_calls']) { - message = message as AIMessage - const toolCalls = message.tool_calls - const parts: ChatPart[] = getContextParts( - Object.fromEntries( - Object.entries(thoughtData).filter( - ([id]) => !toolCalls.some((call) => call.id === id) - ) + const parts: ChatPart[] = getContextParts( + Object.fromEntries( + Object.entries(thoughtData).filter( + ([id]) => !message.tool_calls.some((call) => call.id === id) ) ) + ) - for (const toolCall of toolCalls) { - // tool context: replay context tied to this tool call first. - if (toolCall.id != null) { - parts.push(...getContextParts(thoughtData, toolCall.id)) - } - - const functionCall: ChatFunctionCallingPart['functionCall'] = { - name: toolCall.name, - args: toolCall.args - } - if (!removeId || toolCall.id) { - functionCall.id = toolCall.id - } - const data = thoughtData[toolCall.id] ?? thoughtData - const sig = Array.isArray(data) - ? data.find( - (item) => typeof item?.thoughtSignature === 'string' - )?.thoughtSignature - : data.thoughtSignature - - // tool calls: reattach custom tool calls with their thought signatures. - parts.push({ - functionCall, - ...(typeof sig === 'string' ? { thoughtSignature: sig } : {}) - }) + for (const toolCall of message.tool_calls) { + if (toolCall.id != null) { + parts.push(...getContextParts(thoughtData, toolCall.id)) } - return { - role: 'model', - parts + const functionCall: ChatFunctionCallingPart['functionCall'] = { + name: toolCall.name, + args: toolCall.args + } + if (!removeId || toolCall.id) { + functionCall.id = toolCall.id } + const data = thoughtData[toolCall.id] ?? thoughtData + const sig = Array.isArray(data) + ? data.find((item) => typeof item?.thoughtSignature === 'string') + ?.thoughtSignature + : data.thoughtSignature + + parts.push({ + functionCall, + ...(typeof sig === 'string' ? { thoughtSignature: sig } : {}) + }) + } + + return { role: 'model', parts } +} + +async function processFunctionMessage( + plugin: ChatLunaPlugin, + message: AIMessage | ToolMessage, + removeId: boolean, + agentic: boolean +): Promise { + if (message['tool_calls']) { + return processFunctionCalls(message as AIMessage, removeId) } const finalMessage = message as ToolMessage From e0cae0f34f2a89225e5b286fff22311f85fd8cb0 Mon Sep 17 00:00:00 2001 From: dingyi Date: Sat, 12 Sep 2026 19:16:26 +0800 Subject: [PATCH 5/7] refactor(gemini): simplify agentic video config and conversion --- packages/adapter-gemini/src/index.ts | 7 +- .../src/locales/en-US.schema.yml | 6 +- .../src/locales/zh-CN.schema.yml | 6 +- packages/adapter-gemini/src/requester.ts | 320 ++++------- packages/adapter-gemini/src/types.ts | 84 +-- packages/adapter-gemini/src/utils.ts | 504 +++++++----------- 6 files changed, 363 insertions(+), 564 deletions(-) diff --git a/packages/adapter-gemini/src/index.ts b/packages/adapter-gemini/src/index.ts index 54eac1901..42dae1f45 100644 --- a/packages/adapter-gemini/src/index.ts +++ b/packages/adapter-gemini/src/index.ts @@ -53,7 +53,7 @@ export interface Config extends ChatLunaPlugin.Config { codeExecution: boolean urlContext: boolean imageGeneration: boolean - agenticVideoUnderstanding: boolean + agenticVideo: boolean imageModelSearch: boolean thinkingBudget: number includeThoughts: boolean @@ -124,7 +124,7 @@ export const Config: Schema = Schema.intersect([ imageModelSearch: Schema.boolean().default(false), groundingContentDisplay: Schema.boolean().default(false), imageGeneration: Schema.boolean().default(false), - agenticVideoUnderstanding: Schema.boolean().default(false), + agenticVideo: Schema.boolean().default(false), useCamelCaseSystemInstruction: Schema.boolean().default(false), useCamelCaseMediaFields: Schema.boolean().default(false), nonStreaming: Schema.boolean().default(false) @@ -132,8 +132,7 @@ export const Config: Schema = Schema.intersect([ ]).i18n({ 'zh-CN': require('./locales/zh-CN.schema.yml'), 'en-US': require('./locales/en-US.schema.yml') - // eslint-disable-next-line @typescript-eslint/no-explicit-any -}) as any +}) as Schema export const usage = ` ## Gemini 适配器说明 diff --git a/packages/adapter-gemini/src/locales/en-US.schema.yml b/packages/adapter-gemini/src/locales/en-US.schema.yml index f621d08e7..8a1486dec 100644 --- a/packages/adapter-gemini/src/locales/en-US.schema.yml +++ b/packages/adapter-gemini/src/locales/en-US.schema.yml @@ -33,11 +33,11 @@ $inner: thinkingBudget: 'Thinking budget (-1-24576). (0: dynamic thinking) Higher: more tokens spent on thinking. Currently only supports `gemini-2.5` series models.' groundingContentDisplay: 'Enable display of search results' imageGeneration: 'Enable image generation (only for `gemini-*-image-*` and `gemini-2.5-flash-image-preview` model)' - agenticVideoUnderstanding: 'Enable agentic video understanding for video inputs. The model selectively explores video and audio in segments and can increase the sampling frame rate for fast-moving scenes as needed. This reduces long-video input costs and improves understanding of specific details, but may reduce global understanding of shorter videos and increase response latency. Currently supported by `gemini-3.5-flash-lite` `gemini-3.6-flash` `gemini-3.7-flash` and `gemini-3.8-flash`.' + agenticVideo: 'Enable on-demand video analysis. The model selects segments to examine, reducing long-video input costs. It may miss the overall context of short videos and respond more slowly. Supports Gemini 3.5 Flash Lite and Gemini 3.6, 3.7 and 3.8 Flash.' imageModelSearch: 'Enable search for image generation models. When enabled, supported image models will generate additional variants with a `-search` suffix (e.g. `gemini-3-pro-image-search`).' includeThoughts: 'Enable retrieval of model thoughts' codeExecution: 'Enable code execution tool' urlContext: 'Enable URL context retrieval tool' - useCamelCaseSystemInstruction: 'Use camelCase systemInstruction instead of snake_case system_instruction' - useCamelCaseMediaFields: 'Use camelCase inlineData and mimeType instead of snake_case inline_data and mime_type' + useCamelCaseSystemInstruction: 'Use camelCase field names such as systemInstruction for compatibility with third-party APIs.' + useCamelCaseMediaFields: 'Use camelCase media fields such as inlineData, mimeType and mediaProcessing for compatibility with third-party APIs.' nonStreaming: 'Force disable streaming response. When enabled, requests will always be made in non-streaming mode, even if the stream parameter is configured.' diff --git a/packages/adapter-gemini/src/locales/zh-CN.schema.yml b/packages/adapter-gemini/src/locales/zh-CN.schema.yml index 7dc56242a..c03d325a4 100644 --- a/packages/adapter-gemini/src/locales/zh-CN.schema.yml +++ b/packages/adapter-gemini/src/locales/zh-CN.schema.yml @@ -31,13 +31,13 @@ $inner: temperature: '回复的随机性程度，数值越高，回复越随机（范围：0~2）。' googleSearch: '为模型启用谷歌搜索。' imageGeneration: '为模型启用图像生成。目前仅支持 `gemini-*-image-*` 和 `gemini-2.5-flash-image-preview` 模型。' - agenticVideoUnderstanding: '为视频输入启用 Agentic 视频理解功能。开启后，模型会选择性、可分片地浏览视频的图像、音频信息，并可根据需要，增加高速片段的采样帧率。此功能旨在降低长视频的输入成本，并提升模型对于部分细节的理解能力。但此功能可能降低模型对于较短视频的全局理解能力，并增加响应延迟。目前仅支持 `gemini-3.5-flash-lite` `gemini-3.6-flash` `gemini-3.7-flash` `gemini-3.8-flash`。' + agenticVideo: '启用按需视频分析。模型会挑选片段查看，减少长视频的输入费用，但可能忽略短视频的整体内容，回复也可能更慢。支持 Gemini 3.5 Flash Lite 和 Gemini 3.6、3.7、3.8 Flash。' thinkingBudget: '思考预算，范围：(-1~24576)，设置的数值越大，思考时花费的 Token 越多,-1 为动态思考。目前仅支持 gemini 2.5 系列模型。' groundingContentDisplay: '是否显示谷歌搜索结果。' imageModelSearch: '为图片生成模型启用搜索功能。开启后，支持搜索的图片模型将额外生成带 `-search` 后缀的变体（如 `gemini-3-pro-image-search`）。' includeThoughts: '是否获取模型的思考内容。' codeExecution: '为模型启用代码执行工具。' urlContext: '为模型启用 URL 内容获取工具。' - useCamelCaseSystemInstruction: 使用大写的 systemInstruction 而不是小写的 system_instruction - useCamelCaseMediaFields: 使用大写的 inlineData 和 mimeType，而不是小写的 inline_data 和 mime_type + useCamelCaseSystemInstruction: '使用 systemInstruction 等驼峰字段名，兼容部分第三方接口。' + useCamelCaseMediaFields: '媒体请求使用 inlineData、mimeType、mediaProcessing 等驼峰字段名，兼容部分第三方接口。' nonStreaming: 强制不启用流式返回。开启后，将总是以非流式发起请求，即便配置了 stream 参数。 diff --git a/packages/adapter-gemini/src/requester.ts b/packages/adapter-gemini/src/requester.ts index 1546496a3..fa9788929 100644 --- a/packages/adapter-gemini/src/requester.ts +++ b/packages/adapter-gemini/src/requester.ts @@ -26,22 +26,17 @@ import { readableStreamToAsyncIterable } from 'koishi-plugin-chatluna/utils/stre import * as fetchType from 'undici/types/fetch' import { Config, logger } from '.' import { - ChatFunctionCallingPart, - ChatInlineDataPart, - ChatMessagePart, ChatPart, ChatResponse, - ChatUsageMetadataPart, + ChatThoughtData, CreateEmbeddingResponse, GeminiModelInfo } from './types' import { createChatGenerationParams, getUsage, - isChatResponse, isMediaProcessingPart, - partAsType, - partAsTypeCheck, + isToolContext, prepareModelConfig } from './utils' import { ChatLunaPlugin } from 'koishi-plugin-chatluna/services/chat' @@ -89,8 +84,7 @@ export class GeminiRequester yield new ChatGenerationChunk({ generationInfo: generation.generationInfo, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - message: generation.message as any as BaseMessageChunk, + message: generation.message as BaseMessageChunk, text: generation.text }) @@ -447,7 +441,7 @@ export class GeminiRequester const readableStream = new ReadableStream({ async start(controller) { - if (isChatResponse(response)) { + if ('candidates' in response) { controller.enqueue(response) controller.close() return @@ -546,40 +540,32 @@ export class GeminiRequester } private async *_processChunks(iterable: AsyncIterable) { - let reasoningContent = '' - - let errorCount = 0 - - let functionIndex = 0 + let reasoning = '' + let errors = 0 + let index = 0 for await (const chunk of iterable) { if (isMediaProcessingPart(chunk)) continue - let parsedChunk: ChatUsageMetadataPart | undefined - if ( - (parsedChunk = partAsTypeCheck( - chunk, - (chunk) => chunk['usage'] != null - )) - ) { - const usageMetadata = createUsageMetadata({ - inputTokens: parsedChunk.usage.promptTokens, - outputTokens: parsedChunk.usage.completionTokens, - totalTokens: parsedChunk.usage.totalTokens, - inputImageTokens: parsedChunk.usage.inputImageTokens, - outputImageTokens: parsedChunk.usage.outputImageTokens, - inputAudioTokens: parsedChunk.usage.inputAudioTokens, - outputAudioTokens: parsedChunk.usage.outputAudioTokens, - cacheReadTokens: parsedChunk.usage.cacheReadTokens, - reasoningTokens: parsedChunk.usage.reasoningTokens - }) - + if ('usage' in chunk) { yield { type: 'generation', generation: new ChatGenerationChunk({ message: new AIMessageChunk({ content: '', - usage_metadata: usageMetadata + usage_metadata: createUsageMetadata({ + inputTokens: chunk.usage.promptTokens, + outputTokens: chunk.usage.completionTokens, + totalTokens: chunk.usage.totalTokens, + inputImageTokens: chunk.usage.inputImageTokens, + outputImageTokens: + chunk.usage.outputImageTokens, + inputAudioTokens: chunk.usage.inputAudioTokens, + outputAudioTokens: + chunk.usage.outputAudioTokens, + cacheReadTokens: chunk.usage.cacheReadTokens, + reasoningTokens: chunk.usage.reasoningTokens + }) }), text: '' }) @@ -588,150 +574,105 @@ export class GeminiRequester } try { - const part = Object.assign({}, chunk) - const { updatedContent, updatedReasoning, updatedToolCalling } = - await this._processChunk( - part, - reasoningContent, - functionIndex - ) + let content: MessageContent + if ('text' in chunk && chunk.text) { + if (chunk.thought) { + reasoning += chunk.text + yield { type: 'reasoning', content: reasoning } + continue + } + content = chunk.text + } else if ('inlineData' in chunk && !chunk.thought) { + const image = chunk.inlineData + const storage = this.ctx.chatluna_storage + if (storage == null) { + content = `![image](data:${image.mimeType ?? 'image/png'};base64,${image.data})` + } else { + const hash = await hashString(image.data, 8) + const type = (image.mimeType ?? 'image/png').split( + '/' + )[1] + const file = await storage.createTempFile( + Buffer.from(image.data, 'base64'), + `${hash}.${type}` + ) + content = [{ type: 'image_url', image_url: file.url }] + } + } - if (updatedReasoning !== reasoningContent) { - reasoningContent = updatedReasoning - yield { type: 'reasoning', content: reasoningContent } - continue + const fn = + 'functionCall' in chunk ? chunk.functionCall : undefined + let call: ToolCallChunk + if (fn) { + const fresh = fn.name?.length > 0 + call = { + name: fresh ? fn.name : undefined, + args: + Object.keys(fn.args ?? {}).length > 0 + ? JSON.stringify(fn.args) + : undefined, + id: fresh + ? (fn.id ?? `function_call_${index}`) + : undefined, + index: fresh ? index : index - 1 + } } - if ( - updatedContent || - updatedToolCalling || - chunk['thoughtSignature'] != null || - chunk['toolCall'] != null || - chunk['toolResponse'] != null || - chunk['executableCode'] != null || - chunk['codeExecutionResult'] != null - ) { - const messageChunk = this._createMessageChunk( - updatedContent, - updatedToolCalling, - chunk - ) + const sig = chunk.thoughtSignature + let thought: ChatThoughtData | undefined + if (isToolContext(chunk)) { + thought = { parts: [chunk] } + } else if (sig != null) { + const id = call?.id ?? fn?.id + thought = + id != null + ? { [id]: { thoughtSignature: sig } } + : { parts: [{ thoughtSignature: sig }] } + } - const generationChunk = new ChatGenerationChunk({ - message: messageChunk, - text: getMessageContent(messageChunk.content) ?? '' + if (content || call || thought) { + const image = + this.ctx.chatluna_storage == null && + 'inlineData' in chunk + ? chunk.inlineData + : undefined + const msg = new AIMessageChunk({ + content: content ?? '', + tool_call_chunks: call ? [call] : [], + additional_kwargs: { + images: image + ? [ + `data:${image.mimeType ?? 'image/png'};base64,${image.data}` + ] + : undefined, + thought_data: thought + } }) - - yield { type: 'generation', generation: generationChunk } + yield { + type: 'generation', + generation: new ChatGenerationChunk({ + message: msg, + text: getMessageContent(msg.content) ?? '' + }) + } } - if (updatedToolCalling) { - const fc = chunk['functionCall'] - if (fc?.name && fc.name.length > 0) { - functionIndex++ - } + if (call && fn.name?.length > 0) { + index++ } - } catch (e) { - if (errorCount > 5) { + } catch (err) { + if (errors > 5) { logger.error('error with chunk', chunk) throw new ChatLunaError( ChatLunaErrorCode.API_REQUEST_FAILED, - e + err ) - } else { - errorCount++ - continue } + errors++ } } } - private async _processChunk( - chunk: ChatPart, - reasoningContent: string, - functionIndex: number - ) { - const messagePart = partAsType(chunk) - const chatFunctionCallingPart = - partAsType(chunk) - const imagePart = partAsTypeCheck( - chunk, - (part) => part['inlineData'] != null - ) - - let messageContent: MessageContent - - if (messagePart.text) { - if (messagePart.thought) { - return { - updatedContent: messageContent, - updatedReasoning: reasoningContent + messagePart.text - } - } - messageContent = messagePart.text - } else if (imagePart && !messagePart.thought) { - const storageService = this.ctx.chatluna_storage - if (!storageService) { - messagePart.text = `![image](data:${imagePart.inlineData.mimeType ?? 'image/png'};base64,${imagePart.inlineData.data})` - messageContent = messagePart.text - } else { - const buffer = Buffer.from(imagePart.inlineData.data, 'base64') - - const hash = await hashString(imagePart.inlineData.data, 8) - const type = ( - imagePart.inlineData.mimeType ?? 'image/png' - ).split('/')[1] - const file = await storageService.createTempFile( - buffer, - `${hash}.${type}` - ) - - messagePart.text = `[image:${file.url}]` - messageContent = [ - { - type: 'image_url', - image_url: file.url - } - ] - } - } - - const deltaFunctionCall = chatFunctionCallingPart?.functionCall - let updatedToolCalling: ToolCallChunk - if (deltaFunctionCall) { - const isNew = deltaFunctionCall.name?.length > 0 - updatedToolCalling = this._createToolCallChunk( - deltaFunctionCall, - isNew ? functionIndex : functionIndex - 1 - ) - } - - return { - updatedContent: messageContent, - updatedReasoning: reasoningContent, - updatedToolCalling - } - } - - private _createToolCallChunk( - deltaFunctionCall: ChatFunctionCallingPart['functionCall'], - index: number - ) { - const isNew = deltaFunctionCall.name?.length > 0 - const args = - Object.keys(deltaFunctionCall.args ?? {}).length > 0 - ? JSON.stringify(deltaFunctionCall.args) - : undefined - return { - name: isNew ? deltaFunctionCall.name : undefined, - args, - id: isNew - ? (deltaFunctionCall.id ?? `function_call_${index}`) - : undefined, - index - } satisfies ToolCallChunk - } - private _handleFinalContent( reasoningState: ReasoningState, groundingContent: string @@ -757,58 +698,11 @@ export class GeminiRequester } } - private _createMessageChunk( - content: MessageContent, - functionCall: ToolCallChunk | undefined, - chunk: ChatPart + private _post( + url: string, + data: Record, + params: fetchType.RequestInit = {} ) { - const imagePart = - this.ctx.chatluna_storage != null - ? undefined - : partAsTypeCheck( - chunk, - (part) => part['inlineData'] != null - ) - const messageChunk = new AIMessageChunk({ - content: content ?? '', - tool_call_chunks: [functionCall].filter(Boolean) - }) - const sig = chunk['thoughtSignature'] - let thoughtData: Record | undefined - if ( - chunk['toolCall'] != null || - chunk['toolResponse'] != null || - chunk['executableCode'] != null || - chunk['codeExecutionResult'] != null - ) { - thoughtData = { parts: [chunk] } - } else if (sig != null) { - const id = functionCall?.id ?? chunk['functionCall']?.id - if (id != null) { - thoughtData = { - [id]: { - thoughtSignature: sig - } - } - } else { - thoughtData = { parts: [{ thoughtSignature: sig }] } - } - } - - messageChunk.additional_kwargs = { - images: imagePart - ? [ - `data:${imagePart.inlineData.mimeType ?? 'image/png'};base64,${imagePart.inlineData.data}` - ] - : undefined, - thought_data: thoughtData - } - - return messageChunk - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private _post(url: string, data: any, params: fetchType.RequestInit = {}) { const requestUrl = this._concatUrl(url) for (const key in data) { diff --git a/packages/adapter-gemini/src/types.ts b/packages/adapter-gemini/src/types.ts index 10a3e3d9e..1a5cb9251 100644 --- a/packages/adapter-gemini/src/types.ts +++ b/packages/adapter-gemini/src/types.ts @@ -5,21 +5,23 @@ export interface ChatCompletionResponseMessage { export type BaseChatPart = { thoughtSignature?: string + thought?: boolean } -export type ChatPart = - | (ChatMessagePart & BaseChatPart) - | ChatInlineDataPart - | (ChatFunctionCallingPart & BaseChatPart) - | ChatFunctionResponsePart - | (ChatToolContextPart & BaseChatPart) - | ChatUploadDataPart - // Only used for token - | ChatUsageMetadataPart +export type ChatPart = BaseChatPart & + ( + | ChatMessagePart + | (ChatInlineDataPart & { mediaProcessing?: 'AGENTIC' }) + | ChatFunctionCallingPart + | ChatFunctionResponsePart + | ChatToolContextPart + | (ChatUploadDataPart & { media_processing?: 'AGENTIC' }) + // Only used for token + | ChatUsageMetadataPart + ) export type ChatMessagePart = { text: string - thought?: boolean } export type ChatUsageMetadataPart = { @@ -60,7 +62,6 @@ export type ChatInlineDataPart = { displayName?: string data?: string } - mediaProcessing?: 'AGENTIC' } export type ChatUploadDataPart = { @@ -68,14 +69,12 @@ export type ChatUploadDataPart = { mime_type: string data?: string } - media_processing?: 'AGENTIC' } export type ChatFunctionCallingPart = { functionCall: { name: string - // eslint-disable-next-line @typescript-eslint/no-explicit-any - args?: any + args?: Record id?: string } } @@ -84,26 +83,35 @@ export type ChatFunctionResponsePart = { functionResponse: { name: string response: Record - parts?: (( - | Pick - | Pick - ) & { - mediaProcessing?: never - media_processing?: never - })[] + parts?: (ChatInlineDataPart | ChatUploadDataPart)[] id?: string } } +export type ChatToolCall = { + id?: string + toolType?: string + [key: string]: unknown +} + export type ChatToolContextPart = { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - toolCall?: Record - // eslint-disable-next-line @typescript-eslint/no-explicit-any - toolResponse?: Record - // eslint-disable-next-line @typescript-eslint/no-explicit-any - executableCode?: Record - // eslint-disable-next-line @typescript-eslint/no-explicit-any - codeExecutionResult?: Record + toolCall?: ChatToolCall + toolResponse?: ChatToolCall + executableCode?: Record + codeExecutionResult?: Record +} + +export type ChatThoughtPart = BaseChatPart & ChatToolContextPart + +// Current histories use parts and call IDs; older ones also store a part directly. +export type ChatThoughtData = ChatThoughtPart & { + parts?: ChatThoughtPart[] + [key: string]: + | string + | boolean + | ChatThoughtPart + | ChatThoughtPart[] + | Record } export interface ChatResponse { @@ -149,14 +157,24 @@ export interface ChatResponse { export interface ChatCompletionFunction { name: string description?: string - // eslint-disable-next-line @typescript-eslint/no-explicit-any - parameters?: { [key: string]: any } + parameters?: Record +} + +export interface ChatTool { + functionDeclarations?: ChatCompletionFunction[] + google_search?: { + searchTypes?: { + webSearch: Record + imageSearch: Record + } + } + code_execution?: Record + urlContext?: Record } export interface ChatCompletionMessageFunctionCall { name: string - // eslint-disable-next-line @typescript-eslint/no-explicit-any - args?: any + args?: Record } export interface CreateEmbeddingResponse { diff --git a/packages/adapter-gemini/src/utils.ts b/packages/adapter-gemini/src/utils.ts index 597aca320..e24712416 100644 --- a/packages/adapter-gemini/src/utils.ts +++ b/packages/adapter-gemini/src/utils.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ import { AIMessage, BaseMessage, @@ -12,10 +11,12 @@ import { ChatCompletionFunction, ChatCompletionResponseMessage, ChatCompletionResponseMessageRoleEnum, - ChatFunctionCallingPart, ChatFunctionResponsePart, ChatPart, ChatResponse, + ChatThoughtData, + ChatThoughtPart, + ChatTool, GeminiUsageMetadata } from './types' import { Config, logger } from '.' @@ -41,88 +42,192 @@ export async function langchainMessageToGeminiMessage( plugin: ChatLunaPlugin, model?: string ): Promise { - const match = model?.match( - /gemini[-_\s]*(\d+)(?:[._](\d+))?[-_\s]*flash(?:[-_\s]*(lite))?/i - ) - // Accept provider prefixes/suffixes and future Flash versions. + const cfg = plugin.config const agentic = - plugin.config.agenticVideoUnderstanding && - match != null && - (Number(match[1]) > 3 || - (Number(match[1]) === 3 && - (Number(match[2]) >= 6 || - (Number(match[2]) === 5 && match[3] != null)))) + cfg.agenticVideo && + AGENTIC_VIDEO_MODELS.some((name) => model?.includes(name)) + + async function convert( + content: MessageContentComplex[] + ): Promise { + const parts = await Promise.all( + content.map(async (part): Promise => { + if (isMessageContentText(part)) { + return part.text.length > 0 ? { text: part.text } : null + } + const image = isMessageContentImageUrl(part) + if (!image && !isGeminiFileLikeContent(part)) { + return part as unknown as ChatPart + } + + const media = await readMedia(plugin, part) + if (media == null) return null + const mode = + agentic && !image && media.mimeType.startsWith('video/') + ? 'AGENTIC' + : undefined + return cfg.useCamelCaseMediaFields + ? { inlineData: media, mediaProcessing: mode } + : { + inline_data: { + data: media.data, + mime_type: media.mimeType + }, + media_processing: mode + } + }) + ) + return parts.filter((part) => part != null) + } + const result: ChatCompletionResponseMessage[] = [] for (let i = 0; i < messages.length; i++) { - const message = messages[i] - const role = messageTypeToGeminiRole(message.getType()) - const hasFunctionCall = - (message as AIMessage).tool_calls != null && - (message as AIMessage).tool_calls.length > 0 + const msg = messages[i] + const role = messageTypeToGeminiRole(msg.getType()) if (role === 'function') { - const parts: ChatPart[] = [] - let j = i - while (j < messages.length) { - const msg = messages[j] - if (messageTypeToGeminiRole(msg.getType()) !== 'function') break - parts.push( - ...( - await processFunctionMessage( - plugin, - msg, - plugin.config.useCamelCaseSystemInstruction, - agentic - ) - ).parts + const response: ChatFunctionResponsePart['functionResponse'] = { + name: msg.name, + id: (msg as ToolMessage).tool_call_id || undefined, + response: {} + } + const parts: ChatPart[] = [{ functionResponse: response }] + if (typeof msg.content === 'string') { + response.response = parseJsonArgs(msg.content) + } else { + const texts = msg.content.filter(isMessageContentText) + if (texts.length > 0) { + response.response = parseJsonArgs( + texts.map((part) => part.text).join('') + ) + } + const media = await convert( + msg.content.filter( + (part) => + isMessageContentImageUrl(part) || + isGeminiFileLikeContent(part) + ) ) - j++ + for (const part of media) { + // Gemini requires agentic videos beside the function response. + if ( + ('inlineData' in part && part.mediaProcessing) || + ('inline_data' in part && part.media_processing) + ) { + parts.push(part) + continue + } + if ('inlineData' in part) { + response.parts ??= [] + response.parts.push({ inlineData: part.inlineData }) + } else if ('inline_data' in part) { + response.parts ??= [] + response.parts.push({ inline_data: part.inline_data }) + } + } + } + + // Consecutive tool results form one user turn. + if (i > 0 && messages[i - 1].getType() === 'tool') { + result[result.length - 1].parts.push(...parts) + } else { + result.push({ role: 'user', parts }) } - i = j - 1 - result.push({ role: 'user', parts }) continue } - if (hasFunctionCall) { - result.push( - await processFunctionMessage( - plugin, - message, - plugin.config.useCamelCaseSystemInstruction, - agentic - ) - ) + if ((msg as AIMessage).tool_calls?.length > 0) { + result.push({ + role: 'model', + parts: convertCalls(msg as AIMessage) + }) continue } - const item: ChatCompletionResponseMessage = { role, parts: [] } - const thoughtData: Record = - message.additional_kwargs['thought_data'] ?? {} + const data = (msg.additional_kwargs.thought_data ?? + {}) as ChatThoughtData + const parts = - typeof message.content === 'string' - ? message.content.length > 0 - ? [{ text: message.content }] + typeof msg.content === 'string' + ? msg.content.length > 0 + ? [{ text: msg.content }] : [] - : await processGeminiContentParts( - plugin, - message.content, - agentic - ) - - item.parts = [...getContextParts(thoughtData), ...parts] + : await convert(msg.content) + result.push({ + role, + parts: [ + ...getContextParts([data, ...Object.values(data)]), + ...parts + ] + }) - if (message.additional_kwargs.images != null) { + if (msg.additional_kwargs.images != null) { logger.warn( 'Deprecated: `additional_kwargs.images` is no longer supported. Use `image_url` content parts instead.' ) } - - result.push(item) } return result } +async function readMedia( + plugin: ChatLunaPlugin, + part: MessageContentImageUrl | Parameters[1] +) { + try { + if (isMessageContentImageUrl(part)) { + const url = await fetchImageUrl(plugin, part) + return { + data: url.replace(/^data:image\/\w+;base64,/, ''), + mimeType: + url.match(/^data:([^;]+);base64,/)?.[1] ?? 'image/jpeg' + } + } + const file = await fetchFileLikeUrl(plugin, part) + return { data: file.buffer.toString('base64'), mimeType: file.mimeType } + } catch (err) { + logger.warn(`Failed to fetch ${part.type}`, err) + return null + } +} + +function convertCalls(msg: AIMessage): ChatPart[] { + const calls = msg.tool_calls + const data = (msg.additional_kwargs.thought_data ?? {}) as ChatThoughtData + // Replay shared context once; call-specific context stays with its call. + const shared = { ...data } + for (const call of calls) { + if (call.id != null) delete shared[call.id] + } + const parts: ChatPart[] = getContextParts([ + shared, + ...Object.values(shared) + ]) + for (const call of calls) { + if (call.id != null) { + parts.push(...getContextParts([data[call.id]])) + } + const context = data[call.id] ?? data + const sig = Array.isArray(context) + ? context.find((part) => typeof part?.thoughtSignature === 'string') + ?.thoughtSignature + : typeof context === 'object' + ? context.thoughtSignature + : undefined + parts.push({ + functionCall: { + name: call.name, + args: call.args, + id: call.id || undefined + }, + ...(typeof sig === 'string' ? { thoughtSignature: sig } : {}) + }) + } + + return parts +} + export function extractSystemMessages( messages: ChatCompletionResponseMessage[] ): [ChatCompletionResponseMessage, ChatCompletionResponseMessage[]] { @@ -181,8 +286,8 @@ function parseJsonArgs(args: string): Record { } } -export function isMediaProcessingPart(part: ChatPart): boolean { - const tool = part['toolCall'] ?? part['toolResponse'] +export function isMediaProcessingPart(part: ChatThoughtPart): boolean { + const tool = part.toolCall ?? part.toolResponse // Agentic media steps can omit toolType and cannot be replayed by Gemini. return ( tool != null && @@ -190,176 +295,30 @@ export function isMediaProcessingPart(part: ChatPart): boolean { ) } -function isContextPart(part: any): part is ChatPart { +export function isToolContext(part: unknown): part is ChatThoughtPart { return ( typeof part === 'object' && part != null && - !isMediaProcessingPart(part) && - (part['toolCall'] != null || - part['toolResponse'] != null || - part['executableCode'] != null || - part['codeExecutionResult'] != null) + (('toolCall' in part && part.toolCall != null) || + ('toolResponse' in part && part.toolResponse != null) || + ('executableCode' in part && part.executableCode != null) || + ('codeExecutionResult' in part && part.codeExecutionResult != null)) ) } -function getContextParts(data: Record, id?: string) { - const raw = id != null ? data[id] : [data, ...Object.values(data)] - if (raw == null) return [] - - return (Array.isArray(raw) ? raw.flat() : [raw]).filter(isContextPart) -} - -function processFunctionCalls( - message: AIMessage, - removeId: boolean -): ChatCompletionResponseMessage { - const thoughtData: Record = - message.additional_kwargs['thought_data'] ?? {} - const parts: ChatPart[] = getContextParts( - Object.fromEntries( - Object.entries(thoughtData).filter( - ([id]) => !message.tool_calls.some((call) => call.id === id) - ) - ) - ) - - for (const toolCall of message.tool_calls) { - if (toolCall.id != null) { - parts.push(...getContextParts(thoughtData, toolCall.id)) - } - - const functionCall: ChatFunctionCallingPart['functionCall'] = { - name: toolCall.name, - args: toolCall.args - } - if (!removeId || toolCall.id) { - functionCall.id = toolCall.id - } - const data = thoughtData[toolCall.id] ?? thoughtData - const sig = Array.isArray(data) - ? data.find((item) => typeof item?.thoughtSignature === 'string') - ?.thoughtSignature - : data.thoughtSignature - - parts.push({ - functionCall, - ...(typeof sig === 'string' ? { thoughtSignature: sig } : {}) - }) - } - - return { role: 'model', parts } -} - -async function processFunctionMessage( - plugin: ChatLunaPlugin, - message: AIMessage | ToolMessage, - removeId: boolean, - agentic: boolean -): Promise { - if (message['tool_calls']) { - return processFunctionCalls(message as AIMessage, removeId) - } - - const finalMessage = message as ToolMessage - const media: ChatPart[] = [] - - const functionResponse: ChatFunctionResponsePart['functionResponse'] = { - name: message.name, - response: {} - } - - if (Array.isArray(message.content)) { - const texts = message.content.flatMap((part) => { - if (isMessageContentText(part)) return [part.text] - return [] - }) - - if (texts.length > 0) { - functionResponse.response = parseJsonArgs(texts.join('')) - } - - const parts = await processGeminiContentParts( - plugin, - message.content.filter( - (part) => - isMessageContentImageUrl(part) || - isGeminiFileLikeContent(part) - ), - agentic +function getContextParts(parts: ChatThoughtData[string][]): ChatThoughtPart[] { + // Old histories also store context directly or under individual call IDs. + return parts + .flat() + .filter( + (part): part is ChatThoughtPart => + isToolContext(part) && !isMediaProcessingPart(part) ) - for (const part of parts) { - if (part['mediaProcessing'] || part['media_processing']) { - media.push(part) - } else if ('inlineData' in part) { - ;(functionResponse.parts ??= []).push({ - inlineData: part.inlineData - }) - } else if ('inline_data' in part) { - ;(functionResponse.parts ??= []).push({ - inline_data: part.inline_data - }) - } - } - } else { - functionResponse.response = parseJsonArgs(message.content as string) - } - - if (!removeId || finalMessage.tool_call_id) { - functionResponse.id = finalMessage.tool_call_id - } - - return { - role: 'user', - parts: [ - { - functionResponse - }, - ...media - ] - } } -async function processGeminiImageContent( - plugin: ChatLunaPlugin, - part: MessageContentImageUrl -) { - let url: string - try { - url = await fetchImageUrl(plugin, part) - } catch (e) { - const rawUrl = - typeof part.image_url === 'string' - ? part.image_url - : part.image_url.url - logger.warn(`Failed to fetch image url: ${rawUrl}`, e) - return null - } - - const mineType = url.match(/^data:([^;]+);base64,/)?.[1] ?? 'image/jpeg' - const data = url.replace(/^data:image\/\w+;base64,/, '') - - return createGeminiInlineDataPart(plugin, data, mineType) -} - -type GeminiFileLikeContent = MessageContentComplex & - ( - | { - type: 'file_url' - file_url: string | { url: string; mimeType?: string } - } - | { - type: 'audio_url' - audio_url: string | { url: string; mimeType?: string } - } - | { - type: 'video_url' - video_url: string | { url: string; mimeType?: string } - } - ) - function isGeminiFileLikeContent( part: MessageContentComplex -): part is GeminiFileLikeContent { +): part is Parameters[1] { return ( part != null && typeof part === 'object' && @@ -367,84 +326,19 @@ function isGeminiFileLikeContent( ) } -function createGeminiInlineDataPart( - plugin: ChatLunaPlugin, - data: string, - mimeType: string, - agentic = false -) { - if (plugin.config.useCamelCaseMediaFields) { - return { - inlineData: { data, mimeType }, - mediaProcessing: agentic ? ('AGENTIC' as const) : undefined - } - } - - return { - inline_data: { data, mime_type: mimeType }, - media_processing: agentic ? ('AGENTIC' as const) : undefined - } -} - -async function processGeminiFileLikeContent( - plugin: ChatLunaPlugin, - part: GeminiFileLikeContent, - agentic: boolean -) { - try { - const { buffer, mimeType } = await fetchFileLikeUrl(plugin, part) - return createGeminiInlineDataPart( - plugin, - buffer.toString('base64'), - mimeType, - agentic && mimeType.startsWith('video/') - ) - } catch (e) { - logger.warn(`Failed to fetch ${part.type}`, e) - return null - } -} - -async function processGeminiContentParts( - plugin: ChatLunaPlugin, - content: MessageContentComplex[], - agentic: boolean -): Promise { - const mappedParts = await Promise.all( - content.map(async (part) => { - if (isMessageContentText(part)) { - return part.text.length > 0 ? { text: part.text } : null - } - if (isMessageContentImageUrl(part)) { - return await processGeminiImageContent(plugin, part) - } - if (isGeminiFileLikeContent(part)) { - return await processGeminiFileLikeContent(plugin, part, agentic) - } - return part as unknown as ChatPart - }) - ) - - return mappedParts.filter((part) => part != null) -} - -export function partAsType(part: ChatPart): T { - return part as T -} - -export function partAsTypeCheck( - part: ChatPart, - check: (part: ChatPart & unknown) => boolean -): T | undefined { - return check(part) ? (part as T) : undefined -} - // 不支持 googleSearch / codeExecution / urlContext 的模型列表 const CUSTOM_TOOLS_UNSUPPORTED_MODELS = [ 'gemini-2.0-flash-lite', 'gemini-2.0-flash-exp' ] +const AGENTIC_VIDEO_MODELS = [ + 'gemini-3.5-flash-lite', + 'gemini-3.6-flash', + 'gemini-3.7-flash', + 'gemini-3.8-flash' +] + // 启用 imageGeneration 时同样不支持上述自定义工具的模型列表 const IMAGE_GENERATION_MODELS = [ 'gemini-2.0-flash-exp', @@ -490,7 +384,7 @@ function isImageSearchSupported(model: string): boolean { * - 其余模型使用标准的新版 google_search: {} 格式 */ function appendBuiltinTools( - result: Record[], + result: ChatTool[], googleSearch: boolean, codeExecution: boolean, urlContext: boolean, @@ -524,7 +418,7 @@ export function formatToolsToGeminiAITools( tools: StructuredTool[], config: Config, model: string -): Record { +): ChatTool[] | undefined { // 没有任何工具需要注册时直接返回 if ( tools.length < 1 && @@ -536,7 +430,7 @@ export function formatToolsToGeminiAITools( } const functions = tools.map(formatToolToGeminiAITool) - const result: Record[] = [] + const result: ChatTool[] = [] // --- 处理内置工具（googleSearch / codeExecution / urlContext）--- let { googleSearch, codeExecution, urlContext } = config @@ -589,8 +483,7 @@ export function formatToolToGeminiAITool( return { name: tool.name, description: tool.description, - // any? - parameters + parameters: parameters as ChatCompletionFunction['parameters'] } } @@ -621,15 +514,15 @@ const GEMINI_SCHEMA_KEYS = new Set([ 'anyOf' ]) -function sanitizeGeminiSchema(schema: any): any { +function sanitizeGeminiSchema(schema: unknown): unknown { if (!schema || typeof schema !== 'object' || Array.isArray(schema)) { return schema } - const result: Record = {} + const result: Record = {} for (const [key, value] of Object.entries(schema)) { if (key === 'oneOf' || key === 'anyOf') { - result['anyOf'] = (value as any[]).map(sanitizeGeminiSchema) + result['anyOf'] = (value as unknown[]).map(sanitizeGeminiSchema) continue } if (!GEMINI_SCHEMA_KEYS.has(key)) continue @@ -639,10 +532,9 @@ function sanitizeGeminiSchema(schema: any): any { } if (key === 'properties') { result['properties'] = Object.fromEntries( - Object.entries(value).map(([name, sub]) => [ - name, - sanitizeGeminiSchema(sub) - ]) + Object.entries(value as Record).map( + ([name, sub]) => [name, sanitizeGeminiSchema(sub)] + ) ) continue } @@ -895,10 +787,6 @@ export async function createChatGenerationParams( } } -export function isChatResponse(response: any): response is ChatResponse { - return 'candidates' in response -} - // #region refreshModels helpers export function isGeminiModelName(model: string): boolean { From e4f9215fb94d28ae5a95f3e6000c8ad4e7323798 Mon Sep 17 00:00:00 2001 From: dingyi Date: Sat, 12 Sep 2026 19:28:07 +0800 Subject: [PATCH 6/7] fix(gemini): address review feedback --- packages/adapter-gemini/src/requester.ts | 14 +------------- packages/adapter-gemini/src/utils.ts | 8 +++++++- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/adapter-gemini/src/requester.ts b/packages/adapter-gemini/src/requester.ts index fa9788929..50deb0594 100644 --- a/packages/adapter-gemini/src/requester.ts +++ b/packages/adapter-gemini/src/requester.ts @@ -631,22 +631,10 @@ export class GeminiRequester } if (content || call || thought) { - const image = - this.ctx.chatluna_storage == null && - 'inlineData' in chunk - ? chunk.inlineData - : undefined const msg = new AIMessageChunk({ content: content ?? '', tool_call_chunks: call ? [call] : [], - additional_kwargs: { - images: image - ? [ - `data:${image.mimeType ?? 'image/png'};base64,${image.data}` - ] - : undefined, - thought_data: thought - } + additional_kwargs: { thought_data: thought } }) yield { type: 'generation', diff --git a/packages/adapter-gemini/src/utils.ts b/packages/adapter-gemini/src/utils.ts index e24712416..ecc9dc444 100644 --- a/packages/adapter-gemini/src/utils.ts +++ b/packages/adapter-gemini/src/utils.ts @@ -137,9 +137,15 @@ export async function langchainMessageToGeminiMessage( } if ((msg as AIMessage).tool_calls?.length > 0) { + const text = + typeof msg.content === 'string' + ? msg.content.length > 0 + ? [{ text: msg.content }] + : [] + : await convert(msg.content) result.push({ role: 'model', - parts: convertCalls(msg as AIMessage) + parts: [...text, ...convertCalls(msg as AIMessage)] }) continue } From 287cccec6a7bf26ab36804066860de77c8f6cffd Mon Sep 17 00:00:00 2001 From: CookSleep Date: Sat, 12 Sep 2026 20:31:14 +0800 Subject: [PATCH 7/7] =?UTF-8?q?docs(gemini):=20=E6=9B=B4=E6=96=B0=20Agenti?= =?UTF-8?q?c=20=E8=A7=86=E9=A2=91=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/adapter-gemini/src/locales/zh-CN.schema.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/adapter-gemini/src/locales/zh-CN.schema.yml b/packages/adapter-gemini/src/locales/zh-CN.schema.yml index c03d325a4..9f2238a86 100644 --- a/packages/adapter-gemini/src/locales/zh-CN.schema.yml +++ b/packages/adapter-gemini/src/locales/zh-CN.schema.yml @@ -31,7 +31,7 @@ $inner: temperature: '回复的随机性程度，数值越高，回复越随机（范围：0~2）。' googleSearch: '为模型启用谷歌搜索。' imageGeneration: '为模型启用图像生成。目前仅支持 `gemini-*-image-*` 和 `gemini-2.5-flash-image-preview` 模型。' - agenticVideo: '启用按需视频分析。模型会挑选片段查看，减少长视频的输入费用，但可能忽略短视频的整体内容，回复也可能更慢。支持 Gemini 3.5 Flash Lite 和 Gemini 3.6、3.7、3.8 Flash。' + agenticVideo: '为视频输入启用 Agentic 视频理解功能。开启后，模型可以挑选音频/视频片段查看，或仅查看视频的音频转录文本，减少长视频的输入费用。在需要查看高速片段时，还能提高片段的采样帧率。此功能可能导致模型忽略视频的整体内容，并增加回复延迟。目前仅支持 Gemini 3.5 Flash Lite 和 Gemini 3.6、3.7、3.8 Flash。' thinkingBudget: '思考预算，范围：(-1~24576)，设置的数值越大，思考时花费的 Token 越多,-1 为动态思考。目前仅支持 gemini 2.5 系列模型。' groundingContentDisplay: '是否显示谷歌搜索结果。' imageModelSearch: '为图片生成模型启用搜索功能。开启后，支持搜索的图片模型将额外生成带 `-search` 后缀的变体（如 `gemini-3-pro-image-search`）。'