diff --git a/packages/adapter-gemini/src/index.ts b/packages/adapter-gemini/src/index.ts index f48ef5b97..42dae1f45 100644 --- a/packages/adapter-gemini/src/index.ts +++ b/packages/adapter-gemini/src/index.ts @@ -53,6 +53,7 @@ export interface Config extends ChatLunaPlugin.Config { codeExecution: boolean urlContext: boolean imageGeneration: boolean + agenticVideo: boolean imageModelSearch: boolean thinkingBudget: number includeThoughts: boolean @@ -123,6 +124,7 @@ export const Config: Schema = Schema.intersect([ imageModelSearch: Schema.boolean().default(false), groundingContentDisplay: Schema.boolean().default(false), imageGeneration: Schema.boolean().default(false), + agenticVideo: Schema.boolean().default(false), useCamelCaseSystemInstruction: Schema.boolean().default(false), useCamelCaseMediaFields: Schema.boolean().default(false), nonStreaming: Schema.boolean().default(false) @@ -130,8 +132,7 @@ export const Config: Schema = Schema.intersect([ ]).i18n({ 'zh-CN': require('./locales/zh-CN.schema.yml'), 'en-US': require('./locales/en-US.schema.yml') - // eslint-disable-next-line @typescript-eslint/no-explicit-any -}) as any +}) as Schema export const usage = ` ## Gemini 适配器说明 diff --git a/packages/adapter-gemini/src/locales/en-US.schema.yml b/packages/adapter-gemini/src/locales/en-US.schema.yml index 33ec53f29..8a1486dec 100644 --- a/packages/adapter-gemini/src/locales/en-US.schema.yml +++ b/packages/adapter-gemini/src/locales/en-US.schema.yml @@ -33,10 +33,11 @@ $inner: thinkingBudget: 'Thinking budget (-1-24576). (0: dynamic thinking) Higher: more tokens spent on thinking. Currently only supports `gemini-2.5` series models.' groundingContentDisplay: 'Enable display of search results' imageGeneration: 'Enable image generation (only for `gemini-*-image-*` and `gemini-2.5-flash-image-preview` model)' + agenticVideo: 'Enable on-demand video analysis. The model selects segments to examine, reducing long-video input costs. It may miss the overall context of short videos and respond more slowly. Supports Gemini 3.5 Flash Lite and Gemini 3.6, 3.7 and 3.8 Flash.' imageModelSearch: 'Enable search for image generation models. When enabled, supported image models will generate additional variants with a `-search` suffix (e.g. `gemini-3-pro-image-search`).' includeThoughts: 'Enable retrieval of model thoughts' codeExecution: 'Enable code execution tool' urlContext: 'Enable URL context retrieval tool' - useCamelCaseSystemInstruction: 'Use camelCase systemInstruction instead of snake_case system_instruction' - useCamelCaseMediaFields: 'Use camelCase inlineData and mimeType instead of snake_case inline_data and mime_type' + useCamelCaseSystemInstruction: 'Use camelCase field names such as systemInstruction for compatibility with third-party APIs.' + useCamelCaseMediaFields: 'Use camelCase media fields such as inlineData, mimeType and mediaProcessing for compatibility with third-party APIs.' nonStreaming: 'Force disable streaming response. When enabled, requests will always be made in non-streaming mode, even if the stream parameter is configured.' diff --git a/packages/adapter-gemini/src/locales/zh-CN.schema.yml b/packages/adapter-gemini/src/locales/zh-CN.schema.yml index 684b147aa..9f2238a86 100644 --- a/packages/adapter-gemini/src/locales/zh-CN.schema.yml +++ b/packages/adapter-gemini/src/locales/zh-CN.schema.yml @@ -31,12 +31,13 @@ $inner: temperature: '回复的随机性程度，数值越高，回复越随机（范围：0~2）。' googleSearch: '为模型启用谷歌搜索。' imageGeneration: '为模型启用图像生成。目前仅支持 `gemini-*-image-*` 和 `gemini-2.5-flash-image-preview` 模型。' + agenticVideo: '为视频输入启用 Agentic 视频理解功能。开启后，模型可以挑选音频/视频片段查看，或仅查看视频的音频转录文本，减少长视频的输入费用。在需要查看高速片段时，还能提高片段的采样帧率。此功能可能导致模型忽略视频的整体内容，并增加回复延迟。目前仅支持 Gemini 3.5 Flash Lite 和 Gemini 3.6、3.7、3.8 Flash。' thinkingBudget: '思考预算，范围：(-1~24576)，设置的数值越大，思考时花费的 Token 越多,-1 为动态思考。目前仅支持 gemini 2.5 系列模型。' groundingContentDisplay: '是否显示谷歌搜索结果。' imageModelSearch: '为图片生成模型启用搜索功能。开启后，支持搜索的图片模型将额外生成带 `-search` 后缀的变体（如 `gemini-3-pro-image-search`）。' includeThoughts: '是否获取模型的思考内容。' codeExecution: '为模型启用代码执行工具。' urlContext: '为模型启用 URL 内容获取工具。' - useCamelCaseSystemInstruction: 使用大写的 systemInstruction 而不是小写的 system_instruction - useCamelCaseMediaFields: 使用大写的 inlineData 和 mimeType，而不是小写的 inline_data 和 mime_type + useCamelCaseSystemInstruction: '使用 systemInstruction 等驼峰字段名，兼容部分第三方接口。' + useCamelCaseMediaFields: '媒体请求使用 inlineData、mimeType、mediaProcessing 等驼峰字段名，兼容部分第三方接口。' nonStreaming: 强制不启用流式返回。开启后，将总是以非流式发起请求，即便配置了 stream 参数。 diff --git a/packages/adapter-gemini/src/requester.ts b/packages/adapter-gemini/src/requester.ts index b370640de..50deb0594 100644 --- a/packages/adapter-gemini/src/requester.ts +++ b/packages/adapter-gemini/src/requester.ts @@ -26,21 +26,17 @@ import { readableStreamToAsyncIterable } from 'koishi-plugin-chatluna/utils/stre import * as fetchType from 'undici/types/fetch' import { Config, logger } from '.' import { - ChatFunctionCallingPart, - ChatInlineDataPart, - ChatMessagePart, ChatPart, ChatResponse, - ChatUsageMetadataPart, + ChatThoughtData, CreateEmbeddingResponse, GeminiModelInfo } from './types' import { createChatGenerationParams, getUsage, - isChatResponse, - partAsType, - partAsTypeCheck, + isMediaProcessingPart, + isToolContext, prepareModelConfig } from './utils' import { ChatLunaPlugin } from 'koishi-plugin-chatluna/services/chat' @@ -88,8 +84,7 @@ export class GeminiRequester yield new ChatGenerationChunk({ generationInfo: generation.generationInfo, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - message: generation.message as any as BaseMessageChunk, + message: generation.message as BaseMessageChunk, text: generation.text }) @@ -446,7 +441,7 @@ export class GeminiRequester const readableStream = new ReadableStream({ async start(controller) { - if (isChatResponse(response)) { + if ('candidates' in response) { controller.enqueue(response) controller.close() return @@ -545,38 +540,32 @@ export class GeminiRequester } private async *_processChunks(iterable: AsyncIterable) { - let reasoningContent = '' - - let errorCount = 0 - - let functionIndex = 0 + let reasoning = '' + let errors = 0 + let index = 0 for await (const chunk of iterable) { - let parsedChunk: ChatUsageMetadataPart | undefined - if ( - (parsedChunk = partAsTypeCheck( - chunk, - (chunk) => chunk['usage'] != null - )) - ) { - const usageMetadata = createUsageMetadata({ - inputTokens: parsedChunk.usage.promptTokens, - outputTokens: parsedChunk.usage.completionTokens, - totalTokens: parsedChunk.usage.totalTokens, - inputImageTokens: parsedChunk.usage.inputImageTokens, - outputImageTokens: parsedChunk.usage.outputImageTokens, - inputAudioTokens: parsedChunk.usage.inputAudioTokens, - outputAudioTokens: parsedChunk.usage.outputAudioTokens, - cacheReadTokens: parsedChunk.usage.cacheReadTokens, - reasoningTokens: parsedChunk.usage.reasoningTokens - }) + if (isMediaProcessingPart(chunk)) continue + if ('usage' in chunk) { yield { type: 'generation', generation: new ChatGenerationChunk({ message: new AIMessageChunk({ content: '', - usage_metadata: usageMetadata + usage_metadata: createUsageMetadata({ + inputTokens: chunk.usage.promptTokens, + outputTokens: chunk.usage.completionTokens, + totalTokens: chunk.usage.totalTokens, + inputImageTokens: chunk.usage.inputImageTokens, + outputImageTokens: + chunk.usage.outputImageTokens, + inputAudioTokens: chunk.usage.inputAudioTokens, + outputAudioTokens: + chunk.usage.outputAudioTokens, + cacheReadTokens: chunk.usage.cacheReadTokens, + reasoningTokens: chunk.usage.reasoningTokens + }) }), text: '' }) @@ -585,146 +574,93 @@ export class GeminiRequester } try { - const part = Object.assign({}, chunk) - const { updatedContent, updatedReasoning, updatedToolCalling } = - await this._processChunk( - part, - reasoningContent, - functionIndex - ) + let content: MessageContent + if ('text' in chunk && chunk.text) { + if (chunk.thought) { + reasoning += chunk.text + yield { type: 'reasoning', content: reasoning } + continue + } + content = chunk.text + } else if ('inlineData' in chunk && !chunk.thought) { + const image = chunk.inlineData + const storage = this.ctx.chatluna_storage + if (storage == null) { + content = `![image](data:${image.mimeType ?? 'image/png'};base64,${image.data})` + } else { + const hash = await hashString(image.data, 8) + const type = (image.mimeType ?? 'image/png').split( + '/' + )[1] + const file = await storage.createTempFile( + Buffer.from(image.data, 'base64'), + `${hash}.${type}` + ) + content = [{ type: 'image_url', image_url: file.url }] + } + } - if (updatedReasoning !== reasoningContent) { - reasoningContent = updatedReasoning - yield { type: 'reasoning', content: reasoningContent } - continue + const fn = + 'functionCall' in chunk ? chunk.functionCall : undefined + let call: ToolCallChunk + if (fn) { + const fresh = fn.name?.length > 0 + call = { + name: fresh ? fn.name : undefined, + args: + Object.keys(fn.args ?? {}).length > 0 + ? JSON.stringify(fn.args) + : undefined, + id: fresh + ? (fn.id ?? `function_call_${index}`) + : undefined, + index: fresh ? index : index - 1 + } } - if ( - updatedContent || - updatedToolCalling || - chunk['thoughtSignature'] != null - ) { - const messageChunk = this._createMessageChunk( - updatedContent, - updatedToolCalling, - chunk - ) + const sig = chunk.thoughtSignature + let thought: ChatThoughtData | undefined + if (isToolContext(chunk)) { + thought = { parts: [chunk] } + } else if (sig != null) { + const id = call?.id ?? fn?.id + thought = + id != null + ? { [id]: { thoughtSignature: sig } } + : { parts: [{ thoughtSignature: sig }] } + } - const generationChunk = new ChatGenerationChunk({ - message: messageChunk, - text: getMessageContent(messageChunk.content) ?? '' + if (content || call || thought) { + const msg = new AIMessageChunk({ + content: content ?? '', + tool_call_chunks: call ? [call] : [], + additional_kwargs: { thought_data: thought } }) - - yield { type: 'generation', generation: generationChunk } + yield { + type: 'generation', + generation: new ChatGenerationChunk({ + message: msg, + text: getMessageContent(msg.content) ?? '' + }) + } } - if (updatedToolCalling) { - const fc = chunk['functionCall'] - if (fc?.name && fc.name.length > 0) { - functionIndex++ - } + if (call && fn.name?.length > 0) { + index++ } - } catch (e) { - if (errorCount > 5) { + } catch (err) { + if (errors > 5) { logger.error('error with chunk', chunk) throw new ChatLunaError( ChatLunaErrorCode.API_REQUEST_FAILED, - e + err ) - } else { - errorCount++ - continue } + errors++ } } } - private async _processChunk( - chunk: ChatPart, - reasoningContent: string, - functionIndex: number - ) { - const messagePart = partAsType(chunk) - const chatFunctionCallingPart = - partAsType(chunk) - const imagePart = partAsTypeCheck( - chunk, - (part) => part['inlineData'] != null - ) - - let messageContent: MessageContent - - if (messagePart.text) { - if (messagePart.thought) { - return { - updatedContent: messageContent, - updatedReasoning: reasoningContent + messagePart.text - } - } - messageContent = messagePart.text - } else if (imagePart && !messagePart.thought) { - const storageService = this.ctx.chatluna_storage - if (!storageService) { - messagePart.text = `![image](data:${imagePart.inlineData.mimeType ?? 'image/png'};base64,${imagePart.inlineData.data})` - messageContent = messagePart.text - } else { - const buffer = Buffer.from(imagePart.inlineData.data, 'base64') - - const hash = await hashString(imagePart.inlineData.data, 8) - const type = ( - imagePart.inlineData.mimeType ?? 'image/png' - ).split('/')[1] - const file = await storageService.createTempFile( - buffer, - `${hash}.${type}` - ) - - messagePart.text = `[image:${file.url}]` - messageContent = [ - { - type: 'image_url', - image_url: file.url - } - ] - } - } - - const deltaFunctionCall = chatFunctionCallingPart?.functionCall - let updatedToolCalling: ToolCallChunk - if (deltaFunctionCall) { - const isNew = deltaFunctionCall.name?.length > 0 - updatedToolCalling = this._createToolCallChunk( - deltaFunctionCall, - isNew ? functionIndex : functionIndex - 1 - ) - } - - return { - updatedContent: messageContent, - updatedReasoning: reasoningContent, - updatedToolCalling - } - } - - private _createToolCallChunk( - deltaFunctionCall: ChatFunctionCallingPart['functionCall'], - index: number - ) { - const isNew = deltaFunctionCall.name?.length > 0 - const args = - Object.keys(deltaFunctionCall.args ?? {}).length > 0 - ? JSON.stringify(deltaFunctionCall.args) - : undefined - return { - name: isNew ? deltaFunctionCall.name : undefined, - args, - id: isNew - ? (deltaFunctionCall.id ?? `function_call_${index}`) - : undefined, - index - } satisfies ToolCallChunk - } - private _handleFinalContent( reasoningState: ReasoningState, groundingContent: string @@ -750,67 +686,11 @@ export class GeminiRequester } } - private _createMessageChunk( - content: MessageContent, - functionCall: ToolCallChunk | undefined, - chunk: ChatPart + private _post( + url: string, + data: Record, + params: fetchType.RequestInit = {} ) { - const imagePart = - this.ctx.chatluna_storage != null - ? undefined - : partAsTypeCheck( - chunk, - (part) => part['inlineData'] != null - ) - const messageChunk = new AIMessageChunk({ - content: content ?? '', - tool_call_chunks: [functionCall].filter(Boolean) - }) - const sig = chunk['thoughtSignature'] - let thoughtData: Record | undefined - if (sig != null) { - const id = functionCall?.id ?? chunk['functionCall']?.id - if (id != null) { - thoughtData = { - [id]: { - thoughtSignature: sig - } - } - } else { - const part = { - thoughtSignature: sig, - toolCall: chunk['toolCall'], - toolResponse: chunk['toolResponse'], - executableCode: chunk['executableCode'], - codeExecutionResult: chunk['codeExecutionResult'] - } - const contextId = - chunk['toolCall']?.id ?? - chunk['toolResponse']?.id ?? - chunk['executableCode']?.id ?? - chunk['codeExecutionResult']?.id - - thoughtData = - contextId != null - ? { [contextId]: [part] } - : { parts: [part] } - } - } - - messageChunk.additional_kwargs = { - images: imagePart - ? [ - `data:${imagePart.inlineData.mimeType ?? 'image/png'};base64,${imagePart.inlineData.data}` - ] - : undefined, - thought_data: thoughtData - } - - return messageChunk - } - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private _post(url: string, data: any, params: fetchType.RequestInit = {}) { const requestUrl = this._concatUrl(url) for (const key in data) { diff --git a/packages/adapter-gemini/src/types.ts b/packages/adapter-gemini/src/types.ts index 40c5da189..1a5cb9251 100644 --- a/packages/adapter-gemini/src/types.ts +++ b/packages/adapter-gemini/src/types.ts @@ -5,21 +5,23 @@ export interface ChatCompletionResponseMessage { export type BaseChatPart = { thoughtSignature?: string + thought?: boolean } -export type ChatPart = - | (ChatMessagePart & BaseChatPart) - | ChatInlineDataPart - | (ChatFunctionCallingPart & BaseChatPart) - | ChatFunctionResponsePart - | (ChatToolContextPart & BaseChatPart) - | ChatUploadDataPart - // Only used for token - | ChatUsageMetadataPart +export type ChatPart = BaseChatPart & + ( + | ChatMessagePart + | (ChatInlineDataPart & { mediaProcessing?: 'AGENTIC' }) + | ChatFunctionCallingPart + | ChatFunctionResponsePart + | ChatToolContextPart + | (ChatUploadDataPart & { media_processing?: 'AGENTIC' }) + // Only used for token + | ChatUsageMetadataPart + ) export type ChatMessagePart = { text: string - thought?: boolean } export type ChatUsageMetadataPart = { @@ -72,8 +74,7 @@ export type ChatUploadDataPart = { export type ChatFunctionCallingPart = { functionCall: { name: string - // eslint-disable-next-line @typescript-eslint/no-explicit-any - args?: any + args?: Record id?: string } } @@ -87,15 +88,30 @@ export type ChatFunctionResponsePart = { } } +export type ChatToolCall = { + id?: string + toolType?: string + [key: string]: unknown +} + export type ChatToolContextPart = { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - toolCall?: Record - // eslint-disable-next-line @typescript-eslint/no-explicit-any - toolResponse?: Record - // eslint-disable-next-line @typescript-eslint/no-explicit-any - executableCode?: Record - // eslint-disable-next-line @typescript-eslint/no-explicit-any - codeExecutionResult?: Record + toolCall?: ChatToolCall + toolResponse?: ChatToolCall + executableCode?: Record + codeExecutionResult?: Record +} + +export type ChatThoughtPart = BaseChatPart & ChatToolContextPart + +// Current histories use parts and call IDs; older ones also store a part directly. +export type ChatThoughtData = ChatThoughtPart & { + parts?: ChatThoughtPart[] + [key: string]: + | string + | boolean + | ChatThoughtPart + | ChatThoughtPart[] + | Record } export interface ChatResponse { @@ -141,14 +157,24 @@ export interface ChatResponse { export interface ChatCompletionFunction { name: string description?: string - // eslint-disable-next-line @typescript-eslint/no-explicit-any - parameters?: { [key: string]: any } + parameters?: Record +} + +export interface ChatTool { + functionDeclarations?: ChatCompletionFunction[] + google_search?: { + searchTypes?: { + webSearch: Record + imageSearch: Record + } + } + code_execution?: Record + urlContext?: Record } export interface ChatCompletionMessageFunctionCall { name: string - // eslint-disable-next-line @typescript-eslint/no-explicit-any - args?: any + args?: Record } export interface CreateEmbeddingResponse { diff --git a/packages/adapter-gemini/src/utils.ts b/packages/adapter-gemini/src/utils.ts index 90456c475..ecc9dc444 100644 --- a/packages/adapter-gemini/src/utils.ts +++ b/packages/adapter-gemini/src/utils.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ import { AIMessage, BaseMessage, @@ -12,10 +11,12 @@ import { ChatCompletionFunction, ChatCompletionResponseMessage, ChatCompletionResponseMessageRoleEnum, - ChatFunctionCallingPart, ChatFunctionResponsePart, ChatPart, ChatResponse, + ChatThoughtData, + ChatThoughtPart, + ChatTool, GeminiUsageMetadata } from './types' import { Config, logger } from '.' @@ -41,71 +42,198 @@ export async function langchainMessageToGeminiMessage( plugin: ChatLunaPlugin, model?: string ): Promise { + const cfg = plugin.config + const agentic = + cfg.agenticVideo && + AGENTIC_VIDEO_MODELS.some((name) => model?.includes(name)) + + async function convert( + content: MessageContentComplex[] + ): Promise { + const parts = await Promise.all( + content.map(async (part): Promise => { + if (isMessageContentText(part)) { + return part.text.length > 0 ? { text: part.text } : null + } + const image = isMessageContentImageUrl(part) + if (!image && !isGeminiFileLikeContent(part)) { + return part as unknown as ChatPart + } + + const media = await readMedia(plugin, part) + if (media == null) return null + const mode = + agentic && !image && media.mimeType.startsWith('video/') + ? 'AGENTIC' + : undefined + return cfg.useCamelCaseMediaFields + ? { inlineData: media, mediaProcessing: mode } + : { + inline_data: { + data: media.data, + mime_type: media.mimeType + }, + media_processing: mode + } + }) + ) + return parts.filter((part) => part != null) + } + const result: ChatCompletionResponseMessage[] = [] for (let i = 0; i < messages.length; i++) { - const message = messages[i] - const role = messageTypeToGeminiRole(message.getType()) - const hasFunctionCall = - (message as AIMessage).tool_calls != null && - (message as AIMessage).tool_calls.length > 0 + const msg = messages[i] + const role = messageTypeToGeminiRole(msg.getType()) if (role === 'function') { - const parts: ChatPart[] = [] - let j = i - while (j < messages.length) { - const msg = messages[j] - if (messageTypeToGeminiRole(msg.getType()) !== 'function') break - parts.push( - ...( - await processFunctionMessage( - plugin, - msg, - plugin.config.useCamelCaseSystemInstruction - ) - ).parts + const response: ChatFunctionResponsePart['functionResponse'] = { + name: msg.name, + id: (msg as ToolMessage).tool_call_id || undefined, + response: {} + } + const parts: ChatPart[] = [{ functionResponse: response }] + if (typeof msg.content === 'string') { + response.response = parseJsonArgs(msg.content) + } else { + const texts = msg.content.filter(isMessageContentText) + if (texts.length > 0) { + response.response = parseJsonArgs( + texts.map((part) => part.text).join('') + ) + } + const media = await convert( + msg.content.filter( + (part) => + isMessageContentImageUrl(part) || + isGeminiFileLikeContent(part) + ) ) - j++ + for (const part of media) { + // Gemini requires agentic videos beside the function response. + if ( + ('inlineData' in part && part.mediaProcessing) || + ('inline_data' in part && part.media_processing) + ) { + parts.push(part) + continue + } + if ('inlineData' in part) { + response.parts ??= [] + response.parts.push({ inlineData: part.inlineData }) + } else if ('inline_data' in part) { + response.parts ??= [] + response.parts.push({ inline_data: part.inline_data }) + } + } + } + + // Consecutive tool results form one user turn. + if (i > 0 && messages[i - 1].getType() === 'tool') { + result[result.length - 1].parts.push(...parts) + } else { + result.push({ role: 'user', parts }) } - i = j - 1 - result.push({ role: 'user', parts }) continue } - if (hasFunctionCall) { - result.push( - await processFunctionMessage( - plugin, - message, - plugin.config.useCamelCaseSystemInstruction - ) - ) + if ((msg as AIMessage).tool_calls?.length > 0) { + const text = + typeof msg.content === 'string' + ? msg.content.length > 0 + ? [{ text: msg.content }] + : [] + : await convert(msg.content) + result.push({ + role: 'model', + parts: [...text, ...convertCalls(msg as AIMessage)] + }) continue } - const item: ChatCompletionResponseMessage = { role, parts: [] } - const thoughtData: Record = - message.additional_kwargs['thought_data'] ?? {} + const data = (msg.additional_kwargs.thought_data ?? + {}) as ChatThoughtData + const parts = - typeof message.content === 'string' - ? message.content.length > 0 - ? [{ text: message.content }] + typeof msg.content === 'string' + ? msg.content.length > 0 + ? [{ text: msg.content }] : [] - : await processGeminiContentParts(plugin, message.content) - - item.parts = [...getContextParts(thoughtData), ...parts] + : await convert(msg.content) + result.push({ + role, + parts: [ + ...getContextParts([data, ...Object.values(data)]), + ...parts + ] + }) - if (message.additional_kwargs.images != null) { + if (msg.additional_kwargs.images != null) { logger.warn( 'Deprecated: `additional_kwargs.images` is no longer supported. Use `image_url` content parts instead.' ) } - - result.push(item) } return result } +async function readMedia( + plugin: ChatLunaPlugin, + part: MessageContentImageUrl | Parameters[1] +) { + try { + if (isMessageContentImageUrl(part)) { + const url = await fetchImageUrl(plugin, part) + return { + data: url.replace(/^data:image\/\w+;base64,/, ''), + mimeType: + url.match(/^data:([^;]+);base64,/)?.[1] ?? 'image/jpeg' + } + } + const file = await fetchFileLikeUrl(plugin, part) + return { data: file.buffer.toString('base64'), mimeType: file.mimeType } + } catch (err) { + logger.warn(`Failed to fetch ${part.type}`, err) + return null + } +} + +function convertCalls(msg: AIMessage): ChatPart[] { + const calls = msg.tool_calls + const data = (msg.additional_kwargs.thought_data ?? {}) as ChatThoughtData + // Replay shared context once; call-specific context stays with its call. + const shared = { ...data } + for (const call of calls) { + if (call.id != null) delete shared[call.id] + } + const parts: ChatPart[] = getContextParts([ + shared, + ...Object.values(shared) + ]) + for (const call of calls) { + if (call.id != null) { + parts.push(...getContextParts([data[call.id]])) + } + const context = data[call.id] ?? data + const sig = Array.isArray(context) + ? context.find((part) => typeof part?.thoughtSignature === 'string') + ?.thoughtSignature + : typeof context === 'object' + ? context.thoughtSignature + : undefined + parts.push({ + functionCall: { + name: call.name, + args: call.args, + id: call.id || undefined + }, + ...(typeof sig === 'string' ? { thoughtSignature: sig } : {}) + }) + } + + return parts +} + export function extractSystemMessages( messages: ChatCompletionResponseMessage[] ): [ChatCompletionResponseMessage, ChatCompletionResponseMessage[]] { @@ -164,156 +292,39 @@ function parseJsonArgs(args: string): Record { } } -function isContextPart(part: any): part is ChatPart { +export function isMediaProcessingPart(part: ChatThoughtPart): boolean { + const tool = part.toolCall ?? part.toolResponse + // Agentic media steps can omit toolType and cannot be replayed by Gemini. return ( - typeof part === 'object' && - part != null && - (part['toolCall'] != null || - part['toolResponse'] != null || - part['executableCode'] != null || - part['codeExecutionResult'] != null) + tool != null && + (tool.toolType == null || tool.toolType === 'MEDIA_PROCESSING') ) } -function getContextParts(data: Record, id?: string) { - const parts = data['parts'] ?? [data, ...Object.values(data)] - const raw = id != null ? data[id] : parts - if (raw == null) return [] - - return (Array.isArray(raw) ? raw : [raw]).filter(isContextPart) +export function isToolContext(part: unknown): part is ChatThoughtPart { + return ( + typeof part === 'object' && + part != null && + (('toolCall' in part && part.toolCall != null) || + ('toolResponse' in part && part.toolResponse != null) || + ('executableCode' in part && part.executableCode != null) || + ('codeExecutionResult' in part && part.codeExecutionResult != null)) + ) } -async function processFunctionMessage( - plugin: ChatLunaPlugin, - message: AIMessage | ToolMessage, - removeId: boolean -): Promise { - const thoughtData: Record = - message.additional_kwargs['thought_data'] ?? {} - - if (message['tool_calls']) { - message = message as AIMessage - const toolCalls = message.tool_calls - const parts: ChatPart[] = [] - - for (const toolCall of toolCalls) { - // tool context: replay context tied to this tool call first. - parts.push(...getContextParts(thoughtData, toolCall.id)) - - const functionCall: ChatFunctionCallingPart['functionCall'] = { - name: toolCall.name, - args: toolCall.args - } - if (!removeId || toolCall.id) { - functionCall.id = toolCall.id - } - const data = thoughtData[toolCall.id] ?? thoughtData - const sig = Array.isArray(data) - ? data.find( - (item) => typeof item?.thoughtSignature === 'string' - )?.thoughtSignature - : data.thoughtSignature - - // tool calls: reattach custom tool calls with their thought signatures. - parts.push({ - functionCall, - ...(typeof sig === 'string' ? { thoughtSignature: sig } : {}) - }) - } - - return { - role: 'model', - parts - } - } - - const finalMessage = message as ToolMessage - - const functionResponse: ChatFunctionResponsePart['functionResponse'] = { - name: message.name, - response: {} - } - - if (Array.isArray(message.content)) { - const texts = message.content.flatMap((part) => { - if (isMessageContentText(part)) return [part.text] - return [] - }) - - if (texts.length > 0) { - functionResponse.response = parseJsonArgs(texts.join('')) - } - - const parts = await processGeminiContentParts( - plugin, - message.content.filter( - (part) => - isMessageContentImageUrl(part) || - isGeminiFileLikeContent(part) - ) +function getContextParts(parts: ChatThoughtData[string][]): ChatThoughtPart[] { + // Old histories also store context directly or under individual call IDs. + return parts + .flat() + .filter( + (part): part is ChatThoughtPart => + isToolContext(part) && !isMediaProcessingPart(part) ) - if (parts.length > 0) { - functionResponse.parts = parts - } - } else { - functionResponse.response = parseJsonArgs(message.content as string) - } - - if (!removeId || finalMessage.tool_call_id) { - functionResponse.id = finalMessage.tool_call_id - } - - return { - role: 'user', - parts: [ - { - functionResponse - } - ] - } } -async function processGeminiImageContent( - plugin: ChatLunaPlugin, - part: MessageContentImageUrl -) { - let url: string - try { - url = await fetchImageUrl(plugin, part) - } catch (e) { - const rawUrl = - typeof part.image_url === 'string' - ? part.image_url - : part.image_url.url - logger.warn(`Failed to fetch image url: ${rawUrl}`, e) - return null - } - - const mineType = url.match(/^data:([^;]+);base64,/)?.[1] ?? 'image/jpeg' - const data = url.replace(/^data:image\/\w+;base64,/, '') - - return createGeminiInlineDataPart(plugin, data, mineType) -} - -type GeminiFileLikeContent = MessageContentComplex & - ( - | { - type: 'file_url' - file_url: string | { url: string; mimeType?: string } - } - | { - type: 'audio_url' - audio_url: string | { url: string; mimeType?: string } - } - | { - type: 'video_url' - video_url: string | { url: string; mimeType?: string } - } - ) - function isGeminiFileLikeContent( part: MessageContentComplex -): part is GeminiFileLikeContent { +): part is Parameters[1] { return ( part != null && typeof part === 'object' && @@ -321,78 +332,19 @@ function isGeminiFileLikeContent( ) } -function createGeminiInlineDataPart( - plugin: ChatLunaPlugin, - data: string, - mimeType: string -) { - if (plugin.config.useCamelCaseMediaFields) { - return { - inlineData: { data, mimeType } - } - } - - return { - inline_data: { data, mime_type: mimeType } - } -} - -async function processGeminiFileLikeContent( - plugin: ChatLunaPlugin, - part: GeminiFileLikeContent -) { - try { - const { buffer, mimeType } = await fetchFileLikeUrl(plugin, part) - return createGeminiInlineDataPart( - plugin, - buffer.toString('base64'), - mimeType - ) - } catch (e) { - logger.warn(`Failed to fetch ${part.type}`, e) - return null - } -} - -async function processGeminiContentParts( - plugin: ChatLunaPlugin, - content: MessageContentComplex[] -) { - const mappedParts = await Promise.all( - content.map(async (part) => { - if (isMessageContentText(part)) { - return part.text.length > 0 ? { text: part.text } : null - } - if (isMessageContentImageUrl(part)) { - return await processGeminiImageContent(plugin, part) - } - if (isGeminiFileLikeContent(part)) { - return await processGeminiFileLikeContent(plugin, part) - } - return part as any - }) - ) - - return mappedParts.filter((part) => part != null) -} - -export function partAsType(part: ChatPart): T { - return part as T -} - -export function partAsTypeCheck( - part: ChatPart, - check: (part: ChatPart & unknown) => boolean -): T | undefined { - return check(part) ? (part as T) : undefined -} - // 不支持 googleSearch / codeExecution / urlContext 的模型列表 const CUSTOM_TOOLS_UNSUPPORTED_MODELS = [ 'gemini-2.0-flash-lite', 'gemini-2.0-flash-exp' ] +const AGENTIC_VIDEO_MODELS = [ + 'gemini-3.5-flash-lite', + 'gemini-3.6-flash', + 'gemini-3.7-flash', + 'gemini-3.8-flash' +] + // 启用 imageGeneration 时同样不支持上述自定义工具的模型列表 const IMAGE_GENERATION_MODELS = [ 'gemini-2.0-flash-exp', @@ -438,7 +390,7 @@ function isImageSearchSupported(model: string): boolean { * - 其余模型使用标准的新版 google_search: {} 格式 */ function appendBuiltinTools( - result: Record[], + result: ChatTool[], googleSearch: boolean, codeExecution: boolean, urlContext: boolean, @@ -472,7 +424,7 @@ export function formatToolsToGeminiAITools( tools: StructuredTool[], config: Config, model: string -): Record { +): ChatTool[] | undefined { // 没有任何工具需要注册时直接返回 if ( tools.length < 1 && @@ -484,7 +436,7 @@ export function formatToolsToGeminiAITools( } const functions = tools.map(formatToolToGeminiAITool) - const result: Record[] = [] + const result: ChatTool[] = [] // --- 处理内置工具（googleSearch / codeExecution / urlContext）--- let { googleSearch, codeExecution, urlContext } = config @@ -537,8 +489,7 @@ export function formatToolToGeminiAITool( return { name: tool.name, description: tool.description, - // any? - parameters + parameters: parameters as ChatCompletionFunction['parameters'] } } @@ -569,15 +520,15 @@ const GEMINI_SCHEMA_KEYS = new Set([ 'anyOf' ]) -function sanitizeGeminiSchema(schema: any): any { +function sanitizeGeminiSchema(schema: unknown): unknown { if (!schema || typeof schema !== 'object' || Array.isArray(schema)) { return schema } - const result: Record = {} + const result: Record = {} for (const [key, value] of Object.entries(schema)) { if (key === 'oneOf' || key === 'anyOf') { - result['anyOf'] = (value as any[]).map(sanitizeGeminiSchema) + result['anyOf'] = (value as unknown[]).map(sanitizeGeminiSchema) continue } if (!GEMINI_SCHEMA_KEYS.has(key)) continue @@ -587,10 +538,9 @@ function sanitizeGeminiSchema(schema: any): any { } if (key === 'properties') { result['properties'] = Object.fromEntries( - Object.entries(value).map(([name, sub]) => [ - name, - sanitizeGeminiSchema(sub) - ]) + Object.entries(value as Record).map( + ([name, sub]) => [name, sanitizeGeminiSchema(sub)] + ) ) continue } @@ -843,10 +793,6 @@ export async function createChatGenerationParams( } } -export function isChatResponse(response: any): response is ChatResponse { - return 'candidates' in response -} - // #region refreshModels helpers export function isGeminiModelName(model: string): boolean {