Merge pull request #5581 from ConnectAI-E/feature/gemini-functioncall

google gemini support function call

app/client/platforms/google.ts

@@ -7,21 +7,25 @@ import {
   LLMUsage,
   SpeechOptions,
 } from "../api";
-import { useAccessStore, useAppConfig, useChatStore } from "@/app/store";
+import {
+  useAccessStore,
+  useAppConfig,
+  useChatStore,
+  usePluginStore,
+  ChatMessageTool,
+} from "@/app/store";
+import { stream } from "@/app/utils/chat";
 import { getClientConfig } from "@/app/config/client";
 import { GEMINI_BASE_URL } from "@/app/constant";
-import Locale from "../../locales";
+
-import {
-  EventStreamContentType,
-  fetchEventSource,
-} from "@fortaine/fetch-event-source";
-import { prettyObject } from "@/app/utils/format";
 import {
   getMessageTextContent,
   getMessageImages,
   isVisionModel,
 } from "@/app/utils";
 import { preProcessImageContent } from "@/app/utils/chat";
+import { nanoid } from "nanoid";
+import { RequestPayload } from "./openai";
 import { fetch } from "@/app/utils/stream";
 
 export class GeminiProApi implements LLMApi {
@@ -178,115 +182,81 @@ export class GeminiProApi implements LLMApi {
       );
 
       if (shouldStream) {
-        let responseText = "";
+        const [tools, funcs] = usePluginStore
-        let remainText = "";
+          .getState()
-        let finished = false;
+          .getAsTools(
+            useChatStore.getState().currentSession().mask?.plugin || [],
+          );
+        return stream(
+          chatPath,
+          requestPayload,
+          getHeaders(),
+          // @ts-ignore
+          [{ functionDeclarations: tools.map((tool) => tool.function) }],
+          funcs,
+          controller,
+          // parseSSE
+          (text: string, runTools: ChatMessageTool[]) => {
+            // console.log("parseSSE", text, runTools);
+            const chunkJson = JSON.parse(text);
 
-        const finish = () => {
+            const functionCall = chunkJson?.candidates
-          if (!finished) {
+              ?.at(0)
-            finished = true;
+              ?.content.parts.at(0)?.functionCall;
-            options.onFinish(responseText + remainText);
+            if (functionCall) {
-          }
+              const { name, args } = functionCall;
-        };
+              runTools.push({
-
+                id: nanoid(),
-        // animate response to make it looks smooth
+                type: "function",
-        function animateResponseText() {
+                function: {
-          if (finished || controller.signal.aborted) {
+                  name,
-            responseText += remainText;
+                  arguments: JSON.stringify(args), // utils.chat call function, using JSON.parse
-            finish();
+                },
-            return;
+              });
-          }
+            }
-
+            return chunkJson?.candidates?.at(0)?.content.parts.at(0)?.text;
-          if (remainText.length > 0) {
+          },
-            const fetchCount = Math.max(1, Math.round(remainText.length / 60));
+          // processToolMessage, include tool_calls message and tool call results
-            const fetchText = remainText.slice(0, fetchCount);
+          (
-            responseText += fetchText;
+            requestPayload: RequestPayload,
-            remainText = remainText.slice(fetchCount);
+            toolCallMessage: any,
-            options.onUpdate?.(responseText, fetchText);
+            toolCallResult: any[],
-          }
+          ) => {
-
+            // @ts-ignore
-          requestAnimationFrame(animateResponseText);
+            requestPayload?.contents?.splice(
-        }
+              // @ts-ignore
-
+              requestPayload?.contents?.length,
-        // start animaion
+              0,
-        animateResponseText();
+              {
-
+                role: "model",
-        controller.signal.onabort = finish;
+                parts: toolCallMessage.tool_calls.map(
-
+                  (tool: ChatMessageTool) => ({
-        fetchEventSource(chatPath, {
+                    functionCall: {
-          fetch: fetch as any,
+                      name: tool?.function?.name,
-          ...chatPayload,
+                      args: JSON.parse(tool?.function?.arguments as string),
-          async onopen(res) {
+                    },
-            clearTimeout(requestTimeoutId);
+                  }),
-            const contentType = res.headers.get("content-type");
+                ),
-            console.log(
+              },
-              "[Gemini] request response content type: ",
+              // @ts-ignore
-              contentType,
+              ...toolCallResult.map((result) => ({
+                role: "function",
+                parts: [
+                  {
+                    functionResponse: {
+                      name: result.name,
+                      response: {
+                        name: result.name,
+                        content: result.content, // TODO just text content...
+                      },
+                    },
+                  },
+                ],
+              })),
             );
-
-            if (contentType?.startsWith("text/plain")) {
-              responseText = await res.clone().text();
-              return finish();
-            }
-
-            if (
-              !res.ok ||
-              !res.headers
-                .get("content-type")
-                ?.startsWith(EventStreamContentType) ||
-              res.status !== 200
-            ) {
-              const responseTexts = [responseText];
-              let extraInfo = await res.clone().text();
-              try {
-                const resJson = await res.clone().json();
-                extraInfo = prettyObject(resJson);
-              } catch {}
-
-              if (res.status === 401) {
-                responseTexts.push(Locale.Error.Unauthorized);
-              }
-
-              if (extraInfo) {
-                responseTexts.push(extraInfo);
-              }
-
-              responseText = responseTexts.join("\n\n");
-
-              return finish();
-            }
           },
-          onmessage(msg) {
+          options,
-            if (msg.data === "[DONE]" || finished) {
+        );
-              return finish();
-            }
-            const text = msg.data;
-            try {
-              const json = JSON.parse(text);
-              const delta = apiClient.extractMessage(json);
-
-              if (delta) {
-                remainText += delta;
-              }
-
-              const blockReason = json?.promptFeedback?.blockReason;
-              if (blockReason) {
-                // being blocked
-                console.log(`[Google] [Safety Ratings] result:`, blockReason);
-              }
-            } catch (e) {
-              console.error("[Request] parse error", text, msg);
-            }
-          },
-          onclose() {
-            finish();
-          },
-          onerror(e) {
-            options.onError?.(e);
-            throw e;
-          },
-          openWhenHidden: true,
-        });
       } else {
         const res = await fetch(chatPath, chatPayload);
         clearTimeout(requestTimeoutId);

app/utils.ts

@@ -285,6 +285,9 @@ export function showPlugins(provider: ServiceProvider, model: string) {
   if (provider == ServiceProvider.Anthropic && !model.includes("claude-2")) {
     return true;
   }
+  if (provider == ServiceProvider.Google && !model.includes("vision")) {
+    return true;
+  }
   return false;
 }
 

app/utils/chat.ts

@@ -250,6 +250,7 @@ export function stream(
                 return e.toString();
               })
               .then((content) => ({
+                name: tool.function.name,
                 role: "tool",
                 content,
                 tool_call_id: tool.id,