import { ApiPath, Google, REQUEST_TIMEOUT_MS } from "@/app/constant"; import { ChatOptions, getHeaders, LLMApi, LLMModel, LLMUsage, SpeechOptions, } from "../api"; import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; import { getClientConfig } from "@/app/config/client"; import { DEFAULT_API_HOST } from "@/app/constant"; import Locale from "../../locales"; import { EventStreamContentType, fetchEventSource, } from "@fortaine/fetch-event-source"; import { prettyObject } from "@/app/utils/format"; import { getMessageTextContent, getMessageImages, isVisionModel, } from "@/app/utils"; import { preProcessImageContent } from "@/app/utils/chat"; export class GeminiProApi implements LLMApi { path(path: string): string { const accessStore = useAccessStore.getState(); let baseUrl = ""; if (accessStore.useCustomConfig) { baseUrl = accessStore.googleUrl; } const isApp = !!getClientConfig()?.isApp; if (baseUrl.length === 0) { baseUrl = isApp ? DEFAULT_API_HOST + `/api/proxy/google` : ApiPath.Google; } if (baseUrl.endsWith("/")) { baseUrl = baseUrl.slice(0, baseUrl.length - 1); } if (!baseUrl.startsWith("http") && !baseUrl.startsWith(ApiPath.Google)) { baseUrl = "https://" + baseUrl; } console.log("[Proxy Endpoint] ", baseUrl, path); let chatPath = [baseUrl, path].join("/"); chatPath += chatPath.includes("?") ? "&alt=sse" : "?alt=sse"; return chatPath; } extractMessage(res: any) { console.log("[Response] gemini-pro response: ", res); return ( res?.candidates?.at(0)?.content?.parts.at(0)?.text || res?.error?.message || "" ); } speech(options: SpeechOptions): Promise { throw new Error("Method not implemented."); } async chat(options: ChatOptions): Promise { const apiClient = this; let multimodal = false; // try get base64image from local cache image_url const _messages: ChatOptions["messages"] = []; for (const v of options.messages) { const content = await preProcessImageContent(v.content); _messages.push({ role: v.role, content }); } const messages = _messages.map((v) => { let parts: any[] = [{ text: getMessageTextContent(v) }]; if (isVisionModel(options.config.model)) { const images = getMessageImages(v); if (images.length > 0) { multimodal = true; parts = parts.concat( images.map((image) => { const imageType = image.split(";")[0].split(":")[1]; const imageData = image.split(",")[1]; return { inline_data: { mime_type: imageType, data: imageData, }, }; }), ); } } return { role: v.role.replace("assistant", "model").replace("system", "user"), parts: parts, }; }); // google requires that role in neighboring messages must not be the same for (let i = 0; i < messages.length - 1; ) { // Check if current and next item both have the role "model" if (messages[i].role === messages[i + 1].role) { // Concatenate the 'parts' of the current and next item messages[i].parts = messages[i].parts.concat(messages[i + 1].parts); // Remove the next item messages.splice(i + 1, 1); } else { // Move to the next item i++; } } // if (visionModel && messages.length > 1) { // options.onError?.(new Error("Multiturn chat is not enabled for models/gemini-pro-vision")); // } const accessStore = useAccessStore.getState(); const modelConfig = { ...useAppConfig.getState().modelConfig, ...useChatStore.getState().currentSession().mask.modelConfig, ...{ model: options.config.model, }, }; const requestPayload = { contents: messages, generationConfig: { // stopSequences: [ // "Title" // ], temperature: modelConfig.temperature, maxOutputTokens: modelConfig.max_tokens, topP: modelConfig.top_p, // "topK": modelConfig.top_k, }, safetySettings: [ { category: "HARM_CATEGORY_HARASSMENT", threshold: accessStore.googleSafetySettings, }, { category: "HARM_CATEGORY_HATE_SPEECH", threshold: accessStore.googleSafetySettings, }, { category: "HARM_CATEGORY_SEXUALLY_EXPLICIT", threshold: accessStore.googleSafetySettings, }, { category: "HARM_CATEGORY_DANGEROUS_CONTENT", threshold: accessStore.googleSafetySettings, }, ], }; let shouldStream = !!options.config.stream; const controller = new AbortController(); options.onController?.(controller); try { // https://github.com/google-gemini/cookbook/blob/main/quickstarts/rest/Streaming_REST.ipynb const chatPath = this.path(Google.ChatPath(modelConfig.model)); const chatPayload = { method: "POST", body: JSON.stringify(requestPayload), signal: controller.signal, headers: getHeaders(), }; // make a fetch request const requestTimeoutId = setTimeout( () => controller.abort(), REQUEST_TIMEOUT_MS, ); if (shouldStream) { let responseText = ""; let remainText = ""; let finished = false; const finish = () => { if (!finished) { finished = true; options.onFinish(responseText + remainText); } }; // animate response to make it looks smooth function animateResponseText() { if (finished || controller.signal.aborted) { responseText += remainText; finish(); return; } if (remainText.length > 0) { const fetchCount = Math.max(1, Math.round(remainText.length / 60)); const fetchText = remainText.slice(0, fetchCount); responseText += fetchText; remainText = remainText.slice(fetchCount); options.onUpdate?.(responseText, fetchText); } requestAnimationFrame(animateResponseText); } // start animaion animateResponseText(); controller.signal.onabort = finish; fetchEventSource(chatPath, { ...chatPayload, async onopen(res) { clearTimeout(requestTimeoutId); const contentType = res.headers.get("content-type"); console.log( "[Gemini] request response content type: ", contentType, ); if (contentType?.startsWith("text/plain")) { responseText = await res.clone().text(); return finish(); } if ( !res.ok || !res.headers .get("content-type") ?.startsWith(EventStreamContentType) || res.status !== 200 ) { const responseTexts = [responseText]; let extraInfo = await res.clone().text(); try { const resJson = await res.clone().json(); extraInfo = prettyObject(resJson); } catch {} if (res.status === 401) { responseTexts.push(Locale.Error.Unauthorized); } if (extraInfo) { responseTexts.push(extraInfo); } responseText = responseTexts.join("\n\n"); return finish(); } }, onmessage(msg) { if (msg.data === "[DONE]" || finished) { return finish(); } const text = msg.data; try { const json = JSON.parse(text); const delta = apiClient.extractMessage(json); if (delta) { remainText += delta; } const blockReason = json?.promptFeedback?.blockReason; if (blockReason) { // being blocked console.log(`[Google] [Safety Ratings] result:`, blockReason); } } catch (e) { console.error("[Request] parse error", text, msg); } }, onclose() { finish(); }, onerror(e) { options.onError?.(e); throw e; }, openWhenHidden: true, }); } else { const res = await fetch(chatPath, chatPayload); clearTimeout(requestTimeoutId); const resJson = await res.json(); if (resJson?.promptFeedback?.blockReason) { // being blocked options.onError?.( new Error( "Message is being blocked for reason: " + resJson.promptFeedback.blockReason, ), ); } const message = apiClient.extractMessage(resJson); options.onFinish(message); } } catch (e) { console.log("[Request] failed to make a chat request", e); options.onError?.(e as Error); } } usage(): Promise { throw new Error("Method not implemented."); } async models(): Promise { return []; } }