From 98a11e56d2c55d7d89dfc4c8905045781863bf98 Mon Sep 17 00:00:00 2001
From: suruiqiang <bestsanmao@qq.com>
Date: Tue, 11 Feb 2025 12:46:46 +0800
Subject: [PATCH] support alibaba and bytedance's reasoning_content
---
app/client/platforms/alibaba.ts | 220 ++++++++++++++----------------
app/client/platforms/bytedance.ts | 205 +++++++++++++---------------
2 files changed, 200 insertions(+), 225 deletions(-)
diff --git a/app/client/platforms/alibaba.ts b/app/client/platforms/alibaba.ts
index 6fe69e87a..13cb558f9 100644
--- a/app/client/platforms/alibaba.ts
+++ b/app/client/platforms/alibaba.ts
@@ -5,8 +5,14 @@ import {
ALIBABA_BASE_URL,
REQUEST_TIMEOUT_MS,
} from "@/app/constant";
-import { useAccessStore, useAppConfig, useChatStore } from "@/app/store";
-
+import {
+ useAccessStore,
+ useAppConfig,
+ useChatStore,
+ ChatMessageTool,
+ usePluginStore,
+} from "@/app/store";
+import { streamWithThink } from "@/app/utils/chat";
import {
ChatOptions,
getHeaders,
@@ -15,14 +21,11 @@ import {
SpeechOptions,
MultimodalContent,
} from "../api";
-import Locale from "../../locales";
-import {
- EventStreamContentType,
- fetchEventSource,
-} from "@fortaine/fetch-event-source";
-import { prettyObject } from "@/app/utils/format";
import { getClientConfig } from "@/app/config/client";
-import { getMessageTextContent } from "@/app/utils";
+import {
+ getMessageTextContent,
+ getMessageTextContentWithoutThinking,
+} from "@/app/utils";
import { fetch } from "@/app/utils/stream";
export interface OpenAIListModelResponse {
@@ -92,7 +95,10 @@ export class QwenApi implements LLMApi {
async chat(options: ChatOptions) {
const messages = options.messages.map((v) => ({
role: v.role,
- content: getMessageTextContent(v),
+ content:
+ v.role === "assistant"
+ ? getMessageTextContentWithoutThinking(v)
+ : getMessageTextContent(v),
}));
const modelConfig = {
@@ -122,15 +128,17 @@ export class QwenApi implements LLMApi {
options.onController?.(controller);
try {
+ const headers = {
+ ...getHeaders(),
+ "X-DashScope-SSE": shouldStream ? "enable" : "disable",
+ };
+
const chatPath = this.path(Alibaba.ChatPath);
const chatPayload = {
method: "POST",
body: JSON.stringify(requestPayload),
signal: controller.signal,
- headers: {
- ...getHeaders(),
- "X-DashScope-SSE": shouldStream ? "enable" : "disable",
- },
+ headers: headers,
};
// make a fetch request
@@ -140,116 +148,96 @@ export class QwenApi implements LLMApi {
);
if (shouldStream) {
- let responseText = "";
- let remainText = "";
- let finished = false;
- let responseRes: Response;
-
- // animate response to make it looks smooth
- function animateResponseText() {
- if (finished || controller.signal.aborted) {
- responseText += remainText;
- console.log("[Response Animation] finished");
- if (responseText?.length === 0) {
- options.onError?.(new Error("empty response from server"));
- }
- return;
- }
-
- if (remainText.length > 0) {
- const fetchCount = Math.max(1, Math.round(remainText.length / 60));
- const fetchText = remainText.slice(0, fetchCount);
- responseText += fetchText;
- remainText = remainText.slice(fetchCount);
- options.onUpdate?.(responseText, fetchText);
- }
-
- requestAnimationFrame(animateResponseText);
- }
-
- // start animaion
- animateResponseText();
-
- const finish = () => {
- if (!finished) {
- finished = true;
- options.onFinish(responseText + remainText, responseRes);
- }
- };
-
- controller.signal.onabort = finish;
-
- fetchEventSource(chatPath, {
- fetch: fetch as any,
- ...chatPayload,
- async onopen(res) {
- clearTimeout(requestTimeoutId);
- const contentType = res.headers.get("content-type");
- console.log(
- "[Alibaba] request response content type: ",
- contentType,
- );
- responseRes = res;
-
- if (contentType?.startsWith("text/plain")) {
- responseText = await res.clone().text();
- return finish();
+ const [tools, funcs] = usePluginStore
+ .getState()
+ .getAsTools(
+ useChatStore.getState().currentSession().mask?.plugin || [],
+ );
+ return streamWithThink(
+ chatPath,
+ requestPayload,
+ headers,
+ tools as any,
+ funcs,
+ controller,
+ // parseSSE
+ (text: string, runTools: ChatMessageTool[]) => {
+ // console.log("parseSSE", text, runTools);
+ const json = JSON.parse(text);
+ const choices = json.output.choices as Array<{
+ message: {
+ content: string | null;
+ tool_calls: ChatMessageTool[];
+ reasoning_content: string | null;
+ };
+ }>;
+ const tool_calls = choices[0]?.message?.tool_calls;
+ if (tool_calls?.length > 0) {
+ const index = tool_calls[0]?.index;
+ const id = tool_calls[0]?.id;
+ const args = tool_calls[0]?.function?.arguments;
+ if (id) {
+ runTools.push({
+ id,
+ type: tool_calls[0]?.type,
+ function: {
+ name: tool_calls[0]?.function?.name as string,
+ arguments: args,
+ },
+ });
+ } else {
+ // @ts-ignore
+ runTools[index]["function"]["arguments"] += args;
+ }
}
+ const reasoning = choices[0]?.message?.reasoning_content;
+ const content = choices[0]?.message?.content;
+ // Skip if both content and reasoning_content are empty or null
if (
- !res.ok ||
- !res.headers
- .get("content-type")
- ?.startsWith(EventStreamContentType) ||
- res.status !== 200
+ (!reasoning || reasoning.trim().length === 0) &&
+ (!content || content.trim().length === 0)
) {
- const responseTexts = [responseText];
- let extraInfo = await res.clone().text();
- try {
- const resJson = await res.clone().json();
- extraInfo = prettyObject(resJson);
- } catch {}
-
- if (res.status === 401) {
- responseTexts.push(Locale.Error.Unauthorized);
- }
-
- if (extraInfo) {
- responseTexts.push(extraInfo);
- }
-
- responseText = responseTexts.join("\n\n");
-
- return finish();
+ return {
+ isThinking: false,
+ content: "",
+ };
}
- },
- onmessage(msg) {
- if (msg.data === "[DONE]" || finished) {
- return finish();
- }
- const text = msg.data;
- try {
- const json = JSON.parse(text);
- const choices = json.output.choices as Array<{
- message: { content: string };
- }>;
- const delta = choices[0]?.message?.content;
- if (delta) {
- remainText += delta;
- }
- } catch (e) {
- console.error("[Request] parse error", text, msg);
+
+ if (reasoning && reasoning.trim().length > 0) {
+ return {
+ isThinking: true,
+ content: reasoning,
+ };
+ } else if (content && content.trim().length > 0) {
+ return {
+ isThinking: false,
+ content: content,
+ };
}
+
+ return {
+ isThinking: false,
+ content: "",
+ };
},
- onclose() {
- finish();
+ // processToolMessage, include tool_calls message and tool call results
+ (
+ requestPayload: RequestPayload,
+ toolCallMessage: any,
+ toolCallResult: any[],
+ ) => {
+ // @ts-ignore
+ requestPayload?.messages?.splice(
+ // @ts-ignore
+ requestPayload?.messages?.length,
+ 0,
+ toolCallMessage,
+ ...toolCallResult,
+ );
},
- onerror(e) {
- options.onError?.(e);
- throw e;
- },
- openWhenHidden: true,
- });
+ options,
+ );
} else {
const res = await fetch(chatPath, chatPayload);
clearTimeout(requestTimeoutId);
diff --git a/app/client/platforms/bytedance.ts b/app/client/platforms/bytedance.ts
index c2f128128..5d7ddebeb 100644
--- a/app/client/platforms/bytedance.ts
+++ b/app/client/platforms/bytedance.ts
@@ -5,7 +5,13 @@ import {
BYTEDANCE_BASE_URL,
REQUEST_TIMEOUT_MS,
} from "@/app/constant";
-import { useAccessStore, useAppConfig, useChatStore } from "@/app/store";
+import {
+ useAccessStore,
+ useAppConfig,
+ useChatStore,
+ ChatMessageTool,
+ usePluginStore,
+} from "@/app/store";
import {
ChatOptions,
@@ -15,14 +21,11 @@ import {
MultimodalContent,
SpeechOptions,
} from "../api";
-import Locale from "../../locales";
-import {
- EventStreamContentType,
- fetchEventSource,
-} from "@fortaine/fetch-event-source";
-import { prettyObject } from "@/app/utils/format";
+
+import { streamWithThink } from "@/app/utils/chat";
import { getClientConfig } from "@/app/config/client";
import { preProcessImageContent } from "@/app/utils/chat";
+import { getMessageTextContentWithoutThinking } from "@/app/utils";
import { fetch } from "@/app/utils/stream";
export interface OpenAIListModelResponse {
@@ -86,7 +89,10 @@ export class DoubaoApi implements LLMApi {
async chat(options: ChatOptions) {
const messages: ChatOptions["messages"] = [];
for (const v of options.messages) {
- const content = await preProcessImageContent(v.content);
+ const content =
+ v.role === "assistant"
+ ? getMessageTextContentWithoutThinking(v)
+ : await preProcessImageContent(v.content);
messages.push({ role: v.role, content });
}
@@ -128,115 +134,96 @@ export class DoubaoApi implements LLMApi {
);
if (shouldStream) {
- let responseText = "";
- let remainText = "";
- let finished = false;
- let responseRes: Response;
-
- // animate response to make it looks smooth
- function animateResponseText() {
- if (finished || controller.signal.aborted) {
- responseText += remainText;
- console.log("[Response Animation] finished");
- if (responseText?.length === 0) {
- options.onError?.(new Error("empty response from server"));
- }
- return;
- }
-
- if (remainText.length > 0) {
- const fetchCount = Math.max(1, Math.round(remainText.length / 60));
- const fetchText = remainText.slice(0, fetchCount);
- responseText += fetchText;
- remainText = remainText.slice(fetchCount);
- options.onUpdate?.(responseText, fetchText);
- }
-
- requestAnimationFrame(animateResponseText);
- }
-
- // start animaion
- animateResponseText();
-
- const finish = () => {
- if (!finished) {
- finished = true;
- options.onFinish(responseText + remainText, responseRes);
- }
- };
-
- controller.signal.onabort = finish;
-
- fetchEventSource(chatPath, {
- fetch: fetch as any,
- ...chatPayload,
- async onopen(res) {
- clearTimeout(requestTimeoutId);
- const contentType = res.headers.get("content-type");
- console.log(
- "[ByteDance] request response content type: ",
- contentType,
- );
- responseRes = res;
- if (contentType?.startsWith("text/plain")) {
- responseText = await res.clone().text();
- return finish();
+ const [tools, funcs] = usePluginStore
+ .getState()
+ .getAsTools(
+ useChatStore.getState().currentSession().mask?.plugin || [],
+ );
+ return streamWithThink(
+ chatPath,
+ requestPayload,
+ getHeaders(),
+ tools as any,
+ funcs,
+ controller,
+ // parseSSE
+ (text: string, runTools: ChatMessageTool[]) => {
+ // console.log("parseSSE", text, runTools);
+ const json = JSON.parse(text);
+ const choices = json.choices as Array<{
+ delta: {
+ content: string | null;
+ tool_calls: ChatMessageTool[];
+ reasoning_content: string | null;
+ };
+ }>;
+ const tool_calls = choices[0]?.delta?.tool_calls;
+ if (tool_calls?.length > 0) {
+ const index = tool_calls[0]?.index;
+ const id = tool_calls[0]?.id;
+ const args = tool_calls[0]?.function?.arguments;
+ if (id) {
+ runTools.push({
+ id,
+ type: tool_calls[0]?.type,
+ function: {
+ name: tool_calls[0]?.function?.name as string,
+ arguments: args,
+ },
+ });
+ } else {
+ // @ts-ignore
+ runTools[index]["function"]["arguments"] += args;
+ }
}
+ const reasoning = choices[0]?.delta?.reasoning_content;
+ const content = choices[0]?.delta?.content;
+ // Skip if both content and reasoning_content are empty or null
if (
- !res.ok ||
- !res.headers
- .get("content-type")
- ?.startsWith(EventStreamContentType) ||
- res.status !== 200
+ (!reasoning || reasoning.trim().length === 0) &&
+ (!content || content.trim().length === 0)
) {
- const responseTexts = [responseText];
- let extraInfo = await res.clone().text();
- try {
- const resJson = await res.clone().json();
- extraInfo = prettyObject(resJson);
- } catch {}
-
- if (res.status === 401) {
- responseTexts.push(Locale.Error.Unauthorized);
- }
-
- if (extraInfo) {
- responseTexts.push(extraInfo);
- }
-
- responseText = responseTexts.join("\n\n");
-
- return finish();
+ return {
+ isThinking: false,
+ content: "",
+ };
}
- },
- onmessage(msg) {
- if (msg.data === "[DONE]" || finished) {
- return finish();
- }
- const text = msg.data;
- try {
- const json = JSON.parse(text);
- const choices = json.choices as Array<{
- delta: { content: string };
- }>;
- const delta = choices[0]?.delta?.content;
- if (delta) {
- remainText += delta;
- }
- } catch (e) {
- console.error("[Request] parse error", text, msg);
+
+ if (reasoning && reasoning.trim().length > 0) {
+ return {
+ isThinking: true,
+ content: reasoning,
+ };
+ } else if (content && content.trim().length > 0) {
+ return {
+ isThinking: false,
+ content: content,
+ };
}
+
+ return {
+ isThinking: false,
+ content: "",
+ };
},
- onclose() {
- finish();
+ // processToolMessage, include tool_calls message and tool call results
+ (
+ requestPayload: RequestPayload,
+ toolCallMessage: any,
+ toolCallResult: any[],
+ ) => {
+ // @ts-ignore
+ requestPayload?.messages?.splice(
+ // @ts-ignore
+ requestPayload?.messages?.length,
+ 0,
+ toolCallMessage,
+ ...toolCallResult,
+ );
},
- onerror(e) {
- options.onError?.(e);
- throw e;
- },
- openWhenHidden: true,
- });
+ options,
+ );
} else {
const res = await fetch(chatPath, chatPayload);
clearTimeout(requestTimeoutId);