Merge pull request #4942 from ConnectAI-E/feature/alibaba

feat: qwen
2024-07-09 21:51:16 +08:00 · 2024-07-09 21:51:16 +08:00 · 844025ec14
parent 94bc91c554 044c16da4c
commit 844025ec14
13 changed files with 570 additions and 1 deletions
--- a/README.md
+++ b/README.md
@ -233,6 +233,14 @@ ByteDance Api Key.

 ByteDance Api Url.

+### `ALIBABA_API_KEY` (optional)
+
+Alibaba Cloud Api Key.
+
+### `ALIBABA_URL` (optional)
+
+Alibaba Cloud Api Url.
+
 ### `HIDE_USER_API_KEY` (optional)

 > Default: Empty
--- a/README_CN.md
+++ b/README_CN.md
@ -147,6 +147,14 @@ ByteDance Api Key.

 ByteDance Api Url.

+### `ALIBABA_API_KEY` (可选)
+
+阿里云（千问）Api Key.
+
+### `ALIBABA_URL` (可选)
+
+阿里云（千问）Api Url.
+
 ### `HIDE_USER_API_KEY` （可选）

 如果你不想让用户自行填入 API Key，将此环境变量设置为 1 即可。
--- a/app/api/alibaba/[...path]/route.ts
+++ b/app/api/alibaba/[...path]/route.ts
@ -0,0 +1,170 @@
+import { getServerSideConfig } from "@/app/config/server";
+import {
+  Alibaba,
+  ALIBABA_BASE_URL,
+  ApiPath,
+  ModelProvider,
+  ServiceProvider,
+} from "@/app/constant";
+import { prettyObject } from "@/app/utils/format";
+import { NextRequest, NextResponse } from "next/server";
+import { auth } from "@/app/api/auth";
+import { isModelAvailableInServer } from "@/app/utils/model";
+import type { RequestPayload } from "@/app/client/platforms/openai";
+
+const serverConfig = getServerSideConfig();
+
+async function handle(
+  req: NextRequest,
+  { params }: { params: { path: string[] } },
+) {
+  console.log("[Alibaba Route] params ", params);
+
+  if (req.method === "OPTIONS") {
+    return NextResponse.json({ body: "OK" }, { status: 200 });
+  }
+
+  const authResult = auth(req, ModelProvider.Qwen);
+  if (authResult.error) {
+    return NextResponse.json(authResult, {
+      status: 401,
+    });
+  }
+
+  try {
+    const response = await request(req);
+    return response;
+  } catch (e) {
+    console.error("[Alibaba] ", e);
+    return NextResponse.json(prettyObject(e));
+  }
+}
+
+export const GET = handle;
+export const POST = handle;
+
+export const runtime = "edge";
+export const preferredRegion = [
+  "arn1",
+  "bom1",
+  "cdg1",
+  "cle1",
+  "cpt1",
+  "dub1",
+  "fra1",
+  "gru1",
+  "hnd1",
+  "iad1",
+  "icn1",
+  "kix1",
+  "lhr1",
+  "pdx1",
+  "sfo1",
+  "sin1",
+  "syd1",
+];
+
+async function request(req: NextRequest) {
+  const controller = new AbortController();
+
+  // alibaba use base url or just remove the path
+  let path = `${req.nextUrl.pathname}`.replaceAll(ApiPath.Alibaba, "");
+
+  let baseUrl = serverConfig.alibabaUrl || ALIBABA_BASE_URL;
+
+  if (!baseUrl.startsWith("http")) {
+    baseUrl = `https://${baseUrl}`;
+  }
+
+  if (baseUrl.endsWith("/")) {
+    baseUrl = baseUrl.slice(0, -1);
+  }
+
+  console.log("[Proxy] ", path);
+  console.log("[Base Url]", baseUrl);
+
+  const timeoutId = setTimeout(
+    () => {
+      controller.abort();
+    },
+    10 * 60 * 1000,
+  );
+
+  const fetchUrl = `${baseUrl}${path}`;
+
+  const clonedBody = await req.text();
+
+  const { messages, model, stream, top_p, ...rest } = JSON.parse(
+    clonedBody,
+  ) as RequestPayload;
+
+  const requestBody = {
+    model,
+    input: {
+      messages,
+    },
+    parameters: {
+      ...rest,
+      top_p: top_p === 1 ? 0.99 : top_p, // qwen top_p is should be < 1
+      result_format: "message",
+      incremental_output: true,
+    },
+  };
+
+  const fetchOptions: RequestInit = {
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: req.headers.get("Authorization") ?? "",
+      "X-DashScope-SSE": stream ? "enable" : "disable",
+    },
+    method: req.method,
+    body: JSON.stringify(requestBody),
+    redirect: "manual",
+    // @ts-ignore
+    duplex: "half",
+    signal: controller.signal,
+  };
+
+  // #1815 try to refuse some request to some models
+  if (serverConfig.customModels && req.body) {
+    try {
+      // not undefined and is false
+      if (
+        isModelAvailableInServer(
+          serverConfig.customModels,
+          model as string,
+          ServiceProvider.Alibaba as string,
+        )
+      ) {
+        return NextResponse.json(
+          {
+            error: true,
+            message: `you are not allowed to use ${model} model`,
+          },
+          {
+            status: 403,
+          },
+        );
+      }
+    } catch (e) {
+      console.error(`[Alibaba] filter`, e);
+    }
+  }
+  try {
+    const res = await fetch(fetchUrl, fetchOptions);
+
+    // to prevent browser prompt for credentials
+    const newHeaders = new Headers(res.headers);
+    newHeaders.delete("www-authenticate");
+    // to disable nginx buffering
+    newHeaders.set("X-Accel-Buffering", "no");
+
+    return new Response(res.body, {
+      status: res.status,
+      statusText: res.statusText,
+      headers: newHeaders,
+    });
+  } finally {
+    clearTimeout(timeoutId);
+  }
+}
--- a/app/api/auth.ts
+++ b/app/api/auth.ts
@ -79,6 +79,9 @@ export function auth(req: NextRequest, modelProvider: ModelProvider) {
      case ModelProvider.Ernie:
        systemApiKey = serverConfig.baiduApiKey;
        break;
+      case ModelProvider.Qwen:
+        systemApiKey = serverConfig.alibabaApiKey;
+        break;
      case ModelProvider.GPT:
      default:
        if (req.nextUrl.pathname.includes("azure/deployments")) {
--- a/app/client/api.ts
+++ b/app/client/api.ts
@ -11,6 +11,7 @@ import { GeminiProApi } from "./platforms/google";
 import { ClaudeApi } from "./platforms/anthropic";
 import { ErnieApi } from "./platforms/baidu";
 import { DoubaoApi } from "./platforms/bytedance";
+import { QwenApi } from "./platforms/alibaba";

 export const ROLES = ["system", "user", "assistant"] as const;
 export type MessageRole = (typeof ROLES)[number];
@ -113,6 +114,9 @@ export class ClientApi {
      case ModelProvider.Doubao:
        this.llm = new DoubaoApi();
        break;
+      case ModelProvider.Qwen:
+        this.llm = new QwenApi();
+        break;
      default:
        this.llm = new ChatGPTApi();
    }
@ -181,6 +185,7 @@ export function getHeaders() {
    const isAnthropic = modelConfig.providerName === ServiceProvider.Anthropic;
    const isBaidu = modelConfig.providerName == ServiceProvider.Baidu;
    const isByteDance = modelConfig.providerName === ServiceProvider.ByteDance;
+    const isAlibaba = modelConfig.providerName === ServiceProvider.Alibaba;
    const isEnabledAccessControl = accessStore.enabledAccessControl();
    const apiKey = isGoogle
      ? accessStore.googleApiKey
@ -190,6 +195,8 @@ export function getHeaders() {
      ? accessStore.anthropicApiKey
      : isByteDance
      ? accessStore.bytedanceApiKey
+      : isAlibaba
+      ? accessStore.alibabaApiKey
      : accessStore.openaiApiKey;
    return {
      isGoogle,
@ -197,6 +204,7 @@ export function getHeaders() {
      isAnthropic,
      isBaidu,
      isByteDance,
+      isAlibaba,
      apiKey,
      isEnabledAccessControl,
    };
@ -253,6 +261,8 @@ export function getClientApi(provider: ServiceProvider): ClientApi {
      return new ClientApi(ModelProvider.Ernie);
    case ServiceProvider.ByteDance:
      return new ClientApi(ModelProvider.Doubao);
+    case ServiceProvider.Alibaba:
+      return new ClientApi(ModelProvider.Qwen);
    default:
      return new ClientApi(ModelProvider.GPT);
  }
--- a/app/client/platforms/alibaba.ts
+++ b/app/client/platforms/alibaba.ts
@ -0,0 +1,255 @@
+"use client";
+import {
+  ApiPath,
+  Alibaba,
+  ALIBABA_BASE_URL,
+  REQUEST_TIMEOUT_MS,
+} from "@/app/constant";
+import { useAccessStore, useAppConfig, useChatStore } from "@/app/store";
+
+import {
+  ChatOptions,
+  getHeaders,
+  LLMApi,
+  LLMModel,
+  MultimodalContent,
+} from "../api";
+import Locale from "../../locales";
+import {
+  EventStreamContentType,
+  fetchEventSource,
+} from "@fortaine/fetch-event-source";
+import { prettyObject } from "@/app/utils/format";
+import { getClientConfig } from "@/app/config/client";
+import { getMessageTextContent, isVisionModel } from "@/app/utils";
+
+export interface OpenAIListModelResponse {
+  object: string;
+  data: Array<{
+    id: string;
+    object: string;
+    root: string;
+  }>;
+}
+
+interface RequestPayload {
+  messages: {
+    role: "system" | "user" | "assistant";
+    content: string | MultimodalContent[];
+  }[];
+  stream?: boolean;
+  model: string;
+  temperature: number;
+  presence_penalty: number;
+  frequency_penalty: number;
+  top_p: number;
+  max_tokens?: number;
+}
+
+export class QwenApi implements LLMApi {
+  path(path: string): string {
+    const accessStore = useAccessStore.getState();
+
+    let baseUrl = "";
+
+    if (accessStore.useCustomConfig) {
+      baseUrl = accessStore.alibabaUrl;
+    }
+
+    if (baseUrl.length === 0) {
+      const isApp = !!getClientConfig()?.isApp;
+      baseUrl = isApp ? ALIBABA_BASE_URL : ApiPath.Alibaba;
+    }
+
+    if (baseUrl.endsWith("/")) {
+      baseUrl = baseUrl.slice(0, baseUrl.length - 1);
+    }
+    if (!baseUrl.startsWith("http") && !baseUrl.startsWith(ApiPath.Alibaba)) {
+      baseUrl = "https://" + baseUrl;
+    }
+
+    console.log("[Proxy Endpoint] ", baseUrl, path);
+
+    return [baseUrl, path].join("/");
+  }
+
+  extractMessage(res: any) {
+    return res?.output?.choices?.at(0)?.message?.content ?? "";
+  }
+
+  async chat(options: ChatOptions) {
+    const messages = options.messages.map((v) => ({
+      role: v.role,
+      content: getMessageTextContent(v),
+    }));
+
+    const modelConfig = {
+      ...useAppConfig.getState().modelConfig,
+      ...useChatStore.getState().currentSession().mask.modelConfig,
+      ...{
+        model: options.config.model,
+      },
+    };
+
+    const requestPayload: RequestPayload = {
+      messages,
+      stream: options.config.stream,
+      model: modelConfig.model,
+      temperature: modelConfig.temperature,
+      presence_penalty: modelConfig.presence_penalty,
+      frequency_penalty: modelConfig.frequency_penalty,
+      top_p: modelConfig.top_p,
+    };
+
+    const shouldStream = !!options.config.stream;
+    const controller = new AbortController();
+    options.onController?.(controller);
+
+    try {
+      const chatPath = this.path(Alibaba.ChatPath);
+      const chatPayload = {
+        method: "POST",
+        body: JSON.stringify(requestPayload),
+        signal: controller.signal,
+        headers: getHeaders(),
+      };
+
+      // make a fetch request
+      const requestTimeoutId = setTimeout(
+        () => controller.abort(),
+        REQUEST_TIMEOUT_MS,
+      );
+
+      if (shouldStream) {
+        let responseText = "";
+        let remainText = "";
+        let finished = false;
+
+        // animate response to make it looks smooth
+        function animateResponseText() {
+          if (finished || controller.signal.aborted) {
+            responseText += remainText;
+            console.log("[Response Animation] finished");
+            if (responseText?.length === 0) {
+              options.onError?.(new Error("empty response from server"));
+            }
+            return;
+          }
+
+          if (remainText.length > 0) {
+            const fetchCount = Math.max(1, Math.round(remainText.length / 60));
+            const fetchText = remainText.slice(0, fetchCount);
+            responseText += fetchText;
+            remainText = remainText.slice(fetchCount);
+            options.onUpdate?.(responseText, fetchText);
+          }
+
+          requestAnimationFrame(animateResponseText);
+        }
+
+        // start animaion
+        animateResponseText();
+
+        const finish = () => {
+          if (!finished) {
+            finished = true;
+            options.onFinish(responseText + remainText);
+          }
+        };
+
+        controller.signal.onabort = finish;
+
+        fetchEventSource(chatPath, {
+          ...chatPayload,
+          async onopen(res) {
+            clearTimeout(requestTimeoutId);
+            const contentType = res.headers.get("content-type");
+            console.log(
+              "[Alibaba] request response content type: ",
+              contentType,
+            );
+
+            if (contentType?.startsWith("text/plain")) {
+              responseText = await res.clone().text();
+              return finish();
+            }
+
+            if (
+              !res.ok ||
+              !res.headers
+                .get("content-type")
+                ?.startsWith(EventStreamContentType) ||
+              res.status !== 200
+            ) {
+              const responseTexts = [responseText];
+              let extraInfo = await res.clone().text();
+              try {
+                const resJson = await res.clone().json();
+                extraInfo = prettyObject(resJson);
+              } catch {}
+
+              if (res.status === 401) {
+                responseTexts.push(Locale.Error.Unauthorized);
+              }
+
+              if (extraInfo) {
+                responseTexts.push(extraInfo);
+              }
+
+              responseText = responseTexts.join("\n\n");
+
+              return finish();
+            }
+          },
+          onmessage(msg) {
+            if (msg.data === "[DONE]" || finished) {
+              return finish();
+            }
+            const text = msg.data;
+            try {
+              const json = JSON.parse(text);
+              const choices = json.output.choices as Array<{
+                message: { content: string };
+              }>;
+              const delta = choices[0]?.message?.content;
+              if (delta) {
+                remainText += delta;
+              }
+            } catch (e) {
+              console.error("[Request] parse error", text, msg);
+            }
+          },
+          onclose() {
+            finish();
+          },
+          onerror(e) {
+            options.onError?.(e);
+            throw e;
+          },
+          openWhenHidden: true,
+        });
+      } else {
+        const res = await fetch(chatPath, chatPayload);
+        clearTimeout(requestTimeoutId);
+
+        const resJson = await res.json();
+        const message = this.extractMessage(resJson);
+        options.onFinish(message);
+      }
+    } catch (e) {
+      console.log("[Request] failed to make a chat request", e);
+      options.onError?.(e as Error);
+    }
+  }
+  async usage() {
+    return {
+      used: 0,
+      total: 0,
+    };
+  }
+
+  async models(): Promise<LLMModel[]> {
+    return [];
+  }
+}
+export { Alibaba };
--- a/app/client/platforms/openai.ts
+++ b/app/client/platforms/openai.ts
@ -42,7 +42,7 @@ export interface OpenAIListModelResponse {
  }>;
 }

-interface RequestPayload {
+export interface RequestPayload {
  messages: {
    role: "system" | "user" | "assistant";
    content: string | MultimodalContent[];
--- a/app/components/settings.tsx
+++ b/app/components/settings.tsx
@ -55,6 +55,7 @@ import {
  Azure,
  Baidu,
  ByteDance,
+  Alibaba,
  Google,
  OPENAI_BASE_URL,
  Path,
@ -1295,6 +1296,50 @@ export function Settings() {
                      </ListItem>
                    </>
                  )}
+
+                  {accessStore.provider === ServiceProvider.Alibaba && (
+                    <>
+                      <ListItem
+                        title={Locale.Settings.Access.Alibaba.Endpoint.Title}
+                        subTitle={
+                          Locale.Settings.Access.Alibaba.Endpoint.SubTitle +
+                          Alibaba.ExampleEndpoint
+                        }
+                      >
+                        <input
+                          type="text"
+                          value={accessStore.alibabaUrl}
+                          placeholder={Alibaba.ExampleEndpoint}
+                          onChange={(e) =>
+                            accessStore.update(
+                              (access) =>
+                                (access.alibabaUrl = e.currentTarget.value),
+                            )
+                          }
+                        ></input>
+                      </ListItem>
+                      <ListItem
+                        title={Locale.Settings.Access.Alibaba.ApiKey.Title}
+                        subTitle={
+                          Locale.Settings.Access.Alibaba.ApiKey.SubTitle
+                        }
+                      >
+                        <PasswordInput
+                          value={accessStore.alibabaApiKey}
+                          type="text"
+                          placeholder={
+                            Locale.Settings.Access.Alibaba.ApiKey.Placeholder
+                          }
+                          onChange={(e) => {
+                            accessStore.update(
+                              (access) =>
+                                (access.alibabaApiKey = e.currentTarget.value),
+                            );
+                          }}
+                        />
+                      </ListItem>
+                    </>
+                  )}
                </>
              )}
            </>
--- a/app/config/server.ts
+++ b/app/config/server.ts
@ -49,6 +49,10 @@ declare global {
      BYTEDANCE_URL?: string;
      BYTEDANCE_API_KEY?: string;

+      // alibaba only
+      ALIBABA_URL?: string;
+      ALIBABA_API_KEY?: string;
+
      // custom template for preprocessing user input
      DEFAULT_INPUT_TEMPLATE?: string;
    }
@ -106,8 +110,10 @@ export const getServerSideConfig = () => {
  const isAzure = !!process.env.AZURE_URL;
  const isGoogle = !!process.env.GOOGLE_API_KEY;
  const isAnthropic = !!process.env.ANTHROPIC_API_KEY;
+
  const isBaidu = !!process.env.BAIDU_API_KEY;
  const isBytedance = !!process.env.BYTEDANCE_API_KEY;
+  const isAlibaba = !!process.env.ALIBABA_API_KEY;
  // const apiKeyEnvVar = process.env.OPENAI_API_KEY ?? "";
  // const apiKeys = apiKeyEnvVar.split(",").map((v) => v.trim());
  // const randomIndex = Math.floor(Math.random() * apiKeys.length);
@ -148,6 +154,10 @@ export const getServerSideConfig = () => {
    bytedanceApiKey: getApiKey(process.env.BYTEDANCE_API_KEY),
    bytedanceUrl: process.env.BYTEDANCE_URL,

+    isAlibaba,
+    alibabaUrl: process.env.ALIBABA_URL,
+    alibabaApiKey: getApiKey(process.env.ALIBABA_API_KEY),
+
    gtmId: process.env.GTM_ID,

    needCode: ACCESS_CODES.size > 0,
--- a/app/constant.ts
+++ b/app/constant.ts
@ -19,6 +19,8 @@ export const BAIDU_OATUH_URL = `${BAIDU_BASE_URL}/oauth/2.0/token`;

 export const BYTEDANCE_BASE_URL = "https://ark.cn-beijing.volces.com";

+export const ALIBABA_BASE_URL = "https://dashscope.aliyuncs.com/api/";
+
 export enum Path {
  Home = "/",
  Chat = "/chat",
@ -35,6 +37,7 @@ export enum ApiPath {
  Anthropic = "/api/anthropic",
  Baidu = "/api/baidu",
  ByteDance = "/api/bytedance",
+  Alibaba = "/api/alibaba",
 }

 export enum SlotID {
@ -80,6 +83,7 @@ export enum ServiceProvider {
  Anthropic = "Anthropic",
  Baidu = "Baidu",
  ByteDance = "ByteDance",
+  Alibaba = "Alibaba",
 }

 export enum ModelProvider {
@ -88,6 +92,7 @@ export enum ModelProvider {
  Claude = "Claude",
  Ernie = "Ernie",
  Doubao = "Doubao",
+  Qwen = "Qwen",
 }

 export const Anthropic = {
@ -137,6 +142,11 @@ export const ByteDance = {
  ChatPath: "api/v3/chat/completions",
 };

+export const Alibaba = {
+  ExampleEndpoint: ALIBABA_BASE_URL,
+  ChatPath: "v1/services/aigc/text-generation/generation",
+};
+
 export const DEFAULT_INPUT_TEMPLATE = `{{input}}`; // input / time / model / lang
 // export const DEFAULT_SYSTEM_TEMPLATE = `
 // You are ChatGPT, a large language model trained by {{ServiceProvider}}.
@ -225,6 +235,16 @@ const bytedanceModels = [
  "Doubao-pro-128k",
 ];

+const alibabaModes = [
+  "qwen-turbo",
+  "qwen-plus",
+  "qwen-max",
+  "qwen-max-0428",
+  "qwen-max-0403",
+  "qwen-max-0107",
+  "qwen-max-longcontext",
+];
+
 export const DEFAULT_MODELS = [
  ...openaiModels.map((name) => ({
    name,
@ -280,6 +300,15 @@ export const DEFAULT_MODELS = [
      providerType: "bytedance",
    },
  })),
+  ...alibabaModes.map((name) => ({
+    name,
+    available: true,
+    provider: {
+      id: "alibaba",
+      providerName: "Alibaba",
+      providerType: "alibaba",
+    },
+  })),
 ] as const;

 export const CHAT_PAGE_SIZE = 15;
--- a/app/locales/cn.ts
+++ b/app/locales/cn.ts
@ -374,6 +374,17 @@ const cn = {
          SubTitle: "样例：",
        },
      },
+      Alibaba: {
+        ApiKey: {
+          Title: "接口密钥",
+          SubTitle: "使用自定义阿里云API Key",
+          Placeholder: "Alibaba Cloud API Key",
+        },
+        Endpoint: {
+          Title: "接口地址",
+          SubTitle: "样例：",
+        },
+      },
      CustomModel: {
        Title: "自定义模型名",
        SubTitle: "增加自定义模型可选项，使用英文逗号隔开",
--- a/app/locales/en.ts
+++ b/app/locales/en.ts
@ -361,6 +361,17 @@ const en: LocaleType = {
          SubTitle: "Example:",
        },
      },
+      Alibaba: {
+        ApiKey: {
+          Title: "Alibaba API Key",
+          SubTitle: "Use a custom Alibaba Cloud API Key",
+          Placeholder: "Alibaba Cloud API Key",
+        },
+        Endpoint: {
+          Title: "Endpoint Address",
+          SubTitle: "Example:",
+        },
+      },
      CustomModel: {
        Title: "Custom Models",
        SubTitle: "Custom model options, seperated by comma",
--- a/app/store/access.ts
+++ b/app/store/access.ts
@ -56,6 +56,10 @@ const DEFAULT_ACCESS_STATE = {
  bytedanceApiKey: "",
  bytedanceUrl: "",

+  // alibaba
+  alibabaUrl: "",
+  alibabaApiKey: "",
+
  // server config
  needCode: true,
  hideUserApiKey: false,
@ -100,6 +104,10 @@ export const useAccessStore = createPersistStore(
      return ensure(get(), ["bytedanceApiKey"]);
    },

+    isValidAlibaba() {
+      return ensure(get(), ["alibabaApiKey"]);
+    },
+
    isAuthorized() {
      this.fetch();

@ -111,6 +119,7 @@ export const useAccessStore = createPersistStore(
        this.isValidAnthropic() ||
        this.isValidBaidu() ||
        this.isValidByteDance() ||
+        this.isValidAlibaba() ||
        !this.enabledAccessControl() ||
        (this.enabledAccessControl() && ensure(get(), ["accessCode"]))
      );