feat: realtime config

app/components/chat.tsx

@@ -793,11 +793,13 @@ export function ChatActions(props: {
         )}
       </>
       <div className={styles["chat-input-actions-end"]}>
-        <ChatAction
+        {config.realtimeConfig.enable && (
-          onClick={() => props.setShowChatSidePanel(true)}
+          <ChatAction
-          text={"Realtime Chat"}
+            onClick={() => props.setShowChatSidePanel(true)}
-          icon={<HeadphoneIcon />}
+            text={"Realtime Chat"}
-        />
+            icon={<HeadphoneIcon />}
+          />
+        )}
       </div>
     </div>
   );

app/components/realtime-chat/realtime-chat.tsx

@@ -1,4 +1,3 @@
-import { useDebouncedCallback } from "use-debounce";
 import VoiceIcon from "@/app/icons/voice.svg";
 import VoiceOffIcon from "@/app/icons/voice-off.svg";
 import PowerIcon from "@/app/icons/power.svg";
@@ -8,12 +7,7 @@ import clsx from "clsx";
 
 import { useState, useRef, useEffect } from "react";
 
-import {
+import { useChatStore, createMessage, useAppConfig } from "@/app/store";
-  useAccessStore,
-  useChatStore,
-  ChatMessage,
-  createMessage,
-} from "@/app/store";
 
 import { IconButton } from "@/app/components/button";
 
@@ -23,7 +17,6 @@ import {
   RTInputAudioItem,
   RTResponse,
   TurnDetection,
-  Voice,
 } from "rt-client";
 import { AudioHandler } from "@/app/lib/audio";
 import { uploadImage } from "@/app/utils/chat";
@@ -39,41 +32,40 @@ export function RealtimeChat({
   onStartVoice,
   onPausedVoice,
 }: RealtimeChatProps) {
-  const currentItemId = useRef<string>("");
-  const currentBotMessage = useRef<ChatMessage | null>();
-  const currentUserMessage = useRef<ChatMessage | null>();
-  const accessStore = useAccessStore.getState();
   const chatStore = useChatStore();
   const session = chatStore.currentSession();
-
+  const config = useAppConfig();
   const [status, setStatus] = useState("");
   const [isRecording, setIsRecording] = useState(false);
   const [isConnected, setIsConnected] = useState(false);
   const [isConnecting, setIsConnecting] = useState(false);
   const [modality, setModality] = useState("audio");
-  const [isAzure, setIsAzure] = useState(false);
-  const [endpoint, setEndpoint] = useState("");
-  const [deployment, setDeployment] = useState("");
   const [useVAD, setUseVAD] = useState(true);
-  const [voice, setVoice] = useState<Voice>("alloy");
-  const [temperature, setTemperature] = useState(0.9);
 
   const clientRef = useRef<RTClient | null>(null);
   const audioHandlerRef = useRef<AudioHandler | null>(null);
+  const initRef = useRef(false);
 
-  const apiKey = accessStore.openaiApiKey;
+  const temperature = config.realtimeConfig.temperature;
+  const apiKey = config.realtimeConfig.apiKey;
+  const model = config.realtimeConfig.model;
+  const azure = config.realtimeConfig.provider === "Azure";
+  const azureEndpoint = config.realtimeConfig.azure.endpoint;
+  const azureDeployment = config.realtimeConfig.azure.deployment;
+  const voice = config.realtimeConfig.voice;
 
   const handleConnect = async () => {
     if (isConnecting) return;
     if (!isConnected) {
       try {
         setIsConnecting(true);
-        clientRef.current = isAzure
+        clientRef.current = azure
-          ? new RTClient(new URL(endpoint), { key: apiKey }, { deployment })
+          ? new RTClient(
-          : new RTClient(
+              new URL(azureEndpoint),
               { key: apiKey },
-              { model: "gpt-4o-realtime-preview-2024-10-01" },
+              { deployment: azureDeployment },
-            );
+            )
+          : new RTClient({ key: apiKey }, { model });
         const modalities: Modality[] =
           modality === "audio" ? ["text", "audio"] : ["text"];
         const turnDetection: TurnDetection = useVAD
@@ -191,7 +183,6 @@ export function RealtimeChat({
         const blob = audioHandlerRef.current?.savePlayFile();
         uploadImage(blob!).then((audio_url) => {
           botMessage.audio_url = audio_url;
-          // botMessage.date = new Date().toLocaleString();
           // update text and audio_url
           chatStore.updateTargetSession(session, (session) => {
             session.messages = session.messages.concat();
@@ -258,31 +249,32 @@ export function RealtimeChat({
     }
   };
 
-  useEffect(
+  useEffect(() => {
-    useDebouncedCallback(() => {
+    // 防止重复初始化
-      const initAudioHandler = async () => {
+    if (initRef.current) return;
-        const handler = new AudioHandler();
+    initRef.current = true;
-        await handler.initialize();
-        audioHandlerRef.current = handler;
-        await handleConnect();
-        await toggleRecording();
-      };
 
-      initAudioHandler().catch((error) => {
+    const initAudioHandler = async () => {
-        setStatus(error);
+      const handler = new AudioHandler();
-        console.error(error);
+      await handler.initialize();
-      });
+      audioHandlerRef.current = handler;
+      await handleConnect();
+      await toggleRecording();
+    };
 
-      return () => {
+    initAudioHandler().catch((error) => {
-        if (isRecording) {
+      setStatus(error);
-          toggleRecording();
+      console.error(error);
-        }
+    });
-        audioHandlerRef.current?.close().catch(console.error);
+
-        disconnect();
+    return () => {
-      };
+      if (isRecording) {
-    }),
+        toggleRecording();
-    [],
+      }
-  );
+      audioHandlerRef.current?.close().catch(console.error);
+      disconnect();
+    };
+  }, []);
 
   // update session params
   useEffect(() => {
@@ -304,7 +296,7 @@ export function RealtimeChat({
     <div className={styles["realtime-chat"]}>
       <div
         className={clsx(styles["circle-mic"], {
-          [styles["pulse"]]: true,
+          [styles["pulse"]]: isRecording,
         })}
       >
         <div className={styles["icon-center"]}></div>
@@ -312,10 +304,11 @@ export function RealtimeChat({
       <div className={styles["bottom-icons"]}>
         <div>
           <IconButton
-            icon={isRecording ? <VoiceOffIcon /> : <VoiceIcon />}
+            icon={isRecording ? <VoiceIcon /> : <VoiceOffIcon />}
             onClick={toggleRecording}
             disabled={!isConnected}
-            type={isRecording ? "danger" : isConnected ? "primary" : null}
+            shadow
+            bordered
           />
         </div>
         <div className={styles["icon-center"]}>{status}</div>
@@ -323,7 +316,8 @@ export function RealtimeChat({
           <IconButton
             icon={<PowerIcon />}
             onClick={handleClose}
-            type={isConnecting || isConnected ? "danger" : "primary"}
+            shadow
+            bordered
           />
         </div>
       </div>

app/components/realtime-chat/realtime-config.tsx

@@ -0,0 +1,173 @@
+import { RealtimeConfig } from "@/app/store";
+
+import Locale from "@/app/locales";
+import { ListItem, Select, PasswordInput } from "@/app/components/ui-lib";
+
+import { InputRange } from "@/app/components/input-range";
+import { Voice } from "rt-client";
+import { ServiceProvider } from "@/app/constant";
+
+const providers = [ServiceProvider.OpenAI, ServiceProvider.Azure];
+
+const models = ["gpt-4o-realtime-preview-2024-10-01"];
+
+const voice = ["alloy", "shimmer", "echo"];
+
+export function RealtimeConfigList(props: {
+  realtimeConfig: RealtimeConfig;
+  updateConfig: (updater: (config: RealtimeConfig) => void) => void;
+}) {
+  const azureConfigComponent = props.realtimeConfig.provider ===
+    ServiceProvider.Azure && (
+    <>
+      <ListItem
+        title={Locale.Settings.Realtime.Azure.Endpoint.Title}
+        subTitle={Locale.Settings.Realtime.Azure.Endpoint.SubTitle}
+      >
+        <input
+          value={props.realtimeConfig?.azure?.endpoint}
+          type="text"
+          placeholder={Locale.Settings.Realtime.Azure.Endpoint.Title}
+          onChange={(e) => {
+            props.updateConfig(
+              (config) => (config.azure.endpoint = e.currentTarget.value),
+            );
+          }}
+        />
+      </ListItem>
+      <ListItem
+        title={Locale.Settings.Realtime.Azure.Deployment.Title}
+        subTitle={Locale.Settings.Realtime.Azure.Deployment.SubTitle}
+      >
+        <input
+          value={props.realtimeConfig?.azure?.deployment}
+          type="text"
+          placeholder={Locale.Settings.Realtime.Azure.Deployment.Title}
+          onChange={(e) => {
+            props.updateConfig(
+              (config) => (config.azure.deployment = e.currentTarget.value),
+            );
+          }}
+        />
+      </ListItem>
+    </>
+  );
+
+  return (
+    <>
+      <ListItem
+        title={Locale.Settings.Realtime.Enable.Title}
+        subTitle={Locale.Settings.Realtime.Enable.SubTitle}
+      >
+        <input
+          type="checkbox"
+          checked={props.realtimeConfig.enable}
+          onChange={(e) =>
+            props.updateConfig(
+              (config) => (config.enable = e.currentTarget.checked),
+            )
+          }
+        ></input>
+      </ListItem>
+
+      {props.realtimeConfig.enable && (
+        <>
+          <ListItem
+            title={Locale.Settings.Realtime.Provider.Title}
+            subTitle={Locale.Settings.Realtime.Provider.SubTitle}
+          >
+            <Select
+              aria-label={Locale.Settings.Realtime.Provider.Title}
+              value={props.realtimeConfig.provider}
+              onChange={(e) => {
+                props.updateConfig(
+                  (config) =>
+                    (config.provider = e.target.value as ServiceProvider),
+                );
+              }}
+            >
+              {providers.map((v, i) => (
+                <option value={v} key={i}>
+                  {v}
+                </option>
+              ))}
+            </Select>
+          </ListItem>
+          <ListItem
+            title={Locale.Settings.Realtime.Model.Title}
+            subTitle={Locale.Settings.Realtime.Model.SubTitle}
+          >
+            <Select
+              aria-label={Locale.Settings.Realtime.Model.Title}
+              value={props.realtimeConfig.model}
+              onChange={(e) => {
+                props.updateConfig((config) => (config.model = e.target.value));
+              }}
+            >
+              {models.map((v, i) => (
+                <option value={v} key={i}>
+                  {v}
+                </option>
+              ))}
+            </Select>
+          </ListItem>
+          <ListItem
+            title={Locale.Settings.Realtime.ApiKey.Title}
+            subTitle={Locale.Settings.Realtime.ApiKey.SubTitle}
+          >
+            <PasswordInput
+              aria={Locale.Settings.ShowPassword}
+              aria-label={Locale.Settings.Realtime.ApiKey.Title}
+              value={props.realtimeConfig.apiKey}
+              type="text"
+              placeholder={Locale.Settings.Realtime.ApiKey.Placeholder}
+              onChange={(e) => {
+                props.updateConfig(
+                  (config) => (config.apiKey = e.currentTarget.value),
+                );
+              }}
+            />
+          </ListItem>
+          {azureConfigComponent}
+          <ListItem
+            title={Locale.Settings.TTS.Voice.Title}
+            subTitle={Locale.Settings.TTS.Voice.SubTitle}
+          >
+            <Select
+              value={props.realtimeConfig.voice}
+              onChange={(e) => {
+                props.updateConfig(
+                  (config) => (config.voice = e.currentTarget.value as Voice),
+                );
+              }}
+            >
+              {voice.map((v, i) => (
+                <option value={v} key={i}>
+                  {v}
+                </option>
+              ))}
+            </Select>
+          </ListItem>
+          <ListItem
+            title={Locale.Settings.Realtime.Temperature.Title}
+            subTitle={Locale.Settings.Realtime.Temperature.SubTitle}
+          >
+            <InputRange
+              aria={Locale.Settings.Temperature.Title}
+              value={props.realtimeConfig?.temperature?.toFixed(1)}
+              min="0.6"
+              max="1"
+              step="0.1"
+              onChange={(e) => {
+                props.updateConfig(
+                  (config) =>
+                    (config.temperature = e.currentTarget.valueAsNumber),
+                );
+              }}
+            ></InputRange>
+          </ListItem>
+        </>
+      )}
+    </>
+  );
+}

app/components/settings.tsx

@@ -85,6 +85,7 @@ import { nanoid } from "nanoid";
 import { useMaskStore } from "../store/mask";
 import { ProviderType } from "../utils/cloud";
 import { TTSConfigList } from "./tts-config";
+import { RealtimeConfigList } from "./realtime-chat/realtime-config";
 
 function EditPromptModal(props: { id: string; onClose: () => void }) {
   const promptStore = usePromptStore();
@@ -1799,7 +1800,18 @@ export function Settings() {
         {shouldShowPromptModal && (
           <UserPromptModal onClose={() => setShowPromptModal(false)} />
         )}
-
+        <List>
+          <RealtimeConfigList
+            realtimeConfig={config.realtimeConfig}
+            updateConfig={(updater) => {
+              const realtimeConfig = { ...config.realtimeConfig };
+              updater(realtimeConfig);
+              config.update(
+                (config) => (config.realtimeConfig = realtimeConfig),
+              );
+            }}
+          />
+        </List>
         <List>
           <TTSConfigList
             ttsConfig={config.ttsConfig}

app/locales/cn.ts

@@ -562,6 +562,39 @@ const cn = {
         SubTitle: "生成语音的速度",
       },
     },
+    Realtime: {
+      Enable: {
+        Title: "实时聊天",
+        SubTitle: "开启实时聊天功能",
+      },
+      Provider: {
+        Title: "模型服务商",
+        SubTitle: "切换不同的服务商",
+      },
+      Model: {
+        Title: "模型",
+        SubTitle: "选择一个模型",
+      },
+      ApiKey: {
+        Title: "API Key",
+        SubTitle: "API Key",
+        Placeholder: "API Key",
+      },
+      Azure: {
+        Endpoint: {
+          Title: "接口地址",
+          SubTitle: "接口地址",
+        },
+        Deployment: {
+          Title: "部署",
+          SubTitle: "Deployment",
+        },
+      },
+      Temperature: {
+        Title: "随机性 (temperature)",
+        SubTitle: "值越大，回复越随机",
+      },
+    },
   },
   Store: {
     DefaultTopic: "新的聊天",

app/locales/en.ts

@@ -570,6 +570,39 @@ const en: LocaleType = {
       },
       Engine: "TTS Engine",
     },
+    Realtime: {
+      Enable: {
+        Title: "Realtime Chat",
+        SubTitle: "Enable realtime chat feature",
+      },
+      Provider: {
+        Title: "Model Provider",
+        SubTitle: "Switch between different providers",
+      },
+      Model: {
+        Title: "Model",
+        SubTitle: "Select a model",
+      },
+      ApiKey: {
+        Title: "API Key",
+        SubTitle: "API Key",
+        Placeholder: "API Key",
+      },
+      Azure: {
+        Endpoint: {
+          Title: "Endpoint",
+          SubTitle: "Endpoint",
+        },
+        Deployment: {
+          Title: "Deployment",
+          SubTitle: "Deployment",
+        },
+      },
+      Temperature: {
+        Title: "Randomness (temperature)",
+        SubTitle: "Higher values result in more random responses",
+      },
+    },
   },
   Store: {
     DefaultTopic: "New Conversation",

app/store/config.ts

@@ -15,6 +15,7 @@ import {
   ServiceProvider,
 } from "../constant";
 import { createPersistStore } from "../utils/store";
+import type { Voice } from "rt-client";
 
 export type ModelType = (typeof DEFAULT_MODELS)[number]["name"];
 export type TTSModelType = (typeof DEFAULT_TTS_MODELS)[number];
@@ -90,12 +91,26 @@ export const DEFAULT_CONFIG = {
     voice: DEFAULT_TTS_VOICE,
     speed: 1.0,
   },
+
+  realtimeConfig: {
+    enable: false,
+    provider: "OpenAI" as ServiceProvider,
+    model: "gpt-4o-realtime-preview-2024-10-01",
+    apiKey: "",
+    azure: {
+      endpoint: "",
+      deployment: "",
+    },
+    temperature: 0.9,
+    voice: "alloy" as Voice,
+  },
 };
 
 export type ChatConfig = typeof DEFAULT_CONFIG;
 
 export type ModelConfig = ChatConfig["modelConfig"];
 export type TTSConfig = ChatConfig["ttsConfig"];
+export type RealtimeConfig = ChatConfig["realtimeConfig"];
 
 export function limitNumber(
   x: number,