refactor(api): update file_url field in MultimodalContent interface

app/client/api.ts

@@ -29,11 +29,14 @@ export const TTSModels = ["tts-1", "tts-1-hd"] as const;
 export type ChatModel = ModelType;
 
 export interface MultimodalContent {
-  type: "text" | "image_url";
+  type: "text" | "image_url" | "file_url";
   text?: string;
   image_url?: {
     url: string;
   };
+  file_url?: {
+    url: string;
+  };
 }
 
 export interface RequestMessage {

app/components/chat.module.scss

@@ -57,7 +57,7 @@
   display: flex;
   flex-direction: column;
   //row-gap: 11px;
-  height: 64px;
+  max-height: 64px;
 }
 
 .attach-file {

app/components/chat.tsx

@@ -1041,6 +1041,7 @@ function _Chat() {
       .onUserInput(userInput, attachImages, attachFiles)
       .then(() => setIsLoading(false));
     setAttachImages([]);
+    setAttachFiles([]);
     chatStore.setLastInput(userInput);
     setUserInput("");
     setPromptHints([]);

app/store/chat.ts

@@ -350,20 +350,26 @@ export const useChatStore = createPersistStore(
         const modelConfig = session.mask.modelConfig;
 
         //read file content from the url
-        if (attachFiles && attachFiles.length > 0) {
-          content += " file content: \n";
-          for (let i = 0; i < attachFiles.length; i++) {
-            content += await readFileContent(attachFiles[i]);
-          }
-        }
-
         const userContent = fillTemplateWith(content, modelConfig);
         console.log("[User Input] after template: ", userContent);
 
         let mContent: string | MultimodalContent[] = userContent;
+        let displayContent: string | MultimodalContent[] = userContent;
+
+        if (attachFiles && attachFiles.length > 0) {
+          let fileContent = userContent + " file content: \n";
+          for (let i = 0; i < attachFiles.length; i++) {
+            fileContent += await readFileContent(attachFiles[i]);
+          }
 
           if (attachImages && attachImages.length > 0) {
             mContent = [
+              {
+                type: "text",
+                text: fileContent,
+              },
+            ];
+            displayContent = [
               {
                 type: "text",
                 text: userContent,
@@ -379,7 +385,22 @@ export const useChatStore = createPersistStore(
                 };
               }),
             );
+            displayContent = displayContent.concat(
+              attachFiles.map((url) => {
+                return {
+                  type: "file_url",
+                  file_url: {
+                    url: url,
+                  },
+                };
+              }),
+            );
+          } else {
+            mContent = fileContent;
+            displayContent = userContent;
           }
+        }
+
         let userMessage: ChatMessage = createMessage({
           role: "user",
           content: mContent,
@@ -400,7 +421,8 @@ export const useChatStore = createPersistStore(
         get().updateCurrentSession((session) => {
           const savedUserMessage = {
             ...userMessage,
-            content: mContent,
+            //content: mContent,
+            content: displayContent,
           };
           session.messages = session.messages.concat([
             savedUserMessage,