diff --git a/app/client/api.ts b/app/client/api.ts index 8285b4d9f..f0176cb33 100644 --- a/app/client/api.ts +++ b/app/client/api.ts @@ -29,11 +29,14 @@ export const TTSModels = ["tts-1", "tts-1-hd"] as const; export type ChatModel = ModelType; export interface MultimodalContent { - type: "text" | "image_url"; + type: "text" | "image_url" | "file_url"; text?: string; image_url?: { url: string; }; + file_url?: { + url: string; + }; } export interface RequestMessage { diff --git a/app/components/chat.module.scss b/app/components/chat.module.scss index 30f0b94cc..4314370d2 100644 --- a/app/components/chat.module.scss +++ b/app/components/chat.module.scss @@ -57,7 +57,7 @@ display: flex; flex-direction: column; //row-gap: 11px; - height: 64px; + max-height: 64px; } .attach-file { diff --git a/app/components/chat.tsx b/app/components/chat.tsx index eee63faab..439e1d934 100644 --- a/app/components/chat.tsx +++ b/app/components/chat.tsx @@ -1041,6 +1041,7 @@ function _Chat() { .onUserInput(userInput, attachImages, attachFiles) .then(() => setIsLoading(false)); setAttachImages([]); + setAttachFiles([]); chatStore.setLastInput(userInput); setUserInput(""); setPromptHints([]); diff --git a/app/store/chat.ts b/app/store/chat.ts index 4e6f09304..9256e56b5 100644 --- a/app/store/chat.ts +++ b/app/store/chat.ts @@ -350,36 +350,57 @@ export const useChatStore = createPersistStore( const modelConfig = session.mask.modelConfig; //read file content from the url - if (attachFiles && attachFiles.length > 0) { - content += " file content: \n"; - for (let i = 0; i < attachFiles.length; i++) { - content += await readFileContent(attachFiles[i]); - } - } - const userContent = fillTemplateWith(content, modelConfig); console.log("[User Input] after template: ", userContent); let mContent: string | MultimodalContent[] = userContent; + let displayContent: string | MultimodalContent[] = userContent; - if (attachImages && attachImages.length > 0) { - mContent = [ - { - type: "text", - text: userContent, - }, - ]; - mContent = mContent.concat( - attachImages.map((url) => { - return { - type: "image_url", - image_url: { - url: url, - }, - }; - }), - ); + if (attachFiles && attachFiles.length > 0) { + let fileContent = userContent + " file content: \n"; + for (let i = 0; i < attachFiles.length; i++) { + fileContent += await readFileContent(attachFiles[i]); + } + + if (attachImages && attachImages.length > 0) { + mContent = [ + { + type: "text", + text: fileContent, + }, + ]; + displayContent = [ + { + type: "text", + text: userContent, + }, + ]; + mContent = mContent.concat( + attachImages.map((url) => { + return { + type: "image_url", + image_url: { + url: url, + }, + }; + }), + ); + displayContent = displayContent.concat( + attachFiles.map((url) => { + return { + type: "file_url", + file_url: { + url: url, + }, + }; + }), + ); + } else { + mContent = fileContent; + displayContent = userContent; + } } + let userMessage: ChatMessage = createMessage({ role: "user", content: mContent, @@ -400,7 +421,8 @@ export const useChatStore = createPersistStore( get().updateCurrentSession((session) => { const savedUserMessage = { ...userMessage, - content: mContent, + //content: mContent, + content: displayContent, }; session.messages = session.messages.concat([ savedUserMessage,