diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 01fa35e82..000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -title: "[Bug] " -labels: '' -assignees: '' - ---- - -**Describe the bug** -A clear and concise description of what the bug is. - -**To Reproduce** -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. - -**Deployment** -- [ ] Docker -- [ ] Vercel -- [ ] Server - -**Desktop (please complete the following information):** - - OS: [e.g. iOS] - - Browser [e.g. chrome, safari] - - Version [e.g. 22] - -**Smartphone (please complete the following information):** - - Device: [e.g. iPhone6] - - OS: [e.g. iOS8.1] - - Browser [e.g. stock browser, safari] - - Version [e.g. 22] - -**Additional Logs** -Add any logs about the problem here. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 000000000..bdba257d2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,146 @@ +name: Bug report +description: Create a report to help us improve +title: "[Bug] " +labels: ["bug"] + +body: + - type: markdown + attributes: + value: "## Describe the bug" + - type: textarea + id: bug-description + attributes: + label: "Bug Description" + description: "A clear and concise description of what the bug is." + placeholder: "Explain the bug..." + validations: + required: true + + - type: markdown + attributes: + value: "## To Reproduce" + - type: textarea + id: steps-to-reproduce + attributes: + label: "Steps to Reproduce" + description: "Steps to reproduce the behavior:" + placeholder: | + 1. Go to '...' + 2. Click on '....' + 3. Scroll down to '....' + 4. See error + validations: + required: true + + - type: markdown + attributes: + value: "## Expected behavior" + - type: textarea + id: expected-behavior + attributes: + label: "Expected Behavior" + description: "A clear and concise description of what you expected to happen." + placeholder: "Describe what you expected to happen..." + validations: + required: true + + - type: markdown + attributes: + value: "## Screenshots" + - type: textarea + id: screenshots + attributes: + label: "Screenshots" + description: "If applicable, add screenshots to help explain your problem." + placeholder: "Paste your screenshots here or write 'N/A' if not applicable..." + validations: + required: false + + - type: markdown + attributes: + value: "## Deployment" + - type: checkboxes + id: deployment + attributes: + label: "Deployment Method" + description: "Please select the deployment method you are using." + options: + - label: "Docker" + - label: "Vercel" + - label: "Server" + + - type: markdown + attributes: + value: "## Desktop (please complete the following information):" + - type: input + id: desktop-os + attributes: + label: "Desktop OS" + description: "Your desktop operating system." + placeholder: "e.g., Windows 10" + validations: + required: false + - type: input + id: desktop-browser + attributes: + label: "Desktop Browser" + description: "Your desktop browser." + placeholder: "e.g., Chrome, Safari" + validations: + required: false + - type: input + id: desktop-version + attributes: + label: "Desktop Browser Version" + description: "Version of your desktop browser." + placeholder: "e.g., 89.0" + validations: + required: false + + - type: markdown + attributes: + value: "## Smartphone (please complete the following information):" + - type: input + id: smartphone-device + attributes: + label: "Smartphone Device" + description: "Your smartphone device." + placeholder: "e.g., iPhone X" + validations: + required: false + - type: input + id: smartphone-os + attributes: + label: "Smartphone OS" + description: "Your smartphone operating system." + placeholder: "e.g., iOS 14.4" + validations: + required: false + - type: input + id: smartphone-browser + attributes: + label: "Smartphone Browser" + description: "Your smartphone browser." + placeholder: "e.g., Safari" + validations: + required: false + - type: input + id: smartphone-version + attributes: + label: "Smartphone Browser Version" + description: "Version of your smartphone browser." + placeholder: "e.g., 14" + validations: + required: false + + - type: markdown + attributes: + value: "## Additional Logs" + - type: textarea + id: additional-logs + attributes: + label: "Additional Logs" + description: "Add any logs about the problem here." + placeholder: "Paste any relevant logs here..." + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 25c36ab67..000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: "[Feature] " -labels: '' -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000..499781330 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,53 @@ +name: Feature request +description: Suggest an idea for this project +title: "[Feature Request]: " +labels: ["enhancement"] + +body: + - type: markdown + attributes: + value: "## Is your feature request related to a problem? Please describe." + - type: textarea + id: problem-description + attributes: + label: Problem Description + description: "A clear and concise description of what the problem is. Example: I'm always frustrated when [...]" + placeholder: "Explain the problem you are facing..." + validations: + required: true + + - type: markdown + attributes: + value: "## Describe the solution you'd like" + - type: textarea + id: desired-solution + attributes: + label: Solution Description + description: A clear and concise description of what you want to happen. + placeholder: "Describe the solution you'd like..." + validations: + required: true + + - type: markdown + attributes: + value: "## Describe alternatives you've considered" + - type: textarea + id: alternatives-considered + attributes: + label: Alternatives Considered + description: A clear and concise description of any alternative solutions or features you've considered. + placeholder: "Describe any alternative solutions or features you've considered..." + validations: + required: false + + - type: markdown + attributes: + value: "## Additional context" + - type: textarea + id: additional-context + attributes: + label: Additional Context + description: Add any other context or screenshots about the feature request here. + placeholder: "Add any other context or screenshots about the feature request here..." + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/功能建议.md b/.github/ISSUE_TEMPLATE/功能建议.md deleted file mode 100644 index 3fc3d0769..000000000 --- a/.github/ISSUE_TEMPLATE/功能建议.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -name: 功能建议 -about: 请告诉我们你的灵光一闪 -title: "[Feature] " -labels: '' -assignees: '' - ---- - -> 为了提高交流效率，我们设立了官方 QQ 群和 QQ 频道，如果你在使用或者搭建过程中遇到了任何问题，请先第一时间加群或者频道咨询解决，除非是可以稳定复现的 Bug 或者较为有创意的功能建议，否则请不要随意往 Issue 区发送低质无意义帖子。 - -> [点击加入官方群聊](https://github.com/Yidadaa/ChatGPT-Next-Web/discussions/1724) - -**这个功能与现有的问题有关吗？** -如果有关，请在此列出链接或者描述问题。 - -**你想要什么功能或者有什么建议？** -尽管告诉我们。 - -**有没有可以参考的同类竞品？** -可以给出参考产品的链接或者截图。 - -**其他信息** -可以说说你的其他考虑。 diff --git a/.github/ISSUE_TEMPLATE/反馈问题.md b/.github/ISSUE_TEMPLATE/反馈问题.md deleted file mode 100644 index 270263f06..000000000 --- a/.github/ISSUE_TEMPLATE/反馈问题.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -name: 反馈问题 -about: 请告诉我们你遇到的问题 -title: "[Bug] " -labels: '' -assignees: '' - ---- - -> 为了提高交流效率，我们设立了官方 QQ 群和 QQ 频道，如果你在使用或者搭建过程中遇到了任何问题，请先第一时间加群或者频道咨询解决，除非是可以稳定复现的 Bug 或者较为有创意的功能建议，否则请不要随意往 Issue 区发送低质无意义帖子。 - -> [点击加入官方群聊](https://github.com/Yidadaa/ChatGPT-Next-Web/discussions/1724) - -**反馈须知** - -⚠️ 注意：不遵循此模板的任何帖子都会被立即关闭，如果没有提供下方的信息，我们无法定位你的问题。 - -请在下方中括号内输入 x 来表示你已经知晓相关内容。 -- [ ] 我确认已经在 [常见问题](https://github.com/Yidadaa/ChatGPT-Next-Web/blob/main/docs/faq-cn.md) 中搜索了此次反馈的问题，没有找到解答； -- [ ] 我确认已经在 [Issues](https://github.com/Yidadaa/ChatGPT-Next-Web/issues) 列表（包括已经 Close 的）中搜索了此次反馈的问题，没有找到解答。 -- [ ] 我确认已经在 [Vercel 使用教程](https://github.com/Yidadaa/ChatGPT-Next-Web/blob/main/docs/vercel-cn.md) 中搜索了此次反馈的问题，没有找到解答。 - -**描述问题** -请在此描述你遇到了什么问题。 - -**如何复现** -请告诉我们你是通过什么操作触发的该问题。 - -**截图** -请在此提供控制台截图、屏幕截图或者服务端的 log 截图。 - -**一些必要的信息** - - 系统：[比如 windows 10/ macos 12/ linux / android 11 / ios 16] - - 浏览器： [比如 chrome, safari] - - 版本： [填写设置页面的版本号] - - 部署方式：[比如 vercel、docker 或者服务器部署] diff --git a/README.md b/README.md index 230184a59..28e5c849f 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,9 @@ [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/Hk-Gosuto/ChatGPT-Next-Web-LangChain) +> [!WARNING] +> 本项目插件功能基于 [OpenAI API 函数调用](https://platform.openai.com/docs/guides/function-calling) 功能实现，转发 GitHub Copilot 接口或类似实现的模拟接口并不能正常调用插件功能！ + ![cover](./docs/images/gpt-vision-example.jpg) ![plugin-example](./docs/images/plugin-example.png) @@ -37,7 +40,8 @@ - 支持 OpenAI TTS（文本转语音）https://github.com/Hk-Gosuto/ChatGPT-Next-Web-LangChain/issues/208 - 支持 GPT-4V(视觉) 模型 - - 需要配置对象存储服务，请参考 [对象存储服务配置指南](./docs/s3-oss.md) 配置 + - ~~需要配置对象存储服务，请参考 [对象存储服务配置指南](./docs/s3-oss.md) 配置~~ + - 已同步上游仓库视觉模型调用方式（压缩图片），不过这里还是会有撑爆 localstorage 的风险（https://github.com/Hk-Gosuto/ChatGPT-Next-Web-LangChain/issues/77#issuecomment-1846410078），后续会兼容两种形式的图片本地存储，如果配置了对象存储会优先使用对象存储。 - 基于 [LangChain](https://github.com/hwchase17/langchainjs) 实现的插件功能，目前支持以下插件，未来会添加更多 - 搜索（优先级：`GoogleCustomSearch > SerpAPI > BingSerpAPI > ChooseSearchEngine > DuckDuckGo`） @@ -106,7 +110,7 @@ - 配置密钥 `GOOGLE_API_KEY` ，key 可以在这里获取：https://ai.google.dev/tutorials/setup - 配置自定义接口地址（可选） `GOOGLE_BASE_URL`，可以使用我的这个项目搭建一个基于 vercel 的代理服务：[google-gemini-vercel-proxy](https://github.com/Hk-Gosuto/google-gemini-vercel-proxy) - 常见问题参考：[Gemini Prompting FAQs](https://js.langchain.com/docs/integrations/chat/google_generativeai#gemini-prompting-faqs) - - gemini-pro-vision 模型需要配置对象存储服务，请参考 [对象存储服务配置指南](./docs/s3-oss.md) 配置 + - ~~gemini-pro-vision 模型需要配置对象存储服务，请参考 [对象存储服务配置指南](./docs/s3-oss.md) 配置~~ - ⚠ gemini-pro-vision 注意事项 https://github.com/Hk-Gosuto/ChatGPT-Next-Web-LangChain/issues/203 ： - 每次对话必须包含图像数据，不然会出现 `Add an image to use models/gemini-pro-vision, or switch your model to a text model.` 错误。 - 只支持单轮对话，多轮对话会出现 `Multiturn chat is not enabled for models/gemini-pro-vision` 错误。 @@ -134,12 +138,17 @@ 最新版本中已经移除上面两个模型。 -- [ ] 支持添加自定义插件 +- [ ] ~~支持添加自定义插件~~ -## 最新动态 +- [ ] 支持其他类型文件上传 https://github.com/Hk-Gosuto/ChatGPT-Next-Web-LangChain/issues/77 -- 🚀 v2.9.5 正式版本发布 -- 🚀 v2.9.1-plugin-preview 预览版发布。 +- [ ] 支持 Azure Storage https://github.com/Hk-Gosuto/ChatGPT-Next-Web-LangChain/issues/217 + +- [ ] 支持语音输入 https://github.com/Hk-Gosuto/ChatGPT-Next-Web-LangChain/issues/208 + +- [ ] 支持 Fooocus-API 插件 https://github.com/Hk-Gosuto/ChatGPT-Next-Web-LangChain/issues/58 + +- [ ] 支持在 UI 配置插件需要的 Key https://github.com/Hk-Gosuto/ChatGPT-Next-Web-LangChain/issues/70 ## 开始使用 diff --git a/app/api/google/[...path]/route.ts b/app/api/google/[...path]/route.ts index 3132f7b91..54aca26fa 100644 --- a/app/api/google/[...path]/route.ts +++ b/app/api/google/[...path]/route.ts @@ -17,7 +17,7 @@ async function handle( const serverConfig = getServerSideConfig(); - let baseUrl = serverConfig.googleBaseUrl || GOOGLE_BASE_URL; + let baseUrl = serverConfig.googleUrl || GOOGLE_BASE_URL; if (!baseUrl.startsWith("http")) { baseUrl = `https://${baseUrl}`; diff --git a/app/client/api.ts b/app/client/api.ts index 84be55dbf..c39504139 100644 --- a/app/client/api.ts +++ b/app/client/api.ts @@ -16,10 +16,17 @@ export const Models = ["gpt-3.5-turbo", "gpt-4"] as const; export const TTSModels = ["tts-1", "tts-1-hd"] as const; export type ChatModel = ModelType; +export interface MultimodalContent { + type: "text" | "image_url"; + text?: string; + image_url?: { + url: string; + }; +} + export interface RequestMessage { role: MessageRole; - content: string; - image_url?: string; + content: string | MultimodalContent[]; } export interface LLMConfig { diff --git a/app/client/platforms/google.ts b/app/client/platforms/google.ts index a0fe79ab8..65e609a6e 100644 --- a/app/client/platforms/google.ts +++ b/app/client/platforms/google.ts @@ -1,9 +1,4 @@ -import { - ApiPath, - Google, - REQUEST_TIMEOUT_MS, - ServiceProvider, -} from "@/app/constant"; +import { Google, REQUEST_TIMEOUT_MS } from "@/app/constant"; import { AgentChatOptions, ChatOptions, @@ -14,13 +9,13 @@ import { SpeechOptions, } from "../api"; import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; -import axios from "axios"; - -const getImageBase64Data = async (url: string) => { - const response = await axios.get(url, { responseType: "arraybuffer" }); - const base64 = Buffer.from(response.data, "binary").toString("base64"); - return base64; -}; +import { getClientConfig } from "@/app/config/client"; +import { DEFAULT_API_HOST } from "@/app/constant"; +import { + getMessageTextContent, + getMessageImages, + isVisionModel, +} from "@/app/utils"; export class GeminiProApi implements LLMApi { speech(options: SpeechOptions): Promise { @@ -39,32 +34,34 @@ export class GeminiProApi implements LLMApi { ); } async chat(options: ChatOptions): Promise { - const messages: any[] = []; - if (options.config.model.includes("vision")) { - for (const v of options.messages) { - let message: any = { - role: v.role.replace("assistant", "model").replace("system", "user"), - parts: [{ text: v.content }], - }; - if (v.image_url) { - var base64Data = await getImageBase64Data(v.image_url); - message.parts.push({ - inline_data: { - mime_type: "image/jpeg", - data: base64Data, - }, - }); + // const apiClient = this; + const visionModel = isVisionModel(options.config.model); + let multimodal = false; + const messages = options.messages.map((v) => { + let parts: any[] = [{ text: getMessageTextContent(v) }]; + if (visionModel) { + const images = getMessageImages(v); + if (images.length > 0) { + multimodal = true; + parts = parts.concat( + images.map((image) => { + const imageType = image.split(";")[0].split(":")[1]; + const imageData = image.split(",")[1]; + return { + inline_data: { + mime_type: imageType, + data: imageData, + }, + }; + }), + ); } - messages.push(message); } - } else { - options.messages.map((v) => - messages.push({ - role: v.role.replace("assistant", "model").replace("system", "user"), - parts: [{ text: v.content }], - }), - ); - } + return { + role: v.role.replace("assistant", "model").replace("system", "user"), + parts: parts, + }; + }); // google requires that role in neighboring messages must not be the same for (let i = 0; i < messages.length - 1; ) { @@ -79,7 +76,9 @@ export class GeminiProApi implements LLMApi { i++; } } - + // if (visionModel && messages.length > 1) { + // options.onError?.(new Error("Multiturn chat is not enabled for models/gemini-pro-vision")); + // } const modelConfig = { ...useAppConfig.getState().modelConfig, ...useChatStore.getState().currentSession().mask.modelConfig, @@ -118,15 +117,28 @@ export class GeminiProApi implements LLMApi { ], }; - console.log("[Request] google payload: ", requestPayload); + const accessStore = useAccessStore.getState(); + let baseUrl = accessStore.googleUrl; + const isApp = !!getClientConfig()?.isApp; - const shouldStream = !!options.config.stream; + let shouldStream = !!options.config.stream; const controller = new AbortController(); options.onController?.(controller); try { - const chatPath = this.path( - Google.ChatPath.replace("{{model}}", options.config.model), - ); + let googleChatPath = visionModel + ? Google.VisionChatPath + : Google.ChatPath; + let chatPath = this.path(googleChatPath); + + if (!baseUrl) { + baseUrl = isApp + ? DEFAULT_API_HOST + "/api/proxy/google/" + googleChatPath + : chatPath; + } + + if (isApp) { + baseUrl += `?key=${accessStore.googleApiKey}`; + } const chatPayload = { method: "POST", body: JSON.stringify(requestPayload), @@ -142,10 +154,6 @@ export class GeminiProApi implements LLMApi { if (shouldStream) { let responseText = ""; let remainText = ""; - let streamChatPath = chatPath.replace( - "generateContent", - "streamGenerateContent", - ); let finished = false; let existingTexts: string[] = []; @@ -175,11 +183,12 @@ export class GeminiProApi implements LLMApi { // start animaion animateResponseText(); - fetch(streamChatPath, chatPayload) - .then(async (response) => { - if (!response.ok) { - throw new Error(await response?.text()); - } + + fetch( + baseUrl.replace("generateContent", "streamGenerateContent"), + chatPayload, + ) + .then((response) => { const reader = response?.body?.getReader(); const decoder = new TextDecoder(); let partialData = ""; @@ -189,6 +198,19 @@ export class GeminiProApi implements LLMApi { value, }): Promise { if (done) { + if (response.status !== 200) { + try { + let data = JSON.parse(ensureProperEnding(partialData)); + if (data && data[0].error) { + options.onError?.(new Error(data[0].error.message)); + } else { + options.onError?.(new Error("Request failed")); + } + } catch (_) { + options.onError?.(new Error("Request failed")); + } + } + console.log("Stream complete"); // options.onFinish(responseText + remainText); finished = true; @@ -230,7 +252,7 @@ export class GeminiProApi implements LLMApi { options.onError?.(error as Error); }); } else { - const res = await fetch(chatPath, chatPayload); + const res = await fetch(baseUrl, chatPayload); clearTimeout(requestTimeoutId); const resJson = await res.json(); @@ -259,30 +281,7 @@ export class GeminiProApi implements LLMApi { return []; } path(path: string): string { - const accessStore = useAccessStore.getState(); - const isGoogle = - accessStore.useCustomConfig && - accessStore.provider === ServiceProvider.Google; - - if (isGoogle && !accessStore.isValidGoogle()) { - throw Error( - "incomplete google config, please check it in your settings page", - ); - } - - let baseUrl = isGoogle ? accessStore.googleBaseUrl : ApiPath.GoogleAI; - - if (baseUrl.length === 0) { - baseUrl = ApiPath.GoogleAI; - } - if (baseUrl.endsWith("/")) { - baseUrl = baseUrl.slice(0, baseUrl.length - 1); - } - if (!baseUrl.startsWith("http") && !baseUrl.startsWith(ApiPath.GoogleAI)) { - baseUrl = "https://" + baseUrl; - } - - return [baseUrl, path].join("/"); + return "/api/google/" + path; } } diff --git a/app/client/platforms/openai.ts b/app/client/platforms/openai.ts index eeeac31c9..3ed7f3e94 100644 --- a/app/client/platforms/openai.ts +++ b/app/client/platforms/openai.ts @@ -16,6 +16,7 @@ import { LLMApi, LLMModel, LLMUsage, + MultimodalContent, SpeechOptions, } from "../api"; import Locale from "../../locales"; @@ -26,7 +27,11 @@ import { import { prettyObject } from "@/app/utils/format"; import { getClientConfig } from "@/app/config/client"; import { makeAzurePath } from "@/app/azure"; -import axios from "axios"; +import { + getMessageTextContent, + getMessageImages, + isVisionModel, +} from "@/app/utils"; export interface OpenAIListModelResponse { object: string; @@ -120,49 +125,11 @@ export class ChatGPTApi implements LLMApi { } async chat(options: ChatOptions) { - const messages: any[] = []; - - const getImageBase64Data = async (url: string) => { - const response = await axios.get(url, { responseType: "arraybuffer" }); - const base64 = Buffer.from(response.data, "binary").toString("base64"); - return base64; - }; - if (options.config.model === "gpt-4-vision-preview") { - for (const v of options.messages) { - let message: { - role: string; - content: { - type: string; - text?: string; - image_url?: { url: string }; - }[]; - } = { - role: v.role, - content: [], - }; - message.content.push({ - type: "text", - text: v.content, - }); - if (v.image_url) { - var base64Data = await getImageBase64Data(v.image_url); - message.content.push({ - type: "image_url", - image_url: { - url: `data:image/jpeg;base64,${base64Data}`, - }, - }); - } - messages.push(message); - } - } else { - options.messages.map((v) => - messages.push({ - role: v.role, - content: v.content, - }), - ); - } + const visionModel = isVisionModel(options.config.model); + const messages = options.messages.map((v) => ({ + role: v.role, + content: visionModel ? v.content : getMessageTextContent(v), + })); const modelConfig = { ...useAppConfig.getState().modelConfig, @@ -186,6 +153,16 @@ export class ChatGPTApi implements LLMApi { // Please do not ask me why not send max_tokens, no reason, this param is just shit, I dont want to explain anymore. }; + // add max_tokens to vision model + if (visionModel) { + Object.defineProperty(requestPayload, "max_tokens", { + enumerable: true, + configurable: true, + writable: true, + value: modelConfig.max_tokens, + }); + } + console.log("[Request] openai payload: ", requestPayload); const shouldStream = !!options.config.stream; diff --git a/app/components/chat.module.scss b/app/components/chat.module.scss index 9c0d2963b..00d400a05 100644 --- a/app/components/chat.module.scss +++ b/app/components/chat.module.scss @@ -1,5 +1,47 @@ @import "../styles/animation.scss"; +.attach-images { + position: absolute; + left: 30px; + bottom: 32px; + display: flex; +} + +.attach-image { + cursor: default; + width: 64px; + height: 64px; + border: rgba($color: #888, $alpha: 0.2) 1px solid; + border-radius: 5px; + margin-right: 10px; + background-size: cover; + background-position: center; + background-color: var(--white); + + .attach-image-mask { + width: 100%; + height: 100%; + opacity: 0; + transition: all ease 0.2s; + } + + .attach-image-mask:hover { + opacity: 1; + } + + .delete-image { + width: 24px; + height: 24px; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + border-radius: 5px; + float: right; + background-color: var(--white); + } +} + .chat-input-actions { display: flex; flex-wrap: wrap; @@ -190,12 +232,10 @@ animation: slide-in ease 0.3s; - $linear: linear-gradient( - to right, - rgba(0, 0, 0, 0), - rgba(0, 0, 0, 1), - rgba(0, 0, 0, 0) - ); + $linear: linear-gradient(to right, + rgba(0, 0, 0, 0), + rgba(0, 0, 0, 1), + rgba(0, 0, 0, 0)); mask-image: $linear; @mixin show { @@ -328,7 +368,7 @@ } } -.chat-message-user > .chat-message-container { +.chat-message-user>.chat-message-container { align-items: flex-end; } @@ -350,6 +390,7 @@ padding: 7px; } } + /* Specific styles for iOS devices */ @media screen and (max-device-width: 812px) and (-webkit-min-device-pixel-ratio: 2) { @supports (-webkit-touch-callout: none) { @@ -413,6 +454,64 @@ transition: all ease 0.3s; } +.chat-message-item-image { + width: 100%; + margin-top: 10px; +} + +.chat-message-item-images { + width: 100%; + display: grid; + justify-content: left; + grid-gap: 10px; + grid-template-columns: repeat(var(--image-count), auto); + margin-top: 10px; +} + +.chat-message-item-image-multi { + object-fit: cover; + background-size: cover; + background-position: center; + background-repeat: no-repeat; +} + +.chat-message-item-image, +.chat-message-item-image-multi { + box-sizing: border-box; + border-radius: 10px; + border: rgba($color: #888, $alpha: 0.2) 1px solid; +} + + +@media only screen and (max-width: 600px) { + $calc-image-width: calc(100vw/3*2/var(--image-count)); + + .chat-message-item-image-multi { + width: $calc-image-width; + height: $calc-image-width; + } + + .chat-message-item-image { + max-width: calc(100vw/3*2); + } +} + +@media screen and (min-width: 600px) { + $max-image-width: calc(calc(1200px - var(--sidebar-width))/3*2/var(--image-count)); + $image-width: calc(calc(var(--window-width) - var(--sidebar-width))/3*2/var(--image-count)); + + .chat-message-item-image-multi { + width: $image-width; + height: $image-width; + max-width: $max-image-width; + max-height: $max-image-width; + } + + .chat-message-item-image { + max-width: calc(calc(1200px - var(--sidebar-width))/3*2); + } +} + .chat-message-action-date { font-size: 12px; opacity: 0.2; @@ -427,7 +526,7 @@ z-index: 1; } -.chat-message-user > .chat-message-container > .chat-message-item { +.chat-message-user>.chat-message-container>.chat-message-item { background-color: var(--second); &:hover { @@ -492,6 +591,7 @@ @include single-line(); } + .hint-content { font-size: 12px; @@ -506,15 +606,26 @@ } .chat-input-panel-inner { + cursor: text; display: flex; flex: 1; + border-radius: 10px; + border: var(--border-in-light); +} + +.chat-input-panel-inner-attach { + padding-bottom: 80px; +} + +.chat-input-panel-inner:has(.chat-input:focus) { + border: 1px solid var(--primary); } .chat-input { height: 100%; width: 100%; border-radius: 10px; - border: var(--border-in-light); + border: none; box-shadow: 0 -2px 5px rgba(0, 0, 0, 0.03); background-color: var(--white); color: var(--black); @@ -526,9 +637,7 @@ min-height: 68px; } -.chat-input:focus { - border: 1px solid var(--primary); -} +.chat-input:focus {} .chat-input-send { background-color: var(--primary); @@ -580,9 +689,4 @@ .chat-input-send { bottom: 30px; } - - .chat-input-image { - bottom: 84px; - right: 32px; - } } diff --git a/app/components/chat.tsx b/app/components/chat.tsx index 5ce13de93..4b04b3126 100644 --- a/app/components/chat.tsx +++ b/app/components/chat.tsx @@ -6,6 +6,7 @@ import React, { useMemo, useCallback, Fragment, + RefObject, } from "react"; import SendWhiteIcon from "../icons/send-white.svg"; @@ -17,6 +18,7 @@ import CopyIcon from "../icons/copy.svg"; import SpeakIcon from "../icons/speak.svg"; import SpeakStopIcon from "../icons/speak-stop.svg"; import LoadingIcon from "../icons/three-dots.svg"; +import LoadingButtonIcon from "../icons/loading.svg"; import PromptIcon from "../icons/prompt.svg"; import MaskIcon from "../icons/mask.svg"; import MaxIcon from "../icons/max.svg"; @@ -24,7 +26,7 @@ import MinIcon from "../icons/min.svg"; import ResetIcon from "../icons/reload.svg"; import BreakIcon from "../icons/break.svg"; import SettingsIcon from "../icons/chat-settings.svg"; -import ClearIcon from "../icons/clear.svg"; +import DeleteIcon from "../icons/clear.svg"; import CloseIcon from "../icons/close.svg"; import PinIcon from "../icons/pin.svg"; import EditIcon from "../icons/rename.svg"; @@ -33,6 +35,7 @@ import CancelIcon from "../icons/cancel.svg"; import EnablePluginIcon from "../icons/plugin_enable.svg"; import DisablePluginIcon from "../icons/plugin_disable.svg"; import UploadIcon from "../icons/upload.svg"; +import ImageIcon from "../icons/image.svg"; import LightIcon from "../icons/light.svg"; import DarkIcon from "../icons/dark.svg"; @@ -60,6 +63,10 @@ import { selectOrCopy, autoGrowTextArea, useMobileScreen, + getMessageTextContent, + getMessageImages, + isVisionModel, + compressImage, } from "../utils"; import dynamic from "next/dynamic"; @@ -101,6 +108,7 @@ import { useAllModels } from "../utils/hooks"; import Image from "next/image"; import { ClientApi } from "../client/api"; import { createTTSPlayer } from "../utils/audio"; +import { MultimodalContent } from "../client/api"; const ttsPlayer = createTTSPlayer(); @@ -408,11 +416,13 @@ function ChatAction(props: { ); } -function useScrollToBottom() { +function useScrollToBottom( + scrollRef: RefObject, + detach: boolean = false, +) { // for auto-scroll - const scrollRef = useRef(null); - const [autoScroll, setAutoScroll] = useState(true); + const [autoScroll, setAutoScroll] = useState(true); function scrollDomToBottom() { const dom = scrollRef.current; if (dom) { @@ -425,7 +435,7 @@ function useScrollToBottom() { // auto scroll useEffect(() => { - if (autoScroll) { + if (autoScroll && !detach) { scrollDomToBottom(); } }); @@ -439,18 +449,20 @@ function useScrollToBottom() { } export function ChatActions(props: { + uploadImage: () => void; + setAttachImages: (images: string[]) => void; + setUploading: (uploading: boolean) => void; showPromptModal: () => void; scrollToBottom: () => void; showPromptHints: () => void; imageSelected: (img: any) => void; hitBottom: boolean; + uploading: boolean; }) { const config = useAppConfig(); const navigate = useNavigate(); const chatStore = useChatStore(); - const [uploadLoading, setUploadLoading] = useState(false); - // switch Plugins const usePlugins = chatStore.currentSession().mask.usePlugins; function switchUsePlugins() { @@ -473,33 +485,6 @@ export function ChatActions(props: { const couldStop = ChatControllerPool.hasPending(); const stopAll = () => ChatControllerPool.stopAll(); - function selectImage() { - document.getElementById("chat-image-file-select-upload")?.click(); - } - - function closeImageButton() { - document.getElementById("chat-input-image-close")?.click(); - } - - const onImageSelected = async (e: any) => { - const file = e.target.files[0]; - if (!file) return; - const api = new ClientApi(); - setUploadLoading(true); - const uploadFile = await api.file - .upload(file) - .catch((e) => { - console.error("[Upload]", e); - showToast(prettyObject(e)); - }) - .finally(() => setUploadLoading(false)); - props.imageSelected({ - fileName: uploadFile.fileName, - fileUrl: uploadFile.filePath, - }); - e.target.value = null; - }; - // switch model const currentModel = chatStore.currentSession().mask.modelConfig.model; const allModels = useAllModels(); @@ -508,8 +493,16 @@ export function ChatActions(props: { [allModels], ); const [showModelSelector, setShowModelSelector] = useState(false); + const [showUploadImage, setShowUploadImage] = useState(false); useEffect(() => { + const show = isVisionModel(currentModel); + setShowUploadImage(show); + if (!show) { + props.setAttachImages([]); + props.setUploading(false); + } + // if current model is not available // switch to first available model const isUnavaliableModel = !models.some((m) => m.name === currentModel); @@ -520,37 +513,7 @@ export function ChatActions(props: { ); showToast(nextModel); } - const onPaste = (event: ClipboardEvent) => { - const items = event.clipboardData?.items || []; - const api = new ClientApi(); - for (let i = 0; i < items.length; i++) { - if (items[i].type.indexOf("image") === -1) continue; - const file = items[i].getAsFile(); - if (file !== null) { - setUploadLoading(true); - api.file - .upload(file) - .then((uploadFile) => { - props.imageSelected({ - fileName: uploadFile.fileName, - fileUrl: uploadFile.filePath, - }); - }) - .catch((e) => { - console.error("[Upload]", e); - showToast(prettyObject(e)); - }) - .finally(() => setUploadLoading(false)); - } - } - }; - if (currentModel.includes("vision")) { - window.addEventListener("paste", onPaste); - return () => { - window.removeEventListener("paste", onPaste); - }; - } - }, [chatStore, currentModel, models, props]); + }, [chatStore, currentModel, models]); return (
@@ -577,6 +540,13 @@ export function ChatActions(props: { /> )} + {showUploadImage && ( + : } + /> + )} } /> - : } /> )} - {currentModel.includes("vision") && ( - } - innerNode={ - - } - /> - )} {showModelSelector && ( { session.mask.modelConfig.model = s[0] as ModelType; session.mask.syncGlobalConfig = false; - closeImageButton(); }); showToast(s[0]); }} @@ -746,6 +697,14 @@ export function EditMessageModal(props: { onClose: () => void }) { ); } +export function DeleteImageButton(props: { deleteImage: () => void }) { + return ( +
+ +
+ ); +} + function _Chat() { type RenderMessage = ChatMessage & { preview?: boolean }; @@ -761,10 +720,22 @@ function _Chat() { const [userImage, setUserImage] = useState(); const [isLoading, setIsLoading] = useState(false); const { submitKey, shouldSubmit } = useSubmitHandler(); - const { scrollRef, setAutoScroll, scrollDomToBottom } = useScrollToBottom(); + const scrollRef = useRef(null); + const isScrolledToBottom = scrollRef?.current + ? Math.abs( + scrollRef.current.scrollHeight - + (scrollRef.current.scrollTop + scrollRef.current.clientHeight), + ) <= 1 + : false; + const { setAutoScroll, scrollDomToBottom } = useScrollToBottom( + scrollRef, + isScrolledToBottom, + ); const [hitBottom, setHitBottom] = useState(true); const isMobileScreen = useMobileScreen(); const navigate = useNavigate(); + const [attachImages, setAttachImages] = useState([]); + const [uploading, setUploading] = useState(false); // prompt hints const promptStore = usePromptStore(); @@ -843,8 +814,9 @@ function _Chat() { } setIsLoading(true); chatStore - .onUserInput(userInput, userImage?.fileUrl) + .onUserInput(userInput, attachImages) .then(() => setIsLoading(false)); + setAttachImages([]); localStorage.setItem(LAST_INPUT_KEY, userInput); localStorage.setItem(LAST_INPUT_IMAGE_KEY, userImage); setUserInput(""); @@ -925,9 +897,9 @@ function _Chat() { }; const onRightClick = (e: any, message: ChatMessage) => { // copy to clipboard - if (selectOrCopy(e.currentTarget, message.content)) { + if (selectOrCopy(e.currentTarget, getMessageTextContent(message))) { if (userInput.length === 0) { - setUserInput(message.content); + setUserInput(getMessageTextContent(message)); } e.preventDefault(); @@ -995,9 +967,9 @@ function _Chat() { // resend the message setIsLoading(true); - chatStore - .onUserInput(userMessage.content, userMessage.image_url) - .then(() => setIsLoading(false)); + const textContent = getMessageTextContent(userMessage); + const images = getMessageImages(userMessage); + chatStore.onUserInput(textContent, images).then(() => setIsLoading(false)); inputRef.current?.focus(); }; @@ -1085,7 +1057,6 @@ function _Chat() { ...createMessage({ role: "user", content: userInput, - image_url: userImage?.fileUrl, }), preview: true, }, @@ -1139,7 +1110,6 @@ function _Chat() { setHitBottom(isHitBottom); setAutoScroll(isHitBottom); }; - function scrollToBottom() { setMsgRenderIndex(renderMessages.length - CHAT_PAGE_SIZE); scrollDomToBottom(); @@ -1225,6 +1195,94 @@ function _Chat() { // eslint-disable-next-line react-hooks/exhaustive-deps }, []); + const handlePaste = useCallback( + async (event: React.ClipboardEvent) => { + const currentModel = chatStore.currentSession().mask.modelConfig.model; + if (!isVisionModel(currentModel)) { + return; + } + const items = (event.clipboardData || window.clipboardData).items; + for (const item of items) { + if (item.kind === "file" && item.type.startsWith("image/")) { + event.preventDefault(); + const file = item.getAsFile(); + if (file) { + const images: string[] = []; + images.push(...attachImages); + images.push( + ...(await new Promise((res, rej) => { + setUploading(true); + const imagesData: string[] = []; + compressImage(file, 256 * 1024) + .then((dataUrl) => { + imagesData.push(dataUrl); + setUploading(false); + res(imagesData); + }) + .catch((e) => { + setUploading(false); + rej(e); + }); + })), + ); + const imagesLength = images.length; + + if (imagesLength > 3) { + images.splice(3, imagesLength - 3); + } + setAttachImages(images); + } + } + } + }, + [attachImages, chatStore], + ); + + async function uploadImage() { + const images: string[] = []; + images.push(...attachImages); + + images.push( + ...(await new Promise((res, rej) => { + const fileInput = document.createElement("input"); + fileInput.type = "file"; + fileInput.accept = + "image/png, image/jpeg, image/webp, image/heic, image/heif"; + fileInput.multiple = true; + fileInput.onchange = (event: any) => { + setUploading(true); + const files = event.target.files; + const imagesData: string[] = []; + for (let i = 0; i < files.length; i++) { + const file = event.target.files[i]; + compressImage(file, 256 * 1024) + .then((dataUrl) => { + imagesData.push(dataUrl); + if ( + imagesData.length === 3 || + imagesData.length === files.length + ) { + setUploading(false); + res(imagesData); + } + }) + .catch((e) => { + setUploading(false); + rej(e); + }); + } + }; + fileInput.click(); + })), + ); + + const imagesLength = images.length; + if (imagesLength > 3) { + images.splice(3, imagesLength - 3); + } + setAttachImages(images); + } + const textareaMinHeight = userImage ? 121 : 68; return ( @@ -1333,15 +1391,29 @@ function _Chat() { onClick={async () => { const newMessage = await showPrompt( Locale.Chat.Actions.Edit, - message.content, + getMessageTextContent(message), 10, ); + let newContent: string | MultimodalContent[] = + newMessage; + const images = getMessageImages(message); + if (images.length > 0) { + newContent = [{ type: "text", text: newMessage }]; + for (let i = 0; i < images.length; i++) { + newContent.push({ + type: "image_url", + image_url: { + url: images[i], + }, + }); + } + } chatStore.updateCurrentSession((session) => { const m = session.mask.context .concat(session.messages) .find((m) => m.id === message.id); if (m) { - m.content = newMessage; + m.content = newContent; } }); }} @@ -1384,7 +1456,7 @@ function _Chat() { } + icon={} onClick={() => onDelete(message.id ?? i)} /> @@ -1396,7 +1468,11 @@ function _Chat() { } - onClick={() => copyToClipboard(message.content)} + onClick={() => + copyToClipboard( + getMessageTextContent(message), + ) + } /> {config.ttsConfig.enable && ( ) } - onClick={() => openaiSpeech(message.content)} + onClick={() => + openaiSpeech(getMessageTextContent(message)) + } /> )} @@ -1450,8 +1528,7 @@ function _Chat() { )}
onRightClick(e, message)} onDoubleClickCapture={() => { if (!isMobileScreen) return; - setUserInput(message.content); + setUserInput(getMessageTextContent(message)); }} fontSize={fontSize} parentRef={scrollRef} defaultShow={i >= messages.length - 6} /> + {getMessageImages(message).length == 1 && ( + + )} + {getMessageImages(message).length > 1 && ( +
+ {getMessageImages(message).map((image, index) => { + return ( + + ); + })} +
+ )}
{!isUser && message.model?.includes("vision") && (
setShowPromptModal(true)} scrollToBottom={scrollToBottom} hitBottom={hitBottom} + uploading={uploading} showPromptHints={() => { // Click again to close if (promptHints.length > 0) { @@ -1515,8 +1626,16 @@ function _Chat() { setUserImage(img); }} /> -
+