diff --git a/.dockerignore b/.dockerignore index 60da41dd8..f7f6fbfe4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,4 +5,7 @@ .env *.key -*.key.pub \ No newline at end of file +*.key.pub + +# upload files +/uploads \ No newline at end of file diff --git a/.env.template b/.env.template index cf02aeedc..d3b3a8fa4 100644 --- a/.env.template +++ b/.env.template @@ -50,4 +50,4 @@ DISABLE_FAST_LINK= # (optional) # Default: 1 # If your project is not deployed on Vercel, set this value to 1. -NEXT_PUBLIC_ENABLE_NODEJS_PLUGIN=1 +NEXT_PUBLIC_ENABLE_NODEJS_PLUGIN=1 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 69f4360c1..b4dc097b8 100644 --- a/.gitignore +++ b/.gitignore @@ -45,5 +45,5 @@ dev *.key *.key.pub -/public/uploads +/uploads .vercel diff --git a/README.md b/README.md index f07089d91..1ec498a41 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,11 @@ - 配置密钥 `GOOGLE_API_KEY` ,key 可以在这里获取:https://ai.google.dev/tutorials/setup - 配置自定义接口地址(可选) `GOOGLE_BASE_URL`,可以使用我的这个项目搭建一个基于 vercel 的代理服务:[google-gemini-vercel-proxy](https://github.com/Hk-Gosuto/google-gemini-vercel-proxy) - 常见问题参考:[Gemini Prompting FAQs](https://js.langchain.com/docs/integrations/chat/google_generativeai#gemini-prompting-faqs) + +- 非 Vercel 运行环境下支持本地存储 + + - 如果你的程序运行在非 Vercel 环境,不配置 `S3_ENDPOINT` 和 `R2_ACCOUNT_ID` 参数,默认上传的文件将存储在 `/app/uploads` 文件夹中 + ## 开发计划 @@ -113,25 +118,16 @@ 不配置时默认使用 `DuckDuckGo` 作为搜索插件。 - [x] 插件列表页面开发 + - [x] 支持开关指定插件 -- [ ] 支持添加自定义插件 + - [x] 支持 Agent 参数配置( ~~agentType~~, maxIterations, returnIntermediateSteps 等) + - [x] 支持 ChatSession 级别插件功能开关 仅在使用非 `0301` 和 `0314` 版本模型时会出现插件开关,其它模型默认为关闭状态,开关也不会显示。 - -## 已知问题 -- [x] ~~使用插件时需将模型切换为 `0613` 版本模型,如:`gpt-3.5-turbo-0613`~~ - - 尝试使用 `chat-conversational-react-description` 等类型的 `agent` 使用插件时效果并不理想,不再考虑支持其它版本的模型。 - - 限制修改为非 `0301` 和 `0314` 模型均可调用插件。 [#10](https://github.com/Hk-Gosuto/ChatGPT-Next-Web-LangChain/issues/10) -- [x] `SERPAPI_API_KEY` 目前为必填,后续会支持使用 DuckDuckGo 替换搜索插件 -- [x] Agent 不支持自定义接口地址 -- [x] ~~部分场景下插件会调用失败~~ - - 问题出现在使用 [Calculator](https://js.langchain.com/docs/api/tools_calculator/classes/Calculator) 进行计算时的参数错误,暂时无法干预。 -- [x] 插件调用失败后无反馈 + +- [ ] 支持添加自定义插件 ## 最新动态 diff --git a/app/api/file/[...path]/route.ts b/app/api/file/[...path]/route.ts index 8b4f9547a..f6fbfd52f 100644 --- a/app/api/file/[...path]/route.ts +++ b/app/api/file/[...path]/route.ts @@ -1,5 +1,7 @@ +import { getServerSideConfig } from "@/app/config/server"; +import LocalFileStorage from "@/app/utils/local_file_storage"; +import S3FileStorage from "@/app/utils/s3_file_storage"; import { NextRequest, NextResponse } from "next/server"; -import S3FileStorage from "../../../utils/s3_file_storage"; async function handle( req: NextRequest, @@ -10,12 +12,22 @@ async function handle( } try { - var file = await S3FileStorage.get(params.path[0]); - return new Response(file?.transformToWebStream(), { - headers: { - "Content-Type": "image/png", - }, - }); + const serverConfig = getServerSideConfig(); + if (serverConfig.isStoreFileToLocal) { + var fileBuffer = await LocalFileStorage.get(params.path[0]); + return new Response(fileBuffer, { + headers: { + "Content-Type": "image/png", + }, + }); + } else { + var file = await S3FileStorage.get(params.path[0]); + return new Response(file?.transformToWebStream(), { + headers: { + "Content-Type": "image/png", + }, + }); + } } catch (e) { return new Response("not found", { status: 404, @@ -25,5 +37,5 @@ async function handle( export const GET = handle; -export const runtime = "edge"; +export const runtime = "nodejs"; export const revalidate = 0; diff --git a/app/api/file/upload/route.ts b/app/api/file/upload/route.ts index 7b37066a7..65991477a 100644 --- a/app/api/file/upload/route.ts +++ b/app/api/file/upload/route.ts @@ -1,7 +1,9 @@ import { NextRequest, NextResponse } from "next/server"; -import { auth } from "../../auth"; -import S3FileStorage from "../../../utils/s3_file_storage"; import { ModelProvider } from "@/app/constant"; +import { auth } from "@/app/api/auth"; +import LocalFileStorage from "@/app/utils/local_file_storage"; +import { getServerSideConfig } from "@/app/config/server"; +import S3FileStorage from "@/app/utils/s3_file_storage"; async function handle(req: NextRequest) { if (req.method === "OPTIONS") { @@ -31,10 +33,17 @@ async function handle(req: NextRequest) { const buffer = Buffer.from(imageData); var fileName = `${Date.now()}.png`; - await S3FileStorage.put(fileName, buffer); + var filePath = ""; + const serverConfig = getServerSideConfig(); + if (serverConfig.isStoreFileToLocal) { + filePath = await LocalFileStorage.put(fileName, buffer); + } else { + filePath = await S3FileStorage.put(fileName, buffer); + } return NextResponse.json( { fileName: fileName, + filePath: filePath, }, { status: 200, @@ -55,4 +64,4 @@ async function handle(req: NextRequest) { export const POST = handle; -export const runtime = "edge"; +export const runtime = "nodejs"; diff --git a/app/api/langchain-tools/dalle_image_generator_node.ts b/app/api/langchain-tools/dalle_image_generator_node.ts new file mode 100644 index 000000000..98d4d0e0f --- /dev/null +++ b/app/api/langchain-tools/dalle_image_generator_node.ts @@ -0,0 +1,22 @@ +import { getServerSideConfig } from "@/app/config/server"; +import { DallEAPIWrapper } from "./dalle_image_generator"; +import S3FileStorage from "@/app/utils/s3_file_storage"; +import LocalFileStorage from "@/app/utils/local_file_storage"; + +export class DallEAPINodeWrapper extends DallEAPIWrapper { + async saveImageFromUrl(url: string) { + const response = await fetch(url); + const content = await response.arrayBuffer(); + const buffer = Buffer.from(content); + + var filePath = ""; + const serverConfig = getServerSideConfig(); + var fileName = `${Date.now()}.png`; + if (serverConfig.isStoreFileToLocal) { + filePath = await LocalFileStorage.put(fileName, buffer); + } else { + filePath = await S3FileStorage.put(fileName, buffer); + } + return filePath; + } +} diff --git a/app/api/langchain-tools/edge_tools.ts b/app/api/langchain-tools/edge_tools.ts index 1e23bfc81..b7b0cd21e 100644 --- a/app/api/langchain-tools/edge_tools.ts +++ b/app/api/langchain-tools/edge_tools.ts @@ -4,13 +4,8 @@ import { StableDiffusionWrapper } from "@/app/api/langchain-tools/stable_diffusi import { BaseLanguageModel } from "langchain/dist/base_language"; import { Calculator } from "langchain/tools/calculator"; import { WebBrowser } from "langchain/tools/webbrowser"; -import { BaiduSearch } from "@/app/api/langchain-tools/baidu_search"; -import { DuckDuckGo } from "@/app/api/langchain-tools/duckduckgo_search"; -import { GoogleSearch } from "@/app/api/langchain-tools/google_search"; -import { Tool, DynamicTool } from "langchain/tools"; -import * as langchainTools from "langchain/tools"; import { Embeddings } from "langchain/dist/embeddings/base.js"; -import { WolframAlphaTool } from "./wolframalpha"; +import { WolframAlphaTool } from "@/app/api/langchain-tools/wolframalpha"; export class EdgeTool { private apiKey: string | undefined; @@ -52,7 +47,6 @@ export class EdgeTool { const arxivAPITool = new ArxivAPIWrapper(); const wolframAlphaTool = new WolframAlphaTool(); let tools = [ - // searchTool, calculatorTool, webBrowserTool, dallEAPITool, diff --git a/app/api/langchain-tools/nodejs_tools.ts b/app/api/langchain-tools/nodejs_tools.ts index af01dd2f2..411ce6330 100644 --- a/app/api/langchain-tools/nodejs_tools.ts +++ b/app/api/langchain-tools/nodejs_tools.ts @@ -1,7 +1,12 @@ import { BaseLanguageModel } from "langchain/dist/base_language"; import { PDFBrowser } from "@/app/api/langchain-tools/pdf_browser"; - import { Embeddings } from "langchain/dist/embeddings/base.js"; +import { ArxivAPIWrapper } from "@/app/api/langchain-tools/arxiv"; +import { DallEAPINodeWrapper } from "@/app/api/langchain-tools/dalle_image_generator_node"; +import { StableDiffusionNodeWrapper } from "@/app/api/langchain-tools/stable_diffusion_image_generator_node"; +import { Calculator } from "langchain/tools/calculator"; +import { WebBrowser } from "langchain/tools/webbrowser"; +import { WolframAlphaTool } from "@/app/api/langchain-tools/wolframalpha"; export class NodeJSTool { private apiKey: string | undefined; @@ -29,7 +34,29 @@ export class NodeJSTool { } async getCustomTools(): Promise { + const webBrowserTool = new WebBrowser({ + model: this.model, + embeddings: this.embeddings, + }); + const calculatorTool = new Calculator(); + const dallEAPITool = new DallEAPINodeWrapper( + this.apiKey, + this.baseUrl, + this.callback, + ); + const stableDiffusionTool = new StableDiffusionNodeWrapper(); + const arxivAPITool = new ArxivAPIWrapper(); + const wolframAlphaTool = new WolframAlphaTool(); const pdfBrowserTool = new PDFBrowser(this.model, this.embeddings); - return [pdfBrowserTool]; + let tools = [ + calculatorTool, + webBrowserTool, + dallEAPITool, + stableDiffusionTool, + arxivAPITool, + wolframAlphaTool, + pdfBrowserTool, + ]; + return tools; } } diff --git a/app/api/langchain-tools/stable_diffusion_image_generator.ts b/app/api/langchain-tools/stable_diffusion_image_generator.ts index f64025454..cf551fba8 100644 --- a/app/api/langchain-tools/stable_diffusion_image_generator.ts +++ b/app/api/langchain-tools/stable_diffusion_image_generator.ts @@ -8,6 +8,13 @@ export class StableDiffusionWrapper extends Tool { super(); } + async saveImage(imageBase64: string) { + const buffer = Buffer.from(imageBase64, "base64"); + var fileName = `${Date.now()}.png`; + const filePath = await S3FileStorage.put(fileName, buffer); + return filePath; + } + /** @ignore */ async _call(prompt: string) { let url = process.env.STABLE_DIFFUSION_API_URL; @@ -40,8 +47,7 @@ export class StableDiffusionWrapper extends Tool { const json = await response.json(); let imageBase64 = json.images[0]; if (!imageBase64) return "No image was generated"; - const buffer = Buffer.from(imageBase64, "base64"); - const filePath = await S3FileStorage.put(`${Date.now()}.png`, buffer); + const filePath = await this.saveImage(imageBase64); console.log(`[${this.name}]`, filePath); return filePath; } diff --git a/app/api/langchain-tools/stable_diffusion_image_generator_node.ts b/app/api/langchain-tools/stable_diffusion_image_generator_node.ts new file mode 100644 index 000000000..43519aaa3 --- /dev/null +++ b/app/api/langchain-tools/stable_diffusion_image_generator_node.ts @@ -0,0 +1,19 @@ +import S3FileStorage from "@/app/utils/s3_file_storage"; +import { StableDiffusionWrapper } from "./stable_diffusion_image_generator"; +import { getServerSideConfig } from "@/app/config/server"; +import LocalFileStorage from "@/app/utils/local_file_storage"; + +export class StableDiffusionNodeWrapper extends StableDiffusionWrapper { + async saveImage(imageBase64: string) { + var filePath = ""; + var fileName = `${Date.now()}.png`; + const buffer = Buffer.from(imageBase64, "base64"); + const serverConfig = getServerSideConfig(); + if (serverConfig.isStoreFileToLocal) { + filePath = await LocalFileStorage.put(fileName, buffer); + } else { + filePath = await S3FileStorage.put(fileName, buffer); + } + return filePath; + } +} diff --git a/app/api/langchain/tool/agent/nodejs/route.ts b/app/api/langchain/tool/agent/nodejs/route.ts index 63cee7b53..33af30c90 100644 --- a/app/api/langchain/tool/agent/nodejs/route.ts +++ b/app/api/langchain/tool/agent/nodejs/route.ts @@ -55,13 +55,6 @@ async function handle(req: NextRequest) { ); }; - var edgeTool = new EdgeTool( - apiKey, - baseUrl, - model, - embeddings, - dalleCallback, - ); var nodejsTool = new NodeJSTool( apiKey, baseUrl, @@ -69,9 +62,8 @@ async function handle(req: NextRequest) { embeddings, dalleCallback, ); - var edgeTools = await edgeTool.getCustomTools(); var nodejsTools = await nodejsTool.getCustomTools(); - var tools = [...edgeTools, ...nodejsTools]; + var tools = [...nodejsTools]; return await agentApi.getApiHandler(req, reqBody, tools); } catch (e) { return new Response(JSON.stringify({ error: (e as any).message }), { diff --git a/app/client/platforms/utils.ts b/app/client/platforms/utils.ts index 6920dcc57..9973bb19a 100644 --- a/app/client/platforms/utils.ts +++ b/app/client/platforms/utils.ts @@ -1,11 +1,12 @@ import { getHeaders } from "../api"; export class FileApi { - async upload(file: any): Promise { + async upload(file: any): Promise { const formData = new FormData(); formData.append("file", file); var headers = getHeaders(true); - var res = await fetch("/api/file/upload", { + const api = "/api/file/upload"; + var res = await fetch(api, { method: "POST", body: formData, headers: { @@ -14,6 +15,9 @@ export class FileApi { }); const resJson = await res.json(); console.log(resJson); - return resJson.fileName; + return { + fileName: resJson.fileName, + filePath: resJson.filePath, + }; } } diff --git a/app/components/chat.tsx b/app/components/chat.tsx index c1364701e..b71aabd2d 100644 --- a/app/components/chat.tsx +++ b/app/components/chat.tsx @@ -465,10 +465,10 @@ export function ChatActions(props: { const onImageSelected = async (e: any) => { const file = e.target.files[0]; const api = new ClientApi(); - const fileName = await api.file.upload(file); + const uploadFile = await api.file.upload(file); props.imageSelected({ - fileName, - fileUrl: `/api/file/${fileName}`, + fileName: uploadFile.fileName, + fileUrl: uploadFile.filePath, }); e.target.value = null; }; diff --git a/app/config/server.ts b/app/config/server.ts index 3bc5e5c67..eb4a9cb9b 100644 --- a/app/config/server.ts +++ b/app/config/server.ts @@ -101,5 +101,10 @@ export const getServerSideConfig = () => { hideBalanceQuery: !process.env.ENABLE_BALANCE_QUERY, disableFastLink: !!process.env.DISABLE_FAST_LINK, customModels, + + isStoreFileToLocal: + !!process.env.NEXT_PUBLIC_ENABLE_NODEJS_PLUGIN && + !process.env.R2_ACCOUNT_ID && + !process.env.S3_ENDPOINT, }; }; diff --git a/app/utils/local_file_storage.ts b/app/utils/local_file_storage.ts new file mode 100644 index 000000000..5fde30a35 --- /dev/null +++ b/app/utils/local_file_storage.ts @@ -0,0 +1,29 @@ +import fs from "fs"; +import path from "path"; + +export default class LocalFileStorage { + static async get(fileName: string) { + const filePath = path.resolve(`./uploads`, fileName); + const file = fs.readFileSync(filePath); + if (!file) { + throw new Error("not found."); + } + return file; + } + + static async put(fileName: string, data: Buffer) { + try { + const filePath = path.resolve(`./uploads`, fileName); + const dir = path.dirname(filePath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + await fs.promises.writeFile(filePath, data); + console.log("[LocalFileStorage]", filePath); + return `/api/file/${fileName}`; + } catch (e) { + console.error("[LocalFileStorage]", e); + throw e; + } + } +} diff --git a/app/utils/s3_file_storage.ts b/app/utils/s3_file_storage.ts index 139e145fd..c09b2a5c1 100644 --- a/app/utils/s3_file_storage.ts +++ b/app/utils/s3_file_storage.ts @@ -55,7 +55,7 @@ export default class S3FileStorage { { expiresIn: 60 }, ); - console.log(signedUrl); + console.log("[S3]", signedUrl); try { await fetch(signedUrl, { diff --git a/docker-compose.yml b/docker-compose.yml index c3b38547b..fac8851ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,8 @@ services: profiles: [ "no-proxy" ] container_name: chatgpt-next-web image: gosuto/chatgpt-next-web-langchain + volumes: + - next_chat_upload:/app/uploads ports: - 3000:3000 environment: @@ -22,6 +24,8 @@ services: profiles: [ "proxy" ] container_name: chatgpt-next-web-proxy image: gosuto/chatgpt-next-web-langchain + volumes: + - next_chat_upload:/app/uploads ports: - 3000:3000 environment: @@ -36,3 +40,6 @@ services: - ENABLE_BALANCE_QUERY=$ENABLE_BALANCE_QUERY - DISABLE_FAST_LINK=$DISABLE_FAST_LINK - OPENAI_SB=$OPENAI_SB + +volumes: + next_chat_upload: \ No newline at end of file