feat: 1. using cache storage store image data; 2. get base64image before chat to api #5013

app/client/platforms/alibaba.ts

@@ -21,7 +21,7 @@ import {
 } from "@fortaine/fetch-event-source";
 import { prettyObject } from "@/app/utils/format";
 import { getClientConfig } from "@/app/config/client";
-import { getMessageTextContent, isVisionModel } from "@/app/utils";
+import { getMessageTextContent } from "@/app/utils";
 
 export interface OpenAIListModelResponse {
   object: string;

app/client/platforms/anthropic.ts

@@ -12,6 +12,7 @@ import {
 import Locale from "../../locales";
 import { prettyObject } from "@/app/utils/format";
 import { getMessageTextContent, isVisionModel } from "@/app/utils";
+import { preProcessImageContent } from "@/app/utils/chat";
 import { cloudflareAIGatewayUrl } from "@/app/utils/cloudflare";
 
 export type MultiBlockContent = {
@@ -93,7 +94,12 @@ export class ClaudeApi implements LLMApi {
       },
     };
 
-    const messages = [...options.messages];
+    // try get base64image from local cache image_url
+    const messages = [];
+    for (const v of options.messages) {
+      const content = await preProcessImageContent(v.content);
+      messages.push({ role: v.role, content });
+    }
 
     const keys = ["system", "user"];
 
@@ -135,6 +141,7 @@ export class ClaudeApi implements LLMApi {
           content: content
             .filter((v) => v.image_url || v.text)
             .map(({ type, text, image_url }) => {
+              console.log("process message", type, text, image_url);
               if (type === "text") {
                 return {
                   type,

app/client/platforms/google.ts

@@ -14,6 +14,7 @@ import {
   getMessageImages,
   isVisionModel,
 } from "@/app/utils";
+import { preProcessImageContent } from "@/app/utils/chat";
 
 export class GeminiProApi implements LLMApi {
   path(path: string): string {
@@ -56,7 +57,14 @@ export class GeminiProApi implements LLMApi {
   async chat(options: ChatOptions): Promise<void> {
     const apiClient = this;
     let multimodal = false;
-    const messages = options.messages.map((v) => {
+
+    // try get base64image from local cache image_url
+    const _messages = [];
+    for (const v of options.messages) {
+      const content = await preProcessImageContent(v.content);
+      _messages.push({ role: v.role, content });
+    }
+    const messages = _messages.map((v) => {
       let parts: any[] = [{ text: getMessageTextContent(v) }];
       if (isVisionModel(options.config.model)) {
         const images = getMessageImages(v);

app/client/platforms/openai.ts

@@ -11,6 +11,7 @@ import {
 } from "@/app/constant";
 import { useAccessStore, useAppConfig, useChatStore } from "@/app/store";
 import { collectModelsWithDefaultModel } from "@/app/utils/model";
+import { preProcessImageContent } from "@/app/utils/chat";
 import { cloudflareAIGatewayUrl } from "@/app/utils/cloudflare";
 
 import {
@@ -105,10 +106,13 @@ export class ChatGPTApi implements LLMApi {
 
   async chat(options: ChatOptions) {
     const visionModel = isVisionModel(options.config.model);
-    const messages = options.messages.map((v) => ({
-      role: v.role,
-      content: visionModel ? v.content : getMessageTextContent(v),
-    }));
+    const messages = [];
+    for (const v of options.messages) {
+      const content = visionModel
+        ? await preProcessImageContent(v.content)
+        : getMessageTextContent(v);
+      messages.push({ role: v.role, content });
+    }
 
     const modelConfig = {
       ...useAppConfig.getState().modelConfig,

app/components/chat.tsx

@@ -61,7 +61,7 @@ import {
   isVisionModel,
 } from "../utils";
 
-import { compressImage } from "@/app/utils/chat";
+import { uploadImage as uploadImageRemote } from "@/app/utils/chat";
 
 import dynamic from "next/dynamic";
 
@@ -1167,7 +1167,7 @@ function _Chat() {
               ...(await new Promise<string[]>((res, rej) => {
                 setUploading(true);
                 const imagesData: string[] = [];
-                compressImage(file, 256 * 1024)
+                uploadImageRemote(file)
                   .then((dataUrl) => {
                     imagesData.push(dataUrl);
                     setUploading(false);
@@ -1209,7 +1209,7 @@ function _Chat() {
           const imagesData: string[] = [];
           for (let i = 0; i < files.length; i++) {
             const file = event.target.files[i];
-            compressImage(file, 256 * 1024)
+            uploadImageRemote(file)
               .then((dataUrl) => {
                 imagesData.push(dataUrl);
                 if (

app/constant.ts

@@ -21,6 +21,9 @@ export const BYTEDANCE_BASE_URL = "https://ark.cn-beijing.volces.com";
 
 export const ALIBABA_BASE_URL = "https://dashscope.aliyuncs.com/api/";
 
+export const CACHE_URL_PREFIX = "/api/cache";
+export const UPLOAD_URL = `${CACHE_URL_PREFIX}/upload`;
+
 export enum Path {
   Home = "/",
   Chat = "/chat",
@@ -239,7 +242,7 @@ const baiduModels = [
   "ernie-speed-128k",
   "ernie-speed-8k",
   "ernie-lite-8k",
-  "ernie-tiny-8k"
+  "ernie-tiny-8k",
 ];
 
 const bytedanceModels = [

app/utils/chat.ts

@@ -1,4 +1,5 @@
-import heic2any from "heic2any";
+import { CACHE_URL_PREFIX, UPLOAD_URL } from "@/app/constant";
+// import heic2any from "heic2any";
 
 export function compressImage(file: File, maxSize: number): Promise<string> {
   return new Promise((resolve, reject) => {
@@ -40,6 +41,7 @@ export function compressImage(file: File, maxSize: number): Promise<string> {
     reader.onerror = reject;
 
     if (file.type.includes("heic")) {
+      const heic2any = require("heic2any");
       heic2any({ blob: file, toType: "image/jpeg" })
         .then((blob) => {
           reader.readAsDataURL(blob as Blob);
@@ -52,3 +54,78 @@ export function compressImage(file: File, maxSize: number): Promise<string> {
     reader.readAsDataURL(file);
   });
 }
+
+export async function preProcessImageContent(
+  content: RequestMessage["content"],
+) {
+  if (typeof content === "string") {
+    return content;
+  }
+  const result = [];
+  for (const part of content) {
+    if (part?.type == "image_url" && part?.image_url?.url) {
+      const url = await cacheImageToBase64Image(part?.image_url?.url);
+      result.push({ type: part.type, image_url: { url } });
+    } else {
+      result.push({ ...part });
+    }
+  }
+  return result;
+}
+
+const imageCaches = {};
+export function cacheImageToBase64Image(imageUrl: string) {
+  if (imageUrl.includes(CACHE_URL_PREFIX)) {
+    if (!imageCaches[imageUrl]) {
+      const reader = new FileReader();
+      return fetch(imageUrl, {
+        method: "GET",
+        mode: "cors",
+        credentials: "include",
+      })
+        .then((res) => res.blob())
+        .then(
+          (blob) => (imageCaches[imageUrl] = compressImage(blob, 256 * 1024)),
+        ); // compressImage
+    }
+    return Promise.resolve(imageCaches[imageUrl]);
+  }
+  return imageUrl;
+}
+
+export function base64Image2Blob(base64Data: string, contentType: string) {
+  const byteCharacters = atob(base64Data);
+  const byteNumbers = new Array(byteCharacters.length);
+  for (let i = 0; i < byteCharacters.length; i++) {
+    byteNumbers[i] = byteCharacters.charCodeAt(i);
+  }
+  const byteArray = new Uint8Array(byteNumbers);
+  return new Blob([byteArray], { type: contentType });
+}
+
+export function uploadImage(file: File): Promise<string> {
+  const body = new FormData();
+  body.append("file", file);
+  return fetch(UPLOAD_URL, {
+    method: "post",
+    body,
+    mode: "cors",
+    credentials: "include",
+  })
+    .then((res) => res.json())
+    .then((res) => {
+      console.log("res", res);
+      if (res?.code == 0 && res?.data) {
+        return res?.data;
+      }
+      throw Error(`upload Error: ${res?.msg}`);
+    });
+}
+
+export function removeImage(imageUrl: string) {
+  return fetch(imageUrl, {
+    method: "DELETE",
+    mode: "cors",
+    credentials: "include",
+  });
+}