Add vision support (#4076)

app/utils.ts

@@ -1,12 +1,16 @@
 import { useEffect, useState } from "react";
 import { showToast } from "./components/ui-lib";
 import Locale from "./locales";
+import { RequestMessage } from "./client/api";
+import { DEFAULT_MODELS } from "./constant";
 
 export function trimTopic(topic: string) {
   // Fix an issue where double quotes still show in the Indonesian language
   // This will remove the specified punctuation from the end of the string
   // and also trim quotes from both the start and end if they exist.
-  return topic.replace(/^["“”]+|["“”]+$/g, "").replace(/[，。！？”“"、,.!?]*$/, "");
+  return topic
+    .replace(/^["“”]+|["“”]+$/g, "")
+    .replace(/[，。！？”“"、,.!?]*$/, "");
 }
 
 export async function copyToClipboard(text: string) {
@@ -40,8 +44,8 @@ export async function downloadAs(text: string, filename: string) {
       defaultPath: `${filename}`,
       filters: [
         {
-          name: `${filename.split('.').pop()} files`,
-          extensions: [`${filename.split('.').pop()}`],
+          name: `${filename.split(".").pop()} files`,
+          extensions: [`${filename.split(".").pop()}`],
         },
         {
           name: "All Files",
@@ -54,7 +58,7 @@ export async function downloadAs(text: string, filename: string) {
       try {
         await window.__TAURI__.fs.writeBinaryFile(
           result,
-          new Uint8Array([...text].map((c) => c.charCodeAt(0)))
+          new Uint8Array([...text].map((c) => c.charCodeAt(0))),
         );
         showToast(Locale.Download.Success);
       } catch (error) {
@@ -69,16 +73,59 @@ export async function downloadAs(text: string, filename: string) {
       "href",
       "data:text/plain;charset=utf-8," + encodeURIComponent(text),
     );
-  element.setAttribute("download", filename);
+    element.setAttribute("download", filename);
 
-  element.style.display = "none";
-  document.body.appendChild(element);
+    element.style.display = "none";
+    document.body.appendChild(element);
 
-  element.click();
+    element.click();
 
-  document.body.removeChild(element);
+    document.body.removeChild(element);
+  }
 }
+
+export function compressImage(file: File, maxSize: number): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = (readerEvent: any) => {
+      const image = new Image();
+      image.onload = () => {
+        let canvas = document.createElement("canvas");
+        let ctx = canvas.getContext("2d");
+        let width = image.width;
+        let height = image.height;
+        let quality = 0.9;
+        let dataUrl;
+
+        do {
+          canvas.width = width;
+          canvas.height = height;
+          ctx?.clearRect(0, 0, canvas.width, canvas.height);
+          ctx?.drawImage(image, 0, 0, width, height);
+          dataUrl = canvas.toDataURL("image/jpeg", quality);
+
+          if (dataUrl.length < maxSize) break;
+
+          if (quality > 0.5) {
+            // Prioritize quality reduction
+            quality -= 0.1;
+          } else {
+            // Then reduce the size
+            width *= 0.9;
+            height *= 0.9;
+          }
+        } while (dataUrl.length > maxSize);
+
+        resolve(dataUrl);
+      };
+      image.onerror = reject;
+      image.src = readerEvent.target.result;
+    };
+    reader.onerror = reject;
+    reader.readAsDataURL(file);
+  });
 }
+
 export function readFromFile() {
   return new Promise<string>((res, rej) => {
     const fileInput = document.createElement("input");
@@ -212,8 +259,41 @@ export function getCSSVar(varName: string) {
 export function isMacOS(): boolean {
   if (typeof window !== "undefined") {
     let userAgent = window.navigator.userAgent.toLocaleLowerCase();
-    const macintosh = /iphone|ipad|ipod|macintosh/.test(userAgent)
-    return !!macintosh
+    const macintosh = /iphone|ipad|ipod|macintosh/.test(userAgent);
+    return !!macintosh;
   }
-  return false
+  return false;
+}
+
+export function getMessageTextContent(message: RequestMessage) {
+  if (typeof message.content === "string") {
+    return message.content;
+  }
+  for (const c of message.content) {
+    if (c.type === "text") {
+      return c.text ?? "";
+    }
+  }
+  return "";
+}
+
+export function getMessageImages(message: RequestMessage): string[] {
+  if (typeof message.content === "string") {
+    return [];
+  }
+  const urls: string[] = [];
+  for (const c of message.content) {
+    if (c.type === "image_url") {
+      urls.push(c.image_url?.url ?? "");
+    }
+  }
+  return urls;
+}
+
+export function isVisionModel(model: string) {
+  return (
+    model.startsWith("gpt-4-vision") ||
+    model.startsWith("gemini-pro-vision") ||
+    !DEFAULT_MODELS.find((m) => m.name == model)
+  );
 }