diff --git a/app/client/api.ts b/app/client/api.ts
index a826af12a..389c72e13 100644
--- a/app/client/api.ts
+++ b/app/client/api.ts
@@ -43,6 +43,7 @@ export interface MultimodalContent {
export interface UploadFile {
name: string;
url: string;
+ tokenCount?: string;
}
export interface RequestMessage {
diff --git a/app/components/chat.tsx b/app/components/chat.tsx
index 89f84ece4..617350f6b 100644
--- a/app/components/chat.tsx
+++ b/app/components/chat.tsx
@@ -74,6 +74,7 @@ import {
isDalle3,
showPlugins,
safeLocalStorage,
+ countTokens,
} from "../utils";
import type { UploadFile } from "../client/api";
@@ -1489,11 +1490,15 @@ function _Chat() {
setUploading(true);
const files = event.target.files;
const imagesData: UploadFile[] = [];
- for (let i = 0; i < files.length; i++) {
- const file = event.target.files[i];
- uploadImageRemote(file)
- .then((dataUrl) => {
- imagesData.push({ name: file.name, url: dataUrl });
+ (async () => {
+ for (let i = 0; i < files.length; i++) {
+ const file = files[i];
+ try {
+ const dataUrl = await uploadImageRemote(file);
+ const fileData: UploadFile = { name: file.name, url: dataUrl };
+ const tokenCount = await countTokens(fileData);
+ fileData.tokenCount = tokenCount;
+ imagesData.push(fileData);
if (
imagesData.length === 3 ||
imagesData.length === files.length
@@ -1501,12 +1506,12 @@ function _Chat() {
setUploading(false);
res(imagesData);
}
- })
- .catch((e) => {
+ } catch (e) {
setUploading(false);
rej(e);
- });
- }
+ }
+ }
+ })();
};
fileInput.click();
})),
@@ -1945,7 +1950,6 @@ function _Chat() {
.pop()
?.toLowerCase() as DefaultExtensionType;
const style = defaultStyles[extension];
-
return (
- {file.name}
+ {file.name} {file.tokenCount}
);
@@ -2082,22 +2086,22 @@ function _Chat() {
{attachImages.length == 0 && (
- {file.name}
+ {file.name} {file.tokenCount}
)}
{attachImages.length == 1 && (
- {file.name}
+ {file.name} {file.tokenCount}
)}
{attachImages.length == 2 && (
- {file.name}
+ {file.name} {file.tokenCount}
)}
{attachImages.length == 3 && (
- {file.name}
+ {file.name} {file.tokenCount}
)}
diff --git a/app/store/chat.ts b/app/store/chat.ts
index 8a9839e24..ecdb2d46c 100644
--- a/app/store/chat.ts
+++ b/app/store/chat.ts
@@ -19,7 +19,7 @@ import {
StoreKey,
} from "../constant";
import Locale, { getLang } from "../locales";
-import { isDalle3, safeLocalStorage } from "../utils";
+import { isDalle3, safeLocalStorage, readFileContent } from "../utils";
import { prettyObject } from "../utils/format";
import { createPersistStore } from "../utils/store";
import { estimateTokenLength } from "../utils/token";
@@ -154,23 +154,6 @@ function fillTemplateWith(input: string, modelConfig: ModelConfig) {
return output;
}
-const readFileContent = async (file: UploadFile): Promise => {
- try {
- const response = await fetch(file.url);
- if (!response.ok) {
- throw new Error(
- `Failed to fetch content from ${file.url}: ${response.statusText}`,
- );
- }
- const content = await response.text();
- const result = file.name + "\n" + content;
- return result;
- } catch (error) {
- console.error("Error reading file content:", error);
- return "";
- }
-};
-
const DEFAULT_CHAT_STATE = {
sessions: [createEmptySession()],
currentSessionIndex: 0,
diff --git a/app/utils.ts b/app/utils.ts
index 68c3f8291..479803cfb 100644
--- a/app/utils.ts
+++ b/app/utils.ts
@@ -17,6 +17,54 @@ export function trimTopic(topic: string) {
);
}
+export const readFileContent = async (file: UploadFile): Promise => {
+ try {
+ const response = await fetch(file.url);
+ if (!response.ok) {
+ throw new Error(
+ `Failed to fetch content from ${file.url}: ${response.statusText}`,
+ );
+ }
+ const content = await response.text();
+ const result = file.name + "\n" + content;
+ return result;
+ } catch (error) {
+ console.error("Error reading file content:", error);
+ return "";
+ }
+};
+
+export const countTokens = async (file: UploadFile) => {
+ const text = await readFileContent(file);
+ let totalTokens = 0;
+
+ for (let i = 0; i < text.length; i++) {
+ const char = text[i];
+ const nextChar = text[i + 1];
+
+ if (char === " " && nextChar === " ") {
+ totalTokens += 0.081;
+ } else if ("NORabcdefghilnopqrstuvy ".includes(char)) {
+ totalTokens += 0.202;
+ } else if ("CHLMPQSTUVfkmspwx".includes(char)) {
+ totalTokens += 0.237;
+ } else if ("-.ABDEFGIKWY_\\r\\tz{ü".includes(char)) {
+ totalTokens += 0.304;
+ } else if ("!{{input}}(/;=JX`j\\n}ö".includes(char)) {
+ totalTokens += 0.416;
+ } else if ('"#%)*+56789<>?@Z[\\]^|§«äç’'.includes(char)) {
+ totalTokens += 0.479;
+ } else if (",01234:~Üß".includes(char) || char.charCodeAt(0) > 255) {
+ totalTokens += 0.658;
+ } else {
+ totalTokens += 0.98;
+ }
+ }
+ let totalTokenCount = (totalTokens / 1000).toFixed(2).toString() + "K";
+ console.log(totalTokenCount);
+ return totalTokenCount;
+};
+
export async function copyToClipboard(text: string) {
try {
if (window.__TAURI__) {