From 74ac8b612b7b7a568e18f51c31932603b033b681 Mon Sep 17 00:00:00 2001 From: Sheng Fan Date: Tue, 9 Apr 2024 22:04:37 +0800 Subject: [PATCH] add video conclusion plugin --- .../bilibili_vid_conclusion.ts | 138 ++++++++++++++++++ app/api/langchain-tools/nodejs_tools.ts | 3 + app/plugins/cn.ts | 10 ++ 3 files changed, 151 insertions(+) create mode 100644 app/api/langchain-tools/bilibili_vid_conclusion.ts diff --git a/app/api/langchain-tools/bilibili_vid_conclusion.ts b/app/api/langchain-tools/bilibili_vid_conclusion.ts new file mode 100644 index 000000000..4bd6b1864 --- /dev/null +++ b/app/api/langchain-tools/bilibili_vid_conclusion.ts @@ -0,0 +1,138 @@ +import { Tool } from "@langchain/core/tools"; +import { getRandomUserAgent } from "./ua_tools"; +import { encWbi, getWbiKeys } from "./bili_wbi_tools"; + +export interface Headers { + [key: string]: string; +} + +export interface RequestTool { + headers: Headers; + maxOutputLength?: number; + timeout: number; +} + +export class BilibiliVideoConclusionTool extends Tool implements RequestTool { + name = "bilibili_video_conclusion"; + + maxOutputLength = Infinity; + + timeout = 90000; + + constructor( + public headers: Headers = {}, + { maxOutputLength }: { maxOutputLength?: number } = {}, + { timeout }: { timeout?: number } = {}, + ) { + super(...arguments); + + this.maxOutputLength = maxOutputLength ?? this.maxOutputLength; + this.timeout = timeout ?? this.timeout; + } + + /** @ignore */ + async _call(query: string) { + try { + var [videoAid, pid] = query.split(","); + // check if arguments are valid + // is videoAid a string of numbers? + if (!(/^\d+$/.test(videoAid) || /^av\d+$/.test(videoAid))) { + throw new Error( + "Invalid videoAid: It should be a string of numbers. If a BVid or a short link is given, please convert it to Aid using av{Aid} format using BiliVideoInfo tool.", + ); + } + if (!/^\d+$/.test(pid)) { + throw new Error( + "Invalid pid: it should be a number representing the page number of the video, starting from 1.", + ); + } + + if (videoAid.startsWith("av")) videoAid = videoAid.slice(2); + + let result = await this.fetchVideoInfo(+videoAid, +pid); + // console.log(result) + return result; + } catch (error) { + console.error(error); + return (error as Error).toString(); + } + } + + async fetchVideoInfo(aid: number, pid: number): Promise { + const headers = new Headers(); + headers.append("User-Agent", getRandomUserAgent()); + headers.append("Referer", "https://www.bilibili.com/video/av" + aid); + headers.append("Origin", "https://www.bilibili.com"); + headers.append("Cookie", process.env.BILIBILI_COOKIES || ""); + + const vidinfo_resp = await this.fetchWithTimeout( + `https://api.bilibili.com/x/web-interface/view?aid=${aid}`, + { + headers: headers, + }, + ); + + const vidinfo = await vidinfo_resp.json(); + const mid = vidinfo.data.owner.mid; + const cid = vidinfo.data.pages[pid - 1].cid; + + // get https://api.bilibili.com/x/web-interface/view/conclusion/get?aid={aid}&cid={cid}&up_mid={mid} + const { img_key, sub_key } = await getWbiKeys(); + var queryStr = encWbi( + { + aid: aid, + cid: cid, + up_mid: mid, + }, + img_key, + sub_key, + ); + + const resp = await this.fetchWithTimeout( + `https://api.bilibili.com/x/web-interface/view/conclusion/get?${queryStr}`, + { + headers: headers, + }, + ); + + const data = await resp.json(); + const model_result = data.data.model_result; + switch (model_result.result_type) { + case 0: + return "BiliAPI returned result: Unable to provide a conclusion for this video, or the conclusion cannot be generated for this video due to the video quality or the video content."; + case 1: + return model_result.summary; + case 2: + var resText = + model_result.summary + "\n\nOutline (generated by BiliAPI): \n"; + + for (var i = 0; i < model_result.outline.length; i++) { + let ol = model_result.outline[i]; + resText += `## [position: ${ol.timestamp}s] ${ol.title}\n`; + ol.part_outline.forEach((olpart: any) => { + resText += `- [position: ${olpart.timestamp}s] ${olpart.content}`; + }); + } + return resText; + default: + return "BiliAPI returned unknown result type. Please report this issue to the developer."; + } + } + + async fetchWithTimeout( + resource: RequestInfo | URL, + options = {}, + timeout: number = 30000, + ) { + const controller = new AbortController(); + const id = setTimeout(() => controller.abort(), timeout); + const response = await fetch(resource, { + ...options, + signal: controller.signal, + }); + clearTimeout(id); + return response; + } + + description = `A tool to fetch the conclusion of a Bilibili video. It uses the BiliAPI to fetch the conclusion of the video. Input string should be in the format of "videoAid,pid" where videoAid is the Aid of the video and pid is the page number of the video, starting from 1.`; +} diff --git a/app/api/langchain-tools/nodejs_tools.ts b/app/api/langchain-tools/nodejs_tools.ts index f4033df53..4389b7c46 100644 --- a/app/api/langchain-tools/nodejs_tools.ts +++ b/app/api/langchain-tools/nodejs_tools.ts @@ -11,6 +11,7 @@ import { BilibiliVideoInfoTool } from "./bilibili_vid_info"; import { BilibiliVideoSearchTool } from "./bilibili_vid_search"; import { BilibiliMusicRecognitionTool } from "./bilibili_music_recognition"; import { RAGSearch } from "./rag_search"; +import { BilibiliVideoConclusionTool } from "./bilibili_vid_conclusion"; export class NodeJSTool { private apiKey: string | undefined; @@ -57,6 +58,7 @@ export class NodeJSTool { const bilibiliVideoInfoTool = new BilibiliVideoInfoTool(); const bilibiliVideoSearchTool = new BilibiliVideoSearchTool(); const bilibiliMusicRecognitionTool = new BilibiliMusicRecognitionTool(); + const bilibiliVideoConclusionTool = new BilibiliVideoConclusionTool(); let tools = [ calculatorTool, webBrowserTool, @@ -68,6 +70,7 @@ export class NodeJSTool { bilibiliVideoInfoTool, bilibiliVideoSearchTool, bilibiliMusicRecognitionTool, + bilibiliVideoConclusionTool, ]; if (!!process.env.ENABLE_RAG) { tools.push(new RAGSearch(this.sessionId, this.model, this.ragEmbeddings)); diff --git a/app/plugins/cn.ts b/app/plugins/cn.ts index 8595d2395..606f96560 100644 --- a/app/plugins/cn.ts +++ b/app/plugins/cn.ts @@ -62,6 +62,16 @@ export const CN_PLUGINS: BuiltinPlugin[] = [ enable: true, onlyNodeRuntime: true, }, + { + name: "Bilibili视频总结", + toolName: "bilibili_video_conclusion", + lang: "cn", + description: "通过Bilibili视频ID进行视频总结。", + builtin: true, + createdAt: 1712394126000, + enable: true, + onlyNodeRuntime: true, + }, { name: "维基百科", toolName: "WikipediaQueryRun",