diff options
| -rw-r--r-- | apps/web/components/settings/AISettings.tsx | 11 | ||||
| -rw-r--r-- | apps/workers/workers/inference/summarize.ts | 2 | ||||
| -rw-r--r-- | apps/workers/workers/inference/tagging.ts | 6 | ||||
| -rw-r--r-- | packages/shared/prompts.ts | 126 | ||||
| -rw-r--r-- | packages/trpc/routers/bookmarks.ts | 2 |
5 files changed, 110 insertions, 37 deletions
diff --git a/apps/web/components/settings/AISettings.tsx b/apps/web/components/settings/AISettings.tsx index 6ec71a91..beaa93dc 100644 --- a/apps/web/components/settings/AISettings.tsx +++ b/apps/web/components/settings/AISettings.tsx @@ -29,8 +29,8 @@ import { z } from "zod"; import { buildImagePrompt, - buildSummaryPrompt, - buildTextPrompt, + buildSummaryPromptUntruncated, + buildTextPromptUntruncated, } from "@karakeep/shared/prompts"; import { zNewPromptSchema, @@ -301,6 +301,7 @@ export function PromptDemo() { const { t } = useTranslation(); const { data: prompts } = api.prompts.list.useQuery(); const clientConfig = useClientConfig(); + return ( <div className="flex flex-col gap-2"> <div className="mb-4 w-full text-xl font-medium sm:w-1/3"> @@ -308,7 +309,7 @@ export function PromptDemo() { </div> <p>{t("settings.ai.text_prompt")}</p> <code className="whitespace-pre-wrap rounded-md bg-muted p-3 text-sm text-muted-foreground"> - {buildTextPrompt( + {buildTextPromptUntruncated( clientConfig.inference.inferredTagLang, (prompts ?? []) .filter( @@ -316,7 +317,6 @@ export function PromptDemo() { ) .map((p) => p.text), "\n<CONTENT_HERE>\n", - /* context length */ 1024 /* The value here doesn't matter */, ).trim()} </code> <p>{t("settings.ai.images_prompt")}</p> @@ -332,13 +332,12 @@ export function PromptDemo() { </code> <p>{t("settings.ai.summarization_prompt")}</p> <code className="whitespace-pre-wrap rounded-md bg-muted p-3 text-sm text-muted-foreground"> - {buildSummaryPrompt( + {buildSummaryPromptUntruncated( clientConfig.inference.inferredTagLang, (prompts ?? []) .filter((p) => p.appliesTo == "summary") .map((p) => p.text), "\n<CONTENT_HERE>\n", - /* context length */ 1024 /* The value here doesn't matter */, ).trim()} </code> </div> diff --git a/apps/workers/workers/inference/summarize.ts b/apps/workers/workers/inference/summarize.ts index 0fa24869..23636961 100644 --- a/apps/workers/workers/inference/summarize.ts +++ b/apps/workers/workers/inference/summarize.ts @@ -105,7 +105,7 @@ URL: ${link.url ?? ""} }, }); - const summaryPrompt = buildSummaryPrompt( + const summaryPrompt = await buildSummaryPrompt( serverConfig.inference.inferredTagLang, prompts.map((p) => p.text), textToSummarize, diff --git a/apps/workers/workers/inference/tagging.ts b/apps/workers/workers/inference/tagging.ts index 50a03b00..5a79fd22 100644 --- a/apps/workers/workers/inference/tagging.ts +++ b/apps/workers/workers/inference/tagging.ts @@ -94,7 +94,7 @@ async function buildPrompt( ); return null; } - return buildTextPrompt( + return await buildTextPrompt( serverConfig.inference.inferredTagLang, prompts, `URL: ${bookmark.link.url} @@ -106,7 +106,7 @@ Content: ${content ?? ""}`, } if (bookmark.text) { - return buildTextPrompt( + return await buildTextPrompt( serverConfig.inference.inferredTagLang, prompts, bookmark.text.text ?? "", @@ -215,7 +215,7 @@ async function inferTagsFromPDF( inferenceClient: InferenceClient, abortSignal: AbortSignal, ) { - const prompt = buildTextPrompt( + const prompt = await buildTextPrompt( serverConfig.inference.inferredTagLang, await fetchCustomPrompts(bookmark.userId, "text"), `Content: ${bookmark.asset.content}`, diff --git a/packages/shared/prompts.ts b/packages/shared/prompts.ts index 0b79eb9a..5a6a705e 100644 --- a/packages/shared/prompts.ts +++ b/packages/shared/prompts.ts @@ -1,6 +1,19 @@ -import { getEncoding } from "js-tiktoken"; +import type { Tiktoken } from "js-tiktoken"; -const encoding = getEncoding("o200k_base"); +let encoding: Tiktoken | null = null; + +/** + * Lazy load the encoding to avoid loading the tiktoken data into memory + * until it's actually needed + */ +async function getEncodingInstance(): Promise<Tiktoken> { + if (!encoding) { + // Dynamic import to lazy load the tiktoken module + const { getEncoding } = await import("js-tiktoken"); + encoding = getEncoding("o200k_base"); + } + return encoding; +} /** * Remove duplicate whitespaces to avoid tokenization issues @@ -9,17 +22,22 @@ function preprocessContent(content: string) { return content.replace(/(\s){10,}/g, "$1"); } -function calculateNumTokens(text: string) { - return encoding.encode(text).length; +async function calculateNumTokens(text: string): Promise<number> { + const enc = await getEncodingInstance(); + return enc.encode(text).length; } -function truncateContent(content: string, length: number) { - const tokens = encoding.encode(content); +async function truncateContent( + content: string, + length: number, +): Promise<string> { + const enc = await getEncodingInstance(); + const tokens = enc.encode(content); if (tokens.length <= length) { return content; } const truncatedTokens = tokens.slice(0, length); - return encoding.decode(truncatedTokens); + return enc.decode(truncatedTokens); } export function buildImagePrompt(lang: string, customPrompts: string[]) { @@ -35,14 +53,15 @@ ${customPrompts && customPrompts.map((p) => `- ${p}`).join("\n")} You must respond in valid JSON with the key "tags" and the value is list of tags. Don't wrap the response in a markdown code.`; } -export function buildTextPrompt( +/** + * Construct tagging prompt for text content + */ +function constructTextTaggingPrompt( lang: string, customPrompts: string[], content: string, - contextLength: number, -) { - content = preprocessContent(content); - const constructPrompt = (c: string) => ` +): string { + return ` You are an expert whose responsibility is to help with automatic tagging for a read-it-later app. Please analyze the TEXT_CONTENT below and suggest relevant tags that describe its key themes, topics, and main ideas. The rules are: - Aim for a variety of tags, including broad categories, specific keywords, and potential sub-genres. @@ -54,30 +73,85 @@ Please analyze the TEXT_CONTENT below and suggest relevant tags that describe it ${customPrompts && customPrompts.map((p) => `- ${p}`).join("\n")} <TEXT_CONTENT> -${c} +${content} </TEXT_CONTENT> You must respond in JSON with the key "tags" and the value is an array of string tags.`; - - const promptSize = calculateNumTokens(constructPrompt("")); - const truncatedContent = truncateContent(content, contextLength - promptSize); - return constructPrompt(truncatedContent); } -export function buildSummaryPrompt( +/** + * Construct summary prompt + */ +function constructSummaryPrompt( lang: string, customPrompts: string[], content: string, - contextLength: number, -) { - content = preprocessContent(content); - const constructPrompt = (c: string) => ` +): string { + return ` Summarize the following content responding ONLY with the summary. You MUST follow the following rules: - Summary must be in 3-4 sentences. - The summary must be in ${lang}. ${customPrompts && customPrompts.map((p) => `- ${p}`).join("\n")} - ${c}`; + ${content}`; +} + +/** + * Build text tagging prompt without truncation (for previews/UI) + */ +export function buildTextPromptUntruncated( + lang: string, + customPrompts: string[], + content: string, +): string { + return constructTextTaggingPrompt( + lang, + customPrompts, + preprocessContent(content), + ); +} + +export async function buildTextPrompt( + lang: string, + customPrompts: string[], + content: string, + contextLength: number, +): Promise<string> { + content = preprocessContent(content); + const promptTemplate = constructTextTaggingPrompt(lang, customPrompts, ""); + const promptSize = await calculateNumTokens(promptTemplate); + const truncatedContent = await truncateContent( + content, + contextLength - promptSize, + ); + return constructTextTaggingPrompt(lang, customPrompts, truncatedContent); +} + +export async function buildSummaryPrompt( + lang: string, + customPrompts: string[], + content: string, + contextLength: number, +): Promise<string> { + content = preprocessContent(content); + const promptTemplate = constructSummaryPrompt(lang, customPrompts, ""); + const promptSize = await calculateNumTokens(promptTemplate); + const truncatedContent = await truncateContent( + content, + contextLength - promptSize, + ); + return constructSummaryPrompt(lang, customPrompts, truncatedContent); +} - const promptSize = calculateNumTokens(constructPrompt("")); - const truncatedContent = truncateContent(content, contextLength - promptSize); - return constructPrompt(truncatedContent); +/** + * Build summary prompt without truncation (for previews/UI) + */ +export function buildSummaryPromptUntruncated( + lang: string, + customPrompts: string[], + content: string, +): string { + return constructSummaryPrompt( + lang, + customPrompts, + preprocessContent(content), + ); } diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts index 05ffa240..f68d5ada 100644 --- a/packages/trpc/routers/bookmarks.ts +++ b/packages/trpc/routers/bookmarks.ts @@ -954,7 +954,7 @@ Author: ${bookmark.author ?? ""} }, }); - const summaryPrompt = buildSummaryPrompt( + const summaryPrompt = await buildSummaryPrompt( serverConfig.inference.inferredTagLang, prompts.map((p) => p.text), bookmarkDetails, |