diff options
25 files changed, 2327 insertions, 123 deletions
diff --git a/apps/web/components/admin/BackgroundJobs.tsx b/apps/web/components/admin/BackgroundJobs.tsx index 217e2ad9..ac5885ef 100644 --- a/apps/web/components/admin/BackgroundJobs.tsx +++ b/apps/web/components/admin/BackgroundJobs.tsx @@ -127,7 +127,7 @@ function AdminActions() { variant="destructive" loading={isInferencePending} onClick={() => - reRunInferenceOnAllBookmarks({ taggingStatus: "failure" }) + reRunInferenceOnAllBookmarks({ type: "tag", status: "failure" }) } > {t("admin.actions.regenerate_ai_tags_for_failed_bookmarks_only")} @@ -135,12 +135,32 @@ function AdminActions() { <ActionButton variant="destructive" loading={isInferencePending} - onClick={() => reRunInferenceOnAllBookmarks({ taggingStatus: "all" })} + onClick={() => + reRunInferenceOnAllBookmarks({ type: "tag", status: "all" }) + } > {t("admin.actions.regenerate_ai_tags_for_all_bookmarks")} </ActionButton> <ActionButton variant="destructive" + loading={isInferencePending} + onClick={() => + reRunInferenceOnAllBookmarks({ type: "summarize", status: "failure" }) + } + > + {t("admin.actions.regenerate_ai_summaries_for_failed_bookmarks_only")} + </ActionButton> + <ActionButton + variant="destructive" + loading={isInferencePending} + onClick={() => + reRunInferenceOnAllBookmarks({ type: "summarize", status: "all" }) + } + > + {t("admin.actions.regenerate_ai_summaries_for_all_bookmarks")} + </ActionButton> + <ActionButton + variant="destructive" loading={isReindexPending} onClick={() => reindexBookmarks()} > diff --git a/apps/web/lib/i18n/locales/en/translation.json b/apps/web/lib/i18n/locales/en/translation.json index c26c9523..1eef3ac4 100644 --- a/apps/web/lib/i18n/locales/en/translation.json +++ b/apps/web/lib/i18n/locales/en/translation.json @@ -249,6 +249,8 @@ "without_inference": "Without Inference", "regenerate_ai_tags_for_failed_bookmarks_only": "Regenerate AI Tags for Failed Bookmarks Only", "regenerate_ai_tags_for_all_bookmarks": "Regenerate AI Tags for All Bookmarks", + "regenerate_ai_summaries_for_failed_bookmarks_only": "Regenerate AI Summaries for Failed Bookmarks Only", + "regenerate_ai_summaries_for_all_bookmarks": "Regenerate AI Summaries for All Bookmarks", "reindex_all_bookmarks": "Reindex All Bookmarks", "compact_assets": "Compact Assets", "reprocess_assets_fix_mode": "Reprocess Assets (Fix Mode)" diff --git a/apps/workers/index.ts b/apps/workers/index.ts index 208666c7..1cc1ce49 100644 --- a/apps/workers/index.ts +++ b/apps/workers/index.ts @@ -1,20 +1,19 @@ import "dotenv/config"; -import { AssetPreprocessingWorker } from "assetPreprocessingWorker"; -import { FeedRefreshingWorker, FeedWorker } from "feedWorker"; -import { RuleEngineWorker } from "ruleEngineWorker"; -import { TidyAssetsWorker } from "tidyAssetsWorker"; - import serverConfig from "@karakeep/shared/config"; import logger from "@karakeep/shared/logger"; import { runQueueDBMigrations } from "@karakeep/shared/queues"; -import { CrawlerWorker } from "./crawlerWorker"; import { shutdownPromise } from "./exit"; -import { OpenAiWorker } from "./openaiWorker"; -import { SearchIndexingWorker } from "./searchWorker"; -import { VideoWorker } from "./videoWorker"; -import { WebhookWorker } from "./webhookWorker"; +import { AssetPreprocessingWorker } from "./workers/assetPreprocessingWorker"; +import { CrawlerWorker } from "./workers/crawlerWorker"; +import { FeedRefreshingWorker, FeedWorker } from "./workers/feedWorker"; +import { OpenAiWorker } from "./workers/inference/inferenceWorker"; +import { RuleEngineWorker } from "./workers/ruleEngineWorker"; +import { SearchIndexingWorker } from "./workers/searchWorker"; +import { TidyAssetsWorker } from "./workers/tidyAssetsWorker"; +import { VideoWorker } from "./workers/videoWorker"; +import { WebhookWorker } from "./workers/webhookWorker"; async function main() { logger.info(`Workers version: ${serverConfig.serverVersion ?? "not set"}`); @@ -22,7 +21,7 @@ async function main() { const [ crawler, - openai, + inference, search, tidyAssets, video, @@ -46,7 +45,7 @@ async function main() { await Promise.any([ Promise.all([ crawler.run(), - openai.run(), + inference.run(), search.run(), tidyAssets.run(), video.run(), @@ -58,12 +57,12 @@ async function main() { shutdownPromise, ]); logger.info( - "Shutting down crawler, openai, tidyAssets, video, feed, assetPreprocessing, webhook, ruleEngine and search workers ...", + "Shutting down crawler, inference, tidyAssets, video, feed, assetPreprocessing, webhook, ruleEngine and search workers ...", ); FeedRefreshingWorker.stop(); crawler.stop(); - openai.stop(); + inference.stop(); search.stop(); tidyAssets.stop(); video.stop(); diff --git a/apps/workers/assetPreprocessingWorker.ts b/apps/workers/workers/assetPreprocessingWorker.ts index a678b706..0c0b7aec 100644 --- a/apps/workers/assetPreprocessingWorker.ts +++ b/apps/workers/workers/assetPreprocessingWorker.ts @@ -330,6 +330,7 @@ async function run(req: DequeuedJob<AssetPreprocessingRequest>) { if (!isFixMode || anythingChanged) { await OpenAIQueue.enqueue({ bookmarkId, + type: "tag", }); // Update the search index diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts index a40cbe53..b928e145 100644 --- a/apps/workers/crawlerWorker.ts +++ b/apps/workers/workers/crawlerWorker.ts @@ -859,6 +859,11 @@ async function runCrawler(job: DequeuedJob<ZCrawlLinkRequest>) { if (job.data.runInference !== false) { await OpenAIQueue.enqueue({ bookmarkId, + type: "tag", + }); + await OpenAIQueue.enqueue({ + bookmarkId, + type: "summarize", }); } diff --git a/apps/workers/feedWorker.ts b/apps/workers/workers/feedWorker.ts index 1eaba0c3..1eaba0c3 100644 --- a/apps/workers/feedWorker.ts +++ b/apps/workers/workers/feedWorker.ts diff --git a/apps/workers/workers/inference/inferenceWorker.ts b/apps/workers/workers/inference/inferenceWorker.ts new file mode 100644 index 00000000..f7492c8b --- /dev/null +++ b/apps/workers/workers/inference/inferenceWorker.ts @@ -0,0 +1,100 @@ +import { eq } from "drizzle-orm"; +import { DequeuedJob, Runner } from "liteque"; + +import type { ZOpenAIRequest } from "@karakeep/shared/queues"; +import { db } from "@karakeep/db"; +import { bookmarks } from "@karakeep/db/schema"; +import serverConfig from "@karakeep/shared/config"; +import { InferenceClientFactory } from "@karakeep/shared/inference"; +import logger from "@karakeep/shared/logger"; +import { OpenAIQueue, zOpenAIRequestSchema } from "@karakeep/shared/queues"; + +import { runSummarization } from "./summarize"; +import { runTagging } from "./tagging"; + +async function attemptMarkStatus( + jobData: object | undefined, + status: "success" | "failure", +) { + if (!jobData) { + return; + } + try { + const request = zOpenAIRequestSchema.parse(jobData); + await db + .update(bookmarks) + .set({ + ...(request.type === "summarize" + ? { summarizationStatus: status } + : {}), + ...(request.type === "tag" ? { taggingStatus: status } : {}), + }) + .where(eq(bookmarks.id, request.bookmarkId)); + } catch (e) { + logger.error(`Something went wrong when marking the tagging status: ${e}`); + } +} + +export class OpenAiWorker { + static build() { + logger.info("Starting inference worker ..."); + const worker = new Runner<ZOpenAIRequest>( + OpenAIQueue, + { + run: runOpenAI, + onComplete: async (job) => { + const jobId = job.id; + logger.info(`[inference][${jobId}] Completed successfully`); + await attemptMarkStatus(job.data, "success"); + }, + onError: async (job) => { + const jobId = job.id; + logger.error( + `[inference][${jobId}] inference job failed: ${job.error}\n${job.error.stack}`, + ); + if (job.numRetriesLeft == 0) { + await attemptMarkStatus(job?.data, "failure"); + } + }, + }, + { + concurrency: 1, + pollIntervalMs: 1000, + timeoutSecs: serverConfig.inference.jobTimeoutSec, + }, + ); + + return worker; + } +} + +async function runOpenAI(job: DequeuedJob<ZOpenAIRequest>) { + const jobId = job.id; + + const inferenceClient = InferenceClientFactory.build(); + if (!inferenceClient) { + logger.debug( + `[inference][${jobId}] No inference client configured, nothing to do now`, + ); + return; + } + + const request = zOpenAIRequestSchema.safeParse(job.data); + if (!request.success) { + throw new Error( + `[inference][${jobId}] Got malformed job request: ${request.error.toString()}`, + ); + } + + const { bookmarkId } = request.data; + switch (request.data.type) { + case "summarize": + await runSummarization(bookmarkId, job, inferenceClient); + break; + case "tag": + await runTagging(bookmarkId, job, inferenceClient); + break; + default: + throw new Error(`Unknown inference type: ${request.data.type}`); + } +} diff --git a/apps/workers/workers/inference/summarize.ts b/apps/workers/workers/inference/summarize.ts new file mode 100644 index 00000000..a832fe0a --- /dev/null +++ b/apps/workers/workers/inference/summarize.ts @@ -0,0 +1,123 @@ +import { and, eq } from "drizzle-orm"; +import { DequeuedJob } from "liteque"; + +import { db } from "@karakeep/db"; +import { bookmarks, customPrompts } from "@karakeep/db/schema"; +import serverConfig from "@karakeep/shared/config"; +import { InferenceClient } from "@karakeep/shared/inference"; +import logger from "@karakeep/shared/logger"; +import { buildSummaryPrompt } from "@karakeep/shared/prompts"; +import { triggerSearchReindex, ZOpenAIRequest } from "@karakeep/shared/queues"; +import { BookmarkTypes } from "@karakeep/shared/types/bookmarks"; + +async function fetchBookmarkDetailsForSummary(bookmarkId: string) { + const bookmark = await db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, bookmarkId), + columns: { id: true, userId: true, type: true }, + with: { + link: { + columns: { + title: true, + description: true, + content: true, + publisher: true, + author: true, + url: true, + }, + }, + // If assets (like PDFs with extracted text) should be summarized, extend here + }, + }); + + if (!bookmark) { + throw new Error(`Bookmark with id ${bookmarkId} not found`); + } + return bookmark; +} + +export async function runSummarization( + bookmarkId: string, + job: DequeuedJob<ZOpenAIRequest>, + inferenceClient: InferenceClient, +) { + if (!serverConfig.inference.enableAutoSummarization) { + logger.info( + `[inference][${job.id}] Skipping summarization job for bookmark with id "${bookmarkId}" because it's disabled in the config.`, + ); + return; + } + const jobId = job.id; + + logger.info( + `[inference][${jobId}] Starting a summary job for bookmark with id "${bookmarkId}"`, + ); + + const bookmarkData = await fetchBookmarkDetailsForSummary(bookmarkId); + + let textToSummarize = ""; + if (bookmarkData.type === BookmarkTypes.LINK && bookmarkData.link) { + const link = bookmarkData.link; + textToSummarize = ` +Title: ${link.title ?? ""} +Description: ${link.description ?? ""} +Content: ${link.content ?? ""} +Publisher: ${link.publisher ?? ""} +Author: ${link.author ?? ""} +URL: ${link.url ?? ""} +`; + } else { + logger.warn( + `[inference][${jobId}] Bookmark ${bookmarkId} (type: ${bookmarkData.type}) is not a LINK or TEXT type with content, or content is missing. Skipping summary.`, + ); + return; + } + + if (!textToSummarize.trim()) { + logger.info( + `[inference][${jobId}] No content to summarize for bookmark ${bookmarkId}.`, + ); + return; + } + + const prompts = await db.query.customPrompts.findMany({ + where: and( + eq(customPrompts.userId, bookmarkData.userId), + eq(customPrompts.appliesTo, "summary"), + ), + columns: { + text: true, + }, + }); + + const summaryPrompt = buildSummaryPrompt( + serverConfig.inference.inferredTagLang, + prompts.map((p) => p.text), + textToSummarize, + serverConfig.inference.contextLength, + ); + + const summaryResult = await inferenceClient.inferFromText(summaryPrompt, { + schema: null, // Summaries are typically free-form text + abortSignal: job.abortSignal, + }); + + if (!summaryResult.response) { + throw new Error( + `[inference][${jobId}] Failed to summarize bookmark ${bookmarkId}, empty response from inference client.`, + ); + } + + logger.info( + `[inference][${jobId}] Generated summary for bookmark "${bookmarkId}" using ${summaryResult.totalTokens} tokens.`, + ); + + await db + .update(bookmarks) + .set({ + summary: summaryResult.response, + modifiedAt: new Date(), + }) + .where(eq(bookmarks.id, bookmarkId)); + + await triggerSearchReindex(bookmarkId); +} diff --git a/apps/workers/openaiWorker.ts b/apps/workers/workers/inference/tagging.ts index c8b2770e..35c366c7 100644 --- a/apps/workers/openaiWorker.ts +++ b/apps/workers/workers/inference/tagging.ts @@ -1,5 +1,5 @@ import { and, Column, eq, inArray, sql } from "drizzle-orm"; -import { DequeuedJob, Runner } from "liteque"; +import { DequeuedJob } from "liteque"; import { buildImpersonatingTRPCClient } from "trpc"; import { z } from "zod"; @@ -14,15 +14,12 @@ import { } from "@karakeep/db/schema"; import { readAsset } from "@karakeep/shared/assetdb"; import serverConfig from "@karakeep/shared/config"; -import { InferenceClientFactory } from "@karakeep/shared/inference"; import logger from "@karakeep/shared/logger"; import { buildImagePrompt, buildTextPrompt } from "@karakeep/shared/prompts"; import { - OpenAIQueue, triggerRuleEngineOnEvent, triggerSearchReindex, triggerWebhook, - zOpenAIRequestSchema, } from "@karakeep/shared/queues"; const openAIResponseSchema = z.object({ @@ -39,60 +36,6 @@ function tagNormalizer(col: Column) { sql: sql`lower(replace(replace(replace(${col}, ' ', ''), '-', ''), '_', ''))`, }; } - -async function attemptMarkTaggingStatus( - jobData: object | undefined, - status: "success" | "failure", -) { - if (!jobData) { - return; - } - try { - const request = zOpenAIRequestSchema.parse(jobData); - await db - .update(bookmarks) - .set({ - taggingStatus: status, - }) - .where(eq(bookmarks.id, request.bookmarkId)); - } catch (e) { - logger.error(`Something went wrong when marking the tagging status: ${e}`); - } -} - -export class OpenAiWorker { - static build() { - logger.info("Starting inference worker ..."); - const worker = new Runner<ZOpenAIRequest>( - OpenAIQueue, - { - run: runOpenAI, - onComplete: async (job) => { - const jobId = job.id; - logger.info(`[inference][${jobId}] Completed successfully`); - await attemptMarkTaggingStatus(job.data, "success"); - }, - onError: async (job) => { - const jobId = job.id; - logger.error( - `[inference][${jobId}] inference job failed: ${job.error}\n${job.error.stack}`, - ); - if (job.numRetriesLeft == 0) { - await attemptMarkTaggingStatus(job?.data, "failure"); - } - }, - }, - { - concurrency: 1, - pollIntervalMs: 1000, - timeoutSecs: serverConfig.inference.jobTimeoutSec, - }, - ); - - return worker; - } -} - async function buildPrompt( bookmark: NonNullable<Awaited<ReturnType<typeof fetchBookmark>>>, ) { @@ -128,17 +71,6 @@ Content: ${content ?? ""}`, throw new Error("Unknown bookmark type"); } -async function fetchBookmark(linkId: string) { - return await db.query.bookmarks.findFirst({ - where: eq(bookmarks.id, linkId), - with: { - link: true, - text: true, - asset: true, - }, - }); -} - async function inferTagsFromImage( jobId: string, bookmark: NonNullable<Awaited<ReturnType<typeof fetchBookmark>>>, @@ -416,25 +348,29 @@ async function connectTags( }); } -async function runOpenAI(job: DequeuedJob<ZOpenAIRequest>) { - const jobId = job.id; +async function fetchBookmark(linkId: string) { + return await db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, linkId), + with: { + link: true, + text: true, + asset: true, + }, + }); +} - const inferenceClient = InferenceClientFactory.build(); - if (!inferenceClient) { - logger.debug( - `[inference][${jobId}] No inference client configured, nothing to do now`, +export async function runTagging( + bookmarkId: string, + job: DequeuedJob<ZOpenAIRequest>, + inferenceClient: InferenceClient, +) { + if (!serverConfig.inference.enableAutoTagging) { + logger.info( + `[inference][${job.id}] Skipping tagging job for bookmark with id "${bookmarkId}" because it's disabled in the config.`, ); return; } - - const request = zOpenAIRequestSchema.safeParse(job.data); - if (!request.success) { - throw new Error( - `[inference][${jobId}] Got malformed job request: ${request.error.toString()}`, - ); - } - - const { bookmarkId } = request.data; + const jobId = job.id; const bookmark = await fetchBookmark(bookmarkId); if (!bookmark) { throw new Error( diff --git a/apps/workers/ruleEngineWorker.ts b/apps/workers/workers/ruleEngineWorker.ts index 427cc383..427cc383 100644 --- a/apps/workers/ruleEngineWorker.ts +++ b/apps/workers/workers/ruleEngineWorker.ts diff --git a/apps/workers/searchWorker.ts b/apps/workers/workers/searchWorker.ts index e7b827a9..e7b827a9 100644 --- a/apps/workers/searchWorker.ts +++ b/apps/workers/workers/searchWorker.ts diff --git a/apps/workers/tidyAssetsWorker.ts b/apps/workers/workers/tidyAssetsWorker.ts index d4c8abdb..d4c8abdb 100644 --- a/apps/workers/tidyAssetsWorker.ts +++ b/apps/workers/workers/tidyAssetsWorker.ts diff --git a/apps/workers/videoWorker.ts b/apps/workers/workers/videoWorker.ts index b8f85ddf..3fa3e49e 100644 --- a/apps/workers/videoWorker.ts +++ b/apps/workers/workers/videoWorker.ts @@ -21,8 +21,8 @@ import { zvideoRequestSchema,
} from "@karakeep/shared/queues";
-import { withTimeout } from "./utils";
-import { getBookmarkDetails, updateAsset } from "./workerUtils";
+import { withTimeout } from "../utils";
+import { getBookmarkDetails, updateAsset } from "../workerUtils";
const TMP_FOLDER = path.join(os.tmpdir(), "video_downloads");
diff --git a/apps/workers/webhookWorker.ts b/apps/workers/workers/webhookWorker.ts index 9d3ed2c1..9d3ed2c1 100644 --- a/apps/workers/webhookWorker.ts +++ b/apps/workers/workers/webhookWorker.ts diff --git a/docs/docs/03-configuration.md b/docs/docs/03-configuration.md index c6fa024e..ac8883d4 100644 --- a/docs/docs/03-configuration.md +++ b/docs/docs/03-configuration.md @@ -26,17 +26,17 @@ Only OIDC compliant OAuth providers are supported! For information on how to set When setting up OAuth, the allowed redirect URLs configured at the provider should be set to `<KARAKEEP_ADDRESS>/api/auth/callback/custom` where `<KARAKEEP_ADDRESS>` is the address you configured in `NEXTAUTH_URL` (for example: `https://try.karakeep.app/api/auth/callback/custom`). ::: -| Name | Required | Default | Description | -| ------------------------------------------- | -------- | ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| DISABLE_SIGNUPS | No | false | If enabled, no new signups will be allowed and the signup button will be disabled in the UI | -| DISABLE_PASSWORD_AUTH | No | false | If enabled, only signups and logins using OAuth are allowed and the signup button and login form for local accounts will be disabled in the UI | -| OAUTH_WELLKNOWN_URL | No | Not set | The "wellknown Url" for openid-configuration as provided by the OAuth provider | -| OAUTH_CLIENT_SECRET | No | Not set | The "Client Secret" as provided by the OAuth provider | -| OAUTH_CLIENT_ID | No | Not set | The "Client ID" as provided by the OAuth provider | -| OAUTH_SCOPE | No | "openid email profile" | "Full list of scopes to request (space delimited)" | -| OAUTH_PROVIDER_NAME | No | "Custom Provider" | The name of your provider. Will be shown on the signup page as "Sign in with `<name>`" | +| Name | Required | Default | Description | +| ------------------------------------------- | -------- | ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| DISABLE_SIGNUPS | No | false | If enabled, no new signups will be allowed and the signup button will be disabled in the UI | +| DISABLE_PASSWORD_AUTH | No | false | If enabled, only signups and logins using OAuth are allowed and the signup button and login form for local accounts will be disabled in the UI | +| OAUTH_WELLKNOWN_URL | No | Not set | The "wellknown Url" for openid-configuration as provided by the OAuth provider | +| OAUTH_CLIENT_SECRET | No | Not set | The "Client Secret" as provided by the OAuth provider | +| OAUTH_CLIENT_ID | No | Not set | The "Client ID" as provided by the OAuth provider | +| OAUTH_SCOPE | No | "openid email profile" | "Full list of scopes to request (space delimited)" | +| OAUTH_PROVIDER_NAME | No | "Custom Provider" | The name of your provider. Will be shown on the signup page as "Sign in with `<name>`" | | OAUTH_ALLOW_DANGEROUS_EMAIL_ACCOUNT_LINKING | No | false | Whether existing accounts in karakeep stored in the database should automatically be linked with your OAuth account. Only enable it if you trust the OAuth provider! | -| OAUTH_TIMEOUT | No | 3500 | The wait time in milliseconds for the OAuth provider response. Increase this if you are having `outgoing request timed out` errors | +| OAUTH_TIMEOUT | No | 3500 | The wait time in milliseconds for the OAuth provider response. Increase this if you are having `outgoing request timed out` errors | For more information on `OAUTH_ALLOW_DANGEROUS_EMAIL_ACCOUNT_LINKING`, check the [next-auth.js documentation](https://next-auth.js.org/configuration/providers/oauth#allowdangerousemailaccountlinking-option). @@ -61,6 +61,8 @@ Either `OPENAI_API_KEY` or `OLLAMA_BASE_URL` need to be set for automatic taggin | EMBEDDING_TEXT_MODEL | No | text-embedding-3-small | The model to be used for generating embeddings for the text. | | INFERENCE_CONTEXT_LENGTH | No | 2048 | The max number of tokens that we'll pass to the inference model. If your content is larger than this size, it'll be truncated to fit. The larger this value, the more of the content will be used in tag inference, but the more expensive the inference will be (money-wise on openAI and resource-wise on ollama). Check the model you're using for its max supported content size. | | INFERENCE_LANG | No | english | The language in which the tags will be generated. | +| INFERENCE_ENABLE_AUTO_TAGGING | No | true | Whether automatic AI tagging is enabled or disabled. | +| INFERENCE_ENABLE_AUTO_SUMMARIZATION | No | false | Whether automatic AI summarization is enabled or disabled. | | INFERENCE_JOB_TIMEOUT_SEC | No | 30 | How long to wait for the inference job to finish before timing out. If you're running ollama without powerful GPUs, you might want to increase the timeout a bit. | | INFERENCE_FETCH_TIMEOUT_SEC | No | 300 | \[Ollama Only\] The timeout of the fetch request to the ollama server. If your inference requests take longer than the default 5mins, you might want to increase this timeout. | | INFERENCE_SUPPORTS_STRUCTURED_OUTPUT | No | Not set | \[DEPRECATED\] Whether the inference model supports structured output or not. Use INFERENCE_OUTPUT_SCHEMA instead. Setting this to true translates to INFERENCE_OUTPUT_SCHEMA=structured, and to false translates to INFERENCE_OUTPUT_SCHEMA=plain. | diff --git a/packages/db/drizzle/0047_add_summarization_status.sql b/packages/db/drizzle/0047_add_summarization_status.sql new file mode 100644 index 00000000..6e8b40d0 --- /dev/null +++ b/packages/db/drizzle/0047_add_summarization_status.sql @@ -0,0 +1 @@ +ALTER TABLE `bookmarks` ADD `summarizationStatus` text DEFAULT 'pending';
\ No newline at end of file diff --git a/packages/db/drizzle/meta/0047_snapshot.json b/packages/db/drizzle/meta/0047_snapshot.json new file mode 100644 index 00000000..f6908043 --- /dev/null +++ b/packages/db/drizzle/meta/0047_snapshot.json @@ -0,0 +1,1967 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "cd8ff1a8-c7bb-4576-9bec-2644785c894e", + "prevId": "7a950001-03e7-43f2-8ef5-272dfe82d223", + "tables": { + "account": { + "name": "account", + "columns": { + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_type": { + "name": "token_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "session_state": { + "name": "session_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "account_userId_user_id_fk": { + "name": "account_userId_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "account_provider_providerAccountId_pk": { + "columns": [ + "provider", + "providerAccountId" + ], + "name": "account_provider_providerAccountId_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apiKey": { + "name": "apiKey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyId": { + "name": "keyId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "apiKey_keyId_unique": { + "name": "apiKey_keyId_unique", + "columns": [ + "keyId" + ], + "isUnique": true + }, + "apiKey_name_userId_unique": { + "name": "apiKey_name_userId_unique", + "columns": [ + "name", + "userId" + ], + "isUnique": true + } + }, + "foreignKeys": { + "apiKey_userId_user_id_fk": { + "name": "apiKey_userId_user_id_fk", + "tableFrom": "apiKey", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "assets": { + "name": "assets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "assetType": { + "name": "assetType", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "contentType": { + "name": "contentType", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fileName": { + "name": "fileName", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "assets_bookmarkId_idx": { + "name": "assets_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + }, + "assets_assetType_idx": { + "name": "assets_assetType_idx", + "columns": [ + "assetType" + ], + "isUnique": false + }, + "assets_userId_idx": { + "name": "assets_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "assets_bookmarkId_bookmarks_id_fk": { + "name": "assets_bookmarkId_bookmarks_id_fk", + "tableFrom": "assets", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "assets_userId_user_id_fk": { + "name": "assets_userId_user_id_fk", + "tableFrom": "assets", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarkAssets": { + "name": "bookmarkAssets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "assetType": { + "name": "assetType", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "assetId": { + "name": "assetId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fileName": { + "name": "fileName", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sourceUrl": { + "name": "sourceUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "bookmarkAssets_id_bookmarks_id_fk": { + "name": "bookmarkAssets_id_bookmarks_id_fk", + "tableFrom": "bookmarkAssets", + "tableTo": "bookmarks", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarkLinks": { + "name": "bookmarkLinks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "publisher": { + "name": "publisher", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "datePublished": { + "name": "datePublished", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dateModified": { + "name": "dateModified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "imageUrl": { + "name": "imageUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "favicon": { + "name": "favicon", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "htmlContent": { + "name": "htmlContent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "crawledAt": { + "name": "crawledAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "crawlStatus": { + "name": "crawlStatus", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "crawlStatusCode": { + "name": "crawlStatusCode", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 200 + } + }, + "indexes": { + "bookmarkLinks_url_idx": { + "name": "bookmarkLinks_url_idx", + "columns": [ + "url" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarkLinks_id_bookmarks_id_fk": { + "name": "bookmarkLinks_id_bookmarks_id_fk", + "tableFrom": "bookmarkLinks", + "tableTo": "bookmarks", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarkLists": { + "name": "bookmarkLists", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "query": { + "name": "query", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "parentId": { + "name": "parentId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bookmarkLists_userId_idx": { + "name": "bookmarkLists_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "bookmarkLists_userId_id_idx": { + "name": "bookmarkLists_userId_id_idx", + "columns": [ + "userId", + "id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "bookmarkLists_userId_user_id_fk": { + "name": "bookmarkLists_userId_user_id_fk", + "tableFrom": "bookmarkLists", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "bookmarkLists_parentId_bookmarkLists_id_fk": { + "name": "bookmarkLists_parentId_bookmarkLists_id_fk", + "tableFrom": "bookmarkLists", + "tableTo": "bookmarkLists", + "columnsFrom": [ + "parentId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarkTags": { + "name": "bookmarkTags", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "bookmarkTags_name_idx": { + "name": "bookmarkTags_name_idx", + "columns": [ + "name" + ], + "isUnique": false + }, + "bookmarkTags_userId_idx": { + "name": "bookmarkTags_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "bookmarkTags_userId_name_unique": { + "name": "bookmarkTags_userId_name_unique", + "columns": [ + "userId", + "name" + ], + "isUnique": true + }, + "bookmarkTags_userId_id_idx": { + "name": "bookmarkTags_userId_id_idx", + "columns": [ + "userId", + "id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "bookmarkTags_userId_user_id_fk": { + "name": "bookmarkTags_userId_user_id_fk", + "tableFrom": "bookmarkTags", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarkTexts": { + "name": "bookmarkTexts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sourceUrl": { + "name": "sourceUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "bookmarkTexts_id_bookmarks_id_fk": { + "name": "bookmarkTexts_id_bookmarks_id_fk", + "tableFrom": "bookmarkTexts", + "tableTo": "bookmarks", + "columnsFrom": [ + "id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarks": { + "name": "bookmarks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "modifiedAt": { + "name": "modifiedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "archived": { + "name": "archived", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "favourited": { + "name": "favourited", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "taggingStatus": { + "name": "taggingStatus", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "summarizationStatus": { + "name": "summarizationStatus", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "bookmarks_userId_idx": { + "name": "bookmarks_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "bookmarks_archived_idx": { + "name": "bookmarks_archived_idx", + "columns": [ + "archived" + ], + "isUnique": false + }, + "bookmarks_favourited_idx": { + "name": "bookmarks_favourited_idx", + "columns": [ + "favourited" + ], + "isUnique": false + }, + "bookmarks_createdAt_idx": { + "name": "bookmarks_createdAt_idx", + "columns": [ + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarks_userId_user_id_fk": { + "name": "bookmarks_userId_user_id_fk", + "tableFrom": "bookmarks", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "bookmarksInLists": { + "name": "bookmarksInLists", + "columns": { + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "listId": { + "name": "listId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedAt": { + "name": "addedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "bookmarksInLists_bookmarkId_idx": { + "name": "bookmarksInLists_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + }, + "bookmarksInLists_listId_idx": { + "name": "bookmarksInLists_listId_idx", + "columns": [ + "listId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "bookmarksInLists_bookmarkId_bookmarks_id_fk": { + "name": "bookmarksInLists_bookmarkId_bookmarks_id_fk", + "tableFrom": "bookmarksInLists", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "bookmarksInLists_listId_bookmarkLists_id_fk": { + "name": "bookmarksInLists_listId_bookmarkLists_id_fk", + "tableFrom": "bookmarksInLists", + "tableTo": "bookmarkLists", + "columnsFrom": [ + "listId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "bookmarksInLists_bookmarkId_listId_pk": { + "columns": [ + "bookmarkId", + "listId" + ], + "name": "bookmarksInLists_bookmarkId_listId_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "config": { + "name": "config", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "customPrompts": { + "name": "customPrompts", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appliesTo": { + "name": "appliesTo", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "customPrompts_userId_idx": { + "name": "customPrompts_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "customPrompts_userId_user_id_fk": { + "name": "customPrompts_userId_user_id_fk", + "tableFrom": "customPrompts", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "highlights": { + "name": "highlights", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "startOffset": { + "name": "startOffset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endOffset": { + "name": "endOffset", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'yellow'" + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "highlights_bookmarkId_idx": { + "name": "highlights_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + }, + "highlights_userId_idx": { + "name": "highlights_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "highlights_bookmarkId_bookmarks_id_fk": { + "name": "highlights_bookmarkId_bookmarks_id_fk", + "tableFrom": "highlights", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "highlights_userId_user_id_fk": { + "name": "highlights_userId_user_id_fk", + "tableFrom": "highlights", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "rssFeedImports": { + "name": "rssFeedImports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entryId": { + "name": "entryId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rssFeedId": { + "name": "rssFeedId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "rssFeedImports_feedIdIdx_idx": { + "name": "rssFeedImports_feedIdIdx_idx", + "columns": [ + "rssFeedId" + ], + "isUnique": false + }, + "rssFeedImports_entryIdIdx_idx": { + "name": "rssFeedImports_entryIdIdx_idx", + "columns": [ + "entryId" + ], + "isUnique": false + }, + "rssFeedImports_rssFeedId_entryId_unique": { + "name": "rssFeedImports_rssFeedId_entryId_unique", + "columns": [ + "rssFeedId", + "entryId" + ], + "isUnique": true + } + }, + "foreignKeys": { + "rssFeedImports_rssFeedId_rssFeeds_id_fk": { + "name": "rssFeedImports_rssFeedId_rssFeeds_id_fk", + "tableFrom": "rssFeedImports", + "tableTo": "rssFeeds", + "columnsFrom": [ + "rssFeedId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "rssFeedImports_bookmarkId_bookmarks_id_fk": { + "name": "rssFeedImports_bookmarkId_bookmarks_id_fk", + "tableFrom": "rssFeedImports", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "rssFeeds": { + "name": "rssFeeds", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lastFetchedAt": { + "name": "lastFetchedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastFetchedStatus": { + "name": "lastFetchedStatus", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pending'" + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "rssFeeds_userId_idx": { + "name": "rssFeeds_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "rssFeeds_userId_user_id_fk": { + "name": "rssFeeds_userId_user_id_fk", + "tableFrom": "rssFeeds", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ruleEngineActions": { + "name": "ruleEngineActions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ruleId": { + "name": "ruleId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "listId": { + "name": "listId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tagId": { + "name": "tagId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "ruleEngineActions_userId_idx": { + "name": "ruleEngineActions_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "ruleEngineActions_ruleId_idx": { + "name": "ruleEngineActions_ruleId_idx", + "columns": [ + "ruleId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "ruleEngineActions_userId_user_id_fk": { + "name": "ruleEngineActions_userId_user_id_fk", + "tableFrom": "ruleEngineActions", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "ruleEngineActions_ruleId_ruleEngineRules_id_fk": { + "name": "ruleEngineActions_ruleId_ruleEngineRules_id_fk", + "tableFrom": "ruleEngineActions", + "tableTo": "ruleEngineRules", + "columnsFrom": [ + "ruleId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "ruleEngineActions_userId_tagId_fk": { + "name": "ruleEngineActions_userId_tagId_fk", + "tableFrom": "ruleEngineActions", + "tableTo": "bookmarkTags", + "columnsFrom": [ + "userId", + "tagId" + ], + "columnsTo": [ + "userId", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "ruleEngineActions_userId_listId_fk": { + "name": "ruleEngineActions_userId_listId_fk", + "tableFrom": "ruleEngineActions", + "tableTo": "bookmarkLists", + "columnsFrom": [ + "userId", + "listId" + ], + "columnsTo": [ + "userId", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ruleEngineRules": { + "name": "ruleEngineRules", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "condition": { + "name": "condition", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "listId": { + "name": "listId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tagId": { + "name": "tagId", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "ruleEngine_userId_idx": { + "name": "ruleEngine_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "ruleEngineRules_userId_user_id_fk": { + "name": "ruleEngineRules_userId_user_id_fk", + "tableFrom": "ruleEngineRules", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "ruleEngineRules_userId_tagId_fk": { + "name": "ruleEngineRules_userId_tagId_fk", + "tableFrom": "ruleEngineRules", + "tableTo": "bookmarkTags", + "columnsFrom": [ + "userId", + "tagId" + ], + "columnsTo": [ + "userId", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "ruleEngineRules_userId_listId_fk": { + "name": "ruleEngineRules_userId_listId_fk", + "tableFrom": "ruleEngineRules", + "tableTo": "bookmarkLists", + "columnsFrom": [ + "userId", + "listId" + ], + "columnsTo": [ + "userId", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "sessionToken": { + "name": "sessionToken", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "session_userId_user_id_fk": { + "name": "session_userId_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tagsOnBookmarks": { + "name": "tagsOnBookmarks", + "columns": { + "bookmarkId": { + "name": "bookmarkId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tagId": { + "name": "tagId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attachedAt": { + "name": "attachedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attachedBy": { + "name": "attachedBy", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "tagsOnBookmarks_tagId_idx": { + "name": "tagsOnBookmarks_tagId_idx", + "columns": [ + "tagId" + ], + "isUnique": false + }, + "tagsOnBookmarks_bookmarkId_idx": { + "name": "tagsOnBookmarks_bookmarkId_idx", + "columns": [ + "bookmarkId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "tagsOnBookmarks_bookmarkId_bookmarks_id_fk": { + "name": "tagsOnBookmarks_bookmarkId_bookmarks_id_fk", + "tableFrom": "tagsOnBookmarks", + "tableTo": "bookmarks", + "columnsFrom": [ + "bookmarkId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "tagsOnBookmarks_tagId_bookmarkTags_id_fk": { + "name": "tagsOnBookmarks_tagId_bookmarkTags_id_fk", + "tableFrom": "tagsOnBookmarks", + "tableTo": "bookmarkTags", + "columnsFrom": [ + "tagId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "tagsOnBookmarks_bookmarkId_tagId_pk": { + "columns": [ + "bookmarkId", + "tagId" + ], + "name": "tagsOnBookmarks_bookmarkId_tagId_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "salt": { + "name": "salt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'user'" + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verificationToken": { + "name": "verificationToken", + "columns": { + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verificationToken_identifier_token_pk": { + "columns": [ + "identifier", + "token" + ], + "name": "verificationToken_identifier_token_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhooks": { + "name": "webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "events": { + "name": "events", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhooks_userId_idx": { + "name": "webhooks_userId_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "webhooks_userId_user_id_fk": { + "name": "webhooks_userId_user_id_fk", + "tableFrom": "webhooks", + "tableTo": "user", + "columnsFrom": [ + "userId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +}
\ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index 9e89f54c..0d7870de 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -330,6 +330,13 @@ "when": 1746902541511, "tag": "0046_add_rss_feed_enabled_col", "breakpoints": true + }, + { + "idx": 47, + "version": "6", + "when": 1747598543992, + "tag": "0047_add_summarization_status", + "breakpoints": true } ] }
\ No newline at end of file diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 6cfca10c..b5938989 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -123,6 +123,9 @@ export const bookmarks = sqliteTable( taggingStatus: text("taggingStatus", { enum: ["pending", "failure", "success"], }).default("pending"), + summarizationStatus: text("summarizationStatus", { + enum: ["pending", "failure", "success"], + }).default("pending"), summary: text("summary"), note: text("note"), type: text("type", { diff --git a/packages/shared-react/utils/bookmarkUtils.ts b/packages/shared-react/utils/bookmarkUtils.ts index 1f840f78..08a6a5e9 100644 --- a/packages/shared-react/utils/bookmarkUtils.ts +++ b/packages/shared-react/utils/bookmarkUtils.ts @@ -35,8 +35,19 @@ export function isBookmarkStillTagging(bookmark: ZBookmark) { ); } +export function isBookmarkStillSummarizing(bookmark: ZBookmark) { + return ( + bookmark.summarizationStatus == "pending" && + Date.now().valueOf() - bookmark.createdAt.valueOf() < MAX_LOADING_MSEC + ); +} + export function isBookmarkStillLoading(bookmark: ZBookmark) { - return isBookmarkStillTagging(bookmark) || isBookmarkStillCrawling(bookmark); + return ( + isBookmarkStillTagging(bookmark) || + isBookmarkStillCrawling(bookmark) || + isBookmarkStillSummarizing(bookmark) + ); } export function getSourceUrl(bookmark: ZBookmark) { diff --git a/packages/shared/config.ts b/packages/shared/config.ts index 046583c6..a4548348 100644 --- a/packages/shared/config.ts +++ b/packages/shared/config.ts @@ -40,6 +40,8 @@ const allEnv = z.object({ INFERENCE_OUTPUT_SCHEMA: z .enum(["structured", "json", "plain"]) .default("structured"), + INFERENCE_ENABLE_AUTO_TAGGING: stringBool("true"), + INFERENCE_ENABLE_AUTO_SUMMARIZATION: stringBool("false"), OCR_CACHE_DIR: z.string().optional(), OCR_LANGS: z .string() @@ -120,6 +122,8 @@ const serverConfigSchema = allEnv.transform((val) => { ? ("structured" as const) : ("plain" as const) : val.INFERENCE_OUTPUT_SCHEMA, + enableAutoTagging: val.INFERENCE_ENABLE_AUTO_TAGGING, + enableAutoSummarization: val.INFERENCE_ENABLE_AUTO_SUMMARIZATION, }, embedding: { textModel: val.EMBEDDING_TEXT_MODEL, diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts index 571df568..bbf69428 100644 --- a/packages/shared/queues.ts +++ b/packages/shared/queues.ts @@ -32,9 +32,10 @@ export const LinkCrawlerQueue = new SqliteQueue<ZCrawlLinkRequest>( }, ); -// OpenAI Worker +// Inference Worker export const zOpenAIRequestSchema = z.object({ bookmarkId: z.string(), + type: z.enum(["summarize", "tag"]).default("tag"), }); export type ZOpenAIRequest = z.infer<typeof zOpenAIRequestSchema>; @@ -195,7 +196,7 @@ export async function triggerWebhook( }); } -// RuleEgine worker +// RuleEngine worker export const zRuleEngineRequestSchema = z.object({ bookmarkId: z.string(), events: z.array(zRuleEngineEventSchema), diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts index 709fd431..5fe77278 100644 --- a/packages/shared/types/bookmarks.ts +++ b/packages/shared/types/bookmarks.ts @@ -87,6 +87,7 @@ export const zBareBookmarkSchema = z.object({ archived: z.boolean(), favourited: z.boolean(), taggingStatus: z.enum(["success", "failure", "pending"]).nullable(), + summarizationStatus: z.enum(["success", "failure", "pending"]).nullable(), note: z.string().nullish(), summary: z.string().nullish(), }); diff --git a/packages/trpc/routers/admin.ts b/packages/trpc/routers/admin.ts index e4985b5c..91f4a34f 100644 --- a/packages/trpc/routers/admin.ts +++ b/packages/trpc/routers/admin.ts @@ -1,5 +1,5 @@ import { TRPCError } from "@trpc/server"; -import { count, eq, sum } from "drizzle-orm"; +import { count, eq, or, sum } from "drizzle-orm"; import { z } from "zod"; import { assets, bookmarkLinks, bookmarks, users } from "@karakeep/db/schema"; @@ -129,11 +129,21 @@ export const adminAppRouter = router({ ctx.db .select({ value: count() }) .from(bookmarks) - .where(eq(bookmarks.taggingStatus, "pending")), + .where( + or( + eq(bookmarks.taggingStatus, "pending"), + eq(bookmarks.summarizationStatus, "pending"), + ), + ), ctx.db .select({ value: count() }) .from(bookmarks) - .where(eq(bookmarks.taggingStatus, "failure")), + .where( + or( + eq(bookmarks.taggingStatus, "failure"), + eq(bookmarks.summarizationStatus, "failure"), + ), + ), // Tidy Assets TidyAssetsQueue.stats(), @@ -233,7 +243,8 @@ export const adminAppRouter = router({ reRunInferenceOnAllBookmarks: adminProcedure .input( z.object({ - taggingStatus: z.enum(["success", "failure", "all"]), + type: z.enum(["tag", "summarize"]), + status: z.enum(["success", "failure", "all"]), }), ) .mutation(async ({ input, ctx }) => { @@ -241,13 +252,22 @@ export const adminAppRouter = router({ columns: { id: true, }, - ...(input.taggingStatus === "all" - ? {} - : { where: eq(bookmarks.taggingStatus, input.taggingStatus) }), + ...{ + tag: + input.status === "all" + ? {} + : { where: eq(bookmarks.taggingStatus, input.status) }, + summarize: + input.status === "all" + ? {} + : { where: eq(bookmarks.summarizationStatus, input.status) }, + }[input.type], }); await Promise.all( - bookmarkIds.map((b) => OpenAIQueue.enqueue({ bookmarkId: b.id })), + bookmarkIds.map((b) => + OpenAIQueue.enqueue({ bookmarkId: b.id, type: input.type }), + ), ); }), tidyAssets: adminProcedure.mutation(async () => { diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts index 88386657..de5bd4c2 100644 --- a/packages/trpc/routers/bookmarks.ts +++ b/packages/trpc/routers/bookmarks.ts @@ -420,6 +420,7 @@ export const bookmarksAppRouter = router({ case BookmarkTypes.TEXT: { await OpenAIQueue.enqueue({ bookmarkId: bookmark.id, + type: "tag", }); break; } |