diff options
Diffstat (limited to 'packages/trpc/routers/admin.ts')
| -rw-r--r-- | packages/trpc/routers/admin.ts | 248 |
1 files changed, 235 insertions, 13 deletions
diff --git a/packages/trpc/routers/admin.ts b/packages/trpc/routers/admin.ts index 463d2ddf..f64e071a 100644 --- a/packages/trpc/routers/admin.ts +++ b/packages/trpc/routers/admin.ts @@ -9,7 +9,9 @@ import { AssetPreprocessingQueue, FeedQueue, LinkCrawlerQueue, + LowPriorityCrawlerQueue, OpenAIQueue, + QueuePriority, SearchIndexingQueue, triggerSearchReindex, VideoWorkerQueue, @@ -17,6 +19,7 @@ import { zAdminMaintenanceTaskSchema, } from "@karakeep/shared-server"; import serverConfig from "@karakeep/shared/config"; +import logger from "@karakeep/shared/logger"; import { PluginManager, PluginType } from "@karakeep/shared/plugins"; import { getSearchClient } from "@karakeep/shared/search"; import { @@ -24,9 +27,11 @@ import { updateUserSchema, zAdminCreateUserSchema, } from "@karakeep/shared/types/admin"; +import { BookmarkTypes } from "@karakeep/shared/types/bookmarks"; import { generatePasswordSalt, hashPassword } from "../auth"; import { adminProcedure, router } from "../index"; +import { Bookmark } from "../models/bookmarks"; import { User } from "../models/users"; export const adminAppRouter = router({ @@ -86,6 +91,7 @@ export const adminAppRouter = router({ const [ // Crawls queuedCrawls, + queuedLowPriorityCrawls, [{ value: pendingCrawls }], [{ value: failedCrawls }], @@ -114,6 +120,7 @@ export const adminAppRouter = router({ ] = await Promise.all([ // Crawls LinkCrawlerQueue.stats(), + LowPriorityCrawlerQueue.stats(), ctx.db .select({ value: count() }) .from(bookmarkLinks) @@ -165,7 +172,11 @@ export const adminAppRouter = router({ return { crawlStats: { - queued: queuedCrawls.pending + queuedCrawls.pending_retry, + queued: + queuedCrawls.pending + + queuedCrawls.pending_retry + + queuedLowPriorityCrawls.pending + + queuedLowPriorityCrawls.pending_retry, pending: pendingCrawls, failed: failedCrawls, }, @@ -201,7 +212,7 @@ export const adminAppRouter = router({ recrawlLinks: adminProcedure .input( z.object({ - crawlStatus: z.enum(["success", "failure", "all"]), + crawlStatus: z.enum(["success", "failure", "pending", "all"]), runInference: z.boolean(), }), ) @@ -217,10 +228,15 @@ export const adminAppRouter = router({ await Promise.all( bookmarkIds.map((b) => - LinkCrawlerQueue.enqueue({ - bookmarkId: b.id, - runInference: input.runInference, - }), + LowPriorityCrawlerQueue.enqueue( + { + bookmarkId: b.id, + runInference: input.runInference, + }, + { + priority: QueuePriority.Low, + }, + ), ), ); }), @@ -233,7 +249,13 @@ export const adminAppRouter = router({ }, }); - await Promise.all(bookmarkIds.map((b) => triggerSearchReindex(b.id))); + await Promise.all( + bookmarkIds.map((b) => + triggerSearchReindex(b.id, { + priority: QueuePriority.Low, + }), + ), + ); }), reprocessAssetsFixMode: adminProcedure.mutation(async ({ ctx }) => { const bookmarkIds = await ctx.db.query.bookmarkAssets.findMany({ @@ -244,10 +266,15 @@ export const adminAppRouter = router({ await Promise.all( bookmarkIds.map((b) => - AssetPreprocessingQueue.enqueue({ - bookmarkId: b.id, - fixMode: true, - }), + AssetPreprocessingQueue.enqueue( + { + bookmarkId: b.id, + fixMode: true, + }, + { + priority: QueuePriority.Low, + }, + ), ), ); }), @@ -255,7 +282,7 @@ export const adminAppRouter = router({ .input( z.object({ type: z.enum(["tag", "summarize"]), - status: z.enum(["success", "failure", "all"]), + status: z.enum(["success", "failure", "pending", "all"]), }), ) .mutation(async ({ input, ctx }) => { @@ -277,7 +304,12 @@ export const adminAppRouter = router({ await Promise.all( bookmarkIds.map((b) => - OpenAIQueue.enqueue({ bookmarkId: b.id, type: input.type }), + OpenAIQueue.enqueue( + { bookmarkId: b.id, type: input.type }, + { + priority: QueuePriority.Low, + }, + ), ), ); }), @@ -537,4 +569,194 @@ export const adminAppRouter = router({ queue: queueStatus, }; }), + getBookmarkDebugInfo: adminProcedure + .input(z.object({ bookmarkId: z.string() })) + .output( + z.object({ + id: z.string(), + type: z.enum([ + BookmarkTypes.LINK, + BookmarkTypes.TEXT, + BookmarkTypes.ASSET, + ]), + source: z + .enum([ + "api", + "web", + "extension", + "cli", + "mobile", + "singlefile", + "rss", + "import", + ]) + .nullable(), + createdAt: z.date(), + modifiedAt: z.date().nullable(), + title: z.string().nullable(), + summary: z.string().nullable(), + taggingStatus: z.enum(["pending", "failure", "success"]).nullable(), + summarizationStatus: z + .enum(["pending", "failure", "success"]) + .nullable(), + userId: z.string(), + linkInfo: z + .object({ + url: z.string(), + crawlStatus: z.enum(["pending", "failure", "success"]), + crawlStatusCode: z.number().nullable(), + crawledAt: z.date().nullable(), + hasHtmlContent: z.boolean(), + hasContentAsset: z.boolean(), + htmlContentPreview: z.string().nullable(), + }) + .nullable(), + textInfo: z + .object({ + hasText: z.boolean(), + sourceUrl: z.string().nullable(), + }) + .nullable(), + assetInfo: z + .object({ + assetType: z.enum(["image", "pdf"]), + hasContent: z.boolean(), + fileName: z.string().nullable(), + }) + .nullable(), + tags: z.array( + z.object({ + id: z.string(), + name: z.string(), + attachedBy: z.enum(["ai", "human"]), + }), + ), + assets: z.array( + z.object({ + id: z.string(), + assetType: z.string(), + size: z.number(), + url: z.string().nullable(), + }), + ), + }), + ) + .query(async ({ input, ctx }) => { + logger.info( + `[admin] Admin ${ctx.user.id} accessed debug info for bookmark ${input.bookmarkId}`, + ); + + return await Bookmark.buildDebugInfo(ctx, input.bookmarkId); + }), + adminRecrawlBookmark: adminProcedure + .input(z.object({ bookmarkId: z.string() })) + .mutation(async ({ input, ctx }) => { + // Verify bookmark exists and is a link + const bookmark = await ctx.db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, input.bookmarkId), + }); + + if (!bookmark) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Bookmark not found", + }); + } + + if (bookmark.type !== BookmarkTypes.LINK) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Only link bookmarks can be recrawled", + }); + } + + await LowPriorityCrawlerQueue.enqueue( + { + bookmarkId: input.bookmarkId, + }, + { + priority: QueuePriority.Low, + groupId: "admin", + }, + ); + }), + adminReindexBookmark: adminProcedure + .input(z.object({ bookmarkId: z.string() })) + .mutation(async ({ input, ctx }) => { + // Verify bookmark exists + const bookmark = await ctx.db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, input.bookmarkId), + }); + + if (!bookmark) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Bookmark not found", + }); + } + + await triggerSearchReindex(input.bookmarkId, { + priority: QueuePriority.Low, + groupId: "admin", + }); + }), + adminRetagBookmark: adminProcedure + .input(z.object({ bookmarkId: z.string() })) + .mutation(async ({ input, ctx }) => { + // Verify bookmark exists + const bookmark = await ctx.db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, input.bookmarkId), + }); + + if (!bookmark) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Bookmark not found", + }); + } + + await OpenAIQueue.enqueue( + { + bookmarkId: input.bookmarkId, + type: "tag", + }, + { + priority: QueuePriority.Low, + groupId: "admin", + }, + ); + }), + adminResummarizeBookmark: adminProcedure + .input(z.object({ bookmarkId: z.string() })) + .mutation(async ({ input, ctx }) => { + // Verify bookmark exists and is a link + const bookmark = await ctx.db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, input.bookmarkId), + }); + + if (!bookmark) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Bookmark not found", + }); + } + + if (bookmark.type !== BookmarkTypes.LINK) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Only link bookmarks can be summarized", + }); + } + + await OpenAIQueue.enqueue( + { + bookmarkId: input.bookmarkId, + type: "summarize", + }, + { + priority: QueuePriority.Low, + groupId: "admin", + }, + ); + }), }); |