import { count, eq } from "drizzle-orm"; import { z } from "zod"; import { bookmarkLinks, bookmarks, users } from "@hoarder/db/schema"; import { LinkCrawlerQueue, OpenAIQueue, SearchIndexingQueue, triggerSearchReindex, } from "@hoarder/shared/queues"; import { adminProcedure, router } from "../index"; export const adminAppRouter = router({ stats: adminProcedure .output( z.object({ numUsers: z.number(), numBookmarks: z.number(), crawlStats: z.object({ queuedInRedis: z.number(), pending: z.number(), failed: z.number(), }), inferenceStats: z.object({ queuedInRedis: z.number(), pending: z.number(), failed: z.number(), }), indexingStats: z.object({ queuedInRedis: z.number(), }), }), ) .query(async ({ ctx }) => { const [ [{ value: numUsers }], [{ value: numBookmarks }], // Crawls pendingCrawlsInRedis, [{ value: pendingCrawls }], [{ value: failedCrawls }], // Indexing pendingIndexingInRedis, // Inference pendingInferenceInRedis, [{ value: pendingInference }], [{ value: failedInference }], ] = await Promise.all([ ctx.db.select({ value: count() }).from(users), ctx.db.select({ value: count() }).from(bookmarks), // Crawls LinkCrawlerQueue.getWaitingCount(), ctx.db .select({ value: count() }) .from(bookmarkLinks) .where(eq(bookmarkLinks.crawlStatus, "pending")), ctx.db .select({ value: count() }) .from(bookmarkLinks) .where(eq(bookmarkLinks.crawlStatus, "failure")), // Indexing SearchIndexingQueue.getWaitingCount(), // Inference OpenAIQueue.getWaitingCount(), ctx.db .select({ value: count() }) .from(bookmarks) .where(eq(bookmarks.taggingStatus, "pending")), ctx.db .select({ value: count() }) .from(bookmarks) .where(eq(bookmarks.taggingStatus, "failure")), ]); return { numUsers, numBookmarks, crawlStats: { queuedInRedis: pendingCrawlsInRedis, pending: pendingCrawls, failed: failedCrawls, }, inferenceStats: { queuedInRedis: pendingInferenceInRedis, pending: pendingInference, failed: failedInference, }, indexingStats: { queuedInRedis: pendingIndexingInRedis, }, }; }), recrawlLinks: adminProcedure .input( z.object({ crawlStatus: z.enum(["success", "failure", "all"]), runInference: z.boolean(), }), ) .mutation(async ({ ctx, input }) => { const bookmarkIds = await ctx.db.query.bookmarkLinks.findMany({ columns: { id: true, }, ...(input.crawlStatus === "all" ? {} : { where: eq(bookmarkLinks.crawlStatus, input.crawlStatus) }), }); await Promise.all( bookmarkIds.map((b) => LinkCrawlerQueue.add("crawl", { bookmarkId: b.id, runInference: input.runInference, }), ), ); }), reindexAllBookmarks: adminProcedure.mutation(async ({ ctx }) => { const bookmarkIds = await ctx.db.query.bookmarks.findMany({ columns: { id: true, }, }); await Promise.all(bookmarkIds.map((b) => triggerSearchReindex(b.id))); }), });