From bbd65fd6123f7d1a93d1f6a68f2b933d53ec3c23 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sun, 8 Feb 2026 02:11:51 +0000 Subject: feat: Add separate queue for import link crawling (#2452) * feat: add separate queue for import link crawling --------- Co-authored-by: Claude --- packages/trpc/routers/bookmarks.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'packages/trpc') diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts index 0bbf4fb7..565558c3 100644 --- a/packages/trpc/routers/bookmarks.ts +++ b/packages/trpc/routers/bookmarks.ts @@ -19,6 +19,7 @@ import { import { AssetPreprocessingQueue, LinkCrawlerQueue, + LowPriorityCrawlerQueue, OpenAIQueue, QueuePriority, QuotaService, @@ -282,7 +283,12 @@ export const bookmarksAppRouter = router({ switch (bookmark.content.type) { case BookmarkTypes.LINK: { // The crawling job triggers openai when it's done - await LinkCrawlerQueue.enqueue( + // Use a separate queue for low priority crawling to avoid impacting main queue parallelism + const crawlerQueue = + input.crawlPriority === "low" + ? LowPriorityCrawlerQueue + : LinkCrawlerQueue; + await crawlerQueue.enqueue( { bookmarkId: bookmark.id, }, -- cgit v1.2.3-70-g09d2