diff options
Diffstat (limited to 'packages')
| -rw-r--r-- | packages/shared-server/src/queues.ts | 12 | ||||
| -rw-r--r-- | packages/trpc/routers/bookmarks.ts | 8 |
2 files changed, 19 insertions, 1 deletions
diff --git a/packages/shared-server/src/queues.ts b/packages/shared-server/src/queues.ts index fd9dac83..4d4a61d6 100644 --- a/packages/shared-server/src/queues.ts +++ b/packages/shared-server/src/queues.ts @@ -96,6 +96,18 @@ export const LinkCrawlerQueue = createDeferredQueue<ZCrawlLinkRequest>( }, ); +// Separate queue for low priority link crawling (e.g. imports) +// This prevents low priority crawling from impacting the parallelism of the main queue +export const LowPriorityCrawlerQueue = createDeferredQueue<ZCrawlLinkRequest>( + "low_priority_crawler_queue", + { + defaultJobArgs: { + numRetries: 5, + }, + keepFailedJobs: false, + }, +); + // Inference Worker export const zOpenAIRequestSchema = z.object({ bookmarkId: z.string(), diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts index 0bbf4fb7..565558c3 100644 --- a/packages/trpc/routers/bookmarks.ts +++ b/packages/trpc/routers/bookmarks.ts @@ -19,6 +19,7 @@ import { import { AssetPreprocessingQueue, LinkCrawlerQueue, + LowPriorityCrawlerQueue, OpenAIQueue, QueuePriority, QuotaService, @@ -282,7 +283,12 @@ export const bookmarksAppRouter = router({ switch (bookmark.content.type) { case BookmarkTypes.LINK: { // The crawling job triggers openai when it's done - await LinkCrawlerQueue.enqueue( + // Use a separate queue for low priority crawling to avoid impacting main queue parallelism + const crawlerQueue = + input.crawlPriority === "low" + ? LowPriorityCrawlerQueue + : LinkCrawlerQueue; + await crawlerQueue.enqueue( { bookmarkId: bookmark.id, }, |