From bbd65fd6123f7d1a93d1f6a68f2b933d53ec3c23 Mon Sep 17 00:00:00 2001
From: Mohamed Bassem <me@mbassem.com>
Date: Sun, 8 Feb 2026 02:11:51 +0000
Subject: feat: Add separate queue for import link crawling (#2452)

* feat: add separate queue for import link crawling

---------

Co-authored-by: Claude <noreply@anthropic.com>
---
 packages/shared-server/src/queues.ts | 12 ++++++++++++
 1 file changed, 12 insertions(+)

(limited to 'packages/shared-server/src/queues.ts')

diff --git a/packages/shared-server/src/queues.ts b/packages/shared-server/src/queues.ts
index fd9dac83..4d4a61d6 100644
--- a/packages/shared-server/src/queues.ts
+++ b/packages/shared-server/src/queues.ts
@@ -96,6 +96,18 @@ export const LinkCrawlerQueue = createDeferredQueue<ZCrawlLinkRequest>(
   },
 );
 
+// Separate queue for low priority link crawling (e.g. imports)
+// This prevents low priority crawling from impacting the parallelism of the main queue
+export const LowPriorityCrawlerQueue = createDeferredQueue<ZCrawlLinkRequest>(
+  "low_priority_crawler_queue",
+  {
+    defaultJobArgs: {
+      numRetries: 5,
+    },
+    keepFailedJobs: false,
+  },
+);
+
 // Inference Worker
 export const zOpenAIRequestSchema = z.object({
   bookmarkId: z.string(),
-- 
cgit v1.2.3-70-g09d2