feat: Add separate queue for import link crawling (#2452)

* feat: add separate queue for import link crawling

---------

Co-authored-by: Claude <noreply@anthropic.com>

1 files changed, 6 insertions, 1 deletions

diff --git a/apps/workers/index.ts b/apps/workers/index.ts
index 931a505f..c7b9533d 100644
--- a/apps/workers/index.ts
+++ b/apps/workers/index.ts
@@ -10,6 +10,7 @@ import {
   initTracing,
   LinkCrawlerQueue,
   loadAllPlugins,
+  LowPriorityCrawlerQueue,
   OpenAIQueue,
   prepareQueue,
   RuleEngineQueue,
@@ -38,7 +39,11 @@ import { WebhookWorker } from "./workers/webhookWorker";
 const workerBuilders = {
   crawler: async () => {
     await LinkCrawlerQueue.ensureInit();
-    return CrawlerWorker.build();
+    return CrawlerWorker.build(LinkCrawlerQueue);
+  },
+  lowPriorityCrawler: async () => {
+    await LowPriorityCrawlerQueue.ensureInit();
+    return CrawlerWorker.build(LowPriorityCrawlerQueue);
   },
   inference: async () => {
     await OpenAIQueue.ensureInit();