1 files changed, 42 insertions, 16 deletions
diff --git a/workers/index.ts b/workers/index.ts
index 76c6f03f..bf092953 100644
--- a/workers/index.ts
+++ b/workers/index.ts
@@ -2,31 +2,57 @@ import { Worker } from "bullmq";
 
 import {
   LinkCrawlerQueue,
+  OpenAIQueue,
   ZCrawlLinkRequest,
+  ZOpenAIRequest,
   queueConnectionDetails,
 } from "@remember/shared/queues";
 import logger from "@remember/shared/logger";
 import runCrawler from "./crawler";
+import runOpenAI from "./openai";
 
-logger.info("Starting crawler worker ...");
+function crawlerWorker() {
+  logger.info("Starting crawler worker ...");
+  const worker = new Worker<ZCrawlLinkRequest, void>(
+    LinkCrawlerQueue.name,
+    runCrawler,
+    {
+      connection: queueConnectionDetails,
+      autorun: false,
+    },
+  );
 
-const crawlerWorker = new Worker<ZCrawlLinkRequest, void>(
-  LinkCrawlerQueue.name,
-  runCrawler,
-  {
+  worker.on("completed", (job) => {
+    const jobId = job?.id || "unknown";
+    logger.info(`[Crawler][${jobId}] Completed successfully`);
+  });
+
+  worker.on("failed", (job, error) => {
+    const jobId = job?.id || "unknown";
+    logger.error(`[Crawler][${jobId}] Crawling job failed: ${error}`);
+  });
+
+  return worker;
+}
+
+function openaiWorker() {
+  logger.info("Starting openai worker ...");
+  const worker = new Worker<ZOpenAIRequest, void>(OpenAIQueue.name, runOpenAI, {
     connection: queueConnectionDetails,
     autorun: false,
-  },
-);
+  });
+
+  worker.on("completed", (job) => {
+    const jobId = job?.id || "unknown";
+    logger.info(`[openai][${jobId}] Completed successfully`);
+  });
 
-crawlerWorker.on("completed", (job) => {
-  const jobId = job?.id || "unknown";
-  logger.info(`[Crawler][${jobId}] Completed successfully`);
-});
+  worker.on("failed", (job, error) => {
+    const jobId = job?.id || "unknown";
+    logger.error(`[openai][${jobId}] openai job failed: ${error}`);
+  });
 
-crawlerWorker.on("failed", (job, error) => {
-  const jobId = job?.id || "unknown";
-  logger.error(`[Crawler][${jobId}] Crawling job failed: ${error}`);
-});
+  return worker;
+}
 
-await Promise.all([crawlerWorker.run()]);
+await Promise.all([crawlerWorker().run(), openaiWorker().run()]);