From 8970b3a5375ccfd9b41c8a08722a2fc6bbbe3af9 Mon Sep 17 00:00:00 2001
From: MohamedBassem <me@mbassem.com>
Date: Wed, 7 Feb 2024 21:05:57 +0000
Subject: [feature] Add openAI integration for extracting tags from articles

---
 workers/index.ts | 58 ++++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 42 insertions(+), 16 deletions(-)

(limited to 'workers/index.ts')

diff --git a/workers/index.ts b/workers/index.ts
index 76c6f03f..bf092953 100644
--- a/workers/index.ts
+++ b/workers/index.ts
@@ -2,31 +2,57 @@ import { Worker } from "bullmq";
 
 import {
   LinkCrawlerQueue,
+  OpenAIQueue,
   ZCrawlLinkRequest,
+  ZOpenAIRequest,
   queueConnectionDetails,
 } from "@remember/shared/queues";
 import logger from "@remember/shared/logger";
 import runCrawler from "./crawler";
+import runOpenAI from "./openai";
 
-logger.info("Starting crawler worker ...");
+function crawlerWorker() {
+  logger.info("Starting crawler worker ...");
+  const worker = new Worker<ZCrawlLinkRequest, void>(
+    LinkCrawlerQueue.name,
+    runCrawler,
+    {
+      connection: queueConnectionDetails,
+      autorun: false,
+    },
+  );
 
-const crawlerWorker = new Worker<ZCrawlLinkRequest, void>(
-  LinkCrawlerQueue.name,
-  runCrawler,
-  {
+  worker.on("completed", (job) => {
+    const jobId = job?.id || "unknown";
+    logger.info(`[Crawler][${jobId}] Completed successfully`);
+  });
+
+  worker.on("failed", (job, error) => {
+    const jobId = job?.id || "unknown";
+    logger.error(`[Crawler][${jobId}] Crawling job failed: ${error}`);
+  });
+
+  return worker;
+}
+
+function openaiWorker() {
+  logger.info("Starting openai worker ...");
+  const worker = new Worker<ZOpenAIRequest, void>(OpenAIQueue.name, runOpenAI, {
     connection: queueConnectionDetails,
     autorun: false,
-  },
-);
+  });
+
+  worker.on("completed", (job) => {
+    const jobId = job?.id || "unknown";
+    logger.info(`[openai][${jobId}] Completed successfully`);
+  });
 
-crawlerWorker.on("completed", (job) => {
-  const jobId = job?.id || "unknown";
-  logger.info(`[Crawler][${jobId}] Completed successfully`);
-});
+  worker.on("failed", (job, error) => {
+    const jobId = job?.id || "unknown";
+    logger.error(`[openai][${jobId}] openai job failed: ${error}`);
+  });
 
-crawlerWorker.on("failed", (job, error) => {
-  const jobId = job?.id || "unknown";
-  logger.error(`[Crawler][${jobId}] Crawling job failed: ${error}`);
-});
+  return worker;
+}
 
-await Promise.all([crawlerWorker.run()]);
+await Promise.all([crawlerWorker().run(), openaiWorker().run()]);
-- 
cgit v1.2.3-70-g09d2