From 230cafb6dfc8d3bad57d84ef13c3669f5bf5331a Mon Sep 17 00:00:00 2001
From: MohamedBassem <me@mbassem.com>
Date: Sun, 11 Feb 2024 16:53:17 +0000
Subject: fix: Fix build for workers package and add it to CI

---
 packages/workers/index.ts | 67 +++++++----------------------------------------
 1 file changed, 9 insertions(+), 58 deletions(-)

(limited to 'packages/workers/index.ts')

diff --git a/packages/workers/index.ts b/packages/workers/index.ts
index d16c42eb..a58b2edf 100644
--- a/packages/workers/index.ts
+++ b/packages/workers/index.ts
@@ -1,65 +1,16 @@
-import { Worker } from "bullmq";
-
 import dotenv from "dotenv";
-
-import {
-  LinkCrawlerQueue,
-  OpenAIQueue,
-  ZCrawlLinkRequest,
-  ZOpenAIRequest,
-  queueConnectionDetails,
-} from "@remember/shared/queues";
-import logger from "@remember/shared/logger";
-import runCrawler from "./crawler";
-import runOpenAI from "./openai";
-
-function crawlerWorker() {
-  logger.info("Starting crawler worker ...");
-  const worker = new Worker<ZCrawlLinkRequest, void>(
-    LinkCrawlerQueue.name,
-    runCrawler,
-    {
-      connection: queueConnectionDetails,
-      autorun: false,
-    },
-  );
-
-  worker.on("completed", (job) => {
-    const jobId = job?.id || "unknown";
-    logger.info(`[Crawler][${jobId}] Completed successfully`);
-  });
-
-  worker.on("failed", (job, error) => {
-    const jobId = job?.id || "unknown";
-    logger.error(`[Crawler][${jobId}] Crawling job failed: ${error}`);
-  });
-
-  return worker;
-}
-
-function openaiWorker() {
-  logger.info("Starting openai worker ...");
-  const worker = new Worker<ZOpenAIRequest, void>(OpenAIQueue.name, runOpenAI, {
-    connection: queueConnectionDetails,
-    autorun: false,
-  });
-
-  worker.on("completed", (job) => {
-    const jobId = job?.id || "unknown";
-    logger.info(`[openai][${jobId}] Completed successfully`);
-  });
-
-  worker.on("failed", (job, error) => {
-    const jobId = job?.id || "unknown";
-    logger.error(`[openai][${jobId}] openai job failed: ${error}`);
-  });
-
-  return worker;
-}
+import { CrawlerWorker } from "./crawler";
+import { OpenAiWorker } from "./openai";
 
 async function main() {
   dotenv.config();
-  await Promise.all([crawlerWorker().run(), openaiWorker().run()]);
+
+  const [crawler, openai] = [
+    await CrawlerWorker.build(),
+    await OpenAiWorker.build(),
+  ];
+
+  await Promise.all([crawler.run(), openai.run()]);
 }
 
 main();
-- 
cgit v1.2.3-70-g09d2