From e035c2fd1067a06d4774c64ae54548f664490f9d Mon Sep 17 00:00:00 2001 From: MohamedBassem Date: Tue, 6 Feb 2024 17:00:26 +0000 Subject: Init package and start bullmq workers --- crawler/crawler.ts | 6 ++++++ crawler/main.ts | 17 +++++++++++++++++ crawler/package.json | 8 ++++++++ 3 files changed, 31 insertions(+) create mode 100644 crawler/crawler.ts create mode 100644 crawler/main.ts create mode 100644 crawler/package.json (limited to 'crawler') diff --git a/crawler/crawler.ts b/crawler/crawler.ts new file mode 100644 index 00000000..58127331 --- /dev/null +++ b/crawler/crawler.ts @@ -0,0 +1,6 @@ +import logger from "@remember/shared/logger"; +import { Job } from "bullmq"; + +export default async function runCrawler(job: Job) { + logger.info(`[Crawler] Got a new job: ${job.name}`); +} diff --git a/crawler/main.ts b/crawler/main.ts new file mode 100644 index 00000000..7d1c0f11 --- /dev/null +++ b/crawler/main.ts @@ -0,0 +1,17 @@ +import { Worker } from "bullmq"; + +import { + LinkCrawlerQueue, + queueConnectionDetails, +} from "@remember/shared/queues"; +import logger from "@remember/shared/logger"; +import runCrawler from "./crawler"; + +logger.info("Starting crawler worker ..."); + +const crawlerWorker = new Worker(LinkCrawlerQueue.name, runCrawler, { + connection: queueConnectionDetails, + autorun: false, +}); + +await Promise.all([crawlerWorker]); diff --git a/crawler/package.json b/crawler/package.json new file mode 100644 index 00000000..67e38cff --- /dev/null +++ b/crawler/package.json @@ -0,0 +1,8 @@ +{ + "name": "@remember/crawler", + "version": "0.1.0", + "private": true, + "dependencies": { + "@remember/shared": "workspace:*" + } +} -- cgit v1.2.3-70-g09d2