crawler/index.ts


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

import { Worker } from "bullmq";

import {
  LinkCrawlerQueue,
  ZCrawlLinkRequest,
  queueConnectionDetails,
} from "@remember/shared/queues";
import logger from "@remember/shared/logger";
import runCrawler from "./crawler";

logger.info("Starting crawler worker ...");

const crawlerWorker = new Worker<ZCrawlLinkRequest, void>(
  LinkCrawlerQueue.name,
  runCrawler,
  {
    connection: queueConnectionDetails,
    autorun: false,
  },
);

crawlerWorker.on("completed", (job) => {
  const jobId = job?.id || "unknown";
  logger.info(`[Crawler][${jobId}] Completed successfully`);
});

crawlerWorker.on("failed", (job, error) => {
  const jobId = job?.id || "unknown";
  logger.error(`[Crawler][${jobId}] Crawling job failed: ${error}`);
});

await Promise.all([crawlerWorker.run()]);