1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
|
import { Worker } from "bullmq";
import {
LinkCrawlerQueue,
OpenAIQueue,
ZCrawlLinkRequest,
ZOpenAIRequest,
queueConnectionDetails,
} from "@remember/shared/queues";
import logger from "@remember/shared/logger";
import runCrawler from "./crawler";
import runOpenAI from "./openai";
function crawlerWorker() {
logger.info("Starting crawler worker ...");
const worker = new Worker<ZCrawlLinkRequest, void>(
LinkCrawlerQueue.name,
runCrawler,
{
connection: queueConnectionDetails,
autorun: false,
},
);
worker.on("completed", (job) => {
const jobId = job?.id || "unknown";
logger.info(`[Crawler][${jobId}] Completed successfully`);
});
worker.on("failed", (job, error) => {
const jobId = job?.id || "unknown";
logger.error(`[Crawler][${jobId}] Crawling job failed: ${error}`);
});
return worker;
}
function openaiWorker() {
logger.info("Starting openai worker ...");
const worker = new Worker<ZOpenAIRequest, void>(OpenAIQueue.name, runOpenAI, {
connection: queueConnectionDetails,
autorun: false,
});
worker.on("completed", (job) => {
const jobId = job?.id || "unknown";
logger.info(`[openai][${jobId}] Completed successfully`);
});
worker.on("failed", (job, error) => {
const jobId = job?.id || "unknown";
logger.error(`[openai][${jobId}] openai job failed: ${error}`);
});
return worker;
}
await Promise.all([crawlerWorker().run(), openaiWorker().run()]);
|
