diff options
| -rw-r--r-- | apps/workers/crawlerWorker.ts | 3 | ||||
| -rw-r--r-- | apps/workers/utils.ts | 16 |
2 files changed, 18 insertions, 1 deletions
diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts index 87b785ad..a969ab86 100644 --- a/apps/workers/crawlerWorker.ts +++ b/apps/workers/crawlerWorker.ts @@ -20,6 +20,7 @@ import metascraperUrl from "metascraper-url"; import puppeteer from "puppeteer-extra"; import AdblockerPlugin from "puppeteer-extra-plugin-adblocker"; import StealthPlugin from "puppeteer-extra-plugin-stealth"; +import { withTimeout } from "utils"; import type { ZCrawlLinkRequest } from "@hoarder/shared/queues"; import { db } from "@hoarder/db"; @@ -110,7 +111,7 @@ export class CrawlerWorker { logger.info("Starting crawler worker ..."); const worker = new Worker<ZCrawlLinkRequest, void>( LinkCrawlerQueue.name, - runCrawler, + withTimeout(runCrawler, /* timeoutSec */ 30), { connection: queueConnectionDetails, autorun: false, diff --git a/apps/workers/utils.ts b/apps/workers/utils.ts new file mode 100644 index 00000000..2f56d3f0 --- /dev/null +++ b/apps/workers/utils.ts @@ -0,0 +1,16 @@ +export function withTimeout<T, Ret>( + func: (param: T) => Promise<Ret>, + timeoutSec: number, +) { + return async (param: T): Promise<Ret> => { + return await Promise.race([ + func(param), + new Promise<Ret>((_resolve, reject) => + setTimeout( + () => reject(new Error(`Timed-out after ${timeoutSec} secs`)), + timeoutSec * 1000, + ), + ), + ]); + }; +} |