fix(workers): Add a timeout to the crawling job to prevent it from getting stuck. Fixes #63

2 files changed, 18 insertions, 1 deletions

diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts
index 87b785ad..a969ab86 100644
--- a/apps/workers/crawlerWorker.ts
+++ b/apps/workers/crawlerWorker.ts
@@ -20,6 +20,7 @@ import metascraperUrl from "metascraper-url";
 import puppeteer from "puppeteer-extra";
 import AdblockerPlugin from "puppeteer-extra-plugin-adblocker";
 import StealthPlugin from "puppeteer-extra-plugin-stealth";
+import { withTimeout } from "utils";
 
 import type { ZCrawlLinkRequest } from "@hoarder/shared/queues";
 import { db } from "@hoarder/db";
@@ -110,7 +111,7 @@ export class CrawlerWorker {
     logger.info("Starting crawler worker ...");
     const worker = new Worker<ZCrawlLinkRequest, void>(
       LinkCrawlerQueue.name,
-      runCrawler,
+      withTimeout(runCrawler, /* timeoutSec */ 30),
       {
         connection: queueConnectionDetails,
         autorun: false,
diff --git a/apps/workers/utils.ts b/apps/workers/utils.ts
new file mode 100644
index 00000000..2f56d3f0
--- /dev/null
+++ b/apps/workers/utils.ts
@@ -0,0 +1,16 @@
+export function withTimeout<T, Ret>(
+  func: (param: T) => Promise<Ret>,
+  timeoutSec: number,
+) {
+  return async (param: T): Promise<Ret> => {
+    return await Promise.race([
+      func(param),
+      new Promise<Ret>((_resolve, reject) =>
+        setTimeout(
+          () => reject(new Error(`Timed-out after ${timeoutSec} secs`)),
+          timeoutSec * 1000,
+        ),
+      ),
+    ]);
+  };
+}