From 238c2967b269ca0f66d8e759c6a0234107e1fd1e Mon Sep 17 00:00:00 2001
From: MohamedBassem <me@mbassem.com>
Date: Thu, 11 Apr 2024 15:03:31 +0300
Subject: fix: Increase default navigation timeout to 30s, make it configurable
 and add retries to crawling jobs

---
 apps/workers/crawlerWorker.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'apps/workers/crawlerWorker.ts')

diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts
index c9a1189c..eec8cd98 100644
--- a/apps/workers/crawlerWorker.ts
+++ b/apps/workers/crawlerWorker.ts
@@ -170,7 +170,7 @@ async function crawlPage(jobId: string, url: string) {
     const page = await context.newPage();
 
     await page.goto(url, {
-      timeout: 10000, // 10 seconds
+      timeout: serverConfig.crawler.navigateTimeoutSec * 1000,
     });
     logger.info(
       `[Crawler][${jobId}] Successfully navigated to "${url}". Waiting for the page to load ...`,
-- 
cgit v1.2.3-70-g09d2