From 238c2967b269ca0f66d8e759c6a0234107e1fd1e Mon Sep 17 00:00:00 2001
From: MohamedBassem <me@mbassem.com>
Date: Thu, 11 Apr 2024 15:03:31 +0300
Subject: fix: Increase default navigation timeout to 30s, make it configurable
 and add retries to crawling jobs

---
 packages/shared/config.ts |  2 ++
 packages/shared/queues.ts | 11 ++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

(limited to 'packages/shared')

diff --git a/packages/shared/config.ts b/packages/shared/config.ts
index 4e444908..41173433 100644
--- a/packages/shared/config.ts
+++ b/packages/shared/config.ts
@@ -21,6 +21,7 @@ const allEnv = z.object({
   CRAWLER_HEADLESS_BROWSER: stringBool("true"),
   BROWSER_WEB_URL: z.string().url().optional(),
   CRAWLER_JOB_TIMEOUT_SEC: z.number().default(60),
+  CRAWLER_NAVIGATE_TIMEOUT_SEC: z.number().default(30),
   MEILI_ADDR: z.string().optional(),
   MEILI_MASTER_KEY: z.string().default(""),
   LOG_LEVEL: z.string().default("debug"),
@@ -58,6 +59,7 @@ const serverConfigSchema = allEnv.transform((val) => {
       headlessBrowser: val.CRAWLER_HEADLESS_BROWSER,
       browserWebUrl: val.BROWSER_WEB_URL,
       jobTimeoutSec: val.CRAWLER_JOB_TIMEOUT_SEC,
+      navigateTimeoutSec: val.CRAWLER_NAVIGATE_TIMEOUT_SEC,
     },
     meilisearch: val.MEILI_ADDR
       ? {
diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts
index 146c19c6..6d5fdd5f 100644
--- a/packages/shared/queues.ts
+++ b/packages/shared/queues.ts
@@ -17,7 +17,16 @@ export type ZCrawlLinkRequest = z.infer<typeof zCrawlLinkRequestSchema>;
 
 export const LinkCrawlerQueue = new Queue<ZCrawlLinkRequest, void>(
   "link_crawler_queue",
-  { connection: queueConnectionDetails },
+  {
+    connection: queueConnectionDetails,
+    defaultJobOptions: {
+      attempts: 5,
+      backoff: {
+        type: "exponential",
+        delay: 1000,
+      },
+    },
+  },
 );
 
 // OpenAI Worker
-- 
cgit v1.2.3-70-g09d2