From 238c2967b269ca0f66d8e759c6a0234107e1fd1e Mon Sep 17 00:00:00 2001
From: MohamedBassem <me@mbassem.com>
Date: Thu, 11 Apr 2024 15:03:31 +0300
Subject: fix: Increase default navigation timeout to 30s, make it configurable
 and add retries to crawling jobs

---
 packages/shared/queues.ts | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'packages/shared/queues.ts')

diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts
index 146c19c6..6d5fdd5f 100644
--- a/packages/shared/queues.ts
+++ b/packages/shared/queues.ts
@@ -17,7 +17,16 @@ export type ZCrawlLinkRequest = z.infer<typeof zCrawlLinkRequestSchema>;
 
 export const LinkCrawlerQueue = new Queue<ZCrawlLinkRequest, void>(
   "link_crawler_queue",
-  { connection: queueConnectionDetails },
+  {
+    connection: queueConnectionDetails,
+    defaultJobOptions: {
+      attempts: 5,
+      backoff: {
+        type: "exponential",
+        delay: 1000,
+      },
+    },
+  },
 );
 
 // OpenAI Worker
-- 
cgit v1.2.3-70-g09d2