fix: Increase default navigation timeout to 30s, make it configurable and add retries to crawling jobs

1 files changed, 10 insertions, 1 deletions

diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts
index 146c19c6..6d5fdd5f 100644
--- a/packages/shared/queues.ts
+++ b/packages/shared/queues.ts
@@ -17,7 +17,16 @@ export type ZCrawlLinkRequest = z.infer<typeof zCrawlLinkRequestSchema>;
 
 export const LinkCrawlerQueue = new Queue<ZCrawlLinkRequest, void>(
   "link_crawler_queue",
-  { connection: queueConnectionDetails },
+  {
+    connection: queueConnectionDetails,
+    defaultJobOptions: {
+      attempts: 5,
+      backoff: {
+        type: "exponential",
+        delay: 1000,
+      },
+    },
+  },
 );
 
 // OpenAI Worker