diff options
Diffstat (limited to 'packages/shared')
| -rw-r--r-- | packages/shared/config.ts | 2 | ||||
| -rw-r--r-- | packages/shared/queues.ts | 11 |
2 files changed, 12 insertions, 1 deletions
diff --git a/packages/shared/config.ts b/packages/shared/config.ts index 4e444908..41173433 100644 --- a/packages/shared/config.ts +++ b/packages/shared/config.ts @@ -21,6 +21,7 @@ const allEnv = z.object({ CRAWLER_HEADLESS_BROWSER: stringBool("true"), BROWSER_WEB_URL: z.string().url().optional(), CRAWLER_JOB_TIMEOUT_SEC: z.number().default(60), + CRAWLER_NAVIGATE_TIMEOUT_SEC: z.number().default(30), MEILI_ADDR: z.string().optional(), MEILI_MASTER_KEY: z.string().default(""), LOG_LEVEL: z.string().default("debug"), @@ -58,6 +59,7 @@ const serverConfigSchema = allEnv.transform((val) => { headlessBrowser: val.CRAWLER_HEADLESS_BROWSER, browserWebUrl: val.BROWSER_WEB_URL, jobTimeoutSec: val.CRAWLER_JOB_TIMEOUT_SEC, + navigateTimeoutSec: val.CRAWLER_NAVIGATE_TIMEOUT_SEC, }, meilisearch: val.MEILI_ADDR ? { diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts index 146c19c6..6d5fdd5f 100644 --- a/packages/shared/queues.ts +++ b/packages/shared/queues.ts @@ -17,7 +17,16 @@ export type ZCrawlLinkRequest = z.infer<typeof zCrawlLinkRequestSchema>; export const LinkCrawlerQueue = new Queue<ZCrawlLinkRequest, void>( "link_crawler_queue", - { connection: queueConnectionDetails }, + { + connection: queueConnectionDetails, + defaultJobOptions: { + attempts: 5, + backoff: { + type: "exponential", + delay: 1000, + }, + }, + }, ); // OpenAI Worker |