diff options
| author | MohamedBassem <me@mbassem.com> | 2024-04-11 15:03:31 +0300 |
|---|---|---|
| committer | MohamedBassem <me@mbassem.com> | 2024-04-11 15:03:31 +0300 |
| commit | 238c2967b269ca0f66d8e759c6a0234107e1fd1e (patch) | |
| tree | 1e8590a829bd6b24950ac56eb0a21450c8ce3332 /packages/shared | |
| parent | be622e5594ecb21c82bb6066a82c86e0917bcc35 (diff) | |
| download | karakeep-238c2967b269ca0f66d8e759c6a0234107e1fd1e.tar.zst | |
fix: Increase default navigation timeout to 30s, make it configurable and add retries to crawling jobs
Diffstat (limited to 'packages/shared')
| -rw-r--r-- | packages/shared/config.ts | 2 | ||||
| -rw-r--r-- | packages/shared/queues.ts | 11 |
2 files changed, 12 insertions, 1 deletions
diff --git a/packages/shared/config.ts b/packages/shared/config.ts index 4e444908..41173433 100644 --- a/packages/shared/config.ts +++ b/packages/shared/config.ts @@ -21,6 +21,7 @@ const allEnv = z.object({ CRAWLER_HEADLESS_BROWSER: stringBool("true"), BROWSER_WEB_URL: z.string().url().optional(), CRAWLER_JOB_TIMEOUT_SEC: z.number().default(60), + CRAWLER_NAVIGATE_TIMEOUT_SEC: z.number().default(30), MEILI_ADDR: z.string().optional(), MEILI_MASTER_KEY: z.string().default(""), LOG_LEVEL: z.string().default("debug"), @@ -58,6 +59,7 @@ const serverConfigSchema = allEnv.transform((val) => { headlessBrowser: val.CRAWLER_HEADLESS_BROWSER, browserWebUrl: val.BROWSER_WEB_URL, jobTimeoutSec: val.CRAWLER_JOB_TIMEOUT_SEC, + navigateTimeoutSec: val.CRAWLER_NAVIGATE_TIMEOUT_SEC, }, meilisearch: val.MEILI_ADDR ? { diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts index 146c19c6..6d5fdd5f 100644 --- a/packages/shared/queues.ts +++ b/packages/shared/queues.ts @@ -17,7 +17,16 @@ export type ZCrawlLinkRequest = z.infer<typeof zCrawlLinkRequestSchema>; export const LinkCrawlerQueue = new Queue<ZCrawlLinkRequest, void>( "link_crawler_queue", - { connection: queueConnectionDetails }, + { + connection: queueConnectionDetails, + defaultJobOptions: { + attempts: 5, + backoff: { + type: "exponential", + delay: 1000, + }, + }, + }, ); // OpenAI Worker |