diff options
| author | MohamedBassem <me@mbassem.com> | 2024-04-26 11:06:54 +0100 |
|---|---|---|
| committer | MohamedBassem <me@mbassem.com> | 2024-04-26 11:06:54 +0100 |
| commit | a91aff23eaa3616aec331ef0267863ed0b0e594b (patch) | |
| tree | 09a449ec43b632762af1ef7125cf707b8fe45d81 /packages/shared | |
| parent | 9dace185acff4002aec8265fc010db49d91c7d7f (diff) | |
| download | karakeep-a91aff23eaa3616aec331ef0267863ed0b0e594b.tar.zst | |
feature(crawler): Allow increasing crawler concurrency and configure storing images and screenshots
Diffstat (limited to '')
| -rw-r--r-- | packages/shared/config.ts | 6 |
1 files changed, 6 insertions, 0 deletions
diff --git a/packages/shared/config.ts b/packages/shared/config.ts index 41173433..a31c48b4 100644 --- a/packages/shared/config.ts +++ b/packages/shared/config.ts @@ -22,6 +22,9 @@ const allEnv = z.object({ BROWSER_WEB_URL: z.string().url().optional(), CRAWLER_JOB_TIMEOUT_SEC: z.number().default(60), CRAWLER_NAVIGATE_TIMEOUT_SEC: z.number().default(30), + CRAWLER_NUM_WORKERS: z.coerce.number().default(1), + CRAWLER_DOWNLOAD_BANNER_IMAGE: stringBool("true"), + CRAWLER_STORE_SCREENSHOT: stringBool("true"), MEILI_ADDR: z.string().optional(), MEILI_MASTER_KEY: z.string().default(""), LOG_LEVEL: z.string().default("debug"), @@ -56,10 +59,13 @@ const serverConfigSchema = allEnv.transform((val) => { redisDBIdx: val.REDIS_DB_IDX, }, crawler: { + numWorkers: val.CRAWLER_NUM_WORKERS, headlessBrowser: val.CRAWLER_HEADLESS_BROWSER, browserWebUrl: val.BROWSER_WEB_URL, jobTimeoutSec: val.CRAWLER_JOB_TIMEOUT_SEC, navigateTimeoutSec: val.CRAWLER_NAVIGATE_TIMEOUT_SEC, + downloadBannerImage: val.CRAWLER_DOWNLOAD_BANNER_IMAGE, + storeScreenshot: val.CRAWLER_STORE_SCREENSHOT, }, meilisearch: val.MEILI_ADDR ? { |