From a91aff23eaa3616aec331ef0267863ed0b0e594b Mon Sep 17 00:00:00 2001
From: MohamedBassem <me@mbassem.com>
Date: Fri, 26 Apr 2024 11:06:54 +0100
Subject: feature(crawler): Allow increasing crawler concurrency and configure
 storing images and screenshots

---
 packages/shared/config.ts | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'packages/shared/config.ts')

diff --git a/packages/shared/config.ts b/packages/shared/config.ts
index 41173433..a31c48b4 100644
--- a/packages/shared/config.ts
+++ b/packages/shared/config.ts
@@ -22,6 +22,9 @@ const allEnv = z.object({
   BROWSER_WEB_URL: z.string().url().optional(),
   CRAWLER_JOB_TIMEOUT_SEC: z.number().default(60),
   CRAWLER_NAVIGATE_TIMEOUT_SEC: z.number().default(30),
+  CRAWLER_NUM_WORKERS: z.coerce.number().default(1),
+  CRAWLER_DOWNLOAD_BANNER_IMAGE: stringBool("true"),
+  CRAWLER_STORE_SCREENSHOT: stringBool("true"),
   MEILI_ADDR: z.string().optional(),
   MEILI_MASTER_KEY: z.string().default(""),
   LOG_LEVEL: z.string().default("debug"),
@@ -56,10 +59,13 @@ const serverConfigSchema = allEnv.transform((val) => {
       redisDBIdx: val.REDIS_DB_IDX,
     },
     crawler: {
+      numWorkers: val.CRAWLER_NUM_WORKERS,
       headlessBrowser: val.CRAWLER_HEADLESS_BROWSER,
       browserWebUrl: val.BROWSER_WEB_URL,
       jobTimeoutSec: val.CRAWLER_JOB_TIMEOUT_SEC,
       navigateTimeoutSec: val.CRAWLER_NAVIGATE_TIMEOUT_SEC,
+      downloadBannerImage: val.CRAWLER_DOWNLOAD_BANNER_IMAGE,
+      storeScreenshot: val.CRAWLER_STORE_SCREENSHOT,
     },
     meilisearch: val.MEILI_ADDR
       ? {
-- 
cgit v1.2.3-70-g09d2