diff options
Diffstat (limited to 'apps/workers')
| -rw-r--r-- | apps/workers/crawlerWorker.ts | 28 |
1 files changed, 18 insertions, 10 deletions
diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts index 5798b98c..e75a8586 100644 --- a/apps/workers/crawlerWorker.ts +++ b/apps/workers/crawlerWorker.ts @@ -322,23 +322,31 @@ async function crawlPage( let screenshot: Buffer | undefined = undefined; if (serverConfig.crawler.storeScreenshot) { - screenshot = await Promise.race<Buffer | undefined>([ - page - .screenshot({ + try { + screenshot = await Promise.race<Buffer>([ + page.screenshot({ // If you change this, you need to change the asset type in the store function. type: "png", encoding: "binary", fullPage: serverConfig.crawler.fullPageScreenshot, - }) - .catch(() => undefined), - new Promise((f) => setTimeout(f, 5000)), - ]); - if (!screenshot) { - logger.warn(`[Crawler][${jobId}] Failed to capture the screenshot.`); - } else { + }), + new Promise((_, reject) => + setTimeout( + () => + reject( + "TIMED_OUT, consider increasing CRAWLER_SCREENSHOT_TIMEOUT_SEC", + ), + serverConfig.crawler.screenshotTimeoutSec * 1000, + ), + ), + ]); logger.info( `[Crawler][${jobId}] Finished capturing page content and a screenshot. FullPageScreenshot: ${serverConfig.crawler.fullPageScreenshot}`, ); + } catch (e) { + logger.warn( + `[Crawler][${jobId}] Failed to capture the screenshot. Reason: ${e}`, + ); } } |