fix(workers): Set a modern user agent and update the default viewport size

1 files changed, 7 insertions, 0 deletions

diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts
index 890127c6..2d1c7564 100644
--- a/apps/workers/crawlerWorker.ts
+++ b/apps/workers/crawlerWorker.ts
@@ -68,6 +68,10 @@ async function launchBrowser() {
         );
         browser = await puppeteer.connect({
           browserURL: webUrl.toString(),
+          defaultViewport: {
+            width: 1440,
+            height: 900,
+          },
         });
       } else {
         logger.info(`Launching a new browser instance`);
@@ -195,6 +199,9 @@ async function crawlPage(jobId: string, url: string) {
 
   try {
     const page = await context.newPage();
+    await page.setUserAgent(
+      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    );
 
     await page.goto(url, {
       timeout: serverConfig.crawler.navigateTimeoutSec * 1000,