feat: Stop downloading video/audio in playwright

1 files changed, 19 insertions, 0 deletions

diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts
index 65130a01..e6ae201e 100644
--- a/apps/workers/workers/crawlerWorker.ts
+++ b/apps/workers/workers/crawlerWorker.ts
@@ -452,6 +452,25 @@ async function crawlPage(
       await globalBlocker.enableBlockingInPage(page);
     }
 
+    // Block audio/video resources
+    await page.route("**/*", (route) => {
+      const request = route.request();
+      const resourceType = request.resourceType();
+
+      // Block audio/video resources
+      if (
+        resourceType === "media" ||
+        request.headers()["content-type"]?.includes("video/") ||
+        request.headers()["content-type"]?.includes("audio/")
+      ) {
+        route.abort();
+        return;
+      }
+
+      // Continue with other requests
+      route.continue();
+    });
+
     // Navigate to the target URL
     logger.info(`[Crawler][${jobId}] Navigating to "${url}"`);
     const response = await Promise.race([