fix: Use puppeteer adblocker to block cookies notices

1 files changed, 6 insertions, 0 deletions

diff --git a/packages/workers/crawler.ts b/packages/workers/crawler.ts
index f1ee07f3..fbbee730 100644
--- a/packages/workers/crawler.ts
+++ b/packages/workers/crawler.ts
@@ -18,6 +18,7 @@ import { db } from "@hoarder/db";
 import { Browser } from "puppeteer";
 import puppeteer from "puppeteer-extra";
 import StealthPlugin from "puppeteer-extra-plugin-stealth";
+import AdblockerPlugin from "puppeteer-extra-plugin-adblocker";
 
 import metascraper from "metascraper";
 
@@ -70,6 +71,11 @@ async function launchBrowser() {
 export class CrawlerWorker {
   static async build() {
     puppeteer.use(StealthPlugin());
+    puppeteer.use(
+      AdblockerPlugin({
+        blockTrackersAndAnnoyances: true,
+      }),
+    );
     await launchBrowser();
 
     logger.info("Starting crawler worker ...");