From 591358c5a54919b7b8fc0819679c11ffe1d3472e Mon Sep 17 00:00:00 2001
From: MohamedBassem <me@mbassem.com>
Date: Tue, 5 Mar 2024 00:33:59 +0000
Subject: fix: Use puppeteer adblocker to block cookies notices

---
 packages/workers/crawler.ts   | 6 ++++++
 packages/workers/package.json | 1 +
 2 files changed, 7 insertions(+)

(limited to 'packages/workers')

diff --git a/packages/workers/crawler.ts b/packages/workers/crawler.ts
index f1ee07f3..fbbee730 100644
--- a/packages/workers/crawler.ts
+++ b/packages/workers/crawler.ts
@@ -18,6 +18,7 @@ import { db } from "@hoarder/db";
 import { Browser } from "puppeteer";
 import puppeteer from "puppeteer-extra";
 import StealthPlugin from "puppeteer-extra-plugin-stealth";
+import AdblockerPlugin from "puppeteer-extra-plugin-adblocker";
 
 import metascraper from "metascraper";
 
@@ -70,6 +71,11 @@ async function launchBrowser() {
 export class CrawlerWorker {
   static async build() {
     puppeteer.use(StealthPlugin());
+    puppeteer.use(
+      AdblockerPlugin({
+        blockTrackersAndAnnoyances: true,
+      }),
+    );
     await launchBrowser();
 
     logger.info("Starting crawler worker ...");
diff --git a/packages/workers/package.json b/packages/workers/package.json
index a7b62462..f2fc164c 100644
--- a/packages/workers/package.json
+++ b/packages/workers/package.json
@@ -26,6 +26,7 @@
     "openai": "^4.26.1",
     "puppeteer": "^22.0.0",
     "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-adblocker": "^2.13.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "tsx": "^4.7.1",
     "typescript": "^5",
-- 
cgit v1.2.3-70-g09d2