diff options
| author | MohamedBassem <me@mbassem.com> | 2024-03-05 00:33:59 +0000 |
|---|---|---|
| committer | MohamedBassem <me@mbassem.com> | 2024-03-05 00:33:59 +0000 |
| commit | 591358c5a54919b7b8fc0819679c11ffe1d3472e (patch) | |
| tree | 9095a4c8aba866f2c7d75408a20432920518b22e /packages/workers | |
| parent | fdd60a5764910619f9de6e8671767bafebaf3d66 (diff) | |
| download | karakeep-591358c5a54919b7b8fc0819679c11ffe1d3472e.tar.zst | |
fix: Use puppeteer adblocker to block cookies notices
Diffstat (limited to 'packages/workers')
| -rw-r--r-- | packages/workers/crawler.ts | 6 | ||||
| -rw-r--r-- | packages/workers/package.json | 1 |
2 files changed, 7 insertions, 0 deletions
diff --git a/packages/workers/crawler.ts b/packages/workers/crawler.ts index f1ee07f3..fbbee730 100644 --- a/packages/workers/crawler.ts +++ b/packages/workers/crawler.ts @@ -18,6 +18,7 @@ import { db } from "@hoarder/db"; import { Browser } from "puppeteer"; import puppeteer from "puppeteer-extra"; import StealthPlugin from "puppeteer-extra-plugin-stealth"; +import AdblockerPlugin from "puppeteer-extra-plugin-adblocker"; import metascraper from "metascraper"; @@ -70,6 +71,11 @@ async function launchBrowser() { export class CrawlerWorker { static async build() { puppeteer.use(StealthPlugin()); + puppeteer.use( + AdblockerPlugin({ + blockTrackersAndAnnoyances: true, + }), + ); await launchBrowser(); logger.info("Starting crawler worker ..."); diff --git a/packages/workers/package.json b/packages/workers/package.json index a7b62462..f2fc164c 100644 --- a/packages/workers/package.json +++ b/packages/workers/package.json @@ -26,6 +26,7 @@ "openai": "^4.26.1", "puppeteer": "^22.0.0", "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-adblocker": "^2.13.6", "puppeteer-extra-plugin-stealth": "^2.11.2", "tsx": "^4.7.1", "typescript": "^5", |