[feature] Use puppeteer for fetching websites

1 files changed, 32 insertions, 4 deletions

diff --git a/packages/workers/crawler.ts b/packages/workers/crawler.ts
index d5479025..1cb82f31 100644
--- a/packages/workers/crawler.ts
+++ b/packages/workers/crawler.ts
@@ -8,6 +8,10 @@ import { Job } from "bullmq";
 
 import prisma from "@remember/db";
 
+import { Browser } from "puppeteer";
+import puppeteer from "puppeteer-extra";
+import StealthPlugin from "puppeteer-extra-plugin-stealth";
+
 import metascraper from "metascraper";
 
 import metascraperDescription from "metascraper-description";
@@ -15,15 +19,40 @@ import metascraperImage from "metascraper-image";
 import metascraperLogo from "metascraper-logo-favicon";
 import metascraperTitle from "metascraper-title";
 import metascraperUrl from "metascraper-url";
+import metascraperTwitter from "metascraper-twitter";
+import metascraperReadability from "metascraper-readability";
 
 const metascraperParser = metascraper([
+  metascraperReadability(),
+  metascraperTitle(),
   metascraperDescription(),
+  metascraperTwitter(),
   metascraperImage(),
   metascraperLogo(),
-  metascraperTitle(),
   metascraperUrl(),
 ]);
 
+let browser: Browser;
+(async () => {
+  puppeteer.use(StealthPlugin());
+  // TODO: Configure the browser mode via an env variable
+  browser = await puppeteer.launch({ headless: true });
+})();
+
+async function crawlPage(url: string) {
+  const context = await browser.createBrowserContext();
+  const page = await context.newPage();
+
+  await page.goto(url, {
+    timeout: 10000, // 10 seconds
+    waitUntil: "networkidle2",
+  });
+
+  const htmlContent = await page.content();
+  await context.close();
+  return htmlContent;
+}
+
 export default async function runCrawler(job: Job<ZCrawlLinkRequest, void>) {
   const jobId = job.id || "unknown";
 
@@ -42,12 +71,11 @@ export default async function runCrawler(job: Job<ZCrawlLinkRequest, void>) {
   );
   // TODO(IMPORTANT): Run security validations on the input URL (e.g. deny localhost, etc)
 
-  const resp = await fetch(url);
-  const respBody = await resp.text();
+  const htmlContent = await crawlPage(url);
 
   const meta = await metascraperParser({
     url,
-    html: respBody,
+    html: htmlContent,
   });
 
   await prisma.bookmarkedLink.update({