1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
|
import logger from "@remember/shared/logger";
import {
OpenAIQueue,
ZCrawlLinkRequest,
zCrawlLinkRequestSchema,
} from "@remember/shared/queues";
import { Job } from "bullmq";
import prisma from "@remember/db";
import { Browser } from "puppeteer";
import puppeteer from "puppeteer-extra";
import StealthPlugin from "puppeteer-extra-plugin-stealth";
import metascraper from "metascraper";
import metascraperDescription from "metascraper-description";
import metascraperImage from "metascraper-image";
import metascraperLogo from "metascraper-logo-favicon";
import metascraperTitle from "metascraper-title";
import metascraperUrl from "metascraper-url";
import metascraperTwitter from "metascraper-twitter";
import metascraperReadability from "metascraper-readability";
const metascraperParser = metascraper([
metascraperReadability(),
metascraperTitle(),
metascraperDescription(),
metascraperTwitter(),
metascraperImage(),
metascraperLogo(),
metascraperUrl(),
]);
let browser: Browser;
(async () => {
puppeteer.use(StealthPlugin());
// TODO: Configure the browser mode via an env variable
browser = await puppeteer.launch({ headless: true });
})();
async function crawlPage(url: string) {
const context = await browser.createBrowserContext();
const page = await context.newPage();
await page.goto(url, {
timeout: 10000, // 10 seconds
waitUntil: "networkidle2",
});
const htmlContent = await page.content();
await context.close();
return htmlContent;
}
export default async function runCrawler(job: Job<ZCrawlLinkRequest, void>) {
const jobId = job.id || "unknown";
const request = zCrawlLinkRequestSchema.safeParse(job.data);
if (!request.success) {
logger.error(
`[Crawler][${jobId}] Got malformed job request: ${request.error.toString()}`,
);
return;
}
const { url, bookmarkId } = request.data;
logger.info(
`[Crawler][${jobId}] Will crawl "${url}" for link with id "${bookmarkId}"`,
);
// TODO(IMPORTANT): Run security validations on the input URL (e.g. deny localhost, etc)
const htmlContent = await crawlPage(url);
const meta = await metascraperParser({
url,
html: htmlContent,
});
await prisma.bookmarkedLink.update({
where: {
id: bookmarkId,
},
data: {
title: meta.title,
description: meta.description,
imageUrl: meta.image,
favicon: meta.logo,
crawledAt: new Date(),
},
});
// Enqueue openai job
OpenAIQueue.add("openai", {
bookmarkId,
});
}
|
