aboutsummaryrefslogtreecommitdiffstats
path: root/packages/workers/crawler.ts
diff options
context:
space:
mode:
authorMohamedBassem <me@mbassem.com>2024-02-09 01:50:35 +0000
committerMohamedBassem <me@mbassem.com>2024-02-09 01:57:31 +0000
commit08a5694e451218f1bcb2ad9eb42fd93250afbb96 (patch)
tree2cc2351f26b0ab98268db4bc463c0c3aa3f78a3b /packages/workers/crawler.ts
parentc5bfa5000f178475d0b019b5a960916134b2ecfb (diff)
downloadkarakeep-08a5694e451218f1bcb2ad9eb42fd93250afbb96.tar.zst
[refactor] Extract the bookmark model to be a high level model to support other type of bookmarks
Diffstat (limited to 'packages/workers/crawler.ts')
-rw-r--r--packages/workers/crawler.ts32
1 files changed, 9 insertions, 23 deletions
diff --git a/packages/workers/crawler.ts b/packages/workers/crawler.ts
index 817bba45..c76b2c42 100644
--- a/packages/workers/crawler.ts
+++ b/packages/workers/crawler.ts
@@ -29,10 +29,10 @@ export default async function runCrawler(job: Job<ZCrawlLinkRequest, void>) {
return;
}
- const { url, linkId } = request.data;
+ const { url, bookmarkId } = request.data;
logger.info(
- `[Crawler][${jobId}] Will crawl "${url}" for link with id "${linkId}"`,
+ `[Crawler][${jobId}] Will crawl "${url}" for link with id "${bookmarkId}"`,
);
// TODO(IMPORTANT): Run security validations on the input URL (e.g. deny localhost, etc)
@@ -46,33 +46,19 @@ export default async function runCrawler(job: Job<ZCrawlLinkRequest, void>) {
await prisma.bookmarkedLink.update({
where: {
- id: linkId,
+ id: bookmarkId,
},
data: {
- details: {
- upsert: {
- create: {
- title: meta.title,
- description: meta.description,
- imageUrl: meta.image,
- favicon: meta.logo,
- },
- update: {
- title: meta.title,
- description: meta.description,
- imageUrl: meta.image,
- favicon: meta.logo,
- },
- },
- },
- },
- include: {
- details: true,
+ title: meta.title,
+ description: meta.description,
+ imageUrl: meta.image,
+ favicon: meta.logo,
+ crawledAt: new Date(),
},
});
// Enqueue openai job
OpenAIQueue.add("openai", {
- linkId,
+ bookmarkId,
});
}