diff options
Diffstat (limited to 'packages/workers')
| -rw-r--r-- | packages/workers/crawler.ts | 20 | ||||
| -rw-r--r-- | packages/workers/index.ts | 3 | ||||
| -rw-r--r-- | packages/workers/openai.ts | 82 | ||||
| -rw-r--r-- | packages/workers/package.json | 10 | ||||
| -rw-r--r-- | packages/workers/tsconfig.json | 7 |
5 files changed, 55 insertions, 67 deletions
diff --git a/packages/workers/crawler.ts b/packages/workers/crawler.ts index a4d8d05c..bfb46218 100644 --- a/packages/workers/crawler.ts +++ b/packages/workers/crawler.ts @@ -10,7 +10,7 @@ import { import { Worker } from "bullmq"; import { Job } from "bullmq"; -import { prisma } from "@hoarder/db"; +import { db } from "@hoarder/db"; import { Browser } from "puppeteer"; import puppeteer from "puppeteer-extra"; @@ -28,6 +28,8 @@ import metascraperReadability from "metascraper-readability"; import { Mutex } from "async-mutex"; import assert from "assert"; import serverConfig from "@hoarder/shared/config"; +import { bookmarkLinks } from "@hoarder/db/schema"; +import { eq } from "drizzle-orm"; const metascraperParser = metascraper([ metascraperReadability(), @@ -91,8 +93,8 @@ export class CrawlerWorker { } async function getBookmarkUrl(bookmarkId: string) { - const bookmark = await prisma.bookmarkedLink.findUnique({ - where: { id: bookmarkId }, + const bookmark = await db.query.bookmarkLinks.findFirst({ + where: eq(bookmarkLinks.id, bookmarkId), }); if (!bookmark) { @@ -155,18 +157,16 @@ async function runCrawler(job: Job<ZCrawlLinkRequest, void>) { html: htmlContent, }); - await prisma.bookmarkedLink.update({ - where: { - id: bookmarkId, - }, - data: { + await db + .update(bookmarkLinks) + .set({ title: meta.title, description: meta.description, imageUrl: meta.image, favicon: meta.logo, crawledAt: new Date(), - }, - }); + }) + .where(eq(bookmarkLinks.id, bookmarkId)); // Enqueue openai job OpenAIQueue.add("openai", { diff --git a/packages/workers/index.ts b/packages/workers/index.ts index c021ec67..67be7af2 100644 --- a/packages/workers/index.ts +++ b/packages/workers/index.ts @@ -1,5 +1,4 @@ -import dotenv from "dotenv"; -dotenv.config(); +import "dotenv/config"; import { CrawlerWorker } from "./crawler"; import { OpenAiWorker } from "./openai"; diff --git a/packages/workers/openai.ts b/packages/workers/openai.ts index 8972eb66..ed4c72e8 100644 --- a/packages/workers/openai.ts +++ b/packages/workers/openai.ts @@ -1,4 +1,4 @@ -import { prisma, BookmarkedLink } from "@hoarder/db"; +import { db } from "@hoarder/db"; import logger from "@hoarder/shared/logger"; import serverConfig from "@hoarder/shared/config"; import { @@ -11,6 +11,13 @@ import { Job } from "bullmq"; import OpenAI from "openai"; import { z } from "zod"; import { Worker } from "bullmq"; +import { + bookmarkLinks, + bookmarkTags, + bookmarks, + tagsOnBookmarks, +} from "@hoarder/db/schema"; +import { eq } from "drizzle-orm"; const openAIResponseSchema = z.object({ tags: z.array(z.string()), @@ -57,17 +64,19 @@ Description: ${description} } async function fetchBookmark(linkId: string) { - return await prisma.bookmark.findUnique({ - where: { - id: linkId, - }, - include: { + return await db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, linkId), + with: { link: true, }, }); } -async function inferTags(jobId: string, link: BookmarkedLink, openai: OpenAI) { +async function inferTags( + jobId: string, + link: typeof bookmarkLinks.$inferSelect, + openai: OpenAI, +) { const linkDescription = link?.description; if (!linkDescription) { throw new Error( @@ -111,51 +120,26 @@ async function inferTags(jobId: string, link: BookmarkedLink, openai: OpenAI) { } async function createTags(tags: string[], userId: string) { - const existingTags = await prisma.bookmarkTags.findMany({ - select: { - id: true, - name: true, - }, - where: { - userId, - name: { - in: tags, - }, - }, - }); - - const existingTagSet = new Set<string>(existingTags.map((t) => t.name)); - - const newTags = tags.filter((t) => !existingTagSet.has(t)); - - // TODO: Prisma doesn't support createMany in Sqlite - const newTagObjects = await Promise.all( - newTags.map((t) => { - return prisma.bookmarkTags.create({ - data: { - name: t, - userId: userId, - }, - }); - }), - ); - - return existingTags.map((t) => t.id).concat(newTagObjects.map((t) => t.id)); + const res = await db + .insert(bookmarkTags) + .values( + tags.map((t) => ({ + name: t, + userId, + })), + ) + .onConflictDoNothing() + .returning({ id: bookmarkTags.id }); + return res.map((r) => r.id); } async function connectTags(bookmarkId: string, tagIds: string[]) { - // TODO: Prisma doesn't support createMany in Sqlite - // TODO: This could fail on refetch if the tags are already there - await Promise.all( - tagIds.map((tagId) => { - return prisma.tagsOnBookmarks.create({ - data: { - tagId, - bookmarkId, - attachedBy: "ai", - }, - }); - }), + await db.insert(tagsOnBookmarks).values( + tagIds.map((tagId) => ({ + tagId, + bookmarkId, + attachedBy: "ai" as const, + })), ); } diff --git a/packages/workers/package.json b/packages/workers/package.json index 48510531..b2170441 100644 --- a/packages/workers/package.json +++ b/packages/workers/package.json @@ -10,6 +10,7 @@ "async-mutex": "^0.4.1", "bullmq": "^5.1.9", "dotenv": "^16.4.1", + "drizzle-orm": "^0.29.4", "metascraper": "^5.43.4", "metascraper-description": "^5.43.4", "metascraper-image": "^5.43.4", @@ -23,17 +24,16 @@ "puppeteer": "^22.0.0", "puppeteer-extra": "^3.3.6", "puppeteer-extra-plugin-stealth": "^2.11.2", - "ts-node": "^10.9.2", + "tsx": "^4.7.1", "typescript": "^5", "zod": "^3.22.4" }, "devDependencies": { - "@types/metascraper": "^5.14.3", - "nodemon": "^3.0.3" + "@types/metascraper": "^5.14.3" }, "scripts": { - "start": "nodemon index.ts", - "start:prod": "ts-node -T index.ts", + "start": "tsx watch index.ts", + "start:prod": "tsx index.ts", "typecheck": "tsc --noEmit" } } diff --git a/packages/workers/tsconfig.json b/packages/workers/tsconfig.json index 5ab467a9..cf49c407 100644 --- a/packages/workers/tsconfig.json +++ b/packages/workers/tsconfig.json @@ -2,5 +2,10 @@ "$schema": "https://json.schemastore.org/tsconfig", "extends": "@tsconfig/node21/tsconfig.json", "include": ["**/*.ts"], - "exclude": ["node_modules"] + "exclude": ["node_modules"], + "compilerOptions": { + "module": "ESNext", + "moduleResolution": "node", + "esModuleInterop": true + } } |