Implement metadata fetching logic in the crawler

10 files changed, 39 insertions, 226 deletions

diff --git a/web/app/api/v1/links/route.ts b/web/app/api/v1/links/route.ts
index 97bfa3de..990b6c02 100644
--- a/web/app/api/v1/links/route.ts
+++ b/web/app/api/v1/links/route.ts
@@ -1,7 +1,9 @@
 import { authOptions } from "@/lib/auth";
-import prisma from "@/lib/prisma";
+import { LinkCrawlerQueue } from "@remember/shared/queues";
+import prisma from "@remember/db";
+
 import {
-  ZNewBookmarkedLinkRequest,
+  zNewBookmarkedLinkRequestSchema,
   ZGetLinksResponse,
   ZBookmarkedLink,
 } from "@/lib/types/api/links";
@@ -15,7 +17,9 @@ export async function POST(request: NextRequest) {
     return new Response(null, { status: 401 });
   }
 
-  const linkRequest = ZNewBookmarkedLinkRequest.safeParse(await request.json());
+  const linkRequest = zNewBookmarkedLinkRequestSchema.safeParse(
+    await request.json(),
+  );
 
   if (!linkRequest.success) {
     return NextResponse.json(
@@ -33,8 +37,13 @@ export async function POST(request: NextRequest) {
     },
   });
 
-  let response: ZBookmarkedLink = { ...link };
+  // Enqueue crawling request
+  await LinkCrawlerQueue.add("crawl", {
+    linkId: link.id,
+    url: link.url,
+  });
 
+  let response: ZBookmarkedLink = { ...link };
   return NextResponse.json(response, { status: 201 });
 }
 
@@ -57,6 +66,7 @@ export async function GET() {
           title: true,
           description: true,
           imageUrl: true,
+          favicon: true,
         },
       },
     },
diff --git a/web/app/page.tsx b/web/app/page.tsx
index 2df40508..b78fe389 100644
--- a/web/app/page.tsx
+++ b/web/app/page.tsx
@@ -1,7 +1,16 @@
+"use client";
+
+import { useCallback } from "react";
 import { LoginButton } from "../components/auth/login";
 import { LogoutButton } from "../components/auth/logout";
 
 export default function Home() {
+  const addUrl = useCallback(async () => {
+    await fetch("/api/v1/links", {
+      method: "POST",
+      body: JSON.stringify({ url: "https://news.ycombinator.com/news" }),
+    });
+  }, []);
   return (
     <main className="flex min-h-screen flex-col items-center justify-between p-24">
       <div>
@@ -9,6 +18,9 @@ export default function Home() {
         <br />
         <br />
         <LogoutButton />
+        <br />
+        <br />
+        <button onClick={addUrl}>Add URL</button>
       </div>
     </main>
   );
diff --git a/web/lib/auth.ts b/web/lib/auth.ts
index 8b6527ec..cd6404de 100644
--- a/web/lib/auth.ts
+++ b/web/lib/auth.ts
@@ -2,7 +2,7 @@ import NextAuth, { NextAuthOptions } from "next-auth";
 import { PrismaAdapter } from "@next-auth/prisma-adapter";
 import AuthentikProvider from "next-auth/providers/authentik";
 import serverConfig from "@/lib/config";
-import prisma from "@/lib/prisma";
+import prisma from "@remember/db";
 
 let providers = [];
 
diff --git a/web/lib/prisma.ts b/web/lib/prisma.ts
deleted file mode 100644
index b5bf6ce8..00000000
--- a/web/lib/prisma.ts
+++ /dev/null
@@ -1,5 +0,0 @@
-import { PrismaClient } from "@prisma/client";
-
-const prisma = new PrismaClient();
-
-export default prisma;
diff --git a/web/lib/types/api/links.ts b/web/lib/types/api/links.ts
index 465fe133..48214f9a 100644
--- a/web/lib/types/api/links.ts
+++ b/web/lib/types/api/links.ts
@@ -1,6 +1,6 @@
 import { z } from "zod";
 
-export const ZBookmarkedLink = z.object({
+export const zBookmarkedLinkSchema = z.object({
   id: z.string(),
   url: z.string().url(),
   createdAt: z.coerce.date(),
@@ -8,18 +8,21 @@ export const ZBookmarkedLink = z.object({
   details: z
     .object({
       title: z.string(),
-      description: z.string(),
-      imageUrl: z.string().url(),
+      description: z.string().optional(),
+      imageUrl: z.string().url().optional(),
+      favicon: z.string().url().optional(),
     })
     .nullish(),
 });
-export type ZBookmarkedLink = z.infer<typeof ZBookmarkedLink>;
+export type ZBookmarkedLink = z.infer<typeof zBookmarkedLinkSchema>;
 
 // POST /v1/links
-export const ZNewBookmarkedLinkRequest = ZBookmarkedLink.pick({ url: true });
+export const zNewBookmarkedLinkRequestSchema = zBookmarkedLinkSchema.pick({
+  url: true,
+});
 
 // GET /v1/links
-export const ZGetLinksResponse = z.object({
-  links: z.array(ZBookmarkedLink),
+export const zGetLinksResponseSchema = z.object({
+  links: z.array(zBookmarkedLinkSchema),
 });
-export type ZGetLinksResponse = z.infer<typeof ZGetLinksResponse>;
+export type ZGetLinksResponse = z.infer<typeof zGetLinksResponseSchema>;
diff --git a/web/package.json b/web/package.json
index 6dcbff47..6a043a77 100644
--- a/web/package.json
+++ b/web/package.json
@@ -1,4 +1,5 @@
 {
+  "$schema": "https://json.schemastore.org/package.json",
   "name": "@remember/web",
   "version": "0.1.0",
   "private": true,
@@ -18,7 +19,6 @@
     "next": "14.1.0",
     "next-auth": "^4.24.5",
     "prettier": "^3.2.5",
-    "prisma": "^5.9.1",
     "react": "^18",
     "react-dom": "^18",
     "tailwind-merge": "^2.2.1",
diff --git a/web/prisma/migrations/20240205153748_add_users/migration.sql b/web/prisma/migrations/20240205153748_add_users/migration.sql
deleted file mode 100644
index cbf47073..00000000
--- a/web/prisma/migrations/20240205153748_add_users/migration.sql
+++ /dev/null
@@ -1,56 +0,0 @@
--- CreateTable
-CREATE TABLE "Account" (
-    "id" TEXT NOT NULL PRIMARY KEY,
-    "userId" TEXT NOT NULL,
-    "type" TEXT NOT NULL,
-    "provider" TEXT NOT NULL,
-    "providerAccountId" TEXT NOT NULL,
-    "refresh_token" TEXT,
-    "access_token" TEXT,
-    "expires_at" INTEGER,
-    "token_type" TEXT,
-    "scope" TEXT,
-    "id_token" TEXT,
-    "session_state" TEXT,
-    CONSTRAINT "Account_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User" ("id") ON DELETE CASCADE ON UPDATE CASCADE
-);
-
--- CreateTable
-CREATE TABLE "Session" (
-    "id" TEXT NOT NULL PRIMARY KEY,
-    "sessionToken" TEXT NOT NULL,
-    "userId" TEXT NOT NULL,
-    "expires" DATETIME NOT NULL,
-    CONSTRAINT "Session_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User" ("id") ON DELETE CASCADE ON UPDATE CASCADE
-);
-
--- CreateTable
-CREATE TABLE "User" (
-    "id" TEXT NOT NULL PRIMARY KEY,
-    "name" TEXT,
-    "email" TEXT,
-    "emailVerified" DATETIME,
-    "image" TEXT
-);
-
--- CreateTable
-CREATE TABLE "VerificationToken" (
-    "identifier" TEXT NOT NULL,
-    "token" TEXT NOT NULL,
-    "expires" DATETIME NOT NULL
-);
-
--- CreateIndex
-CREATE UNIQUE INDEX "Account_provider_providerAccountId_key" ON "Account"("provider", "providerAccountId");
-
--- CreateIndex
-CREATE UNIQUE INDEX "Session_sessionToken_key" ON "Session"("sessionToken");
-
--- CreateIndex
-CREATE UNIQUE INDEX "User_email_key" ON "User"("email");
-
--- CreateIndex
-CREATE UNIQUE INDEX "VerificationToken_token_key" ON "VerificationToken"("token");
-
--- CreateIndex
-CREATE UNIQUE INDEX "VerificationToken_identifier_token_key" ON "VerificationToken"("identifier", "token");
diff --git a/web/prisma/migrations/20240206000813_add_links/migration.sql b/web/prisma/migrations/20240206000813_add_links/migration.sql
deleted file mode 100644
index 38c8d938..00000000
--- a/web/prisma/migrations/20240206000813_add_links/migration.sql
+++ /dev/null
@@ -1,43 +0,0 @@
--- CreateTable
-CREATE TABLE "BookmarkedLink" (
-    "id" TEXT NOT NULL PRIMARY KEY,
-    "url" TEXT NOT NULL,
-    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    "userId" TEXT NOT NULL,
-    CONSTRAINT "BookmarkedLink_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User" ("id") ON DELETE CASCADE ON UPDATE CASCADE
-);
-
--- CreateTable
-CREATE TABLE "BookmarkedLinkDetails" (
-    "id" TEXT NOT NULL PRIMARY KEY,
-    "title" TEXT NOT NULL,
-    "description" TEXT NOT NULL,
-    "imageUrl" TEXT NOT NULL,
-    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    CONSTRAINT "BookmarkedLinkDetails_id_fkey" FOREIGN KEY ("id") REFERENCES "BookmarkedLink" ("id") ON DELETE CASCADE ON UPDATE CASCADE
-);
-
--- CreateTable
-CREATE TABLE "BookmarkTags" (
-    "id" TEXT NOT NULL PRIMARY KEY,
-    "name" TEXT NOT NULL,
-    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    "userId" TEXT NOT NULL,
-    CONSTRAINT "BookmarkTags_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User" ("id") ON DELETE CASCADE ON UPDATE CASCADE
-);
-
--- CreateTable
-CREATE TABLE "TagsOnLinks" (
-    "linkId" TEXT NOT NULL,
-    "tagId" TEXT NOT NULL,
-    "attachedAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    "bookmarkTagsId" TEXT NOT NULL,
-    CONSTRAINT "TagsOnLinks_linkId_fkey" FOREIGN KEY ("linkId") REFERENCES "BookmarkedLink" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
-    CONSTRAINT "TagsOnLinks_tagId_fkey" FOREIGN KEY ("tagId") REFERENCES "BookmarkTags" ("id") ON DELETE CASCADE ON UPDATE CASCADE
-);
-
--- CreateIndex
-CREATE UNIQUE INDEX "BookmarkTags_name_key" ON "BookmarkTags"("name");
-
--- CreateIndex
-CREATE UNIQUE INDEX "TagsOnLinks_linkId_tagId_key" ON "TagsOnLinks"("linkId", "tagId");
diff --git a/web/prisma/migrations/migration_lock.toml b/web/prisma/migrations/migration_lock.toml
deleted file mode 100644
index e5e5c470..00000000
--- a/web/prisma/migrations/migration_lock.toml
+++ /dev/null
@@ -1,3 +0,0 @@
-# Please do not edit this file manually
-# It should be added in your version-control system (i.e. Git)
-provider = "sqlite"
\ No newline at end of file
diff --git a/web/prisma/schema.prisma b/web/prisma/schema.prisma
deleted file mode 100644
index 54be3eae..00000000
--- a/web/prisma/schema.prisma
+++ /dev/null
@@ -1,105 +0,0 @@
-// This is your Prisma schema file,
-// learn more about it in the docs: https://pris.ly/d/prisma-schema
-
-generator client {
-  provider = "prisma-client-js"
-}
-
-datasource db {
-  provider = "sqlite"
-  url      = env("DATABASE_URL")
-}
-
-model Account {
-  id                String  @id @default(cuid())
-  userId            String
-  type              String
-  provider          String
-  providerAccountId String
-  refresh_token     String?
-  access_token      String?
-  expires_at        Int?
-  token_type        String?
-  scope             String?
-  id_token          String?
-  session_state     String?
-
-  user User @relation(fields: [userId], references: [id], onDelete: Cascade)
-
-  @@unique([provider, providerAccountId])
-}
-
-model Session {
-  id           String   @id @default(cuid())
-  sessionToken String   @unique
-  userId       String
-  expires      DateTime
-  user         User     @relation(fields: [userId], references: [id], onDelete: Cascade)
-}
-
-model User {
-  id            String           @id @default(cuid())
-  name          String?
-  email         String?          @unique
-  emailVerified DateTime?
-  image         String?
-  accounts      Account[]
-  sessions      Session[]
-  links         BookmarkedLink[]
-  tags          BookmarkTags[]
-}
-
-model VerificationToken {
-  identifier String
-  token      String   @unique
-  expires    DateTime
-
-  @@unique([identifier, token])
-}
-
-model BookmarkedLink {
-  id        String   @id @default(cuid())
-  url       String
-  createdAt DateTime @default(now())
-
-  userId String
-
-  details BookmarkedLinkDetails?
-
-  user User          @relation(fields: [userId], references: [id], onDelete: Cascade)
-  tags TagsOnLinks[]
-}
-
-model BookmarkedLinkDetails {
-  id          String   @id
-  title       String
-  description String
-  imageUrl    String
-  createdAt   DateTime @default(now())
-
-  link BookmarkedLink @relation(fields: [id], references: [id], onDelete: Cascade)
-}
-
-model BookmarkTags {
-  id        String   @id @default(cuid())
-  name      String   @unique
-  createdAt DateTime @default(now())
-
-  userId String
-
-  user          User          @relation(fields: [userId], references: [id], onDelete: Cascade)
-  attachedLinks TagsOnLinks[]
-}
-
-model TagsOnLinks {
-  link   BookmarkedLink @relation(fields: [linkId], references: [id], onDelete: Cascade)
-  linkId String
-
-  tag   BookmarkTags @relation(fields: [tagId], references: [id], onDelete: Cascade)
-  tagId String
-
-  attachedAt     DateTime @default(now())
-  bookmarkTagsId String
-
-  @@unique([linkId, tagId])
-}