Implement metadata fetching logic in the crawler

10 files changed, 43 insertions, 17 deletions

diff --git a/web/lib/prisma.ts b/db/index.ts
index b5bf6ce8..b5bf6ce8 100644
--- a/web/lib/prisma.ts
+++ b/db/index.ts
diff --git a/web/prisma/migrations/20240205153748_add_users/migration.sql b/db/prisma/migrations/20240205153748_add_users/migration.sql
index cbf47073..cbf47073 100644
--- a/web/prisma/migrations/20240205153748_add_users/migration.sql
+++ b/db/prisma/migrations/20240205153748_add_users/migration.sql
diff --git a/web/prisma/migrations/20240206000813_add_links/migration.sql b/db/prisma/migrations/20240206000813_add_links/migration.sql
index 38c8d938..38c8d938 100644
--- a/web/prisma/migrations/20240206000813_add_links/migration.sql
+++ b/db/prisma/migrations/20240206000813_add_links/migration.sql
diff --git a/web/prisma/migrations/migration_lock.toml b/db/prisma/migrations/migration_lock.toml
index e5e5c470..e5e5c470 100644
--- a/web/prisma/migrations/migration_lock.toml
+++ b/db/prisma/migrations/migration_lock.toml
diff --git a/web/prisma/schema.prisma b/db/prisma/schema.prisma
index 54be3eae..f5b83b66 100644
--- a/web/prisma/schema.prisma
+++ b/db/prisma/schema.prisma
@@ -72,9 +72,10 @@ model BookmarkedLink {
 
 model BookmarkedLinkDetails {
   id          String   @id
-  title       String
-  description String
-  imageUrl    String
+  title       String?
+  description String?
+  imageUrl    String?
+  favicon     String?
   createdAt   DateTime @default(now())
 
   link BookmarkedLink @relation(fields: [id], references: [id], onDelete: Cascade)
diff --git a/web/app/api/v1/links/route.ts b/web/app/api/v1/links/route.ts
index 97bfa3de..990b6c02 100644
--- a/web/app/api/v1/links/route.ts
+++ b/web/app/api/v1/links/route.ts
@@ -1,7 +1,9 @@
 import { authOptions } from "@/lib/auth";
-import prisma from "@/lib/prisma";
+import { LinkCrawlerQueue } from "@remember/shared/queues";
+import prisma from "@remember/db";
+
 import {
-  ZNewBookmarkedLinkRequest,
+  zNewBookmarkedLinkRequestSchema,
   ZGetLinksResponse,
   ZBookmarkedLink,
 } from "@/lib/types/api/links";
@@ -15,7 +17,9 @@ export async function POST(request: NextRequest) {
     return new Response(null, { status: 401 });
   }
 
-  const linkRequest = ZNewBookmarkedLinkRequest.safeParse(await request.json());
+  const linkRequest = zNewBookmarkedLinkRequestSchema.safeParse(
+    await request.json(),
+  );
 
   if (!linkRequest.success) {
     return NextResponse.json(
@@ -33,8 +37,13 @@ export async function POST(request: NextRequest) {
     },
   });
 
-  let response: ZBookmarkedLink = { ...link };
+  // Enqueue crawling request
+  await LinkCrawlerQueue.add("crawl", {
+    linkId: link.id,
+    url: link.url,
+  });
 
+  let response: ZBookmarkedLink = { ...link };
   return NextResponse.json(response, { status: 201 });
 }
 
@@ -57,6 +66,7 @@ export async function GET() {
           title: true,
           description: true,
           imageUrl: true,
+          favicon: true,
         },
       },
     },
diff --git a/web/app/page.tsx b/web/app/page.tsx
index 2df40508..b78fe389 100644
--- a/web/app/page.tsx
+++ b/web/app/page.tsx
@@ -1,7 +1,16 @@
+"use client";
+
+import { useCallback } from "react";
 import { LoginButton } from "../components/auth/login";
 import { LogoutButton } from "../components/auth/logout";
 
 export default function Home() {
+  const addUrl = useCallback(async () => {
+    await fetch("/api/v1/links", {
+      method: "POST",
+      body: JSON.stringify({ url: "https://news.ycombinator.com/news" }),
+    });
+  }, []);
   return (
     <main className="flex min-h-screen flex-col items-center justify-between p-24">
       <div>
@@ -9,6 +18,9 @@ export default function Home() {
         <br />
         <br />
         <LogoutButton />
+        <br />
+        <br />
+        <button onClick={addUrl}>Add URL</button>
       </div>
     </main>
   );
diff --git a/web/lib/auth.ts b/web/lib/auth.ts
index 8b6527ec..cd6404de 100644
--- a/web/lib/auth.ts
+++ b/web/lib/auth.ts
@@ -2,7 +2,7 @@ import NextAuth, { NextAuthOptions } from "next-auth";
 import { PrismaAdapter } from "@next-auth/prisma-adapter";
 import AuthentikProvider from "next-auth/providers/authentik";
 import serverConfig from "@/lib/config";
-import prisma from "@/lib/prisma";
+import prisma from "@remember/db";
 
 let providers = [];
 
diff --git a/web/lib/types/api/links.ts b/web/lib/types/api/links.ts
index 465fe133..48214f9a 100644
--- a/web/lib/types/api/links.ts
+++ b/web/lib/types/api/links.ts
@@ -1,6 +1,6 @@
 import { z } from "zod";
 
-export const ZBookmarkedLink = z.object({
+export const zBookmarkedLinkSchema = z.object({
   id: z.string(),
   url: z.string().url(),
   createdAt: z.coerce.date(),
@@ -8,18 +8,21 @@ export const ZBookmarkedLink = z.object({
   details: z
     .object({
       title: z.string(),
-      description: z.string(),
-      imageUrl: z.string().url(),
+      description: z.string().optional(),
+      imageUrl: z.string().url().optional(),
+      favicon: z.string().url().optional(),
     })
     .nullish(),
 });
-export type ZBookmarkedLink = z.infer<typeof ZBookmarkedLink>;
+export type ZBookmarkedLink = z.infer<typeof zBookmarkedLinkSchema>;
 
 // POST /v1/links
-export const ZNewBookmarkedLinkRequest = ZBookmarkedLink.pick({ url: true });
+export const zNewBookmarkedLinkRequestSchema = zBookmarkedLinkSchema.pick({
+  url: true,
+});
 
 // GET /v1/links
-export const ZGetLinksResponse = z.object({
-  links: z.array(ZBookmarkedLink),
+export const zGetLinksResponseSchema = z.object({
+  links: z.array(zBookmarkedLinkSchema),
 });
-export type ZGetLinksResponse = z.infer<typeof ZGetLinksResponse>;
+export type ZGetLinksResponse = z.infer<typeof zGetLinksResponseSchema>;
diff --git a/web/package.json b/web/package.json
index 6dcbff47..6a043a77 100644
--- a/web/package.json
+++ b/web/package.json
@@ -1,4 +1,5 @@
 {
+  "$schema": "https://json.schemastore.org/package.json",
   "name": "@remember/web",
   "version": "0.1.0",
   "private": true,
@@ -18,7 +19,6 @@
     "next": "14.1.0",
     "next-auth": "^4.24.5",
     "prettier": "^3.2.5",
-    "prisma": "^5.9.1",
     "react": "^18",
     "react-dom": "^18",
     "tailwind-merge": "^2.2.1",