refactor: Prepare for pagination by dropping querying bookmarks by id

16 files changed, 155 insertions, 100 deletions

diff --git a/apps/mobile/app/dashboard/(tabs)/index.tsx b/apps/mobile/app/dashboard/(tabs)/index.tsx
index 1a17d472..7f70af6b 100644
--- a/apps/mobile/app/dashboard/(tabs)/index.tsx
+++ b/apps/mobile/app/dashboard/(tabs)/index.tsx
@@ -2,9 +2,9 @@ import { Platform, SafeAreaView, View } from "react-native";
 import * as Haptics from "expo-haptics";
 import { useRouter } from "expo-router";
 import BookmarkList from "@/components/bookmarks/BookmarkList";
+import PageTitle from "@/components/ui/PageTitle";
 import { MenuView } from "@react-native-menu/menu";
 import { SquarePen } from "lucide-react-native";
-import PageTitle from "@/components/ui/PageTitle";
 
 function HeaderRight() {
   const router = useRouter();
@@ -49,7 +49,7 @@ export default function Home() {
   return (
     <SafeAreaView>
       <BookmarkList
-        archived={false}
+        query={{ archived: false }}
         header={
           <View className="flex flex-row justify-between">
             <PageTitle title="Home" />
diff --git a/apps/mobile/app/dashboard/(tabs)/search.tsx b/apps/mobile/app/dashboard/(tabs)/search.tsx
index 76e9aef9..25fc53d5 100644
--- a/apps/mobile/app/dashboard/(tabs)/search.tsx
+++ b/apps/mobile/app/dashboard/(tabs)/search.tsx
@@ -21,7 +21,7 @@ export default function Search() {
     <SafeAreaView>
       {data && (
         <BookmarkList
-          ids={data.bookmarks.map((b) => b.id)}
+          query={{ids: data.bookmarks.map((b) => b.id)}}
           header={
             <View>
               <PageTitle title="Search" />
diff --git a/apps/mobile/app/dashboard/archive.tsx b/apps/mobile/app/dashboard/archive.tsx
index 5c86c6fc..98a03631 100644
--- a/apps/mobile/app/dashboard/archive.tsx
+++ b/apps/mobile/app/dashboard/archive.tsx
@@ -5,7 +5,7 @@ import PageTitle from "@/components/ui/PageTitle";
 export default function Archive() {
   return (
     <SafeAreaView>
-      <BookmarkList archived header={<PageTitle title="🗄️ Archive" />} />
+      <BookmarkList query={{archived: true}} header={<PageTitle title="🗄️ Archive" />} />
     </SafeAreaView>
   );
 }
diff --git a/apps/mobile/app/dashboard/favourites.tsx b/apps/mobile/app/dashboard/favourites.tsx
index 6025d514..f62d561e 100644
--- a/apps/mobile/app/dashboard/favourites.tsx
+++ b/apps/mobile/app/dashboard/favourites.tsx
@@ -6,8 +6,10 @@ export default function Favourites() {
   return (
     <SafeAreaView>
       <BookmarkList
-        archived={false}
-        favourited
+        query={{
+          archived: false,
+          favourited: true,
+        }}
         header={<PageTitle title="⭐️ Favourites" />}
       />
     </SafeAreaView>
diff --git a/apps/mobile/app/dashboard/lists/[slug].tsx b/apps/mobile/app/dashboard/lists/[slug].tsx
index 0d1c01dc..8596b49f 100644
--- a/apps/mobile/app/dashboard/lists/[slug].tsx
+++ b/apps/mobile/app/dashboard/lists/[slug].tsx
@@ -24,8 +24,10 @@ export default function ListView() {
       {list ? (
         <View>
           <BookmarkList
-            archived={false}
-            ids={list.bookmarks}
+            query={{
+              archived: false,
+              listId: list.id,
+            }}
             header={<PageTitle title={`${list.icon} ${list.name}`} />}
           />
         </View>
diff --git a/apps/mobile/app/dashboard/tags/[slug].tsx b/apps/mobile/app/dashboard/tags/[slug].tsx
index 2d37b172..cb6e2ef4 100644
--- a/apps/mobile/app/dashboard/tags/[slug].tsx
+++ b/apps/mobile/app/dashboard/tags/[slug].tsx
@@ -25,8 +25,10 @@ export default function TagView() {
       {tag ? (
         <View>
           <BookmarkList
-            archived={false}
-            ids={tag.bookmarks}
+            query={{
+              archived: false,
+              tagId: tag.id,
+            }}
             header={<PageTitle title={tag.name} />}
           />
         </View>
diff --git a/apps/mobile/components/bookmarks/BookmarkList.tsx b/apps/mobile/components/bookmarks/BookmarkList.tsx
index 04a3d922..e7b5e5f2 100644
--- a/apps/mobile/components/bookmarks/BookmarkList.tsx
+++ b/apps/mobile/components/bookmarks/BookmarkList.tsx
@@ -4,18 +4,16 @@ import Animated, { LinearTransition } from "react-native-reanimated";
 import { api } from "@/lib/trpc";
 import { useScrollToTop } from "@react-navigation/native";
 
+import type { ZGetBookmarksRequest } from "@hoarder/trpc/types/bookmarks";
+
 import FullPageSpinner from "../ui/FullPageSpinner";
 import BookmarkCard from "./BookmarkCard";
 
 export default function BookmarkList({
-  favourited,
-  archived,
-  ids,
-  header
+  query,
+  header,
 }: {
-  favourited?: boolean;
-  archived?: boolean;
-  ids?: string[];
+  query: ZGetBookmarksRequest;
   header?: React.ReactElement;
 }) {
   const apiUtils = api.useUtils();
@@ -23,11 +21,7 @@ export default function BookmarkList({
   const flatListRef = useRef(null);
   useScrollToTop(flatListRef);
   const { data, isPending, isPlaceholderData } =
-    api.bookmarks.getBookmarks.useQuery({
-      favourited,
-      archived,
-      ids,
-    });
+    api.bookmarks.getBookmarks.useQuery(query);
 
   useEffect(() => {
     setRefreshing(isPending || isPlaceholderData);
diff --git a/apps/web/app/dashboard/lists/[listId]/page.tsx b/apps/web/app/dashboard/lists/[listId]/page.tsx
index 4e35c377..b9a26053 100644
--- a/apps/web/app/dashboard/lists/[listId]/page.tsx
+++ b/apps/web/app/dashboard/lists/[listId]/page.tsx
@@ -1,6 +1,6 @@
 import { notFound, redirect } from "next/navigation";
+import BookmarksGrid from "@/components/dashboard/bookmarks/BookmarksGrid";
 import DeleteListButton from "@/components/dashboard/lists/DeleteListButton";
-import ListView from "@/components/dashboard/lists/ListView";
 import { api } from "@/server/api/client";
 import { getServerAuthSession } from "@/server/auth";
 import { TRPCError } from "@trpc/server";
@@ -27,7 +27,10 @@ export default async function ListPage({
     throw e;
   }
 
-  const bookmarks = await api.bookmarks.getBookmarks({ ids: list.bookmarks });
+  const bookmarks = await api.bookmarks.getBookmarks({
+    listId: list.id,
+    archived: false,
+  });
 
   return (
     <div className="container flex flex-col gap-3">
@@ -38,7 +41,10 @@ export default async function ListPage({
         <DeleteListButton list={list} />
       </div>
       <hr />
-      <ListView list={list} bookmarks={bookmarks.bookmarks} />
+      <BookmarksGrid
+        query={{ listId: list.id, archived: false }}
+        bookmarks={bookmarks.bookmarks}
+      />
     </div>
   );
 }
diff --git a/apps/web/app/dashboard/tags/[tagName]/page.tsx b/apps/web/app/dashboard/tags/[tagName]/page.tsx
index 0c5c1c1f..dee29c5e 100644
--- a/apps/web/app/dashboard/tags/[tagName]/page.tsx
+++ b/apps/web/app/dashboard/tags/[tagName]/page.tsx
@@ -28,8 +28,8 @@ export default async function TagPage({
   }
 
   const query = {
-    ids: tag.bookmarks,
     archived: false,
+    tagId: tag.id,
   };
 
   const bookmarks = await api.bookmarks.getBookmarks(query);
diff --git a/apps/web/components/dashboard/bookmarks/AddToListModal.tsx b/apps/web/components/dashboard/bookmarks/AddToListModal.tsx
index 6242aa27..b8cce66d 100644
--- a/apps/web/components/dashboard/bookmarks/AddToListModal.tsx
+++ b/apps/web/components/dashboard/bookmarks/AddToListModal.tsx
@@ -52,7 +52,6 @@ export default function AddToListModal({
   const { data: lists, isPending: isFetchingListsPending } =
     api.lists.list.useQuery();
 
-  const listInvalidationFunction = api.useUtils().lists.get.invalidate;
   const bookmarksInvalidationFunction =
     api.useUtils().bookmarks.getBookmarks.invalidate;
 
@@ -62,8 +61,8 @@ export default function AddToListModal({
         toast({
           description: "List has been updated!",
         });
-        listInvalidationFunction({ listId: req.listId });
-        bookmarksInvalidationFunction();
+        setOpen(false);
+        bookmarksInvalidationFunction({ listId: req.listId });
       },
       onError: (e) => {
         if (e.data?.code == "BAD_REQUEST") {
diff --git a/apps/web/components/dashboard/bookmarks/TagsEditor.tsx b/apps/web/components/dashboard/bookmarks/TagsEditor.tsx
index 8bfbce19..12c0dcd0 100644
--- a/apps/web/components/dashboard/bookmarks/TagsEditor.tsx
+++ b/apps/web/components/dashboard/bookmarks/TagsEditor.tsx
@@ -75,6 +75,7 @@ export function TagsEditor({ bookmark }: { bookmark: ZBookmark }) {
         description: "Tags has been updated!",
       });
       bookmarkInvalidationFunction({ bookmarkId: bookmark.id });
+      // TODO(bug) Invalidate the tag views as well
     },
     onError: () => {
       toast({
diff --git a/apps/web/components/dashboard/lists/ListView.tsx b/apps/web/components/dashboard/lists/ListView.tsx
deleted file mode 100644
index beeea7f1..00000000
--- a/apps/web/components/dashboard/lists/ListView.tsx
+++ /dev/null
@@ -1,26 +0,0 @@
-"use client";
-
-import BookmarksGrid from "@/components/dashboard/bookmarks/BookmarksGrid";
-import { api } from "@/lib/trpc";
-
-import type { ZBookmark } from "@hoarder/trpc/types/bookmarks";
-import type { ZBookmarkListWithBookmarks } from "@hoarder/trpc/types/lists";
-
-export default function ListView({
-  bookmarks,
-  list: initialData,
-}: {
-  list: ZBookmarkListWithBookmarks;
-  bookmarks: ZBookmark[];
-}) {
-  const { data } = api.lists.get.useQuery(
-    { listId: initialData.id },
-    {
-      initialData,
-    },
-  );
-
-  return (
-    <BookmarksGrid query={{ ids: data.bookmarks }} bookmarks={bookmarks} />
-  );
-}
diff --git a/packages/trpc/package.json b/packages/trpc/package.json
index c930da4a..411397dc 100644
--- a/packages/trpc/package.json
+++ b/packages/trpc/package.json
@@ -15,6 +15,7 @@
     "bcryptjs": "^2.4.3",
     "drizzle-orm": "^0.29.4",
     "superjson": "^2.2.1",
+    "tiny-invariant": "^1.3.3",
     "zod": "^3.22.4"
   },
   "devDependencies": {
diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts
index 1c6208a9..f91c8c6a 100644
--- a/packages/trpc/routers/bookmarks.ts
+++ b/packages/trpc/routers/bookmarks.ts
@@ -1,21 +1,15 @@
+import { experimental_trpcMiddleware, TRPCError } from "@trpc/server";
+import { and, desc, eq, exists, inArray } from "drizzle-orm";
+import invariant from "tiny-invariant";
 import { z } from "zod";
-import { Context, authedProcedure, router } from "../index";
-import { getSearchIdxClient } from "@hoarder/shared/search";
-import {
-  ZBookmark,
-  ZBookmarkContent,
-  zBareBookmarkSchema,
-  zBookmarkSchema,
-  zGetBookmarksRequestSchema,
-  zGetBookmarksResponseSchema,
-  zNewBookmarkRequestSchema,
-  zUpdateBookmarksRequestSchema,
-} from "../types/bookmarks";
+
+import { db as DONT_USE_db } from "@hoarder/db";
 import {
   bookmarkLinks,
+  bookmarks,
+  bookmarksInLists,
   bookmarkTags,
   bookmarkTexts,
-  bookmarks,
   tagsOnBookmarks,
 } from "@hoarder/db/schema";
 import {
@@ -23,11 +17,20 @@ import {
   OpenAIQueue,
   SearchIndexingQueue,
 } from "@hoarder/shared/queues";
-import { TRPCError, experimental_trpcMiddleware } from "@trpc/server";
-import { and, desc, eq, inArray } from "drizzle-orm";
-import { ZBookmarkTags } from "../types/tags";
+import { getSearchIdxClient } from "@hoarder/shared/search";
 
-import { db as DONT_USE_db } from "@hoarder/db";
+import { authedProcedure, Context, router } from "../index";
+import {
+  zBareBookmarkSchema,
+  ZBookmark,
+  ZBookmarkContent,
+  zBookmarkSchema,
+  zGetBookmarksRequestSchema,
+  zGetBookmarksResponseSchema,
+  zNewBookmarkRequestSchema,
+  zUpdateBookmarksRequestSchema,
+} from "../types/bookmarks";
+import { ZBookmarkTags } from "../types/tags";
 
 const ensureBookmarkOwnership = experimental_trpcMiddleware<{
   ctx: Context;
@@ -79,16 +82,18 @@ async function dummyDrizzleReturnType() {
   return x;
 }
 
-function toZodSchema(
-  bookmark: Awaited<ReturnType<typeof dummyDrizzleReturnType>>,
-): ZBookmark {
+type BookmarkQueryReturnType = Awaited<
+  ReturnType<typeof dummyDrizzleReturnType>
+>;
+
+function toZodSchema(bookmark: BookmarkQueryReturnType): ZBookmark {
   const { tagsOnBookmarks, link, text, ...rest } = bookmark;
 
   let content: ZBookmarkContent;
   if (link) {
     content = { type: "link", ...link };
   } else if (text) {
-    content = { type: "text", text: text.text || "" };
+    content = { type: "text", text: text.text ?? "" };
   } else {
     throw new Error("Unknown content type");
   }
@@ -147,7 +152,7 @@ export const bookmarksAppRouter = router({
               )[0];
               content = {
                 type: "text",
-                text: text.text || "",
+                text: text.text ?? "",
               };
               break;
             }
@@ -347,30 +352,90 @@ export const bookmarksAppRouter = router({
       if (input.ids && input.ids.length == 0) {
         return { bookmarks: [] };
       }
-      const results = await ctx.db.query.bookmarks.findMany({
-        where: and(
-          eq(bookmarks.userId, ctx.user.id),
-          input.archived !== undefined
-            ? eq(bookmarks.archived, input.archived)
-            : undefined,
-          input.favourited !== undefined
-            ? eq(bookmarks.favourited, input.favourited)
-            : undefined,
-          input.ids ? inArray(bookmarks.id, input.ids) : undefined,
-        ),
-        orderBy: [desc(bookmarks.createdAt)],
-        with: {
-          tagsOnBookmarks: {
-            with: {
-              tag: true,
-            },
-          },
-          link: true,
-          text: true,
+      // TODO: Consider not inlining the tags in the response of getBookmarks as this query is getting kinda expensive
+      const results = await ctx.db
+        .select()
+        .from(bookmarks)
+        .where(
+          and(
+            eq(bookmarks.userId, ctx.user.id),
+            input.archived !== undefined
+              ? eq(bookmarks.archived, input.archived)
+              : undefined,
+            input.favourited !== undefined
+              ? eq(bookmarks.favourited, input.favourited)
+              : undefined,
+            input.ids ? inArray(bookmarks.id, input.ids) : undefined,
+            input.tagId !== undefined
+              ? exists(
+                ctx.db
+                  .select()
+                  .from(tagsOnBookmarks)
+                  .where(
+                    and(
+                      eq(tagsOnBookmarks.bookmarkId, bookmarks.id),
+                      eq(tagsOnBookmarks.tagId, input.tagId),
+                    ),
+                  ),
+              )
+              : undefined,
+            input.listId !== undefined
+              ? exists(
+                ctx.db
+                  .select()
+                  .from(bookmarksInLists)
+                  .where(
+                    and(
+                      eq(bookmarksInLists.bookmarkId, bookmarks.id),
+                      eq(bookmarksInLists.listId, input.listId),
+                    ),
+                  ),
+              )
+              : undefined,
+          ),
+        )
+        .leftJoin(tagsOnBookmarks, eq(bookmarks.id, tagsOnBookmarks.bookmarkId))
+        .leftJoin(bookmarkTags, eq(tagsOnBookmarks.tagId, bookmarkTags.id))
+        .leftJoin(bookmarkLinks, eq(bookmarkLinks.id, bookmarks.id))
+        .leftJoin(bookmarkTexts, eq(bookmarkTexts.id, bookmarks.id))
+        .orderBy(desc(bookmarks.createdAt));
+
+      const bookmarksRes = results.reduce<Record<string, ZBookmark>>(
+        (acc, row) => {
+          const bookmarkId = row.bookmarks.id;
+          if (!acc[bookmarkId]) {
+            let content: ZBookmarkContent;
+            if (row.bookmarkLinks) {
+              content = { type: "link", ...row.bookmarkLinks };
+            } else if (row.bookmarkTexts) {
+              content = { type: "text", text: row.bookmarkTexts.text ?? "" };
+            } else {
+              throw new Error("Unknown content type");
+            }
+            acc[bookmarkId] = {
+              ...row.bookmarks,
+              content,
+              tags: [],
+            };
+          }
+
+          if (row.bookmarkTags) {
+            invariant(
+              row.tagsOnBookmarks,
+              "if bookmark tag is set, its many-to-many relation must also be set",
+            );
+            acc[bookmarkId].tags.push({
+              ...row.bookmarkTags,
+              attachedBy: row.tagsOnBookmarks.attachedBy,
+            });
+          }
+
+          return acc;
         },
-      });
+        {},
+      );
 
-      return { bookmarks: results.map(toZodSchema) };
+      return { bookmarks: Object.values(bookmarksRes) };
     }),
 
   updateTags: authedProcedure
@@ -442,7 +507,7 @@ export const bookmarksAppRouter = router({
           .insert(tagsOnBookmarks)
           .values(
             allIds.map((i) => ({
-              tagId: i as string,
+              tagId: i,
               bookmarkId: input.bookmarkId,
               attachedBy: "human" as const,
               userId: ctx.user.id,
diff --git a/packages/trpc/types/bookmarks.ts b/packages/trpc/types/bookmarks.ts
index b61ab0e0..e23d6b4b 100644
--- a/packages/trpc/types/bookmarks.ts
+++ b/packages/trpc/types/bookmarks.ts
@@ -51,6 +51,8 @@ export const zGetBookmarksRequestSchema = z.object({
   ids: z.array(z.string()).optional(),
   archived: z.boolean().optional(),
   favourited: z.boolean().optional(),
+  tagId: z.string().optional(),
+  listId: z.string().optional(),
 });
 export type ZGetBookmarksRequest = z.infer<typeof zGetBookmarksRequestSchema>;
 
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 696c318d..75e23664 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -616,6 +616,9 @@ importers:
       superjson:
         specifier: ^2.2.1
         version: 2.2.1
+      tiny-invariant:
+        specifier: ^1.3.3
+        version: 1.3.3
       zod:
         specifier: ^3.22.4
         version: 3.22.4
@@ -16160,6 +16163,10 @@ packages:
       next-tick: 1.1.0
     dev: true
 
+  /tiny-invariant@1.3.3:
+    resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==}
+    dev: false
+
   /tinybench@2.6.0:
     resolution: {integrity: sha512-N8hW3PG/3aOoZAN5V/NSAEDz0ZixDSSt5b/a05iqtpgfLWMSVuCo7w0k2vVvEjdrIoeGqZzweX2WlyioNIHchA==}
     dev: true