fix: Dont download html content by default in the bookmark grid. Fixes #1198

14 files changed, 112 insertions, 36 deletions

diff --git a/apps/mobile/app/dashboard/bookmarks/[slug]/info.tsx b/apps/mobile/app/dashboard/bookmarks/[slug]/info.tsx
index ebd3a1e2..6d02af53 100644
--- a/apps/mobile/app/dashboard/bookmarks/[slug]/info.tsx
+++ b/apps/mobile/app/dashboard/bookmarks/[slug]/info.tsx
@@ -147,7 +147,9 @@ const ViewBookmarkPage = () => {
     data: bookmark,
     isPending,
     refetch,
-  } = useAutoRefreshingBookmarkQuery({ bookmarkId: slug });
+  } = useAutoRefreshingBookmarkQuery({
+    bookmarkId: slug,
+  });
 
   if (isPending) {
     return <FullPageSpinner />;
diff --git a/apps/mobile/components/bookmarks/UpdatingBookmarkList.tsx b/apps/mobile/components/bookmarks/UpdatingBookmarkList.tsx
index 115d44c9..e627ee16 100644
--- a/apps/mobile/components/bookmarks/UpdatingBookmarkList.tsx
+++ b/apps/mobile/components/bookmarks/UpdatingBookmarkList.tsx
@@ -11,7 +11,7 @@ export default function UpdatingBookmarkList({
   query,
   header,
 }: {
-  query: Omit<ZGetBookmarksRequest, "sortOrder">; // Sort order is not supported in mobile yet
+  query: Omit<ZGetBookmarksRequest, "sortOrder" | "includeContent">; // Sort order is not supported in mobile yet
   header?: React.ReactElement;
 }) {
   const apiUtils = api.useUtils();
@@ -24,7 +24,7 @@ export default function UpdatingBookmarkList({
     isFetchingNextPage,
     refetch,
   } = api.bookmarks.getBookmarks.useInfiniteQuery(
-    { ...query, useCursorV2: true },
+    { ...query, useCursorV2: true, includeContent: false },
     {
       initialCursor: null,
       getNextPageParam: (lastPage) => lastPage.nextCursor,
diff --git a/apps/web/app/api/bookmarks/export/route.tsx b/apps/web/app/api/bookmarks/export/route.tsx
index 4e04757f..669fb1c7 100644
--- a/apps/web/app/api/bookmarks/export/route.tsx
+++ b/apps/web/app/api/bookmarks/export/route.tsx
@@ -13,6 +13,7 @@ export async function GET(request: Request) {
   const req = {
     limit: MAX_NUM_BOOKMARKS_PER_PAGE,
     useCursorV2: true,
+    includeContent: true,
   };
 
   let resp = await api.bookmarks.getBookmarks(req);
diff --git a/apps/web/app/api/v1/bookmarks/[bookmarkId]/route.ts b/apps/web/app/api/v1/bookmarks/[bookmarkId]/route.ts
index db78f17c..fa551894 100644
--- a/apps/web/app/api/v1/bookmarks/[bookmarkId]/route.ts
+++ b/apps/web/app/api/v1/bookmarks/[bookmarkId]/route.ts
@@ -3,6 +3,8 @@ import { buildHandler } from "@/app/api/v1/utils/handler";
 
 import { zUpdateBookmarksRequestSchema } from "@karakeep/shared/types/bookmarks";
 
+import { zGetBookmarkSearchParamsSchema } from "../../utils/types";
+
 export const dynamic = "force-dynamic";
 
 export const GET = (
@@ -11,9 +13,11 @@ export const GET = (
 ) =>
   buildHandler({
     req,
-    handler: async ({ api }) => {
+    searchParamsSchema: zGetBookmarkSearchParamsSchema,
+    handler: async ({ api, searchParams }) => {
       const bookmark = await api.bookmarks.getBookmark({
         bookmarkId: params.bookmarkId,
+        includeContent: searchParams.includeContent,
       });
       return { status: 200, resp: bookmark };
     },
diff --git a/apps/web/app/api/v1/bookmarks/route.ts b/apps/web/app/api/v1/bookmarks/route.ts
index 7b6954c6..1605d2b5 100644
--- a/apps/web/app/api/v1/bookmarks/route.ts
+++ b/apps/web/app/api/v1/bookmarks/route.ts
@@ -16,6 +16,8 @@ export const GET = (req: NextRequest) =>
       .object({
         favourited: zStringBool.optional(),
         archived: zStringBool.optional(),
+        // TODO: Change the default to false in a couple of releases.
+        includeContent: zStringBool.optional().default("true"),
       })
       .and(zPagination),
     handler: async ({ api, searchParams }) => {
diff --git a/apps/web/app/api/v1/lists/[listId]/bookmarks/route.ts b/apps/web/app/api/v1/lists/[listId]/bookmarks/route.ts
index 72d4aa5f..3977413a 100644
--- a/apps/web/app/api/v1/lists/[listId]/bookmarks/route.ts
+++ b/apps/web/app/api/v1/lists/[listId]/bookmarks/route.ts
@@ -1,13 +1,14 @@
 import { NextRequest } from "next/server";
 import { buildHandler } from "@/app/api/v1/utils/handler";
 import { adaptPagination, zPagination } from "@/app/api/v1/utils/pagination";
+import { zGetBookmarkSearchParamsSchema } from "@/app/api/v1/utils/types";
 
 export const dynamic = "force-dynamic";
 
 export const GET = (req: NextRequest, params: { params: { listId: string } }) =>
   buildHandler({
     req,
-    searchParamsSchema: zPagination,
+    searchParamsSchema: zPagination.and(zGetBookmarkSearchParamsSchema),
     handler: async ({ api, searchParams }) => {
       const bookmarks = await api.bookmarks.getBookmarks({
         listId: params.params.listId,
diff --git a/apps/web/app/api/v1/tags/[tagId]/bookmarks/route.ts b/apps/web/app/api/v1/tags/[tagId]/bookmarks/route.ts
index 98133ec7..cfc0af51 100644
--- a/apps/web/app/api/v1/tags/[tagId]/bookmarks/route.ts
+++ b/apps/web/app/api/v1/tags/[tagId]/bookmarks/route.ts
@@ -1,6 +1,7 @@
 import { NextRequest } from "next/server";
 import { buildHandler } from "@/app/api/v1/utils/handler";
 import { adaptPagination, zPagination } from "@/app/api/v1/utils/pagination";
+import { zGetBookmarkSearchParamsSchema } from "@/app/api/v1/utils/types";
 
 export const dynamic = "force-dynamic";
 
@@ -10,7 +11,7 @@ export const GET = (
 ) =>
   buildHandler({
     req,
-    searchParamsSchema: zPagination,
+    searchParamsSchema: zPagination.and(zGetBookmarkSearchParamsSchema),
     handler: async ({ api, searchParams }) => {
       const bookmarks = await api.bookmarks.getBookmarks({
         tagId: params.tagId,
diff --git a/apps/web/app/api/v1/utils/types.ts b/apps/web/app/api/v1/utils/types.ts
index c0e20dff..f0fe6231 100644
--- a/apps/web/app/api/v1/utils/types.ts
+++ b/apps/web/app/api/v1/utils/types.ts
@@ -4,3 +4,8 @@ export const zStringBool = z
   .string()
   .refine((val) => val === "true" || val === "false", "Must be true or false")
   .transform((val) => val === "true");
+
+export const zGetBookmarkSearchParamsSchema = z.object({
+  // TODO: Change the default to false in a couple of releases.
+  includeContent: zStringBool.optional().default("true"),
+});
diff --git a/apps/web/components/dashboard/bookmarks/Bookmarks.tsx b/apps/web/components/dashboard/bookmarks/Bookmarks.tsx
index af2e4990..9f7a900e 100644
--- a/apps/web/components/dashboard/bookmarks/Bookmarks.tsx
+++ b/apps/web/components/dashboard/bookmarks/Bookmarks.tsx
@@ -13,7 +13,7 @@ export default async function Bookmarks({
   showDivider,
   showEditorCard = false,
 }: {
-  query: Omit<ZGetBookmarksRequest, "sortOrder">; // Sort order is handled by the store
+  query: Omit<ZGetBookmarksRequest, "sortOrder" | "includeContent">; // Sort order is handled by the store
   header?: React.ReactNode;
   showDivider?: boolean;
   showEditorCard?: boolean;
@@ -23,7 +23,9 @@ export default async function Bookmarks({
     redirect("/");
   }
 
-  const bookmarks = await api.bookmarks.getBookmarks(query);
+  const bookmarks = await api.bookmarks.getBookmarks({
+    ...query,
+  });
 
   return (
     <div className="flex flex-col gap-3">
diff --git a/apps/web/components/dashboard/bookmarks/UpdatableBookmarksGrid.tsx b/apps/web/components/dashboard/bookmarks/UpdatableBookmarksGrid.tsx
index 03ea9708..da65b9d9 100644
--- a/apps/web/components/dashboard/bookmarks/UpdatableBookmarksGrid.tsx
+++ b/apps/web/components/dashboard/bookmarks/UpdatableBookmarksGrid.tsx
@@ -18,14 +18,14 @@ export default function UpdatableBookmarksGrid({
   bookmarks: initialBookmarks,
   showEditorCard = false,
 }: {
-  query: Omit<ZGetBookmarksRequest, "sortOrder">; // Sort order is handled by the store
+  query: Omit<ZGetBookmarksRequest, "sortOrder" | "includeContent">; // Sort order is handled by the store
   bookmarks: ZGetBookmarksResponse;
   showEditorCard?: boolean;
   itemsPerPage?: number;
 }) {
   const sortOrder = useSortOrderStore((state) => state.sortOrder);
 
-  const finalQuery = { ...query, sortOrder };
+  const finalQuery = { ...query, sortOrder, includeContent: false };
 
   const { data, fetchNextPage, hasNextPage, isFetchingNextPage, refetch } =
     api.bookmarks.getBookmarks.useInfiniteQuery(
diff --git a/apps/web/components/dashboard/bookmarks/action-buttons/ArchiveBookmarkButton.tsx b/apps/web/components/dashboard/bookmarks/action-buttons/ArchiveBookmarkButton.tsx
index 3b8da82f..d45cfc82 100644
--- a/apps/web/components/dashboard/bookmarks/action-buttons/ArchiveBookmarkButton.tsx
+++ b/apps/web/components/dashboard/bookmarks/action-buttons/ArchiveBookmarkButton.tsx
@@ -15,7 +15,14 @@ const ArchiveBookmarkButton = React.forwardRef<
   HTMLButtonElement,
   ArchiveBookmarkButtonProps
 >(({ bookmarkId, onDone, ...props }, ref) => {
-  const { data } = api.bookmarks.getBookmark.useQuery({ bookmarkId });
+  const { data } = api.bookmarks.getBookmark.useQuery(
+    { bookmarkId },
+    {
+      select: (data) => ({
+        archived: data.archived,
+      }),
+    },
+  );
 
   const { mutate: updateBookmark, isPending: isArchivingBookmark } =
     useUpdateBookmark({
diff --git a/apps/web/components/dashboard/preview/LinkContentSection.tsx b/apps/web/components/dashboard/preview/LinkContentSection.tsx
index f37f110e..dd419fcd 100644
--- a/apps/web/components/dashboard/preview/LinkContentSection.tsx
+++ b/apps/web/components/dashboard/preview/LinkContentSection.tsx
@@ -1,6 +1,7 @@
 import { useState } from "react";
 import Image from "next/image";
 import BookmarkHTMLHighlighter from "@/components/dashboard/preview/BookmarkHtmlHighlighter";
+import { FullPageSpinner } from "@/components/ui/full-page-spinner";
 import {
   Select,
   SelectContent,
@@ -52,16 +53,23 @@ function ScreenshotSection({ link }: { link: ZBookmarkedLink }) {
   );
 }
 
-function CachedContentSection({
-  bookmarkId,
-  link,
-}: {
-  bookmarkId: string;
-  link: ZBookmarkedLink;
-}) {
-  const { data } = api.highlights.getForBookmark.useQuery({
+function CachedContentSection({ bookmarkId }: { bookmarkId: string }) {
+  const { data: highlights } = api.highlights.getForBookmark.useQuery({
     bookmarkId,
   });
+  const { data: cachedContent, isPending: isCachedContentLoading } =
+    api.bookmarks.getBookmark.useQuery(
+      {
+        bookmarkId,
+        includeContent: true,
+      },
+      {
+        select: (data) =>
+          data.content.type == BookmarkTypes.LINK
+            ? data.content.htmlContent
+            : null,
+      },
+    );
 
   const { mutate: createHighlight } = useCreateHighlight({
     onSuccess: () => {
@@ -106,16 +114,18 @@ function CachedContentSection({
   });
 
   let content;
-  if (!link.htmlContent) {
+  if (isCachedContentLoading) {
+    content = <FullPageSpinner />;
+  } else if (!cachedContent) {
     content = (
       <div className="text-destructive">Failed to fetch link content ...</div>
     );
   } else {
     content = (
       <BookmarkHTMLHighlighter
-        htmlContent={link.htmlContent || ""}
+        htmlContent={cachedContent || ""}
         className="prose mx-auto dark:prose-invert"
-        highlights={data?.highlights ?? []}
+        highlights={highlights?.highlights ?? []}
         onDeleteHighlight={(h) =>
           deleteHighlight({
             highlightId: h.id,
@@ -171,9 +181,7 @@ export default function LinkContentSection({
 
   let content;
   if (section === "cached") {
-    content = (
-      <CachedContentSection bookmarkId={bookmark.id} link={bookmark.content} />
-    );
+    content = <CachedContentSection bookmarkId={bookmark.id} />;
   } else if (section === "archive") {
     content = <FullPageArchiveSection link={bookmark.content} />;
   } else if (section === "video") {
diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts
index 25c40bbc..8df0e375 100644
--- a/packages/shared/types/bookmarks.ts
+++ b/packages/shared/types/bookmarks.ts
@@ -179,6 +179,7 @@ export const zGetBookmarksRequestSchema = z.object({
   // servers.
   useCursorV2: z.boolean().optional(),
   sortOrder: zSortOrder.optional().default("desc"),
+  includeContent: z.boolean().optional().default(false),
 });
 export type ZGetBookmarksRequest = z.infer<typeof zGetBookmarksRequestSchema>;
 
diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts
index 7ad98b37..a02fb691 100644
--- a/packages/trpc/routers/bookmarks.ts
+++ b/packages/trpc/routers/bookmarks.ts
@@ -103,7 +103,11 @@ export const ensureBookmarkOwnership = experimental_trpcMiddleware<{
   return opts.next();
 });
 
-async function getBookmark(ctx: AuthedContext, bookmarkId: string) {
+async function getBookmark(
+  ctx: AuthedContext,
+  bookmarkId: string,
+  includeContent: boolean,
+) {
   const bookmark = await ctx.db.query.bookmarks.findFirst({
     where: and(eq(bookmarks.userId, ctx.user.id), eq(bookmarks.id, bookmarkId)),
     with: {
@@ -125,7 +129,7 @@ async function getBookmark(ctx: AuthedContext, bookmarkId: string) {
     });
   }
 
-  return toZodSchema(bookmark);
+  return toZodSchema(bookmark, includeContent);
 }
 
 async function attemptToDedupLink(ctx: AuthedContext, url: string) {
@@ -140,7 +144,7 @@ async function attemptToDedupLink(ctx: AuthedContext, url: string) {
   if (result.length == 0) {
     return null;
   }
-  return getBookmark(ctx, result[0].id);
+  return getBookmark(ctx, result[0].id, /* includeContent: */ false);
 }
 
 async function dummyDrizzleReturnType() {
@@ -184,7 +188,10 @@ async function cleanupAssetForBookmark(
   );
 }
 
-function toZodSchema(bookmark: BookmarkQueryReturnType): ZBookmark {
+function toZodSchema(
+  bookmark: BookmarkQueryReturnType,
+  includeContent: boolean,
+): ZBookmark {
   const { tagsOnBookmarks, link, text, asset, assets, ...rest } = bookmark;
 
   let content: ZBookmarkContent = {
@@ -207,12 +214,23 @@ function toZodSchema(bookmark: BookmarkQueryReturnType): ZBookmark {
       )?.id,
       videoAssetId: assets.find((a) => a.assetType == AssetTypes.LINK_VIDEO)
         ?.id,
-      ...link,
+      url: link.url,
+      title: link.title,
+      description: link.description,
+      imageUrl: link.imageUrl,
+      favicon: link.favicon,
+      htmlContent: includeContent ? link.htmlContent : null,
+      crawledAt: link.crawledAt,
+      author: link.author,
+      publisher: link.publisher,
+      datePublished: link.datePublished,
+      dateModified: link.dateModified,
     };
   }
   if (bookmark.text) {
     content = {
       type: BookmarkTypes.TEXT,
+      // It's ok to include the text content as it's usually not big and is used to render the text bookmark card.
       text: text.text ?? "",
       sourceUrl: text.sourceUrl,
     };
@@ -225,7 +243,7 @@ function toZodSchema(bookmark: BookmarkQueryReturnType): ZBookmark {
       fileName: asset.fileName,
       sourceUrl: asset.sourceUrl,
       size: assets.find((a) => a.id == asset.assetId)?.size,
-      content: asset.content,
+      content: includeContent ? asset.content : null,
     };
   }
 
@@ -549,7 +567,11 @@ export const bookmarksAppRouter = router({
       });
 
       // Refetch the updated bookmark data to return the full object
-      const updatedBookmark = await getBookmark(ctx, input.bookmarkId);
+      const updatedBookmark = await getBookmark(
+        ctx,
+        input.bookmarkId,
+        /* includeContent: */ false,
+      );
 
       // Trigger re-indexing and webhooks
       await triggerSearchReindex(input.bookmarkId);
@@ -653,12 +675,13 @@ export const bookmarksAppRouter = router({
     .input(
       z.object({
         bookmarkId: z.string(),
+        includeContent: z.boolean().optional().default(false),
       }),
     )
     .output(zBookmarkSchema)
     .use(ensureBookmarkOwnership)
     .query(async ({ input, ctx }) => {
-      return await getBookmark(ctx, input.bookmarkId);
+      return await getBookmark(ctx, input.bookmarkId, input.includeContent);
     }),
   searchBookmarks: authedProcedure
     .input(zSearchBookmarksRequestSchema)
@@ -738,7 +761,9 @@ export const bookmarksAppRouter = router({
       results.sort((a, b) => idToRank[b.id] - idToRank[a.id]);
 
       return {
-        bookmarks: results.map(toZodSchema),
+        bookmarks: results.map((b) =>
+          toZodSchema(b, /* includeContent: */ false),
+        ),
         nextCursor:
           resp.hits.length + resp.offset >= resp.estimatedTotalHits
             ? null
@@ -865,7 +890,22 @@ export const bookmarksAppRouter = router({
           if (!acc[bookmarkId]) {
             let content: ZBookmarkContent;
             if (row.bookmarkLinks) {
-              content = { type: BookmarkTypes.LINK, ...row.bookmarkLinks };
+              content = {
+                type: BookmarkTypes.LINK,
+                url: row.bookmarkLinks.url,
+                title: row.bookmarkLinks.title,
+                description: row.bookmarkLinks.description,
+                imageUrl: row.bookmarkLinks.imageUrl,
+                favicon: row.bookmarkLinks.favicon,
+                htmlContent: input.includeContent
+                  ? row.bookmarkLinks.htmlContent
+                  : null,
+                crawledAt: row.bookmarkLinks.crawledAt,
+                author: row.bookmarkLinks.author,
+                publisher: row.bookmarkLinks.publisher,
+                datePublished: row.bookmarkLinks.datePublished,
+                dateModified: row.bookmarkLinks.dateModified,
+              };
             } else if (row.bookmarkTexts) {
               content = {
                 type: BookmarkTypes.TEXT,
@@ -880,7 +920,9 @@ export const bookmarksAppRouter = router({
                 fileName: row.bookmarkAssets.fileName,
                 sourceUrl: row.bookmarkAssets.sourceUrl ?? null,
                 size: null, // This will get filled in the asset loop
-                content: row.bookmarkAssets.content ?? null,
+                content: input.includeContent
+                  ? (row.bookmarkAssets.content ?? null)
+                  : null,
               };
             } else {
               content = {