feat: Generate RSS feeds from lists (#1507)

* refactor: Move bookmark utils from shared-react to shared

* Expose RSS feeds for lists

* Add e2e tests

* Slightly improve the look of the share dialog

* allow specifying a limit in the rss endpoint

2 files changed, 462 insertions, 2 deletions

diff --git a/packages/trpc/models/bookmarks.ts b/packages/trpc/models/bookmarks.ts
new file mode 100644
index 00000000..524749f9
--- /dev/null
+++ b/packages/trpc/models/bookmarks.ts
@@ -0,0 +1,365 @@
+import { TRPCError } from "@trpc/server";
+import {
+  and,
+  asc,
+  desc,
+  eq,
+  exists,
+  gt,
+  gte,
+  inArray,
+  lt,
+  lte,
+  or,
+} from "drizzle-orm";
+import invariant from "tiny-invariant";
+import { z } from "zod";
+
+import {
+  assets,
+  AssetTypes,
+  bookmarkAssets,
+  bookmarkLinks,
+  bookmarks,
+  bookmarksInLists,
+  bookmarkTags,
+  bookmarkTexts,
+  rssFeedImportsTable,
+  tagsOnBookmarks,
+} from "@karakeep/db/schema";
+import {
+  BookmarkTypes,
+  DEFAULT_NUM_BOOKMARKS_PER_PAGE,
+  ZBookmark,
+  ZBookmarkContent,
+  zGetBookmarksRequestSchema,
+  ZPublicBookmark,
+} from "@karakeep/shared/types/bookmarks";
+import { ZCursor } from "@karakeep/shared/types/pagination";
+import { getBookmarkTitle } from "@karakeep/shared/utils/bookmarkUtils";
+
+import { AuthedContext } from "..";
+import { mapDBAssetTypeToUserType } from "../lib/attachments";
+import { List } from "./lists";
+import { PrivacyAware } from "./privacy";
+
+export class Bookmark implements PrivacyAware {
+  protected constructor(
+    protected ctx: AuthedContext,
+    public bookmark: ZBookmark & { userId: string },
+  ) {}
+
+  ensureCanAccess(ctx: AuthedContext): void {
+    if (this.bookmark.userId != ctx.user.id) {
+      throw new TRPCError({
+        code: "FORBIDDEN",
+        message: "User is not allowed to access resource",
+      });
+    }
+  }
+
+  static fromData(ctx: AuthedContext, data: ZBookmark) {
+    return new Bookmark(ctx, {
+      ...data,
+      userId: ctx.user.id,
+    });
+  }
+
+  static async loadMulti(
+    ctx: AuthedContext,
+    input: z.infer<typeof zGetBookmarksRequestSchema>,
+  ): Promise<{
+    bookmarks: Bookmark[];
+    nextCursor: ZCursor | null;
+  }> {
+    if (input.ids && input.ids.length == 0) {
+      return { bookmarks: [], nextCursor: null };
+    }
+    if (!input.limit) {
+      input.limit = DEFAULT_NUM_BOOKMARKS_PER_PAGE;
+    }
+    if (input.listId) {
+      const list = await List.fromId(ctx, input.listId);
+      if (list.type === "smart") {
+        input.ids = await list.getBookmarkIds();
+        delete input.listId;
+      }
+    }
+
+    const sq = ctx.db.$with("bookmarksSq").as(
+      ctx.db
+        .select()
+        .from(bookmarks)
+        .where(
+          and(
+            eq(bookmarks.userId, ctx.user.id),
+            input.archived !== undefined
+              ? eq(bookmarks.archived, input.archived)
+              : undefined,
+            input.favourited !== undefined
+              ? eq(bookmarks.favourited, input.favourited)
+              : undefined,
+            input.ids ? inArray(bookmarks.id, input.ids) : undefined,
+            input.tagId !== undefined
+              ? exists(
+                  ctx.db
+                    .select()
+                    .from(tagsOnBookmarks)
+                    .where(
+                      and(
+                        eq(tagsOnBookmarks.bookmarkId, bookmarks.id),
+                        eq(tagsOnBookmarks.tagId, input.tagId),
+                      ),
+                    ),
+                )
+              : undefined,
+            input.rssFeedId !== undefined
+              ? exists(
+                  ctx.db
+                    .select()
+                    .from(rssFeedImportsTable)
+                    .where(
+                      and(
+                        eq(rssFeedImportsTable.bookmarkId, bookmarks.id),
+                        eq(rssFeedImportsTable.rssFeedId, input.rssFeedId),
+                      ),
+                    ),
+                )
+              : undefined,
+            input.listId !== undefined
+              ? exists(
+                  ctx.db
+                    .select()
+                    .from(bookmarksInLists)
+                    .where(
+                      and(
+                        eq(bookmarksInLists.bookmarkId, bookmarks.id),
+                        eq(bookmarksInLists.listId, input.listId),
+                      ),
+                    ),
+                )
+              : undefined,
+            input.cursor
+              ? input.sortOrder === "asc"
+                ? or(
+                    gt(bookmarks.createdAt, input.cursor.createdAt),
+                    and(
+                      eq(bookmarks.createdAt, input.cursor.createdAt),
+                      gte(bookmarks.id, input.cursor.id),
+                    ),
+                  )
+                : or(
+                    lt(bookmarks.createdAt, input.cursor.createdAt),
+                    and(
+                      eq(bookmarks.createdAt, input.cursor.createdAt),
+                      lte(bookmarks.id, input.cursor.id),
+                    ),
+                  )
+              : undefined,
+          ),
+        )
+        .limit(input.limit + 1)
+        .orderBy(
+          input.sortOrder === "asc"
+            ? asc(bookmarks.createdAt)
+            : desc(bookmarks.createdAt),
+          desc(bookmarks.id),
+        ),
+    );
+    // TODO: Consider not inlining the tags in the response of getBookmarks as this query is getting kinda expensive
+    const results = await ctx.db
+      .with(sq)
+      .select()
+      .from(sq)
+      .leftJoin(tagsOnBookmarks, eq(sq.id, tagsOnBookmarks.bookmarkId))
+      .leftJoin(bookmarkTags, eq(tagsOnBookmarks.tagId, bookmarkTags.id))
+      .leftJoin(bookmarkLinks, eq(bookmarkLinks.id, sq.id))
+      .leftJoin(bookmarkTexts, eq(bookmarkTexts.id, sq.id))
+      .leftJoin(bookmarkAssets, eq(bookmarkAssets.id, sq.id))
+      .leftJoin(assets, eq(assets.bookmarkId, sq.id))
+      .orderBy(desc(sq.createdAt), desc(sq.id));
+
+    const bookmarksRes = results.reduce<Record<string, ZBookmark>>(
+      (acc, row) => {
+        const bookmarkId = row.bookmarksSq.id;
+        if (!acc[bookmarkId]) {
+          let content: ZBookmarkContent;
+          if (row.bookmarkLinks) {
+            content = {
+              type: BookmarkTypes.LINK,
+              url: row.bookmarkLinks.url,
+              title: row.bookmarkLinks.title,
+              description: row.bookmarkLinks.description,
+              imageUrl: row.bookmarkLinks.imageUrl,
+              favicon: row.bookmarkLinks.favicon,
+              htmlContent: input.includeContent
+                ? row.bookmarkLinks.htmlContent
+                : null,
+              crawledAt: row.bookmarkLinks.crawledAt,
+              author: row.bookmarkLinks.author,
+              publisher: row.bookmarkLinks.publisher,
+              datePublished: row.bookmarkLinks.datePublished,
+              dateModified: row.bookmarkLinks.dateModified,
+            };
+          } else if (row.bookmarkTexts) {
+            content = {
+              type: BookmarkTypes.TEXT,
+              text: row.bookmarkTexts.text ?? "",
+              sourceUrl: row.bookmarkTexts.sourceUrl ?? null,
+            };
+          } else if (row.bookmarkAssets) {
+            content = {
+              type: BookmarkTypes.ASSET,
+              assetId: row.bookmarkAssets.assetId,
+              assetType: row.bookmarkAssets.assetType,
+              fileName: row.bookmarkAssets.fileName,
+              sourceUrl: row.bookmarkAssets.sourceUrl ?? null,
+              size: null, // This will get filled in the asset loop
+              content: input.includeContent
+                ? (row.bookmarkAssets.content ?? null)
+                : null,
+            };
+          } else {
+            content = {
+              type: BookmarkTypes.UNKNOWN,
+            };
+          }
+          acc[bookmarkId] = {
+            ...row.bookmarksSq,
+            content,
+            tags: [],
+            assets: [],
+          };
+        }
+
+        if (
+          row.bookmarkTags &&
+          // Duplicates may occur because of the join, so we need to make sure we're not adding the same tag twice
+          !acc[bookmarkId].tags.some((t) => t.id == row.bookmarkTags!.id)
+        ) {
+          invariant(
+            row.tagsOnBookmarks,
+            "if bookmark tag is set, its many-to-many relation must also be set",
+          );
+          acc[bookmarkId].tags.push({
+            ...row.bookmarkTags,
+            attachedBy: row.tagsOnBookmarks.attachedBy,
+          });
+        }
+
+        if (
+          row.assets &&
+          !acc[bookmarkId].assets.some((a) => a.id == row.assets!.id)
+        ) {
+          if (acc[bookmarkId].content.type == BookmarkTypes.LINK) {
+            const content = acc[bookmarkId].content;
+            invariant(content.type == BookmarkTypes.LINK);
+            if (row.assets.assetType == AssetTypes.LINK_SCREENSHOT) {
+              content.screenshotAssetId = row.assets.id;
+            }
+            if (row.assets.assetType == AssetTypes.LINK_FULL_PAGE_ARCHIVE) {
+              content.fullPageArchiveAssetId = row.assets.id;
+            }
+            if (row.assets.assetType == AssetTypes.LINK_BANNER_IMAGE) {
+              content.imageAssetId = row.assets.id;
+            }
+            if (row.assets.assetType == AssetTypes.LINK_VIDEO) {
+              content.videoAssetId = row.assets.id;
+            }
+            if (row.assets.assetType == AssetTypes.LINK_PRECRAWLED_ARCHIVE) {
+              content.precrawledArchiveAssetId = row.assets.id;
+            }
+            acc[bookmarkId].content = content;
+          }
+          if (acc[bookmarkId].content.type == BookmarkTypes.ASSET) {
+            const content = acc[bookmarkId].content;
+            if (row.assets.id == content.assetId) {
+              // If this is the bookmark's main aset, caputure its size.
+              content.size = row.assets.size;
+            }
+          }
+          acc[bookmarkId].assets.push({
+            id: row.assets.id,
+            assetType: mapDBAssetTypeToUserType(row.assets.assetType),
+          });
+        }
+
+        return acc;
+      },
+      {},
+    );
+
+    const bookmarksArr = Object.values(bookmarksRes);
+
+    bookmarksArr.sort((a, b) => {
+      if (a.createdAt != b.createdAt) {
+        return input.sortOrder === "asc"
+          ? a.createdAt.getTime() - b.createdAt.getTime()
+          : b.createdAt.getTime() - a.createdAt.getTime();
+      } else {
+        return b.id.localeCompare(a.id);
+      }
+    });
+
+    let nextCursor = null;
+    if (bookmarksArr.length > input.limit) {
+      const nextItem = bookmarksArr.pop()!;
+      nextCursor = {
+        id: nextItem.id,
+        createdAt: nextItem.createdAt,
+      };
+    }
+
+    return {
+      bookmarks: bookmarksArr.map((b) => Bookmark.fromData(ctx, b)),
+      nextCursor,
+    };
+  }
+
+  asZBookmark(): ZBookmark {
+    return this.bookmark;
+  }
+
+  asPublicBookmark(): ZPublicBookmark {
+    const getContent = (
+      content: ZBookmarkContent,
+    ): ZPublicBookmark["content"] => {
+      switch (content.type) {
+        case BookmarkTypes.LINK: {
+          return {
+            type: BookmarkTypes.LINK,
+            url: content.url,
+          };
+        }
+        case BookmarkTypes.TEXT: {
+          return {
+            type: BookmarkTypes.TEXT,
+            text: content.text,
+          };
+        }
+        case BookmarkTypes.ASSET: {
+          return {
+            type: BookmarkTypes.ASSET,
+            assetType: content.assetType,
+            assetId: content.assetId,
+            fileName: content.fileName,
+            sourceUrl: content.sourceUrl,
+          };
+        }
+        default: {
+          throw new Error("Unknown bookmark content type");
+        }
+      }
+    };
+
+    // WARNING: Everything below is exposed in the public APIs, don't use spreads!
+    return {
+      id: this.bookmark.id,
+      createdAt: this.bookmark.createdAt,
+      modifiedAt: this.bookmark.modifiedAt,
+      title: getBookmarkTitle(this.bookmark),
+      tags: this.bookmark.tags.map((t) => t.name),
+      content: getContent(this.bookmark.content),
+    };
+  }
+}
diff --git a/packages/trpc/models/lists.ts b/packages/trpc/models/lists.ts
index 21b23593..4413a8cd 100644
--- a/packages/trpc/models/lists.ts
+++ b/packages/trpc/models/lists.ts
@@ -1,3 +1,4 @@
+import crypto from "node:crypto";
 import { TRPCError } from "@trpc/server";
 import { and, count, eq } from "drizzle-orm";
 import invariant from "tiny-invariant";
@@ -13,8 +14,10 @@ import {
   zNewBookmarkListSchema,
 } from "@karakeep/shared/types/lists";
 
-import { AuthedContext } from "..";
+import { AuthedContext, Context } from "..";
+import { buildImpersonatingAuthedContext } from "../lib/impersonate";
 import { getBookmarkIdsFromMatcher } from "../lib/search";
+import { Bookmark } from "./bookmarks";
 import { PrivacyAware } from "./privacy";
 
 export abstract class List implements PrivacyAware {
@@ -58,6 +61,52 @@ export abstract class List implements PrivacyAware {
     }
   }
 
+  static async getForRss(
+    ctx: Context,
+    listId: string,
+    token: string,
+    pagination: {
+      limit: number;
+    },
+  ) {
+    const listdb = await ctx.db.query.bookmarkLists.findFirst({
+      where: and(
+        eq(bookmarkLists.id, listId),
+        eq(bookmarkLists.rssToken, token),
+      ),
+    });
+    if (!listdb) {
+      throw new TRPCError({
+        code: "NOT_FOUND",
+        message: "List not found",
+      });
+    }
+
+    // The token here acts as an authed context, so we can create
+    // an impersonating context for the list owner as long as
+    // we don't leak the context.
+
+    const authedCtx = await buildImpersonatingAuthedContext(listdb.userId);
+    const list = List.fromData(authedCtx, listdb);
+    const bookmarkIds = await list.getBookmarkIds();
+
+    const bookmarks = await Bookmark.loadMulti(authedCtx, {
+      ids: bookmarkIds,
+      includeContent: false,
+      limit: pagination.limit,
+      sortOrder: "desc",
+    });
+
+    return {
+      list: {
+        icon: list.list.icon,
+        name: list.list.name,
+        description: list.list.description,
+      },
+      bookmarks: bookmarks.bookmarks.map((b) => b.asPublicBookmark()),
+    };
+  }
+
   static async create(
     ctx: AuthedContext,
     input: z.infer<typeof zNewBookmarkListSchema>,
@@ -79,6 +128,9 @@ export abstract class List implements PrivacyAware {
 
   static async getAll(ctx: AuthedContext): Promise<(ManualList | SmartList)[]> {
     const lists = await ctx.db.query.bookmarkLists.findMany({
+      columns: {
+        rssToken: false,
+      },
       where: and(eq(bookmarkLists.userId, ctx.user.id)),
     });
     return lists.map((l) => this.fromData(ctx, l));
@@ -88,7 +140,11 @@ export abstract class List implements PrivacyAware {
     const lists = await ctx.db.query.bookmarksInLists.findMany({
       where: and(eq(bookmarksInLists.bookmarkId, bookmarkId)),
       with: {
-        list: true,
+        list: {
+          columns: {
+            rssToken: false,
+          },
+        },
       },
     });
     invariant(lists.map((l) => l.list.userId).every((id) => id == ctx.user.id));
@@ -143,6 +199,45 @@ export abstract class List implements PrivacyAware {
     this.list = result[0];
   }
 
+  private async setRssToken(token: string | null) {
+    const result = await this.ctx.db
+      .update(bookmarkLists)
+      .set({ rssToken: token })
+      .where(
+        and(
+          eq(bookmarkLists.id, this.list.id),
+          eq(bookmarkLists.userId, this.ctx.user.id),
+        ),
+      )
+      .returning();
+    if (result.length == 0) {
+      throw new TRPCError({ code: "NOT_FOUND" });
+    }
+    return result[0].rssToken;
+  }
+
+  async getRssToken(): Promise<string | null> {
+    const [result] = await this.ctx.db
+      .select({ rssToken: bookmarkLists.rssToken })
+      .from(bookmarkLists)
+      .where(
+        and(
+          eq(bookmarkLists.id, this.list.id),
+          eq(bookmarkLists.userId, this.ctx.user.id),
+        ),
+      )
+      .limit(1);
+    return result.rssToken ?? null;
+  }
+
+  async regenRssToken() {
+    return await this.setRssToken(crypto.randomBytes(32).toString("hex"));
+  }
+
+  async clearRssToken() {
+    await this.setRssToken(null);
+  }
+
   abstract get type(): "manual" | "smart";
   abstract getBookmarkIds(ctx: AuthedContext): Promise<string[]>;
   abstract getSize(ctx: AuthedContext): Promise<number>;