From 9695bba2e993b48ae333da622fa459dbaacb9349 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sat, 31 May 2025 18:46:04 +0100 Subject: feat: Generate RSS feeds from lists (#1507) * refactor: Move bookmark utils from shared-react to shared * Expose RSS feeds for lists * Add e2e tests * Slightly improve the look of the share dialog * allow specifying a limit in the rss endpoint --- packages/trpc/models/bookmarks.ts | 365 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 packages/trpc/models/bookmarks.ts (limited to 'packages/trpc/models/bookmarks.ts') diff --git a/packages/trpc/models/bookmarks.ts b/packages/trpc/models/bookmarks.ts new file mode 100644 index 00000000..524749f9 --- /dev/null +++ b/packages/trpc/models/bookmarks.ts @@ -0,0 +1,365 @@ +import { TRPCError } from "@trpc/server"; +import { + and, + asc, + desc, + eq, + exists, + gt, + gte, + inArray, + lt, + lte, + or, +} from "drizzle-orm"; +import invariant from "tiny-invariant"; +import { z } from "zod"; + +import { + assets, + AssetTypes, + bookmarkAssets, + bookmarkLinks, + bookmarks, + bookmarksInLists, + bookmarkTags, + bookmarkTexts, + rssFeedImportsTable, + tagsOnBookmarks, +} from "@karakeep/db/schema"; +import { + BookmarkTypes, + DEFAULT_NUM_BOOKMARKS_PER_PAGE, + ZBookmark, + ZBookmarkContent, + zGetBookmarksRequestSchema, + ZPublicBookmark, +} from "@karakeep/shared/types/bookmarks"; +import { ZCursor } from "@karakeep/shared/types/pagination"; +import { getBookmarkTitle } from "@karakeep/shared/utils/bookmarkUtils"; + +import { AuthedContext } from ".."; +import { mapDBAssetTypeToUserType } from "../lib/attachments"; +import { List } from "./lists"; +import { PrivacyAware } from "./privacy"; + +export class Bookmark implements PrivacyAware { + protected constructor( + protected ctx: AuthedContext, + public bookmark: ZBookmark & { userId: string }, + ) {} + + ensureCanAccess(ctx: AuthedContext): void { + if (this.bookmark.userId != ctx.user.id) { + throw new TRPCError({ + code: "FORBIDDEN", + message: "User is not allowed to access resource", + }); + } + } + + static fromData(ctx: AuthedContext, data: ZBookmark) { + return new Bookmark(ctx, { + ...data, + userId: ctx.user.id, + }); + } + + static async loadMulti( + ctx: AuthedContext, + input: z.infer, + ): Promise<{ + bookmarks: Bookmark[]; + nextCursor: ZCursor | null; + }> { + if (input.ids && input.ids.length == 0) { + return { bookmarks: [], nextCursor: null }; + } + if (!input.limit) { + input.limit = DEFAULT_NUM_BOOKMARKS_PER_PAGE; + } + if (input.listId) { + const list = await List.fromId(ctx, input.listId); + if (list.type === "smart") { + input.ids = await list.getBookmarkIds(); + delete input.listId; + } + } + + const sq = ctx.db.$with("bookmarksSq").as( + ctx.db + .select() + .from(bookmarks) + .where( + and( + eq(bookmarks.userId, ctx.user.id), + input.archived !== undefined + ? eq(bookmarks.archived, input.archived) + : undefined, + input.favourited !== undefined + ? eq(bookmarks.favourited, input.favourited) + : undefined, + input.ids ? inArray(bookmarks.id, input.ids) : undefined, + input.tagId !== undefined + ? exists( + ctx.db + .select() + .from(tagsOnBookmarks) + .where( + and( + eq(tagsOnBookmarks.bookmarkId, bookmarks.id), + eq(tagsOnBookmarks.tagId, input.tagId), + ), + ), + ) + : undefined, + input.rssFeedId !== undefined + ? exists( + ctx.db + .select() + .from(rssFeedImportsTable) + .where( + and( + eq(rssFeedImportsTable.bookmarkId, bookmarks.id), + eq(rssFeedImportsTable.rssFeedId, input.rssFeedId), + ), + ), + ) + : undefined, + input.listId !== undefined + ? exists( + ctx.db + .select() + .from(bookmarksInLists) + .where( + and( + eq(bookmarksInLists.bookmarkId, bookmarks.id), + eq(bookmarksInLists.listId, input.listId), + ), + ), + ) + : undefined, + input.cursor + ? input.sortOrder === "asc" + ? or( + gt(bookmarks.createdAt, input.cursor.createdAt), + and( + eq(bookmarks.createdAt, input.cursor.createdAt), + gte(bookmarks.id, input.cursor.id), + ), + ) + : or( + lt(bookmarks.createdAt, input.cursor.createdAt), + and( + eq(bookmarks.createdAt, input.cursor.createdAt), + lte(bookmarks.id, input.cursor.id), + ), + ) + : undefined, + ), + ) + .limit(input.limit + 1) + .orderBy( + input.sortOrder === "asc" + ? asc(bookmarks.createdAt) + : desc(bookmarks.createdAt), + desc(bookmarks.id), + ), + ); + // TODO: Consider not inlining the tags in the response of getBookmarks as this query is getting kinda expensive + const results = await ctx.db + .with(sq) + .select() + .from(sq) + .leftJoin(tagsOnBookmarks, eq(sq.id, tagsOnBookmarks.bookmarkId)) + .leftJoin(bookmarkTags, eq(tagsOnBookmarks.tagId, bookmarkTags.id)) + .leftJoin(bookmarkLinks, eq(bookmarkLinks.id, sq.id)) + .leftJoin(bookmarkTexts, eq(bookmarkTexts.id, sq.id)) + .leftJoin(bookmarkAssets, eq(bookmarkAssets.id, sq.id)) + .leftJoin(assets, eq(assets.bookmarkId, sq.id)) + .orderBy(desc(sq.createdAt), desc(sq.id)); + + const bookmarksRes = results.reduce>( + (acc, row) => { + const bookmarkId = row.bookmarksSq.id; + if (!acc[bookmarkId]) { + let content: ZBookmarkContent; + if (row.bookmarkLinks) { + content = { + type: BookmarkTypes.LINK, + url: row.bookmarkLinks.url, + title: row.bookmarkLinks.title, + description: row.bookmarkLinks.description, + imageUrl: row.bookmarkLinks.imageUrl, + favicon: row.bookmarkLinks.favicon, + htmlContent: input.includeContent + ? row.bookmarkLinks.htmlContent + : null, + crawledAt: row.bookmarkLinks.crawledAt, + author: row.bookmarkLinks.author, + publisher: row.bookmarkLinks.publisher, + datePublished: row.bookmarkLinks.datePublished, + dateModified: row.bookmarkLinks.dateModified, + }; + } else if (row.bookmarkTexts) { + content = { + type: BookmarkTypes.TEXT, + text: row.bookmarkTexts.text ?? "", + sourceUrl: row.bookmarkTexts.sourceUrl ?? null, + }; + } else if (row.bookmarkAssets) { + content = { + type: BookmarkTypes.ASSET, + assetId: row.bookmarkAssets.assetId, + assetType: row.bookmarkAssets.assetType, + fileName: row.bookmarkAssets.fileName, + sourceUrl: row.bookmarkAssets.sourceUrl ?? null, + size: null, // This will get filled in the asset loop + content: input.includeContent + ? (row.bookmarkAssets.content ?? null) + : null, + }; + } else { + content = { + type: BookmarkTypes.UNKNOWN, + }; + } + acc[bookmarkId] = { + ...row.bookmarksSq, + content, + tags: [], + assets: [], + }; + } + + if ( + row.bookmarkTags && + // Duplicates may occur because of the join, so we need to make sure we're not adding the same tag twice + !acc[bookmarkId].tags.some((t) => t.id == row.bookmarkTags!.id) + ) { + invariant( + row.tagsOnBookmarks, + "if bookmark tag is set, its many-to-many relation must also be set", + ); + acc[bookmarkId].tags.push({ + ...row.bookmarkTags, + attachedBy: row.tagsOnBookmarks.attachedBy, + }); + } + + if ( + row.assets && + !acc[bookmarkId].assets.some((a) => a.id == row.assets!.id) + ) { + if (acc[bookmarkId].content.type == BookmarkTypes.LINK) { + const content = acc[bookmarkId].content; + invariant(content.type == BookmarkTypes.LINK); + if (row.assets.assetType == AssetTypes.LINK_SCREENSHOT) { + content.screenshotAssetId = row.assets.id; + } + if (row.assets.assetType == AssetTypes.LINK_FULL_PAGE_ARCHIVE) { + content.fullPageArchiveAssetId = row.assets.id; + } + if (row.assets.assetType == AssetTypes.LINK_BANNER_IMAGE) { + content.imageAssetId = row.assets.id; + } + if (row.assets.assetType == AssetTypes.LINK_VIDEO) { + content.videoAssetId = row.assets.id; + } + if (row.assets.assetType == AssetTypes.LINK_PRECRAWLED_ARCHIVE) { + content.precrawledArchiveAssetId = row.assets.id; + } + acc[bookmarkId].content = content; + } + if (acc[bookmarkId].content.type == BookmarkTypes.ASSET) { + const content = acc[bookmarkId].content; + if (row.assets.id == content.assetId) { + // If this is the bookmark's main aset, caputure its size. + content.size = row.assets.size; + } + } + acc[bookmarkId].assets.push({ + id: row.assets.id, + assetType: mapDBAssetTypeToUserType(row.assets.assetType), + }); + } + + return acc; + }, + {}, + ); + + const bookmarksArr = Object.values(bookmarksRes); + + bookmarksArr.sort((a, b) => { + if (a.createdAt != b.createdAt) { + return input.sortOrder === "asc" + ? a.createdAt.getTime() - b.createdAt.getTime() + : b.createdAt.getTime() - a.createdAt.getTime(); + } else { + return b.id.localeCompare(a.id); + } + }); + + let nextCursor = null; + if (bookmarksArr.length > input.limit) { + const nextItem = bookmarksArr.pop()!; + nextCursor = { + id: nextItem.id, + createdAt: nextItem.createdAt, + }; + } + + return { + bookmarks: bookmarksArr.map((b) => Bookmark.fromData(ctx, b)), + nextCursor, + }; + } + + asZBookmark(): ZBookmark { + return this.bookmark; + } + + asPublicBookmark(): ZPublicBookmark { + const getContent = ( + content: ZBookmarkContent, + ): ZPublicBookmark["content"] => { + switch (content.type) { + case BookmarkTypes.LINK: { + return { + type: BookmarkTypes.LINK, + url: content.url, + }; + } + case BookmarkTypes.TEXT: { + return { + type: BookmarkTypes.TEXT, + text: content.text, + }; + } + case BookmarkTypes.ASSET: { + return { + type: BookmarkTypes.ASSET, + assetType: content.assetType, + assetId: content.assetId, + fileName: content.fileName, + sourceUrl: content.sourceUrl, + }; + } + default: { + throw new Error("Unknown bookmark content type"); + } + } + }; + + // WARNING: Everything below is exposed in the public APIs, don't use spreads! + return { + id: this.bookmark.id, + createdAt: this.bookmark.createdAt, + modifiedAt: this.bookmark.modifiedAt, + title: getBookmarkTitle(this.bookmark), + tags: this.bookmark.tags.map((t) => t.name), + content: getContent(this.bookmark.content), + }; + } +} -- cgit v1.2.3-70-g09d2