From 9695bba2e993b48ae333da622fa459dbaacb9349 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sat, 31 May 2025 18:46:04 +0100 Subject: feat: Generate RSS feeds from lists (#1507) * refactor: Move bookmark utils from shared-react to shared * Expose RSS feeds for lists * Add e2e tests * Slightly improve the look of the share dialog * allow specifying a limit in the rss endpoint --- packages/trpc/models/bookmarks.ts | 365 ++++++++++++++++++++++++++++++++++++++ packages/trpc/models/lists.ts | 99 ++++++++++- 2 files changed, 462 insertions(+), 2 deletions(-) create mode 100644 packages/trpc/models/bookmarks.ts (limited to 'packages/trpc/models') diff --git a/packages/trpc/models/bookmarks.ts b/packages/trpc/models/bookmarks.ts new file mode 100644 index 00000000..524749f9 --- /dev/null +++ b/packages/trpc/models/bookmarks.ts @@ -0,0 +1,365 @@ +import { TRPCError } from "@trpc/server"; +import { + and, + asc, + desc, + eq, + exists, + gt, + gte, + inArray, + lt, + lte, + or, +} from "drizzle-orm"; +import invariant from "tiny-invariant"; +import { z } from "zod"; + +import { + assets, + AssetTypes, + bookmarkAssets, + bookmarkLinks, + bookmarks, + bookmarksInLists, + bookmarkTags, + bookmarkTexts, + rssFeedImportsTable, + tagsOnBookmarks, +} from "@karakeep/db/schema"; +import { + BookmarkTypes, + DEFAULT_NUM_BOOKMARKS_PER_PAGE, + ZBookmark, + ZBookmarkContent, + zGetBookmarksRequestSchema, + ZPublicBookmark, +} from "@karakeep/shared/types/bookmarks"; +import { ZCursor } from "@karakeep/shared/types/pagination"; +import { getBookmarkTitle } from "@karakeep/shared/utils/bookmarkUtils"; + +import { AuthedContext } from ".."; +import { mapDBAssetTypeToUserType } from "../lib/attachments"; +import { List } from "./lists"; +import { PrivacyAware } from "./privacy"; + +export class Bookmark implements PrivacyAware { + protected constructor( + protected ctx: AuthedContext, + public bookmark: ZBookmark & { userId: string }, + ) {} + + ensureCanAccess(ctx: AuthedContext): void { + if (this.bookmark.userId != ctx.user.id) { + throw new TRPCError({ + code: "FORBIDDEN", + message: "User is not allowed to access resource", + }); + } + } + + static fromData(ctx: AuthedContext, data: ZBookmark) { + return new Bookmark(ctx, { + ...data, + userId: ctx.user.id, + }); + } + + static async loadMulti( + ctx: AuthedContext, + input: z.infer, + ): Promise<{ + bookmarks: Bookmark[]; + nextCursor: ZCursor | null; + }> { + if (input.ids && input.ids.length == 0) { + return { bookmarks: [], nextCursor: null }; + } + if (!input.limit) { + input.limit = DEFAULT_NUM_BOOKMARKS_PER_PAGE; + } + if (input.listId) { + const list = await List.fromId(ctx, input.listId); + if (list.type === "smart") { + input.ids = await list.getBookmarkIds(); + delete input.listId; + } + } + + const sq = ctx.db.$with("bookmarksSq").as( + ctx.db + .select() + .from(bookmarks) + .where( + and( + eq(bookmarks.userId, ctx.user.id), + input.archived !== undefined + ? eq(bookmarks.archived, input.archived) + : undefined, + input.favourited !== undefined + ? eq(bookmarks.favourited, input.favourited) + : undefined, + input.ids ? inArray(bookmarks.id, input.ids) : undefined, + input.tagId !== undefined + ? exists( + ctx.db + .select() + .from(tagsOnBookmarks) + .where( + and( + eq(tagsOnBookmarks.bookmarkId, bookmarks.id), + eq(tagsOnBookmarks.tagId, input.tagId), + ), + ), + ) + : undefined, + input.rssFeedId !== undefined + ? exists( + ctx.db + .select() + .from(rssFeedImportsTable) + .where( + and( + eq(rssFeedImportsTable.bookmarkId, bookmarks.id), + eq(rssFeedImportsTable.rssFeedId, input.rssFeedId), + ), + ), + ) + : undefined, + input.listId !== undefined + ? exists( + ctx.db + .select() + .from(bookmarksInLists) + .where( + and( + eq(bookmarksInLists.bookmarkId, bookmarks.id), + eq(bookmarksInLists.listId, input.listId), + ), + ), + ) + : undefined, + input.cursor + ? input.sortOrder === "asc" + ? or( + gt(bookmarks.createdAt, input.cursor.createdAt), + and( + eq(bookmarks.createdAt, input.cursor.createdAt), + gte(bookmarks.id, input.cursor.id), + ), + ) + : or( + lt(bookmarks.createdAt, input.cursor.createdAt), + and( + eq(bookmarks.createdAt, input.cursor.createdAt), + lte(bookmarks.id, input.cursor.id), + ), + ) + : undefined, + ), + ) + .limit(input.limit + 1) + .orderBy( + input.sortOrder === "asc" + ? asc(bookmarks.createdAt) + : desc(bookmarks.createdAt), + desc(bookmarks.id), + ), + ); + // TODO: Consider not inlining the tags in the response of getBookmarks as this query is getting kinda expensive + const results = await ctx.db + .with(sq) + .select() + .from(sq) + .leftJoin(tagsOnBookmarks, eq(sq.id, tagsOnBookmarks.bookmarkId)) + .leftJoin(bookmarkTags, eq(tagsOnBookmarks.tagId, bookmarkTags.id)) + .leftJoin(bookmarkLinks, eq(bookmarkLinks.id, sq.id)) + .leftJoin(bookmarkTexts, eq(bookmarkTexts.id, sq.id)) + .leftJoin(bookmarkAssets, eq(bookmarkAssets.id, sq.id)) + .leftJoin(assets, eq(assets.bookmarkId, sq.id)) + .orderBy(desc(sq.createdAt), desc(sq.id)); + + const bookmarksRes = results.reduce>( + (acc, row) => { + const bookmarkId = row.bookmarksSq.id; + if (!acc[bookmarkId]) { + let content: ZBookmarkContent; + if (row.bookmarkLinks) { + content = { + type: BookmarkTypes.LINK, + url: row.bookmarkLinks.url, + title: row.bookmarkLinks.title, + description: row.bookmarkLinks.description, + imageUrl: row.bookmarkLinks.imageUrl, + favicon: row.bookmarkLinks.favicon, + htmlContent: input.includeContent + ? row.bookmarkLinks.htmlContent + : null, + crawledAt: row.bookmarkLinks.crawledAt, + author: row.bookmarkLinks.author, + publisher: row.bookmarkLinks.publisher, + datePublished: row.bookmarkLinks.datePublished, + dateModified: row.bookmarkLinks.dateModified, + }; + } else if (row.bookmarkTexts) { + content = { + type: BookmarkTypes.TEXT, + text: row.bookmarkTexts.text ?? "", + sourceUrl: row.bookmarkTexts.sourceUrl ?? null, + }; + } else if (row.bookmarkAssets) { + content = { + type: BookmarkTypes.ASSET, + assetId: row.bookmarkAssets.assetId, + assetType: row.bookmarkAssets.assetType, + fileName: row.bookmarkAssets.fileName, + sourceUrl: row.bookmarkAssets.sourceUrl ?? null, + size: null, // This will get filled in the asset loop + content: input.includeContent + ? (row.bookmarkAssets.content ?? null) + : null, + }; + } else { + content = { + type: BookmarkTypes.UNKNOWN, + }; + } + acc[bookmarkId] = { + ...row.bookmarksSq, + content, + tags: [], + assets: [], + }; + } + + if ( + row.bookmarkTags && + // Duplicates may occur because of the join, so we need to make sure we're not adding the same tag twice + !acc[bookmarkId].tags.some((t) => t.id == row.bookmarkTags!.id) + ) { + invariant( + row.tagsOnBookmarks, + "if bookmark tag is set, its many-to-many relation must also be set", + ); + acc[bookmarkId].tags.push({ + ...row.bookmarkTags, + attachedBy: row.tagsOnBookmarks.attachedBy, + }); + } + + if ( + row.assets && + !acc[bookmarkId].assets.some((a) => a.id == row.assets!.id) + ) { + if (acc[bookmarkId].content.type == BookmarkTypes.LINK) { + const content = acc[bookmarkId].content; + invariant(content.type == BookmarkTypes.LINK); + if (row.assets.assetType == AssetTypes.LINK_SCREENSHOT) { + content.screenshotAssetId = row.assets.id; + } + if (row.assets.assetType == AssetTypes.LINK_FULL_PAGE_ARCHIVE) { + content.fullPageArchiveAssetId = row.assets.id; + } + if (row.assets.assetType == AssetTypes.LINK_BANNER_IMAGE) { + content.imageAssetId = row.assets.id; + } + if (row.assets.assetType == AssetTypes.LINK_VIDEO) { + content.videoAssetId = row.assets.id; + } + if (row.assets.assetType == AssetTypes.LINK_PRECRAWLED_ARCHIVE) { + content.precrawledArchiveAssetId = row.assets.id; + } + acc[bookmarkId].content = content; + } + if (acc[bookmarkId].content.type == BookmarkTypes.ASSET) { + const content = acc[bookmarkId].content; + if (row.assets.id == content.assetId) { + // If this is the bookmark's main aset, caputure its size. + content.size = row.assets.size; + } + } + acc[bookmarkId].assets.push({ + id: row.assets.id, + assetType: mapDBAssetTypeToUserType(row.assets.assetType), + }); + } + + return acc; + }, + {}, + ); + + const bookmarksArr = Object.values(bookmarksRes); + + bookmarksArr.sort((a, b) => { + if (a.createdAt != b.createdAt) { + return input.sortOrder === "asc" + ? a.createdAt.getTime() - b.createdAt.getTime() + : b.createdAt.getTime() - a.createdAt.getTime(); + } else { + return b.id.localeCompare(a.id); + } + }); + + let nextCursor = null; + if (bookmarksArr.length > input.limit) { + const nextItem = bookmarksArr.pop()!; + nextCursor = { + id: nextItem.id, + createdAt: nextItem.createdAt, + }; + } + + return { + bookmarks: bookmarksArr.map((b) => Bookmark.fromData(ctx, b)), + nextCursor, + }; + } + + asZBookmark(): ZBookmark { + return this.bookmark; + } + + asPublicBookmark(): ZPublicBookmark { + const getContent = ( + content: ZBookmarkContent, + ): ZPublicBookmark["content"] => { + switch (content.type) { + case BookmarkTypes.LINK: { + return { + type: BookmarkTypes.LINK, + url: content.url, + }; + } + case BookmarkTypes.TEXT: { + return { + type: BookmarkTypes.TEXT, + text: content.text, + }; + } + case BookmarkTypes.ASSET: { + return { + type: BookmarkTypes.ASSET, + assetType: content.assetType, + assetId: content.assetId, + fileName: content.fileName, + sourceUrl: content.sourceUrl, + }; + } + default: { + throw new Error("Unknown bookmark content type"); + } + } + }; + + // WARNING: Everything below is exposed in the public APIs, don't use spreads! + return { + id: this.bookmark.id, + createdAt: this.bookmark.createdAt, + modifiedAt: this.bookmark.modifiedAt, + title: getBookmarkTitle(this.bookmark), + tags: this.bookmark.tags.map((t) => t.name), + content: getContent(this.bookmark.content), + }; + } +} diff --git a/packages/trpc/models/lists.ts b/packages/trpc/models/lists.ts index 21b23593..4413a8cd 100644 --- a/packages/trpc/models/lists.ts +++ b/packages/trpc/models/lists.ts @@ -1,3 +1,4 @@ +import crypto from "node:crypto"; import { TRPCError } from "@trpc/server"; import { and, count, eq } from "drizzle-orm"; import invariant from "tiny-invariant"; @@ -13,8 +14,10 @@ import { zNewBookmarkListSchema, } from "@karakeep/shared/types/lists"; -import { AuthedContext } from ".."; +import { AuthedContext, Context } from ".."; +import { buildImpersonatingAuthedContext } from "../lib/impersonate"; import { getBookmarkIdsFromMatcher } from "../lib/search"; +import { Bookmark } from "./bookmarks"; import { PrivacyAware } from "./privacy"; export abstract class List implements PrivacyAware { @@ -58,6 +61,52 @@ export abstract class List implements PrivacyAware { } } + static async getForRss( + ctx: Context, + listId: string, + token: string, + pagination: { + limit: number; + }, + ) { + const listdb = await ctx.db.query.bookmarkLists.findFirst({ + where: and( + eq(bookmarkLists.id, listId), + eq(bookmarkLists.rssToken, token), + ), + }); + if (!listdb) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "List not found", + }); + } + + // The token here acts as an authed context, so we can create + // an impersonating context for the list owner as long as + // we don't leak the context. + + const authedCtx = await buildImpersonatingAuthedContext(listdb.userId); + const list = List.fromData(authedCtx, listdb); + const bookmarkIds = await list.getBookmarkIds(); + + const bookmarks = await Bookmark.loadMulti(authedCtx, { + ids: bookmarkIds, + includeContent: false, + limit: pagination.limit, + sortOrder: "desc", + }); + + return { + list: { + icon: list.list.icon, + name: list.list.name, + description: list.list.description, + }, + bookmarks: bookmarks.bookmarks.map((b) => b.asPublicBookmark()), + }; + } + static async create( ctx: AuthedContext, input: z.infer, @@ -79,6 +128,9 @@ export abstract class List implements PrivacyAware { static async getAll(ctx: AuthedContext): Promise<(ManualList | SmartList)[]> { const lists = await ctx.db.query.bookmarkLists.findMany({ + columns: { + rssToken: false, + }, where: and(eq(bookmarkLists.userId, ctx.user.id)), }); return lists.map((l) => this.fromData(ctx, l)); @@ -88,7 +140,11 @@ export abstract class List implements PrivacyAware { const lists = await ctx.db.query.bookmarksInLists.findMany({ where: and(eq(bookmarksInLists.bookmarkId, bookmarkId)), with: { - list: true, + list: { + columns: { + rssToken: false, + }, + }, }, }); invariant(lists.map((l) => l.list.userId).every((id) => id == ctx.user.id)); @@ -143,6 +199,45 @@ export abstract class List implements PrivacyAware { this.list = result[0]; } + private async setRssToken(token: string | null) { + const result = await this.ctx.db + .update(bookmarkLists) + .set({ rssToken: token }) + .where( + and( + eq(bookmarkLists.id, this.list.id), + eq(bookmarkLists.userId, this.ctx.user.id), + ), + ) + .returning(); + if (result.length == 0) { + throw new TRPCError({ code: "NOT_FOUND" }); + } + return result[0].rssToken; + } + + async getRssToken(): Promise { + const [result] = await this.ctx.db + .select({ rssToken: bookmarkLists.rssToken }) + .from(bookmarkLists) + .where( + and( + eq(bookmarkLists.id, this.list.id), + eq(bookmarkLists.userId, this.ctx.user.id), + ), + ) + .limit(1); + return result.rssToken ?? null; + } + + async regenRssToken() { + return await this.setRssToken(crypto.randomBytes(32).toString("hex")); + } + + async clearRssToken() { + await this.setRssToken(null); + } + abstract get type(): "manual" | "smart"; abstract getBookmarkIds(ctx: AuthedContext): Promise; abstract getSize(ctx: AuthedContext): Promise; -- cgit v1.2.3-70-g09d2