diff options
Diffstat (limited to '')
36 files changed, 3657 insertions, 838 deletions
diff --git a/packages/trpc/auth.ts b/packages/trpc/auth.ts index d252bebb..764a0904 100644 --- a/packages/trpc/auth.ts +++ b/packages/trpc/auth.ts @@ -125,6 +125,19 @@ export async function authenticateApiKey(key: string, database: Context["db"]) { throw new Error("Invalid API Key"); } + // Update lastUsedAt with 10-minute throttle to avoid excessive DB writes + const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000); + if (!apiKey.lastUsedAt || apiKey.lastUsedAt < tenMinutesAgo) { + // Fire and forget - don't await to avoid blocking the auth response + database + .update(apiKeys) + .set({ lastUsedAt: new Date() }) + .where(eq(apiKeys.id, apiKey.id)) + .catch((err) => { + console.error("Failed to update API key lastUsedAt:", err); + }); + } + return apiKey.user; } diff --git a/packages/trpc/email.ts b/packages/trpc/email.ts index 3c0b8b39..15e1ef74 100644 --- a/packages/trpc/email.ts +++ b/packages/trpc/email.ts @@ -1,17 +1,15 @@ import { createTransport } from "nodemailer"; +import { getTracer, withSpan } from "@karakeep/shared-server"; import serverConfig from "@karakeep/shared/config"; -export async function sendVerificationEmail( - email: string, - name: string, - token: string, -) { +const tracer = getTracer("@karakeep/trpc"); + +function buildTransporter() { if (!serverConfig.email.smtp) { throw new Error("SMTP is not configured"); } - - const transporter = createTransport({ + return createTransport({ host: serverConfig.email.smtp.host, port: serverConfig.email.smtp.port, secure: serverConfig.email.smtp.secure, @@ -23,14 +21,52 @@ export async function sendVerificationEmail( } : undefined, }); +} - const verificationUrl = `${serverConfig.publicUrl}/verify-email?token=${encodeURIComponent(token)}&email=${encodeURIComponent(email)}`; +type Transporter = ReturnType<typeof buildTransporter>; - const mailOptions = { - from: serverConfig.email.smtp.from, - to: email, - subject: "Verify your email address", - html: ` +type Fn<Args extends unknown[] = unknown[]> = ( + transport: Transporter, + ...args: Args +) => Promise<void>; + +interface TracingOptions { + silentFail?: boolean; +} + +function withTracing<Args extends unknown[]>( + name: string, + fn: Fn<Args>, + options: TracingOptions = {}, +) { + return async (...args: Args): Promise<void> => { + if (options.silentFail && !serverConfig.email.smtp) { + return; + } + const transporter = buildTransporter(); + await withSpan(tracer, name, {}, () => fn(transporter, ...args)); + }; +} + +export const sendVerificationEmail = withTracing( + "sendVerificationEmail", + async ( + transporter: Transporter, + email: string, + name: string, + token: string, + redirectUrl?: string, + ) => { + let verificationUrl = `${serverConfig.publicUrl}/verify-email?token=${encodeURIComponent(token)}&email=${encodeURIComponent(email)}`; + if (redirectUrl) { + verificationUrl += `&redirectUrl=${encodeURIComponent(redirectUrl)}`; + } + + const mailOptions = { + from: serverConfig.email.smtp!.from, + to: email, + subject: "Verify your email address", + html: ` <div style="font-family: Arial, sans-serif; max-width: 600px; margin: 0 auto;"> <h2>Welcome to Karakeep, ${name}!</h2> <p>Please verify your email address by clicking the link below:</p> @@ -45,7 +81,7 @@ export async function sendVerificationEmail( <p>If you didn't create an account with us, please ignore this email.</p> </div> `, - text: ` + text: ` Welcome to Karakeep, ${name}! Please verify your email address by visiting this link: @@ -55,40 +91,27 @@ This link will expire in 24 hours. If you didn't create an account with us, please ignore this email. `, - }; + }; - await transporter.sendMail(mailOptions); -} + await transporter.sendMail(mailOptions); + }, +); -export async function sendInviteEmail( - email: string, - token: string, - inviterName: string, -) { - if (!serverConfig.email.smtp) { - throw new Error("SMTP is not configured"); - } - - const transporter = createTransport({ - host: serverConfig.email.smtp.host, - port: serverConfig.email.smtp.port, - secure: serverConfig.email.smtp.secure, - auth: - serverConfig.email.smtp.user && serverConfig.email.smtp.password - ? { - user: serverConfig.email.smtp.user, - pass: serverConfig.email.smtp.password, - } - : undefined, - }); - - const inviteUrl = `${serverConfig.publicUrl}/invite/${encodeURIComponent(token)}`; +export const sendInviteEmail = withTracing( + "sendInviteEmail", + async ( + transporter: Transporter, + email: string, + token: string, + inviterName: string, + ) => { + const inviteUrl = `${serverConfig.publicUrl}/invite/${encodeURIComponent(token)}`; - const mailOptions = { - from: serverConfig.email.smtp.from, - to: email, - subject: "You've been invited to join Karakeep", - html: ` + const mailOptions = { + from: serverConfig.email.smtp!.from, + to: email, + subject: "You've been invited to join Karakeep", + html: ` <div style="font-family: Arial, sans-serif; max-width: 600px; margin: 0 auto;"> <h2>You've been invited to join Karakeep!</h2> <p>${inviterName} has invited you to join Karakeep, the bookmark everything app.</p> @@ -104,7 +127,7 @@ export async function sendInviteEmail( <p>If you weren't expecting this invitation, you can safely ignore this email.</p> </div> `, - text: ` + text: ` You've been invited to join Karakeep! ${inviterName} has invited you to join Karakeep, a powerful bookmarking and content organization platform. @@ -116,40 +139,27 @@ ${inviteUrl} If you weren't expecting this invitation, you can safely ignore this email. `, - }; + }; - await transporter.sendMail(mailOptions); -} + await transporter.sendMail(mailOptions); + }, +); -export async function sendPasswordResetEmail( - email: string, - name: string, - token: string, -) { - if (!serverConfig.email.smtp) { - throw new Error("SMTP is not configured"); - } - - const transporter = createTransport({ - host: serverConfig.email.smtp.host, - port: serverConfig.email.smtp.port, - secure: serverConfig.email.smtp.secure, - auth: - serverConfig.email.smtp.user && serverConfig.email.smtp.password - ? { - user: serverConfig.email.smtp.user, - pass: serverConfig.email.smtp.password, - } - : undefined, - }); - - const resetUrl = `${serverConfig.publicUrl}/reset-password?token=${encodeURIComponent(token)}`; +export const sendPasswordResetEmail = withTracing( + "sendPasswordResetEmail", + async ( + transporter: Transporter, + email: string, + name: string, + token: string, + ) => { + const resetUrl = `${serverConfig.publicUrl}/reset-password?token=${encodeURIComponent(token)}`; - const mailOptions = { - from: serverConfig.email.smtp.from, - to: email, - subject: "Reset your password", - html: ` + const mailOptions = { + from: serverConfig.email.smtp!.from, + to: email, + subject: "Reset your password", + html: ` <div style="font-family: Arial, sans-serif; max-width: 600px; margin: 0 auto;"> <h2>Password Reset Request</h2> <p>Hi ${name},</p> @@ -165,7 +175,7 @@ export async function sendPasswordResetEmail( <p>If you didn't request a password reset, please ignore this email. Your password will remain unchanged.</p> </div> `, - text: ` + text: ` Hi ${name}, You requested to reset your password for your Karakeep account. Visit this link to reset your password: @@ -175,42 +185,28 @@ This link will expire in 1 hour. If you didn't request a password reset, please ignore this email. Your password will remain unchanged. `, - }; - - await transporter.sendMail(mailOptions); -} - -export async function sendListInvitationEmail( - email: string, - inviterName: string, - listName: string, - listId: string, -) { - if (!serverConfig.email.smtp) { - // Silently fail if email is not configured - return; - } + }; - const transporter = createTransport({ - host: serverConfig.email.smtp.host, - port: serverConfig.email.smtp.port, - secure: serverConfig.email.smtp.secure, - auth: - serverConfig.email.smtp.user && serverConfig.email.smtp.password - ? { - user: serverConfig.email.smtp.user, - pass: serverConfig.email.smtp.password, - } - : undefined, - }); + await transporter.sendMail(mailOptions); + }, +); - const inviteUrl = `${serverConfig.publicUrl}/dashboard/lists?pendingInvitation=${encodeURIComponent(listId)}`; +export const sendListInvitationEmail = withTracing( + "sendListInvitationEmail", + async ( + transporter: Transporter, + email: string, + inviterName: string, + listName: string, + listId: string, + ) => { + const inviteUrl = `${serverConfig.publicUrl}/dashboard/lists?pendingInvitation=${encodeURIComponent(listId)}`; - const mailOptions = { - from: serverConfig.email.smtp.from, - to: email, - subject: `${inviterName} invited you to collaborate on "${listName}"`, - html: ` + const mailOptions = { + from: serverConfig.email.smtp!.from, + to: email, + subject: `${inviterName} invited you to collaborate on "${listName}"`, + html: ` <div style="font-family: Arial, sans-serif; max-width: 600px; margin: 0 auto;"> <h2>You've been invited to collaborate on a list!</h2> <p>${inviterName} has invited you to collaborate on the list <strong>"${listName}"</strong> in Karakeep.</p> @@ -226,7 +222,7 @@ export async function sendListInvitationEmail( <p>If you weren't expecting this invitation, you can safely ignore this email or decline it in your dashboard.</p> </div> `, - text: ` + text: ` You've been invited to collaborate on a list! ${inviterName} has invited you to collaborate on the list "${listName}" in Karakeep. @@ -238,7 +234,9 @@ You can accept or decline this invitation from your Karakeep dashboard. If you weren't expecting this invitation, you can safely ignore this email or decline it in your dashboard. `, - }; + }; - await transporter.sendMail(mailOptions); -} + await transporter.sendMail(mailOptions); + }, + { silentFail: true }, +); diff --git a/packages/trpc/index.ts b/packages/trpc/index.ts index 555ca3ba..178703f0 100644 --- a/packages/trpc/index.ts +++ b/packages/trpc/index.ts @@ -6,6 +6,7 @@ import type { db } from "@karakeep/db"; import serverConfig from "@karakeep/shared/config"; import { createRateLimitMiddleware } from "./lib/rateLimit"; +import { createTracingMiddleware } from "./lib/tracing"; import { apiErrorsTotalCounter, apiRequestDurationSummary, @@ -86,7 +87,9 @@ export const procedure = t.procedure }); end(); return res; - }); + }) + // OpenTelemetry tracing middleware + .use(createTracingMiddleware()); // Default public procedure rate limiting export const publicProcedure = procedure.use( diff --git a/packages/trpc/lib/__tests__/ruleEngine.test.ts b/packages/trpc/lib/__tests__/ruleEngine.test.ts index ede22ec6..b737e3a5 100644 --- a/packages/trpc/lib/__tests__/ruleEngine.test.ts +++ b/packages/trpc/lib/__tests__/ruleEngine.test.ts @@ -126,6 +126,7 @@ describe("RuleEngine", () => { .values({ userId, type: BookmarkTypes.LINK, + title: "Example Bookmark Title", favourited: false, archived: false, }) @@ -171,10 +172,9 @@ describe("RuleEngine", () => { expect(engine).toBeInstanceOf(RuleEngine); }); - it("should throw an error if bookmark is not found", async () => { - await expect( - RuleEngine.forBookmark(ctx, "nonexistent-bookmark"), - ).rejects.toThrow("Bookmark nonexistent-bookmark not found"); + it("should return null if bookmark is not found", async () => { + const engine = await RuleEngine.forBookmark(ctx, "nonexistent-bookmark"); + expect(engine).toBeNull(); }); it("should load rules associated with the bookmark's user", async () => { @@ -188,7 +188,7 @@ describe("RuleEngine", () => { actions: [{ type: "addTag", tagId: tagId2 }], }); - const engine = await RuleEngine.forBookmark(ctx, bookmarkId); + const engine = (await RuleEngine.forBookmark(ctx, bookmarkId))!; // @ts-expect-error Accessing private property for test verification expect(engine.rules).toHaveLength(1); // @ts-expect-error Accessing private property for test verification @@ -200,7 +200,7 @@ describe("RuleEngine", () => { let engine: RuleEngine; beforeEach(async () => { - engine = await RuleEngine.forBookmark(ctx, bookmarkId); + engine = (await RuleEngine.forBookmark(ctx, bookmarkId))!; }); it("should return true for urlContains condition", () => { @@ -219,6 +219,54 @@ describe("RuleEngine", () => { expect(engine.doesBookmarkMatchConditions(condition)).toBe(false); }); + it("should return false for urlDoesNotContain condition when URL contains string", () => { + const condition: RuleEngineCondition = { + type: "urlDoesNotContain", + str: "example.com", + }; + expect(engine.doesBookmarkMatchConditions(condition)).toBe(false); + }); + + it("should return true for urlDoesNotContain condition when URL does not contain string", () => { + const condition: RuleEngineCondition = { + type: "urlDoesNotContain", + str: "nonexistent", + }; + expect(engine.doesBookmarkMatchConditions(condition)).toBe(true); + }); + + it("should return true for titleContains condition", () => { + const condition: RuleEngineCondition = { + type: "titleContains", + str: "Example", + }; + expect(engine.doesBookmarkMatchConditions(condition)).toBe(true); + }); + + it("should return false for titleContains condition mismatch", () => { + const condition: RuleEngineCondition = { + type: "titleContains", + str: "nonexistent", + }; + expect(engine.doesBookmarkMatchConditions(condition)).toBe(false); + }); + + it("should return false for titleDoesNotContain condition when title contains string", () => { + const condition: RuleEngineCondition = { + type: "titleDoesNotContain", + str: "Example", + }; + expect(engine.doesBookmarkMatchConditions(condition)).toBe(false); + }); + + it("should return true for titleDoesNotContain condition when title does not contain string", () => { + const condition: RuleEngineCondition = { + type: "titleDoesNotContain", + str: "nonexistent", + }; + expect(engine.doesBookmarkMatchConditions(condition)).toBe(true); + }); + it("should return true for importedFromFeed condition", () => { const condition: RuleEngineCondition = { type: "importedFromFeed", @@ -271,7 +319,7 @@ describe("RuleEngine", () => { .update(bookmarks) .set({ favourited: true }) .where(eq(bookmarks.id, bookmarkId)); - const updatedEngine = await RuleEngine.forBookmark(ctx, bookmarkId); + const updatedEngine = (await RuleEngine.forBookmark(ctx, bookmarkId))!; const condition: RuleEngineCondition = { type: "isFavourited" }; expect(updatedEngine.doesBookmarkMatchConditions(condition)).toBe(true); }); @@ -286,7 +334,7 @@ describe("RuleEngine", () => { .update(bookmarks) .set({ archived: true }) .where(eq(bookmarks.id, bookmarkId)); - const updatedEngine = await RuleEngine.forBookmark(ctx, bookmarkId); + const updatedEngine = (await RuleEngine.forBookmark(ctx, bookmarkId))!; const condition: RuleEngineCondition = { type: "isArchived" }; expect(updatedEngine.doesBookmarkMatchConditions(condition)).toBe(true); }); @@ -354,7 +402,7 @@ describe("RuleEngine", () => { } as Omit<RuleEngineRule, "id"> & { userId: string }; ruleId = await seedRule(tmp); testRule = { ...tmp, id: ruleId }; - engine = await RuleEngine.forBookmark(ctx, bookmarkId); + engine = (await RuleEngine.forBookmark(ctx, bookmarkId))!; }); it("should evaluate rule successfully when event and conditions match", async () => { @@ -443,7 +491,7 @@ describe("RuleEngine", () => { let engine: RuleEngine; beforeEach(async () => { - engine = await RuleEngine.forBookmark(ctx, bookmarkId); + engine = (await RuleEngine.forBookmark(ctx, bookmarkId))!; }); it("should execute addTag action", async () => { @@ -625,7 +673,7 @@ describe("RuleEngine", () => { actions: [{ type: "addToList", listId: listId1 }], }); - engine = await RuleEngine.forBookmark(ctx, bookmarkId); + engine = (await RuleEngine.forBookmark(ctx, bookmarkId))!; }); it("should process event and return only results for matching, enabled rules", async () => { diff --git a/packages/trpc/lib/__tests__/search.test.ts b/packages/trpc/lib/__tests__/search.test.ts index ee8bfb60..d39e27a6 100644 --- a/packages/trpc/lib/__tests__/search.test.ts +++ b/packages/trpc/lib/__tests__/search.test.ts @@ -240,6 +240,61 @@ describe("getBookmarkIdsFromMatcher", () => { expect(result.sort()).toEqual(["b2", "b3", "b4", "b5"]); }); + it("should handle listName matcher when multiple lists share the same name", async () => { + await mockCtx.db.insert(bookmarkLists).values({ + id: "l5", + userId: testUserId, + name: "list1", + icon: "đ", + type: "manual", + }); + await mockCtx.db.insert(bookmarksInLists).values({ + bookmarkId: "b2", + listId: "l5", + }); + + const matcher: Matcher = { + type: "listName", + listName: "list1", + inverse: false, + }; + const result = await getBookmarkIdsFromMatcher(mockCtx, matcher); + expect(result.sort()).toEqual(["b1", "b2", "b6"]); + }); + + it("should handle inverse listName matcher when multiple lists share the same name", async () => { + await mockCtx.db.insert(bookmarkLists).values({ + id: "l5", + userId: testUserId, + name: "list1", + icon: "đ", + type: "manual", + }); + await mockCtx.db.insert(bookmarksInLists).values({ + bookmarkId: "b2", + listId: "l5", + }); + + const matcher: Matcher = { + type: "listName", + listName: "list1", + inverse: true, + }; + const result = await getBookmarkIdsFromMatcher(mockCtx, matcher); + expect(result.sort()).toEqual(["b3", "b4", "b5"]); + }); + + it("should return empty when inverse listName references a missing list", async () => { + const matcher: Matcher = { + type: "listName", + listName: "does-not-exist", + inverse: true, + }; + + const result = await getBookmarkIdsFromMatcher(mockCtx, matcher); + expect(result).toEqual([]); + }); + it("should handle archived matcher", async () => { const matcher: Matcher = { type: "archived", archived: true }; const result = await getBookmarkIdsFromMatcher(mockCtx, matcher); diff --git a/packages/trpc/lib/attachments.ts b/packages/trpc/lib/attachments.ts index 25d9be94..f3170c22 100644 --- a/packages/trpc/lib/attachments.ts +++ b/packages/trpc/lib/attachments.ts @@ -9,6 +9,7 @@ import { export function mapDBAssetTypeToUserType(assetType: AssetTypes): ZAssetType { const map: Record<AssetTypes, z.infer<typeof zAssetTypesSchema>> = { [AssetTypes.LINK_SCREENSHOT]: "screenshot", + [AssetTypes.LINK_PDF]: "pdf", [AssetTypes.ASSET_SCREENSHOT]: "assetScreenshot", [AssetTypes.LINK_FULL_PAGE_ARCHIVE]: "fullPageArchive", [AssetTypes.LINK_PRECRAWLED_ARCHIVE]: "precrawledArchive", @@ -17,6 +18,7 @@ export function mapDBAssetTypeToUserType(assetType: AssetTypes): ZAssetType { [AssetTypes.LINK_HTML_CONTENT]: "linkHtmlContent", [AssetTypes.BOOKMARK_ASSET]: "bookmarkAsset", [AssetTypes.USER_UPLOADED]: "userUploaded", + [AssetTypes.AVATAR]: "avatar", [AssetTypes.BACKUP]: "unknown", // Backups are not displayed as regular assets [AssetTypes.UNKNOWN]: "bannerImage", }; @@ -28,6 +30,7 @@ export function mapSchemaAssetTypeToDB( ): AssetTypes { const map: Record<ZAssetType, AssetTypes> = { screenshot: AssetTypes.LINK_SCREENSHOT, + pdf: AssetTypes.LINK_PDF, assetScreenshot: AssetTypes.ASSET_SCREENSHOT, fullPageArchive: AssetTypes.LINK_FULL_PAGE_ARCHIVE, precrawledArchive: AssetTypes.LINK_PRECRAWLED_ARCHIVE, @@ -36,6 +39,7 @@ export function mapSchemaAssetTypeToDB( bookmarkAsset: AssetTypes.BOOKMARK_ASSET, linkHtmlContent: AssetTypes.LINK_HTML_CONTENT, userUploaded: AssetTypes.USER_UPLOADED, + avatar: AssetTypes.AVATAR, unknown: AssetTypes.UNKNOWN, }; return map[assetType]; @@ -44,6 +48,7 @@ export function mapSchemaAssetTypeToDB( export function humanFriendlyNameForAssertType(type: ZAssetType) { const map: Record<ZAssetType, string> = { screenshot: "Screenshot", + pdf: "PDF", assetScreenshot: "Asset Screenshot", fullPageArchive: "Full Page Archive", precrawledArchive: "Precrawled Archive", @@ -52,6 +57,7 @@ export function humanFriendlyNameForAssertType(type: ZAssetType) { bookmarkAsset: "Bookmark Asset", linkHtmlContent: "HTML Content", userUploaded: "User Uploaded File", + avatar: "Avatar", unknown: "Unknown", }; return map[type]; @@ -60,6 +66,7 @@ export function humanFriendlyNameForAssertType(type: ZAssetType) { export function isAllowedToAttachAsset(type: ZAssetType) { const map: Record<ZAssetType, boolean> = { screenshot: true, + pdf: true, assetScreenshot: true, fullPageArchive: false, precrawledArchive: true, @@ -68,6 +75,7 @@ export function isAllowedToAttachAsset(type: ZAssetType) { bookmarkAsset: false, linkHtmlContent: false, userUploaded: true, + avatar: false, unknown: false, }; return map[type]; @@ -76,6 +84,7 @@ export function isAllowedToAttachAsset(type: ZAssetType) { export function isAllowedToDetachAsset(type: ZAssetType) { const map: Record<ZAssetType, boolean> = { screenshot: true, + pdf: true, assetScreenshot: true, fullPageArchive: true, precrawledArchive: true, @@ -84,6 +93,7 @@ export function isAllowedToDetachAsset(type: ZAssetType) { bookmarkAsset: false, linkHtmlContent: false, userUploaded: true, + avatar: false, unknown: false, }; return map[type]; diff --git a/packages/trpc/lib/ruleEngine.ts b/packages/trpc/lib/ruleEngine.ts index c191619b..acfd747e 100644 --- a/packages/trpc/lib/ruleEngine.ts +++ b/packages/trpc/lib/ruleEngine.ts @@ -3,6 +3,7 @@ import { and, eq } from "drizzle-orm"; import { bookmarks, tagsOnBookmarks } from "@karakeep/db/schema"; import { LinkCrawlerQueue } from "@karakeep/shared-server"; +import { BookmarkTypes } from "@karakeep/shared/types/bookmarks"; import { RuleEngineAction, RuleEngineCondition, @@ -21,6 +22,7 @@ async function fetchBookmark(db: AuthedContext["db"], bookmarkId: string) { link: { columns: { url: true, + title: true, }, }, text: true, @@ -60,13 +62,26 @@ export class RuleEngine { private rules: RuleEngineRule[], ) {} - static async forBookmark(ctx: AuthedContext, bookmarkId: string) { + private get bookmarkTitle(): string { + return ( + this.bookmark.title ?? + (this.bookmark.type === BookmarkTypes.LINK + ? this.bookmark.link?.title + : "") ?? + "" + ); + } + + static async forBookmark( + ctx: AuthedContext, + bookmarkId: string, + ): Promise<RuleEngine | null> { const [bookmark, rules] = await Promise.all([ fetchBookmark(ctx.db, bookmarkId), RuleEngineRuleModel.getAll(ctx), ]); if (!bookmark) { - throw new Error(`Bookmark ${bookmarkId} not found`); + return null; } return new RuleEngine( ctx, @@ -83,6 +98,18 @@ export class RuleEngine { case "urlContains": { return (this.bookmark.link?.url ?? "").includes(condition.str); } + case "urlDoesNotContain": { + return ( + this.bookmark.type == BookmarkTypes.LINK && + !(this.bookmark.link?.url ?? "").includes(condition.str) + ); + } + case "titleContains": { + return this.bookmarkTitle.includes(condition.str); + } + case "titleDoesNotContain": { + return !this.bookmarkTitle.includes(condition.str); + } case "importedFromFeed": { return this.bookmark.rssFeeds.some( (f) => f.rssFeedId === condition.feedId, diff --git a/packages/trpc/lib/search.ts b/packages/trpc/lib/search.ts index 88f10f22..d0f529f5 100644 --- a/packages/trpc/lib/search.ts +++ b/packages/trpc/lib/search.ts @@ -4,6 +4,7 @@ import { exists, gt, gte, + inArray, isNotNull, isNull, like, @@ -11,6 +12,7 @@ import { lte, ne, notExists, + notInArray, notLike, or, } from "drizzle-orm"; @@ -89,10 +91,13 @@ function union(vals: BookmarkQueryReturnType[][]): BookmarkQueryReturnType[] { } async function getIds( - db: AuthedContext["db"], - userId: string, + ctx: AuthedContext, matcher: Matcher, + visitedListIds = new Set<string>(), ): Promise<BookmarkQueryReturnType[]> { + const { db } = ctx; + const userId = ctx.user.id; + switch (matcher.type) { case "tagName": { const comp = matcher.inverse ? notExists : exists; @@ -139,29 +144,54 @@ async function getIds( ); } case "listName": { - const comp = matcher.inverse ? notExists : exists; + // First, look up the list by name + const lists = await db.query.bookmarkLists.findMany({ + where: and( + eq(bookmarkLists.userId, userId), + eq(bookmarkLists.name, matcher.listName), + ), + }); + + if (lists.length === 0) { + // No matching lists + return []; + } + + // Use List model to resolve list membership (manual and smart) + // Import dynamically to avoid circular dependency + const { List } = await import("../models/lists"); + const listBookmarkIds = [ + ...new Set( + ( + await Promise.all( + lists.map(async (list) => { + const listModel = await List.fromId(ctx, list.id); + return await listModel.getBookmarkIds(visitedListIds); + }), + ) + ).flat(), + ), + ]; + + if (listBookmarkIds.length === 0) { + if (matcher.inverse) { + return db + .selectDistinct({ id: bookmarks.id }) + .from(bookmarks) + .where(eq(bookmarks.userId, userId)); + } + return []; + } + return db .selectDistinct({ id: bookmarks.id }) .from(bookmarks) .where( and( eq(bookmarks.userId, userId), - comp( - db - .select() - .from(bookmarksInLists) - .innerJoin( - bookmarkLists, - eq(bookmarksInLists.listId, bookmarkLists.id), - ) - .where( - and( - eq(bookmarksInLists.bookmarkId, bookmarks.id), - eq(bookmarkLists.userId, userId), - eq(bookmarkLists.name, matcher.listName), - ), - ), - ), + matcher.inverse + ? notInArray(bookmarks.id, listBookmarkIds) + : inArray(bookmarks.id, listBookmarkIds), ), ); } @@ -373,15 +403,31 @@ async function getIds( ), ); } + case "source": { + return db + .select({ id: bookmarks.id }) + .from(bookmarks) + .where( + and( + eq(bookmarks.userId, userId), + matcher.inverse + ? or( + ne(bookmarks.source, matcher.source), + isNull(bookmarks.source), + ) + : eq(bookmarks.source, matcher.source), + ), + ); + } case "and": { const vals = await Promise.all( - matcher.matchers.map((m) => getIds(db, userId, m)), + matcher.matchers.map((m) => getIds(ctx, m, visitedListIds)), ); return intersect(vals); } case "or": { const vals = await Promise.all( - matcher.matchers.map((m) => getIds(db, userId, m)), + matcher.matchers.map((m) => getIds(ctx, m, visitedListIds)), ); return union(vals); } @@ -395,7 +441,8 @@ async function getIds( export async function getBookmarkIdsFromMatcher( ctx: AuthedContext, matcher: Matcher, + visitedListIds = new Set<string>(), ): Promise<string[]> { - const results = await getIds(ctx.db, ctx.user.id, matcher); + const results = await getIds(ctx, matcher, visitedListIds); return results.map((r) => r.id); } diff --git a/packages/trpc/lib/tracing.ts b/packages/trpc/lib/tracing.ts new file mode 100644 index 00000000..7b4fb39f --- /dev/null +++ b/packages/trpc/lib/tracing.ts @@ -0,0 +1,63 @@ +import { SpanKind } from "@opentelemetry/api"; + +import { + getTracer, + setSpanAttributes, + withSpan, +} from "@karakeep/shared-server"; +import serverConfig from "@karakeep/shared/config"; + +import type { Context } from "../index"; + +const tracer = getTracer("@karakeep/trpc"); + +/** + * tRPC middleware that creates a span for each procedure call. + * This integrates OpenTelemetry tracing into the tRPC layer. + */ +export function createTracingMiddleware() { + return async function tracingMiddleware<T>(opts: { + ctx: Context; + type: "query" | "mutation" | "subscription"; + path: string; + input: unknown; + next: () => Promise<T>; + }): Promise<T> { + // Skip if tracing is disabled + if (!serverConfig.tracing.enabled) { + return opts.next(); + } + + const spanName = `trpc.${opts.type}.${opts.path}`; + + return withSpan( + tracer, + spanName, + { + kind: SpanKind.SERVER, + attributes: { + "rpc.system": "trpc", + "rpc.method": opts.path, + "rpc.type": opts.type, + "user.id": opts.ctx.user?.id ?? "anonymous", + "user.role": opts.ctx.user?.role ?? "none", + }, + }, + async () => { + return await opts.next(); + }, + ); + }; +} + +/** + * Helper to add tracing attributes within a tRPC procedure. + * Use this to add custom attributes to the current span. + */ +export function addTracingAttributes( + attributes: Record<string, string | number | boolean>, +): void { + if (serverConfig.tracing.enabled) { + setSpanAttributes(attributes); + } +} diff --git a/packages/trpc/models/assets.ts b/packages/trpc/models/assets.ts new file mode 100644 index 00000000..f97cfffb --- /dev/null +++ b/packages/trpc/models/assets.ts @@ -0,0 +1,282 @@ +import { TRPCError } from "@trpc/server"; +import { and, desc, eq, sql } from "drizzle-orm"; +import { z } from "zod"; + +import { assets } from "@karakeep/db/schema"; +import { deleteAsset } from "@karakeep/shared/assetdb"; +import serverConfig from "@karakeep/shared/config"; +import { createSignedToken } from "@karakeep/shared/signedTokens"; +import { zAssetSignedTokenSchema } from "@karakeep/shared/types/assets"; +import { zAssetTypesSchema } from "@karakeep/shared/types/bookmarks"; +import { getAssetUrl } from "@karakeep/shared/utils/assetUtils"; + +import { AuthedContext } from ".."; +import { + isAllowedToAttachAsset, + isAllowedToDetachAsset, + mapDBAssetTypeToUserType, + mapSchemaAssetTypeToDB, +} from "../lib/attachments"; +import { BareBookmark } from "./bookmarks"; + +export class Asset { + constructor( + protected ctx: AuthedContext, + public asset: typeof assets.$inferSelect, + ) {} + + static async fromId(ctx: AuthedContext, id: string): Promise<Asset> { + const assetdb = await ctx.db.query.assets.findFirst({ + where: eq(assets.id, id), + }); + + if (!assetdb) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Asset not found", + }); + } + + const asset = new Asset(ctx, assetdb); + + if (!(await asset.canUserView())) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Asset not found", + }); + } + + return asset; + } + + static async list( + ctx: AuthedContext, + input: { + limit: number; + cursor: number | null; + }, + ) { + const page = input.cursor ?? 1; + const [results, totalCount] = await Promise.all([ + ctx.db + .select() + .from(assets) + .where(eq(assets.userId, ctx.user.id)) + .orderBy(desc(assets.size)) + .limit(input.limit) + .offset((page - 1) * input.limit), + ctx.db + .select({ count: sql<number>`count(*)` }) + .from(assets) + .where(eq(assets.userId, ctx.user.id)), + ]); + + return { + assets: results.map((a) => ({ + ...a, + assetType: mapDBAssetTypeToUserType(a.assetType), + })), + nextCursor: page * input.limit < totalCount[0].count ? page + 1 : null, + totalCount: totalCount[0].count, + }; + } + + static async attachAsset( + ctx: AuthedContext, + input: { + bookmarkId: string; + asset: { + id: string; + assetType: z.infer<typeof zAssetTypesSchema>; + }; + }, + ) { + const [asset] = await Promise.all([ + Asset.fromId(ctx, input.asset.id), + this.ensureBookmarkOwnership(ctx, input.bookmarkId), + ]); + asset.ensureOwnership(); + + if (!isAllowedToAttachAsset(input.asset.assetType)) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "You can't attach this type of asset", + }); + } + + const [updatedAsset] = await ctx.db + .update(assets) + .set({ + assetType: mapSchemaAssetTypeToDB(input.asset.assetType), + bookmarkId: input.bookmarkId, + }) + .where(and(eq(assets.id, input.asset.id), eq(assets.userId, ctx.user.id))) + .returning(); + + return { + id: updatedAsset.id, + assetType: mapDBAssetTypeToUserType(updatedAsset.assetType), + fileName: updatedAsset.fileName, + }; + } + + static async replaceAsset( + ctx: AuthedContext, + input: { + bookmarkId: string; + oldAssetId: string; + newAssetId: string; + }, + ) { + const [oldAsset, newAsset] = await Promise.all([ + Asset.fromId(ctx, input.oldAssetId), + Asset.fromId(ctx, input.newAssetId), + this.ensureBookmarkOwnership(ctx, input.bookmarkId), + ]); + oldAsset.ensureOwnership(); + newAsset.ensureOwnership(); + + if ( + !isAllowedToAttachAsset( + mapDBAssetTypeToUserType(oldAsset.asset.assetType), + ) + ) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "You can't attach this type of asset", + }); + } + + await ctx.db.transaction(async (tx) => { + await tx.delete(assets).where(eq(assets.id, input.oldAssetId)); + await tx + .update(assets) + .set({ + bookmarkId: input.bookmarkId, + assetType: oldAsset.asset.assetType, + }) + .where(eq(assets.id, input.newAssetId)); + }); + + await deleteAsset({ + userId: ctx.user.id, + assetId: input.oldAssetId, + }).catch(() => ({})); + } + + static async detachAsset( + ctx: AuthedContext, + input: { + bookmarkId: string; + assetId: string; + }, + ) { + const [asset] = await Promise.all([ + Asset.fromId(ctx, input.assetId), + this.ensureBookmarkOwnership(ctx, input.bookmarkId), + ]); + + if ( + !isAllowedToDetachAsset(mapDBAssetTypeToUserType(asset.asset.assetType)) + ) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "You can't detach this type of asset", + }); + } + + const result = await ctx.db + .delete(assets) + .where( + and( + eq(assets.id, input.assetId), + eq(assets.bookmarkId, input.bookmarkId), + ), + ); + if (result.changes == 0) { + throw new TRPCError({ code: "NOT_FOUND" }); + } + await deleteAsset({ userId: ctx.user.id, assetId: input.assetId }).catch( + () => ({}), + ); + } + + private static async ensureBookmarkOwnership( + ctx: AuthedContext, + bookmarkId: string, + ) { + const bookmark = await BareBookmark.bareFromId(ctx, bookmarkId); + bookmark.ensureOwnership(); + } + + ensureOwnership() { + if (this.asset.userId != this.ctx.user.id) { + throw new TRPCError({ + code: "FORBIDDEN", + message: "User is not allowed to access resource", + }); + } + } + + static async ensureOwnership(ctx: AuthedContext, assetId: string) { + return (await Asset.fromId(ctx, assetId)).ensureOwnership(); + } + + async canUserView(): Promise<boolean> { + // Asset owner can always view it + if (this.asset.userId === this.ctx.user.id) { + return true; + } + + // Avatars are always public + if (this.asset.assetType === "avatar") { + return true; + } + + // If asset is attached to a bookmark, check bookmark access permissions + if (this.asset.bookmarkId) { + try { + // This throws if the user doesn't have access to the bookmark + await BareBookmark.bareFromId(this.ctx, this.asset.bookmarkId); + return true; + } catch (e) { + if (e instanceof TRPCError && e.code === "FORBIDDEN") { + return false; + } + throw e; + } + } + + return false; + } + + async ensureCanView() { + if (!(await this.canUserView())) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Asset not found", + }); + } + } + + getUrl() { + return getAssetUrl(this.asset.id); + } + + static getPublicSignedAssetUrl( + assetId: string, + assetOwnerId: string, + expireAt: number, + ) { + const payload: z.infer<typeof zAssetSignedTokenSchema> = { + assetId, + userId: assetOwnerId, + }; + const signedToken = createSignedToken( + payload, + serverConfig.signingSecret(), + expireAt, + ); + return `${serverConfig.publicApiUrl}/public/assets/${assetId}?token=${signedToken}`; + } +} diff --git a/packages/trpc/models/bookmarks.ts b/packages/trpc/models/bookmarks.ts index 07fa8693..c8cd1f00 100644 --- a/packages/trpc/models/bookmarks.ts +++ b/packages/trpc/models/bookmarks.ts @@ -4,13 +4,14 @@ import { asc, desc, eq, - exists, + getTableColumns, gt, gte, inArray, lt, lte, or, + SQL, } from "drizzle-orm"; import invariant from "tiny-invariant"; import { z } from "zod"; @@ -21,23 +22,16 @@ import { AssetTypes, bookmarkAssets, bookmarkLinks, - bookmarkLists, bookmarks, bookmarksInLists, bookmarkTags, bookmarkTexts, - listCollaborators, rssFeedImportsTable, tagsOnBookmarks, } from "@karakeep/db/schema"; import { SearchIndexingQueue, triggerWebhook } from "@karakeep/shared-server"; import { deleteAsset, readAsset } from "@karakeep/shared/assetdb"; -import serverConfig from "@karakeep/shared/config"; -import { - createSignedToken, - getAlignedExpiry, -} from "@karakeep/shared/signedTokens"; -import { zAssetSignedTokenSchema } from "@karakeep/shared/types/assets"; +import { getAlignedExpiry } from "@karakeep/shared/signedTokens"; import { BookmarkTypes, DEFAULT_NUM_BOOKMARKS_PER_PAGE, @@ -56,6 +50,7 @@ import { htmlToPlainText } from "@karakeep/shared/utils/htmlUtils"; import { AuthedContext } from ".."; import { mapDBAssetTypeToUserType } from "../lib/attachments"; +import { Asset } from "./assets"; import { List } from "./lists"; async function dummyDrizzleReturnType() { @@ -162,6 +157,7 @@ export class Bookmark extends BareBookmark { screenshotAssetId: assets.find( (a) => a.assetType == AssetTypes.LINK_SCREENSHOT, )?.id, + pdfAssetId: assets.find((a) => a.assetType == AssetTypes.LINK_PDF)?.id, fullPageArchiveAssetId: assets.find( (a) => a.assetType == AssetTypes.LINK_FULL_PAGE_ARCHIVE, )?.id, @@ -182,6 +178,7 @@ export class Bookmark extends BareBookmark { ? await Bookmark.getBookmarkHtmlContent(link, bookmark.userId) : null, crawledAt: link.crawledAt, + crawlStatus: link.crawlStatus, author: link.author, publisher: link.publisher, datePublished: link.datePublished, @@ -270,6 +267,130 @@ export class Bookmark extends BareBookmark { return new Bookmark(ctx, data); } + static async buildDebugInfo(ctx: AuthedContext, bookmarkId: string) { + // Verify the user is an admin + if (ctx.user.role !== "admin") { + throw new TRPCError({ + code: "FORBIDDEN", + message: "Admin access required", + }); + } + + const PRIVACY_REDACTED_ASSET_TYPES = new Set<AssetTypes>([ + AssetTypes.USER_UPLOADED, + AssetTypes.BOOKMARK_ASSET, + ]); + + const bookmark = await ctx.db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, bookmarkId), + with: { + link: true, + text: true, + asset: true, + tagsOnBookmarks: { + with: { + tag: true, + }, + }, + assets: true, + }, + }); + + if (!bookmark) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Bookmark not found", + }); + } + + // Build link info + let linkInfo = null; + if (bookmark.link) { + const htmlContentPreview = await (async () => { + try { + const content = await Bookmark.getBookmarkHtmlContent( + bookmark.link!, + bookmark.userId, + ); + return content ? content.substring(0, 1000) : null; + } catch { + return null; + } + })(); + + linkInfo = { + url: bookmark.link.url, + crawlStatus: bookmark.link.crawlStatus ?? "pending", + crawlStatusCode: bookmark.link.crawlStatusCode, + crawledAt: bookmark.link.crawledAt, + hasHtmlContent: !!bookmark.link.htmlContent, + hasContentAsset: !!bookmark.link.contentAssetId, + htmlContentPreview, + }; + } + + // Build text info + let textInfo = null; + if (bookmark.text) { + textInfo = { + hasText: !!bookmark.text.text, + sourceUrl: bookmark.text.sourceUrl, + }; + } + + // Build asset info + let assetInfo = null; + if (bookmark.asset) { + assetInfo = { + assetType: bookmark.asset.assetType, + hasContent: !!bookmark.asset.content, + fileName: bookmark.asset.fileName, + }; + } + + // Build tags + const tags = bookmark.tagsOnBookmarks.map((t) => ({ + id: t.tag.id, + name: t.tag.name, + attachedBy: t.attachedBy, + })); + + // Build assets list with signed URLs (exclude userUploaded) + const assetsWithUrls = bookmark.assets.map((a) => { + // Generate signed token with 10 mins expiry + const expiresAt = Date.now() + 10 * 60 * 1000; // 10 mins + // Exclude userUploaded assets for privacy reasons + const url = !PRIVACY_REDACTED_ASSET_TYPES.has(a.assetType) + ? Asset.getPublicSignedAssetUrl(a.id, bookmark.userId, expiresAt) + : null; + + return { + id: a.id, + assetType: a.assetType, + size: a.size, + url, + }; + }); + + return { + id: bookmark.id, + type: bookmark.type, + source: bookmark.source, + createdAt: bookmark.createdAt, + modifiedAt: bookmark.modifiedAt, + title: bookmark.title, + summary: bookmark.summary, + taggingStatus: bookmark.taggingStatus, + summarizationStatus: bookmark.summarizationStatus, + userId: bookmark.userId, + linkInfo, + textInfo, + assetInfo, + tags, + assets: assetsWithUrls, + }; + } + static async loadMulti( ctx: AuthedContext, input: z.infer<typeof zGetBookmarksRequestSchema>, @@ -283,6 +404,21 @@ export class Bookmark extends BareBookmark { if (!input.limit) { input.limit = DEFAULT_NUM_BOOKMARKS_PER_PAGE; } + + // Validate that only one of listId, tagId, or rssFeedId is specified + // Combined filters are not supported as they would require different query strategies + const filterCount = [input.listId, input.tagId, input.rssFeedId].filter( + (f) => f !== undefined, + ).length; + if (filterCount > 1) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: + "Cannot filter by multiple of listId, tagId, and rssFeedId simultaneously", + }); + } + + // Handle smart lists by converting to bookmark IDs if (input.listId) { const list = await List.fromId(ctx, input.listId); if (list.type === "smart") { @@ -291,121 +427,132 @@ export class Bookmark extends BareBookmark { } } - const sq = ctx.db.$with("bookmarksSq").as( - ctx.db - .select() - .from(bookmarks) - .where( + // Build cursor condition for pagination + const buildCursorCondition = ( + createdAtCol: typeof bookmarks.createdAt, + idCol: typeof bookmarks.id, + ): SQL | undefined => { + if (!input.cursor) return undefined; + + if (input.sortOrder === "asc") { + return or( + gt(createdAtCol, input.cursor.createdAt), and( - // Access control: User can access bookmarks if they either: - // 1. Own the bookmark (always) - // 2. The bookmark is in a specific shared list being viewed - // When listId is specified, we need special handling to show all bookmarks in that list - input.listId !== undefined - ? // If querying a specific list, check if user has access to that list - or( - eq(bookmarks.userId, ctx.user.id), - // User is the owner of the list being queried - exists( - ctx.db - .select() - .from(bookmarkLists) - .where( - and( - eq(bookmarkLists.id, input.listId), - eq(bookmarkLists.userId, ctx.user.id), - ), - ), - ), - // User is a collaborator on the list being queried - exists( - ctx.db - .select() - .from(listCollaborators) - .where( - and( - eq(listCollaborators.listId, input.listId), - eq(listCollaborators.userId, ctx.user.id), - ), - ), - ), - ) - : // If not querying a specific list, only show bookmarks the user owns - // Shared bookmarks should only appear when viewing the specific shared list - eq(bookmarks.userId, ctx.user.id), - input.archived !== undefined - ? eq(bookmarks.archived, input.archived) - : undefined, - input.favourited !== undefined - ? eq(bookmarks.favourited, input.favourited) - : undefined, - input.ids ? inArray(bookmarks.id, input.ids) : undefined, - input.tagId !== undefined - ? exists( - ctx.db - .select() - .from(tagsOnBookmarks) - .where( - and( - eq(tagsOnBookmarks.bookmarkId, bookmarks.id), - eq(tagsOnBookmarks.tagId, input.tagId), - ), - ), - ) - : undefined, - input.rssFeedId !== undefined - ? exists( - ctx.db - .select() - .from(rssFeedImportsTable) - .where( - and( - eq(rssFeedImportsTable.bookmarkId, bookmarks.id), - eq(rssFeedImportsTable.rssFeedId, input.rssFeedId), - ), - ), - ) - : undefined, - input.listId !== undefined - ? exists( - ctx.db - .select() - .from(bookmarksInLists) - .where( - and( - eq(bookmarksInLists.bookmarkId, bookmarks.id), - eq(bookmarksInLists.listId, input.listId), - ), - ), - ) - : undefined, - input.cursor - ? input.sortOrder === "asc" - ? or( - gt(bookmarks.createdAt, input.cursor.createdAt), - and( - eq(bookmarks.createdAt, input.cursor.createdAt), - gte(bookmarks.id, input.cursor.id), - ), - ) - : or( - lt(bookmarks.createdAt, input.cursor.createdAt), - and( - eq(bookmarks.createdAt, input.cursor.createdAt), - lte(bookmarks.id, input.cursor.id), - ), - ) - : undefined, + eq(createdAtCol, input.cursor.createdAt), + gte(idCol, input.cursor.id), ), - ) - .limit(input.limit + 1) - .orderBy( - input.sortOrder === "asc" - ? asc(bookmarks.createdAt) - : desc(bookmarks.createdAt), - desc(bookmarks.id), + ); + } + return or( + lt(createdAtCol, input.cursor.createdAt), + and( + eq(createdAtCol, input.cursor.createdAt), + lte(idCol, input.cursor.id), ), - ); + ); + }; + + // Build common filter conditions (archived, favourited, ids) + const buildCommonFilters = (): (SQL | undefined)[] => [ + input.archived !== undefined + ? eq(bookmarks.archived, input.archived) + : undefined, + input.favourited !== undefined + ? eq(bookmarks.favourited, input.favourited) + : undefined, + input.ids ? inArray(bookmarks.id, input.ids) : undefined, + ]; + + // Build ORDER BY clause + const buildOrderBy = () => + [ + input.sortOrder === "asc" + ? asc(bookmarks.createdAt) + : desc(bookmarks.createdAt), + desc(bookmarks.id), + ] as const; + + // Choose query strategy based on filters + // Strategy: Use the most selective filter as the driving table + let sq; + + if (input.listId !== undefined) { + // PATH: List filter - start from bookmarksInLists (more selective) + // Access control is already verified by List.fromId() called above + sq = ctx.db.$with("bookmarksSq").as( + ctx.db + .select(getTableColumns(bookmarks)) + .from(bookmarksInLists) + .innerJoin(bookmarks, eq(bookmarks.id, bookmarksInLists.bookmarkId)) + .where( + and( + eq(bookmarksInLists.listId, input.listId), + ...buildCommonFilters(), + buildCursorCondition(bookmarks.createdAt, bookmarks.id), + ), + ) + .limit(input.limit + 1) + .orderBy(...buildOrderBy()), + ); + } else if (input.tagId !== undefined) { + // PATH: Tag filter - start from tagsOnBookmarks (more selective) + sq = ctx.db.$with("bookmarksSq").as( + ctx.db + .select(getTableColumns(bookmarks)) + .from(tagsOnBookmarks) + .innerJoin(bookmarks, eq(bookmarks.id, tagsOnBookmarks.bookmarkId)) + .where( + and( + eq(tagsOnBookmarks.tagId, input.tagId), + eq(bookmarks.userId, ctx.user.id), // Access control + ...buildCommonFilters(), + buildCursorCondition(bookmarks.createdAt, bookmarks.id), + ), + ) + .limit(input.limit + 1) + .orderBy(...buildOrderBy()), + ); + } else if (input.rssFeedId !== undefined) { + // PATH: RSS feed filter - start from rssFeedImportsTable (more selective) + sq = ctx.db.$with("bookmarksSq").as( + ctx.db + .select(getTableColumns(bookmarks)) + .from(rssFeedImportsTable) + .innerJoin( + bookmarks, + eq(bookmarks.id, rssFeedImportsTable.bookmarkId), + ) + .where( + and( + eq(rssFeedImportsTable.rssFeedId, input.rssFeedId), + eq(bookmarks.userId, ctx.user.id), // Access control + ...buildCommonFilters(), + buildCursorCondition(bookmarks.createdAt, bookmarks.id), + ), + ) + .limit(input.limit + 1) + .orderBy(...buildOrderBy()), + ); + } else { + // PATH: No list/tag/rssFeed filter - query bookmarks directly + // Uses composite index: bookmarks_userId_createdAt_id_idx (or archived/favourited variants) + sq = ctx.db.$with("bookmarksSq").as( + ctx.db + .select() + .from(bookmarks) + .where( + and( + eq(bookmarks.userId, ctx.user.id), + ...buildCommonFilters(), + buildCursorCondition(bookmarks.createdAt, bookmarks.id), + ), + ) + .limit(input.limit + 1) + .orderBy(...buildOrderBy()), + ); + } + + // Execute the query with joins for related data // TODO: Consider not inlining the tags in the response of getBookmarks as this query is getting kinda expensive const results = await ctx.db .with(sq) @@ -438,6 +585,7 @@ export class Bookmark extends BareBookmark { : row.bookmarkLinks.htmlContent : null, contentAssetId: row.bookmarkLinks.contentAssetId, + crawlStatus: row.bookmarkLinks.crawlStatus, crawledAt: row.bookmarkLinks.crawledAt, author: row.bookmarkLinks.author, publisher: row.bookmarkLinks.publisher, @@ -500,6 +648,9 @@ export class Bookmark extends BareBookmark { if (row.assets.assetType == AssetTypes.LINK_SCREENSHOT) { content.screenshotAssetId = row.assets.id; } + if (row.assets.assetType == AssetTypes.LINK_PDF) { + content.pdfAssetId = row.assets.id; + } if (row.assets.assetType == AssetTypes.LINK_FULL_PAGE_ARCHIVE) { content.fullPageArchiveAssetId = row.assets.id; } @@ -610,17 +761,12 @@ export class Bookmark extends BareBookmark { asPublicBookmark(): ZPublicBookmark { const getPublicSignedAssetUrl = (assetId: string) => { - const payload: z.infer<typeof zAssetSignedTokenSchema> = { + // Tokens will expire in 1 hour and will have a grace period of 15mins + return Asset.getPublicSignedAssetUrl( assetId, - userId: this.ctx.user.id, - }; - const signedToken = createSignedToken( - payload, - serverConfig.signingSecret(), - // Tokens will expire in 1 hour and will have a grace period of 15mins - getAlignedExpiry(/* interval */ 3600, /* grace */ 900), + this.bookmark.userId, + getAlignedExpiry(3600, 900), ); - return `${serverConfig.publicApiUrl}/public/assets/${assetId}?token=${signedToken}`; }; const getContent = ( content: ZBookmarkContent, diff --git a/packages/trpc/models/feeds.ts b/packages/trpc/models/feeds.ts index c0828bbf..ea22da8f 100644 --- a/packages/trpc/models/feeds.ts +++ b/packages/trpc/models/feeds.ts @@ -1,8 +1,9 @@ import { TRPCError } from "@trpc/server"; -import { and, eq } from "drizzle-orm"; +import { and, count, eq } from "drizzle-orm"; import { z } from "zod"; import { rssFeedsTable } from "@karakeep/db/schema"; +import serverConfig from "@karakeep/shared/config"; import { zFeedSchema, zNewFeedSchema, @@ -44,6 +45,20 @@ export class Feed { ctx: AuthedContext, input: z.infer<typeof zNewFeedSchema>, ): Promise<Feed> { + // Check if user has reached the maximum number of feeds + const [feedCount] = await ctx.db + .select({ count: count() }) + .from(rssFeedsTable) + .where(eq(rssFeedsTable.userId, ctx.user.id)); + + const maxFeeds = serverConfig.feeds.maxRssFeedsPerUser; + if (feedCount.count >= maxFeeds) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: `Maximum number of RSS feeds (${maxFeeds}) reached`, + }); + } + const [result] = await ctx.db .insert(rssFeedsTable) .values({ diff --git a/packages/trpc/models/importSessions.ts b/packages/trpc/models/importSessions.ts index c324cf7f..ee0eb5b2 100644 --- a/packages/trpc/models/importSessions.ts +++ b/packages/trpc/models/importSessions.ts @@ -2,12 +2,7 @@ import { TRPCError } from "@trpc/server"; import { and, count, eq } from "drizzle-orm"; import { z } from "zod"; -import { - bookmarkLinks, - bookmarks, - importSessionBookmarks, - importSessions, -} from "@karakeep/db/schema"; +import { importSessions, importStagingBookmarks } from "@karakeep/db/schema"; import { zCreateImportSessionRequestSchema, ZImportSession, @@ -81,38 +76,17 @@ export class ImportSession { ); } - async attachBookmark(bookmarkId: string): Promise<void> { - await this.ctx.db.insert(importSessionBookmarks).values({ - importSessionId: this.session.id, - bookmarkId, - }); - } - async getWithStats(): Promise<ZImportSessionWithStats> { - // Get bookmark counts by status + // Count by staging status - this now reflects the true state since + // items stay in "processing" until downstream crawl/tag is complete const statusCounts = await this.ctx.db .select({ - crawlStatus: bookmarkLinks.crawlStatus, - taggingStatus: bookmarks.taggingStatus, + status: importStagingBookmarks.status, count: count(), }) - .from(importSessionBookmarks) - .innerJoin( - importSessions, - eq(importSessions.id, importSessionBookmarks.importSessionId), - ) - .leftJoin(bookmarks, eq(bookmarks.id, importSessionBookmarks.bookmarkId)) - .leftJoin( - bookmarkLinks, - eq(bookmarkLinks.id, importSessionBookmarks.bookmarkId), - ) - .where( - and( - eq(importSessionBookmarks.importSessionId, this.session.id), - eq(importSessions.userId, this.ctx.user.id), - ), - ) - .groupBy(bookmarkLinks.crawlStatus, bookmarks.taggingStatus); + .from(importStagingBookmarks) + .where(eq(importStagingBookmarks.importSessionId, this.session.id)) + .groupBy(importStagingBookmarks.status); const stats = { totalBookmarks: 0, @@ -122,41 +96,27 @@ export class ImportSession { processingBookmarks: 0, }; - statusCounts.forEach((statusCount) => { - const { crawlStatus, taggingStatus, count } = statusCount; - - stats.totalBookmarks += count; - - const isCrawlFailure = crawlStatus === "failure"; - const isTagFailure = taggingStatus === "failure"; - if (isCrawlFailure || isTagFailure) { - stats.failedBookmarks += count; - return; - } - - const isCrawlPending = crawlStatus === "pending"; - const isTagPending = taggingStatus === "pending"; - if (isCrawlPending || isTagPending) { - stats.pendingBookmarks += count; - return; - } - - const isCrawlSuccessfulOrNotRequired = - crawlStatus === "success" || crawlStatus === null; - const isTagSuccessfulOrUnknown = - taggingStatus === "success" || taggingStatus === null; + statusCounts.forEach(({ status, count: itemCount }) => { + stats.totalBookmarks += itemCount; - if (isCrawlSuccessfulOrNotRequired && isTagSuccessfulOrUnknown) { - stats.completedBookmarks += count; - } else { - // Fallback to pending to avoid leaving imports unclassified - stats.pendingBookmarks += count; + switch (status) { + case "pending": + stats.pendingBookmarks += itemCount; + break; + case "processing": + stats.processingBookmarks += itemCount; + break; + case "completed": + stats.completedBookmarks += itemCount; + break; + case "failed": + stats.failedBookmarks += itemCount; + break; } }); return { ...this.session, - status: stats.pendingBookmarks > 0 ? "in_progress" : "completed", ...stats, }; } @@ -179,4 +139,92 @@ export class ImportSession { }); } } + + async stageBookmarks( + bookmarks: { + type: "link" | "text" | "asset"; + url?: string; + title?: string; + content?: string; + note?: string; + tags: string[]; + listIds: string[]; + sourceAddedAt?: Date; + }[], + ): Promise<void> { + if (this.session.status !== "staging") { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Session not in staging status", + }); + } + + // Filter out invalid bookmarks (link without url, text without content) + const validBookmarks = bookmarks.filter((bookmark) => { + if (bookmark.type === "link" && !bookmark.url) return false; + if (bookmark.type === "text" && !bookmark.content) return false; + return true; + }); + + if (validBookmarks.length === 0) { + return; + } + + await this.ctx.db.insert(importStagingBookmarks).values( + validBookmarks.map((bookmark) => ({ + importSessionId: this.session.id, + type: bookmark.type, + url: bookmark.url, + title: bookmark.title, + content: bookmark.content, + note: bookmark.note, + tags: bookmark.tags, + listIds: bookmark.listIds, + sourceAddedAt: bookmark.sourceAddedAt, + status: "pending" as const, + })), + ); + } + + async finalize(): Promise<void> { + if (this.session.status !== "staging") { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Session not in staging status", + }); + } + + await this.ctx.db + .update(importSessions) + .set({ status: "pending" }) + .where(eq(importSessions.id, this.session.id)); + } + + async pause(): Promise<void> { + if (!["pending", "running"].includes(this.session.status)) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Session cannot be paused in current status", + }); + } + + await this.ctx.db + .update(importSessions) + .set({ status: "paused" }) + .where(eq(importSessions.id, this.session.id)); + } + + async resume(): Promise<void> { + if (this.session.status !== "paused") { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Session not paused", + }); + } + + await this.ctx.db + .update(importSessions) + .set({ status: "pending" }) + .where(eq(importSessions.id, this.session.id)); + } } diff --git a/packages/trpc/models/listInvitations.ts b/packages/trpc/models/listInvitations.ts index 6bdc8ffa..2e17fa2e 100644 --- a/packages/trpc/models/listInvitations.ts +++ b/packages/trpc/models/listInvitations.ts @@ -372,6 +372,7 @@ export class ListInvitation { // This protects user privacy until they accept name: "Pending User", email: invitation.user.email || "", + image: null, }, })); } diff --git a/packages/trpc/models/lists.ts b/packages/trpc/models/lists.ts index 0968492a..10d7d9bf 100644 --- a/packages/trpc/models/lists.ts +++ b/packages/trpc/models/lists.ts @@ -719,6 +719,7 @@ export abstract class List { id: true, name: true, email: true, + image: true, }, }, }, @@ -738,6 +739,7 @@ export abstract class List { id: true, name: true, email: true, + image: true, }, }); @@ -754,6 +756,7 @@ export abstract class List { name: c.user.name, // Only show email to the owner for privacy email: isOwner ? c.user.email : null, + image: c.user.image, }, }; }); @@ -766,6 +769,7 @@ export abstract class List { name: owner.name, // Only show owner email to the owner for privacy email: isOwner ? owner.email : null, + image: owner.image, } : null, }; @@ -805,8 +809,8 @@ export abstract class List { } abstract get type(): "manual" | "smart"; - abstract getBookmarkIds(ctx: AuthedContext): Promise<string[]>; - abstract getSize(ctx: AuthedContext): Promise<number>; + abstract getBookmarkIds(visitedListIds?: Set<string>): Promise<string[]>; + abstract getSize(): Promise<number>; abstract addBookmark(bookmarkId: string): Promise<void>; abstract removeBookmark(bookmarkId: string): Promise<void>; abstract mergeInto( @@ -816,6 +820,8 @@ export abstract class List { } export class SmartList extends List { + private static readonly MAX_VISITED_LISTS = 30; + parsedQuery: ReturnType<typeof parseSearchQuery> | null = null; constructor(ctx: AuthedContext, list: ZBookmarkList & { userId: string }) { @@ -843,12 +849,27 @@ export class SmartList extends List { return this.parsedQuery; } - async getBookmarkIds(): Promise<string[]> { + async getBookmarkIds(visitedListIds = new Set<string>()): Promise<string[]> { + if (visitedListIds.size >= SmartList.MAX_VISITED_LISTS) { + return []; + } + + if (visitedListIds.has(this.list.id)) { + return []; + } + + const newVisitedListIds = new Set(visitedListIds); + newVisitedListIds.add(this.list.id); + const parsedQuery = this.getParsedQuery(); if (!parsedQuery.matcher) { return []; } - return await getBookmarkIdsFromMatcher(this.ctx, parsedQuery.matcher); + return await getBookmarkIdsFromMatcher( + this.ctx, + parsedQuery.matcher, + newVisitedListIds, + ); } async getSize(): Promise<number> { @@ -894,7 +915,7 @@ export class ManualList extends List { return this.list.type; } - async getBookmarkIds(): Promise<string[]> { + async getBookmarkIds(_visitedListIds?: Set<string>): Promise<string[]> { const results = await this.ctx.db .select({ id: bookmarksInLists.bookmarkId }) .from(bookmarksInLists) diff --git a/packages/trpc/models/tags.ts b/packages/trpc/models/tags.ts index 55532077..1d8f90b9 100644 --- a/packages/trpc/models/tags.ts +++ b/packages/trpc/models/tags.ts @@ -85,6 +85,7 @@ export class Tag { ctx: AuthedContext, opts: { nameContains?: string; + ids?: string[]; attachedBy?: "ai" | "human" | "none"; sortBy?: "name" | "usage" | "relevance"; pagination?: { @@ -119,6 +120,9 @@ export class Tag { opts.nameContains ? like(bookmarkTags.name, `%${opts.nameContains}%`) : undefined, + opts.ids && opts.ids.length > 0 + ? inArray(bookmarkTags.id, opts.ids) + : undefined, ), ) .groupBy(bookmarkTags.id, bookmarkTags.name) diff --git a/packages/trpc/models/users.ts b/packages/trpc/models/users.ts index a1f32f02..3340956a 100644 --- a/packages/trpc/models/users.ts +++ b/packages/trpc/models/users.ts @@ -1,12 +1,13 @@ import { randomBytes } from "crypto"; import { TRPCError } from "@trpc/server"; -import { and, count, desc, eq, gte, sql } from "drizzle-orm"; +import { and, count, desc, eq, gte, lte, sql } from "drizzle-orm"; import invariant from "tiny-invariant"; import { z } from "zod"; import { SqliteError } from "@karakeep/db"; import { assets, + AssetTypes, bookmarkLinks, bookmarkLists, bookmarks, @@ -17,7 +18,7 @@ import { users, verificationTokens, } from "@karakeep/db/schema"; -import { deleteUserAssets } from "@karakeep/shared/assetdb"; +import { deleteAsset, deleteUserAssets } from "@karakeep/shared/assetdb"; import serverConfig from "@karakeep/shared/config"; import { zResetPasswordSchema, @@ -26,6 +27,7 @@ import { zUserSettingsSchema, zUserStatsResponseSchema, zWhoAmIResponseSchema, + zWrappedStatsResponseSchema, } from "@karakeep/shared/types/users"; import { AuthedContext, Context } from ".."; @@ -59,7 +61,7 @@ export class User { static async create( ctx: Context, - input: z.infer<typeof zSignUpSchema>, + input: z.infer<typeof zSignUpSchema> & { redirectUrl?: string }, role?: "user" | "admin", ) { const salt = generatePasswordSalt(); @@ -74,7 +76,12 @@ export class User { if (serverConfig.auth.emailVerificationRequired) { const token = await User.genEmailVerificationToken(ctx.db, input.email); try { - await sendVerificationEmail(input.email, input.name, token); + await sendVerificationEmail( + input.email, + input.name, + token, + input.redirectUrl, + ); } catch (error) { console.error("Failed to send verification email:", error); } @@ -225,6 +232,7 @@ export class User { static async resendVerificationEmail( ctx: Context, email: string, + redirectUrl?: string, ): Promise<void> { if ( !serverConfig.auth.emailVerificationRequired || @@ -253,7 +261,7 @@ export class User { const token = await User.genEmailVerificationToken(ctx.db, email); try { - await sendVerificationEmail(email, user.name, token); + await sendVerificationEmail(email, user.name, token, redirectUrl); } catch (error) { console.error("Failed to send verification email:", error); throw new TRPCError({ @@ -433,6 +441,14 @@ export class User { backupsEnabled: true, backupsFrequency: true, backupsRetentionDays: true, + readerFontSize: true, + readerLineHeight: true, + readerFontFamily: true, + autoTaggingEnabled: true, + autoSummarizationEnabled: true, + tagStyle: true, + curatedTagIds: true, + inferredTagLang: true, }, }); @@ -450,6 +466,14 @@ export class User { backupsEnabled: settings.backupsEnabled, backupsFrequency: settings.backupsFrequency, backupsRetentionDays: settings.backupsRetentionDays, + readerFontSize: settings.readerFontSize, + readerLineHeight: settings.readerLineHeight, + readerFontFamily: settings.readerFontFamily, + autoTaggingEnabled: settings.autoTaggingEnabled, + autoSummarizationEnabled: settings.autoSummarizationEnabled, + tagStyle: settings.tagStyle ?? "as-generated", + curatedTagIds: settings.curatedTagIds ?? null, + inferredTagLang: settings.inferredTagLang, }; } @@ -472,10 +496,116 @@ export class User { backupsEnabled: input.backupsEnabled, backupsFrequency: input.backupsFrequency, backupsRetentionDays: input.backupsRetentionDays, + readerFontSize: input.readerFontSize, + readerLineHeight: input.readerLineHeight, + readerFontFamily: input.readerFontFamily, + autoTaggingEnabled: input.autoTaggingEnabled, + autoSummarizationEnabled: input.autoSummarizationEnabled, + tagStyle: input.tagStyle, + curatedTagIds: input.curatedTagIds, + inferredTagLang: input.inferredTagLang, }) .where(eq(users.id, this.user.id)); } + async updateAvatar(assetId: string | null): Promise<void> { + const previousImage = this.user.image ?? null; + const [asset, previousAsset] = await Promise.all([ + assetId + ? this.ctx.db.query.assets.findFirst({ + where: and(eq(assets.id, assetId), eq(assets.userId, this.user.id)), + columns: { + id: true, + bookmarkId: true, + contentType: true, + assetType: true, + }, + }) + : Promise.resolve(null), + previousImage && previousImage !== assetId + ? this.ctx.db.query.assets.findFirst({ + where: and( + eq(assets.id, previousImage), + eq(assets.userId, this.user.id), + ), + columns: { + id: true, + bookmarkId: true, + }, + }) + : Promise.resolve(null), + ]); + + if (assetId) { + if (!asset) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Avatar asset not found", + }); + } + + if (asset.bookmarkId) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Avatar asset must not be attached to a bookmark", + }); + } + + if (asset.contentType && !asset.contentType.startsWith("image/")) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Avatar asset must be an image", + }); + } + + if ( + asset.assetType !== AssetTypes.AVATAR && + asset.assetType !== AssetTypes.UNKNOWN + ) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Avatar asset type is not supported", + }); + } + + if (asset.assetType !== AssetTypes.AVATAR) { + await this.ctx.db + .update(assets) + .set({ assetType: AssetTypes.AVATAR }) + .where(eq(assets.id, asset.id)); + } + } + if (previousImage === assetId) { + return; + } + + await this.ctx.db.transaction(async (tx) => { + await tx + .update(users) + .set({ image: assetId }) + .where(eq(users.id, this.user.id)); + + if (!previousImage || previousImage === assetId) { + return; + } + + if (previousAsset && !previousAsset.bookmarkId) { + await tx.delete(assets).where(eq(assets.id, previousAsset.id)); + } + }); + + this.user.image = assetId; + + if (!previousImage || previousImage === assetId) { + return; + } + + await deleteAsset({ + userId: this.user.id, + assetId: previousImage, + }).catch(() => ({})); + } + async getStats(): Promise<z.infer<typeof zUserStatsResponseSchema>> { const userObj = await this.ctx.db.query.users.findFirst({ where: eq(users.id, this.user.id), @@ -553,23 +683,23 @@ export class User { // Top domains this.ctx.db .select({ - domain: sql<string>`CASE - WHEN ${bookmarkLinks.url} LIKE 'https://%' THEN - CASE + domain: sql<string>`CASE + WHEN ${bookmarkLinks.url} LIKE 'https://%' THEN + CASE WHEN INSTR(SUBSTR(${bookmarkLinks.url}, 9), '/') > 0 THEN SUBSTR(${bookmarkLinks.url}, 9, INSTR(SUBSTR(${bookmarkLinks.url}, 9), '/') - 1) ELSE SUBSTR(${bookmarkLinks.url}, 9) END - WHEN ${bookmarkLinks.url} LIKE 'http://%' THEN - CASE + WHEN ${bookmarkLinks.url} LIKE 'http://%' THEN + CASE WHEN INSTR(SUBSTR(${bookmarkLinks.url}, 8), '/') > 0 THEN SUBSTR(${bookmarkLinks.url}, 8, INSTR(SUBSTR(${bookmarkLinks.url}, 8), '/') - 1) ELSE SUBSTR(${bookmarkLinks.url}, 8) END - ELSE - CASE + ELSE + CASE WHEN INSTR(${bookmarkLinks.url}, '/') > 0 THEN SUBSTR(${bookmarkLinks.url}, 1, INSTR(${bookmarkLinks.url}, '/') - 1) ELSE @@ -582,23 +712,23 @@ export class User { .innerJoin(bookmarks, eq(bookmarks.id, bookmarkLinks.id)) .where(eq(bookmarks.userId, this.user.id)) .groupBy( - sql`CASE - WHEN ${bookmarkLinks.url} LIKE 'https://%' THEN - CASE + sql`CASE + WHEN ${bookmarkLinks.url} LIKE 'https://%' THEN + CASE WHEN INSTR(SUBSTR(${bookmarkLinks.url}, 9), '/') > 0 THEN SUBSTR(${bookmarkLinks.url}, 9, INSTR(SUBSTR(${bookmarkLinks.url}, 9), '/') - 1) ELSE SUBSTR(${bookmarkLinks.url}, 9) END - WHEN ${bookmarkLinks.url} LIKE 'http://%' THEN - CASE + WHEN ${bookmarkLinks.url} LIKE 'http://%' THEN + CASE WHEN INSTR(SUBSTR(${bookmarkLinks.url}, 8), '/') > 0 THEN SUBSTR(${bookmarkLinks.url}, 8, INSTR(SUBSTR(${bookmarkLinks.url}, 8), '/') - 1) ELSE SUBSTR(${bookmarkLinks.url}, 8) END - ELSE - CASE + ELSE + CASE WHEN INSTR(${bookmarkLinks.url}, '/') > 0 THEN SUBSTR(${bookmarkLinks.url}, 1, INSTR(${bookmarkLinks.url}, '/') - 1) ELSE @@ -750,11 +880,324 @@ export class User { }; } + async hasWrapped(): Promise<boolean> { + // Check for bookmarks created in 2025 + const yearStart = new Date("2025-01-01T00:00:00Z"); + const yearEnd = new Date("2025-12-31T23:59:59Z"); + + const [{ numBookmarks }] = await this.ctx.db + .select({ + numBookmarks: count(bookmarks.id), + }) + .from(bookmarks) + .where( + and( + eq(bookmarks.userId, this.user.id), + gte(bookmarks.createdAt, yearStart), + lte(bookmarks.createdAt, yearEnd), + ), + ); + + return numBookmarks >= 20; + } + + async getWrappedStats( + year: number, + ): Promise<z.infer<typeof zWrappedStatsResponseSchema>> { + const userObj = await this.ctx.db.query.users.findFirst({ + where: eq(users.id, this.user.id), + columns: { + timezone: true, + }, + }); + const userTimezone = userObj?.timezone || "UTC"; + + // Define year range for 2025 + const yearStart = new Date(`${year}-01-01T00:00:00Z`); + const yearEnd = new Date(`${year}-12-31T23:59:59Z`); + + const yearFilter = and( + eq(bookmarks.userId, this.user.id), + gte(bookmarks.createdAt, yearStart), + lte(bookmarks.createdAt, yearEnd), + ); + + const [ + [{ totalBookmarks }], + [{ totalFavorites }], + [{ totalArchived }], + [{ numTags }], + [{ numLists }], + [{ numHighlights }], + firstBookmarkResult, + bookmarksByType, + topDomains, + topTags, + bookmarksBySource, + bookmarkTimestamps, + ] = await Promise.all([ + // Total bookmarks in year + this.ctx.db + .select({ totalBookmarks: count() }) + .from(bookmarks) + .where(yearFilter), + + // Total favorites in year + this.ctx.db + .select({ totalFavorites: count() }) + .from(bookmarks) + .where(and(yearFilter, eq(bookmarks.favourited, true))), + + // Total archived in year + this.ctx.db + .select({ totalArchived: count() }) + .from(bookmarks) + .where(and(yearFilter, eq(bookmarks.archived, true))), + + // Total unique tags (created in year) + this.ctx.db + .select({ numTags: count() }) + .from(bookmarkTags) + .where( + and( + eq(bookmarkTags.userId, this.user.id), + gte(bookmarkTags.createdAt, yearStart), + lte(bookmarkTags.createdAt, yearEnd), + ), + ), + + // Total lists (created in year) + this.ctx.db + .select({ numLists: count() }) + .from(bookmarkLists) + .where( + and( + eq(bookmarkLists.userId, this.user.id), + gte(bookmarkLists.createdAt, yearStart), + lte(bookmarkLists.createdAt, yearEnd), + ), + ), + + // Total highlights (created in year) + this.ctx.db + .select({ numHighlights: count() }) + .from(highlights) + .where( + and( + eq(highlights.userId, this.user.id), + gte(highlights.createdAt, yearStart), + lte(highlights.createdAt, yearEnd), + ), + ), + + // First bookmark of the year + this.ctx.db + .select({ + id: bookmarks.id, + title: bookmarks.title, + createdAt: bookmarks.createdAt, + type: bookmarks.type, + }) + .from(bookmarks) + .where(yearFilter) + .orderBy(bookmarks.createdAt) + .limit(1), + + // Bookmarks by type + this.ctx.db + .select({ + type: bookmarks.type, + count: count(), + }) + .from(bookmarks) + .where(yearFilter) + .groupBy(bookmarks.type), + + // Top 5 domains + this.ctx.db + .select({ + domain: sql<string>`CASE + WHEN ${bookmarkLinks.url} LIKE 'https://%' THEN + CASE + WHEN INSTR(SUBSTR(${bookmarkLinks.url}, 9), '/') > 0 THEN + SUBSTR(${bookmarkLinks.url}, 9, INSTR(SUBSTR(${bookmarkLinks.url}, 9), '/') - 1) + ELSE + SUBSTR(${bookmarkLinks.url}, 9) + END + WHEN ${bookmarkLinks.url} LIKE 'http://%' THEN + CASE + WHEN INSTR(SUBSTR(${bookmarkLinks.url}, 8), '/') > 0 THEN + SUBSTR(${bookmarkLinks.url}, 8, INSTR(SUBSTR(${bookmarkLinks.url}, 8), '/') - 1) + ELSE + SUBSTR(${bookmarkLinks.url}, 8) + END + ELSE + CASE + WHEN INSTR(${bookmarkLinks.url}, '/') > 0 THEN + SUBSTR(${bookmarkLinks.url}, 1, INSTR(${bookmarkLinks.url}, '/') - 1) + ELSE + ${bookmarkLinks.url} + END + END`, + count: count(), + }) + .from(bookmarkLinks) + .innerJoin(bookmarks, eq(bookmarks.id, bookmarkLinks.id)) + .where(yearFilter) + .groupBy( + sql`CASE + WHEN ${bookmarkLinks.url} LIKE 'https://%' THEN + CASE + WHEN INSTR(SUBSTR(${bookmarkLinks.url}, 9), '/') > 0 THEN + SUBSTR(${bookmarkLinks.url}, 9, INSTR(SUBSTR(${bookmarkLinks.url}, 9), '/') - 1) + ELSE + SUBSTR(${bookmarkLinks.url}, 9) + END + WHEN ${bookmarkLinks.url} LIKE 'http://%' THEN + CASE + WHEN INSTR(SUBSTR(${bookmarkLinks.url}, 8), '/') > 0 THEN + SUBSTR(${bookmarkLinks.url}, 8, INSTR(SUBSTR(${bookmarkLinks.url}, 8), '/') - 1) + ELSE + SUBSTR(${bookmarkLinks.url}, 8) + END + ELSE + CASE + WHEN INSTR(${bookmarkLinks.url}, '/') > 0 THEN + SUBSTR(${bookmarkLinks.url}, 1, INSTR(${bookmarkLinks.url}, '/') - 1) + ELSE + ${bookmarkLinks.url} + END + END`, + ) + .orderBy(desc(count())) + .limit(5), + + // Top 5 tags (used in bookmarks created this year) + this.ctx.db + .select({ + name: bookmarkTags.name, + count: count(), + }) + .from(bookmarkTags) + .innerJoin(tagsOnBookmarks, eq(tagsOnBookmarks.tagId, bookmarkTags.id)) + .innerJoin(bookmarks, eq(bookmarks.id, tagsOnBookmarks.bookmarkId)) + .where(yearFilter) + .groupBy(bookmarkTags.name) + .orderBy(desc(count())) + .limit(5), + + // Bookmarks by source + this.ctx.db + .select({ + source: bookmarks.source, + count: count(), + }) + .from(bookmarks) + .where(yearFilter) + .groupBy(bookmarks.source) + .orderBy(desc(count())), + + // All bookmark timestamps in the year for activity calculations + this.ctx.db + .select({ + createdAt: bookmarks.createdAt, + }) + .from(bookmarks) + .where(yearFilter), + ]); + + // Process bookmarks by type + const bookmarkTypeMap = { link: 0, text: 0, asset: 0 }; + bookmarksByType.forEach((item) => { + if (item.type in bookmarkTypeMap) { + bookmarkTypeMap[item.type as keyof typeof bookmarkTypeMap] = item.count; + } + }); + + // Process timestamps with user timezone for hourly/daily activity + const hourCounts = Array.from({ length: 24 }, () => 0); + const dayCounts = Array.from({ length: 7 }, () => 0); + const monthCounts = Array.from({ length: 12 }, () => 0); + const dayCounts_full: Record<string, number> = {}; + + bookmarkTimestamps.forEach(({ createdAt }) => { + if (createdAt) { + const date = new Date(createdAt); + const userDate = new Date( + date.toLocaleString("en-US", { timeZone: userTimezone }), + ); + + const hour = userDate.getHours(); + const day = userDate.getDay(); + const month = userDate.getMonth(); + const dateKey = userDate.toISOString().split("T")[0]; + + hourCounts[hour]++; + dayCounts[day]++; + monthCounts[month]++; + dayCounts_full[dateKey] = (dayCounts_full[dateKey] || 0) + 1; + } + }); + + // Find peak hour and day + const peakHour = hourCounts.indexOf(Math.max(...hourCounts)); + const peakDayOfWeek = dayCounts.indexOf(Math.max(...dayCounts)); + + // Find most active day + let mostActiveDay: { date: string; count: number } | null = null; + if (Object.keys(dayCounts_full).length > 0) { + const sortedDays = Object.entries(dayCounts_full).sort( + ([, a], [, b]) => b - a, + ); + mostActiveDay = { + date: sortedDays[0][0], + count: sortedDays[0][1], + }; + } + + // Monthly activity + const monthlyActivity = Array.from({ length: 12 }, (_, i) => ({ + month: i + 1, + count: monthCounts[i], + })); + + // First bookmark + const firstBookmark = + firstBookmarkResult.length > 0 + ? { + id: firstBookmarkResult[0].id, + title: firstBookmarkResult[0].title, + createdAt: firstBookmarkResult[0].createdAt, + type: firstBookmarkResult[0].type, + } + : null; + + return { + year, + totalBookmarks: totalBookmarks || 0, + totalFavorites: totalFavorites || 0, + totalArchived: totalArchived || 0, + totalHighlights: numHighlights || 0, + totalTags: numTags || 0, + totalLists: numLists || 0, + firstBookmark, + mostActiveDay, + topDomains: topDomains.filter((d) => d.domain && d.domain.length > 0), + topTags, + bookmarksByType: bookmarkTypeMap, + bookmarksBySource, + monthlyActivity, + peakHour, + peakDayOfWeek, + }; + } + asWhoAmI(): z.infer<typeof zWhoAmIResponseSchema> { return { id: this.user.id, name: this.user.name, email: this.user.email, + image: this.user.image, localUser: this.user.password !== null, }; } diff --git a/packages/trpc/models/webhooks.ts b/packages/trpc/models/webhooks.ts index d2d9c19c..12281ec7 100644 --- a/packages/trpc/models/webhooks.ts +++ b/packages/trpc/models/webhooks.ts @@ -1,8 +1,9 @@ import { TRPCError } from "@trpc/server"; -import { and, eq } from "drizzle-orm"; +import { and, count, eq } from "drizzle-orm"; import { z } from "zod"; import { webhooksTable } from "@karakeep/db/schema"; +import serverConfig from "@karakeep/shared/config"; import { zNewWebhookSchema, zUpdateWebhookSchema, @@ -44,6 +45,20 @@ export class Webhook { ctx: AuthedContext, input: z.infer<typeof zNewWebhookSchema>, ): Promise<Webhook> { + // Check if user has reached the maximum number of webhooks + const [webhookCount] = await ctx.db + .select({ count: count() }) + .from(webhooksTable) + .where(eq(webhooksTable.userId, ctx.user.id)); + + const maxWebhooks = serverConfig.webhook.maxWebhooksPerUser; + if (webhookCount.count >= maxWebhooks) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: `Maximum number of webhooks (${maxWebhooks}) reached`, + }); + } + const [result] = await ctx.db .insert(webhooksTable) .values({ diff --git a/packages/trpc/package.json b/packages/trpc/package.json index d9fa12c0..d50a2174 100644 --- a/packages/trpc/package.json +++ b/packages/trpc/package.json @@ -17,7 +17,7 @@ "@karakeep/plugins": "workspace:*", "@karakeep/shared": "workspace:*", "@karakeep/shared-server": "workspace:*", - "@trpc/server": "^11.4.3", + "@trpc/server": "^11.9.0", "bcryptjs": "^2.4.3", "deep-equal": "^2.2.3", "drizzle-orm": "^0.44.2", diff --git a/packages/trpc/routers/_app.ts b/packages/trpc/routers/_app.ts index bae69130..9e20bb7e 100644 --- a/packages/trpc/routers/_app.ts +++ b/packages/trpc/routers/_app.ts @@ -4,6 +4,7 @@ import { apiKeysAppRouter } from "./apiKeys"; import { assetsAppRouter } from "./assets"; import { backupsAppRouter } from "./backups"; import { bookmarksAppRouter } from "./bookmarks"; +import { configAppRouter } from "./config"; import { feedsAppRouter } from "./feeds"; import { highlightsAppRouter } from "./highlights"; import { importSessionsRouter } from "./importSessions"; @@ -35,6 +36,7 @@ export const appRouter = router({ invites: invitesAppRouter, publicBookmarks: publicBookmarks, subscriptions: subscriptionsRouter, + config: configAppRouter, }); // export type definition of API export type AppRouter = typeof appRouter; diff --git a/packages/trpc/routers/admin.test.ts b/packages/trpc/routers/admin.test.ts new file mode 100644 index 00000000..2f80d9c0 --- /dev/null +++ b/packages/trpc/routers/admin.test.ts @@ -0,0 +1,265 @@ +import { eq } from "drizzle-orm"; +import { assert, beforeEach, describe, expect, test } from "vitest"; + +import { bookmarkLinks, users } from "@karakeep/db/schema"; +import { BookmarkTypes } from "@karakeep/shared/types/bookmarks"; + +import type { CustomTestContext } from "../testUtils"; +import { buildTestContext, getApiCaller } from "../testUtils"; + +beforeEach<CustomTestContext>(async (context) => { + const testContext = await buildTestContext(true); + Object.assign(context, testContext); +}); + +describe("Admin Routes", () => { + describe("getBookmarkDebugInfo", () => { + test<CustomTestContext>("admin can access bookmark debug info for link bookmark", async ({ + apiCallers, + db, + }) => { + // Create an admin user + const adminUser = await db + .insert(users) + .values({ + name: "Admin User", + email: "admin@test.com", + role: "admin", + }) + .returning(); + const adminApi = getApiCaller( + db, + adminUser[0].id, + adminUser[0].email, + "admin", + ); + + // Create a bookmark as a regular user + const bookmark = await apiCallers[0].bookmarks.createBookmark({ + url: "https://example.com", + type: BookmarkTypes.LINK, + }); + + // Update the bookmark link with some metadata + await db + .update(bookmarkLinks) + .set({ + crawlStatus: "success", + crawlStatusCode: 200, + crawledAt: new Date(), + htmlContent: "<html><body>Test content</body></html>", + title: "Test Title", + description: "Test Description", + }) + .where(eq(bookmarkLinks.id, bookmark.id)); + + // Admin should be able to access debug info + const debugInfo = await adminApi.admin.getBookmarkDebugInfo({ + bookmarkId: bookmark.id, + }); + + expect(debugInfo.id).toEqual(bookmark.id); + expect(debugInfo.type).toEqual(BookmarkTypes.LINK); + expect(debugInfo.linkInfo).toBeDefined(); + assert(debugInfo.linkInfo); + expect(debugInfo.linkInfo.url).toEqual("https://example.com"); + expect(debugInfo.linkInfo.crawlStatus).toEqual("success"); + expect(debugInfo.linkInfo.crawlStatusCode).toEqual(200); + expect(debugInfo.linkInfo.hasHtmlContent).toEqual(true); + expect(debugInfo.linkInfo.htmlContentPreview).toBeDefined(); + expect(debugInfo.linkInfo.htmlContentPreview).toContain("Test content"); + }); + + test<CustomTestContext>("admin can access bookmark debug info for text bookmark", async ({ + apiCallers, + db, + }) => { + // Create an admin user + const adminUser = await db + .insert(users) + .values({ + name: "Admin User", + email: "admin@test.com", + role: "admin", + }) + .returning(); + const adminApi = getApiCaller( + db, + adminUser[0].id, + adminUser[0].email, + "admin", + ); + + // Create a text bookmark + const bookmark = await apiCallers[0].bookmarks.createBookmark({ + text: "This is a test text bookmark", + type: BookmarkTypes.TEXT, + }); + + // Admin should be able to access debug info + const debugInfo = await adminApi.admin.getBookmarkDebugInfo({ + bookmarkId: bookmark.id, + }); + + expect(debugInfo.id).toEqual(bookmark.id); + expect(debugInfo.type).toEqual(BookmarkTypes.TEXT); + expect(debugInfo.textInfo).toBeDefined(); + assert(debugInfo.textInfo); + expect(debugInfo.textInfo.hasText).toEqual(true); + }); + + test<CustomTestContext>("admin can see bookmark tags in debug info", async ({ + apiCallers, + db, + }) => { + // Create an admin user + const adminUser = await db + .insert(users) + .values({ + name: "Admin User", + email: "admin@test.com", + role: "admin", + }) + .returning(); + const adminApi = getApiCaller( + db, + adminUser[0].id, + adminUser[0].email, + "admin", + ); + + // Create a bookmark with tags + const bookmark = await apiCallers[0].bookmarks.createBookmark({ + url: "https://example.com", + type: BookmarkTypes.LINK, + }); + + // Add tags to the bookmark + await apiCallers[0].bookmarks.updateTags({ + bookmarkId: bookmark.id, + attach: [{ tagName: "test-tag-1" }, { tagName: "test-tag-2" }], + detach: [], + }); + + // Admin should be able to see tags in debug info + const debugInfo = await adminApi.admin.getBookmarkDebugInfo({ + bookmarkId: bookmark.id, + }); + + expect(debugInfo.tags).toHaveLength(2); + expect(debugInfo.tags.map((t) => t.name).sort()).toEqual([ + "test-tag-1", + "test-tag-2", + ]); + expect(debugInfo.tags[0].attachedBy).toEqual("human"); + }); + + test<CustomTestContext>("non-admin user cannot access bookmark debug info", async ({ + apiCallers, + }) => { + // Create a bookmark + const bookmark = await apiCallers[0].bookmarks.createBookmark({ + url: "https://example.com", + type: BookmarkTypes.LINK, + }); + + // Non-admin user should not be able to access debug info + // The admin procedure itself will throw FORBIDDEN + await expect(() => + apiCallers[0].admin.getBookmarkDebugInfo({ bookmarkId: bookmark.id }), + ).rejects.toThrow(/FORBIDDEN/); + }); + + test<CustomTestContext>("debug info includes asset URLs with signed tokens", async ({ + apiCallers, + db, + }) => { + // Create an admin user + const adminUser = await db + .insert(users) + .values({ + name: "Admin User", + email: "admin@test.com", + role: "admin", + }) + .returning(); + const adminApi = getApiCaller( + db, + adminUser[0].id, + adminUser[0].email, + "admin", + ); + + // Create a bookmark + const bookmark = await apiCallers[0].bookmarks.createBookmark({ + url: "https://example.com", + type: BookmarkTypes.LINK, + }); + + // Get debug info + const debugInfo = await adminApi.admin.getBookmarkDebugInfo({ + bookmarkId: bookmark.id, + }); + + // Check that assets array is present + expect(debugInfo.assets).toBeDefined(); + expect(Array.isArray(debugInfo.assets)).toBe(true); + + // If there are assets, check that they have signed URLs + if (debugInfo.assets.length > 0) { + const asset = debugInfo.assets[0]; + expect(asset.url).toBeDefined(); + expect(asset.url).toContain("/api/public/assets/"); + expect(asset.url).toContain("token="); + } + }); + + test<CustomTestContext>("debug info truncates HTML content preview", async ({ + apiCallers, + db, + }) => { + // Create an admin user + const adminUser = await db + .insert(users) + .values({ + name: "Admin User", + email: "admin@test.com", + role: "admin", + }) + .returning(); + const adminApi = getApiCaller( + db, + adminUser[0].id, + adminUser[0].email, + "admin", + ); + + // Create a bookmark + const bookmark = await apiCallers[0].bookmarks.createBookmark({ + url: "https://example.com", + type: BookmarkTypes.LINK, + }); + + // Create a large HTML content + const largeContent = "<html><body>" + "x".repeat(2000) + "</body></html>"; + await db + .update(bookmarkLinks) + .set({ + htmlContent: largeContent, + }) + .where(eq(bookmarkLinks.id, bookmark.id)); + + // Get debug info + const debugInfo = await adminApi.admin.getBookmarkDebugInfo({ + bookmarkId: bookmark.id, + }); + + // Check that HTML preview is truncated to 1000 characters + assert(debugInfo.linkInfo); + expect(debugInfo.linkInfo.htmlContentPreview).toBeDefined(); + expect(debugInfo.linkInfo.htmlContentPreview!.length).toBeLessThanOrEqual( + 1000, + ); + }); + }); +}); diff --git a/packages/trpc/routers/admin.ts b/packages/trpc/routers/admin.ts index 463d2ddf..f64e071a 100644 --- a/packages/trpc/routers/admin.ts +++ b/packages/trpc/routers/admin.ts @@ -9,7 +9,9 @@ import { AssetPreprocessingQueue, FeedQueue, LinkCrawlerQueue, + LowPriorityCrawlerQueue, OpenAIQueue, + QueuePriority, SearchIndexingQueue, triggerSearchReindex, VideoWorkerQueue, @@ -17,6 +19,7 @@ import { zAdminMaintenanceTaskSchema, } from "@karakeep/shared-server"; import serverConfig from "@karakeep/shared/config"; +import logger from "@karakeep/shared/logger"; import { PluginManager, PluginType } from "@karakeep/shared/plugins"; import { getSearchClient } from "@karakeep/shared/search"; import { @@ -24,9 +27,11 @@ import { updateUserSchema, zAdminCreateUserSchema, } from "@karakeep/shared/types/admin"; +import { BookmarkTypes } from "@karakeep/shared/types/bookmarks"; import { generatePasswordSalt, hashPassword } from "../auth"; import { adminProcedure, router } from "../index"; +import { Bookmark } from "../models/bookmarks"; import { User } from "../models/users"; export const adminAppRouter = router({ @@ -86,6 +91,7 @@ export const adminAppRouter = router({ const [ // Crawls queuedCrawls, + queuedLowPriorityCrawls, [{ value: pendingCrawls }], [{ value: failedCrawls }], @@ -114,6 +120,7 @@ export const adminAppRouter = router({ ] = await Promise.all([ // Crawls LinkCrawlerQueue.stats(), + LowPriorityCrawlerQueue.stats(), ctx.db .select({ value: count() }) .from(bookmarkLinks) @@ -165,7 +172,11 @@ export const adminAppRouter = router({ return { crawlStats: { - queued: queuedCrawls.pending + queuedCrawls.pending_retry, + queued: + queuedCrawls.pending + + queuedCrawls.pending_retry + + queuedLowPriorityCrawls.pending + + queuedLowPriorityCrawls.pending_retry, pending: pendingCrawls, failed: failedCrawls, }, @@ -201,7 +212,7 @@ export const adminAppRouter = router({ recrawlLinks: adminProcedure .input( z.object({ - crawlStatus: z.enum(["success", "failure", "all"]), + crawlStatus: z.enum(["success", "failure", "pending", "all"]), runInference: z.boolean(), }), ) @@ -217,10 +228,15 @@ export const adminAppRouter = router({ await Promise.all( bookmarkIds.map((b) => - LinkCrawlerQueue.enqueue({ - bookmarkId: b.id, - runInference: input.runInference, - }), + LowPriorityCrawlerQueue.enqueue( + { + bookmarkId: b.id, + runInference: input.runInference, + }, + { + priority: QueuePriority.Low, + }, + ), ), ); }), @@ -233,7 +249,13 @@ export const adminAppRouter = router({ }, }); - await Promise.all(bookmarkIds.map((b) => triggerSearchReindex(b.id))); + await Promise.all( + bookmarkIds.map((b) => + triggerSearchReindex(b.id, { + priority: QueuePriority.Low, + }), + ), + ); }), reprocessAssetsFixMode: adminProcedure.mutation(async ({ ctx }) => { const bookmarkIds = await ctx.db.query.bookmarkAssets.findMany({ @@ -244,10 +266,15 @@ export const adminAppRouter = router({ await Promise.all( bookmarkIds.map((b) => - AssetPreprocessingQueue.enqueue({ - bookmarkId: b.id, - fixMode: true, - }), + AssetPreprocessingQueue.enqueue( + { + bookmarkId: b.id, + fixMode: true, + }, + { + priority: QueuePriority.Low, + }, + ), ), ); }), @@ -255,7 +282,7 @@ export const adminAppRouter = router({ .input( z.object({ type: z.enum(["tag", "summarize"]), - status: z.enum(["success", "failure", "all"]), + status: z.enum(["success", "failure", "pending", "all"]), }), ) .mutation(async ({ input, ctx }) => { @@ -277,7 +304,12 @@ export const adminAppRouter = router({ await Promise.all( bookmarkIds.map((b) => - OpenAIQueue.enqueue({ bookmarkId: b.id, type: input.type }), + OpenAIQueue.enqueue( + { bookmarkId: b.id, type: input.type }, + { + priority: QueuePriority.Low, + }, + ), ), ); }), @@ -537,4 +569,194 @@ export const adminAppRouter = router({ queue: queueStatus, }; }), + getBookmarkDebugInfo: adminProcedure + .input(z.object({ bookmarkId: z.string() })) + .output( + z.object({ + id: z.string(), + type: z.enum([ + BookmarkTypes.LINK, + BookmarkTypes.TEXT, + BookmarkTypes.ASSET, + ]), + source: z + .enum([ + "api", + "web", + "extension", + "cli", + "mobile", + "singlefile", + "rss", + "import", + ]) + .nullable(), + createdAt: z.date(), + modifiedAt: z.date().nullable(), + title: z.string().nullable(), + summary: z.string().nullable(), + taggingStatus: z.enum(["pending", "failure", "success"]).nullable(), + summarizationStatus: z + .enum(["pending", "failure", "success"]) + .nullable(), + userId: z.string(), + linkInfo: z + .object({ + url: z.string(), + crawlStatus: z.enum(["pending", "failure", "success"]), + crawlStatusCode: z.number().nullable(), + crawledAt: z.date().nullable(), + hasHtmlContent: z.boolean(), + hasContentAsset: z.boolean(), + htmlContentPreview: z.string().nullable(), + }) + .nullable(), + textInfo: z + .object({ + hasText: z.boolean(), + sourceUrl: z.string().nullable(), + }) + .nullable(), + assetInfo: z + .object({ + assetType: z.enum(["image", "pdf"]), + hasContent: z.boolean(), + fileName: z.string().nullable(), + }) + .nullable(), + tags: z.array( + z.object({ + id: z.string(), + name: z.string(), + attachedBy: z.enum(["ai", "human"]), + }), + ), + assets: z.array( + z.object({ + id: z.string(), + assetType: z.string(), + size: z.number(), + url: z.string().nullable(), + }), + ), + }), + ) + .query(async ({ input, ctx }) => { + logger.info( + `[admin] Admin ${ctx.user.id} accessed debug info for bookmark ${input.bookmarkId}`, + ); + + return await Bookmark.buildDebugInfo(ctx, input.bookmarkId); + }), + adminRecrawlBookmark: adminProcedure + .input(z.object({ bookmarkId: z.string() })) + .mutation(async ({ input, ctx }) => { + // Verify bookmark exists and is a link + const bookmark = await ctx.db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, input.bookmarkId), + }); + + if (!bookmark) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Bookmark not found", + }); + } + + if (bookmark.type !== BookmarkTypes.LINK) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Only link bookmarks can be recrawled", + }); + } + + await LowPriorityCrawlerQueue.enqueue( + { + bookmarkId: input.bookmarkId, + }, + { + priority: QueuePriority.Low, + groupId: "admin", + }, + ); + }), + adminReindexBookmark: adminProcedure + .input(z.object({ bookmarkId: z.string() })) + .mutation(async ({ input, ctx }) => { + // Verify bookmark exists + const bookmark = await ctx.db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, input.bookmarkId), + }); + + if (!bookmark) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Bookmark not found", + }); + } + + await triggerSearchReindex(input.bookmarkId, { + priority: QueuePriority.Low, + groupId: "admin", + }); + }), + adminRetagBookmark: adminProcedure + .input(z.object({ bookmarkId: z.string() })) + .mutation(async ({ input, ctx }) => { + // Verify bookmark exists + const bookmark = await ctx.db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, input.bookmarkId), + }); + + if (!bookmark) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Bookmark not found", + }); + } + + await OpenAIQueue.enqueue( + { + bookmarkId: input.bookmarkId, + type: "tag", + }, + { + priority: QueuePriority.Low, + groupId: "admin", + }, + ); + }), + adminResummarizeBookmark: adminProcedure + .input(z.object({ bookmarkId: z.string() })) + .mutation(async ({ input, ctx }) => { + // Verify bookmark exists and is a link + const bookmark = await ctx.db.query.bookmarks.findFirst({ + where: eq(bookmarks.id, input.bookmarkId), + }); + + if (!bookmark) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Bookmark not found", + }); + } + + if (bookmark.type !== BookmarkTypes.LINK) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Only link bookmarks can be summarized", + }); + } + + await OpenAIQueue.enqueue( + { + bookmarkId: input.bookmarkId, + type: "summarize", + }, + { + priority: QueuePriority.Low, + groupId: "admin", + }, + ); + }), }); diff --git a/packages/trpc/routers/apiKeys.ts b/packages/trpc/routers/apiKeys.ts index 763bc23a..90de824a 100644 --- a/packages/trpc/routers/apiKeys.ts +++ b/packages/trpc/routers/apiKeys.ts @@ -1,5 +1,5 @@ import { TRPCError } from "@trpc/server"; -import { and, eq } from "drizzle-orm"; +import { and, desc, eq } from "drizzle-orm"; import { z } from "zod"; import { apiKeys } from "@karakeep/db/schema"; @@ -83,6 +83,7 @@ export const apiKeysAppRouter = router({ name: z.string(), createdAt: z.date(), keyId: z.string(), + lastUsedAt: z.date().nullish(), }), ), }), @@ -94,8 +95,10 @@ export const apiKeysAppRouter = router({ id: true, name: true, createdAt: true, + lastUsedAt: true, keyId: true, }, + orderBy: desc(apiKeys.createdAt), }); return { keys: resp }; }), diff --git a/packages/trpc/routers/assets.ts b/packages/trpc/routers/assets.ts index 7be85446..c75f1e2e 100644 --- a/packages/trpc/routers/assets.ts +++ b/packages/trpc/routers/assets.ts @@ -1,57 +1,20 @@ -import { TRPCError } from "@trpc/server"; -import { and, desc, eq, sql } from "drizzle-orm"; import { z } from "zod"; -import { assets, bookmarks } from "@karakeep/db/schema"; -import { deleteAsset } from "@karakeep/shared/assetdb"; import { zAssetSchema, zAssetTypesSchema, } from "@karakeep/shared/types/bookmarks"; -import { authedProcedure, Context, router } from "../index"; -import { - isAllowedToAttachAsset, - isAllowedToDetachAsset, - mapDBAssetTypeToUserType, - mapSchemaAssetTypeToDB, -} from "../lib/attachments"; +import { authedProcedure, router } from "../index"; +import { Asset } from "../models/assets"; import { ensureBookmarkOwnership } from "./bookmarks"; -export const ensureAssetOwnership = async (opts: { - ctx: Context; - assetId: string; -}) => { - const asset = await opts.ctx.db.query.assets.findFirst({ - where: eq(bookmarks.id, opts.assetId), - }); - if (!opts.ctx.user) { - throw new TRPCError({ - code: "UNAUTHORIZED", - message: "User is not authorized", - }); - } - if (!asset) { - throw new TRPCError({ - code: "NOT_FOUND", - message: "Asset not found", - }); - } - if (asset.userId != opts.ctx.user.id) { - throw new TRPCError({ - code: "FORBIDDEN", - message: "User is not allowed to access resource", - }); - } - return asset; -}; - export const assetsAppRouter = router({ list: authedProcedure .input( z.object({ limit: z.number().min(1).max(100).default(20), - cursor: z.number().nullish(), // page number + cursor: z.number().nullish(), }), ) .output( @@ -71,29 +34,10 @@ export const assetsAppRouter = router({ }), ) .query(async ({ input, ctx }) => { - const page = input.cursor ?? 1; - const [results, totalCount] = await Promise.all([ - ctx.db - .select() - .from(assets) - .where(eq(assets.userId, ctx.user.id)) - .orderBy(desc(assets.size)) - .limit(input.limit) - .offset((page - 1) * input.limit), - ctx.db - .select({ count: sql<number>`count(*)` }) - .from(assets) - .where(eq(assets.userId, ctx.user.id)), - ]); - - return { - assets: results.map((a) => ({ - ...a, - assetType: mapDBAssetTypeToUserType(a.assetType), - })), - nextCursor: page * input.limit < totalCount[0].count ? page + 1 : null, - totalCount: totalCount[0].count, - }; + return await Asset.list(ctx, { + limit: input.limit, + cursor: input.cursor ?? null, + }); }), attachAsset: authedProcedure .input( @@ -108,29 +52,7 @@ export const assetsAppRouter = router({ .output(zAssetSchema) .use(ensureBookmarkOwnership) .mutation(async ({ input, ctx }) => { - await ensureAssetOwnership({ ctx, assetId: input.asset.id }); - if (!isAllowedToAttachAsset(input.asset.assetType)) { - throw new TRPCError({ - code: "BAD_REQUEST", - message: "You can't attach this type of asset", - }); - } - const [updatedAsset] = await ctx.db - .update(assets) - .set({ - assetType: mapSchemaAssetTypeToDB(input.asset.assetType), - bookmarkId: input.bookmarkId, - }) - .where( - and(eq(assets.id, input.asset.id), eq(assets.userId, ctx.user.id)), - ) - .returning(); - - return { - id: updatedAsset.id, - assetType: mapDBAssetTypeToUserType(updatedAsset.assetType), - fileName: updatedAsset.fileName, - }; + return await Asset.attachAsset(ctx, input); }), replaceAsset: authedProcedure .input( @@ -143,41 +65,7 @@ export const assetsAppRouter = router({ .output(z.void()) .use(ensureBookmarkOwnership) .mutation(async ({ input, ctx }) => { - await Promise.all([ - ensureAssetOwnership({ ctx, assetId: input.oldAssetId }), - ensureAssetOwnership({ ctx, assetId: input.newAssetId }), - ]); - const [oldAsset] = await ctx.db - .select() - .from(assets) - .where( - and(eq(assets.id, input.oldAssetId), eq(assets.userId, ctx.user.id)), - ) - .limit(1); - if ( - !isAllowedToAttachAsset(mapDBAssetTypeToUserType(oldAsset.assetType)) - ) { - throw new TRPCError({ - code: "BAD_REQUEST", - message: "You can't attach this type of asset", - }); - } - - await ctx.db.transaction(async (tx) => { - await tx.delete(assets).where(eq(assets.id, input.oldAssetId)); - await tx - .update(assets) - .set({ - bookmarkId: input.bookmarkId, - assetType: oldAsset.assetType, - }) - .where(eq(assets.id, input.newAssetId)); - }); - - await deleteAsset({ - userId: ctx.user.id, - assetId: input.oldAssetId, - }).catch(() => ({})); + await Asset.replaceAsset(ctx, input); }), detachAsset: authedProcedure .input( @@ -189,34 +77,6 @@ export const assetsAppRouter = router({ .output(z.void()) .use(ensureBookmarkOwnership) .mutation(async ({ input, ctx }) => { - await ensureAssetOwnership({ ctx, assetId: input.assetId }); - const [oldAsset] = await ctx.db - .select() - .from(assets) - .where( - and(eq(assets.id, input.assetId), eq(assets.userId, ctx.user.id)), - ); - if ( - !isAllowedToDetachAsset(mapDBAssetTypeToUserType(oldAsset.assetType)) - ) { - throw new TRPCError({ - code: "BAD_REQUEST", - message: "You can't deattach this type of asset", - }); - } - const result = await ctx.db - .delete(assets) - .where( - and( - eq(assets.id, input.assetId), - eq(assets.bookmarkId, input.bookmarkId), - ), - ); - if (result.changes == 0) { - throw new TRPCError({ code: "NOT_FOUND" }); - } - await deleteAsset({ userId: ctx.user.id, assetId: input.assetId }).catch( - () => ({}), - ); + await Asset.detachAsset(ctx, input); }), }); diff --git a/packages/trpc/routers/bookmarks.test.ts b/packages/trpc/routers/bookmarks.test.ts index c272e015..aaee5447 100644 --- a/packages/trpc/routers/bookmarks.test.ts +++ b/packages/trpc/routers/bookmarks.test.ts @@ -331,6 +331,198 @@ describe("Bookmark Routes", () => { ).rejects.toThrow(/You must provide either a tagId or a tagName/); }); + test<CustomTestContext>("update tags - comprehensive edge cases", async ({ + apiCallers, + }) => { + const api = apiCallers[0].bookmarks; + + // Create two bookmarks + const bookmark1 = await api.createBookmark({ + url: "https://bookmark1.com", + type: BookmarkTypes.LINK, + }); + const bookmark2 = await api.createBookmark({ + url: "https://bookmark2.com", + type: BookmarkTypes.LINK, + }); + + // Test 1: Attach tags by name to bookmark1 (creates new tags) + await api.updateTags({ + bookmarkId: bookmark1.id, + attach: [{ tagName: "existing-tag" }, { tagName: "shared-tag" }], + detach: [], + }); + + let b1 = await api.getBookmark({ bookmarkId: bookmark1.id }); + expect(b1.tags.map((t) => t.name).sort()).toEqual([ + "existing-tag", + "shared-tag", + ]); + + const existingTagId = b1.tags.find((t) => t.name === "existing-tag")!.id; + const sharedTagId = b1.tags.find((t) => t.name === "shared-tag")!.id; + + // Test 2: Attach existing tag by ID to bookmark2 (tag already exists in DB from bookmark1) + await api.updateTags({ + bookmarkId: bookmark2.id, + attach: [{ tagId: existingTagId }], + detach: [], + }); + + let b2 = await api.getBookmark({ bookmarkId: bookmark2.id }); + expect(b2.tags.map((t) => t.name)).toEqual(["existing-tag"]); + + // Test 3: Attach existing tag by NAME to bookmark2 (tag already exists in DB) + await api.updateTags({ + bookmarkId: bookmark2.id, + attach: [{ tagName: "shared-tag" }], + detach: [], + }); + + b2 = await api.getBookmark({ bookmarkId: bookmark2.id }); + expect(b2.tags.map((t) => t.name).sort()).toEqual([ + "existing-tag", + "shared-tag", + ]); + + // Test 4: Re-attaching the same tag (idempotency) - should be no-op + await api.updateTags({ + bookmarkId: bookmark2.id, + attach: [{ tagId: existingTagId }], + detach: [], + }); + + b2 = await api.getBookmark({ bookmarkId: bookmark2.id }); + expect(b2.tags.map((t) => t.name).sort()).toEqual([ + "existing-tag", + "shared-tag", + ]); + + // Test 5: Detach non-existent tag by name (should be no-op) + await api.updateTags({ + bookmarkId: bookmark2.id, + attach: [], + detach: [{ tagName: "non-existent-tag" }], + }); + + b2 = await api.getBookmark({ bookmarkId: bookmark2.id }); + expect(b2.tags.map((t) => t.name).sort()).toEqual([ + "existing-tag", + "shared-tag", + ]); + + // Test 6: Mixed attach/detach with pre-existing tags + await api.updateTags({ + bookmarkId: bookmark2.id, + attach: [{ tagName: "new-tag" }, { tagId: sharedTagId }], // sharedTagId already attached + detach: [{ tagName: "existing-tag" }], + }); + + b2 = await api.getBookmark({ bookmarkId: bookmark2.id }); + expect(b2.tags.map((t) => t.name).sort()).toEqual([ + "new-tag", + "shared-tag", + ]); + + // Test 7: Detach by ID and re-attach by name in same operation + await api.updateTags({ + bookmarkId: bookmark2.id, + attach: [{ tagName: "new-tag" }], // Already exists, should be idempotent + detach: [{ tagId: sharedTagId }], + }); + + b2 = await api.getBookmark({ bookmarkId: bookmark2.id }); + expect(b2.tags.map((t) => t.name).sort()).toEqual(["new-tag"]); + + // Verify bookmark1 still has its original tags (operations on bookmark2 didn't affect it) + b1 = await api.getBookmark({ bookmarkId: bookmark1.id }); + expect(b1.tags.map((t) => t.name).sort()).toEqual([ + "existing-tag", + "shared-tag", + ]); + + // Test 8: Attach same tag multiple times in one operation (deduplication) + await api.updateTags({ + bookmarkId: bookmark1.id, + attach: [{ tagName: "duplicate-test" }, { tagName: "duplicate-test" }], + detach: [], + }); + + b1 = await api.getBookmark({ bookmarkId: bookmark1.id }); + const duplicateTagCount = b1.tags.filter( + (t) => t.name === "duplicate-test", + ).length; + expect(duplicateTagCount).toEqual(1); // Should only be attached once + }); + + test<CustomTestContext>("updateTags with attachedBy field", async ({ + apiCallers, + }) => { + const api = apiCallers[0].bookmarks; + const bookmark = await api.createBookmark({ + url: "https://bookmark.com", + type: BookmarkTypes.LINK, + }); + + // Test 1: Attach tags with different attachedBy values + await api.updateTags({ + bookmarkId: bookmark.id, + attach: [ + { tagName: "ai-tag", attachedBy: "ai" }, + { tagName: "human-tag", attachedBy: "human" }, + { tagName: "default-tag" }, // Should default to "human" + ], + detach: [], + }); + + let b = await api.getBookmark({ bookmarkId: bookmark.id }); + expect(b.tags.length).toEqual(3); + + const aiTag = b.tags.find((t) => t.name === "ai-tag"); + const humanTag = b.tags.find((t) => t.name === "human-tag"); + const defaultTag = b.tags.find((t) => t.name === "default-tag"); + + expect(aiTag?.attachedBy).toEqual("ai"); + expect(humanTag?.attachedBy).toEqual("human"); + expect(defaultTag?.attachedBy).toEqual("human"); + + // Test 2: Attach existing tag by ID with different attachedBy + // First detach the ai-tag + await api.updateTags({ + bookmarkId: bookmark.id, + attach: [], + detach: [{ tagId: aiTag!.id }], + }); + + // Re-attach the same tag but as human + await api.updateTags({ + bookmarkId: bookmark.id, + attach: [{ tagId: aiTag!.id, attachedBy: "human" }], + detach: [], + }); + + b = await api.getBookmark({ bookmarkId: bookmark.id }); + const reAttachedTag = b.tags.find((t) => t.id === aiTag!.id); + expect(reAttachedTag?.attachedBy).toEqual("human"); + + // Test 3: Attach existing tag by name with AI attachedBy + const bookmark2 = await api.createBookmark({ + url: "https://bookmark2.com", + type: BookmarkTypes.LINK, + }); + + await api.updateTags({ + bookmarkId: bookmark2.id, + attach: [{ tagName: "ai-tag", attachedBy: "ai" }], + detach: [], + }); + + const b2 = await api.getBookmark({ bookmarkId: bookmark2.id }); + const aiTagOnB2 = b2.tags.find((t) => t.name === "ai-tag"); + expect(aiTagOnB2?.attachedBy).toEqual("ai"); + expect(aiTagOnB2?.id).toEqual(aiTag!.id); // Should be the same tag + }); + test<CustomTestContext>("update bookmark text", async ({ apiCallers }) => { const api = apiCallers[0].bookmarks; const createdBookmark = await api.createBookmark({ diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts index 15ded2bd..782566cf 100644 --- a/packages/trpc/routers/bookmarks.ts +++ b/packages/trpc/routers/bookmarks.ts @@ -1,6 +1,5 @@ import { experimental_trpcMiddleware, TRPCError } from "@trpc/server"; import { and, eq, gt, inArray, lt, or } from "drizzle-orm"; -import invariant from "tiny-invariant"; import { z } from "zod"; import type { ZBookmarkContent } from "@karakeep/shared/types/bookmarks"; @@ -15,11 +14,14 @@ import { bookmarkTexts, customPrompts, tagsOnBookmarks, + users, } from "@karakeep/db/schema"; import { AssetPreprocessingQueue, LinkCrawlerQueue, + LowPriorityCrawlerQueue, OpenAIQueue, + QueuePriority, QuotaService, triggerRuleEngineOnEvent, triggerSearchReindex, @@ -28,7 +30,7 @@ import { import { SUPPORTED_BOOKMARK_ASSET_TYPES } from "@karakeep/shared/assetdb"; import serverConfig from "@karakeep/shared/config"; import { InferenceClientFactory } from "@karakeep/shared/inference"; -import { buildSummaryPrompt } from "@karakeep/shared/prompts"; +import { buildSummaryPrompt } from "@karakeep/shared/prompts.server"; import { EnqueueOptions } from "@karakeep/shared/queueing"; import { FilterQuery, getSearchClient } from "@karakeep/shared/search"; import { parseSearchQuery } from "@karakeep/shared/searchQueryParser"; @@ -49,9 +51,8 @@ import { normalizeTagName } from "@karakeep/shared/utils/tag"; import type { AuthedContext } from "../index"; import { authedProcedure, createRateLimitMiddleware, router } from "../index"; import { getBookmarkIdsFromMatcher } from "../lib/search"; +import { Asset } from "../models/assets"; import { BareBookmark, Bookmark } from "../models/bookmarks"; -import { ImportSession } from "../models/importSessions"; -import { ensureAssetOwnership } from "./assets"; export const ensureBookmarkOwnership = experimental_trpcMiddleware<{ ctx: AuthedContext; @@ -121,173 +122,173 @@ export const bookmarksAppRouter = router({ // This doesn't 100% protect from duplicates because of races, but it's more than enough for this usecase. const alreadyExists = await attemptToDedupLink(ctx, input.url); if (alreadyExists) { - if (input.importSessionId) { - const session = await ImportSession.fromId( - ctx, - input.importSessionId, - ); - await session.attachBookmark(alreadyExists.id); - } return { ...alreadyExists, alreadyExists: true }; } } - // Check user quota - const quotaResult = await QuotaService.canCreateBookmark( - ctx.db, - ctx.user.id, - ); - if (!quotaResult.result) { - throw new TRPCError({ - code: "FORBIDDEN", - message: quotaResult.error, - }); - } - - const bookmark = await ctx.db.transaction(async (tx) => { - const bookmark = ( - await tx - .insert(bookmarks) - .values({ - userId: ctx.user.id, - title: input.title, - type: input.type, - archived: input.archived, - favourited: input.favourited, - note: input.note, - summary: input.summary, - createdAt: input.createdAt, - source: input.source, - }) - .returning() - )[0]; + const bookmark = await ctx.db.transaction( + async (tx) => { + // Check user quota + const quotaResult = await QuotaService.canCreateBookmark( + tx, + ctx.user.id, + ); + if (!quotaResult.result) { + throw new TRPCError({ + code: "FORBIDDEN", + message: quotaResult.error, + }); + } + const bookmark = ( + await tx + .insert(bookmarks) + .values({ + userId: ctx.user.id, + title: input.title, + type: input.type, + archived: input.archived, + favourited: input.favourited, + note: input.note, + summary: input.summary, + createdAt: input.createdAt, + source: input.source, + // Only links currently support summarization. Let's set the status to null for other types for now. + summarizationStatus: + input.type === BookmarkTypes.LINK ? "pending" : null, + }) + .returning() + )[0]; - let content: ZBookmarkContent; + let content: ZBookmarkContent; - switch (input.type) { - case BookmarkTypes.LINK: { - const link = ( - await tx - .insert(bookmarkLinks) + switch (input.type) { + case BookmarkTypes.LINK: { + const link = ( + await tx + .insert(bookmarkLinks) + .values({ + id: bookmark.id, + url: input.url.trim(), + }) + .returning() + )[0]; + if (input.precrawledArchiveId) { + await Asset.ensureOwnership(ctx, input.precrawledArchiveId); + await tx + .update(assets) + .set({ + bookmarkId: bookmark.id, + assetType: AssetTypes.LINK_PRECRAWLED_ARCHIVE, + }) + .where( + and( + eq(assets.id, input.precrawledArchiveId), + eq(assets.userId, ctx.user.id), + ), + ); + } + content = { + type: BookmarkTypes.LINK, + ...link, + }; + break; + } + case BookmarkTypes.TEXT: { + const text = ( + await tx + .insert(bookmarkTexts) + .values({ + id: bookmark.id, + text: input.text, + sourceUrl: input.sourceUrl, + }) + .returning() + )[0]; + content = { + type: BookmarkTypes.TEXT, + text: text.text ?? "", + sourceUrl: text.sourceUrl, + }; + break; + } + case BookmarkTypes.ASSET: { + const [asset] = await tx + .insert(bookmarkAssets) .values({ id: bookmark.id, - url: input.url.trim(), + assetType: input.assetType, + assetId: input.assetId, + content: null, + metadata: null, + fileName: input.fileName ?? null, + sourceUrl: null, }) - .returning() - )[0]; - if (input.precrawledArchiveId) { - await ensureAssetOwnership({ - ctx, - assetId: input.precrawledArchiveId, - }); + .returning(); + const uploadedAsset = await Asset.fromId(ctx, input.assetId); + uploadedAsset.ensureOwnership(); + if ( + !uploadedAsset.asset.contentType || + !SUPPORTED_BOOKMARK_ASSET_TYPES.has( + uploadedAsset.asset.contentType, + ) + ) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "Unsupported asset type", + }); + } await tx .update(assets) .set({ bookmarkId: bookmark.id, - assetType: AssetTypes.LINK_PRECRAWLED_ARCHIVE, + assetType: AssetTypes.BOOKMARK_ASSET, }) .where( and( - eq(assets.id, input.precrawledArchiveId), + eq(assets.id, input.assetId), eq(assets.userId, ctx.user.id), ), ); + content = { + type: BookmarkTypes.ASSET, + assetType: asset.assetType, + assetId: asset.assetId, + }; + break; } - content = { - type: BookmarkTypes.LINK, - ...link, - }; - break; } - case BookmarkTypes.TEXT: { - const text = ( - await tx - .insert(bookmarkTexts) - .values({ - id: bookmark.id, - text: input.text, - sourceUrl: input.sourceUrl, - }) - .returning() - )[0]; - content = { - type: BookmarkTypes.TEXT, - text: text.text ?? "", - sourceUrl: text.sourceUrl, - }; - break; - } - case BookmarkTypes.ASSET: { - const [asset] = await tx - .insert(bookmarkAssets) - .values({ - id: bookmark.id, - assetType: input.assetType, - assetId: input.assetId, - content: null, - metadata: null, - fileName: input.fileName ?? null, - sourceUrl: null, - }) - .returning(); - const uploadedAsset = await ensureAssetOwnership({ - ctx, - assetId: input.assetId, - }); - if ( - !uploadedAsset.contentType || - !SUPPORTED_BOOKMARK_ASSET_TYPES.has(uploadedAsset.contentType) - ) { - throw new TRPCError({ - code: "BAD_REQUEST", - message: "Unsupported asset type", - }); - } - await tx - .update(assets) - .set({ - bookmarkId: bookmark.id, - assetType: AssetTypes.BOOKMARK_ASSET, - }) - .where( - and( - eq(assets.id, input.assetId), - eq(assets.userId, ctx.user.id), - ), - ); - content = { - type: BookmarkTypes.ASSET, - assetType: asset.assetType, - assetId: asset.assetId, - }; - break; - } - } - return { - alreadyExists: false, - tags: [] as ZBookmarkTags[], - assets: [], - content, - ...bookmark, - }; - }); - - if (input.importSessionId) { - const session = await ImportSession.fromId(ctx, input.importSessionId); - await session.attachBookmark(bookmark.id); - } + return { + alreadyExists: false, + tags: [] as ZBookmarkTags[], + assets: [], + content, + ...bookmark, + }; + }, + { + behavior: "immediate", + }, + ); const enqueueOpts: EnqueueOptions = { // The lower the priority number, the sooner the job will be processed - priority: input.crawlPriority === "low" ? 50 : 0, + priority: + input.crawlPriority === "low" + ? QueuePriority.Low + : QueuePriority.Default, groupId: ctx.user.id, }; switch (bookmark.content.type) { case BookmarkTypes.LINK: { // The crawling job triggers openai when it's done - await LinkCrawlerQueue.enqueue( + // Use a separate queue for low priority crawling to avoid impacting main queue parallelism + const crawlerQueue = + input.crawlPriority === "low" + ? LowPriorityCrawlerQueue + : LinkCrawlerQueue; + await crawlerQueue.enqueue( { bookmarkId: bookmark.id, }, @@ -317,22 +318,24 @@ export const bookmarksAppRouter = router({ } } - await triggerRuleEngineOnEvent( - bookmark.id, - [ - { - type: "bookmarkAdded", - }, - ], - enqueueOpts, - ); - await triggerSearchReindex(bookmark.id, enqueueOpts); - await triggerWebhook( - bookmark.id, - "created", - /* userId */ undefined, - enqueueOpts, - ); + await Promise.all([ + triggerRuleEngineOnEvent( + bookmark.id, + [ + { + type: "bookmarkAdded", + }, + ], + enqueueOpts, + ), + triggerSearchReindex(bookmark.id, enqueueOpts), + triggerWebhook( + bookmark.id, + "created", + /* userId */ undefined, + enqueueOpts, + ), + ]); return bookmark; }), @@ -487,13 +490,14 @@ export const bookmarksAppRouter = router({ })), ); } - // Trigger re-indexing and webhooks - await triggerSearchReindex(input.bookmarkId, { - groupId: ctx.user.id, - }); - await triggerWebhook(input.bookmarkId, "edited", ctx.user.id, { - groupId: ctx.user.id, - }); + await Promise.all([ + triggerSearchReindex(input.bookmarkId, { + groupId: ctx.user.id, + }), + triggerWebhook(input.bookmarkId, "edited", ctx.user.id, { + groupId: ctx.user.id, + }), + ]); return updatedBookmark; }), @@ -532,12 +536,14 @@ export const bookmarksAppRouter = router({ ), ); }); - await triggerSearchReindex(input.bookmarkId, { - groupId: ctx.user.id, - }); - await triggerWebhook(input.bookmarkId, "edited", ctx.user.id, { - groupId: ctx.user.id, - }); + await Promise.all([ + triggerSearchReindex(input.bookmarkId, { + groupId: ctx.user.id, + }), + triggerWebhook(input.bookmarkId, "edited", ctx.user.id, { + groupId: ctx.user.id, + }), + ]); }), deleteBookmark: authedProcedure @@ -559,24 +565,20 @@ export const bookmarksAppRouter = router({ z.object({ bookmarkId: z.string(), archiveFullPage: z.boolean().optional().default(false), + storePdf: z.boolean().optional().default(false), }), ) .use(ensureBookmarkOwnership) .mutation(async ({ input, ctx }) => { - await ctx.db - .update(bookmarkLinks) - .set({ - crawlStatus: "pending", - crawlStatusCode: null, - }) - .where(eq(bookmarkLinks.id, input.bookmarkId)); - await LinkCrawlerQueue.enqueue( + await LowPriorityCrawlerQueue.enqueue( { bookmarkId: input.bookmarkId, archiveFullPage: input.archiveFullPage, + storePdf: input.storePdf, }, { groupId: ctx.user.id, + priority: QueuePriority.Low, }, ); }), @@ -711,36 +713,109 @@ export const bookmarksAppRouter = router({ ) .use(ensureBookmarkOwnership) .mutation(async ({ input, ctx }) => { - const res = await ctx.db.transaction(async (tx) => { - // Detaches - const idsToRemove: string[] = []; - if (input.detach.length > 0) { - const namesToRemove: string[] = []; - input.detach.forEach((detachInfo) => { - if (detachInfo.tagId) { - idsToRemove.push(detachInfo.tagId); - } - if (detachInfo.tagName) { - namesToRemove.push(detachInfo.tagName); - } - }); + // Helper function to fetch tag IDs and their names from a list of tag identifiers + const fetchTagIdsWithNames = async ( + tagIdentifiers: { tagId?: string; tagName?: string }[], + ): Promise<{ id: string; name: string }[]> => { + const tagIds = tagIdentifiers.flatMap((t) => + t.tagId ? [t.tagId] : [], + ); + const tagNames = tagIdentifiers.flatMap((t) => + t.tagName ? [t.tagName] : [], + ); - if (namesToRemove.length > 0) { - ( - await tx.query.bookmarkTags.findMany({ - where: and( - eq(bookmarkTags.userId, ctx.user.id), - inArray(bookmarkTags.name, namesToRemove), - ), - columns: { - id: true, - }, - }) - ).forEach((tag) => { - idsToRemove.push(tag.id); - }); + // Fetch tag IDs in parallel + const [byIds, byNames] = await Promise.all([ + tagIds.length > 0 + ? ctx.db + .select({ id: bookmarkTags.id, name: bookmarkTags.name }) + .from(bookmarkTags) + .where( + and( + eq(bookmarkTags.userId, ctx.user.id), + inArray(bookmarkTags.id, tagIds), + ), + ) + : Promise.resolve([]), + tagNames.length > 0 + ? ctx.db + .select({ id: bookmarkTags.id, name: bookmarkTags.name }) + .from(bookmarkTags) + .where( + and( + eq(bookmarkTags.userId, ctx.user.id), + inArray(bookmarkTags.name, tagNames), + ), + ) + : Promise.resolve([]), + ]); + + // Union results and deduplicate by tag ID + const seen = new Set<string>(); + const results: { id: string; name: string }[] = []; + + for (const tag of [...byIds, ...byNames]) { + if (!seen.has(tag.id)) { + seen.add(tag.id); + results.push({ id: tag.id, name: tag.name }); } + } + + return results; + }; + + // Normalize tag names and create new tags outside transaction to reduce transaction duration + const normalizedAttachTags = input.attach.map((tag) => ({ + tagId: tag.tagId, + tagName: tag.tagName ? normalizeTagName(tag.tagName) : undefined, + attachedBy: tag.attachedBy, + })); + + { + // Create new tags + const toAddTagNames = normalizedAttachTags + .flatMap((i) => (i.tagName ? [i.tagName] : [])) + .filter((n) => n.length > 0); // drop empty results + + if (toAddTagNames.length > 0) { + await ctx.db + .insert(bookmarkTags) + .values( + toAddTagNames.map((name) => ({ name, userId: ctx.user.id })), + ) + .onConflictDoNothing(); + } + } + + // Fetch tag IDs for attachment/detachment now that we know that they all exist + const [attachTagsWithNames, detachTagsWithNames] = await Promise.all([ + fetchTagIdsWithNames(normalizedAttachTags), + fetchTagIdsWithNames(input.detach), + ]); + + // Build the attachedBy map from the fetched results + const tagIdToAttachedBy = new Map<string, "ai" | "human">(); + + for (const fetchedTag of attachTagsWithNames) { + // Find the corresponding input tag + const inputTag = normalizedAttachTags.find( + (t) => + (t.tagId && t.tagId === fetchedTag.id) || + (t.tagName && t.tagName === fetchedTag.name), + ); + + if (inputTag) { + tagIdToAttachedBy.set(fetchedTag.id, inputTag.attachedBy); + } + } + + // Extract just the IDs for the transaction + const allIdsToAttach = attachTagsWithNames.map((t) => t.id); + const idsToRemove = detachTagsWithNames.map((t) => t.id); + const res = await ctx.db.transaction(async (tx) => { + // Detaches + if (idsToRemove.length > 0) { await tx .delete(tagsOnBookmarks) .where( @@ -751,67 +826,21 @@ export const bookmarksAppRouter = router({ ); } - if (input.attach.length == 0) { - return { - bookmarkId: input.bookmarkId, - attached: [], - detached: idsToRemove, - }; - } - - const toAddTagNames = input.attach - .flatMap((i) => (i.tagName ? [i.tagName] : [])) - .map(normalizeTagName) // strip leading # - .filter((n) => n.length > 0); // drop empty results - - const toAddTagIds = input.attach.flatMap((i) => - i.tagId ? [i.tagId] : [], - ); - - // New Tags - if (toAddTagNames.length > 0) { + // Attach tags + if (allIdsToAttach.length > 0) { await tx - .insert(bookmarkTags) + .insert(tagsOnBookmarks) .values( - toAddTagNames.map((name) => ({ name, userId: ctx.user.id })), + allIdsToAttach.map((i) => ({ + tagId: i, + bookmarkId: input.bookmarkId, + attachedBy: tagIdToAttachedBy.get(i) ?? "human", + })), ) - .onConflictDoNothing() - .returning(); + .onConflictDoNothing(); } - // If there is nothing to add, the "or" statement will become useless and - // the query below will simply select all the existing tags for this user and assign them to the bookmark - invariant(toAddTagNames.length > 0 || toAddTagIds.length > 0); - const allIds = ( - await tx.query.bookmarkTags.findMany({ - where: and( - eq(bookmarkTags.userId, ctx.user.id), - or( - toAddTagIds.length > 0 - ? inArray(bookmarkTags.id, toAddTagIds) - : undefined, - toAddTagNames.length > 0 - ? inArray(bookmarkTags.name, toAddTagNames) - : undefined, - ), - ), - columns: { - id: true, - }, - }) - ).map((t) => t.id); - - await tx - .insert(tagsOnBookmarks) - .values( - allIds.map((i) => ({ - tagId: i, - bookmarkId: input.bookmarkId, - attachedBy: "human" as const, - userId: ctx.user.id, - })), - ) - .onConflictDoNothing(); + // Update bookmark modified timestamp await tx .update(bookmarks) .set({ modifiedAt: new Date() }) @@ -824,7 +853,7 @@ export const bookmarksAppRouter = router({ return { bookmarkId: input.bookmarkId, - attached: allIds, + attached: allIdsToAttach, detached: idsToRemove, }; }); @@ -958,8 +987,15 @@ Author: ${bookmark.author ?? ""} }, }); + const userSettings = await ctx.db.query.users.findFirst({ + where: eq(users.id, ctx.user.id), + columns: { + inferredTagLang: true, + }, + }); + const summaryPrompt = await buildSummaryPrompt( - serverConfig.inference.inferredTagLang, + userSettings?.inferredTagLang ?? serverConfig.inference.inferredTagLang, prompts.map((p) => p.text), bookmarkDetails, serverConfig.inference.contextLength, @@ -981,12 +1017,14 @@ Author: ${bookmark.author ?? ""} summary: summary.response, }) .where(eq(bookmarks.id, input.bookmarkId)); - await triggerSearchReindex(input.bookmarkId, { - groupId: ctx.user.id, - }); - await triggerWebhook(input.bookmarkId, "edited", ctx.user.id, { - groupId: ctx.user.id, - }); + await Promise.all([ + triggerSearchReindex(input.bookmarkId, { + groupId: ctx.user.id, + }), + triggerWebhook(input.bookmarkId, "edited", ctx.user.id, { + groupId: ctx.user.id, + }), + ]); return { bookmarkId: input.bookmarkId, diff --git a/packages/trpc/routers/config.ts b/packages/trpc/routers/config.ts new file mode 100644 index 00000000..8d09a2ce --- /dev/null +++ b/packages/trpc/routers/config.ts @@ -0,0 +1,10 @@ +import { clientConfig } from "@karakeep/shared/config"; +import { zClientConfigSchema } from "@karakeep/shared/types/config"; + +import { publicProcedure, router } from "../index"; + +export const configAppRouter = router({ + clientConfig: publicProcedure + .output(zClientConfigSchema) + .query(() => clientConfig), +}); diff --git a/packages/trpc/routers/feeds.test.ts b/packages/trpc/routers/feeds.test.ts new file mode 100644 index 00000000..e80aab0a --- /dev/null +++ b/packages/trpc/routers/feeds.test.ts @@ -0,0 +1,154 @@ +import { beforeEach, describe, expect, test } from "vitest"; + +import type { CustomTestContext } from "../testUtils"; +import { defaultBeforeEach } from "../testUtils"; + +beforeEach<CustomTestContext>(defaultBeforeEach(true)); + +describe("Feed Routes", () => { + test<CustomTestContext>("create feed", async ({ apiCallers }) => { + const api = apiCallers[0].feeds; + const newFeed = await api.create({ + name: "Test Feed", + url: "https://example.com/feed.xml", + enabled: true, + }); + + expect(newFeed).toBeDefined(); + expect(newFeed.name).toEqual("Test Feed"); + expect(newFeed.url).toEqual("https://example.com/feed.xml"); + expect(newFeed.enabled).toBe(true); + }); + + test<CustomTestContext>("update feed", async ({ apiCallers }) => { + const api = apiCallers[0].feeds; + + // First, create a feed to update + const createdFeed = await api.create({ + name: "Test Feed", + url: "https://example.com/feed.xml", + enabled: true, + }); + + // Update it + const updatedFeed = await api.update({ + feedId: createdFeed.id, + name: "Updated Feed", + url: "https://updated-example.com/feed.xml", + enabled: false, + }); + + expect(updatedFeed.name).toEqual("Updated Feed"); + expect(updatedFeed.url).toEqual("https://updated-example.com/feed.xml"); + expect(updatedFeed.enabled).toBe(false); + + // Test updating a non-existent feed + await expect(() => + api.update({ + feedId: "non-existent-id", + name: "Fail", + url: "https://fail.com", + enabled: true, + }), + ).rejects.toThrow(/Feed not found/); + }); + + test<CustomTestContext>("list feeds", async ({ apiCallers }) => { + const api = apiCallers[0].feeds; + + // Create a couple of feeds + await api.create({ + name: "Feed 1", + url: "https://example1.com/feed.xml", + enabled: true, + }); + await api.create({ + name: "Feed 2", + url: "https://example2.com/feed.xml", + enabled: true, + }); + + const result = await api.list(); + expect(result.feeds).toBeDefined(); + expect(result.feeds.length).toBeGreaterThanOrEqual(2); + expect(result.feeds.some((f) => f.name === "Feed 1")).toBe(true); + expect(result.feeds.some((f) => f.name === "Feed 2")).toBe(true); + }); + + test<CustomTestContext>("delete feed", async ({ apiCallers }) => { + const api = apiCallers[0].feeds; + + // Create a feed to delete + const createdFeed = await api.create({ + name: "Test Feed", + url: "https://example.com/feed.xml", + enabled: true, + }); + + // Delete it + await api.delete({ feedId: createdFeed.id }); + + // Verify it's deleted + await expect(() => + api.update({ + feedId: createdFeed.id, + name: "Updated", + url: "https://updated.com", + enabled: true, + }), + ).rejects.toThrow(/Feed not found/); + }); + + test<CustomTestContext>("privacy for feeds", async ({ apiCallers }) => { + const user1Feed = await apiCallers[0].feeds.create({ + name: "User 1 Feed", + url: "https://user1-feed.com/feed.xml", + enabled: true, + }); + const user2Feed = await apiCallers[1].feeds.create({ + name: "User 2 Feed", + url: "https://user2-feed.com/feed.xml", + enabled: true, + }); + + // User 1 should not access User 2's feed + await expect(() => + apiCallers[0].feeds.delete({ feedId: user2Feed.id }), + ).rejects.toThrow(/User is not allowed to access resource/); + await expect(() => + apiCallers[0].feeds.update({ + feedId: user2Feed.id, + name: "Fail", + url: "https://fail.com", + enabled: true, + }), + ).rejects.toThrow(/User is not allowed to access resource/); + + // List should only show the correct user's feeds + const user1List = await apiCallers[0].feeds.list(); + expect(user1List.feeds.some((f) => f.id === user1Feed.id)).toBe(true); + expect(user1List.feeds.some((f) => f.id === user2Feed.id)).toBe(false); + }); + + test<CustomTestContext>("feed limit enforcement", async ({ apiCallers }) => { + const api = apiCallers[0].feeds; + + // Create 1000 feeds (the maximum) + for (let i = 0; i < 1000; i++) { + await api.create({ + name: `Feed ${i}`, + url: `https://example${i}.com/feed.xml`, + enabled: true, + }); + } + + // The 1001st feed should fail + await expect(() => + api.create({ + name: "Feed 1001", + url: "https://example1001.com/feed.xml", + enabled: true, + }), + ).rejects.toThrow(/Maximum number of RSS feeds \(1000\) reached/); + }); +}); diff --git a/packages/trpc/routers/importSessions.test.ts b/packages/trpc/routers/importSessions.test.ts index 9ef0de6f..f257ad3b 100644 --- a/packages/trpc/routers/importSessions.test.ts +++ b/packages/trpc/routers/importSessions.test.ts @@ -1,12 +1,13 @@ -import { eq } from "drizzle-orm"; import { beforeEach, describe, expect, test } from "vitest"; import { z } from "zod"; -import { bookmarks } from "@karakeep/db/schema"; import { - BookmarkTypes, - zNewBookmarkRequestSchema, -} from "@karakeep/shared/types/bookmarks"; + bookmarkLinks, + bookmarks, + bookmarkTexts, + importStagingBookmarks, +} from "@karakeep/db/schema"; +import { BookmarkTypes } from "@karakeep/shared/types/bookmarks"; import { zCreateImportSessionRequestSchema, zDeleteImportSessionRequestSchema, @@ -20,17 +21,6 @@ import { defaultBeforeEach } from "../testUtils"; beforeEach<CustomTestContext>(defaultBeforeEach(true)); describe("ImportSessions Routes", () => { - async function createTestBookmark(api: APICallerType, sessionId: string) { - const newBookmarkInput: z.infer<typeof zNewBookmarkRequestSchema> = { - type: BookmarkTypes.TEXT, - text: "Test bookmark text", - importSessionId: sessionId, - }; - const createdBookmark = - await api.bookmarks.createBookmark(newBookmarkInput); - return createdBookmark.id; - } - async function createTestList(api: APICallerType) { const newListInput: z.infer<typeof zNewBookmarkListSchema> = { name: "Test Import List", @@ -98,8 +88,15 @@ describe("ImportSessions Routes", () => { const session = await api.importSessions.createImportSession({ name: "Test Import Session", }); - await createTestBookmark(api, session.id); - await createTestBookmark(api, session.id); + + // Stage bookmarks using the staging flow + await api.importSessions.stageImportedBookmarks({ + importSessionId: session.id, + bookmarks: [ + { type: "text", content: "Test bookmark 1", tags: [], listIds: [] }, + { type: "text", content: "Test bookmark 2", tags: [], listIds: [] }, + ], + }); const statsInput: z.infer<typeof zGetImportSessionStatsRequestSchema> = { importSessionId: session.id, @@ -110,7 +107,7 @@ describe("ImportSessions Routes", () => { expect(stats).toMatchObject({ id: session.id, name: "Test Import Session", - status: "in_progress", + status: "staging", totalBookmarks: 2, pendingBookmarks: 2, completedBookmarks: 0, @@ -119,31 +116,191 @@ describe("ImportSessions Routes", () => { }); }); - test<CustomTestContext>("marks text-only imports as completed when tagging succeeds", async ({ + test<CustomTestContext>("stats reflect crawl and tagging status for completed staging bookmarks", async ({ apiCallers, db, }) => { const api = apiCallers[0]; + const session = await api.importSessions.createImportSession({ - name: "Text Import Session", + name: "Test Import Session", }); - const bookmarkId = await createTestBookmark(api, session.id); - await db - .update(bookmarks) - .set({ taggingStatus: "success" }) - .where(eq(bookmarks.id, bookmarkId)); + // Create bookmarks with different crawl/tag statuses + const user = (await db.query.users.findFirst())!; + + // 1. Link bookmark: crawl success, tag success -> completed + const [completedLinkBookmark] = await db + .insert(bookmarks) + .values({ + userId: user.id, + type: BookmarkTypes.LINK, + taggingStatus: "success", + }) + .returning(); + await db.insert(bookmarkLinks).values({ + id: completedLinkBookmark.id, + url: "https://example.com/1", + crawlStatus: "success", + }); + + // 2. Link bookmark: crawl pending, tag success -> processing + const [crawlPendingBookmark] = await db + .insert(bookmarks) + .values({ + userId: user.id, + type: BookmarkTypes.LINK, + taggingStatus: "success", + }) + .returning(); + await db.insert(bookmarkLinks).values({ + id: crawlPendingBookmark.id, + url: "https://example.com/2", + crawlStatus: "pending", + }); + + // 3. Text bookmark: tag pending -> processing + const [tagPendingBookmark] = await db + .insert(bookmarks) + .values({ + userId: user.id, + type: BookmarkTypes.TEXT, + taggingStatus: "pending", + }) + .returning(); + await db.insert(bookmarkTexts).values({ + id: tagPendingBookmark.id, + text: "Test text", + }); + + // 4. Link bookmark: crawl failure -> failed + const [crawlFailedBookmark] = await db + .insert(bookmarks) + .values({ + userId: user.id, + type: BookmarkTypes.LINK, + taggingStatus: "success", + }) + .returning(); + await db.insert(bookmarkLinks).values({ + id: crawlFailedBookmark.id, + url: "https://example.com/3", + crawlStatus: "failure", + }); + + // 5. Text bookmark: tag failure -> failed + const [tagFailedBookmark] = await db + .insert(bookmarks) + .values({ + userId: user.id, + type: BookmarkTypes.TEXT, + taggingStatus: "failure", + }) + .returning(); + await db.insert(bookmarkTexts).values({ + id: tagFailedBookmark.id, + text: "Test text 2", + }); + + // 6. Text bookmark: tag success (no crawl needed) -> completed + const [completedTextBookmark] = await db + .insert(bookmarks) + .values({ + userId: user.id, + type: BookmarkTypes.TEXT, + taggingStatus: "success", + }) + .returning(); + await db.insert(bookmarkTexts).values({ + id: completedTextBookmark.id, + text: "Test text 3", + }); + + // Create staging bookmarks in different states + // Note: With the new import worker design, items stay in "processing" until + // crawl/tag is done. Only then do they move to "completed". + await db.insert(importStagingBookmarks).values([ + // Staging pending -> pendingBookmarks + { + importSessionId: session.id, + type: "text", + content: "pending staging", + status: "pending", + }, + // Staging processing (no bookmark yet) -> processingBookmarks + { + importSessionId: session.id, + type: "text", + content: "processing staging", + status: "processing", + }, + // Staging failed -> failedBookmarks + { + importSessionId: session.id, + type: "text", + content: "failed staging", + status: "failed", + }, + // Staging completed + crawl/tag success -> completedBookmarks + { + importSessionId: session.id, + type: "link", + url: "https://example.com/1", + status: "completed", + resultBookmarkId: completedLinkBookmark.id, + }, + // Staging processing + crawl pending -> processingBookmarks (waiting for crawl) + { + importSessionId: session.id, + type: "link", + url: "https://example.com/2", + status: "processing", + resultBookmarkId: crawlPendingBookmark.id, + }, + // Staging processing + tag pending -> processingBookmarks (waiting for tag) + { + importSessionId: session.id, + type: "text", + content: "tag pending", + status: "processing", + resultBookmarkId: tagPendingBookmark.id, + }, + // Staging completed + crawl failure -> completedBookmarks (failure is terminal) + { + importSessionId: session.id, + type: "link", + url: "https://example.com/3", + status: "completed", + resultBookmarkId: crawlFailedBookmark.id, + }, + // Staging completed + tag failure -> completedBookmarks (failure is terminal) + { + importSessionId: session.id, + type: "text", + content: "tag failed", + status: "completed", + resultBookmarkId: tagFailedBookmark.id, + }, + // Staging completed + tag success (text, no crawl) -> completedBookmarks + { + importSessionId: session.id, + type: "text", + content: "completed text", + status: "completed", + resultBookmarkId: completedTextBookmark.id, + }, + ]); const stats = await api.importSessions.getImportSessionStats({ importSessionId: session.id, }); expect(stats).toMatchObject({ - completedBookmarks: 1, - pendingBookmarks: 0, - failedBookmarks: 0, - totalBookmarks: 1, - status: "completed", + totalBookmarks: 9, + pendingBookmarks: 1, // staging pending + processingBookmarks: 3, // staging processing (no bookmark) + crawl pending + tag pending + completedBookmarks: 4, // link success + text success + crawl failure + tag failure + failedBookmarks: 1, // staging failed }); }); @@ -215,7 +372,7 @@ describe("ImportSessions Routes", () => { ).rejects.toThrow("Import session not found"); }); - test<CustomTestContext>("cannot attach other user's bookmark", async ({ + test<CustomTestContext>("cannot stage other user's session", async ({ apiCallers, }) => { const api1 = apiCallers[0]; @@ -228,7 +385,17 @@ describe("ImportSessions Routes", () => { // User 1 tries to attach User 2's bookmark await expect( - createTestBookmark(api2, session.id), // User 2's bookmark + api2.importSessions.stageImportedBookmarks({ + importSessionId: session.id, + bookmarks: [ + { + type: "text", + content: "Test bookmark", + tags: [], + listIds: [], + }, + ], + }), ).rejects.toThrow("Import session not found"); }); }); diff --git a/packages/trpc/routers/importSessions.ts b/packages/trpc/routers/importSessions.ts index 4bdc4f29..62263bdd 100644 --- a/packages/trpc/routers/importSessions.ts +++ b/packages/trpc/routers/importSessions.ts @@ -1,5 +1,8 @@ +import { experimental_trpcMiddleware } from "@trpc/server"; +import { and, eq, gt } from "drizzle-orm"; import { z } from "zod"; +import { importStagingBookmarks } from "@karakeep/db/schema"; import { zCreateImportSessionRequestSchema, zDeleteImportSessionRequestSchema, @@ -9,9 +12,26 @@ import { zListImportSessionsResponseSchema, } from "@karakeep/shared/types/importSessions"; +import type { AuthedContext } from "../index"; import { authedProcedure, router } from "../index"; import { ImportSession } from "../models/importSessions"; +const ensureImportSessionAccess = experimental_trpcMiddleware<{ + ctx: AuthedContext; + input: { importSessionId: string }; +}>().create(async (opts) => { + const importSession = await ImportSession.fromId( + opts.ctx, + opts.input.importSessionId, + ); + return opts.next({ + ctx: { + ...opts.ctx, + importSession, + }, + }); +}); + export const importSessionsRouter = router({ createImportSession: authedProcedure .input(zCreateImportSessionRequestSchema) @@ -45,4 +65,93 @@ export const importSessionsRouter = router({ await session.delete(); return { success: true }; }), + + stageImportedBookmarks: authedProcedure + .input( + z.object({ + importSessionId: z.string(), + bookmarks: z + .array( + z.object({ + type: z.enum(["link", "text", "asset"]), + url: z.string().optional(), + title: z.string().optional(), + content: z.string().optional(), + note: z.string().optional(), + tags: z.array(z.string()).default([]), + listIds: z.array(z.string()).default([]), + sourceAddedAt: z.date().optional(), + }), + ) + .max(50), + }), + ) + .use(ensureImportSessionAccess) + .mutation(async ({ input, ctx }) => { + await ctx.importSession.stageBookmarks(input.bookmarks); + }), + + finalizeImportStaging: authedProcedure + .input(z.object({ importSessionId: z.string() })) + .use(ensureImportSessionAccess) + .mutation(async ({ ctx }) => { + await ctx.importSession.finalize(); + }), + + pauseImportSession: authedProcedure + .input(z.object({ importSessionId: z.string() })) + .use(ensureImportSessionAccess) + .mutation(async ({ ctx }) => { + await ctx.importSession.pause(); + }), + + resumeImportSession: authedProcedure + .input(z.object({ importSessionId: z.string() })) + .use(ensureImportSessionAccess) + .mutation(async ({ ctx }) => { + await ctx.importSession.resume(); + }), + + getImportSessionResults: authedProcedure + .input( + z.object({ + importSessionId: z.string(), + filter: z + .enum(["all", "accepted", "rejected", "skipped_duplicate", "pending"]) + .optional(), + cursor: z.string().optional(), + limit: z.number().default(50), + }), + ) + .use(ensureImportSessionAccess) + .query(async ({ ctx, input }) => { + const results = await ctx.db + .select() + .from(importStagingBookmarks) + .where( + and( + eq( + importStagingBookmarks.importSessionId, + ctx.importSession.session.id, + ), + input.filter && input.filter !== "all" + ? input.filter === "pending" + ? eq(importStagingBookmarks.status, "pending") + : eq(importStagingBookmarks.result, input.filter) + : undefined, + input.cursor + ? gt(importStagingBookmarks.id, input.cursor) + : undefined, + ), + ) + .orderBy(importStagingBookmarks.id) + .limit(input.limit + 1); + + // Return with pagination info + const hasMore = results.length > input.limit; + return { + items: results.slice(0, input.limit), + nextCursor: hasMore ? results[input.limit - 1].id : null, + }; + }), }); diff --git a/packages/trpc/routers/lists.test.ts b/packages/trpc/routers/lists.test.ts index 8797b35e..214df32a 100644 --- a/packages/trpc/routers/lists.test.ts +++ b/packages/trpc/routers/lists.test.ts @@ -594,3 +594,385 @@ describe("recursive delete", () => { expect(lists.lists.find((l) => l.id === child.id)).toBeUndefined(); }); }); + +describe("Nested smart lists", () => { + test<CustomTestContext>("smart list can reference another smart list", async ({ + apiCallers, + }) => { + const api = apiCallers[0]; + + // Create a bookmark that is favourited + const bookmark1 = await api.bookmarks.createBookmark({ + type: BookmarkTypes.TEXT, + text: "Favourited bookmark", + }); + await api.bookmarks.updateBookmark({ + bookmarkId: bookmark1.id, + favourited: true, + }); + + // Create a bookmark that is not favourited + const bookmark2 = await api.bookmarks.createBookmark({ + type: BookmarkTypes.TEXT, + text: "Non-favourited bookmark", + }); + + // Create a smart list that matches favourited bookmarks + await api.lists.create({ + name: "Favourites", + type: "smart", + query: "is:fav", + icon: "â", + }); + + // Create a smart list that references the first smart list + const smartListB = await api.lists.create({ + name: "From Favourites", + type: "smart", + query: "list:Favourites", + icon: "đ", + }); + + // Get bookmarks from the nested smart list + const bookmarksInSmartListB = await api.bookmarks.getBookmarks({ + listId: smartListB.id, + }); + + // Should contain the favourited bookmark + expect(bookmarksInSmartListB.bookmarks.length).toBe(1); + expect(bookmarksInSmartListB.bookmarks[0].id).toBe(bookmark1.id); + + // Verify bookmark2 is not in the nested smart list + expect( + bookmarksInSmartListB.bookmarks.find((b) => b.id === bookmark2.id), + ).toBeUndefined(); + }); + + test<CustomTestContext>("nested smart lists with multiple levels", async ({ + apiCallers, + }) => { + const api = apiCallers[0]; + + // Create a bookmark that is archived + const bookmark = await api.bookmarks.createBookmark({ + type: BookmarkTypes.TEXT, + text: "Archived bookmark", + }); + await api.bookmarks.updateBookmark({ + bookmarkId: bookmark.id, + archived: true, + }); + + // Create smart list A: matches archived bookmarks + await api.lists.create({ + name: "Archived", + type: "smart", + query: "is:archived", + icon: "đĻ", + }); + + // Create smart list B: references list A + await api.lists.create({ + name: "Level1", + type: "smart", + query: "list:Archived", + icon: "1ī¸âŖ", + }); + + // Create smart list C: references list B (3 levels deep) + const smartListC = await api.lists.create({ + name: "Level2", + type: "smart", + query: "list:Level1", + icon: "2ī¸âŖ", + }); + + // Get bookmarks from the deepest nested smart list + const bookmarksInSmartListC = await api.bookmarks.getBookmarks({ + listId: smartListC.id, + }); + + // Should contain the archived bookmark + expect(bookmarksInSmartListC.bookmarks.length).toBe(1); + expect(bookmarksInSmartListC.bookmarks[0].id).toBe(bookmark.id); + }); + + test<CustomTestContext>("smart list with inverse reference to another smart list", async ({ + apiCallers, + }) => { + const api = apiCallers[0]; + + // Create two bookmarks + const favouritedBookmark = await api.bookmarks.createBookmark({ + type: BookmarkTypes.TEXT, + text: "Favourited bookmark", + }); + await api.bookmarks.updateBookmark({ + bookmarkId: favouritedBookmark.id, + favourited: true, + }); + + const normalBookmark = await api.bookmarks.createBookmark({ + type: BookmarkTypes.TEXT, + text: "Normal bookmark", + }); + + // Create a smart list that matches favourited bookmarks + await api.lists.create({ + name: "Favourites", + type: "smart", + query: "is:fav", + icon: "â", + }); + + // Create a smart list with negative reference to Favourites + const notInFavourites = await api.lists.create({ + name: "Not In Favourites", + type: "smart", + query: "-list:Favourites", + icon: "â", + }); + + // Get bookmarks from the smart list + const bookmarksNotInFav = await api.bookmarks.getBookmarks({ + listId: notInFavourites.id, + }); + + // Should contain only the non-favourited bookmark + expect(bookmarksNotInFav.bookmarks.length).toBe(1); + expect(bookmarksNotInFav.bookmarks[0].id).toBe(normalBookmark.id); + }); + + test<CustomTestContext>("circular reference between smart lists returns empty", async ({ + apiCallers, + }) => { + const api = apiCallers[0]; + + // Create a bookmark + const bookmark = await api.bookmarks.createBookmark({ + type: BookmarkTypes.TEXT, + text: "Test bookmark", + }); + await api.bookmarks.updateBookmark({ + bookmarkId: bookmark.id, + favourited: true, + }); + + // Create smart list A that references smart list B + const smartListA = await api.lists.create({ + name: "ListA", + type: "smart", + query: "list:ListB", + icon: "đ
°ī¸", + }); + + // Create smart list B that references smart list A (circular!) + await api.lists.create({ + name: "ListB", + type: "smart", + query: "list:ListA", + icon: "đ
ąī¸", + }); + + // Querying ListA should return empty because of the circular reference + const bookmarksInListA = await api.bookmarks.getBookmarks({ + listId: smartListA.id, + }); + + // Should be empty due to circular reference detection + expect(bookmarksInListA.bookmarks.length).toBe(0); + }); + + test<CustomTestContext>("self-referencing smart list returns empty", async ({ + apiCallers, + }) => { + const api = apiCallers[0]; + + // Create a bookmark + await api.bookmarks.createBookmark({ + type: BookmarkTypes.TEXT, + text: "Test bookmark", + }); + + // Create a smart list that references itself + const selfRefList = await api.lists.create({ + name: "SelfRef", + type: "smart", + query: "list:SelfRef", + icon: "đ", + }); + + // Querying should return empty because of self-reference + const bookmarks = await api.bookmarks.getBookmarks({ + listId: selfRefList.id, + }); + + expect(bookmarks.bookmarks.length).toBe(0); + }); + + test<CustomTestContext>("three-way circular reference returns empty", async ({ + apiCallers, + }) => { + const api = apiCallers[0]; + + // Create a bookmark + await api.bookmarks.createBookmark({ + type: BookmarkTypes.TEXT, + text: "Test bookmark", + }); + + // Create three smart lists with circular references: A -> B -> C -> A + const listA = await api.lists.create({ + name: "CircularA", + type: "smart", + query: "list:CircularB", + icon: "đ
°ī¸", + }); + + await api.lists.create({ + name: "CircularB", + type: "smart", + query: "list:CircularC", + icon: "đ
ąī¸", + }); + + await api.lists.create({ + name: "CircularC", + type: "smart", + query: "list:CircularA", + icon: "Šī¸", + }); + + // Querying any of them should return empty due to circular reference + const bookmarksInListA = await api.bookmarks.getBookmarks({ + listId: listA.id, + }); + + expect(bookmarksInListA.bookmarks.length).toBe(0); + }); + + test<CustomTestContext>("smart list traversal above max visited lists returns empty", async ({ + apiCallers, + }) => { + const api = apiCallers[0]; + + const bookmark = await api.bookmarks.createBookmark({ + type: BookmarkTypes.TEXT, + text: "Depth test bookmark", + }); + + const manualList = await api.lists.create({ + name: "DepthBaseManual", + type: "manual", + icon: "đ", + }); + await api.lists.addToList({ + listId: manualList.id, + bookmarkId: bookmark.id, + }); + + const maxVisitedLists = 30; + const overLimitChainLength = maxVisitedLists + 1; + + for (let i = overLimitChainLength; i >= 2; i--) { + await api.lists.create({ + name: `DepthL${i}`, + type: "smart", + query: + i === overLimitChainLength + ? "list:DepthBaseManual" + : `list:DepthL${i + 1}`, + icon: "D", + }); + } + + const depthRoot = await api.lists.create({ + name: "DepthL1", + type: "smart", + query: "list:DepthL2", + icon: "D", + }); + + const bookmarksInRoot = await api.bookmarks.getBookmarks({ + listId: depthRoot.id, + }); + + expect(bookmarksInRoot.bookmarks.length).toBe(0); + }); + + test<CustomTestContext>("smart list references non-existent list returns empty", async ({ + apiCallers, + }) => { + const api = apiCallers[0]; + + // Create a bookmark + await api.bookmarks.createBookmark({ + type: BookmarkTypes.TEXT, + text: "Test bookmark", + }); + + // Create a smart list that references a non-existent list + const smartList = await api.lists.create({ + name: "RefNonExistent", + type: "smart", + query: "list:NonExistentList", + icon: "â", + }); + + // Should return empty since the referenced list doesn't exist + const bookmarks = await api.bookmarks.getBookmarks({ + listId: smartList.id, + }); + + expect(bookmarks.bookmarks.length).toBe(0); + }); + + test<CustomTestContext>("smart list can reference manual list", async ({ + apiCallers, + }) => { + const api = apiCallers[0]; + + // Create bookmarks + const bookmark1 = await api.bookmarks.createBookmark({ + type: BookmarkTypes.TEXT, + text: "Bookmark in manual list", + }); + const bookmark2 = await api.bookmarks.createBookmark({ + type: BookmarkTypes.TEXT, + text: "Bookmark not in list", + }); + + // Create a manual list and add bookmark1 + const manualList = await api.lists.create({ + name: "ManualList", + type: "manual", + icon: "đ", + }); + await api.lists.addToList({ + listId: manualList.id, + bookmarkId: bookmark1.id, + }); + + // Create a smart list that references the manual list + const smartList = await api.lists.create({ + name: "SmartRefManual", + type: "smart", + query: "list:ManualList", + icon: "đ", + }); + + // Get bookmarks from the smart list + const bookmarksInSmartList = await api.bookmarks.getBookmarks({ + listId: smartList.id, + }); + + // Should contain only bookmark1 + expect(bookmarksInSmartList.bookmarks.length).toBe(1); + expect(bookmarksInSmartList.bookmarks[0].id).toBe(bookmark1.id); + + // Verify bookmark2 is not in the smart list + expect( + bookmarksInSmartList.bookmarks.find((b) => b.id === bookmark2.id), + ).toBeUndefined(); + }); +}); diff --git a/packages/trpc/routers/lists.ts b/packages/trpc/routers/lists.ts index 296679f3..bca3dc53 100644 --- a/packages/trpc/routers/lists.ts +++ b/packages/trpc/routers/lists.ts @@ -302,6 +302,7 @@ export const listsAppRouter = router({ id: z.string(), name: z.string(), email: z.string().nullable(), + image: z.string().nullable(), }), }), ), @@ -310,6 +311,7 @@ export const listsAppRouter = router({ id: z.string(), name: z.string(), email: z.string().nullable(), + image: z.string().nullable(), }) .nullable(), }), diff --git a/packages/trpc/routers/tags.ts b/packages/trpc/routers/tags.ts index d4cfbe8c..5713c192 100644 --- a/packages/trpc/routers/tags.ts +++ b/packages/trpc/routers/tags.ts @@ -102,6 +102,7 @@ export const tagsAppRouter = router({ .query(async ({ ctx, input }) => { return await Tag.getAll(ctx, { nameContains: input.nameContains, + ids: input.ids, attachedBy: input.attachedBy, sortBy: input.sortBy, pagination: input.limit diff --git a/packages/trpc/routers/users.test.ts b/packages/trpc/routers/users.test.ts index a2f2be9f..d8ec90f9 100644 --- a/packages/trpc/routers/users.test.ts +++ b/packages/trpc/routers/users.test.ts @@ -158,6 +158,18 @@ describe("User Routes", () => { backupsEnabled: false, backupsFrequency: "weekly", backupsRetentionDays: 30, + + // Reader settings + readerFontFamily: null, + readerFontSize: null, + readerLineHeight: null, + + // AI Settings + autoSummarizationEnabled: null, + autoTaggingEnabled: null, + curatedTagIds: null, + inferredTagLang: null, + tagStyle: "titlecase-spaces", }); // Update settings @@ -166,6 +178,17 @@ describe("User Routes", () => { backupsEnabled: true, backupsFrequency: "daily", backupsRetentionDays: 7, + + // Reader settings + readerFontFamily: "serif", + readerFontSize: 12, + readerLineHeight: 1.5, + + // AI Settings + autoSummarizationEnabled: true, + autoTaggingEnabled: true, + inferredTagLang: "en", + tagStyle: "lowercase-underscores", }); // Verify updated settings @@ -177,6 +200,18 @@ describe("User Routes", () => { backupsEnabled: true, backupsFrequency: "daily", backupsRetentionDays: 7, + + // Reader settings + readerFontFamily: "serif", + readerFontSize: 12, + readerLineHeight: 1.5, + + // AI Settings + autoSummarizationEnabled: true, + autoTaggingEnabled: true, + curatedTagIds: null, + inferredTagLang: "en", + tagStyle: "lowercase-underscores", }); // Test invalid update (e.g., empty input, if schema enforces it) @@ -915,6 +950,81 @@ describe("User Routes", () => { }); }); + describe("Update Avatar", () => { + test<CustomTestContext>("updateAvatar - promotes unknown asset", async ({ + db, + unauthedAPICaller, + }) => { + const user = await unauthedAPICaller.users.create({ + name: "Avatar Reject", + email: "avatar-reject@test.com", + password: "pass1234", + confirmPassword: "pass1234", + }); + const caller = getApiCaller(db, user.id, user.email, user.role || "user"); + + await db.insert(assets).values({ + id: "avatar-asset-2", + assetType: AssetTypes.UNKNOWN, + userId: user.id, + contentType: "image/png", + size: 12, + fileName: "avatar.png", + bookmarkId: null, + }); + + await caller.users.updateAvatar({ assetId: "avatar-asset-2" }); + + const updatedAsset = await db + .select() + .from(assets) + .where(eq(assets.id, "avatar-asset-2")) + .then((rows) => rows[0]); + + expect(updatedAsset?.assetType).toBe(AssetTypes.AVATAR); + }); + + test<CustomTestContext>("updateAvatar - deletes avatar asset", async ({ + db, + unauthedAPICaller, + }) => { + const user = await unauthedAPICaller.users.create({ + name: "Avatar Delete", + email: "avatar-delete@test.com", + password: "pass1234", + confirmPassword: "pass1234", + }); + const caller = getApiCaller(db, user.id, user.email, user.role || "user"); + + await db.insert(assets).values({ + id: "avatar-asset-3", + assetType: AssetTypes.UNKNOWN, + userId: user.id, + contentType: "image/png", + size: 12, + fileName: "avatar.png", + bookmarkId: null, + }); + + await caller.users.updateAvatar({ assetId: "avatar-asset-3" }); + await caller.users.updateAvatar({ assetId: null }); + + const updatedUser = await db + .select() + .from(users) + .where(eq(users.id, user.id)) + .then((rows) => rows[0]); + const remainingAsset = await db + .select() + .from(assets) + .where(eq(assets.id, "avatar-asset-3")) + .then((rows) => rows[0]); + + expect(updatedUser?.image).toBeNull(); + expect(remainingAsset).toBeUndefined(); + }); + }); + describe("Who Am I", () => { test<CustomTestContext>("whoami - returns user info", async ({ db, @@ -1008,6 +1118,7 @@ describe("User Routes", () => { "resend@test.com", "Test User", expect.any(String), // token + undefined, // redirectUrl ); }); diff --git a/packages/trpc/routers/users.ts b/packages/trpc/routers/users.ts index d3bc06d9..c11a0ffd 100644 --- a/packages/trpc/routers/users.ts +++ b/packages/trpc/routers/users.ts @@ -9,7 +9,9 @@ import { zUserSettingsSchema, zUserStatsResponseSchema, zWhoAmIResponseSchema, + zWrappedStatsResponseSchema, } from "@karakeep/shared/types/users"; +import { validateRedirectUrl } from "@karakeep/shared/utils/redirectUrl"; import { adminProcedure, @@ -30,7 +32,7 @@ export const usersAppRouter = router({ maxRequests: 3, }), ) - .input(zSignUpSchema) + .input(zSignUpSchema.and(z.object({ redirectUrl: z.string().optional() }))) .output( z.object({ id: z.string(), @@ -64,7 +66,11 @@ export const usersAppRouter = router({ }); } } - const user = await User.create(ctx, input); + const validatedRedirectUrl = validateRedirectUrl(input.redirectUrl); + const user = await User.create(ctx, { + ...input, + redirectUrl: validatedRedirectUrl, + }); return { id: user.id, name: user.name, @@ -136,6 +142,24 @@ export const usersAppRouter = router({ const user = await User.fromCtx(ctx); return await user.getStats(); }), + wrapped: authedProcedure + .output(zWrappedStatsResponseSchema) + .query(async ({ ctx }) => { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "This endpoint is currently disabled", + }); + const user = await User.fromCtx(ctx); + return await user.getWrappedStats(2025); + }), + hasWrapped: authedProcedure.output(z.boolean()).query(async ({ ctx }) => { + throw new TRPCError({ + code: "BAD_REQUEST", + message: "This endpoint is currently disabled", + }); + const user = await User.fromCtx(ctx); + return await user.hasWrapped(); + }), settings: authedProcedure .output(zUserSettingsSchema) .query(async ({ ctx }) => { @@ -148,6 +172,16 @@ export const usersAppRouter = router({ const user = await User.fromCtx(ctx); await user.updateSettings(input); }), + updateAvatar: authedProcedure + .input( + z.object({ + assetId: z.string().nullable(), + }), + ) + .mutation(async ({ input, ctx }) => { + const user = await User.fromCtx(ctx); + await user.updateAvatar(input.assetId); + }), verifyEmail: publicProcedure .use( createRateLimitMiddleware({ @@ -177,10 +211,16 @@ export const usersAppRouter = router({ .input( z.object({ email: z.string().email(), + redirectUrl: z.string().optional(), }), ) .mutation(async ({ input, ctx }) => { - await User.resendVerificationEmail(ctx, input.email); + const validatedRedirectUrl = validateRedirectUrl(input.redirectUrl); + await User.resendVerificationEmail( + ctx, + input.email, + validatedRedirectUrl, + ); return { success: true }; }), forgotPassword: publicProcedure diff --git a/packages/trpc/routers/webhooks.test.ts b/packages/trpc/routers/webhooks.test.ts index 5a136a31..de27b11e 100644 --- a/packages/trpc/routers/webhooks.test.ts +++ b/packages/trpc/routers/webhooks.test.ts @@ -125,4 +125,26 @@ describe("Webhook Routes", () => { false, ); }); + + test<CustomTestContext>("webhook limit enforcement", async ({ + apiCallers, + }) => { + const api = apiCallers[0].webhooks; + + // Create 100 webhooks (the maximum) + for (let i = 0; i < 100; i++) { + await api.create({ + url: `https://example${i}.com/webhook`, + events: ["created"], + }); + } + + // The 101st webhook should fail + await expect(() => + api.create({ + url: "https://example101.com/webhook", + events: ["created"], + }), + ).rejects.toThrow(/Maximum number of webhooks \(100\) reached/); + }); }); |