From 4a580d713621f99abb8baabc9b847ce039d44842 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sat, 4 Oct 2025 13:40:24 +0100 Subject: feat: Revamp import experience (#2001) * WIP: import v2 * remove new session button * don't redirect after import * store and lint to root list * models + tests * redesign the progress * simplify the import session for ow * drop status from session schema * split the import session page * i18n * fix test * remove pagination * fix some colors in darkmode * one last fix * add privacy filter * privacy check * fix interactivity of import progress * fix test --- packages/trpc/models/importSessions.ts | 180 +++++++++++++++++++++++ packages/trpc/routers/_app.ts | 2 + packages/trpc/routers/bookmarks.ts | 15 +- packages/trpc/routers/importSessions.test.ts | 204 +++++++++++++++++++++++++++ packages/trpc/routers/importSessions.ts | 48 +++++++ 5 files changed, 448 insertions(+), 1 deletion(-) create mode 100644 packages/trpc/models/importSessions.ts create mode 100644 packages/trpc/routers/importSessions.test.ts create mode 100644 packages/trpc/routers/importSessions.ts (limited to 'packages/trpc') diff --git a/packages/trpc/models/importSessions.ts b/packages/trpc/models/importSessions.ts new file mode 100644 index 00000000..270c2bce --- /dev/null +++ b/packages/trpc/models/importSessions.ts @@ -0,0 +1,180 @@ +import { TRPCError } from "@trpc/server"; +import { and, count, eq } from "drizzle-orm"; +import { z } from "zod"; + +import { + bookmarkLinks, + bookmarks, + importSessionBookmarks, + importSessions, +} from "@karakeep/db/schema"; +import { + zCreateImportSessionRequestSchema, + ZImportSession, + ZImportSessionWithStats, +} from "@karakeep/shared/types/importSessions"; + +import type { AuthedContext } from "../index"; +import { PrivacyAware } from "./privacy"; + +export class ImportSession implements PrivacyAware { + protected constructor( + protected ctx: AuthedContext, + public session: ZImportSession, + ) {} + + static async fromId( + ctx: AuthedContext, + importSessionId: string, + ): Promise { + const session = await ctx.db.query.importSessions.findFirst({ + where: and( + eq(importSessions.id, importSessionId), + eq(importSessions.userId, ctx.user.id), + ), + }); + + if (!session) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Import session not found", + }); + } + + return new ImportSession(ctx, session); + } + + static async create( + ctx: AuthedContext, + input: z.infer, + ): Promise { + const [session] = await ctx.db + .insert(importSessions) + .values({ + name: input.name, + userId: ctx.user.id, + rootListId: input.rootListId, + }) + .returning(); + + return new ImportSession(ctx, session); + } + + static async getAll(ctx: AuthedContext): Promise { + const sessions = await ctx.db.query.importSessions.findMany({ + where: eq(importSessions.userId, ctx.user.id), + orderBy: (importSessions, { desc }) => [desc(importSessions.createdAt)], + limit: 50, + }); + + return sessions.map((session) => new ImportSession(ctx, session)); + } + + static async getAllWithStats( + ctx: AuthedContext, + ): Promise { + const sessions = await this.getAll(ctx); + + return await Promise.all( + sessions.map(async (session) => { + return await session.getWithStats(); + }), + ); + } + + ensureCanAccess(ctx: AuthedContext): void { + if (this.session.userId !== ctx.user.id) { + throw new TRPCError({ + code: "FORBIDDEN", + message: "User is not allowed to access this import session", + }); + } + } + + async attachBookmark(bookmarkId: string): Promise { + await this.ctx.db.insert(importSessionBookmarks).values({ + importSessionId: this.session.id, + bookmarkId, + }); + } + + async getWithStats(): Promise { + // Get bookmark counts by status + const statusCounts = await this.ctx.db + .select({ + crawlStatus: bookmarkLinks.crawlStatus, + taggingStatus: bookmarks.taggingStatus, + count: count(), + }) + .from(importSessionBookmarks) + .innerJoin( + importSessions, + eq(importSessions.id, importSessionBookmarks.importSessionId), + ) + .leftJoin(bookmarks, eq(bookmarks.id, importSessionBookmarks.bookmarkId)) + .leftJoin( + bookmarkLinks, + eq(bookmarkLinks.id, importSessionBookmarks.bookmarkId), + ) + .where( + and( + eq(importSessionBookmarks.importSessionId, this.session.id), + eq(importSessions.userId, this.ctx.user.id), + ), + ) + .groupBy(bookmarkLinks.crawlStatus, bookmarks.taggingStatus); + + const stats = { + totalBookmarks: 0, + completedBookmarks: 0, + failedBookmarks: 0, + pendingBookmarks: 0, + processingBookmarks: 0, + }; + + statusCounts.forEach((statusCount) => { + stats.totalBookmarks += statusCount.count; + if ( + statusCount.crawlStatus === "success" && + statusCount.taggingStatus === "success" + ) { + stats.completedBookmarks += statusCount.count; + } else if ( + statusCount.crawlStatus === "failure" || + statusCount.taggingStatus === "failure" + ) { + stats.failedBookmarks += statusCount.count; + } else if ( + statusCount.crawlStatus === "pending" || + statusCount.taggingStatus === "pending" + ) { + stats.pendingBookmarks += statusCount.count; + } + }); + + return { + ...this.session, + status: stats.pendingBookmarks > 0 ? "in_progress" : "completed", + ...stats, + }; + } + + async delete(): Promise { + // Delete the session (cascade will handle the bookmarks) + const result = await this.ctx.db + .delete(importSessions) + .where( + and( + eq(importSessions.id, this.session.id), + eq(importSessions.userId, this.ctx.user.id), + ), + ); + + if (result.changes === 0) { + throw new TRPCError({ + code: "NOT_FOUND", + message: "Import session not found", + }); + } + } +} diff --git a/packages/trpc/routers/_app.ts b/packages/trpc/routers/_app.ts index 651c8d88..1d548ee4 100644 --- a/packages/trpc/routers/_app.ts +++ b/packages/trpc/routers/_app.ts @@ -5,6 +5,7 @@ import { assetsAppRouter } from "./assets"; import { bookmarksAppRouter } from "./bookmarks"; import { feedsAppRouter } from "./feeds"; import { highlightsAppRouter } from "./highlights"; +import { importSessionsRouter } from "./importSessions"; import { invitesAppRouter } from "./invites"; import { listsAppRouter } from "./lists"; import { promptsAppRouter } from "./prompts"; @@ -25,6 +26,7 @@ export const appRouter = router({ admin: adminAppRouter, feeds: feedsAppRouter, highlights: highlightsAppRouter, + importSessions: importSessionsRouter, webhooks: webhooksAppRouter, assets: assetsAppRouter, rules: rulesAppRouter, diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts index 3399bf19..be3664b3 100644 --- a/packages/trpc/routers/bookmarks.ts +++ b/packages/trpc/routers/bookmarks.ts @@ -59,6 +59,7 @@ import { authedProcedure, createRateLimitMiddleware, router } from "../index"; import { mapDBAssetTypeToUserType } from "../lib/attachments"; import { getBookmarkIdsFromMatcher } from "../lib/search"; import { Bookmark } from "../models/bookmarks"; +import { ImportSession } from "../models/importSessions"; import { ensureAssetOwnership } from "./assets"; export const ensureBookmarkOwnership = experimental_trpcMiddleware<{ @@ -272,6 +273,13 @@ export const bookmarksAppRouter = router({ // This doesn't 100% protect from duplicates because of races, but it's more than enough for this usecase. const alreadyExists = await attemptToDedupLink(ctx, input.url); if (alreadyExists) { + if (input.importSessionId) { + const session = await ImportSession.fromId( + ctx, + input.importSessionId, + ); + await session.attachBookmark(alreadyExists.id); + } return { ...alreadyExists, alreadyExists: true }; } } @@ -416,12 +424,16 @@ export const bookmarksAppRouter = router({ }; }); + if (input.importSessionId) { + const session = await ImportSession.fromId(ctx, input.importSessionId); + await session.attachBookmark(bookmark.id); + } + const enqueueOpts: EnqueueOptions = { // The lower the priority number, the sooner the job will be processed priority: input.crawlPriority === "low" ? 50 : 0, }; - // Enqueue crawling request switch (bookmark.content.type) { case BookmarkTypes.LINK: { // The crawling job triggers openai when it's done @@ -454,6 +466,7 @@ export const bookmarksAppRouter = router({ break; } } + await triggerRuleEngineOnEvent( bookmark.id, [ diff --git a/packages/trpc/routers/importSessions.test.ts b/packages/trpc/routers/importSessions.test.ts new file mode 100644 index 00000000..b28d1421 --- /dev/null +++ b/packages/trpc/routers/importSessions.test.ts @@ -0,0 +1,204 @@ +import { beforeEach, describe, expect, test } from "vitest"; +import { z } from "zod"; + +import { + BookmarkTypes, + zNewBookmarkRequestSchema, +} from "@karakeep/shared/types/bookmarks"; +import { + zCreateImportSessionRequestSchema, + zDeleteImportSessionRequestSchema, + zGetImportSessionStatsRequestSchema, +} from "@karakeep/shared/types/importSessions"; +import { zNewBookmarkListSchema } from "@karakeep/shared/types/lists"; + +import type { APICallerType, CustomTestContext } from "../testUtils"; +import { defaultBeforeEach } from "../testUtils"; + +beforeEach(defaultBeforeEach(true)); + +describe("ImportSessions Routes", () => { + async function createTestBookmark(api: APICallerType, sessionId: string) { + const newBookmarkInput: z.infer = { + type: BookmarkTypes.TEXT, + text: "Test bookmark text", + importSessionId: sessionId, + }; + const createdBookmark = + await api.bookmarks.createBookmark(newBookmarkInput); + return createdBookmark.id; + } + + async function createTestList(api: APICallerType) { + const newListInput: z.infer = { + name: "Test Import List", + description: "A test list for imports", + icon: "📋", + type: "manual", + }; + const createdList = await api.lists.create(newListInput); + return createdList.id; + } + + test("create import session", async ({ apiCallers }) => { + const api = apiCallers[0].importSessions; + const listId = await createTestList(apiCallers[0]); + + const newSessionInput: z.infer = { + name: "Test Import Session", + rootListId: listId, + }; + + const createdSession = await api.createImportSession(newSessionInput); + + expect(createdSession).toMatchObject({ + id: expect.any(String), + }); + + // Verify session appears in list + const sessions = await api.listImportSessions({}); + const sessionFromList = sessions.sessions.find( + (s) => s.id === createdSession.id, + ); + expect(sessionFromList).toBeDefined(); + expect(sessionFromList?.name).toEqual(newSessionInput.name); + expect(sessionFromList?.rootListId).toEqual(listId); + }); + + test("create import session without rootListId", async ({ + apiCallers, + }) => { + const api = apiCallers[0].importSessions; + + const newSessionInput: z.infer = { + name: "Test Import Session", + }; + + const createdSession = await api.createImportSession(newSessionInput); + + expect(createdSession).toMatchObject({ + id: expect.any(String), + }); + + // Verify session appears in list + const sessions = await api.listImportSessions({}); + const sessionFromList = sessions.sessions.find( + (s) => s.id === createdSession.id, + ); + expect(sessionFromList?.rootListId).toBeNull(); + }); + + test("get import session stats", async ({ + apiCallers, + }) => { + const api = apiCallers[0]; + + const session = await api.importSessions.createImportSession({ + name: "Test Import Session", + }); + await createTestBookmark(api, session.id); + await createTestBookmark(api, session.id); + + const statsInput: z.infer = { + importSessionId: session.id, + }; + + const stats = await api.importSessions.getImportSessionStats(statsInput); + + expect(stats).toMatchObject({ + id: session.id, + name: "Test Import Session", + status: "in_progress", + totalBookmarks: 2, + pendingBookmarks: 2, + completedBookmarks: 0, + failedBookmarks: 0, + processingBookmarks: 0, + }); + }); + + test("list import sessions returns all sessions", async ({ + apiCallers, + }) => { + const api = apiCallers[0].importSessions; + + const sessionNames = ["Session 1", "Session 2", "Session 3"]; + for (const name of sessionNames) { + await api.createImportSession({ name }); + } + + const result = await api.listImportSessions({}); + + expect(result.sessions).toHaveLength(3); + expect(result.sessions.map((session) => session.name)).toEqual( + sessionNames, + ); + expect( + result.sessions.every((session) => session.totalBookmarks === 0), + ).toBe(true); + }); + + test("delete import session", async ({ apiCallers }) => { + const api = apiCallers[0].importSessions; + + const session = await api.createImportSession({ + name: "Session to Delete", + }); + + const deleteInput: z.infer = { + importSessionId: session.id, + }; + + const result = await api.deleteImportSession(deleteInput); + expect(result.success).toBe(true); + + // Verify session no longer exists + await expect( + api.getImportSessionStats({ + importSessionId: session.id, + }), + ).rejects.toThrow("Import session not found"); + }); + + test("cannot access other user's session", async ({ + apiCallers, + }) => { + const api1 = apiCallers[0].importSessions; + const api2 = apiCallers[1].importSessions; + + // User 1 creates a session + const session = await api1.createImportSession({ + name: "User 1 Session", + }); + + // User 2 tries to access it + await expect( + api2.getImportSessionStats({ + importSessionId: session.id, + }), + ).rejects.toThrow("Import session not found"); + + await expect( + api2.deleteImportSession({ + importSessionId: session.id, + }), + ).rejects.toThrow("Import session not found"); + }); + + test("cannot attach other user's bookmark", async ({ + apiCallers, + }) => { + const api1 = apiCallers[0]; + const api2 = apiCallers[1]; + + // User 1 creates session and bookmark + const session = await api1.importSessions.createImportSession({ + name: "User 1 Session", + }); + + // User 1 tries to attach User 2's bookmark + await expect( + createTestBookmark(api2, session.id), // User 2's bookmark + ).rejects.toThrow("Import session not found"); + }); +}); diff --git a/packages/trpc/routers/importSessions.ts b/packages/trpc/routers/importSessions.ts new file mode 100644 index 00000000..4bdc4f29 --- /dev/null +++ b/packages/trpc/routers/importSessions.ts @@ -0,0 +1,48 @@ +import { z } from "zod"; + +import { + zCreateImportSessionRequestSchema, + zDeleteImportSessionRequestSchema, + zGetImportSessionStatsRequestSchema, + zImportSessionWithStatsSchema, + zListImportSessionsRequestSchema, + zListImportSessionsResponseSchema, +} from "@karakeep/shared/types/importSessions"; + +import { authedProcedure, router } from "../index"; +import { ImportSession } from "../models/importSessions"; + +export const importSessionsRouter = router({ + createImportSession: authedProcedure + .input(zCreateImportSessionRequestSchema) + .output(z.object({ id: z.string() })) + .mutation(async ({ input, ctx }) => { + const session = await ImportSession.create(ctx, input); + return { id: session.session.id }; + }), + + getImportSessionStats: authedProcedure + .input(zGetImportSessionStatsRequestSchema) + .output(zImportSessionWithStatsSchema) + .query(async ({ input, ctx }) => { + const session = await ImportSession.fromId(ctx, input.importSessionId); + return await session.getWithStats(); + }), + + listImportSessions: authedProcedure + .input(zListImportSessionsRequestSchema) + .output(zListImportSessionsResponseSchema) + .query(async ({ ctx }) => { + const sessions = await ImportSession.getAllWithStats(ctx); + return { sessions }; + }), + + deleteImportSession: authedProcedure + .input(zDeleteImportSessionRequestSchema) + .output(z.object({ success: z.boolean() })) + .mutation(async ({ input, ctx }) => { + const session = await ImportSession.fromId(ctx, input.importSessionId); + await session.delete(); + return { success: true }; + }), +}); -- cgit v1.2.3-70-g09d2