feat: Revamp import experience (#2001)

* WIP: import v2

* remove new session button

* don't redirect after import

* store and lint to root list

* models + tests

* redesign the progress

* simplify the import session for ow

* drop status from session schema

* split the import session page

* i18n

* fix test

* remove pagination

* fix some colors in darkmode

* one last fix

* add privacy filter

* privacy check

* fix interactivity of import progress

* fix test

4 files changed, 268 insertions, 1 deletions

diff --git a/packages/trpc/routers/_app.ts b/packages/trpc/routers/_app.ts
index 651c8d88..1d548ee4 100644
--- a/packages/trpc/routers/_app.ts
+++ b/packages/trpc/routers/_app.ts
@@ -5,6 +5,7 @@ import { assetsAppRouter } from "./assets";
 import { bookmarksAppRouter } from "./bookmarks";
 import { feedsAppRouter } from "./feeds";
 import { highlightsAppRouter } from "./highlights";
+import { importSessionsRouter } from "./importSessions";
 import { invitesAppRouter } from "./invites";
 import { listsAppRouter } from "./lists";
 import { promptsAppRouter } from "./prompts";
@@ -25,6 +26,7 @@ export const appRouter = router({
   admin: adminAppRouter,
   feeds: feedsAppRouter,
   highlights: highlightsAppRouter,
+  importSessions: importSessionsRouter,
   webhooks: webhooksAppRouter,
   assets: assetsAppRouter,
   rules: rulesAppRouter,
diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts
index 3399bf19..be3664b3 100644
--- a/packages/trpc/routers/bookmarks.ts
+++ b/packages/trpc/routers/bookmarks.ts
@@ -59,6 +59,7 @@ import { authedProcedure, createRateLimitMiddleware, router } from "../index";
 import { mapDBAssetTypeToUserType } from "../lib/attachments";
 import { getBookmarkIdsFromMatcher } from "../lib/search";
 import { Bookmark } from "../models/bookmarks";
+import { ImportSession } from "../models/importSessions";
 import { ensureAssetOwnership } from "./assets";
 
 export const ensureBookmarkOwnership = experimental_trpcMiddleware<{
@@ -272,6 +273,13 @@ export const bookmarksAppRouter = router({
         // This doesn't 100% protect from duplicates because of races, but it's more than enough for this usecase.
         const alreadyExists = await attemptToDedupLink(ctx, input.url);
         if (alreadyExists) {
+          if (input.importSessionId) {
+            const session = await ImportSession.fromId(
+              ctx,
+              input.importSessionId,
+            );
+            await session.attachBookmark(alreadyExists.id);
+          }
           return { ...alreadyExists, alreadyExists: true };
         }
       }
@@ -416,12 +424,16 @@ export const bookmarksAppRouter = router({
         };
       });
 
+      if (input.importSessionId) {
+        const session = await ImportSession.fromId(ctx, input.importSessionId);
+        await session.attachBookmark(bookmark.id);
+      }
+
       const enqueueOpts: EnqueueOptions = {
         // The lower the priority number, the sooner the job will be processed
         priority: input.crawlPriority === "low" ? 50 : 0,
       };
 
-      // Enqueue crawling request
       switch (bookmark.content.type) {
         case BookmarkTypes.LINK: {
           // The crawling job triggers openai when it's done
@@ -454,6 +466,7 @@ export const bookmarksAppRouter = router({
           break;
         }
       }
+
       await triggerRuleEngineOnEvent(
         bookmark.id,
         [
diff --git a/packages/trpc/routers/importSessions.test.ts b/packages/trpc/routers/importSessions.test.ts
new file mode 100644
index 00000000..b28d1421
--- /dev/null
+++ b/packages/trpc/routers/importSessions.test.ts
@@ -0,0 +1,204 @@
+import { beforeEach, describe, expect, test } from "vitest";
+import { z } from "zod";
+
+import {
+  BookmarkTypes,
+  zNewBookmarkRequestSchema,
+} from "@karakeep/shared/types/bookmarks";
+import {
+  zCreateImportSessionRequestSchema,
+  zDeleteImportSessionRequestSchema,
+  zGetImportSessionStatsRequestSchema,
+} from "@karakeep/shared/types/importSessions";
+import { zNewBookmarkListSchema } from "@karakeep/shared/types/lists";
+
+import type { APICallerType, CustomTestContext } from "../testUtils";
+import { defaultBeforeEach } from "../testUtils";
+
+beforeEach<CustomTestContext>(defaultBeforeEach(true));
+
+describe("ImportSessions Routes", () => {
+  async function createTestBookmark(api: APICallerType, sessionId: string) {
+    const newBookmarkInput: z.infer<typeof zNewBookmarkRequestSchema> = {
+      type: BookmarkTypes.TEXT,
+      text: "Test bookmark text",
+      importSessionId: sessionId,
+    };
+    const createdBookmark =
+      await api.bookmarks.createBookmark(newBookmarkInput);
+    return createdBookmark.id;
+  }
+
+  async function createTestList(api: APICallerType) {
+    const newListInput: z.infer<typeof zNewBookmarkListSchema> = {
+      name: "Test Import List",
+      description: "A test list for imports",
+      icon: "📋",
+      type: "manual",
+    };
+    const createdList = await api.lists.create(newListInput);
+    return createdList.id;
+  }
+
+  test<CustomTestContext>("create import session", async ({ apiCallers }) => {
+    const api = apiCallers[0].importSessions;
+    const listId = await createTestList(apiCallers[0]);
+
+    const newSessionInput: z.infer<typeof zCreateImportSessionRequestSchema> = {
+      name: "Test Import Session",
+      rootListId: listId,
+    };
+
+    const createdSession = await api.createImportSession(newSessionInput);
+
+    expect(createdSession).toMatchObject({
+      id: expect.any(String),
+    });
+
+    // Verify session appears in list
+    const sessions = await api.listImportSessions({});
+    const sessionFromList = sessions.sessions.find(
+      (s) => s.id === createdSession.id,
+    );
+    expect(sessionFromList).toBeDefined();
+    expect(sessionFromList?.name).toEqual(newSessionInput.name);
+    expect(sessionFromList?.rootListId).toEqual(listId);
+  });
+
+  test<CustomTestContext>("create import session without rootListId", async ({
+    apiCallers,
+  }) => {
+    const api = apiCallers[0].importSessions;
+
+    const newSessionInput: z.infer<typeof zCreateImportSessionRequestSchema> = {
+      name: "Test Import Session",
+    };
+
+    const createdSession = await api.createImportSession(newSessionInput);
+
+    expect(createdSession).toMatchObject({
+      id: expect.any(String),
+    });
+
+    // Verify session appears in list
+    const sessions = await api.listImportSessions({});
+    const sessionFromList = sessions.sessions.find(
+      (s) => s.id === createdSession.id,
+    );
+    expect(sessionFromList?.rootListId).toBeNull();
+  });
+
+  test<CustomTestContext>("get import session stats", async ({
+    apiCallers,
+  }) => {
+    const api = apiCallers[0];
+
+    const session = await api.importSessions.createImportSession({
+      name: "Test Import Session",
+    });
+    await createTestBookmark(api, session.id);
+    await createTestBookmark(api, session.id);
+
+    const statsInput: z.infer<typeof zGetImportSessionStatsRequestSchema> = {
+      importSessionId: session.id,
+    };
+
+    const stats = await api.importSessions.getImportSessionStats(statsInput);
+
+    expect(stats).toMatchObject({
+      id: session.id,
+      name: "Test Import Session",
+      status: "in_progress",
+      totalBookmarks: 2,
+      pendingBookmarks: 2,
+      completedBookmarks: 0,
+      failedBookmarks: 0,
+      processingBookmarks: 0,
+    });
+  });
+
+  test<CustomTestContext>("list import sessions returns all sessions", async ({
+    apiCallers,
+  }) => {
+    const api = apiCallers[0].importSessions;
+
+    const sessionNames = ["Session 1", "Session 2", "Session 3"];
+    for (const name of sessionNames) {
+      await api.createImportSession({ name });
+    }
+
+    const result = await api.listImportSessions({});
+
+    expect(result.sessions).toHaveLength(3);
+    expect(result.sessions.map((session) => session.name)).toEqual(
+      sessionNames,
+    );
+    expect(
+      result.sessions.every((session) => session.totalBookmarks === 0),
+    ).toBe(true);
+  });
+
+  test<CustomTestContext>("delete import session", async ({ apiCallers }) => {
+    const api = apiCallers[0].importSessions;
+
+    const session = await api.createImportSession({
+      name: "Session to Delete",
+    });
+
+    const deleteInput: z.infer<typeof zDeleteImportSessionRequestSchema> = {
+      importSessionId: session.id,
+    };
+
+    const result = await api.deleteImportSession(deleteInput);
+    expect(result.success).toBe(true);
+
+    // Verify session no longer exists
+    await expect(
+      api.getImportSessionStats({
+        importSessionId: session.id,
+      }),
+    ).rejects.toThrow("Import session not found");
+  });
+
+  test<CustomTestContext>("cannot access other user's session", async ({
+    apiCallers,
+  }) => {
+    const api1 = apiCallers[0].importSessions;
+    const api2 = apiCallers[1].importSessions;
+
+    // User 1 creates a session
+    const session = await api1.createImportSession({
+      name: "User 1 Session",
+    });
+
+    // User 2 tries to access it
+    await expect(
+      api2.getImportSessionStats({
+        importSessionId: session.id,
+      }),
+    ).rejects.toThrow("Import session not found");
+
+    await expect(
+      api2.deleteImportSession({
+        importSessionId: session.id,
+      }),
+    ).rejects.toThrow("Import session not found");
+  });
+
+  test<CustomTestContext>("cannot attach other user's bookmark", async ({
+    apiCallers,
+  }) => {
+    const api1 = apiCallers[0];
+    const api2 = apiCallers[1];
+
+    // User 1 creates session and bookmark
+    const session = await api1.importSessions.createImportSession({
+      name: "User 1 Session",
+    });
+
+    // User 1 tries to attach User 2's bookmark
+    await expect(
+      createTestBookmark(api2, session.id), // User 2's bookmark
+    ).rejects.toThrow("Import session not found");
+  });
+});
diff --git a/packages/trpc/routers/importSessions.ts b/packages/trpc/routers/importSessions.ts
new file mode 100644
index 00000000..4bdc4f29
--- /dev/null
+++ b/packages/trpc/routers/importSessions.ts
@@ -0,0 +1,48 @@
+import { z } from "zod";
+
+import {
+  zCreateImportSessionRequestSchema,
+  zDeleteImportSessionRequestSchema,
+  zGetImportSessionStatsRequestSchema,
+  zImportSessionWithStatsSchema,
+  zListImportSessionsRequestSchema,
+  zListImportSessionsResponseSchema,
+} from "@karakeep/shared/types/importSessions";
+
+import { authedProcedure, router } from "../index";
+import { ImportSession } from "../models/importSessions";
+
+export const importSessionsRouter = router({
+  createImportSession: authedProcedure
+    .input(zCreateImportSessionRequestSchema)
+    .output(z.object({ id: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      const session = await ImportSession.create(ctx, input);
+      return { id: session.session.id };
+    }),
+
+  getImportSessionStats: authedProcedure
+    .input(zGetImportSessionStatsRequestSchema)
+    .output(zImportSessionWithStatsSchema)
+    .query(async ({ input, ctx }) => {
+      const session = await ImportSession.fromId(ctx, input.importSessionId);
+      return await session.getWithStats();
+    }),
+
+  listImportSessions: authedProcedure
+    .input(zListImportSessionsRequestSchema)
+    .output(zListImportSessionsResponseSchema)
+    .query(async ({ ctx }) => {
+      const sessions = await ImportSession.getAllWithStats(ctx);
+      return { sessions };
+    }),
+
+  deleteImportSession: authedProcedure
+    .input(zDeleteImportSessionRequestSchema)
+    .output(z.object({ success: z.boolean() }))
+    .mutation(async ({ input, ctx }) => {
+      const session = await ImportSession.fromId(ctx, input.importSessionId);
+      await session.delete();
+      return { success: true };
+    }),
+});