feat: Revamp import experience (#2001)

* WIP: import v2

* remove new session button

* don't redirect after import

* store and lint to root list

* models + tests

* redesign the progress

* simplify the import session for ow

* drop status from session schema

* split the import session page

* i18n

* fix test

* remove pagination

* fix some colors in darkmode

* one last fix

* add privacy filter

* privacy check

* fix interactivity of import progress

* fix test

5 files changed, 448 insertions, 1 deletions

diff --git a/packages/trpc/models/importSessions.ts b/packages/trpc/models/importSessions.ts
new file mode 100644
index 00000000..270c2bce
--- /dev/null
+++ b/packages/trpc/models/importSessions.ts
@@ -0,0 +1,180 @@
+import { TRPCError } from "@trpc/server";
+import { and, count, eq } from "drizzle-orm";
+import { z } from "zod";
+
+import {
+  bookmarkLinks,
+  bookmarks,
+  importSessionBookmarks,
+  importSessions,
+} from "@karakeep/db/schema";
+import {
+  zCreateImportSessionRequestSchema,
+  ZImportSession,
+  ZImportSessionWithStats,
+} from "@karakeep/shared/types/importSessions";
+
+import type { AuthedContext } from "../index";
+import { PrivacyAware } from "./privacy";
+
+export class ImportSession implements PrivacyAware {
+  protected constructor(
+    protected ctx: AuthedContext,
+    public session: ZImportSession,
+  ) {}
+
+  static async fromId(
+    ctx: AuthedContext,
+    importSessionId: string,
+  ): Promise<ImportSession> {
+    const session = await ctx.db.query.importSessions.findFirst({
+      where: and(
+        eq(importSessions.id, importSessionId),
+        eq(importSessions.userId, ctx.user.id),
+      ),
+    });
+
+    if (!session) {
+      throw new TRPCError({
+        code: "NOT_FOUND",
+        message: "Import session not found",
+      });
+    }
+
+    return new ImportSession(ctx, session);
+  }
+
+  static async create(
+    ctx: AuthedContext,
+    input: z.infer<typeof zCreateImportSessionRequestSchema>,
+  ): Promise<ImportSession> {
+    const [session] = await ctx.db
+      .insert(importSessions)
+      .values({
+        name: input.name,
+        userId: ctx.user.id,
+        rootListId: input.rootListId,
+      })
+      .returning();
+
+    return new ImportSession(ctx, session);
+  }
+
+  static async getAll(ctx: AuthedContext): Promise<ImportSession[]> {
+    const sessions = await ctx.db.query.importSessions.findMany({
+      where: eq(importSessions.userId, ctx.user.id),
+      orderBy: (importSessions, { desc }) => [desc(importSessions.createdAt)],
+      limit: 50,
+    });
+
+    return sessions.map((session) => new ImportSession(ctx, session));
+  }
+
+  static async getAllWithStats(
+    ctx: AuthedContext,
+  ): Promise<ZImportSessionWithStats[]> {
+    const sessions = await this.getAll(ctx);
+
+    return await Promise.all(
+      sessions.map(async (session) => {
+        return await session.getWithStats();
+      }),
+    );
+  }
+
+  ensureCanAccess(ctx: AuthedContext): void {
+    if (this.session.userId !== ctx.user.id) {
+      throw new TRPCError({
+        code: "FORBIDDEN",
+        message: "User is not allowed to access this import session",
+      });
+    }
+  }
+
+  async attachBookmark(bookmarkId: string): Promise<void> {
+    await this.ctx.db.insert(importSessionBookmarks).values({
+      importSessionId: this.session.id,
+      bookmarkId,
+    });
+  }
+
+  async getWithStats(): Promise<ZImportSessionWithStats> {
+    // Get bookmark counts by status
+    const statusCounts = await this.ctx.db
+      .select({
+        crawlStatus: bookmarkLinks.crawlStatus,
+        taggingStatus: bookmarks.taggingStatus,
+        count: count(),
+      })
+      .from(importSessionBookmarks)
+      .innerJoin(
+        importSessions,
+        eq(importSessions.id, importSessionBookmarks.importSessionId),
+      )
+      .leftJoin(bookmarks, eq(bookmarks.id, importSessionBookmarks.bookmarkId))
+      .leftJoin(
+        bookmarkLinks,
+        eq(bookmarkLinks.id, importSessionBookmarks.bookmarkId),
+      )
+      .where(
+        and(
+          eq(importSessionBookmarks.importSessionId, this.session.id),
+          eq(importSessions.userId, this.ctx.user.id),
+        ),
+      )
+      .groupBy(bookmarkLinks.crawlStatus, bookmarks.taggingStatus);
+
+    const stats = {
+      totalBookmarks: 0,
+      completedBookmarks: 0,
+      failedBookmarks: 0,
+      pendingBookmarks: 0,
+      processingBookmarks: 0,
+    };
+
+    statusCounts.forEach((statusCount) => {
+      stats.totalBookmarks += statusCount.count;
+      if (
+        statusCount.crawlStatus === "success" &&
+        statusCount.taggingStatus === "success"
+      ) {
+        stats.completedBookmarks += statusCount.count;
+      } else if (
+        statusCount.crawlStatus === "failure" ||
+        statusCount.taggingStatus === "failure"
+      ) {
+        stats.failedBookmarks += statusCount.count;
+      } else if (
+        statusCount.crawlStatus === "pending" ||
+        statusCount.taggingStatus === "pending"
+      ) {
+        stats.pendingBookmarks += statusCount.count;
+      }
+    });
+
+    return {
+      ...this.session,
+      status: stats.pendingBookmarks > 0 ? "in_progress" : "completed",
+      ...stats,
+    };
+  }
+
+  async delete(): Promise<void> {
+    // Delete the session (cascade will handle the bookmarks)
+    const result = await this.ctx.db
+      .delete(importSessions)
+      .where(
+        and(
+          eq(importSessions.id, this.session.id),
+          eq(importSessions.userId, this.ctx.user.id),
+        ),
+      );
+
+    if (result.changes === 0) {
+      throw new TRPCError({
+        code: "NOT_FOUND",
+        message: "Import session not found",
+      });
+    }
+  }
+}
diff --git a/packages/trpc/routers/_app.ts b/packages/trpc/routers/_app.ts
index 651c8d88..1d548ee4 100644
--- a/packages/trpc/routers/_app.ts
+++ b/packages/trpc/routers/_app.ts
@@ -5,6 +5,7 @@ import { assetsAppRouter } from "./assets";
 import { bookmarksAppRouter } from "./bookmarks";
 import { feedsAppRouter } from "./feeds";
 import { highlightsAppRouter } from "./highlights";
+import { importSessionsRouter } from "./importSessions";
 import { invitesAppRouter } from "./invites";
 import { listsAppRouter } from "./lists";
 import { promptsAppRouter } from "./prompts";
@@ -25,6 +26,7 @@ export const appRouter = router({
   admin: adminAppRouter,
   feeds: feedsAppRouter,
   highlights: highlightsAppRouter,
+  importSessions: importSessionsRouter,
   webhooks: webhooksAppRouter,
   assets: assetsAppRouter,
   rules: rulesAppRouter,
diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts
index 3399bf19..be3664b3 100644
--- a/packages/trpc/routers/bookmarks.ts
+++ b/packages/trpc/routers/bookmarks.ts
@@ -59,6 +59,7 @@ import { authedProcedure, createRateLimitMiddleware, router } from "../index";
 import { mapDBAssetTypeToUserType } from "../lib/attachments";
 import { getBookmarkIdsFromMatcher } from "../lib/search";
 import { Bookmark } from "../models/bookmarks";
+import { ImportSession } from "../models/importSessions";
 import { ensureAssetOwnership } from "./assets";
 
 export const ensureBookmarkOwnership = experimental_trpcMiddleware<{
@@ -272,6 +273,13 @@ export const bookmarksAppRouter = router({
         // This doesn't 100% protect from duplicates because of races, but it's more than enough for this usecase.
         const alreadyExists = await attemptToDedupLink(ctx, input.url);
         if (alreadyExists) {
+          if (input.importSessionId) {
+            const session = await ImportSession.fromId(
+              ctx,
+              input.importSessionId,
+            );
+            await session.attachBookmark(alreadyExists.id);
+          }
           return { ...alreadyExists, alreadyExists: true };
         }
       }
@@ -416,12 +424,16 @@ export const bookmarksAppRouter = router({
         };
       });
 
+      if (input.importSessionId) {
+        const session = await ImportSession.fromId(ctx, input.importSessionId);
+        await session.attachBookmark(bookmark.id);
+      }
+
       const enqueueOpts: EnqueueOptions = {
         // The lower the priority number, the sooner the job will be processed
         priority: input.crawlPriority === "low" ? 50 : 0,
       };
 
-      // Enqueue crawling request
       switch (bookmark.content.type) {
         case BookmarkTypes.LINK: {
           // The crawling job triggers openai when it's done
@@ -454,6 +466,7 @@ export const bookmarksAppRouter = router({
           break;
         }
       }
+
       await triggerRuleEngineOnEvent(
         bookmark.id,
         [
diff --git a/packages/trpc/routers/importSessions.test.ts b/packages/trpc/routers/importSessions.test.ts
new file mode 100644
index 00000000..b28d1421
--- /dev/null
+++ b/packages/trpc/routers/importSessions.test.ts
@@ -0,0 +1,204 @@
+import { beforeEach, describe, expect, test } from "vitest";
+import { z } from "zod";
+
+import {
+  BookmarkTypes,
+  zNewBookmarkRequestSchema,
+} from "@karakeep/shared/types/bookmarks";
+import {
+  zCreateImportSessionRequestSchema,
+  zDeleteImportSessionRequestSchema,
+  zGetImportSessionStatsRequestSchema,
+} from "@karakeep/shared/types/importSessions";
+import { zNewBookmarkListSchema } from "@karakeep/shared/types/lists";
+
+import type { APICallerType, CustomTestContext } from "../testUtils";
+import { defaultBeforeEach } from "../testUtils";
+
+beforeEach<CustomTestContext>(defaultBeforeEach(true));
+
+describe("ImportSessions Routes", () => {
+  async function createTestBookmark(api: APICallerType, sessionId: string) {
+    const newBookmarkInput: z.infer<typeof zNewBookmarkRequestSchema> = {
+      type: BookmarkTypes.TEXT,
+      text: "Test bookmark text",
+      importSessionId: sessionId,
+    };
+    const createdBookmark =
+      await api.bookmarks.createBookmark(newBookmarkInput);
+    return createdBookmark.id;
+  }
+
+  async function createTestList(api: APICallerType) {
+    const newListInput: z.infer<typeof zNewBookmarkListSchema> = {
+      name: "Test Import List",
+      description: "A test list for imports",
+      icon: "📋",
+      type: "manual",
+    };
+    const createdList = await api.lists.create(newListInput);
+    return createdList.id;
+  }
+
+  test<CustomTestContext>("create import session", async ({ apiCallers }) => {
+    const api = apiCallers[0].importSessions;
+    const listId = await createTestList(apiCallers[0]);
+
+    const newSessionInput: z.infer<typeof zCreateImportSessionRequestSchema> = {
+      name: "Test Import Session",
+      rootListId: listId,
+    };
+
+    const createdSession = await api.createImportSession(newSessionInput);
+
+    expect(createdSession).toMatchObject({
+      id: expect.any(String),
+    });
+
+    // Verify session appears in list
+    const sessions = await api.listImportSessions({});
+    const sessionFromList = sessions.sessions.find(
+      (s) => s.id === createdSession.id,
+    );
+    expect(sessionFromList).toBeDefined();
+    expect(sessionFromList?.name).toEqual(newSessionInput.name);
+    expect(sessionFromList?.rootListId).toEqual(listId);
+  });
+
+  test<CustomTestContext>("create import session without rootListId", async ({
+    apiCallers,
+  }) => {
+    const api = apiCallers[0].importSessions;
+
+    const newSessionInput: z.infer<typeof zCreateImportSessionRequestSchema> = {
+      name: "Test Import Session",
+    };
+
+    const createdSession = await api.createImportSession(newSessionInput);
+
+    expect(createdSession).toMatchObject({
+      id: expect.any(String),
+    });
+
+    // Verify session appears in list
+    const sessions = await api.listImportSessions({});
+    const sessionFromList = sessions.sessions.find(
+      (s) => s.id === createdSession.id,
+    );
+    expect(sessionFromList?.rootListId).toBeNull();
+  });
+
+  test<CustomTestContext>("get import session stats", async ({
+    apiCallers,
+  }) => {
+    const api = apiCallers[0];
+
+    const session = await api.importSessions.createImportSession({
+      name: "Test Import Session",
+    });
+    await createTestBookmark(api, session.id);
+    await createTestBookmark(api, session.id);
+
+    const statsInput: z.infer<typeof zGetImportSessionStatsRequestSchema> = {
+      importSessionId: session.id,
+    };
+
+    const stats = await api.importSessions.getImportSessionStats(statsInput);
+
+    expect(stats).toMatchObject({
+      id: session.id,
+      name: "Test Import Session",
+      status: "in_progress",
+      totalBookmarks: 2,
+      pendingBookmarks: 2,
+      completedBookmarks: 0,
+      failedBookmarks: 0,
+      processingBookmarks: 0,
+    });
+  });
+
+  test<CustomTestContext>("list import sessions returns all sessions", async ({
+    apiCallers,
+  }) => {
+    const api = apiCallers[0].importSessions;
+
+    const sessionNames = ["Session 1", "Session 2", "Session 3"];
+    for (const name of sessionNames) {
+      await api.createImportSession({ name });
+    }
+
+    const result = await api.listImportSessions({});
+
+    expect(result.sessions).toHaveLength(3);
+    expect(result.sessions.map((session) => session.name)).toEqual(
+      sessionNames,
+    );
+    expect(
+      result.sessions.every((session) => session.totalBookmarks === 0),
+    ).toBe(true);
+  });
+
+  test<CustomTestContext>("delete import session", async ({ apiCallers }) => {
+    const api = apiCallers[0].importSessions;
+
+    const session = await api.createImportSession({
+      name: "Session to Delete",
+    });
+
+    const deleteInput: z.infer<typeof zDeleteImportSessionRequestSchema> = {
+      importSessionId: session.id,
+    };
+
+    const result = await api.deleteImportSession(deleteInput);
+    expect(result.success).toBe(true);
+
+    // Verify session no longer exists
+    await expect(
+      api.getImportSessionStats({
+        importSessionId: session.id,
+      }),
+    ).rejects.toThrow("Import session not found");
+  });
+
+  test<CustomTestContext>("cannot access other user's session", async ({
+    apiCallers,
+  }) => {
+    const api1 = apiCallers[0].importSessions;
+    const api2 = apiCallers[1].importSessions;
+
+    // User 1 creates a session
+    const session = await api1.createImportSession({
+      name: "User 1 Session",
+    });
+
+    // User 2 tries to access it
+    await expect(
+      api2.getImportSessionStats({
+        importSessionId: session.id,
+      }),
+    ).rejects.toThrow("Import session not found");
+
+    await expect(
+      api2.deleteImportSession({
+        importSessionId: session.id,
+      }),
+    ).rejects.toThrow("Import session not found");
+  });
+
+  test<CustomTestContext>("cannot attach other user's bookmark", async ({
+    apiCallers,
+  }) => {
+    const api1 = apiCallers[0];
+    const api2 = apiCallers[1];
+
+    // User 1 creates session and bookmark
+    const session = await api1.importSessions.createImportSession({
+      name: "User 1 Session",
+    });
+
+    // User 1 tries to attach User 2's bookmark
+    await expect(
+      createTestBookmark(api2, session.id), // User 2's bookmark
+    ).rejects.toThrow("Import session not found");
+  });
+});
diff --git a/packages/trpc/routers/importSessions.ts b/packages/trpc/routers/importSessions.ts
new file mode 100644
index 00000000..4bdc4f29
--- /dev/null
+++ b/packages/trpc/routers/importSessions.ts
@@ -0,0 +1,48 @@
+import { z } from "zod";
+
+import {
+  zCreateImportSessionRequestSchema,
+  zDeleteImportSessionRequestSchema,
+  zGetImportSessionStatsRequestSchema,
+  zImportSessionWithStatsSchema,
+  zListImportSessionsRequestSchema,
+  zListImportSessionsResponseSchema,
+} from "@karakeep/shared/types/importSessions";
+
+import { authedProcedure, router } from "../index";
+import { ImportSession } from "../models/importSessions";
+
+export const importSessionsRouter = router({
+  createImportSession: authedProcedure
+    .input(zCreateImportSessionRequestSchema)
+    .output(z.object({ id: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      const session = await ImportSession.create(ctx, input);
+      return { id: session.session.id };
+    }),
+
+  getImportSessionStats: authedProcedure
+    .input(zGetImportSessionStatsRequestSchema)
+    .output(zImportSessionWithStatsSchema)
+    .query(async ({ input, ctx }) => {
+      const session = await ImportSession.fromId(ctx, input.importSessionId);
+      return await session.getWithStats();
+    }),
+
+  listImportSessions: authedProcedure
+    .input(zListImportSessionsRequestSchema)
+    .output(zListImportSessionsResponseSchema)
+    .query(async ({ ctx }) => {
+      const sessions = await ImportSession.getAllWithStats(ctx);
+      return { sessions };
+    }),
+
+  deleteImportSession: authedProcedure
+    .input(zDeleteImportSessionRequestSchema)
+    .output(z.object({ success: z.boolean() }))
+    .mutation(async ({ input, ctx }) => {
+      const session = await ImportSession.fromId(ctx, input.importSessionId);
+      await session.delete();
+      return { success: true };
+    }),
+});