7 files changed, 453 insertions, 22 deletions
diff --git a/packages/e2e_tests/fixtures/test.pdf b/packages/e2e_tests/fixtures/test.pdf
new file mode 100644
index 00000000..008f870f
--- /dev/null
+++ b/packages/e2e_tests/fixtures/test.pdf
@@ -0,0 +1,38 @@
+%PDF-1.4
+1 0 obj
+<<
+/Type /Catalog
+/Pages 2 0 R
+>>
+endobj
+
+2 0 obj
+<<
+/Type /Pages
+/Kids [3 0 R]
+/Count 1
+>>
+endobj
+
+3 0 obj
+<<
+/Type /Page
+/Parent 2 0 R
+/MediaBox [0 0 612 792]
+>>
+endobj
+
+xref
+0 4
+0000000000 65535 f 
+0000000010 00000 n 
+0000000053 00000 n 
+0000000125 00000 n 
+trailer
+<<
+/Size 4
+/Root 1 0 R
+>>
+startxref
+173
+%%EOF
diff --git a/packages/e2e_tests/package.json b/packages/e2e_tests/package.json
index 45d512bb..d93318aa 100644
--- a/packages/e2e_tests/package.json
+++ b/packages/e2e_tests/package.json
@@ -19,7 +19,7 @@
     "@karakeep/sdk": "workspace:*",
     "@karakeep/shared": "workspace:^0.1.0",
     "@karakeep/trpc": "workspace:^0.1.0",
-    "@trpc/client": "^11.4.3",
+    "@trpc/client": "^11.9.0",
     "superjson": "^2.2.1",
     "zod": "^3.24.2"
   },
diff --git a/packages/e2e_tests/tests/api/assets.test.ts b/packages/e2e_tests/tests/api/assets.test.ts
index accc5cf0..62b9a1e2 100644
--- a/packages/e2e_tests/tests/api/assets.test.ts
+++ b/packages/e2e_tests/tests/api/assets.test.ts
@@ -3,6 +3,7 @@ import { assert, beforeEach, describe, expect, inject, it } from "vitest";
 import { createKarakeepClient } from "@karakeep/sdk";
 
 import { createTestUser, uploadTestAsset } from "../../utils/api";
+import { createTestPdfFile } from "../../utils/assets";
 
 describe("Assets API", () => {
   const port = inject("karakeepPort");
@@ -27,9 +28,7 @@ describe("Assets API", () => {
 
   it("should upload and retrieve an asset", async () => {
     // Create a test file
-    const file = new File(["test content"], "test.pdf", {
-      type: "application/pdf",
-    });
+    const file = createTestPdfFile();
 
     // Upload the asset
     const uploadResponse = await uploadTestAsset(apiKey, port, file);
@@ -52,9 +51,7 @@ describe("Assets API", () => {
 
   it("should attach an asset to a bookmark", async () => {
     // Create a test file
-    const file = new File(["test content"], "test.pdf", {
-      type: "application/pdf",
-    });
+    const file = createTestPdfFile();
 
     // Upload the asset
     const uploadResponse = await uploadTestAsset(apiKey, port, file);
@@ -91,9 +88,7 @@ describe("Assets API", () => {
 
   it("should delete asset when deleting bookmark", async () => {
     // Create a test file
-    const file = new File(["test content"], "test.pdf", {
-      type: "application/pdf",
-    });
+    const file = createTestPdfFile();
 
     // Upload the asset
     const uploadResponse = await uploadTestAsset(apiKey, port, file);
@@ -154,9 +149,7 @@ describe("Assets API", () => {
       throw new Error("Bookmark creation failed");
     }
 
-    const file = new File(["test content"], "test.pdf", {
-      type: "application/pdf",
-    });
+    const file = createTestPdfFile();
 
     // Upload the asset
     const uploadResponse1 = await uploadTestAsset(apiKey, port, file);
diff --git a/packages/e2e_tests/tests/api/bookmarks.test.ts b/packages/e2e_tests/tests/api/bookmarks.test.ts
index a6bc85e3..7c14ee61 100644
--- a/packages/e2e_tests/tests/api/bookmarks.test.ts
+++ b/packages/e2e_tests/tests/api/bookmarks.test.ts
@@ -289,6 +289,76 @@ describe("Bookmarks API", () => {
     expect(removeTagsRes.status).toBe(200);
   });
 
+  it("should manage tags with attachedBy field", async () => {
+    // Create a new bookmark
+    const { data: createdBookmark, error: createError } = await client.POST(
+      "/bookmarks",
+      {
+        body: {
+          type: "text",
+          title: "Test Bookmark for attachedBy",
+          text: "Testing attachedBy field",
+        },
+      },
+    );
+
+    if (createError) {
+      console.error("Error creating bookmark:", createError);
+      throw createError;
+    }
+    if (!createdBookmark) {
+      throw new Error("Bookmark creation failed");
+    }
+
+    // Add tags with different attachedBy values
+    const { data: addTagsResponse, response: addTagsRes } = await client.POST(
+      "/bookmarks/{bookmarkId}/tags",
+      {
+        params: {
+          path: {
+            bookmarkId: createdBookmark.id,
+          },
+        },
+        body: {
+          tags: [
+            { tagName: "ai-tag", attachedBy: "ai" },
+            { tagName: "human-tag", attachedBy: "human" },
+            { tagName: "default-tag" }, // Should default to "human"
+          ],
+        },
+      },
+    );
+
+    expect(addTagsRes.status).toBe(200);
+    expect(addTagsResponse!.attached.length).toBe(3);
+
+    // Get the bookmark and verify the attachedBy values
+    const { data: retrievedBookmark } = await client.GET(
+      "/bookmarks/{bookmarkId}",
+      {
+        params: {
+          path: {
+            bookmarkId: createdBookmark.id,
+          },
+        },
+      },
+    );
+
+    expect(retrievedBookmark!.tags.length).toBe(3);
+
+    const aiTag = retrievedBookmark!.tags.find((t) => t.name === "ai-tag");
+    const humanTag = retrievedBookmark!.tags.find(
+      (t) => t.name === "human-tag",
+    );
+    const defaultTag = retrievedBookmark!.tags.find(
+      (t) => t.name === "default-tag",
+    );
+
+    expect(aiTag?.attachedBy).toBe("ai");
+    expect(humanTag?.attachedBy).toBe("human");
+    expect(defaultTag?.attachedBy).toBe("human");
+  });
+
   it("should get lists for a bookmark", async () => {
     const { data: createdBookmark } = await client.POST("/bookmarks", {
       body: {
diff --git a/packages/e2e_tests/tests/api/public.test.ts b/packages/e2e_tests/tests/api/public.test.ts
index 59f50495..5025d078 100644
--- a/packages/e2e_tests/tests/api/public.test.ts
+++ b/packages/e2e_tests/tests/api/public.test.ts
@@ -6,6 +6,7 @@ import { zAssetSignedTokenSchema } from "@karakeep/shared/types/assets";
 import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
 
 import { createTestUser, uploadTestAsset } from "../../utils/api";
+import { createTestPdfFile } from "../../utils/assets";
 import { waitUntil } from "../../utils/general";
 import { getTrpcClient } from "../../utils/trpc";
 
@@ -43,9 +44,7 @@ describe("Public API", () => {
     });
 
     // Create a second bookmark with an asset
-    const file = new File(["test content"], "test.pdf", {
-      type: "application/pdf",
-    });
+    const file = createTestPdfFile();
 
     const uploadResponse = await uploadTestAsset(currentApiKey, port, file);
     const createBookmark2 = await trpcClient.bookmarks.createBookmark.mutate({
@@ -160,9 +159,7 @@ describe("Public API", () => {
       const assetUpload = await uploadTestAsset(
         apiKey,
         port,
-        new File(["test content for token validation"], "token_test.pdf", {
-          type: "application/pdf",
-        }),
+        createTestPdfFile("token_test.pdf"),
       );
       assetId = assetUpload.assetId;
     });
@@ -246,9 +243,7 @@ describe("Public API", () => {
       const anotherAssetUpload = await uploadTestAsset(
         apiKey, // Same user
         port,
-        new File(["other content"], "other_asset.pdf", {
-          type: "application/pdf",
-        }),
+        createTestPdfFile("other_asset.pdf"),
       );
       const anotherAssetId = anotherAssetUpload.assetId;
 
diff --git a/packages/e2e_tests/tests/workers/import.test.ts b/packages/e2e_tests/tests/workers/import.test.ts
new file mode 100644
index 00000000..e40200f2
--- /dev/null
+++ b/packages/e2e_tests/tests/workers/import.test.ts
@@ -0,0 +1,324 @@
+import { assert, beforeEach, describe, expect, inject, it } from "vitest";
+
+import { createKarakeepClient } from "@karakeep/sdk";
+
+import { createTestUser } from "../../utils/api";
+import { waitUntil } from "../../utils/general";
+import { getTrpcClient } from "../../utils/trpc";
+
+describe("Import Worker Tests", () => {
+  const port = inject("karakeepPort");
+
+  if (!port) {
+    throw new Error("Missing required environment variables");
+  }
+
+  let client: ReturnType<typeof createKarakeepClient>;
+  let trpc: ReturnType<typeof getTrpcClient>;
+  let apiKey: string;
+
+  beforeEach(async () => {
+    apiKey = await createTestUser();
+    client = createKarakeepClient({
+      baseUrl: `http://localhost:${port}/api/v1/`,
+      headers: {
+        "Content-Type": "application/json",
+        authorization: `Bearer ${apiKey}`,
+      },
+    });
+    trpc = getTrpcClient(apiKey);
+  });
+
+  it("should import 15 bookmarks of different types", async () => {
+    // Create lists first (lists require IDs, not paths)
+    const { data: parentList } = await client.POST("/lists", {
+      body: {
+        name: "Import Test List",
+        icon: "folder",
+      },
+    });
+    assert(parentList);
+
+    const { data: nestedList } = await client.POST("/lists", {
+      body: {
+        name: "Nested",
+        icon: "folder",
+        parentId: parentList.id,
+      },
+    });
+    assert(nestedList);
+
+    // Create a root list that all imported bookmarks will be attached to
+    const { data: rootList } = await client.POST("/lists", {
+      body: {
+        name: "Import Root List",
+        icon: "folder",
+      },
+    });
+    assert(rootList);
+
+    // Create an import session with rootListId
+    const { id: importSessionId } =
+      await trpc.importSessions.createImportSession.mutate({
+        name: "E2E Test Import",
+        rootListId: rootList.id,
+      });
+    assert(importSessionId);
+
+    // Define 15 bookmarks of different types
+    const bookmarksToImport = [
+      // Links (7 total, with varying metadata)
+      {
+        type: "link" as const,
+        url: "http://nginx:80/hello.html",
+      },
+      {
+        type: "link" as const,
+        url: "http://nginx:80/page1.html",
+        title: "Page 1 Title",
+      },
+      {
+        type: "link" as const,
+        url: "http://nginx:80/page2.html",
+        title: "Page 2 with Note",
+        note: "This is a note for page 2",
+      },
+      {
+        type: "link" as const,
+        url: "http://nginx:80/page3.html",
+        title: "Page 3 with Tags",
+        tags: ["tag1", "tag2"],
+      },
+      {
+        type: "link" as const,
+        url: "http://nginx:80/page4.html",
+        title: "Page 4 with List",
+        listIds: [parentList.id],
+      },
+      {
+        type: "link" as const,
+        url: "http://nginx:80/page5.html",
+        title: "Page 5 with Source Date",
+        sourceAddedAt: new Date("2024-01-15T10:30:00Z"),
+      },
+      {
+        type: "link" as const,
+        url: "http://nginx:80/page6.html",
+        title: "Page 6 Full Metadata",
+        note: "Full metadata note",
+        tags: ["imported", "full"],
+        listIds: [nestedList.id],
+        sourceAddedAt: new Date("2024-02-20T15:45:00Z"),
+      },
+
+      // Text bookmarks (5 total)
+      {
+        type: "text" as const,
+        content: "This is a basic text bookmark content.",
+      },
+      {
+        type: "text" as const,
+        title: "Text with Title",
+        content: "Text bookmark with a title.",
+      },
+      {
+        type: "text" as const,
+        title: "Text with Tags",
+        content: "Text bookmark with tags attached.",
+        tags: ["text-tag", "imported"],
+      },
+      {
+        type: "text" as const,
+        title: "Text with Note",
+        content: "Text bookmark content here.",
+        note: "A note attached to this text bookmark.",
+      },
+      {
+        type: "text" as const,
+        title: "Text Full Metadata",
+        content: "Text bookmark with all metadata fields.",
+        note: "Complete text note",
+        tags: ["complete", "text"],
+        listIds: [parentList.id],
+        sourceAddedAt: new Date("2024-03-10T08:00:00Z"),
+      },
+
+      // Duplicates (3 total - same URLs as earlier links)
+      {
+        type: "link" as const,
+        url: "http://nginx:80/hello.html", // Duplicate of link #1
+        title: "Duplicate of Hello",
+      },
+      {
+        type: "link" as const,
+        url: "http://nginx:80/page1.html", // Duplicate of link #2
+        title: "Duplicate of Page 1",
+      },
+      {
+        type: "link" as const,
+        url: "http://nginx:80/page2.html", // Duplicate of link #3
+        title: "Duplicate of Page 2",
+        note: "Different note but same URL",
+      },
+    ];
+
+    // Stage all bookmarks
+    await trpc.importSessions.stageImportedBookmarks.mutate({
+      importSessionId,
+      bookmarks: bookmarksToImport,
+    });
+
+    // Finalize the import to trigger processing
+    await trpc.importSessions.finalizeImportStaging.mutate({
+      importSessionId,
+    });
+
+    // Wait for all bookmarks to be processed
+    await waitUntil(
+      async () => {
+        const stats = await trpc.importSessions.getImportSessionStats.query({
+          importSessionId,
+        });
+        const allProcessed =
+          stats.completedBookmarks + stats.failedBookmarks ===
+          stats.totalBookmarks;
+        console.log(
+          `Import progress: ${stats.completedBookmarks} completed, ${stats.failedBookmarks} failed, ${stats.totalBookmarks} total`,
+        );
+        return allProcessed && stats.totalBookmarks === 15;
+      },
+      "All bookmarks are processed",
+      120000, // 2 minutes timeout
+    );
+
+    // Get final stats
+    const finalStats = await trpc.importSessions.getImportSessionStats.query({
+      importSessionId,
+    });
+
+    expect(finalStats.totalBookmarks).toBe(15);
+    expect(finalStats.completedBookmarks).toBe(15);
+    expect(finalStats.failedBookmarks).toBe(0);
+    expect(finalStats.status).toBe("completed");
+
+    // Get results by filter
+    const acceptedResults =
+      await trpc.importSessions.getImportSessionResults.query({
+        importSessionId,
+        filter: "accepted",
+        limit: 50,
+      });
+
+    const duplicateResults =
+      await trpc.importSessions.getImportSessionResults.query({
+        importSessionId,
+        filter: "skipped_duplicate",
+        limit: 50,
+      });
+
+    // We expect 12 accepted (7 links + 5 text) and 3 duplicates
+    expect(acceptedResults.items.length).toBe(12);
+    expect(duplicateResults.items.length).toBe(3);
+
+    // Verify duplicates reference the original bookmarks
+    for (const dup of duplicateResults.items) {
+      expect(dup.resultBookmarkId).toBeDefined();
+      expect(dup.result).toBe("skipped_duplicate");
+    }
+
+    // Verify accepted bookmarks have resultBookmarkId
+    for (const accepted of acceptedResults.items) {
+      expect(accepted.resultBookmarkId).toBeDefined();
+      expect(accepted.result).toBe("accepted");
+    }
+
+    // Verify actual bookmarks were created via the API
+    const { data: allBookmarks } = await client.GET("/bookmarks", {
+      params: {
+        query: {
+          limit: 50,
+        },
+      },
+    });
+    assert(allBookmarks);
+
+    // Should have 12 unique bookmarks (7 links + 5 text)
+    expect(allBookmarks.bookmarks.length).toBe(12);
+
+    // Verify link bookmarks
+    const linkBookmarks = allBookmarks.bookmarks.filter(
+      (b) => b.content.type === "link",
+    );
+    expect(linkBookmarks.length).toBe(7);
+
+    // Verify text bookmarks
+    const textBookmarks = allBookmarks.bookmarks.filter(
+      (b) => b.content.type === "text",
+    );
+    expect(textBookmarks.length).toBe(5);
+
+    // Verify tags were created
+    const { data: tagsResponse } = await client.GET("/tags", {});
+    assert(tagsResponse);
+    const tagNames = tagsResponse.tags.map((t) => t.name);
+    expect(tagNames).toContain("tag1");
+    expect(tagNames).toContain("tag2");
+    expect(tagNames).toContain("imported");
+    expect(tagNames).toContain("text-tag");
+    expect(tagNames).toContain("complete");
+
+    // Verify tags are actually attached to bookmarks
+    // Find a bookmark with tags and verify
+    const bookmarkWithTags = allBookmarks.bookmarks.find((b) =>
+      b.tags.some((t) => t.name === "tag1"),
+    );
+    assert(bookmarkWithTags, "Should find a bookmark with tag1");
+    expect(bookmarkWithTags.tags.map((t) => t.name)).toContain("tag1");
+    expect(bookmarkWithTags.tags.map((t) => t.name)).toContain("tag2");
+
+    // Verify "imported" tag is on multiple bookmarks (used in link and text)
+    const bookmarksWithImportedTag = allBookmarks.bookmarks.filter((b) =>
+      b.tags.some((t) => t.name === "imported"),
+    );
+    expect(bookmarksWithImportedTag.length).toBeGreaterThanOrEqual(2);
+
+    // Verify bookmarks are actually in the lists
+    const { data: parentListBookmarks } = await client.GET(
+      "/lists/{listId}/bookmarks",
+      {
+        params: {
+          path: { listId: parentList.id },
+        },
+      },
+    );
+    assert(parentListBookmarks);
+    // Should have bookmarks with listIds containing parentList.id
+    expect(parentListBookmarks.bookmarks.length).toBeGreaterThanOrEqual(2);
+
+    // Verify nested list has bookmarks
+    const { data: nestedListBookmarks } = await client.GET(
+      "/lists/{listId}/bookmarks",
+      {
+        params: {
+          path: { listId: nestedList.id },
+        },
+      },
+    );
+    assert(nestedListBookmarks);
+    // Should have the bookmark with listIds containing nestedList.id
+    expect(nestedListBookmarks.bookmarks.length).toBeGreaterThanOrEqual(1);
+
+    // Verify ALL imported bookmarks are in the root list (via rootListId)
+    const { data: rootListBookmarks } = await client.GET(
+      "/lists/{listId}/bookmarks",
+      {
+        params: {
+          path: { listId: rootList.id },
+        },
+      },
+    );
+    assert(rootListBookmarks);
+    // All 12 unique bookmarks should be in the root list
+    expect(rootListBookmarks.bookmarks.length).toBe(12);
+  });
+});
diff --git a/packages/e2e_tests/utils/assets.ts b/packages/e2e_tests/utils/assets.ts
new file mode 100644
index 00000000..efcfec81
--- /dev/null
+++ b/packages/e2e_tests/utils/assets.ts
@@ -0,0 +1,11 @@
+import * as fs from "fs";
+import * as path from "path";
+
+const pdfFixturePath = path.join(__dirname, "..", "fixtures", "test.pdf");
+const pdfContent = fs.readFileSync(pdfFixturePath);
+
+export function createTestPdfFile(fileName = "test.pdf"): File {
+  return new File([pdfContent], fileName, {
+    type: "application/pdf",
+  });
+}