aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--apps/web/app/api/bookmarks/export/route.tsx8
-rw-r--r--apps/web/lib/hooks/useBookmarkImport.ts272
-rw-r--r--packages/open-api/karakeep-openapi-spec.json8
-rw-r--r--packages/shared/import-export/exporters.ts (renamed from apps/web/lib/exportBookmarks.ts)2
-rw-r--r--packages/shared/import-export/importer.test.ts392
-rw-r--r--packages/shared/import-export/importer.ts158
-rw-r--r--packages/shared/import-export/index.ts3
-rw-r--r--packages/shared/import-export/parsers.ts (renamed from apps/web/lib/importBookmarkParser.ts)80
-rw-r--r--packages/shared/types/bookmarks.ts6
-rw-r--r--packages/shared/types/lists.ts23
10 files changed, 713 insertions, 239 deletions
diff --git a/apps/web/app/api/bookmarks/export/route.tsx b/apps/web/app/api/bookmarks/export/route.tsx
index 47fdeebc..ad309877 100644
--- a/apps/web/app/api/bookmarks/export/route.tsx
+++ b/apps/web/app/api/bookmarks/export/route.tsx
@@ -1,12 +1,12 @@
import { NextRequest } from "next/server";
+import { api, createContextFromRequest } from "@/server/api/client";
+import { z } from "zod";
+
import {
toExportFormat,
toNetscapeFormat,
zExportSchema,
-} from "@/lib/exportBookmarks";
-import { api, createContextFromRequest } from "@/server/api/client";
-import { z } from "zod";
-
+} from "@karakeep/shared/import-export";
import { MAX_NUM_BOOKMARKS_PER_PAGE } from "@karakeep/shared/types/bookmarks";
export const dynamic = "force-dynamic";
diff --git a/apps/web/lib/hooks/useBookmarkImport.ts b/apps/web/lib/hooks/useBookmarkImport.ts
index 7e5f6111..de515677 100644
--- a/apps/web/lib/hooks/useBookmarkImport.ts
+++ b/apps/web/lib/hooks/useBookmarkImport.ts
@@ -4,16 +4,6 @@ import { useState } from "react";
import { useRouter } from "next/navigation";
import { toast } from "@/components/ui/use-toast";
import { useTranslation } from "@/lib/i18n/client";
-import {
- deduplicateBookmarks,
- ParsedBookmark,
- parseKarakeepBookmarkFile,
- parseLinkwardenBookmarkFile,
- parseNetscapeBookmarkFile,
- parseOmnivoreBookmarkFile,
- parsePocketBookmarkFile,
- parseTabSessionManagerStateFile,
-} from "@/lib/importBookmarkParser";
import { useMutation } from "@tanstack/react-query";
import {
@@ -24,16 +14,15 @@ import {
useAddBookmarkToList,
useCreateBookmarkList,
} from "@karakeep/shared-react/hooks/lists";
-import { limitConcurrency } from "@karakeep/shared/concurrency";
-import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
-
-export type ImportSource =
- | "html"
- | "pocket"
- | "omnivore"
- | "karakeep"
- | "linkwarden"
- | "tab-session-manager";
+import {
+ importBookmarksFromFile,
+ ImportSource,
+ ParsedBookmark,
+} from "@karakeep/shared/import-export";
+import {
+ BookmarkTypes,
+ MAX_BOOKMARK_TITLE_LENGTH,
+} from "@karakeep/shared/types/bookmarks";
export interface ImportProgress {
done: number;
@@ -53,53 +42,6 @@ export function useBookmarkImport() {
const { mutateAsync: addToList } = useAddBookmarkToList();
const { mutateAsync: updateTags } = useUpdateBookmarkTags();
- const { mutateAsync: parseAndCreateBookmark } = useMutation({
- mutationFn: async (toImport: {
- bookmark: ParsedBookmark;
- listIds: string[];
- }) => {
- const bookmark = toImport.bookmark;
- if (bookmark.content === undefined) {
- throw new Error("Content is undefined");
- }
- const created = await createBookmark({
- crawlPriority: "low",
- title: bookmark.title,
- createdAt: bookmark.addDate
- ? new Date(bookmark.addDate * 1000)
- : undefined,
- note: bookmark.notes,
- archived: bookmark.archived,
- ...(bookmark.content.type === BookmarkTypes.LINK
- ? {
- type: BookmarkTypes.LINK,
- url: bookmark.content.url,
- }
- : {
- type: BookmarkTypes.TEXT,
- text: bookmark.content.text,
- }),
- });
-
- await Promise.all([
- ...toImport.listIds.map((listId) =>
- addToList({
- bookmarkId: created.id,
- listId,
- }),
- ),
- bookmark.tags.length > 0
- ? updateTags({
- bookmarkId: created.id,
- attach: bookmark.tags.map((t) => ({ tagName: t })),
- detach: [],
- })
- : undefined,
- ]);
- return created;
- },
- });
-
const uploadBookmarkFileMutation = useMutation({
mutationFn: async ({
file,
@@ -108,138 +50,87 @@ export function useBookmarkImport() {
file: File;
source: ImportSource;
}) => {
- if (source === "html") {
- return await parseNetscapeBookmarkFile(file);
- } else if (source === "pocket") {
- return await parsePocketBookmarkFile(file);
- } else if (source === "karakeep") {
- return await parseKarakeepBookmarkFile(file);
- } else if (source === "omnivore") {
- return await parseOmnivoreBookmarkFile(file);
- } else if (source === "linkwarden") {
- return await parseLinkwardenBookmarkFile(file);
- } else if (source === "tab-session-manager") {
- return await parseTabSessionManagerStateFile(file);
- } else {
- throw new Error("Unknown source");
- }
- },
- onSuccess: async (parsedBookmarks) => {
- if (parsedBookmarks.length === 0) {
- toast({ description: "No bookmarks found in the file." });
- return;
- }
-
- const rootList = await createList({
- name: t("settings.import.imported_bookmarks"),
- icon: "⬆️",
- });
-
- const finalBookmarksToImport = deduplicateBookmarks(parsedBookmarks);
-
- setImportProgress({ done: 0, total: finalBookmarksToImport.length });
-
- const allRequiredPaths = new Set<string>();
- for (const bookmark of finalBookmarksToImport) {
- for (const path of bookmark.paths) {
- if (path && path.length > 0) {
- for (let i = 1; i <= path.length; i++) {
- const subPath = path.slice(0, i);
- const pathKey = subPath.join("/");
- allRequiredPaths.add(pathKey);
- }
- }
- }
- }
-
- const allRequiredPathsArray = Array.from(allRequiredPaths).sort(
- (a, b) => a.split("/").length - b.split("/").length,
- );
-
- const pathMap: Record<string, string> = {};
- pathMap[""] = rootList.id;
-
- for (const pathKey of allRequiredPathsArray) {
- const parts = pathKey.split("/");
- const parentKey = parts.slice(0, -1).join("/");
- const parentId = pathMap[parentKey] || rootList.id;
-
- const folderName = parts[parts.length - 1];
- const folderList = await createList({
- name: folderName,
- parentId: parentId,
- icon: "📁",
- });
- pathMap[pathKey] = folderList.id;
- }
-
- const importPromises = finalBookmarksToImport.map(
- (bookmark) => async () => {
- const listIds = bookmark.paths.map(
- (path) => pathMap[path.join("/")] || rootList.id,
- );
- if (listIds.length === 0) {
- listIds.push(rootList.id);
- }
-
- try {
- const created = await parseAndCreateBookmark({
- bookmark: bookmark,
+ const result = await importBookmarksFromFile(
+ {
+ file,
+ source,
+ rootListName: t("settings.import.imported_bookmarks"),
+ deps: {
+ createList: createList,
+ createBookmark: async (bookmark: ParsedBookmark) => {
+ if (bookmark.content === undefined) {
+ throw new Error("Content is undefined");
+ }
+ const created = await createBookmark({
+ crawlPriority: "low",
+ title: bookmark.title.substring(0, MAX_BOOKMARK_TITLE_LENGTH),
+ createdAt: bookmark.addDate
+ ? new Date(bookmark.addDate * 1000)
+ : undefined,
+ note: bookmark.notes,
+ archived: bookmark.archived,
+ ...(bookmark.content.type === BookmarkTypes.LINK
+ ? {
+ type: BookmarkTypes.LINK,
+ url: bookmark.content.url,
+ }
+ : {
+ type: BookmarkTypes.TEXT,
+ text: bookmark.content.text,
+ }),
+ });
+ return created as { id: string; alreadyExists?: boolean };
+ },
+ addBookmarkToLists: async ({
+ bookmarkId,
listIds,
- });
-
- setImportProgress((prev) => {
- const newDone = (prev?.done ?? 0) + 1;
- return {
- done: newDone,
- total: finalBookmarksToImport.length,
- };
- });
- return { status: "fulfilled" as const, value: created };
- } catch {
- setImportProgress((prev) => {
- const newDone = (prev?.done ?? 0) + 1;
- return {
- done: newDone,
- total: finalBookmarksToImport.length,
- };
- });
- return { status: "rejected" as const };
- }
+ }: {
+ bookmarkId: string;
+ listIds: string[];
+ }) => {
+ await Promise.all(
+ listIds.map((listId) =>
+ addToList({
+ bookmarkId,
+ listId,
+ }),
+ ),
+ );
+ },
+ updateBookmarkTags: async ({
+ bookmarkId,
+ tags,
+ }: {
+ bookmarkId: string;
+ tags: string[];
+ }) => {
+ if (tags.length > 0) {
+ await updateTags({
+ bookmarkId,
+ attach: tags.map((t) => ({ tagName: t })),
+ detach: [],
+ });
+ }
+ },
+ },
+ onProgress: (done, total) => setImportProgress({ done, total }),
},
+ {},
);
-
- const CONCURRENCY_LIMIT = 20;
- const resultsPromises = limitConcurrency(
- importPromises,
- CONCURRENCY_LIMIT,
- );
-
- const results = await Promise.all(resultsPromises);
-
- let successes = 0;
- let failures = 0;
- let alreadyExisted = 0;
-
- for (const result of results) {
- if (result.status === "fulfilled") {
- if (result.value.alreadyExists) {
- alreadyExisted++;
- } else {
- successes++;
- }
- } else {
- failures++;
- }
+ return result;
+ },
+ onSuccess: async (result) => {
+ if (result.counts.total === 0) {
+ toast({ description: "No bookmarks found in the file." });
+ return;
}
-
+ const { successes, failures, alreadyExisted } = result.counts;
if (successes > 0 || alreadyExisted > 0) {
toast({
description: `Imported ${successes} bookmarks and skipped ${alreadyExisted} bookmarks that already existed`,
variant: "default",
});
}
-
if (failures > 0) {
toast({
description: `Failed to import ${failures} bookmarks. Check console for details.`,
@@ -247,7 +138,8 @@ export function useBookmarkImport() {
});
}
- router.push(`/dashboard/lists/${rootList.id}`);
+ if (result.rootListId)
+ router.push(`/dashboard/lists/${result.rootListId}`);
},
onError: (error) => {
setImportProgress(null);
diff --git a/packages/open-api/karakeep-openapi-spec.json b/packages/open-api/karakeep-openapi-spec.json
index 2b5b9ed2..83a5b811 100644
--- a/packages/open-api/karakeep-openapi-spec.json
+++ b/packages/open-api/karakeep-openapi-spec.json
@@ -1813,12 +1813,12 @@
"name": {
"type": "string",
"minLength": 1,
- "maxLength": 40
+ "maxLength": 100
},
"description": {
"type": "string",
"minLength": 0,
- "maxLength": 100
+ "maxLength": 500
},
"icon": {
"type": "string"
@@ -2006,13 +2006,13 @@
"name": {
"type": "string",
"minLength": 1,
- "maxLength": 40
+ "maxLength": 100
},
"description": {
"type": "string",
"nullable": true,
"minLength": 0,
- "maxLength": 100
+ "maxLength": 500
},
"icon": {
"type": "string"
diff --git a/apps/web/lib/exportBookmarks.ts b/packages/shared/import-export/exporters.ts
index 5dc26e78..967937a4 100644
--- a/apps/web/lib/exportBookmarks.ts
+++ b/packages/shared/import-export/exporters.ts
@@ -1,6 +1,6 @@
import { z } from "zod";
-import { BookmarkTypes, ZBookmark } from "@karakeep/shared/types/bookmarks";
+import { BookmarkTypes, ZBookmark } from "../types/bookmarks";
export const zExportBookmarkSchema = z.object({
createdAt: z.number(),
diff --git a/packages/shared/import-export/importer.test.ts b/packages/shared/import-export/importer.test.ts
new file mode 100644
index 00000000..2ea63846
--- /dev/null
+++ b/packages/shared/import-export/importer.test.ts
@@ -0,0 +1,392 @@
+import { describe, expect, it, vi } from "vitest";
+
+import { importBookmarksFromFile, ParsedBookmark } from ".";
+
+const fakeFile = {
+ text: vi.fn().mockResolvedValue("fake file content"),
+} as unknown as File;
+
+describe("importBookmarksFromFile", () => {
+ it("creates root list, folders and imports bookmarks with progress", async () => {
+ const parsers = {
+ pocket: vi.fn().mockReturnValue([
+ {
+ title: "GitHub Repository",
+ content: { type: "link", url: "https://github.com/example/repo" },
+ tags: ["dev", "github"],
+ addDate: 100,
+ paths: [["Development", "Projects"]],
+ },
+ {
+ title: "My Notes",
+ content: { type: "text", text: "Important notes about the project" },
+ tags: ["notes"],
+ addDate: 200,
+ paths: [["Personal"]],
+ notes: "Additional context",
+ archived: true,
+ },
+ {
+ title: "Blog Post",
+ content: { type: "link", url: "https://example.com/blog" },
+ tags: ["reading", "tech"],
+ addDate: 300,
+ paths: [["Reading", "Tech"]],
+ },
+ {
+ title: "No Category Item",
+ content: { type: "link", url: "https://example.com/misc" },
+ tags: [],
+ addDate: 400,
+ paths: [],
+ },
+ {
+ title: "Duplicate URL Test",
+ content: { type: "link", url: "https://github.com/example/repo" },
+ tags: ["duplicate"],
+ addDate: 50, // Earlier date
+ paths: [["Development", "Duplicates"]],
+ },
+ ]),
+ };
+
+ const createdLists: { name: string; icon: string; parentId?: string }[] =
+ [];
+ const createList = vi.fn(
+ async (input: { name: string; icon: string; parentId?: string }) => {
+ createdLists.push(input);
+ return {
+ id: `${input.parentId ? input.parentId + "/" : ""}${input.name}`,
+ };
+ },
+ );
+
+ const createdBookmarks: ParsedBookmark[] = [];
+ const addedToLists: { bookmarkId: string; listIds: string[] }[] = [];
+ const updatedTags: { bookmarkId: string; tags: string[] }[] = [];
+
+ const createBookmark = vi.fn(async (bookmark: ParsedBookmark) => {
+ createdBookmarks.push(bookmark);
+ return {
+ id: `bookmark-${createdBookmarks.length}`,
+ alreadyExists: false,
+ };
+ });
+
+ const addBookmarkToLists = vi.fn(
+ async (input: { bookmarkId: string; listIds: string[] }) => {
+ addedToLists.push(input);
+ },
+ );
+
+ const updateBookmarkTags = vi.fn(
+ async (input: { bookmarkId: string; tags: string[] }) => {
+ updatedTags.push(input);
+ },
+ );
+
+ const progress: number[] = [];
+ const res = await importBookmarksFromFile(
+ {
+ file: fakeFile,
+ source: "pocket",
+ rootListName: "Imported",
+ deps: {
+ createList,
+ createBookmark,
+ addBookmarkToLists,
+ updateBookmarkTags,
+ },
+ onProgress: (d, t) => progress.push(d / t),
+ },
+ { parsers },
+ );
+
+ expect(res.rootListId).toBe("Imported");
+ expect(res.counts).toEqual({
+ successes: 5,
+ failures: 0,
+ alreadyExisted: 0,
+ total: 5, // Using custom parser, no deduplication
+ });
+ // Root + all unique folders from paths
+ expect(createdLists).toEqual([
+ { name: "Imported", icon: "⬆️" },
+ { name: "Development", parentId: "Imported", icon: "📁" },
+ { name: "Personal", parentId: "Imported", icon: "📁" },
+ { name: "Reading", parentId: "Imported", icon: "📁" },
+ { name: "Projects", parentId: "Imported/Development", icon: "📁" },
+ { name: "Tech", parentId: "Imported/Reading", icon: "📁" },
+ { name: "Duplicates", parentId: "Imported/Development", icon: "📁" },
+ ]);
+ // Verify we have 5 created bookmarks (no deduplication with custom parser)
+ expect(createdBookmarks).toHaveLength(5);
+ // Verify GitHub bookmark exists (will be two separate bookmarks since no deduplication)
+ const githubBookmarks = createdBookmarks.filter(
+ (bookmark) =>
+ bookmark.content?.type === "link" &&
+ bookmark.content.url === "https://github.com/example/repo",
+ );
+ expect(githubBookmarks).toHaveLength(2);
+ // Verify text bookmark exists
+ const textBookmark = createdBookmarks.find(
+ (bookmark) => bookmark.content?.type === "text",
+ );
+ expect(textBookmark).toBeDefined();
+ expect(textBookmark!.archived).toBe(true);
+ expect(textBookmark!.notes).toBe("Additional context");
+ // Verify bookmark with no path goes to root
+ const noCategoryBookmark = createdBookmarks.find(
+ (bookmark) =>
+ bookmark.content?.type === "link" &&
+ bookmark.content.url === "https://example.com/misc",
+ );
+ expect(noCategoryBookmark).toBeDefined();
+ // Find the corresponding list assignment for this bookmark
+ const noCategoryBookmarkId = `bookmark-${createdBookmarks.indexOf(noCategoryBookmark!) + 1}`;
+ const listAssignment = addedToLists.find(
+ (a) => a.bookmarkId === noCategoryBookmarkId,
+ );
+ expect(listAssignment!.listIds).toEqual(["Imported"]);
+
+ // Verify that tags were updated for bookmarks that have tags
+ expect(updatedTags.length).toBeGreaterThan(0);
+ expect(progress).toContain(0);
+ expect(progress.at(-1)).toBe(1);
+ });
+
+ it("returns zero counts and null rootListId when no bookmarks", async () => {
+ const parsers = { html: vi.fn().mockReturnValue([]) };
+ const res = await importBookmarksFromFile(
+ {
+ file: fakeFile,
+ source: "html",
+ rootListName: "Imported",
+ deps: {
+ createList: vi.fn(),
+ createBookmark: vi.fn(),
+ addBookmarkToLists: vi.fn(),
+ updateBookmarkTags: vi.fn(),
+ },
+ },
+ { parsers },
+ );
+ expect(res).toEqual({
+ counts: { successes: 0, failures: 0, alreadyExisted: 0, total: 0 },
+ rootListId: null,
+ });
+ });
+
+ it("continues import when individual bookmarks fail", async () => {
+ const parsers = {
+ pocket: vi.fn().mockReturnValue([
+ {
+ title: "Success Bookmark 1",
+ content: { type: "link", url: "https://example.com/success1" },
+ tags: ["success"],
+ addDate: 100,
+ paths: [["Success"]],
+ },
+ {
+ title: "Failure Bookmark",
+ content: { type: "link", url: "https://example.com/failure" },
+ tags: ["failure"],
+ addDate: 200,
+ paths: [["Failure"]],
+ },
+ {
+ title: "Success Bookmark 2",
+ content: { type: "link", url: "https://example.com/success2" },
+ tags: ["success"],
+ addDate: 300,
+ paths: [["Success"]],
+ },
+ ]),
+ };
+
+ const createdLists: { name: string; icon: string; parentId?: string }[] =
+ [];
+ const createList = vi.fn(
+ async (input: { name: string; icon: string; parentId?: string }) => {
+ createdLists.push(input);
+ return {
+ id: `${input.parentId ? input.parentId + "/" : ""}${input.name}`,
+ };
+ },
+ );
+
+ const createdBookmarks: ParsedBookmark[] = [];
+ const addedToLists: { bookmarkId: string; listIds: string[] }[] = [];
+ const updatedTags: { bookmarkId: string; tags: string[] }[] = [];
+
+ const createBookmark = vi.fn(async (bookmark: ParsedBookmark) => {
+ // Simulate failure for the "Failure Bookmark"
+ if (bookmark.title === "Failure Bookmark") {
+ throw new Error("Simulated bookmark creation failure");
+ }
+
+ createdBookmarks.push(bookmark);
+ return {
+ id: `bookmark-${createdBookmarks.length}`,
+ alreadyExists: false,
+ };
+ });
+
+ const addBookmarkToLists = vi.fn(
+ async (input: { bookmarkId: string; listIds: string[] }) => {
+ addedToLists.push(input);
+ },
+ );
+
+ const updateBookmarkTags = vi.fn(
+ async (input: { bookmarkId: string; tags: string[] }) => {
+ updatedTags.push(input);
+ },
+ );
+
+ const progress: number[] = [];
+ const res = await importBookmarksFromFile(
+ {
+ file: fakeFile,
+ source: "pocket",
+ rootListName: "Imported",
+ deps: {
+ createList,
+ createBookmark,
+ addBookmarkToLists,
+ updateBookmarkTags,
+ },
+ onProgress: (d, t) => progress.push(d / t),
+ },
+ { parsers },
+ );
+
+ // Should still create the root list
+ expect(res.rootListId).toBe("Imported");
+
+ // Should track both successes and failures
+ expect(res.counts).toEqual({
+ successes: 2, // Two successful bookmarks
+ failures: 1, // One failed bookmark
+ alreadyExisted: 0,
+ total: 3,
+ });
+
+ // Should create folders for all bookmarks (including failed ones)
+ expect(createdLists).toEqual([
+ { name: "Imported", icon: "⬆️" },
+ { name: "Success", parentId: "Imported", icon: "📁" },
+ { name: "Failure", parentId: "Imported", icon: "📁" },
+ ]);
+
+ // Only successful bookmarks should be created
+ expect(createdBookmarks).toHaveLength(2);
+ expect(createdBookmarks.map((b) => b.title)).toEqual([
+ "Success Bookmark 1",
+ "Success Bookmark 2",
+ ]);
+
+ // Only successful bookmarks should be added to lists and have tags updated
+ expect(addedToLists).toHaveLength(2);
+ expect(updatedTags).toHaveLength(2);
+
+ // Progress should complete even with failures
+ expect(progress).toContain(0);
+ expect(progress.at(-1)).toBe(1);
+ });
+
+ it("handles failures in different stages of bookmark import", async () => {
+ const parsers = {
+ pocket: vi.fn().mockReturnValue([
+ {
+ title: "Success Bookmark",
+ content: { type: "link", url: "https://example.com/success" },
+ tags: ["success"],
+ addDate: 100,
+ paths: [["Success"]],
+ },
+ {
+ title: "Fail at List Assignment",
+ content: { type: "link", url: "https://example.com/fail-list" },
+ tags: ["fail"],
+ addDate: 200,
+ paths: [["Failure"]],
+ },
+ {
+ title: "Fail at Tag Update",
+ content: { type: "link", url: "https://example.com/fail-tag" },
+ tags: ["fail-tag"],
+ addDate: 300,
+ paths: [["Failure"]],
+ },
+ ]),
+ };
+
+ const createList = vi.fn(
+ async (input: { name: string; icon: string; parentId?: string }) => {
+ return {
+ id: `${input.parentId ? input.parentId + "/" : ""}${input.name}`,
+ };
+ },
+ );
+
+ let bookmarkIdCounter = 1;
+ const createBookmark = vi.fn(async () => {
+ return { id: `bookmark-${bookmarkIdCounter++}`, alreadyExists: false };
+ });
+
+ const addBookmarkToLists = vi.fn(
+ async (input: { bookmarkId: string; listIds: string[] }) => {
+ // Simulate failure for specific bookmark
+ if (input.bookmarkId === "bookmark-2") {
+ throw new Error("Failed to add bookmark to lists");
+ }
+ },
+ );
+
+ const updateBookmarkTags = vi.fn(
+ async (input: { bookmarkId: string; tags: string[] }) => {
+ // Simulate failure for specific bookmark
+ if (input.bookmarkId === "bookmark-3") {
+ throw new Error("Failed to update bookmark tags");
+ }
+ },
+ );
+
+ const progress: number[] = [];
+ const res = await importBookmarksFromFile(
+ {
+ file: fakeFile,
+ source: "pocket",
+ rootListName: "Imported",
+ deps: {
+ createList,
+ createBookmark,
+ addBookmarkToLists,
+ updateBookmarkTags,
+ },
+ onProgress: (d, t) => progress.push(d / t),
+ },
+ { parsers },
+ );
+
+ expect(res.rootListId).toBe("Imported");
+
+ // All bookmarks are created successfully, but 2 fail in post-processing
+ expect(res.counts).toEqual({
+ successes: 1, // Only one fully successful bookmark
+ failures: 2, // Two failed in post-processing steps
+ alreadyExisted: 0,
+ total: 3,
+ });
+
+ // All bookmarks should be created (failures happen after bookmark creation)
+ expect(createBookmark).toHaveBeenCalledTimes(3);
+
+ // addBookmarkToLists should be called 3 times (but one fails)
+ expect(addBookmarkToLists).toHaveBeenCalledTimes(3);
+
+ // updateBookmarkTags should be called 2 times (once fails at list assignment, one fails at tag update)
+ expect(updateBookmarkTags).toHaveBeenCalledTimes(2);
+ });
+});
diff --git a/packages/shared/import-export/importer.ts b/packages/shared/import-export/importer.ts
new file mode 100644
index 00000000..88c0c3bc
--- /dev/null
+++ b/packages/shared/import-export/importer.ts
@@ -0,0 +1,158 @@
+import { limitConcurrency } from "../concurrency";
+import { MAX_LIST_NAME_LENGTH } from "../types/lists";
+import { ImportSource, ParsedBookmark, parseImportFile } from "./parsers";
+
+export interface ImportCounts {
+ successes: number;
+ failures: number;
+ alreadyExisted: number;
+ total: number;
+}
+
+export interface ImportDeps {
+ createList: (input: {
+ name: string;
+ icon: string;
+ parentId?: string;
+ }) => Promise<{ id: string }>;
+ createBookmark: (
+ bookmark: ParsedBookmark,
+ ) => Promise<{ id: string; alreadyExists?: boolean }>;
+ addBookmarkToLists: (input: {
+ bookmarkId: string;
+ listIds: string[];
+ }) => Promise<void>;
+ updateBookmarkTags: (input: {
+ bookmarkId: string;
+ tags: string[];
+ }) => Promise<void>;
+}
+
+export interface ImportOptions {
+ concurrencyLimit?: number;
+ parsers?: Partial<
+ Record<ImportSource, (textContent: string) => ParsedBookmark[]>
+ >;
+}
+
+export interface ImportResult {
+ counts: ImportCounts;
+ rootListId: string | null;
+}
+
+export async function importBookmarksFromFile(
+ {
+ file,
+ source,
+ rootListName,
+ deps,
+ onProgress,
+ }: {
+ file: { text: () => Promise<string> };
+ source: ImportSource;
+ rootListName: string;
+ deps: ImportDeps;
+ onProgress?: (done: number, total: number) => void;
+ },
+ options: ImportOptions = {},
+): Promise<ImportResult> {
+ const { concurrencyLimit = 20, parsers } = options;
+
+ const textContent = await file.text();
+ const parsedBookmarks = parsers?.[source]
+ ? parsers[source]!(textContent)
+ : parseImportFile(source, textContent);
+ if (parsedBookmarks.length === 0) {
+ return {
+ counts: { successes: 0, failures: 0, alreadyExisted: 0, total: 0 },
+ rootListId: null,
+ };
+ }
+
+ const rootList = await deps.createList({ name: rootListName, icon: "⬆️" });
+
+ onProgress?.(0, parsedBookmarks.length);
+
+ const PATH_DELIMITER = "$$__$$";
+
+ // Build required paths
+ const allRequiredPaths = new Set<string>();
+ for (const bookmark of parsedBookmarks) {
+ for (const path of bookmark.paths) {
+ if (path && path.length > 0) {
+ for (let i = 1; i <= path.length; i++) {
+ const subPath = path.slice(0, i);
+ const pathKey = subPath.join(PATH_DELIMITER);
+ allRequiredPaths.add(pathKey);
+ }
+ }
+ }
+ }
+
+ const allRequiredPathsArray = Array.from(allRequiredPaths).sort(
+ (a, b) => a.split(PATH_DELIMITER).length - b.split(PATH_DELIMITER).length,
+ );
+
+ const pathMap: Record<string, string> = { "": rootList.id };
+
+ for (const pathKey of allRequiredPathsArray) {
+ const parts = pathKey.split(PATH_DELIMITER);
+ const parentKey = parts.slice(0, -1).join(PATH_DELIMITER);
+ const parentId = pathMap[parentKey] || rootList.id;
+
+ const folderName = parts[parts.length - 1];
+ const folderList = await deps.createList({
+ name: folderName.substring(0, MAX_LIST_NAME_LENGTH),
+ parentId,
+ icon: "📁",
+ });
+ pathMap[pathKey] = folderList.id;
+ }
+
+ const importPromises = parsedBookmarks.map((bookmark) => async () => {
+ const listIds = bookmark.paths.map(
+ (path) => pathMap[path.join(PATH_DELIMITER)] || rootList.id,
+ );
+ if (listIds.length === 0) listIds.push(rootList.id);
+
+ const created = await deps.createBookmark(bookmark);
+ await deps.addBookmarkToLists({ bookmarkId: created.id, listIds });
+ if (bookmark.tags && bookmark.tags.length > 0) {
+ await deps.updateBookmarkTags({
+ bookmarkId: created.id,
+ tags: bookmark.tags,
+ });
+ }
+
+ return created;
+ });
+
+ const resultsPromises = limitConcurrency(importPromises, concurrencyLimit);
+ const results = await Promise.allSettled(resultsPromises);
+
+ let successes = 0;
+ let failures = 0;
+ let alreadyExisted = 0;
+
+ let done = 0;
+ for (const r of results) {
+ if (r.status === "fulfilled") {
+ if (r.value.alreadyExists) alreadyExisted++;
+ else successes++;
+ } else {
+ failures++;
+ }
+ done += 1;
+ onProgress?.(done, parsedBookmarks.length);
+ }
+
+ return {
+ counts: {
+ successes,
+ failures,
+ alreadyExisted,
+ total: parsedBookmarks.length,
+ },
+ rootListId: rootList.id,
+ };
+}
diff --git a/packages/shared/import-export/index.ts b/packages/shared/import-export/index.ts
new file mode 100644
index 00000000..2d720d0b
--- /dev/null
+++ b/packages/shared/import-export/index.ts
@@ -0,0 +1,3 @@
+export * from "./exporters";
+export * from "./importer";
+export type { ImportSource, ParsedBookmark } from "./parsers";
diff --git a/apps/web/lib/importBookmarkParser.ts b/packages/shared/import-export/parsers.ts
index 44fe872c..c969c615 100644
--- a/apps/web/lib/importBookmarkParser.ts
+++ b/packages/shared/import-export/parsers.ts
@@ -1,11 +1,19 @@
// Copied from https://gist.github.com/devster31/4e8c6548fd16ffb75c02e6f24e27f9b9
+
import * as cheerio from "cheerio";
import { parse } from "csv-parse/sync";
import { z } from "zod";
-import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
+import { BookmarkTypes } from "../types/bookmarks";
+import { zExportSchema } from "./exporters";
-import { zExportSchema } from "./exportBookmarks";
+export type ImportSource =
+ | "html"
+ | "pocket"
+ | "omnivore"
+ | "karakeep"
+ | "linkwarden"
+ | "tab-session-manager";
export interface ParsedBookmark {
title: string;
@@ -19,11 +27,7 @@ export interface ParsedBookmark {
paths: string[][];
}
-export async function parseNetscapeBookmarkFile(
- file: File,
-): Promise<ParsedBookmark[]> {
- const textContent = await file.text();
-
+function parseNetscapeBookmarkFile(textContent: string): ParsedBookmark[] {
if (!textContent.startsWith("<!DOCTYPE NETSCAPE-Bookmark-file-1>")) {
throw Error("The uploaded html file does not seem to be a bookmark file");
}
@@ -66,11 +70,7 @@ export async function parseNetscapeBookmarkFile(
.get();
}
-export async function parsePocketBookmarkFile(
- file: File,
-): Promise<ParsedBookmark[]> {
- const textContent = await file.text();
-
+function parsePocketBookmarkFile(textContent: string): ParsedBookmark[] {
const records = parse(textContent, {
columns: true,
skip_empty_lines: true,
@@ -94,11 +94,7 @@ export async function parsePocketBookmarkFile(
});
}
-export async function parseKarakeepBookmarkFile(
- file: File,
-): Promise<ParsedBookmark[]> {
- const textContent = await file.text();
-
+function parseKarakeepBookmarkFile(textContent: string): ParsedBookmark[] {
const parsed = zExportSchema.safeParse(JSON.parse(textContent));
if (!parsed.success) {
throw new Error(
@@ -131,10 +127,7 @@ export async function parseKarakeepBookmarkFile(
});
}
-export async function parseOmnivoreBookmarkFile(
- file: File,
-): Promise<ParsedBookmark[]> {
- const textContent = await file.text();
+function parseOmnivoreBookmarkFile(textContent: string): ParsedBookmark[] {
const zOmnivoreExportSchema = z.array(
z.object({
title: z.string(),
@@ -164,10 +157,7 @@ export async function parseOmnivoreBookmarkFile(
});
}
-export async function parseLinkwardenBookmarkFile(
- file: File,
-): Promise<ParsedBookmark[]> {
- const textContent = await file.text();
+function parseLinkwardenBookmarkFile(textContent: string): ParsedBookmark[] {
const zLinkwardenExportSchema = z.object({
collections: z.array(
z.object({
@@ -201,11 +191,9 @@ export async function parseLinkwardenBookmarkFile(
});
}
-export async function parseTabSessionManagerStateFile(
- file: File,
-): Promise<ParsedBookmark[]> {
- const textContent = await file.text();
-
+function parseTabSessionManagerStateFile(
+ textContent: string,
+): ParsedBookmark[] {
const zTab = z.object({
url: z.string(),
title: z.string(),
@@ -242,9 +230,7 @@ export async function parseTabSessionManagerStateFile(
);
}
-export function deduplicateBookmarks(
- bookmarks: ParsedBookmark[],
-): ParsedBookmark[] {
+function deduplicateBookmarks(bookmarks: ParsedBookmark[]): ParsedBookmark[] {
const deduplicatedBookmarksMap = new Map<string, ParsedBookmark>();
const textBookmarks: ParsedBookmark[] = [];
@@ -284,3 +270,31 @@ export function deduplicateBookmarks(
return [...deduplicatedBookmarksMap.values(), ...textBookmarks];
}
+
+export function parseImportFile(
+ source: ImportSource,
+ textContent: string,
+): ParsedBookmark[] {
+ let result: ParsedBookmark[];
+ switch (source) {
+ case "html":
+ result = parseNetscapeBookmarkFile(textContent);
+ break;
+ case "pocket":
+ result = parsePocketBookmarkFile(textContent);
+ break;
+ case "karakeep":
+ result = parseKarakeepBookmarkFile(textContent);
+ break;
+ case "omnivore":
+ result = parseOmnivoreBookmarkFile(textContent);
+ break;
+ case "linkwarden":
+ result = parseLinkwardenBookmarkFile(textContent);
+ break;
+ case "tab-session-manager":
+ result = parseTabSessionManagerStateFile(textContent);
+ break;
+ }
+ return deduplicateBookmarks(result);
+}
diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts
index f96cf0c5..a22e7632 100644
--- a/packages/shared/types/bookmarks.ts
+++ b/packages/shared/types/bookmarks.ts
@@ -3,7 +3,7 @@ import { z } from "zod";
import { zCursorV2 } from "./pagination";
import { zBookmarkTagSchema } from "./tags";
-const MAX_TITLE_LENGTH = 1000;
+export const MAX_BOOKMARK_TITLE_LENGTH = 1000;
export const enum BookmarkTypes {
LINK = "link",
@@ -133,7 +133,7 @@ export type ZBookmarkTypeAsset = z.infer<typeof zBookmarkTypeAssetSchema>;
// POST /v1/bookmarks
export const zNewBookmarkRequestSchema = z
.object({
- title: z.string().max(MAX_TITLE_LENGTH).nullish(),
+ title: z.string().max(MAX_BOOKMARK_TITLE_LENGTH).nullish(),
archived: z.boolean().optional(),
favourited: z.boolean().optional(),
note: z.string().optional(),
@@ -202,7 +202,7 @@ export const zUpdateBookmarksRequestSchema = z.object({
favourited: z.boolean().optional(),
summary: z.string().nullish(),
note: z.string().optional(),
- title: z.string().max(MAX_TITLE_LENGTH).nullish(),
+ title: z.string().max(MAX_BOOKMARK_TITLE_LENGTH).nullish(),
createdAt: z.coerce.date().optional(),
// Link specific fields (optional)
url: z.string().url().optional(),
diff --git a/packages/shared/types/lists.ts b/packages/shared/types/lists.ts
index 51fb458c..59abb007 100644
--- a/packages/shared/types/lists.ts
+++ b/packages/shared/types/lists.ts
@@ -2,16 +2,25 @@ import { z } from "zod";
import { parseSearchQuery } from "../searchQueryParser";
+export const MAX_LIST_NAME_LENGTH = 100;
+export const MAX_LIST_DESCRIPTION_LENGTH = 500;
+
export const zNewBookmarkListSchema = z
.object({
name: z
.string()
.min(1, "List name can't be empty")
- .max(40, "List name is at most 40 chars"),
+ .max(
+ MAX_LIST_NAME_LENGTH,
+ `List name is at most ${MAX_LIST_NAME_LENGTH} chars`,
+ ),
description: z
.string()
.min(0, "Description can be empty")
- .max(100, "Description can have at most 100 chars")
+ .max(
+ MAX_LIST_DESCRIPTION_LENGTH,
+ `Description can have at most ${MAX_LIST_DESCRIPTION_LENGTH} chars`,
+ )
.optional(),
icon: z.string(),
type: z.enum(["manual", "smart"]).optional().default("manual"),
@@ -57,12 +66,18 @@ export const zEditBookmarkListSchema = z.object({
name: z
.string()
.min(1, "List name can't be empty")
- .max(40, "List name is at most 40 chars")
+ .max(
+ MAX_LIST_NAME_LENGTH,
+ `List name is at most ${MAX_LIST_NAME_LENGTH} chars`,
+ )
.optional(),
description: z
.string()
.min(0, "Description can be empty")
- .max(100, "Description can have at most 100 chars")
+ .max(
+ MAX_LIST_DESCRIPTION_LENGTH,
+ `Description can have at most ${MAX_LIST_DESCRIPTION_LENGTH} chars`,
+ )
.nullish(),
icon: z.string().optional(),
parentId: z.string().nullish(),
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2026-01-29 11:01:47 +0000