fix: handle list with slashes in their names and truncate long list names. fixes #1597

4 files changed, 134 insertions, 228 deletions

diff --git a/apps/web/app/api/bookmarks/export/route.tsx b/apps/web/app/api/bookmarks/export/route.tsx
index 47fdeebc..ad309877 100644
--- a/apps/web/app/api/bookmarks/export/route.tsx
+++ b/apps/web/app/api/bookmarks/export/route.tsx
@@ -1,12 +1,12 @@
 import { NextRequest } from "next/server";
+import { api, createContextFromRequest } from "@/server/api/client";
+import { z } from "zod";
+
 import {
   toExportFormat,
   toNetscapeFormat,
   zExportSchema,
-} from "@/lib/exportBookmarks";
-import { api, createContextFromRequest } from "@/server/api/client";
-import { z } from "zod";
-
+} from "@karakeep/shared/import-export";
 import { MAX_NUM_BOOKMARKS_PER_PAGE } from "@karakeep/shared/types/bookmarks";
 
 export const dynamic = "force-dynamic";
diff --git a/apps/web/lib/hooks/useBookmarkImport.ts b/apps/web/lib/hooks/useBookmarkImport.ts
index 7e5f6111..de515677 100644
--- a/apps/web/lib/hooks/useBookmarkImport.ts
+++ b/apps/web/lib/hooks/useBookmarkImport.ts
@@ -4,16 +4,6 @@ import { useState } from "react";
 import { useRouter } from "next/navigation";
 import { toast } from "@/components/ui/use-toast";
 import { useTranslation } from "@/lib/i18n/client";
-import {
-  deduplicateBookmarks,
-  ParsedBookmark,
-  parseKarakeepBookmarkFile,
-  parseLinkwardenBookmarkFile,
-  parseNetscapeBookmarkFile,
-  parseOmnivoreBookmarkFile,
-  parsePocketBookmarkFile,
-  parseTabSessionManagerStateFile,
-} from "@/lib/importBookmarkParser";
 import { useMutation } from "@tanstack/react-query";
 
 import {
@@ -24,16 +14,15 @@ import {
   useAddBookmarkToList,
   useCreateBookmarkList,
 } from "@karakeep/shared-react/hooks/lists";
-import { limitConcurrency } from "@karakeep/shared/concurrency";
-import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
-
-export type ImportSource =
-  | "html"
-  | "pocket"
-  | "omnivore"
-  | "karakeep"
-  | "linkwarden"
-  | "tab-session-manager";
+import {
+  importBookmarksFromFile,
+  ImportSource,
+  ParsedBookmark,
+} from "@karakeep/shared/import-export";
+import {
+  BookmarkTypes,
+  MAX_BOOKMARK_TITLE_LENGTH,
+} from "@karakeep/shared/types/bookmarks";
 
 export interface ImportProgress {
   done: number;
@@ -53,53 +42,6 @@ export function useBookmarkImport() {
   const { mutateAsync: addToList } = useAddBookmarkToList();
   const { mutateAsync: updateTags } = useUpdateBookmarkTags();
 
-  const { mutateAsync: parseAndCreateBookmark } = useMutation({
-    mutationFn: async (toImport: {
-      bookmark: ParsedBookmark;
-      listIds: string[];
-    }) => {
-      const bookmark = toImport.bookmark;
-      if (bookmark.content === undefined) {
-        throw new Error("Content is undefined");
-      }
-      const created = await createBookmark({
-        crawlPriority: "low",
-        title: bookmark.title,
-        createdAt: bookmark.addDate
-          ? new Date(bookmark.addDate * 1000)
-          : undefined,
-        note: bookmark.notes,
-        archived: bookmark.archived,
-        ...(bookmark.content.type === BookmarkTypes.LINK
-          ? {
-              type: BookmarkTypes.LINK,
-              url: bookmark.content.url,
-            }
-          : {
-              type: BookmarkTypes.TEXT,
-              text: bookmark.content.text,
-            }),
-      });
-
-      await Promise.all([
-        ...toImport.listIds.map((listId) =>
-          addToList({
-            bookmarkId: created.id,
-            listId,
-          }),
-        ),
-        bookmark.tags.length > 0
-          ? updateTags({
-              bookmarkId: created.id,
-              attach: bookmark.tags.map((t) => ({ tagName: t })),
-              detach: [],
-            })
-          : undefined,
-      ]);
-      return created;
-    },
-  });
-
   const uploadBookmarkFileMutation = useMutation({
     mutationFn: async ({
       file,
@@ -108,138 +50,87 @@ export function useBookmarkImport() {
       file: File;
       source: ImportSource;
     }) => {
-      if (source === "html") {
-        return await parseNetscapeBookmarkFile(file);
-      } else if (source === "pocket") {
-        return await parsePocketBookmarkFile(file);
-      } else if (source === "karakeep") {
-        return await parseKarakeepBookmarkFile(file);
-      } else if (source === "omnivore") {
-        return await parseOmnivoreBookmarkFile(file);
-      } else if (source === "linkwarden") {
-        return await parseLinkwardenBookmarkFile(file);
-      } else if (source === "tab-session-manager") {
-        return await parseTabSessionManagerStateFile(file);
-      } else {
-        throw new Error("Unknown source");
-      }
-    },
-    onSuccess: async (parsedBookmarks) => {
-      if (parsedBookmarks.length === 0) {
-        toast({ description: "No bookmarks found in the file." });
-        return;
-      }
-
-      const rootList = await createList({
-        name: t("settings.import.imported_bookmarks"),
-        icon: "⬆️",
-      });
-
-      const finalBookmarksToImport = deduplicateBookmarks(parsedBookmarks);
-
-      setImportProgress({ done: 0, total: finalBookmarksToImport.length });
-
-      const allRequiredPaths = new Set<string>();
-      for (const bookmark of finalBookmarksToImport) {
-        for (const path of bookmark.paths) {
-          if (path && path.length > 0) {
-            for (let i = 1; i <= path.length; i++) {
-              const subPath = path.slice(0, i);
-              const pathKey = subPath.join("/");
-              allRequiredPaths.add(pathKey);
-            }
-          }
-        }
-      }
-
-      const allRequiredPathsArray = Array.from(allRequiredPaths).sort(
-        (a, b) => a.split("/").length - b.split("/").length,
-      );
-
-      const pathMap: Record<string, string> = {};
-      pathMap[""] = rootList.id;
-
-      for (const pathKey of allRequiredPathsArray) {
-        const parts = pathKey.split("/");
-        const parentKey = parts.slice(0, -1).join("/");
-        const parentId = pathMap[parentKey] || rootList.id;
-
-        const folderName = parts[parts.length - 1];
-        const folderList = await createList({
-          name: folderName,
-          parentId: parentId,
-          icon: "📁",
-        });
-        pathMap[pathKey] = folderList.id;
-      }
-
-      const importPromises = finalBookmarksToImport.map(
-        (bookmark) => async () => {
-          const listIds = bookmark.paths.map(
-            (path) => pathMap[path.join("/")] || rootList.id,
-          );
-          if (listIds.length === 0) {
-            listIds.push(rootList.id);
-          }
-
-          try {
-            const created = await parseAndCreateBookmark({
-              bookmark: bookmark,
+      const result = await importBookmarksFromFile(
+        {
+          file,
+          source,
+          rootListName: t("settings.import.imported_bookmarks"),
+          deps: {
+            createList: createList,
+            createBookmark: async (bookmark: ParsedBookmark) => {
+              if (bookmark.content === undefined) {
+                throw new Error("Content is undefined");
+              }
+              const created = await createBookmark({
+                crawlPriority: "low",
+                title: bookmark.title.substring(0, MAX_BOOKMARK_TITLE_LENGTH),
+                createdAt: bookmark.addDate
+                  ? new Date(bookmark.addDate * 1000)
+                  : undefined,
+                note: bookmark.notes,
+                archived: bookmark.archived,
+                ...(bookmark.content.type === BookmarkTypes.LINK
+                  ? {
+                      type: BookmarkTypes.LINK,
+                      url: bookmark.content.url,
+                    }
+                  : {
+                      type: BookmarkTypes.TEXT,
+                      text: bookmark.content.text,
+                    }),
+              });
+              return created as { id: string; alreadyExists?: boolean };
+            },
+            addBookmarkToLists: async ({
+              bookmarkId,
               listIds,
-            });
-
-            setImportProgress((prev) => {
-              const newDone = (prev?.done ?? 0) + 1;
-              return {
-                done: newDone,
-                total: finalBookmarksToImport.length,
-              };
-            });
-            return { status: "fulfilled" as const, value: created };
-          } catch {
-            setImportProgress((prev) => {
-              const newDone = (prev?.done ?? 0) + 1;
-              return {
-                done: newDone,
-                total: finalBookmarksToImport.length,
-              };
-            });
-            return { status: "rejected" as const };
-          }
+            }: {
+              bookmarkId: string;
+              listIds: string[];
+            }) => {
+              await Promise.all(
+                listIds.map((listId) =>
+                  addToList({
+                    bookmarkId,
+                    listId,
+                  }),
+                ),
+              );
+            },
+            updateBookmarkTags: async ({
+              bookmarkId,
+              tags,
+            }: {
+              bookmarkId: string;
+              tags: string[];
+            }) => {
+              if (tags.length > 0) {
+                await updateTags({
+                  bookmarkId,
+                  attach: tags.map((t) => ({ tagName: t })),
+                  detach: [],
+                });
+              }
+            },
+          },
+          onProgress: (done, total) => setImportProgress({ done, total }),
         },
+        {},
       );
-
-      const CONCURRENCY_LIMIT = 20;
-      const resultsPromises = limitConcurrency(
-        importPromises,
-        CONCURRENCY_LIMIT,
-      );
-
-      const results = await Promise.all(resultsPromises);
-
-      let successes = 0;
-      let failures = 0;
-      let alreadyExisted = 0;
-
-      for (const result of results) {
-        if (result.status === "fulfilled") {
-          if (result.value.alreadyExists) {
-            alreadyExisted++;
-          } else {
-            successes++;
-          }
-        } else {
-          failures++;
-        }
+      return result;
+    },
+    onSuccess: async (result) => {
+      if (result.counts.total === 0) {
+        toast({ description: "No bookmarks found in the file." });
+        return;
       }
-
+      const { successes, failures, alreadyExisted } = result.counts;
       if (successes > 0 || alreadyExisted > 0) {
         toast({
           description: `Imported ${successes} bookmarks and skipped ${alreadyExisted} bookmarks that already existed`,
           variant: "default",
         });
       }
-
       if (failures > 0) {
         toast({
           description: `Failed to import ${failures} bookmarks. Check console for details.`,
@@ -247,7 +138,8 @@ export function useBookmarkImport() {
         });
       }
 
-      router.push(`/dashboard/lists/${rootList.id}`);
+      if (result.rootListId)
+        router.push(`/dashboard/lists/${result.rootListId}`);
     },
     onError: (error) => {
       setImportProgress(null);
diff --git a/apps/web/lib/exportBookmarks.ts b/packages/shared/import-export/exporters.ts
index 5dc26e78..967937a4 100644
--- a/apps/web/lib/exportBookmarks.ts
+++ b/packages/shared/import-export/exporters.ts
@@ -1,6 +1,6 @@
 import { z } from "zod";
 
-import { BookmarkTypes, ZBookmark } from "@karakeep/shared/types/bookmarks";
+import { BookmarkTypes, ZBookmark } from "../types/bookmarks";
 
 export const zExportBookmarkSchema = z.object({
   createdAt: z.number(),
diff --git a/apps/web/lib/importBookmarkParser.ts b/packages/shared/import-export/parsers.ts
index 44fe872c..c969c615 100644
--- a/apps/web/lib/importBookmarkParser.ts
+++ b/packages/shared/import-export/parsers.ts
@@ -1,11 +1,19 @@
 // Copied from https://gist.github.com/devster31/4e8c6548fd16ffb75c02e6f24e27f9b9
+
 import * as cheerio from "cheerio";
 import { parse } from "csv-parse/sync";
 import { z } from "zod";
 
-import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
+import { BookmarkTypes } from "../types/bookmarks";
+import { zExportSchema } from "./exporters";
 
-import { zExportSchema } from "./exportBookmarks";
+export type ImportSource =
+  | "html"
+  | "pocket"
+  | "omnivore"
+  | "karakeep"
+  | "linkwarden"
+  | "tab-session-manager";
 
 export interface ParsedBookmark {
   title: string;
@@ -19,11 +27,7 @@ export interface ParsedBookmark {
   paths: string[][];
 }
 
-export async function parseNetscapeBookmarkFile(
-  file: File,
-): Promise<ParsedBookmark[]> {
-  const textContent = await file.text();
-
+function parseNetscapeBookmarkFile(textContent: string): ParsedBookmark[] {
   if (!textContent.startsWith("<!DOCTYPE NETSCAPE-Bookmark-file-1>")) {
     throw Error("The uploaded html file does not seem to be a bookmark file");
   }
@@ -66,11 +70,7 @@ export async function parseNetscapeBookmarkFile(
     .get();
 }
 
-export async function parsePocketBookmarkFile(
-  file: File,
-): Promise<ParsedBookmark[]> {
-  const textContent = await file.text();
-
+function parsePocketBookmarkFile(textContent: string): ParsedBookmark[] {
   const records = parse(textContent, {
     columns: true,
     skip_empty_lines: true,
@@ -94,11 +94,7 @@ export async function parsePocketBookmarkFile(
   });
 }
 
-export async function parseKarakeepBookmarkFile(
-  file: File,
-): Promise<ParsedBookmark[]> {
-  const textContent = await file.text();
-
+function parseKarakeepBookmarkFile(textContent: string): ParsedBookmark[] {
   const parsed = zExportSchema.safeParse(JSON.parse(textContent));
   if (!parsed.success) {
     throw new Error(
@@ -131,10 +127,7 @@ export async function parseKarakeepBookmarkFile(
   });
 }
 
-export async function parseOmnivoreBookmarkFile(
-  file: File,
-): Promise<ParsedBookmark[]> {
-  const textContent = await file.text();
+function parseOmnivoreBookmarkFile(textContent: string): ParsedBookmark[] {
   const zOmnivoreExportSchema = z.array(
     z.object({
       title: z.string(),
@@ -164,10 +157,7 @@ export async function parseOmnivoreBookmarkFile(
   });
 }
 
-export async function parseLinkwardenBookmarkFile(
-  file: File,
-): Promise<ParsedBookmark[]> {
-  const textContent = await file.text();
+function parseLinkwardenBookmarkFile(textContent: string): ParsedBookmark[] {
   const zLinkwardenExportSchema = z.object({
     collections: z.array(
       z.object({
@@ -201,11 +191,9 @@ export async function parseLinkwardenBookmarkFile(
   });
 }
 
-export async function parseTabSessionManagerStateFile(
-  file: File,
-): Promise<ParsedBookmark[]> {
-  const textContent = await file.text();
-
+function parseTabSessionManagerStateFile(
+  textContent: string,
+): ParsedBookmark[] {
   const zTab = z.object({
     url: z.string(),
     title: z.string(),
@@ -242,9 +230,7 @@ export async function parseTabSessionManagerStateFile(
   );
 }
 
-export function deduplicateBookmarks(
-  bookmarks: ParsedBookmark[],
-): ParsedBookmark[] {
+function deduplicateBookmarks(bookmarks: ParsedBookmark[]): ParsedBookmark[] {
   const deduplicatedBookmarksMap = new Map<string, ParsedBookmark>();
   const textBookmarks: ParsedBookmark[] = [];
 
@@ -284,3 +270,31 @@ export function deduplicateBookmarks(
 
   return [...deduplicatedBookmarksMap.values(), ...textBookmarks];
 }
+
+export function parseImportFile(
+  source: ImportSource,
+  textContent: string,
+): ParsedBookmark[] {
+  let result: ParsedBookmark[];
+  switch (source) {
+    case "html":
+      result = parseNetscapeBookmarkFile(textContent);
+      break;
+    case "pocket":
+      result = parsePocketBookmarkFile(textContent);
+      break;
+    case "karakeep":
+      result = parseKarakeepBookmarkFile(textContent);
+      break;
+    case "omnivore":
+      result = parseOmnivoreBookmarkFile(textContent);
+      break;
+    case "linkwarden":
+      result = parseLinkwardenBookmarkFile(textContent);
+      break;
+    case "tab-session-manager":
+      result = parseTabSessionManagerStateFile(textContent);
+      break;
+  }
+  return deduplicateBookmarks(result);
+}