From aecbe6ae8b3dbc7bcdcf33f1c8c086dafb77eb24 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sat, 30 Aug 2025 15:26:02 +0000 Subject: fix: handle list with slashes in their names and truncate long list names. fixes #1597 --- apps/web/app/api/bookmarks/export/route.tsx | 8 +- apps/web/lib/exportBookmarks.ts | 111 ------- apps/web/lib/hooks/useBookmarkImport.ts | 272 ++++++----------- apps/web/lib/importBookmarkParser.ts | 286 ------------------ packages/open-api/karakeep-openapi-spec.json | 8 +- packages/shared/import-export/exporters.ts | 111 +++++++ packages/shared/import-export/importer.test.ts | 392 +++++++++++++++++++++++++ packages/shared/import-export/importer.ts | 158 ++++++++++ packages/shared/import-export/index.ts | 3 + packages/shared/import-export/parsers.ts | 300 +++++++++++++++++++ packages/shared/types/bookmarks.ts | 6 +- packages/shared/types/lists.ts | 23 +- 12 files changed, 1076 insertions(+), 602 deletions(-) delete mode 100644 apps/web/lib/exportBookmarks.ts delete mode 100644 apps/web/lib/importBookmarkParser.ts create mode 100644 packages/shared/import-export/exporters.ts create mode 100644 packages/shared/import-export/importer.test.ts create mode 100644 packages/shared/import-export/importer.ts create mode 100644 packages/shared/import-export/index.ts create mode 100644 packages/shared/import-export/parsers.ts diff --git a/apps/web/app/api/bookmarks/export/route.tsx b/apps/web/app/api/bookmarks/export/route.tsx index 47fdeebc..ad309877 100644 --- a/apps/web/app/api/bookmarks/export/route.tsx +++ b/apps/web/app/api/bookmarks/export/route.tsx @@ -1,12 +1,12 @@ import { NextRequest } from "next/server"; +import { api, createContextFromRequest } from "@/server/api/client"; +import { z } from "zod"; + import { toExportFormat, toNetscapeFormat, zExportSchema, -} from "@/lib/exportBookmarks"; -import { api, createContextFromRequest } from "@/server/api/client"; -import { z } from "zod"; - +} from "@karakeep/shared/import-export"; import { MAX_NUM_BOOKMARKS_PER_PAGE } from "@karakeep/shared/types/bookmarks"; export const dynamic = "force-dynamic"; diff --git a/apps/web/lib/exportBookmarks.ts b/apps/web/lib/exportBookmarks.ts deleted file mode 100644 index 5dc26e78..00000000 --- a/apps/web/lib/exportBookmarks.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { z } from "zod"; - -import { BookmarkTypes, ZBookmark } from "@karakeep/shared/types/bookmarks"; - -export const zExportBookmarkSchema = z.object({ - createdAt: z.number(), - title: z.string().nullable(), - tags: z.array(z.string()), - content: z - .discriminatedUnion("type", [ - z.object({ - type: z.literal(BookmarkTypes.LINK), - url: z.string(), - }), - z.object({ - type: z.literal(BookmarkTypes.TEXT), - text: z.string(), - }), - ]) - .nullable(), - note: z.string().nullable(), - archived: z.boolean().optional().default(false), -}); - -export const zExportSchema = z.object({ - bookmarks: z.array(zExportBookmarkSchema), -}); - -export function toExportFormat( - bookmark: ZBookmark, -): z.infer { - let content = null; - switch (bookmark.content.type) { - case BookmarkTypes.LINK: { - content = { - type: bookmark.content.type, - url: bookmark.content.url, - }; - break; - } - case BookmarkTypes.TEXT: { - content = { - type: bookmark.content.type, - text: bookmark.content.text, - }; - break; - } - // Exclude asset types for now - } - return { - createdAt: Math.floor(bookmark.createdAt.getTime() / 1000), - title: - bookmark.title ?? - (bookmark.content.type === BookmarkTypes.LINK - ? (bookmark.content.title ?? null) - : null), - tags: bookmark.tags.map((t) => t.name), - content, - note: bookmark.note ?? null, - archived: bookmark.archived, - }; -} - -export function toNetscapeFormat(bookmarks: ZBookmark[]): string { - const header = ` - - -Bookmarks -

Bookmarks

-

`; - - const footer = `

`; - - const bookmarkEntries = bookmarks - .map((bookmark) => { - if (bookmark.content?.type !== BookmarkTypes.LINK) { - return ""; - } - const addDate = bookmark.createdAt - ? `ADD_DATE="${Math.floor(bookmark.createdAt.getTime() / 1000)}"` - : ""; - - const tagNames = bookmark.tags.map((t) => t.name).join(","); - const tags = tagNames.length > 0 ? `TAGS="${tagNames}"` : ""; - - const encodedUrl = encodeURI(bookmark.content.url); - const displayTitle = bookmark.title ?? bookmark.content.url; - const encodedTitle = escapeHtml(displayTitle); - - return `

${encodedTitle}`; - }) - .filter(Boolean) - .join("\n"); - - return `${header}\n${bookmarkEntries}\n${footer}`; -} - -function escapeHtml(input: string): string { - const escapeMap: Record = { - "&": "&", - "'": "'", - "`": "`", - '"': """, - "<": "<", - ">": ">", - }; - - return input.replace(/[&'`"<>]/g, (match) => escapeMap[match] || ""); -} diff --git a/apps/web/lib/hooks/useBookmarkImport.ts b/apps/web/lib/hooks/useBookmarkImport.ts index 7e5f6111..de515677 100644 --- a/apps/web/lib/hooks/useBookmarkImport.ts +++ b/apps/web/lib/hooks/useBookmarkImport.ts @@ -4,16 +4,6 @@ import { useState } from "react"; import { useRouter } from "next/navigation"; import { toast } from "@/components/ui/use-toast"; import { useTranslation } from "@/lib/i18n/client"; -import { - deduplicateBookmarks, - ParsedBookmark, - parseKarakeepBookmarkFile, - parseLinkwardenBookmarkFile, - parseNetscapeBookmarkFile, - parseOmnivoreBookmarkFile, - parsePocketBookmarkFile, - parseTabSessionManagerStateFile, -} from "@/lib/importBookmarkParser"; import { useMutation } from "@tanstack/react-query"; import { @@ -24,16 +14,15 @@ import { useAddBookmarkToList, useCreateBookmarkList, } from "@karakeep/shared-react/hooks/lists"; -import { limitConcurrency } from "@karakeep/shared/concurrency"; -import { BookmarkTypes } from "@karakeep/shared/types/bookmarks"; - -export type ImportSource = - | "html" - | "pocket" - | "omnivore" - | "karakeep" - | "linkwarden" - | "tab-session-manager"; +import { + importBookmarksFromFile, + ImportSource, + ParsedBookmark, +} from "@karakeep/shared/import-export"; +import { + BookmarkTypes, + MAX_BOOKMARK_TITLE_LENGTH, +} from "@karakeep/shared/types/bookmarks"; export interface ImportProgress { done: number; @@ -53,53 +42,6 @@ export function useBookmarkImport() { const { mutateAsync: addToList } = useAddBookmarkToList(); const { mutateAsync: updateTags } = useUpdateBookmarkTags(); - const { mutateAsync: parseAndCreateBookmark } = useMutation({ - mutationFn: async (toImport: { - bookmark: ParsedBookmark; - listIds: string[]; - }) => { - const bookmark = toImport.bookmark; - if (bookmark.content === undefined) { - throw new Error("Content is undefined"); - } - const created = await createBookmark({ - crawlPriority: "low", - title: bookmark.title, - createdAt: bookmark.addDate - ? new Date(bookmark.addDate * 1000) - : undefined, - note: bookmark.notes, - archived: bookmark.archived, - ...(bookmark.content.type === BookmarkTypes.LINK - ? { - type: BookmarkTypes.LINK, - url: bookmark.content.url, - } - : { - type: BookmarkTypes.TEXT, - text: bookmark.content.text, - }), - }); - - await Promise.all([ - ...toImport.listIds.map((listId) => - addToList({ - bookmarkId: created.id, - listId, - }), - ), - bookmark.tags.length > 0 - ? updateTags({ - bookmarkId: created.id, - attach: bookmark.tags.map((t) => ({ tagName: t })), - detach: [], - }) - : undefined, - ]); - return created; - }, - }); - const uploadBookmarkFileMutation = useMutation({ mutationFn: async ({ file, @@ -108,138 +50,87 @@ export function useBookmarkImport() { file: File; source: ImportSource; }) => { - if (source === "html") { - return await parseNetscapeBookmarkFile(file); - } else if (source === "pocket") { - return await parsePocketBookmarkFile(file); - } else if (source === "karakeep") { - return await parseKarakeepBookmarkFile(file); - } else if (source === "omnivore") { - return await parseOmnivoreBookmarkFile(file); - } else if (source === "linkwarden") { - return await parseLinkwardenBookmarkFile(file); - } else if (source === "tab-session-manager") { - return await parseTabSessionManagerStateFile(file); - } else { - throw new Error("Unknown source"); - } - }, - onSuccess: async (parsedBookmarks) => { - if (parsedBookmarks.length === 0) { - toast({ description: "No bookmarks found in the file." }); - return; - } - - const rootList = await createList({ - name: t("settings.import.imported_bookmarks"), - icon: "⬆️", - }); - - const finalBookmarksToImport = deduplicateBookmarks(parsedBookmarks); - - setImportProgress({ done: 0, total: finalBookmarksToImport.length }); - - const allRequiredPaths = new Set(); - for (const bookmark of finalBookmarksToImport) { - for (const path of bookmark.paths) { - if (path && path.length > 0) { - for (let i = 1; i <= path.length; i++) { - const subPath = path.slice(0, i); - const pathKey = subPath.join("/"); - allRequiredPaths.add(pathKey); - } - } - } - } - - const allRequiredPathsArray = Array.from(allRequiredPaths).sort( - (a, b) => a.split("/").length - b.split("/").length, - ); - - const pathMap: Record = {}; - pathMap[""] = rootList.id; - - for (const pathKey of allRequiredPathsArray) { - const parts = pathKey.split("/"); - const parentKey = parts.slice(0, -1).join("/"); - const parentId = pathMap[parentKey] || rootList.id; - - const folderName = parts[parts.length - 1]; - const folderList = await createList({ - name: folderName, - parentId: parentId, - icon: "📁", - }); - pathMap[pathKey] = folderList.id; - } - - const importPromises = finalBookmarksToImport.map( - (bookmark) => async () => { - const listIds = bookmark.paths.map( - (path) => pathMap[path.join("/")] || rootList.id, - ); - if (listIds.length === 0) { - listIds.push(rootList.id); - } - - try { - const created = await parseAndCreateBookmark({ - bookmark: bookmark, + const result = await importBookmarksFromFile( + { + file, + source, + rootListName: t("settings.import.imported_bookmarks"), + deps: { + createList: createList, + createBookmark: async (bookmark: ParsedBookmark) => { + if (bookmark.content === undefined) { + throw new Error("Content is undefined"); + } + const created = await createBookmark({ + crawlPriority: "low", + title: bookmark.title.substring(0, MAX_BOOKMARK_TITLE_LENGTH), + createdAt: bookmark.addDate + ? new Date(bookmark.addDate * 1000) + : undefined, + note: bookmark.notes, + archived: bookmark.archived, + ...(bookmark.content.type === BookmarkTypes.LINK + ? { + type: BookmarkTypes.LINK, + url: bookmark.content.url, + } + : { + type: BookmarkTypes.TEXT, + text: bookmark.content.text, + }), + }); + return created as { id: string; alreadyExists?: boolean }; + }, + addBookmarkToLists: async ({ + bookmarkId, listIds, - }); - - setImportProgress((prev) => { - const newDone = (prev?.done ?? 0) + 1; - return { - done: newDone, - total: finalBookmarksToImport.length, - }; - }); - return { status: "fulfilled" as const, value: created }; - } catch { - setImportProgress((prev) => { - const newDone = (prev?.done ?? 0) + 1; - return { - done: newDone, - total: finalBookmarksToImport.length, - }; - }); - return { status: "rejected" as const }; - } + }: { + bookmarkId: string; + listIds: string[]; + }) => { + await Promise.all( + listIds.map((listId) => + addToList({ + bookmarkId, + listId, + }), + ), + ); + }, + updateBookmarkTags: async ({ + bookmarkId, + tags, + }: { + bookmarkId: string; + tags: string[]; + }) => { + if (tags.length > 0) { + await updateTags({ + bookmarkId, + attach: tags.map((t) => ({ tagName: t })), + detach: [], + }); + } + }, + }, + onProgress: (done, total) => setImportProgress({ done, total }), }, + {}, ); - - const CONCURRENCY_LIMIT = 20; - const resultsPromises = limitConcurrency( - importPromises, - CONCURRENCY_LIMIT, - ); - - const results = await Promise.all(resultsPromises); - - let successes = 0; - let failures = 0; - let alreadyExisted = 0; - - for (const result of results) { - if (result.status === "fulfilled") { - if (result.value.alreadyExists) { - alreadyExisted++; - } else { - successes++; - } - } else { - failures++; - } + return result; + }, + onSuccess: async (result) => { + if (result.counts.total === 0) { + toast({ description: "No bookmarks found in the file." }); + return; } - + const { successes, failures, alreadyExisted } = result.counts; if (successes > 0 || alreadyExisted > 0) { toast({ description: `Imported ${successes} bookmarks and skipped ${alreadyExisted} bookmarks that already existed`, variant: "default", }); } - if (failures > 0) { toast({ description: `Failed to import ${failures} bookmarks. Check console for details.`, @@ -247,7 +138,8 @@ export function useBookmarkImport() { }); } - router.push(`/dashboard/lists/${rootList.id}`); + if (result.rootListId) + router.push(`/dashboard/lists/${result.rootListId}`); }, onError: (error) => { setImportProgress(null); diff --git a/apps/web/lib/importBookmarkParser.ts b/apps/web/lib/importBookmarkParser.ts deleted file mode 100644 index 44fe872c..00000000 --- a/apps/web/lib/importBookmarkParser.ts +++ /dev/null @@ -1,286 +0,0 @@ -// Copied from https://gist.github.com/devster31/4e8c6548fd16ffb75c02e6f24e27f9b9 -import * as cheerio from "cheerio"; -import { parse } from "csv-parse/sync"; -import { z } from "zod"; - -import { BookmarkTypes } from "@karakeep/shared/types/bookmarks"; - -import { zExportSchema } from "./exportBookmarks"; - -export interface ParsedBookmark { - title: string; - content?: - | { type: BookmarkTypes.LINK; url: string } - | { type: BookmarkTypes.TEXT; text: string }; - tags: string[]; - addDate?: number; - notes?: string; - archived?: boolean; - paths: string[][]; -} - -export async function parseNetscapeBookmarkFile( - file: File, -): Promise { - const textContent = await file.text(); - - if (!textContent.startsWith("")) { - throw Error("The uploaded html file does not seem to be a bookmark file"); - } - - const $ = cheerio.load(textContent); - - return $("a") - .map(function (_index, a) { - const $a = $(a); - const addDate = $a.attr("add_date"); - let tags: string[] = []; - - const tagsStr = $a.attr("tags"); - try { - tags = tagsStr && tagsStr.length > 0 ? tagsStr.split(",") : []; - } catch { - /* empty */ - } - const url = $a.attr("href"); - - // Build folder path by traversing up the hierarchy - const path: string[] = []; - let current = $a.parent(); - while (current && current.length > 0) { - const h3 = current.find("> h3").first(); - if (h3.length > 0) { - path.unshift(h3.text()); - } - current = current.parent(); - } - - return { - title: $a.text(), - content: url ? { type: BookmarkTypes.LINK as const, url } : undefined, - tags, - addDate: typeof addDate === "undefined" ? undefined : parseInt(addDate), - paths: [path], - }; - }) - .get(); -} - -export async function parsePocketBookmarkFile( - file: File, -): Promise { - const textContent = await file.text(); - - const records = parse(textContent, { - columns: true, - skip_empty_lines: true, - }) as { - title: string; - url: string; - time_added: string; - tags: string; - status?: string; - }[]; - - return records.map((record) => { - return { - title: record.title, - content: { type: BookmarkTypes.LINK as const, url: record.url }, - tags: record.tags.length > 0 ? record.tags.split("|") : [], - addDate: parseInt(record.time_added), - archived: record.status === "archive", - paths: [], // TODO - }; - }); -} - -export async function parseKarakeepBookmarkFile( - file: File, -): Promise { - const textContent = await file.text(); - - const parsed = zExportSchema.safeParse(JSON.parse(textContent)); - if (!parsed.success) { - throw new Error( - `The uploaded JSON file contains an invalid bookmark file: ${parsed.error.toString()}`, - ); - } - - return parsed.data.bookmarks.map((bookmark) => { - let content = undefined; - if (bookmark.content?.type == BookmarkTypes.LINK) { - content = { - type: BookmarkTypes.LINK as const, - url: bookmark.content.url, - }; - } else if (bookmark.content?.type == BookmarkTypes.TEXT) { - content = { - type: BookmarkTypes.TEXT as const, - text: bookmark.content.text, - }; - } - return { - title: bookmark.title ?? "", - content, - tags: bookmark.tags, - addDate: bookmark.createdAt, - notes: bookmark.note ?? undefined, - archived: bookmark.archived, - paths: [], // TODO - }; - }); -} - -export async function parseOmnivoreBookmarkFile( - file: File, -): Promise { - const textContent = await file.text(); - const zOmnivoreExportSchema = z.array( - z.object({ - title: z.string(), - url: z.string(), - labels: z.array(z.string()), - savedAt: z.coerce.date(), - state: z.string().optional(), - }), - ); - - const parsed = zOmnivoreExportSchema.safeParse(JSON.parse(textContent)); - if (!parsed.success) { - throw new Error( - `The uploaded JSON file contains an invalid omnivore bookmark file: ${parsed.error.toString()}`, - ); - } - - return parsed.data.map((bookmark) => { - return { - title: bookmark.title ?? "", - content: { type: BookmarkTypes.LINK as const, url: bookmark.url }, - tags: bookmark.labels, - addDate: bookmark.savedAt.getTime() / 1000, - archived: bookmark.state === "Archived", - paths: [], - }; - }); -} - -export async function parseLinkwardenBookmarkFile( - file: File, -): Promise { - const textContent = await file.text(); - const zLinkwardenExportSchema = z.object({ - collections: z.array( - z.object({ - links: z.array( - z.object({ - name: z.string(), - url: z.string(), - tags: z.array(z.object({ name: z.string() })), - createdAt: z.coerce.date(), - }), - ), - }), - ), - }); - - const parsed = zLinkwardenExportSchema.safeParse(JSON.parse(textContent)); - if (!parsed.success) { - throw new Error( - `The uploaded JSON file contains an invalid Linkwarden bookmark file: ${parsed.error.toString()}`, - ); - } - - return parsed.data.collections.flatMap((collection) => { - return collection.links.map((bookmark) => ({ - title: bookmark.name ?? "", - content: { type: BookmarkTypes.LINK as const, url: bookmark.url }, - tags: bookmark.tags.map((tag) => tag.name), - addDate: bookmark.createdAt.getTime() / 1000, - paths: [], // TODO - })); - }); -} - -export async function parseTabSessionManagerStateFile( - file: File, -): Promise { - const textContent = await file.text(); - - const zTab = z.object({ - url: z.string(), - title: z.string(), - lastAccessed: z.number(), - }); - - const zSession = z.object({ - windows: z.record(z.string(), z.record(z.string(), zTab)), - date: z.number(), - }); - - const zTabSessionManagerSchema = z.array(zSession); - - const parsed = zTabSessionManagerSchema.safeParse(JSON.parse(textContent)); - if (!parsed.success) { - throw new Error( - `The uploaded JSON file contains an invalid Tab Session Manager bookmark file: ${parsed.error.toString()}`, - ); - } - - // Get the object in data that has the most recent `date` - const { windows } = parsed.data.reduce((prev, curr) => - prev.date > curr.date ? prev : curr, - ); - - return Object.values(windows).flatMap((window) => - Object.values(window).map((tab) => ({ - title: tab.title, - content: { type: BookmarkTypes.LINK as const, url: tab.url }, - tags: [], - addDate: tab.lastAccessed, - paths: [], // Tab Session Manager doesn't have folders - })), - ); -} - -export function deduplicateBookmarks( - bookmarks: ParsedBookmark[], -): ParsedBookmark[] { - const deduplicatedBookmarksMap = new Map(); - const textBookmarks: ParsedBookmark[] = []; - - for (const bookmark of bookmarks) { - if (bookmark.content?.type === BookmarkTypes.LINK) { - const url = bookmark.content.url; - if (deduplicatedBookmarksMap.has(url)) { - const existing = deduplicatedBookmarksMap.get(url)!; - // Merge tags - existing.tags = [...new Set([...existing.tags, ...bookmark.tags])]; - // Merge paths - existing.paths = [...existing.paths, ...bookmark.paths]; - const existingDate = existing.addDate ?? Infinity; - const newDate = bookmark.addDate ?? Infinity; - if (newDate < existingDate) { - existing.addDate = bookmark.addDate; - } - // Append notes if both exist - if (existing.notes && bookmark.notes) { - existing.notes = `${existing.notes}\n---\n${bookmark.notes}`; - } else if (bookmark.notes) { - existing.notes = bookmark.notes; - } - // For archived status, prefer archived if either is archived - if (bookmark.archived === true) { - existing.archived = true; - } - // Title: keep existing one for simplicity - } else { - deduplicatedBookmarksMap.set(url, bookmark); - } - } else { - // Keep text bookmarks as they are (no URL to dedupe on) - textBookmarks.push(bookmark); - } - } - - return [...deduplicatedBookmarksMap.values(), ...textBookmarks]; -} diff --git a/packages/open-api/karakeep-openapi-spec.json b/packages/open-api/karakeep-openapi-spec.json index 2b5b9ed2..83a5b811 100644 --- a/packages/open-api/karakeep-openapi-spec.json +++ b/packages/open-api/karakeep-openapi-spec.json @@ -1813,12 +1813,12 @@ "name": { "type": "string", "minLength": 1, - "maxLength": 40 + "maxLength": 100 }, "description": { "type": "string", "minLength": 0, - "maxLength": 100 + "maxLength": 500 }, "icon": { "type": "string" @@ -2006,13 +2006,13 @@ "name": { "type": "string", "minLength": 1, - "maxLength": 40 + "maxLength": 100 }, "description": { "type": "string", "nullable": true, "minLength": 0, - "maxLength": 100 + "maxLength": 500 }, "icon": { "type": "string" diff --git a/packages/shared/import-export/exporters.ts b/packages/shared/import-export/exporters.ts new file mode 100644 index 00000000..967937a4 --- /dev/null +++ b/packages/shared/import-export/exporters.ts @@ -0,0 +1,111 @@ +import { z } from "zod"; + +import { BookmarkTypes, ZBookmark } from "../types/bookmarks"; + +export const zExportBookmarkSchema = z.object({ + createdAt: z.number(), + title: z.string().nullable(), + tags: z.array(z.string()), + content: z + .discriminatedUnion("type", [ + z.object({ + type: z.literal(BookmarkTypes.LINK), + url: z.string(), + }), + z.object({ + type: z.literal(BookmarkTypes.TEXT), + text: z.string(), + }), + ]) + .nullable(), + note: z.string().nullable(), + archived: z.boolean().optional().default(false), +}); + +export const zExportSchema = z.object({ + bookmarks: z.array(zExportBookmarkSchema), +}); + +export function toExportFormat( + bookmark: ZBookmark, +): z.infer { + let content = null; + switch (bookmark.content.type) { + case BookmarkTypes.LINK: { + content = { + type: bookmark.content.type, + url: bookmark.content.url, + }; + break; + } + case BookmarkTypes.TEXT: { + content = { + type: bookmark.content.type, + text: bookmark.content.text, + }; + break; + } + // Exclude asset types for now + } + return { + createdAt: Math.floor(bookmark.createdAt.getTime() / 1000), + title: + bookmark.title ?? + (bookmark.content.type === BookmarkTypes.LINK + ? (bookmark.content.title ?? null) + : null), + tags: bookmark.tags.map((t) => t.name), + content, + note: bookmark.note ?? null, + archived: bookmark.archived, + }; +} + +export function toNetscapeFormat(bookmarks: ZBookmark[]): string { + const header = ` + + +Bookmarks +

Bookmarks

+

`; + + const footer = `

`; + + const bookmarkEntries = bookmarks + .map((bookmark) => { + if (bookmark.content?.type !== BookmarkTypes.LINK) { + return ""; + } + const addDate = bookmark.createdAt + ? `ADD_DATE="${Math.floor(bookmark.createdAt.getTime() / 1000)}"` + : ""; + + const tagNames = bookmark.tags.map((t) => t.name).join(","); + const tags = tagNames.length > 0 ? `TAGS="${tagNames}"` : ""; + + const encodedUrl = encodeURI(bookmark.content.url); + const displayTitle = bookmark.title ?? bookmark.content.url; + const encodedTitle = escapeHtml(displayTitle); + + return `

${encodedTitle}`; + }) + .filter(Boolean) + .join("\n"); + + return `${header}\n${bookmarkEntries}\n${footer}`; +} + +function escapeHtml(input: string): string { + const escapeMap: Record = { + "&": "&", + "'": "'", + "`": "`", + '"': """, + "<": "<", + ">": ">", + }; + + return input.replace(/[&'`"<>]/g, (match) => escapeMap[match] || ""); +} diff --git a/packages/shared/import-export/importer.test.ts b/packages/shared/import-export/importer.test.ts new file mode 100644 index 00000000..2ea63846 --- /dev/null +++ b/packages/shared/import-export/importer.test.ts @@ -0,0 +1,392 @@ +import { describe, expect, it, vi } from "vitest"; + +import { importBookmarksFromFile, ParsedBookmark } from "."; + +const fakeFile = { + text: vi.fn().mockResolvedValue("fake file content"), +} as unknown as File; + +describe("importBookmarksFromFile", () => { + it("creates root list, folders and imports bookmarks with progress", async () => { + const parsers = { + pocket: vi.fn().mockReturnValue([ + { + title: "GitHub Repository", + content: { type: "link", url: "https://github.com/example/repo" }, + tags: ["dev", "github"], + addDate: 100, + paths: [["Development", "Projects"]], + }, + { + title: "My Notes", + content: { type: "text", text: "Important notes about the project" }, + tags: ["notes"], + addDate: 200, + paths: [["Personal"]], + notes: "Additional context", + archived: true, + }, + { + title: "Blog Post", + content: { type: "link", url: "https://example.com/blog" }, + tags: ["reading", "tech"], + addDate: 300, + paths: [["Reading", "Tech"]], + }, + { + title: "No Category Item", + content: { type: "link", url: "https://example.com/misc" }, + tags: [], + addDate: 400, + paths: [], + }, + { + title: "Duplicate URL Test", + content: { type: "link", url: "https://github.com/example/repo" }, + tags: ["duplicate"], + addDate: 50, // Earlier date + paths: [["Development", "Duplicates"]], + }, + ]), + }; + + const createdLists: { name: string; icon: string; parentId?: string }[] = + []; + const createList = vi.fn( + async (input: { name: string; icon: string; parentId?: string }) => { + createdLists.push(input); + return { + id: `${input.parentId ? input.parentId + "/" : ""}${input.name}`, + }; + }, + ); + + const createdBookmarks: ParsedBookmark[] = []; + const addedToLists: { bookmarkId: string; listIds: string[] }[] = []; + const updatedTags: { bookmarkId: string; tags: string[] }[] = []; + + const createBookmark = vi.fn(async (bookmark: ParsedBookmark) => { + createdBookmarks.push(bookmark); + return { + id: `bookmark-${createdBookmarks.length}`, + alreadyExists: false, + }; + }); + + const addBookmarkToLists = vi.fn( + async (input: { bookmarkId: string; listIds: string[] }) => { + addedToLists.push(input); + }, + ); + + const updateBookmarkTags = vi.fn( + async (input: { bookmarkId: string; tags: string[] }) => { + updatedTags.push(input); + }, + ); + + const progress: number[] = []; + const res = await importBookmarksFromFile( + { + file: fakeFile, + source: "pocket", + rootListName: "Imported", + deps: { + createList, + createBookmark, + addBookmarkToLists, + updateBookmarkTags, + }, + onProgress: (d, t) => progress.push(d / t), + }, + { parsers }, + ); + + expect(res.rootListId).toBe("Imported"); + expect(res.counts).toEqual({ + successes: 5, + failures: 0, + alreadyExisted: 0, + total: 5, // Using custom parser, no deduplication + }); + // Root + all unique folders from paths + expect(createdLists).toEqual([ + { name: "Imported", icon: "⬆️" }, + { name: "Development", parentId: "Imported", icon: "📁" }, + { name: "Personal", parentId: "Imported", icon: "📁" }, + { name: "Reading", parentId: "Imported", icon: "📁" }, + { name: "Projects", parentId: "Imported/Development", icon: "📁" }, + { name: "Tech", parentId: "Imported/Reading", icon: "📁" }, + { name: "Duplicates", parentId: "Imported/Development", icon: "📁" }, + ]); + // Verify we have 5 created bookmarks (no deduplication with custom parser) + expect(createdBookmarks).toHaveLength(5); + // Verify GitHub bookmark exists (will be two separate bookmarks since no deduplication) + const githubBookmarks = createdBookmarks.filter( + (bookmark) => + bookmark.content?.type === "link" && + bookmark.content.url === "https://github.com/example/repo", + ); + expect(githubBookmarks).toHaveLength(2); + // Verify text bookmark exists + const textBookmark = createdBookmarks.find( + (bookmark) => bookmark.content?.type === "text", + ); + expect(textBookmark).toBeDefined(); + expect(textBookmark!.archived).toBe(true); + expect(textBookmark!.notes).toBe("Additional context"); + // Verify bookmark with no path goes to root + const noCategoryBookmark = createdBookmarks.find( + (bookmark) => + bookmark.content?.type === "link" && + bookmark.content.url === "https://example.com/misc", + ); + expect(noCategoryBookmark).toBeDefined(); + // Find the corresponding list assignment for this bookmark + const noCategoryBookmarkId = `bookmark-${createdBookmarks.indexOf(noCategoryBookmark!) + 1}`; + const listAssignment = addedToLists.find( + (a) => a.bookmarkId === noCategoryBookmarkId, + ); + expect(listAssignment!.listIds).toEqual(["Imported"]); + + // Verify that tags were updated for bookmarks that have tags + expect(updatedTags.length).toBeGreaterThan(0); + expect(progress).toContain(0); + expect(progress.at(-1)).toBe(1); + }); + + it("returns zero counts and null rootListId when no bookmarks", async () => { + const parsers = { html: vi.fn().mockReturnValue([]) }; + const res = await importBookmarksFromFile( + { + file: fakeFile, + source: "html", + rootListName: "Imported", + deps: { + createList: vi.fn(), + createBookmark: vi.fn(), + addBookmarkToLists: vi.fn(), + updateBookmarkTags: vi.fn(), + }, + }, + { parsers }, + ); + expect(res).toEqual({ + counts: { successes: 0, failures: 0, alreadyExisted: 0, total: 0 }, + rootListId: null, + }); + }); + + it("continues import when individual bookmarks fail", async () => { + const parsers = { + pocket: vi.fn().mockReturnValue([ + { + title: "Success Bookmark 1", + content: { type: "link", url: "https://example.com/success1" }, + tags: ["success"], + addDate: 100, + paths: [["Success"]], + }, + { + title: "Failure Bookmark", + content: { type: "link", url: "https://example.com/failure" }, + tags: ["failure"], + addDate: 200, + paths: [["Failure"]], + }, + { + title: "Success Bookmark 2", + content: { type: "link", url: "https://example.com/success2" }, + tags: ["success"], + addDate: 300, + paths: [["Success"]], + }, + ]), + }; + + const createdLists: { name: string; icon: string; parentId?: string }[] = + []; + const createList = vi.fn( + async (input: { name: string; icon: string; parentId?: string }) => { + createdLists.push(input); + return { + id: `${input.parentId ? input.parentId + "/" : ""}${input.name}`, + }; + }, + ); + + const createdBookmarks: ParsedBookmark[] = []; + const addedToLists: { bookmarkId: string; listIds: string[] }[] = []; + const updatedTags: { bookmarkId: string; tags: string[] }[] = []; + + const createBookmark = vi.fn(async (bookmark: ParsedBookmark) => { + // Simulate failure for the "Failure Bookmark" + if (bookmark.title === "Failure Bookmark") { + throw new Error("Simulated bookmark creation failure"); + } + + createdBookmarks.push(bookmark); + return { + id: `bookmark-${createdBookmarks.length}`, + alreadyExists: false, + }; + }); + + const addBookmarkToLists = vi.fn( + async (input: { bookmarkId: string; listIds: string[] }) => { + addedToLists.push(input); + }, + ); + + const updateBookmarkTags = vi.fn( + async (input: { bookmarkId: string; tags: string[] }) => { + updatedTags.push(input); + }, + ); + + const progress: number[] = []; + const res = await importBookmarksFromFile( + { + file: fakeFile, + source: "pocket", + rootListName: "Imported", + deps: { + createList, + createBookmark, + addBookmarkToLists, + updateBookmarkTags, + }, + onProgress: (d, t) => progress.push(d / t), + }, + { parsers }, + ); + + // Should still create the root list + expect(res.rootListId).toBe("Imported"); + + // Should track both successes and failures + expect(res.counts).toEqual({ + successes: 2, // Two successful bookmarks + failures: 1, // One failed bookmark + alreadyExisted: 0, + total: 3, + }); + + // Should create folders for all bookmarks (including failed ones) + expect(createdLists).toEqual([ + { name: "Imported", icon: "⬆️" }, + { name: "Success", parentId: "Imported", icon: "📁" }, + { name: "Failure", parentId: "Imported", icon: "📁" }, + ]); + + // Only successful bookmarks should be created + expect(createdBookmarks).toHaveLength(2); + expect(createdBookmarks.map((b) => b.title)).toEqual([ + "Success Bookmark 1", + "Success Bookmark 2", + ]); + + // Only successful bookmarks should be added to lists and have tags updated + expect(addedToLists).toHaveLength(2); + expect(updatedTags).toHaveLength(2); + + // Progress should complete even with failures + expect(progress).toContain(0); + expect(progress.at(-1)).toBe(1); + }); + + it("handles failures in different stages of bookmark import", async () => { + const parsers = { + pocket: vi.fn().mockReturnValue([ + { + title: "Success Bookmark", + content: { type: "link", url: "https://example.com/success" }, + tags: ["success"], + addDate: 100, + paths: [["Success"]], + }, + { + title: "Fail at List Assignment", + content: { type: "link", url: "https://example.com/fail-list" }, + tags: ["fail"], + addDate: 200, + paths: [["Failure"]], + }, + { + title: "Fail at Tag Update", + content: { type: "link", url: "https://example.com/fail-tag" }, + tags: ["fail-tag"], + addDate: 300, + paths: [["Failure"]], + }, + ]), + }; + + const createList = vi.fn( + async (input: { name: string; icon: string; parentId?: string }) => { + return { + id: `${input.parentId ? input.parentId + "/" : ""}${input.name}`, + }; + }, + ); + + let bookmarkIdCounter = 1; + const createBookmark = vi.fn(async () => { + return { id: `bookmark-${bookmarkIdCounter++}`, alreadyExists: false }; + }); + + const addBookmarkToLists = vi.fn( + async (input: { bookmarkId: string; listIds: string[] }) => { + // Simulate failure for specific bookmark + if (input.bookmarkId === "bookmark-2") { + throw new Error("Failed to add bookmark to lists"); + } + }, + ); + + const updateBookmarkTags = vi.fn( + async (input: { bookmarkId: string; tags: string[] }) => { + // Simulate failure for specific bookmark + if (input.bookmarkId === "bookmark-3") { + throw new Error("Failed to update bookmark tags"); + } + }, + ); + + const progress: number[] = []; + const res = await importBookmarksFromFile( + { + file: fakeFile, + source: "pocket", + rootListName: "Imported", + deps: { + createList, + createBookmark, + addBookmarkToLists, + updateBookmarkTags, + }, + onProgress: (d, t) => progress.push(d / t), + }, + { parsers }, + ); + + expect(res.rootListId).toBe("Imported"); + + // All bookmarks are created successfully, but 2 fail in post-processing + expect(res.counts).toEqual({ + successes: 1, // Only one fully successful bookmark + failures: 2, // Two failed in post-processing steps + alreadyExisted: 0, + total: 3, + }); + + // All bookmarks should be created (failures happen after bookmark creation) + expect(createBookmark).toHaveBeenCalledTimes(3); + + // addBookmarkToLists should be called 3 times (but one fails) + expect(addBookmarkToLists).toHaveBeenCalledTimes(3); + + // updateBookmarkTags should be called 2 times (once fails at list assignment, one fails at tag update) + expect(updateBookmarkTags).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/shared/import-export/importer.ts b/packages/shared/import-export/importer.ts new file mode 100644 index 00000000..88c0c3bc --- /dev/null +++ b/packages/shared/import-export/importer.ts @@ -0,0 +1,158 @@ +import { limitConcurrency } from "../concurrency"; +import { MAX_LIST_NAME_LENGTH } from "../types/lists"; +import { ImportSource, ParsedBookmark, parseImportFile } from "./parsers"; + +export interface ImportCounts { + successes: number; + failures: number; + alreadyExisted: number; + total: number; +} + +export interface ImportDeps { + createList: (input: { + name: string; + icon: string; + parentId?: string; + }) => Promise<{ id: string }>; + createBookmark: ( + bookmark: ParsedBookmark, + ) => Promise<{ id: string; alreadyExists?: boolean }>; + addBookmarkToLists: (input: { + bookmarkId: string; + listIds: string[]; + }) => Promise; + updateBookmarkTags: (input: { + bookmarkId: string; + tags: string[]; + }) => Promise; +} + +export interface ImportOptions { + concurrencyLimit?: number; + parsers?: Partial< + Record ParsedBookmark[]> + >; +} + +export interface ImportResult { + counts: ImportCounts; + rootListId: string | null; +} + +export async function importBookmarksFromFile( + { + file, + source, + rootListName, + deps, + onProgress, + }: { + file: { text: () => Promise }; + source: ImportSource; + rootListName: string; + deps: ImportDeps; + onProgress?: (done: number, total: number) => void; + }, + options: ImportOptions = {}, +): Promise { + const { concurrencyLimit = 20, parsers } = options; + + const textContent = await file.text(); + const parsedBookmarks = parsers?.[source] + ? parsers[source]!(textContent) + : parseImportFile(source, textContent); + if (parsedBookmarks.length === 0) { + return { + counts: { successes: 0, failures: 0, alreadyExisted: 0, total: 0 }, + rootListId: null, + }; + } + + const rootList = await deps.createList({ name: rootListName, icon: "⬆️" }); + + onProgress?.(0, parsedBookmarks.length); + + const PATH_DELIMITER = "$$__$$"; + + // Build required paths + const allRequiredPaths = new Set(); + for (const bookmark of parsedBookmarks) { + for (const path of bookmark.paths) { + if (path && path.length > 0) { + for (let i = 1; i <= path.length; i++) { + const subPath = path.slice(0, i); + const pathKey = subPath.join(PATH_DELIMITER); + allRequiredPaths.add(pathKey); + } + } + } + } + + const allRequiredPathsArray = Array.from(allRequiredPaths).sort( + (a, b) => a.split(PATH_DELIMITER).length - b.split(PATH_DELIMITER).length, + ); + + const pathMap: Record = { "": rootList.id }; + + for (const pathKey of allRequiredPathsArray) { + const parts = pathKey.split(PATH_DELIMITER); + const parentKey = parts.slice(0, -1).join(PATH_DELIMITER); + const parentId = pathMap[parentKey] || rootList.id; + + const folderName = parts[parts.length - 1]; + const folderList = await deps.createList({ + name: folderName.substring(0, MAX_LIST_NAME_LENGTH), + parentId, + icon: "📁", + }); + pathMap[pathKey] = folderList.id; + } + + const importPromises = parsedBookmarks.map((bookmark) => async () => { + const listIds = bookmark.paths.map( + (path) => pathMap[path.join(PATH_DELIMITER)] || rootList.id, + ); + if (listIds.length === 0) listIds.push(rootList.id); + + const created = await deps.createBookmark(bookmark); + await deps.addBookmarkToLists({ bookmarkId: created.id, listIds }); + if (bookmark.tags && bookmark.tags.length > 0) { + await deps.updateBookmarkTags({ + bookmarkId: created.id, + tags: bookmark.tags, + }); + } + + return created; + }); + + const resultsPromises = limitConcurrency(importPromises, concurrencyLimit); + const results = await Promise.allSettled(resultsPromises); + + let successes = 0; + let failures = 0; + let alreadyExisted = 0; + + let done = 0; + for (const r of results) { + if (r.status === "fulfilled") { + if (r.value.alreadyExists) alreadyExisted++; + else successes++; + } else { + failures++; + } + done += 1; + onProgress?.(done, parsedBookmarks.length); + } + + return { + counts: { + successes, + failures, + alreadyExisted, + total: parsedBookmarks.length, + }, + rootListId: rootList.id, + }; +} diff --git a/packages/shared/import-export/index.ts b/packages/shared/import-export/index.ts new file mode 100644 index 00000000..2d720d0b --- /dev/null +++ b/packages/shared/import-export/index.ts @@ -0,0 +1,3 @@ +export * from "./exporters"; +export * from "./importer"; +export type { ImportSource, ParsedBookmark } from "./parsers"; diff --git a/packages/shared/import-export/parsers.ts b/packages/shared/import-export/parsers.ts new file mode 100644 index 00000000..c969c615 --- /dev/null +++ b/packages/shared/import-export/parsers.ts @@ -0,0 +1,300 @@ +// Copied from https://gist.github.com/devster31/4e8c6548fd16ffb75c02e6f24e27f9b9 + +import * as cheerio from "cheerio"; +import { parse } from "csv-parse/sync"; +import { z } from "zod"; + +import { BookmarkTypes } from "../types/bookmarks"; +import { zExportSchema } from "./exporters"; + +export type ImportSource = + | "html" + | "pocket" + | "omnivore" + | "karakeep" + | "linkwarden" + | "tab-session-manager"; + +export interface ParsedBookmark { + title: string; + content?: + | { type: BookmarkTypes.LINK; url: string } + | { type: BookmarkTypes.TEXT; text: string }; + tags: string[]; + addDate?: number; + notes?: string; + archived?: boolean; + paths: string[][]; +} + +function parseNetscapeBookmarkFile(textContent: string): ParsedBookmark[] { + if (!textContent.startsWith("")) { + throw Error("The uploaded html file does not seem to be a bookmark file"); + } + + const $ = cheerio.load(textContent); + + return $("a") + .map(function (_index, a) { + const $a = $(a); + const addDate = $a.attr("add_date"); + let tags: string[] = []; + + const tagsStr = $a.attr("tags"); + try { + tags = tagsStr && tagsStr.length > 0 ? tagsStr.split(",") : []; + } catch { + /* empty */ + } + const url = $a.attr("href"); + + // Build folder path by traversing up the hierarchy + const path: string[] = []; + let current = $a.parent(); + while (current && current.length > 0) { + const h3 = current.find("> h3").first(); + if (h3.length > 0) { + path.unshift(h3.text()); + } + current = current.parent(); + } + + return { + title: $a.text(), + content: url ? { type: BookmarkTypes.LINK as const, url } : undefined, + tags, + addDate: typeof addDate === "undefined" ? undefined : parseInt(addDate), + paths: [path], + }; + }) + .get(); +} + +function parsePocketBookmarkFile(textContent: string): ParsedBookmark[] { + const records = parse(textContent, { + columns: true, + skip_empty_lines: true, + }) as { + title: string; + url: string; + time_added: string; + tags: string; + status?: string; + }[]; + + return records.map((record) => { + return { + title: record.title, + content: { type: BookmarkTypes.LINK as const, url: record.url }, + tags: record.tags.length > 0 ? record.tags.split("|") : [], + addDate: parseInt(record.time_added), + archived: record.status === "archive", + paths: [], // TODO + }; + }); +} + +function parseKarakeepBookmarkFile(textContent: string): ParsedBookmark[] { + const parsed = zExportSchema.safeParse(JSON.parse(textContent)); + if (!parsed.success) { + throw new Error( + `The uploaded JSON file contains an invalid bookmark file: ${parsed.error.toString()}`, + ); + } + + return parsed.data.bookmarks.map((bookmark) => { + let content = undefined; + if (bookmark.content?.type == BookmarkTypes.LINK) { + content = { + type: BookmarkTypes.LINK as const, + url: bookmark.content.url, + }; + } else if (bookmark.content?.type == BookmarkTypes.TEXT) { + content = { + type: BookmarkTypes.TEXT as const, + text: bookmark.content.text, + }; + } + return { + title: bookmark.title ?? "", + content, + tags: bookmark.tags, + addDate: bookmark.createdAt, + notes: bookmark.note ?? undefined, + archived: bookmark.archived, + paths: [], // TODO + }; + }); +} + +function parseOmnivoreBookmarkFile(textContent: string): ParsedBookmark[] { + const zOmnivoreExportSchema = z.array( + z.object({ + title: z.string(), + url: z.string(), + labels: z.array(z.string()), + savedAt: z.coerce.date(), + state: z.string().optional(), + }), + ); + + const parsed = zOmnivoreExportSchema.safeParse(JSON.parse(textContent)); + if (!parsed.success) { + throw new Error( + `The uploaded JSON file contains an invalid omnivore bookmark file: ${parsed.error.toString()}`, + ); + } + + return parsed.data.map((bookmark) => { + return { + title: bookmark.title ?? "", + content: { type: BookmarkTypes.LINK as const, url: bookmark.url }, + tags: bookmark.labels, + addDate: bookmark.savedAt.getTime() / 1000, + archived: bookmark.state === "Archived", + paths: [], + }; + }); +} + +function parseLinkwardenBookmarkFile(textContent: string): ParsedBookmark[] { + const zLinkwardenExportSchema = z.object({ + collections: z.array( + z.object({ + links: z.array( + z.object({ + name: z.string(), + url: z.string(), + tags: z.array(z.object({ name: z.string() })), + createdAt: z.coerce.date(), + }), + ), + }), + ), + }); + + const parsed = zLinkwardenExportSchema.safeParse(JSON.parse(textContent)); + if (!parsed.success) { + throw new Error( + `The uploaded JSON file contains an invalid Linkwarden bookmark file: ${parsed.error.toString()}`, + ); + } + + return parsed.data.collections.flatMap((collection) => { + return collection.links.map((bookmark) => ({ + title: bookmark.name ?? "", + content: { type: BookmarkTypes.LINK as const, url: bookmark.url }, + tags: bookmark.tags.map((tag) => tag.name), + addDate: bookmark.createdAt.getTime() / 1000, + paths: [], // TODO + })); + }); +} + +function parseTabSessionManagerStateFile( + textContent: string, +): ParsedBookmark[] { + const zTab = z.object({ + url: z.string(), + title: z.string(), + lastAccessed: z.number(), + }); + + const zSession = z.object({ + windows: z.record(z.string(), z.record(z.string(), zTab)), + date: z.number(), + }); + + const zTabSessionManagerSchema = z.array(zSession); + + const parsed = zTabSessionManagerSchema.safeParse(JSON.parse(textContent)); + if (!parsed.success) { + throw new Error( + `The uploaded JSON file contains an invalid Tab Session Manager bookmark file: ${parsed.error.toString()}`, + ); + } + + // Get the object in data that has the most recent `date` + const { windows } = parsed.data.reduce((prev, curr) => + prev.date > curr.date ? prev : curr, + ); + + return Object.values(windows).flatMap((window) => + Object.values(window).map((tab) => ({ + title: tab.title, + content: { type: BookmarkTypes.LINK as const, url: tab.url }, + tags: [], + addDate: tab.lastAccessed, + paths: [], // Tab Session Manager doesn't have folders + })), + ); +} + +function deduplicateBookmarks(bookmarks: ParsedBookmark[]): ParsedBookmark[] { + const deduplicatedBookmarksMap = new Map(); + const textBookmarks: ParsedBookmark[] = []; + + for (const bookmark of bookmarks) { + if (bookmark.content?.type === BookmarkTypes.LINK) { + const url = bookmark.content.url; + if (deduplicatedBookmarksMap.has(url)) { + const existing = deduplicatedBookmarksMap.get(url)!; + // Merge tags + existing.tags = [...new Set([...existing.tags, ...bookmark.tags])]; + // Merge paths + existing.paths = [...existing.paths, ...bookmark.paths]; + const existingDate = existing.addDate ?? Infinity; + const newDate = bookmark.addDate ?? Infinity; + if (newDate < existingDate) { + existing.addDate = bookmark.addDate; + } + // Append notes if both exist + if (existing.notes && bookmark.notes) { + existing.notes = `${existing.notes}\n---\n${bookmark.notes}`; + } else if (bookmark.notes) { + existing.notes = bookmark.notes; + } + // For archived status, prefer archived if either is archived + if (bookmark.archived === true) { + existing.archived = true; + } + // Title: keep existing one for simplicity + } else { + deduplicatedBookmarksMap.set(url, bookmark); + } + } else { + // Keep text bookmarks as they are (no URL to dedupe on) + textBookmarks.push(bookmark); + } + } + + return [...deduplicatedBookmarksMap.values(), ...textBookmarks]; +} + +export function parseImportFile( + source: ImportSource, + textContent: string, +): ParsedBookmark[] { + let result: ParsedBookmark[]; + switch (source) { + case "html": + result = parseNetscapeBookmarkFile(textContent); + break; + case "pocket": + result = parsePocketBookmarkFile(textContent); + break; + case "karakeep": + result = parseKarakeepBookmarkFile(textContent); + break; + case "omnivore": + result = parseOmnivoreBookmarkFile(textContent); + break; + case "linkwarden": + result = parseLinkwardenBookmarkFile(textContent); + break; + case "tab-session-manager": + result = parseTabSessionManagerStateFile(textContent); + break; + } + return deduplicateBookmarks(result); +} diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts index f96cf0c5..a22e7632 100644 --- a/packages/shared/types/bookmarks.ts +++ b/packages/shared/types/bookmarks.ts @@ -3,7 +3,7 @@ import { z } from "zod"; import { zCursorV2 } from "./pagination"; import { zBookmarkTagSchema } from "./tags"; -const MAX_TITLE_LENGTH = 1000; +export const MAX_BOOKMARK_TITLE_LENGTH = 1000; export const enum BookmarkTypes { LINK = "link", @@ -133,7 +133,7 @@ export type ZBookmarkTypeAsset = z.infer; // POST /v1/bookmarks export const zNewBookmarkRequestSchema = z .object({ - title: z.string().max(MAX_TITLE_LENGTH).nullish(), + title: z.string().max(MAX_BOOKMARK_TITLE_LENGTH).nullish(), archived: z.boolean().optional(), favourited: z.boolean().optional(), note: z.string().optional(), @@ -202,7 +202,7 @@ export const zUpdateBookmarksRequestSchema = z.object({ favourited: z.boolean().optional(), summary: z.string().nullish(), note: z.string().optional(), - title: z.string().max(MAX_TITLE_LENGTH).nullish(), + title: z.string().max(MAX_BOOKMARK_TITLE_LENGTH).nullish(), createdAt: z.coerce.date().optional(), // Link specific fields (optional) url: z.string().url().optional(), diff --git a/packages/shared/types/lists.ts b/packages/shared/types/lists.ts index 51fb458c..59abb007 100644 --- a/packages/shared/types/lists.ts +++ b/packages/shared/types/lists.ts @@ -2,16 +2,25 @@ import { z } from "zod"; import { parseSearchQuery } from "../searchQueryParser"; +export const MAX_LIST_NAME_LENGTH = 100; +export const MAX_LIST_DESCRIPTION_LENGTH = 500; + export const zNewBookmarkListSchema = z .object({ name: z .string() .min(1, "List name can't be empty") - .max(40, "List name is at most 40 chars"), + .max( + MAX_LIST_NAME_LENGTH, + `List name is at most ${MAX_LIST_NAME_LENGTH} chars`, + ), description: z .string() .min(0, "Description can be empty") - .max(100, "Description can have at most 100 chars") + .max( + MAX_LIST_DESCRIPTION_LENGTH, + `Description can have at most ${MAX_LIST_DESCRIPTION_LENGTH} chars`, + ) .optional(), icon: z.string(), type: z.enum(["manual", "smart"]).optional().default("manual"), @@ -57,12 +66,18 @@ export const zEditBookmarkListSchema = z.object({ name: z .string() .min(1, "List name can't be empty") - .max(40, "List name is at most 40 chars") + .max( + MAX_LIST_NAME_LENGTH, + `List name is at most ${MAX_LIST_NAME_LENGTH} chars`, + ) .optional(), description: z .string() .min(0, "Description can be empty") - .max(100, "Description can have at most 100 chars") + .max( + MAX_LIST_DESCRIPTION_LENGTH, + `Description can have at most ${MAX_LIST_DESCRIPTION_LENGTH} chars`, + ) .nullish(), icon: z.string().optional(), parentId: z.string().nullish(), -- cgit v1.2.3-70-g09d2