From aecbe6ae8b3dbc7bcdcf33f1c8c086dafb77eb24 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sat, 30 Aug 2025 15:26:02 +0000 Subject: fix: handle list with slashes in their names and truncate long list names. fixes #1597 --- packages/shared/import-export/exporters.ts | 111 +++++++ packages/shared/import-export/importer.test.ts | 392 +++++++++++++++++++++++++ packages/shared/import-export/importer.ts | 158 ++++++++++ packages/shared/import-export/index.ts | 3 + packages/shared/import-export/parsers.ts | 300 +++++++++++++++++++ packages/shared/types/bookmarks.ts | 6 +- packages/shared/types/lists.ts | 23 +- 7 files changed, 986 insertions(+), 7 deletions(-) create mode 100644 packages/shared/import-export/exporters.ts create mode 100644 packages/shared/import-export/importer.test.ts create mode 100644 packages/shared/import-export/importer.ts create mode 100644 packages/shared/import-export/index.ts create mode 100644 packages/shared/import-export/parsers.ts (limited to 'packages/shared') diff --git a/packages/shared/import-export/exporters.ts b/packages/shared/import-export/exporters.ts new file mode 100644 index 00000000..967937a4 --- /dev/null +++ b/packages/shared/import-export/exporters.ts @@ -0,0 +1,111 @@ +import { z } from "zod"; + +import { BookmarkTypes, ZBookmark } from "../types/bookmarks"; + +export const zExportBookmarkSchema = z.object({ + createdAt: z.number(), + title: z.string().nullable(), + tags: z.array(z.string()), + content: z + .discriminatedUnion("type", [ + z.object({ + type: z.literal(BookmarkTypes.LINK), + url: z.string(), + }), + z.object({ + type: z.literal(BookmarkTypes.TEXT), + text: z.string(), + }), + ]) + .nullable(), + note: z.string().nullable(), + archived: z.boolean().optional().default(false), +}); + +export const zExportSchema = z.object({ + bookmarks: z.array(zExportBookmarkSchema), +}); + +export function toExportFormat( + bookmark: ZBookmark, +): z.infer { + let content = null; + switch (bookmark.content.type) { + case BookmarkTypes.LINK: { + content = { + type: bookmark.content.type, + url: bookmark.content.url, + }; + break; + } + case BookmarkTypes.TEXT: { + content = { + type: bookmark.content.type, + text: bookmark.content.text, + }; + break; + } + // Exclude asset types for now + } + return { + createdAt: Math.floor(bookmark.createdAt.getTime() / 1000), + title: + bookmark.title ?? + (bookmark.content.type === BookmarkTypes.LINK + ? (bookmark.content.title ?? null) + : null), + tags: bookmark.tags.map((t) => t.name), + content, + note: bookmark.note ?? null, + archived: bookmark.archived, + }; +} + +export function toNetscapeFormat(bookmarks: ZBookmark[]): string { + const header = ` + + +Bookmarks +

Bookmarks

+

`; + + const footer = `

`; + + const bookmarkEntries = bookmarks + .map((bookmark) => { + if (bookmark.content?.type !== BookmarkTypes.LINK) { + return ""; + } + const addDate = bookmark.createdAt + ? `ADD_DATE="${Math.floor(bookmark.createdAt.getTime() / 1000)}"` + : ""; + + const tagNames = bookmark.tags.map((t) => t.name).join(","); + const tags = tagNames.length > 0 ? `TAGS="${tagNames}"` : ""; + + const encodedUrl = encodeURI(bookmark.content.url); + const displayTitle = bookmark.title ?? bookmark.content.url; + const encodedTitle = escapeHtml(displayTitle); + + return `

${encodedTitle}`; + }) + .filter(Boolean) + .join("\n"); + + return `${header}\n${bookmarkEntries}\n${footer}`; +} + +function escapeHtml(input: string): string { + const escapeMap: Record = { + "&": "&", + "'": "'", + "`": "`", + '"': """, + "<": "<", + ">": ">", + }; + + return input.replace(/[&'`"<>]/g, (match) => escapeMap[match] || ""); +} diff --git a/packages/shared/import-export/importer.test.ts b/packages/shared/import-export/importer.test.ts new file mode 100644 index 00000000..2ea63846 --- /dev/null +++ b/packages/shared/import-export/importer.test.ts @@ -0,0 +1,392 @@ +import { describe, expect, it, vi } from "vitest"; + +import { importBookmarksFromFile, ParsedBookmark } from "."; + +const fakeFile = { + text: vi.fn().mockResolvedValue("fake file content"), +} as unknown as File; + +describe("importBookmarksFromFile", () => { + it("creates root list, folders and imports bookmarks with progress", async () => { + const parsers = { + pocket: vi.fn().mockReturnValue([ + { + title: "GitHub Repository", + content: { type: "link", url: "https://github.com/example/repo" }, + tags: ["dev", "github"], + addDate: 100, + paths: [["Development", "Projects"]], + }, + { + title: "My Notes", + content: { type: "text", text: "Important notes about the project" }, + tags: ["notes"], + addDate: 200, + paths: [["Personal"]], + notes: "Additional context", + archived: true, + }, + { + title: "Blog Post", + content: { type: "link", url: "https://example.com/blog" }, + tags: ["reading", "tech"], + addDate: 300, + paths: [["Reading", "Tech"]], + }, + { + title: "No Category Item", + content: { type: "link", url: "https://example.com/misc" }, + tags: [], + addDate: 400, + paths: [], + }, + { + title: "Duplicate URL Test", + content: { type: "link", url: "https://github.com/example/repo" }, + tags: ["duplicate"], + addDate: 50, // Earlier date + paths: [["Development", "Duplicates"]], + }, + ]), + }; + + const createdLists: { name: string; icon: string; parentId?: string }[] = + []; + const createList = vi.fn( + async (input: { name: string; icon: string; parentId?: string }) => { + createdLists.push(input); + return { + id: `${input.parentId ? input.parentId + "/" : ""}${input.name}`, + }; + }, + ); + + const createdBookmarks: ParsedBookmark[] = []; + const addedToLists: { bookmarkId: string; listIds: string[] }[] = []; + const updatedTags: { bookmarkId: string; tags: string[] }[] = []; + + const createBookmark = vi.fn(async (bookmark: ParsedBookmark) => { + createdBookmarks.push(bookmark); + return { + id: `bookmark-${createdBookmarks.length}`, + alreadyExists: false, + }; + }); + + const addBookmarkToLists = vi.fn( + async (input: { bookmarkId: string; listIds: string[] }) => { + addedToLists.push(input); + }, + ); + + const updateBookmarkTags = vi.fn( + async (input: { bookmarkId: string; tags: string[] }) => { + updatedTags.push(input); + }, + ); + + const progress: number[] = []; + const res = await importBookmarksFromFile( + { + file: fakeFile, + source: "pocket", + rootListName: "Imported", + deps: { + createList, + createBookmark, + addBookmarkToLists, + updateBookmarkTags, + }, + onProgress: (d, t) => progress.push(d / t), + }, + { parsers }, + ); + + expect(res.rootListId).toBe("Imported"); + expect(res.counts).toEqual({ + successes: 5, + failures: 0, + alreadyExisted: 0, + total: 5, // Using custom parser, no deduplication + }); + // Root + all unique folders from paths + expect(createdLists).toEqual([ + { name: "Imported", icon: "⬆️" }, + { name: "Development", parentId: "Imported", icon: "📁" }, + { name: "Personal", parentId: "Imported", icon: "📁" }, + { name: "Reading", parentId: "Imported", icon: "📁" }, + { name: "Projects", parentId: "Imported/Development", icon: "📁" }, + { name: "Tech", parentId: "Imported/Reading", icon: "📁" }, + { name: "Duplicates", parentId: "Imported/Development", icon: "📁" }, + ]); + // Verify we have 5 created bookmarks (no deduplication with custom parser) + expect(createdBookmarks).toHaveLength(5); + // Verify GitHub bookmark exists (will be two separate bookmarks since no deduplication) + const githubBookmarks = createdBookmarks.filter( + (bookmark) => + bookmark.content?.type === "link" && + bookmark.content.url === "https://github.com/example/repo", + ); + expect(githubBookmarks).toHaveLength(2); + // Verify text bookmark exists + const textBookmark = createdBookmarks.find( + (bookmark) => bookmark.content?.type === "text", + ); + expect(textBookmark).toBeDefined(); + expect(textBookmark!.archived).toBe(true); + expect(textBookmark!.notes).toBe("Additional context"); + // Verify bookmark with no path goes to root + const noCategoryBookmark = createdBookmarks.find( + (bookmark) => + bookmark.content?.type === "link" && + bookmark.content.url === "https://example.com/misc", + ); + expect(noCategoryBookmark).toBeDefined(); + // Find the corresponding list assignment for this bookmark + const noCategoryBookmarkId = `bookmark-${createdBookmarks.indexOf(noCategoryBookmark!) + 1}`; + const listAssignment = addedToLists.find( + (a) => a.bookmarkId === noCategoryBookmarkId, + ); + expect(listAssignment!.listIds).toEqual(["Imported"]); + + // Verify that tags were updated for bookmarks that have tags + expect(updatedTags.length).toBeGreaterThan(0); + expect(progress).toContain(0); + expect(progress.at(-1)).toBe(1); + }); + + it("returns zero counts and null rootListId when no bookmarks", async () => { + const parsers = { html: vi.fn().mockReturnValue([]) }; + const res = await importBookmarksFromFile( + { + file: fakeFile, + source: "html", + rootListName: "Imported", + deps: { + createList: vi.fn(), + createBookmark: vi.fn(), + addBookmarkToLists: vi.fn(), + updateBookmarkTags: vi.fn(), + }, + }, + { parsers }, + ); + expect(res).toEqual({ + counts: { successes: 0, failures: 0, alreadyExisted: 0, total: 0 }, + rootListId: null, + }); + }); + + it("continues import when individual bookmarks fail", async () => { + const parsers = { + pocket: vi.fn().mockReturnValue([ + { + title: "Success Bookmark 1", + content: { type: "link", url: "https://example.com/success1" }, + tags: ["success"], + addDate: 100, + paths: [["Success"]], + }, + { + title: "Failure Bookmark", + content: { type: "link", url: "https://example.com/failure" }, + tags: ["failure"], + addDate: 200, + paths: [["Failure"]], + }, + { + title: "Success Bookmark 2", + content: { type: "link", url: "https://example.com/success2" }, + tags: ["success"], + addDate: 300, + paths: [["Success"]], + }, + ]), + }; + + const createdLists: { name: string; icon: string; parentId?: string }[] = + []; + const createList = vi.fn( + async (input: { name: string; icon: string; parentId?: string }) => { + createdLists.push(input); + return { + id: `${input.parentId ? input.parentId + "/" : ""}${input.name}`, + }; + }, + ); + + const createdBookmarks: ParsedBookmark[] = []; + const addedToLists: { bookmarkId: string; listIds: string[] }[] = []; + const updatedTags: { bookmarkId: string; tags: string[] }[] = []; + + const createBookmark = vi.fn(async (bookmark: ParsedBookmark) => { + // Simulate failure for the "Failure Bookmark" + if (bookmark.title === "Failure Bookmark") { + throw new Error("Simulated bookmark creation failure"); + } + + createdBookmarks.push(bookmark); + return { + id: `bookmark-${createdBookmarks.length}`, + alreadyExists: false, + }; + }); + + const addBookmarkToLists = vi.fn( + async (input: { bookmarkId: string; listIds: string[] }) => { + addedToLists.push(input); + }, + ); + + const updateBookmarkTags = vi.fn( + async (input: { bookmarkId: string; tags: string[] }) => { + updatedTags.push(input); + }, + ); + + const progress: number[] = []; + const res = await importBookmarksFromFile( + { + file: fakeFile, + source: "pocket", + rootListName: "Imported", + deps: { + createList, + createBookmark, + addBookmarkToLists, + updateBookmarkTags, + }, + onProgress: (d, t) => progress.push(d / t), + }, + { parsers }, + ); + + // Should still create the root list + expect(res.rootListId).toBe("Imported"); + + // Should track both successes and failures + expect(res.counts).toEqual({ + successes: 2, // Two successful bookmarks + failures: 1, // One failed bookmark + alreadyExisted: 0, + total: 3, + }); + + // Should create folders for all bookmarks (including failed ones) + expect(createdLists).toEqual([ + { name: "Imported", icon: "⬆️" }, + { name: "Success", parentId: "Imported", icon: "📁" }, + { name: "Failure", parentId: "Imported", icon: "📁" }, + ]); + + // Only successful bookmarks should be created + expect(createdBookmarks).toHaveLength(2); + expect(createdBookmarks.map((b) => b.title)).toEqual([ + "Success Bookmark 1", + "Success Bookmark 2", + ]); + + // Only successful bookmarks should be added to lists and have tags updated + expect(addedToLists).toHaveLength(2); + expect(updatedTags).toHaveLength(2); + + // Progress should complete even with failures + expect(progress).toContain(0); + expect(progress.at(-1)).toBe(1); + }); + + it("handles failures in different stages of bookmark import", async () => { + const parsers = { + pocket: vi.fn().mockReturnValue([ + { + title: "Success Bookmark", + content: { type: "link", url: "https://example.com/success" }, + tags: ["success"], + addDate: 100, + paths: [["Success"]], + }, + { + title: "Fail at List Assignment", + content: { type: "link", url: "https://example.com/fail-list" }, + tags: ["fail"], + addDate: 200, + paths: [["Failure"]], + }, + { + title: "Fail at Tag Update", + content: { type: "link", url: "https://example.com/fail-tag" }, + tags: ["fail-tag"], + addDate: 300, + paths: [["Failure"]], + }, + ]), + }; + + const createList = vi.fn( + async (input: { name: string; icon: string; parentId?: string }) => { + return { + id: `${input.parentId ? input.parentId + "/" : ""}${input.name}`, + }; + }, + ); + + let bookmarkIdCounter = 1; + const createBookmark = vi.fn(async () => { + return { id: `bookmark-${bookmarkIdCounter++}`, alreadyExists: false }; + }); + + const addBookmarkToLists = vi.fn( + async (input: { bookmarkId: string; listIds: string[] }) => { + // Simulate failure for specific bookmark + if (input.bookmarkId === "bookmark-2") { + throw new Error("Failed to add bookmark to lists"); + } + }, + ); + + const updateBookmarkTags = vi.fn( + async (input: { bookmarkId: string; tags: string[] }) => { + // Simulate failure for specific bookmark + if (input.bookmarkId === "bookmark-3") { + throw new Error("Failed to update bookmark tags"); + } + }, + ); + + const progress: number[] = []; + const res = await importBookmarksFromFile( + { + file: fakeFile, + source: "pocket", + rootListName: "Imported", + deps: { + createList, + createBookmark, + addBookmarkToLists, + updateBookmarkTags, + }, + onProgress: (d, t) => progress.push(d / t), + }, + { parsers }, + ); + + expect(res.rootListId).toBe("Imported"); + + // All bookmarks are created successfully, but 2 fail in post-processing + expect(res.counts).toEqual({ + successes: 1, // Only one fully successful bookmark + failures: 2, // Two failed in post-processing steps + alreadyExisted: 0, + total: 3, + }); + + // All bookmarks should be created (failures happen after bookmark creation) + expect(createBookmark).toHaveBeenCalledTimes(3); + + // addBookmarkToLists should be called 3 times (but one fails) + expect(addBookmarkToLists).toHaveBeenCalledTimes(3); + + // updateBookmarkTags should be called 2 times (once fails at list assignment, one fails at tag update) + expect(updateBookmarkTags).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/shared/import-export/importer.ts b/packages/shared/import-export/importer.ts new file mode 100644 index 00000000..88c0c3bc --- /dev/null +++ b/packages/shared/import-export/importer.ts @@ -0,0 +1,158 @@ +import { limitConcurrency } from "../concurrency"; +import { MAX_LIST_NAME_LENGTH } from "../types/lists"; +import { ImportSource, ParsedBookmark, parseImportFile } from "./parsers"; + +export interface ImportCounts { + successes: number; + failures: number; + alreadyExisted: number; + total: number; +} + +export interface ImportDeps { + createList: (input: { + name: string; + icon: string; + parentId?: string; + }) => Promise<{ id: string }>; + createBookmark: ( + bookmark: ParsedBookmark, + ) => Promise<{ id: string; alreadyExists?: boolean }>; + addBookmarkToLists: (input: { + bookmarkId: string; + listIds: string[]; + }) => Promise; + updateBookmarkTags: (input: { + bookmarkId: string; + tags: string[]; + }) => Promise; +} + +export interface ImportOptions { + concurrencyLimit?: number; + parsers?: Partial< + Record ParsedBookmark[]> + >; +} + +export interface ImportResult { + counts: ImportCounts; + rootListId: string | null; +} + +export async function importBookmarksFromFile( + { + file, + source, + rootListName, + deps, + onProgress, + }: { + file: { text: () => Promise }; + source: ImportSource; + rootListName: string; + deps: ImportDeps; + onProgress?: (done: number, total: number) => void; + }, + options: ImportOptions = {}, +): Promise { + const { concurrencyLimit = 20, parsers } = options; + + const textContent = await file.text(); + const parsedBookmarks = parsers?.[source] + ? parsers[source]!(textContent) + : parseImportFile(source, textContent); + if (parsedBookmarks.length === 0) { + return { + counts: { successes: 0, failures: 0, alreadyExisted: 0, total: 0 }, + rootListId: null, + }; + } + + const rootList = await deps.createList({ name: rootListName, icon: "⬆️" }); + + onProgress?.(0, parsedBookmarks.length); + + const PATH_DELIMITER = "$$__$$"; + + // Build required paths + const allRequiredPaths = new Set(); + for (const bookmark of parsedBookmarks) { + for (const path of bookmark.paths) { + if (path && path.length > 0) { + for (let i = 1; i <= path.length; i++) { + const subPath = path.slice(0, i); + const pathKey = subPath.join(PATH_DELIMITER); + allRequiredPaths.add(pathKey); + } + } + } + } + + const allRequiredPathsArray = Array.from(allRequiredPaths).sort( + (a, b) => a.split(PATH_DELIMITER).length - b.split(PATH_DELIMITER).length, + ); + + const pathMap: Record = { "": rootList.id }; + + for (const pathKey of allRequiredPathsArray) { + const parts = pathKey.split(PATH_DELIMITER); + const parentKey = parts.slice(0, -1).join(PATH_DELIMITER); + const parentId = pathMap[parentKey] || rootList.id; + + const folderName = parts[parts.length - 1]; + const folderList = await deps.createList({ + name: folderName.substring(0, MAX_LIST_NAME_LENGTH), + parentId, + icon: "📁", + }); + pathMap[pathKey] = folderList.id; + } + + const importPromises = parsedBookmarks.map((bookmark) => async () => { + const listIds = bookmark.paths.map( + (path) => pathMap[path.join(PATH_DELIMITER)] || rootList.id, + ); + if (listIds.length === 0) listIds.push(rootList.id); + + const created = await deps.createBookmark(bookmark); + await deps.addBookmarkToLists({ bookmarkId: created.id, listIds }); + if (bookmark.tags && bookmark.tags.length > 0) { + await deps.updateBookmarkTags({ + bookmarkId: created.id, + tags: bookmark.tags, + }); + } + + return created; + }); + + const resultsPromises = limitConcurrency(importPromises, concurrencyLimit); + const results = await Promise.allSettled(resultsPromises); + + let successes = 0; + let failures = 0; + let alreadyExisted = 0; + + let done = 0; + for (const r of results) { + if (r.status === "fulfilled") { + if (r.value.alreadyExists) alreadyExisted++; + else successes++; + } else { + failures++; + } + done += 1; + onProgress?.(done, parsedBookmarks.length); + } + + return { + counts: { + successes, + failures, + alreadyExisted, + total: parsedBookmarks.length, + }, + rootListId: rootList.id, + }; +} diff --git a/packages/shared/import-export/index.ts b/packages/shared/import-export/index.ts new file mode 100644 index 00000000..2d720d0b --- /dev/null +++ b/packages/shared/import-export/index.ts @@ -0,0 +1,3 @@ +export * from "./exporters"; +export * from "./importer"; +export type { ImportSource, ParsedBookmark } from "./parsers"; diff --git a/packages/shared/import-export/parsers.ts b/packages/shared/import-export/parsers.ts new file mode 100644 index 00000000..c969c615 --- /dev/null +++ b/packages/shared/import-export/parsers.ts @@ -0,0 +1,300 @@ +// Copied from https://gist.github.com/devster31/4e8c6548fd16ffb75c02e6f24e27f9b9 + +import * as cheerio from "cheerio"; +import { parse } from "csv-parse/sync"; +import { z } from "zod"; + +import { BookmarkTypes } from "../types/bookmarks"; +import { zExportSchema } from "./exporters"; + +export type ImportSource = + | "html" + | "pocket" + | "omnivore" + | "karakeep" + | "linkwarden" + | "tab-session-manager"; + +export interface ParsedBookmark { + title: string; + content?: + | { type: BookmarkTypes.LINK; url: string } + | { type: BookmarkTypes.TEXT; text: string }; + tags: string[]; + addDate?: number; + notes?: string; + archived?: boolean; + paths: string[][]; +} + +function parseNetscapeBookmarkFile(textContent: string): ParsedBookmark[] { + if (!textContent.startsWith("")) { + throw Error("The uploaded html file does not seem to be a bookmark file"); + } + + const $ = cheerio.load(textContent); + + return $("a") + .map(function (_index, a) { + const $a = $(a); + const addDate = $a.attr("add_date"); + let tags: string[] = []; + + const tagsStr = $a.attr("tags"); + try { + tags = tagsStr && tagsStr.length > 0 ? tagsStr.split(",") : []; + } catch { + /* empty */ + } + const url = $a.attr("href"); + + // Build folder path by traversing up the hierarchy + const path: string[] = []; + let current = $a.parent(); + while (current && current.length > 0) { + const h3 = current.find("> h3").first(); + if (h3.length > 0) { + path.unshift(h3.text()); + } + current = current.parent(); + } + + return { + title: $a.text(), + content: url ? { type: BookmarkTypes.LINK as const, url } : undefined, + tags, + addDate: typeof addDate === "undefined" ? undefined : parseInt(addDate), + paths: [path], + }; + }) + .get(); +} + +function parsePocketBookmarkFile(textContent: string): ParsedBookmark[] { + const records = parse(textContent, { + columns: true, + skip_empty_lines: true, + }) as { + title: string; + url: string; + time_added: string; + tags: string; + status?: string; + }[]; + + return records.map((record) => { + return { + title: record.title, + content: { type: BookmarkTypes.LINK as const, url: record.url }, + tags: record.tags.length > 0 ? record.tags.split("|") : [], + addDate: parseInt(record.time_added), + archived: record.status === "archive", + paths: [], // TODO + }; + }); +} + +function parseKarakeepBookmarkFile(textContent: string): ParsedBookmark[] { + const parsed = zExportSchema.safeParse(JSON.parse(textContent)); + if (!parsed.success) { + throw new Error( + `The uploaded JSON file contains an invalid bookmark file: ${parsed.error.toString()}`, + ); + } + + return parsed.data.bookmarks.map((bookmark) => { + let content = undefined; + if (bookmark.content?.type == BookmarkTypes.LINK) { + content = { + type: BookmarkTypes.LINK as const, + url: bookmark.content.url, + }; + } else if (bookmark.content?.type == BookmarkTypes.TEXT) { + content = { + type: BookmarkTypes.TEXT as const, + text: bookmark.content.text, + }; + } + return { + title: bookmark.title ?? "", + content, + tags: bookmark.tags, + addDate: bookmark.createdAt, + notes: bookmark.note ?? undefined, + archived: bookmark.archived, + paths: [], // TODO + }; + }); +} + +function parseOmnivoreBookmarkFile(textContent: string): ParsedBookmark[] { + const zOmnivoreExportSchema = z.array( + z.object({ + title: z.string(), + url: z.string(), + labels: z.array(z.string()), + savedAt: z.coerce.date(), + state: z.string().optional(), + }), + ); + + const parsed = zOmnivoreExportSchema.safeParse(JSON.parse(textContent)); + if (!parsed.success) { + throw new Error( + `The uploaded JSON file contains an invalid omnivore bookmark file: ${parsed.error.toString()}`, + ); + } + + return parsed.data.map((bookmark) => { + return { + title: bookmark.title ?? "", + content: { type: BookmarkTypes.LINK as const, url: bookmark.url }, + tags: bookmark.labels, + addDate: bookmark.savedAt.getTime() / 1000, + archived: bookmark.state === "Archived", + paths: [], + }; + }); +} + +function parseLinkwardenBookmarkFile(textContent: string): ParsedBookmark[] { + const zLinkwardenExportSchema = z.object({ + collections: z.array( + z.object({ + links: z.array( + z.object({ + name: z.string(), + url: z.string(), + tags: z.array(z.object({ name: z.string() })), + createdAt: z.coerce.date(), + }), + ), + }), + ), + }); + + const parsed = zLinkwardenExportSchema.safeParse(JSON.parse(textContent)); + if (!parsed.success) { + throw new Error( + `The uploaded JSON file contains an invalid Linkwarden bookmark file: ${parsed.error.toString()}`, + ); + } + + return parsed.data.collections.flatMap((collection) => { + return collection.links.map((bookmark) => ({ + title: bookmark.name ?? "", + content: { type: BookmarkTypes.LINK as const, url: bookmark.url }, + tags: bookmark.tags.map((tag) => tag.name), + addDate: bookmark.createdAt.getTime() / 1000, + paths: [], // TODO + })); + }); +} + +function parseTabSessionManagerStateFile( + textContent: string, +): ParsedBookmark[] { + const zTab = z.object({ + url: z.string(), + title: z.string(), + lastAccessed: z.number(), + }); + + const zSession = z.object({ + windows: z.record(z.string(), z.record(z.string(), zTab)), + date: z.number(), + }); + + const zTabSessionManagerSchema = z.array(zSession); + + const parsed = zTabSessionManagerSchema.safeParse(JSON.parse(textContent)); + if (!parsed.success) { + throw new Error( + `The uploaded JSON file contains an invalid Tab Session Manager bookmark file: ${parsed.error.toString()}`, + ); + } + + // Get the object in data that has the most recent `date` + const { windows } = parsed.data.reduce((prev, curr) => + prev.date > curr.date ? prev : curr, + ); + + return Object.values(windows).flatMap((window) => + Object.values(window).map((tab) => ({ + title: tab.title, + content: { type: BookmarkTypes.LINK as const, url: tab.url }, + tags: [], + addDate: tab.lastAccessed, + paths: [], // Tab Session Manager doesn't have folders + })), + ); +} + +function deduplicateBookmarks(bookmarks: ParsedBookmark[]): ParsedBookmark[] { + const deduplicatedBookmarksMap = new Map(); + const textBookmarks: ParsedBookmark[] = []; + + for (const bookmark of bookmarks) { + if (bookmark.content?.type === BookmarkTypes.LINK) { + const url = bookmark.content.url; + if (deduplicatedBookmarksMap.has(url)) { + const existing = deduplicatedBookmarksMap.get(url)!; + // Merge tags + existing.tags = [...new Set([...existing.tags, ...bookmark.tags])]; + // Merge paths + existing.paths = [...existing.paths, ...bookmark.paths]; + const existingDate = existing.addDate ?? Infinity; + const newDate = bookmark.addDate ?? Infinity; + if (newDate < existingDate) { + existing.addDate = bookmark.addDate; + } + // Append notes if both exist + if (existing.notes && bookmark.notes) { + existing.notes = `${existing.notes}\n---\n${bookmark.notes}`; + } else if (bookmark.notes) { + existing.notes = bookmark.notes; + } + // For archived status, prefer archived if either is archived + if (bookmark.archived === true) { + existing.archived = true; + } + // Title: keep existing one for simplicity + } else { + deduplicatedBookmarksMap.set(url, bookmark); + } + } else { + // Keep text bookmarks as they are (no URL to dedupe on) + textBookmarks.push(bookmark); + } + } + + return [...deduplicatedBookmarksMap.values(), ...textBookmarks]; +} + +export function parseImportFile( + source: ImportSource, + textContent: string, +): ParsedBookmark[] { + let result: ParsedBookmark[]; + switch (source) { + case "html": + result = parseNetscapeBookmarkFile(textContent); + break; + case "pocket": + result = parsePocketBookmarkFile(textContent); + break; + case "karakeep": + result = parseKarakeepBookmarkFile(textContent); + break; + case "omnivore": + result = parseOmnivoreBookmarkFile(textContent); + break; + case "linkwarden": + result = parseLinkwardenBookmarkFile(textContent); + break; + case "tab-session-manager": + result = parseTabSessionManagerStateFile(textContent); + break; + } + return deduplicateBookmarks(result); +} diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts index f96cf0c5..a22e7632 100644 --- a/packages/shared/types/bookmarks.ts +++ b/packages/shared/types/bookmarks.ts @@ -3,7 +3,7 @@ import { z } from "zod"; import { zCursorV2 } from "./pagination"; import { zBookmarkTagSchema } from "./tags"; -const MAX_TITLE_LENGTH = 1000; +export const MAX_BOOKMARK_TITLE_LENGTH = 1000; export const enum BookmarkTypes { LINK = "link", @@ -133,7 +133,7 @@ export type ZBookmarkTypeAsset = z.infer; // POST /v1/bookmarks export const zNewBookmarkRequestSchema = z .object({ - title: z.string().max(MAX_TITLE_LENGTH).nullish(), + title: z.string().max(MAX_BOOKMARK_TITLE_LENGTH).nullish(), archived: z.boolean().optional(), favourited: z.boolean().optional(), note: z.string().optional(), @@ -202,7 +202,7 @@ export const zUpdateBookmarksRequestSchema = z.object({ favourited: z.boolean().optional(), summary: z.string().nullish(), note: z.string().optional(), - title: z.string().max(MAX_TITLE_LENGTH).nullish(), + title: z.string().max(MAX_BOOKMARK_TITLE_LENGTH).nullish(), createdAt: z.coerce.date().optional(), // Link specific fields (optional) url: z.string().url().optional(), diff --git a/packages/shared/types/lists.ts b/packages/shared/types/lists.ts index 51fb458c..59abb007 100644 --- a/packages/shared/types/lists.ts +++ b/packages/shared/types/lists.ts @@ -2,16 +2,25 @@ import { z } from "zod"; import { parseSearchQuery } from "../searchQueryParser"; +export const MAX_LIST_NAME_LENGTH = 100; +export const MAX_LIST_DESCRIPTION_LENGTH = 500; + export const zNewBookmarkListSchema = z .object({ name: z .string() .min(1, "List name can't be empty") - .max(40, "List name is at most 40 chars"), + .max( + MAX_LIST_NAME_LENGTH, + `List name is at most ${MAX_LIST_NAME_LENGTH} chars`, + ), description: z .string() .min(0, "Description can be empty") - .max(100, "Description can have at most 100 chars") + .max( + MAX_LIST_DESCRIPTION_LENGTH, + `Description can have at most ${MAX_LIST_DESCRIPTION_LENGTH} chars`, + ) .optional(), icon: z.string(), type: z.enum(["manual", "smart"]).optional().default("manual"), @@ -57,12 +66,18 @@ export const zEditBookmarkListSchema = z.object({ name: z .string() .min(1, "List name can't be empty") - .max(40, "List name is at most 40 chars") + .max( + MAX_LIST_NAME_LENGTH, + `List name is at most ${MAX_LIST_NAME_LENGTH} chars`, + ) .optional(), description: z .string() .min(0, "Description can be empty") - .max(100, "Description can have at most 100 chars") + .max( + MAX_LIST_DESCRIPTION_LENGTH, + `Description can have at most ${MAX_LIST_DESCRIPTION_LENGTH} chars`, + ) .nullish(), icon: z.string().optional(), parentId: z.string().nullish(), -- cgit v1.2.3-70-g09d2