From 3c838ddb26c1e86d3f201ce71f13c834be705f69 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Wed, 4 Feb 2026 09:44:18 +0000 Subject: feat: Import workflow v3 (#2378) * feat: import workflow v3 * batch stage * revert migration * cleanups * pr comments * move to models * add allowed workers * e2e tests * import list ids * add missing indicies * merge test * more fixes * add resume/pause to UI * fix ui states * fix tests * simplify progress tracking * remove backpressure * fix list imports * fix race on claiming bookmarks * remove the codex file --- apps/web/lib/hooks/useBookmarkImport.ts | 104 ++++---------------------- apps/web/lib/hooks/useImportSessions.ts | 56 +++++++++++++- apps/web/lib/i18n/locales/en/translation.json | 11 ++- 3 files changed, 77 insertions(+), 94 deletions(-) (limited to 'apps/web/lib') diff --git a/apps/web/lib/hooks/useBookmarkImport.ts b/apps/web/lib/hooks/useBookmarkImport.ts index c0681924..35c04c1b 100644 --- a/apps/web/lib/hooks/useBookmarkImport.ts +++ b/apps/web/lib/hooks/useBookmarkImport.ts @@ -5,25 +5,13 @@ import { toast } from "@/components/ui/sonner"; import { useTranslation } from "@/lib/i18n/client"; import { useMutation, useQueryClient } from "@tanstack/react-query"; -import { - useCreateBookmarkWithPostHook, - useUpdateBookmarkTags, -} from "@karakeep/shared-react/hooks/bookmarks"; -import { - useAddBookmarkToList, - useCreateBookmarkList, -} from "@karakeep/shared-react/hooks/lists"; +import { useCreateBookmarkList } from "@karakeep/shared-react/hooks/lists"; import { useTRPC } from "@karakeep/shared-react/trpc"; import { importBookmarksFromFile, ImportSource, - ParsedBookmark, parseImportFile, } from "@karakeep/shared/import-export"; -import { - BookmarkTypes, - MAX_BOOKMARK_TITLE_LENGTH, -} from "@karakeep/shared/types/bookmarks"; import { useCreateImportSession } from "./useImportSessions"; @@ -43,10 +31,13 @@ export function useBookmarkImport() { const queryClient = useQueryClient(); const { mutateAsync: createImportSession } = useCreateImportSession(); - const { mutateAsync: createBookmark } = useCreateBookmarkWithPostHook(); const { mutateAsync: createList } = useCreateBookmarkList(); - const { mutateAsync: addToList } = useAddBookmarkToList(); - const { mutateAsync: updateTags } = useUpdateBookmarkTags(); + const { mutateAsync: stageImportedBookmarks } = useMutation( + api.importSessions.stageImportedBookmarks.mutationOptions(), + ); + const { mutateAsync: finalizeImportStaging } = useMutation( + api.importSessions.finalizeImportStaging.mutationOptions(), + ); const uploadBookmarkFileMutation = useMutation({ mutationFn: async ({ @@ -86,7 +77,6 @@ export function useBookmarkImport() { } // Proceed with import if quota check passes - // Use a custom parser to avoid re-parsing the file const result = await importBookmarksFromFile( { file, @@ -95,65 +85,9 @@ export function useBookmarkImport() { deps: { createImportSession, createList, - createBookmark: async ( - bookmark: ParsedBookmark, - sessionId: string, - ) => { - if (bookmark.content === undefined) { - throw new Error("Content is undefined"); - } - const created = await createBookmark({ - crawlPriority: "low", - title: bookmark.title.substring(0, MAX_BOOKMARK_TITLE_LENGTH), - createdAt: bookmark.addDate - ? new Date(bookmark.addDate * 1000) - : undefined, - note: bookmark.notes, - archived: bookmark.archived, - importSessionId: sessionId, - source: "import", - ...(bookmark.content.type === BookmarkTypes.LINK - ? { - type: BookmarkTypes.LINK, - url: bookmark.content.url, - } - : { - type: BookmarkTypes.TEXT, - text: bookmark.content.text, - }), - }); - return created as { id: string; alreadyExists?: boolean }; - }, - addBookmarkToLists: async ({ - bookmarkId, - listIds, - }: { - bookmarkId: string; - listIds: string[]; - }) => { - await Promise.all( - listIds.map((listId) => - addToList({ - bookmarkId, - listId, - }), - ), - ); - }, - updateBookmarkTags: async ({ - bookmarkId, - tags, - }: { - bookmarkId: string; - tags: string[]; - }) => { - if (tags.length > 0) { - await updateTags({ - bookmarkId, - attach: tags.map((t) => ({ tagName: t })), - detach: [], - }); - } + stageImportedBookmarks, + finalizeImportStaging: async (sessionId: string) => { + await finalizeImportStaging({ importSessionId: sessionId }); }, }, onProgress: (done, total) => setImportProgress({ done, total }), @@ -174,19 +108,11 @@ export function useBookmarkImport() { toast({ description: "No bookmarks found in the file." }); return; } - const { successes, failures, alreadyExisted } = result.counts; - if (successes > 0 || alreadyExisted > 0) { - toast({ - description: `Imported ${successes} bookmarks into import session. Background processing will start automatically.`, - variant: "default", - }); - } - if (failures > 0) { - toast({ - description: `Failed to import ${failures} bookmarks. Check console for details.`, - variant: "destructive", - }); - } + + toast({ + description: `Staged ${result.counts.total} bookmarks for import. Background processing will start automatically.`, + variant: "default", + }); }, onError: (error) => { setImportProgress(null); diff --git a/apps/web/lib/hooks/useImportSessions.ts b/apps/web/lib/hooks/useImportSessions.ts index 133bb29b..4d095c0b 100644 --- a/apps/web/lib/hooks/useImportSessions.ts +++ b/apps/web/lib/hooks/useImportSessions.ts @@ -46,7 +46,11 @@ export function useImportSessionStats(importSessionId: string) { importSessionId, }, { - refetchInterval: 5000, // Refetch every 5 seconds to show progress + refetchInterval: (q) => + !q.state.data || + !["completed", "failed"].includes(q.state.data.status) + ? 5000 + : false, // Refetch every 5 seconds to show progress enabled: !!importSessionId, }, ), @@ -77,3 +81,53 @@ export function useDeleteImportSession() { }), ); } + +export function usePauseImportSession() { + const api = useTRPC(); + const queryClient = useQueryClient(); + + return useMutation( + api.importSessions.pauseImportSession.mutationOptions({ + onSuccess: () => { + queryClient.invalidateQueries( + api.importSessions.listImportSessions.pathFilter(), + ); + toast({ + description: "Import session paused", + variant: "default", + }); + }, + onError: (error) => { + toast({ + description: error.message || "Failed to pause import session", + variant: "destructive", + }); + }, + }), + ); +} + +export function useResumeImportSession() { + const api = useTRPC(); + const queryClient = useQueryClient(); + + return useMutation( + api.importSessions.resumeImportSession.mutationOptions({ + onSuccess: () => { + queryClient.invalidateQueries( + api.importSessions.listImportSessions.pathFilter(), + ); + toast({ + description: "Import session resumed", + variant: "default", + }); + }, + onError: (error) => { + toast({ + description: error.message || "Failed to resume import session", + variant: "destructive", + }); + }, + }), + ); +} diff --git a/apps/web/lib/i18n/locales/en/translation.json b/apps/web/lib/i18n/locales/en/translation.json index ce607920..59d2098e 100644 --- a/apps/web/lib/i18n/locales/en/translation.json +++ b/apps/web/lib/i18n/locales/en/translation.json @@ -416,11 +416,12 @@ "created_at": "Created {{time}}", "progress": "Progress", "status": { + "staging": "Staging", "pending": "Pending", - "in_progress": "In progress", + "running": "Running", + "paused": "Paused", "completed": "Completed", - "failed": "Failed", - "processing": "Processing" + "failed": "Failed" }, "badges": { "pending": "{{count}} pending", @@ -432,7 +433,9 @@ "view_list": "View List", "delete_dialog_title": "Delete Import Session", "delete_dialog_description": "Are you sure you want to delete \"{{name}}\"? This action cannot be undone. The bookmarks themselves will not be deleted.", - "delete_session": "Delete Session" + "delete_session": "Delete Session", + "pause_session": "Pause", + "resume_session": "Resume" }, "backups": { "backups": "Backups", -- cgit v1.2.3-70-g09d2