From 86a4b3966504507afd6c3adbb6a1246cafd39d83 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sat, 29 Nov 2025 14:53:31 +0000 Subject: feat: Add automated bookmark backup feature (#2182) * feat: Add automated bookmark backup system Implements a comprehensive automated backup feature for user bookmarks with the following capabilities: Database Schema: - Add backupSettings table to store user backup preferences (enabled, frequency, retention) - Add backups table to track backup records with status and metadata - Add BACKUP asset type for storing compressed backup files - Add migration 0066_add_backup_tables.sql Background Workers: - Implement BackupSchedulingWorker cron job (runs daily at midnight UTC) - Create BackupWorker to process individual backup jobs - Deterministic scheduling spreads backup jobs across 24 hours based on user ID hash - Support for daily and weekly backup frequencies - Automated retention cleanup to delete old backups based on user settings Export & Compression: - Reuse existing export functionality for bookmark data - Compress exports using Node.js built-in zlib (gzip level 9) - Store compressed backups as assets with proper metadata - Track backup size and bookmark count for statistics tRPC API: - backups.getSettings - Retrieve user backup configuration - backups.updateSettings - Update backup preferences - backups.list - List all user backups with metadata - backups.get - Get specific backup details - backups.delete - Delete a backup - backups.download - Download backup file (base64 encoded) - backups.triggerBackup - Manually trigger backup creation UI Components: - BackupSettings component with configuration form - Enable/disable automatic backups toggle - Frequency selection (daily/weekly) - Retention period configuration (1-365 days) - Backup list table with download and delete actions - Manual backup trigger button - Display backup stats (size, bookmark count, status) - Added backups page to settings navigation Technical Details: - Uses Restate queue system for distributed job processing - Implements idempotency keys to prevent duplicate backups - Background worker concurrency: 2 jobs at a time - 10-minute timeout for large backup exports - Proper error handling and logging throughout - Type-safe implementation with Zod schemas * refactor: simplify backup settings and asset handling - Move backup settings from separate table to user table columns - Update BackupSettings model to use static methods with users table - Remove download mutation in favor of direct asset links - Implement proper quota checks using QuotaService.checkStorageQuota - Update UI to use new property names and direct asset downloads - Update shared types to match new schema Key changes: - backupSettingsTable removed, settings now in users table - Backup downloads use direct /api/assets/{id} links - Quota properly validated before creating backup assets - Cleaner separation of concerns in tRPC models * migration * use zip instead of gzip * fix drizzle * fix settings * streaming json * remove more dead code * add e2e tests * return backup * poll for backups * more fixes * more fixes * fix test * fix UI * fix delete asset * fix ui * redirect for backup download * cleanups * fix idempotency * fix tests * add ratelimit * add error handling for background backups * i18n * model changes --------- Co-authored-by: Claude --- apps/web/components/settings/BackupSettings.tsx | 423 ++++++++++++++++++++++++ 1 file changed, 423 insertions(+) create mode 100644 apps/web/components/settings/BackupSettings.tsx (limited to 'apps/web/components') diff --git a/apps/web/components/settings/BackupSettings.tsx b/apps/web/components/settings/BackupSettings.tsx new file mode 100644 index 00000000..18a80993 --- /dev/null +++ b/apps/web/components/settings/BackupSettings.tsx @@ -0,0 +1,423 @@ +"use client"; + +import React from "react"; +import Link from "next/link"; +import { ActionButton } from "@/components/ui/action-button"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { FullPageSpinner } from "@/components/ui/full-page-spinner"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; +import { toast } from "@/components/ui/use-toast"; +import { useTranslation } from "@/lib/i18n/client"; +import { api } from "@/lib/trpc"; +import { useUserSettings } from "@/lib/userSettings"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { + CheckCircle, + Download, + Play, + Save, + Trash2, + XCircle, +} from "lucide-react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; + +import { useUpdateUserSettings } from "@karakeep/shared-react/hooks/users"; +import { zBackupSchema } from "@karakeep/shared/types/backups"; +import { zUpdateBackupSettingsSchema } from "@karakeep/shared/types/users"; +import { getAssetUrl } from "@karakeep/shared/utils/assetUtils"; + +import ActionConfirmingDialog from "../ui/action-confirming-dialog"; +import { Button } from "../ui/button"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "../ui/table"; +import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"; + +function BackupConfigurationForm() { + const { t } = useTranslation(); + + const settings = useUserSettings(); + const { mutate: updateSettings, isPending: isUpdating } = + useUpdateUserSettings({ + onSuccess: () => { + toast({ + description: t("settings.info.user_settings.user_settings_updated"), + }); + }, + onError: () => { + toast({ + description: t("common.something_went_wrong"), + variant: "destructive", + }); + }, + }); + + const form = useForm>({ + resolver: zodResolver(zUpdateBackupSettingsSchema), + values: settings + ? { + backupsEnabled: settings.backupsEnabled, + backupsFrequency: settings.backupsFrequency, + backupsRetentionDays: settings.backupsRetentionDays, + } + : undefined, + }); + + return ( +
+

+ {t("settings.backups.configuration.title")} +

+
+ { + updateSettings(value); + })} + > + ( + +
+ + {t( + "settings.backups.configuration.enable_automatic_backups", + )} + + + {t( + "settings.backups.configuration.enable_automatic_backups_description", + )} + +
+ + + + + )} + /> + + ( + + + {t("settings.backups.configuration.backup_frequency")} + + + + + + {t( + "settings.backups.configuration.backup_frequency_description", + )} + + + + )} + /> + + ( + + + {t("settings.backups.configuration.retention_period")} + + + field.onChange(parseInt(e.target.value))} + /> + + + {t( + "settings.backups.configuration.retention_period_description", + )} + + + + )} + /> + + + + {t("settings.backups.configuration.save_settings")} + + + +
+ ); +} + +function BackupRow({ backup }: { backup: z.infer }) { + const { t } = useTranslation(); + const apiUtils = api.useUtils(); + + const { mutate: deleteBackup, isPending: isDeleting } = + api.backups.delete.useMutation({ + onSuccess: () => { + toast({ + description: t("settings.backups.toasts.backup_deleted"), + }); + apiUtils.backups.list.invalidate(); + }, + onError: (error) => { + toast({ + description: `Error: ${error.message}`, + variant: "destructive", + }); + }, + }); + + const formatSize = (bytes: number) => { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; + }; + + return ( + + {backup.createdAt.toLocaleString()} + + {backup.status === "pending" + ? "-" + : backup.bookmarkCount.toLocaleString()} + + + {backup.status === "pending" ? "-" : formatSize(backup.size)} + + + {backup.status === "success" ? ( + + + {t("settings.backups.list.status.success")} + + ) : backup.status === "failure" ? ( + + + + + {t("settings.backups.list.status.failed")} + + + {backup.errorMessage} + + ) : ( + +
+ {t("settings.backups.list.status.pending")} + + )} + + + {backup.assetId && ( + + + + + + {t("settings.backups.list.actions.download_backup")} + + + )} + ( + deleteBackup({ backupId: backup.id })} + className="items-center" + type="button" + > + + {t("settings.backups.list.actions.delete_backup")} + + )} + > + + + + + ); +} + +function BackupsList() { + const { t } = useTranslation(); + const apiUtils = api.useUtils(); + const { data: backups, isLoading } = api.backups.list.useQuery(undefined, { + refetchInterval: (query) => { + const data = query.state.data; + // Poll every 3 seconds if there's a pending backup, otherwise don't poll + return data?.backups.some((backup) => backup.status === "pending") + ? 3000 + : false; + }, + }); + + const { mutate: triggerBackup, isPending: isTriggering } = + api.backups.triggerBackup.useMutation({ + onSuccess: () => { + toast({ + description: t("settings.backups.toasts.backup_queued"), + }); + apiUtils.backups.list.invalidate(); + }, + onError: (error) => { + toast({ + description: `Error: ${error.message}`, + variant: "destructive", + }); + }, + }); + + return ( +
+
+
+ + {t("settings.backups.list.title")} + + triggerBackup()} + loading={isTriggering} + variant="default" + className="items-center" + > + + {t("settings.backups.list.create_backup_now")} + +
+ + {isLoading && } + + {backups && backups.backups.length === 0 && ( +

+ {t("settings.backups.list.no_backups")} +

+ )} + + {backups && backups.backups.length > 0 && ( + + + + + {t("settings.backups.list.table.created_at")} + + + {t("settings.backups.list.table.bookmarks")} + + {t("settings.backups.list.table.size")} + {t("settings.backups.list.table.status")} + + {t("settings.backups.list.table.actions")} + + + + + {backups.backups.map((backup) => ( + + ))} + +
+ )} +
+
+ ); +} + +export default function BackupSettings() { + return ( +
+ + +
+ ); +} -- cgit v1.2.3-70-g09d2