From 86a4b3966504507afd6c3adbb6a1246cafd39d83 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sat, 29 Nov 2025 14:53:31 +0000 Subject: feat: Add automated bookmark backup feature (#2182) * feat: Add automated bookmark backup system Implements a comprehensive automated backup feature for user bookmarks with the following capabilities: Database Schema: - Add backupSettings table to store user backup preferences (enabled, frequency, retention) - Add backups table to track backup records with status and metadata - Add BACKUP asset type for storing compressed backup files - Add migration 0066_add_backup_tables.sql Background Workers: - Implement BackupSchedulingWorker cron job (runs daily at midnight UTC) - Create BackupWorker to process individual backup jobs - Deterministic scheduling spreads backup jobs across 24 hours based on user ID hash - Support for daily and weekly backup frequencies - Automated retention cleanup to delete old backups based on user settings Export & Compression: - Reuse existing export functionality for bookmark data - Compress exports using Node.js built-in zlib (gzip level 9) - Store compressed backups as assets with proper metadata - Track backup size and bookmark count for statistics tRPC API: - backups.getSettings - Retrieve user backup configuration - backups.updateSettings - Update backup preferences - backups.list - List all user backups with metadata - backups.get - Get specific backup details - backups.delete - Delete a backup - backups.download - Download backup file (base64 encoded) - backups.triggerBackup - Manually trigger backup creation UI Components: - BackupSettings component with configuration form - Enable/disable automatic backups toggle - Frequency selection (daily/weekly) - Retention period configuration (1-365 days) - Backup list table with download and delete actions - Manual backup trigger button - Display backup stats (size, bookmark count, status) - Added backups page to settings navigation Technical Details: - Uses Restate queue system for distributed job processing - Implements idempotency keys to prevent duplicate backups - Background worker concurrency: 2 jobs at a time - 10-minute timeout for large backup exports - Proper error handling and logging throughout - Type-safe implementation with Zod schemas * refactor: simplify backup settings and asset handling - Move backup settings from separate table to user table columns - Update BackupSettings model to use static methods with users table - Remove download mutation in favor of direct asset links - Implement proper quota checks using QuotaService.checkStorageQuota - Update UI to use new property names and direct asset downloads - Update shared types to match new schema Key changes: - backupSettingsTable removed, settings now in users table - Backup downloads use direct /api/assets/{id} links - Quota properly validated before creating backup assets - Cleaner separation of concerns in tRPC models * migration * use zip instead of gzip * fix drizzle * fix settings * streaming json * remove more dead code * add e2e tests * return backup * poll for backups * more fixes * more fixes * fix test * fix UI * fix delete asset * fix ui * redirect for backup download * cleanups * fix idempotency * fix tests * add ratelimit * add error handling for background backups * i18n * model changes --------- Co-authored-by: Claude --- apps/web/app/settings/backups/page.tsx | 17 + apps/web/app/settings/layout.tsx | 6 + apps/web/components/settings/BackupSettings.tsx | 423 ++++++++++++++++++++++++ apps/web/lib/i18n/locales/en/translation.json | 49 +++ apps/web/lib/userSettings.tsx | 3 + 5 files changed, 498 insertions(+) create mode 100644 apps/web/app/settings/backups/page.tsx create mode 100644 apps/web/components/settings/BackupSettings.tsx (limited to 'apps/web') diff --git a/apps/web/app/settings/backups/page.tsx b/apps/web/app/settings/backups/page.tsx new file mode 100644 index 00000000..fc263089 --- /dev/null +++ b/apps/web/app/settings/backups/page.tsx @@ -0,0 +1,17 @@ +"use client"; + +import BackupSettings from "@/components/settings/BackupSettings"; +import { useTranslation } from "@/lib/i18n/client"; + +export default function BackupsPage() { + const { t } = useTranslation(); + return ( +
+

{t("settings.backups.page_title")}

+

+ {t("settings.backups.page_description")} +

+ +
+ ); +} diff --git a/apps/web/app/settings/layout.tsx b/apps/web/app/settings/layout.tsx index 982ac61a..1c7d25ac 100644 --- a/apps/web/app/settings/layout.tsx +++ b/apps/web/app/settings/layout.tsx @@ -7,6 +7,7 @@ import { TFunction } from "i18next"; import { ArrowLeft, BarChart3, + CloudDownload, CreditCard, Download, GitBranch, @@ -67,6 +68,11 @@ const settingsSidebarItems = ( icon: , path: "/settings/feeds", }, + { + name: t("settings.backups.backups"), + icon: , + path: "/settings/backups", + }, { name: t("settings.import.import_export"), icon: , diff --git a/apps/web/components/settings/BackupSettings.tsx b/apps/web/components/settings/BackupSettings.tsx new file mode 100644 index 00000000..18a80993 --- /dev/null +++ b/apps/web/components/settings/BackupSettings.tsx @@ -0,0 +1,423 @@ +"use client"; + +import React from "react"; +import Link from "next/link"; +import { ActionButton } from "@/components/ui/action-button"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { FullPageSpinner } from "@/components/ui/full-page-spinner"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; +import { toast } from "@/components/ui/use-toast"; +import { useTranslation } from "@/lib/i18n/client"; +import { api } from "@/lib/trpc"; +import { useUserSettings } from "@/lib/userSettings"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { + CheckCircle, + Download, + Play, + Save, + Trash2, + XCircle, +} from "lucide-react"; +import { useForm } from "react-hook-form"; +import { z } from "zod"; + +import { useUpdateUserSettings } from "@karakeep/shared-react/hooks/users"; +import { zBackupSchema } from "@karakeep/shared/types/backups"; +import { zUpdateBackupSettingsSchema } from "@karakeep/shared/types/users"; +import { getAssetUrl } from "@karakeep/shared/utils/assetUtils"; + +import ActionConfirmingDialog from "../ui/action-confirming-dialog"; +import { Button } from "../ui/button"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "../ui/table"; +import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"; + +function BackupConfigurationForm() { + const { t } = useTranslation(); + + const settings = useUserSettings(); + const { mutate: updateSettings, isPending: isUpdating } = + useUpdateUserSettings({ + onSuccess: () => { + toast({ + description: t("settings.info.user_settings.user_settings_updated"), + }); + }, + onError: () => { + toast({ + description: t("common.something_went_wrong"), + variant: "destructive", + }); + }, + }); + + const form = useForm>({ + resolver: zodResolver(zUpdateBackupSettingsSchema), + values: settings + ? { + backupsEnabled: settings.backupsEnabled, + backupsFrequency: settings.backupsFrequency, + backupsRetentionDays: settings.backupsRetentionDays, + } + : undefined, + }); + + return ( +
+

+ {t("settings.backups.configuration.title")} +

+
+ { + updateSettings(value); + })} + > + ( + +
+ + {t( + "settings.backups.configuration.enable_automatic_backups", + )} + + + {t( + "settings.backups.configuration.enable_automatic_backups_description", + )} + +
+ + + + + )} + /> + + ( + + + {t("settings.backups.configuration.backup_frequency")} + + + + + + {t( + "settings.backups.configuration.backup_frequency_description", + )} + + + + )} + /> + + ( + + + {t("settings.backups.configuration.retention_period")} + + + field.onChange(parseInt(e.target.value))} + /> + + + {t( + "settings.backups.configuration.retention_period_description", + )} + + + + )} + /> + + + + {t("settings.backups.configuration.save_settings")} + + + +
+ ); +} + +function BackupRow({ backup }: { backup: z.infer }) { + const { t } = useTranslation(); + const apiUtils = api.useUtils(); + + const { mutate: deleteBackup, isPending: isDeleting } = + api.backups.delete.useMutation({ + onSuccess: () => { + toast({ + description: t("settings.backups.toasts.backup_deleted"), + }); + apiUtils.backups.list.invalidate(); + }, + onError: (error) => { + toast({ + description: `Error: ${error.message}`, + variant: "destructive", + }); + }, + }); + + const formatSize = (bytes: number) => { + if (bytes < 1024) return `${bytes} B`; + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)} KB`; + return `${(bytes / (1024 * 1024)).toFixed(2)} MB`; + }; + + return ( + + {backup.createdAt.toLocaleString()} + + {backup.status === "pending" + ? "-" + : backup.bookmarkCount.toLocaleString()} + + + {backup.status === "pending" ? "-" : formatSize(backup.size)} + + + {backup.status === "success" ? ( + + + {t("settings.backups.list.status.success")} + + ) : backup.status === "failure" ? ( + + + + + {t("settings.backups.list.status.failed")} + + + {backup.errorMessage} + + ) : ( + +
+ {t("settings.backups.list.status.pending")} + + )} + + + {backup.assetId && ( + + + + + + {t("settings.backups.list.actions.download_backup")} + + + )} + ( + deleteBackup({ backupId: backup.id })} + className="items-center" + type="button" + > + + {t("settings.backups.list.actions.delete_backup")} + + )} + > + + + + + ); +} + +function BackupsList() { + const { t } = useTranslation(); + const apiUtils = api.useUtils(); + const { data: backups, isLoading } = api.backups.list.useQuery(undefined, { + refetchInterval: (query) => { + const data = query.state.data; + // Poll every 3 seconds if there's a pending backup, otherwise don't poll + return data?.backups.some((backup) => backup.status === "pending") + ? 3000 + : false; + }, + }); + + const { mutate: triggerBackup, isPending: isTriggering } = + api.backups.triggerBackup.useMutation({ + onSuccess: () => { + toast({ + description: t("settings.backups.toasts.backup_queued"), + }); + apiUtils.backups.list.invalidate(); + }, + onError: (error) => { + toast({ + description: `Error: ${error.message}`, + variant: "destructive", + }); + }, + }); + + return ( +
+
+
+ + {t("settings.backups.list.title")} + + triggerBackup()} + loading={isTriggering} + variant="default" + className="items-center" + > + + {t("settings.backups.list.create_backup_now")} + +
+ + {isLoading && } + + {backups && backups.backups.length === 0 && ( +

+ {t("settings.backups.list.no_backups")} +

+ )} + + {backups && backups.backups.length > 0 && ( + + + + + {t("settings.backups.list.table.created_at")} + + + {t("settings.backups.list.table.bookmarks")} + + {t("settings.backups.list.table.size")} + {t("settings.backups.list.table.status")} + + {t("settings.backups.list.table.actions")} + + + + + {backups.backups.map((backup) => ( + + ))} + +
+ )} +
+
+ ); +} + +export default function BackupSettings() { + return ( +
+ + +
+ ); +} diff --git a/apps/web/lib/i18n/locales/en/translation.json b/apps/web/lib/i18n/locales/en/translation.json index bc69f710..43d45cb5 100644 --- a/apps/web/lib/i18n/locales/en/translation.json +++ b/apps/web/lib/i18n/locales/en/translation.json @@ -359,6 +359,55 @@ "delete_dialog_title": "Delete Import Session", "delete_dialog_description": "Are you sure you want to delete \"{{name}}\"? This action cannot be undone. The bookmarks themselves will not be deleted.", "delete_session": "Delete Session" + }, + "backups": { + "backups": "Backups", + "page_title": "Backups", + "page_description": "Automatically create and manage backups of your bookmarks. Backups are compressed and stored securely.", + "configuration": { + "title": "Backup Configuration", + "enable_automatic_backups": "Enable Automatic Backups", + "enable_automatic_backups_description": "Automatically create backups of your bookmarks", + "backup_frequency": "Backup Frequency", + "backup_frequency_description": "How often backups should be created", + "retention_period": "Retention Period (days)", + "retention_period_description": "How many days to keep backups before deleting them", + "frequency": { + "daily": "Daily", + "weekly": "Weekly" + }, + "select_frequency": "Select frequency", + "save_settings": "Save Settings" + }, + "list": { + "title": "Your Backups", + "create_backup_now": "Create Backup Now", + "no_backups": "You don't have any backups yet. Enable automatic backups or create one manually.", + "table": { + "created_at": "Created At", + "bookmarks": "Bookmarks", + "size": "Size", + "status": "Status", + "actions": "Actions" + }, + "status": { + "success": "Success", + "failed": "Failed", + "pending": "Pending" + }, + "actions": { + "download_backup": "Download Backup", + "delete_backup": "Delete Backup" + } + }, + "dialogs": { + "delete_backup_title": "Delete Backup?", + "delete_backup_description": "Are you sure you want to delete this backup? This action cannot be undone." + }, + "toasts": { + "backup_queued": "Backup job has been queued! It will be processed shortly.", + "backup_deleted": "Backup has been deleted!" + } } }, "admin": { diff --git a/apps/web/lib/userSettings.tsx b/apps/web/lib/userSettings.tsx index 2ab7db2b..c7a133b7 100644 --- a/apps/web/lib/userSettings.tsx +++ b/apps/web/lib/userSettings.tsx @@ -10,6 +10,9 @@ export const UserSettingsContext = createContext({ bookmarkClickAction: "open_original_link", archiveDisplayBehaviour: "show", timezone: "UTC", + backupsEnabled: false, + backupsFrequency: "daily", + backupsRetentionDays: 7, }); export function UserSettingsContextProvider({ -- cgit v1.2.3-70-g09d2