feat: Add automated bookmark backup feature (#2182)

* feat: Add automated bookmark backup system

Implements a comprehensive automated backup feature for user bookmarks with the following capabilities:

Database Schema:
- Add backupSettings table to store user backup preferences (enabled, frequency, retention)
- Add backups table to track backup records with status and metadata
- Add BACKUP asset type for storing compressed backup files
- Add migration 0066_add_backup_tables.sql

Background Workers:
- Implement BackupSchedulingWorker cron job (runs daily at midnight UTC)
- Create BackupWorker to process individual backup jobs
- Deterministic scheduling spreads backup jobs across 24 hours based on user ID hash
- Support for daily and weekly backup frequencies
- Automated retention cleanup to delete old backups based on user settings

Export & Compression:
- Reuse existing export functionality for bookmark data
- Compress exports using Node.js built-in zlib (gzip level 9)
- Store compressed backups as assets with proper metadata
- Track backup size and bookmark count for statistics

tRPC API:
- backups.getSettings - Retrieve user backup configuration
- backups.updateSettings - Update backup preferences
- backups.list - List all user backups with metadata
- backups.get - Get specific backup details
- backups.delete - Delete a backup
- backups.download - Download backup file (base64 encoded)
- backups.triggerBackup - Manually trigger backup creation

UI Components:
- BackupSettings component with configuration form
- Enable/disable automatic backups toggle
- Frequency selection (daily/weekly)
- Retention period configuration (1-365 days)
- Backup list table with download and delete actions
- Manual backup trigger button
- Display backup stats (size, bookmark count, status)
- Added backups page to settings navigation

Technical Details:
- Uses Restate queue system for distributed job processing
- Implements idempotency keys to prevent duplicate backups
- Background worker concurrency: 2 jobs at a time
- 10-minute timeout for large backup exports
- Proper error handling and logging throughout
- Type-safe implementation with Zod schemas

* refactor: simplify backup settings and asset handling

- Move backup settings from separate table to user table columns
- Update BackupSettings model to use static methods with users table
- Remove download mutation in favor of direct asset links
- Implement proper quota checks using QuotaService.checkStorageQuota
- Update UI to use new property names and direct asset downloads
- Update shared types to match new schema

Key changes:
- backupSettingsTable removed, settings now in users table
- Backup downloads use direct /api/assets/{id} links
- Quota properly validated before creating backup assets
- Cleaner separation of concerns in tRPC models

* migration

* use zip instead of gzip

* fix drizzle

* fix settings

* streaming json

* remove more dead code

* add e2e tests

* return backup

* poll for backups

* more fixes

* more fixes

* fix test

* fix UI

* fix delete asset

* fix ui

* redirect for backup download

* cleanups

* fix idempotency

* fix tests

* add ratelimit

* add error handling for background backups

* i18n

* model changes

---------

Co-authored-by: Claude <noreply@anthropic.com>

1 files changed, 307 insertions, 0 deletions

diff --git a/packages/sdk/src/karakeep-api.d.ts b/packages/sdk/src/karakeep-api.d.ts
index 7960e982..d327f1fa 100644
--- a/packages/sdk/src/karakeep-api.d.ts
+++ b/packages/sdk/src/karakeep-api.d.ts
@@ -68,6 +68,16 @@ export interface paths {
             /** @enum {string} */
             crawlPriority?: "low" | "normal";
             importSessionId?: string;
+            /** @enum {string} */
+            source?:
+              | "api"
+              | "web"
+              | "cli"
+              | "mobile"
+              | "extension"
+              | "singlefile"
+              | "rss"
+              | "import";
           } & (
             | {
                 /** @enum {string} */
@@ -322,6 +332,18 @@ export interface paths {
               summarizationStatus: "success" | "failure" | "pending" | null;
               note?: string | null;
               summary?: string | null;
+              /** @enum {string|null} */
+              source?:
+                | "api"
+                | "web"
+                | "cli"
+                | "mobile"
+                | "extension"
+                | "singlefile"
+                | "rss"
+                | "import"
+                | null;
+              userId: string;
             };
           };
         };
@@ -384,6 +406,18 @@ export interface paths {
               summarizationStatus: "success" | "failure" | "pending" | null;
               note?: string | null;
               summary?: string | null;
+              /** @enum {string|null} */
+              source?:
+                | "api"
+                | "web"
+                | "cli"
+                | "mobile"
+                | "extension"
+                | "singlefile"
+                | "rss"
+                | "import"
+                | null;
+              userId: string;
             };
           };
         };
@@ -668,6 +702,7 @@ export interface paths {
               | "video"
               | "bookmarkAsset"
               | "precrawledArchive"
+              | "userUploaded"
               | "unknown";
           };
         };
@@ -691,7 +726,9 @@ export interface paths {
                 | "video"
                 | "bookmarkAsset"
                 | "precrawledArchive"
+                | "userUploaded"
                 | "unknown";
+              fileName?: string | null;
             };
           };
         };
@@ -1684,6 +1721,7 @@ export interface paths {
           "application/json": {
             /** @enum {string} */
             color?: "yellow" | "red" | "green" | "blue";
+            note?: string | null;
           };
         };
       };
@@ -1822,6 +1860,20 @@ export interface paths {
                 name: string;
                 count: number;
               }[];
+              bookmarksBySource: {
+                /** @enum {string|null} */
+                source:
+                  | "api"
+                  | "web"
+                  | "cli"
+                  | "mobile"
+                  | "extension"
+                  | "singlefile"
+                  | "rss"
+                  | "import"
+                  | null;
+                count: number;
+              }[];
             };
           };
         };
@@ -2017,6 +2069,241 @@ export interface paths {
     patch?: never;
     trace?: never;
   };
+  "/backups": {
+    parameters: {
+      query?: never;
+      header?: never;
+      path?: never;
+      cookie?: never;
+    };
+    /**
+     * Get all backups
+     * @description Get all backups
+     */
+    get: {
+      parameters: {
+        query?: never;
+        header?: never;
+        path?: never;
+        cookie?: never;
+      };
+      requestBody?: never;
+      responses: {
+        /** @description Object with all backups data. */
+        200: {
+          headers: {
+            [name: string]: unknown;
+          };
+          content: {
+            "application/json": {
+              backups: {
+                id: string;
+                userId: string;
+                assetId: string;
+                createdAt: string;
+                size: number;
+                bookmarkCount: number;
+                /** @enum {string} */
+                status: "pending" | "success" | "failure";
+                errorMessage?: string | null;
+              }[];
+            };
+          };
+        };
+      };
+    };
+    put?: never;
+    /**
+     * Trigger a new backup
+     * @description Trigger a new backup
+     */
+    post: {
+      parameters: {
+        query?: never;
+        header?: never;
+        path?: never;
+        cookie?: never;
+      };
+      requestBody?: never;
+      responses: {
+        /** @description Backup created successfully */
+        201: {
+          headers: {
+            [name: string]: unknown;
+          };
+          content: {
+            "application/json": {
+              id: string;
+              userId: string;
+              assetId: string;
+              createdAt: string;
+              size: number;
+              bookmarkCount: number;
+              /** @enum {string} */
+              status: "pending" | "success" | "failure";
+              errorMessage?: string | null;
+            };
+          };
+        };
+      };
+    };
+    delete?: never;
+    options?: never;
+    head?: never;
+    patch?: never;
+    trace?: never;
+  };
+  "/backups/{backupId}": {
+    parameters: {
+      query?: never;
+      header?: never;
+      path?: never;
+      cookie?: never;
+    };
+    /**
+     * Get a single backup
+     * @description Get backup by its id
+     */
+    get: {
+      parameters: {
+        query?: never;
+        header?: never;
+        path: {
+          backupId: components["parameters"]["BackupId"];
+        };
+        cookie?: never;
+      };
+      requestBody?: never;
+      responses: {
+        /** @description Object with backup data. */
+        200: {
+          headers: {
+            [name: string]: unknown;
+          };
+          content: {
+            "application/json": {
+              id: string;
+              userId: string;
+              assetId: string;
+              createdAt: string;
+              size: number;
+              bookmarkCount: number;
+              /** @enum {string} */
+              status: "pending" | "success" | "failure";
+              errorMessage?: string | null;
+            };
+          };
+        };
+        /** @description Backup not found */
+        404: {
+          headers: {
+            [name: string]: unknown;
+          };
+          content: {
+            "application/json": {
+              code: string;
+              message: string;
+            };
+          };
+        };
+      };
+    };
+    put?: never;
+    post?: never;
+    /**
+     * Delete a backup
+     * @description Delete backup by its id
+     */
+    delete: {
+      parameters: {
+        query?: never;
+        header?: never;
+        path: {
+          backupId: components["parameters"]["BackupId"];
+        };
+        cookie?: never;
+      };
+      requestBody?: never;
+      responses: {
+        /** @description No content - the backup was deleted */
+        204: {
+          headers: {
+            [name: string]: unknown;
+          };
+          content?: never;
+        };
+        /** @description Backup not found */
+        404: {
+          headers: {
+            [name: string]: unknown;
+          };
+          content: {
+            "application/json": {
+              code: string;
+              message: string;
+            };
+          };
+        };
+      };
+    };
+    options?: never;
+    head?: never;
+    patch?: never;
+    trace?: never;
+  };
+  "/backups/{backupId}/download": {
+    parameters: {
+      query?: never;
+      header?: never;
+      path?: never;
+      cookie?: never;
+    };
+    /**
+     * Download a backup
+     * @description Download backup file
+     */
+    get: {
+      parameters: {
+        query?: never;
+        header?: never;
+        path: {
+          backupId: components["parameters"]["BackupId"];
+        };
+        cookie?: never;
+      };
+      requestBody?: never;
+      responses: {
+        /** @description Backup file (zip archive) */
+        200: {
+          headers: {
+            [name: string]: unknown;
+          };
+          content: {
+            "application/zip": unknown;
+          };
+        };
+        /** @description Backup not found */
+        404: {
+          headers: {
+            [name: string]: unknown;
+          };
+          content: {
+            "application/json": {
+              code: string;
+              message: string;
+            };
+          };
+        };
+      };
+    };
+    put?: never;
+    post?: never;
+    delete?: never;
+    options?: never;
+    head?: never;
+    patch?: never;
+    trace?: never;
+  };
 }
 export type webhooks = Record<string, never>;
 export interface components {
@@ -2031,6 +2318,8 @@ export interface components {
     HighlightId: string;
     /** @example ieidlxygmwj87oxz5hxttoc8 */
     AssetId: string;
+    /** @example ieidlxygmwj87oxz5hxttoc8 */
+    BackupId: string;
     Bookmark: {
       id: string;
       createdAt: string;
@@ -2044,6 +2333,18 @@ export interface components {
       summarizationStatus: "success" | "failure" | "pending" | null;
       note?: string | null;
       summary?: string | null;
+      /** @enum {string|null} */
+      source?:
+        | "api"
+        | "web"
+        | "cli"
+        | "mobile"
+        | "extension"
+        | "singlefile"
+        | "rss"
+        | "import"
+        | null;
+      userId: string;
       tags: {
         id: string;
         name: string;
@@ -2105,7 +2406,9 @@ export interface components {
           | "video"
           | "bookmarkAsset"
           | "precrawledArchive"
+          | "userUploaded"
           | "unknown";
+        fileName?: string | null;
       }[];
     };
     PaginatedBookmarks: {
@@ -2126,6 +2429,9 @@ export interface components {
       type: "manual" | "smart";
       query?: string | null;
       public: boolean;
+      hasCollaborators: boolean;
+      /** @enum {string} */
+      userRole: "owner" | "editor" | "viewer" | "public";
     };
     Highlight: {
       bookmarkId: string;
@@ -2170,6 +2476,7 @@ export interface components {
     TagId: components["schemas"]["TagId"];
     HighlightId: components["schemas"]["HighlightId"];
     AssetId: components["schemas"]["AssetId"];
+    BackupId: components["schemas"]["BackupId"];
   };
   requestBodies: never;
   headers: never;