From 86a4b3966504507afd6c3adbb6a1246cafd39d83 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sat, 29 Nov 2025 14:53:31 +0000 Subject: feat: Add automated bookmark backup feature (#2182) * feat: Add automated bookmark backup system Implements a comprehensive automated backup feature for user bookmarks with the following capabilities: Database Schema: - Add backupSettings table to store user backup preferences (enabled, frequency, retention) - Add backups table to track backup records with status and metadata - Add BACKUP asset type for storing compressed backup files - Add migration 0066_add_backup_tables.sql Background Workers: - Implement BackupSchedulingWorker cron job (runs daily at midnight UTC) - Create BackupWorker to process individual backup jobs - Deterministic scheduling spreads backup jobs across 24 hours based on user ID hash - Support for daily and weekly backup frequencies - Automated retention cleanup to delete old backups based on user settings Export & Compression: - Reuse existing export functionality for bookmark data - Compress exports using Node.js built-in zlib (gzip level 9) - Store compressed backups as assets with proper metadata - Track backup size and bookmark count for statistics tRPC API: - backups.getSettings - Retrieve user backup configuration - backups.updateSettings - Update backup preferences - backups.list - List all user backups with metadata - backups.get - Get specific backup details - backups.delete - Delete a backup - backups.download - Download backup file (base64 encoded) - backups.triggerBackup - Manually trigger backup creation UI Components: - BackupSettings component with configuration form - Enable/disable automatic backups toggle - Frequency selection (daily/weekly) - Retention period configuration (1-365 days) - Backup list table with download and delete actions - Manual backup trigger button - Display backup stats (size, bookmark count, status) - Added backups page to settings navigation Technical Details: - Uses Restate queue system for distributed job processing - Implements idempotency keys to prevent duplicate backups - Background worker concurrency: 2 jobs at a time - 10-minute timeout for large backup exports - Proper error handling and logging throughout - Type-safe implementation with Zod schemas * refactor: simplify backup settings and asset handling - Move backup settings from separate table to user table columns - Update BackupSettings model to use static methods with users table - Remove download mutation in favor of direct asset links - Implement proper quota checks using QuotaService.checkStorageQuota - Update UI to use new property names and direct asset downloads - Update shared types to match new schema Key changes: - backupSettingsTable removed, settings now in users table - Backup downloads use direct /api/assets/{id} links - Quota properly validated before creating backup assets - Cleaner separation of concerns in tRPC models * migration * use zip instead of gzip * fix drizzle * fix settings * streaming json * remove more dead code * add e2e tests * return backup * poll for backups * more fixes * more fixes * fix test * fix UI * fix delete asset * fix ui * redirect for backup download * cleanups * fix idempotency * fix tests * add ratelimit * add error handling for background backups * i18n * model changes --------- Co-authored-by: Claude --- packages/sdk/src/karakeep-api.d.ts | 307 +++++++++++++++++++++++++++++++++++++ 1 file changed, 307 insertions(+) (limited to 'packages/sdk/src') diff --git a/packages/sdk/src/karakeep-api.d.ts b/packages/sdk/src/karakeep-api.d.ts index 7960e982..d327f1fa 100644 --- a/packages/sdk/src/karakeep-api.d.ts +++ b/packages/sdk/src/karakeep-api.d.ts @@ -68,6 +68,16 @@ export interface paths { /** @enum {string} */ crawlPriority?: "low" | "normal"; importSessionId?: string; + /** @enum {string} */ + source?: + | "api" + | "web" + | "cli" + | "mobile" + | "extension" + | "singlefile" + | "rss" + | "import"; } & ( | { /** @enum {string} */ @@ -322,6 +332,18 @@ export interface paths { summarizationStatus: "success" | "failure" | "pending" | null; note?: string | null; summary?: string | null; + /** @enum {string|null} */ + source?: + | "api" + | "web" + | "cli" + | "mobile" + | "extension" + | "singlefile" + | "rss" + | "import" + | null; + userId: string; }; }; }; @@ -384,6 +406,18 @@ export interface paths { summarizationStatus: "success" | "failure" | "pending" | null; note?: string | null; summary?: string | null; + /** @enum {string|null} */ + source?: + | "api" + | "web" + | "cli" + | "mobile" + | "extension" + | "singlefile" + | "rss" + | "import" + | null; + userId: string; }; }; }; @@ -668,6 +702,7 @@ export interface paths { | "video" | "bookmarkAsset" | "precrawledArchive" + | "userUploaded" | "unknown"; }; }; @@ -691,7 +726,9 @@ export interface paths { | "video" | "bookmarkAsset" | "precrawledArchive" + | "userUploaded" | "unknown"; + fileName?: string | null; }; }; }; @@ -1684,6 +1721,7 @@ export interface paths { "application/json": { /** @enum {string} */ color?: "yellow" | "red" | "green" | "blue"; + note?: string | null; }; }; }; @@ -1822,6 +1860,20 @@ export interface paths { name: string; count: number; }[]; + bookmarksBySource: { + /** @enum {string|null} */ + source: + | "api" + | "web" + | "cli" + | "mobile" + | "extension" + | "singlefile" + | "rss" + | "import" + | null; + count: number; + }[]; }; }; }; @@ -2017,6 +2069,241 @@ export interface paths { patch?: never; trace?: never; }; + "/backups": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get all backups + * @description Get all backups + */ + get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Object with all backups data. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + backups: { + id: string; + userId: string; + assetId: string; + createdAt: string; + size: number; + bookmarkCount: number; + /** @enum {string} */ + status: "pending" | "success" | "failure"; + errorMessage?: string | null; + }[]; + }; + }; + }; + }; + }; + put?: never; + /** + * Trigger a new backup + * @description Trigger a new backup + */ + post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Backup created successfully */ + 201: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + id: string; + userId: string; + assetId: string; + createdAt: string; + size: number; + bookmarkCount: number; + /** @enum {string} */ + status: "pending" | "success" | "failure"; + errorMessage?: string | null; + }; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/backups/{backupId}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get a single backup + * @description Get backup by its id + */ + get: { + parameters: { + query?: never; + header?: never; + path: { + backupId: components["parameters"]["BackupId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Object with backup data. */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + id: string; + userId: string; + assetId: string; + createdAt: string; + size: number; + bookmarkCount: number; + /** @enum {string} */ + status: "pending" | "success" | "failure"; + errorMessage?: string | null; + }; + }; + }; + /** @description Backup not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + code: string; + message: string; + }; + }; + }; + }; + }; + put?: never; + post?: never; + /** + * Delete a backup + * @description Delete backup by its id + */ + delete: { + parameters: { + query?: never; + header?: never; + path: { + backupId: components["parameters"]["BackupId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description No content - the backup was deleted */ + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Backup not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + code: string; + message: string; + }; + }; + }; + }; + }; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/backups/{backupId}/download": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Download a backup + * @description Download backup file + */ + get: { + parameters: { + query?: never; + header?: never; + path: { + backupId: components["parameters"]["BackupId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Backup file (zip archive) */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/zip": unknown; + }; + }; + /** @description Backup not found */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + code: string; + message: string; + }; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; } export type webhooks = Record; export interface components { @@ -2031,6 +2318,8 @@ export interface components { HighlightId: string; /** @example ieidlxygmwj87oxz5hxttoc8 */ AssetId: string; + /** @example ieidlxygmwj87oxz5hxttoc8 */ + BackupId: string; Bookmark: { id: string; createdAt: string; @@ -2044,6 +2333,18 @@ export interface components { summarizationStatus: "success" | "failure" | "pending" | null; note?: string | null; summary?: string | null; + /** @enum {string|null} */ + source?: + | "api" + | "web" + | "cli" + | "mobile" + | "extension" + | "singlefile" + | "rss" + | "import" + | null; + userId: string; tags: { id: string; name: string; @@ -2105,7 +2406,9 @@ export interface components { | "video" | "bookmarkAsset" | "precrawledArchive" + | "userUploaded" | "unknown"; + fileName?: string | null; }[]; }; PaginatedBookmarks: { @@ -2126,6 +2429,9 @@ export interface components { type: "manual" | "smart"; query?: string | null; public: boolean; + hasCollaborators: boolean; + /** @enum {string} */ + userRole: "owner" | "editor" | "viewer" | "public"; }; Highlight: { bookmarkId: string; @@ -2170,6 +2476,7 @@ export interface components { TagId: components["schemas"]["TagId"]; HighlightId: components["schemas"]["HighlightId"]; AssetId: components["schemas"]["AssetId"]; + BackupId: components["schemas"]["BackupId"]; }; requestBodies: never; headers: never; -- cgit v1.2.3-70-g09d2