From 86a4b3966504507afd6c3adbb6a1246cafd39d83 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sat, 29 Nov 2025 14:53:31 +0000 Subject: feat: Add automated bookmark backup feature (#2182) * feat: Add automated bookmark backup system Implements a comprehensive automated backup feature for user bookmarks with the following capabilities: Database Schema: - Add backupSettings table to store user backup preferences (enabled, frequency, retention) - Add backups table to track backup records with status and metadata - Add BACKUP asset type for storing compressed backup files - Add migration 0066_add_backup_tables.sql Background Workers: - Implement BackupSchedulingWorker cron job (runs daily at midnight UTC) - Create BackupWorker to process individual backup jobs - Deterministic scheduling spreads backup jobs across 24 hours based on user ID hash - Support for daily and weekly backup frequencies - Automated retention cleanup to delete old backups based on user settings Export & Compression: - Reuse existing export functionality for bookmark data - Compress exports using Node.js built-in zlib (gzip level 9) - Store compressed backups as assets with proper metadata - Track backup size and bookmark count for statistics tRPC API: - backups.getSettings - Retrieve user backup configuration - backups.updateSettings - Update backup preferences - backups.list - List all user backups with metadata - backups.get - Get specific backup details - backups.delete - Delete a backup - backups.download - Download backup file (base64 encoded) - backups.triggerBackup - Manually trigger backup creation UI Components: - BackupSettings component with configuration form - Enable/disable automatic backups toggle - Frequency selection (daily/weekly) - Retention period configuration (1-365 days) - Backup list table with download and delete actions - Manual backup trigger button - Display backup stats (size, bookmark count, status) - Added backups page to settings navigation Technical Details: - Uses Restate queue system for distributed job processing - Implements idempotency keys to prevent duplicate backups - Background worker concurrency: 2 jobs at a time - 10-minute timeout for large backup exports - Proper error handling and logging throughout - Type-safe implementation with Zod schemas * refactor: simplify backup settings and asset handling - Move backup settings from separate table to user table columns - Update BackupSettings model to use static methods with users table - Remove download mutation in favor of direct asset links - Implement proper quota checks using QuotaService.checkStorageQuota - Update UI to use new property names and direct asset downloads - Update shared types to match new schema Key changes: - backupSettingsTable removed, settings now in users table - Backup downloads use direct /api/assets/{id} links - Quota properly validated before creating backup assets - Cleaner separation of concerns in tRPC models * migration * use zip instead of gzip * fix drizzle * fix settings * streaming json * remove more dead code * add e2e tests * return backup * poll for backups * more fixes * more fixes * fix test * fix UI * fix delete asset * fix ui * redirect for backup download * cleanups * fix idempotency * fix tests * add ratelimit * add error handling for background backups * i18n * model changes --------- Co-authored-by: Claude --- packages/open-api/karakeep-openapi-spec.json | 347 +++++++++++++++++++++++++++ 1 file changed, 347 insertions(+) (limited to 'packages/open-api/karakeep-openapi-spec.json') diff --git a/packages/open-api/karakeep-openapi-spec.json b/packages/open-api/karakeep-openapi-spec.json index c151f42d..4532cd98 100644 --- a/packages/open-api/karakeep-openapi-spec.json +++ b/packages/open-api/karakeep-openapi-spec.json @@ -45,6 +45,10 @@ "type": "string", "example": "ieidlxygmwj87oxz5hxttoc8" }, + "BackupId": { + "type": "string", + "example": "ieidlxygmwj87oxz5hxttoc8" + }, "Bookmark": { "type": "object", "properties": { @@ -596,6 +600,14 @@ "required": true, "name": "assetId", "in": "path" + }, + "BackupId": { + "schema": { + "$ref": "#/components/schemas/BackupId" + }, + "required": true, + "name": "backupId", + "in": "path" } } }, @@ -3703,6 +3715,341 @@ } } } + }, + "/backups": { + "get": { + "description": "Get all backups", + "summary": "Get all backups", + "tags": [ + "Backups" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "200": { + "description": "Object with all backups data.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "backups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "userId": { + "type": "string" + }, + "assetId": { + "type": "string", + "nullable": true + }, + "createdAt": { + "type": "string" + }, + "size": { + "type": "number" + }, + "bookmarkCount": { + "type": "number" + }, + "status": { + "type": "string", + "enum": [ + "pending", + "success", + "failure" + ] + }, + "errorMessage": { + "type": "string", + "nullable": true + } + }, + "required": [ + "id", + "userId", + "assetId", + "createdAt", + "size", + "bookmarkCount", + "status" + ] + } + } + }, + "required": [ + "backups" + ] + } + } + } + } + } + }, + "post": { + "description": "Trigger a new backup", + "summary": "Trigger a new backup", + "tags": [ + "Backups" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "responses": { + "201": { + "description": "Backup created successfully", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "userId": { + "type": "string" + }, + "assetId": { + "type": "string", + "nullable": true + }, + "createdAt": { + "type": "string" + }, + "size": { + "type": "number" + }, + "bookmarkCount": { + "type": "number" + }, + "status": { + "type": "string", + "enum": [ + "pending", + "success", + "failure" + ] + }, + "errorMessage": { + "type": "string", + "nullable": true + } + }, + "required": [ + "id", + "userId", + "assetId", + "createdAt", + "size", + "bookmarkCount", + "status" + ] + } + } + } + } + } + } + }, + "/backups/{backupId}": { + "get": { + "description": "Get backup by its id", + "summary": "Get a single backup", + "tags": [ + "Backups" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/BackupId" + } + ], + "responses": { + "200": { + "description": "Object with backup data.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "userId": { + "type": "string" + }, + "assetId": { + "type": "string", + "nullable": true + }, + "createdAt": { + "type": "string" + }, + "size": { + "type": "number" + }, + "bookmarkCount": { + "type": "number" + }, + "status": { + "type": "string", + "enum": [ + "pending", + "success", + "failure" + ] + }, + "errorMessage": { + "type": "string", + "nullable": true + } + }, + "required": [ + "id", + "userId", + "assetId", + "createdAt", + "size", + "bookmarkCount", + "status" + ] + } + } + } + }, + "404": { + "description": "Backup not found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "required": [ + "code", + "message" + ] + } + } + } + } + } + }, + "delete": { + "description": "Delete backup by its id", + "summary": "Delete a backup", + "tags": [ + "Backups" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/BackupId" + } + ], + "responses": { + "204": { + "description": "No content - the backup was deleted" + }, + "404": { + "description": "Backup not found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "required": [ + "code", + "message" + ] + } + } + } + } + } + } + }, + "/backups/{backupId}/download": { + "get": { + "description": "Download backup file", + "summary": "Download a backup", + "tags": [ + "Backups" + ], + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "$ref": "#/components/parameters/BackupId" + } + ], + "responses": { + "200": { + "description": "Backup file (zip archive)", + "content": { + "application/zip": { + "schema": {} + } + } + }, + "404": { + "description": "Backup not found", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "message": { + "type": "string" + } + }, + "required": [ + "code", + "message" + ] + } + } + } + } + } + } } } } \ No newline at end of file -- cgit v1.2.3-70-g09d2