From 86a4b3966504507afd6c3adbb6a1246cafd39d83 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sat, 29 Nov 2025 14:53:31 +0000 Subject: feat: Add automated bookmark backup feature (#2182) * feat: Add automated bookmark backup system Implements a comprehensive automated backup feature for user bookmarks with the following capabilities: Database Schema: - Add backupSettings table to store user backup preferences (enabled, frequency, retention) - Add backups table to track backup records with status and metadata - Add BACKUP asset type for storing compressed backup files - Add migration 0066_add_backup_tables.sql Background Workers: - Implement BackupSchedulingWorker cron job (runs daily at midnight UTC) - Create BackupWorker to process individual backup jobs - Deterministic scheduling spreads backup jobs across 24 hours based on user ID hash - Support for daily and weekly backup frequencies - Automated retention cleanup to delete old backups based on user settings Export & Compression: - Reuse existing export functionality for bookmark data - Compress exports using Node.js built-in zlib (gzip level 9) - Store compressed backups as assets with proper metadata - Track backup size and bookmark count for statistics tRPC API: - backups.getSettings - Retrieve user backup configuration - backups.updateSettings - Update backup preferences - backups.list - List all user backups with metadata - backups.get - Get specific backup details - backups.delete - Delete a backup - backups.download - Download backup file (base64 encoded) - backups.triggerBackup - Manually trigger backup creation UI Components: - BackupSettings component with configuration form - Enable/disable automatic backups toggle - Frequency selection (daily/weekly) - Retention period configuration (1-365 days) - Backup list table with download and delete actions - Manual backup trigger button - Display backup stats (size, bookmark count, status) - Added backups page to settings navigation Technical Details: - Uses Restate queue system for distributed job processing - Implements idempotency keys to prevent duplicate backups - Background worker concurrency: 2 jobs at a time - 10-minute timeout for large backup exports - Proper error handling and logging throughout - Type-safe implementation with Zod schemas * refactor: simplify backup settings and asset handling - Move backup settings from separate table to user table columns - Update BackupSettings model to use static methods with users table - Remove download mutation in favor of direct asset links - Implement proper quota checks using QuotaService.checkStorageQuota - Update UI to use new property names and direct asset downloads - Update shared types to match new schema Key changes: - backupSettingsTable removed, settings now in users table - Backup downloads use direct /api/assets/{id} links - Quota properly validated before creating backup assets - Cleaner separation of concerns in tRPC models * migration * use zip instead of gzip * fix drizzle * fix settings * streaming json * remove more dead code * add e2e tests * return backup * poll for backups * more fixes * more fixes * fix test * fix UI * fix delete asset * fix ui * redirect for backup download * cleanups * fix idempotency * fix tests * add ratelimit * add error handling for background backups * i18n * model changes --------- Co-authored-by: Claude --- packages/e2e_tests/tests/api/backups.test.ts | 285 +++++++++++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 packages/e2e_tests/tests/api/backups.test.ts (limited to 'packages/e2e_tests/tests') diff --git a/packages/e2e_tests/tests/api/backups.test.ts b/packages/e2e_tests/tests/api/backups.test.ts new file mode 100644 index 00000000..51c16c5e --- /dev/null +++ b/packages/e2e_tests/tests/api/backups.test.ts @@ -0,0 +1,285 @@ +import AdmZip from "adm-zip"; +import { beforeEach, describe, expect, inject, it } from "vitest"; + +import { createKarakeepClient } from "@karakeep/sdk"; + +import { createTestUser } from "../../utils/api"; + +describe("Backups API", () => { + const port = inject("karakeepPort"); + + if (!port) { + throw new Error("Missing required environment variables"); + } + + let client: ReturnType; + let apiKey: string; + + beforeEach(async () => { + apiKey = await createTestUser(); + client = createKarakeepClient({ + baseUrl: `http://localhost:${port}/api/v1/`, + headers: { + "Content-Type": "application/json", + authorization: `Bearer ${apiKey}`, + }, + }); + }); + + it("should list backups", async () => { + const { data: backupsData, response } = await client.GET("/backups"); + + expect(response.status).toBe(200); + expect(backupsData).toBeDefined(); + expect(backupsData!.backups).toBeDefined(); + expect(Array.isArray(backupsData!.backups)).toBe(true); + }); + + it("should trigger a backup and return the backup record", async () => { + const { data: backup, response } = await client.POST("/backups"); + + expect(response.status).toBe(201); + expect(backup).toBeDefined(); + expect(backup!.id).toBeDefined(); + expect(backup!.userId).toBeDefined(); + expect(backup!.assetId).toBeDefined(); + expect(backup!.status).toBe("pending"); + expect(backup!.size).toBe(0); + expect(backup!.bookmarkCount).toBe(0); + + // Verify the backup appears in the list + const { data: backupsData } = await client.GET("/backups"); + expect(backupsData).toBeDefined(); + expect(backupsData!.backups).toBeDefined(); + expect(backupsData!.backups.some((b) => b.id === backup!.id)).toBe(true); + }); + + it("should get and delete a backup", async () => { + // First trigger a backup + const { data: createdBackup } = await client.POST("/backups"); + expect(createdBackup).toBeDefined(); + + const backupId = createdBackup!.id; + + // Get the specific backup + const { data: backup, response: getResponse } = await client.GET( + "/backups/{backupId}", + { + params: { + path: { + backupId, + }, + }, + }, + ); + + expect(getResponse.status).toBe(200); + expect(backup).toBeDefined(); + expect(backup!.id).toBe(backupId); + expect(backup!.userId).toBeDefined(); + expect(backup!.assetId).toBeDefined(); + expect(backup!.status).toBe("pending"); + + // Delete the backup + const { response: deleteResponse } = await client.DELETE( + "/backups/{backupId}", + { + params: { + path: { + backupId, + }, + }, + }, + ); + + expect(deleteResponse.status).toBe(204); + + // Verify it's deleted + const { response: getDeletedResponse } = await client.GET( + "/backups/{backupId}", + { + params: { + path: { + backupId, + }, + }, + }, + ); + + expect(getDeletedResponse.status).toBe(404); + }); + + it("should return 404 for non-existent backup", async () => { + const { response } = await client.GET("/backups/{backupId}", { + params: { + path: { + backupId: "non-existent-backup-id", + }, + }, + }); + + expect(response.status).toBe(404); + }); + + it("should return 404 when deleting non-existent backup", async () => { + const { response } = await client.DELETE("/backups/{backupId}", { + params: { + path: { + backupId: "non-existent-backup-id", + }, + }, + }); + + expect(response.status).toBe(404); + }); + + it("should handle multiple backups", async () => { + // Trigger multiple backups + const { data: backup1 } = await client.POST("/backups"); + const { data: backup2 } = await client.POST("/backups"); + + expect(backup1).toBeDefined(); + expect(backup2).toBeDefined(); + expect(backup1!.id).not.toBe(backup2!.id); + + // Get all backups + const { data: backupsData, response } = await client.GET("/backups"); + + expect(response.status).toBe(200); + expect(backupsData).toBeDefined(); + expect(backupsData!.backups).toBeDefined(); + expect(Array.isArray(backupsData!.backups)).toBe(true); + expect(backupsData!.backups.length).toBeGreaterThanOrEqual(2); + expect(backupsData!.backups.some((b) => b.id === backup1!.id)).toBe(true); + expect(backupsData!.backups.some((b) => b.id === backup2!.id)).toBe(true); + }); + + it("should validate full backup lifecycle", async () => { + // Step 1: Create some test bookmarks + const bookmarks = []; + for (let i = 0; i < 3; i++) { + const { data: bookmark } = await client.POST("/bookmarks", { + body: { + type: "text", + title: `Test Bookmark ${i + 1}`, + text: `This is test bookmark number ${i + 1}`, + }, + }); + expect(bookmark).toBeDefined(); + bookmarks.push(bookmark!); + } + + // Step 2: Trigger a backup + const { data: createdBackup, response: createResponse } = + await client.POST("/backups"); + + expect(createResponse.status).toBe(201); + expect(createdBackup).toBeDefined(); + expect(createdBackup!.id).toBeDefined(); + expect(createdBackup!.status).toBe("pending"); + expect(createdBackup!.bookmarkCount).toBe(0); + expect(createdBackup!.size).toBe(0); + + const backupId = createdBackup!.id; + + // Step 3: Poll until backup is completed or failed + let backup; + let attempts = 0; + const maxAttempts = 60; // Wait up to 60 seconds + const pollInterval = 1000; // Poll every second + + while (attempts < maxAttempts) { + const { data: currentBackup } = await client.GET("/backups/{backupId}", { + params: { + path: { + backupId, + }, + }, + }); + + backup = currentBackup; + + if (backup!.status === "success" || backup!.status === "failure") { + break; + } + + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + attempts++; + } + + // Step 4: Verify backup completed successfully + expect(backup).toBeDefined(); + expect(backup!.status).toBe("success"); + expect(backup!.bookmarkCount).toBeGreaterThanOrEqual(3); + expect(backup!.size).toBeGreaterThan(0); + expect(backup!.errorMessage).toBeNull(); + + // Step 5: Download the backup + const downloadResponse = await fetch( + `http://localhost:${port}/api/v1/backups/${backupId}/download`, + { + headers: { + authorization: `Bearer ${apiKey}`, + }, + }, + ); + + expect(downloadResponse.status).toBe(200); + expect(downloadResponse.headers.get("content-type")).toContain( + "application/zip", + ); + + const backupBlob = await downloadResponse.blob(); + expect(backupBlob.size).toBeGreaterThan(0); + expect(backupBlob.size).toBe(backup!.size); + + // Step 6: Unzip and validate the backup contents + const arrayBuffer = await backupBlob.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + + // Verify it's a valid ZIP file (starts with PK signature) + expect(buffer[0]).toBe(0x50); // 'P' + expect(buffer[1]).toBe(0x4b); // 'K' + + // Unzip the backup file + const zip = new AdmZip(buffer); + const zipEntries = zip.getEntries(); + + // Should contain exactly one JSON file + expect(zipEntries.length).toBe(1); + const jsonEntry = zipEntries[0]; + expect(jsonEntry.entryName).toMatch(/^karakeep-backup-.*\.json$/); + + // Extract and parse the JSON content + const jsonContent = jsonEntry.getData().toString("utf8"); + const backupData = JSON.parse(jsonContent); + + // Validate the backup structure + expect(backupData).toBeDefined(); + expect(backupData.bookmarks).toBeDefined(); + expect(Array.isArray(backupData.bookmarks)).toBe(true); + expect(backupData.bookmarks.length).toBeGreaterThanOrEqual(3); + + // Validate that our test bookmarks are in the backup + const backupTitles = backupData.bookmarks.map( + (b: { title: string }) => b.title, + ); + expect(backupTitles).toContain("Test Bookmark 1"); + expect(backupTitles).toContain("Test Bookmark 2"); + expect(backupTitles).toContain("Test Bookmark 3"); + + // Validate bookmark structure + const firstBookmark = backupData.bookmarks[0]; + expect(firstBookmark).toHaveProperty("content"); + expect(firstBookmark.content).toHaveProperty("type"); + + // Step 7: Verify the backup appears in the list with updated status + const { data: backupsData } = await client.GET("/backups"); + const listedBackup = backupsData!.backups.find((b) => b.id === backupId); + + expect(listedBackup).toBeDefined(); + expect(listedBackup!.status).toBe("success"); + expect(listedBackup!.bookmarkCount).toBe(backup!.bookmarkCount); + expect(listedBackup!.size).toBe(backup!.size); + }); +}); -- cgit v1.2.3-70-g09d2