From 86a4b3966504507afd6c3adbb6a1246cafd39d83 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sat, 29 Nov 2025 14:53:31 +0000 Subject: feat: Add automated bookmark backup feature (#2182) * feat: Add automated bookmark backup system Implements a comprehensive automated backup feature for user bookmarks with the following capabilities: Database Schema: - Add backupSettings table to store user backup preferences (enabled, frequency, retention) - Add backups table to track backup records with status and metadata - Add BACKUP asset type for storing compressed backup files - Add migration 0066_add_backup_tables.sql Background Workers: - Implement BackupSchedulingWorker cron job (runs daily at midnight UTC) - Create BackupWorker to process individual backup jobs - Deterministic scheduling spreads backup jobs across 24 hours based on user ID hash - Support for daily and weekly backup frequencies - Automated retention cleanup to delete old backups based on user settings Export & Compression: - Reuse existing export functionality for bookmark data - Compress exports using Node.js built-in zlib (gzip level 9) - Store compressed backups as assets with proper metadata - Track backup size and bookmark count for statistics tRPC API: - backups.getSettings - Retrieve user backup configuration - backups.updateSettings - Update backup preferences - backups.list - List all user backups with metadata - backups.get - Get specific backup details - backups.delete - Delete a backup - backups.download - Download backup file (base64 encoded) - backups.triggerBackup - Manually trigger backup creation UI Components: - BackupSettings component with configuration form - Enable/disable automatic backups toggle - Frequency selection (daily/weekly) - Retention period configuration (1-365 days) - Backup list table with download and delete actions - Manual backup trigger button - Display backup stats (size, bookmark count, status) - Added backups page to settings navigation Technical Details: - Uses Restate queue system for distributed job processing - Implements idempotency keys to prevent duplicate backups - Background worker concurrency: 2 jobs at a time - 10-minute timeout for large backup exports - Proper error handling and logging throughout - Type-safe implementation with Zod schemas * refactor: simplify backup settings and asset handling - Move backup settings from separate table to user table columns - Update BackupSettings model to use static methods with users table - Remove download mutation in favor of direct asset links - Implement proper quota checks using QuotaService.checkStorageQuota - Update UI to use new property names and direct asset downloads - Update shared types to match new schema Key changes: - backupSettingsTable removed, settings now in users table - Backup downloads use direct /api/assets/{id} links - Quota properly validated before creating backup assets - Cleaner separation of concerns in tRPC models * migration * use zip instead of gzip * fix drizzle * fix settings * streaming json * remove more dead code * add e2e tests * return backup * poll for backups * more fixes * more fixes * fix test * fix UI * fix delete asset * fix ui * redirect for backup download * cleanups * fix idempotency * fix tests * add ratelimit * add error handling for background backups * i18n * model changes --------- Co-authored-by: Claude --- packages/db/schema.ts | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) (limited to 'packages/db/schema.ts') diff --git a/packages/db/schema.ts b/packages/db/schema.ts index d5fd162d..8f259d04 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -58,6 +58,17 @@ export const users = sqliteTable("user", { .notNull() .default("show"), timezone: text("timezone").default("UTC"), + + // Backup Settings + backupsEnabled: integer("backupsEnabled", { mode: "boolean" }) + .notNull() + .default(false), + backupsFrequency: text("backupsFrequency", { + enum: ["daily", "weekly"], + }) + .notNull() + .default("weekly"), + backupsRetentionDays: integer("backupsRetentionDays").notNull().default(30), }); export const accounts = sqliteTable( @@ -229,6 +240,7 @@ export const enum AssetTypes { LINK_HTML_CONTENT = "linkHtmlContent", BOOKMARK_ASSET = "bookmarkAsset", USER_UPLOADED = "userUploaded", + BACKUP = "backup", UNKNOWN = "unknown", } @@ -248,6 +260,7 @@ export const assets = sqliteTable( AssetTypes.LINK_HTML_CONTENT, AssetTypes.BOOKMARK_ASSET, AssetTypes.USER_UPLOADED, + AssetTypes.BACKUP, AssetTypes.UNKNOWN, ], }).notNull(), @@ -574,6 +587,35 @@ export const rssFeedImportsTable = sqliteTable( ], ); +export const backupsTable = sqliteTable( + "backups", + { + id: text("id") + .notNull() + .primaryKey() + .$defaultFn(() => createId()), + userId: text("userId") + .notNull() + .references(() => users.id, { onDelete: "cascade" }), + assetId: text("assetId").references(() => assets.id, { + onDelete: "cascade", + }), + createdAt: createdAtField(), + size: integer("size").notNull(), + bookmarkCount: integer("bookmarkCount").notNull(), + status: text("status", { + enum: ["pending", "success", "failure"], + }) + .notNull() + .default("pending"), + errorMessage: text("errorMessage"), + }, + (b) => [ + index("backups_userId_idx").on(b.userId), + index("backups_createdAt_idx").on(b.createdAt), + ], +); + export const config = sqliteTable("config", { key: text("key").notNull().primaryKey(), value: text("value").notNull(), @@ -766,6 +808,7 @@ export const userRelations = relations(users, ({ many, one }) => ({ subscription: one(subscriptions), importSessions: many(importSessions), listCollaborations: many(listCollaborators), + backups: many(backupsTable), listInvitations: many(listInvitations), })); @@ -989,3 +1032,14 @@ export const importSessionBookmarksRelations = relations( }), }), ); + +export const backupsRelations = relations(backupsTable, ({ one }) => ({ + user: one(users, { + fields: [backupsTable.userId], + references: [users.id], + }), + asset: one(assets, { + fields: [backupsTable.assetId], + references: [assets.id], + }), +})); -- cgit v1.2.3-70-g09d2