feat(cli): Implement a full account dump archive

1 files changed, 377 insertions, 0 deletions

diff --git a/apps/cli/src/commands/dump.ts b/apps/cli/src/commands/dump.ts
new file mode 100644
index 00000000..2e126654
--- /dev/null
+++ b/apps/cli/src/commands/dump.ts
@@ -0,0 +1,377 @@
+import { spawn } from "node:child_process";
+import fsp from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { getGlobalOptions } from "@/lib/globals";
+import { printErrorMessageWithReason, printStatusMessage } from "@/lib/output";
+import { getAPIClient } from "@/lib/trpc";
+import { Command } from "@commander-js/extra-typings";
+import chalk from "chalk";
+
+import type { ZBookmark } from "@karakeep/shared/types/bookmarks";
+import type { ZBookmarkList } from "@karakeep/shared/types/lists";
+import { MAX_NUM_BOOKMARKS_PER_PAGE } from "@karakeep/shared/types/bookmarks";
+import { ZCursor } from "@karakeep/shared/types/pagination";
+
+const OK = chalk.green("✓");
+const FAIL = chalk.red("✗");
+const DOTS = chalk.gray("…");
+
+function line(msg: string) {
+  console.log(msg);
+}
+
+function stepStart(title: string) {
+  console.log(`${chalk.cyan(title)} ${DOTS}`);
+}
+
+function stepEndSuccess(extra?: string) {
+  process.stdout.write(`${OK}${extra ? " " + chalk.gray(extra) : ""}\n`);
+}
+
+function stepEndFail(extra?: string) {
+  process.stdout.write(`${FAIL}${extra ? " " + chalk.gray(extra) : ""}\n`);
+}
+
+function progressUpdate(
+  prefix: string,
+  current: number,
+  total?: number,
+  suffix?: string,
+) {
+  const totalPart = total != null ? `/${total}` : "";
+  const text = `${chalk.gray(prefix)} ${current}${totalPart}${suffix ? " " + chalk.gray(suffix) : ""}`;
+  if (process.stdout.isTTY) {
+    try {
+      process.stdout.clearLine(0);
+      process.stdout.cursorTo(0);
+      process.stdout.write(text);
+      return;
+    } catch {
+      // ignore failures
+    }
+  }
+  console.log(text);
+}
+
+function progressDone() {
+  process.stdout.write("\n");
+}
+
+async function ensureDir(p: string) {
+  await fsp.mkdir(p, { recursive: true });
+}
+
+async function writeJson(filePath: string, data: unknown) {
+  const dir = path.dirname(filePath);
+  await ensureDir(dir);
+  await fsp.writeFile(filePath, JSON.stringify(data, null, 2), "utf-8");
+}
+
+async function writeJsonl(
+  filePath: string,
+  items: AsyncIterable<unknown> | Iterable<unknown>,
+) {
+  const dir = path.dirname(filePath);
+  await ensureDir(dir);
+  const fh = await fsp.open(filePath, "w");
+  try {
+    for await (const item of items) {
+      await fh.write(JSON.stringify(item) + "\n");
+    }
+  } finally {
+    await fh.close();
+  }
+}
+
+async function createTarGz(srcDir: string, outFile: string): Promise<void> {
+  await ensureDir(path.dirname(outFile));
+  await new Promise<void>((resolve, reject) => {
+    const tar = spawn("tar", ["-czf", outFile, "-C", srcDir, "."], {
+      stdio: "inherit",
+    });
+    tar.on("error", reject);
+    tar.on("close", (code) => {
+      if (code === 0) resolve();
+      else reject(new Error(`tar exited with code ${code}`));
+    });
+  });
+}
+
+export const dumpCmd = new Command()
+  .name("dump")
+  .description("dump all account data and assets into an archive")
+  .option("--output <file>", "output archive path (.tar.gz)")
+  .option(
+    "--batch-size <n>",
+    `number of bookmarks per page (max ${MAX_NUM_BOOKMARKS_PER_PAGE})`,
+    (v) => Math.min(Number(v || 50), MAX_NUM_BOOKMARKS_PER_PAGE),
+    50,
+  )
+  .action(async (opts) => {
+    const api = getAPIClient();
+    const globals = getGlobalOptions();
+
+    const ts = new Date()
+      .toISOString()
+      .replace(/[:.]/g, "-")
+      .replace("T", "_")
+      .replace("Z", "Z");
+    const workRoot = await fsp.mkdtemp(
+      path.join(os.tmpdir(), `karakeep-dump-${ts}-`),
+    );
+    const outFile = opts.output ?? path.resolve(`karakeep-dump-${ts}.tar.gz`);
+
+    try {
+      line("");
+      line(`${chalk.bold("Karakeep Dump")}`);
+      line(`${chalk.gray("Server:")} ${globals.serverAddr}`);
+      line(`${chalk.gray("Output:")} ${outFile}`);
+      line("");
+
+      // Manifest skeleton
+      const whoami = await api.users.whoami.query();
+      const manifest = {
+        format: "karakeep.dump",
+        version: 1,
+        exportedAt: new Date().toISOString(),
+        server: globals.serverAddr,
+        user: {
+          id: whoami.id,
+          email: whoami.email,
+          name: whoami.name,
+        },
+        counts: {
+          bookmarks: 0,
+          assets: 0,
+          lists: 0,
+          tags: 0,
+          rules: 0,
+          feeds: 0,
+          webhooks: 0,
+          prompts: 0,
+        },
+      };
+
+      // 1) User settings
+      stepStart("Exporting user settings");
+      const settings = await api.users.settings.query();
+      await writeJson(path.join(workRoot, "users", "settings.json"), settings);
+      stepEndSuccess();
+
+      // 2) Lists
+      stepStart("Exporting lists");
+      const { lists } = await api.lists.list.query();
+      await writeJson(path.join(workRoot, "lists", "index.json"), lists);
+      manifest.counts.lists = lists.length;
+      stepEndSuccess();
+
+      // 3) Tags
+      stepStart("Exporting tags");
+      const { tags } = await api.tags.list.query();
+      await writeJson(path.join(workRoot, "tags", "index.json"), tags);
+      manifest.counts.tags = tags.length;
+      stepEndSuccess();
+
+      // 4) Rules
+      stepStart("Exporting rules");
+      const { rules } = await api.rules.list.query();
+      await writeJson(path.join(workRoot, "rules", "index.json"), rules);
+      manifest.counts.rules = rules.length;
+      stepEndSuccess();
+
+      // 5) Feeds
+      stepStart("Exporting feeds");
+      const { feeds } = await api.feeds.list.query();
+      await writeJson(path.join(workRoot, "feeds", "index.json"), feeds);
+      manifest.counts.feeds = feeds.length;
+      stepEndSuccess();
+
+      // 6) Prompts
+      stepStart("Exporting AI prompts");
+      const prompts = await api.prompts.list.query();
+      await writeJson(path.join(workRoot, "prompts", "index.json"), prompts);
+      manifest.counts.prompts = prompts.length;
+      stepEndSuccess();
+
+      // 7) Webhooks
+      stepStart("Exporting webhooks");
+      const webhooks = await api.webhooks.list.query();
+      await writeJson(
+        path.join(workRoot, "webhooks", "index.json"),
+        webhooks.webhooks,
+      );
+      manifest.counts.webhooks = webhooks.webhooks.length;
+      stepEndSuccess();
+
+      // 8) Bookmarks (JSONL + list membership)
+      stepStart("Exporting bookmarks (metadata/content)");
+      const bookmarkJsonl = path.join(workRoot, "bookmarks", "index.jsonl");
+      let bookmarksExported = 0;
+      const bookmarkIterator = async function* (): AsyncGenerator<ZBookmark> {
+        let cursor: ZCursor | null = null;
+        do {
+          const resp = await api.bookmarks.getBookmarks.query({
+            includeContent: true,
+            limit: Number(opts.batchSize) || 50,
+            cursor,
+            useCursorV2: true,
+          });
+          for (const b of resp.bookmarks) {
+            yield b;
+            bookmarksExported++;
+            progressUpdate("Bookmarks", bookmarksExported);
+          }
+          cursor = resp.nextCursor;
+        } while (cursor);
+      };
+      await writeJsonl(bookmarkJsonl, bookmarkIterator());
+      progressDone();
+      manifest.counts.bookmarks = bookmarksExported;
+      stepEndSuccess();
+
+      // 9) List membership (listId -> [bookmarkId])
+      stepStart("Exporting list membership");
+      const membership = await buildListMembership(api, lists, (p, t) =>
+        progressUpdate("Lists scanned", p, t),
+      );
+      progressDone();
+      await writeJson(
+        path.join(workRoot, "lists", "membership.json"),
+        Object.fromEntries(membership.entries()),
+      );
+      stepEndSuccess();
+
+      // 10) Assets: index + files
+      stepStart("Exporting assets (binary files)");
+      const assetsDir = path.join(workRoot, "assets", "files");
+      await ensureDir(assetsDir);
+      const assetsIndex: {
+        id: string;
+        assetType: string;
+        size: number;
+        contentType: string | null;
+        fileName: string | null;
+        bookmarkId: string | null;
+        filePath: string; // relative inside archive
+      }[] = [];
+      let assetPageCursor: number | null | undefined = null;
+      let downloaded = 0;
+      let totalAssets: number | undefined = undefined;
+      do {
+        const resp = await api.assets.list.query({
+          limit: 50,
+          cursor: assetPageCursor ?? undefined,
+        });
+        if (totalAssets == null) totalAssets = resp.totalCount;
+        for (const a of resp.assets) {
+          const relPath = path.join("assets", "files", a.id);
+          const absPath = path.join(workRoot, relPath);
+          try {
+            await downloadAsset(
+              globals.serverAddr,
+              globals.apiKey,
+              a.id,
+              absPath,
+            );
+            assetsIndex.push({
+              id: a.id,
+              assetType: a.assetType,
+              size: a.size,
+              contentType: a.contentType,
+              fileName: a.fileName,
+              bookmarkId: a.bookmarkId,
+              filePath: relPath.replace(/\\/g, "/"),
+            });
+            downloaded++;
+            progressUpdate("Assets", downloaded, totalAssets);
+          } catch (e) {
+            printErrorMessageWithReason(
+              `Failed to download asset "${a.id}"`,
+              e as object,
+            );
+          }
+        }
+        assetPageCursor = resp.nextCursor;
+      } while (assetPageCursor);
+      progressDone();
+      manifest.counts.assets = downloaded;
+      await writeJson(path.join(workRoot, "assets", "index.json"), assetsIndex);
+      stepEndSuccess();
+
+      // 11) Manifest
+      stepStart("Writing manifest");
+      await writeJson(path.join(workRoot, "manifest.json"), manifest);
+      stepEndSuccess();
+
+      // 12) Create archive
+      stepStart("Creating archive");
+      await createTarGz(workRoot, outFile);
+      stepEndSuccess();
+
+      printStatusMessage(true, `Dump completed. File: ${outFile}`);
+    } catch (error) {
+      stepEndFail();
+      printErrorMessageWithReason("Dump failed", error as object);
+      throw error;
+    } finally {
+      // Best-effort cleanup of temp directory
+      try {
+        await fsp.rm(workRoot, { recursive: true, force: true });
+      } catch {
+        // ignore
+      }
+    }
+  });
+
+async function buildListMembership(
+  api: ReturnType<typeof getAPIClient>,
+  lists: ZBookmarkList[],
+  onProgress?: (processed: number, total: number) => void,
+) {
+  const result = new Map<string, string[]>(); // listId -> [bookmarkId]
+  let processed = 0;
+  for (const l of lists) {
+    // Only manual lists have explicit membership
+    if (l.type !== "manual") {
+      processed++;
+      onProgress?.(processed, lists.length);
+      continue;
+    }
+    let cursor: ZCursor | null = null;
+    const ids: string[] = [];
+    do {
+      const resp = await api.bookmarks.getBookmarks.query({
+        listId: l.id,
+        limit: MAX_NUM_BOOKMARKS_PER_PAGE,
+        cursor,
+        includeContent: false,
+        useCursorV2: true,
+      });
+      for (const b of resp.bookmarks) ids.push(b.id);
+      cursor = resp.nextCursor;
+    } while (cursor);
+    result.set(l.id, ids);
+    processed++;
+    onProgress?.(processed, lists.length);
+  }
+  return result;
+}
+
+async function downloadAsset(
+  serverAddr: string,
+  apiKey: string,
+  assetId: string,
+  destFile: string,
+) {
+  const url = `${serverAddr}/api/assets/${assetId}`;
+  const resp = await fetch(url, {
+    headers: { authorization: `Bearer ${apiKey}` },
+  });
+  if (!resp.ok) {
+    throw new Error(`HTTP ${resp.status} ${resp.statusText}`);
+  }
+  const arrayBuf = await resp.arrayBuffer();
+  await ensureDir(path.dirname(destFile));
+  await fsp.writeFile(destFile, Buffer.from(arrayBuf));
+}