path: root/apps/cli/src/commands/dump.ts

import { spawn } from "node:child_process";
import fsp from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { getGlobalOptions } from "@/lib/globals";
import { printErrorMessageWithReason, printStatusMessage } from "@/lib/output";
import { getAPIClient } from "@/lib/trpc";
import { Command } from "@commander-js/extra-typings";
import chalk from "chalk";

import type { ZBookmark } from "@karakeep/shared/types/bookmarks";
import type { ZBookmarkList } from "@karakeep/shared/types/lists";
import { MAX_NUM_BOOKMARKS_PER_PAGE } from "@karakeep/shared/types/bookmarks";
import { ZCursor } from "@karakeep/shared/types/pagination";

const OK = chalk.green("✓");
const FAIL = chalk.red("✗");
const DOTS = chalk.gray("…");

function line(msg: string) {
  console.log(msg);
}

function stepStart(title: string) {
  console.log(`${chalk.cyan(title)} ${DOTS}`);
}

function stepEndSuccess(extra?: string) {
  process.stdout.write(`${OK}${extra ? " " + chalk.gray(extra) : ""}\n`);
}

function stepEndFail(extra?: string) {
  process.stdout.write(`${FAIL}${extra ? " " + chalk.gray(extra) : ""}\n`);
}

function progressUpdate(
  prefix: string,
  current: number,
  total?: number,
  suffix?: string,
) {
  const totalPart = total != null ? `/${total}` : "";
  const text = `${chalk.gray(prefix)} ${current}${totalPart}${suffix ? " " + chalk.gray(suffix) : ""}`;
  if (process.stdout.isTTY) {
    try {
      process.stdout.clearLine(0);
      process.stdout.cursorTo(0);
      process.stdout.write(text);
      return;
    } catch {
      // ignore failures
    }
  }
  console.log(text);
}

function progressDone() {
  process.stdout.write("\n");
}

async function ensureDir(p: string) {
  await fsp.mkdir(p, { recursive: true });
}

async function writeJson(filePath: string, data: unknown) {
  const dir = path.dirname(filePath);
  await ensureDir(dir);
  await fsp.writeFile(filePath, JSON.stringify(data, null, 2), "utf-8");
}

async function writeJsonl(
  filePath: string,
  items: AsyncIterable<unknown> | Iterable<unknown>,
) {
  const dir = path.dirname(filePath);
  await ensureDir(dir);
  const fh = await fsp.open(filePath, "w");
  try {
    for await (const item of items) {
      await fh.write(JSON.stringify(item) + "\n");
    }
  } finally {
    await fh.close();
  }
}

async function createTarGz(srcDir: string, outFile: string): Promise<void> {
  await ensureDir(path.dirname(outFile));
  await new Promise<void>((resolve, reject) => {
    const tar = spawn("tar", ["-czf", outFile, "-C", srcDir, "."], {
      stdio: "inherit",
    });
    tar.on("error", reject);
    tar.on("close", (code) => {
      if (code === 0) resolve();
      else reject(new Error(`tar exited with code ${code}`));
    });
  });
}

export const dumpCmd = new Command()
  .name("dump")
  .description("dump all account data and assets into an archive")
  .option("--output <file>", "output archive path (.tar.gz)")
  .option(
    "--batch-size <n>",
    `number of bookmarks per page (max ${MAX_NUM_BOOKMARKS_PER_PAGE})`,
    (v) => Math.min(Number(v || 50), MAX_NUM_BOOKMARKS_PER_PAGE),
    50,
  )
  .action(async (opts) => {
    const api = getAPIClient();
    const globals = getGlobalOptions();

    const ts = new Date()
      .toISOString()
      .replace(/[:.]/g, "-")
      .replace("T", "_")
      .replace("Z", "Z");
    const workRoot = await fsp.mkdtemp(
      path.join(os.tmpdir(), `karakeep-dump-${ts}-`),
    );
    const outFile = opts.output ?? path.resolve(`karakeep-dump-${ts}.tar.gz`);

    try {
      line("");
      line(`${chalk.bold("Karakeep Dump")}`);
      line(`${chalk.gray("Server:")} ${globals.serverAddr}`);
      line(`${chalk.gray("Output:")} ${outFile}`);
      line("");

      // Manifest skeleton
      const whoami = await api.users.whoami.query();
      const manifest = {
        format: "karakeep.dump",
        version: 1,
        exportedAt: new Date().toISOString(),
        server: globals.serverAddr,
        user: {
          id: whoami.id,
          email: whoami.email,
          name: whoami.name,
        },
        counts: {
          bookmarks: 0,
          assets: 0,
          lists: 0,
          tags: 0,
          rules: 0,
          feeds: 0,
          webhooks: 0,
          prompts: 0,
        },
      };

      // 1) User settings
      stepStart("Exporting user settings");
      const settings = await api.users.settings.query();
      await writeJson(path.join(workRoot, "users", "settings.json"), settings);
      stepEndSuccess();

      // 2) Lists
      stepStart("Exporting lists");
      const { lists } = await api.lists.list.query();
      await writeJson(path.join(workRoot, "lists", "index.json"), lists);
      manifest.counts.lists = lists.length;
      stepEndSuccess();

      // 3) Tags
      stepStart("Exporting tags");
      const { tags } = await api.tags.list.query();
      await writeJson(path.join(workRoot, "tags", "index.json"), tags);
      manifest.counts.tags = tags.length;
      stepEndSuccess();

      // 4) Rules
      stepStart("Exporting rules");
      const { rules } = await api.rules.list.query();
      await writeJson(path.join(workRoot, "rules", "index.json"), rules);
      manifest.counts.rules = rules.length;
      stepEndSuccess();

      // 5) Feeds
      stepStart("Exporting feeds");
      const { feeds } = await api.feeds.list.query();
      await writeJson(path.join(workRoot, "feeds", "index.json"), feeds);
      manifest.counts.feeds = feeds.length;
      stepEndSuccess();

      // 6) Prompts
      stepStart("Exporting AI prompts");
      const prompts = await api.prompts.list.query();
      await writeJson(path.join(workRoot, "prompts", "index.json"), prompts);
      manifest.counts.prompts = prompts.length;
      stepEndSuccess();

      // 7) Webhooks
      stepStart("Exporting webhooks");
      const webhooks = await api.webhooks.list.query();
      await writeJson(
        path.join(workRoot, "webhooks", "index.json"),
        webhooks.webhooks,
      );
      manifest.counts.webhooks = webhooks.webhooks.length;
      stepEndSuccess();

      // 8) Bookmarks (JSONL + list membership)
      stepStart("Exporting bookmarks (metadata/content)");
      const bookmarkJsonl = path.join(workRoot, "bookmarks", "index.jsonl");
      let bookmarksExported = 0;
      const bookmarkIterator = async function* (): AsyncGenerator<ZBookmark> {
        let cursor: ZCursor | null = null;
        do {
          const resp = await api.bookmarks.getBookmarks.query({
            includeContent: true,
            limit: Number(opts.batchSize) || 50,
            cursor,
            useCursorV2: true,
          });
          for (const b of resp.bookmarks) {
            yield b;
            bookmarksExported++;
            progressUpdate("Bookmarks", bookmarksExported);
          }
          cursor = resp.nextCursor;
        } while (cursor);
      };
      await writeJsonl(bookmarkJsonl, bookmarkIterator());
      progressDone();
      manifest.counts.bookmarks = bookmarksExported;
      stepEndSuccess();

      // 9) List membership (listId -> [bookmarkId])
      stepStart("Exporting list membership");
      const membership = await buildListMembership(api, lists, (p, t) =>
        progressUpdate("Lists scanned", p, t),
      );
      progressDone();
      await writeJson(
        path.join(workRoot, "lists", "membership.json"),
        Object.fromEntries(membership.entries()),
      );
      stepEndSuccess();

      // 10) Assets: index + files
      stepStart("Exporting assets (binary files)");
      const assetsDir = path.join(workRoot, "assets", "files");
      await ensureDir(assetsDir);
      const assetsIndex: {
        id: string;
        assetType: string;
        size: number;
        contentType: string | null;
        fileName: string | null;
        bookmarkId: string | null;
        filePath: string; // relative inside archive
      }[] = [];
      let assetPageCursor: number | null | undefined = null;
      let downloaded = 0;
      let totalAssets: number | undefined = undefined;
      do {
        const resp = await api.assets.list.query({
          limit: 50,
          cursor: assetPageCursor ?? undefined,
        });
        if (totalAssets == null) totalAssets = resp.totalCount;
        for (const a of resp.assets) {
          const relPath = path.join("assets", "files", a.id);
          const absPath = path.join(workRoot, relPath);
          try {
            await downloadAsset(
              globals.serverAddr,
              globals.apiKey,
              a.id,
              absPath,
            );
            assetsIndex.push({
              id: a.id,
              assetType: a.assetType,
              size: a.size,
              contentType: a.contentType,
              fileName: a.fileName,
              bookmarkId: a.bookmarkId,
              filePath: relPath.replace(/\\/g, "/"),
            });
            downloaded++;
            progressUpdate("Assets", downloaded, totalAssets);
          } catch (e) {
            printErrorMessageWithReason(
              `Failed to download asset "${a.id}"`,
              e as object,
            );
          }
        }
        assetPageCursor = resp.nextCursor;
      } while (assetPageCursor);
      progressDone();
      manifest.counts.assets = downloaded;
      await writeJson(path.join(workRoot, "assets", "index.json"), assetsIndex);
      stepEndSuccess();

      // 11) Manifest
      stepStart("Writing manifest");
      await writeJson(path.join(workRoot, "manifest.json"), manifest);
      stepEndSuccess();

      // 12) Create archive
      stepStart("Creating archive");
      await createTarGz(workRoot, outFile);
      stepEndSuccess();

      printStatusMessage(true, `Dump completed. File: ${outFile}`);
    } catch (error) {
      stepEndFail();
      printErrorMessageWithReason("Dump failed", error as object);
      throw error;
    } finally {
      // Best-effort cleanup of temp directory
      try {
        await fsp.rm(workRoot, { recursive: true, force: true });
      } catch {
        // ignore
      }
    }
  });

async function buildListMembership(
  api: ReturnType<typeof getAPIClient>,
  lists: ZBookmarkList[],
  onProgress?: (processed: number, total: number) => void,
) {
  const result = new Map<string, string[]>(); // listId -> [bookmarkId]
  let processed = 0;
  for (const l of lists) {
    // Only manual lists have explicit membership
    if (l.type !== "manual") {
      processed++;
      onProgress?.(processed, lists.length);
      continue;
    }
    let cursor: ZCursor | null = null;
    const ids: string[] = [];
    do {
      const resp = await api.bookmarks.getBookmarks.query({
        listId: l.id,
        limit: MAX_NUM_BOOKMARKS_PER_PAGE,
        cursor,
        includeContent: false,
        useCursorV2: true,
      });
      for (const b of resp.bookmarks) ids.push(b.id);
      cursor = resp.nextCursor;
    } while (cursor);
    result.set(l.id, ids);
    processed++;
    onProgress?.(processed, lists.length);
  }
  return result;
}

async function downloadAsset(
  serverAddr: string,
  apiKey: string,
  assetId: string,
  destFile: string,
) {
  const url = `${serverAddr}/api/assets/${assetId}`;
  const resp = await fetch(url, {
    headers: { authorization: `Bearer ${apiKey}` },
  });
  if (!resp.ok) {
    throw new Error(`HTTP ${resp.status} ${resp.statusText}`);
  }
  const arrayBuf = await resp.arrayBuffer();
  await ensureDir(path.dirname(destFile));
  await fsp.writeFile(destFile, Buffer.from(arrayBuf));
}