diff options
Diffstat (limited to 'apps/cli/src')
| -rw-r--r-- | apps/cli/src/commands/dump.ts | 377 | ||||
| -rw-r--r-- | apps/cli/src/index.ts | 2 |
2 files changed, 379 insertions, 0 deletions
diff --git a/apps/cli/src/commands/dump.ts b/apps/cli/src/commands/dump.ts new file mode 100644 index 00000000..2e126654 --- /dev/null +++ b/apps/cli/src/commands/dump.ts @@ -0,0 +1,377 @@ +import { spawn } from "node:child_process"; +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { getGlobalOptions } from "@/lib/globals"; +import { printErrorMessageWithReason, printStatusMessage } from "@/lib/output"; +import { getAPIClient } from "@/lib/trpc"; +import { Command } from "@commander-js/extra-typings"; +import chalk from "chalk"; + +import type { ZBookmark } from "@karakeep/shared/types/bookmarks"; +import type { ZBookmarkList } from "@karakeep/shared/types/lists"; +import { MAX_NUM_BOOKMARKS_PER_PAGE } from "@karakeep/shared/types/bookmarks"; +import { ZCursor } from "@karakeep/shared/types/pagination"; + +const OK = chalk.green("✓"); +const FAIL = chalk.red("✗"); +const DOTS = chalk.gray("…"); + +function line(msg: string) { + console.log(msg); +} + +function stepStart(title: string) { + console.log(`${chalk.cyan(title)} ${DOTS}`); +} + +function stepEndSuccess(extra?: string) { + process.stdout.write(`${OK}${extra ? " " + chalk.gray(extra) : ""}\n`); +} + +function stepEndFail(extra?: string) { + process.stdout.write(`${FAIL}${extra ? " " + chalk.gray(extra) : ""}\n`); +} + +function progressUpdate( + prefix: string, + current: number, + total?: number, + suffix?: string, +) { + const totalPart = total != null ? `/${total}` : ""; + const text = `${chalk.gray(prefix)} ${current}${totalPart}${suffix ? " " + chalk.gray(suffix) : ""}`; + if (process.stdout.isTTY) { + try { + process.stdout.clearLine(0); + process.stdout.cursorTo(0); + process.stdout.write(text); + return; + } catch { + // ignore failures + } + } + console.log(text); +} + +function progressDone() { + process.stdout.write("\n"); +} + +async function ensureDir(p: string) { + await fsp.mkdir(p, { recursive: true }); +} + +async function writeJson(filePath: string, data: unknown) { + const dir = path.dirname(filePath); + await ensureDir(dir); + await fsp.writeFile(filePath, JSON.stringify(data, null, 2), "utf-8"); +} + +async function writeJsonl( + filePath: string, + items: AsyncIterable<unknown> | Iterable<unknown>, +) { + const dir = path.dirname(filePath); + await ensureDir(dir); + const fh = await fsp.open(filePath, "w"); + try { + for await (const item of items) { + await fh.write(JSON.stringify(item) + "\n"); + } + } finally { + await fh.close(); + } +} + +async function createTarGz(srcDir: string, outFile: string): Promise<void> { + await ensureDir(path.dirname(outFile)); + await new Promise<void>((resolve, reject) => { + const tar = spawn("tar", ["-czf", outFile, "-C", srcDir, "."], { + stdio: "inherit", + }); + tar.on("error", reject); + tar.on("close", (code) => { + if (code === 0) resolve(); + else reject(new Error(`tar exited with code ${code}`)); + }); + }); +} + +export const dumpCmd = new Command() + .name("dump") + .description("dump all account data and assets into an archive") + .option("--output <file>", "output archive path (.tar.gz)") + .option( + "--batch-size <n>", + `number of bookmarks per page (max ${MAX_NUM_BOOKMARKS_PER_PAGE})`, + (v) => Math.min(Number(v || 50), MAX_NUM_BOOKMARKS_PER_PAGE), + 50, + ) + .action(async (opts) => { + const api = getAPIClient(); + const globals = getGlobalOptions(); + + const ts = new Date() + .toISOString() + .replace(/[:.]/g, "-") + .replace("T", "_") + .replace("Z", "Z"); + const workRoot = await fsp.mkdtemp( + path.join(os.tmpdir(), `karakeep-dump-${ts}-`), + ); + const outFile = opts.output ?? path.resolve(`karakeep-dump-${ts}.tar.gz`); + + try { + line(""); + line(`${chalk.bold("Karakeep Dump")}`); + line(`${chalk.gray("Server:")} ${globals.serverAddr}`); + line(`${chalk.gray("Output:")} ${outFile}`); + line(""); + + // Manifest skeleton + const whoami = await api.users.whoami.query(); + const manifest = { + format: "karakeep.dump", + version: 1, + exportedAt: new Date().toISOString(), + server: globals.serverAddr, + user: { + id: whoami.id, + email: whoami.email, + name: whoami.name, + }, + counts: { + bookmarks: 0, + assets: 0, + lists: 0, + tags: 0, + rules: 0, + feeds: 0, + webhooks: 0, + prompts: 0, + }, + }; + + // 1) User settings + stepStart("Exporting user settings"); + const settings = await api.users.settings.query(); + await writeJson(path.join(workRoot, "users", "settings.json"), settings); + stepEndSuccess(); + + // 2) Lists + stepStart("Exporting lists"); + const { lists } = await api.lists.list.query(); + await writeJson(path.join(workRoot, "lists", "index.json"), lists); + manifest.counts.lists = lists.length; + stepEndSuccess(); + + // 3) Tags + stepStart("Exporting tags"); + const { tags } = await api.tags.list.query(); + await writeJson(path.join(workRoot, "tags", "index.json"), tags); + manifest.counts.tags = tags.length; + stepEndSuccess(); + + // 4) Rules + stepStart("Exporting rules"); + const { rules } = await api.rules.list.query(); + await writeJson(path.join(workRoot, "rules", "index.json"), rules); + manifest.counts.rules = rules.length; + stepEndSuccess(); + + // 5) Feeds + stepStart("Exporting feeds"); + const { feeds } = await api.feeds.list.query(); + await writeJson(path.join(workRoot, "feeds", "index.json"), feeds); + manifest.counts.feeds = feeds.length; + stepEndSuccess(); + + // 6) Prompts + stepStart("Exporting AI prompts"); + const prompts = await api.prompts.list.query(); + await writeJson(path.join(workRoot, "prompts", "index.json"), prompts); + manifest.counts.prompts = prompts.length; + stepEndSuccess(); + + // 7) Webhooks + stepStart("Exporting webhooks"); + const webhooks = await api.webhooks.list.query(); + await writeJson( + path.join(workRoot, "webhooks", "index.json"), + webhooks.webhooks, + ); + manifest.counts.webhooks = webhooks.webhooks.length; + stepEndSuccess(); + + // 8) Bookmarks (JSONL + list membership) + stepStart("Exporting bookmarks (metadata/content)"); + const bookmarkJsonl = path.join(workRoot, "bookmarks", "index.jsonl"); + let bookmarksExported = 0; + const bookmarkIterator = async function* (): AsyncGenerator<ZBookmark> { + let cursor: ZCursor | null = null; + do { + const resp = await api.bookmarks.getBookmarks.query({ + includeContent: true, + limit: Number(opts.batchSize) || 50, + cursor, + useCursorV2: true, + }); + for (const b of resp.bookmarks) { + yield b; + bookmarksExported++; + progressUpdate("Bookmarks", bookmarksExported); + } + cursor = resp.nextCursor; + } while (cursor); + }; + await writeJsonl(bookmarkJsonl, bookmarkIterator()); + progressDone(); + manifest.counts.bookmarks = bookmarksExported; + stepEndSuccess(); + + // 9) List membership (listId -> [bookmarkId]) + stepStart("Exporting list membership"); + const membership = await buildListMembership(api, lists, (p, t) => + progressUpdate("Lists scanned", p, t), + ); + progressDone(); + await writeJson( + path.join(workRoot, "lists", "membership.json"), + Object.fromEntries(membership.entries()), + ); + stepEndSuccess(); + + // 10) Assets: index + files + stepStart("Exporting assets (binary files)"); + const assetsDir = path.join(workRoot, "assets", "files"); + await ensureDir(assetsDir); + const assetsIndex: { + id: string; + assetType: string; + size: number; + contentType: string | null; + fileName: string | null; + bookmarkId: string | null; + filePath: string; // relative inside archive + }[] = []; + let assetPageCursor: number | null | undefined = null; + let downloaded = 0; + let totalAssets: number | undefined = undefined; + do { + const resp = await api.assets.list.query({ + limit: 50, + cursor: assetPageCursor ?? undefined, + }); + if (totalAssets == null) totalAssets = resp.totalCount; + for (const a of resp.assets) { + const relPath = path.join("assets", "files", a.id); + const absPath = path.join(workRoot, relPath); + try { + await downloadAsset( + globals.serverAddr, + globals.apiKey, + a.id, + absPath, + ); + assetsIndex.push({ + id: a.id, + assetType: a.assetType, + size: a.size, + contentType: a.contentType, + fileName: a.fileName, + bookmarkId: a.bookmarkId, + filePath: relPath.replace(/\\/g, "/"), + }); + downloaded++; + progressUpdate("Assets", downloaded, totalAssets); + } catch (e) { + printErrorMessageWithReason( + `Failed to download asset "${a.id}"`, + e as object, + ); + } + } + assetPageCursor = resp.nextCursor; + } while (assetPageCursor); + progressDone(); + manifest.counts.assets = downloaded; + await writeJson(path.join(workRoot, "assets", "index.json"), assetsIndex); + stepEndSuccess(); + + // 11) Manifest + stepStart("Writing manifest"); + await writeJson(path.join(workRoot, "manifest.json"), manifest); + stepEndSuccess(); + + // 12) Create archive + stepStart("Creating archive"); + await createTarGz(workRoot, outFile); + stepEndSuccess(); + + printStatusMessage(true, `Dump completed. File: ${outFile}`); + } catch (error) { + stepEndFail(); + printErrorMessageWithReason("Dump failed", error as object); + throw error; + } finally { + // Best-effort cleanup of temp directory + try { + await fsp.rm(workRoot, { recursive: true, force: true }); + } catch { + // ignore + } + } + }); + +async function buildListMembership( + api: ReturnType<typeof getAPIClient>, + lists: ZBookmarkList[], + onProgress?: (processed: number, total: number) => void, +) { + const result = new Map<string, string[]>(); // listId -> [bookmarkId] + let processed = 0; + for (const l of lists) { + // Only manual lists have explicit membership + if (l.type !== "manual") { + processed++; + onProgress?.(processed, lists.length); + continue; + } + let cursor: ZCursor | null = null; + const ids: string[] = []; + do { + const resp = await api.bookmarks.getBookmarks.query({ + listId: l.id, + limit: MAX_NUM_BOOKMARKS_PER_PAGE, + cursor, + includeContent: false, + useCursorV2: true, + }); + for (const b of resp.bookmarks) ids.push(b.id); + cursor = resp.nextCursor; + } while (cursor); + result.set(l.id, ids); + processed++; + onProgress?.(processed, lists.length); + } + return result; +} + +async function downloadAsset( + serverAddr: string, + apiKey: string, + assetId: string, + destFile: string, +) { + const url = `${serverAddr}/api/assets/${assetId}`; + const resp = await fetch(url, { + headers: { authorization: `Bearer ${apiKey}` }, + }); + if (!resp.ok) { + throw new Error(`HTTP ${resp.status} ${resp.statusText}`); + } + const arrayBuf = await resp.arrayBuffer(); + await ensureDir(path.dirname(destFile)); + await fsp.writeFile(destFile, Buffer.from(arrayBuf)); +} diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index c698169f..df7d9512 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -1,4 +1,5 @@ import { bookmarkCmd } from "@/commands/bookmarks"; +import { dumpCmd } from "@/commands/dump"; import { listsCmd } from "@/commands/lists"; import { migrateCmd } from "@/commands/migrate"; import { tagsCmd } from "@/commands/tags"; @@ -36,6 +37,7 @@ program.addCommand(tagsCmd); program.addCommand(whoamiCmd); program.addCommand(migrateCmd); program.addCommand(wipeCmd); +program.addCommand(dumpCmd); setGlobalOptions(program.opts()); |