7 files changed, 592 insertions, 0 deletions
diff --git a/packages/benchmarks/src/benchmarks.ts b/packages/benchmarks/src/benchmarks.ts
new file mode 100644
index 00000000..f2883246
--- /dev/null
+++ b/packages/benchmarks/src/benchmarks.ts
@@ -0,0 +1,158 @@
+import type { TaskResult } from "tinybench";
+import { Bench } from "tinybench";
+
+import type { SeedResult } from "./seed";
+import { logInfo, logStep, logSuccess } from "./log";
+import { formatMs, formatNumber } from "./utils";
+
+// Type guard for completed task results
+type CompletedTaskResult = Extract<TaskResult, { state: "completed" }>;
+
+export interface BenchmarkRow {
+  name: string;
+  ops: number;
+  mean: number;
+  p75: number;
+  p99: number;
+  samples: number;
+}
+
+export interface BenchmarkOptions {
+  timeMs?: number;
+  warmupMs?: number;
+}
+
+export async function runBenchmarks(
+  seed: SeedResult,
+  options?: BenchmarkOptions,
+): Promise<BenchmarkRow[]> {
+  const bench = new Bench({
+    time: options?.timeMs ?? 1000,
+    warmupTime: options?.warmupMs ?? 300,
+  });
+
+  const sampleTag = seed.tags[0];
+  const sampleList = seed.lists[0];
+  const sampleIds = seed.bookmarks.slice(0, 50).map((b) => b.id);
+
+  bench.add("bookmarks.getBookmarks (page)", async () => {
+    await seed.trpc.bookmarks.getBookmarks.query({
+      limit: 50,
+    });
+  });
+
+  if (sampleTag) {
+    bench.add("bookmarks.getBookmarks (tag filter)", async () => {
+      await seed.trpc.bookmarks.getBookmarks.query({
+        limit: 50,
+        tagId: sampleTag.id,
+      });
+    });
+  }
+
+  if (sampleList) {
+    bench.add("bookmarks.getBookmarks (list filter)", async () => {
+      await seed.trpc.bookmarks.getBookmarks.query({
+        limit: 50,
+        listId: sampleList.id,
+      });
+    });
+  }
+
+  if (sampleList && sampleIds.length > 0) {
+    bench.add("lists.getListsOfBookmark", async () => {
+      await seed.trpc.lists.getListsOfBookmark.query({
+        bookmarkId: sampleIds[0],
+      });
+    });
+  }
+
+  bench.add("bookmarks.searchBookmarks", async () => {
+    await seed.trpc.bookmarks.searchBookmarks.query({
+      text: seed.searchTerm,
+      limit: 20,
+    });
+  });
+
+  bench.add("bookmarks.getBookmarks (by ids)", async () => {
+    await seed.trpc.bookmarks.getBookmarks.query({
+      ids: sampleIds.slice(0, 20),
+      includeContent: false,
+    });
+  });
+
+  logStep("Running benchmarks");
+  await bench.run();
+  logSuccess("Benchmarks complete");
+
+  const rows = bench.tasks
+    .map((task) => {
+      const result = task.result;
+
+      // Check for errored state
+      if ("error" in result) {
+        console.error(`\n⚠️  Benchmark "${task.name}" failed with error:`);
+        console.error(result.error);
+        return null;
+      }
+
+      // Check if task completed successfully
+      if (result.state !== "completed") {
+        console.warn(
+          `\n⚠️  Benchmark "${task.name}" did not complete. State: ${result.state}`,
+        );
+        return null;
+      }
+
+      return toRow(task.name, result);
+    })
+    .filter(Boolean) as BenchmarkRow[];
+
+  renderTable(rows);
+  logInfo(
+    "ops/s uses tinybench's hz metric; durations are recorded in milliseconds.",
+  );
+
+  return rows;
+}
+
+function toRow(name: string, result: CompletedTaskResult): BenchmarkRow {
+  // The statistics are now in result.latency and result.throughput
+  const latency = result.latency;
+  const throughput = result.throughput;
+
+  return {
+    name,
+    ops: throughput.mean, // ops/s is the mean throughput
+    mean: latency.mean,
+    p75: latency.p75,
+    p99: latency.p99,
+    samples: latency.samplesCount,
+  };
+}
+
+function renderTable(rows: BenchmarkRow[]): void {
+  const headers = ["Benchmark", "ops/s", "avg", "p75", "p99", "samples"];
+
+  const data = rows.map((row) => [
+    row.name,
+    formatNumber(row.ops, 1),
+    formatMs(row.mean),
+    formatMs(row.p75),
+    formatMs(row.p99),
+    String(row.samples),
+  ]);
+
+  const columnWidths = headers.map((header, index) =>
+    Math.max(header.length, ...data.map((row) => row[index].length)),
+  );
+
+  const formatRow = (cells: string[]): string =>
+    cells.map((cell, index) => cell.padEnd(columnWidths[index])).join("  ");
+
+  console.log("");
+  console.log(formatRow(headers));
+  console.log(columnWidths.map((width) => "-".repeat(width)).join("  "));
+  data.forEach((row) => console.log(formatRow(row)));
+  console.log("");
+}
diff --git a/packages/benchmarks/src/index.ts b/packages/benchmarks/src/index.ts
new file mode 100644
index 00000000..9633da6e
--- /dev/null
+++ b/packages/benchmarks/src/index.ts
@@ -0,0 +1,88 @@
+import { runBenchmarks } from "./benchmarks";
+import { logInfo, logStep, logSuccess, logWarn } from "./log";
+import { seedData } from "./seed";
+import { startContainers } from "./startContainers";
+
+interface CliConfig {
+  bookmarkCount: number;
+  tagCount: number;
+  listCount: number;
+  concurrency: number;
+  keepContainers: boolean;
+  timeMs: number;
+  warmupMs: number;
+}
+
+function numberFromEnv(key: string, fallback: number): number {
+  const raw = process.env[key];
+  if (!raw) return fallback;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+function loadConfig(): CliConfig {
+  return {
+    bookmarkCount: numberFromEnv("BENCH_BOOKMARKS", 400),
+    tagCount: numberFromEnv("BENCH_TAGS", 25),
+    listCount: numberFromEnv("BENCH_LISTS", 6),
+    concurrency: numberFromEnv("BENCH_SEED_CONCURRENCY", 12),
+    keepContainers: process.env.BENCH_KEEP_CONTAINERS === "1",
+    timeMs: numberFromEnv("BENCH_TIME_MS", 1000),
+    warmupMs: numberFromEnv("BENCH_WARMUP_MS", 300),
+  };
+}
+
+async function main() {
+  const config = loadConfig();
+
+  logStep("Benchmark configuration");
+  logInfo(`Bookmarks:    ${config.bookmarkCount}`);
+  logInfo(`Tags:         ${config.tagCount}`);
+  logInfo(`Lists:        ${config.listCount}`);
+  logInfo(`Seed concur.: ${config.concurrency}`);
+  logInfo(`Time per case:${config.timeMs}ms (warmup ${config.warmupMs}ms)`);
+  logInfo(`Keep containers after run: ${config.keepContainers ? "yes" : "no"}`);
+
+  const running = await startContainers();
+
+  const stopContainers = async () => {
+    if (config.keepContainers) {
+      logWarn(
+        `Skipping docker compose shutdown (BENCH_KEEP_CONTAINERS=1). Port ${running.port} stays up.`,
+      );
+      return;
+    }
+    await running.stop();
+  };
+
+  const handleSignal = async (signal: NodeJS.Signals) => {
+    logWarn(`Received ${signal}, shutting down...`);
+    await stopContainers();
+    process.exit(1);
+  };
+
+  process.on("SIGINT", handleSignal);
+  process.on("SIGTERM", handleSignal);
+
+  try {
+    const seedResult = await seedData({
+      bookmarkCount: config.bookmarkCount,
+      tagCount: config.tagCount,
+      listCount: config.listCount,
+      concurrency: config.concurrency,
+    });
+
+    await runBenchmarks(seedResult, {
+      timeMs: config.timeMs,
+      warmupMs: config.warmupMs,
+    });
+    logSuccess("All done");
+  } catch (error) {
+    logWarn("Benchmark run failed");
+    console.error(error);
+  } finally {
+    await stopContainers();
+  }
+}
+
+main();
diff --git a/packages/benchmarks/src/log.ts b/packages/benchmarks/src/log.ts
new file mode 100644
index 00000000..08bb0afa
--- /dev/null
+++ b/packages/benchmarks/src/log.ts
@@ -0,0 +1,22 @@
+const ICONS = {
+  step: "==",
+  info: "--",
+  success: "OK",
+  warn: "!!",
+};
+
+export function logStep(title: string): void {
+  console.log(`\n${ICONS.step} ${title}`);
+}
+
+export function logInfo(message: string): void {
+  console.log(`  ${ICONS.info} ${message}`);
+}
+
+export function logSuccess(message: string): void {
+  console.log(`  ${ICONS.success} ${message}`);
+}
+
+export function logWarn(message: string): void {
+  console.log(`  ${ICONS.warn} ${message}`);
+}
diff --git a/packages/benchmarks/src/seed.ts b/packages/benchmarks/src/seed.ts
new file mode 100644
index 00000000..286a1f66
--- /dev/null
+++ b/packages/benchmarks/src/seed.ts
@@ -0,0 +1,171 @@
+import pLimit from "p-limit";
+
+import type { ZBookmarkList } from "@karakeep/shared/types/lists";
+import type { ZTagBasic } from "@karakeep/shared/types/tags";
+import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
+
+import { logInfo, logStep, logSuccess } from "./log";
+import { getTrpcClient, TrpcClient } from "./trpc";
+import { waitUntil } from "./utils";
+
+export interface SeedConfig {
+  bookmarkCount: number;
+  tagCount: number;
+  listCount: number;
+  concurrency: number;
+}
+
+export interface SeededBookmark {
+  id: string;
+  tags: ZTagBasic[];
+  listId?: string;
+  title: string | null | undefined;
+}
+
+export interface SeedResult {
+  apiKey: string;
+  trpc: TrpcClient;
+  tags: ZTagBasic[];
+  lists: ZBookmarkList[];
+  bookmarks: SeededBookmark[];
+  searchTerm: string;
+}
+
+const TOPICS = [
+  "performance",
+  "search",
+  "reading",
+  "workflow",
+  "api",
+  "workers",
+  "backend",
+  "frontend",
+  "productivity",
+  "cli",
+];
+
+export async function seedData(config: SeedConfig): Promise<SeedResult> {
+  const authlessClient = getTrpcClient();
+  const email = `benchmarks+${Date.now()}@example.com`;
+  const password = "benchmarks1234";
+
+  logStep("Creating benchmark user and API key");
+  await authlessClient.users.create.mutate({
+    name: "Benchmark User",
+    email,
+    password,
+    confirmPassword: password,
+  });
+  const { key } = await authlessClient.apiKeys.exchange.mutate({
+    email,
+    password,
+    keyName: "benchmark-key",
+  });
+
+  const trpc = getTrpcClient(key);
+  logSuccess("User ready");
+
+  logStep(`Creating ${config.tagCount} tags`);
+  const tags: ZTagBasic[] = [];
+  for (let i = 0; i < config.tagCount; i++) {
+    const tag = await trpc.tags.create.mutate({
+      name: `topic-${i + 1}`,
+    });
+    tags.push(tag);
+  }
+  logSuccess("Tags created");
+
+  logStep(`Creating ${config.listCount} lists`);
+  const lists: ZBookmarkList[] = [];
+  for (let i = 0; i < config.listCount; i++) {
+    const list = await trpc.lists.create.mutate({
+      name: `List ${i + 1}`,
+      description: `Auto-generated benchmark list #${i + 1}`,
+      icon: "bookmark",
+    });
+    lists.push(list);
+  }
+  logSuccess("Lists created");
+
+  logStep(`Creating ${config.bookmarkCount} bookmarks`);
+  const limit = pLimit(config.concurrency);
+  const bookmarks: SeededBookmark[] = [];
+
+  await Promise.all(
+    Array.from({ length: config.bookmarkCount }).map((_, index) =>
+      limit(async () => {
+        const topic = TOPICS[index % TOPICS.length];
+        const createdAt = new Date(Date.now() - index * 3000);
+        const bookmark = await trpc.bookmarks.createBookmark.mutate({
+          type: BookmarkTypes.LINK,
+          url: `https://example.com/${topic}/${index}`,
+          title: `Benchmark ${topic} article ${index}`,
+          source: "api",
+          summary: `Benchmark dataset entry about ${topic} performance and organization.`,
+          favourited: index % 7 === 0,
+          archived: false,
+          createdAt,
+        });
+
+        const primaryTag = tags[index % tags.length];
+        const secondaryTag = tags[(index + 5) % tags.length];
+        const attachedTags = [primaryTag, secondaryTag];
+        await trpc.bookmarks.updateTags.mutate({
+          bookmarkId: bookmark.id,
+          attach: attachedTags.map((tag) => ({
+            tagId: tag.id,
+            tagName: tag.name,
+          })),
+          detach: [],
+        });
+
+        let listId: string | undefined;
+        if (lists.length > 0) {
+          const list = lists[index % lists.length];
+          await trpc.lists.addToList.mutate({
+            listId: list.id,
+            bookmarkId: bookmark.id,
+          });
+          listId = list.id;
+        }
+
+        bookmarks.push({
+          id: bookmark.id,
+          tags: attachedTags,
+          listId,
+          title: bookmark.title,
+        });
+      }),
+    ),
+  );
+  logSuccess("Bookmarks created");
+
+  const searchTerm = "benchmark";
+  logStep("Waiting for search index to be ready");
+  await waitUntil(
+    async () => {
+      const results = await trpc.bookmarks.searchBookmarks.query({
+        text: searchTerm,
+        limit: 1,
+      });
+      return results.bookmarks.length > 0;
+    },
+    "search data to be indexed",
+    120_000,
+    2_000,
+  );
+  logSuccess("Search index warmed up");
+
+  logInfo(
+    `Seeded ${bookmarks.length} bookmarks across ${tags.length} tags and ${lists.length} lists`,
+  );
+
+  return {
+    apiKey: key,
+    trpc,
+    tags,
+    lists,
+    bookmarks,
+    searchTerm,
+  };
+}
diff --git a/packages/benchmarks/src/startContainers.ts b/packages/benchmarks/src/startContainers.ts
new file mode 100644
index 00000000..ed4e0250
--- /dev/null
+++ b/packages/benchmarks/src/startContainers.ts
@@ -0,0 +1,96 @@
+import { execSync } from "child_process";
+import net from "net";
+import path from "path";
+import { fileURLToPath } from "url";
+
+import { logInfo, logStep, logSuccess, logWarn } from "./log";
+import { sleep, waitUntil } from "./utils";
+
+async function getRandomPort(): Promise<number> {
+  const server = net.createServer();
+  return new Promise<number>((resolve, reject) => {
+    server.unref();
+    server.on("error", reject);
+    server.listen(0, () => {
+      const port = (server.address() as net.AddressInfo).port;
+      server.close(() => resolve(port));
+    });
+  });
+}
+
+async function waitForHealthy(port: number): Promise<void> {
+  await waitUntil(
+    async () => {
+      const res = await fetch(`http://localhost:${port}/api/health`);
+      return res.status === 200;
+    },
+    "Karakeep stack to become healthy",
+    60_000,
+    1_000,
+  );
+}
+
+async function captureDockerLogs(composeDir: string): Promise<void> {
+  const logsDir = path.join(composeDir, "setup", "docker-logs");
+  try {
+    execSync(`mkdir -p "${logsDir}"`, { cwd: composeDir });
+  } catch {
+    // ignore
+  }
+
+  const services = ["web", "meilisearch", "chrome", "nginx", "minio"];
+  for (const service of services) {
+    try {
+      execSync(
+        `/bin/sh -c 'docker compose logs ${service} > "${logsDir}/${service}.log" 2>&1'`,
+        {
+          cwd: composeDir,
+          stdio: "ignore",
+        },
+      );
+      logInfo(`Captured logs for ${service}`);
+    } catch (error) {
+      logWarn(`Failed to capture logs for ${service}: ${error}`);
+    }
+  }
+}
+
+export interface RunningContainers {
+  port: number;
+  stop: () => Promise<void>;
+}
+
+export async function startContainers(): Promise<RunningContainers> {
+  const __dirname = path.dirname(fileURLToPath(import.meta.url));
+  const composeDir = path.join(__dirname, "..");
+  const port = await getRandomPort();
+  const skipBuild =
+    process.env.BENCH_NO_BUILD === "1" || process.env.BENCH_SKIP_BUILD === "1";
+  const buildArg = skipBuild ? "" : "--build";
+
+  logStep(`Starting docker compose on port ${port}`);
+  execSync(`docker compose up ${buildArg} -d`, {
+    cwd: composeDir,
+    stdio: "inherit",
+    env: { ...process.env, KARAKEEP_PORT: String(port) },
+  });
+
+  logInfo("Waiting for services to report healthy...");
+  await waitForHealthy(port);
+  await sleep(5_000);
+  logSuccess("Containers are ready");
+
+  process.env.KARAKEEP_PORT = String(port);
+
+  let stopped = false;
+  const stop = async (): Promise<void> => {
+    if (stopped) return;
+    stopped = true;
+    logStep("Collecting docker logs");
+    await captureDockerLogs(composeDir);
+    logStep("Stopping docker compose");
+    execSync("docker compose down", { cwd: composeDir, stdio: "inherit" });
+  };
+
+  return { port, stop };
+}
diff --git a/packages/benchmarks/src/trpc.ts b/packages/benchmarks/src/trpc.ts
new file mode 100644
index 00000000..3a8cfe37
--- /dev/null
+++ b/packages/benchmarks/src/trpc.ts
@@ -0,0 +1,26 @@
+import { createTRPCClient, httpBatchLink } from "@trpc/client";
+import superjson from "superjson";
+
+import type { AppRouter } from "@karakeep/trpc/routers/_app";
+
+export type TrpcClient = ReturnType<typeof getTrpcClient>;
+
+export function getTrpcClient(apiKey?: string) {
+  if (!process.env.KARAKEEP_PORT) {
+    throw new Error("KARAKEEP_PORT is not set. Did you start the containers?");
+  }
+
+  return createTRPCClient<AppRouter>({
+    links: [
+      httpBatchLink({
+        transformer: superjson,
+        url: `http://localhost:${process.env.KARAKEEP_PORT}/api/trpc`,
+        headers() {
+          return {
+            authorization: apiKey ? `Bearer ${apiKey}` : undefined,
+          };
+        },
+      }),
+    ],
+  });
+}
diff --git a/packages/benchmarks/src/utils.ts b/packages/benchmarks/src/utils.ts
new file mode 100644
index 00000000..cfb00723
--- /dev/null
+++ b/packages/benchmarks/src/utils.ts
@@ -0,0 +1,31 @@
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function waitUntil(
+  fn: () => Promise<boolean>,
+  description: string,
+  timeoutMs = 60000,
+  intervalMs = 1000,
+): Promise<void> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    try {
+      if (await fn()) {
+        return;
+      }
+    } catch {
+      // Ignore and retry
+    }
+    await sleep(intervalMs);
+  }
+  throw new Error(`${description} timed out after ${timeoutMs}ms`);
+}
+
+export function formatNumber(num: number, fractionDigits = 2): string {
+  return num.toFixed(fractionDigits);
+}
+
+export function formatMs(ms: number): string {
+  return `${formatNumber(ms, ms >= 10 ? 1 : 2)} ms`;
+}