chore: add benchmarks (#2229)

* chore: add benchmarks

* upgrade deps

* fixes

* lint

15 files changed, 789 insertions, 0 deletions

diff --git a/.dockerignore b/.dockerignore
index 4affdb95..615cb7e2 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -18,6 +18,7 @@ apps/mobile
 apps/landing
 apps/browser-extension
 packages/e2e_tests
+packages/benchmarks
 
 # Aider
 .aider*
diff --git a/packages/benchmarks/.gitignore b/packages/benchmarks/.gitignore
new file mode 100644
index 00000000..0cb27f19
--- /dev/null
+++ b/packages/benchmarks/.gitignore
@@ -0,0 +1,2 @@
+# Docker logs captured during test runs
+setup/docker-logs/
diff --git a/packages/benchmarks/README.md b/packages/benchmarks/README.md
new file mode 100644
index 00000000..2d6cb06a
--- /dev/null
+++ b/packages/benchmarks/README.md
@@ -0,0 +1,32 @@
+# Karakeep Benchmarks
+
+This package spins up a production-like Karakeep stack in Docker, seeds it with a sizeable dataset, then benchmarks a handful of high-signal APIs.
+
+## Usage
+
+```bash
+pnpm --filter @karakeep/benchmarks bench
+```
+
+The command will:
+
+- Start the docker-compose stack on a random free port
+- Create a dedicated benchmark user, tags, lists, and hundreds of bookmarks
+- Run a suite of benchmarks (create, list, search, and list metadata calls)
+- Print a table with ops/sec and latency percentiles
+- Tear down the containers and capture logs (unless you opt out)
+
+## Configuration
+
+Control the run via environment variables:
+
+- `BENCH_BOOKMARKS` (default `400`): number of bookmarks to seed
+- `BENCH_TAGS` (default `25`): number of tags to seed
+- `BENCH_LISTS` (default `6`): number of lists to seed
+- `BENCH_SEED_CONCURRENCY` (default `12`): concurrent seeding operations
+- `BENCH_TIME_MS` (default `1000`): time per benchmark case
+- `BENCH_WARMUP_MS` (default `300`): warmup time per case
+- `BENCH_NO_BUILD=1`: reuse existing docker images instead of rebuilding
+- `BENCH_KEEP_CONTAINERS=1`: leave the stack running after the run
+
+The stack uses the package-local `docker-compose.yml` and serves a tiny HTML fixture from `setup/html`.
diff --git a/packages/benchmarks/docker-compose.yml b/packages/benchmarks/docker-compose.yml
new file mode 100644
index 00000000..c74b45d6
--- /dev/null
+++ b/packages/benchmarks/docker-compose.yml
@@ -0,0 +1,54 @@
+services:
+  web:
+    build:
+      dockerfile: docker/Dockerfile
+      context: ../../
+      target: aio
+    restart: unless-stopped
+    ports:
+      - "${KARAKEEP_PORT:-3000}:3000"
+    environment:
+      DATA_DIR: /tmp
+      NEXTAUTH_SECRET: secret
+      NEXTAUTH_URL: http://localhost:${KARAKEEP_PORT:-3000}
+      MEILI_MASTER_KEY: dummy
+      MEILI_ADDR: http://meilisearch:7700
+      BROWSER_WEB_URL: http://chrome:9222
+      CRAWLER_NUM_WORKERS: 6
+      CRAWLER_ALLOWED_INTERNAL_HOSTNAMES: nginx
+  meilisearch:
+    image: getmeili/meilisearch:v1.13.3
+    restart: unless-stopped
+    environment:
+      MEILI_NO_ANALYTICS: "true"
+      MEILI_MASTER_KEY: dummy
+  chrome:
+    image: gcr.io/zenika-hub/alpine-chrome:124
+    restart: unless-stopped
+    command:
+      - --no-sandbox
+      - --disable-gpu
+      - --disable-dev-shm-usage
+      - --remote-debugging-address=0.0.0.0
+      - --remote-debugging-port=9222
+      - --hide-scrollbars
+  nginx:
+    image: nginx:alpine
+    restart: unless-stopped
+    volumes:
+      - ./setup/html:/usr/share/nginx/html
+  minio:
+    image: minio/minio:latest
+    restart: unless-stopped
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    command: server /data --console-address ":9001"
+    volumes:
+      - minio_data:/data
+
+volumes:
+  minio_data:
diff --git a/packages/benchmarks/package.json b/packages/benchmarks/package.json
new file mode 100644
index 00000000..52862862
--- /dev/null
+++ b/packages/benchmarks/package.json
@@ -0,0 +1,32 @@
+{
+  "$schema": "https://json.schemastore.org/package.json",
+  "name": "@karakeep/benchmarks",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "bench": "tsx src/index.ts",
+    "lint": "oxlint .",
+    "lint:fix": "oxlint . --fix",
+    "format": "prettier . --cache --ignore-path ../../.prettierignore --check",
+    "format:fix": "prettier . --cache --write --ignore-path ../../.prettierignore",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "@karakeep/shared": "workspace:^0.1.0",
+    "@karakeep/trpc": "workspace:^0.1.0",
+    "@trpc/client": "^11.4.3",
+    "p-limit": "^7.2.0",
+    "superjson": "^2.2.1",
+    "tinybench": "^6.0.0",
+    "zod": "^3.24.2"
+  },
+  "devDependencies": {
+    "@karakeep/prettier-config": "workspace:^0.1.0",
+    "@karakeep/tsconfig": "workspace:^0.1.0",
+    "oxlint": "^1.29.0",
+    "prettier": "^3.4.2",
+    "tsx": "^4.8.1"
+  },
+  "prettier": "@karakeep/prettier-config"
+}
diff --git a/packages/benchmarks/setup/html/hello.html b/packages/benchmarks/setup/html/hello.html
new file mode 100644
index 00000000..25a7f9b5
--- /dev/null
+++ b/packages/benchmarks/setup/html/hello.html
@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Benchmarks Fixture</title>
+  </head>
+  <body>
+    <h1>Karakeep Benchmarks</h1>
+    <p>This page is served by the nginx container during benchmarks.</p>
+  </body>
+</html>
diff --git a/packages/benchmarks/src/benchmarks.ts b/packages/benchmarks/src/benchmarks.ts
new file mode 100644
index 00000000..f2883246
--- /dev/null
+++ b/packages/benchmarks/src/benchmarks.ts
@@ -0,0 +1,158 @@
+import type { TaskResult } from "tinybench";
+import { Bench } from "tinybench";
+
+import type { SeedResult } from "./seed";
+import { logInfo, logStep, logSuccess } from "./log";
+import { formatMs, formatNumber } from "./utils";
+
+// Type guard for completed task results
+type CompletedTaskResult = Extract<TaskResult, { state: "completed" }>;
+
+export interface BenchmarkRow {
+  name: string;
+  ops: number;
+  mean: number;
+  p75: number;
+  p99: number;
+  samples: number;
+}
+
+export interface BenchmarkOptions {
+  timeMs?: number;
+  warmupMs?: number;
+}
+
+export async function runBenchmarks(
+  seed: SeedResult,
+  options?: BenchmarkOptions,
+): Promise<BenchmarkRow[]> {
+  const bench = new Bench({
+    time: options?.timeMs ?? 1000,
+    warmupTime: options?.warmupMs ?? 300,
+  });
+
+  const sampleTag = seed.tags[0];
+  const sampleList = seed.lists[0];
+  const sampleIds = seed.bookmarks.slice(0, 50).map((b) => b.id);
+
+  bench.add("bookmarks.getBookmarks (page)", async () => {
+    await seed.trpc.bookmarks.getBookmarks.query({
+      limit: 50,
+    });
+  });
+
+  if (sampleTag) {
+    bench.add("bookmarks.getBookmarks (tag filter)", async () => {
+      await seed.trpc.bookmarks.getBookmarks.query({
+        limit: 50,
+        tagId: sampleTag.id,
+      });
+    });
+  }
+
+  if (sampleList) {
+    bench.add("bookmarks.getBookmarks (list filter)", async () => {
+      await seed.trpc.bookmarks.getBookmarks.query({
+        limit: 50,
+        listId: sampleList.id,
+      });
+    });
+  }
+
+  if (sampleList && sampleIds.length > 0) {
+    bench.add("lists.getListsOfBookmark", async () => {
+      await seed.trpc.lists.getListsOfBookmark.query({
+        bookmarkId: sampleIds[0],
+      });
+    });
+  }
+
+  bench.add("bookmarks.searchBookmarks", async () => {
+    await seed.trpc.bookmarks.searchBookmarks.query({
+      text: seed.searchTerm,
+      limit: 20,
+    });
+  });
+
+  bench.add("bookmarks.getBookmarks (by ids)", async () => {
+    await seed.trpc.bookmarks.getBookmarks.query({
+      ids: sampleIds.slice(0, 20),
+      includeContent: false,
+    });
+  });
+
+  logStep("Running benchmarks");
+  await bench.run();
+  logSuccess("Benchmarks complete");
+
+  const rows = bench.tasks
+    .map((task) => {
+      const result = task.result;
+
+      // Check for errored state
+      if ("error" in result) {
+        console.error(`\n⚠️  Benchmark "${task.name}" failed with error:`);
+        console.error(result.error);
+        return null;
+      }
+
+      // Check if task completed successfully
+      if (result.state !== "completed") {
+        console.warn(
+          `\n⚠️  Benchmark "${task.name}" did not complete. State: ${result.state}`,
+        );
+        return null;
+      }
+
+      return toRow(task.name, result);
+    })
+    .filter(Boolean) as BenchmarkRow[];
+
+  renderTable(rows);
+  logInfo(
+    "ops/s uses tinybench's hz metric; durations are recorded in milliseconds.",
+  );
+
+  return rows;
+}
+
+function toRow(name: string, result: CompletedTaskResult): BenchmarkRow {
+  // The statistics are now in result.latency and result.throughput
+  const latency = result.latency;
+  const throughput = result.throughput;
+
+  return {
+    name,
+    ops: throughput.mean, // ops/s is the mean throughput
+    mean: latency.mean,
+    p75: latency.p75,
+    p99: latency.p99,
+    samples: latency.samplesCount,
+  };
+}
+
+function renderTable(rows: BenchmarkRow[]): void {
+  const headers = ["Benchmark", "ops/s", "avg", "p75", "p99", "samples"];
+
+  const data = rows.map((row) => [
+    row.name,
+    formatNumber(row.ops, 1),
+    formatMs(row.mean),
+    formatMs(row.p75),
+    formatMs(row.p99),
+    String(row.samples),
+  ]);
+
+  const columnWidths = headers.map((header, index) =>
+    Math.max(header.length, ...data.map((row) => row[index].length)),
+  );
+
+  const formatRow = (cells: string[]): string =>
+    cells.map((cell, index) => cell.padEnd(columnWidths[index])).join("  ");
+
+  console.log("");
+  console.log(formatRow(headers));
+  console.log(columnWidths.map((width) => "-".repeat(width)).join("  "));
+  data.forEach((row) => console.log(formatRow(row)));
+  console.log("");
+}
diff --git a/packages/benchmarks/src/index.ts b/packages/benchmarks/src/index.ts
new file mode 100644
index 00000000..9633da6e
--- /dev/null
+++ b/packages/benchmarks/src/index.ts
@@ -0,0 +1,88 @@
+import { runBenchmarks } from "./benchmarks";
+import { logInfo, logStep, logSuccess, logWarn } from "./log";
+import { seedData } from "./seed";
+import { startContainers } from "./startContainers";
+
+interface CliConfig {
+  bookmarkCount: number;
+  tagCount: number;
+  listCount: number;
+  concurrency: number;
+  keepContainers: boolean;
+  timeMs: number;
+  warmupMs: number;
+}
+
+function numberFromEnv(key: string, fallback: number): number {
+  const raw = process.env[key];
+  if (!raw) return fallback;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+function loadConfig(): CliConfig {
+  return {
+    bookmarkCount: numberFromEnv("BENCH_BOOKMARKS", 400),
+    tagCount: numberFromEnv("BENCH_TAGS", 25),
+    listCount: numberFromEnv("BENCH_LISTS", 6),
+    concurrency: numberFromEnv("BENCH_SEED_CONCURRENCY", 12),
+    keepContainers: process.env.BENCH_KEEP_CONTAINERS === "1",
+    timeMs: numberFromEnv("BENCH_TIME_MS", 1000),
+    warmupMs: numberFromEnv("BENCH_WARMUP_MS", 300),
+  };
+}
+
+async function main() {
+  const config = loadConfig();
+
+  logStep("Benchmark configuration");
+  logInfo(`Bookmarks:    ${config.bookmarkCount}`);
+  logInfo(`Tags:         ${config.tagCount}`);
+  logInfo(`Lists:        ${config.listCount}`);
+  logInfo(`Seed concur.: ${config.concurrency}`);
+  logInfo(`Time per case:${config.timeMs}ms (warmup ${config.warmupMs}ms)`);
+  logInfo(`Keep containers after run: ${config.keepContainers ? "yes" : "no"}`);
+
+  const running = await startContainers();
+
+  const stopContainers = async () => {
+    if (config.keepContainers) {
+      logWarn(
+        `Skipping docker compose shutdown (BENCH_KEEP_CONTAINERS=1). Port ${running.port} stays up.`,
+      );
+      return;
+    }
+    await running.stop();
+  };
+
+  const handleSignal = async (signal: NodeJS.Signals) => {
+    logWarn(`Received ${signal}, shutting down...`);
+    await stopContainers();
+    process.exit(1);
+  };
+
+  process.on("SIGINT", handleSignal);
+  process.on("SIGTERM", handleSignal);
+
+  try {
+    const seedResult = await seedData({
+      bookmarkCount: config.bookmarkCount,
+      tagCount: config.tagCount,
+      listCount: config.listCount,
+      concurrency: config.concurrency,
+    });
+
+    await runBenchmarks(seedResult, {
+      timeMs: config.timeMs,
+      warmupMs: config.warmupMs,
+    });
+    logSuccess("All done");
+  } catch (error) {
+    logWarn("Benchmark run failed");
+    console.error(error);
+  } finally {
+    await stopContainers();
+  }
+}
+
+main();
diff --git a/packages/benchmarks/src/log.ts b/packages/benchmarks/src/log.ts
new file mode 100644
index 00000000..08bb0afa
--- /dev/null
+++ b/packages/benchmarks/src/log.ts
@@ -0,0 +1,22 @@
+const ICONS = {
+  step: "==",
+  info: "--",
+  success: "OK",
+  warn: "!!",
+};
+
+export function logStep(title: string): void {
+  console.log(`\n${ICONS.step} ${title}`);
+}
+
+export function logInfo(message: string): void {
+  console.log(`  ${ICONS.info} ${message}`);
+}
+
+export function logSuccess(message: string): void {
+  console.log(`  ${ICONS.success} ${message}`);
+}
+
+export function logWarn(message: string): void {
+  console.log(`  ${ICONS.warn} ${message}`);
+}
diff --git a/packages/benchmarks/src/seed.ts b/packages/benchmarks/src/seed.ts
new file mode 100644
index 00000000..286a1f66
--- /dev/null
+++ b/packages/benchmarks/src/seed.ts
@@ -0,0 +1,171 @@
+import pLimit from "p-limit";
+
+import type { ZBookmarkList } from "@karakeep/shared/types/lists";
+import type { ZTagBasic } from "@karakeep/shared/types/tags";
+import { BookmarkTypes } from "@karakeep/shared/types/bookmarks";
+
+import { logInfo, logStep, logSuccess } from "./log";
+import { getTrpcClient, TrpcClient } from "./trpc";
+import { waitUntil } from "./utils";
+
+export interface SeedConfig {
+  bookmarkCount: number;
+  tagCount: number;
+  listCount: number;
+  concurrency: number;
+}
+
+export interface SeededBookmark {
+  id: string;
+  tags: ZTagBasic[];
+  listId?: string;
+  title: string | null | undefined;
+}
+
+export interface SeedResult {
+  apiKey: string;
+  trpc: TrpcClient;
+  tags: ZTagBasic[];
+  lists: ZBookmarkList[];
+  bookmarks: SeededBookmark[];
+  searchTerm: string;
+}
+
+const TOPICS = [
+  "performance",
+  "search",
+  "reading",
+  "workflow",
+  "api",
+  "workers",
+  "backend",
+  "frontend",
+  "productivity",
+  "cli",
+];
+
+export async function seedData(config: SeedConfig): Promise<SeedResult> {
+  const authlessClient = getTrpcClient();
+  const email = `benchmarks+${Date.now()}@example.com`;
+  const password = "benchmarks1234";
+
+  logStep("Creating benchmark user and API key");
+  await authlessClient.users.create.mutate({
+    name: "Benchmark User",
+    email,
+    password,
+    confirmPassword: password,
+  });
+  const { key } = await authlessClient.apiKeys.exchange.mutate({
+    email,
+    password,
+    keyName: "benchmark-key",
+  });
+
+  const trpc = getTrpcClient(key);
+  logSuccess("User ready");
+
+  logStep(`Creating ${config.tagCount} tags`);
+  const tags: ZTagBasic[] = [];
+  for (let i = 0; i < config.tagCount; i++) {
+    const tag = await trpc.tags.create.mutate({
+      name: `topic-${i + 1}`,
+    });
+    tags.push(tag);
+  }
+  logSuccess("Tags created");
+
+  logStep(`Creating ${config.listCount} lists`);
+  const lists: ZBookmarkList[] = [];
+  for (let i = 0; i < config.listCount; i++) {
+    const list = await trpc.lists.create.mutate({
+      name: `List ${i + 1}`,
+      description: `Auto-generated benchmark list #${i + 1}`,
+      icon: "bookmark",
+    });
+    lists.push(list);
+  }
+  logSuccess("Lists created");
+
+  logStep(`Creating ${config.bookmarkCount} bookmarks`);
+  const limit = pLimit(config.concurrency);
+  const bookmarks: SeededBookmark[] = [];
+
+  await Promise.all(
+    Array.from({ length: config.bookmarkCount }).map((_, index) =>
+      limit(async () => {
+        const topic = TOPICS[index % TOPICS.length];
+        const createdAt = new Date(Date.now() - index * 3000);
+        const bookmark = await trpc.bookmarks.createBookmark.mutate({
+          type: BookmarkTypes.LINK,
+          url: `https://example.com/${topic}/${index}`,
+          title: `Benchmark ${topic} article ${index}`,
+          source: "api",
+          summary: `Benchmark dataset entry about ${topic} performance and organization.`,
+          favourited: index % 7 === 0,
+          archived: false,
+          createdAt,
+        });
+
+        const primaryTag = tags[index % tags.length];
+        const secondaryTag = tags[(index + 5) % tags.length];
+        const attachedTags = [primaryTag, secondaryTag];
+        await trpc.bookmarks.updateTags.mutate({
+          bookmarkId: bookmark.id,
+          attach: attachedTags.map((tag) => ({
+            tagId: tag.id,
+            tagName: tag.name,
+          })),
+          detach: [],
+        });
+
+        let listId: string | undefined;
+        if (lists.length > 0) {
+          const list = lists[index % lists.length];
+          await trpc.lists.addToList.mutate({
+            listId: list.id,
+            bookmarkId: bookmark.id,
+          });
+          listId = list.id;
+        }
+
+        bookmarks.push({
+          id: bookmark.id,
+          tags: attachedTags,
+          listId,
+          title: bookmark.title,
+        });
+      }),
+    ),
+  );
+  logSuccess("Bookmarks created");
+
+  const searchTerm = "benchmark";
+  logStep("Waiting for search index to be ready");
+  await waitUntil(
+    async () => {
+      const results = await trpc.bookmarks.searchBookmarks.query({
+        text: searchTerm,
+        limit: 1,
+      });
+      return results.bookmarks.length > 0;
+    },
+    "search data to be indexed",
+    120_000,
+    2_000,
+  );
+  logSuccess("Search index warmed up");
+
+  logInfo(
+    `Seeded ${bookmarks.length} bookmarks across ${tags.length} tags and ${lists.length} lists`,
+  );
+
+  return {
+    apiKey: key,
+    trpc,
+    tags,
+    lists,
+    bookmarks,
+    searchTerm,
+  };
+}
diff --git a/packages/benchmarks/src/startContainers.ts b/packages/benchmarks/src/startContainers.ts
new file mode 100644
index 00000000..ed4e0250
--- /dev/null
+++ b/packages/benchmarks/src/startContainers.ts
@@ -0,0 +1,96 @@
+import { execSync } from "child_process";
+import net from "net";
+import path from "path";
+import { fileURLToPath } from "url";
+
+import { logInfo, logStep, logSuccess, logWarn } from "./log";
+import { sleep, waitUntil } from "./utils";
+
+async function getRandomPort(): Promise<number> {
+  const server = net.createServer();
+  return new Promise<number>((resolve, reject) => {
+    server.unref();
+    server.on("error", reject);
+    server.listen(0, () => {
+      const port = (server.address() as net.AddressInfo).port;
+      server.close(() => resolve(port));
+    });
+  });
+}
+
+async function waitForHealthy(port: number): Promise<void> {
+  await waitUntil(
+    async () => {
+      const res = await fetch(`http://localhost:${port}/api/health`);
+      return res.status === 200;
+    },
+    "Karakeep stack to become healthy",
+    60_000,
+    1_000,
+  );
+}
+
+async function captureDockerLogs(composeDir: string): Promise<void> {
+  const logsDir = path.join(composeDir, "setup", "docker-logs");
+  try {
+    execSync(`mkdir -p "${logsDir}"`, { cwd: composeDir });
+  } catch {
+    // ignore
+  }
+
+  const services = ["web", "meilisearch", "chrome", "nginx", "minio"];
+  for (const service of services) {
+    try {
+      execSync(
+        `/bin/sh -c 'docker compose logs ${service} > "${logsDir}/${service}.log" 2>&1'`,
+        {
+          cwd: composeDir,
+          stdio: "ignore",
+        },
+      );
+      logInfo(`Captured logs for ${service}`);
+    } catch (error) {
+      logWarn(`Failed to capture logs for ${service}: ${error}`);
+    }
+  }
+}
+
+export interface RunningContainers {
+  port: number;
+  stop: () => Promise<void>;
+}
+
+export async function startContainers(): Promise<RunningContainers> {
+  const __dirname = path.dirname(fileURLToPath(import.meta.url));
+  const composeDir = path.join(__dirname, "..");
+  const port = await getRandomPort();
+  const skipBuild =
+    process.env.BENCH_NO_BUILD === "1" || process.env.BENCH_SKIP_BUILD === "1";
+  const buildArg = skipBuild ? "" : "--build";
+
+  logStep(`Starting docker compose on port ${port}`);
+  execSync(`docker compose up ${buildArg} -d`, {
+    cwd: composeDir,
+    stdio: "inherit",
+    env: { ...process.env, KARAKEEP_PORT: String(port) },
+  });
+
+  logInfo("Waiting for services to report healthy...");
+  await waitForHealthy(port);
+  await sleep(5_000);
+  logSuccess("Containers are ready");
+
+  process.env.KARAKEEP_PORT = String(port);
+
+  let stopped = false;
+  const stop = async (): Promise<void> => {
+    if (stopped) return;
+    stopped = true;
+    logStep("Collecting docker logs");
+    await captureDockerLogs(composeDir);
+    logStep("Stopping docker compose");
+    execSync("docker compose down", { cwd: composeDir, stdio: "inherit" });
+  };
+
+  return { port, stop };
+}
diff --git a/packages/benchmarks/src/trpc.ts b/packages/benchmarks/src/trpc.ts
new file mode 100644
index 00000000..3a8cfe37
--- /dev/null
+++ b/packages/benchmarks/src/trpc.ts
@@ -0,0 +1,26 @@
+import { createTRPCClient, httpBatchLink } from "@trpc/client";
+import superjson from "superjson";
+
+import type { AppRouter } from "@karakeep/trpc/routers/_app";
+
+export type TrpcClient = ReturnType<typeof getTrpcClient>;
+
+export function getTrpcClient(apiKey?: string) {
+  if (!process.env.KARAKEEP_PORT) {
+    throw new Error("KARAKEEP_PORT is not set. Did you start the containers?");
+  }
+
+  return createTRPCClient<AppRouter>({
+    links: [
+      httpBatchLink({
+        transformer: superjson,
+        url: `http://localhost:${process.env.KARAKEEP_PORT}/api/trpc`,
+        headers() {
+          return {
+            authorization: apiKey ? `Bearer ${apiKey}` : undefined,
+          };
+        },
+      }),
+    ],
+  });
+}
diff --git a/packages/benchmarks/src/utils.ts b/packages/benchmarks/src/utils.ts
new file mode 100644
index 00000000..cfb00723
--- /dev/null
+++ b/packages/benchmarks/src/utils.ts
@@ -0,0 +1,31 @@
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+export async function waitUntil(
+  fn: () => Promise<boolean>,
+  description: string,
+  timeoutMs = 60000,
+  intervalMs = 1000,
+): Promise<void> {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    try {
+      if (await fn()) {
+        return;
+      }
+    } catch {
+      // Ignore and retry
+    }
+    await sleep(intervalMs);
+  }
+  throw new Error(`${description} timed out after ${timeoutMs}ms`);
+}
+
+export function formatNumber(num: number, fractionDigits = 2): string {
+  return num.toFixed(fractionDigits);
+}
+
+export function formatMs(ms: number): string {
+  return `${formatNumber(ms, ms >= 10 ? 1 : 2)} ms`;
+}
diff --git a/packages/benchmarks/tsconfig.json b/packages/benchmarks/tsconfig.json
new file mode 100644
index 00000000..ae9547cb
--- /dev/null
+++ b/packages/benchmarks/tsconfig.json
@@ -0,0 +1,9 @@
+{
+  "$schema": "https://json.schemastore.org/tsconfig",
+  "extends": "@karakeep/tsconfig/node.json",
+  "include": ["src/**/*.ts"],
+  "exclude": ["node_modules"],
+  "compilerOptions": {
+    "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
+  }
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 7f95735a..d87dc5ff 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1061,6 +1061,46 @@ importers:
         specifier: ^3.2.4
         version: 3.2.4(@types/debug@4.1.12)(@types/node@22.15.30)(happy-dom@20.0.8)(jiti@2.4.2)(jsdom@27.0.1(postcss@8.5.6))(lightningcss@1.30.1)(sass@1.89.1)(terser@5.41.0)(tsx@4.20.3)(yaml@2.8.0)
 
+  packages/benchmarks:
+    dependencies:
+      '@karakeep/shared':
+        specifier: workspace:^0.1.0
+        version: link:../shared
+      '@karakeep/trpc':
+        specifier: workspace:^0.1.0
+        version: link:../trpc
+      '@trpc/client':
+        specifier: ^11.4.3
+        version: 11.4.3(@trpc/server@11.4.3(typescript@5.9.3))(typescript@5.9.3)
+      p-limit:
+        specifier: ^7.2.0
+        version: 7.2.0
+      superjson:
+        specifier: ^2.2.1
+        version: 2.2.1
+      tinybench:
+        specifier: ^6.0.0
+        version: 6.0.0
+      zod:
+        specifier: ^3.24.2
+        version: 3.24.2
+    devDependencies:
+      '@karakeep/prettier-config':
+        specifier: workspace:^0.1.0
+        version: link:../../tooling/prettier
+      '@karakeep/tsconfig':
+        specifier: workspace:^0.1.0
+        version: link:../../tooling/typescript
+      oxlint:
+        specifier: ^1.29.0
+        version: 1.29.0
+      prettier:
+        specifier: ^3.4.2
+        version: 3.4.2
+      tsx:
+        specifier: ^4.8.1
+        version: 4.20.3
+
   packages/db:
     dependencies:
       '@auth/core':
@@ -11280,6 +11320,10 @@ packages:
     resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==}
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
 
+  p-limit@7.2.0:
+    resolution: {integrity: sha512-ATHLtwoTNDloHRFFxFJdHnG6n2WUeFjaR8XQMFdKIv0xkXjrER8/iG9iu265jOM95zXHAfv9oTkqhrfbIzosrQ==}
+    engines: {node: '>=20'}
+
   p-locate@4.1.0:
     resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==}
     engines: {node: '>=8'}
@@ -13843,6 +13887,10 @@ packages:
   tinybench@2.9.0:
     resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==}
 
+  tinybench@6.0.0:
+    resolution: {integrity: sha512-BWlWpVbbZXaYjRV0twGLNQO00Zj4HA/sjLOQP2IvzQqGwRGp+2kh7UU3ijyJ3ywFRogYDRbiHDMrUOfaMnN56g==}
+    engines: {node: '>=20.0.0'}
+
   tinyexec@0.3.2:
     resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==}
 
@@ -27802,6 +27850,10 @@ snapshots:
     dependencies:
       yocto-queue: 1.2.1
 
+  p-limit@7.2.0:
+    dependencies:
+      yocto-queue: 1.2.1
+
   p-locate@4.1.0:
     dependencies:
       p-limit: 2.3.0
@@ -30897,6 +30949,8 @@ snapshots:
 
   tinybench@2.9.0: {}
 
+  tinybench@6.0.0: {}
+
   tinyexec@0.3.2: {}
 
   tinyexec@1.0.1: {}