From 265b677302fb1f63e6311adcd97685aeb1a99f82 Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Tue, 9 Dec 2025 11:36:07 +0000 Subject: chore: Allowing multi user benchmarks and adding more coverage --- packages/benchmarks/src/benchmarks.ts | 70 ++++++++++++++++++++++ packages/benchmarks/src/index.ts | 10 +++- packages/benchmarks/src/seed.ts | 108 +++++++++++++++++++++++++--------- 3 files changed, 157 insertions(+), 31 deletions(-) (limited to 'packages/benchmarks') diff --git a/packages/benchmarks/src/benchmarks.ts b/packages/benchmarks/src/benchmarks.ts index f2883246..5f3601e4 100644 --- a/packages/benchmarks/src/benchmarks.ts +++ b/packages/benchmarks/src/benchmarks.ts @@ -81,6 +81,76 @@ export async function runBenchmarks( }); }); + // Benchmark with cursor (without listId) + { + const firstPage = await seed.trpc.bookmarks.getBookmarks.query({ + limit: 50, + }); + bench.add("bookmarks.getBookmarks (with cursor)", async () => { + if (firstPage.nextCursor) { + await seed.trpc.bookmarks.getBookmarks.query({ + limit: 50, + cursor: firstPage.nextCursor, + }); + } + }); + } + + // Benchmark with cursor and listId + if (sampleList) { + const firstPage = await seed.trpc.bookmarks.getBookmarks.query({ + limit: 50, + listId: sampleList.id, + }); + bench.add("bookmarks.getBookmarks (cursor + list filter)", async () => { + if (firstPage.nextCursor) { + await seed.trpc.bookmarks.getBookmarks.query({ + limit: 50, + listId: sampleList.id, + cursor: firstPage.nextCursor, + }); + } + }); + } + + // Benchmark with archived filter + bench.add("bookmarks.getBookmarks (archived filter)", async () => { + await seed.trpc.bookmarks.getBookmarks.query({ + limit: 50, + archived: true, + }); + }); + + // Benchmark with favourited filter + bench.add("bookmarks.getBookmarks (favourited filter)", async () => { + await seed.trpc.bookmarks.getBookmarks.query({ + limit: 50, + favourited: true, + }); + }); + + // Benchmark with archived and list filter combined + if (sampleList) { + bench.add("bookmarks.getBookmarks (archived + list filter)", async () => { + await seed.trpc.bookmarks.getBookmarks.query({ + limit: 50, + archived: true, + listId: sampleList.id, + }); + }); + } + + // Benchmark with favourited and list filter combined + if (sampleList) { + bench.add("bookmarks.getBookmarks (favourited + list filter)", async () => { + await seed.trpc.bookmarks.getBookmarks.query({ + limit: 50, + favourited: true, + listId: sampleList.id, + }); + }); + } + logStep("Running benchmarks"); await bench.run(); logSuccess("Benchmarks complete"); diff --git a/packages/benchmarks/src/index.ts b/packages/benchmarks/src/index.ts index 9633da6e..bc9d74a0 100644 --- a/packages/benchmarks/src/index.ts +++ b/packages/benchmarks/src/index.ts @@ -8,6 +8,7 @@ interface CliConfig { tagCount: number; listCount: number; concurrency: number; + userCount: number; keepContainers: boolean; timeMs: number; warmupMs: number; @@ -26,6 +27,7 @@ function loadConfig(): CliConfig { tagCount: numberFromEnv("BENCH_TAGS", 25), listCount: numberFromEnv("BENCH_LISTS", 6), concurrency: numberFromEnv("BENCH_SEED_CONCURRENCY", 12), + userCount: numberFromEnv("BENCH_USERS", 3), keepContainers: process.env.BENCH_KEEP_CONTAINERS === "1", timeMs: numberFromEnv("BENCH_TIME_MS", 1000), warmupMs: numberFromEnv("BENCH_WARMUP_MS", 300), @@ -36,9 +38,10 @@ async function main() { const config = loadConfig(); logStep("Benchmark configuration"); - logInfo(`Bookmarks: ${config.bookmarkCount}`); - logInfo(`Tags: ${config.tagCount}`); - logInfo(`Lists: ${config.listCount}`); + logInfo(`Users: ${config.userCount}`); + logInfo(`Bookmarks: ${config.bookmarkCount} per user`); + logInfo(`Tags: ${config.tagCount} per user`); + logInfo(`Lists: ${config.listCount} per user`); logInfo(`Seed concur.: ${config.concurrency}`); logInfo(`Time per case:${config.timeMs}ms (warmup ${config.warmupMs}ms)`); logInfo(`Keep containers after run: ${config.keepContainers ? "yes" : "no"}`); @@ -70,6 +73,7 @@ async function main() { tagCount: config.tagCount, listCount: config.listCount, concurrency: config.concurrency, + userCount: config.userCount, }); await runBenchmarks(seedResult, { diff --git a/packages/benchmarks/src/seed.ts b/packages/benchmarks/src/seed.ts index 286a1f66..dfb14347 100644 --- a/packages/benchmarks/src/seed.ts +++ b/packages/benchmarks/src/seed.ts @@ -13,6 +13,7 @@ export interface SeedConfig { tagCount: number; listCount: number; concurrency: number; + userCount: number; } export interface SeededBookmark { @@ -22,13 +23,24 @@ export interface SeededBookmark { title: string | null | undefined; } -export interface SeedResult { +export interface UserSeedData { apiKey: string; trpc: TrpcClient; + email: string; tags: ZTagBasic[]; lists: ZBookmarkList[]; bookmarks: SeededBookmark[]; +} + +export interface SeedResult { + users: UserSeedData[]; searchTerm: string; + // For backwards compatibility, expose the first user's data + apiKey: string; + trpc: TrpcClient; + tags: ZTagBasic[]; + lists: ZBookmarkList[]; + bookmarks: SeededBookmark[]; } const TOPICS = [ @@ -44,14 +56,18 @@ const TOPICS = [ "cli", ]; -export async function seedData(config: SeedConfig): Promise { - const authlessClient = getTrpcClient(); - const email = `benchmarks+${Date.now()}@example.com`; +async function seedUserData( + authlessClient: TrpcClient, + userIndex: number, + config: SeedConfig, + timestamp: number, +): Promise { + const email = `benchmarks+${timestamp}+user${userIndex}@example.com`; const password = "benchmarks1234"; - logStep("Creating benchmark user and API key"); + logStep(`Creating user ${userIndex + 1}/${config.userCount}`); await authlessClient.users.create.mutate({ - name: "Benchmark User", + name: `Benchmark User ${userIndex + 1}`, email, password, confirmPassword: password, @@ -59,35 +75,37 @@ export async function seedData(config: SeedConfig): Promise { const { key } = await authlessClient.apiKeys.exchange.mutate({ email, password, - keyName: "benchmark-key", + keyName: `benchmark-key-${userIndex}`, }); const trpc = getTrpcClient(key); - logSuccess("User ready"); + logSuccess(`User ${userIndex + 1} ready`); - logStep(`Creating ${config.tagCount} tags`); + logStep(`Creating ${config.tagCount} tags for user ${userIndex + 1}`); const tags: ZTagBasic[] = []; for (let i = 0; i < config.tagCount; i++) { const tag = await trpc.tags.create.mutate({ - name: `topic-${i + 1}`, + name: `user${userIndex}-topic-${i + 1}`, }); tags.push(tag); } - logSuccess("Tags created"); + logSuccess(`Tags created for user ${userIndex + 1}`); - logStep(`Creating ${config.listCount} lists`); + logStep(`Creating ${config.listCount} lists for user ${userIndex + 1}`); const lists: ZBookmarkList[] = []; for (let i = 0; i < config.listCount; i++) { const list = await trpc.lists.create.mutate({ - name: `List ${i + 1}`, - description: `Auto-generated benchmark list #${i + 1}`, + name: `User ${userIndex + 1} List ${i + 1}`, + description: `Auto-generated benchmark list #${i + 1} for user ${userIndex + 1}`, icon: "bookmark", }); lists.push(list); } - logSuccess("Lists created"); + logSuccess(`Lists created for user ${userIndex + 1}`); - logStep(`Creating ${config.bookmarkCount} bookmarks`); + logStep( + `Creating ${config.bookmarkCount} bookmarks for user ${userIndex + 1}`, + ); const limit = pLimit(config.concurrency); const bookmarks: SeededBookmark[] = []; @@ -98,12 +116,12 @@ export async function seedData(config: SeedConfig): Promise { const createdAt = new Date(Date.now() - index * 3000); const bookmark = await trpc.bookmarks.createBookmark.mutate({ type: BookmarkTypes.LINK, - url: `https://example.com/${topic}/${index}`, - title: `Benchmark ${topic} article ${index}`, + url: `https://example.com/user${userIndex}/${topic}/${index}`, + title: `User ${userIndex + 1} ${topic} article ${index}`, source: "api", - summary: `Benchmark dataset entry about ${topic} performance and organization.`, + summary: `Benchmark dataset entry about ${topic} for user ${userIndex + 1}.`, favourited: index % 7 === 0, - archived: false, + archived: index % 11 === 0, createdAt, }); @@ -138,13 +156,37 @@ export async function seedData(config: SeedConfig): Promise { }), ), ); - logSuccess("Bookmarks created"); + logSuccess(`Bookmarks created for user ${userIndex + 1}`); + + return { + apiKey: key, + trpc, + email, + tags, + lists, + bookmarks, + }; +} + +export async function seedData(config: SeedConfig): Promise { + const authlessClient = getTrpcClient(); + const timestamp = Date.now(); + + logInfo(`Seeding data for ${config.userCount} users`); + const users: UserSeedData[] = []; + + // Create all users sequentially to avoid race conditions + for (let i = 0; i < config.userCount; i++) { + const userData = await seedUserData(authlessClient, i, config, timestamp); + users.push(userData); + } const searchTerm = "benchmark"; logStep("Waiting for search index to be ready"); + // Use the first user's client to check search readiness await waitUntil( async () => { - const results = await trpc.bookmarks.searchBookmarks.query({ + const results = await users[0].trpc.bookmarks.searchBookmarks.query({ text: searchTerm, limit: 1, }); @@ -156,16 +198,26 @@ export async function seedData(config: SeedConfig): Promise { ); logSuccess("Search index warmed up"); + const totalBookmarks = users.reduce( + (sum, user) => sum + user.bookmarks.length, + 0, + ); + const totalTags = users.reduce((sum, user) => sum + user.tags.length, 0); + const totalLists = users.reduce((sum, user) => sum + user.lists.length, 0); + logInfo( - `Seeded ${bookmarks.length} bookmarks across ${tags.length} tags and ${lists.length} lists`, + `Seeded ${totalBookmarks} bookmarks across ${totalTags} tags and ${totalLists} lists for ${config.userCount} users`, ); + // Return first user's data for backwards compatibility + const firstUser = users[0]; return { - apiKey: key, - trpc, - tags, - lists, - bookmarks, + users, searchTerm, + apiKey: firstUser.apiKey, + trpc: firstUser.trpc, + tags: firstUser.tags, + lists: firstUser.lists, + bookmarks: firstUser.bookmarks, }; } -- cgit v1.2.3-70-g09d2