Commit b94896a0
| SHA | b94896a0f8fa43b957a9bdd6ab57ada0ab8101af |
|---|---|
| Author | MohamedBassem <me at mbassem dot com> |
| Author Date | 2025-07-27 19:37 +0100 |
| Committer | MohamedBassem <me at mbassem dot com> |
| Commit Date | 2025-07-27 19:37 +0100 |
| Parent(s) | 7bb7f18fbf8e (diff) |
| Tree | ed8f79ce7d40 |
refactor: Extract meilisearch as a plugin
26 file(s) changed, 524 insertions(+), 155 deletions(-)
apps/web/app/dashboard/layout.tsx
diff --git a/apps/web/app/dashboard/layout.tsx b/apps/web/app/dashboard/layout.tsx index 670286ea..1471bfde 100644 --- a/apps/web/app/dashboard/layout.tsx +++ b/apps/web/app/dashboard/layout.tsx @@ -17,7 +17,7 @@ import { Tag, } from "lucide-react"; -import serverConfig from "@karakeep/shared/config"; +import { PluginManager, PluginType } from "@karakeep/shared/plugins"; export default async function Dashboard({ children, @@ -43,7 +43,7 @@ export default async function Dashboard({ icon: <Home size={18} />, path: "/dashboard/bookmarks", }, - serverConfig.search.meilisearch + PluginManager.isRegistered(PluginType.Search) ? [ { name: t("common.search"),
apps/web/app/layout.tsx
diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsx index e8673c78..55cab4fd 100644 --- a/apps/web/app/layout.tsx +++ b/apps/web/app/layout.tsx @@ -2,6 +2,8 @@ import type { Metadata } from "next"; import { Inter } from "next/font/google"; import { NuqsAdapter } from "nuqs/adapters/next/app"; +import { loadAllPlugins } from "@karakeep/shared-server"; + import "@karakeep/tailwind-config/globals.css"; import type { Viewport } from "next"; @@ -14,6 +16,8 @@ import { ReactQueryDevtools } from "@tanstack/react-query-devtools"; import { clientConfig } from "@karakeep/shared/config"; +await loadAllPlugins(); + const inter = Inter({ subsets: ["latin"], fallback: ["sans-serif"],
apps/web/package.json
diff --git a/apps/web/package.json b/apps/web/package.json index 3721d3b2..778a7419 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -25,6 +25,7 @@ "@karakeep/db": "workspace:^0.1.0", "@karakeep/shared": "workspace:^0.1.0", "@karakeep/shared-react": "workspace:^0.1.0", + "@karakeep/shared-server": "workspace:^0.1.0", "@karakeep/trpc": "workspace:^0.1.0", "@lexical/list": "^0.20.2", "@lexical/markdown": "^0.20.2",
apps/workers/index.ts
diff --git a/apps/workers/index.ts b/apps/workers/index.ts index 1cc1ce49..a21b9c2d 100644 --- a/apps/workers/index.ts +++ b/apps/workers/index.ts @@ -1,5 +1,6 @@ import "dotenv/config"; +import { loadAllPlugins } from "@karakeep/shared-server"; import serverConfig from "@karakeep/shared/config"; import logger from "@karakeep/shared/logger"; import { runQueueDBMigrations } from "@karakeep/shared/queues"; @@ -16,6 +17,7 @@ import { VideoWorker } from "./workers/videoWorker"; import { WebhookWorker } from "./workers/webhookWorker"; async function main() { + await loadAllPlugins(); logger.info(`Workers version: ${serverConfig.serverVersion ?? "not set"}`); runQueueDBMigrations();
apps/workers/package.json
diff --git a/apps/workers/package.json b/apps/workers/package.json index aa30878a..a771c710 100644 --- a/apps/workers/package.json +++ b/apps/workers/package.json @@ -8,6 +8,7 @@ "@ghostery/adblocker-playwright": "^2.5.1", "@karakeep/db": "workspace:^0.1.0", "@karakeep/shared": "workspace:^0.1.0", + "@karakeep/shared-server": "workspace:^0.1.0", "@karakeep/trpc": "workspace:^0.1.0", "@karakeep/tsconfig": "workspace:^0.1.0", "@mozilla/readability": "^0.6.0",
apps/workers/workers/searchWorker.ts
diff --git a/apps/workers/workers/searchWorker.ts b/apps/workers/workers/searchWorker.ts index 74fcfc42..4c924ceb 100644 --- a/apps/workers/workers/searchWorker.ts +++ b/apps/workers/workers/searchWorker.ts @@ -10,7 +10,11 @@ import { SearchIndexingQueue, zSearchIndexingRequestSchema, } from "@karakeep/shared/queues"; -import { getSearchIdxClient } from "@karakeep/shared/search"; +import { + BookmarkSearchDocument, + getSearchClient, + SearchIndexClient, +} from "@karakeep/shared/search"; import { Bookmark } from "@karakeep/trpc/models/bookmarks"; export class SearchIndexingWorker { @@ -44,20 +48,7 @@ export class SearchIndexingWorker { } } -async function ensureTaskSuccess( - searchClient: NonNullable<Awaited<ReturnType<typeof getSearchIdxClient>>>, - taskUid: number, -) { - const task = await searchClient.waitForTask(taskUid); - if (task.error) { - throw new Error(`Search task failed: ${task.error.message}`); - } -} - -async function runIndex( - searchClient: NonNullable<Awaited<ReturnType<typeof getSearchIdxClient>>>, - bookmarkId: string, -) { +async function runIndex(searchClient: SearchIndexClient, bookmarkId: string) { const bookmark = await db.query.bookmarks.findFirst({ where: eq(bookmarks.id, bookmarkId), with: { @@ -76,53 +67,43 @@ async function runIndex( throw new Error(`Bookmark ${bookmarkId} not found`); } - const task = await searchClient.addDocuments( - [ - { - id: bookmark.id, - userId: bookmark.userId, - ...(bookmark.link - ? { - url: bookmark.link.url, - linkTitle: bookmark.link.title, - description: bookmark.link.description, - content: await Bookmark.getBookmarkPlainTextContent( - bookmark.link, - bookmark.userId, - ), - publisher: bookmark.link.publisher, - author: bookmark.link.author, - datePublished: bookmark.link.datePublished, - dateModified: bookmark.link.dateModified, - } - : undefined), - ...(bookmark.asset - ? { - content: bookmark.asset.content, - metadata: bookmark.asset.metadata, - } - : undefined), - ...(bookmark.text ? { content: bookmark.text.text } : undefined), - note: bookmark.note, - summary: bookmark.summary, - title: bookmark.title, - createdAt: bookmark.createdAt.toISOString(), - tags: bookmark.tagsOnBookmarks.map((t) => t.tag.name), - }, - ], - { - primaryKey: "id", - }, - ); - await ensureTaskSuccess(searchClient, task.taskUid); + const document: BookmarkSearchDocument = { + id: bookmark.id, + userId: bookmark.userId, + ...(bookmark.link + ? { + url: bookmark.link.url, + linkTitle: bookmark.link.title, + description: bookmark.link.description, + content: await Bookmark.getBookmarkPlainTextContent( + bookmark.link, + bookmark.userId, + ), + publisher: bookmark.link.publisher, + author: bookmark.link.author, + datePublished: bookmark.link.datePublished, + dateModified: bookmark.link.dateModified, + } + : {}), + ...(bookmark.asset + ? { + content: bookmark.asset.content, + metadata: bookmark.asset.metadata, + } + : {}), + ...(bookmark.text ? { content: bookmark.text.text } : {}), + note: bookmark.note, + summary: bookmark.summary, + title: bookmark.title, + createdAt: bookmark.createdAt.toISOString(), + tags: bookmark.tagsOnBookmarks.map((t) => t.tag.name), + }; + + await searchClient.addDocuments([document]); } -async function runDelete( - searchClient: NonNullable<Awaited<ReturnType<typeof getSearchIdxClient>>>, - bookmarkId: string, -) { - const task = await searchClient.deleteDocument(bookmarkId); - await ensureTaskSuccess(searchClient, task.taskUid); +async function runDelete(searchClient: SearchIndexClient, bookmarkId: string) { + await searchClient.deleteDocument(bookmarkId); } async function runSearchIndexing(job: DequeuedJob<ZSearchIndexingRequest>) { @@ -135,7 +116,7 @@ async function runSearchIndexing(job: DequeuedJob<ZSearchIndexingRequest>) { ); } - const searchClient = await getSearchIdxClient(); + const searchClient = await getSearchClient(); if (!searchClient) { logger.debug( `[search][${jobId}] Search is not configured, nothing to do now`,
packages/plugins-search-meilisearch/.oxlintrc.json
diff --git a/packages/plugins-search-meilisearch/.oxlintrc.json b/packages/plugins-search-meilisearch/.oxlintrc.json new file mode 100644 index 00000000..79ba0255 --- /dev/null +++ b/packages/plugins-search-meilisearch/.oxlintrc.json @@ -0,0 +1,19 @@ +{ + "$schema": "../../node_modules/oxlint/configuration_schema.json", + "extends": [ + "../../tooling/oxlint/oxlint-base.json" + ], + "env": { + "builtin": true, + "commonjs": true + }, + "ignorePatterns": [ + "**/*.config.js", + "**/*.config.cjs", + "**/.eslintrc.cjs", + "**/.next", + "**/dist", + "**/build", + "**/pnpm-lock.yaml" + ] +}
packages/plugins-search-meilisearch/index.ts
diff --git a/packages/plugins-search-meilisearch/index.ts b/packages/plugins-search-meilisearch/index.ts new file mode 100644 index 00000000..3496d52f --- /dev/null +++ b/packages/plugins-search-meilisearch/index.ts @@ -0,0 +1,12 @@ +// Auto-register the MeiliSearch provider when this package is imported +import { PluginManager, PluginType } from "@karakeep/shared/plugins"; + +import { MeiliSearchProvider } from "./src"; + +if (MeiliSearchProvider.isConfigured()) { + PluginManager.register({ + type: PluginType.Search, + name: "MeiliSearch", + provider: new MeiliSearchProvider(), + }); +}
packages/plugins-search-meilisearch/package.json
diff --git a/packages/plugins-search-meilisearch/package.json b/packages/plugins-search-meilisearch/package.json new file mode 100644 index 00000000..3bc9db80 --- /dev/null +++ b/packages/plugins-search-meilisearch/package.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json.schemastore.org/package.json", + "name": "@karakeep/plugins-search-meilisearch", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "typecheck": "tsc --noEmit", + "format": "prettier . --ignore-path ../../.prettierignore", + "format:fix": "prettier . --write --ignore-path ../../.prettierignore", + "lint": "oxlint .", + "lint:fix": "oxlint . --fix", + "test": "vitest" + }, + "dependencies": { + "@karakeep/shared": "workspace:*", + "meilisearch": "^0.45.0" + }, + "devDependencies": { + "@karakeep/prettier-config": "workspace:^0.1.0", + "@karakeep/tsconfig": "workspace:^0.1.0", + "vite-tsconfig-paths": "^4.3.1", + "vitest": "^3.2.4" + }, + "prettier": "@karakeep/prettier-config" +}
packages/plugins-search-meilisearch/src/env.ts
diff --git a/packages/plugins-search-meilisearch/src/env.ts b/packages/plugins-search-meilisearch/src/env.ts new file mode 100644 index 00000000..c06fdd55 --- /dev/null +++ b/packages/plugins-search-meilisearch/src/env.ts @@ -0,0 +1,8 @@ +import { z } from "zod"; + +export const envConfig = z + .object({ + MEILI_ADDR: z.string().optional(), + MEILI_MASTER_KEY: z.string().default(""), + }) + .parse(process.env);
packages/plugins-search-meilisearch/src/index.ts
diff --git a/packages/plugins-search-meilisearch/src/index.ts b/packages/plugins-search-meilisearch/src/index.ts new file mode 100644 index 00000000..6153a9c4 --- /dev/null +++ b/packages/plugins-search-meilisearch/src/index.ts @@ -0,0 +1,174 @@ +import type { Index } from "meilisearch"; +import { MeiliSearch } from "meilisearch"; + +import type { + BookmarkSearchDocument, + SearchIndexClient, + SearchOptions, + SearchResponse, +} from "@karakeep/shared/search"; +import { PluginProvider } from "@karakeep/shared/plugins"; + +import { envConfig } from "./env"; + +class MeiliSearchIndexClient implements SearchIndexClient { + constructor(private index: Index<BookmarkSearchDocument>) {} + + async addDocuments(documents: BookmarkSearchDocument[]): Promise<void> { + const task = await this.index.addDocuments(documents, { + primaryKey: "id", + }); + await this.index.waitForTask(task.taskUid); + const taskResult = await this.index.getTask(task.taskUid); + if (taskResult.error) { + throw new Error( + `MeiliSearch add documents failed: ${taskResult.error.message}`, + ); + } + } + + async updateDocuments(documents: BookmarkSearchDocument[]): Promise<void> { + const task = await this.index.updateDocuments(documents, { + primaryKey: "id", + }); + await this.index.waitForTask(task.taskUid); + const taskResult = await this.index.getTask(task.taskUid); + if (taskResult.error) { + throw new Error( + `MeiliSearch update documents failed: ${taskResult.error.message}`, + ); + } + } + + async deleteDocument(id: string): Promise<void> { + const task = await this.index.deleteDocument(id); + await this.index.waitForTask(task.taskUid); + const taskResult = await this.index.getTask(task.taskUid); + if (taskResult.error) { + throw new Error( + `MeiliSearch delete document failed: ${taskResult.error.message}`, + ); + } + } + + async deleteDocuments(ids: string[]): Promise<void> { + const task = await this.index.deleteDocuments(ids); + await this.index.waitForTask(task.taskUid); + const taskResult = await this.index.getTask(task.taskUid); + if (taskResult.error) { + throw new Error( + `MeiliSearch delete documents failed: ${taskResult.error.message}`, + ); + } + } + + async search(options: SearchOptions): Promise<SearchResponse> { + const result = await this.index.search(options.query, { + filter: options.filter, + limit: options.limit, + offset: options.offset, + sort: options.sort, + }); + + return { + hits: result.hits.map((hit) => ({ + id: hit.id, + score: hit._rankingScore, + })), + totalHits: result.estimatedTotalHits ?? 0, + processingTimeMs: result.processingTimeMs, + }; + } + + async clearIndex(): Promise<void> { + const task = await this.index.deleteAllDocuments(); + await this.index.waitForTask(task.taskUid); + const taskResult = await this.index.getTask(task.taskUid); + if (taskResult.error) { + throw new Error( + `MeiliSearch clear index failed: ${taskResult.error.message}`, + ); + } + } +} + +export class MeiliSearchProvider implements PluginProvider<SearchIndexClient> { + private client: MeiliSearch | undefined; + private indexClient: SearchIndexClient | undefined; + private readonly indexName = "bookmarks"; + + constructor() { + if (MeiliSearchProvider.isConfigured()) { + this.client = new MeiliSearch({ + host: envConfig.MEILI_ADDR!, + apiKey: envConfig.MEILI_MASTER_KEY, + }); + } + } + + static isConfigured(): boolean { + return !!envConfig.MEILI_ADDR; + } + + async getClient(): Promise<SearchIndexClient | null> { + if (this.indexClient) { + return this.indexClient; + } + + if (!this.client) { + return null; + } + + const indices = await this.client.getIndexes(); + let indexFound = indices.results.find((i) => i.uid === this.indexName); + + if (!indexFound) { + const idx = await this.client.createIndex(this.indexName, { + primaryKey: "id", + }); + await this.client.waitForTask(idx.taskUid); + indexFound = await this.client.getIndex<BookmarkSearchDocument>( + this.indexName, + ); + } + + await this.configureIndex(indexFound); + this.indexClient = new MeiliSearchIndexClient(indexFound); + return this.indexClient; + } + + private async configureIndex( + index: Index<BookmarkSearchDocument>, + ): Promise<void> { + const desiredFilterableAttributes = ["id", "userId"].sort(); + const desiredSortableAttributes = ["createdAt"].sort(); + + const settings = await index.getSettings(); + + if ( + JSON.stringify(settings.filterableAttributes?.sort()) !== + JSON.stringify(desiredFilterableAttributes) + ) { + console.log( + `[meilisearch] Updating desired filterable attributes to ${desiredFilterableAttributes} from ${settings.filterableAttributes}`, + ); + const taskId = await index.updateFilterableAttributes( + desiredFilterableAttributes, + ); + await this.client!.waitForTask(taskId.taskUid); + } + + if ( + JSON.stringify(settings.sortableAttributes?.sort()) !== + JSON.stringify(desiredSortableAttributes) + ) { + console.log( + `[meilisearch] Updating desired sortable attributes to ${desiredSortableAttributes} from ${settings.sortableAttributes}`, + ); + const taskId = await index.updateSortableAttributes( + desiredSortableAttributes, + ); + await this.client!.waitForTask(taskId.taskUid); + } + } +}
packages/plugins-search-meilisearch/tsconfig.json
diff --git a/packages/plugins-search-meilisearch/tsconfig.json b/packages/plugins-search-meilisearch/tsconfig.json new file mode 100644 index 00000000..a795b96a --- /dev/null +++ b/packages/plugins-search-meilisearch/tsconfig.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "@karakeep/tsconfig/node.json", + "include": ["**/*.ts"], + "exclude": ["node_modules"], + "compilerOptions": { + "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json" + } +}
packages/shared-server/.oxlintrc.json
diff --git a/packages/shared-server/.oxlintrc.json b/packages/shared-server/.oxlintrc.json new file mode 100644 index 00000000..79ba0255 --- /dev/null +++ b/packages/shared-server/.oxlintrc.json @@ -0,0 +1,19 @@ +{ + "$schema": "../../node_modules/oxlint/configuration_schema.json", + "extends": [ + "../../tooling/oxlint/oxlint-base.json" + ], + "env": { + "builtin": true, + "commonjs": true + }, + "ignorePatterns": [ + "**/*.config.js", + "**/*.config.cjs", + "**/.eslintrc.cjs", + "**/.next", + "**/dist", + "**/build", + "**/pnpm-lock.yaml" + ] +}
packages/shared-server/index.ts
diff --git a/packages/shared-server/index.ts b/packages/shared-server/index.ts new file mode 100644 index 00000000..3bd16e17 --- /dev/null +++ b/packages/shared-server/index.ts @@ -0,0 +1 @@ +export * from "./src";
packages/shared-server/package.json
diff --git a/packages/shared-server/package.json b/packages/shared-server/package.json new file mode 100644 index 00000000..8ac98e21 --- /dev/null +++ b/packages/shared-server/package.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json.schemastore.org/package.json", + "name": "@karakeep/shared-server", + "version": "0.1.0", + "private": true, + "type": "module", + "dependencies": { + "@karakeep/plugins-search-meilisearch": "workspace:^0.1.0", + "@karakeep/shared": "workspace:^0.1.0" + }, + "devDependencies": { + "@karakeep/prettier-config": "workspace:^0.1.0", + "@karakeep/tsconfig": "workspace:^0.1.0" + }, + "scripts": { + "typecheck": "tsc --noEmit", + "format": "prettier . --cache --ignore-path ../../.prettierignore --check", + "format:fix": "prettier . --cache --write --ignore-path ../../.prettierignore", + "lint": "oxlint .", + "lint:fix": "oxlint . --fix", + "test": "vitest" + }, + "main": "index.ts", + "exports": { + ".": "./index.ts" + }, + "prettier": "@karakeep/prettier-config" +}
packages/shared-server/src/index.ts
diff --git a/packages/shared-server/src/index.ts b/packages/shared-server/src/index.ts new file mode 100644 index 00000000..a17576ad --- /dev/null +++ b/packages/shared-server/src/index.ts @@ -0,0 +1 @@ +export { loadAllPlugins } from "./plugins";
packages/shared-server/src/plugins.ts
diff --git a/packages/shared-server/src/plugins.ts b/packages/shared-server/src/plugins.ts new file mode 100644 index 00000000..86a0b344 --- /dev/null +++ b/packages/shared-server/src/plugins.ts @@ -0,0 +1,12 @@ +import { PluginManager } from "@karakeep/shared/plugins"; + +let pluginsLoaded = false; +export async function loadAllPlugins() { + if (pluginsLoaded) { + return; + } + // Load plugins here. Order of plugin loading matter. + await import("@karakeep/plugins-search-meilisearch"); + PluginManager.logAllPlugins(); + pluginsLoaded = true; +}
packages/shared-server/tsconfig.json
diff --git a/packages/shared-server/tsconfig.json b/packages/shared-server/tsconfig.json new file mode 100644 index 00000000..9bb09964 --- /dev/null +++ b/packages/shared-server/tsconfig.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "@karakeep/tsconfig/node.json", + "include": ["src/**/*.ts"], + "exclude": ["node_modules"], + "compilerOptions": { + "rootDir": "src/", + "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json" + }, +}
packages/shared/config.ts
diff --git a/packages/shared/config.ts b/packages/shared/config.ts index 8a41f6b5..a71014f0 100644 --- a/packages/shared/config.ts +++ b/packages/shared/config.ts @@ -76,8 +76,6 @@ const allEnv = z.object({ .default("") .transform((t) => t.split("%%").filter((a) => a)), CRAWLER_SCREENSHOT_TIMEOUT_SEC: z.coerce.number().default(5), - MEILI_ADDR: z.string().optional(), - MEILI_MASTER_KEY: z.string().default(""), LOG_LEVEL: z.string().default("debug"), DEMO_MODE: stringBool("false"), DEMO_MODE_EMAIL: z.string().optional(), @@ -231,12 +229,6 @@ const serverConfigSchema = allEnv }, search: { numWorkers: val.SEARCH_NUM_WORKERS, - meilisearch: val.MEILI_ADDR - ? { - address: val.MEILI_ADDR, - key: val.MEILI_MASTER_KEY, - } - : undefined, }, logLevel: val.LOG_LEVEL, demoMode: val.DEMO_MODE
packages/shared/package.json
diff --git a/packages/shared/package.json b/packages/shared/package.json index 70859911..a0e6d2e8 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -10,7 +10,6 @@ "html-to-text": "^9.0.5", "js-tiktoken": "^1.0.20", "liteque": "^0.5.0", - "meilisearch": "^0.37.0", "nodemailer": "^7.0.4", "ollama": "^0.5.14", "openai": "^4.86.1",
packages/shared/plugins.ts
diff --git a/packages/shared/plugins.ts b/packages/shared/plugins.ts new file mode 100644 index 00000000..2ce5826a --- /dev/null +++ b/packages/shared/plugins.ts @@ -0,0 +1,64 @@ +// Implementation inspired from Outline + +import logger from "./logger"; +import { SearchIndexClient } from "./search"; + +export enum PluginType { + Search = "search", +} + +interface PluginTypeMap { + [PluginType.Search]: SearchIndexClient; +} + +export interface TPlugin<T extends PluginType> { + type: T; + name: string; + provider: PluginProvider<PluginTypeMap[T]>; +} + +export interface PluginProvider<T> { + getClient(): Promise<T | null>; +} + +export class PluginManager { + private static providers = new Map<PluginType, TPlugin<PluginType>[]>(); + + static register<T extends PluginType>(plugin: TPlugin<T>): void { + const p = PluginManager.providers.get(plugin.type); + if (!p) { + PluginManager.providers.set(plugin.type, [plugin]); + return; + } + p.push(plugin); + } + + static async getClient<T extends PluginType>( + type: T, + ): Promise<PluginTypeMap[T] | null> { + const provider = PluginManager.providers.get(type); + if (!provider) { + return null; + } + return await provider[provider.length - 1].provider.getClient(); + } + + static isRegistered<T extends PluginType>(type: T): boolean { + return !!PluginManager.providers.get(type); + } + + static logAllPlugins() { + logger.info("Plugins (Last one wins):"); + for (const type of Object.values(PluginType)) { + logger.info(` ${type}:`); + const plugins = PluginManager.providers.get(type); + if (!plugins) { + logger.info(" - None"); + continue; + } + for (const plugin of plugins) { + logger.info(` - ${plugin.name}`); + } + } + } +}
packages/shared/search.ts
diff --git a/packages/shared/search.ts b/packages/shared/search.ts index 2c6904b2..2afc9763 100644 --- a/packages/shared/search.ts +++ b/packages/shared/search.ts @@ -1,10 +1,8 @@ -import type { Index } from "meilisearch"; -import { MeiliSearch } from "meilisearch"; import { z } from "zod"; -import serverConfig from "./config"; +import { PluginManager, PluginType } from "./plugins"; -export const zBookmarkIdxSchema = z.object({ +export const zBookmarkSearchDocument = z.object({ id: z.string(), userId: z.string(), url: z.string().nullish(), @@ -24,68 +22,36 @@ export const zBookmarkIdxSchema = z.object({ dateModified: z.date().nullish(), }); -export type ZBookmarkIdx = z.infer<typeof zBookmarkIdxSchema>; +export type BookmarkSearchDocument = z.infer<typeof zBookmarkSearchDocument>; -let searchClient: MeiliSearch | undefined; - -if (serverConfig.search.meilisearch) { - searchClient = new MeiliSearch({ - host: serverConfig.search.meilisearch.address, - apiKey: serverConfig.search.meilisearch.key, - }); +export interface SearchResult { + id: string; + score?: number; } -const BOOKMARKS_IDX_NAME = "bookmarks"; - -let idxClient: Index<ZBookmarkIdx> | undefined; - -export async function getSearchIdxClient(): Promise<Index<ZBookmarkIdx> | null> { - if (idxClient) { - return idxClient; - } - if (!searchClient) { - return null; - } - - const indicies = await searchClient.getIndexes(); - let idxFound = indicies.results.find((i) => i.uid == BOOKMARKS_IDX_NAME); - if (!idxFound) { - const idx = await searchClient.createIndex(BOOKMARKS_IDX_NAME, { - primaryKey: "id", - }); - await searchClient.waitForTask(idx.taskUid); - idxFound = await searchClient.getIndex<ZBookmarkIdx>(BOOKMARKS_IDX_NAME); - } +export interface SearchOptions { + query: string; + filter?: string[]; + limit?: number; + offset?: number; + sort?: string[]; +} - const desiredFilterableAttributes = ["id", "userId"].sort(); - const desiredSortableAttributes = ["createdAt"].sort(); +export interface SearchResponse { + hits: SearchResult[]; + totalHits: number; + processingTimeMs: number; +} - const settings = await idxFound.getSettings(); - if ( - JSON.stringify(settings.filterableAttributes?.sort()) != - JSON.stringify(desiredFilterableAttributes) - ) { - console.log( - `[meilisearch] Updating desired filterable attributes to ${desiredFilterableAttributes} from ${settings.filterableAttributes}`, - ); - const taskId = await idxFound.updateFilterableAttributes( - desiredFilterableAttributes, - ); - await searchClient.waitForTask(taskId.taskUid); - } +export interface SearchIndexClient { + addDocuments(documents: BookmarkSearchDocument[]): Promise<void>; + updateDocuments(documents: BookmarkSearchDocument[]): Promise<void>; + deleteDocument(id: string): Promise<void>; + deleteDocuments(ids: string[]): Promise<void>; + search(options: SearchOptions): Promise<SearchResponse>; + clearIndex(): Promise<void>; +} - if ( - JSON.stringify(settings.sortableAttributes?.sort()) != - JSON.stringify(desiredSortableAttributes) - ) { - console.log( - `[meilisearch] Updating desired sortable attributes to ${desiredSortableAttributes} from ${settings.sortableAttributes}`, - ); - const taskId = await idxFound.updateSortableAttributes( - desiredSortableAttributes, - ); - await searchClient.waitForTask(taskId.taskUid); - } - idxClient = idxFound; - return idxFound; +export async function getSearchClient(): Promise<SearchIndexClient | null> { + return PluginManager.getClient(PluginType.Search); }
packages/trpc/package.json
diff --git a/packages/trpc/package.json b/packages/trpc/package.json index 8054e7c5..1da46e9a 100644 --- a/packages/trpc/package.json +++ b/packages/trpc/package.json @@ -14,6 +14,7 @@ }, "dependencies": { "@karakeep/db": "workspace:*", + "@karakeep/plugins-search-meilisearch": "workspace:*", "@karakeep/shared": "workspace:*", "@trpc/server": "^11.4.3", "bcryptjs": "^2.4.3",
packages/trpc/routers/admin.ts
diff --git a/packages/trpc/routers/admin.ts b/packages/trpc/routers/admin.ts index 9fd77f1d..1b069b9e 100644 --- a/packages/trpc/routers/admin.ts +++ b/packages/trpc/routers/admin.ts @@ -14,7 +14,7 @@ import { VideoWorkerQueue, WebhookQueue, } from "@karakeep/shared/queues"; -import { getSearchIdxClient } from "@karakeep/shared/search"; +import { getSearchClient } from "@karakeep/shared/search"; import { resetPasswordSchema, updateUserSchema, @@ -219,8 +219,8 @@ export const adminAppRouter = router({ ); }), reindexAllBookmarks: adminProcedure.mutation(async ({ ctx }) => { - const searchIdx = await getSearchIdxClient(); - await searchIdx?.deleteAllDocuments(); + const searchIdx = await getSearchClient(); + await searchIdx?.clearIndex(); const bookmarkIds = await ctx.db.query.bookmarks.findMany({ columns: { id: true,
packages/trpc/routers/bookmarks.ts
diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts index 9aa9ec1e..298f0961 100644 --- a/packages/trpc/routers/bookmarks.ts +++ b/packages/trpc/routers/bookmarks.ts @@ -38,7 +38,7 @@ import { triggerSearchReindex, triggerWebhook, } from "@karakeep/shared/queues"; -import { getSearchIdxClient } from "@karakeep/shared/search"; +import { getSearchClient } from "@karakeep/shared/search"; import { parseSearchQuery } from "@karakeep/shared/searchQueryParser"; import { BookmarkTypes, @@ -761,7 +761,7 @@ export const bookmarksAppRouter = router({ input.limit = DEFAULT_NUM_BOOKMARKS_PER_PAGE; } const sortOrder = input.sortOrder || "relevance"; - const client = await getSearchIdxClient(); + const client = await getSearchClient(); if (!client) { throw new TRPCError({ code: "INTERNAL_SERVER_ERROR", @@ -788,10 +788,9 @@ export const bookmarksAppRouter = router({ */ const createdAtSortOrder = sortOrder === "relevance" ? "desc" : sortOrder; - const resp = await client.search(parsedQuery.text, { + const resp = await client.search({ + query: parsedQuery.text, filter, - showRankingScore: true, - attributesToRetrieve: ["id"], sort: [`createdAt:${createdAtSortOrder}`], limit: input.limit, ...(input.cursor @@ -805,7 +804,7 @@ export const bookmarksAppRouter = router({ return { bookmarks: [], nextCursor: null }; } const idToRank = resp.hits.reduce<Record<string, number>>((acc, r) => { - acc[r.id] = r._rankingScore!; + acc[r.id] = r.score || 0; return acc; }, {}); const results = await ctx.db.query.bookmarks.findMany({ @@ -846,11 +845,11 @@ export const bookmarksAppRouter = router({ results.map((b) => toZodSchema(b, input.includeContent)), ), nextCursor: - resp.hits.length + resp.offset >= resp.estimatedTotalHits + resp.hits.length + (input.cursor?.offset || 0) >= resp.totalHits ? null : { ver: 1 as const, - offset: resp.hits.length + resp.offset, + offset: resp.hits.length + (input.cursor?.offset || 0), }, }; }),
pnpm-lock.yaml
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 550025ce..57cb33d6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -501,6 +501,9 @@ importers: '@karakeep/shared-react': specifier: workspace:^0.1.0 version: link:../../packages/shared-react + '@karakeep/shared-server': + specifier: workspace:^0.1.0 + version: link:../../packages/shared-server '@karakeep/trpc': specifier: workspace:^0.1.0 version: link:../../packages/trpc @@ -763,6 +766,9 @@ importers: '@karakeep/shared': specifier: workspace:^0.1.0 version: link:../../packages/shared + '@karakeep/shared-server': + specifier: workspace:^0.1.0 + version: link:../../packages/shared-server '@karakeep/trpc': specifier: workspace:^0.1.0 version: link:../../packages/trpc @@ -1100,6 +1106,28 @@ importers: specifier: ^4.8.1 version: 4.20.3 + packages/plugins-search-meilisearch: + dependencies: + '@karakeep/shared': + specifier: workspace:* + version: link:../shared + meilisearch: + specifier: ^0.45.0 + version: 0.45.0 + devDependencies: + '@karakeep/prettier-config': + specifier: workspace:^0.1.0 + version: link:../../tooling/prettier + '@karakeep/tsconfig': + specifier: workspace:^0.1.0 + version: link:../../tooling/typescript + vite-tsconfig-paths: + specifier: ^4.3.1 + version: 4.3.2(typescript@5.8.3)(vite@7.0.6(@types/node@22.15.30)(jiti@2.4.2)(lightningcss@1.30.1)(sass@1.89.1)(terser@5.41.0)(tsx@4.20.3)(yaml@2.8.0)) + vitest: + specifier: ^3.2.4 + version: 3.2.4(@types/debug@4.1.12)(@types/node@22.15.30)(happy-dom@17.4.9)(jiti@2.4.2)(jsdom@26.1.0)(lightningcss@1.30.1)(sass@1.89.1)(terser@5.41.0)(tsx@4.20.3)(yaml@2.8.0) + packages/sdk: dependencies: openapi-fetch: @@ -1145,9 +1173,6 @@ importers: liteque: specifier: ^0.5.0 version: 0.5.0(@opentelemetry/api@1.9.0)(@types/better-sqlite3@7.6.13)(@types/react@19.1.8)(better-sqlite3@11.3.0)(react@18.3.1) - meilisearch: - specifier: ^0.37.0 - version: 0.37.0(encoding@0.1.13) nodemailer: specifier: ^7.0.4 version: 7.0.4 @@ -1220,11 +1245,30 @@ importers: specifier: workspace:^0.1.0 version: link:../../tooling/typescript + packages/shared-server: + dependencies: + '@karakeep/plugins-search-meilisearch': + specifier: workspace:^0.1.0 + version: link:../plugins-search-meilisearch + '@karakeep/shared': + specifier: workspace:^0.1.0 + version: link:../shared + devDependencies: + '@karakeep/prettier-config': + specifier: workspace:^0.1.0 + version: link:../../tooling/prettier + '@karakeep/tsconfig': + specifier: workspace:^0.1.0 + version: link:../../tooling/typescript + packages/trpc: dependencies: '@karakeep/db': specifier: workspace:* version: link:../db + '@karakeep/plugins-search-meilisearch': + specifier: workspace:* + version: link:../plugins-search-meilisearch '@karakeep/shared': specifier: workspace:* version: link:../shared @@ -10127,8 +10171,8 @@ packages: resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} engines: {node: '>= 0.8'} - meilisearch@0.37.0: - resolution: {integrity: sha512-LdbK6JmRghCawrmWKJSEQF0OiE82md+YqJGE/U2JcCD8ROwlhTx0KM6NX4rQt0u0VpV0QZVG9umYiu3CSSIJAQ==} + meilisearch@0.45.0: + resolution: {integrity: sha512-+zCzEqE+CumY4icB0Vox180adZqaNtnr60hJWGiEdmol5eWmksfY8rYsTcz87styXC2ZOg+2yF56gdH6oyIBTA==} memfs@3.5.3: resolution: {integrity: sha512-UERzLsxzllchadvbPs5aolHh65ISpKpM+ccLbOJ8/vvpBKmAWf+la7dXFy7Mr0ySHbdHrFv5kGFCUHHe6GFEmw==} @@ -26800,11 +26844,7 @@ snapshots: media-typer@1.1.0: {} - meilisearch@0.37.0(encoding@0.1.13): - dependencies: - cross-fetch: 3.2.0(encoding@0.1.13) - transitivePeerDependencies: - - encoding + meilisearch@0.45.0: {} memfs@3.5.3: dependencies: