From b94896a0f8fa43b957a9bdd6ab57ada0ab8101af Mon Sep 17 00:00:00 2001 From: MohamedBassem Date: Sun, 27 Jul 2025 19:37:11 +0100 Subject: refactor: Extract meilisearch as a plugin --- packages/shared/config.ts | 8 ---- packages/shared/package.json | 1 - packages/shared/plugins.ts | 64 +++++++++++++++++++++++++++++++ packages/shared/search.ts | 90 ++++++++++++++------------------------------ 4 files changed, 92 insertions(+), 71 deletions(-) create mode 100644 packages/shared/plugins.ts (limited to 'packages/shared') diff --git a/packages/shared/config.ts b/packages/shared/config.ts index 8a41f6b5..a71014f0 100644 --- a/packages/shared/config.ts +++ b/packages/shared/config.ts @@ -76,8 +76,6 @@ const allEnv = z.object({ .default("") .transform((t) => t.split("%%").filter((a) => a)), CRAWLER_SCREENSHOT_TIMEOUT_SEC: z.coerce.number().default(5), - MEILI_ADDR: z.string().optional(), - MEILI_MASTER_KEY: z.string().default(""), LOG_LEVEL: z.string().default("debug"), DEMO_MODE: stringBool("false"), DEMO_MODE_EMAIL: z.string().optional(), @@ -231,12 +229,6 @@ const serverConfigSchema = allEnv }, search: { numWorkers: val.SEARCH_NUM_WORKERS, - meilisearch: val.MEILI_ADDR - ? { - address: val.MEILI_ADDR, - key: val.MEILI_MASTER_KEY, - } - : undefined, }, logLevel: val.LOG_LEVEL, demoMode: val.DEMO_MODE diff --git a/packages/shared/package.json b/packages/shared/package.json index 70859911..a0e6d2e8 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -10,7 +10,6 @@ "html-to-text": "^9.0.5", "js-tiktoken": "^1.0.20", "liteque": "^0.5.0", - "meilisearch": "^0.37.0", "nodemailer": "^7.0.4", "ollama": "^0.5.14", "openai": "^4.86.1", diff --git a/packages/shared/plugins.ts b/packages/shared/plugins.ts new file mode 100644 index 00000000..2ce5826a --- /dev/null +++ b/packages/shared/plugins.ts @@ -0,0 +1,64 @@ +// Implementation inspired from Outline + +import logger from "./logger"; +import { SearchIndexClient } from "./search"; + +export enum PluginType { + Search = "search", +} + +interface PluginTypeMap { + [PluginType.Search]: SearchIndexClient; +} + +export interface TPlugin { + type: T; + name: string; + provider: PluginProvider; +} + +export interface PluginProvider { + getClient(): Promise; +} + +export class PluginManager { + private static providers = new Map[]>(); + + static register(plugin: TPlugin): void { + const p = PluginManager.providers.get(plugin.type); + if (!p) { + PluginManager.providers.set(plugin.type, [plugin]); + return; + } + p.push(plugin); + } + + static async getClient( + type: T, + ): Promise { + const provider = PluginManager.providers.get(type); + if (!provider) { + return null; + } + return await provider[provider.length - 1].provider.getClient(); + } + + static isRegistered(type: T): boolean { + return !!PluginManager.providers.get(type); + } + + static logAllPlugins() { + logger.info("Plugins (Last one wins):"); + for (const type of Object.values(PluginType)) { + logger.info(` ${type}:`); + const plugins = PluginManager.providers.get(type); + if (!plugins) { + logger.info(" - None"); + continue; + } + for (const plugin of plugins) { + logger.info(` - ${plugin.name}`); + } + } + } +} diff --git a/packages/shared/search.ts b/packages/shared/search.ts index 2c6904b2..2afc9763 100644 --- a/packages/shared/search.ts +++ b/packages/shared/search.ts @@ -1,10 +1,8 @@ -import type { Index } from "meilisearch"; -import { MeiliSearch } from "meilisearch"; import { z } from "zod"; -import serverConfig from "./config"; +import { PluginManager, PluginType } from "./plugins"; -export const zBookmarkIdxSchema = z.object({ +export const zBookmarkSearchDocument = z.object({ id: z.string(), userId: z.string(), url: z.string().nullish(), @@ -24,68 +22,36 @@ export const zBookmarkIdxSchema = z.object({ dateModified: z.date().nullish(), }); -export type ZBookmarkIdx = z.infer; +export type BookmarkSearchDocument = z.infer; -let searchClient: MeiliSearch | undefined; - -if (serverConfig.search.meilisearch) { - searchClient = new MeiliSearch({ - host: serverConfig.search.meilisearch.address, - apiKey: serverConfig.search.meilisearch.key, - }); +export interface SearchResult { + id: string; + score?: number; } -const BOOKMARKS_IDX_NAME = "bookmarks"; - -let idxClient: Index | undefined; - -export async function getSearchIdxClient(): Promise | null> { - if (idxClient) { - return idxClient; - } - if (!searchClient) { - return null; - } - - const indicies = await searchClient.getIndexes(); - let idxFound = indicies.results.find((i) => i.uid == BOOKMARKS_IDX_NAME); - if (!idxFound) { - const idx = await searchClient.createIndex(BOOKMARKS_IDX_NAME, { - primaryKey: "id", - }); - await searchClient.waitForTask(idx.taskUid); - idxFound = await searchClient.getIndex(BOOKMARKS_IDX_NAME); - } +export interface SearchOptions { + query: string; + filter?: string[]; + limit?: number; + offset?: number; + sort?: string[]; +} - const desiredFilterableAttributes = ["id", "userId"].sort(); - const desiredSortableAttributes = ["createdAt"].sort(); +export interface SearchResponse { + hits: SearchResult[]; + totalHits: number; + processingTimeMs: number; +} - const settings = await idxFound.getSettings(); - if ( - JSON.stringify(settings.filterableAttributes?.sort()) != - JSON.stringify(desiredFilterableAttributes) - ) { - console.log( - `[meilisearch] Updating desired filterable attributes to ${desiredFilterableAttributes} from ${settings.filterableAttributes}`, - ); - const taskId = await idxFound.updateFilterableAttributes( - desiredFilterableAttributes, - ); - await searchClient.waitForTask(taskId.taskUid); - } +export interface SearchIndexClient { + addDocuments(documents: BookmarkSearchDocument[]): Promise; + updateDocuments(documents: BookmarkSearchDocument[]): Promise; + deleteDocument(id: string): Promise; + deleteDocuments(ids: string[]): Promise; + search(options: SearchOptions): Promise; + clearIndex(): Promise; +} - if ( - JSON.stringify(settings.sortableAttributes?.sort()) != - JSON.stringify(desiredSortableAttributes) - ) { - console.log( - `[meilisearch] Updating desired sortable attributes to ${desiredSortableAttributes} from ${settings.sortableAttributes}`, - ); - const taskId = await idxFound.updateSortableAttributes( - desiredSortableAttributes, - ); - await searchClient.waitForTask(taskId.taskUid); - } - idxClient = idxFound; - return idxFound; +export async function getSearchClient(): Promise { + return PluginManager.getClient(PluginType.Search); } -- cgit v1.2.3-70-g09d2