refactor: Extract meilisearch as a plugin

4 files changed, 92 insertions, 71 deletions

diff --git a/packages/shared/config.ts b/packages/shared/config.ts
index 8a41f6b5..a71014f0 100644
--- a/packages/shared/config.ts
+++ b/packages/shared/config.ts
@@ -76,8 +76,6 @@ const allEnv = z.object({
     .default("")
     .transform((t) => t.split("%%").filter((a) => a)),
   CRAWLER_SCREENSHOT_TIMEOUT_SEC: z.coerce.number().default(5),
-  MEILI_ADDR: z.string().optional(),
-  MEILI_MASTER_KEY: z.string().default(""),
   LOG_LEVEL: z.string().default("debug"),
   DEMO_MODE: stringBool("false"),
   DEMO_MODE_EMAIL: z.string().optional(),
@@ -231,12 +229,6 @@ const serverConfigSchema = allEnv
       },
       search: {
         numWorkers: val.SEARCH_NUM_WORKERS,
-        meilisearch: val.MEILI_ADDR
-          ? {
-              address: val.MEILI_ADDR,
-              key: val.MEILI_MASTER_KEY,
-            }
-          : undefined,
       },
       logLevel: val.LOG_LEVEL,
       demoMode: val.DEMO_MODE
diff --git a/packages/shared/package.json b/packages/shared/package.json
index 70859911..a0e6d2e8 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -10,7 +10,6 @@
     "html-to-text": "^9.0.5",
     "js-tiktoken": "^1.0.20",
     "liteque": "^0.5.0",
-    "meilisearch": "^0.37.0",
     "nodemailer": "^7.0.4",
     "ollama": "^0.5.14",
     "openai": "^4.86.1",
diff --git a/packages/shared/plugins.ts b/packages/shared/plugins.ts
new file mode 100644
index 00000000..2ce5826a
--- /dev/null
+++ b/packages/shared/plugins.ts
@@ -0,0 +1,64 @@
+// Implementation inspired from Outline
+
+import logger from "./logger";
+import { SearchIndexClient } from "./search";
+
+export enum PluginType {
+  Search = "search",
+}
+
+interface PluginTypeMap {
+  [PluginType.Search]: SearchIndexClient;
+}
+
+export interface TPlugin<T extends PluginType> {
+  type: T;
+  name: string;
+  provider: PluginProvider<PluginTypeMap[T]>;
+}
+
+export interface PluginProvider<T> {
+  getClient(): Promise<T | null>;
+}
+
+export class PluginManager {
+  private static providers = new Map<PluginType, TPlugin<PluginType>[]>();
+
+  static register<T extends PluginType>(plugin: TPlugin<T>): void {
+    const p = PluginManager.providers.get(plugin.type);
+    if (!p) {
+      PluginManager.providers.set(plugin.type, [plugin]);
+      return;
+    }
+    p.push(plugin);
+  }
+
+  static async getClient<T extends PluginType>(
+    type: T,
+  ): Promise<PluginTypeMap[T] | null> {
+    const provider = PluginManager.providers.get(type);
+    if (!provider) {
+      return null;
+    }
+    return await provider[provider.length - 1].provider.getClient();
+  }
+
+  static isRegistered<T extends PluginType>(type: T): boolean {
+    return !!PluginManager.providers.get(type);
+  }
+
+  static logAllPlugins() {
+    logger.info("Plugins (Last one wins):");
+    for (const type of Object.values(PluginType)) {
+      logger.info(`  ${type}:`);
+      const plugins = PluginManager.providers.get(type);
+      if (!plugins) {
+        logger.info("    - None");
+        continue;
+      }
+      for (const plugin of plugins) {
+        logger.info(`    - ${plugin.name}`);
+      }
+    }
+  }
+}
diff --git a/packages/shared/search.ts b/packages/shared/search.ts
index 2c6904b2..2afc9763 100644
--- a/packages/shared/search.ts
+++ b/packages/shared/search.ts
@@ -1,10 +1,8 @@
-import type { Index } from "meilisearch";
-import { MeiliSearch } from "meilisearch";
 import { z } from "zod";
 
-import serverConfig from "./config";
+import { PluginManager, PluginType } from "./plugins";
 
-export const zBookmarkIdxSchema = z.object({
+export const zBookmarkSearchDocument = z.object({
   id: z.string(),
   userId: z.string(),
   url: z.string().nullish(),
@@ -24,68 +22,36 @@ export const zBookmarkIdxSchema = z.object({
   dateModified: z.date().nullish(),
 });
 
-export type ZBookmarkIdx = z.infer<typeof zBookmarkIdxSchema>;
+export type BookmarkSearchDocument = z.infer<typeof zBookmarkSearchDocument>;
 
-let searchClient: MeiliSearch | undefined;
-
-if (serverConfig.search.meilisearch) {
-  searchClient = new MeiliSearch({
-    host: serverConfig.search.meilisearch.address,
-    apiKey: serverConfig.search.meilisearch.key,
-  });
+export interface SearchResult {
+  id: string;
+  score?: number;
 }
 
-const BOOKMARKS_IDX_NAME = "bookmarks";
-
-let idxClient: Index<ZBookmarkIdx> | undefined;
-
-export async function getSearchIdxClient(): Promise<Index<ZBookmarkIdx> | null> {
-  if (idxClient) {
-    return idxClient;
-  }
-  if (!searchClient) {
-    return null;
-  }
-
-  const indicies = await searchClient.getIndexes();
-  let idxFound = indicies.results.find((i) => i.uid == BOOKMARKS_IDX_NAME);
-  if (!idxFound) {
-    const idx = await searchClient.createIndex(BOOKMARKS_IDX_NAME, {
-      primaryKey: "id",
-    });
-    await searchClient.waitForTask(idx.taskUid);
-    idxFound = await searchClient.getIndex<ZBookmarkIdx>(BOOKMARKS_IDX_NAME);
-  }
+export interface SearchOptions {
+  query: string;
+  filter?: string[];
+  limit?: number;
+  offset?: number;
+  sort?: string[];
+}
 
-  const desiredFilterableAttributes = ["id", "userId"].sort();
-  const desiredSortableAttributes = ["createdAt"].sort();
+export interface SearchResponse {
+  hits: SearchResult[];
+  totalHits: number;
+  processingTimeMs: number;
+}
 
-  const settings = await idxFound.getSettings();
-  if (
-    JSON.stringify(settings.filterableAttributes?.sort()) !=
-    JSON.stringify(desiredFilterableAttributes)
-  ) {
-    console.log(
-      `[meilisearch] Updating desired filterable attributes to ${desiredFilterableAttributes} from ${settings.filterableAttributes}`,
-    );
-    const taskId = await idxFound.updateFilterableAttributes(
-      desiredFilterableAttributes,
-    );
-    await searchClient.waitForTask(taskId.taskUid);
-  }
+export interface SearchIndexClient {
+  addDocuments(documents: BookmarkSearchDocument[]): Promise<void>;
+  updateDocuments(documents: BookmarkSearchDocument[]): Promise<void>;
+  deleteDocument(id: string): Promise<void>;
+  deleteDocuments(ids: string[]): Promise<void>;
+  search(options: SearchOptions): Promise<SearchResponse>;
+  clearIndex(): Promise<void>;
+}
 
-  if (
-    JSON.stringify(settings.sortableAttributes?.sort()) !=
-    JSON.stringify(desiredSortableAttributes)
-  ) {
-    console.log(
-      `[meilisearch] Updating desired sortable attributes to ${desiredSortableAttributes} from ${settings.sortableAttributes}`,
-    );
-    const taskId = await idxFound.updateSortableAttributes(
-      desiredSortableAttributes,
-    );
-    await searchClient.waitForTask(taskId.taskUid);
-  }
-  idxClient = idxFound;
-  return idxFound;
+export async function getSearchClient(): Promise<SearchIndexClient | null> {
+  return PluginManager.getClient(PluginType.Search);
 }