6 files changed, 172 insertions, 0 deletions
diff --git a/packages/shared/config.ts b/packages/shared/config.ts
new file mode 100644
index 00000000..1dee4c4d
--- /dev/null
+++ b/packages/shared/config.ts
@@ -0,0 +1,42 @@
+function buildAuthentikConfig() {
+  const { AUTHENTIK_ID, AUTHENTIK_SECRET, AUTHENTIK_ISSUER } = process.env;
+
+  if (!AUTHENTIK_ID || !AUTHENTIK_SECRET || !AUTHENTIK_ISSUER) {
+    return undefined;
+  }
+
+  return {
+    clientId: AUTHENTIK_ID,
+    clientSecret: AUTHENTIK_SECRET,
+    issuer: AUTHENTIK_ISSUER,
+  };
+}
+
+const serverConfig = {
+  apiUrl: process.env.API_URL || "http://localhost:3000",
+  auth: {
+    authentik: buildAuthentikConfig(),
+  },
+  openAI: {
+    apiKey: process.env.OPENAI_API_KEY,
+  },
+  bullMQ: {
+    redisHost: process.env.REDIS_HOST || "localhost",
+    redisPort: parseInt(process.env.REDIS_PORT || "6379"),
+  },
+  crawler: {
+    headlessBrowser: (process.env.CRAWLER_HEADLESS_BROWSER ?? "true") == "true",
+    browserExecutablePath: process.env.BROWSER_EXECUTABLE_PATH, // If not set, the system's browser will be used
+    browserUserDataDir: process.env.BROWSER_USER_DATA_DIR,
+  },
+  meilisearch: process.env.MEILI_ADDR
+    ? {
+        address: process.env.MEILI_ADDR || "http://127.0.0.1:7700",
+        key: process.env.MEILI_MASTER_KEY || "",
+      }
+    : undefined,
+  logLevel: process.env.LOG_LEVEL || "debug",
+  demoMode: (process.env.DEMO_MODE ?? "false") == "true",
+};
+
+export default serverConfig;
diff --git a/packages/shared/index.ts b/packages/shared/index.ts
new file mode 100644
index 00000000..8b93520f
--- /dev/null
+++ b/packages/shared/index.ts
@@ -0,0 +1 @@
+export * as Queues from "./queues.ts";
diff --git a/packages/shared/logger.ts b/packages/shared/logger.ts
new file mode 100644
index 00000000..471ec7ab
--- /dev/null
+++ b/packages/shared/logger.ts
@@ -0,0 +1,16 @@
+import winston from "winston";
+import serverConfig from "./config";
+
+const logger = winston.createLogger({
+  level: serverConfig.logLevel,
+  format: winston.format.combine(
+    winston.format.timestamp(),
+    winston.format.colorize(),
+    winston.format.printf(
+      (info) => `${info.timestamp} ${info.level}: ${info.message}`,
+    ),
+  ),
+  transports: [new winston.transports.Console()],
+});
+
+export default logger;
diff --git a/packages/shared/package.json b/packages/shared/package.json
new file mode 100644
index 00000000..0b3a8078
--- /dev/null
+++ b/packages/shared/package.json
@@ -0,0 +1,12 @@
+{
+  "$schema": "https://json.schemastore.org/package.json",
+  "name": "@hoarder/shared",
+  "version": "0.1.0",
+  "private": true,
+  "dependencies": {
+    "meilisearch": "^0.37.0",
+    "winston": "^3.11.0",
+    "zod": "^3.22.4"
+  },
+  "main": "index.ts"
+}
diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts
new file mode 100644
index 00000000..a2cbeceb
--- /dev/null
+++ b/packages/shared/queues.ts
@@ -0,0 +1,51 @@
+import { Queue } from "bullmq";
+import { z } from "zod";
+import serverConfig from "./config";
+
+export const queueConnectionDetails = {
+  host: serverConfig.bullMQ.redisHost,
+  port: serverConfig.bullMQ.redisPort,
+};
+
+// Link Crawler
+export const zCrawlLinkRequestSchema = z.object({
+  bookmarkId: z.string(),
+});
+export type ZCrawlLinkRequest = z.infer<typeof zCrawlLinkRequestSchema>;
+
+export const LinkCrawlerQueue = new Queue<ZCrawlLinkRequest, void>(
+  "link_crawler_queue",
+  { connection: queueConnectionDetails },
+);
+
+// OpenAI Worker
+export const zOpenAIRequestSchema = z.object({
+  bookmarkId: z.string(),
+});
+export type ZOpenAIRequest = z.infer<typeof zOpenAIRequestSchema>;
+
+export const OpenAIQueue = new Queue<ZOpenAIRequest, void>("openai_queue", {
+  connection: queueConnectionDetails,
+});
+
+// Search Indexing Worker
+export const zSearchIndexingRequestSchema = z.object({
+  bookmarkId: z.string(),
+  type: z.enum(["index", "delete"]),
+});
+export type ZSearchIndexingRequest = z.infer<
+  typeof zSearchIndexingRequestSchema
+>;
+export const SearchIndexingQueue = new Queue<ZSearchIndexingRequest, void>(
+  "searching_indexing",
+  {
+    connection: queueConnectionDetails,
+    defaultJobOptions: {
+      attempts: 5,
+      backoff: {
+        type: "exponential",
+        delay: 1000,
+      },
+    },
+  },
+);
diff --git a/packages/shared/search.ts b/packages/shared/search.ts
new file mode 100644
index 00000000..3bdf1ad1
--- /dev/null
+++ b/packages/shared/search.ts
@@ -0,0 +1,50 @@
+import { MeiliSearch, Index } from "meilisearch";
+import serverConfig from "./config";
+import { z } from "zod";
+
+export const zBookmarkIdxSchema = z.object({
+  id: z.string(),
+  userId: z.string(),
+  url: z.string().nullish(),
+  title: z.string().nullish(),
+  description: z.string().nullish(),
+  content: z.string().nullish(),
+  tags: z.array(z.string()).default([]),
+});
+
+export type ZBookmarkIdx = z.infer<typeof zBookmarkIdxSchema>;
+
+let searchClient: MeiliSearch | undefined;
+
+if (serverConfig.meilisearch) {
+  searchClient = new MeiliSearch({
+    host: serverConfig.meilisearch.address,
+    apiKey: serverConfig.meilisearch.key,
+  });
+}
+
+const BOOKMARKS_IDX_NAME = "bookmarks";
+
+let idxClient: Index<ZBookmarkIdx> | undefined;
+
+export async function getSearchIdxClient(): Promise<Index<ZBookmarkIdx> | null> {
+  if (idxClient) {
+    return idxClient;
+  }
+  if (!searchClient) {
+    return null;
+  }
+
+  const indicies = await searchClient.getIndexes();
+  let idxFound = indicies.results.find((i) => i.uid == BOOKMARKS_IDX_NAME);
+  if (!idxFound) {
+    const idx = await searchClient.createIndex(BOOKMARKS_IDX_NAME, {
+      primaryKey: "id",
+    });
+    await searchClient.waitForTask(idx.taskUid);
+    idxFound = await searchClient.getIndex<ZBookmarkIdx>(BOOKMARKS_IDX_NAME);
+    const taskId = await idxFound.updateFilterableAttributes(["id", "userId"]);
+    await searchClient.waitForTask(taskId.taskUid);
+  }
+  return idxFound;
+}