refactor: Replace the usage of bullMQ with the hoarder sqlite-based queue (#309)

13 files changed, 128 insertions, 344 deletions

diff --git a/README.md b/README.md
index ba0c5843..601120b9 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,6 @@ The demo is seeded with some content, but it's in read-only mode to prevent abus
 - [tRPC](https://trpc.io) for client->server communication.
 - [Puppeteer](https://pptr.dev/) for crawling the bookmarks.
 - [OpenAI](https://openai.com/) because AI is so hot right now.
-- [BullMQ](https://bullmq.io) for scheduling the background jobs.
 - [Meilisearch](https://meilisearch.com) for the full content search.
 
 ## Why did I build it?
diff --git a/apps/web/components/dashboard/admin/ServerStats.tsx b/apps/web/components/dashboard/admin/ServerStats.tsx
index 06e3421f..e95dc437 100644
--- a/apps/web/components/dashboard/admin/ServerStats.tsx
+++ b/apps/web/components/dashboard/admin/ServerStats.tsx
@@ -106,19 +106,19 @@ export default function ServerStats() {
           <TableBody>
             <TableRow>
               <TableCell className="lg:w-2/3">Crawling Jobs</TableCell>
-              <TableCell>{serverStats.crawlStats.queuedInRedis}</TableCell>
+              <TableCell>{serverStats.crawlStats.queued}</TableCell>
               <TableCell>{serverStats.crawlStats.pending}</TableCell>
               <TableCell>{serverStats.crawlStats.failed}</TableCell>
             </TableRow>
             <TableRow>
               <TableCell>Indexing Jobs</TableCell>
-              <TableCell>{serverStats.indexingStats.queuedInRedis}</TableCell>
+              <TableCell>{serverStats.indexingStats.queued}</TableCell>
               <TableCell>-</TableCell>
               <TableCell>-</TableCell>
             </TableRow>
             <TableRow>
               <TableCell>Inference Jobs</TableCell>
-              <TableCell>{serverStats.inferenceStats.queuedInRedis}</TableCell>
+              <TableCell>{serverStats.inferenceStats.queued}</TableCell>
               <TableCell>{serverStats.inferenceStats.pending}</TableCell>
               <TableCell>{serverStats.inferenceStats.failed}</TableCell>
             </TableRow>
diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts
index ddf61fc8..a1917523 100644
--- a/apps/workers/crawlerWorker.ts
+++ b/apps/workers/crawlerWorker.ts
@@ -1,11 +1,9 @@
 import assert from "assert";
 import * as dns from "dns";
 import * as path from "node:path";
-import type { Job } from "bullmq";
 import type { Browser } from "puppeteer";
 import { Readability } from "@mozilla/readability";
 import { Mutex } from "async-mutex";
-import { Worker } from "bullmq";
 import DOMPurify from "dompurify";
 import { eq } from "drizzle-orm";
 import { execa } from "execa";
@@ -34,6 +32,7 @@ import {
   bookmarkLinks,
   bookmarks,
 } from "@hoarder/db/schema";
+import { DequeuedJob, Runner } from "@hoarder/queue";
 import {
   ASSET_TYPES,
   deleteAsset,
@@ -48,7 +47,6 @@ import logger from "@hoarder/shared/logger";
 import {
   LinkCrawlerQueue,
   OpenAIQueue,
-  queueConnectionDetails,
   triggerSearchReindex,
   zCrawlLinkRequestSchema,
 } from "@hoarder/shared/queues";
@@ -153,37 +151,37 @@ export class CrawlerWorker {
     }
 
     logger.info("Starting crawler worker ...");
-    const worker = new Worker<ZCrawlLinkRequest, void>(
-      LinkCrawlerQueue.name,
-      withTimeout(
-        runCrawler,
-        /* timeoutSec */ serverConfig.crawler.jobTimeoutSec,
-      ),
+    const worker = new Runner<ZCrawlLinkRequest>(
+      LinkCrawlerQueue,
       {
+        run: withTimeout(
+          runCrawler,
+          /* timeoutSec */ serverConfig.crawler.jobTimeoutSec,
+        ),
+        onComplete: async (job) => {
+          const jobId = job?.id ?? "unknown";
+          logger.info(`[Crawler][${jobId}] Completed successfully`);
+          const bookmarkId = job?.data.bookmarkId;
+          if (bookmarkId) {
+            await changeBookmarkStatus(bookmarkId, "success");
+          }
+        },
+        onError: async (job) => {
+          const jobId = job?.id ?? "unknown";
+          logger.error(`[Crawler][${jobId}] Crawling job failed: ${job.error}`);
+          const bookmarkId = job.data?.bookmarkId;
+          if (bookmarkId) {
+            await changeBookmarkStatus(bookmarkId, "failure");
+          }
+        },
+      },
+      {
+        pollIntervalMs: 1000,
+        timeoutSecs: serverConfig.crawler.jobTimeoutSec,
         concurrency: serverConfig.crawler.numWorkers,
-        connection: queueConnectionDetails,
-        autorun: false,
       },
     );
 
-    worker.on("completed", (job) => {
-      const jobId = job?.id ?? "unknown";
-      logger.info(`[Crawler][${jobId}] Completed successfully`);
-      const bookmarkId = job?.data.bookmarkId;
-      if (bookmarkId) {
-        changeBookmarkStatus(bookmarkId, "success");
-      }
-    });
-
-    worker.on("failed", (job, error) => {
-      const jobId = job?.id ?? "unknown";
-      logger.error(`[Crawler][${jobId}] Crawling job failed: ${error}`);
-      const bookmarkId = job?.data.bookmarkId;
-      if (bookmarkId) {
-        changeBookmarkStatus(bookmarkId, "failure");
-      }
-    });
-
     return worker;
   }
 }
@@ -600,7 +598,7 @@ async function crawlAndParseUrl(
   };
 }
 
-async function runCrawler(job: Job<ZCrawlLinkRequest, void>) {
+async function runCrawler(job: DequeuedJob<ZCrawlLinkRequest>) {
   const jobId = job.id ?? "unknown";
 
   const request = zCrawlLinkRequestSchema.safeParse(job.data);
@@ -655,7 +653,7 @@ async function runCrawler(job: Job<ZCrawlLinkRequest, void>) {
 
   // Enqueue openai job (if not set, assume it's true for backward compatibility)
   if (job.data.runInference !== false) {
-    OpenAIQueue.add("openai", {
+    OpenAIQueue.enqueue({
       bookmarkId,
     });
   }
diff --git a/apps/workers/index.ts b/apps/workers/index.ts
index 687d9ced..39741aa8 100644
--- a/apps/workers/index.ts
+++ b/apps/workers/index.ts
@@ -2,6 +2,7 @@ import "dotenv/config";
 
 import serverConfig from "@hoarder/shared/config";
 import logger from "@hoarder/shared/logger";
+import { runQueueDBMigrations } from "@hoarder/shared/queues";
 
 import { CrawlerWorker } from "./crawlerWorker";
 import { shutdownPromise } from "./exit";
@@ -10,6 +11,8 @@ import { SearchIndexingWorker } from "./searchWorker";
 
 async function main() {
   logger.info(`Workers version: ${serverConfig.serverVersion ?? "not set"}`);
+  runQueueDBMigrations();
+
   const [crawler, openai, search] = [
     await CrawlerWorker.build(),
     OpenAiWorker.build(),
diff --git a/apps/workers/openaiWorker.ts b/apps/workers/openaiWorker.ts
index 776d6828..9e6e2f23 100644
--- a/apps/workers/openaiWorker.ts
+++ b/apps/workers/openaiWorker.ts
@@ -1,5 +1,3 @@
-import type { Job } from "bullmq";
-import { Worker } from "bullmq";
 import { and, Column, eq, inArray, sql } from "drizzle-orm";
 import { z } from "zod";
 
@@ -11,12 +9,12 @@ import {
   bookmarkTags,
   tagsOnBookmarks,
 } from "@hoarder/db/schema";
+import { DequeuedJob, Runner } from "@hoarder/queue";
 import { readAsset } from "@hoarder/shared/assetdb";
 import serverConfig from "@hoarder/shared/config";
 import logger from "@hoarder/shared/logger";
 import {
   OpenAIQueue,
-  queueConnectionDetails,
   triggerSearchReindex,
   zOpenAIRequestSchema,
 } from "@hoarder/shared/queues";
@@ -63,27 +61,30 @@ async function attemptMarkTaggingStatus(
 export class OpenAiWorker {
   static build() {
     logger.info("Starting inference worker ...");
-    const worker = new Worker<ZOpenAIRequest, void>(
-      OpenAIQueue.name,
-      runOpenAI,
+    const worker = new Runner<ZOpenAIRequest>(
+      OpenAIQueue,
       {
-        connection: queueConnectionDetails,
-        autorun: false,
+        run: runOpenAI,
+        onComplete: async (job) => {
+          const jobId = job?.id ?? "unknown";
+          logger.info(`[inference][${jobId}] Completed successfully`);
+          await attemptMarkTaggingStatus(job?.data, "success");
+        },
+        onError: async (job) => {
+          const jobId = job?.id ?? "unknown";
+          logger.error(
+            `[inference][${jobId}] inference job failed: ${job.error}`,
+          );
+          await attemptMarkTaggingStatus(job?.data, "failure");
+        },
+      },
+      {
+        concurrency: 1,
+        pollIntervalMs: 1000,
+        timeoutSecs: 30,
       },
     );
 
-    worker.on("completed", (job) => {
-      const jobId = job?.id ?? "unknown";
-      logger.info(`[inference][${jobId}] Completed successfully`);
-      attemptMarkTaggingStatus(job?.data, "success");
-    });
-
-    worker.on("failed", (job, error) => {
-      const jobId = job?.id ?? "unknown";
-      logger.error(`[inference][${jobId}] inference job failed: ${error}`);
-      attemptMarkTaggingStatus(job?.data, "failure");
-    });
-
     return worker;
   }
 }
@@ -361,7 +362,7 @@ async function connectTags(
   });
 }
 
-async function runOpenAI(job: Job<ZOpenAIRequest, void>) {
+async function runOpenAI(job: DequeuedJob<ZOpenAIRequest>) {
   const jobId = job.id ?? "unknown";
 
   const inferenceClient = InferenceClientFactory.build();
diff --git a/apps/workers/package.json b/apps/workers/package.json
index b74f9ec9..471606f2 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -7,10 +7,10 @@
     "@hoarder/db": "workspace:^0.1.0",
     "@hoarder/shared": "workspace:^0.1.0",
     "@hoarder/tsconfig": "workspace:^0.1.0",
+    "@hoarder/queue": "workspace:^0.1.0",
     "@mozilla/readability": "^0.5.0",
     "@tsconfig/node21": "^21.0.1",
     "async-mutex": "^0.4.1",
-    "bullmq": "^5.1.9",
     "dompurify": "^3.0.9",
     "dotenv": "^16.4.1",
     "drizzle-orm": "^0.29.4",
diff --git a/apps/workers/searchWorker.ts b/apps/workers/searchWorker.ts
index 75c057c6..56c2aac4 100644
--- a/apps/workers/searchWorker.ts
+++ b/apps/workers/searchWorker.ts
@@ -1,13 +1,11 @@
-import type { Job } from "bullmq";
-import { Worker } from "bullmq";
 import { eq } from "drizzle-orm";
 
 import type { ZSearchIndexingRequest } from "@hoarder/shared/queues";
 import { db } from "@hoarder/db";
 import { bookmarks } from "@hoarder/db/schema";
+import { DequeuedJob, Runner } from "@hoarder/queue";
 import logger from "@hoarder/shared/logger";
 import {
-  queueConnectionDetails,
   SearchIndexingQueue,
   zSearchIndexingRequestSchema,
 } from "@hoarder/shared/queues";
@@ -16,25 +14,28 @@ import { getSearchIdxClient } from "@hoarder/shared/search";
 export class SearchIndexingWorker {
   static build() {
     logger.info("Starting search indexing worker ...");
-    const worker = new Worker<ZSearchIndexingRequest, void>(
-      SearchIndexingQueue.name,
-      runSearchIndexing,
+    const worker = new Runner<ZSearchIndexingRequest>(
+      SearchIndexingQueue,
       {
-        connection: queueConnectionDetails,
-        autorun: false,
+        run: runSearchIndexing,
+        onComplete: (job) => {
+          const jobId = job?.id ?? "unknown";
+          logger.info(`[search][${jobId}] Completed successfully`);
+          return Promise.resolve();
+        },
+        onError: (job) => {
+          const jobId = job?.id ?? "unknown";
+          logger.error(`[search][${jobId}] search job failed: ${job.error}`);
+          return Promise.resolve();
+        },
+      },
+      {
+        concurrency: 1,
+        pollIntervalMs: 1000,
+        timeoutSecs: 30,
       },
     );
 
-    worker.on("completed", (job) => {
-      const jobId = job?.id ?? "unknown";
-      logger.info(`[search][${jobId}] Completed successfully`);
-    });
-
-    worker.on("failed", (job, error) => {
-      const jobId = job?.id ?? "unknown";
-      logger.error(`[search][${jobId}] search job failed: ${error}`);
-    });
-
     return worker;
   }
 }
@@ -112,7 +113,7 @@ async function runDelete(
   await ensureTaskSuccess(searchClient, task.taskUid);
 }
 
-async function runSearchIndexing(job: Job<ZSearchIndexingRequest, void>) {
+async function runSearchIndexing(job: DequeuedJob<ZSearchIndexingRequest>) {
   const jobId = job.id ?? "unknown";
 
   const request = zSearchIndexingRequestSchema.safeParse(job.data);
diff --git a/packages/shared/config.ts b/packages/shared/config.ts
index 2c739a0c..28bfdbdb 100644
--- a/packages/shared/config.ts
+++ b/packages/shared/config.ts
@@ -15,10 +15,6 @@ const allEnv = z.object({
   OLLAMA_BASE_URL: z.string().url().optional(),
   INFERENCE_TEXT_MODEL: z.string().default("gpt-3.5-turbo-0125"),
   INFERENCE_IMAGE_MODEL: z.string().default("gpt-4o-2024-05-13"),
-  REDIS_HOST: z.string().default("localhost"),
-  REDIS_PORT: z.coerce.number().default(6379),
-  REDIS_DB_IDX: z.coerce.number().optional(),
-  REDIS_PASSWORD: z.string().optional(),
   CRAWLER_HEADLESS_BROWSER: stringBool("true"),
   BROWSER_WEB_URL: z.string().url().optional(),
   BROWSER_WEBSOCKET_URL: z.string().url().optional(),
@@ -58,12 +54,6 @@ const serverConfigSchema = allEnv.transform((val) => {
       imageModel: val.INFERENCE_IMAGE_MODEL,
       inferredTagLang: val.INFERENCE_LANG,
     },
-    bullMQ: {
-      redisHost: val.REDIS_HOST,
-      redisPort: val.REDIS_PORT,
-      redisDBIdx: val.REDIS_DB_IDX,
-      redisPassword: val.REDIS_PASSWORD,
-    },
     crawler: {
       numWorkers: val.CRAWLER_NUM_WORKERS,
       headlessBrowser: val.CRAWLER_HEADLESS_BROWSER,
diff --git a/packages/shared/package.json b/packages/shared/package.json
index 032f3db5..2b1ae973 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -5,7 +5,7 @@
   "private": true,
   "type": "module",
   "dependencies": {
-    "bullmq": "^5.1.9",
+    "@hoarder/queue": "workspace:^0.1.0",
     "meilisearch": "^0.37.0",
     "winston": "^3.11.0",
     "zod": "^3.22.4"
diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts
index 2b890755..8747fb3f 100644
--- a/packages/shared/queues.ts
+++ b/packages/shared/queues.ts
@@ -1,14 +1,17 @@
-import { Queue } from "bullmq";
+import path from "node:path";
 import { z } from "zod";
 
+import { buildDBClient, migrateDB, SqliteQueue } from "@hoarder/queue";
+
 import serverConfig from "./config";
 
-export const queueConnectionDetails = {
-  host: serverConfig.bullMQ.redisHost,
-  port: serverConfig.bullMQ.redisPort,
-  db: serverConfig.bullMQ.redisDBIdx,
-  password: serverConfig.bullMQ.redisPassword,
-};
+const QUEUE_DB_PATH = path.join(serverConfig.dataDir, "queue.db");
+
+const queueDB = buildDBClient(QUEUE_DB_PATH);
+
+export function runQueueDBMigrations() {
+  migrateDB(queueDB);
+}
 
 // Link Crawler
 export const zCrawlLinkRequestSchema = z.object({
@@ -17,16 +20,12 @@ export const zCrawlLinkRequestSchema = z.object({
 });
 export type ZCrawlLinkRequest = z.infer<typeof zCrawlLinkRequestSchema>;
 
-export const LinkCrawlerQueue = new Queue<ZCrawlLinkRequest, void>(
+export const LinkCrawlerQueue = new SqliteQueue<ZCrawlLinkRequest>(
   "link_crawler_queue",
+  queueDB,
   {
-    connection: queueConnectionDetails,
-    defaultJobOptions: {
-      attempts: 5,
-      backoff: {
-        type: "exponential",
-        delay: 1000,
-      },
+    defaultJobArgs: {
+      numRetries: 5,
     },
   },
 );
@@ -37,16 +36,15 @@ export const zOpenAIRequestSchema = z.object({
 });
 export type ZOpenAIRequest = z.infer<typeof zOpenAIRequestSchema>;
 
-export const OpenAIQueue = new Queue<ZOpenAIRequest, void>("openai_queue", {
-  connection: queueConnectionDetails,
-  defaultJobOptions: {
-    attempts: 3,
-    backoff: {
-      type: "exponential",
-      delay: 500,
+export const OpenAIQueue = new SqliteQueue<ZOpenAIRequest>(
+  "openai_queue",
+  queueDB,
+  {
+    defaultJobArgs: {
+      numRetries: 3,
     },
   },
-});
+);
 
 // Search Indexing Worker
 export const zSearchIndexingRequestSchema = z.object({
@@ -56,29 +54,25 @@ export const zSearchIndexingRequestSchema = z.object({
 export type ZSearchIndexingRequest = z.infer<
   typeof zSearchIndexingRequestSchema
 >;
-export const SearchIndexingQueue = new Queue<ZSearchIndexingRequest, void>(
+export const SearchIndexingQueue = new SqliteQueue<ZSearchIndexingRequest>(
   "searching_indexing",
+  queueDB,
   {
-    connection: queueConnectionDetails,
-    defaultJobOptions: {
-      attempts: 5,
-      backoff: {
-        type: "exponential",
-        delay: 1000,
-      },
+    defaultJobArgs: {
+      numRetries: 5,
     },
   },
 );
 
 export function triggerSearchReindex(bookmarkId: string) {
-  SearchIndexingQueue.add("search_indexing", {
+  SearchIndexingQueue.enqueue({
     bookmarkId,
     type: "index",
   });
 }
 
 export function triggerSearchDeletion(bookmarkId: string) {
-  SearchIndexingQueue.add("search_indexing", {
+  SearchIndexingQueue.enqueue({
     bookmarkId: bookmarkId,
     type: "delete",
   });
diff --git a/packages/trpc/routers/admin.ts b/packages/trpc/routers/admin.ts
index 05831b92..14cb4ac9 100644
--- a/packages/trpc/routers/admin.ts
+++ b/packages/trpc/routers/admin.ts
@@ -18,17 +18,17 @@ export const adminAppRouter = router({
         numUsers: z.number(),
         numBookmarks: z.number(),
         crawlStats: z.object({
-          queuedInRedis: z.number(),
+          queued: z.number(),
           pending: z.number(),
           failed: z.number(),
         }),
         inferenceStats: z.object({
-          queuedInRedis: z.number(),
+          queued: z.number(),
           pending: z.number(),
           failed: z.number(),
         }),
         indexingStats: z.object({
-          queuedInRedis: z.number(),
+          queued: z.number(),
         }),
       }),
     )
@@ -38,15 +38,15 @@ export const adminAppRouter = router({
         [{ value: numBookmarks }],
 
         // Crawls
-        pendingCrawlsInRedis,
+        queuedCrawls,
         [{ value: pendingCrawls }],
         [{ value: failedCrawls }],
 
         // Indexing
-        pendingIndexingInRedis,
+        queuedIndexing,
 
         // Inference
-        pendingInferenceInRedis,
+        queuedInferences,
         [{ value: pendingInference }],
         [{ value: failedInference }],
       ] = await Promise.all([
@@ -54,7 +54,7 @@ export const adminAppRouter = router({
         ctx.db.select({ value: count() }).from(bookmarks),
 
         // Crawls
-        LinkCrawlerQueue.getWaitingCount(),
+        LinkCrawlerQueue.stats(),
         ctx.db
           .select({ value: count() })
           .from(bookmarkLinks)
@@ -65,10 +65,10 @@ export const adminAppRouter = router({
           .where(eq(bookmarkLinks.crawlStatus, "failure")),
 
         // Indexing
-        SearchIndexingQueue.getWaitingCount(),
+        SearchIndexingQueue.stats(),
 
         // Inference
-        OpenAIQueue.getWaitingCount(),
+        OpenAIQueue.stats(),
         ctx.db
           .select({ value: count() })
           .from(bookmarks)
@@ -83,17 +83,17 @@ export const adminAppRouter = router({
         numUsers,
         numBookmarks,
         crawlStats: {
-          queuedInRedis: pendingCrawlsInRedis,
+          queued: queuedCrawls.pending + queuedCrawls.pending_retry,
           pending: pendingCrawls,
           failed: failedCrawls,
         },
         inferenceStats: {
-          queuedInRedis: pendingInferenceInRedis,
+          queued: queuedInferences.pending + queuedInferences.pending_retry,
           pending: pendingInference,
           failed: failedInference,
         },
         indexingStats: {
-          queuedInRedis: pendingIndexingInRedis,
+          queued: queuedIndexing.pending + queuedIndexing.pending_retry,
         },
       };
     }),
@@ -116,7 +116,7 @@ export const adminAppRouter = router({
 
       await Promise.all(
         bookmarkIds.map((b) =>
-          LinkCrawlerQueue.add("crawl", {
+          LinkCrawlerQueue.enqueue({
             bookmarkId: b.id,
             runInference: input.runInference,
           }),
diff --git a/packages/trpc/routers/bookmarks.ts b/packages/trpc/routers/bookmarks.ts
index 1e5e7dfc..43bb4db7 100644
--- a/packages/trpc/routers/bookmarks.ts
+++ b/packages/trpc/routers/bookmarks.ts
@@ -310,14 +310,14 @@ export const bookmarksAppRouter = router({
       switch (bookmark.content.type) {
         case BookmarkTypes.LINK: {
           // The crawling job triggers openai when it's done
-          await LinkCrawlerQueue.add("crawl", {
+          await LinkCrawlerQueue.enqueue({
             bookmarkId: bookmark.id,
           });
           break;
         }
         case BookmarkTypes.TEXT:
         case BookmarkTypes.ASSET: {
-          await OpenAIQueue.add("openai", {
+          await OpenAIQueue.enqueue({
             bookmarkId: bookmark.id,
           });
           break;
@@ -418,7 +418,7 @@ export const bookmarksAppRouter = router({
     .input(z.object({ bookmarkId: z.string() }))
     .use(ensureBookmarkOwnership)
     .mutation(async ({ input }) => {
-      await LinkCrawlerQueue.add("crawl", {
+      await LinkCrawlerQueue.enqueue({
         bookmarkId: input.bookmarkId,
       });
     }),
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 1ddf872a..33c8bf06 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -644,6 +644,9 @@ importers:
       '@hoarder/db':
         specifier: workspace:^0.1.0
         version: link:../../packages/db
+      '@hoarder/queue':
+        specifier: workspace:^0.1.0
+        version: link:../../packages/queue
       '@hoarder/shared':
         specifier: workspace:^0.1.0
         version: link:../../packages/shared
@@ -659,9 +662,6 @@ importers:
       async-mutex:
         specifier: ^0.4.1
         version: 0.4.1
-      bullmq:
-        specifier: ^5.1.9
-        version: 5.3.3
       dompurify:
         specifier: ^3.0.9
         version: 3.0.9
@@ -873,9 +873,9 @@ importers:
 
   packages/shared:
     dependencies:
-      bullmq:
-        specifier: ^5.1.9
-        version: 5.3.3
+      '@hoarder/queue':
+        specifier: workspace:^0.1.0
+        version: link:../queue
       meilisearch:
         specifier: ^0.37.0
         version: 0.37.0
@@ -2810,9 +2810,6 @@ packages:
     cpu: [x64]
     os: [win32]
 
-  '@ioredis/commands@1.2.0':
-    resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}
-
   '@isaacs/cliui@8.0.2':
     resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
     engines: {node: '>=12'}
@@ -2898,36 +2895,6 @@ packages:
     resolution: {integrity: sha512-Z+CZ3QaosfFaTqvhQsIktyGrjFjSC0Fa4EMph4mqKnWhmyoGICsV/8QK+8HpXut6zV7zwfWwqDmEjtk1Qf6EgQ==}
     engines: {node: '>=14.0.0'}
 
-  '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2':
-    resolution: {integrity: sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==}
-    cpu: [arm64]
-    os: [darwin]
-
-  '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.2':
-    resolution: {integrity: sha512-lwriRAHm1Yg4iDf23Oxm9n/t5Zpw1lVnxYU3HnJPTi2lJRkKTrps1KVgvL6m7WvmhYVt/FIsssWay+k45QHeuw==}
-    cpu: [x64]
-    os: [darwin]
-
-  '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.2':
-    resolution: {integrity: sha512-FU20Bo66/f7He9Fp9sP2zaJ1Q8L9uLPZQDub/WlUip78JlPeMbVL8546HbZfcW9LNciEXc8d+tThSJjSC+tmsg==}
-    cpu: [arm64]
-    os: [linux]
-
-  '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.2':
-    resolution: {integrity: sha512-MOI9Dlfrpi2Cuc7i5dXdxPbFIgbDBGgKR5F2yWEa6FVEtSWncfVNKW5AKjImAQ6CZlBK9tympdsZJ2xThBiWWA==}
-    cpu: [arm]
-    os: [linux]
-
-  '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.2':
-    resolution: {integrity: sha512-gsWNDCklNy7Ajk0vBBf9jEx04RUxuDQfBse918Ww+Qb9HCPoGzS+XJTLe96iN3BVK7grnLiYghP/M4L8VsaHeA==}
-    cpu: [x64]
-    os: [linux]
-
-  '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.2':
-    resolution: {integrity: sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==}
-    cpu: [x64]
-    os: [win32]
-
   '@next/env@14.1.4':
     resolution: {integrity: sha512-e7X7bbn3Z6DWnDi75UWn+REgAbLEqxI8Tq2pkFOFAMpWAWApz/YCUhtWMWn410h8Q2fYiYL7Yg5OlxMOCfFjJQ==}
 
@@ -5123,9 +5090,6 @@ packages:
   builtins@1.0.3:
     resolution: {integrity: sha512-uYBjakWipfaO/bXI7E8rq6kpwHRZK5cNYrUv2OzZSI/FvmdMyXJ2tG9dKcjEC5YHmHpUAwsargWIZNWdxb/bnQ==}
 
-  bullmq@5.3.3:
-    resolution: {integrity: sha512-Gc/68HxiCHLMPBiGIqtINxcf8HER/5wvBYMY/6x3tFejlvldUBFaAErMTLDv4TnPsTyzNPrfBKmFCEM58uVnJg==}
-
   busboy@1.6.0:
     resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==}
     engines: {node: '>=10.16.0'}
@@ -5400,10 +5364,6 @@ packages:
     resolution: {integrity: sha512-m3iNNWpd9rl3jvvcBnu70ylMdrXt8Vlq4HYadnU5fwcOtvkSQWPmj7amUcDT2qYI7risszBjI5AUIUox9D16pg==}
     engines: {node: '>=6'}
 
-  cluster-key-slot@1.1.2:
-    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
-    engines: {node: '>=0.10.0'}
-
   cmdk@1.0.0:
     resolution: {integrity: sha512-gDzVf0a09TvoJ5jnuPvygTB77+XdOSwEmJ88L6XPFPlv7T3RxbP9jgenfylrAMD0+Le1aO0nVjQUzl2g+vjz5Q==}
     peerDependencies:
@@ -5650,10 +5610,6 @@ packages:
       typescript:
         optional: true
 
-  cron-parser@4.9.0:
-    resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==}
-    engines: {node: '>=12.0.0'}
-
   cross-fetch@3.1.8:
     resolution: {integrity: sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==}
 
@@ -5948,10 +5904,6 @@ packages:
   denodeify@1.2.1:
     resolution: {integrity: sha512-KNTihKNmQENUZeKu5fzfpzRqR5S2VMp4gl9RFHiWzj9DfvYQPMJ6XHKNaQxaGCXwPk6y9yme3aUoaiAe+KX+vg==}
 
-  denque@2.1.0:
-    resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==}
-    engines: {node: '>=0.10'}
-
   depd@1.1.2:
     resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==}
     engines: {node: '>= 0.6'}
@@ -7614,10 +7566,6 @@ packages:
   invariant@2.2.4:
     resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==}
 
-  ioredis@5.3.2:
-    resolution: {integrity: sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==}
-    engines: {node: '>=12.22.0'}
-
   ip-address@9.0.5:
     resolution: {integrity: sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==}
     engines: {node: '>= 12'}
@@ -8362,12 +8310,6 @@ packages:
   lodash.debounce@4.0.8:
     resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==}
 
-  lodash.defaults@4.2.0:
-    resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==}
-
-  lodash.isarguments@3.1.0:
-    resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==}
-
   lodash.isplainobject@4.0.6:
     resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==}
 
@@ -8465,10 +8407,6 @@ packages:
     peerDependencies:
       react: ^16.5.1 || ^17.0.0 || ^18.0.0
 
-  luxon@3.4.4:
-    resolution: {integrity: sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==}
-    engines: {node: '>=12'}
-
   magic-string@0.25.9:
     resolution: {integrity: sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==}
 
@@ -9040,13 +8978,6 @@ packages:
   ms@2.1.3:
     resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
 
-  msgpackr-extract@3.0.2:
-    resolution: {integrity: sha512-SdzXp4kD/Qf8agZ9+iTu6eql0m3kWm1A2y1hkpTeVNENutaB0BwHlSvAIaMxwntmRUAUjon2V4L8Z/njd0Ct8A==}
-    hasBin: true
-
-  msgpackr@1.10.1:
-    resolution: {integrity: sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==}
-
   multicast-dns@7.2.5:
     resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==}
     hasBin: true
@@ -9178,10 +9109,6 @@ packages:
     resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==}
     engines: {node: '>= 6.13.0'}
 
-  node-gyp-build-optional-packages@5.0.7:
-    resolution: {integrity: sha512-YlCCc6Wffkx0kHkmam79GKvDQ6x+QZkMjFGrIMxgFNILFvGSbCp2fCBC55pGTT9gVaz8Na5CLmxt/urtzRv36w==}
-    hasBin: true
-
   node-gyp@10.0.1:
     resolution: {integrity: sha512-gg3/bHehQfZivQVfqIyy8wTdSymF9yTyP4CJifK73imyNMU8AIGQE2pUa7dNWfmMeG9cDVF2eehiRMv0LC1iAg==}
     engines: {node: ^16.14.0 || >=18.0.0}
@@ -10571,14 +10498,6 @@ packages:
     resolution: {integrity: sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==}
     engines: {node: '>=6.0.0'}
 
-  redis-errors@1.2.0:
-    resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==}
-    engines: {node: '>=4'}
-
-  redis-parser@3.0.0:
-    resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==}
-    engines: {node: '>=4'}
-
   reflect.getprototypeof@1.0.5:
     resolution: {integrity: sha512-62wgfC8dJWrmxv44CA36pLDnP6KKl3Vhxb7PL+8+qrrFMMoJij4vgiMP8zV4O8+CBMXY1mHxI5fITGHXFHVmQQ==}
     engines: {node: '>= 0.4'}
@@ -11159,9 +11078,6 @@ packages:
     resolution: {integrity: sha512-KJP1OCML99+8fhOHxwwzyWrlUuVX5GQ0ZpJTd1DFXhdkrvg1szxfHhawXUZ3g9TkXORQd4/WG68jMlQZ2p8wlg==}
     engines: {node: '>=6'}
 
-  standard-as-callback@2.1.0:
-    resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==}
-
   statuses@1.5.0:
     resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==}
     engines: {node: '>= 0.6'}
@@ -11972,10 +11888,6 @@ packages:
     resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==}
     hasBin: true
 
-  uuid@9.0.1:
-    resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==}
-    hasBin: true
-
   valid-url@1.0.9:
     resolution: {integrity: sha512-QQDsV8OnSf5Uc30CKSwG9lnhMPe6exHtTXLRYX8uMwKENy640pU+2BgBL0LRbDh/eYRahNCS7aewCx0wf3NYVA==}
 
@@ -16111,9 +16023,6 @@ snapshots:
     dev: false
     optional: true
 
-  '@ioredis/commands@1.2.0':
-    dev: false
-
   '@isaacs/cliui@8.0.2':
     dependencies:
       string-width: 5.1.2
@@ -16295,30 +16204,6 @@ snapshots:
   '@mozilla/readability@0.5.0':
     dev: false
 
-  '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2':
-    dev: false
-    optional: true
-
-  '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.2':
-    dev: false
-    optional: true
-
-  '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.2':
-    dev: false
-    optional: true
-
-  '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.2':
-    dev: false
-    optional: true
-
-  '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.2':
-    dev: false
-    optional: true
-
-  '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.2':
-    dev: false
-    optional: true
-
   '@next/env@14.1.4':
     dev: false
 
@@ -19502,22 +19387,6 @@ snapshots:
   builtins@1.0.3:
     dev: false
 
-  bullmq@5.3.3:
-    dependencies:
-      cron-parser: 4.9.0
-      fast-glob: 3.3.2
-      ioredis: 5.3.2
-      lodash: 4.17.21
-      minimatch: 9.0.3
-      msgpackr: 1.10.1
-      node-abort-controller: 3.1.1
-      semver: 7.6.0
-      tslib: 2.6.2
-      uuid: 9.0.1
-    transitivePeerDependencies:
-      - supports-color
-    dev: false
-
   busboy@1.6.0:
     dependencies:
       streamsearch: 1.1.0
@@ -19904,9 +19773,6 @@ snapshots:
   clsx@2.1.0:
     dev: false
 
-  cluster-key-slot@1.1.2:
-    dev: false
-
   cmdk@1.0.0(@types/react-dom@18.2.19)(@types/react@18.2.58)(react-dom@18.2.0(react@18.2.0))(react@18.2.0):
     dependencies:
       '@radix-ui/react-dialog': 1.0.5(@types/react-dom@18.2.19)(@types/react@18.2.58)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)
@@ -20197,11 +20063,6 @@ snapshots:
       typescript: 5.3.3
     dev: false
 
-  cron-parser@4.9.0:
-    dependencies:
-      luxon: 3.4.4
-    dev: false
-
   cross-fetch@3.1.8:
     dependencies:
       node-fetch: 2.7.0
@@ -20565,9 +20426,6 @@ snapshots:
   denodeify@1.2.1:
     dev: false
 
-  denque@2.1.0:
-    dev: false
-
   depd@1.1.2:
     dev: false
 
@@ -22953,21 +22811,6 @@ snapshots:
     dependencies:
       loose-envify: 1.4.0
 
-  ioredis@5.3.2:
-    dependencies:
-      '@ioredis/commands': 1.2.0
-      cluster-key-slot: 1.1.2
-      debug: 4.3.4
-      denque: 2.1.0
-      lodash.defaults: 4.2.0
-      lodash.isarguments: 3.1.0
-      redis-errors: 1.2.0
-      redis-parser: 3.0.0
-      standard-as-callback: 2.1.0
-    transitivePeerDependencies:
-      - supports-color
-    dev: false
-
   ip-address@9.0.5:
     dependencies:
       jsbn: 1.1.0
@@ -23806,12 +23649,6 @@ snapshots:
   lodash.debounce@4.0.8:
     dev: false
 
-  lodash.defaults@4.2.0:
-    dev: false
-
-  lodash.isarguments@3.1.0:
-    dev: false
-
   lodash.isplainobject@4.0.6: {}
 
   lodash.memoize@4.1.2:
@@ -23922,9 +23759,6 @@ snapshots:
       react: 18.2.0
     dev: false
 
-  luxon@3.4.4:
-    dev: false
-
   magic-string@0.25.9:
     dependencies:
       sourcemap-codec: 1.4.8
@@ -25057,24 +24891,6 @@ snapshots:
 
   ms@2.1.3: {}
 
-  msgpackr-extract@3.0.2:
-    dependencies:
-      node-gyp-build-optional-packages: 5.0.7
-    optionalDependencies:
-      '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.2
-      '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.2
-      '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.2
-      '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.2
-      '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.2
-      '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.2
-    dev: false
-    optional: true
-
-  msgpackr@1.10.1:
-    optionalDependencies:
-      msgpackr-extract: 3.0.2
-    dev: false
-
   multicast-dns@7.2.5:
     dependencies:
       dns-packet: 5.6.1
@@ -25262,10 +25078,6 @@ snapshots:
   node-forge@1.3.1:
     dev: false
 
-  node-gyp-build-optional-packages@5.0.7:
-    dev: false
-    optional: true
-
   node-gyp@10.0.1:
     dependencies:
       env-paths: 2.2.1
@@ -27061,14 +26873,6 @@ snapshots:
       minimatch: 3.1.2
     dev: false
 
-  redis-errors@1.2.0:
-    dev: false
-
-  redis-parser@3.0.0:
-    dependencies:
-      redis-errors: 1.2.0
-    dev: false
-
   reflect.getprototypeof@1.0.5:
     dependencies:
       call-bind: 1.0.7
@@ -27898,9 +27702,6 @@ snapshots:
       type-fest: 0.7.1
     dev: false
 
-  standard-as-callback@2.1.0:
-    dev: false
-
   statuses@1.5.0:
     dev: false
 
@@ -28860,9 +28661,6 @@ snapshots:
   uuid@8.3.2:
     dev: false
 
-  uuid@9.0.1:
-    dev: false
-
   valid-url@1.0.9:
     dev: false