fix: Prioritize crawling user added links over bulk imports. fixes #1717

3 files changed, 33 insertions, 37 deletions

diff --git a/packages/shared/package.json b/packages/shared/package.json
index 0210e24f..fbdd6651 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -9,7 +9,7 @@
     "glob": "^11.0.0",
     "html-to-text": "^9.0.5",
     "js-tiktoken": "^1.0.20",
-    "liteque": "^0.3.2",
+    "liteque": "^0.4.1",
     "meilisearch": "^0.37.0",
     "nodemailer": "^7.0.4",
     "ollama": "^0.5.14",
diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts
index a2fdc6b4..6cc5dd83 100644
--- a/packages/shared/queues.ts
+++ b/packages/shared/queues.ts
@@ -1,5 +1,5 @@
 import path from "node:path";
-import { buildDBClient, migrateDB, SqliteQueue } from "liteque";
+import { buildDBClient, EnqueueOptions, migrateDB, SqliteQueue } from "liteque";
 import { z } from "zod";
 
 import serverConfig from "./config";
@@ -86,25 +86,17 @@ export const TidyAssetsQueue = new SqliteQueue<ZTidyAssetsRequest>(
   },
 );
 
-export async function triggerSearchReindex(bookmarkId: string) {
-  await SearchIndexingQueue.enqueue({
-    bookmarkId,
-    type: "index",
-  });
-}
-
-export async function triggerSearchDeletion(bookmarkId: string) {
-  await SearchIndexingQueue.enqueue({
-    bookmarkId: bookmarkId,
-    type: "delete",
-  });
-}
-
-export async function triggerReprocessingFixMode(bookmarkId: string) {
-  await AssetPreprocessingQueue.enqueue({
-    bookmarkId,
-    fixMode: true,
-  });
+export async function triggerSearchReindex(
+  bookmarkId: string,
+  opts?: EnqueueOptions,
+) {
+  await SearchIndexingQueue.enqueue(
+    {
+      bookmarkId,
+      type: "index",
+    },
+    opts,
+  );
 }
 
 export const zvideoRequestSchema = z.object({
@@ -124,13 +116,6 @@ export const VideoWorkerQueue = new SqliteQueue<ZVideoRequest>(
   },
 );
 
-export async function triggerVideoWorker(bookmarkId: string, url: string) {
-  await VideoWorkerQueue.enqueue({
-    bookmarkId,
-    url,
-  });
-}
-
 // Feed Worker
 export const zFeedRequestSchema = z.object({
   feedId: z.string(),
@@ -191,12 +176,16 @@ export async function triggerWebhook(
   bookmarkId: string,
   operation: ZWebhookRequest["operation"],
   userId?: string,
+  opts?: EnqueueOptions,
 ) {
-  await WebhookQueue.enqueue({
-    bookmarkId,
-    userId,
-    operation,
-  });
+  await WebhookQueue.enqueue(
+    {
+      bookmarkId,
+      userId,
+      operation,
+    },
+    opts,
+  );
 }
 
 // RuleEngine worker
@@ -219,9 +208,13 @@ export const RuleEngineQueue = new SqliteQueue<ZRuleEngineRequest>(
 export async function triggerRuleEngineOnEvent(
   bookmarkId: string,
   events: z.infer<typeof zRuleEngineEventSchema>[],
+  opts?: EnqueueOptions,
 ) {
-  await RuleEngineQueue.enqueue({
-    events,
-    bookmarkId,
-  });
+  await RuleEngineQueue.enqueue(
+    {
+      events,
+      bookmarkId,
+    },
+    opts,
+  );
 }
diff --git a/packages/shared/types/bookmarks.ts b/packages/shared/types/bookmarks.ts
index f648bce5..f96cf0c5 100644
--- a/packages/shared/types/bookmarks.ts
+++ b/packages/shared/types/bookmarks.ts
@@ -139,6 +139,9 @@ export const zNewBookmarkRequestSchema = z
     note: z.string().optional(),
     summary: z.string().optional(),
     createdAt: z.coerce.date().optional(),
+    // A mechanism to prioritize crawling of bookmarks depending on whether
+    // they were created by a user interaction or by a bulk import.
+    crawlPriority: z.enum(["low", "normal"]).optional(),
   })
   .and(
     z.discriminatedUnion("type", [