diff options
| author | Mohamed Bassem <me@mbassem.com> | 2026-02-08 02:11:51 +0000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-02-08 02:11:51 +0000 |
| commit | bbd65fd6123f7d1a93d1f6a68f2b933d53ec3c23 (patch) | |
| tree | 1f23b04b4ca8fbccdea2d61e7c2c0a8b991d87d1 /packages/shared-server/src | |
| parent | 7d53e2e458cba7153dea27c625ca1bb534952ddf (diff) | |
| download | karakeep-bbd65fd6123f7d1a93d1f6a68f2b933d53ec3c23.tar.zst | |
feat: Add separate queue for import link crawling (#2452)
* feat: add separate queue for import link crawling
---------
Co-authored-by: Claude <noreply@anthropic.com>
Diffstat (limited to 'packages/shared-server/src')
| -rw-r--r-- | packages/shared-server/src/queues.ts | 12 |
1 files changed, 12 insertions, 0 deletions
diff --git a/packages/shared-server/src/queues.ts b/packages/shared-server/src/queues.ts index fd9dac83..4d4a61d6 100644 --- a/packages/shared-server/src/queues.ts +++ b/packages/shared-server/src/queues.ts @@ -96,6 +96,18 @@ export const LinkCrawlerQueue = createDeferredQueue<ZCrawlLinkRequest>( }, ); +// Separate queue for low priority link crawling (e.g. imports) +// This prevents low priority crawling from impacting the parallelism of the main queue +export const LowPriorityCrawlerQueue = createDeferredQueue<ZCrawlLinkRequest>( + "low_priority_crawler_queue", + { + defaultJobArgs: { + numRetries: 5, + }, + keepFailedJobs: false, + }, +); + // Inference Worker export const zOpenAIRequestSchema = z.object({ bookmarkId: z.string(), |