feat: Restate-based queue plugin (#2011)

* WIP: Initial restate integration

* add retry

* add delay + idempotency

* implement concurrency limits

* add admin stats

* add todos

* add id provider

* handle onComplete failures

* add tests

* add pub key and fix logging

* add priorities

* fail call after retries

* more fixes

* fix retries left

* some refactoring

* fix package.json

* upgrade sdk

* some test cleanups

4 files changed, 11 insertions, 4 deletions

diff --git a/packages/shared-server/package.json b/packages/shared-server/package.json
index 9c1b52a8..d18eb4a0 100644
--- a/packages/shared-server/package.json
+++ b/packages/shared-server/package.json
@@ -7,6 +7,7 @@
   "dependencies": {
     "@karakeep/db": "workspace:^0.1.0",
     "@karakeep/plugins-queue-liteque": "workspace:^0.1.0",
+    "@karakeep/plugins-queue-restate": "workspace:^0.1.0",
     "@karakeep/plugins-search-meilisearch": "workspace:^0.1.0",
     "@karakeep/shared": "workspace:^0.1.0"
   },
diff --git a/packages/shared-server/src/plugins.ts b/packages/shared-server/src/plugins.ts
index b6a88462..de08b9b0 100644
--- a/packages/shared-server/src/plugins.ts
+++ b/packages/shared-server/src/plugins.ts
@@ -8,6 +8,7 @@ export async function loadAllPlugins() {
   // Load plugins here. Order of plugin loading matter.
   // Queue provider(s)
   await import("@karakeep/plugins-queue-liteque");
+  await import("@karakeep/plugins-queue-restate");
   await import("@karakeep/plugins-search-meilisearch");
   PluginManager.logAllPlugins();
   pluginsLoaded = true;
diff --git a/packages/shared-server/src/queues.ts b/packages/shared-server/src/queues.ts
index 1c4e0452..c9f8276d 100644
--- a/packages/shared-server/src/queues.ts
+++ b/packages/shared-server/src/queues.ts
@@ -8,8 +8,12 @@ import { loadAllPlugins } from ".";
 await loadAllPlugins();
 const QUEUE_CLIENT = await getQueueClient();
 
-export function runQueueDBMigrations() {
-  QUEUE_CLIENT.init();
+export async function prepareQueue() {
+  await QUEUE_CLIENT.prepare();
+}
+
+export async function startQueue() {
+  await QUEUE_CLIENT.start();
 }
 
 // Link Crawler
diff --git a/packages/shared/queueing.ts b/packages/shared/queueing.ts
index dfe3b31a..e401972b 100644
--- a/packages/shared/queueing.ts
+++ b/packages/shared/queueing.ts
@@ -3,7 +3,6 @@ import { ZodType } from "zod";
 import { PluginManager, PluginType } from "./plugins";
 
 export interface EnqueueOptions {
-  numRetries?: number;
   idempotencyKey?: string;
   priority?: number;
   delayMs?: number;
@@ -47,6 +46,7 @@ export interface RunnerOptions<T> {
 }
 
 export interface Queue<T> {
+  opts: QueueOptions;
   name(): string;
   enqueue(payload: T, options?: EnqueueOptions): Promise<string | undefined>;
   stats(): Promise<{
@@ -65,7 +65,8 @@ export interface Runner<_T> {
 }
 
 export interface QueueClient {
-  init(): Promise<void>;
+  prepare(): Promise<void>;
+  start(): Promise<void>;
   createQueue<T>(name: string, options: QueueOptions): Queue<T>;
   createRunner<T>(
     queue: Queue<T>,