aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMohamed Bassem <me@mbassem.com>2024-10-27 23:37:54 +0000
committerMohamed Bassem <me@mbassem.com>2024-10-27 23:40:10 +0000
commita746e9a38e53e4a9114d786c7fb1b3ef4ca67870 (patch)
treebf1d819f6faca01bc7c3db1efe91c0879caaf4a8
parenteb7da996a7c2d617d276f296cac07a6fd5648664 (diff)
downloadkarakeep-a746e9a38e53e4a9114d786c7fb1b3ef4ca67870.tar.zst
deps: Extract the queue implementation into its own repos
-rw-r--r--apps/workers/crawlerWorker.ts2
-rw-r--r--apps/workers/openaiWorker.ts2
-rw-r--r--apps/workers/package.json2
-rw-r--r--apps/workers/searchWorker.ts2
-rw-r--r--apps/workers/tidyAssetsWorker.ts2
-rw-r--r--packages/queue/db.ts19
-rw-r--r--packages/queue/drizzle.config.ts10
-rw-r--r--packages/queue/drizzle/0000_wonderful_talisman.sql18
-rw-r--r--packages/queue/drizzle/meta/0000_snapshot.json130
-rw-r--r--packages/queue/drizzle/meta/_journal.json13
-rw-r--r--packages/queue/index.ts6
-rw-r--r--packages/queue/options.ts22
-rw-r--r--packages/queue/package.json35
-rw-r--r--packages/queue/queue.ts146
-rw-r--r--packages/queue/runner.test.ts440
-rw-r--r--packages/queue/runner.ts115
-rw-r--r--packages/queue/schema.ts36
-rw-r--r--packages/queue/tsconfig.json9
-rw-r--r--packages/queue/types.ts11
-rw-r--r--packages/queue/vitest.config.ts13
-rw-r--r--packages/shared/package.json2
-rw-r--r--packages/shared/queues.ts3
-rw-r--r--pnpm-lock.yaml363
23 files changed, 65 insertions, 1336 deletions
diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts
index 74413c63..ca0f6608 100644
--- a/apps/workers/crawlerWorker.ts
+++ b/apps/workers/crawlerWorker.ts
@@ -9,6 +9,7 @@ import { eq } from "drizzle-orm";
import { execa } from "execa";
import { isShuttingDown } from "exit";
import { JSDOM } from "jsdom";
+import { DequeuedJob, Runner } from "liteque";
import metascraper from "metascraper";
import metascraperAmazon from "metascraper-amazon";
import metascraperDescription from "metascraper-description";
@@ -32,7 +33,6 @@ import {
bookmarkLinks,
bookmarks,
} from "@hoarder/db/schema";
-import { DequeuedJob, Runner } from "@hoarder/queue";
import {
ASSET_TYPES,
deleteAsset,
diff --git a/apps/workers/openaiWorker.ts b/apps/workers/openaiWorker.ts
index 4fe74f44..5a4fd69d 100644
--- a/apps/workers/openaiWorker.ts
+++ b/apps/workers/openaiWorker.ts
@@ -1,4 +1,5 @@
import { and, Column, eq, inArray, sql } from "drizzle-orm";
+import { DequeuedJob, Runner } from "liteque";
import { z } from "zod";
import type { InferenceClient } from "@hoarder/shared/inference";
@@ -11,7 +12,6 @@ import {
customPrompts,
tagsOnBookmarks,
} from "@hoarder/db/schema";
-import { DequeuedJob, Runner } from "@hoarder/queue";
import { readAsset } from "@hoarder/shared/assetdb";
import serverConfig from "@hoarder/shared/config";
import { InferenceClientFactory } from "@hoarder/shared/inference";
diff --git a/apps/workers/package.json b/apps/workers/package.json
index 289f7315..7f64e715 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -5,7 +5,6 @@
"private": true,
"dependencies": {
"@hoarder/db": "workspace:^0.1.0",
- "@hoarder/queue": "workspace:^0.1.0",
"@hoarder/shared": "workspace:^0.1.0",
"@hoarder/tsconfig": "workspace:^0.1.0",
"@mozilla/readability": "^0.5.0",
@@ -16,6 +15,7 @@
"drizzle-orm": "^0.33.0",
"execa": "9.3.1",
"jsdom": "^24.0.0",
+ "liteque": "^0.1.3",
"metascraper": "^5.45.24",
"metascraper-amazon": "^5.45.22",
"metascraper-description": "^5.45.22",
diff --git a/apps/workers/searchWorker.ts b/apps/workers/searchWorker.ts
index f793f3f6..d2f1dffc 100644
--- a/apps/workers/searchWorker.ts
+++ b/apps/workers/searchWorker.ts
@@ -1,9 +1,9 @@
import { eq } from "drizzle-orm";
+import { DequeuedJob, Runner } from "liteque";
import type { ZSearchIndexingRequest } from "@hoarder/shared/queues";
import { db } from "@hoarder/db";
import { bookmarks } from "@hoarder/db/schema";
-import { DequeuedJob, Runner } from "@hoarder/queue";
import logger from "@hoarder/shared/logger";
import {
SearchIndexingQueue,
diff --git a/apps/workers/tidyAssetsWorker.ts b/apps/workers/tidyAssetsWorker.ts
index bc14aab9..c70736f2 100644
--- a/apps/workers/tidyAssetsWorker.ts
+++ b/apps/workers/tidyAssetsWorker.ts
@@ -1,8 +1,8 @@
import { eq } from "drizzle-orm";
+import { DequeuedJob, Runner } from "liteque";
import { db } from "@hoarder/db";
import { assets } from "@hoarder/db/schema";
-import { DequeuedJob, Runner } from "@hoarder/queue";
import { deleteAsset, getAllAssets } from "@hoarder/shared/assetdb";
import logger from "@hoarder/shared/logger";
import {
diff --git a/packages/queue/db.ts b/packages/queue/db.ts
deleted file mode 100644
index f1412fef..00000000
--- a/packages/queue/db.ts
+++ /dev/null
@@ -1,19 +0,0 @@
-import path from "node:path";
-import Database from "better-sqlite3";
-import { BetterSQLite3Database, drizzle } from "drizzle-orm/better-sqlite3";
-import { migrate } from "drizzle-orm/better-sqlite3/migrator";
-
-import * as schema from "./schema";
-
-export function buildDBClient(dbPath: string, runMigrations = false) {
- const sqlite = new Database(dbPath);
- const db = drizzle(sqlite, { schema });
- if (runMigrations) {
- migrateDB(db);
- }
- return db;
-}
-
-export function migrateDB(db: BetterSQLite3Database<typeof schema>) {
- migrate(db, { migrationsFolder: path.join(__dirname, "drizzle") });
-}
diff --git a/packages/queue/drizzle.config.ts b/packages/queue/drizzle.config.ts
deleted file mode 100644
index 6ef01d1b..00000000
--- a/packages/queue/drizzle.config.ts
+++ /dev/null
@@ -1,10 +0,0 @@
-import type { Config } from "drizzle-kit";
-
-export default {
- schema: "./schema.ts",
- out: "./drizzle",
- driver: "better-sqlite",
- dbCredentials: {
- url: "data.db",
- },
-} satisfies Config;
diff --git a/packages/queue/drizzle/0000_wonderful_talisman.sql b/packages/queue/drizzle/0000_wonderful_talisman.sql
deleted file mode 100644
index e042ab92..00000000
--- a/packages/queue/drizzle/0000_wonderful_talisman.sql
+++ /dev/null
@@ -1,18 +0,0 @@
-CREATE TABLE `tasks` (
- `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
- `queue` text NOT NULL,
- `payload` text NOT NULL,
- `createdAt` integer NOT NULL,
- `status` text DEFAULT 'pending' NOT NULL,
- `expireAt` integer,
- `allocationId` text NOT NULL,
- `numRunsLeft` integer NOT NULL,
- `maxNumRuns` integer NOT NULL
-);
---> statement-breakpoint
-CREATE INDEX `tasks_queue_idx` ON `tasks` (`queue`);--> statement-breakpoint
-CREATE INDEX `tasks_status_idx` ON `tasks` (`status`);--> statement-breakpoint
-CREATE INDEX `tasks_expire_at_idx` ON `tasks` (`expireAt`);--> statement-breakpoint
-CREATE INDEX `tasks_num_runs_left_idx` ON `tasks` (`numRunsLeft`);--> statement-breakpoint
-CREATE INDEX `tasks_max_num_runs_idx` ON `tasks` (`maxNumRuns`);--> statement-breakpoint
-CREATE INDEX `tasks_allocation_id_idx` ON `tasks` (`allocationId`); \ No newline at end of file
diff --git a/packages/queue/drizzle/meta/0000_snapshot.json b/packages/queue/drizzle/meta/0000_snapshot.json
deleted file mode 100644
index 57c7c2f4..00000000
--- a/packages/queue/drizzle/meta/0000_snapshot.json
+++ /dev/null
@@ -1,130 +0,0 @@
-{
- "version": "5",
- "dialect": "sqlite",
- "id": "3094773c-0138-46b2-b617-4b10093b0f53",
- "prevId": "00000000-0000-0000-0000-000000000000",
- "tables": {
- "tasks": {
- "name": "tasks",
- "columns": {
- "id": {
- "name": "id",
- "type": "integer",
- "primaryKey": true,
- "notNull": true,
- "autoincrement": true
- },
- "queue": {
- "name": "queue",
- "type": "text",
- "primaryKey": false,
- "notNull": true,
- "autoincrement": false
- },
- "payload": {
- "name": "payload",
- "type": "text",
- "primaryKey": false,
- "notNull": true,
- "autoincrement": false
- },
- "createdAt": {
- "name": "createdAt",
- "type": "integer",
- "primaryKey": false,
- "notNull": true,
- "autoincrement": false
- },
- "status": {
- "name": "status",
- "type": "text",
- "primaryKey": false,
- "notNull": true,
- "autoincrement": false,
- "default": "'pending'"
- },
- "expireAt": {
- "name": "expireAt",
- "type": "integer",
- "primaryKey": false,
- "notNull": false,
- "autoincrement": false
- },
- "allocationId": {
- "name": "allocationId",
- "type": "text",
- "primaryKey": false,
- "notNull": true,
- "autoincrement": false
- },
- "numRunsLeft": {
- "name": "numRunsLeft",
- "type": "integer",
- "primaryKey": false,
- "notNull": true,
- "autoincrement": false
- },
- "maxNumRuns": {
- "name": "maxNumRuns",
- "type": "integer",
- "primaryKey": false,
- "notNull": true,
- "autoincrement": false
- }
- },
- "indexes": {
- "tasks_queue_idx": {
- "name": "tasks_queue_idx",
- "columns": [
- "queue"
- ],
- "isUnique": false
- },
- "tasks_status_idx": {
- "name": "tasks_status_idx",
- "columns": [
- "status"
- ],
- "isUnique": false
- },
- "tasks_expire_at_idx": {
- "name": "tasks_expire_at_idx",
- "columns": [
- "expireAt"
- ],
- "isUnique": false
- },
- "tasks_num_runs_left_idx": {
- "name": "tasks_num_runs_left_idx",
- "columns": [
- "numRunsLeft"
- ],
- "isUnique": false
- },
- "tasks_max_num_runs_idx": {
- "name": "tasks_max_num_runs_idx",
- "columns": [
- "maxNumRuns"
- ],
- "isUnique": false
- },
- "tasks_allocation_id_idx": {
- "name": "tasks_allocation_id_idx",
- "columns": [
- "allocationId"
- ],
- "isUnique": false
- }
- },
- "foreignKeys": {},
- "compositePrimaryKeys": {},
- "uniqueConstraints": {}
- }
- },
- "enums": {},
- "_meta": {
- "schemas": {},
- "tables": {},
- "columns": {}
- }
-} \ No newline at end of file
diff --git a/packages/queue/drizzle/meta/_journal.json b/packages/queue/drizzle/meta/_journal.json
deleted file mode 100644
index 2b14f895..00000000
--- a/packages/queue/drizzle/meta/_journal.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "version": "5",
- "dialect": "sqlite",
- "entries": [
- {
- "idx": 0,
- "version": "5",
- "when": 1720992922192,
- "tag": "0000_wonderful_talisman",
- "breakpoints": true
- }
- ]
-} \ No newline at end of file
diff --git a/packages/queue/index.ts b/packages/queue/index.ts
deleted file mode 100644
index c9144f29..00000000
--- a/packages/queue/index.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-export { SqliteQueue } from "./queue";
-export { buildDBClient, migrateDB } from "./db";
-export type { SqliteQueueOptions, RunnerOptions, RunnerFuncs } from "./options";
-export { Runner } from "./runner";
-
-export type { DequeuedJob, DequeuedJobError } from "./types";
diff --git a/packages/queue/options.ts b/packages/queue/options.ts
deleted file mode 100644
index 18f8e52d..00000000
--- a/packages/queue/options.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import { ZodType } from "zod";
-
-import { DequeuedJob, DequeuedJobError } from "./types";
-
-export interface SqliteQueueOptions {
- defaultJobArgs: {
- numRetries: number;
- };
-}
-
-export interface RunnerFuncs<T> {
- run: (job: DequeuedJob<T>) => Promise<void>;
- onComplete?: (job: DequeuedJob<T>) => Promise<void>;
- onError?: (job: DequeuedJobError<T>) => Promise<void>;
-}
-
-export interface RunnerOptions<T> {
- pollIntervalMs: number;
- timeoutSecs: number;
- concurrency: number;
- validator?: ZodType<T>;
-}
diff --git a/packages/queue/package.json b/packages/queue/package.json
deleted file mode 100644
index 146a88b7..00000000
--- a/packages/queue/package.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
- "$schema": "https://json.schemastore.org/package.json",
- "name": "@hoarder/queue",
- "version": "0.1.0",
- "private": true,
- "type": "module",
- "dependencies": {
- "async-mutex": "^0.4.1",
- "better-sqlite3": "^11.3.0",
- "drizzle-orm": "^0.33.0",
- "zod": "^3.22.4"
- },
- "devDependencies": {
- "@hoarder/eslint-config": "workspace:^0.2.0",
- "@hoarder/prettier-config": "workspace:^0.1.0",
- "@hoarder/tsconfig": "workspace:^0.1.0",
- "@types/better-sqlite3": "^7.6.11",
- "drizzle-kit": "^0.20.14",
- "vitest": "^1.3.1"
- },
- "scripts": {
- "typecheck": "tsc --noEmit",
- "test": "vitest",
- "format": "prettier . --ignore-path ../../.prettierignore",
- "lint": "eslint ."
- },
- "main": "index.ts",
- "eslintConfig": {
- "root": true,
- "extends": [
- "@hoarder/eslint-config/base"
- ]
- },
- "prettier": "@hoarder/prettier-config"
-}
diff --git a/packages/queue/queue.ts b/packages/queue/queue.ts
deleted file mode 100644
index ad486468..00000000
--- a/packages/queue/queue.ts
+++ /dev/null
@@ -1,146 +0,0 @@
-import assert from "node:assert";
-import { and, asc, count, eq, gt, lt, or } from "drizzle-orm";
-
-import { buildDBClient } from "./db";
-import { SqliteQueueOptions } from "./options";
-import { Job, tasksTable } from "./schema";
-
-// generate random id
-function generateAllocationId() {
- return Math.random().toString(36).substring(2, 15);
-}
-
-export class SqliteQueue<T> {
- queueName: string;
- db: ReturnType<typeof buildDBClient>;
- options: SqliteQueueOptions;
-
- constructor(
- name: string,
- db: ReturnType<typeof buildDBClient>,
- options: SqliteQueueOptions,
- ) {
- this.queueName = name;
- this.options = options;
- this.db = db;
- }
-
- name() {
- return this.queueName;
- }
-
- async enqueue(payload: T): Promise<Job> {
- const job = await this.db
- .insert(tasksTable)
- .values({
- queue: this.queueName,
- payload: JSON.stringify(payload),
- numRunsLeft: this.options.defaultJobArgs.numRetries + 1,
- maxNumRuns: this.options.defaultJobArgs.numRetries + 1,
- allocationId: generateAllocationId(),
- })
- .returning();
-
- return job[0];
- }
-
- async stats() {
- const res = await this.db
- .select({ status: tasksTable.status, count: count() })
- .from(tasksTable)
- .where(eq(tasksTable.queue, this.queueName))
- .groupBy(tasksTable.status);
-
- return res.reduce(
- (acc, r) => {
- acc[r.status] += r.count;
- return acc;
- },
- {
- pending: 0,
- pending_retry: 0,
- running: 0,
- failed: 0,
- },
- );
- }
-
- async attemptDequeue(options: { timeoutSecs: number }): Promise<Job | null> {
- return await this.db.transaction(async (txn) => {
- const jobs = await txn
- .select()
- .from(tasksTable)
- .where(
- and(
- eq(tasksTable.queue, this.queueName),
- gt(tasksTable.numRunsLeft, 0),
- or(
- // Not picked by a worker yet
- eq(tasksTable.status, "pending"),
-
- // Failed but still has attempts left
- eq(tasksTable.status, "pending_retry"),
-
- // Expired and still has attempts left
- and(
- eq(tasksTable.status, "running"),
- lt(tasksTable.expireAt, new Date()),
- ),
- ),
- ),
- )
- .orderBy(asc(tasksTable.createdAt))
- .limit(1);
-
- if (jobs.length == 0) {
- return null;
- }
- assert(jobs.length == 1);
- const job = jobs[0];
-
- const result = await txn
- .update(tasksTable)
- .set({
- status: "running",
- numRunsLeft: job.numRunsLeft - 1,
- allocationId: generateAllocationId(),
- expireAt: new Date(new Date().getTime() + options.timeoutSecs * 1000),
- })
- .where(
- and(
- eq(tasksTable.id, job.id),
-
- // The compare and swap is necessary to avoid race conditions
- eq(tasksTable.allocationId, job.allocationId),
- ),
- )
- .returning();
- if (result.length == 0) {
- return null;
- }
- assert(result.length == 1);
- return result[0];
- });
- }
-
- async finalize(
- id: number,
- alloctionId: string,
- status: "completed" | "pending_retry" | "failed",
- ) {
- if (status == "completed") {
- await this.db
- .delete(tasksTable)
- .where(
- and(eq(tasksTable.id, id), eq(tasksTable.allocationId, alloctionId)),
- );
- } else {
- await this.db
- .update(tasksTable)
- .set({ status: status, expireAt: null })
- .where(
- and(eq(tasksTable.id, id), eq(tasksTable.allocationId, alloctionId)),
- );
- }
- }
-}
diff --git a/packages/queue/runner.test.ts b/packages/queue/runner.test.ts
deleted file mode 100644
index 7777b422..00000000
--- a/packages/queue/runner.test.ts
+++ /dev/null
@@ -1,440 +0,0 @@
-/* eslint-disable @typescript-eslint/require-await */
-import { Semaphore } from "async-mutex";
-import { eq } from "drizzle-orm";
-import { describe, expect, test } from "vitest";
-import { z } from "zod";
-
-import {
- buildDBClient,
- DequeuedJob,
- DequeuedJobError,
- Runner,
- RunnerOptions,
- SqliteQueue,
-} from "./";
-import { tasksTable } from "./schema";
-
-class Baton {
- semaphore: Semaphore;
- constructor() {
- this.semaphore = new Semaphore(0);
- this.reset();
- }
- post() {
- this.semaphore.setValue(100000);
- }
-
- async wait() {
- await this.semaphore.acquire();
- }
-
- reset() {
- this.semaphore.setValue(-Infinity);
- }
-}
-
-class Barrier {
- semaphore: Semaphore;
- baton: Baton;
- constructor(numParticipants: number) {
- this.semaphore = new Semaphore(numParticipants * -1 + 1);
- this.baton = new Baton();
- this.reset(numParticipants * -1 + 1);
- }
-
- async notifyReachedAndWait() {
- this.semaphore.release();
- await this.baton.wait();
- }
-
- async waitUntilAllReached() {
- await this.semaphore.waitForUnlock();
- }
-
- allowParticipantsToProceed() {
- this.baton.post();
- }
-
- reset(numParticipants: number) {
- this.semaphore.setValue(numParticipants);
- this.baton.reset();
- }
-}
-
-const defaultRunnerOpts = {
- pollIntervalMs: 100,
- timeoutSecs: 100,
- concurrency: 2,
- validator: z.object({
- increment: z.number(),
- succeedAfter: z.number().optional().default(0),
- blockForSec: z.number().optional().default(0),
- }),
-};
-
-interface Work {
- increment: number;
- succeedAfter?: number;
- blockForSec?: number;
-}
-
-interface Results {
- result: number;
- numCalled: number;
- numCompleted: number;
- numFailed: number;
-}
-
-async function waitUntilAllSettled(queue: SqliteQueue<Work>) {
- let stats = await queue.stats();
- while (stats.running > 0 || stats.pending > 0 || stats.pending_retry > 0) {
- await new Promise((resolve) => setTimeout(resolve, 100));
- stats = await queue.stats();
- console.log(stats);
- }
-}
-
-function buildRunner(
- queue: SqliteQueue<Work>,
- opts: RunnerOptions<Work>,
- barrier: Barrier,
- inputResults?: Results,
-) {
- const results = inputResults ?? {
- result: 0,
- numCalled: 0,
- numCompleted: 0,
- numFailed: 0,
- };
- const runner = new Runner<Work>(
- queue,
- {
- run: async (job: DequeuedJob<Work>) => {
- console.log("STARTED:", job);
- results.numCalled++;
- await barrier.notifyReachedAndWait();
- if (job.runNumber < (job.data.succeedAfter ?? 0)) {
- throw new Error("Failed");
- }
- if (job.data.blockForSec !== undefined) {
- await new Promise((resolve) =>
- setTimeout(resolve, job.data.blockForSec! * 1000),
- );
- }
- results.result += job.data.increment;
- },
- onComplete: async (job: DequeuedJob<Work>) => {
- console.log("COMPLETED:", job);
- results.numCompleted++;
- },
- onError: async (job: DequeuedJobError<Work>) => {
- console.log("FAILED:", job);
- results.numFailed++;
- },
- },
- opts,
- );
-
- return { runner, results };
-}
-
-describe("SqiteQueueRunner", () => {
- test("should run jobs with correct concurrency", async () => {
- const queue = new SqliteQueue<Work>(
- "queue1",
- buildDBClient(":memory:", true),
- {
- defaultJobArgs: {
- numRetries: 0,
- },
- },
- );
-
- const barrier = new Barrier(2);
- const { runner, results } = buildRunner(
- queue,
- { ...defaultRunnerOpts, concurrency: 2 },
- barrier,
- );
-
- await queue.enqueue({ increment: 1 });
- await queue.enqueue({ increment: 2 });
- await queue.enqueue({ increment: 3 });
-
- expect(await queue.stats()).toEqual({
- pending: 3,
- running: 0,
- pending_retry: 0,
- failed: 0,
- });
-
- const runnerPromise = runner.runUntilEmpty();
-
- // Wait until all runners reach the synchronization point
- await barrier.waitUntilAllReached();
-
- // Ensure that we have two "running" jobs given the concurrency of 2
- expect(await queue.stats()).toEqual({
- pending: 1,
- running: 2,
- pending_retry: 0,
- failed: 0,
- });
-
- // Allow jobs to proceed
- barrier.allowParticipantsToProceed();
-
- // Wait until all jobs are consumed
- await runnerPromise;
-
- expect(await queue.stats()).toEqual({
- pending: 0,
- running: 0,
- pending_retry: 0,
- failed: 0,
- });
-
- expect(results.result).toEqual(6);
- expect(results.numCalled).toEqual(3);
- expect(results.numCompleted).toEqual(3);
- expect(results.numFailed).toEqual(0);
- });
-
- test("should retry errors", async () => {
- const queue = new SqliteQueue<Work>(
- "queue1",
- buildDBClient(":memory:", true),
- {
- defaultJobArgs: {
- numRetries: 2,
- },
- },
- );
-
- const barrier = new Barrier(0);
- barrier.allowParticipantsToProceed();
- const { runner, results } = buildRunner(queue, defaultRunnerOpts, barrier);
-
- await queue.enqueue({ increment: 1, succeedAfter: 2 });
- await queue.enqueue({ increment: 1, succeedAfter: 10 });
- await queue.enqueue({ increment: 3, succeedAfter: 0 });
-
- const runnerPromise = runner.runUntilEmpty();
-
- // Wait until all jobs are consumed
- await runnerPromise;
-
- expect(await queue.stats()).toEqual({
- pending: 0,
- pending_retry: 0,
- running: 0,
- failed: 1,
- });
-
- expect(results.result).toEqual(4);
- expect(results.numCalled).toEqual(7);
- expect(results.numCompleted).toEqual(2);
- expect(results.numFailed).toEqual(1);
- });
-
- test("timeouts are respected", async () => {
- const queue = new SqliteQueue<Work>(
- "queue1",
- buildDBClient(":memory:", true),
- {
- defaultJobArgs: {
- numRetries: 1,
- },
- },
- );
-
- const barrier = new Barrier(1);
- barrier.allowParticipantsToProceed();
- const { runner: runner, results } = buildRunner(
- queue,
- { ...defaultRunnerOpts, concurrency: 1, timeoutSecs: 1 },
- barrier,
- );
-
- await queue.enqueue({ increment: 1, blockForSec: 10 });
- await runner.runUntilEmpty();
-
- expect(await queue.stats()).toEqual({
- pending: 0,
- pending_retry: 0,
- running: 0,
- failed: 1,
- });
-
- expect(results.result).toEqual(0);
- expect(results.numCalled).toEqual(2);
- expect(results.numCompleted).toEqual(0);
- expect(results.numFailed).toEqual(1);
- });
-
- test("serialization errors", async () => {
- const queue = new SqliteQueue<Work>(
- "queue1",
- buildDBClient(":memory:", true),
- {
- defaultJobArgs: {
- numRetries: 1,
- },
- },
- );
-
- const job = await queue.enqueue({ increment: 1 });
- // Corrupt the payload
- await queue.db
- .update(tasksTable)
- .set({ payload: "{}" })
- .where(eq(tasksTable.id, job.id));
-
- const barrier = new Barrier(1);
- barrier.allowParticipantsToProceed();
- const { runner, results } = buildRunner(
- queue,
- { ...defaultRunnerOpts, concurrency: 1 },
- barrier,
- );
-
- const p = runner.run();
- await waitUntilAllSettled(queue);
- runner.stop();
- await p;
-
- expect(await queue.stats()).toEqual({
- pending: 0,
- pending_retry: 0,
- running: 0,
- failed: 1,
- });
-
- expect(results.result).toEqual(0);
- expect(results.numCalled).toEqual(0);
- expect(results.numCompleted).toEqual(0);
- expect(results.numFailed).toEqual(1);
- });
-
- test("concurrent runners", async () => {
- const queue = new SqliteQueue<Work>(
- "queue1",
- buildDBClient(":memory:", true),
- {
- defaultJobArgs: {
- numRetries: 0,
- },
- },
- );
-
- await queue.enqueue({ increment: 1 });
- await queue.enqueue({ increment: 2 });
- await queue.enqueue({ increment: 3 });
-
- const barrier = new Barrier(3);
- const { runner: runner1, results } = buildRunner(
- queue,
- { ...defaultRunnerOpts, concurrency: 1 },
- barrier,
- );
- const { runner: runner2 } = buildRunner(
- queue,
- { ...defaultRunnerOpts, concurrency: 1 },
- barrier,
- results,
- );
- const { runner: runner3 } = buildRunner(
- queue,
- { ...defaultRunnerOpts, concurrency: 1 },
- barrier,
- results,
- );
-
- const runPromises = Promise.all([
- runner1.run(),
- runner2.run(),
- runner3.run(),
- ]);
-
- await barrier.waitUntilAllReached();
-
- expect(await queue.stats()).toEqual({
- pending: 0,
- pending_retry: 0,
- running: 3,
- failed: 0,
- });
-
- barrier.allowParticipantsToProceed();
-
- runner1.stop();
- runner2.stop();
- runner3.stop();
-
- await runPromises;
-
- expect(results.result).toEqual(6);
- expect(results.numCalled).toEqual(3);
- expect(results.numCompleted).toEqual(3);
- expect(results.numFailed).toEqual(0);
- });
-
- test("large test", async () => {
- const db = buildDBClient(":memory:", true);
- const queue1 = new SqliteQueue<Work>("queue1", db, {
- defaultJobArgs: {
- numRetries: 0,
- },
- });
- const queue2 = new SqliteQueue<Work>("queue2", db, {
- defaultJobArgs: {
- numRetries: 0,
- },
- });
-
- const barrier = new Barrier(0);
- barrier.allowParticipantsToProceed();
- const results = {
- result: 0,
- numCalled: 0,
- numCompleted: 0,
- numFailed: 0,
- };
- const runners = [];
- const runnerPromises = [];
-
- for (let i = 0; i < 10; i++) {
- const { runner } = buildRunner(
- i % 2 == 0 ? queue1 : queue2,
- { ...defaultRunnerOpts, concurrency: 2 },
- barrier,
- results,
- );
- runners.push(runner);
- runnerPromises.push(runner.run());
- }
-
- {
- const enqueuePromises = [];
- for (let i = 0; i < 1000; i++) {
- enqueuePromises.push(
- (i % 2 == 0 ? queue1 : queue2).enqueue({ increment: i }),
- );
- }
- await Promise.all(enqueuePromises);
- }
-
- await Promise.all([
- waitUntilAllSettled(queue1),
- waitUntilAllSettled(queue2),
- ]);
-
- runners.forEach((runner) => runner.stop());
- await Promise.all(runnerPromises);
-
- expect(results.result).toEqual(499500);
- expect(results.numCalled).toEqual(1000);
- expect(results.numCompleted).toEqual(1000);
- expect(results.numFailed).toEqual(0);
- });
-});
diff --git a/packages/queue/runner.ts b/packages/queue/runner.ts
deleted file mode 100644
index 1a90f969..00000000
--- a/packages/queue/runner.ts
+++ /dev/null
@@ -1,115 +0,0 @@
-import assert from "node:assert";
-import { Semaphore } from "async-mutex";
-
-import { RunnerFuncs, RunnerOptions } from "./options";
-import { SqliteQueue } from "./queue";
-import { Job } from "./schema";
-import { DequeuedJob } from "./types";
-
-export class Runner<T> {
- queue: SqliteQueue<T>;
- funcs: RunnerFuncs<T>;
- opts: RunnerOptions<T>;
- stopping = false;
-
- constructor(
- queue: SqliteQueue<T>,
- funcs: RunnerFuncs<T>,
- opts: RunnerOptions<T>,
- ) {
- this.queue = queue;
- this.funcs = funcs;
- this.opts = opts;
- }
-
- async run() {
- return this.runImpl(false);
- }
-
- stop() {
- this.stopping = true;
- }
-
- async runUntilEmpty() {
- return this.runImpl(true);
- }
-
- async runImpl(breakOnEmpty: boolean) {
- const semaphore = new Semaphore(this.opts.concurrency);
- const inFlight = new Map<number, Promise<void>>();
- while (!this.stopping) {
- await semaphore.waitForUnlock();
- const job = await this.queue.attemptDequeue({
- timeoutSecs: this.opts.timeoutSecs,
- });
- if (!job && breakOnEmpty && inFlight.size == 0) {
- // No more jobs to process, and no ongoing jobs.
- break;
- }
- if (!job) {
- await new Promise((resolve) =>
- setTimeout(resolve, this.opts.pollIntervalMs),
- );
- continue;
- }
- const [_, release] = await semaphore.acquire();
- inFlight.set(
- job.id,
- this.runOnce(job).finally(() => {
- inFlight.delete(job.id);
- release();
- }),
- );
- }
- await Promise.allSettled(inFlight.values());
- }
-
- async runOnce(job: Job) {
- assert(job.allocationId);
-
- let parsed: T;
- try {
- parsed = JSON.parse(job.payload) as T;
- if (this.opts.validator) {
- parsed = this.opts.validator.parse(parsed);
- }
- } catch (e) {
- if (job.numRunsLeft <= 0) {
- await this.funcs.onError?.({
- id: job.id.toString(),
- error: e as Error,
- });
- await this.queue.finalize(job.id, job.allocationId, "failed");
- } else {
- await this.queue.finalize(job.id, job.allocationId, "pending_retry");
- }
- return;
- }
-
- const dequeuedJob: DequeuedJob<T> = {
- id: job.id.toString(),
- data: parsed,
- runNumber: job.maxNumRuns - job.numRunsLeft - 1,
- };
- try {
- await Promise.race([
- this.funcs.run(dequeuedJob),
- new Promise((_, reject) =>
- setTimeout(
- () => reject(new Error("Timeout")),
- this.opts.timeoutSecs * 1000,
- ),
- ),
- ]);
- await this.funcs.onComplete?.(dequeuedJob);
- await this.queue.finalize(job.id, job.allocationId, "completed");
- } catch (e) {
- if (job.numRunsLeft <= 0) {
- await this.funcs.onError?.({ ...dequeuedJob, error: e as Error });
- await this.queue.finalize(job.id, job.allocationId, "failed");
- } else {
- await this.queue.finalize(job.id, job.allocationId, "pending_retry");
- }
- }
- }
-}
diff --git a/packages/queue/schema.ts b/packages/queue/schema.ts
deleted file mode 100644
index 377c6b1c..00000000
--- a/packages/queue/schema.ts
+++ /dev/null
@@ -1,36 +0,0 @@
-import { index, integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
-
-function createdAtField() {
- return integer("createdAt", { mode: "timestamp" })
- .notNull()
- .$defaultFn(() => new Date());
-}
-
-export const tasksTable = sqliteTable(
- "tasks",
- {
- id: integer("id").notNull().primaryKey({ autoIncrement: true }),
- queue: text("queue").notNull(),
- payload: text("payload").notNull(),
- createdAt: createdAtField(),
- status: text("status", {
- enum: ["pending", "running", "pending_retry", "failed"],
- })
- .notNull()
- .default("pending"),
- expireAt: integer("expireAt", { mode: "timestamp" }),
- allocationId: text("allocationId").notNull(),
- numRunsLeft: integer("numRunsLeft").notNull(),
- maxNumRuns: integer("maxNumRuns").notNull(),
- },
- (tasks) => ({
- queueIdx: index("tasks_queue_idx").on(tasks.queue),
- statusIdx: index("tasks_status_idx").on(tasks.status),
- expireAtIdx: index("tasks_expire_at_idx").on(tasks.expireAt),
- numRunsLeftIdx: index("tasks_num_runs_left_idx").on(tasks.numRunsLeft),
- maxNumRunsIdx: index("tasks_max_num_runs_idx").on(tasks.maxNumRuns),
- allocationIdIdx: index("tasks_allocation_id_idx").on(tasks.allocationId),
- }),
-);
-
-export type Job = typeof tasksTable.$inferSelect;
diff --git a/packages/queue/tsconfig.json b/packages/queue/tsconfig.json
deleted file mode 100644
index 71bf61e7..00000000
--- a/packages/queue/tsconfig.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "$schema": "https://json.schemastore.org/tsconfig",
- "extends": "@hoarder/tsconfig/node.json",
- "include": ["**/*.ts"],
- "exclude": ["node_modules"],
- "compilerOptions": {
- "tsBuildInfoFile": "node_modules/.cache/tsbuildinfo.json"
- },
-}
diff --git a/packages/queue/types.ts b/packages/queue/types.ts
deleted file mode 100644
index 01975cc7..00000000
--- a/packages/queue/types.ts
+++ /dev/null
@@ -1,11 +0,0 @@
-export interface DequeuedJob<T> {
- id: string;
- data: T;
- runNumber: number;
-}
-
-export interface DequeuedJobError<T> {
- id: string;
- data?: T;
- error: Error;
-}
diff --git a/packages/queue/vitest.config.ts b/packages/queue/vitest.config.ts
deleted file mode 100644
index a206cfc4..00000000
--- a/packages/queue/vitest.config.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-/// <reference types="vitest" />
-
-import { defineConfig } from "vitest/config";
-
-// https://vitejs.dev/config/
-export default defineConfig({
- plugins: [],
- test: {
- alias: {
- "@/*": "./*",
- },
- },
-});
diff --git a/packages/shared/package.json b/packages/shared/package.json
index f6774263..a8d636f7 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -5,8 +5,8 @@
"private": true,
"type": "module",
"dependencies": {
- "@hoarder/queue": "workspace:^0.1.0",
"glob": "^11.0.0",
+ "liteque": "^0.1.3",
"meilisearch": "^0.37.0",
"ollama": "^0.5.9",
"openai": "^4.67.1",
diff --git a/packages/shared/queues.ts b/packages/shared/queues.ts
index 6b04b988..0cb30aae 100644
--- a/packages/shared/queues.ts
+++ b/packages/shared/queues.ts
@@ -1,8 +1,7 @@
import path from "node:path";
+import { buildDBClient, migrateDB, SqliteQueue } from "liteque";
import { z } from "zod";
-import { buildDBClient, migrateDB, SqliteQueue } from "@hoarder/queue";
-
import serverConfig from "./config";
const QUEUE_DB_PATH = path.join(serverConfig.dataDir, "queue.db");
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 04cb0c62..1ea23f8e 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -686,9 +686,6 @@ importers:
'@hoarder/db':
specifier: workspace:^0.1.0
version: link:../../packages/db
- '@hoarder/queue':
- specifier: workspace:^0.1.0
- version: link:../../packages/queue
'@hoarder/shared':
specifier: workspace:^0.1.0
version: link:../../packages/shared
@@ -719,6 +716,9 @@ importers:
jsdom:
specifier: ^24.0.0
version: 24.0.0
+ liteque:
+ specifier: ^0.1.3
+ version: 0.1.3(better-sqlite3@11.3.0)
metascraper:
specifier: ^5.45.24
version: 5.45.24
@@ -904,48 +904,14 @@ importers:
specifier: ^4.7.1
version: 4.7.1
- packages/queue:
- dependencies:
- async-mutex:
- specifier: ^0.4.1
- version: 0.4.1
- better-sqlite3:
- specifier: ^11.3.0
- version: 11.3.0
- drizzle-orm:
- specifier: ^0.33.0
- version: 0.33.0(@types/better-sqlite3@7.6.11)(better-sqlite3@11.3.0)
- zod:
- specifier: ^3.22.4
- version: 3.22.4
- devDependencies:
- '@hoarder/eslint-config':
- specifier: workspace:^0.2.0
- version: link:../../tooling/eslint
- '@hoarder/prettier-config':
- specifier: workspace:^0.1.0
- version: link:../../tooling/prettier
- '@hoarder/tsconfig':
- specifier: workspace:^0.1.0
- version: link:../../tooling/typescript
- '@types/better-sqlite3':
- specifier: ^7.6.11
- version: 7.6.11
- drizzle-kit:
- specifier: ^0.20.14
- version: 0.20.14
- vitest:
- specifier: ^1.3.1
- version: 1.3.1(@types/node@20.11.20)
-
packages/shared:
dependencies:
- '@hoarder/queue':
- specifier: workspace:^0.1.0
- version: link:../queue
glob:
specifier: ^11.0.0
version: 11.0.0
+ liteque:
+ specifier: ^0.1.3
+ version: 0.1.3(better-sqlite3@11.3.0)
meilisearch:
specifier: ^0.37.0
version: 0.37.0
@@ -2287,9 +2253,6 @@ packages:
'@drizzle-team/brocli@0.10.1':
resolution: {integrity: sha512-AHy0vjc+n/4w/8Mif+w86qpppHuF3AyXbcWW+R/W7GNA3F5/p2nuhlkCJaTXSLZheB4l1rtHzOfr9A7NwoR/Zg==}
- '@drizzle-team/studio@0.0.39':
- resolution: {integrity: sha512-c5Hkm7MmQC2n5qAsKShjQrHoqlfGslB8+qWzsGGZ+2dHMRTNG60UuzalF0h0rvBax5uzPXuGkYLGaQ+TUX3yMw==}
-
'@egjs/hammerjs@2.0.17':
resolution: {integrity: sha512-XQsZgjm2EcVUiZQf11UBJQfmZeEmOW8DpI1gsFeln6w0ae0ii4dMQEQ0kjl6DspdWX1aGY1/loyXnP0JS06e/A==}
engines: {node: '>=0.8.0'}
@@ -5471,10 +5434,6 @@ packages:
resolution: {integrity: sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==}
engines: {node: '>=10'}
- cli-color@2.0.3:
- resolution: {integrity: sha512-OkoZnxyC4ERN3zLzZaY9Emb7f/MhBOIpePv0Ycok0fJYT+Ouo00UBEIwsVsr0yoow++n5YWlSUgST9GKhNHiRQ==}
- engines: {node: '>=0.10'}
-
cli-cursor@2.1.0:
resolution: {integrity: sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw==}
engines: {node: '>=4'}
@@ -5931,9 +5890,6 @@ packages:
csstype@3.1.3:
resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==}
- d@1.0.1:
- resolution: {integrity: sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==}
-
dag-map@1.0.2:
resolution: {integrity: sha512-+LSAiGFwQ9dRnRdOeaj7g47ZFJcOUPukAP8J3A3fuZ1g9Y44BG+P1sgApjLXTQPOzC4+7S9Wr8kXsfpINM4jpw==}
@@ -6141,9 +6097,6 @@ packages:
resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==}
engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
- difflib@0.2.4:
- resolution: {integrity: sha512-9YVwmMb0wQHQNr5J9m6BSj6fk4pfGITGQOOs+D9Fl+INODWFOfvhIU1hNv6GgR1RBoC/9NJcwu77zShxV0kT7w==}
-
dir-glob@3.0.1:
resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==}
engines: {node: '>=8'}
@@ -6251,14 +6204,6 @@ packages:
resolution: {integrity: sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==}
engines: {node: '>=12'}
- dreamopt@0.8.0:
- resolution: {integrity: sha512-vyJTp8+mC+G+5dfgsY+r3ckxlz+QMX40VjPQsZc5gxVAxLmi64TBoVkP54A/pRAXMXsbu2GMMBrZPxNv23waMg==}
- engines: {node: '>=0.4.0'}
-
- drizzle-kit@0.20.14:
- resolution: {integrity: sha512-0fHv3YIEaUcSVPSGyaaBfOi9bmpajjhbJNdPsRMIUvYdLVxBu9eGjH8mRc3Qk7HVmEidFc/lhG1YyJhoXrn5yA==}
- hasBin: true
-
drizzle-kit@0.24.2:
resolution: {integrity: sha512-nXOaTSFiuIaTMhS8WJC2d4EBeIcN9OSt2A2cyFbQYBAZbi7lRsVGJNqDpEwPqYfJz38yxbY/UtbvBBahBfnExQ==}
hasBin: true
@@ -6429,10 +6374,6 @@ packages:
resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==}
engines: {node: '>=6'}
- env-paths@3.0.0:
- resolution: {integrity: sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==}
- engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
-
envinfo@7.11.1:
resolution: {integrity: sha512-8PiZgZNIB4q/Lw4AhOvAfB/ityHAd2bli3lESSWmWSzSsl5dKpy5N1d1Rfkd2teq/g9xN90lc6o98DOjMeYHpg==}
engines: {node: '>=4'}
@@ -6490,19 +6431,6 @@ packages:
resolution: {integrity: sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==}
engines: {node: '>= 0.4'}
- es5-ext@0.10.63:
- resolution: {integrity: sha512-hUCZd2Byj/mNKjfP9jXrdVZ62B8KuA/VoK7X8nUh5qT+AxDmcbvZz041oDVZdbIN1qW6XY9VDNwzkvKnZvK2TQ==}
- engines: {node: '>=0.10'}
-
- es6-iterator@2.0.3:
- resolution: {integrity: sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==}
-
- es6-symbol@3.1.3:
- resolution: {integrity: sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==}
-
- es6-weak-map@2.0.3:
- resolution: {integrity: sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==}
-
esbuild-register@3.5.0:
resolution: {integrity: sha512-+4G/XmakeBAsvJuDugJvtyF1x+XJT4FMocynNpxrvEBViirpfUn2PgNpCHedfWhF4WokNsO/OvMKrmJOIJsI5A==}
peerDependencies:
@@ -6688,10 +6616,6 @@ packages:
deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options.
hasBin: true
- esniff@2.0.1:
- resolution: {integrity: sha512-kTUIGKQ/mDPFoJ0oVfcmyJn4iBDRptjNVIzwIFR7tqWXdVI9xfA2RMwY/gbSpJG3lkdWNEjLap/NqVHZiJsdfg==}
- engines: {node: '>=0.10'}
-
espree@9.6.1:
resolution: {integrity: sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==}
engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
@@ -6761,9 +6685,6 @@ packages:
resolution: {integrity: sha512-EzV94NYKoO09GLXGjXj9JIlXijVck4ONSr5wiCWDvhsvj5jxSrzTmRU/9C1DyB6uToszLs8aifA6NQ7lEQdvFw==}
engines: {node: '>= 0.8'}
- event-emitter@0.3.5:
- resolution: {integrity: sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==}
-
event-target-shim@5.0.1:
resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==}
engines: {node: '>=6'}
@@ -6970,9 +6891,6 @@ packages:
resolution: {integrity: sha512-6VyCijWQ+9O7WuVMTRBTl+cjNNIzD5cY5mQ1WM8r/LEkI2u8EYpOotESNwzNlyCn3g+dmjKYI6BmNneSr/FSRw==}
engines: {node: '>= 0.10.0'}
- ext@1.7.0:
- resolution: {integrity: sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==}
-
extend-shallow@2.0.1:
resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==}
engines: {node: '>=0.10.0'}
@@ -7419,11 +7337,6 @@ packages:
resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==}
deprecated: Glob versions prior to v9 are no longer supported
- glob@8.1.0:
- resolution: {integrity: sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==}
- engines: {node: '>=12'}
- deprecated: Glob versions prior to v9 are no longer supported
-
global-dirs@3.0.1:
resolution: {integrity: sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==}
engines: {node: '>=10'}
@@ -7507,9 +7420,6 @@ packages:
handle-thing@2.0.1:
resolution: {integrity: sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==}
- hanji@0.0.5:
- resolution: {integrity: sha512-Abxw1Lq+TnYiL4BueXqMau222fPSPMFtya8HdpWsz/xVAhifXou71mPh/kY2+08RgFcVccjG3uZHs6K5HAe3zw==}
-
has-bigints@1.0.2:
resolution: {integrity: sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==}
@@ -7591,9 +7501,6 @@ packages:
resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==}
hasBin: true
- heap@0.2.7:
- resolution: {integrity: sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==}
-
hermes-estree@0.15.0:
resolution: {integrity: sha512-lLYvAd+6BnOqWdnNbP/Q8xfl8LOGw4wVjfrNd9Gt8eoFzhNBRVD95n4l2ksfMVOoxuVyegs85g83KS9QOsxbVQ==}
@@ -8115,9 +8022,6 @@ packages:
is-potential-custom-element-name@1.0.1:
resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==}
- is-promise@2.2.2:
- resolution: {integrity: sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==}
-
is-reference@3.0.2:
resolution: {integrity: sha512-v3rht/LgVcsdZa3O2Nqs+NMowLOxeOm7Ay9+/ARQ2F+qEoANRcqrjAZKGN0v8ymUetZGgkp26LTnGT7H0Qo9Pg==}
@@ -8384,10 +8288,6 @@ packages:
json-buffer@3.0.1:
resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==}
- json-diff@0.9.0:
- resolution: {integrity: sha512-cVnggDrVkAAA3OvFfHpFEhOnmcsUpleEKq4d4O8sQWWSH40MBrWstKigVB1kGrgLWzuom+7rRdaCsnBD6VyObQ==}
- hasBin: true
-
json-parse-better-errors@1.0.2:
resolution: {integrity: sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==}
@@ -8628,6 +8528,11 @@ packages:
resolution: {integrity: sha512-wUayTU8MS827Dam6MxgD72Ui+KOSF+u/eIqpatOtjnvgJ0+mnDq33uC2M7J0tPK+upe/DpUAuK4JUU89iBoNKQ==}
engines: {node: '>=4'}
+ liteque@0.1.3:
+ resolution: {integrity: sha512-DRdJ974/XomYtPXtoyWlUpC5UObRG0Rn/dncgLnJVR/tv084QPix2mGLcVWVR/2794EyO7Cnvt0VqWgpOeFuAQ==}
+ peerDependencies:
+ better-sqlite3: '>=7'
+
loader-runner@4.3.0:
resolution: {integrity: sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==}
engines: {node: '>=6.11.5'}
@@ -8764,9 +8669,6 @@ packages:
resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==}
engines: {node: '>=12'}
- lru-queue@0.1.0:
- resolution: {integrity: sha512-BpdYkt9EvGl8OfWHDQPISVpcl5xZthb+XPsbELj5AQXxIC8IriDZIQYjBJPEm5rS420sjZ0TLEzRcq5KdBhYrQ==}
-
lucide-react-native@0.354.0:
resolution: {integrity: sha512-uNouh+JwusCJPyQ7YnoDD8lTmO+5T4dsh1MAjmnAYxv+cr8c8ZlN84rsbBeYsWDEaqLsrG8PMBFnIrvkcHX8zA==}
peerDependencies:
@@ -8929,9 +8831,6 @@ packages:
memoize-one@6.0.0:
resolution: {integrity: sha512-rkpe71W0N0c0Xz6QD0eJETuWAJGnJ9afsl1srmwPrI+yBCkge5EycXXbYRyvL29zZVUWQCY7InPRCv3GDXuZNw==}
- memoizee@0.4.15:
- resolution: {integrity: sha512-UBWmJpLZd5STPm7PMUlOw/TSy972M+z8gcyQ5veOnSDRREz/0bmpyTfKt3/51DhEBqCZQn1udM/5flcSPYhkdQ==}
-
memory-cache@0.2.0:
resolution: {integrity: sha512-OcjA+jzjOYzKmKS6IQVALHLVz+rNTMPoJvCztFaZxwG14wtAW7VRZjwTQu06vKCYOxh4jVnik7ya0SXTB0W+xA==}
@@ -9287,10 +9186,6 @@ packages:
resolution: {integrity: sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==}
engines: {node: '>=10'}
- minimatch@7.4.6:
- resolution: {integrity: sha512-sBz8G/YjVniEz6lKPNpKxXwazJe4c19fEfV2GDMX6AjFz+MX9uDWIZW8XreVhkFW3fkIdTv/gxWr/Kks5FFAVw==}
- engines: {node: '>=10'}
-
minimatch@9.0.3:
resolution: {integrity: sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==}
engines: {node: '>=16 || 14 >=14.17'}
@@ -9465,9 +9360,6 @@ packages:
react: ^16.8 || ^17 || ^18
react-dom: ^16.8 || ^17 || ^18
- next-tick@1.1.0:
- resolution: {integrity: sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==}
-
next@14.2.13:
resolution: {integrity: sha512-BseY9YNw8QJSwLYD7hlZzl6QVDoSFHL/URN5K64kVEVpCsSOWeyjbIGK+dZUaRViHTaMQX8aqmnn0PHBbGZezg==}
engines: {node: '>=18.17.0'}
@@ -12074,9 +11966,6 @@ packages:
thunky@1.1.0:
resolution: {integrity: sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==}
- timers-ext@0.1.7:
- resolution: {integrity: sha512-b85NUNzTSdodShTIbky6ZF02e8STtVVfD+fu4aXXShEELpozH+bCpJLYMPZbsABN2wDH7fJpqIoXxJpzbf0NqQ==}
-
tiny-invariant@1.3.3:
resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==}
@@ -12285,12 +12174,6 @@ packages:
resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==}
engines: {node: '>= 0.6'}
- type@1.2.0:
- resolution: {integrity: sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==}
-
- type@2.7.2:
- resolution: {integrity: sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==}
-
typed-array-buffer@1.0.2:
resolution: {integrity: sha512-gEymJYKZtKXzzBzM4jqa9w6Q1Jjm7x2d+sh19AdsD4wqnMPDYyvwpsIc2Q/835kHuo3BEQ7CjelGhfTsoBb2MQ==}
engines: {node: '>= 0.4'}
@@ -12858,9 +12741,6 @@ packages:
wonka@4.0.15:
resolution: {integrity: sha512-U0IUQHKXXn6PFo9nqsHphVCE5m3IntqZNB9Jjn7EB1lrR7YTDY3YWgFvEvwniTzXSvOH/XMzAZaIfJF/LvHYXg==}
- wordwrap@1.0.0:
- resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==}
-
workbox-background-sync@6.6.0:
resolution: {integrity: sha512-jkf4ZdgOJxC9u2vztxLuPT/UjlH7m/nWRQ/MgGL0v8BJHoZdVGJd18Kck+a0e55wGXdqyHO+4IQTk0685g4MUw==}
@@ -15932,11 +15812,6 @@ snapshots:
'@drizzle-team/brocli@0.10.1':
dev: true
- '@drizzle-team/studio@0.0.39':
- dependencies:
- superjson: 2.2.1
- dev: true
-
'@egjs/hammerjs@2.0.17':
dependencies:
'@types/hammerjs': 2.0.45
@@ -20270,7 +20145,8 @@ snapshots:
camelcase@6.3.0:
dev: false
- camelcase@7.0.1: {}
+ camelcase@7.0.1:
+ dev: false
camelize@1.0.1:
dev: false
@@ -20477,15 +20353,6 @@ snapshots:
cli-boxes@3.0.0:
dev: false
- cli-color@2.0.3:
- dependencies:
- d: 1.0.1
- es5-ext: 0.10.63
- es6-iterator: 2.0.3
- memoizee: 0.4.15
- timers-ext: 0.1.7
- dev: true
-
cli-cursor@2.1.0:
dependencies:
restore-cursor: 2.0.0
@@ -20654,7 +20521,8 @@ snapshots:
commander@8.3.0:
dev: false
- commander@9.5.0: {}
+ commander@9.5.0:
+ dev: false
common-path-prefix@3.0.0:
dev: false
@@ -21054,12 +20922,6 @@ snapshots:
csstype@3.1.3: {}
- d@1.0.1:
- dependencies:
- es5-ext: 0.10.63
- type: 1.2.0
- dev: true
-
dag-map@1.0.2:
dev: false
@@ -21284,11 +21146,6 @@ snapshots:
diff-sequences@29.6.3:
dev: true
- difflib@0.2.4:
- dependencies:
- heap: 0.2.7
- dev: true
-
dir-glob@3.0.1:
dependencies:
path-type: 4.0.0
@@ -21526,31 +21383,6 @@ snapshots:
dotenv@16.4.5:
dev: false
- dreamopt@0.8.0:
- dependencies:
- wordwrap: 1.0.0
- dev: true
-
- drizzle-kit@0.20.14:
- dependencies:
- '@drizzle-team/studio': 0.0.39
- '@esbuild-kit/esm-loader': 2.6.5
- camelcase: 7.0.1
- chalk: 5.3.0
- commander: 9.5.0
- env-paths: 3.0.0
- esbuild: 0.19.12
- esbuild-register: 3.5.0(esbuild@0.19.12)
- glob: 8.1.0
- hanji: 0.0.5
- json-diff: 0.9.0
- minimatch: 7.4.6
- semver: 7.6.0
- zod: 3.22.4
- transitivePeerDependencies:
- - supports-color
- dev: true
-
drizzle-kit@0.24.2:
dependencies:
'@drizzle-team/brocli': 0.10.1
@@ -21650,9 +21482,6 @@ snapshots:
env-paths@2.2.1:
dev: false
- env-paths@3.0.0:
- dev: true
-
envinfo@7.11.1:
dev: false
@@ -21769,35 +21598,6 @@ snapshots:
is-date-object: 1.0.5
is-symbol: 1.0.4
- es5-ext@0.10.63:
- dependencies:
- es6-iterator: 2.0.3
- es6-symbol: 3.1.3
- esniff: 2.0.1
- next-tick: 1.1.0
- dev: true
-
- es6-iterator@2.0.3:
- dependencies:
- d: 1.0.1
- es5-ext: 0.10.63
- es6-symbol: 3.1.3
- dev: true
-
- es6-symbol@3.1.3:
- dependencies:
- d: 1.0.1
- ext: 1.7.0
- dev: true
-
- es6-weak-map@2.0.3:
- dependencies:
- d: 1.0.1
- es5-ext: 0.10.63
- es6-iterator: 2.0.3
- es6-symbol: 3.1.3
- dev: true
-
esbuild-register@3.5.0(esbuild@0.19.12):
dependencies:
debug: 4.3.4
@@ -22147,14 +21947,6 @@ snapshots:
transitivePeerDependencies:
- supports-color
- esniff@2.0.1:
- dependencies:
- d: 1.0.1
- es5-ext: 0.10.63
- event-emitter: 0.3.5
- type: 2.7.2
- dev: true
-
espree@9.6.1:
dependencies:
acorn: 8.11.3
@@ -22230,12 +22022,6 @@ snapshots:
require-like: 0.1.2
dev: false
- event-emitter@0.3.5:
- dependencies:
- d: 1.0.1
- es5-ext: 0.10.63
- dev: true
-
event-target-shim@5.0.1:
dev: false
@@ -22613,11 +22399,6 @@ snapshots:
- supports-color
dev: false
- ext@1.7.0:
- dependencies:
- type: 2.7.2
- dev: true
-
extend-shallow@2.0.1:
dependencies:
is-extendable: 0.1.1
@@ -23196,15 +22977,6 @@ snapshots:
once: 1.4.0
path-is-absolute: 1.0.1
- glob@8.1.0:
- dependencies:
- fs.realpath: 1.0.0
- inflight: 1.0.6
- inherits: 2.0.4
- minimatch: 5.1.6
- once: 1.4.0
- dev: true
-
global-dirs@3.0.1:
dependencies:
ini: 2.0.0
@@ -23333,12 +23105,6 @@ snapshots:
handle-thing@2.0.1:
dev: false
- hanji@0.0.5:
- dependencies:
- lodash.throttle: 4.1.1
- sisteransi: 1.0.5
- dev: true
-
has-bigints@1.0.2: {}
has-flag@3.0.0: {}
@@ -23501,9 +23267,6 @@ snapshots:
he@1.2.0:
dev: false
- heap@0.2.7:
- dev: true
-
hermes-estree@0.15.0:
dev: false
@@ -24092,9 +23855,6 @@ snapshots:
is-potential-custom-element-name@1.0.1:
dev: false
- is-promise@2.2.2:
- dev: true
-
is-reference@3.0.2:
dependencies:
'@types/estree': 1.0.5
@@ -24486,13 +24246,6 @@ snapshots:
json-buffer@3.0.1: {}
- json-diff@0.9.0:
- dependencies:
- cli-color: 2.0.3
- difflib: 0.2.4
- dreamopt: 0.8.0
- dev: true
-
json-parse-better-errors@1.0.2:
dev: false
@@ -24743,6 +24496,42 @@ snapshots:
liquid-json@0.3.1:
dev: false
+ liteque@0.1.3(better-sqlite3@11.3.0):
+ dependencies:
+ async-mutex: 0.4.1
+ better-sqlite3: 11.3.0
+ drizzle-orm: 0.33.0(@types/react@18.2.58)(better-sqlite3@11.3.0)(react@18.2.0)
+ zod: 3.22.4
+ transitivePeerDependencies:
+ - '@aws-sdk/client-rds-data'
+ - '@cloudflare/workers-types'
+ - '@electric-sql/pglite'
+ - '@libsql/client'
+ - '@neondatabase/serverless'
+ - '@op-engineering/op-sqlite'
+ - '@opentelemetry/api'
+ - '@planetscale/database'
+ - '@prisma/client'
+ - '@tidbcloud/serverless'
+ - '@types/better-sqlite3'
+ - '@types/pg'
+ - '@types/react'
+ - '@types/sql.js'
+ - '@vercel/postgres'
+ - '@xata.io/client'
+ - bun-types
+ - expo-sqlite
+ - knex
+ - kysely
+ - mysql2
+ - pg
+ - postgres
+ - prisma
+ - react
+ - sql.js
+ - sqlite3
+ dev: false
+
loader-runner@4.3.0: {}
loader-utils@2.0.4:
@@ -24808,7 +24597,8 @@ snapshots:
lodash.sortby@4.7.0:
dev: false
- lodash.throttle@4.1.1: {}
+ lodash.throttle@4.1.1:
+ dev: false
lodash.truncate@4.4.2:
dev: true
@@ -24894,11 +24684,6 @@ snapshots:
lru-cache@7.18.3:
dev: false
- lru-queue@0.1.0:
- dependencies:
- es5-ext: 0.10.63
- dev: true
-
lucide-react-native@0.354.0(react-native-svg@15.6.0(react-native@0.73.4(@babel/core@7.23.9)(@babel/preset-env@7.24.0(@babel/core@7.23.9))(react@18.2.0))(react@18.2.0))(react-native@0.73.4(@babel/core@7.23.9)(@babel/preset-env@7.24.0(@babel/core@7.23.9))(react@18.2.0))(react@18.2.0):
dependencies:
react: 18.2.0
@@ -25239,18 +25024,6 @@ snapshots:
memoize-one@6.0.0:
dev: false
- memoizee@0.4.15:
- dependencies:
- d: 1.0.1
- es5-ext: 0.10.63
- es6-weak-map: 2.0.3
- event-emitter: 0.3.5
- is-promise: 2.2.2
- lru-queue: 0.1.0
- next-tick: 1.1.0
- timers-ext: 0.1.7
- dev: true
-
memory-cache@0.2.0:
dev: false
@@ -25971,11 +25744,7 @@ snapshots:
minimatch@5.1.6:
dependencies:
brace-expansion: 2.0.1
-
- minimatch@7.4.6:
- dependencies:
- brace-expansion: 2.0.1
- dev: true
+ dev: false
minimatch@9.0.3:
dependencies:
@@ -26205,9 +25974,6 @@ snapshots:
react-dom: 18.2.0(react@18.2.0)
dev: false
- next-tick@1.1.0:
- dev: true
-
next@14.2.13(@babel/core@7.24.0)(react-dom@18.2.0(react@18.2.0))(react@18.2.0):
dependencies:
'@next/env': 14.2.13
@@ -29020,7 +28786,8 @@ snapshots:
totalist: 3.0.1
dev: false
- sisteransi@1.0.5: {}
+ sisteransi@1.0.5:
+ dev: false
sitemap@7.1.1:
dependencies:
@@ -29694,12 +29461,6 @@ snapshots:
thunky@1.1.0:
dev: false
- timers-ext@0.1.7:
- dependencies:
- es5-ext: 0.10.63
- next-tick: 1.1.0
- dev: true
-
tiny-invariant@1.3.3:
dev: false
@@ -29905,12 +29666,6 @@ snapshots:
mime-types: 2.1.35
dev: false
- type@1.2.0:
- dev: true
-
- type@2.7.2:
- dev: true
-
typed-array-buffer@1.0.2:
dependencies:
call-bind: 1.0.7
@@ -30678,9 +30433,6 @@ snapshots:
wonka@4.0.15:
dev: false
- wordwrap@1.0.0:
- dev: true
-
workbox-background-sync@6.6.0:
dependencies:
idb: 7.1.1
@@ -30988,7 +30740,8 @@ snapshots:
zlibjs@0.3.1:
dev: false
- zod@3.22.4: {}
+ zod@3.22.4:
+ dev: false
zustand@4.5.1(@types/react@18.2.58)(react@18.2.0):
dependencies: