feature: Schedule RSS feed refreshes every hour

5 files changed, 66 insertions, 11 deletions

diff --git a/apps/workers/feedWorker.ts b/apps/workers/feedWorker.ts
index 1bd24641..2f2c80f5 100644
--- a/apps/workers/feedWorker.ts
+++ b/apps/workers/feedWorker.ts
@@ -1,5 +1,6 @@
 import { and, eq, inArray } from "drizzle-orm";
 import { DequeuedJob, Runner } from "liteque";
+import cron from "node-cron";
 import Parser from "rss-parser";
 import { buildImpersonatingTRPCClient } from "trpc";
 
@@ -10,6 +11,35 @@ import logger from "@hoarder/shared/logger";
 import { FeedQueue } from "@hoarder/shared/queues";
 import { BookmarkTypes } from "@hoarder/shared/types/bookmarks";
 
+export const FeedRefreshingWorker = cron.schedule(
+  "0 * * * *",
+  () => {
+    logger.info("[feed] Scheduling feed refreshing jobs ...");
+    db.query.rssFeedsTable
+      .findMany({
+        columns: {
+          id: true,
+        },
+      })
+      .then((feeds) => {
+        for (const feed of feeds) {
+          FeedQueue.enqueue(
+            {
+              feedId: feed.id,
+            },
+            {
+              idempotencyKey: feed.id,
+            },
+          );
+        }
+      });
+  },
+  {
+    runOnInit: false,
+    scheduled: false,
+  },
+);
+
 export class FeedWorker {
   static build() {
     logger.info("Starting feed worker ...");
diff --git a/apps/workers/index.ts b/apps/workers/index.ts
index c8978adc..e333ab0d 100644
--- a/apps/workers/index.ts
+++ b/apps/workers/index.ts
@@ -1,6 +1,6 @@
 import "dotenv/config";
 
-import { FeedWorker } from "feedWorker";
+import { FeedRefreshingWorker, FeedWorker } from "feedWorker";
 import { TidyAssetsWorker } from "tidyAssetsWorker";
 
 import serverConfig from "@hoarder/shared/config";
@@ -25,6 +25,7 @@ async function main() {
     VideoWorker.build(),
     FeedWorker.build(),
   ];
+  FeedRefreshingWorker.start();
 
   await Promise.any([
     Promise.all([
@@ -41,6 +42,7 @@ async function main() {
     "Shutting down crawler, openai, tidyAssets, video, feed and search workers ...",
   );
 
+  FeedRefreshingWorker.stop();
   crawler.stop();
   openai.stop();
   search.stop();
diff --git a/apps/workers/package.json b/apps/workers/package.json
index a7579319..597c5b14 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -16,7 +16,7 @@
     "drizzle-orm": "^0.33.0",
     "execa": "9.3.1",
     "jsdom": "^24.0.0",
-    "liteque": "^0.1.3",
+    "liteque": "^0.2.0",
     "metascraper": "^5.45.24",
     "metascraper-amazon": "^5.45.22",
     "metascraper-description": "^5.45.22",
@@ -27,6 +27,7 @@
     "metascraper-title": "^5.45.22",
     "metascraper-twitter": "^5.45.6",
     "metascraper-url": "^5.45.22",
+    "node-cron": "^3.0.3",
     "pdf2json": "^3.0.5",
     "pdfjs-dist": "^4.0.379",
     "puppeteer": "^22.0.0",
@@ -44,7 +45,8 @@
     "@hoarder/prettier-config": "workspace:^0.1.0",
     "@types/dompurify": "^3.0.5",
     "@types/jsdom": "^21.1.6",
-    "@types/metascraper": "^5.14.3"
+    "@types/metascraper": "^5.14.3",
+    "@types/node-cron": "^3.0.11"
   },
   "scripts": {
     "start": "tsx watch index.ts",
diff --git a/packages/shared/package.json b/packages/shared/package.json
index a8d636f7..32ffa737 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -6,7 +6,7 @@
   "type": "module",
   "dependencies": {
     "glob": "^11.0.0",
-    "liteque": "^0.1.3",
+    "liteque": "^0.2.0",
     "meilisearch": "^0.37.0",
     "ollama": "^0.5.9",
     "openai": "^4.67.1",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 83cff91d..ab0ac31b 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -720,8 +720,8 @@ importers:
         specifier: ^24.0.0
         version: 24.0.0
       liteque:
-        specifier: ^0.1.3
-        version: 0.1.3(better-sqlite3@11.3.0)
+        specifier: ^0.2.0
+        version: 0.2.0(better-sqlite3@11.3.0)
       metascraper:
         specifier: ^5.45.24
         version: 5.45.24
@@ -752,6 +752,9 @@ importers:
       metascraper-url:
         specifier: ^5.45.22
         version: 5.45.22
+      node-cron:
+        specifier: ^3.0.3
+        version: 3.0.3
       pdf2json:
         specifier: ^3.0.5
         version: 3.0.5
@@ -801,6 +804,9 @@ importers:
       '@types/metascraper':
         specifier: ^5.14.3
         version: 5.14.3
+      '@types/node-cron':
+        specifier: ^3.0.11
+        version: 3.0.11
 
   docs:
     dependencies:
@@ -916,8 +922,8 @@ importers:
         specifier: ^11.0.0
         version: 11.0.0
       liteque:
-        specifier: ^0.1.3
-        version: 0.1.3(better-sqlite3@11.3.0)
+        specifier: ^0.2.0
+        version: 0.2.0(better-sqlite3@11.3.0)
       meilisearch:
         specifier: ^0.37.0
         version: 0.37.0
@@ -4348,6 +4354,9 @@ packages:
   '@types/ms@0.7.34':
     resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==}
 
+  '@types/node-cron@3.0.11':
+    resolution: {integrity: sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==}
+
   '@types/node-fetch@2.6.11':
     resolution: {integrity: sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==}
 
@@ -8534,8 +8543,8 @@ packages:
     resolution: {integrity: sha512-wUayTU8MS827Dam6MxgD72Ui+KOSF+u/eIqpatOtjnvgJ0+mnDq33uC2M7J0tPK+upe/DpUAuK4JUU89iBoNKQ==}
     engines: {node: '>=4'}
 
-  liteque@0.1.3:
-    resolution: {integrity: sha512-DRdJ974/XomYtPXtoyWlUpC5UObRG0Rn/dncgLnJVR/tv084QPix2mGLcVWVR/2794EyO7Cnvt0VqWgpOeFuAQ==}
+  liteque@0.2.0:
+    resolution: {integrity: sha512-OgRTJmGq5pyIhMOk7QhiEGuRP1Cu9hvdbs+kSwrx7RY0V0Up0vuoMM062V0HJdk+qmJKY2P4JXHYGYOSMCAEnw==}
     peerDependencies:
       better-sqlite3: '>=7'
 
@@ -9401,6 +9410,10 @@ packages:
   node-abort-controller@3.1.1:
     resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==}
 
+  node-cron@3.0.3:
+    resolution: {integrity: sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==}
+    engines: {node: '>=6.0.0'}
+
   node-dir@0.1.17:
     resolution: {integrity: sha512-tmPX422rYgofd4epzrNoOXiE8XFZYOcCq1vD7MAXCDO+O+zndlA2ztdKKMa+EeuBG5tHETpr4ml4RGgpqDCCAg==}
     engines: {node: '>= 0.10.5'}
@@ -18743,6 +18756,9 @@ snapshots:
 
   '@types/ms@0.7.34': {}
 
+  '@types/node-cron@3.0.11':
+    dev: true
+
   '@types/node-fetch@2.6.11':
     dependencies:
       '@types/node': 20.11.20
@@ -24509,7 +24525,7 @@ snapshots:
   liquid-json@0.3.1:
     dev: false
 
-  liteque@0.1.3(better-sqlite3@11.3.0):
+  liteque@0.2.0(better-sqlite3@11.3.0):
     dependencies:
       async-mutex: 0.4.1
       better-sqlite3: 11.3.0
@@ -26033,6 +26049,11 @@ snapshots:
   node-abort-controller@3.1.1:
     dev: false
 
+  node-cron@3.0.3:
+    dependencies:
+      uuid: 8.3.2
+    dev: false
+
   node-dir@0.1.17:
     dependencies:
       minimatch: 3.1.2