From d34b538a496e3c01f6c49fe23e4360a841ceee3a Mon Sep 17 00:00:00 2001
From: Mohamed Bassem <me@mbassem.com>
Date: Sun, 3 Nov 2024 18:33:52 +0000
Subject: feature: Schedule RSS feed refreshes every hour

---
 apps/workers/feedWorker.ts | 30 ++++++++++++++++++++++++++++++
 apps/workers/index.ts      |  4 +++-
 apps/workers/package.json  |  6 ++++--
 3 files changed, 37 insertions(+), 3 deletions(-)

(limited to 'apps/workers')

diff --git a/apps/workers/feedWorker.ts b/apps/workers/feedWorker.ts
index 1bd24641..2f2c80f5 100644
--- a/apps/workers/feedWorker.ts
+++ b/apps/workers/feedWorker.ts
@@ -1,5 +1,6 @@
 import { and, eq, inArray } from "drizzle-orm";
 import { DequeuedJob, Runner } from "liteque";
+import cron from "node-cron";
 import Parser from "rss-parser";
 import { buildImpersonatingTRPCClient } from "trpc";
 
@@ -10,6 +11,35 @@ import logger from "@hoarder/shared/logger";
 import { FeedQueue } from "@hoarder/shared/queues";
 import { BookmarkTypes } from "@hoarder/shared/types/bookmarks";
 
+export const FeedRefreshingWorker = cron.schedule(
+  "0 * * * *",
+  () => {
+    logger.info("[feed] Scheduling feed refreshing jobs ...");
+    db.query.rssFeedsTable
+      .findMany({
+        columns: {
+          id: true,
+        },
+      })
+      .then((feeds) => {
+        for (const feed of feeds) {
+          FeedQueue.enqueue(
+            {
+              feedId: feed.id,
+            },
+            {
+              idempotencyKey: feed.id,
+            },
+          );
+        }
+      });
+  },
+  {
+    runOnInit: false,
+    scheduled: false,
+  },
+);
+
 export class FeedWorker {
   static build() {
     logger.info("Starting feed worker ...");
diff --git a/apps/workers/index.ts b/apps/workers/index.ts
index c8978adc..e333ab0d 100644
--- a/apps/workers/index.ts
+++ b/apps/workers/index.ts
@@ -1,6 +1,6 @@
 import "dotenv/config";
 
-import { FeedWorker } from "feedWorker";
+import { FeedRefreshingWorker, FeedWorker } from "feedWorker";
 import { TidyAssetsWorker } from "tidyAssetsWorker";
 
 import serverConfig from "@hoarder/shared/config";
@@ -25,6 +25,7 @@ async function main() {
     VideoWorker.build(),
     FeedWorker.build(),
   ];
+  FeedRefreshingWorker.start();
 
   await Promise.any([
     Promise.all([
@@ -41,6 +42,7 @@ async function main() {
     "Shutting down crawler, openai, tidyAssets, video, feed and search workers ...",
   );
 
+  FeedRefreshingWorker.stop();
   crawler.stop();
   openai.stop();
   search.stop();
diff --git a/apps/workers/package.json b/apps/workers/package.json
index a7579319..597c5b14 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -16,7 +16,7 @@
     "drizzle-orm": "^0.33.0",
     "execa": "9.3.1",
     "jsdom": "^24.0.0",
-    "liteque": "^0.1.3",
+    "liteque": "^0.2.0",
     "metascraper": "^5.45.24",
     "metascraper-amazon": "^5.45.22",
     "metascraper-description": "^5.45.22",
@@ -27,6 +27,7 @@
     "metascraper-title": "^5.45.22",
     "metascraper-twitter": "^5.45.6",
     "metascraper-url": "^5.45.22",
+    "node-cron": "^3.0.3",
     "pdf2json": "^3.0.5",
     "pdfjs-dist": "^4.0.379",
     "puppeteer": "^22.0.0",
@@ -44,7 +45,8 @@
     "@hoarder/prettier-config": "workspace:^0.1.0",
     "@types/dompurify": "^3.0.5",
     "@types/jsdom": "^21.1.6",
-    "@types/metascraper": "^5.14.3"
+    "@types/metascraper": "^5.14.3",
+    "@types/node-cron": "^3.0.11"
   },
   "scripts": {
     "start": "tsx watch index.ts",
-- 
cgit v1.2.3-70-g09d2