From d34b538a496e3c01f6c49fe23e4360a841ceee3a Mon Sep 17 00:00:00 2001 From: Mohamed Bassem Date: Sun, 3 Nov 2024 18:33:52 +0000 Subject: feature: Schedule RSS feed refreshes every hour --- apps/workers/feedWorker.ts | 30 ++++++++++++++++++++++++++++++ apps/workers/index.ts | 4 +++- apps/workers/package.json | 6 ++++-- packages/shared/package.json | 2 +- pnpm-lock.yaml | 35 ++++++++++++++++++++++++++++------- 5 files changed, 66 insertions(+), 11 deletions(-) diff --git a/apps/workers/feedWorker.ts b/apps/workers/feedWorker.ts index 1bd24641..2f2c80f5 100644 --- a/apps/workers/feedWorker.ts +++ b/apps/workers/feedWorker.ts @@ -1,5 +1,6 @@ import { and, eq, inArray } from "drizzle-orm"; import { DequeuedJob, Runner } from "liteque"; +import cron from "node-cron"; import Parser from "rss-parser"; import { buildImpersonatingTRPCClient } from "trpc"; @@ -10,6 +11,35 @@ import logger from "@hoarder/shared/logger"; import { FeedQueue } from "@hoarder/shared/queues"; import { BookmarkTypes } from "@hoarder/shared/types/bookmarks"; +export const FeedRefreshingWorker = cron.schedule( + "0 * * * *", + () => { + logger.info("[feed] Scheduling feed refreshing jobs ..."); + db.query.rssFeedsTable + .findMany({ + columns: { + id: true, + }, + }) + .then((feeds) => { + for (const feed of feeds) { + FeedQueue.enqueue( + { + feedId: feed.id, + }, + { + idempotencyKey: feed.id, + }, + ); + } + }); + }, + { + runOnInit: false, + scheduled: false, + }, +); + export class FeedWorker { static build() { logger.info("Starting feed worker ..."); diff --git a/apps/workers/index.ts b/apps/workers/index.ts index c8978adc..e333ab0d 100644 --- a/apps/workers/index.ts +++ b/apps/workers/index.ts @@ -1,6 +1,6 @@ import "dotenv/config"; -import { FeedWorker } from "feedWorker"; +import { FeedRefreshingWorker, FeedWorker } from "feedWorker"; import { TidyAssetsWorker } from "tidyAssetsWorker"; import serverConfig from "@hoarder/shared/config"; @@ -25,6 +25,7 @@ async function main() { VideoWorker.build(), FeedWorker.build(), ]; + FeedRefreshingWorker.start(); await Promise.any([ Promise.all([ @@ -41,6 +42,7 @@ async function main() { "Shutting down crawler, openai, tidyAssets, video, feed and search workers ...", ); + FeedRefreshingWorker.stop(); crawler.stop(); openai.stop(); search.stop(); diff --git a/apps/workers/package.json b/apps/workers/package.json index a7579319..597c5b14 100644 --- a/apps/workers/package.json +++ b/apps/workers/package.json @@ -16,7 +16,7 @@ "drizzle-orm": "^0.33.0", "execa": "9.3.1", "jsdom": "^24.0.0", - "liteque": "^0.1.3", + "liteque": "^0.2.0", "metascraper": "^5.45.24", "metascraper-amazon": "^5.45.22", "metascraper-description": "^5.45.22", @@ -27,6 +27,7 @@ "metascraper-title": "^5.45.22", "metascraper-twitter": "^5.45.6", "metascraper-url": "^5.45.22", + "node-cron": "^3.0.3", "pdf2json": "^3.0.5", "pdfjs-dist": "^4.0.379", "puppeteer": "^22.0.0", @@ -44,7 +45,8 @@ "@hoarder/prettier-config": "workspace:^0.1.0", "@types/dompurify": "^3.0.5", "@types/jsdom": "^21.1.6", - "@types/metascraper": "^5.14.3" + "@types/metascraper": "^5.14.3", + "@types/node-cron": "^3.0.11" }, "scripts": { "start": "tsx watch index.ts", diff --git a/packages/shared/package.json b/packages/shared/package.json index a8d636f7..32ffa737 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -6,7 +6,7 @@ "type": "module", "dependencies": { "glob": "^11.0.0", - "liteque": "^0.1.3", + "liteque": "^0.2.0", "meilisearch": "^0.37.0", "ollama": "^0.5.9", "openai": "^4.67.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 83cff91d..ab0ac31b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -720,8 +720,8 @@ importers: specifier: ^24.0.0 version: 24.0.0 liteque: - specifier: ^0.1.3 - version: 0.1.3(better-sqlite3@11.3.0) + specifier: ^0.2.0 + version: 0.2.0(better-sqlite3@11.3.0) metascraper: specifier: ^5.45.24 version: 5.45.24 @@ -752,6 +752,9 @@ importers: metascraper-url: specifier: ^5.45.22 version: 5.45.22 + node-cron: + specifier: ^3.0.3 + version: 3.0.3 pdf2json: specifier: ^3.0.5 version: 3.0.5 @@ -801,6 +804,9 @@ importers: '@types/metascraper': specifier: ^5.14.3 version: 5.14.3 + '@types/node-cron': + specifier: ^3.0.11 + version: 3.0.11 docs: dependencies: @@ -916,8 +922,8 @@ importers: specifier: ^11.0.0 version: 11.0.0 liteque: - specifier: ^0.1.3 - version: 0.1.3(better-sqlite3@11.3.0) + specifier: ^0.2.0 + version: 0.2.0(better-sqlite3@11.3.0) meilisearch: specifier: ^0.37.0 version: 0.37.0 @@ -4348,6 +4354,9 @@ packages: '@types/ms@0.7.34': resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} + '@types/node-cron@3.0.11': + resolution: {integrity: sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==} + '@types/node-fetch@2.6.11': resolution: {integrity: sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==} @@ -8534,8 +8543,8 @@ packages: resolution: {integrity: sha512-wUayTU8MS827Dam6MxgD72Ui+KOSF+u/eIqpatOtjnvgJ0+mnDq33uC2M7J0tPK+upe/DpUAuK4JUU89iBoNKQ==} engines: {node: '>=4'} - liteque@0.1.3: - resolution: {integrity: sha512-DRdJ974/XomYtPXtoyWlUpC5UObRG0Rn/dncgLnJVR/tv084QPix2mGLcVWVR/2794EyO7Cnvt0VqWgpOeFuAQ==} + liteque@0.2.0: + resolution: {integrity: sha512-OgRTJmGq5pyIhMOk7QhiEGuRP1Cu9hvdbs+kSwrx7RY0V0Up0vuoMM062V0HJdk+qmJKY2P4JXHYGYOSMCAEnw==} peerDependencies: better-sqlite3: '>=7' @@ -9401,6 +9410,10 @@ packages: node-abort-controller@3.1.1: resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + node-cron@3.0.3: + resolution: {integrity: sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==} + engines: {node: '>=6.0.0'} + node-dir@0.1.17: resolution: {integrity: sha512-tmPX422rYgofd4epzrNoOXiE8XFZYOcCq1vD7MAXCDO+O+zndlA2ztdKKMa+EeuBG5tHETpr4ml4RGgpqDCCAg==} engines: {node: '>= 0.10.5'} @@ -18743,6 +18756,9 @@ snapshots: '@types/ms@0.7.34': {} + '@types/node-cron@3.0.11': + dev: true + '@types/node-fetch@2.6.11': dependencies: '@types/node': 20.11.20 @@ -24509,7 +24525,7 @@ snapshots: liquid-json@0.3.1: dev: false - liteque@0.1.3(better-sqlite3@11.3.0): + liteque@0.2.0(better-sqlite3@11.3.0): dependencies: async-mutex: 0.4.1 better-sqlite3: 11.3.0 @@ -26033,6 +26049,11 @@ snapshots: node-abort-controller@3.1.1: dev: false + node-cron@3.0.3: + dependencies: + uuid: 8.3.2 + dev: false + node-dir@0.1.17: dependencies: minimatch: 3.1.2 -- cgit v1.2.3-70-g09d2