diff options
Diffstat (limited to 'apps')
| -rw-r--r-- | apps/workers/package.json | 1 | ||||
| -rw-r--r-- | apps/workers/workers/crawlerWorker.ts | 13 |
2 files changed, 14 insertions, 0 deletions
diff --git a/apps/workers/package.json b/apps/workers/package.json index 1b5b2c95..fa7ffeac 100644 --- a/apps/workers/package.json +++ b/apps/workers/package.json @@ -41,6 +41,7 @@ "metascraper-title": "^5.49.5", "metascraper-twitter": "^5.45.6", "metascraper-url": "^5.49.5", + "metascraper-youtube": "^5.49.7", "node-cron": "^3.0.3", "node-fetch": "^3.3.2", "pdf2json": "^3.1.5", diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts index 44f8f0c3..cb55523f 100644 --- a/apps/workers/workers/crawlerWorker.ts +++ b/apps/workers/workers/crawlerWorker.ts @@ -26,6 +26,7 @@ import metascraperPublisher from "metascraper-publisher"; import metascraperTitle from "metascraper-title"; import metascraperTwitter from "metascraper-twitter"; import metascraperUrl from "metascraper-url"; +import metascraperYoutube from "metascraper-youtube"; import { crawlerStatusCodeCounter, workerStatsCounter } from "metrics"; import { fetchWithProxy, @@ -125,6 +126,18 @@ const metascraperParser = metascraper([ datePublished: true, }), metascraperAmazon(), + metascraperYoutube({ + gotOpts: { + agent: { + http: serverConfig.proxy.httpProxy + ? new HttpProxyAgent(getRandomProxy(serverConfig.proxy.httpProxy)) + : undefined, + https: serverConfig.proxy.httpsProxy + ? new HttpsProxyAgent(getRandomProxy(serverConfig.proxy.httpsProxy)) + : undefined, + }, + }, + }), metascraperReddit(), metascraperAuthor(), metascraperPublisher(), |