diff options
Diffstat (limited to 'apps')
| -rw-r--r-- | apps/workers/workers/crawlerWorker.ts | 15 |
1 files changed, 14 insertions, 1 deletions
diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts index ae7cfcab..357ae976 100644 --- a/apps/workers/workers/crawlerWorker.ts +++ b/apps/workers/workers/crawlerWorker.ts @@ -12,6 +12,8 @@ import DOMPurify from "dompurify"; import { eq } from "drizzle-orm"; import { execa } from "execa"; import { exitAbortController } from "exit"; +import { HttpProxyAgent } from "http-proxy-agent"; +import { HttpsProxyAgent } from "https-proxy-agent"; import { JSDOM, VirtualConsole } from "jsdom"; import metascraper from "metascraper"; import metascraperAmazon from "metascraper-amazon"; @@ -129,7 +131,18 @@ const metascraperParser = metascraper([ metascraperDescription(), metascraperTwitter(), metascraperImage(), - metascraperLogo(), + metascraperLogo({ + gotOpts: { + agent: { + http: serverConfig.proxy.httpProxy + ? new HttpProxyAgent(getRandomProxy(serverConfig.proxy.httpProxy)) + : undefined, + https: serverConfig.proxy.httpsProxy + ? new HttpsProxyAgent(getRandomProxy(serverConfig.proxy.httpsProxy)) + : undefined, + }, + }, + }), metascraperUrl(), ]); |
