aboutsummaryrefslogtreecommitdiffstats
path: root/apps
diff options
context:
space:
mode:
Diffstat (limited to 'apps')
-rw-r--r--apps/workers/workers/crawlerWorker.ts15
1 files changed, 14 insertions, 1 deletions
diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts
index ae7cfcab..357ae976 100644
--- a/apps/workers/workers/crawlerWorker.ts
+++ b/apps/workers/workers/crawlerWorker.ts
@@ -12,6 +12,8 @@ import DOMPurify from "dompurify";
import { eq } from "drizzle-orm";
import { execa } from "execa";
import { exitAbortController } from "exit";
+import { HttpProxyAgent } from "http-proxy-agent";
+import { HttpsProxyAgent } from "https-proxy-agent";
import { JSDOM, VirtualConsole } from "jsdom";
import metascraper from "metascraper";
import metascraperAmazon from "metascraper-amazon";
@@ -129,7 +131,18 @@ const metascraperParser = metascraper([
metascraperDescription(),
metascraperTwitter(),
metascraperImage(),
- metascraperLogo(),
+ metascraperLogo({
+ gotOpts: {
+ agent: {
+ http: serverConfig.proxy.httpProxy
+ ? new HttpProxyAgent(getRandomProxy(serverConfig.proxy.httpProxy))
+ : undefined,
+ https: serverConfig.proxy.httpsProxy
+ ? new HttpsProxyAgent(getRandomProxy(serverConfig.proxy.httpsProxy))
+ : undefined,
+ },
+ },
+ }),
metascraperUrl(),
]);