feat: Add proper proxy support. fixes #1265

2 files changed, 87 insertions, 9 deletions

diff --git a/apps/workers/package.json b/apps/workers/package.json
index 93a26718..43e479c8 100644
--- a/apps/workers/package.json
+++ b/apps/workers/package.json
@@ -16,6 +16,8 @@
     "dotenv": "^16.4.1",
     "drizzle-orm": "^0.44.2",
     "execa": "9.3.1",
+    "http-proxy-agent": "^7.0.2",
+    "https-proxy-agent": "^7.0.6",
     "jsdom": "^24.0.0",
     "liteque": "^0.4.1",
     "metascraper": "^5.46.18",
diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts
index edd1d8f1..05bce103 100644
--- a/apps/workers/workers/crawlerWorker.ts
+++ b/apps/workers/workers/crawlerWorker.ts
@@ -9,6 +9,8 @@ import DOMPurify from "dompurify";
 import { eq } from "drizzle-orm";
 import { execa } from "execa";
 import { isShuttingDown } from "exit";
+import { HttpProxyAgent } from "http-proxy-agent";
+import { HttpsProxyAgent } from "https-proxy-agent";
 import { JSDOM, VirtualConsole } from "jsdom";
 import { DequeuedJob, EnqueueOptions, Runner } from "liteque";
 import metascraper from "metascraper";
@@ -23,7 +25,7 @@ import metascraperTitle from "metascraper-title";
 import metascraperTwitter from "metascraper-twitter";
 import metascraperUrl from "metascraper-url";
 import fetch from "node-fetch";
-import { Browser } from "playwright";
+import { Browser, BrowserContextOptions } from "playwright";
 import { chromium } from "playwright-extra";
 import StealthPlugin from "puppeteer-extra-plugin-stealth";
 import { withTimeout } from "utils";
@@ -85,6 +87,76 @@ const metascraperParser = metascraper([
   metascraperUrl(),
 ]);
 
+function getProxyAgent(url: string) {
+  const { proxy } = serverConfig;
+
+  if (!proxy.httpProxy && !proxy.httpsProxy) {
+    return undefined;
+  }
+
+  const urlObj = new URL(url);
+  const protocol = urlObj.protocol;
+
+  // Check if URL should bypass proxy
+  if (proxy.noProxy) {
+    const noProxyList = proxy.noProxy.split(",").map((host) => host.trim());
+    const hostname = urlObj.hostname;
+
+    for (const noProxyHost of noProxyList) {
+      if (
+        noProxyHost === hostname ||
+        (noProxyHost.startsWith(".") && hostname.endsWith(noProxyHost)) ||
+        hostname.endsWith("." + noProxyHost)
+      ) {
+        return undefined;
+      }
+    }
+  }
+
+  if (protocol === "https:" && proxy.httpsProxy) {
+    return new HttpsProxyAgent(proxy.httpsProxy);
+  } else if (protocol === "http:" && proxy.httpProxy) {
+    return new HttpProxyAgent(proxy.httpProxy);
+  } else if (proxy.httpProxy) {
+    // Fallback to HTTP proxy for HTTPS if HTTPS proxy not configured
+    return new HttpProxyAgent(proxy.httpProxy);
+  }
+
+  return undefined;
+}
+
+function getPlaywrightProxyConfig(): BrowserContextOptions["proxy"] {
+  const { proxy } = serverConfig;
+
+  if (!proxy.httpProxy && !proxy.httpsProxy) {
+    return undefined;
+  }
+
+  // Use HTTPS proxy if available, otherwise fall back to HTTP proxy
+  const proxyUrl = proxy.httpsProxy || proxy.httpProxy;
+  if (!proxyUrl) {
+    // Unreachable, but TypeScript doesn't know that
+    return undefined;
+  }
+
+  const parsed = new URL(proxyUrl);
+
+  return {
+    server: proxyUrl,
+    username: parsed.username,
+    password: parsed.password,
+    bypass: proxy.noProxy,
+  };
+}
+
+const fetchWithProxy = (url: string, options: Record<string, unknown> = {}) => {
+  const agent = getProxyAgent(url);
+  if (agent) {
+    options.agent = agent;
+  }
+  return fetch(url, options);
+};
+
 let globalBrowser: Browser | undefined;
 let globalBlocker: PlaywrightBlocker | undefined;
 // Guards the interactions with the browser instance.
@@ -163,11 +235,14 @@ export class CrawlerWorker {
     if (serverConfig.crawler.enableAdblocker) {
       try {
         logger.info("[crawler] Loading adblocker ...");
-        globalBlocker = await PlaywrightBlocker.fromPrebuiltFull(fetch, {
-          path: path.join(os.tmpdir(), "karakeep_adblocker.bin"),
-          read: fs.readFile,
-          write: fs.writeFile,
-        });
+        globalBlocker = await PlaywrightBlocker.fromPrebuiltFull(
+          fetchWithProxy,
+          {
+            path: path.join(os.tmpdir(), "karakeep_adblocker.bin"),
+            read: fs.readFile,
+            write: fs.writeFile,
+          },
+        );
       } catch (e) {
         logger.error(
           `[crawler] Failed to load adblocker. Will not be blocking ads: ${e}`,
@@ -258,7 +333,7 @@ async function browserlessCrawlPage(
   logger.info(
     `[Crawler][${jobId}] Running in browserless mode. Will do a plain http request to "${url}". Screenshots will be disabled.`,
   );
-  const response = await fetch(url, {
+  const response = await fetchWithProxy(url, {
     signal: AbortSignal.any([AbortSignal.timeout(5000), abortSignal]),
   });
   logger.info(
@@ -296,6 +371,7 @@ async function crawlPage(
     viewport: { width: 1440, height: 900 },
     userAgent:
       "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    proxy: getPlaywrightProxyConfig(),
   });
   try {
     // Create a new page in the context
@@ -479,7 +555,7 @@ async function downloadAndStoreFile(
 ) {
   try {
     logger.info(`[Crawler][${jobId}] Downloading ${fileType} from "${url}"`);
-    const response = await fetch(url, {
+    const response = await fetchWithProxy(url, {
       signal: abortSignal,
     });
     if (!response.ok) {
@@ -617,7 +693,7 @@ async function getContentType(
     logger.info(
       `[Crawler][${jobId}] Attempting to determine the content-type for the url ${url}`,
     );
-    const response = await fetch(url, {
+    const response = await fetchWithProxy(url, {
       method: "HEAD",
       signal: AbortSignal.any([AbortSignal.timeout(5000), abortSignal]),
     });