diff options
| -rw-r--r-- | apps/workers/crawlerWorker.ts | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts index cce409e5..c9a1189c 100644 --- a/apps/workers/crawlerWorker.ts +++ b/apps/workers/crawlerWorker.ts @@ -222,6 +222,9 @@ async function runCrawler(job: Job<ZCrawlLinkRequest, void>) { const meta = await metascraperParser({ url, html: htmlContent, + // We don't want to validate the URL again as we've already done it by visiting the page. + // This was added because URL validation fails if the URL ends with a question mark (e.g. empty query params). + validateUrl: false, }); logger.info(`[Crawler][${jobId}] Done parsing the content of the page.`); |