diff options
| -rw-r--r-- | apps/workers/metrics.ts | 7 | ||||
| -rw-r--r-- | apps/workers/workers/crawlerWorker.ts | 7 |
2 files changed, 13 insertions, 1 deletions
diff --git a/apps/workers/metrics.ts b/apps/workers/metrics.ts index b9ed2c70..3dc4d2c0 100644 --- a/apps/workers/metrics.ts +++ b/apps/workers/metrics.ts @@ -15,4 +15,11 @@ export const workerStatsCounter = new Counter({ labelNames: ["worker_name", "status"], }); +export const crawlerStatusCodeCounter = new Counter({ + name: "karakeep_crawler_status_codes_total", + help: "HTTP status codes encountered during crawling", + labelNames: ["status_code"], +}); + registry.registerMetric(workerStatsCounter); +registry.registerMetric(crawlerStatusCodeCounter); diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts index 07a74757..0ab3a3cc 100644 --- a/apps/workers/workers/crawlerWorker.ts +++ b/apps/workers/workers/crawlerWorker.ts @@ -26,7 +26,7 @@ import metascraperPublisher from "metascraper-publisher"; import metascraperTitle from "metascraper-title"; import metascraperTwitter from "metascraper-twitter"; import metascraperUrl from "metascraper-url"; -import { workerStatsCounter } from "metrics"; +import { crawlerStatusCodeCounter, workerStatsCounter } from "metrics"; import { fetchWithProxy, getRandomProxy, @@ -1099,6 +1099,11 @@ async function crawlAndParseUrl( const { htmlContent, screenshot, statusCode, url: browserUrl } = result; + // Track status code in Prometheus + if (statusCode !== null) { + crawlerStatusCodeCounter.labels(statusCode.toString()).inc(); + } + const meta = await Promise.race([ extractMetadata(htmlContent, browserUrl, jobId), abortPromise(abortSignal), |