diff options
| author | Mohamed Bassem <me@mbassem.com> | 2025-11-09 21:20:54 +0000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2025-11-09 21:20:54 +0000 |
| commit | 4cf0856e39c4d69037a6c1a4c3a2a7f803b364a7 (patch) | |
| tree | 30c0381668b56edc56daf6840968d239a8201fc0 /packages/shared | |
| parent | b28cd03a4a5f95f429a1429a59319c8a9ac986f8 (diff) | |
| download | karakeep-4cf0856e39c4d69037a6c1a4c3a2a7f803b364a7.tar.zst | |
feat: add crawler domain rate limiting (#2115)
Diffstat (limited to 'packages/shared')
| -rw-r--r-- | packages/shared/config.ts | 10 | ||||
| -rw-r--r-- | packages/shared/ratelimiting.ts | 7 |
2 files changed, 13 insertions, 4 deletions
diff --git a/packages/shared/config.ts b/packages/shared/config.ts index a62f0fb6..d4a951f1 100644 --- a/packages/shared/config.ts +++ b/packages/shared/config.ts @@ -106,6 +106,8 @@ const allEnv = z.object({ .transform((t) => t.split("%%").filter((a) => a)), CRAWLER_SCREENSHOT_TIMEOUT_SEC: z.coerce.number().default(5), CRAWLER_IP_VALIDATION_DNS_RESOLVER_TIMEOUT_SEC: z.coerce.number().default(1), + CRAWLER_DOMAIN_RATE_LIMIT_WINDOW_MS: z.coerce.number().min(1).optional(), + CRAWLER_DOMAIN_RATE_LIMIT_MAX_REQUESTS: z.coerce.number().min(1).optional(), LOG_LEVEL: z.string().default("debug"), NO_COLOR: stringBool("false"), DEMO_MODE: stringBool("false"), @@ -299,6 +301,14 @@ const serverConfigSchema = allEnv.transform((val, ctx) => { dnsResolverTimeoutSec: val.CRAWLER_IP_VALIDATION_DNS_RESOLVER_TIMEOUT_SEC, }, + domainRatelimiting: + val.CRAWLER_DOMAIN_RATE_LIMIT_WINDOW_MS !== undefined && + val.CRAWLER_DOMAIN_RATE_LIMIT_MAX_REQUESTS !== undefined + ? { + windowMs: val.CRAWLER_DOMAIN_RATE_LIMIT_WINDOW_MS, + maxRequests: val.CRAWLER_DOMAIN_RATE_LIMIT_MAX_REQUESTS, + } + : null, }, ocr: { langs: val.OCR_LANGS, diff --git a/packages/shared/ratelimiting.ts b/packages/shared/ratelimiting.ts index 3b22310b..ee5988b4 100644 --- a/packages/shared/ratelimiting.ts +++ b/packages/shared/ratelimiting.ts @@ -6,10 +6,9 @@ export interface RateLimitConfig { maxRequests: number; } -export interface RateLimitResult { - allowed: boolean; - resetInSeconds?: number; -} +export type RateLimitResult = + | { allowed: true } + | { allowed: false; resetInSeconds: number }; export interface RateLimitClient { /** |