diff options
| -rw-r--r-- | apps/workers/crawlerWorker.ts | 2 | ||||
| -rw-r--r-- | apps/workers/openaiWorker.ts | 22 | ||||
| -rw-r--r-- | packages/db/drizzle.config.ts | 5 | ||||
| -rw-r--r-- | packages/db/package.json | 1 | ||||
| -rw-r--r-- | packages/shared/config.ts | 103 | ||||
| -rw-r--r-- | pnpm-lock.yaml | 3 | ||||
| -rw-r--r-- | tooling/eslint/base.js | 1 |
7 files changed, 91 insertions, 46 deletions
diff --git a/apps/workers/crawlerWorker.ts b/apps/workers/crawlerWorker.ts index 282f5f43..3f7bff94 100644 --- a/apps/workers/crawlerWorker.ts +++ b/apps/workers/crawlerWorker.ts @@ -83,7 +83,7 @@ async function launchBrowser() { }, 5000); return; } - browser.on("disconnected", async () => { + browser.on("disconnected", async (): Promise<void> => { if (isShuttingDown) { logger.info( "The puppeteer browser got disconnected. But we're shutting down so won't restart it.", diff --git a/apps/workers/openaiWorker.ts b/apps/workers/openaiWorker.ts index 9e6cd553..5f785f2f 100644 --- a/apps/workers/openaiWorker.ts +++ b/apps/workers/openaiWorker.ts @@ -41,7 +41,7 @@ async function attemptMarkTaggingStatus( } export class OpenAiWorker { - static async build() { + static build() { logger.info("Starting openai worker ..."); const worker = new Worker<ZOpenAIRequest, void>( OpenAIQueue.name, @@ -52,14 +52,14 @@ export class OpenAiWorker { }, ); - worker.on("completed", async (job) => { - const jobId = job?.id || "unknown"; + worker.on("completed", async (job): Promise<void> => { + const jobId = job?.id ?? "unknown"; logger.info(`[openai][${jobId}] Completed successfully`); await attemptMarkTaggingStatus(job?.data, "success"); }); - worker.on("failed", async (job, error) => { - const jobId = job?.id || "unknown"; + worker.on("failed", async (job, error): Promise<void> => { + const jobId = job?.id ?? "unknown"; logger.error(`[openai][${jobId}] openai job failed: ${error}`); await attemptMarkTaggingStatus(job?.data, "failure"); }); @@ -152,7 +152,7 @@ async function inferTagsFromImage( const base64 = asset.toString('base64'); const chatCompletion = await openai.chat.completions.create({ - model: "gpt-4-vision-preview", + model: serverConfig.inference.imageModel, messages: [ { role: "user", @@ -185,7 +185,7 @@ async function inferTagsFromText( ) { const chatCompletion = await openai.chat.completions.create({ messages: [{ role: "system", content: buildPrompt(bookmark) }], - model: "gpt-3.5-turbo-0125", + model: serverConfig.inference.textModel, response_format: { type: "json_object" }, }); @@ -290,11 +290,11 @@ async function connectTags( } async function runOpenAI(job: Job<ZOpenAIRequest, void>) { - const jobId = job.id || "unknown"; + const jobId = job.id ?? "unknown"; - const { openAI } = serverConfig; + const { inference } = serverConfig; - if (!openAI.apiKey) { + if (!inference.openAIApiKey) { logger.debug( `[openai][${jobId}] OpenAI is not configured, nothing to do now`, ); @@ -302,7 +302,7 @@ async function runOpenAI(job: Job<ZOpenAIRequest, void>) { } const openai = new OpenAI({ - apiKey: openAI.apiKey, + apiKey: inference.openAIApiKey, }); const request = zOpenAIRequestSchema.safeParse(job.data); diff --git a/packages/db/drizzle.config.ts b/packages/db/drizzle.config.ts index 16aa286f..25a7675f 100644 --- a/packages/db/drizzle.config.ts +++ b/packages/db/drizzle.config.ts @@ -1,8 +1,9 @@ import "dotenv/config"; import type { Config } from "drizzle-kit"; +import serverConfig from "@hoarder/shared/config"; -const databaseURL = process.env.DATA_DIR - ? `${process.env.DATA_DIR}/db.db` +const databaseURL = serverConfig.dataDir + ? `${serverConfig.dataDir}/db.db` : "./db.db"; export default { diff --git a/packages/db/package.json b/packages/db/package.json index 811bd026..85e91e1f 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@auth/core": "^0.27.0", + "@hoarder/shared": "workspace:*", "@paralleldrive/cuid2": "^2.2.2", "better-sqlite3": "^9.4.3", "dotenv": "^16.4.1", diff --git a/packages/shared/config.ts b/packages/shared/config.ts index 4bee1ccf..5d83b4f0 100644 --- a/packages/shared/config.ts +++ b/packages/shared/config.ts @@ -1,41 +1,80 @@ -const serverConfig = { - apiUrl: process.env.API_URL ?? "http://localhost:3000", - auth: { - disableSignups: (process.env.DISABLE_SIGNUPS ?? "false") == "true", - }, - openAI: { - apiKey: process.env.OPENAI_API_KEY, - }, - bullMQ: { - redisHost: process.env.REDIS_HOST ?? "localhost", - redisPort: parseInt(process.env.REDIS_PORT ?? "6379"), - }, - crawler: { - headlessBrowser: (process.env.CRAWLER_HEADLESS_BROWSER ?? "true") == "true", - browserExecutablePath: process.env.BROWSER_EXECUTABLE_PATH, // If not set, the system's browser will be used - browserUserDataDir: process.env.BROWSER_USER_DATA_DIR, - browserWebUrl: process.env.BROWSER_WEB_URL, - }, - meilisearch: process.env.MEILI_ADDR - ? { - address: process.env.MEILI_ADDR ?? "http://127.0.0.1:7700", - key: process.env.MEILI_MASTER_KEY ?? "", - } - : undefined, - logLevel: process.env.LOG_LEVEL ?? "debug", - demoMode: (process.env.DEMO_MODE ?? "false") == "true" ? { - email: process.env.DEMO_MODE_EMAIL, - password: process.env.DEMO_MODE_PASSWORD, - }: undefined, - dataDir: process.env.DATA_DIR ?? "", -}; +import { z } from "zod"; + +const stringBool = (defaultValue: string) => + z + .string() + .default(defaultValue) + .refine((s) => s === "true" || s === "false") + .transform((s) => s === "true"); + +const allEnv = z.object({ + API_URL: z.string().url().default("http://localhost:3000"), + DISABLE_SIGNUPS: stringBool("false"), + OPENAI_API_KEY: z.string().optional(), + OPENAI_BASE_URL: z.string().url().optional(), + INFERENCE_TEXT_MODEL: z.string().default("gpt-3.5-turbo-0125"), + INFERENCE_IMAGE_MODEL: z.string().default("gpt-4-vision-preview"), + REDIS_HOST: z.string().default("localhost"), + REDIS_PORT: z.coerce.number().default(6379), + CRAWLER_HEADLESS_BROWSER: stringBool("true"), + BROWSER_EXECUTABLE_PATH: z.string().optional(), // If not set, the system's browser will be used + BROWSER_USER_DATA_DIR: z.string().optional(), + BROWSER_WEB_URL: z.string().url().optional(), + MEILI_ADDR: z.string().optional(), + MEILI_MASTER_KEY: z.string().default(""), + LOG_LEVEL: z.string().default("debug"), + DEMO_MODE: stringBool("false"), + DEMO_MODE_EMAIL: z.string().optional(), + DEMO_MODE_PASSWORD: z.string().optional(), + DATA_DIR: z.string().default(""), +}); +const serverConfigSchema = allEnv.transform((val) => { + return { + apiUrl: val.API_URL, + auth: { + disableSignups: val.DISABLE_SIGNUPS, + }, + inference: { + openAIApiKey: val.OPENAI_API_KEY, + openAIBaseUrl: val.OPENAI_BASE_URL, + textModel: val.INFERENCE_TEXT_MODEL, + imageModel: val.INFERENCE_IMAGE_MODEL, + }, + bullMQ: { + redisHost: val.REDIS_HOST, + redisPort: val.REDIS_PORT, + }, + crawler: { + headlessBrowser: val.CRAWLER_HEADLESS_BROWSER, + browserExecutablePath: val.BROWSER_EXECUTABLE_PATH, + browserUserDataDir: val.BROWSER_USER_DATA_DIR, + browserWebUrl: val.BROWSER_WEB_URL, + }, + meilisearch: val.MEILI_ADDR + ? { + address: val.MEILI_ADDR, + key: val.MEILI_MASTER_KEY, + } + : undefined, + logLevel: val.LOG_LEVEL, + demoMode: val.DEMO_MODE + ? { + email: val.DEMO_MODE_EMAIL, + password: val.DEMO_MODE_PASSWORD, + } + : undefined, + dataDir: val.DATA_DIR, + }; +}); + +const serverConfig = serverConfigSchema.parse(process.env); // Always explicitly pick up stuff from server config to avoid accidentally leaking stuff export const clientConfig = { demoMode: serverConfig.demoMode, auth: { disableSignups: serverConfig.auth.disableSignups, - } + }, }; export type ClientConfig = typeof clientConfig; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f372fea2..d5dd4f75 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -661,6 +661,9 @@ importers: '@auth/core': specifier: ^0.27.0 version: 0.27.0 + '@hoarder/shared': + specifier: workspace:* + version: link:../shared '@paralleldrive/cuid2': specifier: ^2.2.2 version: 2.2.2 diff --git a/tooling/eslint/base.js b/tooling/eslint/base.js index 2bc62548..7a4f9377 100644 --- a/tooling/eslint/base.js +++ b/tooling/eslint/base.js @@ -30,6 +30,7 @@ const config = { "import/consistent-type-specifier-style": ["error", "prefer-top-level"], "@typescript-eslint/no-floating-promises": "off", "@typescript-eslint/unbound-method": "off", + "@typescript-eslint/restrict-template-expressions": "off", }, ignorePatterns: [ "**/*.config.js", |