diff options
| author | MohamedBassem <me@mbassem.com> | 2024-10-12 17:25:01 +0000 |
|---|---|---|
| committer | MohamedBassem <me@mbassem.com> | 2024-10-12 17:37:42 +0000 |
| commit | 1b09682685f54f29957163be9b9f9fc2de3b49cc (patch) | |
| tree | 7f10a7635cf984acd45147c24ec3e1d35798e8ba /apps/workers/inference.ts | |
| parent | c16173ea0fdbf6cc47b13756c0a77e8399669055 (diff) | |
| download | karakeep-1b09682685f54f29957163be9b9f9fc2de3b49cc.tar.zst | |
feature: Allow customizing the inference's context length
Diffstat (limited to 'apps/workers/inference.ts')
| -rw-r--r-- | apps/workers/inference.ts | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/apps/workers/inference.ts b/apps/workers/inference.ts index 071f4742..41ceffd6 100644 --- a/apps/workers/inference.ts +++ b/apps/workers/inference.ts @@ -104,6 +104,9 @@ class OllamaInferenceClient implements InferenceClient { format: "json", stream: true, keep_alive: serverConfig.inference.ollamaKeepAlive, + options: { + num_ctx: serverConfig.inference.contextLength, + }, messages: [ { role: "user", content: prompt, images: image ? [image] : undefined }, ], |