From 1b09682685f54f29957163be9b9f9fc2de3b49cc Mon Sep 17 00:00:00 2001
From: MohamedBassem <me@mbassem.com>
Date: Sat, 12 Oct 2024 17:25:01 +0000
Subject: feature: Allow customizing the inference's context length

---
 apps/workers/inference.ts | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'apps/workers/inference.ts')

diff --git a/apps/workers/inference.ts b/apps/workers/inference.ts
index 071f4742..41ceffd6 100644
--- a/apps/workers/inference.ts
+++ b/apps/workers/inference.ts
@@ -104,6 +104,9 @@ class OllamaInferenceClient implements InferenceClient {
       format: "json",
       stream: true,
       keep_alive: serverConfig.inference.ollamaKeepAlive,
+      options: {
+        num_ctx: serverConfig.inference.contextLength,
+      },
       messages: [
         { role: "user", content: prompt, images: image ? [image] : undefined },
       ],
-- 
cgit v1.2.3-70-g09d2