From 70d572096706cb2d7f5d3f29b51e6ba1d86578c6 Mon Sep 17 00:00:00 2001
From: Mohamed Bassem <me@mbassem.com>
Date: Sun, 18 May 2025 13:43:48 +0000
Subject: fix: Use proper tokenizer when truncating for inference. Fixes #1405

---
 packages/shared/package.json | 1 +
 1 file changed, 1 insertion(+)

(limited to 'packages/shared/package.json')

diff --git a/packages/shared/package.json b/packages/shared/package.json
index a262267a..478fb77d 100644
--- a/packages/shared/package.json
+++ b/packages/shared/package.json
@@ -6,6 +6,7 @@
   "type": "module",
   "dependencies": {
     "glob": "^11.0.0",
+    "js-tiktoken": "^1.0.20",
     "liteque": "^0.3.2",
     "meilisearch": "^0.37.0",
     "ollama": "^0.5.14",
-- 
cgit v1.2.3-70-g09d2