vet

commit 7976386d63388c49e8375cd0103795bacf1def68
parent 1d79724eaee33ab2dde9c08e49b3ceda9a25848c
Author: andrewlaack-collab <andrew.laack@imbue.com>
Date:   Wed, 18 Feb 2026 22:24:33 +0000

Fixed token blocking issue (#102)

Co-authored-by: Andrew Laack <andrew@laack.co>
Diffstat:
M
uv.lock
 | 
2
+-
M
vet/imbue_core/agents/llm_apis/anthropic_api.py
 | 
2
+-
M
vet/imbue_core/agents/llm_apis/openai_api.py
 | 
2
+-

3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/uv.lock b/uv.lock
@@ -1511,7 +1511,7 @@ wheels = [
 
 [[package]]
 name = "verify-everything"
-version = "0.1.8"
+version = "0.1.10"
 source = { editable = "." }
 dependencies = [
     { name = "anthropic" },
diff --git a/vet/imbue_core/agents/llm_apis/anthropic_api.py b/vet/imbue_core/agents/llm_apis/anthropic_api.py
@@ -362,7 +362,7 @@ def get_anthropic_tokenizer() -> tiktoken.Encoding:
 
 
 def count_anthropic_tokens(text: str) -> int:
-    return int(len(get_anthropic_tokenizer().encode(text)) * 1.1)
+    return int(len(get_anthropic_tokenizer().encode(text, disallowed_special=())) * 1.1)
 
 
 SystemMessageParam = TextBlockParam
diff --git a/vet/imbue_core/agents/llm_apis/openai_api.py b/vet/imbue_core/agents/llm_apis/openai_api.py
@@ -333,7 +333,7 @@ def get_openai_tokenizer(model_name: str) -> tiktoken.Encoding:
 
 
 def count_openai_tokens(text: str, model_name: str) -> int:
-    return len(get_openai_tokenizer(model_name).encode(text))
+    return len(get_openai_tokenizer(model_name).encode(text, disallowed_special=()))
 
 
 @contextmanager