From f435f2267c015ffd97ff5eea9ad8c8051ea0dc0f Mon Sep 17 00:00:00 2001 From: Joshua Snyder Date: Tue, 11 Apr 2023 20:02:28 +0200 Subject: [PATCH] Use tiktoken for Python 3.8 (#2709) Fixes issue #2677 `tiktoken` is supported for Python 3.8, so there is no need to use the fallback GPT-2 tokenizer. --- langchain/chat_models/openai.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langchain/chat_models/openai.py b/langchain/chat_models/openai.py index f31d3931..eb4b0315 100644 --- a/langchain/chat_models/openai.py +++ b/langchain/chat_models/openai.py @@ -327,8 +327,8 @@ class ChatOpenAI(BaseChatModel): def get_num_tokens(self, text: str) -> int: """Calculate num tokens with tiktoken package.""" - # tiktoken NOT supported for Python 3.8 or below - if sys.version_info[1] <= 8: + # tiktoken NOT supported for Python 3.7 or below + if sys.version_info[1] <= 7: return super().get_num_tokens(text) try: import tiktoken