Use tiktoken for Python 3.8 (#2709)

Fixes issue #2677

`tiktoken` is supported for Python 3.8, so there is no need to use the
fallback GPT-2 tokenizer.
fix_agent_callbacks
Joshua Snyder 1 year ago committed by GitHub
parent 186ca9d3e4
commit f435f2267c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -327,8 +327,8 @@ class ChatOpenAI(BaseChatModel):
def get_num_tokens(self, text: str) -> int:
"""Calculate num tokens with tiktoken package."""
# tiktoken NOT supported for Python 3.8 or below
if sys.version_info[1] <= 8:
# tiktoken NOT supported for Python 3.7 or below
if sys.version_info[1] <= 7:
return super().get_num_tokens(text)
try:
import tiktoken

Loading…
Cancel
Save