diff --git a/langchain/llms/openai.py b/langchain/llms/openai.py index 8a54c5d3..32edad00 100644 --- a/langchain/llms/openai.py +++ b/langchain/llms/openai.py @@ -446,15 +446,9 @@ class BaseOpenAI(BaseLLM): "This is needed in order to calculate get_num_tokens. " "Please install it with `pip install tiktoken`." ) - encoder = "gpt2" - if self.model_name in ("text-davinci-003", "text-davinci-002"): - encoder = "p50k_base" - if self.model_name.startswith("code"): - encoder = "p50k_base" - # create a GPT-3 encoder instance - enc = tiktoken.get_encoding(encoder) - - # encode the text using the GPT-3 encoder + + enc = tiktoken.encoding_for_model(self.model_name) + tokenized_text = enc.encode(text) # calculate the number of tokens in the encoded text