mirror of https://github.com/xtekky/gpt4free
parent
79761c6e42
commit
2345588d38
@ -1,9 +1,9 @@
|
||||
import tiktoken
|
||||
from typing import Union
|
||||
# import tiktoken
|
||||
# from typing import Union
|
||||
|
||||
def tokenize(text: str, model: str = 'gpt-3.5-turbo') -> Union[int, str]:
|
||||
encoding = tiktoken.encoding_for_model(model)
|
||||
encoded = encoding.encode(text)
|
||||
num_tokens = len(encoded)
|
||||
# def tokenize(text: str, model: str = 'gpt-3.5-turbo') -> Union[int, str]:
|
||||
# encoding = tiktoken.encoding_for_model(model)
|
||||
# encoded = encoding.encode(text)
|
||||
# num_tokens = len(encoded)
|
||||
|
||||
return num_tokens, encoded
|
||||
# return num_tokens, encoded
|
Loading…
Reference in New Issue