mirror of
https://github.com/xtekky/gpt4free.git
synced 2024-11-05 00:01:00 +00:00
9 lines
286 B
Python
9 lines
286 B
Python
# import tiktoken
|
|
# from typing import Union
|
|
|
|
# def tokenize(text: str, model: str = 'gpt-3.5-turbo') -> Union[int, str]:
|
|
# encoding = tiktoken.encoding_for_model(model)
|
|
# encoded = encoding.encode(text)
|
|
# num_tokens = len(encoded)
|
|
|
|
# return num_tokens, encoded |