mirror of
https://github.com/xtekky/gpt4free.git
synced 2024-11-05 00:01:00 +00:00
9 lines
272 B
Python
9 lines
272 B
Python
|
import tiktoken
|
||
|
|
from typing import Union
|
||
|
|
|
||
|
|
def tokenize(text: str, model: str = 'gpt-3.5-turbo') -> Union[int, str]:
|
||
|
|
encoding = tiktoken.encoding_for_model(model)
|
||
|
|
encoded = encoding.encode(text)
|
||
|
|
num_tokens = len(encoded)
|
||
|
|
|
||
|
|
return num_tokens, encoded
|