mirror of
https://github.com/arc53/DocsGPT
synced 2024-11-09 19:10:53 +00:00
6 lines
175 B
Python
6 lines
175 B
Python
|
from transformers import GPT2TokenizerFast
|
||
|
|
|
||
|
|
|
||
|
|
def count_tokens(string):
|
||
|
|
tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
|
||
|
|
return len(tokenizer(string)['input_ids'])
|