fix: Update count_tokens function in utils.py

pull/939/head
Alex 2 weeks ago
parent de0193fffc
commit 431755144e

@ -1,6 +1,6 @@
from transformers import GPT2TokenizerFast
tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
def count_tokens(string):
tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
return len(tokenizer(string)['input_ids'])
Loading…
Cancel
Save