mirror of https://github.com/arc53/DocsGPT
fix: Update count_tokens function in utils.py
parent
de0193fffc
commit
431755144e
@ -1,6 +1,6 @@
|
|||||||
from transformers import GPT2TokenizerFast
|
from transformers import GPT2TokenizerFast
|
||||||
|
|
||||||
|
tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
|
||||||
def count_tokens(string):
|
def count_tokens(string):
|
||||||
tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
|
|
||||||
return len(tokenizer(string)['input_ids'])
|
return len(tokenizer(string)['input_ids'])
|
Loading…
Reference in New Issue