diff --git a/application/utils.py b/application/utils.py index ac98efc..f050201 100644 --- a/application/utils.py +++ b/application/utils.py @@ -1,6 +1,6 @@ from transformers import GPT2TokenizerFast - +tokenizer = GPT2TokenizerFast.from_pretrained('gpt2') def count_tokens(string): - tokenizer = GPT2TokenizerFast.from_pretrained('gpt2') + return len(tokenizer(string)['input_ids']) \ No newline at end of file