mirror of https://github.com/arc53/DocsGPT
Merge pull request #958 from arc53/feat-pre-loading-embeds
chore: Update Docker build platforms for application and frontend and…pull/920/merge
commit
7f7f48ad56
@ -1,6 +1,6 @@
|
||||
from transformers import GPT2TokenizerFast
|
||||
|
||||
tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
|
||||
tokenizer.model_max_length = 100000
|
||||
def count_tokens(string):
|
||||
|
||||
return len(tokenizer(string)['input_ids'])
|
Loading…
Reference in New Issue